Imported Upstream version 1.3.0upstream/1.3.0

1600 files changed, 132551 insertions, 31912 deletions

diff --git a/.exrc b/.exrc
new file mode 100644
index 000000000..37755ede8
--- /dev/null
+++ b/.exrc
@@ -0,0 +1,7 @@
+"VIM settings to match QEMU coding style.  They are activated by adding the
+"following settings (without the " symbol) as last two lines in $HOME/.vimrc:
+"set secure
+"set exrc
+set expandtab
+set shiftwidth=4
+set smarttab
diff --git a/.gitignore b/.gitignore
index 824c0d24d..bd6ba1c71 100644
--- a/.gitignore
+++ b/.gitignore
@@ -12,8 +12,6 @@ trace-dtrace.dtrace
 *-linux-user
 *-bsd-user
 libdis*
-libhw32
-libhw64
 libuser
 linux-headers/asm
 qapi-generated
diff --git a/.gitmodules b/.gitmodules
deleted file mode 100644
index eca876f85..000000000
--- a/.gitmodules
+++ /dev/null
@@ -1,21 +0,0 @@
-[submodule "roms/vgabios"]
-	path = roms/vgabios
-	url = git://git.qemu.org/vgabios.git/
-[submodule "roms/seabios"]
-	path = roms/seabios
-	url = git://git.qemu.org/seabios.git/
-[submodule "roms/SLOF"]
-	path = roms/SLOF
-	url = git://git.qemu.org/SLOF.git
-[submodule "roms/ipxe"]
-	path = roms/ipxe
-	url = git://git.qemu.org/ipxe.git
-[submodule "roms/openbios"]
-	path = roms/openbios
-	url = git://git.qemu.org/openbios.git
-[submodule "roms/qemu-palcode"]
-	path = roms/qemu-palcode
-	url = git://repo.or.cz/qemu-palcode.git
-[submodule "roms/sgabios"]
-	path = roms/sgabios
-	url = git://git.qemu.org/sgabios.git
diff --git a/HACKING b/HACKING
index 471cf1d19..89a6b3ad4 100644
--- a/HACKING
+++ b/HACKING
@@ -32,7 +32,7 @@ mandatory for VMState fields.
 
 Don't use Linux kernel internal types like u32, __u32 or __le32.
 
-Use target_phys_addr_t for guest physical addresses except pcibus_t
+Use hwaddr for guest physical addresses except pcibus_t
 for PCI addresses.  In addition, ram_addr_t is a QEMU internal address
 space that maps guest RAM physical addresses into an intermediate
 address space that can map to host virtual address spaces.  Generally
@@ -91,10 +91,11 @@ emulators.
 
 4. String manipulation
 
-Do not use the strncpy function.  According to the man page, it does
-*not* guarantee a NULL-terminated buffer, which makes it extremely dangerous
-to use.  Instead, use functionally equivalent function:
-void pstrcpy(char *buf, int buf_size, const char *str)
+Do not use the strncpy function.  As mentioned in the man page, it does *not*
+guarantee a NULL-terminated buffer, which makes it extremely dangerous to use.
+It also zeros trailing destination bytes out to the specified length.  Instead,
+use this similar function when possible, but note its different signature:
+void pstrcpy(char *dest, int dest_buf_size, const char *src)
 
 Don't use strcat because it can't check for buffer overflows, but:
 char *pstrcat(char *buf, int buf_size, const char *s)
diff --git a/MAINTAINERS b/MAINTAINERS
index 6d864c1ce..2ede20d60 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -268,6 +268,7 @@ S: Maintained
 F: hw/xilinx_zynq.c
 F: hw/zynq_slcr.c
 F: hw/cadence_*
+F: hw/xilinx_spips.c
 
 CRIS Machines
 -------------
@@ -349,9 +350,31 @@ PowerPC Machines
 405
 M: Alexander Graf <agraf@suse.de>
 L: qemu-ppc@nongnu.org
-S: Maintained
+S: Odd Fixes
 F: hw/ppc405_boards.c
 
+Bamboo
+M: Alexander Graf <agraf@suse.de>
+L: qemu-ppc@nongnu.org
+S: Odd Fixes
+F: hw/ppc440_bamboo.c
+
+e500
+M: Alexander Graf <agraf@suse.de>
+M: Scott Wood <scottwood@freescale.com>
+L: qemu-ppc@nongnu.org
+S: Supported
+F: hw/ppc/e500.[hc]
+F: hw/ppc/e500plat.c
+
+mpc8544ds
+M: Alexander Graf <agraf@suse.de>
+M: Scott Wood <scottwood@freescale.com>
+L: qemu-ppc@nongnu.org
+S: Supported
+F: hw/ppc/mpc8544ds.c
+F: hw/mpc8544_guts.c
+
 New World
 M: Alexander Graf <agraf@suse.de>
 L: qemu-ppc@nongnu.org
@@ -374,6 +397,19 @@ S: Odd Fixes
 F: hw/ppc_prep.c
 F: hw/prep_pci.[hc]
 
+sPAPR
+M: David Gibson <david@gibson.dropbear.id.au>
+M: Alexander Graf <agraf@suse.de>
+L: qemu-ppc@nongnu.org
+S: Supported
+F: hw/spapr*
+
+virtex_ml507
+M: Edgar E. Iglesias <edgar.iglesias@gmail.com>
+L: qemu-ppc@nongnu.org
+S: Odd Fixes
+F: hw/virtex_ml507.c
+
 SH4 Machines
 ------------
 R2D
@@ -398,6 +434,12 @@ M: Blue Swirl <blauwirbel@gmail.com>
 S: Maintained
 F: hw/sun4u.c
 
+Leon3
+M: Fabien Chouteau <chouteau@adacore.com>
+S: Maintained
+F: hw/leon3.c
+F: hw/grlib*
+
 S390 Machines
 -------------
 S390 Virtio
@@ -451,6 +493,19 @@ S: Supported
 F: hw/pci*
 F: hw/piix*
 
+ppc4xx
+M: Alexander Graf <agraf@suse.de>
+L: qemu-ppc@nongnu.org
+S: Odd Fixes
+F: hw/ppc4xx*.[hc]
+
+ppce500
+M: Alexander Graf <agraf@suse.de>
+M: Scott Wood <scottwood@freescale.com>
+L: qemu-ppc@nongnu.org
+S: Supported
+F: hw/ppce500_*
+
 SCSI
 M: Paolo Bonzini <pbonzini@redhat.com>
 S: Supported
@@ -463,11 +518,22 @@ M: Paul Brook <paul@codesourcery.com>
 S: Odd Fixes
 F: hw/lsi53c895a.c
 
+SSI
+M: Peter Crosthwaite <peter.crosthwaite@petalogix.com>
+S: Maintained
+F: hw/ssi.*
+F: hw/m25p80.c
+
 USB
 M: Gerd Hoffmann <kraxel@redhat.com>
 S: Maintained
 F: hw/usb*
 
+VFIO
+M: Alex Williamson <alex.williamson@redhat.com>
+S: Supported
+F: hw/vfio*
+
 vhost
 M: Michael S. Tsirkin <mst@redhat.com>
 S: Supported
@@ -487,6 +553,7 @@ T: git git://github.com/kvaneesh/QEMU.git
 
 virtio-blk
 M: Kevin Wolf <kwolf@redhat.com>
+M: Stefan Hajnoczi <stefanha@redhat.com>
 S: Supported
 F: hw/virtio-blk*
 
@@ -506,6 +573,7 @@ F: hw/xilinx_intc.c
 F: hw/xilinx_ethlite.c
 F: hw/xilinx_timer.c
 F: hw/xilinx.h
+F: hw/xilinx_spi.c
 
 Subsystems
 ----------
@@ -516,6 +584,7 @@ F: audio/
 
 Block
 M: Kevin Wolf <kwolf@redhat.com>
+M: Stefan Hajnoczi <stefanha@redhat.com>
 S: Supported
 F: block*
 F: block/
@@ -525,6 +594,12 @@ M: Anthony Liguori <aliguori@us.ibm.com>
 S: Maintained
 F: qemu-char.c
 
+CPU
+M: Andreas Färber <afaerber@suse.de>
+S: Supported
+F: qom/cpu.c
+F: include/qemu/cpu.h
+
 Device Tree
 M: Peter Crosthwaite <peter.crosthwaite@petalogix.com>
 M: Alexander Graf <agraf@suse.de>
@@ -568,7 +643,7 @@ F: monitor.c
 
 Network device layer
 M: Anthony Liguori <aliguori@us.ibm.com>
-M: Stefan Hajnoczi <stefanha@gmail.com>
+M: Stefan Hajnoczi <stefanha@redhat.com>
 S: Maintained
 F: net/
 T: git git://github.com/stefanha/qemu.git net
@@ -588,7 +663,7 @@ F: slirp/
 T: git git://git.kiszka.org/qemu.git queues/slirp
 
 Tracing
-M: Stefan Hajnoczi <stefanha@gmail.com>
+M: Stefan Hajnoczi <stefanha@redhat.com>
 S: Maintained
 F: trace/
 F: scripts/tracetool.py
diff --git a/Makefile b/Makefile
index 1cd5bc80d..9ecbcbb0a 100644
--- a/Makefile
+++ b/Makefile
@@ -8,15 +8,28 @@ ifneq ($(wildcard config-host.mak),)
 # Put the all: rule here so that config-host.mak can contain dependencies.
 all:
 include config-host.mak
+
+# Check that we're not trying to do an out-of-tree build from
+# a tree that's been used for an in-tree build.
+ifneq ($(realpath $(SRC_PATH)),$(realpath .))
+ifneq ($(wildcard $(SRC_PATH)/config-host.mak),)
+$(error This is an out of tree build but your source tree ($(SRC_PATH)) \
+seems to have been used for an in-tree build. You can fix this by running \
+"make distclean && rm -rf *-linux-user *-softmmu" in your source tree)
+endif
+endif
+
 include $(SRC_PATH)/rules.mak
 config-host.mak: $(SRC_PATH)/configure
 	@echo $@ is out-of-date, running configure
 	@sed -n "/.*Configured with/s/[^:]*: //p" $@ | sh
 else
 config-host.mak:
+ifneq ($(filter-out %clean,$(MAKECMDGOALS)),$(if $(MAKECMDGOALS),,fail))
 	@echo "Please call configure before running make!"
 	@exit 1
 endif
+endif
 
 GENERATED_HEADERS = config-host.h trace.h qemu-options.def
 ifeq ($(TRACE_BACKEND),dtrace)
@@ -52,8 +65,13 @@ SUBDIR_MAKEFLAGS=$(if $(V),,--no-print-directory) BUILD_DIR=$(BUILD_DIR)
 SUBDIR_DEVICES_MAK=$(patsubst %, %/config-devices.mak, $(TARGET_DIRS))
 SUBDIR_DEVICES_MAK_DEP=$(patsubst %, %/config-devices.mak.d, $(TARGET_DIRS))
 
+ifeq ($(SUBDIR_DEVICES_MAK),)
+config-all-devices.mak:
+	$(call quiet-command,echo '# no devices' > $@,"  GEN   $@")
+else
 config-all-devices.mak: $(SUBDIR_DEVICES_MAK)
 	$(call quiet-command,cat $(SUBDIR_DEVICES_MAK) | grep =y | sort -u > $@,"  GEN   $@")
+endif
 
 -include $(SUBDIR_DEVICES_MAK_DEP)
 
@@ -100,6 +118,17 @@ endif
 
 subdir-libcacard: $(oslib-obj-y) $(trace-obj-y) qemu-timer-common.o
 
+subdir-pixman: pixman/Makefile
+	$(call quiet-command,$(MAKE) $(SUBDIR_MAKEFLAGS) -C pixman V="$(V)" all,)
+
+pixman/Makefile: $(SRC_PATH)/pixman/configure
+	(cd pixman; CFLAGS="$(CFLAGS) -fPIC" $(SRC_PATH)/pixman/configure $(AUTOCONF_HOST) --disable-gtk --disable-shared --enable-static)
+
+$(SRC_PATH)/pixman/configure:
+	(cd $(SRC_PATH)/pixman; autoreconf -v --install)
+
+$(SUBDIR_RULES): libqemustub.a
+
 $(filter %-softmmu,$(SUBDIR_RULES)): $(universal-obj-y) $(trace-obj-y) $(common-obj-y) $(extra-obj-y) subdir-libdis
 
 $(filter %-user,$(SUBDIR_RULES)): $(universal-obj-y) $(trace-obj-y) subdir-libdis-user subdir-libuser
@@ -130,6 +159,12 @@ version.o: $(SRC_PATH)/version.rc config-host.h
 	$(call quiet-command,$(WINDRES) -I. -o $@ $<,"  RC    $(TARGET_DIR)$@")
 
 version-obj-$(CONFIG_WIN32) += version.o
+
+######################################################################
+# Build library with stubs
+
+libqemustub.a: $(stub-obj-y)
+
 ######################################################################
 # Support building shared library libcacard
 
@@ -153,17 +188,16 @@ endif
 qemu-img.o: qemu-img-cmds.h
 
 tools-obj-y = $(oslib-obj-y) $(trace-obj-y) qemu-tool.o qemu-timer.o \
-	qemu-timer-common.o main-loop.o notify.o \
-	iohandler.o cutils.o iov.o async.o
+	main-loop.o iohandler.o error.o
 tools-obj-$(CONFIG_POSIX) += compatfd.o
 
-qemu-img$(EXESUF): qemu-img.o $(tools-obj-y) $(block-obj-y)
-qemu-nbd$(EXESUF): qemu-nbd.o $(tools-obj-y) $(block-obj-y)
-qemu-io$(EXESUF): qemu-io.o cmd.o $(tools-obj-y) $(block-obj-y)
+qemu-img$(EXESUF): qemu-img.o $(tools-obj-y) $(block-obj-y) libqemustub.a
+qemu-nbd$(EXESUF): qemu-nbd.o $(tools-obj-y) $(block-obj-y) libqemustub.a
+qemu-io$(EXESUF): qemu-io.o cmd.o $(tools-obj-y) $(block-obj-y) libqemustub.a
 
 qemu-bridge-helper$(EXESUF): qemu-bridge-helper.o
 
-vscclient$(EXESUF): $(libcacard-y) $(oslib-obj-y) $(trace-obj-y) $(tools-obj-y) qemu-timer-common.o libcacard/vscclient.o
+vscclient$(EXESUF): $(libcacard-y) $(oslib-obj-y) $(trace-obj-y) libcacard/vscclient.o libqemustub.a
 	$(call quiet-command,$(CC) $(LDFLAGS) -o $@ $^ $(libcacard_libs) $(LIBS),"  LINK  $@")
 
 fsdev/virtfs-proxy-helper$(EXESUF): fsdev/virtfs-proxy-helper.o fsdev/virtio-9p-marshal.o oslib-posix.o $(trace-obj-y)
@@ -206,9 +240,9 @@ $(SRC_PATH)/qapi-schema.json $(SRC_PATH)/scripts/qapi-commands.py $(qapi-py)
 QGALIB_GEN=$(addprefix qga/qapi-generated/, qga-qapi-types.h qga-qapi-visit.h qga-qmp-commands.h)
 $(qga-obj-y) qemu-ga.o: $(QGALIB_GEN)
 
-qemu-ga$(EXESUF): qemu-ga.o $(qga-obj-y) $(tools-obj-y) $(qapi-obj-y) $(qobject-obj-y) $(version-obj-y)
+qemu-ga$(EXESUF): qemu-ga.o $(qga-obj-y) $(oslib-obj-y) $(trace-obj-y) $(qapi-obj-y) $(qobject-obj-y) $(version-obj-y) libqemustub.a
 
-QEMULIBS=libhw32 libhw64 libuser libdis libdis-user
+QEMULIBS=libuser libdis libdis-user
 
 clean:
 # avoid old build problems by removing potentially incorrect old files
@@ -252,6 +286,7 @@ distclean: clean
 	for d in $(TARGET_DIRS) $(QEMULIBS); do \
 	rm -rf $$d || exit 1 ; \
         done
+	if test -f pixman/config.log; then make -C pixman distclean; fi
 
 KEYMAPS=da     en-gb  et  fr     fr-ch  is  lt  modifiers  no  pt-br  sv \
 ar      de     en-us  fi  fr-be  hr     it  lv  nl         pl  ru     th \
@@ -297,7 +332,6 @@ install-confdir:
 
 install-sysconfig: install-datadir install-confdir
 	$(INSTALL_DATA) $(SRC_PATH)/sysconfigs/target/target-x86_64.conf "$(DESTDIR)$(qemu_confdir)"
-	$(INSTALL_DATA) $(SRC_PATH)/sysconfigs/target/cpus-x86_64.conf "$(DESTDIR)$(qemu_datadir)"
 
 install: all $(if $(BUILD_DOCS),install-doc) install-sysconfig install-datadir
 	$(INSTALL_DIR) "$(DESTDIR)$(bindir)"
@@ -398,7 +432,9 @@ qemu-doc.dvi qemu-doc.html qemu-doc.info qemu-doc.pdf: \
 
 # Add a dependency on the generated files, so that they are always
 # rebuilt before other object files
+ifneq ($(filter-out %clean,$(MAKECMDGOALS)),$(if $(MAKECMDGOALS),,fail))
 Makefile: $(GENERATED_HEADERS)
+endif
 
 # Include automatically generated dependency files
 # Dependencies in Makefile.objs files come from our recursive subdir rules
diff --git a/Makefile.hw b/Makefile.hw
deleted file mode 100644
index 59f5b4835..000000000
--- a/Makefile.hw
+++ /dev/null
@@ -1,23 +0,0 @@
-# Makefile for qemu target independent devices.
-
-include ../config-host.mak
-include ../config-all-devices.mak
-include config.mak
-include $(SRC_PATH)/rules.mak
-
-.PHONY: all
-
-$(call set-vpath, $(SRC_PATH))
-
-QEMU_CFLAGS+=-I..
-QEMU_CFLAGS += -I$(SRC_PATH)/include
-
-include $(SRC_PATH)/Makefile.objs
-
-all: $(hw-obj-y)
-# Dummy command so that make thinks it has done something
-	@true
-
-clean:
-	rm -f $(addsuffix *.o, $(sort $(dir $(hw-obj-y))))
-	rm -f $(addsuffix *.d, $(sort $(dir $(hw-obj-y))))
diff --git a/Makefile.objs b/Makefile.objs
index 441275730..3c7abca43 100644
--- a/Makefile.objs
+++ b/Makefile.objs
@@ -1,4 +1,8 @@
 #######################################################################
+# Stub library, linked in tools
+stub-obj-y = stubs/
+
+#######################################################################
 # Target-independent parts used in system and user emulation
 universal-obj-y =
 universal-obj-y += qemu-log.o
@@ -19,7 +23,7 @@ universal-obj-y += $(qom-obj-y)
 
 #######################################################################
 # oslib-obj-y is code depending on the OS (win32 vs posix)
-oslib-obj-y = osdep.o
+oslib-obj-y = osdep.o cutils.o qemu-timer-common.o
 oslib-obj-$(CONFIG_WIN32) += oslib-win32.o qemu-thread-win32.o
 oslib-obj-$(CONFIG_POSIX) += oslib-posix.o qemu-thread-posix.o
 
@@ -41,12 +45,14 @@ coroutine-obj-$(CONFIG_WIN32) += coroutine-win32.o
 #######################################################################
 # block-obj-y is code used by both qemu system emulation and qemu-img
 
-block-obj-y = cutils.o iov.o cache-utils.o qemu-option.o module.o async.o
-block-obj-y += nbd.o block.o aio.o aes.o qemu-config.o qemu-progress.o qemu-sockets.o
+block-obj-y = iov.o cache-utils.o qemu-option.o module.o async.o
+block-obj-y += nbd.o block.o blockjob.o aes.o qemu-config.o
+block-obj-y += thread-pool.o qemu-progress.o qemu-sockets.o uri.o notify.o
 block-obj-y += $(coroutine-obj-y) $(qobject-obj-y) $(version-obj-y)
-block-obj-$(CONFIG_POSIX) += posix-aio-compat.o
-block-obj-$(CONFIG_LINUX_AIO) += linux-aio.o
+block-obj-$(CONFIG_POSIX) += event_notifier-posix.o aio-posix.o
+block-obj-$(CONFIG_WIN32) += event_notifier-win32.o aio-win32.o
 block-obj-y += block/
+block-obj-y += $(qapi-obj-y) qapi-types.o qapi-visit.o
 
 ifeq ($(CONFIG_VIRTIO)$(CONFIG_VIRTFS)$(CONFIG_PCI),yyy)
 # Lots of the fsdev/9pcode is pulled in by vl.c via qemu_fsdev_add.
@@ -59,10 +65,11 @@ endif
 # suppress *all* target specific code in case of system emulation, i.e. a
 # single QEMU executable should support all CPUs and machines.
 
-common-obj-y = $(block-obj-y) blockdev.o
+common-obj-y = $(block-obj-y) blockdev.o blockdev-nbd.o block/
 common-obj-y += net.o net/
 common-obj-y += qom/
 common-obj-y += readline.o console.o cursor.o
+common-obj-y += qemu-pixman.o
 common-obj-y += $(oslib-obj-y)
 common-obj-$(CONFIG_WIN32) += os-win32.o
 common-obj-$(CONFIG_POSIX) += os-posix.o
@@ -75,7 +82,6 @@ common-obj-y += input.o
 common-obj-y += buffered_file.o migration.o migration-tcp.o
 common-obj-y += qemu-char.o #aio.o
 common-obj-y += block-migration.o iohandler.o
-common-obj-y += pflib.o
 common-obj-y += bitmap.o bitops.o
 common-obj-y += page_cache.o
 
@@ -89,13 +95,17 @@ common-obj-y += hw/
 common-obj-y += ui/
 common-obj-y += bt-host.o bt-vhci.o
 
-common-obj-y += iov.o acl.o
+common-obj-y += dma-helpers.o
+common-obj-y += acl.o
 common-obj-$(CONFIG_POSIX) += compatfd.o
-common-obj-y += notify.o event_notifier.o
 common-obj-y += qemu-timer.o qemu-timer-common.o
+common-obj-y += qtest.o
+common-obj-y += vl.o
 
 common-obj-$(CONFIG_SLIRP) += slirp/
 
+common-obj-y += backends/
+
 ######################################################################
 # libseccomp
 ifeq ($(CONFIG_SECCOMP),y)
@@ -108,18 +118,13 @@ endif
 user-obj-y =
 user-obj-y += envlist.o path.o
 user-obj-y += tcg-runtime.o host-utils.o
-user-obj-y += cutils.o iov.o cache-utils.o
+user-obj-y += cache-utils.o
 user-obj-y += module.o
 user-obj-y += qemu-user.o
 user-obj-y += $(trace-obj-y)
 user-obj-y += qom/
 
 ######################################################################
-# libhw
-
-hw-obj-y = vl.o dma-helpers.o qtest.o hw/
-
-######################################################################
 # libdis
 # NOTE: the disassembler code is only needed for debugging
 
@@ -228,9 +233,8 @@ universal-obj-y += $(qapi-obj-y)
 ######################################################################
 # guest agent
 
-qga-obj-y = qga/ qemu-ga.o module.o
-qga-obj-$(CONFIG_WIN32) += oslib-win32.o
-qga-obj-$(CONFIG_POSIX) += oslib-posix.o qemu-sockets.o qemu-option.o
+qga-obj-y = qga/ qemu-ga.o module.o qemu-tool.o
+qga-obj-$(CONFIG_POSIX) += qemu-sockets.o qemu-option.o
 
 vl.o: QEMU_CFLAGS+=$(GPROF_CFLAGS)
 
@@ -239,11 +243,11 @@ vl.o: QEMU_CFLAGS+=$(SDL_CFLAGS)
 QEMU_CFLAGS+=$(GLIB_CFLAGS)
 
 nested-vars += \
-	hw-obj-y \
+	stub-obj-y \
 	qga-obj-y \
-	block-obj-y \
 	qom-obj-y \
 	qapi-obj-y \
+	block-obj-y \
 	user-obj-y \
 	common-obj-y \
 	extra-obj-y
diff --git a/Makefile.target b/Makefile.target
index 7892a8df6..927347bac 100644
--- a/Makefile.target
+++ b/Makefile.target
@@ -4,9 +4,6 @@ include ../config-host.mak
 include config-devices.mak
 include config-target.mak
 include $(SRC_PATH)/rules.mak
-ifneq ($(HWDIR),)
-include $(HWDIR)/config.mak
-endif
 
 $(call set-vpath, $(SRC_PATH))
 ifdef CONFIG_LINUX
@@ -80,14 +77,6 @@ obj-$(CONFIG_GDBSTUB_XML) += gdbstub-xml.o
 
 tci-dis.o: QEMU_CFLAGS += -I$(SRC_PATH)/tcg -I$(SRC_PATH)/tcg/tci
 
-# HELPER_CFLAGS is used for all the legacy code compiled with static register
-# variables
-user-exec.o: QEMU_CFLAGS += $(HELPER_CFLAGS)
-
-# Note: this is a workaround. The real fix is to avoid compiling
-# cpu_signal_handler() in user-exec.c.
-%/signal.o: QEMU_CFLAGS += $(HELPER_CFLAGS)
-
 #########################################################
 # Linux user emulator target
 
@@ -154,6 +143,9 @@ GENERATED_HEADERS += hmp-commands.h qmp-commands-old.h
 
 endif # CONFIG_SOFTMMU
 
+# Workaround for http://gcc.gnu.org/PR55489, see configure.
+%/translate.o: QEMU_CFLAGS += $(TRANSLATE_OPT_CFLAGS)
+
 nested-vars += obj-y
 
 # This resolves all nested paths, so it must come last
@@ -165,7 +157,6 @@ all-obj-y += $(addprefix ../, $(universal-obj-y))
 ifdef CONFIG_SOFTMMU
 all-obj-y += $(addprefix ../, $(common-obj-y))
 all-obj-y += $(addprefix ../libdis/, $(libdis-y))
-all-obj-y += $(addprefix $(HWDIR)/, $(hw-obj-y))
 all-obj-y += $(addprefix ../, $(trace-obj-y))
 else
 all-obj-y += $(addprefix ../libuser/, $(user-obj-y))
@@ -174,12 +165,12 @@ endif #CONFIG_LINUX_USER
 
 ifdef QEMU_PROGW
 # The linker builds a windows executable. Make also a console executable.
-$(QEMU_PROGW): $(all-obj-y)
+$(QEMU_PROGW): $(all-obj-y) ../libqemustub.a
 	$(call LINK,$^)
 $(QEMU_PROG): $(QEMU_PROGW)
 	$(call quiet-command,$(OBJCOPY) --subsystem console $(QEMU_PROGW) $(QEMU_PROG),"  GEN   $(TARGET_DIR)$(QEMU_PROG)")
 else
-$(QEMU_PROG): $(all-obj-y)
+$(QEMU_PROG): $(all-obj-y) ../libqemustub.a
 	$(call LINK,$^)
 endif
 
diff --git a/QMP/qemu-ga-client b/QMP/qemu-ga-client
new file mode 100755
index 000000000..46676c375
--- /dev/null
+++ b/QMP/qemu-ga-client
@@ -0,0 +1,299 @@
+#!/usr/bin/python
+
+# QEMU Guest Agent Client
+#
+# Copyright (C) 2012 Ryota Ozaki <ozaki.ryota@gmail.com>
+#
+# This work is licensed under the terms of the GNU GPL, version 2.  See
+# the COPYING file in the top-level directory.
+#
+# Usage:
+#
+# Start QEMU with:
+#
+# # qemu [...] -chardev socket,path=/tmp/qga.sock,server,nowait,id=qga0 \
+#   -device virtio-serial -device virtserialport,chardev=qga0,name=org.qemu.guest_agent.0
+#
+# Run the script:
+#
+# $ qemu-ga-client --address=/tmp/qga.sock <command> [args...]
+#
+# or
+#
+# $ export QGA_CLIENT_ADDRESS=/tmp/qga.sock
+# $ qemu-ga-client <command> [args...]
+#
+# For example:
+#
+# $ qemu-ga-client cat /etc/resolv.conf
+# # Generated by NetworkManager
+# nameserver 10.0.2.3
+# $ qemu-ga-client fsfreeze status
+# thawed
+# $ qemu-ga-client fsfreeze freeze
+# 2 filesystems frozen
+#
+# See also: http://wiki.qemu.org/Features/QAPI/GuestAgent
+#
+
+import base64
+import random
+
+import qmp
+
+
+class QemuGuestAgent(qmp.QEMUMonitorProtocol):
+    def __getattr__(self, name):
+        def wrapper(**kwds):
+            return self.command('guest-' + name.replace('_', '-'), **kwds)
+        return wrapper
+
+
+class QemuGuestAgentClient:
+    error = QemuGuestAgent.error
+
+    def __init__(self, address):
+        self.qga = QemuGuestAgent(address)
+        self.qga.connect(negotiate=False)
+
+    def sync(self, timeout=3):
+        # Avoid being blocked forever
+        if not self.ping(timeout):
+            raise EnvironmentError('Agent seems not alive')
+        uid = random.randint(0, (1 << 32) - 1)
+        while True:
+            ret = self.qga.sync(id=uid)
+            if isinstance(ret, int) and int(ret) == uid:
+                break
+
+    def __file_read_all(self, handle):
+        eof = False
+        data = ''
+        while not eof:
+            ret = self.qga.file_read(handle=handle, count=1024)
+            _data = base64.b64decode(ret['buf-b64'])
+            data += _data
+            eof = ret['eof']
+        return data
+
+    def read(self, path):
+        handle = self.qga.file_open(path=path)
+        try:
+            data = self.__file_read_all(handle)
+        finally:
+            self.qga.file_close(handle=handle)
+        return data
+
+    def info(self):
+        info = self.qga.info()
+
+        msgs = []
+        msgs.append('version: ' + info['version'])
+        msgs.append('supported_commands:')
+        enabled = [c['name'] for c in info['supported_commands'] if c['enabled']]
+        msgs.append('\tenabled: ' + ', '.join(enabled))
+        disabled = [c['name'] for c in info['supported_commands'] if not c['enabled']]
+        msgs.append('\tdisabled: ' + ', '.join(disabled))
+
+        return '\n'.join(msgs)
+
+    def __gen_ipv4_netmask(self, prefixlen):
+        mask = int('1' * prefixlen + '0' * (32 - prefixlen), 2)
+        return '.'.join([str(mask >> 24),
+                         str((mask >> 16) & 0xff),
+                         str((mask >> 8) & 0xff),
+                         str(mask & 0xff)])
+
+    def ifconfig(self):
+        nifs = self.qga.network_get_interfaces()
+
+        msgs = []
+        for nif in nifs:
+            msgs.append(nif['name'] + ':')
+            if 'ip-addresses' in nif:
+                for ipaddr in nif['ip-addresses']:
+                    if ipaddr['ip-address-type'] == 'ipv4':
+                        addr = ipaddr['ip-address']
+                        mask = self.__gen_ipv4_netmask(int(ipaddr['prefix']))
+                        msgs.append("\tinet %s  netmask %s" % (addr, mask))
+                    elif ipaddr['ip-address-type'] == 'ipv6':
+                        addr = ipaddr['ip-address']
+                        prefix = ipaddr['prefix']
+                        msgs.append("\tinet6 %s  prefixlen %s" % (addr, prefix))
+            if nif['hardware-address'] != '00:00:00:00:00:00':
+                msgs.append("\tether " + nif['hardware-address'])
+
+        return '\n'.join(msgs)
+
+    def ping(self, timeout):
+        self.qga.settimeout(timeout)
+        try:
+            self.qga.ping()
+        except self.qga.timeout:
+            return False
+        return True
+
+    def fsfreeze(self, cmd):
+        if cmd not in ['status', 'freeze', 'thaw']:
+            raise StandardError('Invalid command: ' + cmd)
+
+        return getattr(self.qga, 'fsfreeze' + '_' + cmd)()
+
+    def fstrim(self, minimum=0):
+        return getattr(self.qga, 'fstrim')(minimum=minimum)
+
+    def suspend(self, mode):
+        if mode not in ['disk', 'ram', 'hybrid']:
+            raise StandardError('Invalid mode: ' + mode)
+
+        try:
+            getattr(self.qga, 'suspend' + '_' + mode)()
+            # On error exception will raise
+        except self.qga.timeout:
+            # On success command will timed out
+            return
+
+    def shutdown(self, mode='powerdown'):
+        if mode not in ['powerdown', 'halt', 'reboot']:
+            raise StandardError('Invalid mode: ' + mode)
+
+        try:
+            self.qga.shutdown(mode=mode)
+        except self.qga.timeout:
+            return
+
+
+def _cmd_cat(client, args):
+    if len(args) != 1:
+        print('Invalid argument')
+        print('Usage: cat <file>')
+        sys.exit(1)
+    print(client.read(args[0]))
+
+
+def _cmd_fsfreeze(client, args):
+    usage = 'Usage: fsfreeze status|freeze|thaw'
+    if len(args) != 1:
+        print('Invalid argument')
+        print(usage)
+        sys.exit(1)
+    if args[0] not in ['status', 'freeze', 'thaw']:
+        print('Invalid command: ' + args[0])
+        print(usage)
+        sys.exit(1)
+    cmd = args[0]
+    ret = client.fsfreeze(cmd)
+    if cmd == 'status':
+        print(ret)
+    elif cmd == 'freeze':
+        print("%d filesystems frozen" % ret)
+    else:
+        print("%d filesystems thawed" % ret)
+
+
+def _cmd_fstrim(client, args):
+    if len(args) == 0:
+        minimum = 0
+    else:
+        minimum = int(args[0])
+    print(client.fstrim(minimum))
+
+
+def _cmd_ifconfig(client, args):
+    print(client.ifconfig())
+
+
+def _cmd_info(client, args):
+    print(client.info())
+
+
+def _cmd_ping(client, args):
+    if len(args) == 0:
+        timeout = 3
+    else:
+        timeout = float(args[0])
+    alive = client.ping(timeout)
+    if not alive:
+        print("Not responded in %s sec" % args[0])
+        sys.exit(1)
+
+
+def _cmd_suspend(client, args):
+    usage = 'Usage: suspend disk|ram|hybrid'
+    if len(args) != 1:
+        print('Less argument')
+        print(usage)
+        sys.exit(1)
+    if args[0] not in ['disk', 'ram', 'hybrid']:
+        print('Invalid command: ' + args[0])
+        print(usage)
+        sys.exit(1)
+    client.suspend(args[0])
+
+
+def _cmd_shutdown(client, args):
+    client.shutdown()
+_cmd_powerdown = _cmd_shutdown
+
+
+def _cmd_halt(client, args):
+    client.shutdown('halt')
+
+
+def _cmd_reboot(client, args):
+    client.shutdown('reboot')
+
+
+commands = [m.replace('_cmd_', '') for m in dir() if '_cmd_' in m]
+
+
+def main(address, cmd, args):
+    if not os.path.exists(address):
+        print('%s not found' % address)
+        sys.exit(1)
+
+    if cmd not in commands:
+        print('Invalid command: ' + cmd)
+        print('Available commands: ' + ', '.join(commands))
+        sys.exit(1)
+
+    try:
+        client = QemuGuestAgentClient(address)
+    except QemuGuestAgent.error, e:
+        import errno
+
+        print(e)
+        if e.errno == errno.ECONNREFUSED:
+            print('Hint: qemu is not running?')
+        sys.exit(1)
+
+    if cmd != 'ping':
+        client.sync()
+
+    globals()['_cmd_' + cmd](client, args)
+
+
+if __name__ == '__main__':
+    import sys
+    import os
+    import optparse
+
+    address = os.environ['QGA_CLIENT_ADDRESS'] if 'QGA_CLIENT_ADDRESS' in os.environ else None
+
+    usage = "%prog [--address=<unix_path>|<ipv4_address>] <command> [args...]\n"
+    usage += '<command>: ' + ', '.join(commands)
+    parser = optparse.OptionParser(usage=usage)
+    parser.add_option('--address', action='store', type='string',
+                      default=address, help='Specify a ip:port pair or a unix socket path')
+    options, args = parser.parse_args()
+
+    address = options.address
+    if address is None:
+        parser.error('address is not specified')
+        sys.exit(1)
+
+    if len(args) == 0:
+        parser.error('Less argument')
+        sys.exit(1)
+
+    main(address, args[0], args[1:])
diff --git a/QMP/qmp-events.txt b/QMP/qmp-events.txt
index 287805825..b2698e415 100644
--- a/QMP/qmp-events.txt
+++ b/QMP/qmp-events.txt
@@ -50,7 +50,8 @@ Emitted when a block job has been cancelled.
 
 Data:
 
-- "type":     Job type ("stream" for image streaming, json-string)
+- "type":     Job type (json-string; "stream" for image streaming
+                                     "commit" for block commit)
 - "device":   Device name (json-string)
 - "len":      Maximum progress value (json-int)
 - "offset":   Current progress value (json-int)
@@ -73,7 +74,8 @@ Emitted when a block job has completed.
 
 Data:
 
-- "type":     Job type ("stream" for image streaming, json-string)
+- "type":     Job type (json-string; "stream" for image streaming
+                                     "commit" for block commit)
 - "device":   Device name (json-string)
 - "len":      Maximum progress value (json-int)
 - "offset":   Current progress value (json-int)
@@ -94,6 +96,46 @@ Example:
                "speed": 0 },
      "timestamp": { "seconds": 1267061043, "microseconds": 959568 } }
 
+BLOCK_JOB_ERROR
+---------------
+
+Emitted when a block job encounters an error.
+
+Data:
+
+- "device": device name (json-string)
+- "operation": I/O operation (json-string, "read" or "write")
+- "action": action that has been taken, it's one of the following (json-string):
+    "ignore": error has been ignored, the job may fail later
+    "report": error will be reported and the job canceled
+    "stop": error caused job to be paused
+
+Example:
+
+{ "event": "BLOCK_JOB_ERROR",
+    "data": { "device": "ide0-hd1",
+              "operation": "write",
+              "action": "stop" },
+    "timestamp": { "seconds": 1265044230, "microseconds": 450486 } }
+
+BLOCK_JOB_READY
+---------------
+
+Emitted when a block job is ready to complete.
+
+Data:
+
+- "device": device name (json-string)
+
+Example:
+
+{ "event": "BLOCK_JOB_READY",
+    "data": { "device": "ide0-hd1" },
+    "timestamp": { "seconds": 1265044230, "microseconds": 450486 } }
+
+Note: The "ready to complete" status is always reset by a BLOCK_JOB_ERROR
+event.
+
 DEVICE_TRAY_MOVED
 -----------------
 
diff --git a/QMP/qmp-shell b/QMP/qmp-shell
index 42dabc8c6..24b665c8c 100755
--- a/QMP/qmp-shell
+++ b/QMP/qmp-shell
@@ -33,6 +33,7 @@
 import qmp
 import readline
 import sys
+import pprint
 
 class QMPCompleter(list):
     def complete(self, text, state):
@@ -52,10 +53,11 @@ class QMPShellBadPort(QMPShellError):
 # TODO: QMPShell's interface is a bit ugly (eg. _fill_completion() and
 #       _execute_cmd()). Let's design a better one.
 class QMPShell(qmp.QEMUMonitorProtocol):
-    def __init__(self, address):
+    def __init__(self, address, pp=None):
         qmp.QEMUMonitorProtocol.__init__(self, self.__get_address(address))
         self._greeting = None
         self._completer = None
+        self._pp = pp
 
     def __get_address(self, arg):
         """
@@ -114,7 +116,11 @@ class QMPShell(qmp.QEMUMonitorProtocol):
         if resp is None:
             print 'Disconnected'
             return False
-        print resp
+
+        if self._pp is not None:
+            self._pp.pprint(resp)
+        else:
+            print resp
         return True
 
     def connect(self):
@@ -222,22 +228,36 @@ def die(msg):
 def fail_cmdline(option=None):
     if option:
         sys.stderr.write('ERROR: bad command-line option \'%s\'\n' % option)
-    sys.stderr.write('qemu-shell [ -H ] < UNIX socket path> | < TCP address:port >\n')
+    sys.stderr.write('qemu-shell [ -p ] [ -H ] < UNIX socket path> | < TCP address:port >\n')
     sys.exit(1)
 
 def main():
     addr = ''
+    qemu = None
+    hmp = False
+    pp = None
+
     try:
-        if len(sys.argv) == 2:
-            qemu = QMPShell(sys.argv[1])
-            addr = sys.argv[1]
-        elif len(sys.argv) == 3:
-            if sys.argv[1] != '-H':
-                fail_cmdline(sys.argv[1])
-            qemu = HMPShell(sys.argv[2])
-            addr = sys.argv[2]
-        else:
-                fail_cmdline()
+        for arg in sys.argv[1:]:
+            if arg == "-H":
+                if qemu is not None:
+                    fail_cmdline(arg)
+                hmp = True
+            elif arg == "-p":
+                if pp is not None:
+                    fail_cmdline(arg)
+                pp = pprint.PrettyPrinter(indent=4)
+            else:
+                if qemu is not None:
+                    fail_cmdline(arg)
+                if hmp:
+                    qemu = HMPShell(arg)
+                else:
+                    qemu = QMPShell(arg, pp)
+                addr = arg
+
+        if qemu is None:
+            fail_cmdline()
     except QMPShellBadPort:
         die('bad port number in command-line')
 
diff --git a/QMP/qmp.py b/QMP/qmp.py
index 36ecc1dfa..c551df1ed 100644
--- a/QMP/qmp.py
+++ b/QMP/qmp.py
@@ -49,7 +49,6 @@ class QEMUMonitorProtocol:
         return socket.socket(family, socket.SOCK_STREAM)
 
     def __negotiate_capabilities(self):
-        self.__sockfile = self.__sock.makefile()
         greeting = self.__json_read()
         if greeting is None or not greeting.has_key('QMP'):
             raise QMPConnectError
@@ -73,7 +72,7 @@ class QEMUMonitorProtocol:
 
     error = socket.error
 
-    def connect(self):
+    def connect(self, negotiate=True):
         """
         Connect to the QMP Monitor and perform capabilities negotiation.
 
@@ -83,7 +82,9 @@ class QEMUMonitorProtocol:
         @raise QMPCapabilitiesError if fails to negotiate capabilities
         """
         self.__sock.connect(self.__address)
-        return self.__negotiate_capabilities()
+        self.__sockfile = self.__sock.makefile()
+        if negotiate:
+            return self.__negotiate_capabilities()
 
     def accept(self):
         """
@@ -95,6 +96,7 @@ class QEMUMonitorProtocol:
         @raise QMPCapabilitiesError if fails to negotiate capabilities
         """
         self.__sock, _ = self.__sock.accept()
+        self.__sockfile = self.__sock.makefile()
         return self.__negotiate_capabilities()
 
     def cmd_obj(self, qmp_cmd):
@@ -134,6 +136,26 @@ class QEMUMonitorProtocol:
             raise Exception(ret['error']['desc'])
         return ret['return']
 
+    def pull_event(self, wait=False):
+        """
+        Get and delete the first available QMP event.
+
+        @param wait: block until an event is available (bool)
+        """
+        self.__sock.setblocking(0)
+        try:
+            self.__json_read()
+        except socket.error, err:
+            if err[0] == errno.EAGAIN:
+                # No data available
+                pass
+        self.__sock.setblocking(1)
+        if not self.__events and wait:
+            self.__json_read(only_event=True)
+        event = self.__events[0]
+        del self.__events[0]
+        return event
+
     def get_events(self, wait=False):
         """
         Get a list of available QMP events.
@@ -161,3 +183,8 @@ class QEMUMonitorProtocol:
     def close(self):
         self.__sock.close()
         self.__sockfile.close()
+
+    timeout = socket.timeout
+
+    def settimeout(self, timeout):
+        self.__sock.settimeout(timeout)
diff --git a/VERSION b/VERSION
index 26aaba0e8..f0bb29e76 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-1.2.0
+1.3.0
diff --git a/aio-posix.c b/aio-posix.c
new file mode 100644
index 000000000..05cc84e12
--- /dev/null
+++ b/aio-posix.c
@@ -0,0 +1,268 @@
+/*
+ * QEMU aio implementation
+ *
+ * Copyright IBM, Corp. 2008
+ *
+ * Authors:
+ *  Anthony Liguori   <aliguori@us.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ *
+ * Contributions after 2012-01-13 are licensed under the terms of the
+ * GNU GPL, version 2 or (at your option) any later version.
+ */
+
+#include "qemu-common.h"
+#include "block.h"
+#include "qemu-queue.h"
+#include "qemu_socket.h"
+
+struct AioHandler
+{
+    GPollFD pfd;
+    IOHandler *io_read;
+    IOHandler *io_write;
+    AioFlushHandler *io_flush;
+    int deleted;
+    void *opaque;
+    QLIST_ENTRY(AioHandler) node;
+};
+
+static AioHandler *find_aio_handler(AioContext *ctx, int fd)
+{
+    AioHandler *node;
+
+    QLIST_FOREACH(node, &ctx->aio_handlers, node) {
+        if (node->pfd.fd == fd)
+            if (!node->deleted)
+                return node;
+    }
+
+    return NULL;
+}
+
+void aio_set_fd_handler(AioContext *ctx,
+                        int fd,
+                        IOHandler *io_read,
+                        IOHandler *io_write,
+                        AioFlushHandler *io_flush,
+                        void *opaque)
+{
+    AioHandler *node;
+
+    node = find_aio_handler(ctx, fd);
+
+    /* Are we deleting the fd handler? */
+    if (!io_read && !io_write) {
+        if (node) {
+            g_source_remove_poll(&ctx->source, &node->pfd);
+
+            /* If the lock is held, just mark the node as deleted */
+            if (ctx->walking_handlers) {
+                node->deleted = 1;
+                node->pfd.revents = 0;
+            } else {
+                /* Otherwise, delete it for real.  We can't just mark it as
+                 * deleted because deleted nodes are only cleaned up after
+                 * releasing the walking_handlers lock.
+                 */
+                QLIST_REMOVE(node, node);
+                g_free(node);
+            }
+        }
+    } else {
+        if (node == NULL) {
+            /* Alloc and insert if it's not already there */
+            node = g_malloc0(sizeof(AioHandler));
+            node->pfd.fd = fd;
+            QLIST_INSERT_HEAD(&ctx->aio_handlers, node, node);
+
+            g_source_add_poll(&ctx->source, &node->pfd);
+        }
+        /* Update handler with latest information */
+        node->io_read = io_read;
+        node->io_write = io_write;
+        node->io_flush = io_flush;
+        node->opaque = opaque;
+
+        node->pfd.events = (io_read ? G_IO_IN | G_IO_HUP : 0);
+        node->pfd.events |= (io_write ? G_IO_OUT : 0);
+    }
+
+    aio_notify(ctx);
+}
+
+void aio_set_event_notifier(AioContext *ctx,
+                            EventNotifier *notifier,
+                            EventNotifierHandler *io_read,
+                            AioFlushEventNotifierHandler *io_flush)
+{
+    aio_set_fd_handler(ctx, event_notifier_get_fd(notifier),
+                       (IOHandler *)io_read, NULL,
+                       (AioFlushHandler *)io_flush, notifier);
+}
+
+bool aio_pending(AioContext *ctx)
+{
+    AioHandler *node;
+
+    QLIST_FOREACH(node, &ctx->aio_handlers, node) {
+        int revents;
+
+        /*
+         * FIXME: right now we cannot get G_IO_HUP and G_IO_ERR because
+         * main-loop.c is still select based (due to the slirp legacy).
+         * If main-loop.c ever switches to poll, G_IO_ERR should be
+         * tested too.  Dispatching G_IO_ERR to both handlers should be
+         * okay, since handlers need to be ready for spurious wakeups.
+         */
+        revents = node->pfd.revents & node->pfd.events;
+        if (revents & (G_IO_IN | G_IO_HUP | G_IO_ERR) && node->io_read) {
+            return true;
+        }
+        if (revents & (G_IO_OUT | G_IO_ERR) && node->io_write) {
+            return true;
+        }
+    }
+
+    return false;
+}
+
+bool aio_poll(AioContext *ctx, bool blocking)
+{
+    static struct timeval tv0;
+    AioHandler *node;
+    fd_set rdfds, wrfds;
+    int max_fd = -1;
+    int ret;
+    bool busy, progress;
+
+    progress = false;
+
+    /*
+     * If there are callbacks left that have been queued, we need to call then.
+     * Do not call select in this case, because it is possible that the caller
+     * does not need a complete flush (as is the case for qemu_aio_wait loops).
+     */
+    if (aio_bh_poll(ctx)) {
+        blocking = false;
+        progress = true;
+    }
+
+    /*
+     * Then dispatch any pending callbacks from the GSource.
+     *
+     * We have to walk very carefully in case qemu_aio_set_fd_handler is
+     * called while we're walking.
+     */
+    node = QLIST_FIRST(&ctx->aio_handlers);
+    while (node) {
+        AioHandler *tmp;
+        int revents;
+
+        ctx->walking_handlers++;
+
+        revents = node->pfd.revents & node->pfd.events;
+        node->pfd.revents = 0;
+
+        /* See comment in aio_pending.  */
+        if (revents & (G_IO_IN | G_IO_HUP | G_IO_ERR) && node->io_read) {
+            node->io_read(node->opaque);
+            progress = true;
+        }
+        if (revents & (G_IO_OUT | G_IO_ERR) && node->io_write) {
+            node->io_write(node->opaque);
+            progress = true;
+        }
+
+        tmp = node;
+        node = QLIST_NEXT(node, node);
+
+        ctx->walking_handlers--;
+
+        if (!ctx->walking_handlers && tmp->deleted) {
+            QLIST_REMOVE(tmp, node);
+            g_free(tmp);
+        }
+    }
+
+    if (progress && !blocking) {
+        return true;
+    }
+
+    ctx->walking_handlers++;
+
+    FD_ZERO(&rdfds);
+    FD_ZERO(&wrfds);
+
+    /* fill fd sets */
+    busy = false;
+    QLIST_FOREACH(node, &ctx->aio_handlers, node) {
+        /* If there aren't pending AIO operations, don't invoke callbacks.
+         * Otherwise, if there are no AIO requests, qemu_aio_wait() would
+         * wait indefinitely.
+         */
+        if (!node->deleted && node->io_flush) {
+            if (node->io_flush(node->opaque) == 0) {
+                continue;
+            }
+            busy = true;
+        }
+        if (!node->deleted && node->io_read) {
+            FD_SET(node->pfd.fd, &rdfds);
+            max_fd = MAX(max_fd, node->pfd.fd + 1);
+        }
+        if (!node->deleted && node->io_write) {
+            FD_SET(node->pfd.fd, &wrfds);
+            max_fd = MAX(max_fd, node->pfd.fd + 1);
+        }
+    }
+
+    ctx->walking_handlers--;
+
+    /* No AIO operations?  Get us out of here */
+    if (!busy) {
+        return progress;
+    }
+
+    /* wait until next event */
+    ret = select(max_fd, &rdfds, &wrfds, NULL, blocking ? NULL : &tv0);
+
+    /* if we have any readable fds, dispatch event */
+    if (ret > 0) {
+        /* we have to walk very carefully in case
+         * qemu_aio_set_fd_handler is called while we're walking */
+        node = QLIST_FIRST(&ctx->aio_handlers);
+        while (node) {
+            AioHandler *tmp;
+
+            ctx->walking_handlers++;
+
+            if (!node->deleted &&
+                FD_ISSET(node->pfd.fd, &rdfds) &&
+                node->io_read) {
+                node->io_read(node->opaque);
+                progress = true;
+            }
+            if (!node->deleted &&
+                FD_ISSET(node->pfd.fd, &wrfds) &&
+                node->io_write) {
+                node->io_write(node->opaque);
+                progress = true;
+            }
+
+            tmp = node;
+            node = QLIST_NEXT(node, node);
+
+            ctx->walking_handlers--;
+
+            if (!ctx->walking_handlers && tmp->deleted) {
+                QLIST_REMOVE(tmp, node);
+                g_free(tmp);
+            }
+        }
+    }
+
+    return progress;
+}
diff --git a/aio-win32.c b/aio-win32.c
new file mode 100644
index 000000000..cec464663
--- /dev/null
+++ b/aio-win32.c
@@ -0,0 +1,218 @@
+/*
+ * QEMU aio implementation
+ *
+ * Copyright IBM Corp., 2008
+ * Copyright Red Hat Inc., 2012
+ *
+ * Authors:
+ *  Anthony Liguori   <aliguori@us.ibm.com>
+ *  Paolo Bonzini     <pbonzini@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ *
+ * Contributions after 2012-01-13 are licensed under the terms of the
+ * GNU GPL, version 2 or (at your option) any later version.
+ */
+
+#include "qemu-common.h"
+#include "block.h"
+#include "qemu-queue.h"
+#include "qemu_socket.h"
+
+struct AioHandler {
+    EventNotifier *e;
+    EventNotifierHandler *io_notify;
+    AioFlushEventNotifierHandler *io_flush;
+    GPollFD pfd;
+    int deleted;
+    QLIST_ENTRY(AioHandler) node;
+};
+
+void aio_set_event_notifier(AioContext *ctx,
+                            EventNotifier *e,
+                            EventNotifierHandler *io_notify,
+                            AioFlushEventNotifierHandler *io_flush)
+{
+    AioHandler *node;
+
+    QLIST_FOREACH(node, &ctx->aio_handlers, node) {
+        if (node->e == e && !node->deleted) {
+            break;
+        }
+    }
+
+    /* Are we deleting the fd handler? */
+    if (!io_notify) {
+        if (node) {
+            g_source_remove_poll(&ctx->source, &node->pfd);
+
+            /* If the lock is held, just mark the node as deleted */
+            if (ctx->walking_handlers) {
+                node->deleted = 1;
+                node->pfd.revents = 0;
+            } else {
+                /* Otherwise, delete it for real.  We can't just mark it as
+                 * deleted because deleted nodes are only cleaned up after
+                 * releasing the walking_handlers lock.
+                 */
+                QLIST_REMOVE(node, node);
+                g_free(node);
+            }
+        }
+    } else {
+        if (node == NULL) {
+            /* Alloc and insert if it's not already there */
+            node = g_malloc0(sizeof(AioHandler));
+            node->e = e;
+            node->pfd.fd = (uintptr_t)event_notifier_get_handle(e);
+            node->pfd.events = G_IO_IN;
+            QLIST_INSERT_HEAD(&ctx->aio_handlers, node, node);
+
+            g_source_add_poll(&ctx->source, &node->pfd);
+        }
+        /* Update handler with latest information */
+        node->io_notify = io_notify;
+        node->io_flush = io_flush;
+    }
+
+    aio_notify(ctx);
+}
+
+bool aio_pending(AioContext *ctx)
+{
+    AioHandler *node;
+
+    QLIST_FOREACH(node, &ctx->aio_handlers, node) {
+        if (node->pfd.revents && node->io_notify) {
+            return true;
+        }
+    }
+
+    return false;
+}
+
+bool aio_poll(AioContext *ctx, bool blocking)
+{
+    AioHandler *node;
+    HANDLE events[MAXIMUM_WAIT_OBJECTS + 1];
+    bool busy, progress;
+    int count;
+
+    progress = false;
+
+    /*
+     * If there are callbacks left that have been queued, we need to call then.
+     * Do not call select in this case, because it is possible that the caller
+     * does not need a complete flush (as is the case for qemu_aio_wait loops).
+     */
+    if (aio_bh_poll(ctx)) {
+        blocking = false;
+        progress = true;
+    }
+
+    /*
+     * Then dispatch any pending callbacks from the GSource.
+     *
+     * We have to walk very carefully in case qemu_aio_set_fd_handler is
+     * called while we're walking.
+     */
+    node = QLIST_FIRST(&ctx->aio_handlers);
+    while (node) {
+        AioHandler *tmp;
+
+        ctx->walking_handlers++;
+
+        if (node->pfd.revents && node->io_notify) {
+            node->pfd.revents = 0;
+            node->io_notify(node->e);
+            progress = true;
+        }
+
+        tmp = node;
+        node = QLIST_NEXT(node, node);
+
+        ctx->walking_handlers--;
+
+        if (!ctx->walking_handlers && tmp->deleted) {
+            QLIST_REMOVE(tmp, node);
+            g_free(tmp);
+        }
+    }
+
+    if (progress && !blocking) {
+        return true;
+    }
+
+    ctx->walking_handlers++;
+
+    /* fill fd sets */
+    busy = false;
+    count = 0;
+    QLIST_FOREACH(node, &ctx->aio_handlers, node) {
+        /* If there aren't pending AIO operations, don't invoke callbacks.
+         * Otherwise, if there are no AIO requests, qemu_aio_wait() would
+         * wait indefinitely.
+         */
+        if (!node->deleted && node->io_flush) {
+            if (node->io_flush(node->e) == 0) {
+                continue;
+            }
+            busy = true;
+        }
+        if (!node->deleted && node->io_notify) {
+            events[count++] = event_notifier_get_handle(node->e);
+        }
+    }
+
+    ctx->walking_handlers--;
+
+    /* No AIO operations?  Get us out of here */
+    if (!busy) {
+        return progress;
+    }
+
+    /* wait until next event */
+    while (count > 0) {
+        int timeout = blocking ? INFINITE : 0;
+        int ret = WaitForMultipleObjects(count, events, FALSE, timeout);
+
+        /* if we have any signaled events, dispatch event */
+        if ((DWORD) (ret - WAIT_OBJECT_0) >= count) {
+            break;
+        }
+
+        blocking = false;
+
+        /* we have to walk very carefully in case
+         * qemu_aio_set_fd_handler is called while we're walking */
+        node = QLIST_FIRST(&ctx->aio_handlers);
+        while (node) {
+            AioHandler *tmp;
+
+            ctx->walking_handlers++;
+
+            if (!node->deleted &&
+                event_notifier_get_handle(node->e) == events[ret - WAIT_OBJECT_0] &&
+                node->io_notify) {
+                node->io_notify(node->e);
+                progress = true;
+            }
+
+            tmp = node;
+            node = QLIST_NEXT(node, node);
+
+            ctx->walking_handlers--;
+
+            if (!ctx->walking_handlers && tmp->deleted) {
+                QLIST_REMOVE(tmp, node);
+                g_free(tmp);
+            }
+        }
+
+        /* Try again, but only call each handler once.  */
+        events[ret - WAIT_OBJECT_0] = events[--count];
+    }
+
+    return progress;
+}
diff --git a/aio.c b/aio.c
deleted file mode 100644
index 0a9eb10c7..000000000
--- a/aio.c
+++ /dev/null
@@ -1,194 +0,0 @@
-/*
- * QEMU aio implementation
- *
- * Copyright IBM, Corp. 2008
- *
- * Authors:
- *  Anthony Liguori   <aliguori@us.ibm.com>
- *
- * This work is licensed under the terms of the GNU GPL, version 2.  See
- * the COPYING file in the top-level directory.
- *
- * Contributions after 2012-01-13 are licensed under the terms of the
- * GNU GPL, version 2 or (at your option) any later version.
- */
-
-#include "qemu-common.h"
-#include "block.h"
-#include "qemu-queue.h"
-#include "qemu_socket.h"
-
-typedef struct AioHandler AioHandler;
-
-/* The list of registered AIO handlers */
-static QLIST_HEAD(, AioHandler) aio_handlers;
-
-/* This is a simple lock used to protect the aio_handlers list.  Specifically,
- * it's used to ensure that no callbacks are removed while we're walking and
- * dispatching callbacks.
- */
-static int walking_handlers;
-
-struct AioHandler
-{
-    int fd;
-    IOHandler *io_read;
-    IOHandler *io_write;
-    AioFlushHandler *io_flush;
-    int deleted;
-    void *opaque;
-    QLIST_ENTRY(AioHandler) node;
-};
-
-static AioHandler *find_aio_handler(int fd)
-{
-    AioHandler *node;
-
-    QLIST_FOREACH(node, &aio_handlers, node) {
-        if (node->fd == fd)
-            if (!node->deleted)
-                return node;
-    }
-
-    return NULL;
-}
-
-int qemu_aio_set_fd_handler(int fd,
-                            IOHandler *io_read,
-                            IOHandler *io_write,
-                            AioFlushHandler *io_flush,
-                            void *opaque)
-{
-    AioHandler *node;
-
-    node = find_aio_handler(fd);
-
-    /* Are we deleting the fd handler? */
-    if (!io_read && !io_write) {
-        if (node) {
-            /* If the lock is held, just mark the node as deleted */
-            if (walking_handlers)
-                node->deleted = 1;
-            else {
-                /* Otherwise, delete it for real.  We can't just mark it as
-                 * deleted because deleted nodes are only cleaned up after
-                 * releasing the walking_handlers lock.
-                 */
-                QLIST_REMOVE(node, node);
-                g_free(node);
-            }
-        }
-    } else {
-        if (node == NULL) {
-            /* Alloc and insert if it's not already there */
-            node = g_malloc0(sizeof(AioHandler));
-            node->fd = fd;
-            QLIST_INSERT_HEAD(&aio_handlers, node, node);
-        }
-        /* Update handler with latest information */
-        node->io_read = io_read;
-        node->io_write = io_write;
-        node->io_flush = io_flush;
-        node->opaque = opaque;
-    }
-
-    qemu_set_fd_handler2(fd, NULL, io_read, io_write, opaque);
-
-    return 0;
-}
-
-void qemu_aio_flush(void)
-{
-    while (qemu_aio_wait());
-}
-
-bool qemu_aio_wait(void)
-{
-    AioHandler *node;
-    fd_set rdfds, wrfds;
-    int max_fd = -1;
-    int ret;
-    bool busy;
-
-    /*
-     * If there are callbacks left that have been queued, we need to call then.
-     * Do not call select in this case, because it is possible that the caller
-     * does not need a complete flush (as is the case for qemu_aio_wait loops).
-     */
-    if (qemu_bh_poll()) {
-        return true;
-    }
-
-    walking_handlers = 1;
-
-    FD_ZERO(&rdfds);
-    FD_ZERO(&wrfds);
-
-    /* fill fd sets */
-    busy = false;
-    QLIST_FOREACH(node, &aio_handlers, node) {
-        /* If there aren't pending AIO operations, don't invoke callbacks.
-         * Otherwise, if there are no AIO requests, qemu_aio_wait() would
-         * wait indefinitely.
-         */
-        if (node->io_flush) {
-            if (node->io_flush(node->opaque) == 0) {
-                continue;
-            }
-            busy = true;
-        }
-        if (!node->deleted && node->io_read) {
-            FD_SET(node->fd, &rdfds);
-            max_fd = MAX(max_fd, node->fd + 1);
-        }
-        if (!node->deleted && node->io_write) {
-            FD_SET(node->fd, &wrfds);
-            max_fd = MAX(max_fd, node->fd + 1);
-        }
-    }
-
-    walking_handlers = 0;
-
-    /* No AIO operations?  Get us out of here */
-    if (!busy) {
-        return false;
-    }
-
-    /* wait until next event */
-    ret = select(max_fd, &rdfds, &wrfds, NULL, NULL);
-
-    /* if we have any readable fds, dispatch event */
-    if (ret > 0) {
-        walking_handlers = 1;
-
-        /* we have to walk very carefully in case
-         * qemu_aio_set_fd_handler is called while we're walking */
-        node = QLIST_FIRST(&aio_handlers);
-        while (node) {
-            AioHandler *tmp;
-
-            if (!node->deleted &&
-                FD_ISSET(node->fd, &rdfds) &&
-                node->io_read) {
-                node->io_read(node->opaque);
-            }
-            if (!node->deleted &&
-                FD_ISSET(node->fd, &wrfds) &&
-                node->io_write) {
-                node->io_write(node->opaque);
-            }
-
-            tmp = node;
-            node = QLIST_NEXT(node, node);
-
-            if (tmp->deleted) {
-                QLIST_REMOVE(tmp, node);
-                g_free(tmp);
-            }
-        }
-
-        walking_handlers = 0;
-    }
-
-    return true;
-}
diff --git a/arch_init.c b/arch_init.c
index 5a1173e29..e6effe809 100644
--- a/arch_init.c
+++ b/arch_init.c
@@ -31,6 +31,8 @@
 #include "config.h"
 #include "monitor.h"
 #include "sysemu.h"
+#include "bitops.h"
+#include "bitmap.h"
 #include "arch_init.h"
 #include "audio/audio.h"
 #include "hw/pc.h"
@@ -45,6 +47,7 @@
 #include "hw/pcspk.h"
 #include "qemu/page_cache.h"
 #include "qmp-commands.h"
+#include "trace.h"
 
 #ifdef DEBUG_ARCH_INIT
 #define DPRINTF(fmt, ...) \
@@ -136,7 +139,6 @@ static struct defconfig_file {
     /* Indicates it is an user config file (disabled by -no-user-config) */
     bool userconfig;
 } default_config_files[] = {
-    { CONFIG_QEMU_DATADIR "/cpus-" TARGET_ARCH ".conf",  false },
     { CONFIG_QEMU_CONFDIR "/qemu.conf",                   true },
     { CONFIG_QEMU_CONFDIR "/target-" TARGET_ARCH ".conf", true },
     { NULL }, /* end of list */
@@ -331,6 +333,78 @@ static int save_xbzrle_page(QEMUFile *f, uint8_t *current_data,
 
 static RAMBlock *last_block;
 static ram_addr_t last_offset;
+static unsigned long *migration_bitmap;
+static uint64_t migration_dirty_pages;
+
+static inline bool migration_bitmap_test_and_reset_dirty(MemoryRegion *mr,
+                                                         ram_addr_t offset)
+{
+    bool ret;
+    int nr = (mr->ram_addr + offset) >> TARGET_PAGE_BITS;
+
+    ret = test_and_clear_bit(nr, migration_bitmap);
+
+    if (ret) {
+        migration_dirty_pages--;
+    }
+    return ret;
+}
+
+static inline bool migration_bitmap_set_dirty(MemoryRegion *mr,
+                                              ram_addr_t offset)
+{
+    bool ret;
+    int nr = (mr->ram_addr + offset) >> TARGET_PAGE_BITS;
+
+    ret = test_and_set_bit(nr, migration_bitmap);
+
+    if (!ret) {
+        migration_dirty_pages++;
+    }
+    return ret;
+}
+
+static void migration_bitmap_sync(void)
+{
+    RAMBlock *block;
+    ram_addr_t addr;
+    uint64_t num_dirty_pages_init = migration_dirty_pages;
+    MigrationState *s = migrate_get_current();
+    static int64_t start_time;
+    static int64_t num_dirty_pages_period;
+    int64_t end_time;
+
+    if (!start_time) {
+        start_time = qemu_get_clock_ms(rt_clock);
+    }
+
+    trace_migration_bitmap_sync_start();
+    memory_global_sync_dirty_bitmap(get_system_memory());
+
+    QLIST_FOREACH(block, &ram_list.blocks, next) {
+        for (addr = 0; addr < block->length; addr += TARGET_PAGE_SIZE) {
+            if (memory_region_get_dirty(block->mr, addr, TARGET_PAGE_SIZE,
+                                        DIRTY_MEMORY_MIGRATION)) {
+                migration_bitmap_set_dirty(block->mr, addr);
+            }
+        }
+        memory_region_reset_dirty(block->mr, 0, block->length,
+                                  DIRTY_MEMORY_MIGRATION);
+    }
+    trace_migration_bitmap_sync_end(migration_dirty_pages
+                                    - num_dirty_pages_init);
+    num_dirty_pages_period += migration_dirty_pages - num_dirty_pages_init;
+    end_time = qemu_get_clock_ms(rt_clock);
+
+    /* more than 1 second = 1000 millisecons */
+    if (end_time > start_time + 1000) {
+        s->dirty_pages_rate = num_dirty_pages_period * 1000
+            / (end_time - start_time);
+        start_time = end_time;
+        num_dirty_pages_period = 0;
+    }
+}
+
 
 /*
  * ram_save_block: Writes a page of memory to the stream f
@@ -353,14 +427,10 @@ static int ram_save_block(QEMUFile *f, bool last_stage)
 
     do {
         mr = block->mr;
-        if (memory_region_get_dirty(mr, offset, TARGET_PAGE_SIZE,
-                                    DIRTY_MEMORY_MIGRATION)) {
+        if (migration_bitmap_test_and_reset_dirty(mr, offset)) {
             uint8_t *p;
             int cont = (block == last_block) ? RAM_SAVE_FLAG_CONTINUE : 0;
 
-            memory_region_reset_dirty(mr, offset, TARGET_PAGE_SIZE,
-                                      DIRTY_MEMORY_MIGRATION);
-
             p = memory_region_get_ram_ptr(mr) + offset;
 
             if (is_dup_page(p)) {
@@ -410,7 +480,7 @@ static uint64_t bytes_transferred;
 
 static ram_addr_t ram_save_remaining(void)
 {
-    return ram_list.dirty_pages;
+    return migration_dirty_pages;
 }
 
 uint64_t ram_bytes_remaining(void)
@@ -482,17 +552,27 @@ static void ram_migration_cancel(void *opaque)
     migration_end();
 }
 
+
+static void reset_ram_globals(void)
+{
+    last_block = NULL;
+    last_offset = 0;
+    sort_ram_list();
+}
+
 #define MAX_WAIT 50 /* ms, half buffered_file limit */
 
 static int ram_save_setup(QEMUFile *f, void *opaque)
 {
-    ram_addr_t addr;
     RAMBlock *block;
+    int64_t ram_pages = last_ram_offset() >> TARGET_PAGE_BITS;
+
+    migration_bitmap = bitmap_new(ram_pages);
+    bitmap_set(migration_bitmap, 1, ram_pages);
+    migration_dirty_pages = ram_pages;
 
     bytes_transferred = 0;
-    last_block = NULL;
-    last_offset = 0;
-    sort_ram_list();
+    reset_ram_globals();
 
     if (migrate_use_xbzrle()) {
         XBZRLE.cache = cache_init(migrate_xbzrle_cache_size() /
@@ -507,17 +587,8 @@ static int ram_save_setup(QEMUFile *f, void *opaque)
         acct_clear();
     }
 
-    /* Make sure all dirty bits are set */
-    QLIST_FOREACH(block, &ram_list.blocks, next) {
-        for (addr = 0; addr < block->length; addr += TARGET_PAGE_SIZE) {
-            if (!memory_region_get_dirty(block->mr, addr, TARGET_PAGE_SIZE,
-                                         DIRTY_MEMORY_MIGRATION)) {
-                memory_region_set_dirty(block->mr, addr, TARGET_PAGE_SIZE);
-            }
-        }
-    }
-
     memory_global_dirty_log_start();
+    migration_bitmap_sync();
 
     qemu_put_be64(f, ram_bytes_total() | RAM_SAVE_FLAG_MEM_SIZE);
 
@@ -538,7 +609,8 @@ static int ram_save_iterate(QEMUFile *f, void *opaque)
     double bwidth = 0;
     int ret;
     int i;
-    uint64_t expected_time;
+    uint64_t expected_downtime;
+    MigrationState *s = migrate_get_current();
 
     bytes_transferred_last = bytes_transferred;
     bwidth = qemu_get_clock_ns(rt_clock);
@@ -562,7 +634,7 @@ static int ram_save_iterate(QEMUFile *f, void *opaque)
         if ((i & 63) == 0) {
             uint64_t t1 = (qemu_get_clock_ns(rt_clock) - bwidth) / 1000000;
             if (t1 > MAX_WAIT) {
-                DPRINTF("big wait: " PRIu64 " milliseconds, %d iterations\n",
+                DPRINTF("big wait: %" PRIu64 " milliseconds, %d iterations\n",
                         t1, i);
                 break;
             }
@@ -577,31 +649,32 @@ static int ram_save_iterate(QEMUFile *f, void *opaque)
     bwidth = qemu_get_clock_ns(rt_clock) - bwidth;
     bwidth = (bytes_transferred - bytes_transferred_last) / bwidth;
 
-    /* if we haven't transferred anything this round, force expected_time to a
-     * a very high value, but without crashing */
+    /* if we haven't transferred anything this round, force
+     * expected_downtime to a very high value, but without
+     * crashing */
     if (bwidth == 0) {
         bwidth = 0.000001;
     }
 
     qemu_put_be64(f, RAM_SAVE_FLAG_EOS);
 
-    expected_time = ram_save_remaining() * TARGET_PAGE_SIZE / bwidth;
+    expected_downtime = ram_save_remaining() * TARGET_PAGE_SIZE / bwidth;
+    DPRINTF("ram_save_live: expected(%" PRIu64 ") <= max(" PRIu64 ")?\n",
+            expected_downtime, migrate_max_downtime());
 
-    DPRINTF("ram_save_live: expected(" PRIu64 ") <= max(" PRIu64 ")?\n",
-            expected_time, migrate_max_downtime());
+    if (expected_downtime <= migrate_max_downtime()) {
+        migration_bitmap_sync();
+        expected_downtime = ram_save_remaining() * TARGET_PAGE_SIZE / bwidth;
+        s->expected_downtime = expected_downtime / 1000000; /* ns -> ms */
 
-    if (expected_time <= migrate_max_downtime()) {
-        memory_global_sync_dirty_bitmap(get_system_memory());
-        expected_time = ram_save_remaining() * TARGET_PAGE_SIZE / bwidth;
-
-        return expected_time <= migrate_max_downtime();
+        return expected_downtime <= migrate_max_downtime();
     }
     return 0;
 }
 
 static int ram_save_complete(QEMUFile *f, void *opaque)
 {
-    memory_global_sync_dirty_bitmap(get_system_memory());
+    migration_bitmap_sync();
 
     /* try transferring iterative blocks of memory */
 
@@ -620,6 +693,9 @@ static int ram_save_complete(QEMUFile *f, void *opaque)
 
     qemu_put_be64(f, RAM_SAVE_FLAG_EOS);
 
+    g_free(migration_bitmap);
+    migration_bitmap = NULL;
+
     return 0;
 }
 
@@ -799,8 +875,8 @@ static int ram_load(QEMUFile *f, void *opaque, int version_id)
     } while (!(flags & RAM_SAVE_FLAG_EOS));
 
 done:
-    DPRINTF("Completed load of VM with exit code %d seq iteration " PRIu64 "\n",
-            ret, seq_iter);
+    DPRINTF("Completed load of VM with exit code %d seq iteration "
+            "%" PRIu64 "\n", ret, seq_iter);
     return ret;
 }
 
@@ -922,11 +998,16 @@ void select_soundhw(const char *optarg)
     if (is_help_option(optarg)) {
     show_valid_cards:
 
+#ifdef HAS_AUDIO_CHOICE
         printf("Valid sound card names (comma separated):\n");
         for (c = soundhw; c->name; ++c) {
             printf ("%-11s %s\n", c->name, c->descr);
         }
         printf("\n-soundhw all will enable all of the above\n");
+#else
+        printf("Machine has no user-selectable audio hardware "
+               "(it may or may not have always-present audio hardware).\n");
+#endif
         exit(!is_help_option(optarg));
     }
     else {
diff --git a/arch_init.h b/arch_init.h
index d9c572aee..5fc780c63 100644
--- a/arch_init.h
+++ b/arch_init.h
@@ -34,6 +34,6 @@ int tcg_available(void);
 int kvm_available(void);
 int xen_available(void);
 
-CpuDefinitionInfoList GCC_WEAK_DECL *arch_query_cpu_definitions(Error **errp);
+CpuDefinitionInfoList *arch_query_cpu_definitions(Error **errp);
 
 #endif
diff --git a/async.c b/async.c
index 85cc6410c..3f0e8f367 100644
--- a/async.c
+++ b/async.c
@@ -26,13 +26,11 @@
 #include "qemu-aio.h"
 #include "main-loop.h"
 
-/* Anchor of the list of Bottom Halves belonging to the context */
-static struct QEMUBH *first_bh;
-
 /***********************************************************/
 /* bottom halves (can be seen as timers which expire ASAP) */
 
 struct QEMUBH {
+    AioContext *ctx;
     QEMUBHFunc *cb;
     void *opaque;
     QEMUBH *next;
@@ -41,27 +39,27 @@ struct QEMUBH {
     bool deleted;
 };
 
-QEMUBH *qemu_bh_new(QEMUBHFunc *cb, void *opaque)
+QEMUBH *aio_bh_new(AioContext *ctx, QEMUBHFunc *cb, void *opaque)
 {
     QEMUBH *bh;
     bh = g_malloc0(sizeof(QEMUBH));
+    bh->ctx = ctx;
     bh->cb = cb;
     bh->opaque = opaque;
-    bh->next = first_bh;
-    first_bh = bh;
+    bh->next = ctx->first_bh;
+    ctx->first_bh = bh;
     return bh;
 }
 
-int qemu_bh_poll(void)
+int aio_bh_poll(AioContext *ctx)
 {
     QEMUBH *bh, **bhp, *next;
     int ret;
-    static int nesting = 0;
 
-    nesting++;
+    ctx->walking_bh++;
 
     ret = 0;
-    for (bh = first_bh; bh; bh = next) {
+    for (bh = ctx->first_bh; bh; bh = next) {
         next = bh->next;
         if (!bh->deleted && bh->scheduled) {
             bh->scheduled = 0;
@@ -72,11 +70,11 @@ int qemu_bh_poll(void)
         }
     }
 
-    nesting--;
+    ctx->walking_bh--;
 
     /* remove deleted bhs */
-    if (!nesting) {
-        bhp = &first_bh;
+    if (!ctx->walking_bh) {
+        bhp = &ctx->first_bh;
         while (*bhp) {
             bh = *bhp;
             if (bh->deleted) {
@@ -105,8 +103,7 @@ void qemu_bh_schedule(QEMUBH *bh)
         return;
     bh->scheduled = 1;
     bh->idle = 0;
-    /* stop the currently executing CPU to execute the BH ASAP */
-    qemu_notify_event();
+    aio_notify(bh->ctx);
 }
 
 void qemu_bh_cancel(QEMUBH *bh)
@@ -120,23 +117,106 @@ void qemu_bh_delete(QEMUBH *bh)
     bh->deleted = 1;
 }
 
-void qemu_bh_update_timeout(uint32_t *timeout)
+static gboolean
+aio_ctx_prepare(GSource *source, gint    *timeout)
 {
+    AioContext *ctx = (AioContext *) source;
     QEMUBH *bh;
 
-    for (bh = first_bh; bh; bh = bh->next) {
+    for (bh = ctx->first_bh; bh; bh = bh->next) {
         if (!bh->deleted && bh->scheduled) {
             if (bh->idle) {
                 /* idle bottom halves will be polled at least
                  * every 10ms */
-                *timeout = MIN(10, *timeout);
+                *timeout = 10;
             } else {
                 /* non-idle bottom halves will be executed
                  * immediately */
                 *timeout = 0;
-                break;
+                return true;
             }
         }
     }
+
+    return false;
+}
+
+static gboolean
+aio_ctx_check(GSource *source)
+{
+    AioContext *ctx = (AioContext *) source;
+    QEMUBH *bh;
+
+    for (bh = ctx->first_bh; bh; bh = bh->next) {
+        if (!bh->deleted && bh->scheduled) {
+            return true;
+	}
+    }
+    return aio_pending(ctx);
+}
+
+static gboolean
+aio_ctx_dispatch(GSource     *source,
+                 GSourceFunc  callback,
+                 gpointer     user_data)
+{
+    AioContext *ctx = (AioContext *) source;
+
+    assert(callback == NULL);
+    aio_poll(ctx, false);
+    return true;
+}
+
+static void
+aio_ctx_finalize(GSource     *source)
+{
+    AioContext *ctx = (AioContext *) source;
+
+    aio_set_event_notifier(ctx, &ctx->notifier, NULL, NULL);
+    event_notifier_cleanup(&ctx->notifier);
+}
+
+static GSourceFuncs aio_source_funcs = {
+    aio_ctx_prepare,
+    aio_ctx_check,
+    aio_ctx_dispatch,
+    aio_ctx_finalize
+};
+
+GSource *aio_get_g_source(AioContext *ctx)
+{
+    g_source_ref(&ctx->source);
+    return &ctx->source;
+}
+
+void aio_notify(AioContext *ctx)
+{
+    event_notifier_set(&ctx->notifier);
+}
+
+AioContext *aio_context_new(void)
+{
+    AioContext *ctx;
+    ctx = (AioContext *) g_source_new(&aio_source_funcs, sizeof(AioContext));
+    event_notifier_init(&ctx->notifier, false);
+    aio_set_event_notifier(ctx, &ctx->notifier, 
+                           (EventNotifierHandler *)
+                           event_notifier_test_and_clear, NULL);
+
+    return ctx;
 }
 
+void aio_context_ref(AioContext *ctx)
+{
+    g_source_ref(&ctx->source);
+}
+
+void aio_context_unref(AioContext *ctx)
+{
+    g_source_unref(&ctx->source);
+}
+
+void aio_flush(AioContext *ctx)
+{
+    while (aio_poll(ctx, true));
+}
diff --git a/audio/audio_template.h b/audio/audio_template.h
index 519432a7c..16f788077 100644
--- a/audio/audio_template.h
+++ b/audio/audio_template.h
@@ -410,15 +410,15 @@ SW *glue (AUD_open_, TYPE) (
     SW *old_sw = NULL;
 #endif
 
-    ldebug ("open %s, freq %d, nchannels %d, fmt %d\n",
-            name, as->freq, as->nchannels, as->fmt);
-
     if (audio_bug (AUDIO_FUNC, !card || !name || !callback_fn || !as)) {
         dolog ("card=%p name=%p callback_fn=%p as=%p\n",
                card, name, callback_fn, as);
         goto fail;
     }
 
+    ldebug ("open %s, freq %d, nchannels %d, fmt %d\n",
+            name, as->freq, as->nchannels, as->fmt);
+
     if (audio_bug (AUDIO_FUNC, audio_validate_settings (as))) {
         audio_print_settings (as);
         goto fail;
diff --git a/backends/Makefile.objs b/backends/Makefile.objs
new file mode 100644
index 000000000..883676106
--- /dev/null
+++ b/backends/Makefile.objs
@@ -0,0 +1,2 @@
+common-obj-y += rng.o rng-egd.o
+common-obj-$(CONFIG_POSIX) += rng-random.o
diff --git a/backends/rng-egd.c b/backends/rng-egd.c
new file mode 100644
index 000000000..ad8473777
--- /dev/null
+++ b/backends/rng-egd.c
@@ -0,0 +1,224 @@
+/*
+ * QEMU Random Number Generator Backend
+ *
+ * Copyright IBM, Corp. 2012
+ *
+ * Authors:
+ *  Anthony Liguori   <aliguori@us.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#include "qemu/rng.h"
+#include "qemu-char.h"
+#include "qerror.h"
+#include "hw/qdev.h" /* just for DEFINE_PROP_CHR */
+
+#define TYPE_RNG_EGD "rng-egd"
+#define RNG_EGD(obj) OBJECT_CHECK(RngEgd, (obj), TYPE_RNG_EGD)
+
+typedef struct RngEgd
+{
+    RngBackend parent;
+
+    CharDriverState *chr;
+    char *chr_name;
+
+    GSList *requests;
+} RngEgd;
+
+typedef struct RngRequest
+{
+    EntropyReceiveFunc *receive_entropy;
+    uint8_t *data;
+    void *opaque;
+    size_t offset;
+    size_t size;
+} RngRequest;
+
+static void rng_egd_request_entropy(RngBackend *b, size_t size,
+                                    EntropyReceiveFunc *receive_entropy,
+                                    void *opaque)
+{
+    RngEgd *s = RNG_EGD(b);
+    RngRequest *req;
+
+    req = g_malloc(sizeof(*req));
+
+    req->offset = 0;
+    req->size = size;
+    req->receive_entropy = receive_entropy;
+    req->opaque = opaque;
+    req->data = g_malloc(req->size);
+
+    while (size > 0) {
+        uint8_t header[2];
+        uint8_t len = MIN(size, 255);
+
+        /* synchronous entropy request */
+        header[0] = 0x02;
+        header[1] = len;
+
+        qemu_chr_fe_write(s->chr, header, sizeof(header));
+
+        size -= len;
+    }
+
+    s->requests = g_slist_append(s->requests, req);
+}
+
+static void rng_egd_free_request(RngRequest *req)
+{
+    g_free(req->data);
+    g_free(req);
+}
+
+static int rng_egd_chr_can_read(void *opaque)
+{
+    RngEgd *s = RNG_EGD(opaque);
+    GSList *i;
+    int size = 0;
+
+    for (i = s->requests; i; i = i->next) {
+        RngRequest *req = i->data;
+        size += req->size - req->offset;
+    }
+
+    return size;
+}
+
+static void rng_egd_chr_read(void *opaque, const uint8_t *buf, int size)
+{
+    RngEgd *s = RNG_EGD(opaque);
+
+    while (size > 0 && s->requests) {
+        RngRequest *req = s->requests->data;
+        int len = MIN(size, req->size - req->offset);
+
+        memcpy(req->data + req->offset, buf, len);
+        req->offset += len;
+        size -= len;
+
+        if (req->offset == req->size) {
+            s->requests = g_slist_remove_link(s->requests, s->requests);
+
+            req->receive_entropy(req->opaque, req->data, req->size);
+
+            rng_egd_free_request(req);
+        }
+    }
+}
+
+static void rng_egd_free_requests(RngEgd *s)
+{
+    GSList *i;
+
+    for (i = s->requests; i; i = i->next) {
+        rng_egd_free_request(i->data);
+    }
+
+    g_slist_free(s->requests);
+    s->requests = NULL;
+}
+
+static void rng_egd_cancel_requests(RngBackend *b)
+{
+    RngEgd *s = RNG_EGD(b);
+
+    /* We simply delete the list of pending requests.  If there is data in the 
+     * queue waiting to be read, this is okay, because there will always be
+     * more data than we requested originally
+     */
+    rng_egd_free_requests(s);
+}
+
+static void rng_egd_opened(RngBackend *b, Error **errp)
+{
+    RngEgd *s = RNG_EGD(b);
+
+    if (s->chr_name == NULL) {
+        error_set(errp, QERR_INVALID_PARAMETER_VALUE,
+                  "chardev", "a valid character device");
+        return;
+    }
+
+    s->chr = qemu_chr_find(s->chr_name);
+    if (s->chr == NULL) {
+        error_set(errp, QERR_DEVICE_NOT_FOUND, s->chr_name);
+        return;
+    }
+
+    /* FIXME we should resubmit pending requests when the CDS reconnects. */
+    qemu_chr_add_handlers(s->chr, rng_egd_chr_can_read, rng_egd_chr_read,
+                          NULL, s);
+}
+
+static void rng_egd_set_chardev(Object *obj, const char *value, Error **errp)
+{
+    RngBackend *b = RNG_BACKEND(obj);
+    RngEgd *s = RNG_EGD(b);
+
+    if (b->opened) {
+        error_set(errp, QERR_PERMISSION_DENIED);
+    } else {
+        g_free(s->chr_name);
+        s->chr_name = g_strdup(value);
+    }
+}
+
+static char *rng_egd_get_chardev(Object *obj, Error **errp)
+{
+    RngEgd *s = RNG_EGD(obj);
+
+    if (s->chr && s->chr->label) {
+        return g_strdup(s->chr->label);
+    }
+
+    return NULL;
+}
+
+static void rng_egd_init(Object *obj)
+{
+    object_property_add_str(obj, "chardev",
+                            rng_egd_get_chardev, rng_egd_set_chardev,
+                            NULL);
+}
+
+static void rng_egd_finalize(Object *obj)
+{
+    RngEgd *s = RNG_EGD(obj);
+
+    if (s->chr) {
+        qemu_chr_add_handlers(s->chr, NULL, NULL, NULL, NULL);
+    }
+
+    g_free(s->chr_name);
+
+    rng_egd_free_requests(s);
+}
+
+static void rng_egd_class_init(ObjectClass *klass, void *data)
+{
+    RngBackendClass *rbc = RNG_BACKEND_CLASS(klass);
+
+    rbc->request_entropy = rng_egd_request_entropy;
+    rbc->cancel_requests = rng_egd_cancel_requests;
+    rbc->opened = rng_egd_opened;
+}
+
+static TypeInfo rng_egd_info = {
+    .name = TYPE_RNG_EGD,
+    .parent = TYPE_RNG_BACKEND,
+    .instance_size = sizeof(RngEgd),
+    .class_init = rng_egd_class_init,
+    .instance_init = rng_egd_init,
+    .instance_finalize = rng_egd_finalize,
+};
+
+static void register_types(void)
+{
+    type_register_static(&rng_egd_info);
+}
+
+type_init(register_types);
diff --git a/backends/rng-random.c b/backends/rng-random.c
new file mode 100644
index 000000000..9c9923b2a
--- /dev/null
+++ b/backends/rng-random.c
@@ -0,0 +1,161 @@
+/*
+ * QEMU Random Number Generator Backend
+ *
+ * Copyright IBM, Corp. 2012
+ *
+ * Authors:
+ *  Anthony Liguori   <aliguori@us.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#include "qemu/rng-random.h"
+#include "qemu/rng.h"
+#include "qerror.h"
+#include "main-loop.h"
+
+struct RndRandom
+{
+    RngBackend parent;
+
+    int fd;
+    char *filename;
+
+    EntropyReceiveFunc *receive_func;
+    void *opaque;
+    size_t size;
+};
+
+/**
+ * A simple and incomplete backend to request entropy from /dev/random.
+ *
+ * This backend exposes an additional "filename" property that can be used to
+ * set the filename to use to open the backend.
+ */
+
+static void entropy_available(void *opaque)
+{
+    RndRandom *s = RNG_RANDOM(opaque);
+    uint8_t buffer[s->size];
+    ssize_t len;
+
+    len = read(s->fd, buffer, s->size);
+    g_assert(len != -1);
+
+    s->receive_func(s->opaque, buffer, len);
+    s->receive_func = NULL;
+
+    qemu_set_fd_handler(s->fd, NULL, NULL, NULL);
+}
+
+static void rng_random_request_entropy(RngBackend *b, size_t size,
+                                        EntropyReceiveFunc *receive_entropy,
+                                        void *opaque)
+{
+    RndRandom *s = RNG_RANDOM(b);
+
+    if (s->receive_func) {
+        s->receive_func(s->opaque, NULL, 0);
+    }
+
+    s->receive_func = receive_entropy;
+    s->opaque = opaque;
+    s->size = size;
+
+    qemu_set_fd_handler(s->fd, entropy_available, NULL, s);
+}
+
+static void rng_random_opened(RngBackend *b, Error **errp)
+{
+    RndRandom *s = RNG_RANDOM(b);
+
+    if (s->filename == NULL) {
+        error_set(errp, QERR_INVALID_PARAMETER_VALUE,
+                  "filename", "a valid filename");
+    } else {
+        s->fd = open(s->filename, O_RDONLY | O_NONBLOCK);
+
+        if (s->fd == -1) {
+            error_set(errp, QERR_OPEN_FILE_FAILED, s->filename);
+        }
+    }
+}
+
+static char *rng_random_get_filename(Object *obj, Error **errp)
+{
+    RndRandom *s = RNG_RANDOM(obj);
+
+    if (s->filename) {
+        return g_strdup(s->filename);
+    }
+
+    return NULL;
+}
+
+static void rng_random_set_filename(Object *obj, const char *filename,
+                                 Error **errp)
+{
+    RngBackend *b = RNG_BACKEND(obj);
+    RndRandom *s = RNG_RANDOM(obj);
+
+    if (b->opened) {
+        error_set(errp, QERR_PERMISSION_DENIED);
+        return;
+    }
+
+    if (s->filename) {
+        g_free(s->filename);
+    }
+
+    s->filename = g_strdup(filename);
+}
+
+static void rng_random_init(Object *obj)
+{
+    RndRandom *s = RNG_RANDOM(obj);
+
+    object_property_add_str(obj, "filename",
+                            rng_random_get_filename,
+                            rng_random_set_filename,
+                            NULL);
+
+    s->filename = g_strdup("/dev/random");
+}
+
+static void rng_random_finalize(Object *obj)
+{
+    RndRandom *s = RNG_RANDOM(obj);
+
+    qemu_set_fd_handler(s->fd, NULL, NULL, NULL);
+
+    if (s->fd != -1) {
+        close(s->fd);
+    }
+
+    g_free(s->filename);
+}
+
+static void rng_random_class_init(ObjectClass *klass, void *data)
+{
+    RngBackendClass *rbc = RNG_BACKEND_CLASS(klass);
+
+    rbc->request_entropy = rng_random_request_entropy;
+    rbc->opened = rng_random_opened;
+}
+
+static TypeInfo rng_random_info = {
+    .name = TYPE_RNG_RANDOM,
+    .parent = TYPE_RNG_BACKEND,
+    .instance_size = sizeof(RndRandom),
+    .class_init = rng_random_class_init,
+    .instance_init = rng_random_init,
+    .instance_finalize = rng_random_finalize,
+};
+
+static void register_types(void)
+{
+    type_register_static(&rng_random_info);
+}
+
+type_init(register_types);
diff --git a/backends/rng.c b/backends/rng.c
new file mode 100644
index 000000000..06f261180
--- /dev/null
+++ b/backends/rng.c
@@ -0,0 +1,93 @@
+/*
+ * QEMU Random Number Generator Backend
+ *
+ * Copyright IBM, Corp. 2012
+ *
+ * Authors:
+ *  Anthony Liguori   <aliguori@us.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#include "qemu/rng.h"
+#include "qerror.h"
+
+void rng_backend_request_entropy(RngBackend *s, size_t size,
+                                 EntropyReceiveFunc *receive_entropy,
+                                 void *opaque)
+{
+    RngBackendClass *k = RNG_BACKEND_GET_CLASS(s);
+
+    if (k->request_entropy) {
+        k->request_entropy(s, size, receive_entropy, opaque);
+    }
+}
+
+void rng_backend_cancel_requests(RngBackend *s)
+{
+    RngBackendClass *k = RNG_BACKEND_GET_CLASS(s);
+
+    if (k->cancel_requests) {
+        k->cancel_requests(s);
+    }
+}
+
+static bool rng_backend_prop_get_opened(Object *obj, Error **errp)
+{
+    RngBackend *s = RNG_BACKEND(obj);
+
+    return s->opened;
+}
+
+void rng_backend_open(RngBackend *s, Error **errp)
+{
+    object_property_set_bool(OBJECT(s), true, "opened", errp);
+}
+
+static void rng_backend_prop_set_opened(Object *obj, bool value, Error **errp)
+{
+    RngBackend *s = RNG_BACKEND(obj);
+    RngBackendClass *k = RNG_BACKEND_GET_CLASS(s);
+
+    if (value == s->opened) {
+        return;
+    }
+
+    if (!value && s->opened) {
+        error_set(errp, QERR_PERMISSION_DENIED);
+        return;
+    }
+
+    if (k->opened) {
+        k->opened(s, errp);
+    }
+
+    if (!error_is_set(errp)) {
+        s->opened = value;
+    }
+}
+
+static void rng_backend_init(Object *obj)
+{
+    object_property_add_bool(obj, "opened",
+                             rng_backend_prop_get_opened,
+                             rng_backend_prop_set_opened,
+                             NULL);
+}
+
+static TypeInfo rng_backend_info = {
+    .name = TYPE_RNG_BACKEND,
+    .parent = TYPE_OBJECT,
+    .instance_size = sizeof(RngBackend),
+    .instance_init = rng_backend_init,
+    .class_size = sizeof(RngBackendClass),
+    .abstract = true,
+};
+
+static void register_types(void)
+{
+    type_register_static(&rng_backend_info);
+}
+
+type_init(register_types);
diff --git a/block-migration.c b/block-migration.c
index 7def8ab19..71b9601e0 100644
--- a/block-migration.c
+++ b/block-migration.c
@@ -423,20 +423,23 @@ static int mig_save_device_dirty(QEMUFile *f, BlkMigDevState *bmds,
 
 error:
     DPRINTF("Error reading sector %" PRId64 "\n", sector);
-    qemu_file_set_error(f, ret);
     g_free(blk->buf);
     g_free(blk);
-    return 0;
+    return ret;
 }
 
+/* return value:
+ * 0: too much data for max_downtime
+ * 1: few enough data for max_downtime
+*/
 static int blk_mig_save_dirty_block(QEMUFile *f, int is_async)
 {
     BlkMigDevState *bmds;
-    int ret = 0;
+    int ret = 1;
 
     QSIMPLEQ_FOREACH(bmds, &block_mig_state.bmds_list, entry) {
-        if (mig_save_device_dirty(f, bmds, is_async) == 0) {
-            ret = 1;
+        ret = mig_save_device_dirty(f, bmds, is_async);
+        if (ret <= 0) {
             break;
         }
     }
@@ -444,9 +447,10 @@ static int blk_mig_save_dirty_block(QEMUFile *f, int is_async)
     return ret;
 }
 
-static void flush_blks(QEMUFile* f)
+static int flush_blks(QEMUFile *f)
 {
     BlkMigBlock *blk;
+    int ret = 0;
 
     DPRINTF("%s Enter submitted %d read_done %d transferred %d\n",
             __FUNCTION__, block_mig_state.submitted, block_mig_state.read_done,
@@ -457,7 +461,7 @@ static void flush_blks(QEMUFile* f)
             break;
         }
         if (blk->ret < 0) {
-            qemu_file_set_error(f, blk->ret);
+            ret = blk->ret;
             break;
         }
         blk_send(f, blk);
@@ -474,6 +478,7 @@ static void flush_blks(QEMUFile* f)
     DPRINTF("%s Exit submitted %d read_done %d transferred %d\n", __FUNCTION__,
             block_mig_state.submitted, block_mig_state.read_done,
             block_mig_state.transferred);
+    return ret;
 }
 
 static int64_t get_remaining_dirty(void)
@@ -519,6 +524,8 @@ static void blk_mig_cleanup(void)
     BlkMigDevState *bmds;
     BlkMigBlock *blk;
 
+    bdrv_drain_all();
+
     set_dirty_tracking(0);
 
     while ((bmds = QSIMPLEQ_FIRST(&block_mig_state.bmds_list)) != NULL) {
@@ -553,9 +560,7 @@ static int block_save_setup(QEMUFile *f, void *opaque)
     /* start track dirty blocks */
     set_dirty_tracking(1);
 
-    flush_blks(f);
-
-    ret = qemu_file_get_error(f);
+    ret = flush_blks(f);
     if (ret) {
         blk_mig_cleanup();
         return ret;
@@ -575,9 +580,7 @@ static int block_save_iterate(QEMUFile *f, void *opaque)
     DPRINTF("Enter save live iterate submitted %d transferred %d\n",
             block_mig_state.submitted, block_mig_state.transferred);
 
-    flush_blks(f);
-
-    ret = qemu_file_get_error(f);
+    ret = flush_blks(f);
     if (ret) {
         blk_mig_cleanup();
         return ret;
@@ -596,16 +599,19 @@ static int block_save_iterate(QEMUFile *f, void *opaque)
                 block_mig_state.bulk_completed = 1;
             }
         } else {
-            if (blk_mig_save_dirty_block(f, 1) == 0) {
+            ret = blk_mig_save_dirty_block(f, 1);
+            if (ret != 0) {
                 /* no more dirty blocks */
                 break;
             }
         }
     }
+    if (ret) {
+        blk_mig_cleanup();
+        return ret;
+    }
 
-    flush_blks(f);
-
-    ret = qemu_file_get_error(f);
+    ret = flush_blks(f);
     if (ret) {
         blk_mig_cleanup();
         return ret;
@@ -623,9 +629,7 @@ static int block_save_complete(QEMUFile *f, void *opaque)
     DPRINTF("Enter save live complete submitted %d transferred %d\n",
             block_mig_state.submitted, block_mig_state.transferred);
 
-    flush_blks(f);
-
-    ret = qemu_file_get_error(f);
+    ret = flush_blks(f);
     if (ret) {
         blk_mig_cleanup();
         return ret;
@@ -637,18 +641,16 @@ static int block_save_complete(QEMUFile *f, void *opaque)
        all async read completed */
     assert(block_mig_state.submitted == 0);
 
-    while (blk_mig_save_dirty_block(f, 0) != 0) {
-        /* Do nothing */
-    }
-    blk_mig_cleanup();
-
-    /* report completion */
-    qemu_put_be64(f, (100 << BDRV_SECTOR_BITS) | BLK_MIG_FLAG_PROGRESS);
+    do {
+        ret = blk_mig_save_dirty_block(f, 0);
+    } while (ret == 0);
 
-    ret = qemu_file_get_error(f);
+    blk_mig_cleanup();
     if (ret) {
         return ret;
     }
+    /* report completion */
+    qemu_put_be64(f, (100 << BDRV_SECTOR_BITS) | BLK_MIG_FLAG_PROGRESS);
 
     DPRINTF("Block migration completed\n");
 
diff --git a/block.c b/block.c
index 470bdcc1f..c05875fe3 100644
--- a/block.c
+++ b/block.c
@@ -26,8 +26,11 @@
 #include "trace.h"
 #include "monitor.h"
 #include "block_int.h"
+#include "blockjob.h"
 #include "module.h"
 #include "qjson.h"
+#include "sysemu.h"
+#include "notify.h"
 #include "qemu-coroutine.h"
 #include "qmp-commands.h"
 #include "qemu-timer.h"
@@ -310,9 +313,16 @@ BlockDriverState *bdrv_new(const char *device_name)
         QTAILQ_INSERT_TAIL(&bdrv_states, bs, list);
     }
     bdrv_iostatus_disable(bs);
+    notifier_list_init(&bs->close_notifiers);
+
     return bs;
 }
 
+void bdrv_add_close_notifier(BlockDriverState *bs, Notifier *notify)
+{
+    notifier_list_add(&bs->close_notifiers, notify);
+}
+
 BlockDriver *bdrv_find_format(const char *format_name)
 {
     BlockDriver *drv1;
@@ -377,7 +387,8 @@ int bdrv_create(BlockDriver *drv, const char* filename,
     };
 
     if (!drv->bdrv_create) {
-        return -ENOTSUP;
+        ret = -ENOTSUP;
+        goto out;
     }
 
     if (qemu_in_coroutine()) {
@@ -392,8 +403,9 @@ int bdrv_create(BlockDriver *drv, const char* filename,
     }
 
     ret = cco.ret;
-    g_free(cco.filename);
 
+out:
+    g_free(cco.filename);
     return ret;
 }
 
@@ -433,7 +445,11 @@ int get_tmp_filename(char *filename, int size)
         return -EOVERFLOW;
     }
     fd = mkstemp(filename);
-    if (fd < 0 || close(fd)) {
+    if (fd < 0) {
+        return -errno;
+    }
+    if (close(fd) != 0) {
+        unlink(filename);
         return -errno;
     }
     return 0;
@@ -664,7 +680,7 @@ static int bdrv_open_common(BlockDriverState *bs, const char *filename,
         open_flags |= BDRV_O_RDWR;
     }
 
-    bs->keep_read_only = bs->read_only = !(open_flags & BDRV_O_RDWR);
+    bs->read_only = !(open_flags & BDRV_O_RDWR);
 
     /* Open the image, either directly or using a protocol */
     if (drv->bdrv_file_open) {
@@ -728,6 +744,42 @@ int bdrv_file_open(BlockDriverState **pbs, const char *filename, int flags)
     return 0;
 }
 
+int bdrv_open_backing_file(BlockDriverState *bs)
+{
+    char backing_filename[PATH_MAX];
+    int back_flags, ret;
+    BlockDriver *back_drv = NULL;
+
+    if (bs->backing_hd != NULL) {
+        return 0;
+    }
+
+    bs->open_flags &= ~BDRV_O_NO_BACKING;
+    if (bs->backing_file[0] == '\0') {
+        return 0;
+    }
+
+    bs->backing_hd = bdrv_new("");
+    bdrv_get_full_backing_filename(bs, backing_filename,
+                                   sizeof(backing_filename));
+
+    if (bs->backing_format[0] != '\0') {
+        back_drv = bdrv_find_format(bs->backing_format);
+    }
+
+    /* backing files always opened read-only */
+    back_flags = bs->open_flags & ~(BDRV_O_RDWR | BDRV_O_SNAPSHOT);
+
+    ret = bdrv_open(bs->backing_hd, backing_filename, back_flags, back_drv);
+    if (ret < 0) {
+        bdrv_delete(bs->backing_hd);
+        bs->backing_hd = NULL;
+        bs->open_flags |= BDRV_O_NO_BACKING;
+        return ret;
+    }
+    return 0;
+}
+
 /*
  * Opens a disk image (raw, qcow2, vmdk, ...)
  */
@@ -735,7 +787,8 @@ int bdrv_open(BlockDriverState *bs, const char *filename, int flags,
               BlockDriver *drv)
 {
     int ret;
-    char tmp_filename[PATH_MAX];
+    /* TODO: extra byte is a hack to ensure MAX_PATH space on Windows. */
+    char tmp_filename[PATH_MAX + 1];
 
     if (flags & BDRV_O_SNAPSHOT) {
         BlockDriverState *bs1;
@@ -804,6 +857,10 @@ int bdrv_open(BlockDriverState *bs, const char *filename, int flags,
         goto unlink_and_fail;
     }
 
+    if (flags & BDRV_O_RDWR) {
+        flags |= BDRV_O_ALLOW_RDWR;
+    }
+
     /* Open the image */
     ret = bdrv_open_common(bs, filename, flags, drv);
     if (ret < 0) {
@@ -811,34 +868,12 @@ int bdrv_open(BlockDriverState *bs, const char *filename, int flags,
     }
 
     /* If there is a backing file, use it */
-    if ((flags & BDRV_O_NO_BACKING) == 0 && bs->backing_file[0] != '\0') {
-        char backing_filename[PATH_MAX];
-        int back_flags;
-        BlockDriver *back_drv = NULL;
-
-        bs->backing_hd = bdrv_new("");
-        bdrv_get_full_backing_filename(bs, backing_filename,
-                                       sizeof(backing_filename));
-
-        if (bs->backing_format[0] != '\0') {
-            back_drv = bdrv_find_format(bs->backing_format);
-        }
-
-        /* backing files always opened read-only */
-        back_flags =
-            flags & ~(BDRV_O_RDWR | BDRV_O_SNAPSHOT | BDRV_O_NO_BACKING);
-
-        ret = bdrv_open(bs->backing_hd, backing_filename, back_flags, back_drv);
+    if ((flags & BDRV_O_NO_BACKING) == 0) {
+        ret = bdrv_open_backing_file(bs);
         if (ret < 0) {
             bdrv_close(bs);
             return ret;
         }
-        if (bs->is_temporary) {
-            bs->backing_hd->keep_read_only = !(flags & BDRV_O_RDWR);
-        } else {
-            /* base image inherits from "parent" */
-            bs->backing_hd->keep_read_only = bs->keep_read_only;
-        }
     }
 
     if (!bdrv_key_required(bs)) {
@@ -859,15 +894,248 @@ unlink_and_fail:
     return ret;
 }
 
+typedef struct BlockReopenQueueEntry {
+     bool prepared;
+     BDRVReopenState state;
+     QSIMPLEQ_ENTRY(BlockReopenQueueEntry) entry;
+} BlockReopenQueueEntry;
+
+/*
+ * Adds a BlockDriverState to a simple queue for an atomic, transactional
+ * reopen of multiple devices.
+ *
+ * bs_queue can either be an existing BlockReopenQueue that has had QSIMPLE_INIT
+ * already performed, or alternatively may be NULL a new BlockReopenQueue will
+ * be created and initialized. This newly created BlockReopenQueue should be
+ * passed back in for subsequent calls that are intended to be of the same
+ * atomic 'set'.
+ *
+ * bs is the BlockDriverState to add to the reopen queue.
+ *
+ * flags contains the open flags for the associated bs
+ *
+ * returns a pointer to bs_queue, which is either the newly allocated
+ * bs_queue, or the existing bs_queue being used.
+ *
+ */
+BlockReopenQueue *bdrv_reopen_queue(BlockReopenQueue *bs_queue,
+                                    BlockDriverState *bs, int flags)
+{
+    assert(bs != NULL);
+
+    BlockReopenQueueEntry *bs_entry;
+    if (bs_queue == NULL) {
+        bs_queue = g_new0(BlockReopenQueue, 1);
+        QSIMPLEQ_INIT(bs_queue);
+    }
+
+    if (bs->file) {
+        bdrv_reopen_queue(bs_queue, bs->file, flags);
+    }
+
+    bs_entry = g_new0(BlockReopenQueueEntry, 1);
+    QSIMPLEQ_INSERT_TAIL(bs_queue, bs_entry, entry);
+
+    bs_entry->state.bs = bs;
+    bs_entry->state.flags = flags;
+
+    return bs_queue;
+}
+
+/*
+ * Reopen multiple BlockDriverStates atomically & transactionally.
+ *
+ * The queue passed in (bs_queue) must have been built up previous
+ * via bdrv_reopen_queue().
+ *
+ * Reopens all BDS specified in the queue, with the appropriate
+ * flags.  All devices are prepared for reopen, and failure of any
+ * device will cause all device changes to be abandonded, and intermediate
+ * data cleaned up.
+ *
+ * If all devices prepare successfully, then the changes are committed
+ * to all devices.
+ *
+ */
+int bdrv_reopen_multiple(BlockReopenQueue *bs_queue, Error **errp)
+{
+    int ret = -1;
+    BlockReopenQueueEntry *bs_entry, *next;
+    Error *local_err = NULL;
+
+    assert(bs_queue != NULL);
+
+    bdrv_drain_all();
+
+    QSIMPLEQ_FOREACH(bs_entry, bs_queue, entry) {
+        if (bdrv_reopen_prepare(&bs_entry->state, bs_queue, &local_err)) {
+            error_propagate(errp, local_err);
+            goto cleanup;
+        }
+        bs_entry->prepared = true;
+    }
+
+    /* If we reach this point, we have success and just need to apply the
+     * changes
+     */
+    QSIMPLEQ_FOREACH(bs_entry, bs_queue, entry) {
+        bdrv_reopen_commit(&bs_entry->state);
+    }
+
+    ret = 0;
+
+cleanup:
+    QSIMPLEQ_FOREACH_SAFE(bs_entry, bs_queue, entry, next) {
+        if (ret && bs_entry->prepared) {
+            bdrv_reopen_abort(&bs_entry->state);
+        }
+        g_free(bs_entry);
+    }
+    g_free(bs_queue);
+    return ret;
+}
+
+
+/* Reopen a single BlockDriverState with the specified flags. */
+int bdrv_reopen(BlockDriverState *bs, int bdrv_flags, Error **errp)
+{
+    int ret = -1;
+    Error *local_err = NULL;
+    BlockReopenQueue *queue = bdrv_reopen_queue(NULL, bs, bdrv_flags);
+
+    ret = bdrv_reopen_multiple(queue, &local_err);
+    if (local_err != NULL) {
+        error_propagate(errp, local_err);
+    }
+    return ret;
+}
+
+
+/*
+ * Prepares a BlockDriverState for reopen. All changes are staged in the
+ * 'opaque' field of the BDRVReopenState, which is used and allocated by
+ * the block driver layer .bdrv_reopen_prepare()
+ *
+ * bs is the BlockDriverState to reopen
+ * flags are the new open flags
+ * queue is the reopen queue
+ *
+ * Returns 0 on success, non-zero on error.  On error errp will be set
+ * as well.
+ *
+ * On failure, bdrv_reopen_abort() will be called to clean up any data.
+ * It is the responsibility of the caller to then call the abort() or
+ * commit() for any other BDS that have been left in a prepare() state
+ *
+ */
+int bdrv_reopen_prepare(BDRVReopenState *reopen_state, BlockReopenQueue *queue,
+                        Error **errp)
+{
+    int ret = -1;
+    Error *local_err = NULL;
+    BlockDriver *drv;
+
+    assert(reopen_state != NULL);
+    assert(reopen_state->bs->drv != NULL);
+    drv = reopen_state->bs->drv;
+
+    /* if we are to stay read-only, do not allow permission change
+     * to r/w */
+    if (!(reopen_state->bs->open_flags & BDRV_O_ALLOW_RDWR) &&
+        reopen_state->flags & BDRV_O_RDWR) {
+        error_set(errp, QERR_DEVICE_IS_READ_ONLY,
+                  reopen_state->bs->device_name);
+        goto error;
+    }
+
+
+    ret = bdrv_flush(reopen_state->bs);
+    if (ret) {
+        error_set(errp, ERROR_CLASS_GENERIC_ERROR, "Error (%s) flushing drive",
+                  strerror(-ret));
+        goto error;
+    }
+
+    if (drv->bdrv_reopen_prepare) {
+        ret = drv->bdrv_reopen_prepare(reopen_state, queue, &local_err);
+        if (ret) {
+            if (local_err != NULL) {
+                error_propagate(errp, local_err);
+            } else {
+                error_set(errp, QERR_OPEN_FILE_FAILED,
+                          reopen_state->bs->filename);
+            }
+            goto error;
+        }
+    } else {
+        /* It is currently mandatory to have a bdrv_reopen_prepare()
+         * handler for each supported drv. */
+        error_set(errp, QERR_BLOCK_FORMAT_FEATURE_NOT_SUPPORTED,
+                  drv->format_name, reopen_state->bs->device_name,
+                 "reopening of file");
+        ret = -1;
+        goto error;
+    }
+
+    ret = 0;
+
+error:
+    return ret;
+}
+
+/*
+ * Takes the staged changes for the reopen from bdrv_reopen_prepare(), and
+ * makes them final by swapping the staging BlockDriverState contents into
+ * the active BlockDriverState contents.
+ */
+void bdrv_reopen_commit(BDRVReopenState *reopen_state)
+{
+    BlockDriver *drv;
+
+    assert(reopen_state != NULL);
+    drv = reopen_state->bs->drv;
+    assert(drv != NULL);
+
+    /* If there are any driver level actions to take */
+    if (drv->bdrv_reopen_commit) {
+        drv->bdrv_reopen_commit(reopen_state);
+    }
+
+    /* set BDS specific flags now */
+    reopen_state->bs->open_flags         = reopen_state->flags;
+    reopen_state->bs->enable_write_cache = !!(reopen_state->flags &
+                                              BDRV_O_CACHE_WB);
+    reopen_state->bs->read_only = !(reopen_state->flags & BDRV_O_RDWR);
+}
+
+/*
+ * Abort the reopen, and delete and free the staged changes in
+ * reopen_state
+ */
+void bdrv_reopen_abort(BDRVReopenState *reopen_state)
+{
+    BlockDriver *drv;
+
+    assert(reopen_state != NULL);
+    drv = reopen_state->bs->drv;
+    assert(drv != NULL);
+
+    if (drv->bdrv_reopen_abort) {
+        drv->bdrv_reopen_abort(reopen_state);
+    }
+}
+
+
 void bdrv_close(BlockDriverState *bs)
 {
     bdrv_flush(bs);
-    if (bs->drv) {
-        if (bs->job) {
-            block_job_cancel_sync(bs->job);
-        }
-        bdrv_drain_all();
+    if (bs->job) {
+        block_job_cancel_sync(bs->job);
+    }
+    bdrv_drain_all();
+    notifier_list_notify(&bs->close_notifiers, bs);
 
+    if (bs->drv) {
         if (bs == bs_snapshots) {
             bs_snapshots = NULL;
         }
@@ -897,10 +1165,10 @@ void bdrv_close(BlockDriverState *bs)
             bdrv_delete(bs->file);
             bs->file = NULL;
         }
-
-        bdrv_dev_change_media_cb(bs, false);
     }
 
+    bdrv_dev_change_media_cb(bs, false);
+
     /*throttling disk I/O limits*/
     if (bs->io_limits_enabled) {
         bdrv_io_limits_disable(bs);
@@ -1152,7 +1420,8 @@ void bdrv_set_dev_ops(BlockDriverState *bs, const BlockDevOps *ops,
 }
 
 void bdrv_emit_qmp_error_event(const BlockDriverState *bdrv,
-                               BlockQMPEventAction action, int is_read)
+                               enum MonitorEvent ev,
+                               BlockErrorAction action, bool is_read)
 {
     QObject *data;
     const char *action_str;
@@ -1175,7 +1444,7 @@ void bdrv_emit_qmp_error_event(const BlockDriverState *bdrv,
                               bdrv->device_name,
                               action_str,
                               is_read ? "read" : "write");
-    monitor_protocol_event(QEVENT_BLOCK_IO_ERROR, data);
+    monitor_protocol_event(ev, data);
 
     qobject_decref(data);
 }
@@ -1265,13 +1534,11 @@ int bdrv_check(BlockDriverState *bs, BdrvCheckResult *res, BdrvCheckMode fix)
 int bdrv_commit(BlockDriverState *bs)
 {
     BlockDriver *drv = bs->drv;
-    BlockDriver *backing_drv;
     int64_t sector, total_sectors;
     int n, ro, open_flags;
-    int ret = 0, rw_ret = 0;
+    int ret = 0;
     uint8_t *buf;
-    char filename[1024];
-    BlockDriverState *bs_rw, *bs_ro;
+    char filename[PATH_MAX];
 
     if (!drv)
         return -ENOMEDIUM;
@@ -1280,42 +1547,19 @@ int bdrv_commit(BlockDriverState *bs)
         return -ENOTSUP;
     }
 
-    if (bs->backing_hd->keep_read_only) {
-        return -EACCES;
-    }
-
     if (bdrv_in_use(bs) || bdrv_in_use(bs->backing_hd)) {
         return -EBUSY;
     }
 
-    backing_drv = bs->backing_hd->drv;
     ro = bs->backing_hd->read_only;
-    strncpy(filename, bs->backing_hd->filename, sizeof(filename));
+    /* Use pstrcpy (not strncpy): filename must be NUL-terminated. */
+    pstrcpy(filename, sizeof(filename), bs->backing_hd->filename);
     open_flags =  bs->backing_hd->open_flags;
 
     if (ro) {
-        /* re-open as RW */
-        bdrv_delete(bs->backing_hd);
-        bs->backing_hd = NULL;
-        bs_rw = bdrv_new("");
-        rw_ret = bdrv_open(bs_rw, filename, open_flags | BDRV_O_RDWR,
-            backing_drv);
-        if (rw_ret < 0) {
-            bdrv_delete(bs_rw);
-            /* try to re-open read-only */
-            bs_ro = bdrv_new("");
-            ret = bdrv_open(bs_ro, filename, open_flags & ~BDRV_O_RDWR,
-                backing_drv);
-            if (ret < 0) {
-                bdrv_delete(bs_ro);
-                /* drive not functional anymore */
-                bs->drv = NULL;
-                return ret;
-            }
-            bs->backing_hd = bs_ro;
-            return rw_ret;
+        if (bdrv_reopen(bs->backing_hd, open_flags | BDRV_O_RDWR, NULL)) {
+            return -EACCES;
         }
-        bs->backing_hd = bs_rw;
     }
 
     total_sectors = bdrv_getlength(bs) >> BDRV_SECTOR_BITS;
@@ -1352,20 +1596,8 @@ ro_cleanup:
     g_free(buf);
 
     if (ro) {
-        /* re-open as RO */
-        bdrv_delete(bs->backing_hd);
-        bs->backing_hd = NULL;
-        bs_ro = bdrv_new("");
-        ret = bdrv_open(bs_ro, filename, open_flags & ~BDRV_O_RDWR,
-            backing_drv);
-        if (ret < 0) {
-            bdrv_delete(bs_ro);
-            /* drive not functional anymore */
-            bs->drv = NULL;
-            return ret;
-        }
-        bs->backing_hd = bs_ro;
-        bs->backing_hd->keep_read_only = 0;
+        /* ignoring error return here */
+        bdrv_reopen(bs->backing_hd, open_flags & ~BDRV_O_RDWR, NULL);
     }
 
     return ret;
@@ -1528,6 +1760,149 @@ int bdrv_change_backing_file(BlockDriverState *bs,
     return ret;
 }
 
+/*
+ * Finds the image layer in the chain that has 'bs' as its backing file.
+ *
+ * active is the current topmost image.
+ *
+ * Returns NULL if bs is not found in active's image chain,
+ * or if active == bs.
+ */
+BlockDriverState *bdrv_find_overlay(BlockDriverState *active,
+                                    BlockDriverState *bs)
+{
+    BlockDriverState *overlay = NULL;
+    BlockDriverState *intermediate;
+
+    assert(active != NULL);
+    assert(bs != NULL);
+
+    /* if bs is the same as active, then by definition it has no overlay
+     */
+    if (active == bs) {
+        return NULL;
+    }
+
+    intermediate = active;
+    while (intermediate->backing_hd) {
+        if (intermediate->backing_hd == bs) {
+            overlay = intermediate;
+            break;
+        }
+        intermediate = intermediate->backing_hd;
+    }
+
+    return overlay;
+}
+
+typedef struct BlkIntermediateStates {
+    BlockDriverState *bs;
+    QSIMPLEQ_ENTRY(BlkIntermediateStates) entry;
+} BlkIntermediateStates;
+
+
+/*
+ * Drops images above 'base' up to and including 'top', and sets the image
+ * above 'top' to have base as its backing file.
+ *
+ * Requires that the overlay to 'top' is opened r/w, so that the backing file
+ * information in 'bs' can be properly updated.
+ *
+ * E.g., this will convert the following chain:
+ * bottom <- base <- intermediate <- top <- active
+ *
+ * to
+ *
+ * bottom <- base <- active
+ *
+ * It is allowed for bottom==base, in which case it converts:
+ *
+ * base <- intermediate <- top <- active
+ *
+ * to
+ *
+ * base <- active
+ *
+ * Error conditions:
+ *  if active == top, that is considered an error
+ *
+ */
+int bdrv_drop_intermediate(BlockDriverState *active, BlockDriverState *top,
+                           BlockDriverState *base)
+{
+    BlockDriverState *intermediate;
+    BlockDriverState *base_bs = NULL;
+    BlockDriverState *new_top_bs = NULL;
+    BlkIntermediateStates *intermediate_state, *next;
+    int ret = -EIO;
+
+    QSIMPLEQ_HEAD(states_to_delete, BlkIntermediateStates) states_to_delete;
+    QSIMPLEQ_INIT(&states_to_delete);
+
+    if (!top->drv || !base->drv) {
+        goto exit;
+    }
+
+    new_top_bs = bdrv_find_overlay(active, top);
+
+    if (new_top_bs == NULL) {
+        /* we could not find the image above 'top', this is an error */
+        goto exit;
+    }
+
+    /* special case of new_top_bs->backing_hd already pointing to base - nothing
+     * to do, no intermediate images */
+    if (new_top_bs->backing_hd == base) {
+        ret = 0;
+        goto exit;
+    }
+
+    intermediate = top;
+
+    /* now we will go down through the list, and add each BDS we find
+     * into our deletion queue, until we hit the 'base'
+     */
+    while (intermediate) {
+        intermediate_state = g_malloc0(sizeof(BlkIntermediateStates));
+        intermediate_state->bs = intermediate;
+        QSIMPLEQ_INSERT_TAIL(&states_to_delete, intermediate_state, entry);
+
+        if (intermediate->backing_hd == base) {
+            base_bs = intermediate->backing_hd;
+            break;
+        }
+        intermediate = intermediate->backing_hd;
+    }
+    if (base_bs == NULL) {
+        /* something went wrong, we did not end at the base. safely
+         * unravel everything, and exit with error */
+        goto exit;
+    }
+
+    /* success - we can delete the intermediate states, and link top->base */
+    ret = bdrv_change_backing_file(new_top_bs, base_bs->filename,
+                                   base_bs->drv ? base_bs->drv->format_name : "");
+    if (ret) {
+        goto exit;
+    }
+    new_top_bs->backing_hd = base_bs;
+
+
+    QSIMPLEQ_FOREACH_SAFE(intermediate_state, &states_to_delete, entry, next) {
+        /* so that bdrv_close() does not recursively close the chain */
+        intermediate_state->bs->backing_hd = NULL;
+        bdrv_delete(intermediate_state->bs);
+    }
+    ret = 0;
+
+exit:
+    QSIMPLEQ_FOREACH_SAFE(intermediate_state, &states_to_delete, entry, next) {
+        g_free(intermediate_state);
+    }
+    return ret;
+}
+
+
 static int bdrv_check_byte_request(BlockDriverState *bs, int64_t offset,
                                    size_t size)
 {
@@ -2026,7 +2401,7 @@ static int coroutine_fn bdrv_co_do_writev(BlockDriverState *bs,
     }
 
     if (bs->dirty_bitmap) {
-        set_dirty_bitmap(bs, sector_num, nb_sectors, 1);
+        bdrv_set_dirty(bs, sector_num, nb_sectors);
     }
 
     if (bs->wr_highest_sector < sector_num + nb_sectors - 1) {
@@ -2134,18 +2509,51 @@ void bdrv_set_io_limits(BlockDriverState *bs,
     bs->io_limits_enabled = bdrv_io_limits_enabled(bs);
 }
 
-void bdrv_set_on_error(BlockDriverState *bs, BlockErrorAction on_read_error,
-                       BlockErrorAction on_write_error)
+void bdrv_set_on_error(BlockDriverState *bs, BlockdevOnError on_read_error,
+                       BlockdevOnError on_write_error)
 {
     bs->on_read_error = on_read_error;
     bs->on_write_error = on_write_error;
 }
 
-BlockErrorAction bdrv_get_on_error(BlockDriverState *bs, int is_read)
+BlockdevOnError bdrv_get_on_error(BlockDriverState *bs, bool is_read)
 {
     return is_read ? bs->on_read_error : bs->on_write_error;
 }
 
+BlockErrorAction bdrv_get_error_action(BlockDriverState *bs, bool is_read, int error)
+{
+    BlockdevOnError on_err = is_read ? bs->on_read_error : bs->on_write_error;
+
+    switch (on_err) {
+    case BLOCKDEV_ON_ERROR_ENOSPC:
+        return (error == ENOSPC) ? BDRV_ACTION_STOP : BDRV_ACTION_REPORT;
+    case BLOCKDEV_ON_ERROR_STOP:
+        return BDRV_ACTION_STOP;
+    case BLOCKDEV_ON_ERROR_REPORT:
+        return BDRV_ACTION_REPORT;
+    case BLOCKDEV_ON_ERROR_IGNORE:
+        return BDRV_ACTION_IGNORE;
+    default:
+        abort();
+    }
+}
+
+/* This is done by device models because, while the block layer knows
+ * about the error, it does not know whether an operation comes from
+ * the device or the block layer (from a job, for example).
+ */
+void bdrv_error_action(BlockDriverState *bs, BlockErrorAction action,
+                       bool is_read, int error)
+{
+    assert(error >= 0);
+    bdrv_emit_qmp_error_event(bs, QEVENT_BLOCK_IO_ERROR, action, is_read);
+    if (action == BDRV_ACTION_STOP) {
+        vm_stop(RUN_STATE_IO_ERROR);
+        bdrv_iostatus_set_err(bs, error);
+    }
+}
+
 int bdrv_is_read_only(BlockDriverState *bs)
 {
     return bs->read_only;
@@ -2164,6 +2572,13 @@ int bdrv_enable_write_cache(BlockDriverState *bs)
 void bdrv_set_enable_write_cache(BlockDriverState *bs, bool wce)
 {
     bs->enable_write_cache = wce;
+
+    /* so a reopen() will preserve wce */
+    if (wce) {
+        bs->open_flags |= BDRV_O_CACHE_WB;
+    } else {
+        bs->open_flags &= ~BDRV_O_CACHE_WB;
+    }
 }
 
 int bdrv_is_encrypted(BlockDriverState *bs)
@@ -2414,76 +2829,82 @@ int coroutine_fn bdrv_co_is_allocated_above(BlockDriverState *top,
     return 0;
 }
 
-BlockInfoList *qmp_query_block(Error **errp)
+BlockInfo *bdrv_query_info(BlockDriverState *bs)
 {
-    BlockInfoList *head = NULL, *cur_item = NULL;
-    BlockDriverState *bs;
+    BlockInfo *info = g_malloc0(sizeof(*info));
+    info->device = g_strdup(bs->device_name);
+    info->type = g_strdup("unknown");
+    info->locked = bdrv_dev_is_medium_locked(bs);
+    info->removable = bdrv_dev_has_removable_media(bs);
 
-    QTAILQ_FOREACH(bs, &bdrv_states, list) {
-        BlockInfoList *info = g_malloc0(sizeof(*info));
+    if (bdrv_dev_has_removable_media(bs)) {
+        info->has_tray_open = true;
+        info->tray_open = bdrv_dev_is_tray_open(bs);
+    }
 
-        info->value = g_malloc0(sizeof(*info->value));
-        info->value->device = g_strdup(bs->device_name);
-        info->value->type = g_strdup("unknown");
-        info->value->locked = bdrv_dev_is_medium_locked(bs);
-        info->value->removable = bdrv_dev_has_removable_media(bs);
+    if (bdrv_iostatus_is_enabled(bs)) {
+        info->has_io_status = true;
+        info->io_status = bs->iostatus;
+    }
 
-        if (bdrv_dev_has_removable_media(bs)) {
-            info->value->has_tray_open = true;
-            info->value->tray_open = bdrv_dev_is_tray_open(bs);
+    if (bs->dirty_bitmap) {
+        info->has_dirty = true;
+        info->dirty = g_malloc0(sizeof(*info->dirty));
+        info->dirty->count = bdrv_get_dirty_count(bs) *
+            BDRV_SECTORS_PER_DIRTY_CHUNK * BDRV_SECTOR_SIZE;
+    }
+
+    if (bs->drv) {
+        info->has_inserted = true;
+        info->inserted = g_malloc0(sizeof(*info->inserted));
+        info->inserted->file = g_strdup(bs->filename);
+        info->inserted->ro = bs->read_only;
+        info->inserted->drv = g_strdup(bs->drv->format_name);
+        info->inserted->encrypted = bs->encrypted;
+        info->inserted->encryption_key_missing = bdrv_key_required(bs);
+
+        if (bs->backing_file[0]) {
+            info->inserted->has_backing_file = true;
+            info->inserted->backing_file = g_strdup(bs->backing_file);
         }
 
-        if (bdrv_iostatus_is_enabled(bs)) {
-            info->value->has_io_status = true;
-            info->value->io_status = bs->iostatus;
+        info->inserted->backing_file_depth = bdrv_get_backing_file_depth(bs);
+
+        if (bs->io_limits_enabled) {
+            info->inserted->bps =
+                           bs->io_limits.bps[BLOCK_IO_LIMIT_TOTAL];
+            info->inserted->bps_rd =
+                           bs->io_limits.bps[BLOCK_IO_LIMIT_READ];
+            info->inserted->bps_wr =
+                           bs->io_limits.bps[BLOCK_IO_LIMIT_WRITE];
+            info->inserted->iops =
+                           bs->io_limits.iops[BLOCK_IO_LIMIT_TOTAL];
+            info->inserted->iops_rd =
+                           bs->io_limits.iops[BLOCK_IO_LIMIT_READ];
+            info->inserted->iops_wr =
+                           bs->io_limits.iops[BLOCK_IO_LIMIT_WRITE];
         }
+    }
+    return info;
+}
 
-        if (bs->drv) {
-            info->value->has_inserted = true;
-            info->value->inserted = g_malloc0(sizeof(*info->value->inserted));
-            info->value->inserted->file = g_strdup(bs->filename);
-            info->value->inserted->ro = bs->read_only;
-            info->value->inserted->drv = g_strdup(bs->drv->format_name);
-            info->value->inserted->encrypted = bs->encrypted;
-            info->value->inserted->encryption_key_missing = bdrv_key_required(bs);
-            if (bs->backing_file[0]) {
-                info->value->inserted->has_backing_file = true;
-                info->value->inserted->backing_file = g_strdup(bs->backing_file);
-            }
+BlockInfoList *qmp_query_block(Error **errp)
+{
+    BlockInfoList *head = NULL, **p_next = &head;
+    BlockDriverState *bs;
 
-            info->value->inserted->backing_file_depth =
-                bdrv_get_backing_file_depth(bs);
-
-            if (bs->io_limits_enabled) {
-                info->value->inserted->bps =
-                               bs->io_limits.bps[BLOCK_IO_LIMIT_TOTAL];
-                info->value->inserted->bps_rd =
-                               bs->io_limits.bps[BLOCK_IO_LIMIT_READ];
-                info->value->inserted->bps_wr =
-                               bs->io_limits.bps[BLOCK_IO_LIMIT_WRITE];
-                info->value->inserted->iops =
-                               bs->io_limits.iops[BLOCK_IO_LIMIT_TOTAL];
-                info->value->inserted->iops_rd =
-                               bs->io_limits.iops[BLOCK_IO_LIMIT_READ];
-                info->value->inserted->iops_wr =
-                               bs->io_limits.iops[BLOCK_IO_LIMIT_WRITE];
-            }
-        }
+    QTAILQ_FOREACH(bs, &bdrv_states, list) {
+        BlockInfoList *info = g_malloc0(sizeof(*info));
+        info->value = bdrv_query_info(bs);
 
-        /* XXX: waiting for the qapi to support GSList */
-        if (!cur_item) {
-            head = cur_item = info;
-        } else {
-            cur_item->next = info;
-            cur_item = info;
-        }
+        *p_next = info;
+        p_next = &info->next;
     }
 
     return head;
 }
 
-/* Consider exposing this as a full fledged QMP command */
-static BlockStats *qmp_query_blockstat(const BlockDriverState *bs, Error **errp)
+BlockStats *bdrv_query_stats(const BlockDriverState *bs)
 {
     BlockStats *s;
 
@@ -2507,7 +2928,7 @@ static BlockStats *qmp_query_blockstat(const BlockDriverState *bs, Error **errp)
 
     if (bs->file) {
         s->has_parent = true;
-        s->parent = qmp_query_blockstat(bs->file, NULL);
+        s->parent = bdrv_query_stats(bs->file);
     }
 
     return s;
@@ -2515,20 +2936,15 @@ static BlockStats *qmp_query_blockstat(const BlockDriverState *bs, Error **errp)
 
 BlockStatsList *qmp_query_blockstats(Error **errp)
 {
-    BlockStatsList *head = NULL, *cur_item = NULL;
+    BlockStatsList *head = NULL, **p_next = &head;
     BlockDriverState *bs;
 
     QTAILQ_FOREACH(bs, &bdrv_states, list) {
         BlockStatsList *info = g_malloc0(sizeof(*info));
-        info->value = qmp_query_blockstat(bs, NULL);
+        info->value = bdrv_query_stats(bs);
 
-        /* XXX: waiting for the qapi to support GSList */
-        if (!cur_item) {
-            head = cur_item = info;
-        } else {
-            cur_item->next = info;
-            cur_item = info;
-        }
+        *p_next = info;
+        p_next = &info->next;
     }
 
     return head;
@@ -2561,9 +2977,7 @@ int bdrv_write_compressed(BlockDriverState *bs, int64_t sector_num,
     if (bdrv_check_request(bs, sector_num, nb_sectors))
         return -EIO;
 
-    if (bs->dirty_bitmap) {
-        set_dirty_bitmap(bs, sector_num, nb_sectors, 1);
-    }
+    assert(!bs->dirty_bitmap);
 
     return drv->bdrv_write_compressed(bs, sector_num, buf, nb_sectors);
 }
@@ -2740,22 +3154,70 @@ int bdrv_snapshot_load_tmp(BlockDriverState *bs,
     return -ENOTSUP;
 }
 
+/* backing_file can either be relative, or absolute, or a protocol.  If it is
+ * relative, it must be relative to the chain.  So, passing in bs->filename
+ * from a BDS as backing_file should not be done, as that may be relative to
+ * the CWD rather than the chain. */
 BlockDriverState *bdrv_find_backing_image(BlockDriverState *bs,
         const char *backing_file)
 {
-    if (!bs->drv) {
+    char *filename_full = NULL;
+    char *backing_file_full = NULL;
+    char *filename_tmp = NULL;
+    int is_protocol = 0;
+    BlockDriverState *curr_bs = NULL;
+    BlockDriverState *retval = NULL;
+
+    if (!bs || !bs->drv || !backing_file) {
         return NULL;
     }
 
-    if (bs->backing_hd) {
-        if (strcmp(bs->backing_file, backing_file) == 0) {
-            return bs->backing_hd;
+    filename_full     = g_malloc(PATH_MAX);
+    backing_file_full = g_malloc(PATH_MAX);
+    filename_tmp      = g_malloc(PATH_MAX);
+
+    is_protocol = path_has_protocol(backing_file);
+
+    for (curr_bs = bs; curr_bs->backing_hd; curr_bs = curr_bs->backing_hd) {
+
+        /* If either of the filename paths is actually a protocol, then
+         * compare unmodified paths; otherwise make paths relative */
+        if (is_protocol || path_has_protocol(curr_bs->backing_file)) {
+            if (strcmp(backing_file, curr_bs->backing_file) == 0) {
+                retval = curr_bs->backing_hd;
+                break;
+            }
         } else {
-            return bdrv_find_backing_image(bs->backing_hd, backing_file);
+            /* If not an absolute filename path, make it relative to the current
+             * image's filename path */
+            path_combine(filename_tmp, PATH_MAX, curr_bs->filename,
+                         backing_file);
+
+            /* We are going to compare absolute pathnames */
+            if (!realpath(filename_tmp, filename_full)) {
+                continue;
+            }
+
+            /* We need to make sure the backing filename we are comparing against
+             * is relative to the current image filename (or absolute) */
+            path_combine(filename_tmp, PATH_MAX, curr_bs->filename,
+                         curr_bs->backing_file);
+
+            if (!realpath(filename_tmp, backing_file_full)) {
+                continue;
+            }
+
+            if (strcmp(backing_file_full, filename_full) == 0) {
+                retval = curr_bs->backing_hd;
+                break;
+            }
         }
     }
 
-    return NULL;
+    g_free(filename_full);
+    g_free(backing_file_full);
+    g_free(filename_tmp);
+    return retval;
 }
 
 int bdrv_get_backing_file_depth(BlockDriverState *bs)
@@ -2771,6 +3233,22 @@ int bdrv_get_backing_file_depth(BlockDriverState *bs)
     return 1 + bdrv_get_backing_file_depth(bs->backing_hd);
 }
 
+BlockDriverState *bdrv_find_base(BlockDriverState *bs)
+{
+    BlockDriverState *curr_bs = NULL;
+
+    if (!bs) {
+        return NULL;
+    }
+
+    curr_bs = bs;
+
+    while (curr_bs->backing_hd) {
+        curr_bs = curr_bs->backing_hd;
+    }
+    return curr_bs;
+}
+
 #define NB_SUFFIXES 4
 
 char *get_human_readable_size(char *buf, int buf_size, int64_t size)
@@ -3044,7 +3522,7 @@ int bdrv_aio_multiwrite(BlockDriverState *bs, BlockRequest *reqs, int num_reqs)
 
 void bdrv_aio_cancel(BlockDriverAIOCB *acb)
 {
-    acb->pool->cancel(acb);
+    acb->aiocb_info->cancel(acb);
 }
 
 /* block I/O throttling */
@@ -3234,7 +3712,7 @@ static void bdrv_aio_cancel_em(BlockDriverAIOCB *blockacb)
     qemu_aio_release(acb);
 }
 
-static AIOPool bdrv_em_aio_pool = {
+static const AIOCBInfo bdrv_em_aiocb_info = {
     .aiocb_size         = sizeof(BlockDriverAIOCBSync),
     .cancel             = bdrv_aio_cancel_em,
 };
@@ -3263,7 +3741,7 @@ static BlockDriverAIOCB *bdrv_aio_rw_vector(BlockDriverState *bs,
 {
     BlockDriverAIOCBSync *acb;
 
-    acb = qemu_aio_get(&bdrv_em_aio_pool, bs, cb, opaque);
+    acb = qemu_aio_get(&bdrv_em_aiocb_info, bs, cb, opaque);
     acb->is_write = is_write;
     acb->qiov = qiov;
     acb->bounce = qemu_blockalign(bs, qiov->size);
@@ -3308,7 +3786,7 @@ static void bdrv_aio_co_cancel_em(BlockDriverAIOCB *blockacb)
     qemu_aio_flush();
 }
 
-static AIOPool bdrv_em_co_aio_pool = {
+static const AIOCBInfo bdrv_em_co_aiocb_info = {
     .aiocb_size         = sizeof(BlockDriverAIOCBCoroutine),
     .cancel             = bdrv_aio_co_cancel_em,
 };
@@ -3351,7 +3829,7 @@ static BlockDriverAIOCB *bdrv_co_aio_rw_vector(BlockDriverState *bs,
     Coroutine *co;
     BlockDriverAIOCBCoroutine *acb;
 
-    acb = qemu_aio_get(&bdrv_em_co_aio_pool, bs, cb, opaque);
+    acb = qemu_aio_get(&bdrv_em_co_aiocb_info, bs, cb, opaque);
     acb->req.sector = sector_num;
     acb->req.nb_sectors = nb_sectors;
     acb->req.qiov = qiov;
@@ -3381,7 +3859,7 @@ BlockDriverAIOCB *bdrv_aio_flush(BlockDriverState *bs,
     Coroutine *co;
     BlockDriverAIOCBCoroutine *acb;
 
-    acb = qemu_aio_get(&bdrv_em_co_aio_pool, bs, cb, opaque);
+    acb = qemu_aio_get(&bdrv_em_co_aiocb_info, bs, cb, opaque);
     co = qemu_coroutine_create(bdrv_aio_flush_co_entry);
     qemu_coroutine_enter(co, acb);
 
@@ -3407,7 +3885,7 @@ BlockDriverAIOCB *bdrv_aio_discard(BlockDriverState *bs,
 
     trace_bdrv_aio_discard(bs, sector_num, nb_sectors, opaque);
 
-    acb = qemu_aio_get(&bdrv_em_co_aio_pool, bs, cb, opaque);
+    acb = qemu_aio_get(&bdrv_em_co_aiocb_info, bs, cb, opaque);
     acb->req.sector = sector_num;
     acb->req.nb_sectors = nb_sectors;
     co = qemu_coroutine_create(bdrv_aio_discard_co_entry);
@@ -3427,18 +3905,13 @@ void bdrv_init_with_whitelist(void)
     bdrv_init();
 }
 
-void *qemu_aio_get(AIOPool *pool, BlockDriverState *bs,
+void *qemu_aio_get(const AIOCBInfo *aiocb_info, BlockDriverState *bs,
                    BlockDriverCompletionFunc *cb, void *opaque)
 {
     BlockDriverAIOCB *acb;
 
-    if (pool->free_aiocb) {
-        acb = pool->free_aiocb;
-        pool->free_aiocb = acb->next;
-    } else {
-        acb = g_malloc0(pool->aiocb_size);
-        acb->pool = pool;
-    }
+    acb = g_slice_alloc(aiocb_info->aiocb_size);
+    acb->aiocb_info = aiocb_info;
     acb->bs = bs;
     acb->cb = cb;
     acb->opaque = opaque;
@@ -3447,10 +3920,8 @@ void *qemu_aio_get(AIOPool *pool, BlockDriverState *bs,
 
 void qemu_aio_release(void *p)
 {
-    BlockDriverAIOCB *acb = (BlockDriverAIOCB *)p;
-    AIOPool *pool = acb->pool;
-    acb->next = pool->free_aiocb;
-    pool->free_aiocb = acb;
+    BlockDriverAIOCB *acb = p;
+    g_slice_free1(acb->aiocb_info->aiocb_size, acb);
 }
 
 /**************************************************************/
@@ -3806,13 +4277,54 @@ int bdrv_get_dirty(BlockDriverState *bs, int64_t sector)
 
     if (bs->dirty_bitmap &&
         (sector << BDRV_SECTOR_BITS) < bdrv_getlength(bs)) {
-        return !!(bs->dirty_bitmap[chunk / (sizeof(unsigned long) * 8)] &
-            (1UL << (chunk % (sizeof(unsigned long) * 8))));
+        return !!(bs->dirty_bitmap[chunk / BITS_PER_LONG] &
+            (1UL << (chunk % BITS_PER_LONG)));
     } else {
         return 0;
     }
 }
 
+int64_t bdrv_get_next_dirty(BlockDriverState *bs, int64_t sector)
+{
+    int64_t chunk;
+    int bit, elem;
+
+    /* Avoid an infinite loop.  */
+    assert(bs->dirty_count > 0);
+
+    sector = (sector | (BDRV_SECTORS_PER_DIRTY_CHUNK - 1)) + 1;
+    chunk = sector / (int64_t)BDRV_SECTORS_PER_DIRTY_CHUNK;
+
+    QEMU_BUILD_BUG_ON(sizeof(bs->dirty_bitmap[0]) * 8 != BITS_PER_LONG);
+    elem = chunk / BITS_PER_LONG;
+    bit = chunk % BITS_PER_LONG;
+    for (;;) {
+        if (sector >= bs->total_sectors) {
+            sector = 0;
+            bit = elem = 0;
+        }
+        if (bit == 0 && bs->dirty_bitmap[elem] == 0) {
+            sector += BDRV_SECTORS_PER_DIRTY_CHUNK * BITS_PER_LONG;
+            elem++;
+        } else {
+            if (bs->dirty_bitmap[elem] & (1UL << bit)) {
+                return sector;
+            }
+            sector += BDRV_SECTORS_PER_DIRTY_CHUNK;
+            if (++bit == BITS_PER_LONG) {
+                bit = 0;
+                elem++;
+            }
+        }
+    }
+}
+
+void bdrv_set_dirty(BlockDriverState *bs, int64_t cur_sector,
+                    int nr_sectors)
+{
+    set_dirty_bitmap(bs, cur_sector, nr_sectors, 1);
+}
+
 void bdrv_reset_dirty(BlockDriverState *bs, int64_t cur_sector,
                       int nr_sectors)
 {
@@ -3846,9 +4358,9 @@ void bdrv_iostatus_enable(BlockDriverState *bs)
 bool bdrv_iostatus_is_enabled(const BlockDriverState *bs)
 {
     return (bs->iostatus_enabled &&
-           (bs->on_write_error == BLOCK_ERR_STOP_ENOSPC ||
-            bs->on_write_error == BLOCK_ERR_STOP_ANY    ||
-            bs->on_read_error == BLOCK_ERR_STOP_ANY));
+           (bs->on_write_error == BLOCKDEV_ON_ERROR_ENOSPC ||
+            bs->on_write_error == BLOCKDEV_ON_ERROR_STOP   ||
+            bs->on_read_error == BLOCKDEV_ON_ERROR_STOP));
 }
 
 void bdrv_iostatus_disable(BlockDriverState *bs)
@@ -3860,17 +4372,16 @@ void bdrv_iostatus_reset(BlockDriverState *bs)
 {
     if (bdrv_iostatus_is_enabled(bs)) {
         bs->iostatus = BLOCK_DEVICE_IO_STATUS_OK;
+        if (bs->job) {
+            block_job_iostatus_reset(bs->job);
+        }
     }
 }
 
-/* XXX: Today this is set by device models because it makes the implementation
-   quite simple. However, the block layer knows about the error, so it's
-   possible to implement this without device models being involved */
 void bdrv_iostatus_set_err(BlockDriverState *bs, int error)
 {
-    if (bdrv_iostatus_is_enabled(bs) &&
-        bs->iostatus == BLOCK_DEVICE_IO_STATUS_OK) {
-        assert(error >= 0);
+    assert(bdrv_iostatus_is_enabled(bs));
+    if (bs->iostatus == BLOCK_DEVICE_IO_STATUS_OK) {
         bs->iostatus = error == ENOSPC ? BLOCK_DEVICE_IO_STATUS_NOSPACE :
                                          BLOCK_DEVICE_IO_STATUS_FAILED;
     }
@@ -4044,130 +4555,3 @@ out:
 
     return ret;
 }
-
-void *block_job_create(const BlockJobType *job_type, BlockDriverState *bs,
-                       int64_t speed, BlockDriverCompletionFunc *cb,
-                       void *opaque, Error **errp)
-{
-    BlockJob *job;
-
-    if (bs->job || bdrv_in_use(bs)) {
-        error_set(errp, QERR_DEVICE_IN_USE, bdrv_get_device_name(bs));
-        return NULL;
-    }
-    bdrv_set_in_use(bs, 1);
-
-    job = g_malloc0(job_type->instance_size);
-    job->job_type      = job_type;
-    job->bs            = bs;
-    job->cb            = cb;
-    job->opaque        = opaque;
-    job->busy          = true;
-    bs->job = job;
-
-    /* Only set speed when necessary to avoid NotSupported error */
-    if (speed != 0) {
-        Error *local_err = NULL;
-
-        block_job_set_speed(job, speed, &local_err);
-        if (error_is_set(&local_err)) {
-            bs->job = NULL;
-            g_free(job);
-            bdrv_set_in_use(bs, 0);
-            error_propagate(errp, local_err);
-            return NULL;
-        }
-    }
-    return job;
-}
-
-void block_job_complete(BlockJob *job, int ret)
-{
-    BlockDriverState *bs = job->bs;
-
-    assert(bs->job == job);
-    job->cb(job->opaque, ret);
-    bs->job = NULL;
-    g_free(job);
-    bdrv_set_in_use(bs, 0);
-}
-
-void block_job_set_speed(BlockJob *job, int64_t speed, Error **errp)
-{
-    Error *local_err = NULL;
-
-    if (!job->job_type->set_speed) {
-        error_set(errp, QERR_NOT_SUPPORTED);
-        return;
-    }
-    job->job_type->set_speed(job, speed, &local_err);
-    if (error_is_set(&local_err)) {
-        error_propagate(errp, local_err);
-        return;
-    }
-
-    job->speed = speed;
-}
-
-void block_job_cancel(BlockJob *job)
-{
-    job->cancelled = true;
-    if (job->co && !job->busy) {
-        qemu_coroutine_enter(job->co, NULL);
-    }
-}
-
-bool block_job_is_cancelled(BlockJob *job)
-{
-    return job->cancelled;
-}
-
-struct BlockCancelData {
-    BlockJob *job;
-    BlockDriverCompletionFunc *cb;
-    void *opaque;
-    bool cancelled;
-    int ret;
-};
-
-static void block_job_cancel_cb(void *opaque, int ret)
-{
-    struct BlockCancelData *data = opaque;
-
-    data->cancelled = block_job_is_cancelled(data->job);
-    data->ret = ret;
-    data->cb(data->opaque, ret);
-}
-
-int block_job_cancel_sync(BlockJob *job)
-{
-    struct BlockCancelData data;
-    BlockDriverState *bs = job->bs;
-
-    assert(bs->job == job);
-
-    /* Set up our own callback to store the result and chain to
-     * the original callback.
-     */
-    data.job = job;
-    data.cb = job->cb;
-    data.opaque = job->opaque;
-    data.ret = -EINPROGRESS;
-    job->cb = block_job_cancel_cb;
-    job->opaque = &data;
-    block_job_cancel(job);
-    while (data.ret == -EINPROGRESS) {
-        qemu_aio_wait();
-    }
-    return (data.cancelled && data.ret == 0) ? -ECANCELED : data.ret;
-}
-
-void block_job_sleep_ns(BlockJob *job, QEMUClock *clock, int64_t ns)
-{
-    /* Check cancellation *before* setting busy = false, too!  */
-    if (!block_job_is_cancelled(job)) {
-        job->busy = false;
-        co_sleep_ns(clock, ns);
-        job->busy = true;
-    }
-}
diff --git a/block.h b/block.h
index 2e2be1107..722c62059 100644
--- a/block.h
+++ b/block.h
@@ -6,9 +6,11 @@
 #include "qemu-option.h"
 #include "qemu-coroutine.h"
 #include "qobject.h"
+#include "qapi-types.h"
 
 /* block.c */
 typedef struct BlockDriver BlockDriver;
+typedef struct BlockJob BlockJob;
 
 typedef struct BlockDriverInfo {
     /* in bytes, 0 if irrelevant */
@@ -80,6 +82,7 @@ typedef struct BlockDevOps {
 #define BDRV_O_COPY_ON_READ 0x0400 /* copy read backing sectors into image */
 #define BDRV_O_INCOMING    0x0800  /* consistency hint for incoming migration */
 #define BDRV_O_CHECK       0x1000  /* open solely for consistency check */
+#define BDRV_O_ALLOW_RDWR  0x2000  /* allow reopen to change from r/o to r/w */
 
 #define BDRV_O_CACHE_MASK  (BDRV_O_NOCACHE | BDRV_O_CACHE_WB | BDRV_O_NO_FLUSH)
 
@@ -88,21 +91,23 @@ typedef struct BlockDevOps {
 #define BDRV_SECTOR_MASK   ~(BDRV_SECTOR_SIZE - 1)
 
 typedef enum {
-    BLOCK_ERR_REPORT, BLOCK_ERR_IGNORE, BLOCK_ERR_STOP_ENOSPC,
-    BLOCK_ERR_STOP_ANY
+    BDRV_ACTION_REPORT, BDRV_ACTION_IGNORE, BDRV_ACTION_STOP
 } BlockErrorAction;
 
-typedef enum {
-    BDRV_ACTION_REPORT, BDRV_ACTION_IGNORE, BDRV_ACTION_STOP
-} BlockQMPEventAction;
+typedef QSIMPLEQ_HEAD(BlockReopenQueue, BlockReopenQueueEntry) BlockReopenQueue;
+
+typedef struct BDRVReopenState {
+    BlockDriverState *bs;
+    int flags;
+    void *opaque;
+} BDRVReopenState;
+
 
 void bdrv_iostatus_enable(BlockDriverState *bs);
 void bdrv_iostatus_reset(BlockDriverState *bs);
 void bdrv_iostatus_disable(BlockDriverState *bs);
 bool bdrv_iostatus_is_enabled(const BlockDriverState *bs);
 void bdrv_iostatus_set_err(BlockDriverState *bs, int error);
-void bdrv_emit_qmp_error_event(const BlockDriverState *bdrv,
-                               BlockQMPEventAction action, int is_read);
 void bdrv_info_print(Monitor *mon, const QObject *data);
 void bdrv_info(Monitor *mon, QObject **ret_data);
 void bdrv_stats_print(Monitor *mon, const QObject *data);
@@ -128,9 +133,19 @@ void bdrv_append(BlockDriverState *bs_new, BlockDriverState *bs_top);
 void bdrv_delete(BlockDriverState *bs);
 int bdrv_parse_cache_flags(const char *mode, int *flags);
 int bdrv_file_open(BlockDriverState **pbs, const char *filename, int flags);
+int bdrv_open_backing_file(BlockDriverState *bs);
 int bdrv_open(BlockDriverState *bs, const char *filename, int flags,
               BlockDriver *drv);
+BlockReopenQueue *bdrv_reopen_queue(BlockReopenQueue *bs_queue,
+                                    BlockDriverState *bs, int flags);
+int bdrv_reopen_multiple(BlockReopenQueue *bs_queue, Error **errp);
+int bdrv_reopen(BlockDriverState *bs, int bdrv_flags, Error **errp);
+int bdrv_reopen_prepare(BDRVReopenState *reopen_state,
+                        BlockReopenQueue *queue, Error **errp);
+void bdrv_reopen_commit(BDRVReopenState *reopen_state);
+void bdrv_reopen_abort(BDRVReopenState *reopen_state);
 void bdrv_close(BlockDriverState *bs);
+void bdrv_add_close_notifier(BlockDriverState *bs, Notifier *notify);
 int bdrv_attach_dev(BlockDriverState *bs, void *dev);
 void bdrv_attach_dev_nofail(BlockDriverState *bs, void *dev);
 void bdrv_detach_dev(BlockDriverState *bs, void *dev);
@@ -185,6 +200,11 @@ int bdrv_commit_all(void);
 int bdrv_change_backing_file(BlockDriverState *bs,
     const char *backing_file, const char *backing_fmt);
 void bdrv_register(BlockDriver *bdrv);
+int bdrv_drop_intermediate(BlockDriverState *active, BlockDriverState *top,
+                           BlockDriverState *base);
+BlockDriverState *bdrv_find_overlay(BlockDriverState *active,
+                                    BlockDriverState *bs);
+BlockDriverState *bdrv_find_base(BlockDriverState *bs);
 
 
 typedef struct BdrvCheckResult {
@@ -259,9 +279,12 @@ int bdrv_has_zero_init(BlockDriverState *bs);
 int bdrv_is_allocated(BlockDriverState *bs, int64_t sector_num, int nb_sectors,
                       int *pnum);
 
-void bdrv_set_on_error(BlockDriverState *bs, BlockErrorAction on_read_error,
-                       BlockErrorAction on_write_error);
-BlockErrorAction bdrv_get_on_error(BlockDriverState *bs, int is_read);
+void bdrv_set_on_error(BlockDriverState *bs, BlockdevOnError on_read_error,
+                       BlockdevOnError on_write_error);
+BlockdevOnError bdrv_get_on_error(BlockDriverState *bs, bool is_read);
+BlockErrorAction bdrv_get_error_action(BlockDriverState *bs, bool is_read, int error);
+void bdrv_error_action(BlockDriverState *bs, BlockErrorAction action,
+                       bool is_read, int error);
 int bdrv_is_read_only(BlockDriverState *bs);
 int bdrv_is_sg(BlockDriverState *bs);
 int bdrv_enable_write_cache(BlockDriverState *bs);
@@ -292,6 +315,8 @@ void bdrv_get_backing_filename(BlockDriverState *bs,
                                char *filename, int filename_size);
 void bdrv_get_full_backing_filename(BlockDriverState *bs,
                                     char *dest, size_t sz);
+BlockInfo *bdrv_query_info(BlockDriverState *s);
+BlockStats *bdrv_query_stats(const BlockDriverState *bs);
 int bdrv_can_snapshot(BlockDriverState *bs);
 int bdrv_is_snapshot(BlockDriverState *bs);
 BlockDriverState *bdrv_snapshots(void);
@@ -329,8 +354,9 @@ void *qemu_blockalign(BlockDriverState *bs, size_t size);
 
 void bdrv_set_dirty_tracking(BlockDriverState *bs, int enable);
 int bdrv_get_dirty(BlockDriverState *bs, int64_t sector);
-void bdrv_reset_dirty(BlockDriverState *bs, int64_t cur_sector,
-                      int nr_sectors);
+void bdrv_set_dirty(BlockDriverState *bs, int64_t cur_sector, int nr_sectors);
+void bdrv_reset_dirty(BlockDriverState *bs, int64_t cur_sector, int nr_sectors);
+int64_t bdrv_get_next_dirty(BlockDriverState *bs, int64_t sector);
 int64_t bdrv_get_dirty_count(BlockDriverState *bs);
 
 void bdrv_enable_copy_on_read(BlockDriverState *bs);
diff --git a/block/Makefile.objs b/block/Makefile.objs
index b5754d39b..7f015105b 100644
--- a/block/Makefile.objs
+++ b/block/Makefile.objs
@@ -2,10 +2,19 @@ block-obj-y += raw.o cow.o qcow.o vdi.o vmdk.o cloop.o dmg.o bochs.o vpc.o vvfat
 block-obj-y += qcow2.o qcow2-refcount.o qcow2-cluster.o qcow2-snapshot.o qcow2-cache.o
 block-obj-y += qed.o qed-gencb.o qed-l2-cache.o qed-table.o qed-cluster.o
 block-obj-y += qed-check.o
-block-obj-y += parallels.o nbd.o blkdebug.o sheepdog.o blkverify.o
-block-obj-y += stream.o
-block-obj-$(CONFIG_WIN32) += raw-win32.o
+block-obj-y += parallels.o blkdebug.o blkverify.o
+block-obj-$(CONFIG_WIN32) += raw-win32.o win32-aio.o
 block-obj-$(CONFIG_POSIX) += raw-posix.o
+block-obj-$(CONFIG_LINUX_AIO) += linux-aio.o
+
+ifeq ($(CONFIG_POSIX),y)
+block-obj-y += nbd.o sheepdog.o
 block-obj-$(CONFIG_LIBISCSI) += iscsi.o
 block-obj-$(CONFIG_CURL) += curl.o
 block-obj-$(CONFIG_RBD) += rbd.o
+block-obj-$(CONFIG_GLUSTERFS) += gluster.o
+endif
+
+common-obj-y += stream.o
+common-obj-y += commit.o
+common-obj-y += mirror.o
diff --git a/block/blkdebug.c b/block/blkdebug.c
index 59dcea065..d61ece86a 100644
--- a/block/blkdebug.c
+++ b/block/blkdebug.c
@@ -28,6 +28,7 @@
 
 typedef struct BDRVBlkdebugState {
     int state;
+    int new_state;
     QLIST_HEAD(, BlkdebugRule) rules[BLKDBG_EVENT_MAX];
     QSIMPLEQ_HEAD(, BlkdebugRule) active_rules;
 } BDRVBlkdebugState;
@@ -40,7 +41,7 @@ typedef struct BlkdebugAIOCB {
 
 static void blkdebug_aio_cancel(BlockDriverAIOCB *blockacb);
 
-static AIOPool blkdebug_aio_pool = {
+static const AIOCBInfo blkdebug_aiocb_info = {
     .aiocb_size = sizeof(BlkdebugAIOCB),
     .cancel     = blkdebug_aio_cancel,
 };
@@ -334,7 +335,7 @@ static BlockDriverAIOCB *inject_error(BlockDriverState *bs,
         return NULL;
     }
 
-    acb = qemu_aio_get(&blkdebug_aio_pool, bs, cb, opaque);
+    acb = qemu_aio_get(&blkdebug_aiocb_info, bs, cb, opaque);
     acb->ret = -error;
 
     bh = qemu_bh_new(error_callback_bh, acb);
@@ -403,12 +404,12 @@ static void blkdebug_close(BlockDriverState *bs)
 }
 
 static bool process_rule(BlockDriverState *bs, struct BlkdebugRule *rule,
-    int old_state, bool injected)
+    bool injected)
 {
     BDRVBlkdebugState *s = bs->opaque;
 
     /* Only process rules for the current state */
-    if (rule->state && rule->state != old_state) {
+    if (rule->state && rule->state != s->state) {
         return injected;
     }
 
@@ -423,7 +424,7 @@ static bool process_rule(BlockDriverState *bs, struct BlkdebugRule *rule,
         break;
 
     case ACTION_SET_STATE:
-        s->state = rule->options.set_state.new_state;
+        s->new_state = rule->options.set_state.new_state;
         break;
     }
     return injected;
@@ -433,15 +434,16 @@ static void blkdebug_debug_event(BlockDriverState *bs, BlkDebugEvent event)
 {
     BDRVBlkdebugState *s = bs->opaque;
     struct BlkdebugRule *rule;
-    int old_state = s->state;
     bool injected;
 
     assert((int)event >= 0 && event < BLKDBG_EVENT_MAX);
 
     injected = false;
+    s->new_state = s->state;
     QLIST_FOREACH(rule, &s->rules[event], next) {
-        injected = process_rule(bs, rule, old_state, injected);
+        injected = process_rule(bs, rule, injected);
     }
+    s->state = s->new_state;
 }
 
 static int64_t blkdebug_getlength(BlockDriverState *bs)
diff --git a/block/blkverify.c b/block/blkverify.c
index 9d5f1ec5b..4beede77a 100644
--- a/block/blkverify.c
+++ b/block/blkverify.c
@@ -48,7 +48,7 @@ static void blkverify_aio_cancel(BlockDriverAIOCB *blockacb)
     }
 }
 
-static AIOPool blkverify_aio_pool = {
+static const AIOCBInfo blkverify_aiocb_info = {
     .aiocb_size         = sizeof(BlkverifyAIOCB),
     .cancel             = blkverify_aio_cancel,
 };
@@ -233,7 +233,7 @@ static BlkverifyAIOCB *blkverify_aio_get(BlockDriverState *bs, bool is_write,
                                          BlockDriverCompletionFunc *cb,
                                          void *opaque)
 {
-    BlkverifyAIOCB *acb = qemu_aio_get(&blkverify_aio_pool, bs, cb, opaque);
+    BlkverifyAIOCB *acb = qemu_aio_get(&blkverify_aiocb_info, bs, cb, opaque);
 
     acb->bh = NULL;
     acb->is_write = is_write;
diff --git a/block/commit.c b/block/commit.c
new file mode 100644
index 000000000..fae79582d
--- /dev/null
+++ b/block/commit.c
@@ -0,0 +1,259 @@
+/*
+ * Live block commit
+ *
+ * Copyright Red Hat, Inc. 2012
+ *
+ * Authors:
+ *  Jeff Cody   <jcody@redhat.com>
+ *  Based on stream.c by Stefan Hajnoczi
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ *
+ */
+
+#include "trace.h"
+#include "block_int.h"
+#include "blockjob.h"
+#include "qemu/ratelimit.h"
+
+enum {
+    /*
+     * Size of data buffer for populating the image file.  This should be large
+     * enough to process multiple clusters in a single call, so that populating
+     * contiguous regions of the image is efficient.
+     */
+    COMMIT_BUFFER_SIZE = 512 * 1024, /* in bytes */
+};
+
+#define SLICE_TIME 100000000ULL /* ns */
+
+typedef struct CommitBlockJob {
+    BlockJob common;
+    RateLimit limit;
+    BlockDriverState *active;
+    BlockDriverState *top;
+    BlockDriverState *base;
+    BlockdevOnError on_error;
+    int base_flags;
+    int orig_overlay_flags;
+} CommitBlockJob;
+
+static int coroutine_fn commit_populate(BlockDriverState *bs,
+                                        BlockDriverState *base,
+                                        int64_t sector_num, int nb_sectors,
+                                        void *buf)
+{
+    int ret = 0;
+
+    ret = bdrv_read(bs, sector_num, buf, nb_sectors);
+    if (ret) {
+        return ret;
+    }
+
+    ret = bdrv_write(base, sector_num, buf, nb_sectors);
+    if (ret) {
+        return ret;
+    }
+
+    return 0;
+}
+
+static void coroutine_fn commit_run(void *opaque)
+{
+    CommitBlockJob *s = opaque;
+    BlockDriverState *active = s->active;
+    BlockDriverState *top = s->top;
+    BlockDriverState *base = s->base;
+    BlockDriverState *overlay_bs = NULL;
+    int64_t sector_num, end;
+    int ret = 0;
+    int n = 0;
+    void *buf;
+    int bytes_written = 0;
+    int64_t base_len;
+
+    ret = s->common.len = bdrv_getlength(top);
+
+
+    if (s->common.len < 0) {
+        goto exit_restore_reopen;
+    }
+
+    ret = base_len = bdrv_getlength(base);
+    if (base_len < 0) {
+        goto exit_restore_reopen;
+    }
+
+    if (base_len < s->common.len) {
+        ret = bdrv_truncate(base, s->common.len);
+        if (ret) {
+            goto exit_restore_reopen;
+        }
+    }
+
+    overlay_bs = bdrv_find_overlay(active, top);
+
+    end = s->common.len >> BDRV_SECTOR_BITS;
+    buf = qemu_blockalign(top, COMMIT_BUFFER_SIZE);
+
+    for (sector_num = 0; sector_num < end; sector_num += n) {
+        uint64_t delay_ns = 0;
+        bool copy;
+
+wait:
+        /* Note that even when no rate limit is applied we need to yield
+         * with no pending I/O here so that qemu_aio_flush() returns.
+         */
+        block_job_sleep_ns(&s->common, rt_clock, delay_ns);
+        if (block_job_is_cancelled(&s->common)) {
+            break;
+        }
+        /* Copy if allocated above the base */
+        ret = bdrv_co_is_allocated_above(top, base, sector_num,
+                                         COMMIT_BUFFER_SIZE / BDRV_SECTOR_SIZE,
+                                         &n);
+        copy = (ret == 1);
+        trace_commit_one_iteration(s, sector_num, n, ret);
+        if (copy) {
+            if (s->common.speed) {
+                delay_ns = ratelimit_calculate_delay(&s->limit, n);
+                if (delay_ns > 0) {
+                    goto wait;
+                }
+            }
+            ret = commit_populate(top, base, sector_num, n, buf);
+            bytes_written += n * BDRV_SECTOR_SIZE;
+        }
+        if (ret < 0) {
+            if (s->on_error == BLOCKDEV_ON_ERROR_STOP ||
+                s->on_error == BLOCKDEV_ON_ERROR_REPORT||
+                (s->on_error == BLOCKDEV_ON_ERROR_ENOSPC && ret == -ENOSPC)) {
+                goto exit_free_buf;
+            } else {
+                n = 0;
+                continue;
+            }
+        }
+        /* Publish progress */
+        s->common.offset += n * BDRV_SECTOR_SIZE;
+    }
+
+    ret = 0;
+
+    if (!block_job_is_cancelled(&s->common) && sector_num == end) {
+        /* success */
+        ret = bdrv_drop_intermediate(active, top, base);
+    }
+
+exit_free_buf:
+    qemu_vfree(buf);
+
+exit_restore_reopen:
+    /* restore base open flags here if appropriate (e.g., change the base back
+     * to r/o). These reopens do not need to be atomic, since we won't abort
+     * even on failure here */
+    if (s->base_flags != bdrv_get_flags(base)) {
+        bdrv_reopen(base, s->base_flags, NULL);
+    }
+    if (s->orig_overlay_flags != bdrv_get_flags(overlay_bs)) {
+        bdrv_reopen(overlay_bs, s->orig_overlay_flags, NULL);
+    }
+
+    block_job_completed(&s->common, ret);
+}
+
+static void commit_set_speed(BlockJob *job, int64_t speed, Error **errp)
+{
+    CommitBlockJob *s = container_of(job, CommitBlockJob, common);
+
+    if (speed < 0) {
+        error_set(errp, QERR_INVALID_PARAMETER, "speed");
+        return;
+    }
+    ratelimit_set_speed(&s->limit, speed / BDRV_SECTOR_SIZE, SLICE_TIME);
+}
+
+static BlockJobType commit_job_type = {
+    .instance_size = sizeof(CommitBlockJob),
+    .job_type      = "commit",
+    .set_speed     = commit_set_speed,
+};
+
+void commit_start(BlockDriverState *bs, BlockDriverState *base,
+                  BlockDriverState *top, int64_t speed,
+                  BlockdevOnError on_error, BlockDriverCompletionFunc *cb,
+                  void *opaque, Error **errp)
+{
+    CommitBlockJob *s;
+    BlockReopenQueue *reopen_queue = NULL;
+    int orig_overlay_flags;
+    int orig_base_flags;
+    BlockDriverState *overlay_bs;
+    Error *local_err = NULL;
+
+    if ((on_error == BLOCKDEV_ON_ERROR_STOP ||
+         on_error == BLOCKDEV_ON_ERROR_ENOSPC) &&
+        !bdrv_iostatus_is_enabled(bs)) {
+        error_set(errp, QERR_INVALID_PARAMETER_COMBINATION);
+        return;
+    }
+
+    /* Once we support top == active layer, remove this check */
+    if (top == bs) {
+        error_setg(errp,
+                   "Top image as the active layer is currently unsupported");
+        return;
+    }
+
+    if (top == base) {
+        error_setg(errp, "Invalid files for merge: top and base are the same");
+        return;
+    }
+
+    overlay_bs = bdrv_find_overlay(bs, top);
+
+    if (overlay_bs == NULL) {
+        error_setg(errp, "Could not find overlay image for %s:", top->filename);
+        return;
+    }
+
+    orig_base_flags    = bdrv_get_flags(base);
+    orig_overlay_flags = bdrv_get_flags(overlay_bs);
+
+    /* convert base & overlay_bs to r/w, if necessary */
+    if (!(orig_base_flags & BDRV_O_RDWR)) {
+        reopen_queue = bdrv_reopen_queue(reopen_queue, base,
+                                         orig_base_flags | BDRV_O_RDWR);
+    }
+    if (!(orig_overlay_flags & BDRV_O_RDWR)) {
+        reopen_queue = bdrv_reopen_queue(reopen_queue, overlay_bs,
+                                         orig_overlay_flags | BDRV_O_RDWR);
+    }
+    if (reopen_queue) {
+        bdrv_reopen_multiple(reopen_queue, &local_err);
+        if (local_err != NULL) {
+            error_propagate(errp, local_err);
+            return;
+        }
+    }
+
+
+    s = block_job_create(&commit_job_type, bs, speed, cb, opaque, errp);
+    if (!s) {
+        return;
+    }
+
+    s->base   = base;
+    s->top    = top;
+    s->active = bs;
+
+    s->base_flags          = orig_base_flags;
+    s->orig_overlay_flags  = orig_overlay_flags;
+
+    s->on_error = on_error;
+    s->common.co = qemu_coroutine_create(commit_run);
+
+    trace_commit_start(bs, base, top, s, s->common.co, opaque);
+    qemu_coroutine_enter(s->common.co, s);
+}
diff --git a/block/curl.c b/block/curl.c
index e7c3634d3..1179484de 100644
--- a/block/curl.c
+++ b/block/curl.c
@@ -438,7 +438,7 @@ static void curl_aio_cancel(BlockDriverAIOCB *blockacb)
     // Do we have to implement canceling? Seems to work without...
 }
 
-static AIOPool curl_aio_pool = {
+static const AIOCBInfo curl_aiocb_info = {
     .aiocb_size         = sizeof(CURLAIOCB),
     .cancel             = curl_aio_cancel,
 };
@@ -505,7 +505,7 @@ static BlockDriverAIOCB *curl_aio_readv(BlockDriverState *bs,
 {
     CURLAIOCB *acb;
 
-    acb = qemu_aio_get(&curl_aio_pool, bs, cb, opaque);
+    acb = qemu_aio_get(&curl_aiocb_info, bs, cb, opaque);
 
     acb->qiov = qiov;
     acb->sector_num = sector_num;
@@ -542,8 +542,7 @@ static void curl_close(BlockDriverState *bs)
     }
     if (s->multi)
         curl_multi_cleanup(s->multi);
-    if (s->url)
-        free(s->url);
+    g_free(s->url);
 }
 
 static int64_t curl_getlength(BlockDriverState *bs)
diff --git a/block/gluster.c b/block/gluster.c
new file mode 100644
index 000000000..1c90174b1
--- /dev/null
+++ b/block/gluster.c
@@ -0,0 +1,624 @@
+/*
+ * GlusterFS backend for QEMU
+ *
+ * Copyright (C) 2012 Bharata B Rao <bharata@linux.vnet.ibm.com>
+ *
+ * Pipe handling mechanism in AIO implementation is derived from
+ * block/rbd.c. Hence,
+ *
+ * Copyright (C) 2010-2011 Christian Brunner <chb@muc.de>,
+ *                         Josh Durgin <josh.durgin@dreamhost.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ *
+ * Contributions after 2012-01-13 are licensed under the terms of the
+ * GNU GPL, version 2 or (at your option) any later version.
+ */
+#include <glusterfs/api/glfs.h>
+#include "block_int.h"
+#include "qemu_socket.h"
+#include "uri.h"
+
+typedef struct GlusterAIOCB {
+    BlockDriverAIOCB common;
+    int64_t size;
+    int ret;
+    bool *finished;
+    QEMUBH *bh;
+} GlusterAIOCB;
+
+typedef struct BDRVGlusterState {
+    struct glfs *glfs;
+    int fds[2];
+    struct glfs_fd *fd;
+    int qemu_aio_count;
+    int event_reader_pos;
+    GlusterAIOCB *event_acb;
+} BDRVGlusterState;
+
+#define GLUSTER_FD_READ  0
+#define GLUSTER_FD_WRITE 1
+
+typedef struct GlusterConf {
+    char *server;
+    int port;
+    char *volname;
+    char *image;
+    char *transport;
+} GlusterConf;
+
+static void qemu_gluster_gconf_free(GlusterConf *gconf)
+{
+    g_free(gconf->server);
+    g_free(gconf->volname);
+    g_free(gconf->image);
+    g_free(gconf->transport);
+    g_free(gconf);
+}
+
+static int parse_volume_options(GlusterConf *gconf, char *path)
+{
+    char *p, *q;
+
+    if (!path) {
+        return -EINVAL;
+    }
+
+    /* volume */
+    p = q = path + strspn(path, "/");
+    p += strcspn(p, "/");
+    if (*p == '\0') {
+        return -EINVAL;
+    }
+    gconf->volname = g_strndup(q, p - q);
+
+    /* image */
+    p += strspn(p, "/");
+    if (*p == '\0') {
+        return -EINVAL;
+    }
+    gconf->image = g_strdup(p);
+    return 0;
+}
+
+/*
+ * file=gluster[+transport]://[server[:port]]/volname/image[?socket=...]
+ *
+ * 'gluster' is the protocol.
+ *
+ * 'transport' specifies the transport type used to connect to gluster
+ * management daemon (glusterd). Valid transport types are
+ * tcp, unix and rdma. If a transport type isn't specified, then tcp
+ * type is assumed.
+ *
+ * 'server' specifies the server where the volume file specification for
+ * the given volume resides. This can be either hostname, ipv4 address
+ * or ipv6 address. ipv6 address needs to be within square brackets [ ].
+ * If transport type is 'unix', then 'server' field should not be specifed.
+ * The 'socket' field needs to be populated with the path to unix domain
+ * socket.
+ *
+ * 'port' is the port number on which glusterd is listening. This is optional
+ * and if not specified, QEMU will send 0 which will make gluster to use the
+ * default port. If the transport type is unix, then 'port' should not be
+ * specified.
+ *
+ * 'volname' is the name of the gluster volume which contains the VM image.
+ *
+ * 'image' is the path to the actual VM image that resides on gluster volume.
+ *
+ * Examples:
+ *
+ * file=gluster://1.2.3.4/testvol/a.img
+ * file=gluster+tcp://1.2.3.4/testvol/a.img
+ * file=gluster+tcp://1.2.3.4:24007/testvol/dir/a.img
+ * file=gluster+tcp://[1:2:3:4:5:6:7:8]/testvol/dir/a.img
+ * file=gluster+tcp://[1:2:3:4:5:6:7:8]:24007/testvol/dir/a.img
+ * file=gluster+tcp://server.domain.com:24007/testvol/dir/a.img
+ * file=gluster+unix:///testvol/dir/a.img?socket=/tmp/glusterd.socket
+ * file=gluster+rdma://1.2.3.4:24007/testvol/a.img
+ */
+static int qemu_gluster_parseuri(GlusterConf *gconf, const char *filename)
+{
+    URI *uri;
+    QueryParams *qp = NULL;
+    bool is_unix = false;
+    int ret = 0;
+
+    uri = uri_parse(filename);
+    if (!uri) {
+        return -EINVAL;
+    }
+
+    /* transport */
+    if (!strcmp(uri->scheme, "gluster")) {
+        gconf->transport = g_strdup("tcp");
+    } else if (!strcmp(uri->scheme, "gluster+tcp")) {
+        gconf->transport = g_strdup("tcp");
+    } else if (!strcmp(uri->scheme, "gluster+unix")) {
+        gconf->transport = g_strdup("unix");
+        is_unix = true;
+    } else if (!strcmp(uri->scheme, "gluster+rdma")) {
+        gconf->transport = g_strdup("rdma");
+    } else {
+        ret = -EINVAL;
+        goto out;
+    }
+
+    ret = parse_volume_options(gconf, uri->path);
+    if (ret < 0) {
+        goto out;
+    }
+
+    qp = query_params_parse(uri->query);
+    if (qp->n > 1 || (is_unix && !qp->n) || (!is_unix && qp->n)) {
+        ret = -EINVAL;
+        goto out;
+    }
+
+    if (is_unix) {
+        if (uri->server || uri->port) {
+            ret = -EINVAL;
+            goto out;
+        }
+        if (strcmp(qp->p[0].name, "socket")) {
+            ret = -EINVAL;
+            goto out;
+        }
+        gconf->server = g_strdup(qp->p[0].value);
+    } else {
+        gconf->server = g_strdup(uri->server);
+        gconf->port = uri->port;
+    }
+
+out:
+    if (qp) {
+        query_params_free(qp);
+    }
+    uri_free(uri);
+    return ret;
+}
+
+static struct glfs *qemu_gluster_init(GlusterConf *gconf, const char *filename)
+{
+    struct glfs *glfs = NULL;
+    int ret;
+    int old_errno;
+
+    ret = qemu_gluster_parseuri(gconf, filename);
+    if (ret < 0) {
+        error_report("Usage: file=gluster[+transport]://[server[:port]]/"
+            "volname/image[?socket=...]");
+        errno = -ret;
+        goto out;
+    }
+
+    glfs = glfs_new(gconf->volname);
+    if (!glfs) {
+        goto out;
+    }
+
+    ret = glfs_set_volfile_server(glfs, gconf->transport, gconf->server,
+            gconf->port);
+    if (ret < 0) {
+        goto out;
+    }
+
+    /*
+     * TODO: Use GF_LOG_ERROR instead of hard code value of 4 here when
+     * GlusterFS makes GF_LOG_* macros available to libgfapi users.
+     */
+    ret = glfs_set_logging(glfs, "-", 4);
+    if (ret < 0) {
+        goto out;
+    }
+
+    ret = glfs_init(glfs);
+    if (ret) {
+        error_report("Gluster connection failed for server=%s port=%d "
+             "volume=%s image=%s transport=%s\n", gconf->server, gconf->port,
+             gconf->volname, gconf->image, gconf->transport);
+        goto out;
+    }
+    return glfs;
+
+out:
+    if (glfs) {
+        old_errno = errno;
+        glfs_fini(glfs);
+        errno = old_errno;
+    }
+    return NULL;
+}
+
+static void qemu_gluster_complete_aio(GlusterAIOCB *acb, BDRVGlusterState *s)
+{
+    int ret;
+    bool *finished = acb->finished;
+    BlockDriverCompletionFunc *cb = acb->common.cb;
+    void *opaque = acb->common.opaque;
+
+    if (!acb->ret || acb->ret == acb->size) {
+        ret = 0; /* Success */
+    } else if (acb->ret < 0) {
+        ret = acb->ret; /* Read/Write failed */
+    } else {
+        ret = -EIO; /* Partial read/write - fail it */
+    }
+
+    s->qemu_aio_count--;
+    qemu_aio_release(acb);
+    cb(opaque, ret);
+    if (finished) {
+        *finished = true;
+    }
+}
+
+static void qemu_gluster_aio_event_reader(void *opaque)
+{
+    BDRVGlusterState *s = opaque;
+    ssize_t ret;
+
+    do {
+        char *p = (char *)&s->event_acb;
+
+        ret = read(s->fds[GLUSTER_FD_READ], p + s->event_reader_pos,
+                   sizeof(s->event_acb) - s->event_reader_pos);
+        if (ret > 0) {
+            s->event_reader_pos += ret;
+            if (s->event_reader_pos == sizeof(s->event_acb)) {
+                s->event_reader_pos = 0;
+                qemu_gluster_complete_aio(s->event_acb, s);
+            }
+        }
+    } while (ret < 0 && errno == EINTR);
+}
+
+static int qemu_gluster_aio_flush_cb(void *opaque)
+{
+    BDRVGlusterState *s = opaque;
+
+    return (s->qemu_aio_count > 0);
+}
+
+static int qemu_gluster_open(BlockDriverState *bs, const char *filename,
+    int bdrv_flags)
+{
+    BDRVGlusterState *s = bs->opaque;
+    int open_flags = O_BINARY;
+    int ret = 0;
+    GlusterConf *gconf = g_malloc0(sizeof(GlusterConf));
+
+    s->glfs = qemu_gluster_init(gconf, filename);
+    if (!s->glfs) {
+        ret = -errno;
+        goto out;
+    }
+
+    if (bdrv_flags & BDRV_O_RDWR) {
+        open_flags |= O_RDWR;
+    } else {
+        open_flags |= O_RDONLY;
+    }
+
+    if ((bdrv_flags & BDRV_O_NOCACHE)) {
+        open_flags |= O_DIRECT;
+    }
+
+    s->fd = glfs_open(s->glfs, gconf->image, open_flags);
+    if (!s->fd) {
+        ret = -errno;
+        goto out;
+    }
+
+    ret = qemu_pipe(s->fds);
+    if (ret < 0) {
+        ret = -errno;
+        goto out;
+    }
+    fcntl(s->fds[GLUSTER_FD_READ], F_SETFL, O_NONBLOCK);
+    qemu_aio_set_fd_handler(s->fds[GLUSTER_FD_READ],
+        qemu_gluster_aio_event_reader, NULL, qemu_gluster_aio_flush_cb, s);
+
+out:
+    qemu_gluster_gconf_free(gconf);
+    if (!ret) {
+        return ret;
+    }
+    if (s->fd) {
+        glfs_close(s->fd);
+    }
+    if (s->glfs) {
+        glfs_fini(s->glfs);
+    }
+    return ret;
+}
+
+static int qemu_gluster_create(const char *filename,
+        QEMUOptionParameter *options)
+{
+    struct glfs *glfs;
+    struct glfs_fd *fd;
+    int ret = 0;
+    int64_t total_size = 0;
+    GlusterConf *gconf = g_malloc0(sizeof(GlusterConf));
+
+    glfs = qemu_gluster_init(gconf, filename);
+    if (!glfs) {
+        ret = -errno;
+        goto out;
+    }
+
+    while (options && options->name) {
+        if (!strcmp(options->name, BLOCK_OPT_SIZE)) {
+            total_size = options->value.n / BDRV_SECTOR_SIZE;
+        }
+        options++;
+    }
+
+    fd = glfs_creat(glfs, gconf->image,
+        O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, S_IRUSR | S_IWUSR);
+    if (!fd) {
+        ret = -errno;
+    } else {
+        if (glfs_ftruncate(fd, total_size * BDRV_SECTOR_SIZE) != 0) {
+            ret = -errno;
+        }
+        if (glfs_close(fd) != 0) {
+            ret = -errno;
+        }
+    }
+out:
+    qemu_gluster_gconf_free(gconf);
+    if (glfs) {
+        glfs_fini(glfs);
+    }
+    return ret;
+}
+
+static void qemu_gluster_aio_cancel(BlockDriverAIOCB *blockacb)
+{
+    GlusterAIOCB *acb = (GlusterAIOCB *)blockacb;
+    bool finished = false;
+
+    acb->finished = &finished;
+    while (!finished) {
+        qemu_aio_wait();
+    }
+}
+
+static const AIOCBInfo gluster_aiocb_info = {
+    .aiocb_size = sizeof(GlusterAIOCB),
+    .cancel = qemu_gluster_aio_cancel,
+};
+
+static void gluster_finish_aiocb(struct glfs_fd *fd, ssize_t ret, void *arg)
+{
+    GlusterAIOCB *acb = (GlusterAIOCB *)arg;
+    BlockDriverState *bs = acb->common.bs;
+    BDRVGlusterState *s = bs->opaque;
+    int retval;
+
+    acb->ret = ret;
+    retval = qemu_write_full(s->fds[GLUSTER_FD_WRITE], &acb, sizeof(acb));
+    if (retval != sizeof(acb)) {
+        /*
+         * Gluster AIO callback thread failed to notify the waiting
+         * QEMU thread about IO completion.
+         *
+         * Complete this IO request and make the disk inaccessible for
+         * subsequent reads and writes.
+         */
+        error_report("Gluster failed to notify QEMU about IO completion");
+
+        qemu_mutex_lock_iothread(); /* We are in gluster thread context */
+        acb->common.cb(acb->common.opaque, -EIO);
+        qemu_aio_release(acb);
+        s->qemu_aio_count--;
+        close(s->fds[GLUSTER_FD_READ]);
+        close(s->fds[GLUSTER_FD_WRITE]);
+        qemu_aio_set_fd_handler(s->fds[GLUSTER_FD_READ], NULL, NULL, NULL,
+            NULL);
+        bs->drv = NULL; /* Make the disk inaccessible */
+        qemu_mutex_unlock_iothread();
+    }
+}
+
+static BlockDriverAIOCB *qemu_gluster_aio_rw(BlockDriverState *bs,
+        int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
+        BlockDriverCompletionFunc *cb, void *opaque, int write)
+{
+    int ret;
+    GlusterAIOCB *acb;
+    BDRVGlusterState *s = bs->opaque;
+    size_t size;
+    off_t offset;
+
+    offset = sector_num * BDRV_SECTOR_SIZE;
+    size = nb_sectors * BDRV_SECTOR_SIZE;
+    s->qemu_aio_count++;
+
+    acb = qemu_aio_get(&gluster_aiocb_info, bs, cb, opaque);
+    acb->size = size;
+    acb->ret = 0;
+    acb->finished = NULL;
+
+    if (write) {
+        ret = glfs_pwritev_async(s->fd, qiov->iov, qiov->niov, offset, 0,
+            &gluster_finish_aiocb, acb);
+    } else {
+        ret = glfs_preadv_async(s->fd, qiov->iov, qiov->niov, offset, 0,
+            &gluster_finish_aiocb, acb);
+    }
+
+    if (ret < 0) {
+        goto out;
+    }
+    return &acb->common;
+
+out:
+    s->qemu_aio_count--;
+    qemu_aio_release(acb);
+    return NULL;
+}
+
+static BlockDriverAIOCB *qemu_gluster_aio_readv(BlockDriverState *bs,
+        int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
+        BlockDriverCompletionFunc *cb, void *opaque)
+{
+    return qemu_gluster_aio_rw(bs, sector_num, qiov, nb_sectors, cb, opaque, 0);
+}
+
+static BlockDriverAIOCB *qemu_gluster_aio_writev(BlockDriverState *bs,
+        int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
+        BlockDriverCompletionFunc *cb, void *opaque)
+{
+    return qemu_gluster_aio_rw(bs, sector_num, qiov, nb_sectors, cb, opaque, 1);
+}
+
+static BlockDriverAIOCB *qemu_gluster_aio_flush(BlockDriverState *bs,
+        BlockDriverCompletionFunc *cb, void *opaque)
+{
+    int ret;
+    GlusterAIOCB *acb;
+    BDRVGlusterState *s = bs->opaque;
+
+    acb = qemu_aio_get(&gluster_aiocb_info, bs, cb, opaque);
+    acb->size = 0;
+    acb->ret = 0;
+    acb->finished = NULL;
+    s->qemu_aio_count++;
+
+    ret = glfs_fsync_async(s->fd, &gluster_finish_aiocb, acb);
+    if (ret < 0) {
+        goto out;
+    }
+    return &acb->common;
+
+out:
+    s->qemu_aio_count--;
+    qemu_aio_release(acb);
+    return NULL;
+}
+
+static int64_t qemu_gluster_getlength(BlockDriverState *bs)
+{
+    BDRVGlusterState *s = bs->opaque;
+    int64_t ret;
+
+    ret = glfs_lseek(s->fd, 0, SEEK_END);
+    if (ret < 0) {
+        return -errno;
+    } else {
+        return ret;
+    }
+}
+
+static int64_t qemu_gluster_allocated_file_size(BlockDriverState *bs)
+{
+    BDRVGlusterState *s = bs->opaque;
+    struct stat st;
+    int ret;
+
+    ret = glfs_fstat(s->fd, &st);
+    if (ret < 0) {
+        return -errno;
+    } else {
+        return st.st_blocks * 512;
+    }
+}
+
+static void qemu_gluster_close(BlockDriverState *bs)
+{
+    BDRVGlusterState *s = bs->opaque;
+
+    close(s->fds[GLUSTER_FD_READ]);
+    close(s->fds[GLUSTER_FD_WRITE]);
+    qemu_aio_set_fd_handler(s->fds[GLUSTER_FD_READ], NULL, NULL, NULL, NULL);
+
+    if (s->fd) {
+        glfs_close(s->fd);
+        s->fd = NULL;
+    }
+    glfs_fini(s->glfs);
+}
+
+static QEMUOptionParameter qemu_gluster_create_options[] = {
+    {
+        .name = BLOCK_OPT_SIZE,
+        .type = OPT_SIZE,
+        .help = "Virtual disk size"
+    },
+    { NULL }
+};
+
+static BlockDriver bdrv_gluster = {
+    .format_name                  = "gluster",
+    .protocol_name                = "gluster",
+    .instance_size                = sizeof(BDRVGlusterState),
+    .bdrv_file_open               = qemu_gluster_open,
+    .bdrv_close                   = qemu_gluster_close,
+    .bdrv_create                  = qemu_gluster_create,
+    .bdrv_getlength               = qemu_gluster_getlength,
+    .bdrv_get_allocated_file_size = qemu_gluster_allocated_file_size,
+    .bdrv_aio_readv               = qemu_gluster_aio_readv,
+    .bdrv_aio_writev              = qemu_gluster_aio_writev,
+    .bdrv_aio_flush               = qemu_gluster_aio_flush,
+    .create_options               = qemu_gluster_create_options,
+};
+
+static BlockDriver bdrv_gluster_tcp = {
+    .format_name                  = "gluster",
+    .protocol_name                = "gluster+tcp",
+    .instance_size                = sizeof(BDRVGlusterState),
+    .bdrv_file_open               = qemu_gluster_open,
+    .bdrv_close                   = qemu_gluster_close,
+    .bdrv_create                  = qemu_gluster_create,
+    .bdrv_getlength               = qemu_gluster_getlength,
+    .bdrv_get_allocated_file_size = qemu_gluster_allocated_file_size,
+    .bdrv_aio_readv               = qemu_gluster_aio_readv,
+    .bdrv_aio_writev              = qemu_gluster_aio_writev,
+    .bdrv_aio_flush               = qemu_gluster_aio_flush,
+    .create_options               = qemu_gluster_create_options,
+};
+
+static BlockDriver bdrv_gluster_unix = {
+    .format_name                  = "gluster",
+    .protocol_name                = "gluster+unix",
+    .instance_size                = sizeof(BDRVGlusterState),
+    .bdrv_file_open               = qemu_gluster_open,
+    .bdrv_close                   = qemu_gluster_close,
+    .bdrv_create                  = qemu_gluster_create,
+    .bdrv_getlength               = qemu_gluster_getlength,
+    .bdrv_get_allocated_file_size = qemu_gluster_allocated_file_size,
+    .bdrv_aio_readv               = qemu_gluster_aio_readv,
+    .bdrv_aio_writev              = qemu_gluster_aio_writev,
+    .bdrv_aio_flush               = qemu_gluster_aio_flush,
+    .create_options               = qemu_gluster_create_options,
+};
+
+static BlockDriver bdrv_gluster_rdma = {
+    .format_name                  = "gluster",
+    .protocol_name                = "gluster+rdma",
+    .instance_size                = sizeof(BDRVGlusterState),
+    .bdrv_file_open               = qemu_gluster_open,
+    .bdrv_close                   = qemu_gluster_close,
+    .bdrv_create                  = qemu_gluster_create,
+    .bdrv_getlength               = qemu_gluster_getlength,
+    .bdrv_get_allocated_file_size = qemu_gluster_allocated_file_size,
+    .bdrv_aio_readv               = qemu_gluster_aio_readv,
+    .bdrv_aio_writev              = qemu_gluster_aio_writev,
+    .bdrv_aio_flush               = qemu_gluster_aio_flush,
+    .create_options               = qemu_gluster_create_options,
+};
+
+static void bdrv_gluster_init(void)
+{
+    bdrv_register(&bdrv_gluster_rdma);
+    bdrv_register(&bdrv_gluster_unix);
+    bdrv_register(&bdrv_gluster_tcp);
+    bdrv_register(&bdrv_gluster);
+}
+
+block_init(bdrv_gluster_init);
diff --git a/block/iscsi.c b/block/iscsi.c
index 0b96165ec..c0b70b3d3 100644
--- a/block/iscsi.c
+++ b/block/iscsi.c
@@ -65,13 +65,6 @@ typedef struct IscsiAIOCB {
 #endif
 } IscsiAIOCB;
 
-struct IscsiTask {
-    IscsiLun *iscsilun;
-    BlockDriverState *bs;
-    int status;
-    int complete;
-};
-
 static void
 iscsi_bh_cb(void *p)
 {
@@ -133,7 +126,7 @@ iscsi_aio_cancel(BlockDriverAIOCB *blockacb)
     }
 }
 
-static AIOPool iscsi_aio_pool = {
+static const AIOCBInfo iscsi_aiocb_info = {
     .aiocb_size         = sizeof(IscsiAIOCB),
     .cancel             = iscsi_aio_cancel,
 };
@@ -167,12 +160,6 @@ iscsi_set_events(IscsiLun *iscsilun)
 
     }
 
-    /* If we just added an event, the callback might be delayed
-     * unless we call qemu_notify_event().
-     */
-    if (ev & ~iscsilun->events) {
-        qemu_notify_event();
-    }
     iscsilun->events = ev;
 }
 
@@ -240,7 +227,7 @@ iscsi_aio_writev(BlockDriverState *bs, int64_t sector_num,
     uint64_t lba;
     struct iscsi_data data;
 
-    acb = qemu_aio_get(&iscsi_aio_pool, bs, cb, opaque);
+    acb = qemu_aio_get(&iscsi_aiocb_info, bs, cb, opaque);
     trace_iscsi_aio_writev(iscsi, sector_num, nb_sectors, opaque, acb);
 
     acb->iscsilun = iscsilun;
@@ -268,10 +255,6 @@ iscsi_aio_writev(BlockDriverState *bs, int64_t sector_num,
     acb->task->xfer_dir = SCSI_XFER_WRITE;
     acb->task->cdb_size = 16;
     acb->task->cdb[0] = 0x8a;
-    if (!(bs->open_flags & BDRV_O_CACHE_WB)) {
-        /* set FUA on writes when cache mode is write through */
-        acb->task->cdb[1] |= 0x04;
-    }
     lba = sector_qemu2lun(sector_num, iscsilun);
     *(uint32_t *)&acb->task->cdb[2]  = htonl(lba >> 32);
     *(uint32_t *)&acb->task->cdb[6]  = htonl(lba & 0xffffffff);
@@ -335,7 +318,7 @@ iscsi_aio_readv(BlockDriverState *bs, int64_t sector_num,
 
     qemu_read_size = BDRV_SECTOR_SIZE * (size_t)nb_sectors;
 
-    acb = qemu_aio_get(&iscsi_aio_pool, bs, cb, opaque);
+    acb = qemu_aio_get(&iscsi_aiocb_info, bs, cb, opaque);
     trace_iscsi_aio_readv(iscsi, sector_num, nb_sectors, opaque, acb);
 
     acb->iscsilun = iscsilun;
@@ -390,7 +373,7 @@ iscsi_aio_readv(BlockDriverState *bs, int64_t sector_num,
         *(uint16_t *)&acb->task->cdb[7] = htons(num_sectors);
         break;
     }
-    
+
     if (iscsi_scsi_command_async(iscsi, iscsilun->lun, acb->task,
                                  iscsi_aio_read16_cb,
                                  NULL,
@@ -440,7 +423,7 @@ iscsi_aio_flush(BlockDriverState *bs,
     struct iscsi_context *iscsi = iscsilun->iscsi;
     IscsiAIOCB *acb;
 
-    acb = qemu_aio_get(&iscsi_aio_pool, bs, cb, opaque);
+    acb = qemu_aio_get(&iscsi_aiocb_info, bs, cb, opaque);
 
     acb->iscsilun = iscsilun;
     acb->canceled   = 0;
@@ -493,7 +476,7 @@ iscsi_aio_discard(BlockDriverState *bs,
     IscsiAIOCB *acb;
     struct unmap_list list[1];
 
-    acb = qemu_aio_get(&iscsi_aio_pool, bs, cb, opaque);
+    acb = qemu_aio_get(&iscsi_aiocb_info, bs, cb, opaque);
 
     acb->iscsilun = iscsilun;
     acb->canceled   = 0;
@@ -568,7 +551,7 @@ static BlockDriverAIOCB *iscsi_aio_ioctl(BlockDriverState *bs,
 
     assert(req == SG_IO);
 
-    acb = qemu_aio_get(&iscsi_aio_pool, bs, cb, opaque);
+    acb = qemu_aio_get(&iscsi_aiocb_info, bs, cb, opaque);
 
     acb->iscsilun = iscsilun;
     acb->canceled    = 0;
@@ -628,9 +611,17 @@ static BlockDriverAIOCB *iscsi_aio_ioctl(BlockDriverState *bs,
     return &acb->common;
 }
 
+
+static void ioctl_cb(void *opaque, int status)
+{
+    int *p_status = opaque;
+    *p_status = status;
+}
+
 static int iscsi_ioctl(BlockDriverState *bs, unsigned long int req, void *buf)
 {
     IscsiLun *iscsilun = bs->opaque;
+    int status;
 
     switch (req) {
     case SG_GET_VERSION_NUM:
@@ -639,6 +630,15 @@ static int iscsi_ioctl(BlockDriverState *bs, unsigned long int req, void *buf)
     case SG_GET_SCSI_ID:
         ((struct sg_scsi_id *)buf)->scsi_type = iscsilun->type;
         break;
+    case SG_IO:
+        status = -EINPROGRESS;
+        iscsi_aio_ioctl(bs, req, buf, ioctl_cb, &status);
+
+        while (status == -EINPROGRESS) {
+            qemu_aio_wait();
+        }
+
+        return 0;
     default:
         return -1;
     }
@@ -658,163 +658,6 @@ iscsi_getlength(BlockDriverState *bs)
     return len;
 }
 
-static void
-iscsi_readcapacity16_cb(struct iscsi_context *iscsi, int status,
-                        void *command_data, void *opaque)
-{
-    struct IscsiTask *itask = opaque;
-    struct scsi_readcapacity16 *rc16;
-    struct scsi_task *task = command_data;
-
-    if (status != 0) {
-        error_report("iSCSI: Failed to read capacity of iSCSI lun. %s",
-                     iscsi_get_error(iscsi));
-        itask->status   = 1;
-        itask->complete = 1;
-        scsi_free_scsi_task(task);
-        return;
-    }
-
-    rc16 = scsi_datain_unmarshall(task);
-    if (rc16 == NULL) {
-        error_report("iSCSI: Failed to unmarshall readcapacity16 data.");
-        itask->status   = 1;
-        itask->complete = 1;
-        scsi_free_scsi_task(task);
-        return;
-    }
-
-    itask->iscsilun->block_size = rc16->block_length;
-    itask->iscsilun->num_blocks = rc16->returned_lba + 1;
-    itask->bs->total_sectors    = itask->iscsilun->num_blocks *
-                               itask->iscsilun->block_size / BDRV_SECTOR_SIZE ;
-
-    itask->status   = 0;
-    itask->complete = 1;
-    scsi_free_scsi_task(task);
-}
-
-static void
-iscsi_readcapacity10_cb(struct iscsi_context *iscsi, int status,
-                        void *command_data, void *opaque)
-{
-    struct IscsiTask *itask = opaque;
-    struct scsi_readcapacity10 *rc10;
-    struct scsi_task *task = command_data;
-
-    if (status != 0) {
-        error_report("iSCSI: Failed to read capacity of iSCSI lun. %s",
-                     iscsi_get_error(iscsi));
-        itask->status   = 1;
-        itask->complete = 1;
-        scsi_free_scsi_task(task);
-        return;
-    }
-
-    rc10 = scsi_datain_unmarshall(task);
-    if (rc10 == NULL) {
-        error_report("iSCSI: Failed to unmarshall readcapacity10 data.");
-        itask->status   = 1;
-        itask->complete = 1;
-        scsi_free_scsi_task(task);
-        return;
-    }
-
-    itask->iscsilun->block_size = rc10->block_size;
-    if (rc10->lba == 0) {
-        /* blank disk loaded */
-        itask->iscsilun->num_blocks = 0;
-    } else {
-        itask->iscsilun->num_blocks = rc10->lba + 1;
-    }
-    itask->bs->total_sectors    = itask->iscsilun->num_blocks *
-                               itask->iscsilun->block_size / BDRV_SECTOR_SIZE ;
-
-    itask->status   = 0;
-    itask->complete = 1;
-    scsi_free_scsi_task(task);
-}
-
-static void
-iscsi_inquiry_cb(struct iscsi_context *iscsi, int status, void *command_data,
-                 void *opaque)
-{
-    struct IscsiTask *itask = opaque;
-    struct scsi_task *task = command_data;
-    struct scsi_inquiry_standard *inq;
-
-    if (status != 0) {
-        itask->status   = 1;
-        itask->complete = 1;
-        scsi_free_scsi_task(task);
-        return;
-    }
-
-    inq = scsi_datain_unmarshall(task);
-    if (inq == NULL) {
-        error_report("iSCSI: Failed to unmarshall inquiry data.");
-        itask->status   = 1;
-        itask->complete = 1;
-        scsi_free_scsi_task(task);
-        return;
-    }
-
-    itask->iscsilun->type = inq->periperal_device_type;
-
-    scsi_free_scsi_task(task);
-
-    switch (itask->iscsilun->type) {
-    case TYPE_DISK:
-        task = iscsi_readcapacity16_task(iscsi, itask->iscsilun->lun,
-                                   iscsi_readcapacity16_cb, opaque);
-        if (task == NULL) {
-            error_report("iSCSI: failed to send readcapacity16 command.");
-            itask->status   = 1;
-            itask->complete = 1;
-            return;
-        }
-        break;
-    case TYPE_ROM:
-        task = iscsi_readcapacity10_task(iscsi, itask->iscsilun->lun,
-                                   0, 0,
-                                   iscsi_readcapacity10_cb, opaque);
-        if (task == NULL) {
-            error_report("iSCSI: failed to send readcapacity16 command.");
-            itask->status   = 1;
-            itask->complete = 1;
-            return;
-        }
-        break;
-    default:
-        itask->status   = 0;
-        itask->complete = 1;
-    }
-}
-
-static void
-iscsi_connect_cb(struct iscsi_context *iscsi, int status, void *command_data,
-                 void *opaque)
-{
-    struct IscsiTask *itask = opaque;
-    struct scsi_task *task;
-
-    if (status != 0) {
-        itask->status   = 1;
-        itask->complete = 1;
-        return;
-    }
-
-    task = iscsi_inquiry_task(iscsi, itask->iscsilun->lun,
-                              0, 0, 36,
-                              iscsi_inquiry_cb, opaque);
-    if (task == NULL) {
-        error_report("iSCSI: failed to send inquiry command.");
-        itask->status   = 1;
-        itask->complete = 1;
-        return;
-    }
-}
-
 static int parse_chap(struct iscsi_context *iscsi, const char *target)
 {
     QemuOptsList *list;
@@ -927,7 +770,10 @@ static int iscsi_open(BlockDriverState *bs, const char *filename, int flags)
     IscsiLun *iscsilun = bs->opaque;
     struct iscsi_context *iscsi = NULL;
     struct iscsi_url *iscsi_url = NULL;
-    struct IscsiTask task;
+    struct scsi_task *task = NULL;
+    struct scsi_inquiry_standard *inq = NULL;
+    struct scsi_readcapacity10 *rc10 = NULL;
+    struct scsi_readcapacity16 *rc16 = NULL;
     char *initiator_name = NULL;
     int ret;
 
@@ -940,8 +786,7 @@ static int iscsi_open(BlockDriverState *bs, const char *filename, int flags)
 
     iscsi_url = iscsi_parse_full_url(iscsi, filename);
     if (iscsi_url == NULL) {
-        error_report("Failed to parse URL : %s %s", filename,
-                     iscsi_get_error(iscsi));
+        error_report("Failed to parse URL : %s", filename);
         ret = -EINVAL;
         goto out;
     }
@@ -991,33 +836,80 @@ static int iscsi_open(BlockDriverState *bs, const char *filename, int flags)
     /* check if we got HEADER_DIGEST via the options */
     parse_header_digest(iscsi, iscsi_url->target);
 
-    task.iscsilun = iscsilun;
-    task.status = 0;
-    task.complete = 0;
-    task.bs = bs;
+    if (iscsi_full_connect_sync(iscsi, iscsi_url->portal, iscsi_url->lun) != 0) {
+        error_report("iSCSI: Failed to connect to LUN : %s",
+            iscsi_get_error(iscsi));
+        ret = -EINVAL;
+        goto out;
+    }
 
     iscsilun->iscsi = iscsi;
     iscsilun->lun   = iscsi_url->lun;
 
-    if (iscsi_full_connect_async(iscsi, iscsi_url->portal, iscsi_url->lun,
-                                 iscsi_connect_cb, &task)
-        != 0) {
-        error_report("iSCSI: Failed to start async connect.");
+    task = iscsi_inquiry_sync(iscsi, iscsilun->lun, 0, 0, 36);
+
+    if (task == NULL || task->status != SCSI_STATUS_GOOD) {
+        error_report("iSCSI: failed to send inquiry command.");
         ret = -EINVAL;
         goto out;
     }
 
-    while (!task.complete) {
-        iscsi_set_events(iscsilun);
-        qemu_aio_wait();
-    }
-    if (task.status != 0) {
-        error_report("iSCSI: Failed to connect to LUN : %s",
-                     iscsi_get_error(iscsi));
+    inq = scsi_datain_unmarshall(task);
+    if (inq == NULL) {
+        error_report("iSCSI: Failed to unmarshall inquiry data.");
         ret = -EINVAL;
         goto out;
     }
 
+    iscsilun->type = inq->periperal_device_type;
+
+    scsi_free_scsi_task(task);
+
+    switch (iscsilun->type) {
+    case TYPE_DISK:
+        task = iscsi_readcapacity16_sync(iscsi, iscsilun->lun);
+        if (task == NULL || task->status != SCSI_STATUS_GOOD) {
+            error_report("iSCSI: failed to send readcapacity16 command.");
+            ret = -EINVAL;
+            goto out;
+        }
+        rc16 = scsi_datain_unmarshall(task);
+        if (rc16 == NULL) {
+            error_report("iSCSI: Failed to unmarshall readcapacity16 data.");
+            ret = -EINVAL;
+            goto out;
+        }
+        iscsilun->block_size = rc16->block_length;
+        iscsilun->num_blocks = rc16->returned_lba + 1;
+        break;
+    case TYPE_ROM:
+        task = iscsi_readcapacity10_sync(iscsi, iscsilun->lun, 0, 0);
+        if (task == NULL || task->status != SCSI_STATUS_GOOD) {
+            error_report("iSCSI: failed to send readcapacity10 command.");
+            ret = -EINVAL;
+            goto out;
+        }
+        rc10 = scsi_datain_unmarshall(task);
+        if (rc10 == NULL) {
+            error_report("iSCSI: Failed to unmarshall readcapacity10 data.");
+            ret = -EINVAL;
+            goto out;
+        }
+        iscsilun->block_size = rc10->block_size;
+        if (rc10->lba == 0) {
+            /* blank disk loaded */
+            iscsilun->num_blocks = 0;
+        } else {
+            iscsilun->num_blocks = rc10->lba + 1;
+        }
+        break;
+    default:
+        break;
+    }
+
+    bs->total_sectors    = iscsilun->num_blocks *
+                           iscsilun->block_size / BDRV_SECTOR_SIZE ;
+
     /* Medium changer or tape. We dont have any emulation for this so this must
      * be sg ioctl compatible. We force it to be sg, otherwise qemu will try
      * to read from the device to guess the image format.
@@ -1036,6 +928,9 @@ out:
     if (iscsi_url != NULL) {
         iscsi_destroy_url(iscsi_url);
     }
+    if (task != NULL) {
+        scsi_free_scsi_task(task);
+    }
 
     if (ret) {
         if (iscsi != NULL) {
@@ -1056,6 +951,11 @@ static void iscsi_close(BlockDriverState *bs)
     memset(iscsilun, 0, sizeof(IscsiLun));
 }
 
+static int iscsi_has_zero_init(BlockDriverState *bs)
+{
+    return 0;
+}
+
 static BlockDriver bdrv_iscsi = {
     .format_name     = "iscsi",
     .protocol_name   = "iscsi",
@@ -1071,6 +971,7 @@ static BlockDriver bdrv_iscsi = {
     .bdrv_aio_flush  = iscsi_aio_flush,
 
     .bdrv_aio_discard = iscsi_aio_discard,
+    .bdrv_has_zero_init = iscsi_has_zero_init,
 
 #ifdef __linux__
     .bdrv_ioctl       = iscsi_ioctl,
diff --git a/linux-aio.c b/block/linux-aio.c
index ce9b5d4be..91ef86324 100644
--- a/linux-aio.c
+++ b/block/linux-aio.c
@@ -9,9 +9,10 @@
  */
 #include "qemu-common.h"
 #include "qemu-aio.h"
-#include "block/raw-posix-aio.h"
+#include "qemu-queue.h"
+#include "block/raw-aio.h"
+#include "event_notifier.h"
 
-#include <sys/eventfd.h>
 #include <libaio.h>
 
 /*
@@ -37,7 +38,7 @@ struct qemu_laiocb {
 
 struct qemu_laio_state {
     io_context_t ctx;
-    int efd;
+    EventNotifier e;
     int count;
 };
 
@@ -76,29 +77,17 @@ static void qemu_laio_process_completion(struct qemu_laio_state *s,
     qemu_aio_release(laiocb);
 }
 
-static void qemu_laio_completion_cb(void *opaque)
+static void qemu_laio_completion_cb(EventNotifier *e)
 {
-    struct qemu_laio_state *s = opaque;
+    struct qemu_laio_state *s = container_of(e, struct qemu_laio_state, e);
 
-    while (1) {
+    while (event_notifier_test_and_clear(&s->e)) {
         struct io_event events[MAX_EVENTS];
-        uint64_t val;
-        ssize_t ret;
         struct timespec ts = { 0 };
         int nevents, i;
 
         do {
-            ret = read(s->efd, &val, sizeof(val));
-        } while (ret == -1 && errno == EINTR);
-
-        if (ret == -1 && errno == EAGAIN)
-            break;
-
-        if (ret != 8)
-            break;
-
-        do {
-            nevents = io_getevents(s->ctx, val, MAX_EVENTS, events, &ts);
+            nevents = io_getevents(s->ctx, MAX_EVENTS, MAX_EVENTS, events, &ts);
         } while (nevents == -EINTR);
 
         for (i = 0; i < nevents; i++) {
@@ -112,9 +101,9 @@ static void qemu_laio_completion_cb(void *opaque)
     }
 }
 
-static int qemu_laio_flush_cb(void *opaque)
+static int qemu_laio_flush_cb(EventNotifier *e)
 {
-    struct qemu_laio_state *s = opaque;
+    struct qemu_laio_state *s = container_of(e, struct qemu_laio_state, e);
 
     return (s->count > 0) ? 1 : 0;
 }
@@ -146,11 +135,12 @@ static void laio_cancel(BlockDriverAIOCB *blockacb)
      * We might be able to do this slightly more optimal by removing the
      * O_NONBLOCK flag.
      */
-    while (laiocb->ret == -EINPROGRESS)
-        qemu_laio_completion_cb(laiocb->ctx);
+    while (laiocb->ret == -EINPROGRESS) {
+        qemu_laio_completion_cb(&laiocb->ctx->e);
+    }
 }
 
-static AIOPool laio_pool = {
+static const AIOCBInfo laio_aiocb_info = {
     .aiocb_size         = sizeof(struct qemu_laiocb),
     .cancel             = laio_cancel,
 };
@@ -164,7 +154,7 @@ BlockDriverAIOCB *laio_submit(BlockDriverState *bs, void *aio_ctx, int fd,
     struct iocb *iocbs;
     off_t offset = sector_num * 512;
 
-    laiocb = qemu_aio_get(&laio_pool, bs, cb, opaque);
+    laiocb = qemu_aio_get(&laio_aiocb_info, bs, cb, opaque);
     laiocb->nbytes = nb_sectors * 512;
     laiocb->ctx = s;
     laiocb->ret = -EINPROGRESS;
@@ -186,7 +176,7 @@ BlockDriverAIOCB *laio_submit(BlockDriverState *bs, void *aio_ctx, int fd,
                         __func__, type);
         goto out_free_aiocb;
     }
-    io_set_eventfd(&laiocb->iocb, s->efd);
+    io_set_eventfd(&laiocb->iocb, event_notifier_get_fd(&s->e));
     s->count++;
 
     if (io_submit(s->ctx, 1, &iocbs) < 0)
@@ -205,21 +195,21 @@ void *laio_init(void)
     struct qemu_laio_state *s;
 
     s = g_malloc0(sizeof(*s));
-    s->efd = eventfd(0, 0);
-    if (s->efd == -1)
+    if (event_notifier_init(&s->e, false) < 0) {
         goto out_free_state;
-    fcntl(s->efd, F_SETFL, O_NONBLOCK);
+    }
 
-    if (io_setup(MAX_EVENTS, &s->ctx) != 0)
+    if (io_setup(MAX_EVENTS, &s->ctx) != 0) {
         goto out_close_efd;
+    }
 
-    qemu_aio_set_fd_handler(s->efd, qemu_laio_completion_cb, NULL,
-        qemu_laio_flush_cb, s);
+    qemu_aio_set_event_notifier(&s->e, qemu_laio_completion_cb,
+                                qemu_laio_flush_cb);
 
     return s;
 
 out_close_efd:
-    close(s->efd);
+    event_notifier_cleanup(&s->e);
 out_free_state:
     g_free(s);
     return NULL;
diff --git a/block/mirror.c b/block/mirror.c
new file mode 100644
index 000000000..d6618a4b3
--- /dev/null
+++ b/block/mirror.c
@@ -0,0 +1,322 @@
+/*
+ * Image mirroring
+ *
+ * Copyright Red Hat, Inc. 2012
+ *
+ * Authors:
+ *  Paolo Bonzini  <pbonzini@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ *
+ */
+
+#include "trace.h"
+#include "blockjob.h"
+#include "block_int.h"
+#include "qemu/ratelimit.h"
+
+enum {
+    /*
+     * Size of data buffer for populating the image file.  This should be large
+     * enough to process multiple clusters in a single call, so that populating
+     * contiguous regions of the image is efficient.
+     */
+    BLOCK_SIZE = 512 * BDRV_SECTORS_PER_DIRTY_CHUNK, /* in bytes */
+};
+
+#define SLICE_TIME 100000000ULL /* ns */
+
+typedef struct MirrorBlockJob {
+    BlockJob common;
+    RateLimit limit;
+    BlockDriverState *target;
+    MirrorSyncMode mode;
+    BlockdevOnError on_source_error, on_target_error;
+    bool synced;
+    bool should_complete;
+    int64_t sector_num;
+    uint8_t *buf;
+} MirrorBlockJob;
+
+static BlockErrorAction mirror_error_action(MirrorBlockJob *s, bool read,
+                                            int error)
+{
+    s->synced = false;
+    if (read) {
+        return block_job_error_action(&s->common, s->common.bs,
+                                      s->on_source_error, true, error);
+    } else {
+        return block_job_error_action(&s->common, s->target,
+                                      s->on_target_error, false, error);
+    }
+}
+
+static int coroutine_fn mirror_iteration(MirrorBlockJob *s,
+                                         BlockErrorAction *p_action)
+{
+    BlockDriverState *source = s->common.bs;
+    BlockDriverState *target = s->target;
+    QEMUIOVector qiov;
+    int ret, nb_sectors;
+    int64_t end;
+    struct iovec iov;
+
+    end = s->common.len >> BDRV_SECTOR_BITS;
+    s->sector_num = bdrv_get_next_dirty(source, s->sector_num);
+    nb_sectors = MIN(BDRV_SECTORS_PER_DIRTY_CHUNK, end - s->sector_num);
+    bdrv_reset_dirty(source, s->sector_num, nb_sectors);
+
+    /* Copy the dirty cluster.  */
+    iov.iov_base = s->buf;
+    iov.iov_len  = nb_sectors * 512;
+    qemu_iovec_init_external(&qiov, &iov, 1);
+
+    trace_mirror_one_iteration(s, s->sector_num, nb_sectors);
+    ret = bdrv_co_readv(source, s->sector_num, nb_sectors, &qiov);
+    if (ret < 0) {
+        *p_action = mirror_error_action(s, true, -ret);
+        goto fail;
+    }
+    ret = bdrv_co_writev(target, s->sector_num, nb_sectors, &qiov);
+    if (ret < 0) {
+        *p_action = mirror_error_action(s, false, -ret);
+        s->synced = false;
+        goto fail;
+    }
+    return 0;
+
+fail:
+    /* Try again later.  */
+    bdrv_set_dirty(source, s->sector_num, nb_sectors);
+    return ret;
+}
+
+static void coroutine_fn mirror_run(void *opaque)
+{
+    MirrorBlockJob *s = opaque;
+    BlockDriverState *bs = s->common.bs;
+    int64_t sector_num, end;
+    int ret = 0;
+    int n;
+
+    if (block_job_is_cancelled(&s->common)) {
+        goto immediate_exit;
+    }
+
+    s->common.len = bdrv_getlength(bs);
+    if (s->common.len < 0) {
+        block_job_completed(&s->common, s->common.len);
+        return;
+    }
+
+    end = s->common.len >> BDRV_SECTOR_BITS;
+    s->buf = qemu_blockalign(bs, BLOCK_SIZE);
+
+    if (s->mode != MIRROR_SYNC_MODE_NONE) {
+        /* First part, loop on the sectors and initialize the dirty bitmap.  */
+        BlockDriverState *base;
+        base = s->mode == MIRROR_SYNC_MODE_FULL ? NULL : bs->backing_hd;
+        for (sector_num = 0; sector_num < end; ) {
+            int64_t next = (sector_num | (BDRV_SECTORS_PER_DIRTY_CHUNK - 1)) + 1;
+            ret = bdrv_co_is_allocated_above(bs, base,
+                                             sector_num, next - sector_num, &n);
+
+            if (ret < 0) {
+                goto immediate_exit;
+            }
+
+            assert(n > 0);
+            if (ret == 1) {
+                bdrv_set_dirty(bs, sector_num, n);
+                sector_num = next;
+            } else {
+                sector_num += n;
+            }
+        }
+    }
+
+    s->sector_num = -1;
+    for (;;) {
+        uint64_t delay_ns;
+        int64_t cnt;
+        bool should_complete;
+
+        cnt = bdrv_get_dirty_count(bs);
+        if (cnt != 0) {
+            BlockErrorAction action = BDRV_ACTION_REPORT;
+            ret = mirror_iteration(s, &action);
+            if (ret < 0 && action == BDRV_ACTION_REPORT) {
+                goto immediate_exit;
+            }
+            cnt = bdrv_get_dirty_count(bs);
+        }
+
+        should_complete = false;
+        if (cnt == 0) {
+            trace_mirror_before_flush(s);
+            ret = bdrv_flush(s->target);
+            if (ret < 0) {
+                if (mirror_error_action(s, false, -ret) == BDRV_ACTION_REPORT) {
+                    goto immediate_exit;
+                }
+            } else {
+                /* We're out of the streaming phase.  From now on, if the job
+                 * is cancelled we will actually complete all pending I/O and
+                 * report completion.  This way, block-job-cancel will leave
+                 * the target in a consistent state.
+                 */
+                s->common.offset = end * BDRV_SECTOR_SIZE;
+                if (!s->synced) {
+                    block_job_ready(&s->common);
+                    s->synced = true;
+                }
+
+                should_complete = s->should_complete ||
+                    block_job_is_cancelled(&s->common);
+                cnt = bdrv_get_dirty_count(bs);
+            }
+        }
+
+        if (cnt == 0 && should_complete) {
+            /* The dirty bitmap is not updated while operations are pending.
+             * If we're about to exit, wait for pending operations before
+             * calling bdrv_get_dirty_count(bs), or we may exit while the
+             * source has dirty data to copy!
+             *
+             * Note that I/O can be submitted by the guest while
+             * mirror_populate runs.
+             */
+            trace_mirror_before_drain(s, cnt);
+            bdrv_drain_all();
+            cnt = bdrv_get_dirty_count(bs);
+        }
+
+        ret = 0;
+        trace_mirror_before_sleep(s, cnt, s->synced);
+        if (!s->synced) {
+            /* Publish progress */
+            s->common.offset = end * BDRV_SECTOR_SIZE - cnt * BLOCK_SIZE;
+
+            if (s->common.speed) {
+                delay_ns = ratelimit_calculate_delay(&s->limit, BDRV_SECTORS_PER_DIRTY_CHUNK);
+            } else {
+                delay_ns = 0;
+            }
+
+            /* Note that even when no rate limit is applied we need to yield
+             * with no pending I/O here so that qemu_aio_flush() returns.
+             */
+            block_job_sleep_ns(&s->common, rt_clock, delay_ns);
+            if (block_job_is_cancelled(&s->common)) {
+                break;
+            }
+        } else if (!should_complete) {
+            delay_ns = (cnt == 0 ? SLICE_TIME : 0);
+            block_job_sleep_ns(&s->common, rt_clock, delay_ns);
+        } else if (cnt == 0) {
+            /* The two disks are in sync.  Exit and report successful
+             * completion.
+             */
+            assert(QLIST_EMPTY(&bs->tracked_requests));
+            s->common.cancelled = false;
+            break;
+        }
+    }
+
+immediate_exit:
+    g_free(s->buf);
+    bdrv_set_dirty_tracking(bs, false);
+    bdrv_iostatus_disable(s->target);
+    if (s->should_complete && ret == 0) {
+        if (bdrv_get_flags(s->target) != bdrv_get_flags(s->common.bs)) {
+            bdrv_reopen(s->target, bdrv_get_flags(s->common.bs), NULL);
+        }
+        bdrv_swap(s->target, s->common.bs);
+    }
+    bdrv_close(s->target);
+    bdrv_delete(s->target);
+    block_job_completed(&s->common, ret);
+}
+
+static void mirror_set_speed(BlockJob *job, int64_t speed, Error **errp)
+{
+    MirrorBlockJob *s = container_of(job, MirrorBlockJob, common);
+
+    if (speed < 0) {
+        error_set(errp, QERR_INVALID_PARAMETER, "speed");
+        return;
+    }
+    ratelimit_set_speed(&s->limit, speed / BDRV_SECTOR_SIZE, SLICE_TIME);
+}
+
+static void mirror_iostatus_reset(BlockJob *job)
+{
+    MirrorBlockJob *s = container_of(job, MirrorBlockJob, common);
+
+    bdrv_iostatus_reset(s->target);
+}
+
+static void mirror_complete(BlockJob *job, Error **errp)
+{
+    MirrorBlockJob *s = container_of(job, MirrorBlockJob, common);
+    int ret;
+
+    ret = bdrv_open_backing_file(s->target);
+    if (ret < 0) {
+        char backing_filename[PATH_MAX];
+        bdrv_get_full_backing_filename(s->target, backing_filename,
+                                       sizeof(backing_filename));
+        error_set(errp, QERR_OPEN_FILE_FAILED, backing_filename);
+        return;
+    }
+    if (!s->synced) {
+        error_set(errp, QERR_BLOCK_JOB_NOT_READY, job->bs->device_name);
+        return;
+    }
+
+    s->should_complete = true;
+    block_job_resume(job);
+}
+
+static BlockJobType mirror_job_type = {
+    .instance_size = sizeof(MirrorBlockJob),
+    .job_type      = "mirror",
+    .set_speed     = mirror_set_speed,
+    .iostatus_reset= mirror_iostatus_reset,
+    .complete      = mirror_complete,
+};
+
+void mirror_start(BlockDriverState *bs, BlockDriverState *target,
+                  int64_t speed, MirrorSyncMode mode,
+                  BlockdevOnError on_source_error,
+                  BlockdevOnError on_target_error,
+                  BlockDriverCompletionFunc *cb,
+                  void *opaque, Error **errp)
+{
+    MirrorBlockJob *s;
+
+    if ((on_source_error == BLOCKDEV_ON_ERROR_STOP ||
+         on_source_error == BLOCKDEV_ON_ERROR_ENOSPC) &&
+        !bdrv_iostatus_is_enabled(bs)) {
+        error_set(errp, QERR_INVALID_PARAMETER, "on-source-error");
+        return;
+    }
+
+    s = block_job_create(&mirror_job_type, bs, speed, cb, opaque, errp);
+    if (!s) {
+        return;
+    }
+
+    s->on_source_error = on_source_error;
+    s->on_target_error = on_target_error;
+    s->target = target;
+    s->mode = mode;
+    bdrv_set_dirty_tracking(bs, true);
+    bdrv_set_enable_write_cache(s->target, true);
+    bdrv_set_on_error(s->target, on_target_error, on_target_error);
+    bdrv_iostatus_enable(s->target);
+    s->common.co = qemu_coroutine_create(mirror_run);
+    trace_mirror_start(bs, s, s->common.co, opaque);
+    qemu_coroutine_enter(s->common.co, s);
+}
diff --git a/block/nbd.c b/block/nbd.c
index 2bce47bf7..e87c24817 100644
--- a/block/nbd.c
+++ b/block/nbd.c
@@ -28,6 +28,7 @@
 
 #include "qemu-common.h"
 #include "nbd.h"
+#include "uri.h"
 #include "block_int.h"
 #include "module.h"
 #include "qemu_socket.h"
@@ -55,7 +56,6 @@ typedef struct BDRVNBDState {
     uint32_t nbdflags;
     off_t size;
     size_t blocksize;
-    char *export_name; /* An NBD server may export several devices */
 
     CoMutex send_mutex;
     CoMutex free_sema;
@@ -65,13 +65,75 @@ typedef struct BDRVNBDState {
     Coroutine *recv_coroutine[MAX_NBD_REQUESTS];
     struct nbd_reply reply;
 
-    /* If it begins with  '/', this is a UNIX domain socket. Otherwise,
-     * it's a string of the form <hostname|ip4|\[ip6\]>:port
-     */
+    int is_unix;
     char *host_spec;
+    char *export_name; /* An NBD server may export several devices */
 } BDRVNBDState;
 
-static int nbd_config(BDRVNBDState *s, const char *filename, int flags)
+static int nbd_parse_uri(BDRVNBDState *s, const char *filename)
+{
+    URI *uri;
+    const char *p;
+    QueryParams *qp = NULL;
+    int ret = 0;
+
+    uri = uri_parse(filename);
+    if (!uri) {
+        return -EINVAL;
+    }
+
+    /* transport */
+    if (!strcmp(uri->scheme, "nbd")) {
+        s->is_unix = false;
+    } else if (!strcmp(uri->scheme, "nbd+tcp")) {
+        s->is_unix = false;
+    } else if (!strcmp(uri->scheme, "nbd+unix")) {
+        s->is_unix = true;
+    } else {
+        ret = -EINVAL;
+        goto out;
+    }
+
+    p = uri->path ? uri->path : "/";
+    p += strspn(p, "/");
+    if (p[0]) {
+        s->export_name = g_strdup(p);
+    }
+
+    qp = query_params_parse(uri->query);
+    if (qp->n > 1 || (s->is_unix && !qp->n) || (!s->is_unix && qp->n)) {
+        ret = -EINVAL;
+        goto out;
+    }
+
+    if (s->is_unix) {
+        /* nbd+unix:///export?socket=path */
+        if (uri->server || uri->port || strcmp(qp->p[0].name, "socket")) {
+            ret = -EINVAL;
+            goto out;
+        }
+        s->host_spec = g_strdup(qp->p[0].value);
+    } else {
+        /* nbd[+tcp]://host:port/export */
+        if (!uri->server) {
+            ret = -EINVAL;
+            goto out;
+        }
+        if (!uri->port) {
+            uri->port = NBD_DEFAULT_PORT;
+        }
+        s->host_spec = g_strdup_printf("%s:%d", uri->server, uri->port);
+    }
+
+out:
+    if (qp) {
+        query_params_free(qp);
+    }
+    uri_free(uri);
+    return ret;
+}
+
+static int nbd_config(BDRVNBDState *s, const char *filename)
 {
     char *file;
     char *export_name;
@@ -79,6 +141,10 @@ static int nbd_config(BDRVNBDState *s, const char *filename, int flags)
     const char *unixpath;
     int err = -EINVAL;
 
+    if (strstr(filename, "://")) {
+        return nbd_parse_uri(s, filename);
+    }
+
     file = g_strdup(filename);
 
     export_name = strstr(file, EN_OPTSTR);
@@ -98,11 +164,10 @@ static int nbd_config(BDRVNBDState *s, const char *filename, int flags)
 
     /* are we a UNIX or TCP socket? */
     if (strstart(host_spec, "unix:", &unixpath)) {
-        if (unixpath[0] != '/') { /* We demand  an absolute path*/
-            goto out;
-        }
+        s->is_unix = true;
         s->host_spec = g_strdup(unixpath);
     } else {
+        s->is_unix = false;
         s->host_spec = g_strdup(host_spec);
     }
 
@@ -262,7 +327,7 @@ static int nbd_establish_connection(BlockDriverState *bs)
     off_t size;
     size_t blocksize;
 
-    if (s->host_spec[0] == '/') {
+    if (s->is_unix) {
         sock = unix_socket_outgoing(s->host_spec);
     } else {
         sock = tcp_socket_outgoing_spec(s->host_spec);
@@ -320,7 +385,7 @@ static int nbd_open(BlockDriverState *bs, const char* filename, int flags)
     qemu_co_mutex_init(&s->free_sema);
 
     /* Pop the config into our state object. Exit if invalid. */
-    result = nbd_config(s, filename, flags);
+    result = nbd_config(s, filename);
     if (result != 0) {
         return result;
     }
@@ -498,6 +563,33 @@ static int64_t nbd_getlength(BlockDriverState *bs)
 
 static BlockDriver bdrv_nbd = {
     .format_name         = "nbd",
+    .protocol_name       = "nbd",
+    .instance_size       = sizeof(BDRVNBDState),
+    .bdrv_file_open      = nbd_open,
+    .bdrv_co_readv       = nbd_co_readv,
+    .bdrv_co_writev      = nbd_co_writev,
+    .bdrv_close          = nbd_close,
+    .bdrv_co_flush_to_os = nbd_co_flush,
+    .bdrv_co_discard     = nbd_co_discard,
+    .bdrv_getlength      = nbd_getlength,
+};
+
+static BlockDriver bdrv_nbd_tcp = {
+    .format_name         = "nbd",
+    .protocol_name       = "nbd+tcp",
+    .instance_size       = sizeof(BDRVNBDState),
+    .bdrv_file_open      = nbd_open,
+    .bdrv_co_readv       = nbd_co_readv,
+    .bdrv_co_writev      = nbd_co_writev,
+    .bdrv_close          = nbd_close,
+    .bdrv_co_flush_to_os = nbd_co_flush,
+    .bdrv_co_discard     = nbd_co_discard,
+    .bdrv_getlength      = nbd_getlength,
+};
+
+static BlockDriver bdrv_nbd_unix = {
+    .format_name         = "nbd",
+    .protocol_name       = "nbd+unix",
     .instance_size       = sizeof(BDRVNBDState),
     .bdrv_file_open      = nbd_open,
     .bdrv_co_readv       = nbd_co_readv,
@@ -506,12 +598,13 @@ static BlockDriver bdrv_nbd = {
     .bdrv_co_flush_to_os = nbd_co_flush,
     .bdrv_co_discard     = nbd_co_discard,
     .bdrv_getlength      = nbd_getlength,
-    .protocol_name       = "nbd",
 };
 
 static void bdrv_nbd_init(void)
 {
     bdrv_register(&bdrv_nbd);
+    bdrv_register(&bdrv_nbd_tcp);
+    bdrv_register(&bdrv_nbd_unix);
 }
 
 block_init(bdrv_nbd_init);
diff --git a/block/qcow.c b/block/qcow.c
index 7b5ab87d2..b239c82ae 100644
--- a/block/qcow.c
+++ b/block/qcow.c
@@ -197,6 +197,15 @@ static int qcow_open(BlockDriverState *bs, int flags)
     return ret;
 }
 
+
+/* We have nothing to do for QCOW reopen, stubs just return
+ * success */
+static int qcow_reopen_prepare(BDRVReopenState *state,
+                               BlockReopenQueue *queue, Error **errp)
+{
+    return 0;
+}
+
 static int qcow_set_key(BlockDriverState *bs, const char *key)
 {
     BDRVQcowState *s = bs->opaque;
@@ -868,6 +877,7 @@ static BlockDriver bdrv_qcow = {
     .bdrv_probe		= qcow_probe,
     .bdrv_open		= qcow_open,
     .bdrv_close		= qcow_close,
+    .bdrv_reopen_prepare = qcow_reopen_prepare,
     .bdrv_create	= qcow_create,
 
     .bdrv_co_readv          = qcow_co_readv,
diff --git a/block/qcow2-refcount.c b/block/qcow2-refcount.c
index 5e3f9153f..96224d1af 100644
--- a/block/qcow2-refcount.c
+++ b/block/qcow2-refcount.c
@@ -301,7 +301,8 @@ static int alloc_refcount_block(BlockDriverState *bs,
     uint64_t last_table_size;
     uint64_t blocks_clusters;
     do {
-        uint64_t table_clusters = size_to_clusters(s, table_size);
+        uint64_t table_clusters =
+            size_to_clusters(s, table_size * sizeof(uint64_t));
         blocks_clusters = 1 +
             ((table_clusters + refcount_block_clusters - 1)
             / refcount_block_clusters);
diff --git a/block/qcow2.c b/block/qcow2.c
index 8f183f146..c1ff31f48 100644
--- a/block/qcow2.c
+++ b/block/qcow2.c
@@ -52,6 +52,7 @@ typedef struct {
     uint32_t magic;
     uint32_t len;
 } QCowExtension;
+
 #define  QCOW2_EXT_MAGIC_END 0
 #define  QCOW2_EXT_MAGIC_BACKING_FORMAT 0xE2792ACA
 #define  QCOW2_EXT_MAGIC_FEATURE_TABLE 0x6803f857
@@ -558,6 +559,14 @@ static int qcow2_set_key(BlockDriverState *bs, const char *key)
     return 0;
 }
 
+/* We have nothing to do for QCOW2 reopen, stubs just return
+ * success */
+static int qcow2_reopen_prepare(BDRVReopenState *state,
+                                BlockReopenQueue *queue, Error **errp)
+{
+    return 0;
+}
+
 static int coroutine_fn qcow2_co_is_allocated(BlockDriverState *bs,
         int64_t sector_num, int nb_sectors, int *pnum)
 {
@@ -1087,6 +1096,7 @@ int qcow2_update_header(BlockDriverState *bs)
             goto fail;
         }
 
+        /* Using strncpy is ok here, since buf is not NUL-terminated. */
         strncpy(buf, bs->backing_file, buflen);
 
         header->backing_file_offset = cpu_to_be64(buf - ((char*) header));
@@ -1679,6 +1689,7 @@ static BlockDriver bdrv_qcow2 = {
     .bdrv_probe         = qcow2_probe,
     .bdrv_open          = qcow2_open,
     .bdrv_close         = qcow2_close,
+    .bdrv_reopen_prepare  = qcow2_reopen_prepare,
     .bdrv_create        = qcow2_create,
     .bdrv_co_is_allocated = qcow2_co_is_allocated,
     .bdrv_set_key       = qcow2_set_key,
diff --git a/block/qed-table.c b/block/qed-table.c
index ce07b0554..de845ec3d 100644
--- a/block/qed-table.c
+++ b/block/qed-table.c
@@ -103,7 +103,6 @@ static void qed_write_table_cb(void *opaque, int ret)
 out:
     qemu_vfree(write_table_cb->table);
     gencb_complete(&write_table_cb->gencb, ret);
-    return;
 }
 
 /**
diff --git a/block/qed.c b/block/qed.c
index 21cb23987..0b5374a20 100644
--- a/block/qed.c
+++ b/block/qed.c
@@ -30,7 +30,7 @@ static void qed_aio_cancel(BlockDriverAIOCB *blockacb)
     }
 }
 
-static AIOPool qed_aio_pool = {
+static const AIOCBInfo qed_aiocb_info = {
     .aiocb_size         = sizeof(QEDAIOCB),
     .cancel             = qed_aio_cancel,
 };
@@ -505,6 +505,14 @@ out:
     return ret;
 }
 
+/* We have nothing to do for QED reopen, stubs just return
+ * success */
+static int bdrv_qed_reopen_prepare(BDRVReopenState *state,
+                                   BlockReopenQueue *queue, Error **errp)
+{
+    return 0;
+}
+
 static void bdrv_qed_close(BlockDriverState *bs)
 {
     BDRVQEDState *s = bs->opaque;
@@ -1303,7 +1311,7 @@ static BlockDriverAIOCB *qed_aio_setup(BlockDriverState *bs,
                                        BlockDriverCompletionFunc *cb,
                                        void *opaque, int flags)
 {
-    QEDAIOCB *acb = qemu_aio_get(&qed_aio_pool, bs, cb, opaque);
+    QEDAIOCB *acb = qemu_aio_get(&qed_aiocb_info, bs, cb, opaque);
 
     trace_qed_aio_setup(bs->opaque, acb, sector_num, nb_sectors,
                         opaque, flags);
@@ -1564,6 +1572,7 @@ static BlockDriver bdrv_qed = {
     .bdrv_rebind              = bdrv_qed_rebind,
     .bdrv_open                = bdrv_qed_open,
     .bdrv_close               = bdrv_qed_close,
+    .bdrv_reopen_prepare      = bdrv_qed_reopen_prepare,
     .bdrv_create              = bdrv_qed_create,
     .bdrv_co_is_allocated     = bdrv_qed_co_is_allocated,
     .bdrv_make_empty          = bdrv_qed_make_empty,
diff --git a/block/raw-posix-aio.h b/block/raw-aio.h
index ba118f616..e77f36114 100644
--- a/block/raw-posix-aio.h
+++ b/block/raw-aio.h
@@ -1,5 +1,5 @@
 /*
- * QEMU Posix block I/O backend AIO support
+ * Declarations for AIO in the raw protocol
  *
  * Copyright IBM, Corp. 2008
  *
@@ -12,8 +12,8 @@
  * Contributions after 2012-01-13 are licensed under the terms of the
  * GNU GPL, version 2 or (at your option) any later version.
  */
-#ifndef QEMU_RAW_POSIX_AIO_H
-#define QEMU_RAW_POSIX_AIO_H
+#ifndef QEMU_RAW_AIO_H
+#define QEMU_RAW_AIO_H
 
 /* AIO request types */
 #define QEMU_AIO_READ         0x0001
@@ -27,19 +27,22 @@
 #define QEMU_AIO_MISALIGNED   0x1000
 
 
-/* posix-aio-compat.c - thread pool based implementation */
-int paio_init(void);
-BlockDriverAIOCB *paio_submit(BlockDriverState *bs, int fd,
-        int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
-        BlockDriverCompletionFunc *cb, void *opaque, int type);
-BlockDriverAIOCB *paio_ioctl(BlockDriverState *bs, int fd,
-        unsigned long int req, void *buf,
-        BlockDriverCompletionFunc *cb, void *opaque);
-
 /* linux-aio.c - Linux native implementation */
+#ifdef CONFIG_LINUX_AIO
 void *laio_init(void);
 BlockDriverAIOCB *laio_submit(BlockDriverState *bs, void *aio_ctx, int fd,
         int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
         BlockDriverCompletionFunc *cb, void *opaque, int type);
+#endif
+
+#ifdef _WIN32
+typedef struct QEMUWin32AIOState QEMUWin32AIOState;
+QEMUWin32AIOState *win32_aio_init(void);
+int win32_aio_attach(QEMUWin32AIOState *aio, HANDLE hfile);
+BlockDriverAIOCB *win32_aio_submit(BlockDriverState *bs,
+        QEMUWin32AIOState *aio, HANDLE hfile,
+        int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
+        BlockDriverCompletionFunc *cb, void *opaque, int type);
+#endif
 
-#endif /* QEMU_RAW_POSIX_AIO_H */
+#endif /* QEMU_RAW_AIO_H */
diff --git a/block/raw-posix.c b/block/raw-posix.c
index 6be20b192..550c81f22 100644
--- a/block/raw-posix.c
+++ b/block/raw-posix.c
@@ -27,7 +27,10 @@
 #include "qemu-log.h"
 #include "block_int.h"
 #include "module.h"
-#include "block/raw-posix-aio.h"
+#include "trace.h"
+#include "thread-pool.h"
+#include "iov.h"
+#include "raw-aio.h"
 
 #if defined(__APPLE__) && (__MACH__)
 #include <paths.h>
@@ -133,16 +136,36 @@ typedef struct BDRVRawState {
     int use_aio;
     void *aio_ctx;
 #endif
-    uint8_t *aligned_buf;
-    unsigned aligned_buf_size;
 #ifdef CONFIG_XFS
     bool is_xfs : 1;
 #endif
 } BDRVRawState;
 
+typedef struct BDRVRawReopenState {
+    int fd;
+    int open_flags;
+#ifdef CONFIG_LINUX_AIO
+    int use_aio;
+#endif
+} BDRVRawReopenState;
+
 static int fd_open(BlockDriverState *bs);
 static int64_t raw_getlength(BlockDriverState *bs);
 
+typedef struct RawPosixAIOData {
+    BlockDriverState *bs;
+    int aio_fildes;
+    union {
+        struct iovec *aio_iov;
+        void *aio_ioctl_buf;
+    };
+    int aio_niov;
+    size_t aio_nbytes;
+#define aio_ioctl_cmd   aio_nbytes /* for QEMU_AIO_IOCTL */
+    off_t aio_offset;
+    int aio_type;
+} RawPosixAIOData;
+
 #if defined(__FreeBSD__) || defined(__FreeBSD_kernel__)
 static int cdrom_reopen(BlockDriverState *bs);
 #endif
@@ -185,6 +208,57 @@ static int raw_normalize_devicepath(const char **filename)
 }
 #endif
 
+static void raw_parse_flags(int bdrv_flags, int *open_flags)
+{
+    assert(open_flags != NULL);
+
+    *open_flags |= O_BINARY;
+    *open_flags &= ~O_ACCMODE;
+    if (bdrv_flags & BDRV_O_RDWR) {
+        *open_flags |= O_RDWR;
+    } else {
+        *open_flags |= O_RDONLY;
+    }
+
+    /* Use O_DSYNC for write-through caching, no flags for write-back caching,
+     * and O_DIRECT for no caching. */
+    if ((bdrv_flags & BDRV_O_NOCACHE)) {
+        *open_flags |= O_DIRECT;
+    }
+}
+
+#ifdef CONFIG_LINUX_AIO
+static int raw_set_aio(void **aio_ctx, int *use_aio, int bdrv_flags)
+{
+    int ret = -1;
+    assert(aio_ctx != NULL);
+    assert(use_aio != NULL);
+    /*
+     * Currently Linux do AIO only for files opened with O_DIRECT
+     * specified so check NOCACHE flag too
+     */
+    if ((bdrv_flags & (BDRV_O_NOCACHE|BDRV_O_NATIVE_AIO)) ==
+                      (BDRV_O_NOCACHE|BDRV_O_NATIVE_AIO)) {
+
+        /* if non-NULL, laio_init() has already been run */
+        if (*aio_ctx == NULL) {
+            *aio_ctx = laio_init();
+            if (!*aio_ctx) {
+                goto error;
+            }
+        }
+        *use_aio = 1;
+    } else {
+        *use_aio = 0;
+    }
+
+    ret = 0;
+
+error:
+    return ret;
+}
+#endif
+
 static int raw_open_common(BlockDriverState *bs, const char *filename,
                            int bdrv_flags, int open_flags)
 {
@@ -196,20 +270,8 @@ static int raw_open_common(BlockDriverState *bs, const char *filename,
         return ret;
     }
 
-    s->open_flags = open_flags | O_BINARY;
-    s->open_flags &= ~O_ACCMODE;
-    if (bdrv_flags & BDRV_O_RDWR) {
-        s->open_flags |= O_RDWR;
-    } else {
-        s->open_flags |= O_RDONLY;
-    }
-
-    /* Use O_DSYNC for write-through caching, no flags for write-back caching,
-     * and O_DIRECT for no caching. */
-    if ((bdrv_flags & BDRV_O_NOCACHE))
-        s->open_flags |= O_DIRECT;
-    if (!(bdrv_flags & BDRV_O_CACHE_WB))
-        s->open_flags |= O_DSYNC;
+    s->open_flags = open_flags;
+    raw_parse_flags(bdrv_flags, &s->open_flags);
 
     s->fd = -1;
     fd = qemu_open(filename, s->open_flags, 0644);
@@ -220,45 +282,13 @@ static int raw_open_common(BlockDriverState *bs, const char *filename,
         return ret;
     }
     s->fd = fd;
-    s->aligned_buf = NULL;
-
-    if ((bdrv_flags & BDRV_O_NOCACHE)) {
-        /*
-         * Allocate a buffer for read/modify/write cycles.  Chose the size
-         * pessimistically as we don't know the block size yet.
-         */
-        s->aligned_buf_size = 32 * MAX_BLOCKSIZE;
-        s->aligned_buf = qemu_memalign(MAX_BLOCKSIZE, s->aligned_buf_size);
-        if (s->aligned_buf == NULL) {
-            goto out_close;
-        }
-    }
-
-    /* We're falling back to POSIX AIO in some cases so init always */
-    if (paio_init() < 0) {
-        goto out_free_buf;
-    }
 
 #ifdef CONFIG_LINUX_AIO
-    /*
-     * Currently Linux do AIO only for files opened with O_DIRECT
-     * specified so check NOCACHE flag too
-     */
-    if ((bdrv_flags & (BDRV_O_NOCACHE|BDRV_O_NATIVE_AIO)) ==
-                      (BDRV_O_NOCACHE|BDRV_O_NATIVE_AIO)) {
-
-        s->aio_ctx = laio_init();
-        if (!s->aio_ctx) {
-            goto out_free_buf;
-        }
-        s->use_aio = 1;
-    } else
-#endif
-    {
-#ifdef CONFIG_LINUX_AIO
-        s->use_aio = 0;
-#endif
+    if (raw_set_aio(&s->aio_ctx, &s->use_aio, bdrv_flags)) {
+        qemu_close(fd);
+        return -errno;
     }
+#endif
 
 #ifdef CONFIG_XFS
     if (platform_test_xfs_fd(s->fd)) {
@@ -267,12 +297,6 @@ static int raw_open_common(BlockDriverState *bs, const char *filename,
 #endif
 
     return 0;
-
-out_free_buf:
-    qemu_vfree(s->aligned_buf);
-out_close:
-    qemu_close(fd);
-    return -errno;
 }
 
 static int raw_open(BlockDriverState *bs, const char *filename, int flags)
@@ -283,6 +307,113 @@ static int raw_open(BlockDriverState *bs, const char *filename, int flags)
     return raw_open_common(bs, filename, flags, 0);
 }
 
+static int raw_reopen_prepare(BDRVReopenState *state,
+                              BlockReopenQueue *queue, Error **errp)
+{
+    BDRVRawState *s;
+    BDRVRawReopenState *raw_s;
+    int ret = 0;
+
+    assert(state != NULL);
+    assert(state->bs != NULL);
+
+    s = state->bs->opaque;
+
+    state->opaque = g_malloc0(sizeof(BDRVRawReopenState));
+    raw_s = state->opaque;
+
+#ifdef CONFIG_LINUX_AIO
+    raw_s->use_aio = s->use_aio;
+
+    /* we can use s->aio_ctx instead of a copy, because the use_aio flag is
+     * valid in the 'false' condition even if aio_ctx is set, and raw_set_aio()
+     * won't override aio_ctx if aio_ctx is non-NULL */
+    if (raw_set_aio(&s->aio_ctx, &raw_s->use_aio, state->flags)) {
+        return -1;
+    }
+#endif
+
+    if (s->type == FTYPE_FD || s->type == FTYPE_CD) {
+        raw_s->open_flags |= O_NONBLOCK;
+    }
+
+    raw_parse_flags(state->flags, &raw_s->open_flags);
+
+    raw_s->fd = -1;
+
+    int fcntl_flags = O_APPEND | O_ASYNC | O_NONBLOCK;
+#ifdef O_NOATIME
+    fcntl_flags |= O_NOATIME;
+#endif
+
+    if ((raw_s->open_flags & ~fcntl_flags) == (s->open_flags & ~fcntl_flags)) {
+        /* dup the original fd */
+        /* TODO: use qemu fcntl wrapper */
+#ifdef F_DUPFD_CLOEXEC
+        raw_s->fd = fcntl(s->fd, F_DUPFD_CLOEXEC, 0);
+#else
+        raw_s->fd = dup(s->fd);
+        if (raw_s->fd != -1) {
+            qemu_set_cloexec(raw_s->fd);
+        }
+#endif
+        if (raw_s->fd >= 0) {
+            ret = fcntl_setfl(raw_s->fd, raw_s->open_flags);
+            if (ret) {
+                qemu_close(raw_s->fd);
+                raw_s->fd = -1;
+            }
+        }
+    }
+
+    /* If we cannot use fcntl, or fcntl failed, fall back to qemu_open() */
+    if (raw_s->fd == -1) {
+        assert(!(raw_s->open_flags & O_CREAT));
+        raw_s->fd = qemu_open(state->bs->filename, raw_s->open_flags);
+        if (raw_s->fd == -1) {
+            ret = -1;
+        }
+    }
+    return ret;
+}
+
+
+static void raw_reopen_commit(BDRVReopenState *state)
+{
+    BDRVRawReopenState *raw_s = state->opaque;
+    BDRVRawState *s = state->bs->opaque;
+
+    s->open_flags = raw_s->open_flags;
+
+    qemu_close(s->fd);
+    s->fd = raw_s->fd;
+#ifdef CONFIG_LINUX_AIO
+    s->use_aio = raw_s->use_aio;
+#endif
+
+    g_free(state->opaque);
+    state->opaque = NULL;
+}
+
+
+static void raw_reopen_abort(BDRVReopenState *state)
+{
+    BDRVRawReopenState *raw_s = state->opaque;
+
+     /* nothing to do if NULL, we didn't get far enough */
+    if (raw_s == NULL) {
+        return;
+    }
+
+    if (raw_s->fd >= 0) {
+        qemu_close(raw_s->fd);
+        raw_s->fd = -1;
+    }
+    g_free(state->opaque);
+    state->opaque = NULL;
+}
+
+
 /* XXX: use host sector size if necessary with:
 #ifdef DIOCGSECTORSIZE
         {
@@ -316,6 +447,283 @@ static int qiov_is_aligned(BlockDriverState *bs, QEMUIOVector *qiov)
     return 1;
 }
 
+static ssize_t handle_aiocb_ioctl(RawPosixAIOData *aiocb)
+{
+    int ret;
+
+    ret = ioctl(aiocb->aio_fildes, aiocb->aio_ioctl_cmd, aiocb->aio_ioctl_buf);
+    if (ret == -1) {
+        return -errno;
+    }
+
+    /*
+     * This looks weird, but the aio code only considers a request
+     * successful if it has written the full number of bytes.
+     *
+     * Now we overload aio_nbytes as aio_ioctl_cmd for the ioctl command,
+     * so in fact we return the ioctl command here to make posix_aio_read()
+     * happy..
+     */
+    return aiocb->aio_nbytes;
+}
+
+static ssize_t handle_aiocb_flush(RawPosixAIOData *aiocb)
+{
+    int ret;
+
+    ret = qemu_fdatasync(aiocb->aio_fildes);
+    if (ret == -1) {
+        return -errno;
+    }
+    return 0;
+}
+
+#ifdef CONFIG_PREADV
+
+static bool preadv_present = true;
+
+static ssize_t
+qemu_preadv(int fd, const struct iovec *iov, int nr_iov, off_t offset)
+{
+    return preadv(fd, iov, nr_iov, offset);
+}
+
+static ssize_t
+qemu_pwritev(int fd, const struct iovec *iov, int nr_iov, off_t offset)
+{
+    return pwritev(fd, iov, nr_iov, offset);
+}
+
+#else
+
+static bool preadv_present = false;
+
+static ssize_t
+qemu_preadv(int fd, const struct iovec *iov, int nr_iov, off_t offset)
+{
+    return -ENOSYS;
+}
+
+static ssize_t
+qemu_pwritev(int fd, const struct iovec *iov, int nr_iov, off_t offset)
+{
+    return -ENOSYS;
+}
+
+#endif
+
+static ssize_t handle_aiocb_rw_vector(RawPosixAIOData *aiocb)
+{
+    ssize_t len;
+
+    do {
+        if (aiocb->aio_type & QEMU_AIO_WRITE)
+            len = qemu_pwritev(aiocb->aio_fildes,
+                               aiocb->aio_iov,
+                               aiocb->aio_niov,
+                               aiocb->aio_offset);
+         else
+            len = qemu_preadv(aiocb->aio_fildes,
+                              aiocb->aio_iov,
+                              aiocb->aio_niov,
+                              aiocb->aio_offset);
+    } while (len == -1 && errno == EINTR);
+
+    if (len == -1) {
+        return -errno;
+    }
+    return len;
+}
+
+/*
+ * Read/writes the data to/from a given linear buffer.
+ *
+ * Returns the number of bytes handles or -errno in case of an error. Short
+ * reads are only returned if the end of the file is reached.
+ */
+static ssize_t handle_aiocb_rw_linear(RawPosixAIOData *aiocb, char *buf)
+{
+    ssize_t offset = 0;
+    ssize_t len;
+
+    while (offset < aiocb->aio_nbytes) {
+        if (aiocb->aio_type & QEMU_AIO_WRITE) {
+            len = pwrite(aiocb->aio_fildes,
+                         (const char *)buf + offset,
+                         aiocb->aio_nbytes - offset,
+                         aiocb->aio_offset + offset);
+        } else {
+            len = pread(aiocb->aio_fildes,
+                        buf + offset,
+                        aiocb->aio_nbytes - offset,
+                        aiocb->aio_offset + offset);
+        }
+        if (len == -1 && errno == EINTR) {
+            continue;
+        } else if (len == -1) {
+            offset = -errno;
+            break;
+        } else if (len == 0) {
+            break;
+        }
+        offset += len;
+    }
+
+    return offset;
+}
+
+static ssize_t handle_aiocb_rw(RawPosixAIOData *aiocb)
+{
+    ssize_t nbytes;
+    char *buf;
+
+    if (!(aiocb->aio_type & QEMU_AIO_MISALIGNED)) {
+        /*
+         * If there is just a single buffer, and it is properly aligned
+         * we can just use plain pread/pwrite without any problems.
+         */
+        if (aiocb->aio_niov == 1) {
+             return handle_aiocb_rw_linear(aiocb, aiocb->aio_iov->iov_base);
+        }
+        /*
+         * We have more than one iovec, and all are properly aligned.
+         *
+         * Try preadv/pwritev first and fall back to linearizing the
+         * buffer if it's not supported.
+         */
+        if (preadv_present) {
+            nbytes = handle_aiocb_rw_vector(aiocb);
+            if (nbytes == aiocb->aio_nbytes ||
+                (nbytes < 0 && nbytes != -ENOSYS)) {
+                return nbytes;
+            }
+            preadv_present = false;
+        }
+
+        /*
+         * XXX(hch): short read/write.  no easy way to handle the reminder
+         * using these interfaces.  For now retry using plain
+         * pread/pwrite?
+         */
+    }
+
+    /*
+     * Ok, we have to do it the hard way, copy all segments into
+     * a single aligned buffer.
+     */
+    buf = qemu_blockalign(aiocb->bs, aiocb->aio_nbytes);
+    if (aiocb->aio_type & QEMU_AIO_WRITE) {
+        char *p = buf;
+        int i;
+
+        for (i = 0; i < aiocb->aio_niov; ++i) {
+            memcpy(p, aiocb->aio_iov[i].iov_base, aiocb->aio_iov[i].iov_len);
+            p += aiocb->aio_iov[i].iov_len;
+        }
+    }
+
+    nbytes = handle_aiocb_rw_linear(aiocb, buf);
+    if (!(aiocb->aio_type & QEMU_AIO_WRITE)) {
+        char *p = buf;
+        size_t count = aiocb->aio_nbytes, copy;
+        int i;
+
+        for (i = 0; i < aiocb->aio_niov && count; ++i) {
+            copy = count;
+            if (copy > aiocb->aio_iov[i].iov_len) {
+                copy = aiocb->aio_iov[i].iov_len;
+            }
+            memcpy(aiocb->aio_iov[i].iov_base, p, copy);
+            p     += copy;
+            count -= copy;
+        }
+    }
+    qemu_vfree(buf);
+
+    return nbytes;
+}
+
+static int aio_worker(void *arg)
+{
+    RawPosixAIOData *aiocb = arg;
+    ssize_t ret = 0;
+
+    switch (aiocb->aio_type & QEMU_AIO_TYPE_MASK) {
+    case QEMU_AIO_READ:
+        ret = handle_aiocb_rw(aiocb);
+        if (ret >= 0 && ret < aiocb->aio_nbytes && aiocb->bs->growable) {
+            iov_memset(aiocb->aio_iov, aiocb->aio_niov, ret,
+                      0, aiocb->aio_nbytes - ret);
+
+            ret = aiocb->aio_nbytes;
+        }
+        if (ret == aiocb->aio_nbytes) {
+            ret = 0;
+        } else if (ret >= 0 && ret < aiocb->aio_nbytes) {
+            ret = -EINVAL;
+        }
+        break;
+    case QEMU_AIO_WRITE:
+        ret = handle_aiocb_rw(aiocb);
+        if (ret == aiocb->aio_nbytes) {
+            ret = 0;
+        } else if (ret >= 0 && ret < aiocb->aio_nbytes) {
+            ret = -EINVAL;
+        }
+        break;
+    case QEMU_AIO_FLUSH:
+        ret = handle_aiocb_flush(aiocb);
+        break;
+    case QEMU_AIO_IOCTL:
+        ret = handle_aiocb_ioctl(aiocb);
+        break;
+    default:
+        fprintf(stderr, "invalid aio request (0x%x)\n", aiocb->aio_type);
+        ret = -EINVAL;
+        break;
+    }
+
+    g_slice_free(RawPosixAIOData, aiocb);
+    return ret;
+}
+
+static BlockDriverAIOCB *paio_submit(BlockDriverState *bs, int fd,
+        int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
+        BlockDriverCompletionFunc *cb, void *opaque, int type)
+{
+    RawPosixAIOData *acb = g_slice_new(RawPosixAIOData);
+
+    acb->bs = bs;
+    acb->aio_type = type;
+    acb->aio_fildes = fd;
+
+    if (qiov) {
+        acb->aio_iov = qiov->iov;
+        acb->aio_niov = qiov->niov;
+    }
+    acb->aio_nbytes = nb_sectors * 512;
+    acb->aio_offset = sector_num * 512;
+
+    trace_paio_submit(acb, opaque, sector_num, nb_sectors, type);
+    return thread_pool_submit_aio(aio_worker, acb, cb, opaque);
+}
+
+static BlockDriverAIOCB *paio_ioctl(BlockDriverState *bs, int fd,
+        unsigned long int req, void *buf,
+        BlockDriverCompletionFunc *cb, void *opaque)
+{
+    RawPosixAIOData *acb = g_slice_new(RawPosixAIOData);
+
+    acb->bs = bs;
+    acb->aio_type = QEMU_AIO_IOCTL;
+    acb->aio_fildes = fd;
+    acb->aio_offset = 0;
+    acb->aio_ioctl_buf = buf;
+    acb->aio_ioctl_cmd = req;
+
+    return thread_pool_submit_aio(aio_worker, acb, cb, opaque);
+}
+
 static BlockDriverAIOCB *raw_aio_submit(BlockDriverState *bs,
         int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
         BlockDriverCompletionFunc *cb, void *opaque, int type)
@@ -330,7 +738,7 @@ static BlockDriverAIOCB *raw_aio_submit(BlockDriverState *bs,
      * boundary.  Check if this is the case or tell the low-level
      * driver that it needs to copy the buffer.
      */
-    if (s->aligned_buf) {
+    if ((bs->open_flags & BDRV_O_NOCACHE)) {
         if (!qiov_is_aligned(bs, qiov)) {
             type |= QEMU_AIO_MISALIGNED;
 #ifdef CONFIG_LINUX_AIO
@@ -378,8 +786,6 @@ static void raw_close(BlockDriverState *bs)
     if (s->fd >= 0) {
         qemu_close(s->fd);
         s->fd = -1;
-        if (s->aligned_buf != NULL)
-            qemu_vfree(s->aligned_buf);
     }
 }
 
@@ -735,6 +1141,9 @@ static BlockDriver bdrv_file = {
     .instance_size = sizeof(BDRVRawState),
     .bdrv_probe = NULL, /* no probe for protocols */
     .bdrv_file_open = raw_open,
+    .bdrv_reopen_prepare = raw_reopen_prepare,
+    .bdrv_reopen_commit = raw_reopen_commit,
+    .bdrv_reopen_abort = raw_reopen_abort,
     .bdrv_close = raw_close,
     .bdrv_create = raw_create,
     .bdrv_co_discard = raw_co_discard,
@@ -1004,6 +1413,9 @@ static BlockDriver bdrv_host_device = {
     .bdrv_probe_device  = hdev_probe_device,
     .bdrv_file_open     = hdev_open,
     .bdrv_close         = raw_close,
+    .bdrv_reopen_prepare = raw_reopen_prepare,
+    .bdrv_reopen_commit  = raw_reopen_commit,
+    .bdrv_reopen_abort   = raw_reopen_abort,
     .bdrv_create        = hdev_create,
     .create_options     = raw_create_options,
     .bdrv_has_zero_init = hdev_has_zero_init,
@@ -1125,6 +1537,9 @@ static BlockDriver bdrv_host_floppy = {
     .bdrv_probe_device	= floppy_probe_device,
     .bdrv_file_open     = floppy_open,
     .bdrv_close         = raw_close,
+    .bdrv_reopen_prepare = raw_reopen_prepare,
+    .bdrv_reopen_commit  = raw_reopen_commit,
+    .bdrv_reopen_abort   = raw_reopen_abort,
     .bdrv_create        = hdev_create,
     .create_options     = raw_create_options,
     .bdrv_has_zero_init = hdev_has_zero_init,
@@ -1224,6 +1639,9 @@ static BlockDriver bdrv_host_cdrom = {
     .bdrv_probe_device	= cdrom_probe_device,
     .bdrv_file_open     = cdrom_open,
     .bdrv_close         = raw_close,
+    .bdrv_reopen_prepare = raw_reopen_prepare,
+    .bdrv_reopen_commit  = raw_reopen_commit,
+    .bdrv_reopen_abort   = raw_reopen_abort,
     .bdrv_create        = hdev_create,
     .create_options     = raw_create_options,
     .bdrv_has_zero_init = hdev_has_zero_init,
@@ -1343,6 +1761,9 @@ static BlockDriver bdrv_host_cdrom = {
     .bdrv_probe_device	= cdrom_probe_device,
     .bdrv_file_open     = cdrom_open,
     .bdrv_close         = raw_close,
+    .bdrv_reopen_prepare = raw_reopen_prepare,
+    .bdrv_reopen_commit  = raw_reopen_commit,
+    .bdrv_reopen_abort   = raw_reopen_abort,
     .bdrv_create        = hdev_create,
     .create_options     = raw_create_options,
     .bdrv_has_zero_init = hdev_has_zero_init,
diff --git a/block/raw-win32.c b/block/raw-win32.c
index c56bf8337..0c05c58c5 100644
--- a/block/raw-win32.c
+++ b/block/raw-win32.c
@@ -25,6 +25,10 @@
 #include "qemu-timer.h"
 #include "block_int.h"
 #include "module.h"
+#include "raw-aio.h"
+#include "trace.h"
+#include "thread-pool.h"
+#include "iov.h"
 #include <windows.h>
 #include <winioctl.h>
 
@@ -32,12 +36,130 @@
 #define FTYPE_CD     1
 #define FTYPE_HARDDISK 2
 
+static QEMUWin32AIOState *aio;
+
+typedef struct RawWin32AIOData {
+    BlockDriverState *bs;
+    HANDLE hfile;
+    struct iovec *aio_iov;
+    int aio_niov;
+    size_t aio_nbytes;
+    off64_t aio_offset;
+    int aio_type;
+} RawWin32AIOData;
+
 typedef struct BDRVRawState {
     HANDLE hfile;
     int type;
     char drive_path[16]; /* format: "d:\" */
+    QEMUWin32AIOState *aio;
 } BDRVRawState;
 
+/*
+ * Read/writes the data to/from a given linear buffer.
+ *
+ * Returns the number of bytes handles or -errno in case of an error. Short
+ * reads are only returned if the end of the file is reached.
+ */
+static size_t handle_aiocb_rw(RawWin32AIOData *aiocb)
+{
+    size_t offset = 0;
+    int i;
+
+    for (i = 0; i < aiocb->aio_niov; i++) {
+        OVERLAPPED ov;
+        DWORD ret, ret_count, len;
+
+        memset(&ov, 0, sizeof(ov));
+        ov.Offset = (aiocb->aio_offset + offset);
+        ov.OffsetHigh = (aiocb->aio_offset + offset) >> 32;
+        len = aiocb->aio_iov[i].iov_len;
+        if (aiocb->aio_type & QEMU_AIO_WRITE) {
+            ret = WriteFile(aiocb->hfile, aiocb->aio_iov[i].iov_base,
+                            len, &ret_count, &ov);
+        } else {
+            ret = ReadFile(aiocb->hfile, aiocb->aio_iov[i].iov_base,
+                           len, &ret_count, &ov);
+        }
+        if (!ret) {
+            ret_count = 0;
+        }
+        if (ret_count != len) {
+            break;
+        }
+        offset += len;
+    }
+
+    return offset;
+}
+
+static int aio_worker(void *arg)
+{
+    RawWin32AIOData *aiocb = arg;
+    ssize_t ret = 0;
+    size_t count;
+
+    switch (aiocb->aio_type & QEMU_AIO_TYPE_MASK) {
+    case QEMU_AIO_READ:
+        count = handle_aiocb_rw(aiocb);
+        if (count < aiocb->aio_nbytes && aiocb->bs->growable) {
+            /* A short read means that we have reached EOF. Pad the buffer
+             * with zeros for bytes after EOF. */
+            iov_memset(aiocb->aio_iov, aiocb->aio_niov, count,
+                      0, aiocb->aio_nbytes - count);
+
+            count = aiocb->aio_nbytes;
+        }
+        if (count == aiocb->aio_nbytes) {
+            ret = 0;
+        } else {
+            ret = -EINVAL;
+        }
+        break;
+    case QEMU_AIO_WRITE:
+        count = handle_aiocb_rw(aiocb);
+        if (count == aiocb->aio_nbytes) {
+            count = 0;
+        } else {
+            count = -EINVAL;
+        }
+        break;
+    case QEMU_AIO_FLUSH:
+        if (!FlushFileBuffers(aiocb->hfile)) {
+            return -EIO;
+        }
+        break;
+    default:
+        fprintf(stderr, "invalid aio request (0x%x)\n", aiocb->aio_type);
+        ret = -EINVAL;
+        break;
+    }
+
+    g_slice_free(RawWin32AIOData, aiocb);
+    return ret;
+}
+
+static BlockDriverAIOCB *paio_submit(BlockDriverState *bs, HANDLE hfile,
+        int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
+        BlockDriverCompletionFunc *cb, void *opaque, int type)
+{
+    RawWin32AIOData *acb = g_slice_new(RawWin32AIOData);
+
+    acb->bs = bs;
+    acb->hfile = hfile;
+    acb->aio_type = type;
+
+    if (qiov) {
+        acb->aio_iov = qiov->iov;
+        acb->aio_niov = qiov->niov;
+    }
+    acb->aio_nbytes = nb_sectors * 512;
+    acb->aio_offset = sector_num * 512;
+
+    trace_paio_submit(acb, opaque, sector_num, nb_sectors, type);
+    return thread_pool_submit_aio(aio_worker, acb, cb, opaque);
+}
+
 int qemu_ftruncate64(int fd, int64_t length)
 {
     LARGE_INTEGER li;
@@ -77,6 +199,26 @@ static int set_sparse(int fd)
 				 NULL, 0, NULL, 0, &returned, NULL);
 }
 
+static void raw_parse_flags(int flags, int *access_flags, DWORD *overlapped)
+{
+    assert(access_flags != NULL);
+    assert(overlapped != NULL);
+
+    if (flags & BDRV_O_RDWR) {
+        *access_flags = GENERIC_READ | GENERIC_WRITE;
+    } else {
+        *access_flags = GENERIC_READ;
+    }
+
+    *overlapped = FILE_ATTRIBUTE_NORMAL;
+    if (flags & BDRV_O_NATIVE_AIO) {
+        *overlapped |= FILE_FLAG_OVERLAPPED;
+    }
+    if (flags & BDRV_O_NOCACHE) {
+        *overlapped |= FILE_FLAG_NO_BUFFERING;
+    }
+}
+
 static int raw_open(BlockDriverState *bs, const char *filename, int flags)
 {
     BDRVRawState *s = bs->opaque;
@@ -85,17 +227,15 @@ static int raw_open(BlockDriverState *bs, const char *filename, int flags)
 
     s->type = FTYPE_FILE;
 
-    if (flags & BDRV_O_RDWR) {
-        access_flags = GENERIC_READ | GENERIC_WRITE;
-    } else {
-        access_flags = GENERIC_READ;
+    raw_parse_flags(flags, &access_flags, &overlapped);
+    
+    if ((flags & BDRV_O_NATIVE_AIO) && aio == NULL) {
+        aio = win32_aio_init();
+        if (aio == NULL) {
+            return -EINVAL;
+        }
     }
 
-    overlapped = FILE_ATTRIBUTE_NORMAL;
-    if (flags & BDRV_O_NOCACHE)
-        overlapped |= FILE_FLAG_NO_BUFFERING;
-    if (!(flags & BDRV_O_CACHE_WB))
-        overlapped |= FILE_FLAG_WRITE_THROUGH;
     s->hfile = CreateFile(filename, access_flags,
                           FILE_SHARE_READ, NULL,
                           OPEN_EXISTING, overlapped, NULL);
@@ -104,64 +244,53 @@ static int raw_open(BlockDriverState *bs, const char *filename, int flags)
 
         if (err == ERROR_ACCESS_DENIED)
             return -EACCES;
-        return -1;
+        return -EINVAL;
+    }
+
+    if (flags & BDRV_O_NATIVE_AIO) {
+        int ret = win32_aio_attach(aio, s->hfile);
+        if (ret < 0) {
+            CloseHandle(s->hfile);
+            return ret;
+        }
+        s->aio = aio;
     }
     return 0;
 }
 
-static int raw_read(BlockDriverState *bs, int64_t sector_num,
-                    uint8_t *buf, int nb_sectors)
+static BlockDriverAIOCB *raw_aio_readv(BlockDriverState *bs,
+                         int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
+                         BlockDriverCompletionFunc *cb, void *opaque)
 {
     BDRVRawState *s = bs->opaque;
-    OVERLAPPED ov;
-    DWORD ret_count;
-    int ret;
-    int64_t offset = sector_num * 512;
-    int count = nb_sectors * 512;
-
-    memset(&ov, 0, sizeof(ov));
-    ov.Offset = offset;
-    ov.OffsetHigh = offset >> 32;
-    ret = ReadFile(s->hfile, buf, count, &ret_count, &ov);
-    if (!ret)
-        return ret_count;
-    if (ret_count == count)
-        ret_count = 0;
-    return ret_count;
+    if (s->aio) {
+        return win32_aio_submit(bs, s->aio, s->hfile, sector_num, qiov,
+                                nb_sectors, cb, opaque, QEMU_AIO_READ); 
+    } else {
+        return paio_submit(bs, s->hfile, sector_num, qiov, nb_sectors,
+                           cb, opaque, QEMU_AIO_READ);
+    }
 }
 
-static int raw_write(BlockDriverState *bs, int64_t sector_num,
-                     const uint8_t *buf, int nb_sectors)
+static BlockDriverAIOCB *raw_aio_writev(BlockDriverState *bs,
+                          int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
+                          BlockDriverCompletionFunc *cb, void *opaque)
 {
     BDRVRawState *s = bs->opaque;
-    OVERLAPPED ov;
-    DWORD ret_count;
-    int ret;
-    int64_t offset = sector_num * 512;
-    int count = nb_sectors * 512;
-
-    memset(&ov, 0, sizeof(ov));
-    ov.Offset = offset;
-    ov.OffsetHigh = offset >> 32;
-    ret = WriteFile(s->hfile, buf, count, &ret_count, &ov);
-    if (!ret)
-        return ret_count;
-    if (ret_count == count)
-        ret_count = 0;
-    return ret_count;
+    if (s->aio) {
+        return win32_aio_submit(bs, s->aio, s->hfile, sector_num, qiov,
+                                nb_sectors, cb, opaque, QEMU_AIO_WRITE); 
+    } else {
+        return paio_submit(bs, s->hfile, sector_num, qiov, nb_sectors,
+                           cb, opaque, QEMU_AIO_WRITE);
+    }
 }
 
-static int raw_flush(BlockDriverState *bs)
+static BlockDriverAIOCB *raw_aio_flush(BlockDriverState *bs,
+                         BlockDriverCompletionFunc *cb, void *opaque)
 {
     BDRVRawState *s = bs->opaque;
-    int ret;
-
-    ret = FlushFileBuffers(s->hfile);
-    if (ret == 0) {
-        return -EIO;
-    }
-
-    return 0;
+    return paio_submit(bs, s->hfile, 0, NULL, 0, cb, opaque, QEMU_AIO_FLUSH);
 }
 
 static void raw_close(BlockDriverState *bs)
@@ -282,9 +411,9 @@ static BlockDriver bdrv_file = {
     .bdrv_close		= raw_close,
     .bdrv_create	= raw_create,
 
-    .bdrv_read              = raw_read,
-    .bdrv_write             = raw_write,
-    .bdrv_co_flush_to_disk  = raw_flush,
+    .bdrv_aio_readv     = raw_aio_readv,
+    .bdrv_aio_writev    = raw_aio_writev,
+    .bdrv_aio_flush     = raw_aio_flush,
 
     .bdrv_truncate	= raw_truncate,
     .bdrv_getlength	= raw_getlength,
@@ -374,18 +503,10 @@ static int hdev_open(BlockDriverState *bs, const char *filename, int flags)
     }
     s->type = find_device_type(bs, filename);
 
-    if (flags & BDRV_O_RDWR) {
-        access_flags = GENERIC_READ | GENERIC_WRITE;
-    } else {
-        access_flags = GENERIC_READ;
-    }
+    raw_parse_flags(flags, &access_flags, &overlapped);
+
     create_flags = OPEN_EXISTING;
 
-    overlapped = FILE_ATTRIBUTE_NORMAL;
-    if (flags & BDRV_O_NOCACHE)
-        overlapped |= FILE_FLAG_NO_BUFFERING;
-    if (!(flags & BDRV_O_CACHE_WB))
-        overlapped |= FILE_FLAG_WRITE_THROUGH;
     s->hfile = CreateFile(filename, access_flags,
                           FILE_SHARE_READ, NULL,
                           create_flags, overlapped, NULL);
@@ -413,9 +534,9 @@ static BlockDriver bdrv_host_device = {
     .bdrv_close		= raw_close,
     .bdrv_has_zero_init = hdev_has_zero_init,
 
-    .bdrv_read              = raw_read,
-    .bdrv_write             = raw_write,
-    .bdrv_co_flush_to_disk  = raw_flush,
+    .bdrv_aio_readv     = raw_aio_readv,
+    .bdrv_aio_writev    = raw_aio_writev,
+    .bdrv_aio_flush     = raw_aio_flush,
 
     .bdrv_getlength	= raw_getlength,
     .bdrv_get_allocated_file_size
diff --git a/block/raw.c b/block/raw.c
index ff34ea41e..253e949b8 100644
--- a/block/raw.c
+++ b/block/raw.c
@@ -9,6 +9,14 @@ static int raw_open(BlockDriverState *bs, int flags)
     return 0;
 }
 
+/* We have nothing to do for raw reopen, stubs just return
+ * success */
+static int raw_reopen_prepare(BDRVReopenState *state,
+                              BlockReopenQueue *queue,  Error **errp)
+{
+    return 0;
+}
+
 static int coroutine_fn raw_co_readv(BlockDriverState *bs, int64_t sector_num,
                                      int nb_sectors, QEMUIOVector *qiov)
 {
@@ -115,6 +123,8 @@ static BlockDriver bdrv_raw = {
     .bdrv_open          = raw_open,
     .bdrv_close         = raw_close,
 
+    .bdrv_reopen_prepare  = raw_reopen_prepare,
+
     .bdrv_co_readv          = raw_co_readv,
     .bdrv_co_writev         = raw_co_writev,
     .bdrv_co_is_allocated   = raw_co_is_allocated,
diff --git a/block/rbd.c b/block/rbd.c
index 5a0f79fc8..f3becc7a8 100644
--- a/block/rbd.c
+++ b/block/rbd.c
@@ -69,7 +69,7 @@ typedef enum {
 typedef struct RBDAIOCB {
     BlockDriverAIOCB common;
     QEMUBH *bh;
-    int ret;
+    int64_t ret;
     QEMUIOVector *qiov;
     char *bounce;
     RBDAIOCmd cmd;
@@ -86,7 +86,7 @@ typedef struct RADOSCB {
     int done;
     int64_t size;
     char *buf;
-    int ret;
+    int64_t ret;
 } RADOSCB;
 
 #define RBD_FD_READ 0
@@ -487,12 +487,6 @@ static int qemu_rbd_open(BlockDriverState *bs, const char *filename, int flags)
         rados_conf_set(s->cluster, "rbd_cache", "false");
     } else {
         rados_conf_set(s->cluster, "rbd_cache", "true");
-        if (!(flags & BDRV_O_CACHE_WB)) {
-            r = rados_conf_set(s->cluster, "rbd_cache_max_dirty", "0");
-            if (r < 0) {
-                rados_conf_set(s->cluster, "rbd_cache", "false");
-            }
-        }
     }
 
     if (strstr(conf, "conf=") == NULL) {
@@ -576,7 +570,7 @@ static void qemu_rbd_aio_cancel(BlockDriverAIOCB *blockacb)
     acb->cancelled = 1;
 }
 
-static AIOPool rbd_aio_pool = {
+static const AIOCBInfo rbd_aiocb_info = {
     .aiocb_size = sizeof(RBDAIOCB),
     .cancel = qemu_rbd_aio_cancel,
 };
@@ -678,7 +672,7 @@ static BlockDriverAIOCB *rbd_start_aio(BlockDriverState *bs,
 
     BDRVRBDState *s = bs->opaque;
 
-    acb = qemu_aio_get(&rbd_aio_pool, bs, cb, opaque);
+    acb = qemu_aio_get(&rbd_aiocb_info, bs, cb, opaque);
     acb->cmd = cmd;
     acb->qiov = qiov;
     if (cmd == RBD_AIO_DISCARD) {
diff --git a/block/sheepdog.c b/block/sheepdog.c
index df4f44107..a48f58cfe 100644
--- a/block/sheepdog.c
+++ b/block/sheepdog.c
@@ -201,12 +201,12 @@ static inline uint64_t fnv_64a_buf(void *buf, size_t len, uint64_t hval)
     return hval;
 }
 
-static inline int is_data_obj_writable(SheepdogInode *inode, unsigned int idx)
+static inline bool is_data_obj_writable(SheepdogInode *inode, unsigned int idx)
 {
     return inode->vdi_id == inode->data_vdi_id[idx];
 }
 
-static inline int is_data_obj(uint64_t oid)
+static inline bool is_data_obj(uint64_t oid)
 {
     return !(VDI_BIT & oid);
 }
@@ -231,7 +231,7 @@ static inline uint64_t vid_to_data_oid(uint32_t vid, uint32_t idx)
     return ((uint64_t)vid << VDI_SPACE_SHIFT) | idx;
 }
 
-static inline int is_snapshot(struct SheepdogInode *inode)
+static inline bool is_snapshot(struct SheepdogInode *inode)
 {
     return !!inode->snap_ctime;
 }
@@ -281,7 +281,7 @@ struct SheepdogAIOCB {
     Coroutine *coroutine;
     void (*aio_done_func)(SheepdogAIOCB *);
 
-    int canceled;
+    bool canceled;
     int nr_pending;
 };
 
@@ -292,8 +292,8 @@ typedef struct BDRVSheepdogState {
     uint32_t max_dirty_data_idx;
 
     char name[SD_MAX_VDI_LEN];
-    int is_snapshot;
-    uint8_t cache_enabled;
+    bool is_snapshot;
+    bool cache_enabled;
 
     char *addr;
     char *port;
@@ -417,10 +417,10 @@ static void sd_aio_cancel(BlockDriverAIOCB *blockacb)
      */
     acb->ret = -EIO;
     qemu_coroutine_enter(acb->coroutine, NULL);
-    acb->canceled = 1;
+    acb->canceled = true;
 }
 
-static AIOPool sd_aio_pool = {
+static const AIOCBInfo sd_aiocb_info = {
     .aiocb_size = sizeof(SheepdogAIOCB),
     .cancel = sd_aio_cancel,
 };
@@ -431,7 +431,7 @@ static SheepdogAIOCB *sd_aio_setup(BlockDriverState *bs, QEMUIOVector *qiov,
 {
     SheepdogAIOCB *acb;
 
-    acb = qemu_aio_get(&sd_aio_pool, bs, cb, opaque);
+    acb = qemu_aio_get(&sd_aiocb_info, bs, cb, opaque);
 
     acb->qiov = qiov;
 
@@ -439,7 +439,7 @@ static SheepdogAIOCB *sd_aio_setup(BlockDriverState *bs, QEMUIOVector *qiov,
     acb->nb_sectors = nb_sectors;
 
     acb->aio_done_func = NULL;
-    acb->canceled = 0;
+    acb->canceled = false;
     acb->coroutine = qemu_coroutine_self();
     acb->ret = 0;
     acb->nr_pending = 0;
@@ -613,7 +613,7 @@ static int do_req(int sockfd, SheepdogReq *hdr, void *data,
 }
 
 static int coroutine_fn add_aio_request(BDRVSheepdogState *s, AIOReq *aio_req,
-                           struct iovec *iov, int niov, int create,
+                           struct iovec *iov, int niov, bool create,
                            enum AIOCBState aiocb_type);
 
 
@@ -646,7 +646,7 @@ static void coroutine_fn send_pending_req(BDRVSheepdogState *s, uint64_t oid)
         QLIST_REMOVE(aio_req, aio_siblings);
         QLIST_INSERT_HEAD(&s->inflight_aio_head, aio_req, aio_siblings);
         ret = add_aio_request(s, aio_req, acb->qiov->iov,
-                              acb->qiov->niov, 0, acb->aiocb_type);
+                              acb->qiov->niov, false, acb->aiocb_type);
         if (ret < 0) {
             error_report("add_aio_request is failed");
             free_aio_req(s, aio_req);
@@ -866,14 +866,14 @@ static int parse_vdiname(BDRVSheepdogState *s, const char *filename,
         s->port = 0;
     }
 
-    strncpy(vdi, p, SD_MAX_VDI_LEN);
+    pstrcpy(vdi, SD_MAX_VDI_LEN, p);
 
     p = strchr(vdi, ':');
     if (p) {
         *p++ = '\0';
         *snapid = strtoul(p, NULL, 10);
         if (*snapid == 0) {
-            strncpy(tag, p, SD_MAX_VDI_TAG_LEN);
+            pstrcpy(tag, SD_MAX_VDI_TAG_LEN, p);
         }
     } else {
         *snapid = CURRENT_VDI_ID; /* search current vdi */
@@ -900,7 +900,10 @@ static int find_vdi_name(BDRVSheepdogState *s, char *filename, uint32_t snapid,
         return fd;
     }
 
-    memset(buf, 0, sizeof(buf));
+    /* This pair of strncpy calls ensures that the buffer is zero-filled,
+     * which is desirable since we'll soon be sending those bytes, and
+     * don't want the send_req to read uninitialized data.
+     */
     strncpy(buf, filename, SD_MAX_VDI_LEN);
     strncpy(buf + SD_MAX_VDI_LEN, tag, SD_MAX_VDI_TAG_LEN);
 
@@ -940,7 +943,7 @@ out:
 }
 
 static int coroutine_fn add_aio_request(BDRVSheepdogState *s, AIOReq *aio_req,
-                           struct iovec *iov, int niov, int create,
+                           struct iovec *iov, int niov, bool create,
                            enum AIOCBState aiocb_type)
 {
     int nr_copies = s->inode.nr_copies;
@@ -1019,7 +1022,7 @@ static int coroutine_fn add_aio_request(BDRVSheepdogState *s, AIOReq *aio_req,
 
 static int read_write_object(int fd, char *buf, uint64_t oid, int copies,
                              unsigned int datalen, uint64_t offset,
-                             int write, int create, uint8_t cache)
+                             bool write, bool create, bool cache)
 {
     SheepdogObjReq hdr;
     SheepdogObjRsp *rsp = (SheepdogObjRsp *)&hdr;
@@ -1068,18 +1071,18 @@ static int read_write_object(int fd, char *buf, uint64_t oid, int copies,
 }
 
 static int read_object(int fd, char *buf, uint64_t oid, int copies,
-                       unsigned int datalen, uint64_t offset, uint8_t cache)
+                       unsigned int datalen, uint64_t offset, bool cache)
 {
-    return read_write_object(fd, buf, oid, copies, datalen, offset, 0, 0,
-                             cache);
+    return read_write_object(fd, buf, oid, copies, datalen, offset, false,
+                             false, cache);
 }
 
 static int write_object(int fd, char *buf, uint64_t oid, int copies,
-                        unsigned int datalen, uint64_t offset, int create,
-                        uint8_t cache)
+                        unsigned int datalen, uint64_t offset, bool create,
+                        bool cache)
 {
-    return read_write_object(fd, buf, oid, copies, datalen, offset, 1, create,
-                             cache);
+    return read_write_object(fd, buf, oid, copies, datalen, offset, true,
+                             create, cache);
 }
 
 static int sd_open(BlockDriverState *bs, const char *filename, int flags)
@@ -1114,19 +1117,17 @@ static int sd_open(BlockDriverState *bs, const char *filename, int flags)
         goto out;
     }
 
-    if (flags & BDRV_O_CACHE_WB) {
-        s->cache_enabled = 1;
-        s->flush_fd = connect_to_sdog(s->addr, s->port);
-        if (s->flush_fd < 0) {
-            error_report("failed to connect");
-            ret = s->flush_fd;
-            goto out;
-        }
+    s->cache_enabled = true;
+    s->flush_fd = connect_to_sdog(s->addr, s->port);
+    if (s->flush_fd < 0) {
+        error_report("failed to connect");
+        ret = s->flush_fd;
+        goto out;
     }
 
     if (snapid || tag[0] != '\0') {
         dprintf("%" PRIx32 " snapshot inode was open.\n", vid);
-        s->is_snapshot = 1;
+        s->is_snapshot = true;
     }
 
     fd = connect_to_sdog(s->addr, s->port);
@@ -1151,7 +1152,7 @@ static int sd_open(BlockDriverState *bs, const char *filename, int flags)
     s->max_dirty_data_idx = 0;
 
     bs->total_sectors = s->inode.vdi_size / SECTOR_SIZE;
-    strncpy(s->name, vdi, sizeof(s->name));
+    pstrcpy(s->name, sizeof(s->name), vdi);
     qemu_co_mutex_init(&s->lock);
     g_free(buf);
     return 0;
@@ -1179,8 +1180,11 @@ static int do_sd_create(char *filename, int64_t vdi_size,
         return fd;
     }
 
+    /* FIXME: would it be better to fail (e.g., return -EIO) when filename
+     * does not fit in buf?  For now, just truncate and avoid buffer overrun.
+     */
     memset(buf, 0, sizeof(buf));
-    strncpy(buf, filename, SD_MAX_VDI_LEN);
+    pstrcpy(buf, sizeof(buf), filename);
 
     memset(&hdr, 0, sizeof(hdr));
     hdr.opcode = SD_OP_NEW_VDI;
@@ -1266,7 +1270,7 @@ static int sd_create(const char *filename, QEMUOptionParameter *options)
     BDRVSheepdogState *s;
     char vdi[SD_MAX_VDI_LEN], tag[SD_MAX_VDI_TAG_LEN];
     uint32_t snapid;
-    int prealloc = 0;
+    bool prealloc = false;
     const char *vdiname;
 
     s = g_malloc0(sizeof(BDRVSheepdogState));
@@ -1288,9 +1292,9 @@ static int sd_create(const char *filename, QEMUOptionParameter *options)
             backing_file = options->value.s;
         } else if (!strcmp(options->name, BLOCK_OPT_PREALLOC)) {
             if (!options->value.s || !strcmp(options->value.s, "off")) {
-                prealloc = 0;
+                prealloc = false;
             } else if (!strcmp(options->value.s, "full")) {
-                prealloc = 1;
+                prealloc = true;
             } else {
                 error_report("Invalid preallocation mode: '%s'",
                              options->value.s);
@@ -1418,7 +1422,7 @@ static int sd_truncate(BlockDriverState *bs, int64_t offset)
     datalen = SD_INODE_SIZE - sizeof(s->inode.data_vdi_id);
     s->inode.vdi_size = offset;
     ret = write_object(fd, (char *)&s->inode, vid_to_vdi_oid(s->inode.vdi_id),
-                       s->inode.nr_copies, datalen, 0, 0, s->cache_enabled);
+                       s->inode.nr_copies, datalen, 0, false, s->cache_enabled);
     close(fd);
 
     if (ret < 0) {
@@ -1457,7 +1461,7 @@ static void coroutine_fn sd_write_done(SheepdogAIOCB *acb)
         aio_req = alloc_aio_req(s, acb, vid_to_vdi_oid(s->inode.vdi_id),
                                 data_len, offset, 0, 0, offset);
         QLIST_INSERT_HEAD(&s->inflight_aio_head, aio_req, aio_siblings);
-        ret = add_aio_request(s, aio_req, &iov, 1, 0, AIOCB_WRITE_UDATA);
+        ret = add_aio_request(s, aio_req, &iov, 1, false, AIOCB_WRITE_UDATA);
         if (ret) {
             free_aio_req(s, aio_req);
             acb->ret = -EIO;
@@ -1511,7 +1515,7 @@ static int sd_create_branch(BDRVSheepdogState *s)
 
     memcpy(&s->inode, buf, sizeof(s->inode));
 
-    s->is_snapshot = 0;
+    s->is_snapshot = false;
     ret = 0;
     dprintf("%" PRIx32 " was newly created.\n", s->inode.vdi_id);
 
@@ -1566,7 +1570,7 @@ static int coroutine_fn sd_co_rw_vector(void *p)
     while (done != total) {
         uint8_t flags = 0;
         uint64_t old_oid = 0;
-        int create = 0;
+        bool create = false;
 
         oid = vid_to_data_oid(inode->data_vdi_id[idx], idx);
 
@@ -1581,10 +1585,10 @@ static int coroutine_fn sd_co_rw_vector(void *p)
             break;
         case AIOCB_WRITE_UDATA:
             if (!inode->data_vdi_id[idx]) {
-                create = 1;
+                create = true;
             } else if (!is_data_obj_writable(inode, idx)) {
                 /* Copy-On-Write */
-                create = 1;
+                create = true;
                 old_oid = oid;
                 flags = SD_FLAG_CMD_COW;
             }
@@ -1718,7 +1722,7 @@ static int coroutine_fn sd_co_flush_to_disk(BlockDriverState *bs)
     if (rsp->result == SD_RES_INVALID_PARMS) {
         dprintf("disable write cache since the server doesn't support it\n");
 
-        s->cache_enabled = 0;
+        s->cache_enabled = false;
         closesocket(s->flush_fd);
         return 0;
     }
@@ -1754,6 +1758,9 @@ static int sd_snapshot_create(BlockDriverState *bs, QEMUSnapshotInfo *sn_info)
 
     s->inode.vm_state_size = sn_info->vm_state_size;
     s->inode.vm_clock_nsec = sn_info->vm_clock_nsec;
+    /* It appears that inode.tag does not require a NUL terminator,
+     * which means this use of strncpy is ok.
+     */
     strncpy(s->inode.tag, sn_info->name, sizeof(s->inode.tag));
     /* we don't need to update entire object */
     datalen = SD_INODE_SIZE - sizeof(s->inode.data_vdi_id);
@@ -1766,7 +1773,7 @@ static int sd_snapshot_create(BlockDriverState *bs, QEMUSnapshotInfo *sn_info)
     }
 
     ret = write_object(fd, (char *)&s->inode, vid_to_vdi_oid(s->inode.vdi_id),
-                       s->inode.nr_copies, datalen, 0, 0, s->cache_enabled);
+                       s->inode.nr_copies, datalen, 0, false, s->cache_enabled);
     if (ret < 0) {
         error_report("failed to write snapshot's inode.");
         goto cleanup;
@@ -1813,13 +1820,13 @@ static int sd_snapshot_goto(BlockDriverState *bs, const char *snapshot_id)
 
     memcpy(old_s, s, sizeof(BDRVSheepdogState));
 
-    memset(vdi, 0, sizeof(vdi));
-    strncpy(vdi, s->name, sizeof(vdi));
+    pstrcpy(vdi, sizeof(vdi), s->name);
 
-    memset(tag, 0, sizeof(tag));
     snapid = strtoul(snapshot_id, NULL, 10);
-    if (!snapid) {
-        strncpy(tag, s->name, sizeof(tag));
+    if (snapid) {
+        tag[0] = 0;
+    } else {
+        pstrcpy(tag, sizeof(tag), s->name);
     }
 
     ret = find_vdi_name(s, vdi, snapid, tag, &vid, 1);
@@ -1853,7 +1860,7 @@ static int sd_snapshot_goto(BlockDriverState *bs, const char *snapshot_id)
         goto out;
     }
 
-    s->is_snapshot = 1;
+    s->is_snapshot = true;
 
     g_free(buf);
     g_free(old_s);
@@ -1948,8 +1955,9 @@ static int sd_snapshot_list(BlockDriverState *bs, QEMUSnapshotInfo **psn_tab)
 
             snprintf(sn_tab[found].id_str, sizeof(sn_tab[found].id_str), "%u",
                      inode.snap_id);
-            strncpy(sn_tab[found].name, inode.tag,
-                    MIN(sizeof(sn_tab[found].name), sizeof(inode.tag)));
+            pstrcpy(sn_tab[found].name,
+                    MIN(sizeof(sn_tab[found].name), sizeof(inode.tag)),
+                    inode.tag);
             found++;
         }
     }
@@ -1970,8 +1978,8 @@ out:
 static int do_load_save_vmstate(BDRVSheepdogState *s, uint8_t *data,
                                 int64_t pos, int size, int load)
 {
-    int fd, create;
-    int ret = 0, remaining = size;
+    bool create;
+    int fd, ret = 0, remaining = size;
     unsigned int data_len;
     uint64_t vmstate_oid;
     uint32_t vdi_index;
@@ -1986,7 +1994,7 @@ static int do_load_save_vmstate(BDRVSheepdogState *s, uint8_t *data,
         vdi_index = pos / SD_DATA_OBJ_SIZE;
         offset = pos % SD_DATA_OBJ_SIZE;
 
-        data_len = MIN(remaining, SD_DATA_OBJ_SIZE);
+        data_len = MIN(remaining, SD_DATA_OBJ_SIZE - offset);
 
         vmstate_oid = vid_to_vmstate_oid(s->inode.vdi_id, vdi_index);
 
@@ -2007,6 +2015,7 @@ static int do_load_save_vmstate(BDRVSheepdogState *s, uint8_t *data,
         }
 
         pos += data_len;
+        data += data_len;
         remaining -= data_len;
     }
     ret = size;
diff --git a/block/stream.c b/block/stream.c
index c4f87dd5b..0c0fc7a13 100644
--- a/block/stream.c
+++ b/block/stream.c
@@ -13,6 +13,7 @@
 
 #include "trace.h"
 #include "block_int.h"
+#include "blockjob.h"
 #include "qemu/ratelimit.h"
 
 enum {
@@ -30,6 +31,7 @@ typedef struct StreamBlockJob {
     BlockJob common;
     RateLimit limit;
     BlockDriverState *base;
+    BlockdevOnError on_error;
     char backing_file_id[1024];
 } StreamBlockJob;
 
@@ -77,13 +79,14 @@ static void coroutine_fn stream_run(void *opaque)
     BlockDriverState *bs = s->common.bs;
     BlockDriverState *base = s->base;
     int64_t sector_num, end;
+    int error = 0;
     int ret = 0;
     int n = 0;
     void *buf;
 
     s->common.len = bdrv_getlength(bs);
     if (s->common.len < 0) {
-        block_job_complete(&s->common, s->common.len);
+        block_job_completed(&s->common, s->common.len);
         return;
     }
 
@@ -141,7 +144,19 @@ wait:
             ret = stream_populate(bs, sector_num, n, buf);
         }
         if (ret < 0) {
-            break;
+            BlockErrorAction action =
+                block_job_error_action(&s->common, s->common.bs, s->on_error,
+                                       true, -ret);
+            if (action == BDRV_ACTION_STOP) {
+                n = 0;
+                continue;
+            }
+            if (error == 0) {
+                error = ret;
+            }
+            if (action == BDRV_ACTION_REPORT) {
+                break;
+            }
         }
         ret = 0;
 
@@ -153,6 +168,9 @@ wait:
         bdrv_disable_copy_on_read(bs);
     }
 
+    /* Do not remove the backing file if an error was there but ignored.  */
+    ret = error;
+
     if (!block_job_is_cancelled(&s->common) && sector_num == end && ret == 0) {
         const char *base_id = NULL, *base_fmt = NULL;
         if (base) {
@@ -166,7 +184,7 @@ wait:
     }
 
     qemu_vfree(buf);
-    block_job_complete(&s->common, ret);
+    block_job_completed(&s->common, ret);
 }
 
 static void stream_set_speed(BlockJob *job, int64_t speed, Error **errp)
@@ -188,11 +206,19 @@ static BlockJobType stream_job_type = {
 
 void stream_start(BlockDriverState *bs, BlockDriverState *base,
                   const char *base_id, int64_t speed,
+                  BlockdevOnError on_error,
                   BlockDriverCompletionFunc *cb,
                   void *opaque, Error **errp)
 {
     StreamBlockJob *s;
 
+    if ((on_error == BLOCKDEV_ON_ERROR_STOP ||
+         on_error == BLOCKDEV_ON_ERROR_ENOSPC) &&
+        !bdrv_iostatus_is_enabled(bs)) {
+        error_set(errp, QERR_INVALID_PARAMETER, "on-error");
+        return;
+    }
+
     s = block_job_create(&stream_job_type, bs, speed, cb, opaque, errp);
     if (!s) {
         return;
@@ -203,6 +229,7 @@ void stream_start(BlockDriverState *bs, BlockDriverState *base,
         pstrcpy(s->backing_file_id, sizeof(s->backing_file_id), base_id);
     }
 
+    s->on_error = on_error;
     s->common.co = qemu_coroutine_create(stream_run);
     trace_stream_start(bs, base, s, s->common.co, opaque);
     qemu_coroutine_enter(s->common.co, s);
diff --git a/block/vdi.c b/block/vdi.c
index c4f1529db..c8330b7ea 100644
--- a/block/vdi.c
+++ b/block/vdi.c
@@ -60,9 +60,6 @@
 /* TODO: move uuid emulation to some central place in QEMU. */
 #include "sysemu.h"     /* UUID_FMT */
 typedef unsigned char uuid_t[16];
-void uuid_generate(uuid_t out);
-int uuid_is_null(const uuid_t uu);
-void uuid_unparse(const uuid_t uu, char *out);
 #endif
 
 /* Code configuration options. */
@@ -124,18 +121,18 @@ void uuid_unparse(const uuid_t uu, char *out);
 #define VDI_IS_ALLOCATED(X) ((X) < VDI_DISCARDED)
 
 #if !defined(CONFIG_UUID)
-void uuid_generate(uuid_t out)
+static inline void uuid_generate(uuid_t out)
 {
     memset(out, 0, sizeof(uuid_t));
 }
 
-int uuid_is_null(const uuid_t uu)
+static inline int uuid_is_null(const uuid_t uu)
 {
     uuid_t null_uuid = { 0 };
     return memcmp(uu, null_uuid, sizeof(uuid_t)) == 0;
 }
 
-void uuid_unparse(const uuid_t uu, char *out)
+static inline void uuid_unparse(const uuid_t uu, char *out)
 {
     snprintf(out, 37, UUID_FMT,
             uu[0], uu[1], uu[2], uu[3], uu[4], uu[5], uu[6], uu[7],
@@ -454,6 +451,12 @@ static int vdi_open(BlockDriverState *bs, int flags)
     return -1;
 }
 
+static int vdi_reopen_prepare(BDRVReopenState *state,
+                              BlockReopenQueue *queue, Error **errp)
+{
+    return 0;
+}
+
 static int coroutine_fn vdi_co_is_allocated(BlockDriverState *bs,
         int64_t sector_num, int nb_sectors, int *pnum)
 {
@@ -628,7 +631,6 @@ static int vdi_create(const char *filename, QEMUOptionParameter *options)
     VdiHeader header;
     size_t i;
     size_t bmap_size;
-    uint32_t *bmap;
 
     logout("\n");
 
@@ -693,21 +695,21 @@ static int vdi_create(const char *filename, QEMUOptionParameter *options)
         result = -errno;
     }
 
-    bmap = NULL;
     if (bmap_size > 0) {
-        bmap = (uint32_t *)g_malloc0(bmap_size);
-    }
-    for (i = 0; i < blocks; i++) {
-        if (image_type == VDI_TYPE_STATIC) {
-            bmap[i] = i;
-        } else {
-            bmap[i] = VDI_UNALLOCATED;
+        uint32_t *bmap = g_malloc0(bmap_size);
+        for (i = 0; i < blocks; i++) {
+            if (image_type == VDI_TYPE_STATIC) {
+                bmap[i] = i;
+            } else {
+                bmap[i] = VDI_UNALLOCATED;
+            }
         }
+        if (write(fd, bmap, bmap_size) < 0) {
+            result = -errno;
+        }
+        g_free(bmap);
     }
-    if (write(fd, bmap, bmap_size) < 0) {
-        result = -errno;
-    }
-    g_free(bmap);
+
     if (image_type == VDI_TYPE_STATIC) {
         if (ftruncate(fd, sizeof(header) + bmap_size + blocks * block_size)) {
             result = -errno;
@@ -762,6 +764,7 @@ static BlockDriver bdrv_vdi = {
     .bdrv_probe = vdi_probe,
     .bdrv_open = vdi_open,
     .bdrv_close = vdi_close,
+    .bdrv_reopen_prepare = vdi_reopen_prepare,
     .bdrv_create = vdi_create,
     .bdrv_co_is_allocated = vdi_co_is_allocated,
     .bdrv_make_empty = vdi_make_empty,
diff --git a/block/vmdk.c b/block/vmdk.c
index bba4c61a7..51398c0c0 100644
--- a/block/vmdk.c
+++ b/block/vmdk.c
@@ -300,6 +300,40 @@ static int vmdk_is_cid_valid(BlockDriverState *bs)
     return 1;
 }
 
+/* Queue extents, if any, for reopen() */
+static int vmdk_reopen_prepare(BDRVReopenState *state,
+                               BlockReopenQueue *queue, Error **errp)
+{
+    BDRVVmdkState *s;
+    int ret = -1;
+    int i;
+    VmdkExtent *e;
+
+    assert(state != NULL);
+    assert(state->bs != NULL);
+
+    if (queue == NULL) {
+        error_set(errp, ERROR_CLASS_GENERIC_ERROR,
+                 "No reopen queue for VMDK extents");
+        goto exit;
+    }
+
+    s = state->bs->opaque;
+
+    assert(s != NULL);
+
+    for (i = 0; i < s->num_extents; i++) {
+        e = &s->extents[i];
+        if (e->file != state->bs->file) {
+            bdrv_reopen_queue(queue, e->file, state->flags);
+        }
+    }
+    ret = 0;
+
+exit:
+    return ret;
+}
+
 static int vmdk_parent_open(BlockDriverState *bs)
 {
     char *p_name;
@@ -1058,6 +1092,7 @@ static int vmdk_read(BlockDriverState *bs, int64_t sector_num,
     BDRVVmdkState *s = bs->opaque;
     int ret;
     uint64_t n, index_in_cluster;
+    uint64_t extent_begin_sector, extent_relative_sector_num;
     VmdkExtent *extent = NULL;
     uint64_t cluster_offset;
 
@@ -1069,7 +1104,9 @@ static int vmdk_read(BlockDriverState *bs, int64_t sector_num,
         ret = get_cluster_offset(
                             bs, extent, NULL,
                             sector_num << 9, 0, &cluster_offset);
-        index_in_cluster = sector_num % extent->cluster_sectors;
+        extent_begin_sector = extent->end_sector - extent->sectors;
+        extent_relative_sector_num = sector_num - extent_begin_sector;
+        index_in_cluster = extent_relative_sector_num % extent->cluster_sectors;
         n = extent->cluster_sectors - index_in_cluster;
         if (n > nb_sectors) {
             n = nb_sectors;
@@ -1120,6 +1157,7 @@ static int vmdk_write(BlockDriverState *bs, int64_t sector_num,
     VmdkExtent *extent = NULL;
     int n, ret;
     int64_t index_in_cluster;
+    uint64_t extent_begin_sector, extent_relative_sector_num;
     uint64_t cluster_offset;
     VmdkMetaData m_data;
 
@@ -1162,7 +1200,9 @@ static int vmdk_write(BlockDriverState *bs, int64_t sector_num,
         if (ret) {
             return -EINVAL;
         }
-        index_in_cluster = sector_num % extent->cluster_sectors;
+        extent_begin_sector = extent->end_sector - extent->sectors;
+        extent_relative_sector_num = sector_num - extent_begin_sector;
+        index_in_cluster = extent_relative_sector_num % extent->cluster_sectors;
         n = extent->cluster_sectors - index_in_cluster;
         if (n > nb_sectors) {
             n = nb_sectors;
@@ -1374,8 +1414,7 @@ static int relative_path(char *dest, int dest_size,
         return -1;
     }
     if (path_is_absolute(target)) {
-        dest[dest_size - 1] = '\0';
-        strncpy(dest, target, dest_size - 1);
+        pstrcpy(dest, dest_size, target);
         return 0;
     }
     while (base[i] == target[i]) {
@@ -1646,6 +1685,7 @@ static BlockDriver bdrv_vmdk = {
     .instance_size  = sizeof(BDRVVmdkState),
     .bdrv_probe     = vmdk_probe,
     .bdrv_open      = vmdk_open,
+    .bdrv_reopen_prepare = vmdk_reopen_prepare,
     .bdrv_read      = vmdk_co_read,
     .bdrv_write     = vmdk_co_write,
     .bdrv_close     = vmdk_close,
diff --git a/block/vpc.c b/block/vpc.c
index c0b82c4f5..b6bf52f14 100644
--- a/block/vpc.c
+++ b/block/vpc.c
@@ -265,6 +265,12 @@ static int vpc_open(BlockDriverState *bs, int flags)
     return err;
 }
 
+static int vpc_reopen_prepare(BDRVReopenState *state,
+                              BlockReopenQueue *queue, Error **errp)
+{
+    return 0;
+}
+
 /*
  * Returns the absolute byte offset of the given sector in the image file.
  * If the sector is not allocated, -1 is returned instead.
@@ -783,6 +789,7 @@ static BlockDriver bdrv_vpc = {
     .bdrv_probe     = vpc_probe,
     .bdrv_open      = vpc_open,
     .bdrv_close     = vpc_close,
+    .bdrv_reopen_prepare = vpc_reopen_prepare,
     .bdrv_create    = vpc_create,
 
     .bdrv_read              = vpc_co_read,
diff --git a/block/win32-aio.c b/block/win32-aio.c
new file mode 100644
index 000000000..4704ee06c
--- /dev/null
+++ b/block/win32-aio.c
@@ -0,0 +1,226 @@
+/*
+ * Block driver for RAW files (win32)
+ *
+ * Copyright (c) 2006 Fabrice Bellard
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#include "qemu-common.h"
+#include "qemu-timer.h"
+#include "block_int.h"
+#include "module.h"
+#include "qemu-common.h"
+#include "qemu-aio.h"
+#include "raw-aio.h"
+#include "event_notifier.h"
+#include <windows.h>
+#include <winioctl.h>
+
+#define FTYPE_FILE 0
+#define FTYPE_CD     1
+#define FTYPE_HARDDISK 2
+
+struct QEMUWin32AIOState {
+    HANDLE hIOCP;
+    EventNotifier e;
+    int count;
+};
+
+typedef struct QEMUWin32AIOCB {
+    BlockDriverAIOCB common;
+    struct QEMUWin32AIOState *ctx;
+    int nbytes;
+    OVERLAPPED ov;
+    QEMUIOVector *qiov;
+    void *buf;
+    bool is_read;
+    bool is_linear;
+} QEMUWin32AIOCB;
+
+/*
+ * Completes an AIO request (calls the callback and frees the ACB).
+ */
+static void win32_aio_process_completion(QEMUWin32AIOState *s,
+    QEMUWin32AIOCB *waiocb, DWORD count)
+{
+    int ret;
+    s->count--;
+
+    if (waiocb->ov.Internal != 0) {
+        ret = -EIO;
+    } else {
+        ret = 0;
+        if (count < waiocb->nbytes) {
+            /* Short reads mean EOF, pad with zeros. */
+            if (waiocb->is_read) {
+                qemu_iovec_memset(waiocb->qiov, count, 0,
+                    waiocb->qiov->size - count);
+            } else {
+                ret = -EINVAL;
+            }
+       }
+    }
+
+    if (!waiocb->is_linear) {
+        if (ret == 0 && waiocb->is_read) {
+            QEMUIOVector *qiov = waiocb->qiov;
+            char *p = waiocb->buf;
+            int i;
+
+            for (i = 0; i < qiov->niov; ++i) {
+                memcpy(p, qiov->iov[i].iov_base, qiov->iov[i].iov_len);
+                p += qiov->iov[i].iov_len;
+            }
+            g_free(waiocb->buf);
+        }
+    }
+
+
+    waiocb->common.cb(waiocb->common.opaque, ret);
+    qemu_aio_release(waiocb);
+}
+
+static void win32_aio_completion_cb(EventNotifier *e)
+{
+    QEMUWin32AIOState *s = container_of(e, QEMUWin32AIOState, e);
+    DWORD count;
+    ULONG_PTR key;
+    OVERLAPPED *ov;
+
+    event_notifier_test_and_clear(&s->e);
+    while (GetQueuedCompletionStatus(s->hIOCP, &count, &key, &ov, 0)) {
+        QEMUWin32AIOCB *waiocb = container_of(ov, QEMUWin32AIOCB, ov);
+
+        win32_aio_process_completion(s, waiocb, count);
+    }
+}
+
+static int win32_aio_flush_cb(EventNotifier *e)
+{
+    QEMUWin32AIOState *s = container_of(e, QEMUWin32AIOState, e);
+
+    return (s->count > 0) ? 1 : 0;
+}
+
+static void win32_aio_cancel(BlockDriverAIOCB *blockacb)
+{
+    QEMUWin32AIOCB *waiocb = (QEMUWin32AIOCB *)blockacb;
+
+    /*
+     * CancelIoEx is only supported in Vista and newer.  For now, just
+     * wait for completion.
+     */
+    while (!HasOverlappedIoCompleted(&waiocb->ov)) {
+        qemu_aio_wait();
+    }
+}
+
+static const AIOCBInfo win32_aiocb_info = {
+    .aiocb_size         = sizeof(QEMUWin32AIOCB),
+    .cancel             = win32_aio_cancel,
+};
+
+BlockDriverAIOCB *win32_aio_submit(BlockDriverState *bs,
+        QEMUWin32AIOState *aio, HANDLE hfile,
+        int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
+        BlockDriverCompletionFunc *cb, void *opaque, int type)
+{
+    struct QEMUWin32AIOCB *waiocb;
+    uint64_t offset = sector_num * 512;
+    DWORD rc;
+
+    waiocb = qemu_aio_get(&win32_aiocb_info, bs, cb, opaque);
+    waiocb->nbytes = nb_sectors * 512;
+    waiocb->qiov = qiov;
+    waiocb->is_read = (type == QEMU_AIO_READ);
+
+    if (qiov->niov > 1) {
+        waiocb->buf = qemu_blockalign(bs, qiov->size);
+        if (type & QEMU_AIO_WRITE) {
+            char *p = waiocb->buf;
+            int i;
+
+            for (i = 0; i < qiov->niov; ++i) {
+                memcpy(p, qiov->iov[i].iov_base, qiov->iov[i].iov_len);
+                p += qiov->iov[i].iov_len;
+            }
+        }
+        waiocb->is_linear = false;
+    } else {
+        waiocb->buf = qiov->iov[0].iov_base;
+        waiocb->is_linear = true;
+    }
+
+    memset(&waiocb->ov, 0, sizeof(waiocb->ov));
+    waiocb->ov.Offset = (DWORD)offset;
+    waiocb->ov.OffsetHigh = (DWORD)(offset >> 32);
+    waiocb->ov.hEvent = event_notifier_get_handle(&aio->e);
+
+    aio->count++;
+
+    if (type & QEMU_AIO_READ) {
+        rc = ReadFile(hfile, waiocb->buf, waiocb->nbytes, NULL, &waiocb->ov);
+    } else {
+        rc = WriteFile(hfile, waiocb->buf, waiocb->nbytes, NULL, &waiocb->ov);
+    }
+    if(rc == 0 && GetLastError() != ERROR_IO_PENDING) {
+        goto out_dec_count;
+    }
+    return &waiocb->common;
+
+out_dec_count:
+    aio->count--;
+    qemu_aio_release(waiocb);
+    return NULL;
+}
+
+int win32_aio_attach(QEMUWin32AIOState *aio, HANDLE hfile)
+{
+    if (CreateIoCompletionPort(hfile, aio->hIOCP, (ULONG_PTR) 0, 0) == NULL) {
+        return -EINVAL;
+    } else {
+        return 0;
+    }
+}
+
+QEMUWin32AIOState *win32_aio_init(void)
+{
+    QEMUWin32AIOState *s;
+
+    s = g_malloc0(sizeof(*s));
+    if (event_notifier_init(&s->e, false) < 0) {
+        goto out_free_state;
+    }
+
+    s->hIOCP = CreateIoCompletionPort(INVALID_HANDLE_VALUE, NULL, 0, 0);
+    if (s->hIOCP == NULL) {
+        goto out_close_efd;
+    }
+
+    qemu_aio_set_event_notifier(&s->e, win32_aio_completion_cb,
+                                win32_aio_flush_cb);
+
+    return s;
+
+out_close_efd:
+    event_notifier_cleanup(&s->e);
+out_free_state:
+    g_free(s);
+    return NULL;
+}
diff --git a/block_int.h b/block_int.h
index 4452f6f39..9deedb811 100644
--- a/block_int.h
+++ b/block_int.h
@@ -31,6 +31,7 @@
 #include "qemu-timer.h"
 #include "qapi-types.h"
 #include "qerror.h"
+#include "monitor.h"
 
 #define BLOCK_FLAG_ENCRYPT          1
 #define BLOCK_FLAG_COMPAT6          4
@@ -67,78 +68,18 @@ typedef struct BlockIOBaseValue {
     uint64_t ios[2];
 } BlockIOBaseValue;
 
-typedef struct BlockJob BlockJob;
-
-/**
- * BlockJobType:
- *
- * A class type for block job objects.
- */
-typedef struct BlockJobType {
-    /** Derived BlockJob struct size */
-    size_t instance_size;
-
-    /** String describing the operation, part of query-block-jobs QMP API */
-    const char *job_type;
-
-    /** Optional callback for job types that support setting a speed limit */
-    void (*set_speed)(BlockJob *job, int64_t speed, Error **errp);
-} BlockJobType;
-
-/**
- * BlockJob:
- *
- * Long-running operation on a BlockDriverState.
- */
-struct BlockJob {
-    /** The job type, including the job vtable.  */
-    const BlockJobType *job_type;
-
-    /** The block device on which the job is operating.  */
-    BlockDriverState *bs;
-
-    /**
-     * The coroutine that executes the job.  If not NULL, it is
-     * reentered when busy is false and the job is cancelled.
-     */
-    Coroutine *co;
-
-    /**
-     * Set to true if the job should cancel itself.  The flag must
-     * always be tested just before toggling the busy flag from false
-     * to true.  After a job has been cancelled, it should only yield
-     * if #qemu_aio_wait will ("sooner or later") reenter the coroutine.
-     */
-    bool cancelled;
-
-    /**
-     * Set to false by the job while it is in a quiescent state, where
-     * no I/O is pending and the job has yielded on any condition
-     * that is not detected by #qemu_aio_wait, such as a timer.
-     */
-    bool busy;
-
-    /** Offset that is published by the query-block-jobs QMP API */
-    int64_t offset;
-
-    /** Length that is published by the query-block-jobs QMP API */
-    int64_t len;
-
-    /** Speed that was set with @block_job_set_speed.  */
-    int64_t speed;
-
-    /** The completion function that will be called when the job completes.  */
-    BlockDriverCompletionFunc *cb;
-
-    /** The opaque value that is passed to the completion function.  */
-    void *opaque;
-};
-
 struct BlockDriver {
     const char *format_name;
     int instance_size;
     int (*bdrv_probe)(const uint8_t *buf, int buf_size, const char *filename);
     int (*bdrv_probe_device)(const char *filename);
+
+    /* For handling image reopen for split or non-split files */
+    int (*bdrv_reopen_prepare)(BDRVReopenState *reopen_state,
+                               BlockReopenQueue *queue, Error **errp);
+    void (*bdrv_reopen_commit)(BDRVReopenState *reopen_state);
+    void (*bdrv_reopen_abort)(BDRVReopenState *reopen_state);
+
     int (*bdrv_open)(BlockDriverState *bs, int flags);
     int (*bdrv_file_open)(BlockDriverState *bs, const char *filename, int flags);
     int (*bdrv_read)(BlockDriverState *bs, int64_t sector_num,
@@ -268,7 +209,6 @@ struct BlockDriverState {
     int64_t total_sectors; /* if we are reading a disk image, give its
                               size in sectors */
     int read_only; /* if true, the media is read only */
-    int keep_read_only; /* if true, the media was requested to stay read only */
     int open_flags; /* flags used to open the file, re-used for re-open */
     int encrypted; /* if true, the media is encrypted */
     int valid_key; /* if true, a valid encryption key has been set */
@@ -293,6 +233,8 @@ struct BlockDriverState {
     BlockDriverState *backing_hd;
     BlockDriverState *file;
 
+    NotifierList close_notifiers;
+
     /* number of in-flight copy-on-read requests */
     unsigned int copy_on_read_in_flight;
 
@@ -323,7 +265,7 @@ struct BlockDriverState {
 
     /* NOTE: the following infos are only hints for real hardware
        drivers. They are not used by the block driver */
-    BlockErrorAction on_read_error, on_write_error;
+    BlockdevOnError on_read_error, on_write_error;
     bool iostatus_enabled;
     BlockDeviceIoStatus iostatus;
     char device_name[32];
@@ -336,6 +278,7 @@ struct BlockDriverState {
 
     /* long-running background operation */
     BlockJob *job;
+
 };
 
 int get_tmp_filename(char *filename, int size);
@@ -346,113 +289,71 @@ void bdrv_set_io_limits(BlockDriverState *bs,
 #ifdef _WIN32
 int is_windows_drive(const char *filename);
 #endif
+void bdrv_emit_qmp_error_event(const BlockDriverState *bdrv,
+                               enum MonitorEvent ev,
+                               BlockErrorAction action, bool is_read);
 
 /**
- * block_job_create:
- * @job_type: The class object for the newly-created job.
- * @bs: The block
+ * stream_start:
+ * @bs: Block device to operate on.
+ * @base: Block device that will become the new base, or %NULL to
+ * flatten the whole backing file chain onto @bs.
+ * @base_id: The file name that will be written to @bs as the new
+ * backing file if the job completes.  Ignored if @base is %NULL.
  * @speed: The maximum speed, in bytes per second, or 0 for unlimited.
+ * @on_error: The action to take upon error.
  * @cb: Completion function for the job.
  * @opaque: Opaque pointer value passed to @cb.
  * @errp: Error object.
  *
- * Create a new long-running block device job and return it.  The job
- * will call @cb asynchronously when the job completes.  Note that
- * @bs may have been closed at the time the @cb it is called.  If
- * this is the case, the job may be reported as either cancelled or
- * completed.
- *
- * This function is not part of the public job interface; it should be
- * called from a wrapper that is specific to the job type.
- */
-void *block_job_create(const BlockJobType *job_type, BlockDriverState *bs,
-                       int64_t speed, BlockDriverCompletionFunc *cb,
-                       void *opaque, Error **errp);
-
-/**
- * block_job_sleep_ns:
- * @job: The job that calls the function.
- * @clock: The clock to sleep on.
- * @ns: How many nanoseconds to stop for.
- *
- * Put the job to sleep (assuming that it wasn't canceled) for @ns
- * nanoseconds.  Canceling the job will interrupt the wait immediately.
- */
-void block_job_sleep_ns(BlockJob *job, QEMUClock *clock, int64_t ns);
-
-/**
- * block_job_complete:
- * @job: The job being completed.
- * @ret: The status code.
- *
- * Call the completion function that was registered at creation time, and
- * free @job.
+ * Start a streaming operation on @bs.  Clusters that are unallocated
+ * in @bs, but allocated in any image between @base and @bs (both
+ * exclusive) will be written to @bs.  At the end of a successful
+ * streaming job, the backing file of @bs will be changed to
+ * @base_id in the written image and to @base in the live BlockDriverState.
  */
-void block_job_complete(BlockJob *job, int ret);
+void stream_start(BlockDriverState *bs, BlockDriverState *base,
+                  const char *base_id, int64_t speed, BlockdevOnError on_error,
+                  BlockDriverCompletionFunc *cb,
+                  void *opaque, Error **errp);
 
 /**
- * block_job_set_speed:
- * @job: The job to set the speed for.
- * @speed: The new value
+ * commit_start:
+ * @bs: Top Block device
+ * @base: Block device that will be written into, and become the new top
+ * @speed: The maximum speed, in bytes per second, or 0 for unlimited.
+ * @on_error: The action to take upon error.
+ * @cb: Completion function for the job.
+ * @opaque: Opaque pointer value passed to @cb.
  * @errp: Error object.
  *
- * Set a rate-limiting parameter for the job; the actual meaning may
- * vary depending on the job type.
- */
-void block_job_set_speed(BlockJob *job, int64_t speed, Error **errp);
-
-/**
- * block_job_cancel:
- * @job: The job to be canceled.
- *
- * Asynchronously cancel the specified job.
- */
-void block_job_cancel(BlockJob *job);
-
-/**
- * block_job_is_cancelled:
- * @job: The job being queried.
- *
- * Returns whether the job is scheduled for cancellation.
- */
-bool block_job_is_cancelled(BlockJob *job);
-
-/**
- * block_job_cancel:
- * @job: The job to be canceled.
- *
- * Asynchronously cancel the job and wait for it to reach a quiescent
- * state.  Note that the completion callback will still be called
- * asynchronously, hence it is *not* valid to call #bdrv_delete
- * immediately after #block_job_cancel_sync.  Users of block jobs
- * will usually protect the BlockDriverState objects with a reference
- * count, should this be a concern.
- *
- * Returns the return value from the job if the job actually completed
- * during the call, or -ECANCELED if it was canceled.
  */
-int block_job_cancel_sync(BlockJob *job);
+void commit_start(BlockDriverState *bs, BlockDriverState *base,
+                 BlockDriverState *top, int64_t speed,
+                 BlockdevOnError on_error, BlockDriverCompletionFunc *cb,
+                 void *opaque, Error **errp);
 
-/**
- * stream_start:
+/*
+ * mirror_start:
  * @bs: Block device to operate on.
- * @base: Block device that will become the new base, or %NULL to
- * flatten the whole backing file chain onto @bs.
- * @base_id: The file name that will be written to @bs as the new
- * backing file if the job completes.  Ignored if @base is %NULL.
+ * @target: Block device to write to.
  * @speed: The maximum speed, in bytes per second, or 0 for unlimited.
+ * @mode: Whether to collapse all images in the chain to the target.
+ * @on_source_error: The action to take upon error reading from the source.
+ * @on_target_error: The action to take upon error writing to the target.
  * @cb: Completion function for the job.
  * @opaque: Opaque pointer value passed to @cb.
  * @errp: Error object.
  *
- * Start a streaming operation on @bs.  Clusters that are unallocated
- * in @bs, but allocated in any image between @base and @bs (both
- * exclusive) will be written to @bs.  At the end of a successful
- * streaming job, the backing file of @bs will be changed to
- * @base_id in the written image and to @base in the live BlockDriverState.
+ * Start a mirroring operation on @bs.  Clusters that are allocated
+ * in @bs will be written to @bs until the job is cancelled or
+ * manually completed.  At the end of a successful mirroring job,
+ * @bs will be switched to read from @target.
  */
-void stream_start(BlockDriverState *bs, BlockDriverState *base,
-                  const char *base_id, int64_t speed,
+void mirror_start(BlockDriverState *bs, BlockDriverState *target,
+                  int64_t speed, MirrorSyncMode mode,
+                  BlockdevOnError on_source_error,
+                  BlockdevOnError on_target_error,
                   BlockDriverCompletionFunc *cb,
                   void *opaque, Error **errp);
 
diff --git a/blockdev-nbd.c b/blockdev-nbd.c
new file mode 100644
index 000000000..6b26bbf8c
--- /dev/null
+++ b/blockdev-nbd.c
@@ -0,0 +1,133 @@
+/*
+ * Serving QEMU block devices via NBD
+ *
+ * Copyright (c) 2012 Red Hat, Inc.
+ *
+ * Author: Paolo Bonzini <pbonzini@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * later.  See the COPYING file in the top-level directory.
+ */
+
+#include "blockdev.h"
+#include "hw/block-common.h"
+#include "monitor.h"
+#include "qerror.h"
+#include "sysemu.h"
+#include "qmp-commands.h"
+#include "trace.h"
+#include "nbd.h"
+#include "qemu_socket.h"
+
+static int server_fd = -1;
+
+static void nbd_accept(void *opaque)
+{
+    struct sockaddr_in addr;
+    socklen_t addr_len = sizeof(addr);
+
+    int fd = accept(server_fd, (struct sockaddr *)&addr, &addr_len);
+    if (fd >= 0) {
+        nbd_client_new(NULL, fd, nbd_client_put);
+    }
+}
+
+void qmp_nbd_server_start(SocketAddress *addr, Error **errp)
+{
+    if (server_fd != -1) {
+        error_setg(errp, "NBD server already running");
+        return;
+    }
+
+    server_fd = socket_listen(addr, errp);
+    if (server_fd != -1) {
+        qemu_set_fd_handler2(server_fd, NULL, nbd_accept, NULL, NULL);
+    }
+}
+
+/* Hook into the BlockDriverState notifiers to close the export when
+ * the file is closed.
+ */
+typedef struct NBDCloseNotifier {
+    Notifier n;
+    NBDExport *exp;
+    QTAILQ_ENTRY(NBDCloseNotifier) next;
+} NBDCloseNotifier;
+
+static QTAILQ_HEAD(, NBDCloseNotifier) close_notifiers =
+    QTAILQ_HEAD_INITIALIZER(close_notifiers);
+
+static void nbd_close_notifier(Notifier *n, void *data)
+{
+    NBDCloseNotifier *cn = DO_UPCAST(NBDCloseNotifier, n, n);
+
+    notifier_remove(&cn->n);
+    QTAILQ_REMOVE(&close_notifiers, cn, next);
+
+    nbd_export_close(cn->exp);
+    nbd_export_put(cn->exp);
+    g_free(cn);
+}
+
+static void nbd_server_put_ref(NBDExport *exp)
+{
+    BlockDriverState *bs = nbd_export_get_blockdev(exp);
+    drive_put_ref(drive_get_by_blockdev(bs));
+}
+
+void qmp_nbd_server_add(const char *device, bool has_writable, bool writable,
+                        Error **errp)
+{
+    BlockDriverState *bs;
+    NBDExport *exp;
+    NBDCloseNotifier *n;
+
+    if (server_fd == -1) {
+        error_setg(errp, "NBD server not running");
+        return;
+    }
+
+    if (nbd_export_find(device)) {
+        error_setg(errp, "NBD server already exporting device '%s'", device);
+        return;
+    }
+
+    bs = bdrv_find(device);
+    if (!bs) {
+        error_set(errp, QERR_DEVICE_NOT_FOUND, device);
+        return;
+    }
+
+    if (!has_writable) {
+        writable = false;
+    }
+    if (bdrv_is_read_only(bs)) {
+        writable = false;
+    }
+
+    exp = nbd_export_new(bs, 0, -1, writable ? 0 : NBD_FLAG_READ_ONLY,
+                         nbd_server_put_ref);
+
+    nbd_export_set_name(exp, device);
+    drive_get_ref(drive_get_by_blockdev(bs));
+
+    n = g_malloc0(sizeof(NBDCloseNotifier));
+    n->n.notify = nbd_close_notifier;
+    n->exp = exp;
+    bdrv_add_close_notifier(bs, &n->n);
+    QTAILQ_INSERT_TAIL(&close_notifiers, n, next);
+}
+
+void qmp_nbd_server_stop(Error **errp)
+{
+    while (!QTAILQ_EMPTY(&close_notifiers)) {
+        NBDCloseNotifier *cn = QTAILQ_FIRST(&close_notifiers);
+        nbd_close_notifier(&cn->n, nbd_export_get_blockdev(cn->exp));
+    }
+
+    if (server_fd != -1) {
+        qemu_set_fd_handler2(server_fd, NULL, NULL, NULL, NULL);
+        close(server_fd);
+        server_fd = -1;
+    }
+}
diff --git a/blockdev.c b/blockdev.c
index 7c83baa35..e73fd6e38 100644
--- a/blockdev.c
+++ b/blockdev.c
@@ -9,6 +9,7 @@
 
 #include "blockdev.h"
 #include "hw/block-common.h"
+#include "blockjob.h"
 #include "monitor.h"
 #include "qerror.h"
 #include "qemu-option.h"
@@ -237,16 +238,16 @@ static void drive_put_ref_bh_schedule(DriveInfo *dinfo)
     qemu_bh_schedule(s->bh);
 }
 
-static int parse_block_error_action(const char *buf, int is_read)
+static int parse_block_error_action(const char *buf, bool is_read)
 {
     if (!strcmp(buf, "ignore")) {
-        return BLOCK_ERR_IGNORE;
+        return BLOCKDEV_ON_ERROR_IGNORE;
     } else if (!is_read && !strcmp(buf, "enospc")) {
-        return BLOCK_ERR_STOP_ENOSPC;
+        return BLOCKDEV_ON_ERROR_ENOSPC;
     } else if (!strcmp(buf, "stop")) {
-        return BLOCK_ERR_STOP_ANY;
+        return BLOCKDEV_ON_ERROR_STOP;
     } else if (!strcmp(buf, "report")) {
-        return BLOCK_ERR_REPORT;
+        return BLOCKDEV_ON_ERROR_REPORT;
     } else {
         error_report("'%s' invalid %s error action",
                      buf, is_read ? "read" : "write");
@@ -432,7 +433,13 @@ DriveInfo *drive_init(QemuOpts *opts, int default_to_scsi)
         return NULL;
     }
 
-    on_write_error = BLOCK_ERR_STOP_ENOSPC;
+    if (qemu_opt_get(opts, "boot") != NULL) {
+        fprintf(stderr, "qemu-kvm: boot=on|off is deprecated and will be "
+                "ignored. Future versions will reject this parameter. Please "
+                "update your scripts.\n");
+    }
+
+    on_write_error = BLOCKDEV_ON_ERROR_ENOSPC;
     if ((buf = qemu_opt_get(opts, "werror")) != NULL) {
         if (type != IF_IDE && type != IF_SCSI && type != IF_VIRTIO && type != IF_NONE) {
             error_report("werror is not supported by this bus type");
@@ -445,7 +452,7 @@ DriveInfo *drive_init(QemuOpts *opts, int default_to_scsi)
         }
     }
 
-    on_read_error = BLOCK_ERR_REPORT;
+    on_read_error = BLOCKDEV_ON_ERROR_REPORT;
     if ((buf = qemu_opt_get(opts, "rerror")) != NULL) {
         if (type != IF_IDE && type != IF_VIRTIO && type != IF_SCSI && type != IF_NONE) {
             error_report("rerror is not supported by this bus type");
@@ -527,6 +534,8 @@ DriveInfo *drive_init(QemuOpts *opts, int default_to_scsi)
                      if_name[type], mediastr, unit_id);
     }
     dinfo->bdrv = bdrv_new(dinfo->id);
+    dinfo->bdrv->open_flags = snapshot ? BDRV_O_SNAPSHOT : 0;
+    dinfo->bdrv->read_only = ro;
     dinfo->devaddr = devaddr;
     dinfo->type = type;
     dinfo->bus = bus_id;
@@ -803,6 +812,11 @@ void qmp_transaction(BlockdevActionList *dev_list, Error **errp)
     QSIMPLEQ_FOREACH(states, &snap_bdrv_states, entry) {
         /* This removes our old bs from the bdrv_states, and adds the new bs */
         bdrv_append(states->new_bs, states->old_bs);
+        /* We don't need (or want) to use the transactional
+         * bdrv_reopen_multiple() across all the entries at once, because we
+         * don't want to abort all of them if one of them fails the reopen */
+        bdrv_reopen(states->new_bs, states->new_bs->open_flags & ~BDRV_O_RDWR,
+                    NULL);
     }
 
     /* success */
@@ -822,7 +836,6 @@ exit:
     QSIMPLEQ_FOREACH_SAFE(states, &snap_bdrv_states, entry, next) {
         g_free(states);
     }
-    return;
 }
 
 
@@ -1049,26 +1062,12 @@ void qmp_block_resize(const char *device, int64_t size, Error **errp)
     }
 }
 
-static QObject *qobject_from_block_job(BlockJob *job)
-{
-    return qobject_from_jsonf("{ 'type': %s,"
-                              "'device': %s,"
-                              "'len': %" PRId64 ","
-                              "'offset': %" PRId64 ","
-                              "'speed': %" PRId64 " }",
-                              job->job_type->job_type,
-                              bdrv_get_device_name(job->bs),
-                              job->len,
-                              job->offset,
-                              job->speed);
-}
-
-static void block_stream_cb(void *opaque, int ret)
+static void block_job_cb(void *opaque, int ret)
 {
     BlockDriverState *bs = opaque;
     QObject *obj;
 
-    trace_block_stream_cb(bs, bs->job, ret);
+    trace_block_job_cb(bs, bs->job, ret);
 
     assert(bs->job);
     obj = qobject_from_block_job(bs->job);
@@ -1088,13 +1087,18 @@ static void block_stream_cb(void *opaque, int ret)
 }
 
 void qmp_block_stream(const char *device, bool has_base,
-                      const char *base, bool has_speed,
-                      int64_t speed, Error **errp)
+                      const char *base, bool has_speed, int64_t speed,
+                      bool has_on_error, BlockdevOnError on_error,
+                      Error **errp)
 {
     BlockDriverState *bs;
     BlockDriverState *base_bs = NULL;
     Error *local_err = NULL;
 
+    if (!has_on_error) {
+        on_error = BLOCKDEV_ON_ERROR_REPORT;
+    }
+
     bs = bdrv_find(device);
     if (!bs) {
         error_set(errp, QERR_DEVICE_NOT_FOUND, device);
@@ -1110,7 +1114,7 @@ void qmp_block_stream(const char *device, bool has_base,
     }
 
     stream_start(bs, base_bs, base, has_speed ? speed : 0,
-                 block_stream_cb, bs, &local_err);
+                 on_error, block_job_cb, bs, &local_err);
     if (error_is_set(&local_err)) {
         error_propagate(errp, local_err);
         return;
@@ -1124,6 +1128,199 @@ void qmp_block_stream(const char *device, bool has_base,
     trace_qmp_block_stream(bs, bs->job);
 }
 
+void qmp_block_commit(const char *device,
+                      bool has_base, const char *base, const char *top,
+                      bool has_speed, int64_t speed,
+                      Error **errp)
+{
+    BlockDriverState *bs;
+    BlockDriverState *base_bs, *top_bs;
+    Error *local_err = NULL;
+    /* This will be part of the QMP command, if/when the
+     * BlockdevOnError change for blkmirror makes it in
+     */
+    BlockdevOnError on_error = BLOCKDEV_ON_ERROR_REPORT;
+
+    /* drain all i/o before commits */
+    bdrv_drain_all();
+
+    bs = bdrv_find(device);
+    if (!bs) {
+        error_set(errp, QERR_DEVICE_NOT_FOUND, device);
+        return;
+    }
+
+    /* default top_bs is the active layer */
+    top_bs = bs;
+
+    if (top) {
+        if (strcmp(bs->filename, top) != 0) {
+            top_bs = bdrv_find_backing_image(bs, top);
+        }
+    }
+
+    if (top_bs == NULL) {
+        error_setg(errp, "Top image file %s not found", top ? top : "NULL");
+        return;
+    }
+
+    if (has_base && base) {
+        base_bs = bdrv_find_backing_image(top_bs, base);
+    } else {
+        base_bs = bdrv_find_base(top_bs);
+    }
+
+    if (base_bs == NULL) {
+        error_set(errp, QERR_BASE_NOT_FOUND, base ? base : "NULL");
+        return;
+    }
+
+    commit_start(bs, base_bs, top_bs, speed, on_error, block_job_cb, bs,
+                &local_err);
+    if (local_err != NULL) {
+        error_propagate(errp, local_err);
+        return;
+    }
+    /* Grab a reference so hotplug does not delete the BlockDriverState from
+     * underneath us.
+     */
+    drive_get_ref(drive_get_by_blockdev(bs));
+}
+
+void qmp_drive_mirror(const char *device, const char *target,
+                      bool has_format, const char *format,
+                      enum MirrorSyncMode sync,
+                      bool has_mode, enum NewImageMode mode,
+                      bool has_speed, int64_t speed,
+                      bool has_on_source_error, BlockdevOnError on_source_error,
+                      bool has_on_target_error, BlockdevOnError on_target_error,
+                      Error **errp)
+{
+    BlockDriverInfo bdi;
+    BlockDriverState *bs;
+    BlockDriverState *source, *target_bs;
+    BlockDriver *proto_drv;
+    BlockDriver *drv = NULL;
+    Error *local_err = NULL;
+    int flags;
+    uint64_t size;
+    int ret;
+
+    if (!has_speed) {
+        speed = 0;
+    }
+    if (!has_on_source_error) {
+        on_source_error = BLOCKDEV_ON_ERROR_REPORT;
+    }
+    if (!has_on_target_error) {
+        on_target_error = BLOCKDEV_ON_ERROR_REPORT;
+    }
+    if (!has_mode) {
+        mode = NEW_IMAGE_MODE_ABSOLUTE_PATHS;
+    }
+
+    bs = bdrv_find(device);
+    if (!bs) {
+        error_set(errp, QERR_DEVICE_NOT_FOUND, device);
+        return;
+    }
+
+    if (!bdrv_is_inserted(bs)) {
+        error_set(errp, QERR_DEVICE_HAS_NO_MEDIUM, device);
+        return;
+    }
+
+    if (!has_format) {
+        format = mode == NEW_IMAGE_MODE_EXISTING ? NULL : bs->drv->format_name;
+    }
+    if (format) {
+        drv = bdrv_find_format(format);
+        if (!drv) {
+            error_set(errp, QERR_INVALID_BLOCK_FORMAT, format);
+            return;
+        }
+    }
+
+    if (bdrv_in_use(bs)) {
+        error_set(errp, QERR_DEVICE_IN_USE, device);
+        return;
+    }
+
+    flags = bs->open_flags | BDRV_O_RDWR;
+    source = bs->backing_hd;
+    if (!source && sync == MIRROR_SYNC_MODE_TOP) {
+        sync = MIRROR_SYNC_MODE_FULL;
+    }
+
+    proto_drv = bdrv_find_protocol(target);
+    if (!proto_drv) {
+        error_set(errp, QERR_INVALID_BLOCK_FORMAT, format);
+        return;
+    }
+
+    if (sync == MIRROR_SYNC_MODE_FULL && mode != NEW_IMAGE_MODE_EXISTING) {
+        /* create new image w/o backing file */
+        assert(format && drv);
+        bdrv_get_geometry(bs, &size);
+        size *= 512;
+        ret = bdrv_img_create(target, format,
+                              NULL, NULL, NULL, size, flags);
+    } else {
+        switch (mode) {
+        case NEW_IMAGE_MODE_EXISTING:
+            ret = 0;
+            break;
+        case NEW_IMAGE_MODE_ABSOLUTE_PATHS:
+            /* create new image with backing file */
+            ret = bdrv_img_create(target, format,
+                                  source->filename,
+                                  source->drv->format_name,
+                                  NULL, -1, flags);
+            break;
+        default:
+            abort();
+        }
+    }
+
+    if (ret) {
+        error_set(errp, QERR_OPEN_FILE_FAILED, target);
+        return;
+    }
+
+    target_bs = bdrv_new("");
+    ret = bdrv_open(target_bs, target, flags | BDRV_O_NO_BACKING, drv);
+
+    if (ret < 0) {
+        bdrv_delete(target_bs);
+        error_set(errp, QERR_OPEN_FILE_FAILED, target);
+        return;
+    }
+
+    /* We need a backing file if we will copy parts of a cluster.  */
+    if (bdrv_get_info(target_bs, &bdi) >= 0 && bdi.cluster_size != 0 &&
+        bdi.cluster_size >= BDRV_SECTORS_PER_DIRTY_CHUNK * 512) {
+        ret = bdrv_open_backing_file(target_bs);
+        if (ret < 0) {
+            bdrv_delete(target_bs);
+            error_set(errp, QERR_OPEN_FILE_FAILED, target);
+            return;
+        }
+    }
+
+    mirror_start(bs, target_bs, speed, sync, on_source_error, on_target_error,
+                 block_job_cb, bs, &local_err);
+    if (local_err != NULL) {
+        bdrv_delete(target_bs);
+        error_propagate(errp, local_err);
+        return;
+    }
+
+    /* Grab a reference so hotplug does not delete the BlockDriverState from
+     * underneath us.
+     */
+    drive_get_ref(drive_get_by_blockdev(bs));
+}
+
 static BlockJob *find_block_job(const char *device)
 {
     BlockDriverState *bs;
@@ -1140,19 +1337,28 @@ void qmp_block_job_set_speed(const char *device, int64_t speed, Error **errp)
     BlockJob *job = find_block_job(device);
 
     if (!job) {
-        error_set(errp, QERR_DEVICE_NOT_ACTIVE, device);
+        error_set(errp, QERR_BLOCK_JOB_NOT_ACTIVE, device);
         return;
     }
 
     block_job_set_speed(job, speed, errp);
 }
 
-void qmp_block_job_cancel(const char *device, Error **errp)
+void qmp_block_job_cancel(const char *device,
+                          bool has_force, bool force, Error **errp)
 {
     BlockJob *job = find_block_job(device);
 
+    if (!has_force) {
+        force = false;
+    }
+
     if (!job) {
-        error_set(errp, QERR_DEVICE_NOT_ACTIVE, device);
+        error_set(errp, QERR_BLOCK_JOB_NOT_ACTIVE, device);
+        return;
+    }
+    if (job->paused && !force) {
+        error_set(errp, QERR_BLOCK_JOB_PAUSED, device);
         return;
     }
 
@@ -1160,25 +1366,53 @@ void qmp_block_job_cancel(const char *device, Error **errp)
     block_job_cancel(job);
 }
 
+void qmp_block_job_pause(const char *device, Error **errp)
+{
+    BlockJob *job = find_block_job(device);
+
+    if (!job) {
+        error_set(errp, QERR_BLOCK_JOB_NOT_ACTIVE, device);
+        return;
+    }
+
+    trace_qmp_block_job_pause(job);
+    block_job_pause(job);
+}
+
+void qmp_block_job_resume(const char *device, Error **errp)
+{
+    BlockJob *job = find_block_job(device);
+
+    if (!job) {
+        error_set(errp, QERR_BLOCK_JOB_NOT_ACTIVE, device);
+        return;
+    }
+
+    trace_qmp_block_job_resume(job);
+    block_job_resume(job);
+}
+
+void qmp_block_job_complete(const char *device, Error **errp)
+{
+    BlockJob *job = find_block_job(device);
+
+    if (!job) {
+        error_set(errp, QERR_BLOCK_JOB_NOT_ACTIVE, device);
+        return;
+    }
+
+    trace_qmp_block_job_complete(job);
+    block_job_complete(job, errp);
+}
+
 static void do_qmp_query_block_jobs_one(void *opaque, BlockDriverState *bs)
 {
     BlockJobInfoList **prev = opaque;
     BlockJob *job = bs->job;
 
     if (job) {
-        BlockJobInfoList *elem;
-        BlockJobInfo *info = g_new(BlockJobInfo, 1);
-        *info = (BlockJobInfo){
-            .type   = g_strdup(job->job_type->job_type),
-            .device = g_strdup(bdrv_get_device_name(bs)),
-            .len    = job->len,
-            .offset = job->offset,
-            .speed  = job->speed,
-        };
-
-        elem = g_new0(BlockJobInfoList, 1);
-        elem->value = info;
-
+        BlockJobInfoList *elem = g_new0(BlockJobInfoList, 1);
+        elem->value = block_job_query(bs->job);
         (*prev)->next = elem;
         *prev = elem;
     }
diff --git a/blockjob.c b/blockjob.c
new file mode 100644
index 000000000..cda12c693
--- /dev/null
+++ b/blockjob.c
@@ -0,0 +1,283 @@
+/*
+ * QEMU System Emulator block driver
+ *
+ * Copyright (c) 2011 IBM Corp.
+ * Copyright (c) 2012 Red Hat, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "config-host.h"
+#include "qemu-common.h"
+#include "trace.h"
+#include "monitor.h"
+#include "block.h"
+#include "blockjob.h"
+#include "block_int.h"
+#include "qjson.h"
+#include "qemu-coroutine.h"
+#include "qmp-commands.h"
+#include "qemu-timer.h"
+
+void *block_job_create(const BlockJobType *job_type, BlockDriverState *bs,
+                       int64_t speed, BlockDriverCompletionFunc *cb,
+                       void *opaque, Error **errp)
+{
+    BlockJob *job;
+
+    if (bs->job || bdrv_in_use(bs)) {
+        error_set(errp, QERR_DEVICE_IN_USE, bdrv_get_device_name(bs));
+        return NULL;
+    }
+    bdrv_set_in_use(bs, 1);
+
+    job = g_malloc0(job_type->instance_size);
+    job->job_type      = job_type;
+    job->bs            = bs;
+    job->cb            = cb;
+    job->opaque        = opaque;
+    job->busy          = true;
+    bs->job = job;
+
+    /* Only set speed when necessary to avoid NotSupported error */
+    if (speed != 0) {
+        Error *local_err = NULL;
+
+        block_job_set_speed(job, speed, &local_err);
+        if (error_is_set(&local_err)) {
+            bs->job = NULL;
+            g_free(job);
+            bdrv_set_in_use(bs, 0);
+            error_propagate(errp, local_err);
+            return NULL;
+        }
+    }
+    return job;
+}
+
+void block_job_completed(BlockJob *job, int ret)
+{
+    BlockDriverState *bs = job->bs;
+
+    assert(bs->job == job);
+    job->cb(job->opaque, ret);
+    bs->job = NULL;
+    g_free(job);
+    bdrv_set_in_use(bs, 0);
+}
+
+void block_job_set_speed(BlockJob *job, int64_t speed, Error **errp)
+{
+    Error *local_err = NULL;
+
+    if (!job->job_type->set_speed) {
+        error_set(errp, QERR_NOT_SUPPORTED);
+        return;
+    }
+    job->job_type->set_speed(job, speed, &local_err);
+    if (error_is_set(&local_err)) {
+        error_propagate(errp, local_err);
+        return;
+    }
+
+    job->speed = speed;
+}
+
+void block_job_complete(BlockJob *job, Error **errp)
+{
+    if (job->paused || job->cancelled || !job->job_type->complete) {
+        error_set(errp, QERR_BLOCK_JOB_NOT_READY, job->bs->device_name);
+        return;
+    }
+
+    job->job_type->complete(job, errp);
+}
+
+void block_job_pause(BlockJob *job)
+{
+    job->paused = true;
+}
+
+bool block_job_is_paused(BlockJob *job)
+{
+    return job->paused;
+}
+
+void block_job_resume(BlockJob *job)
+{
+    job->paused = false;
+    block_job_iostatus_reset(job);
+    if (job->co && !job->busy) {
+        qemu_coroutine_enter(job->co, NULL);
+    }
+}
+
+void block_job_cancel(BlockJob *job)
+{
+    job->cancelled = true;
+    block_job_resume(job);
+}
+
+bool block_job_is_cancelled(BlockJob *job)
+{
+    return job->cancelled;
+}
+
+void block_job_iostatus_reset(BlockJob *job)
+{
+    job->iostatus = BLOCK_DEVICE_IO_STATUS_OK;
+    if (job->job_type->iostatus_reset) {
+        job->job_type->iostatus_reset(job);
+    }
+}
+
+struct BlockCancelData {
+    BlockJob *job;
+    BlockDriverCompletionFunc *cb;
+    void *opaque;
+    bool cancelled;
+    int ret;
+};
+
+static void block_job_cancel_cb(void *opaque, int ret)
+{
+    struct BlockCancelData *data = opaque;
+
+    data->cancelled = block_job_is_cancelled(data->job);
+    data->ret = ret;
+    data->cb(data->opaque, ret);
+}
+
+int block_job_cancel_sync(BlockJob *job)
+{
+    struct BlockCancelData data;
+    BlockDriverState *bs = job->bs;
+
+    assert(bs->job == job);
+
+    /* Set up our own callback to store the result and chain to
+     * the original callback.
+     */
+    data.job = job;
+    data.cb = job->cb;
+    data.opaque = job->opaque;
+    data.ret = -EINPROGRESS;
+    job->cb = block_job_cancel_cb;
+    job->opaque = &data;
+    block_job_cancel(job);
+    while (data.ret == -EINPROGRESS) {
+        qemu_aio_wait();
+    }
+    return (data.cancelled && data.ret == 0) ? -ECANCELED : data.ret;
+}
+
+void block_job_sleep_ns(BlockJob *job, QEMUClock *clock, int64_t ns)
+{
+    assert(job->busy);
+
+    /* Check cancellation *before* setting busy = false, too!  */
+    if (block_job_is_cancelled(job)) {
+        return;
+    }
+
+    job->busy = false;
+    if (block_job_is_paused(job)) {
+        qemu_coroutine_yield();
+    } else {
+        co_sleep_ns(clock, ns);
+    }
+    job->busy = true;
+}
+
+BlockJobInfo *block_job_query(BlockJob *job)
+{
+    BlockJobInfo *info = g_new0(BlockJobInfo, 1);
+    info->type      = g_strdup(job->job_type->job_type);
+    info->device    = g_strdup(bdrv_get_device_name(job->bs));
+    info->len       = job->len;
+    info->busy      = job->busy;
+    info->paused    = job->paused;
+    info->offset    = job->offset;
+    info->speed     = job->speed;
+    info->io_status = job->iostatus;
+    return info;
+}
+
+static void block_job_iostatus_set_err(BlockJob *job, int error)
+{
+    if (job->iostatus == BLOCK_DEVICE_IO_STATUS_OK) {
+        job->iostatus = error == ENOSPC ? BLOCK_DEVICE_IO_STATUS_NOSPACE :
+                                          BLOCK_DEVICE_IO_STATUS_FAILED;
+    }
+}
+
+
+QObject *qobject_from_block_job(BlockJob *job)
+{
+    return qobject_from_jsonf("{ 'type': %s,"
+                              "'device': %s,"
+                              "'len': %" PRId64 ","
+                              "'offset': %" PRId64 ","
+                              "'speed': %" PRId64 " }",
+                              job->job_type->job_type,
+                              bdrv_get_device_name(job->bs),
+                              job->len,
+                              job->offset,
+                              job->speed);
+}
+
+void block_job_ready(BlockJob *job)
+{
+    QObject *data = qobject_from_block_job(job);
+    monitor_protocol_event(QEVENT_BLOCK_JOB_READY, data);
+    qobject_decref(data);
+}
+
+BlockErrorAction block_job_error_action(BlockJob *job, BlockDriverState *bs,
+                                        BlockdevOnError on_err,
+                                        int is_read, int error)
+{
+    BlockErrorAction action;
+
+    switch (on_err) {
+    case BLOCKDEV_ON_ERROR_ENOSPC:
+        action = (error == ENOSPC) ? BDRV_ACTION_STOP : BDRV_ACTION_REPORT;
+        break;
+    case BLOCKDEV_ON_ERROR_STOP:
+        action = BDRV_ACTION_STOP;
+        break;
+    case BLOCKDEV_ON_ERROR_REPORT:
+        action = BDRV_ACTION_REPORT;
+        break;
+    case BLOCKDEV_ON_ERROR_IGNORE:
+        action = BDRV_ACTION_IGNORE;
+        break;
+    default:
+        abort();
+    }
+    bdrv_emit_qmp_error_event(job->bs, QEVENT_BLOCK_JOB_ERROR, action, is_read);
+    if (action == BDRV_ACTION_STOP) {
+        block_job_pause(job);
+        block_job_iostatus_set_err(job, error);
+        if (bs != job->bs) {
+            bdrv_iostatus_set_err(bs, error);
+        }
+    }
+    return action;
+}
diff --git a/blockjob.h b/blockjob.h
new file mode 100644
index 000000000..3792b73e5
--- /dev/null
+++ b/blockjob.h
@@ -0,0 +1,278 @@
+/*
+ * Declarations for long-running block device operations
+ *
+ * Copyright (c) 2011 IBM Corp.
+ * Copyright (c) 2012 Red Hat, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#ifndef BLOCKJOB_H
+#define BLOCKJOB_H 1
+
+#include "block.h"
+
+/**
+ * BlockJobType:
+ *
+ * A class type for block job objects.
+ */
+typedef struct BlockJobType {
+    /** Derived BlockJob struct size */
+    size_t instance_size;
+
+    /** String describing the operation, part of query-block-jobs QMP API */
+    const char *job_type;
+
+    /** Optional callback for job types that support setting a speed limit */
+    void (*set_speed)(BlockJob *job, int64_t speed, Error **errp);
+
+    /** Optional callback for job types that need to forward I/O status reset */
+    void (*iostatus_reset)(BlockJob *job);
+
+    /**
+     * Optional callback for job types whose completion must be triggered
+     * manually.
+     */
+    void (*complete)(BlockJob *job, Error **errp);
+} BlockJobType;
+
+/**
+ * BlockJob:
+ *
+ * Long-running operation on a BlockDriverState.
+ */
+struct BlockJob {
+    /** The job type, including the job vtable.  */
+    const BlockJobType *job_type;
+
+    /** The block device on which the job is operating.  */
+    BlockDriverState *bs;
+
+    /**
+     * The coroutine that executes the job.  If not NULL, it is
+     * reentered when busy is false and the job is cancelled.
+     */
+    Coroutine *co;
+
+    /**
+     * Set to true if the job should cancel itself.  The flag must
+     * always be tested just before toggling the busy flag from false
+     * to true.  After a job has been cancelled, it should only yield
+     * if #qemu_aio_wait will ("sooner or later") reenter the coroutine.
+     */
+    bool cancelled;
+
+    /**
+     * Set to true if the job is either paused, or will pause itself
+     * as soon as possible (if busy == true).
+     */
+    bool paused;
+
+    /**
+     * Set to false by the job while it is in a quiescent state, where
+     * no I/O is pending and the job has yielded on any condition
+     * that is not detected by #qemu_aio_wait, such as a timer.
+     */
+    bool busy;
+
+    /** Status that is published by the query-block-jobs QMP API */
+    BlockDeviceIoStatus iostatus;
+
+    /** Offset that is published by the query-block-jobs QMP API */
+    int64_t offset;
+
+    /** Length that is published by the query-block-jobs QMP API */
+    int64_t len;
+
+    /** Speed that was set with @block_job_set_speed.  */
+    int64_t speed;
+
+    /** The completion function that will be called when the job completes.  */
+    BlockDriverCompletionFunc *cb;
+
+    /** The opaque value that is passed to the completion function.  */
+    void *opaque;
+};
+
+/**
+ * block_job_create:
+ * @job_type: The class object for the newly-created job.
+ * @bs: The block
+ * @speed: The maximum speed, in bytes per second, or 0 for unlimited.
+ * @cb: Completion function for the job.
+ * @opaque: Opaque pointer value passed to @cb.
+ * @errp: Error object.
+ *
+ * Create a new long-running block device job and return it.  The job
+ * will call @cb asynchronously when the job completes.  Note that
+ * @bs may have been closed at the time the @cb it is called.  If
+ * this is the case, the job may be reported as either cancelled or
+ * completed.
+ *
+ * This function is not part of the public job interface; it should be
+ * called from a wrapper that is specific to the job type.
+ */
+void *block_job_create(const BlockJobType *job_type, BlockDriverState *bs,
+                       int64_t speed, BlockDriverCompletionFunc *cb,
+                       void *opaque, Error **errp);
+
+/**
+ * block_job_sleep_ns:
+ * @job: The job that calls the function.
+ * @clock: The clock to sleep on.
+ * @ns: How many nanoseconds to stop for.
+ *
+ * Put the job to sleep (assuming that it wasn't canceled) for @ns
+ * nanoseconds.  Canceling the job will interrupt the wait immediately.
+ */
+void block_job_sleep_ns(BlockJob *job, QEMUClock *clock, int64_t ns);
+
+/**
+ * block_job_completed:
+ * @job: The job being completed.
+ * @ret: The status code.
+ *
+ * Call the completion function that was registered at creation time, and
+ * free @job.
+ */
+void block_job_completed(BlockJob *job, int ret);
+
+/**
+ * block_job_set_speed:
+ * @job: The job to set the speed for.
+ * @speed: The new value
+ * @errp: Error object.
+ *
+ * Set a rate-limiting parameter for the job; the actual meaning may
+ * vary depending on the job type.
+ */
+void block_job_set_speed(BlockJob *job, int64_t speed, Error **errp);
+
+/**
+ * block_job_cancel:
+ * @job: The job to be canceled.
+ *
+ * Asynchronously cancel the specified job.
+ */
+void block_job_cancel(BlockJob *job);
+
+/**
+ * block_job_complete:
+ * @job: The job to be completed.
+ * @errp: Error object.
+ *
+ * Asynchronously complete the specified job.
+ */
+void block_job_complete(BlockJob *job, Error **errp);
+
+/**
+ * block_job_is_cancelled:
+ * @job: The job being queried.
+ *
+ * Returns whether the job is scheduled for cancellation.
+ */
+bool block_job_is_cancelled(BlockJob *job);
+
+/**
+ * block_job_query:
+ * @job: The job to get information about.
+ *
+ * Return information about a job.
+ */
+BlockJobInfo *block_job_query(BlockJob *job);
+
+/**
+ * block_job_pause:
+ * @job: The job to be paused.
+ *
+ * Asynchronously pause the specified job.
+ */
+void block_job_pause(BlockJob *job);
+
+/**
+ * block_job_resume:
+ * @job: The job to be resumed.
+ *
+ * Resume the specified job.
+ */
+void block_job_resume(BlockJob *job);
+
+/**
+ * qobject_from_block_job:
+ * @job: The job whose information is requested.
+ *
+ * Return a QDict corresponding to @job's query-block-jobs entry.
+ */
+QObject *qobject_from_block_job(BlockJob *job);
+
+/**
+ * block_job_ready:
+ * @job: The job which is now ready to complete.
+ *
+ * Send a BLOCK_JOB_READY event for the specified job.
+ */
+void block_job_ready(BlockJob *job);
+
+/**
+ * block_job_is_paused:
+ * @job: The job being queried.
+ *
+ * Returns whether the job is currently paused, or will pause
+ * as soon as it reaches a sleeping point.
+ */
+bool block_job_is_paused(BlockJob *job);
+
+/**
+ * block_job_cancel_sync:
+ * @job: The job to be canceled.
+ *
+ * Synchronously cancel the job.  The completion callback is called
+ * before the function returns.  The job may actually complete
+ * instead of canceling itself; the circumstances under which this
+ * happens depend on the kind of job that is active.
+ *
+ * Returns the return value from the job if the job actually completed
+ * during the call, or -ECANCELED if it was canceled.
+ */
+int block_job_cancel_sync(BlockJob *job);
+
+/**
+ * block_job_iostatus_reset:
+ * @job: The job whose I/O status should be reset.
+ *
+ * Reset I/O status on @job and on BlockDriverState objects it uses,
+ * other than job->bs.
+ */
+void block_job_iostatus_reset(BlockJob *job);
+
+/**
+ * block_job_error_action:
+ * @job: The job to signal an error for.
+ * @bs: The block device on which to set an I/O error.
+ * @on_err: The error action setting.
+ * @is_read: Whether the operation was a read.
+ * @error: The error that was reported.
+ *
+ * Report an I/O error for a block job and possibly stop the VM.  Return the
+ * action that was selected based on @on_err and @error.
+ */
+BlockErrorAction block_job_error_action(BlockJob *job, BlockDriverState *bs,
+                                        BlockdevOnError on_err,
+                                        int is_read, int error);
+#endif
diff --git a/buffered_file.c b/buffered_file.c
index f170aa046..bd0f61d8c 100644
--- a/buffered_file.c
+++ b/buffered_file.c
@@ -23,11 +23,7 @@
 
 typedef struct QEMUFileBuffered
 {
-    BufferedPutFunc *put_buffer;
-    BufferedPutReadyFunc *put_ready;
-    BufferedWaitForUnfreezeFunc *wait_for_unfreeze;
-    BufferedCloseFunc *close;
-    void *opaque;
+    MigrationState *migration_state;
     QEMUFile *file;
     int freeze_output;
     size_t bytes_xfer;
@@ -50,70 +46,60 @@ static void buffered_append(QEMUFileBuffered *s,
                             const uint8_t *buf, size_t size)
 {
     if (size > (s->buffer_capacity - s->buffer_size)) {
-        void *tmp;
-
         DPRINTF("increasing buffer capacity from %zu by %zu\n",
                 s->buffer_capacity, size + 1024);
 
         s->buffer_capacity += size + 1024;
 
-        tmp = g_realloc(s->buffer, s->buffer_capacity);
-        if (tmp == NULL) {
-            fprintf(stderr, "qemu file buffer expansion failed\n");
-            exit(1);
-        }
-
-        s->buffer = tmp;
+        s->buffer = g_realloc(s->buffer, s->buffer_capacity);
     }
 
     memcpy(s->buffer + s->buffer_size, buf, size);
     s->buffer_size += size;
 }
 
-static void buffered_flush(QEMUFileBuffered *s)
+static ssize_t buffered_flush(QEMUFileBuffered *s)
 {
     size_t offset = 0;
-    int error;
-
-    error = qemu_file_get_error(s->file);
-    if (error != 0) {
-        DPRINTF("flush when error, bailing: %s\n", strerror(-error));
-        return;
-    }
+    ssize_t ret = 0;
 
     DPRINTF("flushing %zu byte(s) of data\n", s->buffer_size);
 
-    while (offset < s->buffer_size) {
-        ssize_t ret;
+    while (s->bytes_xfer < s->xfer_limit && offset < s->buffer_size) {
 
-        ret = s->put_buffer(s->opaque, s->buffer + offset,
-                            s->buffer_size - offset);
+        ret = migrate_fd_put_buffer(s->migration_state, s->buffer + offset,
+                                    s->buffer_size - offset);
         if (ret == -EAGAIN) {
             DPRINTF("backend not ready, freezing\n");
+            ret = 0;
             s->freeze_output = 1;
             break;
         }
 
         if (ret <= 0) {
             DPRINTF("error flushing data, %zd\n", ret);
-            qemu_file_set_error(s->file, ret);
             break;
         } else {
             DPRINTF("flushed %zd byte(s)\n", ret);
             offset += ret;
+            s->bytes_xfer += ret;
         }
     }
 
     DPRINTF("flushed %zu of %zu byte(s)\n", offset, s->buffer_size);
     memmove(s->buffer, s->buffer + offset, s->buffer_size - offset);
     s->buffer_size -= offset;
+
+    if (ret < 0) {
+        return ret;
+    }
+    return offset;
 }
 
 static int buffered_put_buffer(void *opaque, const uint8_t *buf, int64_t pos, int size)
 {
     QEMUFileBuffered *s = opaque;
-    int offset = 0, error;
-    ssize_t ret;
+    ssize_t error;
 
     DPRINTF("putting %d bytes at %" PRId64 "\n", size, pos);
 
@@ -126,65 +112,54 @@ static int buffered_put_buffer(void *opaque, const uint8_t *buf, int64_t pos, in
     DPRINTF("unfreezing output\n");
     s->freeze_output = 0;
 
-    buffered_flush(s);
-
-    while (!s->freeze_output && offset < size) {
-        if (s->bytes_xfer > s->xfer_limit) {
-            DPRINTF("transfer limit exceeded when putting\n");
-            break;
-        }
-
-        ret = s->put_buffer(s->opaque, buf + offset, size - offset);
-        if (ret == -EAGAIN) {
-            DPRINTF("backend not ready, freezing\n");
-            s->freeze_output = 1;
-            break;
-        }
-
-        if (ret <= 0) {
-            DPRINTF("error putting\n");
-            qemu_file_set_error(s->file, ret);
-            offset = -EINVAL;
-            break;
-        }
-
-        DPRINTF("put %zd byte(s)\n", ret);
-        offset += ret;
-        s->bytes_xfer += ret;
+    if (size > 0) {
+        DPRINTF("buffering %d bytes\n", size - offset);
+        buffered_append(s, buf, size);
     }
 
-    if (offset >= 0) {
-        DPRINTF("buffering %d bytes\n", size - offset);
-        buffered_append(s, buf + offset, size - offset);
-        offset = size;
+    error = buffered_flush(s);
+    if (error < 0) {
+        DPRINTF("buffered flush error. bailing: %s\n", strerror(-error));
+        return error;
     }
 
     if (pos == 0 && size == 0) {
         DPRINTF("file is ready\n");
-        if (s->bytes_xfer <= s->xfer_limit) {
+        if (!s->freeze_output && s->bytes_xfer < s->xfer_limit) {
             DPRINTF("notifying client\n");
-            s->put_ready(s->opaque);
+            migrate_fd_put_ready(s->migration_state);
         }
     }
 
-    return offset;
+    return size;
 }
 
 static int buffered_close(void *opaque)
 {
     QEMUFileBuffered *s = opaque;
-    int ret;
+    ssize_t ret = 0;
+    int ret2;
 
     DPRINTF("closing\n");
 
+    s->xfer_limit = INT_MAX;
     while (!qemu_file_get_error(s->file) && s->buffer_size) {
-        buffered_flush(s);
-        if (s->freeze_output)
-            s->wait_for_unfreeze(s->opaque);
+        ret = buffered_flush(s);
+        if (ret < 0) {
+            break;
+        }
+        if (s->freeze_output) {
+            ret = migrate_fd_wait_for_unfreeze(s->migration_state);
+            if (ret < 0) {
+                break;
+            }
+        }
     }
 
-    ret = s->close(s->opaque);
-
+    ret2 = migrate_fd_close(s->migration_state);
+    if (ret >= 0) {
+        ret = ret2;
+    }
     qemu_del_timer(s->timer);
     qemu_free_timer(s->timer);
     g_free(s->buffer);
@@ -199,6 +174,13 @@ static int buffered_close(void *opaque)
  *   1: Time to stop
  *   negative: There has been an error
  */
+static int buffered_get_fd(void *opaque)
+{
+    QEMUFileBuffered *s = opaque;
+
+    return qemu_get_fd(s->file);
+}
+
 static int buffered_rate_limit(void *opaque)
 {
     QEMUFileBuffered *s = opaque;
@@ -256,34 +238,28 @@ static void buffered_rate_tick(void *opaque)
 
     s->bytes_xfer = 0;
 
-    buffered_flush(s);
-
-    /* Add some checks around this */
-    s->put_ready(s->opaque);
+    buffered_put_buffer(s, NULL, 0, 0);
 }
 
-QEMUFile *qemu_fopen_ops_buffered(void *opaque,
-                                  size_t bytes_per_sec,
-                                  BufferedPutFunc *put_buffer,
-                                  BufferedPutReadyFunc *put_ready,
-                                  BufferedWaitForUnfreezeFunc *wait_for_unfreeze,
-                                  BufferedCloseFunc *close)
+static const QEMUFileOps buffered_file_ops = {
+    .get_fd =         buffered_get_fd,
+    .put_buffer =     buffered_put_buffer,
+    .close =          buffered_close,
+    .rate_limit =     buffered_rate_limit,
+    .get_rate_limit = buffered_get_rate_limit,
+    .set_rate_limit = buffered_set_rate_limit,
+};
+
+QEMUFile *qemu_fopen_ops_buffered(MigrationState *migration_state)
 {
     QEMUFileBuffered *s;
 
     s = g_malloc0(sizeof(*s));
 
-    s->opaque = opaque;
-    s->xfer_limit = bytes_per_sec / 10;
-    s->put_buffer = put_buffer;
-    s->put_ready = put_ready;
-    s->wait_for_unfreeze = wait_for_unfreeze;
-    s->close = close;
-
-    s->file = qemu_fopen_ops(s, buffered_put_buffer, NULL,
-                             buffered_close, buffered_rate_limit,
-                             buffered_set_rate_limit,
-			     buffered_get_rate_limit);
+    s->migration_state = migration_state;
+    s->xfer_limit = migration_state->bandwidth_limit / 10;
+
+    s->file = qemu_fopen_ops(s, &buffered_file_ops);
 
     s->timer = qemu_new_timer_ms(rt_clock, buffered_rate_tick, s);
 
diff --git a/buffered_file.h b/buffered_file.h
index 98d358bae..ef010febf 100644
--- a/buffered_file.h
+++ b/buffered_file.h
@@ -15,16 +15,8 @@
 #define QEMU_BUFFERED_FILE_H
 
 #include "hw/hw.h"
+#include "migration.h"
 
-typedef ssize_t (BufferedPutFunc)(void *opaque, const void *data, size_t size);
-typedef void (BufferedPutReadyFunc)(void *opaque);
-typedef void (BufferedWaitForUnfreezeFunc)(void *opaque);
-typedef int (BufferedCloseFunc)(void *opaque);
-
-QEMUFile *qemu_fopen_ops_buffered(void *opaque, size_t xfer_limit,
-                                  BufferedPutFunc *put_buffer,
-                                  BufferedPutReadyFunc *put_ready,
-                                  BufferedWaitForUnfreezeFunc *wait_for_unfreeze,
-                                  BufferedCloseFunc *close);
+QEMUFile *qemu_fopen_ops_buffered(MigrationState *migration_state);
 
 #endif
diff --git a/compiler.h b/compiler.h
index 07ba1f811..2f7998b6c 100644
--- a/compiler.h
+++ b/compiler.h
@@ -44,14 +44,12 @@
    /* Use gnu_printf when supported (qemu uses standard format strings). */
 #  define GCC_ATTR __attribute__((__unused__, format(gnu_printf, 1, 2)))
 #  define GCC_FMT_ATTR(n, m) __attribute__((format(gnu_printf, n, m)))
+#  if defined(_WIN32)
+    /* Map __printf__ to __gnu_printf__ because we want standard format strings
+     * even when MinGW or GLib include files use __printf__. */
+#   define __printf__ __gnu_printf__
+#  endif
 # endif
-#if defined(_WIN32)
-#define GCC_WEAK __attribute__((weak))
-#define GCC_WEAK_DECL GCC_WEAK
-#else
-#define GCC_WEAK __attribute__((weak))
-#define GCC_WEAK_DECL
-#endif
 #else
 #define GCC_ATTR /**/
 #define GCC_FMT_ATTR(n, m)
diff --git a/configure b/configure
index d97fd8170..994f7310b 100755
--- a/configure
+++ b/configure
@@ -111,14 +111,12 @@ source_path=`dirname "$0"`
 cpu=""
 interp_prefix="/usr/gnemul/qemu-%M"
 static="no"
-sparc_cpu=""
 cross_prefix=""
 audio_drv_list=""
 audio_card_list="ac97 es1370 sb16 hda"
 audio_possible_cards="ac97 es1370 sb16 cs4231a adlib gus hda"
 block_drv_whitelist=""
 host_cc="gcc"
-helper_cflags=""
 libs_softmmu=""
 libs_tools=""
 audio_pt_int=""
@@ -127,7 +125,8 @@ cc_i386=i386-pc-linux-gnu-gcc
 libs_qga=""
 debug_info="yes"
 
-target_list=""
+# Don't accept a target_list environment variable.
+unset target_list
 
 # Default value for a variable defining feature "foo".
 #  * foo="no"  feature will only be used if --enable-foo arg is given
@@ -148,6 +147,7 @@ curses=""
 docs=""
 fdt=""
 nptl=""
+pixman=""
 sdl=""
 virtfs=""
 vnc="yes"
@@ -183,8 +183,10 @@ datadir="\${prefix}/share"
 qemu_docdir="\${prefix}/share/doc/qemu"
 bindir="\${prefix}/bin"
 libdir="\${prefix}/lib"
+libexecdir="\${prefix}/libexec"
 includedir="\${prefix}/include"
 sysconfdir="\${prefix}/etc"
+local_statedir="\${prefix}/var"
 confsuffix="/qemu"
 slirp="yes"
 fmod_lib=""
@@ -198,7 +200,7 @@ cocoa="no"
 softmmu="yes"
 linux_user="no"
 bsd_user="no"
-guest_base=""
+guest_base="yes"
 uname_release=""
 mixemu="no"
 aix="no"
@@ -216,9 +218,11 @@ usb_redir=""
 opengl=""
 zlib="yes"
 guest_agent="yes"
+want_tools="yes"
 libiscsi=""
 coroutine=""
 seccomp=""
+glusterfs=""
 
 # parse CC options first
 for opt do
@@ -240,21 +244,6 @@ for opt do
   ;;
   --disable-debug-info) debug_info="no"
   ;;
-  --sparc_cpu=*)
-    sparc_cpu="$optarg"
-    case $sparc_cpu in
-    v7|v8|v8plus|v8plusa)
-      cpu="sparc"
-    ;;
-    v9)
-      cpu="sparc64"
-    ;;
-    *)
-      echo "undefined SPARC architecture. Exiting";
-      exit 1
-    ;;
-    esac
-  ;;
   esac
 done
 # OS specific
@@ -342,8 +331,6 @@ elif check_define __i386__ ; then
 elif check_define __x86_64__ ; then
   cpu="x86_64"
 elif check_define __sparc__ ; then
-  # We can't check for 64 bit (when gcc is biarch) or V8PLUSA
-  # They must be specified using --sparc_cpu
   if check_define __arch64__ ; then
     cpu="sparc64"
   else
@@ -569,6 +556,7 @@ EOF
   qemu_docdir="\${prefix}"
   bindir="\${prefix}"
   sysconfdir="\${prefix}"
+  local_statedir="\${prefix}"
   confsuffix=""
   libs_qga="-lws2_32 -lwinmm -lpowrprof $libs_qga"
 fi
@@ -633,6 +621,8 @@ for opt do
   ;;
   --libdir=*) libdir="$optarg"
   ;;
+  --libexecdir=*) libexecdir="$optarg"
+  ;;
   --includedir=*) includedir="$optarg"
   ;;
   --datadir=*) datadir="$optarg"
@@ -643,7 +633,9 @@ for opt do
   ;;
   --sysconfdir=*) sysconfdir="$optarg"
   ;;
-  --sbindir=*|--libexecdir=*|--sharedstatedir=*|--localstatedir=*|\
+  --localstatedir=*) local_statedir="$optarg"
+  ;;
+  --sbindir=*|--sharedstatedir=*|\
   --oldincludedir=*|--datarootdir=*|--infodir=*|--localedir=*|\
   --htmldir=*|--dvidir=*|--pdfdir=*|--psdir=*)
     # These switches are silently ignored, for compatibility with
@@ -651,6 +643,10 @@ for opt do
     # configure to be used by RPM and similar macros that set
     # lots of directory switches by default.
   ;;
+  --with-system-pixman) pixman="system"
+  ;;
+  --without-system-pixman) pixman="internal"
+  ;;
   --disable-sdl) sdl="no"
   ;;
   --enable-sdl) sdl="yes"
@@ -789,8 +785,6 @@ for opt do
   ;;
   --enable-uname-release=*) uname_release="$optarg"
   ;;
-  --sparc_cpu=*)
-  ;;
   --enable-werror) werror="yes"
   ;;
   --disable-werror) werror="no"
@@ -865,92 +859,53 @@ for opt do
   ;;
   --disable-guest-agent) guest_agent="no"
   ;;
+  --enable-tools) want_tools="yes"
+  ;;
+  --disable-tools) want_tools="no"
+  ;;
   --enable-seccomp) seccomp="yes"
   ;;
   --disable-seccomp) seccomp="no"
   ;;
+  --disable-glusterfs) glusterfs="no"
+  ;;
+  --enable-glusterfs) glusterfs="yes"
+  ;;
   *) echo "ERROR: unknown option $opt"; show_help="yes"
   ;;
   esac
 done
 
-#
-# If cpu ~= sparc and  sparc_cpu hasn't been defined, plug in the right
-# QEMU_CFLAGS/LDFLAGS (assume sparc_v8plus for 32-bit and sparc_v9 for 64-bit)
-#
-host_guest_base="no"
 case "$cpu" in
-    sparc) case $sparc_cpu in
-           v7|v8)
-             QEMU_CFLAGS="-mcpu=${sparc_cpu} -D__sparc_${sparc_cpu}__ $QEMU_CFLAGS"
-           ;;
-           v8plus|v8plusa)
-             QEMU_CFLAGS="-mcpu=ultrasparc -D__sparc_${sparc_cpu}__ $QEMU_CFLAGS"
-           ;;
-           *) # sparc_cpu not defined in the command line
-             QEMU_CFLAGS="-mcpu=ultrasparc -D__sparc_v8plus__ $QEMU_CFLAGS"
-           esac
+    sparc)
            LDFLAGS="-m32 $LDFLAGS"
-           QEMU_CFLAGS="-m32 -ffixed-g2 -ffixed-g3 $QEMU_CFLAGS"
-           if test "$solaris" = "no" ; then
-             QEMU_CFLAGS="-ffixed-g1 -ffixed-g6 $QEMU_CFLAGS"
-             helper_cflags="-ffixed-i0"
-           fi
+           QEMU_CFLAGS="-m32 -mcpu=ultrasparc $QEMU_CFLAGS"
            ;;
     sparc64)
-           QEMU_CFLAGS="-m64 -mcpu=ultrasparc -D__sparc_v9__ $QEMU_CFLAGS"
            LDFLAGS="-m64 $LDFLAGS"
-           QEMU_CFLAGS="-ffixed-g5 -ffixed-g6 -ffixed-g7 $QEMU_CFLAGS"
-           if test "$solaris" != "no" ; then
-             QEMU_CFLAGS="-ffixed-g1 $QEMU_CFLAGS"
-           fi
+           QEMU_CFLAGS="-m64 -mcpu=ultrasparc $QEMU_CFLAGS"
            ;;
     s390)
            QEMU_CFLAGS="-m31 -march=z990 $QEMU_CFLAGS"
            LDFLAGS="-m31 $LDFLAGS"
-           host_guest_base="yes"
            ;;
     s390x)
            QEMU_CFLAGS="-m64 -march=z990 $QEMU_CFLAGS"
            LDFLAGS="-m64 $LDFLAGS"
-           host_guest_base="yes"
            ;;
     i386)
            QEMU_CFLAGS="-m32 $QEMU_CFLAGS"
            LDFLAGS="-m32 $LDFLAGS"
            cc_i386='$(CC) -m32'
-           helper_cflags="-fomit-frame-pointer"
-           host_guest_base="yes"
            ;;
     x86_64)
            QEMU_CFLAGS="-m64 $QEMU_CFLAGS"
            LDFLAGS="-m64 $LDFLAGS"
            cc_i386='$(CC) -m32'
-           host_guest_base="yes"
-           ;;
-    arm*)
-           host_guest_base="yes"
-           ;;
-    ppc*)
-           host_guest_base="yes"
-           ;;
-    mips*)
-           host_guest_base="yes"
-           ;;
-    ia64*)
-           host_guest_base="yes"
-           ;;
-    hppa*)
-           host_guest_base="yes"
-           ;;
-    unicore32*)
-           host_guest_base="yes"
            ;;
+    # No special flags required for other host CPUs
 esac
 
-[ -z "$guest_base" ] && guest_base="$host_guest_base"
-
-
 default_target_list=""
 
 # these targets are portable
@@ -1055,6 +1010,7 @@ echo "  --datadir=PATH           install firmware in PATH$confsuffix"
 echo "  --docdir=PATH            install documentation in PATH$confsuffix"
 echo "  --bindir=PATH            install binaries in PATH"
 echo "  --sysconfdir=PATH        install config in PATH$confsuffix"
+echo "  --localstatedir=PATH     install local state in PATH"
 echo "  --with-confsuffix=SUFFIX suffix for QEMU data inside datadir and sysconfdir [$confsuffix]"
 echo "  --enable-debug-tcg       enable TCG debugging"
 echo "  --disable-debug-tcg      disable TCG debugging (default)"
@@ -1161,6 +1117,8 @@ echo "  --disable-seccomp        disable seccomp support"
 echo "  --enable-seccomp         enables seccomp support"
 echo "  --with-coroutine=BACKEND coroutine backend. Supported options:"
 echo "                           gthread, ucontext, sigaltstack, windows"
+echo "  --enable-glusterfs       enable GlusterFS backend"
+echo "  --disable-glusterfs      disable GlusterFS backend"
 echo ""
 echo "NOTE: The object files are built at the place where configure is launched"
 exit 1
@@ -1207,6 +1165,7 @@ gcc_flags="-Wold-style-declaration -Wold-style-definition -Wtype-limits"
 gcc_flags="-Wformat-security -Wformat-y2k -Winit-self -Wignored-qualifiers $gcc_flags"
 gcc_flags="-Wmissing-include-dirs -Wempty-body -Wnested-externs $gcc_flags"
 gcc_flags="-fstack-protector-all -Wendif-labels $gcc_flags"
+gcc_flags="-Wno-initializer-overrides $gcc_flags"
 # Note that we do not add -Werror to gcc_flags here, because that would
 # enable it for all configure tests. If a configure test failed due
 # to -Werror this would just silently disable some features,
@@ -1215,11 +1174,30 @@ cat > $TMPC << EOF
 int main(void) { return 0; }
 EOF
 for flag in $gcc_flags; do
-    if compile_prog "-Werror $flag" "" ; then
+    # Use the positive sense of the flag when testing for -Wno-wombat
+    # support (gcc will happily accept the -Wno- form of unknown
+    # warning options).
+    optflag="$(echo $flag | sed -e 's/^-Wno-/-W/')"
+    if compile_prog "-Werror $optflag" "" ; then
 	QEMU_CFLAGS="$QEMU_CFLAGS $flag"
     fi
 done
 
+# Workaround for http://gcc.gnu.org/PR55489.  Happens with -fPIE/-fPIC and
+# large functions that use global variables.  The bug is in all releases of
+# GCC, but it became particularly acute in 4.6.x and 4.7.x.  It is fixed in
+# 4.7.3 and 4.8.0.  We should be able to delete this at the end of 2013.
+cat > $TMPC << EOF
+#if __GNUC__ == 4 && (__GNUC_MINOR__ == 6 || (__GNUC_MINOR__ == 7 && __GNUC_PATCHLEVEL__ <= 2))
+int main(void) { return 0; }
+#else
+#error No bug in this compiler.
+#endif
+EOF
+if compile_prog "-Werror -fno-gcse" "" ; then
+  TRANSLATE_OPT_CFLAGS=-fno-gcse
+fi
+
 if test "$static" = "yes" ; then
   if test "$pie" = "yes" ; then
     echo "static and pie are mutually incompatible"
@@ -1314,15 +1292,11 @@ if ! "$python" -c 'import sys; sys.exit(sys.version_info < (2,4) or sys.version_
   exit 1
 fi
 
-if test -z "$target_list" ; then
+if test -z "${target_list+xxx}" ; then
     target_list="$default_target_list"
 else
     target_list=`echo "$target_list" | sed -e 's/,/ /g'`
 fi
-if test -z "$target_list" ; then
-    echo "No targets enabled"
-    exit 1
-fi
 # see if system emulation was really requested
 case " $target_list " in
   *"-softmmu "*) softmmu=yes
@@ -1347,7 +1321,7 @@ if test -z "$cross_prefix" ; then
 # big/little endian test
 cat > $TMPC << EOF
 #include <inttypes.h>
-int main(int argc, char ** argv){
+int main(void) {
         volatile uint32_t i=0x01234567;
         return (*((uint8_t*)(&i))) == 0x67;
 }
@@ -1424,14 +1398,14 @@ fi
 # libseccomp check
 
 if test "$seccomp" != "no" ; then
-    if $pkg_config libseccomp --modversion >/dev/null 2>&1; then
+    if $pkg_config --atleast-version=1.0.0 libseccomp --modversion >/dev/null 2>&1; then
         LIBS=`$pkg_config --libs libseccomp`
 	seccomp="yes"
     else
-	seccomp="no"
 	if test "$seccomp" = "yes"; then
             feature_not_found "libseccomp"
 	fi
+	seccomp="no"
     fi
 fi
 ##########################################
@@ -2141,6 +2115,33 @@ else
 fi
 
 ##########################################
+# pixman support probe
+
+if test "$pixman" = ""; then
+  if $pkg_config pixman-1 > /dev/null 2>&1; then
+    pixman="system"
+  else
+    pixman="internal"
+  fi
+fi
+if test "$pixman" = "system"; then
+  pixman_cflags=`$pkg_config --cflags pixman-1 2>/dev/null`
+  pixman_libs=`$pkg_config --libs pixman-1 2>/dev/null`
+else
+  if test ! -d ${source_path}/pixman/pixman; then
+    echo "ERROR: pixman not present. Your options:"
+    echo "  (1) Prefered: Install the pixman devel package (any recent"
+    echo "      distro should have packages as Xorg needs pixman too)."
+    echo "  (2) Fetch the pixman submodule, using:"
+    echo "      git submodule update --init pixman"
+    exit 1
+  fi
+  mkdir -p pixman/pixman
+  pixman_cflags="-I\$(SRC_PATH)/pixman/pixman -I\$(BUILD_DIR)/pixman/pixman"
+  pixman_libs="-L\$(BUILD_DIR)/pixman/pixman/.libs -lpixman-1"
+fi
+
+##########################################
 # libcap probe
 
 if test "$cap" != "no" ; then
@@ -2340,6 +2341,29 @@ EOF
   fi
 fi
 
+##########################################
+# glusterfs probe
+if test "$glusterfs" != "no" ; then
+  cat > $TMPC <<EOF
+#include <glusterfs/api/glfs.h>
+int main(void) {
+    (void) glfs_new("volume");
+    return 0;
+}
+EOF
+  glusterfs_libs="-lgfapi -lgfrpc -lgfxdr"
+  if compile_prog "" "$glusterfs_libs" ; then
+    glusterfs=yes
+    libs_tools="$glusterfs_libs $libs_tools"
+    libs_softmmu="$glusterfs_libs $libs_softmmu"
+  else
+    if test "$glusterfs" = "yes" ; then
+      feature_not_found "GlusterFS backend support"
+    fi
+    glusterfs=no
+  fi
+fi
+
 #
 # Check for xxxat() functions when we are building linux-user
 # emulator.  This is done because older glibc versions don't
@@ -2426,8 +2450,7 @@ cat > $TMPC << EOF
 int main(void)
 {
     int pipefd[2];
-    pipe2(pipefd, O_CLOEXEC);
-    return 0;
+    return pipe2(pipefd, O_CLOEXEC);
 }
 EOF
 if compile_prog "" "" ; then
@@ -2668,17 +2691,44 @@ fi
 
 
 ##########################################
+# Do we need libm
+cat > $TMPC << EOF
+#include <math.h>
+int main(void) { return isnan(sin(0.0)); }
+EOF
+if compile_prog "" "" ; then
+  :
+elif compile_prog "" "-lm" ; then
+  LIBS="-lm $LIBS"
+  libs_qga="-lm $libs_qga"
+else
+  echo
+  echo "Error: libm check failed"
+  echo
+  exit 1
+fi
+
+##########################################
 # Do we need librt
+# uClibc provides 2 versions of clock_gettime(), one with realtime
+# support and one without. This means that the clock_gettime() don't
+# need -lrt. We still need it for timer_create() so we check for this
+# function in addition.
 cat > $TMPC <<EOF
 #include <signal.h>
 #include <time.h>
-int main(void) { return clock_gettime(CLOCK_REALTIME, NULL); }
+int main(void) {
+  timer_create(CLOCK_REALTIME, NULL, NULL);
+  return clock_gettime(CLOCK_REALTIME, NULL);
+}
 EOF
 
 if compile_prog "" "" ; then
   :
-elif compile_prog "" "-lrt" ; then
+# we need pthread for static linking. use previous pthread test result
+elif compile_prog "" "-lrt $pthread_lib" ; then
   LIBS="-lrt $LIBS"
+  libs_qga="-lrt $libs_qga"
 fi
 
 if test "$darwin" != "yes" -a "$mingw32" != "yes" -a "$solaris" != yes -a \
@@ -2695,12 +2745,14 @@ int main(void) { spice_server_new(); return 0; }
 EOF
   spice_cflags=$($pkg_config --cflags spice-protocol spice-server 2>/dev/null)
   spice_libs=$($pkg_config --libs spice-protocol spice-server 2>/dev/null)
-  if $pkg_config --atleast-version=0.8.2 spice-server >/dev/null 2>&1 && \
-     $pkg_config --atleast-version=0.8.1 spice-protocol > /dev/null 2>&1 && \
+  if $pkg_config --atleast-version=0.12.0 spice-server >/dev/null 2>&1 && \
+     $pkg_config --atleast-version=0.12.2 spice-protocol > /dev/null 2>&1 && \
      compile_prog "$spice_cflags" "$spice_libs" ; then
     spice="yes"
     libs_softmmu="$libs_softmmu $spice_libs"
     QEMU_CFLAGS="$QEMU_CFLAGS $spice_cflags"
+    spice_protocol_version=$($pkg_config --modversion spice-protocol)
+    spice_server_version=$($pkg_config --modversion spice-server)
   else
     if test "$spice" = "yes" ; then
       feature_not_found "spice"
@@ -2750,12 +2802,12 @@ fi
 
 # check for usbredirparser for usb network redirection support
 if test "$usb_redir" != "no" ; then
-    if $pkg_config --atleast-version=0.3.4 libusbredirparser >/dev/null 2>&1 ; then
+    if $pkg_config --atleast-version=0.5.3 libusbredirparser-0.5 >/dev/null 2>&1 ; then
         usb_redir="yes"
-        usb_redir_cflags=$($pkg_config --cflags libusbredirparser 2>/dev/null)
-        usb_redir_libs=$($pkg_config --libs libusbredirparser 2>/dev/null)
+        usb_redir_cflags=$($pkg_config --cflags libusbredirparser-0.5 2>/dev/null)
+        usb_redir_libs=$($pkg_config --libs libusbredirparser-0.5 2>/dev/null)
         QEMU_CFLAGS="$QEMU_CFLAGS $usb_redir_cflags"
-        LIBS="$LIBS $usb_redir_libs"
+        libs_softmmu="$libs_softmmu $usb_redir_libs"
     else
         if test "$usb_redir" = "yes"; then
             feature_not_found "usb-redir"
@@ -2812,6 +2864,24 @@ if compile_prog "" "" ; then
 fi
 
 ##########################################
+# check if we have usable SIGEV_THREAD_ID
+
+sigev_thread_id=no
+cat > $TMPC << EOF
+#include <signal.h>
+int main(void) {
+  struct sigevent ev;
+  ev.sigev_notify = SIGEV_THREAD_ID;
+  ev._sigev_un._tid = 0;
+  asm volatile("" : : "g"(&ev));
+  return 0;
+}
+EOF
+if compile_prog "" "" ; then
+    sigev_thread_id=yes
+fi
+
+##########################################
 # check if trace backend exists
 
 $python "$source_path/scripts/tracetool.py" "--backend=$trace_backend" --check-backend  > /dev/null 2> /dev/null
@@ -2868,7 +2938,7 @@ static int sfaa(int *ptr)
   return __sync_fetch_and_and(ptr, 0);
 }
 
-int main(int argc, char **argv)
+int main(void)
 {
   int val = 42;
   sfaa(&val);
@@ -2963,11 +3033,12 @@ if compile_prog "-Werror" "" ; then
 fi
 
 ########################################
-# check if we have valgrind/valgrind.h
+# check if we have valgrind/valgrind.h and valgrind/memcheck.h
 
 valgrind_h=no
 cat > $TMPC << EOF
 #include <valgrind/valgrind.h>
+#include <valgrind/memcheck.h>
 int main(void) {
   return 0;
 }
@@ -3039,9 +3110,14 @@ fi
 qemu_confdir=$sysconfdir$confsuffix
 qemu_datadir=$datadir$confsuffix
 
-tools=
-if test "$softmmu" = yes ; then
+tools=""
+if test "$want_tools" = "yes" ; then
   tools="qemu-img\$(EXESUF) qemu-io\$(EXESUF) $tools"
+  if [ "$linux" = "yes" -o "$bsd" = "yes" -o "$solaris" = "yes" ] ; then
+    tools="qemu-nbd\$(EXESUF) $tools"
+  fi
+fi
+if test "$softmmu" = yes ; then
   if test "$virtfs" != no ; then
     if test "$cap" = yes && test "$linux" = yes && test "$attr" = yes ; then
       virtfs=yes
@@ -3055,14 +3131,13 @@ if test "$softmmu" = yes ; then
     fi
   fi
   if [ "$linux" = "yes" -o "$bsd" = "yes" -o "$solaris" = "yes" ] ; then
-      tools="qemu-nbd\$(EXESUF) $tools"
     if [ "$guest_agent" = "yes" ]; then
       tools="qemu-ga\$(EXESUF) $tools"
     fi
   fi
-fi
-if test "$smartcard_nss" = "yes" ; then
-  tools="vscclient\$(EXESUF) $tools"
+  if test "$smartcard_nss" = "yes" ; then
+    tools="vscclient\$(EXESUF) $tools"
+  fi
 fi
 
 # Mac OS X ships with a broken assembler
@@ -3076,12 +3151,18 @@ if test "$cpu" = "ppc64" -a "$targetos" != "Darwin" ; then
   roms="$roms spapr-rtas"
 fi
 
+# add pixman flags after all config tests are done
+QEMU_CFLAGS="$QEMU_CFLAGS $pixman_cflags"
+libs_softmmu="$libs_softmmu $pixman_libs"
+
 echo "Install prefix    $prefix"
 echo "BIOS directory    `eval echo $qemu_datadir`"
 echo "binary directory  `eval echo $bindir`"
 echo "library directory `eval echo $libdir`"
+echo "libexec directory `eval echo $libexecdir`"
 echo "include directory `eval echo $includedir`"
 echo "config directory  `eval echo $sysconfdir`"
+echo "local state directory   `eval echo $local_statedir`"
 if test "$mingw32" = "no" ; then
 echo "Manual directory  `eval echo $mandir`"
 echo "ELF interp prefix $interp_prefix"
@@ -3112,6 +3193,7 @@ echo "-Werror enabled   $werror"
 if test "$darwin" = "yes" ; then
     echo "Cocoa support     $cocoa"
 fi
+echo "pixman            $pixman"
 echo "SDL support       $sdl"
 echo "curses support    $curses"
 echo "curl support      $curl"
@@ -3151,12 +3233,13 @@ echo "preadv support    $preadv"
 echo "fdatasync         $fdatasync"
 echo "madvise           $madvise"
 echo "posix_madvise     $posix_madvise"
+echo "sigev_thread_id   $sigev_thread_id"
 echo "uuid support      $uuid"
 echo "libcap-ng support $cap_ng"
 echo "vhost-net support $vhost_net"
 echo "Trace backend     $trace_backend"
 echo "Trace output file $trace_file-<pid>"
-echo "spice support     $spice"
+echo "spice support     $spice ($spice_protocol_version/$spice_server_version)"
 echo "rbd support       $rbd"
 echo "xfsctl support    $xfs"
 echo "nss used          $smartcard_nss"
@@ -3166,6 +3249,7 @@ echo "libiscsi support  $libiscsi"
 echo "build guest agent $guest_agent"
 echo "seccomp support   $seccomp"
 echo "coroutine backend $coroutine_backend"
+echo "GlusterFS support $glusterfs"
 
 if test "$sdl_too_old" = "yes"; then
 echo "-> Your SDL version is too old - please upgrade to have SDL support"
@@ -3183,14 +3267,15 @@ echo all: >> $config_host_mak
 echo "prefix=$prefix" >> $config_host_mak
 echo "bindir=$bindir" >> $config_host_mak
 echo "libdir=$libdir" >> $config_host_mak
+echo "libexecdir=$libexecdir" >> $config_host_mak
 echo "includedir=$includedir" >> $config_host_mak
 echo "mandir=$mandir" >> $config_host_mak
 echo "sysconfdir=$sysconfdir" >> $config_host_mak
 echo "qemu_confdir=$qemu_confdir" >> $config_host_mak
 echo "qemu_datadir=$qemu_datadir" >> $config_host_mak
 echo "qemu_docdir=$qemu_docdir" >> $config_host_mak
-echo "libexecdir=\${prefix}/libexec" >> $config_host_mak
-echo "CONFIG_QEMU_HELPERDIR=\"$prefix/libexec\"" >> $config_host_mak
+echo "qemu_localstatedir=$local_statedir" >> $config_host_mak
+echo "qemu_helperdir=$libexecdir" >> $config_host_mak
 
 echo "ARCH=$ARCH" >> $config_host_mak
 if test "$debug_tcg" = "yes" ; then
@@ -3433,6 +3518,9 @@ fi
 if test "$posix_madvise" = "yes" ; then
   echo "CONFIG_POSIX_MADVISE=y" >> $config_host_mak
 fi
+if test "$sigev_thread_id" = "yes" ; then
+  echo "CONFIG_SIGEV_THREAD_ID=y" >> $config_host_mak
+fi
 
 if test "$spice" = "yes" ; then
   echo "CONFIG_SPICE=y" >> $config_host_mak
@@ -3504,6 +3592,10 @@ if test "$has_environ" = "yes" ; then
   echo "CONFIG_HAS_ENVIRON=y" >> $config_host_mak
 fi
 
+if test "$glusterfs" = "yes" ; then
+  echo "CONFIG_GLUSTERFS=y" >> $config_host_mak
+fi
+
 # USB host support
 case "$usb" in
 linux)
@@ -3572,7 +3664,11 @@ if test "$sparse" = "yes" ; then
   echo "HOST_CC      := REAL_CC=\"\$(HOST_CC)\" cgcc"  >> $config_host_mak
   echo "QEMU_CFLAGS  += -Wbitwise -Wno-transparent-union -Wno-old-initializer -Wno-non-pointer-null" >> $config_host_mak
 fi
-echo "HELPER_CFLAGS=$helper_cflags" >> $config_host_mak
+if test "$cross_prefix" != ""; then
+  echo "AUTOCONF_HOST := --host=${cross_prefix%-}"     >> $config_host_mak
+else
+  echo "AUTOCONF_HOST := "                             >> $config_host_mak
+fi
 echo "LDFLAGS=$LDFLAGS" >> $config_host_mak
 echo "ARLIBS_BEGIN=$arlibs_begin" >> $config_host_mak
 echo "ARLIBS_END=$arlibs_end" >> $config_host_mak
@@ -3581,6 +3677,7 @@ echo "LIBS_TOOLS+=$libs_tools" >> $config_host_mak
 echo "EXESUF=$EXESUF" >> $config_host_mak
 echo "LIBS_QGA+=$libs_qga" >> $config_host_mak
 echo "POD2MAN=$POD2MAN" >> $config_host_mak
+echo "TRANSLATE_OPT_CFLAGS=$TRANSLATE_OPT_CFLAGS" >> $config_host_mak
 
 # generate list of library paths for linker script
 
@@ -3683,15 +3780,12 @@ TARGET_ABI_DIR=""
 
 case "$target_arch2" in
   i386)
-    target_phys_bits=64
   ;;
   x86_64)
     TARGET_BASE_ARCH=i386
-    target_phys_bits=64
     target_long_alignment=8
   ;;
   alpha)
-    target_phys_bits=64
     target_long_alignment=8
     target_nptl="yes"
   ;;
@@ -3700,22 +3794,18 @@ case "$target_arch2" in
     bflt="yes"
     target_nptl="yes"
     gdb_xml_files="arm-core.xml arm-vfp.xml arm-vfp3.xml arm-neon.xml"
-    target_phys_bits=64
     target_llong_alignment=4
     target_libs_softmmu="$fdt_libs"
   ;;
   cris)
     target_nptl="yes"
-    target_phys_bits=32
   ;;
   lm32)
-    target_phys_bits=32
     target_libs_softmmu="$opengl_libs"
   ;;
   m68k)
     bflt="yes"
     gdb_xml_files="cf-core.xml cf-fp.xml"
-    target_phys_bits=32
     target_int_alignment=2
     target_long_alignment=2
     target_llong_alignment=2
@@ -3724,36 +3814,30 @@ case "$target_arch2" in
     TARGET_ARCH=microblaze
     bflt="yes"
     target_nptl="yes"
-    target_phys_bits=32
     target_libs_softmmu="$fdt_libs"
   ;;
   mips|mipsel)
     TARGET_ARCH=mips
     echo "TARGET_ABI_MIPSO32=y" >> $config_target_mak
     target_nptl="yes"
-    target_phys_bits=64
   ;;
   mipsn32|mipsn32el)
     TARGET_ARCH=mipsn32
     TARGET_BASE_ARCH=mips
     echo "TARGET_ABI_MIPSN32=y" >> $config_target_mak
-    target_phys_bits=64
   ;;
   mips64|mips64el)
     TARGET_ARCH=mips64
     TARGET_BASE_ARCH=mips
     echo "TARGET_ABI_MIPSN64=y" >> $config_target_mak
-    target_phys_bits=64
     target_long_alignment=8
   ;;
   or32)
     TARGET_ARCH=openrisc
     TARGET_BASE_ARCH=openrisc
-    target_phys_bits=32
   ;;
   ppc)
     gdb_xml_files="power-core.xml power-fpu.xml power-altivec.xml power-spe.xml"
-    target_phys_bits=64
     target_nptl="yes"
     target_libs_softmmu="$fdt_libs"
   ;;
@@ -3761,7 +3845,6 @@ case "$target_arch2" in
     TARGET_BASE_ARCH=ppc
     TARGET_ABI_DIR=ppc
     gdb_xml_files="power-core.xml power-fpu.xml power-altivec.xml power-spe.xml"
-    target_phys_bits=64
     target_nptl="yes"
     target_libs_softmmu="$fdt_libs"
   ;;
@@ -3769,7 +3852,6 @@ case "$target_arch2" in
     TARGET_BASE_ARCH=ppc
     TARGET_ABI_DIR=ppc
     gdb_xml_files="power64-core.xml power-fpu.xml power-altivec.xml power-spe.xml"
-    target_phys_bits=64
     target_long_alignment=8
     target_libs_softmmu="$fdt_libs"
   ;;
@@ -3779,21 +3861,17 @@ case "$target_arch2" in
     TARGET_ABI_DIR=ppc
     echo "TARGET_ABI32=y" >> $config_target_mak
     gdb_xml_files="power64-core.xml power-fpu.xml power-altivec.xml power-spe.xml"
-    target_phys_bits=64
     target_libs_softmmu="$fdt_libs"
   ;;
   sh4|sh4eb)
     TARGET_ARCH=sh4
     bflt="yes"
     target_nptl="yes"
-    target_phys_bits=32
   ;;
   sparc)
-    target_phys_bits=64
   ;;
   sparc64)
     TARGET_BASE_ARCH=sparc
-    target_phys_bits=64
     target_long_alignment=8
   ;;
   sparc32plus)
@@ -3801,19 +3879,15 @@ case "$target_arch2" in
     TARGET_BASE_ARCH=sparc
     TARGET_ABI_DIR=sparc
     echo "TARGET_ABI32=y" >> $config_target_mak
-    target_phys_bits=64
   ;;
   s390x)
     target_nptl="yes"
-    target_phys_bits=64
     target_long_alignment=8
   ;;
   unicore32)
-    target_phys_bits=32
   ;;
   xtensa|xtensaeb)
     TARGET_ARCH=xtensa
-    target_phys_bits=32
   ;;
   *)
     echo "Unsupported target CPU"
@@ -3827,17 +3901,19 @@ fi
 
 symlink "$source_path/Makefile.target" "$target_dir/Makefile"
 
-
-case "$target_arch2" in
-  alpha | i386 | or32 | sparc* | x86_64 | xtensa* | ppc*)
-    echo "CONFIG_TCG_PASS_AREG0=y" >> $config_target_mak
-  ;;
-esac
-
 upper() {
     echo "$@"| LC_ALL=C tr '[a-z]' '[A-Z]'
 }
 
+case "$cpu" in
+  i386|x86_64|ppc)
+    # The TCG interpreter currently does not support ld/st optimization.
+    if test "$tcg_interpreter" = "no" ; then
+        echo "CONFIG_QEMU_LDST_OPTIMIZATION=y" >> $config_target_mak
+    fi
+  ;;
+esac
+
 echo "TARGET_SHORT_ALIGNMENT=$target_short_alignment" >> $config_target_mak
 echo "TARGET_INT_ALIGNMENT=$target_int_alignment" >> $config_target_mak
 echo "TARGET_LONG_ALIGNMENT=$target_long_alignment" >> $config_target_mak
@@ -3855,7 +3931,6 @@ echo "TARGET_ABI_DIR=$TARGET_ABI_DIR" >> $config_target_mak
 case "$target_arch2" in
   i386|x86_64)
     if test "$xen" = "yes" -a "$target_softmmu" = "yes" ; then
-      target_phys_bits=64
       echo "CONFIG_XEN=y" >> $config_target_mak
       if test "$xen_pci_passthrough" = yes; then
         echo "CONFIG_XEN_PCI_PASSTHROUGH=y" >> "$config_target_mak"
@@ -3895,11 +3970,8 @@ if test "$target_bigendian" = "yes" ; then
   echo "TARGET_WORDS_BIGENDIAN=y" >> $config_target_mak
 fi
 if test "$target_softmmu" = "yes" ; then
-  echo "TARGET_PHYS_ADDR_BITS=$target_phys_bits" >> $config_target_mak
   echo "CONFIG_SOFTMMU=y" >> $config_target_mak
   echo "LIBS+=$libs_softmmu $target_libs_softmmu" >> $config_target_mak
-  echo "HWDIR=../libhw$target_phys_bits" >> $config_target_mak
-  echo "subdir-$target: subdir-libhw$target_phys_bits" >> $config_host_mak
   if test "$smartcard_nss" = "yes" ; then
     echo "subdir-$target: subdir-libcacard" >> $config_host_mak
   fi
@@ -4085,10 +4157,6 @@ fi
 
 if test "$target_linux_user" = "yes" -o "$target_bsd_user" = "yes" ; then
   case "$ARCH" in
-  sparc)
-    # -static is used to avoid g1/g3 usage by the dynamic linker
-    ldflags="$linker_script -static $ldflags"
-    ;;
   alpha | s390x)
     # The default placement of the application is fine.
     ;;
@@ -4104,6 +4172,10 @@ echo "QEMU_INCLUDES+=$includes" >> $config_target_mak
 
 done # for target in $targets
 
+if [ "$pixman" = "internal" ]; then
+  echo "config-host.h: subdir-pixman" >> $config_host_mak
+fi
+
 # build tree in object directory in case the source is not in the current directory
 DIRS="tests tests/tcg tests/tcg/cris tests/tcg/lm32"
 DIRS="$DIRS pc-bios/optionrom pc-bios/spapr-rtas"
@@ -4145,12 +4217,6 @@ for rom in seabios vgabios ; do
     echo "LD=$ld" >> $config_mak
 done
 
-for hwlib in 32 64; do
-  d=libhw$hwlib
-  symlink "$source_path/Makefile.hw" "$d/Makefile"
-  echo "QEMU_CFLAGS+=-DTARGET_PHYS_ADDR_BITS=$hwlib" > $d/config.mak
-done
-
 d=libuser
 symlink "$source_path/Makefile.user" "$d/Makefile"
 
diff --git a/console.c b/console.c
index 3b5cabba3..048b48e72 100644
--- a/console.c
+++ b/console.c
@@ -24,6 +24,7 @@
 #include "qemu-common.h"
 #include "console.h"
 #include "qemu-timer.h"
+#include "qmp-commands.h"
 
 //#define DEBUG_CONSOLE
 #define DEFAULT_BACKSCROLL 512
@@ -113,20 +114,20 @@ typedef enum {
     TEXT_CONSOLE_FIXED_SIZE
 } console_type_t;
 
-/* ??? This is mis-named.
-   It is used for both text and graphical consoles.  */
-struct TextConsole {
+struct QemuConsole {
     int index;
     console_type_t console_type;
     DisplayState *ds;
+
     /* Graphic console state.  */
     vga_hw_update_ptr hw_update;
     vga_hw_invalidate_ptr hw_invalidate;
     vga_hw_screen_dump_ptr hw_screen_dump;
     vga_hw_text_update_ptr hw_text_update;
     void *hw;
-
     int g_width, g_height;
+
+    /* Text console state */
     int width;
     int height;
     int total_height;
@@ -160,8 +161,8 @@ struct TextConsole {
 };
 
 static DisplayState *display_state;
-static TextConsole *active_console;
-static TextConsole *consoles[MAX_CONSOLES];
+static QemuConsole *active_console;
+static QemuConsole *consoles[MAX_CONSOLES];
 static int nb_consoles = 0;
 
 void vga_hw_update(void)
@@ -176,9 +177,9 @@ void vga_hw_invalidate(void)
         active_console->hw_invalidate(active_console->hw);
 }
 
-void vga_hw_screen_dump(const char *filename)
+void qmp_screendump(const char *filename, Error **errp)
 {
-    TextConsole *previous_active_console;
+    QemuConsole *previous_active_console;
     bool cswitch;
 
     previous_active_console = active_console;
@@ -190,9 +191,9 @@ void vga_hw_screen_dump(const char *filename)
         console_select(0);
     }
     if (consoles[0] && consoles[0]->hw_screen_dump) {
-        consoles[0]->hw_screen_dump(consoles[0]->hw, filename, cswitch);
+        consoles[0]->hw_screen_dump(consoles[0]->hw, filename, cswitch, errp);
     } else {
-        error_report("screen dump not implemented");
+        error_setg(errp, "device doesn't support screendump\n");
     }
 
     if (cswitch) {
@@ -520,7 +521,7 @@ static void vga_putcharxy(DisplayState *ds, int x, int y, int ch,
     }
 }
 
-static void text_console_resize(TextConsole *s)
+static void text_console_resize(QemuConsole *s)
 {
     TextCell *cells, *c, *c1;
     int w1, x, y, last_width;
@@ -552,7 +553,7 @@ static void text_console_resize(TextConsole *s)
     s->cells = cells;
 }
 
-static inline void text_update_xy(TextConsole *s, int x, int y)
+static inline void text_update_xy(QemuConsole *s, int x, int y)
 {
     s->text_x[0] = MIN(s->text_x[0], x);
     s->text_x[1] = MAX(s->text_x[1], x);
@@ -560,7 +561,7 @@ static inline void text_update_xy(TextConsole *s, int x, int y)
     s->text_y[1] = MAX(s->text_y[1], y);
 }
 
-static void invalidate_xy(TextConsole *s, int x, int y)
+static void invalidate_xy(QemuConsole *s, int x, int y)
 {
     if (s->update_x0 > x * FONT_WIDTH)
         s->update_x0 = x * FONT_WIDTH;
@@ -572,7 +573,7 @@ static void invalidate_xy(TextConsole *s, int x, int y)
         s->update_y1 = (y + 1) * FONT_HEIGHT;
 }
 
-static void update_xy(TextConsole *s, int x, int y)
+static void update_xy(QemuConsole *s, int x, int y)
 {
     TextCell *c;
     int y1, y2;
@@ -596,7 +597,7 @@ static void update_xy(TextConsole *s, int x, int y)
     }
 }
 
-static void console_show_cursor(TextConsole *s, int show)
+static void console_show_cursor(QemuConsole *s, int show)
 {
     TextCell *c;
     int y, y1;
@@ -630,42 +631,45 @@ static void console_show_cursor(TextConsole *s, int show)
     }
 }
 
-static void console_refresh(TextConsole *s)
+static void console_refresh(QemuConsole *s)
 {
     TextCell *c;
     int x, y, y1;
 
     if (s != active_console)
         return;
-    if (!ds_get_bits_per_pixel(s->ds)) {
+
+    if (s->ds->have_text) {
         s->text_x[0] = 0;
         s->text_y[0] = 0;
         s->text_x[1] = s->width - 1;
         s->text_y[1] = s->height - 1;
         s->cursor_invalidate = 1;
-        return;
     }
 
-    vga_fill_rect(s->ds, 0, 0, ds_get_width(s->ds), ds_get_height(s->ds),
-                  color_table[0][COLOR_BLACK]);
-    y1 = s->y_displayed;
-    for(y = 0; y < s->height; y++) {
-        c = s->cells + y1 * s->width;
-        for(x = 0; x < s->width; x++) {
-            vga_putcharxy(s->ds, x, y, c->ch,
-                          &(c->t_attrib));
-            c++;
+    if (s->ds->have_gfx) {
+        vga_fill_rect(s->ds, 0, 0, ds_get_width(s->ds), ds_get_height(s->ds),
+                      color_table[0][COLOR_BLACK]);
+        y1 = s->y_displayed;
+        for (y = 0; y < s->height; y++) {
+            c = s->cells + y1 * s->width;
+            for (x = 0; x < s->width; x++) {
+                vga_putcharxy(s->ds, x, y, c->ch,
+                              &(c->t_attrib));
+                c++;
+            }
+            if (++y1 == s->total_height) {
+                y1 = 0;
+            }
         }
-        if (++y1 == s->total_height)
-            y1 = 0;
+        console_show_cursor(s, 1);
+        dpy_gfx_update(s->ds, 0, 0, ds_get_width(s->ds), ds_get_height(s->ds));
     }
-    console_show_cursor(s, 1);
-    dpy_update(s->ds, 0, 0, ds_get_width(s->ds), ds_get_height(s->ds));
 }
 
 static void console_scroll(int ydelta)
 {
-    TextConsole *s;
+    QemuConsole *s;
     int i, y1;
 
     s = active_console;
@@ -697,7 +701,7 @@ static void console_scroll(int ydelta)
     console_refresh(s);
 }
 
-static void console_put_lf(TextConsole *s)
+static void console_put_lf(QemuConsole *s)
 {
     TextCell *c;
     int x, y1;
@@ -748,7 +752,7 @@ static void console_put_lf(TextConsole *s)
  * NOTE: I know this code is not very efficient (checking every color for it
  * self) but it is more readable and better maintainable.
  */
-static void console_handle_escape(TextConsole *s)
+static void console_handle_escape(QemuConsole *s)
 {
     int i;
 
@@ -841,7 +845,7 @@ static void console_handle_escape(TextConsole *s)
     }
 }
 
-static void console_clear_xy(TextConsole *s, int x, int y)
+static void console_clear_xy(QemuConsole *s, int x, int y)
 {
     int y1 = (s->y_base + y) % s->total_height;
     TextCell *c = &s->cells[y1 * s->width + x];
@@ -851,7 +855,7 @@ static void console_clear_xy(TextConsole *s, int x, int y)
 }
 
 /* set cursor, checking bounds */
-static void set_cursor(TextConsole *s, int x, int y)
+static void set_cursor(QemuConsole *s, int x, int y)
 {
     if (x < 0) {
         x = 0;
@@ -870,7 +874,7 @@ static void set_cursor(TextConsole *s, int x, int y)
     s->y = y;
 }
 
-static void console_putchar(TextConsole *s, int ch)
+static void console_putchar(QemuConsole *s, int ch)
 {
     TextCell *c;
     int y1, i;
@@ -937,8 +941,11 @@ static void console_putchar(TextConsole *s, int ch)
     case TTY_STATE_CSI: /* handle escape sequence parameters */
         if (ch >= '0' && ch <= '9') {
             if (s->nb_esc_params < MAX_ESC_PARAMS) {
-                s->esc_params[s->nb_esc_params] =
-                    s->esc_params[s->nb_esc_params] * 10 + ch - '0';
+                int *param = &s->esc_params[s->nb_esc_params];
+                int digit = (ch - '0');
+
+                *param = (*param <= (INT_MAX - digit) / 10) ?
+                         *param * 10 + digit : INT_MAX;
             }
         } else {
             if (s->nb_esc_params < MAX_ESC_PARAMS)
@@ -1074,7 +1081,7 @@ static void console_putchar(TextConsole *s, int ch)
 
 void console_select(unsigned int index)
 {
-    TextConsole *s;
+    QemuConsole *s;
 
     if (index >= MAX_CONSOLES)
         return;
@@ -1090,24 +1097,24 @@ void console_select(unsigned int index)
             qemu_del_timer(active_console->cursor_timer);
         }
         active_console = s;
-        if (ds_get_bits_per_pixel(s->ds)) {
+        if (ds->have_gfx) {
             ds->surface = qemu_resize_displaysurface(ds, s->g_width, s->g_height);
-        } else {
-            s->ds->surface->width = s->width;
-            s->ds->surface->height = s->height;
+            dpy_gfx_resize(ds);
+        }
+        if (ds->have_text) {
+            dpy_text_resize(ds, s->width, s->height);
         }
         if (s->cursor_timer) {
             qemu_mod_timer(s->cursor_timer,
                    qemu_get_clock_ms(rt_clock) + CONSOLE_CURSOR_PERIOD / 2);
         }
-        dpy_resize(s->ds);
         vga_hw_invalidate();
     }
 }
 
 static int console_puts(CharDriverState *chr, const uint8_t *buf, int len)
 {
-    TextConsole *s = chr->opaque;
+    QemuConsole *s = chr->opaque;
     int i;
 
     s->update_x0 = s->width * FONT_WIDTH;
@@ -1119,17 +1126,17 @@ static int console_puts(CharDriverState *chr, const uint8_t *buf, int len)
         console_putchar(s, buf[i]);
     }
     console_show_cursor(s, 1);
-    if (ds_get_bits_per_pixel(s->ds) && s->update_x0 < s->update_x1) {
-        dpy_update(s->ds, s->update_x0, s->update_y0,
-                   s->update_x1 - s->update_x0,
-                   s->update_y1 - s->update_y0);
+    if (s->ds->have_gfx && s->update_x0 < s->update_x1) {
+        dpy_gfx_update(s->ds, s->update_x0, s->update_y0,
+                       s->update_x1 - s->update_x0,
+                       s->update_y1 - s->update_y0);
     }
     return len;
 }
 
 static void kbd_send_chars(void *opaque)
 {
-    TextConsole *s = opaque;
+    QemuConsole *s = opaque;
     int len;
     uint8_t buf[16];
 
@@ -1152,7 +1159,7 @@ static void kbd_send_chars(void *opaque)
 /* called when an ascii key is pressed */
 void kbd_put_keysym(int keysym)
 {
-    TextConsole *s;
+    QemuConsole *s;
     uint8_t buf[16], *q;
     int c;
 
@@ -1207,7 +1214,7 @@ void kbd_put_keysym(int keysym)
 
 static void text_console_invalidate(void *opaque)
 {
-    TextConsole *s = (TextConsole *) opaque;
+    QemuConsole *s = (QemuConsole *) opaque;
     if (!ds_get_bits_per_pixel(s->ds) && s->console_type == TEXT_CONSOLE) {
         s->g_width = ds_get_width(s->ds);
         s->g_height = ds_get_height(s->ds);
@@ -1218,7 +1225,7 @@ static void text_console_invalidate(void *opaque)
 
 static void text_console_update(void *opaque, console_ch_t *chardata)
 {
-    TextConsole *s = (TextConsole *) opaque;
+    QemuConsole *s = (QemuConsole *) opaque;
     int i, j, src;
 
     if (s->text_x[0] <= s->text_x[1]) {
@@ -1230,23 +1237,23 @@ static void text_console_update(void *opaque, console_ch_t *chardata)
                                 (s->cells[src].t_attrib.fgcol << 12) |
                                 (s->cells[src].t_attrib.bgcol << 8) |
                                 (s->cells[src].t_attrib.bold << 21));
-        dpy_update(s->ds, s->text_x[0], s->text_y[0],
-                   s->text_x[1] - s->text_x[0], i - s->text_y[0]);
+        dpy_text_update(s->ds, s->text_x[0], s->text_y[0],
+                        s->text_x[1] - s->text_x[0], i - s->text_y[0]);
         s->text_x[0] = s->width;
         s->text_y[0] = s->height;
         s->text_x[1] = 0;
         s->text_y[1] = 0;
     }
     if (s->cursor_invalidate) {
-        dpy_cursor(s->ds, s->x, s->y);
+        dpy_text_cursor(s->ds, s->x, s->y);
         s->cursor_invalidate = 0;
     }
 }
 
-static TextConsole *get_graphic_console(DisplayState *ds)
+static QemuConsole *get_graphic_console(DisplayState *ds)
 {
     int i;
-    TextConsole *s;
+    QemuConsole *s;
     for (i = 0; i < nb_consoles; i++) {
         s = consoles[i];
         if (s->console_type == GRAPHIC_CONSOLE && s->ds == ds)
@@ -1255,14 +1262,14 @@ static TextConsole *get_graphic_console(DisplayState *ds)
     return NULL;
 }
 
-static TextConsole *new_console(DisplayState *ds, console_type_t console_type)
+static QemuConsole *new_console(DisplayState *ds, console_type_t console_type)
 {
-    TextConsole *s;
+    QemuConsole *s;
     int i;
 
     if (nb_consoles >= MAX_CONSOLES)
         return NULL;
-    s = g_malloc0(sizeof(TextConsole));
+    s = g_malloc0(sizeof(QemuConsole));
     if (!active_console || ((active_console->console_type != GRAPHIC_CONSOLE) &&
         (console_type == GRAPHIC_CONSOLE))) {
         active_console = s;
@@ -1287,85 +1294,86 @@ static TextConsole *new_console(DisplayState *ds, console_type_t console_type)
     return s;
 }
 
-static DisplaySurface* defaultallocator_create_displaysurface(int width, int height)
+static void qemu_alloc_display(DisplaySurface *surface, int width, int height,
+                               int linesize, PixelFormat pf, int newflags)
 {
-    DisplaySurface *surface = (DisplaySurface*) g_malloc0(sizeof(DisplaySurface));
+    surface->pf = pf;
 
-    int linesize = width * 4;
-    qemu_alloc_display(surface, width, height, linesize,
-                       qemu_default_pixelformat(32), 0);
-    return surface;
+    qemu_pixman_image_unref(surface->image);
+    surface->image = NULL;
+
+    surface->format = qemu_pixman_get_format(&pf);
+    assert(surface->format != 0);
+    surface->image = pixman_image_create_bits(surface->format,
+                                              width, height,
+                                              NULL, linesize);
+    assert(surface->image != NULL);
+
+    surface->flags = newflags | QEMU_ALLOCATED_FLAG;
+#ifdef HOST_WORDS_BIGENDIAN
+    surface->flags |= QEMU_BIG_ENDIAN_FLAG;
+#endif
 }
 
-static DisplaySurface* defaultallocator_resize_displaysurface(DisplaySurface *surface,
-                                          int width, int height)
+DisplaySurface *qemu_create_displaysurface(DisplayState *ds,
+                                           int width, int height)
 {
+    DisplaySurface *surface = g_new0(DisplaySurface, 1);
+
     int linesize = width * 4;
     qemu_alloc_display(surface, width, height, linesize,
                        qemu_default_pixelformat(32), 0);
     return surface;
 }
 
-void qemu_alloc_display(DisplaySurface *surface, int width, int height,
-                        int linesize, PixelFormat pf, int newflags)
+DisplaySurface *qemu_resize_displaysurface(DisplayState *ds,
+                                           int width, int height)
 {
-    void *data;
-    surface->width = width;
-    surface->height = height;
-    surface->linesize = linesize;
-    surface->pf = pf;
-    if (surface->flags & QEMU_ALLOCATED_FLAG) {
-        data = g_realloc(surface->data,
-                            surface->linesize * surface->height);
-    } else {
-        data = g_malloc(surface->linesize * surface->height);
-    }
-    surface->data = (uint8_t *)data;
-    surface->flags = newflags | QEMU_ALLOCATED_FLAG;
-#ifdef HOST_WORDS_BIGENDIAN
-    surface->flags |= QEMU_BIG_ENDIAN_FLAG;
-#endif
+    int linesize = width * 4;
+
+    trace_displaysurface_resize(ds, ds->surface, width, height);
+    qemu_alloc_display(ds->surface, width, height, linesize,
+                       qemu_default_pixelformat(32), 0);
+    return ds->surface;
 }
 
-DisplaySurface* qemu_create_displaysurface_from(int width, int height, int bpp,
-                                              int linesize, uint8_t *data)
+DisplaySurface *qemu_create_displaysurface_from(int width, int height, int bpp,
+                                                int linesize, uint8_t *data)
 {
-    DisplaySurface *surface = (DisplaySurface*) g_malloc0(sizeof(DisplaySurface));
+    DisplaySurface *surface = g_new0(DisplaySurface, 1);
 
-    surface->width = width;
-    surface->height = height;
-    surface->linesize = linesize;
     surface->pf = qemu_default_pixelformat(bpp);
+
+    surface->format = qemu_pixman_get_format(&surface->pf);
+    assert(surface->format != 0);
+    surface->image = pixman_image_create_bits(surface->format,
+                                              width, height,
+                                              (void *)data, linesize);
+    assert(surface->image != NULL);
+
 #ifdef HOST_WORDS_BIGENDIAN
     surface->flags = QEMU_BIG_ENDIAN_FLAG;
 #endif
-    surface->data = data;
 
     return surface;
 }
 
-static void defaultallocator_free_displaysurface(DisplaySurface *surface)
+void qemu_free_displaysurface(DisplayState *ds)
 {
-    if (surface == NULL)
+    trace_displaysurface_free(ds, ds->surface);
+    if (ds->surface == NULL) {
         return;
-    if (surface->flags & QEMU_ALLOCATED_FLAG)
-        g_free(surface->data);
-    g_free(surface);
+    }
+    qemu_pixman_image_unref(ds->surface->image);
+    g_free(ds->surface);
 }
 
-static struct DisplayAllocator default_allocator = {
-    defaultallocator_create_displaysurface,
-    defaultallocator_resize_displaysurface,
-    defaultallocator_free_displaysurface
-};
-
 static void dumb_display_init(void)
 {
     DisplayState *ds = g_malloc0(sizeof(DisplayState));
     int width = 640;
     int height = 480;
 
-    ds->allocator = &default_allocator;
     if (is_fixedsize_console()) {
         width = active_console->g_width;
         height = active_console->g_height;
@@ -1395,29 +1403,16 @@ DisplayState *get_displaystate(void)
     return display_state;
 }
 
-DisplayAllocator *register_displayallocator(DisplayState *ds, DisplayAllocator *da)
-{
-    if(ds->allocator ==  &default_allocator) {
-        DisplaySurface *surf;
-        surf = da->create_displaysurface(ds_get_width(ds), ds_get_height(ds));
-        defaultallocator_free_displaysurface(ds->surface);
-        ds->surface = surf;
-        ds->allocator = da;
-    }
-    return ds->allocator;
-}
-
 DisplayState *graphic_console_init(vga_hw_update_ptr update,
                                    vga_hw_invalidate_ptr invalidate,
                                    vga_hw_screen_dump_ptr screen_dump,
                                    vga_hw_text_update_ptr text_update,
                                    void *opaque)
 {
-    TextConsole *s;
+    QemuConsole *s;
     DisplayState *ds;
 
     ds = (DisplayState *) g_malloc0(sizeof(DisplayState));
-    ds->allocator = &default_allocator; 
     ds->surface = qemu_create_displaysurface(ds, 640, 480);
 
     s = new_console(ds, GRAPHIC_CONSOLE);
@@ -1459,14 +1454,14 @@ void console_color_init(DisplayState *ds)
 
 static void text_console_set_echo(CharDriverState *chr, bool echo)
 {
-    TextConsole *s = chr->opaque;
+    QemuConsole *s = chr->opaque;
 
     s->echo = echo;
 }
 
 static void text_console_update_cursor(void *opaque)
 {
-    TextConsole *s = opaque;
+    QemuConsole *s = opaque;
 
     s->cursor_visible_phase = !s->cursor_visible_phase;
     vga_hw_invalidate();
@@ -1476,7 +1471,7 @@ static void text_console_update_cursor(void *opaque)
 
 static void text_console_do_init(CharDriverState *chr, DisplayState *ds)
 {
-    TextConsole *s;
+    QemuConsole *s;
     static int color_inited;
 
     s = chr->opaque;
@@ -1539,7 +1534,7 @@ static void text_console_do_init(CharDriverState *chr, DisplayState *ds)
 CharDriverState *text_console_init(QemuOpts *opts)
 {
     CharDriverState *chr;
-    TextConsole *s;
+    QemuConsole *s;
     unsigned width;
     unsigned height;
 
@@ -1585,14 +1580,14 @@ void text_consoles_set_display(DisplayState *ds)
 
 void qemu_console_resize(DisplayState *ds, int width, int height)
 {
-    TextConsole *s = get_graphic_console(ds);
+    QemuConsole *s = get_graphic_console(ds);
     if (!s) return;
 
     s->g_width = width;
     s->g_height = height;
     if (is_graphic_console()) {
         ds->surface = qemu_resize_displaysurface(ds, width, height);
-        dpy_resize(ds);
+        dpy_gfx_resize(ds);
     }
 }
 
@@ -1600,7 +1595,7 @@ void qemu_console_copy(DisplayState *ds, int src_x, int src_y,
                        int dst_x, int dst_y, int w, int h)
 {
     if (is_graphic_console()) {
-        dpy_copy(ds, src_x, src_y, dst_x, dst_y, w, h);
+        dpy_gfx_copy(ds, src_x, src_y, dst_x, dst_y, w, h);
     }
 }
 
@@ -1611,7 +1606,7 @@ PixelFormat qemu_different_endianness_pixelformat(int bpp)
     memset(&pf, 0x00, sizeof(PixelFormat));
 
     pf.bits_per_pixel = bpp;
-    pf.bytes_per_pixel = bpp / 8;
+    pf.bytes_per_pixel = DIV_ROUND_UP(bpp, 8);
     pf.depth = bpp == 32 ? 24 : bpp;
 
     switch (bpp) {
@@ -1660,13 +1655,12 @@ PixelFormat qemu_default_pixelformat(int bpp)
     memset(&pf, 0x00, sizeof(PixelFormat));
 
     pf.bits_per_pixel = bpp;
-    pf.bytes_per_pixel = bpp / 8;
+    pf.bytes_per_pixel = DIV_ROUND_UP(bpp, 8);
     pf.depth = bpp == 32 ? 24 : bpp;
 
     switch (bpp) {
         case 15:
             pf.bits_per_pixel = 16;
-            pf.bytes_per_pixel = 2;
             pf.rmask = 0x00007c00;
             pf.gmask = 0x000003E0;
             pf.bmask = 0x0000001F;
@@ -1712,18 +1706,15 @@ PixelFormat qemu_default_pixelformat(int bpp)
             pf.rmask = 0x00FF0000;
             pf.gmask = 0x0000FF00;
             pf.bmask = 0x000000FF;
-            pf.amax = 255;
             pf.rmax = 255;
             pf.gmax = 255;
             pf.bmax = 255;
-            pf.ashift = 24;
             pf.rshift = 16;
             pf.gshift = 8;
             pf.bshift = 0;
             pf.rbits = 8;
             pf.gbits = 8;
             pf.bbits = 8;
-            pf.abits = 8;
             break;
         default:
             break;
diff --git a/console.h b/console.h
index 4334db5ca..50a0512f3 100644
--- a/console.h
+++ b/console.h
@@ -2,10 +2,13 @@
 #define CONSOLE_H
 
 #include "qemu-char.h"
+#include "qemu-pixman.h"
 #include "qdict.h"
 #include "notify.h"
 #include "monitor.h"
 #include "trace.h"
+#include "qapi-types.h"
+#include "error.h"
 
 /* keyboard/mouse support */
 
@@ -105,7 +108,6 @@ void kbd_put_keysym(int keysym);
 
 #define QEMU_BIG_ENDIAN_FLAG    0x01
 #define QEMU_ALLOCATED_FLAG     0x02
-#define QEMU_REALPIXELS_FLAG    0x04
 
 struct PixelFormat {
     uint8_t bits_per_pixel;
@@ -118,11 +120,9 @@ struct PixelFormat {
 };
 
 struct DisplaySurface {
+    pixman_format_code_t format;
+    pixman_image_t *image;
     uint8_t flags;
-    int width;
-    int height;
-    int linesize;        /* bytes per line */
-    uint8_t *data;
 
     struct PixelFormat pf;
 };
@@ -152,35 +152,32 @@ struct DisplayChangeListener {
     int idle;
     uint64_t gui_timer_interval;
 
-    void (*dpy_update)(struct DisplayState *s, int x, int y, int w, int h);
-    void (*dpy_resize)(struct DisplayState *s);
-    void (*dpy_setdata)(struct DisplayState *s);
     void (*dpy_refresh)(struct DisplayState *s);
-    void (*dpy_copy)(struct DisplayState *s, int src_x, int src_y,
-                     int dst_x, int dst_y, int w, int h);
-    void (*dpy_fill)(struct DisplayState *s, int x, int y,
-                     int w, int h, uint32_t c);
+
+    void (*dpy_gfx_update)(struct DisplayState *s, int x, int y, int w, int h);
+    void (*dpy_gfx_resize)(struct DisplayState *s);
+    void (*dpy_gfx_setdata)(struct DisplayState *s);
+    void (*dpy_gfx_copy)(struct DisplayState *s, int src_x, int src_y,
+                         int dst_x, int dst_y, int w, int h);
+
     void (*dpy_text_cursor)(struct DisplayState *s, int x, int y);
+    void (*dpy_text_resize)(struct DisplayState *s, int w, int h);
+    void (*dpy_text_update)(struct DisplayState *s, int x, int y, int w, int h);
 
-    struct DisplayChangeListener *next;
-};
+    void (*dpy_mouse_set)(struct DisplayState *s, int x, int y, int on);
+    void (*dpy_cursor_define)(struct DisplayState *s, QEMUCursor *cursor);
 
-struct DisplayAllocator {
-    DisplaySurface* (*create_displaysurface)(int width, int height);
-    DisplaySurface* (*resize_displaysurface)(DisplaySurface *surface, int width, int height);
-    void (*free_displaysurface)(DisplaySurface *surface);
+    QLIST_ENTRY(DisplayChangeListener) next;
 };
 
 struct DisplayState {
     struct DisplaySurface *surface;
     void *opaque;
     struct QEMUTimer *gui_timer;
+    bool have_gfx;
+    bool have_text;
 
-    struct DisplayAllocator* allocator;
-    struct DisplayChangeListener* listeners;
-
-    void (*mouse_set)(int x, int y, int on);
-    void (*cursor_define)(QEMUCursor *cursor);
+    QLIST_HEAD(, DisplayChangeListener) listeners;
 
     struct DisplayState *next;
 };
@@ -189,143 +186,220 @@ void register_displaystate(DisplayState *ds);
 DisplayState *get_displaystate(void);
 DisplaySurface* qemu_create_displaysurface_from(int width, int height, int bpp,
                                                 int linesize, uint8_t *data);
-void qemu_alloc_display(DisplaySurface *surface, int width, int height,
-                        int linesize, PixelFormat pf, int newflags);
 PixelFormat qemu_different_endianness_pixelformat(int bpp);
 PixelFormat qemu_default_pixelformat(int bpp);
 
-DisplayAllocator *register_displayallocator(DisplayState *ds, DisplayAllocator *da);
+DisplaySurface *qemu_create_displaysurface(DisplayState *ds,
+                                           int width, int height);
+DisplaySurface *qemu_resize_displaysurface(DisplayState *ds,
+                                           int width, int height);
+void qemu_free_displaysurface(DisplayState *ds);
 
-static inline DisplaySurface* qemu_create_displaysurface(DisplayState *ds, int width, int height)
+static inline int is_surface_bgr(DisplaySurface *surface)
 {
-    return ds->allocator->create_displaysurface(width, height);    
+    if (surface->pf.bits_per_pixel == 32 && surface->pf.rshift == 0)
+        return 1;
+    else
+        return 0;
 }
 
-static inline DisplaySurface* qemu_resize_displaysurface(DisplayState *ds, int width, int height)
+static inline int is_buffer_shared(DisplaySurface *surface)
 {
-    trace_displaysurface_resize(ds, ds->surface, width, height);
-    return ds->allocator->resize_displaysurface(ds->surface, width, height);
+    return !(surface->flags & QEMU_ALLOCATED_FLAG);
 }
 
-static inline void qemu_free_displaysurface(DisplayState *ds)
+void gui_setup_refresh(DisplayState *ds);
+
+static inline void register_displaychangelistener(DisplayState *ds, DisplayChangeListener *dcl)
 {
-    trace_displaysurface_free(ds, ds->surface);
-    ds->allocator->free_displaysurface(ds->surface);
+    QLIST_INSERT_HEAD(&ds->listeners, dcl, next);
+    gui_setup_refresh(ds);
+    if (dcl->dpy_gfx_resize) {
+        dcl->dpy_gfx_resize(ds);
+    }
 }
 
-static inline int is_surface_bgr(DisplaySurface *surface)
+static inline void unregister_displaychangelistener(DisplayState *ds,
+                                                    DisplayChangeListener *dcl)
 {
-    if (surface->pf.bits_per_pixel == 32 && surface->pf.rshift == 0)
-        return 1;
-    else
-        return 0;
+    QLIST_REMOVE(dcl, next);
+    gui_setup_refresh(ds);
 }
 
-static inline int is_buffer_shared(DisplaySurface *surface)
+static inline void dpy_gfx_update(DisplayState *s, int x, int y, int w, int h)
 {
-    return (!(surface->flags & QEMU_ALLOCATED_FLAG) &&
-            !(surface->flags & QEMU_REALPIXELS_FLAG));
+    struct DisplayChangeListener *dcl;
+    QLIST_FOREACH(dcl, &s->listeners, next) {
+        if (dcl->dpy_gfx_update) {
+            dcl->dpy_gfx_update(s, x, y, w, h);
+        }
+    }
 }
 
-static inline void register_displaychangelistener(DisplayState *ds, DisplayChangeListener *dcl)
+static inline void dpy_gfx_resize(DisplayState *s)
 {
-    dcl->next = ds->listeners;
-    ds->listeners = dcl;
+    struct DisplayChangeListener *dcl;
+    QLIST_FOREACH(dcl, &s->listeners, next) {
+        if (dcl->dpy_gfx_resize) {
+            dcl->dpy_gfx_resize(s);
+        }
+    }
 }
 
-static inline void dpy_update(DisplayState *s, int x, int y, int w, int h)
+static inline void dpy_gfx_setdata(DisplayState *s)
 {
-    struct DisplayChangeListener *dcl = s->listeners;
-    while (dcl != NULL) {
-        dcl->dpy_update(s, x, y, w, h);
-        dcl = dcl->next;
+    struct DisplayChangeListener *dcl;
+    QLIST_FOREACH(dcl, &s->listeners, next) {
+        if (dcl->dpy_gfx_setdata) {
+            dcl->dpy_gfx_setdata(s);
+        }
     }
 }
 
-static inline void dpy_resize(DisplayState *s)
+static inline void dpy_refresh(DisplayState *s)
 {
-    struct DisplayChangeListener *dcl = s->listeners;
-    while (dcl != NULL) {
-        dcl->dpy_resize(s);
-        dcl = dcl->next;
+    struct DisplayChangeListener *dcl;
+    QLIST_FOREACH(dcl, &s->listeners, next) {
+        if (dcl->dpy_refresh) {
+            dcl->dpy_refresh(s);
+        }
     }
 }
 
-static inline void dpy_setdata(DisplayState *s)
+static inline void dpy_gfx_copy(struct DisplayState *s, int src_x, int src_y,
+                             int dst_x, int dst_y, int w, int h)
 {
-    struct DisplayChangeListener *dcl = s->listeners;
-    while (dcl != NULL) {
-        if (dcl->dpy_setdata) dcl->dpy_setdata(s);
-        dcl = dcl->next;
+    struct DisplayChangeListener *dcl;
+    QLIST_FOREACH(dcl, &s->listeners, next) {
+        if (dcl->dpy_gfx_copy) {
+            dcl->dpy_gfx_copy(s, src_x, src_y, dst_x, dst_y, w, h);
+        } else { /* TODO */
+            dcl->dpy_gfx_update(s, dst_x, dst_y, w, h);
+        }
     }
 }
 
-static inline void dpy_refresh(DisplayState *s)
+static inline void dpy_text_cursor(struct DisplayState *s, int x, int y)
+{
+    struct DisplayChangeListener *dcl;
+    QLIST_FOREACH(dcl, &s->listeners, next) {
+        if (dcl->dpy_text_cursor) {
+            dcl->dpy_text_cursor(s, x, y);
+        }
+    }
+}
+
+static inline void dpy_text_update(DisplayState *s, int x, int y, int w, int h)
 {
-    struct DisplayChangeListener *dcl = s->listeners;
-    while (dcl != NULL) {
-        if (dcl->dpy_refresh) dcl->dpy_refresh(s);
-        dcl = dcl->next;
+    struct DisplayChangeListener *dcl;
+    QLIST_FOREACH(dcl, &s->listeners, next) {
+        if (dcl->dpy_text_update) {
+            dcl->dpy_text_update(s, x, y, w, h);
+        }
     }
 }
 
-static inline void dpy_copy(struct DisplayState *s, int src_x, int src_y,
-                             int dst_x, int dst_y, int w, int h) {
-    struct DisplayChangeListener *dcl = s->listeners;
-    while (dcl != NULL) {
-        if (dcl->dpy_copy)
-            dcl->dpy_copy(s, src_x, src_y, dst_x, dst_y, w, h);
-        else /* TODO */
-            dcl->dpy_update(s, dst_x, dst_y, w, h);
-        dcl = dcl->next;
+static inline void dpy_text_resize(DisplayState *s, int w, int h)
+{
+    struct DisplayChangeListener *dcl;
+    QLIST_FOREACH(dcl, &s->listeners, next) {
+        if (dcl->dpy_text_resize) {
+            dcl->dpy_text_resize(s, w, h);
+        }
     }
 }
 
-static inline void dpy_fill(struct DisplayState *s, int x, int y,
-                             int w, int h, uint32_t c) {
-    struct DisplayChangeListener *dcl = s->listeners;
-    while (dcl != NULL) {
-        if (dcl->dpy_fill) dcl->dpy_fill(s, x, y, w, h, c);
-        dcl = dcl->next;
+static inline void dpy_mouse_set(struct DisplayState *s, int x, int y, int on)
+{
+    struct DisplayChangeListener *dcl;
+    QLIST_FOREACH(dcl, &s->listeners, next) {
+        if (dcl->dpy_mouse_set) {
+            dcl->dpy_mouse_set(s, x, y, on);
+        }
+    }
+}
+
+static inline void dpy_cursor_define(struct DisplayState *s, QEMUCursor *cursor)
+{
+    struct DisplayChangeListener *dcl;
+    QLIST_FOREACH(dcl, &s->listeners, next) {
+        if (dcl->dpy_cursor_define) {
+            dcl->dpy_cursor_define(s, cursor);
+        }
     }
 }
 
-static inline void dpy_cursor(struct DisplayState *s, int x, int y) {
-    struct DisplayChangeListener *dcl = s->listeners;
-    while (dcl != NULL) {
-        if (dcl->dpy_text_cursor) dcl->dpy_text_cursor(s, x, y);
-        dcl = dcl->next;
+static inline bool dpy_cursor_define_supported(struct DisplayState *s)
+{
+    struct DisplayChangeListener *dcl;
+    QLIST_FOREACH(dcl, &s->listeners, next) {
+        if (dcl->dpy_cursor_define) {
+            return true;
+        }
     }
+    return false;
 }
 
 static inline int ds_get_linesize(DisplayState *ds)
 {
-    return ds->surface->linesize;
+    return pixman_image_get_stride(ds->surface->image);
 }
 
 static inline uint8_t* ds_get_data(DisplayState *ds)
 {
-    return ds->surface->data;
+    return (void *)pixman_image_get_data(ds->surface->image);
 }
 
 static inline int ds_get_width(DisplayState *ds)
 {
-    return ds->surface->width;
+    return pixman_image_get_width(ds->surface->image);
 }
 
 static inline int ds_get_height(DisplayState *ds)
 {
-    return ds->surface->height;
+    return pixman_image_get_height(ds->surface->image);
 }
 
 static inline int ds_get_bits_per_pixel(DisplayState *ds)
 {
-    return ds->surface->pf.bits_per_pixel;
+    int bits = PIXMAN_FORMAT_BPP(ds->surface->format);
+    return bits;
 }
 
 static inline int ds_get_bytes_per_pixel(DisplayState *ds)
 {
-    return ds->surface->pf.bytes_per_pixel;
+    int bits = PIXMAN_FORMAT_BPP(ds->surface->format);
+    return (bits + 7) / 8;
+}
+
+static inline pixman_format_code_t ds_get_format(DisplayState *ds)
+{
+    return ds->surface->format;
+}
+
+static inline pixman_image_t *ds_get_image(DisplayState *ds)
+{
+    return ds->surface->image;
+}
+
+static inline int ds_get_depth(DisplayState *ds)
+{
+    return ds->surface->pf.depth;
+}
+
+static inline int ds_get_rmask(DisplayState *ds)
+{
+    return ds->surface->pf.rmask;
+}
+
+static inline int ds_get_gmask(DisplayState *ds)
+{
+    return ds->surface->pf.gmask;
+}
+
+static inline int ds_get_bmask(DisplayState *ds)
+{
+    return ds->surface->pf.bmask;
 }
 
 #ifdef CONFIG_CURSES
@@ -343,7 +417,8 @@ static inline void console_write_ch(console_ch_t *dest, uint32_t ch)
 
 typedef void (*vga_hw_update_ptr)(void *);
 typedef void (*vga_hw_invalidate_ptr)(void *);
-typedef void (*vga_hw_screen_dump_ptr)(void *, const char *, bool cswitch);
+typedef void (*vga_hw_screen_dump_ptr)(void *, const char *, bool cswitch,
+                                       Error **errp);
 typedef void (*vga_hw_text_update_ptr)(void *, console_ch_t *);
 
 DisplayState *graphic_console_init(vga_hw_update_ptr update,
@@ -354,7 +429,6 @@ DisplayState *graphic_console_init(vga_hw_update_ptr update,
 
 void vga_hw_update(void);
 void vga_hw_invalidate(void);
-void vga_hw_screen_dump(const char *filename);
 void vga_hw_text_update(console_ch_t *chardata);
 
 int is_graphic_console(void);
@@ -375,10 +449,8 @@ void cocoa_display_init(DisplayState *ds, int full_screen);
 
 /* vnc.c */
 void vnc_display_init(DisplayState *ds);
-void vnc_display_close(DisplayState *ds);
-int vnc_display_open(DisplayState *ds, const char *display);
+void vnc_display_open(DisplayState *ds, const char *display, Error **errp);
 void vnc_display_add_client(DisplayState *ds, int csock, int skipauth);
-int vnc_display_disable_login(DisplayState *ds);
 char *vnc_display_local_addr(DisplayState *ds);
 #ifdef CONFIG_VNC
 int vnc_display_password(DisplayState *ds, const char *password);
@@ -397,4 +469,8 @@ static inline int vnc_display_pw_expire(DisplayState *ds, time_t expires)
 /* curses.c */
 void curses_display_init(DisplayState *ds, int full_screen);
 
+/* input.c */
+int index_from_key(const char *key);
+int index_from_keycode(int code);
+
 #endif
diff --git a/coroutine-sigaltstack.c b/coroutine-sigaltstack.c
index 861e87805..39dbaa5da 100644
--- a/coroutine-sigaltstack.c
+++ b/coroutine-sigaltstack.c
@@ -171,8 +171,8 @@ static Coroutine *coroutine_new(void)
     CoroutineThreadState *coTS;
     struct sigaction sa;
     struct sigaction osa;
-    struct sigaltstack ss;
-    struct sigaltstack oss;
+    stack_t ss;
+    stack_t oss;
     sigset_t sigs;
     sigset_t osigs;
     jmp_buf old_env;
diff --git a/cpu-all.h b/cpu-all.h
index 5e07d2850..c9c51b83a 100644
--- a/cpu-all.h
+++ b/cpu-all.h
@@ -260,14 +260,6 @@ extern unsigned long reserved_va;
 #define stfl(p, v) stfl_raw(p, v)
 #define stfq(p, v) stfq_raw(p, v)
 
-#ifndef CONFIG_TCG_PASS_AREG0
-#define ldub_code(p) ldub_raw(p)
-#define ldsb_code(p) ldsb_raw(p)
-#define lduw_code(p) lduw_raw(p)
-#define ldsw_code(p) ldsw_raw(p)
-#define ldl_code(p) ldl_raw(p)
-#define ldq_code(p) ldq_raw(p)
-#else
 #define cpu_ldub_code(env1, p) ldub_raw(p)
 #define cpu_ldsb_code(env1, p) ldsb_raw(p)
 #define cpu_lduw_code(env1, p) lduw_raw(p)
@@ -296,7 +288,6 @@ extern unsigned long reserved_va;
 #define cpu_stw_kernel(env, addr, data) stw_raw(addr, data)
 #define cpu_stl_kernel(env, addr, data) stl_raw(addr, data)
 #define cpu_stq_kernel(env, addr, data) stq_raw(addr, data)
-#endif
 
 #define ldub_kernel(p) ldub_raw(p)
 #define ldsb_kernel(p) ldsb_raw(p)
@@ -313,7 +304,6 @@ extern unsigned long reserved_va;
 #define stfl_kernel(p, v) stfl_raw(p, v)
 #define stfq_kernel(p, vt) stfq_raw(p, v)
 
-#ifdef CONFIG_TCG_PASS_AREG0
 #define cpu_ldub_data(env, addr) ldub_raw(addr)
 #define cpu_lduw_data(env, addr) lduw_raw(addr)
 #define cpu_ldl_data(env, addr) ldl_raw(addr)
@@ -321,7 +311,6 @@ extern unsigned long reserved_va;
 #define cpu_stb_data(env, addr, data) stb_raw(addr, data)
 #define cpu_stw_data(env, addr, data) stw_raw(addr, data)
 #define cpu_stl_data(env, addr, data) stl_raw(addr, data)
-#endif
 #endif /* defined(CONFIG_USER_ONLY) */
 
 /* page related stuff */
@@ -367,6 +356,9 @@ CPUArchState *cpu_copy(CPUArchState *env);
 CPUArchState *qemu_get_cpu(int cpu);
 
 #define CPU_DUMP_CODE 0x00010000
+#define CPU_DUMP_FPU 0x00020000 /* dump FPU register state, not just integer */
+/* dump info about TCG QEMU's condition code optimization state */
+#define CPU_DUMP_CCOP 0x00040000
 
 void cpu_dump_state(CPUArchState *env, FILE *f, fprintf_function cpu_fprintf,
                     int flags);
@@ -446,8 +438,6 @@ void cpu_reset_interrupt(CPUArchState *env, int mask);
 
 void cpu_exit(CPUArchState *s);
 
-bool qemu_cpu_has_work(CPUArchState *env);
-
 /* Breakpoint/watchpoint flags */
 #define BP_MEM_READ           0x01
 #define BP_MEM_WRITE          0x02
@@ -474,15 +464,13 @@ void cpu_watchpoint_remove_all(CPUArchState *env, int mask);
 #define SSTEP_NOTIMER 0x4  /* Do not Timers while single stepping */
 
 void cpu_single_step(CPUArchState *env, int enabled);
-int cpu_is_stopped(CPUArchState *env);
-void run_on_cpu(CPUArchState *env, void (*func)(void *data), void *data);
 
 #if !defined(CONFIG_USER_ONLY)
 
 /* Return the physical page corresponding to a virtual one. Use it
    only for debugging because no protection checks are done. Return -1
    if no page found. */
-target_phys_addr_t cpu_get_phys_page_debug(CPUArchState *env, target_ulong addr);
+hwaddr cpu_get_phys_page_debug(CPUArchState *env, target_ulong addr);
 
 /* memory API */
 
@@ -508,7 +496,6 @@ typedef struct RAMBlock {
 typedef struct RAMList {
     uint8_t *phys_dirty;
     QLIST_HEAD(, RAMBlock) blocks;
-    uint64_t dirty_pages;
 } RAMList;
 extern RAMList ram_list;
 
@@ -526,6 +513,7 @@ extern int mem_prealloc;
 #define TLB_MMIO        (1 << 5)
 
 void dump_exec_info(FILE *f, fprintf_function cpu_fprintf);
+ram_addr_t last_ram_offset(void);
 #endif /* !CONFIG_USER_ONLY */
 
 int cpu_memory_rw_debug(CPUArchState *env, target_ulong addr,
diff --git a/cpu-common.h b/cpu-common.h
index 85548de5e..d2fbafac9 100644
--- a/cpu-common.h
+++ b/cpu-common.h
@@ -3,7 +3,7 @@
 
 /* CPU interfaces that are target independent.  */
 
-#include "targphys.h"
+#include "hwaddr.h"
 
 #ifndef NEED_CPU_H
 #include "poison.h"
@@ -21,7 +21,7 @@ enum device_endian {
 };
 
 /* address in the RAM (different from a physical address) */
-#if defined(CONFIG_XEN_BACKEND) && TARGET_PHYS_ADDR_BITS == 64
+#if defined(CONFIG_XEN_BACKEND)
 typedef uint64_t ram_addr_t;
 #  define RAM_ADDR_MAX UINT64_MAX
 #  define RAM_ADDR_FMT "%" PRIx64
@@ -33,43 +33,38 @@ typedef uintptr_t ram_addr_t;
 
 /* memory API */
 
-typedef void CPUWriteMemoryFunc(void *opaque, target_phys_addr_t addr, uint32_t value);
-typedef uint32_t CPUReadMemoryFunc(void *opaque, target_phys_addr_t addr);
+typedef void CPUWriteMemoryFunc(void *opaque, hwaddr addr, uint32_t value);
+typedef uint32_t CPUReadMemoryFunc(void *opaque, hwaddr addr);
 
 void qemu_ram_remap(ram_addr_t addr, ram_addr_t length);
 /* This should only be used for ram local to a device.  */
 void *qemu_get_ram_ptr(ram_addr_t addr);
-void *qemu_ram_ptr_length(ram_addr_t addr, ram_addr_t *size);
-/* Same but slower, to use for migration, where the order of
- * RAMBlocks must not change. */
-void *qemu_safe_ram_ptr(ram_addr_t addr);
 void qemu_put_ram_ptr(void *addr);
 /* This should not be used by devices.  */
 int qemu_ram_addr_from_host(void *ptr, ram_addr_t *ram_addr);
 ram_addr_t qemu_ram_addr_from_host_nofail(void *ptr);
 void qemu_ram_set_idstr(ram_addr_t addr, const char *name, DeviceState *dev);
 
-void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,
+void cpu_physical_memory_rw(hwaddr addr, uint8_t *buf,
                             int len, int is_write);
-static inline void cpu_physical_memory_read(target_phys_addr_t addr,
+static inline void cpu_physical_memory_read(hwaddr addr,
                                             void *buf, int len)
 {
     cpu_physical_memory_rw(addr, buf, len, 0);
 }
-static inline void cpu_physical_memory_write(target_phys_addr_t addr,
+static inline void cpu_physical_memory_write(hwaddr addr,
                                              const void *buf, int len)
 {
     cpu_physical_memory_rw(addr, (void *)buf, len, 1);
 }
-void *cpu_physical_memory_map(target_phys_addr_t addr,
-                              target_phys_addr_t *plen,
+void *cpu_physical_memory_map(hwaddr addr,
+                              hwaddr *plen,
                               int is_write);
-void cpu_physical_memory_unmap(void *buffer, target_phys_addr_t len,
-                               int is_write, target_phys_addr_t access_len);
+void cpu_physical_memory_unmap(void *buffer, hwaddr len,
+                               int is_write, hwaddr access_len);
 void *cpu_register_map_client(void *opaque, void (*callback)(void *opaque));
-void cpu_unregister_map_client(void *cookie);
 
-bool cpu_physical_memory_is_io(target_phys_addr_t phys_addr);
+bool cpu_physical_memory_is_io(hwaddr phys_addr);
 
 /* Coalesced MMIO regions are areas where write operations can be reordered.
  * This usually implies that write operations are side-effect free.  This allows
@@ -78,33 +73,33 @@ bool cpu_physical_memory_is_io(target_phys_addr_t phys_addr);
  */
 void qemu_flush_coalesced_mmio_buffer(void);
 
-uint32_t ldub_phys(target_phys_addr_t addr);
-uint32_t lduw_le_phys(target_phys_addr_t addr);
-uint32_t lduw_be_phys(target_phys_addr_t addr);
-uint32_t ldl_le_phys(target_phys_addr_t addr);
-uint32_t ldl_be_phys(target_phys_addr_t addr);
-uint64_t ldq_le_phys(target_phys_addr_t addr);
-uint64_t ldq_be_phys(target_phys_addr_t addr);
-void stb_phys(target_phys_addr_t addr, uint32_t val);
-void stw_le_phys(target_phys_addr_t addr, uint32_t val);
-void stw_be_phys(target_phys_addr_t addr, uint32_t val);
-void stl_le_phys(target_phys_addr_t addr, uint32_t val);
-void stl_be_phys(target_phys_addr_t addr, uint32_t val);
-void stq_le_phys(target_phys_addr_t addr, uint64_t val);
-void stq_be_phys(target_phys_addr_t addr, uint64_t val);
+uint32_t ldub_phys(hwaddr addr);
+uint32_t lduw_le_phys(hwaddr addr);
+uint32_t lduw_be_phys(hwaddr addr);
+uint32_t ldl_le_phys(hwaddr addr);
+uint32_t ldl_be_phys(hwaddr addr);
+uint64_t ldq_le_phys(hwaddr addr);
+uint64_t ldq_be_phys(hwaddr addr);
+void stb_phys(hwaddr addr, uint32_t val);
+void stw_le_phys(hwaddr addr, uint32_t val);
+void stw_be_phys(hwaddr addr, uint32_t val);
+void stl_le_phys(hwaddr addr, uint32_t val);
+void stl_be_phys(hwaddr addr, uint32_t val);
+void stq_le_phys(hwaddr addr, uint64_t val);
+void stq_be_phys(hwaddr addr, uint64_t val);
 
 #ifdef NEED_CPU_H
-uint32_t lduw_phys(target_phys_addr_t addr);
-uint32_t ldl_phys(target_phys_addr_t addr);
-uint64_t ldq_phys(target_phys_addr_t addr);
-void stl_phys_notdirty(target_phys_addr_t addr, uint32_t val);
-void stq_phys_notdirty(target_phys_addr_t addr, uint64_t val);
-void stw_phys(target_phys_addr_t addr, uint32_t val);
-void stl_phys(target_phys_addr_t addr, uint32_t val);
-void stq_phys(target_phys_addr_t addr, uint64_t val);
+uint32_t lduw_phys(hwaddr addr);
+uint32_t ldl_phys(hwaddr addr);
+uint64_t ldq_phys(hwaddr addr);
+void stl_phys_notdirty(hwaddr addr, uint32_t val);
+void stq_phys_notdirty(hwaddr addr, uint64_t val);
+void stw_phys(hwaddr addr, uint32_t val);
+void stl_phys(hwaddr addr, uint32_t val);
+void stq_phys(hwaddr addr, uint64_t val);
 #endif
 
-void cpu_physical_memory_write_rom(target_phys_addr_t addr,
+void cpu_physical_memory_write_rom(hwaddr addr,
                                    const uint8_t *buf, int len);
 
 extern struct MemoryRegion io_mem_ram;
diff --git a/cpu-defs.h b/cpu-defs.h
index 4018b88a1..3669241fa 100644
--- a/cpu-defs.h
+++ b/cpu-defs.h
@@ -29,7 +29,7 @@
 #include <signal.h>
 #include "osdep.h"
 #include "qemu-queue.h"
-#include "targphys.h"
+#include "hwaddr.h"
 
 #ifndef TARGET_LONG_BITS
 #error TARGET_LONG_BITS must be defined before including this header
@@ -111,7 +111,7 @@ extern int CPUTLBEntry_wrong_size[sizeof(CPUTLBEntry) == (1 << CPU_TLB_ENTRY_BIT
 #define CPU_COMMON_TLB \
     /* The meaning of the MMU modes is defined in the target code. */   \
     CPUTLBEntry tlb_table[NB_MMU_MODES][CPU_TLB_SIZE];                  \
-    target_phys_addr_t iotlb[NB_MMU_MODES][CPU_TLB_SIZE];               \
+    hwaddr iotlb[NB_MMU_MODES][CPU_TLB_SIZE];               \
     target_ulong tlb_flush_addr;                                        \
     target_ulong tlb_flush_mask;
 
@@ -201,15 +201,9 @@ typedef struct CPUWatchpoint {
     int nr_cores;  /* number of cores within this CPU package */        \
     int nr_threads;/* number of threads within this CPU */              \
     int running; /* Nonzero if cpu is currently running(usermode).  */  \
-    int thread_id;                                                      \
     /* user data */                                                     \
     void *opaque;                                                       \
                                                                         \
-    uint32_t created;                                                   \
-    uint32_t stop;   /* Stop request */                                 \
-    uint32_t stopped; /* Artificially stopped */                        \
-    struct QemuCond *halt_cond;                                         \
-    struct qemu_work_item *queued_work_first, *queued_work_last;        \
     const char *cpu_model_str;                                          \
     struct KVMState *kvm_state;                                         \
     struct kvm_run *kvm_run;                                            \
diff --git a/cpu-exec.c b/cpu-exec.c
index 134b3c4fc..904ee73c7 100644
--- a/cpu-exec.c
+++ b/cpu-exec.c
@@ -27,9 +27,9 @@ int tb_invalidated_flag;
 
 //#define CONFIG_DEBUG_EXEC
 
-bool qemu_cpu_has_work(CPUArchState *env)
+bool qemu_cpu_has_work(CPUState *cpu)
 {
-    return cpu_has_work(env);
+    return cpu_has_work(cpu);
 }
 
 void cpu_loop_exit(CPUArchState *env)
@@ -181,16 +181,14 @@ volatile sig_atomic_t exit_request;
 
 int cpu_exec(CPUArchState *env)
 {
-#ifdef TARGET_PPC
     CPUState *cpu = ENV_GET_CPU(env);
-#endif
     int ret, interrupt_request;
     TranslationBlock *tb;
     uint8_t *tc_ptr;
     tcg_target_ulong next_tb;
 
     if (env->halted) {
-        if (!cpu_has_work(env)) {
+        if (!cpu_has_work(cpu)) {
             return EXCP_HALTED;
         }
 
@@ -552,7 +550,7 @@ int cpu_exec(CPUArchState *env)
 #if defined(TARGET_I386)
                     env->eflags = env->eflags | cpu_cc_compute_all(env, CC_OP)
                         | (DF & DF_MASK);
-                    log_cpu_state(env, X86_DUMP_CCOP);
+                    log_cpu_state(env, CPU_DUMP_CCOP);
                     env->eflags &= ~(DF_MASK | CC_O | CC_S | CC_Z | CC_A | CC_P | CC_C);
 #elif defined(TARGET_M68K)
                     cpu_m68k_flush_flags(env, env->cc_op);
diff --git a/cpus.c b/cpus.c
index e476a3cd5..d9c332fcb 100644
--- a/cpus.c
+++ b/cpus.c
@@ -64,13 +64,15 @@ static CPUArchState *next_cpu;
 
 static bool cpu_thread_is_idle(CPUArchState *env)
 {
-    if (env->stop || env->queued_work_first) {
+    CPUState *cpu = ENV_GET_CPU(env);
+
+    if (cpu->stop || cpu->queued_work_first) {
         return false;
     }
-    if (env->stopped || !runstate_is_running()) {
+    if (cpu->stopped || !runstate_is_running()) {
         return true;
     }
-    if (!env->halted || qemu_cpu_has_work(env) ||
+    if (!env->halted || qemu_cpu_has_work(cpu) ||
         kvm_async_interrupts_enabled()) {
         return false;
     }
@@ -395,11 +397,7 @@ void hw_error(const char *fmt, ...)
     fprintf(stderr, "\n");
     for(env = first_cpu; env != NULL; env = env->next_cpu) {
         fprintf(stderr, "CPU #%d:\n", env->cpu_index);
-#ifdef TARGET_I386
-        cpu_dump_state(env, stderr, fprintf, X86_DUMP_FPU);
-#else
-        cpu_dump_state(env, stderr, fprintf, 0);
-#endif
+        cpu_dump_state(env, stderr, fprintf, CPU_DUMP_FPU);
     }
     va_end(ap);
     abort();
@@ -432,9 +430,9 @@ void cpu_synchronize_all_post_init(void)
     }
 }
 
-int cpu_is_stopped(CPUArchState *env)
+bool cpu_is_stopped(CPUState *cpu)
 {
-    return !runstate_is_running() || env->stopped;
+    return !runstate_is_running() || cpu->stopped;
 }
 
 static void do_vm_stop(RunState state)
@@ -450,22 +448,24 @@ static void do_vm_stop(RunState state)
     }
 }
 
-static int cpu_can_run(CPUArchState *env)
+static bool cpu_can_run(CPUState *cpu)
 {
-    if (env->stop) {
-        return 0;
+    if (cpu->stop) {
+        return false;
     }
-    if (env->stopped || !runstate_is_running()) {
-        return 0;
+    if (cpu->stopped || !runstate_is_running()) {
+        return false;
     }
-    return 1;
+    return true;
 }
 
 static void cpu_handle_guest_debug(CPUArchState *env)
 {
+    CPUState *cpu = ENV_GET_CPU(env);
+
     gdb_set_stop_cpu(env);
     qemu_system_debug_request();
-    env->stopped = 1;
+    cpu->stopped = true;
 }
 
 static void cpu_signal(int sig)
@@ -613,7 +613,7 @@ static void qemu_tcg_init_cpu_signals(void)
 }
 #endif /* _WIN32 */
 
-QemuMutex qemu_global_mutex;
+static QemuMutex qemu_global_mutex;
 static QemuCond qemu_io_proceeded_cond;
 static bool iothread_requesting_mutex;
 
@@ -640,27 +640,27 @@ void qemu_init_cpu_loop(void)
     qemu_thread_get_self(&io_thread);
 }
 
-void run_on_cpu(CPUArchState *env, void (*func)(void *data), void *data)
+void run_on_cpu(CPUState *cpu, void (*func)(void *data), void *data)
 {
     struct qemu_work_item wi;
 
-    if (qemu_cpu_is_self(env)) {
+    if (qemu_cpu_is_self(cpu)) {
         func(data);
         return;
     }
 
     wi.func = func;
     wi.data = data;
-    if (!env->queued_work_first) {
-        env->queued_work_first = &wi;
+    if (cpu->queued_work_first == NULL) {
+        cpu->queued_work_first = &wi;
     } else {
-        env->queued_work_last->next = &wi;
+        cpu->queued_work_last->next = &wi;
     }
-    env->queued_work_last = &wi;
+    cpu->queued_work_last = &wi;
     wi.next = NULL;
     wi.done = false;
 
-    qemu_cpu_kick(env);
+    qemu_cpu_kick(cpu);
     while (!wi.done) {
         CPUArchState *self_env = cpu_single_env;
 
@@ -669,33 +669,31 @@ void run_on_cpu(CPUArchState *env, void (*func)(void *data), void *data)
     }
 }
 
-static void flush_queued_work(CPUArchState *env)
+static void flush_queued_work(CPUState *cpu)
 {
     struct qemu_work_item *wi;
 
-    if (!env->queued_work_first) {
+    if (cpu->queued_work_first == NULL) {
         return;
     }
 
-    while ((wi = env->queued_work_first)) {
-        env->queued_work_first = wi->next;
+    while ((wi = cpu->queued_work_first)) {
+        cpu->queued_work_first = wi->next;
         wi->func(wi->data);
         wi->done = true;
     }
-    env->queued_work_last = NULL;
+    cpu->queued_work_last = NULL;
     qemu_cond_broadcast(&qemu_work_cond);
 }
 
-static void qemu_wait_io_event_common(CPUArchState *env)
+static void qemu_wait_io_event_common(CPUState *cpu)
 {
-    CPUState *cpu = ENV_GET_CPU(env);
-
-    if (env->stop) {
-        env->stop = 0;
-        env->stopped = 1;
+    if (cpu->stop) {
+        cpu->stop = false;
+        cpu->stopped = true;
         qemu_cond_signal(&qemu_pause_cond);
     }
-    flush_queued_work(env);
+    flush_queued_work(cpu);
     cpu->thread_kicked = false;
 }
 
@@ -715,18 +713,20 @@ static void qemu_tcg_wait_io_event(void)
     }
 
     for (env = first_cpu; env != NULL; env = env->next_cpu) {
-        qemu_wait_io_event_common(env);
+        qemu_wait_io_event_common(ENV_GET_CPU(env));
     }
 }
 
 static void qemu_kvm_wait_io_event(CPUArchState *env)
 {
+    CPUState *cpu = ENV_GET_CPU(env);
+
     while (cpu_thread_is_idle(env)) {
-        qemu_cond_wait(env->halt_cond, &qemu_global_mutex);
+        qemu_cond_wait(cpu->halt_cond, &qemu_global_mutex);
     }
 
     qemu_kvm_eat_signals(env);
-    qemu_wait_io_event_common(env);
+    qemu_wait_io_event_common(cpu);
 }
 
 static void *qemu_kvm_cpu_thread_fn(void *arg)
@@ -737,7 +737,7 @@ static void *qemu_kvm_cpu_thread_fn(void *arg)
 
     qemu_mutex_lock(&qemu_global_mutex);
     qemu_thread_get_self(cpu->thread);
-    env->thread_id = qemu_get_thread_id();
+    cpu->thread_id = qemu_get_thread_id();
     cpu_single_env = env;
 
     r = kvm_init_vcpu(env);
@@ -749,11 +749,11 @@ static void *qemu_kvm_cpu_thread_fn(void *arg)
     qemu_kvm_init_cpu_signals(env);
 
     /* signal CPU creation */
-    env->created = 1;
+    cpu->created = true;
     qemu_cond_signal(&qemu_cpu_cond);
 
     while (1) {
-        if (cpu_can_run(env)) {
+        if (cpu_can_run(cpu)) {
             r = kvm_cpu_exec(env);
             if (r == EXCP_DEBUG) {
                 cpu_handle_guest_debug(env);
@@ -778,13 +778,13 @@ static void *qemu_dummy_cpu_thread_fn(void *arg)
 
     qemu_mutex_lock_iothread();
     qemu_thread_get_self(cpu->thread);
-    env->thread_id = qemu_get_thread_id();
+    cpu->thread_id = qemu_get_thread_id();
 
     sigemptyset(&waitset);
     sigaddset(&waitset, SIG_IPI);
 
     /* signal CPU creation */
-    env->created = 1;
+    cpu->created = true;
     qemu_cond_signal(&qemu_cpu_cond);
 
     cpu_single_env = env;
@@ -801,7 +801,7 @@ static void *qemu_dummy_cpu_thread_fn(void *arg)
         }
         qemu_mutex_lock_iothread();
         cpu_single_env = env;
-        qemu_wait_io_event_common(env);
+        qemu_wait_io_event_common(cpu);
     }
 
     return NULL;
@@ -812,8 +812,8 @@ static void tcg_exec_all(void);
 
 static void *qemu_tcg_cpu_thread_fn(void *arg)
 {
-    CPUArchState *env = arg;
-    CPUState *cpu = ENV_GET_CPU(env);
+    CPUState *cpu = arg;
+    CPUArchState *env;
 
     qemu_tcg_init_cpu_signals();
     qemu_thread_get_self(cpu->thread);
@@ -821,18 +821,19 @@ static void *qemu_tcg_cpu_thread_fn(void *arg)
     /* signal CPU creation */
     qemu_mutex_lock(&qemu_global_mutex);
     for (env = first_cpu; env != NULL; env = env->next_cpu) {
-        env->thread_id = qemu_get_thread_id();
-        env->created = 1;
+        cpu = ENV_GET_CPU(env);
+        cpu->thread_id = qemu_get_thread_id();
+        cpu->created = true;
     }
     qemu_cond_signal(&qemu_cpu_cond);
 
     /* wait for initial kick-off after machine start */
-    while (first_cpu->stopped) {
+    while (ENV_GET_CPU(first_cpu)->stopped) {
         qemu_cond_wait(tcg_halt_cond, &qemu_global_mutex);
 
         /* process any pending work */
         for (env = first_cpu; env != NULL; env = env->next_cpu) {
-            qemu_wait_io_event_common(env);
+            qemu_wait_io_event_common(ENV_GET_CPU(env));
         }
     }
 
@@ -847,9 +848,8 @@ static void *qemu_tcg_cpu_thread_fn(void *arg)
     return NULL;
 }
 
-static void qemu_cpu_kick_thread(CPUArchState *env)
+static void qemu_cpu_kick_thread(CPUState *cpu)
 {
-    CPUState *cpu = ENV_GET_CPU(env);
 #ifndef _WIN32
     int err;
 
@@ -859,7 +859,7 @@ static void qemu_cpu_kick_thread(CPUArchState *env)
         exit(1);
     }
 #else /* _WIN32 */
-    if (!qemu_cpu_is_self(env)) {
+    if (!qemu_cpu_is_self(cpu)) {
         SuspendThread(cpu->hThread);
         cpu_signal(0);
         ResumeThread(cpu->hThread);
@@ -867,14 +867,11 @@ static void qemu_cpu_kick_thread(CPUArchState *env)
 #endif
 }
 
-void qemu_cpu_kick(void *_env)
+void qemu_cpu_kick(CPUState *cpu)
 {
-    CPUArchState *env = _env;
-    CPUState *cpu = ENV_GET_CPU(env);
-
-    qemu_cond_broadcast(env->halt_cond);
+    qemu_cond_broadcast(cpu->halt_cond);
     if (!tcg_enabled() && !cpu->thread_kicked) {
-        qemu_cpu_kick_thread(env);
+        qemu_cpu_kick_thread(cpu);
         cpu->thread_kicked = true;
     }
 }
@@ -886,7 +883,7 @@ void qemu_cpu_kick_self(void)
     CPUState *cpu_single_cpu = ENV_GET_CPU(cpu_single_env);
 
     if (!cpu_single_cpu->thread_kicked) {
-        qemu_cpu_kick_thread(cpu_single_env);
+        qemu_cpu_kick_thread(cpu_single_cpu);
         cpu_single_cpu->thread_kicked = true;
     }
 #else
@@ -894,14 +891,16 @@ void qemu_cpu_kick_self(void)
 #endif
 }
 
-int qemu_cpu_is_self(void *_env)
+bool qemu_cpu_is_self(CPUState *cpu)
 {
-    CPUArchState *env = _env;
-    CPUState *cpu = ENV_GET_CPU(env);
-
     return qemu_thread_is_self(cpu->thread);
 }
 
+static bool qemu_in_vcpu_thread(void)
+{
+    return cpu_single_env && qemu_cpu_is_self(ENV_GET_CPU(cpu_single_env));
+}
+
 void qemu_mutex_lock_iothread(void)
 {
     if (!tcg_enabled()) {
@@ -909,7 +908,7 @@ void qemu_mutex_lock_iothread(void)
     } else {
         iothread_requesting_mutex = true;
         if (qemu_mutex_trylock(&qemu_global_mutex)) {
-            qemu_cpu_kick_thread(first_cpu);
+            qemu_cpu_kick_thread(ENV_GET_CPU(first_cpu));
             qemu_mutex_lock(&qemu_global_mutex);
         }
         iothread_requesting_mutex = false;
@@ -927,7 +926,8 @@ static int all_vcpus_paused(void)
     CPUArchState *penv = first_cpu;
 
     while (penv) {
-        if (!penv->stopped) {
+        CPUState *pcpu = ENV_GET_CPU(penv);
+        if (!pcpu->stopped) {
             return 0;
         }
         penv = penv->next_cpu;
@@ -942,17 +942,19 @@ void pause_all_vcpus(void)
 
     qemu_clock_enable(vm_clock, false);
     while (penv) {
-        penv->stop = 1;
-        qemu_cpu_kick(penv);
+        CPUState *pcpu = ENV_GET_CPU(penv);
+        pcpu->stop = true;
+        qemu_cpu_kick(pcpu);
         penv = penv->next_cpu;
     }
 
-    if (!qemu_thread_is_self(&io_thread)) {
+    if (qemu_in_vcpu_thread()) {
         cpu_stop_current();
         if (!kvm_enabled()) {
             while (penv) {
-                penv->stop = 0;
-                penv->stopped = 1;
+                CPUState *pcpu = ENV_GET_CPU(penv);
+                pcpu->stop = 0;
+                pcpu->stopped = true;
                 penv = penv->next_cpu;
             }
             return;
@@ -963,7 +965,7 @@ void pause_all_vcpus(void)
         qemu_cond_wait(&qemu_pause_cond, &qemu_global_mutex);
         penv = first_cpu;
         while (penv) {
-            qemu_cpu_kick(penv);
+            qemu_cpu_kick(ENV_GET_CPU(penv));
             penv = penv->next_cpu;
         }
     }
@@ -975,36 +977,34 @@ void resume_all_vcpus(void)
 
     qemu_clock_enable(vm_clock, true);
     while (penv) {
-        penv->stop = 0;
-        penv->stopped = 0;
-        qemu_cpu_kick(penv);
+        CPUState *pcpu = ENV_GET_CPU(penv);
+        pcpu->stop = false;
+        pcpu->stopped = false;
+        qemu_cpu_kick(pcpu);
         penv = penv->next_cpu;
     }
 }
 
-static void qemu_tcg_init_vcpu(void *_env)
+static void qemu_tcg_init_vcpu(CPUState *cpu)
 {
-    CPUArchState *env = _env;
-    CPUState *cpu = ENV_GET_CPU(env);
-
     /* share a single thread for all cpus with TCG */
     if (!tcg_cpu_thread) {
         cpu->thread = g_malloc0(sizeof(QemuThread));
-        env->halt_cond = g_malloc0(sizeof(QemuCond));
-        qemu_cond_init(env->halt_cond);
-        tcg_halt_cond = env->halt_cond;
-        qemu_thread_create(cpu->thread, qemu_tcg_cpu_thread_fn, env,
+        cpu->halt_cond = g_malloc0(sizeof(QemuCond));
+        qemu_cond_init(cpu->halt_cond);
+        tcg_halt_cond = cpu->halt_cond;
+        qemu_thread_create(cpu->thread, qemu_tcg_cpu_thread_fn, cpu,
                            QEMU_THREAD_JOINABLE);
 #ifdef _WIN32
         cpu->hThread = qemu_thread_get_handle(cpu->thread);
 #endif
-        while (env->created == 0) {
+        while (!cpu->created) {
             qemu_cond_wait(&qemu_cpu_cond, &qemu_global_mutex);
         }
         tcg_cpu_thread = cpu->thread;
     } else {
         cpu->thread = tcg_cpu_thread;
-        env->halt_cond = tcg_halt_cond;
+        cpu->halt_cond = tcg_halt_cond;
     }
 }
 
@@ -1013,11 +1013,11 @@ static void qemu_kvm_start_vcpu(CPUArchState *env)
     CPUState *cpu = ENV_GET_CPU(env);
 
     cpu->thread = g_malloc0(sizeof(QemuThread));
-    env->halt_cond = g_malloc0(sizeof(QemuCond));
-    qemu_cond_init(env->halt_cond);
+    cpu->halt_cond = g_malloc0(sizeof(QemuCond));
+    qemu_cond_init(cpu->halt_cond);
     qemu_thread_create(cpu->thread, qemu_kvm_cpu_thread_fn, env,
                        QEMU_THREAD_JOINABLE);
-    while (env->created == 0) {
+    while (!cpu->created) {
         qemu_cond_wait(&qemu_cpu_cond, &qemu_global_mutex);
     }
 }
@@ -1027,11 +1027,11 @@ static void qemu_dummy_start_vcpu(CPUArchState *env)
     CPUState *cpu = ENV_GET_CPU(env);
 
     cpu->thread = g_malloc0(sizeof(QemuThread));
-    env->halt_cond = g_malloc0(sizeof(QemuCond));
-    qemu_cond_init(env->halt_cond);
+    cpu->halt_cond = g_malloc0(sizeof(QemuCond));
+    qemu_cond_init(cpu->halt_cond);
     qemu_thread_create(cpu->thread, qemu_dummy_cpu_thread_fn, env,
                        QEMU_THREAD_JOINABLE);
-    while (env->created == 0) {
+    while (!cpu->created) {
         qemu_cond_wait(&qemu_cpu_cond, &qemu_global_mutex);
     }
 }
@@ -1039,14 +1039,15 @@ static void qemu_dummy_start_vcpu(CPUArchState *env)
 void qemu_init_vcpu(void *_env)
 {
     CPUArchState *env = _env;
+    CPUState *cpu = ENV_GET_CPU(env);
 
     env->nr_cores = smp_cores;
     env->nr_threads = smp_threads;
-    env->stopped = 1;
+    cpu->stopped = true;
     if (kvm_enabled()) {
         qemu_kvm_start_vcpu(env);
     } else if (tcg_enabled()) {
-        qemu_tcg_init_vcpu(env);
+        qemu_tcg_init_vcpu(cpu);
     } else {
         qemu_dummy_start_vcpu(env);
     }
@@ -1055,8 +1056,9 @@ void qemu_init_vcpu(void *_env)
 void cpu_stop_current(void)
 {
     if (cpu_single_env) {
-        cpu_single_env->stop = 0;
-        cpu_single_env->stopped = 1;
+        CPUState *cpu_single_cpu = ENV_GET_CPU(cpu_single_env);
+        cpu_single_cpu->stop = false;
+        cpu_single_cpu->stopped = true;
         cpu_exit(cpu_single_env);
         qemu_cond_signal(&qemu_pause_cond);
     }
@@ -1064,7 +1066,7 @@ void cpu_stop_current(void)
 
 void vm_stop(RunState state)
 {
-    if (!qemu_thread_is_self(&io_thread)) {
+    if (qemu_in_vcpu_thread()) {
         qemu_system_vmstop_request(state);
         /*
          * FIXME: should not return to device code in case
@@ -1137,17 +1139,18 @@ static void tcg_exec_all(void)
     }
     for (; next_cpu != NULL && !exit_request; next_cpu = next_cpu->next_cpu) {
         CPUArchState *env = next_cpu;
+        CPUState *cpu = ENV_GET_CPU(env);
 
         qemu_clock_enable(vm_clock,
                           (env->singlestep_enabled & SSTEP_NOTIMER) == 0);
 
-        if (cpu_can_run(env)) {
+        if (cpu_can_run(cpu)) {
             r = tcg_cpu_exec(env);
             if (r == EXCP_DEBUG) {
                 cpu_handle_guest_debug(env);
                 break;
             }
-        } else if (env->stop || env->stopped) {
+        } else if (cpu->stop || cpu->stopped) {
             break;
         }
     }
@@ -1192,10 +1195,8 @@ void set_cpu_log_filename(const char *optarg)
 void list_cpus(FILE *f, fprintf_function cpu_fprintf, const char *optarg)
 {
     /* XXX: implement xxx_cpu_list for targets that still miss it */
-#if defined(cpu_list_id)
-    cpu_list_id(f, cpu_fprintf, optarg);
-#elif defined(cpu_list)
-    cpu_list(f, cpu_fprintf); /* deprecated */
+#if defined(cpu_list)
+    cpu_list(f, cpu_fprintf);
 #endif
 }
 
@@ -1204,7 +1205,8 @@ CpuInfoList *qmp_query_cpus(Error **errp)
     CpuInfoList *head = NULL, *cur_item = NULL;
     CPUArchState *env;
 
-    for(env = first_cpu; env != NULL; env = env->next_cpu) {
+    for (env = first_cpu; env != NULL; env = env->next_cpu) {
+        CPUState *cpu = ENV_GET_CPU(env);
         CpuInfoList *info;
 
         cpu_synchronize_state(env);
@@ -1214,7 +1216,7 @@ CpuInfoList *qmp_query_cpus(Error **errp)
         info->value->CPU = env->cpu_index;
         info->value->current = (env == first_cpu);
         info->value->halted = env->halted;
-        info->value->thread_id = env->thread_id;
+        info->value->thread_id = cpu->thread_id;
 #if defined(TARGET_I386)
         info->value->has_pc = true;
         info->value->pc = env->eip + env->segs[R_CS].base;
diff --git a/cputlb.c b/cputlb.c
index d3e7b257a..d6d037261 100644
--- a/cputlb.c
+++ b/cputlb.c
@@ -21,11 +21,11 @@
 #include "cpu.h"
 #include "exec-all.h"
 #include "memory.h"
+#include "exec-memory.h"
 
 #include "cputlb.h"
 
-#define WANT_EXEC_OBSOLETE
-#include "exec-obsolete.h"
+#include "memory-internal.h"
 
 //#define DEBUG_TLB
 //#define DEBUG_TLB_CHECK
@@ -237,7 +237,7 @@ static void tlb_add_large_page(CPUArchState *env, target_ulong vaddr,
    is permitted. Only a single TARGET_PAGE_SIZE region is mapped, the
    supplied size is only used by tlb_flush_page.  */
 void tlb_set_page(CPUArchState *env, target_ulong vaddr,
-                  target_phys_addr_t paddr, int prot,
+                  hwaddr paddr, int prot,
                   int mmu_idx, target_ulong size)
 {
     MemoryRegionSection *section;
@@ -246,13 +246,13 @@ void tlb_set_page(CPUArchState *env, target_ulong vaddr,
     target_ulong code_address;
     uintptr_t addend;
     CPUTLBEntry *te;
-    target_phys_addr_t iotlb;
+    hwaddr iotlb;
 
     assert(size >= TARGET_PAGE_SIZE);
     if (size != TARGET_PAGE_SIZE) {
         tlb_add_large_page(env, vaddr, size);
     }
-    section = phys_page_find(paddr >> TARGET_PAGE_BITS);
+    section = phys_page_find(address_space_memory.dispatch, paddr >> TARGET_PAGE_BITS);
 #if defined(DEBUG_TLB)
     printf("tlb_set_page: vaddr=" TARGET_FMT_lx " paddr=0x" TARGET_FMT_plx
            " prot=%x idx=%d pd=0x%08lx\n",
@@ -325,11 +325,7 @@ tb_page_addr_t get_page_addr_code(CPUArchState *env1, target_ulong addr)
     mmu_idx = cpu_mmu_index(env1);
     if (unlikely(env1->tlb_table[mmu_idx][page_index].addr_code !=
                  (addr & TARGET_PAGE_MASK))) {
-#ifdef CONFIG_TCG_PASS_AREG0
         cpu_ldub_code(env1, addr);
-#else
-        ldub_code(addr);
-#endif
     }
     pd = env1->iotlb[mmu_idx][page_index] & ~TARGET_PAGE_MASK;
     mr = iotlb_to_region(pd);
@@ -348,7 +344,6 @@ tb_page_addr_t get_page_addr_code(CPUArchState *env1, target_ulong addr)
 #define MMUSUFFIX _cmmu
 #undef GETPC
 #define GETPC() ((uintptr_t)0)
-#define env cpu_single_env
 #define SOFTMMU_CODE_ACCESS
 
 #define SHIFT 0
diff --git a/cputlb.h b/cputlb.h
index 2dc2c96cd..733c885a1 100644
--- a/cputlb.h
+++ b/cputlb.h
@@ -26,17 +26,18 @@ void tlb_unprotect_code_phys(CPUArchState *env, ram_addr_t ram_addr,
                              target_ulong vaddr);
 void tlb_reset_dirty_range(CPUTLBEntry *tlb_entry, uintptr_t start,
                            uintptr_t length);
-MemoryRegionSection *phys_page_find(target_phys_addr_t index);
+MemoryRegionSection *phys_page_find(struct AddressSpaceDispatch *d,
+                                    hwaddr index);
 void cpu_tlb_reset_dirty_all(ram_addr_t start1, ram_addr_t length);
 void tlb_set_dirty(CPUArchState *env, target_ulong vaddr);
 extern int tlb_flush_count;
 
 /* exec.c */
 void tb_flush_jmp_cache(CPUArchState *env, target_ulong addr);
-target_phys_addr_t memory_region_section_get_iotlb(CPUArchState *env,
+hwaddr memory_region_section_get_iotlb(CPUArchState *env,
                                                    MemoryRegionSection *section,
                                                    target_ulong vaddr,
-                                                   target_phys_addr_t paddr,
+                                                   hwaddr paddr,
                                                    int prot,
                                                    target_ulong *address);
 bool memory_region_is_unassigned(MemoryRegion *mr);
diff --git a/cutils.c b/cutils.c
index 8ef648f4b..4f0692f78 100644
--- a/cutils.c
+++ b/cutils.c
@@ -115,7 +115,7 @@ time_t mktimegm(struct tm *tm)
         m += 12;
         y--;
     }
-    t = 86400 * (d + (153 * m - 457) / 5 + 365 * y + y / 4 - y / 100 + 
+    t = 86400ULL * (d + (153 * m - 457) / 5 + 365 * y + y / 4 - y / 100 + 
                  y / 400 - 719469);
     t += 3600 * tm->tm_hour + 60 * tm->tm_min + tm->tm_sec;
     return t;
@@ -142,109 +142,6 @@ int qemu_fdatasync(int fd)
 #endif
 }
 
-/* io vectors */
-
-void qemu_iovec_init(QEMUIOVector *qiov, int alloc_hint)
-{
-    qiov->iov = g_malloc(alloc_hint * sizeof(struct iovec));
-    qiov->niov = 0;
-    qiov->nalloc = alloc_hint;
-    qiov->size = 0;
-}
-
-void qemu_iovec_init_external(QEMUIOVector *qiov, struct iovec *iov, int niov)
-{
-    int i;
-
-    qiov->iov = iov;
-    qiov->niov = niov;
-    qiov->nalloc = -1;
-    qiov->size = 0;
-    for (i = 0; i < niov; i++)
-        qiov->size += iov[i].iov_len;
-}
-
-void qemu_iovec_add(QEMUIOVector *qiov, void *base, size_t len)
-{
-    assert(qiov->nalloc != -1);
-
-    if (qiov->niov == qiov->nalloc) {
-        qiov->nalloc = 2 * qiov->nalloc + 1;
-        qiov->iov = g_realloc(qiov->iov, qiov->nalloc * sizeof(struct iovec));
-    }
-    qiov->iov[qiov->niov].iov_base = base;
-    qiov->iov[qiov->niov].iov_len = len;
-    qiov->size += len;
-    ++qiov->niov;
-}
-
-/*
- * Concatenates (partial) iovecs from src to the end of dst.
- * It starts copying after skipping `soffset' bytes at the
- * beginning of src and adds individual vectors from src to
- * dst copies up to `sbytes' bytes total, or up to the end
- * of src if it comes first.  This way, it is okay to specify
- * very large value for `sbytes' to indicate "up to the end
- * of src".
- * Only vector pointers are processed, not the actual data buffers.
- */
-void qemu_iovec_concat(QEMUIOVector *dst,
-                       QEMUIOVector *src, size_t soffset, size_t sbytes)
-{
-    int i;
-    size_t done;
-    struct iovec *siov = src->iov;
-    assert(dst->nalloc != -1);
-    assert(src->size >= soffset);
-    for (i = 0, done = 0; done < sbytes && i < src->niov; i++) {
-        if (soffset < siov[i].iov_len) {
-            size_t len = MIN(siov[i].iov_len - soffset, sbytes - done);
-            qemu_iovec_add(dst, siov[i].iov_base + soffset, len);
-            done += len;
-            soffset = 0;
-        } else {
-            soffset -= siov[i].iov_len;
-        }
-    }
-    /* return done; */
-}
-
-void qemu_iovec_destroy(QEMUIOVector *qiov)
-{
-    assert(qiov->nalloc != -1);
-
-    qemu_iovec_reset(qiov);
-    g_free(qiov->iov);
-    qiov->nalloc = 0;
-    qiov->iov = NULL;
-}
-
-void qemu_iovec_reset(QEMUIOVector *qiov)
-{
-    assert(qiov->nalloc != -1);
-
-    qiov->niov = 0;
-    qiov->size = 0;
-}
-
-size_t qemu_iovec_to_buf(QEMUIOVector *qiov, size_t offset,
-                         void *buf, size_t bytes)
-{
-    return iov_to_buf(qiov->iov, qiov->niov, offset, buf, bytes);
-}
-
-size_t qemu_iovec_from_buf(QEMUIOVector *qiov, size_t offset,
-                           const void *buf, size_t bytes)
-{
-    return iov_from_buf(qiov->iov, qiov->niov, offset, buf, bytes);
-}
-
-size_t qemu_iovec_memset(QEMUIOVector *qiov, size_t offset,
-                         int fillc, size_t bytes)
-{
-    return iov_memset(qiov->iov, qiov->niov, offset, fillc, bytes);
-}
-
 /*
  * Checks if a buffer is all zeroes
  *
@@ -383,11 +280,6 @@ int qemu_parse_fd(const char *param)
     return fd;
 }
 
-int qemu_parse_fdset(const char *param)
-{
-    return qemu_parse_fd(param);
-}
-
 /* round down to the nearest power of 2*/
 int64_t pow2floor(int64_t value)
 {
diff --git a/def-helper.h b/def-helper.h
index b98ff69e4..022a9ceb6 100644
--- a/def-helper.h
+++ b/def-helper.h
@@ -128,6 +128,8 @@
 #define DEF_HELPER_5(name, ret, t1, t2, t3, t4, t5) \
     DEF_HELPER_FLAGS_5(name, 0, ret, t1, t2, t3, t4, t5)
 
+/* MAX_OPC_PARAM_IARGS must be set to n if last entry is DEF_HELPER_FLAGS_n. */
+
 #endif /* DEF_HELPER_H */
 
 #ifndef GEN_HELPER
diff --git a/default-configs/arm-softmmu.mak b/default-configs/arm-softmmu.mak
index f335a725f..2f1a5c994 100644
--- a/default-configs/arm-softmmu.mak
+++ b/default-configs/arm-softmmu.mak
@@ -22,6 +22,7 @@ CONFIG_ADS7846=y
 CONFIG_MAX111X=y
 CONFIG_SSI=y
 CONFIG_SSI_SD=y
+CONFIG_SSI_M25P80=y
 CONFIG_LAN9118=y
 CONFIG_SMC91C111=y
 CONFIG_DS1338=y
diff --git a/default-configs/microblaze-softmmu.mak b/default-configs/microblaze-softmmu.mak
index 64c9485de..2f442e5ae 100644
--- a/default-configs/microblaze-softmmu.mak
+++ b/default-configs/microblaze-softmmu.mak
@@ -5,3 +5,5 @@ CONFIG_PFLASH_CFI01=y
 CONFIG_SERIAL=y
 CONFIG_XILINX=y
 CONFIG_XILINX_AXI=y
+CONFIG_SSI=y
+CONFIG_SSI_M25P80=y
diff --git a/default-configs/microblazeel-softmmu.mak b/default-configs/microblazeel-softmmu.mak
index a9622760c..af9a3cde0 100644
--- a/default-configs/microblazeel-softmmu.mak
+++ b/default-configs/microblazeel-softmmu.mak
@@ -5,3 +5,5 @@ CONFIG_PFLASH_CFI01=y
 CONFIG_SERIAL=y
 CONFIG_XILINX=y
 CONFIG_XILINX_AXI=y
+CONFIG_SSI=y
+CONFIG_SSI_M25P80=y
diff --git a/default-configs/pci.mak b/default-configs/pci.mak
index 69e18f142..ae9d1eb48 100644
--- a/default-configs/pci.mak
+++ b/default-configs/pci.mak
@@ -19,3 +19,5 @@ CONFIG_IDE_PCI=y
 CONFIG_AHCI=y
 CONFIG_ESP=y
 CONFIG_ESP_PCI=y
+CONFIG_SERIAL=y
+CONFIG_SERIAL_PCI=y
diff --git a/default-configs/sparc64-softmmu.mak b/default-configs/sparc64-softmmu.mak
index c9a36c1e4..03e8b4271 100644
--- a/default-configs/sparc64-softmmu.mak
+++ b/default-configs/sparc64-softmmu.mak
@@ -6,7 +6,6 @@ CONFIG_M48T59=y
 CONFIG_PTIMER=y
 CONFIG_VGA=y
 CONFIG_VGA_PCI=y
-CONFIG_VGA_CIRRUS=y
 CONFIG_SERIAL=y
 CONFIG_PARALLEL=y
 CONFIG_PCKBD=y
diff --git a/device_tree.c b/device_tree.c
index d7a9b6bb8..a9236133c 100644
--- a/device_tree.c
+++ b/device_tree.c
@@ -304,3 +304,18 @@ int qemu_devtree_add_subnode(void *fdt, const char *name)
     g_free(dupname);
     return retval;
 }
+
+void qemu_devtree_dumpdtb(void *fdt, int size)
+{
+    QemuOpts *machine_opts;
+
+    machine_opts = qemu_opts_find(qemu_find_opts("machine"), 0);
+    if (machine_opts) {
+        const char *dumpdtb = qemu_opt_get(machine_opts, "dumpdtb");
+        if (dumpdtb) {
+            /* Dump the dtb to a file and quit */
+            exit(g_file_set_contents(dumpdtb, fdt, size, NULL) ? 0 : 1);
+        }
+    }
+
+}
diff --git a/device_tree.h b/device_tree.h
index f7a3e6cfc..f0b3f35e0 100644
--- a/device_tree.h
+++ b/device_tree.h
@@ -49,4 +49,6 @@ int qemu_devtree_add_subnode(void *fdt, const char *name);
                              sizeof(qdt_tmp));                                \
     } while (0)
 
+void qemu_devtree_dumpdtb(void *fdt, int size);
+
 #endif /* __DEVICE_TREE_H__ */
diff --git a/disas.c b/disas.c
index 7b2acc994..6da1dd09f 100644
--- a/disas.c
+++ b/disas.c
@@ -7,6 +7,11 @@
 #include "cpu.h"
 #include "disas.h"
 
+typedef struct CPUDebug {
+    struct disassemble_info info;
+    CPUArchState *env;
+} CPUDebug;
+
 /* Filled in by elfload.c.  Simplistic, but will do for now. */
 struct syminfo *syminfos = NULL;
 
@@ -32,7 +37,9 @@ target_read_memory (bfd_vma memaddr,
                     int length,
                     struct disassemble_info *info)
 {
-    cpu_memory_rw_debug(cpu_single_env, memaddr, myaddr, length, 0);
+    CPUDebug *s = container_of(info, CPUDebug, info);
+
+    cpu_memory_rw_debug(s->env, memaddr, myaddr, length, 0);
     return 0;
 }
 
@@ -158,32 +165,35 @@ print_insn_thumb1(bfd_vma pc, disassemble_info *info)
     ppc  - nonzero means little endian
     other targets - unused
  */
-void target_disas(FILE *out, target_ulong code, target_ulong size, int flags)
+void target_disas(FILE *out, CPUArchState *env, target_ulong code,
+                  target_ulong size, int flags)
 {
     target_ulong pc;
     int count;
-    struct disassemble_info disasm_info;
+    CPUDebug s;
     int (*print_insn)(bfd_vma pc, disassemble_info *info);
 
-    INIT_DISASSEMBLE_INFO(disasm_info, out, fprintf);
+    INIT_DISASSEMBLE_INFO(s.info, out, fprintf);
 
-    disasm_info.read_memory_func = target_read_memory;
-    disasm_info.buffer_vma = code;
-    disasm_info.buffer_length = size;
-    disasm_info.print_address_func = generic_print_target_address;
+    s.env = env;
+    s.info.read_memory_func = target_read_memory;
+    s.info.buffer_vma = code;
+    s.info.buffer_length = size;
+    s.info.print_address_func = generic_print_target_address;
 
 #ifdef TARGET_WORDS_BIGENDIAN
-    disasm_info.endian = BFD_ENDIAN_BIG;
+    s.info.endian = BFD_ENDIAN_BIG;
 #else
-    disasm_info.endian = BFD_ENDIAN_LITTLE;
+    s.info.endian = BFD_ENDIAN_LITTLE;
 #endif
 #if defined(TARGET_I386)
-    if (flags == 2)
-        disasm_info.mach = bfd_mach_x86_64;
-    else if (flags == 1)
-        disasm_info.mach = bfd_mach_i386_i8086;
-    else
-        disasm_info.mach = bfd_mach_i386_i386;
+    if (flags == 2) {
+        s.info.mach = bfd_mach_x86_64;
+    } else if (flags == 1) {
+        s.info.mach = bfd_mach_i386_i8086;
+    } else {
+        s.info.mach = bfd_mach_i386_i386;
+    }
     print_insn = print_insn_i386;
 #elif defined(TARGET_ARM)
     if (flags & 1) {
@@ -193,27 +203,28 @@ void target_disas(FILE *out, target_ulong code, target_ulong size, int flags)
     }
     if (flags & 2) {
 #ifdef TARGET_WORDS_BIGENDIAN
-        disasm_info.endian = BFD_ENDIAN_LITTLE;
+        s.info.endian = BFD_ENDIAN_LITTLE;
 #else
-        disasm_info.endian = BFD_ENDIAN_BIG;
+        s.info.endian = BFD_ENDIAN_BIG;
 #endif
     }
 #elif defined(TARGET_SPARC)
     print_insn = print_insn_sparc;
 #ifdef TARGET_SPARC64
-    disasm_info.mach = bfd_mach_sparc_v9b;
+    s.info.mach = bfd_mach_sparc_v9b;
 #endif
 #elif defined(TARGET_PPC)
-    if (flags >> 16)
-        disasm_info.endian = BFD_ENDIAN_LITTLE;
+    if (flags >> 16) {
+        s.info.endian = BFD_ENDIAN_LITTLE;
+    }
     if (flags & 0xFFFF) {
         /* If we have a precise definitions of the instructions set, use it */
-        disasm_info.mach = flags & 0xFFFF;
+        s.info.mach = flags & 0xFFFF;
     } else {
 #ifdef TARGET_PPC64
-        disasm_info.mach = bfd_mach_ppc64;
+        s.info.mach = bfd_mach_ppc64;
 #else
-        disasm_info.mach = bfd_mach_ppc;
+        s.info.mach = bfd_mach_ppc;
 #endif
     }
     print_insn = print_insn_ppc;
@@ -226,27 +237,27 @@ void target_disas(FILE *out, target_ulong code, target_ulong size, int flags)
     print_insn = print_insn_little_mips;
 #endif
 #elif defined(TARGET_SH4)
-    disasm_info.mach = bfd_mach_sh4;
+    s.info.mach = bfd_mach_sh4;
     print_insn = print_insn_sh;
 #elif defined(TARGET_ALPHA)
-    disasm_info.mach = bfd_mach_alpha_ev6;
+    s.info.mach = bfd_mach_alpha_ev6;
     print_insn = print_insn_alpha;
 #elif defined(TARGET_CRIS)
     if (flags != 32) {
-        disasm_info.mach = bfd_mach_cris_v0_v10;
+        s.info.mach = bfd_mach_cris_v0_v10;
         print_insn = print_insn_crisv10;
     } else {
-        disasm_info.mach = bfd_mach_cris_v32;
+        s.info.mach = bfd_mach_cris_v32;
         print_insn = print_insn_crisv32;
     }
 #elif defined(TARGET_S390X)
-    disasm_info.mach = bfd_mach_s390_64;
+    s.info.mach = bfd_mach_s390_64;
     print_insn = print_insn_s390;
 #elif defined(TARGET_MICROBLAZE)
-    disasm_info.mach = bfd_arch_microblaze;
+    s.info.mach = bfd_arch_microblaze;
     print_insn = print_insn_microblaze;
 #elif defined(TARGET_LM32)
-    disasm_info.mach = bfd_mach_lm32;
+    s.info.mach = bfd_mach_lm32;
     print_insn = print_insn_lm32;
 #else
     fprintf(out, "0x" TARGET_FMT_lx
@@ -256,14 +267,14 @@ void target_disas(FILE *out, target_ulong code, target_ulong size, int flags)
 
     for (pc = code; size > 0; pc += count, size -= count) {
 	fprintf(out, "0x" TARGET_FMT_lx ":  ", pc);
-	count = print_insn(pc, &disasm_info);
+	count = print_insn(pc, &s.info);
 #if 0
         {
             int i;
             uint8_t b;
             fprintf(out, " {");
             for(i = 0; i < count; i++) {
-                target_read_memory(pc + i, &b, 1, &disasm_info);
+                target_read_memory(pc + i, &b, 1, &s.info);
                 fprintf(out, " %02x", b);
             }
             fprintf(out, " }");
@@ -287,28 +298,28 @@ void disas(FILE *out, void *code, unsigned long size)
 {
     uintptr_t pc;
     int count;
-    struct disassemble_info disasm_info;
+    CPUDebug s;
     int (*print_insn)(bfd_vma pc, disassemble_info *info);
 
-    INIT_DISASSEMBLE_INFO(disasm_info, out, fprintf);
-    disasm_info.print_address_func = generic_print_host_address;
+    INIT_DISASSEMBLE_INFO(s.info, out, fprintf);
+    s.info.print_address_func = generic_print_host_address;
 
-    disasm_info.buffer = code;
-    disasm_info.buffer_vma = (uintptr_t)code;
-    disasm_info.buffer_length = size;
+    s.info.buffer = code;
+    s.info.buffer_vma = (uintptr_t)code;
+    s.info.buffer_length = size;
 
 #ifdef HOST_WORDS_BIGENDIAN
-    disasm_info.endian = BFD_ENDIAN_BIG;
+    s.info.endian = BFD_ENDIAN_BIG;
 #else
-    disasm_info.endian = BFD_ENDIAN_LITTLE;
+    s.info.endian = BFD_ENDIAN_LITTLE;
 #endif
 #if defined(CONFIG_TCG_INTERPRETER)
     print_insn = print_insn_tci;
 #elif defined(__i386__)
-    disasm_info.mach = bfd_mach_i386_i386;
+    s.info.mach = bfd_mach_i386_i386;
     print_insn = print_insn_i386;
 #elif defined(__x86_64__)
-    disasm_info.mach = bfd_mach_x86_64;
+    s.info.mach = bfd_mach_x86_64;
     print_insn = print_insn_i386;
 #elif defined(_ARCH_PPC)
     print_insn = print_insn_ppc;
@@ -316,9 +327,7 @@ void disas(FILE *out, void *code, unsigned long size)
     print_insn = print_insn_alpha;
 #elif defined(__sparc__)
     print_insn = print_insn_sparc;
-#if defined(__sparc_v8plus__) || defined(__sparc_v8plusa__) || defined(__sparc_v9__)
-    disasm_info.mach = bfd_mach_sparc_v9b;
-#endif
+    s.info.mach = bfd_mach_sparc_v9b;
 #elif defined(__arm__)
     print_insn = print_insn_arm;
 #elif defined(__MIPSEB__)
@@ -340,7 +349,7 @@ void disas(FILE *out, void *code, unsigned long size)
 #endif
     for (pc = (uintptr_t)code; size > 0; pc += count, size -= count) {
         fprintf(out, "0x%08" PRIxPTR ":  ", pc);
-	count = print_insn(pc, &disasm_info);
+        count = print_insn(pc, &s.info);
 	fprintf(out, "\n");
 	if (count < 0)
 	    break;
@@ -368,16 +377,17 @@ const char *lookup_symbol(target_ulong orig_addr)
 #include "monitor.h"
 
 static int monitor_disas_is_physical;
-static CPUArchState *monitor_disas_env;
 
 static int
 monitor_read_memory (bfd_vma memaddr, bfd_byte *myaddr, int length,
                      struct disassemble_info *info)
 {
+    CPUDebug *s = container_of(info, CPUDebug, info);
+
     if (monitor_disas_is_physical) {
         cpu_physical_memory_read(memaddr, myaddr, length);
     } else {
-        cpu_memory_rw_debug(monitor_disas_env, memaddr,myaddr, length, 0);
+        cpu_memory_rw_debug(s->env, memaddr,myaddr, length, 0);
     }
     return 0;
 }
@@ -396,30 +406,31 @@ void monitor_disas(Monitor *mon, CPUArchState *env,
                    target_ulong pc, int nb_insn, int is_physical, int flags)
 {
     int count, i;
-    struct disassemble_info disasm_info;
+    CPUDebug s;
     int (*print_insn)(bfd_vma pc, disassemble_info *info);
 
-    INIT_DISASSEMBLE_INFO(disasm_info, (FILE *)mon, monitor_fprintf);
+    INIT_DISASSEMBLE_INFO(s.info, (FILE *)mon, monitor_fprintf);
 
-    monitor_disas_env = env;
+    s.env = env;
     monitor_disas_is_physical = is_physical;
-    disasm_info.read_memory_func = monitor_read_memory;
-    disasm_info.print_address_func = generic_print_target_address;
+    s.info.read_memory_func = monitor_read_memory;
+    s.info.print_address_func = generic_print_target_address;
 
-    disasm_info.buffer_vma = pc;
+    s.info.buffer_vma = pc;
 
 #ifdef TARGET_WORDS_BIGENDIAN
-    disasm_info.endian = BFD_ENDIAN_BIG;
+    s.info.endian = BFD_ENDIAN_BIG;
 #else
-    disasm_info.endian = BFD_ENDIAN_LITTLE;
+    s.info.endian = BFD_ENDIAN_LITTLE;
 #endif
 #if defined(TARGET_I386)
-    if (flags == 2)
-        disasm_info.mach = bfd_mach_x86_64;
-    else if (flags == 1)
-        disasm_info.mach = bfd_mach_i386_i8086;
-    else
-        disasm_info.mach = bfd_mach_i386_i386;
+    if (flags == 2) {
+        s.info.mach = bfd_mach_x86_64;
+    } else if (flags == 1) {
+        s.info.mach = bfd_mach_i386_i8086;
+    } else {
+        s.info.mach = bfd_mach_i386_i386;
+    }
     print_insn = print_insn_i386;
 #elif defined(TARGET_ARM)
     print_insn = print_insn_arm;
@@ -428,13 +439,13 @@ void monitor_disas(Monitor *mon, CPUArchState *env,
 #elif defined(TARGET_SPARC)
     print_insn = print_insn_sparc;
 #ifdef TARGET_SPARC64
-    disasm_info.mach = bfd_mach_sparc_v9b;
+    s.info.mach = bfd_mach_sparc_v9b;
 #endif
 #elif defined(TARGET_PPC)
 #ifdef TARGET_PPC64
-    disasm_info.mach = bfd_mach_ppc64;
+    s.info.mach = bfd_mach_ppc64;
 #else
-    disasm_info.mach = bfd_mach_ppc;
+    s.info.mach = bfd_mach_ppc;
 #endif
     print_insn = print_insn_ppc;
 #elif defined(TARGET_M68K)
@@ -446,13 +457,13 @@ void monitor_disas(Monitor *mon, CPUArchState *env,
     print_insn = print_insn_little_mips;
 #endif
 #elif defined(TARGET_SH4)
-    disasm_info.mach = bfd_mach_sh4;
+    s.info.mach = bfd_mach_sh4;
     print_insn = print_insn_sh;
 #elif defined(TARGET_S390X)
-    disasm_info.mach = bfd_mach_s390_64;
+    s.info.mach = bfd_mach_s390_64;
     print_insn = print_insn_s390;
 #elif defined(TARGET_LM32)
-    disasm_info.mach = bfd_mach_lm32;
+    s.info.mach = bfd_mach_lm32;
     print_insn = print_insn_lm32;
 #else
     monitor_printf(mon, "0x" TARGET_FMT_lx
@@ -462,7 +473,7 @@ void monitor_disas(Monitor *mon, CPUArchState *env,
 
     for(i = 0; i < nb_insn; i++) {
 	monitor_printf(mon, "0x" TARGET_FMT_lx ":  ", pc);
-	count = print_insn(pc, &disasm_info);
+        count = print_insn(pc, &s.info);
 	monitor_printf(mon, "\n");
 	if (count < 0)
 	    break;
diff --git a/disas.h b/disas.h
index 3ab42af0b..c13ca9a3a 100644
--- a/disas.h
+++ b/disas.h
@@ -6,7 +6,8 @@
 #ifdef NEED_CPU_H
 /* Disassemble this for me please... (debugging). */
 void disas(FILE *out, void *code, unsigned long size);
-void target_disas(FILE *out, target_ulong code, target_ulong size, int flags);
+void target_disas(FILE *out, CPUArchState *env, target_ulong code,
+                  target_ulong size, int flags);
 
 void monitor_disas(Monitor *mon, CPUArchState *env,
                    target_ulong pc, int nb_insn, int is_physical, int flags);
@@ -22,7 +23,7 @@ struct elf64_sym;
 #if defined(CONFIG_USER_ONLY)
 typedef const char *(*lookup_symbol_t)(struct syminfo *s, target_ulong orig_addr);
 #else
-typedef const char *(*lookup_symbol_t)(struct syminfo *s, target_phys_addr_t orig_addr);
+typedef const char *(*lookup_symbol_t)(struct syminfo *s, hwaddr orig_addr);
 #endif
 
 struct syminfo {
diff --git a/dma-helpers.c b/dma-helpers.c
index 433d8b21b..4f5fb649e 100644
--- a/dma-helpers.c
+++ b/dma-helpers.c
@@ -14,7 +14,8 @@
 
 /* #define DEBUG_IOMMU */
 
-static void do_dma_memory_set(dma_addr_t addr, uint8_t c, dma_addr_t len)
+static void do_dma_memory_set(AddressSpace *as,
+                              dma_addr_t addr, uint8_t c, dma_addr_t len)
 {
 #define FILLBUF_SIZE 512
     uint8_t fillbuf[FILLBUF_SIZE];
@@ -23,7 +24,7 @@ static void do_dma_memory_set(dma_addr_t addr, uint8_t c, dma_addr_t len)
     memset(fillbuf, c, FILLBUF_SIZE);
     while (len > 0) {
         l = len < FILLBUF_SIZE ? len : FILLBUF_SIZE;
-        cpu_physical_memory_rw(addr, fillbuf, l, true);
+        address_space_rw(as, addr, fillbuf, l, true);
         len -= l;
         addr += l;
     }
@@ -36,7 +37,7 @@ int dma_memory_set(DMAContext *dma, dma_addr_t addr, uint8_t c, dma_addr_t len)
     if (dma_has_iommu(dma)) {
         return iommu_dma_memory_set(dma, addr, c, len);
     }
-    do_dma_memory_set(addr, c, len);
+    do_dma_memory_set(dma->as, addr, c, len);
 
     return 0;
 }
@@ -194,7 +195,7 @@ static void dma_aio_cancel(BlockDriverAIOCB *acb)
     dma_complete(dbs, 0);
 }
 
-static AIOPool dma_aio_pool = {
+static const AIOCBInfo dma_aiocb_info = {
     .aiocb_size         = sizeof(DMAAIOCB),
     .cancel             = dma_aio_cancel,
 };
@@ -204,7 +205,7 @@ BlockDriverAIOCB *dma_bdrv_io(
     DMAIOFunc *io_func, BlockDriverCompletionFunc *cb,
     void *opaque, DMADirection dir)
 {
-    DMAAIOCB *dbs = qemu_aio_get(&dma_aio_pool, bs, cb, opaque);
+    DMAAIOCB *dbs = qemu_aio_get(&dma_aiocb_info, bs, cb, opaque);
 
     trace_dma_bdrv_io(dbs, bs, sector_num, (dir == DMA_DIRECTION_TO_DEVICE));
 
@@ -280,7 +281,7 @@ void dma_acct_start(BlockDriverState *bs, BlockAcctCookie *cookie,
 bool iommu_dma_memory_valid(DMAContext *dma, dma_addr_t addr, dma_addr_t len,
                             DMADirection dir)
 {
-    target_phys_addr_t paddr, plen;
+    hwaddr paddr, plen;
 
 #ifdef DEBUG_IOMMU
     fprintf(stderr, "dma_memory_check context=%p addr=0x" DMA_ADDR_FMT
@@ -307,7 +308,7 @@ bool iommu_dma_memory_valid(DMAContext *dma, dma_addr_t addr, dma_addr_t len,
 int iommu_dma_memory_rw(DMAContext *dma, dma_addr_t addr,
                         void *buf, dma_addr_t len, DMADirection dir)
 {
-    target_phys_addr_t paddr, plen;
+    hwaddr paddr, plen;
     int err;
 
 #ifdef DEBUG_IOMMU
@@ -332,8 +333,7 @@ int iommu_dma_memory_rw(DMAContext *dma, dma_addr_t addr,
             plen = len;
         }
 
-        cpu_physical_memory_rw(paddr, buf, plen,
-                               dir == DMA_DIRECTION_FROM_DEVICE);
+        address_space_rw(dma->as, paddr, buf, plen, dir == DMA_DIRECTION_FROM_DEVICE);
 
         len -= plen;
         addr += plen;
@@ -346,7 +346,7 @@ int iommu_dma_memory_rw(DMAContext *dma, dma_addr_t addr,
 int iommu_dma_memory_set(DMAContext *dma, dma_addr_t addr, uint8_t c,
                          dma_addr_t len)
 {
-    target_phys_addr_t paddr, plen;
+    hwaddr paddr, plen;
     int err;
 
 #ifdef DEBUG_IOMMU
@@ -366,7 +366,7 @@ int iommu_dma_memory_set(DMAContext *dma, dma_addr_t addr, uint8_t c,
             plen = len;
         }
 
-        do_dma_memory_set(paddr, c, plen);
+        do_dma_memory_set(dma->as, paddr, c, plen);
 
         len -= plen;
         addr += plen;
@@ -375,13 +375,14 @@ int iommu_dma_memory_set(DMAContext *dma, dma_addr_t addr, uint8_t c,
     return 0;
 }
 
-void dma_context_init(DMAContext *dma, DMATranslateFunc translate,
+void dma_context_init(DMAContext *dma, AddressSpace *as, DMATranslateFunc translate,
                       DMAMapFunc map, DMAUnmapFunc unmap)
 {
 #ifdef DEBUG_IOMMU
     fprintf(stderr, "dma_context_init(%p, %p, %p, %p)\n",
             dma, translate, map, unmap);
 #endif
+    dma->as = as;
     dma->translate = translate;
     dma->map = map;
     dma->unmap = unmap;
@@ -391,7 +392,7 @@ void *iommu_dma_memory_map(DMAContext *dma, dma_addr_t addr, dma_addr_t *len,
                            DMADirection dir)
 {
     int err;
-    target_phys_addr_t paddr, plen;
+    hwaddr paddr, plen;
     void *buf;
 
     if (dma->map) {
@@ -407,14 +408,13 @@ void *iommu_dma_memory_map(DMAContext *dma, dma_addr_t addr, dma_addr_t *len,
     /*
      * If this is true, the virtual region is contiguous,
      * but the translated physical region isn't. We just
-     * clamp *len, much like cpu_physical_memory_map() does.
+     * clamp *len, much like address_space_map() does.
      */
     if (plen < *len) {
         *len = plen;
     }
 
-    buf = cpu_physical_memory_map(paddr, &plen,
-                                  dir == DMA_DIRECTION_FROM_DEVICE);
+    buf = address_space_map(dma->as, paddr, &plen, dir == DMA_DIRECTION_FROM_DEVICE);
     *len = plen;
 
     return buf;
@@ -428,8 +428,7 @@ void iommu_dma_memory_unmap(DMAContext *dma, void *buffer, dma_addr_t len,
         return;
     }
 
-    cpu_physical_memory_unmap(buffer, len,
-                              dir == DMA_DIRECTION_FROM_DEVICE,
-                              access_len);
+    address_space_unmap(dma->as, buffer, len, dir == DMA_DIRECTION_FROM_DEVICE,
+                        access_len);
 
 }
diff --git a/dma.h b/dma.h
index f35c4b663..eedf87838 100644
--- a/dma.h
+++ b/dma.h
@@ -11,6 +11,7 @@
 #define DMA_H
 
 #include <stdio.h>
+#include "memory.h"
 #include "hw/hw.h"
 #include "block.h"
 #include "kvm.h"
@@ -31,7 +32,7 @@ struct QEMUSGList {
     DMAContext *dma;
 };
 
-#if defined(TARGET_PHYS_ADDR_BITS)
+#ifndef CONFIG_USER_ONLY
 
 /*
  * When an IOMMU is present, bus addresses become distinct from
@@ -47,8 +48,8 @@ typedef uint64_t dma_addr_t;
 
 typedef int DMATranslateFunc(DMAContext *dma,
                              dma_addr_t addr,
-                             target_phys_addr_t *paddr,
-                             target_phys_addr_t *len,
+                             hwaddr *paddr,
+                             hwaddr *len,
                              DMADirection dir);
 typedef void* DMAMapFunc(DMAContext *dma,
                          dma_addr_t addr,
@@ -61,11 +62,17 @@ typedef void DMAUnmapFunc(DMAContext *dma,
                           dma_addr_t access_len);
 
 struct DMAContext {
+    AddressSpace *as;
     DMATranslateFunc *translate;
     DMAMapFunc *map;
     DMAUnmapFunc *unmap;
 };
 
+/* A global DMA context corresponding to the address_space_memory
+ * AddressSpace, for sysbus devices which do DMA.
+ */
+extern DMAContext dma_context_memory;
+
 static inline void dma_barrier(DMAContext *dma, DMADirection dir)
 {
     /*
@@ -93,7 +100,7 @@ static inline void dma_barrier(DMAContext *dma, DMADirection dir)
 
 static inline bool dma_has_iommu(DMAContext *dma)
 {
-    return !!dma;
+    return dma && dma->translate;
 }
 
 /* Checks that the given range of addresses is valid for DMA.  This is
@@ -120,8 +127,7 @@ static inline int dma_memory_rw_relaxed(DMAContext *dma, dma_addr_t addr,
 {
     if (!dma_has_iommu(dma)) {
         /* Fast-path for no IOMMU */
-        cpu_physical_memory_rw(addr, buf, len,
-                               dir == DMA_DIRECTION_FROM_DEVICE);
+        address_space_rw(dma->as, addr, buf, len, dir == DMA_DIRECTION_FROM_DEVICE);
         return 0;
     } else {
         return iommu_dma_memory_rw(dma, addr, buf, len, dir);
@@ -176,11 +182,10 @@ static inline void *dma_memory_map(DMAContext *dma,
                                    DMADirection dir)
 {
     if (!dma_has_iommu(dma)) {
-        target_phys_addr_t xlen = *len;
+        hwaddr xlen = *len;
         void *p;
 
-        p = cpu_physical_memory_map(addr, &xlen,
-                                    dir == DMA_DIRECTION_FROM_DEVICE);
+        p = address_space_map(dma->as, addr, &xlen, dir == DMA_DIRECTION_FROM_DEVICE);
         *len = xlen;
         return p;
     } else {
@@ -196,9 +201,8 @@ static inline void dma_memory_unmap(DMAContext *dma,
                                     DMADirection dir, dma_addr_t access_len)
 {
     if (!dma_has_iommu(dma)) {
-        cpu_physical_memory_unmap(buffer, (target_phys_addr_t)len,
-                                  dir == DMA_DIRECTION_FROM_DEVICE,
-                                  access_len);
+        address_space_unmap(dma->as, buffer, (hwaddr)len,
+                            dir == DMA_DIRECTION_FROM_DEVICE, access_len);
     } else {
         iommu_dma_memory_unmap(dma, buffer, len, dir, access_len);
     }
@@ -242,7 +246,7 @@ DEFINE_LDST_DMA(q, q, 64, be);
 
 #undef DEFINE_LDST_DMA
 
-void dma_context_init(DMAContext *dma, DMATranslateFunc translate,
+void dma_context_init(DMAContext *dma, AddressSpace *as, DMATranslateFunc translate,
                       DMAMapFunc map, DMAUnmapFunc unmap);
 
 struct ScatterGatherEntry {
diff --git a/docs/qemupciserial.inf b/docs/qemupciserial.inf
new file mode 100644
index 000000000..3474310a4
--- /dev/null
+++ b/docs/qemupciserial.inf
@@ -0,0 +1,109 @@
+; qemupciserial.inf for QEMU, based on MSPORTS.INF
+
+; The driver itself is shipped with Windows (serial.sys).  This is
+; just a inf file to tell windows which pci id the serial pci card
+; emulated by qemu has, and to apply a name tag to it which windows
+; will show in the device manager.
+
+; Installing the driver: Go to device manager.  You should find a "pci
+; serial card" tagged with a yellow question mark.  Open properties.
+; Pick "update driver".  Then "select driver manually".  Pick "Ports
+; (Com+Lpt)" from the list.  Click "Have a disk".  Select this file.
+; Procedure may vary a bit depending on the windows version.
+
+; FIXME: This file covers the single port version only.
+
+[Version]
+Signature="$CHICAGO$"
+Class=Ports
+ClassGuid={4D36E978-E325-11CE-BFC1-08002BE10318}
+Provider=%QEMU%
+DriverVer=09/24/2012,1.3.0
+
+[SourceDisksNames]
+3426=windows cd
+
+[SourceDisksFiles]
+serial.sys 		= 3426
+serenum.sys 		= 3426
+
+[DestinationDirs]
+DefaultDestDir  = 11        ;LDID_SYS
+ComPort.NT.Copy = 12        ;DIRID_DRIVERS
+SerialEnumerator.NT.Copy=12 ;DIRID_DRIVERS
+
+; Drivers
+;----------------------------------------------------------
+[Manufacturer]
+%QEMU%=QEMU,NTx86
+
+[QEMU.NTx86]
+%QEMU-PCI_SERIAL.DeviceDesc% = ComPort, "PCI\VEN_1b36&DEV_0002&CC_0700"
+
+; COM sections
+;----------------------------------------------------------
+[ComPort.AddReg]
+HKR,,PortSubClass,1,01
+
+[ComPort.NT]
+AddReg=ComPort.AddReg, ComPort.NT.AddReg
+LogConfig=caa
+SyssetupPnPFlags = 1
+
+[ComPort.NT.HW]
+AddReg=ComPort.NT.HW.AddReg
+
+[ComPort.NT.AddReg]
+HKR,,EnumPropPages32,,"MsPorts.dll,SerialPortPropPageProvider"
+
+[ComPort.NT.HW.AddReg]
+HKR,,"UpperFilters",0x00010000,"serenum"
+
+;-------------- Service installation
+; Port Driver (function driver for this device)
+[ComPort.NT.Services]
+AddService = Serial, 0x00000002, Serial_Service_Inst, Serial_EventLog_Inst
+AddService = Serenum,,Serenum_Service_Inst
+
+; -------------- Serial Port Driver install sections
+[Serial_Service_Inst]
+DisplayName    = %Serial.SVCDESC%
+ServiceType    = 1               ; SERVICE_KERNEL_DRIVER
+StartType      = 1               ; SERVICE_SYSTEM_START (this driver may do detection)
+ErrorControl   = 0               ; SERVICE_ERROR_IGNORE
+ServiceBinary  = %12%\serial.sys
+LoadOrderGroup = Extended base
+
+; -------------- Serenum Driver install section
+[Serenum_Service_Inst]
+DisplayName    = %Serenum.SVCDESC%
+ServiceType    = 1               ; SERVICE_KERNEL_DRIVER
+StartType      = 3               ; SERVICE_DEMAND_START
+ErrorControl   = 1               ; SERVICE_ERROR_NORMAL
+ServiceBinary  = %12%\serenum.sys
+LoadOrderGroup = PNP Filter
+
+[Serial_EventLog_Inst]
+AddReg = Serial_EventLog_AddReg
+
+[Serial_EventLog_AddReg]
+HKR,,EventMessageFile,0x00020000,"%%SystemRoot%%\System32\IoLogMsg.dll;%%SystemRoot%%\System32\drivers\serial.sys"
+HKR,,TypesSupported,0x00010001,7
+
+; The following sections are COM port resource configs.
+; Section name format means:
+; Char 1 = c (COM port)
+; Char 2 = I/O config: 1 (3f8), 2 (2f8), 3 (3e8), 4 (2e8), a (any)
+; Char 3 = IRQ config: #, a (any)
+
+[caa]                   ; Any base, any IRQ
+ConfigPriority=HARDRECONFIG
+IOConfig=8@100-ffff%fff8(3ff::)
+IRQConfig=S:3,4,5,7,9,10,11,12,14,15
+
+[Strings]
+QEMU="QEMU"
+QEMU-PCI_SERIAL.DeviceDesc="QEMU Serial PCI Card"
+
+Serial.SVCDESC   = "Serial port driver"
+Serenum.SVCDESC = "Serenum Filter Driver"
diff --git a/docs/specs/pci-serial.txt b/docs/specs/pci-serial.txt
new file mode 100644
index 000000000..66c761f2b
--- /dev/null
+++ b/docs/specs/pci-serial.txt
@@ -0,0 +1,34 @@
+
+QEMU pci serial devices
+=======================
+
+There is one single-port variant and two muliport-variants.  Linux
+guests out-of-the box with all cards.  There is a Windows inf file
+(docs/qemupciserial.inf) to setup the single-port card in Windows
+guests.
+
+
+single-port card
+----------------
+
+Name:   pci-serial
+PCI ID: 1b36:0002
+
+PCI Region 0:
+   IO bar, 8 bytes long, with the 16550 uart mapped to it.
+   Interrupt is wired to pin A.
+
+
+multiport cards
+---------------
+
+Name:   pci-serial-2x
+PCI ID: 1b36:0003
+
+Name:   pci-serial-4x
+PCI ID: 1b36:0004
+
+PCI Region 0:
+   IO bar, with two/four 16550 uart mapped after each other.
+   The first is at offset 0, second at offset 8, ...
+   Interrupt is wired to pin A.
diff --git a/docs/specs/ppc-spapr-hcalls.txt b/docs/specs/ppc-spapr-hcalls.txt
index 52ba8d42a..667b3fa00 100644
--- a/docs/specs/ppc-spapr-hcalls.txt
+++ b/docs/specs/ppc-spapr-hcalls.txt
@@ -31,7 +31,7 @@ Arguments:
 
 Returns:
 
-  H_SUCCESS   : Successully called the RTAS function (RTAS result
+  H_SUCCESS   : Successfully called the RTAS function (RTAS result
                 will have been stored in the parameter block)
   H_PARAMETER : Unknown token
 
diff --git a/docs/specs/standard-vga.txt b/docs/specs/standard-vga.txt
new file mode 100644
index 000000000..8a4c1e93c
--- /dev/null
+++ b/docs/specs/standard-vga.txt
@@ -0,0 +1,65 @@
+
+QEMU Standard VGA
+=================
+
+Exists in two variants, for isa and pci.
+
+command line switches:
+    -vga std            [ picks isa for -M isapc, otherwise pci ]
+    -device VGA         [ pci variant ]
+    -device isa-vga     [ isa variant ]
+
+
+PCI spec
+--------
+
+Applies to the pci variant only for obvious reasons.
+
+PCI ID: 1234:1111
+
+PCI Region 0:
+   Framebuffer memory, 16 MB in size (by default).
+   Size is tunable via vga_mem_mb property.
+
+PCI Region 1:
+   Reserved (so we have the option to make the framebuffer bar 64bit).
+
+PCI Region 2:
+   MMIO bar, 4096 bytes in size (qemu 1.3+)
+
+PCI ROM Region:
+   Holds the vgabios (qemu 0.14+).
+
+
+IO ports used
+-------------
+
+03c0 - 03df : standard vga ports
+01ce        : bochs vbe interface index port
+01cf        : bochs vbe interface data port (x86 only)
+01d0        : bochs vbe interface data port
+
+
+Memory regions used
+-------------------
+
+0xe0000000 : Framebuffer memory, isa variant only.
+
+The pci variant used to mirror the framebuffer bar here, qemu 0.14+
+stops doing that (except when in -M pc-$old compat mode).
+
+
+MMIO area spec
+--------------
+
+Likewise applies to the pci variant only for obvious reasons.
+
+0000 - 03ff : reserved, for possible virtio extension.
+0400 - 041f : vga ioports (0x3c0 -> 0x3df), remapped 1:1.
+              word access is supported, bytes are written
+              in little endia order (aka index port first),
+              so indexed registers can be updated with a
+              single mmio write (and thus only one vmexit).
+0500 - 0515 : bochs dispi interface registers, mapped flat
+              without index/data ports.  Use (index << 1)
+              as offset for (16bit) register access.
diff --git a/docs/tracing.txt b/docs/tracing.txt
index c54113336..453cc4a63 100644
--- a/docs/tracing.txt
+++ b/docs/tracing.txt
@@ -139,6 +139,10 @@ having a common prefix in a batch. For example, virtio-blk trace events could
 be enabled using:
   trace-event virtio_blk_* on
 
+If a line in the "-trace events=<file>" file begins with a '-', the trace event
+will be disabled instead of enabled.  This is useful when a wildcard was used
+to enable an entire family of events but one noisy event needs to be disabled.
+
 == Trace backends ==
 
 The "tracetool" script automates tedious trace event code generation and also
@@ -185,15 +189,6 @@ records the char* pointer value instead of the string that is pointed to.
 
 ==== Monitor commands ====
 
-* info trace
-  Display the contents of trace buffer.  This command dumps the trace buffer
-  with simple formatting.  For full pretty-printing, use the simpletrace.py
-  script on a binary trace file.
-
-  The trace buffer is written into until full.  The full trace buffer is
-  flushed and emptied.  This means the 'info trace' will display few or no
-  entries if the buffer has just been flushed.
-
 * trace-file on|off|flush|set <path>
   Enable/disable/flush the trace file or set the trace file name.
 
diff --git a/docs/usb2.txt b/docs/usb2.txt
index d17e3c004..43dacdec2 100644
--- a/docs/usb2.txt
+++ b/docs/usb2.txt
@@ -58,11 +58,11 @@ try ...
 xhci controller support
 -----------------------
 
-There also is xhci host controller support available.  It got alot
+There is also xhci host controller support available.  It got a lot
 less testing than ehci and there are a bunch of known limitations, so
 ehci may work better for you.  On the other hand the xhci hardware
 design is much more virtualization-friendly, thus xhci emulation uses
-less ressources (especially cpu).  If you wanna give xhci a try
+less resources (especially cpu).  If you want to give xhci a try
 use this to add the host controller ...
 
     qemu -device nec-usb-xhci,id=xhci
diff --git a/dump.c b/dump.c
index 2bf8d8d99..5640c2c2f 100644
--- a/dump.c
+++ b/dump.c
@@ -15,7 +15,7 @@
 #include "elf.h"
 #include "cpu.h"
 #include "cpu-all.h"
-#include "targphys.h"
+#include "hwaddr.h"
 #include "monitor.h"
 #include "kvm.h"
 #include "dump.h"
@@ -66,7 +66,7 @@ typedef struct DumpState {
     bool have_section;
     bool resume;
     size_t note_size;
-    target_phys_addr_t memory_offset;
+    hwaddr memory_offset;
     int fd;
 
     RAMBlock *block;
@@ -100,18 +100,11 @@ static void dump_error(DumpState *s, const char *reason)
 static int fd_write_vmcore(void *buf, size_t size, void *opaque)
 {
     DumpState *s = opaque;
-    int fd = s->fd;
-    size_t writen_size;
+    size_t written_size;
 
-    /* The fd may be passed from user, and it can be non-blocked */
-    while (size) {
-        writen_size = qemu_write_full(fd, buf, size);
-        if (writen_size != size && errno != EAGAIN) {
-            return -1;
-        }
-
-        buf += writen_size;
-        size -= writen_size;
+    written_size = qemu_write_full(s->fd, buf, size);
+    if (written_size != size) {
+        return -1;
     }
 
     return 0;
@@ -194,7 +187,7 @@ static int write_elf32_header(DumpState *s)
 }
 
 static int write_elf64_load(DumpState *s, MemoryMapping *memory_mapping,
-                            int phdr_index, target_phys_addr_t offset)
+                            int phdr_index, hwaddr offset)
 {
     Elf64_Phdr phdr;
     int ret;
@@ -223,7 +216,7 @@ static int write_elf64_load(DumpState *s, MemoryMapping *memory_mapping,
 }
 
 static int write_elf32_load(DumpState *s, MemoryMapping *memory_mapping,
-                            int phdr_index, target_phys_addr_t offset)
+                            int phdr_index, hwaddr offset)
 {
     Elf32_Phdr phdr;
     int ret;
@@ -255,7 +248,7 @@ static int write_elf64_note(DumpState *s)
 {
     Elf64_Phdr phdr;
     int endian = s->dump_info.d_endian;
-    target_phys_addr_t begin = s->memory_offset - s->note_size;
+    hwaddr begin = s->memory_offset - s->note_size;
     int ret;
 
     memset(&phdr, 0, sizeof(Elf64_Phdr));
@@ -303,7 +296,7 @@ static int write_elf64_notes(DumpState *s)
 
 static int write_elf32_note(DumpState *s)
 {
-    target_phys_addr_t begin = s->memory_offset - s->note_size;
+    hwaddr begin = s->memory_offset - s->note_size;
     Elf32_Phdr phdr;
     int endian = s->dump_info.d_endian;
     int ret;
@@ -421,11 +414,11 @@ static int write_memory(DumpState *s, RAMBlock *block, ram_addr_t start,
 }
 
 /* get the memory's offset in the vmcore */
-static target_phys_addr_t get_offset(target_phys_addr_t phys_addr,
+static hwaddr get_offset(hwaddr phys_addr,
                                      DumpState *s)
 {
     RAMBlock *block;
-    target_phys_addr_t offset = s->memory_offset;
+    hwaddr offset = s->memory_offset;
     int64_t size_in_block, start;
 
     if (s->has_filter) {
@@ -470,7 +463,7 @@ static target_phys_addr_t get_offset(target_phys_addr_t phys_addr,
 
 static int write_elf_loads(DumpState *s)
 {
-    target_phys_addr_t offset;
+    hwaddr offset;
     MemoryMapping *memory_mapping;
     uint32_t phdr_index = 1;
     int ret;
@@ -836,9 +829,8 @@ void qmp_dump_guest_memory(bool paging, const char *file, bool has_begin,
 
 #if !defined(WIN32)
     if (strstart(file, "fd:", &p)) {
-        fd = monitor_get_fd(cur_mon, p);
+        fd = monitor_get_fd(cur_mon, p, errp);
         if (fd == -1) {
-            error_set(errp, QERR_FD_NOT_FOUND, p);
             return;
         }
     }
diff --git a/dyngen-exec.h b/dyngen-exec.h
deleted file mode 100644
index 083e20bed..000000000
--- a/dyngen-exec.h
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- *  dyngen defines for micro operation code
- *
- *  Copyright (c) 2003 Fabrice Bellard
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, see <http://www.gnu.org/licenses/>.
- */
-#if !defined(__DYNGEN_EXEC_H__)
-#define __DYNGEN_EXEC_H__
-
-#if defined(CONFIG_TCG_INTERPRETER)
-/* The TCG interpreter does not need a special register AREG0,
- * but it is possible to use one by defining AREG0.
- * On i386, register edi seems to work. */
-/* Run without special register AREG0 or use a value defined elsewhere. */
-#elif defined(__i386__)
-#define AREG0 "ebp"
-#elif defined(__x86_64__)
-#define AREG0 "r14"
-#elif defined(_ARCH_PPC)
-#define AREG0 "r27"
-#elif defined(__arm__)
-#define AREG0 "r6"
-#elif defined(__hppa__)
-#define AREG0 "r17"
-#elif defined(__mips__)
-#define AREG0 "s0"
-#elif defined(__sparc__)
-#ifdef CONFIG_SOLARIS
-#define AREG0 "g2"
-#else
-#ifdef __sparc_v9__
-#define AREG0 "g5"
-#else
-#define AREG0 "g6"
-#endif
-#endif
-#elif defined(__s390__)
-#define AREG0 "r10"
-#elif defined(__alpha__)
-/* Note $15 is the frame pointer, so anything in op-i386.c that would
-   require a frame pointer, like alloca, would probably loose.  */
-#define AREG0 "$15"
-#elif defined(__mc68000)
-#define AREG0 "%a5"
-#elif defined(__ia64__)
-#define AREG0 "r7"
-#else
-#error unsupported CPU
-#endif
-
-#if defined(AREG0)
-register CPUArchState *env asm(AREG0);
-#else
-/* TODO: Try env = cpu_single_env. */
-extern CPUArchState *env;
-#endif
-
-#endif /* !defined(__DYNGEN_EXEC_H__) */
diff --git a/error.c b/error.c
index 1f05fc466..128d88cd9 100644
--- a/error.c
+++ b/error.c
@@ -43,6 +43,34 @@ void error_set(Error **errp, ErrorClass err_class, const char *fmt, ...)
     *errp = err;
 }
 
+void error_set_errno(Error **errp, int os_errno, ErrorClass err_class,
+                     const char *fmt, ...)
+{
+    Error *err;
+    char *msg1;
+    va_list ap;
+
+    if (errp == NULL) {
+        return;
+    }
+    assert(*errp == NULL);
+
+    err = g_malloc0(sizeof(*err));
+
+    va_start(ap, fmt);
+    msg1 = g_strdup_vprintf(fmt, ap);
+    if (os_errno != 0) {
+        err->msg = g_strdup_printf("%s: %s", msg1, strerror(os_errno));
+        g_free(msg1);
+    } else {
+        err->msg = msg1;
+    }
+    va_end(ap);
+    err->err_class = err_class;
+
+    *errp = err;
+}
+
 Error *error_copy(const Error *err)
 {
     Error *err_new;
diff --git a/error.h b/error.h
index 96fc20328..4d52e7369 100644
--- a/error.h
+++ b/error.h
@@ -30,6 +30,21 @@ typedef struct Error Error;
 void error_set(Error **err, ErrorClass err_class, const char *fmt, ...) GCC_FMT_ATTR(3, 4);
 
 /**
+ * Set an indirect pointer to an error given a ErrorClass value and a
+ * printf-style human message, followed by a strerror() string if
+ * @os_error is not zero.
+ */
+void error_set_errno(Error **err, int os_error, ErrorClass err_class, const char *fmt, ...) GCC_FMT_ATTR(4, 5);
+
+/**
+ * Same as error_set(), but sets a generic error
+ */
+#define error_setg(err, fmt, ...) \
+    error_set(err, ERROR_CLASS_GENERIC_ERROR, fmt, ## __VA_ARGS__)
+#define error_setg_errno(err, os_error, fmt, ...) \
+    error_set_errno(err, os_error, ERROR_CLASS_GENERIC_ERROR, fmt, ## __VA_ARGS__)
+
+/**
  * Returns true if an indirect pointer to an error is pointing to a valid
  * error object.
  */
diff --git a/event_notifier-posix.c b/event_notifier-posix.c
new file mode 100644
index 000000000..6f3239a3f
--- /dev/null
+++ b/event_notifier-posix.c
@@ -0,0 +1,120 @@
+/*
+ * event notifier support
+ *
+ * Copyright Red Hat, Inc. 2010
+ *
+ * Authors:
+ *  Michael S. Tsirkin <mst@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#include "qemu-common.h"
+#include "event_notifier.h"
+#include "qemu-char.h"
+
+#ifdef CONFIG_EVENTFD
+#include <sys/eventfd.h>
+#endif
+
+void event_notifier_init_fd(EventNotifier *e, int fd)
+{
+    e->rfd = fd;
+    e->wfd = fd;
+}
+
+int event_notifier_init(EventNotifier *e, int active)
+{
+    int fds[2];
+    int ret;
+
+#ifdef CONFIG_EVENTFD
+    ret = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC);
+#else
+    ret = -1;
+    errno = ENOSYS;
+#endif
+    if (ret >= 0) {
+        e->rfd = e->wfd = ret;
+    } else {
+        if (errno != ENOSYS) {
+            return -errno;
+        }
+        if (qemu_pipe(fds) < 0) {
+            return -errno;
+        }
+        ret = fcntl_setfl(fds[0], O_NONBLOCK);
+        if (ret < 0) {
+            ret = -errno;
+            goto fail;
+        }
+        ret = fcntl_setfl(fds[1], O_NONBLOCK);
+        if (ret < 0) {
+            ret = -errno;
+            goto fail;
+        }
+        e->rfd = fds[0];
+        e->wfd = fds[1];
+    }
+    if (active) {
+        event_notifier_set(e);
+    }
+    return 0;
+
+fail:
+    close(fds[0]);
+    close(fds[1]);
+    return ret;
+}
+
+void event_notifier_cleanup(EventNotifier *e)
+{
+    if (e->rfd != e->wfd) {
+        close(e->rfd);
+    }
+    close(e->wfd);
+}
+
+int event_notifier_get_fd(EventNotifier *e)
+{
+    return e->rfd;
+}
+
+int event_notifier_set_handler(EventNotifier *e,
+                               EventNotifierHandler *handler)
+{
+    return qemu_set_fd_handler(e->rfd, (IOHandler *)handler, NULL, e);
+}
+
+int event_notifier_set(EventNotifier *e)
+{
+    static const uint64_t value = 1;
+    ssize_t ret;
+
+    do {
+        ret = write(e->wfd, &value, sizeof(value));
+    } while (ret < 0 && errno == EINTR);
+
+    /* EAGAIN is fine, a read must be pending.  */
+    if (ret < 0 && errno != EAGAIN) {
+        return -errno;
+    }
+    return 0;
+}
+
+int event_notifier_test_and_clear(EventNotifier *e)
+{
+    int value;
+    ssize_t len;
+    char buffer[512];
+
+    /* Drain the notify pipe.  For eventfd, only 8 bytes will be read.  */
+    value = 0;
+    do {
+        len = read(e->rfd, buffer, sizeof(buffer));
+        value |= (len > 0);
+    } while ((len == -1 && errno == EINTR) || len == sizeof(buffer));
+
+    return value;
+}
diff --git a/event_notifier-win32.c b/event_notifier-win32.c
new file mode 100644
index 000000000..4ed21c2a7
--- /dev/null
+++ b/event_notifier-win32.c
@@ -0,0 +1,59 @@
+/*
+ * event notifier support
+ *
+ * Copyright Red Hat, Inc. 2010
+ *
+ * Authors:
+ *  Michael S. Tsirkin <mst@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#include "qemu-common.h"
+#include "event_notifier.h"
+#include "main-loop.h"
+
+int event_notifier_init(EventNotifier *e, int active)
+{
+    e->event = CreateEvent(NULL, TRUE, FALSE, NULL);
+    assert(e->event);
+    return 0;
+}
+
+void event_notifier_cleanup(EventNotifier *e)
+{
+    CloseHandle(e->event);
+}
+
+HANDLE event_notifier_get_handle(EventNotifier *e)
+{
+    return e->event;
+}
+
+int event_notifier_set_handler(EventNotifier *e,
+                               EventNotifierHandler *handler)
+{
+    if (handler) {
+        return qemu_add_wait_object(e->event, (IOHandler *)handler, e);
+    } else {
+        qemu_del_wait_object(e->event, (IOHandler *)handler, e);
+        return 0;
+    }
+}
+
+int event_notifier_set(EventNotifier *e)
+{
+    SetEvent(e->event);
+    return 0;
+}
+
+int event_notifier_test_and_clear(EventNotifier *e)
+{
+    int ret = WaitForSingleObject(e->event, 0);
+    if (ret == WAIT_OBJECT_0) {
+        ResetEvent(e->event);
+        return true;
+    }
+    return false;
+}
diff --git a/event_notifier.c b/event_notifier.c
deleted file mode 100644
index 2c207e139..000000000
--- a/event_notifier.c
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * event notifier support
- *
- * Copyright Red Hat, Inc. 2010
- *
- * Authors:
- *  Michael S. Tsirkin <mst@redhat.com>
- *
- * This work is licensed under the terms of the GNU GPL, version 2 or later.
- * See the COPYING file in the top-level directory.
- */
-
-#include "qemu-common.h"
-#include "event_notifier.h"
-#include "qemu-char.h"
-
-#ifdef CONFIG_EVENTFD
-#include <sys/eventfd.h>
-#endif
-
-void event_notifier_init_fd(EventNotifier *e, int fd)
-{
-    e->fd = fd;
-}
-
-int event_notifier_init(EventNotifier *e, int active)
-{
-#ifdef CONFIG_EVENTFD
-    int fd = eventfd(!!active, EFD_NONBLOCK | EFD_CLOEXEC);
-    if (fd < 0)
-        return -errno;
-    e->fd = fd;
-    return 0;
-#else
-    return -ENOSYS;
-#endif
-}
-
-void event_notifier_cleanup(EventNotifier *e)
-{
-    close(e->fd);
-}
-
-int event_notifier_get_fd(EventNotifier *e)
-{
-    return e->fd;
-}
-
-int event_notifier_set_handler(EventNotifier *e,
-                               EventNotifierHandler *handler)
-{
-    return qemu_set_fd_handler(e->fd, (IOHandler *)handler, NULL, e);
-}
-
-int event_notifier_set(EventNotifier *e)
-{
-    uint64_t value = 1;
-    int r = write(e->fd, &value, sizeof(value));
-    return r == sizeof(value);
-}
-
-int event_notifier_test_and_clear(EventNotifier *e)
-{
-    uint64_t value;
-    int r = read(e->fd, &value, sizeof(value));
-    return r == sizeof(value);
-}
diff --git a/event_notifier.h b/event_notifier.h
index f0ec2f217..88b57af7c 100644
--- a/event_notifier.h
+++ b/event_notifier.h
@@ -15,18 +15,32 @@
 
 #include "qemu-common.h"
 
+#ifdef _WIN32
+#include <windows.h>
+#endif
+
 struct EventNotifier {
-    int fd;
+#ifdef _WIN32
+    HANDLE event;
+#else
+    int rfd;
+    int wfd;
+#endif
 };
 
 typedef void EventNotifierHandler(EventNotifier *);
 
-void event_notifier_init_fd(EventNotifier *, int fd);
 int event_notifier_init(EventNotifier *, int active);
 void event_notifier_cleanup(EventNotifier *);
-int event_notifier_get_fd(EventNotifier *);
 int event_notifier_set(EventNotifier *);
 int event_notifier_test_and_clear(EventNotifier *);
 int event_notifier_set_handler(EventNotifier *, EventNotifierHandler *);
 
+#ifdef CONFIG_POSIX
+void event_notifier_init_fd(EventNotifier *, int fd);
+int event_notifier_get_fd(EventNotifier *);
+#else
+HANDLE event_notifier_get_handle(EventNotifier *);
+#endif
+
 #endif
diff --git a/exec-all.h b/exec-all.h
index c5ec8e115..21aacdab5 100644
--- a/exec-all.h
+++ b/exec-all.h
@@ -51,7 +51,7 @@ typedef struct TranslationBlock TranslationBlock;
 #else
 #define MAX_OPC_PARAM_PER_ARG 1
 #endif
-#define MAX_OPC_PARAM_IARGS 4
+#define MAX_OPC_PARAM_IARGS 5
 #define MAX_OPC_PARAM_OARGS 1
 #define MAX_OPC_PARAM_ARGS (MAX_OPC_PARAM_IARGS + MAX_OPC_PARAM_OARGS)
 
@@ -103,9 +103,9 @@ void tb_invalidate_phys_range(tb_page_addr_t start, tb_page_addr_t end,
 void tlb_flush_page(CPUArchState *env, target_ulong addr);
 void tlb_flush(CPUArchState *env, int flush_global);
 void tlb_set_page(CPUArchState *env, target_ulong vaddr,
-                  target_phys_addr_t paddr, int prot,
+                  hwaddr paddr, int prot,
                   int mmu_idx, target_ulong size);
-void tb_invalidate_phys_addr(target_phys_addr_t addr);
+void tb_invalidate_phys_addr(hwaddr addr);
 #else
 static inline void tlb_flush_page(CPUArchState *env, target_ulong addr)
 {
@@ -121,8 +121,6 @@ static inline void tlb_flush(CPUArchState *env, int flush_global)
 #define CODE_GEN_PHYS_HASH_BITS     15
 #define CODE_GEN_PHYS_HASH_SIZE     (1 << CODE_GEN_PHYS_HASH_BITS)
 
-#define MIN_CODE_GEN_BUFFER_SIZE     (1024 * 1024)
-
 /* estimated block size for TB allocation */
 /* XXX: use a per code average code fragment size and modulate it
    according to the host CPU */
@@ -132,9 +130,10 @@ static inline void tlb_flush(CPUArchState *env, int flush_global)
 #define CODE_GEN_AVG_BLOCK_SIZE 64
 #endif
 
-#if defined(_ARCH_PPC) || defined(__x86_64__) || defined(__arm__) || defined(__i386__)
-#define USE_DIRECT_JUMP
-#elif defined(CONFIG_TCG_INTERPRETER)
+#if defined(__arm__) || defined(_ARCH_PPC) \
+    || defined(__x86_64__) || defined(__i386__) \
+    || defined(__sparc__) \
+    || defined(CONFIG_TCG_INTERPRETER)
 #define USE_DIRECT_JUMP
 #endif
 
@@ -195,8 +194,6 @@ static inline unsigned int tb_phys_hash_func(tb_page_addr_t pc)
 
 void tb_free(TranslationBlock *tb);
 void tb_flush(CPUArchState *env);
-void tb_link_page(TranslationBlock *tb,
-                  tb_page_addr_t phys_pc, tb_page_addr_t phys_page2);
 void tb_phys_invalidate(TranslationBlock *tb, tb_page_addr_t page_addr);
 
 extern TranslationBlock *tb_phys_hash[CODE_GEN_PHYS_HASH_SIZE];
@@ -244,6 +241,8 @@ static inline void tb_set_jmp_target1(uintptr_t jmp_addr, uintptr_t addr)
     __asm __volatile__ ("swi 0x9f0002" : : "r" (_beg), "r" (_end), "r" (_flg));
 #endif
 }
+#elif defined(__sparc__)
+void tb_set_jmp_target1(uintptr_t jmp_addr, uintptr_t addr);
 #else
 #error tb_set_jmp_target1 is missing
 #endif
@@ -291,9 +290,11 @@ extern int tb_invalidated_flag;
 /* The return address may point to the start of the next instruction.
    Subtracting one gets us the call instruction itself.  */
 #if defined(CONFIG_TCG_INTERPRETER)
-/* Alpha and SH4 user mode emulations and Softmmu call GETPC().
+/* Softmmu, Alpha, MIPS, SH4 and SPARC user mode emulations call GETPC().
    For all others, GETPC remains undefined (which makes TCI a little faster. */
-# if defined(CONFIG_SOFTMMU) || defined(TARGET_ALPHA) || defined(TARGET_SH4)
+# if defined(CONFIG_SOFTMMU) || \
+    defined(TARGET_ALPHA) || defined(TARGET_MIPS) || \
+    defined(TARGET_SH4) || defined(TARGET_SPARC)
 extern uintptr_t tci_tb_ptr;
 #  define GETPC() tci_tb_ptr
 # endif
@@ -308,12 +309,51 @@ extern uintptr_t tci_tb_ptr;
 # define GETPC() ((uintptr_t)__builtin_return_address(0) - 1)
 #endif
 
+#if defined(CONFIG_QEMU_LDST_OPTIMIZATION) && defined(CONFIG_SOFTMMU)
+/* qemu_ld/st optimization split code generation to fast and slow path, thus,
+   it needs special handling for an MMU helper which is called from the slow
+   path, to get the fast path's pc without any additional argument.
+   It uses a tricky solution which embeds the fast path pc into the slow path.
+
+   Code flow in slow path:
+   (1) pre-process
+   (2) call MMU helper
+   (3) jump to (5)
+   (4) fast path information (implementation specific)
+   (5) post-process (e.g. stack adjust)
+   (6) jump to corresponding code of the next of fast path
+ */
+# if defined(__i386__) || defined(__x86_64__)
+/* To avoid broken disassembling, long jmp is used for embedding fast path pc,
+   so that the destination is the next code of fast path, though this jmp is
+   never executed.
+
+   call MMU helper
+   jmp POST_PROC (2byte)    <- GETRA()
+   jmp NEXT_CODE (5byte)
+   POST_PROCESS ...         <- GETRA() + 7
+ */
+#  define GETRA() ((uintptr_t)__builtin_return_address(0))
+#  define GETPC_LDST() ((uintptr_t)(GETRA() + 7 + \
+                                    *(int32_t *)((void *)GETRA() + 3) - 1))
+# elif defined (_ARCH_PPC) && !defined (_ARCH_PPC64)
+#  define GETRA() ((uintptr_t)__builtin_return_address(0))
+#  define GETPC_LDST() ((uintptr_t) ((*(int32_t *)(GETRA() - 4)) - 1))
+# else
+#  error "CONFIG_QEMU_LDST_OPTIMIZATION needs GETPC_LDST() implementation!"
+# endif
+bool is_tcg_gen_code(uintptr_t pc_ptr);
+# define GETPC_EXT() (is_tcg_gen_code(GETRA()) ? GETPC_LDST() : GETPC())
+#else
+# define GETPC_EXT() GETPC()
+#endif
+
 #if !defined(CONFIG_USER_ONLY)
 
-struct MemoryRegion *iotlb_to_region(target_phys_addr_t index);
-uint64_t io_mem_read(struct MemoryRegion *mr, target_phys_addr_t addr,
+struct MemoryRegion *iotlb_to_region(hwaddr index);
+uint64_t io_mem_read(struct MemoryRegion *mr, hwaddr addr,
                      unsigned size);
-void io_mem_write(struct MemoryRegion *mr, target_phys_addr_t addr,
+void io_mem_write(struct MemoryRegion *mr, hwaddr addr,
                   uint64_t value, unsigned size);
 
 void tlb_fill(CPUArchState *env1, target_ulong addr, int is_write, int mmu_idx,
@@ -323,9 +363,6 @@ void tlb_fill(CPUArchState *env1, target_ulong addr, int is_write, int mmu_idx,
 
 #define ACCESS_TYPE (NB_MMU_MODES + 1)
 #define MEMSUFFIX _code
-#ifndef CONFIG_TCG_PASS_AREG0
-#define env cpu_single_env
-#endif
 
 #define DATA_SIZE 1
 #include "softmmu_header.h"
@@ -341,7 +378,6 @@ void tlb_fill(CPUArchState *env1, target_ulong addr, int is_write, int mmu_idx,
 
 #undef ACCESS_TYPE
 #undef MEMSUFFIX
-#undef env
 
 #endif
 
diff --git a/exec-memory.h b/exec-memory.h
index 1cd92eec7..ac1d07dfe 100644
--- a/exec-memory.h
+++ b/exec-memory.h
@@ -33,11 +33,8 @@ MemoryRegion *get_system_memory(void);
  */
 MemoryRegion *get_system_io(void);
 
-/* Set the root memory region.  This region is the system memory map. */
-void set_system_memory_map(MemoryRegion *mr);
-
-/* Set the I/O memory region.  This region is the I/O memory map. */
-void set_system_io_map(MemoryRegion *mr);
+extern AddressSpace address_space_memory;
+extern AddressSpace address_space_io;
 
 #endif
 
diff --git a/exec.c b/exec.c
index 58347664b..8435de0bd 100644
--- a/exec.c
+++ b/exec.c
@@ -34,6 +34,7 @@
 #include "hw/xen.h"
 #include "qemu-timer.h"
 #include "memory.h"
+#include "dma.h"
 #include "exec-memory.h"
 #if defined(CONFIG_USER_ONLY)
 #include <qemu.h>
@@ -59,8 +60,7 @@
 
 #include "cputlb.h"
 
-#define WANT_EXEC_OBSOLETE
-#include "exec-obsolete.h"
+#include "memory-internal.h"
 
 //#define DEBUG_TB_INVALIDATE
 //#define DEBUG_FLUSH
@@ -86,26 +86,11 @@ static int nb_tbs;
 /* any access to the tbs or the page table must use this lock */
 spinlock_t tb_lock = SPIN_LOCK_UNLOCKED;
 
-#if defined(__arm__) || defined(__sparc_v9__)
-/* The prologue must be reachable with a direct jump. ARM and Sparc64
- have limited branch ranges (possibly also PPC) so place it in a
- section close to code segment. */
-#define code_gen_section                                \
-    __attribute__((__section__(".gen_code")))           \
-    __attribute__((aligned (32)))
-#elif defined(_WIN32) && !defined(_WIN64)
-#define code_gen_section                                \
-    __attribute__((aligned (16)))
-#else
-#define code_gen_section                                \
-    __attribute__((aligned (32)))
-#endif
-
-uint8_t code_gen_prologue[1024] code_gen_section;
+uint8_t *code_gen_prologue;
 static uint8_t *code_gen_buffer;
-static unsigned long code_gen_buffer_size;
+static size_t code_gen_buffer_size;
 /* threshold to flush the translated code buffer */
-static unsigned long code_gen_buffer_max_size;
+static size_t code_gen_buffer_max_size;
 static uint8_t *code_gen_ptr;
 
 #if !defined(CONFIG_USER_ONLY)
@@ -117,6 +102,10 @@ RAMList ram_list = { .blocks = QLIST_HEAD_INITIALIZER(ram_list.blocks) };
 static MemoryRegion *system_memory;
 static MemoryRegion *system_io;
 
+AddressSpace address_space_io;
+AddressSpace address_space_memory;
+DMAContext dma_context_memory;
+
 MemoryRegion io_mem_ram, io_mem_rom, io_mem_unassigned, io_mem_notdirty;
 static MemoryRegion io_mem_subpage_ram;
 
@@ -185,7 +174,6 @@ uintptr_t qemu_host_page_mask;
 static void *l1_map[V_L1_SIZE];
 
 #if !defined(CONFIG_USER_ONLY)
-typedef struct PhysPageEntry PhysPageEntry;
 
 static MemoryRegionSection *phys_sections;
 static unsigned phys_sections_nb, phys_sections_nb_alloc;
@@ -194,34 +182,27 @@ static uint16_t phys_section_notdirty;
 static uint16_t phys_section_rom;
 static uint16_t phys_section_watch;
 
-struct PhysPageEntry {
-    uint16_t is_leaf : 1;
-     /* index into phys_sections (is_leaf) or phys_map_nodes (!is_leaf) */
-    uint16_t ptr : 15;
-};
-
 /* Simple allocator for PhysPageEntry nodes */
 static PhysPageEntry (*phys_map_nodes)[L2_SIZE];
 static unsigned phys_map_nodes_nb, phys_map_nodes_nb_alloc;
 
 #define PHYS_MAP_NODE_NIL (((uint16_t)~0) >> 1)
 
-/* This is a multi-level map on the physical address space.
-   The bottom level has pointers to MemoryRegionSections.  */
-static PhysPageEntry phys_map = { .ptr = PHYS_MAP_NODE_NIL, .is_leaf = 0 };
-
 static void io_mem_init(void);
 static void memory_map_init(void);
+static void *qemu_safe_ram_ptr(ram_addr_t addr);
 
 static MemoryRegion io_mem_watch;
 #endif
+static void tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc,
+                         tb_page_addr_t phys_page2);
 
 /* statistics */
 static int tb_flush_count;
 static int tb_phys_invalidate_count;
 
 #ifdef _WIN32
-static void map_exec(void *addr, long size)
+static inline void map_exec(void *addr, long size)
 {
     DWORD old_protect;
     VirtualProtect(addr, size,
@@ -229,7 +210,7 @@ static void map_exec(void *addr, long size)
     
 }
 #else
-static void map_exec(void *addr, long size)
+static inline void map_exec(void *addr, long size)
 {
     unsigned long start, end, page_size;
     
@@ -422,13 +403,13 @@ static void phys_map_nodes_reset(void)
 }
 
 
-static void phys_page_set_level(PhysPageEntry *lp, target_phys_addr_t *index,
-                                target_phys_addr_t *nb, uint16_t leaf,
+static void phys_page_set_level(PhysPageEntry *lp, hwaddr *index,
+                                hwaddr *nb, uint16_t leaf,
                                 int level)
 {
     PhysPageEntry *p;
     int i;
-    target_phys_addr_t step = (target_phys_addr_t)1 << (level * L2_BITS);
+    hwaddr step = (hwaddr)1 << (level * L2_BITS);
 
     if (!lp->is_leaf && lp->ptr == PHYS_MAP_NODE_NIL) {
         lp->ptr = phys_map_node_alloc();
@@ -457,18 +438,19 @@ static void phys_page_set_level(PhysPageEntry *lp, target_phys_addr_t *index,
     }
 }
 
-static void phys_page_set(target_phys_addr_t index, target_phys_addr_t nb,
+static void phys_page_set(AddressSpaceDispatch *d,
+                          hwaddr index, hwaddr nb,
                           uint16_t leaf)
 {
     /* Wildly overreserve - it doesn't matter much. */
     phys_map_node_reserve(3 * P_L2_LEVELS);
 
-    phys_page_set_level(&phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);
+    phys_page_set_level(&d->phys_map, &index, &nb, leaf, P_L2_LEVELS - 1);
 }
 
-MemoryRegionSection *phys_page_find(target_phys_addr_t index)
+MemoryRegionSection *phys_page_find(AddressSpaceDispatch *d, hwaddr index)
 {
-    PhysPageEntry lp = phys_map;
+    PhysPageEntry lp = d->phys_map;
     PhysPageEntry *p;
     int i;
     uint16_t s_index = phys_section_unassigned;
@@ -497,113 +479,142 @@ bool memory_region_is_unassigned(MemoryRegion *mr)
 #define mmap_unlock() do { } while(0)
 #endif
 
-#define DEFAULT_CODE_GEN_BUFFER_SIZE (32 * 1024 * 1024)
-
 #if defined(CONFIG_USER_ONLY)
 /* Currently it is not recommended to allocate big chunks of data in
-   user mode. It will change when a dedicated libc will be used */
+   user mode. It will change when a dedicated libc will be used.  */
+/* ??? 64-bit hosts ought to have no problem mmaping data outside the
+   region in which the guest needs to run.  Revisit this.  */
 #define USE_STATIC_CODE_GEN_BUFFER
 #endif
 
-#ifdef USE_STATIC_CODE_GEN_BUFFER
-static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE]
-               __attribute__((aligned (CODE_GEN_ALIGN)));
+/* ??? Should configure for this, not list operating systems here.  */
+#if (defined(__linux__) \
+    || defined(__FreeBSD__) || defined(__FreeBSD_kernel__) \
+    || defined(__DragonFly__) || defined(__OpenBSD__) \
+    || defined(__NetBSD__))
+# define USE_MMAP
 #endif
 
-static void code_gen_alloc(unsigned long tb_size)
-{
-#ifdef USE_STATIC_CODE_GEN_BUFFER
-    code_gen_buffer = static_code_gen_buffer;
-    code_gen_buffer_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
-    map_exec(code_gen_buffer, code_gen_buffer_size);
-#else
-    code_gen_buffer_size = tb_size;
-    if (code_gen_buffer_size == 0) {
-#if defined(CONFIG_USER_ONLY)
-        code_gen_buffer_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
-#else
-        /* XXX: needs adjustments */
-        code_gen_buffer_size = (unsigned long)(ram_size / 4);
-#endif
-    }
-    if (code_gen_buffer_size < MIN_CODE_GEN_BUFFER_SIZE)
-        code_gen_buffer_size = MIN_CODE_GEN_BUFFER_SIZE;
-    /* The code gen buffer location may have constraints depending on
-       the host cpu and OS */
-#if defined(__linux__) 
-    {
-        int flags;
-        void *start = NULL;
+/* Minimum size of the code gen buffer.  This number is randomly chosen,
+   but not so small that we can't have a fair number of TB's live.  */
+#define MIN_CODE_GEN_BUFFER_SIZE     (1024u * 1024)
 
-        flags = MAP_PRIVATE | MAP_ANONYMOUS;
+/* Maximum size of the code gen buffer we'd like to use.  Unless otherwise
+   indicated, this is constrained by the range of direct branches on the
+   host cpu, as used by the TCG implementation of goto_tb.  */
 #if defined(__x86_64__)
-        flags |= MAP_32BIT;
-        /* Cannot map more than that */
-        if (code_gen_buffer_size > (800 * 1024 * 1024))
-            code_gen_buffer_size = (800 * 1024 * 1024);
-#elif defined(__sparc_v9__)
-        // Map the buffer below 2G, so we can use direct calls and branches
-        flags |= MAP_FIXED;
-        start = (void *) 0x60000000UL;
-        if (code_gen_buffer_size > (512 * 1024 * 1024))
-            code_gen_buffer_size = (512 * 1024 * 1024);
+# define MAX_CODE_GEN_BUFFER_SIZE  (2ul * 1024 * 1024 * 1024)
+#elif defined(__sparc__)
+# define MAX_CODE_GEN_BUFFER_SIZE  (2ul * 1024 * 1024 * 1024)
 #elif defined(__arm__)
-        /* Keep the buffer no bigger than 16MB to branch between blocks */
-        if (code_gen_buffer_size > 16 * 1024 * 1024)
-            code_gen_buffer_size = 16 * 1024 * 1024;
+# define MAX_CODE_GEN_BUFFER_SIZE  (16u * 1024 * 1024)
 #elif defined(__s390x__)
-        /* Map the buffer so that we can use direct calls and branches.  */
-        /* We have a +- 4GB range on the branches; leave some slop.  */
-        if (code_gen_buffer_size > (3ul * 1024 * 1024 * 1024)) {
-            code_gen_buffer_size = 3ul * 1024 * 1024 * 1024;
-        }
-        start = (void *)0x90000000UL;
+  /* We have a +- 4GB range on the branches; leave some slop.  */
+# define MAX_CODE_GEN_BUFFER_SIZE  (3ul * 1024 * 1024 * 1024)
+#else
+# define MAX_CODE_GEN_BUFFER_SIZE  ((size_t)-1)
 #endif
-        code_gen_buffer = mmap(start, code_gen_buffer_size,
-                               PROT_WRITE | PROT_READ | PROT_EXEC,
-                               flags, -1, 0);
-        if (code_gen_buffer == MAP_FAILED) {
-            fprintf(stderr, "Could not allocate dynamic translator buffer\n");
-            exit(1);
-        }
-    }
-#elif defined(__FreeBSD__) || defined(__FreeBSD_kernel__) \
-    || defined(__DragonFly__) || defined(__OpenBSD__) \
-    || defined(__NetBSD__)
-    {
-        int flags;
-        void *addr = NULL;
-        flags = MAP_PRIVATE | MAP_ANONYMOUS;
-#if defined(__x86_64__)
-        /* FreeBSD doesn't have MAP_32BIT, use MAP_FIXED and assume
-         * 0x40000000 is free */
-        flags |= MAP_FIXED;
-        addr = (void *)0x40000000;
-        /* Cannot map more than that */
-        if (code_gen_buffer_size > (800 * 1024 * 1024))
-            code_gen_buffer_size = (800 * 1024 * 1024);
-#elif defined(__sparc_v9__)
-        // Map the buffer below 2G, so we can use direct calls and branches
-        flags |= MAP_FIXED;
-        addr = (void *) 0x60000000UL;
-        if (code_gen_buffer_size > (512 * 1024 * 1024)) {
-            code_gen_buffer_size = (512 * 1024 * 1024);
-        }
+
+#define DEFAULT_CODE_GEN_BUFFER_SIZE_1 (32u * 1024 * 1024)
+
+#define DEFAULT_CODE_GEN_BUFFER_SIZE \
+  (DEFAULT_CODE_GEN_BUFFER_SIZE_1 < MAX_CODE_GEN_BUFFER_SIZE \
+   ? DEFAULT_CODE_GEN_BUFFER_SIZE_1 : MAX_CODE_GEN_BUFFER_SIZE)
+
+static inline size_t size_code_gen_buffer(size_t tb_size)
+{
+    /* Size the buffer.  */
+    if (tb_size == 0) {
+#ifdef USE_STATIC_CODE_GEN_BUFFER
+        tb_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
+#else
+        /* ??? Needs adjustments.  */
+        /* ??? If we relax the requirement that CONFIG_USER_ONLY use the
+           static buffer, we could size this on RESERVED_VA, on the text
+           segment size of the executable, or continue to use the default.  */
+        tb_size = (unsigned long)(ram_size / 4);
 #endif
-        code_gen_buffer = mmap(addr, code_gen_buffer_size,
-                               PROT_WRITE | PROT_READ | PROT_EXEC, 
-                               flags, -1, 0);
-        if (code_gen_buffer == MAP_FAILED) {
-            fprintf(stderr, "Could not allocate dynamic translator buffer\n");
-            exit(1);
-        }
     }
+    if (tb_size < MIN_CODE_GEN_BUFFER_SIZE) {
+        tb_size = MIN_CODE_GEN_BUFFER_SIZE;
+    }
+    if (tb_size > MAX_CODE_GEN_BUFFER_SIZE) {
+        tb_size = MAX_CODE_GEN_BUFFER_SIZE;
+    }
+    code_gen_buffer_size = tb_size;
+    return tb_size;
+}
+
+#ifdef USE_STATIC_CODE_GEN_BUFFER
+static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE]
+    __attribute__((aligned(CODE_GEN_ALIGN)));
+
+static inline void *alloc_code_gen_buffer(void)
+{
+    map_exec(static_code_gen_buffer, code_gen_buffer_size);
+    return static_code_gen_buffer;
+}
+#elif defined(USE_MMAP)
+static inline void *alloc_code_gen_buffer(void)
+{
+    int flags = MAP_PRIVATE | MAP_ANONYMOUS;
+    uintptr_t start = 0;
+    void *buf;
+
+    /* Constrain the position of the buffer based on the host cpu.
+       Note that these addresses are chosen in concert with the
+       addresses assigned in the relevant linker script file.  */
+# if defined(__PIE__) || defined(__PIC__)
+    /* Don't bother setting a preferred location if we're building
+       a position-independent executable.  We're more likely to get
+       an address near the main executable if we let the kernel
+       choose the address.  */
+# elif defined(__x86_64__) && defined(MAP_32BIT)
+    /* Force the memory down into low memory with the executable.
+       Leave the choice of exact location with the kernel.  */
+    flags |= MAP_32BIT;
+    /* Cannot expect to map more than 800MB in low memory.  */
+    if (code_gen_buffer_size > 800u * 1024 * 1024) {
+        code_gen_buffer_size = 800u * 1024 * 1024;
+    }
+# elif defined(__sparc__)
+    start = 0x40000000ul;
+# elif defined(__s390x__)
+    start = 0x90000000ul;
+# endif
+
+    buf = mmap((void *)start, code_gen_buffer_size,
+               PROT_WRITE | PROT_READ | PROT_EXEC, flags, -1, 0);
+    return buf == MAP_FAILED ? NULL : buf;
+}
 #else
-    code_gen_buffer = g_malloc(code_gen_buffer_size);
-    map_exec(code_gen_buffer, code_gen_buffer_size);
-#endif
-#endif /* !USE_STATIC_CODE_GEN_BUFFER */
-    map_exec(code_gen_prologue, sizeof(code_gen_prologue));
+static inline void *alloc_code_gen_buffer(void)
+{
+    void *buf = g_malloc(code_gen_buffer_size);
+    if (buf) {
+        map_exec(buf, code_gen_buffer_size);
+    }
+    return buf;
+}
+#endif /* USE_STATIC_CODE_GEN_BUFFER, USE_MMAP */
+
+static inline void code_gen_alloc(size_t tb_size)
+{
+    code_gen_buffer_size = size_code_gen_buffer(tb_size);
+    code_gen_buffer = alloc_code_gen_buffer();
+    if (code_gen_buffer == NULL) {
+        fprintf(stderr, "Could not allocate dynamic translator buffer\n");
+        exit(1);
+    }
+
+    /* Steal room for the prologue at the end of the buffer.  This ensures
+       (via the MAX_CODE_GEN_BUFFER_SIZE limits above) that direct branches
+       from TB's to the prologue are going to be in range.  It also means
+       that we don't need to mark (additional) portions of the data segment
+       as executable.  */
+    code_gen_prologue = code_gen_buffer + code_gen_buffer_size - 1024;
+    code_gen_buffer_size -= 1024;
+
     code_gen_buffer_max_size = code_gen_buffer_size -
         (TCG_MAX_OP_SIZE * OPC_BUF_SIZE);
     code_gen_max_blocks = code_gen_buffer_size / CODE_GEN_AVG_BLOCK_SIZE;
@@ -683,6 +694,9 @@ CPUArchState *qemu_get_cpu(int cpu)
 
 void cpu_exec_init(CPUArchState *env)
 {
+#ifndef CONFIG_USER_ONLY
+    CPUState *cpu = ENV_GET_CPU(env);
+#endif
     CPUArchState **penv;
     int cpu_index;
 
@@ -701,7 +715,7 @@ void cpu_exec_init(CPUArchState *env)
     QTAILQ_INIT(&env->breakpoints);
     QTAILQ_INIT(&env->watchpoints);
 #ifndef CONFIG_USER_ONLY
-    env->thread_id = qemu_get_thread_id();
+    cpu->thread_id = qemu_get_thread_id();
 #endif
     *penv = env;
 #if defined(CONFIG_USER_ONLY)
@@ -1343,8 +1357,8 @@ static inline void tb_alloc_page(TranslationBlock *tb,
 
 /* add a new TB and link it to the physical page tables. phys_page2 is
    (-1) to indicate that only one page contains the TB. */
-void tb_link_page(TranslationBlock *tb,
-                  tb_page_addr_t phys_pc, tb_page_addr_t phys_page2)
+static void tb_link_page(TranslationBlock *tb, tb_page_addr_t phys_pc,
+                         tb_page_addr_t phys_page2)
 {
     unsigned int h;
     TranslationBlock **ptb;
@@ -1381,6 +1395,17 @@ void tb_link_page(TranslationBlock *tb,
     mmap_unlock();
 }
 
+#if defined(CONFIG_QEMU_LDST_OPTIMIZATION) && defined(CONFIG_SOFTMMU)
+/* check whether the given addr is in TCG generated code buffer or not */
+bool is_tcg_gen_code(uintptr_t tc_ptr)
+{
+    /* This can be called during code generation, code_gen_buffer_max_size
+       is used instead of code_gen_ptr for upper boundary checking */
+    return (tc_ptr >= (uintptr_t)code_gen_buffer &&
+            tc_ptr < (uintptr_t)(code_gen_buffer + code_gen_buffer_max_size));
+}
+#endif
+
 /* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
    tb[1].tc_ptr. Return NULL if not found */
 TranslationBlock *tb_find_pc(uintptr_t tc_ptr)
@@ -1467,12 +1492,12 @@ static void breakpoint_invalidate(CPUArchState *env, target_ulong pc)
     tb_invalidate_phys_page_range(pc, pc + 1, 0);
 }
 #else
-void tb_invalidate_phys_addr(target_phys_addr_t addr)
+void tb_invalidate_phys_addr(hwaddr addr)
 {
     ram_addr_t ram_addr;
     MemoryRegionSection *section;
 
-    section = phys_page_find(addr >> TARGET_PAGE_BITS);
+    section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
     if (!(memory_region_is_ram(section->mr)
           || (section->mr->rom_device && section->mr->readable))) {
         return;
@@ -1687,6 +1712,7 @@ static void cpu_unlink_tb(CPUArchState *env)
 /* mask must never be zero, except for A20 change call */
 static void tcg_handle_interrupt(CPUArchState *env, int mask)
 {
+    CPUState *cpu = ENV_GET_CPU(env);
     int old_mask;
 
     old_mask = env->interrupt_request;
@@ -1696,8 +1722,8 @@ static void tcg_handle_interrupt(CPUArchState *env, int mask)
      * If called from iothread context, wake the target cpu in
      * case its halted.
      */
-    if (!qemu_cpu_is_self(env)) {
-        qemu_cpu_kick(env);
+    if (!qemu_cpu_is_self(cpu)) {
+        qemu_cpu_kick(cpu);
         return;
     }
 
@@ -1744,20 +1770,12 @@ void cpu_abort(CPUArchState *env, const char *fmt, ...)
     fprintf(stderr, "qemu: fatal: ");
     vfprintf(stderr, fmt, ap);
     fprintf(stderr, "\n");
-#ifdef TARGET_I386
-    cpu_dump_state(env, stderr, fprintf, X86_DUMP_FPU | X86_DUMP_CCOP);
-#else
-    cpu_dump_state(env, stderr, fprintf, 0);
-#endif
+    cpu_dump_state(env, stderr, fprintf, CPU_DUMP_FPU | CPU_DUMP_CCOP);
     if (qemu_log_enabled()) {
         qemu_log("qemu: fatal: ");
         qemu_log_vprintf(fmt, ap2);
         qemu_log("\n");
-#ifdef TARGET_I386
-        log_cpu_state(env, X86_DUMP_FPU | X86_DUMP_CCOP);
-#else
-        log_cpu_state(env, 0);
-#endif
+        log_cpu_state(env, CPU_DUMP_FPU | CPU_DUMP_CCOP);
         qemu_log_flush();
         qemu_log_close();
     }
@@ -1861,21 +1879,21 @@ void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
     }
 }
 
-int cpu_physical_memory_set_dirty_tracking(int enable)
+static int cpu_physical_memory_set_dirty_tracking(int enable)
 {
     int ret = 0;
     in_migration = enable;
     return ret;
 }
 
-target_phys_addr_t memory_region_section_get_iotlb(CPUArchState *env,
+hwaddr memory_region_section_get_iotlb(CPUArchState *env,
                                                    MemoryRegionSection *section,
                                                    target_ulong vaddr,
-                                                   target_phys_addr_t paddr,
+                                                   hwaddr paddr,
                                                    int prot,
                                                    target_ulong *address)
 {
-    target_phys_addr_t iotlb;
+    hwaddr iotlb;
     CPUWatchpoint *wp;
 
     if (memory_region_is_ram(section->mr)) {
@@ -2178,13 +2196,13 @@ int page_unprotect(target_ulong address, uintptr_t pc, void *puc)
 #define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
 typedef struct subpage_t {
     MemoryRegion iomem;
-    target_phys_addr_t base;
+    hwaddr base;
     uint16_t sub_section[TARGET_PAGE_SIZE];
 } subpage_t;
 
 static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
                              uint16_t section);
-static subpage_t *subpage_init(target_phys_addr_t base);
+static subpage_t *subpage_init(hwaddr base);
 static void destroy_page_desc(uint16_t section_index)
 {
     MemoryRegionSection *section = &phys_sections[section_index];
@@ -2218,9 +2236,9 @@ static void destroy_l2_mapping(PhysPageEntry *lp, unsigned level)
     lp->ptr = PHYS_MAP_NODE_NIL;
 }
 
-static void destroy_all_mappings(void)
+static void destroy_all_mappings(AddressSpaceDispatch *d)
 {
-    destroy_l2_mapping(&phys_map, P_L2_LEVELS - 1);
+    destroy_l2_mapping(&d->phys_map, P_L2_LEVELS - 1);
     phys_map_nodes_reset();
 }
 
@@ -2240,24 +2258,24 @@ static void phys_sections_clear(void)
     phys_sections_nb = 0;
 }
 
-static void register_subpage(MemoryRegionSection *section)
+static void register_subpage(AddressSpaceDispatch *d, MemoryRegionSection *section)
 {
     subpage_t *subpage;
-    target_phys_addr_t base = section->offset_within_address_space
+    hwaddr base = section->offset_within_address_space
         & TARGET_PAGE_MASK;
-    MemoryRegionSection *existing = phys_page_find(base >> TARGET_PAGE_BITS);
+    MemoryRegionSection *existing = phys_page_find(d, base >> TARGET_PAGE_BITS);
     MemoryRegionSection subsection = {
         .offset_within_address_space = base,
         .size = TARGET_PAGE_SIZE,
     };
-    target_phys_addr_t start, end;
+    hwaddr start, end;
 
     assert(existing->mr->subpage || existing->mr == &io_mem_unassigned);
 
     if (!(existing->mr->subpage)) {
         subpage = subpage_init(base);
         subsection.mr = &subpage->iomem;
-        phys_page_set(base >> TARGET_PAGE_BITS, 1,
+        phys_page_set(d, base >> TARGET_PAGE_BITS, 1,
                       phys_section_add(&subsection));
     } else {
         subpage = container_of(existing->mr, subpage_t, iomem);
@@ -2268,23 +2286,23 @@ static void register_subpage(MemoryRegionSection *section)
 }
 
 
-static void register_multipage(MemoryRegionSection *section)
+static void register_multipage(AddressSpaceDispatch *d, MemoryRegionSection *section)
 {
-    target_phys_addr_t start_addr = section->offset_within_address_space;
+    hwaddr start_addr = section->offset_within_address_space;
     ram_addr_t size = section->size;
-    target_phys_addr_t addr;
+    hwaddr addr;
     uint16_t section_index = phys_section_add(section);
 
     assert(size);
 
     addr = start_addr;
-    phys_page_set(addr >> TARGET_PAGE_BITS, size >> TARGET_PAGE_BITS,
+    phys_page_set(d, addr >> TARGET_PAGE_BITS, size >> TARGET_PAGE_BITS,
                   section_index);
 }
 
-void cpu_register_physical_memory_log(MemoryRegionSection *section,
-                                      bool readonly)
+static void mem_add(MemoryListener *listener, MemoryRegionSection *section)
 {
+    AddressSpaceDispatch *d = container_of(listener, AddressSpaceDispatch, listener);
     MemoryRegionSection now = *section, remain = *section;
 
     if ((now.offset_within_address_space & ~TARGET_PAGE_MASK)
@@ -2292,7 +2310,7 @@ void cpu_register_physical_memory_log(MemoryRegionSection *section,
         now.size = MIN(TARGET_PAGE_ALIGN(now.offset_within_address_space)
                        - now.offset_within_address_space,
                        now.size);
-        register_subpage(&now);
+        register_subpage(d, &now);
         remain.size -= now.size;
         remain.offset_within_address_space += now.size;
         remain.offset_within_region += now.size;
@@ -2301,10 +2319,10 @@ void cpu_register_physical_memory_log(MemoryRegionSection *section,
         now = remain;
         if (remain.offset_within_region & ~TARGET_PAGE_MASK) {
             now.size = TARGET_PAGE_SIZE;
-            register_subpage(&now);
+            register_subpage(d, &now);
         } else {
             now.size &= TARGET_PAGE_MASK;
-            register_multipage(&now);
+            register_multipage(d, &now);
         }
         remain.size -= now.size;
         remain.offset_within_address_space += now.size;
@@ -2312,23 +2330,10 @@ void cpu_register_physical_memory_log(MemoryRegionSection *section,
     }
     now = remain;
     if (now.size) {
-        register_subpage(&now);
+        register_subpage(d, &now);
     }
 }
 
-
-void qemu_register_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size)
-{
-    if (kvm_enabled())
-        kvm_coalesce_mmio_region(addr, size);
-}
-
-void qemu_unregister_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size)
-{
-    if (kvm_enabled())
-        kvm_uncoalesce_mmio_region(addr, size);
-}
-
 void qemu_flush_coalesced_mmio_buffer(void)
 {
     if (kvm_enabled())
@@ -2464,7 +2469,7 @@ static ram_addr_t find_ram_offset(ram_addr_t size)
     return offset;
 }
 
-static ram_addr_t last_ram_offset(void)
+ram_addr_t last_ram_offset(void)
 {
     RAMBlock *block;
     ram_addr_t last = 0;
@@ -2525,6 +2530,19 @@ void qemu_ram_set_idstr(ram_addr_t addr, const char *name, DeviceState *dev)
     }
 }
 
+static int memory_try_enable_merging(void *addr, size_t len)
+{
+    QemuOpts *opts;
+
+    opts = qemu_opts_find(qemu_find_opts("machine"), 0);
+    if (opts && !qemu_opt_get_bool(opts, "mem-merge", true)) {
+        /* disabled by the user */
+        return 0;
+    }
+
+    return qemu_madvise(addr, len, QEMU_MADV_MERGEABLE);
+}
+
 ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
                                    MemoryRegion *mr)
 {
@@ -2544,7 +2562,7 @@ ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
             new_block->host = file_ram_alloc(new_block, size, mem_path);
             if (!new_block->host) {
                 new_block->host = qemu_vmalloc(size);
-                qemu_madvise(new_block->host, size, QEMU_MADV_MERGEABLE);
+                memory_try_enable_merging(new_block->host, size);
             }
 #else
             fprintf(stderr, "-mem-path option unsupported\n");
@@ -2559,7 +2577,7 @@ ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
             } else {
                 new_block->host = qemu_vmalloc(size);
             }
-            qemu_madvise(new_block->host, size, QEMU_MADV_MERGEABLE);
+            memory_try_enable_merging(new_block->host, size);
         }
     }
     new_block->length = size;
@@ -2573,6 +2591,7 @@ ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
     cpu_physical_memory_set_dirty_range(new_block->offset, size, 0xff);
 
     qemu_ram_setup_dump(new_block->host, size);
+    qemu_madvise(new_block->host, size, QEMU_MADV_HUGEPAGE);
 
     if (kvm_enabled())
         kvm_setup_guest_memory(new_block->host, size);
@@ -2689,7 +2708,7 @@ void qemu_ram_remap(ram_addr_t addr, ram_addr_t length)
                             length, addr);
                     exit(1);
                 }
-                qemu_madvise(vaddr, length, QEMU_MADV_MERGEABLE);
+                memory_try_enable_merging(vaddr, length);
                 qemu_ram_setup_dump(vaddr, length);
             }
             return;
@@ -2742,7 +2761,7 @@ void *qemu_get_ram_ptr(ram_addr_t addr)
 /* Return a host pointer to ram allocated with qemu_ram_alloc.
  * Same as qemu_get_ram_ptr but avoid reordering ramblocks.
  */
-void *qemu_safe_ram_ptr(ram_addr_t addr)
+static void *qemu_safe_ram_ptr(ram_addr_t addr)
 {
     RAMBlock *block;
 
@@ -2772,7 +2791,7 @@ void *qemu_safe_ram_ptr(ram_addr_t addr)
 
 /* Return a host pointer to guest's ram. Similar to qemu_get_ram_ptr
  * but takes a size argument */
-void *qemu_ram_ptr_length(ram_addr_t addr, ram_addr_t *size)
+static void *qemu_ram_ptr_length(ram_addr_t addr, ram_addr_t *size)
 {
     if (*size == 0) {
         return NULL;
@@ -2837,7 +2856,7 @@ ram_addr_t qemu_ram_addr_from_host_nofail(void *ptr)
     return ram_addr;
 }
 
-static uint64_t unassigned_mem_read(void *opaque, target_phys_addr_t addr,
+static uint64_t unassigned_mem_read(void *opaque, hwaddr addr,
                                     unsigned size)
 {
 #ifdef DEBUG_UNASSIGNED
@@ -2849,7 +2868,7 @@ static uint64_t unassigned_mem_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void unassigned_mem_write(void *opaque, target_phys_addr_t addr,
+static void unassigned_mem_write(void *opaque, hwaddr addr,
                                  uint64_t val, unsigned size)
 {
 #ifdef DEBUG_UNASSIGNED
@@ -2866,13 +2885,13 @@ static const MemoryRegionOps unassigned_mem_ops = {
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-static uint64_t error_mem_read(void *opaque, target_phys_addr_t addr,
+static uint64_t error_mem_read(void *opaque, hwaddr addr,
                                unsigned size)
 {
     abort();
 }
 
-static void error_mem_write(void *opaque, target_phys_addr_t addr,
+static void error_mem_write(void *opaque, hwaddr addr,
                             uint64_t value, unsigned size)
 {
     abort();
@@ -2890,7 +2909,7 @@ static const MemoryRegionOps rom_mem_ops = {
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-static void notdirty_mem_write(void *opaque, target_phys_addr_t ram_addr,
+static void notdirty_mem_write(void *opaque, hwaddr ram_addr,
                                uint64_t val, unsigned size)
 {
     int dirty_flags;
@@ -2977,7 +2996,7 @@ static void check_watchpoint(int offset, int len_mask, int flags)
 /* Watchpoint access routines.  Watchpoints are inserted using TLB tricks,
    so these check for a hit then pass through to the normal out-of-line
    phys routines.  */
-static uint64_t watch_mem_read(void *opaque, target_phys_addr_t addr,
+static uint64_t watch_mem_read(void *opaque, hwaddr addr,
                                unsigned size)
 {
     check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_READ);
@@ -2989,7 +3008,7 @@ static uint64_t watch_mem_read(void *opaque, target_phys_addr_t addr,
     }
 }
 
-static void watch_mem_write(void *opaque, target_phys_addr_t addr,
+static void watch_mem_write(void *opaque, hwaddr addr,
                             uint64_t val, unsigned size)
 {
     check_watchpoint(addr & ~TARGET_PAGE_MASK, ~(size - 1), BP_MEM_WRITE);
@@ -3013,7 +3032,7 @@ static const MemoryRegionOps watch_mem_ops = {
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-static uint64_t subpage_read(void *opaque, target_phys_addr_t addr,
+static uint64_t subpage_read(void *opaque, hwaddr addr,
                              unsigned len)
 {
     subpage_t *mmio = opaque;
@@ -3031,7 +3050,7 @@ static uint64_t subpage_read(void *opaque, target_phys_addr_t addr,
     return io_mem_read(section->mr, addr, len);
 }
 
-static void subpage_write(void *opaque, target_phys_addr_t addr,
+static void subpage_write(void *opaque, hwaddr addr,
                           uint64_t value, unsigned len)
 {
     subpage_t *mmio = opaque;
@@ -3056,7 +3075,7 @@ static const MemoryRegionOps subpage_ops = {
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-static uint64_t subpage_ram_read(void *opaque, target_phys_addr_t addr,
+static uint64_t subpage_ram_read(void *opaque, hwaddr addr,
                                  unsigned size)
 {
     ram_addr_t raddr = addr;
@@ -3069,7 +3088,7 @@ static uint64_t subpage_ram_read(void *opaque, target_phys_addr_t addr,
     }
 }
 
-static void subpage_ram_write(void *opaque, target_phys_addr_t addr,
+static void subpage_ram_write(void *opaque, hwaddr addr,
                               uint64_t value, unsigned size)
 {
     ram_addr_t raddr = addr;
@@ -3113,7 +3132,7 @@ static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
     return 0;
 }
 
-static subpage_t *subpage_init(target_phys_addr_t base)
+static subpage_t *subpage_init(hwaddr base)
 {
     subpage_t *mmio;
 
@@ -3144,7 +3163,7 @@ static uint16_t dummy_section(MemoryRegion *mr)
     return phys_section_add(&section);
 }
 
-MemoryRegion *iotlb_to_region(target_phys_addr_t index)
+MemoryRegion *iotlb_to_region(hwaddr index)
 {
     return phys_sections[index & ~TARGET_PAGE_MASK].mr;
 }
@@ -3163,18 +3182,24 @@ static void io_mem_init(void)
                           "watch", UINT64_MAX);
 }
 
+static void mem_begin(MemoryListener *listener)
+{
+    AddressSpaceDispatch *d = container_of(listener, AddressSpaceDispatch, listener);
+
+    destroy_all_mappings(d);
+    d->phys_map.ptr = PHYS_MAP_NODE_NIL;
+}
+
 static void core_begin(MemoryListener *listener)
 {
-    destroy_all_mappings();
     phys_sections_clear();
-    phys_map.ptr = PHYS_MAP_NODE_NIL;
     phys_section_unassigned = dummy_section(&io_mem_unassigned);
     phys_section_notdirty = dummy_section(&io_mem_notdirty);
     phys_section_rom = dummy_section(&io_mem_rom);
     phys_section_watch = dummy_section(&io_mem_watch);
 }
 
-static void core_commit(MemoryListener *listener)
+static void tcg_commit(MemoryListener *listener)
 {
     CPUArchState *env;
 
@@ -3186,38 +3211,6 @@ static void core_commit(MemoryListener *listener)
     }
 }
 
-static void core_region_add(MemoryListener *listener,
-                            MemoryRegionSection *section)
-{
-    cpu_register_physical_memory_log(section, section->readonly);
-}
-
-static void core_region_del(MemoryListener *listener,
-                            MemoryRegionSection *section)
-{
-}
-
-static void core_region_nop(MemoryListener *listener,
-                            MemoryRegionSection *section)
-{
-    cpu_register_physical_memory_log(section, section->readonly);
-}
-
-static void core_log_start(MemoryListener *listener,
-                           MemoryRegionSection *section)
-{
-}
-
-static void core_log_stop(MemoryListener *listener,
-                          MemoryRegionSection *section)
-{
-}
-
-static void core_log_sync(MemoryListener *listener,
-                          MemoryRegionSection *section)
-{
-}
-
 static void core_log_global_start(MemoryListener *listener)
 {
     cpu_physical_memory_set_dirty_tracking(1);
@@ -3228,26 +3221,6 @@ static void core_log_global_stop(MemoryListener *listener)
     cpu_physical_memory_set_dirty_tracking(0);
 }
 
-static void core_eventfd_add(MemoryListener *listener,
-                             MemoryRegionSection *section,
-                             bool match_data, uint64_t data, EventNotifier *e)
-{
-}
-
-static void core_eventfd_del(MemoryListener *listener,
-                             MemoryRegionSection *section,
-                             bool match_data, uint64_t data, EventNotifier *e)
-{
-}
-
-static void io_begin(MemoryListener *listener)
-{
-}
-
-static void io_commit(MemoryListener *listener)
-{
-}
-
 static void io_region_add(MemoryListener *listener,
                           MemoryRegionSection *section)
 {
@@ -3266,90 +3239,66 @@ static void io_region_del(MemoryListener *listener,
     isa_unassign_ioport(section->offset_within_address_space, section->size);
 }
 
-static void io_region_nop(MemoryListener *listener,
-                          MemoryRegionSection *section)
-{
-}
-
-static void io_log_start(MemoryListener *listener,
-                         MemoryRegionSection *section)
-{
-}
-
-static void io_log_stop(MemoryListener *listener,
-                        MemoryRegionSection *section)
-{
-}
-
-static void io_log_sync(MemoryListener *listener,
-                        MemoryRegionSection *section)
-{
-}
-
-static void io_log_global_start(MemoryListener *listener)
-{
-}
-
-static void io_log_global_stop(MemoryListener *listener)
-{
-}
-
-static void io_eventfd_add(MemoryListener *listener,
-                           MemoryRegionSection *section,
-                           bool match_data, uint64_t data, EventNotifier *e)
-{
-}
-
-static void io_eventfd_del(MemoryListener *listener,
-                           MemoryRegionSection *section,
-                           bool match_data, uint64_t data, EventNotifier *e)
-{
-}
-
 static MemoryListener core_memory_listener = {
     .begin = core_begin,
-    .commit = core_commit,
-    .region_add = core_region_add,
-    .region_del = core_region_del,
-    .region_nop = core_region_nop,
-    .log_start = core_log_start,
-    .log_stop = core_log_stop,
-    .log_sync = core_log_sync,
     .log_global_start = core_log_global_start,
     .log_global_stop = core_log_global_stop,
-    .eventfd_add = core_eventfd_add,
-    .eventfd_del = core_eventfd_del,
-    .priority = 0,
+    .priority = 1,
 };
 
 static MemoryListener io_memory_listener = {
-    .begin = io_begin,
-    .commit = io_commit,
     .region_add = io_region_add,
     .region_del = io_region_del,
-    .region_nop = io_region_nop,
-    .log_start = io_log_start,
-    .log_stop = io_log_stop,
-    .log_sync = io_log_sync,
-    .log_global_start = io_log_global_start,
-    .log_global_stop = io_log_global_stop,
-    .eventfd_add = io_eventfd_add,
-    .eventfd_del = io_eventfd_del,
     .priority = 0,
 };
 
+static MemoryListener tcg_memory_listener = {
+    .commit = tcg_commit,
+};
+
+void address_space_init_dispatch(AddressSpace *as)
+{
+    AddressSpaceDispatch *d = g_new(AddressSpaceDispatch, 1);
+
+    d->phys_map  = (PhysPageEntry) { .ptr = PHYS_MAP_NODE_NIL, .is_leaf = 0 };
+    d->listener = (MemoryListener) {
+        .begin = mem_begin,
+        .region_add = mem_add,
+        .region_nop = mem_add,
+        .priority = 0,
+    };
+    as->dispatch = d;
+    memory_listener_register(&d->listener, as);
+}
+
+void address_space_destroy_dispatch(AddressSpace *as)
+{
+    AddressSpaceDispatch *d = as->dispatch;
+
+    memory_listener_unregister(&d->listener);
+    destroy_l2_mapping(&d->phys_map, P_L2_LEVELS - 1);
+    g_free(d);
+    as->dispatch = NULL;
+}
+
 static void memory_map_init(void)
 {
     system_memory = g_malloc(sizeof(*system_memory));
     memory_region_init(system_memory, "system", INT64_MAX);
-    set_system_memory_map(system_memory);
+    address_space_init(&address_space_memory, system_memory);
+    address_space_memory.name = "memory";
 
     system_io = g_malloc(sizeof(*system_io));
     memory_region_init(system_io, "io", 65536);
-    set_system_io_map(system_io);
+    address_space_init(&address_space_io, system_io);
+    address_space_io.name = "I/O";
+
+    memory_listener_register(&core_memory_listener, &address_space_memory);
+    memory_listener_register(&io_memory_listener, &address_space_io);
+    memory_listener_register(&tcg_memory_listener, &address_space_memory);
 
-    memory_listener_register(&core_memory_listener, system_memory);
-    memory_listener_register(&io_memory_listener, system_io);
+    dma_context_init(&dma_context_memory, &address_space_memory,
+                     NULL, NULL, NULL);
 }
 
 MemoryRegion *get_system_memory(void)
@@ -3406,13 +3355,27 @@ int cpu_memory_rw_debug(CPUArchState *env, target_ulong addr,
 }
 
 #else
-void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,
-                            int len, int is_write)
+
+static void invalidate_and_set_dirty(hwaddr addr,
+                                     hwaddr length)
 {
+    if (!cpu_physical_memory_is_dirty(addr)) {
+        /* invalidate code */
+        tb_invalidate_phys_page_range(addr, addr + length, 0);
+        /* set dirty bit */
+        cpu_physical_memory_set_dirty_flags(addr, (0xff & ~CODE_DIRTY_FLAG));
+    }
+    xen_modified_memory(addr, length);
+}
+
+void address_space_rw(AddressSpace *as, hwaddr addr, uint8_t *buf,
+                      int len, bool is_write)
+{
+    AddressSpaceDispatch *d = as->dispatch;
     int l;
     uint8_t *ptr;
     uint32_t val;
-    target_phys_addr_t page;
+    hwaddr page;
     MemoryRegionSection *section;
 
     while (len > 0) {
@@ -3420,11 +3383,11 @@ void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,
         l = (page + TARGET_PAGE_SIZE) - addr;
         if (l > len)
             l = len;
-        section = phys_page_find(page >> TARGET_PAGE_BITS);
+        section = phys_page_find(d, page >> TARGET_PAGE_BITS);
 
         if (is_write) {
             if (!memory_region_is_ram(section->mr)) {
-                target_phys_addr_t addr1;
+                hwaddr addr1;
                 addr1 = memory_region_section_addr(section, addr);
                 /* XXX: could force cpu_single_env to NULL to avoid
                    potential bugs */
@@ -3451,19 +3414,13 @@ void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,
                 /* RAM case */
                 ptr = qemu_get_ram_ptr(addr1);
                 memcpy(ptr, buf, l);
-                if (!cpu_physical_memory_is_dirty(addr1)) {
-                    /* invalidate code */
-                    tb_invalidate_phys_page_range(addr1, addr1 + l, 0);
-                    /* set dirty bit */
-                    cpu_physical_memory_set_dirty_flags(
-                        addr1, (0xff & ~CODE_DIRTY_FLAG));
-                }
+                invalidate_and_set_dirty(addr1, l);
                 qemu_put_ram_ptr(ptr);
             }
         } else {
             if (!(memory_region_is_ram(section->mr) ||
                   memory_region_is_romd(section->mr))) {
-                target_phys_addr_t addr1;
+                hwaddr addr1;
                 /* I/O case */
                 addr1 = memory_region_section_addr(section, addr);
                 if (l >= 4 && ((addr1 & 3) == 0)) {
@@ -3497,13 +3454,39 @@ void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,
     }
 }
 
+void address_space_write(AddressSpace *as, hwaddr addr,
+                         const uint8_t *buf, int len)
+{
+    address_space_rw(as, addr, (uint8_t *)buf, len, true);
+}
+
+/**
+ * address_space_read: read from an address space.
+ *
+ * @as: #AddressSpace to be accessed
+ * @addr: address within that address space
+ * @buf: buffer with the data transferred
+ */
+void address_space_read(AddressSpace *as, hwaddr addr, uint8_t *buf, int len)
+{
+    address_space_rw(as, addr, buf, len, false);
+}
+
+
+void cpu_physical_memory_rw(hwaddr addr, uint8_t *buf,
+                            int len, int is_write)
+{
+    return address_space_rw(&address_space_memory, addr, buf, len, is_write);
+}
+
 /* used for ROM loading : can write in RAM and ROM */
-void cpu_physical_memory_write_rom(target_phys_addr_t addr,
+void cpu_physical_memory_write_rom(hwaddr addr,
                                    const uint8_t *buf, int len)
 {
+    AddressSpaceDispatch *d = address_space_memory.dispatch;
     int l;
     uint8_t *ptr;
-    target_phys_addr_t page;
+    hwaddr page;
     MemoryRegionSection *section;
 
     while (len > 0) {
@@ -3511,7 +3494,7 @@ void cpu_physical_memory_write_rom(target_phys_addr_t addr,
         l = (page + TARGET_PAGE_SIZE) - addr;
         if (l > len)
             l = len;
-        section = phys_page_find(page >> TARGET_PAGE_BITS);
+        section = phys_page_find(d, page >> TARGET_PAGE_BITS);
 
         if (!(memory_region_is_ram(section->mr) ||
               memory_region_is_romd(section->mr))) {
@@ -3523,6 +3506,7 @@ void cpu_physical_memory_write_rom(target_phys_addr_t addr,
             /* ROM/RAM case */
             ptr = qemu_get_ram_ptr(addr1);
             memcpy(ptr, buf, l);
+            invalidate_and_set_dirty(addr1, l);
             qemu_put_ram_ptr(ptr);
         }
         len -= l;
@@ -3533,8 +3517,8 @@ void cpu_physical_memory_write_rom(target_phys_addr_t addr,
 
 typedef struct {
     void *buffer;
-    target_phys_addr_t addr;
-    target_phys_addr_t len;
+    hwaddr addr;
+    hwaddr len;
 } BounceBuffer;
 
 static BounceBuffer bounce;
@@ -3558,7 +3542,7 @@ void *cpu_register_map_client(void *opaque, void (*callback)(void *opaque))
     return client;
 }
 
-void cpu_unregister_map_client(void *_client)
+static void cpu_unregister_map_client(void *_client)
 {
     MapClient *client = (MapClient *)_client;
 
@@ -3584,14 +3568,16 @@ static void cpu_notify_map_clients(void)
  * Use cpu_register_map_client() to know when retrying the map operation is
  * likely to succeed.
  */
-void *cpu_physical_memory_map(target_phys_addr_t addr,
-                              target_phys_addr_t *plen,
-                              int is_write)
-{
-    target_phys_addr_t len = *plen;
-    target_phys_addr_t todo = 0;
+void *address_space_map(AddressSpace *as,
+                        hwaddr addr,
+                        hwaddr *plen,
+                        bool is_write)
+{
+    AddressSpaceDispatch *d = as->dispatch;
+    hwaddr len = *plen;
+    hwaddr todo = 0;
     int l;
-    target_phys_addr_t page;
+    hwaddr page;
     MemoryRegionSection *section;
     ram_addr_t raddr = RAM_ADDR_MAX;
     ram_addr_t rlen;
@@ -3602,7 +3588,7 @@ void *cpu_physical_memory_map(target_phys_addr_t addr,
         l = (page + TARGET_PAGE_SIZE) - addr;
         if (l > len)
             l = len;
-        section = phys_page_find(page >> TARGET_PAGE_BITS);
+        section = phys_page_find(d, page >> TARGET_PAGE_BITS);
 
         if (!(memory_region_is_ram(section->mr) && !section->readonly)) {
             if (todo || bounce.buffer) {
@@ -3612,7 +3598,7 @@ void *cpu_physical_memory_map(target_phys_addr_t addr,
             bounce.addr = addr;
             bounce.len = l;
             if (!is_write) {
-                cpu_physical_memory_read(addr, bounce.buffer, l);
+                address_space_read(as, addr, bounce.buffer, l);
             }
 
             *plen = l;
@@ -3633,12 +3619,12 @@ void *cpu_physical_memory_map(target_phys_addr_t addr,
     return ret;
 }
 
-/* Unmaps a memory region previously mapped by cpu_physical_memory_map().
+/* Unmaps a memory region previously mapped by address_space_map().
  * Will also mark the memory as dirty if is_write == 1.  access_len gives
  * the amount of memory that was actually read or written by the caller.
  */
-void cpu_physical_memory_unmap(void *buffer, target_phys_addr_t len,
-                               int is_write, target_phys_addr_t access_len)
+void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len,
+                         int is_write, hwaddr access_len)
 {
     if (buffer != bounce.buffer) {
         if (is_write) {
@@ -3648,13 +3634,7 @@ void cpu_physical_memory_unmap(void *buffer, target_phys_addr_t len,
                 l = TARGET_PAGE_SIZE;
                 if (l > access_len)
                     l = access_len;
-                if (!cpu_physical_memory_is_dirty(addr1)) {
-                    /* invalidate code */
-                    tb_invalidate_phys_page_range(addr1, addr1 + l, 0);
-                    /* set dirty bit */
-                    cpu_physical_memory_set_dirty_flags(
-                        addr1, (0xff & ~CODE_DIRTY_FLAG));
-                }
+                invalidate_and_set_dirty(addr1, l);
                 addr1 += l;
                 access_len -= l;
             }
@@ -3665,22 +3645,35 @@ void cpu_physical_memory_unmap(void *buffer, target_phys_addr_t len,
         return;
     }
     if (is_write) {
-        cpu_physical_memory_write(bounce.addr, bounce.buffer, access_len);
+        address_space_write(as, bounce.addr, bounce.buffer, access_len);
     }
     qemu_vfree(bounce.buffer);
     bounce.buffer = NULL;
     cpu_notify_map_clients();
 }
 
+void *cpu_physical_memory_map(hwaddr addr,
+                              hwaddr *plen,
+                              int is_write)
+{
+    return address_space_map(&address_space_memory, addr, plen, is_write);
+}
+
+void cpu_physical_memory_unmap(void *buffer, hwaddr len,
+                               int is_write, hwaddr access_len)
+{
+    return address_space_unmap(&address_space_memory, buffer, len, is_write, access_len);
+}
+
 /* warning: addr must be aligned */
-static inline uint32_t ldl_phys_internal(target_phys_addr_t addr,
+static inline uint32_t ldl_phys_internal(hwaddr addr,
                                          enum device_endian endian)
 {
     uint8_t *ptr;
     uint32_t val;
     MemoryRegionSection *section;
 
-    section = phys_page_find(addr >> TARGET_PAGE_BITS);
+    section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
 
     if (!(memory_region_is_ram(section->mr) ||
           memory_region_is_romd(section->mr))) {
@@ -3716,30 +3709,30 @@ static inline uint32_t ldl_phys_internal(target_phys_addr_t addr,
     return val;
 }
 
-uint32_t ldl_phys(target_phys_addr_t addr)
+uint32_t ldl_phys(hwaddr addr)
 {
     return ldl_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
 }
 
-uint32_t ldl_le_phys(target_phys_addr_t addr)
+uint32_t ldl_le_phys(hwaddr addr)
 {
     return ldl_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
 }
 
-uint32_t ldl_be_phys(target_phys_addr_t addr)
+uint32_t ldl_be_phys(hwaddr addr)
 {
     return ldl_phys_internal(addr, DEVICE_BIG_ENDIAN);
 }
 
 /* warning: addr must be aligned */
-static inline uint64_t ldq_phys_internal(target_phys_addr_t addr,
+static inline uint64_t ldq_phys_internal(hwaddr addr,
                                          enum device_endian endian)
 {
     uint8_t *ptr;
     uint64_t val;
     MemoryRegionSection *section;
 
-    section = phys_page_find(addr >> TARGET_PAGE_BITS);
+    section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
 
     if (!(memory_region_is_ram(section->mr) ||
           memory_region_is_romd(section->mr))) {
@@ -3775,23 +3768,23 @@ static inline uint64_t ldq_phys_internal(target_phys_addr_t addr,
     return val;
 }
 
-uint64_t ldq_phys(target_phys_addr_t addr)
+uint64_t ldq_phys(hwaddr addr)
 {
     return ldq_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
 }
 
-uint64_t ldq_le_phys(target_phys_addr_t addr)
+uint64_t ldq_le_phys(hwaddr addr)
 {
     return ldq_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
 }
 
-uint64_t ldq_be_phys(target_phys_addr_t addr)
+uint64_t ldq_be_phys(hwaddr addr)
 {
     return ldq_phys_internal(addr, DEVICE_BIG_ENDIAN);
 }
 
 /* XXX: optimize */
-uint32_t ldub_phys(target_phys_addr_t addr)
+uint32_t ldub_phys(hwaddr addr)
 {
     uint8_t val;
     cpu_physical_memory_read(addr, &val, 1);
@@ -3799,14 +3792,14 @@ uint32_t ldub_phys(target_phys_addr_t addr)
 }
 
 /* warning: addr must be aligned */
-static inline uint32_t lduw_phys_internal(target_phys_addr_t addr,
+static inline uint32_t lduw_phys_internal(hwaddr addr,
                                           enum device_endian endian)
 {
     uint8_t *ptr;
     uint64_t val;
     MemoryRegionSection *section;
 
-    section = phys_page_find(addr >> TARGET_PAGE_BITS);
+    section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
 
     if (!(memory_region_is_ram(section->mr) ||
           memory_region_is_romd(section->mr))) {
@@ -3842,17 +3835,17 @@ static inline uint32_t lduw_phys_internal(target_phys_addr_t addr,
     return val;
 }
 
-uint32_t lduw_phys(target_phys_addr_t addr)
+uint32_t lduw_phys(hwaddr addr)
 {
     return lduw_phys_internal(addr, DEVICE_NATIVE_ENDIAN);
 }
 
-uint32_t lduw_le_phys(target_phys_addr_t addr)
+uint32_t lduw_le_phys(hwaddr addr)
 {
     return lduw_phys_internal(addr, DEVICE_LITTLE_ENDIAN);
 }
 
-uint32_t lduw_be_phys(target_phys_addr_t addr)
+uint32_t lduw_be_phys(hwaddr addr)
 {
     return lduw_phys_internal(addr, DEVICE_BIG_ENDIAN);
 }
@@ -3860,12 +3853,12 @@ uint32_t lduw_be_phys(target_phys_addr_t addr)
 /* warning: addr must be aligned. The ram page is not masked as dirty
    and the code inside is not invalidated. It is useful if the dirty
    bits are used to track modified PTEs */
-void stl_phys_notdirty(target_phys_addr_t addr, uint32_t val)
+void stl_phys_notdirty(hwaddr addr, uint32_t val)
 {
     uint8_t *ptr;
     MemoryRegionSection *section;
 
-    section = phys_page_find(addr >> TARGET_PAGE_BITS);
+    section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
 
     if (!memory_region_is_ram(section->mr) || section->readonly) {
         addr = memory_region_section_addr(section, addr);
@@ -3892,12 +3885,12 @@ void stl_phys_notdirty(target_phys_addr_t addr, uint32_t val)
     }
 }
 
-void stq_phys_notdirty(target_phys_addr_t addr, uint64_t val)
+void stq_phys_notdirty(hwaddr addr, uint64_t val)
 {
     uint8_t *ptr;
     MemoryRegionSection *section;
 
-    section = phys_page_find(addr >> TARGET_PAGE_BITS);
+    section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
 
     if (!memory_region_is_ram(section->mr) || section->readonly) {
         addr = memory_region_section_addr(section, addr);
@@ -3920,13 +3913,13 @@ void stq_phys_notdirty(target_phys_addr_t addr, uint64_t val)
 }
 
 /* warning: addr must be aligned */
-static inline void stl_phys_internal(target_phys_addr_t addr, uint32_t val,
+static inline void stl_phys_internal(hwaddr addr, uint32_t val,
                                      enum device_endian endian)
 {
     uint8_t *ptr;
     MemoryRegionSection *section;
 
-    section = phys_page_find(addr >> TARGET_PAGE_BITS);
+    section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
 
     if (!memory_region_is_ram(section->mr) || section->readonly) {
         addr = memory_region_section_addr(section, addr);
@@ -3960,46 +3953,40 @@ static inline void stl_phys_internal(target_phys_addr_t addr, uint32_t val,
             stl_p(ptr, val);
             break;
         }
-        if (!cpu_physical_memory_is_dirty(addr1)) {
-            /* invalidate code */
-            tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
-            /* set dirty bit */
-            cpu_physical_memory_set_dirty_flags(addr1,
-                (0xff & ~CODE_DIRTY_FLAG));
-        }
+        invalidate_and_set_dirty(addr1, 4);
     }
 }
 
-void stl_phys(target_phys_addr_t addr, uint32_t val)
+void stl_phys(hwaddr addr, uint32_t val)
 {
     stl_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
 }
 
-void stl_le_phys(target_phys_addr_t addr, uint32_t val)
+void stl_le_phys(hwaddr addr, uint32_t val)
 {
     stl_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
 }
 
-void stl_be_phys(target_phys_addr_t addr, uint32_t val)
+void stl_be_phys(hwaddr addr, uint32_t val)
 {
     stl_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
 }
 
 /* XXX: optimize */
-void stb_phys(target_phys_addr_t addr, uint32_t val)
+void stb_phys(hwaddr addr, uint32_t val)
 {
     uint8_t v = val;
     cpu_physical_memory_write(addr, &v, 1);
 }
 
 /* warning: addr must be aligned */
-static inline void stw_phys_internal(target_phys_addr_t addr, uint32_t val,
+static inline void stw_phys_internal(hwaddr addr, uint32_t val,
                                      enum device_endian endian)
 {
     uint8_t *ptr;
     MemoryRegionSection *section;
 
-    section = phys_page_find(addr >> TARGET_PAGE_BITS);
+    section = phys_page_find(address_space_memory.dispatch, addr >> TARGET_PAGE_BITS);
 
     if (!memory_region_is_ram(section->mr) || section->readonly) {
         addr = memory_region_section_addr(section, addr);
@@ -4033,45 +4020,39 @@ static inline void stw_phys_internal(target_phys_addr_t addr, uint32_t val,
             stw_p(ptr, val);
             break;
         }
-        if (!cpu_physical_memory_is_dirty(addr1)) {
-            /* invalidate code */
-            tb_invalidate_phys_page_range(addr1, addr1 + 2, 0);
-            /* set dirty bit */
-            cpu_physical_memory_set_dirty_flags(addr1,
-                (0xff & ~CODE_DIRTY_FLAG));
-        }
+        invalidate_and_set_dirty(addr1, 2);
     }
 }
 
-void stw_phys(target_phys_addr_t addr, uint32_t val)
+void stw_phys(hwaddr addr, uint32_t val)
 {
     stw_phys_internal(addr, val, DEVICE_NATIVE_ENDIAN);
 }
 
-void stw_le_phys(target_phys_addr_t addr, uint32_t val)
+void stw_le_phys(hwaddr addr, uint32_t val)
 {
     stw_phys_internal(addr, val, DEVICE_LITTLE_ENDIAN);
 }
 
-void stw_be_phys(target_phys_addr_t addr, uint32_t val)
+void stw_be_phys(hwaddr addr, uint32_t val)
 {
     stw_phys_internal(addr, val, DEVICE_BIG_ENDIAN);
 }
 
 /* XXX: optimize */
-void stq_phys(target_phys_addr_t addr, uint64_t val)
+void stq_phys(hwaddr addr, uint64_t val)
 {
     val = tswap64(val);
     cpu_physical_memory_write(addr, &val, 8);
 }
 
-void stq_le_phys(target_phys_addr_t addr, uint64_t val)
+void stq_le_phys(hwaddr addr, uint64_t val)
 {
     val = cpu_to_le64(val);
     cpu_physical_memory_write(addr, &val, 8);
 }
 
-void stq_be_phys(target_phys_addr_t addr, uint64_t val)
+void stq_be_phys(hwaddr addr, uint64_t val)
 {
     val = cpu_to_be64(val);
     cpu_physical_memory_write(addr, &val, 8);
@@ -4082,7 +4063,7 @@ int cpu_memory_rw_debug(CPUArchState *env, target_ulong addr,
                         uint8_t *buf, int len, int is_write)
 {
     int l;
-    target_phys_addr_t phys_addr;
+    hwaddr phys_addr;
     target_ulong page;
 
     while (len > 0) {
@@ -4195,7 +4176,7 @@ void dump_exec_info(FILE *f, fprintf_function cpu_fprintf)
     }
     /* XXX: avoid using doubles ? */
     cpu_fprintf(f, "Translation buffer state:\n");
-    cpu_fprintf(f, "gen code size       %td/%ld\n",
+    cpu_fprintf(f, "gen code size       %td/%zd\n",
                 code_gen_ptr - code_gen_buffer, code_gen_buffer_max_size);
     cpu_fprintf(f, "TB count            %d/%d\n", 
                 nb_tbs, code_gen_max_blocks);
@@ -4237,11 +4218,12 @@ bool virtio_is_big_endian(void)
 #endif
 
 #ifndef CONFIG_USER_ONLY
-bool cpu_physical_memory_is_io(target_phys_addr_t phys_addr)
+bool cpu_physical_memory_is_io(hwaddr phys_addr)
 {
     MemoryRegionSection *section;
 
-    section = phys_page_find(phys_addr >> TARGET_PAGE_BITS);
+    section = phys_page_find(address_space_memory.dispatch,
+                             phys_addr >> TARGET_PAGE_BITS);
 
     return !(memory_region_is_ram(section->mr) ||
              memory_region_is_romd(section->mr));
diff --git a/fpu/softfloat-specialize.h b/fpu/softfloat-specialize.h
index 490245004..518f694a6 100644
--- a/fpu/softfloat-specialize.h
+++ b/fpu/softfloat-specialize.h
@@ -41,6 +41,13 @@ these four paragraphs for those parts of this code that are retained.
 #define SNAN_BIT_IS_ONE		0
 #endif
 
+#if defined(TARGET_XTENSA)
+/* Define for architectures which deviate from IEEE in not supporting
+ * signaling NaNs (so all NaNs are treated as quiet).
+ */
+#define NO_SIGNALING_NANS 1
+#endif
+
 /*----------------------------------------------------------------------------
 | The pattern for a default generated half-precision NaN.
 *----------------------------------------------------------------------------*/
@@ -57,7 +64,8 @@ const float16 float16_default_nan = const_float16(0xFE00);
 *----------------------------------------------------------------------------*/
 #if defined(TARGET_SPARC)
 const float32 float32_default_nan = const_float32(0x7FFFFFFF);
-#elif defined(TARGET_PPC) || defined(TARGET_ARM) || defined(TARGET_ALPHA)
+#elif defined(TARGET_PPC) || defined(TARGET_ARM) || defined(TARGET_ALPHA) || \
+      defined(TARGET_XTENSA)
 const float32 float32_default_nan = const_float32(0x7FC00000);
 #elif SNAN_BIT_IS_ONE
 const float32 float32_default_nan = const_float32(0x7FBFFFFF);
@@ -127,6 +135,17 @@ typedef struct {
     uint64_t high, low;
 } commonNaNT;
 
+#ifdef NO_SIGNALING_NANS
+int float16_is_quiet_nan(float16 a_)
+{
+    return float16_is_any_nan(a_);
+}
+
+int float16_is_signaling_nan(float16 a_)
+{
+    return 0;
+}
+#else
 /*----------------------------------------------------------------------------
 | Returns 1 if the half-precision floating-point value `a' is a quiet
 | NaN; otherwise returns 0.
@@ -156,6 +175,7 @@ int float16_is_signaling_nan(float16 a_)
     return (((a >> 9) & 0x3F) == 0x3E) && (a & 0x1FF);
 #endif
 }
+#endif
 
 /*----------------------------------------------------------------------------
 | Returns a quiet NaN if the half-precision floating point value `a' is a
@@ -217,6 +237,17 @@ static float16 commonNaNToFloat16(commonNaNT a STATUS_PARAM)
     }
 }
 
+#ifdef NO_SIGNALING_NANS
+int float32_is_quiet_nan(float32 a_)
+{
+    return float32_is_any_nan(a_);
+}
+
+int float32_is_signaling_nan(float32 a_)
+{
+    return 0;
+}
+#else
 /*----------------------------------------------------------------------------
 | Returns 1 if the single-precision floating-point value `a' is a quiet
 | NaN; otherwise returns 0.
@@ -246,6 +277,7 @@ int float32_is_signaling_nan( float32 a_ )
     return ( ( ( a>>22 ) & 0x1FF ) == 0x1FE ) && ( a & 0x003FFFFF );
 #endif
 }
+#endif
 
 /*----------------------------------------------------------------------------
 | Returns a quiet NaN if the single-precision floating point value `a' is a
@@ -372,7 +404,7 @@ static int pickNaN(flag aIsQNaN, flag aIsSNaN, flag bIsQNaN, flag bIsSNaN,
         return 1;
     }
 }
-#elif defined(TARGET_PPC)
+#elif defined(TARGET_PPC) || defined(TARGET_XTENSA)
 static int pickNaN(flag aIsQNaN, flag aIsSNaN, flag bIsQNaN, flag bIsSNaN,
                    flag aIsLargerSignificand)
 {
@@ -454,6 +486,33 @@ static int pickNaNMulAdd(flag aIsQNaN, flag aIsSNaN, flag bIsQNaN, flag bIsSNaN,
         return 1;
     }
 }
+#elif defined(TARGET_MIPS)
+static int pickNaNMulAdd(flag aIsQNaN, flag aIsSNaN, flag bIsQNaN, flag bIsSNaN,
+                         flag cIsQNaN, flag cIsSNaN, flag infzero STATUS_PARAM)
+{
+    /* For MIPS, the (inf,zero,qnan) case sets InvalidOp and returns
+     * the default NaN
+     */
+    if (infzero) {
+        float_raise(float_flag_invalid STATUS_VAR);
+        return 3;
+    }
+
+    /* Prefer sNaN over qNaN, in the a, b, c order. */
+    if (aIsSNaN) {
+        return 0;
+    } else if (bIsSNaN) {
+        return 1;
+    } else if (cIsSNaN) {
+        return 2;
+    } else if (aIsQNaN) {
+        return 0;
+    } else if (bIsQNaN) {
+        return 1;
+    } else {
+        return 2;
+    }
+}
 #elif defined(TARGET_PPC)
 static int pickNaNMulAdd(flag aIsQNaN, flag aIsSNaN, flag bIsQNaN, flag bIsSNaN,
                          flag cIsQNaN, flag cIsSNaN, flag infzero STATUS_PARAM)
@@ -586,6 +645,17 @@ static float32 propagateFloat32MulAddNaN(float32 a, float32 b,
     }
 }
 
+#ifdef NO_SIGNALING_NANS
+int float64_is_quiet_nan(float64 a_)
+{
+    return float64_is_any_nan(a_);
+}
+
+int float64_is_signaling_nan(float64 a_)
+{
+    return 0;
+}
+#else
 /*----------------------------------------------------------------------------
 | Returns 1 if the double-precision floating-point value `a' is a quiet
 | NaN; otherwise returns 0.
@@ -619,6 +689,7 @@ int float64_is_signaling_nan( float64 a_ )
         && ( a & LIT64( 0x0007FFFFFFFFFFFF ) );
 #endif
 }
+#endif
 
 /*----------------------------------------------------------------------------
 | Returns a quiet NaN if the double-precision floating point value `a' is a
@@ -773,6 +844,17 @@ static float64 propagateFloat64MulAddNaN(float64 a, float64 b,
     }
 }
 
+#ifdef NO_SIGNALING_NANS
+int floatx80_is_quiet_nan(floatx80 a_)
+{
+    return floatx80_is_any_nan(a_);
+}
+
+int floatx80_is_signaling_nan(floatx80 a_)
+{
+    return 0;
+}
+#else
 /*----------------------------------------------------------------------------
 | Returns 1 if the extended double-precision floating-point value `a' is a
 | quiet NaN; otherwise returns 0. This slightly differs from the same
@@ -816,6 +898,7 @@ int floatx80_is_signaling_nan( floatx80 a )
         && ( a.low == aLow );
 #endif
 }
+#endif
 
 /*----------------------------------------------------------------------------
 | Returns a quiet NaN if the extended double-precision floating point value
@@ -929,6 +1012,17 @@ static floatx80 propagateFloatx80NaN( floatx80 a, floatx80 b STATUS_PARAM)
     }
 }
 
+#ifdef NO_SIGNALING_NANS
+int float128_is_quiet_nan(float128 a_)
+{
+    return float128_is_any_nan(a_);
+}
+
+int float128_is_signaling_nan(float128 a_)
+{
+    return 0;
+}
+#else
 /*----------------------------------------------------------------------------
 | Returns 1 if the quadruple-precision floating-point value `a' is a quiet
 | NaN; otherwise returns 0.
@@ -964,6 +1058,7 @@ int float128_is_signaling_nan( float128 a )
         && ( a.low || ( a.high & LIT64( 0x00007FFFFFFFFFFF ) ) );
 #endif
 }
+#endif
 
 /*----------------------------------------------------------------------------
 | Returns a quiet NaN if the quadruple-precision floating point value `a' is
diff --git a/fpu/softfloat.c b/fpu/softfloat.c
index b29256a8e..841314686 100644
--- a/fpu/softfloat.c
+++ b/fpu/softfloat.c
@@ -1238,7 +1238,7 @@ float32 uint64_to_float32( uint64 a STATUS_PARAM )
     if ( a == 0 ) return float32_zero;
     shiftCount = countLeadingZeros64( a ) - 40;
     if ( 0 <= shiftCount ) {
-        return packFloat32( 1 > 0, 0x95 - shiftCount, a<<shiftCount );
+        return packFloat32(0, 0x95 - shiftCount, a<<shiftCount);
     }
     else {
         shiftCount += 7;
@@ -1248,7 +1248,7 @@ float32 uint64_to_float32( uint64 a STATUS_PARAM )
         else {
             a <<= shiftCount;
         }
-        return roundAndPackFloat32( 1 > 0, 0x9C - shiftCount, a STATUS_VAR );
+        return roundAndPackFloat32(0, 0x9C - shiftCount, a STATUS_VAR);
     }
 }
 
@@ -3007,7 +3007,7 @@ float32 float16_to_float32(float16 a, flag ieee STATUS_PARAM)
         if (aSig) {
             return commonNaNToFloat32(float16ToCommonNaN(a STATUS_VAR) STATUS_VAR);
         }
-        return packFloat32(aSign, 0xff, aSig << 13);
+        return packFloat32(aSign, 0xff, 0);
     }
     if (aExp == 0) {
         int8 shiftCount;
diff --git a/fpu/softfloat.h b/fpu/softfloat.h
index feec3a180..d8999b311 100644
--- a/fpu/softfloat.h
+++ b/fpu/softfloat.h
@@ -219,7 +219,7 @@ void float_raise( int8 flags STATUS_PARAM);
 enum {
     float_muladd_negate_c = 1,
     float_muladd_negate_product = 2,
-    float_muladd_negate_result = 3,
+    float_muladd_negate_result = 4,
 };
 
 /*----------------------------------------------------------------------------
@@ -251,6 +251,11 @@ int float16_is_quiet_nan( float16 );
 int float16_is_signaling_nan( float16 );
 float16 float16_maybe_silence_nan( float16 );
 
+INLINE int float16_is_any_nan(float16 a)
+{
+    return ((float16_val(a) & ~0x8000) > 0x7c00);
+}
+
 /*----------------------------------------------------------------------------
 | The pattern for a default generated half-precision NaN.
 *----------------------------------------------------------------------------*/
diff --git a/fsdev/qemu-fsdev-dummy.c b/fsdev/qemu-fsdev-dummy.c
index 4e700dd4e..300f2758b 100644
--- a/fsdev/qemu-fsdev-dummy.c
+++ b/fsdev/qemu-fsdev-dummy.c
@@ -14,6 +14,7 @@
 #include <string.h>
 #include "qemu-fsdev.h"
 #include "qemu-config.h"
+#include "module.h"
 
 int qemu_fsdev_add(QemuOpts *opts)
 {
diff --git a/gdbstub.c b/gdbstub.c
index 5d37dd98f..d02ec7538 100644
--- a/gdbstub.c
+++ b/gdbstub.c
@@ -1226,33 +1226,48 @@ static int cpu_gdb_write_register(CPUOpenRISCState *env,
 
 static int cpu_gdb_read_register(CPUSH4State *env, uint8_t *mem_buf, int n)
 {
-    if (n < 8) {
+    switch (n) {
+    case 0 ... 7:
         if ((env->sr & (SR_MD | SR_RB)) == (SR_MD | SR_RB)) {
             GET_REGL(env->gregs[n + 16]);
         } else {
             GET_REGL(env->gregs[n]);
         }
-    } else if (n < 16) {
+    case 8 ... 15:
         GET_REGL(env->gregs[n]);
-    } else if (n >= 25 && n < 41) {
-	GET_REGL(env->fregs[(n - 25) + ((env->fpscr & FPSCR_FR) ? 16 : 0)]);
-    } else if (n >= 43 && n < 51) {
-	GET_REGL(env->gregs[n - 43]);
-    } else if (n >= 51 && n < 59) {
-	GET_REGL(env->gregs[n - (51 - 16)]);
-    }
-    switch (n) {
-    case 16: GET_REGL(env->pc);
-    case 17: GET_REGL(env->pr);
-    case 18: GET_REGL(env->gbr);
-    case 19: GET_REGL(env->vbr);
-    case 20: GET_REGL(env->mach);
-    case 21: GET_REGL(env->macl);
-    case 22: GET_REGL(env->sr);
-    case 23: GET_REGL(env->fpul);
-    case 24: GET_REGL(env->fpscr);
-    case 41: GET_REGL(env->ssr);
-    case 42: GET_REGL(env->spc);
+    case 16:
+        GET_REGL(env->pc);
+    case 17:
+        GET_REGL(env->pr);
+    case 18:
+        GET_REGL(env->gbr);
+    case 19:
+        GET_REGL(env->vbr);
+    case 20:
+        GET_REGL(env->mach);
+    case 21:
+        GET_REGL(env->macl);
+    case 22:
+        GET_REGL(env->sr);
+    case 23:
+        GET_REGL(env->fpul);
+    case 24:
+        GET_REGL(env->fpscr);
+    case 25 ... 40:
+        if (env->fpscr & FPSCR_FR) {
+            stfl_p(mem_buf, env->fregs[n - 9]);
+        } else {
+            stfl_p(mem_buf, env->fregs[n - 25]);
+        }
+        return 4;
+    case 41:
+        GET_REGL(env->ssr);
+    case 42:
+        GET_REGL(env->spc);
+    case 43 ... 50:
+        GET_REGL(env->gregs[n - 43]);
+    case 51 ... 58:
+        GET_REGL(env->gregs[n - (51 - 16)]);
     }
 
     return 0;
@@ -1260,42 +1275,63 @@ static int cpu_gdb_read_register(CPUSH4State *env, uint8_t *mem_buf, int n)
 
 static int cpu_gdb_write_register(CPUSH4State *env, uint8_t *mem_buf, int n)
 {
-    uint32_t tmp;
-
-    tmp = ldl_p(mem_buf);
-
-    if (n < 8) {
+    switch (n) {
+    case 0 ... 7:
         if ((env->sr & (SR_MD | SR_RB)) == (SR_MD | SR_RB)) {
-            env->gregs[n + 16] = tmp;
+            env->gregs[n + 16] = ldl_p(mem_buf);
         } else {
-            env->gregs[n] = tmp;
+            env->gregs[n] = ldl_p(mem_buf);
         }
-	return 4;
-    } else if (n < 16) {
-        env->gregs[n] = tmp;
-	return 4;
-    } else if (n >= 25 && n < 41) {
-	env->fregs[(n - 25) + ((env->fpscr & FPSCR_FR) ? 16 : 0)] = tmp;
-	return 4;
-    } else if (n >= 43 && n < 51) {
-	env->gregs[n - 43] = tmp;
-	return 4;
-    } else if (n >= 51 && n < 59) {
-	env->gregs[n - (51 - 16)] = tmp;
-	return 4;
-    }
-    switch (n) {
-    case 16: env->pc = tmp; break;
-    case 17: env->pr = tmp; break;
-    case 18: env->gbr = tmp; break;
-    case 19: env->vbr = tmp; break;
-    case 20: env->mach = tmp; break;
-    case 21: env->macl = tmp; break;
-    case 22: env->sr = tmp; break;
-    case 23: env->fpul = tmp; break;
-    case 24: env->fpscr = tmp; break;
-    case 41: env->ssr = tmp; break;
-    case 42: env->spc = tmp; break;
+        break;
+    case 8 ... 15:
+        env->gregs[n] = ldl_p(mem_buf);
+        break;
+    case 16:
+        env->pc = ldl_p(mem_buf);
+        break;
+    case 17:
+        env->pr = ldl_p(mem_buf);
+        break;
+    case 18:
+        env->gbr = ldl_p(mem_buf);
+        break;
+    case 19:
+        env->vbr = ldl_p(mem_buf);
+        break;
+    case 20:
+        env->mach = ldl_p(mem_buf);
+        break;
+    case 21:
+        env->macl = ldl_p(mem_buf);
+        break;
+    case 22:
+        env->sr = ldl_p(mem_buf);
+        break;
+    case 23:
+        env->fpul = ldl_p(mem_buf);
+        break;
+    case 24:
+        env->fpscr = ldl_p(mem_buf);
+        break;
+    case 25 ... 40:
+        if (env->fpscr & FPSCR_FR) {
+            env->fregs[n - 9] = ldfl_p(mem_buf);
+        } else {
+            env->fregs[n - 25] = ldfl_p(mem_buf);
+        }
+        break;
+    case 41:
+        env->ssr = ldl_p(mem_buf);
+        break;
+    case 42:
+        env->spc = ldl_p(mem_buf);
+        break;
+    case 43 ... 50:
+        env->gregs[n - 43] = ldl_p(mem_buf);
+        break;
+    case 51 ... 58:
+        env->gregs[n - (51 - 16)] = ldl_p(mem_buf);
+        break;
     default: return 0;
     }
 
@@ -1660,6 +1696,10 @@ static int cpu_gdb_read_register(CPUXtensaState *env, uint8_t *mem_buf, int n)
         GET_REG32(env->uregs[reg->targno & 0xff]);
         break;
 
+    case 4: /*f*/
+        GET_REG32(float32_val(env->fregs[reg->targno & 0x0f]));
+        break;
+
     case 8: /*a*/
         GET_REG32(env->regs[reg->targno & 0x0f]);
         break;
@@ -1700,6 +1740,10 @@ static int cpu_gdb_write_register(CPUXtensaState *env, uint8_t *mem_buf, int n)
         env->uregs[reg->targno & 0xff] = tmp;
         break;
 
+    case 4: /*f*/
+        env->fregs[reg->targno & 0x0f] = make_float32(tmp);
+        break;
+
     case 8: /*a*/
         env->regs[reg->targno & 0x0f] = tmp;
         break;
diff --git a/gen-icount.h b/gen-icount.h
index 430cb446d..248cf5b16 100644
--- a/gen-icount.h
+++ b/gen-icount.h
@@ -16,7 +16,7 @@ static inline void gen_icount_start(void)
     count = tcg_temp_local_new_i32();
     tcg_gen_ld_i32(count, cpu_env, offsetof(CPUArchState, icount_decr.u32));
     /* This is a horrid hack to allow fixing up the value later.  */
-    icount_arg = gen_opparam_ptr + 1;
+    icount_arg = tcg_ctx.gen_opparam_ptr + 1;
     tcg_gen_subi_i32(count, count, 0xdeadbeef);
 
     tcg_gen_brcondi_i32(TCG_COND_LT, count, 0, icount_label);
diff --git a/hmp-commands.hx b/hmp-commands.hx
index f6104b099..010b8c9ba 100644
--- a/hmp-commands.hx
+++ b/hmp-commands.hx
@@ -99,16 +99,60 @@ ETEXI
 
     {
         .name       = "block_job_cancel",
-        .args_type  = "device:B",
-        .params     = "device",
-        .help       = "stop an active background block operation",
+        .args_type  = "force:-f,device:B",
+        .params     = "[-f] device",
+        .help       = "stop an active background block operation (use -f"
+                      "\n\t\t\t if the operation is currently paused)",
         .mhandler.cmd = hmp_block_job_cancel,
     },
 
 STEXI
 @item block_job_cancel
 @findex block_job_cancel
-Stop an active block streaming operation.
+Stop an active background block operation (streaming, mirroring).
+ETEXI
+
+    {
+        .name       = "block_job_complete",
+        .args_type  = "device:B",
+        .params     = "device",
+        .help       = "stop an active background block operation",
+        .mhandler.cmd = hmp_block_job_complete,
+    },
+
+STEXI
+@item block_job_complete
+@findex block_job_complete
+Manually trigger completion of an active background block operation.
+For mirroring, this will switch the device to the destination path.
+ETEXI
+
+    {
+        .name       = "block_job_pause",
+        .args_type  = "device:B",
+        .params     = "device",
+        .help       = "pause an active background block operation",
+        .mhandler.cmd = hmp_block_job_pause,
+    },
+
+STEXI
+@item block_job_pause
+@findex block_job_pause
+Pause an active block streaming operation.
+ETEXI
+
+    {
+        .name       = "block_job_resume",
+        .args_type  = "device:B",
+        .params     = "device",
+        .help       = "resume a paused background block operation",
+        .mhandler.cmd = hmp_block_job_resume,
+    },
+
+STEXI
+@item block_job_resume
+@findex block_job_resume
+Resume a paused block streaming operation.
 ETEXI
 
     {
@@ -194,8 +238,7 @@ ETEXI
         .args_type  = "filename:F",
         .params     = "filename",
         .help       = "save screen into PPM image 'filename'",
-        .user_print = monitor_user_noop,
-        .mhandler.cmd_new = do_screen_dump,
+        .mhandler.cmd = hmp_screen_dump,
     },
 
 STEXI
@@ -502,19 +545,19 @@ ETEXI
 
     {
         .name       = "sendkey",
-        .args_type  = "string:s,hold_time:i?",
+        .args_type  = "keys:s,hold-time:i?",
         .params     = "keys [hold_ms]",
         .help       = "send keys to the VM (e.g. 'sendkey ctrl-alt-f1', default hold time=100 ms)",
-        .mhandler.cmd = do_sendkey,
+        .mhandler.cmd = hmp_send_key,
     },
 
 STEXI
 @item sendkey @var{keys}
 @findex sendkey
 
-Send @var{keys} to the emulator. @var{keys} could be the name of the
-key or @code{#} followed by the raw value in either decimal or hexadecimal
-format. Use @code{-} to press several keys simultaneously. Example:
+Send @var{keys} to the guest. @var{keys} could be the name of the
+key or the raw value in hexadecimal format. Use @code{-} to press
+several keys simultaneously. Example:
 @example
 sendkey ctrl-alt-f1
 @end example
@@ -915,12 +958,11 @@ ETEXI
 #if defined(CONFIG_HAVE_CORE_DUMP)
     {
         .name       = "dump-guest-memory",
-        .args_type  = "paging:-p,protocol:s,begin:i?,length:i?",
-        .params     = "[-p] protocol [begin] [length]",
+        .args_type  = "paging:-p,filename:F,begin:i?,length:i?",
+        .params     = "[-p] filename [begin] [length]",
         .help       = "dump guest memory to file"
                       "\n\t\t\t begin(optional): the starting physical address"
                       "\n\t\t\t length(optional): the memory size, in bytes",
-        .user_print = monitor_user_noop,
         .mhandler.cmd = hmp_dump_guest_memory,
     },
 
@@ -930,8 +972,7 @@ STEXI
 @findex dump-guest-memory
 Dump guest memory to @var{protocol}. The file can be processed with crash or
 gdb.
-  protocol: destination file(started with "file:") or destination file
-            descriptor (started with "fd:")
+  filename: dump file name
     paging: do paging to get guest's memory mapping
      begin: the starting physical address. It's optional, and should be
             specified with length together.
@@ -963,6 +1004,27 @@ Snapshot device, using snapshot file as target if provided
 ETEXI
 
     {
+        .name       = "drive_mirror",
+        .args_type  = "reuse:-n,full:-f,device:B,target:s,format:s?",
+        .params     = "[-n] [-f] device target [format]",
+        .help       = "initiates live storage\n\t\t\t"
+                      "migration for a device. The device's contents are\n\t\t\t"
+                      "copied to the new image file, including data that\n\t\t\t"
+                      "is written after the command is started.\n\t\t\t"
+                      "The -n flag requests QEMU to reuse the image found\n\t\t\t"
+                      "in new-image-file, instead of recreating it from scratch.\n\t\t\t"
+                      "The -f flag requests QEMU to copy the whole disk,\n\t\t\t"
+                      "so that the result does not need a backing file.\n\t\t\t",
+        .mhandler.cmd = hmp_drive_mirror,
+    },
+STEXI
+@item drive_mirror
+@findex drive_mirror
+Start mirroring a block device's writes to a new destination,
+using the specified target.
+ETEXI
+
+    {
         .name       = "drive_add",
         .args_type  = "pci_addr:s,opts:s",
         .params     = "[[<domain>:]<bus>:]<slot>\n"
@@ -1248,6 +1310,51 @@ Remove all matches from the access control list, and set the default
 policy back to @code{deny}.
 ETEXI
 
+    {
+        .name       = "nbd_server_start",
+        .args_type  = "all:-a,writable:-w,uri:s",
+        .params     = "nbd_server_start [-a] [-w] host:port",
+        .help       = "serve block devices on the given host and port",
+        .mhandler.cmd = hmp_nbd_server_start,
+    },
+STEXI
+@item nbd_server_start @var{host}:@var{port}
+@findex nbd_server_start
+Start an NBD server on the given host and/or port.  If the @option{-a}
+option is included, all of the virtual machine's block devices that
+have an inserted media on them are automatically exported; in this case,
+the @option{-w} option makes the devices writable too.
+ETEXI
+
+    {
+        .name       = "nbd_server_add",
+        .args_type  = "writable:-w,device:B",
+        .params     = "nbd_server_add [-w] device",
+        .help       = "export a block device via NBD",
+        .mhandler.cmd = hmp_nbd_server_add,
+    },
+STEXI
+@item nbd_server_add @var{device}
+@findex nbd_server_add
+Export a block device through QEMU's NBD server, which must be started
+beforehand with @command{nbd_server_start}.  The @option{-w} option makes the
+exported device writable too.
+ETEXI
+
+    {
+        .name       = "nbd_server_stop",
+        .args_type  = "",
+        .params     = "nbd_server_stop",
+        .help       = "stop serving block devices using the NBD protocol",
+        .mhandler.cmd = hmp_nbd_server_stop,
+    },
+STEXI
+@item nbd_server_stop
+@findex nbd_server_stop
+Stop the QEMU embedded NBD server.
+ETEXI
+
+
 #if defined(TARGET_I386)
 
     {
@@ -1466,13 +1573,6 @@ show roms
 @end table
 ETEXI
 
-#ifdef CONFIG_TRACE_SIMPLE
-STEXI
-@item info trace
-show contents of trace buffer
-ETEXI
-#endif
-
 STEXI
 @item info trace-events
 show available trace events and their state
diff --git a/hmp.c b/hmp.c
index 81c8acb67..180ba2bfd 100644
--- a/hmp.c
+++ b/hmp.c
@@ -18,7 +18,9 @@
 #include "qemu-option.h"
 #include "qemu-timer.h"
 #include "qmp-commands.h"
+#include "qemu_socket.h"
 #include "monitor.h"
+#include "console.h"
 
 static void hmp_handle_error(Monitor *mon, Error **errp)
 {
@@ -151,6 +153,14 @@ void hmp_info_migrate(Monitor *mon)
         monitor_printf(mon, "Migration status: %s\n", info->status);
         monitor_printf(mon, "total time: %" PRIu64 " milliseconds\n",
                        info->total_time);
+        if (info->has_expected_downtime) {
+            monitor_printf(mon, "expected downtime: %" PRIu64 " milliseconds\n",
+                           info->expected_downtime);
+        }
+        if (info->has_downtime) {
+            monitor_printf(mon, "downtime: %" PRIu64 " milliseconds\n",
+                           info->downtime);
+        }
     }
 
     if (info->has_ram) {
@@ -166,6 +176,10 @@ void hmp_info_migrate(Monitor *mon)
                        info->ram->normal);
         monitor_printf(mon, "normal bytes: %" PRIu64 " kbytes\n",
                        info->ram->normal_bytes >> 10);
+        if (info->ram->dirty_pages_rate) {
+            monitor_printf(mon, "dirty pages rate: %" PRIu64 " pages\n",
+                           info->ram->dirty_pages_rate);
+        }
     }
 
     if (info->has_disk) {
@@ -232,20 +246,19 @@ void hmp_info_cpus(Monitor *mon)
             active = '*';
         }
 
-        monitor_printf(mon, "%c CPU #%" PRId64 ": ", active, cpu->value->CPU);
+        monitor_printf(mon, "%c CPU #%" PRId64 ":", active, cpu->value->CPU);
 
         if (cpu->value->has_pc) {
-            monitor_printf(mon, "pc=0x%016" PRIx64, cpu->value->pc);
+            monitor_printf(mon, " pc=0x%016" PRIx64, cpu->value->pc);
         }
         if (cpu->value->has_nip) {
-            monitor_printf(mon, "nip=0x%016" PRIx64, cpu->value->nip);
+            monitor_printf(mon, " nip=0x%016" PRIx64, cpu->value->nip);
         }
         if (cpu->value->has_npc) {
-            monitor_printf(mon, "pc=0x%016" PRIx64, cpu->value->pc);
-            monitor_printf(mon, "npc=0x%016" PRIx64, cpu->value->npc);
+            monitor_printf(mon, " npc=0x%016" PRIx64, cpu->value->npc);
         }
         if (cpu->value->has_PC) {
-            monitor_printf(mon, "PC=0x%016" PRIx64, cpu->value->PC);
+            monitor_printf(mon, " PC=0x%016" PRIx64, cpu->value->PC);
         }
 
         if (cpu->value->halted) {
@@ -413,6 +426,8 @@ void hmp_info_spice(Monitor *mon)
         monitor_printf(mon, "     address: %s:%" PRId64 " [tls]\n",
                        info->host, info->tls_port);
     }
+    monitor_printf(mon, "    migrated: %s\n",
+                   info->migrated ? "true" : "false");
     monitor_printf(mon, "        auth: %s\n", info->auth);
     monitor_printf(mon, "    compiled: %s\n", info->compiled_version);
     monitor_printf(mon, "  mouse-mode: %s\n",
@@ -756,6 +771,35 @@ void hmp_block_resize(Monitor *mon, const QDict *qdict)
     hmp_handle_error(mon, &errp);
 }
 
+void hmp_drive_mirror(Monitor *mon, const QDict *qdict)
+{
+    const char *device = qdict_get_str(qdict, "device");
+    const char *filename = qdict_get_str(qdict, "target");
+    const char *format = qdict_get_try_str(qdict, "format");
+    int reuse = qdict_get_try_bool(qdict, "reuse", 0);
+    int full = qdict_get_try_bool(qdict, "full", 0);
+    enum NewImageMode mode;
+    Error *errp = NULL;
+
+    if (!filename) {
+        error_set(&errp, QERR_MISSING_PARAMETER, "target");
+        hmp_handle_error(mon, &errp);
+        return;
+    }
+
+    if (reuse) {
+        mode = NEW_IMAGE_MODE_EXISTING;
+    } else {
+        mode = NEW_IMAGE_MODE_ABSOLUTE_PATHS;
+    }
+
+    qmp_drive_mirror(device, filename, !!format, format,
+                     full ? MIRROR_SYNC_MODE_FULL : MIRROR_SYNC_MODE_TOP,
+                     true, mode, false, 0,
+                     false, 0, false, 0, &errp);
+    hmp_handle_error(mon, &errp);
+}
+
 void hmp_snapshot_blkdev(Monitor *mon, const QDict *qdict)
 {
     const char *device = qdict_get_str(qdict, "device");
@@ -927,7 +971,8 @@ void hmp_block_stream(Monitor *mon, const QDict *qdict)
     int64_t speed = qdict_get_try_int(qdict, "speed", 0);
 
     qmp_block_stream(device, base != NULL, base,
-                     qdict_haskey(qdict, "speed"), speed, &error);
+                     qdict_haskey(qdict, "speed"), speed,
+                     BLOCKDEV_ON_ERROR_REPORT, true, &error);
 
     hmp_handle_error(mon, &error);
 }
@@ -947,8 +992,39 @@ void hmp_block_job_cancel(Monitor *mon, const QDict *qdict)
 {
     Error *error = NULL;
     const char *device = qdict_get_str(qdict, "device");
+    bool force = qdict_get_try_bool(qdict, "force", 0);
 
-    qmp_block_job_cancel(device, &error);
+    qmp_block_job_cancel(device, true, force, &error);
+
+    hmp_handle_error(mon, &error);
+}
+
+void hmp_block_job_pause(Monitor *mon, const QDict *qdict)
+{
+    Error *error = NULL;
+    const char *device = qdict_get_str(qdict, "device");
+
+    qmp_block_job_pause(device, &error);
+
+    hmp_handle_error(mon, &error);
+}
+
+void hmp_block_job_resume(Monitor *mon, const QDict *qdict)
+{
+    Error *error = NULL;
+    const char *device = qdict_get_str(qdict, "device");
+
+    qmp_block_job_resume(device, &error);
+
+    hmp_handle_error(mon, &error);
+}
+
+void hmp_block_job_complete(Monitor *mon, const QDict *qdict)
+{
+    Error *error = NULL;
+    const char *device = qdict_get_str(qdict, "device");
+
+    qmp_block_job_complete(device, &error);
 
     hmp_handle_error(mon, &error);
 }
@@ -1039,11 +1115,12 @@ void hmp_dump_guest_memory(Monitor *mon, const QDict *qdict)
 {
     Error *errp = NULL;
     int paging = qdict_get_try_bool(qdict, "paging", 0);
-    const char *file = qdict_get_str(qdict, "protocol");
+    const char *file = qdict_get_str(qdict, "filename");
     bool has_begin = qdict_haskey(qdict, "begin");
     bool has_length = qdict_haskey(qdict, "length");
     int64_t begin = 0;
     int64_t length = 0;
+    char *prot;
 
     if (has_begin) {
         begin = qdict_get_int(qdict, "begin");
@@ -1052,9 +1129,12 @@ void hmp_dump_guest_memory(Monitor *mon, const QDict *qdict)
         length = qdict_get_int(qdict, "length");
     }
 
-    qmp_dump_guest_memory(paging, file, has_begin, begin, has_length, length,
+    prot = g_strconcat("file:", file, NULL);
+
+    qmp_dump_guest_memory(paging, prot, has_begin, begin, has_length, length,
                           &errp);
     hmp_handle_error(mon, &errp);
+    g_free(prot);
 }
 
 void hmp_netdev_add(Monitor *mon, const QDict *qdict)
@@ -1102,3 +1182,156 @@ void hmp_closefd(Monitor *mon, const QDict *qdict)
     qmp_closefd(fdname, &errp);
     hmp_handle_error(mon, &errp);
 }
+
+void hmp_send_key(Monitor *mon, const QDict *qdict)
+{
+    const char *keys = qdict_get_str(qdict, "keys");
+    KeyValueList *keylist, *head = NULL, *tmp = NULL;
+    int has_hold_time = qdict_haskey(qdict, "hold-time");
+    int hold_time = qdict_get_try_int(qdict, "hold-time", -1);
+    Error *err = NULL;
+    char keyname_buf[16];
+    char *separator;
+    int keyname_len;
+
+    while (1) {
+        separator = strchr(keys, '-');
+        keyname_len = separator ? separator - keys : strlen(keys);
+        pstrcpy(keyname_buf, sizeof(keyname_buf), keys);
+
+        /* Be compatible with old interface, convert user inputted "<" */
+        if (!strncmp(keyname_buf, "<", 1) && keyname_len == 1) {
+            pstrcpy(keyname_buf, sizeof(keyname_buf), "less");
+            keyname_len = 4;
+        }
+        keyname_buf[keyname_len] = 0;
+
+        keylist = g_malloc0(sizeof(*keylist));
+        keylist->value = g_malloc0(sizeof(*keylist->value));
+
+        if (!head) {
+            head = keylist;
+        }
+        if (tmp) {
+            tmp->next = keylist;
+        }
+        tmp = keylist;
+
+        if (strstart(keyname_buf, "0x", NULL)) {
+            char *endp;
+            int value = strtoul(keyname_buf, &endp, 0);
+            if (*endp != '\0') {
+                goto err_out;
+            }
+            keylist->value->kind = KEY_VALUE_KIND_NUMBER;
+            keylist->value->number = value;
+        } else {
+            int idx = index_from_key(keyname_buf);
+            if (idx == Q_KEY_CODE_MAX) {
+                goto err_out;
+            }
+            keylist->value->kind = KEY_VALUE_KIND_QCODE;
+            keylist->value->qcode = idx;
+        }
+
+        if (!separator) {
+            break;
+        }
+        keys = separator + 1;
+    }
+
+    qmp_send_key(head, has_hold_time, hold_time, &err);
+    hmp_handle_error(mon, &err);
+
+out:
+    qapi_free_KeyValueList(head);
+    return;
+
+err_out:
+    monitor_printf(mon, "invalid parameter: %s\n", keyname_buf);
+    goto out;
+}
+
+void hmp_screen_dump(Monitor *mon, const QDict *qdict)
+{
+    const char *filename = qdict_get_str(qdict, "filename");
+    Error *err = NULL;
+
+    qmp_screendump(filename, &err);
+    hmp_handle_error(mon, &err);
+}
+
+void hmp_nbd_server_start(Monitor *mon, const QDict *qdict)
+{
+    const char *uri = qdict_get_str(qdict, "uri");
+    int writable = qdict_get_try_bool(qdict, "writable", 0);
+    int all = qdict_get_try_bool(qdict, "all", 0);
+    Error *local_err = NULL;
+    BlockInfoList *block_list, *info;
+    SocketAddress *addr;
+
+    if (writable && !all) {
+        error_setg(&local_err, "-w only valid together with -a");
+        goto exit;
+    }
+
+    /* First check if the address is valid and start the server.  */
+    addr = socket_parse(uri, &local_err);
+    if (local_err != NULL) {
+        goto exit;
+    }
+
+    qmp_nbd_server_start(addr, &local_err);
+    qapi_free_SocketAddress(addr);
+    if (local_err != NULL) {
+        goto exit;
+    }
+
+    if (!all) {
+        return;
+    }
+
+    /* Then try adding all block devices.  If one fails, close all and
+     * exit.
+     */
+    block_list = qmp_query_block(NULL);
+
+    for (info = block_list; info; info = info->next) {
+        if (!info->value->has_inserted) {
+            continue;
+        }
+
+        qmp_nbd_server_add(info->value->device, true, writable, &local_err);
+
+        if (local_err != NULL) {
+            qmp_nbd_server_stop(NULL);
+            break;
+        }
+    }
+
+    qapi_free_BlockInfoList(block_list);
+
+exit:
+    hmp_handle_error(mon, &local_err);
+}
+
+void hmp_nbd_server_add(Monitor *mon, const QDict *qdict)
+{
+    const char *device = qdict_get_str(qdict, "device");
+    int writable = qdict_get_try_bool(qdict, "writable", 0);
+    Error *local_err = NULL;
+
+    qmp_nbd_server_add(device, true, writable, &local_err);
+
+    if (local_err != NULL) {
+        hmp_handle_error(mon, &local_err);
+    }
+}
+
+void hmp_nbd_server_stop(Monitor *mon, const QDict *qdict)
+{
+    Error *errp = NULL;
+
+    qmp_nbd_server_stop(&errp);
+    hmp_handle_error(mon, &errp);
+}
diff --git a/hmp.h b/hmp.h
index 7dd93bf0f..0ab03be98 100644
--- a/hmp.h
+++ b/hmp.h
@@ -51,6 +51,7 @@ void hmp_block_passwd(Monitor *mon, const QDict *qdict);
 void hmp_balloon(Monitor *mon, const QDict *qdict);
 void hmp_block_resize(Monitor *mon, const QDict *qdict);
 void hmp_snapshot_blkdev(Monitor *mon, const QDict *qdict);
+void hmp_drive_mirror(Monitor *mon, const QDict *qdict);
 void hmp_migrate_cancel(Monitor *mon, const QDict *qdict);
 void hmp_migrate_set_downtime(Monitor *mon, const QDict *qdict);
 void hmp_migrate_set_speed(Monitor *mon, const QDict *qdict);
@@ -64,6 +65,9 @@ void hmp_block_set_io_throttle(Monitor *mon, const QDict *qdict);
 void hmp_block_stream(Monitor *mon, const QDict *qdict);
 void hmp_block_job_set_speed(Monitor *mon, const QDict *qdict);
 void hmp_block_job_cancel(Monitor *mon, const QDict *qdict);
+void hmp_block_job_pause(Monitor *mon, const QDict *qdict);
+void hmp_block_job_resume(Monitor *mon, const QDict *qdict);
+void hmp_block_job_complete(Monitor *mon, const QDict *qdict);
 void hmp_migrate(Monitor *mon, const QDict *qdict);
 void hmp_device_del(Monitor *mon, const QDict *qdict);
 void hmp_dump_guest_memory(Monitor *mon, const QDict *qdict);
@@ -71,5 +75,10 @@ void hmp_netdev_add(Monitor *mon, const QDict *qdict);
 void hmp_netdev_del(Monitor *mon, const QDict *qdict);
 void hmp_getfd(Monitor *mon, const QDict *qdict);
 void hmp_closefd(Monitor *mon, const QDict *qdict);
+void hmp_send_key(Monitor *mon, const QDict *qdict);
+void hmp_screen_dump(Monitor *mon, const QDict *qdict);
+void hmp_nbd_server_start(Monitor *mon, const QDict *qdict);
+void hmp_nbd_server_add(Monitor *mon, const QDict *qdict);
+void hmp_nbd_server_stop(Monitor *mon, const QDict *qdict);
 
 #endif
diff --git a/hw/9pfs/Makefile.objs b/hw/9pfs/Makefile.objs
index 972df2405..1e9b595cb 100644
--- a/hw/9pfs/Makefile.objs
+++ b/hw/9pfs/Makefile.objs
@@ -1,9 +1,9 @@
-hw-obj-y  = virtio-9p.o
-hw-obj-y += virtio-9p-local.o virtio-9p-xattr.o
-hw-obj-y += virtio-9p-xattr-user.o virtio-9p-posix-acl.o
-hw-obj-y += virtio-9p-coth.o cofs.o codir.o cofile.o
-hw-obj-y += coxattr.o virtio-9p-synth.o
-hw-obj-$(CONFIG_OPEN_BY_HANDLE) +=  virtio-9p-handle.o
-hw-obj-y += virtio-9p-proxy.o
+common-obj-y  = virtio-9p.o
+common-obj-y += virtio-9p-local.o virtio-9p-xattr.o
+common-obj-y += virtio-9p-xattr-user.o virtio-9p-posix-acl.o
+common-obj-y += virtio-9p-coth.o cofs.o codir.o cofile.o
+common-obj-y += coxattr.o virtio-9p-synth.o
+common-obj-$(CONFIG_OPEN_BY_HANDLE) +=  virtio-9p-handle.o
+common-obj-y += virtio-9p-proxy.o
 
 obj-y += virtio-9p-device.o
diff --git a/hw/9pfs/virtio-9p-posix-acl.c b/hw/9pfs/virtio-9p-posix-acl.c
index a1948e3af..c064017b1 100644
--- a/hw/9pfs/virtio-9p-posix-acl.c
+++ b/hw/9pfs/virtio-9p-posix-acl.c
@@ -44,7 +44,8 @@ static ssize_t mp_pacl_listxattr(FsContext *ctx, const char *path,
         return -1;
     }
 
-    strncpy(value, ACL_ACCESS, len);
+    /* len includes the trailing NUL */
+    memcpy(value, ACL_ACCESS, len);
     return 0;
 }
 
@@ -95,7 +96,8 @@ static ssize_t mp_dacl_listxattr(FsContext *ctx, const char *path,
         return -1;
     }
 
-    strncpy(value, ACL_DEFAULT, len);
+    /* len includes the trailing NUL */
+    memcpy(value, ACL_ACCESS, len);
     return 0;
 }
 
diff --git a/hw/9pfs/virtio-9p-synth.c b/hw/9pfs/virtio-9p-synth.c
index 92e0b09d3..e95a856d2 100644
--- a/hw/9pfs/virtio-9p-synth.c
+++ b/hw/9pfs/virtio-9p-synth.c
@@ -58,7 +58,7 @@ static V9fsSynthNode *v9fs_add_dir_node(V9fsSynthNode *parent, int mode,
         node->attr->read  = NULL;
     }
     node->private = node;
-    strncpy(node->name, name, sizeof(node->name));
+    pstrcpy(node->name, sizeof(node->name), name);
     QLIST_INSERT_HEAD_RCU(&parent->child, node, sibling);
     return node;
 }
@@ -132,7 +132,7 @@ int qemu_v9fs_synth_add_file(V9fsSynthNode *parent, int mode,
     node->attr->write  = write;
     node->attr->mode   = mode;
     node->private      = arg;
-    strncpy(node->name, name, sizeof(node->name));
+    pstrcpy(node->name, sizeof(node->name), name);
     QLIST_INSERT_HEAD_RCU(&parent->child, node, sibling);
     ret = 0;
 err_out:
diff --git a/hw/9pfs/virtio-9p-xattr-user.c b/hw/9pfs/virtio-9p-xattr-user.c
index 5044a3e5a..5bb602007 100644
--- a/hw/9pfs/virtio-9p-xattr-user.c
+++ b/hw/9pfs/virtio-9p-xattr-user.c
@@ -61,7 +61,8 @@ static ssize_t mp_user_listxattr(FsContext *ctx, const char *path,
         return -1;
     }
 
-    strncpy(value, name, name_size);
+    /* name_size includes the trailing NUL. */
+    memcpy(value, name, name_size);
     return name_size;
 }
 
diff --git a/hw/9pfs/virtio-9p-xattr.c b/hw/9pfs/virtio-9p-xattr.c
index 7f08f6e17..a83960676 100644
--- a/hw/9pfs/virtio-9p-xattr.c
+++ b/hw/9pfs/virtio-9p-xattr.c
@@ -53,7 +53,8 @@ ssize_t pt_listxattr(FsContext *ctx, const char *path,
         return -1;
     }
 
-    strncpy(value, name, name_size);
+    /* no need for strncpy: name_size is strlen(name)+1 */
+    memcpy(value, name, name_size);
     return name_size;
 }
 
diff --git a/hw/9pfs/virtio-9p.c b/hw/9pfs/virtio-9p.c
index 4b5254011..8b9cdc96e 100644
--- a/hw/9pfs/virtio-9p.c
+++ b/hw/9pfs/virtio-9p.c
@@ -505,7 +505,6 @@ static void virtfs_reset(V9fsPDU *pdu)
         error_report("9pfs:%s: One or more uncluncked fids "
                      "found during reset", __func__);
     }
-    return;
 }
 
 #define P9_QID_TYPE_DIR         0x80
@@ -934,7 +933,6 @@ static void v9fs_version(void *opaque)
 out:
     complete_pdu(s, pdu, offset);
     v9fs_string_free(&version);
-    return;
 }
 
 static void v9fs_attach(void *opaque)
@@ -1314,7 +1312,6 @@ out_nofid:
         g_free(wnames);
         g_free(qids);
     }
-    return;
 }
 
 static int32_t get_iounit(V9fsPDU *pdu, V9fsPath *path)
@@ -2257,7 +2254,6 @@ static void v9fs_flush(void *opaque)
         free_pdu(pdu->s, cancel_pdu);
     }
     complete_pdu(s, pdu, 7);
-    return;
 }
 
 static void v9fs_link(void *opaque)
@@ -2763,7 +2759,6 @@ out:
     put_fid(pdu, fidp);
 out_nofid:
     complete_pdu(s, pdu, retval);
-    return;
 }
 
 static void v9fs_mknod(void *opaque)
diff --git a/hw/Makefile.objs b/hw/Makefile.objs
index 6dfebd297..d581d8d6d 100644
--- a/hw/Makefile.objs
+++ b/hw/Makefile.objs
@@ -1,140 +1,145 @@
-hw-obj-y = usb/ ide/
-hw-obj-y += loader.o
-hw-obj-$(CONFIG_VIRTIO) += virtio-console.o
-hw-obj-$(CONFIG_VIRTIO_PCI) += virtio-pci.o
-hw-obj-y += fw_cfg.o
-hw-obj-$(CONFIG_PCI) += pci.o pci_bridge.o pci_bridge_dev.o
-hw-obj-$(CONFIG_PCI) += msix.o msi.o
-hw-obj-$(CONFIG_PCI) += shpc.o
-hw-obj-$(CONFIG_PCI) += slotid_cap.o
-hw-obj-$(CONFIG_PCI) += pci_host.o pcie_host.o
-hw-obj-$(CONFIG_PCI) += ioh3420.o xio3130_upstream.o xio3130_downstream.o
-hw-obj-y += watchdog.o
-hw-obj-$(CONFIG_ISA_MMIO) += isa_mmio.o
-hw-obj-$(CONFIG_ECC) += ecc.o
-hw-obj-$(CONFIG_NAND) += nand.o
-hw-obj-$(CONFIG_PFLASH_CFI01) += pflash_cfi01.o
-hw-obj-$(CONFIG_PFLASH_CFI02) += pflash_cfi02.o
-
-hw-obj-$(CONFIG_M48T59) += m48t59.o
-hw-obj-$(CONFIG_ESCC) += escc.o
-hw-obj-$(CONFIG_EMPTY_SLOT) += empty_slot.o
-
-hw-obj-$(CONFIG_SERIAL) += serial.o
-hw-obj-$(CONFIG_PARALLEL) += parallel.o
-hw-obj-$(CONFIG_I8254) += i8254_common.o i8254.o
-hw-obj-$(CONFIG_PCSPK) += pcspk.o
-hw-obj-$(CONFIG_PCKBD) += pckbd.o
-hw-obj-$(CONFIG_FDC) += fdc.o
-hw-obj-$(CONFIG_ACPI) += acpi.o acpi_piix4.o
-hw-obj-$(CONFIG_APM) += pm_smbus.o apm.o
-hw-obj-$(CONFIG_DMA) += dma.o
-hw-obj-$(CONFIG_I82374) += i82374.o
-hw-obj-$(CONFIG_HPET) += hpet.o
-hw-obj-$(CONFIG_APPLESMC) += applesmc.o
-hw-obj-$(CONFIG_SMARTCARD) += ccid-card-passthru.o
-hw-obj-$(CONFIG_SMARTCARD_NSS) += ccid-card-emulated.o
-hw-obj-$(CONFIG_I8259) += i8259_common.o i8259.o
+common-obj-y = usb/ ide/
+common-obj-y += loader.o
+common-obj-$(CONFIG_VIRTIO) += virtio-console.o
+common-obj-$(CONFIG_VIRTIO) += virtio-rng.o
+common-obj-$(CONFIG_VIRTIO_PCI) += virtio-pci.o
+common-obj-y += fw_cfg.o
+common-obj-$(CONFIG_PCI) += pci.o pci_bridge.o pci_bridge_dev.o
+common-obj-$(CONFIG_PCI) += msix.o msi.o
+common-obj-$(CONFIG_PCI) += shpc.o
+common-obj-$(CONFIG_PCI) += slotid_cap.o
+common-obj-$(CONFIG_PCI) += pci_host.o pcie_host.o
+common-obj-$(CONFIG_PCI) += ioh3420.o xio3130_upstream.o xio3130_downstream.o
+common-obj-$(CONFIG_PCI) += i82801b11.o
+common-obj-y += watchdog.o
+common-obj-$(CONFIG_ISA_MMIO) += isa_mmio.o
+common-obj-$(CONFIG_ECC) += ecc.o
+common-obj-$(CONFIG_NAND) += nand.o
+common-obj-$(CONFIG_PFLASH_CFI01) += pflash_cfi01.o
+common-obj-$(CONFIG_PFLASH_CFI02) += pflash_cfi02.o
+
+common-obj-$(CONFIG_M48T59) += m48t59.o
+common-obj-$(CONFIG_ESCC) += escc.o
+common-obj-$(CONFIG_EMPTY_SLOT) += empty_slot.o
+
+common-obj-$(CONFIG_SERIAL) += serial.o serial-isa.o
+common-obj-$(CONFIG_SERIAL_PCI) += serial-pci.o
+common-obj-$(CONFIG_PARALLEL) += parallel.o
+common-obj-$(CONFIG_I8254) += i8254_common.o i8254.o
+common-obj-$(CONFIG_PCSPK) += pcspk.o
+common-obj-$(CONFIG_PCKBD) += pckbd.o
+common-obj-$(CONFIG_FDC) += fdc.o
+common-obj-$(CONFIG_ACPI) += acpi.o acpi_piix4.o acpi_ich9.o smbus_ich9.o
+common-obj-$(CONFIG_APM) += pm_smbus.o apm.o
+common-obj-$(CONFIG_DMA) += dma.o
+common-obj-$(CONFIG_I82374) += i82374.o
+common-obj-$(CONFIG_HPET) += hpet.o
+common-obj-$(CONFIG_APPLESMC) += applesmc.o
+common-obj-$(CONFIG_SMARTCARD) += ccid-card-passthru.o
+common-obj-$(CONFIG_SMARTCARD_NSS) += ccid-card-emulated.o
+common-obj-$(CONFIG_I8259) += i8259_common.o i8259.o
+common-obj-y += fifo.o
+common-obj-y += pam.o
 
 # PPC devices
-hw-obj-$(CONFIG_PREP_PCI) += prep_pci.o
-hw-obj-$(CONFIG_I82378) += i82378.o
+common-obj-$(CONFIG_PREP_PCI) += prep_pci.o
+common-obj-$(CONFIG_I82378) += i82378.o
 # Mac shared devices
-hw-obj-$(CONFIG_MACIO) += macio.o
-hw-obj-$(CONFIG_CUDA) += cuda.o
-hw-obj-$(CONFIG_ADB) += adb.o
-hw-obj-$(CONFIG_MAC_NVRAM) += mac_nvram.o
-hw-obj-$(CONFIG_MAC_DBDMA) += mac_dbdma.o
+common-obj-$(CONFIG_MACIO) += macio.o
+common-obj-$(CONFIG_CUDA) += cuda.o
+common-obj-$(CONFIG_ADB) += adb.o
+common-obj-$(CONFIG_MAC_NVRAM) += mac_nvram.o
+common-obj-$(CONFIG_MAC_DBDMA) += mac_dbdma.o
 # OldWorld PowerMac
-hw-obj-$(CONFIG_HEATHROW_PIC) += heathrow_pic.o
-hw-obj-$(CONFIG_GRACKLE_PCI) += grackle_pci.o
+common-obj-$(CONFIG_HEATHROW_PIC) += heathrow_pic.o
+common-obj-$(CONFIG_GRACKLE_PCI) += grackle_pci.o
 # NewWorld PowerMac
-hw-obj-$(CONFIG_UNIN_PCI) += unin_pci.o
-hw-obj-$(CONFIG_DEC_PCI) += dec_pci.o
+common-obj-$(CONFIG_UNIN_PCI) += unin_pci.o
+common-obj-$(CONFIG_DEC_PCI) += dec_pci.o
 # PowerPC E500 boards
-hw-obj-$(CONFIG_PPCE500_PCI) += ppce500_pci.o
+common-obj-$(CONFIG_PPCE500_PCI) += ppce500_pci.o
 
 # MIPS devices
-hw-obj-$(CONFIG_PIIX4) += piix4.o
-hw-obj-$(CONFIG_G364FB) += g364fb.o
-hw-obj-$(CONFIG_JAZZ_LED) += jazz_led.o
+common-obj-$(CONFIG_PIIX4) += piix4.o
+common-obj-$(CONFIG_G364FB) += g364fb.o
+common-obj-$(CONFIG_JAZZ_LED) += jazz_led.o
 
 # Xilinx devices
-hw-obj-$(CONFIG_XILINX) += xilinx_intc.o
-hw-obj-$(CONFIG_XILINX) += xilinx_timer.o
-hw-obj-$(CONFIG_XILINX) += xilinx_uartlite.o
-hw-obj-$(CONFIG_XILINX_AXI) += xilinx_axidma.o
-hw-obj-$(CONFIG_XILINX_AXI) += xilinx_axienet.o
-hw-obj-$(CONFIG_XILINX_AXI) += stream.o
+common-obj-$(CONFIG_XILINX) += xilinx_intc.o
+common-obj-$(CONFIG_XILINX) += xilinx_timer.o
+common-obj-$(CONFIG_XILINX) += xilinx_uartlite.o
+common-obj-$(CONFIG_XILINX_AXI) += xilinx_axidma.o
+common-obj-$(CONFIG_XILINX_AXI) += xilinx_axienet.o
+common-obj-$(CONFIG_XILINX_AXI) += stream.o
 
 # PKUnity SoC devices
-hw-obj-$(CONFIG_PUV3) += puv3_intc.o
-hw-obj-$(CONFIG_PUV3) += puv3_ost.o
-hw-obj-$(CONFIG_PUV3) += puv3_gpio.o
-hw-obj-$(CONFIG_PUV3) += puv3_pm.o
-hw-obj-$(CONFIG_PUV3) += puv3_dma.o
+common-obj-$(CONFIG_PUV3) += puv3_intc.o
+common-obj-$(CONFIG_PUV3) += puv3_ost.o
+common-obj-$(CONFIG_PUV3) += puv3_gpio.o
+common-obj-$(CONFIG_PUV3) += puv3_pm.o
+common-obj-$(CONFIG_PUV3) += puv3_dma.o
 
 # ARM devices
-hw-obj-$(CONFIG_ARM_TIMER) += arm_timer.o
-hw-obj-$(CONFIG_PL011) += pl011.o
-hw-obj-$(CONFIG_PL022) += pl022.o
-hw-obj-$(CONFIG_PL031) += pl031.o
-hw-obj-$(CONFIG_PL041) += pl041.o lm4549.o
-hw-obj-$(CONFIG_PL050) += pl050.o
-hw-obj-$(CONFIG_PL061) += pl061.o
-hw-obj-$(CONFIG_PL080) += pl080.o
-hw-obj-$(CONFIG_PL110) += pl110.o
-hw-obj-$(CONFIG_PL181) += pl181.o
-hw-obj-$(CONFIG_PL190) += pl190.o
-hw-obj-$(CONFIG_PL310) += arm_l2x0.o
-hw-obj-$(CONFIG_VERSATILE_PCI) += versatile_pci.o
-hw-obj-$(CONFIG_VERSATILE_I2C) += versatile_i2c.o
-hw-obj-$(CONFIG_CADENCE) += cadence_uart.o
-hw-obj-$(CONFIG_CADENCE) += cadence_ttc.o
-hw-obj-$(CONFIG_CADENCE) += cadence_gem.o
-hw-obj-$(CONFIG_XGMAC) += xgmac.o
+common-obj-$(CONFIG_ARM_TIMER) += arm_timer.o
+common-obj-$(CONFIG_PL011) += pl011.o
+common-obj-$(CONFIG_PL022) += pl022.o
+common-obj-$(CONFIG_PL031) += pl031.o
+common-obj-$(CONFIG_PL041) += pl041.o lm4549.o
+common-obj-$(CONFIG_PL050) += pl050.o
+common-obj-$(CONFIG_PL061) += pl061.o
+common-obj-$(CONFIG_PL080) += pl080.o
+common-obj-$(CONFIG_PL110) += pl110.o
+common-obj-$(CONFIG_PL181) += pl181.o
+common-obj-$(CONFIG_PL190) += pl190.o
+common-obj-$(CONFIG_PL310) += arm_l2x0.o
+common-obj-$(CONFIG_VERSATILE_PCI) += versatile_pci.o
+common-obj-$(CONFIG_VERSATILE_I2C) += versatile_i2c.o
+common-obj-$(CONFIG_CADENCE) += cadence_uart.o
+common-obj-$(CONFIG_CADENCE) += cadence_ttc.o
+common-obj-$(CONFIG_CADENCE) += cadence_gem.o
+common-obj-$(CONFIG_XGMAC) += xgmac.o
 
 # PCI watchdog devices
-hw-obj-$(CONFIG_PCI) += wdt_i6300esb.o
+common-obj-$(CONFIG_PCI) += wdt_i6300esb.o
 
-hw-obj-$(CONFIG_PCI) += pcie.o pcie_aer.o pcie_port.o
+common-obj-$(CONFIG_PCI) += pcie.o pcie_aer.o pcie_port.o
 
 # PCI network cards
-hw-obj-$(CONFIG_NE2000_PCI) += ne2000.o
-hw-obj-$(CONFIG_EEPRO100_PCI) += eepro100.o
-hw-obj-$(CONFIG_PCNET_PCI) += pcnet-pci.o
-hw-obj-$(CONFIG_PCNET_COMMON) += pcnet.o
-hw-obj-$(CONFIG_E1000_PCI) += e1000.o
-hw-obj-$(CONFIG_RTL8139_PCI) += rtl8139.o
-
-hw-obj-$(CONFIG_SMC91C111) += smc91c111.o
-hw-obj-$(CONFIG_LAN9118) += lan9118.o
-hw-obj-$(CONFIG_NE2000_ISA) += ne2000-isa.o
-hw-obj-$(CONFIG_OPENCORES_ETH) += opencores_eth.o
+common-obj-$(CONFIG_NE2000_PCI) += ne2000.o
+common-obj-$(CONFIG_EEPRO100_PCI) += eepro100.o
+common-obj-$(CONFIG_PCNET_PCI) += pcnet-pci.o
+common-obj-$(CONFIG_PCNET_COMMON) += pcnet.o
+common-obj-$(CONFIG_E1000_PCI) += e1000.o
+common-obj-$(CONFIG_RTL8139_PCI) += rtl8139.o
+
+common-obj-$(CONFIG_SMC91C111) += smc91c111.o
+common-obj-$(CONFIG_LAN9118) += lan9118.o
+common-obj-$(CONFIG_NE2000_ISA) += ne2000-isa.o
+common-obj-$(CONFIG_OPENCORES_ETH) += opencores_eth.o
 
 # SCSI layer
-hw-obj-$(CONFIG_LSI_SCSI_PCI) += lsi53c895a.o
-hw-obj-$(CONFIG_MEGASAS_SCSI_PCI) += megasas.o
-hw-obj-$(CONFIG_ESP) += esp.o
-hw-obj-$(CONFIG_ESP_PCI) += esp-pci.o
+common-obj-$(CONFIG_LSI_SCSI_PCI) += lsi53c895a.o
+common-obj-$(CONFIG_MEGASAS_SCSI_PCI) += megasas.o
+common-obj-$(CONFIG_ESP) += esp.o
+common-obj-$(CONFIG_ESP_PCI) += esp-pci.o
 
-hw-obj-y += sysbus.o isa-bus.o
-hw-obj-y += qdev-addr.o
+common-obj-y += sysbus.o isa-bus.o
+common-obj-y += qdev-addr.o
 
 # VGA
-hw-obj-$(CONFIG_VGA_PCI) += vga-pci.o
-hw-obj-$(CONFIG_VGA_ISA) += vga-isa.o
-hw-obj-$(CONFIG_VGA_ISA_MM) += vga-isa-mm.o
-hw-obj-$(CONFIG_VMWARE_VGA) += vmware_vga.o
-hw-obj-$(CONFIG_VMMOUSE) += vmmouse.o
-hw-obj-$(CONFIG_VGA_CIRRUS) += cirrus_vga.o
+common-obj-$(CONFIG_VGA_PCI) += vga-pci.o
+common-obj-$(CONFIG_VGA_ISA) += vga-isa.o
+common-obj-$(CONFIG_VGA_ISA_MM) += vga-isa-mm.o
+common-obj-$(CONFIG_VMWARE_VGA) += vmware_vga.o
+common-obj-$(CONFIG_VMMOUSE) += vmmouse.o
+common-obj-$(CONFIG_VGA_CIRRUS) += cirrus_vga.o
 
-hw-obj-$(CONFIG_RC4030) += rc4030.o
-hw-obj-$(CONFIG_DP8393X) += dp8393x.o
-hw-obj-$(CONFIG_DS1225Y) += ds1225y.o
-hw-obj-$(CONFIG_MIPSNET) += mipsnet.o
+common-obj-$(CONFIG_RC4030) += rc4030.o
+common-obj-$(CONFIG_DP8393X) += dp8393x.o
+common-obj-$(CONFIG_DS1225Y) += ds1225y.o
+common-obj-$(CONFIG_MIPSNET) += mipsnet.o
 
-hw-obj-y += null-machine.o
+common-obj-y += null-machine.o
 
 # Sound
 sound-obj-y =
@@ -148,9 +153,9 @@ sound-obj-$(CONFIG_HDA) += intel-hda.o hda-audio.o
 
 $(obj)/adlib.o $(obj)/fmopl.o: QEMU_CFLAGS += -DBUILD_Y8950=0
 
-hw-obj-$(CONFIG_SOUND) += $(sound-obj-y)
+common-obj-$(CONFIG_SOUND) += $(sound-obj-y)
 
-hw-obj-$(CONFIG_REALLY_VIRTFS) += 9pfs/
+common-obj-$(CONFIG_REALLY_VIRTFS) += 9pfs/
 
 common-obj-y += usb/
 common-obj-y += irq.o
@@ -173,6 +178,7 @@ common-obj-y += scsi-disk.o cdrom.o hd-geometry.o block-common.o
 common-obj-y += scsi-generic.o scsi-bus.o
 common-obj-y += hid.o
 common-obj-$(CONFIG_SSI) += ssi.o
+common-obj-$(CONFIG_SSI_M25P80) += m25p80.o
 common-obj-$(CONFIG_SSI_SD) += ssi-sd.o
 common-obj-$(CONFIG_SD) += sd.o
 common-obj-y += bt.o bt-l2cap.o bt-sdp.o bt-hci.o bt-hid.o
@@ -198,7 +204,8 @@ obj-$(CONFIG_VGA) += vga.o
 obj-$(CONFIG_SOFTMMU) += device-hotplug.o
 obj-$(CONFIG_XEN) += xen_domainbuild.o xen_machine_pv.o
 
-# Inter-VM PCI shared memory
+# Inter-VM PCI shared memory & VFIO PCI device assignment
 ifeq ($(CONFIG_PCI), y)
 obj-$(CONFIG_KVM) += ivshmem.o
+obj-$(CONFIG_LINUX) += vfio_pci.o
 endif
diff --git a/hw/a9mpcore.c b/hw/a9mpcore.c
index ebd5b2917..824ff0a4d 100644
--- a/hw/a9mpcore.c
+++ b/hw/a9mpcore.c
@@ -26,7 +26,7 @@ typedef struct a9mp_priv_state {
     uint32_t num_irq;
 } a9mp_priv_state;
 
-static uint64_t a9_scu_read(void *opaque, target_phys_addr_t offset,
+static uint64_t a9_scu_read(void *opaque, hwaddr offset,
                             unsigned size)
 {
     a9mp_priv_state *s = (a9mp_priv_state *)opaque;
@@ -57,7 +57,7 @@ static uint64_t a9_scu_read(void *opaque, target_phys_addr_t offset,
     }
 }
 
-static void a9_scu_write(void *opaque, target_phys_addr_t offset,
+static void a9_scu_write(void *opaque, hwaddr offset,
                          uint64_t value, unsigned size)
 {
     a9mp_priv_state *s = (a9mp_priv_state *)opaque;
diff --git a/hw/ac97.c b/hw/ac97.c
index 0f561fa5c..ce6a1dc60 100644
--- a/hw/ac97.c
+++ b/hw/ac97.c
@@ -1226,32 +1226,101 @@ static const VMStateDescription vmstate_ac97 = {
     }
 };
 
-static const MemoryRegionPortio nam_portio[] = {
-    { 0, 256 * 1, 1, .read = nam_readb, },
-    { 0, 256 * 2, 2, .read = nam_readw, },
-    { 0, 256 * 4, 4, .read = nam_readl, },
-    { 0, 256 * 1, 1, .write = nam_writeb, },
-    { 0, 256 * 2, 2, .write = nam_writew, },
-    { 0, 256 * 4, 4, .write = nam_writel, },
-    PORTIO_END_OF_LIST (),
-};
+static uint64_t nam_read(void *opaque, hwaddr addr, unsigned size)
+{
+    if ((addr / size) > 256) {
+        return -1;
+    }
+
+    switch (size) {
+    case 1:
+        return nam_readb(opaque, addr);
+    case 2:
+        return nam_readw(opaque, addr);
+    case 4:
+        return nam_readl(opaque, addr);
+    default:
+        return -1;
+    }
+}
+
+static void nam_write(void *opaque, hwaddr addr, uint64_t val,
+                      unsigned size)
+{
+    if ((addr / size) > 256) {
+        return;
+    }
+
+    switch (size) {
+    case 1:
+        nam_writeb(opaque, addr, val);
+        break;
+    case 2:
+        nam_writew(opaque, addr, val);
+        break;
+    case 4:
+        nam_writel(opaque, addr, val);
+        break;
+    }
+}
 
 static const MemoryRegionOps ac97_io_nam_ops = {
-    .old_portio = nam_portio,
+    .read = nam_read,
+    .write = nam_write,
+    .impl = {
+        .min_access_size = 1,
+        .max_access_size = 4,
+    },
+    .endianness = DEVICE_LITTLE_ENDIAN,
 };
 
-static const MemoryRegionPortio nabm_portio[] = {
-    { 0, 64 * 1, 1, .read = nabm_readb, },
-    { 0, 64 * 2, 2, .read = nabm_readw, },
-    { 0, 64 * 4, 4, .read = nabm_readl, },
-    { 0, 64 * 1, 1, .write = nabm_writeb, },
-    { 0, 64 * 2, 2, .write = nabm_writew, },
-    { 0, 64 * 4, 4, .write = nabm_writel, },
-    PORTIO_END_OF_LIST ()
-};
+static uint64_t nabm_read(void *opaque, hwaddr addr, unsigned size)
+{
+    if ((addr / size) > 64) {
+        return -1;
+    }
+
+    switch (size) {
+    case 1:
+        return nabm_readb(opaque, addr);
+    case 2:
+        return nabm_readw(opaque, addr);
+    case 4:
+        return nabm_readl(opaque, addr);
+    default:
+        return -1;
+    }
+}
+
+static void nabm_write(void *opaque, hwaddr addr, uint64_t val,
+                      unsigned size)
+{
+    if ((addr / size) > 64) {
+        return;
+    }
+
+    switch (size) {
+    case 1:
+        nabm_writeb(opaque, addr, val);
+        break;
+    case 2:
+        nabm_writew(opaque, addr, val);
+        break;
+    case 4:
+        nabm_writel(opaque, addr, val);
+        break;
+    }
+}
+
 
 static const MemoryRegionOps ac97_io_nabm_ops = {
-    .old_portio = nabm_portio,
+    .read = nabm_read,
+    .write = nabm_write,
+    .impl = {
+        .min_access_size = 1,
+        .max_access_size = 4,
+    },
+    .endianness = DEVICE_LITTLE_ENDIAN,
 };
 
 static void ac97_on_reset (void *opaque)
diff --git a/hw/acpi.c b/hw/acpi.c
index f7950be26..f4aca493f 100644
--- a/hw/acpi.c
+++ b/hw/acpi.c
@@ -61,18 +61,6 @@ static int acpi_checksum(const uint8_t *data, int len)
     return (-sum) & 0xff;
 }
 
-/* like strncpy() but zero-fills the tail of destination */
-static void strzcpy(char *dst, const char *src, size_t size)
-{
-    size_t len = strlen(src);
-    if (len >= size) {
-        len = size;
-    } else {
-      memset(dst + len, 0, size - len);
-    }
-    memcpy(dst, src, len);
-}
-
 /* XXX fixme: this function uses obsolete argument parsing interface */
 int acpi_table_add(const char *t)
 {
@@ -157,7 +145,8 @@ int acpi_table_add(const char *t)
     hdr._length = cpu_to_le16(len);
 
     if (get_param_value(buf, sizeof(buf), "sig", t)) {
-        strzcpy(hdr.sig, buf, sizeof(hdr.sig));
+        /* strncpy is justified: the field need not be NUL-terminated. */
+        strncpy(hdr.sig, buf, sizeof(hdr.sig));
         ++changed;
     }
 
@@ -187,12 +176,14 @@ int acpi_table_add(const char *t)
     }
 
     if (get_param_value(buf, sizeof(buf), "oem_id", t)) {
-        strzcpy(hdr.oem_id, buf, sizeof(hdr.oem_id));
+        /* strncpy is justified: the field need not be NUL-terminated. */
+        strncpy(hdr.oem_id, buf, sizeof(hdr.oem_id));
         ++changed;
     }
 
     if (get_param_value(buf, sizeof(buf), "oem_table_id", t)) {
-        strzcpy(hdr.oem_table_id, buf, sizeof(hdr.oem_table_id));
+        /* strncpy is justified: the field need not be NUL-terminated. */
+        strncpy(hdr.oem_table_id, buf, sizeof(hdr.oem_table_id));
         ++changed;
     }
 
@@ -207,7 +198,8 @@ int acpi_table_add(const char *t)
     }
 
     if (get_param_value(buf, sizeof(buf), "asl_compiler_id", t)) {
-        strzcpy(hdr.asl_compiler_id, buf, sizeof(hdr.asl_compiler_id));
+        /* strncpy is justified: the field need not be NUL-terminated. */
+        strncpy(hdr.asl_compiler_id, buf, sizeof(hdr.asl_compiler_id));
         ++changed;
     }
 
diff --git a/hw/acpi_ich9.c b/hw/acpi_ich9.c
new file mode 100644
index 000000000..61034d3bd
--- /dev/null
+++ b/hw/acpi_ich9.c
@@ -0,0 +1,322 @@
+/*
+ * ACPI implementation
+ *
+ * Copyright (c) 2006 Fabrice Bellard
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2 as published by the Free Software Foundation.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>
+ */
+/*
+ *  Copyright (c) 2009 Isaku Yamahata <yamahata at valinux co jp>
+ *                     VA Linux Systems Japan K.K.
+ *  Copyright (C) 2012 Jason Baron <jbaron@redhat.com>
+ *
+ *  This is based on acpi.c.
+ */
+#include "hw.h"
+#include "pc.h"
+#include "pci.h"
+#include "qemu-timer.h"
+#include "sysemu.h"
+#include "acpi.h"
+#include "kvm.h"
+
+#include "ich9.h"
+
+//#define DEBUG
+
+#ifdef DEBUG
+#define ICH9_DEBUG(fmt, ...) \
+do { printf("%s "fmt, __func__, ## __VA_ARGS__); } while (0)
+#else
+#define ICH9_DEBUG(fmt, ...)    do { } while (0)
+#endif
+
+static void pm_ioport_write_fallback(void *opaque, uint32_t addr, int len,
+                                     uint32_t val);
+static uint32_t pm_ioport_read_fallback(void *opaque, uint32_t addr, int len);
+
+static void pm_update_sci(ICH9LPCPMRegs *pm)
+{
+    int sci_level, pm1a_sts;
+
+    pm1a_sts = acpi_pm1_evt_get_sts(&pm->acpi_regs);
+
+    sci_level = (((pm1a_sts & pm->acpi_regs.pm1.evt.en) &
+                  (ACPI_BITMASK_RT_CLOCK_ENABLE |
+                   ACPI_BITMASK_POWER_BUTTON_ENABLE |
+                   ACPI_BITMASK_GLOBAL_LOCK_ENABLE |
+                   ACPI_BITMASK_TIMER_ENABLE)) != 0);
+    qemu_set_irq(pm->irq, sci_level);
+
+    /* schedule a timer interruption if needed */
+    acpi_pm_tmr_update(&pm->acpi_regs,
+                       (pm->acpi_regs.pm1.evt.en & ACPI_BITMASK_TIMER_ENABLE) &&
+                       !(pm1a_sts & ACPI_BITMASK_TIMER_STATUS));
+}
+
+static void ich9_pm_update_sci_fn(ACPIREGS *regs)
+{
+    ICH9LPCPMRegs *pm = container_of(regs, ICH9LPCPMRegs, acpi_regs);
+    pm_update_sci(pm);
+}
+
+static void pm_ioport_writeb(void *opaque, uint32_t addr, uint32_t val)
+{
+    ICH9LPCPMRegs *pm = opaque;
+
+    switch (addr & ICH9_PMIO_MASK) {
+    case ICH9_PMIO_GPE0_STS ... (ICH9_PMIO_GPE0_STS + ICH9_PMIO_GPE0_LEN - 1):
+        acpi_gpe_ioport_writeb(&pm->acpi_regs, addr, val);
+        break;
+    default:
+        break;
+    }
+
+    ICH9_DEBUG("port=0x%04x val=0x%04x\n", addr, val);
+}
+
+static uint32_t pm_ioport_readb(void *opaque, uint32_t addr)
+{
+    ICH9LPCPMRegs *pm = opaque;
+    uint32_t val = 0;
+
+    switch (addr & ICH9_PMIO_MASK) {
+    case ICH9_PMIO_GPE0_STS ... (ICH9_PMIO_GPE0_STS + ICH9_PMIO_GPE0_LEN - 1):
+        val = acpi_gpe_ioport_readb(&pm->acpi_regs, addr);
+        break;
+    default:
+        val = 0;
+        break;
+    }
+    ICH9_DEBUG("port=0x%04x val=0x%04x\n", addr, val);
+    return val;
+}
+
+static void pm_ioport_writew(void *opaque, uint32_t addr, uint32_t val)
+{
+    ICH9LPCPMRegs *pm = opaque;
+
+    switch (addr & ICH9_PMIO_MASK) {
+    case ICH9_PMIO_PM1_STS:
+        acpi_pm1_evt_write_sts(&pm->acpi_regs, val);
+        pm_update_sci(pm);
+        break;
+    case ICH9_PMIO_PM1_EN:
+        pm->acpi_regs.pm1.evt.en = val;
+        pm_update_sci(pm);
+        break;
+    case ICH9_PMIO_PM1_CNT:
+        acpi_pm1_cnt_write(&pm->acpi_regs, val, 0);
+        break;
+    default:
+        pm_ioport_write_fallback(opaque, addr, 2, val);
+        break;
+    }
+    ICH9_DEBUG("port=0x%04x val=0x%04x\n", addr, val);
+}
+
+static uint32_t pm_ioport_readw(void *opaque, uint32_t addr)
+{
+    ICH9LPCPMRegs *pm = opaque;
+    uint32_t val;
+
+    switch (addr & ICH9_PMIO_MASK) {
+    case ICH9_PMIO_PM1_STS:
+        val = acpi_pm1_evt_get_sts(&pm->acpi_regs);
+        break;
+    case ICH9_PMIO_PM1_EN:
+        val = pm->acpi_regs.pm1.evt.en;
+        break;
+    case ICH9_PMIO_PM1_CNT:
+        val = pm->acpi_regs.pm1.cnt.cnt;
+        break;
+    default:
+        val = pm_ioport_read_fallback(opaque, addr, 2);
+        break;
+    }
+    ICH9_DEBUG("port=0x%04x val=0x%04x\n", addr, val);
+    return val;
+}
+
+static void pm_ioport_writel(void *opaque, uint32_t addr, uint32_t val)
+{
+    ICH9LPCPMRegs *pm = opaque;
+
+    switch (addr & ICH9_PMIO_MASK) {
+    case ICH9_PMIO_SMI_EN:
+        pm->smi_en = val;
+        break;
+    default:
+        pm_ioport_write_fallback(opaque, addr, 4, val);
+        break;
+    }
+    ICH9_DEBUG("port=0x%04x val=0x%08x\n", addr, val);
+}
+
+static uint32_t pm_ioport_readl(void *opaque, uint32_t addr)
+{
+    ICH9LPCPMRegs *pm = opaque;
+    uint32_t val;
+
+    switch (addr & ICH9_PMIO_MASK) {
+    case ICH9_PMIO_PM1_TMR:
+        val = acpi_pm_tmr_get(&pm->acpi_regs);
+        break;
+    case ICH9_PMIO_SMI_EN:
+        val = pm->smi_en;
+        break;
+
+    default:
+        val = pm_ioport_read_fallback(opaque, addr, 4);
+        break;
+    }
+    ICH9_DEBUG("port=0x%04x val=0x%08x\n", addr, val);
+    return val;
+}
+
+static void pm_ioport_write_fallback(void *opaque, uint32_t addr, int len,
+                                     uint32_t val)
+ {
+    int subsize = (len == 4) ? 2 : 1;
+    IOPortWriteFunc *ioport_write =
+        (subsize == 2) ? pm_ioport_writew : pm_ioport_writeb;
+
+    int i;
+
+    for (i = 0; i < len; i += subsize) {
+        ioport_write(opaque, addr, val);
+        val >>= 8 * subsize;
+    }
+}
+
+static uint32_t pm_ioport_read_fallback(void *opaque, uint32_t addr, int len)
+{
+    int subsize = (len == 4) ? 2 : 1;
+    IOPortReadFunc *ioport_read =
+        (subsize == 2) ? pm_ioport_readw : pm_ioport_readb;
+
+    uint32_t val;
+    int i;
+
+    val = 0;
+    for (i = 0; i < len; i += subsize) {
+        val <<= 8 * subsize;
+        val |= ioport_read(opaque, addr);
+    }
+
+    return val;
+}
+
+void ich9_pm_iospace_update(ICH9LPCPMRegs *pm, uint32_t pm_io_base)
+{
+    ICH9_DEBUG("to 0x%x\n", pm_io_base);
+
+    assert((pm_io_base & ICH9_PMIO_MASK) == 0);
+
+    if (pm->pm_io_base != 0) {
+        isa_unassign_ioport(pm->pm_io_base, ICH9_PMIO_SIZE);
+    }
+
+    /* don't map at 0 */
+    if (pm_io_base == 0) {
+        return;
+    }
+
+    register_ioport_write(pm_io_base, ICH9_PMIO_SIZE, 1, pm_ioport_writeb, pm);
+    register_ioport_read(pm_io_base, ICH9_PMIO_SIZE, 1, pm_ioport_readb, pm);
+    register_ioport_write(pm_io_base, ICH9_PMIO_SIZE, 2, pm_ioport_writew, pm);
+    register_ioport_read(pm_io_base, ICH9_PMIO_SIZE, 2, pm_ioport_readw, pm);
+    register_ioport_write(pm_io_base, ICH9_PMIO_SIZE, 4, pm_ioport_writel, pm);
+    register_ioport_read(pm_io_base, ICH9_PMIO_SIZE, 4, pm_ioport_readl, pm);
+
+    pm->pm_io_base = pm_io_base;
+    acpi_gpe_blk(&pm->acpi_regs, pm_io_base + ICH9_PMIO_GPE0_STS);
+}
+
+static int ich9_pm_post_load(void *opaque, int version_id)
+{
+    ICH9LPCPMRegs *pm = opaque;
+    uint32_t pm_io_base = pm->pm_io_base;
+    pm->pm_io_base = 0;
+    ich9_pm_iospace_update(pm, pm_io_base);
+    return 0;
+}
+
+#define VMSTATE_GPE_ARRAY(_field, _state)                            \
+ {                                                                   \
+     .name       = (stringify(_field)),                              \
+     .version_id = 0,                                                \
+     .num        = ICH9_PMIO_GPE0_LEN,                               \
+     .info       = &vmstate_info_uint8,                              \
+     .size       = sizeof(uint8_t),                                  \
+     .flags      = VMS_ARRAY | VMS_POINTER,                          \
+     .offset     = vmstate_offset_pointer(_state, _field, uint8_t),  \
+ }
+
+const VMStateDescription vmstate_ich9_pm = {
+    .name = "ich9_pm",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .minimum_version_id_old = 1,
+    .post_load = ich9_pm_post_load,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT16(acpi_regs.pm1.evt.sts, ICH9LPCPMRegs),
+        VMSTATE_UINT16(acpi_regs.pm1.evt.en, ICH9LPCPMRegs),
+        VMSTATE_UINT16(acpi_regs.pm1.cnt.cnt, ICH9LPCPMRegs),
+        VMSTATE_TIMER(acpi_regs.tmr.timer, ICH9LPCPMRegs),
+        VMSTATE_INT64(acpi_regs.tmr.overflow_time, ICH9LPCPMRegs),
+        VMSTATE_GPE_ARRAY(acpi_regs.gpe.sts, ICH9LPCPMRegs),
+        VMSTATE_GPE_ARRAY(acpi_regs.gpe.en, ICH9LPCPMRegs),
+        VMSTATE_UINT32(smi_en, ICH9LPCPMRegs),
+        VMSTATE_UINT32(smi_sts, ICH9LPCPMRegs),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static void pm_reset(void *opaque)
+{
+    ICH9LPCPMRegs *pm = opaque;
+    ich9_pm_iospace_update(pm, 0);
+
+    acpi_pm1_evt_reset(&pm->acpi_regs);
+    acpi_pm1_cnt_reset(&pm->acpi_regs);
+    acpi_pm_tmr_reset(&pm->acpi_regs);
+    acpi_gpe_reset(&pm->acpi_regs);
+
+    if (kvm_enabled()) {
+        /* Mark SMM as already inited to prevent SMM from running. KVM does not
+         * support SMM mode. */
+        pm->smi_en |= ICH9_PMIO_SMI_EN_APMC_EN;
+    }
+
+    pm_update_sci(pm);
+}
+
+static void pm_powerdown_req(Notifier *n, void *opaque)
+{
+    ICH9LPCPMRegs *pm = container_of(n, ICH9LPCPMRegs, powerdown_notifier);
+
+    acpi_pm1_evt_power_down(&pm->acpi_regs);
+}
+
+void ich9_pm_init(ICH9LPCPMRegs *pm, qemu_irq sci_irq, qemu_irq cmos_s3)
+{
+    acpi_pm_tmr_init(&pm->acpi_regs, ich9_pm_update_sci_fn);
+    acpi_pm1_cnt_init(&pm->acpi_regs);
+    acpi_gpe_init(&pm->acpi_regs, ICH9_PMIO_GPE0_LEN);
+
+    pm->irq = sci_irq;
+    qemu_register_reset(pm_reset, pm);
+    pm->powerdown_notifier.notify = pm_powerdown_req;
+    qemu_register_powerdown_notifier(&pm->powerdown_notifier);
+}
diff --git a/hw/acpi_ich9.h b/hw/acpi_ich9.h
new file mode 100644
index 000000000..180c40673
--- /dev/null
+++ b/hw/acpi_ich9.h
@@ -0,0 +1,47 @@
+/*
+ * QEMU GMCH/ICH9 LPC PM Emulation
+ *
+ *  Copyright (c) 2009 Isaku Yamahata <yamahata at valinux co jp>
+ *                     VA Linux Systems Japan K.K.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>
+ */
+
+#ifndef HW_ACPI_ICH9_H
+#define HW_ACPI_ICH9_H
+
+#include "acpi.h"
+
+typedef struct ICH9LPCPMRegs {
+    /*
+     * In ich9 spec says that pm1_cnt register is 32bit width and
+     * that the upper 16bits are reserved and unused.
+     * PM1a_CNT_BLK = 2 in FADT so it is defined as uint16_t.
+     */
+    ACPIREGS acpi_regs;
+    uint32_t smi_en;
+    uint32_t smi_sts;
+
+    qemu_irq irq;      /* SCI */
+
+    uint32_t pm_io_base;
+    Notifier powerdown_notifier;
+} ICH9LPCPMRegs;
+
+void ich9_pm_init(ICH9LPCPMRegs *pm,
+                  qemu_irq sci_irq, qemu_irq cmos_s3_resume);
+void ich9_pm_iospace_update(ICH9LPCPMRegs *pm, uint32_t pm_io_base);
+extern const VMStateDescription vmstate_ich9_pm;
+
+#endif /* HW_ACPI_ICH9_H */
diff --git a/hw/acpi_piix4.c b/hw/acpi_piix4.c
index c56220b36..519269a01 100644
--- a/hw/acpi_piix4.c
+++ b/hw/acpi_piix4.c
@@ -67,6 +67,7 @@ typedef struct PIIX4PMState {
     qemu_irq smi_irq;
     int kvm_enabled;
     Notifier machine_ready;
+    Notifier powerdown_notifier;
 
     /* for pci hotplug */
     struct pci_status pci0_status;
@@ -234,10 +235,9 @@ static int vmstate_acpi_post_load(void *opaque, int version_id)
  {                                                                   \
      .name       = (stringify(_field)),                              \
      .version_id = 0,                                                \
-     .num        = GPE_LEN,                                          \
      .info       = &vmstate_info_uint16,                             \
      .size       = sizeof(uint16_t),                                 \
-     .flags      = VMS_ARRAY | VMS_POINTER,                          \
+     .flags      = VMS_SINGLE | VMS_POINTER,                         \
      .offset     = vmstate_offset_pointer(_state, _field, uint8_t),  \
  }
 
@@ -266,11 +266,54 @@ static const VMStateDescription vmstate_pci_status = {
     }
 };
 
+static int acpi_load_old(QEMUFile *f, void *opaque, int version_id)
+{
+    PIIX4PMState *s = opaque;
+    int ret, i;
+    uint16_t temp;
+
+    ret = pci_device_load(&s->dev, f);
+    if (ret < 0) {
+        return ret;
+    }
+    qemu_get_be16s(f, &s->ar.pm1.evt.sts);
+    qemu_get_be16s(f, &s->ar.pm1.evt.en);
+    qemu_get_be16s(f, &s->ar.pm1.cnt.cnt);
+
+    ret = vmstate_load_state(f, &vmstate_apm, opaque, 1);
+    if (ret) {
+        return ret;
+    }
+
+    qemu_get_timer(f, s->ar.tmr.timer);
+    qemu_get_sbe64s(f, &s->ar.tmr.overflow_time);
+
+    qemu_get_be16s(f, (uint16_t *)s->ar.gpe.sts);
+    for (i = 0; i < 3; i++) {
+        qemu_get_be16s(f, &temp);
+    }
+
+    qemu_get_be16s(f, (uint16_t *)s->ar.gpe.en);
+    for (i = 0; i < 3; i++) {
+        qemu_get_be16s(f, &temp);
+    }
+
+    ret = vmstate_load_state(f, &vmstate_pci_status, opaque, 1);
+    return ret;
+}
+
+/* qemu-kvm 1.2 uses version 3 but advertised as 2
+ * To support incoming qemu-kvm 1.2 migration, change version_id
+ * and minimum_version_id to 2 below (which breaks migration from
+ * qemu 1.2).
+ *
+ */
 static const VMStateDescription vmstate_acpi = {
     .name = "piix4_pm",
-    .version_id = 2,
-    .minimum_version_id = 1,
+    .version_id = 3,
+    .minimum_version_id = 3,
     .minimum_version_id_old = 1,
+    .load_state_old = acpi_load_old,
     .post_load = vmstate_acpi_post_load,
     .fields      = (VMStateField []) {
         VMSTATE_PCI_DEVICE(dev, PIIX4PMState),
@@ -362,9 +405,9 @@ static void piix4_reset(void *opaque)
     piix4_update_hotplug(s);
 }
 
-static void piix4_powerdown(void *opaque, int irq, int power_failing)
+static void piix4_pm_powerdown_req(Notifier *n, void *opaque)
 {
-    PIIX4PMState *s = opaque;
+    PIIX4PMState *s = container_of(n, PIIX4PMState, powerdown_notifier);
 
     assert(s != NULL);
     acpi_pm1_evt_power_down(&s->ar);
@@ -416,7 +459,8 @@ static int piix4_pm_initfn(PCIDevice *dev)
     acpi_pm_tmr_init(&s->ar, pm_tmr_timer);
     acpi_gpe_init(&s->ar, GPE_LEN);
 
-    qemu_system_powerdown = *qemu_allocate_irqs(piix4_powerdown, s, 1);
+    s->powerdown_notifier.notify = piix4_pm_powerdown_req;
+    qemu_register_powerdown_notifier(&s->powerdown_notifier);
 
     pm_smbus_init(&s->dev.qdev, &s->smb);
     s->machine_ready.notify = piix4_pm_machine_ready;
diff --git a/hw/adb.c b/hw/adb.c
index aa15f55dc..3b547f0af 100644
--- a/hw/adb.c
+++ b/hw/adb.c
@@ -108,10 +108,10 @@ int adb_poll(ADBBusState *s, uint8_t *obuf)
     return olen;
 }
 
-ADBDevice *adb_register_device(ADBBusState *s, int devaddr,
-                               ADBDeviceRequest *devreq,
-                               ADBDeviceReset *devreset,
-                               void *opaque)
+static ADBDevice *adb_register_device(ADBBusState *s, int devaddr,
+                                      ADBDeviceRequest *devreq,
+                                      ADBDeviceReset *devreset,
+                                      void *opaque)
 {
     ADBDevice *d;
     if (s->nb_devices >= MAX_ADB_DEVICES)
diff --git a/hw/adb.h b/hw/adb.h
index b2a591c54..5b27da2dd 100644
--- a/hw/adb.h
+++ b/hw/adb.h
@@ -56,10 +56,6 @@ int adb_request(ADBBusState *s, uint8_t *buf_out,
                 const uint8_t *buf, int len);
 int adb_poll(ADBBusState *s, uint8_t *buf_out);
 
-ADBDevice *adb_register_device(ADBBusState *s, int devaddr,
-                               ADBDeviceRequest *devreq,
-                               ADBDeviceReset *devreset,
-                               void *opaque);
 void adb_kbd_init(ADBBusState *bus);
 void adb_mouse_init(ADBBusState *bus);
 
diff --git a/hw/ads7846.c b/hw/ads7846.c
index 41c7f101c..2ea9e55bb 100644
--- a/hw/ads7846.c
+++ b/hw/ads7846.c
@@ -119,11 +119,12 @@ static int ads7856_post_load(void *opaque, int version_id)
 
 static const VMStateDescription vmstate_ads7846 = {
     .name = "ads7846",
-    .version_id = 0,
-    .minimum_version_id = 0,
-    .minimum_version_id_old = 0,
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .minimum_version_id_old = 1,
     .post_load = ads7856_post_load,
     .fields      = (VMStateField[]) {
+        VMSTATE_SSI_SLAVE(ssidev, ADS7846State),
         VMSTATE_INT32_ARRAY(input, ADS7846State, 8),
         VMSTATE_INT32(noise, ADS7846State),
         VMSTATE_INT32(cycle, ADS7846State),
diff --git a/hw/alpha_dp264.c b/hw/alpha_dp264.c
index 9eb939f38..76d8ae8a8 100644
--- a/hw/alpha_dp264.c
+++ b/hw/alpha_dp264.c
@@ -15,6 +15,7 @@
 #include "mc146818rtc.h"
 #include "ide.h"
 #include "i8254.h"
+#include "serial.h"
 
 #define MAX_IDE_BUS 2
 
@@ -42,13 +43,13 @@ static int clipper_pci_map_irq(PCIDevice *d, int irq_num)
     return (slot + 1) * 4 + irq_num;
 }
 
-static void clipper_init(ram_addr_t ram_size,
-                         const char *boot_device,
-                         const char *kernel_filename,
-                         const char *kernel_cmdline,
-                         const char *initrd_filename,
-                         const char *cpu_model)
+static void clipper_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t ram_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
     CPUAlphaState *cpus[4];
     PCIBus *pci_bus;
     ISABus *isa_bus;
@@ -77,7 +78,7 @@ static void clipper_init(ram_addr_t ram_size,
     isa_create_simple(isa_bus, "i8042");
 
     /* VGA setup.  Don't bother loading the bios.  */
-    alpha_pci_vga_setup(pci_bus);
+    pci_vga_init(pci_bus);
 
     /* Serial code setup.  */
     for (i = 0; i < MAX_SERIAL_PORTS; ++i) {
diff --git a/hw/alpha_pci.c b/hw/alpha_pci.c
index ea546f83f..7e7b1d27d 100644
--- a/hw/alpha_pci.c
+++ b/hw/alpha_pci.c
@@ -10,14 +10,12 @@
 #include "alpha_sys.h"
 #include "qemu-log.h"
 #include "sysemu.h"
-#include "vmware_vga.h"
-#include "vga-pci.h"
 
 
 /* PCI IO reads/writes, to byte-word addressable memory.  */
 /* ??? Doesn't handle multiple PCI busses.  */
 
-static uint64_t bw_io_read(void *opaque, target_phys_addr_t addr, unsigned size)
+static uint64_t bw_io_read(void *opaque, hwaddr addr, unsigned size)
 {
     switch (size) {
     case 1:
@@ -30,7 +28,7 @@ static uint64_t bw_io_read(void *opaque, target_phys_addr_t addr, unsigned size)
     abort();
 }
 
-static void bw_io_write(void *opaque, target_phys_addr_t addr,
+static void bw_io_write(void *opaque, hwaddr addr,
                         uint64_t val, unsigned size)
 {
     switch (size) {
@@ -59,14 +57,14 @@ const MemoryRegionOps alpha_pci_bw_io_ops = {
 };
 
 /* PCI config space reads/writes, to byte-word addressable memory.  */
-static uint64_t bw_conf1_read(void *opaque, target_phys_addr_t addr,
+static uint64_t bw_conf1_read(void *opaque, hwaddr addr,
                               unsigned size)
 {
     PCIBus *b = opaque;
     return pci_data_read(b, addr, size);
 }
 
-static void bw_conf1_write(void *opaque, target_phys_addr_t addr,
+static void bw_conf1_write(void *opaque, hwaddr addr,
                            uint64_t val, unsigned size)
 {
     PCIBus *b = opaque;
@@ -85,12 +83,12 @@ const MemoryRegionOps alpha_pci_conf1_ops = {
 
 /* PCI/EISA Interrupt Acknowledge Cycle.  */
 
-static uint64_t iack_read(void *opaque, target_phys_addr_t addr, unsigned size)
+static uint64_t iack_read(void *opaque, hwaddr addr, unsigned size)
 {
     return pic_read_irq(isa_pic);
 }
 
-static void special_write(void *opaque, target_phys_addr_t addr,
+static void special_write(void *opaque, hwaddr addr,
                           uint64_t val, unsigned size)
 {
     qemu_log("pci: special write cycle");
@@ -109,25 +107,3 @@ const MemoryRegionOps alpha_pci_iack_ops = {
         .max_access_size = 4,
     },
 };
-
-void alpha_pci_vga_setup(PCIBus *pci_bus)
-{
-    switch (vga_interface_type) {
-#ifdef CONFIG_SPICE
-    case VGA_QXL:
-        pci_create_simple(pci_bus, -1, "qxl-vga");
-        return;
-#endif
-    case VGA_CIRRUS:
-        pci_cirrus_vga_init(pci_bus);
-        return;
-    case VGA_VMWARE:
-        pci_vmsvga_init(pci_bus);
-        return;
-    }
-    /* If VGA is enabled at all, and one of the above didn't work, then
-       fallback to Standard VGA.  */
-    if (vga_interface_type != VGA_NONE) {
-        pci_vga_init(pci_bus);
-    }
-}
diff --git a/hw/alpha_sys.h b/hw/alpha_sys.h
index de40f8b61..7604d09c8 100644
--- a/hw/alpha_sys.h
+++ b/hw/alpha_sys.h
@@ -19,6 +19,4 @@ extern const MemoryRegionOps alpha_pci_bw_io_ops;
 extern const MemoryRegionOps alpha_pci_conf1_ops;
 extern const MemoryRegionOps alpha_pci_iack_ops;
 
-void alpha_pci_vga_setup(PCIBus *pci_bus);
-
 #endif
diff --git a/hw/alpha_typhoon.c b/hw/alpha_typhoon.c
index b7cf4e290..9b16d9661 100644
--- a/hw/alpha_typhoon.c
+++ b/hw/alpha_typhoon.c
@@ -70,7 +70,7 @@ static void cpu_irq_change(CPUAlphaState *env, uint64_t req)
     }
 }
 
-static uint64_t cchip_read(void *opaque, target_phys_addr_t addr, unsigned size)
+static uint64_t cchip_read(void *opaque, hwaddr addr, unsigned size)
 {
     CPUAlphaState *env = cpu_single_env;
     TyphoonState *s = opaque;
@@ -203,13 +203,13 @@ static uint64_t cchip_read(void *opaque, target_phys_addr_t addr, unsigned size)
     return ret;
 }
 
-static uint64_t dchip_read(void *opaque, target_phys_addr_t addr, unsigned size)
+static uint64_t dchip_read(void *opaque, hwaddr addr, unsigned size)
 {
     /* Skip this.  It's all related to DRAM timing and setup.  */
     return 0;
 }
 
-static uint64_t pchip_read(void *opaque, target_phys_addr_t addr, unsigned size)
+static uint64_t pchip_read(void *opaque, hwaddr addr, unsigned size)
 {
     TyphoonState *s = opaque;
     uint64_t ret = 0;
@@ -306,7 +306,7 @@ static uint64_t pchip_read(void *opaque, target_phys_addr_t addr, unsigned size)
     return ret;
 }
 
-static void cchip_write(void *opaque, target_phys_addr_t addr,
+static void cchip_write(void *opaque, hwaddr addr,
                         uint64_t v32, unsigned size)
 {
     TyphoonState *s = opaque;
@@ -463,13 +463,13 @@ static void cchip_write(void *opaque, target_phys_addr_t addr,
     }
 }
 
-static void dchip_write(void *opaque, target_phys_addr_t addr,
+static void dchip_write(void *opaque, hwaddr addr,
                         uint64_t val, unsigned size)
 {
     /* Skip this.  It's all related to DRAM timing and setup.  */
 }
 
-static void pchip_write(void *opaque, target_phys_addr_t addr,
+static void pchip_write(void *opaque, hwaddr addr,
                         uint64_t v32, unsigned size)
 {
     TyphoonState *s = opaque;
diff --git a/hw/an5206.c b/hw/an5206.c
index 25407c0f5..d887c0e7e 100644
--- a/hw/an5206.c
+++ b/hw/an5206.c
@@ -19,15 +19,15 @@
 
 /* Board init.  */
 
-static void an5206_init(ram_addr_t ram_size,
-                     const char *boot_device,
-                     const char *kernel_filename, const char *kernel_cmdline,
-                     const char *initrd_filename, const char *cpu_model)
+static void an5206_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t ram_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
     CPUM68KState *env;
     int kernel_size;
     uint64_t elf_entry;
-    target_phys_addr_t entry;
+    hwaddr entry;
     MemoryRegion *address_space_mem = get_system_memory();
     MemoryRegion *ram = g_new(MemoryRegion, 1);
     MemoryRegion *sram = g_new(MemoryRegion, 1);
diff --git a/hw/apb_pci.c b/hw/apb_pci.c
index c28411a46..054814fd4 100644
--- a/hw/apb_pci.c
+++ b/hw/apb_pci.c
@@ -87,7 +87,7 @@ typedef struct APBState {
 
 static void pci_apb_set_irq(void *opaque, int irq_num, int level);
 
-static void apb_config_writel (void *opaque, target_phys_addr_t addr,
+static void apb_config_writel (void *opaque, hwaddr addr,
                                uint64_t val, unsigned size)
 {
     APBState *s = opaque;
@@ -152,7 +152,7 @@ static void apb_config_writel (void *opaque, target_phys_addr_t addr,
 }
 
 static uint64_t apb_config_readl (void *opaque,
-                                  target_phys_addr_t addr, unsigned size)
+                                  hwaddr addr, unsigned size)
 {
     APBState *s = opaque;
     uint32_t val;
@@ -212,7 +212,7 @@ static const MemoryRegionOps apb_config_ops = {
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-static void apb_pci_config_write(void *opaque, target_phys_addr_t addr,
+static void apb_pci_config_write(void *opaque, hwaddr addr,
                                  uint64_t val, unsigned size)
 {
     APBState *s = opaque;
@@ -222,7 +222,7 @@ static void apb_pci_config_write(void *opaque, target_phys_addr_t addr,
     pci_data_write(s->bus, addr, val, size);
 }
 
-static uint64_t apb_pci_config_read(void *opaque, target_phys_addr_t addr,
+static uint64_t apb_pci_config_read(void *opaque, hwaddr addr,
                                     unsigned size)
 {
     uint32_t ret;
@@ -234,25 +234,25 @@ static uint64_t apb_pci_config_read(void *opaque, target_phys_addr_t addr,
     return ret;
 }
 
-static void pci_apb_iowriteb (void *opaque, target_phys_addr_t addr,
+static void pci_apb_iowriteb (void *opaque, hwaddr addr,
                                   uint32_t val)
 {
     cpu_outb(addr & IOPORTS_MASK, val);
 }
 
-static void pci_apb_iowritew (void *opaque, target_phys_addr_t addr,
+static void pci_apb_iowritew (void *opaque, hwaddr addr,
                                   uint32_t val)
 {
     cpu_outw(addr & IOPORTS_MASK, bswap16(val));
 }
 
-static void pci_apb_iowritel (void *opaque, target_phys_addr_t addr,
+static void pci_apb_iowritel (void *opaque, hwaddr addr,
                                 uint32_t val)
 {
     cpu_outl(addr & IOPORTS_MASK, bswap32(val));
 }
 
-static uint32_t pci_apb_ioreadb (void *opaque, target_phys_addr_t addr)
+static uint32_t pci_apb_ioreadb (void *opaque, hwaddr addr)
 {
     uint32_t val;
 
@@ -260,7 +260,7 @@ static uint32_t pci_apb_ioreadb (void *opaque, target_phys_addr_t addr)
     return val;
 }
 
-static uint32_t pci_apb_ioreadw (void *opaque, target_phys_addr_t addr)
+static uint32_t pci_apb_ioreadw (void *opaque, hwaddr addr)
 {
     uint32_t val;
 
@@ -268,7 +268,7 @@ static uint32_t pci_apb_ioreadw (void *opaque, target_phys_addr_t addr)
     return val;
 }
 
-static uint32_t pci_apb_ioreadl (void *opaque, target_phys_addr_t addr)
+static uint32_t pci_apb_ioreadl (void *opaque, hwaddr addr)
 {
     uint32_t val;
 
@@ -351,8 +351,8 @@ static int apb_pci_bridge_initfn(PCIDevice *dev)
     return 0;
 }
 
-PCIBus *pci_apb_init(target_phys_addr_t special_base,
-                     target_phys_addr_t mem_base,
+PCIBus *pci_apb_init(hwaddr special_base,
+                     hwaddr mem_base,
                      qemu_irq *ivec_irqs, PCIBus **bus2, PCIBus **bus3,
                      qemu_irq **pbm_irqs)
 {
diff --git a/hw/apb_pci.h b/hw/apb_pci.h
index 55f7c4c3b..736db6118 100644
--- a/hw/apb_pci.h
+++ b/hw/apb_pci.h
@@ -3,8 +3,8 @@
 
 #include "qemu-common.h"
 
-PCIBus *pci_apb_init(target_phys_addr_t special_base,
-                     target_phys_addr_t mem_base,
+PCIBus *pci_apb_init(hwaddr special_base,
+                     hwaddr mem_base,
                      qemu_irq *ivec_irqs, PCIBus **bus2, PCIBus **bus3,
                      qemu_irq **pbm_irqs);
 #endif
diff --git a/hw/apic.c b/hw/apic.c
index 385555eb4..f73fc877a 100644
--- a/hw/apic.c
+++ b/hw/apic.c
@@ -107,7 +107,7 @@ static void apic_sync_vapic(APICCommonState *s, int sync_type)
         length = offsetof(VAPICState, enabled) - offsetof(VAPICState, isr);
 
         if (sync_type & SYNC_TO_VAPIC) {
-            assert(qemu_cpu_is_self(s->cpu_env));
+            assert(qemu_cpu_is_self(CPU(s->cpu)));
 
             vapic_state.tpr = s->tpr;
             vapic_state.enabled = 1;
@@ -151,15 +151,15 @@ static void apic_local_deliver(APICCommonState *s, int vector)
 
     switch ((lvt >> 8) & 7) {
     case APIC_DM_SMI:
-        cpu_interrupt(s->cpu_env, CPU_INTERRUPT_SMI);
+        cpu_interrupt(&s->cpu->env, CPU_INTERRUPT_SMI);
         break;
 
     case APIC_DM_NMI:
-        cpu_interrupt(s->cpu_env, CPU_INTERRUPT_NMI);
+        cpu_interrupt(&s->cpu->env, CPU_INTERRUPT_NMI);
         break;
 
     case APIC_DM_EXTINT:
-        cpu_interrupt(s->cpu_env, CPU_INTERRUPT_HARD);
+        cpu_interrupt(&s->cpu->env, CPU_INTERRUPT_HARD);
         break;
 
     case APIC_DM_FIXED:
@@ -187,7 +187,7 @@ void apic_deliver_pic_intr(DeviceState *d, int level)
             reset_bit(s->irr, lvt & 0xff);
             /* fall through */
         case APIC_DM_EXTINT:
-            cpu_reset_interrupt(s->cpu_env, CPU_INTERRUPT_HARD);
+            cpu_reset_interrupt(&s->cpu->env, CPU_INTERRUPT_HARD);
             break;
         }
     }
@@ -248,18 +248,22 @@ static void apic_bus_deliver(const uint32_t *deliver_bitmask,
 
         case APIC_DM_SMI:
             foreach_apic(apic_iter, deliver_bitmask,
-                cpu_interrupt(apic_iter->cpu_env, CPU_INTERRUPT_SMI) );
+                cpu_interrupt(&apic_iter->cpu->env, CPU_INTERRUPT_SMI)
+            );
             return;
 
         case APIC_DM_NMI:
             foreach_apic(apic_iter, deliver_bitmask,
-                cpu_interrupt(apic_iter->cpu_env, CPU_INTERRUPT_NMI) );
+                cpu_interrupt(&apic_iter->cpu->env, CPU_INTERRUPT_NMI)
+            );
             return;
 
         case APIC_DM_INIT:
             /* normal INIT IPI sent to processors */
             foreach_apic(apic_iter, deliver_bitmask,
-                         cpu_interrupt(apic_iter->cpu_env, CPU_INTERRUPT_INIT) );
+                         cpu_interrupt(&apic_iter->cpu->env,
+                                       CPU_INTERRUPT_INIT)
+            );
             return;
 
         case APIC_DM_EXTINT:
@@ -293,7 +297,7 @@ static void apic_set_base(APICCommonState *s, uint64_t val)
     /* if disabled, cannot be enabled again */
     if (!(val & MSR_IA32_APICBASE_ENABLE)) {
         s->apicbase &= ~MSR_IA32_APICBASE_ENABLE;
-        cpu_clear_apic_feature(s->cpu_env);
+        cpu_clear_apic_feature(&s->cpu->env);
         s->spurious_vec &= ~APIC_SV_ENABLE;
     }
 }
@@ -359,13 +363,15 @@ static int apic_irq_pending(APICCommonState *s)
 /* signal the CPU if an irq is pending */
 static void apic_update_irq(APICCommonState *s)
 {
+    CPUState *cpu = CPU(s->cpu);
+
     if (!(s->spurious_vec & APIC_SV_ENABLE)) {
         return;
     }
-    if (!qemu_cpu_is_self(s->cpu_env)) {
-        cpu_interrupt(s->cpu_env, CPU_INTERRUPT_POLL);
+    if (!qemu_cpu_is_self(cpu)) {
+        cpu_interrupt(&s->cpu->env, CPU_INTERRUPT_POLL);
     } else if (apic_irq_pending(s) > 0) {
-        cpu_interrupt(s->cpu_env, CPU_INTERRUPT_HARD);
+        cpu_interrupt(&s->cpu->env, CPU_INTERRUPT_HARD);
     }
 }
 
@@ -472,18 +478,18 @@ static void apic_get_delivery_bitmask(uint32_t *deliver_bitmask,
 static void apic_startup(APICCommonState *s, int vector_num)
 {
     s->sipi_vector = vector_num;
-    cpu_interrupt(s->cpu_env, CPU_INTERRUPT_SIPI);
+    cpu_interrupt(&s->cpu->env, CPU_INTERRUPT_SIPI);
 }
 
 void apic_sipi(DeviceState *d)
 {
     APICCommonState *s = DO_UPCAST(APICCommonState, busdev.qdev, d);
 
-    cpu_reset_interrupt(s->cpu_env, CPU_INTERRUPT_SIPI);
+    cpu_reset_interrupt(&s->cpu->env, CPU_INTERRUPT_SIPI);
 
     if (!s->wait_for_sipi)
         return;
-    cpu_x86_load_seg_cache_sipi(s->cpu_env, s->sipi_vector);
+    cpu_x86_load_seg_cache_sipi(s->cpu, s->sipi_vector);
     s->wait_for_sipi = 0;
 }
 
@@ -630,25 +636,25 @@ static void apic_timer(void *opaque)
     apic_timer_update(s, s->next_time);
 }
 
-static uint32_t apic_mem_readb(void *opaque, target_phys_addr_t addr)
+static uint32_t apic_mem_readb(void *opaque, hwaddr addr)
 {
     return 0;
 }
 
-static uint32_t apic_mem_readw(void *opaque, target_phys_addr_t addr)
+static uint32_t apic_mem_readw(void *opaque, hwaddr addr)
 {
     return 0;
 }
 
-static void apic_mem_writeb(void *opaque, target_phys_addr_t addr, uint32_t val)
+static void apic_mem_writeb(void *opaque, hwaddr addr, uint32_t val)
 {
 }
 
-static void apic_mem_writew(void *opaque, target_phys_addr_t addr, uint32_t val)
+static void apic_mem_writew(void *opaque, hwaddr addr, uint32_t val)
 {
 }
 
-static uint32_t apic_mem_readl(void *opaque, target_phys_addr_t addr)
+static uint32_t apic_mem_readl(void *opaque, hwaddr addr)
 {
     DeviceState *d;
     APICCommonState *s;
@@ -672,7 +678,7 @@ static uint32_t apic_mem_readl(void *opaque, target_phys_addr_t addr)
     case 0x08:
         apic_sync_vapic(s, SYNC_FROM_VAPIC);
         if (apic_report_tpr_access) {
-            cpu_report_tpr_access(s->cpu_env, TPR_ACCESS_READ);
+            cpu_report_tpr_access(&s->cpu->env, TPR_ACCESS_READ);
         }
         val = s->tpr;
         break;
@@ -732,7 +738,7 @@ static uint32_t apic_mem_readl(void *opaque, target_phys_addr_t addr)
     return val;
 }
 
-static void apic_send_msi(target_phys_addr_t addr, uint32_t data)
+static void apic_send_msi(hwaddr addr, uint32_t data)
 {
     uint8_t dest = (addr & MSI_ADDR_DEST_ID_MASK) >> MSI_ADDR_DEST_ID_SHIFT;
     uint8_t vector = (data & MSI_DATA_VECTOR_MASK) >> MSI_DATA_VECTOR_SHIFT;
@@ -743,7 +749,7 @@ static void apic_send_msi(target_phys_addr_t addr, uint32_t data)
     apic_deliver_irq(dest, dest_mode, delivery, vector, trigger_mode);
 }
 
-static void apic_mem_writel(void *opaque, target_phys_addr_t addr, uint32_t val)
+static void apic_mem_writel(void *opaque, hwaddr addr, uint32_t val)
 {
     DeviceState *d;
     APICCommonState *s;
@@ -774,7 +780,7 @@ static void apic_mem_writel(void *opaque, target_phys_addr_t addr, uint32_t val)
         break;
     case 0x08:
         if (apic_report_tpr_access) {
-            cpu_report_tpr_access(s->cpu_env, TPR_ACCESS_WRITE);
+            cpu_report_tpr_access(&s->cpu->env, TPR_ACCESS_WRITE);
         }
         s->tpr = val;
         apic_sync_vapic(s, SYNC_TO_VAPIC);
diff --git a/hw/apic_common.c b/hw/apic_common.c
index 371f95d90..5f542764e 100644
--- a/hw/apic_common.c
+++ b/hw/apic_common.c
@@ -89,7 +89,7 @@ void apic_enable_tpr_access_reporting(DeviceState *d, bool enable)
     }
 }
 
-void apic_enable_vapic(DeviceState *d, target_phys_addr_t paddr)
+void apic_enable_vapic(DeviceState *d, hwaddr paddr)
 {
     APICCommonState *s = DO_UPCAST(APICCommonState, busdev.qdev, d);
     APICCommonClass *info = APIC_COMMON_GET_CLASS(s);
@@ -103,7 +103,7 @@ void apic_handle_tpr_access_report(DeviceState *d, target_ulong ip,
 {
     APICCommonState *s = DO_UPCAST(APICCommonState, busdev.qdev, d);
 
-    vapic_report_tpr_access(s->vapic, s->cpu_env, ip, access);
+    vapic_report_tpr_access(s->vapic, &s->cpu->env, ip, access);
 }
 
 void apic_report_irq_delivered(int delivered)
@@ -217,7 +217,7 @@ static void apic_reset_common(DeviceState *d)
     APICCommonClass *info = APIC_COMMON_GET_CLASS(s);
     bool bsp;
 
-    bsp = cpu_is_bsp(x86_env_get_cpu(s->cpu_env));
+    bsp = cpu_is_bsp(s->cpu);
     s->apicbase = 0xfee00000 |
         (bsp ? MSR_IA32_APICBASE_BSP : 0) | MSR_IA32_APICBASE_ENABLE;
 
@@ -368,7 +368,6 @@ static const VMStateDescription vmstate_apic_common = {
 
 static Property apic_properties_common[] = {
     DEFINE_PROP_UINT8("id", APICCommonState, id, -1),
-    DEFINE_PROP_PTR("cpu_env", APICCommonState, cpu_env),
     DEFINE_PROP_BIT("vapic", APICCommonState, vapic_control, VAPIC_ENABLE_BIT,
                     true),
     DEFINE_PROP_END_OF_LIST(),
diff --git a/hw/apic_internal.h b/hw/apic_internal.h
index 4d8ff490c..79e2de224 100644
--- a/hw/apic_internal.h
+++ b/hw/apic_internal.h
@@ -95,8 +95,9 @@ typedef struct APICCommonClass
 
 struct APICCommonState {
     SysBusDevice busdev;
+
     MemoryRegion io_memory;
-    void *cpu_env;
+    X86CPU *cpu;
     uint32_t apicbase;
     uint8_t id;
     uint8_t arb_id;
@@ -124,7 +125,7 @@ struct APICCommonState {
 
     uint32_t vapic_control;
     DeviceState *vapic;
-    target_phys_addr_t vapic_paddr; /* note: persistence via kvmvapic */
+    hwaddr vapic_paddr; /* note: persistence via kvmvapic */
 };
 
 typedef struct VAPICState {
@@ -140,7 +141,7 @@ extern bool apic_report_tpr_access;
 void apic_report_irq_delivered(int delivered);
 bool apic_next_timer(APICCommonState *s, int64_t current_time);
 void apic_enable_tpr_access_reporting(DeviceState *d, bool enable);
-void apic_enable_vapic(DeviceState *d, target_phys_addr_t paddr);
+void apic_enable_vapic(DeviceState *d, hwaddr paddr);
 
 void vapic_report_tpr_access(DeviceState *dev, void *cpu, target_ulong ip,
                              TPRAccess access);
diff --git a/hw/arm-misc.h b/hw/arm-misc.h
index bdd8fecc9..d129678b2 100644
--- a/hw/arm-misc.h
+++ b/hw/arm-misc.h
@@ -12,6 +12,7 @@
 #define ARM_MISC_H 1
 
 #include "memory.h"
+#include "hw/irq.h"
 
 /* The CPU is also modeled as an interrupt controller.  */
 #define ARM_PIC_CPU_IRQ 0
@@ -30,15 +31,15 @@ struct arm_boot_info {
     const char *kernel_cmdline;
     const char *initrd_filename;
     const char *dtb_filename;
-    target_phys_addr_t loader_start;
+    hwaddr loader_start;
     /* multicore boards that use the default secondary core boot functions
      * need to put the address of the secondary boot code, the boot reg,
      * and the GIC address in the next 3 values, respectively. boards that
      * have their own boot functions can use these values as they want.
      */
-    target_phys_addr_t smp_loader_start;
-    target_phys_addr_t smp_bootreg_addr;
-    target_phys_addr_t gic_cpu_if_addr;
+    hwaddr smp_loader_start;
+    hwaddr smp_bootreg_addr;
+    hwaddr gic_cpu_if_addr;
     int nb_cpus;
     int board_id;
     int (*atag_board)(const struct arm_boot_info *info, void *p);
@@ -56,8 +57,9 @@ struct arm_boot_info {
                                      const struct arm_boot_info *info);
     /* Used internally by arm_boot.c */
     int is_linux;
-    target_phys_addr_t initrd_size;
-    target_phys_addr_t entry;
+    hwaddr initrd_start;
+    hwaddr initrd_size;
+    hwaddr entry;
 };
 void arm_load_kernel(ARMCPU *cpu, struct arm_boot_info *info);
 
diff --git a/hw/arm/Makefile.objs b/hw/arm/Makefile.objs
index 2b39fb3c8..6d049e7de 100644
--- a/hw/arm/Makefile.objs
+++ b/hw/arm/Makefile.objs
@@ -1,6 +1,7 @@
 obj-y = integratorcp.o versatilepb.o arm_pic.o
 obj-y += arm_boot.o
 obj-y += xilinx_zynq.o zynq_slcr.o
+obj-y += xilinx_spips.o
 obj-y += arm_gic.o arm_gic_common.o
 obj-y += realview_gic.o realview.o arm_sysctl.o arm11mpcore.o a9mpcore.o
 obj-y += exynos4210_gic.o exynos4210_combiner.o exynos4210.o
diff --git a/hw/arm11mpcore.c b/hw/arm11mpcore.c
index 1bff3d328..640ed20a6 100644
--- a/hw/arm11mpcore.c
+++ b/hw/arm11mpcore.c
@@ -27,7 +27,7 @@ typedef struct mpcore_priv_state {
 
 /* Per-CPU private memory mapped IO.  */
 
-static uint64_t mpcore_scu_read(void *opaque, target_phys_addr_t offset,
+static uint64_t mpcore_scu_read(void *opaque, hwaddr offset,
                                 unsigned size)
 {
     mpcore_priv_state *s = (mpcore_priv_state *)opaque;
@@ -44,11 +44,13 @@ static uint64_t mpcore_scu_read(void *opaque, target_phys_addr_t offset,
     case 0x0c: /* Invalidate all.  */
         return 0;
     default:
-        hw_error("mpcore_priv_read: Bad offset %x\n", (int)offset);
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "mpcore_priv_read: Bad offset %x\n", (int)offset);
+        return 0;
     }
 }
 
-static void mpcore_scu_write(void *opaque, target_phys_addr_t offset,
+static void mpcore_scu_write(void *opaque, hwaddr offset,
                              uint64_t value, unsigned size)
 {
     mpcore_priv_state *s = (mpcore_priv_state *)opaque;
@@ -61,7 +63,8 @@ static void mpcore_scu_write(void *opaque, target_phys_addr_t offset,
         /* This is a no-op as cache is not emulated.  */
         break;
     default:
-        hw_error("mpcore_priv_read: Bad offset %x\n", (int)offset);
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "mpcore_priv_read: Bad offset %x\n", (int)offset);
     }
 }
 
@@ -89,7 +92,7 @@ static void mpcore_priv_map_setup(mpcore_priv_state *s)
      * at 0x200, 0x300...
      */
     for (i = 0; i < (s->num_cpu + 1); i++) {
-        target_phys_addr_t offset = 0x100 + (i * 0x100);
+        hwaddr offset = 0x100 + (i * 0x100);
         memory_region_add_subregion(&s->container, offset,
                                     sysbus_mmio_get_region(gicbusdev, i + 1));
     }
@@ -98,7 +101,7 @@ static void mpcore_priv_map_setup(mpcore_priv_state *s)
      */
     for (i = 0; i < (s->num_cpu + 1) * 2; i++) {
         /* Timers at 0x600, 0x700, ...; watchdogs at 0x620, 0x720, ... */
-        target_phys_addr_t offset = 0x600 + (i >> 1) * 0x100 + (i & 1) * 0x20;
+        hwaddr offset = 0x600 + (i >> 1) * 0x100 + (i & 1) * 0x20;
         memory_region_add_subregion(&s->container, offset,
                                     sysbus_mmio_get_region(busdev, i));
     }
diff --git a/hw/arm_boot.c b/hw/arm_boot.c
index a6e914366..92e2cab47 100644
--- a/hw/arm_boot.c
+++ b/hw/arm_boot.c
@@ -18,7 +18,6 @@
 
 #define KERNEL_ARGS_ADDR 0x100
 #define KERNEL_LOAD_ADDR 0x00010000
-#define INITRD_LOAD_ADDR 0x00d00000
 
 /* The worlds second smallest bootloader.  Set r0-r2, then jump to kernel.  */
 static uint32_t bootloader[] = {
@@ -89,8 +88,8 @@ static void default_reset_secondary(ARMCPU *cpu,
 static void set_kernel_args(const struct arm_boot_info *info)
 {
     int initrd_size = info->initrd_size;
-    target_phys_addr_t base = info->loader_start;
-    target_phys_addr_t p;
+    hwaddr base = info->loader_start;
+    hwaddr p;
 
     p = base + KERNEL_ARGS_ADDR;
     /* ATAG_CORE */
@@ -109,7 +108,7 @@ static void set_kernel_args(const struct arm_boot_info *info)
         /* ATAG_INITRD2 */
         WRITE_WORD(p, 4);
         WRITE_WORD(p, 0x54420005);
-        WRITE_WORD(p, info->loader_start + INITRD_LOAD_ADDR);
+        WRITE_WORD(p, info->initrd_start);
         WRITE_WORD(p, initrd_size);
     }
     if (info->kernel_cmdline && *info->kernel_cmdline) {
@@ -142,10 +141,10 @@ static void set_kernel_args(const struct arm_boot_info *info)
 
 static void set_kernel_args_old(const struct arm_boot_info *info)
 {
-    target_phys_addr_t p;
+    hwaddr p;
     const char *s;
     int initrd_size = info->initrd_size;
-    target_phys_addr_t base = info->loader_start;
+    hwaddr base = info->loader_start;
 
     /* see linux/include/asm-arm/setup.h */
     p = base + KERNEL_ARGS_ADDR;
@@ -185,10 +184,11 @@ static void set_kernel_args_old(const struct arm_boot_info *info)
     /* pages_in_vram */
     WRITE_WORD(p, 0);
     /* initrd_start */
-    if (initrd_size)
-        WRITE_WORD(p, info->loader_start + INITRD_LOAD_ADDR);
-    else
+    if (initrd_size) {
+        WRITE_WORD(p, info->initrd_start);
+    } else {
         WRITE_WORD(p, 0);
+    }
     /* initrd_size */
     WRITE_WORD(p, initrd_size);
     /* rd_start */
@@ -213,7 +213,7 @@ static void set_kernel_args_old(const struct arm_boot_info *info)
     }
 }
 
-static int load_dtb(target_phys_addr_t addr, const struct arm_boot_info *binfo)
+static int load_dtb(hwaddr addr, const struct arm_boot_info *binfo)
 {
 #ifdef CONFIG_FDT
     uint32_t *mem_reg_property;
@@ -281,14 +281,13 @@ static int load_dtb(target_phys_addr_t addr, const struct arm_boot_info *binfo)
 
     if (binfo->initrd_size) {
         rc = qemu_devtree_setprop_cell(fdt, "/chosen", "linux,initrd-start",
-                binfo->loader_start + INITRD_LOAD_ADDR);
+                binfo->initrd_start);
         if (rc < 0) {
             fprintf(stderr, "couldn't set /chosen/linux,initrd-start\n");
         }
 
         rc = qemu_devtree_setprop_cell(fdt, "/chosen", "linux,initrd-end",
-                    binfo->loader_start + INITRD_LOAD_ADDR +
-                    binfo->initrd_size);
+                    binfo->initrd_start + binfo->initrd_size);
         if (rc < 0) {
             fprintf(stderr, "couldn't set /chosen/linux,initrd-end\n");
         }
@@ -342,7 +341,7 @@ void arm_load_kernel(ARMCPU *cpu, struct arm_boot_info *info)
     int n;
     int is_linux = 0;
     uint64_t elf_entry;
-    target_phys_addr_t entry;
+    hwaddr entry;
     int big_endian;
     QemuOpts *machine_opts;
 
@@ -375,6 +374,19 @@ void arm_load_kernel(ARMCPU *cpu, struct arm_boot_info *info)
     big_endian = 0;
 #endif
 
+    /* We want to put the initrd far enough into RAM that when the
+     * kernel is uncompressed it will not clobber the initrd. However
+     * on boards without much RAM we must ensure that we still leave
+     * enough room for a decent sized initrd, and on boards with large
+     * amounts of RAM we must avoid the initrd being so far up in RAM
+     * that it is outside lowmem and inaccessible to the kernel.
+     * So for boards with less  than 256MB of RAM we put the initrd
+     * halfway into RAM, and for boards with 256MB of RAM or more we put
+     * the initrd at 128MB.
+     */
+    info->initrd_start = info->loader_start +
+        MIN(info->ram_size / 2, 128 * 1024 * 1024);
+
     /* Assume that raw images are linux kernels, and ELF images are not.  */
     kernel_size = load_elf(info->kernel_filename, NULL, NULL, &elf_entry,
                            NULL, NULL, big_endian, ELF_MACHINE, 1);
@@ -398,10 +410,9 @@ void arm_load_kernel(ARMCPU *cpu, struct arm_boot_info *info)
     if (is_linux) {
         if (info->initrd_filename) {
             initrd_size = load_image_targphys(info->initrd_filename,
-                                              info->loader_start
-                                              + INITRD_LOAD_ADDR,
-                                              info->ram_size
-                                              - INITRD_LOAD_ADDR);
+                                              info->initrd_start,
+                                              info->ram_size -
+                                              info->initrd_start);
             if (initrd_size < 0) {
                 fprintf(stderr, "qemu: could not load initrd '%s'\n",
                         info->initrd_filename);
@@ -419,9 +430,8 @@ void arm_load_kernel(ARMCPU *cpu, struct arm_boot_info *info)
          */
         if (info->dtb_filename) {
             /* Place the DTB after the initrd in memory */
-            target_phys_addr_t dtb_start = TARGET_PAGE_ALIGN(info->loader_start
-                                                             + INITRD_LOAD_ADDR
-                                                             + initrd_size);
+            hwaddr dtb_start = TARGET_PAGE_ALIGN(info->initrd_start +
+                                                 initrd_size);
             if (load_dtb(dtb_start, info)) {
                 exit(1);
             }
diff --git a/hw/arm_gic.c b/hw/arm_gic.c
index 55871fad1..f9e423f15 100644
--- a/hw/arm_gic.c
+++ b/hw/arm_gic.c
@@ -36,7 +36,7 @@ static const uint8_t gic_id[] = {
 
 #define NUM_CPU(s) ((s)->num_cpu)
 
-static inline int gic_get_current_cpu(gic_state *s)
+static inline int gic_get_current_cpu(GICState *s)
 {
     if (s->num_cpu > 1) {
         return cpu_single_env->cpu_index;
@@ -46,7 +46,7 @@ static inline int gic_get_current_cpu(gic_state *s)
 
 /* TODO: Many places that call this routine could be optimized.  */
 /* Update interrupt status after enabled or pending bits have been changed.  */
-void gic_update(gic_state *s)
+void gic_update(GICState *s)
 {
     int best_irq;
     int best_prio;
@@ -84,7 +84,7 @@ void gic_update(gic_state *s)
     }
 }
 
-void gic_set_pending_private(gic_state *s, int cpu, int irq)
+void gic_set_pending_private(GICState *s, int cpu, int irq)
 {
     int cm = 1 << cpu;
 
@@ -105,7 +105,7 @@ static void gic_set_irq(void *opaque, int irq, int level)
      *  [N+32..N+63] : PPI (internal interrupts for CPU 1
      *  ...
      */
-    gic_state *s = (gic_state *)opaque;
+    GICState *s = (GICState *)opaque;
     int cm, target;
     if (irq < (s->num_irq - GIC_INTERNAL)) {
         /* The first external input line is internal interrupt 32.  */
@@ -137,7 +137,7 @@ static void gic_set_irq(void *opaque, int irq, int level)
     gic_update(s);
 }
 
-static void gic_set_running_irq(gic_state *s, int cpu, int irq)
+static void gic_set_running_irq(GICState *s, int cpu, int irq)
 {
     s->running_irq[cpu] = irq;
     if (irq == 1023) {
@@ -148,7 +148,7 @@ static void gic_set_running_irq(gic_state *s, int cpu, int irq)
     gic_update(s);
 }
 
-uint32_t gic_acknowledge_irq(gic_state *s, int cpu)
+uint32_t gic_acknowledge_irq(GICState *s, int cpu)
 {
     int new_irq;
     int cm = 1 << cpu;
@@ -167,7 +167,7 @@ uint32_t gic_acknowledge_irq(gic_state *s, int cpu)
     return new_irq;
 }
 
-void gic_complete_irq(gic_state *s, int cpu, int irq)
+void gic_complete_irq(GICState *s, int cpu, int irq)
 {
     int update = 0;
     int cm = 1 << cpu;
@@ -212,9 +212,9 @@ void gic_complete_irq(gic_state *s, int cpu, int irq)
     }
 }
 
-static uint32_t gic_dist_readb(void *opaque, target_phys_addr_t offset)
+static uint32_t gic_dist_readb(void *opaque, hwaddr offset)
 {
-    gic_state *s = (gic_state *)opaque;
+    GICState *s = (GICState *)opaque;
     uint32_t res;
     int irq;
     int i;
@@ -324,11 +324,12 @@ static uint32_t gic_dist_readb(void *opaque, target_phys_addr_t offset)
     }
     return res;
 bad_reg:
-    hw_error("gic_dist_readb: Bad offset %x\n", (int)offset);
+    qemu_log_mask(LOG_GUEST_ERROR,
+                  "gic_dist_readb: Bad offset %x\n", (int)offset);
     return 0;
 }
 
-static uint32_t gic_dist_readw(void *opaque, target_phys_addr_t offset)
+static uint32_t gic_dist_readw(void *opaque, hwaddr offset)
 {
     uint32_t val;
     val = gic_dist_readb(opaque, offset);
@@ -336,7 +337,7 @@ static uint32_t gic_dist_readw(void *opaque, target_phys_addr_t offset)
     return val;
 }
 
-static uint32_t gic_dist_readl(void *opaque, target_phys_addr_t offset)
+static uint32_t gic_dist_readl(void *opaque, hwaddr offset)
 {
     uint32_t val;
     val = gic_dist_readw(opaque, offset);
@@ -344,10 +345,10 @@ static uint32_t gic_dist_readl(void *opaque, target_phys_addr_t offset)
     return val;
 }
 
-static void gic_dist_writeb(void *opaque, target_phys_addr_t offset,
+static void gic_dist_writeb(void *opaque, hwaddr offset,
                             uint32_t value)
 {
-    gic_state *s = (gic_state *)opaque;
+    GICState *s = (GICState *)opaque;
     int irq;
     int i;
     int cpu;
@@ -487,20 +488,21 @@ static void gic_dist_writeb(void *opaque, target_phys_addr_t offset,
     gic_update(s);
     return;
 bad_reg:
-    hw_error("gic_dist_writeb: Bad offset %x\n", (int)offset);
+    qemu_log_mask(LOG_GUEST_ERROR,
+                  "gic_dist_writeb: Bad offset %x\n", (int)offset);
 }
 
-static void gic_dist_writew(void *opaque, target_phys_addr_t offset,
+static void gic_dist_writew(void *opaque, hwaddr offset,
                             uint32_t value)
 {
     gic_dist_writeb(opaque, offset, value & 0xff);
     gic_dist_writeb(opaque, offset + 1, value >> 8);
 }
 
-static void gic_dist_writel(void *opaque, target_phys_addr_t offset,
+static void gic_dist_writel(void *opaque, hwaddr offset,
                             uint32_t value)
 {
-    gic_state *s = (gic_state *)opaque;
+    GICState *s = (GICState *)opaque;
     if (offset == 0xf00) {
         int cpu;
         int irq;
@@ -539,7 +541,7 @@ static const MemoryRegionOps gic_dist_ops = {
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-static uint32_t gic_cpu_read(gic_state *s, int cpu, int offset)
+static uint32_t gic_cpu_read(GICState *s, int cpu, int offset)
 {
     switch (offset) {
     case 0x00: /* Control */
@@ -556,17 +558,18 @@ static uint32_t gic_cpu_read(gic_state *s, int cpu, int offset)
     case 0x18: /* Highest Pending Interrupt */
         return s->current_pending[cpu];
     default:
-        hw_error("gic_cpu_read: Bad offset %x\n", (int)offset);
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "gic_cpu_read: Bad offset %x\n", (int)offset);
         return 0;
     }
 }
 
-static void gic_cpu_write(gic_state *s, int cpu, int offset, uint32_t value)
+static void gic_cpu_write(GICState *s, int cpu, int offset, uint32_t value)
 {
     switch (offset) {
     case 0x00: /* Control */
         s->cpu_enabled[cpu] = (value & 1);
-        DPRINTF("CPU %d %sabled\n", cpu, s->cpu_enabled ? "En" : "Dis");
+        DPRINTF("CPU %d %sabled\n", cpu, s->cpu_enabled[cpu] ? "En" : "Dis");
         break;
     case 0x04: /* Priority mask */
         s->priority_mask[cpu] = (value & 0xff);
@@ -577,44 +580,45 @@ static void gic_cpu_write(gic_state *s, int cpu, int offset, uint32_t value)
     case 0x10: /* End Of Interrupt */
         return gic_complete_irq(s, cpu, value & 0x3ff);
     default:
-        hw_error("gic_cpu_write: Bad offset %x\n", (int)offset);
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "gic_cpu_write: Bad offset %x\n", (int)offset);
         return;
     }
     gic_update(s);
 }
 
 /* Wrappers to read/write the GIC CPU interface for the current CPU */
-static uint64_t gic_thiscpu_read(void *opaque, target_phys_addr_t addr,
+static uint64_t gic_thiscpu_read(void *opaque, hwaddr addr,
                                  unsigned size)
 {
-    gic_state *s = (gic_state *)opaque;
+    GICState *s = (GICState *)opaque;
     return gic_cpu_read(s, gic_get_current_cpu(s), addr);
 }
 
-static void gic_thiscpu_write(void *opaque, target_phys_addr_t addr,
+static void gic_thiscpu_write(void *opaque, hwaddr addr,
                               uint64_t value, unsigned size)
 {
-    gic_state *s = (gic_state *)opaque;
+    GICState *s = (GICState *)opaque;
     gic_cpu_write(s, gic_get_current_cpu(s), addr, value);
 }
 
 /* Wrappers to read/write the GIC CPU interface for a specific CPU.
- * These just decode the opaque pointer into gic_state* + cpu id.
+ * These just decode the opaque pointer into GICState* + cpu id.
  */
-static uint64_t gic_do_cpu_read(void *opaque, target_phys_addr_t addr,
+static uint64_t gic_do_cpu_read(void *opaque, hwaddr addr,
                                 unsigned size)
 {
-    gic_state **backref = (gic_state **)opaque;
-    gic_state *s = *backref;
+    GICState **backref = (GICState **)opaque;
+    GICState *s = *backref;
     int id = (backref - s->backref);
     return gic_cpu_read(s, id, addr);
 }
 
-static void gic_do_cpu_write(void *opaque, target_phys_addr_t addr,
+static void gic_do_cpu_write(void *opaque, hwaddr addr,
                              uint64_t value, unsigned size)
 {
-    gic_state **backref = (gic_state **)opaque;
-    gic_state *s = *backref;
+    GICState **backref = (GICState **)opaque;
+    GICState *s = *backref;
     int id = (backref - s->backref);
     gic_cpu_write(s, id, addr, value);
 }
@@ -631,7 +635,7 @@ static const MemoryRegionOps gic_cpu_ops = {
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-void gic_init_irqs_and_distributor(gic_state *s, int num_irq)
+void gic_init_irqs_and_distributor(GICState *s, int num_irq)
 {
     int i;
 
@@ -657,7 +661,7 @@ static int arm_gic_init(SysBusDevice *dev)
 {
     /* Device instance init function for the GIC sysbus device */
     int i;
-    gic_state *s = FROM_SYSBUS(gic_state, dev);
+    GICState *s = FROM_SYSBUS(GICState, dev);
     ARMGICClass *agc = ARM_GIC_GET_CLASS(s);
 
     agc->parent_init(dev);
@@ -701,7 +705,7 @@ static void arm_gic_class_init(ObjectClass *klass, void *data)
 static TypeInfo arm_gic_info = {
     .name = TYPE_ARM_GIC,
     .parent = TYPE_ARM_GIC_COMMON,
-    .instance_size = sizeof(gic_state),
+    .instance_size = sizeof(GICState),
     .class_init = arm_gic_class_init,
     .class_size = sizeof(ARMGICClass),
 };
diff --git a/hw/arm_gic_common.c b/hw/arm_gic_common.c
index 360e7823f..8369309d2 100644
--- a/hw/arm_gic_common.c
+++ b/hw/arm_gic_common.c
@@ -22,7 +22,7 @@
 
 static void gic_save(QEMUFile *f, void *opaque)
 {
-    gic_state *s = (gic_state *)opaque;
+    GICState *s = (GICState *)opaque;
     int i;
     int j;
 
@@ -56,7 +56,7 @@ static void gic_save(QEMUFile *f, void *opaque)
 
 static int gic_load(QEMUFile *f, void *opaque, int version_id)
 {
-    gic_state *s = (gic_state *)opaque;
+    GICState *s = (GICState *)opaque;
     int i;
     int j;
 
@@ -96,7 +96,7 @@ static int gic_load(QEMUFile *f, void *opaque, int version_id)
 
 static int arm_gic_common_init(SysBusDevice *dev)
 {
-    gic_state *s = FROM_SYSBUS(gic_state, dev);
+    GICState *s = FROM_SYSBUS(GICState, dev);
     int num_irq = s->num_irq;
 
     if (s->num_cpu > NCPU) {
@@ -123,7 +123,7 @@ static int arm_gic_common_init(SysBusDevice *dev)
 
 static void arm_gic_common_reset(DeviceState *dev)
 {
-    gic_state *s = FROM_SYSBUS(gic_state, sysbus_from_qdev(dev));
+    GICState *s = FROM_SYSBUS(GICState, sysbus_from_qdev(dev));
     int i;
     memset(s->irq_state, 0, GIC_MAXIRQ * sizeof(gic_irq_state));
     for (i = 0 ; i < s->num_cpu; i++) {
@@ -147,13 +147,13 @@ static void arm_gic_common_reset(DeviceState *dev)
 }
 
 static Property arm_gic_common_properties[] = {
-    DEFINE_PROP_UINT32("num-cpu", gic_state, num_cpu, 1),
-    DEFINE_PROP_UINT32("num-irq", gic_state, num_irq, 32),
+    DEFINE_PROP_UINT32("num-cpu", GICState, num_cpu, 1),
+    DEFINE_PROP_UINT32("num-irq", GICState, num_irq, 32),
     /* Revision can be 1 or 2 for GIC architecture specification
      * versions 1 or 2, or 0 to indicate the legacy 11MPCore GIC.
      * (Internally, 0xffffffff also indicates "not a GIC but an NVIC".)
      */
-    DEFINE_PROP_UINT32("revision", gic_state, revision, 1),
+    DEFINE_PROP_UINT32("revision", GICState, revision, 1),
     DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -170,7 +170,7 @@ static void arm_gic_common_class_init(ObjectClass *klass, void *data)
 static TypeInfo arm_gic_common_type = {
     .name = TYPE_ARM_GIC_COMMON,
     .parent = TYPE_SYS_BUS_DEVICE,
-    .instance_size = sizeof(gic_state),
+    .instance_size = sizeof(GICState),
     .class_size = sizeof(ARMGICCommonClass),
     .class_init = arm_gic_common_class_init,
     .abstract = true,
diff --git a/hw/arm_gic_internal.h b/hw/arm_gic_internal.h
index db4fad564..699352ca8 100644
--- a/hw/arm_gic_internal.h
+++ b/hw/arm_gic_internal.h
@@ -69,7 +69,7 @@ typedef struct gic_irq_state {
     unsigned trigger:1; /* nonzero = edge triggered.  */
 } gic_irq_state;
 
-typedef struct gic_state {
+typedef struct GICState {
     SysBusDevice busdev;
     qemu_irq parent_irq[NCPU];
     int enabled;
@@ -92,25 +92,25 @@ typedef struct gic_state {
     /* This is just so we can have an opaque pointer which identifies
      * both this GIC and which CPU interface we should be accessing.
      */
-    struct gic_state *backref[NCPU];
+    struct GICState *backref[NCPU];
     MemoryRegion cpuiomem[NCPU+1]; /* CPU interfaces */
     uint32_t num_irq;
     uint32_t revision;
-} gic_state;
+} GICState;
 
 /* The special cases for the revision property: */
 #define REV_11MPCORE 0
 #define REV_NVIC 0xffffffff
 
-void gic_set_pending_private(gic_state *s, int cpu, int irq);
-uint32_t gic_acknowledge_irq(gic_state *s, int cpu);
-void gic_complete_irq(gic_state *s, int cpu, int irq);
-void gic_update(gic_state *s);
-void gic_init_irqs_and_distributor(gic_state *s, int num_irq);
+void gic_set_pending_private(GICState *s, int cpu, int irq);
+uint32_t gic_acknowledge_irq(GICState *s, int cpu);
+void gic_complete_irq(GICState *s, int cpu, int irq);
+void gic_update(GICState *s);
+void gic_init_irqs_and_distributor(GICState *s, int num_irq);
 
 #define TYPE_ARM_GIC_COMMON "arm_gic_common"
 #define ARM_GIC_COMMON(obj) \
-     OBJECT_CHECK(gic_state, (obj), TYPE_ARM_GIC_COMMON)
+     OBJECT_CHECK(GICState, (obj), TYPE_ARM_GIC_COMMON)
 #define ARM_GIC_COMMON_CLASS(klass) \
      OBJECT_CLASS_CHECK(ARMGICCommonClass, (klass), TYPE_ARM_GIC_COMMON)
 #define ARM_GIC_COMMON_GET_CLASS(obj) \
@@ -122,7 +122,7 @@ typedef struct ARMGICCommonClass {
 
 #define TYPE_ARM_GIC "arm_gic"
 #define ARM_GIC(obj) \
-     OBJECT_CHECK(gic_state, (obj), TYPE_ARM_GIC)
+     OBJECT_CHECK(GICState, (obj), TYPE_ARM_GIC)
 #define ARM_GIC_CLASS(klass) \
      OBJECT_CLASS_CHECK(ARMGICClass, (klass), TYPE_ARM_GIC)
 #define ARM_GIC_GET_CLASS(obj) \
diff --git a/hw/arm_l2x0.c b/hw/arm_l2x0.c
index de6a0863d..6abf0ee16 100644
--- a/hw/arm_l2x0.c
+++ b/hw/arm_l2x0.c
@@ -51,7 +51,7 @@ static const VMStateDescription vmstate_l2x0 = {
 };
 
 
-static uint64_t l2x0_priv_read(void *opaque, target_phys_addr_t offset,
+static uint64_t l2x0_priv_read(void *opaque, hwaddr offset,
                                unsigned size)
 {
     uint32_t cache_data;
@@ -87,13 +87,14 @@ static uint64_t l2x0_priv_read(void *opaque, target_phys_addr_t offset,
     case 0xF80:
         return 0;
     default:
-        fprintf(stderr, "l2x0_priv_read: Bad offset %x\n", (int)offset);
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "l2x0_priv_read: Bad offset %x\n", (int)offset);
         break;
     }
     return 0;
 }
 
-static void l2x0_priv_write(void *opaque, target_phys_addr_t offset,
+static void l2x0_priv_write(void *opaque, hwaddr offset,
                             uint64_t value, unsigned size)
 {
     l2x0_state *s = (l2x0_state *)opaque;
@@ -128,7 +129,8 @@ static void l2x0_priv_write(void *opaque, target_phys_addr_t offset,
     case 0xF80:
         return;
     default:
-        fprintf(stderr, "l2x0_priv_write: Bad offset %x\n", (int)offset);
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "l2x0_priv_write: Bad offset %x\n", (int)offset);
         break;
     }
 }
diff --git a/hw/arm_mptimer.c b/hw/arm_mptimer.c
index fe43cbb5f..679083223 100644
--- a/hw/arm_mptimer.c
+++ b/hw/arm_mptimer.c
@@ -92,7 +92,7 @@ static void timerblock_tick(void *opaque)
     timerblock_update_irq(tb);
 }
 
-static uint64_t timerblock_read(void *opaque, target_phys_addr_t addr,
+static uint64_t timerblock_read(void *opaque, hwaddr addr,
                                 unsigned size)
 {
     timerblock *tb = (timerblock *)opaque;
@@ -120,7 +120,7 @@ static uint64_t timerblock_read(void *opaque, target_phys_addr_t addr,
     }
 }
 
-static void timerblock_write(void *opaque, target_phys_addr_t addr,
+static void timerblock_write(void *opaque, hwaddr addr,
                              uint64_t value, unsigned size)
 {
     timerblock *tb = (timerblock *)opaque;
@@ -159,7 +159,7 @@ static void timerblock_write(void *opaque, target_phys_addr_t addr,
 /* Wrapper functions to implement the "read timer/watchdog for
  * the current CPU" memory regions.
  */
-static uint64_t arm_thistimer_read(void *opaque, target_phys_addr_t addr,
+static uint64_t arm_thistimer_read(void *opaque, hwaddr addr,
                                    unsigned size)
 {
     arm_mptimer_state *s = (arm_mptimer_state *)opaque;
@@ -167,7 +167,7 @@ static uint64_t arm_thistimer_read(void *opaque, target_phys_addr_t addr,
     return timerblock_read(&s->timerblock[id * 2], addr, size);
 }
 
-static void arm_thistimer_write(void *opaque, target_phys_addr_t addr,
+static void arm_thistimer_write(void *opaque, hwaddr addr,
                                 uint64_t value, unsigned size)
 {
     arm_mptimer_state *s = (arm_mptimer_state *)opaque;
@@ -175,7 +175,7 @@ static void arm_thistimer_write(void *opaque, target_phys_addr_t addr,
     timerblock_write(&s->timerblock[id * 2], addr, value, size);
 }
 
-static uint64_t arm_thiswdog_read(void *opaque, target_phys_addr_t addr,
+static uint64_t arm_thiswdog_read(void *opaque, hwaddr addr,
                                   unsigned size)
 {
     arm_mptimer_state *s = (arm_mptimer_state *)opaque;
@@ -183,7 +183,7 @@ static uint64_t arm_thiswdog_read(void *opaque, target_phys_addr_t addr,
     return timerblock_read(&s->timerblock[id * 2 + 1], addr, size);
 }
 
-static void arm_thiswdog_write(void *opaque, target_phys_addr_t addr,
+static void arm_thiswdog_write(void *opaque, hwaddr addr,
                                uint64_t value, unsigned size)
 {
     arm_mptimer_state *s = (arm_mptimer_state *)opaque;
diff --git a/hw/arm_sysctl.c b/hw/arm_sysctl.c
index 5f1237b8c..58eb98216 100644
--- a/hw/arm_sysctl.c
+++ b/hw/arm_sysctl.c
@@ -92,7 +92,7 @@ static void arm_sysctl_reset(DeviceState *d)
     }
 }
 
-static uint64_t arm_sysctl_read(void *opaque, target_phys_addr_t offset,
+static uint64_t arm_sysctl_read(void *opaque, hwaddr offset,
                                 unsigned size)
 {
     arm_sysctl_state *s = (arm_sysctl_state *)opaque;
@@ -184,12 +184,14 @@ static uint64_t arm_sysctl_read(void *opaque, target_phys_addr_t offset,
         return s->sys_cfgstat;
     default:
     bad_reg:
-        printf ("arm_sysctl_read: Bad register offset 0x%x\n", (int)offset);
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "arm_sysctl_read: Bad register offset 0x%x\n",
+                      (int)offset);
         return 0;
     }
 }
 
-static void arm_sysctl_write(void *opaque, target_phys_addr_t offset,
+static void arm_sysctl_write(void *opaque, hwaddr offset,
                              uint64_t val, unsigned size)
 {
     arm_sysctl_state *s = (arm_sysctl_state *)opaque;
@@ -339,7 +341,9 @@ static void arm_sysctl_write(void *opaque, target_phys_addr_t offset,
         return;
     default:
     bad_reg:
-        printf ("arm_sysctl_write: Bad register offset 0x%x\n", (int)offset);
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "arm_sysctl_write: Bad register offset 0x%x\n",
+                      (int)offset);
         return;
     }
 }
diff --git a/hw/arm_timer.c b/hw/arm_timer.c
index e3ecce29f..af339d3d1 100644
--- a/hw/arm_timer.c
+++ b/hw/arm_timer.c
@@ -45,7 +45,7 @@ static void arm_timer_update(arm_timer_state *s)
     }
 }
 
-static uint32_t arm_timer_read(void *opaque, target_phys_addr_t offset)
+static uint32_t arm_timer_read(void *opaque, hwaddr offset)
 {
     arm_timer_state *s = (arm_timer_state *)opaque;
 
@@ -64,7 +64,8 @@ static uint32_t arm_timer_read(void *opaque, target_phys_addr_t offset)
             return 0;
         return s->int_level;
     default:
-        hw_error("%s: Bad offset %x\n", __func__, (int)offset);
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "%s: Bad offset %x\n", __func__, (int)offset);
         return 0;
     }
 }
@@ -87,7 +88,7 @@ static void arm_timer_recalibrate(arm_timer_state *s, int reload)
     ptimer_set_limit(s->timer, limit, reload);
 }
 
-static void arm_timer_write(void *opaque, target_phys_addr_t offset,
+static void arm_timer_write(void *opaque, hwaddr offset,
                             uint32_t value)
 {
     arm_timer_state *s = (arm_timer_state *)opaque;
@@ -131,7 +132,8 @@ static void arm_timer_write(void *opaque, target_phys_addr_t offset,
         arm_timer_recalibrate(s, 0);
         break;
     default:
-        hw_error("%s: Bad offset %x\n", __func__, (int)offset);
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "%s: Bad offset %x\n", __func__, (int)offset);
     }
     arm_timer_update(s);
 }
@@ -202,7 +204,7 @@ static void sp804_set_irq(void *opaque, int irq, int level)
     qemu_set_irq(s->irq, s->level[0] || s->level[1]);
 }
 
-static uint64_t sp804_read(void *opaque, target_phys_addr_t offset,
+static uint64_t sp804_read(void *opaque, hwaddr offset,
                            unsigned size)
 {
     sp804_state *s = (sp804_state *)opaque;
@@ -223,14 +225,18 @@ static uint64_t sp804_read(void *opaque, target_phys_addr_t offset,
     /* Integration Test control registers, which we won't support */
     case 0xf00: /* TimerITCR */
     case 0xf04: /* TimerITOP (strictly write only but..) */
+        qemu_log_mask(LOG_UNIMP,
+                      "%s: integration test registers unimplemented\n",
+                      __func__);
         return 0;
     }
 
-    hw_error("%s: Bad offset %x\n", __func__, (int)offset);
+    qemu_log_mask(LOG_GUEST_ERROR,
+                  "%s: Bad offset %x\n", __func__, (int)offset);
     return 0;
 }
 
-static void sp804_write(void *opaque, target_phys_addr_t offset,
+static void sp804_write(void *opaque, hwaddr offset,
                         uint64_t value, unsigned size)
 {
     sp804_state *s = (sp804_state *)opaque;
@@ -246,7 +252,8 @@ static void sp804_write(void *opaque, target_phys_addr_t offset,
     }
 
     /* Technically we could be writing to the Test Registers, but not likely */
-    hw_error("%s: Bad offset %x\n", __func__, (int)offset);
+    qemu_log_mask(LOG_GUEST_ERROR, "%s: Bad offset %x\n",
+                  __func__, (int)offset);
 }
 
 static const MemoryRegionOps sp804_ops = {
@@ -291,7 +298,7 @@ typedef struct {
     arm_timer_state *timer[3];
 } icp_pit_state;
 
-static uint64_t icp_pit_read(void *opaque, target_phys_addr_t offset,
+static uint64_t icp_pit_read(void *opaque, hwaddr offset,
                              unsigned size)
 {
     icp_pit_state *s = (icp_pit_state *)opaque;
@@ -300,13 +307,13 @@ static uint64_t icp_pit_read(void *opaque, target_phys_addr_t offset,
     /* ??? Don't know the PrimeCell ID for this device.  */
     n = offset >> 8;
     if (n > 2) {
-        hw_error("%s: Bad timer %d\n", __func__, n);
+        qemu_log_mask(LOG_GUEST_ERROR, "%s: Bad timer %d\n", __func__, n);
     }
 
     return arm_timer_read(s->timer[n], offset & 0xff);
 }
 
-static void icp_pit_write(void *opaque, target_phys_addr_t offset,
+static void icp_pit_write(void *opaque, hwaddr offset,
                           uint64_t value, unsigned size)
 {
     icp_pit_state *s = (icp_pit_state *)opaque;
@@ -314,7 +321,7 @@ static void icp_pit_write(void *opaque, target_phys_addr_t offset,
 
     n = offset >> 8;
     if (n > 2) {
-        hw_error("%s: Bad timer %d\n", __func__, n);
+        qemu_log_mask(LOG_GUEST_ERROR, "%s: Bad timer %d\n", __func__, n);
     }
 
     arm_timer_write(s->timer[n], offset & 0xff, value);
diff --git a/hw/armv7m.c b/hw/armv7m.c
index 9f66667c6..ce2ec9b4d 100644
--- a/hw/armv7m.c
+++ b/hw/armv7m.c
@@ -25,14 +25,14 @@ static inline uint32_t bitband_addr(void * opaque, uint32_t addr)
 
 }
 
-static uint32_t bitband_readb(void *opaque, target_phys_addr_t offset)
+static uint32_t bitband_readb(void *opaque, hwaddr offset)
 {
     uint8_t v;
     cpu_physical_memory_read(bitband_addr(opaque, offset), &v, 1);
     return (v & (1 << ((offset >> 2) & 7))) != 0;
 }
 
-static void bitband_writeb(void *opaque, target_phys_addr_t offset,
+static void bitband_writeb(void *opaque, hwaddr offset,
                            uint32_t value)
 {
     uint32_t addr;
@@ -48,7 +48,7 @@ static void bitband_writeb(void *opaque, target_phys_addr_t offset,
     cpu_physical_memory_write(addr, &v, 1);
 }
 
-static uint32_t bitband_readw(void *opaque, target_phys_addr_t offset)
+static uint32_t bitband_readw(void *opaque, hwaddr offset)
 {
     uint32_t addr;
     uint16_t mask;
@@ -60,7 +60,7 @@ static uint32_t bitband_readw(void *opaque, target_phys_addr_t offset)
     return (v & mask) != 0;
 }
 
-static void bitband_writew(void *opaque, target_phys_addr_t offset,
+static void bitband_writew(void *opaque, hwaddr offset,
                            uint32_t value)
 {
     uint32_t addr;
@@ -77,7 +77,7 @@ static void bitband_writew(void *opaque, target_phys_addr_t offset,
     cpu_physical_memory_write(addr, (uint8_t *)&v, 2);
 }
 
-static uint32_t bitband_readl(void *opaque, target_phys_addr_t offset)
+static uint32_t bitband_readl(void *opaque, hwaddr offset)
 {
     uint32_t addr;
     uint32_t mask;
@@ -89,7 +89,7 @@ static uint32_t bitband_readl(void *opaque, target_phys_addr_t offset)
     return (v & mask) != 0;
 }
 
-static void bitband_writel(void *opaque, target_phys_addr_t offset,
+static void bitband_writel(void *opaque, hwaddr offset,
                            uint32_t value)
 {
     uint32_t addr;
diff --git a/hw/armv7m_nvic.c b/hw/armv7m_nvic.c
index 6a0832eb3..f0a2e7b5d 100644
--- a/hw/armv7m_nvic.c
+++ b/hw/armv7m_nvic.c
@@ -17,7 +17,7 @@
 #include "arm_gic_internal.h"
 
 typedef struct {
-    gic_state gic;
+    GICState gic;
     struct {
         uint32_t control;
         uint32_t reload;
@@ -138,9 +138,8 @@ void armv7m_nvic_complete_irq(void *opaque, int irq)
     gic_complete_irq(&s->gic, 0, irq);
 }
 
-static uint32_t nvic_readl(void *opaque, uint32_t offset)
+static uint32_t nvic_readl(nvic_state *s, uint32_t offset)
 {
-    nvic_state *s = (nvic_state *)opaque;
     uint32_t val;
     int irq;
 
@@ -216,14 +215,6 @@ static uint32_t nvic_readl(void *opaque, uint32_t offset)
     case 0xd14: /* Configuration Control.  */
         /* TODO: Implement Configuration Control bits.  */
         return 0;
-    case 0xd18: case 0xd1c: case 0xd20: /* System Handler Priority.  */
-        irq = offset - 0xd14;
-        val = 0;
-        val |= s->gic.priority1[irq++][0];
-        val |= s->gic.priority1[irq++][0] << 8;
-        val |= s->gic.priority1[irq++][0] << 16;
-        val |= s->gic.priority1[irq][0] << 24;
-        return val;
     case 0xd24: /* System Handler Status.  */
         val = 0;
         if (s->gic.irq_state[ARMV7M_EXCP_MEM].active) val |= (1 << 0);
@@ -243,7 +234,7 @@ static uint32_t nvic_readl(void *opaque, uint32_t offset)
         return val;
     case 0xd28: /* Configurable Fault Status.  */
         /* TODO: Implement Fault Status.  */
-        hw_error("Not implemented: Configurable Fault Status.");
+        qemu_log_mask(LOG_UNIMP, "Configurable Fault Status unimplemented\n");
         return 0;
     case 0xd2c: /* Hard Fault Status.  */
     case 0xd30: /* Debug Fault Status.  */
@@ -251,7 +242,8 @@ static uint32_t nvic_readl(void *opaque, uint32_t offset)
     case 0xd38: /* Bus Fault Address.  */
     case 0xd3c: /* Aux Fault Status.  */
         /* TODO: Implement fault status registers.  */
-        goto bad_reg;
+        qemu_log_mask(LOG_UNIMP, "Fault status registers unimplemented\n");
+        return 0;
     case 0xd40: /* PFR0.  */
         return 0x00000030;
     case 0xd44: /* PRF1.  */
@@ -280,14 +272,13 @@ static uint32_t nvic_readl(void *opaque, uint32_t offset)
         return 0x01310102;
     /* TODO: Implement debug registers.  */
     default:
-    bad_reg:
-        hw_error("NVIC: Bad read offset 0x%x\n", offset);
+        qemu_log_mask(LOG_GUEST_ERROR, "NVIC: Bad read offset 0x%x\n", offset);
+        return 0;
     }
 }
 
-static void nvic_writel(void *opaque, uint32_t offset, uint32_t value)
+static void nvic_writel(nvic_state *s, uint32_t offset, uint32_t value)
 {
-    nvic_state *s = (nvic_state *)opaque;
     uint32_t oldval;
     switch (offset) {
     case 0x10: /* SysTick Control and Status.  */
@@ -345,27 +336,17 @@ static void nvic_writel(void *opaque, uint32_t offset, uint32_t value)
     case 0xd0c: /* Application Interrupt/Reset Control.  */
         if ((value >> 16) == 0x05fa) {
             if (value & 2) {
-                hw_error("VECTCLRACTIVE not implemented");
+                qemu_log_mask(LOG_UNIMP, "VECTCLRACTIVE unimplemented\n");
             }
             if (value & 5) {
-                hw_error("System reset");
+                qemu_log_mask(LOG_UNIMP, "AIRCR system reset unimplemented\n");
             }
         }
         break;
     case 0xd10: /* System Control.  */
     case 0xd14: /* Configuration Control.  */
         /* TODO: Implement control registers.  */
-        goto bad_reg;
-    case 0xd18: case 0xd1c: case 0xd20: /* System Handler Priority.  */
-        {
-            int irq;
-            irq = offset - 0xd14;
-            s->gic.priority1[irq++][0] = value & 0xff;
-            s->gic.priority1[irq++][0] = (value >> 8) & 0xff;
-            s->gic.priority1[irq++][0] = (value >> 16) & 0xff;
-            s->gic.priority1[irq][0] = (value >> 24) & 0xff;
-            gic_update(&s->gic);
-        }
+        qemu_log_mask(LOG_UNIMP, "NVIC: SCR and CCR unimplemented\n");
         break;
     case 0xd24: /* System Handler Control.  */
         /* TODO: Real hardware allows you to set/clear the active bits
@@ -380,47 +361,71 @@ static void nvic_writel(void *opaque, uint32_t offset, uint32_t value)
     case 0xd34: /* Mem Manage Address.  */
     case 0xd38: /* Bus Fault Address.  */
     case 0xd3c: /* Aux Fault Status.  */
-        goto bad_reg;
+        qemu_log_mask(LOG_UNIMP,
+                      "NVIC: fault status registers unimplemented\n");
+        break;
     case 0xf00: /* Software Triggered Interrupt Register */
         if ((value & 0x1ff) < s->num_irq) {
             gic_set_pending_private(&s->gic, 0, value & 0x1ff);
         }
         break;
     default:
-    bad_reg:
-        hw_error("NVIC: Bad write offset 0x%x\n", offset);
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "NVIC: Bad write offset 0x%x\n", offset);
     }
 }
 
-static uint64_t nvic_sysreg_read(void *opaque, target_phys_addr_t addr,
+static uint64_t nvic_sysreg_read(void *opaque, hwaddr addr,
                                  unsigned size)
 {
-    /* At the moment we only support the ID registers for byte/word access.
-     * This is not strictly correct as a few of the other registers also
-     * allow byte access.
-     */
+    nvic_state *s = (nvic_state *)opaque;
     uint32_t offset = addr;
-    if (offset >= 0xfe0) {
+    int i;
+    uint32_t val;
+
+    switch (offset) {
+    case 0xd18 ... 0xd23: /* System Handler Priority.  */
+        val = 0;
+        for (i = 0; i < size; i++) {
+            val |= s->gic.priority1[(offset - 0xd14) + i][0] << (i * 8);
+        }
+        return val;
+    case 0xfe0 ... 0xfff: /* ID.  */
         if (offset & 3) {
             return 0;
         }
         return nvic_id[(offset - 0xfe0) >> 2];
     }
     if (size == 4) {
-        return nvic_readl(opaque, offset);
+        return nvic_readl(s, offset);
     }
-    hw_error("NVIC: Bad read of size %d at offset 0x%x\n", size, offset);
+    qemu_log_mask(LOG_GUEST_ERROR,
+                  "NVIC: Bad read of size %d at offset 0x%x\n", size, offset);
+    return 0;
 }
 
-static void nvic_sysreg_write(void *opaque, target_phys_addr_t addr,
+static void nvic_sysreg_write(void *opaque, hwaddr addr,
                               uint64_t value, unsigned size)
 {
+    nvic_state *s = (nvic_state *)opaque;
     uint32_t offset = addr;
+    int i;
+
+    switch (offset) {
+    case 0xd18 ... 0xd23: /* System Handler Priority.  */
+        for (i = 0; i < size; i++) {
+            s->gic.priority1[(offset - 0xd14) + i][0] =
+                (value >> (i * 8)) & 0xff;
+        }
+        gic_update(&s->gic);
+        return;
+    }
     if (size == 4) {
-        nvic_writel(opaque, offset, value);
+        nvic_writel(s, offset, value);
         return;
     }
-    hw_error("NVIC: Bad write of size %d at offset 0x%x\n", size, offset);
+    qemu_log_mask(LOG_GUEST_ERROR,
+                  "NVIC: Bad write of size %d at offset 0x%x\n", size, offset);
 }
 
 static const MemoryRegionOps nvic_sysreg_ops = {
@@ -489,7 +494,8 @@ static int armv7m_nvic_init(SysBusDevice *dev)
      */
     memory_region_init_alias(&s->gic_iomem_alias, "nvic-gic", &s->gic.iomem,
                              0x100, 0xc00);
-    memory_region_add_subregion_overlap(&s->container, 0x100, &s->gic.iomem, 1);
+    memory_region_add_subregion_overlap(&s->container, 0x100,
+                                        &s->gic_iomem_alias, 1);
     /* Map the whole thing into system memory at the location required
      * by the v7M architecture.
      */
@@ -504,9 +510,9 @@ static void armv7m_nvic_instance_init(Object *obj)
      * than our superclass. This function runs after qdev init
      * has set the defaults from the Property array and before
      * any user-specified property setting, so just modify the
-     * value in the gic_state struct.
+     * value in the GICState struct.
      */
-    gic_state *s = ARM_GIC_COMMON(obj);
+    GICState *s = ARM_GIC_COMMON(obj);
     /* The ARM v7m may have anything from 0 to 496 external interrupt
      * IRQ lines. We default to 64. Other boards may differ and should
      * set the num-irq property appropriately.
diff --git a/hw/axis_dev88.c b/hw/axis_dev88.c
index eab6327be..aa1ac9e47 100644
--- a/hw/axis_dev88.c
+++ b/hw/axis_dev88.c
@@ -47,7 +47,7 @@ struct nand_state_t
 };
 
 static struct nand_state_t nand_state;
-static uint64_t nand_read(void *opaque, target_phys_addr_t addr, unsigned size)
+static uint64_t nand_read(void *opaque, hwaddr addr, unsigned size)
 {
     struct nand_state_t *s = opaque;
     uint32_t r;
@@ -62,7 +62,7 @@ static uint64_t nand_read(void *opaque, target_phys_addr_t addr, unsigned size)
 }
 
 static void
-nand_write(void *opaque, target_phys_addr_t addr, uint64_t value,
+nand_write(void *opaque, hwaddr addr, uint64_t value,
            unsigned size)
 {
     struct nand_state_t *s = opaque;
@@ -166,7 +166,7 @@ static struct gpio_state_t
     uint32_t regs[0x5c / 4];
 } gpio_state;
 
-static uint64_t gpio_read(void *opaque, target_phys_addr_t addr, unsigned size)
+static uint64_t gpio_read(void *opaque, hwaddr addr, unsigned size)
 {
     struct gpio_state_t *s = opaque;
     uint32_t r = 0;
@@ -195,7 +195,7 @@ static uint64_t gpio_read(void *opaque, target_phys_addr_t addr, unsigned size)
     D(printf("%s %x=%x\n", __func__, addr, r));
 }
 
-static void gpio_write(void *opaque, target_phys_addr_t addr, uint64_t value,
+static void gpio_write(void *opaque, hwaddr addr, uint64_t value,
                        unsigned size)
 {
     struct gpio_state_t *s = opaque;
@@ -242,11 +242,12 @@ static const MemoryRegionOps gpio_ops = {
 static struct cris_load_info li;
 
 static
-void axisdev88_init (ram_addr_t ram_size,
-                     const char *boot_device,
-                     const char *kernel_filename, const char *kernel_cmdline,
-                     const char *initrd_filename, const char *cpu_model)
+void axisdev88_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t ram_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
     CRISCPU *cpu;
     CPUCRISState *env;
     DeviceState *dev;
diff --git a/hw/blizzard.c b/hw/blizzard.c
index 29074c4fa..1b57eb539 100644
--- a/hw/blizzard.c
+++ b/hw/blizzard.c
@@ -878,8 +878,6 @@ void s1d13745_write_block(void *opaque, int dc,
         len -= 2;
         buf += 2;
     }
-
-    return;
 }
 
 static void blizzard_update_display(void *opaque)
@@ -923,8 +921,8 @@ static void blizzard_update_display(void *opaque)
     for (; y < s->my[1]; y ++, src += bypl, dst += bypl)
         memcpy(dst, src, bwidth);
 
-    dpy_update(s->state, s->mx[0], s->my[0],
-                    s->mx[1] - s->mx[0], y - s->my[0]);
+    dpy_gfx_update(s->state, s->mx[0], s->my[0],
+                   s->mx[1] - s->mx[0], y - s->my[0]);
 
     s->mx[0] = s->x;
     s->mx[1] = 0;
@@ -933,13 +931,13 @@ static void blizzard_update_display(void *opaque)
 }
 
 static void blizzard_screen_dump(void *opaque, const char *filename,
-                                 bool cswitch)
+                                 bool cswitch, Error **errp)
 {
     BlizzardState *s = (BlizzardState *) opaque;
 
     blizzard_update_display(opaque);
     if (s && ds_get_data(s->state))
-        ppm_save(filename, s->state->surface);
+        ppm_save(filename, s->state->surface, errp);
 }
 
 #define DEPTH 8
diff --git a/hw/boards.h b/hw/boards.h
index a2e0a5449..813d0e510 100644
--- a/hw/boards.h
+++ b/hw/boards.h
@@ -5,12 +5,16 @@
 
 #include "qdev.h"
 
-typedef void QEMUMachineInitFunc(ram_addr_t ram_size,
-                                 const char *boot_device,
-                                 const char *kernel_filename,
-                                 const char *kernel_cmdline,
-                                 const char *initrd_filename,
-                                 const char *cpu_model);
+typedef struct QEMUMachineInitArgs {
+    ram_addr_t ram_size;
+    const char *boot_device;
+    const char *kernel_filename;
+    const char *kernel_cmdline;
+    const char *initrd_filename;
+    const char *cpu_model;
+} QEMUMachineInitArgs;
+
+typedef void QEMUMachineInitFunc(QEMUMachineInitArgs *args);
 
 typedef void QEMUMachineResetFunc(void);
 
diff --git a/hw/bonito.c b/hw/bonito.c
index 6084ac48f..0bf6d4aa5 100644
--- a/hw/bonito.c
+++ b/hw/bonito.c
@@ -211,12 +211,12 @@ typedef struct PCIBonitoState
     MemoryRegion iomem_ldma;
     MemoryRegion iomem_cop;
 
-    target_phys_addr_t bonito_pciio_start;
-    target_phys_addr_t bonito_pciio_length;
+    hwaddr bonito_pciio_start;
+    hwaddr bonito_pciio_length;
     int bonito_pciio_handle;
 
-    target_phys_addr_t bonito_localio_start;
-    target_phys_addr_t bonito_localio_length;
+    hwaddr bonito_localio_start;
+    hwaddr bonito_localio_length;
     int bonito_localio_handle;
 
 } PCIBonitoState;
@@ -232,7 +232,7 @@ struct BonitoState {
     PCIBonitoState *pci_dev;
 };
 
-static void bonito_writel(void *opaque, target_phys_addr_t addr,
+static void bonito_writel(void *opaque, hwaddr addr,
                           uint64_t val, unsigned size)
 {
     PCIBonitoState *s = opaque;
@@ -295,7 +295,7 @@ static void bonito_writel(void *opaque, target_phys_addr_t addr,
     }
 }
 
-static uint64_t bonito_readl(void *opaque, target_phys_addr_t addr,
+static uint64_t bonito_readl(void *opaque, hwaddr addr,
                              unsigned size)
 {
     PCIBonitoState *s = opaque;
@@ -322,7 +322,7 @@ static const MemoryRegionOps bonito_ops = {
     },
 };
 
-static void bonito_pciconf_writel(void *opaque, target_phys_addr_t addr,
+static void bonito_pciconf_writel(void *opaque, hwaddr addr,
                                   uint64_t val, unsigned size)
 {
     PCIBonitoState *s = opaque;
@@ -332,7 +332,7 @@ static void bonito_pciconf_writel(void *opaque, target_phys_addr_t addr,
     d->config_write(d, addr, val, 4);
 }
 
-static uint64_t bonito_pciconf_readl(void *opaque, target_phys_addr_t addr,
+static uint64_t bonito_pciconf_readl(void *opaque, hwaddr addr,
                                      unsigned size)
 {
 
@@ -355,7 +355,7 @@ static const MemoryRegionOps bonito_pciconf_ops = {
     },
 };
 
-static uint64_t bonito_ldma_readl(void *opaque, target_phys_addr_t addr,
+static uint64_t bonito_ldma_readl(void *opaque, hwaddr addr,
                                   unsigned size)
 {
     uint32_t val;
@@ -366,7 +366,7 @@ static uint64_t bonito_ldma_readl(void *opaque, target_phys_addr_t addr,
     return val;
 }
 
-static void bonito_ldma_writel(void *opaque, target_phys_addr_t addr,
+static void bonito_ldma_writel(void *opaque, hwaddr addr,
                                uint64_t val, unsigned size)
 {
     PCIBonitoState *s = opaque;
@@ -384,7 +384,7 @@ static const MemoryRegionOps bonito_ldma_ops = {
     },
 };
 
-static uint64_t bonito_cop_readl(void *opaque, target_phys_addr_t addr,
+static uint64_t bonito_cop_readl(void *opaque, hwaddr addr,
                                  unsigned size)
 {
     uint32_t val;
@@ -395,7 +395,7 @@ static uint64_t bonito_cop_readl(void *opaque, target_phys_addr_t addr,
     return val;
 }
 
-static void bonito_cop_writel(void *opaque, target_phys_addr_t addr,
+static void bonito_cop_writel(void *opaque, hwaddr addr,
                               uint64_t val, unsigned size)
 {
     PCIBonitoState *s = opaque;
@@ -413,7 +413,7 @@ static const MemoryRegionOps bonito_cop_ops = {
     },
 };
 
-static uint32_t bonito_sbridge_pciaddr(void *opaque, target_phys_addr_t addr)
+static uint32_t bonito_sbridge_pciaddr(void *opaque, hwaddr addr)
 {
     PCIBonitoState *s = opaque;
     PCIHostState *phb = PCI_HOST_BRIDGE(s->pcihost);
@@ -449,7 +449,7 @@ static uint32_t bonito_sbridge_pciaddr(void *opaque, target_phys_addr_t addr)
     return pciaddr;
 }
 
-static void bonito_spciconf_writeb(void *opaque, target_phys_addr_t addr,
+static void bonito_spciconf_writeb(void *opaque, hwaddr addr,
                                    uint32_t val)
 {
     PCIBonitoState *s = opaque;
@@ -475,7 +475,7 @@ static void bonito_spciconf_writeb(void *opaque, target_phys_addr_t addr,
     pci_set_word(d->config + PCI_STATUS, status);
 }
 
-static void bonito_spciconf_writew(void *opaque, target_phys_addr_t addr,
+static void bonito_spciconf_writew(void *opaque, hwaddr addr,
                                    uint32_t val)
 {
     PCIBonitoState *s = opaque;
@@ -503,7 +503,7 @@ static void bonito_spciconf_writew(void *opaque, target_phys_addr_t addr,
     pci_set_word(d->config + PCI_STATUS, status);
 }
 
-static void bonito_spciconf_writel(void *opaque, target_phys_addr_t addr,
+static void bonito_spciconf_writel(void *opaque, hwaddr addr,
                                    uint32_t val)
 {
     PCIBonitoState *s = opaque;
@@ -531,7 +531,7 @@ static void bonito_spciconf_writel(void *opaque, target_phys_addr_t addr,
     pci_set_word(d->config + PCI_STATUS, status);
 }
 
-static uint32_t bonito_spciconf_readb(void *opaque, target_phys_addr_t addr)
+static uint32_t bonito_spciconf_readb(void *opaque, hwaddr addr)
 {
     PCIBonitoState *s = opaque;
     PCIDevice *d = PCI_DEVICE(s);
@@ -557,7 +557,7 @@ static uint32_t bonito_spciconf_readb(void *opaque, target_phys_addr_t addr)
     return pci_data_read(phb->bus, phb->config_reg, 1);
 }
 
-static uint32_t bonito_spciconf_readw(void *opaque, target_phys_addr_t addr)
+static uint32_t bonito_spciconf_readw(void *opaque, hwaddr addr)
 {
     PCIBonitoState *s = opaque;
     PCIDevice *d = PCI_DEVICE(s);
@@ -585,7 +585,7 @@ static uint32_t bonito_spciconf_readw(void *opaque, target_phys_addr_t addr)
     return pci_data_read(phb->bus, phb->config_reg, 2);
 }
 
-static uint32_t bonito_spciconf_readl(void *opaque, target_phys_addr_t addr)
+static uint32_t bonito_spciconf_readl(void *opaque, hwaddr addr)
 {
     PCIBonitoState *s = opaque;
     PCIDevice *d = PCI_DEVICE(s);
diff --git a/hw/bt-hci.c b/hw/bt-hci.c
index a3a7fb49e..e54cfd781 100644
--- a/hw/bt-hci.c
+++ b/hw/bt-hci.c
@@ -786,7 +786,6 @@ static void bt_hci_lmp_connection_request(struct bt_link_s *link)
     memcpy(&params.dev_class, &link->host->class, sizeof(params.dev_class));
     params.link_type	= ACL_LINK;
     bt_hci_event(hci, EVT_CONN_REQUEST, &params, EVT_CONN_REQUEST_SIZE);
-    return;
 }
 
 static void bt_hci_conn_accept_timeout(void *opaque)
@@ -943,7 +942,6 @@ static int bt_hci_name_req(struct bt_hci_s *hci, bdaddr_t *bdaddr)
 {
     struct bt_device_s *slave;
     evt_remote_name_req_complete params;
-    int len;
 
     for (slave = hci->device.net->slave; slave; slave = slave->next)
         if (slave->page_scan && !bacmp(&slave->bd_addr, bdaddr))
@@ -955,9 +953,7 @@ static int bt_hci_name_req(struct bt_hci_s *hci, bdaddr_t *bdaddr)
 
     params.status       = HCI_SUCCESS;
     bacpy(&params.bdaddr, &slave->bd_addr);
-    len = snprintf(params.name, sizeof(params.name),
-                    "%s", slave->lmp_name ?: "");
-    memset(params.name + len, 0, sizeof(params.name) - len);
+    pstrcpy(params.name, sizeof(params.name), slave->lmp_name ?: "");
     bt_hci_event(hci, EVT_REMOTE_NAME_REQ_COMPLETE,
                     &params, EVT_REMOTE_NAME_REQ_COMPLETE_SIZE);
 
@@ -1388,7 +1384,7 @@ static inline void bt_hci_event_complete_read_local_name(struct bt_hci_s *hci)
     params.status = HCI_SUCCESS;
     memset(params.name, 0, sizeof(params.name));
     if (hci->device.lmp_name)
-        strncpy(params.name, hci->device.lmp_name, sizeof(params.name));
+        pstrcpy(params.name, sizeof(params.name), hci->device.lmp_name);
 
     bt_hci_event_complete(hci, &params, READ_LOCAL_NAME_RP_SIZE);
 }
diff --git a/hw/bt.h b/hw/bt.h
index a48b8d4b1..ebf6a370a 100644
--- a/hw/bt.h
+++ b/hw/bt.h
@@ -23,6 +23,8 @@
  * along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "hw/irq.h"
+
 /* BD Address */
 typedef struct {
     uint8_t b[6];
diff --git a/hw/cadence_gem.c b/hw/cadence_gem.c
index 967f62513..0c037a299 100644
--- a/hw/cadence_gem.c
+++ b/hw/cadence_gem.c
@@ -605,7 +605,7 @@ static int gem_mac_address_filter(GemState *s, const uint8_t *packet)
 static ssize_t gem_receive(NetClientState *nc, const uint8_t *buf, size_t size)
 {
     unsigned    desc[2];
-    target_phys_addr_t packet_desc_addr, last_desc_addr;
+    hwaddr packet_desc_addr, last_desc_addr;
     GemState *s;
     unsigned   rxbufsize, bytes_to_copy;
     unsigned   rxbuf_offset;
@@ -824,7 +824,7 @@ static void gem_transmit_updatestats(GemState *s, const uint8_t *packet,
 static void gem_transmit(GemState *s)
 {
     unsigned    desc[2];
-    target_phys_addr_t packet_desc_addr;
+    hwaddr packet_desc_addr;
     uint8_t     tx_packet[2048];
     uint8_t     *p;
     unsigned    total_bytes;
@@ -1021,7 +1021,7 @@ static void gem_phy_write(GemState *s, unsigned reg_num, uint16_t val)
  * gem_read32:
  * Read a GEM register.
  */
-static uint64_t gem_read(void *opaque, target_phys_addr_t offset, unsigned size)
+static uint64_t gem_read(void *opaque, hwaddr offset, unsigned size)
 {
     GemState *s;
     uint32_t retval;
@@ -1067,7 +1067,7 @@ static uint64_t gem_read(void *opaque, target_phys_addr_t offset, unsigned size)
  * gem_write32:
  * Write a GEM register.
  */
-static void gem_write(void *opaque, target_phys_addr_t offset, uint64_t val,
+static void gem_write(void *opaque, hwaddr offset, uint64_t val,
         unsigned size)
 {
     GemState *s = (GemState *)opaque;
diff --git a/hw/cadence_ttc.c b/hw/cadence_ttc.c
index dd02f86eb..ec78a5218 100644
--- a/hw/cadence_ttc.c
+++ b/hw/cadence_ttc.c
@@ -76,7 +76,7 @@ static void cadence_timer_update(CadenceTimerState *s)
 }
 
 static CadenceTimerState *cadence_timer_from_addr(void *opaque,
-                                        target_phys_addr_t offset)
+                                        hwaddr offset)
 {
     unsigned int index;
     CadenceTTCState *s = (CadenceTTCState *)opaque;
@@ -224,7 +224,7 @@ static void cadence_timer_tick(void *opaque)
     cadence_timer_run(s);
 }
 
-static uint32_t cadence_ttc_read_imp(void *opaque, target_phys_addr_t offset)
+static uint32_t cadence_ttc_read_imp(void *opaque, hwaddr offset)
 {
     CadenceTimerState *s = cadence_timer_from_addr(opaque, offset);
     uint32_t value;
@@ -274,6 +274,7 @@ static uint32_t cadence_ttc_read_imp(void *opaque, target_phys_addr_t offset)
         /* cleared after read */
         value = s->reg_intr;
         s->reg_intr = 0;
+        cadence_timer_update(s);
         return value;
 
     case 0x60: /* interrupt enable */
@@ -296,7 +297,7 @@ static uint32_t cadence_ttc_read_imp(void *opaque, target_phys_addr_t offset)
     }
 }
 
-static uint64_t cadence_ttc_read(void *opaque, target_phys_addr_t offset,
+static uint64_t cadence_ttc_read(void *opaque, hwaddr offset,
     unsigned size)
 {
     uint32_t ret = cadence_ttc_read_imp(opaque, offset);
@@ -305,7 +306,7 @@ static uint64_t cadence_ttc_read(void *opaque, target_phys_addr_t offset,
     return ret;
 }
 
-static void cadence_ttc_write(void *opaque, target_phys_addr_t offset,
+static void cadence_ttc_write(void *opaque, hwaddr offset,
         uint64_t value, unsigned size)
 {
     CadenceTimerState *s = cadence_timer_from_addr(opaque, offset);
@@ -355,7 +356,6 @@ static void cadence_ttc_write(void *opaque, target_phys_addr_t offset,
     case 0x54: /* interrupt register */
     case 0x58:
     case 0x5c:
-        s->reg_intr &= (~value & 0xfff);
         break;
 
     case 0x60: /* interrupt enable */
diff --git a/hw/cadence_uart.c b/hw/cadence_uart.c
index d98e53137..686e6172d 100644
--- a/hw/cadence_uart.c
+++ b/hw/cadence_uart.c
@@ -354,12 +354,12 @@ static void uart_read_rx_fifo(UartState *s, uint32_t *c)
     uart_update_status(s);
 }
 
-static void uart_write(void *opaque, target_phys_addr_t offset,
+static void uart_write(void *opaque, hwaddr offset,
                           uint64_t value, unsigned size)
 {
     UartState *s = (UartState *)opaque;
 
-    DB_PRINT(" offset:%x data:%08x\n", offset, (unsigned)value);
+    DB_PRINT(" offset:%x data:%08x\n", (unsigned)offset, (unsigned)value);
     offset >>= 2;
     switch (offset) {
     case R_IER: /* ier (wts imr) */
@@ -397,20 +397,23 @@ static void uart_write(void *opaque, target_phys_addr_t offset,
     }
 }
 
-static uint64_t uart_read(void *opaque, target_phys_addr_t offset,
+static uint64_t uart_read(void *opaque, hwaddr offset,
         unsigned size)
 {
     UartState *s = (UartState *)opaque;
     uint32_t c = 0;
 
     offset >>= 2;
-    if (offset > R_MAX) {
-        return 0;
+    if (offset >= R_MAX) {
+        c = 0;
     } else if (offset == R_TX_RX) {
         uart_read_rx_fifo(s, &c);
-        return c;
+    } else {
+       c = s->r[offset];
     }
-    return s->r[offset];
+
+    DB_PRINT(" offset:%x data:%08x\n", (unsigned)(offset << 2), (unsigned)c);
+    return c;
 }
 
 static const MemoryRegionOps uart_ops = {
diff --git a/hw/cirrus_vga.c b/hw/cirrus_vga.c
index e8dcc6b88..9bef96e6d 100644
--- a/hw/cirrus_vga.c
+++ b/hw/cirrus_vga.c
@@ -28,7 +28,6 @@
  */
 #include "hw.h"
 #include "pci.h"
-#include "vga-pci.h"
 #include "console.h"
 #include "vga_int.h"
 #include "loader.h"
@@ -43,8 +42,6 @@
 //#define DEBUG_CIRRUS
 //#define DEBUG_BITBLT
 
-#define VGA_RAM_SIZE (8192 * 1024)
-
 /***************************************
  *
  *  definitions
@@ -1953,7 +1950,7 @@ static void cirrus_mem_writeb_mode4and5_16bpp(CirrusVGAState * s,
  ***************************************/
 
 static uint64_t cirrus_vga_mem_read(void *opaque,
-                                    target_phys_addr_t addr,
+                                    hwaddr addr,
                                     uint32_t size)
 {
     CirrusVGAState *s = opaque;
@@ -1997,7 +1994,7 @@ static uint64_t cirrus_vga_mem_read(void *opaque,
 }
 
 static void cirrus_vga_mem_write(void *opaque,
-                                 target_phys_addr_t addr,
+                                 hwaddr addr,
                                  uint64_t mem_value,
                                  uint32_t size)
 {
@@ -2256,7 +2253,7 @@ static void cirrus_cursor_draw_line(VGACommonState *s1, uint8_t *d1, int scr_y)
  *
  ***************************************/
 
-static uint64_t cirrus_linear_read(void *opaque, target_phys_addr_t addr,
+static uint64_t cirrus_linear_read(void *opaque, hwaddr addr,
                                    unsigned size)
 {
     CirrusVGAState *s = opaque;
@@ -2285,7 +2282,7 @@ static uint64_t cirrus_linear_read(void *opaque, target_phys_addr_t addr,
     return ret;
 }
 
-static void cirrus_linear_write(void *opaque, target_phys_addr_t addr,
+static void cirrus_linear_write(void *opaque, hwaddr addr,
                                 uint64_t val, unsigned size)
 {
     CirrusVGAState *s = opaque;
@@ -2334,7 +2331,7 @@ static void cirrus_linear_write(void *opaque, target_phys_addr_t addr,
 
 
 static uint64_t cirrus_linear_bitblt_read(void *opaque,
-                                          target_phys_addr_t addr,
+                                          hwaddr addr,
                                           unsigned size)
 {
     CirrusVGAState *s = opaque;
@@ -2347,7 +2344,7 @@ static uint64_t cirrus_linear_bitblt_read(void *opaque,
 }
 
 static void cirrus_linear_bitblt_write(void *opaque,
-                                       target_phys_addr_t addr,
+                                       hwaddr addr,
                                        uint64_t val,
                                        unsigned size)
 {
@@ -2441,6 +2438,8 @@ static uint32_t cirrus_vga_ioport_read(void *opaque, uint32_t addr)
     VGACommonState *s = &c->vga;
     int val, index;
 
+    qemu_flush_coalesced_mmio_buffer();
+
     if (vga_ioport_invalid(s, addr)) {
 	val = 0xff;
     } else {
@@ -2534,6 +2533,8 @@ static void cirrus_vga_ioport_write(void *opaque, uint32_t addr, uint32_t val)
     VGACommonState *s = &c->vga;
     int index;
 
+    qemu_flush_coalesced_mmio_buffer();
+
     /* check port range access depending on color/monochrome mode */
     if (vga_ioport_invalid(s, addr)) {
 	return;
@@ -2637,7 +2638,7 @@ static void cirrus_vga_ioport_write(void *opaque, uint32_t addr, uint32_t val)
  *
  ***************************************/
 
-static uint64_t cirrus_mmio_read(void *opaque, target_phys_addr_t addr,
+static uint64_t cirrus_mmio_read(void *opaque, hwaddr addr,
                                  unsigned size)
 {
     CirrusVGAState *s = opaque;
@@ -2649,7 +2650,7 @@ static uint64_t cirrus_mmio_read(void *opaque, target_phys_addr_t addr,
     }
 }
 
-static void cirrus_mmio_write(void *opaque, target_phys_addr_t addr,
+static void cirrus_mmio_write(void *opaque, hwaddr addr,
                               uint64_t val, unsigned size)
 {
     CirrusVGAState *s = opaque;
@@ -2853,7 +2854,9 @@ static void cirrus_init_common(CirrusVGAState * s, int device_id, int is_pci,
 
     /* I/O handler for LFB */
     memory_region_init_io(&s->cirrus_linear_io, &cirrus_linear_io_ops, s,
-                          "cirrus-linear-io", VGA_RAM_SIZE);
+                          "cirrus-linear-io", s->vga.vram_size_mb
+                                              * 1024 * 1024);
+    memory_region_set_flush_coalesced(&s->cirrus_linear_io);
 
     /* I/O handler for LFB */
     memory_region_init_io(&s->cirrus_linear_bitblt_io,
@@ -2861,10 +2864,12 @@ static void cirrus_init_common(CirrusVGAState * s, int device_id, int is_pci,
                           s,
                           "cirrus-bitblt-mmio",
                           0x400000);
+    memory_region_set_flush_coalesced(&s->cirrus_linear_bitblt_io);
 
     /* I/O handler for memory-mapped I/O */
     memory_region_init_io(&s->cirrus_mmio_io, &cirrus_mmio_io_ops, s,
                           "cirrus-mmio", CIRRUS_PNPMMIO_SIZE);
+    memory_region_set_flush_coalesced(&s->cirrus_mmio_io);
 
     s->real_vram_size =
         (s->device_id == CIRRUS_ID_CLGD5446) ? 4096 * 1024 : 2048 * 1024;
@@ -2893,7 +2898,6 @@ static int vga_initfn(ISADevice *dev)
     ISACirrusVGAState *d = DO_UPCAST(ISACirrusVGAState, dev, dev);
     VGACommonState *s = &d->cirrus_vga.vga;
 
-    s->vram_size_mb = VGA_RAM_SIZE >> 20;
     vga_common_init(s);
     cirrus_init_common(&d->cirrus_vga, CIRRUS_ID_CLGD5430, 0,
                        isa_address_space(dev));
@@ -2906,6 +2910,12 @@ static int vga_initfn(ISADevice *dev)
     return 0;
 }
 
+static Property isa_vga_cirrus_properties[] = {
+    DEFINE_PROP_UINT32("vgamem_mb", struct ISACirrusVGAState,
+                       cirrus_vga.vga.vram_size_mb, 8),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
 static void isa_cirrus_vga_class_init(ObjectClass *klass, void *data)
 {
     ISADeviceClass *k = ISA_DEVICE_CLASS(klass);
@@ -2913,6 +2923,7 @@ static void isa_cirrus_vga_class_init(ObjectClass *klass, void *data)
 
     dc->vmsd  = &vmstate_cirrus_vga;
     k->init   = vga_initfn;
+    dc->props = isa_vga_cirrus_properties;
 }
 
 static TypeInfo isa_cirrus_vga_info = {
@@ -2936,7 +2947,6 @@ static int pci_cirrus_vga_initfn(PCIDevice *dev)
      int16_t device_id = pc->device_id;
 
      /* setup VGA */
-     s->vga.vram_size_mb = VGA_RAM_SIZE >> 20;
      vga_common_init(&s->vga);
      cirrus_init_common(s, device_id, 1, pci_address_space(dev));
      s->vga.ds = graphic_console_init(s->vga.update, s->vga.invalidate,
@@ -2963,10 +2973,11 @@ static int pci_cirrus_vga_initfn(PCIDevice *dev)
      return 0;
 }
 
-DeviceState *pci_cirrus_vga_init(PCIBus *bus)
-{
-    return &pci_create_simple(bus, -1, "cirrus-vga")->qdev;
-}
+static Property pci_vga_cirrus_properties[] = {
+    DEFINE_PROP_UINT32("vgamem_mb", struct PCICirrusVGAState,
+                       cirrus_vga.vga.vram_size_mb, 8),
+    DEFINE_PROP_END_OF_LIST(),
+};
 
 static void cirrus_vga_class_init(ObjectClass *klass, void *data)
 {
@@ -2981,6 +2992,7 @@ static void cirrus_vga_class_init(ObjectClass *klass, void *data)
     k->class_id = PCI_CLASS_DISPLAY_VGA;
     dc->desc = "Cirrus CLGD 54xx VGA";
     dc->vmsd = &vmstate_pci_cirrus_vga;
+    dc->props = pci_vga_cirrus_properties;
 }
 
 static TypeInfo cirrus_vga_info = {
diff --git a/hw/collie.c b/hw/collie.c
index 56f89a9f2..695982a99 100644
--- a/hw/collie.c
+++ b/hw/collie.c
@@ -23,11 +23,12 @@ static struct arm_boot_info collie_binfo = {
     .ram_size = 0x20000000,
 };
 
-static void collie_init(ram_addr_t ram_size,
-                const char *boot_device,
-                const char *kernel_filename, const char *kernel_cmdline,
-                const char *initrd_filename, const char *cpu_model)
+static void collie_init(QEMUMachineInitArgs *args)
 {
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
     StrongARMState *s;
     DriveInfo *dinfo;
     MemoryRegion *sysmem = get_system_memory();
diff --git a/hw/cris-boot.h b/hw/cris-boot.h
index 0a2c24241..5b17d83cb 100644
--- a/hw/cris-boot.h
+++ b/hw/cris-boot.h
@@ -5,7 +5,7 @@ struct cris_load_info
     const char *cmdline;
     int image_size;
 
-    target_phys_addr_t entry;
+    hwaddr entry;
 };
 
 void cris_load_image(CRISCPU *cpu, struct cris_load_info *li);
diff --git a/hw/cs4231.c b/hw/cs4231.c
index cfec1d9cd..23570d5b4 100644
--- a/hw/cs4231.c
+++ b/hw/cs4231.c
@@ -55,7 +55,7 @@ static void cs_reset(DeviceState *d)
     s->dregs[25] = CS_VER;
 }
 
-static uint64_t cs_mem_read(void *opaque, target_phys_addr_t addr,
+static uint64_t cs_mem_read(void *opaque, hwaddr addr,
                             unsigned size)
 {
     CSState *s = opaque;
@@ -82,7 +82,7 @@ static uint64_t cs_mem_read(void *opaque, target_phys_addr_t addr,
     return ret;
 }
 
-static void cs_mem_write(void *opaque, target_phys_addr_t addr,
+static void cs_mem_write(void *opaque, hwaddr addr,
                          uint64_t val, unsigned size)
 {
     CSState *s = opaque;
diff --git a/hw/cs4231a.c b/hw/cs4231a.c
index e07b9d623..0257fd8d2 100644
--- a/hw/cs4231a.c
+++ b/hw/cs4231a.c
@@ -346,7 +346,7 @@ static void cs_reset_voices (CSState *s, uint32_t val)
     }
 }
 
-static uint64_t cs_read (void *opaque, target_phys_addr_t addr, unsigned size)
+static uint64_t cs_read (void *opaque, hwaddr addr, unsigned size)
 {
     CSState *s = opaque;
     uint32_t saddr, iaddr, ret;
@@ -383,7 +383,7 @@ static uint64_t cs_read (void *opaque, target_phys_addr_t addr, unsigned size)
     return ret;
 }
 
-static void cs_write (void *opaque, target_phys_addr_t addr,
+static void cs_write (void *opaque, hwaddr addr,
                       uint64_t val64, unsigned size)
 {
     CSState *s = opaque;
diff --git a/hw/cuda.c b/hw/cuda.c
index 233ab666d..f1f408b83 100644
--- a/hw/cuda.c
+++ b/hw/cuda.c
@@ -252,7 +252,7 @@ static void cuda_timer1(void *opaque)
     cuda_update_irq(s);
 }
 
-static uint32_t cuda_readb(void *opaque, target_phys_addr_t addr)
+static uint32_t cuda_readb(void *opaque, hwaddr addr)
 {
     CUDAState *s = opaque;
     uint32_t val;
@@ -325,7 +325,7 @@ static uint32_t cuda_readb(void *opaque, target_phys_addr_t addr)
     return val;
 }
 
-static void cuda_writeb(void *opaque, target_phys_addr_t addr, uint32_t val)
+static void cuda_writeb(void *opaque, hwaddr addr, uint32_t val)
 {
     CUDAState *s = opaque;
 
@@ -616,20 +616,20 @@ static void cuda_receive_packet_from_host(CUDAState *s,
     }
 }
 
-static void cuda_writew (void *opaque, target_phys_addr_t addr, uint32_t value)
+static void cuda_writew (void *opaque, hwaddr addr, uint32_t value)
 {
 }
 
-static void cuda_writel (void *opaque, target_phys_addr_t addr, uint32_t value)
+static void cuda_writel (void *opaque, hwaddr addr, uint32_t value)
 {
 }
 
-static uint32_t cuda_readw (void *opaque, target_phys_addr_t addr)
+static uint32_t cuda_readw (void *opaque, hwaddr addr)
 {
     return 0;
 }
 
-static uint32_t cuda_readl (void *opaque, target_phys_addr_t addr)
+static uint32_t cuda_readl (void *opaque, hwaddr addr)
 {
     return 0;
 }
diff --git a/hw/device-hotplug.c b/hw/device-hotplug.c
index 2bdc615b4..eec0fe314 100644
--- a/hw/device-hotplug.c
+++ b/hw/device-hotplug.c
@@ -89,5 +89,4 @@ err:
     if (dinfo) {
         drive_put_ref(dinfo);
     }
-    return;
 }
diff --git a/hw/devices.h b/hw/devices.h
index 1a55c1e90..c60bcabae 100644
--- a/hw/devices.h
+++ b/hw/devices.h
@@ -1,6 +1,8 @@
 #ifndef QEMU_DEVICES_H
 #define QEMU_DEVICES_H
 
+#include "hw/irq.h"
+
 /* ??? Not all users of this file can include cpu-common.h.  */
 struct MemoryRegion;
 
diff --git a/hw/dma.c b/hw/dma.c
index 0a9322daa..d6aeac283 100644
--- a/hw/dma.c
+++ b/hw/dma.c
@@ -411,7 +411,7 @@ void DMA_register_channel (int nchan,
 int DMA_read_memory (int nchan, void *buf, int pos, int len)
 {
     struct dma_regs *r = &dma_controllers[nchan > 3].regs[nchan & 3];
-    target_phys_addr_t addr = ((r->pageh & 0x7f) << 24) | (r->page << 16) | r->now[ADDR];
+    hwaddr addr = ((r->pageh & 0x7f) << 24) | (r->page << 16) | r->now[ADDR];
 
     if (r->mode & 0x20) {
         int i;
@@ -433,7 +433,7 @@ int DMA_read_memory (int nchan, void *buf, int pos, int len)
 int DMA_write_memory (int nchan, void *buf, int pos, int len)
 {
     struct dma_regs *r = &dma_controllers[nchan > 3].regs[nchan & 3];
-    target_phys_addr_t addr = ((r->pageh & 0x7f) << 24) | (r->page << 16) | r->now[ADDR];
+    hwaddr addr = ((r->pageh & 0x7f) << 24) | (r->page << 16) | r->now[ADDR];
 
     if (r->mode & 0x20) {
         int i;
diff --git a/hw/dp8393x.c b/hw/dp8393x.c
index 4fa6eccba..3f6386eee 100644
--- a/hw/dp8393x.c
+++ b/hw/dp8393x.c
@@ -168,7 +168,7 @@ typedef struct dp8393xState {
     int loopback_packet;
 
     /* Memory access */
-    void (*memory_rw)(void *opaque, target_phys_addr_t addr, uint8_t *buf, int len, int is_write);
+    void (*memory_rw)(void *opaque, hwaddr addr, uint8_t *buf, int len, int is_write);
     void* mem_opaque;
 } dp8393xState;
 
@@ -603,7 +603,7 @@ static void dp8393x_watchdog(void *opaque)
     dp8393x_update_irq(s);
 }
 
-static uint32_t dp8393x_readw(void *opaque, target_phys_addr_t addr)
+static uint32_t dp8393x_readw(void *opaque, hwaddr addr)
 {
     dp8393xState *s = opaque;
     int reg;
@@ -616,13 +616,13 @@ static uint32_t dp8393x_readw(void *opaque, target_phys_addr_t addr)
     return read_register(s, reg);
 }
 
-static uint32_t dp8393x_readb(void *opaque, target_phys_addr_t addr)
+static uint32_t dp8393x_readb(void *opaque, hwaddr addr)
 {
     uint16_t v = dp8393x_readw(opaque, addr & ~0x1);
     return (v >> (8 * (addr & 0x1))) & 0xff;
 }
 
-static uint32_t dp8393x_readl(void *opaque, target_phys_addr_t addr)
+static uint32_t dp8393x_readl(void *opaque, hwaddr addr)
 {
     uint32_t v;
     v = dp8393x_readw(opaque, addr);
@@ -630,7 +630,7 @@ static uint32_t dp8393x_readl(void *opaque, target_phys_addr_t addr)
     return v;
 }
 
-static void dp8393x_writew(void *opaque, target_phys_addr_t addr, uint32_t val)
+static void dp8393x_writew(void *opaque, hwaddr addr, uint32_t val)
 {
     dp8393xState *s = opaque;
     int reg;
@@ -644,7 +644,7 @@ static void dp8393x_writew(void *opaque, target_phys_addr_t addr, uint32_t val)
     write_register(s, reg, (uint16_t)val);
 }
 
-static void dp8393x_writeb(void *opaque, target_phys_addr_t addr, uint32_t val)
+static void dp8393x_writeb(void *opaque, hwaddr addr, uint32_t val)
 {
     uint16_t old_val = dp8393x_readw(opaque, addr & ~0x1);
 
@@ -659,7 +659,7 @@ static void dp8393x_writeb(void *opaque, target_phys_addr_t addr, uint32_t val)
     dp8393x_writew(opaque, addr & ~0x1, val);
 }
 
-static void dp8393x_writel(void *opaque, target_phys_addr_t addr, uint32_t val)
+static void dp8393x_writel(void *opaque, hwaddr addr, uint32_t val)
 {
     dp8393x_writew(opaque, addr, val & 0xffff);
     dp8393x_writew(opaque, addr + 2, (val >> 16) & 0xffff);
@@ -879,10 +879,10 @@ static NetClientInfo net_dp83932_info = {
     .cleanup = nic_cleanup,
 };
 
-void dp83932_init(NICInfo *nd, target_phys_addr_t base, int it_shift,
+void dp83932_init(NICInfo *nd, hwaddr base, int it_shift,
                   MemoryRegion *address_space,
                   qemu_irq irq, void* mem_opaque,
-                  void (*memory_rw)(void *opaque, target_phys_addr_t addr, uint8_t *buf, int len, int is_write))
+                  void (*memory_rw)(void *opaque, hwaddr addr, uint8_t *buf, int len, int is_write))
 {
     dp8393xState *s;
 
diff --git a/hw/ds1225y.c b/hw/ds1225y.c
index 2cd355bd0..4b3f69bc6 100644
--- a/hw/ds1225y.c
+++ b/hw/ds1225y.c
@@ -34,7 +34,7 @@ typedef struct {
     uint8_t *contents;
 } NvRamState;
 
-static uint64_t nvram_read(void *opaque, target_phys_addr_t addr, unsigned size)
+static uint64_t nvram_read(void *opaque, hwaddr addr, unsigned size)
 {
     NvRamState *s = opaque;
     uint32_t val;
@@ -44,7 +44,7 @@ static uint64_t nvram_read(void *opaque, target_phys_addr_t addr, unsigned size)
     return val;
 }
 
-static void nvram_write(void *opaque, target_phys_addr_t addr, uint64_t val,
+static void nvram_write(void *opaque, hwaddr addr, uint64_t val,
                         unsigned size)
 {
     NvRamState *s = opaque;
diff --git a/hw/ds1338.c b/hw/ds1338.c
index d590d9c00..b576d5643 100644
--- a/hw/ds1338.c
+++ b/hw/ds1338.c
@@ -12,39 +12,84 @@
 
 #include "i2c.h"
 
+/* Size of NVRAM including both the user-accessible area and the
+ * secondary register area.
+ */
+#define NVRAM_SIZE 64
+
 typedef struct {
     I2CSlave i2c;
-    time_t offset;
-    struct tm now;
-    uint8_t nvram[56];
-    int ptr;
-    int addr_byte;
+    int64_t offset;
+    uint8_t nvram[NVRAM_SIZE];
+    int32_t ptr;
+    bool addr_byte;
 } DS1338State;
 
+static const VMStateDescription vmstate_ds1338 = {
+    .name = "ds1338",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .minimum_version_id_old = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_I2C_SLAVE(i2c, DS1338State),
+        VMSTATE_INT64(offset, DS1338State),
+        VMSTATE_UINT8_ARRAY(nvram, DS1338State, NVRAM_SIZE),
+        VMSTATE_INT32(ptr, DS1338State),
+        VMSTATE_BOOL(addr_byte, DS1338State),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static void capture_current_time(DS1338State *s)
+{
+    /* Capture the current time into the secondary registers
+     * which will be actually read by the data transfer operation.
+     */
+    struct tm now;
+    qemu_get_timedate(&now, s->offset);
+    s->nvram[0] = to_bcd(now.tm_sec);
+    s->nvram[1] = to_bcd(now.tm_min);
+    if (s->nvram[2] & 0x40) {
+        s->nvram[2] = (to_bcd((now.tm_hour % 12)) + 1) | 0x40;
+        if (now.tm_hour >= 12) {
+            s->nvram[2] |= 0x20;
+        }
+    } else {
+        s->nvram[2] = to_bcd(now.tm_hour);
+    }
+    s->nvram[3] = to_bcd(now.tm_wday) + 1;
+    s->nvram[4] = to_bcd(now.tm_mday);
+    s->nvram[5] = to_bcd(now.tm_mon) + 1;
+    s->nvram[6] = to_bcd(now.tm_year - 100);
+}
+
+static void inc_regptr(DS1338State *s)
+{
+    /* The register pointer wraps around after 0x3F; wraparound
+     * causes the current time/date to be retransferred into
+     * the secondary registers.
+     */
+    s->ptr = (s->ptr + 1) & (NVRAM_SIZE - 1);
+    if (!s->ptr) {
+        capture_current_time(s);
+    }
+}
+
 static void ds1338_event(I2CSlave *i2c, enum i2c_event event)
 {
     DS1338State *s = FROM_I2C_SLAVE(DS1338State, i2c);
 
     switch (event) {
     case I2C_START_RECV:
-        qemu_get_timedate(&s->now, s->offset);
-        s->nvram[0] = to_bcd(s->now.tm_sec);
-        s->nvram[1] = to_bcd(s->now.tm_min);
-        if (s->nvram[2] & 0x40) {
-            s->nvram[2] = (to_bcd((s->now.tm_hour % 12)) + 1) | 0x40;
-            if (s->now.tm_hour >= 12) {
-                s->nvram[2] |= 0x20;
-            }
-        } else {
-            s->nvram[2] = to_bcd(s->now.tm_hour);
-        }
-        s->nvram[3] = to_bcd(s->now.tm_wday) + 1;
-        s->nvram[4] = to_bcd(s->now.tm_mday);
-        s->nvram[5] = to_bcd(s->now.tm_mon) + 1;
-        s->nvram[6] = to_bcd(s->now.tm_year - 100);
+        /* In h/w, capture happens on any START condition, not just a
+         * START_RECV, but there is no need to actually capture on
+         * START_SEND, because the guest can't get at that data
+         * without going through a START_RECV which would overwrite it.
+         */
+        capture_current_time(s);
         break;
     case I2C_START_SEND:
-        s->addr_byte = 1;
+        s->addr_byte = true;
         break;
     default:
         break;
@@ -57,7 +102,7 @@ static int ds1338_recv(I2CSlave *i2c)
     uint8_t res;
 
     res  = s->nvram[s->ptr];
-    s->ptr = (s->ptr + 1) & 0xff;
+    inc_regptr(s);
     return res;
 }
 
@@ -65,20 +110,20 @@ static int ds1338_send(I2CSlave *i2c, uint8_t data)
 {
     DS1338State *s = FROM_I2C_SLAVE(DS1338State, i2c);
     if (s->addr_byte) {
-        s->ptr = data;
-        s->addr_byte = 0;
+        s->ptr = data & (NVRAM_SIZE - 1);
+        s->addr_byte = false;
         return 0;
     }
-    s->nvram[s->ptr - 8] = data;
-    if (data < 8) {
-        qemu_get_timedate(&s->now, s->offset);
-        switch(data) {
+    if (s->ptr < 8) {
+        struct tm now;
+        qemu_get_timedate(&now, s->offset);
+        switch(s->ptr) {
         case 0:
             /* TODO: Implement CH (stop) bit.  */
-            s->now.tm_sec = from_bcd(data & 0x7f);
+            now.tm_sec = from_bcd(data & 0x7f);
             break;
         case 1:
-            s->now.tm_min = from_bcd(data & 0x7f);
+            now.tm_min = from_bcd(data & 0x7f);
             break;
         case 2:
             if (data & 0x40) {
@@ -90,27 +135,29 @@ static int ds1338_send(I2CSlave *i2c, uint8_t data)
             } else {
                 data = from_bcd(data);
             }
-            s->now.tm_hour = data;
+            now.tm_hour = data;
             break;
         case 3:
-            s->now.tm_wday = from_bcd(data & 7) - 1;
+            now.tm_wday = from_bcd(data & 7) - 1;
             break;
         case 4:
-            s->now.tm_mday = from_bcd(data & 0x3f);
+            now.tm_mday = from_bcd(data & 0x3f);
             break;
         case 5:
-            s->now.tm_mon = from_bcd(data & 0x1f) - 1;
+            now.tm_mon = from_bcd(data & 0x1f) - 1;
             break;
         case 6:
-            s->now.tm_year = from_bcd(data) + 100;
+            now.tm_year = from_bcd(data) + 100;
             break;
         case 7:
             /* Control register. Currently ignored.  */
             break;
         }
-        s->offset = qemu_timedate_diff(&s->now);
+        s->offset = qemu_timedate_diff(&now);
+    } else {
+        s->nvram[s->ptr] = data;
     }
-    s->ptr = (s->ptr + 1) & 0xff;
+    inc_regptr(s);
     return 0;
 }
 
@@ -121,12 +168,14 @@ static int ds1338_init(I2CSlave *i2c)
 
 static void ds1338_class_init(ObjectClass *klass, void *data)
 {
+    DeviceClass *dc = DEVICE_CLASS(klass);
     I2CSlaveClass *k = I2C_SLAVE_CLASS(klass);
 
     k->init = ds1338_init;
     k->event = ds1338_event;
     k->recv = ds1338_recv;
     k->send = ds1338_send;
+    dc->vmsd = &vmstate_ds1338;
 }
 
 static TypeInfo ds1338_info = {
diff --git a/hw/dummy_m68k.c b/hw/dummy_m68k.c
index 7cc7a99bf..20f790b1a 100644
--- a/hw/dummy_m68k.c
+++ b/hw/dummy_m68k.c
@@ -16,17 +16,17 @@
 
 /* Board init.  */
 
-static void dummy_m68k_init(ram_addr_t ram_size,
-                     const char *boot_device,
-                     const char *kernel_filename, const char *kernel_cmdline,
-                     const char *initrd_filename, const char *cpu_model)
+static void dummy_m68k_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t ram_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
     CPUM68KState *env;
     MemoryRegion *address_space_mem =  get_system_memory();
     MemoryRegion *ram = g_new(MemoryRegion, 1);
     int kernel_size;
     uint64_t elf_entry;
-    target_phys_addr_t entry;
+    hwaddr entry;
 
     if (!cpu_model)
         cpu_model = "cfv4e";
diff --git a/hw/e1000.c b/hw/e1000.c
index ae8a6c552..5537ad2fc 100644
--- a/hw/e1000.c
+++ b/hw/e1000.c
@@ -59,6 +59,9 @@ static int debugflags = DBGBIT(TXERR) | DBGBIT(GENERAL);
 #define PNPMMIO_SIZE      0x20000
 #define MIN_BUF_SIZE      60 /* Min. octets in an ethernet frame sans FCS */
 
+/* this is the size past which hardware will drop packets when setting LPE=0 */
+#define MAXIMUM_ETHERNET_VLAN_SIZE 1522
+
 /*
  * HW models:
  *  E1000_DEV_ID_82540EM works with Windows and Linux
@@ -92,7 +95,6 @@ typedef struct E1000State_st {
 
     uint32_t rxbuf_size;
     uint32_t rxbuf_min_shift;
-    int check_rxov;
     struct e1000_tx {
         unsigned char header[256];
         unsigned char vlan_header[4];
@@ -266,6 +268,8 @@ rxbufsize(uint32_t v)
 static void e1000_reset(void *opaque)
 {
     E1000State *d = opaque;
+    uint8_t *macaddr = d->conf.macaddr.a;
+    int i;
 
     qemu_del_timer(d->autoneg_timer);
     memset(d->phy_reg, 0, sizeof d->phy_reg);
@@ -278,6 +282,14 @@ static void e1000_reset(void *opaque)
     if (d->nic->nc.link_down) {
         e1000_link_down(d);
     }
+
+    /* Some guests expect pre-initialized RAH/RAL (AddrValid flag + MACaddr) */
+    d->mac_reg[RA] = 0;
+    d->mac_reg[RA + 1] = E1000_RAH_AV;
+    for (i = 0; i < 4; i++) {
+        d->mac_reg[RA] |= macaddr[i] << (8 * i);
+        d->mac_reg[RA + 1] |= (i < 2) ? macaddr[i + 4] << (8 * i) : 0;
+    }
 }
 
 static void
@@ -295,6 +307,7 @@ set_rx_control(E1000State *s, int index, uint32_t val)
     s->rxbuf_min_shift = ((val / E1000_RCTL_RDMTS_QUAT) & 3) + 1;
     DBGOUT(RX, "RCTL: %d, mac_reg[RCTL] = 0x%x\n", s->mac_reg[RDT],
            s->mac_reg[RCTL]);
+    qemu_flush_queued_packets(&s->nic->nc);
 }
 
 static void
@@ -740,11 +753,11 @@ static bool e1000_has_rxbufs(E1000State *s, size_t total_size)
     int bufs;
     /* Fast-path short packets */
     if (total_size <= s->rxbuf_size) {
-        return s->mac_reg[RDH] != s->mac_reg[RDT] || !s->check_rxov;
+        return s->mac_reg[RDH] != s->mac_reg[RDT];
     }
     if (s->mac_reg[RDH] < s->mac_reg[RDT]) {
         bufs = s->mac_reg[RDT] - s->mac_reg[RDH];
-    } else if (s->mac_reg[RDH] > s->mac_reg[RDT] || !s->check_rxov) {
+    } else if (s->mac_reg[RDH] > s->mac_reg[RDT]) {
         bufs = s->mac_reg[RDLEN] /  sizeof(struct e1000_rx_desc) +
             s->mac_reg[RDT] - s->mac_reg[RDH];
     } else {
@@ -795,6 +808,13 @@ e1000_receive(NetClientState *nc, const uint8_t *buf, size_t size)
         size = sizeof(min_buf);
     }
 
+    /* Discard oversized packets if !LPE and !SBP. */
+    if (size > MAXIMUM_ETHERNET_VLAN_SIZE
+        && !(s->mac_reg[RCTL] & E1000_RCTL_LPE)
+        && !(s->mac_reg[RCTL] & E1000_RCTL_SBP)) {
+        return size;
+    }
+
     if (!receive_filter(s, buf, size))
         return size;
 
@@ -847,7 +867,6 @@ e1000_receive(NetClientState *nc, const uint8_t *buf, size_t size)
 
         if (++s->mac_reg[RDH] * sizeof(desc) >= s->mac_reg[RDLEN])
             s->mac_reg[RDH] = 0;
-        s->check_rxov = 1;
         /* see comment in start_xmit; same here */
         if (s->mac_reg[RDH] == rdh_start) {
             DBGOUT(RXERR, "RDH wraparound @%x, RDT %x, RDLEN %x\n",
@@ -924,8 +943,10 @@ mac_writereg(E1000State *s, int index, uint32_t val)
 static void
 set_rdt(E1000State *s, int index, uint32_t val)
 {
-    s->check_rxov = 0;
     s->mac_reg[index] = val & 0xffff;
+    if (e1000_has_rxbufs(s, 1)) {
+        qemu_flush_queued_packets(&s->nic->nc);
+    }
 }
 
 static void
@@ -1007,7 +1028,7 @@ static void (*macreg_writeops[])(E1000State *, int, uint32_t) = {
 enum { NWRITEOPS = ARRAY_SIZE(macreg_writeops) };
 
 static void
-e1000_mmio_write(void *opaque, target_phys_addr_t addr, uint64_t val,
+e1000_mmio_write(void *opaque, hwaddr addr, uint64_t val,
                  unsigned size)
 {
     E1000State *s = opaque;
@@ -1024,7 +1045,7 @@ e1000_mmio_write(void *opaque, target_phys_addr_t addr, uint64_t val,
 }
 
 static uint64_t
-e1000_mmio_read(void *opaque, target_phys_addr_t addr, unsigned size)
+e1000_mmio_read(void *opaque, hwaddr addr, unsigned size)
 {
     E1000State *s = opaque;
     unsigned int index = (addr & 0x1ffff) >> 2;
@@ -1047,7 +1068,7 @@ static const MemoryRegionOps e1000_mmio_ops = {
     },
 };
 
-static uint64_t e1000_io_read(void *opaque, target_phys_addr_t addr,
+static uint64_t e1000_io_read(void *opaque, hwaddr addr,
                               unsigned size)
 {
     E1000State *s = opaque;
@@ -1056,7 +1077,7 @@ static uint64_t e1000_io_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void e1000_io_write(void *opaque, target_phys_addr_t addr,
+static void e1000_io_write(void *opaque, hwaddr addr,
                            uint64_t val, unsigned size)
 {
     E1000State *s = opaque;
@@ -1075,11 +1096,23 @@ static bool is_version_1(void *opaque, int version_id)
     return version_id == 1;
 }
 
+static int e1000_post_load(void *opaque, int version_id)
+{
+    E1000State *s = opaque;
+
+    /* nc.link_down can't be migrated, so infer link_down according
+     * to link status bit in mac_reg[STATUS] */
+    s->nic->nc.link_down = (s->mac_reg[STATUS] & E1000_STATUS_LU) == 0;
+
+    return 0;
+}
+
 static const VMStateDescription vmstate_e1000 = {
     .name = "e1000",
     .version_id = 2,
     .minimum_version_id = 1,
     .minimum_version_id_old = 1,
+    .post_load = e1000_post_load,
     .fields      = (VMStateField []) {
         VMSTATE_PCI_DEVICE(dev, E1000State),
         VMSTATE_UNUSED_TEST(is_version_1, 4), /* was instance id */
diff --git a/hw/eccmemctl.c b/hw/eccmemctl.c
index fe1cd9000..000bd08de 100644
--- a/hw/eccmemctl.c
+++ b/hw/eccmemctl.c
@@ -129,7 +129,7 @@ typedef struct ECCState {
     uint32_t version;
 } ECCState;
 
-static void ecc_mem_write(void *opaque, target_phys_addr_t addr, uint64_t val,
+static void ecc_mem_write(void *opaque, hwaddr addr, uint64_t val,
                           unsigned size)
 {
     ECCState *s = opaque;
@@ -172,7 +172,7 @@ static void ecc_mem_write(void *opaque, target_phys_addr_t addr, uint64_t val,
     }
 }
 
-static uint64_t ecc_mem_read(void *opaque, target_phys_addr_t addr,
+static uint64_t ecc_mem_read(void *opaque, hwaddr addr,
                              unsigned size)
 {
     ECCState *s = opaque;
@@ -229,7 +229,7 @@ static const MemoryRegionOps ecc_mem_ops = {
     },
 };
 
-static void ecc_diag_mem_write(void *opaque, target_phys_addr_t addr,
+static void ecc_diag_mem_write(void *opaque, hwaddr addr,
                                uint64_t val, unsigned size)
 {
     ECCState *s = opaque;
@@ -238,7 +238,7 @@ static void ecc_diag_mem_write(void *opaque, target_phys_addr_t addr,
     s->diag[addr & ECC_DIAG_MASK] = val;
 }
 
-static uint64_t ecc_diag_mem_read(void *opaque, target_phys_addr_t addr,
+static uint64_t ecc_diag_mem_read(void *opaque, hwaddr addr,
                                   unsigned size)
 {
     ECCState *s = opaque;
diff --git a/hw/eepro100.c b/hw/eepro100.c
index 50d117e35..a189474d3 100644
--- a/hw/eepro100.c
+++ b/hw/eepro100.c
@@ -1036,6 +1036,7 @@ static void eepro100_ru_command(EEPRO100State * s, uint8_t val)
         }
         set_ru_state(s, ru_ready);
         s->ru_offset = e100_read_reg4(s, SCBPointer);
+        qemu_flush_queued_packets(&s->nic->nc);
         TRACE(OTHER, logout("val=0x%02x (rx start)\n", val));
         break;
     case RX_RESUME:
@@ -1577,7 +1578,7 @@ static void eepro100_write4(EEPRO100State * s, uint32_t addr, uint32_t val)
     }
 }
 
-static uint64_t eepro100_read(void *opaque, target_phys_addr_t addr,
+static uint64_t eepro100_read(void *opaque, hwaddr addr,
                               unsigned size)
 {
     EEPRO100State *s = opaque;
@@ -1590,7 +1591,7 @@ static uint64_t eepro100_read(void *opaque, target_phys_addr_t addr,
     }
 }
 
-static void eepro100_write(void *opaque, target_phys_addr_t addr,
+static void eepro100_write(void *opaque, hwaddr addr,
                            uint64_t data, unsigned size)
 {
     EEPRO100State *s = opaque;
@@ -1770,7 +1771,8 @@ static ssize_t nic_receive(NetClientState *nc, const uint8_t * buf, size_t size)
     if (rfd_command & COMMAND_EL) {
         /* EL bit is set, so this was the last frame. */
         logout("receive: Running out of frames\n");
-        set_ru_state(s, ru_suspended);
+        set_ru_state(s, ru_no_resources);
+        eepro100_rnr_interrupt(s);
     }
     if (rfd_command & COMMAND_S) {
         /* S bit is set. */
diff --git a/hw/elf_ops.h b/hw/elf_ops.h
index fa65ce2f9..531a42553 100644
--- a/hw/elf_ops.h
+++ b/hw/elf_ops.h
@@ -62,7 +62,7 @@ static struct elf_shdr *glue(find_section, SZ)(struct elf_shdr *shdr_table,
 
 static int glue(symfind, SZ)(const void *s0, const void *s1)
 {
-    target_phys_addr_t addr = *(target_phys_addr_t *)s0;
+    hwaddr addr = *(hwaddr *)s0;
     struct elf_sym *sym = (struct elf_sym *)s1;
     int result = 0;
     if (addr < sym->st_value) {
@@ -74,7 +74,7 @@ static int glue(symfind, SZ)(const void *s0, const void *s1)
 }
 
 static const char *glue(lookup_symbol, SZ)(struct syminfo *s,
-                                           target_phys_addr_t orig_addr)
+                                           hwaddr orig_addr)
 {
     struct elf_sym *syms = glue(s->disas_symtab.elf, SZ);
     struct elf_sym *sym;
@@ -269,6 +269,17 @@ static int glue(load_elf, SZ)(const char *name, int fd,
                 addr = ph->p_paddr;
             }
 
+            /* the entry pointer in the ELF header is a virtual
+             * address, if the text segments paddr and vaddr differ
+             * we need to adjust the entry */
+            if (pentry && !translate_fn &&
+                    ph->p_vaddr != ph->p_paddr &&
+                    ehdr.e_entry >= ph->p_vaddr &&
+                    ehdr.e_entry < ph->p_vaddr + ph->p_filesz &&
+                    ph->p_flags & PF_X) {
+                *pentry = ehdr.e_entry - ph->p_vaddr + ph->p_paddr;
+            }
+
             snprintf(label, sizeof(label), "phdr #%d: %s", i, name);
             rom_add_blob_fixed(label, data, mem_size, addr);
 
diff --git a/hw/empty_slot.c b/hw/empty_slot.c
index 099c85e58..23978eb14 100644
--- a/hw/empty_slot.c
+++ b/hw/empty_slot.c
@@ -28,14 +28,14 @@ typedef struct EmptySlot {
     uint64_t size;
 } EmptySlot;
 
-static uint64_t empty_slot_read(void *opaque, target_phys_addr_t addr,
+static uint64_t empty_slot_read(void *opaque, hwaddr addr,
                                 unsigned size)
 {
     DPRINTF("read from " TARGET_FMT_plx "\n", addr);
     return 0;
 }
 
-static void empty_slot_write(void *opaque, target_phys_addr_t addr,
+static void empty_slot_write(void *opaque, hwaddr addr,
                              uint64_t val, unsigned size)
 {
     DPRINTF("write 0x%x to " TARGET_FMT_plx "\n", (unsigned)val, addr);
@@ -47,7 +47,7 @@ static const MemoryRegionOps empty_slot_ops = {
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-void empty_slot_init(target_phys_addr_t addr, uint64_t slot_size)
+void empty_slot_init(hwaddr addr, uint64_t slot_size)
 {
     if (slot_size > 0) {
         /* Only empty slots larger than 0 byte need handling. */
diff --git a/hw/empty_slot.h b/hw/empty_slot.h
index 78dc91de1..4e9e46048 100644
--- a/hw/empty_slot.h
+++ b/hw/empty_slot.h
@@ -1,2 +1,2 @@
 /* empty_slot.c */
-void empty_slot_init(target_phys_addr_t addr, uint64_t slot_size);
+void empty_slot_init(hwaddr addr, uint64_t slot_size);
diff --git a/hw/es1370.c b/hw/es1370.c
index e34234c35..e0c9729dc 100644
--- a/hw/es1370.c
+++ b/hw/es1370.c
@@ -908,18 +908,44 @@ static void es1370_adc_callback (void *opaque, int avail)
     es1370_run_channel (s, ADC_CHANNEL, avail);
 }
 
-static const MemoryRegionPortio es1370_portio[] = {
-    { 0, 0x40 * 4, 1, .write = es1370_writeb, },
-    { 0, 0x40 * 2, 2, .write = es1370_writew, },
-    { 0, 0x40, 4, .write = es1370_writel, },
-    { 0, 0x40 * 4, 1, .read = es1370_readb, },
-    { 0, 0x40 * 2, 2, .read = es1370_readw, },
-    { 0, 0x40, 4, .read = es1370_readl, },
-    PORTIO_END_OF_LIST ()
-};
+static uint64_t es1370_read(void *opaque, hwaddr addr,
+                            unsigned size)
+{
+    switch (size) {
+    case 1:
+        return es1370_readb(opaque, addr);
+    case 2:
+        return es1370_readw(opaque, addr);
+    case 4:
+        return es1370_readl(opaque, addr);
+    default:
+        return -1;
+    }
+}
+
+static void es1370_write(void *opaque, hwaddr addr, uint64_t val,
+                      unsigned size)
+{
+    switch (size) {
+    case 1:
+        es1370_writeb(opaque, addr, val);
+        break;
+    case 2:
+        es1370_writew(opaque, addr, val);
+        break;
+    case 4:
+        es1370_writel(opaque, addr, val);
+        break;
+    }
+}
 
 static const MemoryRegionOps es1370_io_ops = {
-    .old_portio = es1370_portio,
+    .read = es1370_read,
+    .write = es1370_write,
+    .impl = {
+        .min_access_size = 1,
+        .max_access_size = 4,
+    },
     .endianness = DEVICE_LITTLE_ENDIAN,
 };
 
diff --git a/hw/escc.c b/hw/escc.c
index e1f5e73ba..a356613b0 100644
--- a/hw/escc.c
+++ b/hw/escc.c
@@ -463,7 +463,7 @@ static void escc_update_parameters(ChannelState *s)
     qemu_chr_fe_ioctl(s->chr, CHR_IOCTL_SERIAL_SET_PARAMS, &ssp);
 }
 
-static void escc_mem_write(void *opaque, target_phys_addr_t addr,
+static void escc_mem_write(void *opaque, hwaddr addr,
                            uint64_t val, unsigned size)
 {
     SerialState *serial = opaque;
@@ -565,7 +565,7 @@ static void escc_mem_write(void *opaque, target_phys_addr_t addr,
     }
 }
 
-static uint64_t escc_mem_read(void *opaque, target_phys_addr_t addr,
+static uint64_t escc_mem_read(void *opaque, hwaddr addr,
                               unsigned size)
 {
     SerialState *serial = opaque;
@@ -683,7 +683,7 @@ static const VMStateDescription vmstate_escc = {
     }
 };
 
-MemoryRegion *escc_init(target_phys_addr_t base, qemu_irq irqA, qemu_irq irqB,
+MemoryRegion *escc_init(hwaddr base, qemu_irq irqA, qemu_irq irqB,
               CharDriverState *chrA, CharDriverState *chrB,
               int clock, int it_shift)
 {
@@ -846,7 +846,7 @@ static void sunmouse_event(void *opaque,
     put_queue(s, 0);
 }
 
-void slavio_serial_ms_kbd_init(target_phys_addr_t base, qemu_irq irq,
+void slavio_serial_ms_kbd_init(hwaddr base, qemu_irq irq,
                                int disabled, int clock, int it_shift)
 {
     DeviceState *dev;
diff --git a/hw/escc.h b/hw/escc.h
index d1da46fd0..def28947a 100644
--- a/hw/escc.h
+++ b/hw/escc.h
@@ -1,8 +1,8 @@
 /* escc.c */
 #define ESCC_SIZE 4
-MemoryRegion *escc_init(target_phys_addr_t base, qemu_irq irqA, qemu_irq irqB,
+MemoryRegion *escc_init(hwaddr base, qemu_irq irqA, qemu_irq irqB,
               CharDriverState *chrA, CharDriverState *chrB,
               int clock, int it_shift);
 
-void slavio_serial_ms_kbd_init(target_phys_addr_t base, qemu_irq irq,
+void slavio_serial_ms_kbd_init(hwaddr base, qemu_irq irq,
                                int disabled, int clock, int it_shift);
diff --git a/hw/esp-pci.c b/hw/esp-pci.c
index 170e007be..d9a8e59c9 100644
--- a/hw/esp-pci.c
+++ b/hw/esp-pci.c
@@ -159,7 +159,7 @@ static uint32_t esp_pci_dma_read(PCIESPState *pci, uint32_t saddr)
     return val;
 }
 
-static void esp_pci_io_write(void *opaque, target_phys_addr_t addr,
+static void esp_pci_io_write(void *opaque, hwaddr addr,
                              uint64_t val, unsigned int size)
 {
     PCIESPState *pci = opaque;
@@ -202,7 +202,7 @@ static void esp_pci_io_write(void *opaque, target_phys_addr_t addr,
     }
 }
 
-static uint64_t esp_pci_io_read(void *opaque, target_phys_addr_t addr,
+static uint64_t esp_pci_io_read(void *opaque, hwaddr addr,
                                 unsigned int size)
 {
     PCIESPState *pci = opaque;
diff --git a/hw/esp.c b/hw/esp.c
index 84a4e74e2..6d01624a4 100644
--- a/hw/esp.c
+++ b/hw/esp.c
@@ -550,7 +550,7 @@ void esp_reg_write(ESPState *s, uint32_t saddr, uint64_t val)
     s->wregs[saddr] = val;
 }
 
-static bool esp_mem_accepts(void *opaque, target_phys_addr_t addr,
+static bool esp_mem_accepts(void *opaque, hwaddr addr,
                             unsigned size, bool is_write)
 {
     return (size == 1) || (is_write && size == 4);
@@ -585,7 +585,7 @@ typedef struct {
     ESPState esp;
 } SysBusESPState;
 
-static void sysbus_esp_mem_write(void *opaque, target_phys_addr_t addr,
+static void sysbus_esp_mem_write(void *opaque, hwaddr addr,
                                  uint64_t val, unsigned int size)
 {
     SysBusESPState *sysbus = opaque;
@@ -595,7 +595,7 @@ static void sysbus_esp_mem_write(void *opaque, target_phys_addr_t addr,
     esp_reg_write(&sysbus->esp, saddr, val);
 }
 
-static uint64_t sysbus_esp_mem_read(void *opaque, target_phys_addr_t addr,
+static uint64_t sysbus_esp_mem_read(void *opaque, hwaddr addr,
                                     unsigned int size)
 {
     SysBusESPState *sysbus = opaque;
@@ -612,7 +612,7 @@ static const MemoryRegionOps sysbus_esp_mem_ops = {
     .valid.accepts = esp_mem_accepts,
 };
 
-void esp_init(target_phys_addr_t espaddr, int it_shift,
+void esp_init(hwaddr espaddr, int it_shift,
               ESPDMAMemoryReadWriteFunc dma_memory_read,
               ESPDMAMemoryReadWriteFunc dma_memory_write,
               void *dma_opaque, qemu_irq irq, qemu_irq *reset,
diff --git a/hw/esp.h b/hw/esp.h
index fa855e2fd..f15cc7b5b 100644
--- a/hw/esp.h
+++ b/hw/esp.h
@@ -6,7 +6,7 @@
 /* esp.c */
 #define ESP_MAX_DEVS 7
 typedef void (*ESPDMAMemoryReadWriteFunc)(void *opaque, uint8_t *buf, int len);
-void esp_init(target_phys_addr_t espaddr, int it_shift,
+void esp_init(hwaddr espaddr, int it_shift,
               ESPDMAMemoryReadWriteFunc dma_memory_read,
               ESPDMAMemoryReadWriteFunc dma_memory_write,
               void *dma_opaque, qemu_irq irq, qemu_irq *reset,
diff --git a/hw/etraxfs.h b/hw/etraxfs.h
index c62f94b7b..725bb9e14 100644
--- a/hw/etraxfs.h
+++ b/hw/etraxfs.h
@@ -29,7 +29,7 @@ qemu_irq *cris_pic_init_cpu(CPUCRISState *env);
 
 /* Instantiate an ETRAXFS Ethernet MAC.  */
 static inline DeviceState *
-etraxfs_eth_init(NICInfo *nd, target_phys_addr_t base, int phyaddr,
+etraxfs_eth_init(NICInfo *nd, hwaddr base, int phyaddr,
                  void *dma_out, void *dma_in)
 {
     DeviceState *dev;
diff --git a/hw/etraxfs_dma.c b/hw/etraxfs_dma.c
index 332525cab..49221abc1 100644
--- a/hw/etraxfs_dma.c
+++ b/hw/etraxfs_dma.c
@@ -212,7 +212,7 @@ static inline int channel_en(struct fs_dma_ctrl *ctrl, int c)
 		&& ctrl->channels[c].client;
 }
 
-static inline int fs_channel(target_phys_addr_t addr)
+static inline int fs_channel(hwaddr addr)
 {
 	/* Every channel has a 0x2000 ctrl register map.  */
 	return addr >> 13;
@@ -221,7 +221,7 @@ static inline int fs_channel(target_phys_addr_t addr)
 #ifdef USE_THIS_DEAD_CODE
 static void channel_load_g(struct fs_dma_ctrl *ctrl, int c)
 {
-	target_phys_addr_t addr = channel_reg(ctrl, c, RW_GROUP);
+	hwaddr addr = channel_reg(ctrl, c, RW_GROUP);
 
 	/* Load and decode. FIXME: handle endianness.  */
 	cpu_physical_memory_read (addr, 
@@ -253,7 +253,7 @@ static void dump_d(int ch, struct dma_descr_data *d)
 
 static void channel_load_c(struct fs_dma_ctrl *ctrl, int c)
 {
-	target_phys_addr_t addr = channel_reg(ctrl, c, RW_GROUP_DOWN);
+	hwaddr addr = channel_reg(ctrl, c, RW_GROUP_DOWN);
 
 	/* Load and decode. FIXME: handle endianness.  */
 	cpu_physical_memory_read (addr, 
@@ -270,7 +270,7 @@ static void channel_load_c(struct fs_dma_ctrl *ctrl, int c)
 
 static void channel_load_d(struct fs_dma_ctrl *ctrl, int c)
 {
-	target_phys_addr_t addr = channel_reg(ctrl, c, RW_SAVED_DATA);
+	hwaddr addr = channel_reg(ctrl, c, RW_SAVED_DATA);
 
 	/* Load and decode. FIXME: handle endianness.  */
 	D(printf("%s ch=%d addr=" TARGET_FMT_plx "\n", __func__, c, addr));
@@ -284,7 +284,7 @@ static void channel_load_d(struct fs_dma_ctrl *ctrl, int c)
 
 static void channel_store_c(struct fs_dma_ctrl *ctrl, int c)
 {
-	target_phys_addr_t addr = channel_reg(ctrl, c, RW_GROUP_DOWN);
+	hwaddr addr = channel_reg(ctrl, c, RW_GROUP_DOWN);
 
 	/* Encode and store. FIXME: handle endianness.  */
 	D(printf("%s ch=%d addr=" TARGET_FMT_plx "\n", __func__, c, addr));
@@ -296,7 +296,7 @@ static void channel_store_c(struct fs_dma_ctrl *ctrl, int c)
 
 static void channel_store_d(struct fs_dma_ctrl *ctrl, int c)
 {
-	target_phys_addr_t addr = channel_reg(ctrl, c, RW_SAVED_DATA);
+	hwaddr addr = channel_reg(ctrl, c, RW_SAVED_DATA);
 
 	/* Encode and store. FIXME: handle endianness.  */
 	D(printf("%s ch=%d addr=" TARGET_FMT_plx "\n", __func__, c, addr));
@@ -573,14 +573,14 @@ static inline int channel_in_run(struct fs_dma_ctrl *ctrl, int c)
 		return 0;
 }
 
-static uint32_t dma_rinvalid (void *opaque, target_phys_addr_t addr)
+static uint32_t dma_rinvalid (void *opaque, hwaddr addr)
 {
         hw_error("Unsupported short raccess. reg=" TARGET_FMT_plx "\n", addr);
         return 0;
 }
 
 static uint64_t
-dma_read(void *opaque, target_phys_addr_t addr, unsigned int size)
+dma_read(void *opaque, hwaddr addr, unsigned int size)
 {
         struct fs_dma_ctrl *ctrl = opaque;
 	int c;
@@ -612,7 +612,7 @@ dma_read(void *opaque, target_phys_addr_t addr, unsigned int size)
 }
 
 static void
-dma_winvalid (void *opaque, target_phys_addr_t addr, uint32_t value)
+dma_winvalid (void *opaque, hwaddr addr, uint32_t value)
 {
         hw_error("Unsupported short waccess. reg=" TARGET_FMT_plx "\n", addr);
 }
@@ -627,7 +627,7 @@ dma_update_state(struct fs_dma_ctrl *ctrl, int c)
 }
 
 static void
-dma_write(void *opaque, target_phys_addr_t addr,
+dma_write(void *opaque, hwaddr addr,
 	  uint64_t val64, unsigned int size)
 {
         struct fs_dma_ctrl *ctrl = opaque;
@@ -762,7 +762,7 @@ static void DMA_run(void *opaque)
         qemu_bh_schedule_idle(etraxfs_dmac->bh);
 }
 
-void *etraxfs_dmac_init(target_phys_addr_t base, int nr_channels)
+void *etraxfs_dmac_init(hwaddr base, int nr_channels)
 {
 	struct fs_dma_ctrl *ctrl = NULL;
 
diff --git a/hw/etraxfs_dma.h b/hw/etraxfs_dma.h
index 021c52ae7..3fef80fae 100644
--- a/hw/etraxfs_dma.h
+++ b/hw/etraxfs_dma.h
@@ -20,7 +20,7 @@ struct etraxfs_dma_client
 	} client;
 };
 
-void *etraxfs_dmac_init(target_phys_addr_t base, int nr_channels);
+void *etraxfs_dmac_init(hwaddr base, int nr_channels);
 void etraxfs_dmac_connect(void *opaque, int channel, qemu_irq *line,
 			  int input);
 void etraxfs_dmac_connect_client(void *opaque, int c, 
diff --git a/hw/etraxfs_eth.c b/hw/etraxfs_eth.c
index b124f5bb3..3d4242682 100644
--- a/hw/etraxfs_eth.c
+++ b/hw/etraxfs_eth.c
@@ -374,7 +374,7 @@ static void eth_validate_duplex(struct fs_eth *eth)
 }
 
 static uint64_t
-eth_read(void *opaque, target_phys_addr_t addr, unsigned int size)
+eth_read(void *opaque, hwaddr addr, unsigned int size)
 {
 	struct fs_eth *eth = opaque;
 	uint32_t r = 0;
@@ -418,7 +418,7 @@ static void eth_update_ma(struct fs_eth *eth, int ma)
 }
 
 static void
-eth_write(void *opaque, target_phys_addr_t addr,
+eth_write(void *opaque, hwaddr addr,
           uint64_t val64, unsigned int size)
 {
 	struct fs_eth *eth = opaque;
diff --git a/hw/etraxfs_pic.c b/hw/etraxfs_pic.c
index dc27f88ac..62a62a36a 100644
--- a/hw/etraxfs_pic.c
+++ b/hw/etraxfs_pic.c
@@ -79,7 +79,7 @@ static void pic_update(struct etrax_pic *fs)
 }
 
 static uint64_t
-pic_read(void *opaque, target_phys_addr_t addr, unsigned int size)
+pic_read(void *opaque, hwaddr addr, unsigned int size)
 {
     struct etrax_pic *fs = opaque;
     uint32_t rval;
@@ -89,7 +89,7 @@ pic_read(void *opaque, target_phys_addr_t addr, unsigned int size)
     return rval;
 }
 
-static void pic_write(void *opaque, target_phys_addr_t addr,
+static void pic_write(void *opaque, hwaddr addr,
                       uint64_t value, unsigned int size)
 {
     struct etrax_pic *fs = opaque;
diff --git a/hw/etraxfs_ser.c b/hw/etraxfs_ser.c
index 5f16b1783..ee0d72bf8 100644
--- a/hw/etraxfs_ser.c
+++ b/hw/etraxfs_ser.c
@@ -75,7 +75,7 @@ static void ser_update_irq(struct etrax_serial *s)
 }
 
 static uint64_t
-ser_read(void *opaque, target_phys_addr_t addr, unsigned int size)
+ser_read(void *opaque, hwaddr addr, unsigned int size)
 {
     struct etrax_serial *s = opaque;
     D(CPUCRISState *env = s->env);
@@ -110,7 +110,7 @@ ser_read(void *opaque, target_phys_addr_t addr, unsigned int size)
 }
 
 static void
-ser_write(void *opaque, target_phys_addr_t addr,
+ser_write(void *opaque, hwaddr addr,
           uint64_t val64, unsigned int size)
 {
     struct etrax_serial *s = opaque;
diff --git a/hw/etraxfs_timer.c b/hw/etraxfs_timer.c
index 9076a4988..f5601dc7a 100644
--- a/hw/etraxfs_timer.c
+++ b/hw/etraxfs_timer.c
@@ -75,7 +75,7 @@ struct etrax_timer {
 };
 
 static uint64_t
-timer_read(void *opaque, target_phys_addr_t addr, unsigned int size)
+timer_read(void *opaque, hwaddr addr, unsigned int size)
 {
     struct etrax_timer *t = opaque;
     uint32_t r = 0;
@@ -242,7 +242,7 @@ static inline void timer_watchdog_update(struct etrax_timer *t, uint32_t value)
 }
 
 static void
-timer_write(void *opaque, target_phys_addr_t addr,
+timer_write(void *opaque, hwaddr addr,
             uint64_t val64, unsigned int size)
 {
     struct etrax_timer *t = opaque;
diff --git a/hw/exynos4210.h b/hw/exynos4210.h
index a43ba3aed..777f0f5b2 100644
--- a/hw/exynos4210.h
+++ b/hw/exynos4210.h
@@ -128,7 +128,7 @@ void exynos4210_combiner_get_gpioin(Exynos4210Irq *irqs, DeviceState *dev,
 /*
  * exynos4210 UART
  */
-DeviceState *exynos4210_uart_create(target_phys_addr_t addr,
+DeviceState *exynos4210_uart_create(hwaddr addr,
                                     int fifo_size,
                                     int channel,
                                     CharDriverState *chr,
diff --git a/hw/exynos4210_combiner.c b/hw/exynos4210_combiner.c
index 80af22cc3..84d36ed11 100644
--- a/hw/exynos4210_combiner.c
+++ b/hw/exynos4210_combiner.c
@@ -174,7 +174,7 @@ void exynos4210_combiner_get_gpioin(Exynos4210Irq *irqs, DeviceState *dev,
 }
 
 static uint64_t
-exynos4210_combiner_read(void *opaque, target_phys_addr_t offset, unsigned size)
+exynos4210_combiner_read(void *opaque, hwaddr offset, unsigned size)
 {
     struct Exynos4210CombinerState *s =
             (struct Exynos4210CombinerState *)opaque;
@@ -266,7 +266,7 @@ static void exynos4210_combiner_update(void *opaque, uint8_t group_n)
     }
 }
 
-static void exynos4210_combiner_write(void *opaque, target_phys_addr_t offset,
+static void exynos4210_combiner_write(void *opaque, hwaddr offset,
         uint64_t val, unsigned size)
 {
     struct Exynos4210CombinerState *s =
@@ -347,8 +347,6 @@ static void exynos4210_combiner_write(void *opaque, target_phys_addr_t offset,
                 TARGET_FMT_plx "\n", offset);
         break;
     }
-
-    return;
 }
 
 /* Get combiner group and bit from irq number */
@@ -380,8 +378,6 @@ static void exynos4210_combiner_handler(void *opaque, int irq, int level)
     }
 
     exynos4210_combiner_update(s, group_n);
-
-    return;
 }
 
 static void exynos4210_combiner_reset(DeviceState *d)
diff --git a/hw/exynos4210_fimd.c b/hw/exynos4210_fimd.c
index 3313f00a7..f2443ca4a 100644
--- a/hw/exynos4210_fimd.c
+++ b/hw/exynos4210_fimd.c
@@ -290,7 +290,7 @@ struct Exynos4210fimdWindow {
     uint16_t virtpage_offsize;       /* VIDWADD2 register */
     MemoryRegionSection mem_section; /* RAM fragment containing framebuffer */
     uint8_t *host_fb_addr;           /* Host pointer to window's framebuffer */
-    target_phys_addr_t fb_len;       /* Framebuffer length */
+    hwaddr fb_len;       /* Framebuffer length */
 };
 
 typedef struct {
@@ -1110,7 +1110,7 @@ static inline int fimd_get_buffer_id(Exynos4210fimdWindow *w)
 static void fimd_update_memory_section(Exynos4210fimdState *s, unsigned win)
 {
     Exynos4210fimdWindow *w = &s->window[win];
-    target_phys_addr_t fb_start_addr, fb_mapped_len;
+    hwaddr fb_start_addr, fb_mapped_len;
 
     if (!s->enabled || !(w->wincon & FIMD_WINCON_ENWIN) ||
             FIMD_WINDOW_PROTECTED(s->shadowcon, win)) {
@@ -1243,7 +1243,7 @@ static void exynos4210_fimd_update(void *opaque)
     Exynos4210fimdState *s = (Exynos4210fimdState *)opaque;
     Exynos4210fimdWindow *w;
     int i, line;
-    target_phys_addr_t fb_line_addr, inc_size;
+    hwaddr fb_line_addr, inc_size;
     int scrn_height;
     int first_line = -1, last_line = -1, scrn_width;
     bool blend = false;
@@ -1307,7 +1307,7 @@ static void exynos4210_fimd_update(void *opaque)
             fimd_copy_line_toqemu(global_width, s->ifb + global_width * line *
                     RGBA_SIZE, d + global_width * line * bpp);
         }
-        dpy_update(s->console, 0, 0, global_width, global_height);
+        dpy_gfx_update(s->console, 0, 0, global_width, global_height);
     }
     s->invalidate = false;
     s->vidintcon[1] |= FIMD_VIDINT_INTFRMPEND;
@@ -1348,7 +1348,7 @@ static void exynos4210_fimd_reset(DeviceState *d)
     s->hueoffset = 0x01800080;
 }
 
-static void exynos4210_fimd_write(void *opaque, target_phys_addr_t offset,
+static void exynos4210_fimd_write(void *opaque, hwaddr offset,
                               uint64_t val, unsigned size)
 {
     Exynos4210fimdState *s = (Exynos4210fimdState *)opaque;
@@ -1649,7 +1649,7 @@ static void exynos4210_fimd_write(void *opaque, target_phys_addr_t offset,
     }
 }
 
-static uint64_t exynos4210_fimd_read(void *opaque, target_phys_addr_t offset,
+static uint64_t exynos4210_fimd_read(void *opaque, hwaddr offset,
                                   unsigned size)
 {
     Exynos4210fimdState *s = (Exynos4210fimdState *)opaque;
diff --git a/hw/exynos4210_gic.c b/hw/exynos4210_gic.c
index 7d03dd9ae..4fea09873 100644
--- a/hw/exynos4210_gic.c
+++ b/hw/exynos4210_gic.c
@@ -193,8 +193,6 @@ static void exynos4210_irq_handler(void *opaque, int irq, int level)
 
     /* Bypass */
     qemu_set_irq(s->board_irqs[irq], level);
-
-    return;
 }
 
 /*
@@ -410,8 +408,6 @@ static void exynos4210_irq_gate_handler(void *opaque, int irq, int level)
     }
 
     qemu_irq_lower(s->out);
-
-    return;
 }
 
 static void exynos4210_irq_gate_reset(DeviceState *d)
diff --git a/hw/exynos4210_i2c.c b/hw/exynos4210_i2c.c
index 3f72a5c46..1e11d9b48 100644
--- a/hw/exynos4210_i2c.c
+++ b/hw/exynos4210_i2c.c
@@ -129,7 +129,7 @@ static void exynos4210_i2c_data_send(void *opaque)
     exynos4210_i2c_raise_interrupt(s);
 }
 
-static uint64_t exynos4210_i2c_read(void *opaque, target_phys_addr_t offset,
+static uint64_t exynos4210_i2c_read(void *opaque, hwaddr offset,
                                  unsigned size)
 {
     Exynos4210I2CState *s = (Exynos4210I2CState *)opaque;
@@ -168,7 +168,7 @@ static uint64_t exynos4210_i2c_read(void *opaque, target_phys_addr_t offset,
     return value;
 }
 
-static void exynos4210_i2c_write(void *opaque, target_phys_addr_t offset,
+static void exynos4210_i2c_write(void *opaque, hwaddr offset,
                               uint64_t value, unsigned size)
 {
     Exynos4210I2CState *s = (Exynos4210I2CState *)opaque;
diff --git a/hw/exynos4210_mct.c b/hw/exynos4210_mct.c
index 7a22b1f90..e79cd6ac0 100644
--- a/hw/exynos4210_mct.c
+++ b/hw/exynos4210_mct.c
@@ -574,8 +574,6 @@ static void exynos4210_gfrc_event(void *opaque)
     exynos4210_gfrc_set_count(&s->g_timer, distance);
 
     exynos4210_gfrc_start(&s->g_timer);
-
-    return;
 }
 
 /*
@@ -987,7 +985,7 @@ static void exynos4210_mct_reset(DeviceState *d)
 }
 
 /* Multi Core Timer read */
-static uint64_t exynos4210_mct_read(void *opaque, target_phys_addr_t offset,
+static uint64_t exynos4210_mct_read(void *opaque, hwaddr offset,
         unsigned size)
 {
     Exynos4210MCTState *s = (Exynos4210MCTState *)opaque;
@@ -1100,7 +1098,7 @@ static uint64_t exynos4210_mct_read(void *opaque, target_phys_addr_t offset,
 }
 
 /* MCT write */
-static void exynos4210_mct_write(void *opaque, target_phys_addr_t offset,
+static void exynos4210_mct_write(void *opaque, hwaddr offset,
         uint64_t value, unsigned size)
 {
     Exynos4210MCTState *s = (Exynos4210MCTState *)opaque;
diff --git a/hw/exynos4210_pmu.c b/hw/exynos4210_pmu.c
index c12d7501c..a22b8f181 100644
--- a/hw/exynos4210_pmu.c
+++ b/hw/exynos4210_pmu.c
@@ -392,7 +392,7 @@ typedef struct Exynos4210PmuState {
     uint32_t reg[PMU_NUM_OF_REGISTERS];
 } Exynos4210PmuState;
 
-static uint64_t exynos4210_pmu_read(void *opaque, target_phys_addr_t offset,
+static uint64_t exynos4210_pmu_read(void *opaque, hwaddr offset,
                                     unsigned size)
 {
     Exynos4210PmuState *s = (Exynos4210PmuState *)opaque;
@@ -411,7 +411,7 @@ static uint64_t exynos4210_pmu_read(void *opaque, target_phys_addr_t offset,
     return 0;
 }
 
-static void exynos4210_pmu_write(void *opaque, target_phys_addr_t offset,
+static void exynos4210_pmu_write(void *opaque, hwaddr offset,
                                  uint64_t val, unsigned size)
 {
     Exynos4210PmuState *s = (Exynos4210PmuState *)opaque;
diff --git a/hw/exynos4210_pwm.c b/hw/exynos4210_pwm.c
index 0c228280a..5e2872f8f 100644
--- a/hw/exynos4210_pwm.c
+++ b/hw/exynos4210_pwm.c
@@ -208,7 +208,7 @@ static void exynos4210_pwm_tick(void *opaque)
 /*
  * PWM Read
  */
-static uint64_t exynos4210_pwm_read(void *opaque, target_phys_addr_t offset,
+static uint64_t exynos4210_pwm_read(void *opaque, hwaddr offset,
         unsigned size)
 {
     Exynos4210PWMState *s = (Exynos4210PWMState *)opaque;
@@ -259,7 +259,7 @@ static uint64_t exynos4210_pwm_read(void *opaque, target_phys_addr_t offset,
 /*
  * PWM Write
  */
-static void exynos4210_pwm_write(void *opaque, target_phys_addr_t offset,
+static void exynos4210_pwm_write(void *opaque, hwaddr offset,
         uint64_t value, unsigned size)
 {
     Exynos4210PWMState *s = (Exynos4210PWMState *)opaque;
diff --git a/hw/exynos4210_rtc.c b/hw/exynos4210_rtc.c
index 42a4ddc32..c4fbd4946 100644
--- a/hw/exynos4210_rtc.c
+++ b/hw/exynos4210_rtc.c
@@ -299,7 +299,7 @@ static void exynos4210_rtc_1Hz_tick(void *opaque)
 /*
  * RTC Read
  */
-static uint64_t exynos4210_rtc_read(void *opaque, target_phys_addr_t offset,
+static uint64_t exynos4210_rtc_read(void *opaque, hwaddr offset,
         unsigned size)
 {
     uint32_t value = 0;
@@ -376,7 +376,7 @@ static uint64_t exynos4210_rtc_read(void *opaque, target_phys_addr_t offset,
 /*
  * RTC Write
  */
-static void exynos4210_rtc_write(void *opaque, target_phys_addr_t offset,
+static void exynos4210_rtc_write(void *opaque, hwaddr offset,
         uint64_t value, unsigned size)
 {
     Exynos4210RTCState *s = (Exynos4210RTCState *)opaque;
diff --git a/hw/exynos4210_uart.c b/hw/exynos4210_uart.c
index ccc47804f..20dcd9fb5 100644
--- a/hw/exynos4210_uart.c
+++ b/hw/exynos4210_uart.c
@@ -96,7 +96,7 @@
 
 typedef struct Exynos4210UartReg {
     const char         *name; /* the only reason is the debug output */
-    target_phys_addr_t  offset;
+    hwaddr  offset;
     uint32_t            reset_value;
 } Exynos4210UartReg;
 
@@ -184,7 +184,7 @@ typedef struct {
 
 #if DEBUG_UART
 /* Used only for debugging inside PRINT_DEBUG_... macros */
-static const char *exynos4210_uart_regname(target_phys_addr_t  offset)
+static const char *exynos4210_uart_regname(hwaddr  offset)
 {
 
     int regs_number = sizeof(exynos4210_uart_regs) / sizeof(Exynos4210UartReg);
@@ -348,7 +348,7 @@ static void exynos4210_uart_update_parameters(Exynos4210UartState *s)
                 s->channel, speed, parity, data_bits, stop_bits);
 }
 
-static void exynos4210_uart_write(void *opaque, target_phys_addr_t offset,
+static void exynos4210_uart_write(void *opaque, hwaddr offset,
                                uint64_t val, unsigned size)
 {
     Exynos4210UartState *s = (Exynos4210UartState *)opaque;
@@ -423,7 +423,7 @@ static void exynos4210_uart_write(void *opaque, target_phys_addr_t offset,
         break;
     }
 }
-static uint64_t exynos4210_uart_read(void *opaque, target_phys_addr_t offset,
+static uint64_t exynos4210_uart_read(void *opaque, hwaddr offset,
                                   unsigned size)
 {
     Exynos4210UartState *s = (Exynos4210UartState *)opaque;
@@ -581,7 +581,7 @@ static const VMStateDescription vmstate_exynos4210_uart = {
     }
 };
 
-DeviceState *exynos4210_uart_create(target_phys_addr_t addr,
+DeviceState *exynos4210_uart_create(hwaddr addr,
                                  int fifo_size,
                                  int channel,
                                  CharDriverState *chr,
@@ -617,7 +617,7 @@ DeviceState *exynos4210_uart_create(target_phys_addr_t addr,
 
     bus = sysbus_from_qdev(dev);
     qdev_init_nofail(dev);
-    if (addr != (target_phys_addr_t)-1) {
+    if (addr != (hwaddr)-1) {
         sysbus_mmio_map(bus, 0, addr);
     }
     sysbus_connect_irq(bus, 0, irq);
diff --git a/hw/exynos4_boards.c b/hw/exynos4_boards.c
index 4bb0a60cb..bc815bbae 100644
--- a/hw/exynos4_boards.c
+++ b/hw/exynos4_boards.c
@@ -93,11 +93,8 @@ static void lan9215_init(uint32_t base, qemu_irq irq)
     }
 }
 
-static Exynos4210State *exynos4_boards_init_common(
-        const char *kernel_filename,
-        const char *kernel_cmdline,
-        const char *initrd_filename,
-        Exynos4BoardType board_type)
+static Exynos4210State *exynos4_boards_init_common(QEMUMachineInitArgs *args,
+                                                   Exynos4BoardType board_type)
 {
     if (smp_cpus != EXYNOS4210_NCPUS) {
         fprintf(stderr, "%s board supports only %d CPU cores. Ignoring smp_cpus"
@@ -110,9 +107,9 @@ static Exynos4210State *exynos4_boards_init_common(
     exynos4_board_binfo.board_id = exynos4_board_id[board_type];
     exynos4_board_binfo.smp_bootreg_addr =
             exynos4_board_smp_bootreg_addr[board_type];
-    exynos4_board_binfo.kernel_filename = kernel_filename;
-    exynos4_board_binfo.initrd_filename = initrd_filename;
-    exynos4_board_binfo.kernel_cmdline = kernel_cmdline;
+    exynos4_board_binfo.kernel_filename = args->kernel_filename;
+    exynos4_board_binfo.initrd_filename = args->initrd_filename;
+    exynos4_board_binfo.kernel_cmdline = args->kernel_cmdline;
     exynos4_board_binfo.gic_cpu_if_addr =
             EXYNOS4210_SMP_PRIVATE_BASE_ADDR + 0x100;
 
@@ -122,32 +119,25 @@ static Exynos4210State *exynos4_boards_init_common(
             " initrd_filename: %s\n",
             exynos4_board_ram_size[board_type] / 1048576,
             exynos4_board_ram_size[board_type],
-            kernel_filename,
-            kernel_cmdline,
-            initrd_filename);
+            args->kernel_filename,
+            args->kernel_cmdline,
+            args->initrd_filename);
 
     return exynos4210_init(get_system_memory(),
             exynos4_board_ram_size[board_type]);
 }
 
-static void nuri_init(ram_addr_t ram_size,
-        const char *boot_device,
-        const char *kernel_filename, const char *kernel_cmdline,
-        const char *initrd_filename, const char *cpu_model)
+static void nuri_init(QEMUMachineInitArgs *args)
 {
-    exynos4_boards_init_common(kernel_filename, kernel_cmdline,
-                initrd_filename, EXYNOS4_BOARD_NURI);
+    exynos4_boards_init_common(args, EXYNOS4_BOARD_NURI);
 
     arm_load_kernel(arm_env_get_cpu(first_cpu), &exynos4_board_binfo);
 }
 
-static void smdkc210_init(ram_addr_t ram_size,
-        const char *boot_device,
-        const char *kernel_filename, const char *kernel_cmdline,
-        const char *initrd_filename, const char *cpu_model)
+static void smdkc210_init(QEMUMachineInitArgs *args)
 {
-    Exynos4210State *s = exynos4_boards_init_common(kernel_filename,
-            kernel_cmdline, initrd_filename, EXYNOS4_BOARD_SMDKC210);
+    Exynos4210State *s = exynos4_boards_init_common(args,
+                                                    EXYNOS4_BOARD_SMDKC210);
 
     lan9215_init(SMDK_LAN9118_BASE_ADDR,
             qemu_irq_invert(s->irq_table[exynos4210_get_irq(37, 1)]));
diff --git a/hw/fdc.c b/hw/fdc.c
index 08830c1ba..29b5449ff 100644
--- a/hw/fdc.c
+++ b/hw/fdc.c
@@ -327,7 +327,7 @@ static void fdctrl_reset(FDCtrl *fdctrl, int do_irq);
 static void fdctrl_reset_fifo(FDCtrl *fdctrl);
 static int fdctrl_transfer_handler (void *opaque, int nchan,
                                     int dma_pos, int dma_len);
-static void fdctrl_raise_irq(FDCtrl *fdctrl, uint8_t status0);
+static void fdctrl_raise_irq(FDCtrl *fdctrl);
 static FDrive *get_cur_drv(FDCtrl *fdctrl);
 
 static uint32_t fdctrl_read_statusA(FDCtrl *fdctrl);
@@ -349,12 +349,12 @@ enum {
     FD_DIR_SCANE   = 2,
     FD_DIR_SCANL   = 3,
     FD_DIR_SCANH   = 4,
+    FD_DIR_VERIFY  = 5,
 };
 
 enum {
     FD_STATE_MULTI  = 0x01,	/* multi track flag */
     FD_STATE_FORMAT = 0x02,	/* format flag */
-    FD_STATE_SEEK   = 0x04,	/* seek flag */
 };
 
 enum {
@@ -496,7 +496,6 @@ enum {
 };
 
 #define FD_MULTI_TRACK(state) ((state) & FD_STATE_MULTI)
-#define FD_DID_SEEK(state) ((state) & FD_STATE_SEEK)
 #define FD_FORMAT_CMD(state) ((state) & FD_STATE_FORMAT)
 
 struct FDCtrl {
@@ -626,13 +625,13 @@ static void fdctrl_write (void *opaque, uint32_t reg, uint32_t value)
     }
 }
 
-static uint64_t fdctrl_read_mem (void *opaque, target_phys_addr_t reg,
+static uint64_t fdctrl_read_mem (void *opaque, hwaddr reg,
                                  unsigned ize)
 {
     return fdctrl_read(opaque, (uint32_t)reg);
 }
 
-static void fdctrl_write_mem (void *opaque, target_phys_addr_t reg,
+static void fdctrl_write_mem (void *opaque, hwaddr reg,
                               uint64_t value, unsigned size)
 {
     fdctrl_write(opaque, (uint32_t)reg, value);
@@ -799,6 +798,7 @@ static void fdctrl_handle_tc(void *opaque, int irq, int level)
 /* Change IRQ state */
 static void fdctrl_reset_irq(FDCtrl *fdctrl)
 {
+    fdctrl->status0 = 0;
     if (!(fdctrl->sra & FD_SRA_INTPEND))
         return;
     FLOPPY_DPRINTF("Reset interrupt\n");
@@ -806,14 +806,13 @@ static void fdctrl_reset_irq(FDCtrl *fdctrl)
     fdctrl->sra &= ~FD_SRA_INTPEND;
 }
 
-static void fdctrl_raise_irq(FDCtrl *fdctrl, uint8_t status0)
+static void fdctrl_raise_irq(FDCtrl *fdctrl)
 {
     /* Sparc mutation */
     if (fdctrl->sun4m && (fdctrl->msr & FD_MSR_CMDBUSY)) {
         /* XXX: not sure */
         fdctrl->msr &= ~FD_MSR_CMDBUSY;
         fdctrl->msr |= FD_MSR_RQM | FD_MSR_DIO;
-        fdctrl->status0 = status0;
         return;
     }
     if (!(fdctrl->sra & FD_SRA_INTPEND)) {
@@ -822,7 +821,6 @@ static void fdctrl_raise_irq(FDCtrl *fdctrl, uint8_t status0)
     }
 
     fdctrl->reset_sensei = 0;
-    fdctrl->status0 = status0;
     FLOPPY_DPRINTF("Set interrupt status to 0x%02x\n", fdctrl->status0);
 }
 
@@ -851,7 +849,8 @@ static void fdctrl_reset(FDCtrl *fdctrl, int do_irq)
         fd_recalibrate(&fdctrl->drives[i]);
     fdctrl_reset_fifo(fdctrl);
     if (do_irq) {
-        fdctrl_raise_irq(fdctrl, FD_SR0_RDYCHG);
+        fdctrl->status0 |= FD_SR0_RDYCHG;
+        fdctrl_raise_irq(fdctrl);
         fdctrl->reset_sensei = FD_RESET_SENSEI_COUNT;
     }
 }
@@ -1079,15 +1078,12 @@ static void fdctrl_reset_fifo(FDCtrl *fdctrl)
 }
 
 /* Set FIFO status for the host to read */
-static void fdctrl_set_fifo(FDCtrl *fdctrl, int fifo_len, uint8_t status0)
+static void fdctrl_set_fifo(FDCtrl *fdctrl, int fifo_len)
 {
     fdctrl->data_dir = FD_DIR_READ;
     fdctrl->data_len = fifo_len;
     fdctrl->data_pos = 0;
     fdctrl->msr |= FD_MSR_CMDBUSY | FD_MSR_RQM | FD_MSR_DIO;
-    if (status0) {
-        fdctrl_raise_irq(fdctrl, status0);
-    }
 }
 
 /* Set an error: unimplemented/unknown command */
@@ -1096,7 +1092,7 @@ static void fdctrl_unimplemented(FDCtrl *fdctrl, int direction)
     qemu_log_mask(LOG_UNIMP, "fdc: unimplemented command 0x%02x\n",
                   fdctrl->fifo[0]);
     fdctrl->fifo[0] = FD_SR0_INVCMD;
-    fdctrl_set_fifo(fdctrl, 1, 0);
+    fdctrl_set_fifo(fdctrl, 1);
 }
 
 /* Seek to next sector
@@ -1126,11 +1122,13 @@ static int fdctrl_seek_to_next_sect(FDCtrl *fdctrl, FDrive *cur_drv)
             } else {
                 new_head = 0;
                 new_track++;
+                fdctrl->status0 |= FD_SR0_SEEK;
                 if ((cur_drv->flags & FDISK_DBL_SIDES) == 0) {
                     ret = 0;
                 }
             }
         } else {
+            fdctrl->status0 |= FD_SR0_SEEK;
             new_track++;
             ret = 0;
         }
@@ -1150,10 +1148,14 @@ static void fdctrl_stop_transfer(FDCtrl *fdctrl, uint8_t status0,
                                  uint8_t status1, uint8_t status2)
 {
     FDrive *cur_drv;
-
     cur_drv = get_cur_drv(fdctrl);
-    fdctrl->status0 = status0 | FD_SR0_SEEK | (cur_drv->head << 2) |
-                      GET_CUR_DRV(fdctrl);
+
+    fdctrl->status0 &= ~(FD_SR0_DS0 | FD_SR0_DS1 | FD_SR0_HEAD);
+    fdctrl->status0 |= GET_CUR_DRV(fdctrl);
+    if (cur_drv->head) {
+        fdctrl->status0 |= FD_SR0_HEAD;
+    }
+    fdctrl->status0 |= status0;
 
     FLOPPY_DPRINTF("transfer status: %02x %02x %02x (%02x)\n",
                    status0, status1, status2, fdctrl->status0);
@@ -1170,7 +1172,9 @@ static void fdctrl_stop_transfer(FDCtrl *fdctrl, uint8_t status0,
     }
     fdctrl->msr |= FD_MSR_RQM | FD_MSR_DIO;
     fdctrl->msr &= ~FD_MSR_NONDMA;
-    fdctrl_set_fifo(fdctrl, 7, fdctrl->status0);
+
+    fdctrl_set_fifo(fdctrl, 7);
+    fdctrl_raise_irq(fdctrl);
 }
 
 /* Prepare a data transfer (either DMA or FIFO) */
@@ -1178,7 +1182,6 @@ static void fdctrl_start_transfer(FDCtrl *fdctrl, int direction)
 {
     FDrive *cur_drv;
     uint8_t kh, kt, ks;
-    int did_seek = 0;
 
     SET_CUR_DRV(fdctrl, fdctrl->fifo[1] & FD_DOR_SELMASK);
     cur_drv = get_cur_drv(fdctrl);
@@ -1212,7 +1215,7 @@ static void fdctrl_start_transfer(FDCtrl *fdctrl, int direction)
         fdctrl->fifo[5] = ks;
         return;
     case 1:
-        did_seek = 1;
+        fdctrl->status0 |= FD_SR0_SEEK;
         break;
     default:
         break;
@@ -1234,16 +1237,12 @@ static void fdctrl_start_transfer(FDCtrl *fdctrl, int direction)
     /* Set the FIFO state */
     fdctrl->data_dir = direction;
     fdctrl->data_pos = 0;
-    fdctrl->msr |= FD_MSR_CMDBUSY;
+    assert(fdctrl->msr & FD_MSR_CMDBUSY);
     if (fdctrl->fifo[0] & 0x80)
         fdctrl->data_state |= FD_STATE_MULTI;
     else
         fdctrl->data_state &= ~FD_STATE_MULTI;
-    if (did_seek)
-        fdctrl->data_state |= FD_STATE_SEEK;
-    else
-        fdctrl->data_state &= ~FD_STATE_SEEK;
-    if (fdctrl->fifo[5] == 00) {
+    if (fdctrl->fifo[5] == 0) {
         fdctrl->data_len = fdctrl->fifo[8];
     } else {
         int tmp;
@@ -1266,14 +1265,21 @@ static void fdctrl_start_transfer(FDCtrl *fdctrl, int direction)
         if (((direction == FD_DIR_SCANE || direction == FD_DIR_SCANL ||
               direction == FD_DIR_SCANH) && dma_mode == 0) ||
             (direction == FD_DIR_WRITE && dma_mode == 2) ||
-            (direction == FD_DIR_READ && dma_mode == 1)) {
+            (direction == FD_DIR_READ && dma_mode == 1) ||
+            (direction == FD_DIR_VERIFY)) {
             /* No access is allowed until DMA transfer has completed */
             fdctrl->msr &= ~FD_MSR_RQM;
-            /* Now, we just have to wait for the DMA controller to
-             * recall us...
-             */
-            DMA_hold_DREQ(fdctrl->dma_chann);
-            DMA_schedule(fdctrl->dma_chann);
+            if (direction != FD_DIR_VERIFY) {
+                /* Now, we just have to wait for the DMA controller to
+                 * recall us...
+                 */
+                DMA_hold_DREQ(fdctrl->dma_chann);
+                DMA_schedule(fdctrl->dma_chann);
+            } else {
+                /* Start transfer */
+                fdctrl_transfer_handler(fdctrl, fdctrl->dma_chann, 0,
+                                        fdctrl->data_len);
+            }
             return;
         } else {
             FLOPPY_DPRINTF("bad dma_mode=%d direction=%d\n", dma_mode,
@@ -1285,9 +1291,7 @@ static void fdctrl_start_transfer(FDCtrl *fdctrl, int direction)
     if (direction != FD_DIR_WRITE)
         fdctrl->msr |= FD_MSR_DIO;
     /* IO based transfer: calculate len */
-    fdctrl_raise_irq(fdctrl, FD_SR0_SEEK);
-
-    return;
+    fdctrl_raise_irq(fdctrl);
 }
 
 /* Prepare a transfer of deleted data */
@@ -1378,6 +1382,9 @@ static int fdctrl_transfer_handler (void *opaque, int nchan,
                 goto transfer_error;
             }
             break;
+        case FD_DIR_VERIFY:
+            /* VERIFY commands */
+            break;
         default:
             /* SCAN commands */
             {
@@ -1413,8 +1420,6 @@ static int fdctrl_transfer_handler (void *opaque, int nchan,
         fdctrl->data_dir == FD_DIR_SCANL ||
         fdctrl->data_dir == FD_DIR_SCANH)
         status2 = FD_SR2_SEH;
-    if (FD_DID_SEEK(fdctrl->data_state))
-        status0 |= FD_SR0_SEEK;
     fdctrl->data_len -= len;
     fdctrl_stop_transfer(fdctrl, status0, status1, status2);
  transfer_error:
@@ -1460,7 +1465,7 @@ static uint32_t fdctrl_read_data(FDCtrl *fdctrl)
          * then from status mode to command mode
          */
         if (fdctrl->msr & FD_MSR_NONDMA) {
-            fdctrl_stop_transfer(fdctrl, FD_SR0_SEEK, 0x00, 0x00);
+            fdctrl_stop_transfer(fdctrl, 0x00, 0x00, 0x00);
         } else {
             fdctrl_reset_fifo(fdctrl);
             fdctrl_reset_irq(fdctrl);
@@ -1508,7 +1513,7 @@ static void fdctrl_format_sector(FDCtrl *fdctrl)
         fdctrl->fifo[5] = ks;
         return;
     case 1:
-        fdctrl->data_state |= FD_STATE_SEEK;
+        fdctrl->status0 |= FD_SR0_SEEK;
         break;
     default:
         break;
@@ -1522,10 +1527,7 @@ static void fdctrl_format_sector(FDCtrl *fdctrl)
         if (cur_drv->sect == cur_drv->last_sect) {
             fdctrl->data_state &= ~FD_STATE_FORMAT;
             /* Last sector done */
-            if (FD_DID_SEEK(fdctrl->data_state))
-                fdctrl_stop_transfer(fdctrl, FD_SR0_SEEK, 0x00, 0x00);
-            else
-                fdctrl_stop_transfer(fdctrl, 0x00, 0x00, 0x00);
+            fdctrl_stop_transfer(fdctrl, 0x00, 0x00, 0x00);
         } else {
             /* More to do */
             fdctrl->data_pos = 0;
@@ -1538,7 +1540,7 @@ static void fdctrl_handle_lock(FDCtrl *fdctrl, int direction)
 {
     fdctrl->lock = (fdctrl->fifo[0] & 0x80) ? 1 : 0;
     fdctrl->fifo[0] = fdctrl->lock << 4;
-    fdctrl_set_fifo(fdctrl, 1, 0);
+    fdctrl_set_fifo(fdctrl, 1);
 }
 
 static void fdctrl_handle_dumpreg(FDCtrl *fdctrl, int direction)
@@ -1563,20 +1565,20 @@ static void fdctrl_handle_dumpreg(FDCtrl *fdctrl, int direction)
         (cur_drv->perpendicular << 2);
     fdctrl->fifo[8] = fdctrl->config;
     fdctrl->fifo[9] = fdctrl->precomp_trk;
-    fdctrl_set_fifo(fdctrl, 10, 0);
+    fdctrl_set_fifo(fdctrl, 10);
 }
 
 static void fdctrl_handle_version(FDCtrl *fdctrl, int direction)
 {
     /* Controller's version */
     fdctrl->fifo[0] = fdctrl->version;
-    fdctrl_set_fifo(fdctrl, 1, 0);
+    fdctrl_set_fifo(fdctrl, 1);
 }
 
 static void fdctrl_handle_partid(FDCtrl *fdctrl, int direction)
 {
     fdctrl->fifo[0] = 0x41; /* Stepping 1 */
-    fdctrl_set_fifo(fdctrl, 1, 0);
+    fdctrl_set_fifo(fdctrl, 1);
 }
 
 static void fdctrl_handle_restore(FDCtrl *fdctrl, int direction)
@@ -1629,7 +1631,7 @@ static void fdctrl_handle_save(FDCtrl *fdctrl, int direction)
     fdctrl->fifo[12] = fdctrl->pwrd;
     fdctrl->fifo[13] = 0;
     fdctrl->fifo[14] = 0;
-    fdctrl_set_fifo(fdctrl, 15, 0);
+    fdctrl_set_fifo(fdctrl, 15);
 }
 
 static void fdctrl_handle_readid(FDCtrl *fdctrl, int direction)
@@ -1652,7 +1654,6 @@ static void fdctrl_handle_format_track(FDCtrl *fdctrl, int direction)
         fdctrl->data_state |= FD_STATE_MULTI;
     else
         fdctrl->data_state &= ~FD_STATE_MULTI;
-    fdctrl->data_state &= ~FD_STATE_SEEK;
     cur_drv->bps =
         fdctrl->fifo[2] > 7 ? 16384 : 128 << fdctrl->fifo[2];
 #if 0
@@ -1695,7 +1696,7 @@ static void fdctrl_handle_sense_drive_status(FDCtrl *fdctrl, int direction)
         (cur_drv->head << 2) |
         GET_CUR_DRV(fdctrl) |
         0x28;
-    fdctrl_set_fifo(fdctrl, 1, 0);
+    fdctrl_set_fifo(fdctrl, 1);
 }
 
 static void fdctrl_handle_recalibrate(FDCtrl *fdctrl, int direction)
@@ -1707,7 +1708,8 @@ static void fdctrl_handle_recalibrate(FDCtrl *fdctrl, int direction)
     fd_recalibrate(cur_drv);
     fdctrl_reset_fifo(fdctrl);
     /* Raise Interrupt */
-    fdctrl_raise_irq(fdctrl, FD_SR0_SEEK);
+    fdctrl->status0 |= FD_SR0_SEEK;
+    fdctrl_raise_irq(fdctrl);
 }
 
 static void fdctrl_handle_sense_interrupt_status(FDCtrl *fdctrl, int direction)
@@ -1720,7 +1722,7 @@ static void fdctrl_handle_sense_interrupt_status(FDCtrl *fdctrl, int direction)
         fdctrl->reset_sensei--;
     } else if (!(fdctrl->sra & FD_SRA_INTPEND)) {
         fdctrl->fifo[0] = FD_SR0_INVCMD;
-        fdctrl_set_fifo(fdctrl, 1, 0);
+        fdctrl_set_fifo(fdctrl, 1);
         return;
     } else {
         fdctrl->fifo[0] =
@@ -1729,7 +1731,7 @@ static void fdctrl_handle_sense_interrupt_status(FDCtrl *fdctrl, int direction)
     }
 
     fdctrl->fifo[1] = cur_drv->track;
-    fdctrl_set_fifo(fdctrl, 2, 0);
+    fdctrl_set_fifo(fdctrl, 2);
     fdctrl_reset_irq(fdctrl);
     fdctrl->status0 = FD_SR0_RDYCHG;
 }
@@ -1746,7 +1748,8 @@ static void fdctrl_handle_seek(FDCtrl *fdctrl, int direction)
      */
     fd_seek(cur_drv, cur_drv->head, fdctrl->fifo[2], cur_drv->sect, 1);
     /* Raise Interrupt */
-    fdctrl_raise_irq(fdctrl, FD_SR0_SEEK);
+    fdctrl->status0 |= FD_SR0_SEEK;
+    fdctrl_raise_irq(fdctrl);
 }
 
 static void fdctrl_handle_perpendicular_mode(FDCtrl *fdctrl, int direction)
@@ -1771,7 +1774,7 @@ static void fdctrl_handle_powerdown_mode(FDCtrl *fdctrl, int direction)
 {
     fdctrl->pwrd = fdctrl->fifo[1];
     fdctrl->fifo[0] = fdctrl->fifo[1];
-    fdctrl_set_fifo(fdctrl, 1, 0);
+    fdctrl_set_fifo(fdctrl, 1);
 }
 
 static void fdctrl_handle_option(FDCtrl *fdctrl, int direction)
@@ -1790,7 +1793,7 @@ static void fdctrl_handle_drive_specification_command(FDCtrl *fdctrl, int direct
             fdctrl->fifo[0] = fdctrl->fifo[1];
             fdctrl->fifo[2] = 0;
             fdctrl->fifo[3] = 0;
-            fdctrl_set_fifo(fdctrl, 4, 0);
+            fdctrl_set_fifo(fdctrl, 4);
         } else {
             fdctrl_reset_fifo(fdctrl);
         }
@@ -1798,7 +1801,7 @@ static void fdctrl_handle_drive_specification_command(FDCtrl *fdctrl, int direct
         /* ERROR */
         fdctrl->fifo[0] = 0x80 |
             (cur_drv->head << 2) | GET_CUR_DRV(fdctrl);
-        fdctrl_set_fifo(fdctrl, 1, 0);
+        fdctrl_set_fifo(fdctrl, 1);
     }
 }
 
@@ -1817,7 +1820,8 @@ static void fdctrl_handle_relative_seek_in(FDCtrl *fdctrl, int direction)
     }
     fdctrl_reset_fifo(fdctrl);
     /* Raise Interrupt */
-    fdctrl_raise_irq(fdctrl, FD_SR0_SEEK);
+    fdctrl->status0 |= FD_SR0_SEEK;
+    fdctrl_raise_irq(fdctrl);
 }
 
 static void fdctrl_handle_relative_seek_out(FDCtrl *fdctrl, int direction)
@@ -1834,7 +1838,8 @@ static void fdctrl_handle_relative_seek_out(FDCtrl *fdctrl, int direction)
     }
     fdctrl_reset_fifo(fdctrl);
     /* Raise Interrupt */
-    fdctrl_raise_irq(fdctrl, FD_SR0_SEEK);
+    fdctrl->status0 |= FD_SR0_SEEK;
+    fdctrl_raise_irq(fdctrl);
 }
 
 static const struct {
@@ -1856,7 +1861,7 @@ static const struct {
     { FD_CMD_SAVE, 0xff, "SAVE", 0, fdctrl_handle_save }, /* part of READ DELETED DATA */
     { FD_CMD_READ_DELETED, 0x1f, "READ DELETED DATA", 8, fdctrl_start_transfer_del, FD_DIR_READ },
     { FD_CMD_SCAN_EQUAL, 0x1f, "SCAN EQUAL", 8, fdctrl_start_transfer, FD_DIR_SCANE },
-    { FD_CMD_VERIFY, 0x1f, "VERIFY", 8, fdctrl_unimplemented },
+    { FD_CMD_VERIFY, 0x1f, "VERIFY", 8, fdctrl_start_transfer, FD_DIR_VERIFY },
     { FD_CMD_SCAN_LOW_OR_EQUAL, 0x1f, "SCAN LOW OR EQUAL", 8, fdctrl_start_transfer, FD_DIR_SCANL },
     { FD_CMD_SCAN_HIGH_OR_EQUAL, 0x1f, "SCAN HIGH OR EQUAL", 8, fdctrl_start_transfer, FD_DIR_SCANH },
     { FD_CMD_WRITE_DELETED, 0x3f, "WRITE DELETED DATA", 8, fdctrl_start_transfer_del, FD_DIR_WRITE },
@@ -1920,7 +1925,7 @@ static void fdctrl_write_data(FDCtrl *fdctrl, uint32_t value)
          * then from status mode to command mode
          */
         if (fdctrl->data_pos == fdctrl->data_len)
-            fdctrl_stop_transfer(fdctrl, FD_SR0_SEEK, 0x00, 0x00);
+            fdctrl_stop_transfer(fdctrl, 0x00, 0x00, 0x00);
         return;
     }
     if (fdctrl->data_pos == 0) {
@@ -1994,11 +1999,11 @@ static int fdctrl_connect_drives(FDCtrl *fdctrl)
         drive->fdctrl = fdctrl;
 
         if (drive->bs) {
-            if (bdrv_get_on_error(drive->bs, 0) != BLOCK_ERR_STOP_ENOSPC) {
+            if (bdrv_get_on_error(drive->bs, 0) != BLOCKDEV_ON_ERROR_ENOSPC) {
                 error_report("fdc doesn't support drive option werror");
                 return -1;
             }
-            if (bdrv_get_on_error(drive->bs, 1) != BLOCK_ERR_REPORT) {
+            if (bdrv_get_on_error(drive->bs, 1) != BLOCKDEV_ON_ERROR_REPORT) {
                 error_report("fdc doesn't support drive option rerror");
                 return -1;
             }
@@ -2034,7 +2039,7 @@ ISADevice *fdctrl_init_isa(ISABus *bus, DriveInfo **fds)
 }
 
 void fdctrl_init_sysbus(qemu_irq irq, int dma_chann,
-                        target_phys_addr_t mmio_base, DriveInfo **fds)
+                        hwaddr mmio_base, DriveInfo **fds)
 {
     FDCtrl *fdctrl;
     DeviceState *dev;
@@ -2055,7 +2060,7 @@ void fdctrl_init_sysbus(qemu_irq irq, int dma_chann,
     sysbus_mmio_map(&sys->busdev, 0, mmio_base);
 }
 
-void sun4m_fdctrl_init(qemu_irq irq, target_phys_addr_t io_base,
+void sun4m_fdctrl_init(qemu_irq irq, hwaddr io_base,
                        DriveInfo **fds, qemu_irq *fdc_tc)
 {
     DeviceState *dev;
diff --git a/hw/fdc.h b/hw/fdc.h
index b5c9f3107..a8f6f7c85 100644
--- a/hw/fdc.h
+++ b/hw/fdc.h
@@ -15,8 +15,8 @@ typedef enum FDriveType {
 
 ISADevice *fdctrl_init_isa(ISABus *bus, DriveInfo **fds);
 void fdctrl_init_sysbus(qemu_irq irq, int dma_chann,
-                        target_phys_addr_t mmio_base, DriveInfo **fds);
-void sun4m_fdctrl_init(qemu_irq irq, target_phys_addr_t io_base,
+                        hwaddr mmio_base, DriveInfo **fds);
+void sun4m_fdctrl_init(qemu_irq irq, hwaddr io_base,
                        DriveInfo **fds, qemu_irq *fdc_tc);
 
 FDriveType isa_fdc_get_drive_type(ISADevice *fdc, int i);
diff --git a/hw/fifo.c b/hw/fifo.c
new file mode 100644
index 000000000..68a955a77
--- /dev/null
+++ b/hw/fifo.c
@@ -0,0 +1,78 @@
+/*
+ * Generic FIFO component, implemented as a circular buffer.
+ *
+ * Copyright (c) 2012 Peter A. G. Crosthwaite
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version
+ * 2 of the License, or (at your option) any later version.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "fifo.h"
+
+void fifo8_create(Fifo8 *fifo, uint32_t capacity)
+{
+    fifo->data = g_new(uint8_t, capacity);
+    fifo->capacity = capacity;
+    fifo->head = 0;
+    fifo->num = 0;
+}
+
+void fifo8_destroy(Fifo8 *fifo)
+{
+    g_free(fifo->data);
+}
+
+void fifo8_push(Fifo8 *fifo, uint8_t data)
+{
+    if (fifo->num == fifo->capacity) {
+        abort();
+    }
+    fifo->data[(fifo->head + fifo->num) % fifo->capacity] = data;
+    fifo->num++;
+}
+
+uint8_t fifo8_pop(Fifo8 *fifo)
+{
+    uint8_t ret;
+
+    if (fifo->num == 0) {
+        abort();
+    }
+    ret = fifo->data[fifo->head++];
+    fifo->head %= fifo->capacity;
+    fifo->num--;
+    return ret;
+}
+
+void fifo8_reset(Fifo8 *fifo)
+{
+    fifo->num = 0;
+}
+
+bool fifo8_is_empty(Fifo8 *fifo)
+{
+    return (fifo->num == 0);
+}
+
+bool fifo8_is_full(Fifo8 *fifo)
+{
+    return (fifo->num == fifo->capacity);
+}
+
+const VMStateDescription vmstate_fifo8 = {
+    .name = "Fifo8",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .minimum_version_id_old = 1,
+    .fields      = (VMStateField[]) {
+        VMSTATE_VBUFFER_UINT32(data, Fifo8, 1, NULL, 0, capacity),
+        VMSTATE_UINT32(head, Fifo8),
+        VMSTATE_UINT32(num, Fifo8),
+        VMSTATE_END_OF_LIST()
+    }
+};
diff --git a/hw/fifo.h b/hw/fifo.h
new file mode 100644
index 000000000..f23890abf
--- /dev/null
+++ b/hw/fifo.h
@@ -0,0 +1,99 @@
+#ifndef FIFO_H
+#define FIFO_H
+
+#include "hw.h"
+
+typedef struct {
+    /* All fields are private */
+    uint8_t *data;
+    uint32_t capacity;
+    uint32_t head;
+    uint32_t num;
+} Fifo8;
+
+/**
+ * fifo8_create:
+ * @fifo: struct Fifo8 to initialise with new FIFO
+ * @capacity: capacity of the newly created FIFO
+ *
+ * Create a FIFO of the specified size. Clients should call fifo8_destroy()
+ * when finished using the fifo. The FIFO is initially empty.
+ */
+
+void fifo8_create(Fifo8 *fifo, uint32_t capacity);
+
+/**
+ * fifo8_destroy:
+ * @fifo: FIFO to cleanup
+ *
+ * Cleanup a FIFO created with fifo8_create(). Frees memory created for FIFO
+  *storage. The FIFO is no longer usable after this has been called.
+ */
+
+void fifo8_destroy(Fifo8 *fifo);
+
+/**
+ * fifo8_push:
+ * @fifo: FIFO to push to
+ * @data: data byte to push
+ *
+ * Push a data byte to the FIFO. Behaviour is undefined if the FIFO is full.
+ * Clients are responsible for checking for fullness using fifo8_is_full().
+ */
+
+void fifo8_push(Fifo8 *fifo, uint8_t data);
+
+/**
+ * fifo8_pop:
+ * @fifo: fifo to pop from
+ *
+ * Pop a data byte from the FIFO. Behaviour is undefined if the FIFO is empty.
+ * Clients are responsible for checking for emptyness using fifo8_is_empty().
+ *
+ * Returns: The popped data byte.
+ */
+
+uint8_t fifo8_pop(Fifo8 *fifo);
+
+/**
+ * fifo8_reset:
+ * @fifo: FIFO to reset
+ *
+ * Reset a FIFO. All data is discarded and the FIFO is emptied.
+ */
+
+void fifo8_reset(Fifo8 *fifo);
+
+/**
+ * fifo8_is_empty:
+ * @fifo: FIFO to check
+ *
+ * Check if a FIFO is empty.
+ *
+ * Returns: True if the fifo is empty, false otherwise.
+ */
+
+bool fifo8_is_empty(Fifo8 *fifo);
+
+/**
+ * fifo8_is_full:
+ * @fifo: FIFO to check
+ *
+ * Check if a FIFO is full.
+ *
+ * Returns: True if the fifo is full, false otherwise.
+ */
+
+bool fifo8_is_full(Fifo8 *fifo);
+
+extern const VMStateDescription vmstate_fifo8;
+
+#define VMSTATE_FIFO8(_field, _state) {                              \
+    .name       = (stringify(_field)),                               \
+    .size       = sizeof(Fifo8),                                     \
+    .vmsd       = &vmstate_fifo8,                                    \
+    .flags      = VMS_STRUCT,                                        \
+    .offset     = vmstate_offset_value(_state, _field, Fifo8),       \
+}
+
+#endif /* FIFO_H */
diff --git a/hw/flash.h b/hw/flash.h
index 9c9e5265b..d790f3c92 100644
--- a/hw/flash.h
+++ b/hw/flash.h
@@ -5,18 +5,18 @@
 typedef struct pflash_t pflash_t;
 
 /* pflash_cfi01.c */
-pflash_t *pflash_cfi01_register(target_phys_addr_t base,
+pflash_t *pflash_cfi01_register(hwaddr base,
                                 DeviceState *qdev, const char *name,
-                                target_phys_addr_t size,
+                                hwaddr size,
                                 BlockDriverState *bs,
                                 uint32_t sector_len, int nb_blocs, int width,
                                 uint16_t id0, uint16_t id1,
                                 uint16_t id2, uint16_t id3, int be);
 
 /* pflash_cfi02.c */
-pflash_t *pflash_cfi02_register(target_phys_addr_t base,
+pflash_t *pflash_cfi02_register(hwaddr base,
                                 DeviceState *qdev, const char *name,
-                                target_phys_addr_t size,
+                                hwaddr size,
                                 BlockDriverState *bs, uint32_t sector_len,
                                 int nb_blocs, int nb_mappings, int width,
                                 uint16_t id0, uint16_t id1,
diff --git a/hw/framebuffer.c b/hw/framebuffer.c
index 85a00a579..fa0f7863c 100644
--- a/hw/framebuffer.c
+++ b/hw/framebuffer.c
@@ -26,7 +26,7 @@
 void framebuffer_update_display(
     DisplayState *ds,
     MemoryRegion *address_space,
-    target_phys_addr_t base,
+    hwaddr base,
     int cols, /* Width in pixels.  */
     int rows, /* Height in pixels.  */
     int src_width, /* Length of source line, in bytes.  */
@@ -38,7 +38,7 @@ void framebuffer_update_display(
     int *first_row, /* Input and output.  */
     int *last_row /* Output only */)
 {
-    target_phys_addr_t src_len;
+    hwaddr src_len;
     uint8_t *dest;
     uint8_t *src;
     uint8_t *src_base;
@@ -107,5 +107,4 @@ void framebuffer_update_display(
                               DIRTY_MEMORY_VGA);
     *first_row = first;
     *last_row = last;
-    return;
 }
diff --git a/hw/framebuffer.h b/hw/framebuffer.h
index 527a6b85f..46e375b5e 100644
--- a/hw/framebuffer.h
+++ b/hw/framebuffer.h
@@ -10,7 +10,7 @@ typedef void (*drawfn)(void *, uint8_t *, const uint8_t *, int, int);
 void framebuffer_update_display(
     DisplayState *ds,
     MemoryRegion *address_space,
-    target_phys_addr_t base,
+    hwaddr base,
     int cols,
     int rows,
     int src_width,
diff --git a/hw/fw_cfg.c b/hw/fw_cfg.c
index 7b3b5769a..2b92cda98 100644
--- a/hw/fw_cfg.c
+++ b/hw/fw_cfg.c
@@ -183,6 +183,30 @@ static void fw_cfg_bootsplash(FWCfgState *s)
     }
 }
 
+static void fw_cfg_reboot(FWCfgState *s)
+{
+    int reboot_timeout = -1;
+    char *p;
+    const char *temp;
+
+    /* get user configuration */
+    QemuOptsList *plist = qemu_find_opts("boot-opts");
+    QemuOpts *opts = QTAILQ_FIRST(&plist->head);
+    if (opts != NULL) {
+        temp = qemu_opt_get(opts, "reboot-timeout");
+        if (temp != NULL) {
+            p = (char *)temp;
+            reboot_timeout = strtol(p, (char **)&p, 10);
+        }
+    }
+    /* validate the input */
+    if (reboot_timeout > 0xffff) {
+        error_report("reboot timeout is larger than 65535, force it to 65535.");
+        reboot_timeout = 0xffff;
+    }
+    fw_cfg_add_file(s, "etc/boot-fail-wait", g_memdup(&reboot_timeout, 4), 4);
+}
+
 static void fw_cfg_write(FWCfgState *s, uint8_t value)
 {
     int arch = !!(s->cur_entry & FW_CFG_ARCH_LOCAL);
@@ -234,37 +258,37 @@ static uint8_t fw_cfg_read(FWCfgState *s)
     return ret;
 }
 
-static uint64_t fw_cfg_data_mem_read(void *opaque, target_phys_addr_t addr,
+static uint64_t fw_cfg_data_mem_read(void *opaque, hwaddr addr,
                                      unsigned size)
 {
     return fw_cfg_read(opaque);
 }
 
-static void fw_cfg_data_mem_write(void *opaque, target_phys_addr_t addr,
+static void fw_cfg_data_mem_write(void *opaque, hwaddr addr,
                                   uint64_t value, unsigned size)
 {
     fw_cfg_write(opaque, (uint8_t)value);
 }
 
-static void fw_cfg_ctl_mem_write(void *opaque, target_phys_addr_t addr,
+static void fw_cfg_ctl_mem_write(void *opaque, hwaddr addr,
                                  uint64_t value, unsigned size)
 {
     fw_cfg_select(opaque, (uint16_t)value);
 }
 
-static bool fw_cfg_ctl_mem_valid(void *opaque, target_phys_addr_t addr,
+static bool fw_cfg_ctl_mem_valid(void *opaque, hwaddr addr,
                                  unsigned size, bool is_write)
 {
     return is_write && size == 2;
 }
 
-static uint64_t fw_cfg_comb_read(void *opaque, target_phys_addr_t addr,
+static uint64_t fw_cfg_comb_read(void *opaque, hwaddr addr,
                                  unsigned size)
 {
     return fw_cfg_read(opaque);
 }
 
-static void fw_cfg_comb_write(void *opaque, target_phys_addr_t addr,
+static void fw_cfg_comb_write(void *opaque, hwaddr addr,
                               uint64_t value, unsigned size)
 {
     switch (size) {
@@ -277,7 +301,7 @@ static void fw_cfg_comb_write(void *opaque, target_phys_addr_t addr,
     }
 }
 
-static bool fw_cfg_comb_valid(void *opaque, target_phys_addr_t addr,
+static bool fw_cfg_comb_valid(void *opaque, hwaddr addr,
                                   unsigned size, bool is_write)
 {
     return (size == 1) || (is_write && size == 2);
@@ -470,7 +494,7 @@ static void fw_cfg_machine_ready(struct Notifier *n, void *data)
 }
 
 FWCfgState *fw_cfg_init(uint32_t ctl_port, uint32_t data_port,
-                        target_phys_addr_t ctl_addr, target_phys_addr_t data_addr)
+                        hwaddr ctl_addr, hwaddr data_addr)
 {
     DeviceState *dev;
     SysBusDevice *d;
@@ -497,6 +521,7 @@ FWCfgState *fw_cfg_init(uint32_t ctl_port, uint32_t data_port,
     fw_cfg_add_i16(s, FW_CFG_MAX_CPUS, (uint16_t)max_cpus);
     fw_cfg_add_i16(s, FW_CFG_BOOT_MENU, (uint16_t)boot_menu);
     fw_cfg_bootsplash(s);
+    fw_cfg_reboot(s);
 
     s->machine_ready.notify = fw_cfg_machine_ready;
     qemu_add_machine_init_done_notifier(&s->machine_ready);
diff --git a/hw/fw_cfg.h b/hw/fw_cfg.h
index 856bf9199..619a39432 100644
--- a/hw/fw_cfg.h
+++ b/hw/fw_cfg.h
@@ -63,7 +63,7 @@ int fw_cfg_add_callback(FWCfgState *s, uint16_t key, FWCfgCallback callback,
 int fw_cfg_add_file(FWCfgState *s, const char *filename, uint8_t *data,
                     uint32_t len);
 FWCfgState *fw_cfg_init(uint32_t ctl_port, uint32_t data_port,
-                        target_phys_addr_t crl_addr, target_phys_addr_t data_addr);
+                        hwaddr crl_addr, hwaddr data_addr);
 
 #endif /* NO_QEMU_PROTOS */
 
diff --git a/hw/g364fb.c b/hw/g364fb.c
index 3a0b68fba..8192baf1c 100644
--- a/hw/g364fb.c
+++ b/hw/g364fb.c
@@ -197,7 +197,8 @@ static void g364fb_draw_graphic8(G364State *s)
                 reset_dirty(s, page_min, page_max);
                 page_min = (ram_addr_t)-1;
                 page_max = 0;
-                dpy_update(s->ds, xmin, ymin, xmax - xmin + 1, ymax - ymin + 1);
+                dpy_gfx_update(s->ds, xmin, ymin,
+                               xmax - xmin + 1, ymax - ymin + 1);
                 xmin = s->width;
                 xmax = 0;
                 ymin = s->height;
@@ -216,7 +217,7 @@ static void g364fb_draw_graphic8(G364State *s)
 
 done:
     if (page_min != (ram_addr_t)-1) {
-        dpy_update(s->ds, xmin, ymin, xmax - xmin + 1, ymax - ymin + 1);
+        dpy_gfx_update(s->ds, xmin, ymin, xmax - xmin + 1, ymax - ymin + 1);
         reset_dirty(s, page_min, page_max);
     }
 }
@@ -238,7 +239,7 @@ static void g364fb_draw_blank(G364State *s)
         d += ds_get_linesize(s->ds);
     }
 
-    dpy_update(s->ds, 0, 0, s->width, s->height);
+    dpy_gfx_update(s->ds, 0, 0, s->width, s->height);
     s->blanked = 1;
 }
 
@@ -289,10 +290,11 @@ static void g364fb_reset(G364State *s)
     g364fb_invalidate_display(s);
 }
 
-static void g364fb_screen_dump(void *opaque, const char *filename, bool cswitch)
+static void g364fb_screen_dump(void *opaque, const char *filename, bool cswitch,
+                               Error **errp)
 {
     G364State *s = opaque;
-    int y, x;
+    int ret, y, x;
     uint8_t index;
     uint8_t *data_buffer;
     FILE *f;
@@ -300,40 +302,68 @@ static void g364fb_screen_dump(void *opaque, const char *filename, bool cswitch)
     qemu_flush_coalesced_mmio_buffer();
 
     if (s->depth != 8) {
-        error_report("g364: unknown guest depth %d", s->depth);
+        error_setg(errp, "g364: unknown guest depth %d", s->depth);
         return;
     }
 
     f = fopen(filename, "wb");
-    if (!f)
+    if (!f) {
+        error_setg(errp, "failed to open file '%s': %s", filename,
+                   strerror(errno));
         return;
+    }
 
     if (s->ctla & CTLA_FORCE_BLANK) {
         /* blank screen */
-        fprintf(f, "P4\n%d %d\n",
-            s->width, s->height);
+        ret = fprintf(f, "P4\n%d %d\n", s->width, s->height);
+        if (ret < 0) {
+            goto write_err;
+        }
         for (y = 0; y < s->height; y++)
-            for (x = 0; x < s->width; x++)
-                fputc(0, f);
+            for (x = 0; x < s->width; x++) {
+                ret = fputc(0, f);
+                if (ret == EOF) {
+                    goto write_err;
+                }
+            }
     } else {
         data_buffer = s->vram + s->top_of_screen;
-        fprintf(f, "P6\n%d %d\n%d\n",
-            s->width, s->height, 255);
+        ret = fprintf(f, "P6\n%d %d\n%d\n", s->width, s->height, 255);
+        if (ret < 0) {
+            goto write_err;
+        }
         for (y = 0; y < s->height; y++)
             for (x = 0; x < s->width; x++, data_buffer++) {
                 index = *data_buffer;
-                fputc(s->color_palette[index][0], f);
-                fputc(s->color_palette[index][1], f);
-                fputc(s->color_palette[index][2], f);
+                ret = fputc(s->color_palette[index][0], f);
+                if (ret == EOF) {
+                    goto write_err;
+                }
+                ret = fputc(s->color_palette[index][1], f);
+                if (ret == EOF) {
+                    goto write_err;
+                }
+                ret = fputc(s->color_palette[index][2], f);
+                if (ret == EOF) {
+                    goto write_err;
+                }
         }
     }
 
+out:
     fclose(f);
+    return;
+
+write_err:
+    error_setg(errp, "failed to write to file '%s': %s", filename,
+               strerror(errno));
+    unlink(filename);
+    goto out;
 }
 
 /* called for accesses to io ports */
 static uint64_t g364fb_ctrl_read(void *opaque,
-                                 target_phys_addr_t addr,
+                                 hwaddr addr,
                                  unsigned int size)
 {
     G364State *s = opaque;
@@ -395,7 +425,7 @@ static void g364_invalidate_cursor_position(G364State *s)
 }
 
 static void g364fb_ctrl_write(void *opaque,
-                              target_phys_addr_t addr,
+                              hwaddr addr,
                               uint64_t val,
                               unsigned int size)
 {
diff --git a/hw/grlib.h b/hw/grlib.h
index e1c41378d..35c22f599 100644
--- a/hw/grlib.h
+++ b/hw/grlib.h
@@ -41,7 +41,7 @@ void grlib_irqmp_set_irq(void *opaque, int irq, int level);
 void grlib_irqmp_ack(DeviceState *dev, int intno);
 
 static inline
-DeviceState *grlib_irqmp_create(target_phys_addr_t   base,
+DeviceState *grlib_irqmp_create(hwaddr   base,
                                 CPUSPARCState            *env,
                                 qemu_irq           **cpu_irqs,
                                 uint32_t             nr_irqs,
@@ -73,7 +73,7 @@ DeviceState *grlib_irqmp_create(target_phys_addr_t   base,
 /* GPTimer */
 
 static inline
-DeviceState *grlib_gptimer_create(target_phys_addr_t  base,
+DeviceState *grlib_gptimer_create(hwaddr  base,
                                   uint32_t            nr_timers,
                                   uint32_t            freq,
                                   qemu_irq           *cpu_irqs,
@@ -103,7 +103,7 @@ DeviceState *grlib_gptimer_create(target_phys_addr_t  base,
 /* APB UART */
 
 static inline
-DeviceState *grlib_apbuart_create(target_phys_addr_t  base,
+DeviceState *grlib_apbuart_create(hwaddr  base,
                                   CharDriverState    *serial,
                                   qemu_irq            irq)
 {
diff --git a/hw/grlib_apbuart.c b/hw/grlib_apbuart.c
index 73fc9894d..0865764de 100644
--- a/hw/grlib_apbuart.c
+++ b/hw/grlib_apbuart.c
@@ -151,7 +151,7 @@ static void grlib_apbuart_event(void *opaque, int event)
 }
 
 
-static uint64_t grlib_apbuart_read(void *opaque, target_phys_addr_t addr,
+static uint64_t grlib_apbuart_read(void *opaque, hwaddr addr,
                                    unsigned size)
 {
     UART     *uart = opaque;
@@ -181,7 +181,7 @@ static uint64_t grlib_apbuart_read(void *opaque, target_phys_addr_t addr,
     }
 }
 
-static void grlib_apbuart_write(void *opaque, target_phys_addr_t addr,
+static void grlib_apbuart_write(void *opaque, hwaddr addr,
                                 uint64_t value, unsigned size)
 {
     UART          *uart = opaque;
diff --git a/hw/grlib_gptimer.c b/hw/grlib_gptimer.c
index 41770a9e6..2fdccfba0 100644
--- a/hw/grlib_gptimer.c
+++ b/hw/grlib_gptimer.c
@@ -155,11 +155,11 @@ static void grlib_gptimer_hit(void *opaque)
     }
 }
 
-static uint64_t grlib_gptimer_read(void *opaque, target_phys_addr_t addr,
+static uint64_t grlib_gptimer_read(void *opaque, hwaddr addr,
                                    unsigned size)
 {
     GPTimerUnit        *unit  = opaque;
-    target_phys_addr_t  timer_addr;
+    hwaddr  timer_addr;
     int                 id;
     uint32_t            value = 0;
 
@@ -214,11 +214,11 @@ static uint64_t grlib_gptimer_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void grlib_gptimer_write(void *opaque, target_phys_addr_t addr,
+static void grlib_gptimer_write(void *opaque, hwaddr addr,
                                 uint64_t value, unsigned size)
 {
     GPTimerUnit        *unit = opaque;
-    target_phys_addr_t  timer_addr;
+    hwaddr  timer_addr;
     int                 id;
 
     addr &= 0xff;
diff --git a/hw/grlib_irqmp.c b/hw/grlib_irqmp.c
index 0f6e65cf2..23a6a02bc 100644
--- a/hw/grlib_irqmp.c
+++ b/hw/grlib_irqmp.c
@@ -162,7 +162,7 @@ void grlib_irqmp_set_irq(void *opaque, int irq, int level)
     }
 }
 
-static uint64_t grlib_irqmp_read(void *opaque, target_phys_addr_t addr,
+static uint64_t grlib_irqmp_read(void *opaque, hwaddr addr,
                                  unsigned size)
 {
     IRQMP      *irqmp = opaque;
@@ -226,7 +226,7 @@ static uint64_t grlib_irqmp_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void grlib_irqmp_write(void *opaque, target_phys_addr_t addr,
+static void grlib_irqmp_write(void *opaque, hwaddr addr,
                               uint64_t value, unsigned size)
 {
     IRQMP      *irqmp = opaque;
diff --git a/hw/gt64xxx.c b/hw/gt64xxx.c
index e95e66483..95d491d93 100644
--- a/hw/gt64xxx.c
+++ b/hw/gt64xxx.c
@@ -225,8 +225,8 @@
 #define GT_PCI1_SERR1MASK    	(0xca8 >> 2)
 
 #define PCI_MAPPING_ENTRY(regname)            \
-    target_phys_addr_t regname ##_start;      \
-    target_phys_addr_t regname ##_length;     \
+    hwaddr regname ##_start;      \
+    hwaddr regname ##_length;     \
     MemoryRegion regname ##_mem
 
 #define TYPE_GT64120_PCI_HOST_BRIDGE "gt64120"
@@ -245,11 +245,11 @@ typedef struct GT64120State {
 /* Adjust range to avoid touching space which isn't mappable via PCI */
 /* XXX: Hardcoded values for Malta: 0x1e000000 - 0x1f100000
                                     0x1fc00000 - 0x1fd00000  */
-static void check_reserved_space (target_phys_addr_t *start,
-                                  target_phys_addr_t *length)
+static void check_reserved_space (hwaddr *start,
+                                  hwaddr *length)
 {
-    target_phys_addr_t begin = *start;
-    target_phys_addr_t end = *start + *length;
+    hwaddr begin = *start;
+    hwaddr end = *start + *length;
 
     if (end >= 0x1e000000LL && end < 0x1f100000LL)
         end = 0x1e000000LL;
@@ -271,8 +271,8 @@ static void check_reserved_space (target_phys_addr_t *start,
 
 static void gt64120_isd_mapping(GT64120State *s)
 {
-    target_phys_addr_t start = s->regs[GT_ISD] << 21;
-    target_phys_addr_t length = 0x1000;
+    hwaddr start = s->regs[GT_ISD] << 21;
+    hwaddr length = 0x1000;
 
     if (s->ISD_length) {
         memory_region_del_subregion(get_system_memory(), &s->ISD_mem);
@@ -311,7 +311,7 @@ static void gt64120_pci_mapping(GT64120State *s)
     }
 }
 
-static void gt64120_writel (void *opaque, target_phys_addr_t addr,
+static void gt64120_writel (void *opaque, hwaddr addr,
                             uint64_t val, unsigned size)
 {
     GT64120State *s = opaque;
@@ -594,7 +594,7 @@ static void gt64120_writel (void *opaque, target_phys_addr_t addr,
 }
 
 static uint64_t gt64120_readl (void *opaque,
-                               target_phys_addr_t addr, unsigned size)
+                               hwaddr addr, unsigned size)
 {
     GT64120State *s = opaque;
     PCIHostState *phb = PCI_HOST_BRIDGE(s);
diff --git a/hw/gumstix.c b/hw/gumstix.c
index 13a36ea5c..4103a88b8 100644
--- a/hw/gumstix.c
+++ b/hw/gumstix.c
@@ -45,10 +45,7 @@
 
 static const int sector_len = 128 * 1024;
 
-static void connex_init(ram_addr_t ram_size,
-                const char *boot_device,
-                const char *kernel_filename, const char *kernel_cmdline,
-                const char *initrd_filename, const char *cpu_model)
+static void connex_init(QEMUMachineInitArgs *args)
 {
     PXA2xxState *cpu;
     DriveInfo *dinfo;
@@ -84,11 +81,9 @@ static void connex_init(ram_addr_t ram_size,
                     qdev_get_gpio_in(cpu->gpio, 36));
 }
 
-static void verdex_init(ram_addr_t ram_size,
-                const char *boot_device,
-                const char *kernel_filename, const char *kernel_cmdline,
-                const char *initrd_filename, const char *cpu_model)
+static void verdex_init(QEMUMachineInitArgs *args)
 {
+    const char *cpu_model = args->cpu_model;
     PXA2xxState *cpu;
     DriveInfo *dinfo;
     int be;
diff --git a/hw/heathrow_pic.c b/hw/heathrow_pic.c
index 16f48d12e..b9ec8e7b4 100644
--- a/hw/heathrow_pic.c
+++ b/hw/heathrow_pic.c
@@ -63,7 +63,7 @@ static void heathrow_pic_update(HeathrowPICS *s)
     }
 }
 
-static void pic_write(void *opaque, target_phys_addr_t addr,
+static void pic_write(void *opaque, hwaddr addr,
                       uint64_t value, unsigned size)
 {
     HeathrowPICS *s = opaque;
@@ -91,7 +91,7 @@ static void pic_write(void *opaque, target_phys_addr_t addr,
     }
 }
 
-static uint64_t pic_read(void *opaque, target_phys_addr_t addr,
+static uint64_t pic_read(void *opaque, hwaddr addr,
                          unsigned size)
 {
     HeathrowPICS *s = opaque;
diff --git a/hw/highbank.c b/hw/highbank.c
index 11aa1312c..afbb00542 100644
--- a/hw/highbank.c
+++ b/hw/highbank.c
@@ -79,7 +79,7 @@ static void hb_reset_secondary(ARMCPU *cpu, const struct arm_boot_info *info)
 }
 
 #define NUM_REGS      0x200
-static void hb_regs_write(void *opaque, target_phys_addr_t offset,
+static void hb_regs_write(void *opaque, hwaddr offset,
                           uint64_t value, unsigned size)
 {
     uint32_t *regs = opaque;
@@ -95,7 +95,7 @@ static void hb_regs_write(void *opaque, target_phys_addr_t offset,
     regs[offset/4] = value;
 }
 
-static uint64_t hb_regs_read(void *opaque, target_phys_addr_t offset,
+static uint64_t hb_regs_read(void *opaque, hwaddr offset,
                              unsigned size)
 {
     uint32_t *regs = opaque;
@@ -187,11 +187,13 @@ static struct arm_boot_info highbank_binfo;
  * 32-bit host, set the reg value of memory to 0xf7ff00000 in the
  * device tree and pass -m 2047 to QEMU.
  */
-static void highbank_init(ram_addr_t ram_size,
-                     const char *boot_device,
-                     const char *kernel_filename, const char *kernel_cmdline,
-                     const char *initrd_filename, const char *cpu_model)
+static void highbank_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t ram_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
     DeviceState *dev;
     SysBusDevice *busdev;
     qemu_irq *irqp;
diff --git a/hw/hpet.c b/hw/hpet.c
index fd3ddca7f..50ac067ec 100644
--- a/hw/hpet.c
+++ b/hw/hpet.c
@@ -370,20 +370,20 @@ static void hpet_del_timer(HPETTimer *t)
 }
 
 #ifdef HPET_DEBUG
-static uint32_t hpet_ram_readb(void *opaque, target_phys_addr_t addr)
+static uint32_t hpet_ram_readb(void *opaque, hwaddr addr)
 {
     printf("qemu: hpet_read b at %" PRIx64 "\n", addr);
     return 0;
 }
 
-static uint32_t hpet_ram_readw(void *opaque, target_phys_addr_t addr)
+static uint32_t hpet_ram_readw(void *opaque, hwaddr addr)
 {
     printf("qemu: hpet_read w at %" PRIx64 "\n", addr);
     return 0;
 }
 #endif
 
-static uint64_t hpet_ram_read(void *opaque, target_phys_addr_t addr,
+static uint64_t hpet_ram_read(void *opaque, hwaddr addr,
                               unsigned size)
 {
     HPETState *s = opaque;
@@ -455,7 +455,7 @@ static uint64_t hpet_ram_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void hpet_ram_write(void *opaque, target_phys_addr_t addr,
+static void hpet_ram_write(void *opaque, hwaddr addr,
                            uint64_t value, unsigned size)
 {
     int i;
diff --git a/hw/hw.h b/hw/hw.h
index e5cb9bf94..f530f6f41 100644
--- a/hw/hw.h
+++ b/hw/hw.h
@@ -4,14 +4,16 @@
 
 #include "qemu-common.h"
 
-#if defined(TARGET_PHYS_ADDR_BITS) && !defined(NEED_CPU_H)
+#if !defined(CONFIG_USER_ONLY) && !defined(NEED_CPU_H)
 #include "cpu-common.h"
 #endif
 
 #include "ioport.h"
 #include "irq.h"
+#include "qemu-aio.h"
 #include "qemu-file.h"
 #include "vmstate.h"
+#include "qemu-log.h"
 
 #ifdef NEED_CPU_H
 #if TARGET_LONG_BITS == 64
diff --git a/hw/i386/Makefile.objs b/hw/i386/Makefile.objs
index 8c764bbfe..0d3f6a8e8 100644
--- a/hw/i386/Makefile.objs
+++ b/hw/i386/Makefile.objs
@@ -6,6 +6,7 @@ obj-y += pci-hotplug.o smbios.o wdt_ib700.o
 obj-y += debugcon.o multiboot.o
 obj-y += pc_piix.o
 obj-y += pc_sysfw.o
+obj-y += lpc_ich9.o q35.o pc_q35.o
 obj-$(CONFIG_XEN) += xen_platform.o xen_apic.o
 obj-$(CONFIG_XEN_PCI_PASSTHROUGH) += xen-host-pci-device.o
 obj-$(CONFIG_XEN_PCI_PASSTHROUGH) += xen_pt.o xen_pt_config_init.o xen_pt_msi.o
diff --git a/hw/i82378.c b/hw/i82378.c
index 2123c142a..99f35d41e 100644
--- a/hw/i82378.c
+++ b/hw/i82378.c
@@ -59,7 +59,7 @@ static const VMStateDescription vmstate_pci_i82378 = {
     },
 };
 
-static void i82378_io_write(void *opaque, target_phys_addr_t addr,
+static void i82378_io_write(void *opaque, hwaddr addr,
                             uint64_t value, unsigned int size)
 {
     switch (size) {
@@ -83,7 +83,7 @@ static void i82378_io_write(void *opaque, target_phys_addr_t addr,
     }
 }
 
-static uint64_t i82378_io_read(void *opaque, target_phys_addr_t addr,
+static uint64_t i82378_io_read(void *opaque, hwaddr addr,
                                unsigned int size)
 {
     DPRINTF("%s: " TARGET_FMT_plx "\n", __func__, addr);
@@ -105,7 +105,7 @@ static const MemoryRegionOps i82378_io_ops = {
     .endianness = DEVICE_LITTLE_ENDIAN,
 };
 
-static void i82378_mem_write(void *opaque, target_phys_addr_t addr,
+static void i82378_mem_write(void *opaque, hwaddr addr,
                              uint64_t value, unsigned int size)
 {
     switch (size) {
@@ -129,7 +129,7 @@ static void i82378_mem_write(void *opaque, target_phys_addr_t addr,
     }
 }
 
-static uint64_t i82378_mem_read(void *opaque, target_phys_addr_t addr,
+static uint64_t i82378_mem_read(void *opaque, hwaddr addr,
                                 unsigned int size)
 {
     DPRINTF("%s: " TARGET_FMT_plx "\n", __func__, addr);
diff --git a/hw/i8254.c b/hw/i8254.c
index 77bd5e822..bea5f92fd 100644
--- a/hw/i8254.c
+++ b/hw/i8254.c
@@ -111,7 +111,8 @@ static void pit_latch_count(PITChannelState *s)
     }
 }
 
-static void pit_ioport_write(void *opaque, uint32_t addr, uint32_t val)
+static void pit_ioport_write(void *opaque, hwaddr addr,
+                             uint64_t val, unsigned size)
 {
     PITCommonState *pit = opaque;
     int channel, access;
@@ -178,7 +179,8 @@ static void pit_ioport_write(void *opaque, uint32_t addr, uint32_t val)
     }
 }
 
-static uint32_t pit_ioport_read(void *opaque, uint32_t addr)
+static uint64_t pit_ioport_read(void *opaque, hwaddr addr,
+                                unsigned size)
 {
     PITCommonState *pit = opaque;
     int ret, count;
@@ -290,14 +292,14 @@ static void pit_irq_control(void *opaque, int n, int enable)
     }
 }
 
-static const MemoryRegionPortio pit_portio[] = {
-    { 0, 4, 1, .write = pit_ioport_write },
-    { 0, 3, 1, .read = pit_ioport_read },
-    PORTIO_END_OF_LIST()
-};
-
 static const MemoryRegionOps pit_ioport_ops = {
-    .old_portio = pit_portio
+    .read = pit_ioport_read,
+    .write = pit_ioport_write,
+    .impl = {
+        .min_access_size = 1,
+        .max_access_size = 1,
+    },
+    .endianness = DEVICE_LITTLE_ENDIAN,
 };
 
 static void pit_post_load(PITCommonState *s)
diff --git a/hw/i8259.c b/hw/i8259.c
index 53daf7865..af0ba4d7c 100644
--- a/hw/i8259.c
+++ b/hw/i8259.c
@@ -235,7 +235,7 @@ static void pic_reset(DeviceState *dev)
     pic_init_reset(s);
 }
 
-static void pic_ioport_write(void *opaque, target_phys_addr_t addr64,
+static void pic_ioport_write(void *opaque, hwaddr addr64,
                              uint64_t val64, unsigned size)
 {
     PICCommonState *s = opaque;
@@ -329,7 +329,7 @@ static void pic_ioport_write(void *opaque, target_phys_addr_t addr64,
     }
 }
 
-static uint64_t pic_ioport_read(void *opaque, target_phys_addr_t addr,
+static uint64_t pic_ioport_read(void *opaque, hwaddr addr,
                                 unsigned size)
 {
     PICCommonState *s = opaque;
@@ -366,14 +366,14 @@ int pic_get_output(DeviceState *d)
     return (pic_get_irq(s) >= 0);
 }
 
-static void elcr_ioport_write(void *opaque, target_phys_addr_t addr,
+static void elcr_ioport_write(void *opaque, hwaddr addr,
                               uint64_t val, unsigned size)
 {
     PICCommonState *s = opaque;
     s->elcr = val & s->elcr_mask;
 }
 
-static uint64_t elcr_ioport_read(void *opaque, target_phys_addr_t addr,
+static uint64_t elcr_ioport_read(void *opaque, hwaddr addr,
                                  unsigned size)
 {
     PICCommonState *s = opaque;
diff --git a/hw/i8259_internal.h b/hw/i8259_internal.h
index 4137b6170..8785b1da3 100644
--- a/hw/i8259_internal.h
+++ b/hw/i8259_internal.h
@@ -33,7 +33,7 @@ typedef struct PICCommonState PICCommonState;
 
 #define TYPE_PIC_COMMON "pic-common"
 #define PIC_COMMON(obj) \
-     OBJECT_CHECK(PICCommon, (obj), TYPE_PIC_COMMON)
+     OBJECT_CHECK(PICCommonState, (obj), TYPE_PIC_COMMON)
 #define PIC_COMMON_CLASS(klass) \
      OBJECT_CLASS_CHECK(PICCommonClass, (klass), TYPE_PIC_COMMON)
 #define PIC_COMMON_GET_CLASS(obj) \
diff --git a/hw/i82801b11.c b/hw/i82801b11.c
new file mode 100644
index 000000000..3d1f996b2
--- /dev/null
+++ b/hw/i82801b11.c
@@ -0,0 +1,125 @@
+/*
+ * Copyright (c) 2006 Fabrice Bellard
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+/*
+ * QEMU i82801b11 dmi-to-pci Bridge Emulation
+ *
+ *  Copyright (c) 2009, 2010, 2011
+ *                Isaku Yamahata <yamahata at valinux co jp>
+ *                VA Linux Systems Japan K.K.
+ *  Copyright (C) 2012 Jason Baron <jbaron@redhat.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>
+ */
+
+#include "pci.h"
+#include "ich9.h"
+
+
+/*****************************************************************************/
+/* ICH9 DMI-to-PCI bridge */
+#define I82801ba_SSVID_OFFSET   0x50
+#define I82801ba_SSVID_SVID     0
+#define I82801ba_SSVID_SSID     0
+
+typedef struct I82801b11Bridge {
+    PCIBridge br;
+} I82801b11Bridge;
+
+static int i82801b11_bridge_initfn(PCIDevice *d)
+{
+    int rc;
+
+    rc = pci_bridge_initfn(d);
+    if (rc < 0) {
+        return rc;
+    }
+
+    rc = pci_bridge_ssvid_init(d, I82801ba_SSVID_OFFSET,
+                               I82801ba_SSVID_SVID, I82801ba_SSVID_SSID);
+    if (rc < 0) {
+        goto err_bridge;
+    }
+    pci_config_set_prog_interface(d->config, PCI_CLASS_BRDIGE_PCI_INF_SUB);
+    return 0;
+
+err_bridge:
+    pci_bridge_exitfn(d);
+
+    return rc;
+}
+
+static void i82801b11_bridge_class_init(ObjectClass *klass, void *data)
+{
+    PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
+
+    k->is_bridge = 1;
+    k->vendor_id = PCI_VENDOR_ID_INTEL;
+    k->device_id = PCI_DEVICE_ID_INTEL_82801BA_11;
+    k->revision = ICH9_D2P_A2_REVISION;
+    k->init = i82801b11_bridge_initfn;
+}
+
+static const TypeInfo i82801b11_bridge_info = {
+    .name          = "i82801b11-bridge",
+    .parent        = TYPE_PCI_DEVICE,
+    .instance_size = sizeof(I82801b11Bridge),
+    .class_init    = i82801b11_bridge_class_init,
+};
+
+PCIBus *ich9_d2pbr_init(PCIBus *bus, int devfn, int sec_bus)
+{
+    PCIDevice *d;
+    PCIBridge *br;
+    char buf[16];
+    DeviceState *qdev;
+
+    d = pci_create_multifunction(bus, devfn, true, "i82801b11-bridge");
+    if (!d) {
+        return NULL;
+    }
+    br = DO_UPCAST(PCIBridge, dev, d);
+    qdev = &br->dev.qdev;
+
+    snprintf(buf, sizeof(buf), "pci.%d", sec_bus);
+    pci_bridge_map_irq(br, buf, pci_swizzle_map_irq_fn);
+    qdev_init_nofail(qdev);
+
+    return pci_bridge_get_sec_bus(br);
+}
+
+static void d2pbr_register(void)
+{
+    type_register_static(&i82801b11_bridge_info);
+}
+
+type_init(d2pbr_register);
diff --git a/hw/ich9.h b/hw/ich9.h
new file mode 100644
index 000000000..de491350c
--- /dev/null
+++ b/hw/ich9.h
@@ -0,0 +1,207 @@
+#ifndef HW_ICH9_H
+#define HW_ICH9_H
+
+#include "hw.h"
+#include "range.h"
+#include "isa.h"
+#include "sysbus.h"
+#include "pc.h"
+#include "apm.h"
+#include "ioapic.h"
+#include "pci.h"
+#include "pcie_host.h"
+#include "pci_bridge.h"
+#include "acpi.h"
+#include "acpi_ich9.h"
+#include "pam.h"
+#include "pci_internals.h"
+
+void ich9_lpc_set_irq(void *opaque, int irq_num, int level);
+int ich9_lpc_map_irq(PCIDevice *pci_dev, int intx);
+void ich9_lpc_pm_init(PCIDevice *pci_lpc, qemu_irq cmos_s3);
+PCIBus *ich9_d2pbr_init(PCIBus *bus, int devfn, int sec_bus);
+i2c_bus *ich9_smb_init(PCIBus *bus, int devfn, uint32_t smb_io_base);
+
+#define ICH9_CC_SIZE                            (16 * 1024)     /* 16KB */
+
+#define TYPE_ICH9_LPC_DEVICE "ICH9 LPC"
+#define ICH9_LPC_DEVICE(obj) \
+     OBJECT_CHECK(ICH9LPCState, (obj), TYPE_ICH9_LPC_DEVICE)
+
+typedef struct ICH9LPCState {
+    /* ICH9 LPC PCI to ISA bridge */
+    PCIDevice d;
+
+    /* (pci device, intx) -> pirq
+     * In real chipset case, the unused slots are never used
+     * as ICH9 supports only D25-D32 irq routing.
+     * On the other hand in qemu case, any slot/function can be populated
+     * via command line option.
+     * So fallback interrupt routing for any devices in any slots is necessary.
+    */
+    uint8_t irr[PCI_SLOT_MAX][PCI_NUM_PINS];
+
+    APMState apm;
+    ICH9LPCPMRegs pm;
+    uint32_t sci_level; /* track sci level */
+
+    /* 10.1 Chipset Configuration registers(Memory Space)
+     which is pointed by RCBA */
+    uint8_t chip_config[ICH9_CC_SIZE];
+    /* isa bus */
+    ISABus *isa_bus;
+    MemoryRegion rbca_mem;
+
+    qemu_irq *pic;
+    qemu_irq *ioapic;
+} ICH9LPCState;
+
+#define Q35_MASK(bit, ms_bit, ls_bit) \
+((uint##bit##_t)(((1ULL << ((ms_bit) + 1)) - 1) & ~((1ULL << ls_bit) - 1)))
+
+/* ICH9: Chipset Configuration Registers */
+#define ICH9_CC_ADDR_MASK                       (ICH9_CC_SIZE - 1)
+
+#define ICH9_CC
+#define ICH9_CC_D28IP                           0x310C
+#define ICH9_CC_D28IP_SHIFT                     4
+#define ICH9_CC_D28IP_MASK                      0xf
+#define ICH9_CC_D28IP_DEFAULT                   0x00214321
+#define ICH9_CC_D31IR                           0x3140
+#define ICH9_CC_D30IR                           0x3142
+#define ICH9_CC_D29IR                           0x3144
+#define ICH9_CC_D28IR                           0x3146
+#define ICH9_CC_D27IR                           0x3148
+#define ICH9_CC_D26IR                           0x314C
+#define ICH9_CC_D25IR                           0x3150
+#define ICH9_CC_DIR_DEFAULT                     0x3210
+#define ICH9_CC_D30IR_DEFAULT                   0x0
+#define ICH9_CC_DIR_SHIFT                       4
+#define ICH9_CC_DIR_MASK                        0x7
+#define ICH9_CC_OIC                             0x31FF
+#define ICH9_CC_OIC_AEN                         0x1
+
+/* D28:F[0-5] */
+#define ICH9_PCIE_DEV                           28
+#define ICH9_PCIE_FUNC_MAX                      6
+
+
+/* D29:F0 USB UHCI Controller #1 */
+#define ICH9_USB_UHCI1_DEV                      29
+#define ICH9_USB_UHCI1_FUNC                     0
+
+/* D30:F0 DMI-to-PCI brdige */
+#define ICH9_D2P_BRIDGE                         "ICH9 D2P BRIDGE"
+#define ICH9_D2P_BRIDGE_SAVEVM_VERSION          0
+
+#define ICH9_D2P_BRIDGE_DEV                     30
+#define ICH9_D2P_BRIDGE_FUNC                    0
+
+#define ICH9_D2P_SECONDARY_DEFAULT              (256 - 8)
+
+#define ICH9_D2P_A2_REVISION                    0x92
+
+
+/* D31:F1 LPC controller */
+#define ICH9_A2_LPC                             "ICH9 A2 LPC"
+#define ICH9_A2_LPC_SAVEVM_VERSION              0
+
+#define ICH9_LPC_DEV                            31
+#define ICH9_LPC_FUNC                           0
+
+#define ICH9_A2_LPC_REVISION                    0x2
+#define ICH9_LPC_NB_PIRQS                       8       /* PCI A-H */
+
+#define ICH9_LPC_PMBASE                         0x40
+#define ICH9_LPC_PMBASE_BASE_ADDRESS_MASK       Q35_MASK(32, 15, 7)
+#define ICH9_LPC_PMBASE_RTE                     0x1
+#define ICH9_LPC_PMBASE_DEFAULT                 0x1
+#define ICH9_LPC_ACPI_CTRL                      0x44
+#define ICH9_LPC_ACPI_CTRL_ACPI_EN              0x80
+#define ICH9_LPC_ACPI_CTRL_SCI_IRQ_SEL_MASK     Q35_MASK(8, 2, 0)
+#define ICH9_LPC_ACPI_CTRL_9                    0x0
+#define ICH9_LPC_ACPI_CTRL_10                   0x1
+#define ICH9_LPC_ACPI_CTRL_11                   0x2
+#define ICH9_LPC_ACPI_CTRL_20                   0x4
+#define ICH9_LPC_ACPI_CTRL_21                   0x5
+#define ICH9_LPC_ACPI_CTRL_DEFAULT              0x0
+
+#define ICH9_LPC_PIRQA_ROUT                     0x60
+#define ICH9_LPC_PIRQB_ROUT                     0x61
+#define ICH9_LPC_PIRQC_ROUT                     0x62
+#define ICH9_LPC_PIRQD_ROUT                     0x63
+
+#define ICH9_LPC_PIRQE_ROUT                     0x68
+#define ICH9_LPC_PIRQF_ROUT                     0x69
+#define ICH9_LPC_PIRQG_ROUT                     0x6a
+#define ICH9_LPC_PIRQH_ROUT                     0x6b
+
+#define ICH9_LPC_PIRQ_ROUT_IRQEN                0x80
+#define ICH9_LPC_PIRQ_ROUT_MASK                 Q35_MASK(8, 3, 0)
+#define ICH9_LPC_PIRQ_ROUT_DEFAULT              0x80
+
+#define ICH9_LPC_RCBA                           0xf0
+#define ICH9_LPC_RCBA_BA_MASK                   Q35_MASK(32, 31, 14)
+#define ICH9_LPC_RCBA_EN                        0x1
+#define ICH9_LPC_RCBA_DEFAULT                   0x0
+
+#define ICH9_LPC_PIC_NUM_PINS                   16
+#define ICH9_LPC_IOAPIC_NUM_PINS                24
+
+/* D31:F2 SATA Controller #1 */
+#define ICH9_SATA1_DEV                          31
+#define ICH9_SATA1_FUNC                         2
+
+/* D30:F1 power management I/O registers
+   offset from the address ICH9_LPC_PMBASE */
+
+/* ICH9 LPC PM I/O registers are 128 ports and 128-aligned */
+#define ICH9_PMIO_SIZE                          128
+#define ICH9_PMIO_MASK                          (ICH9_PMIO_SIZE - 1)
+
+#define ICH9_PMIO_PM1_STS                       0x00
+#define ICH9_PMIO_PM1_EN                        0x02
+#define ICH9_PMIO_PM1_CNT                       0x04
+#define ICH9_PMIO_PM1_TMR                       0x08
+#define ICH9_PMIO_GPE0_STS                      0x20
+#define ICH9_PMIO_GPE0_EN                       0x28
+#define ICH9_PMIO_GPE0_LEN                      16
+#define ICH9_PMIO_SMI_EN                        0x30
+#define ICH9_PMIO_SMI_EN_APMC_EN                (1 << 5)
+#define ICH9_PMIO_SMI_STS                       0x34
+
+/* FADT ACPI_ENABLE/ACPI_DISABLE */
+#define ICH9_APM_ACPI_ENABLE                    0x2
+#define ICH9_APM_ACPI_DISABLE                   0x3
+
+
+/* D31:F3 SMBus controller */
+#define ICH9_A2_SMB_REVISION                    0x02
+#define ICH9_SMB_PI                             0x00
+
+#define ICH9_SMB_SMBMBAR0                       0x10
+#define ICH9_SMB_SMBMBAR1                       0x14
+#define ICH9_SMB_SMBM_BAR                       0
+#define ICH9_SMB_SMBM_SIZE                      (1 << 8)
+#define ICH9_SMB_SMB_BASE                       0x20
+#define ICH9_SMB_SMB_BASE_BAR                   4
+#define ICH9_SMB_SMB_BASE_SIZE                  (1 << 5)
+#define ICH9_SMB_HOSTC                          0x40
+#define ICH9_SMB_HOSTC_SSRESET                  ((uint8_t)(1 << 3))
+#define ICH9_SMB_HOSTC_I2C_EN                   ((uint8_t)(1 << 2))
+#define ICH9_SMB_HOSTC_SMB_SMI_EN               ((uint8_t)(1 << 1))
+#define ICH9_SMB_HOSTC_HST_EN                   ((uint8_t)(1 << 0))
+
+/* D31:F3 SMBus I/O and memory mapped I/O registers */
+#define ICH9_SMB_DEV                            31
+#define ICH9_SMB_FUNC                           3
+
+#define ICH9_SMB_HST_STS                        0x00
+#define ICH9_SMB_HST_CNT                        0x02
+#define ICH9_SMB_HST_CMD                        0x03
+#define ICH9_SMB_XMIT_SLVA                      0x04
+#define ICH9_SMB_HST_D0                         0x05
+#define ICH9_SMB_HST_D1                         0x06
+#define ICH9_SMB_HOST_BLOCK_DB                  0x07
+
+#endif /* HW_ICH9_H */
diff --git a/hw/ide.h b/hw/ide.h
index 2db4079f6..add742c4a 100644
--- a/hw/ide.h
+++ b/hw/ide.h
@@ -24,7 +24,7 @@ MemoryRegion *pmac_ide_init (DriveInfo **hd_table, qemu_irq irq,
 		   void *dbdma, int channel, qemu_irq dma_irq);
 
 /* ide-mmio.c */
-void mmio_ide_init (target_phys_addr_t membase, target_phys_addr_t membase2,
+void mmio_ide_init (hwaddr membase, hwaddr membase2,
                     MemoryRegion *address_space,
                     qemu_irq irq, int shift,
                     DriveInfo *hd0, DriveInfo *hd1);
diff --git a/hw/ide/Makefile.objs b/hw/ide/Makefile.objs
index cf718dd01..5c8c22aad 100644
--- a/hw/ide/Makefile.objs
+++ b/hw/ide/Makefile.objs
@@ -1,10 +1,10 @@
-hw-obj-$(CONFIG_IDE_CORE) += core.o atapi.o
-hw-obj-$(CONFIG_IDE_QDEV) += qdev.o
-hw-obj-$(CONFIG_IDE_PCI) += pci.o
-hw-obj-$(CONFIG_IDE_ISA) += isa.o
-hw-obj-$(CONFIG_IDE_PIIX) += piix.o
-hw-obj-$(CONFIG_IDE_CMD646) += cmd646.o
-hw-obj-$(CONFIG_IDE_MACIO) += macio.o
-hw-obj-$(CONFIG_IDE_VIA) += via.o
-hw-obj-$(CONFIG_AHCI) += ahci.o
-hw-obj-$(CONFIG_AHCI) += ich.o
+common-obj-$(CONFIG_IDE_CORE) += core.o atapi.o
+common-obj-$(CONFIG_IDE_QDEV) += qdev.o
+common-obj-$(CONFIG_IDE_PCI) += pci.o
+common-obj-$(CONFIG_IDE_ISA) += isa.o
+common-obj-$(CONFIG_IDE_PIIX) += piix.o
+common-obj-$(CONFIG_IDE_CMD646) += cmd646.o
+common-obj-$(CONFIG_IDE_MACIO) += macio.o
+common-obj-$(CONFIG_IDE_VIA) += via.o
+common-obj-$(CONFIG_AHCI) += ahci.o
+common-obj-$(CONFIG_AHCI) += ich.o
diff --git a/hw/ide/ahci.c b/hw/ide/ahci.c
index 5ea3cadb0..67562db04 100644
--- a/hw/ide/ahci.c
+++ b/hw/ide/ahci.c
@@ -174,7 +174,7 @@ static void ahci_trigger_irq(AHCIState *s, AHCIDevice *d,
 
 static void map_page(uint8_t **ptr, uint64_t addr, uint32_t wanted)
 {
-    target_phys_addr_t len = wanted;
+    hwaddr len = wanted;
 
     if (*ptr) {
         cpu_physical_memory_unmap(*ptr, len, 1, len);
@@ -279,7 +279,7 @@ static void  ahci_port_write(AHCIState *s, int port, int offset, uint32_t val)
     }
 }
 
-static uint64_t ahci_mem_read(void *opaque, target_phys_addr_t addr,
+static uint64_t ahci_mem_read(void *opaque, hwaddr addr,
                               unsigned size)
 {
     AHCIState *s = opaque;
@@ -317,7 +317,7 @@ static uint64_t ahci_mem_read(void *opaque, target_phys_addr_t addr,
 
 
 
-static void ahci_mem_write(void *opaque, target_phys_addr_t addr,
+static void ahci_mem_write(void *opaque, hwaddr addr,
                            uint64_t val, unsigned size)
 {
     AHCIState *s = opaque;
@@ -373,7 +373,7 @@ static const MemoryRegionOps ahci_mem_ops = {
     .endianness = DEVICE_LITTLE_ENDIAN,
 };
 
-static uint64_t ahci_idp_read(void *opaque, target_phys_addr_t addr,
+static uint64_t ahci_idp_read(void *opaque, hwaddr addr,
                               unsigned size)
 {
     AHCIState *s = opaque;
@@ -389,7 +389,7 @@ static uint64_t ahci_idp_read(void *opaque, target_phys_addr_t addr,
     }
 }
 
-static void ahci_idp_write(void *opaque, target_phys_addr_t addr,
+static void ahci_idp_write(void *opaque, hwaddr addr,
                            uint64_t val, unsigned size)
 {
     AHCIState *s = opaque;
@@ -1175,7 +1175,6 @@ void ahci_init(AHCIState *s, DeviceState *qdev, DMAContext *dma, int ports)
         ad->port_no = i;
         ad->port.dma = &ad->dma;
         ad->port.dma->ops = &ahci_dma_ops;
-        ad->port_regs.cmd = PORT_CMD_SPIN_UP | PORT_CMD_POWER_ON;
     }
 }
 
@@ -1199,6 +1198,7 @@ void ahci_reset(AHCIState *s)
         pr->irq_stat = 0;
         pr->irq_mask = 0;
         pr->scr_ctl = 0;
+        pr->cmd = PORT_CMD_SPIN_UP | PORT_CMD_POWER_ON;
         ahci_reset_port(s, i);
     }
 }
diff --git a/hw/ide/atapi.c b/hw/ide/atapi.c
index f7f714c72..861fd2bec 100644
--- a/hw/ide/atapi.c
+++ b/hw/ide/atapi.c
@@ -875,6 +875,12 @@ static void cmd_start_stop_unit(IDEState *s, uint8_t* buf)
     int sense;
     bool start = buf[4] & 1;
     bool loej = buf[4] & 2;     /* load on start, eject on !start */
+    int pwrcnd = buf[4] & 0xf0;
+
+    if (pwrcnd) {
+        /* eject/load only happens for power condition == 0 */
+        return;
+    }
 
     if (loej) {
         if (!start && !s->tray_open && s->tray_locked) {
@@ -1118,12 +1124,17 @@ void ide_atapi_cmd(IDEState *s)
      * GET_EVENT_STATUS_NOTIFICATION to detect such tray open/close
      * states rely on this behavior.
      */
-    if (!s->tray_open && bdrv_is_inserted(s->bs) && s->cdrom_changed) {
-        ide_atapi_cmd_error(s, NOT_READY, ASC_MEDIUM_NOT_PRESENT);
+    if (!(atapi_cmd_table[s->io_buffer[0]].flags & ALLOW_UA) &&
+        !s->tray_open && bdrv_is_inserted(s->bs) && s->cdrom_changed) {
+
+        if (s->cdrom_changed == 1) {
+            ide_atapi_cmd_error(s, NOT_READY, ASC_MEDIUM_NOT_PRESENT);
+            s->cdrom_changed = 2;
+        } else {
+            ide_atapi_cmd_error(s, UNIT_ATTENTION, ASC_MEDIUM_MAY_HAVE_CHANGED);
+            s->cdrom_changed = 0;
+        }
 
-        s->cdrom_changed = 0;
-        s->sense_key = UNIT_ATTENTION;
-        s->asc = ASC_MEDIUM_MAY_HAVE_CHANGED;
         return;
     }
 
diff --git a/hw/ide/cmd646.c b/hw/ide/cmd646.c
index e0b944349..804db60ff 100644
--- a/hw/ide/cmd646.c
+++ b/hw/ide/cmd646.c
@@ -43,7 +43,7 @@
 
 static void cmd646_update_irq(PCIIDEState *d);
 
-static uint64_t cmd646_cmd_read(void *opaque, target_phys_addr_t addr,
+static uint64_t cmd646_cmd_read(void *opaque, hwaddr addr,
                                 unsigned size)
 {
     CMD646BAR *cmd646bar = opaque;
@@ -54,7 +54,7 @@ static uint64_t cmd646_cmd_read(void *opaque, target_phys_addr_t addr,
     return ide_status_read(cmd646bar->bus, addr + 2);
 }
 
-static void cmd646_cmd_write(void *opaque, target_phys_addr_t addr,
+static void cmd646_cmd_write(void *opaque, hwaddr addr,
                              uint64_t data, unsigned size)
 {
     CMD646BAR *cmd646bar = opaque;
@@ -71,7 +71,7 @@ static const MemoryRegionOps cmd646_cmd_ops = {
     .endianness = DEVICE_LITTLE_ENDIAN,
 };
 
-static uint64_t cmd646_data_read(void *opaque, target_phys_addr_t addr,
+static uint64_t cmd646_data_read(void *opaque, hwaddr addr,
                                  unsigned size)
 {
     CMD646BAR *cmd646bar = opaque;
@@ -88,7 +88,7 @@ static uint64_t cmd646_data_read(void *opaque, target_phys_addr_t addr,
     return ((uint64_t)1 << (size * 8)) - 1;
 }
 
-static void cmd646_data_write(void *opaque, target_phys_addr_t addr,
+static void cmd646_data_write(void *opaque, hwaddr addr,
                              uint64_t data, unsigned size)
 {
     CMD646BAR *cmd646bar = opaque;
@@ -121,7 +121,7 @@ static void setup_cmd646_bar(PCIIDEState *d, int bus_num)
     memory_region_init_io(&bar->data, &cmd646_data_ops, bar, "cmd646-data", 8);
 }
 
-static uint64_t bmdma_read(void *opaque, target_phys_addr_t addr,
+static uint64_t bmdma_read(void *opaque, hwaddr addr,
                            unsigned size)
 {
     BMDMAState *bm = opaque;
@@ -159,7 +159,7 @@ static uint64_t bmdma_read(void *opaque, target_phys_addr_t addr,
     return val;
 }
 
-static void bmdma_write(void *opaque, target_phys_addr_t addr,
+static void bmdma_write(void *opaque, hwaddr addr,
                         uint64_t val, unsigned size)
 {
     BMDMAState *bm = opaque;
diff --git a/hw/ide/core.c b/hw/ide/core.c
index d65ef3d58..c4f93d0e4 100644
--- a/hw/ide/core.c
+++ b/hw/ide/core.c
@@ -53,8 +53,6 @@ static const int smart_attributes[][12] = {
     { 0x0c, 0x03, 0x00, 0x64, 0x64, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00},
     /* airflow-temperature-celsius */
     { 190,  0x03, 0x00, 0x45, 0x45, 0x1f, 0x00, 0x1f, 0x1f, 0x00, 0x00, 0x32},
-    /* end of list */
-    { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}
 };
 
 static int ide_handle_rw_error(IDEState *s, int error, int op);
@@ -338,7 +336,7 @@ static void trim_aio_cancel(BlockDriverAIOCB *acb)
     qemu_aio_release(iocb);
 }
 
-static AIOPool trim_aio_pool = {
+static const AIOCBInfo trim_aiocb_info = {
     .aiocb_size         = sizeof(TrimAIOCB),
     .cancel             = trim_aio_cancel,
 };
@@ -362,7 +360,7 @@ BlockDriverAIOCB *ide_issue_trim(BlockDriverState *bs,
     TrimAIOCB *iocb;
     int i, j, ret;
 
-    iocb = qemu_aio_get(&trim_aio_pool, bs, cb, opaque);
+    iocb = qemu_aio_get(&trim_aiocb_info, bs, cb, opaque);
     iocb->bh = qemu_bh_new(ide_trim_bh_cb, iocb);
     iocb->ret = 0;
 
@@ -558,32 +556,22 @@ void ide_dma_error(IDEState *s)
 
 static int ide_handle_rw_error(IDEState *s, int error, int op)
 {
-    int is_read = (op & BM_STATUS_RETRY_READ);
-    BlockErrorAction action = bdrv_get_on_error(s->bs, is_read);
+    bool is_read = (op & BM_STATUS_RETRY_READ) != 0;
+    BlockErrorAction action = bdrv_get_error_action(s->bs, is_read, error);
 
-    if (action == BLOCK_ERR_IGNORE) {
-        bdrv_emit_qmp_error_event(s->bs, BDRV_ACTION_IGNORE, is_read);
-        return 0;
-    }
-
-    if ((error == ENOSPC && action == BLOCK_ERR_STOP_ENOSPC)
-            || action == BLOCK_ERR_STOP_ANY) {
+    if (action == BDRV_ACTION_STOP) {
         s->bus->dma->ops->set_unit(s->bus->dma, s->unit);
         s->bus->error_status = op;
-        bdrv_emit_qmp_error_event(s->bs, BDRV_ACTION_STOP, is_read);
-        vm_stop(RUN_STATE_IO_ERROR);
-        bdrv_iostatus_set_err(s->bs, error);
-    } else {
+    } else if (action == BDRV_ACTION_REPORT) {
         if (op & BM_STATUS_DMA_RETRY) {
             dma_buf_commit(s);
             ide_dma_error(s);
         } else {
             ide_rw_error(s);
         }
-        bdrv_emit_qmp_error_event(s->bs, BDRV_ACTION_REPORT, is_read);
     }
-
-    return 1;
+    bdrv_error_action(s->bs, action, is_read, error);
+    return action != BDRV_ACTION_IGNORE;
 }
 
 void ide_dma_cb(void *opaque, int ret)
@@ -591,6 +579,7 @@ void ide_dma_cb(void *opaque, int ret)
     IDEState *s = opaque;
     int n;
     int64_t sector_num;
+    bool stay_active = false;
 
     if (ret < 0) {
         int op = BM_STATUS_DMA_RETRY;
@@ -606,6 +595,14 @@ void ide_dma_cb(void *opaque, int ret)
     }
 
     n = s->io_buffer_size >> 9;
+    if (n > s->nsector) {
+        /* The PRDs were longer than needed for this request. Shorten them so
+         * we don't get a negative remainder. The Active bit must remain set
+         * after the request completes. */
+        n = s->nsector;
+        stay_active = true;
+    }
+
     sector_num = ide_get_sector(s);
     if (n > 0) {
         dma_buf_commit(s);
@@ -628,6 +625,7 @@ void ide_dma_cb(void *opaque, int ret)
     if (s->bus->dma->ops->prepare_buf(s->bus->dma, ide_cmd_is_read(s)) == 0) {
         /* The PRDs were too short. Reset the Active bit, but don't raise an
          * interrupt. */
+        s->status = READY_STAT | SEEK_STAT;
         goto eot;
     }
 
@@ -658,6 +656,9 @@ eot:
         bdrv_acct_done(s->bs, &s->acct);
     }
     ide_set_inactive(s);
+    if (stay_active) {
+        s->bus->dma->ops->add_status(s->bus->dma, BM_STATUS_DMAING);
+    }
 }
 
 static void ide_sector_start_dma(IDEState *s, enum ide_dma_cmd dma_cmd)
@@ -1468,9 +1469,7 @@ void ide_exec_cmd(IDEBus *bus, uint32_t val)
 	case SMART_READ_THRESH:
 		memset(s->io_buffer, 0, 0x200);
 		s->io_buffer[0] = 0x01; /* smart struct version */
-		for (n=0; n<30; n++) {
-		if (smart_attributes[n][0] == 0)
-			break;
+		for (n = 0; n < ARRAY_SIZE(smart_attributes); n++) {
 		s->io_buffer[2+0+(n*12)] = smart_attributes[n][0];
 		s->io_buffer[2+1+(n*12)] = smart_attributes[n][11];
 		}
@@ -1484,10 +1483,7 @@ void ide_exec_cmd(IDEBus *bus, uint32_t val)
 	case SMART_READ_DATA:
 		memset(s->io_buffer, 0, 0x200);
 		s->io_buffer[0] = 0x01; /* smart struct version */
-		for (n=0; n<30; n++) {
-		    if (smart_attributes[n][0] == 0) {
-			break;
-		    }
+		for (n = 0; n < ARRAY_SIZE(smart_attributes); n++) {
 		    int i;
 		    for(i = 0; i < 11; i++) {
 			s->io_buffer[2+i+(n*12)] = smart_attributes[n][i];
@@ -2164,12 +2160,6 @@ static int ide_drive_post_load(void *opaque, int version_id)
 {
     IDEState *s = opaque;
 
-    if (version_id < 3) {
-        if (s->sense_key == UNIT_ATTENTION &&
-            s->asc == ASC_MEDIUM_MAY_HAVE_CHANGED) {
-            s->cdrom_changed = 1;
-        }
-    }
     if (s->identify_set) {
         bdrv_set_enable_write_cache(s->bs, !!(s->identify_data[85] & (1 << 5)));
     }
diff --git a/hw/ide/macio.c b/hw/ide/macio.c
index 848cb3142..d2edcc085 100644
--- a/hw/ide/macio.c
+++ b/hw/ide/macio.c
@@ -76,7 +76,8 @@ static void pmac_ide_atapi_transfer_cb(void *opaque, int ret)
 
     s->io_buffer_size = io->len;
 
-    qemu_sglist_init(&s->sg, io->len / MACIO_PAGE_SIZE + 1, NULL);
+    qemu_sglist_init(&s->sg, io->len / MACIO_PAGE_SIZE + 1,
+                     &dma_context_memory);
     qemu_sglist_add(&s->sg, io->addr, io->len);
     io->addr += io->len;
     io->len = 0;
@@ -89,7 +90,6 @@ static void pmac_ide_atapi_transfer_cb(void *opaque, int ret)
 done:
     bdrv_acct_done(s->bs, &s->acct);
     io->dma_end(opaque);
-    return;
 }
 
 static void pmac_ide_transfer_cb(void *opaque, int ret)
@@ -133,7 +133,8 @@ static void pmac_ide_transfer_cb(void *opaque, int ret)
     s->io_buffer_index = 0;
     s->io_buffer_size = io->len;
 
-    qemu_sglist_init(&s->sg, io->len / MACIO_PAGE_SIZE + 1, NULL);
+    qemu_sglist_init(&s->sg, io->len / MACIO_PAGE_SIZE + 1,
+                     &dma_context_memory);
     qemu_sglist_add(&s->sg, io->addr, io->len);
     io->addr += io->len;
     io->len = 0;
@@ -199,7 +200,7 @@ static void pmac_ide_flush(DBDMA_io *io)
 
 /* PowerMac IDE memory IO */
 static void pmac_ide_writeb (void *opaque,
-                             target_phys_addr_t addr, uint32_t val)
+                             hwaddr addr, uint32_t val)
 {
     MACIOIDEState *d = opaque;
 
@@ -217,7 +218,7 @@ static void pmac_ide_writeb (void *opaque,
     }
 }
 
-static uint32_t pmac_ide_readb (void *opaque,target_phys_addr_t addr)
+static uint32_t pmac_ide_readb (void *opaque,hwaddr addr)
 {
     uint8_t retval;
     MACIOIDEState *d = opaque;
@@ -239,7 +240,7 @@ static uint32_t pmac_ide_readb (void *opaque,target_phys_addr_t addr)
 }
 
 static void pmac_ide_writew (void *opaque,
-                             target_phys_addr_t addr, uint32_t val)
+                             hwaddr addr, uint32_t val)
 {
     MACIOIDEState *d = opaque;
 
@@ -250,7 +251,7 @@ static void pmac_ide_writew (void *opaque,
     }
 }
 
-static uint32_t pmac_ide_readw (void *opaque,target_phys_addr_t addr)
+static uint32_t pmac_ide_readw (void *opaque,hwaddr addr)
 {
     uint16_t retval;
     MACIOIDEState *d = opaque;
@@ -266,7 +267,7 @@ static uint32_t pmac_ide_readw (void *opaque,target_phys_addr_t addr)
 }
 
 static void pmac_ide_writel (void *opaque,
-                             target_phys_addr_t addr, uint32_t val)
+                             hwaddr addr, uint32_t val)
 {
     MACIOIDEState *d = opaque;
 
@@ -277,7 +278,7 @@ static void pmac_ide_writel (void *opaque,
     }
 }
 
-static uint32_t pmac_ide_readl (void *opaque,target_phys_addr_t addr)
+static uint32_t pmac_ide_readl (void *opaque,hwaddr addr)
 {
     uint32_t retval;
     MACIOIDEState *d = opaque;
diff --git a/hw/ide/mmio.c b/hw/ide/mmio.c
index fcfb09eea..bcb26c8b6 100644
--- a/hw/ide/mmio.c
+++ b/hw/ide/mmio.c
@@ -47,7 +47,7 @@ static void mmio_ide_reset(void *opaque)
     ide_bus_reset(&s->bus);
 }
 
-static uint64_t mmio_ide_read(void *opaque, target_phys_addr_t addr,
+static uint64_t mmio_ide_read(void *opaque, hwaddr addr,
                               unsigned size)
 {
     MMIOState *s = opaque;
@@ -58,7 +58,7 @@ static uint64_t mmio_ide_read(void *opaque, target_phys_addr_t addr,
         return ide_data_readw(&s->bus, 0);
 }
 
-static void mmio_ide_write(void *opaque, target_phys_addr_t addr,
+static void mmio_ide_write(void *opaque, hwaddr addr,
                            uint64_t val, unsigned size)
 {
     MMIOState *s = opaque;
@@ -75,14 +75,14 @@ static const MemoryRegionOps mmio_ide_ops = {
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-static uint64_t mmio_ide_status_read(void *opaque, target_phys_addr_t addr,
+static uint64_t mmio_ide_status_read(void *opaque, hwaddr addr,
                                      unsigned size)
 {
     MMIOState *s= opaque;
     return ide_status_read(&s->bus, 0);
 }
 
-static void mmio_ide_cmd_write(void *opaque, target_phys_addr_t addr,
+static void mmio_ide_cmd_write(void *opaque, hwaddr addr,
                                uint64_t val, unsigned size)
 {
     MMIOState *s = opaque;
@@ -107,7 +107,7 @@ static const VMStateDescription vmstate_ide_mmio = {
     }
 };
 
-void mmio_ide_init (target_phys_addr_t membase, target_phys_addr_t membase2,
+void mmio_ide_init (hwaddr membase, hwaddr membase2,
                     MemoryRegion *address_space,
                     qemu_irq irq, int shift,
                     DriveInfo *hd0, DriveInfo *hd1)
diff --git a/hw/ide/pci.c b/hw/ide/pci.c
index 88c0942e3..bcdd70e45 100644
--- a/hw/ide/pci.c
+++ b/hw/ide/pci.c
@@ -188,7 +188,7 @@ static void bmdma_restart_bh(void *opaque)
 {
     BMDMAState *bm = opaque;
     IDEBus *bus = bm->bus;
-    int is_read;
+    bool is_read;
     int error_status;
 
     qemu_bh_delete(bm->bh);
@@ -198,7 +198,7 @@ static void bmdma_restart_bh(void *opaque)
         return;
     }
 
-    is_read = !!(bus->error_status & BM_STATUS_RETRY_READ);
+    is_read = (bus->error_status & BM_STATUS_RETRY_READ) != 0;
 
     /* The error status must be cleared before resubmitting the request: The
      * request may fail again, and this case can only be distinguished if the
@@ -327,7 +327,7 @@ void bmdma_cmd_writeb(BMDMAState *bm, uint32_t val)
     bm->cmd = val & 0x09;
 }
 
-static uint64_t bmdma_addr_read(void *opaque, target_phys_addr_t addr,
+static uint64_t bmdma_addr_read(void *opaque, hwaddr addr,
                                 unsigned width)
 {
     BMDMAState *bm = opaque;
@@ -341,7 +341,7 @@ static uint64_t bmdma_addr_read(void *opaque, target_phys_addr_t addr,
     return data;
 }
 
-static void bmdma_addr_write(void *opaque, target_phys_addr_t addr,
+static void bmdma_addr_write(void *opaque, hwaddr addr,
                              uint64_t data, unsigned width)
 {
     BMDMAState *bm = opaque;
diff --git a/hw/ide/piix.c b/hw/ide/piix.c
index 4ded9ee13..9431badad 100644
--- a/hw/ide/piix.c
+++ b/hw/ide/piix.c
@@ -33,7 +33,7 @@
 
 #include <hw/ide/pci.h>
 
-static uint64_t bmdma_read(void *opaque, target_phys_addr_t addr, unsigned size)
+static uint64_t bmdma_read(void *opaque, hwaddr addr, unsigned size)
 {
     BMDMAState *bm = opaque;
     uint32_t val;
@@ -59,7 +59,7 @@ static uint64_t bmdma_read(void *opaque, target_phys_addr_t addr, unsigned size)
     return val;
 }
 
-static void bmdma_write(void *opaque, target_phys_addr_t addr,
+static void bmdma_write(void *opaque, hwaddr addr,
                         uint64_t val, unsigned size)
 {
     BMDMAState *bm = opaque;
diff --git a/hw/ide/qdev.c b/hw/ide/qdev.c
index 5ea9b8f4b..f2e4ea420 100644
--- a/hw/ide/qdev.c
+++ b/hw/ide/qdev.c
@@ -60,7 +60,7 @@ static char *idebus_get_fw_dev_path(DeviceState *dev)
     snprintf(path, sizeof(path), "%s@%d", qdev_fw_name(dev),
              ((IDEBus*)dev->parent_bus)->bus_id);
 
-    return strdup(path);
+    return g_strdup(path);
 }
 
 static int ide_qdev_init(DeviceState *qdev)
diff --git a/hw/ide/via.c b/hw/ide/via.c
index b20e4f094..efda1733d 100644
--- a/hw/ide/via.c
+++ b/hw/ide/via.c
@@ -33,7 +33,7 @@
 
 #include <hw/ide/pci.h>
 
-static uint64_t bmdma_read(void *opaque, target_phys_addr_t addr,
+static uint64_t bmdma_read(void *opaque, hwaddr addr,
                            unsigned size)
 {
     BMDMAState *bm = opaque;
@@ -60,7 +60,7 @@ static uint64_t bmdma_read(void *opaque, target_phys_addr_t addr,
     return val;
 }
 
-static void bmdma_write(void *opaque, target_phys_addr_t addr,
+static void bmdma_write(void *opaque, hwaddr addr,
                         uint64_t val, unsigned size)
 {
     BMDMAState *bm = opaque;
diff --git a/hw/imx.h b/hw/imx.h
index ccf586fef..ea9e09327 100644
--- a/hw/imx.h
+++ b/hw/imx.h
@@ -11,7 +11,7 @@
 #ifndef IMX_H
 #define IMX_H
 
-void imx_serial_create(int uart, const target_phys_addr_t addr, qemu_irq irq);
+void imx_serial_create(int uart, const hwaddr addr, qemu_irq irq);
 
 typedef enum  {
     NOCLK,
@@ -23,10 +23,10 @@ typedef enum  {
 
 uint32_t imx_clock_frequency(DeviceState *s, IMXClk clock);
 
-void imx_timerp_create(const target_phys_addr_t addr,
+void imx_timerp_create(const hwaddr addr,
                       qemu_irq irq,
                       DeviceState *ccm);
-void imx_timerg_create(const target_phys_addr_t addr,
+void imx_timerg_create(const hwaddr addr,
                       qemu_irq irq,
                       DeviceState *ccm);
 
diff --git a/hw/imx_avic.c b/hw/imx_avic.c
index 4f010e8ee..810979366 100644
--- a/hw/imx_avic.c
+++ b/hw/imx_avic.c
@@ -6,9 +6,9 @@
  *
  * Copyright (c) 2008 OKL
  * Copyright (c) 2011 NICTA Pty Ltd
- * Originally Written by Hans Jiang
+ * Originally written by Hans Jiang
  *
- * This code is licenced under the GPL version 2 or later.  See
+ * This code is licensed under the GPL version 2 or later.  See
  * the COPYING file in the top-level directory.
  *
  * TODO: implement vectors.
@@ -152,7 +152,7 @@ static void imx_avic_set_irq(void *opaque, int irq, int level)
 
 
 static uint64_t imx_avic_read(void *opaque,
-                             target_phys_addr_t offset, unsigned size)
+                             hwaddr offset, unsigned size)
 {
     IMXAVICState *s = (IMXAVICState *)opaque;
 
@@ -259,7 +259,7 @@ static uint64_t imx_avic_read(void *opaque,
     }
 }
 
-static void imx_avic_write(void *opaque, target_phys_addr_t offset,
+static void imx_avic_write(void *opaque, hwaddr offset,
                           uint64_t val, unsigned size)
 {
     IMXAVICState *s = (IMXAVICState *)opaque;
diff --git a/hw/imx_ccm.c b/hw/imx_ccm.c
index 10952c6ea..f2e623cd2 100644
--- a/hw/imx_ccm.c
+++ b/hw/imx_ccm.c
@@ -191,7 +191,7 @@ static void imx_ccm_reset(DeviceState *dev)
     update_clocks(s);
 }
 
-static uint64_t imx_ccm_read(void *opaque, target_phys_addr_t offset,
+static uint64_t imx_ccm_read(void *opaque, hwaddr offset,
                                 unsigned size)
 {
     IMXCCMState *s = (IMXCCMState *)opaque;
@@ -232,7 +232,7 @@ static uint64_t imx_ccm_read(void *opaque, target_phys_addr_t offset,
     return 0;
 }
 
-static void imx_ccm_write(void *opaque, target_phys_addr_t offset,
+static void imx_ccm_write(void *opaque, hwaddr offset,
                           uint64_t value, unsigned size)
 {
     IMXCCMState *s = (IMXCCMState *)opaque;
diff --git a/hw/imx_serial.c b/hw/imx_serial.c
index d4eae430f..dcd125fd2 100644
--- a/hw/imx_serial.c
+++ b/hw/imx_serial.c
@@ -183,7 +183,7 @@ static void imx_serial_reset_at_boot(DeviceState *dev)
 
 }
 
-static uint64_t imx_serial_read(void *opaque, target_phys_addr_t offset,
+static uint64_t imx_serial_read(void *opaque, hwaddr offset,
                                 unsigned size)
 {
     IMXSerialState *s = (IMXSerialState *)opaque;
@@ -244,7 +244,7 @@ static uint64_t imx_serial_read(void *opaque, target_phys_addr_t offset,
     }
 }
 
-static void imx_serial_write(void *opaque, target_phys_addr_t offset,
+static void imx_serial_write(void *opaque, hwaddr offset,
                       uint64_t value, unsigned size)
 {
     IMXSerialState *s = (IMXSerialState *)opaque;
@@ -401,7 +401,7 @@ static int imx_serial_init(SysBusDevice *dev)
     return 0;
 }
 
-void imx_serial_create(int uart, const target_phys_addr_t addr, qemu_irq irq)
+void imx_serial_create(int uart, const hwaddr addr, qemu_irq irq)
 {
     DeviceState *dev;
     SysBusDevice *bus;
@@ -427,7 +427,7 @@ void imx_serial_create(int uart, const target_phys_addr_t addr, qemu_irq irq)
     qdev_prop_set_chr(dev, "chardev", chr);
     bus = sysbus_from_qdev(dev);
     qdev_init_nofail(dev);
-    if (addr != (target_phys_addr_t)-1) {
+    if (addr != (hwaddr)-1) {
         sysbus_mmio_map(bus, 0, addr);
     }
     sysbus_connect_irq(bus, 0, irq);
diff --git a/hw/imx_timer.c b/hw/imx_timer.c
index 16215ccf0..33f33fb41 100644
--- a/hw/imx_timer.c
+++ b/hw/imx_timer.c
@@ -3,10 +3,10 @@
  *
  * Copyright (c) 2008 OK Labs
  * Copyright (c) 2011 NICTA Pty Ltd
- * Originally Written by Hans Jiang
+ * Originally written by Hans Jiang
  * Updated by Peter Chubb
  *
- * This code is licenced under GPL version 2 or later.  See
+ * This code is licensed under GPL version 2 or later.  See
  * the COPYING file in the top-level directory.
  *
  */
@@ -194,7 +194,7 @@ static void imx_timerg_reload(IMXTimerGState *s, uint32_t timeout)
     ptimer_set_count(s->timer, diff_cnt);
 }
 
-static uint64_t imx_timerg_read(void *opaque, target_phys_addr_t offset,
+static uint64_t imx_timerg_read(void *opaque, hwaddr offset,
                                 unsigned size)
 {
     IMXTimerGState *s = (IMXTimerGState *)opaque;
@@ -251,7 +251,7 @@ static void imx_timerg_reset(DeviceState *dev)
     imx_timerg_set_freq(s);
 }
 
-static void imx_timerg_write(void *opaque, target_phys_addr_t offset,
+static void imx_timerg_write(void *opaque, hwaddr offset,
                              uint64_t value, unsigned size)
 {
     IMXTimerGState *s = (IMXTimerGState *)opaque;
@@ -468,7 +468,7 @@ static void imx_timerp_reset(DeviceState *dev)
     ptimer_set_count(s->timer, TIMER_MAX);
 }
 
-static uint64_t imx_timerp_read(void *opaque, target_phys_addr_t offset,
+static uint64_t imx_timerp_read(void *opaque, hwaddr offset,
                                 unsigned size)
 {
     IMXTimerPState *s = (IMXTimerPState *)opaque;
@@ -517,7 +517,7 @@ static void set_timerp_freq(IMXTimerPState *s)
     }
 }
 
-static void imx_timerp_write(void *opaque, target_phys_addr_t offset,
+static void imx_timerp_write(void *opaque, hwaddr offset,
                              uint64_t value, unsigned size)
 {
     IMXTimerPState *s = (IMXTimerPState *)opaque;
@@ -580,7 +580,7 @@ static void imx_timerp_tick(void *opaque)
     imx_timerp_update(s);
 }
 
-void imx_timerp_create(const target_phys_addr_t addr,
+void imx_timerp_create(const hwaddr addr,
                               qemu_irq irq,
                               DeviceState *ccm)
 {
@@ -634,7 +634,7 @@ static int imx_timerp_init(SysBusDevice *dev)
 }
 
 
-void imx_timerg_create(const target_phys_addr_t addr,
+void imx_timerg_create(const hwaddr addr,
                               qemu_irq irq,
                               DeviceState *ccm)
 {
diff --git a/hw/integratorcp.c b/hw/integratorcp.c
index d0e2e9068..77807c39e 100644
--- a/hw/integratorcp.c
+++ b/hw/integratorcp.c
@@ -38,7 +38,7 @@ static uint8_t integrator_spd[128] = {
    0xe, 4, 0x1c, 1, 2, 0x20, 0xc0, 0, 0, 0, 0, 0x30, 0x28, 0x30, 0x28, 0x40
 };
 
-static uint64_t integratorcm_read(void *opaque, target_phys_addr_t offset,
+static uint64_t integratorcm_read(void *opaque, hwaddr offset,
                                   unsigned size)
 {
     integratorcm_state *s = (integratorcm_state *)opaque;
@@ -141,7 +141,7 @@ static void integratorcm_update(integratorcm_state *s)
         hw_error("Core module interrupt\n");
 }
 
-static void integratorcm_write(void *opaque, target_phys_addr_t offset,
+static void integratorcm_write(void *opaque, hwaddr offset,
                                uint64_t value, unsigned size)
 {
     integratorcm_state *s = (integratorcm_state *)opaque;
@@ -295,7 +295,7 @@ static void icp_pic_set_irq(void *opaque, int irq, int level)
     icp_pic_update(s);
 }
 
-static uint64_t icp_pic_read(void *opaque, target_phys_addr_t offset,
+static uint64_t icp_pic_read(void *opaque, hwaddr offset,
                              unsigned size)
 {
     icp_pic_state *s = (icp_pic_state *)opaque;
@@ -324,7 +324,7 @@ static uint64_t icp_pic_read(void *opaque, target_phys_addr_t offset,
     }
 }
 
-static void icp_pic_write(void *opaque, target_phys_addr_t offset,
+static void icp_pic_write(void *opaque, hwaddr offset,
                           uint64_t value, unsigned size)
 {
     icp_pic_state *s = (icp_pic_state *)opaque;
@@ -381,7 +381,7 @@ static int icp_pic_init(SysBusDevice *dev)
 
 /* CP control registers.  */
 
-static uint64_t icp_control_read(void *opaque, target_phys_addr_t offset,
+static uint64_t icp_control_read(void *opaque, hwaddr offset,
                                  unsigned size)
 {
     switch (offset >> 2) {
@@ -399,7 +399,7 @@ static uint64_t icp_control_read(void *opaque, target_phys_addr_t offset,
     }
 }
 
-static void icp_control_write(void *opaque, target_phys_addr_t offset,
+static void icp_control_write(void *opaque, hwaddr offset,
                           uint64_t value, unsigned size)
 {
     switch (offset >> 2) {
@@ -419,7 +419,7 @@ static const MemoryRegionOps icp_control_ops = {
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-static void icp_control_init(target_phys_addr_t base)
+static void icp_control_init(hwaddr base)
 {
     MemoryRegion *io;
 
@@ -438,11 +438,13 @@ static struct arm_boot_info integrator_binfo = {
     .board_id = 0x113,
 };
 
-static void integratorcp_init(ram_addr_t ram_size,
-                     const char *boot_device,
-                     const char *kernel_filename, const char *kernel_cmdline,
-                     const char *initrd_filename, const char *cpu_model)
+static void integratorcp_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t ram_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
     ARMCPU *cpu;
     MemoryRegion *address_space_mem = get_system_memory();
     MemoryRegion *ram = g_new(MemoryRegion, 1);
diff --git a/hw/intel-hda.c b/hw/intel-hda.c
index 127e81888..a68c3685e 100644
--- a/hw/intel-hda.c
+++ b/hw/intel-hda.c
@@ -206,17 +206,11 @@ static void intel_hda_reset(DeviceState *dev);
 
 /* --------------------------------------------------------------------- */
 
-static target_phys_addr_t intel_hda_addr(uint32_t lbase, uint32_t ubase)
+static hwaddr intel_hda_addr(uint32_t lbase, uint32_t ubase)
 {
-    target_phys_addr_t addr;
-
-#if TARGET_PHYS_ADDR_BITS == 32
-    addr = lbase;
-#else
-    addr = ubase;
-    addr <<= 32;
-    addr |= lbase;
-#endif
+    hwaddr addr;
+
+    addr = ((uint64_t)ubase << 32) | lbase;
     return addr;
 }
 
@@ -301,7 +295,7 @@ static int intel_hda_send_command(IntelHDAState *d, uint32_t verb)
 
 static void intel_hda_corb_run(IntelHDAState *d)
 {
-    target_phys_addr_t addr;
+    hwaddr addr;
     uint32_t rp, verb;
 
     if (d->ics & ICH6_IRS_BUSY) {
@@ -338,7 +332,7 @@ static void intel_hda_response(HDACodecDevice *dev, bool solicited, uint32_t res
 {
     HDACodecBus *bus = DO_UPCAST(HDACodecBus, qbus, dev->qdev.parent_bus);
     IntelHDAState *d = container_of(bus, IntelHDAState, codecs);
-    target_phys_addr_t addr;
+    hwaddr addr;
     uint32_t wp, ex;
 
     if (d->ics & ICH6_IRS_BUSY) {
@@ -387,7 +381,7 @@ static bool intel_hda_xfer(HDACodecDevice *dev, uint32_t stnr, bool output,
 {
     HDACodecBus *bus = DO_UPCAST(HDACodecBus, qbus, dev->qdev.parent_bus);
     IntelHDAState *d = container_of(bus, IntelHDAState, codecs);
-    target_phys_addr_t addr;
+    hwaddr addr;
     uint32_t s, copy, left;
     IntelHDAStream *st;
     bool irq = false;
@@ -459,7 +453,7 @@ static bool intel_hda_xfer(HDACodecDevice *dev, uint32_t stnr, bool output,
 
 static void intel_hda_parse_bdl(IntelHDAState *d, IntelHDAStream *st)
 {
-    target_phys_addr_t addr;
+    hwaddr addr;
     uint8_t buf[16];
     uint32_t i;
 
@@ -896,7 +890,7 @@ static const struct IntelHDAReg regtab[] = {
 
 };
 
-static const IntelHDAReg *intel_hda_reg_find(IntelHDAState *d, target_phys_addr_t addr)
+static const IntelHDAReg *intel_hda_reg_find(IntelHDAState *d, hwaddr addr)
 {
     const IntelHDAReg *reg;
 
@@ -1039,7 +1033,7 @@ static void intel_hda_regs_reset(IntelHDAState *d)
 
 /* --------------------------------------------------------------------- */
 
-static void intel_hda_mmio_writeb(void *opaque, target_phys_addr_t addr, uint32_t val)
+static void intel_hda_mmio_writeb(void *opaque, hwaddr addr, uint32_t val)
 {
     IntelHDAState *d = opaque;
     const IntelHDAReg *reg = intel_hda_reg_find(d, addr);
@@ -1047,7 +1041,7 @@ static void intel_hda_mmio_writeb(void *opaque, target_phys_addr_t addr, uint32_
     intel_hda_reg_write(d, reg, val, 0xff);
 }
 
-static void intel_hda_mmio_writew(void *opaque, target_phys_addr_t addr, uint32_t val)
+static void intel_hda_mmio_writew(void *opaque, hwaddr addr, uint32_t val)
 {
     IntelHDAState *d = opaque;
     const IntelHDAReg *reg = intel_hda_reg_find(d, addr);
@@ -1055,7 +1049,7 @@ static void intel_hda_mmio_writew(void *opaque, target_phys_addr_t addr, uint32_
     intel_hda_reg_write(d, reg, val, 0xffff);
 }
 
-static void intel_hda_mmio_writel(void *opaque, target_phys_addr_t addr, uint32_t val)
+static void intel_hda_mmio_writel(void *opaque, hwaddr addr, uint32_t val)
 {
     IntelHDAState *d = opaque;
     const IntelHDAReg *reg = intel_hda_reg_find(d, addr);
@@ -1063,7 +1057,7 @@ static void intel_hda_mmio_writel(void *opaque, target_phys_addr_t addr, uint32_
     intel_hda_reg_write(d, reg, val, 0xffffffff);
 }
 
-static uint32_t intel_hda_mmio_readb(void *opaque, target_phys_addr_t addr)
+static uint32_t intel_hda_mmio_readb(void *opaque, hwaddr addr)
 {
     IntelHDAState *d = opaque;
     const IntelHDAReg *reg = intel_hda_reg_find(d, addr);
@@ -1071,7 +1065,7 @@ static uint32_t intel_hda_mmio_readb(void *opaque, target_phys_addr_t addr)
     return intel_hda_reg_read(d, reg, 0xff);
 }
 
-static uint32_t intel_hda_mmio_readw(void *opaque, target_phys_addr_t addr)
+static uint32_t intel_hda_mmio_readw(void *opaque, hwaddr addr)
 {
     IntelHDAState *d = opaque;
     const IntelHDAReg *reg = intel_hda_reg_find(d, addr);
@@ -1079,7 +1073,7 @@ static uint32_t intel_hda_mmio_readw(void *opaque, target_phys_addr_t addr)
     return intel_hda_reg_read(d, reg, 0xffff);
 }
 
-static uint32_t intel_hda_mmio_readl(void *opaque, target_phys_addr_t addr)
+static uint32_t intel_hda_mmio_readl(void *opaque, hwaddr addr)
 {
     IntelHDAState *d = opaque;
     const IntelHDAReg *reg = intel_hda_reg_find(d, addr);
diff --git a/hw/ioapic.c b/hw/ioapic.c
index e2e4796bb..72730951a 100644
--- a/hw/ioapic.c
+++ b/hw/ioapic.c
@@ -139,7 +139,7 @@ void ioapic_eoi_broadcast(int vector)
 }
 
 static uint64_t
-ioapic_mem_read(void *opaque, target_phys_addr_t addr, unsigned int size)
+ioapic_mem_read(void *opaque, hwaddr addr, unsigned int size)
 {
     IOAPICCommonState *s = opaque;
     int index;
@@ -181,7 +181,7 @@ ioapic_mem_read(void *opaque, target_phys_addr_t addr, unsigned int size)
 }
 
 static void
-ioapic_mem_write(void *opaque, target_phys_addr_t addr, uint64_t val,
+ioapic_mem_write(void *opaque, hwaddr addr, uint64_t val,
                  unsigned int size)
 {
     IOAPICCommonState *s = opaque;
diff --git a/hw/ioh3420.c b/hw/ioh3420.c
index 94a537c9b..4d314733b 100644
--- a/hw/ioh3420.c
+++ b/hw/ioh3420.c
@@ -125,7 +125,6 @@ static int ioh3420_initfn(PCIDevice *d)
     rc = pcie_chassis_add_slot(s);
     if (rc < 0) {
         goto err_pcie_cap;
-        return rc;
     }
     pcie_cap_root_init(d);
     rc = pcie_aer_init(d, IOH_EP_AER_OFFSET);
diff --git a/hw/irq.c b/hw/irq.c
index d413a0b23..f4e2a7804 100644
--- a/hw/irq.c
+++ b/hw/irq.c
@@ -38,24 +38,37 @@ void qemu_set_irq(qemu_irq irq, int level)
     irq->handler(irq->opaque, irq->n, level);
 }
 
-qemu_irq *qemu_allocate_irqs(qemu_irq_handler handler, void *opaque, int n)
+qemu_irq *qemu_extend_irqs(qemu_irq *old, int n_old, qemu_irq_handler handler,
+                           void *opaque, int n)
 {
     qemu_irq *s;
     struct IRQState *p;
     int i;
 
-    s = (qemu_irq *)g_malloc0(sizeof(qemu_irq) * n);
-    p = (struct IRQState *)g_malloc0(sizeof(struct IRQState) * n);
-    for (i = 0; i < n; i++) {
-        p->handler = handler;
-        p->opaque = opaque;
-        p->n = i;
+    if (!old) {
+        n_old = 0;
+    }
+    s = old ? g_renew(qemu_irq, old, n + n_old) : g_new(qemu_irq, n);
+    p = old ? g_renew(struct IRQState, s[0], n + n_old) :
+                g_new(struct IRQState, n);
+    for (i = 0; i < n + n_old; i++) {
+        if (i >= n_old) {
+            p->handler = handler;
+            p->opaque = opaque;
+            p->n = i;
+        }
         s[i] = p;
         p++;
     }
     return s;
 }
 
+qemu_irq *qemu_allocate_irqs(qemu_irq_handler handler, void *opaque, int n)
+{
+    return qemu_extend_irqs(NULL, 0, handler, opaque, n);
+}
+
+
 void qemu_free_irqs(qemu_irq *s)
 {
     g_free(s[0]);
diff --git a/hw/irq.h b/hw/irq.h
index 56c55f0c4..610e6b762 100644
--- a/hw/irq.h
+++ b/hw/irq.h
@@ -3,6 +3,8 @@
 
 /* Generic IRQ/GPIO pin infrastructure.  */
 
+typedef struct IRQState *qemu_irq;
+
 typedef void (*qemu_irq_handler)(void *opaque, int n, int level);
 
 void qemu_set_irq(qemu_irq irq, int level);
@@ -23,8 +25,17 @@ static inline void qemu_irq_pulse(qemu_irq irq)
     qemu_set_irq(irq, 0);
 }
 
-/* Returns an array of N IRQs.  */
+/* Returns an array of N IRQs. Each IRQ is assigned the argument handler and
+ * opaque data.
+ */
 qemu_irq *qemu_allocate_irqs(qemu_irq_handler handler, void *opaque, int n);
+
+/* Extends an Array of IRQs. Old IRQs have their handlers and opaque data
+ * preserved. New IRQs are assigned the argument handler and opaque data.
+ */
+qemu_irq *qemu_extend_irqs(qemu_irq *old, int n_old, qemu_irq_handler handler,
+                                void *opaque, int n);
+
 void qemu_free_irqs(qemu_irq *s);
 
 /* Returns a new IRQ with opposite polarity.  */
diff --git a/hw/isa-bus.c b/hw/isa-bus.c
index f9b237387..685fdc0f8 100644
--- a/hw/isa-bus.c
+++ b/hw/isa-bus.c
@@ -19,11 +19,12 @@
 #include "hw.h"
 #include "monitor.h"
 #include "sysbus.h"
+#include "sysemu.h"
 #include "isa.h"
 #include "exec-memory.h"
 
 static ISABus *isabus;
-target_phys_addr_t isa_mem_base = 0;
+hwaddr isa_mem_base = 0;
 
 static void isabus_dev_print(Monitor *mon, DeviceState *dev, int indent);
 static char *isabus_get_fw_dev_path(DeviceState *dev);
@@ -166,6 +167,25 @@ ISADevice *isa_create_simple(ISABus *bus, const char *name)
     return dev;
 }
 
+ISADevice *isa_vga_init(ISABus *bus)
+{
+    switch (vga_interface_type) {
+    case VGA_CIRRUS:
+        return isa_create_simple(bus, "isa-cirrus-vga");
+    case VGA_QXL:
+        fprintf(stderr, "%s: qxl: no PCI bus\n", __func__);
+        return NULL;
+    case VGA_STD:
+        return isa_create_simple(bus, "isa-vga");
+    case VGA_VMWARE:
+        fprintf(stderr, "%s: vmware_vga: no PCI bus\n", __func__);
+        return NULL;
+    case VGA_NONE:
+    default:
+        return NULL;
+    }
+}
+
 static void isabus_dev_print(Monitor *mon, DeviceState *dev, int indent)
 {
     ISADevice *d = ISA_DEVICE(dev);
@@ -236,7 +256,7 @@ static char *isabus_get_fw_dev_path(DeviceState *dev)
         snprintf(path + off, sizeof(path) - off, "@%04x", d->ioport_id);
     }
 
-    return strdup(path);
+    return g_strdup(path);
 }
 
 MemoryRegion *isa_address_space(ISADevice *dev)
diff --git a/hw/isa.h b/hw/isa.h
index dc970527a..f9382e836 100644
--- a/hw/isa.h
+++ b/hw/isa.h
@@ -47,6 +47,8 @@ ISADevice *isa_create(ISABus *bus, const char *name);
 ISADevice *isa_try_create(ISABus *bus, const char *name);
 ISADevice *isa_create_simple(ISABus *bus, const char *name);
 
+ISADevice *isa_vga_init(ISABus *bus);
+
 /**
  * isa_register_ioport: Install an I/O port region on the ISA bus.
  *
@@ -82,10 +84,10 @@ static inline ISABus *isa_bus_from_device(ISADevice *d)
     return DO_UPCAST(ISABus, qbus, d->qdev.parent_bus);
 }
 
-extern target_phys_addr_t isa_mem_base;
+extern hwaddr isa_mem_base;
 
-void isa_mmio_setup(MemoryRegion *mr, target_phys_addr_t size);
-void isa_mmio_init(target_phys_addr_t base, target_phys_addr_t size);
+void isa_mmio_setup(MemoryRegion *mr, hwaddr size);
+void isa_mmio_init(hwaddr base, hwaddr size);
 
 /* dma.c */
 int DMA_get_channel_mode (int nchan);
diff --git a/hw/isa_mmio.c b/hw/isa_mmio.c
index fd755ab4a..14053960c 100644
--- a/hw/isa_mmio.c
+++ b/hw/isa_mmio.c
@@ -26,35 +26,35 @@
 #include "isa.h"
 #include "exec-memory.h"
 
-static void isa_mmio_writeb (void *opaque, target_phys_addr_t addr,
+static void isa_mmio_writeb (void *opaque, hwaddr addr,
                                   uint32_t val)
 {
     cpu_outb(addr & IOPORTS_MASK, val);
 }
 
-static void isa_mmio_writew(void *opaque, target_phys_addr_t addr,
+static void isa_mmio_writew(void *opaque, hwaddr addr,
                                uint32_t val)
 {
     cpu_outw(addr & IOPORTS_MASK, val);
 }
 
-static void isa_mmio_writel(void *opaque, target_phys_addr_t addr,
+static void isa_mmio_writel(void *opaque, hwaddr addr,
                                uint32_t val)
 {
     cpu_outl(addr & IOPORTS_MASK, val);
 }
 
-static uint32_t isa_mmio_readb (void *opaque, target_phys_addr_t addr)
+static uint32_t isa_mmio_readb (void *opaque, hwaddr addr)
 {
     return cpu_inb(addr & IOPORTS_MASK);
 }
 
-static uint32_t isa_mmio_readw(void *opaque, target_phys_addr_t addr)
+static uint32_t isa_mmio_readw(void *opaque, hwaddr addr)
 {
     return cpu_inw(addr & IOPORTS_MASK);
 }
 
-static uint32_t isa_mmio_readl(void *opaque, target_phys_addr_t addr)
+static uint32_t isa_mmio_readl(void *opaque, hwaddr addr)
 {
     return cpu_inl(addr & IOPORTS_MASK);
 }
@@ -67,12 +67,12 @@ static const MemoryRegionOps isa_mmio_ops = {
     .endianness = DEVICE_LITTLE_ENDIAN,
 };
 
-void isa_mmio_setup(MemoryRegion *mr, target_phys_addr_t size)
+void isa_mmio_setup(MemoryRegion *mr, hwaddr size)
 {
     memory_region_init_io(mr, &isa_mmio_ops, NULL, "isa-mmio", size);
 }
 
-void isa_mmio_init(target_phys_addr_t base, target_phys_addr_t size)
+void isa_mmio_init(hwaddr base, hwaddr size)
 {
     MemoryRegion *mr = g_malloc(sizeof(*mr));
 
diff --git a/hw/ivshmem.c b/hw/ivshmem.c
index 62fe53ae2..f6dbb212f 100644
--- a/hw/ivshmem.c
+++ b/hw/ivshmem.c
@@ -71,6 +71,8 @@ typedef struct IVShmemState {
     MemoryRegion bar;
     MemoryRegion ivshmem;
     uint64_t ivshmem_size; /* size of shared memory region */
+    uint32_t ivshmem_attr;
+    uint32_t ivshmem_64bit;
     int shm_fd; /* shared memory file descriptor */
 
     Peer *peers;
@@ -147,7 +149,6 @@ static void ivshmem_IntrStatus_write(IVShmemState *s, uint32_t val)
     s->intrstatus = val;
 
     ivshmem_update_irq(s, val);
-    return;
 }
 
 static uint32_t ivshmem_IntrStatus_read(IVShmemState *s)
@@ -162,7 +163,7 @@ static uint32_t ivshmem_IntrStatus_read(IVShmemState *s)
     return ret;
 }
 
-static void ivshmem_io_write(void *opaque, target_phys_addr_t addr,
+static void ivshmem_io_write(void *opaque, hwaddr addr,
                              uint64_t val, unsigned size)
 {
     IVShmemState *s = opaque;
@@ -201,7 +202,7 @@ static void ivshmem_io_write(void *opaque, target_phys_addr_t addr,
     }
 }
 
-static uint64_t ivshmem_io_read(void *opaque, target_phys_addr_t addr,
+static uint64_t ivshmem_io_read(void *opaque, hwaddr addr,
                                 unsigned size)
 {
 
@@ -339,7 +340,7 @@ static void create_shared_memory_BAR(IVShmemState *s, int fd) {
     memory_region_add_subregion(&s->bar, 0, &s->ivshmem);
 
     /* region for shared memory */
-    pci_register_bar(&s->dev, 2, PCI_BASE_ADDRESS_SPACE_MEMORY, &s->bar);
+    pci_register_bar(&s->dev, 2, s->ivshmem_attr, &s->bar);
 }
 
 static void ivshmem_add_eventfd(IVShmemState *s, int posn, int i)
@@ -508,8 +509,6 @@ static void ivshmem_read(void *opaque, const uint8_t * buf, int flags)
     if (ivshmem_has_feature(s, IVSHMEM_IOEVENTFD)) {
         ivshmem_add_eventfd(s, incoming_posn, guest_max_eventfd);
     }
-
-    return;
 }
 
 /* Select the MSI-X vectors used by device.
@@ -534,7 +533,6 @@ static void ivshmem_reset(DeviceState *d)
 
     s->intrstatus = 0;
     ivshmem_use_msix(s);
-    return;
 }
 
 static uint64_t ivshmem_get_size(IVShmemState * s) {
@@ -690,6 +688,11 @@ static int pci_ivshmem_init(PCIDevice *dev)
                      &s->ivshmem_mmio);
 
     memory_region_init(&s->bar, "ivshmem-bar2-container", s->ivshmem_size);
+    s->ivshmem_attr = PCI_BASE_ADDRESS_SPACE_MEMORY |
+        PCI_BASE_ADDRESS_MEM_PREFETCH;
+    if (s->ivshmem_64bit) {
+        s->ivshmem_attr |= PCI_BASE_ADDRESS_MEM_TYPE_64;
+    }
 
     if ((s->server_chr != NULL) &&
                         (strncmp(s->server_chr->filename, "unix:", 5) == 0)) {
@@ -715,8 +718,7 @@ static int pci_ivshmem_init(PCIDevice *dev)
         /* allocate/initialize space for interrupt handling */
         s->peers = g_malloc0(s->nb_peers * sizeof(Peer));
 
-        pci_register_bar(&s->dev, 2,
-                         PCI_BASE_ADDRESS_SPACE_MEMORY, &s->bar);
+        pci_register_bar(&s->dev, 2, s->ivshmem_attr, &s->bar);
 
         s->eventfd_chr = g_malloc0(s->vectors * sizeof(CharDriverState *));
 
@@ -786,6 +788,7 @@ static Property ivshmem_properties[] = {
     DEFINE_PROP_BIT("msi", IVShmemState, features, IVSHMEM_MSI, true),
     DEFINE_PROP_STRING("shm", IVShmemState, shmobj),
     DEFINE_PROP_STRING("role", IVShmemState, role),
+    DEFINE_PROP_UINT32("use64", IVShmemState, ivshmem_64bit, 1),
     DEFINE_PROP_END_OF_LIST(),
 };
 
diff --git a/hw/jazz_led.c b/hw/jazz_led.c
index 648652302..640e75ef2 100644
--- a/hw/jazz_led.c
+++ b/hw/jazz_led.c
@@ -39,7 +39,7 @@ typedef struct LedState {
     screen_state_t state;
 } LedState;
 
-static uint64_t jazz_led_read(void *opaque, target_phys_addr_t addr,
+static uint64_t jazz_led_read(void *opaque, hwaddr addr,
                               unsigned int size)
 {
     LedState *s = opaque;
@@ -51,7 +51,7 @@ static uint64_t jazz_led_read(void *opaque, target_phys_addr_t addr,
     return val;
 }
 
-static void jazz_led_write(void *opaque, target_phys_addr_t addr,
+static void jazz_led_write(void *opaque, hwaddr addr,
                            uint64_t val, unsigned int size)
 {
     LedState *s = opaque;
@@ -196,7 +196,7 @@ static void jazz_led_update_display(void *opaque)
     }
 
     s->state = REDRAW_NONE;
-    dpy_update(ds, 0, 0, ds_get_width(ds), ds_get_height(ds));
+    dpy_gfx_update(ds, 0, 0, ds_get_width(ds), ds_get_height(ds));
 }
 
 static void jazz_led_invalidate_display(void *opaque)
@@ -210,7 +210,7 @@ static void jazz_led_text_update(void *opaque, console_ch_t *chardata)
     LedState *s = opaque;
     char buf[2];
 
-    dpy_cursor(s->ds, -1, -1);
+    dpy_text_cursor(s->ds, -1, -1);
     qemu_console_resize(s->ds, 2, 1);
 
     /* TODO: draw the segments */
@@ -218,7 +218,7 @@ static void jazz_led_text_update(void *opaque, console_ch_t *chardata)
     console_write_ch(chardata++, 0x00200100 | buf[0]);
     console_write_ch(chardata++, 0x00200100 | buf[1]);
 
-    dpy_update(s->ds, 0, 0, 2, 1);
+    dpy_text_update(s->ds, 0, 0, 2, 1);
 }
 
 static int jazz_led_post_load(void *opaque, int version_id)
diff --git a/hw/kvm/Makefile.objs b/hw/kvm/Makefile.objs
index 226497a58..f620d7ff8 100644
--- a/hw/kvm/Makefile.objs
+++ b/hw/kvm/Makefile.objs
@@ -1 +1 @@
-obj-$(CONFIG_KVM) += clock.o apic.o i8259.o ioapic.o i8254.o
+obj-$(CONFIG_KVM) += clock.o apic.o i8259.o ioapic.o i8254.o pci-assign.o
diff --git a/hw/kvm/apic.c b/hw/kvm/apic.c
index 80e3e4833..8b65d513d 100644
--- a/hw/kvm/apic.c
+++ b/hw/kvm/apic.c
@@ -104,7 +104,7 @@ static void kvm_apic_enable_tpr_reporting(APICCommonState *s, bool enable)
         .enabled = enable
     };
 
-    kvm_vcpu_ioctl(s->cpu_env, KVM_TPR_ACCESS_REPORTING, &ctl);
+    kvm_vcpu_ioctl(&s->cpu->env, KVM_TPR_ACCESS_REPORTING, &ctl);
 }
 
 static void kvm_apic_vapic_base_update(APICCommonState *s)
@@ -114,7 +114,7 @@ static void kvm_apic_vapic_base_update(APICCommonState *s)
     };
     int ret;
 
-    ret = kvm_vcpu_ioctl(s->cpu_env, KVM_SET_VAPIC_ADDR, &vapid_addr);
+    ret = kvm_vcpu_ioctl(&s->cpu->env, KVM_SET_VAPIC_ADDR, &vapid_addr);
     if (ret < 0) {
         fprintf(stderr, "KVM: setting VAPIC address failed (%s)\n",
                 strerror(-ret));
@@ -125,7 +125,7 @@ static void kvm_apic_vapic_base_update(APICCommonState *s)
 static void do_inject_external_nmi(void *data)
 {
     APICCommonState *s = data;
-    CPUX86State *env = s->cpu_env;
+    CPUX86State *env = &s->cpu->env;
     uint32_t lvt;
     int ret;
 
@@ -143,16 +143,16 @@ static void do_inject_external_nmi(void *data)
 
 static void kvm_apic_external_nmi(APICCommonState *s)
 {
-    run_on_cpu(s->cpu_env, do_inject_external_nmi, s);
+    run_on_cpu(CPU(s->cpu), do_inject_external_nmi, s);
 }
 
-static uint64_t kvm_apic_mem_read(void *opaque, target_phys_addr_t addr,
+static uint64_t kvm_apic_mem_read(void *opaque, hwaddr addr,
                                   unsigned size)
 {
     return ~(uint64_t)0;
 }
 
-static void kvm_apic_mem_write(void *opaque, target_phys_addr_t addr,
+static void kvm_apic_mem_write(void *opaque, hwaddr addr,
                                uint64_t data, unsigned size)
 {
     MSIMessage msg = { .address = addr, .data = data };
diff --git a/hw/kvm/ioapic.c b/hw/kvm/ioapic.c
index 6c3b8fe39..f95c15759 100644
--- a/hw/kvm/ioapic.c
+++ b/hw/kvm/ioapic.c
@@ -15,6 +15,46 @@
 #include "hw/apic_internal.h"
 #include "kvm.h"
 
+/* PC Utility function */
+void kvm_pc_setup_irq_routing(bool pci_enabled)
+{
+    KVMState *s = kvm_state;
+    int i;
+
+    if (kvm_check_extension(s, KVM_CAP_IRQ_ROUTING)) {
+        for (i = 0; i < 8; ++i) {
+            if (i == 2) {
+                continue;
+            }
+            kvm_irqchip_add_irq_route(s, i, KVM_IRQCHIP_PIC_MASTER, i);
+        }
+        for (i = 8; i < 16; ++i) {
+            kvm_irqchip_add_irq_route(s, i, KVM_IRQCHIP_PIC_SLAVE, i - 8);
+        }
+        if (pci_enabled) {
+            for (i = 0; i < 24; ++i) {
+                if (i == 0) {
+                    kvm_irqchip_add_irq_route(s, i, KVM_IRQCHIP_IOAPIC, 2);
+                } else if (i != 2) {
+                    kvm_irqchip_add_irq_route(s, i, KVM_IRQCHIP_IOAPIC, i);
+                }
+            }
+        }
+    }
+}
+
+void kvm_pc_gsi_handler(void *opaque, int n, int level)
+{
+    GSIState *s = opaque;
+
+    if (n < ISA_NUM_IRQS) {
+        /* Kernel will forward to both PIC and IOAPIC */
+        qemu_set_irq(s->i8259_irq[n], level);
+    } else {
+        qemu_set_irq(s->ioapic_irq[n], level);
+    }
+}
+
 typedef struct KVMIOAPICState KVMIOAPICState;
 
 struct KVMIOAPICState {
diff --git a/hw/kvm/pci-assign.c b/hw/kvm/pci-assign.c
new file mode 100644
index 000000000..e80dad009
--- /dev/null
+++ b/hw/kvm/pci-assign.c
@@ -0,0 +1,1905 @@
+/*
+ * Copyright (c) 2007, Neocleus Corporation.
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ *
+ *
+ *  Assign a PCI device from the host to a guest VM.
+ *
+ *  This implementation uses the classic device assignment interface of KVM
+ *  and is only available on x86 hosts. It is expected to be obsoleted by VFIO
+ *  based device assignment.
+ *
+ *  Adapted for KVM (qemu-kvm) by Qumranet. QEMU version was based on qemu-kvm
+ *  revision 4144fe9d48. See its repository for the history.
+ *
+ *  Copyright (c) 2007, Neocleus, Alex Novik (alex@neocleus.com)
+ *  Copyright (c) 2007, Neocleus, Guy Zana (guy@neocleus.com)
+ *  Copyright (C) 2008, Qumranet, Amit Shah (amit.shah@qumranet.com)
+ *  Copyright (C) 2008, Red Hat, Amit Shah (amit.shah@redhat.com)
+ *  Copyright (C) 2008, IBM, Muli Ben-Yehuda (muli@il.ibm.com)
+ */
+#include <stdio.h>
+#include <unistd.h>
+#include <sys/io.h>
+#include <sys/mman.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include "hw/hw.h"
+#include "hw/pc.h"
+#include "qemu-error.h"
+#include "console.h"
+#include "hw/loader.h"
+#include "monitor.h"
+#include "range.h"
+#include "sysemu.h"
+#include "hw/pci.h"
+#include "hw/msi.h"
+#include "kvm_i386.h"
+
+#define MSIX_PAGE_SIZE 0x1000
+
+/* From linux/ioport.h */
+#define IORESOURCE_IO       0x00000100  /* Resource type */
+#define IORESOURCE_MEM      0x00000200
+#define IORESOURCE_IRQ      0x00000400
+#define IORESOURCE_DMA      0x00000800
+#define IORESOURCE_PREFETCH 0x00002000  /* No side effects */
+
+//#define DEVICE_ASSIGNMENT_DEBUG
+
+#ifdef DEVICE_ASSIGNMENT_DEBUG
+#define DEBUG(fmt, ...)                                       \
+    do {                                                      \
+        fprintf(stderr, "%s: " fmt, __func__ , __VA_ARGS__);  \
+    } while (0)
+#else
+#define DEBUG(fmt, ...)
+#endif
+
+typedef struct PCIRegion {
+    int type;           /* Memory or port I/O */
+    int valid;
+    uint64_t base_addr;
+    uint64_t size;    /* size of the region */
+    int resource_fd;
+} PCIRegion;
+
+typedef struct PCIDevRegions {
+    uint8_t bus, dev, func; /* Bus inside domain, device and function */
+    int irq;                /* IRQ number */
+    uint16_t region_number; /* number of active regions */
+
+    /* Port I/O or MMIO Regions */
+    PCIRegion regions[PCI_NUM_REGIONS - 1];
+    int config_fd;
+} PCIDevRegions;
+
+typedef struct AssignedDevRegion {
+    MemoryRegion container;
+    MemoryRegion real_iomem;
+    union {
+        uint8_t *r_virtbase; /* mmapped access address for memory regions */
+        uint32_t r_baseport; /* the base guest port for I/O regions */
+    } u;
+    pcibus_t e_size;    /* emulated size of region in bytes */
+    pcibus_t r_size;    /* real size of region in bytes */
+    PCIRegion *region;
+} AssignedDevRegion;
+
+#define ASSIGNED_DEVICE_PREFER_MSI_BIT  0
+#define ASSIGNED_DEVICE_SHARE_INTX_BIT  1
+
+#define ASSIGNED_DEVICE_PREFER_MSI_MASK (1 << ASSIGNED_DEVICE_PREFER_MSI_BIT)
+#define ASSIGNED_DEVICE_SHARE_INTX_MASK (1 << ASSIGNED_DEVICE_SHARE_INTX_BIT)
+
+typedef struct MSIXTableEntry {
+    uint32_t addr_lo;
+    uint32_t addr_hi;
+    uint32_t data;
+    uint32_t ctrl;
+} MSIXTableEntry;
+
+typedef enum AssignedIRQType {
+    ASSIGNED_IRQ_NONE = 0,
+    ASSIGNED_IRQ_INTX_HOST_INTX,
+    ASSIGNED_IRQ_INTX_HOST_MSI,
+    ASSIGNED_IRQ_MSI,
+    ASSIGNED_IRQ_MSIX
+} AssignedIRQType;
+
+typedef struct AssignedDevice {
+    PCIDevice dev;
+    PCIHostDeviceAddress host;
+    uint32_t dev_id;
+    uint32_t features;
+    int intpin;
+    AssignedDevRegion v_addrs[PCI_NUM_REGIONS - 1];
+    PCIDevRegions real_device;
+    PCIINTxRoute intx_route;
+    AssignedIRQType assigned_irq_type;
+    struct {
+#define ASSIGNED_DEVICE_CAP_MSI (1 << 0)
+#define ASSIGNED_DEVICE_CAP_MSIX (1 << 1)
+        uint32_t available;
+#define ASSIGNED_DEVICE_MSI_ENABLED (1 << 0)
+#define ASSIGNED_DEVICE_MSIX_ENABLED (1 << 1)
+#define ASSIGNED_DEVICE_MSIX_MASKED (1 << 2)
+        uint32_t state;
+    } cap;
+    uint8_t emulate_config_read[PCI_CONFIG_SPACE_SIZE];
+    uint8_t emulate_config_write[PCI_CONFIG_SPACE_SIZE];
+    int msi_virq_nr;
+    int *msi_virq;
+    MSIXTableEntry *msix_table;
+    hwaddr msix_table_addr;
+    uint16_t msix_max;
+    MemoryRegion mmio;
+    char *configfd_name;
+    int32_t bootindex;
+} AssignedDevice;
+
+static void assigned_dev_update_irq_routing(PCIDevice *dev);
+
+static void assigned_dev_load_option_rom(AssignedDevice *dev);
+
+static void assigned_dev_unregister_msix_mmio(AssignedDevice *dev);
+
+static uint64_t assigned_dev_ioport_rw(AssignedDevRegion *dev_region,
+                                       hwaddr addr, int size,
+                                       uint64_t *data)
+{
+    uint64_t val = 0;
+    int fd = dev_region->region->resource_fd;
+
+    if (fd >= 0) {
+        if (data) {
+            DEBUG("pwrite data=%" PRIx64 ", size=%d, e_phys=" TARGET_FMT_plx
+                  ", addr="TARGET_FMT_plx"\n", *data, size, addr, addr);
+            if (pwrite(fd, data, size, addr) != size) {
+                error_report("%s - pwrite failed %s",
+                             __func__, strerror(errno));
+            }
+        } else {
+            if (pread(fd, &val, size, addr) != size) {
+                error_report("%s - pread failed %s",
+                             __func__, strerror(errno));
+                val = (1UL << (size * 8)) - 1;
+            }
+            DEBUG("pread val=%" PRIx64 ", size=%d, e_phys=" TARGET_FMT_plx
+                  ", addr=" TARGET_FMT_plx "\n", val, size, addr, addr);
+        }
+    } else {
+        uint32_t port = addr + dev_region->u.r_baseport;
+
+        if (data) {
+            DEBUG("out data=%" PRIx64 ", size=%d, e_phys=" TARGET_FMT_plx
+                  ", host=%x\n", *data, size, addr, port);
+            switch (size) {
+            case 1:
+                outb(*data, port);
+                break;
+            case 2:
+                outw(*data, port);
+                break;
+            case 4:
+                outl(*data, port);
+                break;
+            }
+        } else {
+            switch (size) {
+            case 1:
+                val = inb(port);
+                break;
+            case 2:
+                val = inw(port);
+                break;
+            case 4:
+                val = inl(port);
+                break;
+            }
+            DEBUG("in data=%" PRIx64 ", size=%d, e_phys=" TARGET_FMT_plx
+                  ", host=%x\n", val, size, addr, port);
+        }
+    }
+    return val;
+}
+
+static void assigned_dev_ioport_write(void *opaque, hwaddr addr,
+                                      uint64_t data, unsigned size)
+{
+    assigned_dev_ioport_rw(opaque, addr, size, &data);
+}
+
+static uint64_t assigned_dev_ioport_read(void *opaque,
+                                         hwaddr addr, unsigned size)
+{
+    return assigned_dev_ioport_rw(opaque, addr, size, NULL);
+}
+
+static uint32_t slow_bar_readb(void *opaque, hwaddr addr)
+{
+    AssignedDevRegion *d = opaque;
+    uint8_t *in = d->u.r_virtbase + addr;
+    uint32_t r;
+
+    r = *in;
+    DEBUG("slow_bar_readl addr=0x" TARGET_FMT_plx " val=0x%08x\n", addr, r);
+
+    return r;
+}
+
+static uint32_t slow_bar_readw(void *opaque, hwaddr addr)
+{
+    AssignedDevRegion *d = opaque;
+    uint16_t *in = (uint16_t *)(d->u.r_virtbase + addr);
+    uint32_t r;
+
+    r = *in;
+    DEBUG("slow_bar_readl addr=0x" TARGET_FMT_plx " val=0x%08x\n", addr, r);
+
+    return r;
+}
+
+static uint32_t slow_bar_readl(void *opaque, hwaddr addr)
+{
+    AssignedDevRegion *d = opaque;
+    uint32_t *in = (uint32_t *)(d->u.r_virtbase + addr);
+    uint32_t r;
+
+    r = *in;
+    DEBUG("slow_bar_readl addr=0x" TARGET_FMT_plx " val=0x%08x\n", addr, r);
+
+    return r;
+}
+
+static void slow_bar_writeb(void *opaque, hwaddr addr, uint32_t val)
+{
+    AssignedDevRegion *d = opaque;
+    uint8_t *out = d->u.r_virtbase + addr;
+
+    DEBUG("slow_bar_writeb addr=0x" TARGET_FMT_plx " val=0x%02x\n", addr, val);
+    *out = val;
+}
+
+static void slow_bar_writew(void *opaque, hwaddr addr, uint32_t val)
+{
+    AssignedDevRegion *d = opaque;
+    uint16_t *out = (uint16_t *)(d->u.r_virtbase + addr);
+
+    DEBUG("slow_bar_writew addr=0x" TARGET_FMT_plx " val=0x%04x\n", addr, val);
+    *out = val;
+}
+
+static void slow_bar_writel(void *opaque, hwaddr addr, uint32_t val)
+{
+    AssignedDevRegion *d = opaque;
+    uint32_t *out = (uint32_t *)(d->u.r_virtbase + addr);
+
+    DEBUG("slow_bar_writel addr=0x" TARGET_FMT_plx " val=0x%08x\n", addr, val);
+    *out = val;
+}
+
+static const MemoryRegionOps slow_bar_ops = {
+    .old_mmio = {
+        .read = { slow_bar_readb, slow_bar_readw, slow_bar_readl, },
+        .write = { slow_bar_writeb, slow_bar_writew, slow_bar_writel, },
+    },
+    .endianness = DEVICE_NATIVE_ENDIAN,
+};
+
+static void assigned_dev_iomem_setup(PCIDevice *pci_dev, int region_num,
+                                     pcibus_t e_size)
+{
+    AssignedDevice *r_dev = DO_UPCAST(AssignedDevice, dev, pci_dev);
+    AssignedDevRegion *region = &r_dev->v_addrs[region_num];
+    PCIRegion *real_region = &r_dev->real_device.regions[region_num];
+
+    if (e_size > 0) {
+        memory_region_init(&region->container, "assigned-dev-container",
+                           e_size);
+        memory_region_add_subregion(&region->container, 0, &region->real_iomem);
+
+        /* deal with MSI-X MMIO page */
+        if (real_region->base_addr <= r_dev->msix_table_addr &&
+                real_region->base_addr + real_region->size >
+                r_dev->msix_table_addr) {
+            uint64_t offset = r_dev->msix_table_addr - real_region->base_addr;
+
+            memory_region_add_subregion_overlap(&region->container,
+                                                offset,
+                                                &r_dev->mmio,
+                                                1);
+        }
+    }
+}
+
+static const MemoryRegionOps assigned_dev_ioport_ops = {
+    .read = assigned_dev_ioport_read,
+    .write = assigned_dev_ioport_write,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+};
+
+static void assigned_dev_ioport_setup(PCIDevice *pci_dev, int region_num,
+                                      pcibus_t size)
+{
+    AssignedDevice *r_dev = DO_UPCAST(AssignedDevice, dev, pci_dev);
+    AssignedDevRegion *region = &r_dev->v_addrs[region_num];
+
+    region->e_size = size;
+    memory_region_init(&region->container, "assigned-dev-container", size);
+    memory_region_init_io(&region->real_iomem, &assigned_dev_ioport_ops,
+                          r_dev->v_addrs + region_num,
+                          "assigned-dev-iomem", size);
+    memory_region_add_subregion(&region->container, 0, &region->real_iomem);
+}
+
+static uint32_t assigned_dev_pci_read(PCIDevice *d, int pos, int len)
+{
+    AssignedDevice *pci_dev = DO_UPCAST(AssignedDevice, dev, d);
+    uint32_t val;
+    ssize_t ret;
+    int fd = pci_dev->real_device.config_fd;
+
+again:
+    ret = pread(fd, &val, len, pos);
+    if (ret != len) {
+        if ((ret < 0) && (errno == EINTR || errno == EAGAIN)) {
+            goto again;
+        }
+
+        hw_error("pci read failed, ret = %zd errno = %d\n", ret, errno);
+    }
+
+    return val;
+}
+
+static uint8_t assigned_dev_pci_read_byte(PCIDevice *d, int pos)
+{
+    return (uint8_t)assigned_dev_pci_read(d, pos, 1);
+}
+
+static void assigned_dev_pci_write(PCIDevice *d, int pos, uint32_t val, int len)
+{
+    AssignedDevice *pci_dev = DO_UPCAST(AssignedDevice, dev, d);
+    ssize_t ret;
+    int fd = pci_dev->real_device.config_fd;
+
+again:
+    ret = pwrite(fd, &val, len, pos);
+    if (ret != len) {
+        if ((ret < 0) && (errno == EINTR || errno == EAGAIN)) {
+            goto again;
+        }
+
+        hw_error("pci write failed, ret = %zd errno = %d\n", ret, errno);
+    }
+}
+
+static void assigned_dev_emulate_config_read(AssignedDevice *dev,
+                                             uint32_t offset, uint32_t len)
+{
+    memset(dev->emulate_config_read + offset, 0xff, len);
+}
+
+static void assigned_dev_direct_config_read(AssignedDevice *dev,
+                                            uint32_t offset, uint32_t len)
+{
+    memset(dev->emulate_config_read + offset, 0, len);
+}
+
+static void assigned_dev_direct_config_write(AssignedDevice *dev,
+                                             uint32_t offset, uint32_t len)
+{
+    memset(dev->emulate_config_write + offset, 0, len);
+}
+
+static uint8_t pci_find_cap_offset(PCIDevice *d, uint8_t cap, uint8_t start)
+{
+    int id;
+    int max_cap = 48;
+    int pos = start ? start : PCI_CAPABILITY_LIST;
+    int status;
+
+    status = assigned_dev_pci_read_byte(d, PCI_STATUS);
+    if ((status & PCI_STATUS_CAP_LIST) == 0) {
+        return 0;
+    }
+
+    while (max_cap--) {
+        pos = assigned_dev_pci_read_byte(d, pos);
+        if (pos < 0x40) {
+            break;
+        }
+
+        pos &= ~3;
+        id = assigned_dev_pci_read_byte(d, pos + PCI_CAP_LIST_ID);
+
+        if (id == 0xff) {
+            break;
+        }
+        if (id == cap) {
+            return pos;
+        }
+
+        pos += PCI_CAP_LIST_NEXT;
+    }
+    return 0;
+}
+
+static int assigned_dev_register_regions(PCIRegion *io_regions,
+                                         unsigned long regions_num,
+                                         AssignedDevice *pci_dev)
+{
+    uint32_t i;
+    PCIRegion *cur_region = io_regions;
+
+    for (i = 0; i < regions_num; i++, cur_region++) {
+        if (!cur_region->valid) {
+            continue;
+        }
+
+        /* handle memory io regions */
+        if (cur_region->type & IORESOURCE_MEM) {
+            int t = cur_region->type & IORESOURCE_PREFETCH
+                ? PCI_BASE_ADDRESS_MEM_PREFETCH
+                : PCI_BASE_ADDRESS_SPACE_MEMORY;
+
+            /* map physical memory */
+            pci_dev->v_addrs[i].u.r_virtbase = mmap(NULL, cur_region->size,
+                                                    PROT_WRITE | PROT_READ,
+                                                    MAP_SHARED,
+                                                    cur_region->resource_fd,
+                                                    (off_t)0);
+
+            if (pci_dev->v_addrs[i].u.r_virtbase == MAP_FAILED) {
+                pci_dev->v_addrs[i].u.r_virtbase = NULL;
+                error_report("%s: Error: Couldn't mmap 0x%" PRIx64 "!",
+                             __func__, cur_region->base_addr);
+                return -1;
+            }
+
+            pci_dev->v_addrs[i].r_size = cur_region->size;
+            pci_dev->v_addrs[i].e_size = 0;
+
+            /* add offset */
+            pci_dev->v_addrs[i].u.r_virtbase +=
+                (cur_region->base_addr & 0xFFF);
+
+            if (cur_region->size & 0xFFF) {
+                error_report("PCI region %d at address 0x%" PRIx64 " has "
+                             "size 0x%" PRIx64 ", which is not a multiple of "
+                             "4K.  You might experience some performance hit "
+                             "due to that.",
+                             i, cur_region->base_addr, cur_region->size);
+                memory_region_init_io(&pci_dev->v_addrs[i].real_iomem,
+                                      &slow_bar_ops, &pci_dev->v_addrs[i],
+                                      "assigned-dev-slow-bar",
+                                      cur_region->size);
+            } else {
+                void *virtbase = pci_dev->v_addrs[i].u.r_virtbase;
+                char name[32];
+                snprintf(name, sizeof(name), "%s.bar%d",
+                         object_get_typename(OBJECT(pci_dev)), i);
+                memory_region_init_ram_ptr(&pci_dev->v_addrs[i].real_iomem,
+                                           name, cur_region->size,
+                                           virtbase);
+                vmstate_register_ram(&pci_dev->v_addrs[i].real_iomem,
+                                     &pci_dev->dev.qdev);
+            }
+
+            assigned_dev_iomem_setup(&pci_dev->dev, i, cur_region->size);
+            pci_register_bar((PCIDevice *) pci_dev, i, t,
+                             &pci_dev->v_addrs[i].container);
+            continue;
+        } else {
+            /* handle port io regions */
+            uint32_t val;
+            int ret;
+
+            /* Test kernel support for ioport resource read/write.  Old
+             * kernels return EIO.  New kernels only allow 1/2/4 byte reads
+             * so should return EINVAL for a 3 byte read */
+            ret = pread(pci_dev->v_addrs[i].region->resource_fd, &val, 3, 0);
+            if (ret >= 0) {
+                error_report("Unexpected return from I/O port read: %d", ret);
+                abort();
+            } else if (errno != EINVAL) {
+                error_report("Kernel doesn't support ioport resource "
+                             "access, hiding this region.");
+                close(pci_dev->v_addrs[i].region->resource_fd);
+                cur_region->valid = 0;
+                continue;
+            }
+
+            pci_dev->v_addrs[i].u.r_baseport = cur_region->base_addr;
+            pci_dev->v_addrs[i].r_size = cur_region->size;
+            pci_dev->v_addrs[i].e_size = 0;
+
+            assigned_dev_ioport_setup(&pci_dev->dev, i, cur_region->size);
+            pci_register_bar((PCIDevice *) pci_dev, i,
+                             PCI_BASE_ADDRESS_SPACE_IO,
+                             &pci_dev->v_addrs[i].container);
+        }
+    }
+
+    /* success */
+    return 0;
+}
+
+static int get_real_id(const char *devpath, const char *idname, uint16_t *val)
+{
+    FILE *f;
+    char name[128];
+    long id;
+
+    snprintf(name, sizeof(name), "%s%s", devpath, idname);
+    f = fopen(name, "r");
+    if (f == NULL) {
+        error_report("%s: %s: %m", __func__, name);
+        return -1;
+    }
+    if (fscanf(f, "%li\n", &id) == 1) {
+        *val = id;
+    } else {
+        return -1;
+    }
+    fclose(f);
+
+    return 0;
+}
+
+static int get_real_vendor_id(const char *devpath, uint16_t *val)
+{
+    return get_real_id(devpath, "vendor", val);
+}
+
+static int get_real_device_id(const char *devpath, uint16_t *val)
+{
+    return get_real_id(devpath, "device", val);
+}
+
+static int get_real_device(AssignedDevice *pci_dev, uint16_t r_seg,
+                           uint8_t r_bus, uint8_t r_dev, uint8_t r_func)
+{
+    char dir[128], name[128];
+    int fd, r = 0, v;
+    FILE *f;
+    uint64_t start, end, size, flags;
+    uint16_t id;
+    PCIRegion *rp;
+    PCIDevRegions *dev = &pci_dev->real_device;
+
+    dev->region_number = 0;
+
+    snprintf(dir, sizeof(dir), "/sys/bus/pci/devices/%04x:%02x:%02x.%x/",
+             r_seg, r_bus, r_dev, r_func);
+
+    snprintf(name, sizeof(name), "%sconfig", dir);
+
+    if (pci_dev->configfd_name && *pci_dev->configfd_name) {
+        dev->config_fd = monitor_handle_fd_param(cur_mon, pci_dev->configfd_name);
+        if (dev->config_fd < 0) {
+            return 1;
+        }
+    } else {
+        dev->config_fd = open(name, O_RDWR);
+
+        if (dev->config_fd == -1) {
+            error_report("%s: %s: %m", __func__, name);
+            return 1;
+        }
+    }
+again:
+    r = read(dev->config_fd, pci_dev->dev.config,
+             pci_config_size(&pci_dev->dev));
+    if (r < 0) {
+        if (errno == EINTR || errno == EAGAIN) {
+            goto again;
+        }
+        error_report("%s: read failed, errno = %d", __func__, errno);
+    }
+
+    /* Restore or clear multifunction, this is always controlled by qemu */
+    if (pci_dev->dev.cap_present & QEMU_PCI_CAP_MULTIFUNCTION) {
+        pci_dev->dev.config[PCI_HEADER_TYPE] |= PCI_HEADER_TYPE_MULTI_FUNCTION;
+    } else {
+        pci_dev->dev.config[PCI_HEADER_TYPE] &= ~PCI_HEADER_TYPE_MULTI_FUNCTION;
+    }
+
+    /* Clear host resource mapping info.  If we choose not to register a
+     * BAR, such as might be the case with the option ROM, we can get
+     * confusing, unwritable, residual addresses from the host here. */
+    memset(&pci_dev->dev.config[PCI_BASE_ADDRESS_0], 0, 24);
+    memset(&pci_dev->dev.config[PCI_ROM_ADDRESS], 0, 4);
+
+    snprintf(name, sizeof(name), "%sresource", dir);
+
+    f = fopen(name, "r");
+    if (f == NULL) {
+        error_report("%s: %s: %m", __func__, name);
+        return 1;
+    }
+
+    for (r = 0; r < PCI_ROM_SLOT; r++) {
+        if (fscanf(f, "%" SCNi64 " %" SCNi64 " %" SCNi64 "\n",
+                   &start, &end, &flags) != 3) {
+            break;
+        }
+
+        rp = dev->regions + r;
+        rp->valid = 0;
+        rp->resource_fd = -1;
+        size = end - start + 1;
+        flags &= IORESOURCE_IO | IORESOURCE_MEM | IORESOURCE_PREFETCH;
+        if (size == 0 || (flags & ~IORESOURCE_PREFETCH) == 0) {
+            continue;
+        }
+        if (flags & IORESOURCE_MEM) {
+            flags &= ~IORESOURCE_IO;
+        } else {
+            flags &= ~IORESOURCE_PREFETCH;
+        }
+        snprintf(name, sizeof(name), "%sresource%d", dir, r);
+        fd = open(name, O_RDWR);
+        if (fd == -1) {
+            continue;
+        }
+        rp->resource_fd = fd;
+
+        rp->type = flags;
+        rp->valid = 1;
+        rp->base_addr = start;
+        rp->size = size;
+        pci_dev->v_addrs[r].region = rp;
+        DEBUG("region %d size %" PRIu64 " start 0x%" PRIx64
+              " type %d resource_fd %d\n",
+              r, rp->size, start, rp->type, rp->resource_fd);
+    }
+
+    fclose(f);
+
+    /* read and fill vendor ID */
+    v = get_real_vendor_id(dir, &id);
+    if (v) {
+        return 1;
+    }
+    pci_dev->dev.config[0] = id & 0xff;
+    pci_dev->dev.config[1] = (id & 0xff00) >> 8;
+
+    /* read and fill device ID */
+    v = get_real_device_id(dir, &id);
+    if (v) {
+        return 1;
+    }
+    pci_dev->dev.config[2] = id & 0xff;
+    pci_dev->dev.config[3] = (id & 0xff00) >> 8;
+
+    pci_word_test_and_clear_mask(pci_dev->emulate_config_write + PCI_COMMAND,
+                                 PCI_COMMAND_MASTER | PCI_COMMAND_INTX_DISABLE);
+
+    dev->region_number = r;
+    return 0;
+}
+
+static void free_msi_virqs(AssignedDevice *dev)
+{
+    int i;
+
+    for (i = 0; i < dev->msi_virq_nr; i++) {
+        if (dev->msi_virq[i] >= 0) {
+            kvm_irqchip_release_virq(kvm_state, dev->msi_virq[i]);
+            dev->msi_virq[i] = -1;
+        }
+    }
+    g_free(dev->msi_virq);
+    dev->msi_virq = NULL;
+    dev->msi_virq_nr = 0;
+}
+
+static void free_assigned_device(AssignedDevice *dev)
+{
+    int i;
+
+    if (dev->cap.available & ASSIGNED_DEVICE_CAP_MSIX) {
+        assigned_dev_unregister_msix_mmio(dev);
+    }
+    for (i = 0; i < dev->real_device.region_number; i++) {
+        PCIRegion *pci_region = &dev->real_device.regions[i];
+        AssignedDevRegion *region = &dev->v_addrs[i];
+
+        if (!pci_region->valid) {
+            continue;
+        }
+        if (pci_region->type & IORESOURCE_IO) {
+            if (region->u.r_baseport) {
+                memory_region_del_subregion(&region->container,
+                                            &region->real_iomem);
+                memory_region_destroy(&region->real_iomem);
+                memory_region_destroy(&region->container);
+            }
+        } else if (pci_region->type & IORESOURCE_MEM) {
+            if (region->u.r_virtbase) {
+                memory_region_del_subregion(&region->container,
+                                            &region->real_iomem);
+
+                /* Remove MSI-X table subregion */
+                if (pci_region->base_addr <= dev->msix_table_addr &&
+                    pci_region->base_addr + pci_region->size >
+                    dev->msix_table_addr) {
+                    memory_region_del_subregion(&region->container,
+                                                &dev->mmio);
+                }
+
+                memory_region_destroy(&region->real_iomem);
+                memory_region_destroy(&region->container);
+                if (munmap(region->u.r_virtbase,
+                           (pci_region->size + 0xFFF) & 0xFFFFF000)) {
+                    error_report("Failed to unmap assigned device region: %s",
+                                 strerror(errno));
+                }
+            }
+        }
+        if (pci_region->resource_fd >= 0) {
+            close(pci_region->resource_fd);
+        }
+    }
+
+    if (dev->real_device.config_fd >= 0) {
+        close(dev->real_device.config_fd);
+    }
+
+    free_msi_virqs(dev);
+}
+
+static void assign_failed_examine(AssignedDevice *dev)
+{
+    char name[PATH_MAX], dir[PATH_MAX], driver[PATH_MAX] = {}, *ns;
+    uint16_t vendor_id, device_id;
+    int r;
+
+    snprintf(dir, sizeof(dir), "/sys/bus/pci/devices/%04x:%02x:%02x.%01x/",
+            dev->host.domain, dev->host.bus, dev->host.slot,
+            dev->host.function);
+
+    snprintf(name, sizeof(name), "%sdriver", dir);
+
+    r = readlink(name, driver, sizeof(driver));
+    if ((r <= 0) || r >= sizeof(driver)) {
+        goto fail;
+    }
+
+    ns = strrchr(driver, '/');
+    if (!ns) {
+        goto fail;
+    }
+
+    ns++;
+
+    if (get_real_vendor_id(dir, &vendor_id) ||
+        get_real_device_id(dir, &device_id)) {
+        goto fail;
+    }
+
+    error_report("*** The driver '%s' is occupying your device "
+                 "%04x:%02x:%02x.%x.",
+                 ns, dev->host.domain, dev->host.bus, dev->host.slot,
+                 dev->host.function);
+    error_report("***");
+    error_report("*** You can try the following commands to free it:");
+    error_report("***");
+    error_report("*** $ echo \"%04x %04x\" > /sys/bus/pci/drivers/pci-stub/"
+                 "new_id", vendor_id, device_id);
+    error_report("*** $ echo \"%04x:%02x:%02x.%x\" > /sys/bus/pci/drivers/"
+                 "%s/unbind",
+                 dev->host.domain, dev->host.bus, dev->host.slot,
+                 dev->host.function, ns);
+    error_report("*** $ echo \"%04x:%02x:%02x.%x\" > /sys/bus/pci/drivers/"
+                 "pci-stub/bind",
+                 dev->host.domain, dev->host.bus, dev->host.slot,
+                 dev->host.function);
+    error_report("*** $ echo \"%04x %04x\" > /sys/bus/pci/drivers/pci-stub"
+                 "/remove_id", vendor_id, device_id);
+    error_report("***");
+
+    return;
+
+fail:
+    error_report("Couldn't find out why.");
+}
+
+static int assign_device(AssignedDevice *dev)
+{
+    uint32_t flags = KVM_DEV_ASSIGN_ENABLE_IOMMU;
+    int r;
+
+    /* Only pass non-zero PCI segment to capable module */
+    if (!kvm_check_extension(kvm_state, KVM_CAP_PCI_SEGMENT) &&
+        dev->host.domain) {
+        error_report("Can't assign device inside non-zero PCI segment "
+                     "as this KVM module doesn't support it.");
+        return -ENODEV;
+    }
+
+    if (!kvm_check_extension(kvm_state, KVM_CAP_IOMMU)) {
+        error_report("No IOMMU found.  Unable to assign device \"%s\"",
+                     dev->dev.qdev.id);
+        return -ENODEV;
+    }
+
+    if (dev->features & ASSIGNED_DEVICE_SHARE_INTX_MASK &&
+        kvm_has_intx_set_mask()) {
+        flags |= KVM_DEV_ASSIGN_PCI_2_3;
+    }
+
+    r = kvm_device_pci_assign(kvm_state, &dev->host, flags, &dev->dev_id);
+    if (r < 0) {
+        error_report("Failed to assign device \"%s\" : %s",
+                     dev->dev.qdev.id, strerror(-r));
+
+        switch (r) {
+        case -EBUSY:
+            assign_failed_examine(dev);
+            break;
+        default:
+            break;
+        }
+    }
+    return r;
+}
+
+static bool check_irqchip_in_kernel(void)
+{
+    if (kvm_irqchip_in_kernel()) {
+        return true;
+    }
+    error_report("pci-assign: error: requires KVM with in-kernel irqchip "
+                 "enabled");
+    return false;
+}
+
+static int assign_intx(AssignedDevice *dev)
+{
+    AssignedIRQType new_type;
+    PCIINTxRoute intx_route;
+    bool intx_host_msi;
+    int r;
+
+    /* Interrupt PIN 0 means don't use INTx */
+    if (assigned_dev_pci_read_byte(&dev->dev, PCI_INTERRUPT_PIN) == 0) {
+        pci_device_set_intx_routing_notifier(&dev->dev, NULL);
+        return 0;
+    }
+
+    if (!check_irqchip_in_kernel()) {
+        return -ENOTSUP;
+    }
+
+    pci_device_set_intx_routing_notifier(&dev->dev,
+                                         assigned_dev_update_irq_routing);
+
+    intx_route = pci_device_route_intx_to_irq(&dev->dev, dev->intpin);
+    assert(intx_route.mode != PCI_INTX_INVERTED);
+
+    if (!pci_intx_route_changed(&dev->intx_route, &intx_route)) {
+        return 0;
+    }
+
+    switch (dev->assigned_irq_type) {
+    case ASSIGNED_IRQ_INTX_HOST_INTX:
+    case ASSIGNED_IRQ_INTX_HOST_MSI:
+        intx_host_msi = dev->assigned_irq_type == ASSIGNED_IRQ_INTX_HOST_MSI;
+        r = kvm_device_intx_deassign(kvm_state, dev->dev_id, intx_host_msi);
+        break;
+    case ASSIGNED_IRQ_MSI:
+        r = kvm_device_msi_deassign(kvm_state, dev->dev_id);
+        break;
+    case ASSIGNED_IRQ_MSIX:
+        r = kvm_device_msix_deassign(kvm_state, dev->dev_id);
+        break;
+    default:
+        r = 0;
+        break;
+    }
+    if (r) {
+        perror("assign_intx: deassignment of previous interrupt failed");
+    }
+    dev->assigned_irq_type = ASSIGNED_IRQ_NONE;
+
+    if (intx_route.mode == PCI_INTX_DISABLED) {
+        dev->intx_route = intx_route;
+        return 0;
+    }
+
+retry:
+    if (dev->features & ASSIGNED_DEVICE_PREFER_MSI_MASK &&
+        dev->cap.available & ASSIGNED_DEVICE_CAP_MSI) {
+        intx_host_msi = true;
+        new_type = ASSIGNED_IRQ_INTX_HOST_MSI;
+    } else {
+        intx_host_msi = false;
+        new_type = ASSIGNED_IRQ_INTX_HOST_INTX;
+    }
+
+    r = kvm_device_intx_assign(kvm_state, dev->dev_id, intx_host_msi,
+                               intx_route.irq);
+    if (r < 0) {
+        if (r == -EIO && !(dev->features & ASSIGNED_DEVICE_PREFER_MSI_MASK) &&
+            dev->cap.available & ASSIGNED_DEVICE_CAP_MSI) {
+            /* Retry with host-side MSI. There might be an IRQ conflict and
+             * either the kernel or the device doesn't support sharing. */
+            error_report("Host-side INTx sharing not supported, "
+                         "using MSI instead.\n"
+                         "Some devices do not to work properly in this mode.");
+            dev->features |= ASSIGNED_DEVICE_PREFER_MSI_MASK;
+            goto retry;
+        }
+        error_report("Failed to assign irq for \"%s\": %s",
+                     dev->dev.qdev.id, strerror(-r));
+        error_report("Perhaps you are assigning a device "
+                     "that shares an IRQ with another device?");
+        return r;
+    }
+
+    dev->intx_route = intx_route;
+    dev->assigned_irq_type = new_type;
+    return r;
+}
+
+static void deassign_device(AssignedDevice *dev)
+{
+    int r;
+
+    r = kvm_device_pci_deassign(kvm_state, dev->dev_id);
+    assert(r == 0);
+}
+
+/* The pci config space got updated. Check if irq numbers have changed
+ * for our devices
+ */
+static void assigned_dev_update_irq_routing(PCIDevice *dev)
+{
+    AssignedDevice *assigned_dev = DO_UPCAST(AssignedDevice, dev, dev);
+    Error *err = NULL;
+    int r;
+
+    r = assign_intx(assigned_dev);
+    if (r < 0) {
+        qdev_unplug(&dev->qdev, &err);
+        assert(!err);
+    }
+}
+
+static void assigned_dev_update_msi(PCIDevice *pci_dev)
+{
+    AssignedDevice *assigned_dev = DO_UPCAST(AssignedDevice, dev, pci_dev);
+    uint8_t ctrl_byte = pci_get_byte(pci_dev->config + pci_dev->msi_cap +
+                                     PCI_MSI_FLAGS);
+    int r;
+
+    /* Some guests gratuitously disable MSI even if they're not using it,
+     * try to catch this by only deassigning irqs if the guest is using
+     * MSI or intends to start. */
+    if (assigned_dev->assigned_irq_type == ASSIGNED_IRQ_MSI ||
+        (ctrl_byte & PCI_MSI_FLAGS_ENABLE)) {
+        r = kvm_device_msi_deassign(kvm_state, assigned_dev->dev_id);
+        /* -ENXIO means no assigned irq */
+        if (r && r != -ENXIO) {
+            perror("assigned_dev_update_msi: deassign irq");
+        }
+
+        free_msi_virqs(assigned_dev);
+
+        assigned_dev->assigned_irq_type = ASSIGNED_IRQ_NONE;
+        pci_device_set_intx_routing_notifier(pci_dev, NULL);
+    }
+
+    if (ctrl_byte & PCI_MSI_FLAGS_ENABLE) {
+        MSIMessage msg = msi_get_message(pci_dev, 0);
+        int virq;
+
+        virq = kvm_irqchip_add_msi_route(kvm_state, msg);
+        if (virq < 0) {
+            perror("assigned_dev_update_msi: kvm_irqchip_add_msi_route");
+            return;
+        }
+
+        assigned_dev->msi_virq = g_malloc(sizeof(*assigned_dev->msi_virq));
+        assigned_dev->msi_virq_nr = 1;
+        assigned_dev->msi_virq[0] = virq;
+        if (kvm_device_msi_assign(kvm_state, assigned_dev->dev_id, virq) < 0) {
+            perror("assigned_dev_update_msi: kvm_device_msi_assign");
+        }
+
+        assigned_dev->intx_route.mode = PCI_INTX_DISABLED;
+        assigned_dev->intx_route.irq = -1;
+        assigned_dev->assigned_irq_type = ASSIGNED_IRQ_MSI;
+    } else {
+        assign_intx(assigned_dev);
+    }
+}
+
+static bool assigned_dev_msix_masked(MSIXTableEntry *entry)
+{
+    return (entry->ctrl & cpu_to_le32(0x1)) != 0;
+}
+
+static int assigned_dev_update_msix_mmio(PCIDevice *pci_dev)
+{
+    AssignedDevice *adev = DO_UPCAST(AssignedDevice, dev, pci_dev);
+    uint16_t entries_nr = 0;
+    int i, r = 0;
+    MSIXTableEntry *entry = adev->msix_table;
+    MSIMessage msg;
+
+    /* Get the usable entry number for allocating */
+    for (i = 0; i < adev->msix_max; i++, entry++) {
+        if (assigned_dev_msix_masked(entry)) {
+            continue;
+        }
+        entries_nr++;
+    }
+
+    DEBUG("MSI-X entries: %d\n", entries_nr);
+
+    /* It's valid to enable MSI-X with all entries masked */
+    if (!entries_nr) {
+        return 0;
+    }
+
+    r = kvm_device_msix_init_vectors(kvm_state, adev->dev_id, entries_nr);
+    if (r != 0) {
+        error_report("fail to set MSI-X entry number for MSIX! %s",
+                     strerror(-r));
+        return r;
+    }
+
+    free_msi_virqs(adev);
+
+    adev->msi_virq_nr = adev->msix_max;
+    adev->msi_virq = g_malloc(adev->msix_max * sizeof(*adev->msi_virq));
+
+    entry = adev->msix_table;
+    for (i = 0; i < adev->msix_max; i++, entry++) {
+        adev->msi_virq[i] = -1;
+
+        if (assigned_dev_msix_masked(entry)) {
+            continue;
+        }
+
+        msg.address = entry->addr_lo | ((uint64_t)entry->addr_hi << 32);
+        msg.data = entry->data;
+        r = kvm_irqchip_add_msi_route(kvm_state, msg);
+        if (r < 0) {
+            return r;
+        }
+        adev->msi_virq[i] = r;
+
+        DEBUG("MSI-X vector %d, gsi %d, addr %08x_%08x, data %08x\n", i,
+              r, entry->addr_hi, entry->addr_lo, entry->data);
+
+        r = kvm_device_msix_set_vector(kvm_state, adev->dev_id, i,
+                                       adev->msi_virq[i]);
+        if (r) {
+            error_report("fail to set MSI-X entry! %s", strerror(-r));
+            break;
+        }
+    }
+
+    return r;
+}
+
+static void assigned_dev_update_msix(PCIDevice *pci_dev)
+{
+    AssignedDevice *assigned_dev = DO_UPCAST(AssignedDevice, dev, pci_dev);
+    uint16_t ctrl_word = pci_get_word(pci_dev->config + pci_dev->msix_cap +
+                                      PCI_MSIX_FLAGS);
+    int r;
+
+    /* Some guests gratuitously disable MSIX even if they're not using it,
+     * try to catch this by only deassigning irqs if the guest is using
+     * MSIX or intends to start. */
+    if ((assigned_dev->assigned_irq_type == ASSIGNED_IRQ_MSIX) ||
+        (ctrl_word & PCI_MSIX_FLAGS_ENABLE)) {
+        r = kvm_device_msix_deassign(kvm_state, assigned_dev->dev_id);
+        /* -ENXIO means no assigned irq */
+        if (r && r != -ENXIO) {
+            perror("assigned_dev_update_msix: deassign irq");
+        }
+
+        free_msi_virqs(assigned_dev);
+
+        assigned_dev->assigned_irq_type = ASSIGNED_IRQ_NONE;
+        pci_device_set_intx_routing_notifier(pci_dev, NULL);
+    }
+
+    if (ctrl_word & PCI_MSIX_FLAGS_ENABLE) {
+        if (assigned_dev_update_msix_mmio(pci_dev) < 0) {
+            perror("assigned_dev_update_msix_mmio");
+            return;
+        }
+
+        if (assigned_dev->msi_virq_nr > 0) {
+            if (kvm_device_msix_assign(kvm_state, assigned_dev->dev_id) < 0) {
+                perror("assigned_dev_enable_msix: assign irq");
+                return;
+            }
+        }
+        assigned_dev->intx_route.mode = PCI_INTX_DISABLED;
+        assigned_dev->intx_route.irq = -1;
+        assigned_dev->assigned_irq_type = ASSIGNED_IRQ_MSIX;
+    } else {
+        assign_intx(assigned_dev);
+    }
+}
+
+static uint32_t assigned_dev_pci_read_config(PCIDevice *pci_dev,
+                                             uint32_t address, int len)
+{
+    AssignedDevice *assigned_dev = DO_UPCAST(AssignedDevice, dev, pci_dev);
+    uint32_t virt_val = pci_default_read_config(pci_dev, address, len);
+    uint32_t real_val, emulate_mask, full_emulation_mask;
+
+    emulate_mask = 0;
+    memcpy(&emulate_mask, assigned_dev->emulate_config_read + address, len);
+    emulate_mask = le32_to_cpu(emulate_mask);
+
+    full_emulation_mask = 0xffffffff >> (32 - len * 8);
+
+    if (emulate_mask != full_emulation_mask) {
+        real_val = assigned_dev_pci_read(pci_dev, address, len);
+        return (virt_val & emulate_mask) | (real_val & ~emulate_mask);
+    } else {
+        return virt_val;
+    }
+}
+
+static void assigned_dev_pci_write_config(PCIDevice *pci_dev, uint32_t address,
+                                          uint32_t val, int len)
+{
+    AssignedDevice *assigned_dev = DO_UPCAST(AssignedDevice, dev, pci_dev);
+    uint16_t old_cmd = pci_get_word(pci_dev->config + PCI_COMMAND);
+    uint32_t emulate_mask, full_emulation_mask;
+    int ret;
+
+    pci_default_write_config(pci_dev, address, val, len);
+
+    if (kvm_has_intx_set_mask() &&
+        range_covers_byte(address, len, PCI_COMMAND + 1)) {
+        bool intx_masked = (pci_get_word(pci_dev->config + PCI_COMMAND) &
+                            PCI_COMMAND_INTX_DISABLE);
+
+        if (intx_masked != !!(old_cmd & PCI_COMMAND_INTX_DISABLE)) {
+            ret = kvm_device_intx_set_mask(kvm_state, assigned_dev->dev_id,
+                                           intx_masked);
+            if (ret) {
+                perror("assigned_dev_pci_write_config: set intx mask");
+            }
+        }
+    }
+    if (assigned_dev->cap.available & ASSIGNED_DEVICE_CAP_MSI) {
+        if (range_covers_byte(address, len,
+                              pci_dev->msi_cap + PCI_MSI_FLAGS)) {
+            assigned_dev_update_msi(pci_dev);
+        }
+    }
+    if (assigned_dev->cap.available & ASSIGNED_DEVICE_CAP_MSIX) {
+        if (range_covers_byte(address, len,
+                              pci_dev->msix_cap + PCI_MSIX_FLAGS + 1)) {
+            assigned_dev_update_msix(pci_dev);
+        }
+    }
+
+    emulate_mask = 0;
+    memcpy(&emulate_mask, assigned_dev->emulate_config_write + address, len);
+    emulate_mask = le32_to_cpu(emulate_mask);
+
+    full_emulation_mask = 0xffffffff >> (32 - len * 8);
+
+    if (emulate_mask != full_emulation_mask) {
+        if (emulate_mask) {
+            val &= ~emulate_mask;
+            val |= assigned_dev_pci_read(pci_dev, address, len) & emulate_mask;
+        }
+        assigned_dev_pci_write(pci_dev, address, val, len);
+    }
+}
+
+static void assigned_dev_setup_cap_read(AssignedDevice *dev, uint32_t offset,
+                                        uint32_t len)
+{
+    assigned_dev_direct_config_read(dev, offset, len);
+    assigned_dev_emulate_config_read(dev, offset + PCI_CAP_LIST_NEXT, 1);
+}
+
+static int assigned_device_pci_cap_init(PCIDevice *pci_dev)
+{
+    AssignedDevice *dev = DO_UPCAST(AssignedDevice, dev, pci_dev);
+    PCIRegion *pci_region = dev->real_device.regions;
+    int ret, pos;
+
+    /* Clear initial capabilities pointer and status copied from hw */
+    pci_set_byte(pci_dev->config + PCI_CAPABILITY_LIST, 0);
+    pci_set_word(pci_dev->config + PCI_STATUS,
+                 pci_get_word(pci_dev->config + PCI_STATUS) &
+                 ~PCI_STATUS_CAP_LIST);
+
+    /* Expose MSI capability
+     * MSI capability is the 1st capability in capability config */
+    pos = pci_find_cap_offset(pci_dev, PCI_CAP_ID_MSI, 0);
+    if (pos != 0 && kvm_check_extension(kvm_state, KVM_CAP_ASSIGN_DEV_IRQ)) {
+        if (!check_irqchip_in_kernel()) {
+            return -ENOTSUP;
+        }
+        dev->cap.available |= ASSIGNED_DEVICE_CAP_MSI;
+        /* Only 32-bit/no-mask currently supported */
+        ret = pci_add_capability(pci_dev, PCI_CAP_ID_MSI, pos, 10);
+        if (ret < 0) {
+            return ret;
+        }
+        pci_dev->msi_cap = pos;
+
+        pci_set_word(pci_dev->config + pos + PCI_MSI_FLAGS,
+                     pci_get_word(pci_dev->config + pos + PCI_MSI_FLAGS) &
+                     PCI_MSI_FLAGS_QMASK);
+        pci_set_long(pci_dev->config + pos + PCI_MSI_ADDRESS_LO, 0);
+        pci_set_word(pci_dev->config + pos + PCI_MSI_DATA_32, 0);
+
+        /* Set writable fields */
+        pci_set_word(pci_dev->wmask + pos + PCI_MSI_FLAGS,
+                     PCI_MSI_FLAGS_QSIZE | PCI_MSI_FLAGS_ENABLE);
+        pci_set_long(pci_dev->wmask + pos + PCI_MSI_ADDRESS_LO, 0xfffffffc);
+        pci_set_word(pci_dev->wmask + pos + PCI_MSI_DATA_32, 0xffff);
+    }
+    /* Expose MSI-X capability */
+    pos = pci_find_cap_offset(pci_dev, PCI_CAP_ID_MSIX, 0);
+    if (pos != 0 && kvm_device_msix_supported(kvm_state)) {
+        int bar_nr;
+        uint32_t msix_table_entry;
+
+        if (!check_irqchip_in_kernel()) {
+            return -ENOTSUP;
+        }
+        dev->cap.available |= ASSIGNED_DEVICE_CAP_MSIX;
+        ret = pci_add_capability(pci_dev, PCI_CAP_ID_MSIX, pos, 12);
+        if (ret < 0) {
+            return ret;
+        }
+        pci_dev->msix_cap = pos;
+
+        pci_set_word(pci_dev->config + pos + PCI_MSIX_FLAGS,
+                     pci_get_word(pci_dev->config + pos + PCI_MSIX_FLAGS) &
+                     PCI_MSIX_FLAGS_QSIZE);
+
+        /* Only enable and function mask bits are writable */
+        pci_set_word(pci_dev->wmask + pos + PCI_MSIX_FLAGS,
+                     PCI_MSIX_FLAGS_ENABLE | PCI_MSIX_FLAGS_MASKALL);
+
+        msix_table_entry = pci_get_long(pci_dev->config + pos + PCI_MSIX_TABLE);
+        bar_nr = msix_table_entry & PCI_MSIX_FLAGS_BIRMASK;
+        msix_table_entry &= ~PCI_MSIX_FLAGS_BIRMASK;
+        dev->msix_table_addr = pci_region[bar_nr].base_addr + msix_table_entry;
+        dev->msix_max = pci_get_word(pci_dev->config + pos + PCI_MSIX_FLAGS);
+        dev->msix_max &= PCI_MSIX_FLAGS_QSIZE;
+        dev->msix_max += 1;
+    }
+
+    /* Minimal PM support, nothing writable, device appears to NAK changes */
+    pos = pci_find_cap_offset(pci_dev, PCI_CAP_ID_PM, 0);
+    if (pos) {
+        uint16_t pmc;
+
+        ret = pci_add_capability(pci_dev, PCI_CAP_ID_PM, pos, PCI_PM_SIZEOF);
+        if (ret < 0) {
+            return ret;
+        }
+
+        assigned_dev_setup_cap_read(dev, pos, PCI_PM_SIZEOF);
+
+        pmc = pci_get_word(pci_dev->config + pos + PCI_CAP_FLAGS);
+        pmc &= (PCI_PM_CAP_VER_MASK | PCI_PM_CAP_DSI);
+        pci_set_word(pci_dev->config + pos + PCI_CAP_FLAGS, pmc);
+
+        /* assign_device will bring the device up to D0, so we don't need
+         * to worry about doing that ourselves here. */
+        pci_set_word(pci_dev->config + pos + PCI_PM_CTRL,
+                     PCI_PM_CTRL_NO_SOFT_RESET);
+
+        pci_set_byte(pci_dev->config + pos + PCI_PM_PPB_EXTENSIONS, 0);
+        pci_set_byte(pci_dev->config + pos + PCI_PM_DATA_REGISTER, 0);
+    }
+
+    pos = pci_find_cap_offset(pci_dev, PCI_CAP_ID_EXP, 0);
+    if (pos) {
+        uint8_t version, size = 0;
+        uint16_t type, devctl, lnksta;
+        uint32_t devcap, lnkcap;
+
+        version = pci_get_byte(pci_dev->config + pos + PCI_EXP_FLAGS);
+        version &= PCI_EXP_FLAGS_VERS;
+        if (version == 1) {
+            size = 0x14;
+        } else if (version == 2) {
+            /*
+             * Check for non-std size, accept reduced size to 0x34,
+             * which is what bcm5761 implemented, violating the
+             * PCIe v3.0 spec that regs should exist and be read as 0,
+             * not optionally provided and shorten the struct size.
+             */
+            size = MIN(0x3c, PCI_CONFIG_SPACE_SIZE - pos);
+            if (size < 0x34) {
+                error_report("%s: Invalid size PCIe cap-id 0x%x",
+                             __func__, PCI_CAP_ID_EXP);
+                return -EINVAL;
+            } else if (size != 0x3c) {
+                error_report("WARNING, %s: PCIe cap-id 0x%x has "
+                             "non-standard size 0x%x; std size should be 0x3c",
+                             __func__, PCI_CAP_ID_EXP, size);
+            }
+        } else if (version == 0) {
+            uint16_t vid, did;
+            vid = pci_get_word(pci_dev->config + PCI_VENDOR_ID);
+            did = pci_get_word(pci_dev->config + PCI_DEVICE_ID);
+            if (vid == PCI_VENDOR_ID_INTEL && did == 0x10ed) {
+                /*
+                 * quirk for Intel 82599 VF with invalid PCIe capability
+                 * version, should really be version 2 (same as PF)
+                 */
+                size = 0x3c;
+            }
+        }
+
+        if (size == 0) {
+            error_report("%s: Unsupported PCI express capability version %d",
+                         __func__, version);
+            return -EINVAL;
+        }
+
+        ret = pci_add_capability(pci_dev, PCI_CAP_ID_EXP, pos, size);
+        if (ret < 0) {
+            return ret;
+        }
+
+        assigned_dev_setup_cap_read(dev, pos, size);
+
+        type = pci_get_word(pci_dev->config + pos + PCI_EXP_FLAGS);
+        type = (type & PCI_EXP_FLAGS_TYPE) >> 4;
+        if (type != PCI_EXP_TYPE_ENDPOINT &&
+            type != PCI_EXP_TYPE_LEG_END && type != PCI_EXP_TYPE_RC_END) {
+            error_report("Device assignment only supports endpoint assignment,"
+                         " device type %d", type);
+            return -EINVAL;
+        }
+
+        /* capabilities, pass existing read-only copy
+         * PCI_EXP_FLAGS_IRQ: updated by hardware, should be direct read */
+
+        /* device capabilities: hide FLR */
+        devcap = pci_get_long(pci_dev->config + pos + PCI_EXP_DEVCAP);
+        devcap &= ~PCI_EXP_DEVCAP_FLR;
+        pci_set_long(pci_dev->config + pos + PCI_EXP_DEVCAP, devcap);
+
+        /* device control: clear all error reporting enable bits, leaving
+         *                 only a few host values.  Note, these are
+         *                 all writable, but not passed to hw.
+         */
+        devctl = pci_get_word(pci_dev->config + pos + PCI_EXP_DEVCTL);
+        devctl = (devctl & (PCI_EXP_DEVCTL_READRQ | PCI_EXP_DEVCTL_PAYLOAD)) |
+                  PCI_EXP_DEVCTL_RELAX_EN | PCI_EXP_DEVCTL_NOSNOOP_EN;
+        pci_set_word(pci_dev->config + pos + PCI_EXP_DEVCTL, devctl);
+        devctl = PCI_EXP_DEVCTL_BCR_FLR | PCI_EXP_DEVCTL_AUX_PME;
+        pci_set_word(pci_dev->wmask + pos + PCI_EXP_DEVCTL, ~devctl);
+
+        /* Clear device status */
+        pci_set_word(pci_dev->config + pos + PCI_EXP_DEVSTA, 0);
+
+        /* Link capabilities, expose links and latencues, clear reporting */
+        lnkcap = pci_get_long(pci_dev->config + pos + PCI_EXP_LNKCAP);
+        lnkcap &= (PCI_EXP_LNKCAP_SLS | PCI_EXP_LNKCAP_MLW |
+                   PCI_EXP_LNKCAP_ASPMS | PCI_EXP_LNKCAP_L0SEL |
+                   PCI_EXP_LNKCAP_L1EL);
+        pci_set_long(pci_dev->config + pos + PCI_EXP_LNKCAP, lnkcap);
+
+        /* Link control, pass existing read-only copy.  Should be writable? */
+
+        /* Link status, only expose current speed and width */
+        lnksta = pci_get_word(pci_dev->config + pos + PCI_EXP_LNKSTA);
+        lnksta &= (PCI_EXP_LNKSTA_CLS | PCI_EXP_LNKSTA_NLW);
+        pci_set_word(pci_dev->config + pos + PCI_EXP_LNKSTA, lnksta);
+
+        if (version >= 2) {
+            /* Slot capabilities, control, status - not needed for endpoints */
+            pci_set_long(pci_dev->config + pos + PCI_EXP_SLTCAP, 0);
+            pci_set_word(pci_dev->config + pos + PCI_EXP_SLTCTL, 0);
+            pci_set_word(pci_dev->config + pos + PCI_EXP_SLTSTA, 0);
+
+            /* Root control, capabilities, status - not needed for endpoints */
+            pci_set_word(pci_dev->config + pos + PCI_EXP_RTCTL, 0);
+            pci_set_word(pci_dev->config + pos + PCI_EXP_RTCAP, 0);
+            pci_set_long(pci_dev->config + pos + PCI_EXP_RTSTA, 0);
+
+            /* Device capabilities/control 2, pass existing read-only copy */
+            /* Link control 2, pass existing read-only copy */
+        }
+    }
+
+    pos = pci_find_cap_offset(pci_dev, PCI_CAP_ID_PCIX, 0);
+    if (pos) {
+        uint16_t cmd;
+        uint32_t status;
+
+        /* Only expose the minimum, 8 byte capability */
+        ret = pci_add_capability(pci_dev, PCI_CAP_ID_PCIX, pos, 8);
+        if (ret < 0) {
+            return ret;
+        }
+
+        assigned_dev_setup_cap_read(dev, pos, 8);
+
+        /* Command register, clear upper bits, including extended modes */
+        cmd = pci_get_word(pci_dev->config + pos + PCI_X_CMD);
+        cmd &= (PCI_X_CMD_DPERR_E | PCI_X_CMD_ERO | PCI_X_CMD_MAX_READ |
+                PCI_X_CMD_MAX_SPLIT);
+        pci_set_word(pci_dev->config + pos + PCI_X_CMD, cmd);
+
+        /* Status register, update with emulated PCI bus location, clear
+         * error bits, leave the rest. */
+        status = pci_get_long(pci_dev->config + pos + PCI_X_STATUS);
+        status &= ~(PCI_X_STATUS_BUS | PCI_X_STATUS_DEVFN);
+        status |= (pci_bus_num(pci_dev->bus) << 8) | pci_dev->devfn;
+        status &= ~(PCI_X_STATUS_SPL_DISC | PCI_X_STATUS_UNX_SPL |
+                    PCI_X_STATUS_SPL_ERR);
+        pci_set_long(pci_dev->config + pos + PCI_X_STATUS, status);
+    }
+
+    pos = pci_find_cap_offset(pci_dev, PCI_CAP_ID_VPD, 0);
+    if (pos) {
+        /* Direct R/W passthrough */
+        ret = pci_add_capability(pci_dev, PCI_CAP_ID_VPD, pos, 8);
+        if (ret < 0) {
+            return ret;
+        }
+
+        assigned_dev_setup_cap_read(dev, pos, 8);
+
+        /* direct write for cap content */
+        assigned_dev_direct_config_write(dev, pos + 2, 6);
+    }
+
+    /* Devices can have multiple vendor capabilities, get them all */
+    for (pos = 0; (pos = pci_find_cap_offset(pci_dev, PCI_CAP_ID_VNDR, pos));
+        pos += PCI_CAP_LIST_NEXT) {
+        uint8_t len = pci_get_byte(pci_dev->config + pos + PCI_CAP_FLAGS);
+        /* Direct R/W passthrough */
+        ret = pci_add_capability(pci_dev, PCI_CAP_ID_VNDR, pos, len);
+        if (ret < 0) {
+            return ret;
+        }
+
+        assigned_dev_setup_cap_read(dev, pos, len);
+
+        /* direct write for cap content */
+        assigned_dev_direct_config_write(dev, pos + 2, len - 2);
+    }
+
+    /* If real and virtual capability list status bits differ, virtualize the
+     * access. */
+    if ((pci_get_word(pci_dev->config + PCI_STATUS) & PCI_STATUS_CAP_LIST) !=
+        (assigned_dev_pci_read_byte(pci_dev, PCI_STATUS) &
+         PCI_STATUS_CAP_LIST)) {
+        dev->emulate_config_read[PCI_STATUS] |= PCI_STATUS_CAP_LIST;
+    }
+
+    return 0;
+}
+
+static uint64_t
+assigned_dev_msix_mmio_read(void *opaque, hwaddr addr,
+                            unsigned size)
+{
+    AssignedDevice *adev = opaque;
+    uint64_t val;
+
+    memcpy(&val, (void *)((uint8_t *)adev->msix_table + addr), size);
+
+    return val;
+}
+
+static void assigned_dev_msix_mmio_write(void *opaque, hwaddr addr,
+                                         uint64_t val, unsigned size)
+{
+    AssignedDevice *adev = opaque;
+    PCIDevice *pdev = &adev->dev;
+    uint16_t ctrl;
+    MSIXTableEntry orig;
+    int i = addr >> 4;
+
+    if (i >= adev->msix_max) {
+        return; /* Drop write */
+    }
+
+    ctrl = pci_get_word(pdev->config + pdev->msix_cap + PCI_MSIX_FLAGS);
+
+    DEBUG("write to MSI-X table offset 0x%lx, val 0x%lx\n", addr, val);
+
+    if (ctrl & PCI_MSIX_FLAGS_ENABLE) {
+        orig = adev->msix_table[i];
+    }
+
+    memcpy((uint8_t *)adev->msix_table + addr, &val, size);
+
+    if (ctrl & PCI_MSIX_FLAGS_ENABLE) {
+        MSIXTableEntry *entry = &adev->msix_table[i];
+
+        if (!assigned_dev_msix_masked(&orig) &&
+            assigned_dev_msix_masked(entry)) {
+            /*
+             * Vector masked, disable it
+             *
+             * XXX It's not clear if we can or should actually attempt
+             * to mask or disable the interrupt.  KVM doesn't have
+             * support for pending bits and kvm_assign_set_msix_entry
+             * doesn't modify the device hardware mask.  Interrupts
+             * while masked are simply not injected to the guest, so
+             * are lost.  Can we get away with always injecting an
+             * interrupt on unmask?
+             */
+        } else if (assigned_dev_msix_masked(&orig) &&
+                   !assigned_dev_msix_masked(entry)) {
+            /* Vector unmasked */
+            if (i >= adev->msi_virq_nr || adev->msi_virq[i] < 0) {
+                /* Previously unassigned vector, start from scratch */
+                assigned_dev_update_msix(pdev);
+                return;
+            } else {
+                /* Update an existing, previously masked vector */
+                MSIMessage msg;
+                int ret;
+
+                msg.address = entry->addr_lo |
+                    ((uint64_t)entry->addr_hi << 32);
+                msg.data = entry->data;
+
+                ret = kvm_irqchip_update_msi_route(kvm_state,
+                                                   adev->msi_virq[i], msg);
+                if (ret) {
+                    error_report("Error updating irq routing entry (%d)", ret);
+                }
+            }
+        }
+    }
+}
+
+static const MemoryRegionOps assigned_dev_msix_mmio_ops = {
+    .read = assigned_dev_msix_mmio_read,
+    .write = assigned_dev_msix_mmio_write,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+    .valid = {
+        .min_access_size = 4,
+        .max_access_size = 8,
+    },
+    .impl = {
+        .min_access_size = 4,
+        .max_access_size = 8,
+    },
+};
+
+static void assigned_dev_msix_reset(AssignedDevice *dev)
+{
+    MSIXTableEntry *entry;
+    int i;
+
+    if (!dev->msix_table) {
+        return;
+    }
+
+    memset(dev->msix_table, 0, MSIX_PAGE_SIZE);
+
+    for (i = 0, entry = dev->msix_table; i < dev->msix_max; i++, entry++) {
+        entry->ctrl = cpu_to_le32(0x1); /* Masked */
+    }
+}
+
+static int assigned_dev_register_msix_mmio(AssignedDevice *dev)
+{
+    dev->msix_table = mmap(NULL, MSIX_PAGE_SIZE, PROT_READ|PROT_WRITE,
+                           MAP_ANONYMOUS|MAP_PRIVATE, 0, 0);
+    if (dev->msix_table == MAP_FAILED) {
+        error_report("fail allocate msix_table! %s", strerror(errno));
+        return -EFAULT;
+    }
+
+    assigned_dev_msix_reset(dev);
+
+    memory_region_init_io(&dev->mmio, &assigned_dev_msix_mmio_ops, dev,
+                          "assigned-dev-msix", MSIX_PAGE_SIZE);
+    return 0;
+}
+
+static void assigned_dev_unregister_msix_mmio(AssignedDevice *dev)
+{
+    if (!dev->msix_table) {
+        return;
+    }
+
+    memory_region_destroy(&dev->mmio);
+
+    if (munmap(dev->msix_table, MSIX_PAGE_SIZE) == -1) {
+        error_report("error unmapping msix_table! %s", strerror(errno));
+    }
+    dev->msix_table = NULL;
+}
+
+static const VMStateDescription vmstate_assigned_device = {
+    .name = "pci-assign",
+    .unmigratable = 1,
+};
+
+static void reset_assigned_device(DeviceState *dev)
+{
+    PCIDevice *pci_dev = DO_UPCAST(PCIDevice, qdev, dev);
+    AssignedDevice *adev = DO_UPCAST(AssignedDevice, dev, pci_dev);
+    char reset_file[64];
+    const char reset[] = "1";
+    int fd, ret;
+
+    /*
+     * If a guest is reset without being shutdown, MSI/MSI-X can still
+     * be running.  We want to return the device to a known state on
+     * reset, so disable those here.  We especially do not want MSI-X
+     * enabled since it lives in MMIO space, which is about to get
+     * disabled.
+     */
+    if (adev->assigned_irq_type == ASSIGNED_IRQ_MSIX) {
+        uint16_t ctrl = pci_get_word(pci_dev->config +
+                                     pci_dev->msix_cap + PCI_MSIX_FLAGS);
+
+        pci_set_word(pci_dev->config + pci_dev->msix_cap + PCI_MSIX_FLAGS,
+                     ctrl & ~PCI_MSIX_FLAGS_ENABLE);
+        assigned_dev_update_msix(pci_dev);
+    } else if (adev->assigned_irq_type == ASSIGNED_IRQ_MSI) {
+        uint8_t ctrl = pci_get_byte(pci_dev->config +
+                                    pci_dev->msi_cap + PCI_MSI_FLAGS);
+
+        pci_set_byte(pci_dev->config + pci_dev->msi_cap + PCI_MSI_FLAGS,
+                     ctrl & ~PCI_MSI_FLAGS_ENABLE);
+        assigned_dev_update_msi(pci_dev);
+    }
+
+    snprintf(reset_file, sizeof(reset_file),
+             "/sys/bus/pci/devices/%04x:%02x:%02x.%01x/reset",
+             adev->host.domain, adev->host.bus, adev->host.slot,
+             adev->host.function);
+
+    /*
+     * Issue a device reset via pci-sysfs.  Note that we use write(2) here
+     * and ignore the return value because some kernels have a bug that
+     * returns 0 rather than bytes written on success, sending us into an
+     * infinite retry loop using other write mechanisms.
+     */
+    fd = open(reset_file, O_WRONLY);
+    if (fd != -1) {
+        ret = write(fd, reset, strlen(reset));
+        (void)ret;
+        close(fd);
+    }
+
+    /*
+     * When a 0 is written to the bus master register, the device is logically
+     * disconnected from the PCI bus. This avoids further DMA transfers.
+     */
+    assigned_dev_pci_write_config(pci_dev, PCI_COMMAND, 0, 1);
+}
+
+static int assigned_initfn(struct PCIDevice *pci_dev)
+{
+    AssignedDevice *dev = DO_UPCAST(AssignedDevice, dev, pci_dev);
+    uint8_t e_intx;
+    int r;
+
+    if (!kvm_enabled()) {
+        error_report("pci-assign: error: requires KVM support");
+        return -1;
+    }
+
+    if (!dev->host.domain && !dev->host.bus && !dev->host.slot &&
+        !dev->host.function) {
+        error_report("pci-assign: error: no host device specified");
+        return -1;
+    }
+
+    /*
+     * Set up basic config space access control. Will be further refined during
+     * device initialization.
+     */
+    assigned_dev_emulate_config_read(dev, 0, PCI_CONFIG_SPACE_SIZE);
+    assigned_dev_direct_config_read(dev, PCI_STATUS, 2);
+    assigned_dev_direct_config_read(dev, PCI_REVISION_ID, 1);
+    assigned_dev_direct_config_read(dev, PCI_CLASS_PROG, 3);
+    assigned_dev_direct_config_read(dev, PCI_CACHE_LINE_SIZE, 1);
+    assigned_dev_direct_config_read(dev, PCI_LATENCY_TIMER, 1);
+    assigned_dev_direct_config_read(dev, PCI_BIST, 1);
+    assigned_dev_direct_config_read(dev, PCI_CARDBUS_CIS, 4);
+    assigned_dev_direct_config_read(dev, PCI_SUBSYSTEM_VENDOR_ID, 2);
+    assigned_dev_direct_config_read(dev, PCI_SUBSYSTEM_ID, 2);
+    assigned_dev_direct_config_read(dev, PCI_CAPABILITY_LIST + 1, 7);
+    assigned_dev_direct_config_read(dev, PCI_MIN_GNT, 1);
+    assigned_dev_direct_config_read(dev, PCI_MAX_LAT, 1);
+    memcpy(dev->emulate_config_write, dev->emulate_config_read,
+           sizeof(dev->emulate_config_read));
+
+    if (get_real_device(dev, dev->host.domain, dev->host.bus,
+                        dev->host.slot, dev->host.function)) {
+        error_report("pci-assign: Error: Couldn't get real device (%s)!",
+                     dev->dev.qdev.id);
+        goto out;
+    }
+
+    if (assigned_device_pci_cap_init(pci_dev) < 0) {
+        goto out;
+    }
+
+    /* intercept MSI-X entry page in the MMIO */
+    if (dev->cap.available & ASSIGNED_DEVICE_CAP_MSIX) {
+        if (assigned_dev_register_msix_mmio(dev)) {
+            goto out;
+        }
+    }
+
+    /* handle real device's MMIO/PIO BARs */
+    if (assigned_dev_register_regions(dev->real_device.regions,
+                                      dev->real_device.region_number,
+                                      dev)) {
+        goto out;
+    }
+
+    /* handle interrupt routing */
+    e_intx = dev->dev.config[PCI_INTERRUPT_PIN] - 1;
+    dev->intpin = e_intx;
+    dev->intx_route.mode = PCI_INTX_DISABLED;
+    dev->intx_route.irq = -1;
+
+    /* assign device to guest */
+    r = assign_device(dev);
+    if (r < 0) {
+        goto out;
+    }
+
+    /* assign legacy INTx to the device */
+    r = assign_intx(dev);
+    if (r < 0) {
+        goto assigned_out;
+    }
+
+    assigned_dev_load_option_rom(dev);
+
+    add_boot_device_path(dev->bootindex, &pci_dev->qdev, NULL);
+
+    return 0;
+
+assigned_out:
+    deassign_device(dev);
+out:
+    free_assigned_device(dev);
+    return -1;
+}
+
+static void assigned_exitfn(struct PCIDevice *pci_dev)
+{
+    AssignedDevice *dev = DO_UPCAST(AssignedDevice, dev, pci_dev);
+
+    deassign_device(dev);
+    free_assigned_device(dev);
+}
+
+static Property assigned_dev_properties[] = {
+    DEFINE_PROP_PCI_HOST_DEVADDR("host", AssignedDevice, host),
+    DEFINE_PROP_BIT("prefer_msi", AssignedDevice, features,
+                    ASSIGNED_DEVICE_PREFER_MSI_BIT, false),
+    DEFINE_PROP_BIT("share_intx", AssignedDevice, features,
+                    ASSIGNED_DEVICE_SHARE_INTX_BIT, true),
+    DEFINE_PROP_INT32("bootindex", AssignedDevice, bootindex, -1),
+    DEFINE_PROP_STRING("configfd", AssignedDevice, configfd_name),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void assign_class_init(ObjectClass *klass, void *data)
+{
+    PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    k->init         = assigned_initfn;
+    k->exit         = assigned_exitfn;
+    k->config_read  = assigned_dev_pci_read_config;
+    k->config_write = assigned_dev_pci_write_config;
+    dc->props       = assigned_dev_properties;
+    dc->vmsd        = &vmstate_assigned_device;
+    dc->reset       = reset_assigned_device;
+    dc->desc        = "KVM-based PCI passthrough";
+}
+
+static const TypeInfo assign_info = {
+    .name               = "kvm-pci-assign",
+    .parent             = TYPE_PCI_DEVICE,
+    .instance_size      = sizeof(AssignedDevice),
+    .class_init         = assign_class_init,
+};
+
+static void assign_register_types(void)
+{
+    type_register_static(&assign_info);
+}
+
+type_init(assign_register_types)
+
+/*
+ * Scan the assigned devices for the devices that have an option ROM, and then
+ * load the corresponding ROM data to RAM. If an error occurs while loading an
+ * option ROM, we just ignore that option ROM and continue with the next one.
+ */
+static void assigned_dev_load_option_rom(AssignedDevice *dev)
+{
+    char name[32], rom_file[64];
+    FILE *fp;
+    uint8_t val;
+    struct stat st;
+    void *ptr;
+
+    /* If loading ROM from file, pci handles it */
+    if (dev->dev.romfile || !dev->dev.rom_bar) {
+        return;
+    }
+
+    snprintf(rom_file, sizeof(rom_file),
+             "/sys/bus/pci/devices/%04x:%02x:%02x.%01x/rom",
+             dev->host.domain, dev->host.bus, dev->host.slot,
+             dev->host.function);
+
+    if (stat(rom_file, &st)) {
+        return;
+    }
+
+    if (access(rom_file, F_OK)) {
+        error_report("pci-assign: Insufficient privileges for %s", rom_file);
+        return;
+    }
+
+    /* Write "1" to the ROM file to enable it */
+    fp = fopen(rom_file, "r+");
+    if (fp == NULL) {
+        return;
+    }
+    val = 1;
+    if (fwrite(&val, 1, 1, fp) != 1) {
+        goto close_rom;
+    }
+    fseek(fp, 0, SEEK_SET);
+
+    snprintf(name, sizeof(name), "%s.rom",
+            object_get_typename(OBJECT(dev)));
+    memory_region_init_ram(&dev->dev.rom, name, st.st_size);
+    vmstate_register_ram(&dev->dev.rom, &dev->dev.qdev);
+    ptr = memory_region_get_ram_ptr(&dev->dev.rom);
+    memset(ptr, 0xff, st.st_size);
+
+    if (!fread(ptr, 1, st.st_size, fp)) {
+        error_report("pci-assign: Cannot read from host %s\n"
+                     "\tDevice option ROM contents are probably invalid "
+                     "(check dmesg).\n\tSkip option ROM probe with rombar=0, "
+                     "or load from file with romfile=", rom_file);
+        memory_region_destroy(&dev->dev.rom);
+        goto close_rom;
+    }
+
+    pci_register_bar(&dev->dev, PCI_ROM_SLOT, 0, &dev->dev.rom);
+    dev->dev.has_rom = true;
+close_rom:
+    /* Write "0" to disable ROM */
+    fseek(fp, 0, SEEK_SET);
+    val = 0;
+    if (!fwrite(&val, 1, 1, fp)) {
+        DEBUG("%s\n", "Failed to disable pci-sysfs rom file");
+    }
+    fclose(fp);
+}
diff --git a/hw/kvmvapic.c b/hw/kvmvapic.c
index 5d83625f4..e04c4011d 100644
--- a/hw/kvmvapic.c
+++ b/hw/kvmvapic.c
@@ -144,7 +144,7 @@ static void update_guest_rom_state(VAPICROMState *s)
 
 static int find_real_tpr_addr(VAPICROMState *s, CPUX86State *env)
 {
-    target_phys_addr_t paddr;
+    hwaddr paddr;
     target_ulong addr;
 
     if (s->state == VAPIC_ACTIVE) {
@@ -269,7 +269,7 @@ instruction_ok:
 
 static int update_rom_mapping(VAPICROMState *s, CPUX86State *env, target_ulong ip)
 {
-    target_phys_addr_t paddr;
+    hwaddr paddr;
     uint32_t rom_state_vaddr;
     uint32_t pos, patch, offset;
 
@@ -350,14 +350,14 @@ static int get_kpcr_number(CPUX86State *env)
 static int vapic_enable(VAPICROMState *s, CPUX86State *env)
 {
     int cpu_number = get_kpcr_number(env);
-    target_phys_addr_t vapic_paddr;
+    hwaddr vapic_paddr;
     static const uint8_t enabled = 1;
 
     if (cpu_number < 0) {
         return -1;
     }
     vapic_paddr = s->vapic_paddr +
-        (((target_phys_addr_t)cpu_number) << VAPIC_CPU_SHIFT);
+        (((hwaddr)cpu_number) << VAPIC_CPU_SHIFT);
     cpu_physical_memory_rw(vapic_paddr + offsetof(VAPICState, enabled),
                            (void *)&enabled, sizeof(enabled), 1);
     apic_enable_vapic(env->apic_state, vapic_paddr);
@@ -384,10 +384,13 @@ static void patch_call(VAPICROMState *s, CPUX86State *env, target_ulong ip,
 
 static void patch_instruction(VAPICROMState *s, CPUX86State *env, target_ulong ip)
 {
-    target_phys_addr_t paddr;
     VAPICHandlers *handlers;
     uint8_t opcode[2];
     uint32_t imm32;
+    TranslationBlock *current_tb;
+    target_ulong current_pc = 0;
+    target_ulong current_cs_base = 0;
+    int current_flags = 0;
 
     if (smp_cpus == 1) {
         handlers = &s->rom_state.up;
@@ -395,6 +398,13 @@ static void patch_instruction(VAPICROMState *s, CPUX86State *env, target_ulong i
         handlers = &s->rom_state.mp;
     }
 
+    if (!kvm_enabled()) {
+        current_tb = tb_find_pc(env->mem_io_pc);
+        cpu_restore_state(current_tb, env, env->mem_io_pc);
+        cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
+                             &current_flags);
+    }
+
     pause_all_vcpus();
 
     cpu_memory_rw_debug(env, ip, opcode, sizeof(opcode), 0);
@@ -430,9 +440,11 @@ static void patch_instruction(VAPICROMState *s, CPUX86State *env, target_ulong i
 
     resume_all_vcpus();
 
-    paddr = cpu_get_phys_page_debug(env, ip);
-    paddr += ip & ~TARGET_PAGE_MASK;
-    tb_invalidate_phys_page_range(paddr, paddr + 1, 1);
+    if (!kvm_enabled()) {
+        env->current_tb = NULL;
+        tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
+        cpu_resume_from_signal(env, NULL);
+    }
 }
 
 void vapic_report_tpr_access(DeviceState *dev, void *cpu, target_ulong ip,
@@ -475,11 +487,13 @@ static void vapic_enable_tpr_reporting(bool enable)
     VAPICEnableTPRReporting info = {
         .enable = enable,
     };
+    X86CPU *cpu;
     CPUX86State *env;
 
     for (env = first_cpu; env != NULL; env = env->next_cpu) {
+        cpu = x86_env_get_cpu(env);
         info.apic = env->apic_state;
-        run_on_cpu(env, vapic_do_enable_tpr_reporting, &info);
+        run_on_cpu(CPU(cpu), vapic_do_enable_tpr_reporting, &info);
     }
 }
 
@@ -500,7 +514,7 @@ static void vapic_reset(DeviceState *dev)
  */
 static int patch_hypercalls(VAPICROMState *s)
 {
-    target_phys_addr_t rom_paddr = s->rom_state_paddr & ROM_BLOCK_MASK;
+    hwaddr rom_paddr = s->rom_state_paddr & ROM_BLOCK_MASK;
     static const uint8_t vmcall_pattern[] = { /* vmcall */
         0xb8, 0x1, 0, 0, 0, 0xf, 0x1, 0xc1
     };
@@ -557,7 +571,7 @@ static int patch_hypercalls(VAPICROMState *s)
  */
 static void vapic_map_rom_writable(VAPICROMState *s)
 {
-    target_phys_addr_t rom_paddr = s->rom_state_paddr & ROM_BLOCK_MASK;
+    hwaddr rom_paddr = s->rom_state_paddr & ROM_BLOCK_MASK;
     MemoryRegionSection section;
     MemoryRegion *as;
     size_t rom_size;
@@ -603,11 +617,11 @@ static int vapic_prepare(VAPICROMState *s)
     return 0;
 }
 
-static void vapic_write(void *opaque, target_phys_addr_t addr, uint64_t data,
+static void vapic_write(void *opaque, hwaddr addr, uint64_t data,
                         unsigned int size)
 {
     CPUX86State *env = cpu_single_env;
-    target_phys_addr_t rom_paddr;
+    hwaddr rom_paddr;
     VAPICROMState *s = opaque;
 
     cpu_synchronize_state(env);
@@ -717,7 +731,7 @@ static int vapic_post_load(void *opaque, int version_id)
     }
     if (s->state == VAPIC_ACTIVE) {
         if (smp_cpus == 1) {
-            run_on_cpu(first_cpu, do_vapic_enable, s);
+            run_on_cpu(ENV_GET_CPU(first_cpu), do_vapic_enable, s);
         } else {
             zero = g_malloc0(s->rom_state.vapic_size);
             cpu_physical_memory_rw(s->vapic_paddr, zero,
diff --git a/hw/kzm.c b/hw/kzm.c
index 6a5e9dfac..687daf3b2 100644
--- a/hw/kzm.c
+++ b/hw/kzm.c
@@ -5,7 +5,7 @@
  * Written by Hans at OK-Labs
  * Updated by Peter Chubb.
  *
- * This code is licenced under the GPL, version 2 or later.
+ * This code is licensed under the GPL, version 2 or later.
  * See the file `COPYING' in the top level directory.
  *
  * It (partially) emulates a Kyoto Microcomputer
@@ -21,7 +21,7 @@
 #include "net.h"
 #include "sysemu.h"
 #include "boards.h"
-#include "pc.h" /* for the FPGA UART that emulates a 16550 */
+#include "serial.h"
 #include "imx.h"
 
     /* Memory map for Kzm Emulation Baseboard:
@@ -70,11 +70,13 @@ static struct arm_boot_info kzm_binfo = {
     .board_id = 1722,
 };
 
-static void kzm_init(ram_addr_t ram_size,
-                     const char *boot_device,
-                     const char *kernel_filename, const char *kernel_cmdline,
-                     const char *initrd_filename, const char *cpu_model)
+static void kzm_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t ram_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
     ARMCPU *cpu;
     MemoryRegion *address_space_mem = get_system_memory();
     MemoryRegion *ram = g_new(MemoryRegion, 1);
diff --git a/hw/lan9118.c b/hw/lan9118.c
index ff0a50be1..f724e1c30 100644
--- a/hw/lan9118.c
+++ b/hw/lan9118.c
@@ -500,7 +500,7 @@ static int lan9118_filter(lan9118_state *s, const uint8_t *addr)
         }
     } else {
         /* Hash matching  */
-        hash = (crc32(~0, addr, 6) >> 26);
+        hash = compute_mcast_idx(addr);
         if (hash & 0x20) {
             return (s->mac_hashh >> (hash & 0x1f)) & 1;
         } else {
@@ -1000,7 +1000,7 @@ static void lan9118_tick(void *opaque)
     lan9118_update(s);
 }
 
-static void lan9118_writel(void *opaque, target_phys_addr_t offset,
+static void lan9118_writel(void *opaque, hwaddr offset,
                            uint64_t val, unsigned size)
 {
     lan9118_state *s = (lan9118_state *)opaque;
@@ -1134,7 +1134,7 @@ static void lan9118_writel(void *opaque, target_phys_addr_t offset,
     lan9118_update(s);
 }
 
-static void lan9118_writew(void *opaque, target_phys_addr_t offset,
+static void lan9118_writew(void *opaque, hwaddr offset,
                            uint32_t val)
 {
     lan9118_state *s = (lan9118_state *)opaque;
@@ -1161,7 +1161,7 @@ static void lan9118_writew(void *opaque, target_phys_addr_t offset,
     }
 }
 
-static void lan9118_16bit_mode_write(void *opaque, target_phys_addr_t offset,
+static void lan9118_16bit_mode_write(void *opaque, hwaddr offset,
                                      uint64_t val, unsigned size)
 {
     switch (size) {
@@ -1176,7 +1176,7 @@ static void lan9118_16bit_mode_write(void *opaque, target_phys_addr_t offset,
     hw_error("lan9118_write: Bad size 0x%x\n", size);
 }
 
-static uint64_t lan9118_readl(void *opaque, target_phys_addr_t offset,
+static uint64_t lan9118_readl(void *opaque, hwaddr offset,
                               unsigned size)
 {
     lan9118_state *s = (lan9118_state *)opaque;
@@ -1250,7 +1250,7 @@ static uint64_t lan9118_readl(void *opaque, target_phys_addr_t offset,
     return 0;
 }
 
-static uint32_t lan9118_readw(void *opaque, target_phys_addr_t offset)
+static uint32_t lan9118_readw(void *opaque, hwaddr offset)
 {
     lan9118_state *s = (lan9118_state *)opaque;
     uint32_t val;
@@ -1278,7 +1278,7 @@ static uint32_t lan9118_readw(void *opaque, target_phys_addr_t offset)
     return val;
 }
 
-static uint64_t lan9118_16bit_mode_read(void *opaque, target_phys_addr_t offset,
+static uint64_t lan9118_16bit_mode_read(void *opaque, hwaddr offset,
                                         unsigned size)
 {
     switch (size) {
diff --git a/hw/lance.c b/hw/lance.c
index 9b98bb849..a3e6dd91d 100644
--- a/hw/lance.c
+++ b/hw/lance.c
@@ -55,7 +55,7 @@ static void parent_lance_reset(void *opaque, int irq, int level)
         pcnet_h_reset(&d->state);
 }
 
-static void lance_mem_write(void *opaque, target_phys_addr_t addr,
+static void lance_mem_write(void *opaque, hwaddr addr,
                             uint64_t val, unsigned size)
 {
     SysBusPCNetState *d = opaque;
@@ -64,7 +64,7 @@ static void lance_mem_write(void *opaque, target_phys_addr_t addr,
     pcnet_ioport_writew(&d->state, addr, val & 0xffff);
 }
 
-static uint64_t lance_mem_read(void *opaque, target_phys_addr_t addr,
+static uint64_t lance_mem_read(void *opaque, hwaddr addr,
                                unsigned size)
 {
     SysBusPCNetState *d = opaque;
diff --git a/hw/leon3.c b/hw/leon3.c
index 878d3aa55..774273828 100644
--- a/hw/leon3.c
+++ b/hw/leon3.c
@@ -94,13 +94,11 @@ static void leon3_set_pil_in(void *opaque, uint32_t pil_in)
     }
 }
 
-static void leon3_generic_hw_init(ram_addr_t  ram_size,
-                                  const char *boot_device,
-                                  const char *kernel_filename,
-                                  const char *kernel_cmdline,
-                                  const char *initrd_filename,
-                                  const char *cpu_model)
+static void leon3_generic_hw_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t ram_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
     SPARCCPU *cpu;
     CPUSPARCState   *env;
     MemoryRegion *address_space_mem = get_system_memory();
@@ -210,7 +208,7 @@ static void leon3_generic_hw_init(ram_addr_t  ram_size,
     }
 }
 
-QEMUMachine leon3_generic_machine = {
+static QEMUMachine leon3_generic_machine = {
     .name     = "leon3_generic",
     .desc     = "Leon-3 generic",
     .init     = leon3_generic_hw_init,
diff --git a/hw/lm32_boards.c b/hw/lm32_boards.c
index b76d8008b..772cb8b05 100644
--- a/hw/lm32_boards.c
+++ b/hw/lm32_boards.c
@@ -32,12 +32,12 @@
 
 typedef struct {
     LM32CPU *cpu;
-    target_phys_addr_t bootstrap_pc;
-    target_phys_addr_t flash_base;
-    target_phys_addr_t hwsetup_base;
-    target_phys_addr_t initrd_base;
+    hwaddr bootstrap_pc;
+    hwaddr flash_base;
+    hwaddr hwsetup_base;
+    hwaddr initrd_base;
     size_t initrd_size;
-    target_phys_addr_t cmdline_base;
+    hwaddr cmdline_base;
 } ResetInfo;
 
 static void cpu_irq_handler(void *opaque, int irq, int level)
@@ -69,12 +69,10 @@ static void main_cpu_reset(void *opaque)
     env->deba = reset_info->flash_base;
 }
 
-static void lm32_evr_init(ram_addr_t ram_size_not_used,
-                          const char *boot_device,
-                          const char *kernel_filename,
-                          const char *kernel_cmdline,
-                          const char *initrd_filename, const char *cpu_model)
+static void lm32_evr_init(QEMUMachineInitArgs *args)
 {
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
     LM32CPU *cpu;
     CPULM32State *env;
     DriveInfo *dinfo;
@@ -85,14 +83,14 @@ static void lm32_evr_init(ram_addr_t ram_size_not_used,
     int i;
 
     /* memory map */
-    target_phys_addr_t flash_base  = 0x04000000;
+    hwaddr flash_base  = 0x04000000;
     size_t flash_sector_size       = 256 * 1024;
     size_t flash_size              = 32 * 1024 * 1024;
-    target_phys_addr_t ram_base    = 0x08000000;
+    hwaddr ram_base    = 0x08000000;
     size_t ram_size                = 64 * 1024 * 1024;
-    target_phys_addr_t timer0_base = 0x80002000;
-    target_phys_addr_t uart0_base  = 0x80006000;
-    target_phys_addr_t timer1_base = 0x8000a000;
+    hwaddr timer0_base = 0x80002000;
+    hwaddr uart0_base  = 0x80006000;
+    hwaddr timer1_base = 0x8000a000;
     int uart0_irq                  = 0;
     int timer0_irq                 = 1;
     int timer1_irq                 = 3;
@@ -159,12 +157,12 @@ static void lm32_evr_init(ram_addr_t ram_size_not_used,
     qemu_register_reset(main_cpu_reset, reset_info);
 }
 
-static void lm32_uclinux_init(ram_addr_t ram_size_not_used,
-                          const char *boot_device,
-                          const char *kernel_filename,
-                          const char *kernel_cmdline,
-                          const char *initrd_filename, const char *cpu_model)
+static void lm32_uclinux_init(QEMUMachineInitArgs *args)
 {
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
     LM32CPU *cpu;
     CPULM32State *env;
     DriveInfo *dinfo;
@@ -176,22 +174,22 @@ static void lm32_uclinux_init(ram_addr_t ram_size_not_used,
     int i;
 
     /* memory map */
-    target_phys_addr_t flash_base   = 0x04000000;
+    hwaddr flash_base   = 0x04000000;
     size_t flash_sector_size        = 256 * 1024;
     size_t flash_size               = 32 * 1024 * 1024;
-    target_phys_addr_t ram_base     = 0x08000000;
+    hwaddr ram_base     = 0x08000000;
     size_t ram_size                 = 64 * 1024 * 1024;
-    target_phys_addr_t uart0_base   = 0x80000000;
-    target_phys_addr_t timer0_base  = 0x80002000;
-    target_phys_addr_t timer1_base  = 0x80010000;
-    target_phys_addr_t timer2_base  = 0x80012000;
+    hwaddr uart0_base   = 0x80000000;
+    hwaddr timer0_base  = 0x80002000;
+    hwaddr timer1_base  = 0x80010000;
+    hwaddr timer2_base  = 0x80012000;
     int uart0_irq                   = 0;
     int timer0_irq                  = 1;
     int timer1_irq                  = 20;
     int timer2_irq                  = 21;
-    target_phys_addr_t hwsetup_base = 0x0bffe000;
-    target_phys_addr_t cmdline_base = 0x0bfff000;
-    target_phys_addr_t initrd_base  = 0x08400000;
+    hwaddr hwsetup_base = 0x0bffe000;
+    hwaddr cmdline_base = 0x0bfff000;
+    hwaddr initrd_base  = 0x08400000;
     size_t initrd_max               = 0x01000000;
 
     reset_info = g_malloc0(sizeof(ResetInfo));
diff --git a/hw/lm32_hwsetup.h b/hw/lm32_hwsetup.h
index 8fc285efc..853e9abc7 100644
--- a/hw/lm32_hwsetup.h
+++ b/hw/lm32_hwsetup.h
@@ -71,7 +71,7 @@ static inline void hwsetup_free(HWSetup *hw)
 }
 
 static inline void hwsetup_create_rom(HWSetup *hw,
-        target_phys_addr_t base)
+        hwaddr base)
 {
     rom_add_blob("hwsetup", hw->data, TARGET_PAGE_SIZE, base);
 }
@@ -96,7 +96,7 @@ static inline void hwsetup_add_tag(HWSetup *hw, enum hwsetup_tag t)
 
 static inline void hwsetup_add_str(HWSetup *hw, const char *str)
 {
-    strncpy(hw->ptr, str, 31); /* make sure last byte is zero */
+    pstrcpy(hw->ptr, 32, str);
     hw->ptr += 32;
 }
 
diff --git a/hw/lm32_sys.c b/hw/lm32_sys.c
index bbe03c41d..a7887d14f 100644
--- a/hw/lm32_sys.c
+++ b/hw/lm32_sys.c
@@ -61,7 +61,7 @@ static void copy_testname(LM32SysState *s)
     s->testname[MAX_TESTNAME_LEN - 1] = '\0';
 }
 
-static void sys_write(void *opaque, target_phys_addr_t addr,
+static void sys_write(void *opaque, hwaddr addr,
                       uint64_t value, unsigned size)
 {
     LM32SysState *s = opaque;
@@ -91,7 +91,7 @@ static void sys_write(void *opaque, target_phys_addr_t addr,
     }
 }
 
-static bool sys_ops_accepts(void *opaque, target_phys_addr_t addr,
+static bool sys_ops_accepts(void *opaque, hwaddr addr,
                             unsigned size, bool is_write)
 {
     return is_write && size == 4;
diff --git a/hw/lm32_timer.c b/hw/lm32_timer.c
index e9450a0ce..a8be9cc16 100644
--- a/hw/lm32_timer.c
+++ b/hw/lm32_timer.c
@@ -72,7 +72,7 @@ static void timer_update_irq(LM32TimerState *s)
     qemu_set_irq(s->irq, state);
 }
 
-static uint64_t timer_read(void *opaque, target_phys_addr_t addr, unsigned size)
+static uint64_t timer_read(void *opaque, hwaddr addr, unsigned size)
 {
     LM32TimerState *s = opaque;
     uint32_t r = 0;
@@ -97,7 +97,7 @@ static uint64_t timer_read(void *opaque, target_phys_addr_t addr, unsigned size)
     return r;
 }
 
-static void timer_write(void *opaque, target_phys_addr_t addr,
+static void timer_write(void *opaque, hwaddr addr,
                         uint64_t value, unsigned size)
 {
     LM32TimerState *s = opaque;
diff --git a/hw/lm32_uart.c b/hw/lm32_uart.c
index 57066e28c..adb928756 100644
--- a/hw/lm32_uart.c
+++ b/hw/lm32_uart.c
@@ -125,7 +125,7 @@ static void uart_update_irq(LM32UartState *s)
     qemu_set_irq(s->irq, irq);
 }
 
-static uint64_t uart_read(void *opaque, target_phys_addr_t addr,
+static uint64_t uart_read(void *opaque, hwaddr addr,
                           unsigned size)
 {
     LM32UartState *s = opaque;
@@ -160,7 +160,7 @@ static uint64_t uart_read(void *opaque, target_phys_addr_t addr,
     return r;
 }
 
-static void uart_write(void *opaque, target_phys_addr_t addr,
+static void uart_write(void *opaque, hwaddr addr,
                        uint64_t value, unsigned size)
 {
     LM32UartState *s = opaque;
diff --git a/hw/lm4549.c b/hw/lm4549.c
index 80b3ec4a5..b3c2d5f25 100644
--- a/hw/lm4549.c
+++ b/hw/lm4549.c
@@ -150,7 +150,7 @@ static void lm4549_audio_out_callback(void *opaque, int free)
     }
 }
 
-uint32_t lm4549_read(lm4549_state *s, target_phys_addr_t offset)
+uint32_t lm4549_read(lm4549_state *s, hwaddr offset)
 {
     uint16_t *regfile = s->regfile;
     uint32_t value = 0;
@@ -165,7 +165,7 @@ uint32_t lm4549_read(lm4549_state *s, target_phys_addr_t offset)
 }
 
 void lm4549_write(lm4549_state *s,
-                  target_phys_addr_t offset, uint32_t value)
+                  hwaddr offset, uint32_t value)
 {
     uint16_t *regfile = s->regfile;
 
@@ -224,7 +224,7 @@ uint32_t lm4549_write_samples(lm4549_state *s, uint32_t left, uint32_t right)
        This model supports 16-bit playback.
     */
 
-    if (s->buffer_level >= LM4549_BUFFER_SIZE) {
+    if (s->buffer_level > LM4549_BUFFER_SIZE - 2) {
         DPRINTF("write_sample Buffer full\n");
         return 0;
     }
diff --git a/hw/lm4549.h b/hw/lm4549.h
index 5948780e0..812a7a444 100644
--- a/hw/lm4549.h
+++ b/hw/lm4549.h
@@ -36,8 +36,8 @@ extern const VMStateDescription vmstate_lm4549_state;
 
 
 void lm4549_init(lm4549_state *s, lm4549_callback data_req, void *opaque);
-uint32_t lm4549_read(lm4549_state *s, target_phys_addr_t offset);
-void lm4549_write(lm4549_state *s, target_phys_addr_t offset, uint32_t value);
+uint32_t lm4549_read(lm4549_state *s, hwaddr offset);
+void lm4549_write(lm4549_state *s, hwaddr offset, uint32_t value);
 uint32_t lm4549_write_samples(lm4549_state *s, uint32_t left, uint32_t right);
 
 #endif /* #ifndef HW_LM4549_H */
diff --git a/hw/loader.c b/hw/loader.c
index 33acc2fda..ba01ca663 100644
--- a/hw/loader.c
+++ b/hw/loader.c
@@ -88,7 +88,7 @@ int load_image(const char *filename, uint8_t *addr)
 
 /* read()-like version */
 ssize_t read_targphys(const char *name,
-                      int fd, target_phys_addr_t dst_addr, size_t nbytes)
+                      int fd, hwaddr dst_addr, size_t nbytes)
 {
     uint8_t *buf;
     ssize_t did;
@@ -103,7 +103,7 @@ ssize_t read_targphys(const char *name,
 
 /* return the size or -1 if error */
 int load_image_targphys(const char *filename,
-                        target_phys_addr_t addr, uint64_t max_sz)
+                        hwaddr addr, uint64_t max_sz)
 {
     int size;
 
@@ -117,7 +117,7 @@ int load_image_targphys(const char *filename,
     return size;
 }
 
-void pstrcpy_targphys(const char *name, target_phys_addr_t dest, int buf_size,
+void pstrcpy_targphys(const char *name, hwaddr dest, int buf_size,
                       const char *source)
 {
     const char *nulp;
@@ -179,8 +179,8 @@ static void bswap_ahdr(struct exec *e)
      : (_N_SEGMENT_ROUND (_N_TXTENDADDR(x, target_page_size), target_page_size)))
 
 
-int load_aout(const char *filename, target_phys_addr_t addr, int max_sz,
-              int bswap_needed, target_phys_addr_t target_page_size)
+int load_aout(const char *filename, hwaddr addr, int max_sz,
+              int bswap_needed, hwaddr target_page_size)
 {
     int fd;
     ssize_t size, ret;
@@ -434,8 +434,8 @@ static ssize_t gunzip(void *dst, size_t dstlen, uint8_t *src,
 }
 
 /* Load a U-Boot image.  */
-int load_uimage(const char *filename, target_phys_addr_t *ep,
-                target_phys_addr_t *loadaddr, int *is_linux)
+int load_uimage(const char *filename, hwaddr *ep,
+                hwaddr *loadaddr, int *is_linux)
 {
     int fd;
     int size;
@@ -539,7 +539,7 @@ struct Rom {
     char *fw_dir;
     char *fw_file;
 
-    target_phys_addr_t addr;
+    hwaddr addr;
     QTAILQ_ENTRY(Rom) next;
 };
 
@@ -565,7 +565,7 @@ static void rom_insert(Rom *rom)
 }
 
 int rom_add_file(const char *file, const char *fw_dir,
-                 target_phys_addr_t addr, int32_t bootindex)
+                 hwaddr addr, int32_t bootindex)
 {
     Rom *rom;
     int rc, fd = -1;
@@ -633,7 +633,7 @@ err:
 }
 
 int rom_add_blob(const char *name, const void *blob, size_t len,
-                 target_phys_addr_t addr)
+                 hwaddr addr)
 {
     Rom *rom;
 
@@ -679,7 +679,7 @@ static void rom_reset(void *unused)
 
 int rom_load_all(void)
 {
-    target_phys_addr_t addr = 0;
+    hwaddr addr = 0;
     MemoryRegionSection section;
     Rom *rom;
 
@@ -709,7 +709,7 @@ void rom_set_fw(void *f)
     fw_cfg = f;
 }
 
-static Rom *find_rom(target_phys_addr_t addr)
+static Rom *find_rom(hwaddr addr)
 {
     Rom *rom;
 
@@ -733,9 +733,9 @@ static Rom *find_rom(target_phys_addr_t addr)
  * a ROM between addr and addr + size is copied. Note that this can involve
  * multiple ROMs, which need not start at addr and need not end at addr + size.
  */
-int rom_copy(uint8_t *dest, target_phys_addr_t addr, size_t size)
+int rom_copy(uint8_t *dest, hwaddr addr, size_t size)
 {
-    target_phys_addr_t end = addr + size;
+    hwaddr end = addr + size;
     uint8_t *s, *d = dest;
     size_t l = 0;
     Rom *rom;
@@ -768,7 +768,7 @@ int rom_copy(uint8_t *dest, target_phys_addr_t addr, size_t size)
     return (d + l) - dest;
 }
 
-void *rom_ptr(target_phys_addr_t addr)
+void *rom_ptr(hwaddr addr)
 {
     Rom *rom;
 
diff --git a/hw/loader.h b/hw/loader.h
index 6da291e31..26480ad8d 100644
--- a/hw/loader.h
+++ b/hw/loader.h
@@ -4,32 +4,32 @@
 /* loader.c */
 int get_image_size(const char *filename);
 int load_image(const char *filename, uint8_t *addr); /* deprecated */
-int load_image_targphys(const char *filename, target_phys_addr_t,
+int load_image_targphys(const char *filename, hwaddr,
                         uint64_t max_sz);
 int load_elf(const char *filename, uint64_t (*translate_fn)(void *, uint64_t),
              void *translate_opaque, uint64_t *pentry, uint64_t *lowaddr,
              uint64_t *highaddr, int big_endian, int elf_machine,
              int clear_lsb);
-int load_aout(const char *filename, target_phys_addr_t addr, int max_sz,
-              int bswap_needed, target_phys_addr_t target_page_size);
-int load_uimage(const char *filename, target_phys_addr_t *ep,
-                target_phys_addr_t *loadaddr, int *is_linux);
+int load_aout(const char *filename, hwaddr addr, int max_sz,
+              int bswap_needed, hwaddr target_page_size);
+int load_uimage(const char *filename, hwaddr *ep,
+                hwaddr *loadaddr, int *is_linux);
 
 ssize_t read_targphys(const char *name,
-                      int fd, target_phys_addr_t dst_addr, size_t nbytes);
+                      int fd, hwaddr dst_addr, size_t nbytes);
 void pstrcpy_targphys(const char *name,
-                      target_phys_addr_t dest, int buf_size,
+                      hwaddr dest, int buf_size,
                       const char *source);
 
 
 int rom_add_file(const char *file, const char *fw_dir,
-                 target_phys_addr_t addr, int32_t bootindex);
+                 hwaddr addr, int32_t bootindex);
 int rom_add_blob(const char *name, const void *blob, size_t len,
-                 target_phys_addr_t addr);
+                 hwaddr addr);
 int rom_load_all(void);
 void rom_set_fw(void *f);
-int rom_copy(uint8_t *dest, target_phys_addr_t addr, size_t size);
-void *rom_ptr(target_phys_addr_t addr);
+int rom_copy(uint8_t *dest, hwaddr addr, size_t size);
+void *rom_ptr(hwaddr addr);
 void do_info_roms(Monitor *mon);
 
 #define rom_add_file_fixed(_f, _a, _i)          \
diff --git a/hw/lpc_ich9.c b/hw/lpc_ich9.c
new file mode 100644
index 000000000..2fc83a496
--- /dev/null
+++ b/hw/lpc_ich9.c
@@ -0,0 +1,525 @@
+/*
+ * Copyright (c) 2006 Fabrice Bellard
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+/*
+ * QEMU ICH9 Emulation
+ *
+ *  Copyright (c) 2009, 2010, 2011
+ *                Isaku Yamahata <yamahata at valinux co jp>
+ *                VA Linux Systems Japan K.K.
+ *  Copyright (C) 2012 Jason Baron <jbaron@redhat.com>
+ *
+ *  This is based on piix_pci.c, but heavily modified.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>
+ */
+
+#include "qemu-common.h"
+#include "hw.h"
+#include "range.h"
+#include "isa.h"
+#include "sysbus.h"
+#include "pc.h"
+#include "apm.h"
+#include "ioapic.h"
+#include "pci.h"
+#include "pcie_host.h"
+#include "pci_bridge.h"
+#include "ich9.h"
+#include "acpi.h"
+#include "acpi_ich9.h"
+#include "pam.h"
+#include "pci_internals.h"
+#include "exec-memory.h"
+
+static int ich9_lpc_sci_irq(ICH9LPCState *lpc);
+
+/*****************************************************************************/
+/* ICH9 LPC PCI to ISA bridge */
+
+static void ich9_lpc_reset(DeviceState *qdev);
+
+/* chipset configuration register
+ * to access chipset configuration registers, pci_[sg]et_{byte, word, long}
+ * are used.
+ * Although it's not pci configuration space, it's little endian as Intel.
+ */
+
+static void ich9_cc_update_ir(uint8_t irr[PCI_NUM_PINS], uint16_t ir)
+{
+    int intx;
+    for (intx = 0; intx < PCI_NUM_PINS; intx++) {
+        irr[intx] = (ir >> (intx * ICH9_CC_DIR_SHIFT)) & ICH9_CC_DIR_MASK;
+    }
+}
+
+static void ich9_cc_update(ICH9LPCState *lpc)
+{
+    int slot;
+    int pci_intx;
+
+    const int reg_offsets[] = {
+        ICH9_CC_D25IR,
+        ICH9_CC_D26IR,
+        ICH9_CC_D27IR,
+        ICH9_CC_D28IR,
+        ICH9_CC_D29IR,
+        ICH9_CC_D30IR,
+        ICH9_CC_D31IR,
+    };
+    const int *offset;
+
+    /* D{25 - 31}IR, but D30IR is read only to 0. */
+    for (slot = 25, offset = reg_offsets; slot < 32; slot++, offset++) {
+        if (slot == 30) {
+            continue;
+        }
+        ich9_cc_update_ir(lpc->irr[slot],
+                          pci_get_word(lpc->chip_config + *offset));
+    }
+
+    /*
+     * D30: DMI2PCI bridge
+     * It is arbitrarily decided how INTx lines of PCI devicesbehind the bridge
+     * are connected to pirq lines. Our choice is PIRQ[E-H].
+     * INT[A-D] are connected to PIRQ[E-H]
+     */
+    for (pci_intx = 0; pci_intx < PCI_NUM_PINS; pci_intx++) {
+        lpc->irr[30][pci_intx] = pci_intx + 4;
+    }
+}
+
+static void ich9_cc_init(ICH9LPCState *lpc)
+{
+    int slot;
+    int intx;
+
+    /* the default irq routing is arbitrary as long as it matches with
+     * acpi irq routing table.
+     * The one that is incompatible with piix_pci(= bochs) one is
+     * intentionally chosen to let the users know that the different
+     * board is used.
+     *
+     * int[A-D] -> pirq[E-F]
+     * avoid pirq A-D because they are used for pci express port
+     */
+    for (slot = 0; slot < PCI_SLOT_MAX; slot++) {
+        for (intx = 0; intx < PCI_NUM_PINS; intx++) {
+            lpc->irr[slot][intx] = (slot + intx) % 4 + 4;
+        }
+    }
+    ich9_cc_update(lpc);
+}
+
+static void ich9_cc_reset(ICH9LPCState *lpc)
+{
+    uint8_t *c = lpc->chip_config;
+
+    memset(lpc->chip_config, 0, sizeof(lpc->chip_config));
+
+    pci_set_long(c + ICH9_CC_D31IR, ICH9_CC_DIR_DEFAULT);
+    pci_set_long(c + ICH9_CC_D30IR, ICH9_CC_D30IR_DEFAULT);
+    pci_set_long(c + ICH9_CC_D29IR, ICH9_CC_DIR_DEFAULT);
+    pci_set_long(c + ICH9_CC_D28IR, ICH9_CC_DIR_DEFAULT);
+    pci_set_long(c + ICH9_CC_D27IR, ICH9_CC_DIR_DEFAULT);
+    pci_set_long(c + ICH9_CC_D26IR, ICH9_CC_DIR_DEFAULT);
+    pci_set_long(c + ICH9_CC_D25IR, ICH9_CC_DIR_DEFAULT);
+
+    ich9_cc_update(lpc);
+}
+
+static void ich9_cc_addr_len(uint64_t *addr, unsigned *len)
+{
+    *addr &= ICH9_CC_ADDR_MASK;
+    if (*addr + *len >= ICH9_CC_SIZE) {
+        *len = ICH9_CC_SIZE - *addr;
+    }
+}
+
+/* val: little endian */
+static void ich9_cc_write(void *opaque, hwaddr addr,
+                          uint64_t val, unsigned len)
+{
+    ICH9LPCState *lpc = (ICH9LPCState *)opaque;
+
+    ich9_cc_addr_len(&addr, &len);
+    memcpy(lpc->chip_config + addr, &val, len);
+    ich9_cc_update(lpc);
+}
+
+/* return value: little endian */
+static uint64_t ich9_cc_read(void *opaque, hwaddr addr,
+                              unsigned len)
+{
+    ICH9LPCState *lpc = (ICH9LPCState *)opaque;
+
+    uint32_t val = 0;
+    ich9_cc_addr_len(&addr, &len);
+    memcpy(&val, lpc->chip_config + addr, len);
+    return val;
+}
+
+/* IRQ routing */
+/* */
+static void ich9_lpc_rout(uint8_t pirq_rout, int *pic_irq, int *pic_dis)
+{
+    *pic_irq = pirq_rout & ICH9_LPC_PIRQ_ROUT_MASK;
+    *pic_dis = pirq_rout & ICH9_LPC_PIRQ_ROUT_IRQEN;
+}
+
+static void ich9_lpc_pic_irq(ICH9LPCState *lpc, int pirq_num,
+                             int *pic_irq, int *pic_dis)
+{
+    switch (pirq_num) {
+    case 0 ... 3: /* A-D */
+        ich9_lpc_rout(lpc->d.config[ICH9_LPC_PIRQA_ROUT + pirq_num],
+                      pic_irq, pic_dis);
+        return;
+    case 4 ... 7: /* E-H */
+        ich9_lpc_rout(lpc->d.config[ICH9_LPC_PIRQE_ROUT + (pirq_num - 4)],
+                      pic_irq, pic_dis);
+        return;
+    default:
+        break;
+    }
+    abort();
+}
+
+/* pic_irq: i8254 irq 0-15 */
+static void ich9_lpc_update_pic(ICH9LPCState *lpc, int pic_irq)
+{
+    int i, pic_level;
+
+    /* The pic level is the logical OR of all the PCI irqs mapped to it */
+    pic_level = 0;
+    for (i = 0; i < ICH9_LPC_NB_PIRQS; i++) {
+        int tmp_irq;
+        int tmp_dis;
+        ich9_lpc_pic_irq(lpc, i, &tmp_irq, &tmp_dis);
+        if (!tmp_dis && pic_irq == tmp_irq) {
+            pic_level |= pci_bus_get_irq_level(lpc->d.bus, i);
+        }
+    }
+    if (pic_irq == ich9_lpc_sci_irq(lpc)) {
+        pic_level |= lpc->sci_level;
+    }
+
+    qemu_set_irq(lpc->pic[pic_irq], pic_level);
+}
+
+/* pirq: pirq[A-H] 0-7*/
+static void ich9_lpc_update_by_pirq(ICH9LPCState *lpc, int pirq)
+{
+    int pic_irq;
+    int pic_dis;
+
+    ich9_lpc_pic_irq(lpc, pirq, &pic_irq, &pic_dis);
+    assert(pic_irq < ICH9_LPC_PIC_NUM_PINS);
+    if (pic_dis) {
+        return;
+    }
+
+    ich9_lpc_update_pic(lpc, pic_irq);
+}
+
+/* APIC mode: GSIx: PIRQ[A-H] -> GSI 16, ... no pirq shares same APIC pins. */
+static int ich9_pirq_to_gsi(int pirq)
+{
+    return pirq + ICH9_LPC_PIC_NUM_PINS;
+}
+
+static int ich9_gsi_to_pirq(int gsi)
+{
+    return gsi - ICH9_LPC_PIC_NUM_PINS;
+}
+
+static void ich9_lpc_update_apic(ICH9LPCState *lpc, int gsi)
+{
+    int level = 0;
+
+    if (gsi >= ICH9_LPC_PIC_NUM_PINS) {
+        level |= pci_bus_get_irq_level(lpc->d.bus, ich9_gsi_to_pirq(gsi));
+    }
+    if (gsi == ich9_lpc_sci_irq(lpc)) {
+        level |= lpc->sci_level;
+    }
+
+    qemu_set_irq(lpc->ioapic[gsi], level);
+}
+
+void ich9_lpc_set_irq(void *opaque, int pirq, int level)
+{
+    ICH9LPCState *lpc = opaque;
+
+    assert(0 <= pirq);
+    assert(pirq < ICH9_LPC_NB_PIRQS);
+
+    ich9_lpc_update_apic(lpc, ich9_pirq_to_gsi(pirq));
+    ich9_lpc_update_by_pirq(lpc, pirq);
+}
+
+/* return the pirq number (PIRQ[A-H]:0-7) corresponding to
+ * a given device irq pin.
+ */
+int ich9_lpc_map_irq(PCIDevice *pci_dev, int intx)
+{
+    BusState *bus = qdev_get_parent_bus(&pci_dev->qdev);
+    PCIBus *pci_bus = PCI_BUS(bus);
+    PCIDevice *lpc_pdev =
+            pci_bus->devices[PCI_DEVFN(ICH9_LPC_DEV, ICH9_LPC_FUNC)];
+    ICH9LPCState *lpc = ICH9_LPC_DEVICE(lpc_pdev);
+
+    return lpc->irr[PCI_SLOT(pci_dev->devfn)][intx];
+}
+
+static int ich9_lpc_sci_irq(ICH9LPCState *lpc)
+{
+    switch (lpc->d.config[ICH9_LPC_ACPI_CTRL] &
+            ICH9_LPC_ACPI_CTRL_SCI_IRQ_SEL_MASK) {
+    case ICH9_LPC_ACPI_CTRL_9:
+        return 9;
+    case ICH9_LPC_ACPI_CTRL_10:
+        return 10;
+    case ICH9_LPC_ACPI_CTRL_11:
+        return 11;
+    case ICH9_LPC_ACPI_CTRL_20:
+        return 20;
+    case ICH9_LPC_ACPI_CTRL_21:
+        return 21;
+    default:
+        /* reserved */
+        break;
+    }
+    return -1;
+}
+
+static void ich9_set_sci(void *opaque, int irq_num, int level)
+{
+    ICH9LPCState *lpc = opaque;
+    int irq;
+
+    assert(irq_num == 0);
+    level = !!level;
+    if (level == lpc->sci_level) {
+        return;
+    }
+    lpc->sci_level = level;
+
+    irq = ich9_lpc_sci_irq(lpc);
+    if (irq < 0) {
+        return;
+    }
+
+    ich9_lpc_update_apic(lpc, irq);
+    if (irq < ICH9_LPC_PIC_NUM_PINS) {
+        ich9_lpc_update_pic(lpc, irq);
+    }
+}
+
+void ich9_lpc_pm_init(PCIDevice *lpc_pci, qemu_irq cmos_s3)
+{
+    ICH9LPCState *lpc = ICH9_LPC_DEVICE(lpc_pci);
+    qemu_irq *sci_irq;
+
+    sci_irq = qemu_allocate_irqs(ich9_set_sci, lpc, 1);
+    ich9_pm_init(&lpc->pm, sci_irq[0], cmos_s3);
+
+    ich9_lpc_reset(&lpc->d.qdev);
+}
+
+/* APM */
+
+static void ich9_apm_ctrl_changed(uint32_t val, void *arg)
+{
+    ICH9LPCState *lpc = arg;
+
+    /* ACPI specs 3.0, 4.7.2.5 */
+    acpi_pm1_cnt_update(&lpc->pm.acpi_regs,
+                        val == ICH9_APM_ACPI_ENABLE,
+                        val == ICH9_APM_ACPI_DISABLE);
+
+    /* SMI_EN = PMBASE + 30. SMI control and enable register */
+    if (lpc->pm.smi_en & ICH9_PMIO_SMI_EN_APMC_EN) {
+        cpu_interrupt(first_cpu, CPU_INTERRUPT_SMI);
+    }
+}
+
+/* config:PMBASE */
+static void
+ich9_lpc_pmbase_update(ICH9LPCState *lpc)
+{
+    uint32_t pm_io_base = pci_get_long(lpc->d.config + ICH9_LPC_PMBASE);
+    pm_io_base &= ICH9_LPC_PMBASE_BASE_ADDRESS_MASK;
+
+    ich9_pm_iospace_update(&lpc->pm, pm_io_base);
+}
+
+/* config:RBCA */
+static void ich9_lpc_rcba_update(ICH9LPCState *lpc, uint32_t rbca_old)
+{
+    uint32_t rbca = pci_get_long(lpc->d.config + ICH9_LPC_RCBA);
+
+    if (rbca_old & ICH9_LPC_RCBA_EN) {
+            memory_region_del_subregion(get_system_memory(), &lpc->rbca_mem);
+    }
+    if (rbca & ICH9_LPC_RCBA_EN) {
+            memory_region_add_subregion_overlap(get_system_memory(),
+                                                rbca & ICH9_LPC_RCBA_BA_MASK,
+                                                &lpc->rbca_mem, 1);
+    }
+}
+
+static int ich9_lpc_post_load(void *opaque, int version_id)
+{
+    ICH9LPCState *lpc = opaque;
+
+    ich9_lpc_pmbase_update(lpc);
+    ich9_lpc_rcba_update(lpc, 0 /* disabled ICH9_LPC_RBCA_EN */);
+    return 0;
+}
+
+static void ich9_lpc_config_write(PCIDevice *d,
+                                  uint32_t addr, uint32_t val, int len)
+{
+    ICH9LPCState *lpc = ICH9_LPC_DEVICE(d);
+    uint32_t rbca_old = pci_get_long(d->config + ICH9_LPC_RCBA);
+
+    pci_default_write_config(d, addr, val, len);
+    if (ranges_overlap(addr, len, ICH9_LPC_PMBASE, 4)) {
+        ich9_lpc_pmbase_update(lpc);
+    }
+    if (ranges_overlap(addr, len, ICH9_LPC_RCBA, 4)) {
+        ich9_lpc_rcba_update(lpc, rbca_old);
+    }
+}
+
+static void ich9_lpc_reset(DeviceState *qdev)
+{
+    PCIDevice *d = PCI_DEVICE(qdev);
+    ICH9LPCState *lpc = ICH9_LPC_DEVICE(d);
+    uint32_t rbca_old = pci_get_long(d->config + ICH9_LPC_RCBA);
+    int i;
+
+    for (i = 0; i < 4; i++) {
+        pci_set_byte(d->config + ICH9_LPC_PIRQA_ROUT + i,
+                     ICH9_LPC_PIRQ_ROUT_DEFAULT);
+    }
+    for (i = 0; i < 4; i++) {
+        pci_set_byte(d->config + ICH9_LPC_PIRQE_ROUT + i,
+                     ICH9_LPC_PIRQ_ROUT_DEFAULT);
+    }
+    pci_set_byte(d->config + ICH9_LPC_ACPI_CTRL, ICH9_LPC_ACPI_CTRL_DEFAULT);
+
+    pci_set_long(d->config + ICH9_LPC_PMBASE, ICH9_LPC_PMBASE_DEFAULT);
+    pci_set_long(d->config + ICH9_LPC_RCBA, ICH9_LPC_RCBA_DEFAULT);
+
+    ich9_cc_reset(lpc);
+
+    ich9_lpc_pmbase_update(lpc);
+    ich9_lpc_rcba_update(lpc, rbca_old);
+
+    lpc->sci_level = 0;
+}
+
+static const MemoryRegionOps rbca_mmio_ops = {
+    .read = ich9_cc_read,
+    .write = ich9_cc_write,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+static int ich9_lpc_initfn(PCIDevice *d)
+{
+    ICH9LPCState *lpc = ICH9_LPC_DEVICE(d);
+    ISABus *isa_bus;
+
+    isa_bus = isa_bus_new(&d->qdev, get_system_io());
+
+    pci_set_long(d->wmask + ICH9_LPC_PMBASE,
+                 ICH9_LPC_PMBASE_BASE_ADDRESS_MASK);
+
+    memory_region_init_io(&lpc->rbca_mem, &rbca_mmio_ops, lpc,
+                            "lpc-rbca-mmio", ICH9_CC_SIZE);
+
+    lpc->isa_bus = isa_bus;
+
+    ich9_cc_init(lpc);
+    apm_init(&lpc->apm, ich9_apm_ctrl_changed, lpc);
+    return 0;
+}
+
+static const VMStateDescription vmstate_ich9_lpc = {
+    .name = "ICH9LPC",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .minimum_version_id_old = 1,
+    .post_load = ich9_lpc_post_load,
+    .fields = (VMStateField[]) {
+        VMSTATE_PCI_DEVICE(d, ICH9LPCState),
+        VMSTATE_STRUCT(apm, ICH9LPCState, 0, vmstate_apm, APMState),
+        VMSTATE_STRUCT(pm, ICH9LPCState, 0, vmstate_ich9_pm, ICH9LPCPMRegs),
+        VMSTATE_UINT8_ARRAY(chip_config, ICH9LPCState, ICH9_CC_SIZE),
+        VMSTATE_UINT32(sci_level, ICH9LPCState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static void ich9_lpc_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
+
+    dc->reset = ich9_lpc_reset;
+    k->init = ich9_lpc_initfn;
+    dc->vmsd = &vmstate_ich9_lpc;
+    dc->no_user = 1;
+    k->config_write = ich9_lpc_config_write;
+    dc->desc = "ICH9 LPC bridge";
+    k->vendor_id = PCI_VENDOR_ID_INTEL;
+    k->device_id = PCI_DEVICE_ID_INTEL_ICH9_8;
+    k->revision = ICH9_A2_LPC_REVISION;
+    k->class_id = PCI_CLASS_BRIDGE_ISA;
+
+}
+
+static const TypeInfo ich9_lpc_info = {
+    .name       = TYPE_ICH9_LPC_DEVICE,
+    .parent     = TYPE_PCI_DEVICE,
+    .instance_size = sizeof(struct ICH9LPCState),
+    .class_init  = ich9_lpc_class_init,
+};
+
+static void ich9_lpc_register(void)
+{
+    type_register_static(&ich9_lpc_info);
+}
+
+type_init(ich9_lpc_register);
diff --git a/hw/lsi53c895a.c b/hw/lsi53c895a.c
index 34afe9674..04f2faef4 100644
--- a/hw/lsi53c895a.c
+++ b/hw/lsi53c895a.c
@@ -1878,7 +1878,7 @@ static void lsi_reg_writeb(LSIState *s, int offset, uint8_t val)
 #undef CASE_SET_REG32
 }
 
-static void lsi_mmio_write(void *opaque, target_phys_addr_t addr,
+static void lsi_mmio_write(void *opaque, hwaddr addr,
                            uint64_t val, unsigned size)
 {
     LSIState *s = opaque;
@@ -1886,7 +1886,7 @@ static void lsi_mmio_write(void *opaque, target_phys_addr_t addr,
     lsi_reg_writeb(s, addr & 0xff, val);
 }
 
-static uint64_t lsi_mmio_read(void *opaque, target_phys_addr_t addr,
+static uint64_t lsi_mmio_read(void *opaque, hwaddr addr,
                               unsigned size)
 {
     LSIState *s = opaque;
@@ -1904,7 +1904,7 @@ static const MemoryRegionOps lsi_mmio_ops = {
     },
 };
 
-static void lsi_ram_write(void *opaque, target_phys_addr_t addr,
+static void lsi_ram_write(void *opaque, hwaddr addr,
                           uint64_t val, unsigned size)
 {
     LSIState *s = opaque;
@@ -1920,7 +1920,7 @@ static void lsi_ram_write(void *opaque, target_phys_addr_t addr,
     s->script_ram[addr >> 2] = newval;
 }
 
-static uint64_t lsi_ram_read(void *opaque, target_phys_addr_t addr,
+static uint64_t lsi_ram_read(void *opaque, hwaddr addr,
                              unsigned size)
 {
     LSIState *s = opaque;
@@ -1939,14 +1939,14 @@ static const MemoryRegionOps lsi_ram_ops = {
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-static uint64_t lsi_io_read(void *opaque, target_phys_addr_t addr,
+static uint64_t lsi_io_read(void *opaque, hwaddr addr,
                             unsigned size)
 {
     LSIState *s = opaque;
     return lsi_reg_readb(s, addr & 0xff);
 }
 
-static void lsi_io_write(void *opaque, target_phys_addr_t addr,
+static void lsi_io_write(void *opaque, hwaddr addr,
                          uint64_t val, unsigned size)
 {
     LSIState *s = opaque;
diff --git a/hw/m25p80.c b/hw/m25p80.c
new file mode 100644
index 000000000..3895e7395
--- /dev/null
+++ b/hw/m25p80.c
@@ -0,0 +1,651 @@
+/*
+ * ST M25P80 emulator. Emulate all SPI flash devices based on the m25p80 command
+ * set. Known devices table current as of Jun/2012 and taken from linux.
+ * See drivers/mtd/devices/m25p80.c.
+ *
+ * Copyright (C) 2011 Edgar E. Iglesias <edgar.iglesias@gmail.com>
+ * Copyright (C) 2012 Peter A. G. Crosthwaite <peter.crosthwaite@petalogix.com>
+ * Copyright (C) 2012 PetaLogix
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 or
+ * (at your option) a later version of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "hw.h"
+#include "blockdev.h"
+#include "ssi.h"
+#include "devices.h"
+
+#ifdef M25P80_ERR_DEBUG
+#define DB_PRINT(...) do { \
+    fprintf(stderr,  ": %s: ", __func__); \
+    fprintf(stderr, ## __VA_ARGS__); \
+    } while (0);
+#else
+    #define DB_PRINT(...)
+#endif
+
+/* Fields for FlashPartInfo->flags */
+
+/* erase capabilities */
+#define ER_4K 1
+#define ER_32K 2
+/* set to allow the page program command to write 0s back to 1. Useful for
+ * modelling EEPROM with SPI flash command set
+ */
+#define WR_1 0x100
+
+typedef struct FlashPartInfo {
+    const char *part_name;
+    /* jedec code. (jedec >> 16) & 0xff is the 1st byte, >> 8 the 2nd etc */
+    uint32_t jedec;
+    /* extended jedec code */
+    uint16_t ext_jedec;
+    /* there is confusion between manufacturers as to what a sector is. In this
+     * device model, a "sector" is the size that is erased by the ERASE_SECTOR
+     * command (opcode 0xd8).
+     */
+    uint32_t sector_size;
+    uint32_t n_sectors;
+    uint32_t page_size;
+    uint8_t flags;
+} FlashPartInfo;
+
+/* adapted from linux */
+
+#define INFO(_part_name, _jedec, _ext_jedec, _sector_size, _n_sectors, _flags)\
+    .part_name = (_part_name),\
+    .jedec = (_jedec),\
+    .ext_jedec = (_ext_jedec),\
+    .sector_size = (_sector_size),\
+    .n_sectors = (_n_sectors),\
+    .page_size = 256,\
+    .flags = (_flags),\
+
+#define JEDEC_NUMONYX 0x20
+#define JEDEC_WINBOND 0xEF
+#define JEDEC_SPANSION 0x01
+
+static const FlashPartInfo known_devices[] = {
+    /* Atmel -- some are (confusingly) marketed as "DataFlash" */
+    { INFO("at25fs010",   0x1f6601,      0,  32 << 10,   4, ER_4K) },
+    { INFO("at25fs040",   0x1f6604,      0,  64 << 10,   8, ER_4K) },
+
+    { INFO("at25df041a",  0x1f4401,      0,  64 << 10,   8, ER_4K) },
+    { INFO("at25df321a",  0x1f4701,      0,  64 << 10,  64, ER_4K) },
+    { INFO("at25df641",   0x1f4800,      0,  64 << 10, 128, ER_4K) },
+
+    { INFO("at26f004",    0x1f0400,      0,  64 << 10,   8, ER_4K) },
+    { INFO("at26df081a",  0x1f4501,      0,  64 << 10,  16, ER_4K) },
+    { INFO("at26df161a",  0x1f4601,      0,  64 << 10,  32, ER_4K) },
+    { INFO("at26df321",   0x1f4700,      0,  64 << 10,  64, ER_4K) },
+
+    /* EON -- en25xxx */
+    { INFO("en25f32",     0x1c3116,      0,  64 << 10,  64, ER_4K) },
+    { INFO("en25p32",     0x1c2016,      0,  64 << 10,  64, 0) },
+    { INFO("en25q32b",    0x1c3016,      0,  64 << 10,  64, 0) },
+    { INFO("en25p64",     0x1c2017,      0,  64 << 10, 128, 0) },
+
+    /* Intel/Numonyx -- xxxs33b */
+    { INFO("160s33b",     0x898911,      0,  64 << 10,  32, 0) },
+    { INFO("320s33b",     0x898912,      0,  64 << 10,  64, 0) },
+    { INFO("640s33b",     0x898913,      0,  64 << 10, 128, 0) },
+
+    /* Macronix */
+    { INFO("mx25l4005a",  0xc22013,      0,  64 << 10,   8, ER_4K) },
+    { INFO("mx25l8005",   0xc22014,      0,  64 << 10,  16, 0) },
+    { INFO("mx25l1606e",  0xc22015,      0,  64 << 10,  32, ER_4K) },
+    { INFO("mx25l3205d",  0xc22016,      0,  64 << 10,  64, 0) },
+    { INFO("mx25l6405d",  0xc22017,      0,  64 << 10, 128, 0) },
+    { INFO("mx25l12805d", 0xc22018,      0,  64 << 10, 256, 0) },
+    { INFO("mx25l12855e", 0xc22618,      0,  64 << 10, 256, 0) },
+    { INFO("mx25l25635e", 0xc22019,      0,  64 << 10, 512, 0) },
+    { INFO("mx25l25655e", 0xc22619,      0,  64 << 10, 512, 0) },
+
+    /* Spansion -- single (large) sector size only, at least
+     * for the chips listed here (without boot sectors).
+     */
+    { INFO("s25sl004a",   0x010212,      0,  64 << 10,   8, 0) },
+    { INFO("s25sl008a",   0x010213,      0,  64 << 10,  16, 0) },
+    { INFO("s25sl016a",   0x010214,      0,  64 << 10,  32, 0) },
+    { INFO("s25sl032a",   0x010215,      0,  64 << 10,  64, 0) },
+    { INFO("s25sl032p",   0x010215, 0x4d00,  64 << 10,  64, ER_4K) },
+    { INFO("s25sl064a",   0x010216,      0,  64 << 10, 128, 0) },
+    { INFO("s25fl256s0",  0x010219, 0x4d00, 256 << 10, 128, 0) },
+    { INFO("s25fl256s1",  0x010219, 0x4d01,  64 << 10, 512, 0) },
+    { INFO("s25fl512s",   0x010220, 0x4d00, 256 << 10, 256, 0) },
+    { INFO("s70fl01gs",   0x010221, 0x4d00, 256 << 10, 256, 0) },
+    { INFO("s25sl12800",  0x012018, 0x0300, 256 << 10,  64, 0) },
+    { INFO("s25sl12801",  0x012018, 0x0301,  64 << 10, 256, 0) },
+    { INFO("s25fl129p0",  0x012018, 0x4d00, 256 << 10,  64, 0) },
+    { INFO("s25fl129p1",  0x012018, 0x4d01,  64 << 10, 256, 0) },
+    { INFO("s25fl016k",   0xef4015,      0,  64 << 10,  32, ER_4K | ER_32K) },
+    { INFO("s25fl064k",   0xef4017,      0,  64 << 10, 128, ER_4K | ER_32K) },
+
+    /* SST -- large erase sizes are "overlays", "sectors" are 4<< 10 */
+    { INFO("sst25vf040b", 0xbf258d,      0,  64 << 10,   8, ER_4K) },
+    { INFO("sst25vf080b", 0xbf258e,      0,  64 << 10,  16, ER_4K) },
+    { INFO("sst25vf016b", 0xbf2541,      0,  64 << 10,  32, ER_4K) },
+    { INFO("sst25vf032b", 0xbf254a,      0,  64 << 10,  64, ER_4K) },
+    { INFO("sst25wf512",  0xbf2501,      0,  64 << 10,   1, ER_4K) },
+    { INFO("sst25wf010",  0xbf2502,      0,  64 << 10,   2, ER_4K) },
+    { INFO("sst25wf020",  0xbf2503,      0,  64 << 10,   4, ER_4K) },
+    { INFO("sst25wf040",  0xbf2504,      0,  64 << 10,   8, ER_4K) },
+
+    /* ST Microelectronics -- newer production may have feature updates */
+    { INFO("m25p05",      0x202010,      0,  32 << 10,   2, 0) },
+    { INFO("m25p10",      0x202011,      0,  32 << 10,   4, 0) },
+    { INFO("m25p20",      0x202012,      0,  64 << 10,   4, 0) },
+    { INFO("m25p40",      0x202013,      0,  64 << 10,   8, 0) },
+    { INFO("m25p80",      0x202014,      0,  64 << 10,  16, 0) },
+    { INFO("m25p16",      0x202015,      0,  64 << 10,  32, 0) },
+    { INFO("m25p32",      0x202016,      0,  64 << 10,  64, 0) },
+    { INFO("m25p64",      0x202017,      0,  64 << 10, 128, 0) },
+    { INFO("m25p128",     0x202018,      0, 256 << 10,  64, 0) },
+
+    { INFO("m45pe10",     0x204011,      0,  64 << 10,   2, 0) },
+    { INFO("m45pe80",     0x204014,      0,  64 << 10,  16, 0) },
+    { INFO("m45pe16",     0x204015,      0,  64 << 10,  32, 0) },
+
+    { INFO("m25pe80",     0x208014,      0,  64 << 10,  16, 0) },
+    { INFO("m25pe16",     0x208015,      0,  64 << 10,  32, ER_4K) },
+
+    { INFO("m25px32",     0x207116,      0,  64 << 10,  64, ER_4K) },
+    { INFO("m25px32-s0",  0x207316,      0,  64 << 10,  64, ER_4K) },
+    { INFO("m25px32-s1",  0x206316,      0,  64 << 10,  64, ER_4K) },
+    { INFO("m25px64",     0x207117,      0,  64 << 10, 128, 0) },
+
+    /* Winbond -- w25x "blocks" are 64k, "sectors" are 4KiB */
+    { INFO("w25x10",      0xef3011,      0,  64 << 10,   2, ER_4K) },
+    { INFO("w25x20",      0xef3012,      0,  64 << 10,   4, ER_4K) },
+    { INFO("w25x40",      0xef3013,      0,  64 << 10,   8, ER_4K) },
+    { INFO("w25x80",      0xef3014,      0,  64 << 10,  16, ER_4K) },
+    { INFO("w25x16",      0xef3015,      0,  64 << 10,  32, ER_4K) },
+    { INFO("w25x32",      0xef3016,      0,  64 << 10,  64, ER_4K) },
+    { INFO("w25q32",      0xef4016,      0,  64 << 10,  64, ER_4K) },
+    { INFO("w25x64",      0xef3017,      0,  64 << 10, 128, ER_4K) },
+    { INFO("w25q64",      0xef4017,      0,  64 << 10, 128, ER_4K) },
+
+    /* Numonyx -- n25q128 */
+    { INFO("n25q128",      0x20ba18,      0,  64 << 10, 256, 0) },
+
+    { },
+};
+
+typedef enum {
+    NOP = 0,
+    WRDI = 0x4,
+    RDSR = 0x5,
+    WREN = 0x6,
+    JEDEC_READ = 0x9f,
+    BULK_ERASE = 0xc7,
+
+    READ = 0x3,
+    FAST_READ = 0xb,
+    DOR = 0x3b,
+    QOR = 0x6b,
+    DIOR = 0xbb,
+    QIOR = 0xeb,
+
+    PP = 0x2,
+    DPP = 0xa2,
+    QPP = 0x32,
+
+    ERASE_4K = 0x20,
+    ERASE_32K = 0x52,
+    ERASE_SECTOR = 0xd8,
+} FlashCMD;
+
+typedef enum {
+    STATE_IDLE,
+    STATE_PAGE_PROGRAM,
+    STATE_READ,
+    STATE_COLLECTING_DATA,
+    STATE_READING_DATA,
+} CMDState;
+
+typedef struct Flash {
+    SSISlave ssidev;
+    uint32_t r;
+
+    BlockDriverState *bdrv;
+
+    uint8_t *storage;
+    uint32_t size;
+    int page_size;
+
+    uint8_t state;
+    uint8_t data[16];
+    uint32_t len;
+    uint32_t pos;
+    uint8_t needed_bytes;
+    uint8_t cmd_in_progress;
+    uint64_t cur_addr;
+    bool write_enable;
+
+    int64_t dirty_page;
+
+    char *part_name;
+    const FlashPartInfo *pi;
+
+} Flash;
+
+static void bdrv_sync_complete(void *opaque, int ret)
+{
+    /* do nothing. Masters do not directly interact with the backing store,
+     * only the working copy so no mutexing required.
+     */
+}
+
+static void flash_sync_page(Flash *s, int page)
+{
+    if (s->bdrv) {
+        int bdrv_sector, nb_sectors;
+        QEMUIOVector iov;
+
+        bdrv_sector = (page * s->pi->page_size) / BDRV_SECTOR_SIZE;
+        nb_sectors = DIV_ROUND_UP(s->pi->page_size, BDRV_SECTOR_SIZE);
+        qemu_iovec_init(&iov, 1);
+        qemu_iovec_add(&iov, s->storage + bdrv_sector * BDRV_SECTOR_SIZE,
+                                                nb_sectors * BDRV_SECTOR_SIZE);
+        bdrv_aio_writev(s->bdrv, bdrv_sector, &iov, nb_sectors,
+                                                bdrv_sync_complete, NULL);
+    }
+}
+
+static inline void flash_sync_area(Flash *s, int64_t off, int64_t len)
+{
+    int64_t start, end, nb_sectors;
+    QEMUIOVector iov;
+
+    if (!s->bdrv) {
+        return;
+    }
+
+    assert(!(len % BDRV_SECTOR_SIZE));
+    start = off / BDRV_SECTOR_SIZE;
+    end = (off + len) / BDRV_SECTOR_SIZE;
+    nb_sectors = end - start;
+    qemu_iovec_init(&iov, 1);
+    qemu_iovec_add(&iov, s->storage + (start * BDRV_SECTOR_SIZE),
+                                        nb_sectors * BDRV_SECTOR_SIZE);
+    bdrv_aio_writev(s->bdrv, start, &iov, nb_sectors, bdrv_sync_complete, NULL);
+}
+
+static void flash_erase(Flash *s, int offset, FlashCMD cmd)
+{
+    uint32_t len;
+    uint8_t capa_to_assert = 0;
+
+    switch (cmd) {
+    case ERASE_4K:
+        len = 4 << 10;
+        capa_to_assert = ER_4K;
+        break;
+    case ERASE_32K:
+        len = 32 << 10;
+        capa_to_assert = ER_32K;
+        break;
+    case ERASE_SECTOR:
+        len = s->pi->sector_size;
+        break;
+    case BULK_ERASE:
+        len = s->size;
+        break;
+    default:
+        abort();
+    }
+
+    DB_PRINT("offset = %#x, len = %d\n", offset, len);
+    if ((s->pi->flags & capa_to_assert) != capa_to_assert) {
+        hw_error("m25p80: %dk erase size not supported by device\n", len);
+    }
+
+    if (!s->write_enable) {
+        DB_PRINT("erase with write protect!\n");
+        return;
+    }
+    memset(s->storage + offset, 0xff, len);
+    flash_sync_area(s, offset, len);
+}
+
+static inline void flash_sync_dirty(Flash *s, int64_t newpage)
+{
+    if (s->dirty_page >= 0 && s->dirty_page != newpage) {
+        flash_sync_page(s, s->dirty_page);
+        s->dirty_page = newpage;
+    }
+}
+
+static inline
+void flash_write8(Flash *s, uint64_t addr, uint8_t data)
+{
+    int64_t page = addr / s->pi->page_size;
+    uint8_t prev = s->storage[s->cur_addr];
+
+    if (!s->write_enable) {
+        DB_PRINT("write with write protect!\n");
+    }
+
+    if ((prev ^ data) & data) {
+        DB_PRINT("programming zero to one! addr=%lx  %x -> %x\n",
+                  addr, prev, data);
+    }
+
+    if (s->pi->flags & WR_1) {
+        s->storage[s->cur_addr] = data;
+    } else {
+        s->storage[s->cur_addr] &= data;
+    }
+
+    flash_sync_dirty(s, page);
+    s->dirty_page = page;
+}
+
+static void complete_collecting_data(Flash *s)
+{
+    s->cur_addr = s->data[0] << 16;
+    s->cur_addr |= s->data[1] << 8;
+    s->cur_addr |= s->data[2];
+
+    switch (s->cmd_in_progress) {
+    case DPP:
+    case QPP:
+    case PP:
+        s->state = STATE_PAGE_PROGRAM;
+        break;
+    case READ:
+    case FAST_READ:
+    case DOR:
+    case QOR:
+    case DIOR:
+    case QIOR:
+        s->state = STATE_READ;
+        break;
+    case ERASE_4K:
+    case ERASE_32K:
+    case ERASE_SECTOR:
+        flash_erase(s, s->cur_addr, s->cmd_in_progress);
+        break;
+    default:
+        break;
+    }
+}
+
+static void decode_new_cmd(Flash *s, uint32_t value)
+{
+    s->cmd_in_progress = value;
+    DB_PRINT("decoded new command:%x\n", value);
+
+    switch (value) {
+
+    case ERASE_4K:
+    case ERASE_32K:
+    case ERASE_SECTOR:
+    case READ:
+    case DPP:
+    case QPP:
+    case PP:
+        s->needed_bytes = 3;
+        s->pos = 0;
+        s->len = 0;
+        s->state = STATE_COLLECTING_DATA;
+        break;
+
+    case FAST_READ:
+    case DOR:
+    case QOR:
+        s->needed_bytes = 4;
+        s->pos = 0;
+        s->len = 0;
+        s->state = STATE_COLLECTING_DATA;
+        break;
+
+    case DIOR:
+        switch ((s->pi->jedec >> 16) & 0xFF) {
+        case JEDEC_WINBOND:
+        case JEDEC_SPANSION:
+            s->needed_bytes = 4;
+            break;
+        case JEDEC_NUMONYX:
+        default:
+            s->needed_bytes = 5;
+        }
+        s->pos = 0;
+        s->len = 0;
+        s->state = STATE_COLLECTING_DATA;
+        break;
+
+    case QIOR:
+        switch ((s->pi->jedec >> 16) & 0xFF) {
+        case JEDEC_WINBOND:
+        case JEDEC_SPANSION:
+            s->needed_bytes = 6;
+            break;
+        case JEDEC_NUMONYX:
+        default:
+            s->needed_bytes = 8;
+        }
+        s->pos = 0;
+        s->len = 0;
+        s->state = STATE_COLLECTING_DATA;
+        break;
+
+    case WRDI:
+        s->write_enable = false;
+        break;
+    case WREN:
+        s->write_enable = true;
+        break;
+
+    case RDSR:
+        s->data[0] = (!!s->write_enable) << 1;
+        s->pos = 0;
+        s->len = 1;
+        s->state = STATE_READING_DATA;
+        break;
+
+    case JEDEC_READ:
+        DB_PRINT("populated jedec code\n");
+        s->data[0] = (s->pi->jedec >> 16) & 0xff;
+        s->data[1] = (s->pi->jedec >> 8) & 0xff;
+        s->data[2] = s->pi->jedec & 0xff;
+        if (s->pi->ext_jedec) {
+            s->data[3] = (s->pi->ext_jedec >> 8) & 0xff;
+            s->data[4] = s->pi->ext_jedec & 0xff;
+            s->len = 5;
+        } else {
+            s->len = 3;
+        }
+        s->pos = 0;
+        s->state = STATE_READING_DATA;
+        break;
+
+    case BULK_ERASE:
+        if (s->write_enable) {
+            DB_PRINT("chip erase\n");
+            flash_erase(s, 0, BULK_ERASE);
+        } else {
+            DB_PRINT("chip erase with write protect!\n");
+        }
+        break;
+    case NOP:
+        break;
+    default:
+        DB_PRINT("Unknown cmd %x\n", value);
+        break;
+    }
+}
+
+static int m25p80_cs(SSISlave *ss, bool select)
+{
+    Flash *s = FROM_SSI_SLAVE(Flash, ss);
+
+    if (select) {
+        s->len = 0;
+        s->pos = 0;
+        s->state = STATE_IDLE;
+        flash_sync_dirty(s, -1);
+    }
+
+    DB_PRINT("%sselect\n", select ? "de" : "");
+
+    return 0;
+}
+
+static uint32_t m25p80_transfer8(SSISlave *ss, uint32_t tx)
+{
+    Flash *s = FROM_SSI_SLAVE(Flash, ss);
+    uint32_t r = 0;
+
+    switch (s->state) {
+
+    case STATE_PAGE_PROGRAM:
+        DB_PRINT("page program cur_addr=%lx data=%x\n", s->cur_addr,
+                 (uint8_t)tx);
+        flash_write8(s, s->cur_addr, (uint8_t)tx);
+        s->cur_addr++;
+        break;
+
+    case STATE_READ:
+        r = s->storage[s->cur_addr];
+        DB_PRINT("READ 0x%lx=%x\n", s->cur_addr, r);
+        s->cur_addr = (s->cur_addr + 1) % s->size;
+        break;
+
+    case STATE_COLLECTING_DATA:
+        s->data[s->len] = (uint8_t)tx;
+        s->len++;
+
+        if (s->len == s->needed_bytes) {
+            complete_collecting_data(s);
+        }
+        break;
+
+    case STATE_READING_DATA:
+        r = s->data[s->pos];
+        s->pos++;
+        if (s->pos == s->len) {
+            s->pos = 0;
+            s->state = STATE_IDLE;
+        }
+        break;
+
+    default:
+    case STATE_IDLE:
+        decode_new_cmd(s, (uint8_t)tx);
+        break;
+    }
+
+    return r;
+}
+
+static int m25p80_init(SSISlave *ss)
+{
+    DriveInfo *dinfo;
+    Flash *s = FROM_SSI_SLAVE(Flash, ss);
+    const FlashPartInfo *i;
+
+    if (!s->part_name) { /* default to actual m25p80 if no partname given */
+        s->part_name = (char *)"m25p80";
+    }
+
+    i = known_devices;
+    for (i = known_devices;; i++) {
+        assert(i);
+        if (!i->part_name) {
+            fprintf(stderr, "Unknown SPI flash part: \"%s\"\n", s->part_name);
+            return 1;
+        } else if (!strcmp(i->part_name, s->part_name)) {
+            s->pi = i;
+            break;
+        }
+    }
+
+    s->size = s->pi->sector_size * s->pi->n_sectors;
+    s->dirty_page = -1;
+    s->storage = qemu_blockalign(s->bdrv, s->size);
+
+    dinfo = drive_get_next(IF_MTD);
+
+    if (dinfo && dinfo->bdrv) {
+        DB_PRINT("Binding to IF_MTD drive\n");
+        s->bdrv = dinfo->bdrv;
+        /* FIXME: Move to late init */
+        if (bdrv_read(s->bdrv, 0, s->storage, DIV_ROUND_UP(s->size,
+                                                    BDRV_SECTOR_SIZE))) {
+            fprintf(stderr, "Failed to initialize SPI flash!\n");
+            return 1;
+        }
+    } else {
+        memset(s->storage, 0xFF, s->size);
+    }
+
+    return 0;
+}
+
+static void m25p80_pre_save(void *opaque)
+{
+    flash_sync_dirty((Flash *)opaque, -1);
+}
+
+static const VMStateDescription vmstate_m25p80 = {
+    .name = "xilinx_spi",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .minimum_version_id_old = 1,
+    .pre_save = m25p80_pre_save,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT8(state, Flash),
+        VMSTATE_UINT8_ARRAY(data, Flash, 16),
+        VMSTATE_UINT32(len, Flash),
+        VMSTATE_UINT32(pos, Flash),
+        VMSTATE_UINT8(needed_bytes, Flash),
+        VMSTATE_UINT8(cmd_in_progress, Flash),
+        VMSTATE_UINT64(cur_addr, Flash),
+        VMSTATE_BOOL(write_enable, Flash),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static Property m25p80_properties[] = {
+    DEFINE_PROP_STRING("partname", Flash, part_name),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void m25p80_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    SSISlaveClass *k = SSI_SLAVE_CLASS(klass);
+
+    k->init = m25p80_init;
+    k->transfer = m25p80_transfer8;
+    k->set_cs = m25p80_cs;
+    k->cs_polarity = SSI_CS_LOW;
+    dc->props = m25p80_properties;
+    dc->vmsd = &vmstate_m25p80;
+}
+
+static const TypeInfo m25p80_info = {
+    .name           = "m25p80",
+    .parent         = TYPE_SSI_SLAVE,
+    .instance_size  = sizeof(Flash),
+    .class_init     = m25p80_class_init,
+};
+
+static void m25p80_register_types(void)
+{
+    type_register_static(&m25p80_info);
+}
+
+type_init(m25p80_register_types)
diff --git a/hw/m48t59.c b/hw/m48t59.c
index dd6cb37ba..7da7e7c82 100644
--- a/hw/m48t59.c
+++ b/hw/m48t59.c
@@ -27,6 +27,7 @@
 #include "sysemu.h"
 #include "sysbus.h"
 #include "isa.h"
+#include "exec-memory.h"
 
 //#define DEBUG_NVRAM
 
@@ -80,6 +81,7 @@ typedef struct M48t59ISAState {
 typedef struct M48t59SysBusState {
     SysBusDevice busdev;
     M48t59State state;
+    MemoryRegion io;
 } M48t59SysBusState;
 
 /* Fake timer functions */
@@ -466,13 +468,6 @@ uint32_t m48t59_read (void *opaque, uint32_t addr)
     return retval;
 }
 
-void m48t59_set_addr (void *opaque, uint32_t addr)
-{
-    M48t59State *NVRAM = opaque;
-
-    NVRAM->addr = addr;
-}
-
 void m48t59_toggle_lock (void *opaque, int lock)
 {
     M48t59State *NVRAM = opaque;
@@ -481,7 +476,8 @@ void m48t59_toggle_lock (void *opaque, int lock)
 }
 
 /* IO access to NVRAM */
-static void NVRAM_writeb (void *opaque, uint32_t addr, uint32_t val)
+static void NVRAM_writeb(void *opaque, hwaddr addr, uint64_t val,
+                         unsigned size)
 {
     M48t59State *NVRAM = opaque;
 
@@ -504,7 +500,7 @@ static void NVRAM_writeb (void *opaque, uint32_t addr, uint32_t val)
     }
 }
 
-static uint32_t NVRAM_readb (void *opaque, uint32_t addr)
+static uint64_t NVRAM_readb(void *opaque, hwaddr addr, unsigned size)
 {
     M48t59State *NVRAM = opaque;
     uint32_t retval;
@@ -522,14 +518,14 @@ static uint32_t NVRAM_readb (void *opaque, uint32_t addr)
     return retval;
 }
 
-static void nvram_writeb (void *opaque, target_phys_addr_t addr, uint32_t value)
+static void nvram_writeb (void *opaque, hwaddr addr, uint32_t value)
 {
     M48t59State *NVRAM = opaque;
 
     m48t59_write(NVRAM, addr, value & 0xff);
 }
 
-static void nvram_writew (void *opaque, target_phys_addr_t addr, uint32_t value)
+static void nvram_writew (void *opaque, hwaddr addr, uint32_t value)
 {
     M48t59State *NVRAM = opaque;
 
@@ -537,7 +533,7 @@ static void nvram_writew (void *opaque, target_phys_addr_t addr, uint32_t value)
     m48t59_write(NVRAM, addr + 1, value & 0xff);
 }
 
-static void nvram_writel (void *opaque, target_phys_addr_t addr, uint32_t value)
+static void nvram_writel (void *opaque, hwaddr addr, uint32_t value)
 {
     M48t59State *NVRAM = opaque;
 
@@ -547,7 +543,7 @@ static void nvram_writel (void *opaque, target_phys_addr_t addr, uint32_t value)
     m48t59_write(NVRAM, addr + 3, value & 0xff);
 }
 
-static uint32_t nvram_readb (void *opaque, target_phys_addr_t addr)
+static uint32_t nvram_readb (void *opaque, hwaddr addr)
 {
     M48t59State *NVRAM = opaque;
     uint32_t retval;
@@ -556,7 +552,7 @@ static uint32_t nvram_readb (void *opaque, target_phys_addr_t addr)
     return retval;
 }
 
-static uint32_t nvram_readw (void *opaque, target_phys_addr_t addr)
+static uint32_t nvram_readw (void *opaque, hwaddr addr)
 {
     M48t59State *NVRAM = opaque;
     uint32_t retval;
@@ -566,7 +562,7 @@ static uint32_t nvram_readw (void *opaque, target_phys_addr_t addr)
     return retval;
 }
 
-static uint32_t nvram_readl (void *opaque, target_phys_addr_t addr)
+static uint32_t nvram_readl (void *opaque, hwaddr addr)
 {
     M48t59State *NVRAM = opaque;
     uint32_t retval;
@@ -626,17 +622,18 @@ static void m48t59_reset_sysbus(DeviceState *d)
     m48t59_reset_common(NVRAM);
 }
 
-static const MemoryRegionPortio m48t59_portio[] = {
-    {0, 4, 1, .read = NVRAM_readb, .write = NVRAM_writeb },
-    PORTIO_END_OF_LIST(),
-};
-
 static const MemoryRegionOps m48t59_io_ops = {
-    .old_portio = m48t59_portio,
+    .read = NVRAM_readb,
+    .write = NVRAM_writeb,
+    .impl = {
+        .min_access_size = 1,
+        .max_access_size = 1,
+    },
+    .endianness = DEVICE_LITTLE_ENDIAN,
 };
 
 /* Initialisation routine */
-M48t59State *m48t59_init(qemu_irq IRQ, target_phys_addr_t mem_base,
+M48t59State *m48t59_init(qemu_irq IRQ, hwaddr mem_base,
                          uint32_t io_base, uint16_t size, int model)
 {
     DeviceState *dev;
@@ -653,9 +650,9 @@ M48t59State *m48t59_init(qemu_irq IRQ, target_phys_addr_t mem_base,
     d = FROM_SYSBUS(M48t59SysBusState, s);
     state = &d->state;
     sysbus_connect_irq(s, 0, IRQ);
+    memory_region_init_io(&d->io, &m48t59_io_ops, state, "m48t59", 4);
     if (io_base != 0) {
-        register_ioport_read(io_base, 0x04, 1, NVRAM_readb, state);
-        register_ioport_write(io_base, 0x04, 1, NVRAM_writeb, state);
+        memory_region_add_subregion(get_system_io(), io_base, &d->io);
     }
     if (mem_base != 0) {
         sysbus_mmio_map(s, 0, mem_base);
diff --git a/hw/mac_dbdma.c b/hw/mac_dbdma.c
index 1791ec12e..e551156af 100644
--- a/hw/mac_dbdma.c
+++ b/hw/mac_dbdma.c
@@ -699,7 +699,7 @@ dbdma_control_write(DBDMA_channel *ch)
         ch->flush(&ch->io);
 }
 
-static void dbdma_write(void *opaque, target_phys_addr_t addr,
+static void dbdma_write(void *opaque, hwaddr addr,
                         uint64_t value, unsigned size)
 {
     int channel = addr >> DBDMA_CHANNEL_SHIFT;
@@ -749,7 +749,7 @@ static void dbdma_write(void *opaque, target_phys_addr_t addr,
     }
 }
 
-static uint64_t dbdma_read(void *opaque, target_phys_addr_t addr,
+static uint64_t dbdma_read(void *opaque, hwaddr addr,
                            unsigned size)
 {
     uint32_t value;
diff --git a/hw/mac_dbdma.h b/hw/mac_dbdma.h
index 6d1abe6aa..bfdb0ddc6 100644
--- a/hw/mac_dbdma.h
+++ b/hw/mac_dbdma.h
@@ -30,7 +30,7 @@ typedef void (*DBDMA_end)(DBDMA_io *io);
 struct DBDMA_io {
     void *opaque;
     void *channel;
-    target_phys_addr_t addr;
+    hwaddr addr;
     int len;
     int is_last;
     int is_dma_out;
diff --git a/hw/mac_nvram.c b/hw/mac_nvram.c
index ed0a2b7ef..a0d14dd3c 100644
--- a/hw/mac_nvram.c
+++ b/hw/mac_nvram.c
@@ -71,7 +71,7 @@ void macio_nvram_write (void *opaque, uint32_t addr, uint32_t val)
 }
 
 /* macio style NVRAM device */
-static void macio_nvram_writeb(void *opaque, target_phys_addr_t addr,
+static void macio_nvram_writeb(void *opaque, hwaddr addr,
                                uint64_t value, unsigned size)
 {
     MacIONVRAMState *s = opaque;
@@ -81,7 +81,7 @@ static void macio_nvram_writeb(void *opaque, target_phys_addr_t addr,
     NVR_DPRINTF("writeb addr %04x val %x\n", (int)addr, value);
 }
 
-static uint64_t macio_nvram_readb(void *opaque, target_phys_addr_t addr,
+static uint64_t macio_nvram_readb(void *opaque, hwaddr addr,
                                   unsigned size)
 {
     MacIONVRAMState *s = opaque;
@@ -116,7 +116,7 @@ static void macio_nvram_reset(void *opaque)
 {
 }
 
-MacIONVRAMState *macio_nvram_init (target_phys_addr_t size,
+MacIONVRAMState *macio_nvram_init (hwaddr size,
                                    unsigned int it_shift)
 {
     MacIONVRAMState *s;
@@ -135,7 +135,7 @@ MacIONVRAMState *macio_nvram_init (target_phys_addr_t size,
 }
 
 void macio_nvram_setup_bar(MacIONVRAMState *s, MemoryRegion *bar,
-                           target_phys_addr_t mem_base)
+                           hwaddr mem_base)
 {
     memory_region_add_subregion(bar, mem_base, &s->mem);
 }
diff --git a/hw/mainstone.c b/hw/mainstone.c
index 97687b6ee..5bbecb730 100644
--- a/hw/mainstone.c
+++ b/hw/mainstone.c
@@ -95,19 +95,18 @@ static struct arm_boot_info mainstone_binfo = {
 };
 
 static void mainstone_common_init(MemoryRegion *address_space_mem,
-                ram_addr_t ram_size,
-                const char *kernel_filename,
-                const char *kernel_cmdline, const char *initrd_filename,
-                const char *cpu_model, enum mainstone_model_e model, int arm_id)
+                                  QEMUMachineInitArgs *args,
+                                  enum mainstone_model_e model, int arm_id)
 {
     uint32_t sector_len = 256 * 1024;
-    target_phys_addr_t mainstone_flash_base[] = { MST_FLASH_0, MST_FLASH_1 };
+    hwaddr mainstone_flash_base[] = { MST_FLASH_0, MST_FLASH_1 };
     PXA2xxState *mpu;
     DeviceState *mst_irq;
     DriveInfo *dinfo;
     int i;
     int be;
     MemoryRegion *rom = g_new(MemoryRegion, 1);
+    const char *cpu_model = args->cpu_model;
 
     if (!cpu_model)
         cpu_model = "pxa270-c5";
@@ -164,20 +163,16 @@ static void mainstone_common_init(MemoryRegion *address_space_mem,
     smc91c111_init(&nd_table[0], MST_ETH_PHYS,
                     qdev_get_gpio_in(mst_irq, ETHERNET_IRQ));
 
-    mainstone_binfo.kernel_filename = kernel_filename;
-    mainstone_binfo.kernel_cmdline = kernel_cmdline;
-    mainstone_binfo.initrd_filename = initrd_filename;
+    mainstone_binfo.kernel_filename = args->kernel_filename;
+    mainstone_binfo.kernel_cmdline = args->kernel_cmdline;
+    mainstone_binfo.initrd_filename = args->initrd_filename;
     mainstone_binfo.board_id = arm_id;
     arm_load_kernel(mpu->cpu, &mainstone_binfo);
 }
 
-static void mainstone_init(ram_addr_t ram_size,
-                const char *boot_device,
-                const char *kernel_filename, const char *kernel_cmdline,
-                const char *initrd_filename, const char *cpu_model)
+static void mainstone_init(QEMUMachineInitArgs *args)
 {
-    mainstone_common_init(get_system_memory(), ram_size, kernel_filename,
-                kernel_cmdline, initrd_filename, cpu_model, mainstone, 0x196);
+    mainstone_common_init(get_system_memory(), args, mainstone, 0x196);
 }
 
 static QEMUMachine mainstone2_machine = {
diff --git a/hw/marvell_88w8618_audio.c b/hw/marvell_88w8618_audio.c
index f6f193744..de16cfa09 100644
--- a/hw/marvell_88w8618_audio.c
+++ b/hw/marvell_88w8618_audio.c
@@ -138,7 +138,7 @@ static void mv88w8618_audio_clock_update(mv88w8618_audio_state *s)
     wm8750_set_bclk_in(s->wm, rate);
 }
 
-static uint64_t mv88w8618_audio_read(void *opaque, target_phys_addr_t offset,
+static uint64_t mv88w8618_audio_read(void *opaque, hwaddr offset,
                                     unsigned size)
 {
     mv88w8618_audio_state *s = opaque;
@@ -164,7 +164,7 @@ static uint64_t mv88w8618_audio_read(void *opaque, target_phys_addr_t offset,
     }
 }
 
-static void mv88w8618_audio_write(void *opaque, target_phys_addr_t offset,
+static void mv88w8618_audio_write(void *opaque, hwaddr offset,
                                   uint64_t value, unsigned size)
 {
     mv88w8618_audio_state *s = opaque;
diff --git a/hw/max111x.c b/hw/max111x.c
index 706d89f4f..67640f109 100644
--- a/hw/max111x.c
+++ b/hw/max111x.c
@@ -99,10 +99,11 @@ static uint32_t max111x_transfer(SSISlave *dev, uint32_t value)
 
 static const VMStateDescription vmstate_max111x = {
     .name = "max111x",
-    .version_id = 0,
-    .minimum_version_id = 0,
-    .minimum_version_id_old = 0,
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .minimum_version_id_old = 1,
     .fields      = (VMStateField[]) {
+        VMSTATE_SSI_SLAVE(ssidev, MAX111xState),
         VMSTATE_UINT8(tb1, MAX111xState),
         VMSTATE_UINT8(rb2, MAX111xState),
         VMSTATE_UINT8(rb3, MAX111xState),
diff --git a/hw/mc146818rtc.c b/hw/mc146818rtc.c
index 3777f858a..c79fca7d6 100644
--- a/hw/mc146818rtc.c
+++ b/hw/mc146818rtc.c
@@ -25,6 +25,7 @@
 #include "qemu-timer.h"
 #include "sysemu.h"
 #include "mc146818rtc.h"
+#include "qapi/qapi-visit-core.h"
 
 #ifdef TARGET_I386
 #include "apic.h"
@@ -45,36 +46,65 @@
 # define DPRINTF_C(format, ...)      do { } while (0)
 #endif
 
+#define NSEC_PER_SEC    1000000000LL
+#define SEC_PER_MIN     60
+#define MIN_PER_HOUR    60
+#define SEC_PER_HOUR    3600
+#define HOUR_PER_DAY    24
+#define SEC_PER_DAY     86400
+
 #define RTC_REINJECT_ON_ACK_COUNT 20
+#define RTC_CLOCK_RATE            32768
+#define UIP_HOLD_LENGTH           (8 * NSEC_PER_SEC / 32768)
 
 typedef struct RTCState {
     ISADevice dev;
     MemoryRegion io;
     uint8_t cmos_data[128];
     uint8_t cmos_index;
-    struct tm current_tm;
     int32_t base_year;
+    uint64_t base_rtc;
+    uint64_t last_update;
+    int64_t offset;
     qemu_irq irq;
     qemu_irq sqw_irq;
     int it_shift;
     /* periodic timer */
     QEMUTimer *periodic_timer;
     int64_t next_periodic_time;
-    /* second update */
-    int64_t next_second_time;
+    /* update-ended timer */
+    QEMUTimer *update_timer;
+    uint64_t next_alarm_time;
     uint16_t irq_reinject_on_ack_count;
     uint32_t irq_coalesced;
     uint32_t period;
     QEMUTimer *coalesced_timer;
-    QEMUTimer *second_timer;
-    QEMUTimer *second_timer2;
     Notifier clock_reset_notifier;
     LostTickPolicy lost_tick_policy;
     Notifier suspend_notifier;
 } RTCState;
 
 static void rtc_set_time(RTCState *s);
-static void rtc_copy_date(RTCState *s);
+static void rtc_update_time(RTCState *s);
+static void rtc_set_cmos(RTCState *s, const struct tm *tm);
+static inline int rtc_from_bcd(RTCState *s, int a);
+static uint64_t get_next_alarm(RTCState *s);
+
+static inline bool rtc_running(RTCState *s)
+{
+    return (!(s->cmos_data[RTC_REG_B] & REG_B_SET) &&
+            (s->cmos_data[RTC_REG_A] & 0x70) <= 0x20);
+}
+
+static uint64_t get_guest_rtc_ns(RTCState *s)
+{
+    uint64_t guest_rtc;
+    uint64_t guest_clock = qemu_get_clock_ns(rtc_clock);
+
+    guest_rtc = s->base_rtc * NSEC_PER_SEC
+                 + guest_clock - s->last_update + s->offset;
+    return guest_rtc;
+}
 
 #ifdef TARGET_I386
 static void rtc_coalesced_timer_update(RTCState *s)
@@ -85,7 +115,7 @@ static void rtc_coalesced_timer_update(RTCState *s)
         /* divide each RTC interval to 2 - 8 smaller intervals */
         int c = MIN(s->irq_coalesced, 7) + 1; 
         int64_t next_clock = qemu_get_clock_ns(rtc_clock) +
-            muldiv64(s->period / c, get_ticks_per_sec(), 32768);
+            muldiv64(s->period / c, get_ticks_per_sec(), RTC_CLOCK_RATE);
         qemu_mod_timer(s->coalesced_timer, next_clock);
     }
 }
@@ -110,7 +140,8 @@ static void rtc_coalesced_timer(void *opaque)
 }
 #endif
 
-static void rtc_timer_update(RTCState *s, int64_t current_time)
+/* handle periodic timer */
+static void periodic_timer_update(RTCState *s, int64_t current_time)
 {
     int period_code, period;
     int64_t cur_clock, next_irq_clock;
@@ -131,10 +162,10 @@ static void rtc_timer_update(RTCState *s, int64_t current_time)
         s->period = period;
 #endif
         /* compute 32 khz clock */
-        cur_clock = muldiv64(current_time, 32768, get_ticks_per_sec());
+        cur_clock = muldiv64(current_time, RTC_CLOCK_RATE, get_ticks_per_sec());
         next_irq_clock = (cur_clock & ~(period - 1)) + period;
         s->next_periodic_time =
-            muldiv64(next_irq_clock, get_ticks_per_sec(), 32768) + 1;
+            muldiv64(next_irq_clock, get_ticks_per_sec(), RTC_CLOCK_RATE) + 1;
         qemu_mod_timer(s->periodic_timer, s->next_periodic_time);
     } else {
 #ifdef TARGET_I386
@@ -148,7 +179,7 @@ static void rtc_periodic_timer(void *opaque)
 {
     RTCState *s = opaque;
 
-    rtc_timer_update(s, s->next_periodic_time);
+    periodic_timer_update(s, s->next_periodic_time);
     s->cmos_data[RTC_REG_C] |= REG_C_PF;
     if (s->cmos_data[RTC_REG_B] & REG_B_PIE) {
         s->cmos_data[RTC_REG_C] |= REG_C_IRQF;
@@ -175,7 +206,186 @@ static void rtc_periodic_timer(void *opaque)
     }
 }
 
-static void cmos_ioport_write(void *opaque, uint32_t addr, uint32_t data)
+/* handle update-ended timer */
+static void check_update_timer(RTCState *s)
+{
+    uint64_t next_update_time;
+    uint64_t guest_nsec;
+    int next_alarm_sec;
+
+    /* From the data sheet: "Holding the dividers in reset prevents
+     * interrupts from operating, while setting the SET bit allows"
+     * them to occur.  However, it will prevent an alarm interrupt
+     * from occurring, because the time of day is not updated.
+     */
+    if ((s->cmos_data[RTC_REG_A] & 0x60) == 0x60) {
+        qemu_del_timer(s->update_timer);
+        return;
+    }
+    if ((s->cmos_data[RTC_REG_C] & REG_C_UF) &&
+        (s->cmos_data[RTC_REG_B] & REG_B_SET)) {
+        qemu_del_timer(s->update_timer);
+        return;
+    }
+    if ((s->cmos_data[RTC_REG_C] & REG_C_UF) &&
+        (s->cmos_data[RTC_REG_C] & REG_C_AF)) {
+        qemu_del_timer(s->update_timer);
+        return;
+    }
+
+    guest_nsec = get_guest_rtc_ns(s) % NSEC_PER_SEC;
+    /* if UF is clear, reprogram to next second */
+    next_update_time = qemu_get_clock_ns(rtc_clock)
+        + NSEC_PER_SEC - guest_nsec;
+
+    /* Compute time of next alarm.  One second is already accounted
+     * for in next_update_time.
+     */
+    next_alarm_sec = get_next_alarm(s);
+    s->next_alarm_time = next_update_time + (next_alarm_sec - 1) * NSEC_PER_SEC;
+
+    if (s->cmos_data[RTC_REG_C] & REG_C_UF) {
+        /* UF is set, but AF is clear.  Program the timer to target
+         * the alarm time.  */
+        next_update_time = s->next_alarm_time;
+    }
+    if (next_update_time != qemu_timer_expire_time_ns(s->update_timer)) {
+        qemu_mod_timer(s->update_timer, next_update_time);
+    }
+}
+
+static inline uint8_t convert_hour(RTCState *s, uint8_t hour)
+{
+    if (!(s->cmos_data[RTC_REG_B] & REG_B_24H)) {
+        hour %= 12;
+        if (s->cmos_data[RTC_HOURS] & 0x80) {
+            hour += 12;
+        }
+    }
+    return hour;
+}
+
+static uint64_t get_next_alarm(RTCState *s)
+{
+    int32_t alarm_sec, alarm_min, alarm_hour, cur_hour, cur_min, cur_sec;
+    int32_t hour, min, sec;
+
+    rtc_update_time(s);
+
+    alarm_sec = rtc_from_bcd(s, s->cmos_data[RTC_SECONDS_ALARM]);
+    alarm_min = rtc_from_bcd(s, s->cmos_data[RTC_MINUTES_ALARM]);
+    alarm_hour = rtc_from_bcd(s, s->cmos_data[RTC_HOURS_ALARM]);
+    alarm_hour = alarm_hour == -1 ? -1 : convert_hour(s, alarm_hour);
+
+    cur_sec = rtc_from_bcd(s, s->cmos_data[RTC_SECONDS]);
+    cur_min = rtc_from_bcd(s, s->cmos_data[RTC_MINUTES]);
+    cur_hour = rtc_from_bcd(s, s->cmos_data[RTC_HOURS]);
+    cur_hour = convert_hour(s, cur_hour);
+
+    if (alarm_hour == -1) {
+        alarm_hour = cur_hour;
+        if (alarm_min == -1) {
+            alarm_min = cur_min;
+            if (alarm_sec == -1) {
+                alarm_sec = cur_sec + 1;
+            } else if (cur_sec > alarm_sec) {
+                alarm_min++;
+            }
+        } else if (cur_min == alarm_min) {
+            if (alarm_sec == -1) {
+                alarm_sec = cur_sec + 1;
+            } else {
+                if (cur_sec > alarm_sec) {
+                    alarm_hour++;
+                }
+            }
+            if (alarm_sec == SEC_PER_MIN) {
+                /* wrap to next hour, minutes is not in don't care mode */
+                alarm_sec = 0;
+                alarm_hour++;
+            }
+        } else if (cur_min > alarm_min) {
+            alarm_hour++;
+        }
+    } else if (cur_hour == alarm_hour) {
+        if (alarm_min == -1) {
+            alarm_min = cur_min;
+            if (alarm_sec == -1) {
+                alarm_sec = cur_sec + 1;
+            } else if (cur_sec > alarm_sec) {
+                alarm_min++;
+            }
+
+            if (alarm_sec == SEC_PER_MIN) {
+                alarm_sec = 0;
+                alarm_min++;
+            }
+            /* wrap to next day, hour is not in don't care mode */
+            alarm_min %= MIN_PER_HOUR;
+        } else if (cur_min == alarm_min) {
+            if (alarm_sec == -1) {
+                alarm_sec = cur_sec + 1;
+            }
+            /* wrap to next day, hours+minutes not in don't care mode */
+            alarm_sec %= SEC_PER_MIN;
+        }
+    }
+
+    /* values that are still don't care fire at the next min/sec */
+    if (alarm_min == -1) {
+        alarm_min = 0;
+    }
+    if (alarm_sec == -1) {
+        alarm_sec = 0;
+    }
+
+    /* keep values in range */
+    if (alarm_sec == SEC_PER_MIN) {
+        alarm_sec = 0;
+        alarm_min++;
+    }
+    if (alarm_min == MIN_PER_HOUR) {
+        alarm_min = 0;
+        alarm_hour++;
+    }
+    alarm_hour %= HOUR_PER_DAY;
+
+    hour = alarm_hour - cur_hour;
+    min = hour * MIN_PER_HOUR + alarm_min - cur_min;
+    sec = min * SEC_PER_MIN + alarm_sec - cur_sec;
+    return sec <= 0 ? sec + SEC_PER_DAY : sec;
+}
+
+static void rtc_update_timer(void *opaque)
+{
+    RTCState *s = opaque;
+    int32_t irqs = REG_C_UF;
+    int32_t new_irqs;
+
+    assert((s->cmos_data[RTC_REG_A] & 0x60) != 0x60);
+
+    /* UIP might have been latched, update time and clear it.  */
+    rtc_update_time(s);
+    s->cmos_data[RTC_REG_A] &= ~REG_A_UIP;
+
+    if (qemu_get_clock_ns(rtc_clock) >= s->next_alarm_time) {
+        irqs |= REG_C_AF;
+        if (s->cmos_data[RTC_REG_B] & REG_B_AIE) {
+            qemu_system_wakeup_request(QEMU_WAKEUP_REASON_RTC);
+        }
+    }
+
+    new_irqs = irqs & ~s->cmos_data[RTC_REG_C];
+    s->cmos_data[RTC_REG_C] |= irqs;
+    if ((new_irqs & s->cmos_data[RTC_REG_B]) != 0) {
+        s->cmos_data[RTC_REG_C] |= REG_C_IRQF;
+        qemu_irq_raise(s->irq);
+    }
+    check_update_timer(s);
+}
+
+static void cmos_ioport_write(void *opaque, hwaddr addr,
+                              uint64_t data, unsigned size)
 {
     RTCState *s = opaque;
 
@@ -189,7 +399,12 @@ static void cmos_ioport_write(void *opaque, uint32_t addr, uint32_t data)
         case RTC_MINUTES_ALARM:
         case RTC_HOURS_ALARM:
             s->cmos_data[s->cmos_index] = data;
+            check_update_timer(s);
             break;
+	case RTC_IBM_PS2_CENTURY_BYTE:
+            s->cmos_index = RTC_CENTURY;
+            /* fall through */
+        case RTC_CENTURY:
         case RTC_SECONDS:
         case RTC_MINUTES:
         case RTC_HOURS:
@@ -199,37 +414,66 @@ static void cmos_ioport_write(void *opaque, uint32_t addr, uint32_t data)
         case RTC_YEAR:
             s->cmos_data[s->cmos_index] = data;
             /* if in set mode, do not update the time */
-            if (!(s->cmos_data[RTC_REG_B] & REG_B_SET)) {
+            if (rtc_running(s)) {
                 rtc_set_time(s);
+                check_update_timer(s);
             }
             break;
         case RTC_REG_A:
+            if ((data & 0x60) == 0x60) {
+                if (rtc_running(s)) {
+                    rtc_update_time(s);
+                }
+                /* What happens to UIP when divider reset is enabled is
+                 * unclear from the datasheet.  Shouldn't matter much
+                 * though.
+                 */
+                s->cmos_data[RTC_REG_A] &= ~REG_A_UIP;
+            } else if (((s->cmos_data[RTC_REG_A] & 0x60) == 0x60) &&
+                    (data & 0x70)  <= 0x20) {
+                /* when the divider reset is removed, the first update cycle
+                 * begins one-half second later*/
+                if (!(s->cmos_data[RTC_REG_B] & REG_B_SET)) {
+                    s->offset = 500000000;
+                    rtc_set_time(s);
+                }
+                s->cmos_data[RTC_REG_A] &= ~REG_A_UIP;
+            }
             /* UIP bit is read only */
             s->cmos_data[RTC_REG_A] = (data & ~REG_A_UIP) |
                 (s->cmos_data[RTC_REG_A] & REG_A_UIP);
-            rtc_timer_update(s, qemu_get_clock_ns(rtc_clock));
+            periodic_timer_update(s, qemu_get_clock_ns(rtc_clock));
+            check_update_timer(s);
             break;
         case RTC_REG_B:
             if (data & REG_B_SET) {
+                /* update cmos to when the rtc was stopping */
+                if (rtc_running(s)) {
+                    rtc_update_time(s);
+                }
                 /* set mode: reset UIP mode */
                 s->cmos_data[RTC_REG_A] &= ~REG_A_UIP;
                 data &= ~REG_B_UIE;
             } else {
                 /* if disabling set mode, update the time */
-                if (s->cmos_data[RTC_REG_B] & REG_B_SET) {
+                if ((s->cmos_data[RTC_REG_B] & REG_B_SET) &&
+                    (s->cmos_data[RTC_REG_A] & 0x70) <= 0x20) {
+                    s->offset = get_guest_rtc_ns(s) % NSEC_PER_SEC;
                     rtc_set_time(s);
                 }
             }
-            if (((s->cmos_data[RTC_REG_B] ^ data) & (REG_B_DM | REG_B_24H)) &&
-                !(data & REG_B_SET)) {
-                /* If the time format has changed and not in set mode,
-                   update the registers immediately. */
-                s->cmos_data[RTC_REG_B] = data;
-                rtc_copy_date(s);
+            /* if an interrupt flag is already set when the interrupt
+             * becomes enabled, raise an interrupt immediately.  */
+            if (data & s->cmos_data[RTC_REG_C] & REG_C_MASK) {
+                s->cmos_data[RTC_REG_C] |= REG_C_IRQF;
+                qemu_irq_raise(s->irq);
             } else {
-                s->cmos_data[RTC_REG_B] = data;
+                s->cmos_data[RTC_REG_C] &= ~REG_C_IRQF;
+                qemu_irq_lower(s->irq);
             }
-            rtc_timer_update(s, qemu_get_clock_ns(rtc_clock));
+            s->cmos_data[RTC_REG_B] = data;
+            periodic_timer_update(s, qemu_get_clock_ns(rtc_clock));
+            check_update_timer(s);
             break;
         case RTC_REG_C:
         case RTC_REG_D:
@@ -253,6 +497,9 @@ static inline int rtc_to_bcd(RTCState *s, int a)
 
 static inline int rtc_from_bcd(RTCState *s, int a)
 {
+    if ((a & 0xc0) == 0xc0) {
+        return -1;
+    }
     if (s->cmos_data[RTC_REG_B] & REG_B_DM) {
         return a;
     } else {
@@ -260,10 +507,8 @@ static inline int rtc_from_bcd(RTCState *s, int a)
     }
 }
 
-static void rtc_set_time(RTCState *s)
+static void rtc_get_time(RTCState *s, struct tm *tm)
 {
-    struct tm *tm = &s->current_tm;
-
     tm->tm_sec = rtc_from_bcd(s, s->cmos_data[RTC_SECONDS]);
     tm->tm_min = rtc_from_bcd(s, s->cmos_data[RTC_MINUTES]);
     tm->tm_hour = rtc_from_bcd(s, s->cmos_data[RTC_HOURS] & 0x7f);
@@ -276,14 +521,24 @@ static void rtc_set_time(RTCState *s)
     tm->tm_wday = rtc_from_bcd(s, s->cmos_data[RTC_DAY_OF_WEEK]) - 1;
     tm->tm_mday = rtc_from_bcd(s, s->cmos_data[RTC_DAY_OF_MONTH]);
     tm->tm_mon = rtc_from_bcd(s, s->cmos_data[RTC_MONTH]) - 1;
-    tm->tm_year = rtc_from_bcd(s, s->cmos_data[RTC_YEAR]) + s->base_year - 1900;
+    tm->tm_year =
+        rtc_from_bcd(s, s->cmos_data[RTC_YEAR]) + s->base_year +
+        rtc_from_bcd(s, s->cmos_data[RTC_CENTURY]) * 100 - 1900;
+}
+
+static void rtc_set_time(RTCState *s)
+{
+    struct tm tm;
+
+    rtc_get_time(s, &tm);
+    s->base_rtc = mktimegm(&tm);
+    s->last_update = qemu_get_clock_ns(rtc_clock);
 
-    rtc_change_mon_event(tm);
+    rtc_change_mon_event(&tm);
 }
 
-static void rtc_copy_date(RTCState *s)
+static void rtc_set_cmos(RTCState *s, const struct tm *tm)
 {
-    const struct tm *tm = &s->current_tm;
     int year;
 
     s->cmos_data[RTC_SECONDS] = rtc_to_bcd(s, tm->tm_sec);
@@ -301,131 +556,53 @@ static void rtc_copy_date(RTCState *s)
     s->cmos_data[RTC_DAY_OF_WEEK] = rtc_to_bcd(s, tm->tm_wday + 1);
     s->cmos_data[RTC_DAY_OF_MONTH] = rtc_to_bcd(s, tm->tm_mday);
     s->cmos_data[RTC_MONTH] = rtc_to_bcd(s, tm->tm_mon + 1);
-    year = (tm->tm_year - s->base_year) % 100;
-    if (year < 0)
-        year += 100;
-    s->cmos_data[RTC_YEAR] = rtc_to_bcd(s, year);
+    year = tm->tm_year + 1900 - s->base_year;
+    s->cmos_data[RTC_YEAR] = rtc_to_bcd(s, year % 100);
+    s->cmos_data[RTC_CENTURY] = rtc_to_bcd(s, year / 100);
 }
 
-/* month is between 0 and 11. */
-static int get_days_in_month(int month, int year)
+static void rtc_update_time(RTCState *s)
 {
-    static const int days_tab[12] = {
-        31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31
-    };
-    int d;
-    if ((unsigned )month >= 12)
-        return 31;
-    d = days_tab[month];
-    if (month == 1) {
-        if ((year % 4) == 0 && ((year % 100) != 0 || (year % 400) == 0))
-            d++;
-    }
-    return d;
-}
+    struct tm ret;
+    time_t guest_sec;
+    int64_t guest_nsec;
 
-/* update 'tm' to the next second */
-static void rtc_next_second(struct tm *tm)
-{
-    int days_in_month;
-
-    tm->tm_sec++;
-    if ((unsigned)tm->tm_sec >= 60) {
-        tm->tm_sec = 0;
-        tm->tm_min++;
-        if ((unsigned)tm->tm_min >= 60) {
-            tm->tm_min = 0;
-            tm->tm_hour++;
-            if ((unsigned)tm->tm_hour >= 24) {
-                tm->tm_hour = 0;
-                /* next day */
-                tm->tm_wday++;
-                if ((unsigned)tm->tm_wday >= 7)
-                    tm->tm_wday = 0;
-                days_in_month = get_days_in_month(tm->tm_mon,
-                                                  tm->tm_year + 1900);
-                tm->tm_mday++;
-                if (tm->tm_mday < 1) {
-                    tm->tm_mday = 1;
-                } else if (tm->tm_mday > days_in_month) {
-                    tm->tm_mday = 1;
-                    tm->tm_mon++;
-                    if (tm->tm_mon >= 12) {
-                        tm->tm_mon = 0;
-                        tm->tm_year++;
-                    }
-                }
-            }
-        }
-    }
-}
-
-
-static void rtc_update_second(void *opaque)
-{
-    RTCState *s = opaque;
-    int64_t delay;
-
-    /* if the oscillator is not in normal operation, we do not update */
-    if ((s->cmos_data[RTC_REG_A] & 0x70) != 0x20) {
-        s->next_second_time += get_ticks_per_sec();
-        qemu_mod_timer(s->second_timer, s->next_second_time);
-    } else {
-        rtc_next_second(&s->current_tm);
+    guest_nsec = get_guest_rtc_ns(s);
+    guest_sec = guest_nsec / NSEC_PER_SEC;
+    gmtime_r(&guest_sec, &ret);
 
-        if (!(s->cmos_data[RTC_REG_B] & REG_B_SET)) {
-            /* update in progress bit */
-            s->cmos_data[RTC_REG_A] |= REG_A_UIP;
-        }
-        /* should be 244 us = 8 / 32768 seconds, but currently the
-           timers do not have the necessary resolution. */
-        delay = (get_ticks_per_sec() * 1) / 100;
-        if (delay < 1)
-            delay = 1;
-        qemu_mod_timer(s->second_timer2,
-                       s->next_second_time + delay);
+    /* Is SET flag of Register B disabled? */
+    if ((s->cmos_data[RTC_REG_B] & REG_B_SET) == 0) {
+        rtc_set_cmos(s, &ret);
     }
 }
 
-static void rtc_update_second2(void *opaque)
+static int update_in_progress(RTCState *s)
 {
-    RTCState *s = opaque;
+    int64_t guest_nsec;
 
-    if (!(s->cmos_data[RTC_REG_B] & REG_B_SET)) {
-        rtc_copy_date(s);
+    if (!rtc_running(s)) {
+        return 0;
     }
-
-    /* check alarm */
-    if (((s->cmos_data[RTC_SECONDS_ALARM] & 0xc0) == 0xc0 ||
-         rtc_from_bcd(s, s->cmos_data[RTC_SECONDS_ALARM]) == s->current_tm.tm_sec) &&
-        ((s->cmos_data[RTC_MINUTES_ALARM] & 0xc0) == 0xc0 ||
-         rtc_from_bcd(s, s->cmos_data[RTC_MINUTES_ALARM]) == s->current_tm.tm_min) &&
-        ((s->cmos_data[RTC_HOURS_ALARM] & 0xc0) == 0xc0 ||
-         rtc_from_bcd(s, s->cmos_data[RTC_HOURS_ALARM]) == s->current_tm.tm_hour)) {
-
-        s->cmos_data[RTC_REG_C] |= REG_C_AF;
-        if (s->cmos_data[RTC_REG_B] & REG_B_AIE) {
-            qemu_system_wakeup_request(QEMU_WAKEUP_REASON_RTC);
-            qemu_irq_raise(s->irq);
-            s->cmos_data[RTC_REG_C] |= REG_C_IRQF;
+    if (qemu_timer_pending(s->update_timer)) {
+        int64_t next_update_time = qemu_timer_expire_time_ns(s->update_timer);
+        /* Latch UIP until the timer expires.  */
+        if (qemu_get_clock_ns(rtc_clock) >= (next_update_time - UIP_HOLD_LENGTH)) {
+            s->cmos_data[RTC_REG_A] |= REG_A_UIP;
+            return 1;
         }
     }
 
-    /* update ended interrupt */
-    s->cmos_data[RTC_REG_C] |= REG_C_UF;
-    if (s->cmos_data[RTC_REG_B] & REG_B_UIE) {
-        s->cmos_data[RTC_REG_C] |= REG_C_IRQF;
-        qemu_irq_raise(s->irq);
+    guest_nsec = get_guest_rtc_ns(s);
+    /* UIP bit will be set at last 244us of every second. */
+    if ((guest_nsec % NSEC_PER_SEC) >= (NSEC_PER_SEC - UIP_HOLD_LENGTH)) {
+        return 1;
     }
-
-    /* clear update in progress bit */
-    s->cmos_data[RTC_REG_A] &= ~REG_A_UIP;
-
-    s->next_second_time += get_ticks_per_sec();
-    qemu_mod_timer(s->second_timer, s->next_second_time);
+    return 0;
 }
 
-static uint32_t cmos_ioport_read(void *opaque, uint32_t addr)
+static uint64_t cmos_ioport_read(void *opaque, hwaddr addr,
+                                 unsigned size)
 {
     RTCState *s = opaque;
     int ret;
@@ -433,6 +610,10 @@ static uint32_t cmos_ioport_read(void *opaque, uint32_t addr)
         return 0xff;
     } else {
         switch(s->cmos_index) {
+	case RTC_IBM_PS2_CENTURY_BYTE:
+            s->cmos_index = RTC_CENTURY;
+            /* fall through */
+        case RTC_CENTURY:
         case RTC_SECONDS:
         case RTC_MINUTES:
         case RTC_HOURS:
@@ -440,15 +621,28 @@ static uint32_t cmos_ioport_read(void *opaque, uint32_t addr)
         case RTC_DAY_OF_MONTH:
         case RTC_MONTH:
         case RTC_YEAR:
+            /* if not in set mode, calibrate cmos before
+             * reading*/
+            if (rtc_running(s)) {
+                rtc_update_time(s);
+            }
             ret = s->cmos_data[s->cmos_index];
             break;
         case RTC_REG_A:
+            if (update_in_progress(s)) {
+                s->cmos_data[s->cmos_index] |= REG_A_UIP;
+            } else {
+                s->cmos_data[s->cmos_index] &= ~REG_A_UIP;
+            }
             ret = s->cmos_data[s->cmos_index];
             break;
         case RTC_REG_C:
             ret = s->cmos_data[s->cmos_index];
             qemu_irq_lower(s->irq);
             s->cmos_data[RTC_REG_C] = 0x00;
+            if (ret & (REG_C_UF | REG_C_AF)) {
+                check_update_timer(s);
+            }
 #ifdef TARGET_I386
             if(s->irq_coalesced &&
                     (s->cmos_data[RTC_REG_B] & REG_B_PIE) &&
@@ -483,37 +677,32 @@ void rtc_set_memory(ISADevice *dev, int addr, int val)
         s->cmos_data[addr] = val;
 }
 
-void rtc_set_date(ISADevice *dev, const struct tm *tm)
-{
-    RTCState *s = DO_UPCAST(RTCState, dev, dev);
-    s->current_tm = *tm;
-    rtc_copy_date(s);
-}
-
-/* PC cmos mappings */
-#define REG_IBM_CENTURY_BYTE        0x32
-#define REG_IBM_PS2_CENTURY_BYTE    0x37
-
 static void rtc_set_date_from_host(ISADevice *dev)
 {
     RTCState *s = DO_UPCAST(RTCState, dev, dev);
     struct tm tm;
-    int val;
 
-    /* set the CMOS date */
     qemu_get_timedate(&tm, 0);
-    rtc_set_date(dev, &tm);
 
-    val = rtc_to_bcd(s, (tm.tm_year / 100) + 19);
-    rtc_set_memory(dev, REG_IBM_CENTURY_BYTE, val);
-    rtc_set_memory(dev, REG_IBM_PS2_CENTURY_BYTE, val);
+    s->base_rtc = mktimegm(&tm);
+    s->last_update = qemu_get_clock_ns(rtc_clock);
+    s->offset = 0;
+
+    /* set the CMOS date */
+    rtc_set_cmos(s, &tm);
 }
 
 static int rtc_post_load(void *opaque, int version_id)
 {
-#ifdef TARGET_I386
     RTCState *s = opaque;
 
+    if (version_id <= 2) {
+        rtc_set_time(s);
+        s->offset = 0;
+        check_update_timer(s);
+    }
+
+#ifdef TARGET_I386
     if (version_id >= 2) {
         if (s->lost_tick_policy == LOST_TICK_SLEW) {
             rtc_coalesced_timer_update(s);
@@ -525,27 +714,24 @@ static int rtc_post_load(void *opaque, int version_id)
 
 static const VMStateDescription vmstate_rtc = {
     .name = "mc146818rtc",
-    .version_id = 2,
+    .version_id = 3,
     .minimum_version_id = 1,
     .minimum_version_id_old = 1,
     .post_load = rtc_post_load,
     .fields      = (VMStateField []) {
         VMSTATE_BUFFER(cmos_data, RTCState),
         VMSTATE_UINT8(cmos_index, RTCState),
-        VMSTATE_INT32(current_tm.tm_sec, RTCState),
-        VMSTATE_INT32(current_tm.tm_min, RTCState),
-        VMSTATE_INT32(current_tm.tm_hour, RTCState),
-        VMSTATE_INT32(current_tm.tm_wday, RTCState),
-        VMSTATE_INT32(current_tm.tm_mday, RTCState),
-        VMSTATE_INT32(current_tm.tm_mon, RTCState),
-        VMSTATE_INT32(current_tm.tm_year, RTCState),
+        VMSTATE_UNUSED(7*4),
         VMSTATE_TIMER(periodic_timer, RTCState),
         VMSTATE_INT64(next_periodic_time, RTCState),
-        VMSTATE_INT64(next_second_time, RTCState),
-        VMSTATE_TIMER(second_timer, RTCState),
-        VMSTATE_TIMER(second_timer2, RTCState),
+        VMSTATE_UNUSED(3*8),
         VMSTATE_UINT32_V(irq_coalesced, RTCState, 2),
         VMSTATE_UINT32_V(period, RTCState, 2),
+        VMSTATE_UINT64_V(base_rtc, RTCState, 3),
+        VMSTATE_UINT64_V(last_update, RTCState, 3),
+        VMSTATE_INT64_V(offset, RTCState, 3),
+        VMSTATE_TIMER_V(update_timer, RTCState, 3),
+        VMSTATE_UINT64_V(next_alarm_time, RTCState, 3),
         VMSTATE_END_OF_LIST()
     }
 };
@@ -556,9 +742,8 @@ static void rtc_notify_clock_reset(Notifier *notifier, void *data)
     int64_t now = *(int64_t *)data;
 
     rtc_set_date_from_host(&s->dev);
-    s->next_second_time = now + (get_ticks_per_sec() * 99) / 100;
-    qemu_mod_timer(s->second_timer2, s->next_second_time);
-    rtc_timer_update(s, now);
+    periodic_timer_update(s, now);
+    check_update_timer(s);
 #ifdef TARGET_I386
     if (s->lost_tick_policy == LOST_TICK_SLEW) {
         rtc_coalesced_timer_update(s);
@@ -580,6 +765,7 @@ static void rtc_reset(void *opaque)
 
     s->cmos_data[RTC_REG_B] &= ~(REG_B_PIE | REG_B_AIE | REG_B_SQWE);
     s->cmos_data[RTC_REG_C] &= ~(REG_C_UF | REG_C_IRQF | REG_C_PF | REG_C_AF);
+    check_update_timer(s);
 
     qemu_irq_lower(s->irq);
 
@@ -590,13 +776,14 @@ static void rtc_reset(void *opaque)
 #endif
 }
 
-static const MemoryRegionPortio cmos_portio[] = {
-    {0, 2, 1, .read = cmos_ioport_read, .write = cmos_ioport_write },
-    PORTIO_END_OF_LIST(),
-};
-
 static const MemoryRegionOps cmos_ops = {
-    .old_portio = cmos_portio
+    .read = cmos_ioport_read,
+    .write = cmos_ioport_write,
+    .impl = {
+        .min_access_size = 1,
+        .max_access_size = 1,
+    },
+    .endianness = DEVICE_LITTLE_ENDIAN,
 };
 
 static void rtc_get_date(Object *obj, Visitor *v, void *opaque,
@@ -604,14 +791,17 @@ static void rtc_get_date(Object *obj, Visitor *v, void *opaque,
 {
     ISADevice *isa = ISA_DEVICE(obj);
     RTCState *s = DO_UPCAST(RTCState, dev, isa);
+    struct tm current_tm;
 
+    rtc_update_time(s);
+    rtc_get_time(s, &current_tm);
     visit_start_struct(v, NULL, "struct tm", name, 0, errp);
-    visit_type_int32(v, &s->current_tm.tm_year, "tm_year", errp);
-    visit_type_int32(v, &s->current_tm.tm_mon, "tm_mon", errp);
-    visit_type_int32(v, &s->current_tm.tm_mday, "tm_mday", errp);
-    visit_type_int32(v, &s->current_tm.tm_hour, "tm_hour", errp);
-    visit_type_int32(v, &s->current_tm.tm_min, "tm_min", errp);
-    visit_type_int32(v, &s->current_tm.tm_sec, "tm_sec", errp);
+    visit_type_int32(v, &current_tm.tm_year, "tm_year", errp);
+    visit_type_int32(v, &current_tm.tm_mon, "tm_mon", errp);
+    visit_type_int32(v, &current_tm.tm_mday, "tm_mday", errp);
+    visit_type_int32(v, &current_tm.tm_hour, "tm_hour", errp);
+    visit_type_int32(v, &current_tm.tm_min, "tm_min", errp);
+    visit_type_int32(v, &current_tm.tm_sec, "tm_sec", errp);
     visit_end_struct(v, errp);
 }
 
@@ -625,6 +815,18 @@ static int rtc_initfn(ISADevice *dev)
     s->cmos_data[RTC_REG_C] = 0x00;
     s->cmos_data[RTC_REG_D] = 0x80;
 
+    /* This is for historical reasons.  The default base year qdev property
+     * was set to 2000 for most machine types before the century byte was
+     * implemented.
+     *
+     * This if statement means that the century byte will be always 0
+     * (at least until 2079...) for base_year = 1980, but will be set
+     * correctly for base_year = 2000.
+     */
+    if (s->base_year == 2000) {
+        s->base_year = 0;
+    }
+
     rtc_set_date_from_host(dev);
 
 #ifdef TARGET_I386
@@ -641,8 +843,8 @@ static int rtc_initfn(ISADevice *dev)
 #endif
 
     s->periodic_timer = qemu_new_timer_ns(rtc_clock, rtc_periodic_timer, s);
-    s->second_timer = qemu_new_timer_ns(rtc_clock, rtc_update_second, s);
-    s->second_timer2 = qemu_new_timer_ns(rtc_clock, rtc_update_second2, s);
+    s->update_timer = qemu_new_timer_ns(rtc_clock, rtc_update_timer, s);
+    check_update_timer(s);
 
     s->clock_reset_notifier.notify = rtc_notify_clock_reset;
     qemu_register_clock_reset_notifier(rtc_clock, &s->clock_reset_notifier);
@@ -650,14 +852,10 @@ static int rtc_initfn(ISADevice *dev)
     s->suspend_notifier.notify = rtc_notify_suspend;
     qemu_register_suspend_notifier(&s->suspend_notifier);
 
-    s->next_second_time =
-        qemu_get_clock_ns(rtc_clock) + (get_ticks_per_sec() * 99) / 100;
-    qemu_mod_timer(s->second_timer2, s->next_second_time);
-
     memory_region_init_io(&s->io, &cmos_ops, s, "rtc", 2);
     isa_register_ioport(dev, &s->io, base);
 
-    qdev_set_legacy_instance_id(&dev->qdev, base, 2);
+    qdev_set_legacy_instance_id(&dev->qdev, base, 3);
     qemu_register_reset(rtc_reset, s);
 
     object_property_add(OBJECT(s), "date", "struct tm",
diff --git a/hw/mc146818rtc_regs.h b/hw/mc146818rtc_regs.h
index 3ab37709f..ccdee42b3 100644
--- a/hw/mc146818rtc_regs.h
+++ b/hw/mc146818rtc_regs.h
@@ -44,6 +44,10 @@
 #define RTC_REG_C               12
 #define RTC_REG_D               13
 
+/* PC cmos mappings */
+#define RTC_CENTURY              0x32
+#define RTC_IBM_PS2_CENTURY_BYTE 0x37
+
 #define REG_A_UIP 0x80
 
 #define REG_B_SET  0x80
@@ -58,5 +62,6 @@
 #define REG_C_IRQF 0x80
 #define REG_C_PF   0x40
 #define REG_C_AF   0x20
+#define REG_C_MASK 0x70
 
 #endif
diff --git a/hw/mcf.h b/hw/mcf.h
index 19a8b5477..f929910f0 100644
--- a/hw/mcf.h
+++ b/hw/mcf.h
@@ -5,23 +5,23 @@
 struct MemoryRegion;
 
 /* mcf_uart.c */
-uint64_t mcf_uart_read(void *opaque, target_phys_addr_t addr,
+uint64_t mcf_uart_read(void *opaque, hwaddr addr,
                        unsigned size);
-void mcf_uart_write(void *opaque, target_phys_addr_t addr,
+void mcf_uart_write(void *opaque, hwaddr addr,
                     uint64_t val, unsigned size);
 void *mcf_uart_init(qemu_irq irq, CharDriverState *chr);
 void mcf_uart_mm_init(struct MemoryRegion *sysmem,
-                      target_phys_addr_t base,
+                      hwaddr base,
                       qemu_irq irq, CharDriverState *chr);
 
 /* mcf_intc.c */
 qemu_irq *mcf_intc_init(struct MemoryRegion *sysmem,
-                        target_phys_addr_t base,
+                        hwaddr base,
                         CPUM68KState *env);
 
 /* mcf_fec.c */
 void mcf_fec_init(struct MemoryRegion *sysmem, NICInfo *nd,
-                  target_phys_addr_t base, qemu_irq *irq);
+                  hwaddr base, qemu_irq *irq);
 
 /* mcf5206.c */
 qemu_irq *mcf5206_init(struct MemoryRegion *sysmem,
diff --git a/hw/mcf5206.c b/hw/mcf5206.c
index 539b39133..510d77047 100644
--- a/hw/mcf5206.c
+++ b/hw/mcf5206.c
@@ -371,14 +371,14 @@ static const int m5206_mbar_width[] =
   /* 1c0-200 */ 1, 1, 1, 1,  1, 1, 1, 1,  1, 1, 1, 1,  1, 1, 1, 1,
 };
 
-static uint32_t m5206_mbar_readw(void *opaque, target_phys_addr_t offset);
-static uint32_t m5206_mbar_readl(void *opaque, target_phys_addr_t offset);
+static uint32_t m5206_mbar_readw(void *opaque, hwaddr offset);
+static uint32_t m5206_mbar_readl(void *opaque, hwaddr offset);
 
-static uint32_t m5206_mbar_readb(void *opaque, target_phys_addr_t offset)
+static uint32_t m5206_mbar_readb(void *opaque, hwaddr offset)
 {
     m5206_mbar_state *s = (m5206_mbar_state *)opaque;
     offset &= 0x3ff;
-    if (offset > 0x200) {
+    if (offset >= 0x200) {
         hw_error("Bad MBAR read offset 0x%x", (int)offset);
     }
     if (m5206_mbar_width[offset >> 2] > 1) {
@@ -392,12 +392,12 @@ static uint32_t m5206_mbar_readb(void *opaque, target_phys_addr_t offset)
     return m5206_mbar_read(s, offset, 1);
 }
 
-static uint32_t m5206_mbar_readw(void *opaque, target_phys_addr_t offset)
+static uint32_t m5206_mbar_readw(void *opaque, hwaddr offset)
 {
     m5206_mbar_state *s = (m5206_mbar_state *)opaque;
     int width;
     offset &= 0x3ff;
-    if (offset > 0x200) {
+    if (offset >= 0x200) {
         hw_error("Bad MBAR read offset 0x%x", (int)offset);
     }
     width = m5206_mbar_width[offset >> 2];
@@ -416,12 +416,12 @@ static uint32_t m5206_mbar_readw(void *opaque, target_phys_addr_t offset)
     return m5206_mbar_read(s, offset, 2);
 }
 
-static uint32_t m5206_mbar_readl(void *opaque, target_phys_addr_t offset)
+static uint32_t m5206_mbar_readl(void *opaque, hwaddr offset)
 {
     m5206_mbar_state *s = (m5206_mbar_state *)opaque;
     int width;
     offset &= 0x3ff;
-    if (offset > 0x200) {
+    if (offset >= 0x200) {
         hw_error("Bad MBAR read offset 0x%x", (int)offset);
     }
     width = m5206_mbar_width[offset >> 2];
@@ -434,18 +434,18 @@ static uint32_t m5206_mbar_readl(void *opaque, target_phys_addr_t offset)
     return m5206_mbar_read(s, offset, 4);
 }
 
-static void m5206_mbar_writew(void *opaque, target_phys_addr_t offset,
+static void m5206_mbar_writew(void *opaque, hwaddr offset,
                               uint32_t value);
-static void m5206_mbar_writel(void *opaque, target_phys_addr_t offset,
+static void m5206_mbar_writel(void *opaque, hwaddr offset,
                               uint32_t value);
 
-static void m5206_mbar_writeb(void *opaque, target_phys_addr_t offset,
+static void m5206_mbar_writeb(void *opaque, hwaddr offset,
                               uint32_t value)
 {
     m5206_mbar_state *s = (m5206_mbar_state *)opaque;
     int width;
     offset &= 0x3ff;
-    if (offset > 0x200) {
+    if (offset >= 0x200) {
         hw_error("Bad MBAR write offset 0x%x", (int)offset);
     }
     width = m5206_mbar_width[offset >> 2];
@@ -463,13 +463,13 @@ static void m5206_mbar_writeb(void *opaque, target_phys_addr_t offset,
     m5206_mbar_write(s, offset, value, 1);
 }
 
-static void m5206_mbar_writew(void *opaque, target_phys_addr_t offset,
+static void m5206_mbar_writew(void *opaque, hwaddr offset,
                               uint32_t value)
 {
     m5206_mbar_state *s = (m5206_mbar_state *)opaque;
     int width;
     offset &= 0x3ff;
-    if (offset > 0x200) {
+    if (offset >= 0x200) {
         hw_error("Bad MBAR write offset 0x%x", (int)offset);
     }
     width = m5206_mbar_width[offset >> 2];
@@ -491,13 +491,13 @@ static void m5206_mbar_writew(void *opaque, target_phys_addr_t offset,
     m5206_mbar_write(s, offset, value, 2);
 }
 
-static void m5206_mbar_writel(void *opaque, target_phys_addr_t offset,
+static void m5206_mbar_writel(void *opaque, hwaddr offset,
                               uint32_t value)
 {
     m5206_mbar_state *s = (m5206_mbar_state *)opaque;
     int width;
     offset &= 0x3ff;
-    if (offset > 0x200) {
+    if (offset >= 0x200) {
         hw_error("Bad MBAR write offset 0x%x", (int)offset);
     }
     width = m5206_mbar_width[offset >> 2];
diff --git a/hw/mcf5208.c b/hw/mcf5208.c
index ee25b1b38..b1db54937 100644
--- a/hw/mcf5208.c
+++ b/hw/mcf5208.c
@@ -45,7 +45,7 @@ static void m5208_timer_update(m5208_timer_state *s)
         qemu_irq_lower(s->irq);
 }
 
-static void m5208_timer_write(void *opaque, target_phys_addr_t offset,
+static void m5208_timer_write(void *opaque, hwaddr offset,
                               uint64_t value, unsigned size)
 {
     m5208_timer_state *s = (m5208_timer_state *)opaque;
@@ -107,7 +107,7 @@ static void m5208_timer_trigger(void *opaque)
     m5208_timer_update(s);
 }
 
-static uint64_t m5208_timer_read(void *opaque, target_phys_addr_t addr,
+static uint64_t m5208_timer_read(void *opaque, hwaddr addr,
                                  unsigned size)
 {
     m5208_timer_state *s = (m5208_timer_state *)opaque;
@@ -130,7 +130,7 @@ static const MemoryRegionOps m5208_timer_ops = {
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-static uint64_t m5208_sys_read(void *opaque, target_phys_addr_t addr,
+static uint64_t m5208_sys_read(void *opaque, hwaddr addr,
                                unsigned size)
 {
     switch (addr) {
@@ -152,7 +152,7 @@ static uint64_t m5208_sys_read(void *opaque, target_phys_addr_t addr,
     }
 }
 
-static void m5208_sys_write(void *opaque, target_phys_addr_t addr,
+static void m5208_sys_write(void *opaque, hwaddr addr,
                             uint64_t value, unsigned size)
 {
     hw_error("m5208_sys_write: Bad offset 0x%x\n", (int)addr);
@@ -187,15 +187,15 @@ static void mcf5208_sys_init(MemoryRegion *address_space, qemu_irq *pic)
     }
 }
 
-static void mcf5208evb_init(ram_addr_t ram_size,
-                     const char *boot_device,
-                     const char *kernel_filename, const char *kernel_cmdline,
-                     const char *initrd_filename, const char *cpu_model)
+static void mcf5208evb_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t ram_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
     CPUM68KState *env;
     int kernel_size;
     uint64_t elf_entry;
-    target_phys_addr_t entry;
+    hwaddr entry;
     qemu_irq *pic;
     MemoryRegion *address_space_mem = get_system_memory();
     MemoryRegion *ram = g_new(MemoryRegion, 1);
diff --git a/hw/mcf_fec.c b/hw/mcf_fec.c
index 2fec5bc73..1ed193c5d 100644
--- a/hw/mcf_fec.c
+++ b/hw/mcf_fec.c
@@ -216,7 +216,7 @@ static void mcf_fec_reset(mcf_fec_state *s)
     s->rfsr = 0x500;
 }
 
-static uint64_t mcf_fec_read(void *opaque, target_phys_addr_t addr,
+static uint64_t mcf_fec_read(void *opaque, hwaddr addr,
                              unsigned size)
 {
     mcf_fec_state *s = (mcf_fec_state *)opaque;
@@ -254,7 +254,7 @@ static uint64_t mcf_fec_read(void *opaque, target_phys_addr_t addr,
     }
 }
 
-static void mcf_fec_write(void *opaque, target_phys_addr_t addr,
+static void mcf_fec_write(void *opaque, hwaddr addr,
                           uint64_t value, unsigned size)
 {
     mcf_fec_state *s = (mcf_fec_state *)opaque;
@@ -458,7 +458,7 @@ static NetClientInfo net_mcf_fec_info = {
 };
 
 void mcf_fec_init(MemoryRegion *sysmem, NICInfo *nd,
-                  target_phys_addr_t base, qemu_irq *irq)
+                  hwaddr base, qemu_irq *irq)
 {
     mcf_fec_state *s;
 
diff --git a/hw/mcf_intc.c b/hw/mcf_intc.c
index cc1a5f376..6ef6dac93 100644
--- a/hw/mcf_intc.c
+++ b/hw/mcf_intc.c
@@ -43,7 +43,7 @@ static void mcf_intc_update(mcf_intc_state *s)
     m68k_set_irq_level(s->env, best_level, s->active_vector);
 }
 
-static uint64_t mcf_intc_read(void *opaque, target_phys_addr_t addr,
+static uint64_t mcf_intc_read(void *opaque, hwaddr addr,
                               unsigned size)
 {
     int offset;
@@ -76,7 +76,7 @@ static uint64_t mcf_intc_read(void *opaque, target_phys_addr_t addr,
     }
 }
 
-static void mcf_intc_write(void *opaque, target_phys_addr_t addr,
+static void mcf_intc_write(void *opaque, hwaddr addr,
                            uint64_t val, unsigned size)
 {
     int offset;
@@ -138,7 +138,7 @@ static const MemoryRegionOps mcf_intc_ops = {
 };
 
 qemu_irq *mcf_intc_init(MemoryRegion *sysmem,
-                        target_phys_addr_t base,
+                        hwaddr base,
                         CPUM68KState *env)
 {
     mcf_intc_state *s;
diff --git a/hw/mcf_uart.c b/hw/mcf_uart.c
index ec6a87f23..d1655f8f2 100644
--- a/hw/mcf_uart.c
+++ b/hw/mcf_uart.c
@@ -66,7 +66,7 @@ static void mcf_uart_update(mcf_uart_state *s)
     qemu_set_irq(s->irq, (s->isr & s->imr) != 0);
 }
 
-uint64_t mcf_uart_read(void *opaque, target_phys_addr_t addr,
+uint64_t mcf_uart_read(void *opaque, hwaddr addr,
                        unsigned size)
 {
     mcf_uart_state *s = (mcf_uart_state *)opaque;
@@ -185,7 +185,7 @@ static void mcf_do_command(mcf_uart_state *s, uint8_t cmd)
     }
 }
 
-void mcf_uart_write(void *opaque, target_phys_addr_t addr,
+void mcf_uart_write(void *opaque, hwaddr addr,
                     uint64_t val, unsigned size)
 {
     mcf_uart_state *s = (mcf_uart_state *)opaque;
@@ -294,7 +294,7 @@ static const MemoryRegionOps mcf_uart_ops = {
 };
 
 void mcf_uart_mm_init(MemoryRegion *sysmem,
-                      target_phys_addr_t base,
+                      hwaddr base,
                       qemu_irq irq,
                       CharDriverState *chr)
 {
diff --git a/hw/megasas.c b/hw/megasas.c
index c728aea69..61b652792 100644
--- a/hw/megasas.c
+++ b/hw/megasas.c
@@ -25,7 +25,6 @@
 #include "iov.h"
 #include "scsi.h"
 #include "scsi-defs.h"
-#include "block_int.h"
 #include "trace.h"
 
 #include "mfi.h"
@@ -59,8 +58,8 @@ typedef struct MegasasCmd {
     uint16_t count;
     uint64_t context;
 
-    target_phys_addr_t pa;
-    target_phys_addr_t pa_size;
+    hwaddr pa;
+    hwaddr pa_size;
     union mfi_frame *frame;
     SCSIRequest *req;
     QEMUSGList qsg;
@@ -277,7 +276,7 @@ static int megasas_build_sense(MegasasCmd *cmd, uint8_t *sense_ptr,
     uint8_t sense_len)
 {
     uint32_t pa_hi = 0, pa_lo;
-    target_phys_addr_t pa;
+    hwaddr pa;
 
     if (sense_len > cmd->frame->header.sense_len) {
         sense_len = cmd->frame->header.sense_len;
@@ -404,7 +403,7 @@ static int megasas_next_index(MegasasState *s, int index, int limit)
 }
 
 static MegasasCmd *megasas_lookup_frame(MegasasState *s,
-    target_phys_addr_t frame)
+    hwaddr frame)
 {
     MegasasCmd *cmd = NULL;
     int num = 0, index;
@@ -424,7 +423,7 @@ static MegasasCmd *megasas_lookup_frame(MegasasState *s,
 }
 
 static MegasasCmd *megasas_next_frame(MegasasState *s,
-    target_phys_addr_t frame)
+    hwaddr frame)
 {
     MegasasCmd *cmd = NULL;
     int num = 0, index;
@@ -452,11 +451,11 @@ static MegasasCmd *megasas_next_frame(MegasasState *s,
 }
 
 static MegasasCmd *megasas_enqueue_frame(MegasasState *s,
-    target_phys_addr_t frame, uint64_t context, int count)
+    hwaddr frame, uint64_t context, int count)
 {
     MegasasCmd *cmd = NULL;
     int frame_size = MFI_FRAME_SIZE * 16;
-    target_phys_addr_t frame_size_p = frame_size;
+    hwaddr frame_size_p = frame_size;
 
     cmd = megasas_next_frame(s, frame);
     /* All frames busy */
@@ -561,7 +560,7 @@ static void megasas_abort_command(MegasasCmd *cmd)
 static int megasas_init_firmware(MegasasState *s, MegasasCmd *cmd)
 {
     uint32_t pa_hi, pa_lo;
-    target_phys_addr_t iq_pa, initq_size;
+    hwaddr iq_pa, initq_size;
     struct mfi_init_qinfo *initq;
     uint32_t flags;
     int ret = MFI_STAT_OK;
@@ -652,7 +651,6 @@ static void megasas_finish_dcmd(MegasasCmd *cmd, uint32_t iov_size)
         }
     }
     cmd->iov_size = 0;
-    return;
 }
 
 static int megasas_ctrl_get_info(MegasasState *s, MegasasCmd *cmd)
@@ -1081,6 +1079,7 @@ static int megasas_dcmd_ld_get_list(MegasasState *s, MegasasCmd *cmd)
         /* Logical device size is in blocks */
         bdrv_get_geometry(conf->bs, &ld_size);
         info.ld_list[num_ld_disks].ld.v.target_id = sdev->id;
+        info.ld_list[num_ld_disks].ld.v.lun_id = sdev->lun;
         info.ld_list[num_ld_disks].state = MFI_LD_STATE_OPTIMAL;
         info.ld_list[num_ld_disks].size = cpu_to_le64(ld_size);
         num_ld_disks++;
@@ -1297,7 +1296,7 @@ static int megasas_dcmd_get_properties(MegasasState *s, MegasasCmd *cmd)
 
 static int megasas_cache_flush(MegasasState *s, MegasasCmd *cmd)
 {
-    qemu_aio_flush();
+    bdrv_drain_all();
     return MFI_STAT_OK;
 }
 
@@ -1772,7 +1771,7 @@ static void megasas_command_cancel(SCSIRequest *req)
 static int megasas_handle_abort(MegasasState *s, MegasasCmd *cmd)
 {
     uint64_t abort_ctx = le64_to_cpu(cmd->frame->abort.abort_context);
-    target_phys_addr_t abort_addr, addr_hi, addr_lo;
+    hwaddr abort_addr, addr_hi, addr_lo;
     MegasasCmd *abort_cmd;
 
     addr_hi = le32_to_cpu(cmd->frame->abort.abort_mfi_addr_hi);
@@ -1862,7 +1861,7 @@ static void megasas_handle_frame(MegasasState *s, uint64_t frame_addr,
     }
 }
 
-static uint64_t megasas_mmio_read(void *opaque, target_phys_addr_t addr,
+static uint64_t megasas_mmio_read(void *opaque, hwaddr addr,
                                   unsigned size)
 {
     MegasasState *s = opaque;
@@ -1898,7 +1897,7 @@ static uint64_t megasas_mmio_read(void *opaque, target_phys_addr_t addr,
     return retval;
 }
 
-static void megasas_mmio_write(void *opaque, target_phys_addr_t addr,
+static void megasas_mmio_write(void *opaque, hwaddr addr,
                                uint64_t val, unsigned size)
 {
     MegasasState *s = opaque;
@@ -1978,13 +1977,13 @@ static const MemoryRegionOps megasas_mmio_ops = {
     }
 };
 
-static uint64_t megasas_port_read(void *opaque, target_phys_addr_t addr,
+static uint64_t megasas_port_read(void *opaque, hwaddr addr,
                                   unsigned size)
 {
     return megasas_mmio_read(opaque, addr & 0xff, size);
 }
 
-static void megasas_port_write(void *opaque, target_phys_addr_t addr,
+static void megasas_port_write(void *opaque, hwaddr addr,
                                uint64_t val, unsigned size)
 {
     megasas_mmio_write(opaque, addr & 0xff, val, size);
@@ -2000,7 +1999,7 @@ static const MemoryRegionOps megasas_port_ops = {
     }
 };
 
-static uint64_t megasas_queue_read(void *opaque, target_phys_addr_t addr,
+static uint64_t megasas_queue_read(void *opaque, hwaddr addr,
                                    unsigned size)
 {
     return 0;
diff --git a/hw/mfi.h b/hw/mfi.h
index 436b6906b..cd8355bad 100644
--- a/hw/mfi.h
+++ b/hw/mfi.h
@@ -1085,7 +1085,7 @@ struct mfi_pd_list {
 union mfi_ld_ref {
     struct {
         uint8_t target_id;
-        uint8_t reserved;
+        uint8_t lun_id;
         uint16_t seq;
     } v;
     uint32_t ref;
diff --git a/hw/microblaze/Makefile.objs b/hw/microblaze/Makefile.objs
index 274d2c543..3028e651c 100644
--- a/hw/microblaze/Makefile.objs
+++ b/hw/microblaze/Makefile.objs
@@ -1,6 +1,7 @@
 obj-y = petalogix_s3adsp1800_mmu.o
 obj-y += petalogix_ml605_mmu.o
 obj-y += microblaze_boot.o
+obj-y += xilinx_spi.o
 
 obj-y += microblaze_pic_cpu.o
 obj-y += xilinx_ethlite.o
diff --git a/hw/microblaze_boot.c b/hw/microblaze_boot.c
index 1030e9c8e..02c349c18 100644
--- a/hw/microblaze_boot.c
+++ b/hw/microblaze_boot.c
@@ -55,7 +55,7 @@ static void main_cpu_reset(void *opaque)
     }
 }
 
-static int microblaze_load_dtb(target_phys_addr_t addr,
+static int microblaze_load_dtb(hwaddr addr,
                                       uint32_t ramsize,
                                       const char *kernel_cmdline,
                                       const char *dtb_filename)
@@ -100,7 +100,7 @@ static uint64_t translate_kernel_address(void *opaque, uint64_t addr)
     return addr - 0x30000000LL;
 }
 
-void microblaze_load_kernel(MicroBlazeCPU *cpu, target_phys_addr_t ddr_base,
+void microblaze_load_kernel(MicroBlazeCPU *cpu, hwaddr ddr_base,
                             uint32_t ramsize, const char *dtb_filename,
                             void (*machine_cpu_reset)(MicroBlazeCPU *))
 {
@@ -149,7 +149,7 @@ void microblaze_load_kernel(MicroBlazeCPU *cpu, target_phys_addr_t ddr_base,
 
         /* If it wasn't an ELF image, try an u-boot image.  */
         if (kernel_size < 0) {
-            target_phys_addr_t uentry, loadaddr;
+            hwaddr uentry, loadaddr;
 
             kernel_size = load_uimage(kernel_filename, &uentry, &loadaddr, 0);
             boot_info.bootstrap_pc = uentry;
diff --git a/hw/microblaze_boot.h b/hw/microblaze_boot.h
index c9a3064d2..c1cf836b9 100644
--- a/hw/microblaze_boot.h
+++ b/hw/microblaze_boot.h
@@ -3,7 +3,7 @@
 
 #include "hw.h"
 
-void microblaze_load_kernel(MicroBlazeCPU *cpu, target_phys_addr_t ddr_base,
+void microblaze_load_kernel(MicroBlazeCPU *cpu, hwaddr ddr_base,
                             uint32_t ramsize, const char *dtb_filename,
                             void (*machine_cpu_reset)(MicroBlazeCPU *));
 
diff --git a/hw/milkymist-ac97.c b/hw/milkymist-ac97.c
index 4414f3973..d87656c9a 100644
--- a/hw/milkymist-ac97.c
+++ b/hw/milkymist-ac97.c
@@ -83,7 +83,7 @@ static void update_voices(MilkymistAC97State *s)
     }
 }
 
-static uint64_t ac97_read(void *opaque, target_phys_addr_t addr,
+static uint64_t ac97_read(void *opaque, hwaddr addr,
                           unsigned size)
 {
     MilkymistAC97State *s = opaque;
@@ -115,7 +115,7 @@ static uint64_t ac97_read(void *opaque, target_phys_addr_t addr,
     return r;
 }
 
-static void ac97_write(void *opaque, target_phys_addr_t addr, uint64_t value,
+static void ac97_write(void *opaque, hwaddr addr, uint64_t value,
                        unsigned size)
 {
     MilkymistAC97State *s = opaque;
diff --git a/hw/milkymist-hpdmc.c b/hw/milkymist-hpdmc.c
index 2da029368..5d120a497 100644
--- a/hw/milkymist-hpdmc.c
+++ b/hw/milkymist-hpdmc.c
@@ -48,7 +48,7 @@ struct MilkymistHpdmcState {
 };
 typedef struct MilkymistHpdmcState MilkymistHpdmcState;
 
-static uint64_t hpdmc_read(void *opaque, target_phys_addr_t addr,
+static uint64_t hpdmc_read(void *opaque, hwaddr addr,
                            unsigned size)
 {
     MilkymistHpdmcState *s = opaque;
@@ -74,7 +74,7 @@ static uint64_t hpdmc_read(void *opaque, target_phys_addr_t addr,
     return r;
 }
 
-static void hpdmc_write(void *opaque, target_phys_addr_t addr, uint64_t value,
+static void hpdmc_write(void *opaque, hwaddr addr, uint64_t value,
                         unsigned size)
 {
     MilkymistHpdmcState *s = opaque;
diff --git a/hw/milkymist-hw.h b/hw/milkymist-hw.h
index 9f358a7d6..96b2a7f86 100644
--- a/hw/milkymist-hw.h
+++ b/hw/milkymist-hw.h
@@ -4,7 +4,7 @@
 #include "qdev.h"
 #include "qdev-addr.h"
 
-static inline DeviceState *milkymist_uart_create(target_phys_addr_t base,
+static inline DeviceState *milkymist_uart_create(hwaddr base,
         qemu_irq irq)
 {
     DeviceState *dev;
@@ -17,7 +17,7 @@ static inline DeviceState *milkymist_uart_create(target_phys_addr_t base,
     return dev;
 }
 
-static inline DeviceState *milkymist_hpdmc_create(target_phys_addr_t base)
+static inline DeviceState *milkymist_hpdmc_create(hwaddr base)
 {
     DeviceState *dev;
 
@@ -28,7 +28,7 @@ static inline DeviceState *milkymist_hpdmc_create(target_phys_addr_t base)
     return dev;
 }
 
-static inline DeviceState *milkymist_memcard_create(target_phys_addr_t base)
+static inline DeviceState *milkymist_memcard_create(hwaddr base)
 {
     DeviceState *dev;
 
@@ -39,7 +39,7 @@ static inline DeviceState *milkymist_memcard_create(target_phys_addr_t base)
     return dev;
 }
 
-static inline DeviceState *milkymist_vgafb_create(target_phys_addr_t base,
+static inline DeviceState *milkymist_vgafb_create(hwaddr base,
         uint32_t fb_offset, uint32_t fb_mask)
 {
     DeviceState *dev;
@@ -53,7 +53,7 @@ static inline DeviceState *milkymist_vgafb_create(target_phys_addr_t base,
     return dev;
 }
 
-static inline DeviceState *milkymist_sysctl_create(target_phys_addr_t base,
+static inline DeviceState *milkymist_sysctl_create(hwaddr base,
         qemu_irq gpio_irq, qemu_irq timer0_irq, qemu_irq timer1_irq,
         uint32_t freq_hz, uint32_t system_id, uint32_t capabilities,
         uint32_t gpio_strappings)
@@ -74,7 +74,7 @@ static inline DeviceState *milkymist_sysctl_create(target_phys_addr_t base,
     return dev;
 }
 
-static inline DeviceState *milkymist_pfpu_create(target_phys_addr_t base,
+static inline DeviceState *milkymist_pfpu_create(hwaddr base,
         qemu_irq irq)
 {
     DeviceState *dev;
@@ -97,7 +97,7 @@ static const int glx_fbconfig_attr[] = {
 };
 #endif
 
-static inline DeviceState *milkymist_tmu2_create(target_phys_addr_t base,
+static inline DeviceState *milkymist_tmu2_create(hwaddr base,
         qemu_irq irq)
 {
 #ifdef CONFIG_OPENGL
@@ -152,7 +152,7 @@ static inline DeviceState *milkymist_tmu2_create(target_phys_addr_t base,
 #endif
 }
 
-static inline DeviceState *milkymist_ac97_create(target_phys_addr_t base,
+static inline DeviceState *milkymist_ac97_create(hwaddr base,
         qemu_irq crrequest_irq, qemu_irq crreply_irq, qemu_irq dmar_irq,
         qemu_irq dmaw_irq)
 {
@@ -169,7 +169,7 @@ static inline DeviceState *milkymist_ac97_create(target_phys_addr_t base,
     return dev;
 }
 
-static inline DeviceState *milkymist_minimac_create(target_phys_addr_t base,
+static inline DeviceState *milkymist_minimac_create(hwaddr base,
         qemu_irq rx_irq, qemu_irq tx_irq)
 {
     DeviceState *dev;
@@ -185,8 +185,8 @@ static inline DeviceState *milkymist_minimac_create(target_phys_addr_t base,
     return dev;
 }
 
-static inline DeviceState *milkymist_minimac2_create(target_phys_addr_t base,
-        target_phys_addr_t buffers_base, qemu_irq rx_irq, qemu_irq tx_irq)
+static inline DeviceState *milkymist_minimac2_create(hwaddr base,
+        hwaddr buffers_base, qemu_irq rx_irq, qemu_irq tx_irq)
 {
     DeviceState *dev;
 
@@ -202,7 +202,7 @@ static inline DeviceState *milkymist_minimac2_create(target_phys_addr_t base,
     return dev;
 }
 
-static inline DeviceState *milkymist_softusb_create(target_phys_addr_t base,
+static inline DeviceState *milkymist_softusb_create(hwaddr base,
         qemu_irq irq, uint32_t pmem_base, uint32_t pmem_size,
         uint32_t dmem_base, uint32_t dmem_size)
 {
diff --git a/hw/milkymist-memcard.c b/hw/milkymist-memcard.c
index 3515c3cd9..ca5df5629 100644
--- a/hw/milkymist-memcard.c
+++ b/hw/milkymist-memcard.c
@@ -117,7 +117,7 @@ static void memcard_sd_command(MilkymistMemcardState *s)
     }
 }
 
-static uint64_t memcard_read(void *opaque, target_phys_addr_t addr,
+static uint64_t memcard_read(void *opaque, hwaddr addr,
                              unsigned size)
 {
     MilkymistMemcardState *s = opaque;
@@ -166,7 +166,7 @@ static uint64_t memcard_read(void *opaque, target_phys_addr_t addr,
     return r;
 }
 
-static void memcard_write(void *opaque, target_phys_addr_t addr, uint64_t value,
+static void memcard_write(void *opaque, hwaddr addr, uint64_t value,
                           unsigned size)
 {
     MilkymistMemcardState *s = opaque;
diff --git a/hw/milkymist-minimac2.c b/hw/milkymist-minimac2.c
index b483a02f2..b204e5f89 100644
--- a/hw/milkymist-minimac2.c
+++ b/hw/milkymist-minimac2.c
@@ -96,7 +96,7 @@ struct MilkymistMinimac2State {
     NICState *nic;
     NICConf conf;
     char *phy_model;
-    target_phys_addr_t buffers_base;
+    hwaddr buffers_base;
     MemoryRegion buffers;
     MemoryRegion regs_region;
 
@@ -323,7 +323,7 @@ static ssize_t minimac2_rx(NetClientState *nc, const uint8_t *buf, size_t size)
 }
 
 static uint64_t
-minimac2_read(void *opaque, target_phys_addr_t addr, unsigned size)
+minimac2_read(void *opaque, hwaddr addr, unsigned size)
 {
     MilkymistMinimac2State *s = opaque;
     uint32_t r = 0;
@@ -352,7 +352,7 @@ minimac2_read(void *opaque, target_phys_addr_t addr, unsigned size)
 }
 
 static void
-minimac2_write(void *opaque, target_phys_addr_t addr, uint64_t value,
+minimac2_write(void *opaque, hwaddr addr, uint64_t value,
                unsigned size)
 {
     MilkymistMinimac2State *s = opaque;
diff --git a/hw/milkymist-pfpu.c b/hw/milkymist-pfpu.c
index 0f9ff4a13..450bab921 100644
--- a/hw/milkymist-pfpu.c
+++ b/hw/milkymist-pfpu.c
@@ -131,7 +131,7 @@ struct MilkymistPFPUState {
 };
 typedef struct MilkymistPFPUState MilkymistPFPUState;
 
-static inline target_phys_addr_t
+static inline hwaddr
 get_dma_address(uint32_t base, uint32_t x, uint32_t y)
 {
     return base + 8 * (128 * y + x);
@@ -225,7 +225,7 @@ static int pfpu_decode_insn(MilkymistPFPUState *s)
     {
         uint32_t a = cpu_to_be32(s->gp_regs[reg_a]);
         uint32_t b = cpu_to_be32(s->gp_regs[reg_b]);
-        target_phys_addr_t dma_ptr =
+        hwaddr dma_ptr =
             get_dma_address(s->regs[R_MESHBASE],
                     s->gp_regs[GPR_X], s->gp_regs[GPR_Y]);
         cpu_physical_memory_write(dma_ptr, (uint8_t *)&a, 4);
@@ -380,7 +380,7 @@ static inline int get_microcode_address(MilkymistPFPUState *s, uint32_t addr)
     return (512 * s->regs[R_CODEPAGE]) + addr - MICROCODE_BEGIN;
 }
 
-static uint64_t pfpu_read(void *opaque, target_phys_addr_t addr,
+static uint64_t pfpu_read(void *opaque, hwaddr addr,
                           unsigned size)
 {
     MilkymistPFPUState *s = opaque;
@@ -420,7 +420,7 @@ static uint64_t pfpu_read(void *opaque, target_phys_addr_t addr,
     return r;
 }
 
-static void pfpu_write(void *opaque, target_phys_addr_t addr, uint64_t value,
+static void pfpu_write(void *opaque, hwaddr addr, uint64_t value,
                        unsigned size)
 {
     MilkymistPFPUState *s = opaque;
diff --git a/hw/milkymist-softusb.c b/hw/milkymist-softusb.c
index ecc2be922..b162b88db 100644
--- a/hw/milkymist-softusb.c
+++ b/hw/milkymist-softusb.c
@@ -71,7 +71,7 @@ struct MilkymistSoftUsbState {
 };
 typedef struct MilkymistSoftUsbState MilkymistSoftUsbState;
 
-static uint64_t softusb_read(void *opaque, target_phys_addr_t addr,
+static uint64_t softusb_read(void *opaque, hwaddr addr,
                              unsigned size)
 {
     MilkymistSoftUsbState *s = opaque;
@@ -95,7 +95,7 @@ static uint64_t softusb_read(void *opaque, target_phys_addr_t addr,
 }
 
 static void
-softusb_write(void *opaque, target_phys_addr_t addr, uint64_t value,
+softusb_write(void *opaque, hwaddr addr, uint64_t value,
               unsigned size)
 {
     MilkymistSoftUsbState *s = opaque;
diff --git a/hw/milkymist-sysctl.c b/hw/milkymist-sysctl.c
index 8878d2bd1..f951ef9ca 100644
--- a/hw/milkymist-sysctl.c
+++ b/hw/milkymist-sysctl.c
@@ -89,7 +89,7 @@ static void sysctl_icap_write(MilkymistSysctlState *s, uint32_t value)
     }
 }
 
-static uint64_t sysctl_read(void *opaque, target_phys_addr_t addr,
+static uint64_t sysctl_read(void *opaque, hwaddr addr,
                             unsigned size)
 {
     MilkymistSysctlState *s = opaque;
@@ -134,7 +134,7 @@ static uint64_t sysctl_read(void *opaque, target_phys_addr_t addr,
     return r;
 }
 
-static void sysctl_write(void *opaque, target_phys_addr_t addr, uint64_t value,
+static void sysctl_write(void *opaque, hwaddr addr, uint64_t value,
                          unsigned size)
 {
     MilkymistSysctlState *s = opaque;
diff --git a/hw/milkymist-tmu2.c b/hw/milkymist-tmu2.c
index 210ceede2..3f9a684ed 100644
--- a/hw/milkymist-tmu2.c
+++ b/hw/milkymist-tmu2.c
@@ -182,9 +182,9 @@ static void tmu2_start(MilkymistTMU2State *s)
     GLXPbuffer pbuffer;
     GLuint texture;
     void *fb;
-    target_phys_addr_t fb_len;
+    hwaddr fb_len;
     void *mesh;
-    target_phys_addr_t mesh_len;
+    hwaddr mesh_len;
     float m;
 
     trace_milkymist_tmu2_start();
@@ -310,7 +310,7 @@ static void tmu2_start(MilkymistTMU2State *s)
     qemu_irq_pulse(s->irq);
 }
 
-static uint64_t tmu2_read(void *opaque, target_phys_addr_t addr,
+static uint64_t tmu2_read(void *opaque, hwaddr addr,
                           unsigned size)
 {
     MilkymistTMU2State *s = opaque;
@@ -372,7 +372,7 @@ static void tmu2_check_registers(MilkymistTMU2State *s)
     }
 }
 
-static void tmu2_write(void *opaque, target_phys_addr_t addr, uint64_t value,
+static void tmu2_write(void *opaque, hwaddr addr, uint64_t value,
                        unsigned size)
 {
     MilkymistTMU2State *s = opaque;
diff --git a/hw/milkymist-uart.c b/hw/milkymist-uart.c
index 291fe3c57..aefa8c7f1 100644
--- a/hw/milkymist-uart.c
+++ b/hw/milkymist-uart.c
@@ -78,7 +78,7 @@ static void uart_update_irq(MilkymistUartState *s)
     }
 }
 
-static uint64_t uart_read(void *opaque, target_phys_addr_t addr,
+static uint64_t uart_read(void *opaque, hwaddr addr,
                           unsigned size)
 {
     MilkymistUartState *s = opaque;
@@ -107,7 +107,7 @@ static uint64_t uart_read(void *opaque, target_phys_addr_t addr,
     return r;
 }
 
-static void uart_write(void *opaque, target_phys_addr_t addr, uint64_t value,
+static void uart_write(void *opaque, hwaddr addr, uint64_t value,
                        unsigned size)
 {
     MilkymistUartState *s = opaque;
diff --git a/hw/milkymist-vgafb.c b/hw/milkymist-vgafb.c
index cd4365d64..833881cc6 100644
--- a/hw/milkymist-vgafb.c
+++ b/hw/milkymist-vgafb.c
@@ -134,7 +134,7 @@ static void vgafb_update_display(void *opaque)
                                &first, &last);
 
     if (first >= 0) {
-        dpy_update(s->ds, 0, first, s->regs[R_HRES], last - first + 1);
+        dpy_gfx_update(s->ds, 0, first, s->regs[R_HRES], last - first + 1);
     }
     s->invalidate = 0;
 }
@@ -155,7 +155,7 @@ static void vgafb_resize(MilkymistVgafbState *s)
     s->invalidate = 1;
 }
 
-static uint64_t vgafb_read(void *opaque, target_phys_addr_t addr,
+static uint64_t vgafb_read(void *opaque, hwaddr addr,
                            unsigned size)
 {
     MilkymistVgafbState *s = opaque;
@@ -193,7 +193,7 @@ static uint64_t vgafb_read(void *opaque, target_phys_addr_t addr,
     return r;
 }
 
-static void vgafb_write(void *opaque, target_phys_addr_t addr, uint64_t value,
+static void vgafb_write(void *opaque, hwaddr addr, uint64_t value,
                         unsigned size)
 {
     MilkymistVgafbState *s = opaque;
diff --git a/hw/milkymist.c b/hw/milkymist.c
index 2e7235b4b..4c8111a74 100644
--- a/hw/milkymist.c
+++ b/hw/milkymist.c
@@ -38,11 +38,11 @@
 
 typedef struct {
     LM32CPU *cpu;
-    target_phys_addr_t bootstrap_pc;
-    target_phys_addr_t flash_base;
-    target_phys_addr_t initrd_base;
+    hwaddr bootstrap_pc;
+    hwaddr flash_base;
+    hwaddr initrd_base;
     size_t initrd_size;
-    target_phys_addr_t cmdline_base;
+    hwaddr cmdline_base;
 } ResetInfo;
 
 static void cpu_irq_handler(void *opaque, int irq, int level)
@@ -73,12 +73,12 @@ static void main_cpu_reset(void *opaque)
 }
 
 static void
-milkymist_init(ram_addr_t ram_size_not_used,
-                          const char *boot_device,
-                          const char *kernel_filename,
-                          const char *kernel_cmdline,
-                          const char *initrd_filename, const char *cpu_model)
+milkymist_init(QEMUMachineInitArgs *args)
 {
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
     LM32CPU *cpu;
     CPULM32State *env;
     int kernel_size;
@@ -91,14 +91,14 @@ milkymist_init(ram_addr_t ram_size_not_used,
     ResetInfo *reset_info;
 
     /* memory map */
-    target_phys_addr_t flash_base   = 0x00000000;
+    hwaddr flash_base   = 0x00000000;
     size_t flash_sector_size        = 128 * 1024;
     size_t flash_size               = 32 * 1024 * 1024;
-    target_phys_addr_t sdram_base   = 0x40000000;
+    hwaddr sdram_base   = 0x40000000;
     size_t sdram_size               = 128 * 1024 * 1024;
 
-    target_phys_addr_t initrd_base  = sdram_base + 0x1002000;
-    target_phys_addr_t cmdline_base = sdram_base + 0x1000000;
+    hwaddr initrd_base  = sdram_base + 0x1002000;
+    hwaddr cmdline_base = sdram_base + 0x1000000;
     size_t initrd_max = sdram_size - 0x1002000;
 
     reset_info = g_malloc0(sizeof(ResetInfo));
diff --git a/hw/mips.h b/hw/mips.h
index a7e6d4cc6..f7e9b7e2c 100644
--- a/hw/mips.h
+++ b/hw/mips.h
@@ -12,7 +12,7 @@ PCIBus *bonito_init(qemu_irq *pic);
 
 /* rc4030.c */
 typedef struct rc4030DMAState *rc4030_dma;
-void rc4030_dma_memory_rw(void *opaque, target_phys_addr_t addr, uint8_t *buf, int len, int is_write);
+void rc4030_dma_memory_rw(void *opaque, hwaddr addr, uint8_t *buf, int len, int is_write);
 void rc4030_dma_read(void *dma, uint8_t *buf, int len);
 void rc4030_dma_write(void *dma, uint8_t *buf, int len);
 
@@ -21,9 +21,9 @@ void *rc4030_init(qemu_irq timer, qemu_irq jazz_bus,
                   MemoryRegion *sysmem);
 
 /* dp8393x.c */
-void dp83932_init(NICInfo *nd, target_phys_addr_t base, int it_shift,
+void dp83932_init(NICInfo *nd, hwaddr base, int it_shift,
                   MemoryRegion *address_space,
                   qemu_irq irq, void* mem_opaque,
-                  void (*memory_rw)(void *opaque, target_phys_addr_t addr, uint8_t *buf, int len, int is_write));
+                  void (*memory_rw)(void *opaque, hwaddr addr, uint8_t *buf, int len, int is_write));
 
 #endif
diff --git a/hw/mips_fulong2e.c b/hw/mips_fulong2e.c
index 38e4b8615..5fcf900e0 100644
--- a/hw/mips_fulong2e.c
+++ b/hw/mips_fulong2e.c
@@ -20,6 +20,7 @@
 
 #include "hw.h"
 #include "pc.h"
+#include "serial.h"
 #include "fdc.h"
 #include "net.h"
 #include "boards.h"
@@ -256,10 +257,13 @@ static void cpu_request_exit(void *opaque, int irq, int level)
     }
 }
 
-static void mips_fulong2e_init(ram_addr_t ram_size, const char *boot_device,
-                        const char *kernel_filename, const char *kernel_cmdline,
-                        const char *initrd_filename, const char *cpu_model)
+static void mips_fulong2e_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t ram_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
     char *filename;
     MemoryRegion *address_space_mem = get_system_memory();
     MemoryRegion *ram = g_new(MemoryRegion, 1);
@@ -392,7 +396,7 @@ static void mips_fulong2e_init(ram_addr_t ram_size, const char *boot_device,
     network_init();
 }
 
-QEMUMachine mips_fulong2e_machine = {
+static QEMUMachine mips_fulong2e_machine = {
     .name = "fulong2e",
     .desc = "Fulong 2e mini pc",
     .init = mips_fulong2e_init,
diff --git a/hw/mips_jazz.c b/hw/mips_jazz.c
index db927f14d..084742724 100644
--- a/hw/mips_jazz.c
+++ b/hw/mips_jazz.c
@@ -26,6 +26,7 @@
 #include "mips.h"
 #include "mips_cpudevs.h"
 #include "pc.h"
+#include "serial.h"
 #include "isa.h"
 #include "fdc.h"
 #include "sysemu.h"
@@ -55,12 +56,12 @@ static void main_cpu_reset(void *opaque)
     cpu_reset(CPU(cpu));
 }
 
-static uint64_t rtc_read(void *opaque, target_phys_addr_t addr, unsigned size)
+static uint64_t rtc_read(void *opaque, hwaddr addr, unsigned size)
 {
     return cpu_inw(0x71);
 }
 
-static void rtc_write(void *opaque, target_phys_addr_t addr,
+static void rtc_write(void *opaque, hwaddr addr,
                       uint64_t val, unsigned size)
 {
     cpu_outw(0x71, val & 0xff);
@@ -72,7 +73,7 @@ static const MemoryRegionOps rtc_ops = {
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-static uint64_t dma_dummy_read(void *opaque, target_phys_addr_t addr,
+static uint64_t dma_dummy_read(void *opaque, hwaddr addr,
                                unsigned size)
 {
     /* Nothing to do. That is only to ensure that
@@ -80,7 +81,7 @@ static uint64_t dma_dummy_read(void *opaque, target_phys_addr_t addr,
     return 0xff;
 }
 
-static void dma_dummy_write(void *opaque, target_phys_addr_t addr,
+static void dma_dummy_write(void *opaque, hwaddr addr,
                             uint64_t val, unsigned size)
 {
     /* Nothing to do. That is only to ensure that
@@ -302,21 +303,19 @@ static void mips_jazz_init(MemoryRegion *address_space,
 }
 
 static
-void mips_magnum_init (ram_addr_t ram_size,
-                       const char *boot_device,
-                       const char *kernel_filename, const char *kernel_cmdline,
-                       const char *initrd_filename, const char *cpu_model)
+void mips_magnum_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t ram_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
         mips_jazz_init(get_system_memory(), get_system_io(),
                        ram_size, cpu_model, JAZZ_MAGNUM);
 }
 
 static
-void mips_pica61_init (ram_addr_t ram_size,
-                       const char *boot_device,
-                       const char *kernel_filename, const char *kernel_cmdline,
-                       const char *initrd_filename, const char *cpu_model)
+void mips_pica61_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t ram_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
     mips_jazz_init(get_system_memory(), get_system_io(),
                    ram_size, cpu_model, JAZZ_PICA61);
 }
diff --git a/hw/mips_malta.c b/hw/mips_malta.c
index ad23f26e5..4d2464a02 100644
--- a/hw/mips_malta.c
+++ b/hw/mips_malta.c
@@ -24,6 +24,7 @@
 
 #include "hw.h"
 #include "pc.h"
+#include "serial.h"
 #include "fdc.h"
 #include "net.h"
 #include "boards.h"
@@ -33,7 +34,6 @@
 #include "mips.h"
 #include "mips_cpudevs.h"
 #include "pci.h"
-#include "vmware_vga.h"
 #include "qemu-char.h"
 #include "sysemu.h"
 #include "arch_init.h"
@@ -48,7 +48,6 @@
 #include "blockdev.h"
 #include "exec-memory.h"
 #include "sysbus.h"             /* SysBusDevice */
-#include "vga-pci.h"
 
 //#define DEBUG_BOARD_INIT
 
@@ -232,7 +231,7 @@ static void eeprom24c0x_write(int scl, int sda)
     eeprom.sda = sda;
 }
 
-static uint64_t malta_fpga_read(void *opaque, target_phys_addr_t addr,
+static uint64_t malta_fpga_read(void *opaque, hwaddr addr,
                                 unsigned size)
 {
     MaltaFPGAState *s = opaque;
@@ -320,7 +319,7 @@ static uint64_t malta_fpga_read(void *opaque, target_phys_addr_t addr,
     return val;
 }
 
-static void malta_fpga_write(void *opaque, target_phys_addr_t addr,
+static void malta_fpga_write(void *opaque, hwaddr addr,
                              uint64_t val, unsigned size)
 {
     MaltaFPGAState *s = opaque;
@@ -442,7 +441,7 @@ static void malta_fpga_led_init(CharDriverState *chr)
 }
 
 static MaltaFPGAState *malta_fpga_init(MemoryRegion *address_space,
-         target_phys_addr_t base, qemu_irq uart_irq, CharDriverState *uart_chr)
+         hwaddr base, qemu_irq uart_irq, CharDriverState *uart_chr)
 {
     MaltaFPGAState *s;
 
@@ -777,11 +776,13 @@ static void cpu_request_exit(void *opaque, int irq, int level)
 }
 
 static
-void mips_malta_init (ram_addr_t ram_size,
-                      const char *boot_device,
-                      const char *kernel_filename, const char *kernel_cmdline,
-                      const char *initrd_filename, const char *cpu_model)
+void mips_malta_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t ram_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
     char *filename;
     pflash_t *fl;
     MemoryRegion *system_memory = get_system_memory();
@@ -860,7 +861,8 @@ void mips_malta_init (ram_addr_t ram_size,
     be = 0;
 #endif
     /* FPGA */
-    malta_fpga_init(system_memory, FPGA_ADDRESS, env->irq[2], serial_hds[2]);
+    /* The CBUS UART is attached to the MIPS CPU INT2 pin, ie interrupt 4 */
+    malta_fpga_init(system_memory, FPGA_ADDRESS, env->irq[4], serial_hds[2]);
 
     /* Load firmware in flash / BIOS. */
     dinfo = drive_get(IF_PFLASH, 0, fl_idx);
@@ -987,13 +989,7 @@ void mips_malta_init (ram_addr_t ram_size,
     network_init();
 
     /* Optional PCI video card */
-    if (cirrus_vga_enabled) {
-        pci_cirrus_vga_init(pci_bus);
-    } else if (vmsvga_enabled) {
-        pci_vmsvga_init(pci_bus);
-    } else if (std_vga_enabled) {
-        pci_vga_init(pci_bus);
-    }
+    pci_vga_init(pci_bus);
 }
 
 static int mips_malta_sysbus_device_init(SysBusDevice *sysbusdev)
diff --git a/hw/mips_mipssim.c b/hw/mips_mipssim.c
index 830f63559..a95a3c1f1 100644
--- a/hw/mips_mipssim.c
+++ b/hw/mips_mipssim.c
@@ -27,7 +27,7 @@
 #include "hw.h"
 #include "mips.h"
 #include "mips_cpudevs.h"
-#include "pc.h"
+#include "serial.h"
 #include "isa.h"
 #include "net.h"
 #include "sysemu.h"
@@ -131,11 +131,13 @@ static void mipsnet_init(int base, qemu_irq irq, NICInfo *nd)
 }
 
 static void
-mips_mipssim_init (ram_addr_t ram_size,
-                   const char *boot_device,
-                   const char *kernel_filename, const char *kernel_cmdline,
-                   const char *initrd_filename, const char *cpu_model)
+mips_mipssim_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t ram_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
     char *filename;
     MemoryRegion *address_space_mem = get_system_memory();
     MemoryRegion *ram = g_new(MemoryRegion, 1);
diff --git a/hw/mips_r4k.c b/hw/mips_r4k.c
index 967a76e53..325098a43 100644
--- a/hw/mips_r4k.c
+++ b/hw/mips_r4k.c
@@ -11,6 +11,7 @@
 #include "mips.h"
 #include "mips_cpudevs.h"
 #include "pc.h"
+#include "serial.h"
 #include "isa.h"
 #include "net.h"
 #include "sysemu.h"
@@ -43,7 +44,7 @@ static struct _loaderparams {
     const char *initrd_filename;
 } loaderparams;
 
-static void mips_qemu_write (void *opaque, target_phys_addr_t addr,
+static void mips_qemu_write (void *opaque, hwaddr addr,
                              uint64_t val, unsigned size)
 {
     if ((addr & 0xffff) == 0 && val == 42)
@@ -52,7 +53,7 @@ static void mips_qemu_write (void *opaque, target_phys_addr_t addr,
         qemu_system_shutdown_request ();
 }
 
-static uint64_t mips_qemu_read (void *opaque, target_phys_addr_t addr,
+static uint64_t mips_qemu_read (void *opaque, hwaddr addr,
                                 unsigned size)
 {
     return 0;
@@ -151,11 +152,13 @@ static void main_cpu_reset(void *opaque)
 
 static const int sector_len = 32 * 1024;
 static
-void mips_r4k_init (ram_addr_t ram_size,
-                    const char *boot_device,
-                    const char *kernel_filename, const char *kernel_cmdline,
-                    const char *initrd_filename, const char *cpu_model)
+void mips_r4k_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t ram_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
     char *filename;
     MemoryRegion *address_space_mem = get_system_memory();
     MemoryRegion *ram = g_new(MemoryRegion, 1);
diff --git a/hw/mipsnet.c b/hw/mipsnet.c
index 28063b110..bece332c4 100644
--- a/hw/mipsnet.c
+++ b/hw/mipsnet.c
@@ -96,7 +96,7 @@ static ssize_t mipsnet_receive(NetClientState *nc, const uint8_t *buf, size_t si
     return size;
 }
 
-static uint64_t mipsnet_ioport_read(void *opaque, target_phys_addr_t addr,
+static uint64_t mipsnet_ioport_read(void *opaque, hwaddr addr,
                                     unsigned int size)
 {
     MIPSnetState *s = opaque;
@@ -142,7 +142,7 @@ static uint64_t mipsnet_ioport_read(void *opaque, target_phys_addr_t addr,
     return ret;
 }
 
-static void mipsnet_ioport_write(void *opaque, target_phys_addr_t addr,
+static void mipsnet_ioport_write(void *opaque, hwaddr addr,
                                  uint64_t val, unsigned int size)
 {
     MIPSnetState *s = opaque;
diff --git a/hw/mpc8544_guts.c b/hw/mpc8544_guts.c
index 13b0dddc1..873cb8cbf 100644
--- a/hw/mpc8544_guts.c
+++ b/hw/mpc8544_guts.c
@@ -58,7 +58,7 @@ struct GutsState {
 
 typedef struct GutsState GutsState;
 
-static uint64_t mpc8544_guts_read(void *opaque, target_phys_addr_t addr,
+static uint64_t mpc8544_guts_read(void *opaque, hwaddr addr,
                                   unsigned size)
 {
     uint32_t value = 0;
@@ -80,7 +80,7 @@ static uint64_t mpc8544_guts_read(void *opaque, target_phys_addr_t addr,
     return value;
 }
 
-static void mpc8544_guts_write(void *opaque, target_phys_addr_t addr,
+static void mpc8544_guts_write(void *opaque, hwaddr addr,
                                uint64_t value, unsigned size)
 {
     addr &= MPC8544_GUTS_MMIO_SIZE - 1;
diff --git a/hw/msi.c b/hw/msi.c
index e2273a09a..33037a80e 100644
--- a/hw/msi.c
+++ b/hw/msi.c
@@ -122,6 +122,31 @@ void msi_set_message(PCIDevice *dev, MSIMessage msg)
     pci_set_word(dev->config + msi_data_off(dev, msi64bit), msg.data);
 }
 
+MSIMessage msi_get_message(PCIDevice *dev, unsigned int vector)
+{
+    uint16_t flags = pci_get_word(dev->config + msi_flags_off(dev));
+    bool msi64bit = flags & PCI_MSI_FLAGS_64BIT;
+    unsigned int nr_vectors = msi_nr_vectors(flags);
+    MSIMessage msg;
+
+    assert(vector < nr_vectors);
+
+    if (msi64bit) {
+        msg.address = pci_get_quad(dev->config + msi_address_lo_off(dev));
+    } else {
+        msg.address = pci_get_long(dev->config + msi_address_lo_off(dev));
+    }
+
+    /* upper bit 31:16 is zero */
+    msg.data = pci_get_word(dev->config + msi_data_off(dev, msi64bit));
+    if (nr_vectors > 1) {
+        msg.data &= ~(nr_vectors - 1);
+        msg.data |= vector;
+    }
+
+    return msg;
+}
+
 bool msi_enabled(const PCIDevice *dev)
 {
     return msi_present(dev) &&
@@ -249,8 +274,7 @@ void msi_notify(PCIDevice *dev, unsigned int vector)
     uint16_t flags = pci_get_word(dev->config + msi_flags_off(dev));
     bool msi64bit = flags & PCI_MSI_FLAGS_64BIT;
     unsigned int nr_vectors = msi_nr_vectors(flags);
-    uint64_t address;
-    uint32_t data;
+    MSIMessage msg;
 
     assert(vector < nr_vectors);
     if (msi_is_masked(dev, vector)) {
@@ -261,24 +285,13 @@ void msi_notify(PCIDevice *dev, unsigned int vector)
         return;
     }
 
-    if (msi64bit) {
-        address = pci_get_quad(dev->config + msi_address_lo_off(dev));
-    } else {
-        address = pci_get_long(dev->config + msi_address_lo_off(dev));
-    }
-
-    /* upper bit 31:16 is zero */
-    data = pci_get_word(dev->config + msi_data_off(dev, msi64bit));
-    if (nr_vectors > 1) {
-        data &= ~(nr_vectors - 1);
-        data |= vector;
-    }
+    msg = msi_get_message(dev, vector);
 
     MSI_DEV_PRINTF(dev,
                    "notify vector 0x%x"
                    " address: 0x%"PRIx64" data: 0x%"PRIx32"\n",
-                   vector, address, data);
-    stl_le_phys(address, data);
+                   vector, msg.address, msg.data);
+    stl_le_phys(msg.address, msg.data);
 }
 
 /* Normally called by pci_default_write_config(). */
diff --git a/hw/msi.h b/hw/msi.h
index 6ec1f99f8..150b09a19 100644
--- a/hw/msi.h
+++ b/hw/msi.h
@@ -32,6 +32,7 @@ struct MSIMessage {
 extern bool msi_supported;
 
 void msi_set_message(PCIDevice *dev, MSIMessage msg);
+MSIMessage msi_get_message(PCIDevice *dev, unsigned int vector);
 bool msi_enabled(const PCIDevice *dev);
 int msi_init(struct PCIDevice *dev, uint8_t offset,
              unsigned int nr_vectors, bool msi64bit, bool msi_per_vector_mask);
diff --git a/hw/msix.c b/hw/msix.c
index d81209413..136ef0937 100644
--- a/hw/msix.c
+++ b/hw/msix.c
@@ -157,7 +157,7 @@ void msix_write_config(PCIDevice *dev, uint32_t addr,
     }
 }
 
-static uint64_t msix_table_mmio_read(void *opaque, target_phys_addr_t addr,
+static uint64_t msix_table_mmio_read(void *opaque, hwaddr addr,
                                      unsigned size)
 {
     PCIDevice *dev = opaque;
@@ -165,7 +165,7 @@ static uint64_t msix_table_mmio_read(void *opaque, target_phys_addr_t addr,
     return pci_get_long(dev->msix_table + addr);
 }
 
-static void msix_table_mmio_write(void *opaque, target_phys_addr_t addr,
+static void msix_table_mmio_write(void *opaque, hwaddr addr,
                                   uint64_t val, unsigned size)
 {
     PCIDevice *dev = opaque;
@@ -188,7 +188,7 @@ static const MemoryRegionOps msix_table_mmio_ops = {
     },
 };
 
-static uint64_t msix_pba_mmio_read(void *opaque, target_phys_addr_t addr,
+static uint64_t msix_pba_mmio_read(void *opaque, hwaddr addr,
                                    unsigned size)
 {
     PCIDevice *dev = opaque;
@@ -366,7 +366,6 @@ void msix_uninit(PCIDevice *dev, MemoryRegion *table_bar, MemoryRegion *pba_bar)
     g_free(dev->msix_entry_used);
     dev->msix_entry_used = NULL;
     dev->cap_present &= ~QEMU_PCI_CAP_MSIX;
-    return;
 }
 
 void msix_uninit_exclusive_bar(PCIDevice *dev)
diff --git a/hw/mst_fpga.c b/hw/mst_fpga.c
index 024192d13..fb4b739c7 100644
--- a/hw/mst_fpga.c
+++ b/hw/mst_fpga.c
@@ -91,7 +91,7 @@ mst_fpga_set_irq(void *opaque, int irq, int level)
 
 
 static uint64_t
-mst_fpga_readb(void *opaque, target_phys_addr_t addr, unsigned size)
+mst_fpga_readb(void *opaque, hwaddr addr, unsigned size)
 {
 	mst_irq_state *s = (mst_irq_state *) opaque;
 
@@ -128,7 +128,7 @@ mst_fpga_readb(void *opaque, target_phys_addr_t addr, unsigned size)
 }
 
 static void
-mst_fpga_writeb(void *opaque, target_phys_addr_t addr, uint64_t value,
+mst_fpga_writeb(void *opaque, hwaddr addr, uint64_t value,
 		unsigned size)
 {
 	mst_irq_state *s = (mst_irq_state *) opaque;
diff --git a/hw/multiboot.c b/hw/multiboot.c
index b1e04c571..09ec5b253 100644
--- a/hw/multiboot.c
+++ b/hw/multiboot.c
@@ -80,15 +80,15 @@ typedef struct {
     /* buffer holding kernel, cmdlines and mb_infos */
     void *mb_buf;
     /* address in target */
-    target_phys_addr_t mb_buf_phys;
+    hwaddr mb_buf_phys;
     /* size of mb_buf in bytes */
     unsigned mb_buf_size;
     /* offset of mb-info's in bytes */
-    target_phys_addr_t offset_mbinfo;
+    hwaddr offset_mbinfo;
     /* offset in buffer for cmdlines in bytes */
-    target_phys_addr_t offset_cmdlines;
+    hwaddr offset_cmdlines;
     /* offset of modules in bytes */
-    target_phys_addr_t offset_mods;
+    hwaddr offset_mods;
     /* available slots for mb modules infos */
     int mb_mods_avail;
     /* currently used slots of mb modules */
@@ -97,7 +97,7 @@ typedef struct {
 
 static uint32_t mb_add_cmdline(MultibootState *s, const char *cmdline)
 {
-    target_phys_addr_t p = s->offset_cmdlines;
+    hwaddr p = s->offset_cmdlines;
     char *b = (char *)s->mb_buf + p;
 
     get_opt_value(b, strlen(cmdline) + 1, cmdline);
@@ -106,8 +106,8 @@ static uint32_t mb_add_cmdline(MultibootState *s, const char *cmdline)
 }
 
 static void mb_add_mod(MultibootState *s,
-                       target_phys_addr_t start, target_phys_addr_t end,
-                       target_phys_addr_t cmdline_phys)
+                       hwaddr start, hwaddr end,
+                       hwaddr cmdline_phys)
 {
     char *p;
     assert(s->mb_mods_count < s->mb_mods_avail);
@@ -276,7 +276,7 @@ int load_multiboot(void *fw_cfg,
             *next_initrd = '\0';
             /* if a space comes after the module filename, treat everything
                after that as parameters */
-            target_phys_addr_t c = mb_add_cmdline(&mbs, initrd_filename);
+            hwaddr c = mb_add_cmdline(&mbs, initrd_filename);
             if ((next_space = strchr(initrd_filename, ' ')))
                 *next_space = '\0';
             mb_debug("multiboot loading module: %s\n", initrd_filename);
diff --git a/hw/musicpal.c b/hw/musicpal.c
index ad725b559..e0c57c84e 100644
--- a/hw/musicpal.c
+++ b/hw/musicpal.c
@@ -15,7 +15,7 @@
 #include "net.h"
 #include "sysemu.h"
 #include "boards.h"
-#include "pc.h"
+#include "serial.h"
 #include "qemu-timer.h"
 #include "ptimer.h"
 #include "block.h"
@@ -266,7 +266,7 @@ static void eth_send(mv88w8618_eth_state *s, int queue_index)
     } while (desc_addr != s->tx_queue[queue_index]);
 }
 
-static uint64_t mv88w8618_eth_read(void *opaque, target_phys_addr_t offset,
+static uint64_t mv88w8618_eth_read(void *opaque, hwaddr offset,
                                    unsigned size)
 {
     mv88w8618_eth_state *s = opaque;
@@ -308,7 +308,7 @@ static uint64_t mv88w8618_eth_read(void *opaque, target_phys_addr_t offset,
     }
 }
 
-static void mv88w8618_eth_write(void *opaque, target_phys_addr_t offset,
+static void mv88w8618_eth_write(void *opaque, hwaddr offset,
                                 uint64_t value, unsigned size)
 {
     mv88w8618_eth_state *s = opaque;
@@ -526,7 +526,7 @@ static void lcd_refresh(void *opaque)
                   ds_get_bits_per_pixel(s->ds));
     }
 
-    dpy_update(s->ds, 0, 0, 128*3, 64*3);
+    dpy_gfx_update(s->ds, 0, 0, 128*3, 64*3);
 }
 
 static void lcd_invalidate(void *opaque)
@@ -540,7 +540,7 @@ static void musicpal_lcd_gpio_brigthness_in(void *opaque, int irq, int level)
     s->brightness |= level << irq;
 }
 
-static uint64_t musicpal_lcd_read(void *opaque, target_phys_addr_t offset,
+static uint64_t musicpal_lcd_read(void *opaque, hwaddr offset,
                                   unsigned size)
 {
     musicpal_lcd_state *s = opaque;
@@ -554,7 +554,7 @@ static uint64_t musicpal_lcd_read(void *opaque, target_phys_addr_t offset,
     }
 }
 
-static void musicpal_lcd_write(void *opaque, target_phys_addr_t offset,
+static void musicpal_lcd_write(void *opaque, hwaddr offset,
                                uint64_t value, unsigned size)
 {
     musicpal_lcd_state *s = opaque;
@@ -682,7 +682,7 @@ static void mv88w8618_pic_set_irq(void *opaque, int irq, int level)
     mv88w8618_pic_update(s);
 }
 
-static uint64_t mv88w8618_pic_read(void *opaque, target_phys_addr_t offset,
+static uint64_t mv88w8618_pic_read(void *opaque, hwaddr offset,
                                    unsigned size)
 {
     mv88w8618_pic_state *s = opaque;
@@ -696,7 +696,7 @@ static uint64_t mv88w8618_pic_read(void *opaque, target_phys_addr_t offset,
     }
 }
 
-static void mv88w8618_pic_write(void *opaque, target_phys_addr_t offset,
+static void mv88w8618_pic_write(void *opaque, hwaddr offset,
                                 uint64_t value, unsigned size)
 {
     mv88w8618_pic_state *s = opaque;
@@ -815,7 +815,7 @@ static void mv88w8618_timer_init(SysBusDevice *dev, mv88w8618_timer_state *s,
     s->ptimer = ptimer_init(bh);
 }
 
-static uint64_t mv88w8618_pit_read(void *opaque, target_phys_addr_t offset,
+static uint64_t mv88w8618_pit_read(void *opaque, hwaddr offset,
                                    unsigned size)
 {
     mv88w8618_pit_state *s = opaque;
@@ -831,7 +831,7 @@ static uint64_t mv88w8618_pit_read(void *opaque, target_phys_addr_t offset,
     }
 }
 
-static void mv88w8618_pit_write(void *opaque, target_phys_addr_t offset,
+static void mv88w8618_pit_write(void *opaque, hwaddr offset,
                                 uint64_t value, unsigned size)
 {
     mv88w8618_pit_state *s = opaque;
@@ -957,7 +957,7 @@ typedef struct mv88w8618_flashcfg_state {
 } mv88w8618_flashcfg_state;
 
 static uint64_t mv88w8618_flashcfg_read(void *opaque,
-                                        target_phys_addr_t offset,
+                                        hwaddr offset,
                                         unsigned size)
 {
     mv88w8618_flashcfg_state *s = opaque;
@@ -971,7 +971,7 @@ static uint64_t mv88w8618_flashcfg_read(void *opaque,
     }
 }
 
-static void mv88w8618_flashcfg_write(void *opaque, target_phys_addr_t offset,
+static void mv88w8618_flashcfg_write(void *opaque, hwaddr offset,
                                      uint64_t value, unsigned size)
 {
     mv88w8618_flashcfg_state *s = opaque;
@@ -1032,7 +1032,7 @@ static TypeInfo mv88w8618_flashcfg_info = {
 
 #define MP_BOARD_REVISION       0x31
 
-static uint64_t musicpal_misc_read(void *opaque, target_phys_addr_t offset,
+static uint64_t musicpal_misc_read(void *opaque, hwaddr offset,
                                    unsigned size)
 {
     switch (offset) {
@@ -1044,7 +1044,7 @@ static uint64_t musicpal_misc_read(void *opaque, target_phys_addr_t offset,
     }
 }
 
-static void musicpal_misc_write(void *opaque, target_phys_addr_t offset,
+static void musicpal_misc_write(void *opaque, hwaddr offset,
                                 uint64_t value, unsigned size)
 {
 }
@@ -1068,7 +1068,7 @@ static void musicpal_misc_init(SysBusDevice *dev)
 #define MP_WLAN_MAGIC1          0x11c
 #define MP_WLAN_MAGIC2          0x124
 
-static uint64_t mv88w8618_wlan_read(void *opaque, target_phys_addr_t offset,
+static uint64_t mv88w8618_wlan_read(void *opaque, hwaddr offset,
                                     unsigned size)
 {
     switch (offset) {
@@ -1084,7 +1084,7 @@ static uint64_t mv88w8618_wlan_read(void *opaque, target_phys_addr_t offset,
     }
 }
 
-static void mv88w8618_wlan_write(void *opaque, target_phys_addr_t offset,
+static void mv88w8618_wlan_write(void *opaque, hwaddr offset,
                                  uint64_t value, unsigned size)
 {
 }
@@ -1202,7 +1202,7 @@ static void musicpal_gpio_pin_event(void *opaque, int pin, int level)
     }
 }
 
-static uint64_t musicpal_gpio_read(void *opaque, target_phys_addr_t offset,
+static uint64_t musicpal_gpio_read(void *opaque, hwaddr offset,
                                    unsigned size)
 {
     musicpal_gpio_state *s = opaque;
@@ -1241,7 +1241,7 @@ static uint64_t musicpal_gpio_read(void *opaque, target_phys_addr_t offset,
     }
 }
 
-static void musicpal_gpio_write(void *opaque, target_phys_addr_t offset,
+static void musicpal_gpio_write(void *opaque, hwaddr offset,
                                 uint64_t value, unsigned size)
 {
     musicpal_gpio_state *s = opaque;
@@ -1508,11 +1508,12 @@ static struct arm_boot_info musicpal_binfo = {
     .board_id = 0x20e,
 };
 
-static void musicpal_init(ram_addr_t ram_size,
-               const char *boot_device,
-               const char *kernel_filename, const char *kernel_cmdline,
-               const char *initrd_filename, const char *cpu_model)
+static void musicpal_init(QEMUMachineInitArgs *args)
 {
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
     ARMCPU *cpu;
     qemu_irq *cpu_pic;
     qemu_irq pic[32];
@@ -1583,7 +1584,7 @@ static void musicpal_init(ram_addr_t ram_size,
          * image is smaller than 32 MB.
          */
 #ifdef TARGET_WORDS_BIGENDIAN
-        pflash_cfi02_register(0-MP_FLASH_SIZE_MAX, NULL,
+        pflash_cfi02_register(0x100000000ULL-MP_FLASH_SIZE_MAX, NULL,
                               "musicpal.flash", flash_size,
                               dinfo->bdrv, 0x10000,
                               (flash_size + 0xffff) >> 16,
@@ -1591,7 +1592,7 @@ static void musicpal_init(ram_addr_t ram_size,
                               2, 0x00BF, 0x236D, 0x0000, 0x0000,
                               0x5555, 0x2AAA, 1);
 #else
-        pflash_cfi02_register(0-MP_FLASH_SIZE_MAX, NULL,
+        pflash_cfi02_register(0x100000000ULL-MP_FLASH_SIZE_MAX, NULL,
                               "musicpal.flash", flash_size,
                               dinfo->bdrv, 0x10000,
                               (flash_size + 0xffff) >> 16,
diff --git a/hw/nand.c b/hw/nand.c
index e9501ae03..01f3adaee 100644
--- a/hw/nand.c
+++ b/hw/nand.c
@@ -654,7 +654,7 @@ static void glue(nand_blk_write_, PAGE_SIZE)(NANDFlashState *s)
         sector = SECTOR(s->addr);
         off = (s->addr & PAGE_MASK) + s->offset;
         soff = SECTOR_OFFSET(s->addr);
-        if (bdrv_read(s->bdrv, sector, iobuf, PAGE_SECTORS) == -1) {
+        if (bdrv_read(s->bdrv, sector, iobuf, PAGE_SECTORS) < 0) {
             printf("%s: read error in sector %" PRIu64 "\n", __func__, sector);
             return;
         }
@@ -666,21 +666,23 @@ static void glue(nand_blk_write_, PAGE_SIZE)(NANDFlashState *s)
                             MIN(OOB_SIZE, off + s->iolen - PAGE_SIZE));
         }
 
-        if (bdrv_write(s->bdrv, sector, iobuf, PAGE_SECTORS) == -1)
+        if (bdrv_write(s->bdrv, sector, iobuf, PAGE_SECTORS) < 0) {
             printf("%s: write error in sector %" PRIu64 "\n", __func__, sector);
+        }
     } else {
         off = PAGE_START(s->addr) + (s->addr & PAGE_MASK) + s->offset;
         sector = off >> 9;
         soff = off & 0x1ff;
-        if (bdrv_read(s->bdrv, sector, iobuf, PAGE_SECTORS + 2) == -1) {
+        if (bdrv_read(s->bdrv, sector, iobuf, PAGE_SECTORS + 2) < 0) {
             printf("%s: read error in sector %" PRIu64 "\n", __func__, sector);
             return;
         }
 
         mem_and(iobuf + soff, s->io, s->iolen);
 
-        if (bdrv_write(s->bdrv, sector, iobuf, PAGE_SECTORS + 2) == -1)
+        if (bdrv_write(s->bdrv, sector, iobuf, PAGE_SECTORS + 2) < 0) {
             printf("%s: write error in sector %" PRIu64 "\n", __func__, sector);
+        }
     }
     s->offset = 0;
 }
@@ -704,31 +706,37 @@ static void glue(nand_blk_erase_, PAGE_SIZE)(NANDFlashState *s)
         i = SECTOR(addr);
         page = SECTOR(addr + (ADDR_SHIFT + s->erase_shift));
         for (; i < page; i ++)
-            if (bdrv_write(s->bdrv, i, iobuf, 1) == -1)
+            if (bdrv_write(s->bdrv, i, iobuf, 1) < 0) {
                 printf("%s: write error in sector %" PRIu64 "\n", __func__, i);
+            }
     } else {
         addr = PAGE_START(addr);
         page = addr >> 9;
-        if (bdrv_read(s->bdrv, page, iobuf, 1) == -1)
+        if (bdrv_read(s->bdrv, page, iobuf, 1) < 0) {
             printf("%s: read error in sector %" PRIu64 "\n", __func__, page);
+        }
         memset(iobuf + (addr & 0x1ff), 0xff, (~addr & 0x1ff) + 1);
-        if (bdrv_write(s->bdrv, page, iobuf, 1) == -1)
+        if (bdrv_write(s->bdrv, page, iobuf, 1) < 0) {
             printf("%s: write error in sector %" PRIu64 "\n", __func__, page);
+        }
 
         memset(iobuf, 0xff, 0x200);
         i = (addr & ~0x1ff) + 0x200;
         for (addr += ((PAGE_SIZE + OOB_SIZE) << s->erase_shift) - 0x200;
                         i < addr; i += 0x200)
-            if (bdrv_write(s->bdrv, i >> 9, iobuf, 1) == -1)
+            if (bdrv_write(s->bdrv, i >> 9, iobuf, 1) < 0) {
                 printf("%s: write error in sector %" PRIu64 "\n",
                        __func__, i >> 9);
+            }
 
         page = i >> 9;
-        if (bdrv_read(s->bdrv, page, iobuf, 1) == -1)
+        if (bdrv_read(s->bdrv, page, iobuf, 1) < 0) {
             printf("%s: read error in sector %" PRIu64 "\n", __func__, page);
+        }
         memset(iobuf, 0xff, ((addr - 1) & 0x1ff) + 1);
-        if (bdrv_write(s->bdrv, page, iobuf, 1) == -1)
+        if (bdrv_write(s->bdrv, page, iobuf, 1) < 0) {
             printf("%s: write error in sector %" PRIu64 "\n", __func__, page);
+        }
     }
 }
 
@@ -740,18 +748,20 @@ static void glue(nand_blk_load_, PAGE_SIZE)(NANDFlashState *s,
 
     if (s->bdrv) {
         if (s->mem_oob) {
-            if (bdrv_read(s->bdrv, SECTOR(addr), s->io, PAGE_SECTORS) == -1)
+            if (bdrv_read(s->bdrv, SECTOR(addr), s->io, PAGE_SECTORS) < 0) {
                 printf("%s: read error in sector %" PRIu64 "\n",
                                 __func__, SECTOR(addr));
+            }
             memcpy(s->io + SECTOR_OFFSET(s->addr) + PAGE_SIZE,
                             s->storage + (PAGE(s->addr) << OOB_SHIFT),
                             OOB_SIZE);
             s->ioaddr = s->io + SECTOR_OFFSET(s->addr) + offset;
         } else {
             if (bdrv_read(s->bdrv, PAGE_START(addr) >> 9,
-                                    s->io, (PAGE_SECTORS + 2)) == -1)
+                                    s->io, (PAGE_SECTORS + 2)) < 0) {
                 printf("%s: read error in sector %" PRIu64 "\n",
                                 __func__, PAGE_START(addr) >> 9);
+            }
             s->ioaddr = s->io + (PAGE_START(addr) & 0x1ff) + offset;
         }
     } else {
diff --git a/hw/ne2000.c b/hw/ne2000.c
index 15605c478..d3dd9a6f2 100644
--- a/hw/ne2000.c
+++ b/hw/ne2000.c
@@ -652,7 +652,7 @@ static const VMStateDescription vmstate_pci_ne2000 = {
     }
 };
 
-static uint64_t ne2000_read(void *opaque, target_phys_addr_t addr,
+static uint64_t ne2000_read(void *opaque, hwaddr addr,
                             unsigned size)
 {
     NE2000State *s = opaque;
@@ -671,7 +671,7 @@ static uint64_t ne2000_read(void *opaque, target_phys_addr_t addr,
     return ((uint64_t)1 << (size * 8)) - 1;
 }
 
-static void ne2000_write(void *opaque, target_phys_addr_t addr,
+static void ne2000_write(void *opaque, hwaddr addr,
                          uint64_t data, unsigned size)
 {
     NE2000State *s = opaque;
diff --git a/hw/nseries.c b/hw/nseries.c
index 4df267032..2de8d21f0 100644
--- a/hw/nseries.c
+++ b/hw/nseries.c
@@ -189,6 +189,17 @@ static void n8x0_nand_setup(struct n800_s *s)
     /* XXX: in theory should also update the OOB for both pages */
 }
 
+static qemu_irq n8x0_system_powerdown;
+
+static void n8x0_powerdown_req(Notifier *n, void *opaque)
+{
+    qemu_irq_raise(n8x0_system_powerdown);
+}
+
+static Notifier n8x0_system_powerdown_notifier = {
+    .notify = n8x0_powerdown_req
+};
+
 static void n8x0_i2c_setup(struct n800_s *s)
 {
     DeviceState *dev;
@@ -201,7 +212,8 @@ static void n8x0_i2c_setup(struct n800_s *s)
                           qdev_get_gpio_in(s->mpu->ih[0],
                                            OMAP_INT_24XX_SYS_NIRQ));
 
-    qemu_system_powerdown = qdev_get_gpio_in(dev, 3);
+    n8x0_system_powerdown = qdev_get_gpio_in(dev, 3);
+    qemu_register_powerdown_notifier(&n8x0_system_powerdown_notifier);
 
     /* Attach a TMP105 PM chip (A0 wired to ground) */
     dev = i2c_create_slave(i2c, "tmp105", N8X0_TMP105_ADDR);
@@ -1272,17 +1284,15 @@ static int n810_atag_setup(const struct arm_boot_info *info, void *p)
     return n8x0_atag_setup(p, 810);
 }
 
-static void n8x0_init(ram_addr_t ram_size, const char *boot_device,
-                const char *kernel_filename,
-                const char *kernel_cmdline, const char *initrd_filename,
-                const char *cpu_model, struct arm_boot_info *binfo, int model)
+static void n8x0_init(QEMUMachineInitArgs *args,
+                      struct arm_boot_info *binfo, int model)
 {
     MemoryRegion *sysmem = get_system_memory();
     struct n800_s *s = (struct n800_s *) g_malloc0(sizeof(*s));
     int sdram_size = binfo->ram_size;
     DisplayState *ds;
 
-    s->mpu = omap2420_mpu_init(sysmem, sdram_size, cpu_model);
+    s->mpu = omap2420_mpu_init(sysmem, sdram_size, args->cpu_model);
 
     /* Setup peripherals
      *
@@ -1322,20 +1332,22 @@ static void n8x0_init(ram_addr_t ram_size, const char *boot_device,
     n8x0_dss_setup(s);
     n8x0_cbus_setup(s);
     n8x0_uart_setup(s);
-    if (usb_enabled)
+    if (usb_enabled(false)) {
         n8x0_usb_setup(s);
+    }
 
-    if (kernel_filename) {
+    if (args->kernel_filename) {
         /* Or at the linux loader.  */
-        binfo->kernel_filename = kernel_filename;
-        binfo->kernel_cmdline = kernel_cmdline;
-        binfo->initrd_filename = initrd_filename;
+        binfo->kernel_filename = args->kernel_filename;
+        binfo->kernel_cmdline = args->kernel_cmdline;
+        binfo->initrd_filename = args->initrd_filename;
         arm_load_kernel(s->mpu->cpu, binfo);
 
         qemu_register_reset(n8x0_boot_init, s);
     }
 
-    if (option_rom[0].name && (boot_device[0] == 'n' || !kernel_filename)) {
+    if (option_rom[0].name &&
+        (args->boot_device[0] == 'n' || !args->kernel_filename)) {
         int rom_size;
         uint8_t nolo_tags[0x10000];
         /* No, wait, better start at the ROM.  */
@@ -1363,7 +1375,7 @@ static void n8x0_init(ram_addr_t ram_size, const char *boot_device,
        size until the guest activates the display.  */
     ds = get_displaystate();
     ds->surface = qemu_resize_displaysurface(ds, 800, 480);
-    dpy_resize(ds);
+    dpy_gfx_resize(ds);
 }
 
 static struct arm_boot_info n800_binfo = {
@@ -1385,24 +1397,14 @@ static struct arm_boot_info n810_binfo = {
     .atag_board = n810_atag_setup,
 };
 
-static void n800_init(ram_addr_t ram_size,
-                const char *boot_device,
-                const char *kernel_filename, const char *kernel_cmdline,
-                const char *initrd_filename, const char *cpu_model)
+static void n800_init(QEMUMachineInitArgs *args)
 {
-    return n8x0_init(ram_size, boot_device,
-                    kernel_filename, kernel_cmdline, initrd_filename,
-                    cpu_model, &n800_binfo, 800);
+    return n8x0_init(args, &n800_binfo, 800);
 }
 
-static void n810_init(ram_addr_t ram_size,
-                const char *boot_device,
-                const char *kernel_filename, const char *kernel_cmdline,
-                const char *initrd_filename, const char *cpu_model)
+static void n810_init(QEMUMachineInitArgs *args)
 {
-    return n8x0_init(ram_size, boot_device,
-                    kernel_filename, kernel_cmdline, initrd_filename,
-                    cpu_model, &n810_binfo, 810);
+    return n8x0_init(args, &n810_binfo, 810);
 }
 
 static QEMUMachine n800_machine = {
diff --git a/hw/null-machine.c b/hw/null-machine.c
index 69910d399..d813c089e 100644
--- a/hw/null-machine.c
+++ b/hw/null-machine.c
@@ -15,12 +15,7 @@
 #include "hw/hw.h"
 #include "hw/boards.h"
 
-static void machine_none_init(ram_addr_t ram_size,
-                              const char *boot_device,
-                              const char *kernel_filename,
-                              const char *kernel_cmdline,
-                              const char *initrd_filename,
-                              const char *cpu_model)
+static void machine_none_init(QEMUMachineInitArgs *args)
 {
 }
 
diff --git a/hw/nvram.h b/hw/nvram.h
index 8924da47d..59337faaa 100644
--- a/hw/nvram.h
+++ b/hw/nvram.h
@@ -10,17 +10,9 @@ typedef struct nvram_t {
     nvram_write_t write_fn;
 } nvram_t;
 
-void NVRAM_set_byte (nvram_t *nvram, uint32_t addr, uint8_t value);
-uint8_t NVRAM_get_byte (nvram_t *nvram, uint32_t addr);
-void NVRAM_set_word (nvram_t *nvram, uint32_t addr, uint16_t value);
-uint16_t NVRAM_get_word (nvram_t *nvram, uint32_t addr);
-void NVRAM_set_lword (nvram_t *nvram, uint32_t addr, uint32_t value);
 uint32_t NVRAM_get_lword (nvram_t *nvram, uint32_t addr);
-void NVRAM_set_string (nvram_t *nvram, uint32_t addr,
-                       const char *str, uint32_t max);
 int NVRAM_get_string (nvram_t *nvram, uint8_t *dst, uint16_t addr, int max);
-void NVRAM_set_crc (nvram_t *nvram, uint32_t addr,
-                    uint32_t start, uint32_t count);
+
 int PPC_NVRAM_set_params (nvram_t *nvram, uint16_t NVRAM_size,
                           const char *arch,
                           uint32_t RAM_size, int boot_device,
@@ -36,8 +28,7 @@ uint32_t m48t59_read (void *private, uint32_t addr);
 void m48t59_toggle_lock (void *private, int lock);
 M48t59State *m48t59_init_isa(ISABus *bus, uint32_t io_base, uint16_t size,
                              int type);
-M48t59State *m48t59_init(qemu_irq IRQ, target_phys_addr_t mem_base,
+M48t59State *m48t59_init(qemu_irq IRQ, hwaddr mem_base,
                          uint32_t io_base, uint16_t size, int type);
-void m48t59_set_addr (void *opaque, uint32_t addr);
 
 #endif /* !NVRAM_H */
diff --git a/hw/omap.h b/hw/omap.h
index 413851bc3..2b383ffc4 100644
--- a/hw/omap.h
+++ b/hw/omap.h
@@ -19,6 +19,7 @@
 #ifndef hw_omap_h
 #include "memory.h"
 # define hw_omap_h		"omap.h"
+#include "hw/irq.h"
 
 # define OMAP_EMIFS_BASE	0x00000000
 # define OMAP2_Q0_BASE		0x00000000
@@ -65,7 +66,7 @@ void omap_clk_reparent(omap_clk clk, omap_clk parent);
 /* OMAP2 l4 Interconnect */
 struct omap_l4_s;
 struct omap_l4_region_s {
-    target_phys_addr_t offset;
+    hwaddr offset;
     size_t size;
     int access;
 };
@@ -80,13 +81,13 @@ struct omap_target_agent_s {
     struct omap_l4_s *bus;
     int regions;
     const struct omap_l4_region_s *start;
-    target_phys_addr_t base;
+    hwaddr base;
     uint32_t component;
     uint32_t control;
     uint32_t status;
 };
 struct omap_l4_s *omap_l4_init(MemoryRegion *address_space,
-                               target_phys_addr_t base, int ta_num);
+                               hwaddr base, int ta_num);
 
 struct omap_target_agent_s;
 struct omap_target_agent_s *omap_l4ta_get(
@@ -94,23 +95,23 @@ struct omap_target_agent_s *omap_l4ta_get(
     const struct omap_l4_region_s *regions,
     const struct omap_l4_agent_info_s *agents,
     int cs);
-target_phys_addr_t omap_l4_attach(struct omap_target_agent_s *ta,
+hwaddr omap_l4_attach(struct omap_target_agent_s *ta,
                                          int region, MemoryRegion *mr);
-target_phys_addr_t omap_l4_region_base(struct omap_target_agent_s *ta,
+hwaddr omap_l4_region_base(struct omap_target_agent_s *ta,
                                        int region);
-target_phys_addr_t omap_l4_region_size(struct omap_target_agent_s *ta,
+hwaddr omap_l4_region_size(struct omap_target_agent_s *ta,
                                        int region);
 
 /* OMAP2 SDRAM controller */
 struct omap_sdrc_s;
 struct omap_sdrc_s *omap_sdrc_init(MemoryRegion *sysmem,
-                                   target_phys_addr_t base);
+                                   hwaddr base);
 void omap_sdrc_reset(struct omap_sdrc_s *s);
 
 /* OMAP2 general purpose memory controller */
 struct omap_gpmc_s;
 struct omap_gpmc_s *omap_gpmc_init(struct omap_mpu_state_s *mpu,
-                                   target_phys_addr_t base,
+                                   hwaddr base,
                                    qemu_irq irq, qemu_irq drq);
 void omap_gpmc_reset(struct omap_gpmc_s *s);
 void omap_gpmc_attach(struct omap_gpmc_s *s, int cs, MemoryRegion *iomem);
@@ -433,11 +434,11 @@ enum omap_dma_model {
 };
 
 struct soc_dma_s;
-struct soc_dma_s *omap_dma_init(target_phys_addr_t base, qemu_irq *irqs,
+struct soc_dma_s *omap_dma_init(hwaddr base, qemu_irq *irqs,
                 MemoryRegion *sysmem,
                 qemu_irq lcd_irq, struct omap_mpu_state_s *mpu, omap_clk clk,
                 enum omap_dma_model model);
-struct soc_dma_s *omap_dma4_init(target_phys_addr_t base, qemu_irq *irqs,
+struct soc_dma_s *omap_dma4_init(hwaddr base, qemu_irq *irqs,
                 MemoryRegion *sysmem,
                 struct omap_mpu_state_s *mpu, int fifo,
                 int chans, omap_clk iclk, omap_clk fclk);
@@ -469,10 +470,10 @@ typedef enum {
 /* Only used in OMAP DMA 3.x gigacells */
 struct omap_dma_lcd_channel_s {
     enum omap_dma_port src;
-    target_phys_addr_t src_f1_top;
-    target_phys_addr_t src_f1_bottom;
-    target_phys_addr_t src_f2_top;
-    target_phys_addr_t src_f2_bottom;
+    hwaddr src_f1_top;
+    hwaddr src_f1_bottom;
+    hwaddr src_f2_top;
+    hwaddr src_f2_bottom;
 
     /* Used in OMAP DMA 3.2 gigacell */
     unsigned char brust_f1;
@@ -508,7 +509,7 @@ struct omap_dma_lcd_channel_s {
     int dual;
 
     int current_frame;
-    target_phys_addr_t phys_framebuffer[2];
+    hwaddr phys_framebuffer[2];
     qemu_irq irq;
     struct omap_mpu_state_s *mpu;
 } *omap_dma_get_lcdch(struct soc_dma_s *s);
@@ -659,7 +660,7 @@ struct omap_synctimer_s *omap_synctimer_init(struct omap_target_agent_s *ta,
 void omap_synctimer_reset(struct omap_synctimer_s *s);
 
 struct omap_uart_s;
-struct omap_uart_s *omap_uart_init(target_phys_addr_t base,
+struct omap_uart_s *omap_uart_init(hwaddr base,
                 qemu_irq irq, omap_clk fclk, omap_clk iclk,
                 qemu_irq txdma, qemu_irq rxdma,
                 const char *label, CharDriverState *chr);
@@ -728,7 +729,7 @@ void omap_tap_init(struct omap_target_agent_s *ta,
 struct omap_lcd_panel_s;
 void omap_lcdc_reset(struct omap_lcd_panel_s *s);
 struct omap_lcd_panel_s *omap_lcdc_init(MemoryRegion *sysmem,
-                                        target_phys_addr_t base,
+                                        hwaddr base,
                                         qemu_irq irq,
                                         struct omap_dma_lcd_channel_s *dma,
                                         omap_clk clk);
@@ -744,7 +745,7 @@ struct omap_dss_s;
 void omap_dss_reset(struct omap_dss_s *s);
 struct omap_dss_s *omap_dss_init(struct omap_target_agent_s *ta,
                 MemoryRegion *sysmem,
-                target_phys_addr_t l3_base,
+                hwaddr l3_base,
                 qemu_irq irq, qemu_irq drq,
                 omap_clk fck1, omap_clk fck2, omap_clk ck54m,
                 omap_clk ick1, omap_clk ick2);
@@ -752,7 +753,7 @@ void omap_rfbi_attach(struct omap_dss_s *s, int cs, struct rfbi_chip_s *chip);
 
 /* omap_mmc.c */
 struct omap_mmc_s;
-struct omap_mmc_s *omap_mmc_init(target_phys_addr_t base,
+struct omap_mmc_s *omap_mmc_init(hwaddr base,
                 MemoryRegion *sysmem,
                 BlockDriverState *bd,
                 qemu_irq irq, qemu_irq dma[], omap_clk clk);
@@ -829,11 +830,11 @@ struct omap_mpu_state_s {
 
     struct omap_dma_port_if_s {
         uint32_t (*read[3])(struct omap_mpu_state_s *s,
-                        target_phys_addr_t offset);
+                        hwaddr offset);
         void (*write[3])(struct omap_mpu_state_s *s,
-                        target_phys_addr_t offset, uint32_t value);
+                        hwaddr offset, uint32_t value);
         int (*addr_valid)(struct omap_mpu_state_s *s,
-                        target_phys_addr_t addr);
+                        hwaddr addr);
     } port[__omap_dma_port_last];
 
     unsigned long sdram_size;
@@ -942,16 +943,16 @@ struct omap_mpu_state_s *omap2420_mpu_init(MemoryRegion *sysmem,
                 unsigned long sdram_size,
                 const char *core);
 
-#define OMAP_FMT_plx "%#08" TARGET_PRIxPHYS
+#define OMAP_FMT_plx "%#08" HWADDR_PRIx
 
-uint32_t omap_badwidth_read8(void *opaque, target_phys_addr_t addr);
-void omap_badwidth_write8(void *opaque, target_phys_addr_t addr,
+uint32_t omap_badwidth_read8(void *opaque, hwaddr addr);
+void omap_badwidth_write8(void *opaque, hwaddr addr,
                 uint32_t value);
-uint32_t omap_badwidth_read16(void *opaque, target_phys_addr_t addr);
-void omap_badwidth_write16(void *opaque, target_phys_addr_t addr,
+uint32_t omap_badwidth_read16(void *opaque, hwaddr addr);
+void omap_badwidth_write16(void *opaque, hwaddr addr,
                 uint32_t value);
-uint32_t omap_badwidth_read32(void *opaque, target_phys_addr_t addr);
-void omap_badwidth_write32(void *opaque, target_phys_addr_t addr,
+uint32_t omap_badwidth_read32(void *opaque, hwaddr addr);
+void omap_badwidth_write32(void *opaque, hwaddr addr,
                 uint32_t value);
 
 void omap_mpu_wakeup(void *opaque, int irq, int req);
diff --git a/hw/omap1.c b/hw/omap1.c
index ad60cc491..4d5815eb0 100644
--- a/hw/omap1.c
+++ b/hw/omap1.c
@@ -26,7 +26,7 @@
 #include "sysbus.h"
 
 /* Should signal the TCMI/GPMC */
-uint32_t omap_badwidth_read8(void *opaque, target_phys_addr_t addr)
+uint32_t omap_badwidth_read8(void *opaque, hwaddr addr)
 {
     uint8_t ret;
 
@@ -35,7 +35,7 @@ uint32_t omap_badwidth_read8(void *opaque, target_phys_addr_t addr)
     return ret;
 }
 
-void omap_badwidth_write8(void *opaque, target_phys_addr_t addr,
+void omap_badwidth_write8(void *opaque, hwaddr addr,
                 uint32_t value)
 {
     uint8_t val8 = value;
@@ -44,7 +44,7 @@ void omap_badwidth_write8(void *opaque, target_phys_addr_t addr,
     cpu_physical_memory_write(addr, (void *) &val8, 1);
 }
 
-uint32_t omap_badwidth_read16(void *opaque, target_phys_addr_t addr)
+uint32_t omap_badwidth_read16(void *opaque, hwaddr addr)
 {
     uint16_t ret;
 
@@ -53,7 +53,7 @@ uint32_t omap_badwidth_read16(void *opaque, target_phys_addr_t addr)
     return ret;
 }
 
-void omap_badwidth_write16(void *opaque, target_phys_addr_t addr,
+void omap_badwidth_write16(void *opaque, hwaddr addr,
                 uint32_t value)
 {
     uint16_t val16 = value;
@@ -62,7 +62,7 @@ void omap_badwidth_write16(void *opaque, target_phys_addr_t addr,
     cpu_physical_memory_write(addr, (void *) &val16, 2);
 }
 
-uint32_t omap_badwidth_read32(void *opaque, target_phys_addr_t addr)
+uint32_t omap_badwidth_read32(void *opaque, hwaddr addr)
 {
     uint32_t ret;
 
@@ -71,7 +71,7 @@ uint32_t omap_badwidth_read32(void *opaque, target_phys_addr_t addr)
     return ret;
 }
 
-void omap_badwidth_write32(void *opaque, target_phys_addr_t addr,
+void omap_badwidth_write32(void *opaque, hwaddr addr,
                 uint32_t value)
 {
     OMAP_32B_REG(addr);
@@ -176,7 +176,7 @@ static void omap_timer_clk_setup(struct omap_mpu_timer_s *timer)
     timer->rate = omap_clk_getrate(timer->clk);
 }
 
-static uint64_t omap_mpu_timer_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_mpu_timer_read(void *opaque, hwaddr addr,
                                     unsigned size)
 {
     struct omap_mpu_timer_s *s = (struct omap_mpu_timer_s *) opaque;
@@ -200,7 +200,7 @@ static uint64_t omap_mpu_timer_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void omap_mpu_timer_write(void *opaque, target_phys_addr_t addr,
+static void omap_mpu_timer_write(void *opaque, hwaddr addr,
                                  uint64_t value, unsigned size)
 {
     struct omap_mpu_timer_s *s = (struct omap_mpu_timer_s *) opaque;
@@ -251,7 +251,7 @@ static void omap_mpu_timer_reset(struct omap_mpu_timer_s *s)
 }
 
 static struct omap_mpu_timer_s *omap_mpu_timer_init(MemoryRegion *system_memory,
-                target_phys_addr_t base,
+                hwaddr base,
                 qemu_irq irq, omap_clk clk)
 {
     struct omap_mpu_timer_s *s = (struct omap_mpu_timer_s *)
@@ -282,7 +282,7 @@ struct omap_watchdog_timer_s {
     int reset;
 };
 
-static uint64_t omap_wd_timer_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_wd_timer_read(void *opaque, hwaddr addr,
                                    unsigned size)
 {
     struct omap_watchdog_timer_s *s = (struct omap_watchdog_timer_s *) opaque;
@@ -307,7 +307,7 @@ static uint64_t omap_wd_timer_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void omap_wd_timer_write(void *opaque, target_phys_addr_t addr,
+static void omap_wd_timer_write(void *opaque, hwaddr addr,
                                 uint64_t value, unsigned size)
 {
     struct omap_watchdog_timer_s *s = (struct omap_watchdog_timer_s *) opaque;
@@ -380,7 +380,7 @@ static void omap_wd_timer_reset(struct omap_watchdog_timer_s *s)
 }
 
 static struct omap_watchdog_timer_s *omap_wd_timer_init(MemoryRegion *memory,
-                target_phys_addr_t base,
+                hwaddr base,
                 qemu_irq irq, omap_clk clk)
 {
     struct omap_watchdog_timer_s *s = (struct omap_watchdog_timer_s *)
@@ -405,7 +405,7 @@ struct omap_32khz_timer_s {
     MemoryRegion iomem;
 };
 
-static uint64_t omap_os_timer_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_os_timer_read(void *opaque, hwaddr addr,
                                    unsigned size)
 {
     struct omap_32khz_timer_s *s = (struct omap_32khz_timer_s *) opaque;
@@ -432,7 +432,7 @@ static uint64_t omap_os_timer_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void omap_os_timer_write(void *opaque, target_phys_addr_t addr,
+static void omap_os_timer_write(void *opaque, hwaddr addr,
                                 uint64_t value, unsigned size)
 {
     struct omap_32khz_timer_s *s = (struct omap_32khz_timer_s *) opaque;
@@ -486,7 +486,7 @@ static void omap_os_timer_reset(struct omap_32khz_timer_s *s)
 }
 
 static struct omap_32khz_timer_s *omap_os_timer_init(MemoryRegion *memory,
-                target_phys_addr_t base,
+                hwaddr base,
                 qemu_irq irq, omap_clk clk)
 {
     struct omap_32khz_timer_s *s = (struct omap_32khz_timer_s *)
@@ -506,7 +506,7 @@ static struct omap_32khz_timer_s *omap_os_timer_init(MemoryRegion *memory,
 }
 
 /* Ultra Low-Power Device Module */
-static uint64_t omap_ulpd_pm_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_ulpd_pm_read(void *opaque, hwaddr addr,
                                   unsigned size)
 {
     struct omap_mpu_state_s *s = (struct omap_mpu_state_s *) opaque;
@@ -573,7 +573,7 @@ static inline void omap_ulpd_req_update(struct omap_mpu_state_s *s,
         omap_clk_canidle(omap_findclk(s, "usb_clk0"), (~value >> 3) & 1);
 }
 
-static void omap_ulpd_pm_write(void *opaque, target_phys_addr_t addr,
+static void omap_ulpd_pm_write(void *opaque, hwaddr addr,
                                uint64_t value, unsigned size)
 {
     struct omap_mpu_state_s *s = (struct omap_mpu_state_s *) opaque;
@@ -726,7 +726,7 @@ static void omap_ulpd_pm_reset(struct omap_mpu_state_s *mpu)
 }
 
 static void omap_ulpd_pm_init(MemoryRegion *system_memory,
-                target_phys_addr_t base,
+                hwaddr base,
                 struct omap_mpu_state_s *mpu)
 {
     memory_region_init_io(&mpu->ulpd_pm_iomem, &omap_ulpd_pm_ops, mpu,
@@ -736,7 +736,7 @@ static void omap_ulpd_pm_init(MemoryRegion *system_memory,
 }
 
 /* OMAP Pin Configuration */
-static uint64_t omap_pin_cfg_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_pin_cfg_read(void *opaque, hwaddr addr,
                                   unsigned size)
 {
     struct omap_mpu_state_s *s = (struct omap_mpu_state_s *) opaque;
@@ -843,7 +843,7 @@ static inline void omap_pin_modconf1_update(struct omap_mpu_state_s *s,
          omap_clk_onoff(omap_findclk(s, "usb_hhc_ck"), (value >> 9) & 1);
 }
 
-static void omap_pin_cfg_write(void *opaque, target_phys_addr_t addr,
+static void omap_pin_cfg_write(void *opaque, hwaddr addr,
                                uint64_t value, unsigned size)
 {
     struct omap_mpu_state_s *s = (struct omap_mpu_state_s *) opaque;
@@ -944,7 +944,7 @@ static void omap_pin_cfg_reset(struct omap_mpu_state_s *mpu)
 }
 
 static void omap_pin_cfg_init(MemoryRegion *system_memory,
-                target_phys_addr_t base,
+                hwaddr base,
                 struct omap_mpu_state_s *mpu)
 {
     memory_region_init_io(&mpu->pin_cfg_iomem, &omap_pin_cfg_ops, mpu,
@@ -954,7 +954,7 @@ static void omap_pin_cfg_init(MemoryRegion *system_memory,
 }
 
 /* Device Identification, Die Identification */
-static uint64_t omap_id_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_id_read(void *opaque, hwaddr addr,
                              unsigned size)
 {
     struct omap_mpu_state_s *s = (struct omap_mpu_state_s *) opaque;
@@ -1001,7 +1001,7 @@ static uint64_t omap_id_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void omap_id_write(void *opaque, target_phys_addr_t addr,
+static void omap_id_write(void *opaque, hwaddr addr,
                           uint64_t value, unsigned size)
 {
     if (size != 4) {
@@ -1035,7 +1035,7 @@ static void omap_id_init(MemoryRegion *memory, struct omap_mpu_state_s *mpu)
 }
 
 /* MPUI Control (Dummy) */
-static uint64_t omap_mpui_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_mpui_read(void *opaque, hwaddr addr,
                                unsigned size)
 {
     struct omap_mpu_state_s *s = (struct omap_mpu_state_s *) opaque;
@@ -1068,7 +1068,7 @@ static uint64_t omap_mpui_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void omap_mpui_write(void *opaque, target_phys_addr_t addr,
+static void omap_mpui_write(void *opaque, hwaddr addr,
                             uint64_t value, unsigned size)
 {
     struct omap_mpu_state_s *s = (struct omap_mpu_state_s *) opaque;
@@ -1109,7 +1109,7 @@ static void omap_mpui_reset(struct omap_mpu_state_s *s)
     s->mpui_ctrl = 0x0003ff1b;
 }
 
-static void omap_mpui_init(MemoryRegion *memory, target_phys_addr_t base,
+static void omap_mpui_init(MemoryRegion *memory, hwaddr base,
                 struct omap_mpu_state_s *mpu)
 {
     memory_region_init_io(&mpu->mpui_iomem, &omap_mpui_ops, mpu,
@@ -1131,7 +1131,7 @@ struct omap_tipb_bridge_s {
     uint16_t enh_control;
 };
 
-static uint64_t omap_tipb_bridge_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_tipb_bridge_read(void *opaque, hwaddr addr,
                                       unsigned size)
 {
     struct omap_tipb_bridge_s *s = (struct omap_tipb_bridge_s *) opaque;
@@ -1161,7 +1161,7 @@ static uint64_t omap_tipb_bridge_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void omap_tipb_bridge_write(void *opaque, target_phys_addr_t addr,
+static void omap_tipb_bridge_write(void *opaque, hwaddr addr,
                                    uint64_t value, unsigned size)
 {
     struct omap_tipb_bridge_s *s = (struct omap_tipb_bridge_s *) opaque;
@@ -1215,7 +1215,7 @@ static void omap_tipb_bridge_reset(struct omap_tipb_bridge_s *s)
 }
 
 static struct omap_tipb_bridge_s *omap_tipb_bridge_init(
-    MemoryRegion *memory, target_phys_addr_t base,
+    MemoryRegion *memory, hwaddr base,
     qemu_irq abort_irq, omap_clk clk)
 {
     struct omap_tipb_bridge_s *s = (struct omap_tipb_bridge_s *)
@@ -1232,7 +1232,7 @@ static struct omap_tipb_bridge_s *omap_tipb_bridge_init(
 }
 
 /* Dummy Traffic Controller's Memory Interface */
-static uint64_t omap_tcmi_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_tcmi_read(void *opaque, hwaddr addr,
                                unsigned size)
 {
     struct omap_mpu_state_s *s = (struct omap_mpu_state_s *) opaque;
@@ -1270,7 +1270,7 @@ static uint64_t omap_tcmi_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void omap_tcmi_write(void *opaque, target_phys_addr_t addr,
+static void omap_tcmi_write(void *opaque, hwaddr addr,
                             uint64_t value, unsigned size)
 {
     struct omap_mpu_state_s *s = (struct omap_mpu_state_s *) opaque;
@@ -1330,7 +1330,7 @@ static void omap_tcmi_reset(struct omap_mpu_state_s *mpu)
     mpu->tcmi_regs[0x40 >> 2] = 0x00000000;
 }
 
-static void omap_tcmi_init(MemoryRegion *memory, target_phys_addr_t base,
+static void omap_tcmi_init(MemoryRegion *memory, hwaddr base,
                 struct omap_mpu_state_s *mpu)
 {
     memory_region_init_io(&mpu->tcmi_iomem, &omap_tcmi_ops, mpu,
@@ -1346,7 +1346,7 @@ struct dpll_ctl_s {
     omap_clk dpll;
 };
 
-static uint64_t omap_dpll_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_dpll_read(void *opaque, hwaddr addr,
                                unsigned size)
 {
     struct dpll_ctl_s *s = (struct dpll_ctl_s *) opaque;
@@ -1362,7 +1362,7 @@ static uint64_t omap_dpll_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void omap_dpll_write(void *opaque, target_phys_addr_t addr,
+static void omap_dpll_write(void *opaque, hwaddr addr,
                             uint64_t value, unsigned size)
 {
     struct dpll_ctl_s *s = (struct dpll_ctl_s *) opaque;
@@ -1412,7 +1412,7 @@ static void omap_dpll_reset(struct dpll_ctl_s *s)
 }
 
 static struct dpll_ctl_s  *omap_dpll_init(MemoryRegion *memory,
-                           target_phys_addr_t base, omap_clk clk)
+                           hwaddr base, omap_clk clk)
 {
     struct dpll_ctl_s *s = g_malloc0(sizeof(*s));
     memory_region_init_io(&s->iomem, &omap_dpll_ops, s, "omap-dpll", 0x100);
@@ -1425,7 +1425,7 @@ static struct dpll_ctl_s  *omap_dpll_init(MemoryRegion *memory,
 }
 
 /* MPU Clock/Reset/Power Mode Control */
-static uint64_t omap_clkm_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_clkm_read(void *opaque, hwaddr addr,
                                unsigned size)
 {
     struct omap_mpu_state_s *s = (struct omap_mpu_state_s *) opaque;
@@ -1627,7 +1627,7 @@ static inline void omap_clkm_ckout1_update(struct omap_mpu_state_s *s,
     }
 }
 
-static void omap_clkm_write(void *opaque, target_phys_addr_t addr,
+static void omap_clkm_write(void *opaque, hwaddr addr,
                             uint64_t value, unsigned size)
 {
     struct omap_mpu_state_s *s = (struct omap_mpu_state_s *) opaque;
@@ -1714,7 +1714,7 @@ static const MemoryRegionOps omap_clkm_ops = {
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-static uint64_t omap_clkdsp_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_clkdsp_read(void *opaque, hwaddr addr,
                                  unsigned size)
 {
     struct omap_mpu_state_s *s = (struct omap_mpu_state_s *) opaque;
@@ -1758,7 +1758,7 @@ static inline void omap_clkdsp_idlect2_update(struct omap_mpu_state_s *s,
     SET_ONOFF("dspxor_ck", 1);				/* EN_XORPCK */
 }
 
-static void omap_clkdsp_write(void *opaque, target_phys_addr_t addr,
+static void omap_clkdsp_write(void *opaque, hwaddr addr,
                               uint64_t value, unsigned size)
 {
     struct omap_mpu_state_s *s = (struct omap_mpu_state_s *) opaque;
@@ -1823,8 +1823,8 @@ static void omap_clkm_reset(struct omap_mpu_state_s *s)
     s->clkm.dsp_rstct2 = 0x0000;
 }
 
-static void omap_clkm_init(MemoryRegion *memory, target_phys_addr_t mpu_base,
-                target_phys_addr_t dsp_base, struct omap_mpu_state_s *s)
+static void omap_clkm_init(MemoryRegion *memory, hwaddr mpu_base,
+                hwaddr dsp_base, struct omap_mpu_state_s *s)
 {
     memory_region_init_io(&s->clkm_iomem, &omap_clkm_ops, s,
                           "omap-clkm", 0x100);
@@ -1903,7 +1903,7 @@ static void omap_mpuio_kbd_update(struct omap_mpuio_s *s)
     s->row_latch = ~rows;
 }
 
-static uint64_t omap_mpuio_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_mpuio_read(void *opaque, hwaddr addr,
                                 unsigned size)
 {
     struct omap_mpuio_s *s = (struct omap_mpuio_s *) opaque;
@@ -1963,7 +1963,7 @@ static uint64_t omap_mpuio_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void omap_mpuio_write(void *opaque, target_phys_addr_t addr,
+static void omap_mpuio_write(void *opaque, hwaddr addr,
                              uint64_t value, unsigned size)
 {
     struct omap_mpuio_s *s = (struct omap_mpuio_s *) opaque;
@@ -2072,7 +2072,7 @@ static void omap_mpuio_onoff(void *opaque, int line, int on)
 }
 
 static struct omap_mpuio_s *omap_mpuio_init(MemoryRegion *memory,
-                target_phys_addr_t base,
+                hwaddr base,
                 qemu_irq kbd_int, qemu_irq gpio_int, qemu_irq wakeup,
                 omap_clk clk)
 {
@@ -2159,7 +2159,7 @@ static void omap_uwire_transfer_start(struct omap_uwire_s *s)
     }
 }
 
-static uint64_t omap_uwire_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_uwire_read(void *opaque, hwaddr addr,
                                 unsigned size)
 {
     struct omap_uwire_s *s = (struct omap_uwire_s *) opaque;
@@ -2193,7 +2193,7 @@ static uint64_t omap_uwire_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void omap_uwire_write(void *opaque, target_phys_addr_t addr,
+static void omap_uwire_write(void *opaque, hwaddr addr,
                              uint64_t value, unsigned size)
 {
     struct omap_uwire_s *s = (struct omap_uwire_s *) opaque;
@@ -2263,7 +2263,7 @@ static void omap_uwire_reset(struct omap_uwire_s *s)
 }
 
 static struct omap_uwire_s *omap_uwire_init(MemoryRegion *system_memory,
-                                            target_phys_addr_t base,
+                                            hwaddr base,
                                             qemu_irq txirq, qemu_irq rxirq,
                                             qemu_irq dma,
                                             omap_clk clk)
@@ -2312,7 +2312,7 @@ static void omap_pwl_update(struct omap_pwl_s *s)
     }
 }
 
-static uint64_t omap_pwl_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_pwl_read(void *opaque, hwaddr addr,
                               unsigned size)
 {
     struct omap_pwl_s *s = (struct omap_pwl_s *) opaque;
@@ -2332,7 +2332,7 @@ static uint64_t omap_pwl_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void omap_pwl_write(void *opaque, target_phys_addr_t addr,
+static void omap_pwl_write(void *opaque, hwaddr addr,
                            uint64_t value, unsigned size)
 {
     struct omap_pwl_s *s = (struct omap_pwl_s *) opaque;
@@ -2381,7 +2381,7 @@ static void omap_pwl_clk_update(void *opaque, int line, int on)
 }
 
 static struct omap_pwl_s *omap_pwl_init(MemoryRegion *system_memory,
-                                        target_phys_addr_t base,
+                                        hwaddr base,
                                         omap_clk clk)
 {
     struct omap_pwl_s *s = g_malloc0(sizeof(*s));
@@ -2405,7 +2405,7 @@ struct omap_pwt_s {
     omap_clk clk;
 };
 
-static uint64_t omap_pwt_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_pwt_read(void *opaque, hwaddr addr,
                               unsigned size)
 {
     struct omap_pwt_s *s = (struct omap_pwt_s *) opaque;
@@ -2427,7 +2427,7 @@ static uint64_t omap_pwt_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void omap_pwt_write(void *opaque, target_phys_addr_t addr,
+static void omap_pwt_write(void *opaque, hwaddr addr,
                            uint64_t value, unsigned size)
 {
     struct omap_pwt_s *s = (struct omap_pwt_s *) opaque;
@@ -2488,7 +2488,7 @@ static void omap_pwt_reset(struct omap_pwt_s *s)
 }
 
 static struct omap_pwt_s *omap_pwt_init(MemoryRegion *system_memory,
-                                        target_phys_addr_t base,
+                                        hwaddr base,
                                         omap_clk clk)
 {
     struct omap_pwt_s *s = g_malloc0(sizeof(*s));
@@ -2536,7 +2536,7 @@ static void omap_rtc_alarm_update(struct omap_rtc_s *s)
         printf("%s: conversion failed\n", __FUNCTION__);
 }
 
-static uint64_t omap_rtc_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_rtc_read(void *opaque, hwaddr addr,
                               unsigned size)
 {
     struct omap_rtc_s *s = (struct omap_rtc_s *) opaque;
@@ -2618,7 +2618,7 @@ static uint64_t omap_rtc_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void omap_rtc_write(void *opaque, target_phys_addr_t addr,
+static void omap_rtc_write(void *opaque, hwaddr addr,
                            uint64_t value, unsigned size)
 {
     struct omap_rtc_s *s = (struct omap_rtc_s *) opaque;
@@ -2901,7 +2901,7 @@ static void omap_rtc_reset(struct omap_rtc_s *s)
 }
 
 static struct omap_rtc_s *omap_rtc_init(MemoryRegion *system_memory,
-                                        target_phys_addr_t base,
+                                        hwaddr base,
                                         qemu_irq timerirq, qemu_irq alarmirq,
                                         omap_clk clk)
 {
@@ -3129,7 +3129,7 @@ static void omap_mcbsp_req_update(struct omap_mcbsp_s *s)
         omap_mcbsp_rx_stop(s);
 }
 
-static uint64_t omap_mcbsp_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_mcbsp_read(void *opaque, hwaddr addr,
                                 unsigned size)
 {
     struct omap_mcbsp_s *s = (struct omap_mcbsp_s *) opaque;
@@ -3227,7 +3227,7 @@ static uint64_t omap_mcbsp_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void omap_mcbsp_writeh(void *opaque, target_phys_addr_t addr,
+static void omap_mcbsp_writeh(void *opaque, hwaddr addr,
                 uint32_t value)
 {
     struct omap_mcbsp_s *s = (struct omap_mcbsp_s *) opaque;
@@ -3365,7 +3365,7 @@ static void omap_mcbsp_writeh(void *opaque, target_phys_addr_t addr,
     OMAP_BAD_REG(addr);
 }
 
-static void omap_mcbsp_writew(void *opaque, target_phys_addr_t addr,
+static void omap_mcbsp_writew(void *opaque, hwaddr addr,
                 uint32_t value)
 {
     struct omap_mcbsp_s *s = (struct omap_mcbsp_s *) opaque;
@@ -3396,7 +3396,7 @@ static void omap_mcbsp_writew(void *opaque, target_phys_addr_t addr,
     omap_badwidth_write16(opaque, addr, value);
 }
 
-static void omap_mcbsp_write(void *opaque, target_phys_addr_t addr,
+static void omap_mcbsp_write(void *opaque, hwaddr addr,
                              uint64_t value, unsigned size)
 {
     switch (size) {
@@ -3432,7 +3432,7 @@ static void omap_mcbsp_reset(struct omap_mcbsp_s *s)
 }
 
 static struct omap_mcbsp_s *omap_mcbsp_init(MemoryRegion *system_memory,
-                                            target_phys_addr_t base,
+                                            hwaddr base,
                                             qemu_irq txirq, qemu_irq rxirq,
                                             qemu_irq *dma, omap_clk clk)
 {
@@ -3547,7 +3547,7 @@ static void omap_lpg_reset(struct omap_lpg_s *s)
     omap_lpg_update(s);
 }
 
-static uint64_t omap_lpg_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_lpg_read(void *opaque, hwaddr addr,
                               unsigned size)
 {
     struct omap_lpg_s *s = (struct omap_lpg_s *) opaque;
@@ -3569,7 +3569,7 @@ static uint64_t omap_lpg_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void omap_lpg_write(void *opaque, target_phys_addr_t addr,
+static void omap_lpg_write(void *opaque, hwaddr addr,
                            uint64_t value, unsigned size)
 {
     struct omap_lpg_s *s = (struct omap_lpg_s *) opaque;
@@ -3613,7 +3613,7 @@ static void omap_lpg_clk_update(void *opaque, int line, int on)
 }
 
 static struct omap_lpg_s *omap_lpg_init(MemoryRegion *system_memory,
-                                        target_phys_addr_t base, omap_clk clk)
+                                        hwaddr base, omap_clk clk)
 {
     struct omap_lpg_s *s = (struct omap_lpg_s *)
             g_malloc0(sizeof(struct omap_lpg_s));
@@ -3631,7 +3631,7 @@ static struct omap_lpg_s *omap_lpg_init(MemoryRegion *system_memory,
 }
 
 /* MPUI Peripheral Bridge configuration */
-static uint64_t omap_mpui_io_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_mpui_io_read(void *opaque, hwaddr addr,
                                   unsigned size)
 {
     if (size != 2) {
@@ -3645,7 +3645,7 @@ static uint64_t omap_mpui_io_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void omap_mpui_io_write(void *opaque, target_phys_addr_t addr,
+static void omap_mpui_io_write(void *opaque, hwaddr addr,
                                uint64_t value, unsigned size)
 {
     /* FIXME: infinite loop */
@@ -3706,8 +3706,8 @@ static void omap1_mpu_reset(void *opaque)
 }
 
 static const struct omap_map_s {
-    target_phys_addr_t phys_dsp;
-    target_phys_addr_t phys_mpu;
+    hwaddr phys_dsp;
+    hwaddr phys_mpu;
     uint32_t size;
     const char *name;
 } omap15xx_dsp_mm[] = {
@@ -3778,38 +3778,38 @@ static const struct dma_irq_map omap1_dma_irq_map[] = {
 
 /* DMA ports for OMAP1 */
 static int omap_validate_emiff_addr(struct omap_mpu_state_s *s,
-                target_phys_addr_t addr)
+                hwaddr addr)
 {
     return range_covers_byte(OMAP_EMIFF_BASE, s->sdram_size, addr);
 }
 
 static int omap_validate_emifs_addr(struct omap_mpu_state_s *s,
-                target_phys_addr_t addr)
+                hwaddr addr)
 {
     return range_covers_byte(OMAP_EMIFS_BASE, OMAP_EMIFF_BASE - OMAP_EMIFS_BASE,
                              addr);
 }
 
 static int omap_validate_imif_addr(struct omap_mpu_state_s *s,
-                target_phys_addr_t addr)
+                hwaddr addr)
 {
     return range_covers_byte(OMAP_IMIF_BASE, s->sram_size, addr);
 }
 
 static int omap_validate_tipb_addr(struct omap_mpu_state_s *s,
-                target_phys_addr_t addr)
+                hwaddr addr)
 {
     return range_covers_byte(0xfffb0000, 0xffff0000 - 0xfffb0000, addr);
 }
 
 static int omap_validate_local_addr(struct omap_mpu_state_s *s,
-                target_phys_addr_t addr)
+                hwaddr addr)
 {
     return range_covers_byte(OMAP_LOCALBUS_BASE, 0x1000000, addr);
 }
 
 static int omap_validate_tipb_mpui_addr(struct omap_mpu_state_s *s,
-                target_phys_addr_t addr)
+                hwaddr addr)
 {
     return range_covers_byte(0xe1010000, 0xe1020004 - 0xe1010000, addr);
 }
diff --git a/hw/omap2.c b/hw/omap2.c
index 4278dd19c..96aba7105 100644
--- a/hw/omap2.c
+++ b/hw/omap2.c
@@ -324,7 +324,7 @@ static void omap_eac_reset(struct omap_eac_s *s)
     omap_eac_interrupt_update(s);
 }
 
-static uint64_t omap_eac_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_eac_read(void *opaque, hwaddr addr,
                               unsigned size)
 {
     struct omap_eac_s *s = (struct omap_eac_s *) opaque;
@@ -440,7 +440,7 @@ static uint64_t omap_eac_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void omap_eac_write(void *opaque, target_phys_addr_t addr,
+static void omap_eac_write(void *opaque, hwaddr addr,
                            uint64_t value, unsigned size)
 {
     struct omap_eac_s *s = (struct omap_eac_s *) opaque;
@@ -644,7 +644,7 @@ static void omap_sti_reset(struct omap_sti_s *s)
     omap_sti_interrupt_update(s);
 }
 
-static uint64_t omap_sti_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_sti_read(void *opaque, hwaddr addr,
                               unsigned size)
 {
     struct omap_sti_s *s = (struct omap_sti_s *) opaque;
@@ -685,7 +685,7 @@ static uint64_t omap_sti_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void omap_sti_write(void *opaque, target_phys_addr_t addr,
+static void omap_sti_write(void *opaque, hwaddr addr,
                            uint64_t value, unsigned size)
 {
     struct omap_sti_s *s = (struct omap_sti_s *) opaque;
@@ -741,14 +741,14 @@ static const MemoryRegionOps omap_sti_ops = {
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-static uint64_t omap_sti_fifo_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_sti_fifo_read(void *opaque, hwaddr addr,
                                    unsigned size)
 {
     OMAP_BAD_REG(addr);
     return 0;
 }
 
-static void omap_sti_fifo_write(void *opaque, target_phys_addr_t addr,
+static void omap_sti_fifo_write(void *opaque, hwaddr addr,
                                 uint64_t value, unsigned size)
 {
     struct omap_sti_s *s = (struct omap_sti_s *) opaque;
@@ -780,7 +780,7 @@ static const MemoryRegionOps omap_sti_fifo_ops = {
 
 static struct omap_sti_s *omap_sti_init(struct omap_target_agent_s *ta,
                 MemoryRegion *sysmem,
-                target_phys_addr_t channel_base, qemu_irq irq, omap_clk clk,
+                hwaddr channel_base, qemu_irq irq, omap_clk clk,
                 CharDriverState *chr)
 {
     struct omap_sti_s *s = (struct omap_sti_s *)
@@ -1040,7 +1040,7 @@ static void omap_prcm_int_update(struct omap_prcm_s *s, int dom)
     /* XXX or is the mask applied before PRCM_IRQSTATUS_* ? */
 }
 
-static uint64_t omap_prcm_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_prcm_read(void *opaque, hwaddr addr,
                                unsigned size)
 {
     struct omap_prcm_s *s = (struct omap_prcm_s *) opaque;
@@ -1352,7 +1352,7 @@ static void omap_prcm_dpll_update(struct omap_prcm_s *s)
     }
 }
 
-static void omap_prcm_write(void *opaque, target_phys_addr_t addr,
+static void omap_prcm_write(void *opaque, hwaddr addr,
                             uint64_t value, unsigned size)
 {
     struct omap_prcm_s *s = (struct omap_prcm_s *) opaque;
@@ -1832,7 +1832,7 @@ struct omap_sysctl_s {
     uint32_t msuspendmux[5];
 };
 
-static uint32_t omap_sysctl_read8(void *opaque, target_phys_addr_t addr)
+static uint32_t omap_sysctl_read8(void *opaque, hwaddr addr)
 {
 
     struct omap_sysctl_s *s = (struct omap_sysctl_s *) opaque;
@@ -1857,7 +1857,7 @@ static uint32_t omap_sysctl_read8(void *opaque, target_phys_addr_t addr)
     return 0;
 }
 
-static uint32_t omap_sysctl_read(void *opaque, target_phys_addr_t addr)
+static uint32_t omap_sysctl_read(void *opaque, hwaddr addr)
 {
     struct omap_sysctl_s *s = (struct omap_sysctl_s *) opaque;
 
@@ -1957,7 +1957,7 @@ static uint32_t omap_sysctl_read(void *opaque, target_phys_addr_t addr)
     return 0;
 }
 
-static void omap_sysctl_write8(void *opaque, target_phys_addr_t addr,
+static void omap_sysctl_write8(void *opaque, hwaddr addr,
                 uint32_t value)
 {
     struct omap_sysctl_s *s = (struct omap_sysctl_s *) opaque;
@@ -1981,7 +1981,7 @@ static void omap_sysctl_write8(void *opaque, target_phys_addr_t addr,
     }
 }
 
-static void omap_sysctl_write(void *opaque, target_phys_addr_t addr,
+static void omap_sysctl_write(void *opaque, hwaddr addr,
                 uint32_t value)
 {
     struct omap_sysctl_s *s = (struct omap_sysctl_s *) opaque;
@@ -2226,7 +2226,7 @@ static void omap2_mpu_reset(void *opaque)
 }
 
 static int omap2_validate_addr(struct omap_mpu_state_s *s,
-                target_phys_addr_t addr)
+                hwaddr addr)
 {
     return 1;
 }
diff --git a/hw/omap_dma.c b/hw/omap_dma.c
index 389cb78de..e619c7b7d 100644
--- a/hw/omap_dma.c
+++ b/hw/omap_dma.c
@@ -31,7 +31,7 @@ struct omap_dma_channel_s {
     int endian_lock[2];
     int translate[2];
     enum omap_dma_port port[2];
-    target_phys_addr_t addr[2];
+    hwaddr addr[2];
     omap_dma_addressing_t mode[2];
     uint32_t elements;
     uint16_t frames;
@@ -78,7 +78,7 @@ struct omap_dma_channel_s {
     struct omap_dma_channel_s *sibling;
 
     struct omap_dma_reg_set_s {
-        target_phys_addr_t src, dest;
+        hwaddr src, dest;
         int frame;
         int element;
         int pck_element;
@@ -914,7 +914,7 @@ static int omap_dma_ch_reg_write(struct omap_dma_s *s,
         break;
 
     case 0x06:	/* SYS_DMA_CSR_CH0 */
-        OMAP_RO_REG((target_phys_addr_t) reg);
+        OMAP_RO_REG((hwaddr) reg);
         break;
 
     case 0x08:	/* SYS_DMA_CSSA_L_CH0 */
@@ -954,7 +954,7 @@ static int omap_dma_ch_reg_write(struct omap_dma_s *s,
         break;
 
     case 0x18:	/* SYS_DMA_CPC_CH0 or DMA_CSAC */
-        OMAP_RO_REG((target_phys_addr_t) reg);
+        OMAP_RO_REG((hwaddr) reg);
         break;
 
     case 0x1c:	/* DMA_CDEI */
@@ -1446,7 +1446,7 @@ static int omap_dma_sys_read(struct omap_dma_s *s, int offset,
     return 0;
 }
 
-static uint64_t omap_dma_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_dma_read(void *opaque, hwaddr addr,
                               unsigned size)
 {
     struct omap_dma_s *s = (struct omap_dma_s *) opaque;
@@ -1494,7 +1494,7 @@ static uint64_t omap_dma_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void omap_dma_write(void *opaque, target_phys_addr_t addr,
+static void omap_dma_write(void *opaque, hwaddr addr,
                            uint64_t value, unsigned size)
 {
     struct omap_dma_s *s = (struct omap_dma_s *) opaque;
@@ -1618,7 +1618,7 @@ static void omap_dma_setcaps(struct omap_dma_s *s)
     }
 }
 
-struct soc_dma_s *omap_dma_init(target_phys_addr_t base, qemu_irq *irqs,
+struct soc_dma_s *omap_dma_init(hwaddr base, qemu_irq *irqs,
                 MemoryRegion *sysmem,
                 qemu_irq lcd_irq, struct omap_mpu_state_s *mpu, omap_clk clk,
                 enum omap_dma_model model)
@@ -1692,7 +1692,7 @@ static void omap_dma_interrupts_4_update(struct omap_dma_s *s)
         qemu_irq_raise(s->irq[3]);
 }
 
-static uint64_t omap_dma4_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_dma4_read(void *opaque, hwaddr addr,
                                unsigned size)
 {
     struct omap_dma_s *s = (struct omap_dma_s *) opaque;
@@ -1842,7 +1842,7 @@ static uint64_t omap_dma4_read(void *opaque, target_phys_addr_t addr,
     }
 }
 
-static void omap_dma4_write(void *opaque, target_phys_addr_t addr,
+static void omap_dma4_write(void *opaque, hwaddr addr,
                             uint64_t value, unsigned size)
 {
     struct omap_dma_s *s = (struct omap_dma_s *) opaque;
@@ -1988,12 +1988,12 @@ static void omap_dma4_write(void *opaque, target_phys_addr_t addr,
         break;
 
     case 0x1c:	/* DMA4_CSSA */
-        ch->addr[0] = (target_phys_addr_t) (uint32_t) value;
+        ch->addr[0] = (hwaddr) (uint32_t) value;
         ch->set_update = 1;
         break;
 
     case 0x20:	/* DMA4_CDSA */
-        ch->addr[1] = (target_phys_addr_t) (uint32_t) value;
+        ch->addr[1] = (hwaddr) (uint32_t) value;
         ch->set_update = 1;
         break;
 
@@ -2040,7 +2040,7 @@ static const MemoryRegionOps omap_dma4_ops = {
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-struct soc_dma_s *omap_dma4_init(target_phys_addr_t base, qemu_irq *irqs,
+struct soc_dma_s *omap_dma4_init(hwaddr base, qemu_irq *irqs,
                 MemoryRegion *sysmem,
                 struct omap_mpu_state_s *mpu, int fifo,
                 int chans, omap_clk iclk, omap_clk fclk)
diff --git a/hw/omap_dss.c b/hw/omap_dss.c
index 86ed6ea5d..1e83726d3 100644
--- a/hw/omap_dss.c
+++ b/hw/omap_dss.c
@@ -60,7 +60,7 @@ struct omap_dss_s {
             int nx;
             int ny;
 
-            target_phys_addr_t addr[3];
+            hwaddr addr[3];
 
             uint32_t attr;
             uint32_t tresh;
@@ -168,7 +168,7 @@ void omap_dss_reset(struct omap_dss_s *s)
     omap_dispc_interrupt_update(s);
 }
 
-static uint64_t omap_diss_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_diss_read(void *opaque, hwaddr addr,
                                unsigned size)
 {
     struct omap_dss_s *s = (struct omap_dss_s *) opaque;
@@ -206,7 +206,7 @@ static uint64_t omap_diss_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void omap_diss_write(void *opaque, target_phys_addr_t addr,
+static void omap_diss_write(void *opaque, hwaddr addr,
                             uint64_t value, unsigned size)
 {
     struct omap_dss_s *s = (struct omap_dss_s *) opaque;
@@ -246,7 +246,7 @@ static const MemoryRegionOps omap_diss_ops = {
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-static uint64_t omap_disc_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_disc_read(void *opaque, hwaddr addr,
                                unsigned size)
 {
     struct omap_dss_s *s = (struct omap_dss_s *) opaque;
@@ -371,7 +371,7 @@ static uint64_t omap_disc_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void omap_disc_write(void *opaque, target_phys_addr_t addr,
+static void omap_disc_write(void *opaque, hwaddr addr,
                             uint64_t value, unsigned size)
 {
     struct omap_dss_s *s = (struct omap_dss_s *) opaque;
@@ -502,11 +502,11 @@ static void omap_disc_write(void *opaque, target_phys_addr_t addr,
         s->dispc.invalidate = 1;
         break;
     case 0x080:	/* DISPC_GFX_BA0 */
-        s->dispc.l[0].addr[0] = (target_phys_addr_t) value;
+        s->dispc.l[0].addr[0] = (hwaddr) value;
         s->dispc.invalidate = 1;
         break;
     case 0x084:	/* DISPC_GFX_BA1 */
-        s->dispc.l[0].addr[1] = (target_phys_addr_t) value;
+        s->dispc.l[0].addr[1] = (hwaddr) value;
         s->dispc.invalidate = 1;
         break;
     case 0x088:	/* DISPC_GFX_POSITION */
@@ -543,7 +543,7 @@ static void omap_disc_write(void *opaque, target_phys_addr_t addr,
         s->dispc.l[0].wininc = value;
         break;
     case 0x0b8:	/* DISPC_GFX_TABLE_BA */
-        s->dispc.l[0].addr[2] = (target_phys_addr_t) value;
+        s->dispc.l[0].addr[2] = (hwaddr) value;
         s->dispc.invalidate = 1;
         break;
 
@@ -602,11 +602,11 @@ static void omap_rfbi_transfer_stop(struct omap_dss_s *s)
 static void omap_rfbi_transfer_start(struct omap_dss_s *s)
 {
     void *data;
-    target_phys_addr_t len;
-    target_phys_addr_t data_addr;
+    hwaddr len;
+    hwaddr data_addr;
     int pitch;
     static void *bounce_buffer;
-    static target_phys_addr_t bounce_len;
+    static hwaddr bounce_len;
 
     if (!s->rfbi.enable || s->rfbi.busy)
         return;
@@ -663,7 +663,7 @@ static void omap_rfbi_transfer_start(struct omap_dss_s *s)
     omap_dispc_interrupt_update(s);
 }
 
-static uint64_t omap_rfbi_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_rfbi_read(void *opaque, hwaddr addr,
                                unsigned size)
 {
     struct omap_dss_s *s = (struct omap_dss_s *) opaque;
@@ -730,7 +730,7 @@ static uint64_t omap_rfbi_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void omap_rfbi_write(void *opaque, target_phys_addr_t addr,
+static void omap_rfbi_write(void *opaque, hwaddr addr,
                             uint64_t value, unsigned size)
 {
     struct omap_dss_s *s = (struct omap_dss_s *) opaque;
@@ -864,7 +864,7 @@ static const MemoryRegionOps omap_rfbi_ops = {
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-static uint64_t omap_venc_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_venc_read(void *opaque, hwaddr addr,
                                unsigned size)
 {
     if (size != 4) {
@@ -924,7 +924,7 @@ static uint64_t omap_venc_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void omap_venc_write(void *opaque, target_phys_addr_t addr,
+static void omap_venc_write(void *opaque, hwaddr addr,
                             uint64_t value, unsigned size)
 {
     if (size != 4) {
@@ -986,7 +986,7 @@ static const MemoryRegionOps omap_venc_ops = {
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-static uint64_t omap_im3_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_im3_read(void *opaque, hwaddr addr,
                               unsigned size)
 {
     if (size != 4) {
@@ -1012,7 +1012,7 @@ static uint64_t omap_im3_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void omap_im3_write(void *opaque, target_phys_addr_t addr,
+static void omap_im3_write(void *opaque, hwaddr addr,
                            uint64_t value, unsigned size)
 {
     if (size != 4) {
@@ -1041,7 +1041,7 @@ static const MemoryRegionOps omap_im3_ops = {
 
 struct omap_dss_s *omap_dss_init(struct omap_target_agent_s *ta,
                 MemoryRegion *sysmem,
-                target_phys_addr_t l3_base,
+                hwaddr l3_base,
                 qemu_irq irq, qemu_irq drq,
                 omap_clk fck1, omap_clk fck2, omap_clk ck54m,
                 omap_clk ick1, omap_clk ick2)
diff --git a/hw/omap_gpio.c b/hw/omap_gpio.c
index 201ff77c3..25655325d 100644
--- a/hw/omap_gpio.c
+++ b/hw/omap_gpio.c
@@ -61,7 +61,7 @@ static void omap_gpio_set(void *opaque, int line, int level)
     }
 }
 
-static uint64_t omap_gpio_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_gpio_read(void *opaque, hwaddr addr,
                                unsigned size)
 {
     struct omap_gpio_s *s = (struct omap_gpio_s *) opaque;
@@ -99,7 +99,7 @@ static uint64_t omap_gpio_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void omap_gpio_write(void *opaque, target_phys_addr_t addr,
+static void omap_gpio_write(void *opaque, hwaddr addr,
                             uint64_t value, unsigned size)
 {
     struct omap_gpio_s *s = (struct omap_gpio_s *) opaque;
@@ -300,7 +300,7 @@ static void omap2_gpio_module_reset(struct omap2_gpio_s *s)
     s->delay = 0;
 }
 
-static uint32_t omap2_gpio_module_read(void *opaque, target_phys_addr_t addr)
+static uint32_t omap2_gpio_module_read(void *opaque, hwaddr addr)
 {
     struct omap2_gpio_s *s = (struct omap2_gpio_s *) opaque;
 
@@ -372,7 +372,7 @@ static uint32_t omap2_gpio_module_read(void *opaque, target_phys_addr_t addr)
     return 0;
 }
 
-static void omap2_gpio_module_write(void *opaque, target_phys_addr_t addr,
+static void omap2_gpio_module_write(void *opaque, hwaddr addr,
                 uint32_t value)
 {
     struct omap2_gpio_s *s = (struct omap2_gpio_s *) opaque;
@@ -514,12 +514,12 @@ static void omap2_gpio_module_write(void *opaque, target_phys_addr_t addr,
     }
 }
 
-static uint32_t omap2_gpio_module_readp(void *opaque, target_phys_addr_t addr)
+static uint32_t omap2_gpio_module_readp(void *opaque, hwaddr addr)
 {
     return omap2_gpio_module_read(opaque, addr & ~3) >> ((addr & 3) << 3);
 }
 
-static void omap2_gpio_module_writep(void *opaque, target_phys_addr_t addr,
+static void omap2_gpio_module_writep(void *opaque, hwaddr addr,
                 uint32_t value)
 {
     uint32_t cur = 0;
@@ -604,7 +604,7 @@ static void omap2_gpif_reset(DeviceState *dev)
     s->gpo = 0;
 }
 
-static uint64_t omap2_gpif_top_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap2_gpif_top_read(void *opaque, hwaddr addr,
                                     unsigned size)
 {
     struct omap2_gpif_s *s = (struct omap2_gpif_s *) opaque;
@@ -633,7 +633,7 @@ static uint64_t omap2_gpif_top_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void omap2_gpif_top_write(void *opaque, target_phys_addr_t addr,
+static void omap2_gpif_top_write(void *opaque, hwaddr addr,
                                  uint64_t value, unsigned size)
 {
     struct omap2_gpif_s *s = (struct omap2_gpif_s *) opaque;
diff --git a/hw/omap_gpmc.c b/hw/omap_gpmc.c
index 2fc413720..1f7c5bc5f 100644
--- a/hw/omap_gpmc.c
+++ b/hw/omap_gpmc.c
@@ -121,7 +121,7 @@ static void omap_gpmc_dma_update(struct omap_gpmc_s *s, int value)
  * all addresses in the region behave like accesses to the relevant
  * GPMC_NAND_DATA_i register (which is actually implemented to call these)
  */
-static uint64_t omap_nand_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_nand_read(void *opaque, hwaddr addr,
                                unsigned size)
 {
     struct omap_gpmc_cs_file_s *f = (struct omap_gpmc_cs_file_s *)opaque;
@@ -200,7 +200,7 @@ static void omap_nand_setio(DeviceState *dev, uint64_t value,
     }
 }
 
-static void omap_nand_write(void *opaque, target_phys_addr_t addr,
+static void omap_nand_write(void *opaque, hwaddr addr,
                             uint64_t value, unsigned size)
 {
     struct omap_gpmc_cs_file_s *f = (struct omap_gpmc_cs_file_s *)opaque;
@@ -281,7 +281,7 @@ static void fill_prefetch_fifo(struct omap_gpmc_s *s)
  * engine is enabled -- all addresses in the region behave alike:
  * data is read or written to the FIFO.
  */
-static uint64_t omap_gpmc_prefetch_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_gpmc_prefetch_read(void *opaque, hwaddr addr,
                                         unsigned size)
 {
     struct omap_gpmc_s *s = (struct omap_gpmc_s *) opaque;
@@ -311,7 +311,7 @@ static uint64_t omap_gpmc_prefetch_read(void *opaque, target_phys_addr_t addr,
     return data;
 }
 
-static void omap_gpmc_prefetch_write(void *opaque, target_phys_addr_t addr,
+static void omap_gpmc_prefetch_write(void *opaque, hwaddr addr,
                                      uint64_t value, unsigned size)
 {
     struct omap_gpmc_s *s = (struct omap_gpmc_s *) opaque;
@@ -484,7 +484,7 @@ void omap_gpmc_reset(struct omap_gpmc_s *s)
         ecc_reset(&s->ecc[i]);
 }
 
-static int gpmc_wordaccess_only(target_phys_addr_t addr)
+static int gpmc_wordaccess_only(hwaddr addr)
 {
     /* Return true if the register offset is to a register that
      * only permits word width accesses.
@@ -502,7 +502,7 @@ static int gpmc_wordaccess_only(target_phys_addr_t addr)
     return 1;
 }
 
-static uint64_t omap_gpmc_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_gpmc_read(void *opaque, hwaddr addr,
                                unsigned size)
 {
     struct omap_gpmc_s *s = (struct omap_gpmc_s *) opaque;
@@ -614,7 +614,7 @@ static uint64_t omap_gpmc_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void omap_gpmc_write(void *opaque, target_phys_addr_t addr,
+static void omap_gpmc_write(void *opaque, hwaddr addr,
                             uint64_t value, unsigned size)
 {
     struct omap_gpmc_s *s = (struct omap_gpmc_s *) opaque;
@@ -819,7 +819,7 @@ static const MemoryRegionOps omap_gpmc_ops = {
 };
 
 struct omap_gpmc_s *omap_gpmc_init(struct omap_mpu_state_s *mpu,
-                                   target_phys_addr_t base,
+                                   hwaddr base,
                                    qemu_irq irq, qemu_irq drq)
 {
     int cs;
diff --git a/hw/omap_gptimer.c b/hw/omap_gptimer.c
index 7a145198a..e39da7406 100644
--- a/hw/omap_gptimer.c
+++ b/hw/omap_gptimer.c
@@ -258,7 +258,7 @@ void omap_gp_timer_reset(struct omap_gp_timer_s *s)
     omap_gp_timer_update(s);
 }
 
-static uint32_t omap_gp_timer_readw(void *opaque, target_phys_addr_t addr)
+static uint32_t omap_gp_timer_readw(void *opaque, hwaddr addr)
 {
     struct omap_gp_timer_s *s = (struct omap_gp_timer_s *) opaque;
 
@@ -324,7 +324,7 @@ static uint32_t omap_gp_timer_readw(void *opaque, target_phys_addr_t addr)
     return 0;
 }
 
-static uint32_t omap_gp_timer_readh(void *opaque, target_phys_addr_t addr)
+static uint32_t omap_gp_timer_readh(void *opaque, hwaddr addr)
 {
     struct omap_gp_timer_s *s = (struct omap_gp_timer_s *) opaque;
     uint32_t ret;
@@ -338,7 +338,7 @@ static uint32_t omap_gp_timer_readh(void *opaque, target_phys_addr_t addr)
     }
 }
 
-static void omap_gp_timer_write(void *opaque, target_phys_addr_t addr,
+static void omap_gp_timer_write(void *opaque, hwaddr addr,
                 uint32_t value)
 {
     struct omap_gp_timer_s *s = (struct omap_gp_timer_s *) opaque;
@@ -438,7 +438,7 @@ static void omap_gp_timer_write(void *opaque, target_phys_addr_t addr,
     }
 }
 
-static void omap_gp_timer_writeh(void *opaque, target_phys_addr_t addr,
+static void omap_gp_timer_writeh(void *opaque, hwaddr addr,
                 uint32_t value)
 {
     struct omap_gp_timer_s *s = (struct omap_gp_timer_s *) opaque;
diff --git a/hw/omap_i2c.c b/hw/omap_i2c.c
index 20bc82e3b..ba08e6400 100644
--- a/hw/omap_i2c.c
+++ b/hw/omap_i2c.c
@@ -149,7 +149,7 @@ static void omap_i2c_reset(DeviceState *dev)
     s->test = 0;
 }
 
-static uint32_t omap_i2c_read(void *opaque, target_phys_addr_t addr)
+static uint32_t omap_i2c_read(void *opaque, hwaddr addr)
 {
     OMAPI2CState *s = opaque;
     int offset = addr & OMAP_MPUI_REG_MASK;
@@ -248,7 +248,7 @@ static uint32_t omap_i2c_read(void *opaque, target_phys_addr_t addr)
     return 0;
 }
 
-static void omap_i2c_write(void *opaque, target_phys_addr_t addr,
+static void omap_i2c_write(void *opaque, hwaddr addr,
                 uint32_t value)
 {
     OMAPI2CState *s = opaque;
@@ -390,7 +390,7 @@ static void omap_i2c_write(void *opaque, target_phys_addr_t addr,
     }
 }
 
-static void omap_i2c_writeb(void *opaque, target_phys_addr_t addr,
+static void omap_i2c_writeb(void *opaque, hwaddr addr,
                 uint32_t value)
 {
     OMAPI2CState *s = opaque;
diff --git a/hw/omap_intc.c b/hw/omap_intc.c
index 5076e07ed..61e0dafbd 100644
--- a/hw/omap_intc.c
+++ b/hw/omap_intc.c
@@ -145,7 +145,7 @@ static void omap_set_intr_noedge(void *opaque, int irq, int req)
         bank->irqs = (bank->inputs &= ~(1 << n)) | bank->swi;
 }
 
-static uint64_t omap_inth_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_inth_read(void *opaque, hwaddr addr,
                                unsigned size)
 {
     struct omap_intr_handler_s *s = (struct omap_intr_handler_s *) opaque;
@@ -223,7 +223,7 @@ static uint64_t omap_inth_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void omap_inth_write(void *opaque, target_phys_addr_t addr,
+static void omap_inth_write(void *opaque, hwaddr addr,
                             uint64_t value, unsigned size)
 {
     struct omap_intr_handler_s *s = (struct omap_intr_handler_s *) opaque;
@@ -396,7 +396,7 @@ static TypeInfo omap_intc_info = {
     .class_init    = omap_intc_class_init,
 };
 
-static uint64_t omap2_inth_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap2_inth_read(void *opaque, hwaddr addr,
                                 unsigned size)
 {
     struct omap_intr_handler_s *s = (struct omap_intr_handler_s *) opaque;
@@ -477,7 +477,7 @@ static uint64_t omap2_inth_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void omap2_inth_write(void *opaque, target_phys_addr_t addr,
+static void omap2_inth_write(void *opaque, hwaddr addr,
                              uint64_t value, unsigned size)
 {
     struct omap_intr_handler_s *s = (struct omap_intr_handler_s *) opaque;
diff --git a/hw/omap_l4.c b/hw/omap_l4.c
index dbad7f67a..09e983f31 100644
--- a/hw/omap_l4.c
+++ b/hw/omap_l4.c
@@ -22,13 +22,13 @@
 
 struct omap_l4_s {
     MemoryRegion *address_space;
-    target_phys_addr_t base;
+    hwaddr base;
     int ta_num;
     struct omap_target_agent_s ta[0];
 };
 
 struct omap_l4_s *omap_l4_init(MemoryRegion *address_space,
-                               target_phys_addr_t base, int ta_num)
+                               hwaddr base, int ta_num)
 {
     struct omap_l4_s *bus = g_malloc0(
                     sizeof(*bus) + ta_num * sizeof(*bus->ta));
@@ -40,19 +40,19 @@ struct omap_l4_s *omap_l4_init(MemoryRegion *address_space,
     return bus;
 }
 
-target_phys_addr_t omap_l4_region_base(struct omap_target_agent_s *ta,
+hwaddr omap_l4_region_base(struct omap_target_agent_s *ta,
                                        int region)
 {
     return ta->bus->base + ta->start[region].offset;
 }
 
-target_phys_addr_t omap_l4_region_size(struct omap_target_agent_s *ta,
+hwaddr omap_l4_region_size(struct omap_target_agent_s *ta,
                                        int region)
 {
     return ta->start[region].size;
 }
 
-static uint64_t omap_l4ta_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_l4ta_read(void *opaque, hwaddr addr,
                                unsigned size)
 {
     struct omap_target_agent_s *s = (struct omap_target_agent_s *) opaque;
@@ -76,7 +76,7 @@ static uint64_t omap_l4ta_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void omap_l4ta_write(void *opaque, target_phys_addr_t addr,
+static void omap_l4ta_write(void *opaque, hwaddr addr,
                             uint64_t value, unsigned size)
 {
     struct omap_target_agent_s *s = (struct omap_target_agent_s *) opaque;
@@ -143,10 +143,10 @@ struct omap_target_agent_s *omap_l4ta_get(struct omap_l4_s *bus,
     return ta;
 }
 
-target_phys_addr_t omap_l4_attach(struct omap_target_agent_s *ta,
+hwaddr omap_l4_attach(struct omap_target_agent_s *ta,
                                          int region, MemoryRegion *mr)
 {
-    target_phys_addr_t base;
+    hwaddr base;
 
     if (region < 0 || region >= ta->regions) {
         fprintf(stderr, "%s: bad io region (%i)\n", __FUNCTION__, region);
diff --git a/hw/omap_lcdc.c b/hw/omap_lcdc.c
index 4a08e9d00..d7ae3032b 100644
--- a/hw/omap_lcdc.c
+++ b/hw/omap_lcdc.c
@@ -117,7 +117,7 @@ static void omap_update_display(void *opaque)
     draw_line_func draw_line;
     int size, height, first, last;
     int width, linesize, step, bpp, frame_offset;
-    target_phys_addr_t frame_base;
+    hwaddr frame_base;
 
     if (!omap_lcd || omap_lcd->plm == 1 ||
                     !omap_lcd->enable || !ds_get_bits_per_pixel(omap_lcd->state))
@@ -219,23 +219,29 @@ static void omap_update_display(void *opaque)
                                draw_line, omap_lcd->palette,
                                &first, &last);
     if (first >= 0) {
-        dpy_update(omap_lcd->state, 0, first, width, last - first + 1);
+        dpy_gfx_update(omap_lcd->state, 0, first, width, last - first + 1);
     }
     omap_lcd->invalidate = 0;
 }
 
-static int ppm_save(const char *filename, uint8_t *data,
-                int w, int h, int linesize)
+static void omap_ppm_save(const char *filename, uint8_t *data,
+                    int w, int h, int linesize, Error **errp)
 {
     FILE *f;
     uint8_t *d, *d1;
     unsigned int v;
-    int y, x, bpp;
+    int ret, y, x, bpp;
 
     f = fopen(filename, "wb");
-    if (!f)
-        return -1;
-    fprintf(f, "P6\n%d %d\n%d\n", w, h, 255);
+    if (!f) {
+        error_setg(errp, "failed to open file '%s': %s", filename,
+                   strerror(errno));
+        return;
+    }
+    ret = fprintf(f, "P6\n%d %d\n%d\n", w, h, 255);
+    if (ret < 0) {
+        goto write_err;
+    }
     d1 = data;
     bpp = linesize / w;
     for (y = 0; y < h; y ++) {
@@ -244,35 +250,61 @@ static int ppm_save(const char *filename, uint8_t *data,
             v = *(uint32_t *) d;
             switch (bpp) {
             case 2:
-                fputc((v >> 8) & 0xf8, f);
-                fputc((v >> 3) & 0xfc, f);
-                fputc((v << 3) & 0xf8, f);
+                ret = fputc((v >> 8) & 0xf8, f);
+                if (ret == EOF) {
+                    goto write_err;
+                }
+                ret = fputc((v >> 3) & 0xfc, f);
+                if (ret == EOF) {
+                    goto write_err;
+                }
+                ret = fputc((v << 3) & 0xf8, f);
+                if (ret == EOF) {
+                    goto write_err;
+                }
                 break;
             case 3:
             case 4:
             default:
-                fputc((v >> 16) & 0xff, f);
-                fputc((v >> 8) & 0xff, f);
-                fputc((v) & 0xff, f);
+                ret = fputc((v >> 16) & 0xff, f);
+                if (ret == EOF) {
+                    goto write_err;
+                }
+                ret = fputc((v >> 8) & 0xff, f);
+                if (ret == EOF) {
+                    goto write_err;
+                }
+                ret = fputc((v) & 0xff, f);
+                if (ret == EOF) {
+                    goto write_err;
+                }
                 break;
             }
             d += bpp;
         }
         d1 += linesize;
     }
+out:
     fclose(f);
-    return 0;
+    return;
+
+write_err:
+    error_setg(errp, "failed to write to file '%s': %s", filename,
+               strerror(errno));
+    unlink(filename);
+    goto out;
 }
 
-static void omap_screen_dump(void *opaque, const char *filename, bool cswitch)
+static void omap_screen_dump(void *opaque, const char *filename, bool cswitch,
+                             Error **errp)
 {
     struct omap_lcd_panel_s *omap_lcd = opaque;
 
     omap_update_display(opaque);
     if (omap_lcd && ds_get_data(omap_lcd->state))
-        ppm_save(filename, ds_get_data(omap_lcd->state),
-                omap_lcd->width, omap_lcd->height,
-                ds_get_linesize(omap_lcd->state));
+        omap_ppm_save(filename, ds_get_data(omap_lcd->state),
+                    omap_lcd->width, omap_lcd->height,
+                    ds_get_linesize(omap_lcd->state), errp);
 }
 
 static void omap_invalidate_display(void *opaque) {
@@ -327,7 +359,7 @@ static void omap_lcd_update(struct omap_lcd_panel_s *s) {
     }
 }
 
-static uint64_t omap_lcdc_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_lcdc_read(void *opaque, hwaddr addr,
                                unsigned size)
 {
     struct omap_lcd_panel_s *s = (struct omap_lcd_panel_s *) opaque;
@@ -360,7 +392,7 @@ static uint64_t omap_lcdc_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void omap_lcdc_write(void *opaque, target_phys_addr_t addr,
+static void omap_lcdc_write(void *opaque, hwaddr addr,
                             uint64_t value, unsigned size)
 {
     struct omap_lcd_panel_s *s = (struct omap_lcd_panel_s *) opaque;
@@ -433,7 +465,7 @@ void omap_lcdc_reset(struct omap_lcd_panel_s *s)
 }
 
 struct omap_lcd_panel_s *omap_lcdc_init(MemoryRegion *sysmem,
-                                        target_phys_addr_t base,
+                                        hwaddr base,
                                         qemu_irq irq,
                                         struct omap_dma_lcd_channel_s *dma,
                                         omap_clk clk)
diff --git a/hw/omap_mmc.c b/hw/omap_mmc.c
index aec028567..7ecd9bd4c 100644
--- a/hw/omap_mmc.c
+++ b/hw/omap_mmc.c
@@ -306,7 +306,7 @@ void omap_mmc_reset(struct omap_mmc_s *host)
     host->clkdiv = 0;
 }
 
-static uint64_t omap_mmc_read(void *opaque, target_phys_addr_t offset,
+static uint64_t omap_mmc_read(void *opaque, hwaddr offset,
                               unsigned size)
 {
     uint16_t i;
@@ -399,7 +399,7 @@ static uint64_t omap_mmc_read(void *opaque, target_phys_addr_t offset,
     return 0;
 }
 
-static void omap_mmc_write(void *opaque, target_phys_addr_t offset,
+static void omap_mmc_write(void *opaque, hwaddr offset,
                            uint64_t value, unsigned size)
 {
     int i;
@@ -572,7 +572,7 @@ static void omap_mmc_cover_cb(void *opaque, int line, int level)
     }
 }
 
-struct omap_mmc_s *omap_mmc_init(target_phys_addr_t base,
+struct omap_mmc_s *omap_mmc_init(hwaddr base,
                 MemoryRegion *sysmem,
                 BlockDriverState *bd,
                 qemu_irq irq, qemu_irq dma[], omap_clk clk)
diff --git a/hw/omap_sdrc.c b/hw/omap_sdrc.c
index 784e32610..b0f3b8e67 100644
--- a/hw/omap_sdrc.c
+++ b/hw/omap_sdrc.c
@@ -31,7 +31,7 @@ void omap_sdrc_reset(struct omap_sdrc_s *s)
     s->config = 0x10;
 }
 
-static uint64_t omap_sdrc_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_sdrc_read(void *opaque, hwaddr addr,
                                unsigned size)
 {
     struct omap_sdrc_s *s = (struct omap_sdrc_s *) opaque;
@@ -86,7 +86,7 @@ static uint64_t omap_sdrc_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void omap_sdrc_write(void *opaque, target_phys_addr_t addr,
+static void omap_sdrc_write(void *opaque, hwaddr addr,
                             uint64_t value, unsigned size)
 {
     struct omap_sdrc_s *s = (struct omap_sdrc_s *) opaque;
@@ -154,7 +154,7 @@ static const MemoryRegionOps omap_sdrc_ops = {
 };
 
 struct omap_sdrc_s *omap_sdrc_init(MemoryRegion *sysmem,
-                                   target_phys_addr_t base)
+                                   hwaddr base)
 {
     struct omap_sdrc_s *s = (struct omap_sdrc_s *)
             g_malloc0(sizeof(struct omap_sdrc_s));
diff --git a/hw/omap_spi.c b/hw/omap_spi.c
index 8f2b697d2..42d5149a2 100644
--- a/hw/omap_spi.c
+++ b/hw/omap_spi.c
@@ -130,7 +130,7 @@ void omap_mcspi_reset(struct omap_mcspi_s *s)
     omap_mcspi_interrupt_update(s);
 }
 
-static uint64_t omap_mcspi_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_mcspi_read(void *opaque, hwaddr addr,
                                 unsigned size)
 {
     struct omap_mcspi_s *s = (struct omap_mcspi_s *) opaque;
@@ -204,7 +204,7 @@ static uint64_t omap_mcspi_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void omap_mcspi_write(void *opaque, target_phys_addr_t addr,
+static void omap_mcspi_write(void *opaque, hwaddr addr,
                              uint64_t value, unsigned size)
 {
     struct omap_mcspi_s *s = (struct omap_mcspi_s *) opaque;
diff --git a/hw/omap_sx1.c b/hw/omap_sx1.c
index abca34192..21a5bbb00 100644
--- a/hw/omap_sx1.c
+++ b/hw/omap_sx1.c
@@ -59,7 +59,7 @@
  * - 1 RTC
  */
 
-static uint64_t static_read(void *opaque, target_phys_addr_t offset,
+static uint64_t static_read(void *opaque, hwaddr offset,
                             unsigned size)
 {
     uint32_t *val = (uint32_t *) opaque;
@@ -68,7 +68,7 @@ static uint64_t static_read(void *opaque, target_phys_addr_t offset,
     return *val >> ((offset & mask) << 3);
 }
 
-static void static_write(void *opaque, target_phys_addr_t offset,
+static void static_write(void *opaque, hwaddr offset,
                          uint64_t value, unsigned size)
 {
 #ifdef SPY
@@ -97,11 +97,7 @@ static struct arm_boot_info sx1_binfo = {
     .board_id = 0x265,
 };
 
-static void sx1_init(ram_addr_t ram_size,
-                const char *boot_device,
-                const char *kernel_filename, const char *kernel_cmdline,
-                const char *initrd_filename, const char *cpu_model,
-                const int version)
+static void sx1_init(QEMUMachineInitArgs *args, const int version)
 {
     struct omap_mpu_state_s *mpu;
     MemoryRegion *address_space = get_system_memory();
@@ -121,7 +117,7 @@ static void sx1_init(ram_addr_t ram_size,
         flash_size = flash2_size;
     }
 
-    mpu = omap310_mpu_init(address_space, sx1_binfo.ram_size, cpu_model);
+    mpu = omap310_mpu_init(address_space, sx1_binfo.ram_size, args->cpu_model);
 
     /* External Flash (EMIFS) */
     memory_region_init_ram(flash, "omap_sx1.flash0-0", flash_size);
@@ -192,16 +188,16 @@ static void sx1_init(ram_addr_t ram_size,
                                 OMAP_CS1_BASE, &cs[1]);
     }
 
-    if (!kernel_filename && !fl_idx) {
+    if (!args->kernel_filename && !fl_idx) {
         fprintf(stderr, "Kernel or Flash image must be specified\n");
         exit(1);
     }
 
     /* Load the kernel.  */
-    if (kernel_filename) {
-        sx1_binfo.kernel_filename = kernel_filename;
-        sx1_binfo.kernel_cmdline = kernel_cmdline;
-        sx1_binfo.initrd_filename = initrd_filename;
+    if (args->kernel_filename) {
+        sx1_binfo.kernel_filename = args->kernel_filename;
+        sx1_binfo.kernel_cmdline = args->kernel_cmdline;
+        sx1_binfo.initrd_filename = args->initrd_filename;
         arm_load_kernel(mpu->cpu, &sx1_binfo);
     }
 
@@ -209,22 +205,14 @@ static void sx1_init(ram_addr_t ram_size,
     //~ qemu_console_resize(ds, 640, 480);
 }
 
-static void sx1_init_v1(ram_addr_t ram_size,
-                const char *boot_device,
-                const char *kernel_filename, const char *kernel_cmdline,
-                const char *initrd_filename, const char *cpu_model)
+static void sx1_init_v1(QEMUMachineInitArgs *args)
 {
-    sx1_init(ram_size, boot_device, kernel_filename,
-                kernel_cmdline, initrd_filename, cpu_model, 1);
+    sx1_init(args, 1);
 }
 
-static void sx1_init_v2(ram_addr_t ram_size,
-                const char *boot_device,
-                const char *kernel_filename, const char *kernel_cmdline,
-                const char *initrd_filename, const char *cpu_model)
+static void sx1_init_v2(QEMUMachineInitArgs *args)
 {
-    sx1_init(ram_size, boot_device, kernel_filename,
-                kernel_cmdline, initrd_filename, cpu_model, 2);
+    sx1_init(args, 2);
 }
 
 static QEMUMachine sx1_machine_v2 = {
diff --git a/hw/omap_synctimer.c b/hw/omap_synctimer.c
index 367f26e3a..7031a8853 100644
--- a/hw/omap_synctimer.c
+++ b/hw/omap_synctimer.c
@@ -36,7 +36,7 @@ void omap_synctimer_reset(struct omap_synctimer_s *s)
     s->val = omap_synctimer_read(s);
 }
 
-static uint32_t omap_synctimer_readw(void *opaque, target_phys_addr_t addr)
+static uint32_t omap_synctimer_readw(void *opaque, hwaddr addr)
 {
     struct omap_synctimer_s *s = (struct omap_synctimer_s *) opaque;
 
@@ -52,7 +52,7 @@ static uint32_t omap_synctimer_readw(void *opaque, target_phys_addr_t addr)
     return 0;
 }
 
-static uint32_t omap_synctimer_readh(void *opaque, target_phys_addr_t addr)
+static uint32_t omap_synctimer_readh(void *opaque, hwaddr addr)
 {
     struct omap_synctimer_s *s = (struct omap_synctimer_s *) opaque;
     uint32_t ret;
@@ -66,7 +66,7 @@ static uint32_t omap_synctimer_readh(void *opaque, target_phys_addr_t addr)
     }
 }
 
-static void omap_synctimer_write(void *opaque, target_phys_addr_t addr,
+static void omap_synctimer_write(void *opaque, hwaddr addr,
                 uint32_t value)
 {
     OMAP_BAD_REG(addr);
diff --git a/hw/omap_tap.c b/hw/omap_tap.c
index 0277c7365..e273e971e 100644
--- a/hw/omap_tap.c
+++ b/hw/omap_tap.c
@@ -22,7 +22,7 @@
 #include "omap.h"
 
 /* TEST-Chip-level TAP */
-static uint64_t omap_tap_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_tap_read(void *opaque, hwaddr addr,
                               unsigned size)
 {
     struct omap_mpu_state_s *s = (struct omap_mpu_state_s *) opaque;
@@ -91,7 +91,7 @@ static uint64_t omap_tap_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void omap_tap_write(void *opaque, target_phys_addr_t addr,
+static void omap_tap_write(void *opaque, hwaddr addr,
                            uint64_t value, unsigned size)
 {
     if (size != 4) {
diff --git a/hw/omap_uart.c b/hw/omap_uart.c
index 167d5c4e5..92f27021b 100644
--- a/hw/omap_uart.c
+++ b/hw/omap_uart.c
@@ -20,14 +20,13 @@
 #include "qemu-char.h"
 #include "hw.h"
 #include "omap.h"
-/* We use pc-style serial ports.  */
-#include "pc.h"
+#include "serial.h"
 #include "exec-memory.h"
 
 /* UARTs */
 struct omap_uart_s {
     MemoryRegion iomem;
-    target_phys_addr_t base;
+    hwaddr base;
     SerialState *serial; /* TODO */
     struct omap_target_agent_s *ta;
     omap_clk fclk;
@@ -51,7 +50,7 @@ void omap_uart_reset(struct omap_uart_s *s)
     s->clksel = 0;
 }
 
-struct omap_uart_s *omap_uart_init(target_phys_addr_t base,
+struct omap_uart_s *omap_uart_init(hwaddr base,
                 qemu_irq irq, omap_clk fclk, omap_clk iclk,
                 qemu_irq txdma, qemu_irq rxdma,
                 const char *label, CharDriverState *chr)
@@ -69,7 +68,7 @@ struct omap_uart_s *omap_uart_init(target_phys_addr_t base,
     return s;
 }
 
-static uint64_t omap_uart_read(void *opaque, target_phys_addr_t addr,
+static uint64_t omap_uart_read(void *opaque, hwaddr addr,
                                unsigned size)
 {
     struct omap_uart_s *s = (struct omap_uart_s *) opaque;
@@ -107,7 +106,7 @@ static uint64_t omap_uart_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void omap_uart_write(void *opaque, target_phys_addr_t addr,
+static void omap_uart_write(void *opaque, hwaddr addr,
                             uint64_t value, unsigned size)
 {
     struct omap_uart_s *s = (struct omap_uart_s *) opaque;
@@ -165,7 +164,7 @@ struct omap_uart_s *omap2_uart_init(MemoryRegion *sysmem,
                 qemu_irq txdma, qemu_irq rxdma,
                 const char *label, CharDriverState *chr)
 {
-    target_phys_addr_t base = omap_l4_attach(ta, 0, NULL);
+    hwaddr base = omap_l4_attach(ta, 0, NULL);
     struct omap_uart_s *s = omap_uart_init(base, irq,
                     fclk, iclk, txdma, rxdma, label, chr);
 
diff --git a/hw/onenand.c b/hw/onenand.c
index db6af682c..1803e4c26 100644
--- a/hw/onenand.c
+++ b/hw/onenand.c
@@ -42,7 +42,7 @@ typedef struct {
         uint16_t ver;
     } id;
     int shift;
-    target_phys_addr_t base;
+    hwaddr base;
     qemu_irq intr;
     qemu_irq rdy;
     BlockDriverState *bdrv;
@@ -351,7 +351,7 @@ static inline int onenand_erase(OneNANDState *s, int sec, int num)
     for (; num > 0; num--, sec++) {
         if (s->bdrv_cur) {
             int erasesec = s->secs_cur + (sec >> 5);
-            if (bdrv_write(s->bdrv_cur, sec, blankbuf, 1)) {
+            if (bdrv_write(s->bdrv_cur, sec, blankbuf, 1) < 0) {
                 goto fail;
             }
             if (bdrv_read(s->bdrv_cur, erasesec, tmpbuf, 1) < 0) {
@@ -588,7 +588,7 @@ static void onenand_command(OneNANDState *s)
     onenand_intr_update(s);
 }
 
-static uint64_t onenand_read(void *opaque, target_phys_addr_t addr,
+static uint64_t onenand_read(void *opaque, hwaddr addr,
                              unsigned size)
 {
     OneNANDState *s = (OneNANDState *) opaque;
@@ -653,7 +653,7 @@ static uint64_t onenand_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void onenand_write(void *opaque, target_phys_addr_t addr,
+static void onenand_write(void *opaque, hwaddr addr,
                           uint64_t value, unsigned size)
 {
     OneNANDState *s = (OneNANDState *) opaque;
@@ -760,7 +760,7 @@ static int onenand_initfn(SysBusDevice *dev)
     OneNANDState *s = (OneNANDState *)dev;
     uint32_t size = 1 << (24 + ((s->id.dev >> 4) & 7));
     void *ram;
-    s->base = (target_phys_addr_t)-1;
+    s->base = (hwaddr)-1;
     s->rdy = NULL;
     s->blocks = size >> BLOCK_SHIFT;
     s->secs = size >> 9;
diff --git a/hw/opencores_eth.c b/hw/opencores_eth.c
index 8c15969e2..b2780b933 100644
--- a/hw/opencores_eth.c
+++ b/hw/opencores_eth.c
@@ -528,7 +528,7 @@ static void open_eth_check_start_xmit(OpenEthState *s)
 }
 
 static uint64_t open_eth_reg_read(void *opaque,
-        target_phys_addr_t addr, unsigned int size)
+        hwaddr addr, unsigned int size)
 {
     static uint32_t (*reg_read[REG_MAX])(OpenEthState *s) = {
     };
@@ -620,7 +620,7 @@ static void open_eth_mii_tx_host_write(OpenEthState *s, uint32_t val)
 }
 
 static void open_eth_reg_write(void *opaque,
-        target_phys_addr_t addr, uint64_t val, unsigned int size)
+        hwaddr addr, uint64_t val, unsigned int size)
 {
     static void (*reg_write[REG_MAX])(OpenEthState *s, uint32_t val) = {
         [MODER] = open_eth_moder_host_write,
@@ -644,7 +644,7 @@ static void open_eth_reg_write(void *opaque,
 }
 
 static uint64_t open_eth_desc_read(void *opaque,
-        target_phys_addr_t addr, unsigned int size)
+        hwaddr addr, unsigned int size)
 {
     OpenEthState *s = opaque;
     uint64_t v = 0;
@@ -656,7 +656,7 @@ static uint64_t open_eth_desc_read(void *opaque,
 }
 
 static void open_eth_desc_write(void *opaque,
-        target_phys_addr_t addr, uint64_t val, unsigned int size)
+        hwaddr addr, uint64_t val, unsigned int size)
 {
     OpenEthState *s = opaque;
 
diff --git a/hw/openpic.c b/hw/openpic.c
index b9d856830..8b3784a6b 100644
--- a/hw/openpic.c
+++ b/hw/openpic.c
@@ -178,9 +178,9 @@ static int get_current_cpu(void)
   return cpu_single_env->cpu_index;
 }
 
-static uint32_t openpic_cpu_read_internal(void *opaque, target_phys_addr_t addr,
+static uint32_t openpic_cpu_read_internal(void *opaque, hwaddr addr,
                                           int idx);
-static void openpic_cpu_write_internal(void *opaque, target_phys_addr_t addr,
+static void openpic_cpu_write_internal(void *opaque, hwaddr addr,
                                        uint32_t val, int idx);
 
 enum {
@@ -596,7 +596,7 @@ static void write_mailbox_register (openpic_t *opp, int n_mbx,
 #endif
 #endif /* 0 : Code provision for Intel model */
 
-static void openpic_gbl_write (void *opaque, target_phys_addr_t addr, uint32_t val)
+static void openpic_gbl_write (void *opaque, hwaddr addr, uint32_t val)
 {
     openpic_t *opp = opaque;
     IRQ_dst_t *dst;
@@ -662,7 +662,7 @@ static void openpic_gbl_write (void *opaque, target_phys_addr_t addr, uint32_t v
     }
 }
 
-static uint32_t openpic_gbl_read (void *opaque, target_phys_addr_t addr)
+static uint32_t openpic_gbl_read (void *opaque, hwaddr addr)
 {
     openpic_t *opp = opaque;
     uint32_t retval;
@@ -826,7 +826,7 @@ static uint32_t openpic_src_read (void *opaque, uint32_t addr)
     return retval;
 }
 
-static void openpic_cpu_write_internal(void *opaque, target_phys_addr_t addr,
+static void openpic_cpu_write_internal(void *opaque, hwaddr addr,
                                        uint32_t val, int idx)
 {
     openpic_t *opp = opaque;
@@ -886,12 +886,12 @@ static void openpic_cpu_write_internal(void *opaque, target_phys_addr_t addr,
     }
 }
 
-static void openpic_cpu_write(void *opaque, target_phys_addr_t addr, uint32_t val)
+static void openpic_cpu_write(void *opaque, hwaddr addr, uint32_t val)
 {
     openpic_cpu_write_internal(opaque, addr, val, (addr & 0x1f000) >> 12);
 }
 
-static uint32_t openpic_cpu_read_internal(void *opaque, target_phys_addr_t addr,
+static uint32_t openpic_cpu_read_internal(void *opaque, hwaddr addr,
                                           int idx)
 {
     openpic_t *opp = opaque;
@@ -970,18 +970,18 @@ static uint32_t openpic_cpu_read_internal(void *opaque, target_phys_addr_t addr,
     return retval;
 }
 
-static uint32_t openpic_cpu_read(void *opaque, target_phys_addr_t addr)
+static uint32_t openpic_cpu_read(void *opaque, hwaddr addr)
 {
     return openpic_cpu_read_internal(opaque, addr, (addr & 0x1f000) >> 12);
 }
 
 static void openpic_buggy_write (void *opaque,
-                                 target_phys_addr_t addr, uint32_t val)
+                                 hwaddr addr, uint32_t val)
 {
     printf("Invalid OPENPIC write access !\n");
 }
 
-static uint32_t openpic_buggy_read (void *opaque, target_phys_addr_t addr)
+static uint32_t openpic_buggy_read (void *opaque, hwaddr addr)
 {
     printf("Invalid OPENPIC read access !\n");
 
@@ -989,7 +989,7 @@ static uint32_t openpic_buggy_read (void *opaque, target_phys_addr_t addr)
 }
 
 static void openpic_writel (void *opaque,
-                            target_phys_addr_t addr, uint32_t val)
+                            hwaddr addr, uint32_t val)
 {
     openpic_t *opp = opaque;
 
@@ -1010,7 +1010,7 @@ static void openpic_writel (void *opaque,
     }
 }
 
-static uint32_t openpic_readl (void *opaque,target_phys_addr_t addr)
+static uint32_t openpic_readl (void *opaque,hwaddr addr)
 {
     openpic_t *opp = opaque;
     uint32_t retval;
@@ -1034,7 +1034,7 @@ static uint32_t openpic_readl (void *opaque,target_phys_addr_t addr)
     return retval;
 }
 
-static uint64_t openpic_read(void *opaque, target_phys_addr_t addr,
+static uint64_t openpic_read(void *opaque, hwaddr addr,
                              unsigned size)
 {
     openpic_t *opp = opaque;
@@ -1045,7 +1045,7 @@ static uint64_t openpic_read(void *opaque, target_phys_addr_t addr,
     }
 }
 
-static void openpic_write(void *opaque, target_phys_addr_t addr,
+static void openpic_write(void *opaque, hwaddr addr,
                           uint64_t data, unsigned size)
 {
     openpic_t *opp = opaque;
@@ -1300,7 +1300,7 @@ static void mpic_reset (void *opaque)
     mpp->glbc = 0x00000000;
 }
 
-static void mpic_timer_write (void *opaque, target_phys_addr_t addr, uint32_t val)
+static void mpic_timer_write (void *opaque, hwaddr addr, uint32_t val)
 {
     openpic_t *mpp = opaque;
     int idx, cpu;
@@ -1333,7 +1333,7 @@ static void mpic_timer_write (void *opaque, target_phys_addr_t addr, uint32_t va
     }
 }
 
-static uint32_t mpic_timer_read (void *opaque, target_phys_addr_t addr)
+static uint32_t mpic_timer_read (void *opaque, hwaddr addr)
 {
     openpic_t *mpp = opaque;
     uint32_t retval;
@@ -1368,7 +1368,7 @@ static uint32_t mpic_timer_read (void *opaque, target_phys_addr_t addr)
     return retval;
 }
 
-static void mpic_src_ext_write (void *opaque, target_phys_addr_t addr,
+static void mpic_src_ext_write (void *opaque, hwaddr addr,
                                 uint32_t val)
 {
     openpic_t *mpp = opaque;
@@ -1390,7 +1390,7 @@ static void mpic_src_ext_write (void *opaque, target_phys_addr_t addr,
     }
 }
 
-static uint32_t mpic_src_ext_read (void *opaque, target_phys_addr_t addr)
+static uint32_t mpic_src_ext_read (void *opaque, hwaddr addr)
 {
     openpic_t *mpp = opaque;
     uint32_t retval;
@@ -1416,7 +1416,7 @@ static uint32_t mpic_src_ext_read (void *opaque, target_phys_addr_t addr)
     return retval;
 }
 
-static void mpic_src_int_write (void *opaque, target_phys_addr_t addr,
+static void mpic_src_int_write (void *opaque, hwaddr addr,
                                 uint32_t val)
 {
     openpic_t *mpp = opaque;
@@ -1438,7 +1438,7 @@ static void mpic_src_int_write (void *opaque, target_phys_addr_t addr,
     }
 }
 
-static uint32_t mpic_src_int_read (void *opaque, target_phys_addr_t addr)
+static uint32_t mpic_src_int_read (void *opaque, hwaddr addr)
 {
     openpic_t *mpp = opaque;
     uint32_t retval;
@@ -1464,7 +1464,7 @@ static uint32_t mpic_src_int_read (void *opaque, target_phys_addr_t addr)
     return retval;
 }
 
-static void mpic_src_msg_write (void *opaque, target_phys_addr_t addr,
+static void mpic_src_msg_write (void *opaque, hwaddr addr,
                                 uint32_t val)
 {
     openpic_t *mpp = opaque;
@@ -1486,7 +1486,7 @@ static void mpic_src_msg_write (void *opaque, target_phys_addr_t addr,
     }
 }
 
-static uint32_t mpic_src_msg_read (void *opaque, target_phys_addr_t addr)
+static uint32_t mpic_src_msg_read (void *opaque, hwaddr addr)
 {
     openpic_t *mpp = opaque;
     uint32_t retval;
@@ -1512,7 +1512,7 @@ static uint32_t mpic_src_msg_read (void *opaque, target_phys_addr_t addr)
     return retval;
 }
 
-static void mpic_src_msi_write (void *opaque, target_phys_addr_t addr,
+static void mpic_src_msi_write (void *opaque, hwaddr addr,
                                 uint32_t val)
 {
     openpic_t *mpp = opaque;
@@ -1533,7 +1533,7 @@ static void mpic_src_msi_write (void *opaque, target_phys_addr_t addr,
         }
     }
 }
-static uint32_t mpic_src_msi_read (void *opaque, target_phys_addr_t addr)
+static uint32_t mpic_src_msi_read (void *opaque, hwaddr addr)
 {
     openpic_t *mpp = opaque;
     uint32_t retval;
@@ -1657,7 +1657,7 @@ static const MemoryRegionOps mpic_msi_ops = {
     .endianness = DEVICE_BIG_ENDIAN,
 };
 
-qemu_irq *mpic_init (MemoryRegion *address_space, target_phys_addr_t base,
+qemu_irq *mpic_init (MemoryRegion *address_space, hwaddr base,
                      int nb_cpus, qemu_irq **irqs, qemu_irq irq_out)
 {
     openpic_t    *mpp;
@@ -1665,7 +1665,7 @@ qemu_irq *mpic_init (MemoryRegion *address_space, target_phys_addr_t base,
     struct {
         const char             *name;
         MemoryRegionOps const  *ops;
-        target_phys_addr_t      start_addr;
+        hwaddr      start_addr;
         ram_addr_t              size;
     } const list[] = {
         {"glb", &mpic_glb_ops, MPIC_GLB_REG_START, MPIC_GLB_REG_SIZE},
diff --git a/hw/openpic.h b/hw/openpic.h
index 855603026..f50a1e42b 100644
--- a/hw/openpic.h
+++ b/hw/openpic.h
@@ -13,6 +13,6 @@ enum {
 
 qemu_irq *openpic_init (MemoryRegion **pmem, int nb_cpus,
                         qemu_irq **irqs, qemu_irq irq_out);
-qemu_irq *mpic_init (MemoryRegion *address_space, target_phys_addr_t base,
+qemu_irq *mpic_init (MemoryRegion *address_space, hwaddr base,
                      int nb_cpus, qemu_irq **irqs, qemu_irq irq_out);
 #endif /* __OPENPIC_H__ */
diff --git a/hw/openrisc_sim.c b/hw/openrisc_sim.c
index 55e97f095..23c66df1f 100644
--- a/hw/openrisc_sim.c
+++ b/hw/openrisc_sim.c
@@ -21,7 +21,8 @@
 #include "hw.h"
 #include "boards.h"
 #include "elf.h"
-#include "pc.h"
+#include "serial.h"
+#include "net.h"
 #include "loader.h"
 #include "exec-memory.h"
 #include "sysemu.h"
@@ -38,8 +39,8 @@ static void main_cpu_reset(void *opaque)
 }
 
 static void openrisc_sim_net_init(MemoryRegion *address_space,
-                                  target_phys_addr_t base,
-                                  target_phys_addr_t descriptors,
+                                  hwaddr base,
+                                  hwaddr descriptors,
                                   qemu_irq irq, NICInfo *nd)
 {
     DeviceState *dev;
@@ -63,7 +64,7 @@ static void cpu_openrisc_load_kernel(ram_addr_t ram_size,
 {
     long kernel_size;
     uint64_t elf_entry;
-    target_phys_addr_t entry;
+    hwaddr entry;
 
     if (kernel_filename && !qtest_enabled()) {
         kernel_size = load_elf(kernel_filename, NULL, NULL,
@@ -90,13 +91,11 @@ static void cpu_openrisc_load_kernel(ram_addr_t ram_size,
     cpu->env.pc = entry;
 }
 
-static void openrisc_sim_init(ram_addr_t ram_size,
-                              const char *boot_device,
-                              const char *kernel_filename,
-                              const char *kernel_cmdline,
-                              const char *initrd_filename,
-                              const char *cpu_model)
+static void openrisc_sim_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t ram_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
    OpenRISCCPU *cpu = NULL;
     MemoryRegion *ram;
     int n;
diff --git a/hw/palm.c b/hw/palm.c
index bacdc90d4..6f6f414e6 100644
--- a/hw/palm.c
+++ b/hw/palm.c
@@ -27,25 +27,25 @@
 #include "loader.h"
 #include "exec-memory.h"
 
-static uint32_t static_readb(void *opaque, target_phys_addr_t offset)
+static uint32_t static_readb(void *opaque, hwaddr offset)
 {
     uint32_t *val = (uint32_t *) opaque;
     return *val >> ((offset & 3) << 3);
 }
 
-static uint32_t static_readh(void *opaque, target_phys_addr_t offset)
+static uint32_t static_readh(void *opaque, hwaddr offset)
 {
     uint32_t *val = (uint32_t *) opaque;
     return *val >> ((offset & 1) << 3);
 }
 
-static uint32_t static_readw(void *opaque, target_phys_addr_t offset)
+static uint32_t static_readw(void *opaque, hwaddr offset)
 {
     uint32_t *val = (uint32_t *) opaque;
     return *val >> ((offset & 0) << 3);
 }
 
-static void static_write(void *opaque, target_phys_addr_t offset,
+static void static_write(void *opaque, hwaddr offset,
                 uint32_t value)
 {
 #ifdef SPY
@@ -190,11 +190,12 @@ static struct arm_boot_info palmte_binfo = {
     .board_id = 0x331,
 };
 
-static void palmte_init(ram_addr_t ram_size,
-                const char *boot_device,
-                const char *kernel_filename, const char *kernel_cmdline,
-                const char *initrd_filename, const char *cpu_model)
+static void palmte_init(QEMUMachineInitArgs *args)
 {
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
     MemoryRegion *address_space_mem = get_system_memory();
     struct omap_mpu_state_s *mpu;
     int flash_size = 0x00800000;
@@ -272,7 +273,7 @@ static void palmte_init(ram_addr_t ram_size,
        will set the size once configured, so this just sets an initial
        size until the guest activates the display.  */
     ds->surface = qemu_resize_displaysurface(ds, 320, 320);
-    dpy_resize(ds);
+    dpy_gfx_resize(ds);
 }
 
 static QEMUMachine palmte_machine = {
diff --git a/hw/pam.c b/hw/pam.c
new file mode 100644
index 000000000..a95e2cfb0
--- /dev/null
+++ b/hw/pam.c
@@ -0,0 +1,87 @@
+/*
+ * QEMU i440FX/PIIX3 PCI Bridge Emulation
+ *
+ * Copyright (c) 2006 Fabrice Bellard
+ * Copyright (c) 2011 Isaku Yamahata <yamahata at valinux co jp>
+ *                    VA Linux Systems Japan K.K.
+ * Copyright (c) 2012 Jason Baron <jbaron@redhat.com>
+ *
+ * Split out from piix_pci.c
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#include "sysemu.h"
+#include "pam.h"
+
+void smram_update(MemoryRegion *smram_region, uint8_t smram,
+                  uint8_t smm_enabled)
+{
+    bool smram_enabled;
+
+    smram_enabled = ((smm_enabled && (smram & SMRAM_G_SMRAME)) ||
+                        (smram & SMRAM_D_OPEN));
+    memory_region_set_enabled(smram_region, !smram_enabled);
+}
+
+void smram_set_smm(uint8_t *host_smm_enabled, int smm, uint8_t smram,
+                   MemoryRegion *smram_region)
+{
+    uint8_t smm_enabled = (smm != 0);
+    if (*host_smm_enabled != smm_enabled) {
+        *host_smm_enabled = smm_enabled;
+        smram_update(smram_region, smram, *host_smm_enabled);
+    }
+}
+
+void init_pam(MemoryRegion *ram_memory, MemoryRegion *system_memory,
+              MemoryRegion *pci_address_space, PAMMemoryRegion *mem,
+              uint32_t start, uint32_t size)
+{
+    int i;
+
+    /* RAM */
+    memory_region_init_alias(&mem->alias[3], "pam-ram", ram_memory,
+                             start, size);
+    /* ROM (XXX: not quite correct) */
+    memory_region_init_alias(&mem->alias[1], "pam-rom", ram_memory,
+                             start, size);
+    memory_region_set_readonly(&mem->alias[1], true);
+
+    /* XXX: should distinguish read/write cases */
+    memory_region_init_alias(&mem->alias[0], "pam-pci", pci_address_space,
+                             start, size);
+    memory_region_init_alias(&mem->alias[2], "pam-pci", pci_address_space,
+                             start, size);
+
+    for (i = 0; i < 4; ++i) {
+        memory_region_set_enabled(&mem->alias[i], false);
+        memory_region_add_subregion_overlap(system_memory, start,
+                                            &mem->alias[i], 1);
+    }
+    mem->current = 0;
+}
+
+void pam_update(PAMMemoryRegion *pam, int idx, uint8_t val)
+{
+    assert(0 <= idx && idx <= 12);
+
+    memory_region_set_enabled(&pam->alias[pam->current], false);
+    pam->current = (val >> ((!(idx & 1)) * 4)) & PAM_ATTR_MASK;
+    memory_region_set_enabled(&pam->alias[pam->current], true);
+}
diff --git a/hw/pam.h b/hw/pam.h
new file mode 100644
index 000000000..2d77ebe0d
--- /dev/null
+++ b/hw/pam.h
@@ -0,0 +1,97 @@
+#ifndef QEMU_PAM_H
+#define QEMU_PAM_H
+
+/*
+ * Copyright (c) 2006 Fabrice Bellard
+ * Copyright (c) 2011 Isaku Yamahata <yamahata at valinux co jp>
+ *               VA Linux Systems Japan K.K.
+ * Copyright (c) 2012 Jason Baron <jbaron@redhat.com>
+ *
+ * Split out from piix_pci.c
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+/*
+ * SMRAM memory area and PAM memory area in Legacy address range for PC.
+ * PAM: Programmable Attribute Map registers
+ *
+ * 0xa0000 - 0xbffff compatible SMRAM
+ *
+ * 0xc0000 - 0xc3fff Expansion area memory segments
+ * 0xc4000 - 0xc7fff
+ * 0xc8000 - 0xcbfff
+ * 0xcc000 - 0xcffff
+ * 0xd0000 - 0xd3fff
+ * 0xd4000 - 0xd7fff
+ * 0xd8000 - 0xdbfff
+ * 0xdc000 - 0xdffff
+ * 0xe0000 - 0xe3fff Extended System BIOS Area Memory Segments
+ * 0xe4000 - 0xe7fff
+ * 0xe8000 - 0xebfff
+ * 0xec000 - 0xeffff
+ *
+ * 0xf0000 - 0xfffff System BIOS Area Memory Segments
+ */
+
+#include "qemu-common.h"
+#include "memory.h"
+
+#define SMRAM_C_BASE    0xa0000
+#define SMRAM_C_END     0xc0000
+#define SMRAM_C_SIZE    0x20000
+
+#define PAM_EXPAN_BASE  0xc0000
+#define PAM_EXPAN_SIZE  0x04000
+
+#define PAM_EXBIOS_BASE 0xe0000
+#define PAM_EXBIOS_SIZE 0x04000
+
+#define PAM_BIOS_BASE   0xf0000
+#define PAM_BIOS_END    0xfffff
+/* 64KB: Intel 3 series express chipset family p. 58*/
+#define PAM_BIOS_SIZE   0x10000
+
+/* PAM registers: log nibble and high nibble*/
+#define PAM_ATTR_WE     ((uint8_t)2)
+#define PAM_ATTR_RE     ((uint8_t)1)
+#define PAM_ATTR_MASK   ((uint8_t)3)
+
+/* SMRAM register */
+#define SMRAM_D_OPEN           ((uint8_t)(1 << 6))
+#define SMRAM_D_CLS            ((uint8_t)(1 << 5))
+#define SMRAM_D_LCK            ((uint8_t)(1 << 4))
+#define SMRAM_G_SMRAME         ((uint8_t)(1 << 3))
+#define SMRAM_C_BASE_SEG_MASK  ((uint8_t)0x7)
+#define SMRAM_C_BASE_SEG       ((uint8_t)0x2)  /* hardwired to b010 */
+
+typedef struct PAMMemoryRegion {
+    MemoryRegion alias[4];  /* index = PAM value */
+    unsigned current;
+} PAMMemoryRegion;
+
+void smram_update(MemoryRegion *smram_region, uint8_t smram,
+                  uint8_t smm_enabled);
+void smram_set_smm(uint8_t *host_smm_enabled, int smm, uint8_t smram,
+                   MemoryRegion *smram_region);
+void init_pam(MemoryRegion *ram, MemoryRegion *system, MemoryRegion *pci,
+              PAMMemoryRegion *mem, uint32_t start, uint32_t size);
+void pam_update(PAMMemoryRegion *mem, int idx, uint8_t val);
+
+#endif /* QEMU_PAM_H */
diff --git a/hw/parallel.c b/hw/parallel.c
index 219f38436..c4705bc89 100644
--- a/hw/parallel.c
+++ b/hw/parallel.c
@@ -511,7 +511,7 @@ static int parallel_isa_initfn(ISADevice *dev)
 }
 
 /* Memory mapped interface */
-static uint32_t parallel_mm_readb (void *opaque, target_phys_addr_t addr)
+static uint32_t parallel_mm_readb (void *opaque, hwaddr addr)
 {
     ParallelState *s = opaque;
 
@@ -519,14 +519,14 @@ static uint32_t parallel_mm_readb (void *opaque, target_phys_addr_t addr)
 }
 
 static void parallel_mm_writeb (void *opaque,
-                                target_phys_addr_t addr, uint32_t value)
+                                hwaddr addr, uint32_t value)
 {
     ParallelState *s = opaque;
 
     parallel_ioport_write_sw(s, addr >> s->it_shift, value & 0xFF);
 }
 
-static uint32_t parallel_mm_readw (void *opaque, target_phys_addr_t addr)
+static uint32_t parallel_mm_readw (void *opaque, hwaddr addr)
 {
     ParallelState *s = opaque;
 
@@ -534,14 +534,14 @@ static uint32_t parallel_mm_readw (void *opaque, target_phys_addr_t addr)
 }
 
 static void parallel_mm_writew (void *opaque,
-                                target_phys_addr_t addr, uint32_t value)
+                                hwaddr addr, uint32_t value)
 {
     ParallelState *s = opaque;
 
     parallel_ioport_write_sw(s, addr >> s->it_shift, value & 0xFFFF);
 }
 
-static uint32_t parallel_mm_readl (void *opaque, target_phys_addr_t addr)
+static uint32_t parallel_mm_readl (void *opaque, hwaddr addr)
 {
     ParallelState *s = opaque;
 
@@ -549,7 +549,7 @@ static uint32_t parallel_mm_readl (void *opaque, target_phys_addr_t addr)
 }
 
 static void parallel_mm_writel (void *opaque,
-                                target_phys_addr_t addr, uint32_t value)
+                                hwaddr addr, uint32_t value)
 {
     ParallelState *s = opaque;
 
@@ -566,7 +566,7 @@ static const MemoryRegionOps parallel_mm_ops = {
 
 /* If fd is zero, it means that the parallel device uses the console */
 bool parallel_mm_init(MemoryRegion *address_space,
-                      target_phys_addr_t base, int it_shift, qemu_irq irq,
+                      hwaddr base, int it_shift, qemu_irq irq,
                       CharDriverState *chr)
 {
     ParallelState *s;
diff --git a/hw/pc.c b/hw/pc.c
index 112739a27..2b5bbbfb3 100644
--- a/hw/pc.c
+++ b/hw/pc.c
@@ -23,11 +23,11 @@
  */
 #include "hw.h"
 #include "pc.h"
+#include "serial.h"
 #include "apic.h"
 #include "fdc.h"
 #include "ide.h"
 #include "pci.h"
-#include "vmware_vga.h"
 #include "monitor.h"
 #include "fw_cfg.h"
 #include "hpet_emul.h"
@@ -51,10 +51,6 @@
 #include "exec-memory.h"
 #include "arch_init.h"
 #include "bitmap.h"
-#include "vga-pci.h"
-
-/* output Bochs bios info messages */
-//#define DEBUG_BIOS
 
 /* debug PC/ISA interrupts */
 //#define DEBUG_IRQ
@@ -75,8 +71,6 @@
 #define FW_CFG_E820_TABLE (FW_CFG_ARCH_LOCAL + 3)
 #define FW_CFG_HPET (FW_CFG_ARCH_LOCAL + 4)
 
-#define MSI_ADDR_BASE 0xfee00000
-
 #define E820_NR_ENTRIES		16
 
 struct e820_entry {
@@ -425,7 +419,8 @@ typedef struct Port92State {
     qemu_irq *a20_out;
 } Port92State;
 
-static void port92_write(void *opaque, uint32_t addr, uint32_t val)
+static void port92_write(void *opaque, hwaddr addr, uint64_t val,
+                         unsigned size)
 {
     Port92State *s = opaque;
 
@@ -437,7 +432,8 @@ static void port92_write(void *opaque, uint32_t addr, uint32_t val)
     }
 }
 
-static uint32_t port92_read(void *opaque, uint32_t addr)
+static uint64_t port92_read(void *opaque, hwaddr addr,
+                            unsigned size)
 {
     Port92State *s = opaque;
     uint32_t ret;
@@ -472,13 +468,14 @@ static void port92_reset(DeviceState *d)
     s->outport &= ~1;
 }
 
-static const MemoryRegionPortio port92_portio[] = {
-    { 0, 1, 1, .read = port92_read, .write = port92_write },
-    PORTIO_END_OF_LIST(),
-};
-
 static const MemoryRegionOps port92_ops = {
-    .old_portio = port92_portio
+    .read = port92_read,
+    .write = port92_write,
+    .impl = {
+        .min_access_size = 1,
+        .max_access_size = 1,
+    },
+    .endianness = DEVICE_LITTLE_ENDIAN,
 };
 
 static int port92_initfn(ISADevice *dev)
@@ -534,17 +531,6 @@ static void bochs_bios_write(void *opaque, uint32_t addr, uint32_t val)
     static int shutdown_index = 0;
 
     switch(addr) {
-        /* Bochs BIOS messages */
-    case 0x400:
-    case 0x401:
-        /* used to be panic, now unused */
-        break;
-    case 0x402:
-    case 0x403:
-#ifdef DEBUG_BIOS
-        fprintf(stderr, "%c", val);
-#endif
-        break;
     case 0x8900:
         /* same as Bochs power off */
         if (val == shutdown_str[shutdown_index]) {
@@ -558,16 +544,9 @@ static void bochs_bios_write(void *opaque, uint32_t addr, uint32_t val)
         }
         break;
 
-        /* LGPL'ed VGA BIOS messages */
     case 0x501:
     case 0x502:
         exit((val << 1) | 1);
-    case 0x500:
-    case 0x503:
-#ifdef DEBUG_BIOS
-        fprintf(stderr, "%c", val);
-#endif
-        break;
     }
 }
 
@@ -596,17 +575,11 @@ static void *bochs_bios_init(void)
     uint64_t *numa_fw_cfg;
     int i, j;
 
-    register_ioport_write(0x400, 1, 2, bochs_bios_write, NULL);
-    register_ioport_write(0x401, 1, 2, bochs_bios_write, NULL);
-    register_ioport_write(0x402, 1, 1, bochs_bios_write, NULL);
-    register_ioport_write(0x403, 1, 1, bochs_bios_write, NULL);
     register_ioport_write(0x8900, 1, 1, bochs_bios_write, NULL);
 
     register_ioport_write(0x501, 1, 1, bochs_bios_write, NULL);
     register_ioport_write(0x501, 1, 2, bochs_bios_write, NULL);
     register_ioport_write(0x502, 1, 2, bochs_bios_write, NULL);
-    register_ioport_write(0x500, 1, 1, bochs_bios_write, NULL);
-    register_ioport_write(0x503, 1, 1, bochs_bios_write, NULL);
 
     fw_cfg = fw_cfg_init(BIOS_CFG_IOPORT, BIOS_CFG_IOPORT + 1, 0, 0);
 
@@ -666,13 +639,13 @@ static void load_linux(void *fw_cfg,
                        const char *kernel_filename,
 		       const char *initrd_filename,
 		       const char *kernel_cmdline,
-                       target_phys_addr_t max_ram_size)
+                       hwaddr max_ram_size)
 {
     uint16_t protocol;
     int setup_size, kernel_size, initrd_size = 0, cmdline_size;
     uint32_t initrd_max;
     uint8_t header[8192], *setup, *kernel, *initrd_data;
-    target_phys_addr_t real_addr, prot_addr, cmdline_addr, initrd_addr = 0;
+    hwaddr real_addr, prot_addr, cmdline_addr, initrd_addr = 0;
     FILE *f;
     char *vmode;
 
@@ -874,35 +847,6 @@ DeviceState *cpu_get_current_apic(void)
     }
 }
 
-static DeviceState *apic_init(void *env, uint8_t apic_id)
-{
-    DeviceState *dev;
-    static int apic_mapped;
-
-    if (kvm_irqchip_in_kernel()) {
-        dev = qdev_create(NULL, "kvm-apic");
-    } else if (xen_enabled()) {
-        dev = qdev_create(NULL, "xen-apic");
-    } else {
-        dev = qdev_create(NULL, "apic");
-    }
-
-    qdev_prop_set_uint8(dev, "id", apic_id);
-    qdev_prop_set_ptr(dev, "cpu_env", env);
-    qdev_init_nofail(dev);
-
-    /* XXX: mapping more APICs at the same memory location */
-    if (apic_mapped == 0) {
-        /* NOTE: the APIC is directly connected to the CPU - it is not
-           on the global memory bus. */
-        /* XXX: what if the base changes? */
-        sysbus_mmio_map(sysbus_from_qdev(dev), 0, MSI_ADDR_BASE);
-        apic_mapped = 1;
-    }
-
-    return dev;
-}
-
 void pc_acpi_smi_interrupt(void *opaque, int irq, int level)
 {
     CPUX86State *s = opaque;
@@ -912,24 +856,6 @@ void pc_acpi_smi_interrupt(void *opaque, int irq, int level)
     }
 }
 
-static X86CPU *pc_new_cpu(const char *cpu_model)
-{
-    X86CPU *cpu;
-    CPUX86State *env;
-
-    cpu = cpu_x86_init(cpu_model);
-    if (cpu == NULL) {
-        fprintf(stderr, "Unable to find x86 CPU definition\n");
-        exit(1);
-    }
-    env = &cpu->env;
-    if ((env->cpuid_features & CPUID_APIC) || smp_cpus > 1) {
-        env->apic_state = apic_init(env, env->cpuid_apic_id);
-    }
-    cpu_reset(CPU(cpu));
-    return cpu;
-}
-
 void pc_cpus_init(const char *cpu_model)
 {
     int i;
@@ -943,8 +869,11 @@ void pc_cpus_init(const char *cpu_model)
 #endif
     }
 
-    for(i = 0; i < smp_cpus; i++) {
-        pc_new_cpu(cpu_model);
+    for (i = 0; i < smp_cpus; i++) {
+        if (!cpu_x86_init(cpu_model)) {
+            fprintf(stderr, "Unable to find x86 CPU definition\n");
+            exit(1);
+        }
     }
 }
 
@@ -1019,34 +948,13 @@ DeviceState *pc_vga_init(ISABus *isa_bus, PCIBus *pci_bus)
 {
     DeviceState *dev = NULL;
 
-    if (cirrus_vga_enabled) {
-        if (pci_bus) {
-            dev = pci_cirrus_vga_init(pci_bus);
-        } else {
-            dev = &isa_create_simple(isa_bus, "isa-cirrus-vga")->qdev;
-        }
-    } else if (vmsvga_enabled) {
-        if (pci_bus) {
-            dev = pci_vmsvga_init(pci_bus);
-        } else {
-            fprintf(stderr, "%s: vmware_vga: no PCI bus\n", __FUNCTION__);
-        }
-#ifdef CONFIG_SPICE
-    } else if (qxl_enabled) {
-        if (pci_bus) {
-            dev = &pci_create_simple(pci_bus, -1, "qxl-vga")->qdev;
-        } else {
-            fprintf(stderr, "%s: qxl: no PCI bus\n", __FUNCTION__);
-        }
-#endif
-    } else if (std_vga_enabled) {
-        if (pci_bus) {
-            dev = pci_vga_init(pci_bus);
-        } else {
-            dev = isa_vga_init(isa_bus);
-        }
+    if (pci_bus) {
+        PCIDevice *pcidev = pci_vga_init(pci_bus);
+        dev = pcidev ? &pcidev->qdev : NULL;
+    } else if (isa_bus) {
+        ISADevice *isadev = isa_vga_init(isa_bus);
+        dev = isadev ? &isadev->qdev : NULL;
     }
-
     return dev;
 }
 
@@ -1150,6 +1058,21 @@ void pc_basic_device_init(ISABus *isa_bus, qemu_irq *gsi,
     *floppy = fdctrl_init_isa(isa_bus, fd);
 }
 
+void pc_nic_init(ISABus *isa_bus, PCIBus *pci_bus)
+{
+    int i;
+
+    for (i = 0; i < nb_nics; i++) {
+        NICInfo *nd = &nd_table[i];
+
+        if (!pci_bus || (nd->model && strcmp(nd->model, "ne2k_isa") == 0)) {
+            pc_init_ne2k_isa(isa_bus, nd);
+        } else {
+            pci_nic_init_nofail(nd, "e1000", NULL);
+        }
+    }
+}
+
 void pc_pci_device_init(PCIBus *pci_bus)
 {
     int max_bus;
@@ -1160,3 +1083,27 @@ void pc_pci_device_init(PCIBus *pci_bus)
         pci_create_simple(pci_bus, -1, "lsi53c895a");
     }
 }
+
+void ioapic_init_gsi(GSIState *gsi_state, const char *parent_name)
+{
+    DeviceState *dev;
+    SysBusDevice *d;
+    unsigned int i;
+
+    if (kvm_irqchip_in_kernel()) {
+        dev = qdev_create(NULL, "kvm-ioapic");
+    } else {
+        dev = qdev_create(NULL, "ioapic");
+    }
+    if (parent_name) {
+        object_property_add_child(object_resolve_path(parent_name, NULL),
+                                  "ioapic", OBJECT(dev), NULL);
+    }
+    qdev_init_nofail(dev);
+    d = sysbus_from_qdev(dev);
+    sysbus_mmio_map(d, 0, 0xfec00000);
+
+    for (i = 0; i < IOAPIC_NUM_PINS; i++) {
+        gsi_state->ioapic_irq[i] = qdev_get_gpio_in(dev, i);
+    }
+}
diff --git a/hw/pc.h b/hw/pc.h
index e4db0715b..2237e8644 100644
--- a/hw/pc.h
+++ b/hw/pc.h
@@ -12,33 +12,6 @@
 
 /* PC-style peripherals (also used by other machines).  */
 
-/* serial.c */
-
-SerialState *serial_init(int base, qemu_irq irq, int baudbase,
-                         CharDriverState *chr);
-SerialState *serial_mm_init(MemoryRegion *address_space,
-                            target_phys_addr_t base, int it_shift,
-                            qemu_irq irq, int baudbase,
-                            CharDriverState *chr, enum device_endian);
-static inline bool serial_isa_init(ISABus *bus, int index,
-                                   CharDriverState *chr)
-{
-    ISADevice *dev;
-
-    dev = isa_try_create(bus, "isa-serial");
-    if (!dev) {
-        return false;
-    }
-    qdev_prop_set_uint32(&dev->qdev, "index", index);
-    qdev_prop_set_chr(&dev->qdev, "chardev", chr);
-    if (qdev_init(&dev->qdev) < 0) {
-        return false;
-    }
-    return true;
-}
-
-void serial_set_frequency(SerialState *s, uint32_t frequency);
-
 /* parallel.c */
 static inline bool parallel_init(ISABus *bus, int index, CharDriverState *chr)
 {
@@ -57,7 +30,7 @@ static inline bool parallel_init(ISABus *bus, int index, CharDriverState *chr)
 }
 
 bool parallel_mm_init(MemoryRegion *address_space,
-                      target_phys_addr_t base, int it_shift, qemu_irq irq,
+                      hwaddr base, int it_shift, qemu_irq irq,
                       CharDriverState *chr);
 
 /* i8259.c */
@@ -95,7 +68,7 @@ void vmmouse_set_data(const uint32_t *data);
 void i8042_init(qemu_irq kbd_irq, qemu_irq mouse_irq, uint32_t io_base);
 void i8042_mm_init(qemu_irq kbd_irq, qemu_irq mouse_irq,
                    MemoryRegion *region, ram_addr_t size,
-                   target_phys_addr_t mask);
+                   hwaddr mask);
 void i8042_isa_mouse_fake_event(void *opaque);
 void i8042_setup_a20_line(ISADevice *dev, qemu_irq *a20_out);
 
@@ -125,11 +98,14 @@ void pc_cmos_init(ram_addr_t ram_size, ram_addr_t above_4g_mem_size,
                   const char *boot_device,
                   ISADevice *floppy, BusState *ide0, BusState *ide1,
                   ISADevice *s);
+void pc_nic_init(ISABus *isa_bus, PCIBus *pci_bus);
 void pc_pci_device_init(PCIBus *pci_bus);
 
 typedef void (*cpu_set_smm_t)(int smm, void *arg);
 void cpu_smm_register(cpu_set_smm_t callback, void *arg);
 
+void ioapic_init_gsi(GSIState *gsi_state, const char *parent_name);
+
 /* acpi.c */
 extern int acpi_enabled;
 extern char *acpi_tables;
@@ -157,10 +133,10 @@ PCIBus *i440fx_init(PCII440FXState **pi440fx_state, int *piix_devfn,
                     MemoryRegion *address_space_mem,
                     MemoryRegion *address_space_io,
                     ram_addr_t ram_size,
-                    target_phys_addr_t pci_hole_start,
-                    target_phys_addr_t pci_hole_size,
-                    target_phys_addr_t pci_hole64_start,
-                    target_phys_addr_t pci_hole64_size,
+                    hwaddr pci_hole_start,
+                    hwaddr pci_hole_size,
+                    hwaddr pci_hole64_start,
+                    hwaddr pci_hole64_size,
                     MemoryRegion *pci_memory,
                     MemoryRegion *ram_memory);
 
@@ -176,21 +152,8 @@ enum vga_retrace_method {
 
 extern enum vga_retrace_method vga_retrace_method;
 
-static inline DeviceState *isa_vga_init(ISABus *bus)
-{
-    ISADevice *dev;
-
-    dev = isa_try_create(bus, "isa-vga");
-    if (!dev) {
-        fprintf(stderr, "Warning: isa-vga not available\n");
-        return NULL;
-    }
-    qdev_init_nofail(&dev->qdev);
-    return &dev->qdev;
-}
-
-int isa_vga_mm_init(target_phys_addr_t vram_base,
-                    target_phys_addr_t ctrl_base, int it_shift,
+int isa_vga_mm_init(hwaddr vram_base,
+                    hwaddr ctrl_base, int it_shift,
                     MemoryRegion *address_space);
 
 /* ne2000.c */
diff --git a/hw/pc_piix.c b/hw/pc_piix.c
index 88ff0411c..aa3e7f40d 100644
--- a/hw/pc_piix.c
+++ b/hw/pc_piix.c
@@ -43,6 +43,7 @@
 #include "xen.h"
 #include "memory.h"
 #include "exec-memory.h"
+#include "cpu.h"
 #ifdef CONFIG_XEN
 #  include <xen/hvm/hvm_info_table.h>
 #endif
@@ -53,70 +54,6 @@ static const int ide_iobase[MAX_IDE_BUS] = { 0x1f0, 0x170 };
 static const int ide_iobase2[MAX_IDE_BUS] = { 0x3f6, 0x376 };
 static const int ide_irq[MAX_IDE_BUS] = { 14, 15 };
 
-static void kvm_piix3_setup_irq_routing(bool pci_enabled)
-{
-#ifdef CONFIG_KVM
-    KVMState *s = kvm_state;
-    int i;
-
-    if (kvm_check_extension(s, KVM_CAP_IRQ_ROUTING)) {
-        for (i = 0; i < 8; ++i) {
-            if (i == 2) {
-                continue;
-            }
-            kvm_irqchip_add_irq_route(s, i, KVM_IRQCHIP_PIC_MASTER, i);
-        }
-        for (i = 8; i < 16; ++i) {
-            kvm_irqchip_add_irq_route(s, i, KVM_IRQCHIP_PIC_SLAVE, i - 8);
-        }
-        if (pci_enabled) {
-            for (i = 0; i < 24; ++i) {
-                if (i == 0) {
-                    kvm_irqchip_add_irq_route(s, i, KVM_IRQCHIP_IOAPIC, 2);
-                } else if (i != 2) {
-                    kvm_irqchip_add_irq_route(s, i, KVM_IRQCHIP_IOAPIC, i);
-                }
-            }
-        }
-    }
-#endif /* CONFIG_KVM */
-}
-
-static void kvm_piix3_gsi_handler(void *opaque, int n, int level)
-{
-    GSIState *s = opaque;
-
-    if (n < ISA_NUM_IRQS) {
-        /* Kernel will forward to both PIC and IOAPIC */
-        qemu_set_irq(s->i8259_irq[n], level);
-    } else {
-        qemu_set_irq(s->ioapic_irq[n], level);
-    }
-}
-
-static void ioapic_init(GSIState *gsi_state)
-{
-    DeviceState *dev;
-    SysBusDevice *d;
-    unsigned int i;
-
-    if (kvm_irqchip_in_kernel()) {
-        dev = qdev_create(NULL, "kvm-ioapic");
-    } else {
-        dev = qdev_create(NULL, "ioapic");
-    }
-    /* FIXME: this should be under the piix3.  */
-    object_property_add_child(object_resolve_path("i440fx", NULL),
-                              "ioapic", OBJECT(dev), NULL);
-    qdev_init_nofail(dev);
-    d = sysbus_from_qdev(dev);
-    sysbus_mmio_map(d, 0, 0xfec00000);
-
-    for (i = 0; i < IOAPIC_NUM_PINS; i++) {
-        gsi_state->ioapic_irq[i] = qdev_get_gpio_in(dev, i);
-    }
-}
-
 /* PC hardware initialisation */
 static void pc_init1(MemoryRegion *system_memory,
                      MemoryRegion *system_io,
@@ -177,13 +114,13 @@ static void pc_init1(MemoryRegion *system_memory,
         fw_cfg = pc_memory_init(system_memory,
                        kernel_filename, kernel_cmdline, initrd_filename,
                        below_4g_mem_size, above_4g_mem_size,
-                       pci_enabled ? rom_memory : system_memory, &ram_memory);
+                       rom_memory, &ram_memory);
     }
 
     gsi_state = g_malloc0(sizeof(*gsi_state));
     if (kvm_irqchip_in_kernel()) {
-        kvm_piix3_setup_irq_routing(pci_enabled);
-        gsi = qemu_allocate_irqs(kvm_piix3_gsi_handler, gsi_state,
+        kvm_pc_setup_irq_routing(pci_enabled);
+        gsi = qemu_allocate_irqs(kvm_pc_gsi_handler, gsi_state,
                                  GSI_NUM_PINS);
     } else {
         gsi = qemu_allocate_irqs(gsi_handler, gsi_state, GSI_NUM_PINS);
@@ -195,7 +132,7 @@ static void pc_init1(MemoryRegion *system_memory,
                               below_4g_mem_size,
                               0x100000000ULL - below_4g_mem_size,
                               0x100000000ULL + above_4g_mem_size,
-                              (sizeof(target_phys_addr_t) == 4
+                              (sizeof(hwaddr) == 4
                                ? 0
                                : ((uint64_t)1 << 62)),
                               pci_memory, ram_memory);
@@ -220,7 +157,7 @@ static void pc_init1(MemoryRegion *system_memory,
         gsi_state->i8259_irq[i] = i8259[i];
     }
     if (pci_enabled) {
-        ioapic_init(gsi_state);
+        ioapic_init_gsi(gsi_state, "i440fx");
     }
 
     pc_register_ferr_irq(gsi[13]);
@@ -233,14 +170,7 @@ static void pc_init1(MemoryRegion *system_memory,
     /* init basic PC hardware */
     pc_basic_device_init(isa_bus, gsi, &rtc_state, &floppy, xen_enabled());
 
-    for(i = 0; i < nb_nics; i++) {
-        NICInfo *nd = &nd_table[i];
-
-        if (!pci_enabled || (nd->model && strcmp(nd->model, "ne2k_isa") == 0))
-            pc_init_ne2k_isa(isa_bus, nd);
-        else
-            pci_nic_init_nofail(nd, "e1000", NULL);
-    }
+    pc_nic_init(isa_bus, pci_bus);
 
     ide_drive_get(hd, MAX_IDE_BUS);
     if (pci_enabled) {
@@ -267,7 +197,7 @@ static void pc_init1(MemoryRegion *system_memory,
     pc_cmos_init(below_4g_mem_size, above_4g_mem_size, boot_device,
                  floppy, idebus[0], idebus[1], rtc_state);
 
-    if (pci_enabled && usb_enabled) {
+    if (pci_enabled && usb_enabled(false)) {
         pci_create_simple(pci_bus, piix3_devfn + 2, "piix3-usb-uhci");
     }
 
@@ -287,13 +217,14 @@ static void pc_init1(MemoryRegion *system_memory,
     }
 }
 
-static void pc_init_pci(ram_addr_t ram_size,
-                        const char *boot_device,
-                        const char *kernel_filename,
-                        const char *kernel_cmdline,
-                        const char *initrd_filename,
-                        const char *cpu_model)
+static void pc_init_pci(QEMUMachineInitArgs *args)
 {
+    ram_addr_t ram_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
+    const char *boot_device = args->boot_device;
     pc_init1(get_system_memory(),
              get_system_io(),
              ram_size, boot_device,
@@ -301,13 +232,20 @@ static void pc_init_pci(ram_addr_t ram_size,
              initrd_filename, cpu_model, 1, 1);
 }
 
-static void pc_init_pci_no_kvmclock(ram_addr_t ram_size,
-                                    const char *boot_device,
-                                    const char *kernel_filename,
-                                    const char *kernel_cmdline,
-                                    const char *initrd_filename,
-                                    const char *cpu_model)
+static void pc_init_pci_1_3(QEMUMachineInitArgs *args)
 {
+    enable_kvm_pv_eoi();
+    pc_init_pci(args);
+}
+
+static void pc_init_pci_no_kvmclock(QEMUMachineInitArgs *args)
+{
+    ram_addr_t ram_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
+    const char *boot_device = args->boot_device;
     pc_init1(get_system_memory(),
              get_system_io(),
              ram_size, boot_device,
@@ -315,13 +253,14 @@ static void pc_init_pci_no_kvmclock(ram_addr_t ram_size,
              initrd_filename, cpu_model, 1, 0);
 }
 
-static void pc_init_isa(ram_addr_t ram_size,
-                        const char *boot_device,
-                        const char *kernel_filename,
-                        const char *kernel_cmdline,
-                        const char *initrd_filename,
-                        const char *cpu_model)
+static void pc_init_isa(QEMUMachineInitArgs *args)
 {
+    ram_addr_t ram_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
+    const char *boot_device = args->boot_device;
     if (cpu_model == NULL)
         cpu_model = "486";
     pc_init1(get_system_memory(),
@@ -332,33 +271,65 @@ static void pc_init_isa(ram_addr_t ram_size,
 }
 
 #ifdef CONFIG_XEN
-static void pc_xen_hvm_init(ram_addr_t ram_size,
-                            const char *boot_device,
-                            const char *kernel_filename,
-                            const char *kernel_cmdline,
-                            const char *initrd_filename,
-                            const char *cpu_model)
+static void pc_xen_hvm_init(QEMUMachineInitArgs *args)
 {
     if (xen_hvm_init() != 0) {
         hw_error("xen hardware virtual machine initialisation failed");
     }
-    pc_init_pci_no_kvmclock(ram_size, boot_device,
-                            kernel_filename, kernel_cmdline,
-                            initrd_filename, cpu_model);
+    pc_init_pci_no_kvmclock(args);
     xen_vcpu_init();
 }
 #endif
 
+static QEMUMachine pc_machine_v1_3 = {
+    .name = "pc-1.3",
+    .alias = "pc",
+    .desc = "Standard PC",
+    .init = pc_init_pci_1_3,
+    .max_cpus = 255,
+    .is_default = 1,
+};
+
+#define PC_COMPAT_1_2 \
+        {\
+            .driver   = "nec-usb-xhci",\
+            .property = "msi",\
+            .value    = "off",\
+        },{\
+            .driver   = "nec-usb-xhci",\
+            .property = "msix",\
+            .value    = "off",\
+        },{\
+            .driver   = "ivshmem",\
+            .property = "use64",\
+            .value    = "0",\
+        },{\
+            .driver   = "qxl",\
+            .property = "revision",\
+            .value    = stringify(3),\
+        },{\
+            .driver   = "qxl-vga",\
+            .property = "revision",\
+            .value    = stringify(3),\
+        },{\
+            .driver   = "VGA",\
+            .property = "mmio",\
+            .value    = "off",\
+        }
+
 static QEMUMachine pc_machine_v1_2 = {
     .name = "pc-1.2",
-    .alias = "pc",
     .desc = "Standard PC",
     .init = pc_init_pci,
     .max_cpus = 255,
-    .is_default = 1,
+    .compat_props = (GlobalProperty[]) {
+        PC_COMPAT_1_2,
+        { /* end of list */ }
+    },
 };
 
 #define PC_COMPAT_1_1 \
+        PC_COMPAT_1_2,\
         {\
             .driver   = "virtio-scsi-pci",\
             .property = "hotplug",\
@@ -655,6 +626,7 @@ static QEMUMachine xenfv_machine = {
 
 static void pc_machine_init(void)
 {
+    qemu_register_machine(&pc_machine_v1_3);
     qemu_register_machine(&pc_machine_v1_2);
     qemu_register_machine(&pc_machine_v1_1);
     qemu_register_machine(&pc_machine_v1_0);
diff --git a/hw/pc_q35.c b/hw/pc_q35.c
new file mode 100644
index 000000000..3429a9ae8
--- /dev/null
+++ b/hw/pc_q35.c
@@ -0,0 +1,223 @@
+/*
+ * Q35 chipset based pc system emulator
+ *
+ * Copyright (c) 2003-2004 Fabrice Bellard
+ * Copyright (c) 2009, 2010
+ *               Isaku Yamahata <yamahata at valinux co jp>
+ *               VA Linux Systems Japan K.K.
+ * Copyright (C) 2012 Jason Baron <jbaron@redhat.com>
+ *
+ * This is based on pc.c, but heavily modified.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#include "hw.h"
+#include "arch_init.h"
+#include "smbus.h"
+#include "boards.h"
+#include "mc146818rtc.h"
+#include "xen.h"
+#include "kvm.h"
+#include "kvm/clock.h"
+#include "q35.h"
+#include "exec-memory.h"
+#include "ich9.h"
+#include "hw/ide/pci.h"
+#include "hw/ide/ahci.h"
+#include "hw/usb.h"
+
+/* ICH9 AHCI has 6 ports */
+#define MAX_SATA_PORTS     6
+
+/* set CMOS shutdown status register (index 0xF) as S3_resume(0xFE)
+ *    BIOS will read it and start S3 resume at POST Entry */
+static void pc_cmos_set_s3_resume(void *opaque, int irq, int level)
+{
+    ISADevice *s = opaque;
+
+    if (level) {
+        rtc_set_memory(s, 0xF, 0xFE);
+    }
+}
+
+/* PC hardware initialisation */
+static void pc_q35_init(QEMUMachineInitArgs *args)
+{
+    ram_addr_t ram_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
+    const char *boot_device = args->boot_device;
+    ram_addr_t below_4g_mem_size, above_4g_mem_size;
+    Q35PCIHost *q35_host;
+    PCIBus *host_bus;
+    PCIDevice *lpc;
+    BusState *idebus[MAX_SATA_PORTS];
+    ISADevice *rtc_state;
+    ISADevice *floppy;
+    MemoryRegion *pci_memory;
+    MemoryRegion *rom_memory;
+    MemoryRegion *ram_memory;
+    GSIState *gsi_state;
+    ISABus *isa_bus;
+    int pci_enabled = 1;
+    qemu_irq *cpu_irq;
+    qemu_irq *gsi;
+    qemu_irq *i8259;
+    int i;
+    ICH9LPCState *ich9_lpc;
+    PCIDevice *ahci;
+    qemu_irq *cmos_s3;
+
+    pc_cpus_init(cpu_model);
+
+    kvmclock_create();
+
+    if (ram_size >= 0xb0000000) {
+        above_4g_mem_size = ram_size - 0xb0000000;
+        below_4g_mem_size = 0xb0000000;
+    } else {
+        above_4g_mem_size = 0;
+        below_4g_mem_size = ram_size;
+    }
+
+    /* pci enabled */
+    if (pci_enabled) {
+        pci_memory = g_new(MemoryRegion, 1);
+        memory_region_init(pci_memory, "pci", INT64_MAX);
+        rom_memory = pci_memory;
+    } else {
+        pci_memory = NULL;
+        rom_memory = get_system_memory();
+    }
+
+    /* allocate ram and load rom/bios */
+    if (!xen_enabled()) {
+        pc_memory_init(get_system_memory(), kernel_filename, kernel_cmdline,
+                       initrd_filename, below_4g_mem_size, above_4g_mem_size,
+                       rom_memory, &ram_memory);
+    }
+
+    /* irq lines */
+    gsi_state = g_malloc0(sizeof(*gsi_state));
+    if (kvm_irqchip_in_kernel()) {
+        kvm_pc_setup_irq_routing(pci_enabled);
+        gsi = qemu_allocate_irqs(kvm_pc_gsi_handler, gsi_state,
+                                 GSI_NUM_PINS);
+    } else {
+        gsi = qemu_allocate_irqs(gsi_handler, gsi_state, GSI_NUM_PINS);
+    }
+
+    /* create pci host bus */
+    q35_host = Q35_HOST_DEVICE(qdev_create(NULL, TYPE_Q35_HOST_DEVICE));
+
+    q35_host->mch.ram_memory = ram_memory;
+    q35_host->mch.pci_address_space = pci_memory;
+    q35_host->mch.system_memory = get_system_memory();
+    q35_host->mch.address_space_io = get_system_io();;
+    q35_host->mch.below_4g_mem_size = below_4g_mem_size;
+    q35_host->mch.above_4g_mem_size = above_4g_mem_size;
+    /* pci */
+    qdev_init_nofail(DEVICE(q35_host));
+    host_bus = q35_host->host.pci.bus;
+    /* create ISA bus */
+    lpc = pci_create_simple_multifunction(host_bus, PCI_DEVFN(ICH9_LPC_DEV,
+                                          ICH9_LPC_FUNC), true,
+                                          TYPE_ICH9_LPC_DEVICE);
+    ich9_lpc = ICH9_LPC_DEVICE(lpc);
+    ich9_lpc->pic = gsi;
+    ich9_lpc->ioapic = gsi_state->ioapic_irq;
+    pci_bus_irqs(host_bus, ich9_lpc_set_irq, ich9_lpc_map_irq, ich9_lpc,
+                 ICH9_LPC_NB_PIRQS);
+    isa_bus = ich9_lpc->isa_bus;
+
+    /*end early*/
+    isa_bus_irqs(isa_bus, gsi);
+
+    if (kvm_irqchip_in_kernel()) {
+        i8259 = kvm_i8259_init(isa_bus);
+    } else if (xen_enabled()) {
+        i8259 = xen_interrupt_controller_init();
+    } else {
+        cpu_irq = pc_allocate_cpu_irq();
+        i8259 = i8259_init(isa_bus, cpu_irq[0]);
+    }
+
+    for (i = 0; i < ISA_NUM_IRQS; i++) {
+        gsi_state->i8259_irq[i] = i8259[i];
+    }
+    if (pci_enabled) {
+        ioapic_init_gsi(gsi_state, NULL);
+    }
+
+    pc_register_ferr_irq(gsi[13]);
+
+    /* init basic PC hardware */
+    pc_basic_device_init(isa_bus, gsi, &rtc_state, &floppy, false);
+
+    /* connect pm stuff to lpc */
+    cmos_s3 = qemu_allocate_irqs(pc_cmos_set_s3_resume, rtc_state, 1);
+    ich9_lpc_pm_init(lpc, *cmos_s3);
+
+    /* ahci and SATA device, for q35 1 ahci controller is built-in */
+    ahci = pci_create_simple_multifunction(host_bus,
+                                           PCI_DEVFN(ICH9_SATA1_DEV,
+                                                     ICH9_SATA1_FUNC),
+                                           true, "ich9-ahci");
+    idebus[0] = qdev_get_child_bus(&ahci->qdev, "ide.0");
+    idebus[1] = qdev_get_child_bus(&ahci->qdev, "ide.1");
+
+    if (usb_enabled(false)) {
+        /* Should we create 6 UHCI according to ich9 spec? */
+        ehci_create_ich9_with_companions(host_bus, 0x1d);
+    }
+
+    /* TODO: Populate SPD eeprom data.  */
+    smbus_eeprom_init(ich9_smb_init(host_bus,
+                                    PCI_DEVFN(ICH9_SMB_DEV, ICH9_SMB_FUNC),
+                                    0xb100),
+                      8, NULL, 0);
+
+    pc_cmos_init(below_4g_mem_size, above_4g_mem_size, boot_device,
+                 floppy, idebus[0], idebus[1], rtc_state);
+
+    /* the rest devices to which pci devfn is automatically assigned */
+    pc_vga_init(isa_bus, host_bus);
+    audio_init(isa_bus, host_bus);
+    pc_nic_init(isa_bus, host_bus);
+    if (pci_enabled) {
+        pc_pci_device_init(host_bus);
+    }
+}
+
+static QEMUMachine pc_q35_machine = {
+    .name = "q35-next",
+    .alias = "q35",
+    .desc = "Q35 chipset PC",
+    .init = pc_q35_init,
+    .max_cpus = 255,
+};
+
+static void pc_q35_machine_init(void)
+{
+    qemu_register_machine(&pc_q35_machine);
+}
+
+machine_init(pc_q35_machine_init);
diff --git a/hw/pc_sysfw.c b/hw/pc_sysfw.c
index b45f0acc7..9d7c5f400 100644
--- a/hw/pc_sysfw.c
+++ b/hw/pc_sysfw.c
@@ -106,7 +106,7 @@ static void pc_system_flash_init(MemoryRegion *rom_memory,
 {
     BlockDriverState *bdrv;
     int64_t size;
-    target_phys_addr_t phys_addr;
+    hwaddr phys_addr;
     int sector_bits, sector_size;
     pflash_t *system_flash;
     MemoryRegion *flash_mem;
diff --git a/hw/pci-hotplug.c b/hw/pci-hotplug.c
index e7fb780a0..0ca5546fc 100644
--- a/hw/pci-hotplug.c
+++ b/hw/pci-hotplug.c
@@ -80,7 +80,13 @@ static int scsi_hot_add(Monitor *mon, DeviceState *adapter,
     SCSIBus *scsibus;
     SCSIDevice *scsidev;
 
-    scsibus = SCSI_BUS(QLIST_FIRST(&adapter->child_bus));
+    scsibus = (SCSIBus *)
+        object_dynamic_cast(OBJECT(QLIST_FIRST(&adapter->child_bus)),
+                            TYPE_SCSI_BUS);
+    if (!scsibus) {
+	error_report("Device is not a SCSI adapter");
+	return -1;
+    }
 
     /*
      * drive_init() tries to find a default for dinfo->unit.  Doesn't
diff --git a/hw/pci.c b/hw/pci.c
index 4d9598480..97a0cd77c 100644
--- a/hw/pci.c
+++ b/hw/pci.c
@@ -33,6 +33,7 @@
 #include "qmp-commands.h"
 #include "msi.h"
 #include "msix.h"
+#include "exec-memory.h"
 
 //#define DEBUG_PCI
 #ifdef DEBUG_PCI
@@ -300,9 +301,9 @@ PCIBus *pci_bus_new(DeviceState *parent, const char *name,
     PCIBus *bus;
 
     bus = g_malloc0(sizeof(*bus));
-    bus->qbus.glib_allocated = true;
     pci_bus_new_inplace(bus, parent, name, address_space_mem,
                         address_space_io, devfn_min);
+    OBJECT(bus)->free = g_free;
     return bus;
 }
 
@@ -366,6 +367,10 @@ static int get_pci_config_device(QEMUFile *f, void *pv, size_t size)
 
     pci_update_mappings(s);
 
+    memory_region_set_enabled(&s->bus_master_enable_region,
+                              pci_get_word(s->config + PCI_COMMAND)
+                              & PCI_COMMAND_MASTER);
+
     g_free(config);
     return 0;
 }
@@ -439,7 +444,7 @@ const VMStateDescription vmstate_pci_device = {
 };
 
 const VMStateDescription vmstate_pcie_device = {
-    .name = "PCIDevice",
+    .name = "PCIEDevice",
     .version_id = 2,
     .minimum_version_id = 1,
     .minimum_version_id_old = 1,
@@ -777,6 +782,17 @@ static PCIDevice *do_pci_register_device(PCIDevice *pci_dev, PCIBus *bus,
     pci_dev->bus = bus;
     if (bus->dma_context_fn) {
         pci_dev->dma = bus->dma_context_fn(bus, bus->dma_context_opaque, devfn);
+    } else {
+        /* FIXME: Make dma_context_fn use MemoryRegions instead, so this path is
+         * taken unconditionally */
+        /* FIXME: inherit memory region from bus creator */
+        memory_region_init_alias(&pci_dev->bus_master_enable_region, "bus master",
+                                 get_system_memory(), 0,
+                                 memory_region_size(get_system_memory()));
+        memory_region_set_enabled(&pci_dev->bus_master_enable_region, false);
+        address_space_init(&pci_dev->bus_master_as, &pci_dev->bus_master_enable_region);
+        pci_dev->dma = g_new(DMAContext, 1);
+        dma_context_init(pci_dev->dma, &pci_dev->bus_master_as, NULL, NULL, NULL);
     }
     pci_dev->devfn = devfn;
     pstrcpy(pci_dev->name, sizeof(pci_dev->name), name);
@@ -830,6 +846,13 @@ static void do_pci_unregister_device(PCIDevice *pci_dev)
     qemu_free_irqs(pci_dev->irq);
     pci_dev->bus->devices[pci_dev->devfn] = NULL;
     pci_config_free(pci_dev);
+
+    if (!pci_dev->bus->dma_context_fn) {
+        address_space_destroy(&pci_dev->bus_master_as);
+        memory_region_destroy(&pci_dev->bus_master_enable_region);
+        g_free(pci_dev->dma);
+        pci_dev->dma = NULL;
+    }
 }
 
 static void pci_unregister_io_regions(PCIDevice *pci_dev)
@@ -968,7 +991,7 @@ static pcibus_t pci_bar_address(PCIDevice *d,
      * to >4G. Check it. TODO: we might need to support
      * it in the future for e.g. PAE.
      */
-    if (last_addr >= TARGET_PHYS_ADDR_MAX) {
+    if (last_addr >= HWADDR_MAX) {
         return PCI_BAR_UNMAPPED;
     }
 
@@ -1051,8 +1074,12 @@ void pci_default_write_config(PCIDevice *d, uint32_t addr, uint32_t val, int l)
         range_covers_byte(addr, l, PCI_COMMAND))
         pci_update_mappings(d);
 
-    if (range_covers_byte(addr, l, PCI_COMMAND))
+    if (range_covers_byte(addr, l, PCI_COMMAND)) {
         pci_update_irq_disabled(d, was_irq_disabled);
+        memory_region_set_enabled(&d->bus_master_enable_region,
+                                  pci_get_word(d->config + PCI_COMMAND)
+                                    & PCI_COMMAND_MASTER);
+    }
 
     msi_write_config(d, addr, val, l);
     msix_write_config(d, addr, val, l);
@@ -1094,10 +1121,21 @@ PCIINTxRoute pci_device_route_intx_to_irq(PCIDevice *dev, int pin)
          pin = bus->map_irq(dev, pin);
          dev = bus->parent_dev;
     } while (dev);
-    assert(bus->route_intx_to_irq);
+
+    if (!bus->route_intx_to_irq) {
+        error_report("PCI: Bug - unimplemented PCI INTx routing (%s)\n",
+                     object_get_typename(OBJECT(bus->qbus.parent)));
+        return (PCIINTxRoute) { PCI_INTX_DISABLED, -1 };
+    }
+
     return bus->route_intx_to_irq(bus->irq_opaque, pin);
 }
 
+bool pci_intx_route_changed(PCIINTxRoute *old, PCIINTxRoute *new)
+{
+    return old->mode != new->mode || old->irq != new->irq;
+}
+
 void pci_bus_fire_intx_routing_notifier(PCIBus *bus)
 {
     PCIDevice *dev;
@@ -1121,6 +1159,24 @@ void pci_device_set_intx_routing_notifier(PCIDevice *dev,
     dev->intx_routing_notifier = notifier;
 }
 
+/*
+ * PCI-to-PCI bridge specification
+ * 9.1: Interrupt routing. Table 9-1
+ *
+ * the PCI Express Base Specification, Revision 2.1
+ * 2.2.8.1: INTx interrutp signaling - Rules
+ *          the Implementation Note
+ *          Table 2-20
+ */
+/*
+ * 0 <= pin <= 3 0 = INTA, 1 = INTB, 2 = INTC, 3 = INTD
+ * 0-origin unlike PCI interrupt pin register.
+ */
+int pci_swizzle_map_irq_fn(PCIDevice *pci_dev, int pin)
+{
+    return (pin + PCI_SLOT(pci_dev->devfn)) % PCI_NUM_PINS;
+}
+
 /***********************************************************/
 /* monitor info on PCI */
 
@@ -1185,6 +1241,7 @@ static const pci_class_desc pci_class_descriptions[] =
     { 0x0c02, "SSA controller", "ssa"},
     { 0x0c03, "USB controller", "usb"},
     { 0x0c04, "Fibre channel controller", "fibre-channel"},
+    { 0x0c05, "SMBus"},
     { 0, NULL}
 };
 
@@ -1474,6 +1531,24 @@ PCIDevice *pci_nic_init_nofail(NICInfo *nd, const char *default_model,
     return res;
 }
 
+PCIDevice *pci_vga_init(PCIBus *bus)
+{
+    switch (vga_interface_type) {
+    case VGA_CIRRUS:
+        return pci_create_simple(bus, -1, "cirrus-vga");
+    case VGA_QXL:
+        return pci_create_simple(bus, -1, "qxl-vga");
+    case VGA_STD:
+        return pci_create_simple(bus, -1, "VGA");
+    case VGA_VMWARE:
+        return pci_create_simple(bus, -1, "vmware-svga");
+    case VGA_NONE:
+    default: /* Other non-PCI types. Checking for unsupported types is already
+                done in vl.c. */
+        return NULL;
+    }
+}
+
 /* Whether a given bus number is in range of the secondary
  * bus of the given bridge device. */
 static bool pci_secondary_bus_in_range(PCIDevice *dev, int bus_num)
@@ -1626,16 +1701,16 @@ PCIDevice *pci_create_simple(PCIBus *bus, int devfn, const char *name)
     return pci_create_simple_multifunction(bus, devfn, false, name);
 }
 
-static int pci_find_space(PCIDevice *pdev, uint8_t size)
+static uint8_t pci_find_space(PCIDevice *pdev, uint8_t size)
 {
-    int config_size = pci_config_size(pdev);
     int offset = PCI_CONFIG_HEADER_SIZE;
     int i;
-    for (i = PCI_CONFIG_HEADER_SIZE; i < config_size; ++i)
+    for (i = PCI_CONFIG_HEADER_SIZE; i < PCI_CONFIG_SPACE_SIZE; ++i) {
         if (pdev->used[i])
             offset = i + 1;
         else if (i - offset + 1 == size)
             return offset;
+    }
     return 0;
 }
 
@@ -1854,7 +1929,7 @@ int pci_add_capability(PCIDevice *pdev, uint8_t cap_id,
     config[PCI_CAP_LIST_NEXT] = pdev->config[PCI_CAPABILITY_LIST];
     pdev->config[PCI_CAPABILITY_LIST] = offset;
     pdev->config[PCI_STATUS] |= PCI_STATUS_CAP_LIST;
-    memset(pdev->used + offset, 0xFF, size);
+    memset(pdev->used + offset, 0xFF, QEMU_ALIGN_UP(size, 4));
     /* Make capability read-only by default */
     memset(pdev->wmask + offset, 0, size);
     /* Check capability by default */
@@ -1874,7 +1949,7 @@ void pci_del_capability(PCIDevice *pdev, uint8_t cap_id, uint8_t size)
     memset(pdev->w1cmask + offset, 0, size);
     /* Clear cmask as device-specific registers can't be checked */
     memset(pdev->cmask + offset, 0, size);
-    memset(pdev->used + offset, 0, size);
+    memset(pdev->used + offset, 0, QEMU_ALIGN_UP(size, 4));
 
     if (!pdev->config[PCI_CAPABILITY_LIST])
         pdev->config[PCI_STATUS] &= ~PCI_STATUS_CAP_LIST;
@@ -1962,7 +2037,7 @@ static char *pcibus_get_fw_dev_path(DeviceState *dev)
                    PCI_SLOT(d->devfn));
     if (PCI_FUNC(d->devfn))
         snprintf(path + off, sizeof(path) + off, ",%x", PCI_FUNC(d->devfn));
-    return strdup(path);
+    return g_strdup(path);
 }
 
 static char *pcibus_get_dev_path(DeviceState *dev)
diff --git a/hw/pci.h b/hw/pci.h
index 4b6ab3d19..4da0c2a4c 100644
--- a/hw/pci.h
+++ b/hw/pci.h
@@ -76,6 +76,7 @@
 #define PCI_DEVICE_ID_VIRTIO_BALLOON     0x1002
 #define PCI_DEVICE_ID_VIRTIO_CONSOLE     0x1003
 #define PCI_DEVICE_ID_VIRTIO_SCSI        0x1004
+#define PCI_DEVICE_ID_VIRTIO_RNG         0x1005
 
 #define FMT_PCIBUS                      PRIx64
 
@@ -211,6 +212,8 @@ struct PCIDevice {
     int32_t devfn;
     char name[64];
     PCIIORegion io_regions[PCI_NUM_REGIONS];
+    AddressSpace bus_master_as;
+    MemoryRegion bus_master_enable_region;
     DMAContext *dma;
 
     /* do not access the following fields */
@@ -316,6 +319,8 @@ void pci_bus_irqs(PCIBus *bus, pci_set_irq_fn set_irq, pci_map_irq_fn map_irq,
                   void *irq_opaque, int nirq);
 int pci_bus_get_irq_level(PCIBus *bus, int irq_num);
 void pci_bus_hotplug(PCIBus *bus, pci_hotplug_fn hotplug, DeviceState *dev);
+/* 0 <= pin <= 3 0 = INTA, 1 = INTB, 2 = INTC, 3 = INTD */
+int pci_swizzle_map_irq_fn(PCIDevice *pci_dev, int pin);
 PCIBus *pci_register_bus(DeviceState *parent, const char *name,
                          pci_set_irq_fn set_irq, pci_map_irq_fn map_irq,
                          void *irq_opaque,
@@ -324,6 +329,7 @@ PCIBus *pci_register_bus(DeviceState *parent, const char *name,
                          uint8_t devfn_min, int nirq);
 void pci_bus_set_route_irq_fn(PCIBus *, pci_route_irq_fn);
 PCIINTxRoute pci_device_route_intx_to_irq(PCIDevice *dev, int pin);
+bool pci_intx_route_changed(PCIINTxRoute *old, PCIINTxRoute *new);
 void pci_bus_fire_intx_routing_notifier(PCIBus *bus);
 void pci_device_set_intx_routing_notifier(PCIDevice *dev,
                                           PCIINTxRoutingNotifier notifier);
@@ -334,6 +340,9 @@ PCIDevice *pci_nic_init(NICInfo *nd, const char *default_model,
                         const char *default_devaddr);
 PCIDevice *pci_nic_init_nofail(NICInfo *nd, const char *default_model,
                                const char *default_devaddr);
+
+PCIDevice *pci_vga_init(PCIBus *bus);
+
 int pci_bus_num(PCIBus *s);
 void pci_for_each_device(PCIBus *bus, int bus_num,
                          void (*fn)(PCIBus *bus, PCIDevice *d, void *opaque),
diff --git a/hw/pci_bridge.c b/hw/pci_bridge.c
index 5c6455f6f..4680501e4 100644
--- a/hw/pci_bridge.c
+++ b/hw/pci_bridge.c
@@ -151,58 +151,63 @@ static void pci_bridge_init_alias(PCIBridge *bridge, MemoryRegion *alias,
     memory_region_add_subregion_overlap(parent_space, base, alias, 1);
 }
 
-static void pci_bridge_cleanup_alias(MemoryRegion *alias,
-                                     MemoryRegion *parent_space)
-{
-    memory_region_del_subregion(parent_space, alias);
-    memory_region_destroy(alias);
-}
-
-static void pci_bridge_region_init(PCIBridge *br)
+static PCIBridgeWindows *pci_bridge_region_init(PCIBridge *br)
 {
     PCIBus *parent = br->dev.bus;
+    PCIBridgeWindows *w = g_new(PCIBridgeWindows, 1);
     uint16_t cmd = pci_get_word(br->dev.config + PCI_COMMAND);
 
-    pci_bridge_init_alias(br, &br->alias_pref_mem,
+    pci_bridge_init_alias(br, &w->alias_pref_mem,
                           PCI_BASE_ADDRESS_MEM_PREFETCH,
                           "pci_bridge_pref_mem",
                           &br->address_space_mem,
                           parent->address_space_mem,
                           cmd & PCI_COMMAND_MEMORY);
-    pci_bridge_init_alias(br, &br->alias_mem,
+    pci_bridge_init_alias(br, &w->alias_mem,
                           PCI_BASE_ADDRESS_SPACE_MEMORY,
                           "pci_bridge_mem",
                           &br->address_space_mem,
                           parent->address_space_mem,
                           cmd & PCI_COMMAND_MEMORY);
-    pci_bridge_init_alias(br, &br->alias_io,
+    pci_bridge_init_alias(br, &w->alias_io,
                           PCI_BASE_ADDRESS_SPACE_IO,
                           "pci_bridge_io",
                           &br->address_space_io,
                           parent->address_space_io,
                           cmd & PCI_COMMAND_IO);
    /* TODO: optinal VGA and VGA palette snooping support. */
+
+    return w;
 }
 
-static void pci_bridge_region_cleanup(PCIBridge *br)
+static void pci_bridge_region_del(PCIBridge *br, PCIBridgeWindows *w)
 {
     PCIBus *parent = br->dev.bus;
-    pci_bridge_cleanup_alias(&br->alias_io,
-                             parent->address_space_io);
-    pci_bridge_cleanup_alias(&br->alias_mem,
-                             parent->address_space_mem);
-    pci_bridge_cleanup_alias(&br->alias_pref_mem,
-                             parent->address_space_mem);
+
+    memory_region_del_subregion(parent->address_space_io, &w->alias_io);
+    memory_region_del_subregion(parent->address_space_mem, &w->alias_mem);
+    memory_region_del_subregion(parent->address_space_mem, &w->alias_pref_mem);
+}
+
+static void pci_bridge_region_cleanup(PCIBridge *br, PCIBridgeWindows *w)
+{
+    memory_region_destroy(&w->alias_io);
+    memory_region_destroy(&w->alias_mem);
+    memory_region_destroy(&w->alias_pref_mem);
+    g_free(w);
 }
 
 static void pci_bridge_update_mappings(PCIBridge *br)
 {
+    PCIBridgeWindows *w = br->windows;
+
     /* Make updates atomic to: handle the case of one VCPU updating the bridge
      * while another accesses an unaffected region. */
     memory_region_transaction_begin();
-    pci_bridge_region_cleanup(br);
-    pci_bridge_region_init(br);
+    pci_bridge_region_del(br, br->windows);
+    br->windows = pci_bridge_region_init(br);
     memory_region_transaction_commit();
+    pci_bridge_region_cleanup(br, w);
 }
 
 /* default write_config function for PCI-to-PCI bridge */
@@ -326,7 +331,7 @@ int pci_bridge_initfn(PCIDevice *dev)
     memory_region_init(&br->address_space_mem, "pci_bridge_pci", INT64_MAX);
     sec_bus->address_space_io = &br->address_space_io;
     memory_region_init(&br->address_space_io, "pci_bridge_io", 65536);
-    pci_bridge_region_init(br);
+    br->windows = pci_bridge_region_init(br);
     QLIST_INIT(&sec_bus->child);
     QLIST_INSERT_HEAD(&parent->child, sec_bus, sibling);
     return 0;
@@ -338,7 +343,8 @@ void pci_bridge_exitfn(PCIDevice *pci_dev)
     PCIBridge *s = DO_UPCAST(PCIBridge, dev, pci_dev);
     assert(QLIST_EMPTY(&s->sec_bus.child));
     QLIST_REMOVE(&s->sec_bus, sibling);
-    pci_bridge_region_cleanup(s);
+    pci_bridge_region_del(s, s->windows);
+    pci_bridge_region_cleanup(s, s->windows);
     memory_region_destroy(&s->address_space_mem);
     memory_region_destroy(&s->address_space_io);
     /* qbus_free() is called automatically by qdev_free() */
diff --git a/hw/pci_host.c b/hw/pci_host.c
index 3950e943c..68e328cd2 100644
--- a/hw/pci_host.c
+++ b/hw/pci_host.c
@@ -94,7 +94,7 @@ uint32_t pci_data_read(PCIBus *s, uint32_t addr, int len)
     return val;
 }
 
-static void pci_host_config_write(void *opaque, target_phys_addr_t addr,
+static void pci_host_config_write(void *opaque, hwaddr addr,
                                   uint64_t val, unsigned len)
 {
     PCIHostState *s = opaque;
@@ -107,7 +107,7 @@ static void pci_host_config_write(void *opaque, target_phys_addr_t addr,
     s->config_reg = val;
 }
 
-static uint64_t pci_host_config_read(void *opaque, target_phys_addr_t addr,
+static uint64_t pci_host_config_read(void *opaque, hwaddr addr,
                                      unsigned len)
 {
     PCIHostState *s = opaque;
@@ -118,7 +118,7 @@ static uint64_t pci_host_config_read(void *opaque, target_phys_addr_t addr,
     return val;
 }
 
-static void pci_host_data_write(void *opaque, target_phys_addr_t addr,
+static void pci_host_data_write(void *opaque, hwaddr addr,
                                 uint64_t val, unsigned len)
 {
     PCIHostState *s = opaque;
@@ -129,7 +129,7 @@ static void pci_host_data_write(void *opaque, target_phys_addr_t addr,
 }
 
 static uint64_t pci_host_data_read(void *opaque,
-                                   target_phys_addr_t addr, unsigned len)
+                                   hwaddr addr, unsigned len)
 {
     PCIHostState *s = opaque;
     uint32_t val;
diff --git a/hw/pci_ids.h b/hw/pci_ids.h
index 301bf1cd8..5df724534 100644
--- a/hw/pci_ids.h
+++ b/hw/pci_ids.h
@@ -31,12 +31,15 @@
 #define PCI_CLASS_SYSTEM_OTHER           0x0880
 
 #define PCI_CLASS_SERIAL_USB             0x0c03
+#define PCI_CLASS_SERIAL_SMBUS           0x0c05
 
 #define PCI_CLASS_BRIDGE_HOST            0x0600
 #define PCI_CLASS_BRIDGE_ISA             0x0601
 #define PCI_CLASS_BRIDGE_PCI             0x0604
+#define PCI_CLASS_BRDIGE_PCI_INF_SUB     0x01
 #define PCI_CLASS_BRIDGE_OTHER           0x0680
 
+#define PCI_CLASS_COMMUNICATION_SERIAL   0x0700
 #define PCI_CLASS_COMMUNICATION_OTHER    0x0780
 
 #define PCI_CLASS_PROCESSOR_CO           0x0b40
@@ -104,6 +107,7 @@
 #define PCI_DEVICE_ID_INTEL_82378        0x0484
 #define PCI_DEVICE_ID_INTEL_82441        0x1237
 #define PCI_DEVICE_ID_INTEL_82801AA_5    0x2415
+#define PCI_DEVICE_ID_INTEL_82801BA_11   0x244e
 #define PCI_DEVICE_ID_INTEL_82801D       0x24CD
 #define PCI_DEVICE_ID_INTEL_ESB_9        0x25ab
 #define PCI_DEVICE_ID_INTEL_82371SB_0    0x7000
@@ -113,6 +117,17 @@
 #define PCI_DEVICE_ID_INTEL_82371AB      0x7111
 #define PCI_DEVICE_ID_INTEL_82371AB_2    0x7112
 #define PCI_DEVICE_ID_INTEL_82371AB_3    0x7113
+
+#define PCI_DEVICE_ID_INTEL_ICH9_0       0x2910
+#define PCI_DEVICE_ID_INTEL_ICH9_1       0x2917
+#define PCI_DEVICE_ID_INTEL_ICH9_2       0x2912
+#define PCI_DEVICE_ID_INTEL_ICH9_3       0x2913
+#define PCI_DEVICE_ID_INTEL_ICH9_4       0x2914
+#define PCI_DEVICE_ID_INTEL_ICH9_5       0x2919
+#define PCI_DEVICE_ID_INTEL_ICH9_6       0x2930
+#define PCI_DEVICE_ID_INTEL_ICH9_7       0x2916
+#define PCI_DEVICE_ID_INTEL_ICH9_8       0x2918
+
 #define PCI_DEVICE_ID_INTEL_82801I_UHCI1 0x2934
 #define PCI_DEVICE_ID_INTEL_82801I_UHCI2 0x2935
 #define PCI_DEVICE_ID_INTEL_82801I_UHCI3 0x2936
@@ -123,6 +138,8 @@
 #define PCI_DEVICE_ID_INTEL_82801I_EHCI2 0x293c
 #define PCI_DEVICE_ID_INTEL_82599_SFP_VF 0x10ed
 
+#define PCI_DEVICE_ID_INTEL_Q35_MCH      0x29c0
+
 #define PCI_VENDOR_ID_XEN               0x5853
 #define PCI_DEVICE_ID_XEN_PLATFORM      0x0001
 
diff --git a/hw/pci_internals.h b/hw/pci_internals.h
index c931b64b4..21d0ce697 100644
--- a/hw/pci_internals.h
+++ b/hw/pci_internals.h
@@ -40,6 +40,19 @@ struct PCIBus {
     int *irq_count;
 };
 
+typedef struct PCIBridgeWindows PCIBridgeWindows;
+
+/*
+ * Aliases for each of the address space windows that the bridge
+ * can forward. Mapped into the bridge's parent's address space,
+ * as subregions.
+ */
+struct PCIBridgeWindows {
+    MemoryRegion alias_pref_mem;
+    MemoryRegion alias_mem;
+    MemoryRegion alias_io;
+};
+
 struct PCIBridge {
     PCIDevice dev;
 
@@ -55,14 +68,9 @@ struct PCIBridge {
      */
     MemoryRegion address_space_mem;
     MemoryRegion address_space_io;
-    /*
-     * Aliases for each of the address space windows that the bridge
-     * can forward. Mapped into the bridge's parent's address space,
-     * as subregions.
-     */
-    MemoryRegion alias_pref_mem;
-    MemoryRegion alias_mem;
-    MemoryRegion alias_io;
+
+    PCIBridgeWindows *windows;
+
     pci_map_irq_fn map_irq;
     const char *bus_name;
 };
diff --git a/hw/pcie.h b/hw/pcie.h
index b8ab0c7b4..4889194ae 100644
--- a/hw/pcie.h
+++ b/hw/pcie.h
@@ -133,7 +133,6 @@ extern const VMStateDescription vmstate_pcie_device;
 
 #define VMSTATE_PCIE_DEVICE(_field, _state) {                        \
     .name       = (stringify(_field)),                               \
-    .version_id = 2,                                                 \
     .size       = sizeof(PCIDevice),                                 \
     .vmsd       = &vmstate_pcie_device,                              \
     .flags      = VMS_STRUCT,                                        \
diff --git a/hw/pcie_aer.c b/hw/pcie_aer.c
index 3b6981c7b..b04c164e2 100644
--- a/hw/pcie_aer.c
+++ b/hw/pcie_aer.c
@@ -738,6 +738,11 @@ void pcie_aer_root_init(PCIDevice *dev)
                  PCI_ERR_ROOT_CMD_EN_MASK);
     pci_set_long(dev->w1cmask + pos + PCI_ERR_ROOT_STATUS,
                  PCI_ERR_ROOT_STATUS_REPORT_MASK);
+    /* PCI_ERR_ROOT_IRQ is RO but devices change it using a
+     * device-specific method.
+     */
+    pci_set_long(dev->cmask + pos + PCI_ERR_ROOT_STATUS,
+                 ~PCI_ERR_ROOT_IRQ);
 }
 
 void pcie_aer_root_reset(PCIDevice *dev)
diff --git a/hw/pcie_host.c b/hw/pcie_host.c
index 28bbe72b3..c257fb43c 100644
--- a/hw/pcie_host.c
+++ b/hw/pcie_host.c
@@ -53,7 +53,7 @@ static inline PCIDevice *pcie_dev_find_by_mmcfg_addr(PCIBus *s,
                            PCIE_MMCFG_DEVFN(mmcfg_addr));
 }
 
-static void pcie_mmcfg_data_write(void *opaque, target_phys_addr_t mmcfg_addr,
+static void pcie_mmcfg_data_write(void *opaque, hwaddr mmcfg_addr,
                                   uint64_t val, unsigned len)
 {
     PCIExpressHost *e = opaque;
@@ -76,7 +76,7 @@ static void pcie_mmcfg_data_write(void *opaque, target_phys_addr_t mmcfg_addr,
 }
 
 static uint64_t pcie_mmcfg_data_read(void *opaque,
-                                     target_phys_addr_t mmcfg_addr,
+                                     hwaddr mmcfg_addr,
                                      unsigned len)
 {
     PCIExpressHost *e = opaque;
@@ -105,16 +105,11 @@ static const MemoryRegionOps pcie_mmcfg_ops = {
 };
 
 /* pcie_host::base_addr == PCIE_BASE_ADDR_UNMAPPED when it isn't mapped. */
-#define PCIE_BASE_ADDR_UNMAPPED  ((target_phys_addr_t)-1ULL)
+#define PCIE_BASE_ADDR_UNMAPPED  ((hwaddr)-1ULL)
 
-int pcie_host_init(PCIExpressHost *e, uint32_t size)
+int pcie_host_init(PCIExpressHost *e)
 {
-    assert(!(size & (size - 1)));       /* power of 2 */
-    assert(size >= PCIE_MMCFG_SIZE_MIN);
-    assert(size <= PCIE_MMCFG_SIZE_MAX);
     e->base_addr = PCIE_BASE_ADDR_UNMAPPED;
-    e->size = size;
-    memory_region_init_io(&e->mmio, &pcie_mmcfg_ops, e, "pcie-mmcfg", e->size);
 
     return 0;
 }
@@ -123,22 +118,44 @@ void pcie_host_mmcfg_unmap(PCIExpressHost *e)
 {
     if (e->base_addr != PCIE_BASE_ADDR_UNMAPPED) {
         memory_region_del_subregion(get_system_memory(), &e->mmio);
+        memory_region_destroy(&e->mmio);
         e->base_addr = PCIE_BASE_ADDR_UNMAPPED;
     }
 }
 
-void pcie_host_mmcfg_map(PCIExpressHost *e, target_phys_addr_t addr)
+void pcie_host_mmcfg_map(PCIExpressHost *e, hwaddr addr,
+                         uint32_t size)
 {
+    assert(!(size & (size - 1)));       /* power of 2 */
+    assert(size >= PCIE_MMCFG_SIZE_MIN);
+    assert(size <= PCIE_MMCFG_SIZE_MAX);
+    e->size = size;
+    memory_region_init_io(&e->mmio, &pcie_mmcfg_ops, e, "pcie-mmcfg", e->size);
     e->base_addr = addr;
     memory_region_add_subregion(get_system_memory(), e->base_addr, &e->mmio);
 }
 
 void pcie_host_mmcfg_update(PCIExpressHost *e,
                             int enable,
-                            target_phys_addr_t addr)
+                            hwaddr addr,
+                            uint32_t size)
 {
     pcie_host_mmcfg_unmap(e);
     if (enable) {
-        pcie_host_mmcfg_map(e, addr);
+        pcie_host_mmcfg_map(e, addr, size);
     }
 }
+
+static const TypeInfo pcie_host_type_info = {
+    .name = TYPE_PCIE_HOST_BRIDGE,
+    .parent = TYPE_PCI_HOST_BRIDGE,
+    .abstract = true,
+    .instance_size = sizeof(PCIExpressHost),
+};
+
+static void pcie_host_register_types(void)
+{
+    type_register_static(&pcie_host_type_info);
+}
+
+type_init(pcie_host_register_types)
diff --git a/hw/pcie_host.h b/hw/pcie_host.h
index 0074508b4..392193530 100644
--- a/hw/pcie_host.h
+++ b/hw/pcie_host.h
@@ -24,26 +24,31 @@
 #include "pci_host.h"
 #include "memory.h"
 
+#define TYPE_PCIE_HOST_BRIDGE "pcie-host-bridge"
+#define PCIE_HOST_BRIDGE(obj) \
+    OBJECT_CHECK(PCIExpressHost, (obj), TYPE_PCIE_HOST_BRIDGE)
+
 struct PCIExpressHost {
     PCIHostState pci;
 
     /* express part */
 
     /* base address where MMCONFIG area is mapped. */
-    target_phys_addr_t  base_addr;
+    hwaddr  base_addr;
 
     /* the size of MMCONFIG area. It's host bridge dependent */
-    target_phys_addr_t  size;
+    hwaddr  size;
 
     /* MMCONFIG mmio area */
     MemoryRegion mmio;
 };
 
-int pcie_host_init(PCIExpressHost *e, uint32_t size);
+int pcie_host_init(PCIExpressHost *e);
 void pcie_host_mmcfg_unmap(PCIExpressHost *e);
-void pcie_host_mmcfg_map(PCIExpressHost *e, target_phys_addr_t addr);
+void pcie_host_mmcfg_map(PCIExpressHost *e, hwaddr addr, uint32_t size);
 void pcie_host_mmcfg_update(PCIExpressHost *e,
                             int enable,
-                            target_phys_addr_t addr);
+                            hwaddr addr,
+                            uint32_t size);
 
 #endif /* PCIE_HOST_H */
diff --git a/hw/pckbd.c b/hw/pckbd.c
index 69857bade..5bb3e0abf 100644
--- a/hw/pckbd.c
+++ b/hw/pckbd.c
@@ -139,7 +139,7 @@ typedef struct KBDState {
     qemu_irq irq_kbd;
     qemu_irq irq_mouse;
     qemu_irq *a20_out;
-    target_phys_addr_t mask;
+    hwaddr mask;
 } KBDState;
 
 /* update irq and KBD_STAT_[MOUSE_]OBF */
@@ -194,7 +194,8 @@ static void kbd_update_aux_irq(void *opaque, int level)
     kbd_update_irq(s);
 }
 
-static uint32_t kbd_read_status(void *opaque, uint32_t addr)
+static uint64_t kbd_read_status(void *opaque, hwaddr addr,
+                                unsigned size)
 {
     KBDState *s = opaque;
     int val;
@@ -223,7 +224,8 @@ static void outport_write(KBDState *s, uint32_t val)
     }
 }
 
-static void kbd_write_command(void *opaque, uint32_t addr, uint32_t val)
+static void kbd_write_command(void *opaque, hwaddr addr,
+                              uint64_t val, unsigned size)
 {
     KBDState *s = opaque;
 
@@ -303,12 +305,13 @@ static void kbd_write_command(void *opaque, uint32_t addr, uint32_t val)
         /* ignore that */
         break;
     default:
-        fprintf(stderr, "qemu: unsupported keyboard cmd=0x%02x\n", val);
+        fprintf(stderr, "qemu: unsupported keyboard cmd=0x%02x\n", (int)val);
         break;
     }
 }
 
-static uint32_t kbd_read_data(void *opaque, uint32_t addr)
+static uint64_t kbd_read_data(void *opaque, hwaddr addr,
+                              unsigned size)
 {
     KBDState *s = opaque;
     uint32_t val;
@@ -322,7 +325,8 @@ static uint32_t kbd_read_data(void *opaque, uint32_t addr)
     return val;
 }
 
-static void kbd_write_data(void *opaque, uint32_t addr, uint32_t val)
+static void kbd_write_data(void *opaque, hwaddr addr,
+                           uint64_t val, unsigned size)
 {
     KBDState *s = opaque;
 
@@ -380,24 +384,24 @@ static const VMStateDescription vmstate_kbd = {
 };
 
 /* Memory mapped interface */
-static uint32_t kbd_mm_readb (void *opaque, target_phys_addr_t addr)
+static uint32_t kbd_mm_readb (void *opaque, hwaddr addr)
 {
     KBDState *s = opaque;
 
     if (addr & s->mask)
-        return kbd_read_status(s, 0) & 0xff;
+        return kbd_read_status(s, 0, 1) & 0xff;
     else
-        return kbd_read_data(s, 0) & 0xff;
+        return kbd_read_data(s, 0, 1) & 0xff;
 }
 
-static void kbd_mm_writeb (void *opaque, target_phys_addr_t addr, uint32_t value)
+static void kbd_mm_writeb (void *opaque, hwaddr addr, uint32_t value)
 {
     KBDState *s = opaque;
 
     if (addr & s->mask)
-        kbd_write_command(s, 0, value & 0xff);
+        kbd_write_command(s, 0, value & 0xff, 1);
     else
-        kbd_write_data(s, 0, value & 0xff);
+        kbd_write_data(s, 0, value & 0xff, 1);
 }
 
 static const MemoryRegionOps i8042_mmio_ops = {
@@ -410,7 +414,7 @@ static const MemoryRegionOps i8042_mmio_ops = {
 
 void i8042_mm_init(qemu_irq kbd_irq, qemu_irq mouse_irq,
                    MemoryRegion *region, ram_addr_t size,
-                   target_phys_addr_t mask)
+                   hwaddr mask)
 {
     KBDState *s = g_malloc0(sizeof(KBDState));
 
@@ -459,22 +463,24 @@ static const VMStateDescription vmstate_kbd_isa = {
     }
 };
 
-static const MemoryRegionPortio i8042_data_portio[] = {
-    { 0, 1, 1, .read = kbd_read_data, .write = kbd_write_data },
-    PORTIO_END_OF_LIST()
-};
-
-static const MemoryRegionPortio i8042_cmd_portio[] = {
-    { 0, 1, 1, .read = kbd_read_status, .write = kbd_write_command },
-    PORTIO_END_OF_LIST()
-};
-
 static const MemoryRegionOps i8042_data_ops = {
-    .old_portio = i8042_data_portio
+    .read = kbd_read_data,
+    .write = kbd_write_data,
+    .impl = {
+        .min_access_size = 1,
+        .max_access_size = 1,
+    },
+    .endianness = DEVICE_LITTLE_ENDIAN,
 };
 
 static const MemoryRegionOps i8042_cmd_ops = {
-    .old_portio = i8042_cmd_portio
+    .read = kbd_read_status,
+    .write = kbd_write_command,
+    .impl = {
+        .min_access_size = 1,
+        .max_access_size = 1,
+    },
+    .endianness = DEVICE_LITTLE_ENDIAN,
 };
 
 static int i8042_initfn(ISADevice *dev)
diff --git a/hw/pcnet-pci.c b/hw/pcnet-pci.c
index 48fd44799..0bf438ffe 100644
--- a/hw/pcnet-pci.c
+++ b/hw/pcnet-pci.c
@@ -71,7 +71,7 @@ static uint32_t pcnet_aprom_readb(void *opaque, uint32_t addr)
     return val;
 }
 
-static uint64_t pcnet_ioport_read(void *opaque, target_phys_addr_t addr,
+static uint64_t pcnet_ioport_read(void *opaque, hwaddr addr,
                                   unsigned size)
 {
     PCNetState *d = opaque;
@@ -98,7 +98,7 @@ static uint64_t pcnet_ioport_read(void *opaque, target_phys_addr_t addr,
     return ((uint64_t)1 << (size * 8)) - 1;
 }
 
-static void pcnet_ioport_write(void *opaque, target_phys_addr_t addr,
+static void pcnet_ioport_write(void *opaque, hwaddr addr,
                                uint64_t data, unsigned size)
 {
     PCNetState *d = opaque;
@@ -130,7 +130,7 @@ static const MemoryRegionOps pcnet_io_ops = {
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-static void pcnet_mmio_writeb(void *opaque, target_phys_addr_t addr, uint32_t val)
+static void pcnet_mmio_writeb(void *opaque, hwaddr addr, uint32_t val)
 {
     PCNetState *d = opaque;
 #ifdef PCNET_DEBUG_IO
@@ -141,7 +141,7 @@ static void pcnet_mmio_writeb(void *opaque, target_phys_addr_t addr, uint32_t va
         pcnet_aprom_writeb(d, addr & 0x0f, val);
 }
 
-static uint32_t pcnet_mmio_readb(void *opaque, target_phys_addr_t addr)
+static uint32_t pcnet_mmio_readb(void *opaque, hwaddr addr)
 {
     PCNetState *d = opaque;
     uint32_t val = -1;
@@ -154,7 +154,7 @@ static uint32_t pcnet_mmio_readb(void *opaque, target_phys_addr_t addr)
     return val;
 }
 
-static void pcnet_mmio_writew(void *opaque, target_phys_addr_t addr, uint32_t val)
+static void pcnet_mmio_writew(void *opaque, hwaddr addr, uint32_t val)
 {
     PCNetState *d = opaque;
 #ifdef PCNET_DEBUG_IO
@@ -170,7 +170,7 @@ static void pcnet_mmio_writew(void *opaque, target_phys_addr_t addr, uint32_t va
     }
 }
 
-static uint32_t pcnet_mmio_readw(void *opaque, target_phys_addr_t addr)
+static uint32_t pcnet_mmio_readw(void *opaque, hwaddr addr)
 {
     PCNetState *d = opaque;
     uint32_t val = -1;
@@ -189,7 +189,7 @@ static uint32_t pcnet_mmio_readw(void *opaque, target_phys_addr_t addr)
     return val;
 }
 
-static void pcnet_mmio_writel(void *opaque, target_phys_addr_t addr, uint32_t val)
+static void pcnet_mmio_writel(void *opaque, hwaddr addr, uint32_t val)
 {
     PCNetState *d = opaque;
 #ifdef PCNET_DEBUG_IO
@@ -207,7 +207,7 @@ static void pcnet_mmio_writel(void *opaque, target_phys_addr_t addr, uint32_t va
     }
 }
 
-static uint32_t pcnet_mmio_readl(void *opaque, target_phys_addr_t addr)
+static uint32_t pcnet_mmio_readl(void *opaque, hwaddr addr)
 {
     PCNetState *d = opaque;
     uint32_t val;
@@ -252,13 +252,13 @@ static const MemoryRegionOps pcnet_mmio_ops = {
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-static void pci_physical_memory_write(void *dma_opaque, target_phys_addr_t addr,
+static void pci_physical_memory_write(void *dma_opaque, hwaddr addr,
                                       uint8_t *buf, int len, int do_bswap)
 {
     pci_dma_write(dma_opaque, addr, buf, len);
 }
 
-static void pci_physical_memory_read(void *dma_opaque, target_phys_addr_t addr,
+static void pci_physical_memory_read(void *dma_opaque, hwaddr addr,
                                      uint8_t *buf, int len, int do_bswap)
 {
     pci_dma_read(dma_opaque, addr, buf, len);
diff --git a/hw/pcnet.c b/hw/pcnet.c
index 40820b363..54eecd01d 100644
--- a/hw/pcnet.c
+++ b/hw/pcnet.c
@@ -293,7 +293,7 @@ struct pcnet_RMD {
         GET_FIELD((R)->msg_length, RMDM, ZEROS))
 
 static inline void pcnet_tmd_load(PCNetState *s, struct pcnet_TMD *tmd,
-                                  target_phys_addr_t addr)
+                                  hwaddr addr)
 {
     if (!BCR_SSIZE32(s)) {
         struct {
@@ -323,7 +323,7 @@ static inline void pcnet_tmd_load(PCNetState *s, struct pcnet_TMD *tmd,
 }
 
 static inline void pcnet_tmd_store(PCNetState *s, const struct pcnet_TMD *tmd,
-                                   target_phys_addr_t addr)
+                                   hwaddr addr)
 {
     if (!BCR_SSIZE32(s)) {
         struct {
@@ -359,7 +359,7 @@ static inline void pcnet_tmd_store(PCNetState *s, const struct pcnet_TMD *tmd,
 }
 
 static inline void pcnet_rmd_load(PCNetState *s, struct pcnet_RMD *rmd,
-                                  target_phys_addr_t addr)
+                                  hwaddr addr)
 {
     if (!BCR_SSIZE32(s)) {
         struct {
@@ -389,7 +389,7 @@ static inline void pcnet_rmd_load(PCNetState *s, struct pcnet_RMD *rmd,
 }
 
 static inline void pcnet_rmd_store(PCNetState *s, struct pcnet_RMD *rmd,
-                                   target_phys_addr_t addr)
+                                   hwaddr addr)
 {
     if (!BCR_SSIZE32(s)) {
         struct {
@@ -660,7 +660,7 @@ static inline int ladr_match(PCNetState *s, const uint8_t *buf, int size)
     return 0;
 }
 
-static inline target_phys_addr_t pcnet_rdra_addr(PCNetState *s, int idx)
+static inline hwaddr pcnet_rdra_addr(PCNetState *s, int idx)
 {
     while (idx < 1) idx += CSR_RCVRL(s);
     return s->rdra + ((CSR_RCVRL(s) - idx) * (BCR_SWSTYLE(s) ? 16 : 8));
@@ -898,19 +898,19 @@ static void pcnet_rdte_poll(PCNetState *s)
     if (s->rdra) {
         int bad = 0;
 #if 1
-        target_phys_addr_t crda = pcnet_rdra_addr(s, CSR_RCVRC(s));
-        target_phys_addr_t nrda = pcnet_rdra_addr(s, -1 + CSR_RCVRC(s));
-        target_phys_addr_t nnrd = pcnet_rdra_addr(s, -2 + CSR_RCVRC(s));
+        hwaddr crda = pcnet_rdra_addr(s, CSR_RCVRC(s));
+        hwaddr nrda = pcnet_rdra_addr(s, -1 + CSR_RCVRC(s));
+        hwaddr nnrd = pcnet_rdra_addr(s, -2 + CSR_RCVRC(s));
 #else
-        target_phys_addr_t crda = s->rdra +
+        hwaddr crda = s->rdra +
             (CSR_RCVRL(s) - CSR_RCVRC(s)) *
             (BCR_SWSTYLE(s) ? 16 : 8 );
         int nrdc = CSR_RCVRC(s)<=1 ? CSR_RCVRL(s) : CSR_RCVRC(s)-1;
-        target_phys_addr_t nrda = s->rdra +
+        hwaddr nrda = s->rdra +
             (CSR_RCVRL(s) - nrdc) *
             (BCR_SWSTYLE(s) ? 16 : 8 );
         int nnrc = nrdc<=1 ? CSR_RCVRL(s) : nrdc-1;
-        target_phys_addr_t nnrd = s->rdra +
+        hwaddr nnrd = s->rdra +
             (CSR_RCVRL(s) - nnrc) *
             (BCR_SWSTYLE(s) ? 16 : 8 );
 #endif
@@ -970,7 +970,7 @@ static int pcnet_tdte_poll(PCNetState *s)
 {
     s->csr[34] = s->csr[35] = 0;
     if (s->tdra) {
-        target_phys_addr_t cxda = s->tdra +
+        hwaddr cxda = s->tdra +
             (CSR_XMTRL(s) - CSR_XMTRC(s)) *
             (BCR_SWSTYLE(s) ? 16 : 8);
         int bad = 0;
@@ -1050,7 +1050,7 @@ ssize_t pcnet_receive(NetClientState *nc, const uint8_t *buf, size_t size_)
         if (!(CSR_CRST(s) & 0x8000) && s->rdra) {
             struct pcnet_RMD rmd;
             int rcvrc = CSR_RCVRC(s)-1,i;
-            target_phys_addr_t nrda;
+            hwaddr nrda;
             for (i = CSR_RCVRL(s)-1; i > 0; i--, rcvrc--) {
                 if (rcvrc <= 1)
                     rcvrc = CSR_RCVRL(s);
@@ -1078,7 +1078,7 @@ ssize_t pcnet_receive(NetClientState *nc, const uint8_t *buf, size_t size_)
             CSR_MISSC(s)++;
         } else {
             uint8_t *src = s->buffer;
-            target_phys_addr_t crda = CSR_CRDA(s);
+            hwaddr crda = CSR_CRDA(s);
             struct pcnet_RMD rmd;
             int pktcount = 0;
 
@@ -1118,7 +1118,7 @@ ssize_t pcnet_receive(NetClientState *nc, const uint8_t *buf, size_t size_)
 
 #define PCNET_RECV_STORE() do {                                 \
     int count = MIN(4096 - GET_FIELD(rmd.buf_length, RMDL, BCNT),remaining); \
-    target_phys_addr_t rbadr = PHYSADDR(s, rmd.rbadr);          \
+    hwaddr rbadr = PHYSADDR(s, rmd.rbadr);          \
     s->phys_mem_write(s->dma_opaque, rbadr, src, count, CSR_BSWP(s)); \
     src += count; remaining -= count;                           \
     SET_FIELD(&rmd.status, RMDS, OWN, 0);                       \
@@ -1129,7 +1129,7 @@ ssize_t pcnet_receive(NetClientState *nc, const uint8_t *buf, size_t size_)
             remaining = size;
             PCNET_RECV_STORE();
             if ((remaining > 0) && CSR_NRDA(s)) {
-                target_phys_addr_t nrda = CSR_NRDA(s);
+                hwaddr nrda = CSR_NRDA(s);
 #ifdef PCNET_DEBUG_RMD
                 PRINT_RMD(&rmd);
 #endif
@@ -1206,7 +1206,7 @@ void pcnet_set_link_status(NetClientState *nc)
 
 static void pcnet_transmit(PCNetState *s)
 {
-    target_phys_addr_t xmit_cxda = 0;
+    hwaddr xmit_cxda = 0;
     int count = CSR_XMTRL(s)-1;
     int add_crc = 0;
 
diff --git a/hw/pcnet.h b/hw/pcnet.h
index d0af54a46..da8c3bde7 100644
--- a/hw/pcnet.h
+++ b/hw/pcnet.h
@@ -42,9 +42,9 @@ struct PCNetState_st {
     MemoryRegion mmio;
     uint8_t buffer[4096];
     qemu_irq irq;
-    void (*phys_mem_read)(void *dma_opaque, target_phys_addr_t addr,
+    void (*phys_mem_read)(void *dma_opaque, hwaddr addr,
                          uint8_t *buf, int len, int do_bswap);
-    void (*phys_mem_write)(void *dma_opaque, target_phys_addr_t addr,
+    void (*phys_mem_write)(void *dma_opaque, hwaddr addr,
                           uint8_t *buf, int len, int do_bswap);
     void *dma_opaque;
     int tx_busy;
diff --git a/hw/pcspk.c b/hw/pcspk.c
index e4303247d..ad6491b0f 100644
--- a/hw/pcspk.c
+++ b/hw/pcspk.c
@@ -121,7 +121,7 @@ int pcspk_audio_init(ISABus *bus)
     return 0;
 }
 
-static uint64_t pcspk_io_read(void *opaque, target_phys_addr_t addr,
+static uint64_t pcspk_io_read(void *opaque, hwaddr addr,
                               unsigned size)
 {
     PCSpkState *s = opaque;
@@ -135,7 +135,7 @@ static uint64_t pcspk_io_read(void *opaque, target_phys_addr_t addr,
        (ch.out << 5);
 }
 
-static void pcspk_io_write(void *opaque, target_phys_addr_t addr, uint64_t val,
+static void pcspk_io_write(void *opaque, hwaddr addr, uint64_t val,
                            unsigned size)
 {
     PCSpkState *s = opaque;
diff --git a/hw/petalogix_ml605_mmu.c b/hw/petalogix_ml605_mmu.c
index dced648f4..3589a4bc7 100644
--- a/hw/petalogix_ml605_mmu.c
+++ b/hw/petalogix_ml605_mmu.c
@@ -34,8 +34,9 @@
 #include "boards.h"
 #include "xilinx.h"
 #include "blockdev.h"
-#include "pc.h"
+#include "serial.h"
 #include "exec-memory.h"
+#include "ssi.h"
 
 #include "microblaze_boot.h"
 #include "microblaze_pic_cpu.h"
@@ -47,6 +48,8 @@
 
 #define BINARY_DEVICE_TREE_FILE "petalogix-ml605.dtb"
 
+#define NUM_SPI_FLASHES 4
+
 #define MEMORY_BASEADDR 0x50000000
 #define FLASH_BASEADDR 0x86000000
 #define INTC_BASEADDR 0x81800000
@@ -70,19 +73,18 @@ static void machine_cpu_reset(MicroBlazeCPU *cpu)
 }
 
 static void
-petalogix_ml605_init(ram_addr_t ram_size,
-                          const char *boot_device,
-                          const char *kernel_filename,
-                          const char *kernel_cmdline,
-                          const char *initrd_filename, const char *cpu_model)
+petalogix_ml605_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t ram_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
     MemoryRegion *address_space_mem = get_system_memory();
     DeviceState *dev, *dma, *eth0;
     MicroBlazeCPU *cpu;
+    SysBusDevice *busdev;
     CPUMBState *env;
     DriveInfo *dinfo;
     int i;
-    target_phys_addr_t ddr_base = MEMORY_BASEADDR;
+    hwaddr ddr_base = MEMORY_BASEADDR;
     MemoryRegion *phys_lmb_bram = g_new(MemoryRegion, 1);
     MemoryRegion *phys_ram = g_new(MemoryRegion, 1);
     qemu_irq irq[32], *cpu_irq;
@@ -139,6 +141,29 @@ petalogix_ml605_init(ram_addr_t ram_size,
     xilinx_axiethernetdma_init(dma, STREAM_SLAVE(eth0),
                                0x84600000, irq[1], irq[0], 100 * 1000000);
 
+    {
+        SSIBus *spi;
+
+        dev = qdev_create(NULL, "xlnx.xps-spi");
+        qdev_prop_set_uint8(dev, "num-ss-bits", NUM_SPI_FLASHES);
+        qdev_init_nofail(dev);
+        busdev = sysbus_from_qdev(dev);
+        sysbus_mmio_map(busdev, 0, 0x40a00000);
+        sysbus_connect_irq(busdev, 0, irq[4]);
+
+        spi = (SSIBus *)qdev_get_child_bus(dev, "spi");
+
+        for (i = 0; i < NUM_SPI_FLASHES; i++) {
+            qemu_irq cs_line;
+
+            dev = ssi_create_slave_no_init(spi, "m25p80");
+            qdev_prop_set_string(dev, "partname", "n25q128");
+            qdev_init_nofail(dev);
+            cs_line = qdev_get_gpio_in(dev, 0);
+            sysbus_connect_irq(busdev, i+1, cs_line);
+        }
+    }
+
     microblaze_load_kernel(cpu, ddr_base, ram_size, BINARY_DEVICE_TREE_FILE,
                                                             machine_cpu_reset);
 
diff --git a/hw/petalogix_s3adsp1800_mmu.c b/hw/petalogix_s3adsp1800_mmu.c
index 2cf68828e..c5fd5e793 100644
--- a/hw/petalogix_s3adsp1800_mmu.c
+++ b/hw/petalogix_s3adsp1800_mmu.c
@@ -57,18 +57,16 @@ static void machine_cpu_reset(MicroBlazeCPU *cpu)
 }
 
 static void
-petalogix_s3adsp1800_init(ram_addr_t ram_size,
-                          const char *boot_device,
-                          const char *kernel_filename,
-                          const char *kernel_cmdline,
-                          const char *initrd_filename, const char *cpu_model)
+petalogix_s3adsp1800_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t ram_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
     DeviceState *dev;
     MicroBlazeCPU *cpu;
     CPUMBState *env;
     DriveInfo *dinfo;
     int i;
-    target_phys_addr_t ddr_base = MEMORY_BASEADDR;
+    hwaddr ddr_base = MEMORY_BASEADDR;
     MemoryRegion *phys_lmb_bram = g_new(MemoryRegion, 1);
     MemoryRegion *phys_ram = g_new(MemoryRegion, 1);
     qemu_irq irq[32], *cpu_irq;
diff --git a/hw/pflash_cfi01.c b/hw/pflash_cfi01.c
index d1c742379..7d040b508 100644
--- a/hw/pflash_cfi01.c
+++ b/hw/pflash_cfi01.c
@@ -41,6 +41,8 @@
 #include "block.h"
 #include "qemu-timer.h"
 #include "exec-memory.h"
+#include "host-utils.h"
+#include "sysbus.h"
 
 #define PFLASH_BUG(fmt, ...) \
 do { \
@@ -59,23 +61,28 @@ do {                                               \
 #endif
 
 struct pflash_t {
+    SysBusDevice busdev;
     BlockDriverState *bs;
-    target_phys_addr_t base;
-    target_phys_addr_t sector_len;
-    target_phys_addr_t total_len;
-    int width;
+    uint32_t nb_blocs;
+    uint64_t sector_len;
+    uint8_t width;
+    uint8_t be;
     int wcycle; /* if 0, the flash is read normally */
     int bypass;
     int ro;
     uint8_t cmd;
     uint8_t status;
-    uint16_t ident[4];
+    uint16_t ident0;
+    uint16_t ident1;
+    uint16_t ident2;
+    uint16_t ident3;
     uint8_t cfi_len;
     uint8_t cfi_table[0x52];
-    target_phys_addr_t counter;
+    hwaddr counter;
     unsigned int writeblock_size;
     QEMUTimer *timer;
     MemoryRegion mem;
+    char *name;
     void *storage;
 };
 
@@ -95,10 +102,10 @@ static void pflash_timer (void *opaque)
     pfl->cmd = 0;
 }
 
-static uint32_t pflash_read (pflash_t *pfl, target_phys_addr_t offset,
+static uint32_t pflash_read (pflash_t *pfl, hwaddr offset,
                              int width, int be)
 {
-    target_phys_addr_t boff;
+    hwaddr boff;
     uint32_t ret;
     uint8_t *p;
 
@@ -167,15 +174,16 @@ static uint32_t pflash_read (pflash_t *pfl, target_phys_addr_t offset,
     case 0x90:
         switch (boff) {
         case 0:
-            ret = pfl->ident[0] << 8 | pfl->ident[1];
+            ret = pfl->ident0 << 8 | pfl->ident1;
             DPRINTF("%s: Manufacturer Code %04x\n", __func__, ret);
             break;
         case 1:
-            ret = pfl->ident[2] << 8 | pfl->ident[3];
+            ret = pfl->ident2 << 8 | pfl->ident3;
             DPRINTF("%s: Device ID Code %04x\n", __func__, ret);
             break;
         default:
-            DPRINTF("%s: Read Device Information boff=%x\n", __func__, boff);
+            DPRINTF("%s: Read Device Information boff=%x\n", __func__,
+                    (unsigned)boff);
             ret = 0;
             break;
         }
@@ -210,7 +218,7 @@ static void pflash_update(pflash_t *pfl, int offset,
     }
 }
 
-static inline void pflash_data_write(pflash_t *pfl, target_phys_addr_t offset,
+static inline void pflash_data_write(pflash_t *pfl, hwaddr offset,
                                      uint32_t value, int width, int be)
 {
     uint8_t *p = pfl->storage;
@@ -248,7 +256,7 @@ static inline void pflash_data_write(pflash_t *pfl, target_phys_addr_t offset,
 
 }
 
-static void pflash_write(pflash_t *pfl, target_phys_addr_t offset,
+static void pflash_write(pflash_t *pfl, hwaddr offset,
                          uint32_t value, int width, int be)
 {
     uint8_t *p;
@@ -278,9 +286,8 @@ static void pflash_write(pflash_t *pfl, target_phys_addr_t offset,
             p = pfl->storage;
             offset &= ~(pfl->sector_len - 1);
 
-            DPRINTF("%s: block erase at " TARGET_FMT_plx " bytes "
-                    TARGET_FMT_plx "\n",
-                    __func__, offset, pfl->sector_len);
+            DPRINTF("%s: block erase at " TARGET_FMT_plx " bytes %x\n",
+                    __func__, offset, (unsigned)pfl->sector_len);
 
             if (!pfl->ro) {
                 memset(p + offset, 0xff, pfl->sector_len);
@@ -320,7 +327,7 @@ static void pflash_write(pflash_t *pfl, target_phys_addr_t offset,
         }
         pfl->wcycle++;
         pfl->cmd = cmd;
-        return;
+        break;
     case 1:
         switch (pfl->cmd) {
         case 0x10: /* Single Byte Program */
@@ -375,7 +382,7 @@ static void pflash_write(pflash_t *pfl, target_phys_addr_t offset,
         default:
             goto error_flash;
         }
-        return;
+        break;
     case 2:
         switch (pfl->cmd) {
         case 0xe8: /* Block write */
@@ -388,7 +395,7 @@ static void pflash_write(pflash_t *pfl, target_phys_addr_t offset,
             pfl->status |= 0x80;
 
             if (!pfl->counter) {
-                target_phys_addr_t mask = pfl->writeblock_size - 1;
+                hwaddr mask = pfl->writeblock_size - 1;
                 mask = ~mask;
 
                 DPRINTF("%s: block write finished\n", __func__);
@@ -406,7 +413,7 @@ static void pflash_write(pflash_t *pfl, target_phys_addr_t offset,
         default:
             goto error_flash;
         }
-        return;
+        break;
     case 3: /* Confirm mode */
         switch (pfl->cmd) {
         case 0xe8: /* Block write */
@@ -422,7 +429,7 @@ static void pflash_write(pflash_t *pfl, target_phys_addr_t offset,
         default:
             goto error_flash;
         }
-        return;
+        break;
     default:
         /* Should never happen */
         DPRINTF("%s: invalid write state\n",  __func__);
@@ -441,61 +448,60 @@ static void pflash_write(pflash_t *pfl, target_phys_addr_t offset,
     pfl->bypass = 0;
     pfl->wcycle = 0;
     pfl->cmd = 0;
-    return;
 }
 
 
-static uint32_t pflash_readb_be(void *opaque, target_phys_addr_t addr)
+static uint32_t pflash_readb_be(void *opaque, hwaddr addr)
 {
     return pflash_read(opaque, addr, 1, 1);
 }
 
-static uint32_t pflash_readb_le(void *opaque, target_phys_addr_t addr)
+static uint32_t pflash_readb_le(void *opaque, hwaddr addr)
 {
     return pflash_read(opaque, addr, 1, 0);
 }
 
-static uint32_t pflash_readw_be(void *opaque, target_phys_addr_t addr)
+static uint32_t pflash_readw_be(void *opaque, hwaddr addr)
 {
     pflash_t *pfl = opaque;
 
     return pflash_read(pfl, addr, 2, 1);
 }
 
-static uint32_t pflash_readw_le(void *opaque, target_phys_addr_t addr)
+static uint32_t pflash_readw_le(void *opaque, hwaddr addr)
 {
     pflash_t *pfl = opaque;
 
     return pflash_read(pfl, addr, 2, 0);
 }
 
-static uint32_t pflash_readl_be(void *opaque, target_phys_addr_t addr)
+static uint32_t pflash_readl_be(void *opaque, hwaddr addr)
 {
     pflash_t *pfl = opaque;
 
     return pflash_read(pfl, addr, 4, 1);
 }
 
-static uint32_t pflash_readl_le(void *opaque, target_phys_addr_t addr)
+static uint32_t pflash_readl_le(void *opaque, hwaddr addr)
 {
     pflash_t *pfl = opaque;
 
     return pflash_read(pfl, addr, 4, 0);
 }
 
-static void pflash_writeb_be(void *opaque, target_phys_addr_t addr,
+static void pflash_writeb_be(void *opaque, hwaddr addr,
                              uint32_t value)
 {
     pflash_write(opaque, addr, value, 1, 1);
 }
 
-static void pflash_writeb_le(void *opaque, target_phys_addr_t addr,
+static void pflash_writeb_le(void *opaque, hwaddr addr,
                              uint32_t value)
 {
     pflash_write(opaque, addr, value, 1, 0);
 }
 
-static void pflash_writew_be(void *opaque, target_phys_addr_t addr,
+static void pflash_writew_be(void *opaque, hwaddr addr,
                              uint32_t value)
 {
     pflash_t *pfl = opaque;
@@ -503,7 +509,7 @@ static void pflash_writew_be(void *opaque, target_phys_addr_t addr,
     pflash_write(pfl, addr, value, 2, 1);
 }
 
-static void pflash_writew_le(void *opaque, target_phys_addr_t addr,
+static void pflash_writew_le(void *opaque, hwaddr addr,
                              uint32_t value)
 {
     pflash_t *pfl = opaque;
@@ -511,7 +517,7 @@ static void pflash_writew_le(void *opaque, target_phys_addr_t addr,
     pflash_write(pfl, addr, value, 2, 0);
 }
 
-static void pflash_writel_be(void *opaque, target_phys_addr_t addr,
+static void pflash_writel_be(void *opaque, hwaddr addr,
                              uint32_t value)
 {
     pflash_t *pfl = opaque;
@@ -519,7 +525,7 @@ static void pflash_writel_be(void *opaque, target_phys_addr_t addr,
     pflash_write(pfl, addr, value, 4, 1);
 }
 
-static void pflash_writel_le(void *opaque, target_phys_addr_t addr,
+static void pflash_writel_le(void *opaque, hwaddr addr,
                              uint32_t value)
 {
     pflash_t *pfl = opaque;
@@ -543,55 +549,13 @@ static const MemoryRegionOps pflash_cfi01_ops_le = {
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-/* Count trailing zeroes of a 32 bits quantity */
-static int ctz32 (uint32_t n)
+static int pflash_cfi01_init(SysBusDevice *dev)
 {
+    pflash_t *pfl = FROM_SYSBUS(typeof(*pfl), dev);
+    uint64_t total_len;
     int ret;
 
-    ret = 0;
-    if (!(n & 0xFFFF)) {
-        ret += 16;
-        n = n >> 16;
-    }
-    if (!(n & 0xFF)) {
-        ret += 8;
-        n = n >> 8;
-    }
-    if (!(n & 0xF)) {
-        ret += 4;
-        n = n >> 4;
-    }
-    if (!(n & 0x3)) {
-        ret += 2;
-        n = n >> 2;
-    }
-    if (!(n & 0x1)) {
-        ret++;
-#if 0 /* This is not necessary as n is never 0 */
-        n = n >> 1;
-#endif
-    }
-#if 0 /* This is not necessary as n is never 0 */
-    if (!n)
-        ret++;
-#endif
-
-    return ret;
-}
-
-pflash_t *pflash_cfi01_register(target_phys_addr_t base,
-                                DeviceState *qdev, const char *name,
-                                target_phys_addr_t size,
-                                BlockDriverState *bs, uint32_t sector_len,
-                                int nb_blocs, int width,
-                                uint16_t id0, uint16_t id1,
-                                uint16_t id2, uint16_t id3, int be)
-{
-    pflash_t *pfl;
-    target_phys_addr_t total_len;
-    int ret;
-
-    total_len = sector_len * nb_blocs;
+    total_len = pfl->sector_len * pfl->nb_blocs;
 
     /* XXX: to be fixed */
 #if 0
@@ -600,27 +564,22 @@ pflash_t *pflash_cfi01_register(target_phys_addr_t base,
         return NULL;
 #endif
 
-    pfl = g_malloc0(sizeof(pflash_t));
-
     memory_region_init_rom_device(
-        &pfl->mem, be ? &pflash_cfi01_ops_be : &pflash_cfi01_ops_le, pfl,
-        name, size);
-    vmstate_register_ram(&pfl->mem, qdev);
+        &pfl->mem, pfl->be ? &pflash_cfi01_ops_be : &pflash_cfi01_ops_le, pfl,
+        pfl->name, total_len);
+    vmstate_register_ram(&pfl->mem, DEVICE(pfl));
     pfl->storage = memory_region_get_ram_ptr(&pfl->mem);
-    memory_region_add_subregion(get_system_memory(), base, &pfl->mem);
+    sysbus_init_mmio(dev, &pfl->mem);
 
-    pfl->bs = bs;
     if (pfl->bs) {
         /* read the initial flash content */
         ret = bdrv_read(pfl->bs, 0, pfl->storage, total_len >> 9);
+
         if (ret < 0) {
-            memory_region_del_subregion(get_system_memory(), &pfl->mem);
-            vmstate_unregister_ram(&pfl->mem, qdev);
+            vmstate_unregister_ram(&pfl->mem, DEVICE(pfl));
             memory_region_destroy(&pfl->mem);
-            g_free(pfl);
-            return NULL;
+            return 1;
         }
-        bdrv_attach_dev_nofail(pfl->bs, pfl);
     }
 
     if (pfl->bs) {
@@ -630,17 +589,9 @@ pflash_t *pflash_cfi01_register(target_phys_addr_t base,
     }
 
     pfl->timer = qemu_new_timer_ns(vm_clock, pflash_timer, pfl);
-    pfl->base = base;
-    pfl->sector_len = sector_len;
-    pfl->total_len = total_len;
-    pfl->width = width;
     pfl->wcycle = 0;
     pfl->cmd = 0;
     pfl->status = 0;
-    pfl->ident[0] = id0;
-    pfl->ident[1] = id1;
-    pfl->ident[2] = id2;
-    pfl->ident[3] = id3;
     /* Hardcoded CFI table */
     pfl->cfi_len = 0x52;
     /* Standard "QRY" string */
@@ -689,7 +640,7 @@ pflash_t *pflash_cfi01_register(target_phys_addr_t base,
     pfl->cfi_table[0x28] = 0x02;
     pfl->cfi_table[0x29] = 0x00;
     /* Max number of bytes in multi-bytes write */
-    if (width == 1) {
+    if (pfl->width == 1) {
         pfl->cfi_table[0x2A] = 0x08;
     } else {
         pfl->cfi_table[0x2A] = 0x0B;
@@ -700,10 +651,10 @@ pflash_t *pflash_cfi01_register(target_phys_addr_t base,
     /* Number of erase block regions (uniform) */
     pfl->cfi_table[0x2C] = 0x01;
     /* Erase block region 1 */
-    pfl->cfi_table[0x2D] = nb_blocs - 1;
-    pfl->cfi_table[0x2E] = (nb_blocs - 1) >> 8;
-    pfl->cfi_table[0x2F] = sector_len >> 8;
-    pfl->cfi_table[0x30] = sector_len >> 16;
+    pfl->cfi_table[0x2D] = pfl->nb_blocs - 1;
+    pfl->cfi_table[0x2E] = (pfl->nb_blocs - 1) >> 8;
+    pfl->cfi_table[0x2F] = pfl->sector_len >> 8;
+    pfl->cfi_table[0x30] = pfl->sector_len >> 16;
 
     /* Extended */
     pfl->cfi_table[0x31] = 'P';
@@ -711,7 +662,7 @@ pflash_t *pflash_cfi01_register(target_phys_addr_t base,
     pfl->cfi_table[0x33] = 'I';
 
     pfl->cfi_table[0x34] = '1';
-    pfl->cfi_table[0x35] = '1';
+    pfl->cfi_table[0x35] = '0';
 
     pfl->cfi_table[0x36] = 0x00;
     pfl->cfi_table[0x37] = 0x00;
@@ -723,6 +674,77 @@ pflash_t *pflash_cfi01_register(target_phys_addr_t base,
     pfl->cfi_table[0x3b] = 0x00;
     pfl->cfi_table[0x3c] = 0x00;
 
+    pfl->cfi_table[0x3f] = 0x01; /* Number of protection fields */
+
+    return 0;
+}
+
+static Property pflash_cfi01_properties[] = {
+    DEFINE_PROP_DRIVE("drive", struct pflash_t, bs),
+    DEFINE_PROP_UINT32("num-blocks", struct pflash_t, nb_blocs, 0),
+    DEFINE_PROP_UINT64("sector-length", struct pflash_t, sector_len, 0),
+    DEFINE_PROP_UINT8("width", struct pflash_t, width, 0),
+    DEFINE_PROP_UINT8("big-endian", struct pflash_t, be, 0),
+    DEFINE_PROP_UINT16("id0", struct pflash_t, ident0, 0),
+    DEFINE_PROP_UINT16("id1", struct pflash_t, ident1, 0),
+    DEFINE_PROP_UINT16("id2", struct pflash_t, ident2, 0),
+    DEFINE_PROP_UINT16("id3", struct pflash_t, ident3, 0),
+    DEFINE_PROP_STRING("name", struct pflash_t, name),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void pflash_cfi01_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
+
+    k->init = pflash_cfi01_init;
+    dc->props = pflash_cfi01_properties;
+}
+
+
+static const TypeInfo pflash_cfi01_info = {
+    .name           = "cfi.pflash01",
+    .parent         = TYPE_SYS_BUS_DEVICE,
+    .instance_size  = sizeof(struct pflash_t),
+    .class_init     = pflash_cfi01_class_init,
+};
+
+static void pflash_cfi01_register_types(void)
+{
+    type_register_static(&pflash_cfi01_info);
+}
+
+type_init(pflash_cfi01_register_types)
+
+pflash_t *pflash_cfi01_register(hwaddr base,
+                                DeviceState *qdev, const char *name,
+                                hwaddr size,
+                                BlockDriverState *bs,
+                                uint32_t sector_len, int nb_blocs, int width,
+                                uint16_t id0, uint16_t id1,
+                                uint16_t id2, uint16_t id3, int be)
+{
+    DeviceState *dev = qdev_create(NULL, "cfi.pflash01");
+    SysBusDevice *busdev = sysbus_from_qdev(dev);
+    pflash_t *pfl = (pflash_t *)object_dynamic_cast(OBJECT(dev),
+                                                    "cfi.pflash01");
+
+    if (bs && qdev_prop_set_drive(dev, "drive", bs)) {
+        abort();
+    }
+    qdev_prop_set_uint32(dev, "num-blocks", nb_blocs);
+    qdev_prop_set_uint64(dev, "sector-length", sector_len);
+    qdev_prop_set_uint8(dev, "width", width);
+    qdev_prop_set_uint8(dev, "big-endian", !!be);
+    qdev_prop_set_uint16(dev, "id0", id0);
+    qdev_prop_set_uint16(dev, "id1", id1);
+    qdev_prop_set_uint16(dev, "id2", id2);
+    qdev_prop_set_uint16(dev, "id3", id3);
+    qdev_prop_set_string(dev, "name", name);
+    qdev_init_nofail(dev);
+
+    sysbus_mmio_map(busdev, 0, base);
     return pfl;
 }
 
diff --git a/hw/pflash_cfi02.c b/hw/pflash_cfi02.c
index 3e2002e4b..f918e3658 100644
--- a/hw/pflash_cfi02.c
+++ b/hw/pflash_cfi02.c
@@ -40,6 +40,8 @@
 #include "qemu-timer.h"
 #include "block.h"
 #include "exec-memory.h"
+#include "host-utils.h"
+#include "sysbus.h"
 
 //#define PFLASH_DEBUG
 #ifdef PFLASH_DEBUG
@@ -54,19 +56,26 @@ do {                                               \
 #define PFLASH_LAZY_ROMD_THRESHOLD 42
 
 struct pflash_t {
+    SysBusDevice busdev;
     BlockDriverState *bs;
-    target_phys_addr_t base;
     uint32_t sector_len;
+    uint32_t nb_blocs;
     uint32_t chip_len;
-    int mappings;
-    int width;
+    uint8_t mappings;
+    uint8_t width;
+    uint8_t be;
     int wcycle; /* if 0, the flash is read normally */
     int bypass;
     int ro;
     uint8_t cmd;
     uint8_t status;
-    uint16_t ident[4];
-    uint16_t unlock_addr[2];
+    /* FIXME: implement array device properties */
+    uint16_t ident0;
+    uint16_t ident1;
+    uint16_t ident2;
+    uint16_t ident3;
+    uint16_t unlock_addr0;
+    uint16_t unlock_addr1;
     uint8_t cfi_len;
     uint8_t cfi_table[0x52];
     QEMUTimer *timer;
@@ -79,6 +88,7 @@ struct pflash_t {
     MemoryRegion orig_mem;
     int rom_mode;
     int read_counter; /* used for lazy switch-back to rom mode */
+    char *name;
     void *storage;
 };
 
@@ -88,7 +98,7 @@ struct pflash_t {
 static void pflash_setup_mappings(pflash_t *pfl)
 {
     unsigned i;
-    target_phys_addr_t size = memory_region_size(&pfl->orig_mem);
+    hwaddr size = memory_region_size(&pfl->orig_mem);
 
     memory_region_init(&pfl->mem, "pflash", pfl->mappings * size);
     pfl->mem_mappings = g_new(MemoryRegion, pfl->mappings);
@@ -121,10 +131,10 @@ static void pflash_timer (void *opaque)
     pfl->cmd = 0;
 }
 
-static uint32_t pflash_read (pflash_t *pfl, target_phys_addr_t offset,
+static uint32_t pflash_read (pflash_t *pfl, hwaddr offset,
                              int width, int be)
 {
-    target_phys_addr_t boff;
+    hwaddr boff;
     uint32_t ret;
     uint8_t *p;
 
@@ -189,16 +199,17 @@ static uint32_t pflash_read (pflash_t *pfl, target_phys_addr_t offset,
         switch (boff) {
         case 0x00:
         case 0x01:
-            ret = pfl->ident[boff & 0x01];
+            ret = boff & 0x01 ? pfl->ident1 : pfl->ident0;
             break;
         case 0x02:
             ret = 0x00; /* Pretend all sectors are unprotected */
             break;
         case 0x0E:
         case 0x0F:
-            if (pfl->ident[2 + (boff & 0x01)] == (uint8_t)-1)
+            ret = boff & 0x01 ? pfl->ident3 : pfl->ident2;
+            if (ret == (uint8_t)-1) {
                 goto flash_read;
-            ret = pfl->ident[2 + (boff & 0x01)];
+            }
             break;
         default:
             goto flash_read;
@@ -241,10 +252,10 @@ static void pflash_update(pflash_t *pfl, int offset,
     }
 }
 
-static void pflash_write (pflash_t *pfl, target_phys_addr_t offset,
+static void pflash_write (pflash_t *pfl, hwaddr offset,
                           uint32_t value, int width, int be)
 {
-    target_phys_addr_t boff;
+    hwaddr boff;
     uint8_t *p;
     uint8_t cmd;
 
@@ -282,9 +293,9 @@ static void pflash_write (pflash_t *pfl, target_phys_addr_t offset,
             pfl->cmd = 0x98;
             return;
         }
-        if (boff != pfl->unlock_addr[0] || cmd != 0xAA) {
+        if (boff != pfl->unlock_addr0 || cmd != 0xAA) {
             DPRINTF("%s: unlock0 failed " TARGET_FMT_plx " %02x %04x\n",
-                    __func__, boff, cmd, pfl->unlock_addr[0]);
+                    __func__, boff, cmd, pfl->unlock_addr0);
             goto reset_flash;
         }
         DPRINTF("%s: unlock sequence started\n", __func__);
@@ -292,7 +303,7 @@ static void pflash_write (pflash_t *pfl, target_phys_addr_t offset,
     case 1:
         /* We started an unlock sequence */
     check_unlock1:
-        if (boff != pfl->unlock_addr[1] || cmd != 0x55) {
+        if (boff != pfl->unlock_addr1 || cmd != 0x55) {
             DPRINTF("%s: unlock1 failed " TARGET_FMT_plx " %02x\n", __func__,
                     boff, cmd);
             goto reset_flash;
@@ -301,7 +312,7 @@ static void pflash_write (pflash_t *pfl, target_phys_addr_t offset,
         break;
     case 2:
         /* We finished an unlock sequence */
-        if (!pfl->bypass && boff != pfl->unlock_addr[0]) {
+        if (!pfl->bypass && boff != pfl->unlock_addr0) {
             DPRINTF("%s: command failed " TARGET_FMT_plx " %02x\n", __func__,
                     boff, cmd);
             goto reset_flash;
@@ -399,7 +410,7 @@ static void pflash_write (pflash_t *pfl, target_phys_addr_t offset,
     case 5:
         switch (cmd) {
         case 0x10:
-            if (boff != pfl->unlock_addr[0]) {
+            if (boff != pfl->unlock_addr0) {
                 DPRINTF("%s: chip erase: invalid address " TARGET_FMT_plx "\n",
                         __func__, offset);
                 goto reset_flash;
@@ -473,61 +484,60 @@ static void pflash_write (pflash_t *pfl, target_phys_addr_t offset,
  do_bypass:
     pfl->wcycle = 2;
     pfl->cmd = 0;
-    return;
 }
 
 
-static uint32_t pflash_readb_be(void *opaque, target_phys_addr_t addr)
+static uint32_t pflash_readb_be(void *opaque, hwaddr addr)
 {
     return pflash_read(opaque, addr, 1, 1);
 }
 
-static uint32_t pflash_readb_le(void *opaque, target_phys_addr_t addr)
+static uint32_t pflash_readb_le(void *opaque, hwaddr addr)
 {
     return pflash_read(opaque, addr, 1, 0);
 }
 
-static uint32_t pflash_readw_be(void *opaque, target_phys_addr_t addr)
+static uint32_t pflash_readw_be(void *opaque, hwaddr addr)
 {
     pflash_t *pfl = opaque;
 
     return pflash_read(pfl, addr, 2, 1);
 }
 
-static uint32_t pflash_readw_le(void *opaque, target_phys_addr_t addr)
+static uint32_t pflash_readw_le(void *opaque, hwaddr addr)
 {
     pflash_t *pfl = opaque;
 
     return pflash_read(pfl, addr, 2, 0);
 }
 
-static uint32_t pflash_readl_be(void *opaque, target_phys_addr_t addr)
+static uint32_t pflash_readl_be(void *opaque, hwaddr addr)
 {
     pflash_t *pfl = opaque;
 
     return pflash_read(pfl, addr, 4, 1);
 }
 
-static uint32_t pflash_readl_le(void *opaque, target_phys_addr_t addr)
+static uint32_t pflash_readl_le(void *opaque, hwaddr addr)
 {
     pflash_t *pfl = opaque;
 
     return pflash_read(pfl, addr, 4, 0);
 }
 
-static void pflash_writeb_be(void *opaque, target_phys_addr_t addr,
+static void pflash_writeb_be(void *opaque, hwaddr addr,
                              uint32_t value)
 {
     pflash_write(opaque, addr, value, 1, 1);
 }
 
-static void pflash_writeb_le(void *opaque, target_phys_addr_t addr,
+static void pflash_writeb_le(void *opaque, hwaddr addr,
                              uint32_t value)
 {
     pflash_write(opaque, addr, value, 1, 0);
 }
 
-static void pflash_writew_be(void *opaque, target_phys_addr_t addr,
+static void pflash_writew_be(void *opaque, hwaddr addr,
                              uint32_t value)
 {
     pflash_t *pfl = opaque;
@@ -535,7 +545,7 @@ static void pflash_writew_be(void *opaque, target_phys_addr_t addr,
     pflash_write(pfl, addr, value, 2, 1);
 }
 
-static void pflash_writew_le(void *opaque, target_phys_addr_t addr,
+static void pflash_writew_le(void *opaque, hwaddr addr,
                              uint32_t value)
 {
     pflash_t *pfl = opaque;
@@ -543,7 +553,7 @@ static void pflash_writew_le(void *opaque, target_phys_addr_t addr,
     pflash_write(pfl, addr, value, 2, 0);
 }
 
-static void pflash_writel_be(void *opaque, target_phys_addr_t addr,
+static void pflash_writel_be(void *opaque, hwaddr addr,
                              uint32_t value)
 {
     pflash_t *pfl = opaque;
@@ -551,7 +561,7 @@ static void pflash_writel_be(void *opaque, target_phys_addr_t addr,
     pflash_write(pfl, addr, value, 4, 1);
 }
 
-static void pflash_writel_le(void *opaque, target_phys_addr_t addr,
+static void pflash_writel_le(void *opaque, hwaddr addr,
                              uint32_t value)
 {
     pflash_t *pfl = opaque;
@@ -575,86 +585,38 @@ static const MemoryRegionOps pflash_cfi02_ops_le = {
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-/* Count trailing zeroes of a 32 bits quantity */
-static int ctz32 (uint32_t n)
+static int pflash_cfi02_init(SysBusDevice *dev)
 {
+    pflash_t *pfl = FROM_SYSBUS(typeof(*pfl), dev);
+    uint32_t chip_len;
     int ret;
 
-    ret = 0;
-    if (!(n & 0xFFFF)) {
-        ret += 16;
-        n = n >> 16;
-    }
-    if (!(n & 0xFF)) {
-        ret += 8;
-        n = n >> 8;
-    }
-    if (!(n & 0xF)) {
-        ret += 4;
-        n = n >> 4;
-    }
-    if (!(n & 0x3)) {
-        ret += 2;
-        n = n >> 2;
-    }
-    if (!(n & 0x1)) {
-        ret++;
-#if 0 /* This is not necessary as n is never 0 */
-        n = n >> 1;
-#endif
-    }
-#if 0 /* This is not necessary as n is never 0 */
-    if (!n)
-        ret++;
-#endif
-
-    return ret;
-}
-
-pflash_t *pflash_cfi02_register(target_phys_addr_t base,
-                                DeviceState *qdev, const char *name,
-                                target_phys_addr_t size,
-                                BlockDriverState *bs, uint32_t sector_len,
-                                int nb_blocs, int nb_mappings, int width,
-                                uint16_t id0, uint16_t id1,
-                                uint16_t id2, uint16_t id3,
-                                uint16_t unlock_addr0, uint16_t unlock_addr1,
-                                int be)
-{
-    pflash_t *pfl;
-    int32_t chip_len;
-    int ret;
-
-    chip_len = sector_len * nb_blocs;
+    chip_len = pfl->sector_len * pfl->nb_blocs;
     /* XXX: to be fixed */
 #if 0
     if (total_len != (8 * 1024 * 1024) && total_len != (16 * 1024 * 1024) &&
         total_len != (32 * 1024 * 1024) && total_len != (64 * 1024 * 1024))
         return NULL;
 #endif
-    pfl = g_malloc0(sizeof(pflash_t));
-    memory_region_init_rom_device(
-        &pfl->orig_mem, be ? &pflash_cfi02_ops_be : &pflash_cfi02_ops_le, pfl,
-        name, size);
-    vmstate_register_ram(&pfl->orig_mem, qdev);
+
+    memory_region_init_rom_device(&pfl->orig_mem, pfl->be ?
+                                  &pflash_cfi02_ops_be : &pflash_cfi02_ops_le,
+                                  pfl, pfl->name, chip_len);
+    vmstate_register_ram(&pfl->orig_mem, DEVICE(pfl));
     pfl->storage = memory_region_get_ram_ptr(&pfl->orig_mem);
-    pfl->base = base;
     pfl->chip_len = chip_len;
-    pfl->mappings = nb_mappings;
-    pfl->bs = bs;
     if (pfl->bs) {
         /* read the initial flash content */
         ret = bdrv_read(pfl->bs, 0, pfl->storage, chip_len >> 9);
         if (ret < 0) {
             g_free(pfl);
-            return NULL;
+            return 1;
         }
-        bdrv_attach_dev_nofail(pfl->bs, pfl);
     }
 
     pflash_setup_mappings(pfl);
     pfl->rom_mode = 1;
-    memory_region_add_subregion(get_system_memory(), pfl->base, &pfl->mem);
+    sysbus_init_mmio(dev, &pfl->mem);
 
     if (pfl->bs) {
         pfl->ro = bdrv_is_read_only(pfl->bs);
@@ -663,17 +625,9 @@ pflash_t *pflash_cfi02_register(target_phys_addr_t base,
     }
 
     pfl->timer = qemu_new_timer_ns(vm_clock, pflash_timer, pfl);
-    pfl->sector_len = sector_len;
-    pfl->width = width;
     pfl->wcycle = 0;
     pfl->cmd = 0;
     pfl->status = 0;
-    pfl->ident[0] = id0;
-    pfl->ident[1] = id1;
-    pfl->ident[2] = id2;
-    pfl->ident[3] = id3;
-    pfl->unlock_addr[0] = unlock_addr0;
-    pfl->unlock_addr[1] = unlock_addr1;
     /* Hardcoded CFI table (mostly from SG29 Spansion flash) */
     pfl->cfi_len = 0x52;
     /* Standard "QRY" string */
@@ -729,10 +683,10 @@ pflash_t *pflash_cfi02_register(target_phys_addr_t base,
     /* Number of erase block regions (uniform) */
     pfl->cfi_table[0x2C] = 0x01;
     /* Erase block region 1 */
-    pfl->cfi_table[0x2D] = nb_blocs - 1;
-    pfl->cfi_table[0x2E] = (nb_blocs - 1) >> 8;
-    pfl->cfi_table[0x2F] = sector_len >> 8;
-    pfl->cfi_table[0x30] = sector_len >> 16;
+    pfl->cfi_table[0x2D] = pfl->nb_blocs - 1;
+    pfl->cfi_table[0x2E] = (pfl->nb_blocs - 1) >> 8;
+    pfl->cfi_table[0x2F] = pfl->sector_len >> 8;
+    pfl->cfi_table[0x30] = pfl->sector_len >> 16;
 
     /* Extended */
     pfl->cfi_table[0x31] = 'P';
@@ -752,5 +706,81 @@ pflash_t *pflash_cfi02_register(target_phys_addr_t base,
     pfl->cfi_table[0x3b] = 0x00;
     pfl->cfi_table[0x3c] = 0x00;
 
+    return 0;
+}
+
+static Property pflash_cfi02_properties[] = {
+    DEFINE_PROP_DRIVE("drive", struct pflash_t, bs),
+    DEFINE_PROP_UINT32("num-blocks", struct pflash_t, nb_blocs, 0),
+    DEFINE_PROP_UINT32("sector-length", struct pflash_t, sector_len, 0),
+    DEFINE_PROP_UINT8("width", struct pflash_t, width, 0),
+    DEFINE_PROP_UINT8("mappings", struct pflash_t, mappings, 0),
+    DEFINE_PROP_UINT8("big-endian", struct pflash_t, be, 0),
+    DEFINE_PROP_UINT16("id0", struct pflash_t, ident0, 0),
+    DEFINE_PROP_UINT16("id1", struct pflash_t, ident1, 0),
+    DEFINE_PROP_UINT16("id2", struct pflash_t, ident2, 0),
+    DEFINE_PROP_UINT16("id3", struct pflash_t, ident3, 0),
+    DEFINE_PROP_UINT16("unlock-addr0", struct pflash_t, unlock_addr0, 0),
+    DEFINE_PROP_UINT16("unlock-addr1", struct pflash_t, unlock_addr1, 0),
+    DEFINE_PROP_STRING("name", struct pflash_t, name),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void pflash_cfi02_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
+
+    k->init = pflash_cfi02_init;
+    dc->props = pflash_cfi02_properties;
+}
+
+static const TypeInfo pflash_cfi02_info = {
+    .name           = "cfi.pflash02",
+    .parent         = TYPE_SYS_BUS_DEVICE,
+    .instance_size  = sizeof(struct pflash_t),
+    .class_init     = pflash_cfi02_class_init,
+};
+
+static void pflash_cfi02_register_types(void)
+{
+    type_register_static(&pflash_cfi02_info);
+}
+
+type_init(pflash_cfi02_register_types)
+
+pflash_t *pflash_cfi02_register(hwaddr base,
+                                DeviceState *qdev, const char *name,
+                                hwaddr size,
+                                BlockDriverState *bs, uint32_t sector_len,
+                                int nb_blocs, int nb_mappings, int width,
+                                uint16_t id0, uint16_t id1,
+                                uint16_t id2, uint16_t id3,
+                                uint16_t unlock_addr0, uint16_t unlock_addr1,
+                                int be)
+{
+    DeviceState *dev = qdev_create(NULL, "cfi.pflash02");
+    SysBusDevice *busdev = sysbus_from_qdev(dev);
+    pflash_t *pfl = (pflash_t *)object_dynamic_cast(OBJECT(dev),
+                                                    "cfi.pflash02");
+
+    if (bs && qdev_prop_set_drive(dev, "drive", bs)) {
+        abort();
+    }
+    qdev_prop_set_uint32(dev, "num-blocks", nb_blocs);
+    qdev_prop_set_uint32(dev, "sector-length", sector_len);
+    qdev_prop_set_uint8(dev, "width", width);
+    qdev_prop_set_uint8(dev, "mappings", nb_mappings);
+    qdev_prop_set_uint8(dev, "big-endian", !!be);
+    qdev_prop_set_uint16(dev, "id0", id0);
+    qdev_prop_set_uint16(dev, "id1", id1);
+    qdev_prop_set_uint16(dev, "id2", id2);
+    qdev_prop_set_uint16(dev, "id3", id3);
+    qdev_prop_set_uint16(dev, "unlock-addr0", unlock_addr0);
+    qdev_prop_set_uint16(dev, "unlock-addr1", unlock_addr1);
+    qdev_prop_set_string(dev, "name", name);
+    qdev_init_nofail(dev);
+
+    sysbus_mmio_map(busdev, 0, base);
     return pfl;
 }
diff --git a/hw/piix_pci.c b/hw/piix_pci.c
index 537fc1973..ba1b3de74 100644
--- a/hw/piix_pci.c
+++ b/hw/piix_pci.c
@@ -30,6 +30,7 @@
 #include "sysbus.h"
 #include "range.h"
 #include "xen.h"
+#include "pam.h"
 
 /*
  * I440FX chipset data sheet.
@@ -68,11 +69,6 @@ typedef struct PIIX3State {
     int32_t pci_irq_levels_vmstate[PIIX_NUM_PIRQS];
 } PIIX3State;
 
-typedef struct PAMMemoryRegion {
-    MemoryRegion mem;
-    bool initialized;
-} PAMMemoryRegion;
-
 struct PCII440FXState {
     PCIDevice dev;
     MemoryRegion *system_memory;
@@ -105,56 +101,16 @@ static int pci_slot_get_pirq(PCIDevice *pci_dev, int pci_intx)
     return (pci_intx + slot_addend) & 3;
 }
 
-static void update_pam(PCII440FXState *d, uint32_t start, uint32_t end, int r,
-                       PAMMemoryRegion *mem)
-{
-    if (mem->initialized) {
-        memory_region_del_subregion(d->system_memory, &mem->mem);
-        memory_region_destroy(&mem->mem);
-    }
-
-    //    printf("ISA mapping %08x-0x%08x: %d\n", start, end, r);
-    switch(r) {
-    case 3:
-        /* RAM */
-        memory_region_init_alias(&mem->mem, "pam-ram", d->ram_memory,
-                                 start, end - start);
-        break;
-    case 1:
-        /* ROM (XXX: not quite correct) */
-        memory_region_init_alias(&mem->mem, "pam-rom", d->ram_memory,
-                                 start, end - start);
-        memory_region_set_readonly(&mem->mem, true);
-        break;
-    case 2:
-    case 0:
-        /* XXX: should distinguish read/write cases */
-        memory_region_init_alias(&mem->mem, "pam-pci", d->pci_address_space,
-                                 start, end - start);
-        break;
-    }
-    memory_region_add_subregion_overlap(d->system_memory,
-                                        start, &mem->mem, 1);
-    mem->initialized = true;
-}
-
 static void i440fx_update_memory_mappings(PCII440FXState *d)
 {
-    int i, r;
-    uint32_t smram;
-    bool smram_enabled;
+    int i;
 
     memory_region_transaction_begin();
-    update_pam(d, 0xf0000, 0x100000, (d->dev.config[I440FX_PAM] >> 4) & 3,
-               &d->pam_regions[0]);
-    for(i = 0; i < 12; i++) {
-        r = (d->dev.config[(i >> 1) + (I440FX_PAM + 1)] >> ((i & 1) * 4)) & 3;
-        update_pam(d, 0xc0000 + 0x4000 * i, 0xc0000 + 0x4000 * (i + 1), r,
-                   &d->pam_regions[i+1]);
+    for (i = 0; i < 13; i++) {
+        pam_update(&d->pam_regions[i], i,
+                   d->dev.config[I440FX_PAM + ((i + 1) / 2)]);
     }
-    smram = d->dev.config[I440FX_SMRAM];
-    smram_enabled = (d->smm_enabled && (smram & 0x08)) || (smram & 0x40);
-    memory_region_set_enabled(&d->smram_region, !smram_enabled);
+    smram_update(&d->smram_region, d->dev.config[I440FX_SMRAM], d->smm_enabled);
     memory_region_transaction_commit();
 }
 
@@ -162,11 +118,10 @@ static void i440fx_set_smm(int val, void *arg)
 {
     PCII440FXState *d = arg;
 
-    val = (val != 0);
-    if (d->smm_enabled != val) {
-        d->smm_enabled = val;
-        i440fx_update_memory_mappings(d);
-    }
+    memory_region_transaction_begin();
+    smram_set_smm(&d->smm_enabled, val, d->dev.config[I440FX_SMRAM],
+                  &d->smram_region);
+    memory_region_transaction_commit();
 }
 
 
@@ -259,10 +214,10 @@ static PCIBus *i440fx_common_init(const char *device_name,
                                   MemoryRegion *address_space_mem,
                                   MemoryRegion *address_space_io,
                                   ram_addr_t ram_size,
-                                  target_phys_addr_t pci_hole_start,
-                                  target_phys_addr_t pci_hole_size,
-                                  target_phys_addr_t pci_hole64_start,
-                                  target_phys_addr_t pci_hole64_size,
+                                  hwaddr pci_hole_start,
+                                  hwaddr pci_hole_size,
+                                  hwaddr pci_hole64_start,
+                                  hwaddr pci_hole64_size,
                                   MemoryRegion *pci_address_space,
                                   MemoryRegion *ram_memory)
 {
@@ -272,6 +227,7 @@ static PCIBus *i440fx_common_init(const char *device_name,
     PCIHostState *s;
     PIIX3State *piix3;
     PCII440FXState *f;
+    unsigned i;
 
     dev = qdev_create(NULL, "i440FX-pcihost");
     s = PCI_HOST_BRIDGE(dev);
@@ -303,6 +259,13 @@ static PCIBus *i440fx_common_init(const char *device_name,
     memory_region_add_subregion_overlap(f->system_memory, 0xa0000,
                                         &f->smram_region, 1);
     memory_region_set_enabled(&f->smram_region, false);
+    init_pam(f->ram_memory, f->system_memory, f->pci_address_space,
+             &f->pam_regions[0], PAM_BIOS_BASE, PAM_BIOS_SIZE);
+    for (i = 0; i < 12; ++i) {
+        init_pam(f->ram_memory, f->system_memory, f->pci_address_space,
+                 &f->pam_regions[i+1], PAM_EXPAN_BASE + i * PAM_EXPAN_SIZE,
+                 PAM_EXPAN_SIZE);
+    }
 
     /* Xen supports additional interrupt routes from the PCI devices to
      * the IOAPIC: the four pins of each PCI device on the bus are also
@@ -341,10 +304,10 @@ PCIBus *i440fx_init(PCII440FXState **pi440fx_state, int *piix3_devfn,
                     MemoryRegion *address_space_mem,
                     MemoryRegion *address_space_io,
                     ram_addr_t ram_size,
-                    target_phys_addr_t pci_hole_start,
-                    target_phys_addr_t pci_hole_size,
-                    target_phys_addr_t pci_hole64_start,
-                    target_phys_addr_t pci_hole64_size,
+                    hwaddr pci_hole_start,
+                    hwaddr pci_hole_size,
+                    hwaddr pci_hole64_start,
+                    hwaddr pci_hole64_size,
                     MemoryRegion *pci_memory, MemoryRegion *ram_memory)
 
 {
diff --git a/hw/pl011.c b/hw/pl011.c
index 3245702df..1f7ce2f94 100644
--- a/hw/pl011.c
+++ b/hw/pl011.c
@@ -54,7 +54,7 @@ static void pl011_update(pl011_state *s)
     qemu_set_irq(s->irq, flags != 0);
 }
 
-static uint64_t pl011_read(void *opaque, target_phys_addr_t offset,
+static uint64_t pl011_read(void *opaque, hwaddr offset,
                            unsigned size)
 {
     pl011_state *s = (pl011_state *)opaque;
@@ -107,7 +107,8 @@ static uint64_t pl011_read(void *opaque, target_phys_addr_t offset,
     case 18: /* UARTDMACR */
         return s->dmacr;
     default:
-        hw_error("pl011_read: Bad offset %x\n", (int)offset);
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "pl011_read: Bad offset %x\n", (int)offset);
         return 0;
     }
 }
@@ -126,7 +127,7 @@ static void pl011_set_read_trigger(pl011_state *s)
         s->read_trigger = 1;
 }
 
-static void pl011_write(void *opaque, target_phys_addr_t offset,
+static void pl011_write(void *opaque, hwaddr offset,
                         uint64_t value, unsigned size)
 {
     pl011_state *s = (pl011_state *)opaque;
@@ -178,11 +179,13 @@ static void pl011_write(void *opaque, target_phys_addr_t offset,
         break;
     case 18: /* UARTDMACR */
         s->dmacr = value;
-        if (value & 3)
-            hw_error("PL011: DMA not implemented\n");
+        if (value & 3) {
+            qemu_log_mask(LOG_UNIMP, "pl011: DMA not implemented\n");
+        }
         break;
     default:
-        hw_error("pl011_write: Bad offset %x\n", (int)offset);
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "pl011_write: Bad offset %x\n", (int)offset);
     }
 }
 
diff --git a/hw/pl022.c b/hw/pl022.c
index 60e35daeb..fbd7ded0c 100644
--- a/hw/pl022.c
+++ b/hw/pl022.c
@@ -130,7 +130,7 @@ static void pl022_xfer(pl022_state *s)
     pl022_update(s);
 }
 
-static uint64_t pl022_read(void *opaque, target_phys_addr_t offset,
+static uint64_t pl022_read(void *opaque, hwaddr offset,
                            unsigned size)
 {
     pl022_state *s = (pl022_state *)opaque;
@@ -168,12 +168,13 @@ static uint64_t pl022_read(void *opaque, target_phys_addr_t offset,
         /* Not implemented.  */
         return 0;
     default:
-        hw_error("pl022_read: Bad offset %x\n", (int)offset);
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "pl022_read: Bad offset %x\n", (int)offset);
         return 0;
     }
 }
 
-static void pl022_write(void *opaque, target_phys_addr_t offset,
+static void pl022_write(void *opaque, hwaddr offset,
                         uint64_t value, unsigned size)
 {
     pl022_state *s = (pl022_state *)opaque;
@@ -211,11 +212,12 @@ static void pl022_write(void *opaque, target_phys_addr_t offset,
         break;
     case 0x20: /* DMACR */
         if (value) {
-            hw_error("pl022: DMA not implemented\n");
+            qemu_log_mask(LOG_UNIMP, "pl022: DMA not implemented\n");
         }
         break;
     default:
-        hw_error("pl022_write: Bad offset %x\n", (int)offset);
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "pl022_write: Bad offset %x\n", (int)offset);
     }
 }
 
diff --git a/hw/pl031.c b/hw/pl031.c
index 9602664da..8bf018328 100644
--- a/hw/pl031.c
+++ b/hw/pl031.c
@@ -95,7 +95,7 @@ static void pl031_set_alarm(pl031_state *s)
     }
 }
 
-static uint64_t pl031_read(void *opaque, target_phys_addr_t offset,
+static uint64_t pl031_read(void *opaque, hwaddr offset,
                            unsigned size)
 {
     pl031_state *s = (pl031_state *)opaque;
@@ -120,18 +120,20 @@ static uint64_t pl031_read(void *opaque, target_phys_addr_t offset,
     case RTC_MIS:
         return s->is & s->im;
     case RTC_ICR:
-        fprintf(stderr, "qemu: pl031_read: Unexpected offset 0x%x\n",
-                (int)offset);
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "pl031: read of write-only register at offset 0x%x\n",
+                      (int)offset);
         break;
     default:
-        hw_error("pl031_read: Bad offset 0x%x\n", (int)offset);
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "pl031_read: Bad offset 0x%x\n", (int)offset);
         break;
     }
 
     return 0;
 }
 
-static void pl031_write(void * opaque, target_phys_addr_t offset,
+static void pl031_write(void * opaque, hwaddr offset,
                         uint64_t value, unsigned size)
 {
     pl031_state *s = (pl031_state *)opaque;
@@ -167,12 +169,14 @@ static void pl031_write(void * opaque, target_phys_addr_t offset,
     case RTC_DR:
     case RTC_MIS:
     case RTC_RIS:
-        fprintf(stderr, "qemu: pl031_write: Unexpected offset 0x%x\n",
-                (int)offset);
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "pl031: write to read-only register at offset 0x%x\n",
+                      (int)offset);
         break;
 
     default:
-        hw_error("pl031_write: Bad offset 0x%x\n", (int)offset);
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "pl031_write: Bad offset 0x%x\n", (int)offset);
         break;
     }
 }
diff --git a/hw/pl041.c b/hw/pl041.c
index b6723be0a..4436d97c5 100644
--- a/hw/pl041.c
+++ b/hw/pl041.c
@@ -97,7 +97,7 @@ static const char *pl041_regs_name[] = {
 
 
 #if defined(PL041_DEBUG_LEVEL)
-static const char *get_reg_name(target_phys_addr_t offset)
+static const char *get_reg_name(hwaddr offset)
 {
     if (offset <= PL041_dr1_7) {
         return pl041_regs_name[offset >> 2];
@@ -327,7 +327,7 @@ static void pl041_request_data(void *opaque)
     pl041_isr1_update(s);
 }
 
-static uint64_t pl041_read(void *opaque, target_phys_addr_t offset,
+static uint64_t pl041_read(void *opaque, hwaddr offset,
                                 unsigned size)
 {
     pl041_state *s = (pl041_state *)opaque;
@@ -361,7 +361,7 @@ static uint64_t pl041_read(void *opaque, target_phys_addr_t offset,
     return value;
 }
 
-static void pl041_write(void *opaque, target_phys_addr_t offset,
+static void pl041_write(void *opaque, hwaddr offset,
                              uint64_t value, unsigned size)
 {
     pl041_state *s = (pl041_state *)opaque;
@@ -536,8 +536,9 @@ static int pl041_init(SysBusDevice *dev)
     default:
         /* NC FIFO depth of 16 is not allowed because its id bits in
            AACIPERIPHID3 overlap with the id for the default NC FIFO depth */
-        fprintf(stderr, "pl041: unsupported non-compact fifo depth [%i]\n",
-                s->fifo_depth);
+        qemu_log_mask(LOG_UNIMP,
+                      "pl041: unsupported non-compact fifo depth [%i]\n",
+                      s->fifo_depth);
         return -1;
     }
 
diff --git a/hw/pl050.c b/hw/pl050.c
index b13924a16..47032f126 100644
--- a/hw/pl050.c
+++ b/hw/pl050.c
@@ -58,7 +58,7 @@ static void pl050_update(void *opaque, int level)
     qemu_set_irq(s->irq, raise);
 }
 
-static uint64_t pl050_read(void *opaque, target_phys_addr_t offset,
+static uint64_t pl050_read(void *opaque, hwaddr offset,
                            unsigned size)
 {
     pl050_state *s = (pl050_state *)opaque;
@@ -95,12 +95,13 @@ static uint64_t pl050_read(void *opaque, target_phys_addr_t offset,
     case 4: /* KMIIR */
         return s->pending | 2;
     default:
-        hw_error("pl050_read: Bad offset %x\n", (int)offset);
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "pl050_read: Bad offset %x\n", (int)offset);
         return 0;
     }
 }
 
-static void pl050_write(void *opaque, target_phys_addr_t offset,
+static void pl050_write(void *opaque, hwaddr offset,
                         uint64_t value, unsigned size)
 {
     pl050_state *s = (pl050_state *)opaque;
@@ -123,7 +124,8 @@ static void pl050_write(void *opaque, target_phys_addr_t offset,
         s->clk = value;
         return;
     default:
-        hw_error("pl050_write: Bad offset %x\n", (int)offset);
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "pl050_write: Bad offset %x\n", (int)offset);
     }
 }
 static const MemoryRegionOps pl050_ops = {
diff --git a/hw/pl061.c b/hw/pl061.c
index 2aac7e8e9..f1ed5ced1 100644
--- a/hw/pl061.c
+++ b/hw/pl061.c
@@ -113,7 +113,7 @@ static void pl061_update(pl061_state *s)
     /* FIXME: Implement input interrupts.  */
 }
 
-static uint64_t pl061_read(void *opaque, target_phys_addr_t offset,
+static uint64_t pl061_read(void *opaque, hwaddr offset,
                            unsigned size)
 {
     pl061_state *s = (pl061_state *)opaque;
@@ -164,12 +164,13 @@ static uint64_t pl061_read(void *opaque, target_phys_addr_t offset,
     case 0x528: /* Analog mode select */
         return s->amsel;
     default:
-        hw_error("pl061_read: Bad offset %x\n", (int)offset);
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "pl061_read: Bad offset %x\n", (int)offset);
         return 0;
     }
 }
 
-static void pl061_write(void *opaque, target_phys_addr_t offset,
+static void pl061_write(void *opaque, hwaddr offset,
                         uint64_t value, unsigned size)
 {
     pl061_state *s = (pl061_state *)opaque;
@@ -239,7 +240,8 @@ static void pl061_write(void *opaque, target_phys_addr_t offset,
         s->amsel = value & 0xff;
         break;
     default:
-        hw_error("pl061_write: Bad offset %x\n", (int)offset);
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "pl061_write: Bad offset %x\n", (int)offset);
     }
     pl061_update(s);
 }
diff --git a/hw/pl080.c b/hw/pl080.c
index b3cf651ab..26150af75 100644
--- a/hw/pl080.c
+++ b/hw/pl080.c
@@ -218,7 +218,7 @@ again:
     }
 }
 
-static uint64_t pl080_read(void *opaque, target_phys_addr_t offset,
+static uint64_t pl080_read(void *opaque, hwaddr offset,
                            unsigned size)
 {
     pl080_state *s = (pl080_state *)opaque;
@@ -281,12 +281,13 @@ static uint64_t pl080_read(void *opaque, target_phys_addr_t offset,
         return s->sync;
     default:
     bad_offset:
-        hw_error("pl080_read: Bad offset %x\n", (int)offset);
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "pl080_read: Bad offset %x\n", (int)offset);
         return 0;
     }
 }
 
-static void pl080_write(void *opaque, target_phys_addr_t offset,
+static void pl080_write(void *opaque, hwaddr offset,
                         uint64_t value, unsigned size)
 {
     pl080_state *s = (pl080_state *)opaque;
@@ -327,12 +328,13 @@ static void pl080_write(void *opaque, target_phys_addr_t offset,
     case 10: /* SoftLBReq */
     case 11: /* SoftLSReq */
         /* ??? Implement these.  */
-        hw_error("pl080_write: Soft DMA not implemented\n");
+        qemu_log_mask(LOG_UNIMP, "pl080_write: Soft DMA not implemented\n");
         break;
     case 12: /* Configuration */
         s->conf = value;
         if (s->conf & (PL080_CONF_M1 | PL080_CONF_M1)) {
-            hw_error("pl080_write: Big-endian DMA not implemented\n");
+            qemu_log_mask(LOG_UNIMP,
+                          "pl080_write: Big-endian DMA not implemented\n");
         }
         pl080_run(s);
         break;
@@ -341,7 +343,8 @@ static void pl080_write(void *opaque, target_phys_addr_t offset,
         break;
     default:
     bad_offset:
-        hw_error("pl080_write: Bad offset %x\n", (int)offset);
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "pl080_write: Bad offset %x\n", (int)offset);
     }
     pl080_update(s);
 }
diff --git a/hw/pl110.c b/hw/pl110.c
index f94608cb6..f869ba60d 100644
--- a/hw/pl110.c
+++ b/hw/pl110.c
@@ -55,8 +55,8 @@ typedef struct {
     enum pl110_bppmode bpp;
     int invalidate;
     uint32_t mux_ctrl;
-    uint32_t pallette[256];
-    uint32_t raw_pallette[128];
+    uint32_t palette[256];
+    uint32_t raw_palette[128];
     qemu_irq irq;
 } pl110_state;
 
@@ -79,8 +79,8 @@ static const VMStateDescription vmstate_pl110 = {
         VMSTATE_INT32(rows, pl110_state),
         VMSTATE_UINT32(bpp, pl110_state),
         VMSTATE_INT32(invalidate, pl110_state),
-        VMSTATE_UINT32_ARRAY(pallette, pl110_state, 256),
-        VMSTATE_UINT32_ARRAY(raw_pallette, pl110_state, 128),
+        VMSTATE_UINT32_ARRAY(palette, pl110_state, 256),
+        VMSTATE_UINT32_ARRAY(raw_palette, pl110_state, 128),
         VMSTATE_UINT32_V(mux_ctrl, pl110_state, 2),
         VMSTATE_END_OF_LIST()
     }
@@ -236,10 +236,10 @@ static void pl110_update_display(void *opaque)
                                s->upbase, s->cols, s->rows,
                                src_width, dest_width, 0,
                                s->invalidate,
-                               fn, s->pallette,
+                               fn, s->palette,
                                &first, &last);
     if (first >= 0) {
-        dpy_update(s->ds, 0, first, s->cols, last - first + 1);
+        dpy_gfx_update(s->ds, 0, first, s->cols, last - first + 1);
     }
     s->invalidate = 0;
 }
@@ -253,13 +253,13 @@ static void pl110_invalidate_display(void * opaque)
     }
 }
 
-static void pl110_update_pallette(pl110_state *s, int n)
+static void pl110_update_palette(pl110_state *s, int n)
 {
     int i;
     uint32_t raw;
     unsigned int r, g, b;
 
-    raw = s->raw_pallette[n];
+    raw = s->raw_palette[n];
     n <<= 1;
     for (i = 0; i < 2; i++) {
         r = (raw & 0x1f) << 3;
@@ -271,17 +271,17 @@ static void pl110_update_pallette(pl110_state *s, int n)
         raw >>= 6;
         switch (ds_get_bits_per_pixel(s->ds)) {
         case 8:
-            s->pallette[n] = rgb_to_pixel8(r, g, b);
+            s->palette[n] = rgb_to_pixel8(r, g, b);
             break;
         case 15:
-            s->pallette[n] = rgb_to_pixel15(r, g, b);
+            s->palette[n] = rgb_to_pixel15(r, g, b);
             break;
         case 16:
-            s->pallette[n] = rgb_to_pixel16(r, g, b);
+            s->palette[n] = rgb_to_pixel16(r, g, b);
             break;
         case 24:
         case 32:
-            s->pallette[n] = rgb_to_pixel32(r, g, b);
+            s->palette[n] = rgb_to_pixel32(r, g, b);
             break;
         }
         n++;
@@ -305,7 +305,7 @@ static void pl110_update(pl110_state *s)
   /* TODO: Implement interrupts.  */
 }
 
-static uint64_t pl110_read(void *opaque, target_phys_addr_t offset,
+static uint64_t pl110_read(void *opaque, hwaddr offset,
                            unsigned size)
 {
     pl110_state *s = (pl110_state *)opaque;
@@ -314,7 +314,7 @@ static uint64_t pl110_read(void *opaque, target_phys_addr_t offset,
         return idregs[s->version][(offset - 0xfe0) >> 2];
     }
     if (offset >= 0x200 && offset < 0x400) {
-        return s->raw_pallette[(offset - 0x200) >> 2];
+        return s->raw_palette[(offset - 0x200) >> 2];
     }
     switch (offset >> 2) {
     case 0: /* LCDTiming0 */
@@ -349,12 +349,13 @@ static uint64_t pl110_read(void *opaque, target_phys_addr_t offset,
     case 12: /* LCDLPCURR */
         return s->lpbase;
     default:
-        hw_error("pl110_read: Bad offset %x\n", (int)offset);
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "pl110_read: Bad offset %x\n", (int)offset);
         return 0;
     }
 }
 
-static void pl110_write(void *opaque, target_phys_addr_t offset,
+static void pl110_write(void *opaque, hwaddr offset,
                         uint64_t val, unsigned size)
 {
     pl110_state *s = (pl110_state *)opaque;
@@ -364,10 +365,10 @@ static void pl110_write(void *opaque, target_phys_addr_t offset,
        is written to.  */
     s->invalidate = 1;
     if (offset >= 0x200 && offset < 0x400) {
-        /* Pallette.  */
+        /* Palette.  */
         n = (offset - 0x200) >> 2;
-        s->raw_pallette[(offset - 0x200) >> 2] = val;
-        pl110_update_pallette(s, n);
+        s->raw_palette[(offset - 0x200) >> 2] = val;
+        pl110_update_palette(s, n);
         return;
     }
     switch (offset >> 2) {
@@ -417,7 +418,8 @@ static void pl110_write(void *opaque, target_phys_addr_t offset,
         pl110_update(s);
         break;
     default:
-        hw_error("pl110_write: Bad offset %x\n", (int)offset);
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "pl110_write: Bad offset %x\n", (int)offset);
     }
 }
 
diff --git a/hw/pl110_template.h b/hw/pl110_template.h
index 1dce32a0c..e738e4a24 100644
--- a/hw/pl110_template.h
+++ b/hw/pl110_template.h
@@ -129,14 +129,14 @@ static drawfn glue(pl110_draw_fn_,BITS)[48] =
 
 static void glue(pl110_draw_line1_,NAME)(void *opaque, uint8_t *d, const uint8_t *src, int width, int deststep)
 {
-    uint32_t *pallette = opaque;
+    uint32_t *palette = opaque;
     uint32_t data;
     while (width > 0) {
         data = *(uint32_t *)src;
 #ifdef SWAP_PIXELS
-#define FN(x, y) COPY_PIXEL(d, pallette[(data >> (y + 7 - (x))) & 1]);
+#define FN(x, y) COPY_PIXEL(d, palette[(data >> (y + 7 - (x))) & 1]);
 #else
-#define FN(x, y) COPY_PIXEL(d, pallette[(data >> ((x) + y)) & 1]);
+#define FN(x, y) COPY_PIXEL(d, palette[(data >> ((x) + y)) & 1]);
 #endif
 #ifdef SWAP_WORDS
         FN_8(24)
@@ -157,14 +157,14 @@ static void glue(pl110_draw_line1_,NAME)(void *opaque, uint8_t *d, const uint8_t
 
 static void glue(pl110_draw_line2_,NAME)(void *opaque, uint8_t *d, const uint8_t *src, int width, int deststep)
 {
-    uint32_t *pallette = opaque;
+    uint32_t *palette = opaque;
     uint32_t data;
     while (width > 0) {
         data = *(uint32_t *)src;
 #ifdef SWAP_PIXELS
-#define FN(x, y) COPY_PIXEL(d, pallette[(data >> (y + 6 - (x)*2)) & 3]);
+#define FN(x, y) COPY_PIXEL(d, palette[(data >> (y + 6 - (x)*2)) & 3]);
 #else
-#define FN(x, y) COPY_PIXEL(d, pallette[(data >> ((x)*2 + y)) & 3]);
+#define FN(x, y) COPY_PIXEL(d, palette[(data >> ((x)*2 + y)) & 3]);
 #endif
 #ifdef SWAP_WORDS
         FN_4(0, 24)
@@ -185,14 +185,14 @@ static void glue(pl110_draw_line2_,NAME)(void *opaque, uint8_t *d, const uint8_t
 
 static void glue(pl110_draw_line4_,NAME)(void *opaque, uint8_t *d, const uint8_t *src, int width, int deststep)
 {
-    uint32_t *pallette = opaque;
+    uint32_t *palette = opaque;
     uint32_t data;
     while (width > 0) {
         data = *(uint32_t *)src;
 #ifdef SWAP_PIXELS
-#define FN(x, y) COPY_PIXEL(d, pallette[(data >> (y + 4 - (x)*4)) & 0xf]);
+#define FN(x, y) COPY_PIXEL(d, palette[(data >> (y + 4 - (x)*4)) & 0xf]);
 #else
-#define FN(x, y) COPY_PIXEL(d, pallette[(data >> ((x)*4 + y)) & 0xf]);
+#define FN(x, y) COPY_PIXEL(d, palette[(data >> ((x)*4 + y)) & 0xf]);
 #endif
 #ifdef SWAP_WORDS
         FN_2(0, 24)
@@ -213,11 +213,11 @@ static void glue(pl110_draw_line4_,NAME)(void *opaque, uint8_t *d, const uint8_t
 
 static void glue(pl110_draw_line8_,NAME)(void *opaque, uint8_t *d, const uint8_t *src, int width, int deststep)
 {
-    uint32_t *pallette = opaque;
+    uint32_t *palette = opaque;
     uint32_t data;
     while (width > 0) {
         data = *(uint32_t *)src;
-#define FN(x) COPY_PIXEL(d, pallette[(data >> (x)) & 0xff]);
+#define FN(x) COPY_PIXEL(d, palette[(data >> (x)) & 0xff]);
 #ifdef SWAP_WORDS
         FN(24)
         FN(16)
diff --git a/hw/pl181.c b/hw/pl181.c
index 7d91fbba1..8a2895ce1 100644
--- a/hw/pl181.c
+++ b/hw/pl181.c
@@ -285,7 +285,7 @@ static void pl181_fifo_run(pl181_state *s)
     }
 }
 
-static uint64_t pl181_read(void *opaque, target_phys_addr_t offset,
+static uint64_t pl181_read(void *opaque, hwaddr offset,
                            unsigned size)
 {
     pl181_state *s = (pl181_state *)opaque;
@@ -352,7 +352,7 @@ static uint64_t pl181_read(void *opaque, target_phys_addr_t offset,
     case 0xa0: case 0xa4: case 0xa8: case 0xac:
     case 0xb0: case 0xb4: case 0xb8: case 0xbc:
         if (s->fifo_len == 0) {
-            fprintf(stderr, "pl181: Unexpected FIFO read\n");
+            qemu_log_mask(LOG_GUEST_ERROR, "pl181: Unexpected FIFO read\n");
             return 0;
         } else {
             uint32_t value;
@@ -363,12 +363,13 @@ static uint64_t pl181_read(void *opaque, target_phys_addr_t offset,
             return value;
         }
     default:
-        hw_error("pl181_read: Bad offset %x\n", (int)offset);
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "pl181_read: Bad offset %x\n", (int)offset);
         return 0;
     }
 }
 
-static void pl181_write(void *opaque, target_phys_addr_t offset,
+static void pl181_write(void *opaque, hwaddr offset,
                         uint64_t value, unsigned size)
 {
     pl181_state *s = (pl181_state *)opaque;
@@ -387,11 +388,11 @@ static void pl181_write(void *opaque, target_phys_addr_t offset,
         s->cmd = value;
         if (s->cmd & PL181_CMD_ENABLE) {
             if (s->cmd & PL181_CMD_INTERRUPT) {
-                fprintf(stderr, "pl181: Interrupt mode not implemented\n");
-                abort();
+                qemu_log_mask(LOG_UNIMP,
+                              "pl181: Interrupt mode not implemented\n");
             } if (s->cmd & PL181_CMD_PENDING) {
-                fprintf(stderr, "pl181: Pending commands not implemented\n");
-                abort();
+                qemu_log_mask(LOG_UNIMP,
+                              "pl181: Pending commands not implemented\n");
             } else {
                 pl181_send_command(s);
                 pl181_fifo_run(s);
@@ -427,14 +428,15 @@ static void pl181_write(void *opaque, target_phys_addr_t offset,
     case 0xa0: case 0xa4: case 0xa8: case 0xac:
     case 0xb0: case 0xb4: case 0xb8: case 0xbc:
         if (s->datacnt == 0) {
-            fprintf(stderr, "pl181: Unexpected FIFO write\n");
+            qemu_log_mask(LOG_GUEST_ERROR, "pl181: Unexpected FIFO write\n");
         } else {
             pl181_fifo_push(s, value);
             pl181_fifo_run(s);
         }
         break;
     default:
-        hw_error("pl181_write: Bad offset %x\n", (int)offset);
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "pl181_write: Bad offset %x\n", (int)offset);
     }
     pl181_update(s);
 }
diff --git a/hw/pl190.c b/hw/pl190.c
index cb50afb9f..40199302a 100644
--- a/hw/pl190.c
+++ b/hw/pl190.c
@@ -85,7 +85,7 @@ static void pl190_update_vectors(pl190_state *s)
     pl190_update(s);
 }
 
-static uint64_t pl190_read(void *opaque, target_phys_addr_t offset,
+static uint64_t pl190_read(void *opaque, hwaddr offset,
                            unsigned size)
 {
     pl190_state *s = (pl190_state *)opaque;
@@ -117,12 +117,18 @@ static uint64_t pl190_read(void *opaque, target_phys_addr_t offset,
         return s->protected;
     case 12: /* VECTADDR */
         /* Read vector address at the start of an ISR.  Increases the
-           current priority level to that of the current interrupt.  */
-        for (i = 0; i < s->priority; i++)
-          {
-            if ((s->level | s->soft_level) & s->prio_mask[i])
-              break;
-          }
+         * current priority level to that of the current interrupt.
+         *
+         * Since an enabled interrupt X at priority P causes prio_mask[Y]
+         * to have bit X set for all Y > P, this loop will stop with
+         * i == the priority of the highest priority set interrupt.
+         */
+        for (i = 0; i < s->priority; i++) {
+            if ((s->level | s->soft_level) & s->prio_mask[i + 1]) {
+                break;
+            }
+        }
+
         /* Reading this value with no pending interrupts is undefined.
            We return the default address.  */
         if (i == PL190_NUM_PRIO)
@@ -137,12 +143,13 @@ static uint64_t pl190_read(void *opaque, target_phys_addr_t offset,
     case 13: /* DEFVECTADDR */
         return s->vect_addr[16];
     default:
-        hw_error("pl190_read: Bad offset %x\n", (int)offset);
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "pl190_read: Bad offset %x\n", (int)offset);
         return 0;
     }
 }
 
-static void pl190_write(void *opaque, target_phys_addr_t offset,
+static void pl190_write(void *opaque, hwaddr offset,
                         uint64_t val, unsigned size)
 {
     pl190_state *s = (pl190_state *)opaque;
@@ -192,11 +199,12 @@ static void pl190_write(void *opaque, target_phys_addr_t offset,
         break;
     case 0xc0: /* ITCR */
         if (val) {
-            hw_error("pl190: Test mode not implemented\n");
+            qemu_log_mask(LOG_UNIMP, "pl190: Test mode not implemented\n");
         }
         break;
     default:
-        hw_error("pl190_write: Bad offset %x\n", (int)offset);
+        qemu_log_mask(LOG_GUEST_ERROR,
+                     "pl190_write: Bad offset %x\n", (int)offset);
         return;
     }
     pl190_update(s);
diff --git a/hw/ppc.c b/hw/ppc.c
index 98546de99..11fd199ea 100644
--- a/hw/ppc.c
+++ b/hw/ppc.c
@@ -75,9 +75,10 @@ void ppc_set_irq(CPUPPCState *env, int n_IRQ, int level)
 }
 
 /* PowerPC 6xx / 7xx internal IRQ controller */
-static void ppc6xx_set_irq (void *opaque, int pin, int level)
+static void ppc6xx_set_irq(void *opaque, int pin, int level)
 {
-    CPUPPCState *env = opaque;
+    PowerPCCPU *cpu = opaque;
+    CPUPPCState *env = &cpu->env;
     int cur_level;
 
     LOG_IRQ("%s: env %p pin %d level %d\n", __func__,
@@ -151,17 +152,20 @@ static void ppc6xx_set_irq (void *opaque, int pin, int level)
     }
 }
 
-void ppc6xx_irq_init (CPUPPCState *env)
+void ppc6xx_irq_init(CPUPPCState *env)
 {
-    env->irq_inputs = (void **)qemu_allocate_irqs(&ppc6xx_set_irq, env,
+    PowerPCCPU *cpu = ppc_env_get_cpu(env);
+
+    env->irq_inputs = (void **)qemu_allocate_irqs(&ppc6xx_set_irq, cpu,
                                                   PPC6xx_INPUT_NB);
 }
 
 #if defined(TARGET_PPC64)
 /* PowerPC 970 internal IRQ controller */
-static void ppc970_set_irq (void *opaque, int pin, int level)
+static void ppc970_set_irq(void *opaque, int pin, int level)
 {
-    CPUPPCState *env = opaque;
+    PowerPCCPU *cpu = opaque;
+    CPUPPCState *env = &cpu->env;
     int cur_level;
 
     LOG_IRQ("%s: env %p pin %d level %d\n", __func__,
@@ -202,7 +206,7 @@ static void ppc970_set_irq (void *opaque, int pin, int level)
             } else {
                 LOG_IRQ("%s: restart the CPU\n", __func__);
                 env->halted = 0;
-                qemu_cpu_kick(env);
+                qemu_cpu_kick(CPU(cpu));
             }
             break;
         case PPC970_INPUT_HRESET:
@@ -233,16 +237,19 @@ static void ppc970_set_irq (void *opaque, int pin, int level)
     }
 }
 
-void ppc970_irq_init (CPUPPCState *env)
+void ppc970_irq_init(CPUPPCState *env)
 {
-    env->irq_inputs = (void **)qemu_allocate_irqs(&ppc970_set_irq, env,
+    PowerPCCPU *cpu = ppc_env_get_cpu(env);
+
+    env->irq_inputs = (void **)qemu_allocate_irqs(&ppc970_set_irq, cpu,
                                                   PPC970_INPUT_NB);
 }
 
 /* POWER7 internal IRQ controller */
-static void power7_set_irq (void *opaque, int pin, int level)
+static void power7_set_irq(void *opaque, int pin, int level)
 {
-    CPUPPCState *env = opaque;
+    PowerPCCPU *cpu = opaque;
+    CPUPPCState *env = &cpu->env;
 
     LOG_IRQ("%s: env %p pin %d level %d\n", __func__,
                 env, pin, level);
@@ -266,17 +273,20 @@ static void power7_set_irq (void *opaque, int pin, int level)
     }
 }
 
-void ppcPOWER7_irq_init (CPUPPCState *env)
+void ppcPOWER7_irq_init(CPUPPCState *env)
 {
-    env->irq_inputs = (void **)qemu_allocate_irqs(&power7_set_irq, env,
+    PowerPCCPU *cpu = ppc_env_get_cpu(env);
+
+    env->irq_inputs = (void **)qemu_allocate_irqs(&power7_set_irq, cpu,
                                                   POWER7_INPUT_NB);
 }
 #endif /* defined(TARGET_PPC64) */
 
 /* PowerPC 40x internal IRQ controller */
-static void ppc40x_set_irq (void *opaque, int pin, int level)
+static void ppc40x_set_irq(void *opaque, int pin, int level)
 {
-    CPUPPCState *env = opaque;
+    PowerPCCPU *cpu = opaque;
+    CPUPPCState *env = &cpu->env;
     int cur_level;
 
     LOG_IRQ("%s: env %p pin %d level %d\n", __func__,
@@ -325,7 +335,7 @@ static void ppc40x_set_irq (void *opaque, int pin, int level)
             } else {
                 LOG_IRQ("%s: restart the CPU\n", __func__);
                 env->halted = 0;
-                qemu_cpu_kick(env);
+                qemu_cpu_kick(CPU(cpu));
             }
             break;
         case PPC40x_INPUT_DEBUG:
@@ -346,16 +356,19 @@ static void ppc40x_set_irq (void *opaque, int pin, int level)
     }
 }
 
-void ppc40x_irq_init (CPUPPCState *env)
+void ppc40x_irq_init(CPUPPCState *env)
 {
+    PowerPCCPU *cpu = ppc_env_get_cpu(env);
+
     env->irq_inputs = (void **)qemu_allocate_irqs(&ppc40x_set_irq,
-                                                  env, PPC40x_INPUT_NB);
+                                                  cpu, PPC40x_INPUT_NB);
 }
 
 /* PowerPC E500 internal IRQ controller */
-static void ppce500_set_irq (void *opaque, int pin, int level)
+static void ppce500_set_irq(void *opaque, int pin, int level)
 {
-    CPUPPCState *env = opaque;
+    PowerPCCPU *cpu = opaque;
+    CPUPPCState *env = &cpu->env;
     int cur_level;
 
     LOG_IRQ("%s: env %p pin %d level %d\n", __func__,
@@ -407,10 +420,12 @@ static void ppce500_set_irq (void *opaque, int pin, int level)
     }
 }
 
-void ppce500_irq_init (CPUPPCState *env)
+void ppce500_irq_init(CPUPPCState *env)
 {
+    PowerPCCPU *cpu = ppc_env_get_cpu(env);
+
     env->irq_inputs = (void **)qemu_allocate_irqs(&ppce500_set_irq,
-                                        env, PPCE500_INPUT_NB);
+                                                  cpu, PPCE500_INPUT_NB);
 }
 /*****************************************************************************/
 /* PowerPC time base and decrementer emulation */
@@ -721,7 +736,7 @@ static void cpu_ppc_hdecr_cb (void *opaque)
     _cpu_ppc_store_hdecr(opaque, 0x00000000, 0xFFFFFFFF, 1);
 }
 
-void cpu_ppc_store_purr (CPUPPCState *env, uint64_t value)
+static void cpu_ppc_store_purr(CPUPPCState *env, uint64_t value)
 {
     ppc_tb_t *tb_env = env->tb_env;
 
@@ -1152,23 +1167,23 @@ static inline void nvram_write (nvram_t *nvram, uint32_t addr, uint32_t val)
     (*nvram->write_fn)(nvram->opaque, addr, val);
 }
 
-void NVRAM_set_byte (nvram_t *nvram, uint32_t addr, uint8_t value)
+static void NVRAM_set_byte(nvram_t *nvram, uint32_t addr, uint8_t value)
 {
     nvram_write(nvram, addr, value);
 }
 
-uint8_t NVRAM_get_byte (nvram_t *nvram, uint32_t addr)
+static uint8_t NVRAM_get_byte(nvram_t *nvram, uint32_t addr)
 {
     return nvram_read(nvram, addr);
 }
 
-void NVRAM_set_word (nvram_t *nvram, uint32_t addr, uint16_t value)
+static void NVRAM_set_word(nvram_t *nvram, uint32_t addr, uint16_t value)
 {
     nvram_write(nvram, addr, value >> 8);
     nvram_write(nvram, addr + 1, value & 0xFF);
 }
 
-uint16_t NVRAM_get_word (nvram_t *nvram, uint32_t addr)
+static uint16_t NVRAM_get_word(nvram_t *nvram, uint32_t addr)
 {
     uint16_t tmp;
 
@@ -1178,7 +1193,7 @@ uint16_t NVRAM_get_word (nvram_t *nvram, uint32_t addr)
     return tmp;
 }
 
-void NVRAM_set_lword (nvram_t *nvram, uint32_t addr, uint32_t value)
+static void NVRAM_set_lword(nvram_t *nvram, uint32_t addr, uint32_t value)
 {
     nvram_write(nvram, addr, value >> 24);
     nvram_write(nvram, addr + 1, (value >> 16) & 0xFF);
@@ -1198,8 +1213,8 @@ uint32_t NVRAM_get_lword (nvram_t *nvram, uint32_t addr)
     return tmp;
 }
 
-void NVRAM_set_string (nvram_t *nvram, uint32_t addr,
-                       const char *str, uint32_t max)
+static void NVRAM_set_string(nvram_t *nvram, uint32_t addr, const char *str,
+                             uint32_t max)
 {
     int i;
 
diff --git a/hw/ppc/Makefile.objs b/hw/ppc/Makefile.objs
index 951e407f1..8fe21235f 100644
--- a/hw/ppc/Makefile.objs
+++ b/hw/ppc/Makefile.objs
@@ -11,6 +11,7 @@ obj-y += ppc_newworld.o
 obj-$(CONFIG_PSERIES) += spapr.o spapr_hcall.o spapr_rtas.o spapr_vio.o
 obj-$(CONFIG_PSERIES) += xics.o spapr_vty.o spapr_llan.o spapr_vscsi.o
 obj-$(CONFIG_PSERIES) += spapr_pci.o pci-hotplug.o spapr_iommu.o
+obj-$(CONFIG_PSERIES) += spapr_events.o
 # PowerPC 4xx boards
 obj-y += ppc4xx_devs.o ppc4xx_pci.o ppc405_uc.o ppc405_boards.o
 obj-y += ppc440_bamboo.o
diff --git a/hw/ppc/e500.c b/hw/ppc/e500.c
index 6f0de6d95..6749ffffb 100644
--- a/hw/ppc/e500.c
+++ b/hw/ppc/e500.c
@@ -19,7 +19,7 @@
 #include "e500.h"
 #include "net.h"
 #include "hw/hw.h"
-#include "hw/pc.h"
+#include "hw/serial.h"
 #include "hw/pci.h"
 #include "hw/boards.h"
 #include "sysemu.h"
@@ -36,7 +36,7 @@
 
 #define BINARY_DEVICE_TREE_FILE    "mpc8544ds.dtb"
 #define UIMAGE_LOAD_BASE           0
-#define DTC_LOAD_PAD               0x500000
+#define DTC_LOAD_PAD               0x1800000
 #define DTC_PAD_MASK               0xFFFFF
 #define INITRD_LOAD_PAD            0x2000000
 #define INITRD_PAD_MASK            0xFFFFFF
@@ -52,7 +52,6 @@
 #define MPC8544_PCI_REGS_BASE      (MPC8544_CCSRBAR_BASE + 0x8000ULL)
 #define MPC8544_PCI_REGS_SIZE      0x1000ULL
 #define MPC8544_PCI_IO             0xE1000000ULL
-#define MPC8544_PCI_IOLEN          0x10000ULL
 #define MPC8544_UTIL_BASE          (MPC8544_CCSRBAR_BASE + 0xe0000ULL)
 #define MPC8544_SPIN_BASE          0xEF000000ULL
 
@@ -108,9 +107,9 @@ static void dt_serial_create(void *fdt, unsigned long long offset,
 
 static int ppce500_load_device_tree(CPUPPCState *env,
                                     PPCE500Params *params,
-                                    target_phys_addr_t addr,
-                                    target_phys_addr_t initrd_base,
-                                    target_phys_addr_t initrd_size)
+                                    hwaddr addr,
+                                    hwaddr initrd_base,
+                                    hwaddr initrd_size)
 {
     int ret = -1;
     uint64_t mem_reg_property[] = { 0, cpu_to_be64(params->ram_size) };
@@ -139,12 +138,10 @@ static int ppce500_load_device_tree(CPUPPCState *env,
             0x0, 0x10000,
         };
     QemuOpts *machine_opts;
-    const char *dumpdtb = NULL;
     const char *dtb_file = NULL;
 
     machine_opts = qemu_opts_find(qemu_find_opts("machine"), 0);
     if (machine_opts) {
-        dumpdtb = qemu_opt_get(machine_opts, "dumpdtb");
         dtb_file = qemu_opt_get(machine_opts, "dtb");
         toplevel_compat = qemu_opt_get(machine_opts, "dt_compatible");
     }
@@ -334,18 +331,7 @@ static int ppce500_load_device_tree(CPUPPCState *env,
     }
 
 done:
-    if (dumpdtb) {
-        /* Dump the dtb to a file and quit */
-        FILE *f = fopen(dumpdtb, "wb");
-        size_t len;
-        len = fwrite(fdt, fdt_size, 1, f);
-        fclose(f);
-        if (len != fdt_size) {
-            exit(1);
-        }
-        exit(0);
-    }
-
+    qemu_devtree_dumpdtb(fdt, fdt_size);
     ret = rom_add_blob_fixed(BINARY_DEVICE_TREE_FILE, fdt, fdt_size, addr);
     if (ret < 0) {
         goto out;
@@ -359,7 +345,7 @@ out:
 }
 
 /* Create -kernel TLB entries for BookE.  */
-static inline target_phys_addr_t booke206_page_size_to_tlb(uint64_t size)
+static inline hwaddr booke206_page_size_to_tlb(uint64_t size)
 {
     return 63 - clz64(size >> 10);
 }
@@ -368,13 +354,17 @@ static void mmubooke_create_initial_mapping(CPUPPCState *env)
 {
     struct boot_info *bi = env->load_info;
     ppcmas_tlb_t *tlb = booke206_get_tlbm(env, 1, 0, 0);
-    target_phys_addr_t size, dt_end;
+    hwaddr size, dt_end;
     int ps;
 
     /* Our initial TLB entry needs to cover everything from 0 to
        the device tree top */
     dt_end = bi->dt_base + bi->dt_size;
     ps = booke206_page_size_to_tlb(dt_end) + 1;
+    if (ps & 1) {
+        /* e500v2 can only do even TLB size bits */
+        ps++;
+    }
     size = (ps << MAS1_TSIZE_SHIFT);
     tlb->mas1 = MAS1_VALID | size;
     tlb->mas2 = 0;
@@ -421,8 +411,8 @@ void ppce500_init(PPCE500Params *params)
     CPUPPCState *env = NULL;
     uint64_t elf_entry;
     uint64_t elf_lowaddr;
-    target_phys_addr_t entry=0;
-    target_phys_addr_t loadaddr=UIMAGE_LOAD_BASE;
+    hwaddr entry=0;
+    hwaddr loadaddr=UIMAGE_LOAD_BASE;
     target_long kernel_size=0;
     target_ulong dt_base = 0;
     target_ulong initrd_base = 0;
@@ -505,7 +495,7 @@ void ppce500_init(PPCE500Params *params)
     if (serial_hds[1]) {
         serial_mm_init(address_space_mem, MPC8544_SERIAL1_REGS_BASE,
                        0, mpic[12+26], 399193,
-                       serial_hds[0], DEVICE_BIG_ENDIAN);
+                       serial_hds[1], DEVICE_BIG_ENDIAN);
     }
 
     /* General Utility device */
@@ -520,7 +510,7 @@ void ppce500_init(PPCE500Params *params)
     if (!pci_bus)
         printf("couldn't create PCI controller!\n");
 
-    isa_mmio_init(MPC8544_PCI_IO, MPC8544_PCI_IOLEN);
+    sysbus_mmio_map(sysbus_from_qdev(dev), 1, MPC8544_PCI_IO);
 
     if (pci_bus) {
         /* Register network interfaces. */
@@ -553,7 +543,8 @@ void ppce500_init(PPCE500Params *params)
 
     /* Load initrd. */
     if (params->initrd_filename) {
-        initrd_base = (kernel_size + INITRD_LOAD_PAD) & ~INITRD_PAD_MASK;
+        initrd_base = (loadaddr + kernel_size + INITRD_LOAD_PAD) &
+            ~INITRD_PAD_MASK;
         initrd_size = load_image_targphys(params->initrd_filename, initrd_base,
                                           ram_size - initrd_base);
 
diff --git a/hw/ppc/e500plat.c b/hw/ppc/e500plat.c
index 60a5cb3bd..4cfb94061 100644
--- a/hw/ppc/e500plat.c
+++ b/hw/ppc/e500plat.c
@@ -25,13 +25,14 @@ static void e500plat_fixup_devtree(PPCE500Params *params, void *fdt)
                          sizeof(compatible));
 }
 
-static void e500plat_init(ram_addr_t ram_size,
-                           const char *boot_device,
-                           const char *kernel_filename,
-                           const char *kernel_cmdline,
-                           const char *initrd_filename,
-                           const char *cpu_model)
+static void e500plat_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t ram_size = args->ram_size;
+    const char *boot_device = args->boot_device;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
     PPCE500Params params = {
         .ram_size = ram_size,
         .boot_device = boot_device,
diff --git a/hw/ppc/mpc8544ds.c b/hw/ppc/mpc8544ds.c
index 984d21cbf..e65166194 100644
--- a/hw/ppc/mpc8544ds.c
+++ b/hw/ppc/mpc8544ds.c
@@ -25,13 +25,14 @@ static void mpc8544ds_fixup_devtree(PPCE500Params *params, void *fdt)
                          sizeof(compatible));
 }
 
-static void mpc8544ds_init(ram_addr_t ram_size,
-                           const char *boot_device,
-                           const char *kernel_filename,
-                           const char *kernel_cmdline,
-                           const char *initrd_filename,
-                           const char *cpu_model)
+static void mpc8544ds_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t ram_size = args->ram_size;
+    const char *boot_device = args->boot_device;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
     PPCE500Params params = {
         .ram_size = ram_size,
         .boot_device = boot_device,
diff --git a/hw/ppc405.h b/hw/ppc405.h
index 1f5dc5fd3..535cbfb33 100644
--- a/hw/ppc405.h
+++ b/hw/ppc405.h
@@ -61,20 +61,20 @@ ram_addr_t ppc405_set_bootinfo (CPUPPCState *env, ppc4xx_bd_info_t *bd,
 
 CPUPPCState *ppc405cr_init(MemoryRegion *address_space_mem,
                         MemoryRegion ram_memories[4],
-                        target_phys_addr_t ram_bases[4],
-                        target_phys_addr_t ram_sizes[4],
+                        hwaddr ram_bases[4],
+                        hwaddr ram_sizes[4],
                         uint32_t sysclk, qemu_irq **picp,
                         int do_init);
 CPUPPCState *ppc405ep_init(MemoryRegion *address_space_mem,
                         MemoryRegion ram_memories[2],
-                        target_phys_addr_t ram_bases[2],
-                        target_phys_addr_t ram_sizes[2],
+                        hwaddr ram_bases[2],
+                        hwaddr ram_sizes[2],
                         uint32_t sysclk, qemu_irq **picp,
                         int do_init);
 /* IBM STBxxx microcontrollers */
 CPUPPCState *ppc_stb025_init (MemoryRegion ram_memories[2],
-                           target_phys_addr_t ram_bases[2],
-                           target_phys_addr_t ram_sizes[2],
+                           hwaddr ram_bases[2],
+                           hwaddr ram_sizes[2],
                            uint32_t sysclk, qemu_irq **picp,
                            ram_addr_t *offsetp);
 
diff --git a/hw/ppc405_boards.c b/hw/ppc405_boards.c
index 476775d05..8dc693f05 100644
--- a/hw/ppc405_boards.c
+++ b/hw/ppc405_boards.c
@@ -60,7 +60,7 @@ struct ref405ep_fpga_t {
     uint8_t reg1;
 };
 
-static uint32_t ref405ep_fpga_readb (void *opaque, target_phys_addr_t addr)
+static uint32_t ref405ep_fpga_readb (void *opaque, hwaddr addr)
 {
     ref405ep_fpga_t *fpga;
     uint32_t ret;
@@ -82,7 +82,7 @@ static uint32_t ref405ep_fpga_readb (void *opaque, target_phys_addr_t addr)
 }
 
 static void ref405ep_fpga_writeb (void *opaque,
-                                  target_phys_addr_t addr, uint32_t value)
+                                  hwaddr addr, uint32_t value)
 {
     ref405ep_fpga_t *fpga;
 
@@ -99,7 +99,7 @@ static void ref405ep_fpga_writeb (void *opaque,
     }
 }
 
-static uint32_t ref405ep_fpga_readw (void *opaque, target_phys_addr_t addr)
+static uint32_t ref405ep_fpga_readw (void *opaque, hwaddr addr)
 {
     uint32_t ret;
 
@@ -110,13 +110,13 @@ static uint32_t ref405ep_fpga_readw (void *opaque, target_phys_addr_t addr)
 }
 
 static void ref405ep_fpga_writew (void *opaque,
-                                  target_phys_addr_t addr, uint32_t value)
+                                  hwaddr addr, uint32_t value)
 {
     ref405ep_fpga_writeb(opaque, addr, (value >> 8) & 0xFF);
     ref405ep_fpga_writeb(opaque, addr + 1, value & 0xFF);
 }
 
-static uint32_t ref405ep_fpga_readl (void *opaque, target_phys_addr_t addr)
+static uint32_t ref405ep_fpga_readl (void *opaque, hwaddr addr)
 {
     uint32_t ret;
 
@@ -129,7 +129,7 @@ static uint32_t ref405ep_fpga_readl (void *opaque, target_phys_addr_t addr)
 }
 
 static void ref405ep_fpga_writel (void *opaque,
-                                  target_phys_addr_t addr, uint32_t value)
+                                  hwaddr addr, uint32_t value)
 {
     ref405ep_fpga_writeb(opaque, addr, (value >> 24) & 0xFF);
     ref405ep_fpga_writeb(opaque, addr + 1, (value >> 16) & 0xFF);
@@ -158,7 +158,7 @@ static void ref405ep_fpga_reset (void *opaque)
     fpga->reg1 = 0x0F;
 }
 
-static void ref405ep_fpga_init (MemoryRegion *sysmem, uint32_t base)
+static void ref405ep_fpga_init(MemoryRegion *sysmem, uint32_t base)
 {
     ref405ep_fpga_t *fpga;
     MemoryRegion *fpga_memory = g_new(MemoryRegion, 1);
@@ -170,13 +170,12 @@ static void ref405ep_fpga_init (MemoryRegion *sysmem, uint32_t base)
     qemu_register_reset(&ref405ep_fpga_reset, fpga);
 }
 
-static void ref405ep_init (ram_addr_t ram_size,
-                           const char *boot_device,
-                           const char *kernel_filename,
-                           const char *kernel_cmdline,
-                           const char *initrd_filename,
-                           const char *cpu_model)
+static void ref405ep_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t ram_size = args->ram_size;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
     char *filename;
     ppc4xx_bd_info_t bd;
     CPUPPCState *env;
@@ -185,7 +184,7 @@ static void ref405ep_init (ram_addr_t ram_size,
     MemoryRegion *sram = g_new(MemoryRegion, 1);
     ram_addr_t bdloc;
     MemoryRegion *ram_memories = g_malloc(2 * sizeof(*ram_memories));
-    target_phys_addr_t ram_bases[2], ram_sizes[2];
+    hwaddr ram_bases[2], ram_sizes[2];
     target_ulong sram_size;
     long bios_size;
     //int phy_addr = 0;
@@ -390,7 +389,7 @@ struct taihu_cpld_t {
     uint8_t reg1;
 };
 
-static uint32_t taihu_cpld_readb (void *opaque, target_phys_addr_t addr)
+static uint32_t taihu_cpld_readb (void *opaque, hwaddr addr)
 {
     taihu_cpld_t *cpld;
     uint32_t ret;
@@ -412,7 +411,7 @@ static uint32_t taihu_cpld_readb (void *opaque, target_phys_addr_t addr)
 }
 
 static void taihu_cpld_writeb (void *opaque,
-                               target_phys_addr_t addr, uint32_t value)
+                               hwaddr addr, uint32_t value)
 {
     taihu_cpld_t *cpld;
 
@@ -429,7 +428,7 @@ static void taihu_cpld_writeb (void *opaque,
     }
 }
 
-static uint32_t taihu_cpld_readw (void *opaque, target_phys_addr_t addr)
+static uint32_t taihu_cpld_readw (void *opaque, hwaddr addr)
 {
     uint32_t ret;
 
@@ -440,13 +439,13 @@ static uint32_t taihu_cpld_readw (void *opaque, target_phys_addr_t addr)
 }
 
 static void taihu_cpld_writew (void *opaque,
-                               target_phys_addr_t addr, uint32_t value)
+                               hwaddr addr, uint32_t value)
 {
     taihu_cpld_writeb(opaque, addr, (value >> 8) & 0xFF);
     taihu_cpld_writeb(opaque, addr + 1, value & 0xFF);
 }
 
-static uint32_t taihu_cpld_readl (void *opaque, target_phys_addr_t addr)
+static uint32_t taihu_cpld_readl (void *opaque, hwaddr addr)
 {
     uint32_t ret;
 
@@ -459,7 +458,7 @@ static uint32_t taihu_cpld_readl (void *opaque, target_phys_addr_t addr)
 }
 
 static void taihu_cpld_writel (void *opaque,
-                               target_phys_addr_t addr, uint32_t value)
+                               hwaddr addr, uint32_t value)
 {
     taihu_cpld_writel(opaque, addr, (value >> 24) & 0xFF);
     taihu_cpld_writel(opaque, addr + 1, (value >> 16) & 0xFF);
@@ -484,7 +483,7 @@ static void taihu_cpld_reset (void *opaque)
     cpld->reg1 = 0x80;
 }
 
-static void taihu_cpld_init (MemoryRegion *sysmem, uint32_t base)
+static void taihu_cpld_init(MemoryRegion *sysmem, uint32_t base)
 {
     taihu_cpld_t *cpld;
     MemoryRegion *cpld_memory = g_new(MemoryRegion, 1);
@@ -495,19 +494,17 @@ static void taihu_cpld_init (MemoryRegion *sysmem, uint32_t base)
     qemu_register_reset(&taihu_cpld_reset, cpld);
 }
 
-static void taihu_405ep_init(ram_addr_t ram_size,
-                             const char *boot_device,
-                             const char *kernel_filename,
-                             const char *kernel_cmdline,
-                             const char *initrd_filename,
-                             const char *cpu_model)
+static void taihu_405ep_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t ram_size = args->ram_size;
+    const char *kernel_filename = args->kernel_filename;
+    const char *initrd_filename = args->initrd_filename;
     char *filename;
     qemu_irq *pic;
     MemoryRegion *sysmem = get_system_memory();
     MemoryRegion *bios;
     MemoryRegion *ram_memories = g_malloc(2 * sizeof(*ram_memories));
-    target_phys_addr_t ram_bases[2], ram_sizes[2];
+    hwaddr ram_bases[2], ram_sizes[2];
     long bios_size;
     target_ulong kernel_base, initrd_base;
     long kernel_size, initrd_size;
diff --git a/hw/ppc405_uc.c b/hw/ppc405_uc.c
index 89e5013b5..0f458ef77 100644
--- a/hw/ppc405_uc.c
+++ b/hw/ppc405_uc.c
@@ -24,7 +24,7 @@
 #include "hw.h"
 #include "ppc.h"
 #include "ppc405.h"
-#include "pc.h"
+#include "serial.h"
 #include "qemu-timer.h"
 #include "sysemu.h"
 #include "qemu-log.h"
@@ -191,7 +191,8 @@ enum {
 typedef struct ppc4xx_pob_t ppc4xx_pob_t;
 struct ppc4xx_pob_t {
     uint32_t bear;
-    uint32_t besr[2];
+    uint32_t besr0;
+    uint32_t besr1;
 };
 
 static uint32_t dcr_read_pob (void *opaque, int dcrn)
@@ -205,8 +206,10 @@ static uint32_t dcr_read_pob (void *opaque, int dcrn)
         ret = pob->bear;
         break;
     case POB0_BESR0:
+        ret = pob->besr0;
+        break;
     case POB0_BESR1:
-        ret = pob->besr[dcrn - POB0_BESR0];
+        ret = pob->besr1;
         break;
     default:
         /* Avoid gcc warning */
@@ -227,9 +230,12 @@ static void dcr_write_pob (void *opaque, int dcrn, uint32_t val)
         /* Read only */
         break;
     case POB0_BESR0:
+        /* Write-clear */
+        pob->besr0 &= ~val;
+        break;
     case POB0_BESR1:
         /* Write-clear */
-        pob->besr[dcrn - POB0_BESR0] &= ~val;
+        pob->besr1 &= ~val;
         break;
     }
 }
@@ -241,8 +247,8 @@ static void ppc4xx_pob_reset (void *opaque)
     pob = opaque;
     /* No error */
     pob->bear = 0x00000000;
-    pob->besr[0] = 0x0000000;
-    pob->besr[1] = 0x0000000;
+    pob->besr0 = 0x0000000;
+    pob->besr1 = 0x0000000;
 }
 
 static void ppc4xx_pob_init(CPUPPCState *env)
@@ -265,7 +271,7 @@ struct ppc4xx_opba_t {
     uint8_t pr;
 };
 
-static uint32_t opba_readb (void *opaque, target_phys_addr_t addr)
+static uint32_t opba_readb (void *opaque, hwaddr addr)
 {
     ppc4xx_opba_t *opba;
     uint32_t ret;
@@ -290,7 +296,7 @@ static uint32_t opba_readb (void *opaque, target_phys_addr_t addr)
 }
 
 static void opba_writeb (void *opaque,
-                         target_phys_addr_t addr, uint32_t value)
+                         hwaddr addr, uint32_t value)
 {
     ppc4xx_opba_t *opba;
 
@@ -311,7 +317,7 @@ static void opba_writeb (void *opaque,
     }
 }
 
-static uint32_t opba_readw (void *opaque, target_phys_addr_t addr)
+static uint32_t opba_readw (void *opaque, hwaddr addr)
 {
     uint32_t ret;
 
@@ -325,7 +331,7 @@ static uint32_t opba_readw (void *opaque, target_phys_addr_t addr)
 }
 
 static void opba_writew (void *opaque,
-                         target_phys_addr_t addr, uint32_t value)
+                         hwaddr addr, uint32_t value)
 {
 #ifdef DEBUG_OPBA
     printf("%s: addr " TARGET_FMT_plx " val %08" PRIx32 "\n", __func__, addr,
@@ -335,7 +341,7 @@ static void opba_writew (void *opaque,
     opba_writeb(opaque, addr + 1, value);
 }
 
-static uint32_t opba_readl (void *opaque, target_phys_addr_t addr)
+static uint32_t opba_readl (void *opaque, hwaddr addr)
 {
     uint32_t ret;
 
@@ -349,7 +355,7 @@ static uint32_t opba_readl (void *opaque, target_phys_addr_t addr)
 }
 
 static void opba_writel (void *opaque,
-                         target_phys_addr_t addr, uint32_t value)
+                         hwaddr addr, uint32_t value)
 {
 #ifdef DEBUG_OPBA
     printf("%s: addr " TARGET_FMT_plx " val %08" PRIx32 "\n", __func__, addr,
@@ -376,7 +382,7 @@ static void ppc4xx_opba_reset (void *opaque)
     opba->pr = 0x11;
 }
 
-static void ppc4xx_opba_init(target_phys_addr_t base)
+static void ppc4xx_opba_init(hwaddr base)
 {
     ppc4xx_opba_t *opba;
 
@@ -732,7 +738,7 @@ struct ppc405_gpio_t {
     uint32_t isr1l;
 };
 
-static uint32_t ppc405_gpio_readb (void *opaque, target_phys_addr_t addr)
+static uint32_t ppc405_gpio_readb (void *opaque, hwaddr addr)
 {
 #ifdef DEBUG_GPIO
     printf("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
@@ -742,7 +748,7 @@ static uint32_t ppc405_gpio_readb (void *opaque, target_phys_addr_t addr)
 }
 
 static void ppc405_gpio_writeb (void *opaque,
-                                target_phys_addr_t addr, uint32_t value)
+                                hwaddr addr, uint32_t value)
 {
 #ifdef DEBUG_GPIO
     printf("%s: addr " TARGET_FMT_plx " val %08" PRIx32 "\n", __func__, addr,
@@ -750,7 +756,7 @@ static void ppc405_gpio_writeb (void *opaque,
 #endif
 }
 
-static uint32_t ppc405_gpio_readw (void *opaque, target_phys_addr_t addr)
+static uint32_t ppc405_gpio_readw (void *opaque, hwaddr addr)
 {
 #ifdef DEBUG_GPIO
     printf("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
@@ -760,7 +766,7 @@ static uint32_t ppc405_gpio_readw (void *opaque, target_phys_addr_t addr)
 }
 
 static void ppc405_gpio_writew (void *opaque,
-                                target_phys_addr_t addr, uint32_t value)
+                                hwaddr addr, uint32_t value)
 {
 #ifdef DEBUG_GPIO
     printf("%s: addr " TARGET_FMT_plx " val %08" PRIx32 "\n", __func__, addr,
@@ -768,7 +774,7 @@ static void ppc405_gpio_writew (void *opaque,
 #endif
 }
 
-static uint32_t ppc405_gpio_readl (void *opaque, target_phys_addr_t addr)
+static uint32_t ppc405_gpio_readl (void *opaque, hwaddr addr)
 {
 #ifdef DEBUG_GPIO
     printf("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
@@ -778,7 +784,7 @@ static uint32_t ppc405_gpio_readl (void *opaque, target_phys_addr_t addr)
 }
 
 static void ppc405_gpio_writel (void *opaque,
-                                target_phys_addr_t addr, uint32_t value)
+                                hwaddr addr, uint32_t value)
 {
 #ifdef DEBUG_GPIO
     printf("%s: addr " TARGET_FMT_plx " val %08" PRIx32 "\n", __func__, addr,
@@ -798,7 +804,7 @@ static void ppc405_gpio_reset (void *opaque)
 {
 }
 
-static void ppc405_gpio_init(target_phys_addr_t base)
+static void ppc405_gpio_init(hwaddr base)
 {
     ppc405_gpio_t *gpio;
 
@@ -1004,7 +1010,7 @@ struct ppc4xx_i2c_t {
     uint8_t directcntl;
 };
 
-static uint32_t ppc4xx_i2c_readb (void *opaque, target_phys_addr_t addr)
+static uint32_t ppc4xx_i2c_readb (void *opaque, hwaddr addr)
 {
     ppc4xx_i2c_t *i2c;
     uint32_t ret;
@@ -1072,7 +1078,7 @@ static uint32_t ppc4xx_i2c_readb (void *opaque, target_phys_addr_t addr)
 }
 
 static void ppc4xx_i2c_writeb (void *opaque,
-                               target_phys_addr_t addr, uint32_t value)
+                               hwaddr addr, uint32_t value)
 {
     ppc4xx_i2c_t *i2c;
 
@@ -1131,7 +1137,7 @@ static void ppc4xx_i2c_writeb (void *opaque,
     }
 }
 
-static uint32_t ppc4xx_i2c_readw (void *opaque, target_phys_addr_t addr)
+static uint32_t ppc4xx_i2c_readw (void *opaque, hwaddr addr)
 {
     uint32_t ret;
 
@@ -1145,7 +1151,7 @@ static uint32_t ppc4xx_i2c_readw (void *opaque, target_phys_addr_t addr)
 }
 
 static void ppc4xx_i2c_writew (void *opaque,
-                               target_phys_addr_t addr, uint32_t value)
+                               hwaddr addr, uint32_t value)
 {
 #ifdef DEBUG_I2C
     printf("%s: addr " TARGET_FMT_plx " val %08" PRIx32 "\n", __func__, addr,
@@ -1155,7 +1161,7 @@ static void ppc4xx_i2c_writew (void *opaque,
     ppc4xx_i2c_writeb(opaque, addr + 1, value);
 }
 
-static uint32_t ppc4xx_i2c_readl (void *opaque, target_phys_addr_t addr)
+static uint32_t ppc4xx_i2c_readl (void *opaque, hwaddr addr)
 {
     uint32_t ret;
 
@@ -1171,7 +1177,7 @@ static uint32_t ppc4xx_i2c_readl (void *opaque, target_phys_addr_t addr)
 }
 
 static void ppc4xx_i2c_writel (void *opaque,
-                               target_phys_addr_t addr, uint32_t value)
+                               hwaddr addr, uint32_t value)
 {
 #ifdef DEBUG_I2C
     printf("%s: addr " TARGET_FMT_plx " val %08" PRIx32 "\n", __func__, addr,
@@ -1207,7 +1213,7 @@ static void ppc4xx_i2c_reset (void *opaque)
     i2c->directcntl = 0x0F;
 }
 
-static void ppc405_i2c_init(target_phys_addr_t base, qemu_irq irq)
+static void ppc405_i2c_init(hwaddr base, qemu_irq irq)
 {
     ppc4xx_i2c_t *i2c;
 
@@ -1239,7 +1245,7 @@ struct ppc4xx_gpt_t {
     uint32_t mask[5];
 };
 
-static uint32_t ppc4xx_gpt_readb (void *opaque, target_phys_addr_t addr)
+static uint32_t ppc4xx_gpt_readb (void *opaque, hwaddr addr)
 {
 #ifdef DEBUG_GPT
     printf("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
@@ -1249,7 +1255,7 @@ static uint32_t ppc4xx_gpt_readb (void *opaque, target_phys_addr_t addr)
 }
 
 static void ppc4xx_gpt_writeb (void *opaque,
-                               target_phys_addr_t addr, uint32_t value)
+                               hwaddr addr, uint32_t value)
 {
 #ifdef DEBUG_I2C
     printf("%s: addr " TARGET_FMT_plx " val %08" PRIx32 "\n", __func__, addr,
@@ -1258,7 +1264,7 @@ static void ppc4xx_gpt_writeb (void *opaque,
     /* XXX: generate a bus fault */
 }
 
-static uint32_t ppc4xx_gpt_readw (void *opaque, target_phys_addr_t addr)
+static uint32_t ppc4xx_gpt_readw (void *opaque, hwaddr addr)
 {
 #ifdef DEBUG_GPT
     printf("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
@@ -1268,7 +1274,7 @@ static uint32_t ppc4xx_gpt_readw (void *opaque, target_phys_addr_t addr)
 }
 
 static void ppc4xx_gpt_writew (void *opaque,
-                               target_phys_addr_t addr, uint32_t value)
+                               hwaddr addr, uint32_t value)
 {
 #ifdef DEBUG_I2C
     printf("%s: addr " TARGET_FMT_plx " val %08" PRIx32 "\n", __func__, addr,
@@ -1329,7 +1335,7 @@ static void ppc4xx_gpt_compute_timer (ppc4xx_gpt_t *gpt)
     /* XXX: TODO */
 }
 
-static uint32_t ppc4xx_gpt_readl (void *opaque, target_phys_addr_t addr)
+static uint32_t ppc4xx_gpt_readl (void *opaque, hwaddr addr)
 {
     ppc4xx_gpt_t *gpt;
     uint32_t ret;
@@ -1385,7 +1391,7 @@ static uint32_t ppc4xx_gpt_readl (void *opaque, target_phys_addr_t addr)
 }
 
 static void ppc4xx_gpt_writel (void *opaque,
-                               target_phys_addr_t addr, uint32_t value)
+                               hwaddr addr, uint32_t value)
 {
     ppc4xx_gpt_t *gpt;
     int idx;
@@ -1482,7 +1488,7 @@ static void ppc4xx_gpt_reset (void *opaque)
     }
 }
 
-static void ppc4xx_gpt_init(target_phys_addr_t base, qemu_irq irqs[5])
+static void ppc4xx_gpt_init(hwaddr base, qemu_irq irqs[5])
 {
     ppc4xx_gpt_t *gpt;
     int i;
@@ -2098,8 +2104,8 @@ static void ppc405cr_cpc_init (CPUPPCState *env, clk_setup_t clk_setup[7],
 
 CPUPPCState *ppc405cr_init(MemoryRegion *address_space_mem,
                         MemoryRegion ram_memories[4],
-                        target_phys_addr_t ram_bases[4],
-                        target_phys_addr_t ram_sizes[4],
+                        hwaddr ram_bases[4],
+                        hwaddr ram_sizes[4],
                         uint32_t sysclk, qemu_irq **picp,
                         int do_init)
 {
@@ -2447,8 +2453,8 @@ static void ppc405ep_cpc_init (CPUPPCState *env, clk_setup_t clk_setup[8],
 
 CPUPPCState *ppc405ep_init(MemoryRegion *address_space_mem,
                         MemoryRegion ram_memories[2],
-                        target_phys_addr_t ram_bases[2],
-                        target_phys_addr_t ram_sizes[2],
+                        hwaddr ram_bases[2],
+                        hwaddr ram_sizes[2],
                         uint32_t sysclk, qemu_irq **picp,
                         int do_init)
 {
diff --git a/hw/ppc440_bamboo.c b/hw/ppc440_bamboo.c
index c19807117..cc85607cb 100644
--- a/hw/ppc440_bamboo.c
+++ b/hw/ppc440_bamboo.c
@@ -23,7 +23,7 @@
 #include "loader.h"
 #include "elf.h"
 #include "exec-memory.h"
-#include "pc.h"
+#include "serial.h"
 #include "ppc.h"
 #include "ppc405.h"
 #include "sysemu.h"
@@ -49,17 +49,17 @@ static const unsigned int ppc440ep_sdram_bank_sizes[] = {
     256<<20, 128<<20, 64<<20, 32<<20, 16<<20, 8<<20, 0
 };
 
-static target_phys_addr_t entry;
+static hwaddr entry;
 
-static int bamboo_load_device_tree(target_phys_addr_t addr,
+static int bamboo_load_device_tree(hwaddr addr,
                                      uint32_t ramsize,
-                                     target_phys_addr_t initrd_base,
-                                     target_phys_addr_t initrd_size,
+                                     hwaddr initrd_base,
+                                     hwaddr initrd_size,
                                      const char *kernel_cmdline)
 {
     int ret = -1;
 #ifdef CONFIG_FDT
-    uint32_t mem_reg_property[] = { 0, 0, ramsize };
+    uint32_t mem_reg_property[] = { 0, 0, cpu_to_be32(ramsize) };
     char *filename;
     int fdt_size;
     void *fdt;
@@ -123,7 +123,7 @@ out:
 /* Create reset TLB entries for BookE, spanning the 32bit addr space.  */
 static void mmubooke_create_initial_mapping(CPUPPCState *env,
                                      target_ulong va,
-                                     target_phys_addr_t pa)
+                                     hwaddr pa)
 {
     ppcemb_tlb_t *tlb = &env->tlb.tlbe[0];
 
@@ -157,19 +157,19 @@ static void main_cpu_reset(void *opaque)
     mmubooke_create_initial_mapping(env, 0, 0);
 }
 
-static void bamboo_init(ram_addr_t ram_size,
-                        const char *boot_device,
-                        const char *kernel_filename,
-                        const char *kernel_cmdline,
-                        const char *initrd_filename,
-                        const char *cpu_model)
+static void bamboo_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t ram_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
     unsigned int pci_irq_nrs[4] = { 28, 27, 26, 25 };
     MemoryRegion *address_space_mem = get_system_memory();
     MemoryRegion *ram_memories
         = g_malloc(PPC440EP_SDRAM_NR_BANKS * sizeof(*ram_memories));
-    target_phys_addr_t ram_bases[PPC440EP_SDRAM_NR_BANKS];
-    target_phys_addr_t ram_sizes[PPC440EP_SDRAM_NR_BANKS];
+    hwaddr ram_bases[PPC440EP_SDRAM_NR_BANKS];
+    hwaddr ram_sizes[PPC440EP_SDRAM_NR_BANKS];
     qemu_irq *pic;
     qemu_irq *irqs;
     PCIBus *pcibus;
@@ -177,7 +177,7 @@ static void bamboo_init(ram_addr_t ram_size,
     CPUPPCState *env;
     uint64_t elf_entry;
     uint64_t elf_lowaddr;
-    target_phys_addr_t loadaddr = 0;
+    hwaddr loadaddr = 0;
     target_long initrd_size = 0;
     DeviceState *dev;
     int success;
diff --git a/hw/ppc4xx.h b/hw/ppc4xx.h
index 5cd78b629..d795ced57 100644
--- a/hw/ppc4xx.h
+++ b/hw/ppc4xx.h
@@ -43,22 +43,22 @@ qemu_irq *ppcuic_init (CPUPPCState *env, qemu_irq *irqs,
 
 ram_addr_t ppc4xx_sdram_adjust(ram_addr_t ram_size, int nr_banks,
                                MemoryRegion ram_memories[],
-                               target_phys_addr_t ram_bases[],
-                               target_phys_addr_t ram_sizes[],
+                               hwaddr ram_bases[],
+                               hwaddr ram_sizes[],
                                const unsigned int sdram_bank_sizes[]);
 
 void ppc4xx_sdram_init (CPUPPCState *env, qemu_irq irq, int nbanks,
                         MemoryRegion ram_memories[],
-                        target_phys_addr_t *ram_bases,
-                        target_phys_addr_t *ram_sizes,
+                        hwaddr *ram_bases,
+                        hwaddr *ram_sizes,
                         int do_init);
 
 #define TYPE_PPC4xx_PCI_HOST_BRIDGE "ppc4xx-pcihost"
 
 PCIBus *ppc4xx_pci_init(CPUPPCState *env, qemu_irq pci_irqs[4],
-                        target_phys_addr_t config_space,
-                        target_phys_addr_t int_ack,
-                        target_phys_addr_t special_cycle,
-                        target_phys_addr_t registers);
+                        hwaddr config_space,
+                        hwaddr int_ack,
+                        hwaddr special_cycle,
+                        hwaddr registers);
 
 #endif /* !defined(PPC_4XX_H) */
diff --git a/hw/ppc4xx_devs.c b/hw/ppc4xx_devs.c
index 41163e607..bac8d8769 100644
--- a/hw/ppc4xx_devs.c
+++ b/hw/ppc4xx_devs.c
@@ -326,8 +326,8 @@ struct ppc4xx_sdram_t {
     int nbanks;
     MemoryRegion containers[4]; /* used for clipping */
     MemoryRegion *ram_memories;
-    target_phys_addr_t ram_bases[4];
-    target_phys_addr_t ram_sizes[4];
+    hwaddr ram_bases[4];
+    hwaddr ram_sizes[4];
     uint32_t besr0;
     uint32_t besr1;
     uint32_t bear;
@@ -348,11 +348,11 @@ enum {
 };
 
 /* XXX: TOFIX: some patches have made this code become inconsistent:
- *      there are type inconsistencies, mixing target_phys_addr_t, target_ulong
+ *      there are type inconsistencies, mixing hwaddr, target_ulong
  *      and uint32_t
  */
-static uint32_t sdram_bcr (target_phys_addr_t ram_base,
-                           target_phys_addr_t ram_size)
+static uint32_t sdram_bcr (hwaddr ram_base,
+                           hwaddr ram_size)
 {
     uint32_t bcr;
 
@@ -389,7 +389,7 @@ static uint32_t sdram_bcr (target_phys_addr_t ram_base,
     return bcr;
 }
 
-static inline target_phys_addr_t sdram_base(uint32_t bcr)
+static inline hwaddr sdram_base(uint32_t bcr)
 {
     return bcr & 0xFF800000;
 }
@@ -646,8 +646,8 @@ static void sdram_reset (void *opaque)
 
 void ppc4xx_sdram_init (CPUPPCState *env, qemu_irq irq, int nbanks,
                         MemoryRegion *ram_memories,
-                        target_phys_addr_t *ram_bases,
-                        target_phys_addr_t *ram_sizes,
+                        hwaddr *ram_bases,
+                        hwaddr *ram_sizes,
                         int do_init)
 {
     ppc4xx_sdram_t *sdram;
@@ -656,12 +656,12 @@ void ppc4xx_sdram_init (CPUPPCState *env, qemu_irq irq, int nbanks,
     sdram->irq = irq;
     sdram->nbanks = nbanks;
     sdram->ram_memories = ram_memories;
-    memset(sdram->ram_bases, 0, 4 * sizeof(target_phys_addr_t));
+    memset(sdram->ram_bases, 0, 4 * sizeof(hwaddr));
     memcpy(sdram->ram_bases, ram_bases,
-           nbanks * sizeof(target_phys_addr_t));
-    memset(sdram->ram_sizes, 0, 4 * sizeof(target_phys_addr_t));
+           nbanks * sizeof(hwaddr));
+    memset(sdram->ram_sizes, 0, 4 * sizeof(hwaddr));
     memcpy(sdram->ram_sizes, ram_sizes,
-           nbanks * sizeof(target_phys_addr_t));
+           nbanks * sizeof(hwaddr));
     qemu_register_reset(&sdram_reset, sdram);
     ppc_dcr_register(env, SDRAM0_CFGADDR,
                      sdram, &dcr_read_sdram, &dcr_write_sdram);
@@ -680,8 +680,8 @@ void ppc4xx_sdram_init (CPUPPCState *env, qemu_irq irq, int nbanks,
  * sizes varies by SoC. */
 ram_addr_t ppc4xx_sdram_adjust(ram_addr_t ram_size, int nr_banks,
                                MemoryRegion ram_memories[],
-                               target_phys_addr_t ram_bases[],
-                               target_phys_addr_t ram_sizes[],
+                               hwaddr ram_bases[],
+                               hwaddr ram_sizes[],
                                const unsigned int sdram_bank_sizes[])
 {
     ram_addr_t size_left = ram_size;
diff --git a/hw/ppc4xx_pci.c b/hw/ppc4xx_pci.c
index a14fd426c..d3ad6a0b7 100644
--- a/hw/ppc4xx_pci.c
+++ b/hw/ppc4xx_pci.c
@@ -92,7 +92,7 @@ typedef struct PPC4xxPCIState PPC4xxPCIState;
 
 #define PCI_ALL_SIZE        (PCI_REG_BASE + PCI_REG_SIZE)
 
-static uint64_t pci4xx_cfgaddr_read(void *opaque, target_phys_addr_t addr,
+static uint64_t pci4xx_cfgaddr_read(void *opaque, hwaddr addr,
                                     unsigned size)
 {
     PPC4xxPCIState *ppc4xx_pci = opaque;
@@ -101,7 +101,7 @@ static uint64_t pci4xx_cfgaddr_read(void *opaque, target_phys_addr_t addr,
     return phb->config_reg;
 }
 
-static void pci4xx_cfgaddr_write(void *opaque, target_phys_addr_t addr,
+static void pci4xx_cfgaddr_write(void *opaque, hwaddr addr,
                                   uint64_t value, unsigned size)
 {
     PPC4xxPCIState *ppc4xx_pci = opaque;
@@ -116,7 +116,7 @@ static const MemoryRegionOps pci4xx_cfgaddr_ops = {
     .endianness = DEVICE_LITTLE_ENDIAN,
 };
 
-static void ppc4xx_pci_reg_write4(void *opaque, target_phys_addr_t offset,
+static void ppc4xx_pci_reg_write4(void *opaque, hwaddr offset,
                                   uint64_t value, unsigned size)
 {
     struct PPC4xxPCIState *pci = opaque;
@@ -184,7 +184,7 @@ static void ppc4xx_pci_reg_write4(void *opaque, target_phys_addr_t offset,
     }
 }
 
-static uint64_t ppc4xx_pci_reg_read4(void *opaque, target_phys_addr_t offset,
+static uint64_t ppc4xx_pci_reg_read4(void *opaque, hwaddr offset,
                                      unsigned size)
 {
     struct PPC4xxPCIState *pci = opaque;
diff --git a/hw/ppc_mac.h b/hw/ppc_mac.h
index 7d084184f..524b2368a 100644
--- a/hw/ppc_mac.h
+++ b/hw/ppc_mac.h
@@ -71,10 +71,10 @@ PCIBus *pci_pmac_u3_init(qemu_irq *pic,
 /* Mac NVRAM */
 typedef struct MacIONVRAMState MacIONVRAMState;
 
-MacIONVRAMState *macio_nvram_init (target_phys_addr_t size,
+MacIONVRAMState *macio_nvram_init (hwaddr size,
                                    unsigned int it_shift);
 void macio_nvram_setup_bar(MacIONVRAMState *s, MemoryRegion *bar,
-                           target_phys_addr_t mem_base);
+                           hwaddr mem_base);
 void pmac_format_nvram_partition (MacIONVRAMState *nvr, int len);
 uint32_t macio_nvram_read (void *opaque, uint32_t addr);
 void macio_nvram_write (void *opaque, uint32_t addr, uint32_t val);
diff --git a/hw/ppc_newworld.c b/hw/ppc_newworld.c
index e95cfe831..664747ead 100644
--- a/hw/ppc_newworld.c
+++ b/hw/ppc_newworld.c
@@ -67,7 +67,6 @@
 #include "hw/usb.h"
 #include "blockdev.h"
 #include "exec-memory.h"
-#include "vga-pci.h"
 
 #define MAX_IDE_BUS 2
 #define CFG_ADDR 0xf0000510
@@ -83,13 +82,13 @@
 #endif
 
 /* UniN device */
-static void unin_write(void *opaque, target_phys_addr_t addr, uint64_t value,
+static void unin_write(void *opaque, hwaddr addr, uint64_t value,
                        unsigned size)
 {
     UNIN_DPRINTF("write addr " TARGET_FMT_plx " val %"PRIx64"\n", addr, value);
 }
 
-static uint64_t unin_read(void *opaque, target_phys_addr_t addr, unsigned size)
+static uint64_t unin_read(void *opaque, hwaddr addr, unsigned size)
 {
     uint32_t value;
 
@@ -116,7 +115,7 @@ static uint64_t translate_kernel_address(void *opaque, uint64_t addr)
     return (addr & 0x0fffffff) + KERNEL_LOAD_ADDR;
 }
 
-static target_phys_addr_t round_page(target_phys_addr_t addr)
+static hwaddr round_page(hwaddr addr)
 {
     return (addr + TARGET_PAGE_SIZE - 1) & TARGET_PAGE_MASK;
 }
@@ -129,13 +128,14 @@ static void ppc_core99_reset(void *opaque)
 }
 
 /* PowerPC Mac99 hardware initialisation */
-static void ppc_core99_init (ram_addr_t ram_size,
-                             const char *boot_device,
-                             const char *kernel_filename,
-                             const char *kernel_cmdline,
-                             const char *initrd_filename,
-                             const char *cpu_model)
+static void ppc_core99_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t ram_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
+    const char *boot_device = args->boot_device;
     PowerPCCPU *cpu = NULL;
     CPUPPCState *env = NULL;
     char *filename;
@@ -143,7 +143,7 @@ static void ppc_core99_init (ram_addr_t ram_size,
     MemoryRegion *unin_memory = g_new(MemoryRegion, 1);
     int linux_boot, i;
     MemoryRegion *ram = g_new(MemoryRegion, 1), *bios = g_new(MemoryRegion, 1);
-    target_phys_addr_t kernel_base, initrd_base, cmdline_base = 0;
+    hwaddr kernel_base, initrd_base, cmdline_base = 0;
     long kernel_size, initrd_size;
     PCIBus *pci_bus;
     MacIONVRAMState *nvr;
@@ -348,10 +348,6 @@ static void ppc_core99_init (ram_addr_t ram_size,
     ide_mem[1] = pmac_ide_init(hd, pic[0x0d], dbdma, 0x16, pic[0x02]);
     ide_mem[2] = pmac_ide_init(&hd[MAX_IDE_DEVS], pic[0x0e], dbdma, 0x1a, pic[0x02]);
 
-    /* cuda also initialize ADB */
-    if (machine_arch == ARCH_MAC99_U3) {
-        usb_enabled = 1;
-    }
     cuda_init(&cuda_mem, pic[0x19]);
 
     adb_kbd_init(&adb_bus);
@@ -360,15 +356,14 @@ static void ppc_core99_init (ram_addr_t ram_size,
     macio_init(pci_bus, PCI_DEVICE_ID_APPLE_UNI_N_KEYL, 0, pic_mem,
                dbdma_mem, cuda_mem, NULL, 3, ide_mem, escc_bar);
 
-    if (usb_enabled) {
+    if (usb_enabled(machine_arch == ARCH_MAC99_U3)) {
         pci_create_simple(pci_bus, -1, "pci-ohci");
-    }
-
-    /* U3 needs to use USB for input because Linux doesn't support via-cuda
-       on PPC64 */
-    if (machine_arch == ARCH_MAC99_U3) {
-        usbdevice_create("keyboard");
-        usbdevice_create("mouse");
+        /* U3 needs to use USB for input because Linux doesn't support via-cuda
+        on PPC64 */
+        if (machine_arch == ARCH_MAC99_U3) {
+            usbdevice_create("keyboard");
+            usbdevice_create("mouse");
+        }
     }
 
     if (graphic_depth != 15 && graphic_depth != 32 && graphic_depth != 8)
diff --git a/hw/ppc_oldworld.c b/hw/ppc_oldworld.c
index 1dcd8a6c3..e8138c091 100644
--- a/hw/ppc_oldworld.c
+++ b/hw/ppc_oldworld.c
@@ -43,7 +43,6 @@
 #include "kvm_ppc.h"
 #include "blockdev.h"
 #include "exec-memory.h"
-#include "vga-pci.h"
 
 #define MAX_IDE_BUS 2
 #define CFG_ADDR 0xf0000510
@@ -60,7 +59,7 @@ static uint64_t translate_kernel_address(void *opaque, uint64_t addr)
     return (addr & 0x0fffffff) + KERNEL_LOAD_ADDR;
 }
 
-static target_phys_addr_t round_page(target_phys_addr_t addr)
+static hwaddr round_page(hwaddr addr)
 {
     return (addr + TARGET_PAGE_SIZE - 1) & TARGET_PAGE_MASK;
 }
@@ -72,13 +71,14 @@ static void ppc_heathrow_reset(void *opaque)
     cpu_reset(CPU(cpu));
 }
 
-static void ppc_heathrow_init (ram_addr_t ram_size,
-                               const char *boot_device,
-                               const char *kernel_filename,
-                               const char *kernel_cmdline,
-                               const char *initrd_filename,
-                               const char *cpu_model)
+static void ppc_heathrow_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t ram_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
+    const char *boot_device = args->boot_device;
     MemoryRegion *sysmem = get_system_memory();
     PowerPCCPU *cpu = NULL;
     CPUPPCState *env = NULL;
@@ -286,7 +286,7 @@ static void ppc_heathrow_init (ram_addr_t ram_size,
     macio_init(pci_bus, PCI_DEVICE_ID_APPLE_343S1201, 1, pic_mem,
                dbdma_mem, cuda_mem, nvr, 2, ide_mem, escc_bar);
 
-    if (usb_enabled) {
+    if (usb_enabled(false)) {
         pci_create_simple(pci_bus, -1, "pci-ohci");
     }
 
diff --git a/hw/ppc_prep.c b/hw/ppc_prep.c
index 592b7b255..bf15730d8 100644
--- a/hw/ppc_prep.c
+++ b/hw/ppc_prep.c
@@ -24,6 +24,7 @@
 #include "hw.h"
 #include "nvram.h"
 #include "pc.h"
+#include "serial.h"
 #include "fdc.h"
 #include "net.h"
 #include "sysemu.h"
@@ -39,7 +40,6 @@
 #include "blockdev.h"
 #include "arch_init.h"
 #include "exec-memory.h"
-#include "vga-pci.h"
 
 //#define HARD_DEBUG_PPC_IO
 //#define DEBUG_PPC_IO
@@ -115,27 +115,27 @@ static struct {
 } XCSR;
 
 static void PPC_XCSR_writeb (void *opaque,
-                             target_phys_addr_t addr, uint32_t value)
+                             hwaddr addr, uint32_t value)
 {
     printf("%s: 0x" TARGET_FMT_plx " => 0x%08" PRIx32 "\n", __func__, addr,
            value);
 }
 
 static void PPC_XCSR_writew (void *opaque,
-                             target_phys_addr_t addr, uint32_t value)
+                             hwaddr addr, uint32_t value)
 {
     printf("%s: 0x" TARGET_FMT_plx " => 0x%08" PRIx32 "\n", __func__, addr,
            value);
 }
 
 static void PPC_XCSR_writel (void *opaque,
-                             target_phys_addr_t addr, uint32_t value)
+                             hwaddr addr, uint32_t value)
 {
     printf("%s: 0x" TARGET_FMT_plx " => 0x%08" PRIx32 "\n", __func__, addr,
            value);
 }
 
-static uint32_t PPC_XCSR_readb (void *opaque, target_phys_addr_t addr)
+static uint32_t PPC_XCSR_readb (void *opaque, hwaddr addr)
 {
     uint32_t retval = 0;
 
@@ -145,7 +145,7 @@ static uint32_t PPC_XCSR_readb (void *opaque, target_phys_addr_t addr)
     return retval;
 }
 
-static uint32_t PPC_XCSR_readw (void *opaque, target_phys_addr_t addr)
+static uint32_t PPC_XCSR_readw (void *opaque, hwaddr addr)
 {
     uint32_t retval = 0;
 
@@ -155,7 +155,7 @@ static uint32_t PPC_XCSR_readw (void *opaque, target_phys_addr_t addr)
     return retval;
 }
 
-static uint32_t PPC_XCSR_readl (void *opaque, target_phys_addr_t addr)
+static uint32_t PPC_XCSR_readl (void *opaque, hwaddr addr)
 {
     uint32_t retval = 0;
 
@@ -343,8 +343,8 @@ static uint32_t PREP_io_800_readb (void *opaque, uint32_t addr)
     return retval;
 }
 
-static inline target_phys_addr_t prep_IO_address(sysctrl_t *sysctrl,
-                                                 target_phys_addr_t addr)
+static inline hwaddr prep_IO_address(sysctrl_t *sysctrl,
+                                                 hwaddr addr)
 {
     if (sysctrl->contiguous_map == 0) {
         /* 64 KB contiguous space for IOs */
@@ -357,7 +357,7 @@ static inline target_phys_addr_t prep_IO_address(sysctrl_t *sysctrl,
     return addr;
 }
 
-static void PPC_prep_io_writeb (void *opaque, target_phys_addr_t addr,
+static void PPC_prep_io_writeb (void *opaque, hwaddr addr,
                                 uint32_t value)
 {
     sysctrl_t *sysctrl = opaque;
@@ -366,7 +366,7 @@ static void PPC_prep_io_writeb (void *opaque, target_phys_addr_t addr,
     cpu_outb(addr, value);
 }
 
-static uint32_t PPC_prep_io_readb (void *opaque, target_phys_addr_t addr)
+static uint32_t PPC_prep_io_readb (void *opaque, hwaddr addr)
 {
     sysctrl_t *sysctrl = opaque;
     uint32_t ret;
@@ -377,7 +377,7 @@ static uint32_t PPC_prep_io_readb (void *opaque, target_phys_addr_t addr)
     return ret;
 }
 
-static void PPC_prep_io_writew (void *opaque, target_phys_addr_t addr,
+static void PPC_prep_io_writew (void *opaque, hwaddr addr,
                                 uint32_t value)
 {
     sysctrl_t *sysctrl = opaque;
@@ -387,7 +387,7 @@ static void PPC_prep_io_writew (void *opaque, target_phys_addr_t addr,
     cpu_outw(addr, value);
 }
 
-static uint32_t PPC_prep_io_readw (void *opaque, target_phys_addr_t addr)
+static uint32_t PPC_prep_io_readw (void *opaque, hwaddr addr)
 {
     sysctrl_t *sysctrl = opaque;
     uint32_t ret;
@@ -399,7 +399,7 @@ static uint32_t PPC_prep_io_readw (void *opaque, target_phys_addr_t addr)
     return ret;
 }
 
-static void PPC_prep_io_writel (void *opaque, target_phys_addr_t addr,
+static void PPC_prep_io_writel (void *opaque, hwaddr addr,
                                 uint32_t value)
 {
     sysctrl_t *sysctrl = opaque;
@@ -409,7 +409,7 @@ static void PPC_prep_io_writel (void *opaque, target_phys_addr_t addr,
     cpu_outl(addr, value);
 }
 
-static uint32_t PPC_prep_io_readl (void *opaque, target_phys_addr_t addr)
+static uint32_t PPC_prep_io_readl (void *opaque, hwaddr addr)
 {
     sysctrl_t *sysctrl = opaque;
     uint32_t ret;
@@ -448,13 +448,14 @@ static void ppc_prep_reset(void *opaque)
 }
 
 /* PowerPC PREP hardware initialisation */
-static void ppc_prep_init (ram_addr_t ram_size,
-                           const char *boot_device,
-                           const char *kernel_filename,
-                           const char *kernel_cmdline,
-                           const char *initrd_filename,
-                           const char *cpu_model)
+static void ppc_prep_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t ram_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
+    const char *boot_device = args->boot_device;
     MemoryRegion *sysmem = get_system_memory();
     PowerPCCPU *cpu = NULL;
     CPUPPCState *env = NULL;
@@ -524,7 +525,7 @@ static void ppc_prep_init (ram_addr_t ram_size,
         bios_size = -1;
     }
     if (bios_size > 0 && bios_size <= BIOS_SIZE) {
-        target_phys_addr_t bios_addr;
+        hwaddr bios_addr;
         bios_size = (bios_size + 0xfff) & ~0xfff;
         bios_addr = (uint32_t)(-bios_size);
         bios_size = load_image_targphys(filename, bios_addr, bios_size);
@@ -660,7 +661,7 @@ static void ppc_prep_init (ram_addr_t ram_size,
     memory_region_add_subregion(sysmem, 0xFEFF0000, xcsr);
 #endif
 
-    if (usb_enabled) {
+    if (usb_enabled(false)) {
         pci_create_simple(pci_bus, -1, "pci-ohci");
     }
 
diff --git a/hw/ppce500_pci.c b/hw/ppce500_pci.c
index 92b1dc053..2ff7438d0 100644
--- a/hw/ppce500_pci.c
+++ b/hw/ppce500_pci.c
@@ -31,6 +31,8 @@
 #define PCIE500_ALL_SIZE      0x1000
 #define PCIE500_REG_SIZE      (PCIE500_ALL_SIZE - PCIE500_REG_BASE)
 
+#define PCIE500_PCI_IOLEN     0x10000ULL
+
 #define PPCE500_PCI_CONFIG_ADDR         0x0
 #define PPCE500_PCI_CONFIG_DATA         0x4
 #define PPCE500_PCI_INTACK              0x8
@@ -87,11 +89,12 @@ struct PPCE500PCIState {
     /* mmio maps */
     MemoryRegion container;
     MemoryRegion iomem;
+    MemoryRegion pio;
 };
 
 typedef struct PPCE500PCIState PPCE500PCIState;
 
-static uint64_t pci_reg_read4(void *opaque, target_phys_addr_t addr,
+static uint64_t pci_reg_read4(void *opaque, hwaddr addr,
                               unsigned size)
 {
     PPCE500PCIState *pci = opaque;
@@ -160,7 +163,7 @@ static uint64_t pci_reg_read4(void *opaque, target_phys_addr_t addr,
     return value;
 }
 
-static void pci_reg_write4(void *opaque, target_phys_addr_t addr,
+static void pci_reg_write4(void *opaque, hwaddr addr,
                            uint64_t value, unsigned size)
 {
     PPCE500PCIState *pci = opaque;
@@ -314,7 +317,6 @@ static int e500_pcihost_initfn(SysBusDevice *dev)
     PCIBus *b;
     int i;
     MemoryRegion *address_space_mem = get_system_memory();
-    MemoryRegion *address_space_io = get_system_io();
 
     h = PCI_HOST_BRIDGE(dev);
     s = PPC_E500_PCI_HOST_BRIDGE(dev);
@@ -323,9 +325,11 @@ static int e500_pcihost_initfn(SysBusDevice *dev)
         sysbus_init_irq(dev, &s->irq[i]);
     }
 
+    memory_region_init(&s->pio, "pci-pio", PCIE500_PCI_IOLEN);
+
     b = pci_register_bus(DEVICE(dev), NULL, mpc85xx_pci_set_irq,
                          mpc85xx_pci_map_irq, s->irq, address_space_mem,
-                         address_space_io, PCI_DEVFN(0x11, 0), 4);
+                         &s->pio, PCI_DEVFN(0x11, 0), 4);
     h->bus = b;
 
     pci_create_simple(b, 0, "e500-host-bridge");
@@ -341,6 +345,7 @@ static int e500_pcihost_initfn(SysBusDevice *dev)
     memory_region_add_subregion(&s->container, PCIE500_CFGDATA, &h->data_mem);
     memory_region_add_subregion(&s->container, PCIE500_REG_BASE, &s->iomem);
     sysbus_init_mmio(dev, &s->container);
+    sysbus_init_mmio(dev, &s->pio);
 
     return 0;
 }
diff --git a/hw/ppce500_spin.c b/hw/ppce500_spin.c
index c5b8e051e..c1a155bd3 100644
--- a/hw/ppce500_spin.c
+++ b/hw/ppce500_spin.c
@@ -49,7 +49,7 @@ typedef struct spin_state {
 } SpinState;
 
 typedef struct spin_kick {
-    CPUPPCState *env;
+    PowerPCCPU *cpu;
     SpinInfo *spin;
 } SpinKick;
 
@@ -68,18 +68,18 @@ static void spin_reset(void *opaque)
 }
 
 /* Create -kernel TLB entries for BookE, linearly spanning 256MB.  */
-static inline target_phys_addr_t booke206_page_size_to_tlb(uint64_t size)
+static inline hwaddr booke206_page_size_to_tlb(uint64_t size)
 {
     return (ffs(size >> 10) - 1) >> 1;
 }
 
 static void mmubooke_create_initial_mapping(CPUPPCState *env,
                                      target_ulong va,
-                                     target_phys_addr_t pa,
-                                     target_phys_addr_t len)
+                                     hwaddr pa,
+                                     hwaddr len)
 {
     ppcmas_tlb_t *tlb = booke206_get_tlbm(env, 1, 0, 1);
-    target_phys_addr_t size;
+    hwaddr size;
 
     size = (booke206_page_size_to_tlb(len) << MAS1_TSIZE_SHIFT);
     tlb->mas1 = MAS1_VALID | size;
@@ -92,10 +92,11 @@ static void mmubooke_create_initial_mapping(CPUPPCState *env,
 static void spin_kick(void *data)
 {
     SpinKick *kick = data;
-    CPUPPCState *env = kick->env;
+    CPUState *cpu = CPU(kick->cpu);
+    CPUPPCState *env = &kick->cpu->env;
     SpinInfo *curspin = kick->spin;
-    target_phys_addr_t map_size = 64 * 1024 * 1024;
-    target_phys_addr_t map_start;
+    hwaddr map_size = 64 * 1024 * 1024;
+    hwaddr map_start;
 
     cpu_synchronize_state(env);
     stl_p(&curspin->pir, env->spr[SPR_PIR]);
@@ -113,11 +114,11 @@ static void spin_kick(void *data)
 
     env->halted = 0;
     env->exception_index = -1;
-    env->stopped = 0;
-    qemu_cpu_kick(env);
+    cpu->stopped = false;
+    qemu_cpu_kick(cpu);
 }
 
-static void spin_write(void *opaque, target_phys_addr_t addr, uint64_t value,
+static void spin_write(void *opaque, hwaddr addr, uint64_t value,
                        unsigned len)
 {
     SpinState *s = opaque;
@@ -158,15 +159,15 @@ static void spin_write(void *opaque, target_phys_addr_t addr, uint64_t value,
     if (!(ldq_p(&curspin->addr) & 1)) {
         /* run CPU */
         SpinKick kick = {
-            .env = env,
+            .cpu = ppc_env_get_cpu(env),
             .spin = curspin,
         };
 
-        run_on_cpu(env, spin_kick, &kick);
+        run_on_cpu(CPU(kick.cpu), spin_kick, &kick);
     }
 }
 
-static uint64_t spin_read(void *opaque, target_phys_addr_t addr, unsigned len)
+static uint64_t spin_read(void *opaque, hwaddr addr, unsigned len)
 {
     SpinState *s = opaque;
     uint8_t *spin_p = &((uint8_t*)s->spin)[addr];
diff --git a/hw/prep_pci.c b/hw/prep_pci.c
index cc44e6198..0bc479cd1 100644
--- a/hw/prep_pci.c
+++ b/hw/prep_pci.c
@@ -44,7 +44,7 @@ typedef struct RavenPCIState {
     PCIDevice dev;
 } RavenPCIState;
 
-static inline uint32_t PPC_PCIIO_config(target_phys_addr_t addr)
+static inline uint32_t PPC_PCIIO_config(hwaddr addr)
 {
     int i;
 
@@ -56,7 +56,7 @@ static inline uint32_t PPC_PCIIO_config(target_phys_addr_t addr)
     return (addr & 0x7ff) |  (i << 11);
 }
 
-static void ppc_pci_io_write(void *opaque, target_phys_addr_t addr,
+static void ppc_pci_io_write(void *opaque, hwaddr addr,
                              uint64_t val, unsigned int size)
 {
     PREPPCIState *s = opaque;
@@ -64,7 +64,7 @@ static void ppc_pci_io_write(void *opaque, target_phys_addr_t addr,
     pci_data_write(phb->bus, PPC_PCIIO_config(addr), val, size);
 }
 
-static uint64_t ppc_pci_io_read(void *opaque, target_phys_addr_t addr,
+static uint64_t ppc_pci_io_read(void *opaque, hwaddr addr,
                                 unsigned int size)
 {
     PREPPCIState *s = opaque;
@@ -78,7 +78,7 @@ static const MemoryRegionOps PPC_PCIIO_ops = {
     .endianness = DEVICE_LITTLE_ENDIAN,
 };
 
-static uint64_t ppc_intack_read(void *opaque, target_phys_addr_t addr,
+static uint64_t ppc_intack_read(void *opaque, hwaddr addr,
                                 unsigned int size)
 {
     return pic_read_irq(isa_pic);
diff --git a/hw/puv3.c b/hw/puv3.c
index 43f7216e4..764799cff 100644
--- a/hw/puv3.c
+++ b/hw/puv3.c
@@ -91,10 +91,12 @@ static void puv3_load_kernel(const char *kernel_filename)
     graphic_console_init(NULL, NULL, NULL, NULL, NULL);
 }
 
-static void puv3_init(ram_addr_t ram_size, const char *boot_device,
-                     const char *kernel_filename, const char *kernel_cmdline,
-                     const char *initrd_filename, const char *cpu_model)
+static void puv3_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t ram_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *initrd_filename = args->initrd_filename;
     CPUUniCore32State *env;
 
     if (initrd_filename) {
diff --git a/hw/puv3_dma.c b/hw/puv3_dma.c
index 85b97bfde..9de63b4c3 100644
--- a/hw/puv3_dma.c
+++ b/hw/puv3_dma.c
@@ -24,7 +24,7 @@ typedef struct {
     uint32_t reg_CFG[PUV3_DMA_CH_NR];
 } PUV3DMAState;
 
-static uint64_t puv3_dma_read(void *opaque, target_phys_addr_t offset,
+static uint64_t puv3_dma_read(void *opaque, hwaddr offset,
         unsigned size)
 {
     PUV3DMAState *s = opaque;
@@ -44,7 +44,7 @@ static uint64_t puv3_dma_read(void *opaque, target_phys_addr_t offset,
     return ret;
 }
 
-static void puv3_dma_write(void *opaque, target_phys_addr_t offset,
+static void puv3_dma_write(void *opaque, hwaddr offset,
         uint64_t value, unsigned size)
 {
     PUV3DMAState *s = opaque;
diff --git a/hw/puv3_gpio.c b/hw/puv3_gpio.c
index 9436e6c62..152248d29 100644
--- a/hw/puv3_gpio.c
+++ b/hw/puv3_gpio.c
@@ -24,7 +24,7 @@ typedef struct {
     uint32_t reg_GPIR;
 } PUV3GPIOState;
 
-static uint64_t puv3_gpio_read(void *opaque, target_phys_addr_t offset,
+static uint64_t puv3_gpio_read(void *opaque, hwaddr offset,
         unsigned size)
 {
     PUV3GPIOState *s = opaque;
@@ -48,7 +48,7 @@ static uint64_t puv3_gpio_read(void *opaque, target_phys_addr_t offset,
     return ret;
 }
 
-static void puv3_gpio_write(void *opaque, target_phys_addr_t offset,
+static void puv3_gpio_write(void *opaque, hwaddr offset,
         uint64_t value, unsigned size)
 {
     PUV3GPIOState *s = opaque;
diff --git a/hw/puv3_intc.c b/hw/puv3_intc.c
index 9e0b975ea..07f564906 100644
--- a/hw/puv3_intc.c
+++ b/hw/puv3_intc.c
@@ -46,7 +46,7 @@ static void puv3_intc_handler(void *opaque, int irq, int level)
     puv3_intc_update(s);
 }
 
-static uint64_t puv3_intc_read(void *opaque, target_phys_addr_t offset,
+static uint64_t puv3_intc_read(void *opaque, hwaddr offset,
         unsigned size)
 {
     PUV3INTCState *s = opaque;
@@ -66,7 +66,7 @@ static uint64_t puv3_intc_read(void *opaque, target_phys_addr_t offset,
     return ret;
 }
 
-static void puv3_intc_write(void *opaque, target_phys_addr_t offset,
+static void puv3_intc_write(void *opaque, hwaddr offset,
         uint64_t value, unsigned size)
 {
     PUV3INTCState *s = opaque;
diff --git a/hw/puv3_ost.c b/hw/puv3_ost.c
index dd30cad0e..14c6f21a7 100644
--- a/hw/puv3_ost.c
+++ b/hw/puv3_ost.c
@@ -28,7 +28,7 @@ typedef struct {
     uint32_t reg_OIER;
 } PUV3OSTState;
 
-static uint64_t puv3_ost_read(void *opaque, target_phys_addr_t offset,
+static uint64_t puv3_ost_read(void *opaque, hwaddr offset,
         unsigned size)
 {
     PUV3OSTState *s = opaque;
@@ -51,7 +51,7 @@ static uint64_t puv3_ost_read(void *opaque, target_phys_addr_t offset,
     return ret;
 }
 
-static void puv3_ost_write(void *opaque, target_phys_addr_t offset,
+static void puv3_ost_write(void *opaque, hwaddr offset,
         uint64_t value, unsigned size)
 {
     PUV3OSTState *s = opaque;
diff --git a/hw/puv3_pm.c b/hw/puv3_pm.c
index 621c96875..87a687afa 100644
--- a/hw/puv3_pm.c
+++ b/hw/puv3_pm.c
@@ -26,7 +26,7 @@ typedef struct {
     uint32_t reg_DIVCFG;
 } PUV3PMState;
 
-static uint64_t puv3_pm_read(void *opaque, target_phys_addr_t offset,
+static uint64_t puv3_pm_read(void *opaque, hwaddr offset,
         unsigned size)
 {
     PUV3PMState *s = opaque;
@@ -74,7 +74,7 @@ static uint64_t puv3_pm_read(void *opaque, target_phys_addr_t offset,
     return ret;
 }
 
-static void puv3_pm_write(void *opaque, target_phys_addr_t offset,
+static void puv3_pm_write(void *opaque, hwaddr offset,
         uint64_t value, unsigned size)
 {
     PUV3PMState *s = opaque;
diff --git a/hw/pxa.h b/hw/pxa.h
index 6a2120555..49ac820ea 100644
--- a/hw/pxa.h
+++ b/hw/pxa.h
@@ -65,28 +65,28 @@
 # define PXA2XX_INTERNAL_SIZE	0x40000
 
 /* pxa2xx_pic.c */
-DeviceState *pxa2xx_pic_init(target_phys_addr_t base, ARMCPU *cpu);
+DeviceState *pxa2xx_pic_init(hwaddr base, ARMCPU *cpu);
 
 /* pxa2xx_gpio.c */
-DeviceState *pxa2xx_gpio_init(target_phys_addr_t base,
+DeviceState *pxa2xx_gpio_init(hwaddr base,
                 CPUARMState *env, DeviceState *pic, int lines);
 void pxa2xx_gpio_read_notifier(DeviceState *dev, qemu_irq handler);
 
 /* pxa2xx_dma.c */
-DeviceState *pxa255_dma_init(target_phys_addr_t base, qemu_irq irq);
-DeviceState *pxa27x_dma_init(target_phys_addr_t base, qemu_irq irq);
+DeviceState *pxa255_dma_init(hwaddr base, qemu_irq irq);
+DeviceState *pxa27x_dma_init(hwaddr base, qemu_irq irq);
 
 /* pxa2xx_lcd.c */
 typedef struct PXA2xxLCDState PXA2xxLCDState;
 PXA2xxLCDState *pxa2xx_lcdc_init(MemoryRegion *sysmem,
-                target_phys_addr_t base, qemu_irq irq);
+                hwaddr base, qemu_irq irq);
 void pxa2xx_lcd_vsync_notifier(PXA2xxLCDState *s, qemu_irq handler);
 void pxa2xx_lcdc_oritentation(void *opaque, int angle);
 
 /* pxa2xx_mmci.c */
 typedef struct PXA2xxMMCIState PXA2xxMMCIState;
 PXA2xxMMCIState *pxa2xx_mmci_init(MemoryRegion *sysmem,
-                target_phys_addr_t base,
+                hwaddr base,
                 BlockDriverState *bd, qemu_irq irq,
                 qemu_irq rx_dma, qemu_irq tx_dma);
 void pxa2xx_mmci_handlers(PXA2xxMMCIState *s, qemu_irq readonly,
@@ -95,7 +95,7 @@ void pxa2xx_mmci_handlers(PXA2xxMMCIState *s, qemu_irq readonly,
 /* pxa2xx_pcmcia.c */
 typedef struct PXA2xxPCMCIAState PXA2xxPCMCIAState;
 PXA2xxPCMCIAState *pxa2xx_pcmcia_init(MemoryRegion *sysmem,
-                                      target_phys_addr_t base);
+                                      hwaddr base);
 int pxa2xx_pcmcia_attach(void *opaque, PCMCIACardState *card);
 int pxa2xx_pcmcia_dettach(void *opaque);
 void pxa2xx_pcmcia_set_irq_cb(void *opaque, qemu_irq irq, qemu_irq cd_irq);
@@ -107,14 +107,14 @@ struct  keymap {
 };
 typedef struct PXA2xxKeyPadState PXA2xxKeyPadState;
 PXA2xxKeyPadState *pxa27x_keypad_init(MemoryRegion *sysmem,
-                                      target_phys_addr_t base,
+                                      hwaddr base,
                                       qemu_irq irq);
 void pxa27x_register_keypad(PXA2xxKeyPadState *kp, struct keymap *map,
                 int size);
 
 /* pxa2xx.c */
 typedef struct PXA2xxI2CState PXA2xxI2CState;
-PXA2xxI2CState *pxa2xx_i2c_init(target_phys_addr_t base,
+PXA2xxI2CState *pxa2xx_i2c_init(hwaddr base,
                 qemu_irq irq, uint32_t page_size);
 i2c_bus *pxa2xx_i2c_bus(PXA2xxI2CState *s);
 
@@ -142,16 +142,16 @@ typedef struct {
     PXA2xxKeyPadState *kp;
 
     /* Power management */
-    target_phys_addr_t pm_base;
+    hwaddr pm_base;
     uint32_t pm_regs[0x40];
 
     /* Clock management */
-    target_phys_addr_t cm_base;
+    hwaddr cm_base;
     uint32_t cm_regs[4];
     uint32_t clkcfg;
 
     /* Memory management */
-    target_phys_addr_t mm_base;
+    hwaddr mm_base;
     uint32_t mm_regs[0x1a];
 
     /* Performance monitoring */
diff --git a/hw/pxa2xx.c b/hw/pxa2xx.c
index d5f1420ed..e61697920 100644
--- a/hw/pxa2xx.c
+++ b/hw/pxa2xx.c
@@ -10,14 +10,14 @@
 #include "sysbus.h"
 #include "pxa.h"
 #include "sysemu.h"
-#include "pc.h"
+#include "serial.h"
 #include "i2c.h"
 #include "ssi.h"
 #include "qemu-char.h"
 #include "blockdev.h"
 
 static struct {
-    target_phys_addr_t io_base;
+    hwaddr io_base;
     int irqn;
 } pxa255_serial[] = {
     { 0x40100000, PXA2XX_PIC_FFUART },
@@ -33,7 +33,7 @@ static struct {
 };
 
 typedef struct PXASSPDef {
-    target_phys_addr_t io_base;
+    hwaddr io_base;
     int irqn;
 } PXASSPDef;
 
@@ -88,7 +88,7 @@ static PXASSPDef pxa27x_ssp[] = {
 #define PCMD0	0x80	/* Power Manager I2C Command register File 0 */
 #define PCMD31	0xfc	/* Power Manager I2C Command register File 31 */
 
-static uint64_t pxa2xx_pm_read(void *opaque, target_phys_addr_t addr,
+static uint64_t pxa2xx_pm_read(void *opaque, hwaddr addr,
                                unsigned size)
 {
     PXA2xxState *s = (PXA2xxState *) opaque;
@@ -107,7 +107,7 @@ static uint64_t pxa2xx_pm_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void pxa2xx_pm_write(void *opaque, target_phys_addr_t addr,
+static void pxa2xx_pm_write(void *opaque, hwaddr addr,
                             uint64_t value, unsigned size)
 {
     PXA2xxState *s = (PXA2xxState *) opaque;
@@ -160,7 +160,7 @@ static const VMStateDescription vmstate_pxa2xx_pm = {
 #define OSCC	0x08	/* Oscillator Configuration register */
 #define CCSR	0x0c	/* Core Clock Status register */
 
-static uint64_t pxa2xx_cm_read(void *opaque, target_phys_addr_t addr,
+static uint64_t pxa2xx_cm_read(void *opaque, hwaddr addr,
                                unsigned size)
 {
     PXA2xxState *s = (PXA2xxState *) opaque;
@@ -181,7 +181,7 @@ static uint64_t pxa2xx_cm_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void pxa2xx_cm_write(void *opaque, target_phys_addr_t addr,
+static void pxa2xx_cm_write(void *opaque, hwaddr addr,
                             uint64_t value, unsigned size)
 {
     PXA2xxState *s = (PXA2xxState *) opaque;
@@ -405,7 +405,7 @@ static void pxa2xx_setup_cp14(PXA2xxState *s)
 #define BSCNTR3		0x60	/* Memory Buffer Strength Control register 3 */
 #define SA1110		0x64	/* SA-1110 Memory Compatibility register */
 
-static uint64_t pxa2xx_mm_read(void *opaque, target_phys_addr_t addr,
+static uint64_t pxa2xx_mm_read(void *opaque, hwaddr addr,
                                unsigned size)
 {
     PXA2xxState *s = (PXA2xxState *) opaque;
@@ -422,7 +422,7 @@ static uint64_t pxa2xx_mm_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void pxa2xx_mm_write(void *opaque, target_phys_addr_t addr,
+static void pxa2xx_mm_write(void *opaque, hwaddr addr,
                             uint64_t value, unsigned size)
 {
     PXA2xxState *s = (PXA2xxState *) opaque;
@@ -567,7 +567,7 @@ static void pxa2xx_ssp_fifo_update(PXA2xxSSPState *s)
     pxa2xx_ssp_int_update(s);
 }
 
-static uint64_t pxa2xx_ssp_read(void *opaque, target_phys_addr_t addr,
+static uint64_t pxa2xx_ssp_read(void *opaque, hwaddr addr,
                                 unsigned size)
 {
     PXA2xxSSPState *s = (PXA2xxSSPState *) opaque;
@@ -613,7 +613,7 @@ static uint64_t pxa2xx_ssp_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void pxa2xx_ssp_write(void *opaque, target_phys_addr_t addr,
+static void pxa2xx_ssp_write(void *opaque, hwaddr addr,
                              uint64_t value64, unsigned size)
 {
     PXA2xxSSPState *s = (PXA2xxSSPState *) opaque;
@@ -943,7 +943,7 @@ static inline void pxa2xx_rtc_pi_tick(void *opaque)
     pxa2xx_rtc_int_update(s);
 }
 
-static uint64_t pxa2xx_rtc_read(void *opaque, target_phys_addr_t addr,
+static uint64_t pxa2xx_rtc_read(void *opaque, hwaddr addr,
                                 unsigned size)
 {
     PXA2xxRTCState *s = (PXA2xxRTCState *) opaque;
@@ -989,7 +989,7 @@ static uint64_t pxa2xx_rtc_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void pxa2xx_rtc_write(void *opaque, target_phys_addr_t addr,
+static void pxa2xx_rtc_write(void *opaque, hwaddr addr,
                              uint64_t value64, unsigned size)
 {
     PXA2xxRTCState *s = (PXA2xxRTCState *) opaque;
@@ -1294,7 +1294,7 @@ static int pxa2xx_i2c_tx(I2CSlave *i2c, uint8_t data)
     return 1;
 }
 
-static uint64_t pxa2xx_i2c_read(void *opaque, target_phys_addr_t addr,
+static uint64_t pxa2xx_i2c_read(void *opaque, hwaddr addr,
                                 unsigned size)
 {
     PXA2xxI2CState *s = (PXA2xxI2CState *) opaque;
@@ -1322,7 +1322,7 @@ static uint64_t pxa2xx_i2c_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void pxa2xx_i2c_write(void *opaque, target_phys_addr_t addr,
+static void pxa2xx_i2c_write(void *opaque, hwaddr addr,
                              uint64_t value64, unsigned size)
 {
     PXA2xxI2CState *s = (PXA2xxI2CState *) opaque;
@@ -1449,7 +1449,7 @@ static TypeInfo pxa2xx_i2c_slave_info = {
     .class_init    = pxa2xx_i2c_slave_class_init,
 };
 
-PXA2xxI2CState *pxa2xx_i2c_init(target_phys_addr_t base,
+PXA2xxI2CState *pxa2xx_i2c_init(hwaddr base,
                 qemu_irq irq, uint32_t region_size)
 {
     DeviceState *dev;
@@ -1572,7 +1572,7 @@ static inline void pxa2xx_i2s_update(PXA2xxI2SState *i2s)
 #define SADIV	0x60	/* Serial Audio Clock Divider register */
 #define SADR	0x80	/* Serial Audio Data register */
 
-static uint64_t pxa2xx_i2s_read(void *opaque, target_phys_addr_t addr,
+static uint64_t pxa2xx_i2s_read(void *opaque, hwaddr addr,
                                 unsigned size)
 {
     PXA2xxI2SState *s = (PXA2xxI2SState *) opaque;
@@ -1604,7 +1604,7 @@ static uint64_t pxa2xx_i2s_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void pxa2xx_i2s_write(void *opaque, target_phys_addr_t addr,
+static void pxa2xx_i2s_write(void *opaque, hwaddr addr,
                              uint64_t value, unsigned size)
 {
     PXA2xxI2SState *s = (PXA2xxI2SState *) opaque;
@@ -1706,7 +1706,7 @@ static void pxa2xx_i2s_data_req(void *opaque, int tx, int rx)
 }
 
 static PXA2xxI2SState *pxa2xx_i2s_init(MemoryRegion *sysmem,
-                target_phys_addr_t base,
+                hwaddr base,
                 qemu_irq irq, qemu_irq rx_dma, qemu_irq tx_dma)
 {
     PXA2xxI2SState *s = (PXA2xxI2SState *)
@@ -1801,7 +1801,7 @@ static inline void pxa2xx_fir_update(PXA2xxFIrState *s)
 #define ICSR1	0x18	/* FICP Status register 1 */
 #define ICFOR	0x1c	/* FICP FIFO Occupancy Status register */
 
-static uint64_t pxa2xx_fir_read(void *opaque, target_phys_addr_t addr,
+static uint64_t pxa2xx_fir_read(void *opaque, hwaddr addr,
                                 unsigned size)
 {
     PXA2xxFIrState *s = (PXA2xxFIrState *) opaque;
@@ -1839,7 +1839,7 @@ static uint64_t pxa2xx_fir_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void pxa2xx_fir_write(void *opaque, target_phys_addr_t addr,
+static void pxa2xx_fir_write(void *opaque, hwaddr addr,
                              uint64_t value64, unsigned size)
 {
     PXA2xxFIrState *s = (PXA2xxFIrState *) opaque;
@@ -1963,7 +1963,7 @@ static int pxa2xx_fir_load(QEMUFile *f, void *opaque, int version_id)
 }
 
 static PXA2xxFIrState *pxa2xx_fir_init(MemoryRegion *sysmem,
-                target_phys_addr_t base,
+                hwaddr base,
                 qemu_irq irq, qemu_irq rx_dma, qemu_irq tx_dma,
                 CharDriverState *chr)
 {
@@ -2108,7 +2108,7 @@ PXA2xxState *pxa270_init(MemoryRegion *address_space,
         s->ssp[i] = (SSIBus *)qdev_get_child_bus(dev, "ssi");
     }
 
-    if (usb_enabled) {
+    if (usb_enabled(false)) {
         sysbus_create_simple("sysbus-ohci", 0x4c000000,
                         qdev_get_gpio_in(s->pic, PXA2XX_PIC_USBH1));
     }
@@ -2239,7 +2239,7 @@ PXA2xxState *pxa255_init(MemoryRegion *address_space, unsigned int sdram_size)
         s->ssp[i] = (SSIBus *)qdev_get_child_bus(dev, "ssi");
     }
 
-    if (usb_enabled) {
+    if (usb_enabled(false)) {
         sysbus_create_simple("sysbus-ohci", 0x4c000000,
                         qdev_get_gpio_in(s->pic, PXA2XX_PIC_USBH1));
     }
diff --git a/hw/pxa2xx_dma.c b/hw/pxa2xx_dma.c
index 031015400..dbea1d209 100644
--- a/hw/pxa2xx_dma.c
+++ b/hw/pxa2xx_dma.c
@@ -147,7 +147,7 @@ static inline void pxa2xx_dma_descriptor_fetch(
                 PXA2xxDMAState *s, int ch)
 {
     uint32_t desc[4];
-    target_phys_addr_t daddr = s->chan[ch].descr & ~0xf;
+    hwaddr daddr = s->chan[ch].descr & ~0xf;
     if ((s->chan[ch].descr & DDADR_BREN) && (s->chan[ch].state & DCSR_CMPST))
         daddr += 32;
 
@@ -251,7 +251,7 @@ static void pxa2xx_dma_run(PXA2xxDMAState *s)
     }
 }
 
-static uint64_t pxa2xx_dma_read(void *opaque, target_phys_addr_t offset,
+static uint64_t pxa2xx_dma_read(void *opaque, hwaddr offset,
                                 unsigned size)
 {
     PXA2xxDMAState *s = (PXA2xxDMAState *) opaque;
@@ -310,7 +310,7 @@ static uint64_t pxa2xx_dma_read(void *opaque, target_phys_addr_t offset,
     return 7;
 }
 
-static void pxa2xx_dma_write(void *opaque, target_phys_addr_t offset,
+static void pxa2xx_dma_write(void *opaque, hwaddr offset,
                              uint64_t value, unsigned size)
 {
     PXA2xxDMAState *s = (PXA2xxDMAState *) opaque;
@@ -473,7 +473,7 @@ static int pxa2xx_dma_init(SysBusDevice *dev)
     return 0;
 }
 
-DeviceState *pxa27x_dma_init(target_phys_addr_t base, qemu_irq irq)
+DeviceState *pxa27x_dma_init(hwaddr base, qemu_irq irq)
 {
     DeviceState *dev;
 
@@ -487,7 +487,7 @@ DeviceState *pxa27x_dma_init(target_phys_addr_t base, qemu_irq irq)
     return dev;
 }
 
-DeviceState *pxa255_dma_init(target_phys_addr_t base, qemu_irq irq)
+DeviceState *pxa255_dma_init(hwaddr base, qemu_irq irq)
 {
     DeviceState *dev;
 
diff --git a/hw/pxa2xx_gpio.c b/hw/pxa2xx_gpio.c
index 3c90c9c4e..7aaf4092d 100644
--- a/hw/pxa2xx_gpio.c
+++ b/hw/pxa2xx_gpio.c
@@ -139,7 +139,7 @@ static void pxa2xx_gpio_handler_update(PXA2xxGPIOInfo *s) {
     }
 }
 
-static uint64_t pxa2xx_gpio_read(void *opaque, target_phys_addr_t offset,
+static uint64_t pxa2xx_gpio_read(void *opaque, hwaddr offset,
                                  unsigned size)
 {
     PXA2xxGPIOInfo *s = (PXA2xxGPIOInfo *) opaque;
@@ -191,7 +191,7 @@ static uint64_t pxa2xx_gpio_read(void *opaque, target_phys_addr_t offset,
     return 0;
 }
 
-static void pxa2xx_gpio_write(void *opaque, target_phys_addr_t offset,
+static void pxa2xx_gpio_write(void *opaque, hwaddr offset,
                               uint64_t value, unsigned size)
 {
     PXA2xxGPIOInfo *s = (PXA2xxGPIOInfo *) opaque;
@@ -249,7 +249,7 @@ static const MemoryRegionOps pxa_gpio_ops = {
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-DeviceState *pxa2xx_gpio_init(target_phys_addr_t base,
+DeviceState *pxa2xx_gpio_init(hwaddr base,
                 CPUARMState *env, DeviceState *pic, int lines)
 {
     DeviceState *dev;
diff --git a/hw/pxa2xx_keypad.c b/hw/pxa2xx_keypad.c
index 59db02584..257984c42 100644
--- a/hw/pxa2xx_keypad.c
+++ b/hw/pxa2xx_keypad.c
@@ -172,10 +172,9 @@ static void pxa27x_keyboard_event (PXA2xxKeyPadState *kp, int keycode)
         kp->kpc |= KPC_MI;
         qemu_irq_raise(kp->irq);
     }
-    return;
 }
 
-static uint64_t pxa2xx_keypad_read(void *opaque, target_phys_addr_t offset,
+static uint64_t pxa2xx_keypad_read(void *opaque, hwaddr offset,
                                    unsigned size)
 {
     PXA2xxKeyPadState *s = (PXA2xxKeyPadState *) opaque;
@@ -237,7 +236,7 @@ static uint64_t pxa2xx_keypad_read(void *opaque, target_phys_addr_t offset,
     return 0;
 }
 
-static void pxa2xx_keypad_write(void *opaque, target_phys_addr_t offset,
+static void pxa2xx_keypad_write(void *opaque, hwaddr offset,
                                 uint64_t value, unsigned size)
 {
     PXA2xxKeyPadState *s = (PXA2xxKeyPadState *) opaque;
@@ -306,7 +305,7 @@ static const VMStateDescription vmstate_pxa2xx_keypad = {
 };
 
 PXA2xxKeyPadState *pxa27x_keypad_init(MemoryRegion *sysmem,
-                                      target_phys_addr_t base,
+                                      hwaddr base,
                                       qemu_irq irq)
 {
     PXA2xxKeyPadState *s;
diff --git a/hw/pxa2xx_lcd.c b/hw/pxa2xx_lcd.c
index ee8bf577c..b53dfaf3c 100644
--- a/hw/pxa2xx_lcd.c
+++ b/hw/pxa2xx_lcd.c
@@ -23,7 +23,7 @@ struct DMAChannel {
     uint8_t up;
     uint8_t palette[1024];
     uint8_t pbuffer[1024];
-    void (*redraw)(PXA2xxLCDState *s, target_phys_addr_t addr,
+    void (*redraw)(PXA2xxLCDState *s, hwaddr addr,
                    int *miny, int *maxy);
 
     uint32_t descriptor;
@@ -291,7 +291,7 @@ static inline void pxa2xx_dma_rdst_set(PXA2xxLCDState *s)
 static void pxa2xx_descriptor_load(PXA2xxLCDState *s)
 {
     PXAFrameDescriptor desc;
-    target_phys_addr_t descptr;
+    hwaddr descptr;
     int i;
 
     for (i = 0; i < PXA_LCDDMA_CHANS; i ++) {
@@ -323,7 +323,7 @@ static void pxa2xx_descriptor_load(PXA2xxLCDState *s)
     }
 }
 
-static uint64_t pxa2xx_lcdc_read(void *opaque, target_phys_addr_t offset,
+static uint64_t pxa2xx_lcdc_read(void *opaque, hwaddr offset,
                                  unsigned size)
 {
     PXA2xxLCDState *s = (PXA2xxLCDState *) opaque;
@@ -417,7 +417,7 @@ static uint64_t pxa2xx_lcdc_read(void *opaque, target_phys_addr_t offset,
     return 0;
 }
 
-static void pxa2xx_lcdc_write(void *opaque, target_phys_addr_t offset,
+static void pxa2xx_lcdc_write(void *opaque, hwaddr offset,
                               uint64_t value, unsigned size)
 {
     PXA2xxLCDState *s = (PXA2xxLCDState *) opaque;
@@ -674,7 +674,7 @@ static void pxa2xx_palette_parse(PXA2xxLCDState *s, int ch, int bpp)
 }
 
 static void pxa2xx_lcdc_dma0_redraw_rot0(PXA2xxLCDState *s,
-                target_phys_addr_t addr, int *miny, int *maxy)
+                hwaddr addr, int *miny, int *maxy)
 {
     int src_width, dest_width;
     drawfn fn = NULL;
@@ -701,7 +701,7 @@ static void pxa2xx_lcdc_dma0_redraw_rot0(PXA2xxLCDState *s,
 }
 
 static void pxa2xx_lcdc_dma0_redraw_rot90(PXA2xxLCDState *s,
-               target_phys_addr_t addr, int *miny, int *maxy)
+               hwaddr addr, int *miny, int *maxy)
 {
     int src_width, dest_width;
     drawfn fn = NULL;
@@ -729,7 +729,7 @@ static void pxa2xx_lcdc_dma0_redraw_rot90(PXA2xxLCDState *s,
 }
 
 static void pxa2xx_lcdc_dma0_redraw_rot180(PXA2xxLCDState *s,
-                target_phys_addr_t addr, int *miny, int *maxy)
+                hwaddr addr, int *miny, int *maxy)
 {
     int src_width, dest_width;
     drawfn fn = NULL;
@@ -759,7 +759,7 @@ static void pxa2xx_lcdc_dma0_redraw_rot180(PXA2xxLCDState *s,
 }
 
 static void pxa2xx_lcdc_dma0_redraw_rot270(PXA2xxLCDState *s,
-               target_phys_addr_t addr, int *miny, int *maxy)
+               hwaddr addr, int *miny, int *maxy)
 {
     int src_width, dest_width;
     drawfn fn = NULL;
@@ -813,7 +813,7 @@ static void pxa2xx_lcdc_resize(PXA2xxLCDState *s)
 static void pxa2xx_update_display(void *opaque)
 {
     PXA2xxLCDState *s = (PXA2xxLCDState *) opaque;
-    target_phys_addr_t fbptr;
+    hwaddr fbptr;
     int miny, maxy;
     int ch;
     if (!(s->control[0] & LCCR0_ENB))
@@ -871,20 +871,20 @@ static void pxa2xx_update_display(void *opaque)
     if (miny >= 0) {
         switch (s->orientation) {
         case 0:
-            dpy_update(s->ds, 0, miny, s->xres, maxy - miny + 1);
+            dpy_gfx_update(s->ds, 0, miny, s->xres, maxy - miny + 1);
             break;
         case 90:
-            dpy_update(s->ds, miny, 0, maxy - miny + 1, s->xres);
+            dpy_gfx_update(s->ds, miny, 0, maxy - miny + 1, s->xres);
             break;
         case 180:
             maxy = s->yres - maxy - 1;
             miny = s->yres - miny - 1;
-            dpy_update(s->ds, 0, maxy, s->xres, miny - maxy + 1);
+            dpy_gfx_update(s->ds, 0, maxy, s->xres, miny - maxy + 1);
             break;
         case 270:
             maxy = s->yres - maxy - 1;
             miny = s->yres - miny - 1;
-            dpy_update(s->ds, maxy, 0, miny - maxy + 1, s->xres);
+            dpy_gfx_update(s->ds, maxy, 0, miny - maxy + 1, s->xres);
             break;
         }
     }
@@ -987,7 +987,7 @@ static const VMStateDescription vmstate_pxa2xx_lcdc = {
 #include "pxa2xx_template.h"
 
 PXA2xxLCDState *pxa2xx_lcdc_init(MemoryRegion *sysmem,
-                                 target_phys_addr_t base, qemu_irq irq)
+                                 hwaddr base, qemu_irq irq)
 {
     PXA2xxLCDState *s;
 
diff --git a/hw/pxa2xx_mmci.c b/hw/pxa2xx_mmci.c
index b505a4cc9..358996871 100644
--- a/hw/pxa2xx_mmci.c
+++ b/hw/pxa2xx_mmci.c
@@ -215,7 +215,7 @@ static void pxa2xx_mmci_wakequeues(PXA2xxMMCIState *s)
     pxa2xx_mmci_fifo_update(s);
 }
 
-static uint32_t pxa2xx_mmci_read(void *opaque, target_phys_addr_t offset)
+static uint32_t pxa2xx_mmci_read(void *opaque, hwaddr offset)
 {
     PXA2xxMMCIState *s = (PXA2xxMMCIState *) opaque;
     uint32_t ret;
@@ -277,7 +277,7 @@ static uint32_t pxa2xx_mmci_read(void *opaque, target_phys_addr_t offset)
 }
 
 static void pxa2xx_mmci_write(void *opaque,
-                target_phys_addr_t offset, uint32_t value)
+                hwaddr offset, uint32_t value)
 {
     PXA2xxMMCIState *s = (PXA2xxMMCIState *) opaque;
 
@@ -386,21 +386,21 @@ static void pxa2xx_mmci_write(void *opaque,
     }
 }
 
-static uint32_t pxa2xx_mmci_readb(void *opaque, target_phys_addr_t offset)
+static uint32_t pxa2xx_mmci_readb(void *opaque, hwaddr offset)
 {
     PXA2xxMMCIState *s = (PXA2xxMMCIState *) opaque;
     s->ac_width = 1;
     return pxa2xx_mmci_read(opaque, offset);
 }
 
-static uint32_t pxa2xx_mmci_readh(void *opaque, target_phys_addr_t offset)
+static uint32_t pxa2xx_mmci_readh(void *opaque, hwaddr offset)
 {
     PXA2xxMMCIState *s = (PXA2xxMMCIState *) opaque;
     s->ac_width = 2;
     return pxa2xx_mmci_read(opaque, offset);
 }
 
-static uint32_t pxa2xx_mmci_readw(void *opaque, target_phys_addr_t offset)
+static uint32_t pxa2xx_mmci_readw(void *opaque, hwaddr offset)
 {
     PXA2xxMMCIState *s = (PXA2xxMMCIState *) opaque;
     s->ac_width = 4;
@@ -408,7 +408,7 @@ static uint32_t pxa2xx_mmci_readw(void *opaque, target_phys_addr_t offset)
 }
 
 static void pxa2xx_mmci_writeb(void *opaque,
-                target_phys_addr_t offset, uint32_t value)
+                hwaddr offset, uint32_t value)
 {
     PXA2xxMMCIState *s = (PXA2xxMMCIState *) opaque;
     s->ac_width = 1;
@@ -416,7 +416,7 @@ static void pxa2xx_mmci_writeb(void *opaque,
 }
 
 static void pxa2xx_mmci_writeh(void *opaque,
-                target_phys_addr_t offset, uint32_t value)
+                hwaddr offset, uint32_t value)
 {
     PXA2xxMMCIState *s = (PXA2xxMMCIState *) opaque;
     s->ac_width = 2;
@@ -424,7 +424,7 @@ static void pxa2xx_mmci_writeh(void *opaque,
 }
 
 static void pxa2xx_mmci_writew(void *opaque,
-                target_phys_addr_t offset, uint32_t value)
+                hwaddr offset, uint32_t value)
 {
     PXA2xxMMCIState *s = (PXA2xxMMCIState *) opaque;
     s->ac_width = 4;
@@ -522,7 +522,7 @@ static int pxa2xx_mmci_load(QEMUFile *f, void *opaque, int version_id)
 }
 
 PXA2xxMMCIState *pxa2xx_mmci_init(MemoryRegion *sysmem,
-                target_phys_addr_t base,
+                hwaddr base,
                 BlockDriverState *bd, qemu_irq irq,
                 qemu_irq rx_dma, qemu_irq tx_dma)
 {
diff --git a/hw/pxa2xx_pcmcia.c b/hw/pxa2xx_pcmcia.c
index b15872a9d..3a79c728a 100644
--- a/hw/pxa2xx_pcmcia.c
+++ b/hw/pxa2xx_pcmcia.c
@@ -27,7 +27,7 @@ struct PXA2xxPCMCIAState {
 };
 
 static uint64_t pxa2xx_pcmcia_common_read(void *opaque,
-                target_phys_addr_t offset, unsigned size)
+                hwaddr offset, unsigned size)
 {
     PXA2xxPCMCIAState *s = (PXA2xxPCMCIAState *) opaque;
 
@@ -38,7 +38,7 @@ static uint64_t pxa2xx_pcmcia_common_read(void *opaque,
     return 0;
 }
 
-static void pxa2xx_pcmcia_common_write(void *opaque, target_phys_addr_t offset,
+static void pxa2xx_pcmcia_common_write(void *opaque, hwaddr offset,
                                        uint64_t value, unsigned size)
 {
     PXA2xxPCMCIAState *s = (PXA2xxPCMCIAState *) opaque;
@@ -49,7 +49,7 @@ static void pxa2xx_pcmcia_common_write(void *opaque, target_phys_addr_t offset,
 }
 
 static uint64_t pxa2xx_pcmcia_attr_read(void *opaque,
-                target_phys_addr_t offset, unsigned size)
+                hwaddr offset, unsigned size)
 {
     PXA2xxPCMCIAState *s = (PXA2xxPCMCIAState *) opaque;
 
@@ -60,7 +60,7 @@ static uint64_t pxa2xx_pcmcia_attr_read(void *opaque,
     return 0;
 }
 
-static void pxa2xx_pcmcia_attr_write(void *opaque, target_phys_addr_t offset,
+static void pxa2xx_pcmcia_attr_write(void *opaque, hwaddr offset,
                                      uint64_t value, unsigned size)
 {
     PXA2xxPCMCIAState *s = (PXA2xxPCMCIAState *) opaque;
@@ -71,7 +71,7 @@ static void pxa2xx_pcmcia_attr_write(void *opaque, target_phys_addr_t offset,
 }
 
 static uint64_t pxa2xx_pcmcia_io_read(void *opaque,
-                target_phys_addr_t offset, unsigned size)
+                hwaddr offset, unsigned size)
 {
     PXA2xxPCMCIAState *s = (PXA2xxPCMCIAState *) opaque;
 
@@ -82,7 +82,7 @@ static uint64_t pxa2xx_pcmcia_io_read(void *opaque,
     return 0;
 }
 
-static void pxa2xx_pcmcia_io_write(void *opaque, target_phys_addr_t offset,
+static void pxa2xx_pcmcia_io_write(void *opaque, hwaddr offset,
                                    uint64_t value, unsigned size)
 {
     PXA2xxPCMCIAState *s = (PXA2xxPCMCIAState *) opaque;
@@ -120,7 +120,7 @@ static void pxa2xx_pcmcia_set_irq(void *opaque, int line, int level)
 }
 
 PXA2xxPCMCIAState *pxa2xx_pcmcia_init(MemoryRegion *sysmem,
-                                      target_phys_addr_t base)
+                                      hwaddr base)
 {
     PXA2xxPCMCIAState *s;
 
diff --git a/hw/pxa2xx_pic.c b/hw/pxa2xx_pic.c
index e1e8830ff..70b2b79d0 100644
--- a/hw/pxa2xx_pic.c
+++ b/hw/pxa2xx_pic.c
@@ -119,7 +119,7 @@ static inline uint32_t pxa2xx_pic_highest(PXA2xxPICState *s) {
     return ichp;
 }
 
-static uint64_t pxa2xx_pic_mem_read(void *opaque, target_phys_addr_t offset,
+static uint64_t pxa2xx_pic_mem_read(void *opaque, hwaddr offset,
                                     unsigned size)
 {
     PXA2xxPICState *s = (PXA2xxPICState *) opaque;
@@ -159,7 +159,7 @@ static uint64_t pxa2xx_pic_mem_read(void *opaque, target_phys_addr_t offset,
     }
 }
 
-static void pxa2xx_pic_mem_write(void *opaque, target_phys_addr_t offset,
+static void pxa2xx_pic_mem_write(void *opaque, hwaddr offset,
                                  uint64_t value, unsigned size)
 {
     PXA2xxPICState *s = (PXA2xxPICState *) opaque;
@@ -257,7 +257,7 @@ static int pxa2xx_pic_post_load(void *opaque, int version_id)
     return 0;
 }
 
-DeviceState *pxa2xx_pic_init(target_phys_addr_t base, ARMCPU *cpu)
+DeviceState *pxa2xx_pic_init(hwaddr base, ARMCPU *cpu)
 {
     CPUARMState *env = &cpu->env;
     DeviceState *dev = qdev_create(NULL, "pxa2xx_pic");
diff --git a/hw/pxa2xx_timer.c b/hw/pxa2xx_timer.c
index 77b033b54..8242d26c3 100644
--- a/hw/pxa2xx_timer.c
+++ b/hw/pxa2xx_timer.c
@@ -149,7 +149,7 @@ static void pxa2xx_timer_update4(void *opaque, uint64_t now_qemu, int n)
     qemu_mod_timer(s->tm4[n].tm.qtimer, new_qemu);
 }
 
-static uint64_t pxa2xx_timer_read(void *opaque, target_phys_addr_t offset,
+static uint64_t pxa2xx_timer_read(void *opaque, hwaddr offset,
                                   unsigned size)
 {
     PXA2xxTimerInfo *s = (PXA2xxTimerInfo *) opaque;
@@ -227,7 +227,7 @@ static uint64_t pxa2xx_timer_read(void *opaque, target_phys_addr_t offset,
     return 0;
 }
 
-static void pxa2xx_timer_write(void *opaque, target_phys_addr_t offset,
+static void pxa2xx_timer_write(void *opaque, hwaddr offset,
                                uint64_t value, unsigned size)
 {
     int i, tm = 0;
diff --git a/hw/q35.c b/hw/q35.c
new file mode 100644
index 000000000..efebc2786
--- /dev/null
+++ b/hw/q35.c
@@ -0,0 +1,309 @@
+/*
+ * QEMU MCH/ICH9 PCI Bridge Emulation
+ *
+ * Copyright (c) 2006 Fabrice Bellard
+ * Copyright (c) 2009, 2010, 2011
+ *               Isaku Yamahata <yamahata at valinux co jp>
+ *               VA Linux Systems Japan K.K.
+ * Copyright (C) 2012 Jason Baron <jbaron@redhat.com>
+ *
+ * This is based on piix_pci.c, but heavily modified.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+#include "hw.h"
+#include "q35.h"
+
+/****************************************************************************
+ * Q35 host
+ */
+
+static int q35_host_init(SysBusDevice *dev)
+{
+    PCIBus *b;
+    PCIHostState *pci = FROM_SYSBUS(PCIHostState, dev);
+    Q35PCIHost *s = Q35_HOST_DEVICE(&dev->qdev);
+
+    memory_region_init_io(&pci->conf_mem, &pci_host_conf_le_ops, pci,
+                          "pci-conf-idx", 4);
+    sysbus_add_io(dev, MCH_HOST_BRIDGE_CONFIG_ADDR, &pci->conf_mem);
+    sysbus_init_ioports(&pci->busdev, MCH_HOST_BRIDGE_CONFIG_ADDR, 4);
+
+    memory_region_init_io(&pci->data_mem, &pci_host_data_le_ops, pci,
+                          "pci-conf-data", 4);
+    sysbus_add_io(dev, MCH_HOST_BRIDGE_CONFIG_DATA, &pci->data_mem);
+    sysbus_init_ioports(&pci->busdev, MCH_HOST_BRIDGE_CONFIG_DATA, 4);
+
+    if (pcie_host_init(&s->host) < 0) {
+        return -1;
+    }
+    b = pci_bus_new(&s->host.pci.busdev.qdev, "pcie.0",
+                    s->mch.pci_address_space, s->mch.address_space_io, 0);
+    s->host.pci.bus = b;
+    qdev_set_parent_bus(DEVICE(&s->mch), BUS(b));
+    qdev_init_nofail(DEVICE(&s->mch));
+
+    return 0;
+}
+
+static Property mch_props[] = {
+    DEFINE_PROP_UINT64("MCFG", Q35PCIHost, host.base_addr,
+                        MCH_HOST_BRIDGE_PCIEXBAR_DEFAULT),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void q35_host_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
+
+    k->init = q35_host_init;
+    dc->props = mch_props;
+}
+
+static void q35_host_initfn(Object *obj)
+{
+    Q35PCIHost *s = Q35_HOST_DEVICE(obj);
+
+    object_initialize(&s->mch, TYPE_MCH_PCI_DEVICE);
+    object_property_add_child(OBJECT(s), "mch", OBJECT(&s->mch), NULL);
+    qdev_prop_set_uint32(DEVICE(&s->mch), "addr", PCI_DEVFN(0, 0));
+    qdev_prop_set_bit(DEVICE(&s->mch), "multifunction", false);
+}
+
+static const TypeInfo q35_host_info = {
+    .name       = TYPE_Q35_HOST_DEVICE,
+    .parent     = TYPE_PCIE_HOST_BRIDGE,
+    .instance_size = sizeof(Q35PCIHost),
+    .instance_init = q35_host_initfn,
+    .class_init = q35_host_class_init,
+};
+
+/****************************************************************************
+ * MCH D0:F0
+ */
+
+/* PCIe MMCFG */
+static void mch_update_pciexbar(MCHPCIState *mch)
+{
+    PCIDevice *pci_dev = &mch->d;
+    BusState *bus = qdev_get_parent_bus(&pci_dev->qdev);
+    DeviceState *qdev = bus->parent;
+    Q35PCIHost *s = Q35_HOST_DEVICE(qdev);
+
+    uint64_t pciexbar;
+    int enable;
+    uint64_t addr;
+    uint64_t addr_mask;
+    uint32_t length;
+
+    pciexbar = pci_get_quad(pci_dev->config + MCH_HOST_BRIDGE_PCIEXBAR);
+    enable = pciexbar & MCH_HOST_BRIDGE_PCIEXBAREN;
+    addr_mask = MCH_HOST_BRIDGE_PCIEXBAR_ADMSK;
+    switch (pciexbar & MCH_HOST_BRIDGE_PCIEXBAR_LENGTH_MASK) {
+    case MCH_HOST_BRIDGE_PCIEXBAR_LENGTH_256M:
+        length = 256 * 1024 * 1024;
+        break;
+    case MCH_HOST_BRIDGE_PCIEXBAR_LENGTH_128M:
+        length = 128 * 1024 * 1024;
+        addr_mask |= MCH_HOST_BRIDGE_PCIEXBAR_128ADMSK |
+            MCH_HOST_BRIDGE_PCIEXBAR_64ADMSK;
+        break;
+    case MCH_HOST_BRIDGE_PCIEXBAR_LENGTH_64M:
+        length = 64 * 1024 * 1024;
+        addr_mask |= MCH_HOST_BRIDGE_PCIEXBAR_64ADMSK;
+        break;
+    case MCH_HOST_BRIDGE_PCIEXBAR_LENGTH_RVD:
+    default:
+        enable = 0;
+        length = 0;
+        abort();
+        break;
+    }
+    addr = pciexbar & addr_mask;
+    pcie_host_mmcfg_update(&s->host, enable, addr, length);
+}
+
+/* PAM */
+static void mch_update_pam(MCHPCIState *mch)
+{
+    int i;
+
+    memory_region_transaction_begin();
+    for (i = 0; i < 13; i++) {
+        pam_update(&mch->pam_regions[i], i,
+                   mch->d.config[MCH_HOST_BRIDGE_PAM0 + ((i + 1) / 2)]);
+    }
+    memory_region_transaction_commit();
+}
+
+/* SMRAM */
+static void mch_update_smram(MCHPCIState *mch)
+{
+    memory_region_transaction_begin();
+    smram_update(&mch->smram_region, mch->d.config[MCH_HOST_BRDIGE_SMRAM],
+                    mch->smm_enabled);
+    memory_region_transaction_commit();
+}
+
+static void mch_set_smm(int smm, void *arg)
+{
+    MCHPCIState *mch = arg;
+
+    memory_region_transaction_begin();
+    smram_set_smm(&mch->smm_enabled, smm, mch->d.config[MCH_HOST_BRDIGE_SMRAM],
+                    &mch->smram_region);
+    memory_region_transaction_commit();
+}
+
+static void mch_write_config(PCIDevice *d,
+                              uint32_t address, uint32_t val, int len)
+{
+    MCHPCIState *mch = MCH_PCI_DEVICE(d);
+
+    /* XXX: implement SMRAM.D_LOCK */
+    pci_default_write_config(d, address, val, len);
+
+    if (ranges_overlap(address, len, MCH_HOST_BRIDGE_PAM0,
+                       MCH_HOST_BRIDGE_PAM_SIZE)) {
+        mch_update_pam(mch);
+    }
+
+    if (ranges_overlap(address, len, MCH_HOST_BRIDGE_PCIEXBAR,
+                       MCH_HOST_BRIDGE_PCIEXBAR_SIZE)) {
+        mch_update_pciexbar(mch);
+    }
+
+    if (ranges_overlap(address, len, MCH_HOST_BRDIGE_SMRAM,
+                       MCH_HOST_BRDIGE_SMRAM_SIZE)) {
+        mch_update_smram(mch);
+    }
+}
+
+static void mch_update(MCHPCIState *mch)
+{
+    mch_update_pciexbar(mch);
+    mch_update_pam(mch);
+    mch_update_smram(mch);
+}
+
+static int mch_post_load(void *opaque, int version_id)
+{
+    MCHPCIState *mch = opaque;
+    mch_update(mch);
+    return 0;
+}
+
+static const VMStateDescription vmstate_mch = {
+    .name = "mch",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .minimum_version_id_old = 1,
+    .post_load = mch_post_load,
+    .fields = (VMStateField []) {
+        VMSTATE_PCI_DEVICE(d, MCHPCIState),
+        VMSTATE_UINT8(smm_enabled, MCHPCIState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static void mch_reset(DeviceState *qdev)
+{
+    PCIDevice *d = PCI_DEVICE(qdev);
+    MCHPCIState *mch = MCH_PCI_DEVICE(d);
+
+    pci_set_quad(d->config + MCH_HOST_BRIDGE_PCIEXBAR,
+                 MCH_HOST_BRIDGE_PCIEXBAR_DEFAULT);
+
+    d->config[MCH_HOST_BRDIGE_SMRAM] = MCH_HOST_BRIDGE_SMRAM_DEFAULT;
+
+    mch_update(mch);
+}
+
+static int mch_init(PCIDevice *d)
+{
+    int i;
+    hwaddr pci_hole64_size;
+    MCHPCIState *mch = MCH_PCI_DEVICE(d);
+
+    /* setup pci memory regions */
+    memory_region_init_alias(&mch->pci_hole, "pci-hole",
+                             mch->pci_address_space,
+                             mch->below_4g_mem_size,
+                             0x100000000ULL - mch->below_4g_mem_size);
+    memory_region_add_subregion(mch->system_memory, mch->below_4g_mem_size,
+                                &mch->pci_hole);
+    pci_hole64_size = (sizeof(hwaddr) == 4 ? 0 :
+                       ((uint64_t)1 << 62));
+    memory_region_init_alias(&mch->pci_hole_64bit, "pci-hole64",
+                             mch->pci_address_space,
+                             0x100000000ULL + mch->above_4g_mem_size,
+                             pci_hole64_size);
+    if (pci_hole64_size) {
+        memory_region_add_subregion(mch->system_memory,
+                                    0x100000000ULL + mch->above_4g_mem_size,
+                                    &mch->pci_hole_64bit);
+    }
+    /* smram */
+    cpu_smm_register(&mch_set_smm, mch);
+    memory_region_init_alias(&mch->smram_region, "smram-region",
+                             mch->pci_address_space, 0xa0000, 0x20000);
+    memory_region_add_subregion_overlap(mch->system_memory, 0xa0000,
+                                        &mch->smram_region, 1);
+    memory_region_set_enabled(&mch->smram_region, false);
+    init_pam(mch->ram_memory, mch->system_memory, mch->pci_address_space,
+             &mch->pam_regions[0], PAM_BIOS_BASE, PAM_BIOS_SIZE);
+    for (i = 0; i < 12; ++i) {
+        init_pam(mch->ram_memory, mch->system_memory, mch->pci_address_space,
+                 &mch->pam_regions[i+1], PAM_EXPAN_BASE + i * PAM_EXPAN_SIZE,
+                 PAM_EXPAN_SIZE);
+    }
+    return 0;
+}
+
+static void mch_class_init(ObjectClass *klass, void *data)
+{
+    PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    k->init = mch_init;
+    k->config_write = mch_write_config;
+    dc->reset = mch_reset;
+    dc->desc = "Host bridge";
+    dc->vmsd = &vmstate_mch;
+    k->vendor_id = PCI_VENDOR_ID_INTEL;
+    k->device_id = PCI_DEVICE_ID_INTEL_Q35_MCH;
+    k->revision = MCH_HOST_BRIDGE_REVISION_DEFUALT;
+    k->class_id = PCI_CLASS_BRIDGE_HOST;
+}
+
+static const TypeInfo mch_info = {
+    .name = TYPE_MCH_PCI_DEVICE,
+    .parent = TYPE_PCI_DEVICE,
+    .instance_size = sizeof(MCHPCIState),
+    .class_init = mch_class_init,
+};
+
+static void q35_register(void)
+{
+    type_register_static(&mch_info);
+    type_register_static(&q35_host_info);
+}
+
+type_init(q35_register);
diff --git a/hw/q35.h b/hw/q35.h
new file mode 100644
index 000000000..e34f7c165
--- /dev/null
+++ b/hw/q35.h
@@ -0,0 +1,150 @@
+/*
+ * q35.h
+ *
+ * Copyright (c) 2009 Isaku Yamahata <yamahata at valinux co jp>
+ *                    VA Linux Systems Japan K.K.
+ * Copyright (C) 2012 Jason Baron <jbaron@redhat.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>
+ */
+
+#ifndef HW_Q35_H
+#define HW_Q35_H
+
+#include "hw.h"
+#include "range.h"
+#include "isa.h"
+#include "sysbus.h"
+#include "pc.h"
+#include "apm.h"
+#include "apic.h"
+#include "pci.h"
+#include "pcie_host.h"
+#include "acpi.h"
+#include "acpi_ich9.h"
+#include "pam.h"
+
+#define TYPE_Q35_HOST_DEVICE "q35-pcihost"
+#define Q35_HOST_DEVICE(obj) \
+     OBJECT_CHECK(Q35PCIHost, (obj), TYPE_Q35_HOST_DEVICE)
+
+#define TYPE_MCH_PCI_DEVICE "mch"
+#define MCH_PCI_DEVICE(obj) \
+     OBJECT_CHECK(MCHPCIState, (obj), TYPE_MCH_PCI_DEVICE)
+
+typedef struct MCHPCIState {
+    PCIDevice d;
+    MemoryRegion *ram_memory;
+    MemoryRegion *pci_address_space;
+    MemoryRegion *system_memory;
+    MemoryRegion *address_space_io;
+    PAMMemoryRegion pam_regions[13];
+    MemoryRegion smram_region;
+    MemoryRegion pci_hole;
+    MemoryRegion pci_hole_64bit;
+    uint8_t smm_enabled;
+    ram_addr_t below_4g_mem_size;
+    ram_addr_t above_4g_mem_size;
+} MCHPCIState;
+
+typedef struct Q35PCIHost {
+    PCIExpressHost host;
+    MCHPCIState mch;
+} Q35PCIHost;
+
+#define Q35_MASK(bit, ms_bit, ls_bit) \
+((uint##bit##_t)(((1ULL << ((ms_bit) + 1)) - 1) & ~((1ULL << ls_bit) - 1)))
+
+/*
+ * gmch part
+ */
+
+/* PCI configuration */
+#define MCH_HOST_BRIDGE                        "MCH"
+
+#define MCH_HOST_BRIDGE_CONFIG_ADDR            0xcf8
+#define MCH_HOST_BRIDGE_CONFIG_DATA            0xcfc
+
+/* D0:F0 configuration space */
+#define MCH_HOST_BRIDGE_REVISION_DEFUALT       0x0
+
+#define MCH_HOST_BRIDGE_PCIEXBAR               0x60    /* 64bit register */
+#define MCH_HOST_BRIDGE_PCIEXBAR_SIZE          8       /* 64bit register */
+#define MCH_HOST_BRIDGE_PCIEXBAR_DEFAULT       0xb0000000
+#define MCH_HOST_BRIDGE_PCIEXBAR_ADMSK         Q35_MASK(64, 35, 28)
+#define MCH_HOST_BRIDGE_PCIEXBAR_128ADMSK      ((uint64_t)(1 << 26))
+#define MCH_HOST_BRIDGE_PCIEXBAR_64ADMSK       ((uint64_t)(1 << 25))
+#define MCH_HOST_BRIDGE_PCIEXBAR_LENGTH_MASK   ((uint64_t)(0x3 << 1))
+#define MCH_HOST_BRIDGE_PCIEXBAR_LENGTH_256M   ((uint64_t)(0x0 << 1))
+#define MCH_HOST_BRIDGE_PCIEXBAR_LENGTH_128M   ((uint64_t)(0x1 << 1))
+#define MCH_HOST_BRIDGE_PCIEXBAR_LENGTH_64M    ((uint64_t)(0x2 << 1))
+#define MCH_HOST_BRIDGE_PCIEXBAR_LENGTH_RVD    ((uint64_t)(0x3 << 1))
+#define MCH_HOST_BRIDGE_PCIEXBAREN             ((uint64_t)1)
+
+#define MCH_HOST_BRIDGE_PAM_NB                 7
+#define MCH_HOST_BRIDGE_PAM_SIZE               7
+#define MCH_HOST_BRIDGE_PAM0                   0x90
+#define MCH_HOST_BRIDGE_PAM_BIOS_AREA          0xf0000
+#define MCH_HOST_BRIDGE_PAM_AREA_SIZE          0x10000 /* 16KB */
+#define MCH_HOST_BRIDGE_PAM1                   0x91
+#define MCH_HOST_BRIDGE_PAM_EXPAN_AREA         0xc0000
+#define MCH_HOST_BRIDGE_PAM_EXPAN_SIZE         0x04000
+#define MCH_HOST_BRIDGE_PAM2                   0x92
+#define MCH_HOST_BRIDGE_PAM3                   0x93
+#define MCH_HOST_BRIDGE_PAM4                   0x94
+#define MCH_HOST_BRIDGE_PAM_EXBIOS_AREA        0xe0000
+#define MCH_HOST_BRIDGE_PAM_EXBIOS_SIZE        0x04000
+#define MCH_HOST_BRIDGE_PAM5                   0x95
+#define MCH_HOST_BRIDGE_PAM6                   0x96
+#define MCH_HOST_BRIDGE_PAM_WE_HI              ((uint8_t)(0x2 << 4))
+#define MCH_HOST_BRIDGE_PAM_RE_HI              ((uint8_t)(0x1 << 4))
+#define MCH_HOST_BRIDGE_PAM_HI_MASK            ((uint8_t)(0x3 << 4))
+#define MCH_HOST_BRIDGE_PAM_WE_LO              ((uint8_t)0x2)
+#define MCH_HOST_BRIDGE_PAM_RE_LO              ((uint8_t)0x1)
+#define MCH_HOST_BRIDGE_PAM_LO_MASK            ((uint8_t)0x3)
+#define MCH_HOST_BRIDGE_PAM_WE                 ((uint8_t)0x2)
+#define MCH_HOST_BRIDGE_PAM_RE                 ((uint8_t)0x1)
+#define MCH_HOST_BRIDGE_PAM_MASK               ((uint8_t)0x3)
+
+#define MCH_HOST_BRDIGE_SMRAM                  0x9d
+#define MCH_HOST_BRDIGE_SMRAM_SIZE             1
+#define MCH_HOST_BRIDGE_SMRAM_DEFAULT          ((uint8_t)0x2)
+#define MCH_HOST_BRIDGE_SMRAM_D_OPEN           ((uint8_t)(1 << 6))
+#define MCH_HOST_BRIDGE_SMRAM_D_CLS            ((uint8_t)(1 << 5))
+#define MCH_HOST_BRIDGE_SMRAM_D_LCK            ((uint8_t)(1 << 4))
+#define MCH_HOST_BRIDGE_SMRAM_G_SMRAME         ((uint8_t)(1 << 3))
+#define MCH_HOST_BRIDGE_SMRAM_C_BASE_SEG_MASK  ((uint8_t)0x7)
+#define MCH_HOST_BRIDGE_SMRAM_C_BASE_SEG       ((uint8_t)0x2)  /* hardwired to b010 */
+#define MCH_HOST_BRIDGE_SMRAM_C_BASE           0xa0000
+#define MCH_HOST_BRIDGE_SMRAM_C_END            0xc0000
+#define MCH_HOST_BRIDGE_SMRAM_C_SIZE           0x20000
+#define MCH_HOST_BRIDGE_UPPER_SYSTEM_BIOS_END  0x100000
+
+#define MCH_HOST_BRIDGE_ESMRAMC                0x9e
+#define MCH_HOST_BRDIGE_ESMRAMC_H_SMRAME       ((uint8_t)(1 << 6))
+#define MCH_HOST_BRDIGE_ESMRAMC_E_SMERR        ((uint8_t)(1 << 5))
+#define MCH_HOST_BRDIGE_ESMRAMC_SM_CACHE       ((uint8_t)(1 << 4))
+#define MCH_HOST_BRDIGE_ESMRAMC_SM_L1          ((uint8_t)(1 << 3))
+#define MCH_HOST_BRDIGE_ESMRAMC_SM_L2          ((uint8_t)(1 << 2))
+#define MCH_HOST_BRDIGE_ESMRAMC_TSEG_SZ_MASK   ((uint8_t)(0x3 << 1))
+#define MCH_HOST_BRDIGE_ESMRAMC_TSEG_SZ_1MB    ((uint8_t)(0x0 << 1))
+#define MCH_HOST_BRDIGE_ESMRAMC_TSEG_SZ_2MB    ((uint8_t)(0x1 << 1))
+#define MCH_HOST_BRDIGE_ESMRAMC_TSEG_SZ_8MB    ((uint8_t)(0x2 << 1))
+#define MCH_HOST_BRDIGE_ESMRAMC_T_EN           ((uint8_t)1)
+
+/* D1:F0 PCIE* port*/
+#define MCH_PCIE_DEV                           1
+#define MCH_PCIE_FUNC                          0
+
+#endif /* HW_Q35_H */
diff --git a/hw/qdev-addr.c b/hw/qdev-addr.c
index b711b6bf9..ea32c31ab 100644
--- a/hw/qdev-addr.c
+++ b/hw/qdev-addr.c
@@ -1,12 +1,13 @@
 #include "qdev.h"
 #include "qdev-addr.h"
-#include "targphys.h"
+#include "hwaddr.h"
+#include "qapi/qapi-visit-core.h"
 
 /* --- target physical address --- */
 
 static int parse_taddr(DeviceState *dev, Property *prop, const char *str)
 {
-    target_phys_addr_t *ptr = qdev_get_prop_ptr(dev, prop);
+    hwaddr *ptr = qdev_get_prop_ptr(dev, prop);
 
     *ptr = strtoull(str, NULL, 16);
     return 0;
@@ -14,7 +15,7 @@ static int parse_taddr(DeviceState *dev, Property *prop, const char *str)
 
 static int print_taddr(DeviceState *dev, Property *prop, char *dest, size_t len)
 {
-    target_phys_addr_t *ptr = qdev_get_prop_ptr(dev, prop);
+    hwaddr *ptr = qdev_get_prop_ptr(dev, prop);
     return snprintf(dest, len, "0x" TARGET_FMT_plx, *ptr);
 }
 
@@ -23,7 +24,7 @@ static void get_taddr(Object *obj, Visitor *v, void *opaque,
 {
     DeviceState *dev = DEVICE(obj);
     Property *prop = opaque;
-    target_phys_addr_t *ptr = qdev_get_prop_ptr(dev, prop);
+    hwaddr *ptr = qdev_get_prop_ptr(dev, prop);
     int64_t value;
 
     value = *ptr;
@@ -35,7 +36,7 @@ static void set_taddr(Object *obj, Visitor *v, void *opaque,
 {
     DeviceState *dev = DEVICE(obj);
     Property *prop = opaque;
-    target_phys_addr_t *ptr = qdev_get_prop_ptr(dev, prop);
+    hwaddr *ptr = qdev_get_prop_ptr(dev, prop);
     Error *local_err = NULL;
     int64_t value;
 
@@ -49,12 +50,12 @@ static void set_taddr(Object *obj, Visitor *v, void *opaque,
         error_propagate(errp, local_err);
         return;
     }
-    if ((uint64_t)value <= (uint64_t) ~(target_phys_addr_t)0) {
+    if ((uint64_t)value <= (uint64_t) ~(hwaddr)0) {
         *ptr = value;
     } else {
         error_set(errp, QERR_PROPERTY_VALUE_OUT_OF_RANGE,
                   dev->id?:"", name, value, (uint64_t) 0,
-                  (uint64_t) ~(target_phys_addr_t)0);
+                  (uint64_t) ~(hwaddr)0);
     }
 }
 
@@ -67,7 +68,7 @@ PropertyInfo qdev_prop_taddr = {
     .set   = set_taddr,
 };
 
-void qdev_prop_set_taddr(DeviceState *dev, const char *name, target_phys_addr_t value)
+void qdev_prop_set_taddr(DeviceState *dev, const char *name, hwaddr value)
 {
     Error *errp = NULL;
     object_property_set_int(OBJECT(dev), value, name, &errp);
diff --git a/hw/qdev-addr.h b/hw/qdev-addr.h
index a0ddf3863..ea5ecb4d7 100644
--- a/hw/qdev-addr.h
+++ b/hw/qdev-addr.h
@@ -1,5 +1,5 @@
 #define DEFINE_PROP_TADDR(_n, _s, _f, _d)                               \
-    DEFINE_PROP_DEFAULT(_n, _s, _f, _d, qdev_prop_taddr, target_phys_addr_t)
+    DEFINE_PROP_DEFAULT(_n, _s, _f, _d, qdev_prop_taddr, hwaddr)
 
 extern PropertyInfo qdev_prop_taddr;
-void qdev_prop_set_taddr(DeviceState *dev, const char *name, target_phys_addr_t value);
+void qdev_prop_set_taddr(DeviceState *dev, const char *name, hwaddr value);
diff --git a/hw/qdev-core.h b/hw/qdev-core.h
new file mode 100644
index 000000000..fff7f0f5a
--- /dev/null
+++ b/hw/qdev-core.h
@@ -0,0 +1,233 @@
+#ifndef QDEV_CORE_H
+#define QDEV_CORE_H
+
+#include "qemu-queue.h"
+#include "qemu-option.h"
+#include "qemu/object.h"
+#include "hw/irq.h"
+#include "error.h"
+
+typedef struct Property Property;
+
+typedef struct PropertyInfo PropertyInfo;
+
+typedef struct CompatProperty CompatProperty;
+
+typedef struct BusState BusState;
+
+typedef struct BusClass BusClass;
+
+enum DevState {
+    DEV_STATE_CREATED = 1,
+    DEV_STATE_INITIALIZED,
+};
+
+enum {
+    DEV_NVECTORS_UNSPECIFIED = -1,
+};
+
+#define TYPE_DEVICE "device"
+#define DEVICE(obj) OBJECT_CHECK(DeviceState, (obj), TYPE_DEVICE)
+#define DEVICE_CLASS(klass) OBJECT_CLASS_CHECK(DeviceClass, (klass), TYPE_DEVICE)
+#define DEVICE_GET_CLASS(obj) OBJECT_GET_CLASS(DeviceClass, (obj), TYPE_DEVICE)
+
+typedef int (*qdev_initfn)(DeviceState *dev);
+typedef int (*qdev_event)(DeviceState *dev);
+typedef void (*qdev_resetfn)(DeviceState *dev);
+
+struct VMStateDescription;
+
+typedef struct DeviceClass {
+    ObjectClass parent_class;
+
+    const char *fw_name;
+    const char *desc;
+    Property *props;
+    int no_user;
+
+    /* callbacks */
+    void (*reset)(DeviceState *dev);
+
+    /* device state */
+    const struct VMStateDescription *vmsd;
+
+    /* Private to qdev / bus.  */
+    qdev_initfn init;
+    qdev_event unplug;
+    qdev_event exit;
+    const char *bus_type;
+} DeviceClass;
+
+/* This structure should not be accessed directly.  We declare it here
+   so that it can be embedded in individual device state structures.  */
+struct DeviceState {
+    Object parent_obj;
+
+    const char *id;
+    enum DevState state;
+    QemuOpts *opts;
+    int hotplugged;
+    BusState *parent_bus;
+    int num_gpio_out;
+    qemu_irq *gpio_out;
+    int num_gpio_in;
+    qemu_irq *gpio_in;
+    QLIST_HEAD(, BusState) child_bus;
+    int num_child_bus;
+    int instance_id_alias;
+    int alias_required_for_version;
+};
+
+#define TYPE_BUS "bus"
+#define BUS(obj) OBJECT_CHECK(BusState, (obj), TYPE_BUS)
+#define BUS_CLASS(klass) OBJECT_CLASS_CHECK(BusClass, (klass), TYPE_BUS)
+#define BUS_GET_CLASS(obj) OBJECT_GET_CLASS(BusClass, (obj), TYPE_BUS)
+
+struct BusClass {
+    ObjectClass parent_class;
+
+    /* FIXME first arg should be BusState */
+    void (*print_dev)(Monitor *mon, DeviceState *dev, int indent);
+    char *(*get_dev_path)(DeviceState *dev);
+    /*
+     * This callback is used to create Open Firmware device path in accordance
+     * with OF spec http://forthworks.com/standards/of1275.pdf. Individual bus
+     * bindings can be found at http://playground.sun.com/1275/bindings/.
+     */
+    char *(*get_fw_dev_path)(DeviceState *dev);
+    int (*reset)(BusState *bus);
+};
+
+typedef struct BusChild {
+    DeviceState *child;
+    int index;
+    QTAILQ_ENTRY(BusChild) sibling;
+} BusChild;
+
+/**
+ * BusState:
+ */
+struct BusState {
+    Object obj;
+    DeviceState *parent;
+    const char *name;
+    int allow_hotplug;
+    int max_index;
+    QTAILQ_HEAD(ChildrenHead, BusChild) children;
+    QLIST_ENTRY(BusState) sibling;
+};
+
+struct Property {
+    const char   *name;
+    PropertyInfo *info;
+    int          offset;
+    uint8_t      bitnr;
+    uint8_t      qtype;
+    int64_t      defval;
+};
+
+struct PropertyInfo {
+    const char *name;
+    const char *legacy_name;
+    const char **enum_table;
+    int (*parse)(DeviceState *dev, Property *prop, const char *str);
+    int (*print)(DeviceState *dev, Property *prop, char *dest, size_t len);
+    ObjectPropertyAccessor *get;
+    ObjectPropertyAccessor *set;
+    ObjectPropertyRelease *release;
+};
+
+typedef struct GlobalProperty {
+    const char *driver;
+    const char *property;
+    const char *value;
+    QTAILQ_ENTRY(GlobalProperty) next;
+} GlobalProperty;
+
+/*** Board API.  This should go away once we have a machine config file.  ***/
+
+DeviceState *qdev_create(BusState *bus, const char *name);
+DeviceState *qdev_try_create(BusState *bus, const char *name);
+int qdev_init(DeviceState *dev) QEMU_WARN_UNUSED_RESULT;
+void qdev_init_nofail(DeviceState *dev);
+void qdev_set_legacy_instance_id(DeviceState *dev, int alias_id,
+                                 int required_for_version);
+void qdev_unplug(DeviceState *dev, Error **errp);
+void qdev_free(DeviceState *dev);
+int qdev_simple_unplug_cb(DeviceState *dev);
+void qdev_machine_creation_done(void);
+bool qdev_machine_modified(void);
+
+qemu_irq qdev_get_gpio_in(DeviceState *dev, int n);
+void qdev_connect_gpio_out(DeviceState *dev, int n, qemu_irq pin);
+
+BusState *qdev_get_child_bus(DeviceState *dev, const char *name);
+
+/*** Device API.  ***/
+
+/* Register device properties.  */
+/* GPIO inputs also double as IRQ sinks.  */
+void qdev_init_gpio_in(DeviceState *dev, qemu_irq_handler handler, int n);
+void qdev_init_gpio_out(DeviceState *dev, qemu_irq *pins, int n);
+
+BusState *qdev_get_parent_bus(DeviceState *dev);
+
+/*** BUS API. ***/
+
+DeviceState *qdev_find_recursive(BusState *bus, const char *id);
+
+/* Returns 0 to walk children, > 0 to skip walk, < 0 to terminate walk. */
+typedef int (qbus_walkerfn)(BusState *bus, void *opaque);
+typedef int (qdev_walkerfn)(DeviceState *dev, void *opaque);
+
+void qbus_create_inplace(BusState *bus, const char *typename,
+                         DeviceState *parent, const char *name);
+BusState *qbus_create(const char *typename, DeviceState *parent, const char *name);
+/* Returns > 0 if either devfn or busfn skip walk somewhere in cursion,
+ *         < 0 if either devfn or busfn terminate walk somewhere in cursion,
+ *           0 otherwise. */
+int qbus_walk_children(BusState *bus, qdev_walkerfn *devfn,
+                       qbus_walkerfn *busfn, void *opaque);
+int qdev_walk_children(DeviceState *dev, qdev_walkerfn *devfn,
+                       qbus_walkerfn *busfn, void *opaque);
+void qdev_reset_all(DeviceState *dev);
+void qbus_reset_all_fn(void *opaque);
+
+void qbus_free(BusState *bus);
+
+#define FROM_QBUS(type, dev) DO_UPCAST(type, qbus, dev)
+
+/* This should go away once we get rid of the NULL bus hack */
+BusState *sysbus_get_default(void);
+
+char *qdev_get_fw_dev_path(DeviceState *dev);
+
+/**
+ * @qdev_machine_init
+ *
+ * Initialize platform devices before machine init.  This is a hack until full
+ * support for composition is added.
+ */
+void qdev_machine_init(void);
+
+/**
+ * @device_reset
+ *
+ * Reset a single device (by calling the reset method).
+ */
+void device_reset(DeviceState *dev);
+
+const struct VMStateDescription *qdev_get_vmsd(DeviceState *dev);
+
+const char *qdev_fw_name(DeviceState *dev);
+
+Object *qdev_get_machine(void);
+
+/* FIXME: make this a link<> */
+void qdev_set_parent_bus(DeviceState *dev, BusState *bus);
+
+extern int qdev_hotplug;
+
+char *qdev_get_dev_path(DeviceState *dev);
+
+#endif
diff --git a/hw/qdev-monitor.c b/hw/qdev-monitor.c
index 33b7f79a9..a1b4d6ae5 100644
--- a/hw/qdev-monitor.c
+++ b/hw/qdev-monitor.c
@@ -44,6 +44,7 @@ static const QDevAlias qdev_alias_table[] = {
     { "virtio-serial-s390", "virtio-serial", QEMU_ARCH_S390X },
     { "lsi53c895a", "lsi" },
     { "ich9-ahci", "ahci" },
+    { "kvm-pci-assign", "pci-assign" },
     { }
 };
 
@@ -288,8 +289,7 @@ static BusState *qbus_find_recursive(BusState *bus, const char *name,
     if (name && (strcmp(bus->name, name) != 0)) {
         match = 0;
     }
-    if (bus_typename &&
-        (strcmp(object_get_typename(OBJECT(bus)), bus_typename) != 0)) {
+    if (bus_typename && !object_dynamic_cast(OBJECT(bus), bus_typename)) {
         match = 0;
     }
     if (match) {
@@ -434,7 +434,7 @@ DeviceState *qdev_device_add(QemuOpts *opts)
         if (!bus) {
             return NULL;
         }
-        if (strcmp(object_get_typename(OBJECT(bus)), k->bus_type) != 0) {
+        if (!object_dynamic_cast(OBJECT(bus), k->bus_type)) {
             qerror_report(QERR_BAD_BUS_FOR_DEVICE,
                           driver, object_get_typename(OBJECT(bus)));
             return NULL;
diff --git a/hw/qdev-monitor.h b/hw/qdev-monitor.h
new file mode 100644
index 000000000..220ceba4c
--- /dev/null
+++ b/hw/qdev-monitor.h
@@ -0,0 +1,16 @@
+#ifndef QEMU_QDEV_MONITOR_H
+#define QEMU_QDEV_MONITOR_H
+
+#include "qdev-core.h"
+#include "monitor.h"
+
+/*** monitor commands ***/
+
+void do_info_qtree(Monitor *mon);
+void do_info_qdm(Monitor *mon);
+int do_device_add(Monitor *mon, const QDict *qdict, QObject **ret_data);
+int do_device_del(Monitor *mon, const QDict *qdict, QObject **ret_data);
+int qdev_device_help(QemuOpts *opts);
+DeviceState *qdev_device_add(QemuOpts *opts);
+
+#endif
diff --git a/hw/qdev-properties.c b/hw/qdev-properties.c
index 8aca0d43f..81d901c6c 100644
--- a/hw/qdev-properties.c
+++ b/hw/qdev-properties.c
@@ -4,6 +4,7 @@
 #include "blockdev.h"
 #include "hw/block-common.h"
 #include "net/hub.h"
+#include "qapi/qapi-visit-core.h"
 
 void *qdev_get_prop_ptr(DeviceState *dev, Property *prop)
 {
diff --git a/hw/qdev-properties.h b/hw/qdev-properties.h
new file mode 100644
index 000000000..5b046abb2
--- /dev/null
+++ b/hw/qdev-properties.h
@@ -0,0 +1,130 @@
+#ifndef QEMU_QDEV_PROPERTIES_H
+#define QEMU_QDEV_PROPERTIES_H
+
+#include "qdev-core.h"
+
+/*** qdev-properties.c ***/
+
+extern PropertyInfo qdev_prop_bit;
+extern PropertyInfo qdev_prop_uint8;
+extern PropertyInfo qdev_prop_uint16;
+extern PropertyInfo qdev_prop_uint32;
+extern PropertyInfo qdev_prop_int32;
+extern PropertyInfo qdev_prop_uint64;
+extern PropertyInfo qdev_prop_hex8;
+extern PropertyInfo qdev_prop_hex32;
+extern PropertyInfo qdev_prop_hex64;
+extern PropertyInfo qdev_prop_string;
+extern PropertyInfo qdev_prop_chr;
+extern PropertyInfo qdev_prop_ptr;
+extern PropertyInfo qdev_prop_macaddr;
+extern PropertyInfo qdev_prop_losttickpolicy;
+extern PropertyInfo qdev_prop_bios_chs_trans;
+extern PropertyInfo qdev_prop_drive;
+extern PropertyInfo qdev_prop_netdev;
+extern PropertyInfo qdev_prop_vlan;
+extern PropertyInfo qdev_prop_pci_devfn;
+extern PropertyInfo qdev_prop_blocksize;
+extern PropertyInfo qdev_prop_pci_host_devaddr;
+
+#define DEFINE_PROP(_name, _state, _field, _prop, _type) { \
+        .name      = (_name),                                    \
+        .info      = &(_prop),                                   \
+        .offset    = offsetof(_state, _field)                    \
+            + type_check(_type,typeof_field(_state, _field)),    \
+        }
+#define DEFINE_PROP_DEFAULT(_name, _state, _field, _defval, _prop, _type) { \
+        .name      = (_name),                                           \
+        .info      = &(_prop),                                          \
+        .offset    = offsetof(_state, _field)                           \
+            + type_check(_type,typeof_field(_state, _field)),           \
+        .qtype     = QTYPE_QINT,                                        \
+        .defval    = (_type)_defval,                                    \
+        }
+#define DEFINE_PROP_BIT(_name, _state, _field, _bit, _defval) {  \
+        .name      = (_name),                                    \
+        .info      = &(qdev_prop_bit),                           \
+        .bitnr    = (_bit),                                      \
+        .offset    = offsetof(_state, _field)                    \
+            + type_check(uint32_t,typeof_field(_state, _field)), \
+        .qtype     = QTYPE_QBOOL,                                \
+        .defval    = (bool)_defval,                              \
+        }
+
+#define DEFINE_PROP_UINT8(_n, _s, _f, _d)                       \
+    DEFINE_PROP_DEFAULT(_n, _s, _f, _d, qdev_prop_uint8, uint8_t)
+#define DEFINE_PROP_UINT16(_n, _s, _f, _d)                      \
+    DEFINE_PROP_DEFAULT(_n, _s, _f, _d, qdev_prop_uint16, uint16_t)
+#define DEFINE_PROP_UINT32(_n, _s, _f, _d)                      \
+    DEFINE_PROP_DEFAULT(_n, _s, _f, _d, qdev_prop_uint32, uint32_t)
+#define DEFINE_PROP_INT32(_n, _s, _f, _d)                      \
+    DEFINE_PROP_DEFAULT(_n, _s, _f, _d, qdev_prop_int32, int32_t)
+#define DEFINE_PROP_UINT64(_n, _s, _f, _d)                      \
+    DEFINE_PROP_DEFAULT(_n, _s, _f, _d, qdev_prop_uint64, uint64_t)
+#define DEFINE_PROP_HEX8(_n, _s, _f, _d)                       \
+    DEFINE_PROP_DEFAULT(_n, _s, _f, _d, qdev_prop_hex8, uint8_t)
+#define DEFINE_PROP_HEX32(_n, _s, _f, _d)                       \
+    DEFINE_PROP_DEFAULT(_n, _s, _f, _d, qdev_prop_hex32, uint32_t)
+#define DEFINE_PROP_HEX64(_n, _s, _f, _d)                       \
+    DEFINE_PROP_DEFAULT(_n, _s, _f, _d, qdev_prop_hex64, uint64_t)
+#define DEFINE_PROP_PCI_DEVFN(_n, _s, _f, _d)                   \
+    DEFINE_PROP_DEFAULT(_n, _s, _f, _d, qdev_prop_pci_devfn, int32_t)
+
+#define DEFINE_PROP_PTR(_n, _s, _f)             \
+    DEFINE_PROP(_n, _s, _f, qdev_prop_ptr, void*)
+#define DEFINE_PROP_CHR(_n, _s, _f)             \
+    DEFINE_PROP(_n, _s, _f, qdev_prop_chr, CharDriverState*)
+#define DEFINE_PROP_STRING(_n, _s, _f)             \
+    DEFINE_PROP(_n, _s, _f, qdev_prop_string, char*)
+#define DEFINE_PROP_NETDEV(_n, _s, _f)             \
+    DEFINE_PROP(_n, _s, _f, qdev_prop_netdev, NetClientState*)
+#define DEFINE_PROP_VLAN(_n, _s, _f)             \
+    DEFINE_PROP(_n, _s, _f, qdev_prop_vlan, NetClientState*)
+#define DEFINE_PROP_DRIVE(_n, _s, _f) \
+    DEFINE_PROP(_n, _s, _f, qdev_prop_drive, BlockDriverState *)
+#define DEFINE_PROP_MACADDR(_n, _s, _f)         \
+    DEFINE_PROP(_n, _s, _f, qdev_prop_macaddr, MACAddr)
+#define DEFINE_PROP_LOSTTICKPOLICY(_n, _s, _f, _d) \
+    DEFINE_PROP_DEFAULT(_n, _s, _f, _d, qdev_prop_losttickpolicy, \
+                        LostTickPolicy)
+#define DEFINE_PROP_BIOS_CHS_TRANS(_n, _s, _f, _d) \
+    DEFINE_PROP_DEFAULT(_n, _s, _f, _d, qdev_prop_bios_chs_trans, int)
+#define DEFINE_PROP_BLOCKSIZE(_n, _s, _f, _d) \
+    DEFINE_PROP_DEFAULT(_n, _s, _f, _d, qdev_prop_blocksize, uint16_t)
+#define DEFINE_PROP_PCI_HOST_DEVADDR(_n, _s, _f) \
+    DEFINE_PROP(_n, _s, _f, qdev_prop_pci_host_devaddr, PCIHostDeviceAddress)
+
+#define DEFINE_PROP_END_OF_LIST()               \
+    {}
+
+/* Set properties between creation and init.  */
+void *qdev_get_prop_ptr(DeviceState *dev, Property *prop);
+int qdev_prop_parse(DeviceState *dev, const char *name, const char *value);
+void qdev_prop_set_bit(DeviceState *dev, const char *name, bool value);
+void qdev_prop_set_uint8(DeviceState *dev, const char *name, uint8_t value);
+void qdev_prop_set_uint16(DeviceState *dev, const char *name, uint16_t value);
+void qdev_prop_set_uint32(DeviceState *dev, const char *name, uint32_t value);
+void qdev_prop_set_int32(DeviceState *dev, const char *name, int32_t value);
+void qdev_prop_set_uint64(DeviceState *dev, const char *name, uint64_t value);
+void qdev_prop_set_string(DeviceState *dev, const char *name, const char *value);
+void qdev_prop_set_chr(DeviceState *dev, const char *name, CharDriverState *value);
+void qdev_prop_set_netdev(DeviceState *dev, const char *name, NetClientState *value);
+int qdev_prop_set_drive(DeviceState *dev, const char *name, BlockDriverState *value) QEMU_WARN_UNUSED_RESULT;
+void qdev_prop_set_drive_nofail(DeviceState *dev, const char *name, BlockDriverState *value);
+void qdev_prop_set_macaddr(DeviceState *dev, const char *name, uint8_t *value);
+void qdev_prop_set_enum(DeviceState *dev, const char *name, int value);
+/* FIXME: Remove opaque pointer properties.  */
+void qdev_prop_set_ptr(DeviceState *dev, const char *name, void *value);
+
+void qdev_prop_register_global_list(GlobalProperty *props);
+void qdev_prop_set_globals(DeviceState *dev);
+void error_set_from_qdev_prop_error(Error **errp, int ret, DeviceState *dev,
+                                    Property *prop, const char *value);
+
+/**
+ * @qdev_property_add_static - add a @Property to a device referencing a
+ * field in a struct.
+ */
+void qdev_property_add_static(DeviceState *dev, Property *prop, Error **errp);
+
+#endif
diff --git a/hw/qdev.c b/hw/qdev.c
index b5a52ac50..788b4da55 100644
--- a/hw/qdev.c
+++ b/hw/qdev.c
@@ -29,12 +29,12 @@
 #include "qdev.h"
 #include "sysemu.h"
 #include "error.h"
+#include "qapi/qapi-visit-core.h"
 
 int qdev_hotplug = 0;
 static bool qdev_hot_added = false;
 static bool qdev_hot_removed = false;
 
-/* Register a new device type.  */
 const VMStateDescription *qdev_get_vmsd(DeviceState *dev)
 {
     DeviceClass *dc = DEVICE_GET_CLASS(dev);
@@ -52,11 +52,6 @@ const char *qdev_fw_name(DeviceState *dev)
     return object_get_typename(OBJECT(dev));
 }
 
-bool qdev_exists(const char *name)
-{
-    return !!object_class_by_name(name);
-}
-
 static void qdev_property_add_legacy(DeviceState *dev, Property *prop,
                                      Error **errp);
 
@@ -291,9 +286,9 @@ BusState *qdev_get_parent_bus(DeviceState *dev)
 
 void qdev_init_gpio_in(DeviceState *dev, qemu_irq_handler handler, int n)
 {
-    assert(dev->num_gpio_in == 0);
-    dev->num_gpio_in = n;
-    dev->gpio_in = qemu_allocate_irqs(handler, dev, n);
+    dev->gpio_in = qemu_extend_irqs(dev->gpio_in, dev->num_gpio_in, handler,
+                                        dev, n);
+    dev->num_gpio_in += n;
 }
 
 void qdev_init_gpio_out(DeviceState *dev, qemu_irq *pins, int n)
@@ -459,7 +454,6 @@ BusState *qbus_create(const char *typename, DeviceState *parent, const char *nam
     BusState *bus;
 
     bus = BUS(object_new(typename));
-    bus->qom_allocated = true;
 
     bus->parent = parent;
     bus->name = name ? g_strdup(name) : NULL;
@@ -470,14 +464,7 @@ BusState *qbus_create(const char *typename, DeviceState *parent, const char *nam
 
 void qbus_free(BusState *bus)
 {
-    if (bus->qom_allocated) {
-        object_delete(OBJECT(bus));
-    } else {
-        object_finalize(OBJECT(bus));
-        if (bus->glib_allocated) {
-            g_free(bus);
-        }
-    }
+    object_delete(OBJECT(bus));
 }
 
 static char *bus_get_fw_dev_path(BusState *bus, DeviceState *dev)
@@ -520,7 +507,7 @@ char* qdev_get_fw_dev_path(DeviceState *dev)
 
     path[l-1] = '\0';
 
-    return strdup(path);
+    return g_strdup(path);
 }
 
 char *qdev_get_dev_path(DeviceState *dev)
@@ -710,9 +697,6 @@ static void device_finalize(Object *obj)
             qemu_opts_del(dev->opts);
         }
     }
-    if (dev->parent_bus) {
-        bus_remove_child(dev->parent_bus, dev);
-    }
 }
 
 static void device_class_base_init(ObjectClass *class, void *data)
@@ -725,6 +709,18 @@ static void device_class_base_init(ObjectClass *class, void *data)
     klass->props = NULL;
 }
 
+static void qdev_remove_from_bus(Object *obj)
+{
+    DeviceState *dev = DEVICE(obj);
+
+    bus_remove_child(dev->parent_bus, dev);
+}
+
+static void device_class_init(ObjectClass *class, void *data)
+{
+    class->unparent = qdev_remove_from_bus;
+}
+
 void device_reset(DeviceState *dev)
 {
     DeviceClass *klass = DEVICE_GET_CLASS(dev);
@@ -752,6 +748,7 @@ static TypeInfo device_type_info = {
     .instance_init = device_initfn,
     .instance_finalize = device_finalize,
     .class_base_init = device_class_base_init,
+    .class_init = device_class_init,
     .abstract = true,
     .class_size = sizeof(DeviceClass),
 };
diff --git a/hw/qdev.h b/hw/qdev.h
index d69919441..365b8d6ca 100644
--- a/hw/qdev.h
+++ b/hw/qdev.h
@@ -1,372 +1,9 @@
 #ifndef QDEV_H
 #define QDEV_H
 
-#include "hw.h"
-#include "qemu-queue.h"
-#include "qemu-char.h"
-#include "qemu-option.h"
-#include "qapi/qapi-visit-core.h"
-#include "qemu/object.h"
-#include "error.h"
-
-typedef struct Property Property;
-
-typedef struct PropertyInfo PropertyInfo;
-
-typedef struct CompatProperty CompatProperty;
-
-typedef struct BusState BusState;
-
-typedef struct BusClass BusClass;
-
-enum DevState {
-    DEV_STATE_CREATED = 1,
-    DEV_STATE_INITIALIZED,
-};
-
-enum {
-    DEV_NVECTORS_UNSPECIFIED = -1,
-};
-
-#define TYPE_DEVICE "device"
-#define DEVICE(obj) OBJECT_CHECK(DeviceState, (obj), TYPE_DEVICE)
-#define DEVICE_CLASS(klass) OBJECT_CLASS_CHECK(DeviceClass, (klass), TYPE_DEVICE)
-#define DEVICE_GET_CLASS(obj) OBJECT_GET_CLASS(DeviceClass, (obj), TYPE_DEVICE)
-
-typedef int (*qdev_initfn)(DeviceState *dev);
-typedef int (*qdev_event)(DeviceState *dev);
-typedef void (*qdev_resetfn)(DeviceState *dev);
-
-typedef struct DeviceClass {
-    ObjectClass parent_class;
-
-    const char *fw_name;
-    const char *desc;
-    Property *props;
-    int no_user;
-
-    /* callbacks */
-    void (*reset)(DeviceState *dev);
-
-    /* device state */
-    const VMStateDescription *vmsd;
-
-    /* Private to qdev / bus.  */
-    qdev_initfn init;
-    qdev_event unplug;
-    qdev_event exit;
-    const char *bus_type;
-} DeviceClass;
-
-/* This structure should not be accessed directly.  We declare it here
-   so that it can be embedded in individual device state structures.  */
-struct DeviceState {
-    Object parent_obj;
-
-    const char *id;
-    enum DevState state;
-    QemuOpts *opts;
-    int hotplugged;
-    BusState *parent_bus;
-    int num_gpio_out;
-    qemu_irq *gpio_out;
-    int num_gpio_in;
-    qemu_irq *gpio_in;
-    QLIST_HEAD(, BusState) child_bus;
-    int num_child_bus;
-    int instance_id_alias;
-    int alias_required_for_version;
-};
-
-#define TYPE_BUS "bus"
-#define BUS(obj) OBJECT_CHECK(BusState, (obj), TYPE_BUS)
-#define BUS_CLASS(klass) OBJECT_CLASS_CHECK(BusClass, (klass), TYPE_BUS)
-#define BUS_GET_CLASS(obj) OBJECT_GET_CLASS(BusClass, (obj), TYPE_BUS)
-
-struct BusClass {
-    ObjectClass parent_class;
-
-    /* FIXME first arg should be BusState */
-    void (*print_dev)(Monitor *mon, DeviceState *dev, int indent);
-    char *(*get_dev_path)(DeviceState *dev);
-    /*
-     * This callback is used to create Open Firmware device path in accordance
-     * with OF spec http://forthworks.com/standards/of1275.pdf. Individual bus
-     * bindings can be found at http://playground.sun.com/1275/bindings/.
-     */
-    char *(*get_fw_dev_path)(DeviceState *dev);
-    int (*reset)(BusState *bus);
-};
-
-typedef struct BusChild {
-    DeviceState *child;
-    int index;
-    QTAILQ_ENTRY(BusChild) sibling;
-} BusChild;
-
-/**
- * BusState:
- * @qom_allocated: Indicates whether the object was allocated by QOM.
- * @glib_allocated: Indicates whether the object was initialized in-place
- * yet is expected to be freed with g_free().
- */
-struct BusState {
-    Object obj;
-    DeviceState *parent;
-    const char *name;
-    int allow_hotplug;
-    bool qom_allocated;
-    bool glib_allocated;
-    int max_index;
-    QTAILQ_HEAD(ChildrenHead, BusChild) children;
-    QLIST_ENTRY(BusState) sibling;
-};
-
-struct Property {
-    const char   *name;
-    PropertyInfo *info;
-    int          offset;
-    uint8_t      bitnr;
-    uint8_t      qtype;
-    int64_t      defval;
-};
-
-struct PropertyInfo {
-    const char *name;
-    const char *legacy_name;
-    const char **enum_table;
-    int (*parse)(DeviceState *dev, Property *prop, const char *str);
-    int (*print)(DeviceState *dev, Property *prop, char *dest, size_t len);
-    ObjectPropertyAccessor *get;
-    ObjectPropertyAccessor *set;
-    ObjectPropertyRelease *release;
-};
-
-typedef struct GlobalProperty {
-    const char *driver;
-    const char *property;
-    const char *value;
-    QTAILQ_ENTRY(GlobalProperty) next;
-} GlobalProperty;
-
-/*** Board API.  This should go away once we have a machine config file.  ***/
-
-DeviceState *qdev_create(BusState *bus, const char *name);
-DeviceState *qdev_try_create(BusState *bus, const char *name);
-bool qdev_exists(const char *name);
-int qdev_device_help(QemuOpts *opts);
-DeviceState *qdev_device_add(QemuOpts *opts);
-int qdev_init(DeviceState *dev) QEMU_WARN_UNUSED_RESULT;
-void qdev_init_nofail(DeviceState *dev);
-void qdev_set_legacy_instance_id(DeviceState *dev, int alias_id,
-                                 int required_for_version);
-void qdev_unplug(DeviceState *dev, Error **errp);
-void qdev_free(DeviceState *dev);
-int qdev_simple_unplug_cb(DeviceState *dev);
-void qdev_machine_creation_done(void);
-bool qdev_machine_modified(void);
-
-qemu_irq qdev_get_gpio_in(DeviceState *dev, int n);
-void qdev_connect_gpio_out(DeviceState *dev, int n, qemu_irq pin);
-
-BusState *qdev_get_child_bus(DeviceState *dev, const char *name);
-
-/*** Device API.  ***/
-
-/* Register device properties.  */
-/* GPIO inputs also double as IRQ sinks.  */
-void qdev_init_gpio_in(DeviceState *dev, qemu_irq_handler handler, int n);
-void qdev_init_gpio_out(DeviceState *dev, qemu_irq *pins, int n);
-
-BusState *qdev_get_parent_bus(DeviceState *dev);
-
-/*** BUS API. ***/
-
-DeviceState *qdev_find_recursive(BusState *bus, const char *id);
-
-/* Returns 0 to walk children, > 0 to skip walk, < 0 to terminate walk. */
-typedef int (qbus_walkerfn)(BusState *bus, void *opaque);
-typedef int (qdev_walkerfn)(DeviceState *dev, void *opaque);
-
-void qbus_create_inplace(BusState *bus, const char *typename,
-                         DeviceState *parent, const char *name);
-BusState *qbus_create(const char *typename, DeviceState *parent, const char *name);
-/* Returns > 0 if either devfn or busfn skip walk somewhere in cursion,
- *         < 0 if either devfn or busfn terminate walk somewhere in cursion,
- *           0 otherwise. */
-int qbus_walk_children(BusState *bus, qdev_walkerfn *devfn,
-                       qbus_walkerfn *busfn, void *opaque);
-int qdev_walk_children(DeviceState *dev, qdev_walkerfn *devfn,
-                       qbus_walkerfn *busfn, void *opaque);
-void qdev_reset_all(DeviceState *dev);
-void qbus_reset_all_fn(void *opaque);
-
-void qbus_free(BusState *bus);
-
-#define FROM_QBUS(type, dev) DO_UPCAST(type, qbus, dev)
-
-/* This should go away once we get rid of the NULL bus hack */
-BusState *sysbus_get_default(void);
-
-/*** monitor commands ***/
-
-void do_info_qtree(Monitor *mon);
-void do_info_qdm(Monitor *mon);
-int do_device_add(Monitor *mon, const QDict *qdict, QObject **ret_data);
-int do_device_del(Monitor *mon, const QDict *qdict, QObject **ret_data);
-
-/*** qdev-properties.c ***/
-
-extern PropertyInfo qdev_prop_bit;
-extern PropertyInfo qdev_prop_uint8;
-extern PropertyInfo qdev_prop_uint16;
-extern PropertyInfo qdev_prop_uint32;
-extern PropertyInfo qdev_prop_int32;
-extern PropertyInfo qdev_prop_uint64;
-extern PropertyInfo qdev_prop_hex8;
-extern PropertyInfo qdev_prop_hex32;
-extern PropertyInfo qdev_prop_hex64;
-extern PropertyInfo qdev_prop_string;
-extern PropertyInfo qdev_prop_chr;
-extern PropertyInfo qdev_prop_ptr;
-extern PropertyInfo qdev_prop_macaddr;
-extern PropertyInfo qdev_prop_losttickpolicy;
-extern PropertyInfo qdev_prop_bios_chs_trans;
-extern PropertyInfo qdev_prop_drive;
-extern PropertyInfo qdev_prop_netdev;
-extern PropertyInfo qdev_prop_vlan;
-extern PropertyInfo qdev_prop_pci_devfn;
-extern PropertyInfo qdev_prop_blocksize;
-extern PropertyInfo qdev_prop_pci_host_devaddr;
-
-#define DEFINE_PROP(_name, _state, _field, _prop, _type) { \
-        .name      = (_name),                                    \
-        .info      = &(_prop),                                   \
-        .offset    = offsetof(_state, _field)                    \
-            + type_check(_type,typeof_field(_state, _field)),    \
-        }
-#define DEFINE_PROP_DEFAULT(_name, _state, _field, _defval, _prop, _type) { \
-        .name      = (_name),                                           \
-        .info      = &(_prop),                                          \
-        .offset    = offsetof(_state, _field)                           \
-            + type_check(_type,typeof_field(_state, _field)),           \
-        .qtype     = QTYPE_QINT,                                        \
-        .defval    = (_type)_defval,                                    \
-        }
-#define DEFINE_PROP_BIT(_name, _state, _field, _bit, _defval) {  \
-        .name      = (_name),                                    \
-        .info      = &(qdev_prop_bit),                           \
-        .bitnr    = (_bit),                                      \
-        .offset    = offsetof(_state, _field)                    \
-            + type_check(uint32_t,typeof_field(_state, _field)), \
-        .qtype     = QTYPE_QBOOL,                                \
-        .defval    = (bool)_defval,                              \
-        }
-
-#define DEFINE_PROP_UINT8(_n, _s, _f, _d)                       \
-    DEFINE_PROP_DEFAULT(_n, _s, _f, _d, qdev_prop_uint8, uint8_t)
-#define DEFINE_PROP_UINT16(_n, _s, _f, _d)                      \
-    DEFINE_PROP_DEFAULT(_n, _s, _f, _d, qdev_prop_uint16, uint16_t)
-#define DEFINE_PROP_UINT32(_n, _s, _f, _d)                      \
-    DEFINE_PROP_DEFAULT(_n, _s, _f, _d, qdev_prop_uint32, uint32_t)
-#define DEFINE_PROP_INT32(_n, _s, _f, _d)                      \
-    DEFINE_PROP_DEFAULT(_n, _s, _f, _d, qdev_prop_int32, int32_t)
-#define DEFINE_PROP_UINT64(_n, _s, _f, _d)                      \
-    DEFINE_PROP_DEFAULT(_n, _s, _f, _d, qdev_prop_uint64, uint64_t)
-#define DEFINE_PROP_HEX8(_n, _s, _f, _d)                       \
-    DEFINE_PROP_DEFAULT(_n, _s, _f, _d, qdev_prop_hex8, uint8_t)
-#define DEFINE_PROP_HEX32(_n, _s, _f, _d)                       \
-    DEFINE_PROP_DEFAULT(_n, _s, _f, _d, qdev_prop_hex32, uint32_t)
-#define DEFINE_PROP_HEX64(_n, _s, _f, _d)                       \
-    DEFINE_PROP_DEFAULT(_n, _s, _f, _d, qdev_prop_hex64, uint64_t)
-#define DEFINE_PROP_PCI_DEVFN(_n, _s, _f, _d)                   \
-    DEFINE_PROP_DEFAULT(_n, _s, _f, _d, qdev_prop_pci_devfn, int32_t)
-
-#define DEFINE_PROP_PTR(_n, _s, _f)             \
-    DEFINE_PROP(_n, _s, _f, qdev_prop_ptr, void*)
-#define DEFINE_PROP_CHR(_n, _s, _f)             \
-    DEFINE_PROP(_n, _s, _f, qdev_prop_chr, CharDriverState*)
-#define DEFINE_PROP_STRING(_n, _s, _f)             \
-    DEFINE_PROP(_n, _s, _f, qdev_prop_string, char*)
-#define DEFINE_PROP_NETDEV(_n, _s, _f)             \
-    DEFINE_PROP(_n, _s, _f, qdev_prop_netdev, NetClientState*)
-#define DEFINE_PROP_VLAN(_n, _s, _f)             \
-    DEFINE_PROP(_n, _s, _f, qdev_prop_vlan, NetClientState*)
-#define DEFINE_PROP_DRIVE(_n, _s, _f) \
-    DEFINE_PROP(_n, _s, _f, qdev_prop_drive, BlockDriverState *)
-#define DEFINE_PROP_MACADDR(_n, _s, _f)         \
-    DEFINE_PROP(_n, _s, _f, qdev_prop_macaddr, MACAddr)
-#define DEFINE_PROP_LOSTTICKPOLICY(_n, _s, _f, _d) \
-    DEFINE_PROP_DEFAULT(_n, _s, _f, _d, qdev_prop_losttickpolicy, \
-                        LostTickPolicy)
-#define DEFINE_PROP_BIOS_CHS_TRANS(_n, _s, _f, _d) \
-    DEFINE_PROP_DEFAULT(_n, _s, _f, _d, qdev_prop_bios_chs_trans, int)
-#define DEFINE_PROP_BLOCKSIZE(_n, _s, _f, _d) \
-    DEFINE_PROP_DEFAULT(_n, _s, _f, _d, qdev_prop_blocksize, uint16_t)
-#define DEFINE_PROP_PCI_HOST_DEVADDR(_n, _s, _f) \
-    DEFINE_PROP(_n, _s, _f, qdev_prop_pci_host_devaddr, PCIHostDeviceAddress)
-
-#define DEFINE_PROP_END_OF_LIST()               \
-    {}
-
-/* Set properties between creation and init.  */
-void *qdev_get_prop_ptr(DeviceState *dev, Property *prop);
-int qdev_prop_parse(DeviceState *dev, const char *name, const char *value);
-void qdev_prop_set_bit(DeviceState *dev, const char *name, bool value);
-void qdev_prop_set_uint8(DeviceState *dev, const char *name, uint8_t value);
-void qdev_prop_set_uint16(DeviceState *dev, const char *name, uint16_t value);
-void qdev_prop_set_uint32(DeviceState *dev, const char *name, uint32_t value);
-void qdev_prop_set_int32(DeviceState *dev, const char *name, int32_t value);
-void qdev_prop_set_uint64(DeviceState *dev, const char *name, uint64_t value);
-void qdev_prop_set_string(DeviceState *dev, const char *name, const char *value);
-void qdev_prop_set_chr(DeviceState *dev, const char *name, CharDriverState *value);
-void qdev_prop_set_netdev(DeviceState *dev, const char *name, NetClientState *value);
-int qdev_prop_set_drive(DeviceState *dev, const char *name, BlockDriverState *value) QEMU_WARN_UNUSED_RESULT;
-void qdev_prop_set_drive_nofail(DeviceState *dev, const char *name, BlockDriverState *value);
-void qdev_prop_set_macaddr(DeviceState *dev, const char *name, uint8_t *value);
-void qdev_prop_set_enum(DeviceState *dev, const char *name, int value);
-/* FIXME: Remove opaque pointer properties.  */
-void qdev_prop_set_ptr(DeviceState *dev, const char *name, void *value);
-
-void qdev_prop_register_global_list(GlobalProperty *props);
-void qdev_prop_set_globals(DeviceState *dev);
-void error_set_from_qdev_prop_error(Error **errp, int ret, DeviceState *dev,
-                                    Property *prop, const char *value);
-
-char *qdev_get_fw_dev_path(DeviceState *dev);
-
-/**
- * @qdev_property_add_static - add a @Property to a device referencing a
- * field in a struct.
- */
-void qdev_property_add_static(DeviceState *dev, Property *prop, Error **errp);
-
-/**
- * @qdev_machine_init
- *
- * Initialize platform devices before machine init.  This is a hack until full
- * support for composition is added.
- */
-void qdev_machine_init(void);
-
-/**
- * @device_reset
- *
- * Reset a single device (by calling the reset method).
- */
-void device_reset(DeviceState *dev);
-
-const VMStateDescription *qdev_get_vmsd(DeviceState *dev);
-
-const char *qdev_fw_name(DeviceState *dev);
-
-Object *qdev_get_machine(void);
-
-/* FIXME: make this a link<> */
-void qdev_set_parent_bus(DeviceState *dev, BusState *bus);
-
-extern int qdev_hotplug;
-
-char *qdev_get_dev_path(DeviceState *dev);
+#include "hw/hw.h"
+#include "qdev-core.h"
+#include "qdev-properties.h"
+#include "qdev-monitor.h"
 
 #endif
diff --git a/hw/qxl-render.c b/hw/qxl-render.c
index e2e3fe2d3..98ecb2140 100644
--- a/hw/qxl-render.c
+++ b/hw/qxl-render.c
@@ -24,7 +24,7 @@
 static void qxl_blit(PCIQXLDevice *qxl, QXLRect *rect)
 {
     uint8_t *src;
-    uint8_t *dst = qxl->vga.ds->surface->data;
+    uint8_t *dst = ds_get_data(qxl->vga.ds);
     int len, i;
 
     if (is_buffer_shared(qxl->vga.ds->surface)) {
@@ -99,7 +99,6 @@ static void qxl_render_update_area_unlocked(PCIQXLDevice *qxl)
 {
     VGACommonState *vga = &qxl->vga;
     int i;
-    DisplaySurface *surface = vga->ds->surface;
 
     if (qxl->guest_primary.resized) {
         qxl->guest_primary.resized = 0;
@@ -112,9 +111,6 @@ static void qxl_render_update_area_unlocked(PCIQXLDevice *qxl)
                qxl->guest_primary.qxl_stride,
                qxl->guest_primary.bytes_pp,
                qxl->guest_primary.bits_pp);
-    }
-    if (surface->width != qxl->guest_primary.surface.width ||
-        surface->height != qxl->guest_primary.surface.height) {
         if (qxl->guest_primary.qxl_stride > 0) {
             qemu_free_displaysurface(vga->ds);
             qemu_create_displaysurface_from(qxl->guest_primary.surface.width,
@@ -127,17 +123,17 @@ static void qxl_render_update_area_unlocked(PCIQXLDevice *qxl)
                     qxl->guest_primary.surface.width,
                     qxl->guest_primary.surface.height);
         }
-        dpy_resize(vga->ds);
+        dpy_gfx_resize(vga->ds);
     }
     for (i = 0; i < qxl->num_dirty_rects; i++) {
         if (qemu_spice_rect_is_empty(qxl->dirty+i)) {
             break;
         }
         qxl_blit(qxl, qxl->dirty+i);
-        dpy_update(vga->ds,
-                   qxl->dirty[i].left, qxl->dirty[i].top,
-                   qxl->dirty[i].right - qxl->dirty[i].left,
-                   qxl->dirty[i].bottom - qxl->dirty[i].top);
+        dpy_gfx_update(vga->ds,
+                       qxl->dirty[i].left, qxl->dirty[i].top,
+                       qxl->dirty[i].right - qxl->dirty[i].left,
+                       qxl->dirty[i].bottom - qxl->dirty[i].top);
     }
     qxl->num_dirty_rects = 0;
 }
@@ -238,7 +234,7 @@ int qxl_render_cursor(PCIQXLDevice *qxl, QXLCommandExt *ext)
         return 1;
     }
 
-    if (!qxl->ssd.ds->mouse_set || !qxl->ssd.ds->cursor_define) {
+    if (!dpy_cursor_define_supported(qxl->ssd.ds)) {
         return 0;
     }
 
diff --git a/hw/qxl.c b/hw/qxl.c
index c2dd3b471..96887c4aa 100644
--- a/hw/qxl.c
+++ b/hw/qxl.c
@@ -18,6 +18,8 @@
  * along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
+#include <zlib.h>
+
 #include "qemu-common.h"
 #include "qemu-timer.h"
 #include "qemu-queue.h"
@@ -136,6 +138,7 @@ static void qxl_ring_set_dirty(PCIQXLDevice *qxl);
 
 void qxl_set_guest_bug(PCIQXLDevice *qxl, const char *msg, ...)
 {
+    trace_qxl_set_guest_bug(qxl->id);
     qxl_send_events(qxl, QXL_INTERRUPT_ERROR);
     qxl->guest_bug = 1;
     if (qxl->guestdebug) {
@@ -196,6 +199,7 @@ static void qxl_spice_destroy_surface_wait(PCIQXLDevice *qxl, uint32_t id,
         spice_qxl_destroy_surface_async(&qxl->ssd.qxl, id, (uintptr_t)cookie);
     } else {
         qxl->ssd.worker->destroy_surface_wait(qxl->ssd.worker, id);
+        qxl_spice_destroy_surface_wait_complete(qxl, id);
     }
 }
 
@@ -231,7 +235,8 @@ static void qxl_spice_destroy_surfaces_complete(PCIQXLDevice *qxl)
 {
     trace_qxl_spice_destroy_surfaces_complete(qxl->id);
     qemu_mutex_lock(&qxl->track_lock);
-    memset(&qxl->guest_surfaces.cmds, 0, sizeof(qxl->guest_surfaces.cmds));
+    memset(qxl->guest_surfaces.cmds, 0,
+           sizeof(qxl->guest_surfaces.cmds) * qxl->ssd.num_surfaces);
     qxl->guest_surfaces.count = 0;
     qemu_mutex_unlock(&qxl->track_lock);
 }
@@ -249,6 +254,32 @@ static void qxl_spice_destroy_surfaces(PCIQXLDevice *qxl, qxl_async_io async)
     }
 }
 
+static void qxl_spice_monitors_config_async(PCIQXLDevice *qxl, int replay)
+{
+    trace_qxl_spice_monitors_config(qxl->id);
+    if (replay) {
+        /*
+         * don't use QXL_COOKIE_TYPE_IO:
+         *  - we are not running yet (post_load), we will assert
+         *    in send_events
+         *  - this is not a guest io, but a reply, so async_io isn't set.
+         */
+        spice_qxl_monitors_config_async(&qxl->ssd.qxl,
+                qxl->guest_monitors_config,
+                MEMSLOT_GROUP_GUEST,
+                (uintptr_t)qxl_cookie_new(
+                    QXL_COOKIE_TYPE_POST_LOAD_MONITORS_CONFIG,
+                    0));
+    } else {
+        qxl->guest_monitors_config = qxl->ram->monitors_config;
+        spice_qxl_monitors_config_async(&qxl->ssd.qxl,
+                qxl->ram->monitors_config,
+                MEMSLOT_GROUP_GUEST,
+                (uintptr_t)qxl_cookie_new(QXL_COOKIE_TYPE_IO,
+                                          QXL_IO_MONITORS_CONFIG_ASYNC));
+    }
+}
+
 void qxl_spice_reset_image_cache(PCIQXLDevice *qxl)
 {
     trace_qxl_spice_reset_image_cache(qxl->id);
@@ -262,6 +293,10 @@ void qxl_spice_reset_cursor(PCIQXLDevice *qxl)
     qemu_mutex_lock(&qxl->track_lock);
     qxl->guest_cursor = 0;
     qemu_mutex_unlock(&qxl->track_lock);
+    if (qxl->ssd.cursor) {
+        cursor_put(qxl->ssd.cursor);
+    }
+    qxl->ssd.cursor = cursor_builtin_hidden();
 }
 
 
@@ -307,7 +342,7 @@ static void init_qxl_rom(PCIQXLDevice *d)
     rom->slot_id_bits  = MEMSLOT_SLOT_BITS;
     rom->slots_start   = 1;
     rom->slots_end     = NUM_MEMSLOTS - 1;
-    rom->n_surfaces    = cpu_to_le32(NUM_SURFACES);
+    rom->n_surfaces    = cpu_to_le32(d->ssd.num_surfaces);
 
     for (i = 0, n = 0; i < ARRAY_SIZE(qxl_modes); i++) {
         fb = qxl_modes[i].y_res * qxl_modes[i].stride;
@@ -411,9 +446,15 @@ static int qxl_track_command(PCIQXLDevice *qxl, struct QXLCommandExt *ext)
         }
         uint32_t id = le32_to_cpu(cmd->surface_id);
 
-        if (id >= NUM_SURFACES) {
+        if (id >= qxl->ssd.num_surfaces) {
             qxl_set_guest_bug(qxl, "QXL_CMD_SURFACE id %d >= %d", id,
-                              NUM_SURFACES);
+                              qxl->ssd.num_surfaces);
+            return 1;
+        }
+        if (cmd->type == QXL_SURFACE_CMD_CREATE &&
+            (cmd->u.surface_create.stride & 0x03) != 0) {
+            qxl_set_guest_bug(qxl, "QXL_CMD_SURFACE stride = %d %% 4 != 0\n",
+                              cmd->u.surface_create.stride);
             return 1;
         }
         qemu_mutex_lock(&qxl->track_lock);
@@ -489,7 +530,7 @@ static void interface_get_init_info(QXLInstance *sin, QXLDevInitInfo *info)
     info->num_memslots_groups = NUM_MEMSLOTS_GROUPS;
     info->internal_groupslot_id = 0;
     info->qxl_ram_size = le32_to_cpu(qxl->shadow_rom.num_pages) << TARGET_PAGE_BITS;
-    info->n_surfaces = NUM_SURFACES;
+    info->n_surfaces = qxl->ssd.num_surfaces;
 }
 
 static const char *qxl_mode_to_string(int mode)
@@ -538,6 +579,7 @@ static const char *io_port_to_string(uint32_t io_port)
                                         = "QXL_IO_DESTROY_ALL_SURFACES_ASYNC",
         [QXL_IO_FLUSH_SURFACES_ASYNC]   = "QXL_IO_FLUSH_SURFACES_ASYNC",
         [QXL_IO_FLUSH_RELEASE]          = "QXL_IO_FLUSH_RELEASE",
+        [QXL_IO_MONITORS_CONFIG_ASYNC]  = "QXL_IO_MONITORS_CONFIG_ASYNC",
     };
     return io_port_to_string[io_port];
 }
@@ -557,9 +599,9 @@ static int interface_get_command(QXLInstance *sin, struct QXLCommandExt *ext)
     case QXL_MODE_VGA:
         ret = false;
         qemu_mutex_lock(&qxl->ssd.lock);
-        if (qxl->ssd.update != NULL) {
-            update = qxl->ssd.update;
-            qxl->ssd.update = NULL;
+        update = QTAILQ_FIRST(&qxl->ssd.updates);
+        if (update != NULL) {
+            QTAILQ_REMOVE(&qxl->ssd.updates, update, next);
             *ext = update->ext;
             ret = true;
         }
@@ -819,6 +861,7 @@ static void interface_async_complete_io(PCIQXLDevice *qxl, QXLCookie *cookie)
     case QXL_IO_DESTROY_PRIMARY_ASYNC:
     case QXL_IO_UPDATE_AREA_ASYNC:
     case QXL_IO_FLUSH_SURFACES_ASYNC:
+    case QXL_IO_MONITORS_CONFIG_ASYNC:
         break;
     case QXL_IO_CREATE_PRIMARY_ASYNC:
         qxl_create_guest_primary_complete(qxl);
@@ -894,6 +937,8 @@ static void interface_async_complete(QXLInstance *sin, uint64_t cookie_token)
     case QXL_COOKIE_TYPE_RENDER_UPDATE_AREA:
         qxl_render_update_area_done(qxl, cookie);
         break;
+    case QXL_COOKIE_TYPE_POST_LOAD_MONITORS_CONFIG:
+        break;
     default:
         fprintf(stderr, "qxl: %s: unexpected cookie type %d\n",
                 __func__, cookie->type);
@@ -901,6 +946,96 @@ static void interface_async_complete(QXLInstance *sin, uint64_t cookie_token)
     }
 }
 
+/* called from spice server thread context only */
+static void interface_set_client_capabilities(QXLInstance *sin,
+                                              uint8_t client_present,
+                                              uint8_t caps[58])
+{
+    PCIQXLDevice *qxl = container_of(sin, PCIQXLDevice, ssd.qxl);
+
+    if (runstate_check(RUN_STATE_INMIGRATE) ||
+        runstate_check(RUN_STATE_POSTMIGRATE)) {
+        return;
+    }
+
+    qxl->shadow_rom.client_present = client_present;
+    memcpy(qxl->shadow_rom.client_capabilities, caps, sizeof(caps));
+    qxl->rom->client_present = client_present;
+    memcpy(qxl->rom->client_capabilities, caps, sizeof(caps));
+    qxl_rom_set_dirty(qxl);
+
+    qxl_send_events(qxl, QXL_INTERRUPT_CLIENT);
+}
+
+static uint32_t qxl_crc32(const uint8_t *p, unsigned len)
+{
+    /*
+     * zlib xors the seed with 0xffffffff, and xors the result
+     * again with 0xffffffff; Both are not done with linux's crc32,
+     * which we want to be compatible with, so undo that.
+     */
+    return crc32(0xffffffff, p, len) ^ 0xffffffff;
+}
+
+/* called from main context only */
+static int interface_client_monitors_config(QXLInstance *sin,
+                                        VDAgentMonitorsConfig *monitors_config)
+{
+    PCIQXLDevice *qxl = container_of(sin, PCIQXLDevice, ssd.qxl);
+    QXLRom *rom = memory_region_get_ram_ptr(&qxl->rom_bar);
+    int i;
+
+    /*
+     * Older windows drivers set int_mask to 0 when their ISR is called,
+     * then later set it to ~0. So it doesn't relate to the actual interrupts
+     * handled. However, they are old, so clearly they don't support this
+     * interrupt
+     */
+    if (qxl->ram->int_mask == 0 || qxl->ram->int_mask == ~0 ||
+        !(qxl->ram->int_mask & QXL_INTERRUPT_CLIENT_MONITORS_CONFIG)) {
+        trace_qxl_client_monitors_config_unsupported_by_guest(qxl->id,
+                                                            qxl->ram->int_mask,
+                                                            monitors_config);
+        return 0;
+    }
+    if (!monitors_config) {
+        return 1;
+    }
+    memset(&rom->client_monitors_config, 0,
+           sizeof(rom->client_monitors_config));
+    rom->client_monitors_config.count = monitors_config->num_of_monitors;
+    /* monitors_config->flags ignored */
+    if (rom->client_monitors_config.count >=
+            ARRAY_SIZE(rom->client_monitors_config.heads)) {
+        trace_qxl_client_monitors_config_capped(qxl->id,
+                                monitors_config->num_of_monitors,
+                                ARRAY_SIZE(rom->client_monitors_config.heads));
+        rom->client_monitors_config.count =
+            ARRAY_SIZE(rom->client_monitors_config.heads);
+    }
+    for (i = 0 ; i < rom->client_monitors_config.count ; ++i) {
+        VDAgentMonConfig *monitor = &monitors_config->monitors[i];
+        QXLURect *rect = &rom->client_monitors_config.heads[i];
+        /* monitor->depth ignored */
+        rect->left = monitor->x;
+        rect->top = monitor->y;
+        rect->right = monitor->x + monitor->width;
+        rect->bottom = monitor->y + monitor->height;
+    }
+    rom->client_monitors_config_crc = qxl_crc32(
+            (const uint8_t *)&rom->client_monitors_config,
+            sizeof(rom->client_monitors_config));
+    trace_qxl_client_monitors_config_crc(qxl->id,
+            sizeof(rom->client_monitors_config),
+            rom->client_monitors_config_crc);
+
+    trace_qxl_interrupt_client_monitors_config(qxl->id,
+                        rom->client_monitors_config.count,
+                        rom->client_monitors_config.heads);
+    qxl_send_events(qxl, QXL_INTERRUPT_CLIENT_MONITORS_CONFIG);
+    return 1;
+}
+
 static const QXLInterface qxl_interface = {
     .base.type               = SPICE_INTERFACE_QXL,
     .base.description        = "qxl gpu",
@@ -922,6 +1057,8 @@ static const QXLInterface qxl_interface = {
     .flush_resources         = interface_flush_resources,
     .async_complete          = interface_async_complete,
     .update_area_complete    = interface_update_area_complete,
+    .set_client_capabilities = interface_set_client_capabilities,
+    .client_monitors_config = interface_client_monitors_config,
 };
 
 static void qxl_enter_vga_mode(PCIQXLDevice *d)
@@ -932,7 +1069,7 @@ static void qxl_enter_vga_mode(PCIQXLDevice *d)
     trace_qxl_enter_vga_mode(d->id);
     qemu_spice_create_host_primary(&d->ssd);
     d->mode = QXL_MODE_VGA;
-    memset(&d->ssd.dirty, 0, sizeof(d->ssd.dirty));
+    dpy_gfx_resize(d->ssd.ds);
     vga_dirty_log_start(&d->vga);
 }
 
@@ -958,9 +1095,10 @@ static void qxl_update_irq(PCIQXLDevice *d)
 static void qxl_check_state(PCIQXLDevice *d)
 {
     QXLRam *ram = d->ram;
+    int spice_display_running = qemu_spice_display_is_running(&d->ssd);
 
-    assert(!d->ssd.running || SPICE_RING_IS_EMPTY(&ram->cmd_ring));
-    assert(!d->ssd.running || SPICE_RING_IS_EMPTY(&ram->cursor_ring));
+    assert(!spice_display_running || SPICE_RING_IS_EMPTY(&ram->cmd_ring));
+    assert(!spice_display_running || SPICE_RING_IS_EMPTY(&ram->cursor_ring));
 }
 
 static void qxl_reset_state(PCIQXLDevice *d)
@@ -1229,6 +1367,12 @@ static void qxl_create_guest_primary(PCIQXLDevice *qxl, int loadvm,
     trace_qxl_create_guest_primary_rest(qxl->id, sc->stride, sc->type,
                                         sc->flags);
 
+    if ((surface.stride & 0x3) != 0) {
+        qxl_set_guest_bug(qxl, "primary surface stride = %d %% 4 != 0",
+                          surface.stride);
+        return;
+    }
+
     surface.mouse_mode = true;
     surface.group_id   = MEMSLOT_GROUP_GUEST;
     if (loadvm) {
@@ -1292,17 +1436,15 @@ static void qxl_set_mode(PCIQXLDevice *d, int modenr, int loadvm)
 
     d->mode = QXL_MODE_COMPAT;
     d->cmdflags = QXL_COMMAND_FLAG_COMPAT;
-#ifdef QXL_COMMAND_FLAG_COMPAT_16BPP /* new in spice 0.6.1 */
     if (mode->bits == 16) {
         d->cmdflags |= QXL_COMMAND_FLAG_COMPAT_16BPP;
     }
-#endif
     d->shadow_rom.mode = cpu_to_le32(modenr);
     d->rom->mode = cpu_to_le32(modenr);
     qxl_rom_set_dirty(d);
 }
 
-static void ioport_write(void *opaque, target_phys_addr_t addr,
+static void ioport_write(void *opaque, hwaddr addr,
                          uint64_t val, unsigned size)
 {
     PCIQXLDevice *d = opaque;
@@ -1310,7 +1452,14 @@ static void ioport_write(void *opaque, target_phys_addr_t addr,
     qxl_async_io async = QXL_SYNC;
     uint32_t orig_io_port = io_port;
 
-    if (d->guest_bug && !io_port == QXL_IO_RESET) {
+    if (d->guest_bug && io_port != QXL_IO_RESET) {
+        return;
+    }
+
+    if (d->revision <= QXL_REVISION_STABLE_V10 &&
+        io_port > QXL_IO_FLUSH_RELEASE) {
+        qxl_set_guest_bug(d, "unsupported io %d for revision %d\n",
+            io_port, d->revision);
         return;
     }
 
@@ -1330,10 +1479,10 @@ static void ioport_write(void *opaque, target_phys_addr_t addr,
             break;
         }
         trace_qxl_io_unexpected_vga_mode(d->id,
-            io_port, io_port_to_string(io_port));
+            addr, val, io_port_to_string(io_port));
         /* be nice to buggy guest drivers */
         if (io_port >= QXL_IO_UPDATE_AREA_ASYNC &&
-            io_port <= QXL_IO_DESTROY_ALL_SURFACES_ASYNC) {
+            io_port < QXL_IO_RANGE_SIZE) {
             qxl_send_events(d, QXL_INTERRUPT_IO_CMD);
         }
         return;
@@ -1361,6 +1510,7 @@ static void ioport_write(void *opaque, target_phys_addr_t addr,
         io_port = QXL_IO_DESTROY_ALL_SURFACES;
         goto async_common;
     case QXL_IO_FLUSH_SURFACES_ASYNC:
+    case QXL_IO_MONITORS_CONFIG_ASYNC:
 async_common:
         async = QXL_ASYNC;
         qemu_mutex_lock(&d->async_lock);
@@ -1385,6 +1535,18 @@ async_common:
         QXLCookie *cookie = NULL;
         QXLRect update = d->ram->update_area;
 
+        if (d->ram->update_surface > d->ssd.num_surfaces) {
+            qxl_set_guest_bug(d, "QXL_IO_UPDATE_AREA: invalid surface id %d\n",
+                              d->ram->update_surface);
+            break;
+        }
+        if (update.left >= update.right || update.top >= update.bottom ||
+            update.left < 0 || update.top < 0) {
+            qxl_set_guest_bug(d,
+                    "QXL_IO_UPDATE_AREA: invalid area (%ux%u)x(%ux%u)\n",
+                    update.left, update.top, update.right, update.bottom);
+            break;
+        }
         if (async == QXL_ASYNC) {
             cookie = qxl_cookie_new(QXL_COOKIE_TYPE_IO,
                                     QXL_IO_UPDATE_AREA_ASYNC);
@@ -1416,6 +1578,7 @@ async_common:
         qxl_set_mode(d, val, 0);
         break;
     case QXL_IO_LOG:
+        trace_qxl_io_log(d->id, d->ram->log_buf);
         if (d->guestdebug) {
             fprintf(stderr, "qxl/guest-%d: %" PRId64 ": %s", d->id,
                     qemu_get_clock_ns(vm_clock), d->ram->log_buf);
@@ -1466,7 +1629,7 @@ async_common:
         }
         break;
     case QXL_IO_DESTROY_SURFACE_WAIT:
-        if (val >= NUM_SURFACES) {
+        if (val >= d->ssd.num_surfaces) {
             qxl_set_guest_bug(d, "QXL_IO_DESTROY_SURFACE (async=%d):"
                              "%" PRIu64 " >= NUM_SURFACES", async, val);
             goto cancel_async;
@@ -1490,6 +1653,9 @@ async_common:
         d->mode = QXL_MODE_UNDEFINED;
         qxl_spice_destroy_surfaces(d, async);
         break;
+    case QXL_IO_MONITORS_CONFIG_ASYNC:
+        qxl_spice_monitors_config_async(d, 0);
+        break;
     default:
         qxl_set_guest_bug(d, "%s: unexpected ioport=0x%x\n", __func__, io_port);
     }
@@ -1503,12 +1669,12 @@ cancel_async:
     }
 }
 
-static uint64_t ioport_read(void *opaque, target_phys_addr_t addr,
+static uint64_t ioport_read(void *opaque, hwaddr addr,
                             unsigned size)
 {
-    PCIQXLDevice *d = opaque;
+    PCIQXLDevice *qxl = opaque;
 
-    trace_qxl_io_read_unexpected(d->id);
+    trace_qxl_io_read_unexpected(qxl->id);
     return 0xff;
 }
 
@@ -1538,7 +1704,14 @@ static void qxl_send_events(PCIQXLDevice *d, uint32_t events)
     uint32_t old_pending;
     uint32_t le_events = cpu_to_le32(events);
 
-    assert(d->ssd.running);
+    trace_qxl_send_events(d->id, events);
+    if (!qemu_spice_display_is_running(&d->ssd)) {
+        /* spice-server tracks guest running state and should not do this */
+        fprintf(stderr, "%s: spice-server bug: guest stopped, ignoring\n",
+                __func__);
+        trace_qxl_send_events_vm_stopped(d->id, events);
+        return;
+    }
     old_pending = __sync_fetch_and_or(&d->ram->int_pending, le_events);
     if ((old_pending & le_events) == le_events) {
         return;
@@ -1595,7 +1768,8 @@ static void qxl_hw_invalidate(void *opaque)
     vga->invalidate(vga);
 }
 
-static void qxl_hw_screen_dump(void *opaque, const char *filename, bool cswitch)
+static void qxl_hw_screen_dump(void *opaque, const char *filename, bool cswitch,
+                               Error **errp)
 {
     PCIQXLDevice *qxl = opaque;
     VGACommonState *vga = &qxl->vga;
@@ -1604,10 +1778,10 @@ static void qxl_hw_screen_dump(void *opaque, const char *filename, bool cswitch)
     case QXL_MODE_COMPAT:
     case QXL_MODE_NATIVE:
         qxl_render_update(qxl);
-        ppm_save(filename, qxl->ssd.ds->surface);
+        ppm_save(filename, qxl->ssd.ds->surface, errp);
         break;
     case QXL_MODE_VGA:
-        vga->screen_dump(vga, filename, cswitch);
+        vga->screen_dump(vga, filename, cswitch, errp);
         break;
     default:
         break;
@@ -1627,7 +1801,7 @@ static void qxl_hw_text_update(void *opaque, console_ch_t *chardata)
 
 static void qxl_dirty_surfaces(PCIQXLDevice *qxl)
 {
-    intptr_t vram_start;
+    uintptr_t vram_start;
     int i;
 
     if (qxl->mode != QXL_MODE_NATIVE && qxl->mode != QXL_MODE_COMPAT) {
@@ -1638,10 +1812,10 @@ static void qxl_dirty_surfaces(PCIQXLDevice *qxl)
     qxl_set_dirty(&qxl->vga.vram, qxl->shadow_rom.draw_area_offset,
                   qxl->shadow_rom.surface0_area_size);
 
-    vram_start =  (intptr_t)memory_region_get_ram_ptr(&qxl->vram_bar);
+    vram_start = (uintptr_t)memory_region_get_ram_ptr(&qxl->vram_bar);
 
     /* dirty the off-screen surfaces */
-    for (i = 0; i < NUM_SURFACES; i++) {
+    for (i = 0; i < qxl->ssd.num_surfaces; i++) {
         QXLSurfaceCmd *cmd;
         intptr_t surface_offset;
         int surface_size;
@@ -1670,7 +1844,6 @@ static void qxl_vm_change_state_handler(void *opaque, int running,
                                         RunState state)
 {
     PCIQXLDevice *qxl = opaque;
-    qemu_spice_vm_change_state_handler(&qxl->ssd, running, state);
 
     if (running) {
         /*
@@ -1713,8 +1886,8 @@ static void display_refresh(struct DisplayState *ds)
 }
 
 static DisplayChangeListener display_listener = {
-    .dpy_update  = display_update,
-    .dpy_resize  = display_resize,
+    .dpy_gfx_update  = display_update,
+    .dpy_gfx_resize  = display_resize,
     .dpy_refresh = display_refresh,
 };
 
@@ -1769,7 +1942,6 @@ static int qxl_init_common(PCIQXLDevice *qxl)
     qxl->mode = QXL_MODE_UNDEFINED;
     qxl->generation = 1;
     qxl->num_memslots = NUM_MEMSLOTS;
-    qxl->num_surfaces = NUM_SURFACES;
     qemu_mutex_init(&qxl->track_lock);
     qemu_mutex_init(&qxl->async_lock);
     qxl->current_async = QXL_UNDEFINED_IO;
@@ -1785,10 +1957,17 @@ static int qxl_init_common(PCIQXLDevice *qxl)
         io_size = 16;
         break;
     case 3: /* qxl-3 */
-    default:
-        pci_device_rev = QXL_DEFAULT_REVISION;
+        pci_device_rev = QXL_REVISION_STABLE_V10;
+        io_size = 32; /* PCI region size must be pow2 */
+        break;
+    case 4: /* qxl-4 */
+        pci_device_rev = QXL_REVISION_STABLE_V12;
         io_size = msb_mask(QXL_IO_RANGE_SIZE * 2 - 1);
         break;
+    default:
+        error_report("Invalid revision %d for qxl device (max %d)",
+                     qxl->revision, QXL_DEFAULT_REVISION);
+        return -1;
     }
 
     pci_set_byte(&config[PCI_REVISION_ID], pci_device_rev);
@@ -1800,6 +1979,7 @@ static int qxl_init_common(PCIQXLDevice *qxl)
     init_qxl_rom(qxl);
     init_qxl_ram(qxl);
 
+    qxl->guest_surfaces.cmds = g_new0(QXLPHYSICAL, qxl->ssd.num_surfaces);
     memory_region_init_ram(&qxl->vram_bar, "qxl.vram", qxl->vram_size);
     vmstate_register_ram(&qxl->vram_bar, &qxl->pci.qdev);
     memory_region_init_alias(&qxl->vram32_bar, "qxl.vram32", &qxl->vram_bar,
@@ -1810,6 +1990,7 @@ static int qxl_init_common(PCIQXLDevice *qxl)
     if (qxl->id == 0) {
         vga_dirty_log_start(&qxl->vga);
     }
+    memory_region_set_flush_coalesced(&qxl->io_bar);
 
 
     pci_register_bar(&qxl->pci, QXL_IO_RANGE_INDEX,
@@ -1848,7 +2029,11 @@ static int qxl_init_common(PCIQXLDevice *qxl)
 
     qxl->ssd.qxl.base.sif = &qxl_interface.base;
     qxl->ssd.qxl.id = qxl->id;
-    qemu_spice_add_interface(&qxl->ssd.qxl.base);
+    if (qemu_spice_add_interface(&qxl->ssd.qxl.base) != 0) {
+        error_report("qxl interface %d.%d not supported by spice-server\n",
+                     SPICE_INTERFACE_QXL_MAJOR, SPICE_INTERFACE_QXL_MINOR);
+        return -1;
+    }
     qemu_add_vm_change_state_handler(qxl_vm_change_state_handler, qxl);
 
     init_pipe_signaling(qxl);
@@ -1864,6 +2049,7 @@ static int qxl_init_primary(PCIDevice *dev)
     PCIQXLDevice *qxl = DO_UPCAST(PCIQXLDevice, pci, dev);
     VGACommonState *vga = &qxl->vga;
     PortioList *qxl_vga_port_list = g_new(PortioList, 1);
+    int rc;
 
     qxl->id = 0;
     qxl_init_ramsize(qxl);
@@ -1878,9 +2064,14 @@ static int qxl_init_primary(PCIDevice *dev)
     qemu_spice_display_init_common(&qxl->ssd, vga->ds);
 
     qxl0 = qxl;
-    register_displaychangelistener(vga->ds, &display_listener);
 
-    return qxl_init_common(qxl);
+    rc = qxl_init_common(qxl);
+    if (rc != 0) {
+        return rc;
+    }
+
+    register_displaychangelistener(vga->ds, &display_listener);
+    return rc;
 }
 
 static int qxl_init_secondary(PCIDevice *dev)
@@ -1955,6 +2146,7 @@ static int qxl_post_load(void *opaque, int version)
 
     switch (newmode) {
     case QXL_MODE_UNDEFINED:
+        qxl_create_memslots(d);
         break;
     case QXL_MODE_VGA:
         qxl_create_memslots(d);
@@ -1965,8 +2157,8 @@ static int qxl_post_load(void *opaque, int version)
         qxl_create_guest_primary(d, 1, QXL_SYNC);
 
         /* replay surface-create and cursor-set commands */
-        cmds = g_malloc0(sizeof(QXLCommandExt) * (NUM_SURFACES + 1));
-        for (in = 0, out = 0; in < NUM_SURFACES; in++) {
+        cmds = g_malloc0(sizeof(QXLCommandExt) * (d->ssd.num_surfaces + 1));
+        for (in = 0, out = 0; in < d->ssd.num_surfaces; in++) {
             if (d->guest_surfaces.cmds[in] == 0) {
                 continue;
             }
@@ -1983,7 +2175,9 @@ static int qxl_post_load(void *opaque, int version)
         }
         qxl_spice_loadvm_commands(d, cmds, out);
         g_free(cmds);
-
+        if (d->guest_monitors_config) {
+            qxl_spice_monitors_config_async(d, 1);
+        }
         break;
     case QXL_MODE_COMPAT:
         /* note: no need to call qxl_create_memslots, qxl_set_mode
@@ -1996,6 +2190,14 @@ static int qxl_post_load(void *opaque, int version)
 
 #define QXL_SAVE_VERSION 21
 
+static bool qxl_monitors_config_needed(void *opaque)
+{
+    PCIQXLDevice *qxl = opaque;
+
+    return qxl->guest_monitors_config != 0;
+}
+
+
 static VMStateDescription qxl_memslot = {
     .name               = "qxl-memslot",
     .version_id         = QXL_SAVE_VERSION,
@@ -2026,6 +2228,16 @@ static VMStateDescription qxl_surface = {
     }
 };
 
+static VMStateDescription qxl_vmstate_monitors_config = {
+    .name               = "qxl/monitors-config",
+    .version_id         = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT64(guest_monitors_config, PCIQXLDevice),
+        VMSTATE_END_OF_LIST()
+    },
+};
+
 static VMStateDescription qxl_vmstate = {
     .name               = "qxl",
     .version_id         = QXL_SAVE_VERSION,
@@ -2033,7 +2245,7 @@ static VMStateDescription qxl_vmstate = {
     .pre_save           = qxl_pre_save,
     .pre_load           = qxl_pre_load,
     .post_load          = qxl_post_load,
-    .fields = (VMStateField []) {
+    .fields = (VMStateField[]) {
         VMSTATE_PCI_DEVICE(pci, PCIQXLDevice),
         VMSTATE_STRUCT(vga, PCIQXLDevice, 0, vmstate_vga_common, VGACommonState),
         VMSTATE_UINT32(shadow_rom.mode, PCIQXLDevice),
@@ -2046,12 +2258,21 @@ static VMStateDescription qxl_vmstate = {
                              qxl_memslot, struct guest_slots),
         VMSTATE_STRUCT(guest_primary.surface, PCIQXLDevice, 0,
                        qxl_surface, QXLSurfaceCreate),
-        VMSTATE_INT32_EQUAL(num_surfaces, PCIQXLDevice),
-        VMSTATE_ARRAY(guest_surfaces.cmds, PCIQXLDevice, NUM_SURFACES, 0,
-                      vmstate_info_uint64, uint64_t),
+        VMSTATE_INT32_EQUAL(ssd.num_surfaces, PCIQXLDevice),
+        VMSTATE_VARRAY_INT32(guest_surfaces.cmds, PCIQXLDevice,
+                             ssd.num_surfaces, 0,
+                             vmstate_info_uint64, uint64_t),
         VMSTATE_UINT64(guest_cursor, PCIQXLDevice),
         VMSTATE_END_OF_LIST()
     },
+    .subsections = (VMStateSubsection[]) {
+        {
+            .vmsd = &qxl_vmstate_monitors_config,
+            .needed = qxl_monitors_config_needed,
+        }, {
+            /* empty */
+        }
+    }
 };
 
 static Property qxl_properties[] = {
@@ -2068,6 +2289,7 @@ static Property qxl_properties[] = {
         DEFINE_PROP_UINT32("vram_size_mb", PCIQXLDevice, vram32_size_mb, -1),
         DEFINE_PROP_UINT32("vram64_size_mb", PCIQXLDevice, vram_size_mb, -1),
         DEFINE_PROP_UINT32("vgamem_mb", PCIQXLDevice, vgamem_size_mb, 16),
+        DEFINE_PROP_INT32("surfaces", PCIQXLDevice, ssd.num_surfaces, 1024),
         DEFINE_PROP_END_OF_LIST(),
 };
 
diff --git a/hw/qxl.h b/hw/qxl.h
index 172baf6cc..e583cfb75 100644
--- a/hw/qxl.h
+++ b/hw/qxl.h
@@ -40,7 +40,6 @@ typedef struct PCIQXLDevice {
     uint32_t           revision;
 
     int32_t            num_memslots;
-    int32_t            num_surfaces;
 
     uint32_t           current_async;
     QemuMutex          async_lock;
@@ -65,12 +64,14 @@ typedef struct PCIQXLDevice {
     } guest_primary;
 
     struct surfaces {
-        QXLPHYSICAL    cmds[NUM_SURFACES];
+        QXLPHYSICAL    *cmds;
         uint32_t       count;
         uint32_t       max;
     } guest_surfaces;
     QXLPHYSICAL        guest_cursor;
 
+    QXLPHYSICAL        guest_monitors_config;
+
     QemuMutex          track_lock;
 
     /* thread signaling */
@@ -128,7 +129,7 @@ typedef struct PCIQXLDevice {
         }                                                               \
     } while (0)
 
-#define QXL_DEFAULT_REVISION QXL_REVISION_STABLE_V10
+#define QXL_DEFAULT_REVISION QXL_REVISION_STABLE_V12
 
 /* qxl.c */
 void *qxl_phys2virt(PCIQXLDevice *qxl, QXLPHYSICAL phys, int group_id);
diff --git a/hw/r2d.c b/hw/r2d.c
index 0f16e81af..66212e9cd 100644
--- a/hw/r2d.c
+++ b/hw/r2d.c
@@ -127,7 +127,7 @@ static void r2d_fpga_irq_set(void *opaque, int n, int level)
     update_irl(fpga);
 }
 
-static uint32_t r2d_fpga_read(void *opaque, target_phys_addr_t addr)
+static uint32_t r2d_fpga_read(void *opaque, hwaddr addr)
 {
     r2d_fpga_t *s = opaque;
 
@@ -146,7 +146,7 @@ static uint32_t r2d_fpga_read(void *opaque, target_phys_addr_t addr)
 }
 
 static void
-r2d_fpga_write(void *opaque, target_phys_addr_t addr, uint32_t value)
+r2d_fpga_write(void *opaque, hwaddr addr, uint32_t value)
 {
     r2d_fpga_t *s = opaque;
 
@@ -178,7 +178,7 @@ static const MemoryRegionOps r2d_fpga_ops = {
 };
 
 static qemu_irq *r2d_fpga_init(MemoryRegion *sysmem,
-                               target_phys_addr_t base, qemu_irq irl)
+                               hwaddr base, qemu_irq irl)
 {
     r2d_fpga_t *s;
 
@@ -219,11 +219,12 @@ static struct QEMU_PACKED
     char kernel_cmdline[256];
 } boot_params;
 
-static void r2d_init(ram_addr_t ram_size,
-              const char *boot_device,
-	      const char *kernel_filename, const char *kernel_cmdline,
-	      const char *initrd_filename, const char *cpu_model)
+static void r2d_init(QEMUMachineInitArgs *args)
 {
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
     SuperHCPU *cpu;
     CPUSH4State *env;
     ResetData *reset_info;
@@ -332,6 +333,8 @@ static void r2d_init(ram_addr_t ram_size,
     }
 
     if (kernel_cmdline) {
+        /* I see no evidence that this .kernel_cmdline buffer requires
+           NUL-termination, so using strncpy should be ok. */
         strncpy(boot_params.kernel_cmdline, kernel_cmdline,
                 sizeof(boot_params.kernel_cmdline));
     }
diff --git a/hw/rc4030.c b/hw/rc4030.c
index 9f39b3050..e0024c87e 100644
--- a/hw/rc4030.c
+++ b/hw/rc4030.c
@@ -112,7 +112,7 @@ static void set_next_tick(rc4030State *s)
 }
 
 /* called for accesses to rc4030 */
-static uint32_t rc4030_readl(void *opaque, target_phys_addr_t addr)
+static uint32_t rc4030_readl(void *opaque, hwaddr addr)
 {
     rc4030State *s = opaque;
     uint32_t val;
@@ -250,7 +250,7 @@ static uint32_t rc4030_readl(void *opaque, target_phys_addr_t addr)
     return val;
 }
 
-static uint32_t rc4030_readw(void *opaque, target_phys_addr_t addr)
+static uint32_t rc4030_readw(void *opaque, hwaddr addr)
 {
     uint32_t v = rc4030_readl(opaque, addr & ~0x3);
     if (addr & 0x2)
@@ -259,13 +259,13 @@ static uint32_t rc4030_readw(void *opaque, target_phys_addr_t addr)
         return v & 0xffff;
 }
 
-static uint32_t rc4030_readb(void *opaque, target_phys_addr_t addr)
+static uint32_t rc4030_readb(void *opaque, hwaddr addr)
 {
     uint32_t v = rc4030_readl(opaque, addr & ~0x3);
     return (v >> (8 * (addr & 0x3))) & 0xff;
 }
 
-static void rc4030_writel(void *opaque, target_phys_addr_t addr, uint32_t val)
+static void rc4030_writel(void *opaque, hwaddr addr, uint32_t val)
 {
     rc4030State *s = opaque;
     addr &= 0x3fff;
@@ -308,7 +308,7 @@ static void rc4030_writel(void *opaque, target_phys_addr_t addr, uint32_t val)
     case 0x0060:
         /* HACK */
         if (s->cache_ltag == 0x80000001 && s->cache_bmask == 0xf0f0f0f) {
-            target_phys_addr_t dest = s->cache_ptag & ~0x1;
+            hwaddr dest = s->cache_ptag & ~0x1;
             dest += (s->cache_maint & 0x3) << 3;
             cpu_physical_memory_write(dest, &val, 4);
         }
@@ -390,7 +390,7 @@ static void rc4030_writel(void *opaque, target_phys_addr_t addr, uint32_t val)
     }
 }
 
-static void rc4030_writew(void *opaque, target_phys_addr_t addr, uint32_t val)
+static void rc4030_writew(void *opaque, hwaddr addr, uint32_t val)
 {
     uint32_t old_val = rc4030_readl(opaque, addr & ~0x3);
 
@@ -401,7 +401,7 @@ static void rc4030_writew(void *opaque, target_phys_addr_t addr, uint32_t val)
     rc4030_writel(opaque, addr & ~0x3, val);
 }
 
-static void rc4030_writeb(void *opaque, target_phys_addr_t addr, uint32_t val)
+static void rc4030_writeb(void *opaque, hwaddr addr, uint32_t val)
 {
     uint32_t old_val = rc4030_readl(opaque, addr & ~0x3);
 
@@ -479,7 +479,7 @@ static void rc4030_periodic_timer(void *opaque)
     qemu_irq_raise(s->timer_irq);
 }
 
-static uint32_t jazzio_readw(void *opaque, target_phys_addr_t addr)
+static uint32_t jazzio_readw(void *opaque, hwaddr addr)
 {
     rc4030State *s = opaque;
     uint32_t val;
@@ -517,14 +517,14 @@ static uint32_t jazzio_readw(void *opaque, target_phys_addr_t addr)
     return val;
 }
 
-static uint32_t jazzio_readb(void *opaque, target_phys_addr_t addr)
+static uint32_t jazzio_readb(void *opaque, hwaddr addr)
 {
     uint32_t v;
     v = jazzio_readw(opaque, addr & ~0x1);
     return (v >> (8 * (addr & 0x1))) & 0xff;
 }
 
-static uint32_t jazzio_readl(void *opaque, target_phys_addr_t addr)
+static uint32_t jazzio_readl(void *opaque, hwaddr addr)
 {
     uint32_t v;
     v = jazzio_readw(opaque, addr);
@@ -532,7 +532,7 @@ static uint32_t jazzio_readl(void *opaque, target_phys_addr_t addr)
     return v;
 }
 
-static void jazzio_writew(void *opaque, target_phys_addr_t addr, uint32_t val)
+static void jazzio_writew(void *opaque, hwaddr addr, uint32_t val)
 {
     rc4030State *s = opaque;
     addr &= 0xfff;
@@ -551,7 +551,7 @@ static void jazzio_writew(void *opaque, target_phys_addr_t addr, uint32_t val)
     }
 }
 
-static void jazzio_writeb(void *opaque, target_phys_addr_t addr, uint32_t val)
+static void jazzio_writeb(void *opaque, hwaddr addr, uint32_t val)
 {
     uint32_t old_val = jazzio_readw(opaque, addr & ~0x1);
 
@@ -566,7 +566,7 @@ static void jazzio_writeb(void *opaque, target_phys_addr_t addr, uint32_t val)
     jazzio_writew(opaque, addr & ~0x1, val);
 }
 
-static void jazzio_writel(void *opaque, target_phys_addr_t addr, uint32_t val)
+static void jazzio_writel(void *opaque, hwaddr addr, uint32_t val)
 {
     jazzio_writew(opaque, addr, val & 0xffff);
     jazzio_writew(opaque, addr + 2, (val >> 16) & 0xffff);
@@ -672,11 +672,11 @@ static void rc4030_save(QEMUFile *f, void *opaque)
     qemu_put_be32(f, s->itr);
 }
 
-void rc4030_dma_memory_rw(void *opaque, target_phys_addr_t addr, uint8_t *buf, int len, int is_write)
+void rc4030_dma_memory_rw(void *opaque, hwaddr addr, uint8_t *buf, int len, int is_write)
 {
     rc4030State *s = opaque;
-    target_phys_addr_t entry_addr;
-    target_phys_addr_t phys_addr;
+    hwaddr entry_addr;
+    hwaddr phys_addr;
     dma_pagetable_entry entry;
     int index;
     int ncpy, i;
@@ -713,7 +713,7 @@ void rc4030_dma_memory_rw(void *opaque, target_phys_addr_t addr, uint8_t *buf, i
 static void rc4030_do_dma(void *opaque, int n, uint8_t *buf, int len, int is_write)
 {
     rc4030State *s = opaque;
-    target_phys_addr_t dma_addr;
+    hwaddr dma_addr;
     int dev_to_mem;
 
     s->dma_regs[n][DMA_REG_ENABLE] &= ~(DMA_FLAG_TC_INTR | DMA_FLAG_MEM_INTR | DMA_FLAG_ADDR_INTR);
diff --git a/hw/realview.c b/hw/realview.c
index 19db4d026..e789c159a 100644
--- a/hw/realview.c
+++ b/hw/realview.c
@@ -44,11 +44,8 @@ static const int realview_board_id[] = {
     0x76d
 };
 
-static void realview_init(ram_addr_t ram_size,
-                     const char *boot_device,
-                     const char *kernel_filename, const char *kernel_cmdline,
-                     const char *initrd_filename, const char *cpu_model,
-                     enum realview_board_type board_type)
+static void realview_init(QEMUMachineInitArgs *args,
+                          enum realview_board_type board_type)
 {
     ARMCPU *cpu = NULL;
     CPUARMState *env;
@@ -73,6 +70,7 @@ static void realview_init(ram_addr_t ram_size,
     uint32_t proc_id = 0;
     uint32_t sys_id;
     ram_addr_t low_ram_size;
+    ram_addr_t ram_size = args->ram_size;
 
     switch (board_type) {
     case BOARD_EB:
@@ -89,7 +87,7 @@ static void realview_init(ram_addr_t ram_size,
         break;
     }
     for (n = 0; n < smp_cpus; n++) {
-        cpu = cpu_arm_init(cpu_model);
+        cpu = cpu_arm_init(args->cpu_model);
         if (!cpu) {
             fprintf(stderr, "Unable to find CPU definition\n");
             exit(1);
@@ -145,7 +143,7 @@ static void realview_init(ram_addr_t ram_size,
     sysbus_mmio_map(sysbus_from_qdev(sysctl), 0, 0x10000000);
 
     if (is_mpcore) {
-        target_phys_addr_t periphbase;
+        hwaddr periphbase;
         dev = qdev_create(NULL, is_pb ? "a9mpcore_priv": "realview_mpcore");
         qdev_prop_set_uint32(dev, "num-cpu", smp_cpus);
         qdev_init_nofail(dev);
@@ -227,7 +225,7 @@ static void realview_init(ram_addr_t ram_size,
         sysbus_connect_irq(busdev, 2, pic[50]);
         sysbus_connect_irq(busdev, 3, pic[51]);
         pci_bus = (PCIBus *)qdev_get_child_bus(dev, "pci");
-        if (usb_enabled) {
+        if (usb_enabled(false)) {
             pci_create_simple(pci_bus, -1, "pci-ohci");
         }
         n = drive_get_max_bus(IF_SCSI);
@@ -321,61 +319,45 @@ static void realview_init(ram_addr_t ram_size,
     memory_region_add_subregion(sysmem, SMP_BOOT_ADDR, ram_hack);
 
     realview_binfo.ram_size = ram_size;
-    realview_binfo.kernel_filename = kernel_filename;
-    realview_binfo.kernel_cmdline = kernel_cmdline;
-    realview_binfo.initrd_filename = initrd_filename;
+    realview_binfo.kernel_filename = args->kernel_filename;
+    realview_binfo.kernel_cmdline = args->kernel_cmdline;
+    realview_binfo.initrd_filename = args->initrd_filename;
     realview_binfo.nb_cpus = smp_cpus;
     realview_binfo.board_id = realview_board_id[board_type];
     realview_binfo.loader_start = (board_type == BOARD_PB_A8 ? 0x70000000 : 0);
     arm_load_kernel(arm_env_get_cpu(first_cpu), &realview_binfo);
 }
 
-static void realview_eb_init(ram_addr_t ram_size,
-                     const char *boot_device,
-                     const char *kernel_filename, const char *kernel_cmdline,
-                     const char *initrd_filename, const char *cpu_model)
+static void realview_eb_init(QEMUMachineInitArgs *args)
 {
-    if (!cpu_model) {
-        cpu_model = "arm926";
+    if (!args->cpu_model) {
+        args->cpu_model = "arm926";
     }
-    realview_init(ram_size, boot_device, kernel_filename, kernel_cmdline,
-                  initrd_filename, cpu_model, BOARD_EB);
+    realview_init(args, BOARD_EB);
 }
 
-static void realview_eb_mpcore_init(ram_addr_t ram_size,
-                     const char *boot_device,
-                     const char *kernel_filename, const char *kernel_cmdline,
-                     const char *initrd_filename, const char *cpu_model)
+static void realview_eb_mpcore_init(QEMUMachineInitArgs *args)
 {
-    if (!cpu_model) {
-        cpu_model = "arm11mpcore";
+    if (!args->cpu_model) {
+        args->cpu_model = "arm11mpcore";
     }
-    realview_init(ram_size, boot_device, kernel_filename, kernel_cmdline,
-                  initrd_filename, cpu_model, BOARD_EB_MPCORE);
+    realview_init(args, BOARD_EB_MPCORE);
 }
 
-static void realview_pb_a8_init(ram_addr_t ram_size,
-                     const char *boot_device,
-                     const char *kernel_filename, const char *kernel_cmdline,
-                     const char *initrd_filename, const char *cpu_model)
+static void realview_pb_a8_init(QEMUMachineInitArgs *args)
 {
-    if (!cpu_model) {
-        cpu_model = "cortex-a8";
+    if (!args->cpu_model) {
+        args->cpu_model = "cortex-a8";
     }
-    realview_init(ram_size, boot_device, kernel_filename, kernel_cmdline,
-                  initrd_filename, cpu_model, BOARD_PB_A8);
+    realview_init(args, BOARD_PB_A8);
 }
 
-static void realview_pbx_a9_init(ram_addr_t ram_size,
-                     const char *boot_device,
-                     const char *kernel_filename, const char *kernel_cmdline,
-                     const char *initrd_filename, const char *cpu_model)
+static void realview_pbx_a9_init(QEMUMachineInitArgs *args)
 {
-    if (!cpu_model) {
-        cpu_model = "cortex-a9";
+    if (!args->cpu_model) {
+        args->cpu_model = "cortex-a9";
     }
-    realview_init(ram_size, boot_device, kernel_filename, kernel_cmdline,
-                  initrd_filename, cpu_model, BOARD_PBX_A9);
+    realview_init(args, BOARD_PBX_A9);
 }
 
 static QEMUMachine realview_eb_machine = {
diff --git a/hw/rtl8139.c b/hw/rtl8139.c
index 844f1b8c3..e3aa8bfb1 100644
--- a/hw/rtl8139.c
+++ b/hw/rtl8139.c
@@ -167,7 +167,7 @@ enum IntrStatusBits {
     PCIErr = 0x8000,
     PCSTimeout = 0x4000,
     RxFIFOOver = 0x40,
-    RxUnderrun = 0x20,
+    RxUnderrun = 0x20, /* Packet Underrun / Link Change */
     RxOverflow = 0x10,
     TxErr = 0x08,
     TxOK = 0x04,
@@ -774,11 +774,7 @@ static void rtl8139_write_buffer(RTL8139State *s, const void *buf, int size)
 #define MIN_BUF_SIZE 60
 static inline dma_addr_t rtl8139_addr64(uint32_t low, uint32_t high)
 {
-#if TARGET_PHYS_ADDR_BITS > 32
-    return low | ((target_phys_addr_t)high << 32);
-#else
-    return low;
-#endif
+    return low | ((uint64_t)high << 32);
 }
 
 /* Workaround for buggy guest driver such as linux who allocates rx
@@ -2459,7 +2455,7 @@ static void rtl8139_TxStatus_write(RTL8139State *s, uint32_t txRegOffset, uint32
 
         if (descriptor == 0 && (val & 0x8))
         {
-            target_phys_addr_t tc_addr = rtl8139_addr64(s->TxStatus[0] & ~0x3f, s->TxStatus[1]);
+            hwaddr tc_addr = rtl8139_addr64(s->TxStatus[0] & ~0x3f, s->TxStatus[1]);
 
             /* dump tally counters to specified memory location */
             RTL8139TallyCounters_dma_write(s, tc_addr);
@@ -3007,7 +3003,8 @@ static uint32_t rtl8139_io_readb(void *opaque, uint8_t addr)
             break;
 
         case MediaStatus:
-            ret = 0xd0;
+            /* The LinkDown bit of MediaStatus is inverse with link status */
+            ret = 0xd0 | (~s->BasicModeStatus & 0x04);
             DPRINTF("MediaStatus read 0x%x\n", ret);
             break;
 
@@ -3190,65 +3187,33 @@ static uint32_t rtl8139_io_readl(void *opaque, uint8_t addr)
 
 /* */
 
-static void rtl8139_ioport_writeb(void *opaque, uint32_t addr, uint32_t val)
+static void rtl8139_mmio_writeb(void *opaque, hwaddr addr, uint32_t val)
 {
     rtl8139_io_writeb(opaque, addr & 0xFF, val);
 }
 
-static void rtl8139_ioport_writew(void *opaque, uint32_t addr, uint32_t val)
+static void rtl8139_mmio_writew(void *opaque, hwaddr addr, uint32_t val)
 {
     rtl8139_io_writew(opaque, addr & 0xFF, val);
 }
 
-static void rtl8139_ioport_writel(void *opaque, uint32_t addr, uint32_t val)
+static void rtl8139_mmio_writel(void *opaque, hwaddr addr, uint32_t val)
 {
     rtl8139_io_writel(opaque, addr & 0xFF, val);
 }
 
-static uint32_t rtl8139_ioport_readb(void *opaque, uint32_t addr)
+static uint32_t rtl8139_mmio_readb(void *opaque, hwaddr addr)
 {
     return rtl8139_io_readb(opaque, addr & 0xFF);
 }
 
-static uint32_t rtl8139_ioport_readw(void *opaque, uint32_t addr)
-{
-    return rtl8139_io_readw(opaque, addr & 0xFF);
-}
-
-static uint32_t rtl8139_ioport_readl(void *opaque, uint32_t addr)
-{
-    return rtl8139_io_readl(opaque, addr & 0xFF);
-}
-
-/* */
-
-static void rtl8139_mmio_writeb(void *opaque, target_phys_addr_t addr, uint32_t val)
-{
-    rtl8139_io_writeb(opaque, addr & 0xFF, val);
-}
-
-static void rtl8139_mmio_writew(void *opaque, target_phys_addr_t addr, uint32_t val)
-{
-    rtl8139_io_writew(opaque, addr & 0xFF, val);
-}
-
-static void rtl8139_mmio_writel(void *opaque, target_phys_addr_t addr, uint32_t val)
-{
-    rtl8139_io_writel(opaque, addr & 0xFF, val);
-}
-
-static uint32_t rtl8139_mmio_readb(void *opaque, target_phys_addr_t addr)
-{
-    return rtl8139_io_readb(opaque, addr & 0xFF);
-}
-
-static uint32_t rtl8139_mmio_readw(void *opaque, target_phys_addr_t addr)
+static uint32_t rtl8139_mmio_readw(void *opaque, hwaddr addr)
 {
     uint32_t val = rtl8139_io_readw(opaque, addr & 0xFF);
     return val;
 }
 
-static uint32_t rtl8139_mmio_readl(void *opaque, target_phys_addr_t addr)
+static uint32_t rtl8139_mmio_readl(void *opaque, hwaddr addr)
 {
     uint32_t val = rtl8139_io_readl(opaque, addr & 0xFF);
     return val;
@@ -3262,6 +3227,10 @@ static int rtl8139_post_load(void *opaque, int version_id)
         s->cplus_enabled = s->CpCmd != 0;
     }
 
+    /* nc.link_down can't be migrated, so infer link_down according
+     * to link status bit in BasicModeStatus */
+    s->nic->nc.link_down = (s->BasicModeStatus & 0x04) == 0;
+
     return 0;
 }
 
@@ -3385,18 +3354,44 @@ static const VMStateDescription vmstate_rtl8139 = {
 /***********************************************************/
 /* PCI RTL8139 definitions */
 
-static const MemoryRegionPortio rtl8139_portio[] = {
-    { 0, 0x100, 1, .read = rtl8139_ioport_readb, },
-    { 0, 0x100, 1, .write = rtl8139_ioport_writeb, },
-    { 0, 0x100, 2, .read = rtl8139_ioport_readw, },
-    { 0, 0x100, 2, .write = rtl8139_ioport_writew, },
-    { 0, 0x100, 4, .read = rtl8139_ioport_readl, },
-    { 0, 0x100, 4, .write = rtl8139_ioport_writel, },
-    PORTIO_END_OF_LIST()
-};
+static void rtl8139_ioport_write(void *opaque, hwaddr addr,
+                                 uint64_t val, unsigned size)
+{
+    switch (size) {
+    case 1:
+        rtl8139_io_writeb(opaque, addr, val);
+        break;
+    case 2:
+        rtl8139_io_writew(opaque, addr, val);
+        break;
+    case 4:
+        rtl8139_io_writel(opaque, addr, val);
+        break;
+    }
+}
+
+static uint64_t rtl8139_ioport_read(void *opaque, hwaddr addr,
+                                    unsigned size)
+{
+    switch (size) {
+    case 1:
+        return rtl8139_io_readb(opaque, addr);
+    case 2:
+        return rtl8139_io_readw(opaque, addr);
+    case 4:
+        return rtl8139_io_readl(opaque, addr);
+    }
+
+    return -1;
+}
 
 static const MemoryRegionOps rtl8139_io_ops = {
-    .old_portio = rtl8139_portio,
+    .read = rtl8139_ioport_read,
+    .write = rtl8139_ioport_write,
+    .impl = {
+        .min_access_size = 1,
+        .max_access_size = 4,
+    },
     .endianness = DEVICE_LITTLE_ENDIAN,
 };
 
@@ -3453,12 +3448,27 @@ static void pci_rtl8139_uninit(PCIDevice *dev)
     qemu_del_net_client(&s->nic->nc);
 }
 
+static void rtl8139_set_link_status(NetClientState *nc)
+{
+    RTL8139State *s = DO_UPCAST(NICState, nc, nc)->opaque;
+
+    if (nc->link_down) {
+        s->BasicModeStatus &= ~0x04;
+    } else {
+        s->BasicModeStatus |= 0x04;
+    }
+
+    s->IntrStatus |= RxUnderrun;
+    rtl8139_update_irq(s);
+}
+
 static NetClientInfo net_rtl8139_info = {
     .type = NET_CLIENT_OPTIONS_KIND_NIC,
     .size = sizeof(NICState),
     .can_receive = rtl8139_can_receive,
     .receive = rtl8139_receive,
     .cleanup = rtl8139_cleanup,
+    .link_status_changed = rtl8139_set_link_status,
 };
 
 static int pci_rtl8139_init(PCIDevice *dev)
diff --git a/hw/s390-virtio-bus.c b/hw/s390-virtio-bus.c
index a24568469..e0ac2d1ec 100644
--- a/hw/s390-virtio-bus.c
+++ b/hw/s390-virtio-bus.c
@@ -26,6 +26,7 @@
 #include "loader.h"
 #include "elf.h"
 #include "hw/virtio.h"
+#include "hw/virtio-rng.h"
 #include "hw/virtio-serial.h"
 #include "hw/virtio-net.h"
 #include "hw/sysbus.h"
@@ -56,7 +57,7 @@ static const VirtIOBindings virtio_s390_bindings;
 static ram_addr_t s390_virtio_device_num_vq(VirtIOS390Device *dev);
 
 /* length of VirtIO device pages */
-const target_phys_addr_t virtio_size = S390_DEVICE_PAGES * TARGET_PAGE_SIZE;
+const hwaddr virtio_size = S390_DEVICE_PAGES * TARGET_PAGE_SIZE;
 
 static void s390_virtio_bus_reset(void *opaque)
 {
@@ -67,7 +68,7 @@ static void s390_virtio_bus_reset(void *opaque)
 void s390_virtio_reset_idx(VirtIOS390Device *dev)
 {
     int i;
-    target_phys_addr_t idx_addr;
+    hwaddr idx_addr;
     uint8_t num_vq;
 
     num_vq = s390_virtio_device_num_vq(dev);
@@ -206,6 +207,18 @@ static int s390_virtio_scsi_init(VirtIOS390Device *dev)
     return s390_virtio_device_init(dev, vdev);
 }
 
+static int s390_virtio_rng_init(VirtIOS390Device *dev)
+{
+    VirtIODevice *vdev;
+
+    vdev = virtio_rng_init((DeviceState *)dev, &dev->rng);
+    if (!vdev) {
+        return -1;
+    }
+
+    return s390_virtio_device_init(dev, vdev);
+}
+
 static uint64_t s390_virtio_device_vq_token(VirtIOS390Device *dev, int vq)
 {
     ram_addr_t token_off;
@@ -448,6 +461,29 @@ static TypeInfo s390_virtio_serial = {
     .class_init    = s390_virtio_serial_class_init,
 };
 
+static void s390_virtio_rng_initfn(Object *obj)
+{
+    VirtIOS390Device *dev = VIRTIO_S390_DEVICE(obj);
+
+    object_property_add_link(obj, "rng", TYPE_RNG_BACKEND,
+                             (Object **)&dev->rng.rng, NULL);
+}
+
+static void s390_virtio_rng_class_init(ObjectClass *klass, void *data)
+{
+    VirtIOS390DeviceClass *k = VIRTIO_S390_DEVICE_CLASS(klass);
+
+    k->init = s390_virtio_rng_init;
+}
+
+static TypeInfo s390_virtio_rng = {
+    .name          = "virtio-rng-s390",
+    .parent        = TYPE_VIRTIO_S390_DEVICE,
+    .instance_size = sizeof(VirtIOS390Device),
+    .instance_init = s390_virtio_rng_initfn,
+    .class_init    = s390_virtio_rng_class_init,
+};
+
 static int s390_virtio_busdev_init(DeviceState *dev)
 {
     VirtIOS390Device *_dev = (VirtIOS390Device *)dev;
@@ -528,6 +564,7 @@ static void s390_virtio_register_types(void)
     type_register_static(&s390_virtio_blk);
     type_register_static(&s390_virtio_net);
     type_register_static(&s390_virtio_scsi);
+    type_register_static(&s390_virtio_rng);
     type_register_static(&s390_virtio_bridge_info);
 }
 
diff --git a/hw/s390-virtio-bus.h b/hw/s390-virtio-bus.h
index 4873134ae..a83afe785 100644
--- a/hw/s390-virtio-bus.h
+++ b/hw/s390-virtio-bus.h
@@ -19,6 +19,7 @@
 
 #include "virtio-blk.h"
 #include "virtio-net.h"
+#include "virtio-rng.h"
 #include "virtio-serial.h"
 #include "virtio-scsi.h"
 
@@ -75,6 +76,7 @@ struct VirtIOS390Device {
     virtio_serial_conf serial;
     virtio_net_conf net;
     VirtIOSCSIConf scsi;
+    VirtIORNGConf rng;
 };
 
 typedef struct VirtIOS390Bus {
diff --git a/hw/s390-virtio.c b/hw/s390-virtio.c
index 47eed35da..ca1bb0981 100644
--- a/hw/s390-virtio.c
+++ b/hw/s390-virtio.c
@@ -32,6 +32,7 @@
 #include "exec-memory.h"
 
 #include "hw/s390-virtio-bus.h"
+#include "hw/s390x/sclp.h"
 
 //#define DEBUG_S390
 
@@ -151,13 +152,13 @@ unsigned s390_del_running_cpu(CPUS390XState *env)
 }
 
 /* PC hardware initialisation */
-static void s390_init(ram_addr_t my_ram_size,
-                      const char *boot_device,
-                      const char *kernel_filename,
-                      const char *kernel_cmdline,
-                      const char *initrd_filename,
-                      const char *cpu_model)
+static void s390_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t my_ram_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
     CPUS390XState *env = NULL;
     MemoryRegion *sysmem = get_system_memory();
     MemoryRegion *ram = g_new(MemoryRegion, 1);
@@ -167,8 +168,8 @@ static void s390_init(ram_addr_t my_ram_size,
     int shift = 0;
     uint8_t *storage_keys;
     void *virtio_region;
-    target_phys_addr_t virtio_region_len;
-    target_phys_addr_t virtio_region_start;
+    hwaddr virtio_region_len;
+    hwaddr virtio_region_start;
     int i;
 
     /* s390x ram size detection needs a 16bit multiplier + an increment. So
@@ -183,6 +184,7 @@ static void s390_init(ram_addr_t my_ram_size,
 
     /* get a BUS */
     s390_bus = s390_virtio_bus_init(&my_ram_size);
+    s390_sclp_init();
 
     /* allocate RAM */
     memory_region_init_ram(ram, "s390.ram", my_ram_size);
@@ -284,8 +286,8 @@ static void s390_init(ram_addr_t my_ram_size,
         }
 
         /* we have to overwrite values in the kernel image, which are "rom" */
-        memcpy(rom_ptr(INITRD_PARM_START), &initrd_offset, 8);
-        memcpy(rom_ptr(INITRD_PARM_SIZE), &initrd_size, 8);
+        stq_p(rom_ptr(INITRD_PARM_START), initrd_offset);
+        stq_p(rom_ptr(INITRD_PARM_SIZE), initrd_size);
     }
 
     if (rom_ptr(KERN_PARM_AREA)) {
diff --git a/hw/s390x/Makefile.objs b/hw/s390x/Makefile.objs
index dcdcac8a8..096dfcd6a 100644
--- a/hw/s390x/Makefile.objs
+++ b/hw/s390x/Makefile.objs
@@ -1,3 +1,6 @@
 obj-y = s390-virtio-bus.o s390-virtio.o
 
 obj-y := $(addprefix ../,$(obj-y))
+obj-y += sclp.o
+obj-y += event-facility.o
+obj-y += sclpquiesce.o sclpconsole.o
diff --git a/hw/s390x/event-facility.c b/hw/s390x/event-facility.c
new file mode 100644
index 000000000..bc9cea9e1
--- /dev/null
+++ b/hw/s390x/event-facility.c
@@ -0,0 +1,399 @@
+/*
+ * SCLP
+ *    Event Facility
+ *       handles SCLP event types
+ *          - Signal Quiesce - system power down
+ *          - ASCII Console Data - VT220 read and write
+ *
+ * Copyright IBM, Corp. 2012
+ *
+ * Authors:
+ *  Heinz Graalfs <graalfs@de.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at your
+ * option) any later version.  See the COPYING file in the top-level directory.
+ *
+ */
+
+#include "monitor.h"
+#include "sysemu.h"
+
+#include "sclp.h"
+#include "event-facility.h"
+
+typedef struct EventTypesBus {
+    BusState qbus;
+} EventTypesBus;
+
+struct SCLPEventFacility {
+    EventTypesBus sbus;
+    DeviceState *qdev;
+    /* guest' receive mask */
+    unsigned int receive_mask;
+};
+
+/* return true if any child has event pending set */
+static bool event_pending(SCLPEventFacility *ef)
+{
+    BusChild *kid;
+    SCLPEvent *event;
+    SCLPEventClass *event_class;
+
+    QTAILQ_FOREACH(kid, &ef->sbus.qbus.children, sibling) {
+        DeviceState *qdev = kid->child;
+        event = DO_UPCAST(SCLPEvent, qdev, qdev);
+        event_class = SCLP_EVENT_GET_CLASS(event);
+        if (event->event_pending &&
+            event_class->get_send_mask() & ef->receive_mask) {
+            return true;
+        }
+    }
+    return false;
+}
+
+static unsigned int get_host_send_mask(SCLPEventFacility *ef)
+{
+    unsigned int mask;
+    BusChild *kid;
+    SCLPEventClass *child;
+
+    mask = 0;
+
+    QTAILQ_FOREACH(kid, &ef->sbus.qbus.children, sibling) {
+        DeviceState *qdev = kid->child;
+        child = SCLP_EVENT_GET_CLASS((SCLPEvent *) qdev);
+        mask |= child->get_send_mask();
+    }
+    return mask;
+}
+
+static unsigned int get_host_receive_mask(SCLPEventFacility *ef)
+{
+    unsigned int mask;
+    BusChild *kid;
+    SCLPEventClass *child;
+
+    mask = 0;
+
+    QTAILQ_FOREACH(kid, &ef->sbus.qbus.children, sibling) {
+        DeviceState *qdev = kid->child;
+        child = SCLP_EVENT_GET_CLASS((SCLPEvent *) qdev);
+        mask |= child->get_receive_mask();
+    }
+    return mask;
+}
+
+static uint16_t write_event_length_check(SCCB *sccb)
+{
+    int slen;
+    unsigned elen = 0;
+    EventBufferHeader *event;
+    WriteEventData *wed = (WriteEventData *) sccb;
+
+    event = (EventBufferHeader *) &wed->ebh;
+    for (slen = sccb_data_len(sccb); slen > 0; slen -= elen) {
+        elen = be16_to_cpu(event->length);
+        if (elen < sizeof(*event) || elen > slen) {
+            return SCLP_RC_EVENT_BUFFER_SYNTAX_ERROR;
+        }
+        event = (void *) event + elen;
+    }
+    if (slen) {
+        return SCLP_RC_INCONSISTENT_LENGTHS;
+    }
+    return SCLP_RC_NORMAL_COMPLETION;
+}
+
+static uint16_t handle_write_event_buf(SCLPEventFacility *ef,
+                                       EventBufferHeader *event_buf, SCCB *sccb)
+{
+    uint16_t rc;
+    BusChild *kid;
+    SCLPEvent *event;
+    SCLPEventClass *ec;
+
+    rc = SCLP_RC_INVALID_FUNCTION;
+
+    QTAILQ_FOREACH(kid, &ef->sbus.qbus.children, sibling) {
+        DeviceState *qdev = kid->child;
+        event = (SCLPEvent *) qdev;
+        ec = SCLP_EVENT_GET_CLASS(event);
+
+        if (ec->write_event_data &&
+            ec->event_type() == event_buf->type) {
+            rc = ec->write_event_data(event, event_buf);
+            break;
+        }
+    }
+    return rc;
+}
+
+static uint16_t handle_sccb_write_events(SCLPEventFacility *ef, SCCB *sccb)
+{
+    uint16_t rc;
+    int slen;
+    unsigned elen = 0;
+    EventBufferHeader *event_buf;
+    WriteEventData *wed = (WriteEventData *) sccb;
+
+    event_buf = &wed->ebh;
+    rc = SCLP_RC_NORMAL_COMPLETION;
+
+    /* loop over all contained event buffers */
+    for (slen = sccb_data_len(sccb); slen > 0; slen -= elen) {
+        elen = be16_to_cpu(event_buf->length);
+
+        /* in case of a previous error mark all trailing buffers
+         * as not accepted */
+        if (rc != SCLP_RC_NORMAL_COMPLETION) {
+            event_buf->flags &= ~(SCLP_EVENT_BUFFER_ACCEPTED);
+        } else {
+            rc = handle_write_event_buf(ef, event_buf, sccb);
+        }
+        event_buf = (void *) event_buf + elen;
+    }
+    return rc;
+}
+
+static void write_event_data(SCLPEventFacility *ef, SCCB *sccb)
+{
+    if (sccb->h.function_code != SCLP_FC_NORMAL_WRITE) {
+        sccb->h.response_code = cpu_to_be16(SCLP_RC_INVALID_FUNCTION);
+        goto out;
+    }
+    if (be16_to_cpu(sccb->h.length) < 8) {
+        sccb->h.response_code = cpu_to_be16(SCLP_RC_INSUFFICIENT_SCCB_LENGTH);
+        goto out;
+    }
+    /* first do a sanity check of the write events */
+    sccb->h.response_code = cpu_to_be16(write_event_length_check(sccb));
+
+    /* if no early error, then execute */
+    if (sccb->h.response_code == be16_to_cpu(SCLP_RC_NORMAL_COMPLETION)) {
+        sccb->h.response_code =
+                cpu_to_be16(handle_sccb_write_events(ef, sccb));
+    }
+
+out:
+    return;
+}
+
+static uint16_t handle_sccb_read_events(SCLPEventFacility *ef, SCCB *sccb,
+                                        unsigned int mask)
+{
+    uint16_t rc;
+    int slen;
+    unsigned elen = 0;
+    BusChild *kid;
+    SCLPEvent *event;
+    SCLPEventClass *ec;
+    EventBufferHeader *event_buf;
+    ReadEventData *red = (ReadEventData *) sccb;
+
+    event_buf = &red->ebh;
+    event_buf->length = 0;
+    slen = sizeof(sccb->data);
+
+    rc = SCLP_RC_NO_EVENT_BUFFERS_STORED;
+
+    QTAILQ_FOREACH(kid, &ef->sbus.qbus.children, sibling) {
+        DeviceState *qdev = kid->child;
+        event = (SCLPEvent *) qdev;
+        ec = SCLP_EVENT_GET_CLASS(event);
+
+        if (mask & ec->get_send_mask()) {
+            if (ec->read_event_data(event, event_buf, &slen)) {
+                rc = SCLP_RC_NORMAL_COMPLETION;
+            }
+        }
+        elen = be16_to_cpu(event_buf->length);
+        event_buf = (void *) event_buf + elen;
+    }
+
+    if (sccb->h.control_mask[2] & SCLP_VARIABLE_LENGTH_RESPONSE) {
+        /* architecture suggests to reset variable-length-response bit */
+        sccb->h.control_mask[2] &= ~SCLP_VARIABLE_LENGTH_RESPONSE;
+        /* with a new length value */
+        sccb->h.length = cpu_to_be16(SCCB_SIZE - slen);
+    }
+    return rc;
+}
+
+static void read_event_data(SCLPEventFacility *ef, SCCB *sccb)
+{
+    unsigned int sclp_active_selection_mask;
+    unsigned int sclp_cp_receive_mask;
+
+    ReadEventData *red = (ReadEventData *) sccb;
+
+    if (be16_to_cpu(sccb->h.length) != SCCB_SIZE) {
+        sccb->h.response_code = cpu_to_be16(SCLP_RC_INSUFFICIENT_SCCB_LENGTH);
+        goto out;
+    }
+
+    sclp_cp_receive_mask = ef->receive_mask;
+
+    /* get active selection mask */
+    switch (sccb->h.function_code) {
+    case SCLP_UNCONDITIONAL_READ:
+        sclp_active_selection_mask = sclp_cp_receive_mask;
+        break;
+    case SCLP_SELECTIVE_READ:
+        if (!(sclp_cp_receive_mask & be32_to_cpu(red->mask))) {
+            sccb->h.response_code =
+                    cpu_to_be16(SCLP_RC_INVALID_SELECTION_MASK);
+            goto out;
+        }
+        sclp_active_selection_mask = be32_to_cpu(red->mask);
+        break;
+    default:
+        sccb->h.response_code = cpu_to_be16(SCLP_RC_INVALID_FUNCTION);
+        goto out;
+    }
+    sccb->h.response_code = cpu_to_be16(
+            handle_sccb_read_events(ef, sccb, sclp_active_selection_mask));
+
+out:
+    return;
+}
+
+static void write_event_mask(SCLPEventFacility *ef, SCCB *sccb)
+{
+    WriteEventMask *we_mask = (WriteEventMask *) sccb;
+
+    /* Attention: We assume that Linux uses 4-byte masks, what it actually
+       does. Architecture allows for masks of variable size, though */
+    if (be16_to_cpu(we_mask->mask_length) != 4) {
+        sccb->h.response_code = cpu_to_be16(SCLP_RC_INVALID_MASK_LENGTH);
+        goto out;
+    }
+
+    /* keep track of the guest's capability masks */
+    ef->receive_mask = be32_to_cpu(we_mask->cp_receive_mask);
+
+    /* return the SCLP's capability masks to the guest */
+    we_mask->send_mask = cpu_to_be32(get_host_send_mask(ef));
+    we_mask->receive_mask = cpu_to_be32(get_host_receive_mask(ef));
+
+    sccb->h.response_code = cpu_to_be16(SCLP_RC_NORMAL_COMPLETION);
+
+out:
+    return;
+}
+
+/* qemu object creation and initialization functions */
+
+#define TYPE_SCLP_EVENTS_BUS "s390-sclp-events-bus"
+
+static void sclp_events_bus_class_init(ObjectClass *klass, void *data)
+{
+}
+
+static const TypeInfo s390_sclp_events_bus_info = {
+    .name = TYPE_SCLP_EVENTS_BUS,
+    .parent = TYPE_BUS,
+    .class_init = sclp_events_bus_class_init,
+};
+
+static void command_handler(SCLPEventFacility *ef, SCCB *sccb, uint64_t code)
+{
+    switch (code) {
+    case SCLP_CMD_READ_EVENT_DATA:
+        read_event_data(ef, sccb);
+        break;
+    case SCLP_CMD_WRITE_EVENT_DATA:
+        write_event_data(ef, sccb);
+        break;
+    case SCLP_CMD_WRITE_EVENT_MASK:
+        write_event_mask(ef, sccb);
+        break;
+    default:
+        sccb->h.response_code = cpu_to_be16(SCLP_RC_INVALID_SCLP_COMMAND);
+        break;
+    }
+}
+
+static int init_event_facility(S390SCLPDevice *sdev)
+{
+    SCLPEventFacility *event_facility;
+    DeviceState *quiesce;
+
+    event_facility = g_malloc0(sizeof(SCLPEventFacility));
+    sdev->ef = event_facility;
+    sdev->sclp_command_handler = command_handler;
+    sdev->event_pending = event_pending;
+
+    /* Spawn a new sclp-events facility */
+    qbus_create_inplace(&event_facility->sbus.qbus,
+                        TYPE_SCLP_EVENTS_BUS, (DeviceState *)sdev, NULL);
+    event_facility->sbus.qbus.allow_hotplug = 0;
+    event_facility->qdev = (DeviceState *) sdev;
+
+    quiesce = qdev_create(&event_facility->sbus.qbus, "sclpquiesce");
+    if (!quiesce) {
+        return -1;
+    }
+    qdev_init_nofail(quiesce);
+
+    return 0;
+}
+
+static void init_event_facility_class(ObjectClass *klass, void *data)
+{
+    S390SCLPDeviceClass *k = SCLP_S390_DEVICE_CLASS(klass);
+
+    k->init = init_event_facility;
+}
+
+static TypeInfo s390_sclp_event_facility_info = {
+    .name          = "s390-sclp-event-facility",
+    .parent        = TYPE_DEVICE_S390_SCLP,
+    .instance_size = sizeof(S390SCLPDevice),
+    .class_init    = init_event_facility_class,
+};
+
+static int event_qdev_init(DeviceState *qdev)
+{
+    SCLPEvent *event = DO_UPCAST(SCLPEvent, qdev, qdev);
+    SCLPEventClass *child = SCLP_EVENT_GET_CLASS(event);
+
+    return child->init(event);
+}
+
+static int event_qdev_exit(DeviceState *qdev)
+{
+    SCLPEvent *event = DO_UPCAST(SCLPEvent, qdev, qdev);
+    SCLPEventClass *child = SCLP_EVENT_GET_CLASS(event);
+    if (child->exit) {
+        child->exit(event);
+    }
+    return 0;
+}
+
+static void event_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+
+    dc->bus_type = TYPE_SCLP_EVENTS_BUS;
+    dc->unplug = qdev_simple_unplug_cb;
+    dc->init = event_qdev_init;
+    dc->exit = event_qdev_exit;
+}
+
+static TypeInfo s390_sclp_event_type_info = {
+    .name = TYPE_SCLP_EVENT,
+    .parent = TYPE_DEVICE,
+    .instance_size = sizeof(SCLPEvent),
+    .class_init = event_class_init,
+    .class_size = sizeof(SCLPEventClass),
+    .abstract = true,
+};
+
+static void register_types(void)
+{
+    type_register_static(&s390_sclp_events_bus_info);
+    type_register_static(&s390_sclp_event_facility_info);
+    type_register_static(&s390_sclp_event_type_info);
+}
+
+type_init(register_types)
diff --git a/hw/s390x/event-facility.h b/hw/s390x/event-facility.h
new file mode 100644
index 000000000..30af0a76a
--- /dev/null
+++ b/hw/s390x/event-facility.h
@@ -0,0 +1,96 @@
+/*
+ * SCLP
+ *    Event Facility definitions
+ *
+ * Copyright IBM, Corp. 2012
+ *
+ * Authors:
+ *  Heinz Graalfs <graalfs@de.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at your
+ * option) any later version.  See the COPYING file in the top-level directory.
+ *
+ */
+
+#ifndef HW_S390_SCLP_EVENT_FACILITY_H
+#define HW_S390_SCLP_EVENT_FACILITY_H
+
+#include <hw/qdev.h>
+#include "qemu-thread.h"
+
+/* SCLP event types */
+#define SCLP_EVENT_ASCII_CONSOLE_DATA           0x1a
+#define SCLP_EVENT_SIGNAL_QUIESCE               0x1d
+
+/* SCLP event masks */
+#define SCLP_EVENT_MASK_SIGNAL_QUIESCE          0x00000008
+#define SCLP_EVENT_MASK_MSG_ASCII               0x00000040
+
+#define SCLP_UNCONDITIONAL_READ                 0x00
+#define SCLP_SELECTIVE_READ                     0x01
+
+#define TYPE_SCLP_EVENT "s390-sclp-event-type"
+#define SCLP_EVENT(obj) \
+     OBJECT_CHECK(SCLPEvent, (obj), TYPE_SCLP_EVENT)
+#define SCLP_EVENT_CLASS(klass) \
+     OBJECT_CLASS_CHECK(SCLPEventClass, (klass), TYPE_SCLP_EVENT)
+#define SCLP_EVENT_GET_CLASS(obj) \
+     OBJECT_GET_CLASS(SCLPEventClass, (obj), TYPE_SCLP_EVENT)
+
+typedef struct WriteEventMask {
+    SCCBHeader h;
+    uint16_t _reserved;
+    uint16_t mask_length;
+    uint32_t cp_receive_mask;
+    uint32_t cp_send_mask;
+    uint32_t send_mask;
+    uint32_t receive_mask;
+} QEMU_PACKED WriteEventMask;
+
+typedef struct EventBufferHeader {
+    uint16_t length;
+    uint8_t  type;
+    uint8_t  flags;
+    uint16_t _reserved;
+} QEMU_PACKED EventBufferHeader;
+
+typedef struct WriteEventData {
+    SCCBHeader h;
+    EventBufferHeader ebh;
+} QEMU_PACKED WriteEventData;
+
+typedef struct ReadEventData {
+    SCCBHeader h;
+    EventBufferHeader ebh;
+    uint32_t mask;
+} QEMU_PACKED ReadEventData;
+
+typedef struct SCLPEvent {
+    DeviceState qdev;
+    bool event_pending;
+    uint32_t event_type;
+    char *name;
+} SCLPEvent;
+
+typedef struct SCLPEventClass {
+    DeviceClass parent_class;
+    int (*init)(SCLPEvent *event);
+    int (*exit)(SCLPEvent *event);
+
+    /* get SCLP's send mask */
+    unsigned int (*get_send_mask)(void);
+
+    /* get SCLP's receive mask */
+    unsigned int (*get_receive_mask)(void);
+
+    int (*read_event_data)(SCLPEvent *event, EventBufferHeader *evt_buf_hdr,
+                           int *slen);
+
+    int (*write_event_data)(SCLPEvent *event, EventBufferHeader *evt_buf_hdr);
+
+    /* returns the supported event type */
+    int (*event_type)(void);
+
+} SCLPEventClass;
+
+#endif
diff --git a/hw/s390x/sclp.c b/hw/s390x/sclp.c
new file mode 100644
index 000000000..5c274fa03
--- /dev/null
+++ b/hw/s390x/sclp.c
@@ -0,0 +1,163 @@
+/*
+ * SCLP Support
+ *
+ * Copyright IBM, Corp. 2012
+ *
+ * Authors:
+ *  Christian Borntraeger <borntraeger@de.ibm.com>
+ *  Heinz Graalfs <graalfs@linux.vnet.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at your
+ * option) any later version.  See the COPYING file in the top-level directory.
+ *
+ */
+
+#include "cpu.h"
+#include "kvm.h"
+#include "memory.h"
+
+#include "sclp.h"
+
+static inline S390SCLPDevice *get_event_facility(void)
+{
+    ObjectProperty *op = object_property_find(qdev_get_machine(),
+                                              "s390-sclp-event-facility",
+                                              NULL);
+    assert(op);
+    return op->opaque;
+}
+
+/* Provide information about the configuration, CPUs and storage */
+static void read_SCP_info(SCCB *sccb)
+{
+    ReadInfo *read_info = (ReadInfo *) sccb;
+    int shift = 0;
+
+    while ((ram_size >> (20 + shift)) > 65535) {
+        shift++;
+    }
+    read_info->rnmax = cpu_to_be16(ram_size >> (20 + shift));
+    read_info->rnsize = 1 << shift;
+    sccb->h.response_code = cpu_to_be16(SCLP_RC_NORMAL_READ_COMPLETION);
+}
+
+static void sclp_execute(SCCB *sccb, uint64_t code)
+{
+    S390SCLPDevice *sdev = get_event_facility();
+
+    switch (code) {
+    case SCLP_CMDW_READ_SCP_INFO:
+    case SCLP_CMDW_READ_SCP_INFO_FORCED:
+        read_SCP_info(sccb);
+        break;
+    default:
+        sdev->sclp_command_handler(sdev->ef, sccb, code);
+        break;
+    }
+}
+
+int sclp_service_call(uint32_t sccb, uint64_t code)
+{
+    int r = 0;
+    SCCB work_sccb;
+
+    hwaddr sccb_len = sizeof(SCCB);
+
+    /* first some basic checks on program checks */
+    if (cpu_physical_memory_is_io(sccb)) {
+        r = -PGM_ADDRESSING;
+        goto out;
+    }
+    if (sccb & ~0x7ffffff8ul) {
+        r = -PGM_SPECIFICATION;
+        goto out;
+    }
+
+    /*
+     * we want to work on a private copy of the sccb, to prevent guests
+     * from playing dirty tricks by modifying the memory content after
+     * the host has checked the values
+     */
+    cpu_physical_memory_read(sccb, &work_sccb, sccb_len);
+
+    /* Valid sccb sizes */
+    if (be16_to_cpu(work_sccb.h.length) < sizeof(SCCBHeader) ||
+        be16_to_cpu(work_sccb.h.length) > SCCB_SIZE) {
+        r = -PGM_SPECIFICATION;
+        goto out;
+    }
+
+    sclp_execute((SCCB *)&work_sccb, code);
+
+    cpu_physical_memory_write(sccb, &work_sccb,
+                              be16_to_cpu(work_sccb.h.length));
+
+    sclp_service_interrupt(sccb);
+
+out:
+    return r;
+}
+
+void sclp_service_interrupt(uint32_t sccb)
+{
+    S390SCLPDevice *sdev = get_event_facility();
+    uint32_t param = sccb & ~3;
+
+    /* Indicate whether an event is still pending */
+    param |= sdev->event_pending(sdev->ef) ? 1 : 0;
+
+    if (!param) {
+        /* No need to send an interrupt, there's nothing to be notified about */
+        return;
+    }
+    s390_sclp_extint(param);
+}
+
+/* qemu object creation and initialization functions */
+
+void s390_sclp_init(void)
+{
+    DeviceState *dev  = qdev_create(NULL, "s390-sclp-event-facility");
+
+    object_property_add_child(qdev_get_machine(), "s390-sclp-event-facility",
+                              OBJECT(dev), NULL);
+    qdev_init_nofail(dev);
+}
+
+static int s390_sclp_dev_init(SysBusDevice *dev)
+{
+    int r;
+    S390SCLPDevice *sdev = (S390SCLPDevice *)dev;
+    S390SCLPDeviceClass *sclp = SCLP_S390_DEVICE_GET_CLASS(dev);
+
+    r = sclp->init(sdev);
+    if (!r) {
+        assert(sdev->event_pending);
+        assert(sdev->sclp_command_handler);
+    }
+
+    return r;
+}
+
+static void s390_sclp_device_class_init(ObjectClass *klass, void *data)
+{
+    SysBusDeviceClass *dc = SYS_BUS_DEVICE_CLASS(klass);
+
+    dc->init = s390_sclp_dev_init;
+}
+
+static TypeInfo s390_sclp_device_info = {
+    .name = TYPE_DEVICE_S390_SCLP,
+    .parent = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(S390SCLPDevice),
+    .class_init = s390_sclp_device_class_init,
+    .class_size = sizeof(S390SCLPDeviceClass),
+    .abstract = true,
+};
+
+static void s390_sclp_register_types(void)
+{
+    type_register_static(&s390_sclp_device_info);
+}
+
+type_init(s390_sclp_register_types)
diff --git a/hw/s390x/sclp.h b/hw/s390x/sclp.h
new file mode 100644
index 000000000..fe89dadd6
--- /dev/null
+++ b/hw/s390x/sclp.h
@@ -0,0 +1,118 @@
+/*
+ * SCLP Support
+ *
+ * Copyright IBM, Corp. 2012
+ *
+ * Authors:
+ *  Christian Borntraeger <borntraeger@de.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at your
+ * option) any later version.  See the COPYING file in the top-level directory.
+ *
+ */
+
+#ifndef HW_S390_SCLP_H
+#define HW_S390_SCLP_H
+
+#include <hw/sysbus.h>
+#include <hw/qdev.h>
+
+/* SCLP command codes */
+#define SCLP_CMDW_READ_SCP_INFO                 0x00020001
+#define SCLP_CMDW_READ_SCP_INFO_FORCED          0x00120001
+#define SCLP_CMD_READ_EVENT_DATA                0x00770005
+#define SCLP_CMD_WRITE_EVENT_DATA               0x00760005
+#define SCLP_CMD_READ_EVENT_DATA                0x00770005
+#define SCLP_CMD_WRITE_EVENT_DATA               0x00760005
+#define SCLP_CMD_WRITE_EVENT_MASK               0x00780005
+
+/* SCLP response codes */
+#define SCLP_RC_NORMAL_READ_COMPLETION          0x0010
+#define SCLP_RC_NORMAL_COMPLETION               0x0020
+#define SCLP_RC_INVALID_SCLP_COMMAND            0x01f0
+#define SCLP_RC_CONTAINED_EQUIPMENT_CHECK       0x0340
+#define SCLP_RC_INSUFFICIENT_SCCB_LENGTH        0x0300
+#define SCLP_RC_INVALID_FUNCTION                0x40f0
+#define SCLP_RC_NO_EVENT_BUFFERS_STORED         0x60f0
+#define SCLP_RC_INVALID_SELECTION_MASK          0x70f0
+#define SCLP_RC_INCONSISTENT_LENGTHS            0x72f0
+#define SCLP_RC_EVENT_BUFFER_SYNTAX_ERROR       0x73f0
+#define SCLP_RC_INVALID_MASK_LENGTH             0x74f0
+
+
+/* Service Call Control Block (SCCB) and its elements */
+
+#define SCCB_SIZE 4096
+
+#define SCLP_VARIABLE_LENGTH_RESPONSE           0x80
+#define SCLP_EVENT_BUFFER_ACCEPTED              0x80
+
+#define SCLP_FC_NORMAL_WRITE                    0
+
+/*
+ * Normally packed structures are not the right thing to do, since all code
+ * must take care of endianess. We cant use ldl_phys and friends for two
+ * reasons, though:
+ * - some of the embedded structures below the SCCB can appear multiple times
+ *   at different locations, so there is no fixed offset
+ * - we work on a private copy of the SCCB, since there are several length
+ *   fields, that would cause a security nightmare if we allow the guest to
+ *   alter the structure while we parse it. We cannot use ldl_p and friends
+ *   either without doing pointer arithmetics
+ * So we have to double check that all users of sclp data structures use the
+ * right endianess wrappers.
+ */
+typedef struct SCCBHeader {
+    uint16_t length;
+    uint8_t function_code;
+    uint8_t control_mask[3];
+    uint16_t response_code;
+} QEMU_PACKED SCCBHeader;
+
+#define SCCB_DATA_LEN (SCCB_SIZE - sizeof(SCCBHeader))
+
+typedef struct ReadInfo {
+    SCCBHeader h;
+    uint16_t rnmax;
+    uint8_t rnsize;
+} QEMU_PACKED ReadInfo;
+
+typedef struct SCCB {
+    SCCBHeader h;
+    char data[SCCB_DATA_LEN];
+ } QEMU_PACKED SCCB;
+
+static inline int sccb_data_len(SCCB *sccb)
+{
+    return be16_to_cpu(sccb->h.length) - sizeof(sccb->h);
+}
+
+#define TYPE_DEVICE_S390_SCLP "s390-sclp-device"
+#define SCLP_S390_DEVICE(obj) \
+     OBJECT_CHECK(S390SCLPDevice, (obj), TYPE_DEVICE_S390_SCLP)
+#define SCLP_S390_DEVICE_CLASS(klass) \
+     OBJECT_CLASS_CHECK(S390SCLPDeviceClass, (klass), \
+             TYPE_DEVICE_S390_SCLP)
+#define SCLP_S390_DEVICE_GET_CLASS(obj) \
+     OBJECT_GET_CLASS(S390SCLPDeviceClass, (obj), \
+             TYPE_DEVICE_S390_SCLP)
+
+typedef struct SCLPEventFacility SCLPEventFacility;
+
+typedef struct S390SCLPDevice {
+    SysBusDevice busdev;
+    SCLPEventFacility *ef;
+    void (*sclp_command_handler)(SCLPEventFacility *ef, SCCB *sccb,
+                                 uint64_t code);
+    bool (*event_pending)(SCLPEventFacility *ef);
+} S390SCLPDevice;
+
+typedef struct S390SCLPDeviceClass {
+    DeviceClass qdev;
+    int (*init)(S390SCLPDevice *sdev);
+} S390SCLPDeviceClass;
+
+void s390_sclp_init(void);
+void sclp_service_interrupt(uint32_t sccb);
+
+#endif
diff --git a/hw/s390x/sclpconsole.c b/hw/s390x/sclpconsole.c
new file mode 100644
index 000000000..0ec5623f5
--- /dev/null
+++ b/hw/s390x/sclpconsole.c
@@ -0,0 +1,306 @@
+/*
+ * SCLP event type
+ *    Ascii Console Data (VT220 Console)
+ *
+ * Copyright IBM, Corp. 2012
+ *
+ * Authors:
+ *  Heinz Graalfs <graalfs@de.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at your
+ * option) any later version.  See the COPYING file in the top-level directory.
+ *
+ */
+
+#include <hw/qdev.h>
+#include "qemu-thread.h"
+
+#include "sclp.h"
+#include "event-facility.h"
+
+typedef struct ASCIIConsoleData {
+    EventBufferHeader ebh;
+    char data[0];
+} QEMU_PACKED ASCIIConsoleData;
+
+/* max size for ASCII data in 4K SCCB page */
+#define SIZE_BUFFER_VT220 4080
+
+typedef struct SCLPConsole {
+    SCLPEvent event;
+    CharDriverState *chr;
+    /* io vector                                                       */
+    uint8_t *iov;           /* iov buffer pointer                      */
+    uint8_t *iov_sclp;      /* pointer to SCLP read offset             */
+    uint8_t *iov_bs;        /* pointer byte stream read offset         */
+    uint32_t iov_data_len;  /* length of byte stream in buffer         */
+    uint32_t iov_sclp_rest; /* length of byte stream not read via SCLP */
+    qemu_irq irq_read_vt220;
+} SCLPConsole;
+
+/* character layer call-back functions */
+
+/* Return number of bytes that fit into iov buffer */
+static int chr_can_read(void *opaque)
+{
+    int can_read;
+    SCLPConsole *scon = opaque;
+
+    can_read = SIZE_BUFFER_VT220 - scon->iov_data_len;
+
+    return can_read;
+}
+
+/* Receive n bytes from character layer, save in iov buffer,
+ * and set event pending */
+static void receive_from_chr_layer(SCLPConsole *scon, const uint8_t *buf,
+                                   int size)
+{
+    assert(scon->iov);
+
+    /* read data must fit into current buffer */
+    assert(size <= SIZE_BUFFER_VT220 - scon->iov_data_len);
+
+    /* put byte-stream from character layer into buffer */
+    memcpy(scon->iov_bs, buf, size);
+    scon->iov_data_len += size;
+    scon->iov_sclp_rest += size;
+    scon->iov_bs += size;
+    scon->event.event_pending = true;
+}
+
+/* Send data from a char device over to the guest */
+static void chr_read(void *opaque, const uint8_t *buf, int size)
+{
+    SCLPConsole *scon = opaque;
+
+    assert(scon);
+
+    receive_from_chr_layer(scon, buf, size);
+    /* trigger SCLP read operation */
+    qemu_irq_raise(scon->irq_read_vt220);
+}
+
+static void chr_event(void *opaque, int event)
+{
+    SCLPConsole *scon = opaque;
+
+    switch (event) {
+    case CHR_EVENT_OPENED:
+        if (!scon->iov) {
+            scon->iov = g_malloc0(SIZE_BUFFER_VT220);
+            scon->iov_sclp = scon->iov;
+            scon->iov_bs = scon->iov;
+            scon->iov_data_len = 0;
+            scon->iov_sclp_rest = 0;
+        }
+        break;
+    case CHR_EVENT_CLOSED:
+        if (scon->iov) {
+            g_free(scon->iov);
+            scon->iov = NULL;
+        }
+        break;
+    }
+}
+
+/* functions to be called by event facility */
+
+static int event_type(void)
+{
+    return SCLP_EVENT_ASCII_CONSOLE_DATA;
+}
+
+static unsigned int send_mask(void)
+{
+    return SCLP_EVENT_MASK_MSG_ASCII;
+}
+
+static unsigned int receive_mask(void)
+{
+    return SCLP_EVENT_MASK_MSG_ASCII;
+}
+
+/* triggered by SCLP's read_event_data -
+ * copy console data byte-stream into provided (SCLP) buffer
+ */
+static void get_console_data(SCLPEvent *event, uint8_t *buf, size_t *size,
+                             int avail)
+{
+    SCLPConsole *cons = DO_UPCAST(SCLPConsole, event, event);
+
+    /* first byte is hex 0 saying an ascii string follows */
+    *buf++ = '\0';
+    avail--;
+    /* if all data fit into provided SCLP buffer */
+    if (avail >= cons->iov_sclp_rest) {
+        /* copy character byte-stream to SCLP buffer */
+        memcpy(buf, cons->iov_sclp, cons->iov_sclp_rest);
+        *size = cons->iov_sclp_rest + 1;
+        cons->iov_sclp = cons->iov;
+        cons->iov_bs = cons->iov;
+        cons->iov_data_len = 0;
+        cons->iov_sclp_rest = 0;
+        event->event_pending = false;
+        /* data provided and no more data pending */
+    } else {
+        /* if provided buffer is too small, just copy part */
+        memcpy(buf, cons->iov_sclp, avail);
+        *size = avail + 1;
+        cons->iov_sclp_rest -= avail;
+        cons->iov_sclp += avail;
+        /* more data pending */
+    }
+}
+
+static int read_event_data(SCLPEvent *event, EventBufferHeader *evt_buf_hdr,
+                           int *slen)
+{
+    int avail;
+    size_t src_len;
+    uint8_t *to;
+    ASCIIConsoleData *acd = (ASCIIConsoleData *) evt_buf_hdr;
+
+    if (!event->event_pending) {
+        /* no data pending */
+        return 0;
+    }
+
+    to = (uint8_t *)&acd->data;
+    avail = *slen - sizeof(ASCIIConsoleData);
+    get_console_data(event, to, &src_len, avail);
+
+    acd->ebh.length = cpu_to_be16(sizeof(ASCIIConsoleData) + src_len);
+    acd->ebh.type = SCLP_EVENT_ASCII_CONSOLE_DATA;
+    acd->ebh.flags |= SCLP_EVENT_BUFFER_ACCEPTED;
+    *slen = avail - src_len;
+
+    return 1;
+}
+
+/* triggered by SCLP's write_event_data
+ *  - write console data into character layer
+ *  returns < 0 if an error occured
+ */
+static ssize_t write_console_data(SCLPEvent *event, const uint8_t *buf,
+                                  size_t len)
+{
+    ssize_t ret = 0;
+    const uint8_t *iov_offset;
+    SCLPConsole *scon = DO_UPCAST(SCLPConsole, event, event);
+
+    if (!scon->chr) {
+        /* If there's no backend, we can just say we consumed all data. */
+        return len;
+    }
+
+    iov_offset = buf;
+    while (len > 0) {
+        ret = qemu_chr_fe_write(scon->chr, buf, len);
+        if (ret == 0) {
+            /* a pty doesn't seem to be connected - no error */
+            len = 0;
+        } else if (ret == -EAGAIN || (ret > 0 && ret < len)) {
+            len -= ret;
+            iov_offset += ret;
+        } else {
+            len = 0;
+        }
+    }
+
+    return ret;
+}
+
+static int write_event_data(SCLPEvent *event, EventBufferHeader *evt_buf_hdr)
+{
+    int rc;
+    int length;
+    ssize_t written;
+    ASCIIConsoleData *acd = (ASCIIConsoleData *) evt_buf_hdr;
+
+    length = be16_to_cpu(evt_buf_hdr->length) - sizeof(EventBufferHeader);
+    written = write_console_data(event, (uint8_t *)acd->data, length);
+
+    rc = SCLP_RC_NORMAL_COMPLETION;
+    /* set event buffer accepted flag */
+    evt_buf_hdr->flags |= SCLP_EVENT_BUFFER_ACCEPTED;
+
+    /* written will be zero if a pty is not connected - don't treat as error */
+    if (written < 0) {
+        /* event buffer not accepted due to error in character layer */
+        evt_buf_hdr->flags &= ~(SCLP_EVENT_BUFFER_ACCEPTED);
+        rc = SCLP_RC_CONTAINED_EQUIPMENT_CHECK;
+    }
+
+    return rc;
+}
+
+static void trigger_ascii_console_data(void *env, int n, int level)
+{
+    sclp_service_interrupt(0);
+}
+
+/* qemu object creation and initialization functions */
+
+/* tell character layer our call-back functions */
+static int console_init(SCLPEvent *event)
+{
+    static bool console_available;
+
+    SCLPConsole *scon = DO_UPCAST(SCLPConsole, event, event);
+
+    if (console_available) {
+        error_report("Multiple VT220 operator consoles are not supported");
+        return -1;
+    }
+    console_available = true;
+    event->event_type = SCLP_EVENT_ASCII_CONSOLE_DATA;
+    if (scon->chr) {
+        qemu_chr_add_handlers(scon->chr, chr_can_read,
+                              chr_read, chr_event, scon);
+    }
+    scon->irq_read_vt220 = *qemu_allocate_irqs(trigger_ascii_console_data,
+                                               NULL, 1);
+
+    return 0;
+}
+
+static int console_exit(SCLPEvent *event)
+{
+    return 0;
+}
+
+static Property console_properties[] = {
+    DEFINE_PROP_CHR("chardev", SCLPConsole, chr),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void console_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    SCLPEventClass *ec = SCLP_EVENT_CLASS(klass);
+
+    dc->props = console_properties;
+    ec->init = console_init;
+    ec->exit = console_exit;
+    ec->get_send_mask = send_mask;
+    ec->get_receive_mask = receive_mask;
+    ec->event_type = event_type;
+    ec->read_event_data = read_event_data;
+    ec->write_event_data = write_event_data;
+}
+
+static TypeInfo sclp_console_info = {
+    .name          = "sclpconsole",
+    .parent        = TYPE_SCLP_EVENT,
+    .instance_size = sizeof(SCLPConsole),
+    .class_init    = console_class_init,
+    .class_size    = sizeof(SCLPEventClass),
+};
+
+static void register_types(void)
+{
+    type_register_static(&sclp_console_info);
+}
+
+type_init(register_types)
diff --git a/hw/s390x/sclpquiesce.c b/hw/s390x/sclpquiesce.c
new file mode 100644
index 000000000..9a773b87f
--- /dev/null
+++ b/hw/s390x/sclpquiesce.c
@@ -0,0 +1,123 @@
+/*
+ * SCLP event type
+ *    Signal Quiesce - trigger system powerdown request
+ *
+ * Copyright IBM, Corp. 2012
+ *
+ * Authors:
+ *  Heinz Graalfs <graalfs@de.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or (at your
+ * option) any later version.  See the COPYING file in the top-level directory.
+ *
+ */
+#include <hw/qdev.h>
+#include "sysemu.h"
+#include "sclp.h"
+#include "event-facility.h"
+
+typedef struct SignalQuiesce {
+    EventBufferHeader ebh;
+    uint16_t timeout;
+    uint8_t unit;
+} QEMU_PACKED SignalQuiesce;
+
+static int event_type(void)
+{
+    return SCLP_EVENT_SIGNAL_QUIESCE;
+}
+
+static unsigned int send_mask(void)
+{
+    return SCLP_EVENT_MASK_SIGNAL_QUIESCE;
+}
+
+static unsigned int receive_mask(void)
+{
+    return 0;
+}
+
+static int read_event_data(SCLPEvent *event, EventBufferHeader *evt_buf_hdr,
+                           int *slen)
+{
+    SignalQuiesce *sq = (SignalQuiesce *) evt_buf_hdr;
+
+    if (*slen < sizeof(SignalQuiesce)) {
+        return 0;
+    }
+
+    if (!event->event_pending) {
+        return 0;
+    }
+    event->event_pending = false;
+
+    sq->ebh.length = cpu_to_be16(sizeof(SignalQuiesce));
+    sq->ebh.type = SCLP_EVENT_SIGNAL_QUIESCE;
+    sq->ebh.flags |= SCLP_EVENT_BUFFER_ACCEPTED;
+    /*
+     * system_powerdown does not have a timeout. Fortunately the
+     * timeout value is currently ignored by Linux, anyway
+     */
+    sq->timeout = cpu_to_be16(0);
+    sq->unit = cpu_to_be16(0);
+    *slen -= sizeof(SignalQuiesce);
+
+    return 1;
+}
+
+typedef struct QuiesceNotifier QuiesceNotifier;
+
+static struct QuiesceNotifier {
+    Notifier notifier;
+    SCLPEvent *event;
+} qn;
+
+static void quiesce_powerdown_req(Notifier *n, void *opaque)
+{
+    QuiesceNotifier *qn = container_of(n, QuiesceNotifier, notifier);
+    SCLPEvent *event = qn->event;
+
+    event->event_pending = true;
+    /* trigger SCLP read operation */
+    sclp_service_interrupt(0);
+}
+
+static int quiesce_init(SCLPEvent *event)
+{
+    event->event_type = SCLP_EVENT_SIGNAL_QUIESCE;
+
+    qn.notifier.notify = quiesce_powerdown_req;
+    qn.event = event;
+
+    qemu_register_powerdown_notifier(&qn.notifier);
+
+    return 0;
+}
+
+static void quiesce_class_init(ObjectClass *klass, void *data)
+{
+    SCLPEventClass *k = SCLP_EVENT_CLASS(klass);
+
+    k->init = quiesce_init;
+
+    k->get_send_mask = send_mask;
+    k->get_receive_mask = receive_mask;
+    k->event_type = event_type;
+    k->read_event_data = read_event_data;
+    k->write_event_data = NULL;
+}
+
+static TypeInfo sclp_quiesce_info = {
+    .name          = "sclpquiesce",
+    .parent        = TYPE_SCLP_EVENT,
+    .instance_size = sizeof(SCLPEvent),
+    .class_init    = quiesce_class_init,
+    .class_size    = sizeof(SCLPEventClass),
+};
+
+static void register_types(void)
+{
+    type_register_static(&sclp_quiesce_info);
+}
+
+type_init(register_types)
diff --git a/hw/sb16.c b/hw/sb16.c
index c81455d7f..523ab0d5f 100644
--- a/hw/sb16.c
+++ b/hw/sb16.c
@@ -822,7 +822,6 @@ static void complete (SB16State *s)
 
     ldebug ("\n");
     s->cmd = -1;
-    return;
 }
 
 static void legacy_reset (SB16State *s)
diff --git a/hw/sbi.c b/hw/sbi.c
index 52982a9a3..ca78a384c 100644
--- a/hw/sbi.c
+++ b/hw/sbi.c
@@ -52,7 +52,7 @@ static void sbi_set_irq(void *opaque, int irq, int level)
 {
 }
 
-static uint64_t sbi_mem_read(void *opaque, target_phys_addr_t addr,
+static uint64_t sbi_mem_read(void *opaque, hwaddr addr,
                              unsigned size)
 {
     SBIState *s = opaque;
@@ -69,7 +69,7 @@ static uint64_t sbi_mem_read(void *opaque, target_phys_addr_t addr,
     return ret;
 }
 
-static void sbi_mem_write(void *opaque, target_phys_addr_t addr,
+static void sbi_mem_write(void *opaque, hwaddr addr,
                           uint64_t val, unsigned dize)
 {
     SBIState *s = opaque;
diff --git a/hw/scsi-bus.c b/hw/scsi-bus.c
index 4981a0243..dfb263121 100644
--- a/hw/scsi-bus.c
+++ b/hw/scsi-bus.c
@@ -801,26 +801,39 @@ static int ata_passthrough_16_xfer_size(SCSIDevice *dev, uint8_t *buf)
     return xfer * unit;
 }
 
-static int scsi_req_length(SCSICommand *cmd, SCSIDevice *dev, uint8_t *buf)
+uint32_t scsi_data_cdb_length(uint8_t *buf)
+{
+    if ((buf[0] >> 5) == 0 && buf[4] == 0) {
+        return 256;
+    } else {
+        return scsi_cdb_length(buf);
+    }
+}
+
+uint32_t scsi_cdb_length(uint8_t *buf)
 {
     switch (buf[0] >> 5) {
     case 0:
-        cmd->xfer = buf[4];
+        return buf[4];
         break;
     case 1:
     case 2:
-        cmd->xfer = lduw_be_p(&buf[7]);
+        return lduw_be_p(&buf[7]);
         break;
     case 4:
-        cmd->xfer = ldl_be_p(&buf[10]) & 0xffffffffULL;
+        return ldl_be_p(&buf[10]) & 0xffffffffULL;
         break;
     case 5:
-        cmd->xfer = ldl_be_p(&buf[6]) & 0xffffffffULL;
+        return ldl_be_p(&buf[6]) & 0xffffffffULL;
         break;
     default:
         return -1;
     }
+}
 
+static int scsi_req_length(SCSICommand *cmd, SCSIDevice *dev, uint8_t *buf)
+{
+    cmd->xfer = scsi_cdb_length(buf);
     switch (buf[0]) {
     case TEST_UNIT_READY:
     case REWIND:
@@ -1710,12 +1723,8 @@ static char *scsibus_get_dev_path(DeviceState *dev)
 static char *scsibus_get_fw_dev_path(DeviceState *dev)
 {
     SCSIDevice *d = SCSI_DEVICE(dev);
-    char path[100];
-
-    snprintf(path, sizeof(path), "channel@%x/%s@%x,%x", d->channel,
-             qdev_fw_name(dev), d->id, d->lun);
-
-    return strdup(path);
+    return g_strdup_printf("channel@%x/%s@%x,%x", d->channel,
+                           qdev_fw_name(dev), d->id, d->lun);
 }
 
 SCSIDevice *scsi_device_find(SCSIBus *bus, int channel, int id, int lun)
diff --git a/hw/scsi-disk.c b/hw/scsi-disk.c
index 1585683bc..49b5686a9 100644
--- a/hw/scsi-disk.c
+++ b/hw/scsi-disk.c
@@ -386,23 +386,11 @@ static void scsi_read_data(SCSIRequest *req)
  */
 static int scsi_handle_rw_error(SCSIDiskReq *r, int error)
 {
-    int is_read = (r->req.cmd.xfer == SCSI_XFER_FROM_DEV);
+    bool is_read = (r->req.cmd.xfer == SCSI_XFER_FROM_DEV);
     SCSIDiskState *s = DO_UPCAST(SCSIDiskState, qdev, r->req.dev);
-    BlockErrorAction action = bdrv_get_on_error(s->qdev.conf.bs, is_read);
+    BlockErrorAction action = bdrv_get_error_action(s->qdev.conf.bs, is_read, error);
 
-    if (action == BLOCK_ERR_IGNORE) {
-        bdrv_emit_qmp_error_event(s->qdev.conf.bs, BDRV_ACTION_IGNORE, is_read);
-        return 0;
-    }
-
-    if ((error == ENOSPC && action == BLOCK_ERR_STOP_ENOSPC)
-            || action == BLOCK_ERR_STOP_ANY) {
-
-        bdrv_emit_qmp_error_event(s->qdev.conf.bs, BDRV_ACTION_STOP, is_read);
-        vm_stop(RUN_STATE_IO_ERROR);
-        bdrv_iostatus_set_err(s->qdev.conf.bs, error);
-        scsi_req_retry(&r->req);
-    } else {
+    if (action == BDRV_ACTION_REPORT) {
         switch (error) {
         case ENOMEDIUM:
             scsi_check_condition(r, SENSE_CODE(NO_MEDIUM));
@@ -417,9 +405,12 @@ static int scsi_handle_rw_error(SCSIDiskReq *r, int error)
             scsi_check_condition(r, SENSE_CODE(IO_ERROR));
             break;
         }
-        bdrv_emit_qmp_error_event(s->qdev.conf.bs, BDRV_ACTION_REPORT, is_read);
     }
-    return 1;
+    bdrv_error_action(s->qdev.conf.bs, action, is_read, error);
+    if (action == BDRV_ACTION_STOP) {
+        scsi_req_retry(&r->req);
+    }
+    return action != BDRV_ACTION_IGNORE;
 }
 
 static void scsi_write_complete(void * opaque, int ret)
@@ -661,7 +652,6 @@ static int scsi_disk_emulate_inquiry(SCSIRequest *req, uint8_t *outbuf)
     if (buflen > SCSI_MAX_INQUIRY_LEN) {
         buflen = SCSI_MAX_INQUIRY_LEN;
     }
-    memset(outbuf, 0, buflen);
 
     outbuf[0] = s->qdev.type & 0x1f;
     outbuf[1] = (s->features & (1 << SCSI_DISK_F_REMOVABLE)) ? 0x80 : 0;
@@ -678,7 +668,7 @@ static int scsi_disk_emulate_inquiry(SCSIRequest *req, uint8_t *outbuf)
      * is actually implemented, but we're good enough.
      */
     outbuf[2] = 5;
-    outbuf[3] = 2; /* Format 2 */
+    outbuf[3] = 2 | 0x10; /* Format 2, HiSup */
 
     if (buflen > 36) {
         outbuf[4] = buflen - 5; /* Additional Length = (Len - 1) - 4 */
@@ -1397,6 +1387,7 @@ invalid_param_len:
 
 static void scsi_disk_emulate_mode_select(SCSIDiskReq *r, uint8_t *inbuf)
 {
+    SCSIDiskState *s = DO_UPCAST(SCSIDiskState, qdev, r->req.dev);
     uint8_t *p = inbuf;
     int cmd = r->req.cmd.buf[0];
     int len = r->req.cmd.xfer;
@@ -1433,6 +1424,14 @@ static void scsi_disk_emulate_mode_select(SCSIDiskReq *r, uint8_t *inbuf)
             return;
         }
     }
+    if (!bdrv_enable_write_cache(s->qdev.conf.bs)) {
+        /* The request is used as the AIO opaque value, so add a ref.  */
+        scsi_req_ref(&r->req);
+        bdrv_acct_start(s->qdev.conf.bs, &r->acct, 0, BDRV_ACCT_FLUSH);
+        r->req.aiocb = bdrv_aio_flush(s->qdev.conf.bs, scsi_aio_complete, r);
+        return;
+    }
+
     scsi_req_complete(&r->req, GOOD);
     return;
 
@@ -1446,7 +1445,22 @@ invalid_param_len:
 
 invalid_field:
     scsi_check_condition(r, SENSE_CODE(INVALID_FIELD));
-    return;
+}
+
+static inline bool check_lba_range(SCSIDiskState *s,
+                                   uint64_t sector_num, uint32_t nb_sectors)
+{
+    /*
+     * The first line tests that no overflow happens when computing the last
+     * sector.  The second line tests that the last accessed sector is in
+     * range.
+     *
+     * Careful, the computations should not underflow for nb_sectors == 0,
+     * and a 0-block read to the first LBA beyond the end of device is
+     * valid.
+     */
+    return (sector_num <= sector_num + nb_sectors &&
+            sector_num + nb_sectors <= s->qdev.max_lba + 1);
 }
 
 typedef struct UnmapCBData {
@@ -1473,8 +1487,7 @@ static void scsi_unmap_complete(void *opaque, int ret)
     if (data->count > 0 && !r->req.io_canceled) {
         sector_num = ldq_be_p(&data->inbuf[0]);
         nb_sectors = ldl_be_p(&data->inbuf[8]) & 0xffffffffULL;
-        if (sector_num > sector_num + nb_sectors ||
-            sector_num + nb_sectors - 1 > s->qdev.max_lba) {
+        if (!check_lba_range(s, sector_num, nb_sectors)) {
             scsi_check_condition(r, SENSE_CODE(LBA_OUT_OF_RANGE));
             goto done;
         }
@@ -1529,7 +1542,6 @@ static void scsi_disk_emulate_unmap(SCSIDiskReq *r, uint8_t *inbuf)
 
 invalid_param_len:
     scsi_check_condition(r, SENSE_CODE(INVALID_PARAM_LEN));
-    return;
 }
 
 static void scsi_disk_emulate_write_data(SCSIRequest *req)
@@ -1592,24 +1604,26 @@ static int32_t scsi_disk_emulate_command(SCSIRequest *req, uint8_t *buf)
         break;
     }
 
+    /*
+     * FIXME: we shouldn't return anything bigger than 4k, but the code
+     * requires the buffer to be as big as req->cmd.xfer in several
+     * places.  So, do not allow CDBs with a very large ALLOCATION
+     * LENGTH.  The real fix would be to modify scsi_read_data and
+     * dma_buf_read, so that they return data beyond the buflen
+     * as all zeros.
+     */
+    if (req->cmd.xfer > 65536) {
+        goto illegal_request;
+    }
+    r->buflen = MAX(4096, req->cmd.xfer);
+
     if (!r->iov.iov_base) {
-        /*
-         * FIXME: we shouldn't return anything bigger than 4k, but the code
-         * requires the buffer to be as big as req->cmd.xfer in several
-         * places.  So, do not allow CDBs with a very large ALLOCATION
-         * LENGTH.  The real fix would be to modify scsi_read_data and
-         * dma_buf_read, so that they return data beyond the buflen
-         * as all zeros.
-         */
-        if (req->cmd.xfer > 65536) {
-            goto illegal_request;
-        }
-        r->buflen = MAX(4096, req->cmd.xfer);
         r->iov.iov_base = qemu_blockalign(s->qdev.conf.bs, r->buflen);
     }
 
     buflen = req->cmd.xfer;
     outbuf = r->iov.iov_base;
+    memset(outbuf, 0, r->buflen);
     switch (req->cmd.buf[0]) {
     case TEST_UNIT_READY:
         assert(!s->tray_open && bdrv_is_inserted(s->qdev.conf.bs));
@@ -1690,12 +1704,14 @@ static int32_t scsi_disk_emulate_command(SCSIRequest *req, uint8_t *buf)
         outbuf[5] = 0;
         outbuf[6] = s->qdev.blocksize >> 8;
         outbuf[7] = 0;
-        buflen = 8;
         break;
     case REQUEST_SENSE:
         /* Just return "NO SENSE".  */
         buflen = scsi_build_sense(NULL, 0, outbuf, r->buflen,
                                   (req->cmd.buf[1] & 1) == 0);
+        if (buflen < 0) {
+            goto illegal_request;
+        }
         break;
     case MECHANISM_STATUS:
         buflen = scsi_emulate_mechanism_status(s, outbuf);
@@ -1766,7 +1782,6 @@ static int32_t scsi_disk_emulate_command(SCSIRequest *req, uint8_t *buf)
             }
 
             /* Protection, exponent and lowest lba field left blank. */
-            buflen = req->cmd.xfer;
             break;
         }
         DPRINTF("Unsupported Service Action In\n");
@@ -1793,17 +1808,13 @@ static int32_t scsi_disk_emulate_command(SCSIRequest *req, uint8_t *buf)
         DPRINTF("Unmap (len %lu)\n", (long)r->req.cmd.xfer);
         break;
     case WRITE_SAME_10:
-        nb_sectors = lduw_be_p(&req->cmd.buf[7]);
-        goto write_same;
     case WRITE_SAME_16:
-        nb_sectors = ldl_be_p(&req->cmd.buf[10]) & 0xffffffffULL;
-    write_same:
+        nb_sectors = scsi_data_cdb_length(r->req.cmd.buf);
         if (bdrv_is_read_only(s->qdev.conf.bs)) {
             scsi_check_condition(r, SENSE_CODE(WRITE_PROTECTED));
             return 0;
         }
-        if (r->req.cmd.lba > r->req.cmd.lba + nb_sectors ||
-            r->req.cmd.lba + nb_sectors - 1 > s->qdev.max_lba) {
+        if (!check_lba_range(s, r->req.cmd.lba, nb_sectors)) {
             goto illegal_lba;
         }
 
@@ -1827,7 +1838,7 @@ static int32_t scsi_disk_emulate_command(SCSIRequest *req, uint8_t *buf)
         return 0;
     }
     assert(!r->req.aiocb);
-    r->iov.iov_len = MIN(buflen, req->cmd.xfer);
+    r->iov.iov_len = MIN(r->buflen, req->cmd.xfer);
     if (r->iov.iov_len == 0) {
         scsi_req_complete(&r->req, GOOD);
     }
@@ -1858,7 +1869,7 @@ static int32_t scsi_disk_dma_command(SCSIRequest *req, uint8_t *buf)
 {
     SCSIDiskReq *r = DO_UPCAST(SCSIDiskReq, req, req);
     SCSIDiskState *s = DO_UPCAST(SCSIDiskState, qdev, req->dev);
-    int32_t len;
+    uint32_t len;
     uint8_t command;
 
     command = buf[0];
@@ -1868,18 +1879,17 @@ static int32_t scsi_disk_dma_command(SCSIRequest *req, uint8_t *buf)
         return 0;
     }
 
+    len = scsi_data_cdb_length(r->req.cmd.buf);
     switch (command) {
     case READ_6:
     case READ_10:
     case READ_12:
     case READ_16:
-        len = r->req.cmd.xfer / s->qdev.blocksize;
-        DPRINTF("Read (sector %" PRId64 ", count %d)\n", r->req.cmd.lba, len);
+        DPRINTF("Read (sector %" PRId64 ", count %u)\n", r->req.cmd.lba, len);
         if (r->req.cmd.buf[1] & 0xe0) {
             goto illegal_request;
         }
-        if (r->req.cmd.lba > r->req.cmd.lba + len ||
-            r->req.cmd.lba + len - 1 > s->qdev.max_lba) {
+        if (!check_lba_range(s, r->req.cmd.lba, len)) {
             goto illegal_lba;
         }
         r->sector = r->req.cmd.lba * (s->qdev.blocksize / 512);
@@ -1900,15 +1910,13 @@ static int32_t scsi_disk_dma_command(SCSIRequest *req, uint8_t *buf)
     case VERIFY_10:
     case VERIFY_12:
     case VERIFY_16:
-        len = r->req.cmd.xfer / s->qdev.blocksize;
-        DPRINTF("Write %s(sector %" PRId64 ", count %d)\n",
+        DPRINTF("Write %s(sector %" PRId64 ", count %u)\n",
                 (command & 0xe) == 0xe ? "And Verify " : "",
                 r->req.cmd.lba, len);
         if (r->req.cmd.buf[1] & 0xe0) {
             goto illegal_request;
         }
-        if (r->req.cmd.lba > r->req.cmd.lba + len ||
-            r->req.cmd.lba + len - 1 > s->qdev.max_lba) {
+        if (!check_lba_range(s, r->req.cmd.lba, len)) {
             goto illegal_lba;
         }
         r->sector = r->req.cmd.lba * (s->qdev.blocksize / 512);
@@ -1965,7 +1973,6 @@ static void scsi_disk_resize_cb(void *opaque)
      * direct-access devices.
      */
     if (s->qdev.type == TYPE_DISK) {
-        scsi_device_set_ua(&s->qdev, SENSE_CODE(CAPACITY_CHANGED));
         scsi_device_report_change(&s->qdev, SENSE_CODE(CAPACITY_CHANGED));
     }
 }
diff --git a/hw/scsi-generic.c b/hw/scsi-generic.c
index a5eb663ec..d9045341b 100644
--- a/hw/scsi-generic.c
+++ b/hw/scsi-generic.c
@@ -400,11 +400,11 @@ static int scsi_generic_initfn(SCSIDevice *s)
         return -1;
     }
 
-    if (bdrv_get_on_error(s->conf.bs, 0) != BLOCK_ERR_STOP_ENOSPC) {
+    if (bdrv_get_on_error(s->conf.bs, 0) != BLOCKDEV_ON_ERROR_ENOSPC) {
         error_report("Device doesn't support drive option werror");
         return -1;
     }
-    if (bdrv_get_on_error(s->conf.bs, 1) != BLOCK_ERR_REPORT) {
+    if (bdrv_get_on_error(s->conf.bs, 1) != BLOCKDEV_ON_ERROR_REPORT) {
         error_report("Device doesn't support drive option rerror");
         return -1;
     }
diff --git a/hw/scsi.h b/hw/scsi.h
index 1aeee4659..b8f73577d 100644
--- a/hw/scsi.h
+++ b/hw/scsi.h
@@ -218,6 +218,8 @@ extern const struct SCSISense sense_code_WRITE_PROTECTED;
 
 #define SENSE_CODE(x) sense_code_ ## x
 
+uint32_t scsi_data_cdb_length(uint8_t *buf);
+uint32_t scsi_cdb_length(uint8_t *buf);
 int scsi_sense_valid(SCSISense sense);
 int scsi_build_sense(uint8_t *in_buf, int in_len,
                      uint8_t *buf, int len, bool fixed);
diff --git a/hw/sd.c b/hw/sd.c
index ec2640754..3c34d43ad 100644
--- a/hw/sd.c
+++ b/hw/sd.c
@@ -55,24 +55,28 @@ typedef enum {
     sd_illegal = -2,
 } sd_rsp_type_t;
 
+enum SDCardModes {
+    sd_inactive,
+    sd_card_identification_mode,
+    sd_data_transfer_mode,
+};
+
+enum SDCardStates {
+    sd_inactive_state = -1,
+    sd_idle_state = 0,
+    sd_ready_state,
+    sd_identification_state,
+    sd_standby_state,
+    sd_transfer_state,
+    sd_sendingdata_state,
+    sd_receivingdata_state,
+    sd_programming_state,
+    sd_disconnect_state,
+};
+
 struct SDState {
-    enum {
-        sd_inactive,
-        sd_card_identification_mode,
-        sd_data_transfer_mode,
-    } mode;
-    enum {
-        sd_inactive_state = -1,
-        sd_idle_state = 0,
-        sd_ready_state,
-        sd_identification_state,
-        sd_standby_state,
-        sd_transfer_state,
-        sd_sendingdata_state,
-        sd_receivingdata_state,
-        sd_programming_state,
-        sd_disconnect_state,
-    } state;
+    uint32_t mode;    /* current card mode, one of SDCardModes */
+    int32_t state;    /* current card state, one of SDCardStates */
     uint32_t ocr;
     uint8_t scr[8];
     uint8_t cid[16];
@@ -83,21 +87,22 @@ struct SDState {
     uint32_t vhs;
     bool wp_switch;
     unsigned long *wp_groups;
+    int32_t wpgrps_size;
     uint64_t size;
-    int blk_len;
+    uint32_t blk_len;
     uint32_t erase_start;
     uint32_t erase_end;
     uint8_t pwd[16];
-    int pwd_len;
-    int function_group[6];
+    uint32_t pwd_len;
+    uint8_t function_group[6];
 
     bool spi;
-    int current_cmd;
+    uint8_t current_cmd;
     /* True if we will handle the next command as an ACMD. Note that this does
      * *not* track the APP_CMD status bit!
      */
     bool expecting_acmd;
-    int blk_written;
+    uint32_t blk_written;
     uint64_t data_start;
     uint32_t data_offset;
     uint8_t data[512];
@@ -421,8 +426,9 @@ static void sd_reset(SDState *sd, BlockDriverState *bdrv)
     if (sd->wp_groups)
         g_free(sd->wp_groups);
     sd->wp_switch = bdrv ? bdrv_is_read_only(bdrv) : false;
-    sd->wp_groups = bitmap_new(sect);
-    memset(sd->function_group, 0, sizeof(int) * 6);
+    sd->wpgrps_size = sect;
+    sd->wp_groups = bitmap_new(sd->wpgrps_size);
+    memset(sd->function_group, 0, sizeof(sd->function_group));
     sd->erase_start = 0;
     sd->erase_end = 0;
     sd->size = size;
@@ -446,6 +452,38 @@ static const BlockDevOps sd_block_ops = {
     .change_media_cb = sd_cardchange,
 };
 
+static const VMStateDescription sd_vmstate = {
+    .name = "sd-card",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT32(mode, SDState),
+        VMSTATE_INT32(state, SDState),
+        VMSTATE_UINT8_ARRAY(cid, SDState, 16),
+        VMSTATE_UINT8_ARRAY(csd, SDState, 16),
+        VMSTATE_UINT16(rca, SDState),
+        VMSTATE_UINT32(card_status, SDState),
+        VMSTATE_PARTIAL_BUFFER(sd_status, SDState, 1),
+        VMSTATE_UINT32(vhs, SDState),
+        VMSTATE_BITMAP(wp_groups, SDState, 0, wpgrps_size),
+        VMSTATE_UINT32(blk_len, SDState),
+        VMSTATE_UINT32(erase_start, SDState),
+        VMSTATE_UINT32(erase_end, SDState),
+        VMSTATE_UINT8_ARRAY(pwd, SDState, 16),
+        VMSTATE_UINT32(pwd_len, SDState),
+        VMSTATE_UINT8_ARRAY(function_group, SDState, 6),
+        VMSTATE_UINT8(current_cmd, SDState),
+        VMSTATE_BOOL(expecting_acmd, SDState),
+        VMSTATE_UINT32(blk_written, SDState),
+        VMSTATE_UINT64(data_start, SDState),
+        VMSTATE_UINT32(data_offset, SDState),
+        VMSTATE_UINT8_ARRAY(data, SDState, 512),
+        VMSTATE_BUFFER_UNSAFE(buf, SDState, 1, 512),
+        VMSTATE_BOOL(enable, SDState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
 /* We do not model the chip select pin, so allow the board to select
    whether card should be in SSI or MMC/SD mode.  It is also up to the
    board to ensure that ssi transfers only occur when the chip select
@@ -463,6 +501,7 @@ SDState *sd_init(BlockDriverState *bs, bool is_spi)
         bdrv_attach_dev_nofail(sd->bdrv, sd);
         bdrv_set_dev_ops(sd->bdrv, &sd_block_ops, sd);
     }
+    vmstate_register(NULL, -1, &sd_vmstate, sd);
     return sd;
 }
 
@@ -476,19 +515,28 @@ void sd_set_cb(SDState *sd, qemu_irq readonly, qemu_irq insert)
 
 static void sd_erase(SDState *sd)
 {
-    int i, start, end;
+    int i;
+    uint64_t erase_start = sd->erase_start;
+    uint64_t erase_end = sd->erase_end;
+
     if (!sd->erase_start || !sd->erase_end) {
         sd->card_status |= ERASE_SEQ_ERROR;
         return;
     }
 
-    start = sd_addr_to_wpnum(sd->erase_start);
-    end = sd_addr_to_wpnum(sd->erase_end);
+    if (extract32(sd->ocr, OCR_CCS_BITN, 1)) {
+        /* High capacity memory card: erase units are 512 byte blocks */
+        erase_start *= 512;
+        erase_end *= 512;
+    }
+
+    erase_start = sd_addr_to_wpnum(erase_start);
+    erase_end = sd_addr_to_wpnum(erase_end);
     sd->erase_start = 0;
     sd->erase_end = 0;
     sd->csd[14] |= 0x40;
 
-    for (i = start; i <= end; i++) {
+    for (i = erase_start; i <= erase_end; i++) {
         if (test_bit(i, sd->wp_groups)) {
             sd->card_status |= WP_ERASE_SKIP;
         }
@@ -567,7 +615,7 @@ static void sd_lock_command(SDState *sd)
             sd->card_status |= LOCK_UNLOCK_FAILED;
             return;
         }
-        bitmap_zero(sd->wp_groups, sd_addr_to_wpnum(sd->size) + 1);
+        bitmap_zero(sd->wp_groups, sd->wpgrps_size);
         sd->csd[14] &= ~0x10;
         sd->card_status &= ~CARD_IS_LOCKED;
         sd->pwd_len = 0;
@@ -1407,7 +1455,7 @@ static void sd_blk_read(SDState *sd, uint64_t addr, uint32_t len)
 
     DPRINTF("sd_blk_read: addr = 0x%08llx, len = %d\n",
             (unsigned long long) addr, len);
-    if (!sd->bdrv || bdrv_read(sd->bdrv, addr >> 9, sd->buf, 1) == -1) {
+    if (!sd->bdrv || bdrv_read(sd->bdrv, addr >> 9, sd->buf, 1) < 0) {
         fprintf(stderr, "sd_blk_read: read error on host side\n");
         return;
     }
@@ -1415,7 +1463,7 @@ static void sd_blk_read(SDState *sd, uint64_t addr, uint32_t len)
     if (end > (addr & ~511) + 512) {
         memcpy(sd->data, sd->buf + (addr & 511), 512 - (addr & 511));
 
-        if (bdrv_read(sd->bdrv, end >> 9, sd->buf, 1) == -1) {
+        if (bdrv_read(sd->bdrv, end >> 9, sd->buf, 1) < 0) {
             fprintf(stderr, "sd_blk_read: read error on host side\n");
             return;
         }
@@ -1429,29 +1477,31 @@ static void sd_blk_write(SDState *sd, uint64_t addr, uint32_t len)
     uint64_t end = addr + len;
 
     if ((addr & 511) || len < 512)
-        if (!sd->bdrv || bdrv_read(sd->bdrv, addr >> 9, sd->buf, 1) == -1) {
+        if (!sd->bdrv || bdrv_read(sd->bdrv, addr >> 9, sd->buf, 1) < 0) {
             fprintf(stderr, "sd_blk_write: read error on host side\n");
             return;
         }
 
     if (end > (addr & ~511) + 512) {
         memcpy(sd->buf + (addr & 511), sd->data, 512 - (addr & 511));
-        if (bdrv_write(sd->bdrv, addr >> 9, sd->buf, 1) == -1) {
+        if (bdrv_write(sd->bdrv, addr >> 9, sd->buf, 1) < 0) {
             fprintf(stderr, "sd_blk_write: write error on host side\n");
             return;
         }
 
-        if (bdrv_read(sd->bdrv, end >> 9, sd->buf, 1) == -1) {
+        if (bdrv_read(sd->bdrv, end >> 9, sd->buf, 1) < 0) {
             fprintf(stderr, "sd_blk_write: read error on host side\n");
             return;
         }
         memcpy(sd->buf, sd->data + 512 - (addr & 511), end & 511);
-        if (bdrv_write(sd->bdrv, end >> 9, sd->buf, 1) == -1)
+        if (bdrv_write(sd->bdrv, end >> 9, sd->buf, 1) < 0) {
             fprintf(stderr, "sd_blk_write: write error on host side\n");
+        }
     } else {
         memcpy(sd->buf + (addr & 511), sd->data, len);
-        if (!sd->bdrv || bdrv_write(sd->bdrv, addr >> 9, sd->buf, 1) == -1)
+        if (!sd->bdrv || bdrv_write(sd->bdrv, addr >> 9, sd->buf, 1) < 0) {
             fprintf(stderr, "sd_blk_write: write error on host side\n");
+        }
     }
 }
 
diff --git a/hw/sd.h b/hw/sd.h
index 4eb9679ac..d9b97e446 100644
--- a/hw/sd.h
+++ b/hw/sd.h
@@ -50,6 +50,7 @@
 #define READY_FOR_DATA		(1 << 8)
 #define APP_CMD			(1 << 5)
 #define AKE_SEQ_ERROR		(1 << 3)
+#define OCR_CCS_BITN        30
 
 typedef enum {
     sd_none = -1,
diff --git a/hw/serial-isa.c b/hw/serial-isa.c
new file mode 100644
index 000000000..96c78f7f8
--- /dev/null
+++ b/hw/serial-isa.c
@@ -0,0 +1,130 @@
+/*
+ * QEMU 16550A UART emulation
+ *
+ * Copyright (c) 2003-2004 Fabrice Bellard
+ * Copyright (c) 2008 Citrix Systems, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "serial.h"
+#include "isa.h"
+
+typedef struct ISASerialState {
+    ISADevice dev;
+    uint32_t index;
+    uint32_t iobase;
+    uint32_t isairq;
+    SerialState state;
+} ISASerialState;
+
+static const int isa_serial_io[MAX_SERIAL_PORTS] = {
+    0x3f8, 0x2f8, 0x3e8, 0x2e8
+};
+static const int isa_serial_irq[MAX_SERIAL_PORTS] = {
+    4, 3, 4, 3
+};
+
+static int serial_isa_initfn(ISADevice *dev)
+{
+    static int index;
+    ISASerialState *isa = DO_UPCAST(ISASerialState, dev, dev);
+    SerialState *s = &isa->state;
+
+    if (isa->index == -1) {
+        isa->index = index;
+    }
+    if (isa->index >= MAX_SERIAL_PORTS) {
+        return -1;
+    }
+    if (isa->iobase == -1) {
+        isa->iobase = isa_serial_io[isa->index];
+    }
+    if (isa->isairq == -1) {
+        isa->isairq = isa_serial_irq[isa->index];
+    }
+    index++;
+
+    s->baudbase = 115200;
+    isa_init_irq(dev, &s->irq, isa->isairq);
+    serial_init_core(s);
+    qdev_set_legacy_instance_id(&dev->qdev, isa->iobase, 3);
+
+    memory_region_init_io(&s->io, &serial_io_ops, s, "serial", 8);
+    isa_register_ioport(dev, &s->io, isa->iobase);
+    return 0;
+}
+
+static const VMStateDescription vmstate_isa_serial = {
+    .name = "serial",
+    .version_id = 3,
+    .minimum_version_id = 2,
+    .fields = (VMStateField[]) {
+        VMSTATE_STRUCT(state, ISASerialState, 0, vmstate_serial, SerialState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static Property serial_isa_properties[] = {
+    DEFINE_PROP_UINT32("index",  ISASerialState, index,   -1),
+    DEFINE_PROP_HEX32("iobase",  ISASerialState, iobase,  -1),
+    DEFINE_PROP_UINT32("irq",    ISASerialState, isairq,  -1),
+    DEFINE_PROP_CHR("chardev",   ISASerialState, state.chr),
+    DEFINE_PROP_UINT32("wakeup", ISASerialState, state.wakeup, 0),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void serial_isa_class_initfn(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    ISADeviceClass *ic = ISA_DEVICE_CLASS(klass);
+    ic->init = serial_isa_initfn;
+    dc->vmsd = &vmstate_isa_serial;
+    dc->props = serial_isa_properties;
+}
+
+static TypeInfo serial_isa_info = {
+    .name          = "isa-serial",
+    .parent        = TYPE_ISA_DEVICE,
+    .instance_size = sizeof(ISASerialState),
+    .class_init    = serial_isa_class_initfn,
+};
+
+static void serial_register_types(void)
+{
+    type_register_static(&serial_isa_info);
+}
+
+type_init(serial_register_types)
+
+bool serial_isa_init(ISABus *bus, int index, CharDriverState *chr)
+{
+    ISADevice *dev;
+
+    dev = isa_try_create(bus, "isa-serial");
+    if (!dev) {
+        return false;
+    }
+    qdev_prop_set_uint32(&dev->qdev, "index", index);
+    qdev_prop_set_chr(&dev->qdev, "chardev", chr);
+    if (qdev_init(&dev->qdev) < 0) {
+        return false;
+    }
+    return true;
+}
diff --git a/hw/serial-pci.c b/hw/serial-pci.c
new file mode 100644
index 000000000..95dc5c8d2
--- /dev/null
+++ b/hw/serial-pci.c
@@ -0,0 +1,252 @@
+/*
+ * QEMU 16550A UART emulation
+ *
+ * Copyright (c) 2003-2004 Fabrice Bellard
+ * Copyright (c) 2008 Citrix Systems, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+/* see docs/specs/pci-serial.txt */
+
+#include "serial.h"
+#include "pci.h"
+
+#define PCI_SERIAL_MAX_PORTS 4
+
+typedef struct PCISerialState {
+    PCIDevice dev;
+    SerialState state;
+} PCISerialState;
+
+typedef struct PCIMultiSerialState {
+    PCIDevice    dev;
+    MemoryRegion iobar;
+    uint32_t     ports;
+    char         *name[PCI_SERIAL_MAX_PORTS];
+    SerialState  state[PCI_SERIAL_MAX_PORTS];
+    uint32_t     level[PCI_SERIAL_MAX_PORTS];
+    qemu_irq     *irqs;
+} PCIMultiSerialState;
+
+static int serial_pci_init(PCIDevice *dev)
+{
+    PCISerialState *pci = DO_UPCAST(PCISerialState, dev, dev);
+    SerialState *s = &pci->state;
+
+    s->baudbase = 115200;
+    serial_init_core(s);
+
+    pci->dev.config[PCI_INTERRUPT_PIN] = 0x01;
+    s->irq = pci->dev.irq[0];
+
+    memory_region_init_io(&s->io, &serial_io_ops, s, "serial", 8);
+    pci_register_bar(&pci->dev, 0, PCI_BASE_ADDRESS_SPACE_IO, &s->io);
+    return 0;
+}
+
+static void multi_serial_irq_mux(void *opaque, int n, int level)
+{
+    PCIMultiSerialState *pci = opaque;
+    int i, pending = 0;
+
+    pci->level[n] = level;
+    for (i = 0; i < pci->ports; i++) {
+        if (pci->level[i]) {
+            pending = 1;
+        }
+    }
+    qemu_set_irq(pci->dev.irq[0], pending);
+}
+
+static int multi_serial_pci_init(PCIDevice *dev)
+{
+    PCIDeviceClass *pc = PCI_DEVICE_GET_CLASS(dev);
+    PCIMultiSerialState *pci = DO_UPCAST(PCIMultiSerialState, dev, dev);
+    SerialState *s;
+    int i;
+
+    switch (pc->device_id) {
+    case 0x0003:
+        pci->ports = 2;
+        break;
+    case 0x0004:
+        pci->ports = 4;
+        break;
+    }
+    assert(pci->ports > 0);
+    assert(pci->ports <= PCI_SERIAL_MAX_PORTS);
+
+    pci->dev.config[PCI_INTERRUPT_PIN] = 0x01;
+    memory_region_init(&pci->iobar, "multiserial", 8 * pci->ports);
+    pci_register_bar(&pci->dev, 0, PCI_BASE_ADDRESS_SPACE_IO, &pci->iobar);
+    pci->irqs = qemu_allocate_irqs(multi_serial_irq_mux, pci,
+                                   pci->ports);
+
+    for (i = 0; i < pci->ports; i++) {
+        s = pci->state + i;
+        s->baudbase = 115200;
+        serial_init_core(s);
+        s->irq = pci->irqs[i];
+        pci->name[i] = g_strdup_printf("uart #%d", i+1);
+        memory_region_init_io(&s->io, &serial_io_ops, s, pci->name[i], 8);
+        memory_region_add_subregion(&pci->iobar, 8 * i, &s->io);
+    }
+    return 0;
+}
+
+static void serial_pci_exit(PCIDevice *dev)
+{
+    PCISerialState *pci = DO_UPCAST(PCISerialState, dev, dev);
+    SerialState *s = &pci->state;
+
+    serial_exit_core(s);
+    memory_region_destroy(&s->io);
+}
+
+static void multi_serial_pci_exit(PCIDevice *dev)
+{
+    PCIMultiSerialState *pci = DO_UPCAST(PCIMultiSerialState, dev, dev);
+    SerialState *s;
+    int i;
+
+    for (i = 0; i < pci->ports; i++) {
+        s = pci->state + i;
+        serial_exit_core(s);
+        memory_region_destroy(&s->io);
+        g_free(pci->name[i]);
+    }
+    memory_region_destroy(&pci->iobar);
+    qemu_free_irqs(pci->irqs);
+}
+
+static const VMStateDescription vmstate_pci_serial = {
+    .name = "pci-serial",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields      = (VMStateField[]) {
+        VMSTATE_PCI_DEVICE(dev, PCISerialState),
+        VMSTATE_STRUCT(state, PCISerialState, 0, vmstate_serial, SerialState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static const VMStateDescription vmstate_pci_multi_serial = {
+    .name = "pci-serial-multi",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields      = (VMStateField[]) {
+        VMSTATE_PCI_DEVICE(dev, PCIMultiSerialState),
+        VMSTATE_STRUCT_ARRAY(state, PCIMultiSerialState, PCI_SERIAL_MAX_PORTS,
+                             0, vmstate_serial, SerialState),
+        VMSTATE_UINT32_ARRAY(level, PCIMultiSerialState, PCI_SERIAL_MAX_PORTS),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static Property serial_pci_properties[] = {
+    DEFINE_PROP_CHR("chardev",  PCISerialState, state.chr),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static Property multi_2x_serial_pci_properties[] = {
+    DEFINE_PROP_CHR("chardev1",  PCIMultiSerialState, state[0].chr),
+    DEFINE_PROP_CHR("chardev2",  PCIMultiSerialState, state[1].chr),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static Property multi_4x_serial_pci_properties[] = {
+    DEFINE_PROP_CHR("chardev1",  PCIMultiSerialState, state[0].chr),
+    DEFINE_PROP_CHR("chardev2",  PCIMultiSerialState, state[1].chr),
+    DEFINE_PROP_CHR("chardev3",  PCIMultiSerialState, state[2].chr),
+    DEFINE_PROP_CHR("chardev4",  PCIMultiSerialState, state[3].chr),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void serial_pci_class_initfn(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    PCIDeviceClass *pc = PCI_DEVICE_CLASS(klass);
+    pc->init = serial_pci_init;
+    pc->exit = serial_pci_exit;
+    pc->vendor_id = 0x1b36; /* Red Hat */
+    pc->device_id = 0x0002;
+    pc->revision = 1;
+    pc->class_id = PCI_CLASS_COMMUNICATION_SERIAL;
+    dc->vmsd = &vmstate_pci_serial;
+    dc->props = serial_pci_properties;
+}
+
+static void multi_2x_serial_pci_class_initfn(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    PCIDeviceClass *pc = PCI_DEVICE_CLASS(klass);
+    pc->init = multi_serial_pci_init;
+    pc->exit = multi_serial_pci_exit;
+    pc->vendor_id = 0x1b36; /* Red Hat */
+    pc->device_id = 0x0003;
+    pc->revision = 1;
+    pc->class_id = PCI_CLASS_COMMUNICATION_SERIAL;
+    dc->vmsd = &vmstate_pci_multi_serial;
+    dc->props = multi_2x_serial_pci_properties;
+}
+
+static void multi_4x_serial_pci_class_initfn(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    PCIDeviceClass *pc = PCI_DEVICE_CLASS(klass);
+    pc->init = multi_serial_pci_init;
+    pc->exit = multi_serial_pci_exit;
+    pc->vendor_id = 0x1b36; /* Red Hat */
+    pc->device_id = 0x0004;
+    pc->revision = 1;
+    pc->class_id = PCI_CLASS_COMMUNICATION_SERIAL;
+    dc->vmsd = &vmstate_pci_multi_serial;
+    dc->props = multi_4x_serial_pci_properties;
+}
+
+static TypeInfo serial_pci_info = {
+    .name          = "pci-serial",
+    .parent        = TYPE_PCI_DEVICE,
+    .instance_size = sizeof(PCISerialState),
+    .class_init    = serial_pci_class_initfn,
+};
+
+static TypeInfo multi_2x_serial_pci_info = {
+    .name          = "pci-serial-2x",
+    .parent        = TYPE_PCI_DEVICE,
+    .instance_size = sizeof(PCIMultiSerialState),
+    .class_init    = multi_2x_serial_pci_class_initfn,
+};
+
+static TypeInfo multi_4x_serial_pci_info = {
+    .name          = "pci-serial-4x",
+    .parent        = TYPE_PCI_DEVICE,
+    .instance_size = sizeof(PCIMultiSerialState),
+    .class_init    = multi_4x_serial_pci_class_initfn,
+};
+
+static void serial_pci_register_types(void)
+{
+    type_register_static(&serial_pci_info);
+    type_register_static(&multi_2x_serial_pci_info);
+    type_register_static(&multi_4x_serial_pci_info);
+}
+
+type_init(serial_pci_register_types)
diff --git a/hw/serial.c b/hw/serial.c
index a421d1e7b..60283eab9 100644
--- a/hw/serial.c
+++ b/hw/serial.c
@@ -22,12 +22,11 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#include "hw.h"
+
+#include "serial.h"
 #include "qemu-char.h"
-#include "isa.h"
-#include "pc.h"
 #include "qemu-timer.h"
-#include "sysemu.h"
+#include "exec-memory.h"
 
 //#define DEBUG_SERIAL
 
@@ -93,8 +92,6 @@
 #define UART_FCR_RFR        0x02    /* RCVR Fifo Reset */
 #define UART_FCR_FE         0x01    /* FIFO Enable */
 
-#define UART_FIFO_LENGTH    16      /* 16550A Fifo Length */
-
 #define XMIT_FIFO           0
 #define RECV_FIFO           1
 #define MAX_XMIT_RETRY      4
@@ -107,64 +104,6 @@ do { fprintf(stderr, "serial: " fmt , ## __VA_ARGS__); } while (0)
 do {} while (0)
 #endif
 
-typedef struct SerialFIFO {
-    uint8_t data[UART_FIFO_LENGTH];
-    uint8_t count;
-    uint8_t itl;                        /* Interrupt Trigger Level */
-    uint8_t tail;
-    uint8_t head;
-} SerialFIFO;
-
-struct SerialState {
-    uint16_t divider;
-    uint8_t rbr; /* receive register */
-    uint8_t thr; /* transmit holding register */
-    uint8_t tsr; /* transmit shift register */
-    uint8_t ier;
-    uint8_t iir; /* read only */
-    uint8_t lcr;
-    uint8_t mcr;
-    uint8_t lsr; /* read only */
-    uint8_t msr; /* read only */
-    uint8_t scr;
-    uint8_t fcr;
-    uint8_t fcr_vmstate; /* we can't write directly this value
-                            it has side effects */
-    /* NOTE: this hidden state is necessary for tx irq generation as
-       it can be reset while reading iir */
-    int thr_ipending;
-    qemu_irq irq;
-    CharDriverState *chr;
-    int last_break_enable;
-    int it_shift;
-    int baudbase;
-    int tsr_retry;
-    uint32_t wakeup;
-
-    uint64_t last_xmit_ts;              /* Time when the last byte was successfully sent out of the tsr */
-    SerialFIFO recv_fifo;
-    SerialFIFO xmit_fifo;
-
-    struct QEMUTimer *fifo_timeout_timer;
-    int timeout_ipending;                   /* timeout interrupt pending state */
-    struct QEMUTimer *transmit_timer;
-
-
-    uint64_t char_transmit_time;               /* time to transmit a char in ticks*/
-    int poll_msl;
-
-    struct QEMUTimer *modem_status_poll;
-    MemoryRegion io;
-};
-
-typedef struct ISASerialState {
-    ISADevice dev;
-    uint32_t index;
-    uint32_t iobase;
-    uint32_t isairq;
-    SerialState state;
-} ISASerialState;
-
 static void serial_receive1(void *opaque, const uint8_t *buf, int size);
 
 static void fifo_clear(SerialState *s, int fifo)
@@ -367,7 +306,8 @@ static void serial_xmit(void *opaque)
 }
 
 
-static void serial_ioport_write(void *opaque, uint32_t addr, uint32_t val)
+static void serial_ioport_write(void *opaque, hwaddr addr, uint64_t val,
+                                unsigned size)
 {
     SerialState *s = opaque;
 
@@ -513,7 +453,7 @@ static void serial_ioport_write(void *opaque, uint32_t addr, uint32_t val)
     }
 }
 
-static uint32_t serial_ioport_read(void *opaque, uint32_t addr)
+static uint64_t serial_ioport_read(void *opaque, hwaddr addr, unsigned size)
 {
     SerialState *s = opaque;
     uint32_t ret;
@@ -682,12 +622,12 @@ static int serial_post_load(void *opaque, int version_id)
         s->fcr_vmstate = 0;
     }
     /* Initialize fcr via setter to perform essential side-effects */
-    serial_ioport_write(s, 0x02, s->fcr_vmstate);
+    serial_ioport_write(s, 0x02, s->fcr_vmstate, 1);
     serial_update_parameters(s);
     return 0;
 }
 
-static const VMStateDescription vmstate_serial = {
+const VMStateDescription vmstate_serial = {
     .name = "serial",
     .version_id = 3,
     .minimum_version_id = 2,
@@ -736,7 +676,7 @@ static void serial_reset(void *opaque)
     qemu_irq_lower(s->irq);
 }
 
-static void serial_init_core(SerialState *s)
+void serial_init_core(SerialState *s)
 {
     if (!s->chr) {
         fprintf(stderr, "Can't create serial device, empty char device\n");
@@ -754,6 +694,12 @@ static void serial_init_core(SerialState *s)
                           serial_event, s);
 }
 
+void serial_exit_core(SerialState *s)
+{
+    qemu_chr_add_handlers(s->chr, NULL, NULL, NULL, NULL);
+    qemu_unregister_reset(serial_reset, s);
+}
+
 /* Change the main reference oscillator frequency. */
 void serial_set_frequency(SerialState *s, uint32_t frequency)
 {
@@ -761,52 +707,14 @@ void serial_set_frequency(SerialState *s, uint32_t frequency)
     serial_update_parameters(s);
 }
 
-static const int isa_serial_io[MAX_SERIAL_PORTS] = { 0x3f8, 0x2f8, 0x3e8, 0x2e8 };
-static const int isa_serial_irq[MAX_SERIAL_PORTS] = { 4, 3, 4, 3 };
-
-static const MemoryRegionPortio serial_portio[] = {
-    { 0, 8, 1, .read = serial_ioport_read, .write = serial_ioport_write },
-    PORTIO_END_OF_LIST()
-};
-
-static const MemoryRegionOps serial_io_ops = {
-    .old_portio = serial_portio
-};
-
-static int serial_isa_initfn(ISADevice *dev)
-{
-    static int index;
-    ISASerialState *isa = DO_UPCAST(ISASerialState, dev, dev);
-    SerialState *s = &isa->state;
-
-    if (isa->index == -1)
-        isa->index = index;
-    if (isa->index >= MAX_SERIAL_PORTS)
-        return -1;
-    if (isa->iobase == -1)
-        isa->iobase = isa_serial_io[isa->index];
-    if (isa->isairq == -1)
-        isa->isairq = isa_serial_irq[isa->index];
-    index++;
-
-    s->baudbase = 115200;
-    isa_init_irq(dev, &s->irq, isa->isairq);
-    serial_init_core(s);
-    qdev_set_legacy_instance_id(&dev->qdev, isa->iobase, 3);
-
-    memory_region_init_io(&s->io, &serial_io_ops, s, "serial", 8);
-    isa_register_ioport(dev, &s->io, isa->iobase);
-    return 0;
-}
-
-static const VMStateDescription vmstate_isa_serial = {
-    .name = "serial",
-    .version_id = 3,
-    .minimum_version_id = 2,
-    .fields      = (VMStateField []) {
-        VMSTATE_STRUCT(state, ISASerialState, 0, vmstate_serial, SerialState),
-        VMSTATE_END_OF_LIST()
-    }
+const MemoryRegionOps serial_io_ops = {
+    .read = serial_ioport_read,
+    .write = serial_ioport_write,
+    .impl = {
+        .min_access_size = 1,
+        .max_access_size = 1,
+    },
+    .endianness = DEVICE_LITTLE_ENDIAN,
 };
 
 SerialState *serial_init(int base, qemu_irq irq, int baudbase,
@@ -823,25 +731,26 @@ SerialState *serial_init(int base, qemu_irq irq, int baudbase,
 
     vmstate_register(NULL, base, &vmstate_serial, s);
 
-    register_ioport_write(base, 8, 1, serial_ioport_write, s);
-    register_ioport_read(base, 8, 1, serial_ioport_read, s);
+    memory_region_init_io(&s->io, &serial_io_ops, s, "serial", 8);
+    memory_region_add_subregion(get_system_io(), base, &s->io);
+
     return s;
 }
 
 /* Memory mapped interface */
-static uint64_t serial_mm_read(void *opaque, target_phys_addr_t addr,
+static uint64_t serial_mm_read(void *opaque, hwaddr addr,
                                unsigned size)
 {
     SerialState *s = opaque;
-    return serial_ioport_read(s, addr >> s->it_shift);
+    return serial_ioport_read(s, addr >> s->it_shift, 1);
 }
 
-static void serial_mm_write(void *opaque, target_phys_addr_t addr,
+static void serial_mm_write(void *opaque, hwaddr addr,
                             uint64_t value, unsigned size)
 {
     SerialState *s = opaque;
     value &= ~0u >> (32 - (size * 8));
-    serial_ioport_write(s, addr >> s->it_shift, value);
+    serial_ioport_write(s, addr >> s->it_shift, value, 1);
 }
 
 static const MemoryRegionOps serial_mm_ops[3] = {
@@ -863,7 +772,7 @@ static const MemoryRegionOps serial_mm_ops[3] = {
 };
 
 SerialState *serial_mm_init(MemoryRegion *address_space,
-                            target_phys_addr_t base, int it_shift,
+                            hwaddr base, int it_shift,
                             qemu_irq irq, int baudbase,
                             CharDriverState *chr, enum device_endian end)
 {
@@ -886,35 +795,3 @@ SerialState *serial_mm_init(MemoryRegion *address_space,
     serial_update_msl(s);
     return s;
 }
-
-static Property serial_isa_properties[] = {
-    DEFINE_PROP_UINT32("index", ISASerialState, index,   -1),
-    DEFINE_PROP_HEX32("iobase", ISASerialState, iobase,  -1),
-    DEFINE_PROP_UINT32("irq",   ISASerialState, isairq,  -1),
-    DEFINE_PROP_CHR("chardev",  ISASerialState, state.chr),
-    DEFINE_PROP_UINT32("wakeup", ISASerialState, state.wakeup, 0),
-    DEFINE_PROP_END_OF_LIST(),
-};
-
-static void serial_isa_class_initfn(ObjectClass *klass, void *data)
-{
-    DeviceClass *dc = DEVICE_CLASS(klass);
-    ISADeviceClass *ic = ISA_DEVICE_CLASS(klass);
-    ic->init = serial_isa_initfn;
-    dc->vmsd = &vmstate_isa_serial;
-    dc->props = serial_isa_properties;
-}
-
-static TypeInfo serial_isa_info = {
-    .name          = "isa-serial",
-    .parent        = TYPE_ISA_DEVICE,
-    .instance_size = sizeof(ISASerialState),
-    .class_init    = serial_isa_class_initfn,
-};
-
-static void serial_register_types(void)
-{
-    type_register_static(&serial_isa_info);
-}
-
-type_init(serial_register_types)
diff --git a/hw/serial.h b/hw/serial.h
new file mode 100644
index 000000000..f1e3c4aaa
--- /dev/null
+++ b/hw/serial.h
@@ -0,0 +1,99 @@
+/*
+ * QEMU 16550A UART emulation
+ *
+ * Copyright (c) 2003-2004 Fabrice Bellard
+ * Copyright (c) 2008 Citrix Systems, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "hw.h"
+#include "sysemu.h"
+#include "memory.h"
+
+#define UART_FIFO_LENGTH    16      /* 16550A Fifo Length */
+
+typedef struct SerialFIFO {
+    uint8_t data[UART_FIFO_LENGTH];
+    uint8_t count;
+    uint8_t itl;                        /* Interrupt Trigger Level */
+    uint8_t tail;
+    uint8_t head;
+} SerialFIFO;
+
+struct SerialState {
+    uint16_t divider;
+    uint8_t rbr; /* receive register */
+    uint8_t thr; /* transmit holding register */
+    uint8_t tsr; /* transmit shift register */
+    uint8_t ier;
+    uint8_t iir; /* read only */
+    uint8_t lcr;
+    uint8_t mcr;
+    uint8_t lsr; /* read only */
+    uint8_t msr; /* read only */
+    uint8_t scr;
+    uint8_t fcr;
+    uint8_t fcr_vmstate; /* we can't write directly this value
+                            it has side effects */
+    /* NOTE: this hidden state is necessary for tx irq generation as
+       it can be reset while reading iir */
+    int thr_ipending;
+    qemu_irq irq;
+    CharDriverState *chr;
+    int last_break_enable;
+    int it_shift;
+    int baudbase;
+    int tsr_retry;
+    uint32_t wakeup;
+
+    /* Time when the last byte was successfully sent out of the tsr */
+    uint64_t last_xmit_ts;
+    SerialFIFO recv_fifo;
+    SerialFIFO xmit_fifo;
+
+    struct QEMUTimer *fifo_timeout_timer;
+    int timeout_ipending;           /* timeout interrupt pending state */
+    struct QEMUTimer *transmit_timer;
+
+
+    uint64_t char_transmit_time;    /* time to transmit a char in ticks */
+    int poll_msl;
+
+    struct QEMUTimer *modem_status_poll;
+    MemoryRegion io;
+};
+
+extern const VMStateDescription vmstate_serial;
+extern const MemoryRegionOps serial_io_ops;
+
+void serial_init_core(SerialState *s);
+void serial_exit_core(SerialState *s);
+void serial_set_frequency(SerialState *s, uint32_t frequency);
+
+/* legacy pre qom */
+SerialState *serial_init(int base, qemu_irq irq, int baudbase,
+                         CharDriverState *chr);
+SerialState *serial_mm_init(MemoryRegion *address_space,
+                            hwaddr base, int it_shift,
+                            qemu_irq irq, int baudbase,
+                            CharDriverState *chr, enum device_endian end);
+
+/* serial-isa.c */
+bool serial_isa_init(ISABus *bus, int index, CharDriverState *chr);
diff --git a/hw/sh.h b/hw/sh.h
index 40df18c5d..77bf8aad2 100644
--- a/hw/sh.h
+++ b/hw/sh.h
@@ -31,7 +31,7 @@ int sh7750_register_io_device(struct SH7750State *s,
 #define TMU012_FEAT_TOCR   (1 << 0)
 #define TMU012_FEAT_3CHAN  (1 << 1)
 #define TMU012_FEAT_EXTCLK (1 << 2)
-void tmu012_init(struct MemoryRegion *sysmem, target_phys_addr_t base,
+void tmu012_init(struct MemoryRegion *sysmem, hwaddr base,
                  int feat, uint32_t freq,
 		 qemu_irq ch0_irq, qemu_irq ch1_irq,
 		 qemu_irq ch2_irq0, qemu_irq ch2_irq1);
@@ -40,7 +40,7 @@ void tmu012_init(struct MemoryRegion *sysmem, target_phys_addr_t base,
 /* sh_serial.c */
 #define SH_SERIAL_FEAT_SCIF (1 << 0)
 void sh_serial_init(MemoryRegion *sysmem,
-                    target_phys_addr_t base, int feat,
+                    hwaddr base, int feat,
 		     uint32_t freq, CharDriverState *chr,
 		     qemu_irq eri_source,
 		     qemu_irq rxi_source,
diff --git a/hw/sh7750.c b/hw/sh7750.c
index e7129283d..8bcf0df96 100644
--- a/hw/sh7750.c
+++ b/hw/sh7750.c
@@ -197,19 +197,19 @@ static void portb_changed(SH7750State * s, uint16_t prev)
  Memory
 **********************************************************************/
 
-static void error_access(const char *kind, target_phys_addr_t addr)
+static void error_access(const char *kind, hwaddr addr)
 {
     fprintf(stderr, "%s to %s (0x" TARGET_FMT_plx ") not supported\n",
 	    kind, regname(addr), addr);
 }
 
-static void ignore_access(const char *kind, target_phys_addr_t addr)
+static void ignore_access(const char *kind, hwaddr addr)
 {
     fprintf(stderr, "%s to %s (0x" TARGET_FMT_plx ") ignored\n",
 	    kind, regname(addr), addr);
 }
 
-static uint32_t sh7750_mem_readb(void *opaque, target_phys_addr_t addr)
+static uint32_t sh7750_mem_readb(void *opaque, hwaddr addr)
 {
     switch (addr) {
     default:
@@ -218,7 +218,7 @@ static uint32_t sh7750_mem_readb(void *opaque, target_phys_addr_t addr)
     }
 }
 
-static uint32_t sh7750_mem_readw(void *opaque, target_phys_addr_t addr)
+static uint32_t sh7750_mem_readw(void *opaque, hwaddr addr)
 {
     SH7750State *s = opaque;
 
@@ -252,7 +252,7 @@ static uint32_t sh7750_mem_readw(void *opaque, target_phys_addr_t addr)
     }
 }
 
-static uint32_t sh7750_mem_readl(void *opaque, target_phys_addr_t addr)
+static uint32_t sh7750_mem_readl(void *opaque, hwaddr addr)
 {
     SH7750State *s = opaque;
 
@@ -301,7 +301,7 @@ static uint32_t sh7750_mem_readl(void *opaque, target_phys_addr_t addr)
 
 #define is_in_sdrmx(a, x) (a >= SH7750_SDMR ## x ## _A7 \
 			&& a <= (SH7750_SDMR ## x ## _A7 + SH7750_SDMR ## x ## _REGNB))
-static void sh7750_mem_writeb(void *opaque, target_phys_addr_t addr,
+static void sh7750_mem_writeb(void *opaque, hwaddr addr,
 			      uint32_t mem_value)
 {
 
@@ -314,7 +314,7 @@ static void sh7750_mem_writeb(void *opaque, target_phys_addr_t addr,
     abort();
 }
 
-static void sh7750_mem_writew(void *opaque, target_phys_addr_t addr,
+static void sh7750_mem_writew(void *opaque, hwaddr addr,
 			      uint32_t mem_value)
 {
     SH7750State *s = opaque;
@@ -366,7 +366,7 @@ static void sh7750_mem_writew(void *opaque, target_phys_addr_t addr,
     }
 }
 
-static void sh7750_mem_writel(void *opaque, target_phys_addr_t addr,
+static void sh7750_mem_writel(void *opaque, hwaddr addr,
 			      uint32_t mem_value)
 {
     SH7750State *s = opaque;
@@ -624,14 +624,14 @@ static struct intc_group groups_irl[] = {
 #define MM_UTLB_DATA     (7)
 #define MM_REGION_TYPE(addr)  ((addr & MM_REGION_MASK) >> 24)
 
-static uint64_t invalid_read(void *opaque, target_phys_addr_t addr)
+static uint64_t invalid_read(void *opaque, hwaddr addr)
 {
     abort();
 
     return 0;
 }
 
-static uint64_t sh7750_mmct_read(void *opaque, target_phys_addr_t addr,
+static uint64_t sh7750_mmct_read(void *opaque, hwaddr addr,
                                  unsigned size)
 {
     SH7750State *s = opaque;
@@ -669,13 +669,13 @@ static uint64_t sh7750_mmct_read(void *opaque, target_phys_addr_t addr,
     return ret;
 }
 
-static void invalid_write(void *opaque, target_phys_addr_t addr,
+static void invalid_write(void *opaque, hwaddr addr,
                           uint64_t mem_value)
 {
     abort();
 }
 
-static void sh7750_mmct_write(void *opaque, target_phys_addr_t addr,
+static void sh7750_mmct_write(void *opaque, hwaddr addr,
                               uint64_t mem_value, unsigned size)
 {
     SH7750State *s = opaque;
diff --git a/hw/sh_intc.c b/hw/sh_intc.c
index 7d31ced85..c3f77d509 100644
--- a/hw/sh_intc.c
+++ b/hw/sh_intc.c
@@ -219,7 +219,7 @@ static void sh_intc_toggle_mask(struct intc_desc *desc, intc_enum id,
 #endif
 }
 
-static uint64_t sh_intc_read(void *opaque, target_phys_addr_t offset,
+static uint64_t sh_intc_read(void *opaque, hwaddr offset,
                              unsigned size)
 {
     struct intc_desc *desc = opaque;
@@ -238,7 +238,7 @@ static uint64_t sh_intc_read(void *opaque, target_phys_addr_t offset,
     return *valuep;
 }
 
-static void sh_intc_write(void *opaque, target_phys_addr_t offset,
+static void sh_intc_write(void *opaque, hwaddr offset,
                           uint64_t value, unsigned size)
 {
     struct intc_desc *desc = opaque;
diff --git a/hw/sh_pci.c b/hw/sh_pci.c
index 0cfac46f7..fdec71b9e 100644
--- a/hw/sh_pci.c
+++ b/hw/sh_pci.c
@@ -41,7 +41,7 @@ typedef struct SHPCIState {
     uint32_t iobr;
 } SHPCIState;
 
-static void sh_pci_reg_write (void *p, target_phys_addr_t addr, uint64_t val,
+static void sh_pci_reg_write (void *p, hwaddr addr, uint64_t val,
                               unsigned size)
 {
     SHPCIState *pcic = p;
@@ -69,7 +69,7 @@ static void sh_pci_reg_write (void *p, target_phys_addr_t addr, uint64_t val,
     }
 }
 
-static uint64_t sh_pci_reg_read (void *p, target_phys_addr_t addr,
+static uint64_t sh_pci_reg_read (void *p, hwaddr addr,
                                  unsigned size)
 {
     SHPCIState *pcic = p;
diff --git a/hw/sh_serial.c b/hw/sh_serial.c
index 1d1883dd2..9da5d08fe 100644
--- a/hw/sh_serial.c
+++ b/hw/sh_serial.c
@@ -78,7 +78,7 @@ static void sh_serial_clear_fifo(sh_serial_state * s)
     s->rx_tail = 0;
 }
 
-static void sh_serial_write(void *opaque, target_phys_addr_t offs,
+static void sh_serial_write(void *opaque, hwaddr offs,
                             uint64_t val, unsigned size)
 {
     sh_serial_state *s = opaque;
@@ -187,11 +187,11 @@ static void sh_serial_write(void *opaque, target_phys_addr_t offs,
     }
 
     fprintf(stderr, "sh_serial: unsupported write to 0x%02"
-            TARGET_PRIxPHYS "\n", offs);
+            HWADDR_PRIx "\n", offs);
     abort();
 }
 
-static uint64_t sh_serial_read(void *opaque, target_phys_addr_t offs,
+static uint64_t sh_serial_read(void *opaque, hwaddr offs,
                                unsigned size)
 {
     sh_serial_state *s = opaque;
@@ -289,7 +289,7 @@ static uint64_t sh_serial_read(void *opaque, target_phys_addr_t offs,
 
     if (ret & ~((1 << 16) - 1)) {
         fprintf(stderr, "sh_serial: unsupported read from 0x%02"
-                TARGET_PRIxPHYS "\n", offs);
+                HWADDR_PRIx "\n", offs);
         abort();
     }
 
@@ -353,7 +353,7 @@ static const MemoryRegionOps sh_serial_ops = {
 };
 
 void sh_serial_init(MemoryRegion *sysmem,
-                    target_phys_addr_t base, int feat,
+                    hwaddr base, int feat,
                     uint32_t freq, CharDriverState *chr,
                     qemu_irq eri_source,
                     qemu_irq rxi_source,
diff --git a/hw/sh_timer.c b/hw/sh_timer.c
index 64bf604ba..c0365b114 100644
--- a/hw/sh_timer.c
+++ b/hw/sh_timer.c
@@ -59,7 +59,7 @@ static void sh_timer_update(sh_timer_state *s)
     s->int_level = new_level;
 }
 
-static uint32_t sh_timer_read(void *opaque, target_phys_addr_t offset)
+static uint32_t sh_timer_read(void *opaque, hwaddr offset)
 {
     sh_timer_state *s = (sh_timer_state *)opaque;
 
@@ -79,7 +79,7 @@ static uint32_t sh_timer_read(void *opaque, target_phys_addr_t offset)
     }
 }
 
-static void sh_timer_write(void *opaque, target_phys_addr_t offset,
+static void sh_timer_write(void *opaque, hwaddr offset,
                             uint32_t value)
 {
     sh_timer_state *s = (sh_timer_state *)opaque;
@@ -222,7 +222,7 @@ typedef struct {
     int feat;
 } tmu012_state;
 
-static uint64_t tmu012_read(void *opaque, target_phys_addr_t offset,
+static uint64_t tmu012_read(void *opaque, hwaddr offset,
                             unsigned size)
 {
     tmu012_state *s = (tmu012_state *)opaque;
@@ -253,7 +253,7 @@ static uint64_t tmu012_read(void *opaque, target_phys_addr_t offset,
     return 0;
 }
 
-static void tmu012_write(void *opaque, target_phys_addr_t offset,
+static void tmu012_write(void *opaque, hwaddr offset,
                         uint64_t value, unsigned size)
 {
     tmu012_state *s = (tmu012_state *)opaque;
@@ -303,7 +303,7 @@ static const MemoryRegionOps tmu012_ops = {
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-void tmu012_init(MemoryRegion *sysmem, target_phys_addr_t base,
+void tmu012_init(MemoryRegion *sysmem, hwaddr base,
                  int feat, uint32_t freq,
 		 qemu_irq ch0_irq, qemu_irq ch1_irq,
 		 qemu_irq ch2_irq0, qemu_irq ch2_irq1)
diff --git a/hw/sharpsl.h b/hw/sharpsl.h
index 0b3a774f2..13981a6d0 100644
--- a/hw/sharpsl.h
+++ b/hw/sharpsl.h
@@ -12,6 +12,6 @@
 /* zaurus.c */
 
 #define SL_PXA_PARAM_BASE	0xa0000a00
-void sl_bootparam_write(target_phys_addr_t ptr);
+void sl_bootparam_write(hwaddr ptr);
 
 #endif
diff --git a/hw/shix.c b/hw/shix.c
index dd9ce174f..b56dd54f7 100644
--- a/hw/shix.c
+++ b/hw/shix.c
@@ -37,11 +37,9 @@
 #define BIOS_FILENAME "shix_bios.bin"
 #define BIOS_ADDRESS 0xA0000000
 
-static void shix_init(ram_addr_t ram_size,
-               const char *boot_device,
-	       const char *kernel_filename, const char *kernel_cmdline,
-	       const char *initrd_filename, const char *cpu_model)
+static void shix_init(QEMUMachineInitArgs *args)
 {
+    const char *cpu_model = args->cpu_model;
     int ret;
     CPUSH4State *env;
     struct SH7750State *s;
diff --git a/hw/shpc.c b/hw/shpc.c
index a5baf246f..4597bbde7 100644
--- a/hw/shpc.c
+++ b/hw/shpc.c
@@ -466,13 +466,13 @@ static int shpc_cap_add_config(PCIDevice *d)
     return 0;
 }
 
-static uint64_t shpc_mmio_read(void *opaque, target_phys_addr_t addr,
+static uint64_t shpc_mmio_read(void *opaque, hwaddr addr,
                                unsigned size)
 {
     return shpc_read(opaque, addr, size);
 }
 
-static void shpc_mmio_write(void *opaque, target_phys_addr_t addr,
+static void shpc_mmio_write(void *opaque, hwaddr addr,
                             uint64_t val, unsigned size)
 {
     shpc_write(opaque, addr, val, size);
diff --git a/hw/slavio_intctl.c b/hw/slavio_intctl.c
index 7fdc3be08..6aafa8b23 100644
--- a/hw/slavio_intctl.c
+++ b/hw/slavio_intctl.c
@@ -78,7 +78,7 @@ typedef struct SLAVIO_INTCTLState {
 static void slavio_check_interrupts(SLAVIO_INTCTLState *s, int set_irqs);
 
 // per-cpu interrupt controller
-static uint64_t slavio_intctl_mem_readl(void *opaque, target_phys_addr_t addr,
+static uint64_t slavio_intctl_mem_readl(void *opaque, hwaddr addr,
                                         unsigned size)
 {
     SLAVIO_CPUINTCTLState *s = opaque;
@@ -98,7 +98,7 @@ static uint64_t slavio_intctl_mem_readl(void *opaque, target_phys_addr_t addr,
     return ret;
 }
 
-static void slavio_intctl_mem_writel(void *opaque, target_phys_addr_t addr,
+static void slavio_intctl_mem_writel(void *opaque, hwaddr addr,
                                      uint64_t val, unsigned size)
 {
     SLAVIO_CPUINTCTLState *s = opaque;
@@ -135,7 +135,7 @@ static const MemoryRegionOps slavio_intctl_mem_ops = {
 };
 
 // master system interrupt controller
-static uint64_t slavio_intctlm_mem_readl(void *opaque, target_phys_addr_t addr,
+static uint64_t slavio_intctlm_mem_readl(void *opaque, hwaddr addr,
                                          unsigned size)
 {
     SLAVIO_INTCTLState *s = opaque;
@@ -161,7 +161,7 @@ static uint64_t slavio_intctlm_mem_readl(void *opaque, target_phys_addr_t addr,
     return ret;
 }
 
-static void slavio_intctlm_mem_writel(void *opaque, target_phys_addr_t addr,
+static void slavio_intctlm_mem_writel(void *opaque, hwaddr addr,
                                       uint64_t val, unsigned size)
 {
     SLAVIO_INTCTLState *s = opaque;
diff --git a/hw/slavio_misc.c b/hw/slavio_misc.c
index 944835e88..682fb457f 100644
--- a/hw/slavio_misc.c
+++ b/hw/slavio_misc.c
@@ -107,7 +107,7 @@ static void slavio_set_power_fail(void *opaque, int irq, int power_failing)
     slavio_misc_update_irq(s);
 }
 
-static void slavio_cfg_mem_writeb(void *opaque, target_phys_addr_t addr,
+static void slavio_cfg_mem_writeb(void *opaque, hwaddr addr,
                                   uint64_t val, unsigned size)
 {
     MiscState *s = opaque;
@@ -117,7 +117,7 @@ static void slavio_cfg_mem_writeb(void *opaque, target_phys_addr_t addr,
     slavio_misc_update_irq(s);
 }
 
-static uint64_t slavio_cfg_mem_readb(void *opaque, target_phys_addr_t addr,
+static uint64_t slavio_cfg_mem_readb(void *opaque, hwaddr addr,
                                      unsigned size)
 {
     MiscState *s = opaque;
@@ -138,7 +138,7 @@ static const MemoryRegionOps slavio_cfg_mem_ops = {
     },
 };
 
-static void slavio_diag_mem_writeb(void *opaque, target_phys_addr_t addr,
+static void slavio_diag_mem_writeb(void *opaque, hwaddr addr,
                                    uint64_t val, unsigned size)
 {
     MiscState *s = opaque;
@@ -147,7 +147,7 @@ static void slavio_diag_mem_writeb(void *opaque, target_phys_addr_t addr,
     s->diag = val & 0xff;
 }
 
-static uint64_t slavio_diag_mem_readb(void *opaque, target_phys_addr_t addr,
+static uint64_t slavio_diag_mem_readb(void *opaque, hwaddr addr,
                                       unsigned size)
 {
     MiscState *s = opaque;
@@ -168,7 +168,7 @@ static const MemoryRegionOps slavio_diag_mem_ops = {
     },
 };
 
-static void slavio_mdm_mem_writeb(void *opaque, target_phys_addr_t addr,
+static void slavio_mdm_mem_writeb(void *opaque, hwaddr addr,
                                   uint64_t val, unsigned size)
 {
     MiscState *s = opaque;
@@ -177,7 +177,7 @@ static void slavio_mdm_mem_writeb(void *opaque, target_phys_addr_t addr,
     s->mctrl = val & 0xff;
 }
 
-static uint64_t slavio_mdm_mem_readb(void *opaque, target_phys_addr_t addr,
+static uint64_t slavio_mdm_mem_readb(void *opaque, hwaddr addr,
                                      unsigned size)
 {
     MiscState *s = opaque;
@@ -198,7 +198,7 @@ static const MemoryRegionOps slavio_mdm_mem_ops = {
     },
 };
 
-static void slavio_aux1_mem_writeb(void *opaque, target_phys_addr_t addr,
+static void slavio_aux1_mem_writeb(void *opaque, hwaddr addr,
                                    uint64_t val, unsigned size)
 {
     MiscState *s = opaque;
@@ -215,7 +215,7 @@ static void slavio_aux1_mem_writeb(void *opaque, target_phys_addr_t addr,
     s->aux1 = val & 0xff;
 }
 
-static uint64_t slavio_aux1_mem_readb(void *opaque, target_phys_addr_t addr,
+static uint64_t slavio_aux1_mem_readb(void *opaque, hwaddr addr,
                                       unsigned size)
 {
     MiscState *s = opaque;
@@ -236,7 +236,7 @@ static const MemoryRegionOps slavio_aux1_mem_ops = {
     },
 };
 
-static void slavio_aux2_mem_writeb(void *opaque, target_phys_addr_t addr,
+static void slavio_aux2_mem_writeb(void *opaque, hwaddr addr,
                                    uint64_t val, unsigned size)
 {
     MiscState *s = opaque;
@@ -252,7 +252,7 @@ static void slavio_aux2_mem_writeb(void *opaque, target_phys_addr_t addr,
     slavio_misc_update_irq(s);
 }
 
-static uint64_t slavio_aux2_mem_readb(void *opaque, target_phys_addr_t addr,
+static uint64_t slavio_aux2_mem_readb(void *opaque, hwaddr addr,
                                       unsigned size)
 {
     MiscState *s = opaque;
@@ -273,7 +273,7 @@ static const MemoryRegionOps slavio_aux2_mem_ops = {
     },
 };
 
-static void apc_mem_writeb(void *opaque, target_phys_addr_t addr,
+static void apc_mem_writeb(void *opaque, hwaddr addr,
                            uint64_t val, unsigned size)
 {
     APCState *s = opaque;
@@ -282,7 +282,7 @@ static void apc_mem_writeb(void *opaque, target_phys_addr_t addr,
     qemu_irq_raise(s->cpu_halt);
 }
 
-static uint64_t apc_mem_readb(void *opaque, target_phys_addr_t addr,
+static uint64_t apc_mem_readb(void *opaque, hwaddr addr,
                               unsigned size)
 {
     uint32_t ret = 0;
@@ -301,7 +301,7 @@ static const MemoryRegionOps apc_mem_ops = {
     }
 };
 
-static uint64_t slavio_sysctrl_mem_readl(void *opaque, target_phys_addr_t addr,
+static uint64_t slavio_sysctrl_mem_readl(void *opaque, hwaddr addr,
                                          unsigned size)
 {
     MiscState *s = opaque;
@@ -318,7 +318,7 @@ static uint64_t slavio_sysctrl_mem_readl(void *opaque, target_phys_addr_t addr,
     return ret;
 }
 
-static void slavio_sysctrl_mem_writel(void *opaque, target_phys_addr_t addr,
+static void slavio_sysctrl_mem_writel(void *opaque, hwaddr addr,
                                       uint64_t val, unsigned size)
 {
     MiscState *s = opaque;
@@ -346,7 +346,7 @@ static const MemoryRegionOps slavio_sysctrl_mem_ops = {
     },
 };
 
-static uint64_t slavio_led_mem_readw(void *opaque, target_phys_addr_t addr,
+static uint64_t slavio_led_mem_readw(void *opaque, hwaddr addr,
                                      unsigned size)
 {
     MiscState *s = opaque;
@@ -363,7 +363,7 @@ static uint64_t slavio_led_mem_readw(void *opaque, target_phys_addr_t addr,
     return ret;
 }
 
-static void slavio_led_mem_writew(void *opaque, target_phys_addr_t addr,
+static void slavio_led_mem_writew(void *opaque, hwaddr addr,
                                   uint64_t val, unsigned size)
 {
     MiscState *s = opaque;
diff --git a/hw/slavio_timer.c b/hw/slavio_timer.c
index 97edebb3b..c07ceb1de 100644
--- a/hw/slavio_timer.c
+++ b/hw/slavio_timer.c
@@ -130,7 +130,7 @@ static void slavio_timer_irq(void *opaque)
     }
 }
 
-static uint64_t slavio_timer_mem_readl(void *opaque, target_phys_addr_t addr,
+static uint64_t slavio_timer_mem_readl(void *opaque, hwaddr addr,
                                        unsigned size)
 {
     TimerContext *tc = opaque;
@@ -190,7 +190,7 @@ static uint64_t slavio_timer_mem_readl(void *opaque, target_phys_addr_t addr,
     return ret;
 }
 
-static void slavio_timer_mem_writel(void *opaque, target_phys_addr_t addr,
+static void slavio_timer_mem_writel(void *opaque, hwaddr addr,
                                     uint64_t val, unsigned size)
 {
     TimerContext *tc = opaque;
diff --git a/hw/sm501.c b/hw/sm501.c
index 786e07629..50324cda5 100644
--- a/hw/sm501.c
+++ b/hw/sm501.c
@@ -24,7 +24,7 @@
 
 #include <stdio.h>
 #include "hw.h"
-#include "pc.h"
+#include "serial.h"
 #include "console.h"
 #include "devices.h"
 #include "sysbus.h"
@@ -456,7 +456,7 @@ typedef struct SM501State {
     DisplayState *ds;
 
     /* status & internal resources */
-    target_phys_addr_t base;
+    hwaddr base;
     uint32_t local_mem_size_index;
     uint8_t * local_mem;
     MemoryRegion local_mem_region;
@@ -726,7 +726,7 @@ static void sm501_2d_operation(SM501State * s)
     }
 }
 
-static uint64_t sm501_system_config_read(void *opaque, target_phys_addr_t addr,
+static uint64_t sm501_system_config_read(void *opaque, hwaddr addr,
                                          unsigned size)
 {
     SM501State * s = (SM501State *)opaque;
@@ -779,7 +779,7 @@ static uint64_t sm501_system_config_read(void *opaque, target_phys_addr_t addr,
     return ret;
 }
 
-static void sm501_system_config_write(void *opaque, target_phys_addr_t addr,
+static void sm501_system_config_write(void *opaque, hwaddr addr,
                                       uint64_t value, unsigned size)
 {
     SM501State * s = (SM501State *)opaque;
@@ -837,7 +837,7 @@ static const MemoryRegionOps sm501_system_config_ops = {
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-static uint32_t sm501_palette_read(void *opaque, target_phys_addr_t addr)
+static uint32_t sm501_palette_read(void *opaque, hwaddr addr)
 {
     SM501State * s = (SM501State *)opaque;
     SM501_DPRINTF("sm501 palette read addr=%x\n", (int)addr);
@@ -850,7 +850,7 @@ static uint32_t sm501_palette_read(void *opaque, target_phys_addr_t addr)
 }
 
 static void sm501_palette_write(void *opaque,
-				target_phys_addr_t addr, uint32_t value)
+				hwaddr addr, uint32_t value)
 {
     SM501State * s = (SM501State *)opaque;
     SM501_DPRINTF("sm501 palette write addr=%x, val=%x\n",
@@ -863,7 +863,7 @@ static void sm501_palette_write(void *opaque,
     *(uint32_t*)&s->dc_palette[addr] = value;
 }
 
-static uint64_t sm501_disp_ctrl_read(void *opaque, target_phys_addr_t addr,
+static uint64_t sm501_disp_ctrl_read(void *opaque, hwaddr addr,
                                      unsigned size)
 {
     SM501State * s = (SM501State *)opaque;
@@ -958,7 +958,7 @@ static uint64_t sm501_disp_ctrl_read(void *opaque, target_phys_addr_t addr,
     return ret;
 }
 
-static void sm501_disp_ctrl_write(void *opaque, target_phys_addr_t addr,
+static void sm501_disp_ctrl_write(void *opaque, hwaddr addr,
                                   uint64_t value, unsigned size)
 {
     SM501State * s = (SM501State *)opaque;
@@ -1073,7 +1073,7 @@ static const MemoryRegionOps sm501_disp_ctrl_ops = {
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-static uint64_t sm501_2d_engine_read(void *opaque, target_phys_addr_t addr,
+static uint64_t sm501_2d_engine_read(void *opaque, hwaddr addr,
                                      unsigned size)
 {
     SM501State * s = (SM501State *)opaque;
@@ -1093,7 +1093,7 @@ static uint64_t sm501_2d_engine_read(void *opaque, target_phys_addr_t addr,
     return ret;
 }
 
-static void sm501_2d_engine_write(void *opaque, target_phys_addr_t addr,
+static void sm501_2d_engine_write(void *opaque, hwaddr addr,
                                   uint64_t value, unsigned size)
 {
     SM501State * s = (SM501State *)opaque;
@@ -1351,7 +1351,7 @@ static void sm501_draw_crt(SM501State * s)
 	} else {
 	    if (y_start >= 0) {
 		/* flush to display */
-		dpy_update(s->ds, 0, y_start, width, y - y_start);
+                dpy_gfx_update(s->ds, 0, y_start, width, y - y_start);
 		y_start = -1;
 	    }
 	}
@@ -1362,7 +1362,7 @@ static void sm501_draw_crt(SM501State * s)
 
     /* complete flush to display */
     if (y_start >= 0)
-	dpy_update(s->ds, 0, y_start, width, y - y_start);
+        dpy_gfx_update(s->ds, 0, y_start, width, y - y_start);
 
     /* clear dirty flags */
     if (page_min != ~0l) {
diff --git a/hw/smbus_ich9.c b/hw/smbus_ich9.c
new file mode 100644
index 000000000..6940583bb
--- /dev/null
+++ b/hw/smbus_ich9.c
@@ -0,0 +1,159 @@
+/*
+ * ACPI implementation
+ *
+ * Copyright (c) 2006 Fabrice Bellard
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2 as published by the Free Software Foundation.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>
+ */
+/*
+ *  Copyright (c) 2009 Isaku Yamahata <yamahata at valinux co jp>
+ *                     VA Linux Systems Japan K.K.
+ *  Copyright (C) 2012 Jason Baron <jbaron@redhat.com>
+ *
+ *  This is based on acpi.c, but heavily rewritten.
+ */
+#include "hw.h"
+#include "pc.h"
+#include "pm_smbus.h"
+#include "pci.h"
+#include "sysemu.h"
+#include "i2c.h"
+#include "smbus.h"
+
+#include "ich9.h"
+
+#define TYPE_ICH9_SMB_DEVICE "ICH9 SMB"
+#define ICH9_SMB_DEVICE(obj) \
+     OBJECT_CHECK(ICH9SMBState, (obj), TYPE_ICH9_SMB_DEVICE)
+
+typedef struct ICH9SMBState {
+    PCIDevice dev;
+
+    PMSMBus smb;
+    MemoryRegion mem_bar;
+} ICH9SMBState;
+
+static const VMStateDescription vmstate_ich9_smbus = {
+    .name = "ich9_smb",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .minimum_version_id_old = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_PCI_DEVICE(dev, struct ICH9SMBState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static void ich9_smb_ioport_writeb(void *opaque, hwaddr addr,
+                                   uint64_t val, unsigned size)
+{
+    ICH9SMBState *s = opaque;
+    uint8_t hostc = s->dev.config[ICH9_SMB_HOSTC];
+
+    if ((hostc & ICH9_SMB_HOSTC_HST_EN) && !(hostc & ICH9_SMB_HOSTC_I2C_EN)) {
+        uint64_t offset = addr - s->dev.io_regions[ICH9_SMB_SMB_BASE_BAR].addr;
+        smb_ioport_writeb(&s->smb, offset, val);
+    }
+}
+
+static uint64_t ich9_smb_ioport_readb(void *opaque, hwaddr addr,
+                                      unsigned size)
+{
+    ICH9SMBState *s = opaque;
+    uint8_t hostc = s->dev.config[ICH9_SMB_HOSTC];
+
+    if ((hostc & ICH9_SMB_HOSTC_HST_EN) && !(hostc & ICH9_SMB_HOSTC_I2C_EN)) {
+        uint64_t offset = addr - s->dev.io_regions[ICH9_SMB_SMB_BASE_BAR].addr;
+        return smb_ioport_readb(&s->smb, offset);
+    }
+
+    return 0xff;
+}
+
+static const MemoryRegionOps lpc_smb_mmio_ops = {
+    .read = ich9_smb_ioport_readb,
+    .write = ich9_smb_ioport_writeb,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+    .impl = {
+        .min_access_size = 1,
+        .max_access_size = 1,
+    },
+};
+
+static int ich9_smbus_initfn(PCIDevice *d)
+{
+    ICH9SMBState *s = ICH9_SMB_DEVICE(d);
+
+    /* TODO? D31IP.SMIP in chipset configuration space */
+    pci_config_set_interrupt_pin(d->config, 0x01); /* interrupt pin 1 */
+
+    pci_set_byte(d->config + ICH9_SMB_HOSTC, 0);
+
+    /*
+     * update parameters based on
+     * paralell_hds[0]
+     * serial_hds[0]
+     * serial_hds[0]
+     * fdc
+     *
+     * Is there any OS that depends on them?
+     */
+
+    /* TODO smb_io_base */
+    pci_set_byte(d->config + ICH9_SMB_HOSTC, 0);
+    /* TODO bar0, bar1: 64bit BAR support*/
+
+    memory_region_init_io(&s->mem_bar, &lpc_smb_mmio_ops, s, "ich9-smbus-bar",
+                            ICH9_SMB_SMB_BASE_SIZE);
+    pci_register_bar(d, ICH9_SMB_SMB_BASE_BAR, PCI_BASE_ADDRESS_SPACE_IO,
+                        &s->mem_bar);
+    pm_smbus_init(&d->qdev, &s->smb);
+    return 0;
+}
+
+static void ich9_smb_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
+
+    k->vendor_id = PCI_VENDOR_ID_INTEL;
+    k->device_id = PCI_DEVICE_ID_INTEL_ICH9_6;
+    k->revision = ICH9_A2_SMB_REVISION;
+    k->class_id = PCI_CLASS_SERIAL_SMBUS;
+    dc->no_user = 1;
+    dc->vmsd = &vmstate_ich9_smbus;
+    dc->desc = "ICH9 SMBUS Bridge";
+    k->init = ich9_smbus_initfn;
+}
+
+i2c_bus *ich9_smb_init(PCIBus *bus, int devfn, uint32_t smb_io_base)
+{
+    PCIDevice *d =
+        pci_create_simple_multifunction(bus, devfn, true, TYPE_ICH9_SMB_DEVICE);
+    ICH9SMBState *s = ICH9_SMB_DEVICE(d);
+    return s->smb.smbus;
+}
+
+static const TypeInfo ich9_smb_info = {
+    .name   = TYPE_ICH9_SMB_DEVICE,
+    .parent = TYPE_PCI_DEVICE,
+    .instance_size = sizeof(ICH9SMBState),
+    .class_init = ich9_smb_class_init,
+};
+
+static void ich9_smb_register(void)
+{
+    type_register_static(&ich9_smb_info);
+}
+
+type_init(ich9_smb_register);
diff --git a/hw/smc91c111.c b/hw/smc91c111.c
index d6ef302c6..4ceed01a1 100644
--- a/hw/smc91c111.c
+++ b/hw/smc91c111.c
@@ -276,7 +276,7 @@ static void smc91c111_reset(DeviceState *dev)
 #define SET_LOW(name, val) s->name = (s->name & 0xff00) | val
 #define SET_HIGH(name, val) s->name = (s->name & 0xff) | (val << 8)
 
-static void smc91c111_writeb(void *opaque, target_phys_addr_t offset,
+static void smc91c111_writeb(void *opaque, hwaddr offset,
                              uint32_t value)
 {
     smc91c111_state *s = (smc91c111_state *)opaque;
@@ -451,7 +451,7 @@ static void smc91c111_writeb(void *opaque, target_phys_addr_t offset,
     hw_error("smc91c111_write: Bad reg %d:%x\n", s->bank, (int)offset);
 }
 
-static uint32_t smc91c111_readb(void *opaque, target_phys_addr_t offset)
+static uint32_t smc91c111_readb(void *opaque, hwaddr offset)
 {
     smc91c111_state *s = (smc91c111_state *)opaque;
 
@@ -595,14 +595,14 @@ static uint32_t smc91c111_readb(void *opaque, target_phys_addr_t offset)
     return 0;
 }
 
-static void smc91c111_writew(void *opaque, target_phys_addr_t offset,
+static void smc91c111_writew(void *opaque, hwaddr offset,
                              uint32_t value)
 {
     smc91c111_writeb(opaque, offset, value & 0xff);
     smc91c111_writeb(opaque, offset + 1, value >> 8);
 }
 
-static void smc91c111_writel(void *opaque, target_phys_addr_t offset,
+static void smc91c111_writel(void *opaque, hwaddr offset,
                              uint32_t value)
 {
     /* 32-bit writes to offset 0xc only actually write to the bank select
@@ -612,7 +612,7 @@ static void smc91c111_writel(void *opaque, target_phys_addr_t offset,
     smc91c111_writew(opaque, offset + 2, value >> 16);
 }
 
-static uint32_t smc91c111_readw(void *opaque, target_phys_addr_t offset)
+static uint32_t smc91c111_readw(void *opaque, hwaddr offset)
 {
     uint32_t val;
     val = smc91c111_readb(opaque, offset);
@@ -620,7 +620,7 @@ static uint32_t smc91c111_readw(void *opaque, target_phys_addr_t offset)
     return val;
 }
 
-static uint32_t smc91c111_readl(void *opaque, target_phys_addr_t offset)
+static uint32_t smc91c111_readl(void *opaque, hwaddr offset)
 {
     uint32_t val;
     val = smc91c111_readw(opaque, offset);
diff --git a/hw/soc_dma.c b/hw/soc_dma.c
index 03bc8468d..50d5f84b4 100644
--- a/hw/soc_dma.c
+++ b/hw/soc_dma.c
@@ -64,7 +64,7 @@ struct dma_s {
 
     struct memmap_entry_s {
         enum soc_dma_port_type type;
-        target_phys_addr_t addr;
+        hwaddr addr;
         union {
            struct {
                void *opaque;
@@ -105,7 +105,7 @@ static void soc_dma_ch_run(void *opaque)
 }
 
 static inline struct memmap_entry_s *soc_dma_lookup(struct dma_s *dma,
-                target_phys_addr_t addr)
+                hwaddr addr)
 {
     struct memmap_entry_s *lo;
     int hi;
@@ -255,7 +255,7 @@ struct soc_dma_s *soc_dma_init(int n)
     return &s->soc;
 }
 
-void soc_dma_port_add_fifo(struct soc_dma_s *soc, target_phys_addr_t virt_base,
+void soc_dma_port_add_fifo(struct soc_dma_s *soc, hwaddr virt_base,
                 soc_dma_io_t fn, void *opaque, int out)
 {
     struct memmap_entry_s *entry;
@@ -308,7 +308,7 @@ void soc_dma_port_add_fifo(struct soc_dma_s *soc, target_phys_addr_t virt_base,
 }
 
 void soc_dma_port_add_mem(struct soc_dma_s *soc, uint8_t *phys_base,
-                target_phys_addr_t virt_base, size_t size)
+                hwaddr virt_base, size_t size)
 {
     struct memmap_entry_s *entry;
     struct dma_s *dma = (struct dma_s *) soc;
diff --git a/hw/soc_dma.h b/hw/soc_dma.h
index 904b26c5a..5948489ea 100644
--- a/hw/soc_dma.h
+++ b/hw/soc_dma.h
@@ -19,6 +19,7 @@
  */
 
 #include "memory.h"
+#include "hw/irq.h"
 
 struct soc_dma_s;
 struct soc_dma_ch_s;
@@ -51,7 +52,7 @@ struct soc_dma_ch_s {
     int bytes;
     /* Initialised by the DMA module, call soc_dma_ch_update after writing.  */
     enum soc_dma_access_type type[2];
-    target_phys_addr_t vaddr[2];	/* Updated by .transfer_fn().  */
+    hwaddr vaddr[2];	/* Updated by .transfer_fn().  */
     /* Private */
     void *paddr[2];
     soc_dma_io_t io_fn[2];
@@ -91,19 +92,19 @@ void soc_dma_ch_update(struct soc_dma_ch_s *ch);
 void soc_dma_reset(struct soc_dma_s *s);
 struct soc_dma_s *soc_dma_init(int n);
 
-void soc_dma_port_add_fifo(struct soc_dma_s *dma, target_phys_addr_t virt_base,
+void soc_dma_port_add_fifo(struct soc_dma_s *dma, hwaddr virt_base,
                 soc_dma_io_t fn, void *opaque, int out);
 void soc_dma_port_add_mem(struct soc_dma_s *dma, uint8_t *phys_base,
-                target_phys_addr_t virt_base, size_t size);
+                hwaddr virt_base, size_t size);
 
 static inline void soc_dma_port_add_fifo_in(struct soc_dma_s *dma,
-                target_phys_addr_t virt_base, soc_dma_io_t fn, void *opaque)
+                hwaddr virt_base, soc_dma_io_t fn, void *opaque)
 {
     return soc_dma_port_add_fifo(dma, virt_base, fn, opaque, 0);
 }
 
 static inline void soc_dma_port_add_fifo_out(struct soc_dma_s *dma,
-                target_phys_addr_t virt_base, soc_dma_io_t fn, void *opaque)
+                hwaddr virt_base, soc_dma_io_t fn, void *opaque)
 {
     return soc_dma_port_add_fifo(dma, virt_base, fn, opaque, 1);
 }
diff --git a/hw/spapr.c b/hw/spapr.c
index c34b767c6..ad3f0ea7f 100644
--- a/hw/spapr.c
+++ b/hw/spapr.c
@@ -46,7 +46,6 @@
 #include "kvm.h"
 #include "kvm_ppc.h"
 #include "pci.h"
-#include "vga-pci.h"
 
 #include "exec-memory.h"
 #include "hw/usb.h"
@@ -85,9 +84,11 @@
 
 #define PHANDLE_XICP            0x00001111
 
+#define HTAB_SIZE(spapr)        (1ULL << ((spapr)->htab_shift))
+
 sPAPREnvironment *spapr;
 
-int spapr_allocate_irq(int hint, enum xics_irq_type type)
+int spapr_allocate_irq(int hint, bool lsi)
 {
     int irq;
 
@@ -103,13 +104,13 @@ int spapr_allocate_irq(int hint, enum xics_irq_type type)
         return 0;
     }
 
-    xics_set_irq_type(spapr->icp, irq, type);
+    xics_set_irq_type(spapr->icp, irq, lsi);
 
     return irq;
 }
 
 /* Allocate block of consequtive IRQs, returns a number of the first */
-int spapr_allocate_irq_block(int num, enum xics_irq_type type)
+int spapr_allocate_irq_block(int num, bool lsi)
 {
     int first = -1;
     int i;
@@ -117,7 +118,7 @@ int spapr_allocate_irq_block(int num, enum xics_irq_type type)
     for (i = 0; i < num; ++i) {
         int irq;
 
-        irq = spapr_allocate_irq(0, type);
+        irq = spapr_allocate_irq(0, lsi);
         if (!irq) {
             return -1;
         }
@@ -134,12 +135,13 @@ int spapr_allocate_irq_block(int num, enum xics_irq_type type)
     return first;
 }
 
-static int spapr_set_associativity(void *fdt, sPAPREnvironment *spapr)
+static int spapr_fixup_cpu_dt(void *fdt, sPAPREnvironment *spapr)
 {
     int ret = 0, offset;
     CPUPPCState *env;
     char cpu_model[32];
     int smt = kvmppc_smt_threads();
+    uint32_t pft_size_prop[] = {0, cpu_to_be32(spapr->htab_shift)};
 
     assert(spapr->cpu_model);
 
@@ -163,8 +165,16 @@ static int spapr_set_associativity(void *fdt, sPAPREnvironment *spapr)
             return offset;
         }
 
-        ret = fdt_setprop(fdt, offset, "ibm,associativity", associativity,
-                          sizeof(associativity));
+        if (nb_numa_nodes > 1) {
+            ret = fdt_setprop(fdt, offset, "ibm,associativity", associativity,
+                              sizeof(associativity));
+            if (ret < 0) {
+                return ret;
+            }
+        }
+
+        ret = fdt_setprop(fdt, offset, "ibm,pft-size",
+                          pft_size_prop, sizeof(pft_size_prop));
         if (ret < 0) {
             return ret;
         }
@@ -206,45 +216,37 @@ static size_t create_page_sizes_prop(CPUPPCState *env, uint32_t *prop,
     return (p - prop) * sizeof(uint32_t);
 }
 
+#define _FDT(exp) \
+    do { \
+        int ret = (exp);                                           \
+        if (ret < 0) {                                             \
+            fprintf(stderr, "qemu: error creating device tree: %s: %s\n", \
+                    #exp, fdt_strerror(ret));                      \
+            exit(1);                                               \
+        }                                                          \
+    } while (0)
+
+
 static void *spapr_create_fdt_skel(const char *cpu_model,
-                                   target_phys_addr_t rma_size,
-                                   target_phys_addr_t initrd_base,
-                                   target_phys_addr_t initrd_size,
-                                   target_phys_addr_t kernel_size,
+                                   hwaddr initrd_base,
+                                   hwaddr initrd_size,
+                                   hwaddr kernel_size,
                                    const char *boot_device,
                                    const char *kernel_cmdline,
-                                   long hash_shift)
+                                   uint32_t epow_irq)
 {
     void *fdt;
     CPUPPCState *env;
-    uint64_t mem_reg_property[2];
     uint32_t start_prop = cpu_to_be32(initrd_base);
     uint32_t end_prop = cpu_to_be32(initrd_base + initrd_size);
-    uint32_t pft_size_prop[] = {0, cpu_to_be32(hash_shift)};
     char hypertas_prop[] = "hcall-pft\0hcall-term\0hcall-dabr\0hcall-interrupt"
         "\0hcall-tce\0hcall-vio\0hcall-splpar\0hcall-bulk";
     char qemu_hypertas_prop[] = "hcall-memop1";
+    uint32_t refpoints[] = {cpu_to_be32(0x4), cpu_to_be32(0x4)};
     uint32_t interrupt_server_ranges_prop[] = {0, cpu_to_be32(smp_cpus)};
-    int i;
     char *modelname;
-    int smt = kvmppc_smt_threads();
+    int i, smt = kvmppc_smt_threads();
     unsigned char vec5[] = {0x0, 0x0, 0x0, 0x0, 0x0, 0x80};
-    uint32_t refpoints[] = {cpu_to_be32(0x4), cpu_to_be32(0x4)};
-    uint32_t associativity[] = {cpu_to_be32(0x4), cpu_to_be32(0x0),
-                                cpu_to_be32(0x0), cpu_to_be32(0x0),
-                                cpu_to_be32(0x0)};
-    char mem_name[32];
-    target_phys_addr_t node0_size, mem_start;
-
-#define _FDT(exp) \
-    do { \
-        int ret = (exp);                                           \
-        if (ret < 0) {                                             \
-            fprintf(stderr, "qemu: error creating device tree: %s: %s\n", \
-                    #exp, fdt_strerror(ret));                      \
-            exit(1);                                               \
-        }                                                          \
-    } while (0)
 
     fdt = g_malloc0(FDT_MAX_SIZE);
     _FDT((fdt_create(fdt, FDT_MAX_SIZE)));
@@ -289,55 +291,6 @@ static void *spapr_create_fdt_skel(const char *cpu_model,
 
     _FDT((fdt_end_node(fdt)));
 
-    /* memory node(s) */
-    node0_size = (nb_numa_nodes > 1) ? node_mem[0] : ram_size;
-    if (rma_size > node0_size) {
-        rma_size = node0_size;
-    }
-
-    /* RMA */
-    mem_reg_property[0] = 0;
-    mem_reg_property[1] = cpu_to_be64(rma_size);
-    _FDT((fdt_begin_node(fdt, "memory@0")));
-    _FDT((fdt_property_string(fdt, "device_type", "memory")));
-    _FDT((fdt_property(fdt, "reg", mem_reg_property,
-        sizeof(mem_reg_property))));
-    _FDT((fdt_property(fdt, "ibm,associativity", associativity,
-        sizeof(associativity))));
-    _FDT((fdt_end_node(fdt)));
-
-    /* RAM: Node 0 */
-    if (node0_size > rma_size) {
-        mem_reg_property[0] = cpu_to_be64(rma_size);
-        mem_reg_property[1] = cpu_to_be64(node0_size - rma_size);
-
-        sprintf(mem_name, "memory@" TARGET_FMT_lx, rma_size);
-        _FDT((fdt_begin_node(fdt, mem_name)));
-        _FDT((fdt_property_string(fdt, "device_type", "memory")));
-        _FDT((fdt_property(fdt, "reg", mem_reg_property,
-                           sizeof(mem_reg_property))));
-        _FDT((fdt_property(fdt, "ibm,associativity", associativity,
-                           sizeof(associativity))));
-        _FDT((fdt_end_node(fdt)));
-    }
-
-    /* RAM: Node 1 and beyond */
-    mem_start = node0_size;
-    for (i = 1; i < nb_numa_nodes; i++) {
-        mem_reg_property[0] = cpu_to_be64(mem_start);
-        mem_reg_property[1] = cpu_to_be64(node_mem[i]);
-        associativity[3] = associativity[4] = cpu_to_be32(i);
-        sprintf(mem_name, "memory@" TARGET_FMT_lx, mem_start);
-        _FDT((fdt_begin_node(fdt, mem_name)));
-        _FDT((fdt_property_string(fdt, "device_type", "memory")));
-        _FDT((fdt_property(fdt, "reg", mem_reg_property,
-            sizeof(mem_reg_property))));
-        _FDT((fdt_property(fdt, "ibm,associativity", associativity,
-            sizeof(associativity))));
-        _FDT((fdt_end_node(fdt)));
-        mem_start += node_mem[i];
-    }
-
     /* cpus */
     _FDT((fdt_begin_node(fdt, "cpus")));
 
@@ -389,8 +342,6 @@ static void *spapr_create_fdt_skel(const char *cpu_model,
         _FDT((fdt_property_cell(fdt, "timebase-frequency", tbfreq)));
         _FDT((fdt_property_cell(fdt, "clock-frequency", cpufreq)));
         _FDT((fdt_property_cell(fdt, "ibm,slb-size", env->slb_nr)));
-        _FDT((fdt_property(fdt, "ibm,pft-size",
-                           pft_size_prop, sizeof(pft_size_prop))));
         _FDT((fdt_property_string(fdt, "status", "okay")));
         _FDT((fdt_property(fdt, "64-bit", NULL, 0)));
 
@@ -453,6 +404,8 @@ static void *spapr_create_fdt_skel(const char *cpu_model,
     _FDT((fdt_property(fdt, "ibm,associativity-reference-points",
         refpoints, sizeof(refpoints))));
 
+    _FDT((fdt_property_cell(fdt, "rtas-error-log-max", RTAS_ERROR_LOG_MAX)));
+
     _FDT((fdt_end_node(fdt)));
 
     /* interrupt controller */
@@ -483,16 +436,81 @@ static void *spapr_create_fdt_skel(const char *cpu_model,
 
     _FDT((fdt_end_node(fdt)));
 
+    /* event-sources */
+    spapr_events_fdt_skel(fdt, epow_irq);
+
     _FDT((fdt_end_node(fdt))); /* close root node */
     _FDT((fdt_finish(fdt)));
 
     return fdt;
 }
 
+static int spapr_populate_memory(sPAPREnvironment *spapr, void *fdt)
+{
+    uint32_t associativity[] = {cpu_to_be32(0x4), cpu_to_be32(0x0),
+                                cpu_to_be32(0x0), cpu_to_be32(0x0),
+                                cpu_to_be32(0x0)};
+    char mem_name[32];
+    hwaddr node0_size, mem_start;
+    uint64_t mem_reg_property[2];
+    int i, off;
+
+    /* memory node(s) */
+    node0_size = (nb_numa_nodes > 1) ? node_mem[0] : ram_size;
+    if (spapr->rma_size > node0_size) {
+        spapr->rma_size = node0_size;
+    }
+
+    /* RMA */
+    mem_reg_property[0] = 0;
+    mem_reg_property[1] = cpu_to_be64(spapr->rma_size);
+    off = fdt_add_subnode(fdt, 0, "memory@0");
+    _FDT(off);
+    _FDT((fdt_setprop_string(fdt, off, "device_type", "memory")));
+    _FDT((fdt_setprop(fdt, off, "reg", mem_reg_property,
+                      sizeof(mem_reg_property))));
+    _FDT((fdt_setprop(fdt, off, "ibm,associativity", associativity,
+                      sizeof(associativity))));
+
+    /* RAM: Node 0 */
+    if (node0_size > spapr->rma_size) {
+        mem_reg_property[0] = cpu_to_be64(spapr->rma_size);
+        mem_reg_property[1] = cpu_to_be64(node0_size - spapr->rma_size);
+
+        sprintf(mem_name, "memory@" TARGET_FMT_lx, spapr->rma_size);
+        off = fdt_add_subnode(fdt, 0, mem_name);
+        _FDT(off);
+        _FDT((fdt_setprop_string(fdt, off, "device_type", "memory")));
+        _FDT((fdt_setprop(fdt, off, "reg", mem_reg_property,
+                          sizeof(mem_reg_property))));
+        _FDT((fdt_setprop(fdt, off, "ibm,associativity", associativity,
+                          sizeof(associativity))));
+    }
+
+    /* RAM: Node 1 and beyond */
+    mem_start = node0_size;
+    for (i = 1; i < nb_numa_nodes; i++) {
+        mem_reg_property[0] = cpu_to_be64(mem_start);
+        mem_reg_property[1] = cpu_to_be64(node_mem[i]);
+        associativity[3] = associativity[4] = cpu_to_be32(i);
+        sprintf(mem_name, "memory@" TARGET_FMT_lx, mem_start);
+        off = fdt_add_subnode(fdt, 0, mem_name);
+        _FDT(off);
+        _FDT((fdt_setprop_string(fdt, off, "device_type", "memory")));
+        _FDT((fdt_setprop(fdt, off, "reg", mem_reg_property,
+                          sizeof(mem_reg_property))));
+        _FDT((fdt_setprop(fdt, off, "ibm,associativity", associativity,
+                          sizeof(associativity))));
+        mem_start += node_mem[i];
+    }
+
+    return 0;
+}
+
 static void spapr_finalize_fdt(sPAPREnvironment *spapr,
-                               target_phys_addr_t fdt_addr,
-                               target_phys_addr_t rtas_addr,
-                               target_phys_addr_t rtas_size)
+                               hwaddr fdt_addr,
+                               hwaddr rtas_addr,
+                               hwaddr rtas_size)
 {
     int ret;
     void *fdt;
@@ -503,6 +521,12 @@ static void spapr_finalize_fdt(sPAPREnvironment *spapr,
     /* open out the base tree into a temp buffer for the final tweaks */
     _FDT((fdt_open_into(spapr->fdt_skel, fdt, FDT_MAX_SIZE)));
 
+    ret = spapr_populate_memory(spapr, fdt);
+    if (ret < 0) {
+        fprintf(stderr, "couldn't setup memory nodes in fdt\n");
+        exit(1);
+    }
+
     ret = spapr_populate_vdevice(spapr->vio_bus, fdt);
     if (ret < 0) {
         fprintf(stderr, "couldn't setup vio devices in fdt\n");
@@ -525,11 +549,9 @@ static void spapr_finalize_fdt(sPAPREnvironment *spapr,
     }
 
     /* Advertise NUMA via ibm,associativity */
-    if (nb_numa_nodes > 1) {
-        ret = spapr_set_associativity(fdt, spapr);
-        if (ret < 0) {
-            fprintf(stderr, "Couldn't set up NUMA device tree properties\n");
-        }
+    ret = spapr_fixup_cpu_dt(fdt, spapr);
+    if (ret < 0) {
+        fprintf(stderr, "Couldn't finalize CPU device tree properties\n");
     }
 
     if (!spapr->has_graphics) {
@@ -554,17 +576,53 @@ static uint64_t translate_kernel_address(void *opaque, uint64_t addr)
     return (addr & 0x0fffffff) + KERNEL_LOAD_ADDR;
 }
 
-static void emulate_spapr_hypercall(CPUPPCState *env)
+static void emulate_spapr_hypercall(PowerPCCPU *cpu)
 {
-    env->gpr[3] = spapr_hypercall(env, env->gpr[3], &env->gpr[4]);
+    CPUPPCState *env = &cpu->env;
+
+    if (msr_pr) {
+        hcall_dprintf("Hypercall made with MSR[PR]=1\n");
+        env->gpr[3] = H_PRIVILEGE;
+    } else {
+        env->gpr[3] = spapr_hypercall(cpu, env->gpr[3], &env->gpr[4]);
+    }
 }
 
-static void spapr_reset(void *opaque)
+static void spapr_reset_htab(sPAPREnvironment *spapr)
 {
-    sPAPREnvironment *spapr = (sPAPREnvironment *)opaque;
+    long shift;
+
+    /* allocate hash page table.  For now we always make this 16mb,
+     * later we should probably make it scale to the size of guest
+     * RAM */
+
+    shift = kvmppc_reset_htab(spapr->htab_shift);
 
-    /* flush out the hash table */
-    memset(spapr->htab, 0, spapr->htab_size);
+    if (shift > 0) {
+        /* Kernel handles htab, we don't need to allocate one */
+        spapr->htab_shift = shift;
+    } else {
+        if (!spapr->htab) {
+            /* Allocate an htab if we don't yet have one */
+            spapr->htab = qemu_memalign(HTAB_SIZE(spapr), HTAB_SIZE(spapr));
+        }
+
+        /* And clear it */
+        memset(spapr->htab, 0, HTAB_SIZE(spapr));
+    }
+
+    /* Update the RMA size if necessary */
+    if (spapr->vrma_adjust) {
+        spapr->rma_size = kvmppc_rma_size(ram_size, spapr->htab_shift);
+    }
+}
+
+static void ppc_spapr_reset(void)
+{
+    /* Reset the hash table & recalc the RMA */
+    spapr_reset_htab(spapr);
+
+    qemu_devices_reset();
 
     /* Load the fdt */
     spapr_finalize_fdt(spapr, spapr->fdt_addr, spapr->rtas_addr,
@@ -581,19 +639,31 @@ static void spapr_reset(void *opaque)
 static void spapr_cpu_reset(void *opaque)
 {
     PowerPCCPU *cpu = opaque;
+    CPUPPCState *env = &cpu->env;
 
     cpu_reset(CPU(cpu));
+
+    /* All CPUs start halted.  CPU0 is unhalted from the machine level
+     * reset code and the rest are explicitly started up by the guest
+     * using an RTAS call */
+    env->halted = 1;
+
+    env->spr[SPR_HIOR] = 0;
+
+    env->external_htab = spapr->htab;
+    env->htab_base = -1;
+    env->htab_mask = HTAB_SIZE(spapr) - 1;
+    env->spr[SPR_SDR1] = (unsigned long)spapr->htab |
+        (spapr->htab_shift - 18);
 }
 
 /* Returns whether we want to use VGA or not */
 static int spapr_vga_init(PCIBus *pci_bus)
 {
     switch (vga_interface_type) {
-    case VGA_STD:
-        pci_vga_init(pci_bus);
-        return 1;
     case VGA_NONE:
-        return 0;
+    case VGA_STD:
+        return pci_vga_init(pci_bus) != NULL;
     default:
         fprintf(stderr, "This vga model is not supported,"
                 "currently it only supports -vga std\n");
@@ -603,24 +673,24 @@ static int spapr_vga_init(PCIBus *pci_bus)
 }
 
 /* pSeries LPAR / sPAPR hardware init */
-static void ppc_spapr_init(ram_addr_t ram_size,
-                           const char *boot_device,
-                           const char *kernel_filename,
-                           const char *kernel_cmdline,
-                           const char *initrd_filename,
-                           const char *cpu_model)
+static void ppc_spapr_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t ram_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
+    const char *boot_device = args->boot_device;
     PowerPCCPU *cpu;
     CPUPPCState *env;
     PCIHostState *phb;
     int i;
     MemoryRegion *sysmem = get_system_memory();
     MemoryRegion *ram = g_new(MemoryRegion, 1);
-    target_phys_addr_t rma_alloc_size, rma_size;
+    hwaddr rma_alloc_size;
     uint32_t initrd_base = 0;
     long kernel_size = 0, initrd_size = 0;
     long load_limit, rtas_limit, fw_size;
-    long pteg_shift = 17;
     char *filename;
 
     msi_supported = true;
@@ -637,20 +707,46 @@ static void ppc_spapr_init(ram_addr_t ram_size,
         hw_error("qemu: Unable to create RMA\n");
         exit(1);
     }
+
     if (rma_alloc_size && (rma_alloc_size < ram_size)) {
-        rma_size = rma_alloc_size;
+        spapr->rma_size = rma_alloc_size;
     } else {
-        rma_size = ram_size;
+        spapr->rma_size = ram_size;
+
+        /* With KVM, we don't actually know whether KVM supports an
+         * unbounded RMA (PR KVM) or is limited by the hash table size
+         * (HV KVM using VRMA), so we always assume the latter
+         *
+         * In that case, we also limit the initial allocations for RTAS
+         * etc... to 256M since we have no way to know what the VRMA size
+         * is going to be as it depends on the size of the hash table
+         * isn't determined yet.
+         */
+        if (kvm_enabled()) {
+            spapr->vrma_adjust = 1;
+            spapr->rma_size = MIN(spapr->rma_size, 0x10000000);
+        }
     }
 
     /* We place the device tree and RTAS just below either the top of the RMA,
      * or just below 2GB, whichever is lowere, so that it can be
      * processed with 32-bit real mode code if necessary */
-    rtas_limit = MIN(rma_size, 0x80000000);
+    rtas_limit = MIN(spapr->rma_size, 0x80000000);
     spapr->rtas_addr = rtas_limit - RTAS_MAX_SIZE;
     spapr->fdt_addr = spapr->rtas_addr - FDT_MAX_SIZE;
     load_limit = spapr->fdt_addr - FW_OVERHEAD;
 
+    /* We aim for a hash table of size 1/128 the size of RAM.  The
+     * normal rule of thumb is 1/64 the size of RAM, but that's much
+     * more than needed for the Linux guests we support. */
+    spapr->htab_shift = 18; /* Minimum architected size */
+    while (spapr->htab_shift <= 46) {
+        if ((1ULL << (spapr->htab_shift + 7)) >= ram_size) {
+            break;
+        }
+        spapr->htab_shift++;
+    }
+
     /* init CPUs */
     if (cpu_model == NULL) {
         cpu_model = kvm_enabled() ? "host" : "POWER7";
@@ -665,11 +761,16 @@ static void ppc_spapr_init(ram_addr_t ram_size,
 
         /* Set time-base frequency to 512 MHz */
         cpu_ppc_tb_init(env, TIMEBASE_FREQ);
-        qemu_register_reset(spapr_cpu_reset, cpu);
 
-        env->hreset_vector = 0x60;
+        /* PAPR always has exception vectors in RAM not ROM */
         env->hreset_excp_prefix = 0;
-        env->gpr[3] = env->cpu_index;
+
+        /* Tell KVM that we're in PAPR mode */
+        if (kvm_enabled()) {
+            kvmppc_set_papr(env);
+        }
+
+        qemu_register_reset(spapr_cpu_reset, cpu);
     }
 
     /* allocate RAM */
@@ -683,27 +784,6 @@ static void ppc_spapr_init(ram_addr_t ram_size,
         memory_region_add_subregion(sysmem, nonrma_base, ram);
     }
 
-    /* allocate hash page table.  For now we always make this 16mb,
-     * later we should probably make it scale to the size of guest
-     * RAM */
-    spapr->htab_size = 1ULL << (pteg_shift + 7);
-    spapr->htab = qemu_memalign(spapr->htab_size, spapr->htab_size);
-
-    for (env = first_cpu; env != NULL; env = env->next_cpu) {
-        env->external_htab = spapr->htab;
-        env->htab_base = -1;
-        env->htab_mask = spapr->htab_size - 1;
-
-        /* Tell KVM that we're in PAPR mode */
-        env->spr[SPR_SDR1] = (unsigned long)spapr->htab |
-                             ((pteg_shift + 7) - 18);
-        env->spr[SPR_HIOR] = 0;
-
-        if (kvm_enabled()) {
-            kvmppc_set_papr(env);
-        }
-    }
-
     filename = qemu_find_file(QEMU_FILE_TYPE_BIOS, "spapr-rtas.bin");
     spapr->rtas_size = load_image_targphys(filename, spapr->rtas_addr,
                                            rtas_limit - spapr->rtas_addr);
@@ -723,6 +803,9 @@ static void ppc_spapr_init(ram_addr_t ram_size,
     spapr->icp = xics_system_init(XICS_IRQS);
     spapr->next_irq = 16;
 
+    /* Set up EPOW events infrastructure */
+    spapr_events_init(spapr);
+
     /* Set up IOMMU */
     spapr_iommu_init();
 
@@ -768,7 +851,7 @@ static void ppc_spapr_init(ram_addr_t ram_size,
         spapr->has_graphics = true;
     }
 
-    if (usb_enabled) {
+    if (usb_enabled(spapr->has_graphics)) {
         pci_create_simple(phb->bus, -1, "pci-ohci");
         if (spapr->has_graphics) {
             usbdevice_create("keyboard");
@@ -776,7 +859,7 @@ static void ppc_spapr_init(ram_addr_t ram_size,
         }
     }
 
-    if (rma_size < (MIN_RMA_SLOF << 20)) {
+    if (spapr->rma_size < (MIN_RMA_SLOF << 20)) {
         fprintf(stderr, "qemu: pSeries SLOF firmware requires >= "
                 "%ldM guest RMA (Real Mode Area memory)\n", MIN_RMA_SLOF);
         exit(1);
@@ -827,26 +910,20 @@ static void ppc_spapr_init(ram_addr_t ram_size,
 
     spapr->entry_point = 0x100;
 
-    /* SLOF will startup the secondary CPUs using RTAS */
-    for (env = first_cpu; env != NULL; env = env->next_cpu) {
-        env->halted = 1;
-    }
-
     /* Prepare the device tree */
-    spapr->fdt_skel = spapr_create_fdt_skel(cpu_model, rma_size,
+    spapr->fdt_skel = spapr_create_fdt_skel(cpu_model,
                                             initrd_base, initrd_size,
                                             kernel_size,
                                             boot_device, kernel_cmdline,
-                                            pteg_shift + 7);
+                                            spapr->epow_irq);
     assert(spapr->fdt_skel != NULL);
-
-    qemu_register_reset(spapr_reset, spapr);
 }
 
 static QEMUMachine spapr_machine = {
     .name = "pseries",
     .desc = "pSeries Logical Partition (PAPR compliant)",
     .init = ppc_spapr_init,
+    .reset = ppc_spapr_reset,
     .max_cpus = MAX_CPUS,
     .no_parallel = 1,
     .use_scsi = 1,
diff --git a/hw/spapr.h b/hw/spapr.h
index ac34a171e..efe7f5758 100644
--- a/hw/spapr.h
+++ b/hw/spapr.h
@@ -13,10 +13,12 @@ typedef struct sPAPREnvironment {
     QLIST_HEAD(, sPAPRPHBState) phbs;
     struct icp_state *icp;
 
-    target_phys_addr_t ram_limit;
+    hwaddr ram_limit;
     void *htab;
-    long htab_size;
-    target_phys_addr_t fdt_addr, rtas_addr;
+    long htab_shift;
+    hwaddr rma_size;
+    int vrma_adjust;
+    hwaddr fdt_addr, rtas_addr;
     long rtas_size;
     void *fdt_skel;
     target_ulong entry_point;
@@ -24,6 +26,9 @@ typedef struct sPAPREnvironment {
     int rtc_offset;
     char *cpu_model;
     bool has_graphics;
+
+    uint32_t epow_irq;
+    Notifier epow_notifier;
 } sPAPREnvironment;
 
 #define H_SUCCESS         0
@@ -281,25 +286,25 @@ extern sPAPREnvironment *spapr;
     do { } while (0)
 #endif
 
-typedef target_ulong (*spapr_hcall_fn)(CPUPPCState *env, sPAPREnvironment *spapr,
+typedef target_ulong (*spapr_hcall_fn)(PowerPCCPU *cpu, sPAPREnvironment *spapr,
                                        target_ulong opcode,
                                        target_ulong *args);
 
 void spapr_register_hypercall(target_ulong opcode, spapr_hcall_fn fn);
-target_ulong spapr_hypercall(CPUPPCState *env, target_ulong opcode,
+target_ulong spapr_hypercall(PowerPCCPU *cpu, target_ulong opcode,
                              target_ulong *args);
 
-int spapr_allocate_irq(int hint, enum xics_irq_type type);
-int spapr_allocate_irq_block(int num, enum xics_irq_type type);
+int spapr_allocate_irq(int hint, bool lsi);
+int spapr_allocate_irq_block(int num, bool lsi);
 
 static inline int spapr_allocate_msi(int hint)
 {
-    return spapr_allocate_irq(hint, XICS_MSI);
+    return spapr_allocate_irq(hint, false);
 }
 
 static inline int spapr_allocate_lsi(int hint)
 {
-    return spapr_allocate_irq(hint, XICS_LSI);
+    return spapr_allocate_irq(hint, true);
 }
 
 static inline uint32_t rtas_ld(target_ulong phys, int n)
@@ -319,8 +324,8 @@ void spapr_rtas_register(const char *name, spapr_rtas_fn fn);
 target_ulong spapr_rtas_call(sPAPREnvironment *spapr,
                              uint32_t token, uint32_t nargs, target_ulong args,
                              uint32_t nret, target_ulong rets);
-int spapr_rtas_device_tree_setup(void *fdt, target_phys_addr_t rtas_addr,
-                                 target_phys_addr_t rtas_size);
+int spapr_rtas_device_tree_setup(void *fdt, hwaddr rtas_addr,
+                                 hwaddr rtas_size);
 
 #define SPAPR_TCE_PAGE_SHIFT   12
 #define SPAPR_TCE_PAGE_SIZE    (1ULL << SPAPR_TCE_PAGE_SHIFT)
@@ -333,9 +338,16 @@ typedef struct sPAPRTCE {
 #define SPAPR_VIO_BASE_LIOBN    0x00000000
 #define SPAPR_PCI_BASE_LIOBN    0x80000000
 
+#define RTAS_ERROR_LOG_MAX      2048
+
+
 void spapr_iommu_init(void);
+void spapr_events_init(sPAPREnvironment *spapr);
+void spapr_events_fdt_skel(void *fdt, uint32_t epow_irq);
 DMAContext *spapr_tce_new_dma_context(uint32_t liobn, size_t window_size);
 void spapr_tce_free(DMAContext *dma);
+void spapr_tce_reset(DMAContext *dma);
+void spapr_tce_set_bypass(DMAContext *dma, bool bypass);
 int spapr_dma_dt(void *fdt, int node_off, const char *propname,
                  uint32_t liobn, uint64_t window, uint32_t size);
 int spapr_tcet_dma_dt(void *fdt, int node_off, const char *propname,
diff --git a/hw/spapr_events.c b/hw/spapr_events.c
new file mode 100644
index 000000000..18ccd4a9e
--- /dev/null
+++ b/hw/spapr_events.c
@@ -0,0 +1,321 @@
+/*
+ * QEMU PowerPC pSeries Logical Partition (aka sPAPR) hardware System Emulator
+ *
+ * RTAS events handling
+ *
+ * Copyright (c) 2012 David Gibson, IBM Corporation.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ *
+ */
+#include "cpu.h"
+#include "sysemu.h"
+#include "qemu-char.h"
+#include "hw/qdev.h"
+#include "device_tree.h"
+
+#include "hw/spapr.h"
+#include "hw/spapr_vio.h"
+
+#include <libfdt.h>
+
+struct rtas_error_log {
+    uint32_t summary;
+#define RTAS_LOG_VERSION_MASK                   0xff000000
+#define   RTAS_LOG_VERSION_6                    0x06000000
+#define RTAS_LOG_SEVERITY_MASK                  0x00e00000
+#define   RTAS_LOG_SEVERITY_ALREADY_REPORTED    0x00c00000
+#define   RTAS_LOG_SEVERITY_FATAL               0x00a00000
+#define   RTAS_LOG_SEVERITY_ERROR               0x00800000
+#define   RTAS_LOG_SEVERITY_ERROR_SYNC          0x00600000
+#define   RTAS_LOG_SEVERITY_WARNING             0x00400000
+#define   RTAS_LOG_SEVERITY_EVENT               0x00200000
+#define   RTAS_LOG_SEVERITY_NO_ERROR            0x00000000
+#define RTAS_LOG_DISPOSITION_MASK               0x00180000
+#define   RTAS_LOG_DISPOSITION_FULLY_RECOVERED  0x00000000
+#define   RTAS_LOG_DISPOSITION_LIMITED_RECOVERY 0x00080000
+#define   RTAS_LOG_DISPOSITION_NOT_RECOVERED    0x00100000
+#define RTAS_LOG_OPTIONAL_PART_PRESENT          0x00040000
+#define RTAS_LOG_INITIATOR_MASK                 0x0000f000
+#define   RTAS_LOG_INITIATOR_UNKNOWN            0x00000000
+#define   RTAS_LOG_INITIATOR_CPU                0x00001000
+#define   RTAS_LOG_INITIATOR_PCI                0x00002000
+#define   RTAS_LOG_INITIATOR_MEMORY             0x00004000
+#define   RTAS_LOG_INITIATOR_HOTPLUG            0x00006000
+#define RTAS_LOG_TARGET_MASK                    0x00000f00
+#define   RTAS_LOG_TARGET_UNKNOWN               0x00000000
+#define   RTAS_LOG_TARGET_CPU                   0x00000100
+#define   RTAS_LOG_TARGET_PCI                   0x00000200
+#define   RTAS_LOG_TARGET_MEMORY                0x00000400
+#define   RTAS_LOG_TARGET_HOTPLUG               0x00000600
+#define RTAS_LOG_TYPE_MASK                      0x000000ff
+#define   RTAS_LOG_TYPE_OTHER                   0x00000000
+#define   RTAS_LOG_TYPE_RETRY                   0x00000001
+#define   RTAS_LOG_TYPE_TCE_ERR                 0x00000002
+#define   RTAS_LOG_TYPE_INTERN_DEV_FAIL         0x00000003
+#define   RTAS_LOG_TYPE_TIMEOUT                 0x00000004
+#define   RTAS_LOG_TYPE_DATA_PARITY             0x00000005
+#define   RTAS_LOG_TYPE_ADDR_PARITY             0x00000006
+#define   RTAS_LOG_TYPE_CACHE_PARITY            0x00000007
+#define   RTAS_LOG_TYPE_ADDR_INVALID            0x00000008
+#define   RTAS_LOG_TYPE_ECC_UNCORR              0x00000009
+#define   RTAS_LOG_TYPE_ECC_CORR                0x0000000a
+#define   RTAS_LOG_TYPE_EPOW                    0x00000040
+    uint32_t extended_length;
+} QEMU_PACKED;
+
+struct rtas_event_log_v6 {
+    uint8_t b0;
+#define RTAS_LOG_V6_B0_VALID                          0x80
+#define RTAS_LOG_V6_B0_UNRECOVERABLE_ERROR            0x40
+#define RTAS_LOG_V6_B0_RECOVERABLE_ERROR              0x20
+#define RTAS_LOG_V6_B0_DEGRADED_OPERATION             0x10
+#define RTAS_LOG_V6_B0_PREDICTIVE_ERROR               0x08
+#define RTAS_LOG_V6_B0_NEW_LOG                        0x04
+#define RTAS_LOG_V6_B0_BIGENDIAN                      0x02
+    uint8_t _resv1;
+    uint8_t b2;
+#define RTAS_LOG_V6_B2_POWERPC_FORMAT                 0x80
+#define RTAS_LOG_V6_B2_LOG_FORMAT_MASK                0x0f
+#define   RTAS_LOG_V6_B2_LOG_FORMAT_PLATFORM_EVENT    0x0e
+    uint8_t _resv2[9];
+    uint32_t company;
+#define RTAS_LOG_V6_COMPANY_IBM                 0x49424d00 /* IBM<null> */
+} QEMU_PACKED;
+
+struct rtas_event_log_v6_section_header {
+    uint16_t section_id;
+    uint16_t section_length;
+    uint8_t section_version;
+    uint8_t section_subtype;
+    uint16_t creator_component_id;
+} QEMU_PACKED;
+
+struct rtas_event_log_v6_maina {
+#define RTAS_LOG_V6_SECTION_ID_MAINA                0x5048 /* PH */
+    struct rtas_event_log_v6_section_header hdr;
+    uint32_t creation_date; /* BCD: YYYYMMDD */
+    uint32_t creation_time; /* BCD: HHMMSS00 */
+    uint8_t _platform1[8];
+    char creator_id;
+    uint8_t _resv1[2];
+    uint8_t section_count;
+    uint8_t _resv2[4];
+    uint8_t _platform2[8];
+    uint32_t plid;
+    uint8_t _platform3[4];
+} QEMU_PACKED;
+
+struct rtas_event_log_v6_mainb {
+#define RTAS_LOG_V6_SECTION_ID_MAINB                0x5548 /* UH */
+    struct rtas_event_log_v6_section_header hdr;
+    uint8_t subsystem_id;
+    uint8_t _platform1;
+    uint8_t event_severity;
+    uint8_t event_subtype;
+    uint8_t _platform2[4];
+    uint8_t _resv1[2];
+    uint16_t action_flags;
+    uint8_t _resv2[4];
+} QEMU_PACKED;
+
+struct rtas_event_log_v6_epow {
+#define RTAS_LOG_V6_SECTION_ID_EPOW                 0x4550 /* EP */
+    struct rtas_event_log_v6_section_header hdr;
+    uint8_t sensor_value;
+#define RTAS_LOG_V6_EPOW_ACTION_RESET                    0
+#define RTAS_LOG_V6_EPOW_ACTION_WARN_COOLING             1
+#define RTAS_LOG_V6_EPOW_ACTION_WARN_POWER               2
+#define RTAS_LOG_V6_EPOW_ACTION_SYSTEM_SHUTDOWN          3
+#define RTAS_LOG_V6_EPOW_ACTION_SYSTEM_HALT              4
+#define RTAS_LOG_V6_EPOW_ACTION_MAIN_ENCLOSURE           5
+#define RTAS_LOG_V6_EPOW_ACTION_POWER_OFF                7
+    uint8_t event_modifier;
+#define RTAS_LOG_V6_EPOW_MODIFIER_NORMAL                 1
+#define RTAS_LOG_V6_EPOW_MODIFIER_ON_UPS                 2
+#define RTAS_LOG_V6_EPOW_MODIFIER_CRITICAL               3
+#define RTAS_LOG_V6_EPOW_MODIFIER_TEMPERATURE            4
+    uint8_t extended_modifier;
+#define RTAS_LOG_V6_EPOW_XMODIFIER_SYSTEM_WIDE           0
+#define RTAS_LOG_V6_EPOW_XMODIFIER_PARTITION_SPECIFIC    1
+    uint8_t _resv;
+    uint64_t reason_code;
+} QEMU_PACKED;
+
+struct epow_log_full {
+    struct rtas_error_log hdr;
+    struct rtas_event_log_v6 v6hdr;
+    struct rtas_event_log_v6_maina maina;
+    struct rtas_event_log_v6_mainb mainb;
+    struct rtas_event_log_v6_epow epow;
+} QEMU_PACKED;
+
+#define EVENT_MASK_INTERNAL_ERRORS           0x80000000
+#define EVENT_MASK_EPOW                      0x40000000
+#define EVENT_MASK_HOTPLUG                   0x10000000
+#define EVENT_MASK_IO                        0x08000000
+
+#define _FDT(exp) \
+    do { \
+        int ret = (exp);                                           \
+        if (ret < 0) {                                             \
+            fprintf(stderr, "qemu: error creating device tree: %s: %s\n", \
+                    #exp, fdt_strerror(ret));                      \
+            exit(1);                                               \
+        }                                                          \
+    } while (0)
+
+void spapr_events_fdt_skel(void *fdt, uint32_t epow_irq)
+{
+    uint32_t epow_irq_ranges[] = {cpu_to_be32(epow_irq), cpu_to_be32(1)};
+    uint32_t epow_interrupts[] = {cpu_to_be32(epow_irq), 0};
+
+    _FDT((fdt_begin_node(fdt, "event-sources")));
+
+    _FDT((fdt_property(fdt, "interrupt-controller", NULL, 0)));
+    _FDT((fdt_property_cell(fdt, "#interrupt-cells", 2)));
+    _FDT((fdt_property(fdt, "interrupt-ranges",
+                       epow_irq_ranges, sizeof(epow_irq_ranges))));
+
+    _FDT((fdt_begin_node(fdt, "epow-events")));
+    _FDT((fdt_property(fdt, "interrupts",
+                       epow_interrupts, sizeof(epow_interrupts))));
+    _FDT((fdt_end_node(fdt)));
+
+    _FDT((fdt_end_node(fdt)));
+}
+
+static struct epow_log_full *pending_epow;
+static uint32_t next_plid;
+
+static void spapr_powerdown_req(Notifier *n, void *opaque)
+{
+    sPAPREnvironment *spapr = container_of(n, sPAPREnvironment, epow_notifier);
+    struct rtas_error_log *hdr;
+    struct rtas_event_log_v6 *v6hdr;
+    struct rtas_event_log_v6_maina *maina;
+    struct rtas_event_log_v6_mainb *mainb;
+    struct rtas_event_log_v6_epow *epow;
+    struct tm tm;
+    int year;
+
+    if (pending_epow) {
+        /* For now, we just throw away earlier events if two come
+         * along before any are consumed.  This is sufficient for our
+         * powerdown messages, but we'll need more if we do more
+         * general error/event logging */
+        g_free(pending_epow);
+    }
+    pending_epow = g_malloc0(sizeof(*pending_epow));
+    hdr = &pending_epow->hdr;
+    v6hdr = &pending_epow->v6hdr;
+    maina = &pending_epow->maina;
+    mainb = &pending_epow->mainb;
+    epow = &pending_epow->epow;
+
+    hdr->summary = cpu_to_be32(RTAS_LOG_VERSION_6
+                               | RTAS_LOG_SEVERITY_EVENT
+                               | RTAS_LOG_DISPOSITION_NOT_RECOVERED
+                               | RTAS_LOG_OPTIONAL_PART_PRESENT
+                               | RTAS_LOG_TYPE_EPOW);
+    hdr->extended_length = cpu_to_be32(sizeof(*pending_epow)
+                                       - sizeof(pending_epow->hdr));
+
+    v6hdr->b0 = RTAS_LOG_V6_B0_VALID | RTAS_LOG_V6_B0_NEW_LOG
+        | RTAS_LOG_V6_B0_BIGENDIAN;
+    v6hdr->b2 = RTAS_LOG_V6_B2_POWERPC_FORMAT
+        | RTAS_LOG_V6_B2_LOG_FORMAT_PLATFORM_EVENT;
+    v6hdr->company = cpu_to_be32(RTAS_LOG_V6_COMPANY_IBM);
+
+    maina->hdr.section_id = cpu_to_be16(RTAS_LOG_V6_SECTION_ID_MAINA);
+    maina->hdr.section_length = cpu_to_be16(sizeof(*maina));
+    /* FIXME: section version, subtype and creator id? */
+    qemu_get_timedate(&tm, spapr->rtc_offset);
+    year = tm.tm_year + 1900;
+    maina->creation_date = cpu_to_be32((to_bcd(year / 100) << 24)
+                                       | (to_bcd(year % 100) << 16)
+                                       | (to_bcd(tm.tm_mon + 1) << 8)
+                                       | to_bcd(tm.tm_mday));
+    maina->creation_time = cpu_to_be32((to_bcd(tm.tm_hour) << 24)
+                                       | (to_bcd(tm.tm_min) << 16)
+                                       | (to_bcd(tm.tm_sec) << 8));
+    maina->creator_id = 'H'; /* Hypervisor */
+    maina->section_count = 3; /* Main-A, Main-B and EPOW */
+    maina->plid = next_plid++;
+
+    mainb->hdr.section_id = cpu_to_be16(RTAS_LOG_V6_SECTION_ID_MAINB);
+    mainb->hdr.section_length = cpu_to_be16(sizeof(*mainb));
+    /* FIXME: section version, subtype and creator id? */
+    mainb->subsystem_id = 0xa0; /* External environment */
+    mainb->event_severity = 0x00; /* Informational / non-error */
+    mainb->event_subtype = 0xd0; /* Normal shutdown */
+
+    epow->hdr.section_id = cpu_to_be16(RTAS_LOG_V6_SECTION_ID_EPOW);
+    epow->hdr.section_length = cpu_to_be16(sizeof(*epow));
+    epow->hdr.section_version = 2; /* includes extended modifier */
+    /* FIXME: section subtype and creator id? */
+    epow->sensor_value = RTAS_LOG_V6_EPOW_ACTION_SYSTEM_SHUTDOWN;
+    epow->event_modifier = RTAS_LOG_V6_EPOW_MODIFIER_NORMAL;
+    epow->extended_modifier = RTAS_LOG_V6_EPOW_XMODIFIER_PARTITION_SPECIFIC;
+
+    qemu_irq_pulse(xics_get_qirq(spapr->icp, spapr->epow_irq));
+}
+
+static void check_exception(sPAPREnvironment *spapr,
+                            uint32_t token, uint32_t nargs,
+                            target_ulong args,
+                            uint32_t nret, target_ulong rets)
+{
+    uint32_t mask, buf, len;
+    uint64_t xinfo;
+
+    if ((nargs < 6) || (nargs > 7) || nret != 1) {
+        rtas_st(rets, 0, -3);
+        return;
+    }
+
+    xinfo = rtas_ld(args, 1);
+    mask = rtas_ld(args, 2);
+    buf = rtas_ld(args, 4);
+    len = rtas_ld(args, 5);
+    if (nargs == 7) {
+        xinfo |= (uint64_t)rtas_ld(args, 6) << 32;
+    }
+
+    if ((mask & EVENT_MASK_EPOW) && pending_epow) {
+        if (sizeof(*pending_epow) < len) {
+            len = sizeof(*pending_epow);
+        }
+
+        cpu_physical_memory_write(buf, pending_epow, len);
+        g_free(pending_epow);
+        pending_epow = NULL;
+        rtas_st(rets, 0, 0);
+    } else {
+        rtas_st(rets, 0, 1);
+    }
+}
+
+void spapr_events_init(sPAPREnvironment *spapr)
+{
+    spapr->epow_irq = spapr_allocate_msi(0);
+    spapr->epow_notifier.notify = spapr_powerdown_req;
+    qemu_register_powerdown_notifier(&spapr->epow_notifier);
+    spapr_rtas_register("check-exception", check_exception);
+}
diff --git a/hw/spapr_hcall.c b/hw/spapr_hcall.c
index a5990a961..63cadb8d9 100644
--- a/hw/spapr_hcall.c
+++ b/hw/spapr_hcall.c
@@ -1,6 +1,5 @@
 #include "sysemu.h"
 #include "cpu.h"
-#include "dyngen-exec.h"
 #include "qemu-char.h"
 #include "sysemu.h"
 #include "qemu-char.h"
@@ -40,22 +39,6 @@
 #define HPTE_V_1TB_SEG          0x4000000000000000ULL
 #define HPTE_V_VRMA_MASK        0x4001ffffff000000ULL
 
-#define HPTE_V_HVLOCK           0x40ULL
-
-static inline int lock_hpte(void *hpte, target_ulong bits)
-{
-    uint64_t pteh;
-
-    pteh = ldq_p(hpte);
-
-    /* We're protected by qemu's global lock here */
-    if (pteh & bits) {
-        return 0;
-    }
-    stq_p(hpte, pteh | HPTE_V_HVLOCK);
-    return 1;
-}
-
 static target_ulong compute_tlbie_rb(target_ulong v, target_ulong r,
                                      target_ulong pte_index)
 {
@@ -92,9 +75,10 @@ static target_ulong compute_tlbie_rb(target_ulong v, target_ulong r,
     return rb;
 }
 
-static target_ulong h_enter(CPUPPCState *env, sPAPREnvironment *spapr,
+static target_ulong h_enter(PowerPCCPU *cpu, sPAPREnvironment *spapr,
                             target_ulong opcode, target_ulong *args)
 {
+    CPUPPCState *env = &cpu->env;
     target_ulong flags = args[0];
     target_ulong pte_index = args[1];
     target_ulong pteh = args[2];
@@ -152,8 +136,7 @@ static target_ulong h_enter(CPUPPCState *env, sPAPREnvironment *spapr,
             if (i == 8) {
                 return H_PTEG_FULL;
             }
-            if (((ldq_p(hpte) & HPTE_V_VALID) == 0) &&
-                lock_hpte(hpte, HPTE_V_HVLOCK | HPTE_V_VALID)) {
+            if ((ldq_p(hpte) & HPTE_V_VALID) == 0) {
                 break;
             }
             hpte += HASH_PTE_SIZE_64;
@@ -161,7 +144,7 @@ static target_ulong h_enter(CPUPPCState *env, sPAPREnvironment *spapr,
     } else {
         i = 0;
         hpte = env->external_htab + (pte_index * HASH_PTE_SIZE_64);
-        if (!lock_hpte(hpte, HPTE_V_HVLOCK | HPTE_V_VALID)) {
+        if (ldq_p(hpte) & HPTE_V_VALID) {
             return H_PTEG_FULL;
         }
     }
@@ -169,7 +152,6 @@ static target_ulong h_enter(CPUPPCState *env, sPAPREnvironment *spapr,
     /* eieio();  FIXME: need some sort of barrier for smp? */
     stq_p(hpte, pteh);
 
-    assert(!(ldq_p(hpte) & HPTE_V_HVLOCK));
     args[0] = pte_index + i;
     return H_SUCCESS;
 }
@@ -194,11 +176,6 @@ static target_ulong remove_hpte(CPUPPCState *env, target_ulong ptex,
     }
 
     hpte = env->external_htab + (ptex * HASH_PTE_SIZE_64);
-    while (!lock_hpte(hpte, HPTE_V_HVLOCK)) {
-        /* We have no real concurrency in qemu soft-emulation, so we
-         * will never actually have a contested lock */
-        assert(0);
-    }
 
     v = ldq_p(hpte);
     r = ldq_p(hpte + (HASH_PTE_SIZE_64/2));
@@ -206,22 +183,20 @@ static target_ulong remove_hpte(CPUPPCState *env, target_ulong ptex,
     if ((v & HPTE_V_VALID) == 0 ||
         ((flags & H_AVPN) && (v & ~0x7fULL) != avpn) ||
         ((flags & H_ANDCOND) && (v & avpn) != 0)) {
-        stq_p(hpte, v & ~HPTE_V_HVLOCK);
-        assert(!(ldq_p(hpte) & HPTE_V_HVLOCK));
         return REMOVE_NOT_FOUND;
     }
-    *vp = v & ~HPTE_V_HVLOCK;
+    *vp = v;
     *rp = r;
     stq_p(hpte, 0);
     rb = compute_tlbie_rb(v, r, ptex);
     ppc_tlb_invalidate_one(env, rb);
-    assert(!(ldq_p(hpte) & HPTE_V_HVLOCK));
     return REMOVE_SUCCESS;
 }
 
-static target_ulong h_remove(CPUPPCState *env, sPAPREnvironment *spapr,
+static target_ulong h_remove(PowerPCCPU *cpu, sPAPREnvironment *spapr,
                              target_ulong opcode, target_ulong *args)
 {
+    CPUPPCState *env = &cpu->env;
     target_ulong flags = args[0];
     target_ulong pte_index = args[1];
     target_ulong avpn = args[2];
@@ -265,9 +240,10 @@ static target_ulong h_remove(CPUPPCState *env, sPAPREnvironment *spapr,
 
 #define H_BULK_REMOVE_MAX_BATCH        4
 
-static target_ulong h_bulk_remove(CPUPPCState *env, sPAPREnvironment *spapr,
+static target_ulong h_bulk_remove(PowerPCCPU *cpu, sPAPREnvironment *spapr,
                                   target_ulong opcode, target_ulong *args)
 {
+    CPUPPCState *env = &cpu->env;
     int i;
 
     for (i = 0; i < H_BULK_REMOVE_MAX_BATCH; i++) {
@@ -311,9 +287,10 @@ static target_ulong h_bulk_remove(CPUPPCState *env, sPAPREnvironment *spapr,
     return H_SUCCESS;
 }
 
-static target_ulong h_protect(CPUPPCState *env, sPAPREnvironment *spapr,
+static target_ulong h_protect(PowerPCCPU *cpu, sPAPREnvironment *spapr,
                               target_ulong opcode, target_ulong *args)
 {
+    CPUPPCState *env = &cpu->env;
     target_ulong flags = args[0];
     target_ulong pte_index = args[1];
     target_ulong avpn = args[2];
@@ -325,19 +302,12 @@ static target_ulong h_protect(CPUPPCState *env, sPAPREnvironment *spapr,
     }
 
     hpte = env->external_htab + (pte_index * HASH_PTE_SIZE_64);
-    while (!lock_hpte(hpte, HPTE_V_HVLOCK)) {
-        /* We have no real concurrency in qemu soft-emulation, so we
-         * will never actually have a contested lock */
-        assert(0);
-    }
 
     v = ldq_p(hpte);
     r = ldq_p(hpte + (HASH_PTE_SIZE_64/2));
 
     if ((v & HPTE_V_VALID) == 0 ||
         ((flags & H_AVPN) && (v & ~0x7fULL) != avpn)) {
-        stq_p(hpte, v & ~HPTE_V_HVLOCK);
-        assert(!(ldq_p(hpte) & HPTE_V_HVLOCK));
         return H_NOT_FOUND;
     }
 
@@ -351,12 +321,11 @@ static target_ulong h_protect(CPUPPCState *env, sPAPREnvironment *spapr,
     ppc_tlb_invalidate_one(env, rb);
     stq_p(hpte + (HASH_PTE_SIZE_64/2), r);
     /* Don't need a memory barrier, due to qemu's global lock */
-    stq_p(hpte, v & ~HPTE_V_HVLOCK);
-    assert(!(ldq_p(hpte) & HPTE_V_HVLOCK));
+    stq_p(hpte, v);
     return H_SUCCESS;
 }
 
-static target_ulong h_set_dabr(CPUPPCState *env, sPAPREnvironment *spapr,
+static target_ulong h_set_dabr(PowerPCCPU *cpu, sPAPREnvironment *spapr,
                                target_ulong opcode, target_ulong *args)
 {
     /* FIXME: actually implement this */
@@ -401,26 +370,26 @@ static target_ulong register_vpa(CPUPPCState *env, target_ulong vpa)
         return H_PARAMETER;
     }
 
-    env->vpa = vpa;
+    env->vpa_addr = vpa;
 
-    tmp = ldub_phys(env->vpa + VPA_SHARED_PROC_OFFSET);
+    tmp = ldub_phys(env->vpa_addr + VPA_SHARED_PROC_OFFSET);
     tmp |= VPA_SHARED_PROC_VAL;
-    stb_phys(env->vpa + VPA_SHARED_PROC_OFFSET, tmp);
+    stb_phys(env->vpa_addr + VPA_SHARED_PROC_OFFSET, tmp);
 
     return H_SUCCESS;
 }
 
 static target_ulong deregister_vpa(CPUPPCState *env, target_ulong vpa)
 {
-    if (env->slb_shadow) {
+    if (env->slb_shadow_addr) {
         return H_RESOURCE;
     }
 
-    if (env->dispatch_trace_log) {
+    if (env->dtl_addr) {
         return H_RESOURCE;
     }
 
-    env->vpa = 0;
+    env->vpa_addr = 0;
     return H_SUCCESS;
 }
 
@@ -442,18 +411,20 @@ static target_ulong register_slb_shadow(CPUPPCState *env, target_ulong addr)
         return H_PARAMETER;
     }
 
-    if (!env->vpa) {
+    if (!env->vpa_addr) {
         return H_RESOURCE;
     }
 
-    env->slb_shadow = addr;
+    env->slb_shadow_addr = addr;
+    env->slb_shadow_size = size;
 
     return H_SUCCESS;
 }
 
 static target_ulong deregister_slb_shadow(CPUPPCState *env, target_ulong addr)
 {
-    env->slb_shadow = 0;
+    env->slb_shadow_addr = 0;
+    env->slb_shadow_size = 0;
     return H_SUCCESS;
 }
 
@@ -472,11 +443,11 @@ static target_ulong register_dtl(CPUPPCState *env, target_ulong addr)
         return H_PARAMETER;
     }
 
-    if (!env->vpa) {
+    if (!env->vpa_addr) {
         return H_RESOURCE;
     }
 
-    env->dispatch_trace_log = addr;
+    env->dtl_addr = addr;
     env->dtl_size = size;
 
     return H_SUCCESS;
@@ -484,13 +455,13 @@ static target_ulong register_dtl(CPUPPCState *env, target_ulong addr)
 
 static target_ulong deregister_dtl(CPUPPCState *env, target_ulong addr)
 {
-    env->dispatch_trace_log = 0;
+    env->dtl_addr = 0;
     env->dtl_size = 0;
 
     return H_SUCCESS;
 }
 
-static target_ulong h_register_vpa(CPUPPCState *env, sPAPREnvironment *spapr,
+static target_ulong h_register_vpa(PowerPCCPU *cpu, sPAPREnvironment *spapr,
                                    target_ulong opcode, target_ulong *args)
 {
     target_ulong flags = args[0];
@@ -538,18 +509,22 @@ static target_ulong h_register_vpa(CPUPPCState *env, sPAPREnvironment *spapr,
     return ret;
 }
 
-static target_ulong h_cede(CPUPPCState *env, sPAPREnvironment *spapr,
+static target_ulong h_cede(PowerPCCPU *cpu, sPAPREnvironment *spapr,
                            target_ulong opcode, target_ulong *args)
 {
+    CPUPPCState *env = &cpu->env;
+
     env->msr |= (1ULL << MSR_EE);
     hreg_compute_hflags(env);
-    if (!cpu_has_work(env)) {
+    if (!cpu_has_work(CPU(cpu))) {
         env->halted = 1;
+        env->exception_index = EXCP_HLT;
+        env->exit_request = 1;
     }
     return H_SUCCESS;
 }
 
-static target_ulong h_rtas(CPUPPCState *env, sPAPREnvironment *spapr,
+static target_ulong h_rtas(PowerPCCPU *cpu, sPAPREnvironment *spapr,
                            target_ulong opcode, target_ulong *args)
 {
     target_ulong rtas_r3 = args[0];
@@ -561,7 +536,7 @@ static target_ulong h_rtas(CPUPPCState *env, sPAPREnvironment *spapr,
                            nret, rtas_r3 + 12 + 4*nargs);
 }
 
-static target_ulong h_logical_load(CPUPPCState *env, sPAPREnvironment *spapr,
+static target_ulong h_logical_load(PowerPCCPU *cpu, sPAPREnvironment *spapr,
                                    target_ulong opcode, target_ulong *args)
 {
     target_ulong size = args[0];
@@ -584,7 +559,7 @@ static target_ulong h_logical_load(CPUPPCState *env, sPAPREnvironment *spapr,
     return H_PARAMETER;
 }
 
-static target_ulong h_logical_store(CPUPPCState *env, sPAPREnvironment *spapr,
+static target_ulong h_logical_store(PowerPCCPU *cpu, sPAPREnvironment *spapr,
                                     target_ulong opcode, target_ulong *args)
 {
     target_ulong size = args[0];
@@ -608,7 +583,7 @@ static target_ulong h_logical_store(CPUPPCState *env, sPAPREnvironment *spapr,
     return H_PARAMETER;
 }
 
-static target_ulong h_logical_memop(CPUPPCState *env, sPAPREnvironment *spapr,
+static target_ulong h_logical_memop(PowerPCCPU *cpu, sPAPREnvironment *spapr,
                                     target_ulong opcode, target_ulong *args)
 {
     target_ulong dst   = args[0]; /* Destination address */
@@ -675,14 +650,14 @@ static target_ulong h_logical_memop(CPUPPCState *env, sPAPREnvironment *spapr,
     return H_SUCCESS;
 }
 
-static target_ulong h_logical_icbi(CPUPPCState *env, sPAPREnvironment *spapr,
+static target_ulong h_logical_icbi(PowerPCCPU *cpu, sPAPREnvironment *spapr,
                                    target_ulong opcode, target_ulong *args)
 {
     /* Nothing to do on emulation, KVM will trap this in the kernel */
     return H_SUCCESS;
 }
 
-static target_ulong h_logical_dcbf(CPUPPCState *env, sPAPREnvironment *spapr,
+static target_ulong h_logical_dcbf(PowerPCCPU *cpu, sPAPREnvironment *spapr,
                                    target_ulong opcode, target_ulong *args)
 {
     /* Nothing to do on emulation, KVM will trap this in the kernel */
@@ -703,35 +678,29 @@ void spapr_register_hypercall(target_ulong opcode, spapr_hcall_fn fn)
     } else {
         assert((opcode >= KVMPPC_HCALL_BASE) && (opcode <= KVMPPC_HCALL_MAX));
 
-
         slot = &kvmppc_hypercall_table[opcode - KVMPPC_HCALL_BASE];
     }
 
-    assert(!(*slot) || (fn == *slot));
+    assert(!(*slot));
     *slot = fn;
 }
 
-target_ulong spapr_hypercall(CPUPPCState *env, target_ulong opcode,
+target_ulong spapr_hypercall(PowerPCCPU *cpu, target_ulong opcode,
                              target_ulong *args)
 {
-    if (msr_pr) {
-        hcall_dprintf("Hypercall made with MSR[PR]=1\n");
-        return H_PRIVILEGE;
-    }
-
     if ((opcode <= MAX_HCALL_OPCODE)
         && ((opcode & 0x3) == 0)) {
         spapr_hcall_fn fn = papr_hypercall_table[opcode / 4];
 
         if (fn) {
-            return fn(env, spapr, opcode, args);
+            return fn(cpu, spapr, opcode, args);
         }
     } else if ((opcode >= KVMPPC_HCALL_BASE) &&
                (opcode <= KVMPPC_HCALL_MAX)) {
         spapr_hcall_fn fn = kvmppc_hypercall_table[opcode - KVMPPC_HCALL_BASE];
 
         if (fn) {
-            return fn(env, spapr, opcode, args);
+            return fn(cpu, spapr, opcode, args);
         }
     }
 
diff --git a/hw/spapr_iommu.c b/hw/spapr_iommu.c
index 53b731773..02d78ccf2 100644
--- a/hw/spapr_iommu.c
+++ b/hw/spapr_iommu.c
@@ -21,6 +21,7 @@
 #include "qdev.h"
 #include "kvm_ppc.h"
 #include "dma.h"
+#include "exec-memory.h"
 
 #include "hw/spapr.h"
 
@@ -42,6 +43,7 @@ struct sPAPRTCETable {
     uint32_t liobn;
     uint32_t window_size;
     sPAPRTCE *table;
+    bool bypass;
     int fd;
     QLIST_ENTRY(sPAPRTCETable) list;
 };
@@ -64,8 +66,8 @@ static sPAPRTCETable *spapr_tce_find_by_liobn(uint32_t liobn)
 
 static int spapr_tce_translate(DMAContext *dma,
                                dma_addr_t addr,
-                               target_phys_addr_t *paddr,
-                               target_phys_addr_t *len,
+                               hwaddr *paddr,
+                               hwaddr *len,
                                DMADirection dir)
 {
     sPAPRTCETable *tcet = DO_UPCAST(sPAPRTCETable, dma, dma);
@@ -78,6 +80,12 @@ static int spapr_tce_translate(DMAContext *dma,
             DMA_ADDR_FMT "\n", tcet->liobn, addr);
 #endif
 
+    if (tcet->bypass) {
+        *paddr = addr;
+        *len = (hwaddr)-1;
+        return 0;
+    }
+
     /* Check if we are in bound */
     if (addr >= tcet->window_size) {
 #ifdef DEBUG_TCE
@@ -117,7 +125,7 @@ DMAContext *spapr_tce_new_dma_context(uint32_t liobn, size_t window_size)
     }
 
     tcet = g_malloc0(sizeof(*tcet));
-    dma_context_init(&tcet->dma, spapr_tce_translate, NULL, NULL);
+    dma_context_init(&tcet->dma, &address_space_memory, spapr_tce_translate, NULL, NULL);
 
     tcet->liobn = liobn;
     tcet->window_size = window_size;
@@ -162,6 +170,23 @@ void spapr_tce_free(DMAContext *dma)
     }
 }
 
+void spapr_tce_set_bypass(DMAContext *dma, bool bypass)
+{
+    sPAPRTCETable *tcet = DO_UPCAST(sPAPRTCETable, dma, dma);
+
+    tcet->bypass = bypass;
+}
+
+void spapr_tce_reset(DMAContext *dma)
+{
+    sPAPRTCETable *tcet = DO_UPCAST(sPAPRTCETable, dma, dma);
+    size_t table_size = (tcet->window_size >> SPAPR_TCE_PAGE_SHIFT)
+        * sizeof(sPAPRTCE);
+
+    tcet->bypass = false;
+    memset(tcet->table, 0, table_size);
+}
+
 static target_ulong put_tce_emu(sPAPRTCETable *tcet, target_ulong ioba,
                                 target_ulong tce)
 {
@@ -179,7 +204,7 @@ static target_ulong put_tce_emu(sPAPRTCETable *tcet, target_ulong ioba,
     return H_SUCCESS;
 }
 
-static target_ulong h_put_tce(CPUPPCState *env, sPAPREnvironment *spapr,
+static target_ulong h_put_tce(PowerPCCPU *cpu, sPAPREnvironment *spapr,
                               target_ulong opcode, target_ulong *args)
 {
     target_ulong liobn = args[0];
diff --git a/hw/spapr_llan.c b/hw/spapr_llan.c
index bd3f131d7..09ad69f6b 100644
--- a/hw/spapr_llan.c
+++ b/hw/spapr_llan.c
@@ -264,7 +264,7 @@ static int check_bd(VIOsPAPRVLANDevice *dev, vlan_bd_t bd,
     return 0;
 }
 
-static target_ulong h_register_logical_lan(CPUPPCState *env,
+static target_ulong h_register_logical_lan(PowerPCCPU *cpu,
                                            sPAPREnvironment *spapr,
                                            target_ulong opcode,
                                            target_ulong *args)
@@ -328,7 +328,7 @@ static target_ulong h_register_logical_lan(CPUPPCState *env,
 }
 
 
-static target_ulong h_free_logical_lan(CPUPPCState *env, sPAPREnvironment *spapr,
+static target_ulong h_free_logical_lan(PowerPCCPU *cpu, sPAPREnvironment *spapr,
                                        target_ulong opcode, target_ulong *args)
 {
     target_ulong reg = args[0];
@@ -349,7 +349,7 @@ static target_ulong h_free_logical_lan(CPUPPCState *env, sPAPREnvironment *spapr
     return H_SUCCESS;
 }
 
-static target_ulong h_add_logical_lan_buffer(CPUPPCState *env,
+static target_ulong h_add_logical_lan_buffer(PowerPCCPU *cpu,
                                              sPAPREnvironment *spapr,
                                              target_ulong opcode,
                                              target_ulong *args)
@@ -398,7 +398,7 @@ static target_ulong h_add_logical_lan_buffer(CPUPPCState *env,
     return H_SUCCESS;
 }
 
-static target_ulong h_send_logical_lan(CPUPPCState *env, sPAPREnvironment *spapr,
+static target_ulong h_send_logical_lan(PowerPCCPU *cpu, sPAPREnvironment *spapr,
                                        target_ulong opcode, target_ulong *args)
 {
     target_ulong reg = args[0];
@@ -467,7 +467,7 @@ static target_ulong h_send_logical_lan(CPUPPCState *env, sPAPREnvironment *spapr
     return H_SUCCESS;
 }
 
-static target_ulong h_multicast_ctrl(CPUPPCState *env, sPAPREnvironment *spapr,
+static target_ulong h_multicast_ctrl(PowerPCCPU *cpu, sPAPREnvironment *spapr,
                                      target_ulong opcode, target_ulong *args)
 {
     target_ulong reg = args[0];
diff --git a/hw/spapr_pci.c b/hw/spapr_pci.c
index 661c05bc3..3c5b855bc 100644
--- a/hw/spapr_pci.c
+++ b/hw/spapr_pci.c
@@ -258,7 +258,7 @@ static int spapr_msicfg_find(sPAPRPHBState *phb, uint32_t config_addr,
  * This is required for msi_notify()/msix_notify() which
  * will write at the addresses via spapr_msi_write().
  */
-static void spapr_msi_setmsg(PCIDevice *pdev, target_phys_addr_t addr,
+static void spapr_msi_setmsg(PCIDevice *pdev, hwaddr addr,
                              bool msix, unsigned req_num)
 {
     unsigned i;
@@ -351,7 +351,7 @@ static void rtas_ibm_change_msi(sPAPREnvironment *spapr,
 
     /* There is no cached config, allocate MSIs */
     if (!phb->msi_table[ndev].nvec) {
-        irq = spapr_allocate_irq_block(req_num, XICS_MSI);
+        irq = spapr_allocate_irq_block(req_num, false);
         if (irq < 0) {
             fprintf(stderr, "Cannot allocate MSIs for device#%d", ndev);
             rtas_st(rets, 0, -1); /* Hardware error */
@@ -439,7 +439,7 @@ static void pci_spapr_set_irq(void *opaque, int irq_num, int level)
     qemu_set_irq(spapr_phb_lsi_qirq(phb, irq_num), level);
 }
 
-static uint64_t spapr_io_read(void *opaque, target_phys_addr_t addr,
+static uint64_t spapr_io_read(void *opaque, hwaddr addr,
                               unsigned size)
 {
     switch (size) {
@@ -453,7 +453,7 @@ static uint64_t spapr_io_read(void *opaque, target_phys_addr_t addr,
     assert(0);
 }
 
-static void spapr_io_write(void *opaque, target_phys_addr_t addr,
+static void spapr_io_write(void *opaque, hwaddr addr,
                            uint64_t data, unsigned size)
 {
     switch (size) {
@@ -483,7 +483,7 @@ static const MemoryRegionOps spapr_io_ops = {
  * data is set to 0.
  * For MSI, the vector number is encoded in least bits in data.
  */
-static void spapr_msi_write(void *opaque, target_phys_addr_t addr,
+static void spapr_msi_write(void *opaque, hwaddr addr,
                             uint64_t data, unsigned size)
 {
     sPAPRPHBState *phb = opaque;
@@ -595,6 +595,15 @@ static int spapr_phb_init(SysBusDevice *s)
     return 0;
 }
 
+static void spapr_phb_reset(DeviceState *qdev)
+{
+    SysBusDevice *s = sysbus_from_qdev(qdev);
+    sPAPRPHBState *sphb = SPAPR_PCI_HOST_BRIDGE(s);
+
+    /* Reset the IOMMU state */
+    spapr_tce_reset(sphb->dma);
+}
+
 static Property spapr_phb_properties[] = {
     DEFINE_PROP_HEX64("buid", sPAPRPHBState, buid, 0),
     DEFINE_PROP_STRING("busname", sPAPRPHBState, busname),
@@ -613,6 +622,7 @@ static void spapr_phb_class_init(ObjectClass *klass, void *data)
 
     sdc->init = spapr_phb_init;
     dc->props = spapr_phb_properties;
+    dc->reset = spapr_phb_reset;
 }
 
 static const TypeInfo spapr_phb_info = {
diff --git a/hw/spapr_pci.h b/hw/spapr_pci.h
index 670dc62a1..e307ac803 100644
--- a/hw/spapr_pci.h
+++ b/hw/spapr_pci.h
@@ -42,8 +42,8 @@ typedef struct sPAPRPHBState {
     char *dtbusname;
 
     MemoryRegion memspace, iospace;
-    target_phys_addr_t mem_win_addr, mem_win_size, io_win_addr, io_win_size;
-    target_phys_addr_t msi_win_addr;
+    hwaddr mem_win_addr, mem_win_size, io_win_addr, io_win_size;
+    hwaddr msi_win_addr;
     MemoryRegion memwindow, iowindow, msiwindow;
 
     uint32_t dma_liobn;
diff --git a/hw/spapr_rtas.c b/hw/spapr_rtas.c
index ae1859515..6d5c48a74 100644
--- a/hw/spapr_rtas.c
+++ b/hw/spapr_rtas.c
@@ -163,6 +163,7 @@ static void rtas_start_cpu(sPAPREnvironment *spapr,
                            uint32_t nret, target_ulong rets)
 {
     target_ulong id, start, r3;
+    CPUState *cpu;
     CPUPPCState *env;
 
     if (nargs != 3 || nret != 1) {
@@ -175,6 +176,8 @@ static void rtas_start_cpu(sPAPREnvironment *spapr,
     r3 = rtas_ld(args, 2);
 
     for (env = first_cpu; env; env = env->next_cpu) {
+        cpu = ENV_GET_CPU(env);
+
         if (env->cpu_index != id) {
             continue;
         }
@@ -184,12 +187,17 @@ static void rtas_start_cpu(sPAPREnvironment *spapr,
             return;
         }
 
+        /* This will make sure qemu state is up to date with kvm, and
+         * mark it dirty so our changes get flushed back before the
+         * new cpu enters */
+        kvm_cpu_synchronize_state(env);
+
         env->msr = (1ULL << MSR_SF) | (1ULL << MSR_ME);
         env->nip = start;
         env->gpr[3] = r3;
         env->halted = 0;
 
-        qemu_cpu_kick(env);
+        qemu_cpu_kick(cpu);
 
         rtas_st(rets, 0, 0);
         return;
@@ -236,6 +244,15 @@ target_ulong spapr_rtas_call(sPAPREnvironment *spapr,
 
 void spapr_rtas_register(const char *name, spapr_rtas_fn fn)
 {
+    int i;
+
+    for (i = 0; i < (rtas_next - rtas_table); i++) {
+        if (strcmp(name, rtas_table[i].name) == 0) {
+            fprintf(stderr, "RTAS call \"%s\" registered twice\n", name);
+            exit(1);
+        }
+    }
+
     assert(rtas_next < (rtas_table + TOKEN_MAX));
 
     rtas_next->name = name;
@@ -244,8 +261,8 @@ void spapr_rtas_register(const char *name, spapr_rtas_fn fn)
     rtas_next++;
 }
 
-int spapr_rtas_device_tree_setup(void *fdt, target_phys_addr_t rtas_addr,
-                                 target_phys_addr_t rtas_size)
+int spapr_rtas_device_tree_setup(void *fdt, hwaddr rtas_addr,
+                                 hwaddr rtas_size)
 {
     int ret;
     int i;
diff --git a/hw/spapr_vio.c b/hw/spapr_vio.c
index 7ca445216..1f19fedd0 100644
--- a/hw/spapr_vio.c
+++ b/hw/spapr_vio.c
@@ -161,7 +161,7 @@ static int vio_make_devnode(VIOsPAPRDevice *dev,
 /*
  * CRQ handling
  */
-static target_ulong h_reg_crq(CPUPPCState *env, sPAPREnvironment *spapr,
+static target_ulong h_reg_crq(PowerPCCPU *cpu, sPAPREnvironment *spapr,
                               target_ulong opcode, target_ulong *args)
 {
     target_ulong reg = args[0];
@@ -219,7 +219,7 @@ static target_ulong free_crq(VIOsPAPRDevice *dev)
     return H_SUCCESS;
 }
 
-static target_ulong h_free_crq(CPUPPCState *env, sPAPREnvironment *spapr,
+static target_ulong h_free_crq(PowerPCCPU *cpu, sPAPREnvironment *spapr,
                                target_ulong opcode, target_ulong *args)
 {
     target_ulong reg = args[0];
@@ -233,7 +233,7 @@ static target_ulong h_free_crq(CPUPPCState *env, sPAPREnvironment *spapr,
     return free_crq(dev);
 }
 
-static target_ulong h_send_crq(CPUPPCState *env, sPAPREnvironment *spapr,
+static target_ulong h_send_crq(PowerPCCPU *cpu, sPAPREnvironment *spapr,
                                target_ulong opcode, target_ulong *args)
 {
     target_ulong reg = args[0];
@@ -256,7 +256,7 @@ static target_ulong h_send_crq(CPUPPCState *env, sPAPREnvironment *spapr,
     return H_HARDWARE;
 }
 
-static target_ulong h_enable_crq(CPUPPCState *env, sPAPREnvironment *spapr,
+static target_ulong h_enable_crq(PowerPCCPU *cpu, sPAPREnvironment *spapr,
                                  target_ulong opcode, target_ulong *args)
 {
     target_ulong reg = args[0];
@@ -316,17 +316,10 @@ int spapr_vio_send_crq(VIOsPAPRDevice *dev, uint8_t *crq)
 
 static void spapr_vio_quiesce_one(VIOsPAPRDevice *dev)
 {
-    VIOsPAPRDeviceClass *pc = VIO_SPAPR_DEVICE_GET_CLASS(dev);
-    uint32_t liobn = SPAPR_VIO_BASE_LIOBN | dev->reg;
-
     if (dev->dma) {
-        spapr_tce_free(dev->dma);
+        spapr_tce_reset(dev->dma);
     }
-    dev->dma = spapr_tce_new_dma_context(liobn, pc->rtce_window_size);
-
-    dev->crq.qladdr = 0;
-    dev->crq.qsize = 0;
-    dev->crq.qnext = 0;
+    free_crq(dev);
 }
 
 static void rtas_set_tce_bypass(sPAPREnvironment *spapr, uint32_t token,
@@ -348,16 +341,14 @@ static void rtas_set_tce_bypass(sPAPREnvironment *spapr, uint32_t token,
         rtas_st(rets, 0, -3);
         return;
     }
-    if (enable) {
-        spapr_tce_free(dev->dma);
-        dev->dma = NULL;
-    } else {
-        VIOsPAPRDeviceClass *pc = VIO_SPAPR_DEVICE_GET_CLASS(dev);
-        uint32_t liobn = SPAPR_VIO_BASE_LIOBN | dev->reg;
 
-        dev->dma = spapr_tce_new_dma_context(liobn, pc->rtce_window_size);
+    if (!dev->dma) {
+        rtas_st(rets, 0, -3);
+        return;
     }
 
+    spapr_tce_set_bypass(dev->dma, !!enable);
+
     rtas_st(rets, 0, 0);
 }
 
@@ -409,9 +400,10 @@ static void spapr_vio_busdev_reset(DeviceState *qdev)
     VIOsPAPRDevice *dev = DO_UPCAST(VIOsPAPRDevice, qdev, qdev);
     VIOsPAPRDeviceClass *pc = VIO_SPAPR_DEVICE_GET_CLASS(dev);
 
-    if (dev->crq.qsize) {
-        free_crq(dev);
-    }
+    /* Shut down the request queue and TCEs if necessary */
+    spapr_vio_quiesce_one(dev);
+
+    dev->signal_state = 0;
 
     if (pc->reset) {
         pc->reset(dev);
@@ -422,7 +414,6 @@ static int spapr_vio_busdev_init(DeviceState *qdev)
 {
     VIOsPAPRDevice *dev = (VIOsPAPRDevice *)qdev;
     VIOsPAPRDeviceClass *pc = VIO_SPAPR_DEVICE_GET_CLASS(dev);
-    uint32_t liobn;
     char *id;
 
     if (dev->reg != -1) {
@@ -464,13 +455,15 @@ static int spapr_vio_busdev_init(DeviceState *qdev)
         return -1;
     }
 
-    liobn = SPAPR_VIO_BASE_LIOBN | dev->reg;
-    dev->dma = spapr_tce_new_dma_context(liobn, pc->rtce_window_size);
+    if (pc->rtce_window_size) {
+        uint32_t liobn = SPAPR_VIO_BASE_LIOBN | dev->reg;
+        dev->dma = spapr_tce_new_dma_context(liobn, pc->rtce_window_size);
+    }
 
     return pc->init(dev);
 }
 
-static target_ulong h_vio_signal(CPUPPCState *env, sPAPREnvironment *spapr,
+static target_ulong h_vio_signal(PowerPCCPU *cpu, sPAPREnvironment *spapr,
                                  target_ulong opcode,
                                  target_ulong *args)
 {
diff --git a/hw/spapr_vio.h b/hw/spapr_vio.h
index ea6aa43e2..cc85d2610 100644
--- a/hw/spapr_vio.h
+++ b/hw/spapr_vio.h
@@ -60,7 +60,6 @@ typedef struct VIOsPAPRDeviceClass {
 struct VIOsPAPRDevice {
     DeviceState qdev;
     uint32_t reg;
-    uint32_t flags;
     uint32_t irq;
     target_ulong signal_state;
     VIOsPAPR_CRQ crq;
@@ -132,7 +131,6 @@ void spapr_vscsi_create(VIOsPAPRBus *bus);
 
 VIOsPAPRDevice *spapr_vty_get_default(VIOsPAPRBus *bus);
 
-int spapr_tce_set_bypass(uint32_t unit, uint32_t enable);
 void spapr_vio_quiesce(void);
 
 #endif /* _HW_SPAPR_VIO_H */
diff --git a/hw/spapr_vscsi.c b/hw/spapr_vscsi.c
index 3cf5844e0..e3d4b237f 100644
--- a/hw/spapr_vscsi.c
+++ b/hw/spapr_vscsi.c
@@ -737,7 +737,7 @@ static int vscsi_send_adapter_info(VSCSIState *s, vscsi_req *req)
 #endif
     memset(&info, 0, sizeof(info));
     strcpy(info.srp_version, SRP_VERSION);
-    strncpy(info.partition_name, "qemu", sizeof("qemu"));
+    memcpy(info.partition_name, "qemu", sizeof("qemu"));
     info.partition_number = cpu_to_be32(0);
     info.mad_version = cpu_to_be32(1);
     info.os_type = cpu_to_be32(2);
diff --git a/hw/spapr_vty.c b/hw/spapr_vty.c
index 5da17a3ff..14f862fba 100644
--- a/hw/spapr_vty.c
+++ b/hw/spapr_vty.c
@@ -70,7 +70,7 @@ static int spapr_vty_init(VIOsPAPRDevice *sdev)
 }
 
 /* Forward declaration */
-static target_ulong h_put_term_char(CPUPPCState *env, sPAPREnvironment *spapr,
+static target_ulong h_put_term_char(PowerPCCPU *cpu, sPAPREnvironment *spapr,
                                     target_ulong opcode, target_ulong *args)
 {
     target_ulong reg = args[0];
@@ -97,7 +97,7 @@ static target_ulong h_put_term_char(CPUPPCState *env, sPAPREnvironment *spapr,
     return H_SUCCESS;
 }
 
-static target_ulong h_get_term_char(CPUPPCState *env, sPAPREnvironment *spapr,
+static target_ulong h_get_term_char(PowerPCCPU *cpu, sPAPREnvironment *spapr,
                                     target_ulong opcode, target_ulong *args)
 {
     target_ulong reg = args[0];
diff --git a/hw/sparc32_dma.c b/hw/sparc32_dma.c
index 1dbf69e80..d11a302f2 100644
--- a/hw/sparc32_dma.c
+++ b/hw/sparc32_dma.c
@@ -78,7 +78,7 @@ enum {
 };
 
 /* Note: on sparc, the lance 16 bit bus is swapped */
-void ledma_memory_read(void *opaque, target_phys_addr_t addr,
+void ledma_memory_read(void *opaque, hwaddr addr,
                        uint8_t *buf, int len, int do_bswap)
 {
     DMAState *s = opaque;
@@ -98,7 +98,7 @@ void ledma_memory_read(void *opaque, target_phys_addr_t addr,
     }
 }
 
-void ledma_memory_write(void *opaque, target_phys_addr_t addr,
+void ledma_memory_write(void *opaque, hwaddr addr,
                         uint8_t *buf, int len, int do_bswap)
 {
     DMAState *s = opaque;
@@ -165,7 +165,7 @@ void espdma_memory_write(void *opaque, uint8_t *buf, int len)
     s->dmaregs[1] += len;
 }
 
-static uint64_t dma_mem_read(void *opaque, target_phys_addr_t addr,
+static uint64_t dma_mem_read(void *opaque, hwaddr addr,
                              unsigned size)
 {
     DMAState *s = opaque;
@@ -182,7 +182,7 @@ static uint64_t dma_mem_read(void *opaque, target_phys_addr_t addr,
     return s->dmaregs[saddr];
 }
 
-static void dma_mem_write(void *opaque, target_phys_addr_t addr,
+static void dma_mem_write(void *opaque, hwaddr addr,
                           uint64_t val, unsigned size)
 {
     DMAState *s = opaque;
diff --git a/hw/sparc32_dma.h b/hw/sparc32_dma.h
index 8b72c37a9..9497b13d3 100644
--- a/hw/sparc32_dma.h
+++ b/hw/sparc32_dma.h
@@ -2,9 +2,9 @@
 #define SPARC32_DMA_H
 
 /* sparc32_dma.c */
-void ledma_memory_read(void *opaque, target_phys_addr_t addr,
+void ledma_memory_read(void *opaque, hwaddr addr,
                        uint8_t *buf, int len, int do_bswap);
-void ledma_memory_write(void *opaque, target_phys_addr_t addr,
+void ledma_memory_write(void *opaque, hwaddr addr,
                         uint8_t *buf, int len, int do_bswap);
 void espdma_memory_read(void *opaque, uint8_t *buf, int len);
 void espdma_memory_write(void *opaque, uint8_t *buf, int len);
diff --git a/hw/spitz.c b/hw/spitz.c
index 20e783519..12e281522 100644
--- a/hw/spitz.c
+++ b/hw/spitz.c
@@ -60,7 +60,7 @@ typedef struct {
     ECCState ecc;
 } SLNANDState;
 
-static uint64_t sl_read(void *opaque, target_phys_addr_t addr, unsigned size)
+static uint64_t sl_read(void *opaque, hwaddr addr, unsigned size)
 {
     SLNANDState *s = (SLNANDState *) opaque;
     int ryby;
@@ -102,7 +102,7 @@ static uint64_t sl_read(void *opaque, target_phys_addr_t addr, unsigned size)
     return 0;
 }
 
-static void sl_write(void *opaque, target_phys_addr_t addr,
+static void sl_write(void *opaque, hwaddr addr,
                      uint64_t value, unsigned size)
 {
     SLNANDState *s = (SLNANDState *) opaque;
@@ -879,15 +879,14 @@ static struct arm_boot_info spitz_binfo = {
     .ram_size = 0x04000000,
 };
 
-static void spitz_common_init(ram_addr_t ram_size,
-                const char *kernel_filename,
-                const char *kernel_cmdline, const char *initrd_filename,
-                const char *cpu_model, enum spitz_model_e model, int arm_id)
+static void spitz_common_init(QEMUMachineInitArgs *args,
+                              enum spitz_model_e model, int arm_id)
 {
     PXA2xxState *mpu;
     DeviceState *scp0, *scp1 = NULL;
     MemoryRegion *address_space_mem = get_system_memory();
     MemoryRegion *rom = g_new(MemoryRegion, 1);
+    const char *cpu_model = args->cpu_model;
 
     if (!cpu_model)
         cpu_model = (model == terrier) ? "pxa270-c5" : "pxa270-c0";
@@ -928,48 +927,32 @@ static void spitz_common_init(ram_addr_t ram_size,
         /* A 4.0 GB microdrive is permanently sitting in CF slot 0.  */
         spitz_microdrive_attach(mpu, 0);
 
-    spitz_binfo.kernel_filename = kernel_filename;
-    spitz_binfo.kernel_cmdline = kernel_cmdline;
-    spitz_binfo.initrd_filename = initrd_filename;
+    spitz_binfo.kernel_filename = args->kernel_filename;
+    spitz_binfo.kernel_cmdline = args->kernel_cmdline;
+    spitz_binfo.initrd_filename = args->initrd_filename;
     spitz_binfo.board_id = arm_id;
     arm_load_kernel(mpu->cpu, &spitz_binfo);
     sl_bootparam_write(SL_PXA_PARAM_BASE);
 }
 
-static void spitz_init(ram_addr_t ram_size,
-                const char *boot_device,
-                const char *kernel_filename, const char *kernel_cmdline,
-                const char *initrd_filename, const char *cpu_model)
+static void spitz_init(QEMUMachineInitArgs *args)
 {
-    spitz_common_init(ram_size, kernel_filename,
-                kernel_cmdline, initrd_filename, cpu_model, spitz, 0x2c9);
+    spitz_common_init(args, spitz, 0x2c9);
 }
 
-static void borzoi_init(ram_addr_t ram_size,
-                const char *boot_device,
-                const char *kernel_filename, const char *kernel_cmdline,
-                const char *initrd_filename, const char *cpu_model)
+static void borzoi_init(QEMUMachineInitArgs *args)
 {
-    spitz_common_init(ram_size, kernel_filename,
-                kernel_cmdline, initrd_filename, cpu_model, borzoi, 0x33f);
+    spitz_common_init(args, borzoi, 0x33f);
 }
 
-static void akita_init(ram_addr_t ram_size,
-                const char *boot_device,
-                const char *kernel_filename, const char *kernel_cmdline,
-                const char *initrd_filename, const char *cpu_model)
+static void akita_init(QEMUMachineInitArgs *args)
 {
-    spitz_common_init(ram_size, kernel_filename,
-                kernel_cmdline, initrd_filename, cpu_model, akita, 0x2e8);
+    spitz_common_init(args, akita, 0x2e8);
 }
 
-static void terrier_init(ram_addr_t ram_size,
-                const char *boot_device,
-                const char *kernel_filename, const char *kernel_cmdline,
-                const char *initrd_filename, const char *cpu_model)
+static void terrier_init(QEMUMachineInitArgs *args)
 {
-    spitz_common_init(ram_size, kernel_filename,
-                kernel_cmdline, initrd_filename, cpu_model, terrier, 0x33f);
+    spitz_common_init(args, terrier, 0x33f);
 }
 
 static QEMUMachine akitapda_machine = {
@@ -1083,10 +1066,11 @@ static TypeInfo spitz_keyboard_info = {
 
 static const VMStateDescription vmstate_corgi_ssp_regs = {
     .name = "corgi-ssp",
-    .version_id = 1,
-    .minimum_version_id = 1,
-    .minimum_version_id_old = 1,
+    .version_id = 2,
+    .minimum_version_id = 2,
+    .minimum_version_id_old = 2,
     .fields = (VMStateField []) {
+        VMSTATE_SSI_SLAVE(ssidev, CorgiSSPState),
         VMSTATE_UINT32_ARRAY(enable, CorgiSSPState, 3),
         VMSTATE_END_OF_LIST(),
     }
@@ -1115,6 +1099,7 @@ static const VMStateDescription vmstate_spitz_lcdtg_regs = {
     .minimum_version_id = 1,
     .minimum_version_id_old = 1,
     .fields = (VMStateField []) {
+        VMSTATE_SSI_SLAVE(ssidev, SpitzLCDTG),
         VMSTATE_UINT32(bl_intensity, SpitzLCDTG),
         VMSTATE_UINT32(bl_power, SpitzLCDTG),
         VMSTATE_END_OF_LIST(),
diff --git a/hw/srp.h b/hw/srp.h
index 3009bd56c..5e0cad5c1 100644
--- a/hw/srp.h
+++ b/hw/srp.h
@@ -177,13 +177,13 @@ struct srp_tsk_mgmt {
     uint8_t    reserved1[6];
     uint64_t   tag;
     uint8_t    reserved2[4];
-    uint64_t   lun QEMU_PACKED;
+    uint64_t   lun;
     uint8_t    reserved3[2];
     uint8_t    tsk_mgmt_func;
     uint8_t    reserved4;
     uint64_t   task_tag;
     uint8_t    reserved5[8];
-};
+} QEMU_PACKED;
 
 /*
  * We need the packed attribute because the SRP spec only aligns the
@@ -198,14 +198,14 @@ struct srp_cmd {
     uint8_t    data_in_desc_cnt;
     uint64_t   tag;
     uint8_t    reserved2[4];
-    uint64_t   lun QEMU_PACKED;
+    uint64_t   lun;
     uint8_t    reserved3;
     uint8_t    task_attr;
     uint8_t    reserved4;
     uint8_t    add_cdb_len;
     uint8_t    cdb[16];
     uint8_t    add_data[0];
-};
+} QEMU_PACKED;
 
 enum {
     SRP_RSP_FLAG_RSPVALID = 1 << 0,
diff --git a/hw/ssd0303.c b/hw/ssd0303.c
index 4e1ee6e12..d7fd828c6 100644
--- a/hw/ssd0303.c
+++ b/hw/ssd0303.c
@@ -252,7 +252,7 @@ static void ssd0303_update_display(void *opaque)
         }
     }
     s->redraw = 0;
-    dpy_update(s->ds, 0, 0, 96 * MAGNIFY, 16 * MAGNIFY);
+    dpy_gfx_update(s->ds, 0, 0, 96 * MAGNIFY, 16 * MAGNIFY);
 }
 
 static void ssd0303_invalidate_display(void * opaque)
diff --git a/hw/ssd0323.c b/hw/ssd0323.c
index b101c5112..4098830c2 100644
--- a/hw/ssd0323.c
+++ b/hw/ssd0323.c
@@ -260,7 +260,7 @@ static void ssd0323_update_display(void *opaque)
         }
     }
     s->redraw = 0;
-    dpy_update(s->ds, 0, 0, 128 * MAGNIFY, 64 * MAGNIFY);
+    dpy_gfx_update(s->ds, 0, 0, 128 * MAGNIFY, 64 * MAGNIFY);
 }
 
 static void ssd0323_invalidate_display(void * opaque)
@@ -279,6 +279,7 @@ static void ssd0323_cd(void *opaque, int n, int level)
 
 static void ssd0323_save(QEMUFile *f, void *opaque)
 {
+    SSISlave *ss = SSI_SLAVE(opaque);
     ssd0323_state *s = (ssd0323_state *)opaque;
     int i;
 
@@ -296,10 +297,13 @@ static void ssd0323_save(QEMUFile *f, void *opaque)
     qemu_put_be32(f, s->remap);
     qemu_put_be32(f, s->mode);
     qemu_put_buffer(f, s->framebuffer, sizeof(s->framebuffer));
+
+    qemu_put_be32(f, ss->cs);
 }
 
 static int ssd0323_load(QEMUFile *f, void *opaque, int version_id)
 {
+    SSISlave *ss = SSI_SLAVE(opaque);
     ssd0323_state *s = (ssd0323_state *)opaque;
     int i;
 
@@ -321,6 +325,8 @@ static int ssd0323_load(QEMUFile *f, void *opaque, int version_id)
     s->mode = qemu_get_be32(f);
     qemu_get_buffer(f, s->framebuffer, sizeof(s->framebuffer));
 
+    ss->cs = qemu_get_be32(f);
+
     return 0;
 }
 
@@ -348,6 +354,7 @@ static void ssd0323_class_init(ObjectClass *klass, void *data)
 
     k->init = ssd0323_init;
     k->transfer = ssd0323_transfer;
+    k->cs_polarity = SSI_CS_HIGH;
 }
 
 static TypeInfo ssd0323_info = {
diff --git a/hw/ssi-sd.c b/hw/ssi-sd.c
index b519bdb29..c5505ee24 100644
--- a/hw/ssi-sd.c
+++ b/hw/ssi-sd.c
@@ -197,6 +197,7 @@ static uint32_t ssi_sd_transfer(SSISlave *dev, uint32_t val)
 
 static void ssi_sd_save(QEMUFile *f, void *opaque)
 {
+    SSISlave *ss = SSI_SLAVE(opaque);
     ssi_sd_state *s = (ssi_sd_state *)opaque;
     int i;
 
@@ -209,10 +210,13 @@ static void ssi_sd_save(QEMUFile *f, void *opaque)
     qemu_put_be32(f, s->arglen);
     qemu_put_be32(f, s->response_pos);
     qemu_put_be32(f, s->stopping);
+
+    qemu_put_be32(f, ss->cs);
 }
 
 static int ssi_sd_load(QEMUFile *f, void *opaque, int version_id)
 {
+    SSISlave *ss = SSI_SLAVE(opaque);
     ssi_sd_state *s = (ssi_sd_state *)opaque;
     int i;
 
@@ -229,6 +233,8 @@ static int ssi_sd_load(QEMUFile *f, void *opaque, int version_id)
     s->response_pos = qemu_get_be32(f);
     s->stopping = qemu_get_be32(f);
 
+    ss->cs = qemu_get_be32(f);
+
     return 0;
 }
 
@@ -250,6 +256,7 @@ static void ssi_sd_class_init(ObjectClass *klass, void *data)
 
     k->init = ssi_sd_init;
     k->transfer = ssi_sd_transfer;
+    k->cs_polarity = SSI_CS_LOW;
 }
 
 static TypeInfo ssi_sd_info = {
diff --git a/hw/ssi.c b/hw/ssi.c
index e5f14a0ce..2b5635715 100644
--- a/hw/ssi.c
+++ b/hw/ssi.c
@@ -2,6 +2,8 @@
  * QEMU Synchronous Serial Interface support
  *
  * Copyright (c) 2009 CodeSourcery.
+ * Copyright (c) 2012 Peter A.G. Crosthwaite (peter.crosthwaite@petalogix.com)
+ * Copyright (c) 2012 PetaLogix Pty Ltd.
  * Written by Paul Brook
  *
  * This code is licensed under the GNU GPL v2.
@@ -25,17 +27,40 @@ static const TypeInfo ssi_bus_info = {
     .instance_size = sizeof(SSIBus),
 };
 
+static void ssi_cs_default(void *opaque, int n, int level)
+{
+    SSISlave *s = SSI_SLAVE(opaque);
+    bool cs = !!level;
+    assert(n == 0);
+    if (s->cs != cs) {
+        SSISlaveClass *ssc = SSI_SLAVE_GET_CLASS(s);
+        if (ssc->set_cs) {
+            ssc->set_cs(s, cs);
+        }
+    }
+    s->cs = cs;
+}
+
+static uint32_t ssi_transfer_raw_default(SSISlave *dev, uint32_t val)
+{
+    SSISlaveClass *ssc = SSI_SLAVE_GET_CLASS(dev);
+
+    if ((dev->cs && ssc->cs_polarity == SSI_CS_HIGH) ||
+            (!dev->cs && ssc->cs_polarity == SSI_CS_LOW) ||
+            ssc->cs_polarity == SSI_CS_NONE) {
+        return ssc->transfer(dev, val);
+    }
+    return 0;
+}
+
 static int ssi_slave_init(DeviceState *dev)
 {
     SSISlave *s = SSI_SLAVE(dev);
     SSISlaveClass *ssc = SSI_SLAVE_GET_CLASS(s);
-    SSIBus *bus;
-    BusChild *kid;
 
-    bus = FROM_QBUS(SSIBus, qdev_get_parent_bus(dev));
-    kid = QTAILQ_FIRST(&bus->qbus.children);
-    if (kid->child != dev || QTAILQ_NEXT(kid, sibling) != NULL) {
-        hw_error("Too many devices on SSI bus");
+    if (ssc->transfer_raw == ssi_transfer_raw_default &&
+            ssc->cs_polarity != SSI_CS_NONE) {
+        qdev_init_gpio_in(&s->qdev, ssi_cs_default, 1);
     }
 
     return ssc->init(s);
@@ -43,9 +68,14 @@ static int ssi_slave_init(DeviceState *dev)
 
 static void ssi_slave_class_init(ObjectClass *klass, void *data)
 {
+    SSISlaveClass *ssc = SSI_SLAVE_CLASS(klass);
     DeviceClass *dc = DEVICE_CLASS(klass);
+
     dc->init = ssi_slave_init;
     dc->bus_type = TYPE_SSI_BUS;
+    if (!ssc->transfer_raw) {
+        ssc->transfer_raw = ssi_transfer_raw_default;
+    }
 }
 
 static TypeInfo ssi_slave_info = {
@@ -56,10 +86,15 @@ static TypeInfo ssi_slave_info = {
     .abstract = true,
 };
 
+DeviceState *ssi_create_slave_no_init(SSIBus *bus, const char *name)
+{
+    return qdev_create(&bus->qbus, name);
+}
+
 DeviceState *ssi_create_slave(SSIBus *bus, const char *name)
 {
-    DeviceState *dev;
-    dev = qdev_create(&bus->qbus, name);
+    DeviceState *dev = ssi_create_slave_no_init(bus, name);
+
     qdev_init_nofail(dev);
     return dev;
 }
@@ -74,18 +109,29 @@ SSIBus *ssi_create_bus(DeviceState *parent, const char *name)
 uint32_t ssi_transfer(SSIBus *bus, uint32_t val)
 {
     BusChild *kid;
-    SSISlave *slave;
     SSISlaveClass *ssc;
+    uint32_t r = 0;
 
-    kid = QTAILQ_FIRST(&bus->qbus.children);
-    if (!kid) {
-        return 0;
+    QTAILQ_FOREACH(kid, &bus->qbus.children, sibling) {
+        SSISlave *slave = SSI_SLAVE(kid->child);
+        ssc = SSI_SLAVE_GET_CLASS(slave);
+        r |= ssc->transfer_raw(slave, val);
     }
-    slave = SSI_SLAVE(kid->child);
-    ssc = SSI_SLAVE_GET_CLASS(slave);
-    return ssc->transfer(slave, val);
+
+    return r;
 }
 
+const VMStateDescription vmstate_ssi_slave = {
+    .name = "SSISlave",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .minimum_version_id_old = 1,
+    .fields      = (VMStateField[]) {
+        VMSTATE_BOOL(cs, SSISlave),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
 static void ssi_slave_register_types(void)
 {
     type_register_static(&ssi_bus_info);
@@ -93,3 +139,36 @@ static void ssi_slave_register_types(void)
 }
 
 type_init(ssi_slave_register_types)
+
+typedef struct SSIAutoConnectArg {
+    qemu_irq **cs_linep;
+    SSIBus *bus;
+} SSIAutoConnectArg;
+
+static int ssi_auto_connect_slave(Object *child, void *opaque)
+{
+    SSIAutoConnectArg *arg = opaque;
+    SSISlave *dev = (SSISlave *)object_dynamic_cast(child, TYPE_SSI_SLAVE);
+    qemu_irq cs_line;
+
+    if (!dev) {
+        return 0;
+    }
+
+    cs_line = qdev_get_gpio_in(DEVICE(dev), 0);
+    qdev_set_parent_bus(DEVICE(dev), &arg->bus->qbus);
+    **arg->cs_linep = cs_line;
+    (*arg->cs_linep)++;
+    return 0;
+}
+
+void ssi_auto_connect_slaves(DeviceState *parent, qemu_irq *cs_line,
+                             SSIBus *bus)
+{
+    SSIAutoConnectArg arg = {
+        .cs_linep = &cs_line,
+        .bus = bus
+    };
+
+    object_child_foreach(OBJECT(parent), ssi_auto_connect_slave, &arg);
+}
diff --git a/hw/ssi.h b/hw/ssi.h
index 06657d7c1..a05d60beb 100644
--- a/hw/ssi.h
+++ b/hw/ssi.h
@@ -23,28 +23,70 @@ typedef struct SSISlave SSISlave;
 #define SSI_SLAVE_GET_CLASS(obj) \
      OBJECT_GET_CLASS(SSISlaveClass, (obj), TYPE_SSI_SLAVE)
 
+typedef enum {
+    SSI_CS_NONE = 0,
+    SSI_CS_LOW,
+    SSI_CS_HIGH,
+} SSICSMode;
+
 /* Slave devices.  */
 typedef struct SSISlaveClass {
     DeviceClass parent_class;
 
     int (*init)(SSISlave *dev);
+
+    /* if you have standard or no CS behaviour, just override transfer.
+     * This is called when the device cs is active (true by default).
+     */
     uint32_t (*transfer)(SSISlave *dev, uint32_t val);
+    /* called when the CS line changes. Optional, devices only need to implement
+     * this if they have side effects associated with the cs line (beyond
+     * tristating the txrx lines).
+     */
+    int (*set_cs)(SSISlave *dev, bool select);
+    /* define whether or not CS exists and is active low/high */
+    SSICSMode cs_polarity;
+
+    /* if you have non-standard CS behaviour override this to take control
+     * of the CS behaviour at the device level. transfer, set_cs, and
+     * cs_polarity are unused if this is overwritten. Transfer_raw will
+     * always be called for the device for every txrx access to the parent bus
+     */
+    uint32_t (*transfer_raw)(SSISlave *dev, uint32_t val);
 } SSISlaveClass;
 
 struct SSISlave {
     DeviceState qdev;
+
+    /* Chip select state */
+    bool cs;
 };
 
 #define SSI_SLAVE_FROM_QDEV(dev) DO_UPCAST(SSISlave, qdev, dev)
 #define FROM_SSI_SLAVE(type, dev) DO_UPCAST(type, ssidev, dev)
 
+extern const VMStateDescription vmstate_ssi_slave;
+
+#define VMSTATE_SSI_SLAVE(_field, _state) {                          \
+    .name       = (stringify(_field)),                               \
+    .size       = sizeof(SSISlave),                                  \
+    .vmsd       = &vmstate_ssi_slave,                                \
+    .flags      = VMS_STRUCT,                                        \
+    .offset     = vmstate_offset_value(_state, _field, SSISlave),    \
+}
+
 DeviceState *ssi_create_slave(SSIBus *bus, const char *name);
+DeviceState *ssi_create_slave_no_init(SSIBus *bus, const char *name);
 
 /* Master interface.  */
 SSIBus *ssi_create_bus(DeviceState *parent, const char *name);
 
 uint32_t ssi_transfer(SSIBus *bus, uint32_t val);
 
+/* Automatically connect all children nodes a spi controller as slaves */
+void ssi_auto_connect_slaves(DeviceState *parent, qemu_irq *cs_lines,
+                             SSIBus *bus);
+
 /* max111x.c */
 void max111x_set_input(DeviceState *dev, int line, uint8_t value);
 
diff --git a/hw/stellaris.c b/hw/stellaris.c
index 562fbbf49..b038f10f3 100644
--- a/hw/stellaris.c
+++ b/hw/stellaris.c
@@ -141,7 +141,7 @@ static void gptm_tick(void *opaque)
     gptm_update_irq(s);
 }
 
-static uint64_t gptm_read(void *opaque, target_phys_addr_t offset,
+static uint64_t gptm_read(void *opaque, hwaddr offset,
                           unsigned size)
 {
     gptm_state *s = (gptm_state *)opaque;
@@ -190,7 +190,7 @@ static uint64_t gptm_read(void *opaque, target_phys_addr_t offset,
     }
 }
 
-static void gptm_write(void *opaque, target_phys_addr_t offset,
+static void gptm_write(void *opaque, hwaddr offset,
                        uint64_t value, unsigned size)
 {
     gptm_state *s = (gptm_state *)opaque;
@@ -410,7 +410,7 @@ static int ssys_board_class(const ssys_state *s)
     }
 }
 
-static uint64_t ssys_read(void *opaque, target_phys_addr_t offset,
+static uint64_t ssys_read(void *opaque, hwaddr offset,
                           unsigned size)
 {
     ssys_state *s = (ssys_state *)opaque;
@@ -515,7 +515,7 @@ static void ssys_calculate_system_clock(ssys_state *s)
     }
 }
 
-static void ssys_write(void *opaque, target_phys_addr_t offset,
+static void ssys_write(void *opaque, hwaddr offset,
                        uint64_t value, unsigned size)
 {
     ssys_state *s = (ssys_state *)opaque;
@@ -701,7 +701,7 @@ typedef struct {
 #define STELLARIS_I2C_MCS_IDLE    0x20
 #define STELLARIS_I2C_MCS_BUSBSY  0x40
 
-static uint64_t stellaris_i2c_read(void *opaque, target_phys_addr_t offset,
+static uint64_t stellaris_i2c_read(void *opaque, hwaddr offset,
                                    unsigned size)
 {
     stellaris_i2c_state *s = (stellaris_i2c_state *)opaque;
@@ -738,7 +738,7 @@ static void stellaris_i2c_update(stellaris_i2c_state *s)
     qemu_set_irq(s->irq, level);
 }
 
-static void stellaris_i2c_write(void *opaque, target_phys_addr_t offset,
+static void stellaris_i2c_write(void *opaque, hwaddr offset,
                                 uint64_t value, unsigned size)
 {
     stellaris_i2c_state *s = (stellaris_i2c_state *)opaque;
@@ -989,7 +989,7 @@ static void stellaris_adc_reset(stellaris_adc_state *s)
     }
 }
 
-static uint64_t stellaris_adc_read(void *opaque, target_phys_addr_t offset,
+static uint64_t stellaris_adc_read(void *opaque, hwaddr offset,
                                    unsigned size)
 {
     stellaris_adc_state *s = (stellaris_adc_state *)opaque;
@@ -1037,7 +1037,7 @@ static uint64_t stellaris_adc_read(void *opaque, target_phys_addr_t offset,
     }
 }
 
-static void stellaris_adc_write(void *opaque, target_phys_addr_t offset,
+static void stellaris_adc_write(void *opaque, hwaddr offset,
                                 uint64_t value, unsigned size)
 {
     stellaris_adc_state *s = (stellaris_adc_state *)opaque;
@@ -1154,57 +1154,6 @@ static int stellaris_adc_init(SysBusDevice *dev)
     return 0;
 }
 
-/* Some boards have both an OLED controller and SD card connected to
-   the same SSI port, with the SD card chip select connected to a
-   GPIO pin.  Technically the OLED chip select is connected to the SSI
-   Fss pin.  We do not bother emulating that as both devices should
-   never be selected simultaneously, and our OLED controller ignores stray
-   0xff commands that occur when deselecting the SD card.  */
-
-typedef struct {
-    SSISlave ssidev;
-    qemu_irq irq;
-    int current_dev;
-    SSIBus *bus[2];
-} stellaris_ssi_bus_state;
-
-static void stellaris_ssi_bus_select(void *opaque, int irq, int level)
-{
-    stellaris_ssi_bus_state *s = (stellaris_ssi_bus_state *)opaque;
-
-    s->current_dev = level;
-}
-
-static uint32_t stellaris_ssi_bus_transfer(SSISlave *dev, uint32_t val)
-{
-    stellaris_ssi_bus_state *s = FROM_SSI_SLAVE(stellaris_ssi_bus_state, dev);
-
-    return ssi_transfer(s->bus[s->current_dev], val);
-}
-
-static const VMStateDescription vmstate_stellaris_ssi_bus = {
-    .name = "stellaris_ssi_bus",
-    .version_id = 1,
-    .minimum_version_id = 1,
-    .minimum_version_id_old = 1,
-    .fields      = (VMStateField[]) {
-        VMSTATE_INT32(current_dev, stellaris_ssi_bus_state),
-        VMSTATE_END_OF_LIST()
-    }
-};
-
-static int stellaris_ssi_bus_init(SSISlave *dev)
-{
-    stellaris_ssi_bus_state *s = FROM_SSI_SLAVE(stellaris_ssi_bus_state, dev);
-
-    s->bus[0] = ssi_create_bus(&dev->qdev, "ssi0");
-    s->bus[1] = ssi_create_bus(&dev->qdev, "ssi1");
-    qdev_init_gpio_in(&dev->qdev, stellaris_ssi_bus_select, 1);
-
-    vmstate_register(&dev->qdev, -1, &vmstate_stellaris_ssi_bus, s);
-    return 0;
-}
-
 /* Board init.  */
 static stellaris_board_info stellaris_boards[] = {
   { "LM3S811EVB",
@@ -1305,19 +1254,25 @@ static void stellaris_init(const char *kernel_filename, const char *cpu_model,
     if (board->dc2 & (1 << 4)) {
         dev = sysbus_create_simple("pl022", 0x40008000, pic[7]);
         if (board->peripherals & BP_OLED_SSI) {
-            DeviceState *mux;
             void *bus;
-
+            DeviceState *sddev;
+            DeviceState *ssddev;
+
+            /* Some boards have both an OLED controller and SD card connected to
+             * the same SSI port, with the SD card chip select connected to a
+             * GPIO pin.  Technically the OLED chip select is connected to the
+             * SSI Fss pin.  We do not bother emulating that as both devices
+             * should never be selected simultaneously, and our OLED controller
+             * ignores stray 0xff commands that occur when deselecting the SD
+             * card.
+             */
             bus = qdev_get_child_bus(dev, "ssi");
-            mux = ssi_create_slave(bus, "evb6965-ssi");
-            gpio_out[GPIO_D][0] = qdev_get_gpio_in(mux, 0);
-
-            bus = qdev_get_child_bus(mux, "ssi0");
-            ssi_create_slave(bus, "ssi-sd");
 
-            bus = qdev_get_child_bus(mux, "ssi1");
-            dev = ssi_create_slave(bus, "ssd0323");
-            gpio_out[GPIO_C][7] = qdev_get_gpio_in(dev, 0);
+            sddev = ssi_create_slave(bus, "ssi-sd");
+            ssddev = ssi_create_slave(bus, "ssd0323");
+            gpio_out[GPIO_D][0] = qemu_irq_split(qdev_get_gpio_in(sddev, 0),
+                                                 qdev_get_gpio_in(ssddev, 0));
+            gpio_out[GPIO_C][7] = qdev_get_gpio_in(ssddev, 1);
 
             /* Make sure the select pin is high.  */
             qemu_irq_raise(gpio_out[GPIO_D][0]);
@@ -1358,19 +1313,17 @@ static void stellaris_init(const char *kernel_filename, const char *cpu_model,
 }
 
 /* FIXME: Figure out how to generate these from stellaris_boards.  */
-static void lm3s811evb_init(ram_addr_t ram_size,
-                     const char *boot_device,
-                     const char *kernel_filename, const char *kernel_cmdline,
-                     const char *initrd_filename, const char *cpu_model)
+static void lm3s811evb_init(QEMUMachineInitArgs *args)
 {
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
     stellaris_init(kernel_filename, cpu_model, &stellaris_boards[0]);
 }
 
-static void lm3s6965evb_init(ram_addr_t ram_size,
-                     const char *boot_device,
-                     const char *kernel_filename, const char *kernel_cmdline,
-                     const char *initrd_filename, const char *cpu_model)
+static void lm3s6965evb_init(QEMUMachineInitArgs *args)
 {
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
     stellaris_init(kernel_filename, cpu_model, &stellaris_boards[1]);
 }
 
@@ -1394,21 +1347,6 @@ static void stellaris_machine_init(void)
 
 machine_init(stellaris_machine_init);
 
-static void stellaris_ssi_bus_class_init(ObjectClass *klass, void *data)
-{
-    SSISlaveClass *k = SSI_SLAVE_CLASS(klass);
-
-    k->init = stellaris_ssi_bus_init;
-    k->transfer = stellaris_ssi_bus_transfer;
-}
-
-static TypeInfo stellaris_ssi_bus_info = {
-    .name          = "evb6965-ssi",
-    .parent        = TYPE_SSI_SLAVE,
-    .instance_size = sizeof(stellaris_ssi_bus_state),
-    .class_init    = stellaris_ssi_bus_class_init,
-};
-
 static void stellaris_i2c_class_init(ObjectClass *klass, void *data)
 {
     SysBusDeviceClass *sdc = SYS_BUS_DEVICE_CLASS(klass);
@@ -1456,7 +1394,6 @@ static void stellaris_register_types(void)
     type_register_static(&stellaris_i2c_info);
     type_register_static(&stellaris_gptm_info);
     type_register_static(&stellaris_adc_info);
-    type_register_static(&stellaris_ssi_bus_info);
 }
 
 type_init(stellaris_register_types)
diff --git a/hw/stellaris_enet.c b/hw/stellaris_enet.c
index bc97280cc..a530b1059 100644
--- a/hw/stellaris_enet.c
+++ b/hw/stellaris_enet.c
@@ -130,7 +130,7 @@ static int stellaris_enet_can_receive(NetClientState *nc)
     return (s->np < 31);
 }
 
-static uint64_t stellaris_enet_read(void *opaque, target_phys_addr_t offset,
+static uint64_t stellaris_enet_read(void *opaque, hwaddr offset,
                                     unsigned size)
 {
     stellaris_enet_state *s = (stellaris_enet_state *)opaque;
@@ -198,7 +198,7 @@ static uint64_t stellaris_enet_read(void *opaque, target_phys_addr_t offset,
     }
 }
 
-static void stellaris_enet_write(void *opaque, target_phys_addr_t offset,
+static void stellaris_enet_write(void *opaque, hwaddr offset,
                                  uint64_t value, unsigned size)
 {
     stellaris_enet_state *s = (stellaris_enet_state *)opaque;
diff --git a/hw/strongarm.c b/hw/strongarm.c
index 7150eeb2d..43855151c 100644
--- a/hw/strongarm.c
+++ b/hw/strongarm.c
@@ -59,7 +59,7 @@
 #endif
 
 static struct {
-    target_phys_addr_t io_base;
+    hwaddr io_base;
     int irq;
 } sa_serial[] = {
     { 0x80010000, SA_PIC_UART1 },
@@ -113,7 +113,7 @@ static void strongarm_pic_set_irq(void *opaque, int irq, int level)
     strongarm_pic_update(s);
 }
 
-static uint64_t strongarm_pic_mem_read(void *opaque, target_phys_addr_t offset,
+static uint64_t strongarm_pic_mem_read(void *opaque, hwaddr offset,
                                        unsigned size)
 {
     StrongARMPICState *s = opaque;
@@ -138,7 +138,7 @@ static uint64_t strongarm_pic_mem_read(void *opaque, target_phys_addr_t offset,
     }
 }
 
-static void strongarm_pic_mem_write(void *opaque, target_phys_addr_t offset,
+static void strongarm_pic_mem_write(void *opaque, hwaddr offset,
                                     uint64_t value, unsigned size)
 {
     StrongARMPICState *s = opaque;
@@ -294,7 +294,7 @@ static inline void strongarm_rtc_hz_tick(void *opaque)
     strongarm_rtc_int_update(s);
 }
 
-static uint64_t strongarm_rtc_read(void *opaque, target_phys_addr_t addr,
+static uint64_t strongarm_rtc_read(void *opaque, hwaddr addr,
                                    unsigned size)
 {
     StrongARMRTCState *s = opaque;
@@ -316,7 +316,7 @@ static uint64_t strongarm_rtc_read(void *opaque, target_phys_addr_t addr,
     }
 }
 
-static void strongarm_rtc_write(void *opaque, target_phys_addr_t addr,
+static void strongarm_rtc_write(void *opaque, hwaddr addr,
                                 uint64_t value, unsigned size)
 {
     StrongARMRTCState *s = opaque;
@@ -517,7 +517,7 @@ static void strongarm_gpio_handler_update(StrongARMGPIOInfo *s)
     s->prev_level = level;
 }
 
-static uint64_t strongarm_gpio_read(void *opaque, target_phys_addr_t offset,
+static uint64_t strongarm_gpio_read(void *opaque, hwaddr offset,
                                     unsigned size)
 {
     StrongARMGPIOInfo *s = opaque;
@@ -559,7 +559,7 @@ static uint64_t strongarm_gpio_read(void *opaque, target_phys_addr_t offset,
     return 0;
 }
 
-static void strongarm_gpio_write(void *opaque, target_phys_addr_t offset,
+static void strongarm_gpio_write(void *opaque, hwaddr offset,
                                  uint64_t value, unsigned size)
 {
     StrongARMGPIOInfo *s = opaque;
@@ -609,7 +609,7 @@ static const MemoryRegionOps strongarm_gpio_ops = {
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-static DeviceState *strongarm_gpio_init(target_phys_addr_t base,
+static DeviceState *strongarm_gpio_init(hwaddr base,
                 DeviceState *pic)
 {
     DeviceState *dev;
@@ -729,7 +729,7 @@ static void strongarm_ppc_handler_update(StrongARMPPCInfo *s)
     s->prev_level = level;
 }
 
-static uint64_t strongarm_ppc_read(void *opaque, target_phys_addr_t offset,
+static uint64_t strongarm_ppc_read(void *opaque, hwaddr offset,
                                    unsigned size)
 {
     StrongARMPPCInfo *s = opaque;
@@ -759,7 +759,7 @@ static uint64_t strongarm_ppc_read(void *opaque, target_phys_addr_t offset,
     return 0;
 }
 
-static void strongarm_ppc_write(void *opaque, target_phys_addr_t offset,
+static void strongarm_ppc_write(void *opaque, hwaddr offset,
                                 uint64_t value, unsigned size)
 {
     StrongARMPPCInfo *s = opaque;
@@ -1095,7 +1095,7 @@ static void strongarm_uart_tx(void *opaque)
     strongarm_uart_update_int_status(s);
 }
 
-static uint64_t strongarm_uart_read(void *opaque, target_phys_addr_t addr,
+static uint64_t strongarm_uart_read(void *opaque, hwaddr addr,
                                     unsigned size)
 {
     StrongARMUARTState *s = opaque;
@@ -1137,7 +1137,7 @@ static uint64_t strongarm_uart_read(void *opaque, target_phys_addr_t addr,
     }
 }
 
-static void strongarm_uart_write(void *opaque, target_phys_addr_t addr,
+static void strongarm_uart_write(void *opaque, hwaddr addr,
                                  uint64_t value, unsigned size)
 {
     StrongARMUARTState *s = opaque;
@@ -1376,7 +1376,7 @@ static void strongarm_ssp_fifo_update(StrongARMSSPState *s)
     strongarm_ssp_int_update(s);
 }
 
-static uint64_t strongarm_ssp_read(void *opaque, target_phys_addr_t addr,
+static uint64_t strongarm_ssp_read(void *opaque, hwaddr addr,
                                    unsigned size)
 {
     StrongARMSSPState *s = opaque;
@@ -1409,7 +1409,7 @@ static uint64_t strongarm_ssp_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void strongarm_ssp_write(void *opaque, target_phys_addr_t addr,
+static void strongarm_ssp_write(void *opaque, hwaddr addr,
                                 uint64_t value, unsigned size)
 {
     StrongARMSSPState *s = opaque;
diff --git a/hw/sun4c_intctl.c b/hw/sun4c_intctl.c
index 8dfa5ecab..702e9f544 100644
--- a/hw/sun4c_intctl.c
+++ b/hw/sun4c_intctl.c
@@ -61,7 +61,7 @@ typedef struct Sun4c_INTCTLState {
 
 static void sun4c_check_interrupts(void *opaque);
 
-static uint64_t sun4c_intctl_mem_read(void *opaque, target_phys_addr_t addr,
+static uint64_t sun4c_intctl_mem_read(void *opaque, hwaddr addr,
                                       unsigned size)
 {
     Sun4c_INTCTLState *s = opaque;
@@ -73,7 +73,7 @@ static uint64_t sun4c_intctl_mem_read(void *opaque, target_phys_addr_t addr,
     return ret;
 }
 
-static void sun4c_intctl_mem_write(void *opaque, target_phys_addr_t addr,
+static void sun4c_intctl_mem_write(void *opaque, hwaddr addr,
                                    uint64_t val, unsigned size)
 {
     Sun4c_INTCTLState *s = opaque;
@@ -94,29 +94,6 @@ static const MemoryRegionOps sun4c_intctl_mem_ops = {
     },
 };
 
-void sun4c_pic_info(Monitor *mon, void *opaque)
-{
-    Sun4c_INTCTLState *s = opaque;
-
-    monitor_printf(mon, "master: pending 0x%2.2x, enabled 0x%2.2x\n",
-                   s->pending, s->reg);
-}
-
-void sun4c_irq_info(Monitor *mon, void *opaque)
-{
-#ifndef DEBUG_IRQ_COUNT
-    monitor_printf(mon, "irq statistic code not compiled.\n");
-#else
-    Sun4c_INTCTLState *s = opaque;
-    int64_t count;
-
-    monitor_printf(mon, "IRQ statistics:\n");
-    count = s->irq_count;
-    if (count > 0)
-        monitor_printf(mon, " %" PRId64 "\n", count);
-#endif
-}
-
 static const uint32_t intbit_to_level[] = { 0, 1, 4, 6, 8, 10, 0, 14, };
 
 static void sun4c_check_interrupts(void *opaque)
diff --git a/hw/sun4m.c b/hw/sun4m.c
index 0f909b5f8..1a786762a 100644
--- a/hw/sun4m.c
+++ b/hw/sun4m.c
@@ -87,16 +87,16 @@
 #define ESCC_CLOCK 4915200
 
 struct sun4m_hwdef {
-    target_phys_addr_t iommu_base, iommu_pad_base, iommu_pad_len, slavio_base;
-    target_phys_addr_t intctl_base, counter_base, nvram_base, ms_kb_base;
-    target_phys_addr_t serial_base, fd_base;
-    target_phys_addr_t afx_base, idreg_base, dma_base, esp_base, le_base;
-    target_phys_addr_t tcx_base, cs_base, apc_base, aux1_base, aux2_base;
-    target_phys_addr_t bpp_base, dbri_base, sx_base;
+    hwaddr iommu_base, iommu_pad_base, iommu_pad_len, slavio_base;
+    hwaddr intctl_base, counter_base, nvram_base, ms_kb_base;
+    hwaddr serial_base, fd_base;
+    hwaddr afx_base, idreg_base, dma_base, esp_base, le_base;
+    hwaddr tcx_base, cs_base, apc_base, aux1_base, aux2_base;
+    hwaddr bpp_base, dbri_base, sx_base;
     struct {
-        target_phys_addr_t reg_base, vram_base;
+        hwaddr reg_base, vram_base;
     } vsimm[MAX_VSIMMS];
-    target_phys_addr_t ecc_base;
+    hwaddr ecc_base;
     uint64_t max_mem;
     const char * const default_cpu_model;
     uint32_t ecc_version;
@@ -108,13 +108,13 @@ struct sun4m_hwdef {
 #define MAX_IOUNITS 5
 
 struct sun4d_hwdef {
-    target_phys_addr_t iounit_bases[MAX_IOUNITS], slavio_base;
-    target_phys_addr_t counter_base, nvram_base, ms_kb_base;
-    target_phys_addr_t serial_base;
-    target_phys_addr_t espdma_base, esp_base;
-    target_phys_addr_t ledma_base, le_base;
-    target_phys_addr_t tcx_base;
-    target_phys_addr_t sbi_base;
+    hwaddr iounit_bases[MAX_IOUNITS], slavio_base;
+    hwaddr counter_base, nvram_base, ms_kb_base;
+    hwaddr serial_base;
+    hwaddr espdma_base, esp_base;
+    hwaddr ledma_base, le_base;
+    hwaddr tcx_base;
+    hwaddr sbi_base;
     uint64_t max_mem;
     const char * const default_cpu_model;
     uint32_t iounit_version;
@@ -123,11 +123,11 @@ struct sun4d_hwdef {
 };
 
 struct sun4c_hwdef {
-    target_phys_addr_t iommu_base, slavio_base;
-    target_phys_addr_t intctl_base, counter_base, nvram_base, ms_kb_base;
-    target_phys_addr_t serial_base, fd_base;
-    target_phys_addr_t idreg_base, dma_base, esp_base, le_base;
-    target_phys_addr_t tcx_base, aux1_base;
+    hwaddr iommu_base, slavio_base;
+    hwaddr intctl_base, counter_base, nvram_base, ms_kb_base;
+    hwaddr serial_base, fd_base;
+    hwaddr idreg_base, dma_base, esp_base, le_base;
+    hwaddr tcx_base, aux1_base;
     uint64_t max_mem;
     const char * const default_cpu_model;
     uint32_t iommu_version;
@@ -253,21 +253,24 @@ void cpu_check_irqs(CPUSPARCState *env)
     }
 }
 
-static void cpu_kick_irq(CPUSPARCState *env)
+static void cpu_kick_irq(SPARCCPU *cpu)
 {
+    CPUSPARCState *env = &cpu->env;
+
     env->halted = 0;
     cpu_check_irqs(env);
-    qemu_cpu_kick(env);
+    qemu_cpu_kick(CPU(cpu));
 }
 
 static void cpu_set_irq(void *opaque, int irq, int level)
 {
-    CPUSPARCState *env = opaque;
+    SPARCCPU *cpu = opaque;
+    CPUSPARCState *env = &cpu->env;
 
     if (level) {
         trace_sun4m_cpu_set_irq_raise(irq);
         env->pil_in |= 1 << irq;
-        cpu_kick_irq(env);
+        cpu_kick_irq(cpu);
     } else {
         trace_sun4m_cpu_set_irq_lower(irq);
         env->pil_in &= ~(1 << irq);
@@ -370,7 +373,7 @@ static unsigned long sun4m_load_kernel(const char *kernel_filename,
     return kernel_size;
 }
 
-static void *iommu_init(target_phys_addr_t addr, uint32_t version, qemu_irq irq)
+static void *iommu_init(hwaddr addr, uint32_t version, qemu_irq irq)
 {
     DeviceState *dev;
     SysBusDevice *s;
@@ -385,7 +388,7 @@ static void *iommu_init(target_phys_addr_t addr, uint32_t version, qemu_irq irq)
     return s;
 }
 
-static void *sparc32_dma_init(target_phys_addr_t daddr, qemu_irq parent_irq,
+static void *sparc32_dma_init(hwaddr daddr, qemu_irq parent_irq,
                               void *iommu, qemu_irq *dev_irq, int is_ledma)
 {
     DeviceState *dev;
@@ -403,7 +406,7 @@ static void *sparc32_dma_init(target_phys_addr_t daddr, qemu_irq parent_irq,
     return s;
 }
 
-static void lance_init(NICInfo *nd, target_phys_addr_t leaddr,
+static void lance_init(NICInfo *nd, hwaddr leaddr,
                        void *dma_opaque, qemu_irq irq)
 {
     DeviceState *dev;
@@ -423,8 +426,8 @@ static void lance_init(NICInfo *nd, target_phys_addr_t leaddr,
     qdev_connect_gpio_out(dma_opaque, 0, reset);
 }
 
-static DeviceState *slavio_intctl_init(target_phys_addr_t addr,
-                                       target_phys_addr_t addrg,
+static DeviceState *slavio_intctl_init(hwaddr addr,
+                                       hwaddr addrg,
                                        qemu_irq **parent_irq)
 {
     DeviceState *dev;
@@ -452,7 +455,7 @@ static DeviceState *slavio_intctl_init(target_phys_addr_t addr,
 #define SYS_TIMER_OFFSET      0x10000ULL
 #define CPU_TIMER_OFFSET(cpu) (0x1000ULL * cpu)
 
-static void slavio_timer_init_all(target_phys_addr_t addr, qemu_irq master_irq,
+static void slavio_timer_init_all(hwaddr addr, qemu_irq master_irq,
                                   qemu_irq *cpu_irqs, unsigned int num_cpus)
 {
     DeviceState *dev;
@@ -467,20 +470,31 @@ static void slavio_timer_init_all(target_phys_addr_t addr, qemu_irq master_irq,
     sysbus_mmio_map(s, 0, addr + SYS_TIMER_OFFSET);
 
     for (i = 0; i < MAX_CPUS; i++) {
-        sysbus_mmio_map(s, i + 1, addr + (target_phys_addr_t)CPU_TIMER_OFFSET(i));
+        sysbus_mmio_map(s, i + 1, addr + (hwaddr)CPU_TIMER_OFFSET(i));
         sysbus_connect_irq(s, i + 1, cpu_irqs[i]);
     }
 }
 
+static qemu_irq  slavio_system_powerdown;
+
+static void slavio_powerdown_req(Notifier *n, void *opaque)
+{
+    qemu_irq_raise(slavio_system_powerdown);
+}
+
+static Notifier slavio_system_powerdown_notifier = {
+    .notify = slavio_powerdown_req
+};
+
 #define MISC_LEDS 0x01600000
 #define MISC_CFG  0x01800000
 #define MISC_DIAG 0x01a00000
 #define MISC_MDM  0x01b00000
 #define MISC_SYS  0x01f00000
 
-static void slavio_misc_init(target_phys_addr_t base,
-                             target_phys_addr_t aux1_base,
-                             target_phys_addr_t aux2_base, qemu_irq irq,
+static void slavio_misc_init(hwaddr base,
+                             hwaddr aux1_base,
+                             hwaddr aux2_base, qemu_irq irq,
                              qemu_irq fdc_tc)
 {
     DeviceState *dev;
@@ -514,10 +528,11 @@ static void slavio_misc_init(target_phys_addr_t base,
     }
     sysbus_connect_irq(s, 0, irq);
     sysbus_connect_irq(s, 1, fdc_tc);
-    qemu_system_powerdown = qdev_get_gpio_in(dev, 0);
+    slavio_system_powerdown = qdev_get_gpio_in(dev, 0);
+    qemu_register_powerdown_notifier(&slavio_system_powerdown_notifier);
 }
 
-static void ecc_init(target_phys_addr_t base, qemu_irq irq, uint32_t version)
+static void ecc_init(hwaddr base, qemu_irq irq, uint32_t version)
 {
     DeviceState *dev;
     SysBusDevice *s;
@@ -533,7 +548,7 @@ static void ecc_init(target_phys_addr_t base, qemu_irq irq, uint32_t version)
     }
 }
 
-static void apc_init(target_phys_addr_t power_base, qemu_irq cpu_halt)
+static void apc_init(hwaddr power_base, qemu_irq cpu_halt)
 {
     DeviceState *dev;
     SysBusDevice *s;
@@ -546,7 +561,7 @@ static void apc_init(target_phys_addr_t power_base, qemu_irq cpu_halt)
     sysbus_connect_irq(s, 0, cpu_halt);
 }
 
-static void tcx_init(target_phys_addr_t addr, int vram_size, int width,
+static void tcx_init(hwaddr addr, int vram_size, int width,
                      int height, int depth)
 {
     DeviceState *dev;
@@ -582,7 +597,7 @@ static void tcx_init(target_phys_addr_t addr, int vram_size, int width,
 /* NCR89C100/MACIO Internal ID register */
 static const uint8_t idreg_data[] = { 0xfe, 0x81, 0x01, 0x03 };
 
-static void idreg_init(target_phys_addr_t addr)
+static void idreg_init(hwaddr addr)
 {
     DeviceState *dev;
     SysBusDevice *s;
@@ -631,7 +646,7 @@ typedef struct AFXState {
 } AFXState;
 
 /* SS-5 TCX AFX register */
-static void afx_init(target_phys_addr_t addr)
+static void afx_init(hwaddr addr)
 {
     DeviceState *dev;
     SysBusDevice *s;
@@ -675,11 +690,11 @@ typedef struct PROMState {
 /* Boot PROM (OpenBIOS) */
 static uint64_t translate_prom_address(void *opaque, uint64_t addr)
 {
-    target_phys_addr_t *base_addr = (target_phys_addr_t *)opaque;
+    hwaddr *base_addr = (hwaddr *)opaque;
     return addr + *base_addr - PROM_VADDR;
 }
 
-static void prom_init(target_phys_addr_t addr, const char *bios_name)
+static void prom_init(hwaddr addr, const char *bios_name)
 {
     DeviceState *dev;
     SysBusDevice *s;
@@ -762,7 +777,7 @@ static int ram_init1(SysBusDevice *dev)
     return 0;
 }
 
-static void ram_init(target_phys_addr_t addr, ram_addr_t RAM_size,
+static void ram_init(hwaddr addr, ram_addr_t RAM_size,
                      uint64_t max_mem)
 {
     DeviceState *dev;
@@ -828,7 +843,7 @@ static void cpu_devinit(const char *cpu_model, unsigned int id,
         qemu_register_reset(secondary_cpu_reset, cpu);
         env->halted = 1;
     }
-    *cpu_irqs = qemu_allocate_irqs(cpu_set_irq, env, MAX_PILS);
+    *cpu_irqs = qemu_allocate_irqs(cpu_set_irq, cpu, MAX_PILS);
     env->prom_addr = prom_addr;
 }
 
@@ -1291,92 +1306,118 @@ static const struct sun4m_hwdef sun4m_hwdefs[] = {
 };
 
 /* SPARCstation 5 hardware initialisation */
-static void ss5_init(ram_addr_t RAM_size,
-                     const char *boot_device,
-                     const char *kernel_filename, const char *kernel_cmdline,
-                     const char *initrd_filename, const char *cpu_model)
+static void ss5_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t RAM_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
+    const char *boot_device = args->boot_device;
     sun4m_hw_init(&sun4m_hwdefs[0], RAM_size, boot_device, kernel_filename,
                   kernel_cmdline, initrd_filename, cpu_model);
 }
 
 /* SPARCstation 10 hardware initialisation */
-static void ss10_init(ram_addr_t RAM_size,
-                      const char *boot_device,
-                      const char *kernel_filename, const char *kernel_cmdline,
-                      const char *initrd_filename, const char *cpu_model)
+static void ss10_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t RAM_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
+    const char *boot_device = args->boot_device;
     sun4m_hw_init(&sun4m_hwdefs[1], RAM_size, boot_device, kernel_filename,
                   kernel_cmdline, initrd_filename, cpu_model);
 }
 
 /* SPARCserver 600MP hardware initialisation */
-static void ss600mp_init(ram_addr_t RAM_size,
-                         const char *boot_device,
-                         const char *kernel_filename,
-                         const char *kernel_cmdline,
-                         const char *initrd_filename, const char *cpu_model)
+static void ss600mp_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t RAM_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
+    const char *boot_device = args->boot_device;
     sun4m_hw_init(&sun4m_hwdefs[2], RAM_size, boot_device, kernel_filename,
                   kernel_cmdline, initrd_filename, cpu_model);
 }
 
 /* SPARCstation 20 hardware initialisation */
-static void ss20_init(ram_addr_t RAM_size,
-                      const char *boot_device,
-                      const char *kernel_filename, const char *kernel_cmdline,
-                      const char *initrd_filename, const char *cpu_model)
+static void ss20_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t RAM_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
+    const char *boot_device = args->boot_device;
     sun4m_hw_init(&sun4m_hwdefs[3], RAM_size, boot_device, kernel_filename,
                   kernel_cmdline, initrd_filename, cpu_model);
 }
 
 /* SPARCstation Voyager hardware initialisation */
-static void vger_init(ram_addr_t RAM_size,
-                      const char *boot_device,
-                      const char *kernel_filename, const char *kernel_cmdline,
-                      const char *initrd_filename, const char *cpu_model)
+static void vger_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t RAM_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
+    const char *boot_device = args->boot_device;
     sun4m_hw_init(&sun4m_hwdefs[4], RAM_size, boot_device, kernel_filename,
                   kernel_cmdline, initrd_filename, cpu_model);
 }
 
 /* SPARCstation LX hardware initialisation */
-static void ss_lx_init(ram_addr_t RAM_size,
-                       const char *boot_device,
-                       const char *kernel_filename, const char *kernel_cmdline,
-                       const char *initrd_filename, const char *cpu_model)
+static void ss_lx_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t RAM_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
+    const char *boot_device = args->boot_device;
     sun4m_hw_init(&sun4m_hwdefs[5], RAM_size, boot_device, kernel_filename,
                   kernel_cmdline, initrd_filename, cpu_model);
 }
 
 /* SPARCstation 4 hardware initialisation */
-static void ss4_init(ram_addr_t RAM_size,
-                     const char *boot_device,
-                     const char *kernel_filename, const char *kernel_cmdline,
-                     const char *initrd_filename, const char *cpu_model)
+static void ss4_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t RAM_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
+    const char *boot_device = args->boot_device;
     sun4m_hw_init(&sun4m_hwdefs[6], RAM_size, boot_device, kernel_filename,
                   kernel_cmdline, initrd_filename, cpu_model);
 }
 
 /* SPARCClassic hardware initialisation */
-static void scls_init(ram_addr_t RAM_size,
-                      const char *boot_device,
-                      const char *kernel_filename, const char *kernel_cmdline,
-                      const char *initrd_filename, const char *cpu_model)
+static void scls_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t RAM_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
+    const char *boot_device = args->boot_device;
     sun4m_hw_init(&sun4m_hwdefs[7], RAM_size, boot_device, kernel_filename,
                   kernel_cmdline, initrd_filename, cpu_model);
 }
 
 /* SPARCbook hardware initialisation */
-static void sbook_init(ram_addr_t RAM_size,
-                       const char *boot_device,
-                       const char *kernel_filename, const char *kernel_cmdline,
-                       const char *initrd_filename, const char *cpu_model)
+static void sbook_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t RAM_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
+    const char *boot_device = args->boot_device;
     sun4m_hw_init(&sun4m_hwdefs[8], RAM_size, boot_device, kernel_filename,
                   kernel_cmdline, initrd_filename, cpu_model);
 }
@@ -1503,7 +1544,7 @@ static const struct sun4d_hwdef sun4d_hwdefs[] = {
     },
 };
 
-static DeviceState *sbi_init(target_phys_addr_t addr, qemu_irq **parent_irq)
+static DeviceState *sbi_init(hwaddr addr, qemu_irq **parent_irq)
 {
     DeviceState *dev;
     SysBusDevice *s;
@@ -1564,7 +1605,7 @@ static void sun4d_hw_init(const struct sun4d_hwdef *hwdef, ram_addr_t RAM_size,
     }
 
     for (i = 0; i < MAX_IOUNITS; i++)
-        if (hwdef->iounit_bases[i] != (target_phys_addr_t)-1)
+        if (hwdef->iounit_bases[i] != (hwaddr)-1)
             iounits[i] = iommu_init(hwdef->iounit_bases[i],
                                     hwdef->iounit_version,
                                     sbi_irq[0]);
@@ -1639,21 +1680,27 @@ static void sun4d_hw_init(const struct sun4d_hwdef *hwdef, ram_addr_t RAM_size,
 }
 
 /* SPARCserver 1000 hardware initialisation */
-static void ss1000_init(ram_addr_t RAM_size,
-                        const char *boot_device,
-                        const char *kernel_filename, const char *kernel_cmdline,
-                        const char *initrd_filename, const char *cpu_model)
+static void ss1000_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t RAM_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
+    const char *boot_device = args->boot_device;
     sun4d_hw_init(&sun4d_hwdefs[0], RAM_size, boot_device, kernel_filename,
                   kernel_cmdline, initrd_filename, cpu_model);
 }
 
 /* SPARCcenter 2000 hardware initialisation */
-static void ss2000_init(ram_addr_t RAM_size,
-                        const char *boot_device,
-                        const char *kernel_filename, const char *kernel_cmdline,
-                        const char *initrd_filename, const char *cpu_model)
+static void ss2000_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t RAM_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
+    const char *boot_device = args->boot_device;
     sun4d_hw_init(&sun4d_hwdefs[1], RAM_size, boot_device, kernel_filename,
                   kernel_cmdline, initrd_filename, cpu_model);
 }
@@ -1697,7 +1744,7 @@ static const struct sun4c_hwdef sun4c_hwdefs[] = {
     },
 };
 
-static DeviceState *sun4c_intctl_init(target_phys_addr_t addr,
+static DeviceState *sun4c_intctl_init(hwaddr addr,
                                       qemu_irq *parent_irq)
 {
     DeviceState *dev;
@@ -1778,7 +1825,7 @@ static void sun4c_hw_init(const struct sun4c_hwdef *hwdef, ram_addr_t RAM_size,
               slavio_irq[1], serial_hds[0], serial_hds[1],
               ESCC_CLOCK, 1);
 
-    if (hwdef->fd_base != (target_phys_addr_t)-1) {
+    if (hwdef->fd_base != (hwaddr)-1) {
         /* there is zero or one floppy drive */
         memset(fd, 0, sizeof(fd));
         fd[0] = drive_get(IF_FLOPPY, 0, 0);
@@ -1833,11 +1880,14 @@ static void sun4c_hw_init(const struct sun4c_hwdef *hwdef, ram_addr_t RAM_size,
 }
 
 /* SPARCstation 2 hardware initialisation */
-static void ss2_init(ram_addr_t RAM_size,
-                     const char *boot_device,
-                     const char *kernel_filename, const char *kernel_cmdline,
-                     const char *initrd_filename, const char *cpu_model)
+static void ss2_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t RAM_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
+    const char *boot_device = args->boot_device;
     sun4c_hw_init(&sun4c_hwdefs[0], RAM_size, boot_device, kernel_filename,
                   kernel_cmdline, initrd_filename, cpu_model);
 }
diff --git a/hw/sun4m.h b/hw/sun4m.h
index 504c3af41..47eb945f0 100644
--- a/hw/sun4m.h
+++ b/hw/sun4m.h
@@ -6,17 +6,17 @@
 /* Devices used by sparc32 system.  */
 
 /* iommu.c */
-void sparc_iommu_memory_rw(void *opaque, target_phys_addr_t addr,
+void sparc_iommu_memory_rw(void *opaque, hwaddr addr,
                                  uint8_t *buf, int len, int is_write);
 static inline void sparc_iommu_memory_read(void *opaque,
-                                           target_phys_addr_t addr,
+                                           hwaddr addr,
                                            uint8_t *buf, int len)
 {
     sparc_iommu_memory_rw(opaque, addr, buf, len, 0);
 }
 
 static inline void sparc_iommu_memory_write(void *opaque,
-                                            target_phys_addr_t addr,
+                                            hwaddr addr,
                                             uint8_t *buf, int len)
 {
     sparc_iommu_memory_rw(opaque, addr, buf, len, 1);
@@ -26,10 +26,6 @@ static inline void sparc_iommu_memory_write(void *opaque,
 void slavio_pic_info(Monitor *mon, DeviceState *dev);
 void slavio_irq_info(Monitor *mon, DeviceState *dev);
 
-/* sun4c_intctl.c */
-void sun4c_pic_info(Monitor *mon, void *opaque);
-void sun4c_irq_info(Monitor *mon, void *opaque);
-
 /* sun4m.c */
 void sun4m_pic_info(Monitor *mon);
 void sun4m_irq_info(Monitor *mon);
diff --git a/hw/sun4m_iommu.c b/hw/sun4m_iommu.c
index ebefa91b7..ce6819e10 100644
--- a/hw/sun4m_iommu.c
+++ b/hw/sun4m_iommu.c
@@ -130,16 +130,16 @@ typedef struct IOMMUState {
     SysBusDevice busdev;
     MemoryRegion iomem;
     uint32_t regs[IOMMU_NREGS];
-    target_phys_addr_t iostart;
+    hwaddr iostart;
     qemu_irq irq;
     uint32_t version;
 } IOMMUState;
 
-static uint64_t iommu_mem_read(void *opaque, target_phys_addr_t addr,
+static uint64_t iommu_mem_read(void *opaque, hwaddr addr,
                                unsigned size)
 {
     IOMMUState *s = opaque;
-    target_phys_addr_t saddr;
+    hwaddr saddr;
     uint32_t ret;
 
     saddr = addr >> 2;
@@ -157,11 +157,11 @@ static uint64_t iommu_mem_read(void *opaque, target_phys_addr_t addr,
     return ret;
 }
 
-static void iommu_mem_write(void *opaque, target_phys_addr_t addr,
+static void iommu_mem_write(void *opaque, hwaddr addr,
                             uint64_t val, unsigned size)
 {
     IOMMUState *s = opaque;
-    target_phys_addr_t saddr;
+    hwaddr saddr;
 
     saddr = addr >> 2;
     trace_sun4m_iommu_mem_writel(saddr, val);
@@ -249,11 +249,11 @@ static const MemoryRegionOps iommu_mem_ops = {
     },
 };
 
-static uint32_t iommu_page_get_flags(IOMMUState *s, target_phys_addr_t addr)
+static uint32_t iommu_page_get_flags(IOMMUState *s, hwaddr addr)
 {
     uint32_t ret;
-    target_phys_addr_t iopte;
-    target_phys_addr_t pa = addr;
+    hwaddr iopte;
+    hwaddr pa = addr;
 
     iopte = s->regs[IOMMU_BASE] << 4;
     addr &= ~s->iostart;
@@ -264,17 +264,17 @@ static uint32_t iommu_page_get_flags(IOMMUState *s, target_phys_addr_t addr)
     return ret;
 }
 
-static target_phys_addr_t iommu_translate_pa(target_phys_addr_t addr,
+static hwaddr iommu_translate_pa(hwaddr addr,
                                              uint32_t pte)
 {
-    target_phys_addr_t pa;
+    hwaddr pa;
 
     pa = ((pte & IOPTE_PAGE) << 4) + (addr & ~IOMMU_PAGE_MASK);
     trace_sun4m_iommu_translate_pa(addr, pa, pte);
     return pa;
 }
 
-static void iommu_bad_addr(IOMMUState *s, target_phys_addr_t addr,
+static void iommu_bad_addr(IOMMUState *s, hwaddr addr,
                            int is_write)
 {
     trace_sun4m_iommu_bad_addr(addr);
@@ -286,12 +286,12 @@ static void iommu_bad_addr(IOMMUState *s, target_phys_addr_t addr,
     qemu_irq_raise(s->irq);
 }
 
-void sparc_iommu_memory_rw(void *opaque, target_phys_addr_t addr,
+void sparc_iommu_memory_rw(void *opaque, hwaddr addr,
                            uint8_t *buf, int len, int is_write)
 {
     int l;
     uint32_t flags;
-    target_phys_addr_t page, phys_addr;
+    hwaddr page, phys_addr;
 
     while (len > 0) {
         page = addr & IOMMU_PAGE_MASK;
diff --git a/hw/sun4u.c b/hw/sun4u.c
index 07cd04273..b2b51e30c 100644
--- a/hw/sun4u.c
+++ b/hw/sun4u.c
@@ -25,6 +25,7 @@
 #include "pci.h"
 #include "apb_pci.h"
 #include "pc.h"
+#include "serial.h"
 #include "nvram.h"
 #include "fdc.h"
 #include "net.h"
@@ -39,7 +40,6 @@
 #include "elf.h"
 #include "blockdev.h"
 #include "exec-memory.h"
-#include "vga-pci.h"
 
 //#define DEBUG_IRQ
 //#define DEBUG_EBUS
@@ -311,16 +311,19 @@ void cpu_check_irqs(CPUSPARCState *env)
     }
 }
 
-static void cpu_kick_irq(CPUSPARCState *env)
+static void cpu_kick_irq(SPARCCPU *cpu)
 {
+    CPUSPARCState *env = &cpu->env;
+
     env->halted = 0;
     cpu_check_irqs(env);
-    qemu_cpu_kick(env);
+    qemu_cpu_kick(CPU(cpu));
 }
 
 static void cpu_set_ivec_irq(void *opaque, int irq, int level)
 {
-    CPUSPARCState *env = opaque;
+    SPARCCPU *cpu = opaque;
+    CPUSPARCState *env = &cpu->env;
 
     if (level) {
         if (!(env->ivec_status & 0x20)) {
@@ -367,7 +370,7 @@ void cpu_get_timer(QEMUFile *f, CPUTimer *s)
     qemu_get_timer(f, s->qtimer);
 }
 
-static CPUTimer* cpu_timer_create(const char* name, CPUSPARCState *env,
+static CPUTimer *cpu_timer_create(const char *name, SPARCCPU *cpu,
                                   QEMUBHFunc *cb, uint32_t frequency,
                                   uint64_t disabled_mask)
 {
@@ -380,7 +383,7 @@ static CPUTimer* cpu_timer_create(const char* name, CPUSPARCState *env,
     timer->disabled = 1;
     timer->clock_offset = qemu_get_clock_ns(vm_clock);
 
-    timer->qtimer = qemu_new_timer_ns(vm_clock, cb, env);
+    timer->qtimer = qemu_new_timer_ns(vm_clock, cb, cpu);
 
     return timer;
 }
@@ -419,7 +422,8 @@ static void main_cpu_reset(void *opaque)
 
 static void tick_irq(void *opaque)
 {
-    CPUSPARCState *env = opaque;
+    SPARCCPU *cpu = opaque;
+    CPUSPARCState *env = &cpu->env;
 
     CPUTimer* timer = env->tick;
 
@@ -431,12 +435,13 @@ static void tick_irq(void *opaque)
     }
 
     env->softint |= SOFTINT_TIMER;
-    cpu_kick_irq(env);
+    cpu_kick_irq(cpu);
 }
 
 static void stick_irq(void *opaque)
 {
-    CPUSPARCState *env = opaque;
+    SPARCCPU *cpu = opaque;
+    CPUSPARCState *env = &cpu->env;
 
     CPUTimer* timer = env->stick;
 
@@ -448,12 +453,13 @@ static void stick_irq(void *opaque)
     }
 
     env->softint |= SOFTINT_STIMER;
-    cpu_kick_irq(env);
+    cpu_kick_irq(cpu);
 }
 
 static void hstick_irq(void *opaque)
 {
-    CPUSPARCState *env = opaque;
+    SPARCCPU *cpu = opaque;
+    CPUSPARCState *env = &cpu->env;
 
     CPUTimer* timer = env->hstick;
 
@@ -465,7 +471,7 @@ static void hstick_irq(void *opaque)
     }
 
     env->softint |= SOFTINT_STIMER;
-    cpu_kick_irq(env);
+    cpu_kick_irq(cpu);
 }
 
 static int64_t cpu_to_timer_ticks(int64_t cpu_ticks, uint32_t frequency)
@@ -626,12 +632,12 @@ typedef struct PROMState {
 
 static uint64_t translate_prom_address(void *opaque, uint64_t addr)
 {
-    target_phys_addr_t *base_addr = (target_phys_addr_t *)opaque;
+    hwaddr *base_addr = (hwaddr *)opaque;
     return addr + *base_addr - PROM_VADDR;
 }
 
 /* Boot PROM (OpenBIOS) */
-static void prom_init(target_phys_addr_t addr, const char *bios_name)
+static void prom_init(hwaddr addr, const char *bios_name)
 {
     DeviceState *dev;
     SysBusDevice *s;
@@ -715,7 +721,7 @@ static int ram_init1(SysBusDevice *dev)
     return 0;
 }
 
-static void ram_init(target_phys_addr_t addr, ram_addr_t RAM_size)
+static void ram_init(hwaddr addr, ram_addr_t RAM_size)
 {
     DeviceState *dev;
     SysBusDevice *s;
@@ -773,13 +779,13 @@ static SPARCCPU *cpu_devinit(const char *cpu_model, const struct hwdef *hwdef)
     }
     env = &cpu->env;
 
-    env->tick = cpu_timer_create("tick", env, tick_irq,
+    env->tick = cpu_timer_create("tick", cpu, tick_irq,
                                   tick_frequency, TICK_NPT_MASK);
 
-    env->stick = cpu_timer_create("stick", env, stick_irq,
+    env->stick = cpu_timer_create("stick", cpu, stick_irq,
                                    stick_frequency, TICK_INT_DIS);
 
-    env->hstick = cpu_timer_create("hstick", env, hstick_irq,
+    env->hstick = cpu_timer_create("hstick", cpu, hstick_irq,
                                     hstick_frequency, TICK_INT_DIS);
 
     reset_info = g_malloc0(sizeof(ResetData));
@@ -798,7 +804,6 @@ static void sun4uv_init(MemoryRegion *address_space_mem,
                         const struct hwdef *hwdef)
 {
     SPARCCPU *cpu;
-    CPUSPARCState *env;
     M48t59State *nvram;
     unsigned int i;
     uint64_t initrd_addr, initrd_size, kernel_addr, kernel_size, kernel_entry;
@@ -811,14 +816,13 @@ static void sun4uv_init(MemoryRegion *address_space_mem,
 
     /* init CPUs */
     cpu = cpu_devinit(cpu_model, hwdef);
-    env = &cpu->env;
 
     /* set up devices */
     ram_init(0, RAM_size);
 
     prom_init(hwdef->prom_addr, bios_name);
 
-    ivec_irqs = qemu_allocate_irqs(cpu_set_ivec_irq, env, IVEC_MAX);
+    ivec_irqs = qemu_allocate_irqs(cpu_set_ivec_irq, cpu, IVEC_MAX);
     pci_bus = pci_apb_init(APB_SPECIAL_BASE, APB_MEM_BASE, ivec_irqs, &pci_bus2,
                            &pci_bus3, &pbm_irqs);
     pci_vga_init(pci_bus);
@@ -930,31 +934,40 @@ static const struct hwdef hwdefs[] = {
 };
 
 /* Sun4u hardware initialisation */
-static void sun4u_init(ram_addr_t RAM_size,
-                       const char *boot_devices,
-                       const char *kernel_filename, const char *kernel_cmdline,
-                       const char *initrd_filename, const char *cpu_model)
-{
+static void sun4u_init(QEMUMachineInitArgs *args)
+{
+    ram_addr_t RAM_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
+    const char *boot_devices = args->boot_device;
     sun4uv_init(get_system_memory(), RAM_size, boot_devices, kernel_filename,
                 kernel_cmdline, initrd_filename, cpu_model, &hwdefs[0]);
 }
 
 /* Sun4v hardware initialisation */
-static void sun4v_init(ram_addr_t RAM_size,
-                       const char *boot_devices,
-                       const char *kernel_filename, const char *kernel_cmdline,
-                       const char *initrd_filename, const char *cpu_model)
-{
+static void sun4v_init(QEMUMachineInitArgs *args)
+{
+    ram_addr_t RAM_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
+    const char *boot_devices = args->boot_device;
     sun4uv_init(get_system_memory(), RAM_size, boot_devices, kernel_filename,
                 kernel_cmdline, initrd_filename, cpu_model, &hwdefs[1]);
 }
 
 /* Niagara hardware initialisation */
-static void niagara_init(ram_addr_t RAM_size,
-                         const char *boot_devices,
-                         const char *kernel_filename, const char *kernel_cmdline,
-                         const char *initrd_filename, const char *cpu_model)
-{
+static void niagara_init(QEMUMachineInitArgs *args)
+{
+    ram_addr_t RAM_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
+    const char *boot_devices = args->boot_device;
     sun4uv_init(get_system_memory(), RAM_size, boot_devices, kernel_filename,
                 kernel_cmdline, initrd_filename, cpu_model, &hwdefs[2]);
 }
diff --git a/hw/sysbus.c b/hw/sysbus.c
index 9d8b1eaf7..ef8ffb660 100644
--- a/hw/sysbus.c
+++ b/hw/sysbus.c
@@ -48,7 +48,7 @@ void sysbus_connect_irq(SysBusDevice *dev, int n, qemu_irq irq)
     }
 }
 
-void sysbus_mmio_map(SysBusDevice *dev, int n, target_phys_addr_t addr)
+void sysbus_mmio_map(SysBusDevice *dev, int n, hwaddr addr)
 {
     assert(n >= 0 && n < dev->num_mmio);
 
@@ -56,7 +56,7 @@ void sysbus_mmio_map(SysBusDevice *dev, int n, target_phys_addr_t addr)
         /* ??? region already mapped here.  */
         return;
     }
-    if (dev->mmio[n].addr != (target_phys_addr_t)-1) {
+    if (dev->mmio[n].addr != (hwaddr)-1) {
         /* Unregister previous mapping.  */
         memory_region_del_subregion(get_system_memory(), dev->mmio[n].memory);
     }
@@ -122,7 +122,7 @@ static int sysbus_device_init(DeviceState *dev)
 }
 
 DeviceState *sysbus_create_varargs(const char *name,
-                                   target_phys_addr_t addr, ...)
+                                   hwaddr addr, ...)
 {
     DeviceState *dev;
     SysBusDevice *s;
@@ -133,7 +133,7 @@ DeviceState *sysbus_create_varargs(const char *name,
     dev = qdev_create(NULL, name);
     s = sysbus_from_qdev(dev);
     qdev_init_nofail(dev);
-    if (addr != (target_phys_addr_t)-1) {
+    if (addr != (hwaddr)-1) {
         sysbus_mmio_map(s, 0, addr);
     }
     va_start(va, addr);
@@ -151,7 +151,7 @@ DeviceState *sysbus_create_varargs(const char *name,
 }
 
 DeviceState *sysbus_try_create_varargs(const char *name,
-                                       target_phys_addr_t addr, ...)
+                                       hwaddr addr, ...)
 {
     DeviceState *dev;
     SysBusDevice *s;
@@ -165,7 +165,7 @@ DeviceState *sysbus_try_create_varargs(const char *name,
     }
     s = sysbus_from_qdev(dev);
     qdev_init_nofail(dev);
-    if (addr != (target_phys_addr_t)-1) {
+    if (addr != (hwaddr)-1) {
         sysbus_mmio_map(s, 0, addr);
     }
     va_start(va, addr);
@@ -185,7 +185,7 @@ DeviceState *sysbus_try_create_varargs(const char *name,
 static void sysbus_dev_print(Monitor *mon, DeviceState *dev, int indent)
 {
     SysBusDevice *s = sysbus_from_qdev(dev);
-    target_phys_addr_t size;
+    hwaddr size;
     int i;
 
     monitor_printf(mon, "%*sirq %d\n", indent, "", s->num_irq);
@@ -211,16 +211,16 @@ static char *sysbus_get_fw_dev_path(DeviceState *dev)
         snprintf(path + off, sizeof(path) - off, "@i%04x", s->pio[0]);
     }
 
-    return strdup(path);
+    return g_strdup(path);
 }
 
-void sysbus_add_memory(SysBusDevice *dev, target_phys_addr_t addr,
+void sysbus_add_memory(SysBusDevice *dev, hwaddr addr,
                        MemoryRegion *mem)
 {
     memory_region_add_subregion(get_system_memory(), addr, mem);
 }
 
-void sysbus_add_memory_overlap(SysBusDevice *dev, target_phys_addr_t addr,
+void sysbus_add_memory_overlap(SysBusDevice *dev, hwaddr addr,
                                MemoryRegion *mem, unsigned priority)
 {
     memory_region_add_subregion_overlap(get_system_memory(), addr, mem,
@@ -232,7 +232,7 @@ void sysbus_del_memory(SysBusDevice *dev, MemoryRegion *mem)
     memory_region_del_subregion(get_system_memory(), mem);
 }
 
-void sysbus_add_io(SysBusDevice *dev, target_phys_addr_t addr,
+void sysbus_add_io(SysBusDevice *dev, hwaddr addr,
                        MemoryRegion *mem)
 {
     memory_region_add_subregion(get_system_io(), addr, mem);
@@ -274,7 +274,7 @@ static void main_system_bus_create(void)
     main_system_bus = g_malloc0(system_bus_info.instance_size);
     qbus_create_inplace(main_system_bus, TYPE_SYSTEM_BUS, NULL,
                         "main-system-bus");
-    main_system_bus->glib_allocated = true;
+    OBJECT(main_system_bus)->free = g_free;
     object_property_add_child(container_get(qdev_get_machine(),
                                             "/unattached"),
                               "sysbus", OBJECT(main_system_bus), NULL);
diff --git a/hw/sysbus.h b/hw/sysbus.h
index acfbcfba5..e58baaae3 100644
--- a/hw/sysbus.h
+++ b/hw/sysbus.h
@@ -36,7 +36,7 @@ struct SysBusDevice {
     qemu_irq *irqp[QDEV_MAX_IRQ];
     int num_mmio;
     struct {
-        target_phys_addr_t addr;
+        hwaddr addr;
         MemoryRegion *memory;
     } mmio[QDEV_MAX_MMIO];
     int num_pio;
@@ -56,31 +56,31 @@ void sysbus_init_ioports(SysBusDevice *dev, pio_addr_t ioport, pio_addr_t size);
 
 
 void sysbus_connect_irq(SysBusDevice *dev, int n, qemu_irq irq);
-void sysbus_mmio_map(SysBusDevice *dev, int n, target_phys_addr_t addr);
-void sysbus_add_memory(SysBusDevice *dev, target_phys_addr_t addr,
+void sysbus_mmio_map(SysBusDevice *dev, int n, hwaddr addr);
+void sysbus_add_memory(SysBusDevice *dev, hwaddr addr,
                        MemoryRegion *mem);
-void sysbus_add_memory_overlap(SysBusDevice *dev, target_phys_addr_t addr,
+void sysbus_add_memory_overlap(SysBusDevice *dev, hwaddr addr,
                                MemoryRegion *mem, unsigned priority);
 void sysbus_del_memory(SysBusDevice *dev, MemoryRegion *mem);
-void sysbus_add_io(SysBusDevice *dev, target_phys_addr_t addr,
+void sysbus_add_io(SysBusDevice *dev, hwaddr addr,
                    MemoryRegion *mem);
 void sysbus_del_io(SysBusDevice *dev, MemoryRegion *mem);
 MemoryRegion *sysbus_address_space(SysBusDevice *dev);
 
 /* Legacy helper function for creating devices.  */
 DeviceState *sysbus_create_varargs(const char *name,
-                                 target_phys_addr_t addr, ...);
+                                 hwaddr addr, ...);
 DeviceState *sysbus_try_create_varargs(const char *name,
-                                       target_phys_addr_t addr, ...);
+                                       hwaddr addr, ...);
 static inline DeviceState *sysbus_create_simple(const char *name,
-                                              target_phys_addr_t addr,
+                                              hwaddr addr,
                                               qemu_irq irq)
 {
     return sysbus_create_varargs(name, addr, irq, NULL);
 }
 
 static inline DeviceState *sysbus_try_create_simple(const char *name,
-                                                    target_phys_addr_t addr,
+                                                    hwaddr addr,
                                                     qemu_irq irq)
 {
     return sysbus_try_create_varargs(name, addr, irq, NULL);
diff --git a/hw/tc6393xb.c b/hw/tc6393xb.c
index 420925ccb..f0320271d 100644
--- a/hw/tc6393xb.c
+++ b/hw/tc6393xb.c
@@ -215,7 +215,7 @@ static void tc6393xb_sub_irq(void *opaque, int line, int level) {
     case SCR_ ##N(1): return s->scr.N[1];       \
     case SCR_ ##N(2): return s->scr.N[2]
 
-static uint32_t tc6393xb_scr_readb(TC6393xbState *s, target_phys_addr_t addr)
+static uint32_t tc6393xb_scr_readb(TC6393xbState *s, hwaddr addr)
 {
     switch (addr) {
         case SCR_REVID:
@@ -276,7 +276,7 @@ static uint32_t tc6393xb_scr_readb(TC6393xbState *s, target_phys_addr_t addr)
     case SCR_ ##N(1): s->scr.N[1] = value; return;   \
     case SCR_ ##N(2): s->scr.N[2] = value; return
 
-static void tc6393xb_scr_writeb(TC6393xbState *s, target_phys_addr_t addr, uint32_t value)
+static void tc6393xb_scr_writeb(TC6393xbState *s, hwaddr addr, uint32_t value)
 {
     switch (addr) {
         SCR_REG_B(ISR);
@@ -327,7 +327,7 @@ static void tc6393xb_nand_irq(TC6393xbState *s) {
             (s->nand.imr & 0x80) && (s->nand.imr & s->nand.isr));
 }
 
-static uint32_t tc6393xb_nand_cfg_readb(TC6393xbState *s, target_phys_addr_t addr) {
+static uint32_t tc6393xb_nand_cfg_readb(TC6393xbState *s, hwaddr addr) {
     switch (addr) {
         case NAND_CFG_COMMAND:
             return s->nand_enable ? 2 : 0;
@@ -340,7 +340,7 @@ static uint32_t tc6393xb_nand_cfg_readb(TC6393xbState *s, target_phys_addr_t add
     fprintf(stderr, "tc6393xb_nand_cfg: unhandled read at %08x\n", (uint32_t) addr);
     return 0;
 }
-static void tc6393xb_nand_cfg_writeb(TC6393xbState *s, target_phys_addr_t addr, uint32_t value) {
+static void tc6393xb_nand_cfg_writeb(TC6393xbState *s, hwaddr addr, uint32_t value) {
     switch (addr) {
         case NAND_CFG_COMMAND:
             s->nand_enable = (value & 0x2);
@@ -357,7 +357,7 @@ static void tc6393xb_nand_cfg_writeb(TC6393xbState *s, target_phys_addr_t addr,
 					(uint32_t) addr, value & 0xff);
 }
 
-static uint32_t tc6393xb_nand_readb(TC6393xbState *s, target_phys_addr_t addr) {
+static uint32_t tc6393xb_nand_readb(TC6393xbState *s, hwaddr addr) {
     switch (addr) {
         case NAND_DATA + 0:
         case NAND_DATA + 1:
@@ -376,7 +376,7 @@ static uint32_t tc6393xb_nand_readb(TC6393xbState *s, target_phys_addr_t addr) {
     fprintf(stderr, "tc6393xb_nand: unhandled read at %08x\n", (uint32_t) addr);
     return 0;
 }
-static void tc6393xb_nand_writeb(TC6393xbState *s, target_phys_addr_t addr, uint32_t value) {
+static void tc6393xb_nand_writeb(TC6393xbState *s, hwaddr addr, uint32_t value) {
 //    fprintf(stderr, "tc6393xb_nand: write at %08x: %02x\n",
 //					(uint32_t) addr, value & 0xff);
     switch (addr) {
@@ -454,7 +454,7 @@ static void tc6393xb_draw_graphic(TC6393xbState *s, int full_update)
             return;
     }
 
-    dpy_update(s->ds, 0, 0, s->scr_width, s->scr_height);
+    dpy_gfx_update(s->ds, 0, 0, s->scr_width, s->scr_height);
 }
 
 static void tc6393xb_draw_blank(TC6393xbState *s, int full_update)
@@ -472,7 +472,7 @@ static void tc6393xb_draw_blank(TC6393xbState *s, int full_update)
         d += ds_get_linesize(s->ds);
     }
 
-    dpy_update(s->ds, 0, 0, s->scr_width, s->scr_height);
+    dpy_gfx_update(s->ds, 0, 0, s->scr_width, s->scr_height);
 }
 
 static void tc6393xb_update_display(void *opaque)
@@ -499,7 +499,7 @@ static void tc6393xb_update_display(void *opaque)
 }
 
 
-static uint64_t tc6393xb_readb(void *opaque, target_phys_addr_t addr,
+static uint64_t tc6393xb_readb(void *opaque, hwaddr addr,
                                unsigned size)
 {
     TC6393xbState *s = opaque;
@@ -522,7 +522,7 @@ static uint64_t tc6393xb_readb(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void tc6393xb_writeb(void *opaque, target_phys_addr_t addr,
+static void tc6393xb_writeb(void *opaque, hwaddr addr,
                             uint64_t value, unsigned size) {
     TC6393xbState *s = opaque;
 
diff --git a/hw/tcx.c b/hw/tcx.c
index ac7dcb428..7aee2a9bd 100644
--- a/hw/tcx.c
+++ b/hw/tcx.c
@@ -36,7 +36,7 @@
 
 typedef struct TCXState {
     SysBusDevice busdev;
-    target_phys_addr_t addr;
+    hwaddr addr;
     DisplayState *ds;
     uint8_t *vram;
     uint32_t *vram24, *cplane;
@@ -56,8 +56,10 @@ typedef struct TCXState {
     uint8_t dac_index, dac_state;
 } TCXState;
 
-static void tcx_screen_dump(void *opaque, const char *filename, bool cswitch);
-static void tcx24_screen_dump(void *opaque, const char *filename, bool cswitch);
+static void tcx_screen_dump(void *opaque, const char *filename, bool cswitch,
+                            Error **errp);
+static void tcx24_screen_dump(void *opaque, const char *filename, bool cswitch,
+                            Error **errp);
 
 static void tcx_set_dirty(TCXState *s)
 {
@@ -266,8 +268,8 @@ static void tcx_update_display(void *opaque)
         } else {
             if (y_start >= 0) {
                 /* flush to display */
-                dpy_update(ts->ds, 0, y_start,
-                           ts->width, y - y_start);
+                dpy_gfx_update(ts->ds, 0, y_start,
+                               ts->width, y - y_start);
                 y_start = -1;
             }
             d += dd * 4;
@@ -276,8 +278,8 @@ static void tcx_update_display(void *opaque)
     }
     if (y_start >= 0) {
         /* flush to display */
-        dpy_update(ts->ds, 0, y_start,
-                   ts->width, y - y_start);
+        dpy_gfx_update(ts->ds, 0, y_start,
+                       ts->width, y - y_start);
     }
     /* reset modified pages */
     if (page_max >= page_min) {
@@ -342,8 +344,8 @@ static void tcx24_update_display(void *opaque)
         } else {
             if (y_start >= 0) {
                 /* flush to display */
-                dpy_update(ts->ds, 0, y_start,
-                           ts->width, y - y_start);
+                dpy_gfx_update(ts->ds, 0, y_start,
+                               ts->width, y - y_start);
                 y_start = -1;
             }
             d += dd * 4;
@@ -354,8 +356,8 @@ static void tcx24_update_display(void *opaque)
     }
     if (y_start >= 0) {
         /* flush to display */
-        dpy_update(ts->ds, 0, y_start,
-                   ts->width, y - y_start);
+        dpy_gfx_update(ts->ds, 0, y_start,
+                       ts->width, y - y_start);
     }
     /* reset modified pages */
     if (page_max >= page_min) {
@@ -430,13 +432,13 @@ static void tcx_reset(DeviceState *d)
     s->dac_state = 0;
 }
 
-static uint64_t tcx_dac_readl(void *opaque, target_phys_addr_t addr,
+static uint64_t tcx_dac_readl(void *opaque, hwaddr addr,
                               unsigned size)
 {
     return 0;
 }
 
-static void tcx_dac_writel(void *opaque, target_phys_addr_t addr, uint64_t val,
+static void tcx_dac_writel(void *opaque, hwaddr addr, uint64_t val,
                            unsigned size)
 {
     TCXState *s = opaque;
@@ -470,7 +472,6 @@ static void tcx_dac_writel(void *opaque, target_phys_addr_t addr, uint64_t val,
     default:
         break;
     }
-    return;
 }
 
 static const MemoryRegionOps tcx_dac_ops = {
@@ -483,13 +484,13 @@ static const MemoryRegionOps tcx_dac_ops = {
     },
 };
 
-static uint64_t dummy_readl(void *opaque, target_phys_addr_t addr,
+static uint64_t dummy_readl(void *opaque, hwaddr addr,
                             unsigned size)
 {
     return 0;
 }
 
-static void dummy_writel(void *opaque, target_phys_addr_t addr,
+static void dummy_writel(void *opaque, hwaddr addr,
                          uint64_t val, unsigned size)
 {
 }
@@ -574,45 +575,76 @@ static int tcx_init1(SysBusDevice *dev)
     return 0;
 }
 
-static void tcx_screen_dump(void *opaque, const char *filename, bool cswitch)
+static void tcx_screen_dump(void *opaque, const char *filename, bool cswitch,
+                            Error **errp)
 {
     TCXState *s = opaque;
     FILE *f;
     uint8_t *d, *d1, v;
-    int y, x;
+    int ret, y, x;
 
     f = fopen(filename, "wb");
-    if (!f)
+    if (!f) {
+        error_setg(errp, "failed to open file '%s': %s", filename,
+                   strerror(errno));
         return;
-    fprintf(f, "P6\n%d %d\n%d\n", s->width, s->height, 255);
+    }
+    ret = fprintf(f, "P6\n%d %d\n%d\n", s->width, s->height, 255);
+    if (ret < 0) {
+        goto write_err;
+    }
     d1 = s->vram;
     for(y = 0; y < s->height; y++) {
         d = d1;
         for(x = 0; x < s->width; x++) {
             v = *d;
-            fputc(s->r[v], f);
-            fputc(s->g[v], f);
-            fputc(s->b[v], f);
+            ret = fputc(s->r[v], f);
+            if (ret == EOF) {
+                goto write_err;
+            }
+            ret = fputc(s->g[v], f);
+            if (ret == EOF) {
+                goto write_err;
+            }
+            ret = fputc(s->b[v], f);
+            if (ret == EOF) {
+                goto write_err;
+            }
             d++;
         }
         d1 += MAXX;
     }
+
+out:
     fclose(f);
     return;
+
+write_err:
+    error_setg(errp, "failed to write to file '%s': %s", filename,
+               strerror(errno));
+    unlink(filename);
+    goto out;
 }
 
-static void tcx24_screen_dump(void *opaque, const char *filename, bool cswitch)
+static void tcx24_screen_dump(void *opaque, const char *filename, bool cswitch,
+                              Error **errp)
 {
     TCXState *s = opaque;
     FILE *f;
     uint8_t *d, *d1, v;
     uint32_t *s24, *cptr, dval;
-    int y, x;
+    int ret, y, x;
 
     f = fopen(filename, "wb");
-    if (!f)
+    if (!f) {
+        error_setg(errp, "failed to open file '%s': %s", filename,
+                   strerror(errno));
         return;
-    fprintf(f, "P6\n%d %d\n%d\n", s->width, s->height, 255);
+    }
+    ret = fprintf(f, "P6\n%d %d\n%d\n", s->width, s->height, 255);
+    if (ret < 0) {
+        goto write_err;
+    }
     d1 = s->vram;
     s24 = s->vram24;
     cptr = s->cplane;
@@ -621,20 +653,46 @@ static void tcx24_screen_dump(void *opaque, const char *filename, bool cswitch)
         for(x = 0; x < s->width; x++, d++, s24++) {
             if ((*cptr++ & 0xff000000) == 0x03000000) { // 24-bit direct
                 dval = *s24 & 0x00ffffff;
-                fputc((dval >> 16) & 0xff, f);
-                fputc((dval >> 8) & 0xff, f);
-                fputc(dval & 0xff, f);
+                ret = fputc((dval >> 16) & 0xff, f);
+                if (ret == EOF) {
+                    goto write_err;
+                }
+                ret = fputc((dval >> 8) & 0xff, f);
+                if (ret == EOF) {
+                    goto write_err;
+                }
+                ret = fputc(dval & 0xff, f);
+                if (ret == EOF) {
+                    goto write_err;
+                }
             } else {
                 v = *d;
-                fputc(s->r[v], f);
-                fputc(s->g[v], f);
-                fputc(s->b[v], f);
+                ret = fputc(s->r[v], f);
+                if (ret == EOF) {
+                    goto write_err;
+                }
+                ret = fputc(s->g[v], f);
+                if (ret == EOF) {
+                    goto write_err;
+                }
+                ret = fputc(s->b[v], f);
+                if (ret == EOF) {
+                    goto write_err;
+                }
             }
         }
         d1 += MAXX;
     }
+
+out:
     fclose(f);
     return;
+
+write_err:
+    error_setg(errp, "failed to write to file '%s': %s", filename,
+               strerror(errno));
+    unlink(filename);
+    goto out;
 }
 
 static Property tcx_properties[] = {
diff --git a/hw/tosa.c b/hw/tosa.c
index 297a8c2ed..512278c24 100644
--- a/hw/tosa.c
+++ b/hw/tosa.c
@@ -205,11 +205,12 @@ static struct arm_boot_info tosa_binfo = {
     .ram_size = 0x04000000,
 };
 
-static void tosa_init(ram_addr_t ram_size,
-                const char *boot_device,
-                const char *kernel_filename, const char *kernel_cmdline,
-                const char *initrd_filename, const char *cpu_model)
+static void tosa_init(QEMUMachineInitArgs *args)
 {
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
     MemoryRegion *address_space_mem = get_system_memory();
     MemoryRegion *rom = g_new(MemoryRegion, 1);
     PXA2xxState *mpu;
diff --git a/hw/tusb6010.c b/hw/tusb6010.c
index 5ba8da6d6..325200bd3 100644
--- a/hw/tusb6010.c
+++ b/hw/tusb6010.c
@@ -281,7 +281,7 @@ static void tusb_gpio_intr_update(TUSBState *s)
 extern CPUReadMemoryFunc * const musb_read[];
 extern CPUWriteMemoryFunc * const musb_write[];
 
-static uint32_t tusb_async_readb(void *opaque, target_phys_addr_t addr)
+static uint32_t tusb_async_readb(void *opaque, hwaddr addr)
 {
     TUSBState *s = (TUSBState *) opaque;
 
@@ -298,7 +298,7 @@ static uint32_t tusb_async_readb(void *opaque, target_phys_addr_t addr)
     return 0;
 }
 
-static uint32_t tusb_async_readh(void *opaque, target_phys_addr_t addr)
+static uint32_t tusb_async_readh(void *opaque, hwaddr addr)
 {
     TUSBState *s = (TUSBState *) opaque;
 
@@ -315,7 +315,7 @@ static uint32_t tusb_async_readh(void *opaque, target_phys_addr_t addr)
     return 0;
 }
 
-static uint32_t tusb_async_readw(void *opaque, target_phys_addr_t addr)
+static uint32_t tusb_async_readw(void *opaque, hwaddr addr)
 {
     TUSBState *s = (TUSBState *) opaque;
     int offset = addr & 0xfff;
@@ -438,7 +438,7 @@ static uint32_t tusb_async_readw(void *opaque, target_phys_addr_t addr)
     return 0;
 }
 
-static void tusb_async_writeb(void *opaque, target_phys_addr_t addr,
+static void tusb_async_writeb(void *opaque, hwaddr addr,
                 uint32_t value)
 {
     TUSBState *s = (TUSBState *) opaque;
@@ -459,7 +459,7 @@ static void tusb_async_writeb(void *opaque, target_phys_addr_t addr,
     }
 }
 
-static void tusb_async_writeh(void *opaque, target_phys_addr_t addr,
+static void tusb_async_writeh(void *opaque, hwaddr addr,
                 uint32_t value)
 {
     TUSBState *s = (TUSBState *) opaque;
@@ -480,7 +480,7 @@ static void tusb_async_writeh(void *opaque, target_phys_addr_t addr,
     }
 }
 
-static void tusb_async_writew(void *opaque, target_phys_addr_t addr,
+static void tusb_async_writew(void *opaque, hwaddr addr,
                 uint32_t value)
 {
     TUSBState *s = (TUSBState *) opaque;
diff --git a/hw/unin_pci.c b/hw/unin_pci.c
index d1cc68062..9981d949d 100644
--- a/hw/unin_pci.c
+++ b/hw/unin_pci.c
@@ -111,7 +111,7 @@ static uint32_t unin_get_config_reg(uint32_t reg, uint32_t addr)
     return retval;
 }
 
-static void unin_data_write(void *opaque, target_phys_addr_t addr,
+static void unin_data_write(void *opaque, hwaddr addr,
                             uint64_t val, unsigned len)
 {
     UNINState *s = opaque;
@@ -123,7 +123,7 @@ static void unin_data_write(void *opaque, target_phys_addr_t addr,
                    val, len);
 }
 
-static uint64_t unin_data_read(void *opaque, target_phys_addr_t addr,
+static uint64_t unin_data_read(void *opaque, hwaddr addr,
                                unsigned len)
 {
     UNINState *s = opaque;
diff --git a/hw/usb.h b/hw/usb.h
index b8fceec89..7d6de69ec 100644
--- a/hw/usb.h
+++ b/hw/usb.h
@@ -38,12 +38,15 @@
 #define USB_TOKEN_IN    0x69 /* device -> host */
 #define USB_TOKEN_OUT   0xe1 /* host -> device */
 
-#define USB_RET_NODEV   (-1)
-#define USB_RET_NAK     (-2)
-#define USB_RET_STALL   (-3)
-#define USB_RET_BABBLE  (-4)
-#define USB_RET_IOERROR (-5)
-#define USB_RET_ASYNC   (-6)
+#define USB_RET_SUCCESS           (0)
+#define USB_RET_NODEV             (-1)
+#define USB_RET_NAK               (-2)
+#define USB_RET_STALL             (-3)
+#define USB_RET_BABBLE            (-4)
+#define USB_RET_IOERROR           (-5)
+#define USB_RET_ASYNC             (-6)
+#define USB_RET_ADD_TO_QUEUE      (-7)
+#define USB_RET_REMOVE_FROM_QUEUE (-8)
 
 #define USB_SPEED_LOW   0
 #define USB_SPEED_FULL  1
@@ -135,8 +138,15 @@
 #define USB_DT_OTHER_SPEED_CONFIG       0x07
 #define USB_DT_DEBUG                    0x0A
 #define USB_DT_INTERFACE_ASSOC          0x0B
+#define USB_DT_BOS                      0x0F
+#define USB_DT_DEVICE_CAPABILITY        0x10
 #define USB_DT_CS_INTERFACE             0x24
 #define USB_DT_CS_ENDPOINT              0x25
+#define USB_DT_ENDPOINT_COMPANION       0x30
+
+#define USB_DEV_CAP_WIRELESS            0x01
+#define USB_DEV_CAP_USB2_EXT            0x02
+#define USB_DEV_CAP_SUPERSPEED          0x03
 
 #define USB_ENDPOINT_XFER_CONTROL	0
 #define USB_ENDPOINT_XFER_ISOC		1
@@ -151,6 +161,7 @@ typedef struct USBBusOps USBBusOps;
 typedef struct USBPort USBPort;
 typedef struct USBDevice USBDevice;
 typedef struct USBPacket USBPacket;
+typedef struct USBCombinedPacket USBCombinedPacket;
 typedef struct USBEndpoint USBEndpoint;
 
 typedef struct USBDesc USBDesc;
@@ -270,22 +281,30 @@ typedef struct USBDeviceClass {
      * Process control request.
      * Called from handle_packet().
      *
-     * Returns length or one of the USB_RET_ codes.
+     * Status gets stored in p->status, and if p->status == USB_RET_SUCCESS
+     * then the number of bytes transfered is stored in p->actual_length
      */
-    int (*handle_control)(USBDevice *dev, USBPacket *p, int request, int value,
-                          int index, int length, uint8_t *data);
+    void (*handle_control)(USBDevice *dev, USBPacket *p, int request, int value,
+                           int index, int length, uint8_t *data);
 
     /*
      * Process data transfers (both BULK and ISOC).
      * Called from handle_packet().
      *
-     * Returns length or one of the USB_RET_ codes.
+     * Status gets stored in p->status, and if p->status == USB_RET_SUCCESS
+     * then the number of bytes transfered is stored in p->actual_length
      */
-    int (*handle_data)(USBDevice *dev, USBPacket *p);
+    void (*handle_data)(USBDevice *dev, USBPacket *p);
 
     void (*set_interface)(USBDevice *dev, int interface,
                           int alt_old, int alt_new);
 
+    /*
+     * Called when the hcd is done queuing packets for an endpoint, only
+     * necessary for devices which can return USB_RET_ADD_TO_QUEUE.
+     */
+    void (*flush_ep_queue)(USBDevice *dev, USBEndpoint *ep);
+
     const char *product_desc;
     const USBDesc *usb_desc;
 } USBDeviceClass;
@@ -336,16 +355,28 @@ struct USBPacket {
     USBEndpoint *ep;
     QEMUIOVector iov;
     uint64_t parameter; /* control transfers */
-    int result; /* transfer length or USB_RET_* status code */
+    bool short_not_ok;
+    bool int_req;
+    int status; /* USB_RET_* status code */
+    int actual_length; /* Number of bytes actually transfered */
     /* Internal use by the USB layer.  */
     USBPacketState state;
+    USBCombinedPacket *combined;
     QTAILQ_ENTRY(USBPacket) queue;
+    QTAILQ_ENTRY(USBPacket) combined_entry;
+};
+
+struct USBCombinedPacket {
+    USBPacket *first;
+    QTAILQ_HEAD(packets_head, USBPacket) packets;
+    QEMUIOVector iov;
 };
 
 void usb_packet_init(USBPacket *p);
 void usb_packet_set_state(USBPacket *p, USBPacketState state);
 void usb_packet_check_state(USBPacket *p, USBPacketState expected);
-void usb_packet_setup(USBPacket *p, int pid, USBEndpoint *ep, uint64_t id);
+void usb_packet_setup(USBPacket *p, int pid, USBEndpoint *ep, uint64_t id,
+                      bool short_not_ok, bool int_req);
 void usb_packet_addbuf(USBPacket *p, void *ptr, size_t len);
 int usb_packet_map(USBPacket *p, QEMUSGList *sgl);
 void usb_packet_unmap(USBPacket *p, QEMUSGList *sgl);
@@ -361,8 +392,9 @@ static inline bool usb_packet_is_inflight(USBPacket *p)
 
 USBDevice *usb_find_device(USBPort *port, uint8_t addr);
 
-int usb_handle_packet(USBDevice *dev, USBPacket *p);
+void usb_handle_packet(USBDevice *dev, USBPacket *p);
 void usb_packet_complete(USBDevice *dev, USBPacket *p);
+void usb_packet_complete_one(USBDevice *dev, USBPacket *p);
 void usb_cancel_packet(USBPacket * p);
 
 void usb_ep_init(USBDevice *dev);
@@ -377,6 +409,12 @@ void usb_ep_set_max_packet_size(USBDevice *dev, int pid, int ep,
                                 uint16_t raw);
 int usb_ep_get_max_packet_size(USBDevice *dev, int pid, int ep);
 void usb_ep_set_pipeline(USBDevice *dev, int pid, int ep, bool enabled);
+USBPacket *usb_ep_find_packet_by_id(USBDevice *dev, int pid, int ep,
+                                    uint64_t id);
+
+void usb_ep_combine_input_packets(USBEndpoint *ep);
+void usb_combined_input_packet_complete(USBDevice *dev, USBPacket *p);
+void usb_combined_packet_cancel(USBDevice *dev, USBPacket *p);
 
 void usb_attach(USBPort *port);
 void usb_detach(USBPort *port);
@@ -489,17 +527,21 @@ void usb_device_handle_attach(USBDevice *dev);
 
 void usb_device_handle_reset(USBDevice *dev);
 
-int usb_device_handle_control(USBDevice *dev, USBPacket *p, int request, int value,
-                              int index, int length, uint8_t *data);
+void usb_device_handle_control(USBDevice *dev, USBPacket *p, int request,
+                               int val, int index, int length, uint8_t *data);
 
-int usb_device_handle_data(USBDevice *dev, USBPacket *p);
+void usb_device_handle_data(USBDevice *dev, USBPacket *p);
 
 void usb_device_set_interface(USBDevice *dev, int interface,
                               int alt_old, int alt_new);
 
+void usb_device_flush_ep_queue(USBDevice *dev, USBEndpoint *ep);
+
 const char *usb_device_get_product_desc(USBDevice *dev);
 
 const USBDesc *usb_device_get_usb_desc(USBDevice *dev);
 
+int ehci_create_ich9_with_companions(PCIBus *bus, int slot);
+
 #endif
 
diff --git a/hw/usb/Makefile.objs b/hw/usb/Makefile.objs
index 4225136d0..5a4eeb6d1 100644
--- a/hw/usb/Makefile.objs
+++ b/hw/usb/Makefile.objs
@@ -1,13 +1,13 @@
-hw-obj-$(CONFIG_USB_UHCI) += hcd-uhci.o
-hw-obj-$(CONFIG_USB_OHCI) += hcd-ohci.o
-hw-obj-$(CONFIG_USB_EHCI) += hcd-ehci.o
-hw-obj-$(CONFIG_USB_XHCI) += hcd-xhci.o
-hw-obj-y += libhw.o
+common-obj-$(CONFIG_USB_UHCI) += hcd-uhci.o
+common-obj-$(CONFIG_USB_OHCI) += hcd-ohci.o
+common-obj-$(CONFIG_USB_EHCI) += hcd-ehci.o hcd-ehci-pci.o hcd-ehci-sysbus.o
+common-obj-$(CONFIG_USB_XHCI) += hcd-xhci.o
+common-obj-y += libhw.o
 
-hw-obj-$(CONFIG_SMARTCARD) += dev-smartcard-reader.o
-hw-obj-$(CONFIG_USB_REDIR) += redirect.o
+common-obj-$(CONFIG_SMARTCARD) += dev-smartcard-reader.o
+common-obj-$(CONFIG_USB_REDIR) += redirect.o
 
-common-obj-y += core.o bus.o desc.o dev-hub.o
+common-obj-y += core.o combined-packet.o bus.o desc.o dev-hub.o
 common-obj-y += host-$(HOST_USB).o dev-bluetooth.o
 common-obj-y += dev-hid.o dev-storage.o dev-wacom.o
 common-obj-y += dev-serial.o dev-network.o dev-audio.o
diff --git a/hw/usb/bus.c b/hw/usb/bus.c
index b649360dd..55d0edd5c 100644
--- a/hw/usb/bus.c
+++ b/hw/usb/bus.c
@@ -140,24 +140,21 @@ void usb_device_handle_reset(USBDevice *dev)
     }
 }
 
-int usb_device_handle_control(USBDevice *dev, USBPacket *p, int request,
-                              int value, int index, int length, uint8_t *data)
+void usb_device_handle_control(USBDevice *dev, USBPacket *p, int request,
+                               int value, int index, int length, uint8_t *data)
 {
     USBDeviceClass *klass = USB_DEVICE_GET_CLASS(dev);
     if (klass->handle_control) {
-        return klass->handle_control(dev, p, request, value, index, length,
-                                         data);
+        klass->handle_control(dev, p, request, value, index, length, data);
     }
-    return -ENOSYS;
 }
 
-int usb_device_handle_data(USBDevice *dev, USBPacket *p)
+void usb_device_handle_data(USBDevice *dev, USBPacket *p)
 {
     USBDeviceClass *klass = USB_DEVICE_GET_CLASS(dev);
     if (klass->handle_data) {
-        return klass->handle_data(dev, p);
+        klass->handle_data(dev, p);
     }
-    return -ENOSYS;
 }
 
 const char *usb_device_get_product_desc(USBDevice *dev)
@@ -181,6 +178,14 @@ void usb_device_set_interface(USBDevice *dev, int interface,
     }
 }
 
+void usb_device_flush_ep_queue(USBDevice *dev, USBEndpoint *ep)
+{
+    USBDeviceClass *klass = USB_DEVICE_GET_CLASS(dev);
+    if (klass->flush_ep_queue) {
+        klass->flush_ep_queue(dev, ep);
+    }
+}
+
 static int usb_qdev_init(DeviceState *qdev)
 {
     USBDevice *dev = USB_DEVICE(qdev);
@@ -585,6 +590,13 @@ USBDevice *usbdevice_create(const char *cmdline)
         return NULL;
     }
 
+    if (!bus) {
+        error_report("Error: no usb bus to attach usbdevice %s, "
+                     "please try -machine usb=on and check that "
+                     "the machine model supports USB", driver);
+        return NULL;
+    }
+
     if (!f->usbdevice_init) {
         if (*params) {
             error_report("usbdevice %s accepts no params", driver);
diff --git a/hw/usb/combined-packet.c b/hw/usb/combined-packet.c
new file mode 100644
index 000000000..4a0c29945
--- /dev/null
+++ b/hw/usb/combined-packet.c
@@ -0,0 +1,186 @@
+/*
+ * QEMU USB packet combining code (for input pipelining)
+ *
+ * Copyright(c) 2012 Red Hat, Inc.
+ *
+ * Red Hat Authors:
+ * Hans de Goede <hdegoede@redhat.com>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or(at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+#include "qemu-common.h"
+#include "hw/usb.h"
+#include "iov.h"
+#include "trace.h"
+
+static void usb_combined_packet_add(USBCombinedPacket *combined, USBPacket *p)
+{
+    qemu_iovec_concat(&combined->iov, &p->iov, 0, p->iov.size);
+    QTAILQ_INSERT_TAIL(&combined->packets, p, combined_entry);
+    p->combined = combined;
+}
+
+/* Note will free combined when the last packet gets removed */
+static void usb_combined_packet_remove(USBCombinedPacket *combined,
+                                       USBPacket *p)
+{
+    assert(p->combined == combined);
+    p->combined = NULL;
+    QTAILQ_REMOVE(&combined->packets, p, combined_entry);
+    if (QTAILQ_EMPTY(&combined->packets)) {
+        g_free(combined);
+    }
+}
+
+/* Also handles completion of non combined packets for pipelined input eps */
+void usb_combined_input_packet_complete(USBDevice *dev, USBPacket *p)
+{
+    USBCombinedPacket *combined = p->combined;
+    USBEndpoint *ep = p->ep;
+    USBPacket *next;
+    int status, actual_length;
+    bool short_not_ok, done = false;
+
+    if (combined == NULL) {
+        usb_packet_complete_one(dev, p);
+        goto leave;
+    }
+
+    assert(combined->first == p && p == QTAILQ_FIRST(&combined->packets));
+
+    status = combined->first->status;
+    actual_length = combined->first->actual_length;
+    short_not_ok = QTAILQ_LAST(&combined->packets, packets_head)->short_not_ok;
+
+    QTAILQ_FOREACH_SAFE(p, &combined->packets, combined_entry, next) {
+        if (!done) {
+            /* Distribute data over uncombined packets */
+            if (actual_length >= p->iov.size) {
+                p->actual_length = p->iov.size;
+            } else {
+                /* Send short or error packet to complete the transfer */
+                p->actual_length = actual_length;
+                done = true;
+            }
+            /* Report status on the last packet */
+            if (done || next == NULL) {
+                p->status = status;
+            } else {
+                p->status = USB_RET_SUCCESS;
+            }
+            p->short_not_ok = short_not_ok;
+            /* Note will free combined when the last packet gets removed! */
+            usb_combined_packet_remove(combined, p);
+            usb_packet_complete_one(dev, p);
+            actual_length -= p->actual_length;
+        } else {
+            /* Remove any leftover packets from the queue */
+            p->status = USB_RET_REMOVE_FROM_QUEUE;
+            /* Note will free combined on the last packet! */
+            dev->port->ops->complete(dev->port, p);
+        }
+    }
+    /* Do not use combined here, it has been freed! */
+leave:
+    /* Check if there are packets in the queue waiting for our completion */
+    usb_ep_combine_input_packets(ep);
+}
+
+/* May only be called for combined packets! */
+void usb_combined_packet_cancel(USBDevice *dev, USBPacket *p)
+{
+    USBCombinedPacket *combined = p->combined;
+    assert(combined != NULL);
+    USBPacket *first = p->combined->first;
+
+    /* Note will free combined on the last packet! */
+    usb_combined_packet_remove(combined, p);
+    if (p == first) {
+        usb_device_cancel_packet(dev, p);
+    }
+}
+
+/*
+ * Large input transfers can get split into multiple input packets, this
+ * function recombines them, removing the short_not_ok checks which all but
+ * the last packet of such splits transfers have, thereby allowing input
+ * transfer pipelining (which we cannot do on short_not_ok transfers)
+ */
+void usb_ep_combine_input_packets(USBEndpoint *ep)
+{
+    USBPacket *p, *u, *next, *prev = NULL, *first = NULL;
+    USBPort *port = ep->dev->port;
+    int totalsize;
+
+    assert(ep->pipeline);
+    assert(ep->pid == USB_TOKEN_IN);
+
+    QTAILQ_FOREACH_SAFE(p, &ep->queue, queue, next) {
+        /* Empty the queue on a halt */
+        if (ep->halted) {
+            p->status = USB_RET_REMOVE_FROM_QUEUE;
+            port->ops->complete(port, p);
+            continue;
+        }
+
+        /* Skip packets already submitted to the device */
+        if (p->state == USB_PACKET_ASYNC) {
+            prev = p;
+            continue;
+        }
+        usb_packet_check_state(p, USB_PACKET_QUEUED);
+
+        /*
+         * If the previous (combined) packet has the short_not_ok flag set
+         * stop, as we must not submit packets to the device after a transfer
+         * ending with short_not_ok packet.
+         */
+        if (prev && prev->short_not_ok) {
+            break;
+        }
+
+        if (first) {
+            if (first->combined == NULL) {
+                USBCombinedPacket *combined = g_new0(USBCombinedPacket, 1);
+
+                combined->first = first;
+                QTAILQ_INIT(&combined->packets);
+                qemu_iovec_init(&combined->iov, 2);
+                usb_combined_packet_add(combined, first);
+            }
+            usb_combined_packet_add(first->combined, p);
+        } else {
+            first = p;
+        }
+
+        /* Is this packet the last one of a (combined) transfer? */
+        totalsize = (p->combined) ? p->combined->iov.size : p->iov.size;
+        if ((p->iov.size % ep->max_packet_size) != 0 || !p->short_not_ok ||
+                next == NULL ||
+                /* Work around for Linux usbfs bulk splitting + migration */
+                (totalsize == 16348 && p->int_req)) {
+            usb_device_handle_data(ep->dev, first);
+            assert(first->status == USB_RET_ASYNC);
+            if (first->combined) {
+                QTAILQ_FOREACH(u, &first->combined->packets, combined_entry) {
+                    usb_packet_set_state(u, USB_PACKET_ASYNC);
+                }
+            } else {
+                usb_packet_set_state(first, USB_PACKET_ASYNC);
+            }
+            first = NULL;
+            prev = p;
+        }
+    }
+}
diff --git a/hw/usb/core.c b/hw/usb/core.c
index 2da38e7fd..52b53108c 100644
--- a/hw/usb/core.c
+++ b/hw/usb/core.c
@@ -97,17 +97,17 @@ void usb_wakeup(USBEndpoint *ep)
 #define SETUP_STATE_ACK   3
 #define SETUP_STATE_PARAM 4
 
-static int do_token_setup(USBDevice *s, USBPacket *p)
+static void do_token_setup(USBDevice *s, USBPacket *p)
 {
     int request, value, index;
-    int ret = 0;
 
     if (p->iov.size != 8) {
-        return USB_RET_STALL;
+        p->status = USB_RET_STALL;
+        return;
     }
 
     usb_packet_copy(p, s->setup_buf, p->iov.size);
-    p->result = 0;
+    p->actual_length = 0;
     s->setup_len   = (s->setup_buf[7] << 8) | s->setup_buf[6];
     s->setup_index = 0;
 
@@ -116,24 +116,26 @@ static int do_token_setup(USBDevice *s, USBPacket *p)
     index   = (s->setup_buf[5] << 8) | s->setup_buf[4];
 
     if (s->setup_buf[0] & USB_DIR_IN) {
-        ret = usb_device_handle_control(s, p, request, value, index,
-                                        s->setup_len, s->data_buf);
-        if (ret == USB_RET_ASYNC) {
-             s->setup_state = SETUP_STATE_SETUP;
-             return USB_RET_ASYNC;
+        usb_device_handle_control(s, p, request, value, index,
+                                  s->setup_len, s->data_buf);
+        if (p->status == USB_RET_ASYNC) {
+            s->setup_state = SETUP_STATE_SETUP;
+        }
+        if (p->status != USB_RET_SUCCESS) {
+            return;
         }
-        if (ret < 0)
-            return ret;
 
-        if (ret < s->setup_len)
-            s->setup_len = ret;
+        if (p->actual_length < s->setup_len) {
+            s->setup_len = p->actual_length;
+        }
         s->setup_state = SETUP_STATE_DATA;
     } else {
         if (s->setup_len > sizeof(s->data_buf)) {
             fprintf(stderr,
                 "usb_generic_handle_packet: ctrl buffer too small (%d > %zu)\n",
                 s->setup_len, sizeof(s->data_buf));
-            return USB_RET_STALL;
+            p->status = USB_RET_STALL;
+            return;
         }
         if (s->setup_len == 0)
             s->setup_state = SETUP_STATE_ACK;
@@ -141,13 +143,12 @@ static int do_token_setup(USBDevice *s, USBPacket *p)
             s->setup_state = SETUP_STATE_DATA;
     }
 
-    return ret;
+    p->actual_length = 8;
 }
 
-static int do_token_in(USBDevice *s, USBPacket *p)
+static void do_token_in(USBDevice *s, USBPacket *p)
 {
     int request, value, index;
-    int ret = 0;
 
     assert(p->ep->nr == 0);
 
@@ -158,19 +159,15 @@ static int do_token_in(USBDevice *s, USBPacket *p)
     switch(s->setup_state) {
     case SETUP_STATE_ACK:
         if (!(s->setup_buf[0] & USB_DIR_IN)) {
-            ret = usb_device_handle_control(s, p, request, value, index,
-                                            s->setup_len, s->data_buf);
-            if (ret == USB_RET_ASYNC) {
-                return USB_RET_ASYNC;
+            usb_device_handle_control(s, p, request, value, index,
+                                      s->setup_len, s->data_buf);
+            if (p->status == USB_RET_ASYNC) {
+                return;
             }
             s->setup_state = SETUP_STATE_IDLE;
-            if (ret > 0)
-                return 0;
-            return ret;
+            p->actual_length = 0;
         }
-
-        /* return 0 byte */
-        return 0;
+        break;
 
     case SETUP_STATE_DATA:
         if (s->setup_buf[0] & USB_DIR_IN) {
@@ -180,20 +177,21 @@ static int do_token_in(USBDevice *s, USBPacket *p)
             }
             usb_packet_copy(p, s->data_buf + s->setup_index, len);
             s->setup_index += len;
-            if (s->setup_index >= s->setup_len)
+            if (s->setup_index >= s->setup_len) {
                 s->setup_state = SETUP_STATE_ACK;
-            return len;
+            }
+            return;
         }
-
         s->setup_state = SETUP_STATE_IDLE;
-        return USB_RET_STALL;
+        p->status = USB_RET_STALL;
+        break;
 
     default:
-        return USB_RET_STALL;
+        p->status = USB_RET_STALL;
     }
 }
 
-static int do_token_out(USBDevice *s, USBPacket *p)
+static void do_token_out(USBDevice *s, USBPacket *p)
 {
     assert(p->ep->nr == 0);
 
@@ -205,7 +203,7 @@ static int do_token_out(USBDevice *s, USBPacket *p)
         } else {
             /* ignore additional output */
         }
-        return 0;
+        break;
 
     case SETUP_STATE_DATA:
         if (!(s->setup_buf[0] & USB_DIR_IN)) {
@@ -215,23 +213,23 @@ static int do_token_out(USBDevice *s, USBPacket *p)
             }
             usb_packet_copy(p, s->data_buf + s->setup_index, len);
             s->setup_index += len;
-            if (s->setup_index >= s->setup_len)
+            if (s->setup_index >= s->setup_len) {
                 s->setup_state = SETUP_STATE_ACK;
-            return len;
+            }
+            return;
         }
-
         s->setup_state = SETUP_STATE_IDLE;
-        return USB_RET_STALL;
+        p->status = USB_RET_STALL;
+        break;
 
     default:
-        return USB_RET_STALL;
+        p->status = USB_RET_STALL;
     }
 }
 
-static int do_parameter(USBDevice *s, USBPacket *p)
+static void do_parameter(USBDevice *s, USBPacket *p)
 {
-    int request, value, index;
-    int i, ret = 0;
+    int i, request, value, index;
 
     for (i = 0; i < 8; i++) {
         s->setup_buf[i] = p->parameter >> (i*8);
@@ -249,27 +247,27 @@ static int do_parameter(USBDevice *s, USBPacket *p)
         fprintf(stderr,
                 "usb_generic_handle_packet: ctrl buffer too small (%d > %zu)\n",
                 s->setup_len, sizeof(s->data_buf));
-        return USB_RET_STALL;
+        p->status = USB_RET_STALL;
+        return;
     }
 
     if (p->pid == USB_TOKEN_OUT) {
         usb_packet_copy(p, s->data_buf, s->setup_len);
     }
 
-    ret = usb_device_handle_control(s, p, request, value, index,
-                                    s->setup_len, s->data_buf);
-    if (ret < 0) {
-        return ret;
+    usb_device_handle_control(s, p, request, value, index,
+                              s->setup_len, s->data_buf);
+    if (p->status == USB_RET_ASYNC) {
+        return;
     }
 
-    if (ret < s->setup_len) {
-        s->setup_len = ret;
+    if (p->actual_length < s->setup_len) {
+        s->setup_len = p->actual_length;
     }
     if (p->pid == USB_TOKEN_IN) {
+        p->actual_length = 0;
         usb_packet_copy(p, s->data_buf, s->setup_len);
     }
-
-    return ret;
 }
 
 /* ctrl complete function for devices which use usb_generic_handle_packet and
@@ -278,30 +276,30 @@ static int do_parameter(USBDevice *s, USBPacket *p)
    usb_packet_complete to complete their async control packets. */
 void usb_generic_async_ctrl_complete(USBDevice *s, USBPacket *p)
 {
-    if (p->result < 0) {
+    if (p->status < 0) {
         s->setup_state = SETUP_STATE_IDLE;
     }
 
     switch (s->setup_state) {
     case SETUP_STATE_SETUP:
-        if (p->result < s->setup_len) {
-            s->setup_len = p->result;
+        if (p->actual_length < s->setup_len) {
+            s->setup_len = p->actual_length;
         }
         s->setup_state = SETUP_STATE_DATA;
-        p->result = 8;
+        p->actual_length = 8;
         break;
 
     case SETUP_STATE_ACK:
         s->setup_state = SETUP_STATE_IDLE;
-        p->result = 0;
+        p->actual_length = 0;
         break;
 
     case SETUP_STATE_PARAM:
-        if (p->result < s->setup_len) {
-            s->setup_len = p->result;
+        if (p->actual_length < s->setup_len) {
+            s->setup_len = p->actual_length;
         }
         if (p->pid == USB_TOKEN_IN) {
-            p->result = 0;
+            p->actual_length = 0;
             usb_packet_copy(p, s->data_buf, s->setup_len);
         }
         break;
@@ -342,40 +340,57 @@ USBDevice *usb_find_device(USBPort *port, uint8_t addr)
     return usb_device_find_device(dev, addr);
 }
 
-static int usb_process_one(USBPacket *p)
+static void usb_process_one(USBPacket *p)
 {
     USBDevice *dev = p->ep->dev;
 
+    /*
+     * Handlers expect status to be initialized to USB_RET_SUCCESS, but it
+     * can be USB_RET_NAK here from a previous usb_process_one() call,
+     * or USB_RET_ASYNC from going through usb_queue_one().
+     */
+    p->status = USB_RET_SUCCESS;
+
     if (p->ep->nr == 0) {
         /* control pipe */
         if (p->parameter) {
-            return do_parameter(dev, p);
+            do_parameter(dev, p);
+            return;
         }
         switch (p->pid) {
         case USB_TOKEN_SETUP:
-            return do_token_setup(dev, p);
+            do_token_setup(dev, p);
+            break;
         case USB_TOKEN_IN:
-            return do_token_in(dev, p);
+            do_token_in(dev, p);
+            break;
         case USB_TOKEN_OUT:
-            return do_token_out(dev, p);
+            do_token_out(dev, p);
+            break;
         default:
-            return USB_RET_STALL;
+            p->status = USB_RET_STALL;
         }
     } else {
         /* data pipe */
-        return usb_device_handle_data(dev, p);
+        usb_device_handle_data(dev, p);
     }
 }
 
-/* Hand over a packet to a device for processing.  Return value
+static void usb_queue_one(USBPacket *p)
+{
+    usb_packet_set_state(p, USB_PACKET_QUEUED);
+    QTAILQ_INSERT_TAIL(&p->ep->queue, p, queue);
+    p->status = USB_RET_ASYNC;
+}
+
+/* Hand over a packet to a device for processing.  p->status ==
    USB_RET_ASYNC indicates the processing isn't finished yet, the
    driver will call usb_packet_complete() when done processing it. */
-int usb_handle_packet(USBDevice *dev, USBPacket *p)
+void usb_handle_packet(USBDevice *dev, USBPacket *p)
 {
-    int ret;
-
     if (dev == NULL) {
-        return USB_RET_NODEV;
+        p->status = USB_RET_NODEV;
+        return;
     }
     assert(dev == p->ep->dev);
     assert(dev->state == USB_STATE_DEFAULT);
@@ -389,34 +404,37 @@ int usb_handle_packet(USBDevice *dev, USBPacket *p)
     }
 
     if (QTAILQ_EMPTY(&p->ep->queue) || p->ep->pipeline) {
-        ret = usb_process_one(p);
-        if (ret == USB_RET_ASYNC) {
+        usb_process_one(p);
+        if (p->status == USB_RET_ASYNC) {
+            assert(p->ep->type != USB_ENDPOINT_XFER_ISOC);
             usb_packet_set_state(p, USB_PACKET_ASYNC);
             QTAILQ_INSERT_TAIL(&p->ep->queue, p, queue);
+        } else if (p->status == USB_RET_ADD_TO_QUEUE) {
+            usb_queue_one(p);
         } else {
             /*
              * When pipelining is enabled usb-devices must always return async,
              * otherwise packets can complete out of order!
              */
-            assert(!p->ep->pipeline);
-            p->result = ret;
-            usb_packet_set_state(p, USB_PACKET_COMPLETE);
+            assert(!p->ep->pipeline || QTAILQ_EMPTY(&p->ep->queue));
+            if (p->status != USB_RET_NAK) {
+                usb_packet_set_state(p, USB_PACKET_COMPLETE);
+            }
         }
     } else {
-        ret = USB_RET_ASYNC;
-        usb_packet_set_state(p, USB_PACKET_QUEUED);
-        QTAILQ_INSERT_TAIL(&p->ep->queue, p, queue);
+        usb_queue_one(p);
     }
-    return ret;
 }
 
-static void __usb_packet_complete(USBDevice *dev, USBPacket *p)
+void usb_packet_complete_one(USBDevice *dev, USBPacket *p)
 {
     USBEndpoint *ep = p->ep;
 
-    assert(p->result != USB_RET_ASYNC && p->result != USB_RET_NAK);
+    assert(QTAILQ_FIRST(&ep->queue) == p);
+    assert(p->status != USB_RET_ASYNC && p->status != USB_RET_NAK);
 
-    if (p->result < 0) {
+    if (p->status != USB_RET_SUCCESS ||
+            (p->short_not_ok && (p->actual_length < p->iov.size))) {
         ep->halted = true;
     }
     usb_packet_set_state(p, USB_PACKET_COMPLETE);
@@ -430,25 +448,28 @@ static void __usb_packet_complete(USBDevice *dev, USBPacket *p)
 void usb_packet_complete(USBDevice *dev, USBPacket *p)
 {
     USBEndpoint *ep = p->ep;
-    int ret;
 
     usb_packet_check_state(p, USB_PACKET_ASYNC);
-    assert(QTAILQ_FIRST(&ep->queue) == p);
-    __usb_packet_complete(dev, p);
+    usb_packet_complete_one(dev, p);
 
-    while (!ep->halted && !QTAILQ_EMPTY(&ep->queue)) {
+    while (!QTAILQ_EMPTY(&ep->queue)) {
         p = QTAILQ_FIRST(&ep->queue);
+        if (ep->halted) {
+            /* Empty the queue on a halt */
+            p->status = USB_RET_REMOVE_FROM_QUEUE;
+            dev->port->ops->complete(dev->port, p);
+            continue;
+        }
         if (p->state == USB_PACKET_ASYNC) {
             break;
         }
         usb_packet_check_state(p, USB_PACKET_QUEUED);
-        ret = usb_process_one(p);
-        if (ret == USB_RET_ASYNC) {
+        usb_process_one(p);
+        if (p->status == USB_RET_ASYNC) {
             usb_packet_set_state(p, USB_PACKET_ASYNC);
             break;
         }
-        p->result = ret;
-        __usb_packet_complete(ep->dev, p);
+        usb_packet_complete_one(ep->dev, p);
     }
 }
 
@@ -520,15 +541,20 @@ void usb_packet_set_state(USBPacket *p, USBPacketState state)
     p->state = state;
 }
 
-void usb_packet_setup(USBPacket *p, int pid, USBEndpoint *ep, uint64_t id)
+void usb_packet_setup(USBPacket *p, int pid, USBEndpoint *ep, uint64_t id,
+                      bool short_not_ok, bool int_req)
 {
     assert(!usb_packet_is_inflight(p));
     assert(p->iov.iov != NULL);
     p->id = id;
     p->pid = pid;
     p->ep = ep;
-    p->result = 0;
+    p->status = USB_RET_SUCCESS;
+    p->actual_length = 0;
     p->parameter = 0;
+    p->short_not_ok = short_not_ok;
+    p->int_req = int_req;
+    p->combined = NULL;
     qemu_iovec_reset(&p->iov);
     usb_packet_set_state(p, USB_PACKET_SETUP);
 }
@@ -540,31 +566,31 @@ void usb_packet_addbuf(USBPacket *p, void *ptr, size_t len)
 
 void usb_packet_copy(USBPacket *p, void *ptr, size_t bytes)
 {
-    assert(p->result >= 0);
-    assert(p->result + bytes <= p->iov.size);
+    assert(p->actual_length >= 0);
+    assert(p->actual_length + bytes <= p->iov.size);
     switch (p->pid) {
     case USB_TOKEN_SETUP:
     case USB_TOKEN_OUT:
-        iov_to_buf(p->iov.iov, p->iov.niov, p->result, ptr, bytes);
+        iov_to_buf(p->iov.iov, p->iov.niov, p->actual_length, ptr, bytes);
         break;
     case USB_TOKEN_IN:
-        iov_from_buf(p->iov.iov, p->iov.niov, p->result, ptr, bytes);
+        iov_from_buf(p->iov.iov, p->iov.niov, p->actual_length, ptr, bytes);
         break;
     default:
         fprintf(stderr, "%s: invalid pid: %x\n", __func__, p->pid);
         abort();
     }
-    p->result += bytes;
+    p->actual_length += bytes;
 }
 
 void usb_packet_skip(USBPacket *p, size_t bytes)
 {
-    assert(p->result >= 0);
-    assert(p->result + bytes <= p->iov.size);
+    assert(p->actual_length >= 0);
+    assert(p->actual_length + bytes <= p->iov.size);
     if (p->pid == USB_TOKEN_IN) {
-        iov_memset(p->iov.iov, p->iov.niov, p->result, 0, bytes);
+        iov_memset(p->iov.iov, p->iov.niov, p->actual_length, 0, bytes);
     }
-    p->result += bytes;
+    p->actual_length += bytes;
 }
 
 void usb_packet_cleanup(USBPacket *p)
@@ -724,3 +750,18 @@ void usb_ep_set_pipeline(USBDevice *dev, int pid, int ep, bool enabled)
     struct USBEndpoint *uep = usb_ep_get(dev, pid, ep);
     uep->pipeline = enabled;
 }
+
+USBPacket *usb_ep_find_packet_by_id(USBDevice *dev, int pid, int ep,
+                                    uint64_t id)
+{
+    struct USBEndpoint *uep = usb_ep_get(dev, pid, ep);
+    USBPacket *p;
+
+    while ((p = QTAILQ_FIRST(&uep->queue)) != NULL) {
+        if (p->id == id) {
+            return p;
+        }
+    }
+
+    return NULL;
+}
diff --git a/hw/usb/desc.c b/hw/usb/desc.c
index 0a9d3c9f6..b7c32333d 100644
--- a/hw/usb/desc.c
+++ b/hw/usb/desc.c
@@ -76,7 +76,8 @@ int usb_desc_device_qualifier(const USBDescDevice *dev,
     return bLength;
 }
 
-int usb_desc_config(const USBDescConfig *conf, uint8_t *dest, size_t len)
+int usb_desc_config(const USBDescConfig *conf, int flags,
+                    uint8_t *dest, size_t len)
 {
     uint8_t  bLength = 0x09;
     uint16_t wTotalLength = 0;
@@ -99,7 +100,7 @@ int usb_desc_config(const USBDescConfig *conf, uint8_t *dest, size_t len)
 
     /* handle grouped interfaces if any */
     for (i = 0; i < conf->nif_groups; i++) {
-        rc = usb_desc_iface_group(&(conf->if_groups[i]),
+        rc = usb_desc_iface_group(&(conf->if_groups[i]), flags,
                                   dest + wTotalLength,
                                   len - wTotalLength);
         if (rc < 0) {
@@ -110,7 +111,8 @@ int usb_desc_config(const USBDescConfig *conf, uint8_t *dest, size_t len)
 
     /* handle normal (ungrouped / no IAD) interfaces if any */
     for (i = 0; i < conf->nif; i++) {
-        rc = usb_desc_iface(conf->ifs + i, dest + wTotalLength, len - wTotalLength);
+        rc = usb_desc_iface(conf->ifs + i, flags,
+                            dest + wTotalLength, len - wTotalLength);
         if (rc < 0) {
             return rc;
         }
@@ -122,8 +124,8 @@ int usb_desc_config(const USBDescConfig *conf, uint8_t *dest, size_t len)
     return wTotalLength;
 }
 
-int usb_desc_iface_group(const USBDescIfaceAssoc *iad, uint8_t *dest,
-                         size_t len)
+int usb_desc_iface_group(const USBDescIfaceAssoc *iad, int flags,
+                         uint8_t *dest, size_t len)
 {
     int pos = 0;
     int i = 0;
@@ -147,7 +149,7 @@ int usb_desc_iface_group(const USBDescIfaceAssoc *iad, uint8_t *dest,
 
     /* handle associated interfaces in this group */
     for (i = 0; i < iad->nif; i++) {
-        int rc = usb_desc_iface(&(iad->ifs[i]), dest + pos, len - pos);
+        int rc = usb_desc_iface(&(iad->ifs[i]), flags, dest + pos, len - pos);
         if (rc < 0) {
             return rc;
         }
@@ -157,7 +159,8 @@ int usb_desc_iface_group(const USBDescIfaceAssoc *iad, uint8_t *dest,
     return pos;
 }
 
-int usb_desc_iface(const USBDescIface *iface, uint8_t *dest, size_t len)
+int usb_desc_iface(const USBDescIface *iface, int flags,
+                   uint8_t *dest, size_t len)
 {
     uint8_t bLength = 0x09;
     int i, rc, pos = 0;
@@ -188,7 +191,7 @@ int usb_desc_iface(const USBDescIface *iface, uint8_t *dest, size_t len)
     }
 
     for (i = 0; i < iface->bNumEndpoints; i++) {
-        rc = usb_desc_endpoint(iface->eps + i, dest + pos, len - pos);
+        rc = usb_desc_endpoint(iface->eps + i, flags, dest + pos, len - pos);
         if (rc < 0) {
             return rc;
         }
@@ -198,13 +201,15 @@ int usb_desc_iface(const USBDescIface *iface, uint8_t *dest, size_t len)
     return pos;
 }
 
-int usb_desc_endpoint(const USBDescEndpoint *ep, uint8_t *dest, size_t len)
+int usb_desc_endpoint(const USBDescEndpoint *ep, int flags,
+                      uint8_t *dest, size_t len)
 {
     uint8_t bLength = ep->is_audio ? 0x09 : 0x07;
     uint8_t extralen = ep->extra ? ep->extra[0] : 0;
+    uint8_t superlen = (flags & USB_DESC_FLAG_SUPER) ? 0x06 : 0;
     USBDescriptor *d = (void *)dest;
 
-    if (len < bLength + extralen) {
+    if (len < bLength + extralen + superlen) {
         return -1;
     }
 
@@ -224,7 +229,21 @@ int usb_desc_endpoint(const USBDescEndpoint *ep, uint8_t *dest, size_t len)
         memcpy(dest + bLength, ep->extra, extralen);
     }
 
-    return bLength + extralen;
+    if (superlen) {
+        USBDescriptor *d = (void *)(dest + bLength + extralen);
+
+        d->bLength                       = 0x06;
+        d->bDescriptorType               = USB_DT_ENDPOINT_COMPANION;
+
+        d->u.super_endpoint.bMaxBurst    = ep->bMaxBurst;
+        d->u.super_endpoint.bmAttributes = ep->bmAttributes_super;
+        d->u.super_endpoint.wBytesPerInterval_lo =
+            usb_lo(ep->wBytesPerInterval);
+        d->u.super_endpoint.wBytesPerInterval_hi =
+            usb_hi(ep->wBytesPerInterval);
+    }
+
+    return bLength + extralen + superlen;
 }
 
 int usb_desc_other(const USBDescOther *desc, uint8_t *dest, size_t len)
@@ -239,6 +258,111 @@ int usb_desc_other(const USBDescOther *desc, uint8_t *dest, size_t len)
     return bLength;
 }
 
+static int usb_desc_cap_usb2_ext(const USBDesc *desc, uint8_t *dest, size_t len)
+{
+    uint8_t  bLength = 0x07;
+    USBDescriptor *d = (void *)dest;
+
+    if (len < bLength) {
+        return -1;
+    }
+
+    d->bLength                          = bLength;
+    d->bDescriptorType                  = USB_DT_DEVICE_CAPABILITY;
+    d->u.cap.bDevCapabilityType         = USB_DEV_CAP_USB2_EXT;
+
+    d->u.cap.u.usb2_ext.bmAttributes_1  = (1 << 1);  /* LPM */
+    d->u.cap.u.usb2_ext.bmAttributes_2  = 0;
+    d->u.cap.u.usb2_ext.bmAttributes_3  = 0;
+    d->u.cap.u.usb2_ext.bmAttributes_4  = 0;
+
+    return bLength;
+}
+
+static int usb_desc_cap_super(const USBDesc *desc, uint8_t *dest, size_t len)
+{
+    uint8_t  bLength = 0x0a;
+    USBDescriptor *d = (void *)dest;
+
+    if (len < bLength) {
+        return -1;
+    }
+
+    d->bLength                           = bLength;
+    d->bDescriptorType                   = USB_DT_DEVICE_CAPABILITY;
+    d->u.cap.bDevCapabilityType          = USB_DEV_CAP_SUPERSPEED;
+
+    d->u.cap.u.super.bmAttributes        = 0;
+    d->u.cap.u.super.wSpeedsSupported_lo = 0;
+    d->u.cap.u.super.wSpeedsSupported_hi = 0;
+    d->u.cap.u.super.bFunctionalitySupport = 0;
+    d->u.cap.u.super.bU1DevExitLat       = 0x0a;
+    d->u.cap.u.super.wU2DevExitLat_lo    = 0x20;
+    d->u.cap.u.super.wU2DevExitLat_hi    = 0;
+
+    if (desc->full) {
+        d->u.cap.u.super.wSpeedsSupported_lo |= (1 << 1);
+        d->u.cap.u.super.bFunctionalitySupport = 1;
+    }
+    if (desc->high) {
+        d->u.cap.u.super.wSpeedsSupported_lo |= (1 << 2);
+        if (!d->u.cap.u.super.bFunctionalitySupport) {
+            d->u.cap.u.super.bFunctionalitySupport = 2;
+        }
+    }
+    if (desc->super) {
+        d->u.cap.u.super.wSpeedsSupported_lo |= (1 << 3);
+        if (!d->u.cap.u.super.bFunctionalitySupport) {
+            d->u.cap.u.super.bFunctionalitySupport = 3;
+        }
+    }
+
+    return bLength;
+}
+
+static int usb_desc_bos(const USBDesc *desc, uint8_t *dest, size_t len)
+{
+    uint8_t  bLength = 0x05;
+    uint16_t wTotalLength = 0;
+    uint8_t  bNumDeviceCaps = 0;
+    USBDescriptor *d = (void *)dest;
+    int rc;
+
+    if (len < bLength) {
+        return -1;
+    }
+
+    d->bLength                      = bLength;
+    d->bDescriptorType              = USB_DT_BOS;
+
+    wTotalLength += bLength;
+
+    if (desc->high != NULL) {
+        rc = usb_desc_cap_usb2_ext(desc, dest + wTotalLength,
+                                   len - wTotalLength);
+        if (rc < 0) {
+            return rc;
+        }
+        wTotalLength += rc;
+        bNumDeviceCaps++;
+    }
+
+    if (desc->super != NULL) {
+        rc = usb_desc_cap_super(desc, dest + wTotalLength,
+                                len - wTotalLength);
+        if (rc < 0) {
+            return rc;
+        }
+        wTotalLength += rc;
+        bNumDeviceCaps++;
+    }
+
+    d->u.bos.wTotalLength_lo = usb_lo(wTotalLength);
+    d->u.bos.wTotalLength_hi = usb_hi(wTotalLength);
+    d->u.bos.bNumDeviceCaps  = bNumDeviceCaps;
+    return wTotalLength;
+}
+
 /* ------------------------------------------------------------------ */
 
 static void usb_desc_ep_init(USBDevice *dev)
@@ -359,6 +483,9 @@ static void usb_desc_setdefaults(USBDevice *dev)
     case USB_SPEED_HIGH:
         dev->device = desc->high;
         break;
+    case USB_SPEED_SUPER:
+        dev->device = desc->super;
+        break;
     }
     usb_desc_set_config(dev, 0);
 }
@@ -376,6 +503,9 @@ void usb_desc_init(USBDevice *dev)
     if (desc->high) {
         dev->speedmask |= USB_SPEED_MASK_HIGH;
     }
+    if (desc->super) {
+        dev->speedmask |= USB_SPEED_MASK_SUPER;
+    }
     usb_desc_setdefaults(dev);
 }
 
@@ -384,7 +514,9 @@ void usb_desc_attach(USBDevice *dev)
     const USBDesc *desc = usb_device_get_usb_desc(dev);
 
     assert(desc != NULL);
-    if (desc->high && (dev->port->speedmask & USB_SPEED_MASK_HIGH)) {
+    if (desc->super && (dev->port->speedmask & USB_SPEED_MASK_SUPER)) {
+        dev->speed = USB_SPEED_SUPER;
+    } else if (desc->high && (dev->port->speedmask & USB_SPEED_MASK_HIGH)) {
         dev->speed = USB_SPEED_HIGH;
     } else if (desc->full && (dev->port->speedmask & USB_SPEED_MASK_FULL)) {
         dev->speed = USB_SPEED_FULL;
@@ -494,14 +626,15 @@ int usb_desc_string(USBDevice *dev, int index, uint8_t *dest, size_t len)
     return pos;
 }
 
-int usb_desc_get_descriptor(USBDevice *dev, int value, uint8_t *dest, size_t len)
+int usb_desc_get_descriptor(USBDevice *dev, USBPacket *p,
+                            int value, uint8_t *dest, size_t len)
 {
     const USBDesc *desc = usb_device_get_usb_desc(dev);
     const USBDescDevice *other_dev;
     uint8_t buf[256];
     uint8_t type = value >> 8;
     uint8_t index = value & 0xff;
-    int ret = -1;
+    int flags, ret = -1;
 
     if (dev->speed == USB_SPEED_HIGH) {
         other_dev = usb_device_get_usb_desc(dev)->full;
@@ -509,6 +642,11 @@ int usb_desc_get_descriptor(USBDevice *dev, int value, uint8_t *dest, size_t len
         other_dev = usb_device_get_usb_desc(dev)->high;
     }
 
+    flags = 0;
+    if (dev->device->bcdUSB >= 0x0300) {
+        flags |= USB_DESC_FLAG_SUPER;
+    }
+
     switch(type) {
     case USB_DT_DEVICE:
         ret = usb_desc_device(&desc->id, dev->device, buf, sizeof(buf));
@@ -516,7 +654,8 @@ int usb_desc_get_descriptor(USBDevice *dev, int value, uint8_t *dest, size_t len
         break;
     case USB_DT_CONFIG:
         if (index < dev->device->bNumConfigurations) {
-            ret = usb_desc_config(dev->device->confs + index, buf, sizeof(buf));
+            ret = usb_desc_config(dev->device->confs + index, flags,
+                                  buf, sizeof(buf));
         }
         trace_usb_desc_config(dev->addr, index, len, ret);
         break;
@@ -524,7 +663,6 @@ int usb_desc_get_descriptor(USBDevice *dev, int value, uint8_t *dest, size_t len
         ret = usb_desc_string(dev, index, buf, sizeof(buf));
         trace_usb_desc_string(dev->addr, index, len, ret);
         break;
-
     case USB_DT_DEVICE_QUALIFIER:
         if (other_dev != NULL) {
             ret = usb_desc_device_qualifier(other_dev, buf, sizeof(buf));
@@ -533,11 +671,16 @@ int usb_desc_get_descriptor(USBDevice *dev, int value, uint8_t *dest, size_t len
         break;
     case USB_DT_OTHER_SPEED_CONFIG:
         if (other_dev != NULL && index < other_dev->bNumConfigurations) {
-            ret = usb_desc_config(other_dev->confs + index, buf, sizeof(buf));
+            ret = usb_desc_config(other_dev->confs + index, flags,
+                                  buf, sizeof(buf));
             buf[0x01] = USB_DT_OTHER_SPEED_CONFIG;
         }
         trace_usb_desc_other_speed_config(dev->addr, index, len, ret);
         break;
+    case USB_DT_BOS:
+        ret = usb_desc_bos(desc, buf, sizeof(buf));
+        trace_usb_desc_bos(dev->addr, len, ret);
+        break;
 
     case USB_DT_DEBUG:
         /* ignore silently */
@@ -554,6 +697,8 @@ int usb_desc_get_descriptor(USBDevice *dev, int value, uint8_t *dest, size_t len
             ret = len;
         }
         memcpy(dest, buf, ret);
+        p->actual_length = ret;
+        ret = 0;
     }
     return ret;
 }
@@ -573,7 +718,7 @@ int usb_desc_handle_control(USBDevice *dev, USBPacket *p,
         break;
 
     case DeviceRequest | USB_REQ_GET_DESCRIPTOR:
-        ret = usb_desc_get_descriptor(dev, value, data, length);
+        ret = usb_desc_get_descriptor(dev, p, value, data, length);
         break;
 
     case DeviceRequest | USB_REQ_GET_CONFIGURATION:
@@ -582,7 +727,8 @@ int usb_desc_handle_control(USBDevice *dev, USBPacket *p,
          * the non zero value of bConfigurationValue.
          */
         data[0] = dev->config ? dev->config->bConfigurationValue : 0;
-        ret = 1;
+        p->actual_length = 1;
+        ret = 0;
         break;
     case DeviceOutRequest | USB_REQ_SET_CONFIGURATION:
         ret = usb_desc_set_config(dev, value);
@@ -607,7 +753,8 @@ int usb_desc_handle_control(USBDevice *dev, USBPacket *p,
             data[0] |= 1 << USB_DEVICE_REMOTE_WAKEUP;
         }
         data[1] = 0x00;
-        ret = 2;
+        p->actual_length = 2;
+        ret = 0;
         break;
     }
     case DeviceOutRequest | USB_REQ_CLEAR_FEATURE:
@@ -630,7 +777,8 @@ int usb_desc_handle_control(USBDevice *dev, USBPacket *p,
             break;
         }
         data[0] = dev->altsetting[index];
-        ret = 1;
+        p->actual_length = 1;
+        ret = 0;
         break;
     case InterfaceOutRequest | USB_REQ_SET_INTERFACE:
         ret = usb_desc_set_interface(dev, index, value);
diff --git a/hw/usb/desc.h b/hw/usb/desc.h
index 7cf544294..ddd3e7485 100644
--- a/hw/usb/desc.h
+++ b/hw/usb/desc.h
@@ -63,6 +63,37 @@ typedef struct USBDescriptor {
             uint8_t           bRefresh;        /* only audio ep */
             uint8_t           bSynchAddress;   /* only audio ep */
         } endpoint;
+        struct {
+            uint8_t           bMaxBurst;
+            uint8_t           bmAttributes;
+            uint8_t           wBytesPerInterval_lo;
+            uint8_t           wBytesPerInterval_hi;
+        } super_endpoint;
+        struct {
+            uint8_t           wTotalLength_lo;
+            uint8_t           wTotalLength_hi;
+            uint8_t           bNumDeviceCaps;
+        } bos;
+        struct {
+            uint8_t           bDevCapabilityType;
+            union {
+                struct {
+                    uint8_t   bmAttributes_1;
+                    uint8_t   bmAttributes_2;
+                    uint8_t   bmAttributes_3;
+                    uint8_t   bmAttributes_4;
+                } usb2_ext;
+                struct {
+                    uint8_t   bmAttributes;
+                    uint8_t   wSpeedsSupported_lo;
+                    uint8_t   wSpeedsSupported_hi;
+                    uint8_t   bFunctionalitySupport;
+                    uint8_t   bU1DevExitLat;
+                    uint8_t   wU2DevExitLat_lo;
+                    uint8_t   wU2DevExitLat_hi;
+                } super;
+            } u;
+        } cap;
     } u;
 } QEMU_PACKED USBDescriptor;
 
@@ -139,6 +170,11 @@ struct USBDescEndpoint {
 
     uint8_t                   is_audio; /* has bRefresh + bSynchAddress */
     uint8_t                   *extra;
+
+    /* superspeed endpoint companion */
+    uint8_t                   bMaxBurst;
+    uint8_t                   bmAttributes_super;
+    uint16_t                  wBytesPerInterval;
 };
 
 struct USBDescOther {
@@ -152,19 +188,25 @@ struct USBDesc {
     USBDescID                 id;
     const USBDescDevice       *full;
     const USBDescDevice       *high;
+    const USBDescDevice       *super;
     const char* const         *str;
 };
 
+#define USB_DESC_FLAG_SUPER (1 << 1)
+
 /* generate usb packages from structs */
 int usb_desc_device(const USBDescID *id, const USBDescDevice *dev,
                     uint8_t *dest, size_t len);
 int usb_desc_device_qualifier(const USBDescDevice *dev,
                               uint8_t *dest, size_t len);
-int usb_desc_config(const USBDescConfig *conf, uint8_t *dest, size_t len);
-int usb_desc_iface_group(const USBDescIfaceAssoc *iad, uint8_t *dest,
-                         size_t len);
-int usb_desc_iface(const USBDescIface *iface, uint8_t *dest, size_t len);
-int usb_desc_endpoint(const USBDescEndpoint *ep, uint8_t *dest, size_t len);
+int usb_desc_config(const USBDescConfig *conf, int flags,
+                    uint8_t *dest, size_t len);
+int usb_desc_iface_group(const USBDescIfaceAssoc *iad, int flags,
+                         uint8_t *dest, size_t len);
+int usb_desc_iface(const USBDescIface *iface, int flags,
+                   uint8_t *dest, size_t len);
+int usb_desc_endpoint(const USBDescEndpoint *ep, int flags,
+                      uint8_t *dest, size_t len);
 int usb_desc_other(const USBDescOther *desc, uint8_t *dest, size_t len);
 
 /* control message emulation helpers */
@@ -174,7 +216,8 @@ void usb_desc_set_string(USBDevice *dev, uint8_t index, const char *str);
 void usb_desc_create_serial(USBDevice *dev);
 const char *usb_desc_get_string(USBDevice *dev, uint8_t index);
 int usb_desc_string(USBDevice *dev, int index, uint8_t *dest, size_t len);
-int usb_desc_get_descriptor(USBDevice *dev, int value, uint8_t *dest, size_t len);
+int usb_desc_get_descriptor(USBDevice *dev, USBPacket *p,
+        int value, uint8_t *dest, size_t len);
 int usb_desc_handle_control(USBDevice *dev, USBPacket *p,
         int request, int value, int index, int length, uint8_t *data);
 
diff --git a/hw/usb/dev-audio.c b/hw/usb/dev-audio.c
index 79b75fb62..b669601c9 100644
--- a/hw/usb/dev-audio.c
+++ b/hw/usb/dev-audio.c
@@ -217,7 +217,7 @@ static const USBDescIface desc_iface[] = {
 };
 
 static const USBDescDevice desc_device = {
-    .bcdUSB                        = 0x0200,
+    .bcdUSB                        = 0x0100,
     .bMaxPacketSize0               = 64,
     .bNumConfigurations            = 1,
     .confs = (USBDescConfig[]) {
@@ -503,7 +503,7 @@ static int usb_audio_set_control(USBAudioState *s, uint8_t attrib,
     return ret;
 }
 
-static int usb_audio_handle_control(USBDevice *dev, USBPacket *p,
+static void usb_audio_handle_control(USBDevice *dev, USBPacket *p,
                                     int request, int value, int index,
                                     int length, uint8_t *data)
 {
@@ -518,7 +518,7 @@ static int usb_audio_handle_control(USBDevice *dev, USBPacket *p,
 
     ret = usb_desc_handle_control(dev, p, request, value, index, length, data);
     if (ret >= 0) {
-        return ret;
+        return;
     }
 
     switch (request) {
@@ -534,6 +534,7 @@ static int usb_audio_handle_control(USBDevice *dev, USBPacket *p,
             }
             goto fail;
         }
+        p->actual_length = ret;
         break;
 
     case ClassInterfaceOutRequest | CR_SET_CUR:
@@ -557,10 +558,9 @@ fail:
                     "request 0x%04x value 0x%04x index 0x%04x length 0x%04x\n",
                     request, value, index, length);
         }
-        ret = USB_RET_STALL;
+        p->status = USB_RET_STALL;
         break;
     }
-    return ret;
 }
 
 static void usb_audio_set_interface(USBDevice *dev, int iface,
@@ -583,50 +583,35 @@ static void usb_audio_handle_reset(USBDevice *dev)
     usb_audio_set_output_altset(s, ALTSET_OFF);
 }
 
-static int usb_audio_handle_dataout(USBAudioState *s, USBPacket *p)
+static void usb_audio_handle_dataout(USBAudioState *s, USBPacket *p)
 {
-    int rc;
-
     if (s->out.altset == ALTSET_OFF) {
-        return USB_RET_STALL;
+        p->status = USB_RET_STALL;
+        return;
     }
 
-    rc = streambuf_put(&s->out.buf, p);
-    if (rc < p->iov.size && s->debug > 1) {
+    streambuf_put(&s->out.buf, p);
+    if (p->actual_length < p->iov.size && s->debug > 1) {
         fprintf(stderr, "usb-audio: output overrun (%zd bytes)\n",
-                p->iov.size - rc);
+                p->iov.size - p->actual_length);
     }
-
-    return 0;
 }
 
-static int usb_audio_handle_data(USBDevice *dev, USBPacket *p)
+static void usb_audio_handle_data(USBDevice *dev, USBPacket *p)
 {
     USBAudioState *s = (USBAudioState *) dev;
-    int ret = 0;
 
-    switch (p->pid) {
-    case USB_TOKEN_OUT:
-        switch (p->ep->nr) {
-        case 1:
-            ret = usb_audio_handle_dataout(s, p);
-            break;
-        default:
-            goto fail;
-        }
-        break;
-
-    default:
-fail:
-        ret = USB_RET_STALL;
-        break;
+    if (p->pid == USB_TOKEN_OUT && p->ep->nr == 1) {
+        usb_audio_handle_dataout(s, p);
+        return;
     }
-    if (ret == USB_RET_STALL && s->debug) {
+
+    p->status = USB_RET_STALL;
+    if (s->debug) {
         fprintf(stderr, "usb-audio: failed data transaction: "
                         "pid 0x%x ep 0x%x len 0x%zx\n",
                         p->pid, p->ep->nr, p->iov.size);
     }
-    return ret;
 }
 
 static void usb_audio_handle_destroy(USBDevice *dev)
diff --git a/hw/usb/dev-bluetooth.c b/hw/usb/dev-bluetooth.c
index 55bc19184..39984f53e 100644
--- a/hw/usb/dev-bluetooth.c
+++ b/hw/usb/dev-bluetooth.c
@@ -27,6 +27,7 @@
 struct USBBtState {
     USBDevice dev;
     struct HCIInfo *hci;
+    USBEndpoint *intr;
 
     int config;
 
@@ -285,13 +286,12 @@ static void usb_bt_fifo_enqueue(struct usb_hci_in_fifo_s *fifo,
     fifo->fifo[off].len = len;
 }
 
-static inline int usb_bt_fifo_dequeue(struct usb_hci_in_fifo_s *fifo,
+static inline void usb_bt_fifo_dequeue(struct usb_hci_in_fifo_s *fifo,
                 USBPacket *p)
 {
     int len;
 
-    if (likely(!fifo->len))
-        return USB_RET_STALL;
+    assert(fifo->len != 0);
 
     len = MIN(p->iov.size, fifo->fifo[fifo->start].len);
     usb_packet_copy(p, fifo->fifo[fifo->start].data, len);
@@ -310,8 +310,6 @@ static inline int usb_bt_fifo_dequeue(struct usb_hci_in_fifo_s *fifo,
         fifo->dstart = 0;
         fifo->dsize = DFIFO_LEN_MASK + 1;
     }
-
-    return len;
 }
 
 static inline void usb_bt_fifo_out_enqueue(struct USBBtState *s,
@@ -363,7 +361,7 @@ static void usb_bt_handle_reset(USBDevice *dev)
     s->outsco.len = 0;
 }
 
-static int usb_bt_handle_control(USBDevice *dev, USBPacket *p,
+static void usb_bt_handle_control(USBDevice *dev, USBPacket *p,
                int request, int value, int index, int length, uint8_t *data)
 {
     struct USBBtState *s = (struct USBBtState *) dev->opaque;
@@ -382,16 +380,15 @@ static int usb_bt_handle_control(USBDevice *dev, USBPacket *p,
             usb_bt_fifo_reset(&s->sco);
             break;
         }
-        return ret;
+        return;
     }
 
-    ret = 0;
     switch (request) {
     case InterfaceRequest | USB_REQ_GET_STATUS:
     case EndpointRequest | USB_REQ_GET_STATUS:
         data[0] = 0x00;
         data[1] = 0x00;
-        ret = 2;
+        p->actual_length = 2;
         break;
     case InterfaceOutRequest | USB_REQ_CLEAR_FEATURE:
     case EndpointOutRequest | USB_REQ_CLEAR_FEATURE:
@@ -407,16 +404,14 @@ static int usb_bt_handle_control(USBDevice *dev, USBPacket *p,
         break;
     default:
     fail:
-        ret = USB_RET_STALL;
+        p->status = USB_RET_STALL;
         break;
     }
-    return ret;
 }
 
-static int usb_bt_handle_data(USBDevice *dev, USBPacket *p)
+static void usb_bt_handle_data(USBDevice *dev, USBPacket *p)
 {
     struct USBBtState *s = (struct USBBtState *) dev->opaque;
-    int ret = 0;
 
     if (!s->config)
         goto fail;
@@ -425,15 +420,27 @@ static int usb_bt_handle_data(USBDevice *dev, USBPacket *p)
     case USB_TOKEN_IN:
         switch (p->ep->nr) {
         case USB_EVT_EP:
-            ret = usb_bt_fifo_dequeue(&s->evt, p);
+            if (s->evt.len == 0) {
+                p->status = USB_RET_NAK;
+                break;
+            }
+            usb_bt_fifo_dequeue(&s->evt, p);
             break;
 
         case USB_ACL_EP:
-            ret = usb_bt_fifo_dequeue(&s->acl, p);
+            if (s->evt.len == 0) {
+                p->status = USB_RET_STALL;
+                break;
+            }
+            usb_bt_fifo_dequeue(&s->acl, p);
             break;
 
         case USB_SCO_EP:
-            ret = usb_bt_fifo_dequeue(&s->sco, p);
+            if (s->evt.len == 0) {
+                p->status = USB_RET_STALL;
+                break;
+            }
+            usb_bt_fifo_dequeue(&s->sco, p);
             break;
 
         default:
@@ -460,11 +467,9 @@ static int usb_bt_handle_data(USBDevice *dev, USBPacket *p)
 
     default:
     fail:
-        ret = USB_RET_STALL;
+        p->status = USB_RET_STALL;
         break;
     }
-
-    return ret;
 }
 
 static void usb_bt_out_hci_packet_event(void *opaque,
@@ -472,6 +477,9 @@ static void usb_bt_out_hci_packet_event(void *opaque,
 {
     struct USBBtState *s = (struct USBBtState *) opaque;
 
+    if (s->evt.len == 0) {
+        usb_wakeup(s->intr);
+    }
     usb_bt_fifo_enqueue(&s->evt, data, len);
 }
 
@@ -494,8 +502,12 @@ static void usb_bt_handle_destroy(USBDevice *dev)
 
 static int usb_bt_initfn(USBDevice *dev)
 {
+    struct USBBtState *s = DO_UPCAST(struct USBBtState, dev, dev);
+
     usb_desc_create_serial(dev);
     usb_desc_init(dev);
+    s->intr = usb_ep_get(dev, USB_TOKEN_IN, USB_EVT_EP);
+
     return 0;
 }
 
diff --git a/hw/usb/dev-hid.c b/hw/usb/dev-hid.c
index b3dcd2310..55266b18e 100644
--- a/hw/usb/dev-hid.c
+++ b/hw/usb/dev-hid.c
@@ -371,7 +371,7 @@ static void usb_hid_handle_reset(USBDevice *dev)
     hid_reset(&us->hid);
 }
 
-static int usb_hid_handle_control(USBDevice *dev, USBPacket *p,
+static void usb_hid_handle_control(USBDevice *dev, USBPacket *p,
                int request, int value, int index, int length, uint8_t *data)
 {
     USBHIDState *us = DO_UPCAST(USBHIDState, dev, dev);
@@ -380,10 +380,9 @@ static int usb_hid_handle_control(USBDevice *dev, USBPacket *p,
 
     ret = usb_desc_handle_control(dev, p, request, value, index, length, data);
     if (ret >= 0) {
-        return ret;
+        return;
     }
 
-    ret = 0;
     switch (request) {
         /* hid specific requests */
     case InterfaceRequest | USB_REQ_GET_DESCRIPTOR:
@@ -392,15 +391,15 @@ static int usb_hid_handle_control(USBDevice *dev, USBPacket *p,
             if (hs->kind == HID_MOUSE) {
 		memcpy(data, qemu_mouse_hid_report_descriptor,
 		       sizeof(qemu_mouse_hid_report_descriptor));
-		ret = sizeof(qemu_mouse_hid_report_descriptor);
+                p->actual_length = sizeof(qemu_mouse_hid_report_descriptor);
             } else if (hs->kind == HID_TABLET) {
                 memcpy(data, qemu_tablet_hid_report_descriptor,
 		       sizeof(qemu_tablet_hid_report_descriptor));
-		ret = sizeof(qemu_tablet_hid_report_descriptor);
+                p->actual_length = sizeof(qemu_tablet_hid_report_descriptor);
             } else if (hs->kind == HID_KEYBOARD) {
                 memcpy(data, qemu_keyboard_hid_report_descriptor,
                        sizeof(qemu_keyboard_hid_report_descriptor));
-                ret = sizeof(qemu_keyboard_hid_report_descriptor);
+                p->actual_length = sizeof(qemu_keyboard_hid_report_descriptor);
             }
             break;
         default:
@@ -409,14 +408,14 @@ static int usb_hid_handle_control(USBDevice *dev, USBPacket *p,
         break;
     case GET_REPORT:
         if (hs->kind == HID_MOUSE || hs->kind == HID_TABLET) {
-            ret = hid_pointer_poll(hs, data, length);
+            p->actual_length = hid_pointer_poll(hs, data, length);
         } else if (hs->kind == HID_KEYBOARD) {
-            ret = hid_keyboard_poll(hs, data, length);
+            p->actual_length = hid_keyboard_poll(hs, data, length);
         }
         break;
     case SET_REPORT:
         if (hs->kind == HID_KEYBOARD) {
-            ret = hid_keyboard_write(hs, data, length);
+            p->actual_length = hid_keyboard_write(hs, data, length);
         } else {
             goto fail;
         }
@@ -425,19 +424,18 @@ static int usb_hid_handle_control(USBDevice *dev, USBPacket *p,
         if (hs->kind != HID_KEYBOARD && hs->kind != HID_MOUSE) {
             goto fail;
         }
-        ret = 1;
         data[0] = hs->protocol;
+        p->actual_length = 1;
         break;
     case SET_PROTOCOL:
         if (hs->kind != HID_KEYBOARD && hs->kind != HID_MOUSE) {
             goto fail;
         }
-        ret = 0;
         hs->protocol = value;
         break;
     case GET_IDLE:
-        ret = 1;
         data[0] = hs->idle;
+        p->actual_length = 1;
         break;
     case SET_IDLE:
         hs->idle = (uint8_t) (value >> 8);
@@ -445,22 +443,20 @@ static int usb_hid_handle_control(USBDevice *dev, USBPacket *p,
         if (hs->kind == HID_MOUSE || hs->kind == HID_TABLET) {
             hid_pointer_activate(hs);
         }
-        ret = 0;
         break;
     default:
     fail:
-        ret = USB_RET_STALL;
+        p->status = USB_RET_STALL;
         break;
     }
-    return ret;
 }
 
-static int usb_hid_handle_data(USBDevice *dev, USBPacket *p)
+static void usb_hid_handle_data(USBDevice *dev, USBPacket *p)
 {
     USBHIDState *us = DO_UPCAST(USBHIDState, dev, dev);
     HIDState *hs = &us->hid;
     uint8_t buf[p->iov.size];
-    int ret = 0;
+    int len = 0;
 
     switch (p->pid) {
     case USB_TOKEN_IN:
@@ -471,15 +467,16 @@ static int usb_hid_handle_data(USBDevice *dev, USBPacket *p)
             }
             if (!hid_has_events(hs) &&
                 (!hs->idle || hs->next_idle_clock - curtime > 0)) {
-                return USB_RET_NAK;
+                p->status = USB_RET_NAK;
+                return;
             }
             hid_set_next_idle(hs, curtime);
             if (hs->kind == HID_MOUSE || hs->kind == HID_TABLET) {
-                ret = hid_pointer_poll(hs, buf, p->iov.size);
+                len = hid_pointer_poll(hs, buf, p->iov.size);
             } else if (hs->kind == HID_KEYBOARD) {
-                ret = hid_keyboard_poll(hs, buf, p->iov.size);
+                len = hid_keyboard_poll(hs, buf, p->iov.size);
             }
-            usb_packet_copy(p, buf, ret);
+            usb_packet_copy(p, buf, len);
         } else {
             goto fail;
         }
@@ -487,10 +484,9 @@ static int usb_hid_handle_data(USBDevice *dev, USBPacket *p)
     case USB_TOKEN_OUT:
     default:
     fail:
-        ret = USB_RET_STALL;
+        p->status = USB_RET_STALL;
         break;
     }
-    return ret;
 }
 
 static void usb_hid_handle_destroy(USBDevice *dev)
diff --git a/hw/usb/dev-hub.c b/hw/usb/dev-hub.c
index 8fd30df0e..9ee60dd41 100644
--- a/hw/usb/dev-hub.c
+++ b/hw/usb/dev-hub.c
@@ -288,7 +288,7 @@ static const char *feature_name(int feature)
     return name[feature] ?: "?";
 }
 
-static int usb_hub_handle_control(USBDevice *dev, USBPacket *p,
+static void usb_hub_handle_control(USBDevice *dev, USBPacket *p,
                int request, int value, int index, int length, uint8_t *data)
 {
     USBHubState *s = (USBHubState *)dev;
@@ -298,7 +298,7 @@ static int usb_hub_handle_control(USBDevice *dev, USBPacket *p,
 
     ret = usb_desc_handle_control(dev, p, request, value, index, length, data);
     if (ret >= 0) {
-        return ret;
+        return;
     }
 
     switch(request) {
@@ -306,7 +306,6 @@ static int usb_hub_handle_control(USBDevice *dev, USBPacket *p,
         if (value == 0 && index != 0x81) { /* clear ep halt */
             goto fail;
         }
-        ret = 0;
         break;
         /* usb specific requests */
     case GetHubStatus:
@@ -314,7 +313,7 @@ static int usb_hub_handle_control(USBDevice *dev, USBPacket *p,
         data[1] = 0;
         data[2] = 0;
         data[3] = 0;
-        ret = 4;
+        p->actual_length = 4;
         break;
     case GetPortStatus:
         {
@@ -331,16 +330,14 @@ static int usb_hub_handle_control(USBDevice *dev, USBPacket *p,
             data[1] = port->wPortStatus >> 8;
             data[2] = port->wPortChange;
             data[3] = port->wPortChange >> 8;
-            ret = 4;
+            p->actual_length = 4;
         }
         break;
     case SetHubFeature:
     case ClearHubFeature:
-        if (value == 0 || value == 1) {
-        } else {
+        if (value != 0 && value != 1) {
             goto fail;
         }
-        ret = 0;
         break;
     case SetPortFeature:
         {
@@ -373,7 +370,6 @@ static int usb_hub_handle_control(USBDevice *dev, USBPacket *p,
             default:
                 goto fail;
             }
-            ret = 0;
         }
         break;
     case ClearPortFeature:
@@ -413,7 +409,6 @@ static int usb_hub_handle_control(USBDevice *dev, USBPacket *p,
             default:
                 goto fail;
             }
-            ret = 0;
         }
         break;
     case GetHubDescriptor:
@@ -437,22 +432,20 @@ static int usb_hub_handle_control(USBDevice *dev, USBPacket *p,
                 var_hub_size++;
             }
 
-            ret = sizeof(qemu_hub_hub_descriptor) + var_hub_size;
-            data[0] = ret;
+            p->actual_length = sizeof(qemu_hub_hub_descriptor) + var_hub_size;
+            data[0] = p->actual_length;
             break;
         }
     default:
     fail:
-        ret = USB_RET_STALL;
+        p->status = USB_RET_STALL;
         break;
     }
-    return ret;
 }
 
-static int usb_hub_handle_data(USBDevice *dev, USBPacket *p)
+static void usb_hub_handle_data(USBDevice *dev, USBPacket *p)
 {
     USBHubState *s = (USBHubState *)dev;
-    int ret;
 
     switch(p->pid) {
     case USB_TOKEN_IN:
@@ -465,7 +458,8 @@ static int usb_hub_handle_data(USBDevice *dev, USBPacket *p)
             if (p->iov.size == 1) { /* FreeBSD workaround */
                 n = 1;
             } else if (n > p->iov.size) {
-                return USB_RET_BABBLE;
+                p->status = USB_RET_BABBLE;
+                return;
             }
             status = 0;
             for(i = 0; i < NUM_PORTS; i++) {
@@ -478,9 +472,8 @@ static int usb_hub_handle_data(USBDevice *dev, USBPacket *p)
                     buf[i] = status >> (8 * i);
                 }
                 usb_packet_copy(p, buf, n);
-                ret = n;
             } else {
-                ret = USB_RET_NAK; /* usb11 11.13.1 */
+                p->status = USB_RET_NAK; /* usb11 11.13.1 */
             }
         } else {
             goto fail;
@@ -489,10 +482,9 @@ static int usb_hub_handle_data(USBDevice *dev, USBPacket *p)
     case USB_TOKEN_OUT:
     default:
     fail:
-        ret = USB_RET_STALL;
+        p->status = USB_RET_STALL;
         break;
     }
-    return ret;
 }
 
 static void usb_hub_handle_destroy(USBDevice *dev)
diff --git a/hw/usb/dev-network.c b/hw/usb/dev-network.c
index c84892c98..14d9e5aa5 100644
--- a/hw/usb/dev-network.c
+++ b/hw/usb/dev-network.c
@@ -1001,6 +1001,13 @@ static int rndis_keepalive_response(USBNetState *s,
     return 0;
 }
 
+/* Prepare to receive the next packet */
+static void usb_net_reset_in_buf(USBNetState *s)
+{
+    s->in_ptr = s->in_len = 0;
+    qemu_flush_queued_packets(&s->nic->nc);
+}
+
 static int rndis_parse(USBNetState *s, uint8_t *data, int length)
 {
     uint32_t msg_type;
@@ -1025,7 +1032,8 @@ static int rndis_parse(USBNetState *s, uint8_t *data, int length)
 
     case RNDIS_RESET_MSG:
         rndis_clear_responsequeue(s);
-        s->out_ptr = s->in_ptr = s->in_len = 0;
+        s->out_ptr = 0;
+        usb_net_reset_in_buf(s);
         return rndis_reset_response(s, (rndis_reset_msg_type *) data);
 
     case RNDIS_KEEPALIVE_MSG:
@@ -1040,7 +1048,7 @@ static void usb_net_handle_reset(USBDevice *dev)
 {
 }
 
-static int usb_net_handle_control(USBDevice *dev, USBPacket *p,
+static void usb_net_handle_control(USBDevice *dev, USBPacket *p,
                int request, int value, int index, int length, uint8_t *data)
 {
     USBNetState *s = (USBNetState *) dev;
@@ -1048,10 +1056,9 @@ static int usb_net_handle_control(USBDevice *dev, USBPacket *p,
 
     ret = usb_desc_handle_control(dev, p, request, value, index, length, data);
     if (ret >= 0) {
-        return ret;
+        return;
     }
 
-    ret = 0;
     switch(request) {
     case ClassInterfaceOutRequest | USB_CDC_SEND_ENCAPSULATED_COMMAND:
         if (!is_rndis(s) || value || index != 0) {
@@ -1070,22 +1077,25 @@ static int usb_net_handle_control(USBDevice *dev, USBPacket *p,
         }
 #endif
         ret = rndis_parse(s, data, length);
+        if (ret < 0) {
+            p->status = ret;
+        }
         break;
 
     case ClassInterfaceRequest | USB_CDC_GET_ENCAPSULATED_RESPONSE:
         if (!is_rndis(s) || value || index != 0) {
             goto fail;
         }
-        ret = rndis_get_response(s, data);
-        if (!ret) {
+        p->actual_length = rndis_get_response(s, data);
+        if (p->actual_length == 0) {
             data[0] = 0;
-            ret = 1;
+            p->actual_length = 1;
         }
 #ifdef TRAFFIC_DEBUG
         {
             unsigned int i;
             fprintf(stderr, "GET_ENCAPSULATED_RESPONSE:");
-            for (i = 0; i < ret; i++) {
+            for (i = 0; i < p->actual_length; i++) {
                 if (!(i & 15))
                     fprintf(stderr, "\n%04x:", i);
                 fprintf(stderr, " %02x", data[i]);
@@ -1100,72 +1110,67 @@ static int usb_net_handle_control(USBDevice *dev, USBPacket *p,
         fprintf(stderr, "usbnet: failed control transaction: "
                         "request 0x%x value 0x%x index 0x%x length 0x%x\n",
                         request, value, index, length);
-        ret = USB_RET_STALL;
+        p->status = USB_RET_STALL;
         break;
     }
-    return ret;
 }
 
-static int usb_net_handle_statusin(USBNetState *s, USBPacket *p)
+static void usb_net_handle_statusin(USBNetState *s, USBPacket *p)
 {
     le32 buf[2];
-    int ret = 8;
 
     if (p->iov.size < 8) {
-        return USB_RET_STALL;
+        p->status = USB_RET_STALL;
+        return;
     }
 
     buf[0] = cpu_to_le32(1);
     buf[1] = cpu_to_le32(0);
     usb_packet_copy(p, buf, 8);
-    if (!s->rndis_resp.tqh_first)
-        ret = USB_RET_NAK;
+    if (!s->rndis_resp.tqh_first) {
+        p->status = USB_RET_NAK;
+    }
 
 #ifdef TRAFFIC_DEBUG
     fprintf(stderr, "usbnet: interrupt poll len %zu return %d",
-            p->iov.size, ret);
-    iov_hexdump(p->iov.iov, p->iov.niov, stderr, "usbnet", ret);
+            p->iov.size, p->status);
+    iov_hexdump(p->iov.iov, p->iov.niov, stderr, "usbnet", p->status);
 #endif
-
-    return ret;
 }
 
-static int usb_net_handle_datain(USBNetState *s, USBPacket *p)
+static void usb_net_handle_datain(USBNetState *s, USBPacket *p)
 {
-    int ret = USB_RET_NAK;
+    int len;
 
     if (s->in_ptr > s->in_len) {
-        s->in_ptr = s->in_len = 0;
-        ret = USB_RET_NAK;
-        return ret;
+        usb_net_reset_in_buf(s);
+        p->status = USB_RET_NAK;
+        return;
     }
     if (!s->in_len) {
-        ret = USB_RET_NAK;
-        return ret;
+        p->status = USB_RET_NAK;
+        return;
     }
-    ret = s->in_len - s->in_ptr;
-    if (ret > p->iov.size) {
-        ret = p->iov.size;
+    len = s->in_len - s->in_ptr;
+    if (len > p->iov.size) {
+        len = p->iov.size;
     }
-    usb_packet_copy(p, &s->in_buf[s->in_ptr], ret);
-    s->in_ptr += ret;
+    usb_packet_copy(p, &s->in_buf[s->in_ptr], len);
+    s->in_ptr += len;
     if (s->in_ptr >= s->in_len &&
-                    (is_rndis(s) || (s->in_len & (64 - 1)) || !ret)) {
+                    (is_rndis(s) || (s->in_len & (64 - 1)) || !len)) {
         /* no short packet necessary */
-        s->in_ptr = s->in_len = 0;
+        usb_net_reset_in_buf(s);
     }
 
 #ifdef TRAFFIC_DEBUG
-    fprintf(stderr, "usbnet: data in len %zu return %d", p->iov.size, ret);
-    iov_hexdump(p->iov.iov, p->iov.niov, stderr, "usbnet", ret);
+    fprintf(stderr, "usbnet: data in len %zu return %d", p->iov.size, len);
+    iov_hexdump(p->iov.iov, p->iov.niov, stderr, "usbnet", len);
 #endif
-
-    return ret;
 }
 
-static int usb_net_handle_dataout(USBNetState *s, USBPacket *p)
+static void usb_net_handle_dataout(USBNetState *s, USBPacket *p)
 {
-    int ret = p->iov.size;
     int sz = sizeof(s->out_buf) - s->out_ptr;
     struct rndis_packet_msg_type *msg =
             (struct rndis_packet_msg_type *) s->out_buf;
@@ -1176,21 +1181,23 @@ static int usb_net_handle_dataout(USBNetState *s, USBPacket *p)
     iov_hexdump(p->iov.iov, p->iov.niov, stderr, "usbnet", p->iov.size);
 #endif
 
-    if (sz > ret)
-        sz = ret;
+    if (sz > p->iov.size) {
+        sz = p->iov.size;
+    }
     usb_packet_copy(p, &s->out_buf[s->out_ptr], sz);
     s->out_ptr += sz;
 
     if (!is_rndis(s)) {
-        if (ret < 64) {
+        if (p->iov.size < 64) {
             qemu_send_packet(&s->nic->nc, s->out_buf, s->out_ptr);
             s->out_ptr = 0;
         }
-        return ret;
+        return;
     }
     len = le32_to_cpu(msg->MessageLength);
-    if (s->out_ptr < 8 || s->out_ptr < len)
-        return ret;
+    if (s->out_ptr < 8 || s->out_ptr < len) {
+        return;
+    }
     if (le32_to_cpu(msg->MessageType) == RNDIS_PACKET_MSG) {
         uint32_t offs = 8 + le32_to_cpu(msg->DataOffset);
         uint32_t size = le32_to_cpu(msg->DataLength);
@@ -1199,24 +1206,21 @@ static int usb_net_handle_dataout(USBNetState *s, USBPacket *p)
     }
     s->out_ptr -= len;
     memmove(s->out_buf, &s->out_buf[len], s->out_ptr);
-
-    return ret;
 }
 
-static int usb_net_handle_data(USBDevice *dev, USBPacket *p)
+static void usb_net_handle_data(USBDevice *dev, USBPacket *p)
 {
     USBNetState *s = (USBNetState *) dev;
-    int ret = 0;
 
     switch(p->pid) {
     case USB_TOKEN_IN:
         switch (p->ep->nr) {
         case 1:
-            ret = usb_net_handle_statusin(s, p);
+            usb_net_handle_statusin(s, p);
             break;
 
         case 2:
-            ret = usb_net_handle_datain(s, p);
+            usb_net_handle_datain(s, p);
             break;
 
         default:
@@ -1227,7 +1231,7 @@ static int usb_net_handle_data(USBDevice *dev, USBPacket *p)
     case USB_TOKEN_OUT:
         switch (p->ep->nr) {
         case 2:
-            ret = usb_net_handle_dataout(s, p);
+            usb_net_handle_dataout(s, p);
             break;
 
         default:
@@ -1237,33 +1241,46 @@ static int usb_net_handle_data(USBDevice *dev, USBPacket *p)
 
     default:
     fail:
-        ret = USB_RET_STALL;
+        p->status = USB_RET_STALL;
         break;
     }
-    if (ret == USB_RET_STALL)
+
+    if (p->status == USB_RET_STALL) {
         fprintf(stderr, "usbnet: failed data transaction: "
                         "pid 0x%x ep 0x%x len 0x%zx\n",
                         p->pid, p->ep->nr, p->iov.size);
-    return ret;
+    }
 }
 
 static ssize_t usbnet_receive(NetClientState *nc, const uint8_t *buf, size_t size)
 {
     USBNetState *s = DO_UPCAST(NICState, nc, nc)->opaque;
-    struct rndis_packet_msg_type *msg;
+    uint8_t *in_buf = s->in_buf;
+    size_t total_size = size;
 
     if (is_rndis(s)) {
-        msg = (struct rndis_packet_msg_type *) s->in_buf;
         if (s->rndis_state != RNDIS_DATA_INITIALIZED) {
             return -1;
         }
-        if (size + sizeof(struct rndis_packet_msg_type) > sizeof(s->in_buf))
-            return -1;
+        total_size += sizeof(struct rndis_packet_msg_type);
+    }
+    if (total_size > sizeof(s->in_buf)) {
+        return -1;
+    }
+
+    /* Only accept packet if input buffer is empty */
+    if (s->in_len > 0) {
+        return 0;
+    }
 
+    if (is_rndis(s)) {
+        struct rndis_packet_msg_type *msg;
+
+        msg = (struct rndis_packet_msg_type *)in_buf;
         memset(msg, 0, sizeof(struct rndis_packet_msg_type));
         msg->MessageType = cpu_to_le32(RNDIS_PACKET_MSG);
-        msg->MessageLength = cpu_to_le32(size + sizeof(struct rndis_packet_msg_type));
-        msg->DataOffset = cpu_to_le32(sizeof(struct rndis_packet_msg_type) - 8);
+        msg->MessageLength = cpu_to_le32(size + sizeof(*msg));
+        msg->DataOffset = cpu_to_le32(sizeof(*msg) - 8);
         msg->DataLength = cpu_to_le32(size);
         /* msg->OOBDataOffset;
          * msg->OOBDataLength;
@@ -1273,14 +1290,11 @@ static ssize_t usbnet_receive(NetClientState *nc, const uint8_t *buf, size_t siz
          * msg->VcHandle;
          * msg->Reserved;
          */
-        memcpy(msg + 1, buf, size);
-        s->in_len = size + sizeof(struct rndis_packet_msg_type);
-    } else {
-        if (size > sizeof(s->in_buf))
-            return -1;
-        memcpy(s->in_buf, buf, size);
-        s->in_len = size;
+        in_buf += sizeof(*msg);
     }
+
+    memcpy(in_buf, buf, size);
+    s->in_len = total_size;
     s->in_ptr = 0;
     return size;
 }
diff --git a/hw/usb/dev-serial.c b/hw/usb/dev-serial.c
index 8aa655286..99b19df1d 100644
--- a/hw/usb/dev-serial.c
+++ b/hw/usb/dev-serial.c
@@ -113,7 +113,7 @@ enum {
 static const USBDescStrings desc_strings = {
     [STR_MANUFACTURER]    = "QEMU",
     [STR_PRODUCT_SERIAL]  = "QEMU USB SERIAL",
-    [STR_PRODUCT_BRAILLE] = "QEMU USB BRAILLE",
+    [STR_PRODUCT_BRAILLE] = "QEMU USB BAUM BRAILLE",
     [STR_SERIALNUMBER]    = "1",
 };
 
@@ -219,7 +219,7 @@ static uint8_t usb_get_modem_lines(USBSerialState *s)
     return ret;
 }
 
-static int usb_serial_handle_control(USBDevice *dev, USBPacket *p,
+static void usb_serial_handle_control(USBDevice *dev, USBPacket *p,
                int request, int value, int index, int length, uint8_t *data)
 {
     USBSerialState *s = (USBSerialState *)dev;
@@ -228,13 +228,11 @@ static int usb_serial_handle_control(USBDevice *dev, USBPacket *p,
     DPRINTF("got control %x, value %x\n",request, value);
     ret = usb_desc_handle_control(dev, p, request, value, index, length, data);
     if (ret >= 0) {
-        return ret;
+        return;
     }
 
-    ret = 0;
     switch (request) {
     case EndpointOutRequest | USB_REQ_CLEAR_FEATURE:
-        ret = 0;
         break;
 
         /* Class specific requests.  */
@@ -323,7 +321,7 @@ static int usb_serial_handle_control(USBDevice *dev, USBPacket *p,
     case DeviceInVendor | FTDI_GET_MDM_ST:
         data[0] = usb_get_modem_lines(s) | 1;
         data[1] = 0;
-        ret = 2;
+        p->actual_length = 2;
         break;
     case DeviceOutVendor | FTDI_SET_EVENT_CHR:
         /* TODO: handle it */
@@ -338,25 +336,23 @@ static int usb_serial_handle_control(USBDevice *dev, USBPacket *p,
         break;
     case DeviceInVendor | FTDI_GET_LATENCY:
         data[0] = s->latency;
-        ret = 1;
+        p->actual_length = 1;
         break;
     default:
     fail:
         DPRINTF("got unsupported/bogus control %x, value %x\n", request, value);
-        ret = USB_RET_STALL;
+        p->status = USB_RET_STALL;
         break;
     }
-    return ret;
 }
 
-static int usb_serial_handle_data(USBDevice *dev, USBPacket *p)
+static void usb_serial_handle_data(USBDevice *dev, USBPacket *p)
 {
     USBSerialState *s = (USBSerialState *)dev;
-    int i, ret = 0;
     uint8_t devep = p->ep->nr;
     struct iovec *iov;
     uint8_t header[2];
-    int first_len, len;
+    int i, first_len, len;
 
     switch (p->pid) {
     case USB_TOKEN_OUT:
@@ -366,6 +362,7 @@ static int usb_serial_handle_data(USBDevice *dev, USBPacket *p)
             iov = p->iov.iov + i;
             qemu_chr_fe_write(s->cs, iov->iov_base, iov->iov_len);
         }
+        p->actual_length = p->iov.size;
         break;
 
     case USB_TOKEN_IN:
@@ -374,7 +371,7 @@ static int usb_serial_handle_data(USBDevice *dev, USBPacket *p)
         first_len = RECV_BUF - s->recv_ptr;
         len = p->iov.size;
         if (len <= 2) {
-            ret = USB_RET_NAK;
+            p->status = USB_RET_NAK;
             break;
         }
         header[0] = usb_get_modem_lines(s) | 1;
@@ -384,7 +381,6 @@ static int usb_serial_handle_data(USBDevice *dev, USBPacket *p)
             s->event_trigger &= ~FTDI_BI;
             header[1] = FTDI_BI;
             usb_packet_copy(p, header, 2);
-            ret = 2;
             break;
         } else {
             header[1] = 0;
@@ -393,7 +389,7 @@ static int usb_serial_handle_data(USBDevice *dev, USBPacket *p)
         if (len > s->recv_used)
             len = s->recv_used;
         if (!len) {
-            ret = USB_RET_NAK;
+            p->status = USB_RET_NAK;
             break;
         }
         if (first_len > len)
@@ -404,29 +400,30 @@ static int usb_serial_handle_data(USBDevice *dev, USBPacket *p)
             usb_packet_copy(p, s->recv_buf, len - first_len);
         s->recv_used -= len;
         s->recv_ptr = (s->recv_ptr + len) % RECV_BUF;
-        ret = len + 2;
         break;
 
     default:
         DPRINTF("Bad token\n");
     fail:
-        ret = USB_RET_STALL;
+        p->status = USB_RET_STALL;
         break;
     }
-
-    return ret;
 }
 
 static void usb_serial_handle_destroy(USBDevice *dev)
 {
     USBSerialState *s = (USBSerialState *)dev;
 
-    qemu_chr_delete(s->cs);
+    qemu_chr_add_handlers(s->cs, NULL, NULL, NULL, NULL);
 }
 
 static int usb_serial_can_read(void *opaque)
 {
     USBSerialState *s = opaque;
+
+    if (!s->dev.attached) {
+        return 0;
+    }
     return RECV_BUF - s->recv_used;
 }
 
@@ -469,8 +466,14 @@ static void usb_serial_event(void *opaque, int event)
         case CHR_EVENT_FOCUS:
             break;
         case CHR_EVENT_OPENED:
-            usb_serial_reset(s);
-            /* TODO: Reset USB port */
+            if (!s->dev.attached) {
+                usb_device_attach(&s->dev);
+            }
+            break;
+        case CHR_EVENT_CLOSED:
+            if (s->dev.attached) {
+                usb_device_detach(&s->dev);
+            }
             break;
     }
 }
@@ -481,6 +484,7 @@ static int usb_serial_initfn(USBDevice *dev)
 
     usb_desc_create_serial(dev);
     usb_desc_init(dev);
+    dev->auto_attach = 0;
 
     if (!s->cs) {
         error_report("Property chardev is required");
@@ -490,6 +494,10 @@ static int usb_serial_initfn(USBDevice *dev)
     qemu_chr_add_handlers(s->cs, usb_serial_can_read, usb_serial_read,
                           usb_serial_event, s);
     usb_serial_handle_reset(dev);
+
+    if (s->cs->opened && !dev->attached) {
+        usb_device_attach(dev);
+    }
     return 0;
 }
 
diff --git a/hw/usb/dev-smartcard-reader.c b/hw/usb/dev-smartcard-reader.c
index 1ea079176..de955b709 100644
--- a/hw/usb/dev-smartcard-reader.c
+++ b/hw/usb/dev-smartcard-reader.c
@@ -635,39 +635,38 @@ static void ccid_handle_reset(USBDevice *dev)
     ccid_reset(s);
 }
 
-static int ccid_handle_control(USBDevice *dev, USBPacket *p, int request,
+static void ccid_handle_control(USBDevice *dev, USBPacket *p, int request,
                                int value, int index, int length, uint8_t *data)
 {
     USBCCIDState *s = DO_UPCAST(USBCCIDState, dev, dev);
-    int ret = 0;
+    int ret;
 
     DPRINTF(s, 1, "got control %x, value %x\n", request, value);
     ret = usb_desc_handle_control(dev, p, request, value, index, length, data);
     if (ret >= 0) {
-        return ret;
+        return;
     }
 
     switch (request) {
         /* Class specific requests.  */
     case InterfaceOutClass | CCID_CONTROL_ABORT:
         DPRINTF(s, 1, "ccid_control abort UNIMPLEMENTED\n");
-        ret = USB_RET_STALL;
+        p->status = USB_RET_STALL;
         break;
     case InterfaceInClass | CCID_CONTROL_GET_CLOCK_FREQUENCIES:
         DPRINTF(s, 1, "ccid_control get clock frequencies UNIMPLEMENTED\n");
-        ret = USB_RET_STALL;
+        p->status = USB_RET_STALL;
         break;
     case InterfaceInClass | CCID_CONTROL_GET_DATA_RATES:
         DPRINTF(s, 1, "ccid_control get data rates UNIMPLEMENTED\n");
-        ret = USB_RET_STALL;
+        p->status = USB_RET_STALL;
         break;
     default:
         DPRINTF(s, 1, "got unsupported/bogus control %x, value %x\n",
                 request, value);
-        ret = USB_RET_STALL;
+        p->status = USB_RET_STALL;
         break;
     }
-    return ret;
 }
 
 static bool ccid_card_inserted(USBCCIDState *s)
@@ -870,18 +869,13 @@ static void ccid_on_apdu_from_guest(USBCCIDState *s, CCID_XferBlock *recv)
     }
 }
 
-/*
- * Handle a single USB_TOKEN_OUT, return value returned to guest.
- * Return value:
- *  0             - all ok
- *  USB_RET_STALL - failed to handle packet
- */
-static int ccid_handle_bulk_out(USBCCIDState *s, USBPacket *p)
+static void ccid_handle_bulk_out(USBCCIDState *s, USBPacket *p)
 {
     CCID_Header *ccid_header;
 
     if (p->iov.size + s->bulk_out_pos > BULK_OUT_DATA_SIZE) {
-        return USB_RET_STALL;
+        p->status = USB_RET_STALL;
+        return;
     }
     ccid_header = (CCID_Header *)s->bulk_out_data;
     usb_packet_copy(p, s->bulk_out_data + s->bulk_out_pos, p->iov.size);
@@ -890,7 +884,7 @@ static int ccid_handle_bulk_out(USBCCIDState *s, USBPacket *p)
         DPRINTF(s, D_VERBOSE,
             "usb-ccid: bulk_in: expecting more packets (%zd/%d)\n",
             p->iov.size, ccid_header->dwLength);
-        return 0;
+        return;
     }
     if (s->bulk_out_pos < 10) {
         DPRINTF(s, 1,
@@ -949,60 +943,52 @@ static int ccid_handle_bulk_out(USBCCIDState *s, USBPacket *p)
         }
     }
     s->bulk_out_pos = 0;
-    return 0;
 }
 
-static int ccid_bulk_in_copy_to_guest(USBCCIDState *s, USBPacket *p)
+static void ccid_bulk_in_copy_to_guest(USBCCIDState *s, USBPacket *p)
 {
-    int ret = 0;
+    int len = 0;
 
-    assert(p->iov.size > 0);
     ccid_bulk_in_get(s);
     if (s->current_bulk_in != NULL) {
-        ret = MIN(s->current_bulk_in->len - s->current_bulk_in->pos,
+        len = MIN(s->current_bulk_in->len - s->current_bulk_in->pos,
                   p->iov.size);
         usb_packet_copy(p, s->current_bulk_in->data +
-                        s->current_bulk_in->pos, ret);
-        s->current_bulk_in->pos += ret;
+                        s->current_bulk_in->pos, len);
+        s->current_bulk_in->pos += len;
         if (s->current_bulk_in->pos == s->current_bulk_in->len) {
             ccid_bulk_in_release(s);
         }
     } else {
         /* return when device has no data - usb 2.0 spec Table 8-4 */
-        ret = USB_RET_NAK;
+        p->status = USB_RET_NAK;
     }
-    if (ret > 0) {
+    if (len) {
         DPRINTF(s, D_MORE_INFO,
                 "%s: %zd/%d req/act to guest (BULK_IN)\n",
-                __func__, p->iov.size, ret);
+                __func__, p->iov.size, len);
     }
-    if (ret != USB_RET_NAK && ret < p->iov.size) {
+    if (len < p->iov.size) {
         DPRINTF(s, 1,
                 "%s: returning short (EREMOTEIO) %d < %zd\n",
-                __func__, ret, p->iov.size);
+                __func__, len, p->iov.size);
     }
-    return ret;
 }
 
-static int ccid_handle_data(USBDevice *dev, USBPacket *p)
+static void ccid_handle_data(USBDevice *dev, USBPacket *p)
 {
     USBCCIDState *s = DO_UPCAST(USBCCIDState, dev, dev);
-    int ret = 0;
     uint8_t buf[2];
 
     switch (p->pid) {
     case USB_TOKEN_OUT:
-        ret = ccid_handle_bulk_out(s, p);
+        ccid_handle_bulk_out(s, p);
         break;
 
     case USB_TOKEN_IN:
         switch (p->ep->nr) {
         case CCID_BULK_IN_EP:
-            if (!p->iov.size) {
-                ret = USB_RET_NAK;
-            } else {
-                ret = ccid_bulk_in_copy_to_guest(s, p);
-            }
+            ccid_bulk_in_copy_to_guest(s, p);
             break;
         case CCID_INT_IN_EP:
             if (s->notify_slot_change) {
@@ -1010,28 +996,27 @@ static int ccid_handle_data(USBDevice *dev, USBPacket *p)
                 buf[0] = CCID_MESSAGE_TYPE_RDR_to_PC_NotifySlotChange;
                 buf[1] = s->bmSlotICCState;
                 usb_packet_copy(p, buf, 2);
-                ret = 2;
                 s->notify_slot_change = false;
                 s->bmSlotICCState &= ~SLOT_0_CHANGED_MASK;
                 DPRINTF(s, D_INFO,
                         "handle_data: int_in: notify_slot_change %X, "
                         "requested len %zd\n",
                         s->bmSlotICCState, p->iov.size);
+            } else {
+                p->status = USB_RET_NAK;
             }
             break;
         default:
             DPRINTF(s, 1, "Bad endpoint\n");
-            ret = USB_RET_STALL;
+            p->status = USB_RET_STALL;
             break;
         }
         break;
     default:
         DPRINTF(s, 1, "Bad token\n");
-        ret = USB_RET_STALL;
+        p->status = USB_RET_STALL;
         break;
     }
-
-    return ret;
 }
 
 static void ccid_handle_destroy(USBDevice *dev)
diff --git a/hw/usb/dev-storage.c b/hw/usb/dev-storage.c
index ff48d9104..50af97109 100644
--- a/hw/usb/dev-storage.c
+++ b/hw/usb/dev-storage.c
@@ -78,6 +78,7 @@ enum {
     STR_SERIALNUMBER,
     STR_CONFIG_FULL,
     STR_CONFIG_HIGH,
+    STR_CONFIG_SUPER,
 };
 
 static const USBDescStrings desc_strings = {
@@ -86,6 +87,7 @@ static const USBDescStrings desc_strings = {
     [STR_SERIALNUMBER] = "1",
     [STR_CONFIG_FULL]  = "Full speed config (usb 1.1)",
     [STR_CONFIG_HIGH]  = "High speed config (usb 2.0)",
+    [STR_CONFIG_SUPER] = "Super speed config (usb 3.0)",
 };
 
 static const USBDescIface desc_iface_full = {
@@ -158,6 +160,43 @@ static const USBDescDevice desc_device_high = {
     },
 };
 
+static const USBDescIface desc_iface_super = {
+    .bInterfaceNumber              = 0,
+    .bNumEndpoints                 = 2,
+    .bInterfaceClass               = USB_CLASS_MASS_STORAGE,
+    .bInterfaceSubClass            = 0x06, /* SCSI */
+    .bInterfaceProtocol            = 0x50, /* Bulk */
+    .eps = (USBDescEndpoint[]) {
+        {
+            .bEndpointAddress      = USB_DIR_IN | 0x01,
+            .bmAttributes          = USB_ENDPOINT_XFER_BULK,
+            .wMaxPacketSize        = 1024,
+            .bMaxBurst             = 15,
+        },{
+            .bEndpointAddress      = USB_DIR_OUT | 0x02,
+            .bmAttributes          = USB_ENDPOINT_XFER_BULK,
+            .wMaxPacketSize        = 1024,
+            .bMaxBurst             = 15,
+        },
+    }
+};
+
+static const USBDescDevice desc_device_super = {
+    .bcdUSB                        = 0x0300,
+    .bMaxPacketSize0               = 9,
+    .bNumConfigurations            = 1,
+    .confs = (USBDescConfig[]) {
+        {
+            .bNumInterfaces        = 1,
+            .bConfigurationValue   = 1,
+            .iConfiguration        = STR_CONFIG_SUPER,
+            .bmAttributes          = 0xc0,
+            .nif = 1,
+            .ifs = &desc_iface_super,
+        },
+    },
+};
+
 static const USBDesc desc = {
     .id = {
         .idVendor          = 0x46f4, /* CRC16() of "QEMU" */
@@ -167,15 +206,16 @@ static const USBDesc desc = {
         .iProduct          = STR_PRODUCT,
         .iSerialNumber     = STR_SERIALNUMBER,
     },
-    .full = &desc_device_full,
-    .high = &desc_device_high,
-    .str  = desc_strings,
+    .full  = &desc_device_full,
+    .high  = &desc_device_high,
+    .super = &desc_device_super,
+    .str   = desc_strings,
 };
 
 static void usb_msd_copy_data(MSDState *s, USBPacket *p)
 {
     uint32_t len;
-    len = p->iov.size - p->result;
+    len = p->iov.size - p->actual_length;
     if (len > s->scsi_len)
         len = s->scsi_len;
     usb_packet_copy(p, scsi_req_get_buf(s->req) + s->scsi_off, len);
@@ -223,7 +263,8 @@ static void usb_msd_transfer_data(SCSIRequest *req, uint32_t len)
     if (p) {
         usb_msd_copy_data(s, p);
         p = s->packet;
-        if (p && p->result == p->iov.size) {
+        if (p && p->actual_length == p->iov.size) {
+            p->status = USB_RET_SUCCESS; /* Clear previous ASYNC status */
             usb_msd_packet_complete(s);
         }
     }
@@ -252,7 +293,7 @@ static void usb_msd_command_complete(SCSIRequest *req, uint32_t status, size_t r
             s->mode = USB_MSDM_CBW;
         } else {
             if (s->data_len) {
-                int len = (p->iov.size - p->result);
+                int len = (p->iov.size - p->actual_length);
                 usb_packet_skip(p, len);
                 s->data_len -= len;
             }
@@ -260,6 +301,7 @@ static void usb_msd_command_complete(SCSIRequest *req, uint32_t status, size_t r
                 s->mode = USB_MSDM_CSW;
             }
         }
+        p->status = USB_RET_SUCCESS; /* Clear previous ASYNC status */
         usb_msd_packet_complete(s);
     } else if (s->data_len == 0) {
         s->mode = USB_MSDM_CSW;
@@ -290,14 +332,14 @@ static void usb_msd_handle_reset(USBDevice *dev)
     assert(s->req == NULL);
 
     if (s->packet) {
-        s->packet->result = USB_RET_STALL;
+        s->packet->status = USB_RET_STALL;
         usb_msd_packet_complete(s);
     }
 
     s->mode = USB_MSDM_CBW;
 }
 
-static int usb_msd_handle_control(USBDevice *dev, USBPacket *p,
+static void usb_msd_handle_control(USBDevice *dev, USBPacket *p,
                int request, int value, int index, int length, uint8_t *data)
 {
     MSDState *s = (MSDState *)dev;
@@ -305,29 +347,25 @@ static int usb_msd_handle_control(USBDevice *dev, USBPacket *p,
 
     ret = usb_desc_handle_control(dev, p, request, value, index, length, data);
     if (ret >= 0) {
-        return ret;
+        return;
     }
 
-    ret = 0;
     switch (request) {
     case EndpointOutRequest | USB_REQ_CLEAR_FEATURE:
-        ret = 0;
         break;
         /* Class specific requests.  */
     case ClassInterfaceOutRequest | MassStorageReset:
         /* Reset state ready for the next CBW.  */
         s->mode = USB_MSDM_CBW;
-        ret = 0;
         break;
     case ClassInterfaceRequest | GetMaxLun:
         data[0] = 0;
-        ret = 1;
+        p->actual_length = 1;
         break;
     default:
-        ret = USB_RET_STALL;
+        p->status = USB_RET_STALL;
         break;
     }
-    return ret;
 }
 
 static void usb_msd_cancel_io(USBDevice *dev, USBPacket *p)
@@ -342,11 +380,10 @@ static void usb_msd_cancel_io(USBDevice *dev, USBPacket *p)
     }
 }
 
-static int usb_msd_handle_data(USBDevice *dev, USBPacket *p)
+static void usb_msd_handle_data(USBDevice *dev, USBPacket *p)
 {
     MSDState *s = (MSDState *)dev;
     uint32_t tag;
-    int ret = 0;
     struct usb_msd_cbw cbw;
     uint8_t devep = p->ep->nr;
 
@@ -393,7 +430,6 @@ static int usb_msd_handle_data(USBDevice *dev, USBPacket *p)
             if (s->req && s->req->cmd.xfer != SCSI_XFER_NONE) {
                 scsi_req_continue(s->req);
             }
-            ret = p->result;
             break;
 
         case USB_MSDM_DATAOUT:
@@ -406,7 +442,7 @@ static int usb_msd_handle_data(USBDevice *dev, USBPacket *p)
                 usb_msd_copy_data(s, p);
             }
             if (le32_to_cpu(s->csw.residue)) {
-                int len = p->iov.size - p->result;
+                int len = p->iov.size - p->actual_length;
                 if (len) {
                     usb_packet_skip(p, len);
                     s->data_len -= len;
@@ -415,12 +451,10 @@ static int usb_msd_handle_data(USBDevice *dev, USBPacket *p)
                     }
                 }
             }
-            if (p->result < p->iov.size) {
+            if (p->actual_length < p->iov.size) {
                 DPRINTF("Deferring packet %p [wait data-out]\n", p);
                 s->packet = p;
-                ret = USB_RET_ASYNC;
-            } else {
-                ret = p->result;
+                p->status = USB_RET_ASYNC;
             }
             break;
 
@@ -441,7 +475,7 @@ static int usb_msd_handle_data(USBDevice *dev, USBPacket *p)
             }
             /* Waiting for SCSI write to complete.  */
             s->packet = p;
-            ret = USB_RET_ASYNC;
+            p->status = USB_RET_ASYNC;
             break;
 
         case USB_MSDM_CSW:
@@ -453,11 +487,10 @@ static int usb_msd_handle_data(USBDevice *dev, USBPacket *p)
                 /* still in flight */
                 DPRINTF("Deferring packet %p [wait status]\n", p);
                 s->packet = p;
-                ret = USB_RET_ASYNC;
+                p->status = USB_RET_ASYNC;
             } else {
                 usb_msd_send_status(s, p);
                 s->mode = USB_MSDM_CBW;
-                ret = 13;
             }
             break;
 
@@ -468,7 +501,7 @@ static int usb_msd_handle_data(USBDevice *dev, USBPacket *p)
                 usb_msd_copy_data(s, p);
             }
             if (le32_to_cpu(s->csw.residue)) {
-                int len = p->iov.size - p->result;
+                int len = p->iov.size - p->actual_length;
                 if (len) {
                     usb_packet_skip(p, len);
                     s->data_len -= len;
@@ -477,12 +510,10 @@ static int usb_msd_handle_data(USBDevice *dev, USBPacket *p)
                     }
                 }
             }
-            if (p->result < p->iov.size) {
+            if (p->actual_length < p->iov.size) {
                 DPRINTF("Deferring packet %p [wait data-in]\n", p);
                 s->packet = p;
-                ret = USB_RET_ASYNC;
-            } else {
-                ret = p->result;
+                p->status = USB_RET_ASYNC;
             }
             break;
 
@@ -495,11 +526,9 @@ static int usb_msd_handle_data(USBDevice *dev, USBPacket *p)
     default:
         DPRINTF("Bad token\n");
     fail:
-        ret = USB_RET_STALL;
+        p->status = USB_RET_STALL;
         break;
     }
-
-    return ret;
 }
 
 static void usb_msd_password_cb(void *opaque, int err)
diff --git a/hw/usb/dev-uas.c b/hw/usb/dev-uas.c
index 5a0057a36..a21b2ba62 100644
--- a/hw/usb/dev-uas.c
+++ b/hw/usb/dev-uas.c
@@ -256,10 +256,10 @@ static void usb_uas_send_status_bh(void *opaque)
 
     uas->status = NULL;
     usb_packet_copy(p, &st->status, st->length);
-    p->result = st->length;
     QTAILQ_REMOVE(&uas->results, st, next);
     g_free(st);
 
+    p->status = USB_RET_SUCCESS; /* Clear previous ASYNC status */
     usb_packet_complete(&uas->dev, p);
 }
 
@@ -349,6 +349,7 @@ static void usb_uas_complete_data_packet(UASRequest *req)
     p = req->data;
     req->data = NULL;
     req->data_async = false;
+    p->status = USB_RET_SUCCESS; /* Clear previous ASYNC status */
     usb_packet_complete(&req->uas->dev, p);
 }
 
@@ -357,16 +358,16 @@ static void usb_uas_copy_data(UASRequest *req)
     uint32_t length;
 
     length = MIN(req->buf_size - req->buf_off,
-                 req->data->iov.size - req->data->result);
+                 req->data->iov.size - req->data->actual_length);
     trace_usb_uas_xfer_data(req->uas->dev.addr, req->tag, length,
-                            req->data->result, req->data->iov.size,
+                            req->data->actual_length, req->data->iov.size,
                             req->buf_off, req->buf_size);
     usb_packet_copy(req->data, scsi_req_get_buf(req->req) + req->buf_off,
                     length);
     req->buf_off += length;
     req->data_off += length;
 
-    if (req->data->result == req->data->iov.size) {
+    if (req->data->actual_length == req->data->iov.size) {
         usb_uas_complete_data_packet(req);
     }
     if (req->buf_size && req->buf_off == req->buf_size) {
@@ -504,17 +505,17 @@ static void usb_uas_handle_reset(USBDevice *dev)
     }
 }
 
-static int usb_uas_handle_control(USBDevice *dev, USBPacket *p,
+static void usb_uas_handle_control(USBDevice *dev, USBPacket *p,
                int request, int value, int index, int length, uint8_t *data)
 {
     int ret;
 
     ret = usb_desc_handle_control(dev, p, request, value, index, length, data);
     if (ret >= 0) {
-        return ret;
+        return;
     }
     fprintf(stderr, "%s: unhandled control request\n", __func__);
-    return USB_RET_STALL;
+    p->status = USB_RET_STALL;
 }
 
 static void usb_uas_cancel_io(USBDevice *dev, USBPacket *p)
@@ -576,7 +577,6 @@ bad_target:
      */
     usb_uas_queue_response(uas, req->tag, UAS_RC_INVALID_INFO_UNIT, 0);
     g_free(req);
-    return;
 }
 
 static void usb_uas_task(UASDevice *uas, uas_ui *ui)
@@ -640,16 +640,15 @@ bad_target:
 
 incorrect_lun:
     usb_uas_queue_response(uas, tag, UAS_RC_INCORRECT_LUN, 0);
-    return;
 }
 
-static int usb_uas_handle_data(USBDevice *dev, USBPacket *p)
+static void usb_uas_handle_data(USBDevice *dev, USBPacket *p)
 {
     UASDevice *uas = DO_UPCAST(UASDevice, dev, dev);
     uas_ui ui;
     UASStatus *st;
     UASRequest *req;
-    int length, ret = 0;
+    int length;
 
     switch (p->ep->nr) {
     case UAS_PIPE_ID_COMMAND:
@@ -658,16 +657,14 @@ static int usb_uas_handle_data(USBDevice *dev, USBPacket *p)
         switch (ui.hdr.id) {
         case UAS_UI_COMMAND:
             usb_uas_command(uas, &ui);
-            ret = length;
             break;
         case UAS_UI_TASK_MGMT:
             usb_uas_task(uas, &ui);
-            ret = length;
             break;
         default:
             fprintf(stderr, "%s: unknown command ui: id 0x%x\n",
                     __func__, ui.hdr.id);
-            ret = USB_RET_STALL;
+            p->status = USB_RET_STALL;
             break;
         }
         break;
@@ -676,11 +673,10 @@ static int usb_uas_handle_data(USBDevice *dev, USBPacket *p)
         if (st == NULL) {
             assert(uas->status == NULL);
             uas->status = p;
-            ret = USB_RET_ASYNC;
+            p->status = USB_RET_ASYNC;
             break;
         }
         usb_packet_copy(p, &st->status, st->length);
-        ret = st->length;
         QTAILQ_REMOVE(&uas->results, st, next);
         g_free(st);
         break;
@@ -689,28 +685,26 @@ static int usb_uas_handle_data(USBDevice *dev, USBPacket *p)
         req = (p->ep->nr == UAS_PIPE_ID_DATA_IN) ? uas->datain : uas->dataout;
         if (req == NULL) {
             fprintf(stderr, "%s: no inflight request\n", __func__);
-            ret = USB_RET_STALL;
+            p->status = USB_RET_STALL;
             break;
         }
         scsi_req_ref(req->req);
         req->data = p;
         usb_uas_copy_data(req);
-        if (p->result == p->iov.size || req->complete) {
+        if (p->actual_length == p->iov.size || req->complete) {
             req->data = NULL;
-            ret = p->result;
         } else {
             req->data_async = true;
-            ret = USB_RET_ASYNC;
+            p->status = USB_RET_ASYNC;
         }
         scsi_req_unref(req->req);
         usb_uas_start_next_transfer(uas);
         break;
     default:
         fprintf(stderr, "%s: invalid endpoint %d\n", __func__, p->ep->nr);
-        ret = USB_RET_STALL;
+        p->status = USB_RET_STALL;
         break;
     }
-    return ret;
 }
 
 static void usb_uas_handle_destroy(USBDevice *dev)
diff --git a/hw/usb/dev-wacom.c b/hw/usb/dev-wacom.c
index ed9a5ee35..08b416daa 100644
--- a/hw/usb/dev-wacom.c
+++ b/hw/usb/dev-wacom.c
@@ -250,7 +250,7 @@ static void usb_wacom_handle_reset(USBDevice *dev)
     s->mode = WACOM_MODE_HID;
 }
 
-static int usb_wacom_handle_control(USBDevice *dev, USBPacket *p,
+static void usb_wacom_handle_control(USBDevice *dev, USBPacket *p,
                int request, int value, int index, int length, uint8_t *data)
 {
     USBWacomState *s = (USBWacomState *) dev;
@@ -258,10 +258,9 @@ static int usb_wacom_handle_control(USBDevice *dev, USBPacket *p,
 
     ret = usb_desc_handle_control(dev, p, request, value, index, length, data);
     if (ret >= 0) {
-        return ret;
+        return;
     }
 
-    ret = 0;
     switch (request) {
     case WACOM_SET_REPORT:
         if (s->mouse_grabbed) {
@@ -269,61 +268,58 @@ static int usb_wacom_handle_control(USBDevice *dev, USBPacket *p,
             s->mouse_grabbed = 0;
         }
         s->mode = data[0];
-        ret = 0;
         break;
     case WACOM_GET_REPORT:
         data[0] = 0;
         data[1] = s->mode;
-        ret = 2;
+        p->actual_length = 2;
         break;
     /* USB HID requests */
     case HID_GET_REPORT:
         if (s->mode == WACOM_MODE_HID)
-            ret = usb_mouse_poll(s, data, length);
+            p->actual_length = usb_mouse_poll(s, data, length);
         else if (s->mode == WACOM_MODE_WACOM)
-            ret = usb_wacom_poll(s, data, length);
+            p->actual_length = usb_wacom_poll(s, data, length);
         break;
     case HID_GET_IDLE:
-        ret = 1;
         data[0] = s->idle;
+        p->actual_length = 1;
         break;
     case HID_SET_IDLE:
         s->idle = (uint8_t) (value >> 8);
-        ret = 0;
         break;
     default:
-        ret = USB_RET_STALL;
+        p->status = USB_RET_STALL;
         break;
     }
-    return ret;
 }
 
-static int usb_wacom_handle_data(USBDevice *dev, USBPacket *p)
+static void usb_wacom_handle_data(USBDevice *dev, USBPacket *p)
 {
     USBWacomState *s = (USBWacomState *) dev;
     uint8_t buf[p->iov.size];
-    int ret = 0;
+    int len = 0;
 
     switch (p->pid) {
     case USB_TOKEN_IN:
         if (p->ep->nr == 1) {
-            if (!(s->changed || s->idle))
-                return USB_RET_NAK;
+            if (!(s->changed || s->idle)) {
+                p->status = USB_RET_NAK;
+                return;
+            }
             s->changed = 0;
             if (s->mode == WACOM_MODE_HID)
-                ret = usb_mouse_poll(s, buf, p->iov.size);
+                len = usb_mouse_poll(s, buf, p->iov.size);
             else if (s->mode == WACOM_MODE_WACOM)
-                ret = usb_wacom_poll(s, buf, p->iov.size);
-            usb_packet_copy(p, buf, ret);
+                len = usb_wacom_poll(s, buf, p->iov.size);
+            usb_packet_copy(p, buf, len);
             break;
         }
         /* Fall through.  */
     case USB_TOKEN_OUT:
     default:
-        ret = USB_RET_STALL;
-        break;
+        p->status = USB_RET_STALL;
     }
-    return ret;
 }
 
 static void usb_wacom_handle_destroy(USBDevice *dev)
diff --git a/hw/usb/hcd-ehci-pci.c b/hw/usb/hcd-ehci-pci.c
new file mode 100644
index 000000000..41dbb539f
--- /dev/null
+++ b/hw/usb/hcd-ehci-pci.c
@@ -0,0 +1,219 @@
+/*
+ * QEMU USB EHCI Emulation
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or(at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "hw/usb/hcd-ehci.h"
+#include "hw/pci.h"
+#include "range.h"
+
+typedef struct EHCIPCIState {
+    PCIDevice pcidev;
+    EHCIState ehci;
+} EHCIPCIState;
+
+typedef struct EHCIPCIInfo {
+    const char *name;
+    uint16_t vendor_id;
+    uint16_t device_id;
+    uint8_t  revision;
+} EHCIPCIInfo;
+
+static int usb_ehci_pci_initfn(PCIDevice *dev)
+{
+    EHCIPCIState *i = DO_UPCAST(EHCIPCIState, pcidev, dev);
+    EHCIState *s = &i->ehci;
+    uint8_t *pci_conf = dev->config;
+
+    pci_set_byte(&pci_conf[PCI_CLASS_PROG], 0x20);
+
+    /* capabilities pointer */
+    pci_set_byte(&pci_conf[PCI_CAPABILITY_LIST], 0x00);
+    /* pci_set_byte(&pci_conf[PCI_CAPABILITY_LIST], 0x50); */
+
+    pci_set_byte(&pci_conf[PCI_INTERRUPT_PIN], 4); /* interrupt pin D */
+    pci_set_byte(&pci_conf[PCI_MIN_GNT], 0);
+    pci_set_byte(&pci_conf[PCI_MAX_LAT], 0);
+
+    /* pci_conf[0x50] = 0x01; *//* power management caps */
+
+    pci_set_byte(&pci_conf[USB_SBRN], USB_RELEASE_2); /* release # (2.1.4) */
+    pci_set_byte(&pci_conf[0x61], 0x20);  /* frame length adjustment (2.1.5) */
+    pci_set_word(&pci_conf[0x62], 0x00);  /* port wake up capability (2.1.6) */
+
+    pci_conf[0x64] = 0x00;
+    pci_conf[0x65] = 0x00;
+    pci_conf[0x66] = 0x00;
+    pci_conf[0x67] = 0x00;
+    pci_conf[0x68] = 0x01;
+    pci_conf[0x69] = 0x00;
+    pci_conf[0x6a] = 0x00;
+    pci_conf[0x6b] = 0x00;  /* USBLEGSUP */
+    pci_conf[0x6c] = 0x00;
+    pci_conf[0x6d] = 0x00;
+    pci_conf[0x6e] = 0x00;
+    pci_conf[0x6f] = 0xc0;  /* USBLEFCTLSTS */
+
+    s->caps[0x09] = 0x68;        /* EECP */
+
+    s->irq = dev->irq[3];
+    s->dma = pci_dma_context(dev);
+
+    s->capsbase = 0x00;
+    s->opregbase = 0x20;
+
+    usb_ehci_initfn(s, DEVICE(dev));
+    pci_register_bar(dev, 0, PCI_BASE_ADDRESS_SPACE_MEMORY, &s->mem);
+
+    return 0;
+}
+
+static void usb_ehci_pci_write_config(PCIDevice *dev, uint32_t addr,
+                                      uint32_t val, int l)
+{
+    EHCIPCIState *i = DO_UPCAST(EHCIPCIState, pcidev, dev);
+    bool busmaster;
+
+    pci_default_write_config(dev, addr, val, l);
+
+    if (!range_covers_byte(addr, l, PCI_COMMAND)) {
+        return;
+    }
+    busmaster = pci_get_word(dev->config + PCI_COMMAND) & PCI_COMMAND_MASTER;
+    i->ehci.dma = busmaster ? pci_dma_context(dev) : NULL;
+}
+
+static Property ehci_pci_properties[] = {
+    DEFINE_PROP_UINT32("maxframes", EHCIPCIState, ehci.maxframes, 128),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static const VMStateDescription vmstate_ehci_pci = {
+    .name        = "ehci",
+    .version_id  = 2,
+    .minimum_version_id  = 1,
+    .fields      = (VMStateField[]) {
+        VMSTATE_PCI_DEVICE(pcidev, EHCIPCIState),
+        VMSTATE_STRUCT(ehci, EHCIPCIState, 2, vmstate_ehci, EHCIState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static void ehci_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
+    EHCIPCIInfo *i = data;
+
+    k->init = usb_ehci_pci_initfn;
+    k->vendor_id = i->vendor_id;
+    k->device_id = i->device_id;
+    k->revision = i->revision;
+    k->class_id = PCI_CLASS_SERIAL_USB;
+    k->config_write = usb_ehci_pci_write_config;
+    k->no_hotplug = 1;
+    dc->vmsd = &vmstate_ehci_pci;
+    dc->props = ehci_pci_properties;
+}
+
+static struct EHCIPCIInfo ehci_pci_info[] = {
+    {
+        .name      = "usb-ehci",
+        .vendor_id = PCI_VENDOR_ID_INTEL,
+        .device_id = PCI_DEVICE_ID_INTEL_82801D, /* ich4 */
+        .revision  = 0x10,
+    },{
+        .name      = "ich9-usb-ehci1", /* 00:1d.7 */
+        .vendor_id = PCI_VENDOR_ID_INTEL,
+        .device_id = PCI_DEVICE_ID_INTEL_82801I_EHCI1,
+        .revision  = 0x03,
+    },{
+        .name      = "ich9-usb-ehci2", /* 00:1a.7 */
+        .vendor_id = PCI_VENDOR_ID_INTEL,
+        .device_id = PCI_DEVICE_ID_INTEL_82801I_EHCI2,
+        .revision  = 0x03,
+    }
+};
+
+static void ehci_pci_register_types(void)
+{
+    TypeInfo ehci_type_info = {
+        .parent        = TYPE_PCI_DEVICE,
+        .instance_size = sizeof(EHCIPCIState),
+        .class_init    = ehci_class_init,
+    };
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(ehci_pci_info); i++) {
+        ehci_type_info.name = ehci_pci_info[i].name;
+        ehci_type_info.class_data = ehci_pci_info + i;
+        type_register(&ehci_type_info);
+    }
+}
+
+type_init(ehci_pci_register_types)
+
+struct ehci_companions {
+    const char *name;
+    int func;
+    int port;
+};
+
+static const struct ehci_companions ich9_1d[] = {
+    { .name = "ich9-usb-uhci1", .func = 0, .port = 0 },
+    { .name = "ich9-usb-uhci2", .func = 1, .port = 2 },
+    { .name = "ich9-usb-uhci3", .func = 2, .port = 4 },
+};
+
+static const struct ehci_companions ich9_1a[] = {
+    { .name = "ich9-usb-uhci4", .func = 0, .port = 0 },
+    { .name = "ich9-usb-uhci5", .func = 1, .port = 2 },
+    { .name = "ich9-usb-uhci6", .func = 2, .port = 4 },
+};
+
+int ehci_create_ich9_with_companions(PCIBus *bus, int slot)
+{
+    const struct ehci_companions *comp;
+    PCIDevice *ehci, *uhci;
+    BusState *usbbus;
+    const char *name;
+    int i;
+
+    switch (slot) {
+    case 0x1d:
+        name = "ich9-usb-ehci1";
+        comp = ich9_1d;
+        break;
+    case 0x1a:
+        name = "ich9-usb-ehci2";
+        comp = ich9_1a;
+        break;
+    default:
+        return -1;
+    }
+
+    ehci = pci_create_multifunction(bus, PCI_DEVFN(slot, 7), true, name);
+    qdev_init_nofail(&ehci->qdev);
+    usbbus = QLIST_FIRST(&ehci->qdev.child_bus);
+
+    for (i = 0; i < 3; i++) {
+        uhci = pci_create_multifunction(bus, PCI_DEVFN(slot, comp[i].func),
+                                        true, comp[i].name);
+        qdev_prop_set_string(&uhci->qdev, "masterbus", usbbus->name);
+        qdev_prop_set_uint32(&uhci->qdev, "firstport", comp[i].port);
+        qdev_init_nofail(&uhci->qdev);
+    }
+    return 0;
+}
diff --git a/hw/usb/hcd-ehci-sysbus.c b/hw/usb/hcd-ehci-sysbus.c
new file mode 100644
index 000000000..803df92f3
--- /dev/null
+++ b/hw/usb/hcd-ehci-sysbus.c
@@ -0,0 +1,78 @@
+/*
+ * QEMU USB EHCI Emulation
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or(at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "hw/usb/hcd-ehci.h"
+#include "hw/sysbus.h"
+
+typedef struct EHCISysBusState {
+    SysBusDevice busdev;
+    EHCIState ehci;
+} EHCISysBusState;
+
+static const VMStateDescription vmstate_ehci_sysbus = {
+    .name        = "ehci-sysbus",
+    .version_id  = 2,
+    .minimum_version_id  = 1,
+    .fields      = (VMStateField[]) {
+        VMSTATE_STRUCT(ehci, EHCISysBusState, 2, vmstate_ehci, EHCIState),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static Property ehci_sysbus_properties[] = {
+    DEFINE_PROP_UINT32("maxframes", EHCISysBusState, ehci.maxframes, 128),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static int usb_ehci_sysbus_initfn(SysBusDevice *dev)
+{
+    EHCISysBusState *i = FROM_SYSBUS(EHCISysBusState, dev);
+    EHCIState *s = &i->ehci;
+
+    s->capsbase = 0x100;
+    s->opregbase = 0x140;
+    s->dma = &dma_context_memory;
+
+    usb_ehci_initfn(s, DEVICE(dev));
+    sysbus_init_irq(dev, &s->irq);
+    sysbus_init_mmio(dev, &s->mem);
+    return 0;
+}
+
+static void ehci_sysbus_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
+
+    k->init = usb_ehci_sysbus_initfn;
+    dc->vmsd = &vmstate_ehci_sysbus;
+    dc->props = ehci_sysbus_properties;
+}
+
+TypeInfo ehci_xlnx_type_info = {
+    .name          = "xlnx,ps7-usb",
+    .parent        = TYPE_SYS_BUS_DEVICE,
+    .instance_size = sizeof(EHCISysBusState),
+    .class_init    = ehci_sysbus_class_init,
+};
+
+static void ehci_sysbus_register_types(void)
+{
+    type_register_static(&ehci_xlnx_type_info);
+}
+
+type_init(ehci_sysbus_register_types)
diff --git a/hw/usb/hcd-ehci.c b/hw/usb/hcd-ehci.c
index 017342b56..7df8e21ec 100644
--- a/hw/usb/hcd-ehci.c
+++ b/hw/usb/hcd-ehci.c
@@ -2,6 +2,11 @@
  * QEMU USB EHCI Emulation
  *
  * Copyright(c) 2008  Emutex Ltd. (address@hidden)
+ * Copyright(c) 2011-2012 Red Hat, Inc.
+ *
+ * Red Hat Authors:
+ * Gerd Hoffmann <kraxel@redhat.com>
+ * Hans de Goede <hdegoede@redhat.com>
  *
  * EHCI project was started by Mark Burkley, with contributions by
  * Niels de Vos.  David S. Ahern continued working on it.  Kevin Wolf,
@@ -22,40 +27,18 @@
  * along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "hw/hw.h"
-#include "qemu-timer.h"
-#include "hw/usb.h"
-#include "hw/pci.h"
-#include "monitor.h"
-#include "trace.h"
-#include "dma.h"
-
-#define EHCI_DEBUG   0
-
-#if EHCI_DEBUG
-#define DPRINTF printf
-#else
-#define DPRINTF(...)
-#endif
-
-/* internal processing - reset HC to try and recover */
-#define USB_RET_PROCERR   (-99)
-
-#define MMIO_SIZE        0x1000
+#include "hw/usb/hcd-ehci.h"
 
 /* Capability Registers Base Address - section 2.2 */
-#define CAPREGBASE       0x0000
-#define CAPLENGTH        CAPREGBASE + 0x0000  // 1-byte, 0x0001 reserved
-#define HCIVERSION       CAPREGBASE + 0x0002  // 2-bytes, i/f version #
-#define HCSPARAMS        CAPREGBASE + 0x0004  // 4-bytes, structural params
-#define HCCPARAMS        CAPREGBASE + 0x0008  // 4-bytes, capability params
+#define CAPLENGTH        0x0000  /* 1-byte, 0x0001 reserved */
+#define HCIVERSION       0x0002  /* 2-bytes, i/f version # */
+#define HCSPARAMS        0x0004  /* 4-bytes, structural params */
+#define HCCPARAMS        0x0008  /* 4-bytes, capability params */
 #define EECP             HCCPARAMS + 1
-#define HCSPPORTROUTE1   CAPREGBASE + 0x000c
-#define HCSPPORTROUTE2   CAPREGBASE + 0x0010
+#define HCSPPORTROUTE1   0x000c
+#define HCSPPORTROUTE2   0x0010
 
-#define OPREGBASE        0x0020        // Operational Registers Base Address
-
-#define USBCMD           OPREGBASE + 0x0000
+#define USBCMD           0x0000
 #define USBCMD_RUNSTOP   (1 << 0)      // run / Stop
 #define USBCMD_HCRESET   (1 << 1)      // HC Reset
 #define USBCMD_FLS       (3 << 2)      // Frame List Size
@@ -69,7 +52,7 @@
 #define USBCMD_ITC       (0x7f << 16)  // Int Threshold Control
 #define USBCMD_ITC_SH    16            // Int Threshold Control Shift
 
-#define USBSTS           OPREGBASE + 0x0004
+#define USBSTS           0x0004
 #define USBSTS_RO_MASK   0x0000003f
 #define USBSTS_INT       (1 << 0)      // USB Interrupt
 #define USBSTS_ERRINT    (1 << 1)      // Error Interrupt
@@ -86,20 +69,17 @@
  *  Interrupt enable bits correspond to the interrupt active bits in USBSTS
  *  so no need to redefine here.
  */
-#define USBINTR              OPREGBASE + 0x0008
+#define USBINTR              0x0008
 #define USBINTR_MASK         0x0000003f
 
-#define FRINDEX              OPREGBASE + 0x000c
-#define CTRLDSSEGMENT        OPREGBASE + 0x0010
-#define PERIODICLISTBASE     OPREGBASE + 0x0014
-#define ASYNCLISTADDR        OPREGBASE + 0x0018
+#define FRINDEX              0x000c
+#define CTRLDSSEGMENT        0x0010
+#define PERIODICLISTBASE     0x0014
+#define ASYNCLISTADDR        0x0018
 #define ASYNCLISTADDR_MASK   0xffffffe0
 
-#define CONFIGFLAG           OPREGBASE + 0x0040
+#define CONFIGFLAG           0x0040
 
-#define PORTSC               (OPREGBASE + 0x0044)
-#define PORTSC_BEGIN         PORTSC
-#define PORTSC_END           (PORTSC + 4 * NB_PORTS)
 /*
  * Bits that are reserved or are read-only are masked out of values
  * written to us by software
@@ -131,9 +111,9 @@
 #define FRAME_TIMER_NS   (1000000000 / FRAME_TIMER_FREQ)
 
 #define NB_MAXINTRATE    8        // Max rate at which controller issues ints
-#define NB_PORTS         6        // Number of downstream ports
 #define BUFF_SIZE        5*4096   // Max bytes to transfer per transaction
 #define MAX_QH           100      // Max allowable queue heads in a chain
+#define MIN_FR_PER_TICK  3        // Min frames to process when catching up
 
 /*  Internal periodic / asynchronous schedule state machine states
  */
@@ -167,274 +147,6 @@ typedef enum {
 #define NLPTR_TYPE_STITD         2     // split xaction, isoc xfer descriptor
 #define NLPTR_TYPE_FSTN          3     // frame span traversal node
 
-
-/*  EHCI spec version 1.0 Section 3.3
- */
-typedef struct EHCIitd {
-    uint32_t next;
-
-    uint32_t transact[8];
-#define ITD_XACT_ACTIVE          (1 << 31)
-#define ITD_XACT_DBERROR         (1 << 30)
-#define ITD_XACT_BABBLE          (1 << 29)
-#define ITD_XACT_XACTERR         (1 << 28)
-#define ITD_XACT_LENGTH_MASK     0x0fff0000
-#define ITD_XACT_LENGTH_SH       16
-#define ITD_XACT_IOC             (1 << 15)
-#define ITD_XACT_PGSEL_MASK      0x00007000
-#define ITD_XACT_PGSEL_SH        12
-#define ITD_XACT_OFFSET_MASK     0x00000fff
-
-    uint32_t bufptr[7];
-#define ITD_BUFPTR_MASK          0xfffff000
-#define ITD_BUFPTR_SH            12
-#define ITD_BUFPTR_EP_MASK       0x00000f00
-#define ITD_BUFPTR_EP_SH         8
-#define ITD_BUFPTR_DEVADDR_MASK  0x0000007f
-#define ITD_BUFPTR_DEVADDR_SH    0
-#define ITD_BUFPTR_DIRECTION     (1 << 11)
-#define ITD_BUFPTR_MAXPKT_MASK   0x000007ff
-#define ITD_BUFPTR_MAXPKT_SH     0
-#define ITD_BUFPTR_MULT_MASK     0x00000003
-#define ITD_BUFPTR_MULT_SH       0
-} EHCIitd;
-
-/*  EHCI spec version 1.0 Section 3.4
- */
-typedef struct EHCIsitd {
-    uint32_t next;                  // Standard next link pointer
-    uint32_t epchar;
-#define SITD_EPCHAR_IO              (1 << 31)
-#define SITD_EPCHAR_PORTNUM_MASK    0x7f000000
-#define SITD_EPCHAR_PORTNUM_SH      24
-#define SITD_EPCHAR_HUBADD_MASK     0x007f0000
-#define SITD_EPCHAR_HUBADDR_SH      16
-#define SITD_EPCHAR_EPNUM_MASK      0x00000f00
-#define SITD_EPCHAR_EPNUM_SH        8
-#define SITD_EPCHAR_DEVADDR_MASK    0x0000007f
-
-    uint32_t uframe;
-#define SITD_UFRAME_CMASK_MASK      0x0000ff00
-#define SITD_UFRAME_CMASK_SH        8
-#define SITD_UFRAME_SMASK_MASK      0x000000ff
-
-    uint32_t results;
-#define SITD_RESULTS_IOC              (1 << 31)
-#define SITD_RESULTS_PGSEL            (1 << 30)
-#define SITD_RESULTS_TBYTES_MASK      0x03ff0000
-#define SITD_RESULTS_TYBYTES_SH       16
-#define SITD_RESULTS_CPROGMASK_MASK   0x0000ff00
-#define SITD_RESULTS_CPROGMASK_SH     8
-#define SITD_RESULTS_ACTIVE           (1 << 7)
-#define SITD_RESULTS_ERR              (1 << 6)
-#define SITD_RESULTS_DBERR            (1 << 5)
-#define SITD_RESULTS_BABBLE           (1 << 4)
-#define SITD_RESULTS_XACTERR          (1 << 3)
-#define SITD_RESULTS_MISSEDUF         (1 << 2)
-#define SITD_RESULTS_SPLITXSTATE      (1 << 1)
-
-    uint32_t bufptr[2];
-#define SITD_BUFPTR_MASK              0xfffff000
-#define SITD_BUFPTR_CURROFF_MASK      0x00000fff
-#define SITD_BUFPTR_TPOS_MASK         0x00000018
-#define SITD_BUFPTR_TPOS_SH           3
-#define SITD_BUFPTR_TCNT_MASK         0x00000007
-
-    uint32_t backptr;                 // Standard next link pointer
-} EHCIsitd;
-
-/*  EHCI spec version 1.0 Section 3.5
- */
-typedef struct EHCIqtd {
-    uint32_t next;                    // Standard next link pointer
-    uint32_t altnext;                 // Standard next link pointer
-    uint32_t token;
-#define QTD_TOKEN_DTOGGLE             (1 << 31)
-#define QTD_TOKEN_TBYTES_MASK         0x7fff0000
-#define QTD_TOKEN_TBYTES_SH           16
-#define QTD_TOKEN_IOC                 (1 << 15)
-#define QTD_TOKEN_CPAGE_MASK          0x00007000
-#define QTD_TOKEN_CPAGE_SH            12
-#define QTD_TOKEN_CERR_MASK           0x00000c00
-#define QTD_TOKEN_CERR_SH             10
-#define QTD_TOKEN_PID_MASK            0x00000300
-#define QTD_TOKEN_PID_SH              8
-#define QTD_TOKEN_ACTIVE              (1 << 7)
-#define QTD_TOKEN_HALT                (1 << 6)
-#define QTD_TOKEN_DBERR               (1 << 5)
-#define QTD_TOKEN_BABBLE              (1 << 4)
-#define QTD_TOKEN_XACTERR             (1 << 3)
-#define QTD_TOKEN_MISSEDUF            (1 << 2)
-#define QTD_TOKEN_SPLITXSTATE         (1 << 1)
-#define QTD_TOKEN_PING                (1 << 0)
-
-    uint32_t bufptr[5];               // Standard buffer pointer
-#define QTD_BUFPTR_MASK               0xfffff000
-#define QTD_BUFPTR_SH                 12
-} EHCIqtd;
-
-/*  EHCI spec version 1.0 Section 3.6
- */
-typedef struct EHCIqh {
-    uint32_t next;                    // Standard next link pointer
-
-    /* endpoint characteristics */
-    uint32_t epchar;
-#define QH_EPCHAR_RL_MASK             0xf0000000
-#define QH_EPCHAR_RL_SH               28
-#define QH_EPCHAR_C                   (1 << 27)
-#define QH_EPCHAR_MPLEN_MASK          0x07FF0000
-#define QH_EPCHAR_MPLEN_SH            16
-#define QH_EPCHAR_H                   (1 << 15)
-#define QH_EPCHAR_DTC                 (1 << 14)
-#define QH_EPCHAR_EPS_MASK            0x00003000
-#define QH_EPCHAR_EPS_SH              12
-#define EHCI_QH_EPS_FULL              0
-#define EHCI_QH_EPS_LOW               1
-#define EHCI_QH_EPS_HIGH              2
-#define EHCI_QH_EPS_RESERVED          3
-
-#define QH_EPCHAR_EP_MASK             0x00000f00
-#define QH_EPCHAR_EP_SH               8
-#define QH_EPCHAR_I                   (1 << 7)
-#define QH_EPCHAR_DEVADDR_MASK        0x0000007f
-#define QH_EPCHAR_DEVADDR_SH          0
-
-    /* endpoint capabilities */
-    uint32_t epcap;
-#define QH_EPCAP_MULT_MASK            0xc0000000
-#define QH_EPCAP_MULT_SH              30
-#define QH_EPCAP_PORTNUM_MASK         0x3f800000
-#define QH_EPCAP_PORTNUM_SH           23
-#define QH_EPCAP_HUBADDR_MASK         0x007f0000
-#define QH_EPCAP_HUBADDR_SH           16
-#define QH_EPCAP_CMASK_MASK           0x0000ff00
-#define QH_EPCAP_CMASK_SH             8
-#define QH_EPCAP_SMASK_MASK           0x000000ff
-#define QH_EPCAP_SMASK_SH             0
-
-    uint32_t current_qtd;             // Standard next link pointer
-    uint32_t next_qtd;                // Standard next link pointer
-    uint32_t altnext_qtd;
-#define QH_ALTNEXT_NAKCNT_MASK        0x0000001e
-#define QH_ALTNEXT_NAKCNT_SH          1
-
-    uint32_t token;                   // Same as QTD token
-    uint32_t bufptr[5];               // Standard buffer pointer
-#define BUFPTR_CPROGMASK_MASK         0x000000ff
-#define BUFPTR_FRAMETAG_MASK          0x0000001f
-#define BUFPTR_SBYTES_MASK            0x00000fe0
-#define BUFPTR_SBYTES_SH              5
-} EHCIqh;
-
-/*  EHCI spec version 1.0 Section 3.7
- */
-typedef struct EHCIfstn {
-    uint32_t next;                    // Standard next link pointer
-    uint32_t backptr;                 // Standard next link pointer
-} EHCIfstn;
-
-typedef struct EHCIPacket EHCIPacket;
-typedef struct EHCIQueue EHCIQueue;
-typedef struct EHCIState EHCIState;
-
-enum async_state {
-    EHCI_ASYNC_NONE = 0,
-    EHCI_ASYNC_INFLIGHT,
-    EHCI_ASYNC_FINISHED,
-};
-
-struct EHCIPacket {
-    EHCIQueue *queue;
-    QTAILQ_ENTRY(EHCIPacket) next;
-
-    EHCIqtd qtd;           /* copy of current QTD (being worked on) */
-    uint32_t qtdaddr;      /* address QTD read from                 */
-
-    USBPacket packet;
-    QEMUSGList sgl;
-    int pid;
-    uint32_t tbytes;
-    enum async_state async;
-    int usb_status;
-};
-
-struct EHCIQueue {
-    EHCIState *ehci;
-    QTAILQ_ENTRY(EHCIQueue) next;
-    uint32_t seen;
-    uint64_t ts;
-    int async;
-    int revalidate;
-
-    /* cached data from guest - needs to be flushed
-     * when guest removes an entry (doorbell, handshake sequence)
-     */
-    EHCIqh qh;             /* copy of current QH (being worked on) */
-    uint32_t qhaddr;       /* address QH read from                 */
-    uint32_t qtdaddr;      /* address QTD read from                */
-    USBDevice *dev;
-    QTAILQ_HEAD(, EHCIPacket) packets;
-};
-
-typedef QTAILQ_HEAD(EHCIQueueHead, EHCIQueue) EHCIQueueHead;
-
-struct EHCIState {
-    PCIDevice dev;
-    USBBus bus;
-    qemu_irq irq;
-    MemoryRegion mem;
-    int companion_count;
-
-    /* properties */
-    uint32_t maxframes;
-
-    /*
-     *  EHCI spec version 1.0 Section 2.3
-     *  Host Controller Operational Registers
-     */
-    union {
-        uint8_t mmio[MMIO_SIZE];
-        struct {
-            uint8_t cap[OPREGBASE];
-            uint32_t usbcmd;
-            uint32_t usbsts;
-            uint32_t usbintr;
-            uint32_t frindex;
-            uint32_t ctrldssegment;
-            uint32_t periodiclistbase;
-            uint32_t asynclistaddr;
-            uint32_t notused[9];
-            uint32_t configflag;
-            uint32_t portsc[NB_PORTS];
-        };
-    };
-
-    /*
-     *  Internal states, shadow registers, etc
-     */
-    QEMUTimer *frame_timer;
-    QEMUBH *async_bh;
-    uint32_t astate;         /* Current state in asynchronous schedule */
-    uint32_t pstate;         /* Current state in periodic schedule     */
-    USBPort ports[NB_PORTS];
-    USBPort *companion_ports[NB_PORTS];
-    uint32_t usbsts_pending;
-    uint32_t usbsts_frindex;
-    EHCIQueueHead aqueues;
-    EHCIQueueHead pqueues;
-
-    /* which address to look at next */
-    uint32_t a_fetch_addr;
-    uint32_t p_fetch_addr;
-
-    USBPacket ipacket;
-    QEMUSGList isgl;
-
-    uint64_t last_run_ns;
-    uint32_t async_stepdown;
-};
-
 #define SET_LAST_RUN_CLOCK(s) \
     (s)->last_run_ns = qemu_get_clock_ns(vm_clock);
 
@@ -466,25 +178,20 @@ static const char *ehci_state_names[] = {
 };
 
 static const char *ehci_mmio_names[] = {
-    [CAPLENGTH]         = "CAPLENGTH",
-    [HCIVERSION]        = "HCIVERSION",
-    [HCSPARAMS]         = "HCSPARAMS",
-    [HCCPARAMS]         = "HCCPARAMS",
     [USBCMD]            = "USBCMD",
     [USBSTS]            = "USBSTS",
     [USBINTR]           = "USBINTR",
     [FRINDEX]           = "FRINDEX",
     [PERIODICLISTBASE]  = "P-LIST BASE",
     [ASYNCLISTADDR]     = "A-LIST ADDR",
-    [PORTSC_BEGIN]      = "PORTSC #0",
-    [PORTSC_BEGIN + 4]  = "PORTSC #1",
-    [PORTSC_BEGIN + 8]  = "PORTSC #2",
-    [PORTSC_BEGIN + 12] = "PORTSC #3",
-    [PORTSC_BEGIN + 16] = "PORTSC #4",
-    [PORTSC_BEGIN + 20] = "PORTSC #5",
     [CONFIGFLAG]        = "CONFIGFLAG",
 };
 
+static int ehci_state_executing(EHCIQueue *q);
+static int ehci_state_writeback(EHCIQueue *q);
+static int ehci_state_advqueue(EHCIQueue *q);
+static int ehci_fill_queue(EHCIPacket *p);
+
 static const char *nr2str(const char **n, size_t len, uint32_t nr)
 {
     if (nr < len && n[nr] != NULL) {
@@ -499,7 +206,7 @@ static const char *state2str(uint32_t state)
     return nr2str(ehci_state_names, ARRAY_SIZE(ehci_state_names), state);
 }
 
-static const char *addr2str(target_phys_addr_t addr)
+static const char *addr2str(hwaddr addr)
 {
     return nr2str(ehci_mmio_names, ARRAY_SIZE(ehci_mmio_names), addr);
 }
@@ -658,7 +365,7 @@ static int ehci_get_fetch_addr(EHCIState *s, int async)
     return async ? s->a_fetch_addr : s->p_fetch_addr;
 }
 
-static void ehci_trace_qh(EHCIQueue *q, target_phys_addr_t addr, EHCIqh *qh)
+static void ehci_trace_qh(EHCIQueue *q, hwaddr addr, EHCIqh *qh)
 {
     /* need three here due to argument count limits */
     trace_usb_ehci_qh_ptrs(q, addr, qh->next,
@@ -676,7 +383,7 @@ static void ehci_trace_qh(EHCIQueue *q, target_phys_addr_t addr, EHCIqh *qh)
                            (bool)(qh->epchar & QH_EPCHAR_I));
 }
 
-static void ehci_trace_qtd(EHCIQueue *q, target_phys_addr_t addr, EHCIqtd *qtd)
+static void ehci_trace_qtd(EHCIQueue *q, hwaddr addr, EHCIqtd *qtd)
 {
     /* need three here due to argument count limits */
     trace_usb_ehci_qtd_ptrs(q, addr, qtd->next, qtd->altnext);
@@ -693,7 +400,7 @@ static void ehci_trace_qtd(EHCIQueue *q, target_phys_addr_t addr, EHCIqtd *qtd)
                             (bool)(qtd->token & QTD_TOKEN_XACTERR));
 }
 
-static void ehci_trace_itd(EHCIState *s, target_phys_addr_t addr, EHCIitd *itd)
+static void ehci_trace_itd(EHCIState *s, hwaddr addr, EHCIitd *itd)
 {
     trace_usb_ehci_itd(addr, itd->next,
                        get_field(itd->bufptr[1], ITD_BUFPTR_MAXPKT),
@@ -702,13 +409,19 @@ static void ehci_trace_itd(EHCIState *s, target_phys_addr_t addr, EHCIitd *itd)
                        get_field(itd->bufptr[0], ITD_BUFPTR_DEVADDR));
 }
 
-static void ehci_trace_sitd(EHCIState *s, target_phys_addr_t addr,
+static void ehci_trace_sitd(EHCIState *s, hwaddr addr,
                             EHCIsitd *sitd)
 {
     trace_usb_ehci_sitd(addr, sitd->next,
                         (bool)(sitd->results & SITD_RESULTS_ACTIVE));
 }
 
+static void ehci_trace_guest_bug(EHCIState *s, const char *message)
+{
+    trace_usb_ehci_guest_bug(message);
+    fprintf(stderr, "ehci warning: %s\n", message);
+}
+
 static inline bool ehci_enabled(EHCIState *s)
 {
     return s->usbcmd & USBCMD_RUNSTOP;
@@ -740,9 +453,29 @@ static EHCIPacket *ehci_alloc_packet(EHCIQueue *q)
 
 static void ehci_free_packet(EHCIPacket *p)
 {
+    if (p->async == EHCI_ASYNC_FINISHED) {
+        EHCIQueue *q = p->queue;
+        int state = ehci_get_state(q->ehci, q->async);
+        /* This is a normal, but rare condition (cancel racing completion) */
+        fprintf(stderr, "EHCI: Warning packet completed but not processed\n");
+        ehci_state_executing(q);
+        ehci_state_writeback(q);
+        if (!(q->qh.token & QTD_TOKEN_HALT)) {
+            ehci_state_advqueue(q);
+        }
+        ehci_set_state(q->ehci, q->async, state);
+        /* state_writeback recurses into us with async == EHCI_ASYNC_NONE!! */
+        return;
+    }
     trace_usb_ehci_packet_action(p->queue, p, "free");
+    if (p->async == EHCI_ASYNC_INITIALIZED) {
+        usb_packet_unmap(&p->packet, &p->sgl);
+        qemu_sglist_destroy(&p->sgl);
+    }
     if (p->async == EHCI_ASYNC_INFLIGHT) {
         usb_cancel_packet(&p->packet);
+        usb_packet_unmap(&p->packet, &p->sgl);
+        qemu_sglist_destroy(&p->sgl);
     }
     QTAILQ_REMOVE(&p->queue->packets, p, next);
     usb_packet_cleanup(&p->packet);
@@ -766,27 +499,45 @@ static EHCIQueue *ehci_alloc_queue(EHCIState *ehci, uint32_t addr, int async)
     return q;
 }
 
-static void ehci_cancel_queue(EHCIQueue *q)
+static int ehci_cancel_queue(EHCIQueue *q)
 {
     EHCIPacket *p;
+    int packets = 0;
 
     p = QTAILQ_FIRST(&q->packets);
     if (p == NULL) {
-        return;
+        return 0;
     }
 
     trace_usb_ehci_queue_action(q, "cancel");
     do {
         ehci_free_packet(p);
+        packets++;
     } while ((p = QTAILQ_FIRST(&q->packets)) != NULL);
+    return packets;
+}
+
+static int ehci_reset_queue(EHCIQueue *q)
+{
+    int packets;
+
+    trace_usb_ehci_queue_action(q, "reset");
+    packets = ehci_cancel_queue(q);
+    q->dev = NULL;
+    q->qtdaddr = 0;
+    return packets;
 }
 
-static void ehci_free_queue(EHCIQueue *q)
+static void ehci_free_queue(EHCIQueue *q, const char *warn)
 {
     EHCIQueueHead *head = q->async ? &q->ehci->aqueues : &q->ehci->pqueues;
+    int cancelled;
 
     trace_usb_ehci_queue_action(q, "free");
-    ehci_cancel_queue(q);
+    cancelled = ehci_cancel_queue(q);
+    if (warn && cancelled > 0) {
+        ehci_trace_guest_bug(q->ehci, warn);
+    }
     QTAILQ_REMOVE(head, q, next);
     g_free(q);
 }
@@ -805,20 +556,10 @@ static EHCIQueue *ehci_find_queue_by_qh(EHCIState *ehci, uint32_t addr,
     return NULL;
 }
 
-static void ehci_queues_tag_unused_async(EHCIState *ehci)
-{
-    EHCIQueue *q;
-
-    QTAILQ_FOREACH(q, &ehci->aqueues, next) {
-        if (!q->seen) {
-            q->revalidate = 1;
-        }
-    }
-}
-
 static void ehci_queues_rip_unused(EHCIState *ehci, int async)
 {
     EHCIQueueHead *head = async ? &ehci->aqueues : &ehci->pqueues;
+    const char *warn = async ? "guest unlinked busy QH" : NULL;
     uint64_t maxage = FRAME_TIMER_NS * ehci->maxframes * 4;
     EHCIQueue *q, *tmp;
 
@@ -831,7 +572,19 @@ static void ehci_queues_rip_unused(EHCIState *ehci, int async)
         if (ehci->last_run_ns < q->ts + maxage) {
             continue;
         }
-        ehci_free_queue(q);
+        ehci_free_queue(q, warn);
+    }
+}
+
+static void ehci_queues_rip_unseen(EHCIState *ehci, int async)
+{
+    EHCIQueueHead *head = async ? &ehci->aqueues : &ehci->pqueues;
+    EHCIQueue *q, *tmp;
+
+    QTAILQ_FOREACH_SAFE(q, head, next, tmp) {
+        if (!q->seen) {
+            ehci_free_queue(q, NULL);
+        }
     }
 }
 
@@ -844,17 +597,18 @@ static void ehci_queues_rip_device(EHCIState *ehci, USBDevice *dev, int async)
         if (q->dev != dev) {
             continue;
         }
-        ehci_free_queue(q);
+        ehci_free_queue(q, NULL);
     }
 }
 
 static void ehci_queues_rip_all(EHCIState *ehci, int async)
 {
     EHCIQueueHead *head = async ? &ehci->aqueues : &ehci->pqueues;
+    const char *warn = async ? "guest stopped busy async schedule" : NULL;
     EHCIQueue *q, *tmp;
 
     QTAILQ_FOREACH_SAFE(q, head, next, tmp) {
-        ehci_free_queue(q);
+        ehci_free_queue(q, warn);
     }
 }
 
@@ -979,7 +733,7 @@ static int ehci_register_companion(USBBus *bus, USBPort *ports[],
     }
 
     s->companion_count++;
-    s->mmio[0x05] = (s->companion_count << 4) | portcount;
+    s->caps[0x05] = (s->companion_count << 4) | portcount;
 
     return 0;
 }
@@ -1024,7 +778,8 @@ static void ehci_reset(void *opaque)
         }
     }
 
-    memset(&s->mmio[OPREGBASE], 0x00, MMIO_SIZE - OPREGBASE);
+    memset(&s->opreg, 0x00, sizeof(s->opreg));
+    memset(&s->portsc, 0x00, sizeof(s->portsc));
 
     s->usbcmd = NB_MAXINTRATE << USBCMD_ITC_SH;
     s->usbsts = USBSTS_HALT;
@@ -1051,50 +806,35 @@ static void ehci_reset(void *opaque)
     qemu_bh_cancel(s->async_bh);
 }
 
-static uint32_t ehci_mem_readb(void *ptr, target_phys_addr_t addr)
+static uint64_t ehci_caps_read(void *ptr, hwaddr addr,
+                               unsigned size)
 {
     EHCIState *s = ptr;
-    uint32_t val;
-
-    val = s->mmio[addr];
-
-    return val;
+    return s->caps[addr];
 }
 
-static uint32_t ehci_mem_readw(void *ptr, target_phys_addr_t addr)
+static uint64_t ehci_opreg_read(void *ptr, hwaddr addr,
+                                unsigned size)
 {
     EHCIState *s = ptr;
     uint32_t val;
 
-    val = s->mmio[addr] | (s->mmio[addr+1] << 8);
-
+    val = s->opreg[addr >> 2];
+    trace_usb_ehci_opreg_read(addr + s->opregbase, addr2str(addr), val);
     return val;
 }
 
-static uint32_t ehci_mem_readl(void *ptr, target_phys_addr_t addr)
+static uint64_t ehci_port_read(void *ptr, hwaddr addr,
+                               unsigned size)
 {
     EHCIState *s = ptr;
     uint32_t val;
 
-    val = s->mmio[addr] | (s->mmio[addr+1] << 8) |
-          (s->mmio[addr+2] << 16) | (s->mmio[addr+3] << 24);
-
-    trace_usb_ehci_mmio_readl(addr, addr2str(addr), val);
+    val = s->portsc[addr >> 2];
+    trace_usb_ehci_portsc_read(addr + PORTSC_BEGIN, addr >> 2, val);
     return val;
 }
 
-static void ehci_mem_writeb(void *ptr, target_phys_addr_t addr, uint32_t val)
-{
-    fprintf(stderr, "EHCI doesn't handle byte writes to MMIO\n");
-    exit(1);
-}
-
-static void ehci_mem_writew(void *ptr, target_phys_addr_t addr, uint32_t val)
-{
-    fprintf(stderr, "EHCI doesn't handle 16-bit writes to MMIO\n");
-    exit(1);
-}
-
 static void handle_port_owner_write(EHCIState *s, int port, uint32_t owner)
 {
     USBDevice *dev = s->ports[port].dev;
@@ -1123,11 +863,17 @@ static void handle_port_owner_write(EHCIState *s, int port, uint32_t owner)
     }
 }
 
-static void handle_port_status_write(EHCIState *s, int port, uint32_t val)
+static void ehci_port_write(void *ptr, hwaddr addr,
+                            uint64_t val, unsigned size)
 {
+    EHCIState *s = ptr;
+    int port = addr >> 2;
     uint32_t *portsc = &s->portsc[port];
+    uint32_t old = *portsc;
     USBDevice *dev = s->ports[port].dev;
 
+    trace_usb_ehci_portsc_write(addr + PORTSC_BEGIN, addr >> 2, val);
+
     /* Clear rwc bits */
     *portsc &= ~(val & PORTSC_RWC_MASK);
     /* The guest may clear, but not set the PED bit */
@@ -1159,39 +905,20 @@ static void handle_port_status_write(EHCIState *s, int port, uint32_t val)
 
     *portsc &= ~PORTSC_RO_MASK;
     *portsc |= val;
+    trace_usb_ehci_portsc_change(addr + PORTSC_BEGIN, addr >> 2, *portsc, old);
 }
 
-static void ehci_mem_writel(void *ptr, target_phys_addr_t addr, uint32_t val)
+static void ehci_opreg_write(void *ptr, hwaddr addr,
+                             uint64_t val, unsigned size)
 {
     EHCIState *s = ptr;
-    uint32_t *mmio = (uint32_t *)(&s->mmio[addr]);
+    uint32_t *mmio = s->opreg + (addr >> 2);
     uint32_t old = *mmio;
     int i;
 
-    trace_usb_ehci_mmio_writel(addr, addr2str(addr), val);
-
-    /* Only aligned reads are allowed on OHCI */
-    if (addr & 3) {
-        fprintf(stderr, "usb-ehci: Mis-aligned write to addr 0x"
-                TARGET_FMT_plx "\n", addr);
-        return;
-    }
-
-    if (addr >= PORTSC && addr < PORTSC + 4 * NB_PORTS) {
-        handle_port_status_write(s, (addr-PORTSC)/4, val);
-        trace_usb_ehci_mmio_change(addr, addr2str(addr), *mmio, old);
-        return;
-    }
-
-    if (addr < OPREGBASE) {
-        fprintf(stderr, "usb-ehci: write attempt to read-only register"
-                TARGET_FMT_plx "\n", addr);
-        return;
-    }
+    trace_usb_ehci_opreg_write(addr + s->opregbase, addr2str(addr), val);
 
-
-    /* Do any register specific pre-write processing here.  */
-    switch(addr) {
+    switch (addr) {
     case USBCMD:
         if (val & USBCMD_HCRESET) {
             ehci_reset(s);
@@ -1202,7 +929,7 @@ static void ehci_mem_writel(void *ptr, target_phys_addr_t addr, uint32_t val)
         /* not supporting dynamic frame list size at the moment */
         if ((val & USBCMD_FLS) && !(s->usbcmd & USBCMD_FLS)) {
             fprintf(stderr, "attempt to set frame list size -- value %d\n",
-                    val & USBCMD_FLS);
+                    (int)val & USBCMD_FLS);
             val &= ~USBCMD_FLS;
         }
 
@@ -1213,6 +940,7 @@ static void ehci_mem_writel(void *ptr, target_phys_addr_t addr, uint32_t val)
              */
             s->async_stepdown = 0;
             qemu_bh_schedule(s->async_bh);
+            trace_usb_ehci_doorbell_ring();
         }
 
         if (((USBCMD_RUNSTOP | USBCMD_PSE | USBCMD_ASE) & val) !=
@@ -1223,7 +951,7 @@ static void ehci_mem_writel(void *ptr, target_phys_addr_t addr, uint32_t val)
             s->usbcmd = val; /* Set usbcmd for ehci_update_halt() */
             ehci_update_halt(s);
             s->async_stepdown = 0;
-            qemu_mod_timer(s->frame_timer, qemu_get_clock_ns(vm_clock));
+            qemu_bh_schedule(s->async_bh);
         }
         break;
 
@@ -1236,6 +964,9 @@ static void ehci_mem_writel(void *ptr, target_phys_addr_t addr, uint32_t val)
 
     case USBINTR:
         val &= USBINTR_MASK;
+        if (ehci_enabled(s) && (USBSTS_FLR & val)) {
+            qemu_bh_schedule(s->async_bh);
+        }
         break;
 
     case FRINDEX:
@@ -1268,24 +999,29 @@ static void ehci_mem_writel(void *ptr, target_phys_addr_t addr, uint32_t val)
     }
 
     *mmio = val;
-    trace_usb_ehci_mmio_change(addr, addr2str(addr), *mmio, old);
+    trace_usb_ehci_opreg_change(addr + s->opregbase, addr2str(addr),
+                                *mmio, old);
 }
 
-
-// TODO : Put in common header file, duplication from usb-ohci.c
-
 /* Get an array of dwords from main memory */
 static inline int get_dwords(EHCIState *ehci, uint32_t addr,
                              uint32_t *buf, int num)
 {
     int i;
 
+    if (!ehci->dma) {
+        ehci_raise_irq(ehci, USBSTS_HSE);
+        ehci->usbcmd &= ~USBCMD_RUNSTOP;
+        trace_usb_ehci_dma_error();
+        return -1;
+    }
+
     for(i = 0; i < num; i++, buf++, addr += sizeof(*buf)) {
-        pci_dma_read(&ehci->dev, addr, buf, sizeof(*buf));
+        dma_memory_read(ehci->dma, addr, buf, sizeof(*buf));
         *buf = le32_to_cpu(*buf);
     }
 
-    return 1;
+    return num;
 }
 
 /* Put an array of dwords in to main memory */
@@ -1294,12 +1030,19 @@ static inline int put_dwords(EHCIState *ehci, uint32_t addr,
 {
     int i;
 
+    if (!ehci->dma) {
+        ehci_raise_irq(ehci, USBSTS_HSE);
+        ehci->usbcmd &= ~USBCMD_RUNSTOP;
+        trace_usb_ehci_dma_error();
+        return -1;
+    }
+
     for(i = 0; i < num; i++, buf++, addr += sizeof(*buf)) {
         uint32_t tmp = cpu_to_le32(*buf);
-        pci_dma_write(&ehci->dev, addr, &tmp, sizeof(tmp));
+        dma_memory_write(ehci->dma, addr, &tmp, sizeof(tmp));
     }
 
-    return 1;
+    return num;
 }
 
 /*
@@ -1379,12 +1122,12 @@ static int ehci_init_transfer(EHCIPacket *p)
     cpage  = get_field(p->qtd.token, QTD_TOKEN_CPAGE);
     bytes  = get_field(p->qtd.token, QTD_TOKEN_TBYTES);
     offset = p->qtd.bufptr[0] & ~QTD_BUFPTR_MASK;
-    pci_dma_sglist_init(&p->sgl, &p->queue->ehci->dev, 5);
+    qemu_sglist_init(&p->sgl, 5, p->queue->ehci->dma);
 
     while (bytes > 0) {
         if (cpage > 4) {
             fprintf(stderr, "cpage out of range (%d)\n", cpage);
-            return USB_RET_PROCERR;
+            return -1;
         }
 
         page  = p->qtd.bufptr[cpage] & QTD_BUFPTR_MASK;
@@ -1402,16 +1145,16 @@ static int ehci_init_transfer(EHCIPacket *p)
     return 0;
 }
 
-static void ehci_finish_transfer(EHCIQueue *q, int status)
+static void ehci_finish_transfer(EHCIQueue *q, int len)
 {
     uint32_t cpage, offset;
 
-    if (status > 0) {
+    if (len > 0) {
         /* update cpage & offset */
         cpage  = get_field(q->qh.token, QTD_TOKEN_CPAGE);
         offset = q->qh.bufptr[0] & ~QTD_BUFPTR_MASK;
 
-        offset += status;
+        offset += len;
         cpage  += offset >> QTD_BUFPTR_SH;
         offset &= ~QTD_BUFPTR_MASK;
 
@@ -1434,10 +1177,16 @@ static void ehci_async_complete_packet(USBPort *port, USBPacket *packet)
     }
 
     p = container_of(packet, EHCIPacket, packet);
-    trace_usb_ehci_packet_action(p->queue, p, "wakeup");
     assert(p->async == EHCI_ASYNC_INFLIGHT);
+
+    if (packet->status == USB_RET_REMOVE_FROM_QUEUE) {
+        trace_usb_ehci_packet_action(p->queue, p, "remove");
+        ehci_free_packet(p);
+        return;
+    }
+
+    trace_usb_ehci_packet_action(p->queue, p, "wakeup");
     p->async = EHCI_ASYNC_FINISHED;
-    p->usb_status = packet->result;
 
     if (p->queue->async) {
         qemu_bh_schedule(p->queue->ehci->async_bh);
@@ -1447,85 +1196,94 @@ static void ehci_async_complete_packet(USBPort *port, USBPacket *packet)
 static void ehci_execute_complete(EHCIQueue *q)
 {
     EHCIPacket *p = QTAILQ_FIRST(&q->packets);
+    uint32_t tbytes;
 
     assert(p != NULL);
     assert(p->qtdaddr == q->qtdaddr);
-    assert(p->async != EHCI_ASYNC_INFLIGHT);
-    p->async = EHCI_ASYNC_NONE;
+    assert(p->async == EHCI_ASYNC_INITIALIZED ||
+           p->async == EHCI_ASYNC_FINISHED);
 
-    DPRINTF("execute_complete: qhaddr 0x%x, next %x, qtdaddr 0x%x, status %d\n",
-            q->qhaddr, q->qh.next, q->qtdaddr, q->usb_status);
+    DPRINTF("execute_complete: qhaddr 0x%x, next 0x%x, qtdaddr 0x%x, "
+            "status %d, actual_length %d\n",
+            q->qhaddr, q->qh.next, q->qtdaddr,
+            p->packet.status, p->packet.actual_length);
 
-    if (p->usb_status < 0) {
-        switch (p->usb_status) {
-        case USB_RET_IOERROR:
-        case USB_RET_NODEV:
-            q->qh.token |= (QTD_TOKEN_HALT | QTD_TOKEN_XACTERR);
-            set_field(&q->qh.token, 0, QTD_TOKEN_CERR);
-            ehci_raise_irq(q->ehci, USBSTS_ERRINT);
-            break;
-        case USB_RET_STALL:
-            q->qh.token |= QTD_TOKEN_HALT;
-            ehci_raise_irq(q->ehci, USBSTS_ERRINT);
-            break;
-        case USB_RET_NAK:
-            set_field(&q->qh.altnext_qtd, 0, QH_ALTNEXT_NAKCNT);
-            return; /* We're not done yet with this transaction */
-        case USB_RET_BABBLE:
-            q->qh.token |= (QTD_TOKEN_HALT | QTD_TOKEN_BABBLE);
-            ehci_raise_irq(q->ehci, USBSTS_ERRINT);
-            break;
-        default:
-            /* should not be triggerable */
-            fprintf(stderr, "USB invalid response %d\n", p->usb_status);
-            assert(0);
-            break;
-        }
-    } else if ((p->usb_status > p->tbytes) && (p->pid == USB_TOKEN_IN)) {
-        p->usb_status = USB_RET_BABBLE;
+    switch (p->packet.status) {
+    case USB_RET_SUCCESS:
+        break;
+    case USB_RET_IOERROR:
+    case USB_RET_NODEV:
+        q->qh.token |= (QTD_TOKEN_HALT | QTD_TOKEN_XACTERR);
+        set_field(&q->qh.token, 0, QTD_TOKEN_CERR);
+        ehci_raise_irq(q->ehci, USBSTS_ERRINT);
+        break;
+    case USB_RET_STALL:
+        q->qh.token |= QTD_TOKEN_HALT;
+        ehci_raise_irq(q->ehci, USBSTS_ERRINT);
+        break;
+    case USB_RET_NAK:
+        set_field(&q->qh.altnext_qtd, 0, QH_ALTNEXT_NAKCNT);
+        return; /* We're not done yet with this transaction */
+    case USB_RET_BABBLE:
         q->qh.token |= (QTD_TOKEN_HALT | QTD_TOKEN_BABBLE);
         ehci_raise_irq(q->ehci, USBSTS_ERRINT);
-    } else {
-        // TODO check 4.12 for splits
+        break;
+    default:
+        /* should not be triggerable */
+        fprintf(stderr, "USB invalid response %d\n", p->packet.status);
+        assert(0);
+        break;
+    }
 
-        if (p->tbytes && p->pid == USB_TOKEN_IN) {
-            p->tbytes -= p->usb_status;
-        } else {
-            p->tbytes = 0;
+    /* TODO check 4.12 for splits */
+    tbytes = get_field(q->qh.token, QTD_TOKEN_TBYTES);
+    if (tbytes && p->pid == USB_TOKEN_IN) {
+        tbytes -= p->packet.actual_length;
+        if (tbytes) {
+            /* 4.15.1.2 must raise int on a short input packet */
+            ehci_raise_irq(q->ehci, USBSTS_INT);
         }
-
-        DPRINTF("updating tbytes to %d\n", p->tbytes);
-        set_field(&q->qh.token, p->tbytes, QTD_TOKEN_TBYTES);
+    } else {
+        tbytes = 0;
     }
-    ehci_finish_transfer(q, p->usb_status);
+    DPRINTF("updating tbytes to %d\n", tbytes);
+    set_field(&q->qh.token, tbytes, QTD_TOKEN_TBYTES);
+
+    ehci_finish_transfer(q, p->packet.actual_length);
     usb_packet_unmap(&p->packet, &p->sgl);
     qemu_sglist_destroy(&p->sgl);
+    p->async = EHCI_ASYNC_NONE;
 
     q->qh.token ^= QTD_TOKEN_DTOGGLE;
     q->qh.token &= ~QTD_TOKEN_ACTIVE;
 
     if (q->qh.token & QTD_TOKEN_IOC) {
         ehci_raise_irq(q->ehci, USBSTS_INT);
+        if (q->async) {
+            q->ehci->int_req_by_async = true;
+        }
     }
 }
 
-// 4.10.3
-
+/* 4.10.3 returns "again" */
 static int ehci_execute(EHCIPacket *p, const char *action)
 {
     USBEndpoint *ep;
-    int ret;
     int endp;
+    bool spd;
+
+    assert(p->async == EHCI_ASYNC_NONE ||
+           p->async == EHCI_ASYNC_INITIALIZED);
 
     if (!(p->qtd.token & QTD_TOKEN_ACTIVE)) {
         fprintf(stderr, "Attempting to execute inactive qtd\n");
-        return USB_RET_PROCERR;
+        return -1;
     }
 
-    p->tbytes = (p->qtd.token & QTD_TOKEN_TBYTES_MASK) >> QTD_TOKEN_TBYTES_SH;
-    if (p->tbytes > BUFF_SIZE) {
-        fprintf(stderr, "Request for more bytes than allowed\n");
-        return USB_RET_PROCERR;
+    if (get_field(p->qtd.token, QTD_TOKEN_TBYTES) > BUFF_SIZE) {
+        ehci_trace_guest_bug(p->queue->ehci,
+                             "guest requested more bytes than allowed");
+        return -1;
     }
 
     p->pid = (p->qtd.token & QTD_TOKEN_PID_MASK) >> QTD_TOKEN_PID_SH;
@@ -1544,29 +1302,34 @@ static int ehci_execute(EHCIPacket *p, const char *action)
         break;
     }
 
-    if (ehci_init_transfer(p) != 0) {
-        return USB_RET_PROCERR;
-    }
-
     endp = get_field(p->queue->qh.epchar, QH_EPCHAR_EP);
     ep = usb_ep_get(p->queue->dev, p->pid, endp);
 
-    usb_packet_setup(&p->packet, p->pid, ep, p->qtdaddr);
-    usb_packet_map(&p->packet, &p->sgl);
+    if (p->async == EHCI_ASYNC_NONE) {
+        if (ehci_init_transfer(p) != 0) {
+            return -1;
+        }
+
+        spd = (p->pid == USB_TOKEN_IN && NLPTR_TBIT(p->qtd.altnext) == 0);
+        usb_packet_setup(&p->packet, p->pid, ep, p->qtdaddr, spd,
+                         (p->qtd.token & QTD_TOKEN_IOC) != 0);
+        usb_packet_map(&p->packet, &p->sgl);
+        p->async = EHCI_ASYNC_INITIALIZED;
+    }
 
     trace_usb_ehci_packet_action(p->queue, p, action);
-    ret = usb_handle_packet(p->queue->dev, &p->packet);
-    DPRINTF("submit: qh %x next %x qtd %x pid %x len %zd "
-            "(total %d) endp %x ret %d\n",
-            q->qhaddr, q->qh.next, q->qtdaddr, q->pid,
-            q->packet.iov.size, q->tbytes, endp, ret);
+    usb_handle_packet(p->queue->dev, &p->packet);
+    DPRINTF("submit: qh 0x%x next 0x%x qtd 0x%x pid 0x%x len %zd endp 0x%x "
+            "status %d actual_length %d\n", p->queue->qhaddr, p->qtd.next,
+            p->qtdaddr, p->pid, p->packet.iov.size, endp, p->packet.status,
+            p->packet.actual_length);
 
-    if (ret > BUFF_SIZE) {
+    if (p->packet.actual_length > BUFF_SIZE) {
         fprintf(stderr, "ret from usb_handle_packet > BUFF_SIZE\n");
-        return USB_RET_PROCERR;
+        return -1;
     }
 
-    return ret;
+    return 1;
 }
 
 /*  4.7.2
@@ -1578,7 +1341,6 @@ static int ehci_process_itd(EHCIState *ehci,
 {
     USBDevice *dev;
     USBEndpoint *ep;
-    int ret;
     uint32_t i, len, pid, dir, devaddr, endp;
     uint32_t pg, off, ptr1, ptr2, max, mult;
 
@@ -1601,10 +1363,10 @@ static int ehci_process_itd(EHCIState *ehci,
             }
 
             if (len > BUFF_SIZE) {
-                return USB_RET_PROCERR;
+                return -1;
             }
 
-            pci_dma_sglist_init(&ehci->isgl, &ehci->dev, 2);
+            qemu_sglist_init(&ehci->isgl, 2, ehci->dma);
             if (off + len > 4096) {
                 /* transfer crosses page border */
                 uint32_t len2 = off + len - 4096;
@@ -1620,48 +1382,48 @@ static int ehci_process_itd(EHCIState *ehci,
             dev = ehci_find_device(ehci, devaddr);
             ep = usb_ep_get(dev, pid, endp);
             if (ep && ep->type == USB_ENDPOINT_XFER_ISOC) {
-                usb_packet_setup(&ehci->ipacket, pid, ep, addr);
+                usb_packet_setup(&ehci->ipacket, pid, ep, addr, false,
+                                 (itd->transact[i] & ITD_XACT_IOC) != 0);
                 usb_packet_map(&ehci->ipacket, &ehci->isgl);
-                ret = usb_handle_packet(dev, &ehci->ipacket);
-                assert(ret != USB_RET_ASYNC);
+                usb_handle_packet(dev, &ehci->ipacket);
                 usb_packet_unmap(&ehci->ipacket, &ehci->isgl);
             } else {
                 DPRINTF("ISOCH: attempt to addess non-iso endpoint\n");
-                ret = USB_RET_NAK;
+                ehci->ipacket.status = USB_RET_NAK;
+                ehci->ipacket.actual_length = 0;
             }
             qemu_sglist_destroy(&ehci->isgl);
 
-            if (ret < 0) {
-                switch (ret) {
-                default:
-                    fprintf(stderr, "Unexpected iso usb result: %d\n", ret);
-                    /* Fall through */
-                case USB_RET_IOERROR:
-                case USB_RET_NODEV:
-                    /* 3.3.2: XACTERR is only allowed on IN transactions */
-                    if (dir) {
-                        itd->transact[i] |= ITD_XACT_XACTERR;
-                        ehci_raise_irq(ehci, USBSTS_ERRINT);
-                    }
-                    break;
-                case USB_RET_BABBLE:
-                    itd->transact[i] |= ITD_XACT_BABBLE;
+            switch (ehci->ipacket.status) {
+            case USB_RET_SUCCESS:
+                break;
+            default:
+                fprintf(stderr, "Unexpected iso usb result: %d\n",
+                        ehci->ipacket.status);
+                /* Fall through */
+            case USB_RET_IOERROR:
+            case USB_RET_NODEV:
+                /* 3.3.2: XACTERR is only allowed on IN transactions */
+                if (dir) {
+                    itd->transact[i] |= ITD_XACT_XACTERR;
                     ehci_raise_irq(ehci, USBSTS_ERRINT);
-                    break;
-                case USB_RET_NAK:
-                    /* no data for us, so do a zero-length transfer */
-                    ret = 0;
-                    break;
                 }
+                break;
+            case USB_RET_BABBLE:
+                itd->transact[i] |= ITD_XACT_BABBLE;
+                ehci_raise_irq(ehci, USBSTS_ERRINT);
+                break;
+            case USB_RET_NAK:
+                /* no data for us, so do a zero-length transfer */
+                ehci->ipacket.actual_length = 0;
+                break;
             }
-            if (ret >= 0) {
-                if (!dir) {
-                    /* OUT */
-                    set_field(&itd->transact[i], len - ret, ITD_XACT_LENGTH);
-                } else {
-                    /* IN */
-                    set_field(&itd->transact[i], ret, ITD_XACT_LENGTH);
-                }
+            if (!dir) {
+                set_field(&itd->transact[i], len - ehci->ipacket.actual_length,
+                          ITD_XACT_LENGTH); /* OUT */
+            } else {
+                set_field(&itd->transact[i], ehci->ipacket.actual_length,
+                          ITD_XACT_LENGTH); /* IN */
             }
             if (itd->transact[i] & ITD_XACT_IOC) {
                 ehci_raise_irq(ehci, USBSTS_INT);
@@ -1692,8 +1454,10 @@ static int ehci_state_waitlisthead(EHCIState *ehci,  int async)
 
     /*  Find the head of the list (4.9.1.1) */
     for(i = 0; i < MAX_QH; i++) {
-        get_dwords(ehci, NLPTR_GET(entry), (uint32_t *) &qh,
-                   sizeof(EHCIqh) >> 2);
+        if (get_dwords(ehci, NLPTR_GET(entry), (uint32_t *) &qh,
+                       sizeof(EHCIqh) >> 2) < 0) {
+            return 0;
+        }
         ehci_trace_qh(NULL, NLPTR_GET(entry), &qh);
 
         if (qh.epchar & QH_EPCHAR_H) {
@@ -1771,7 +1535,7 @@ out:
 static EHCIQueue *ehci_state_fetchqh(EHCIState *ehci, int async)
 {
     EHCIPacket *p;
-    uint32_t entry, devaddr;
+    uint32_t entry, devaddr, endp;
     EHCIQueue *q;
     EHCIqh qh;
 
@@ -1790,28 +1554,38 @@ static EHCIQueue *ehci_state_fetchqh(EHCIState *ehci, int async)
         goto out;
     }
 
-    get_dwords(ehci, NLPTR_GET(q->qhaddr),
-               (uint32_t *) &qh, sizeof(EHCIqh) >> 2);
-    if (q->revalidate && (q->qh.epchar      != qh.epchar ||
-                          q->qh.epcap       != qh.epcap  ||
-                          q->qh.current_qtd != qh.current_qtd)) {
-        ehci_free_queue(q);
-        q = ehci_alloc_queue(ehci, entry, async);
-        q->seen++;
+    if (get_dwords(ehci, NLPTR_GET(q->qhaddr),
+                   (uint32_t *) &qh, sizeof(EHCIqh) >> 2) < 0) {
+        q = NULL;
+        goto out;
+    }
+    ehci_trace_qh(q, NLPTR_GET(q->qhaddr), &qh);
+
+    /*
+     * The overlay area of the qh should never be changed by the guest,
+     * except when idle, in which case the reset is a nop.
+     */
+    devaddr = get_field(qh.epchar, QH_EPCHAR_DEVADDR);
+    endp    = get_field(qh.epchar, QH_EPCHAR_EP);
+    if ((devaddr != get_field(q->qh.epchar, QH_EPCHAR_DEVADDR)) ||
+        (endp    != get_field(q->qh.epchar, QH_EPCHAR_EP)) ||
+        (qh.current_qtd != q->qh.current_qtd) ||
+        (q->async && qh.next_qtd != q->qh.next_qtd) ||
+        (memcmp(&qh.altnext_qtd, &q->qh.altnext_qtd,
+                                 7 * sizeof(uint32_t)) != 0) ||
+        (q->dev != NULL && q->dev->addr != devaddr)) {
+        if (ehci_reset_queue(q) > 0) {
+            ehci_trace_guest_bug(ehci, "guest updated active QH");
+        }
         p = NULL;
     }
     q->qh = qh;
-    q->revalidate = 0;
-    ehci_trace_qh(q, NLPTR_GET(q->qhaddr), &q->qh);
 
-    devaddr = get_field(q->qh.epchar, QH_EPCHAR_DEVADDR);
-    if (q->dev != NULL && q->dev->addr != devaddr) {
-        if (!QTAILQ_EMPTY(&q->packets)) {
-            /* should not happen (guest bug) */
-            ehci_cancel_queue(q);
-        }
-        q->dev = NULL;
+    q->transact_ctr = get_field(q->qh.epcap, QH_EPCAP_MULT);
+    if (q->transact_ctr == 0) { /* Guest bug in some versions of windows */
+        q->transact_ctr = 4;
     }
+
     if (q->dev == NULL) {
         q->dev = ehci_find_device(q->ehci, devaddr);
     }
@@ -1873,8 +1647,10 @@ static int ehci_state_fetchitd(EHCIState *ehci, int async)
     assert(!async);
     entry = ehci_get_fetch_addr(ehci, async);
 
-    get_dwords(ehci, NLPTR_GET(entry), (uint32_t *) &itd,
-               sizeof(EHCIitd) >> 2);
+    if (get_dwords(ehci, NLPTR_GET(entry), (uint32_t *) &itd,
+                   sizeof(EHCIitd) >> 2) < 0) {
+        return -1;
+    }
     ehci_trace_itd(ehci, entry, &itd);
 
     if (ehci_process_itd(ehci, &itd, entry) != 0) {
@@ -1897,8 +1673,10 @@ static int ehci_state_fetchsitd(EHCIState *ehci, int async)
     assert(!async);
     entry = ehci_get_fetch_addr(ehci, async);
 
-    get_dwords(ehci, NLPTR_GET(entry), (uint32_t *)&sitd,
-               sizeof(EHCIsitd) >> 2);
+    if (get_dwords(ehci, NLPTR_GET(entry), (uint32_t *)&sitd,
+                   sizeof(EHCIsitd) >> 2) < 0) {
+        return 0;
+    }
     ehci_trace_sitd(ehci, entry, &sitd);
 
     if (!(sitd.results & SITD_RESULTS_ACTIVE)) {
@@ -1957,20 +1735,23 @@ static int ehci_state_fetchqtd(EHCIQueue *q)
 {
     EHCIqtd qtd;
     EHCIPacket *p;
-    int again = 0;
+    int again = 1;
 
-    get_dwords(q->ehci, NLPTR_GET(q->qtdaddr), (uint32_t *) &qtd,
-               sizeof(EHCIqtd) >> 2);
+    if (get_dwords(q->ehci, NLPTR_GET(q->qtdaddr), (uint32_t *) &qtd,
+                   sizeof(EHCIqtd) >> 2) < 0) {
+        return 0;
+    }
     ehci_trace_qtd(q, NLPTR_GET(q->qtdaddr), &qtd);
 
     p = QTAILQ_FIRST(&q->packets);
     if (p != NULL) {
         if (p->qtdaddr != q->qtdaddr ||
-            (!NLPTR_TBIT(p->qtd.next) && (p->qtd.next != qtd.next)) ||
+            (q->async && !NLPTR_TBIT(p->qtd.next) &&
+                (p->qtd.next != qtd.next)) ||
             (!NLPTR_TBIT(p->qtd.altnext) && (p->qtd.altnext != qtd.altnext)) ||
             p->qtd.bufptr[0] != qtd.bufptr[0]) {
-            /* guest bug: guest updated active QH or qTD underneath us */
             ehci_cancel_queue(q);
+            ehci_trace_guest_bug(q->ehci, "guest updated active QH or qTD");
             p = NULL;
         } else {
             p->qtd = qtd;
@@ -1985,29 +1766,32 @@ static int ehci_state_fetchqtd(EHCIQueue *q)
             p = NULL;
         }
         ehci_set_state(q->ehci, q->async, EST_HORIZONTALQH);
-        again = 1;
     } else if (p != NULL) {
         switch (p->async) {
         case EHCI_ASYNC_NONE:
-            /* Previously nacked packet (likely interrupt ep) */
-           ehci_set_state(q->ehci, q->async, EST_EXECUTE);
-           break;
+        case EHCI_ASYNC_INITIALIZED:
+            /* Not yet executed (MULT), or previously nacked (int) packet */
+            ehci_set_state(q->ehci, q->async, EST_EXECUTE);
+            break;
         case EHCI_ASYNC_INFLIGHT:
-            /* Unfinyshed async handled packet, go horizontal */
+            /* Check if the guest has added new tds to the queue */
+            again = ehci_fill_queue(QTAILQ_LAST(&q->packets, pkts_head));
+            /* Unfinished async handled packet, go horizontal */
             ehci_set_state(q->ehci, q->async, EST_HORIZONTALQH);
             break;
         case EHCI_ASYNC_FINISHED:
-            /* Should never happen, as this case is caught by fetchqh */
+            /*
+             * We get here when advqueue moves to a packet which is already
+             * finished, which can happen with packets queued up by fill_queue
+             */
             ehci_set_state(q->ehci, q->async, EST_EXECUTING);
             break;
         }
-        again = 1;
     } else {
         p = ehci_alloc_packet(q);
         p->qtdaddr = q->qtdaddr;
         p->qtd = qtd;
         ehci_set_state(q->ehci, q->async, EST_EXECUTE);
-        again = 1;
     }
 
     return again;
@@ -2028,22 +1812,32 @@ static int ehci_state_horizqh(EHCIQueue *q)
     return again;
 }
 
-static void ehci_fill_queue(EHCIPacket *p)
+/* Returns "again" */
+static int ehci_fill_queue(EHCIPacket *p)
 {
+    USBEndpoint *ep = p->packet.ep;
     EHCIQueue *q = p->queue;
     EHCIqtd qtd = p->qtd;
     uint32_t qtdaddr;
 
     for (;;) {
-        if (NLPTR_TBIT(qtd.altnext) == 0) {
-            break;
-        }
         if (NLPTR_TBIT(qtd.next) != 0) {
             break;
         }
         qtdaddr = qtd.next;
-        get_dwords(q->ehci, NLPTR_GET(qtdaddr),
-                   (uint32_t *) &qtd, sizeof(EHCIqtd) >> 2);
+        /*
+         * Detect circular td lists, Windows creates these, counting on the
+         * active bit going low after execution to make the queue stop.
+         */
+        QTAILQ_FOREACH(p, &q->packets, next) {
+            if (p->qtdaddr == qtdaddr) {
+                goto leave;
+            }
+        }
+        if (get_dwords(q->ehci, NLPTR_GET(qtdaddr),
+                       (uint32_t *) &qtd, sizeof(EHCIqtd) >> 2) < 0) {
+            return -1;
+        }
         ehci_trace_qtd(q, NLPTR_GET(qtdaddr), &qtd);
         if (!(qtd.token & QTD_TOKEN_ACTIVE)) {
             break;
@@ -2051,10 +1845,15 @@ static void ehci_fill_queue(EHCIPacket *p)
         p = ehci_alloc_packet(q);
         p->qtdaddr = qtdaddr;
         p->qtd = qtd;
-        p->usb_status = ehci_execute(p, "queue");
-        assert(p->usb_status == USB_RET_ASYNC);
+        if (ehci_execute(p, "queue") == -1) {
+            return -1;
+        }
+        assert(p->packet.status == USB_RET_ASYNC);
         p->async = EHCI_ASYNC_INFLIGHT;
     }
+leave:
+    usb_device_flush_ep_queue(ep->dev, ep);
+    return 1;
 }
 
 static int ehci_state_execute(EHCIQueue *q)
@@ -2071,33 +1870,32 @@ static int ehci_state_execute(EHCIQueue *q)
 
     // TODO verify enough time remains in the uframe as in 4.4.1.1
     // TODO write back ptr to async list when done or out of time
-    // TODO Windows does not seem to ever set the MULT field
 
-    if (!q->async) {
-        int transactCtr = get_field(q->qh.epcap, QH_EPCAP_MULT);
-        if (!transactCtr) {
-            ehci_set_state(q->ehci, q->async, EST_HORIZONTALQH);
-            again = 1;
-            goto out;
-        }
+    /* 4.10.3, bottom of page 82, go horizontal on transaction counter == 0 */
+    if (!q->async && q->transact_ctr == 0) {
+        ehci_set_state(q->ehci, q->async, EST_HORIZONTALQH);
+        again = 1;
+        goto out;
     }
 
     if (q->async) {
         ehci_set_usbsts(q->ehci, USBSTS_REC);
     }
 
-    p->usb_status = ehci_execute(p, "process");
-    if (p->usb_status == USB_RET_PROCERR) {
-        again = -1;
+    again = ehci_execute(p, "process");
+    if (again == -1) {
         goto out;
     }
-    if (p->usb_status == USB_RET_ASYNC) {
+    if (p->packet.status == USB_RET_ASYNC) {
         ehci_flush_qh(q);
         trace_usb_ehci_packet_action(p->queue, p, "async");
         p->async = EHCI_ASYNC_INFLIGHT;
         ehci_set_state(q->ehci, q->async, EST_HORIZONTALQH);
-        again = 1;
-        ehci_fill_queue(p);
+        if (q->async) {
+            again = ehci_fill_queue(p);
+        } else {
+            again = 1;
+        }
         goto out;
     }
 
@@ -2117,17 +1915,13 @@ static int ehci_state_executing(EHCIQueue *q)
 
     ehci_execute_complete(q);
 
-    // 4.10.3
-    if (!q->async) {
-        int transactCtr = get_field(q->qh.epcap, QH_EPCAP_MULT);
-        transactCtr--;
-        set_field(&q->qh.epcap, transactCtr, QH_EPCAP_MULT);
-        // 4.10.3, bottom of page 82, should exit this state when transaction
-        // counter decrements to 0
+    /* 4.10.3 */
+    if (!q->async && q->transact_ctr > 0) {
+        q->transact_ctr--;
     }
 
     /* 4.10.5 */
-    if (p->usb_status == USB_RET_NAK) {
+    if (p->packet.status == USB_RET_NAK) {
         ehci_set_state(q->ehci, q->async, EST_HORIZONTALQH);
     } else {
         ehci_set_state(q->ehci, q->async, EST_WRITEBACK);
@@ -2163,19 +1957,6 @@ static int ehci_state_writeback(EHCIQueue *q)
      * bit is clear.
      */
     if (q->qh.token & QTD_TOKEN_HALT) {
-        /*
-         * We should not do any further processing on a halted queue!
-         * This is esp. important for bulk endpoints with pipelining enabled
-         * (redirection to a real USB device), where we must cancel all the
-         * transfers after this one so that:
-         * 1) If they've completed already, they are not processed further
-         *    causing more stalls, originating from the same failed transfer
-         * 2) If still in flight, they are cancelled before the guest does
-         *    a clear stall, otherwise the guest and device can loose sync!
-         */
-        while ((p = QTAILQ_FIRST(&q->packets)) != NULL) {
-            ehci_free_packet(p);
-        }
         ehci_set_state(q->ehci, q->async, EST_HORIZONTALQH);
         again = 1;
     } else {
@@ -2310,8 +2091,8 @@ static void ehci_advance_async_state(EHCIState *ehci)
          */
         if (ehci->usbcmd & USBCMD_IAAD) {
             /* Remove all unseen qhs from the async qhs queue */
-            ehci_queues_tag_unused_async(ehci);
-            DPRINTF("ASYNC: doorbell request acknowledged\n");
+            ehci_queues_rip_unseen(ehci, async);
+            trace_usb_ehci_doorbell_ack();
             ehci->usbcmd &= ~USBCMD_IAAD;
             ehci_raise_irq(ehci, USBSTS_IAA);
         }
@@ -2355,8 +2136,9 @@ static void ehci_advance_periodic_state(EHCIState *ehci)
         }
         list |= ((ehci->frindex & 0x1ff8) >> 1);
 
-        pci_dma_read(&ehci->dev, list, &entry, sizeof entry);
-        entry = le32_to_cpu(entry);
+        if (get_dwords(ehci, list, &entry, 1) < 0) {
+            break;
+        }
 
         DPRINTF("PERIODIC state adv fr=%d.  [%08X] -> %08X\n",
                 ehci->frindex / 8, list, entry);
@@ -2392,7 +2174,7 @@ static void ehci_update_frindex(EHCIState *ehci, int frames)
         if (ehci->frindex == 0x00004000) {
             ehci_raise_irq(ehci, USBSTS_FLR);
             ehci->frindex = 0;
-            if (ehci->usbsts_frindex > 0x00004000) {
+            if (ehci->usbsts_frindex >= 0x00004000) {
                 ehci->usbsts_frindex -= 0x00004000;
             } else {
                 ehci->usbsts_frindex = 0;
@@ -2427,6 +2209,19 @@ static void ehci_frame_timer(void *opaque)
         }
 
         for (i = 0; i < frames; i++) {
+            /*
+             * If we're running behind schedule, we should not catch up
+             * too fast, as that will make some guests unhappy:
+             * 1) We must process a minimum of MIN_FR_PER_TICK frames,
+             *    otherwise we will never catch up
+             * 2) Process frames until the guest has requested an irq (IOC)
+             */
+            if (i >= MIN_FR_PER_TICK) {
+                ehci_commit_irq(ehci);
+                if ((ehci->usbsts & USBINTR_MASK) & ehci->usbintr) {
+                    break;
+                }
+            }
             ehci_update_frindex(ehci, 1);
             ehci_advance_periodic_state(ehci);
             ehci->last_run_ns += FRAME_TIMER_NS;
@@ -2453,28 +2248,48 @@ static void ehci_frame_timer(void *opaque)
         ehci->async_stepdown = 0;
     }
 
+    if (ehci_enabled(ehci) && (ehci->usbintr & USBSTS_FLR)) {
+        need_timer++;
+    }
+
     if (need_timer) {
-        expire_time = t_now + (get_ticks_per_sec()
+        /* If we've raised int, we speed up the timer, so that we quickly
+         * notice any new packets queued up in response */
+        if (ehci->int_req_by_async && (ehci->usbsts & USBSTS_INT)) {
+            expire_time = t_now + get_ticks_per_sec() / (FRAME_TIMER_FREQ * 2);
+            ehci->int_req_by_async = false;
+        } else {
+            expire_time = t_now + (get_ticks_per_sec()
                                * (ehci->async_stepdown+1) / FRAME_TIMER_FREQ);
+        }
         qemu_mod_timer(ehci->frame_timer, expire_time);
     }
 }
 
-static void ehci_async_bh(void *opaque)
-{
-    EHCIState *ehci = opaque;
-    ehci_advance_async_state(ehci);
-}
+static const MemoryRegionOps ehci_mmio_caps_ops = {
+    .read = ehci_caps_read,
+    .valid.min_access_size = 1,
+    .valid.max_access_size = 4,
+    .impl.min_access_size = 1,
+    .impl.max_access_size = 1,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
 
-static const MemoryRegionOps ehci_mem_ops = {
-    .old_mmio = {
-        .read = { ehci_mem_readb, ehci_mem_readw, ehci_mem_readl },
-        .write = { ehci_mem_writeb, ehci_mem_writew, ehci_mem_writel },
-    },
+static const MemoryRegionOps ehci_mmio_opreg_ops = {
+    .read = ehci_opreg_read,
+    .write = ehci_opreg_write,
+    .valid.min_access_size = 4,
+    .valid.max_access_size = 4,
     .endianness = DEVICE_LITTLE_ENDIAN,
 };
 
-static int usb_ehci_initfn(PCIDevice *dev);
+static const MemoryRegionOps ehci_mmio_port_ops = {
+    .read = ehci_port_read,
+    .write = ehci_port_write,
+    .valid.min_access_size = 4,
+    .valid.max_access_size = 4,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
 
 static USBPortOps ehci_port_ops = {
     .attach = ehci_attach,
@@ -2508,13 +2323,38 @@ static int usb_ehci_post_load(void *opaque, int version_id)
     return 0;
 }
 
-static const VMStateDescription vmstate_ehci = {
-    .name        = "ehci",
+static void usb_ehci_vm_state_change(void *opaque, int running, RunState state)
+{
+    EHCIState *ehci = opaque;
+
+    /*
+     * We don't migrate the EHCIQueue-s, instead we rebuild them for the
+     * schedule in guest memory. We must do the rebuilt ASAP, so that
+     * USB-devices which have async handled packages have a packet in the
+     * ep queue to match the completion with.
+     */
+    if (state == RUN_STATE_RUNNING) {
+        ehci_advance_async_state(ehci);
+    }
+
+    /*
+     * The schedule rebuilt from guest memory could cause the migration dest
+     * to miss a QH unlink, and fail to cancel packets, since the unlinked QH
+     * will never have existed on the destination. Therefor we must flush the
+     * async schedule on savevm to catch any not yet noticed unlinks.
+     */
+    if (state == RUN_STATE_SAVE_VM) {
+        ehci_advance_async_state(ehci);
+        ehci_queues_rip_unseen(ehci, 1);
+    }
+}
+
+const VMStateDescription vmstate_ehci = {
+    .name        = "ehci-core",
     .version_id  = 2,
     .minimum_version_id  = 1,
     .post_load   = usb_ehci_post_load,
     .fields      = (VMStateField[]) {
-        VMSTATE_PCI_DEVICE(dev, EHCIState),
         /* mmio registers */
         VMSTATE_UINT32(usbcmd, EHCIState),
         VMSTATE_UINT32(usbsts, EHCIState),
@@ -2545,105 +2385,24 @@ static const VMStateDescription vmstate_ehci = {
     }
 };
 
-static Property ehci_properties[] = {
-    DEFINE_PROP_UINT32("maxframes", EHCIState, maxframes, 128),
-    DEFINE_PROP_END_OF_LIST(),
-};
-
-static void ehci_class_init(ObjectClass *klass, void *data)
-{
-    DeviceClass *dc = DEVICE_CLASS(klass);
-    PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
-
-    k->init = usb_ehci_initfn;
-    k->vendor_id = PCI_VENDOR_ID_INTEL;
-    k->device_id = PCI_DEVICE_ID_INTEL_82801D; /* ich4 */
-    k->revision = 0x10;
-    k->class_id = PCI_CLASS_SERIAL_USB;
-    dc->vmsd = &vmstate_ehci;
-    dc->props = ehci_properties;
-}
-
-static TypeInfo ehci_info = {
-    .name          = "usb-ehci",
-    .parent        = TYPE_PCI_DEVICE,
-    .instance_size = sizeof(EHCIState),
-    .class_init    = ehci_class_init,
-};
-
-static void ich9_ehci_class_init(ObjectClass *klass, void *data)
-{
-    DeviceClass *dc = DEVICE_CLASS(klass);
-    PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
-
-    k->init = usb_ehci_initfn;
-    k->vendor_id = PCI_VENDOR_ID_INTEL;
-    k->device_id = PCI_DEVICE_ID_INTEL_82801I_EHCI1;
-    k->revision = 0x03;
-    k->class_id = PCI_CLASS_SERIAL_USB;
-    dc->vmsd = &vmstate_ehci;
-    dc->props = ehci_properties;
-}
-
-static TypeInfo ich9_ehci_info = {
-    .name          = "ich9-usb-ehci1",
-    .parent        = TYPE_PCI_DEVICE,
-    .instance_size = sizeof(EHCIState),
-    .class_init    = ich9_ehci_class_init,
-};
-
-static int usb_ehci_initfn(PCIDevice *dev)
+void usb_ehci_initfn(EHCIState *s, DeviceState *dev)
 {
-    EHCIState *s = DO_UPCAST(EHCIState, dev, dev);
-    uint8_t *pci_conf = s->dev.config;
     int i;
 
-    pci_set_byte(&pci_conf[PCI_CLASS_PROG], 0x20);
-
-    /* capabilities pointer */
-    pci_set_byte(&pci_conf[PCI_CAPABILITY_LIST], 0x00);
-    //pci_set_byte(&pci_conf[PCI_CAPABILITY_LIST], 0x50);
-
-    pci_set_byte(&pci_conf[PCI_INTERRUPT_PIN], 4); /* interrupt pin D */
-    pci_set_byte(&pci_conf[PCI_MIN_GNT], 0);
-    pci_set_byte(&pci_conf[PCI_MAX_LAT], 0);
-
-    // pci_conf[0x50] = 0x01; // power management caps
-
-    pci_set_byte(&pci_conf[USB_SBRN], USB_RELEASE_2); // release number (2.1.4)
-    pci_set_byte(&pci_conf[0x61], 0x20);  // frame length adjustment (2.1.5)
-    pci_set_word(&pci_conf[0x62], 0x00);  // port wake up capability (2.1.6)
-
-    pci_conf[0x64] = 0x00;
-    pci_conf[0x65] = 0x00;
-    pci_conf[0x66] = 0x00;
-    pci_conf[0x67] = 0x00;
-    pci_conf[0x68] = 0x01;
-    pci_conf[0x69] = 0x00;
-    pci_conf[0x6a] = 0x00;
-    pci_conf[0x6b] = 0x00;  // USBLEGSUP
-    pci_conf[0x6c] = 0x00;
-    pci_conf[0x6d] = 0x00;
-    pci_conf[0x6e] = 0x00;
-    pci_conf[0x6f] = 0xc0;  // USBLEFCTLSTS
-
-    // 2.2 host controller interface version
-    s->mmio[0x00] = (uint8_t) OPREGBASE;
-    s->mmio[0x01] = 0x00;
-    s->mmio[0x02] = 0x00;
-    s->mmio[0x03] = 0x01;        // HC version
-    s->mmio[0x04] = NB_PORTS;    // Number of downstream ports
-    s->mmio[0x05] = 0x00;        // No companion ports at present
-    s->mmio[0x06] = 0x00;
-    s->mmio[0x07] = 0x00;
-    s->mmio[0x08] = 0x80;        // We can cache whole frame, not 64-bit capable
-    s->mmio[0x09] = 0x68;        // EECP
-    s->mmio[0x0a] = 0x00;
-    s->mmio[0x0b] = 0x00;
-
-    s->irq = s->dev.irq[3];
-
-    usb_bus_new(&s->bus, &ehci_bus_ops, &s->dev.qdev);
+    /* 2.2 host controller interface version */
+    s->caps[0x00] = (uint8_t)(s->opregbase - s->capsbase);
+    s->caps[0x01] = 0x00;
+    s->caps[0x02] = 0x00;
+    s->caps[0x03] = 0x01;        /* HC version */
+    s->caps[0x04] = NB_PORTS;    /* Number of downstream ports */
+    s->caps[0x05] = 0x00;        /* No companion ports at present */
+    s->caps[0x06] = 0x00;
+    s->caps[0x07] = 0x00;
+    s->caps[0x08] = 0x80;        /* We can cache whole frame, no 64-bit */
+    s->caps[0x0a] = 0x00;
+    s->caps[0x0b] = 0x00;
+
+    usb_bus_new(&s->bus, &ehci_bus_ops, dev);
     for(i = 0; i < NB_PORTS; i++) {
         usb_register_port(&s->bus, &s->ports[i], s, i, &ehci_port_ops,
                           USB_SPEED_MASK_HIGH);
@@ -2651,27 +2410,28 @@ static int usb_ehci_initfn(PCIDevice *dev)
     }
 
     s->frame_timer = qemu_new_timer_ns(vm_clock, ehci_frame_timer, s);
-    s->async_bh = qemu_bh_new(ehci_async_bh, s);
+    s->async_bh = qemu_bh_new(ehci_frame_timer, s);
     QTAILQ_INIT(&s->aqueues);
     QTAILQ_INIT(&s->pqueues);
     usb_packet_init(&s->ipacket);
 
     qemu_register_reset(ehci_reset, s);
+    qemu_add_vm_change_state_handler(usb_ehci_vm_state_change, s);
 
-    memory_region_init_io(&s->mem, &ehci_mem_ops, s, "ehci", MMIO_SIZE);
-    pci_register_bar(&s->dev, 0, PCI_BASE_ADDRESS_SPACE_MEMORY, &s->mem);
-
-    return 0;
-}
+    memory_region_init(&s->mem, "ehci", MMIO_SIZE);
+    memory_region_init_io(&s->mem_caps, &ehci_mmio_caps_ops, s,
+                          "capabilities", CAPA_SIZE);
+    memory_region_init_io(&s->mem_opreg, &ehci_mmio_opreg_ops, s,
+                          "operational", PORTSC_BEGIN);
+    memory_region_init_io(&s->mem_ports, &ehci_mmio_port_ops, s,
+                          "ports", PORTSC_END - PORTSC_BEGIN);
 
-static void ehci_register_types(void)
-{
-    type_register_static(&ehci_info);
-    type_register_static(&ich9_ehci_info);
+    memory_region_add_subregion(&s->mem, s->capsbase, &s->mem_caps);
+    memory_region_add_subregion(&s->mem, s->opregbase, &s->mem_opreg);
+    memory_region_add_subregion(&s->mem, s->opregbase + PORTSC_BEGIN,
+                                &s->mem_ports);
 }
 
-type_init(ehci_register_types)
-
 /*
  * vim: expandtab ts=4
  */
diff --git a/hw/usb/hcd-ehci.h b/hw/usb/hcd-ehci.h
new file mode 100644
index 000000000..d8078f455
--- /dev/null
+++ b/hw/usb/hcd-ehci.h
@@ -0,0 +1,319 @@
+/*
+ * QEMU USB EHCI Emulation
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or(at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "hw/hw.h"
+#include "qemu-timer.h"
+#include "hw/usb.h"
+#include "monitor.h"
+#include "trace.h"
+#include "dma.h"
+#include "sysemu.h"
+
+#ifndef EHCI_DEBUG
+#define EHCI_DEBUG   0
+#endif
+
+#if EHCI_DEBUG
+#define DPRINTF printf
+#else
+#define DPRINTF(...)
+#endif
+
+#define MMIO_SIZE        0x1000
+#define CAPA_SIZE        0x10
+
+#define PORTSC               0x0044
+#define PORTSC_BEGIN         PORTSC
+#define PORTSC_END           (PORTSC + 4 * NB_PORTS)
+
+#define NB_PORTS         6        /* Number of downstream ports */
+
+typedef struct EHCIPacket EHCIPacket;
+typedef struct EHCIQueue EHCIQueue;
+typedef struct EHCIState EHCIState;
+
+/*  EHCI spec version 1.0 Section 3.3
+ */
+typedef struct EHCIitd {
+    uint32_t next;
+
+    uint32_t transact[8];
+#define ITD_XACT_ACTIVE          (1 << 31)
+#define ITD_XACT_DBERROR         (1 << 30)
+#define ITD_XACT_BABBLE          (1 << 29)
+#define ITD_XACT_XACTERR         (1 << 28)
+#define ITD_XACT_LENGTH_MASK     0x0fff0000
+#define ITD_XACT_LENGTH_SH       16
+#define ITD_XACT_IOC             (1 << 15)
+#define ITD_XACT_PGSEL_MASK      0x00007000
+#define ITD_XACT_PGSEL_SH        12
+#define ITD_XACT_OFFSET_MASK     0x00000fff
+
+    uint32_t bufptr[7];
+#define ITD_BUFPTR_MASK          0xfffff000
+#define ITD_BUFPTR_SH            12
+#define ITD_BUFPTR_EP_MASK       0x00000f00
+#define ITD_BUFPTR_EP_SH         8
+#define ITD_BUFPTR_DEVADDR_MASK  0x0000007f
+#define ITD_BUFPTR_DEVADDR_SH    0
+#define ITD_BUFPTR_DIRECTION     (1 << 11)
+#define ITD_BUFPTR_MAXPKT_MASK   0x000007ff
+#define ITD_BUFPTR_MAXPKT_SH     0
+#define ITD_BUFPTR_MULT_MASK     0x00000003
+#define ITD_BUFPTR_MULT_SH       0
+} EHCIitd;
+
+/*  EHCI spec version 1.0 Section 3.4
+ */
+typedef struct EHCIsitd {
+    uint32_t next;                  /* Standard next link pointer */
+    uint32_t epchar;
+#define SITD_EPCHAR_IO              (1 << 31)
+#define SITD_EPCHAR_PORTNUM_MASK    0x7f000000
+#define SITD_EPCHAR_PORTNUM_SH      24
+#define SITD_EPCHAR_HUBADD_MASK     0x007f0000
+#define SITD_EPCHAR_HUBADDR_SH      16
+#define SITD_EPCHAR_EPNUM_MASK      0x00000f00
+#define SITD_EPCHAR_EPNUM_SH        8
+#define SITD_EPCHAR_DEVADDR_MASK    0x0000007f
+
+    uint32_t uframe;
+#define SITD_UFRAME_CMASK_MASK      0x0000ff00
+#define SITD_UFRAME_CMASK_SH        8
+#define SITD_UFRAME_SMASK_MASK      0x000000ff
+
+    uint32_t results;
+#define SITD_RESULTS_IOC              (1 << 31)
+#define SITD_RESULTS_PGSEL            (1 << 30)
+#define SITD_RESULTS_TBYTES_MASK      0x03ff0000
+#define SITD_RESULTS_TYBYTES_SH       16
+#define SITD_RESULTS_CPROGMASK_MASK   0x0000ff00
+#define SITD_RESULTS_CPROGMASK_SH     8
+#define SITD_RESULTS_ACTIVE           (1 << 7)
+#define SITD_RESULTS_ERR              (1 << 6)
+#define SITD_RESULTS_DBERR            (1 << 5)
+#define SITD_RESULTS_BABBLE           (1 << 4)
+#define SITD_RESULTS_XACTERR          (1 << 3)
+#define SITD_RESULTS_MISSEDUF         (1 << 2)
+#define SITD_RESULTS_SPLITXSTATE      (1 << 1)
+
+    uint32_t bufptr[2];
+#define SITD_BUFPTR_MASK              0xfffff000
+#define SITD_BUFPTR_CURROFF_MASK      0x00000fff
+#define SITD_BUFPTR_TPOS_MASK         0x00000018
+#define SITD_BUFPTR_TPOS_SH           3
+#define SITD_BUFPTR_TCNT_MASK         0x00000007
+
+    uint32_t backptr;                 /* Standard next link pointer */
+} EHCIsitd;
+
+/*  EHCI spec version 1.0 Section 3.5
+ */
+typedef struct EHCIqtd {
+    uint32_t next;                    /* Standard next link pointer */
+    uint32_t altnext;                 /* Standard next link pointer */
+    uint32_t token;
+#define QTD_TOKEN_DTOGGLE             (1 << 31)
+#define QTD_TOKEN_TBYTES_MASK         0x7fff0000
+#define QTD_TOKEN_TBYTES_SH           16
+#define QTD_TOKEN_IOC                 (1 << 15)
+#define QTD_TOKEN_CPAGE_MASK          0x00007000
+#define QTD_TOKEN_CPAGE_SH            12
+#define QTD_TOKEN_CERR_MASK           0x00000c00
+#define QTD_TOKEN_CERR_SH             10
+#define QTD_TOKEN_PID_MASK            0x00000300
+#define QTD_TOKEN_PID_SH              8
+#define QTD_TOKEN_ACTIVE              (1 << 7)
+#define QTD_TOKEN_HALT                (1 << 6)
+#define QTD_TOKEN_DBERR               (1 << 5)
+#define QTD_TOKEN_BABBLE              (1 << 4)
+#define QTD_TOKEN_XACTERR             (1 << 3)
+#define QTD_TOKEN_MISSEDUF            (1 << 2)
+#define QTD_TOKEN_SPLITXSTATE         (1 << 1)
+#define QTD_TOKEN_PING                (1 << 0)
+
+    uint32_t bufptr[5];               /* Standard buffer pointer */
+#define QTD_BUFPTR_MASK               0xfffff000
+#define QTD_BUFPTR_SH                 12
+} EHCIqtd;
+
+/*  EHCI spec version 1.0 Section 3.6
+ */
+typedef struct EHCIqh {
+    uint32_t next;                    /* Standard next link pointer */
+
+    /* endpoint characteristics */
+    uint32_t epchar;
+#define QH_EPCHAR_RL_MASK             0xf0000000
+#define QH_EPCHAR_RL_SH               28
+#define QH_EPCHAR_C                   (1 << 27)
+#define QH_EPCHAR_MPLEN_MASK          0x07FF0000
+#define QH_EPCHAR_MPLEN_SH            16
+#define QH_EPCHAR_H                   (1 << 15)
+#define QH_EPCHAR_DTC                 (1 << 14)
+#define QH_EPCHAR_EPS_MASK            0x00003000
+#define QH_EPCHAR_EPS_SH              12
+#define EHCI_QH_EPS_FULL              0
+#define EHCI_QH_EPS_LOW               1
+#define EHCI_QH_EPS_HIGH              2
+#define EHCI_QH_EPS_RESERVED          3
+
+#define QH_EPCHAR_EP_MASK             0x00000f00
+#define QH_EPCHAR_EP_SH               8
+#define QH_EPCHAR_I                   (1 << 7)
+#define QH_EPCHAR_DEVADDR_MASK        0x0000007f
+#define QH_EPCHAR_DEVADDR_SH          0
+
+    /* endpoint capabilities */
+    uint32_t epcap;
+#define QH_EPCAP_MULT_MASK            0xc0000000
+#define QH_EPCAP_MULT_SH              30
+#define QH_EPCAP_PORTNUM_MASK         0x3f800000
+#define QH_EPCAP_PORTNUM_SH           23
+#define QH_EPCAP_HUBADDR_MASK         0x007f0000
+#define QH_EPCAP_HUBADDR_SH           16
+#define QH_EPCAP_CMASK_MASK           0x0000ff00
+#define QH_EPCAP_CMASK_SH             8
+#define QH_EPCAP_SMASK_MASK           0x000000ff
+#define QH_EPCAP_SMASK_SH             0
+
+    uint32_t current_qtd;             /* Standard next link pointer */
+    uint32_t next_qtd;                /* Standard next link pointer */
+    uint32_t altnext_qtd;
+#define QH_ALTNEXT_NAKCNT_MASK        0x0000001e
+#define QH_ALTNEXT_NAKCNT_SH          1
+
+    uint32_t token;                   /* Same as QTD token */
+    uint32_t bufptr[5];               /* Standard buffer pointer */
+#define BUFPTR_CPROGMASK_MASK         0x000000ff
+#define BUFPTR_FRAMETAG_MASK          0x0000001f
+#define BUFPTR_SBYTES_MASK            0x00000fe0
+#define BUFPTR_SBYTES_SH              5
+} EHCIqh;
+
+/*  EHCI spec version 1.0 Section 3.7
+ */
+typedef struct EHCIfstn {
+    uint32_t next;                    /* Standard next link pointer */
+    uint32_t backptr;                 /* Standard next link pointer */
+} EHCIfstn;
+
+enum async_state {
+    EHCI_ASYNC_NONE = 0,
+    EHCI_ASYNC_INITIALIZED,
+    EHCI_ASYNC_INFLIGHT,
+    EHCI_ASYNC_FINISHED,
+};
+
+struct EHCIPacket {
+    EHCIQueue *queue;
+    QTAILQ_ENTRY(EHCIPacket) next;
+
+    EHCIqtd qtd;           /* copy of current QTD (being worked on) */
+    uint32_t qtdaddr;      /* address QTD read from                 */
+
+    USBPacket packet;
+    QEMUSGList sgl;
+    int pid;
+    enum async_state async;
+};
+
+struct EHCIQueue {
+    EHCIState *ehci;
+    QTAILQ_ENTRY(EHCIQueue) next;
+    uint32_t seen;
+    uint64_t ts;
+    int async;
+    int transact_ctr;
+
+    /* cached data from guest - needs to be flushed
+     * when guest removes an entry (doorbell, handshake sequence)
+     */
+    EHCIqh qh;             /* copy of current QH (being worked on) */
+    uint32_t qhaddr;       /* address QH read from                 */
+    uint32_t qtdaddr;      /* address QTD read from                */
+    USBDevice *dev;
+    QTAILQ_HEAD(pkts_head, EHCIPacket) packets;
+};
+
+typedef QTAILQ_HEAD(EHCIQueueHead, EHCIQueue) EHCIQueueHead;
+
+struct EHCIState {
+    USBBus bus;
+    qemu_irq irq;
+    MemoryRegion mem;
+    DMAContext *dma;
+    MemoryRegion mem_caps;
+    MemoryRegion mem_opreg;
+    MemoryRegion mem_ports;
+    int companion_count;
+    uint16_t capsbase;
+    uint16_t opregbase;
+
+    /* properties */
+    uint32_t maxframes;
+
+    /*
+     *  EHCI spec version 1.0 Section 2.3
+     *  Host Controller Operational Registers
+     */
+    uint8_t caps[CAPA_SIZE];
+    union {
+        uint32_t opreg[PORTSC_BEGIN/sizeof(uint32_t)];
+        struct {
+            uint32_t usbcmd;
+            uint32_t usbsts;
+            uint32_t usbintr;
+            uint32_t frindex;
+            uint32_t ctrldssegment;
+            uint32_t periodiclistbase;
+            uint32_t asynclistaddr;
+            uint32_t notused[9];
+            uint32_t configflag;
+        };
+    };
+    uint32_t portsc[NB_PORTS];
+
+    /*
+     *  Internal states, shadow registers, etc
+     */
+    QEMUTimer *frame_timer;
+    QEMUBH *async_bh;
+    uint32_t astate;         /* Current state in asynchronous schedule */
+    uint32_t pstate;         /* Current state in periodic schedule     */
+    USBPort ports[NB_PORTS];
+    USBPort *companion_ports[NB_PORTS];
+    uint32_t usbsts_pending;
+    uint32_t usbsts_frindex;
+    EHCIQueueHead aqueues;
+    EHCIQueueHead pqueues;
+
+    /* which address to look at next */
+    uint32_t a_fetch_addr;
+    uint32_t p_fetch_addr;
+
+    USBPacket ipacket;
+    QEMUSGList isgl;
+
+    uint64_t last_run_ns;
+    uint32_t async_stepdown;
+    bool int_req_by_async;
+};
+
+extern const VMStateDescription vmstate_ehci;
+
+void usb_ehci_initfn(EHCIState *s, DeviceState *dev);
diff --git a/hw/usb/hcd-musb.c b/hw/usb/hcd-musb.c
index 0bb5c7b19..c707f7a2b 100644
--- a/hw/usb/hcd-musb.c
+++ b/hw/usb/hcd-musb.c
@@ -607,7 +607,6 @@ static void musb_packet(MUSBState *s, MUSBEndPoint *ep,
 {
     USBDevice *dev;
     USBEndpoint *uep;
-    int ret;
     int idx = epnum && dir;
     int ttype;
 
@@ -627,19 +626,24 @@ static void musb_packet(MUSBState *s, MUSBEndPoint *ep,
     dev = usb_find_device(&s->port, ep->faddr[idx]);
     uep = usb_ep_get(dev, pid, ep->type[idx] & 0xf);
     usb_packet_setup(&ep->packey[dir].p, pid, uep,
-                     (dev->addr << 16) | (uep->nr << 8) | pid);
+                     (dev->addr << 16) | (uep->nr << 8) | pid, false, true);
     usb_packet_addbuf(&ep->packey[dir].p, ep->buf[idx], len);
     ep->packey[dir].ep = ep;
     ep->packey[dir].dir = dir;
 
-    ret = usb_handle_packet(dev, &ep->packey[dir].p);
+    usb_handle_packet(dev, &ep->packey[dir].p);
 
-    if (ret == USB_RET_ASYNC) {
+    if (ep->packey[dir].p.status == USB_RET_ASYNC) {
+        usb_device_flush_ep_queue(dev, uep);
         ep->status[dir] = len;
         return;
     }
 
-    ep->status[dir] = ret;
+    if (ep->packey[dir].p.status == USB_RET_SUCCESS) {
+        ep->status[dir] = ep->packey[dir].p.actual_length;
+    } else {
+        ep->status[dir] = ep->packey[dir].p.status;
+    }
     musb_schedule_cb(&s->port, &ep->packey[dir].p);
 }
 
@@ -753,7 +757,6 @@ static void musb_rx_packet_complete(USBPacket *packey, void *opaque)
 
     if (ep->status[1] == USB_RET_STALL) {
         ep->status[1] = 0;
-        packey->result = 0;
 
         ep->csr[1] |= MGC_M_RXCSR_H_RXSTALL;
         if (!epnum)
@@ -792,14 +795,12 @@ static void musb_rx_packet_complete(USBPacket *packey, void *opaque)
     /* TODO: check len for over/underruns of an OUT packet?  */
     /* TODO: perhaps make use of e->ext_size[1] here.  */
 
-    packey->result = ep->status[1];
-
     if (!(ep->csr[1] & (MGC_M_RXCSR_H_RXSTALL | MGC_M_RXCSR_DATAERROR))) {
         ep->csr[1] |= MGC_M_RXCSR_FIFOFULL | MGC_M_RXCSR_RXPKTRDY;
         if (!epnum)
             ep->csr[0] |= MGC_M_CSR0_RXPKTRDY;
 
-        ep->rxcount = packey->result; /* XXX: MIN(packey->len, ep->maxp[1]); */
+        ep->rxcount = ep->status[1]; /* XXX: MIN(packey->len, ep->maxp[1]); */
         /* In DMA mode: assert DMA request for this EP */
     }
 
@@ -1236,7 +1237,7 @@ static void musb_ep_writeh(void *opaque, int ep, int addr, uint16_t value)
 }
 
 /* Generic control */
-static uint32_t musb_readb(void *opaque, target_phys_addr_t addr)
+static uint32_t musb_readb(void *opaque, hwaddr addr)
 {
     MUSBState *s = (MUSBState *) opaque;
     int ep, i;
@@ -1298,7 +1299,7 @@ static uint32_t musb_readb(void *opaque, target_phys_addr_t addr)
     };
 }
 
-static void musb_writeb(void *opaque, target_phys_addr_t addr, uint32_t value)
+static void musb_writeb(void *opaque, hwaddr addr, uint32_t value)
 {
     MUSBState *s = (MUSBState *) opaque;
     int ep;
@@ -1385,7 +1386,7 @@ static void musb_writeb(void *opaque, target_phys_addr_t addr, uint32_t value)
     };
 }
 
-static uint32_t musb_readh(void *opaque, target_phys_addr_t addr)
+static uint32_t musb_readh(void *opaque, hwaddr addr)
 {
     MUSBState *s = (MUSBState *) opaque;
     int ep, i;
@@ -1439,7 +1440,7 @@ static uint32_t musb_readh(void *opaque, target_phys_addr_t addr)
     };
 }
 
-static void musb_writeh(void *opaque, target_phys_addr_t addr, uint32_t value)
+static void musb_writeh(void *opaque, hwaddr addr, uint32_t value)
 {
     MUSBState *s = (MUSBState *) opaque;
     int ep;
@@ -1495,7 +1496,7 @@ static void musb_writeh(void *opaque, target_phys_addr_t addr, uint32_t value)
     };
 }
 
-static uint32_t musb_readw(void *opaque, target_phys_addr_t addr)
+static uint32_t musb_readw(void *opaque, hwaddr addr)
 {
     MUSBState *s = (MUSBState *) opaque;
     int ep;
@@ -1513,7 +1514,7 @@ static uint32_t musb_readw(void *opaque, target_phys_addr_t addr)
     };
 }
 
-static void musb_writew(void *opaque, target_phys_addr_t addr, uint32_t value)
+static void musb_writew(void *opaque, hwaddr addr, uint32_t value)
 {
     MUSBState *s = (MUSBState *) opaque;
     int ep;
diff --git a/hw/usb/hcd-ohci.c b/hw/usb/hcd-ohci.c
index c36184ae4..e16a2ecab 100644
--- a/hw/usb/hcd-ohci.c
+++ b/hw/usb/hcd-ohci.c
@@ -807,18 +807,24 @@ static int ohci_service_iso_td(OHCIState *ohci, struct ohci_ed *ed,
                          DMA_DIRECTION_TO_DEVICE);
     }
 
-    if (completion) {
-        ret = ohci->usb_packet.result;
-    } else {
+    if (!completion) {
+        bool int_req = relative_frame_number == frame_count &&
+                       OHCI_BM(iso_td.flags, TD_DI) == 0;
         dev = ohci_find_device(ohci, OHCI_BM(ed->flags, ED_FA));
         ep = usb_ep_get(dev, pid, OHCI_BM(ed->flags, ED_EN));
-        usb_packet_setup(&ohci->usb_packet, pid, ep, addr);
+        usb_packet_setup(&ohci->usb_packet, pid, ep, addr, false, int_req);
         usb_packet_addbuf(&ohci->usb_packet, ohci->usb_buf, len);
-        ret = usb_handle_packet(dev, &ohci->usb_packet);
-        if (ret == USB_RET_ASYNC) {
+        usb_handle_packet(dev, &ohci->usb_packet);
+        if (ohci->usb_packet.status == USB_RET_ASYNC) {
+            usb_device_flush_ep_queue(dev, ep);
             return 1;
         }
     }
+    if (ohci->usb_packet.status == USB_RET_SUCCESS) {
+        ret = ohci->usb_packet.actual_length;
+    } else {
+        ret = ohci->usb_packet.status;
+    }
 
 #ifdef DEBUG_ISOCH
     printf("so 0x%.8x eo 0x%.8x\nsa 0x%.8x ea 0x%.8x\ndir %s len %zu ret %d\n",
@@ -994,7 +1000,6 @@ static int ohci_service_td(OHCIState *ohci, struct ohci_ed *ed)
     }
 #endif
     if (completion) {
-        ret = ohci->usb_packet.result;
         ohci->async_td = 0;
         ohci->async_complete = 0;
     } else {
@@ -1011,17 +1016,25 @@ static int ohci_service_td(OHCIState *ohci, struct ohci_ed *ed)
         }
         dev = ohci_find_device(ohci, OHCI_BM(ed->flags, ED_FA));
         ep = usb_ep_get(dev, pid, OHCI_BM(ed->flags, ED_EN));
-        usb_packet_setup(&ohci->usb_packet, pid, ep, addr);
+        usb_packet_setup(&ohci->usb_packet, pid, ep, addr, !flag_r,
+                         OHCI_BM(td.flags, TD_DI) == 0);
         usb_packet_addbuf(&ohci->usb_packet, ohci->usb_buf, pktlen);
-        ret = usb_handle_packet(dev, &ohci->usb_packet);
+        usb_handle_packet(dev, &ohci->usb_packet);
 #ifdef DEBUG_PACKET
-        DPRINTF("ret=%d\n", ret);
+        DPRINTF("status=%d\n", ohci->usb_packet.status);
 #endif
-        if (ret == USB_RET_ASYNC) {
+        if (ohci->usb_packet.status == USB_RET_ASYNC) {
+            usb_device_flush_ep_queue(dev, ep);
             ohci->async_td = addr;
             return 1;
         }
     }
+    if (ohci->usb_packet.status == USB_RET_SUCCESS) {
+        ret = ohci->usb_packet.actual_length;
+    } else {
+        ret = ohci->usb_packet.status;
+    }
+
     if (ret >= 0) {
         if (dir == OHCI_TD_DIR_IN) {
             ohci_copy_td(ohci, &td, ohci->usb_buf, ret,
@@ -1470,12 +1483,10 @@ static void ohci_port_set_status(OHCIState *ohci, int portnum, uint32_t val)
 
     if (old_state != port->ctrl)
         ohci_set_interrupt(ohci, OHCI_INTR_RHSC);
-
-    return;
 }
 
 static uint64_t ohci_mem_read(void *opaque,
-                              target_phys_addr_t addr,
+                              hwaddr addr,
                               unsigned size)
 {
     OHCIState *ohci = opaque;
@@ -1598,7 +1609,7 @@ static uint64_t ohci_mem_read(void *opaque,
 }
 
 static void ohci_mem_write(void *opaque,
-                           target_phys_addr_t addr,
+                           hwaddr addr,
                            uint64_t val,
                            unsigned size)
 {
@@ -1848,7 +1859,7 @@ static int ohci_init_pxa(SysBusDevice *dev)
 
     /* Cannot fail as we pass NULL for masterbus */
     usb_ohci_init(&s->ohci, &dev->qdev, s->num_ports, s->dma_offset, NULL, 0,
-                  NULL);
+                  &dma_context_memory);
     sysbus_init_irq(dev, &s->ohci.irq);
     sysbus_init_mmio(dev, &s->ohci.mem);
 
@@ -1871,6 +1882,7 @@ static void ohci_pci_class_init(ObjectClass *klass, void *data)
     k->vendor_id = PCI_VENDOR_ID_APPLE;
     k->device_id = PCI_DEVICE_ID_APPLE_IPID_USB;
     k->class_id = PCI_CLASS_SERIAL_USB;
+    k->no_hotplug = 1;
     dc->desc = "Apple USB Controller";
     dc->props = ohci_pci_properties;
 }
diff --git a/hw/usb/hcd-uhci.c b/hw/usb/hcd-uhci.c
index b0db92145..d053791de 100644
--- a/hw/usb/hcd-uhci.c
+++ b/hw/usb/hcd-uhci.c
@@ -88,6 +88,23 @@ enum {
 typedef struct UHCIState UHCIState;
 typedef struct UHCIAsync UHCIAsync;
 typedef struct UHCIQueue UHCIQueue;
+typedef struct UHCIInfo UHCIInfo;
+typedef struct UHCIPCIDeviceClass UHCIPCIDeviceClass;
+
+struct UHCIInfo {
+    const char *name;
+    uint16_t   vendor_id;
+    uint16_t   device_id;
+    uint8_t    revision;
+    uint8_t    irq_pin;
+    int        (*initfn)(PCIDevice *dev);
+    bool       unplug;
+};
+
+struct UHCIPCIDeviceClass {
+    PCIDeviceClass parent_class;
+    UHCIInfo       info;
+};
 
 /* 
  * Pending async transaction.
@@ -100,16 +117,17 @@ struct UHCIAsync {
     QEMUSGList sgl;
     UHCIQueue *queue;
     QTAILQ_ENTRY(UHCIAsync) next;
-    uint32_t  td;
-    uint8_t   isoc;
+    uint32_t  td_addr;
     uint8_t   done;
 };
 
 struct UHCIQueue {
+    uint32_t  qh_addr;
     uint32_t  token;
     UHCIState *uhci;
+    USBEndpoint *ep;
     QTAILQ_ENTRY(UHCIQueue) next;
-    QTAILQ_HEAD(, UHCIAsync) asyncs;
+    QTAILQ_HEAD(asyncs_head, UHCIAsync) asyncs;
     int8_t    valid;
 };
 
@@ -134,6 +152,7 @@ struct UHCIState {
     QEMUBH *bh;
     uint32_t frame_bytes;
     uint32_t frame_bandwidth;
+    bool completions_only;
     UHCIPort ports[NB_PORTS];
 
     /* Interrupts that should be raised at the end of the current frame.  */
@@ -161,57 +180,94 @@ typedef struct UHCI_QH {
     uint32_t el_link;
 } UHCI_QH;
 
+static void uhci_async_cancel(UHCIAsync *async);
+static void uhci_queue_fill(UHCIQueue *q, UHCI_TD *td);
+
 static inline int32_t uhci_queue_token(UHCI_TD *td)
 {
-    /* covers ep, dev, pid -> identifies the endpoint */
-    return td->token & 0x7ffff;
+    if ((td->token & (0xf << 15)) == 0) {
+        /* ctrl ep, cover ep and dev, not pid! */
+        return td->token & 0x7ff00;
+    } else {
+        /* covers ep, dev, pid -> identifies the endpoint */
+        return td->token & 0x7ffff;
+    }
 }
 
-static UHCIQueue *uhci_queue_get(UHCIState *s, UHCI_TD *td)
+static UHCIQueue *uhci_queue_new(UHCIState *s, uint32_t qh_addr, UHCI_TD *td,
+                                 USBEndpoint *ep)
 {
-    uint32_t token = uhci_queue_token(td);
     UHCIQueue *queue;
 
-    QTAILQ_FOREACH(queue, &s->queues, next) {
-        if (queue->token == token) {
-            return queue;
-        }
-    }
-
     queue = g_new0(UHCIQueue, 1);
     queue->uhci = s;
-    queue->token = token;
+    queue->qh_addr = qh_addr;
+    queue->token = uhci_queue_token(td);
+    queue->ep = ep;
     QTAILQ_INIT(&queue->asyncs);
     QTAILQ_INSERT_HEAD(&s->queues, queue, next);
+    /* valid needs to be large enough to handle 10 frame delay
+     * for initial isochronous requests */
+    queue->valid = 32;
     trace_usb_uhci_queue_add(queue->token);
     return queue;
 }
 
-static void uhci_queue_free(UHCIQueue *queue)
+static void uhci_queue_free(UHCIQueue *queue, const char *reason)
 {
     UHCIState *s = queue->uhci;
+    UHCIAsync *async;
+
+    while (!QTAILQ_EMPTY(&queue->asyncs)) {
+        async = QTAILQ_FIRST(&queue->asyncs);
+        uhci_async_cancel(async);
+    }
 
-    trace_usb_uhci_queue_del(queue->token);
+    trace_usb_uhci_queue_del(queue->token, reason);
     QTAILQ_REMOVE(&s->queues, queue, next);
     g_free(queue);
 }
 
-static UHCIAsync *uhci_async_alloc(UHCIQueue *queue, uint32_t addr)
+static UHCIQueue *uhci_queue_find(UHCIState *s, UHCI_TD *td)
+{
+    uint32_t token = uhci_queue_token(td);
+    UHCIQueue *queue;
+
+    QTAILQ_FOREACH(queue, &s->queues, next) {
+        if (queue->token == token) {
+            return queue;
+        }
+    }
+    return NULL;
+}
+
+static bool uhci_queue_verify(UHCIQueue *queue, uint32_t qh_addr, UHCI_TD *td,
+                              uint32_t td_addr, bool queuing)
+{
+    UHCIAsync *first = QTAILQ_FIRST(&queue->asyncs);
+
+    return queue->qh_addr == qh_addr &&
+           queue->token == uhci_queue_token(td) &&
+           (queuing || !(td->ctrl & TD_CTRL_ACTIVE) || first == NULL ||
+            first->td_addr == td_addr);
+}
+
+static UHCIAsync *uhci_async_alloc(UHCIQueue *queue, uint32_t td_addr)
 {
     UHCIAsync *async = g_new0(UHCIAsync, 1);
 
     async->queue = queue;
-    async->td = addr;
+    async->td_addr = td_addr;
     usb_packet_init(&async->packet);
     pci_dma_sglist_init(&async->sgl, &queue->uhci->dev, 1);
-    trace_usb_uhci_packet_add(async->queue->token, async->td);
+    trace_usb_uhci_packet_add(async->queue->token, async->td_addr);
 
     return async;
 }
 
 static void uhci_async_free(UHCIAsync *async)
 {
-    trace_usb_uhci_packet_del(async->queue->token, async->td);
+    trace_usb_uhci_packet_del(async->queue->token, async->td_addr);
     usb_packet_cleanup(&async->packet);
     qemu_sglist_destroy(&async->sgl);
     g_free(async);
@@ -221,21 +277,24 @@ static void uhci_async_link(UHCIAsync *async)
 {
     UHCIQueue *queue = async->queue;
     QTAILQ_INSERT_TAIL(&queue->asyncs, async, next);
-    trace_usb_uhci_packet_link_async(async->queue->token, async->td);
+    trace_usb_uhci_packet_link_async(async->queue->token, async->td_addr);
 }
 
 static void uhci_async_unlink(UHCIAsync *async)
 {
     UHCIQueue *queue = async->queue;
     QTAILQ_REMOVE(&queue->asyncs, async, next);
-    trace_usb_uhci_packet_unlink_async(async->queue->token, async->td);
+    trace_usb_uhci_packet_unlink_async(async->queue->token, async->td_addr);
 }
 
 static void uhci_async_cancel(UHCIAsync *async)
 {
-    trace_usb_uhci_packet_cancel(async->queue->token, async->td, async->done);
+    uhci_async_unlink(async);
+    trace_usb_uhci_packet_cancel(async->queue->token, async->td_addr,
+                                 async->done);
     if (!async->done)
         usb_cancel_packet(&async->packet);
+    usb_packet_unmap(&async->packet, &async->sgl);
     uhci_async_free(async);
 }
 
@@ -258,34 +317,21 @@ static void uhci_async_validate_begin(UHCIState *s)
 static void uhci_async_validate_end(UHCIState *s)
 {
     UHCIQueue *queue, *n;
-    UHCIAsync *async;
 
     QTAILQ_FOREACH_SAFE(queue, &s->queues, next, n) {
-        if (queue->valid > 0) {
-            continue;
+        if (!queue->valid) {
+            uhci_queue_free(queue, "validate-end");
         }
-        while (!QTAILQ_EMPTY(&queue->asyncs)) {
-            async = QTAILQ_FIRST(&queue->asyncs);
-            uhci_async_unlink(async);
-            uhci_async_cancel(async);
-        }
-        uhci_queue_free(queue);
     }
 }
 
 static void uhci_async_cancel_device(UHCIState *s, USBDevice *dev)
 {
-    UHCIQueue *queue;
-    UHCIAsync *curr, *n;
+    UHCIQueue *queue, *n;
 
-    QTAILQ_FOREACH(queue, &s->queues, next) {
-        QTAILQ_FOREACH_SAFE(curr, &queue->asyncs, next, n) {
-            if (!usb_packet_is_inflight(&curr->packet) ||
-                curr->packet.ep->dev != dev) {
-                continue;
-            }
-            uhci_async_unlink(curr);
-            uhci_async_cancel(curr);
+    QTAILQ_FOREACH_SAFE(queue, &s->queues, next, n) {
+        if (queue->ep->dev == dev) {
+            uhci_queue_free(queue, "cancel-device");
         }
     }
 }
@@ -293,38 +339,24 @@ static void uhci_async_cancel_device(UHCIState *s, USBDevice *dev)
 static void uhci_async_cancel_all(UHCIState *s)
 {
     UHCIQueue *queue, *nq;
-    UHCIAsync *curr, *n;
 
     QTAILQ_FOREACH_SAFE(queue, &s->queues, next, nq) {
-        QTAILQ_FOREACH_SAFE(curr, &queue->asyncs, next, n) {
-            uhci_async_unlink(curr);
-            uhci_async_cancel(curr);
-        }
-        uhci_queue_free(queue);
+        uhci_queue_free(queue, "cancel-all");
     }
 }
 
-static UHCIAsync *uhci_async_find_td(UHCIState *s, uint32_t addr, UHCI_TD *td)
+static UHCIAsync *uhci_async_find_td(UHCIState *s, uint32_t td_addr)
 {
-    uint32_t token = uhci_queue_token(td);
     UHCIQueue *queue;
     UHCIAsync *async;
 
     QTAILQ_FOREACH(queue, &s->queues, next) {
-        if (queue->token == token) {
-            break;
-        }
-    }
-    if (queue == NULL) {
-        return NULL;
-    }
-
-    QTAILQ_FOREACH(async, &queue->asyncs, next) {
-        if (async->td == addr) {
-            return async;
+        QTAILQ_FOREACH(async, &queue->asyncs, next) {
+            if (async->td_addr == td_addr) {
+                return async;
+            }
         }
     }
-
     return NULL;
 }
 
@@ -524,6 +556,10 @@ static void uhci_ioport_writew(void *opaque, uint32_t addr, uint32_t val)
                 }
             }
             port->ctrl &= UHCI_PORT_READ_ONLY;
+            /* enabled may only be set if a device is connected */
+            if (!(port->ctrl & UHCI_PORT_CCS)) {
+                val &= ~UHCI_PORT_EN;
+            }
             port->ctrl |= (val & ~UHCI_PORT_READ_ONLY);
             /* some bits are reset when a '1' is written to them */
             port->ctrl &= ~(val & UHCI_PORT_WRITE_CLEAR);
@@ -695,30 +731,75 @@ static USBDevice *uhci_find_device(UHCIState *s, uint8_t addr)
     return NULL;
 }
 
-static void uhci_async_complete(USBPort *port, USBPacket *packet);
-static void uhci_process_frame(UHCIState *s);
+static void uhci_read_td(UHCIState *s, UHCI_TD *td, uint32_t link)
+{
+    pci_dma_read(&s->dev, link & ~0xf, td, sizeof(*td));
+    le32_to_cpus(&td->link);
+    le32_to_cpus(&td->ctrl);
+    le32_to_cpus(&td->token);
+    le32_to_cpus(&td->buffer);
+}
+
+static int uhci_handle_td_error(UHCIState *s, UHCI_TD *td, uint32_t td_addr,
+                                int status, uint32_t *int_mask)
+{
+    uint32_t queue_token = uhci_queue_token(td);
+    int ret;
+
+    switch (status) {
+    case USB_RET_NAK:
+        td->ctrl |= TD_CTRL_NAK;
+        return TD_RESULT_NEXT_QH;
+
+    case USB_RET_STALL:
+        td->ctrl |= TD_CTRL_STALL;
+        trace_usb_uhci_packet_complete_stall(queue_token, td_addr);
+        ret = TD_RESULT_NEXT_QH;
+        break;
+
+    case USB_RET_BABBLE:
+        td->ctrl |= TD_CTRL_BABBLE | TD_CTRL_STALL;
+        /* frame interrupted */
+        trace_usb_uhci_packet_complete_babble(queue_token, td_addr);
+        ret = TD_RESULT_STOP_FRAME;
+        break;
+
+    case USB_RET_IOERROR:
+    case USB_RET_NODEV:
+    default:
+        td->ctrl |= TD_CTRL_TIMEOUT;
+        td->ctrl &= ~(3 << TD_CTRL_ERROR_SHIFT);
+        trace_usb_uhci_packet_complete_error(queue_token, td_addr);
+        ret = TD_RESULT_NEXT_QH;
+        break;
+    }
+
+    td->ctrl &= ~TD_CTRL_ACTIVE;
+    s->status |= UHCI_STS_USBERR;
+    if (td->ctrl & TD_CTRL_IOC) {
+        *int_mask |= 0x01;
+    }
+    uhci_update_irq(s);
+    return ret;
+}
 
-/* return -1 if fatal error (frame must be stopped)
-          0 if TD successful
-          1 if TD unsuccessful or inactive
-*/
 static int uhci_complete_td(UHCIState *s, UHCI_TD *td, UHCIAsync *async, uint32_t *int_mask)
 {
-    int len = 0, max_len, err, ret;
+    int len = 0, max_len;
     uint8_t pid;
 
     max_len = ((td->token >> 21) + 1) & 0x7ff;
     pid = td->token & 0xff;
 
-    ret = async->packet.result;
-
     if (td->ctrl & TD_CTRL_IOS)
         td->ctrl &= ~TD_CTRL_ACTIVE;
 
-    if (ret < 0)
-        goto out;
+    if (async->packet.status != USB_RET_SUCCESS) {
+        return uhci_handle_td_error(s, td, async->td_addr,
+                                    async->packet.status, int_mask);
+    }
 
-    len = async->packet.result;
+    len = async->packet.actual_length;
     td->ctrl = (td->ctrl & ~0x7ff) | ((len - 1) & 0x7ff);
 
     /* The NAK bit may have been set by a previous frame, so clear it
@@ -729,179 +810,151 @@ static int uhci_complete_td(UHCIState *s, UHCI_TD *td, UHCIAsync *async, uint32_
         *int_mask |= 0x01;
 
     if (pid == USB_TOKEN_IN) {
-        if (len > max_len) {
-            ret = USB_RET_BABBLE;
-            goto out;
-        }
-
         if ((td->ctrl & TD_CTRL_SPD) && len < max_len) {
             *int_mask |= 0x02;
             /* short packet: do not update QH */
             trace_usb_uhci_packet_complete_shortxfer(async->queue->token,
-                                                    async->td);
+                                                     async->td_addr);
             return TD_RESULT_NEXT_QH;
         }
     }
 
     /* success */
-    trace_usb_uhci_packet_complete_success(async->queue->token, async->td);
+    trace_usb_uhci_packet_complete_success(async->queue->token,
+                                           async->td_addr);
     return TD_RESULT_COMPLETE;
+}
 
-out:
-    /*
-     * We should not do any further processing on a queue with errors!
-     * This is esp. important for bulk endpoints with pipelining enabled
-     * (redirection to a real USB device), where we must cancel all the
-     * transfers after this one so that:
-     * 1) If they've completed already, they are not processed further
-     *    causing more stalls, originating from the same failed transfer
-     * 2) If still in flight, they are cancelled before the guest does
-     *    a clear stall, otherwise the guest and device can loose sync!
-     */
-    while (!QTAILQ_EMPTY(&async->queue->asyncs)) {
-        UHCIAsync *as = QTAILQ_FIRST(&async->queue->asyncs);
-        uhci_async_unlink(as);
-        uhci_async_cancel(as);
-    }
-
-    switch(ret) {
-    case USB_RET_STALL:
-        td->ctrl |= TD_CTRL_STALL;
-        td->ctrl &= ~TD_CTRL_ACTIVE;
-        s->status |= UHCI_STS_USBERR;
-        if (td->ctrl & TD_CTRL_IOC) {
-            *int_mask |= 0x01;
-        }
-        uhci_update_irq(s);
-        trace_usb_uhci_packet_complete_stall(async->queue->token, async->td);
-        return TD_RESULT_NEXT_QH;
+static int uhci_handle_td(UHCIState *s, UHCIQueue *q, uint32_t qh_addr,
+                          UHCI_TD *td, uint32_t td_addr, uint32_t *int_mask)
+{
+    int ret, max_len;
+    bool spd;
+    bool queuing = (q != NULL);
+    uint8_t pid = td->token & 0xff;
+    UHCIAsync *async = uhci_async_find_td(s, td_addr);
 
-    case USB_RET_BABBLE:
-        td->ctrl |= TD_CTRL_BABBLE | TD_CTRL_STALL;
-        td->ctrl &= ~TD_CTRL_ACTIVE;
-        s->status |= UHCI_STS_USBERR;
-        if (td->ctrl & TD_CTRL_IOC) {
-            *int_mask |= 0x01;
+    if (async) {
+        if (uhci_queue_verify(async->queue, qh_addr, td, td_addr, queuing)) {
+            assert(q == NULL || q == async->queue);
+            q = async->queue;
+        } else {
+            uhci_queue_free(async->queue, "guest re-used pending td");
+            async = NULL;
         }
-        uhci_update_irq(s);
-        /* frame interrupted */
-        trace_usb_uhci_packet_complete_babble(async->queue->token, async->td);
-        return TD_RESULT_STOP_FRAME;
-
-    case USB_RET_NAK:
-        td->ctrl |= TD_CTRL_NAK;
-        if (pid == USB_TOKEN_SETUP)
-            break;
-        return TD_RESULT_NEXT_QH;
-
-    case USB_RET_IOERROR:
-    case USB_RET_NODEV:
-    default:
-	break;
     }
 
-    /* Retry the TD if error count is not zero */
-
-    td->ctrl |= TD_CTRL_TIMEOUT;
-    err = (td->ctrl >> TD_CTRL_ERROR_SHIFT) & 3;
-    if (err != 0) {
-        err--;
-        if (err == 0) {
-            td->ctrl &= ~TD_CTRL_ACTIVE;
-            s->status |= UHCI_STS_USBERR;
-            if (td->ctrl & TD_CTRL_IOC)
-                *int_mask |= 0x01;
-            uhci_update_irq(s);
-            trace_usb_uhci_packet_complete_error(async->queue->token,
-                                                 async->td);
+    if (q == NULL) {
+        q = uhci_queue_find(s, td);
+        if (q && !uhci_queue_verify(q, qh_addr, td, td_addr, queuing)) {
+            uhci_queue_free(q, "guest re-used qh");
+            q = NULL;
         }
     }
-    td->ctrl = (td->ctrl & ~(3 << TD_CTRL_ERROR_SHIFT)) |
-        (err << TD_CTRL_ERROR_SHIFT);
-    return TD_RESULT_NEXT_QH;
-}
 
-static int uhci_handle_td(UHCIState *s, uint32_t addr, UHCI_TD *td,
-                          uint32_t *int_mask, bool queuing)
-{
-    UHCIAsync *async;
-    int len = 0, max_len;
-    uint8_t pid;
-    USBDevice *dev;
-    USBEndpoint *ep;
+    if (q) {
+        q->valid = 32;
+    }
 
     /* Is active ? */
-    if (!(td->ctrl & TD_CTRL_ACTIVE))
+    if (!(td->ctrl & TD_CTRL_ACTIVE)) {
+        if (async) {
+            /* Guest marked a pending td non-active, cancel the queue */
+            uhci_queue_free(async->queue, "pending td non-active");
+        }
+        /*
+         * ehci11d spec page 22: "Even if the Active bit in the TD is already
+         * cleared when the TD is fetched ... an IOC interrupt is generated"
+         */
+        if (td->ctrl & TD_CTRL_IOC) {
+                *int_mask |= 0x01;
+        }
         return TD_RESULT_NEXT_QH;
+    }
 
-    async = uhci_async_find_td(s, addr, td);
     if (async) {
-        /* Already submitted */
-        async->queue->valid = 32;
-
-        if (!async->done)
-            return TD_RESULT_ASYNC_CONT;
         if (queuing) {
             /* we are busy filling the queue, we are not prepared
                to consume completed packages then, just leave them
                in async state */
             return TD_RESULT_ASYNC_CONT;
         }
+        if (!async->done) {
+            UHCI_TD last_td;
+            UHCIAsync *last = QTAILQ_LAST(&async->queue->asyncs, asyncs_head);
+            /*
+             * While we are waiting for the current td to complete, the guest
+             * may have added more tds to the queue. Note we re-read the td
+             * rather then caching it, as we want to see guest made changes!
+             */
+            uhci_read_td(s, &last_td, last->td_addr);
+            uhci_queue_fill(async->queue, &last_td);
 
+            return TD_RESULT_ASYNC_CONT;
+        }
         uhci_async_unlink(async);
         goto done;
     }
 
+    if (s->completions_only) {
+        return TD_RESULT_ASYNC_CONT;
+    }
+
     /* Allocate new packet */
-    async = uhci_async_alloc(uhci_queue_get(s, td), addr);
+    if (q == NULL) {
+        USBDevice *dev = uhci_find_device(s, (td->token >> 8) & 0x7f);
+        USBEndpoint *ep = usb_ep_get(dev, pid, (td->token >> 15) & 0xf);
 
-    /* valid needs to be large enough to handle 10 frame delay
-     * for initial isochronous requests
-     */
-    async->queue->valid = 32;
-    async->isoc = td->ctrl & TD_CTRL_IOS;
+        if (ep == NULL) {
+            return uhci_handle_td_error(s, td, td_addr, USB_RET_NODEV,
+                                        int_mask);
+        }
+        q = uhci_queue_new(s, qh_addr, td, ep);
+    }
+    async = uhci_async_alloc(q, td_addr);
 
     max_len = ((td->token >> 21) + 1) & 0x7ff;
-    pid = td->token & 0xff;
-
-    dev = uhci_find_device(s, (td->token >> 8) & 0x7f);
-    ep = usb_ep_get(dev, pid, (td->token >> 15) & 0xf);
-    usb_packet_setup(&async->packet, pid, ep, addr);
+    spd = (pid == USB_TOKEN_IN && (td->ctrl & TD_CTRL_SPD) != 0);
+    usb_packet_setup(&async->packet, pid, q->ep, td_addr, spd,
+                     (td->ctrl & TD_CTRL_IOC) != 0);
     qemu_sglist_add(&async->sgl, td->buffer, max_len);
     usb_packet_map(&async->packet, &async->sgl);
 
     switch(pid) {
     case USB_TOKEN_OUT:
     case USB_TOKEN_SETUP:
-        len = usb_handle_packet(dev, &async->packet);
-        if (len >= 0)
-            len = max_len;
+        usb_handle_packet(q->ep->dev, &async->packet);
+        if (async->packet.status == USB_RET_SUCCESS) {
+            async->packet.actual_length = max_len;
+        }
         break;
 
     case USB_TOKEN_IN:
-        len = usb_handle_packet(dev, &async->packet);
+        usb_handle_packet(q->ep->dev, &async->packet);
         break;
 
     default:
         /* invalid pid : frame interrupted */
+        usb_packet_unmap(&async->packet, &async->sgl);
         uhci_async_free(async);
         s->status |= UHCI_STS_HCPERR;
         uhci_update_irq(s);
         return TD_RESULT_STOP_FRAME;
     }
- 
-    if (len == USB_RET_ASYNC) {
+
+    if (async->packet.status == USB_RET_ASYNC) {
         uhci_async_link(async);
+        if (!queuing) {
+            uhci_queue_fill(q, td);
+        }
         return TD_RESULT_ASYNC_START;
     }
 
-    async->packet.result = len;
-
 done:
-    len = uhci_complete_td(s, td, async, int_mask);
+    ret = uhci_complete_td(s, td, async, int_mask);
     usb_packet_unmap(&async->packet, &async->sgl);
     uhci_async_free(async);
-    return len;
+    return ret;
 }
 
 static void uhci_async_complete(USBPort *port, USBPacket *packet)
@@ -909,31 +962,15 @@ static void uhci_async_complete(USBPort *port, USBPacket *packet)
     UHCIAsync *async = container_of(packet, UHCIAsync, packet);
     UHCIState *s = async->queue->uhci;
 
-    if (async->isoc) {
-        UHCI_TD td;
-        uint32_t link = async->td;
-        uint32_t int_mask = 0, val;
-
-        pci_dma_read(&s->dev, link & ~0xf, &td, sizeof(td));
-        le32_to_cpus(&td.link);
-        le32_to_cpus(&td.ctrl);
-        le32_to_cpus(&td.token);
-        le32_to_cpus(&td.buffer);
-
-        uhci_async_unlink(async);
-        uhci_complete_td(s, &td, async, &int_mask);
-        s->pending_int_mask |= int_mask;
-
-        /* update the status bits of the TD */
-        val = cpu_to_le32(td.ctrl);
-        pci_dma_write(&s->dev, (link & ~0xf) + 4, &val, sizeof(val));
-        uhci_async_free(async);
-    } else {
-        async->done = 1;
-        if (s->frame_bytes < s->frame_bandwidth) {
-            qemu_bh_schedule(s->bh);
-        }
+    if (packet->status == USB_RET_REMOVE_FROM_QUEUE) {
+        uhci_async_cancel(async);
+        return;
     }
+
+    async->done = 1;
+    /* Force processing of this packet *now*, needed for migration */
+    s->completions_only = true;
+    qemu_bh_schedule(s->bh);
 }
 
 static int is_valid(uint32_t link)
@@ -978,28 +1015,23 @@ static int qhdb_insert(QhDb *db, uint32_t addr)
     return 0;
 }
 
-static void uhci_fill_queue(UHCIState *s, UHCI_TD *td)
+static void uhci_queue_fill(UHCIQueue *q, UHCI_TD *td)
 {
     uint32_t int_mask = 0;
     uint32_t plink = td->link;
-    uint32_t token = uhci_queue_token(td);
     UHCI_TD ptd;
     int ret;
 
     while (is_valid(plink)) {
-        pci_dma_read(&s->dev, plink & ~0xf, &ptd, sizeof(ptd));
-        le32_to_cpus(&ptd.link);
-        le32_to_cpus(&ptd.ctrl);
-        le32_to_cpus(&ptd.token);
-        le32_to_cpus(&ptd.buffer);
+        uhci_read_td(q->uhci, &ptd, plink);
         if (!(ptd.ctrl & TD_CTRL_ACTIVE)) {
             break;
         }
-        if (uhci_queue_token(&ptd) != token) {
+        if (uhci_queue_token(&ptd) != q->token) {
             break;
         }
         trace_usb_uhci_td_queue(plink & ~0xf, ptd.ctrl, ptd.token);
-        ret = uhci_handle_td(s, plink, &ptd, &int_mask, true);
+        ret = uhci_handle_td(q->uhci, q, q->qh_addr, &ptd, plink, &int_mask);
         if (ret == TD_RESULT_ASYNC_CONT) {
             break;
         }
@@ -1007,6 +1039,7 @@ static void uhci_fill_queue(UHCIState *s, UHCI_TD *td)
         assert(int_mask == 0);
         plink = ptd.link;
     }
+    usb_device_flush_ep_queue(q->ep->dev, q->ep);
 }
 
 static void uhci_process_frame(UHCIState *s)
@@ -1029,7 +1062,7 @@ static void uhci_process_frame(UHCIState *s)
     qhdb_reset(&qhdb);
 
     for (cnt = FRAME_MAX_LOOPS; is_valid(link) && cnt; cnt--) {
-        if (s->frame_bytes >= s->frame_bandwidth) {
+        if (!s->completions_only && s->frame_bytes >= s->frame_bandwidth) {
             /* We've reached the usb 1.1 bandwidth, which is
                1280 bytes/frame, stop processing */
             trace_usb_uhci_frame_stop_bandwidth();
@@ -1075,15 +1108,11 @@ static void uhci_process_frame(UHCIState *s)
         }
 
         /* TD */
-        pci_dma_read(&s->dev, link & ~0xf, &td, sizeof(td));
-        le32_to_cpus(&td.link);
-        le32_to_cpus(&td.ctrl);
-        le32_to_cpus(&td.token);
-        le32_to_cpus(&td.buffer);
+        uhci_read_td(s, &td, link);
         trace_usb_uhci_td_load(curr_qh & ~0xf, link & ~0xf, td.ctrl, td.token);
 
         old_td_ctrl = td.ctrl;
-        ret = uhci_handle_td(s, link, &td, &int_mask, false);
+        ret = uhci_handle_td(s, NULL, curr_qh, &td, link, &int_mask);
         if (old_td_ctrl != td.ctrl) {
             /* update the status bits of the TD */
             val = cpu_to_le32(td.ctrl);
@@ -1102,9 +1131,6 @@ static void uhci_process_frame(UHCIState *s)
 
         case TD_RESULT_ASYNC_START:
             trace_usb_uhci_td_async(curr_qh & ~0xf, link & ~0xf);
-            if (is_valid(td.link)) {
-                uhci_fill_queue(s, &td);
-            }
             link = curr_qh ? qh.link : td.link;
             continue;
 
@@ -1152,6 +1178,7 @@ static void uhci_frame_timer(void *opaque)
     /* prepare the timer for the next frame */
     s->expire_time += (get_ticks_per_sec() / FRAME_TIMER_FREQ);
     s->frame_bytes = 0;
+    s->completions_only = false;
     qemu_bh_cancel(s->bh);
 
     if (!(s->cmd & UHCI_CMD_RS)) {
@@ -1214,6 +1241,7 @@ static USBBusOps uhci_bus_ops = {
 static int usb_uhci_common_initfn(PCIDevice *dev)
 {
     PCIDeviceClass *pc = PCI_DEVICE_GET_CLASS(dev);
+    UHCIPCIDeviceClass *u = container_of(pc, UHCIPCIDeviceClass, parent_class);
     UHCIState *s = DO_UPCAST(UHCIState, dev, dev);
     uint8_t *pci_conf = s->dev.config;
     int i;
@@ -1222,20 +1250,7 @@ static int usb_uhci_common_initfn(PCIDevice *dev)
     /* TODO: reset value should be 0. */
     pci_conf[USB_SBRN] = USB_RELEASE_1; // release number
 
-    switch (pc->device_id) {
-    case PCI_DEVICE_ID_INTEL_82801I_UHCI1:
-        s->irq_pin = 0;  /* A */
-        break;
-    case PCI_DEVICE_ID_INTEL_82801I_UHCI2:
-        s->irq_pin = 1;  /* B */
-        break;
-    case PCI_DEVICE_ID_INTEL_82801I_UHCI3:
-        s->irq_pin = 2;  /* C */
-        break;
-    default:
-        s->irq_pin = 3;  /* D */
-        break;
-    }
+    s->irq_pin = u->info.irq_pin;
     pci_config_set_interrupt_pin(pci_conf, s->irq_pin + 1);
 
     if (s->masterbus) {
@@ -1299,143 +1314,108 @@ static Property uhci_properties[] = {
     DEFINE_PROP_END_OF_LIST(),
 };
 
-static void piix3_uhci_class_init(ObjectClass *klass, void *data)
+static void uhci_class_init(ObjectClass *klass, void *data)
 {
     DeviceClass *dc = DEVICE_CLASS(klass);
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
-
-    k->init = usb_uhci_common_initfn;
-    k->exit = usb_uhci_exit;
-    k->vendor_id = PCI_VENDOR_ID_INTEL;
-    k->device_id = PCI_DEVICE_ID_INTEL_82371SB_2;
-    k->revision = 0x01;
-    k->class_id = PCI_CLASS_SERIAL_USB;
-    dc->vmsd = &vmstate_uhci;
-    dc->props = uhci_properties;
-}
-
-static TypeInfo piix3_uhci_info = {
-    .name          = "piix3-usb-uhci",
-    .parent        = TYPE_PCI_DEVICE,
-    .instance_size = sizeof(UHCIState),
-    .class_init    = piix3_uhci_class_init,
-};
-
-static void piix4_uhci_class_init(ObjectClass *klass, void *data)
-{
-    DeviceClass *dc = DEVICE_CLASS(klass);
-    PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
-
-    k->init = usb_uhci_common_initfn;
-    k->exit = usb_uhci_exit;
-    k->vendor_id = PCI_VENDOR_ID_INTEL;
-    k->device_id = PCI_DEVICE_ID_INTEL_82371AB_2;
-    k->revision = 0x01;
-    k->class_id = PCI_CLASS_SERIAL_USB;
-    dc->vmsd = &vmstate_uhci;
-    dc->props = uhci_properties;
-}
-
-static TypeInfo piix4_uhci_info = {
-    .name          = "piix4-usb-uhci",
-    .parent        = TYPE_PCI_DEVICE,
-    .instance_size = sizeof(UHCIState),
-    .class_init    = piix4_uhci_class_init,
-};
-
-static void vt82c686b_uhci_class_init(ObjectClass *klass, void *data)
-{
-    DeviceClass *dc = DEVICE_CLASS(klass);
-    PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
-
-    k->init = usb_uhci_vt82c686b_initfn;
-    k->exit = usb_uhci_exit;
-    k->vendor_id = PCI_VENDOR_ID_VIA;
-    k->device_id = PCI_DEVICE_ID_VIA_UHCI;
-    k->revision = 0x01;
-    k->class_id = PCI_CLASS_SERIAL_USB;
+    UHCIPCIDeviceClass *u = container_of(k, UHCIPCIDeviceClass, parent_class);
+    UHCIInfo *info = data;
+
+    k->init = info->initfn ? info->initfn : usb_uhci_common_initfn;
+    k->exit = info->unplug ? usb_uhci_exit : NULL;
+    k->vendor_id = info->vendor_id;
+    k->device_id = info->device_id;
+    k->revision  = info->revision;
+    k->class_id  = PCI_CLASS_SERIAL_USB;
+    k->no_hotplug = 1;
     dc->vmsd = &vmstate_uhci;
     dc->props = uhci_properties;
+    u->info = *info;
 }
 
-static TypeInfo vt82c686b_uhci_info = {
-    .name          = "vt82c686b-usb-uhci",
-    .parent        = TYPE_PCI_DEVICE,
-    .instance_size = sizeof(UHCIState),
-    .class_init    = vt82c686b_uhci_class_init,
-};
-
-static void ich9_uhci1_class_init(ObjectClass *klass, void *data)
-{
-    DeviceClass *dc = DEVICE_CLASS(klass);
-    PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
-
-    k->init = usb_uhci_common_initfn;
-    k->vendor_id = PCI_VENDOR_ID_INTEL;
-    k->device_id = PCI_DEVICE_ID_INTEL_82801I_UHCI1;
-    k->revision = 0x03;
-    k->class_id = PCI_CLASS_SERIAL_USB;
-    dc->vmsd = &vmstate_uhci;
-    dc->props = uhci_properties;
-}
-
-static TypeInfo ich9_uhci1_info = {
-    .name          = "ich9-usb-uhci1",
-    .parent        = TYPE_PCI_DEVICE,
-    .instance_size = sizeof(UHCIState),
-    .class_init    = ich9_uhci1_class_init,
-};
-
-static void ich9_uhci2_class_init(ObjectClass *klass, void *data)
-{
-    DeviceClass *dc = DEVICE_CLASS(klass);
-    PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
-
-    k->init = usb_uhci_common_initfn;
-    k->vendor_id = PCI_VENDOR_ID_INTEL;
-    k->device_id = PCI_DEVICE_ID_INTEL_82801I_UHCI2;
-    k->revision = 0x03;
-    k->class_id = PCI_CLASS_SERIAL_USB;
-    dc->vmsd = &vmstate_uhci;
-    dc->props = uhci_properties;
-}
-
-static TypeInfo ich9_uhci2_info = {
-    .name          = "ich9-usb-uhci2",
-    .parent        = TYPE_PCI_DEVICE,
-    .instance_size = sizeof(UHCIState),
-    .class_init    = ich9_uhci2_class_init,
-};
-
-static void ich9_uhci3_class_init(ObjectClass *klass, void *data)
-{
-    DeviceClass *dc = DEVICE_CLASS(klass);
-    PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
-
-    k->init = usb_uhci_common_initfn;
-    k->vendor_id = PCI_VENDOR_ID_INTEL;
-    k->device_id = PCI_DEVICE_ID_INTEL_82801I_UHCI3;
-    k->revision = 0x03;
-    k->class_id = PCI_CLASS_SERIAL_USB;
-    dc->vmsd = &vmstate_uhci;
-    dc->props = uhci_properties;
-}
-
-static TypeInfo ich9_uhci3_info = {
-    .name          = "ich9-usb-uhci3",
-    .parent        = TYPE_PCI_DEVICE,
-    .instance_size = sizeof(UHCIState),
-    .class_init    = ich9_uhci3_class_init,
+static UHCIInfo uhci_info[] = {
+    {
+        .name       = "piix3-usb-uhci",
+        .vendor_id = PCI_VENDOR_ID_INTEL,
+        .device_id = PCI_DEVICE_ID_INTEL_82371SB_2,
+        .revision  = 0x01,
+        .irq_pin   = 3,
+        .unplug    = true,
+    },{
+        .name      = "piix4-usb-uhci",
+        .vendor_id = PCI_VENDOR_ID_INTEL,
+        .device_id = PCI_DEVICE_ID_INTEL_82371AB_2,
+        .revision  = 0x01,
+        .irq_pin   = 3,
+        .unplug    = true,
+    },{
+        .name      = "vt82c686b-usb-uhci",
+        .vendor_id = PCI_VENDOR_ID_VIA,
+        .device_id = PCI_DEVICE_ID_VIA_UHCI,
+        .revision  = 0x01,
+        .irq_pin   = 3,
+        .initfn    = usb_uhci_vt82c686b_initfn,
+        .unplug    = true,
+    },{
+        .name      = "ich9-usb-uhci1", /* 00:1d.0 */
+        .vendor_id = PCI_VENDOR_ID_INTEL,
+        .device_id = PCI_DEVICE_ID_INTEL_82801I_UHCI1,
+        .revision  = 0x03,
+        .irq_pin   = 0,
+        .unplug    = false,
+    },{
+        .name      = "ich9-usb-uhci2", /* 00:1d.1 */
+        .vendor_id = PCI_VENDOR_ID_INTEL,
+        .device_id = PCI_DEVICE_ID_INTEL_82801I_UHCI2,
+        .revision  = 0x03,
+        .irq_pin   = 1,
+        .unplug    = false,
+    },{
+        .name      = "ich9-usb-uhci3", /* 00:1d.2 */
+        .vendor_id = PCI_VENDOR_ID_INTEL,
+        .device_id = PCI_DEVICE_ID_INTEL_82801I_UHCI3,
+        .revision  = 0x03,
+        .irq_pin   = 2,
+        .unplug    = false,
+    },{
+        .name      = "ich9-usb-uhci4", /* 00:1a.0 */
+        .vendor_id = PCI_VENDOR_ID_INTEL,
+        .device_id = PCI_DEVICE_ID_INTEL_82801I_UHCI4,
+        .revision  = 0x03,
+        .irq_pin   = 0,
+        .unplug    = false,
+    },{
+        .name      = "ich9-usb-uhci5", /* 00:1a.1 */
+        .vendor_id = PCI_VENDOR_ID_INTEL,
+        .device_id = PCI_DEVICE_ID_INTEL_82801I_UHCI5,
+        .revision  = 0x03,
+        .irq_pin   = 1,
+        .unplug    = false,
+    },{
+        .name      = "ich9-usb-uhci6", /* 00:1a.2 */
+        .vendor_id = PCI_VENDOR_ID_INTEL,
+        .device_id = PCI_DEVICE_ID_INTEL_82801I_UHCI6,
+        .revision  = 0x03,
+        .irq_pin   = 2,
+        .unplug    = false,
+    }
 };
 
 static void uhci_register_types(void)
 {
-    type_register_static(&piix3_uhci_info);
-    type_register_static(&piix4_uhci_info);
-    type_register_static(&vt82c686b_uhci_info);
-    type_register_static(&ich9_uhci1_info);
-    type_register_static(&ich9_uhci2_info);
-    type_register_static(&ich9_uhci3_info);
+    TypeInfo uhci_type_info = {
+        .parent        = TYPE_PCI_DEVICE,
+        .instance_size = sizeof(UHCIState),
+        .class_size    = sizeof(UHCIPCIDeviceClass),
+        .class_init    = uhci_class_init,
+    };
+    int i;
+
+    for (i = 0; i < ARRAY_SIZE(uhci_info); i++) {
+        uhci_type_info.name = uhci_info[i].name;
+        uhci_type_info.class_data = uhci_info + i;
+        type_register(&uhci_type_info);
+    }
 }
 
 type_init(uhci_register_types)
diff --git a/hw/usb/hcd-xhci.c b/hw/usb/hcd-xhci.c
index 3eb27fadb..efb509e42 100644
--- a/hw/usb/hcd-xhci.c
+++ b/hw/usb/hcd-xhci.c
@@ -23,6 +23,7 @@
 #include "hw/usb.h"
 #include "hw/pci.h"
 #include "hw/msi.h"
+#include "hw/msix.h"
 #include "trace.h"
 
 //#define DEBUG_XHCI
@@ -36,17 +37,14 @@
 #define FIXME() do { fprintf(stderr, "FIXME %s:%d\n", \
                              __func__, __LINE__); abort(); } while (0)
 
-#define MAXSLOTS 8
-#define MAXINTRS 1
+#define MAXPORTS_2 15
+#define MAXPORTS_3 15
 
-#define USB2_PORTS 4
-#define USB3_PORTS 4
-
-#define MAXPORTS (USB2_PORTS+USB3_PORTS)
+#define MAXPORTS (MAXPORTS_2+MAXPORTS_3)
+#define MAXSLOTS 64
+#define MAXINTRS 16
 
 #define TD_QUEUE 24
-#define BG_XFERS 8
-#define BG_PKTS 8
 
 /* Very pessimistic, let's hope it's enough for all cases */
 #define EV_QUEUE (((3*TD_QUEUE)+16)*MAXSLOTS)
@@ -55,24 +53,28 @@
 #define ER_FULL_HACK
 
 #define LEN_CAP         0x40
-#define OFF_OPER        LEN_CAP
 #define LEN_OPER        (0x400 + 0x10 * MAXPORTS)
-#define OFF_RUNTIME     ((OFF_OPER + LEN_OPER + 0x20) & ~0x1f)
-#define LEN_RUNTIME     (0x20 + MAXINTRS * 0x20)
-#define OFF_DOORBELL    (OFF_RUNTIME + LEN_RUNTIME)
+#define LEN_RUNTIME     ((MAXINTRS + 1) * 0x20)
 #define LEN_DOORBELL    ((MAXSLOTS + 1) * 0x20)
 
+#define OFF_OPER        LEN_CAP
+#define OFF_RUNTIME     0x1000
+#define OFF_DOORBELL    0x2000
+#define OFF_MSIX_TABLE  0x3000
+#define OFF_MSIX_PBA    0x3800
 /* must be power of 2 */
-#define LEN_REGS        0x2000
+#define LEN_REGS        0x4000
 
+#if (OFF_OPER + LEN_OPER) > OFF_RUNTIME
+#error Increase OFF_RUNTIME
+#endif
+#if (OFF_RUNTIME + LEN_RUNTIME) > OFF_DOORBELL
+#error Increase OFF_DOORBELL
+#endif
 #if (OFF_DOORBELL + LEN_DOORBELL) > LEN_REGS
 # error Increase LEN_REGS
 #endif
 
-#if MAXINTRS > 1
-# error TODO: only one interrupter supported
-#endif
-
 /* bit definitions */
 #define USBCMD_RS       (1<<0)
 #define USBCMD_HCRST    (1<<1)
@@ -144,6 +146,21 @@ typedef struct XHCITRB {
     bool ccs;
 } XHCITRB;
 
+enum {
+    PLS_U0              =  0,
+    PLS_U1              =  1,
+    PLS_U2              =  2,
+    PLS_U3              =  3,
+    PLS_DISABLED        =  4,
+    PLS_RX_DETECT       =  5,
+    PLS_INACTIVE        =  6,
+    PLS_POLLING         =  7,
+    PLS_RECOVERY        =  8,
+    PLS_HOT_RESET       =  9,
+    PLS_COMPILANCE_MODE = 10,
+    PLS_TEST_MODE       = 11,
+    PLS_RESUME          = 15,
+};
 
 typedef enum TRBType {
     TRB_RESERVED = 0,
@@ -258,6 +275,10 @@ typedef enum TRBCCode {
 
 #define TRB_LK_TC           (1<<1)
 
+#define TRB_INTR_SHIFT          22
+#define TRB_INTR_MASK       0x3ff
+#define TRB_INTR(t)         (((t).status >> TRB_INTR_SHIFT) & TRB_INTR_MASK)
+
 #define EP_TYPE_MASK        0x7
 #define EP_TYPE_SHIFT           3
 
@@ -279,6 +300,18 @@ typedef enum TRBCCode {
 #define SLOT_CONTEXT_ENTRIES_MASK 0x1f
 #define SLOT_CONTEXT_ENTRIES_SHIFT 27
 
+typedef struct XHCIState XHCIState;
+
+#define get_field(data, field)                  \
+    (((data) >> field##_SHIFT) & field##_MASK)
+
+#define set_field(data, newval, field) do {                     \
+        uint32_t val = *data;                                   \
+        val &= ~(field##_MASK << field##_SHIFT);                \
+        val |= ((newval) & field##_MASK) << field##_SHIFT;      \
+        *data = val;                                            \
+    } while (0)
+
 typedef enum EPType {
     ET_INVALID = 0,
     ET_ISO_OUT,
@@ -297,64 +330,68 @@ typedef struct XHCIRing {
 } XHCIRing;
 
 typedef struct XHCIPort {
-    USBPort port;
+    XHCIState *xhci;
     uint32_t portsc;
+    uint32_t portnr;
+    USBPort  *uport;
+    uint32_t speedmask;
+    char name[16];
+    MemoryRegion mem;
 } XHCIPort;
 
-struct XHCIState;
-typedef struct XHCIState XHCIState;
-
 typedef struct XHCITransfer {
     XHCIState *xhci;
     USBPacket packet;
+    QEMUSGList sgl;
     bool running_async;
     bool running_retry;
     bool cancelled;
     bool complete;
-    bool backgrounded;
+    bool int_req;
     unsigned int iso_pkts;
     unsigned int slotid;
     unsigned int epid;
     bool in_xfer;
     bool iso_xfer;
-    bool bg_xfer;
 
     unsigned int trb_count;
     unsigned int trb_alloced;
     XHCITRB *trbs;
 
-    unsigned int data_length;
-    unsigned int data_alloced;
-    uint8_t *data;
-
     TRBCCode status;
 
     unsigned int pkts;
     unsigned int pktsize;
     unsigned int cur_pkt;
+
+    uint64_t mfindex_kick;
 } XHCITransfer;
 
 typedef struct XHCIEPContext {
+    XHCIState *xhci;
+    unsigned int slotid;
+    unsigned int epid;
+
     XHCIRing ring;
     unsigned int next_xfer;
     unsigned int comp_xfer;
     XHCITransfer transfers[TD_QUEUE];
     XHCITransfer *retry;
-    bool bg_running;
-    bool bg_updating;
-    unsigned int next_bg;
-    XHCITransfer bg_transfers[BG_XFERS];
     EPType type;
     dma_addr_t pctx;
     unsigned int max_psize;
-    bool has_bg;
     uint32_t state;
+
+    /* iso xfer scheduling */
+    unsigned int interval;
+    int64_t mfindex_last;
+    QEMUTimer *kick_timer;
 } XHCIEPContext;
 
 typedef struct XHCISlot {
     bool enabled;
     dma_addr_t ctx;
-    unsigned int port;
+    USBPort *uport;
     unsigned int devaddr;
     XHCIEPContext * eps[31];
 } XHCISlot;
@@ -369,15 +406,46 @@ typedef struct XHCIEvent {
     uint8_t epid;
 } XHCIEvent;
 
+typedef struct XHCIInterrupter {
+    uint32_t iman;
+    uint32_t imod;
+    uint32_t erstsz;
+    uint32_t erstba_low;
+    uint32_t erstba_high;
+    uint32_t erdp_low;
+    uint32_t erdp_high;
+
+    bool msix_used, er_pcs, er_full;
+
+    dma_addr_t er_start;
+    uint32_t er_size;
+    unsigned int er_ep_idx;
+
+    XHCIEvent ev_buffer[EV_QUEUE];
+    unsigned int ev_buffer_put;
+    unsigned int ev_buffer_get;
+
+} XHCIInterrupter;
+
 struct XHCIState {
     PCIDevice pci_dev;
     USBBus bus;
     qemu_irq irq;
     MemoryRegion mem;
+    MemoryRegion mem_cap;
+    MemoryRegion mem_oper;
+    MemoryRegion mem_runtime;
+    MemoryRegion mem_doorbell;
     const char *name;
-    uint32_t msi;
     unsigned int devaddr;
 
+    /* properties */
+    uint32_t numports_2;
+    uint32_t numports_3;
+    uint32_t numintrs;
+    uint32_t numslots;
+    uint32_t flags;
+
     /* Operational Registers */
     uint32_t usbcmd;
     uint32_t usbsts;
@@ -388,29 +456,15 @@ struct XHCIState {
     uint32_t dcbaap_high;
     uint32_t config;
 
+    USBPort  uports[MAX(MAXPORTS_2, MAXPORTS_3)];
     XHCIPort ports[MAXPORTS];
     XHCISlot slots[MAXSLOTS];
+    uint32_t numports;
 
     /* Runtime Registers */
-    uint32_t mfindex;
-    /* note: we only support one interrupter */
-    uint32_t iman;
-    uint32_t imod;
-    uint32_t erstsz;
-    uint32_t erstba_low;
-    uint32_t erstba_high;
-    uint32_t erdp_low;
-    uint32_t erdp_high;
-
-    dma_addr_t er_start;
-    uint32_t er_size;
-    bool er_pcs;
-    unsigned int er_ep_idx;
-    bool er_full;
-
-    XHCIEvent ev_buffer[EV_QUEUE];
-    unsigned int ev_buffer_put;
-    unsigned int ev_buffer_get;
+    int64_t mfindex_start;
+    QEMUTimer *mfwrap_timer;
+    XHCIInterrupter intr[MAXINTRS];
 
     XHCIRing cmd_ring;
 };
@@ -422,6 +476,18 @@ typedef struct XHCIEvRingSeg {
     uint32_t rsvd;
 } XHCIEvRingSeg;
 
+enum xhci_flags {
+    XHCI_FLAG_USE_MSI = 1,
+    XHCI_FLAG_USE_MSI_X,
+};
+
+static void xhci_kick_ep(XHCIState *xhci, unsigned int slotid,
+                         unsigned int epid);
+static TRBCCode xhci_disable_ep(XHCIState *xhci, unsigned int slotid,
+                                unsigned int epid);
+static void xhci_event(XHCIState *xhci, XHCIEvent *event, int v);
+static void xhci_write_event(XHCIState *xhci, XHCIEvent *event, int v);
+
 static const char *TRBType_names[] = {
     [TRB_RESERVED]                     = "TRB_RESERVED",
     [TR_NORMAL]                        = "TR_NORMAL",
@@ -460,6 +526,45 @@ static const char *TRBType_names[] = {
     [CR_VENDOR_NEC_CHALLENGE_RESPONSE] = "CR_VENDOR_NEC_CHALLENGE_RESPONSE",
 };
 
+static const char *TRBCCode_names[] = {
+    [CC_INVALID]                       = "CC_INVALID",
+    [CC_SUCCESS]                       = "CC_SUCCESS",
+    [CC_DATA_BUFFER_ERROR]             = "CC_DATA_BUFFER_ERROR",
+    [CC_BABBLE_DETECTED]               = "CC_BABBLE_DETECTED",
+    [CC_USB_TRANSACTION_ERROR]         = "CC_USB_TRANSACTION_ERROR",
+    [CC_TRB_ERROR]                     = "CC_TRB_ERROR",
+    [CC_STALL_ERROR]                   = "CC_STALL_ERROR",
+    [CC_RESOURCE_ERROR]                = "CC_RESOURCE_ERROR",
+    [CC_BANDWIDTH_ERROR]               = "CC_BANDWIDTH_ERROR",
+    [CC_NO_SLOTS_ERROR]                = "CC_NO_SLOTS_ERROR",
+    [CC_INVALID_STREAM_TYPE_ERROR]     = "CC_INVALID_STREAM_TYPE_ERROR",
+    [CC_SLOT_NOT_ENABLED_ERROR]        = "CC_SLOT_NOT_ENABLED_ERROR",
+    [CC_EP_NOT_ENABLED_ERROR]          = "CC_EP_NOT_ENABLED_ERROR",
+    [CC_SHORT_PACKET]                  = "CC_SHORT_PACKET",
+    [CC_RING_UNDERRUN]                 = "CC_RING_UNDERRUN",
+    [CC_RING_OVERRUN]                  = "CC_RING_OVERRUN",
+    [CC_VF_ER_FULL]                    = "CC_VF_ER_FULL",
+    [CC_PARAMETER_ERROR]               = "CC_PARAMETER_ERROR",
+    [CC_BANDWIDTH_OVERRUN]             = "CC_BANDWIDTH_OVERRUN",
+    [CC_CONTEXT_STATE_ERROR]           = "CC_CONTEXT_STATE_ERROR",
+    [CC_NO_PING_RESPONSE_ERROR]        = "CC_NO_PING_RESPONSE_ERROR",
+    [CC_EVENT_RING_FULL_ERROR]         = "CC_EVENT_RING_FULL_ERROR",
+    [CC_INCOMPATIBLE_DEVICE_ERROR]     = "CC_INCOMPATIBLE_DEVICE_ERROR",
+    [CC_MISSED_SERVICE_ERROR]          = "CC_MISSED_SERVICE_ERROR",
+    [CC_COMMAND_RING_STOPPED]          = "CC_COMMAND_RING_STOPPED",
+    [CC_COMMAND_ABORTED]               = "CC_COMMAND_ABORTED",
+    [CC_STOPPED]                       = "CC_STOPPED",
+    [CC_STOPPED_LENGTH_INVALID]        = "CC_STOPPED_LENGTH_INVALID",
+    [CC_MAX_EXIT_LATENCY_TOO_LARGE_ERROR]
+    = "CC_MAX_EXIT_LATENCY_TOO_LARGE_ERROR",
+    [CC_ISOCH_BUFFER_OVERRUN]          = "CC_ISOCH_BUFFER_OVERRUN",
+    [CC_EVENT_LOST_ERROR]              = "CC_EVENT_LOST_ERROR",
+    [CC_UNDEFINED_ERROR]               = "CC_UNDEFINED_ERROR",
+    [CC_INVALID_STREAM_ID_ERROR]       = "CC_INVALID_STREAM_ID_ERROR",
+    [CC_SECONDARY_BANDWIDTH_ERROR]     = "CC_SECONDARY_BANDWIDTH_ERROR",
+    [CC_SPLIT_TRANSACTION_ERROR]       = "CC_SPLIT_TRANSACTION_ERROR",
+};
+
 static const char *lookup_name(uint32_t index, const char **list, uint32_t llen)
 {
     if (index >= llen || list[index] == NULL) {
@@ -474,8 +579,42 @@ static const char *trb_name(XHCITRB *trb)
                        ARRAY_SIZE(TRBType_names));
 }
 
-static void xhci_kick_ep(XHCIState *xhci, unsigned int slotid,
-                         unsigned int epid);
+static const char *event_name(XHCIEvent *event)
+{
+    return lookup_name(event->ccode, TRBCCode_names,
+                       ARRAY_SIZE(TRBCCode_names));
+}
+
+static uint64_t xhci_mfindex_get(XHCIState *xhci)
+{
+    int64_t now = qemu_get_clock_ns(vm_clock);
+    return (now - xhci->mfindex_start) / 125000;
+}
+
+static void xhci_mfwrap_update(XHCIState *xhci)
+{
+    const uint32_t bits = USBCMD_RS | USBCMD_EWE;
+    uint32_t mfindex, left;
+    int64_t now;
+
+    if ((xhci->usbcmd & bits) == bits) {
+        now = qemu_get_clock_ns(vm_clock);
+        mfindex = ((now - xhci->mfindex_start) / 125000) & 0x3fff;
+        left = 0x4000 - mfindex;
+        qemu_mod_timer(xhci->mfwrap_timer, now + left * 125000);
+    } else {
+        qemu_del_timer(xhci->mfwrap_timer);
+    }
+}
+
+static void xhci_mfwrap_timer(void *opaque)
+{
+    XHCIState *xhci = opaque;
+    XHCIEvent wrap = { ER_MFINDEX_WRAP, CC_SUCCESS };
+
+    xhci_event(xhci, &wrap, 0);
+    xhci_mfwrap_update(xhci);
+}
 
 static inline dma_addr_t xhci_addr64(uint32_t low, uint32_t high)
 {
@@ -495,29 +634,134 @@ static inline dma_addr_t xhci_mask64(uint64_t addr)
     }
 }
 
-static void xhci_irq_update(XHCIState *xhci)
+static inline void xhci_dma_read_u32s(XHCIState *xhci, dma_addr_t addr,
+                                      uint32_t *buf, size_t len)
+{
+    int i;
+
+    assert((len % sizeof(uint32_t)) == 0);
+
+    pci_dma_read(&xhci->pci_dev, addr, buf, len);
+
+    for (i = 0; i < (len / sizeof(uint32_t)); i++) {
+        buf[i] = le32_to_cpu(buf[i]);
+    }
+}
+
+static inline void xhci_dma_write_u32s(XHCIState *xhci, dma_addr_t addr,
+                                       uint32_t *buf, size_t len)
+{
+    int i;
+    uint32_t tmp[len / sizeof(uint32_t)];
+
+    assert((len % sizeof(uint32_t)) == 0);
+
+    for (i = 0; i < (len / sizeof(uint32_t)); i++) {
+        tmp[i] = cpu_to_le32(buf[i]);
+    }
+    pci_dma_write(&xhci->pci_dev, addr, tmp, len);
+}
+
+static XHCIPort *xhci_lookup_port(XHCIState *xhci, struct USBPort *uport)
+{
+    int index;
+
+    if (!uport->dev) {
+        return NULL;
+    }
+    switch (uport->dev->speed) {
+    case USB_SPEED_LOW:
+    case USB_SPEED_FULL:
+    case USB_SPEED_HIGH:
+        index = uport->index;
+        break;
+    case USB_SPEED_SUPER:
+        index = uport->index + xhci->numports_2;
+        break;
+    default:
+        return NULL;
+    }
+    return &xhci->ports[index];
+}
+
+static void xhci_intx_update(XHCIState *xhci)
 {
     int level = 0;
 
-    if (xhci->iman & IMAN_IP && xhci->iman & IMAN_IE &&
+    if (msix_enabled(&xhci->pci_dev) ||
+        msi_enabled(&xhci->pci_dev)) {
+        return;
+    }
+
+    if (xhci->intr[0].iman & IMAN_IP &&
+        xhci->intr[0].iman & IMAN_IE &&
         xhci->usbcmd & USBCMD_INTE) {
         level = 1;
     }
 
-    if (xhci->msi && msi_enabled(&xhci->pci_dev)) {
-        if (level) {
-            trace_usb_xhci_irq_msi(0);
-            msi_notify(&xhci->pci_dev, 0);
-        }
+    trace_usb_xhci_irq_intx(level);
+    qemu_set_irq(xhci->irq, level);
+}
+
+static void xhci_msix_update(XHCIState *xhci, int v)
+{
+    bool enabled;
+
+    if (!msix_enabled(&xhci->pci_dev)) {
+        return;
+    }
+
+    enabled = xhci->intr[v].iman & IMAN_IE;
+    if (enabled == xhci->intr[v].msix_used) {
+        return;
+    }
+
+    if (enabled) {
+        trace_usb_xhci_irq_msix_use(v);
+        msix_vector_use(&xhci->pci_dev, v);
+        xhci->intr[v].msix_used = true;
     } else {
-        trace_usb_xhci_irq_intx(level);
-        qemu_set_irq(xhci->irq, level);
+        trace_usb_xhci_irq_msix_unuse(v);
+        msix_vector_unuse(&xhci->pci_dev, v);
+        xhci->intr[v].msix_used = false;
+    }
+}
+
+static void xhci_intr_raise(XHCIState *xhci, int v)
+{
+    xhci->intr[v].erdp_low |= ERDP_EHB;
+    xhci->intr[v].iman |= IMAN_IP;
+    xhci->usbsts |= USBSTS_EINT;
+
+    if (!(xhci->intr[v].iman & IMAN_IE)) {
+        return;
+    }
+
+    if (!(xhci->usbcmd & USBCMD_INTE)) {
+        return;
+    }
+
+    if (msix_enabled(&xhci->pci_dev)) {
+        trace_usb_xhci_irq_msix(v);
+        msix_notify(&xhci->pci_dev, v);
+        return;
+    }
+
+    if (msi_enabled(&xhci->pci_dev)) {
+        trace_usb_xhci_irq_msi(v);
+        msi_notify(&xhci->pci_dev, v);
+        return;
+    }
+
+    if (v == 0) {
+        trace_usb_xhci_irq_intx(1);
+        qemu_set_irq(xhci->irq, 1);
     }
 }
 
 static inline int xhci_running(XHCIState *xhci)
 {
-    return !(xhci->usbsts & USBSTS_HCH) && !xhci->er_full;
+    return !(xhci->usbsts & USBSTS_HCH) && !xhci->intr[0].er_full;
 }
 
 static void xhci_die(XHCIState *xhci)
@@ -526,8 +770,9 @@ static void xhci_die(XHCIState *xhci)
     fprintf(stderr, "xhci: asserted controller error\n");
 }
 
-static void xhci_write_event(XHCIState *xhci, XHCIEvent *event)
+static void xhci_write_event(XHCIState *xhci, XHCIEvent *event, int v)
 {
+    XHCIInterrupter *intr = &xhci->intr[v];
     XHCITRB ev_trb;
     dma_addr_t addr;
 
@@ -535,26 +780,28 @@ static void xhci_write_event(XHCIState *xhci, XHCIEvent *event)
     ev_trb.status = cpu_to_le32(event->length | (event->ccode << 24));
     ev_trb.control = (event->slotid << 24) | (event->epid << 16) |
                      event->flags | (event->type << TRB_TYPE_SHIFT);
-    if (xhci->er_pcs) {
+    if (intr->er_pcs) {
         ev_trb.control |= TRB_C;
     }
     ev_trb.control = cpu_to_le32(ev_trb.control);
 
-    trace_usb_xhci_queue_event(xhci->er_ep_idx, trb_name(&ev_trb),
-                               ev_trb.parameter, ev_trb.status, ev_trb.control);
+    trace_usb_xhci_queue_event(v, intr->er_ep_idx, trb_name(&ev_trb),
+                               event_name(event), ev_trb.parameter,
+                               ev_trb.status, ev_trb.control);
 
-    addr = xhci->er_start + TRB_SIZE*xhci->er_ep_idx;
+    addr = intr->er_start + TRB_SIZE*intr->er_ep_idx;
     pci_dma_write(&xhci->pci_dev, addr, &ev_trb, TRB_SIZE);
 
-    xhci->er_ep_idx++;
-    if (xhci->er_ep_idx >= xhci->er_size) {
-        xhci->er_ep_idx = 0;
-        xhci->er_pcs = !xhci->er_pcs;
+    intr->er_ep_idx++;
+    if (intr->er_ep_idx >= intr->er_size) {
+        intr->er_ep_idx = 0;
+        intr->er_pcs = !intr->er_pcs;
     }
 }
 
-static void xhci_events_update(XHCIState *xhci)
+static void xhci_events_update(XHCIState *xhci, int v)
 {
+    XHCIInterrupter *intr = &xhci->intr[v];
     dma_addr_t erdp;
     unsigned int dp_idx;
     bool do_irq = 0;
@@ -563,122 +810,121 @@ static void xhci_events_update(XHCIState *xhci)
         return;
     }
 
-    erdp = xhci_addr64(xhci->erdp_low, xhci->erdp_high);
-    if (erdp < xhci->er_start ||
-        erdp >= (xhci->er_start + TRB_SIZE*xhci->er_size)) {
+    erdp = xhci_addr64(intr->erdp_low, intr->erdp_high);
+    if (erdp < intr->er_start ||
+        erdp >= (intr->er_start + TRB_SIZE*intr->er_size)) {
         fprintf(stderr, "xhci: ERDP out of bounds: "DMA_ADDR_FMT"\n", erdp);
-        fprintf(stderr, "xhci: ER at "DMA_ADDR_FMT" len %d\n",
-                xhci->er_start, xhci->er_size);
+        fprintf(stderr, "xhci: ER[%d] at "DMA_ADDR_FMT" len %d\n",
+                v, intr->er_start, intr->er_size);
         xhci_die(xhci);
         return;
     }
-    dp_idx = (erdp - xhci->er_start) / TRB_SIZE;
-    assert(dp_idx < xhci->er_size);
+    dp_idx = (erdp - intr->er_start) / TRB_SIZE;
+    assert(dp_idx < intr->er_size);
 
     /* NEC didn't read section 4.9.4 of the spec (v1.0 p139 top Note) and thus
      * deadlocks when the ER is full. Hack it by holding off events until
      * the driver decides to free at least half of the ring */
-    if (xhci->er_full) {
-        int er_free = dp_idx - xhci->er_ep_idx;
+    if (intr->er_full) {
+        int er_free = dp_idx - intr->er_ep_idx;
         if (er_free <= 0) {
-            er_free += xhci->er_size;
+            er_free += intr->er_size;
         }
-        if (er_free < (xhci->er_size/2)) {
+        if (er_free < (intr->er_size/2)) {
             DPRINTF("xhci_events_update(): event ring still "
                     "more than half full (hack)\n");
             return;
         }
     }
 
-    while (xhci->ev_buffer_put != xhci->ev_buffer_get) {
-        assert(xhci->er_full);
-        if (((xhci->er_ep_idx+1) % xhci->er_size) == dp_idx) {
+    while (intr->ev_buffer_put != intr->ev_buffer_get) {
+        assert(intr->er_full);
+        if (((intr->er_ep_idx+1) % intr->er_size) == dp_idx) {
             DPRINTF("xhci_events_update(): event ring full again\n");
 #ifndef ER_FULL_HACK
             XHCIEvent full = {ER_HOST_CONTROLLER, CC_EVENT_RING_FULL_ERROR};
-            xhci_write_event(xhci, &full);
+            xhci_write_event(xhci, &full, v);
 #endif
             do_irq = 1;
             break;
         }
-        XHCIEvent *event = &xhci->ev_buffer[xhci->ev_buffer_get];
-        xhci_write_event(xhci, event);
-        xhci->ev_buffer_get++;
+        XHCIEvent *event = &intr->ev_buffer[intr->ev_buffer_get];
+        xhci_write_event(xhci, event, v);
+        intr->ev_buffer_get++;
         do_irq = 1;
-        if (xhci->ev_buffer_get == EV_QUEUE) {
-            xhci->ev_buffer_get = 0;
+        if (intr->ev_buffer_get == EV_QUEUE) {
+            intr->ev_buffer_get = 0;
         }
     }
 
     if (do_irq) {
-        xhci->erdp_low |= ERDP_EHB;
-        xhci->iman |= IMAN_IP;
-        xhci->usbsts |= USBSTS_EINT;
-        xhci_irq_update(xhci);
+        xhci_intr_raise(xhci, v);
     }
 
-    if (xhci->er_full && xhci->ev_buffer_put == xhci->ev_buffer_get) {
+    if (intr->er_full && intr->ev_buffer_put == intr->ev_buffer_get) {
         DPRINTF("xhci_events_update(): event ring no longer full\n");
-        xhci->er_full = 0;
+        intr->er_full = 0;
     }
-    return;
 }
 
-static void xhci_event(XHCIState *xhci, XHCIEvent *event)
+static void xhci_event(XHCIState *xhci, XHCIEvent *event, int v)
 {
+    XHCIInterrupter *intr;
     dma_addr_t erdp;
     unsigned int dp_idx;
 
-    if (xhci->er_full) {
+    if (v >= xhci->numintrs) {
+        DPRINTF("intr nr out of range (%d >= %d)\n", v, xhci->numintrs);
+        return;
+    }
+    intr = &xhci->intr[v];
+
+    if (intr->er_full) {
         DPRINTF("xhci_event(): ER full, queueing\n");
-        if (((xhci->ev_buffer_put+1) % EV_QUEUE) == xhci->ev_buffer_get) {
+        if (((intr->ev_buffer_put+1) % EV_QUEUE) == intr->ev_buffer_get) {
             fprintf(stderr, "xhci: event queue full, dropping event!\n");
             return;
         }
-        xhci->ev_buffer[xhci->ev_buffer_put++] = *event;
-        if (xhci->ev_buffer_put == EV_QUEUE) {
-            xhci->ev_buffer_put = 0;
+        intr->ev_buffer[intr->ev_buffer_put++] = *event;
+        if (intr->ev_buffer_put == EV_QUEUE) {
+            intr->ev_buffer_put = 0;
         }
         return;
     }
 
-    erdp = xhci_addr64(xhci->erdp_low, xhci->erdp_high);
-    if (erdp < xhci->er_start ||
-        erdp >= (xhci->er_start + TRB_SIZE*xhci->er_size)) {
+    erdp = xhci_addr64(intr->erdp_low, intr->erdp_high);
+    if (erdp < intr->er_start ||
+        erdp >= (intr->er_start + TRB_SIZE*intr->er_size)) {
         fprintf(stderr, "xhci: ERDP out of bounds: "DMA_ADDR_FMT"\n", erdp);
-        fprintf(stderr, "xhci: ER at "DMA_ADDR_FMT" len %d\n",
-                xhci->er_start, xhci->er_size);
+        fprintf(stderr, "xhci: ER[%d] at "DMA_ADDR_FMT" len %d\n",
+                v, intr->er_start, intr->er_size);
         xhci_die(xhci);
         return;
     }
 
-    dp_idx = (erdp - xhci->er_start) / TRB_SIZE;
-    assert(dp_idx < xhci->er_size);
+    dp_idx = (erdp - intr->er_start) / TRB_SIZE;
+    assert(dp_idx < intr->er_size);
 
-    if ((xhci->er_ep_idx+1) % xhci->er_size == dp_idx) {
+    if ((intr->er_ep_idx+1) % intr->er_size == dp_idx) {
         DPRINTF("xhci_event(): ER full, queueing\n");
 #ifndef ER_FULL_HACK
         XHCIEvent full = {ER_HOST_CONTROLLER, CC_EVENT_RING_FULL_ERROR};
         xhci_write_event(xhci, &full);
 #endif
-        xhci->er_full = 1;
-        if (((xhci->ev_buffer_put+1) % EV_QUEUE) == xhci->ev_buffer_get) {
+        intr->er_full = 1;
+        if (((intr->ev_buffer_put+1) % EV_QUEUE) == intr->ev_buffer_get) {
             fprintf(stderr, "xhci: event queue full, dropping event!\n");
             return;
         }
-        xhci->ev_buffer[xhci->ev_buffer_put++] = *event;
-        if (xhci->ev_buffer_put == EV_QUEUE) {
-            xhci->ev_buffer_put = 0;
+        intr->ev_buffer[intr->ev_buffer_put++] = *event;
+        if (intr->ev_buffer_put == EV_QUEUE) {
+            intr->ev_buffer_put = 0;
         }
     } else {
-        xhci_write_event(xhci, event);
+        xhci_write_event(xhci, event, v);
     }
 
-    xhci->erdp_low |= ERDP_EHB;
-    xhci->iman |= IMAN_IP;
-    xhci->usbsts |= USBSTS_EINT;
-
-    xhci_irq_update(xhci);
+    xhci_intr_raise(xhci, v);
 }
 
 static void xhci_ring_init(XHCIState *xhci, XHCIRing *ring,
@@ -770,17 +1016,24 @@ static int xhci_ring_chain_length(XHCIState *xhci, const XHCIRing *ring)
     }
 }
 
-static void xhci_er_reset(XHCIState *xhci)
+static void xhci_er_reset(XHCIState *xhci, int v)
 {
+    XHCIInterrupter *intr = &xhci->intr[v];
     XHCIEvRingSeg seg;
 
+    if (intr->erstsz == 0) {
+        /* disabled */
+        intr->er_start = 0;
+        intr->er_size = 0;
+        return;
+    }
     /* cache the (sole) event ring segment location */
-    if (xhci->erstsz != 1) {
-        fprintf(stderr, "xhci: invalid value for ERSTSZ: %d\n", xhci->erstsz);
+    if (intr->erstsz != 1) {
+        fprintf(stderr, "xhci: invalid value for ERSTSZ: %d\n", intr->erstsz);
         xhci_die(xhci);
         return;
     }
-    dma_addr_t erstba = xhci_addr64(xhci->erstba_low, xhci->erstba_high);
+    dma_addr_t erstba = xhci_addr64(intr->erstba_low, intr->erstba_high);
     pci_dma_read(&xhci->pci_dev, erstba, &seg, sizeof(seg));
     le32_to_cpus(&seg.addr_low);
     le32_to_cpus(&seg.addr_high);
@@ -790,21 +1043,22 @@ static void xhci_er_reset(XHCIState *xhci)
         xhci_die(xhci);
         return;
     }
-    xhci->er_start = xhci_addr64(seg.addr_low, seg.addr_high);
-    xhci->er_size = seg.size;
+    intr->er_start = xhci_addr64(seg.addr_low, seg.addr_high);
+    intr->er_size = seg.size;
 
-    xhci->er_ep_idx = 0;
-    xhci->er_pcs = 1;
-    xhci->er_full = 0;
+    intr->er_ep_idx = 0;
+    intr->er_pcs = 1;
+    intr->er_full = 0;
 
-    DPRINTF("xhci: event ring:" DMA_ADDR_FMT " [%d]\n",
-            xhci->er_start, xhci->er_size);
+    DPRINTF("xhci: event ring[%d]:" DMA_ADDR_FMT " [%d]\n",
+            v, intr->er_start, intr->er_size);
 }
 
 static void xhci_run(XHCIState *xhci)
 {
     trace_usb_xhci_run();
     xhci->usbsts &= ~USBSTS_HCH;
+    xhci->mfindex_start = qemu_get_clock_ns(vm_clock);
 }
 
 static void xhci_stop(XHCIState *xhci)
@@ -818,21 +1072,24 @@ static void xhci_set_ep_state(XHCIState *xhci, XHCIEPContext *epctx,
                               uint32_t state)
 {
     uint32_t ctx[5];
-    if (epctx->state == state) {
-        return;
-    }
 
-    pci_dma_read(&xhci->pci_dev, epctx->pctx, ctx, sizeof(ctx));
+    xhci_dma_read_u32s(xhci, epctx->pctx, ctx, sizeof(ctx));
     ctx[0] &= ~EP_STATE_MASK;
     ctx[0] |= state;
     ctx[2] = epctx->ring.dequeue | epctx->ring.ccs;
     ctx[3] = (epctx->ring.dequeue >> 16) >> 16;
     DPRINTF("xhci: set epctx: " DMA_ADDR_FMT " state=%d dequeue=%08x%08x\n",
             epctx->pctx, state, ctx[3], ctx[2]);
-    pci_dma_write(&xhci->pci_dev, epctx->pctx, ctx, sizeof(ctx));
+    xhci_dma_write_u32s(xhci, epctx->pctx, ctx, sizeof(ctx));
     epctx->state = state;
 }
 
+static void xhci_ep_kick_timer(void *opaque)
+{
+    XHCIEPContext *epctx = opaque;
+    xhci_kick_ep(epctx->xhci, epctx->slotid, epctx->epid);
+}
+
 static TRBCCode xhci_enable_ep(XHCIState *xhci, unsigned int slotid,
                                unsigned int epid, dma_addr_t pctx,
                                uint32_t *ctx)
@@ -843,17 +1100,19 @@ static TRBCCode xhci_enable_ep(XHCIState *xhci, unsigned int slotid,
     int i;
 
     trace_usb_xhci_ep_enable(slotid, epid);
-    assert(slotid >= 1 && slotid <= MAXSLOTS);
+    assert(slotid >= 1 && slotid <= xhci->numslots);
     assert(epid >= 1 && epid <= 31);
 
     slot = &xhci->slots[slotid-1];
     if (slot->eps[epid-1]) {
-        fprintf(stderr, "xhci: slot %d ep %d already enabled!\n", slotid, epid);
-        return CC_TRB_ERROR;
+        xhci_disable_ep(xhci, slotid, epid);
     }
 
     epctx = g_malloc(sizeof(XHCIEPContext));
     memset(epctx, 0, sizeof(XHCIEPContext));
+    epctx->xhci = xhci;
+    epctx->slotid = slotid;
+    epctx->epid = epid;
 
     slot->eps[epid-1] = epctx;
 
@@ -866,16 +1125,16 @@ static TRBCCode xhci_enable_ep(XHCIState *xhci, unsigned int slotid,
     epctx->pctx = pctx;
     epctx->max_psize = ctx[1]>>16;
     epctx->max_psize *= 1+((ctx[1]>>8)&0xff);
-    epctx->has_bg = false;
-    if (epctx->type == ET_ISO_IN) {
-        epctx->has_bg = true;
-    }
     DPRINTF("xhci: endpoint %d.%d max transaction (burst) size is %d\n",
             epid/2, epid%2, epctx->max_psize);
     for (i = 0; i < ARRAY_SIZE(epctx->transfers); i++) {
         usb_packet_init(&epctx->transfers[i].packet);
     }
 
+    epctx->interval = 1 << (ctx[0] >> 16) & 0xff;
+    epctx->mfindex_last = 0;
+    epctx->kick_timer = qemu_new_timer_ns(vm_clock, xhci_ep_kick_timer, epctx);
+
     epctx->state = EP_RUNNING;
     ctx[0] &= ~EP_STATE_MASK;
     ctx[0] |= EP_RUNNING;
@@ -883,13 +1142,42 @@ static TRBCCode xhci_enable_ep(XHCIState *xhci, unsigned int slotid,
     return CC_SUCCESS;
 }
 
+static int xhci_ep_nuke_one_xfer(XHCITransfer *t)
+{
+    int killed = 0;
+
+    if (t->running_async) {
+        usb_cancel_packet(&t->packet);
+        t->running_async = 0;
+        t->cancelled = 1;
+        DPRINTF("xhci: cancelling transfer, waiting for it to complete\n");
+        killed = 1;
+    }
+    if (t->running_retry) {
+        XHCIEPContext *epctx = t->xhci->slots[t->slotid-1].eps[t->epid-1];
+        if (epctx) {
+            epctx->retry = NULL;
+            qemu_del_timer(epctx->kick_timer);
+        }
+        t->running_retry = 0;
+    }
+    if (t->trbs) {
+        g_free(t->trbs);
+    }
+
+    t->trbs = NULL;
+    t->trb_count = t->trb_alloced = 0;
+
+    return killed;
+}
+
 static int xhci_ep_nuke_xfers(XHCIState *xhci, unsigned int slotid,
                                unsigned int epid)
 {
     XHCISlot *slot;
     XHCIEPContext *epctx;
     int i, xferi, killed = 0;
-    assert(slotid >= 1 && slotid <= MAXSLOTS);
+    assert(slotid >= 1 && slotid <= xhci->numslots);
     assert(epid >= 1 && epid <= 31);
 
     DPRINTF("xhci_ep_nuke_xfers(%d, %d)\n", slotid, epid);
@@ -904,53 +1192,9 @@ static int xhci_ep_nuke_xfers(XHCIState *xhci, unsigned int slotid,
 
     xferi = epctx->next_xfer;
     for (i = 0; i < TD_QUEUE; i++) {
-        XHCITransfer *t = &epctx->transfers[xferi];
-        if (t->running_async) {
-            usb_cancel_packet(&t->packet);
-            t->running_async = 0;
-            t->cancelled = 1;
-            DPRINTF("xhci: cancelling transfer %d, waiting for it to complete...\n", i);
-            killed++;
-        }
-        if (t->running_retry) {
-            t->running_retry = 0;
-            epctx->retry = NULL;
-        }
-        if (t->backgrounded) {
-            t->backgrounded = 0;
-        }
-        if (t->trbs) {
-            g_free(t->trbs);
-        }
-        if (t->data) {
-            g_free(t->data);
-        }
-
-        t->trbs = NULL;
-        t->data = NULL;
-        t->trb_count = t->trb_alloced = 0;
-        t->data_length = t->data_alloced = 0;
+        killed += xhci_ep_nuke_one_xfer(&epctx->transfers[xferi]);
         xferi = (xferi + 1) % TD_QUEUE;
     }
-    if (epctx->has_bg) {
-        xferi = epctx->next_bg;
-        for (i = 0; i < BG_XFERS; i++) {
-            XHCITransfer *t = &epctx->bg_transfers[xferi];
-            if (t->running_async) {
-                usb_cancel_packet(&t->packet);
-                t->running_async = 0;
-                t->cancelled = 1;
-                DPRINTF("xhci: cancelling bg transfer %d, waiting for it to complete...\n", i);
-                killed++;
-            }
-            if (t->data) {
-                g_free(t->data);
-            }
-
-            t->data = NULL;
-            xferi = (xferi + 1) % BG_XFERS;
-        }
-    }
     return killed;
 }
 
@@ -961,7 +1205,7 @@ static TRBCCode xhci_disable_ep(XHCIState *xhci, unsigned int slotid,
     XHCIEPContext *epctx;
 
     trace_usb_xhci_ep_disable(slotid, epid);
-    assert(slotid >= 1 && slotid <= MAXSLOTS);
+    assert(slotid >= 1 && slotid <= xhci->numslots);
     assert(epid >= 1 && epid <= 31);
 
     slot = &xhci->slots[slotid-1];
@@ -977,6 +1221,7 @@ static TRBCCode xhci_disable_ep(XHCIState *xhci, unsigned int slotid,
 
     xhci_set_ep_state(xhci, epctx, EP_DISABLED);
 
+    qemu_free_timer(epctx->kick_timer);
     g_free(epctx);
     slot->eps[epid-1] = NULL;
 
@@ -990,7 +1235,7 @@ static TRBCCode xhci_stop_ep(XHCIState *xhci, unsigned int slotid,
     XHCIEPContext *epctx;
 
     trace_usb_xhci_ep_stop(slotid, epid);
-    assert(slotid >= 1 && slotid <= MAXSLOTS);
+    assert(slotid >= 1 && slotid <= xhci->numslots);
 
     if (epid < 1 || epid > 31) {
         fprintf(stderr, "xhci: bad ep %d\n", epid);
@@ -1024,7 +1269,7 @@ static TRBCCode xhci_reset_ep(XHCIState *xhci, unsigned int slotid,
     USBDevice *dev;
 
     trace_usb_xhci_ep_reset(slotid, epid);
-    assert(slotid >= 1 && slotid <= MAXSLOTS);
+    assert(slotid >= 1 && slotid <= xhci->numslots);
 
     if (epid < 1 || epid > 31) {
         fprintf(stderr, "xhci: bad ep %d\n", epid);
@@ -1057,7 +1302,7 @@ static TRBCCode xhci_reset_ep(XHCIState *xhci, unsigned int slotid,
         ep |= 0x80;
     }
 
-    dev = xhci->ports[xhci->slots[slotid-1].port-1].port.dev;
+    dev = xhci->slots[slotid-1].uport->dev;
     if (!dev) {
         return CC_USB_TRANSACTION_ERROR;
     }
@@ -1074,14 +1319,14 @@ static TRBCCode xhci_set_ep_dequeue(XHCIState *xhci, unsigned int slotid,
     XHCIEPContext *epctx;
     dma_addr_t dequeue;
 
-    assert(slotid >= 1 && slotid <= MAXSLOTS);
+    assert(slotid >= 1 && slotid <= xhci->numslots);
 
     if (epid < 1 || epid > 31) {
         fprintf(stderr, "xhci: bad ep %d\n", epid);
         return CC_TRB_ERROR;
     }
 
-    DPRINTF("xhci_set_ep_dequeue(%d, %d, %016"PRIx64")\n", slotid, epid, pdequeue);
+    trace_usb_xhci_ep_set_dequeue(slotid, epid, pdequeue);
     dequeue = xhci_mask64(pdequeue);
 
     slot = &xhci->slots[slotid-1];
@@ -1107,81 +1352,89 @@ static TRBCCode xhci_set_ep_dequeue(XHCIState *xhci, unsigned int slotid,
     return CC_SUCCESS;
 }
 
-static int xhci_xfer_data(XHCITransfer *xfer, uint8_t *data,
-                          unsigned int length, bool in_xfer, bool out_xfer,
-                          bool report)
+static int xhci_xfer_create_sgl(XHCITransfer *xfer, int in_xfer)
 {
-    int i;
-    uint32_t edtla = 0;
-    unsigned int transferred = 0;
-    unsigned int left = length;
-    bool reported = 0;
-    bool shortpkt = 0;
-    XHCIEvent event = {ER_TRANSFER, CC_SUCCESS};
     XHCIState *xhci = xfer->xhci;
+    int i;
 
-    DPRINTF("xhci_xfer_data(len=%d, in_xfer=%d, out_xfer=%d, report=%d)\n",
-            length, in_xfer, out_xfer, report);
-
-    assert(!(in_xfer && out_xfer));
-
+    xfer->int_req = false;
+    pci_dma_sglist_init(&xfer->sgl, &xhci->pci_dev, xfer->trb_count);
     for (i = 0; i < xfer->trb_count; i++) {
         XHCITRB *trb = &xfer->trbs[i];
         dma_addr_t addr;
         unsigned int chunk = 0;
 
+        if (trb->control & TRB_TR_IOC) {
+            xfer->int_req = true;
+        }
+
         switch (TRB_TYPE(*trb)) {
         case TR_DATA:
             if ((!(trb->control & TRB_TR_DIR)) != (!in_xfer)) {
                 fprintf(stderr, "xhci: data direction mismatch for TR_DATA\n");
-                xhci_die(xhci);
-                return transferred;
+                goto err;
             }
             /* fallthrough */
         case TR_NORMAL:
         case TR_ISOCH:
             addr = xhci_mask64(trb->parameter);
             chunk = trb->status & 0x1ffff;
+            if (trb->control & TRB_TR_IDT) {
+                if (chunk > 8 || in_xfer) {
+                    fprintf(stderr, "xhci: invalid immediate data TRB\n");
+                    goto err;
+                }
+                qemu_sglist_add(&xfer->sgl, trb->addr, chunk);
+            } else {
+                qemu_sglist_add(&xfer->sgl, addr, chunk);
+            }
+            break;
+        }
+    }
+
+    return 0;
+
+err:
+    qemu_sglist_destroy(&xfer->sgl);
+    xhci_die(xhci);
+    return -1;
+}
+
+static void xhci_xfer_unmap(XHCITransfer *xfer)
+{
+    usb_packet_unmap(&xfer->packet, &xfer->sgl);
+    qemu_sglist_destroy(&xfer->sgl);
+}
+
+static void xhci_xfer_report(XHCITransfer *xfer)
+{
+    uint32_t edtla = 0;
+    unsigned int left;
+    bool reported = 0;
+    bool shortpkt = 0;
+    XHCIEvent event = {ER_TRANSFER, CC_SUCCESS};
+    XHCIState *xhci = xfer->xhci;
+    int i;
+
+    left = xfer->packet.actual_length;
+
+    for (i = 0; i < xfer->trb_count; i++) {
+        XHCITRB *trb = &xfer->trbs[i];
+        unsigned int chunk = 0;
+
+        switch (TRB_TYPE(*trb)) {
+        case TR_DATA:
+        case TR_NORMAL:
+        case TR_ISOCH:
+            chunk = trb->status & 0x1ffff;
             if (chunk > left) {
                 chunk = left;
-                shortpkt = 1;
-            }
-            if (in_xfer || out_xfer) {
-                if (trb->control & TRB_TR_IDT) {
-                    uint64_t idata;
-                    if (chunk > 8 || in_xfer) {
-                        fprintf(stderr, "xhci: invalid immediate data TRB\n");
-                        xhci_die(xhci);
-                        return transferred;
-                    }
-                    idata = le64_to_cpu(trb->parameter);
-                    memcpy(data, &idata, chunk);
-                } else {
-                    DPRINTF("xhci_xfer_data: r/w(%d) %d bytes at "
-                            DMA_ADDR_FMT "\n", in_xfer, chunk, addr);
-                    if (in_xfer) {
-                        pci_dma_write(&xhci->pci_dev, addr, data, chunk);
-                    } else {
-                        pci_dma_read(&xhci->pci_dev, addr, data, chunk);
-                    }
-#ifdef DEBUG_DATA
-                    unsigned int count = chunk;
-                    int i;
-                    if (count > 16) {
-                        count = 16;
-                    }
-                    DPRINTF(" ::");
-                    for (i = 0; i < count; i++) {
-                        DPRINTF(" %02x", data[i]);
-                    }
-                    DPRINTF("\n");
-#endif
+                if (xfer->status == CC_SUCCESS) {
+                    shortpkt = 1;
                 }
             }
             left -= chunk;
-            data += chunk;
             edtla += chunk;
-            transferred += chunk;
             break;
         case TR_STATUS:
             reported = 0;
@@ -1189,8 +1442,9 @@ static int xhci_xfer_data(XHCITransfer *xfer, uint8_t *data,
             break;
         }
 
-        if (report && !reported && (trb->control & TRB_TR_IOC ||
-            (shortpkt && (trb->control & TRB_TR_ISP)))) {
+        if (!reported && ((trb->control & TRB_TR_IOC) ||
+                          (shortpkt && (trb->control & TRB_TR_ISP)) ||
+                          (xfer->status != CC_SUCCESS && left == 0))) {
             event.slotid = xfer->slotid;
             event.epid = xfer->epid;
             event.length = (trb->status & 0x1ffff) - chunk;
@@ -1208,11 +1462,13 @@ static int xhci_xfer_data(XHCITransfer *xfer, uint8_t *data,
                 DPRINTF("xhci_xfer_data: EDTLA=%d\n", event.length);
                 edtla = 0;
             }
-            xhci_event(xhci, &event);
+            xhci_event(xhci, &event, TRB_INTR(*trb));
             reported = 1;
+            if (xfer->status != CC_SUCCESS) {
+                return;
+            }
         }
     }
-    return transferred;
 }
 
 static void xhci_stall_ep(XHCITransfer *xfer)
@@ -1231,184 +1487,47 @@ static void xhci_stall_ep(XHCITransfer *xfer)
 static int xhci_submit(XHCIState *xhci, XHCITransfer *xfer,
                        XHCIEPContext *epctx);
 
-static void xhci_bg_update(XHCIState *xhci, XHCIEPContext *epctx)
+static int xhci_setup_packet(XHCITransfer *xfer)
 {
-    if (epctx->bg_updating) {
-        return;
-    }
-    DPRINTF("xhci_bg_update(%p, %p)\n", xhci, epctx);
-    assert(epctx->has_bg);
-    DPRINTF("xhci: fg=%d bg=%d\n", epctx->comp_xfer, epctx->next_bg);
-    epctx->bg_updating = 1;
-    while (epctx->transfers[epctx->comp_xfer].backgrounded &&
-           epctx->bg_transfers[epctx->next_bg].complete) {
-        XHCITransfer *fg = &epctx->transfers[epctx->comp_xfer];
-        XHCITransfer *bg = &epctx->bg_transfers[epctx->next_bg];
-#if 0
-        DPRINTF("xhci: completing fg %d from bg %d.%d (stat: %d)\n",
-                epctx->comp_xfer, epctx->next_bg, bg->cur_pkt,
-                bg->usbxfer->iso_packet_desc[bg->cur_pkt].status
-               );
-#endif
-        assert(epctx->type == ET_ISO_IN);
-        assert(bg->iso_xfer);
-        assert(bg->in_xfer);
-        uint8_t *p = bg->data + bg->cur_pkt * bg->pktsize;
-#if 0
-        int len = bg->usbxfer->iso_packet_desc[bg->cur_pkt].actual_length;
-        fg->status = libusb_to_ccode(bg->usbxfer->iso_packet_desc[bg->cur_pkt].status);
-#else
-        int len = 0;
-        FIXME();
-#endif
-        fg->complete = 1;
-        fg->backgrounded = 0;
-
-        if (fg->status == CC_STALL_ERROR) {
-            xhci_stall_ep(fg);
-        }
-
-        xhci_xfer_data(fg, p, len, 1, 0, 1);
-
-        epctx->comp_xfer++;
-        if (epctx->comp_xfer == TD_QUEUE) {
-            epctx->comp_xfer = 0;
-        }
-        DPRINTF("next fg xfer: %d\n", epctx->comp_xfer);
-        bg->cur_pkt++;
-        if (bg->cur_pkt == bg->pkts) {
-            bg->complete = 0;
-            if (xhci_submit(xhci, bg, epctx) < 0) {
-                fprintf(stderr, "xhci: bg resubmit failed\n");
-            }
-            epctx->next_bg++;
-            if (epctx->next_bg == BG_XFERS) {
-                epctx->next_bg = 0;
-            }
-            DPRINTF("next bg xfer: %d\n", epctx->next_bg);
-
-        xhci_kick_ep(xhci, fg->slotid, fg->epid);
-        }
-    }
-    epctx->bg_updating = 0;
-}
-
-#if 0
-static void xhci_xfer_cb(struct libusb_transfer *transfer)
-{
-    XHCIState *xhci;
-    XHCITransfer *xfer;
-
-    xfer = (XHCITransfer *)transfer->user_data;
-    xhci = xfer->xhci;
-
-    DPRINTF("xhci_xfer_cb(slot=%d, ep=%d, status=%d)\n", xfer->slotid,
-            xfer->epid, transfer->status);
-
-    assert(xfer->slotid >= 1 && xfer->slotid <= MAXSLOTS);
-    assert(xfer->epid >= 1 && xfer->epid <= 31);
-
-    if (xfer->cancelled) {
-        DPRINTF("xhci: transfer cancelled, not reporting anything\n");
-        xfer->running = 0;
-        return;
-    }
-
-    XHCIEPContext *epctx;
-    XHCISlot *slot;
-    slot = &xhci->slots[xfer->slotid-1];
-    assert(slot->eps[xfer->epid-1]);
-    epctx = slot->eps[xfer->epid-1];
-
-    if (xfer->bg_xfer) {
-        DPRINTF("xhci: background transfer, updating\n");
-        xfer->complete = 1;
-        xfer->running = 0;
-        xhci_bg_update(xhci, epctx);
-        return;
-    }
-
-    if (xfer->iso_xfer) {
-        transfer->status = transfer->iso_packet_desc[0].status;
-        transfer->actual_length = transfer->iso_packet_desc[0].actual_length;
-    }
-
-    xfer->status = libusb_to_ccode(transfer->status);
-
-    xfer->complete = 1;
-    xfer->running = 0;
-
-    if (transfer->status == LIBUSB_TRANSFER_STALL)
-        xhci_stall_ep(xhci, epctx, xfer);
+    XHCIState *xhci = xfer->xhci;
+    USBDevice *dev;
+    USBEndpoint *ep;
+    int dir;
 
-    DPRINTF("xhci: transfer actual length = %d\n", transfer->actual_length);
+    dir = xfer->in_xfer ? USB_TOKEN_IN : USB_TOKEN_OUT;
 
-    if (xfer->in_xfer) {
-        if (xfer->epid == 1) {
-            xhci_xfer_data(xhci, xfer, xfer->data + 8,
-                           transfer->actual_length, 1, 0, 1);
-        } else {
-            xhci_xfer_data(xhci, xfer, xfer->data,
-                           transfer->actual_length, 1, 0, 1);
-        }
+    if (xfer->packet.ep) {
+        ep = xfer->packet.ep;
+        dev = ep->dev;
     } else {
-        xhci_xfer_data(xhci, xfer, NULL, transfer->actual_length, 0, 0, 1);
-    }
-
-    xhci_kick_ep(xhci, xfer->slotid, xfer->epid);
-}
-
-static int xhci_hle_control(XHCIState *xhci, XHCITransfer *xfer,
-                            uint8_t bmRequestType, uint8_t bRequest,
-                            uint16_t wValue, uint16_t wIndex, uint16_t wLength)
-{
-    uint16_t type_req = (bmRequestType << 8) | bRequest;
-
-    switch (type_req) {
-        case 0x0000 | USB_REQ_SET_CONFIGURATION:
-            DPRINTF("xhci: HLE switch configuration\n");
-            return xhci_switch_config(xhci, xfer->slotid, wValue) == 0;
-        case 0x0100 | USB_REQ_SET_INTERFACE:
-            DPRINTF("xhci: HLE set interface altsetting\n");
-            return xhci_set_iface_alt(xhci, xfer->slotid, wIndex, wValue) == 0;
-        case 0x0200 | USB_REQ_CLEAR_FEATURE:
-            if (wValue == 0) { // endpoint halt
-                DPRINTF("xhci: HLE clear halt\n");
-                return xhci_clear_halt(xhci, xfer->slotid, wIndex);
-            }
-        case 0x0000 | USB_REQ_SET_ADDRESS:
-            fprintf(stderr, "xhci: warn: illegal SET_ADDRESS request\n");
-            return 0;
-        default:
-            return 0;
+        if (!xhci->slots[xfer->slotid-1].uport) {
+            fprintf(stderr, "xhci: slot %d has no device\n",
+                    xfer->slotid);
+            return -1;
+        }
+        dev = xhci->slots[xfer->slotid-1].uport->dev;
+        ep = usb_ep_get(dev, dir, xfer->epid >> 1);
     }
-}
-#endif
-
-static int xhci_setup_packet(XHCITransfer *xfer, USBDevice *dev)
-{
-    USBEndpoint *ep;
-    int dir;
 
-    dir = xfer->in_xfer ? USB_TOKEN_IN : USB_TOKEN_OUT;
-    ep = usb_ep_get(dev, dir, xfer->epid >> 1);
-    usb_packet_setup(&xfer->packet, dir, ep, xfer->trbs[0].addr);
-    usb_packet_addbuf(&xfer->packet, xfer->data, xfer->data_length);
+    xhci_xfer_create_sgl(xfer, dir == USB_TOKEN_IN); /* Also sets int_req */
+    usb_packet_setup(&xfer->packet, dir, ep, xfer->trbs[0].addr, false,
+                     xfer->int_req);
+    usb_packet_map(&xfer->packet, &xfer->sgl);
     DPRINTF("xhci: setup packet pid 0x%x addr %d ep %d\n",
             xfer->packet.pid, dev->addr, ep->nr);
     return 0;
 }
 
-static int xhci_complete_packet(XHCITransfer *xfer, int ret)
+static int xhci_complete_packet(XHCITransfer *xfer)
 {
-    if (ret == USB_RET_ASYNC) {
+    if (xfer->packet.status == USB_RET_ASYNC) {
         trace_usb_xhci_xfer_async(xfer);
         xfer->running_async = 1;
         xfer->running_retry = 0;
         xfer->complete = 0;
         xfer->cancelled = 0;
         return 0;
-    } else if (ret == USB_RET_NAK) {
+    } else if (xfer->packet.status == USB_RET_NAK) {
         trace_usb_xhci_xfer_nak(xfer);
         xfer->running_async = 0;
         xfer->running_retry = 1;
@@ -1419,57 +1538,46 @@ static int xhci_complete_packet(XHCITransfer *xfer, int ret)
         xfer->running_async = 0;
         xfer->running_retry = 0;
         xfer->complete = 1;
+        xhci_xfer_unmap(xfer);
     }
 
-    if (ret >= 0) {
+    if (xfer->packet.status == USB_RET_SUCCESS) {
+        trace_usb_xhci_xfer_success(xfer, xfer->packet.actual_length);
         xfer->status = CC_SUCCESS;
-        xhci_xfer_data(xfer, xfer->data, ret, xfer->in_xfer, 0, 1);
-        trace_usb_xhci_xfer_success(xfer, ret);
+        xhci_xfer_report(xfer);
         return 0;
     }
 
     /* error */
-    trace_usb_xhci_xfer_error(xfer, ret);
-    switch (ret) {
+    trace_usb_xhci_xfer_error(xfer, xfer->packet.status);
+    switch (xfer->packet.status) {
     case USB_RET_NODEV:
         xfer->status = CC_USB_TRANSACTION_ERROR;
-        xhci_xfer_data(xfer, xfer->data, 0, xfer->in_xfer, 0, 1);
+        xhci_xfer_report(xfer);
         xhci_stall_ep(xfer);
         break;
     case USB_RET_STALL:
         xfer->status = CC_STALL_ERROR;
-        xhci_xfer_data(xfer, xfer->data, 0, xfer->in_xfer, 0, 1);
+        xhci_xfer_report(xfer);
         xhci_stall_ep(xfer);
         break;
     default:
-        fprintf(stderr, "%s: FIXME: ret = %d\n", __FUNCTION__, ret);
+        fprintf(stderr, "%s: FIXME: status = %d\n", __func__,
+                xfer->packet.status);
         FIXME();
     }
     return 0;
 }
 
-static USBDevice *xhci_find_device(XHCIPort *port, uint8_t addr)
-{
-    if (!(port->portsc & PORTSC_PED)) {
-        return NULL;
-    }
-    return usb_find_device(&port->port, addr);
-}
-
 static int xhci_fire_ctl_transfer(XHCIState *xhci, XHCITransfer *xfer)
 {
     XHCITRB *trb_setup, *trb_status;
     uint8_t bmRequestType;
-    uint16_t wLength;
-    XHCIPort *port;
-    USBDevice *dev;
-    int ret;
 
     trb_setup = &xfer->trbs[0];
     trb_status = &xfer->trbs[xfer->trb_count-1];
 
-    trace_usb_xhci_xfer_start(xfer, xfer->slotid, xfer->epid,
-                              trb_setup->parameter >> 48);
+    trace_usb_xhci_xfer_start(xfer, xfer->slotid, xfer->epid);
 
     /* at most one Event Data TRB allowed after STATUS */
     if (TRB_TYPE(*trb_status) == TR_EVDATA && xfer->trb_count > 2) {
@@ -1498,93 +1606,87 @@ static int xhci_fire_ctl_transfer(XHCIState *xhci, XHCITransfer *xfer)
     }
 
     bmRequestType = trb_setup->parameter;
-    wLength = trb_setup->parameter >> 48;
-
-    if (xfer->data && xfer->data_alloced < wLength) {
-        xfer->data_alloced = 0;
-        g_free(xfer->data);
-        xfer->data = NULL;
-    }
-    if (!xfer->data) {
-        DPRINTF("xhci: alloc %d bytes data\n", wLength);
-        xfer->data = g_malloc(wLength+1);
-        xfer->data_alloced = wLength;
-    }
-    xfer->data_length = wLength;
-
-    port = &xhci->ports[xhci->slots[xfer->slotid-1].port-1];
-    dev = xhci_find_device(port, xhci->slots[xfer->slotid-1].devaddr);
-    if (!dev) {
-        fprintf(stderr, "xhci: slot %d port %d has no device\n", xfer->slotid,
-                xhci->slots[xfer->slotid-1].port);
-        return -1;
-    }
 
     xfer->in_xfer = bmRequestType & USB_DIR_IN;
     xfer->iso_xfer = false;
 
-    xhci_setup_packet(xfer, dev);
-    xfer->packet.parameter = trb_setup->parameter;
-    if (!xfer->in_xfer) {
-        xhci_xfer_data(xfer, xfer->data, wLength, 0, 1, 0);
+    if (xhci_setup_packet(xfer) < 0) {
+        return -1;
     }
+    xfer->packet.parameter = trb_setup->parameter;
 
-    ret = usb_handle_packet(dev, &xfer->packet);
+    usb_handle_packet(xfer->packet.ep->dev, &xfer->packet);
 
-    xhci_complete_packet(xfer, ret);
+    xhci_complete_packet(xfer);
     if (!xfer->running_async && !xfer->running_retry) {
         xhci_kick_ep(xhci, xfer->slotid, xfer->epid);
     }
     return 0;
 }
 
-static int xhci_submit(XHCIState *xhci, XHCITransfer *xfer, XHCIEPContext *epctx)
+static void xhci_calc_iso_kick(XHCIState *xhci, XHCITransfer *xfer,
+                               XHCIEPContext *epctx, uint64_t mfindex)
 {
-    XHCIPort *port;
-    USBDevice *dev;
-    int ret;
-
-    DPRINTF("xhci_submit(slotid=%d,epid=%d)\n", xfer->slotid, xfer->epid);
-
-    xfer->in_xfer = epctx->type>>2;
-
-    if (xfer->data && xfer->data_alloced < xfer->data_length) {
-        xfer->data_alloced = 0;
-        g_free(xfer->data);
-        xfer->data = NULL;
-    }
-    if (!xfer->data && xfer->data_length) {
-        DPRINTF("xhci: alloc %d bytes data\n", xfer->data_length);
-        xfer->data = g_malloc(xfer->data_length);
-        xfer->data_alloced = xfer->data_length;
-    }
-    if (epctx->type == ET_ISO_IN || epctx->type == ET_ISO_OUT) {
-        if (!xfer->bg_xfer) {
-            xfer->pkts = 1;
+    if (xfer->trbs[0].control & TRB_TR_SIA) {
+        uint64_t asap = ((mfindex + epctx->interval - 1) &
+                         ~(epctx->interval-1));
+        if (asap >= epctx->mfindex_last &&
+            asap <= epctx->mfindex_last + epctx->interval * 4) {
+            xfer->mfindex_kick = epctx->mfindex_last + epctx->interval;
+        } else {
+            xfer->mfindex_kick = asap;
         }
     } else {
-        xfer->pkts = 0;
+        xfer->mfindex_kick = (xfer->trbs[0].control >> TRB_TR_FRAMEID_SHIFT)
+            & TRB_TR_FRAMEID_MASK;
+        xfer->mfindex_kick |= mfindex & ~0x3fff;
+        if (xfer->mfindex_kick < mfindex) {
+            xfer->mfindex_kick += 0x4000;
+        }
     }
+}
 
-    port = &xhci->ports[xhci->slots[xfer->slotid-1].port-1];
-    dev = xhci_find_device(port, xhci->slots[xfer->slotid-1].devaddr);
-    if (!dev) {
-        fprintf(stderr, "xhci: slot %d port %d has no device\n", xfer->slotid,
-                xhci->slots[xfer->slotid-1].port);
-        return -1;
+static void xhci_check_iso_kick(XHCIState *xhci, XHCITransfer *xfer,
+                                XHCIEPContext *epctx, uint64_t mfindex)
+{
+    if (xfer->mfindex_kick > mfindex) {
+        qemu_mod_timer(epctx->kick_timer, qemu_get_clock_ns(vm_clock) +
+                       (xfer->mfindex_kick - mfindex) * 125000);
+        xfer->running_retry = 1;
+    } else {
+        epctx->mfindex_last = xfer->mfindex_kick;
+        qemu_del_timer(epctx->kick_timer);
+        xfer->running_retry = 0;
     }
+}
+
+
+static int xhci_submit(XHCIState *xhci, XHCITransfer *xfer, XHCIEPContext *epctx)
+{
+    uint64_t mfindex;
 
-    xhci_setup_packet(xfer, dev);
+    DPRINTF("xhci_submit(slotid=%d,epid=%d)\n", xfer->slotid, xfer->epid);
+
+    xfer->in_xfer = epctx->type>>2;
 
     switch(epctx->type) {
     case ET_INTR_OUT:
     case ET_INTR_IN:
     case ET_BULK_OUT:
     case ET_BULK_IN:
+        xfer->pkts = 0;
+        xfer->iso_xfer = false;
         break;
     case ET_ISO_OUT:
     case ET_ISO_IN:
-        FIXME();
+        xfer->pkts = 1;
+        xfer->iso_xfer = true;
+        mfindex = xhci_mfindex_get(xhci);
+        xhci_calc_iso_kick(xhci, xfer, epctx, mfindex);
+        xhci_check_iso_kick(xhci, xfer, epctx, mfindex);
+        if (xfer->running_retry) {
+            return -1;
+        }
         break;
     default:
         fprintf(stderr, "xhci: unknown or unhandled EP "
@@ -1593,12 +1695,12 @@ static int xhci_submit(XHCIState *xhci, XHCITransfer *xfer, XHCIEPContext *epctx
         return -1;
     }
 
-    if (!xfer->in_xfer) {
-        xhci_xfer_data(xfer, xfer->data, xfer->data_length, 0, 1, 0);
+    if (xhci_setup_packet(xfer) < 0) {
+        return -1;
     }
-    ret = usb_handle_packet(dev, &xfer->packet);
+    usb_handle_packet(xfer->packet.ep->dev, &xfer->packet);
 
-    xhci_complete_packet(xfer, ret);
+    xhci_complete_packet(xfer);
     if (!xfer->running_async && !xfer->running_retry) {
         xhci_kick_ep(xhci, xfer->slotid, xfer->epid);
     }
@@ -1607,55 +1709,20 @@ static int xhci_submit(XHCIState *xhci, XHCITransfer *xfer, XHCIEPContext *epctx
 
 static int xhci_fire_transfer(XHCIState *xhci, XHCITransfer *xfer, XHCIEPContext *epctx)
 {
-    int i;
-    unsigned int length = 0;
-    XHCITRB *trb;
-
-    for (i = 0; i < xfer->trb_count; i++) {
-        trb = &xfer->trbs[i];
-        if (TRB_TYPE(*trb) == TR_NORMAL || TRB_TYPE(*trb) == TR_ISOCH) {
-            length += trb->status & 0x1ffff;
-        }
-    }
-
-    trace_usb_xhci_xfer_start(xfer, xfer->slotid, xfer->epid, length);
-
-    if (!epctx->has_bg) {
-        xfer->data_length = length;
-        xfer->backgrounded = 0;
-        return xhci_submit(xhci, xfer, epctx);
-    } else {
-        if (!epctx->bg_running) {
-            for (i = 0; i < BG_XFERS; i++) {
-                XHCITransfer *t = &epctx->bg_transfers[i];
-                t->xhci = xhci;
-                t->epid = xfer->epid;
-                t->slotid = xfer->slotid;
-                t->pkts = BG_PKTS;
-                t->pktsize = epctx->max_psize;
-                t->data_length = t->pkts * t->pktsize;
-                t->bg_xfer = 1;
-                if (xhci_submit(xhci, t, epctx) < 0) {
-                    fprintf(stderr, "xhci: bg submit failed\n");
-                    return -1;
-                }
-            }
-            epctx->bg_running = 1;
-        }
-        xfer->backgrounded = 1;
-        xhci_bg_update(xhci, epctx);
-        return 0;
-    }
+    trace_usb_xhci_xfer_start(xfer, xfer->slotid, xfer->epid);
+    return xhci_submit(xhci, xfer, epctx);
 }
 
 static void xhci_kick_ep(XHCIState *xhci, unsigned int slotid, unsigned int epid)
 {
     XHCIEPContext *epctx;
+    USBEndpoint *ep = NULL;
+    uint64_t mfindex;
     int length;
     int i;
 
     trace_usb_xhci_ep_kick(slotid, epid);
-    assert(slotid >= 1 && slotid <= MAXSLOTS);
+    assert(slotid >= 1 && slotid <= xhci->numslots);
     assert(epid >= 1 && epid <= 31);
 
     if (!xhci->slots[slotid-1].enabled) {
@@ -1670,18 +1737,34 @@ static void xhci_kick_ep(XHCIState *xhci, unsigned int slotid, unsigned int epid
     }
 
     if (epctx->retry) {
-        /* retry nak'ed transfer */
         XHCITransfer *xfer = epctx->retry;
-        int result;
 
         trace_usb_xhci_xfer_retry(xfer);
         assert(xfer->running_retry);
-        xhci_setup_packet(xfer, xfer->packet.ep->dev);
-        result = usb_handle_packet(xfer->packet.ep->dev, &xfer->packet);
-        if (result == USB_RET_NAK) {
-            return;
+        if (xfer->iso_xfer) {
+            /* retry delayed iso transfer */
+            mfindex = xhci_mfindex_get(xhci);
+            xhci_check_iso_kick(xhci, xfer, epctx, mfindex);
+            if (xfer->running_retry) {
+                return;
+            }
+            if (xhci_setup_packet(xfer) < 0) {
+                return;
+            }
+            usb_handle_packet(xfer->packet.ep->dev, &xfer->packet);
+            assert(xfer->packet.status != USB_RET_NAK);
+            xhci_complete_packet(xfer);
+        } else {
+            /* retry nak'ed transfer */
+            if (xhci_setup_packet(xfer) < 0) {
+                return;
+            }
+            usb_handle_packet(xfer->packet.ep->dev, &xfer->packet);
+            if (xfer->packet.status == USB_RET_NAK) {
+                return;
+            }
+            xhci_complete_packet(xfer);
         }
-        xhci_complete_packet(xfer, result);
         assert(!xfer->running_retry);
         epctx->retry = NULL;
     }
@@ -1695,7 +1778,7 @@ static void xhci_kick_ep(XHCIState *xhci, unsigned int slotid, unsigned int epid
 
     while (1) {
         XHCITransfer *xfer = &epctx->transfers[epctx->next_xfer];
-        if (xfer->running_async || xfer->running_retry || xfer->backgrounded) {
+        if (xfer->running_async || xfer->running_retry) {
             break;
         }
         length = xhci_ring_chain_length(xhci, &epctx->ring);
@@ -1726,14 +1809,18 @@ static void xhci_kick_ep(XHCIState *xhci, unsigned int slotid, unsigned int epid
         if (epid == 1) {
             if (xhci_fire_ctl_transfer(xhci, xfer) >= 0) {
                 epctx->next_xfer = (epctx->next_xfer + 1) % TD_QUEUE;
+                ep = xfer->packet.ep;
             } else {
                 fprintf(stderr, "xhci: error firing CTL transfer\n");
             }
         } else {
             if (xhci_fire_transfer(xhci, xfer, epctx) >= 0) {
                 epctx->next_xfer = (epctx->next_xfer + 1) % TD_QUEUE;
+                ep = xfer->packet.ep;
             } else {
-                fprintf(stderr, "xhci: error firing data transfer\n");
+                if (!xfer->iso_xfer) {
+                    fprintf(stderr, "xhci: error firing data transfer\n");
+                }
             }
         }
 
@@ -1746,14 +1833,17 @@ static void xhci_kick_ep(XHCIState *xhci, unsigned int slotid, unsigned int epid
             break;
         }
     }
+    if (ep) {
+        usb_device_flush_ep_queue(ep->dev, ep);
+    }
 }
 
 static TRBCCode xhci_enable_slot(XHCIState *xhci, unsigned int slotid)
 {
     trace_usb_xhci_slot_enable(slotid);
-    assert(slotid >= 1 && slotid <= MAXSLOTS);
+    assert(slotid >= 1 && slotid <= xhci->numslots);
     xhci->slots[slotid-1].enabled = 1;
-    xhci->slots[slotid-1].port = 0;
+    xhci->slots[slotid-1].uport = NULL;
     memset(xhci->slots[slotid-1].eps, 0, sizeof(XHCIEPContext*)*31);
 
     return CC_SUCCESS;
@@ -1764,7 +1854,7 @@ static TRBCCode xhci_disable_slot(XHCIState *xhci, unsigned int slotid)
     int i;
 
     trace_usb_xhci_slot_disable(slotid);
-    assert(slotid >= 1 && slotid <= MAXSLOTS);
+    assert(slotid >= 1 && slotid <= xhci->numslots);
 
     for (i = 1; i <= 31; i++) {
         if (xhci->slots[slotid-1].eps[i-1]) {
@@ -1776,32 +1866,57 @@ static TRBCCode xhci_disable_slot(XHCIState *xhci, unsigned int slotid)
     return CC_SUCCESS;
 }
 
+static USBPort *xhci_lookup_uport(XHCIState *xhci, uint32_t *slot_ctx)
+{
+    USBPort *uport;
+    char path[32];
+    int i, pos, port;
+
+    port = (slot_ctx[1]>>16) & 0xFF;
+    port = xhci->ports[port-1].uport->index+1;
+    pos = snprintf(path, sizeof(path), "%d", port);
+    for (i = 0; i < 5; i++) {
+        port = (slot_ctx[0] >> 4*i) & 0x0f;
+        if (!port) {
+            break;
+        }
+        pos += snprintf(path + pos, sizeof(path) - pos, ".%d", port);
+    }
+
+    QTAILQ_FOREACH(uport, &xhci->bus.used, next) {
+        if (strcmp(uport->path, path) == 0) {
+            return uport;
+        }
+    }
+    return NULL;
+}
+
 static TRBCCode xhci_address_slot(XHCIState *xhci, unsigned int slotid,
                                   uint64_t pictx, bool bsr)
 {
     XHCISlot *slot;
+    USBPort *uport;
     USBDevice *dev;
     dma_addr_t ictx, octx, dcbaap;
     uint64_t poctx;
     uint32_t ictl_ctx[2];
     uint32_t slot_ctx[4];
     uint32_t ep0_ctx[5];
-    unsigned int port;
     int i;
     TRBCCode res;
 
     trace_usb_xhci_slot_address(slotid);
-    assert(slotid >= 1 && slotid <= MAXSLOTS);
+    assert(slotid >= 1 && slotid <= xhci->numslots);
 
     dcbaap = xhci_addr64(xhci->dcbaap_low, xhci->dcbaap_high);
-    pci_dma_read(&xhci->pci_dev, dcbaap + 8*slotid, &poctx, sizeof(poctx));
+    poctx = ldq_le_pci_dma(&xhci->pci_dev, dcbaap + 8*slotid);
     ictx = xhci_mask64(pictx);
-    octx = xhci_mask64(le64_to_cpu(poctx));
+    octx = xhci_mask64(poctx);
 
     DPRINTF("xhci: input context at "DMA_ADDR_FMT"\n", ictx);
     DPRINTF("xhci: output context at "DMA_ADDR_FMT"\n", octx);
 
-    pci_dma_read(&xhci->pci_dev, ictx, ictl_ctx, sizeof(ictl_ctx));
+    xhci_dma_read_u32s(xhci, ictx, ictl_ctx, sizeof(ictl_ctx));
 
     if (ictl_ctx[0] != 0x0 || ictl_ctx[1] != 0x3) {
         fprintf(stderr, "xhci: invalid input context control %08x %08x\n",
@@ -1809,8 +1924,8 @@ static TRBCCode xhci_address_slot(XHCIState *xhci, unsigned int slotid,
         return CC_TRB_ERROR;
     }
 
-    pci_dma_read(&xhci->pci_dev, ictx+32, slot_ctx, sizeof(slot_ctx));
-    pci_dma_read(&xhci->pci_dev, ictx+64, ep0_ctx, sizeof(ep0_ctx));
+    xhci_dma_read_u32s(xhci, ictx+32, slot_ctx, sizeof(slot_ctx));
+    xhci_dma_read_u32s(xhci, ictx+64, ep0_ctx, sizeof(ep0_ctx));
 
     DPRINTF("xhci: input slot context: %08x %08x %08x %08x\n",
             slot_ctx[0], slot_ctx[1], slot_ctx[2], slot_ctx[3]);
@@ -1818,27 +1933,31 @@ static TRBCCode xhci_address_slot(XHCIState *xhci, unsigned int slotid,
     DPRINTF("xhci: input ep0 context: %08x %08x %08x %08x %08x\n",
             ep0_ctx[0], ep0_ctx[1], ep0_ctx[2], ep0_ctx[3], ep0_ctx[4]);
 
-    port = (slot_ctx[1]>>16) & 0xFF;
-    dev = xhci->ports[port-1].port.dev;
-
-    if (port < 1 || port > MAXPORTS) {
-        fprintf(stderr, "xhci: bad port %d\n", port);
+    uport = xhci_lookup_uport(xhci, slot_ctx);
+    if (uport == NULL) {
+        fprintf(stderr, "xhci: port not found\n");
         return CC_TRB_ERROR;
-    } else if (!dev) {
-        fprintf(stderr, "xhci: port %d not connected\n", port);
+    }
+
+    dev = uport->dev;
+    if (!dev) {
+        fprintf(stderr, "xhci: port %s not connected\n", uport->path);
         return CC_USB_TRANSACTION_ERROR;
     }
 
-    for (i = 0; i < MAXSLOTS; i++) {
-        if (xhci->slots[i].port == port) {
-            fprintf(stderr, "xhci: port %d already assigned to slot %d\n",
-                    port, i+1);
+    for (i = 0; i < xhci->numslots; i++) {
+        if (i == slotid-1) {
+            continue;
+        }
+        if (xhci->slots[i].uport == uport) {
+            fprintf(stderr, "xhci: port %s already assigned to slot %d\n",
+                    uport->path, i+1);
             return CC_TRB_ERROR;
         }
     }
 
     slot = &xhci->slots[slotid-1];
-    slot->port = port;
+    slot->uport = uport;
     slot->ctx = octx;
 
     if (bsr) {
@@ -1847,6 +1966,7 @@ static TRBCCode xhci_address_slot(XHCIState *xhci, unsigned int slotid,
         slot->devaddr = xhci->devaddr++;
         slot_ctx[3] = (SLOT_ADDRESSED << SLOT_STATE_SHIFT) | slot->devaddr;
         DPRINTF("xhci: device address is %d\n", slot->devaddr);
+        usb_device_reset(dev);
         usb_device_handle_control(dev, NULL,
                                   DeviceOutRequest | USB_REQ_SET_ADDRESS,
                                   slot->devaddr, 0, 0, NULL);
@@ -1859,8 +1979,8 @@ static TRBCCode xhci_address_slot(XHCIState *xhci, unsigned int slotid,
     DPRINTF("xhci: output ep0 context: %08x %08x %08x %08x %08x\n",
             ep0_ctx[0], ep0_ctx[1], ep0_ctx[2], ep0_ctx[3], ep0_ctx[4]);
 
-    pci_dma_write(&xhci->pci_dev, octx, slot_ctx, sizeof(slot_ctx));
-    pci_dma_write(&xhci->pci_dev, octx+32, ep0_ctx, sizeof(ep0_ctx));
+    xhci_dma_write_u32s(xhci, octx, slot_ctx, sizeof(slot_ctx));
+    xhci_dma_write_u32s(xhci, octx+32, ep0_ctx, sizeof(ep0_ctx));
 
     return res;
 }
@@ -1878,7 +1998,7 @@ static TRBCCode xhci_configure_slot(XHCIState *xhci, unsigned int slotid,
     TRBCCode res;
 
     trace_usb_xhci_slot_configure(slotid);
-    assert(slotid >= 1 && slotid <= MAXSLOTS);
+    assert(slotid >= 1 && slotid <= xhci->numslots);
 
     ictx = xhci_mask64(pictx);
     octx = xhci->slots[slotid-1].ctx;
@@ -1893,17 +2013,17 @@ static TRBCCode xhci_configure_slot(XHCIState *xhci, unsigned int slotid,
             }
         }
 
-        pci_dma_read(&xhci->pci_dev, octx, slot_ctx, sizeof(slot_ctx));
+        xhci_dma_read_u32s(xhci, octx, slot_ctx, sizeof(slot_ctx));
         slot_ctx[3] &= ~(SLOT_STATE_MASK << SLOT_STATE_SHIFT);
         slot_ctx[3] |= SLOT_ADDRESSED << SLOT_STATE_SHIFT;
         DPRINTF("xhci: output slot context: %08x %08x %08x %08x\n",
                 slot_ctx[0], slot_ctx[1], slot_ctx[2], slot_ctx[3]);
-        pci_dma_write(&xhci->pci_dev, octx, slot_ctx, sizeof(slot_ctx));
+        xhci_dma_write_u32s(xhci, octx, slot_ctx, sizeof(slot_ctx));
 
         return CC_SUCCESS;
     }
 
-    pci_dma_read(&xhci->pci_dev, ictx, ictl_ctx, sizeof(ictl_ctx));
+    xhci_dma_read_u32s(xhci, ictx, ictl_ctx, sizeof(ictl_ctx));
 
     if ((ictl_ctx[0] & 0x3) != 0x0 || (ictl_ctx[1] & 0x3) != 0x1) {
         fprintf(stderr, "xhci: invalid input context control %08x %08x\n",
@@ -1911,8 +2031,8 @@ static TRBCCode xhci_configure_slot(XHCIState *xhci, unsigned int slotid,
         return CC_TRB_ERROR;
     }
 
-    pci_dma_read(&xhci->pci_dev, ictx+32, islot_ctx, sizeof(islot_ctx));
-    pci_dma_read(&xhci->pci_dev, octx, slot_ctx, sizeof(slot_ctx));
+    xhci_dma_read_u32s(xhci, ictx+32, islot_ctx, sizeof(islot_ctx));
+    xhci_dma_read_u32s(xhci, octx, slot_ctx, sizeof(slot_ctx));
 
     if (SLOT_STATE(slot_ctx[3]) < SLOT_ADDRESSED) {
         fprintf(stderr, "xhci: invalid slot state %08x\n", slot_ctx[3]);
@@ -1924,8 +2044,7 @@ static TRBCCode xhci_configure_slot(XHCIState *xhci, unsigned int slotid,
             xhci_disable_ep(xhci, slotid, i);
         }
         if (ictl_ctx[1] & (1<<i)) {
-            pci_dma_read(&xhci->pci_dev, ictx+32+(32*i), ep_ctx,
-                         sizeof(ep_ctx));
+            xhci_dma_read_u32s(xhci, ictx+32+(32*i), ep_ctx, sizeof(ep_ctx));
             DPRINTF("xhci: input ep%d.%d context: %08x %08x %08x %08x %08x\n",
                     i/2, i%2, ep_ctx[0], ep_ctx[1], ep_ctx[2],
                     ep_ctx[3], ep_ctx[4]);
@@ -1937,7 +2056,7 @@ static TRBCCode xhci_configure_slot(XHCIState *xhci, unsigned int slotid,
             DPRINTF("xhci: output ep%d.%d context: %08x %08x %08x %08x %08x\n",
                     i/2, i%2, ep_ctx[0], ep_ctx[1], ep_ctx[2],
                     ep_ctx[3], ep_ctx[4]);
-            pci_dma_write(&xhci->pci_dev, octx+(32*i), ep_ctx, sizeof(ep_ctx));
+            xhci_dma_write_u32s(xhci, octx+(32*i), ep_ctx, sizeof(ep_ctx));
         }
     }
 
@@ -1949,7 +2068,7 @@ static TRBCCode xhci_configure_slot(XHCIState *xhci, unsigned int slotid,
     DPRINTF("xhci: output slot context: %08x %08x %08x %08x\n",
             slot_ctx[0], slot_ctx[1], slot_ctx[2], slot_ctx[3]);
 
-    pci_dma_write(&xhci->pci_dev, octx, slot_ctx, sizeof(slot_ctx));
+    xhci_dma_write_u32s(xhci, octx, slot_ctx, sizeof(slot_ctx));
 
     return CC_SUCCESS;
 }
@@ -1966,7 +2085,7 @@ static TRBCCode xhci_evaluate_slot(XHCIState *xhci, unsigned int slotid,
     uint32_t slot_ctx[4];
 
     trace_usb_xhci_slot_evaluate(slotid);
-    assert(slotid >= 1 && slotid <= MAXSLOTS);
+    assert(slotid >= 1 && slotid <= xhci->numslots);
 
     ictx = xhci_mask64(pictx);
     octx = xhci->slots[slotid-1].ctx;
@@ -1974,7 +2093,7 @@ static TRBCCode xhci_evaluate_slot(XHCIState *xhci, unsigned int slotid,
     DPRINTF("xhci: input context at "DMA_ADDR_FMT"\n", ictx);
     DPRINTF("xhci: output context at "DMA_ADDR_FMT"\n", octx);
 
-    pci_dma_read(&xhci->pci_dev, ictx, ictl_ctx, sizeof(ictl_ctx));
+    xhci_dma_read_u32s(xhci, ictx, ictl_ctx, sizeof(ictl_ctx));
 
     if (ictl_ctx[0] != 0x0 || ictl_ctx[1] & ~0x3) {
         fprintf(stderr, "xhci: invalid input context control %08x %08x\n",
@@ -1983,12 +2102,12 @@ static TRBCCode xhci_evaluate_slot(XHCIState *xhci, unsigned int slotid,
     }
 
     if (ictl_ctx[1] & 0x1) {
-        pci_dma_read(&xhci->pci_dev, ictx+32, islot_ctx, sizeof(islot_ctx));
+        xhci_dma_read_u32s(xhci, ictx+32, islot_ctx, sizeof(islot_ctx));
 
         DPRINTF("xhci: input slot context: %08x %08x %08x %08x\n",
                 islot_ctx[0], islot_ctx[1], islot_ctx[2], islot_ctx[3]);
 
-        pci_dma_read(&xhci->pci_dev, octx, slot_ctx, sizeof(slot_ctx));
+        xhci_dma_read_u32s(xhci, octx, slot_ctx, sizeof(slot_ctx));
 
         slot_ctx[1] &= ~0xFFFF; /* max exit latency */
         slot_ctx[1] |= islot_ctx[1] & 0xFFFF;
@@ -1998,17 +2117,17 @@ static TRBCCode xhci_evaluate_slot(XHCIState *xhci, unsigned int slotid,
         DPRINTF("xhci: output slot context: %08x %08x %08x %08x\n",
                 slot_ctx[0], slot_ctx[1], slot_ctx[2], slot_ctx[3]);
 
-        pci_dma_write(&xhci->pci_dev, octx, slot_ctx, sizeof(slot_ctx));
+        xhci_dma_write_u32s(xhci, octx, slot_ctx, sizeof(slot_ctx));
     }
 
     if (ictl_ctx[1] & 0x2) {
-        pci_dma_read(&xhci->pci_dev, ictx+64, iep0_ctx, sizeof(iep0_ctx));
+        xhci_dma_read_u32s(xhci, ictx+64, iep0_ctx, sizeof(iep0_ctx));
 
         DPRINTF("xhci: input ep0 context: %08x %08x %08x %08x %08x\n",
                 iep0_ctx[0], iep0_ctx[1], iep0_ctx[2],
                 iep0_ctx[3], iep0_ctx[4]);
 
-        pci_dma_read(&xhci->pci_dev, octx+32, ep0_ctx, sizeof(ep0_ctx));
+        xhci_dma_read_u32s(xhci, octx+32, ep0_ctx, sizeof(ep0_ctx));
 
         ep0_ctx[1] &= ~0xFFFF0000; /* max packet size*/
         ep0_ctx[1] |= iep0_ctx[1] & 0xFFFF0000;
@@ -2016,7 +2135,7 @@ static TRBCCode xhci_evaluate_slot(XHCIState *xhci, unsigned int slotid,
         DPRINTF("xhci: output ep0 context: %08x %08x %08x %08x %08x\n",
                 ep0_ctx[0], ep0_ctx[1], ep0_ctx[2], ep0_ctx[3], ep0_ctx[4]);
 
-        pci_dma_write(&xhci->pci_dev, octx+32, ep0_ctx, sizeof(ep0_ctx));
+        xhci_dma_write_u32s(xhci, octx+32, ep0_ctx, sizeof(ep0_ctx));
     }
 
     return CC_SUCCESS;
@@ -2029,7 +2148,7 @@ static TRBCCode xhci_reset_slot(XHCIState *xhci, unsigned int slotid)
     int i;
 
     trace_usb_xhci_slot_reset(slotid);
-    assert(slotid >= 1 && slotid <= MAXSLOTS);
+    assert(slotid >= 1 && slotid <= xhci->numslots);
 
     octx = xhci->slots[slotid-1].ctx;
 
@@ -2041,12 +2160,12 @@ static TRBCCode xhci_reset_slot(XHCIState *xhci, unsigned int slotid)
         }
     }
 
-    pci_dma_read(&xhci->pci_dev, octx, slot_ctx, sizeof(slot_ctx));
+    xhci_dma_read_u32s(xhci, octx, slot_ctx, sizeof(slot_ctx));
     slot_ctx[3] &= ~(SLOT_STATE_MASK << SLOT_STATE_SHIFT);
     slot_ctx[3] |= SLOT_DEFAULT << SLOT_STATE_SHIFT;
     DPRINTF("xhci: output slot context: %08x %08x %08x %08x\n",
             slot_ctx[0], slot_ctx[1], slot_ctx[2], slot_ctx[3]);
-    pci_dma_write(&xhci->pci_dev, octx, slot_ctx, sizeof(slot_ctx));
+    xhci_dma_write_u32s(xhci, octx, slot_ctx, sizeof(slot_ctx));
 
     return CC_SUCCESS;
 }
@@ -2055,7 +2174,7 @@ static unsigned int xhci_get_slot(XHCIState *xhci, XHCIEvent *event, XHCITRB *tr
 {
     unsigned int slotid;
     slotid = (trb->control >> TRB_CR_SLOTID_SHIFT) & TRB_CR_SLOTID_MASK;
-    if (slotid < 1 || slotid > MAXSLOTS) {
+    if (slotid < 1 || slotid > xhci->numslots) {
         fprintf(stderr, "xhci: bad slot id %d\n", slotid);
         event->ccode = CC_TRB_ERROR;
         return 0;
@@ -2070,7 +2189,7 @@ static unsigned int xhci_get_slot(XHCIState *xhci, XHCIEvent *event, XHCITRB *tr
 static TRBCCode xhci_get_port_bandwidth(XHCIState *xhci, uint64_t pctx)
 {
     dma_addr_t ctx;
-    uint8_t bw_ctx[MAXPORTS+1];
+    uint8_t bw_ctx[xhci->numports+1];
 
     DPRINTF("xhci_get_port_bandwidth()\n");
 
@@ -2080,7 +2199,7 @@ static TRBCCode xhci_get_port_bandwidth(XHCIState *xhci, uint64_t pctx)
 
     /* TODO: actually implement real values here */
     bw_ctx[0] = 0;
-    memset(&bw_ctx[1], 80, MAXPORTS); /* 80% */
+    memset(&bw_ctx[1], 80, xhci->numports); /* 80% */
     pci_dma_write(&xhci->pci_dev, ctx, bw_ctx, sizeof(bw_ctx));
 
     return CC_SUCCESS;
@@ -2147,12 +2266,12 @@ static void xhci_process_commands(XHCIState *xhci)
         event.ptr = addr;
         switch (type) {
         case CR_ENABLE_SLOT:
-            for (i = 0; i < MAXSLOTS; i++) {
+            for (i = 0; i < xhci->numslots; i++) {
                 if (!xhci->slots[i].enabled) {
                     break;
                 }
             }
-            if (i >= MAXSLOTS) {
+            if (i >= xhci->numslots) {
                 fprintf(stderr, "xhci: no device slots available\n");
                 event.ccode = CC_NO_SLOTS_ERROR;
             } else {
@@ -2244,36 +2363,90 @@ static void xhci_process_commands(XHCIState *xhci)
             break;
         }
         event.slotid = slotid;
-        xhci_event(xhci, &event);
+        xhci_event(xhci, &event, 0);
+    }
+}
+
+static bool xhci_port_have_device(XHCIPort *port)
+{
+    if (!port->uport->dev || !port->uport->dev->attached) {
+        return false; /* no device present */
+    }
+    if (!((1 << port->uport->dev->speed) & port->speedmask)) {
+        return false; /* speed mismatch */
+    }
+    return true;
+}
+
+static void xhci_port_notify(XHCIPort *port, uint32_t bits)
+{
+    XHCIEvent ev = { ER_PORT_STATUS_CHANGE, CC_SUCCESS,
+                     port->portnr << 24 };
+
+    if ((port->portsc & bits) == bits) {
+        return;
     }
+    port->portsc |= bits;
+    if (!xhci_running(port->xhci)) {
+        return;
+    }
+    xhci_event(port->xhci, &ev, 0);
 }
 
-static void xhci_update_port(XHCIState *xhci, XHCIPort *port, int is_detach)
+static void xhci_port_update(XHCIPort *port, int is_detach)
 {
-    int nr = port->port.index + 1;
+    uint32_t pls = PLS_RX_DETECT;
 
     port->portsc = PORTSC_PP;
-    if (port->port.dev && port->port.dev->attached && !is_detach) {
+    if (!is_detach && xhci_port_have_device(port)) {
         port->portsc |= PORTSC_CCS;
-        switch (port->port.dev->speed) {
+        switch (port->uport->dev->speed) {
         case USB_SPEED_LOW:
             port->portsc |= PORTSC_SPEED_LOW;
+            pls = PLS_POLLING;
             break;
         case USB_SPEED_FULL:
             port->portsc |= PORTSC_SPEED_FULL;
+            pls = PLS_POLLING;
             break;
         case USB_SPEED_HIGH:
             port->portsc |= PORTSC_SPEED_HIGH;
+            pls = PLS_POLLING;
+            break;
+        case USB_SPEED_SUPER:
+            port->portsc |= PORTSC_SPEED_SUPER;
+            port->portsc |= PORTSC_PED;
+            pls = PLS_U0;
             break;
         }
     }
+    set_field(&port->portsc, pls, PORTSC_PLS);
+    trace_usb_xhci_port_link(port->portnr, pls);
+    xhci_port_notify(port, PORTSC_CSC);
+}
+
+static void xhci_port_reset(XHCIPort *port)
+{
+    trace_usb_xhci_port_reset(port->portnr);
 
-    if (xhci_running(xhci)) {
-        port->portsc |= PORTSC_CSC;
-        XHCIEvent ev = { ER_PORT_STATUS_CHANGE, CC_SUCCESS, nr << 24};
-        xhci_event(xhci, &ev);
-        DPRINTF("xhci: port change event for port %d\n", nr);
+    if (!xhci_port_have_device(port)) {
+        return;
     }
+
+    usb_device_reset(port->uport->dev);
+
+    switch (port->uport->dev->speed) {
+    case USB_SPEED_LOW:
+    case USB_SPEED_FULL:
+    case USB_SPEED_HIGH:
+        set_field(&port->portsc, PLS_U0, PORTSC_PLS);
+        trace_usb_xhci_port_link(port->portnr, PLS_U0);
+        port->portsc |= PORTSC_PED;
+        break;
+    }
+
+    port->portsc &= ~PORTSC_PR;
+    xhci_port_notify(port, PORTSC_PRC);
 }
 
 static void xhci_reset(DeviceState *dev)
@@ -2296,32 +2469,38 @@ static void xhci_reset(DeviceState *dev)
     xhci->config = 0;
     xhci->devaddr = 2;
 
-    for (i = 0; i < MAXSLOTS; i++) {
+    for (i = 0; i < xhci->numslots; i++) {
         xhci_disable_slot(xhci, i+1);
     }
 
-    for (i = 0; i < MAXPORTS; i++) {
-        xhci_update_port(xhci, xhci->ports + i, 0);
+    for (i = 0; i < xhci->numports; i++) {
+        xhci_port_update(xhci->ports + i, 0);
     }
 
-    xhci->mfindex = 0;
-    xhci->iman = 0;
-    xhci->imod = 0;
-    xhci->erstsz = 0;
-    xhci->erstba_low = 0;
-    xhci->erstba_high = 0;
-    xhci->erdp_low = 0;
-    xhci->erdp_high = 0;
+    for (i = 0; i < xhci->numintrs; i++) {
+        xhci->intr[i].iman = 0;
+        xhci->intr[i].imod = 0;
+        xhci->intr[i].erstsz = 0;
+        xhci->intr[i].erstba_low = 0;
+        xhci->intr[i].erstba_high = 0;
+        xhci->intr[i].erdp_low = 0;
+        xhci->intr[i].erdp_high = 0;
+        xhci->intr[i].msix_used = 0;
+
+        xhci->intr[i].er_ep_idx = 0;
+        xhci->intr[i].er_pcs = 1;
+        xhci->intr[i].er_full = 0;
+        xhci->intr[i].ev_buffer_put = 0;
+        xhci->intr[i].ev_buffer_get = 0;
+    }
 
-    xhci->er_ep_idx = 0;
-    xhci->er_pcs = 1;
-    xhci->er_full = 0;
-    xhci->ev_buffer_put = 0;
-    xhci->ev_buffer_get = 0;
+    xhci->mfindex_start = qemu_get_clock_ns(vm_clock);
+    xhci_mfwrap_update(xhci);
 }
 
-static uint32_t xhci_cap_read(XHCIState *xhci, uint32_t reg)
+static uint64_t xhci_cap_read(void *ptr, hwaddr reg, unsigned size)
 {
+    XHCIState *xhci = ptr;
     uint32_t ret;
 
     switch (reg) {
@@ -2329,7 +2508,8 @@ static uint32_t xhci_cap_read(XHCIState *xhci, uint32_t reg)
         ret = 0x01000000 | LEN_CAP;
         break;
     case 0x04: /* HCSPARAMS 1 */
-        ret = (MAXPORTS<<24) | (MAXINTRS<<8) | MAXSLOTS;
+        ret = ((xhci->numports_2+xhci->numports_3)<<24)
+            | (xhci->numintrs<<8) | xhci->numslots;
         break;
     case 0x08: /* HCSPARAMS 2 */
         ret = 0x0000000f;
@@ -2356,10 +2536,10 @@ static uint32_t xhci_cap_read(XHCIState *xhci, uint32_t reg)
         ret = 0x02000402; /* USB 2.0 */
         break;
     case 0x24: /* Supported Protocol:04 */
-        ret = 0x20425455; /* "USB " */
+        ret = 0x20425355; /* "USB " */
         break;
     case 0x28: /* Supported Protocol:08 */
-        ret = 0x00000001 | (USB2_PORTS<<8);
+        ret = 0x00000001 | (xhci->numports_2<<8);
         break;
     case 0x2c: /* Supported Protocol:0c */
         ret = 0x00000000; /* reserved */
@@ -2368,16 +2548,16 @@ static uint32_t xhci_cap_read(XHCIState *xhci, uint32_t reg)
         ret = 0x03000002; /* USB 3.0 */
         break;
     case 0x34: /* Supported Protocol:04 */
-        ret = 0x20425455; /* "USB " */
+        ret = 0x20425355; /* "USB " */
         break;
     case 0x38: /* Supported Protocol:08 */
-        ret = 0x00000000 | (USB2_PORTS+1) | (USB3_PORTS<<8);
+        ret = 0x00000000 | (xhci->numports_2+1) | (xhci->numports_3<<8);
         break;
     case 0x3c: /* Supported Protocol:0c */
         ret = 0x00000000; /* reserved */
         break;
     default:
-        fprintf(stderr, "xhci_cap_read: reg %d unimplemented\n", reg);
+        fprintf(stderr, "xhci_cap_read: reg %d unimplemented\n", (int)reg);
         ret = 0;
     }
 
@@ -2385,20 +2565,14 @@ static uint32_t xhci_cap_read(XHCIState *xhci, uint32_t reg)
     return ret;
 }
 
-static uint32_t xhci_port_read(XHCIState *xhci, uint32_t reg)
+static uint64_t xhci_port_read(void *ptr, hwaddr reg, unsigned size)
 {
-    uint32_t port = reg >> 4;
+    XHCIPort *port = ptr;
     uint32_t ret;
 
-    if (port >= MAXPORTS) {
-        fprintf(stderr, "xhci_port_read: port %d out of bounds\n", port);
-        ret = 0;
-        goto out;
-    }
-
-    switch (reg & 0xf) {
+    switch (reg) {
     case 0x00: /* PORTSC */
-        ret = xhci->ports[port].portsc;
+        ret = port->portsc;
         break;
     case 0x04: /* PORTPMSC */
     case 0x08: /* PORTLI */
@@ -2407,65 +2581,56 @@ static uint32_t xhci_port_read(XHCIState *xhci, uint32_t reg)
     case 0x0c: /* reserved */
     default:
         fprintf(stderr, "xhci_port_read (port %d): reg 0x%x unimplemented\n",
-                port, reg);
+                port->portnr, (uint32_t)reg);
         ret = 0;
     }
 
-out:
-    trace_usb_xhci_port_read(port, reg & 0x0f, ret);
+    trace_usb_xhci_port_read(port->portnr, reg, ret);
     return ret;
 }
 
-static void xhci_port_write(XHCIState *xhci, uint32_t reg, uint32_t val)
+static void xhci_port_write(void *ptr, hwaddr reg,
+                            uint64_t val, unsigned size)
 {
-    uint32_t port = reg >> 4;
+    XHCIPort *port = ptr;
     uint32_t portsc;
 
-    trace_usb_xhci_port_write(port, reg & 0x0f, val);
+    trace_usb_xhci_port_write(port->portnr, reg, val);
 
-    if (port >= MAXPORTS) {
-        fprintf(stderr, "xhci_port_read: port %d out of bounds\n", port);
-        return;
-    }
-
-    switch (reg & 0xf) {
+    switch (reg) {
     case 0x00: /* PORTSC */
-        portsc = xhci->ports[port].portsc;
+        portsc = port->portsc;
         /* write-1-to-clear bits*/
         portsc &= ~(val & (PORTSC_CSC|PORTSC_PEC|PORTSC_WRC|PORTSC_OCC|
                            PORTSC_PRC|PORTSC_PLC|PORTSC_CEC));
         if (val & PORTSC_LWS) {
             /* overwrite PLS only when LWS=1 */
-            portsc &= ~(PORTSC_PLS_MASK << PORTSC_PLS_SHIFT);
-            portsc |= val & (PORTSC_PLS_MASK << PORTSC_PLS_SHIFT);
+            uint32_t pls = get_field(val, PORTSC_PLS);
+            set_field(&portsc, pls, PORTSC_PLS);
+            trace_usb_xhci_port_link(port->portnr, pls);
         }
         /* read/write bits */
         portsc &= ~(PORTSC_PP|PORTSC_WCE|PORTSC_WDE|PORTSC_WOE);
         portsc |= (val & (PORTSC_PP|PORTSC_WCE|PORTSC_WDE|PORTSC_WOE));
+        port->portsc = portsc;
         /* write-1-to-start bits */
         if (val & PORTSC_PR) {
-            DPRINTF("xhci: port %d reset\n", port);
-            usb_device_reset(xhci->ports[port].port.dev);
-            portsc |= PORTSC_PRC | PORTSC_PED;
+            xhci_port_reset(port);
         }
-        xhci->ports[port].portsc = portsc;
         break;
     case 0x04: /* PORTPMSC */
     case 0x08: /* PORTLI */
     default:
         fprintf(stderr, "xhci_port_write (port %d): reg 0x%x unimplemented\n",
-                port, reg);
+                port->portnr, (uint32_t)reg);
     }
 }
 
-static uint32_t xhci_oper_read(XHCIState *xhci, uint32_t reg)
+static uint64_t xhci_oper_read(void *ptr, hwaddr reg, unsigned size)
 {
+    XHCIState *xhci = ptr;
     uint32_t ret;
 
-    if (reg >= 0x400) {
-        return xhci_port_read(xhci, reg - 0x400);
-    }
-
     switch (reg) {
     case 0x00: /* USBCMD */
         ret = xhci->usbcmd;
@@ -2495,7 +2660,7 @@ static uint32_t xhci_oper_read(XHCIState *xhci, uint32_t reg)
         ret = xhci->config;
         break;
     default:
-        fprintf(stderr, "xhci_oper_read: reg 0x%x unimplemented\n", reg);
+        fprintf(stderr, "xhci_oper_read: reg 0x%x unimplemented\n", (int)reg);
         ret = 0;
     }
 
@@ -2503,12 +2668,10 @@ static uint32_t xhci_oper_read(XHCIState *xhci, uint32_t reg)
     return ret;
 }
 
-static void xhci_oper_write(XHCIState *xhci, uint32_t reg, uint32_t val)
+static void xhci_oper_write(void *ptr, hwaddr reg,
+                            uint64_t val, unsigned size)
 {
-    if (reg >= 0x400) {
-        xhci_port_write(xhci, reg - 0x400, val);
-        return;
-    }
+    XHCIState *xhci = ptr;
 
     trace_usb_xhci_oper_write(reg, val);
 
@@ -2520,16 +2683,17 @@ static void xhci_oper_write(XHCIState *xhci, uint32_t reg, uint32_t val)
             xhci_stop(xhci);
         }
         xhci->usbcmd = val & 0xc0f;
+        xhci_mfwrap_update(xhci);
         if (val & USBCMD_HCRST) {
             xhci_reset(&xhci->pci_dev.qdev);
         }
-        xhci_irq_update(xhci);
+        xhci_intx_update(xhci);
         break;
 
     case 0x04: /* USBSTS */
         /* these bits are write-1-to-clear */
         xhci->usbsts &= ~(val & (USBSTS_HSE|USBSTS_EINT|USBSTS_PCD|USBSTS_SRE));
-        xhci_irq_update(xhci);
+        xhci_intx_update(xhci);
         break;
 
     case 0x14: /* DNCTRL */
@@ -2543,7 +2707,7 @@ static void xhci_oper_write(XHCIState *xhci, uint32_t reg, uint32_t val)
         if (xhci->crcr_low & (CRCR_CA|CRCR_CS) && (xhci->crcr_low & CRCR_CRR)) {
             XHCIEvent event = {ER_COMMAND_COMPLETE, CC_COMMAND_RING_STOPPED};
             xhci->crcr_low &= ~CRCR_CRR;
-            xhci_event(xhci, &event);
+            xhci_event(xhci, &event, 0);
             DPRINTF("xhci: command ring stopped (CRCR=%08x)\n", xhci->crcr_low);
         } else {
             dma_addr_t base = xhci_addr64(xhci->crcr_low & ~0x3f, val);
@@ -2561,101 +2725,127 @@ static void xhci_oper_write(XHCIState *xhci, uint32_t reg, uint32_t val)
         xhci->config = val & 0xff;
         break;
     default:
-        fprintf(stderr, "xhci_oper_write: reg 0x%x unimplemented\n", reg);
+        fprintf(stderr, "xhci_oper_write: reg 0x%x unimplemented\n", (int)reg);
     }
 }
 
-static uint32_t xhci_runtime_read(XHCIState *xhci, uint32_t reg)
+static uint64_t xhci_runtime_read(void *ptr, hwaddr reg,
+                                  unsigned size)
 {
-    uint32_t ret;
+    XHCIState *xhci = ptr;
+    uint32_t ret = 0;
 
-    switch (reg) {
-    case 0x00: /* MFINDEX */
-        fprintf(stderr, "xhci_runtime_read: MFINDEX not yet implemented\n");
-        ret = xhci->mfindex;
-        break;
-    case 0x20: /* IMAN */
-        ret = xhci->iman;
-        break;
-    case 0x24: /* IMOD */
-        ret = xhci->imod;
-        break;
-    case 0x28: /* ERSTSZ */
-        ret = xhci->erstsz;
-        break;
-    case 0x30: /* ERSTBA low */
-        ret = xhci->erstba_low;
-        break;
-    case 0x34: /* ERSTBA high */
-        ret = xhci->erstba_high;
-        break;
-    case 0x38: /* ERDP low */
-        ret = xhci->erdp_low;
-        break;
-    case 0x3c: /* ERDP high */
-        ret = xhci->erdp_high;
-        break;
-    default:
-        fprintf(stderr, "xhci_runtime_read: reg 0x%x unimplemented\n", reg);
-        ret = 0;
+    if (reg < 0x20) {
+        switch (reg) {
+        case 0x00: /* MFINDEX */
+            ret = xhci_mfindex_get(xhci) & 0x3fff;
+            break;
+        default:
+            fprintf(stderr, "xhci_runtime_read: reg 0x%x unimplemented\n",
+                    (int)reg);
+            break;
+        }
+    } else {
+        int v = (reg - 0x20) / 0x20;
+        XHCIInterrupter *intr = &xhci->intr[v];
+        switch (reg & 0x1f) {
+        case 0x00: /* IMAN */
+            ret = intr->iman;
+            break;
+        case 0x04: /* IMOD */
+            ret = intr->imod;
+            break;
+        case 0x08: /* ERSTSZ */
+            ret = intr->erstsz;
+            break;
+        case 0x10: /* ERSTBA low */
+            ret = intr->erstba_low;
+            break;
+        case 0x14: /* ERSTBA high */
+            ret = intr->erstba_high;
+            break;
+        case 0x18: /* ERDP low */
+            ret = intr->erdp_low;
+            break;
+        case 0x1c: /* ERDP high */
+            ret = intr->erdp_high;
+            break;
+        }
     }
 
     trace_usb_xhci_runtime_read(reg, ret);
     return ret;
 }
 
-static void xhci_runtime_write(XHCIState *xhci, uint32_t reg, uint32_t val)
+static void xhci_runtime_write(void *ptr, hwaddr reg,
+                               uint64_t val, unsigned size)
 {
-    trace_usb_xhci_runtime_read(reg, val);
+    XHCIState *xhci = ptr;
+    int v = (reg - 0x20) / 0x20;
+    XHCIInterrupter *intr = &xhci->intr[v];
+    trace_usb_xhci_runtime_write(reg, val);
 
-    switch (reg) {
-    case 0x20: /* IMAN */
+    if (reg < 0x20) {
+        fprintf(stderr, "%s: reg 0x%x unimplemented\n", __func__, (int)reg);
+        return;
+    }
+
+    switch (reg & 0x1f) {
+    case 0x00: /* IMAN */
         if (val & IMAN_IP) {
-            xhci->iman &= ~IMAN_IP;
+            intr->iman &= ~IMAN_IP;
+        }
+        intr->iman &= ~IMAN_IE;
+        intr->iman |= val & IMAN_IE;
+        if (v == 0) {
+            xhci_intx_update(xhci);
         }
-        xhci->iman &= ~IMAN_IE;
-        xhci->iman |= val & IMAN_IE;
-        xhci_irq_update(xhci);
+        xhci_msix_update(xhci, v);
         break;
-    case 0x24: /* IMOD */
-        xhci->imod = val;
+    case 0x04: /* IMOD */
+        intr->imod = val;
         break;
-    case 0x28: /* ERSTSZ */
-        xhci->erstsz = val & 0xffff;
+    case 0x08: /* ERSTSZ */
+        intr->erstsz = val & 0xffff;
         break;
-    case 0x30: /* ERSTBA low */
+    case 0x10: /* ERSTBA low */
         /* XXX NEC driver bug: it doesn't align this to 64 bytes
-        xhci->erstba_low = val & 0xffffffc0; */
-        xhci->erstba_low = val & 0xfffffff0;
+        intr->erstba_low = val & 0xffffffc0; */
+        intr->erstba_low = val & 0xfffffff0;
         break;
-    case 0x34: /* ERSTBA high */
-        xhci->erstba_high = val;
-        xhci_er_reset(xhci);
+    case 0x14: /* ERSTBA high */
+        intr->erstba_high = val;
+        xhci_er_reset(xhci, v);
         break;
-    case 0x38: /* ERDP low */
+    case 0x18: /* ERDP low */
         if (val & ERDP_EHB) {
-            xhci->erdp_low &= ~ERDP_EHB;
+            intr->erdp_low &= ~ERDP_EHB;
         }
-        xhci->erdp_low = (val & ~ERDP_EHB) | (xhci->erdp_low & ERDP_EHB);
+        intr->erdp_low = (val & ~ERDP_EHB) | (intr->erdp_low & ERDP_EHB);
         break;
-    case 0x3c: /* ERDP high */
-        xhci->erdp_high = val;
-        xhci_events_update(xhci);
+    case 0x1c: /* ERDP high */
+        intr->erdp_high = val;
+        xhci_events_update(xhci, v);
         break;
     default:
-        fprintf(stderr, "xhci_oper_write: reg 0x%x unimplemented\n", reg);
+        fprintf(stderr, "xhci_oper_write: reg 0x%x unimplemented\n",
+                (int)reg);
     }
 }
 
-static uint32_t xhci_doorbell_read(XHCIState *xhci, uint32_t reg)
+static uint64_t xhci_doorbell_read(void *ptr, hwaddr reg,
+                                   unsigned size)
 {
     /* doorbells always read as 0 */
     trace_usb_xhci_doorbell_read(reg, 0);
     return 0;
 }
 
-static void xhci_doorbell_write(XHCIState *xhci, uint32_t reg, uint32_t val)
+static void xhci_doorbell_write(void *ptr, hwaddr reg,
+                                uint64_t val, unsigned size)
 {
+    XHCIState *xhci = ptr;
+
     trace_usb_xhci_doorbell_write(reg, val);
 
     if (!xhci_running(xhci)) {
@@ -2669,69 +2859,57 @@ static void xhci_doorbell_write(XHCIState *xhci, uint32_t reg, uint32_t val)
         if (val == 0) {
             xhci_process_commands(xhci);
         } else {
-            fprintf(stderr, "xhci: bad doorbell 0 write: 0x%x\n", val);
+            fprintf(stderr, "xhci: bad doorbell 0 write: 0x%x\n",
+                    (uint32_t)val);
         }
     } else {
-        if (reg > MAXSLOTS) {
-            fprintf(stderr, "xhci: bad doorbell %d\n", reg);
+        if (reg > xhci->numslots) {
+            fprintf(stderr, "xhci: bad doorbell %d\n", (int)reg);
         } else if (val > 31) {
-            fprintf(stderr, "xhci: bad doorbell %d write: 0x%x\n", reg, val);
+            fprintf(stderr, "xhci: bad doorbell %d write: 0x%x\n",
+                    (int)reg, (uint32_t)val);
         } else {
             xhci_kick_ep(xhci, reg, val);
         }
     }
 }
 
-static uint64_t xhci_mem_read(void *ptr, target_phys_addr_t addr,
-                              unsigned size)
-{
-    XHCIState *xhci = ptr;
-
-    /* Only aligned reads are allowed on xHCI */
-    if (addr & 3) {
-        fprintf(stderr, "xhci_mem_read: Mis-aligned read\n");
-        return 0;
-    }
-
-    if (addr < LEN_CAP) {
-        return xhci_cap_read(xhci, addr);
-    } else if (addr >= OFF_OPER && addr < (OFF_OPER + LEN_OPER)) {
-        return xhci_oper_read(xhci, addr - OFF_OPER);
-    } else if (addr >= OFF_RUNTIME && addr < (OFF_RUNTIME + LEN_RUNTIME)) {
-        return xhci_runtime_read(xhci, addr - OFF_RUNTIME);
-    } else if (addr >= OFF_DOORBELL && addr < (OFF_DOORBELL + LEN_DOORBELL)) {
-        return xhci_doorbell_read(xhci, addr - OFF_DOORBELL);
-    } else {
-        fprintf(stderr, "xhci_mem_read: Bad offset %x\n", (int)addr);
-        return 0;
-    }
-}
+static const MemoryRegionOps xhci_cap_ops = {
+    .read = xhci_cap_read,
+    .valid.min_access_size = 1,
+    .valid.max_access_size = 4,
+    .impl.min_access_size = 4,
+    .impl.max_access_size = 4,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
 
-static void xhci_mem_write(void *ptr, target_phys_addr_t addr,
-                           uint64_t val, unsigned size)
-{
-    XHCIState *xhci = ptr;
+static const MemoryRegionOps xhci_oper_ops = {
+    .read = xhci_oper_read,
+    .write = xhci_oper_write,
+    .valid.min_access_size = 4,
+    .valid.max_access_size = 4,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
 
-    /* Only aligned writes are allowed on xHCI */
-    if (addr & 3) {
-        fprintf(stderr, "xhci_mem_write: Mis-aligned write\n");
-        return;
-    }
+static const MemoryRegionOps xhci_port_ops = {
+    .read = xhci_port_read,
+    .write = xhci_port_write,
+    .valid.min_access_size = 4,
+    .valid.max_access_size = 4,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
 
-    if (addr >= OFF_OPER && addr < (OFF_OPER + LEN_OPER)) {
-        xhci_oper_write(xhci, addr - OFF_OPER, val);
-    } else if (addr >= OFF_RUNTIME && addr < (OFF_RUNTIME + LEN_RUNTIME)) {
-        xhci_runtime_write(xhci, addr - OFF_RUNTIME, val);
-    } else if (addr >= OFF_DOORBELL && addr < (OFF_DOORBELL + LEN_DOORBELL)) {
-        xhci_doorbell_write(xhci, addr - OFF_DOORBELL, val);
-    } else {
-        fprintf(stderr, "xhci_mem_write: Bad offset %x\n", (int)addr);
-    }
-}
+static const MemoryRegionOps xhci_runtime_ops = {
+    .read = xhci_runtime_read,
+    .write = xhci_runtime_write,
+    .valid.min_access_size = 4,
+    .valid.max_access_size = 4,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
 
-static const MemoryRegionOps xhci_mem_ops = {
-    .read = xhci_mem_read,
-    .write = xhci_mem_write,
+static const MemoryRegionOps xhci_doorbell_ops = {
+    .read = xhci_doorbell_read,
+    .write = xhci_doorbell_write,
     .valid.min_access_size = 4,
     .valid.max_access_size = 4,
     .endianness = DEVICE_LITTLE_ENDIAN,
@@ -2740,53 +2918,57 @@ static const MemoryRegionOps xhci_mem_ops = {
 static void xhci_attach(USBPort *usbport)
 {
     XHCIState *xhci = usbport->opaque;
-    XHCIPort *port = &xhci->ports[usbport->index];
+    XHCIPort *port = xhci_lookup_port(xhci, usbport);
 
-    xhci_update_port(xhci, port, 0);
+    xhci_port_update(port, 0);
 }
 
 static void xhci_detach(USBPort *usbport)
 {
     XHCIState *xhci = usbport->opaque;
-    XHCIPort *port = &xhci->ports[usbport->index];
+    XHCIPort *port = xhci_lookup_port(xhci, usbport);
 
-    xhci_update_port(xhci, port, 1);
+    xhci_port_update(port, 1);
 }
 
 static void xhci_wakeup(USBPort *usbport)
 {
     XHCIState *xhci = usbport->opaque;
-    XHCIPort *port = &xhci->ports[usbport->index];
-    int nr = port->port.index + 1;
-    XHCIEvent ev = { ER_PORT_STATUS_CHANGE, CC_SUCCESS, nr << 24};
-    uint32_t pls;
+    XHCIPort *port = xhci_lookup_port(xhci, usbport);
 
-    pls = (port->portsc >> PORTSC_PLS_SHIFT) & PORTSC_PLS_MASK;
-    if (pls != 3) {
+    if (get_field(port->portsc, PORTSC_PLS) != PLS_U3) {
         return;
     }
-    port->portsc |= 0xf << PORTSC_PLS_SHIFT;
-    if (port->portsc & PORTSC_PLC) {
-        return;
-    }
-    port->portsc |= PORTSC_PLC;
-    xhci_event(xhci, &ev);
+    set_field(&port->portsc, PLS_RESUME, PORTSC_PLS);
+    xhci_port_notify(port, PORTSC_PLC);
 }
 
 static void xhci_complete(USBPort *port, USBPacket *packet)
 {
     XHCITransfer *xfer = container_of(packet, XHCITransfer, packet);
 
-    xhci_complete_packet(xfer, packet->result);
+    if (packet->status == USB_RET_REMOVE_FROM_QUEUE) {
+        xhci_ep_nuke_one_xfer(xfer);
+        return;
+    }
+    xhci_complete_packet(xfer);
     xhci_kick_ep(xfer->xhci, xfer->slotid, xfer->epid);
 }
 
-static void xhci_child_detach(USBPort *port, USBDevice *child)
+static void xhci_child_detach(USBPort *uport, USBDevice *child)
 {
-    FIXME();
+    USBBus *bus = usb_bus_from_device(child);
+    XHCIState *xhci = container_of(bus, XHCIState, bus);
+    int i;
+
+    for (i = 0; i < xhci->numslots; i++) {
+        if (xhci->slots[i].uport == uport) {
+            xhci->slots[i].uport = NULL;
+        }
+    }
 }
 
-static USBPortOps xhci_port_ops = {
+static USBPortOps xhci_uport_ops = {
     .attach   = xhci_attach,
     .detach   = xhci_detach,
     .wakeup   = xhci_wakeup,
@@ -2799,7 +2981,7 @@ static int xhci_find_slotid(XHCIState *xhci, USBDevice *dev)
     XHCISlot *slot;
     int slotid;
 
-    for (slotid = 1; slotid <= MAXSLOTS; slotid++) {
+    for (slotid = 1; slotid <= xhci->numslots; slotid++) {
         slot = &xhci->slots[slotid-1];
         if (slot->devaddr == dev->addr) {
             return slotid;
@@ -2840,28 +3022,51 @@ static USBBusOps xhci_bus_ops = {
 
 static void usb_xhci_init(XHCIState *xhci, DeviceState *dev)
 {
-    int i;
+    XHCIPort *port;
+    int i, usbports, speedmask;
 
     xhci->usbsts = USBSTS_HCH;
 
+    if (xhci->numports_2 > MAXPORTS_2) {
+        xhci->numports_2 = MAXPORTS_2;
+    }
+    if (xhci->numports_3 > MAXPORTS_3) {
+        xhci->numports_3 = MAXPORTS_3;
+    }
+    usbports = MAX(xhci->numports_2, xhci->numports_3);
+    xhci->numports = xhci->numports_2 + xhci->numports_3;
+
     usb_bus_new(&xhci->bus, &xhci_bus_ops, &xhci->pci_dev.qdev);
 
-    for (i = 0; i < MAXPORTS; i++) {
-        memset(&xhci->ports[i], 0, sizeof(xhci->ports[i]));
-        usb_register_port(&xhci->bus, &xhci->ports[i].port, xhci, i,
-                          &xhci_port_ops,
-                          USB_SPEED_MASK_LOW  |
-                          USB_SPEED_MASK_FULL |
-                          USB_SPEED_MASK_HIGH);
-    }
-    for (i = 0; i < MAXSLOTS; i++) {
-        xhci->slots[i].enabled = 0;
+    for (i = 0; i < usbports; i++) {
+        speedmask = 0;
+        if (i < xhci->numports_2) {
+            port = &xhci->ports[i];
+            port->portnr = i + 1;
+            port->uport = &xhci->uports[i];
+            port->speedmask =
+                USB_SPEED_MASK_LOW  |
+                USB_SPEED_MASK_FULL |
+                USB_SPEED_MASK_HIGH;
+            snprintf(port->name, sizeof(port->name), "usb2 port #%d", i+1);
+            speedmask |= port->speedmask;
+        }
+        if (i < xhci->numports_3) {
+            port = &xhci->ports[i + xhci->numports_2];
+            port->portnr = i + 1 + xhci->numports_2;
+            port->uport = &xhci->uports[i];
+            port->speedmask = USB_SPEED_MASK_SUPER;
+            snprintf(port->name, sizeof(port->name), "usb3 port #%d", i+1);
+            speedmask |= port->speedmask;
+        }
+        usb_register_port(&xhci->bus, &xhci->uports[i], xhci, i,
+                          &xhci_uport_ops, speedmask);
     }
 }
 
 static int usb_xhci_initfn(struct PCIDevice *dev)
 {
-    int ret;
+    int i, ret;
 
     XHCIState *xhci = DO_UPCAST(XHCIState, pci_dev, dev);
 
@@ -2872,10 +3077,47 @@ static int usb_xhci_initfn(struct PCIDevice *dev)
 
     usb_xhci_init(xhci, &dev->qdev);
 
+    if (xhci->numintrs > MAXINTRS) {
+        xhci->numintrs = MAXINTRS;
+    }
+    if (xhci->numintrs < 1) {
+        xhci->numintrs = 1;
+    }
+    if (xhci->numslots > MAXSLOTS) {
+        xhci->numslots = MAXSLOTS;
+    }
+    if (xhci->numslots < 1) {
+        xhci->numslots = 1;
+    }
+
+    xhci->mfwrap_timer = qemu_new_timer_ns(vm_clock, xhci_mfwrap_timer, xhci);
+
     xhci->irq = xhci->pci_dev.irq[0];
 
-    memory_region_init_io(&xhci->mem, &xhci_mem_ops, xhci,
-                          "xhci", LEN_REGS);
+    memory_region_init(&xhci->mem, "xhci", LEN_REGS);
+    memory_region_init_io(&xhci->mem_cap, &xhci_cap_ops, xhci,
+                          "capabilities", LEN_CAP);
+    memory_region_init_io(&xhci->mem_oper, &xhci_oper_ops, xhci,
+                          "operational", 0x400);
+    memory_region_init_io(&xhci->mem_runtime, &xhci_runtime_ops, xhci,
+                          "runtime", LEN_RUNTIME);
+    memory_region_init_io(&xhci->mem_doorbell, &xhci_doorbell_ops, xhci,
+                          "doorbell", LEN_DOORBELL);
+
+    memory_region_add_subregion(&xhci->mem, 0,            &xhci->mem_cap);
+    memory_region_add_subregion(&xhci->mem, OFF_OPER,     &xhci->mem_oper);
+    memory_region_add_subregion(&xhci->mem, OFF_RUNTIME,  &xhci->mem_runtime);
+    memory_region_add_subregion(&xhci->mem, OFF_DOORBELL, &xhci->mem_doorbell);
+
+    for (i = 0; i < xhci->numports; i++) {
+        XHCIPort *port = &xhci->ports[i];
+        uint32_t offset = OFF_OPER + 0x400 + 0x10 * i;
+        port->xhci = xhci;
+        memory_region_init_io(&port->mem, &xhci_port_ops, port,
+                              port->name, 0x10);
+        memory_region_add_subregion(&xhci->mem, offset, &port->mem);
+    }
+
     pci_register_bar(&xhci->pci_dev, 0,
                      PCI_BASE_ADDRESS_SPACE_MEMORY|PCI_BASE_ADDRESS_MEM_TYPE_64,
                      &xhci->mem);
@@ -2883,32 +3125,31 @@ static int usb_xhci_initfn(struct PCIDevice *dev)
     ret = pcie_cap_init(&xhci->pci_dev, 0xa0, PCI_EXP_TYPE_ENDPOINT, 0);
     assert(ret >= 0);
 
-    if (xhci->msi) {
-        ret = msi_init(&xhci->pci_dev, 0x70, 1, true, false);
-        assert(ret >= 0);
+    if (xhci->flags & (1 << XHCI_FLAG_USE_MSI)) {
+        msi_init(&xhci->pci_dev, 0x70, xhci->numintrs, true, false);
+    }
+    if (xhci->flags & (1 << XHCI_FLAG_USE_MSI_X)) {
+        msix_init(&xhci->pci_dev, xhci->numintrs,
+                  &xhci->mem, 0, OFF_MSIX_TABLE,
+                  &xhci->mem, 0, OFF_MSIX_PBA,
+                  0x90);
     }
 
     return 0;
 }
 
-static void xhci_write_config(PCIDevice *dev, uint32_t addr, uint32_t val,
-                              int len)
-{
-    XHCIState *xhci = DO_UPCAST(XHCIState, pci_dev, dev);
-
-    pci_default_write_config(dev, addr, val, len);
-    if (xhci->msi) {
-        msi_write_config(dev, addr, val, len);
-    }
-}
-
 static const VMStateDescription vmstate_xhci = {
     .name = "xhci",
     .unmigratable = 1,
 };
 
 static Property xhci_properties[] = {
-    DEFINE_PROP_UINT32("msi", XHCIState, msi, 0),
+    DEFINE_PROP_BIT("msi",      XHCIState, flags, XHCI_FLAG_USE_MSI, true),
+    DEFINE_PROP_BIT("msix",     XHCIState, flags, XHCI_FLAG_USE_MSI_X, true),
+    DEFINE_PROP_UINT32("intrs", XHCIState, numintrs, MAXINTRS),
+    DEFINE_PROP_UINT32("slots", XHCIState, numslots, MAXSLOTS),
+    DEFINE_PROP_UINT32("p2",    XHCIState, numports_2, 4),
+    DEFINE_PROP_UINT32("p3",    XHCIState, numports_3, 4),
     DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -2926,7 +3167,7 @@ static void xhci_class_init(ObjectClass *klass, void *data)
     k->class_id     = PCI_CLASS_SERIAL_USB;
     k->revision     = 0x03;
     k->is_express   = 1;
-    k->config_write = xhci_write_config;
+    k->no_hotplug   = 1;
 }
 
 static TypeInfo xhci_info = {
diff --git a/hw/usb/host-bsd.c b/hw/usb/host-bsd.c
index ec2626662..6473e8b74 100644
--- a/hw/usb/host-bsd.c
+++ b/hw/usb/host-bsd.c
@@ -121,7 +121,7 @@ static void usb_host_handle_reset(USBDevice *dev)
  * -check device states against transfer requests
  *  and return appropriate response
  */
-static int usb_host_handle_control(USBDevice *dev,
+static void usb_host_handle_control(USBDevice *dev,
                                    USBPacket *p,
                                    int request,
                                    int value,
@@ -139,7 +139,6 @@ static int usb_host_handle_control(USBDevice *dev,
 
         /* specific SET_ADDRESS support */
         dev->addr = value;
-        return 0;
     } else if ((request >> 8) == UT_WRITE_DEVICE &&
                (request & 0xff) == UR_SET_CONFIG) {
 
@@ -151,10 +150,8 @@ static int usb_host_handle_control(USBDevice *dev,
             printf("handle_control: failed to set configuration - %s\n",
                    strerror(errno));
 #endif
-            return USB_RET_STALL;
+            p->status = USB_RET_STALL;
         }
-
-        return 0;
     } else if ((request >> 8) == UT_WRITE_INTERFACE &&
                (request & 0xff) == UR_SET_INTERFACE) {
 
@@ -168,10 +165,8 @@ static int usb_host_handle_control(USBDevice *dev,
             printf("handle_control: failed to set alternate interface - %s\n",
                    strerror(errno));
 #endif
-            return USB_RET_STALL;
+            p->status = USB_RET_STALL;
         }
-
-        return 0;
     } else {
         req.ucr_request.bmRequestType = request >> 8;
         req.ucr_request.bRequest = request & 0xff;
@@ -201,14 +196,14 @@ static int usb_host_handle_control(USBDevice *dev,
             printf("handle_control: error after request - %s\n",
                    strerror(errno));
 #endif
-            return USB_RET_NAK; // STALL
+            p->status = USB_RET_NAK; /* STALL */
         } else {
-            return req.ucr_actlen;
+            p->actual_length = req.ucr_actlen;
         }
     }
 }
 
-static int usb_host_handle_data(USBDevice *dev, USBPacket *p)
+static void usb_host_handle_data(USBDevice *dev, USBPacket *p)
 {
     USBHostDevice *s = (USBHostDevice *)dev;
     int ret, fd, mode;
@@ -232,7 +227,8 @@ static int usb_host_handle_data(USBDevice *dev, USBPacket *p)
     fd = ensure_ep_open(s, devep, mode);
     if (fd < 0) {
         sigprocmask(SIG_SETMASK, &old_mask, NULL);
-        return USB_RET_NODEV;
+        p->status = USB_RET_NODEV;
+        return;
     }
 
     if (ioctl(fd, USB_SET_TIMEOUT, &timeout) < 0) {
@@ -267,12 +263,13 @@ static int usb_host_handle_data(USBDevice *dev, USBPacket *p)
         switch(errno) {
         case ETIMEDOUT:
         case EINTR:
-            return USB_RET_NAK;
+            p->status = USB_RET_NAK;
+            break;
         default:
-            return USB_RET_STALL;
+            p->status = USB_RET_STALL;
         }
     } else {
-        return ret;
+        p->actual_length = ret;
     }
 }
 
diff --git a/hw/usb/host-linux.c b/hw/usb/host-linux.c
index 8df92074d..aa77b7704 100644
--- a/hw/usb/host-linux.c
+++ b/hw/usb/host-linux.c
@@ -135,7 +135,7 @@ static int parse_filter(const char *spec, struct USBAutoFilter *f);
 static void usb_host_auto_check(void *unused);
 static int usb_host_read_file(char *line, size_t line_size,
                             const char *device_file, const char *device_name);
-static int usb_linux_update_endp_table(USBHostDevice *s);
+static void usb_linux_update_endp_table(USBHostDevice *s);
 
 static int usb_host_usbfs_type(USBHostDevice *s, USBPacket *p)
 {
@@ -366,28 +366,34 @@ static void async_complete(void *opaque)
         if (p) {
             switch (aurb->urb.status) {
             case 0:
-                p->result += aurb->urb.actual_length;
+                p->actual_length += aurb->urb.actual_length;
+                if (!aurb->more) {
+                    /* Clear previous ASYNC status */
+                    p->status = USB_RET_SUCCESS;
+                }
                 break;
 
             case -EPIPE:
                 set_halt(s, p->pid, p->ep->nr);
-                p->result = USB_RET_STALL;
+                p->status = USB_RET_STALL;
                 break;
 
             case -EOVERFLOW:
-                p->result = USB_RET_BABBLE;
+                p->status = USB_RET_BABBLE;
                 break;
 
             default:
-                p->result = USB_RET_IOERROR;
+                p->status = USB_RET_IOERROR;
                 break;
             }
 
             if (aurb->urb.type == USBDEVFS_URB_TYPE_CONTROL) {
-                trace_usb_host_req_complete(s->bus_num, s->addr, p, p->result);
+                trace_usb_host_req_complete(s->bus_num, s->addr, p,
+                                            p->status, aurb->urb.actual_length);
                 usb_generic_async_ctrl_complete(&s->dev, p);
             } else if (!aurb->more) {
-                trace_usb_host_req_complete(s->bus_num, s->addr, p, p->result);
+                trace_usb_host_req_complete(s->bus_num, s->addr, p,
+                                            p->status, aurb->urb.actual_length);
                 usb_packet_complete(&s->dev, p);
             }
         }
@@ -733,27 +739,31 @@ static void usb_host_stop_n_free_iso(USBHostDevice *s, int pid, uint8_t ep)
     clear_iso_started(s, pid, ep);
 }
 
-static int urb_status_to_usb_ret(int status)
+static void urb_status_to_usb_ret(int status, USBPacket *p)
 {
     switch (status) {
     case -EPIPE:
-        return USB_RET_STALL;
+        p->status = USB_RET_STALL;
+        break;
     case -EOVERFLOW:
-        return USB_RET_BABBLE;
+        p->status = USB_RET_BABBLE;
+        break;
     default:
-        return USB_RET_IOERROR;
+        p->status = USB_RET_IOERROR;
     }
 }
 
-static int usb_host_handle_iso_data(USBHostDevice *s, USBPacket *p, int in)
+static void usb_host_handle_iso_data(USBHostDevice *s, USBPacket *p, int in)
 {
     AsyncURB *aurb;
-    int i, j, ret, max_packet_size, offset, len = 0;
+    int i, j, max_packet_size, offset, len;
     uint8_t *buf;
 
     max_packet_size = p->ep->max_packet_size;
-    if (max_packet_size == 0)
-        return USB_RET_NAK;
+    if (max_packet_size == 0) {
+        p->status = USB_RET_NAK;
+        return;
+    }
 
     aurb = get_iso_urb(s, p->pid, p->ep->nr);
     if (!aurb) {
@@ -766,18 +776,17 @@ static int usb_host_handle_iso_data(USBHostDevice *s, USBPacket *p, int in)
         if (in) {
             /* Check urb status  */
             if (aurb[i].urb.status) {
-                len = urb_status_to_usb_ret(aurb[i].urb.status);
+                urb_status_to_usb_ret(aurb[i].urb.status, p);
                 /* Move to the next urb */
                 aurb[i].iso_frame_idx = ISO_FRAME_DESC_PER_URB - 1;
             /* Check frame status */
             } else if (aurb[i].urb.iso_frame_desc[j].status) {
-                len = urb_status_to_usb_ret(
-                                        aurb[i].urb.iso_frame_desc[j].status);
+                urb_status_to_usb_ret(aurb[i].urb.iso_frame_desc[j].status, p);
             /* Check the frame fits */
             } else if (aurb[i].urb.iso_frame_desc[j].actual_length
                        > p->iov.size) {
                 printf("husb: received iso data is larger then packet\n");
-                len = USB_RET_BABBLE;
+                p->status = USB_RET_BABBLE;
             /* All good copy data over */
             } else {
                 len = aurb[i].urb.iso_frame_desc[j].actual_length;
@@ -792,7 +801,8 @@ static int usb_host_handle_iso_data(USBHostDevice *s, USBPacket *p, int in)
             /* Check the frame fits */
             if (len > max_packet_size) {
                 printf("husb: send iso data is larger then max packet size\n");
-                return USB_RET_NAK;
+                p->status = USB_RET_NAK;
+                return;
             }
 
             /* All good copy data over */
@@ -823,17 +833,16 @@ static int usb_host_handle_iso_data(USBHostDevice *s, USBPacket *p, int in)
         /* (Re)-submit all fully consumed / filled urbs */
         for (i = 0; i < s->iso_urb_count; i++) {
             if (aurb[i].iso_frame_idx == ISO_FRAME_DESC_PER_URB) {
-                ret = ioctl(s->fd, USBDEVFS_SUBMITURB, &aurb[i]);
-                if (ret < 0) {
+                if (ioctl(s->fd, USBDEVFS_SUBMITURB, &aurb[i]) < 0) {
                     perror("USBDEVFS_SUBMITURB");
-                    if (!in || len == 0) {
+                    if (!in || p->status == USB_RET_SUCCESS) {
                         switch(errno) {
                         case ETIMEDOUT:
-                            len = USB_RET_NAK;
+                            p->status = USB_RET_NAK;
                             break;
                         case EPIPE:
                         default:
-                            len = USB_RET_STALL;
+                            p->status = USB_RET_STALL;
                         }
                     }
                     break;
@@ -843,11 +852,9 @@ static int usb_host_handle_iso_data(USBHostDevice *s, USBPacket *p, int in)
             }
         }
     }
-
-    return len;
 }
 
-static int usb_host_handle_data(USBDevice *dev, USBPacket *p)
+static void usb_host_handle_data(USBDevice *dev, USBPacket *p)
 {
     USBHostDevice *s = DO_UPCAST(USBHostDevice, dev, dev);
     struct usbdevfs_urb *urb;
@@ -861,8 +868,10 @@ static int usb_host_handle_data(USBDevice *dev, USBPacket *p)
                             p->ep->nr, p->iov.size);
 
     if (!is_valid(s, p->pid, p->ep->nr)) {
-        trace_usb_host_req_complete(s->bus_num, s->addr, p, USB_RET_NAK);
-        return USB_RET_NAK;
+        p->status = USB_RET_NAK;
+        trace_usb_host_req_complete(s->bus_num, s->addr, p,
+                                    p->status, p->actual_length);
+        return;
     }
 
     if (p->pid == USB_TOKEN_IN) {
@@ -876,14 +885,17 @@ static int usb_host_handle_data(USBDevice *dev, USBPacket *p)
         ret = ioctl(s->fd, USBDEVFS_CLEAR_HALT, &arg);
         if (ret < 0) {
             perror("USBDEVFS_CLEAR_HALT");
-            trace_usb_host_req_complete(s->bus_num, s->addr, p, USB_RET_NAK);
-            return USB_RET_NAK;
+            p->status = USB_RET_NAK;
+            trace_usb_host_req_complete(s->bus_num, s->addr, p,
+                                        p->status, p->actual_length);
+            return;
         }
         clear_halt(s, p->pid, p->ep->nr);
     }
 
     if (is_isoc(s, p->pid, p->ep->nr)) {
-        return usb_host_handle_iso_data(s, p, p->pid == USB_TOKEN_IN);
+        usb_host_handle_iso_data(s, p, p->pid == USB_TOKEN_IN);
+        return;
     }
 
     v = 0;
@@ -931,19 +943,21 @@ static int usb_host_handle_data(USBDevice *dev, USBPacket *p)
 
             switch(errno) {
             case ETIMEDOUT:
+                p->status = USB_RET_NAK;
                 trace_usb_host_req_complete(s->bus_num, s->addr, p,
-                                            USB_RET_NAK);
-                return USB_RET_NAK;
+                                            p->status, p->actual_length);
+                break;
             case EPIPE:
             default:
+                p->status = USB_RET_STALL;
                 trace_usb_host_req_complete(s->bus_num, s->addr, p,
-                                            USB_RET_STALL);
-                return USB_RET_STALL;
+                                            p->status, p->actual_length);
             }
+            return;
         }
     } while (rem > 0);
 
-    return USB_RET_ASYNC;
+    p->status = USB_RET_ASYNC;
 }
 
 static int ctrl_error(void)
@@ -955,14 +969,13 @@ static int ctrl_error(void)
     }
 }
 
-static int usb_host_set_address(USBHostDevice *s, int addr)
+static void usb_host_set_address(USBHostDevice *s, int addr)
 {
     trace_usb_host_set_address(s->bus_num, s->addr, addr);
     s->dev.addr = addr;
-    return 0;
 }
 
-static int usb_host_set_config(USBHostDevice *s, int config)
+static void usb_host_set_config(USBHostDevice *s, int config, USBPacket *p)
 {
     int ret, first = 1;
 
@@ -987,14 +1000,15 @@ again:
     }
 
     if (ret < 0) {
-        return ctrl_error();
+        p->status = ctrl_error();
+        return;
     }
     usb_host_claim_interfaces(s, config);
     usb_linux_update_endp_table(s);
-    return 0;
 }
 
-static int usb_host_set_interface(USBHostDevice *s, int iface, int alt)
+static void usb_host_set_interface(USBHostDevice *s, int iface, int alt,
+                                   USBPacket *p)
 {
     struct usbdevfs_setinterface si;
     int i, ret;
@@ -1011,7 +1025,8 @@ static int usb_host_set_interface(USBHostDevice *s, int iface, int alt)
     }
 
     if (iface >= USB_MAX_INTERFACES) {
-        return USB_RET_STALL;
+        p->status = USB_RET_STALL;
+        return;
     }
 
     si.interface  = iface;
@@ -1022,15 +1037,15 @@ static int usb_host_set_interface(USBHostDevice *s, int iface, int alt)
             iface, alt, ret, errno);
 
     if (ret < 0) {
-        return ctrl_error();
+        p->status = ctrl_error();
+        return;
     }
 
     s->dev.altsetting[iface] = alt;
     usb_linux_update_endp_table(s);
-    return 0;
 }
 
-static int usb_host_handle_control(USBDevice *dev, USBPacket *p,
+static void usb_host_handle_control(USBDevice *dev, USBPacket *p,
                int request, int value, int index, int length, uint8_t *data)
 {
     USBHostDevice *s = DO_UPCAST(USBHostDevice, dev, dev);
@@ -1045,23 +1060,22 @@ static int usb_host_handle_control(USBDevice *dev, USBPacket *p,
 
     /* Note request is (bRequestType << 8) | bRequest */
     trace_usb_host_req_control(s->bus_num, s->addr, p, request, value, index);
-    assert(p->result == 0);
 
     switch (request) {
     case DeviceOutRequest | USB_REQ_SET_ADDRESS:
-        ret = usb_host_set_address(s, value);
-        trace_usb_host_req_emulated(s->bus_num, s->addr, p, ret);
-        return ret;
+        usb_host_set_address(s, value);
+        trace_usb_host_req_emulated(s->bus_num, s->addr, p, p->status);
+        return;
 
     case DeviceOutRequest | USB_REQ_SET_CONFIGURATION:
-        ret = usb_host_set_config(s, value & 0xff);
-        trace_usb_host_req_emulated(s->bus_num, s->addr, p, ret);
-        return ret;
+        usb_host_set_config(s, value & 0xff, p);
+        trace_usb_host_req_emulated(s->bus_num, s->addr, p, p->status);
+        return;
 
     case InterfaceOutRequest | USB_REQ_SET_INTERFACE:
-        ret = usb_host_set_interface(s, index, value);
-        trace_usb_host_req_emulated(s->bus_num, s->addr, p, ret);
-        return ret;
+        usb_host_set_interface(s, index, value, p);
+        trace_usb_host_req_emulated(s->bus_num, s->addr, p, p->status);
+        return;
 
     case EndpointOutRequest | USB_REQ_CLEAR_FEATURE:
         if (value == 0) { /* clear halt */
@@ -1069,16 +1083,16 @@ static int usb_host_handle_control(USBDevice *dev, USBPacket *p,
             ioctl(s->fd, USBDEVFS_CLEAR_HALT, &index);
             clear_halt(s, pid, index & 0x0f);
             trace_usb_host_req_emulated(s->bus_num, s->addr, p, 0);
-            return 0;
+            return;
         }
     }
 
     /* The rest are asynchronous */
-
     if (length > sizeof(dev->data_buf)) {
         fprintf(stderr, "husb: ctrl buffer too small (%d > %zu)\n",
                 length, sizeof(dev->data_buf));
-        return USB_RET_STALL;
+        p->status = USB_RET_STALL;
+        return;
     }
 
     aurb = async_alloc(s);
@@ -1112,18 +1126,20 @@ static int usb_host_handle_control(USBDevice *dev, USBPacket *p,
 
         switch(errno) {
         case ETIMEDOUT:
-            return USB_RET_NAK;
+            p->status = USB_RET_NAK;
+            break;
         case EPIPE:
         default:
-            return USB_RET_STALL;
+            p->status = USB_RET_STALL;
+            break;
         }
+        return;
     }
 
-    return USB_RET_ASYNC;
+    p->status = USB_RET_ASYNC;
 }
 
-/* returns 1 on problem encountered or 0 for success */
-static int usb_linux_update_endp_table(USBHostDevice *s)
+static void usb_linux_update_endp_table(USBHostDevice *s)
 {
     static const char *tname[] = {
         [USB_ENDPOINT_XFER_CONTROL] = "control",
@@ -1149,23 +1165,23 @@ static int usb_linux_update_endp_table(USBHostDevice *s)
         if (d->bLength < 2) {
             trace_usb_host_parse_error(s->bus_num, s->addr,
                                        "descriptor too short");
-            goto error;
+            return;
         }
         if (i + d->bLength > s->descr_len) {
             trace_usb_host_parse_error(s->bus_num, s->addr,
                                        "descriptor too long");
-            goto error;
+            return;
         }
         switch (d->bDescriptorType) {
         case 0:
             trace_usb_host_parse_error(s->bus_num, s->addr,
                                        "invalid descriptor type");
-            goto error;
+            return;
         case USB_DT_DEVICE:
             if (d->bLength < 0x12) {
                 trace_usb_host_parse_error(s->bus_num, s->addr,
                                            "device descriptor too short");
-                goto error;
+                return;
             }
             v = (d->u.device.idVendor_hi << 8) | d->u.device.idVendor_lo;
             p = (d->u.device.idProduct_hi << 8) | d->u.device.idProduct_lo;
@@ -1175,7 +1191,7 @@ static int usb_linux_update_endp_table(USBHostDevice *s)
             if (d->bLength < 0x09) {
                 trace_usb_host_parse_error(s->bus_num, s->addr,
                                            "config descriptor too short");
-                goto error;
+                return;
             }
             configuration = d->u.config.bConfigurationValue;
             active = (configuration == s->dev.configuration);
@@ -1186,7 +1202,7 @@ static int usb_linux_update_endp_table(USBHostDevice *s)
             if (d->bLength < 0x09) {
                 trace_usb_host_parse_error(s->bus_num, s->addr,
                                            "interface descriptor too short");
-                goto error;
+                return;
             }
             interface = d->u.interface.bInterfaceNumber;
             altsetting = d->u.interface.bAlternateSetting;
@@ -1199,7 +1215,7 @@ static int usb_linux_update_endp_table(USBHostDevice *s)
             if (d->bLength < 0x07) {
                 trace_usb_host_parse_error(s->bus_num, s->addr,
                                            "endpoint descriptor too short");
-                goto error;
+                return;
             }
             devep = d->u.endpoint.bEndpointAddress;
             pid = (devep & USB_DIR_IN) ? USB_TOKEN_IN : USB_TOKEN_OUT;
@@ -1207,7 +1223,7 @@ static int usb_linux_update_endp_table(USBHostDevice *s)
             if (ep == 0) {
                 trace_usb_host_parse_error(s->bus_num, s->addr,
                                            "invalid endpoint address");
-                goto error;
+                return;
             }
 
             type = d->u.endpoint.bmAttributes & 0x3;
@@ -1224,7 +1240,8 @@ static int usb_linux_update_endp_table(USBHostDevice *s)
                 usb_ep_set_type(&s->dev, pid, ep, type);
                 usb_ep_set_ifnum(&s->dev, pid, ep, interface);
                 if ((s->options & (1 << USB_HOST_OPT_PIPELINE)) &&
-                    (type == USB_ENDPOINT_XFER_BULK)) {
+                    (type == USB_ENDPOINT_XFER_BULK) &&
+                    (pid == USB_TOKEN_OUT)) {
                     usb_ep_set_pipeline(&s->dev, pid, ep, true);
                 }
 
@@ -1239,11 +1256,6 @@ static int usb_linux_update_endp_table(USBHostDevice *s)
             break;
         }
     }
-    return 0;
-
-error:
-    usb_ep_reset(&s->dev);
-    return 1;
 }
 
 /*
@@ -1330,10 +1342,7 @@ static int usb_host_open(USBHostDevice *dev, int bus_num,
     }
 
     usb_ep_init(&dev->dev);
-    ret = usb_linux_update_endp_table(dev);
-    if (ret) {
-        goto fail;
-    }
+    usb_linux_update_endp_table(dev);
 
     if (speed == -1) {
         struct usbdevfs_connectinfo ci;
@@ -1727,6 +1736,7 @@ static int usb_host_scan(void *opaque, USBScanFunc *func)
 }
 
 static QEMUTimer *usb_auto_timer;
+static VMChangeStateEntry *usb_vmstate;
 
 static int usb_host_auto_scan(void *opaque, int bus_num,
                               int addr, const char *port,
@@ -1781,6 +1791,13 @@ static int usb_host_auto_scan(void *opaque, int bus_num,
     return 0;
 }
 
+static void usb_host_vm_state(void *unused, int running, RunState state)
+{
+    if (running) {
+        usb_host_auto_check(unused);
+    }
+}
+
 static void usb_host_auto_check(void *unused)
 {
     struct USBHostDevice *s;
@@ -1809,6 +1826,9 @@ static void usb_host_auto_check(void *unused)
         }
     }
 
+    if (!usb_vmstate) {
+        usb_vmstate = qemu_add_vm_change_state_handler(usb_host_vm_state, NULL);
+    }
     if (!usb_auto_timer) {
         usb_auto_timer = qemu_new_timer_ms(rt_clock, usb_host_auto_check, NULL);
         if (!usb_auto_timer) {
diff --git a/hw/usb/libhw.c b/hw/usb/libhw.c
index c0de30ea8..24d3cad3a 100644
--- a/hw/usb/libhw.c
+++ b/hw/usb/libhw.c
@@ -28,19 +28,25 @@ int usb_packet_map(USBPacket *p, QEMUSGList *sgl)
 {
     DMADirection dir = (p->pid == USB_TOKEN_IN) ?
         DMA_DIRECTION_FROM_DEVICE : DMA_DIRECTION_TO_DEVICE;
-    dma_addr_t len;
     void *mem;
     int i;
 
     for (i = 0; i < sgl->nsg; i++) {
-        len = sgl->sg[i].len;
-        mem = dma_memory_map(sgl->dma, sgl->sg[i].base, &len, dir);
-        if (!mem) {
-            goto err;
-        }
-        qemu_iovec_add(&p->iov, mem, len);
-        if (len != sgl->sg[i].len) {
-            goto err;
+        dma_addr_t base = sgl->sg[i].base;
+        dma_addr_t len = sgl->sg[i].len;
+
+        while (len) {
+            dma_addr_t xlen = len;
+            mem = dma_memory_map(sgl->dma, base, &xlen, dir);
+            if (!mem) {
+                goto err;
+            }
+            if (xlen > len) {
+                xlen = len;
+            }
+            qemu_iovec_add(&p->iov, mem, xlen);
+            len -= xlen;
+            base += xlen;
         }
     }
     return 0;
diff --git a/hw/usb/redirect.c b/hw/usb/redirect.c
index 10b4fbb3a..490c90fae 100644
--- a/hw/usb/redirect.c
+++ b/hw/usb/redirect.c
@@ -1,7 +1,7 @@
 /*
  * USB redirector usb-guest
  *
- * Copyright (c) 2011 Red Hat, Inc.
+ * Copyright (c) 2011-2012 Red Hat, Inc.
  *
  * Red Hat Authors:
  * Hans de Goede <hdegoede@redhat.com>
@@ -29,6 +29,7 @@
 #include "qemu-timer.h"
 #include "monitor.h"
 #include "sysemu.h"
+#include "iov.h"
 
 #include <dirent.h>
 #include <sys/ioctl.h>
@@ -43,7 +44,6 @@
 #define EP2I(ep_address) (((ep_address & 0x80) >> 3) | (ep_address & 0x0f))
 #define I2EP(i) (((i & 0x10) << 3) | (i & 0x0f))
 
-typedef struct AsyncURB AsyncURB;
 typedef struct USBRedirDevice USBRedirDevice;
 
 /* Struct to hold buffered packets (iso or int input packets) */
@@ -58,6 +58,7 @@ struct endp_data {
     uint8_t type;
     uint8_t interval;
     uint8_t interface; /* bInterfaceNumber this ep belongs to */
+    uint16_t max_packet_size; /* In bytes, not wMaxPacketSize format !! */
     uint8_t iso_started;
     uint8_t iso_error; /* For reporting iso errors to the HC */
     uint8_t interrupt_started;
@@ -65,8 +66,20 @@ struct endp_data {
     uint8_t bufpq_prefilled;
     uint8_t bufpq_dropping_packets;
     QTAILQ_HEAD(, buf_packet) bufpq;
-    int bufpq_size;
-    int bufpq_target_size;
+    int32_t bufpq_size;
+    int32_t bufpq_target_size;
+};
+
+struct PacketIdQueueEntry {
+    uint64_t id;
+    QTAILQ_ENTRY(PacketIdQueueEntry)next;
+};
+
+struct PacketIdQueue {
+    USBRedirDevice *dev;
+    const char *name;
+    QTAILQ_HEAD(, PacketIdQueueEntry) head;
+    int size;
 };
 
 struct USBRedirDevice {
@@ -79,33 +92,21 @@ struct USBRedirDevice {
     /* Data passed from chardev the fd_read cb to the usbredirparser read cb */
     const uint8_t *read_buf;
     int read_buf_size;
-    /* For async handling of open/close */
-    QEMUBH *open_close_bh;
+    /* For async handling of close */
+    QEMUBH *chardev_close_bh;
     /* To delay the usb attach in case of quick chardev close + open */
     QEMUTimer *attach_timer;
     int64_t next_attach_time;
     struct usbredirparser *parser;
     struct endp_data endpoint[MAX_ENDPOINTS];
-    uint32_t packet_id;
-    QTAILQ_HEAD(, AsyncURB) asyncq;
+    struct PacketIdQueue cancelled;
+    struct PacketIdQueue already_in_flight;
     /* Data for device filtering */
     struct usb_redir_device_connect_header device_info;
     struct usb_redir_interface_info_header interface_info;
     struct usbredirfilter_rule *filter_rules;
     int filter_rules_count;
-};
-
-struct AsyncURB {
-    USBRedirDevice *dev;
-    USBPacket *packet;
-    uint32_t packet_id;
-    int get;
-    union {
-        struct usb_redir_control_packet_header control_packet;
-        struct usb_redir_bulk_packet_header bulk_packet;
-        struct usb_redir_interrupt_packet_header interrupt_packet;
-    };
-    QTAILQ_ENTRY(AsyncURB)next;
+    int compatible_speedmask;
 };
 
 static void usbredir_hello(void *priv, struct usb_redir_hello_header *h);
@@ -116,32 +117,34 @@ static void usbredir_interface_info(void *priv,
     struct usb_redir_interface_info_header *interface_info);
 static void usbredir_ep_info(void *priv,
     struct usb_redir_ep_info_header *ep_info);
-static void usbredir_configuration_status(void *priv, uint32_t id,
+static void usbredir_configuration_status(void *priv, uint64_t id,
     struct usb_redir_configuration_status_header *configuration_status);
-static void usbredir_alt_setting_status(void *priv, uint32_t id,
+static void usbredir_alt_setting_status(void *priv, uint64_t id,
     struct usb_redir_alt_setting_status_header *alt_setting_status);
-static void usbredir_iso_stream_status(void *priv, uint32_t id,
+static void usbredir_iso_stream_status(void *priv, uint64_t id,
     struct usb_redir_iso_stream_status_header *iso_stream_status);
-static void usbredir_interrupt_receiving_status(void *priv, uint32_t id,
+static void usbredir_interrupt_receiving_status(void *priv, uint64_t id,
     struct usb_redir_interrupt_receiving_status_header
     *interrupt_receiving_status);
-static void usbredir_bulk_streams_status(void *priv, uint32_t id,
+static void usbredir_bulk_streams_status(void *priv, uint64_t id,
     struct usb_redir_bulk_streams_status_header *bulk_streams_status);
-static void usbredir_control_packet(void *priv, uint32_t id,
+static void usbredir_control_packet(void *priv, uint64_t id,
     struct usb_redir_control_packet_header *control_packet,
     uint8_t *data, int data_len);
-static void usbredir_bulk_packet(void *priv, uint32_t id,
+static void usbredir_bulk_packet(void *priv, uint64_t id,
     struct usb_redir_bulk_packet_header *bulk_packet,
     uint8_t *data, int data_len);
-static void usbredir_iso_packet(void *priv, uint32_t id,
+static void usbredir_iso_packet(void *priv, uint64_t id,
     struct usb_redir_iso_packet_header *iso_packet,
     uint8_t *data, int data_len);
-static void usbredir_interrupt_packet(void *priv, uint32_t id,
+static void usbredir_interrupt_packet(void *priv, uint64_t id,
     struct usb_redir_interrupt_packet_header *interrupt_header,
     uint8_t *data, int data_len);
 
-static int usbredir_handle_status(USBRedirDevice *dev,
-                                       int status, int actual_len);
+static void usbredir_handle_status(USBRedirDevice *dev, USBPacket *p,
+    int status);
+
+#define VERSION "qemu usb-redir guest " QEMU_VERSION
 
 /*
  * Logging stuff
@@ -241,62 +244,144 @@ static int usbredir_write(void *priv, uint8_t *data, int count)
         return 0;
     }
 
+    /* Don't send new data to the chardev until our state is fully synced */
+    if (!runstate_check(RUN_STATE_RUNNING)) {
+        return 0;
+    }
+
     return qemu_chr_fe_write(dev->cs, data, count);
 }
 
 /*
- * Async and buffered packets helpers
+ * Cancelled and buffered packets helpers
  */
 
-static AsyncURB *async_alloc(USBRedirDevice *dev, USBPacket *p)
+static void packet_id_queue_init(struct PacketIdQueue *q,
+    USBRedirDevice *dev, const char *name)
+{
+    q->dev = dev;
+    q->name = name;
+    QTAILQ_INIT(&q->head);
+    q->size = 0;
+}
+
+static void packet_id_queue_add(struct PacketIdQueue *q, uint64_t id)
 {
-    AsyncURB *aurb = (AsyncURB *) g_malloc0(sizeof(AsyncURB));
-    aurb->dev = dev;
-    aurb->packet = p;
-    aurb->packet_id = dev->packet_id;
-    QTAILQ_INSERT_TAIL(&dev->asyncq, aurb, next);
-    dev->packet_id++;
+    USBRedirDevice *dev = q->dev;
+    struct PacketIdQueueEntry *e;
+
+    DPRINTF("adding packet id %"PRIu64" to %s queue\n", id, q->name);
 
-    return aurb;
+    e = g_malloc0(sizeof(struct PacketIdQueueEntry));
+    e->id = id;
+    QTAILQ_INSERT_TAIL(&q->head, e, next);
+    q->size++;
 }
 
-static void async_free(USBRedirDevice *dev, AsyncURB *aurb)
+static int packet_id_queue_remove(struct PacketIdQueue *q, uint64_t id)
 {
-    QTAILQ_REMOVE(&dev->asyncq, aurb, next);
-    g_free(aurb);
+    USBRedirDevice *dev = q->dev;
+    struct PacketIdQueueEntry *e;
+
+    QTAILQ_FOREACH(e, &q->head, next) {
+        if (e->id == id) {
+            DPRINTF("removing packet id %"PRIu64" from %s queue\n",
+                    id, q->name);
+            QTAILQ_REMOVE(&q->head, e, next);
+            q->size--;
+            g_free(e);
+            return 1;
+        }
+    }
+    return 0;
 }
 
-static AsyncURB *async_find(USBRedirDevice *dev, uint32_t packet_id)
+static void packet_id_queue_empty(struct PacketIdQueue *q)
 {
-    AsyncURB *aurb;
+    USBRedirDevice *dev = q->dev;
+    struct PacketIdQueueEntry *e, *next_e;
 
-    QTAILQ_FOREACH(aurb, &dev->asyncq, next) {
-        if (aurb->packet_id == packet_id) {
-            return aurb;
-        }
+    DPRINTF("removing %d packet-ids from %s queue\n", q->size, q->name);
+
+    QTAILQ_FOREACH_SAFE(e, &q->head, next, next_e) {
+        QTAILQ_REMOVE(&q->head, e, next);
+        g_free(e);
     }
-    DPRINTF("could not find async urb for packet_id %u\n", packet_id);
-    return NULL;
+    q->size = 0;
 }
 
 static void usbredir_cancel_packet(USBDevice *udev, USBPacket *p)
 {
     USBRedirDevice *dev = DO_UPCAST(USBRedirDevice, dev, udev);
-    AsyncURB *aurb;
 
-    QTAILQ_FOREACH(aurb, &dev->asyncq, next) {
-        if (p != aurb->packet) {
+    if (p->combined) {
+        usb_combined_packet_cancel(udev, p);
+        return;
+    }
+
+    packet_id_queue_add(&dev->cancelled, p->id);
+    usbredirparser_send_cancel_data_packet(dev->parser, p->id);
+    usbredirparser_do_write(dev->parser);
+}
+
+static int usbredir_is_cancelled(USBRedirDevice *dev, uint64_t id)
+{
+    if (!dev->dev.attached) {
+        return 1; /* Treat everything as cancelled after a disconnect */
+    }
+    return packet_id_queue_remove(&dev->cancelled, id);
+}
+
+static void usbredir_fill_already_in_flight_from_ep(USBRedirDevice *dev,
+    struct USBEndpoint *ep)
+{
+    static USBPacket *p;
+
+    QTAILQ_FOREACH(p, &ep->queue, queue) {
+        /* Skip combined packets, except for the first */
+        if (p->combined && p != p->combined->first) {
             continue;
         }
+        if (p->state == USB_PACKET_ASYNC) {
+            packet_id_queue_add(&dev->already_in_flight, p->id);
+        }
+    }
+}
 
-        DPRINTF("async cancel id %u\n", aurb->packet_id);
-        usbredirparser_send_cancel_data_packet(dev->parser, aurb->packet_id);
-        usbredirparser_do_write(dev->parser);
+static void usbredir_fill_already_in_flight(USBRedirDevice *dev)
+{
+    int ep;
+    struct USBDevice *udev = &dev->dev;
 
-        /* Mark it as dead */
-        aurb->packet = NULL;
-        break;
+    usbredir_fill_already_in_flight_from_ep(dev, &udev->ep_ctl);
+
+    for (ep = 0; ep < USB_MAX_ENDPOINTS; ep++) {
+        usbredir_fill_already_in_flight_from_ep(dev, &udev->ep_in[ep]);
+        usbredir_fill_already_in_flight_from_ep(dev, &udev->ep_out[ep]);
+    }
+}
+
+static int usbredir_already_in_flight(USBRedirDevice *dev, uint64_t id)
+{
+    return packet_id_queue_remove(&dev->already_in_flight, id);
+}
+
+static USBPacket *usbredir_find_packet_by_id(USBRedirDevice *dev,
+    uint8_t ep, uint64_t id)
+{
+    USBPacket *p;
+
+    if (usbredir_is_cancelled(dev, id)) {
+        return NULL;
+    }
+
+    p = usb_ep_find_packet_by_id(&dev->dev,
+                            (ep & USB_DIR_IN) ? USB_TOKEN_IN : USB_TOKEN_OUT,
+                            ep & 0x0f, id);
+    if (p == NULL) {
+        ERROR("could not find packet with id %"PRIu64"\n", id);
     }
+    return p;
 }
 
 static void bufp_alloc(USBRedirDevice *dev,
@@ -360,7 +445,7 @@ static void usbredir_handle_reset(USBDevice *udev)
     usbredirparser_do_write(dev->parser);
 }
 
-static int usbredir_handle_iso_data(USBRedirDevice *dev, USBPacket *p,
+static void usbredir_handle_iso_data(USBRedirDevice *dev, USBPacket *p,
                                      uint8_t ep)
 {
     int status, len;
@@ -417,7 +502,7 @@ static int usbredir_handle_iso_data(USBRedirDevice *dev, USBPacket *p,
                 !dev->endpoint[EP2I(ep)].bufpq_prefilled) {
             if (dev->endpoint[EP2I(ep)].bufpq_size <
                     dev->endpoint[EP2I(ep)].bufpq_target_size) {
-                return usbredir_handle_status(dev, 0, 0);
+                return;
             }
             dev->endpoint[EP2I(ep)].bufpq_prefilled = 1;
         }
@@ -431,27 +516,23 @@ static int usbredir_handle_iso_data(USBRedirDevice *dev, USBPacket *p,
             /* Check iso_error for stream errors, otherwise its an underrun */
             status = dev->endpoint[EP2I(ep)].iso_error;
             dev->endpoint[EP2I(ep)].iso_error = 0;
-            return status ? USB_RET_IOERROR : 0;
+            p->status = status ? USB_RET_IOERROR : USB_RET_SUCCESS;
+            return;
         }
         DPRINTF2("iso-token-in ep %02X status %d len %d queue-size: %d\n", ep,
                  isop->status, isop->len, dev->endpoint[EP2I(ep)].bufpq_size);
 
         status = isop->status;
-        if (status != usb_redir_success) {
-            bufp_free(dev, isop, ep);
-            return USB_RET_IOERROR;
-        }
-
         len = isop->len;
         if (len > p->iov.size) {
             ERROR("received iso data is larger then packet ep %02X (%d > %d)\n",
                   ep, len, (int)p->iov.size);
-            bufp_free(dev, isop, ep);
-            return USB_RET_BABBLE;
+            len = p->iov.size;
+            status = usb_redir_babble;
         }
         usb_packet_copy(p, isop->data, len);
         bufp_free(dev, isop, ep);
-        return len;
+        usbredir_handle_status(dev, p, status);
     } else {
         /* If the stream was not started because of a pending error don't
            send the packet to the usb-host */
@@ -471,7 +552,7 @@ static int usbredir_handle_iso_data(USBRedirDevice *dev, USBPacket *p,
         dev->endpoint[EP2I(ep)].iso_error = 0;
         DPRINTF2("iso-token-out ep %02X status %d len %zd\n", ep, status,
                  p->iov.size);
-        return usbredir_handle_status(dev, status, p->iov.size);
+        usbredir_handle_status(dev, p, status);
     }
 }
 
@@ -489,108 +570,122 @@ static void usbredir_stop_iso_stream(USBRedirDevice *dev, uint8_t ep)
     usbredir_free_bufpq(dev, ep);
 }
 
-static int usbredir_handle_bulk_data(USBRedirDevice *dev, USBPacket *p,
+static void usbredir_handle_bulk_data(USBRedirDevice *dev, USBPacket *p,
                                       uint8_t ep)
 {
-    AsyncURB *aurb = async_alloc(dev, p);
     struct usb_redir_bulk_packet_header bulk_packet;
+    size_t size = (p->combined) ? p->combined->iov.size : p->iov.size;
+
+    DPRINTF("bulk-out ep %02X len %zd id %"PRIu64"\n", ep, size, p->id);
 
-    DPRINTF("bulk-out ep %02X len %zd id %u\n", ep,
-            p->iov.size, aurb->packet_id);
+    if (usbredir_already_in_flight(dev, p->id)) {
+        p->status = USB_RET_ASYNC;
+        return;
+    }
 
     bulk_packet.endpoint  = ep;
-    bulk_packet.length    = p->iov.size;
+    bulk_packet.length    = size;
     bulk_packet.stream_id = 0;
-    aurb->bulk_packet = bulk_packet;
+    bulk_packet.length_high = size >> 16;
+    assert(bulk_packet.length_high == 0 ||
+           usbredirparser_peer_has_cap(dev->parser,
+                                       usb_redir_cap_32bits_bulk_length));
 
     if (ep & USB_DIR_IN) {
-        usbredirparser_send_bulk_packet(dev->parser, aurb->packet_id,
+        usbredirparser_send_bulk_packet(dev->parser, p->id,
                                         &bulk_packet, NULL, 0);
     } else {
-        uint8_t buf[p->iov.size];
-        usb_packet_copy(p, buf, p->iov.size);
-        usbredir_log_data(dev, "bulk data out:", buf, p->iov.size);
-        usbredirparser_send_bulk_packet(dev->parser, aurb->packet_id,
-                                        &bulk_packet, buf, p->iov.size);
+        uint8_t buf[size];
+        if (p->combined) {
+            iov_to_buf(p->combined->iov.iov, p->combined->iov.niov,
+                       0, buf, size);
+        } else {
+            usb_packet_copy(p, buf, size);
+        }
+        usbredir_log_data(dev, "bulk data out:", buf, size);
+        usbredirparser_send_bulk_packet(dev->parser, p->id,
+                                        &bulk_packet, buf, size);
     }
     usbredirparser_do_write(dev->parser);
-    return USB_RET_ASYNC;
+    p->status = USB_RET_ASYNC;
 }
 
-static int usbredir_handle_interrupt_data(USBRedirDevice *dev,
-                                           USBPacket *p, uint8_t ep)
+static void usbredir_handle_interrupt_in_data(USBRedirDevice *dev,
+                                              USBPacket *p, uint8_t ep)
 {
-    if (ep & USB_DIR_IN) {
-        /* Input interrupt endpoint, buffered packet input */
-        struct buf_packet *intp;
-        int status, len;
-
-        if (!dev->endpoint[EP2I(ep)].interrupt_started &&
-                !dev->endpoint[EP2I(ep)].interrupt_error) {
-            struct usb_redir_start_interrupt_receiving_header start_int = {
-                .endpoint = ep,
-            };
-            /* No id, we look at the ep when receiving a status back */
-            usbredirparser_send_start_interrupt_receiving(dev->parser, 0,
-                                                          &start_int);
-            usbredirparser_do_write(dev->parser);
-            DPRINTF("interrupt recv started ep %02X\n", ep);
-            dev->endpoint[EP2I(ep)].interrupt_started = 1;
-            /* We don't really want to drop interrupt packets ever, but
-               having some upper limit to how much we buffer is good. */
-            dev->endpoint[EP2I(ep)].bufpq_target_size = 1000;
-            dev->endpoint[EP2I(ep)].bufpq_dropping_packets = 0;
-        }
+    /* Input interrupt endpoint, buffered packet input */
+    struct buf_packet *intp;
+    int status, len;
 
-        intp = QTAILQ_FIRST(&dev->endpoint[EP2I(ep)].bufpq);
-        if (intp == NULL) {
-            DPRINTF2("interrupt-token-in ep %02X, no intp\n", ep);
-            /* Check interrupt_error for stream errors */
-            status = dev->endpoint[EP2I(ep)].interrupt_error;
-            dev->endpoint[EP2I(ep)].interrupt_error = 0;
-            if (status) {
-                return usbredir_handle_status(dev, status, 0);
-            }
-            return USB_RET_NAK;
-        }
-        DPRINTF("interrupt-token-in ep %02X status %d len %d\n", ep,
-                intp->status, intp->len);
+    if (!dev->endpoint[EP2I(ep)].interrupt_started &&
+            !dev->endpoint[EP2I(ep)].interrupt_error) {
+        struct usb_redir_start_interrupt_receiving_header start_int = {
+            .endpoint = ep,
+        };
+        /* No id, we look at the ep when receiving a status back */
+        usbredirparser_send_start_interrupt_receiving(dev->parser, 0,
+                                                      &start_int);
+        usbredirparser_do_write(dev->parser);
+        DPRINTF("interrupt recv started ep %02X\n", ep);
+        dev->endpoint[EP2I(ep)].interrupt_started = 1;
+        /* We don't really want to drop interrupt packets ever, but
+           having some upper limit to how much we buffer is good. */
+        dev->endpoint[EP2I(ep)].bufpq_target_size = 1000;
+        dev->endpoint[EP2I(ep)].bufpq_dropping_packets = 0;
+    }
 
-        status = intp->status;
-        if (status != usb_redir_success) {
-            bufp_free(dev, intp, ep);
-            return usbredir_handle_status(dev, status, 0);
+    intp = QTAILQ_FIRST(&dev->endpoint[EP2I(ep)].bufpq);
+    if (intp == NULL) {
+        DPRINTF2("interrupt-token-in ep %02X, no intp\n", ep);
+        /* Check interrupt_error for stream errors */
+        status = dev->endpoint[EP2I(ep)].interrupt_error;
+        dev->endpoint[EP2I(ep)].interrupt_error = 0;
+        if (status) {
+            usbredir_handle_status(dev, p, status);
+        } else {
+            p->status = USB_RET_NAK;
         }
+        return;
+    }
+    DPRINTF("interrupt-token-in ep %02X status %d len %d\n", ep,
+            intp->status, intp->len);
 
-        len = intp->len;
-        if (len > p->iov.size) {
-            ERROR("received int data is larger then packet ep %02X\n", ep);
-            bufp_free(dev, intp, ep);
-            return USB_RET_BABBLE;
-        }
-        usb_packet_copy(p, intp->data, len);
-        bufp_free(dev, intp, ep);
-        return len;
-    } else {
-        /* Output interrupt endpoint, normal async operation */
-        AsyncURB *aurb = async_alloc(dev, p);
-        struct usb_redir_interrupt_packet_header interrupt_packet;
-        uint8_t buf[p->iov.size];
-
-        DPRINTF("interrupt-out ep %02X len %zd id %u\n", ep, p->iov.size,
-                aurb->packet_id);
-
-        interrupt_packet.endpoint  = ep;
-        interrupt_packet.length    = p->iov.size;
-        aurb->interrupt_packet     = interrupt_packet;
-
-        usb_packet_copy(p, buf, p->iov.size);
-        usbredir_log_data(dev, "interrupt data out:", buf, p->iov.size);
-        usbredirparser_send_interrupt_packet(dev->parser, aurb->packet_id,
-                                        &interrupt_packet, buf, p->iov.size);
-        usbredirparser_do_write(dev->parser);
-        return USB_RET_ASYNC;
+    status = intp->status;
+    len = intp->len;
+    if (len > p->iov.size) {
+        ERROR("received int data is larger then packet ep %02X\n", ep);
+        len = p->iov.size;
+        status = usb_redir_babble;
     }
+    usb_packet_copy(p, intp->data, len);
+    bufp_free(dev, intp, ep);
+    usbredir_handle_status(dev, p, status);
+}
+
+/*
+ * Handle interrupt out data, the usbredir protocol expects us to do this
+ * async, so that it can report back a completion status. But guests will
+ * expect immediate completion for an interrupt endpoint, and handling this
+ * async causes migration issues. So we report success directly, counting
+ * on the fact that output interrupt packets normally always succeed.
+ */
+static void usbredir_handle_interrupt_out_data(USBRedirDevice *dev,
+                                               USBPacket *p, uint8_t ep)
+{
+    struct usb_redir_interrupt_packet_header interrupt_packet;
+    uint8_t buf[p->iov.size];
+
+    DPRINTF("interrupt-out ep %02X len %zd id %"PRIu64"\n", ep,
+            p->iov.size, p->id);
+
+    interrupt_packet.endpoint  = ep;
+    interrupt_packet.length    = p->iov.size;
+
+    usb_packet_copy(p, buf, p->iov.size);
+    usbredir_log_data(dev, "interrupt data out:", buf, p->iov.size);
+    usbredirparser_send_interrupt_packet(dev->parser, p->id,
+                                    &interrupt_packet, buf, p->iov.size);
+    usbredirparser_do_write(dev->parser);
 }
 
 static void usbredir_stop_interrupt_receiving(USBRedirDevice *dev,
@@ -609,7 +704,7 @@ static void usbredir_stop_interrupt_receiving(USBRedirDevice *dev,
     usbredir_free_bufpq(dev, ep);
 }
 
-static int usbredir_handle_data(USBDevice *udev, USBPacket *p)
+static void usbredir_handle_data(USBDevice *udev, USBPacket *p)
 {
     USBRedirDevice *dev = DO_UPCAST(USBRedirDevice, dev, udev);
     uint8_t ep;
@@ -622,28 +717,47 @@ static int usbredir_handle_data(USBDevice *udev, USBPacket *p)
     switch (dev->endpoint[EP2I(ep)].type) {
     case USB_ENDPOINT_XFER_CONTROL:
         ERROR("handle_data called for control transfer on ep %02X\n", ep);
-        return USB_RET_NAK;
+        p->status = USB_RET_NAK;
+        break;
     case USB_ENDPOINT_XFER_ISOC:
-        return usbredir_handle_iso_data(dev, p, ep);
+        usbredir_handle_iso_data(dev, p, ep);
+        break;
     case USB_ENDPOINT_XFER_BULK:
-        return usbredir_handle_bulk_data(dev, p, ep);
+        if (p->state == USB_PACKET_SETUP && p->pid == USB_TOKEN_IN &&
+                p->ep->pipeline) {
+            p->status = USB_RET_ADD_TO_QUEUE;
+            break;
+        }
+        usbredir_handle_bulk_data(dev, p, ep);
+        break;
     case USB_ENDPOINT_XFER_INT:
-        return usbredir_handle_interrupt_data(dev, p, ep);
+        if (ep & USB_DIR_IN) {
+            usbredir_handle_interrupt_in_data(dev, p, ep);
+        } else {
+            usbredir_handle_interrupt_out_data(dev, p, ep);
+        }
+        break;
     default:
         ERROR("handle_data ep %02X has unknown type %d\n", ep,
               dev->endpoint[EP2I(ep)].type);
-        return USB_RET_NAK;
+        p->status = USB_RET_NAK;
+    }
+}
+
+static void usbredir_flush_ep_queue(USBDevice *dev, USBEndpoint *ep)
+{
+    if (ep->pid == USB_TOKEN_IN && ep->pipeline) {
+        usb_ep_combine_input_packets(ep);
     }
 }
 
-static int usbredir_set_config(USBRedirDevice *dev, USBPacket *p,
+static void usbredir_set_config(USBRedirDevice *dev, USBPacket *p,
                                 int config)
 {
     struct usb_redir_set_configuration_header set_config;
-    AsyncURB *aurb = async_alloc(dev, p);
     int i;
 
-    DPRINTF("set config %d id %u\n", config, aurb->packet_id);
+    DPRINTF("set config %d id %"PRIu64"\n", config, p->id);
 
     for (i = 0; i < MAX_ENDPOINTS; i++) {
         switch (dev->endpoint[i].type) {
@@ -660,33 +774,27 @@ static int usbredir_set_config(USBRedirDevice *dev, USBPacket *p,
     }
 
     set_config.configuration = config;
-    usbredirparser_send_set_configuration(dev->parser, aurb->packet_id,
-                                          &set_config);
+    usbredirparser_send_set_configuration(dev->parser, p->id, &set_config);
     usbredirparser_do_write(dev->parser);
-    return USB_RET_ASYNC;
+    p->status = USB_RET_ASYNC;
 }
 
-static int usbredir_get_config(USBRedirDevice *dev, USBPacket *p)
+static void usbredir_get_config(USBRedirDevice *dev, USBPacket *p)
 {
-    AsyncURB *aurb = async_alloc(dev, p);
+    DPRINTF("get config id %"PRIu64"\n", p->id);
 
-    DPRINTF("get config id %u\n", aurb->packet_id);
-
-    aurb->get = 1;
-    usbredirparser_send_get_configuration(dev->parser, aurb->packet_id);
+    usbredirparser_send_get_configuration(dev->parser, p->id);
     usbredirparser_do_write(dev->parser);
-    return USB_RET_ASYNC;
+    p->status = USB_RET_ASYNC;
 }
 
-static int usbredir_set_interface(USBRedirDevice *dev, USBPacket *p,
+static void usbredir_set_interface(USBRedirDevice *dev, USBPacket *p,
                                    int interface, int alt)
 {
     struct usb_redir_set_alt_setting_header set_alt;
-    AsyncURB *aurb = async_alloc(dev, p);
     int i;
 
-    DPRINTF("set interface %d alt %d id %u\n", interface, alt,
-            aurb->packet_id);
+    DPRINTF("set interface %d alt %d id %"PRIu64"\n", interface, alt, p->id);
 
     for (i = 0; i < MAX_ENDPOINTS; i++) {
         if (dev->endpoint[i].interface == interface) {
@@ -706,58 +814,59 @@ static int usbredir_set_interface(USBRedirDevice *dev, USBPacket *p,
 
     set_alt.interface = interface;
     set_alt.alt = alt;
-    usbredirparser_send_set_alt_setting(dev->parser, aurb->packet_id,
-                                        &set_alt);
+    usbredirparser_send_set_alt_setting(dev->parser, p->id, &set_alt);
     usbredirparser_do_write(dev->parser);
-    return USB_RET_ASYNC;
+    p->status = USB_RET_ASYNC;
 }
 
-static int usbredir_get_interface(USBRedirDevice *dev, USBPacket *p,
+static void usbredir_get_interface(USBRedirDevice *dev, USBPacket *p,
                                    int interface)
 {
     struct usb_redir_get_alt_setting_header get_alt;
-    AsyncURB *aurb = async_alloc(dev, p);
 
-    DPRINTF("get interface %d id %u\n", interface, aurb->packet_id);
+    DPRINTF("get interface %d id %"PRIu64"\n", interface, p->id);
 
     get_alt.interface = interface;
-    aurb->get = 1;
-    usbredirparser_send_get_alt_setting(dev->parser, aurb->packet_id,
-                                        &get_alt);
+    usbredirparser_send_get_alt_setting(dev->parser, p->id, &get_alt);
     usbredirparser_do_write(dev->parser);
-    return USB_RET_ASYNC;
+    p->status = USB_RET_ASYNC;
 }
 
-static int usbredir_handle_control(USBDevice *udev, USBPacket *p,
+static void usbredir_handle_control(USBDevice *udev, USBPacket *p,
         int request, int value, int index, int length, uint8_t *data)
 {
     USBRedirDevice *dev = DO_UPCAST(USBRedirDevice, dev, udev);
     struct usb_redir_control_packet_header control_packet;
-    AsyncURB *aurb;
+
+    if (usbredir_already_in_flight(dev, p->id)) {
+        p->status = USB_RET_ASYNC;
+        return;
+    }
 
     /* Special cases for certain standard device requests */
     switch (request) {
     case DeviceOutRequest | USB_REQ_SET_ADDRESS:
         DPRINTF("set address %d\n", value);
         dev->dev.addr = value;
-        return 0;
+        return;
     case DeviceOutRequest | USB_REQ_SET_CONFIGURATION:
-        return usbredir_set_config(dev, p, value & 0xff);
+        usbredir_set_config(dev, p, value & 0xff);
+        return;
     case DeviceRequest | USB_REQ_GET_CONFIGURATION:
-        return usbredir_get_config(dev, p);
+        usbredir_get_config(dev, p);
+        return;
     case InterfaceOutRequest | USB_REQ_SET_INTERFACE:
-        return usbredir_set_interface(dev, p, index, value);
+        usbredir_set_interface(dev, p, index, value);
+        return;
     case InterfaceRequest | USB_REQ_GET_INTERFACE:
-        return usbredir_get_interface(dev, p, index);
+        usbredir_get_interface(dev, p, index);
+        return;
     }
 
-    /* "Normal" ctrl requests */
-    aurb = async_alloc(dev, p);
-
-    /* Note request is (bRequestType << 8) | bRequest */
-    DPRINTF("ctrl-out type 0x%x req 0x%x val 0x%x index %d len %d id %u\n",
-            request >> 8, request & 0xff, value, index, length,
-            aurb->packet_id);
+    /* Normal ctrl requests, note request is (bRequestType << 8) | bRequest */
+    DPRINTF(
+        "ctrl-out type 0x%x req 0x%x val 0x%x index %d len %d id %"PRIu64"\n",
+        request >> 8, request & 0xff, value, index, length, p->id);
 
     control_packet.request     = request & 0xFF;
     control_packet.requesttype = request >> 8;
@@ -765,18 +874,17 @@ static int usbredir_handle_control(USBDevice *udev, USBPacket *p,
     control_packet.value       = value;
     control_packet.index       = index;
     control_packet.length      = length;
-    aurb->control_packet       = control_packet;
 
     if (control_packet.requesttype & USB_DIR_IN) {
-        usbredirparser_send_control_packet(dev->parser, aurb->packet_id,
+        usbredirparser_send_control_packet(dev->parser, p->id,
                                            &control_packet, NULL, 0);
     } else {
         usbredir_log_data(dev, "ctrl data out:", data, length);
-        usbredirparser_send_control_packet(dev->parser, aurb->packet_id,
+        usbredirparser_send_control_packet(dev->parser, p->id,
                                            &control_packet, data, length);
     }
     usbredirparser_do_write(dev->parser);
-    return USB_RET_ASYNC;
+    p->status = USB_RET_ASYNC;
 }
 
 /*
@@ -784,53 +892,70 @@ static int usbredir_handle_control(USBDevice *udev, USBPacket *p,
  * from within the USBDevice data / control packet callbacks and doing a
  * usb_detach from within these callbacks is not a good idea.
  *
- * So we use a bh handler to take care of close events. We also handle
- * open events from this callback to make sure that a close directly followed
- * by an open gets handled in the right order.
+ * So we use a bh handler to take care of close events.
  */
-static void usbredir_open_close_bh(void *opaque)
+static void usbredir_chardev_close_bh(void *opaque)
 {
     USBRedirDevice *dev = opaque;
-    uint32_t caps[USB_REDIR_CAPS_SIZE] = { 0, };
-    char version[32];
-
-    strcpy(version, "qemu usb-redir guest ");
-    pstrcat(version, sizeof(version), qemu_get_version());
 
     usbredir_device_disconnect(dev);
 
     if (dev->parser) {
+        DPRINTF("destroying usbredirparser\n");
         usbredirparser_destroy(dev->parser);
         dev->parser = NULL;
     }
+}
 
-    if (dev->cs->opened) {
-        dev->parser = qemu_oom_check(usbredirparser_create());
-        dev->parser->priv = dev;
-        dev->parser->log_func = usbredir_log;
-        dev->parser->read_func = usbredir_read;
-        dev->parser->write_func = usbredir_write;
-        dev->parser->hello_func = usbredir_hello;
-        dev->parser->device_connect_func = usbredir_device_connect;
-        dev->parser->device_disconnect_func = usbredir_device_disconnect;
-        dev->parser->interface_info_func = usbredir_interface_info;
-        dev->parser->ep_info_func = usbredir_ep_info;
-        dev->parser->configuration_status_func = usbredir_configuration_status;
-        dev->parser->alt_setting_status_func = usbredir_alt_setting_status;
-        dev->parser->iso_stream_status_func = usbredir_iso_stream_status;
-        dev->parser->interrupt_receiving_status_func =
-            usbredir_interrupt_receiving_status;
-        dev->parser->bulk_streams_status_func = usbredir_bulk_streams_status;
-        dev->parser->control_packet_func = usbredir_control_packet;
-        dev->parser->bulk_packet_func = usbredir_bulk_packet;
-        dev->parser->iso_packet_func = usbredir_iso_packet;
-        dev->parser->interrupt_packet_func = usbredir_interrupt_packet;
-        dev->read_buf = NULL;
-        dev->read_buf_size = 0;
+static void usbredir_create_parser(USBRedirDevice *dev)
+{
+    uint32_t caps[USB_REDIR_CAPS_SIZE] = { 0, };
+    int flags = 0;
+
+    DPRINTF("creating usbredirparser\n");
+
+    dev->parser = qemu_oom_check(usbredirparser_create());
+    dev->parser->priv = dev;
+    dev->parser->log_func = usbredir_log;
+    dev->parser->read_func = usbredir_read;
+    dev->parser->write_func = usbredir_write;
+    dev->parser->hello_func = usbredir_hello;
+    dev->parser->device_connect_func = usbredir_device_connect;
+    dev->parser->device_disconnect_func = usbredir_device_disconnect;
+    dev->parser->interface_info_func = usbredir_interface_info;
+    dev->parser->ep_info_func = usbredir_ep_info;
+    dev->parser->configuration_status_func = usbredir_configuration_status;
+    dev->parser->alt_setting_status_func = usbredir_alt_setting_status;
+    dev->parser->iso_stream_status_func = usbredir_iso_stream_status;
+    dev->parser->interrupt_receiving_status_func =
+        usbredir_interrupt_receiving_status;
+    dev->parser->bulk_streams_status_func = usbredir_bulk_streams_status;
+    dev->parser->control_packet_func = usbredir_control_packet;
+    dev->parser->bulk_packet_func = usbredir_bulk_packet;
+    dev->parser->iso_packet_func = usbredir_iso_packet;
+    dev->parser->interrupt_packet_func = usbredir_interrupt_packet;
+    dev->read_buf = NULL;
+    dev->read_buf_size = 0;
+
+    usbredirparser_caps_set_cap(caps, usb_redir_cap_connect_device_version);
+    usbredirparser_caps_set_cap(caps, usb_redir_cap_filter);
+    usbredirparser_caps_set_cap(caps, usb_redir_cap_ep_info_max_packet_size);
+    usbredirparser_caps_set_cap(caps, usb_redir_cap_64bits_ids);
+    usbredirparser_caps_set_cap(caps, usb_redir_cap_32bits_bulk_length);
+
+    if (runstate_check(RUN_STATE_INMIGRATE)) {
+        flags |= usbredirparser_fl_no_hello;
+    }
+    usbredirparser_init(dev->parser, VERSION, caps, USB_REDIR_CAPS_SIZE,
+                        flags);
+    usbredirparser_do_write(dev->parser);
+}
 
-        usbredirparser_caps_set_cap(caps, usb_redir_cap_connect_device_version);
-        usbredirparser_caps_set_cap(caps, usb_redir_cap_filter);
-        usbredirparser_init(dev->parser, version, caps, USB_REDIR_CAPS_SIZE, 0);
+static void usbredir_reject_device(USBRedirDevice *dev)
+{
+    usbredir_device_disconnect(dev);
+    if (usbredirparser_peer_has_cap(dev->parser, usb_redir_cap_filter)) {
+        usbredirparser_send_filter_reject(dev->parser);
         usbredirparser_do_write(dev->parser);
     }
 }
@@ -839,12 +964,20 @@ static void usbredir_do_attach(void *opaque)
 {
     USBRedirDevice *dev = opaque;
 
+    /* In order to work properly with XHCI controllers we need these caps */
+    if ((dev->dev.port->speedmask & USB_SPEED_MASK_SUPER) && !(
+        usbredirparser_peer_has_cap(dev->parser,
+                                    usb_redir_cap_ep_info_max_packet_size) &&
+        usbredirparser_peer_has_cap(dev->parser,
+                                    usb_redir_cap_64bits_ids))) {
+        ERROR("usb-redir-host lacks capabilities needed for use with XHCI\n");
+        usbredir_reject_device(dev);
+        return;
+    }
+
     if (usb_device_attach(&dev->dev) != 0) {
-        usbredir_device_disconnect(dev);
-        if (usbredirparser_peer_has_cap(dev->parser, usb_redir_cap_filter)) {
-            usbredirparser_send_filter_reject(dev->parser);
-            usbredirparser_do_write(dev->parser);
-        }
+        WARNING("rejecting device due to speed mismatch\n");
+        usbredir_reject_device(dev);
     }
 }
 
@@ -856,13 +989,18 @@ static int usbredir_chardev_can_read(void *opaque)
 {
     USBRedirDevice *dev = opaque;
 
-    if (dev->parser) {
-        /* usbredir_parser_do_read will consume *all* data we give it */
-        return 1024 * 1024;
-    } else {
-        /* usbredir_open_close_bh hasn't handled the open event yet */
+    if (!dev->parser) {
+        WARNING("chardev_can_read called on non open chardev!\n");
+        return 0;
+    }
+
+    /* Don't read new data from the chardev until our state is fully synced */
+    if (!runstate_check(RUN_STATE_RUNNING)) {
         return 0;
     }
+
+    /* usbredir_parser_do_read will consume *all* data we give it */
+    return 1024 * 1024;
 }
 
 static void usbredir_chardev_read(void *opaque, const uint8_t *buf, int size)
@@ -886,8 +1024,15 @@ static void usbredir_chardev_event(void *opaque, int event)
 
     switch (event) {
     case CHR_EVENT_OPENED:
+        DPRINTF("chardev open\n");
+        /* Make sure any pending closes are handled (no-op if none pending) */
+        usbredir_chardev_close_bh(dev);
+        qemu_bh_cancel(dev->chardev_close_bh);
+        usbredir_create_parser(dev);
+        break;
     case CHR_EVENT_CLOSED:
-        qemu_bh_schedule(dev->open_close_bh);
+        DPRINTF("chardev close\n");
+        qemu_bh_schedule(dev->chardev_close_bh);
         break;
     }
 }
@@ -896,6 +1041,15 @@ static void usbredir_chardev_event(void *opaque, int event)
  * init + destroy
  */
 
+static void usbredir_vm_state_change(void *priv, int running, RunState state)
+{
+    USBRedirDevice *dev = priv;
+
+    if (state == RUN_STATE_RUNNING && dev->parser != NULL) {
+        usbredirparser_do_write(dev->parser); /* Flush any pending writes */
+    }
+}
+
 static int usbredir_initfn(USBDevice *udev)
 {
     USBRedirDevice *dev = DO_UPCAST(USBRedirDevice, dev, udev);
@@ -917,10 +1071,11 @@ static int usbredir_initfn(USBDevice *udev)
         }
     }
 
-    dev->open_close_bh = qemu_bh_new(usbredir_open_close_bh, dev);
+    dev->chardev_close_bh = qemu_bh_new(usbredir_chardev_close_bh, dev);
     dev->attach_timer = qemu_new_timer_ms(vm_clock, usbredir_do_attach, dev);
 
-    QTAILQ_INIT(&dev->asyncq);
+    packet_id_queue_init(&dev->cancelled, dev, "cancelled");
+    packet_id_queue_init(&dev->already_in_flight, dev, "already-in-flight");
     for (i = 0; i < MAX_ENDPOINTS; i++) {
         QTAILQ_INIT(&dev->endpoint[i].bufpq);
     }
@@ -928,23 +1083,25 @@ static int usbredir_initfn(USBDevice *udev)
     /* We'll do the attach once we receive the speed from the usb-host */
     udev->auto_attach = 0;
 
+    /* Will be cleared during setup when we find conflicts */
+    dev->compatible_speedmask = USB_SPEED_MASK_FULL | USB_SPEED_MASK_HIGH;
+
     /* Let the backend know we are ready */
     qemu_chr_fe_open(dev->cs);
     qemu_chr_add_handlers(dev->cs, usbredir_chardev_can_read,
                           usbredir_chardev_read, usbredir_chardev_event, dev);
 
+    qemu_add_vm_change_state_handler(usbredir_vm_state_change, dev);
     add_boot_device_path(dev->bootindex, &udev->qdev, NULL);
     return 0;
 }
 
 static void usbredir_cleanup_device_queues(USBRedirDevice *dev)
 {
-    AsyncURB *aurb, *next_aurb;
     int i;
 
-    QTAILQ_FOREACH_SAFE(aurb, &dev->asyncq, next, next_aurb) {
-        async_free(dev, aurb);
-    }
+    packet_id_queue_empty(&dev->cancelled);
+    packet_id_queue_empty(&dev->already_in_flight);
     for (i = 0; i < MAX_ENDPOINTS; i++) {
         usbredir_free_bufpq(dev, I2EP(i));
     }
@@ -957,7 +1114,7 @@ static void usbredir_handle_destroy(USBDevice *udev)
     qemu_chr_fe_close(dev->cs);
     qemu_chr_delete(dev->cs);
     /* Note must be done after qemu_chr_close, as that causes a close event */
-    qemu_bh_delete(dev->open_close_bh);
+    qemu_bh_delete(dev->chardev_close_bh);
 
     qemu_del_timer(dev->attach_timer);
     qemu_free_timer(dev->attach_timer);
@@ -1007,11 +1164,7 @@ static int usbredir_check_filter(USBRedirDevice *dev)
     return 0;
 
 error:
-    usbredir_device_disconnect(dev);
-    if (usbredirparser_peer_has_cap(dev->parser, usb_redir_cap_filter)) {
-        usbredirparser_send_filter_reject(dev->parser);
-        usbredirparser_do_write(dev->parser);
-    }
+    usbredir_reject_device(dev);
     return -1;
 }
 
@@ -1019,26 +1172,34 @@ error:
  * usbredirparser packet complete callbacks
  */
 
-static int usbredir_handle_status(USBRedirDevice *dev,
-                                       int status, int actual_len)
+static void usbredir_handle_status(USBRedirDevice *dev, USBPacket *p,
+    int status)
 {
     switch (status) {
     case usb_redir_success:
-        return actual_len;
+        p->status = USB_RET_SUCCESS; /* Clear previous ASYNC status */
+        break;
     case usb_redir_stall:
-        return USB_RET_STALL;
+        p->status = USB_RET_STALL;
+        break;
     case usb_redir_cancelled:
-        WARNING("returning cancelled packet to HC?\n");
-        return USB_RET_NAK;
+        /*
+         * When the usbredir-host unredirects a device, it will report a status
+         * of cancelled for all pending packets, followed by a disconnect msg.
+         */
+        p->status = USB_RET_IOERROR;
+        break;
     case usb_redir_inval:
         WARNING("got invalid param error from usb-host?\n");
-        return USB_RET_NAK;
+        p->status = USB_RET_IOERROR;
+        break;
     case usb_redir_babble:
-        return USB_RET_BABBLE;
+        p->status = USB_RET_BABBLE;
+        break;
     case usb_redir_ioerror:
     case usb_redir_timeout:
     default:
-        return USB_RET_IOERROR;
+        p->status = USB_RET_IOERROR;
     }
 }
 
@@ -1070,10 +1231,13 @@ static void usbredir_device_connect(void *priv,
     case usb_redir_speed_low:
         speed = "low speed";
         dev->dev.speed = USB_SPEED_LOW;
+        dev->compatible_speedmask &= ~USB_SPEED_MASK_FULL;
+        dev->compatible_speedmask &= ~USB_SPEED_MASK_HIGH;
         break;
     case usb_redir_speed_full:
         speed = "full speed";
         dev->dev.speed = USB_SPEED_FULL;
+        dev->compatible_speedmask &= ~USB_SPEED_MASK_HIGH;
         break;
     case usb_redir_speed_high:
         speed = "high speed";
@@ -1103,7 +1267,7 @@ static void usbredir_device_connect(void *priv,
              device_connect->device_class);
     }
 
-    dev->dev.speedmask = (1 << dev->dev.speed);
+    dev->dev.speedmask = (1 << dev->dev.speed) | dev->compatible_speedmask;
     dev->device_info = *device_connect;
 
     if (usbredir_check_filter(dev)) {
@@ -1124,6 +1288,7 @@ static void usbredir_device_disconnect(void *priv)
     qemu_del_timer(dev->attach_timer);
 
     if (dev->dev.attached) {
+        DPRINTF("detaching device\n");
         usb_device_detach(&dev->dev);
         /*
          * Delay next usb device attach to give the guest a chance to see
@@ -1142,6 +1307,7 @@ static void usbredir_device_disconnect(void *priv)
     dev->interface_info.interface_count = NO_INTERFACE_INFO;
     dev->dev.addr = 0;
     dev->dev.speed = 0;
+    dev->compatible_speedmask = USB_SPEED_MASK_FULL | USB_SPEED_MASK_HIGH;
 }
 
 static void usbredir_interface_info(void *priv,
@@ -1163,25 +1329,78 @@ static void usbredir_interface_info(void *priv,
     }
 }
 
+static void usbredir_mark_speed_incompatible(USBRedirDevice *dev, int speed)
+{
+    dev->compatible_speedmask &= ~(1 << speed);
+    dev->dev.speedmask = (1 << dev->dev.speed) | dev->compatible_speedmask;
+}
+
+static void usbredir_set_pipeline(USBRedirDevice *dev, struct USBEndpoint *uep)
+{
+    if (uep->type != USB_ENDPOINT_XFER_BULK) {
+        return;
+    }
+    if (uep->pid == USB_TOKEN_OUT) {
+        uep->pipeline = true;
+    }
+    if (uep->pid == USB_TOKEN_IN && uep->max_packet_size != 0 &&
+        usbredirparser_peer_has_cap(dev->parser,
+                                    usb_redir_cap_32bits_bulk_length)) {
+        uep->pipeline = true;
+    }
+}
+
+static void usbredir_setup_usb_eps(USBRedirDevice *dev)
+{
+    struct USBEndpoint *usb_ep;
+    int i, pid;
+
+    for (i = 0; i < MAX_ENDPOINTS; i++) {
+        pid = (i & 0x10) ? USB_TOKEN_IN : USB_TOKEN_OUT;
+        usb_ep = usb_ep_get(&dev->dev, pid, i & 0x0f);
+        usb_ep->type = dev->endpoint[i].type;
+        usb_ep->ifnum = dev->endpoint[i].interface;
+        usb_ep->max_packet_size = dev->endpoint[i].max_packet_size;
+        usbredir_set_pipeline(dev, usb_ep);
+    }
+}
+
 static void usbredir_ep_info(void *priv,
     struct usb_redir_ep_info_header *ep_info)
 {
     USBRedirDevice *dev = priv;
-    struct USBEndpoint *usb_ep;
     int i;
 
     for (i = 0; i < MAX_ENDPOINTS; i++) {
         dev->endpoint[i].type = ep_info->type[i];
         dev->endpoint[i].interval = ep_info->interval[i];
         dev->endpoint[i].interface = ep_info->interface[i];
+        if (usbredirparser_peer_has_cap(dev->parser,
+                                     usb_redir_cap_ep_info_max_packet_size)) {
+            dev->endpoint[i].max_packet_size = ep_info->max_packet_size[i];
+        }
         switch (dev->endpoint[i].type) {
         case usb_redir_type_invalid:
             break;
         case usb_redir_type_iso:
+            usbredir_mark_speed_incompatible(dev, USB_SPEED_FULL);
+            usbredir_mark_speed_incompatible(dev, USB_SPEED_HIGH);
+            /* Fall through */
         case usb_redir_type_interrupt:
+            if (!usbredirparser_peer_has_cap(dev->parser,
+                                     usb_redir_cap_ep_info_max_packet_size) ||
+                    ep_info->max_packet_size[i] > 64) {
+                usbredir_mark_speed_incompatible(dev, USB_SPEED_FULL);
+            }
+            if (!usbredirparser_peer_has_cap(dev->parser,
+                                     usb_redir_cap_ep_info_max_packet_size) ||
+                    ep_info->max_packet_size[i] > 1024) {
+                usbredir_mark_speed_incompatible(dev, USB_SPEED_HIGH);
+            }
             if (dev->endpoint[i].interval == 0) {
                 ERROR("Received 0 interval for isoc or irq endpoint\n");
-                usbredir_device_disconnect(dev);
+                usbredir_reject_device(dev);
+                return;
             }
             /* Fall through */
         case usb_redir_type_control:
@@ -1191,78 +1410,69 @@ static void usbredir_ep_info(void *priv,
             break;
         default:
             ERROR("Received invalid endpoint type\n");
-            usbredir_device_disconnect(dev);
+            usbredir_reject_device(dev);
             return;
         }
-        usb_ep = usb_ep_get(&dev->dev,
-                            (i & 0x10) ? USB_TOKEN_IN : USB_TOKEN_OUT,
-                            i & 0x0f);
-        usb_ep->type = dev->endpoint[i].type;
-        usb_ep->ifnum = dev->endpoint[i].interface;
     }
+    /* The new ep info may have caused a speed incompatibility, recheck */
+    if (dev->dev.attached &&
+            !(dev->dev.port->speedmask & dev->dev.speedmask)) {
+        ERROR("Device no longer matches speed after endpoint info change, "
+              "disconnecting!\n");
+        usbredir_reject_device(dev);
+        return;
+    }
+    usbredir_setup_usb_eps(dev);
 }
 
-static void usbredir_configuration_status(void *priv, uint32_t id,
+static void usbredir_configuration_status(void *priv, uint64_t id,
     struct usb_redir_configuration_status_header *config_status)
 {
     USBRedirDevice *dev = priv;
-    AsyncURB *aurb;
-    int len = 0;
+    USBPacket *p;
 
-    DPRINTF("set config status %d config %d id %u\n", config_status->status,
-            config_status->configuration, id);
+    DPRINTF("set config status %d config %d id %"PRIu64"\n",
+            config_status->status, config_status->configuration, id);
 
-    aurb = async_find(dev, id);
-    if (!aurb) {
-        return;
-    }
-    if (aurb->packet) {
-        if (aurb->get) {
+    p = usbredir_find_packet_by_id(dev, 0, id);
+    if (p) {
+        if (dev->dev.setup_buf[0] & USB_DIR_IN) {
             dev->dev.data_buf[0] = config_status->configuration;
-            len = 1;
+            p->actual_length = 1;
         }
-        aurb->packet->result =
-            usbredir_handle_status(dev, config_status->status, len);
-        usb_generic_async_ctrl_complete(&dev->dev, aurb->packet);
+        usbredir_handle_status(dev, p, config_status->status);
+        usb_generic_async_ctrl_complete(&dev->dev, p);
     }
-    async_free(dev, aurb);
 }
 
-static void usbredir_alt_setting_status(void *priv, uint32_t id,
+static void usbredir_alt_setting_status(void *priv, uint64_t id,
     struct usb_redir_alt_setting_status_header *alt_setting_status)
 {
     USBRedirDevice *dev = priv;
-    AsyncURB *aurb;
-    int len = 0;
+    USBPacket *p;
 
-    DPRINTF("alt status %d intf %d alt %d id: %u\n",
-            alt_setting_status->status,
-            alt_setting_status->interface,
+    DPRINTF("alt status %d intf %d alt %d id: %"PRIu64"\n",
+            alt_setting_status->status, alt_setting_status->interface,
             alt_setting_status->alt, id);
 
-    aurb = async_find(dev, id);
-    if (!aurb) {
-        return;
-    }
-    if (aurb->packet) {
-        if (aurb->get) {
+    p = usbredir_find_packet_by_id(dev, 0, id);
+    if (p) {
+        if (dev->dev.setup_buf[0] & USB_DIR_IN) {
             dev->dev.data_buf[0] = alt_setting_status->alt;
-            len = 1;
+            p->actual_length = 1;
         }
-        aurb->packet->result =
-            usbredir_handle_status(dev, alt_setting_status->status, len);
-        usb_generic_async_ctrl_complete(&dev->dev, aurb->packet);
+        usbredir_handle_status(dev, p, alt_setting_status->status);
+        usb_generic_async_ctrl_complete(&dev->dev, p);
     }
-    async_free(dev, aurb);
 }
 
-static void usbredir_iso_stream_status(void *priv, uint32_t id,
+static void usbredir_iso_stream_status(void *priv, uint64_t id,
     struct usb_redir_iso_stream_status_header *iso_stream_status)
 {
     USBRedirDevice *dev = priv;
     uint8_t ep = iso_stream_status->endpoint;
 
-    DPRINTF("iso status %d ep %02X id %u\n", iso_stream_status->status,
+    DPRINTF("iso status %d ep %02X id %"PRIu64"\n", iso_stream_status->status,
             ep, id);
 
     if (!dev->dev.attached || !dev->endpoint[EP2I(ep)].iso_started) {
@@ -1276,14 +1486,14 @@ static void usbredir_iso_stream_status(void *priv, uint32_t id,
     }
 }
 
-static void usbredir_interrupt_receiving_status(void *priv, uint32_t id,
+static void usbredir_interrupt_receiving_status(void *priv, uint64_t id,
     struct usb_redir_interrupt_receiving_status_header
     *interrupt_receiving_status)
 {
     USBRedirDevice *dev = priv;
     uint8_t ep = interrupt_receiving_status->endpoint;
 
-    DPRINTF("interrupt recv status %d ep %02X id %u\n",
+    DPRINTF("interrupt recv status %d ep %02X id %"PRIu64"\n",
             interrupt_receiving_status->status, ep, id);
 
     if (!dev->dev.attached || !dev->endpoint[EP2I(ep)].interrupt_started) {
@@ -1298,107 +1508,102 @@ static void usbredir_interrupt_receiving_status(void *priv, uint32_t id,
     }
 }
 
-static void usbredir_bulk_streams_status(void *priv, uint32_t id,
+static void usbredir_bulk_streams_status(void *priv, uint64_t id,
     struct usb_redir_bulk_streams_status_header *bulk_streams_status)
 {
 }
 
-static void usbredir_control_packet(void *priv, uint32_t id,
+static void usbredir_control_packet(void *priv, uint64_t id,
     struct usb_redir_control_packet_header *control_packet,
     uint8_t *data, int data_len)
 {
     USBRedirDevice *dev = priv;
+    USBPacket *p;
     int len = control_packet->length;
-    AsyncURB *aurb;
 
-    DPRINTF("ctrl-in status %d len %d id %u\n", control_packet->status,
+    DPRINTF("ctrl-in status %d len %d id %"PRIu64"\n", control_packet->status,
             len, id);
 
-    aurb = async_find(dev, id);
-    if (!aurb) {
-        free(data);
-        return;
-    }
-
-    aurb->control_packet.status = control_packet->status;
-    aurb->control_packet.length = control_packet->length;
-    if (memcmp(&aurb->control_packet, control_packet,
-               sizeof(*control_packet))) {
-        ERROR("return control packet mismatch, please report this!\n");
-        len = USB_RET_NAK;
-    }
-
-    if (aurb->packet) {
-        len = usbredir_handle_status(dev, control_packet->status, len);
-        if (len > 0) {
+    /* Fix up USB-3 ep0 maxpacket size to allow superspeed connected devices
+     * to work redirected to a not superspeed capable hcd */
+    if (dev->dev.speed == USB_SPEED_SUPER &&
+            !((dev->dev.port->speedmask & USB_SPEED_MASK_SUPER)) &&
+            control_packet->requesttype == 0x80 &&
+            control_packet->request == 6 &&
+            control_packet->value == 0x100 && control_packet->index == 0 &&
+            data_len >= 18 && data[7] == 9) {
+        data[7] = 64;
+    }
+
+    p = usbredir_find_packet_by_id(dev, 0, id);
+    if (p) {
+        usbredir_handle_status(dev, p, control_packet->status);
+        if (data_len > 0) {
             usbredir_log_data(dev, "ctrl data in:", data, data_len);
-            if (data_len <= sizeof(dev->dev.data_buf)) {
-                memcpy(dev->dev.data_buf, data, data_len);
-            } else {
+            if (data_len > sizeof(dev->dev.data_buf)) {
                 ERROR("ctrl buffer too small (%d > %zu)\n",
                       data_len, sizeof(dev->dev.data_buf));
-                len = USB_RET_STALL;
+                p->status = USB_RET_STALL;
+                data_len = len = sizeof(dev->dev.data_buf);
             }
+            memcpy(dev->dev.data_buf, data, data_len);
         }
-        aurb->packet->result = len;
-        usb_generic_async_ctrl_complete(&dev->dev, aurb->packet);
+        p->actual_length = len;
+        usb_generic_async_ctrl_complete(&dev->dev, p);
     }
-    async_free(dev, aurb);
     free(data);
 }
 
-static void usbredir_bulk_packet(void *priv, uint32_t id,
+static void usbredir_bulk_packet(void *priv, uint64_t id,
     struct usb_redir_bulk_packet_header *bulk_packet,
     uint8_t *data, int data_len)
 {
     USBRedirDevice *dev = priv;
     uint8_t ep = bulk_packet->endpoint;
-    int len = bulk_packet->length;
-    AsyncURB *aurb;
-
-    DPRINTF("bulk-in status %d ep %02X len %d id %u\n", bulk_packet->status,
-            ep, len, id);
-
-    aurb = async_find(dev, id);
-    if (!aurb) {
-        free(data);
-        return;
-    }
+    int len = (bulk_packet->length_high << 16) | bulk_packet->length;
+    USBPacket *p;
 
-    if (aurb->bulk_packet.endpoint != bulk_packet->endpoint ||
-            aurb->bulk_packet.stream_id != bulk_packet->stream_id) {
-        ERROR("return bulk packet mismatch, please report this!\n");
-        len = USB_RET_NAK;
-    }
+    DPRINTF("bulk-in status %d ep %02X len %d id %"PRIu64"\n",
+            bulk_packet->status, ep, len, id);
 
-    if (aurb->packet) {
-        len = usbredir_handle_status(dev, bulk_packet->status, len);
-        if (len > 0) {
+    p = usbredir_find_packet_by_id(dev, ep, id);
+    if (p) {
+        size_t size = (p->combined) ? p->combined->iov.size : p->iov.size;
+        usbredir_handle_status(dev, p, bulk_packet->status);
+        if (data_len > 0) {
             usbredir_log_data(dev, "bulk data in:", data, data_len);
-            if (data_len <= aurb->packet->iov.size) {
-                usb_packet_copy(aurb->packet, data, data_len);
+            if (data_len > size) {
+                ERROR("bulk got more data then requested (%d > %zd)\n",
+                      data_len, p->iov.size);
+                p->status = USB_RET_BABBLE;
+                data_len = len = size;
+            }
+            if (p->combined) {
+                iov_from_buf(p->combined->iov.iov, p->combined->iov.niov,
+                             0, data, data_len);
             } else {
-                ERROR("bulk buffer too small (%d > %zd)\n", data_len,
-                      aurb->packet->iov.size);
-                len = USB_RET_STALL;
+                usb_packet_copy(p, data, data_len);
             }
         }
-        aurb->packet->result = len;
-        usb_packet_complete(&dev->dev, aurb->packet);
+        p->actual_length = len;
+        if (p->pid == USB_TOKEN_IN && p->ep->pipeline) {
+            usb_combined_input_packet_complete(&dev->dev, p);
+        } else {
+            usb_packet_complete(&dev->dev, p);
+        }
     }
-    async_free(dev, aurb);
     free(data);
 }
 
-static void usbredir_iso_packet(void *priv, uint32_t id,
+static void usbredir_iso_packet(void *priv, uint64_t id,
     struct usb_redir_iso_packet_header *iso_packet,
     uint8_t *data, int data_len)
 {
     USBRedirDevice *dev = priv;
     uint8_t ep = iso_packet->endpoint;
 
-    DPRINTF2("iso-in status %d ep %02X len %d id %u\n", iso_packet->status, ep,
-             data_len, id);
+    DPRINTF2("iso-in status %d ep %02X len %d id %"PRIu64"\n",
+             iso_packet->status, ep, data_len, id);
 
     if (dev->endpoint[EP2I(ep)].type != USB_ENDPOINT_XFER_ISOC) {
         ERROR("received iso packet for non iso endpoint %02X\n", ep);
@@ -1416,14 +1621,14 @@ static void usbredir_iso_packet(void *priv, uint32_t id,
     bufp_alloc(dev, data, data_len, iso_packet->status, ep);
 }
 
-static void usbredir_interrupt_packet(void *priv, uint32_t id,
+static void usbredir_interrupt_packet(void *priv, uint64_t id,
     struct usb_redir_interrupt_packet_header *interrupt_packet,
     uint8_t *data, int data_len)
 {
     USBRedirDevice *dev = priv;
     uint8_t ep = interrupt_packet->endpoint;
 
-    DPRINTF("interrupt-in status %d ep %02X len %d id %u\n",
+    DPRINTF("interrupt-in status %d ep %02X len %d id %"PRIu64"\n",
             interrupt_packet->status, ep, data_len, id);
 
     if (dev->endpoint[EP2I(ep)].type != USB_ENDPOINT_XFER_INT) {
@@ -1442,30 +1647,330 @@ static void usbredir_interrupt_packet(void *priv, uint32_t id,
         /* bufp_alloc also adds the packet to the ep queue */
         bufp_alloc(dev, data, data_len, interrupt_packet->status, ep);
     } else {
-        int len = interrupt_packet->length;
-
-        AsyncURB *aurb = async_find(dev, id);
-        if (!aurb) {
-            return;
+        /*
+         * We report output interrupt packets as completed directly upon
+         * submission, so all we can do here if one failed is warn.
+         */
+        if (interrupt_packet->status) {
+            WARNING("interrupt output failed status %d ep %02X id %"PRIu64"\n",
+                    interrupt_packet->status, ep, id);
         }
+    }
+}
 
-        if (aurb->interrupt_packet.endpoint != interrupt_packet->endpoint) {
-            ERROR("return int packet mismatch, please report this!\n");
-            len = USB_RET_NAK;
-        }
+/*
+ * Migration code
+ */
 
-        if (aurb->packet) {
-            aurb->packet->result = usbredir_handle_status(dev,
-                                               interrupt_packet->status, len);
-            usb_packet_complete(&dev->dev, aurb->packet);
-        }
-        async_free(dev, aurb);
+static void usbredir_pre_save(void *priv)
+{
+    USBRedirDevice *dev = priv;
+
+    usbredir_fill_already_in_flight(dev);
+}
+
+static int usbredir_post_load(void *priv, int version_id)
+{
+    USBRedirDevice *dev = priv;
+
+    switch (dev->device_info.speed) {
+    case usb_redir_speed_low:
+        dev->dev.speed = USB_SPEED_LOW;
+        break;
+    case usb_redir_speed_full:
+        dev->dev.speed = USB_SPEED_FULL;
+        break;
+    case usb_redir_speed_high:
+        dev->dev.speed = USB_SPEED_HIGH;
+        break;
+    case usb_redir_speed_super:
+        dev->dev.speed = USB_SPEED_SUPER;
+        break;
+    default:
+        dev->dev.speed = USB_SPEED_FULL;
+    }
+    dev->dev.speedmask = (1 << dev->dev.speed);
+
+    usbredir_setup_usb_eps(dev);
+
+    return 0;
+}
+
+/* For usbredirparser migration */
+static void usbredir_put_parser(QEMUFile *f, void *priv, size_t unused)
+{
+    USBRedirDevice *dev = priv;
+    uint8_t *data;
+    int len;
+
+    if (dev->parser == NULL) {
+        qemu_put_be32(f, 0);
+        return;
+    }
+
+    usbredirparser_serialize(dev->parser, &data, &len);
+    qemu_oom_check(data);
+
+    qemu_put_be32(f, len);
+    qemu_put_buffer(f, data, len);
+
+    free(data);
+}
+
+static int usbredir_get_parser(QEMUFile *f, void *priv, size_t unused)
+{
+    USBRedirDevice *dev = priv;
+    uint8_t *data;
+    int len, ret;
+
+    len = qemu_get_be32(f);
+    if (len == 0) {
+        return 0;
+    }
+
+    /*
+     * If our chardev is not open already at this point the usbredir connection
+     * has been broken (non seamless migration, or restore from disk).
+     *
+     * In this case create a temporary parser to receive the migration data,
+     * and schedule the close_bh to report the device as disconnected to the
+     * guest and to destroy the parser again.
+     */
+    if (dev->parser == NULL) {
+        WARNING("usb-redir connection broken during migration\n");
+        usbredir_create_parser(dev);
+        qemu_bh_schedule(dev->chardev_close_bh);
+    }
+
+    data = g_malloc(len);
+    qemu_get_buffer(f, data, len);
+
+    ret = usbredirparser_unserialize(dev->parser, data, len);
+
+    g_free(data);
+
+    return ret;
+}
+
+static const VMStateInfo usbredir_parser_vmstate_info = {
+    .name = "usb-redir-parser",
+    .put  = usbredir_put_parser,
+    .get  = usbredir_get_parser,
+};
+
+
+/* For buffered packets (iso/irq) queue migration */
+static void usbredir_put_bufpq(QEMUFile *f, void *priv, size_t unused)
+{
+    struct endp_data *endp = priv;
+    struct buf_packet *bufp;
+    int remain = endp->bufpq_size;
+
+    qemu_put_be32(f, endp->bufpq_size);
+    QTAILQ_FOREACH(bufp, &endp->bufpq, next) {
+        qemu_put_be32(f, bufp->len);
+        qemu_put_be32(f, bufp->status);
+        qemu_put_buffer(f, bufp->data, bufp->len);
+        remain--;
+    }
+    assert(remain == 0);
+}
+
+static int usbredir_get_bufpq(QEMUFile *f, void *priv, size_t unused)
+{
+    struct endp_data *endp = priv;
+    struct buf_packet *bufp;
+    int i;
+
+    endp->bufpq_size = qemu_get_be32(f);
+    for (i = 0; i < endp->bufpq_size; i++) {
+        bufp = g_malloc(sizeof(struct buf_packet));
+        bufp->len = qemu_get_be32(f);
+        bufp->status = qemu_get_be32(f);
+        bufp->data = qemu_oom_check(malloc(bufp->len)); /* regular malloc! */
+        qemu_get_buffer(f, bufp->data, bufp->len);
+        QTAILQ_INSERT_TAIL(&endp->bufpq, bufp, next);
+    }
+    return 0;
+}
+
+static const VMStateInfo usbredir_ep_bufpq_vmstate_info = {
+    .name = "usb-redir-bufpq",
+    .put  = usbredir_put_bufpq,
+    .get  = usbredir_get_bufpq,
+};
+
+
+/* For endp_data migration */
+static const VMStateDescription usbredir_ep_vmstate = {
+    .name = "usb-redir-ep",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT8(type, struct endp_data),
+        VMSTATE_UINT8(interval, struct endp_data),
+        VMSTATE_UINT8(interface, struct endp_data),
+        VMSTATE_UINT16(max_packet_size, struct endp_data),
+        VMSTATE_UINT8(iso_started, struct endp_data),
+        VMSTATE_UINT8(iso_error, struct endp_data),
+        VMSTATE_UINT8(interrupt_started, struct endp_data),
+        VMSTATE_UINT8(interrupt_error, struct endp_data),
+        VMSTATE_UINT8(bufpq_prefilled, struct endp_data),
+        VMSTATE_UINT8(bufpq_dropping_packets, struct endp_data),
+        {
+            .name         = "bufpq",
+            .version_id   = 0,
+            .field_exists = NULL,
+            .size         = 0,
+            .info         = &usbredir_ep_bufpq_vmstate_info,
+            .flags        = VMS_SINGLE,
+            .offset       = 0,
+        },
+        VMSTATE_INT32(bufpq_target_size, struct endp_data),
+        VMSTATE_END_OF_LIST()
     }
+};
+
+
+/* For PacketIdQueue migration */
+static void usbredir_put_packet_id_q(QEMUFile *f, void *priv, size_t unused)
+{
+    struct PacketIdQueue *q = priv;
+    USBRedirDevice *dev = q->dev;
+    struct PacketIdQueueEntry *e;
+    int remain = q->size;
+
+    DPRINTF("put_packet_id_q %s size %d\n", q->name, q->size);
+    qemu_put_be32(f, q->size);
+    QTAILQ_FOREACH(e, &q->head, next) {
+        qemu_put_be64(f, e->id);
+        remain--;
+    }
+    assert(remain == 0);
+}
+
+static int usbredir_get_packet_id_q(QEMUFile *f, void *priv, size_t unused)
+{
+    struct PacketIdQueue *q = priv;
+    USBRedirDevice *dev = q->dev;
+    int i, size;
+    uint64_t id;
+
+    size = qemu_get_be32(f);
+    DPRINTF("get_packet_id_q %s size %d\n", q->name, size);
+    for (i = 0; i < size; i++) {
+        id = qemu_get_be64(f);
+        packet_id_queue_add(q, id);
+    }
+    assert(q->size == size);
+    return 0;
 }
 
+static const VMStateInfo usbredir_ep_packet_id_q_vmstate_info = {
+    .name = "usb-redir-packet-id-q",
+    .put  = usbredir_put_packet_id_q,
+    .get  = usbredir_get_packet_id_q,
+};
+
+static const VMStateDescription usbredir_ep_packet_id_queue_vmstate = {
+    .name = "usb-redir-packet-id-queue",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        {
+            .name         = "queue",
+            .version_id   = 0,
+            .field_exists = NULL,
+            .size         = 0,
+            .info         = &usbredir_ep_packet_id_q_vmstate_info,
+            .flags        = VMS_SINGLE,
+            .offset       = 0,
+        },
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+
+/* For usb_redir_device_connect_header migration */
+static const VMStateDescription usbredir_device_info_vmstate = {
+    .name = "usb-redir-device-info",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT8(speed, struct usb_redir_device_connect_header),
+        VMSTATE_UINT8(device_class, struct usb_redir_device_connect_header),
+        VMSTATE_UINT8(device_subclass, struct usb_redir_device_connect_header),
+        VMSTATE_UINT8(device_protocol, struct usb_redir_device_connect_header),
+        VMSTATE_UINT16(vendor_id, struct usb_redir_device_connect_header),
+        VMSTATE_UINT16(product_id, struct usb_redir_device_connect_header),
+        VMSTATE_UINT16(device_version_bcd,
+                       struct usb_redir_device_connect_header),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+
+/* For usb_redir_interface_info_header migration */
+static const VMStateDescription usbredir_interface_info_vmstate = {
+    .name = "usb-redir-interface-info",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_UINT32(interface_count,
+                       struct usb_redir_interface_info_header),
+        VMSTATE_UINT8_ARRAY(interface,
+                            struct usb_redir_interface_info_header, 32),
+        VMSTATE_UINT8_ARRAY(interface_class,
+                            struct usb_redir_interface_info_header, 32),
+        VMSTATE_UINT8_ARRAY(interface_subclass,
+                            struct usb_redir_interface_info_header, 32),
+        VMSTATE_UINT8_ARRAY(interface_protocol,
+                            struct usb_redir_interface_info_header, 32),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+
+/* And finally the USBRedirDevice vmstate itself */
+static const VMStateDescription usbredir_vmstate = {
+    .name = "usb-redir",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .pre_save = usbredir_pre_save,
+    .post_load = usbredir_post_load,
+    .fields = (VMStateField[]) {
+        VMSTATE_USB_DEVICE(dev, USBRedirDevice),
+        VMSTATE_TIMER(attach_timer, USBRedirDevice),
+        {
+            .name         = "parser",
+            .version_id   = 0,
+            .field_exists = NULL,
+            .size         = 0,
+            .info         = &usbredir_parser_vmstate_info,
+            .flags        = VMS_SINGLE,
+            .offset       = 0,
+        },
+        VMSTATE_STRUCT_ARRAY(endpoint, USBRedirDevice, MAX_ENDPOINTS, 1,
+                             usbredir_ep_vmstate, struct endp_data),
+        VMSTATE_STRUCT(cancelled, USBRedirDevice, 1,
+                       usbredir_ep_packet_id_queue_vmstate,
+                       struct PacketIdQueue),
+        VMSTATE_STRUCT(already_in_flight, USBRedirDevice, 1,
+                       usbredir_ep_packet_id_queue_vmstate,
+                       struct PacketIdQueue),
+        VMSTATE_STRUCT(device_info, USBRedirDevice, 1,
+                       usbredir_device_info_vmstate,
+                       struct usb_redir_device_connect_header),
+        VMSTATE_STRUCT(interface_info, USBRedirDevice, 1,
+                       usbredir_interface_info_vmstate,
+                       struct usb_redir_interface_info_header),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
 static Property usbredir_properties[] = {
     DEFINE_PROP_CHR("chardev", USBRedirDevice, cs),
-    DEFINE_PROP_UINT8("debug", USBRedirDevice, debug, 0),
+    DEFINE_PROP_UINT8("debug", USBRedirDevice, debug, usbredirparser_warning),
     DEFINE_PROP_STRING("filter", USBRedirDevice, filter_str),
     DEFINE_PROP_INT32("bootindex", USBRedirDevice, bootindex, -1),
     DEFINE_PROP_END_OF_LIST(),
@@ -1483,6 +1988,8 @@ static void usbredir_class_initfn(ObjectClass *klass, void *data)
     uc->handle_reset   = usbredir_handle_reset;
     uc->handle_data    = usbredir_handle_data;
     uc->handle_control = usbredir_handle_control;
+    uc->flush_ep_queue = usbredir_flush_ep_queue;
+    dc->vmsd           = &usbredir_vmstate;
     dc->props          = usbredir_properties;
 }
 
diff --git a/hw/versatile_i2c.c b/hw/versatile_i2c.c
index 88f530aef..ad71e9d92 100644
--- a/hw/versatile_i2c.c
+++ b/hw/versatile_i2c.c
@@ -32,7 +32,7 @@ typedef struct {
     int in;
 } VersatileI2CState;
 
-static uint64_t versatile_i2c_read(void *opaque, target_phys_addr_t offset,
+static uint64_t versatile_i2c_read(void *opaque, hwaddr offset,
                                    unsigned size)
 {
     VersatileI2CState *s = (VersatileI2CState *)opaque;
@@ -40,12 +40,13 @@ static uint64_t versatile_i2c_read(void *opaque, target_phys_addr_t offset,
     if (offset == 0) {
         return (s->out & 1) | (s->in << 1);
     } else {
-        hw_error("%s: Bad offset 0x%x\n", __func__, (int)offset);
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "%s: Bad offset 0x%x\n", __func__, (int)offset);
         return -1;
     }
 }
 
-static void versatile_i2c_write(void *opaque, target_phys_addr_t offset,
+static void versatile_i2c_write(void *opaque, hwaddr offset,
                                 uint64_t value, unsigned size)
 {
     VersatileI2CState *s = (VersatileI2CState *)opaque;
@@ -58,7 +59,8 @@ static void versatile_i2c_write(void *opaque, target_phys_addr_t offset,
         s->out &= ~value;
         break;
     default:
-        hw_error("%s: Bad offset 0x%x\n", __func__, (int)offset);
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "%s: Bad offset 0x%x\n", __func__, (int)offset);
     }
     bitbang_i2c_set(s->bitbang, BITBANG_I2C_SCL, (s->out & 1) != 0);
     s->in = bitbang_i2c_set(s->bitbang, BITBANG_I2C_SDA, (s->out & 2) != 0);
diff --git a/hw/versatile_pci.c b/hw/versatile_pci.c
index ae53a8b37..e0c3ee36a 100644
--- a/hw/versatile_pci.c
+++ b/hw/versatile_pci.c
@@ -21,18 +21,18 @@ typedef struct {
     MemoryRegion isa;
 } PCIVPBState;
 
-static inline uint32_t vpb_pci_config_addr(target_phys_addr_t addr)
+static inline uint32_t vpb_pci_config_addr(hwaddr addr)
 {
     return addr & 0xffffff;
 }
 
-static void pci_vpb_config_write(void *opaque, target_phys_addr_t addr,
+static void pci_vpb_config_write(void *opaque, hwaddr addr,
                                  uint64_t val, unsigned size)
 {
     pci_data_write(opaque, vpb_pci_config_addr(addr), val, size);
 }
 
-static uint64_t pci_vpb_config_read(void *opaque, target_phys_addr_t addr,
+static uint64_t pci_vpb_config_read(void *opaque, hwaddr addr,
                                     unsigned size)
 {
     uint32_t val;
diff --git a/hw/versatilepb.c b/hw/versatilepb.c
index 7a9203471..25e652b1a 100644
--- a/hw/versatilepb.c
+++ b/hw/versatilepb.c
@@ -81,7 +81,7 @@ static void vpb_sic_set_irq(void *opaque, int irq, int level)
     vpb_sic_update(s);
 }
 
-static uint64_t vpb_sic_read(void *opaque, target_phys_addr_t offset,
+static uint64_t vpb_sic_read(void *opaque, hwaddr offset,
                              unsigned size)
 {
     vpb_sic_state *s = (vpb_sic_state *)opaque;
@@ -103,7 +103,7 @@ static uint64_t vpb_sic_read(void *opaque, target_phys_addr_t offset,
     }
 }
 
-static void vpb_sic_write(void *opaque, target_phys_addr_t offset,
+static void vpb_sic_write(void *opaque, hwaddr offset,
                           uint64_t value, unsigned size)
 {
     vpb_sic_state *s = (vpb_sic_state *)opaque;
@@ -167,11 +167,7 @@ static int vpb_sic_init(SysBusDevice *dev)
 
 static struct arm_boot_info versatile_binfo;
 
-static void versatile_init(ram_addr_t ram_size,
-                     const char *boot_device,
-                     const char *kernel_filename, const char *kernel_cmdline,
-                     const char *initrd_filename, const char *cpu_model,
-                     int board_id)
+static void versatile_init(QEMUMachineInitArgs *args, int board_id)
 {
     ARMCPU *cpu;
     MemoryRegion *sysmem = get_system_memory();
@@ -189,15 +185,15 @@ static void versatile_init(ram_addr_t ram_size,
     int done_smc = 0;
     DriveInfo *dinfo;
 
-    if (!cpu_model) {
-        cpu_model = "arm926";
+    if (!args->cpu_model) {
+        args->cpu_model = "arm926";
     }
-    cpu = cpu_arm_init(cpu_model);
+    cpu = cpu_arm_init(args->cpu_model);
     if (!cpu) {
         fprintf(stderr, "Unable to find CPU definition\n");
         exit(1);
     }
-    memory_region_init_ram(ram, "versatile.ram", ram_size);
+    memory_region_init_ram(ram, "versatile.ram", args->ram_size);
     vmstate_register_ram_global(ram);
     /* ??? RAM should repeat to fill physical memory space.  */
     /* SDRAM at address zero.  */
@@ -211,7 +207,8 @@ static void versatile_init(ram_addr_t ram_size,
 
     cpu_pic = arm_pic_init_cpu(cpu);
     dev = sysbus_create_varargs("pl190", 0x10140000,
-                                cpu_pic[0], cpu_pic[1], NULL);
+                                cpu_pic[ARM_PIC_CPU_IRQ],
+                                cpu_pic[ARM_PIC_CPU_FIQ], NULL);
     for (n = 0; n < 32; n++) {
         pic[n] = qdev_get_gpio_in(dev, n);
     }
@@ -247,7 +244,7 @@ static void versatile_init(ram_addr_t ram_size,
             pci_nic_init_nofail(nd, "rtl8139", NULL);
         }
     }
-    if (usb_enabled) {
+    if (usb_enabled(false)) {
         pci_create_simple(pci_bus, -1, "pci-ohci");
     }
     n = drive_get_max_bus(IF_SCSI);
@@ -265,6 +262,11 @@ static void versatile_init(ram_addr_t ram_size,
     sysbus_create_simple("sp804", 0x101e2000, pic[4]);
     sysbus_create_simple("sp804", 0x101e3000, pic[5]);
 
+    sysbus_create_simple("pl061", 0x101e4000, pic[6]);
+    sysbus_create_simple("pl061", 0x101e5000, pic[7]);
+    sysbus_create_simple("pl061", 0x101e6000, pic[8]);
+    sysbus_create_simple("pl061", 0x101e7000, pic[9]);
+
     /* The versatile/PB actually has a modified Color LCD controller
        that includes hardware cursor support from the PL111.  */
     dev = sysbus_create_simple("pl110_versatile", 0x10120000, pic[16]);
@@ -334,34 +336,22 @@ static void versatile_init(ram_addr_t ram_size,
         fprintf(stderr, "qemu: Error registering flash memory.\n");
     }
 
-    versatile_binfo.ram_size = ram_size;
-    versatile_binfo.kernel_filename = kernel_filename;
-    versatile_binfo.kernel_cmdline = kernel_cmdline;
-    versatile_binfo.initrd_filename = initrd_filename;
+    versatile_binfo.ram_size = args->ram_size;
+    versatile_binfo.kernel_filename = args->kernel_filename;
+    versatile_binfo.kernel_cmdline = args->kernel_cmdline;
+    versatile_binfo.initrd_filename = args->initrd_filename;
     versatile_binfo.board_id = board_id;
     arm_load_kernel(cpu, &versatile_binfo);
 }
 
-static void vpb_init(ram_addr_t ram_size,
-                     const char *boot_device,
-                     const char *kernel_filename, const char *kernel_cmdline,
-                     const char *initrd_filename, const char *cpu_model)
+static void vpb_init(QEMUMachineInitArgs *args)
 {
-    versatile_init(ram_size,
-                   boot_device,
-                   kernel_filename, kernel_cmdline,
-                   initrd_filename, cpu_model, 0x183);
+    versatile_init(args, 0x183);
 }
 
-static void vab_init(ram_addr_t ram_size,
-                     const char *boot_device,
-                     const char *kernel_filename, const char *kernel_cmdline,
-                     const char *initrd_filename, const char *cpu_model)
+static void vab_init(QEMUMachineInitArgs *args)
 {
-    versatile_init(ram_size,
-                   boot_device,
-                   kernel_filename, kernel_cmdline,
-                   initrd_filename, cpu_model, 0x25e);
+    versatile_init(args, 0x25e);
 }
 
 static QEMUMachine versatilepb_machine = {
diff --git a/hw/vexpress.c b/hw/vexpress.c
index b6158447d..d93f057bf 100644
--- a/hw/vexpress.c
+++ b/hw/vexpress.c
@@ -29,8 +29,12 @@
 #include "sysemu.h"
 #include "boards.h"
 #include "exec-memory.h"
+#include "blockdev.h"
+#include "flash.h"
 
 #define VEXPRESS_BOARD_ID 0x8e0
+#define VEXPRESS_FLASH_SIZE (64 * 1024 * 1024)
+#define VEXPRESS_FLASH_SECT_SIZE (256 * 1024)
 
 static struct arm_boot_info vexpress_binfo;
 
@@ -62,7 +66,6 @@ enum {
     VE_COMPACTFLASH,
     VE_CLCD,
     VE_NORFLASH0,
-    VE_NORFLASH0ALIAS,
     VE_NORFLASH1,
     VE_SRAM,
     VE_VIDEORAM,
@@ -71,7 +74,7 @@ enum {
     VE_DAPROM,
 };
 
-static target_phys_addr_t motherboard_legacy_map[] = {
+static hwaddr motherboard_legacy_map[] = {
     /* CS7: 0x10000000 .. 0x10020000 */
     [VE_SYSREGS] = 0x10000000,
     [VE_SP810] = 0x10001000,
@@ -103,10 +106,9 @@ static target_phys_addr_t motherboard_legacy_map[] = {
     [VE_USB] = 0x4f000000,
 };
 
-static target_phys_addr_t motherboard_aseries_map[] = {
-    /* CS0: 0x00000000 .. 0x0c000000 */
-    [VE_NORFLASH0] = 0x00000000,
-    [VE_NORFLASH0ALIAS] = 0x08000000,
+static hwaddr motherboard_aseries_map[] = {
+    /* CS0: 0x08000000 .. 0x0c000000 */
+    [VE_NORFLASH0] = 0x08000000,
     /* CS4: 0x0c000000 .. 0x10000000 */
     [VE_NORFLASH1] = 0x0c000000,
     /* CS5: 0x10000000 .. 0x14000000 */
@@ -148,9 +150,9 @@ typedef void DBoardInitFn(const VEDBoardInfo *daughterboard,
                           qemu_irq *pic, uint32_t *proc_id);
 
 struct VEDBoardInfo {
-    const target_phys_addr_t *motherboard_map;
-    target_phys_addr_t loader_start;
-    const target_phys_addr_t gic_cpu_if_addr;
+    const hwaddr *motherboard_map;
+    hwaddr loader_start;
+    const hwaddr gic_cpu_if_addr;
     DBoardInitFn *init;
 };
 
@@ -346,24 +348,21 @@ static const VEDBoardInfo a15_daughterboard = {
 };
 
 static void vexpress_common_init(const VEDBoardInfo *daughterboard,
-                                 ram_addr_t ram_size,
-                                 const char *boot_device,
-                                 const char *kernel_filename,
-                                 const char *kernel_cmdline,
-                                 const char *initrd_filename,
-                                 const char *cpu_model)
+                                 QEMUMachineInitArgs *args)
 {
     DeviceState *dev, *sysctl, *pl041;
     qemu_irq pic[64];
     uint32_t proc_id;
     uint32_t sys_id;
+    DriveInfo *dinfo;
     ram_addr_t vram_size, sram_size;
     MemoryRegion *sysmem = get_system_memory();
     MemoryRegion *vram = g_new(MemoryRegion, 1);
     MemoryRegion *sram = g_new(MemoryRegion, 1);
-    const target_phys_addr_t *map = daughterboard->motherboard_map;
+    const hwaddr *map = daughterboard->motherboard_map;
 
-    daughterboard->init(daughterboard, ram_size, cpu_model, pic, &proc_id);
+    daughterboard->init(daughterboard, args->ram_size, args->cpu_model,
+                        pic, &proc_id);
 
     /* Motherboard peripherals: the wiring is the same but the
      * addresses vary between the legacy and A-Series memory maps.
@@ -412,9 +411,25 @@ static void vexpress_common_init(const VEDBoardInfo *daughterboard,
 
     sysbus_create_simple("pl111", map[VE_CLCD], pic[14]);
 
-    /* VE_NORFLASH0: not modelled */
-    /* VE_NORFLASH0ALIAS: not modelled */
-    /* VE_NORFLASH1: not modelled */
+    dinfo = drive_get_next(IF_PFLASH);
+    if (!pflash_cfi01_register(map[VE_NORFLASH0], NULL, "vexpress.flash0",
+            VEXPRESS_FLASH_SIZE, dinfo ? dinfo->bdrv : NULL,
+            VEXPRESS_FLASH_SECT_SIZE,
+            VEXPRESS_FLASH_SIZE / VEXPRESS_FLASH_SECT_SIZE, 4,
+            0x00, 0x89, 0x00, 0x18, 0)) {
+        fprintf(stderr, "vexpress: error registering flash 0.\n");
+        exit(1);
+    }
+
+    dinfo = drive_get_next(IF_PFLASH);
+    if (!pflash_cfi01_register(map[VE_NORFLASH1], NULL, "vexpress.flash1",
+            VEXPRESS_FLASH_SIZE, dinfo ? dinfo->bdrv : NULL,
+            VEXPRESS_FLASH_SECT_SIZE,
+            VEXPRESS_FLASH_SIZE / VEXPRESS_FLASH_SECT_SIZE, 4,
+            0x00, 0x89, 0x00, 0x18, 0)) {
+        fprintf(stderr, "vexpress: error registering flash 1.\n");
+        exit(1);
+    }
 
     sram_size = 0x2000000;
     memory_region_init_ram(sram, "vexpress.sram", sram_size);
@@ -435,10 +450,10 @@ static void vexpress_common_init(const VEDBoardInfo *daughterboard,
 
     /* VE_DAPROM: not modelled */
 
-    vexpress_binfo.ram_size = ram_size;
-    vexpress_binfo.kernel_filename = kernel_filename;
-    vexpress_binfo.kernel_cmdline = kernel_cmdline;
-    vexpress_binfo.initrd_filename = initrd_filename;
+    vexpress_binfo.ram_size = args->ram_size;
+    vexpress_binfo.kernel_filename = args->kernel_filename;
+    vexpress_binfo.kernel_cmdline = args->kernel_cmdline;
+    vexpress_binfo.initrd_filename = args->initrd_filename;
     vexpress_binfo.nb_cpus = smp_cpus;
     vexpress_binfo.board_id = VEXPRESS_BOARD_ID;
     vexpress_binfo.loader_start = daughterboard->loader_start;
@@ -448,28 +463,14 @@ static void vexpress_common_init(const VEDBoardInfo *daughterboard,
     arm_load_kernel(arm_env_get_cpu(first_cpu), &vexpress_binfo);
 }
 
-static void vexpress_a9_init(ram_addr_t ram_size,
-                             const char *boot_device,
-                             const char *kernel_filename,
-                             const char *kernel_cmdline,
-                             const char *initrd_filename,
-                             const char *cpu_model)
+static void vexpress_a9_init(QEMUMachineInitArgs *args)
 {
-    vexpress_common_init(&a9_daughterboard,
-                         ram_size, boot_device, kernel_filename,
-                         kernel_cmdline, initrd_filename, cpu_model);
+    vexpress_common_init(&a9_daughterboard, args);
 }
 
-static void vexpress_a15_init(ram_addr_t ram_size,
-                              const char *boot_device,
-                              const char *kernel_filename,
-                              const char *kernel_cmdline,
-                              const char *initrd_filename,
-                              const char *cpu_model)
+static void vexpress_a15_init(QEMUMachineInitArgs *args)
 {
-    vexpress_common_init(&a15_daughterboard,
-                         ram_size, boot_device, kernel_filename,
-                         kernel_cmdline, initrd_filename, cpu_model);
+    vexpress_common_init(&a15_daughterboard, args);
 }
 
 static QEMUMachine vexpress_a9_machine = {
diff --git a/hw/vfio_pci.c b/hw/vfio_pci.c
new file mode 100644
index 000000000..7c27834e0
--- /dev/null
+++ b/hw/vfio_pci.c
@@ -0,0 +1,2115 @@
+/*
+ * vfio based device assignment support
+ *
+ * Copyright Red Hat, Inc. 2012
+ *
+ * Authors:
+ *  Alex Williamson <alex.williamson@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ *
+ * Based on qemu-kvm device-assignment:
+ *  Adapted for KVM by Qumranet.
+ *  Copyright (c) 2007, Neocleus, Alex Novik (alex@neocleus.com)
+ *  Copyright (c) 2007, Neocleus, Guy Zana (guy@neocleus.com)
+ *  Copyright (C) 2008, Qumranet, Amit Shah (amit.shah@qumranet.com)
+ *  Copyright (C) 2008, Red Hat, Amit Shah (amit.shah@redhat.com)
+ *  Copyright (C) 2008, IBM, Muli Ben-Yehuda (muli@il.ibm.com)
+ */
+
+#include <dirent.h>
+#include <unistd.h>
+#include <sys/ioctl.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <linux/vfio.h>
+
+#include "config.h"
+#include "event_notifier.h"
+#include "exec-memory.h"
+#include "kvm.h"
+#include "memory.h"
+#include "msi.h"
+#include "msix.h"
+#include "pci.h"
+#include "qemu-common.h"
+#include "qemu-error.h"
+#include "qemu-queue.h"
+#include "range.h"
+
+/* #define DEBUG_VFIO */
+#ifdef DEBUG_VFIO
+#define DPRINTF(fmt, ...) \
+    do { fprintf(stderr, "vfio: " fmt, ## __VA_ARGS__); } while (0)
+#else
+#define DPRINTF(fmt, ...) \
+    do { } while (0)
+#endif
+
+typedef struct VFIOBAR {
+    off_t fd_offset; /* offset of BAR within device fd */
+    int fd; /* device fd, allows us to pass VFIOBAR as opaque data */
+    MemoryRegion mem; /* slow, read/write access */
+    MemoryRegion mmap_mem; /* direct mapped access */
+    void *mmap;
+    size_t size;
+    uint32_t flags; /* VFIO region flags (rd/wr/mmap) */
+    uint8_t nr; /* cache the BAR number for debug */
+} VFIOBAR;
+
+typedef struct VFIOINTx {
+    bool pending; /* interrupt pending */
+    bool kvm_accel; /* set when QEMU bypass through KVM enabled */
+    uint8_t pin; /* which pin to pull for qemu_set_irq */
+    EventNotifier interrupt; /* eventfd triggered on interrupt */
+    EventNotifier unmask; /* eventfd for unmask on QEMU bypass */
+    PCIINTxRoute route; /* routing info for QEMU bypass */
+    uint32_t mmap_timeout; /* delay to re-enable mmaps after interrupt */
+    QEMUTimer *mmap_timer; /* enable mmaps after periods w/o interrupts */
+} VFIOINTx;
+
+struct VFIODevice;
+
+typedef struct VFIOMSIVector {
+    EventNotifier interrupt; /* eventfd triggered on interrupt */
+    struct VFIODevice *vdev; /* back pointer to device */
+    int virq; /* KVM irqchip route for QEMU bypass */
+    bool use;
+} VFIOMSIVector;
+
+enum {
+    VFIO_INT_NONE = 0,
+    VFIO_INT_INTx = 1,
+    VFIO_INT_MSI  = 2,
+    VFIO_INT_MSIX = 3,
+};
+
+struct VFIOGroup;
+
+typedef struct VFIOContainer {
+    int fd; /* /dev/vfio/vfio, empowered by the attached groups */
+    struct {
+        /* enable abstraction to support various iommu backends */
+        union {
+            MemoryListener listener; /* Used by type1 iommu */
+        };
+        void (*release)(struct VFIOContainer *);
+    } iommu_data;
+    QLIST_HEAD(, VFIOGroup) group_list;
+    QLIST_ENTRY(VFIOContainer) next;
+} VFIOContainer;
+
+/* Cache of MSI-X setup plus extra mmap and memory region for split BAR map */
+typedef struct VFIOMSIXInfo {
+    uint8_t table_bar;
+    uint8_t pba_bar;
+    uint16_t entries;
+    uint32_t table_offset;
+    uint32_t pba_offset;
+    MemoryRegion mmap_mem;
+    void *mmap;
+} VFIOMSIXInfo;
+
+typedef struct VFIODevice {
+    PCIDevice pdev;
+    int fd;
+    VFIOINTx intx;
+    unsigned int config_size;
+    off_t config_offset; /* Offset of config space region within device fd */
+    unsigned int rom_size;
+    off_t rom_offset; /* Offset of ROM region within device fd */
+    int msi_cap_size;
+    VFIOMSIVector *msi_vectors;
+    VFIOMSIXInfo *msix;
+    int nr_vectors; /* Number of MSI/MSIX vectors currently in use */
+    int interrupt; /* Current interrupt type */
+    VFIOBAR bars[PCI_NUM_REGIONS - 1]; /* No ROM */
+    PCIHostDeviceAddress host;
+    QLIST_ENTRY(VFIODevice) next;
+    struct VFIOGroup *group;
+    bool reset_works;
+} VFIODevice;
+
+typedef struct VFIOGroup {
+    int fd;
+    int groupid;
+    VFIOContainer *container;
+    QLIST_HEAD(, VFIODevice) device_list;
+    QLIST_ENTRY(VFIOGroup) next;
+    QLIST_ENTRY(VFIOGroup) container_next;
+} VFIOGroup;
+
+#define MSIX_CAP_LENGTH 12
+
+static QLIST_HEAD(, VFIOContainer)
+    container_list = QLIST_HEAD_INITIALIZER(container_list);
+
+static QLIST_HEAD(, VFIOGroup)
+    group_list = QLIST_HEAD_INITIALIZER(group_list);
+
+static void vfio_disable_interrupts(VFIODevice *vdev);
+static uint32_t vfio_pci_read_config(PCIDevice *pdev, uint32_t addr, int len);
+static void vfio_mmap_set_enabled(VFIODevice *vdev, bool enabled);
+
+/*
+ * Common VFIO interrupt disable
+ */
+static void vfio_disable_irqindex(VFIODevice *vdev, int index)
+{
+    struct vfio_irq_set irq_set = {
+        .argsz = sizeof(irq_set),
+        .flags = VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_TRIGGER,
+        .index = index,
+        .start = 0,
+        .count = 0,
+    };
+
+    ioctl(vdev->fd, VFIO_DEVICE_SET_IRQS, &irq_set);
+}
+
+/*
+ * INTx
+ */
+static void vfio_unmask_intx(VFIODevice *vdev)
+{
+    struct vfio_irq_set irq_set = {
+        .argsz = sizeof(irq_set),
+        .flags = VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_UNMASK,
+        .index = VFIO_PCI_INTX_IRQ_INDEX,
+        .start = 0,
+        .count = 1,
+    };
+
+    ioctl(vdev->fd, VFIO_DEVICE_SET_IRQS, &irq_set);
+}
+
+#ifdef CONFIG_KVM /* Unused outside of CONFIG_KVM code */
+static void vfio_mask_intx(VFIODevice *vdev)
+{
+    struct vfio_irq_set irq_set = {
+        .argsz = sizeof(irq_set),
+        .flags = VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_MASK,
+        .index = VFIO_PCI_INTX_IRQ_INDEX,
+        .start = 0,
+        .count = 1,
+    };
+
+    ioctl(vdev->fd, VFIO_DEVICE_SET_IRQS, &irq_set);
+}
+#endif
+
+/*
+ * Disabling BAR mmaping can be slow, but toggling it around INTx can
+ * also be a huge overhead.  We try to get the best of both worlds by
+ * waiting until an interrupt to disable mmaps (subsequent transitions
+ * to the same state are effectively no overhead).  If the interrupt has
+ * been serviced and the time gap is long enough, we re-enable mmaps for
+ * performance.  This works well for things like graphics cards, which
+ * may not use their interrupt at all and are penalized to an unusable
+ * level by read/write BAR traps.  Other devices, like NICs, have more
+ * regular interrupts and see much better latency by staying in non-mmap
+ * mode.  We therefore set the default mmap_timeout such that a ping
+ * is just enough to keep the mmap disabled.  Users can experiment with
+ * other options with the x-intx-mmap-timeout-ms parameter (a value of
+ * zero disables the timer).
+ */
+static void vfio_intx_mmap_enable(void *opaque)
+{
+    VFIODevice *vdev = opaque;
+
+    if (vdev->intx.pending) {
+        qemu_mod_timer(vdev->intx.mmap_timer,
+                       qemu_get_clock_ms(vm_clock) + vdev->intx.mmap_timeout);
+        return;
+    }
+
+    vfio_mmap_set_enabled(vdev, true);
+}
+
+static void vfio_intx_interrupt(void *opaque)
+{
+    VFIODevice *vdev = opaque;
+
+    if (!event_notifier_test_and_clear(&vdev->intx.interrupt)) {
+        return;
+    }
+
+    DPRINTF("%s(%04x:%02x:%02x.%x) Pin %c\n", __func__, vdev->host.domain,
+            vdev->host.bus, vdev->host.slot, vdev->host.function,
+            'A' + vdev->intx.pin);
+
+    vdev->intx.pending = true;
+    qemu_set_irq(vdev->pdev.irq[vdev->intx.pin], 1);
+    vfio_mmap_set_enabled(vdev, false);
+    if (vdev->intx.mmap_timeout) {
+        qemu_mod_timer(vdev->intx.mmap_timer,
+                       qemu_get_clock_ms(vm_clock) + vdev->intx.mmap_timeout);
+    }
+}
+
+static void vfio_eoi(VFIODevice *vdev)
+{
+    if (!vdev->intx.pending) {
+        return;
+    }
+
+    DPRINTF("%s(%04x:%02x:%02x.%x) EOI\n", __func__, vdev->host.domain,
+            vdev->host.bus, vdev->host.slot, vdev->host.function);
+
+    vdev->intx.pending = false;
+    qemu_set_irq(vdev->pdev.irq[vdev->intx.pin], 0);
+    vfio_unmask_intx(vdev);
+}
+
+static void vfio_enable_intx_kvm(VFIODevice *vdev)
+{
+#ifdef CONFIG_KVM
+    struct kvm_irqfd irqfd = {
+        .fd = event_notifier_get_fd(&vdev->intx.interrupt),
+        .gsi = vdev->intx.route.irq,
+        .flags = KVM_IRQFD_FLAG_RESAMPLE,
+    };
+    struct vfio_irq_set *irq_set;
+    int ret, argsz;
+    int32_t *pfd;
+
+    if (!kvm_irqchip_in_kernel() ||
+        vdev->intx.route.mode != PCI_INTX_ENABLED ||
+        !kvm_check_extension(kvm_state, KVM_CAP_IRQFD_RESAMPLE)) {
+        return;
+    }
+
+    /* Get to a known interrupt state */
+    qemu_set_fd_handler(irqfd.fd, NULL, NULL, vdev);
+    vfio_mask_intx(vdev);
+    vdev->intx.pending = false;
+    qemu_set_irq(vdev->pdev.irq[vdev->intx.pin], 0);
+
+    /* Get an eventfd for resample/unmask */
+    if (event_notifier_init(&vdev->intx.unmask, 0)) {
+        error_report("vfio: Error: event_notifier_init failed eoi\n");
+        goto fail;
+    }
+
+    /* KVM triggers it, VFIO listens for it */
+    irqfd.resamplefd = event_notifier_get_fd(&vdev->intx.unmask);
+
+    if (kvm_vm_ioctl(kvm_state, KVM_IRQFD, &irqfd)) {
+        error_report("vfio: Error: Failed to setup resample irqfd: %m\n");
+        goto fail_irqfd;
+    }
+
+    argsz = sizeof(*irq_set) + sizeof(*pfd);
+
+    irq_set = g_malloc0(argsz);
+    irq_set->argsz = argsz;
+    irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_UNMASK;
+    irq_set->index = VFIO_PCI_INTX_IRQ_INDEX;
+    irq_set->start = 0;
+    irq_set->count = 1;
+    pfd = (int32_t *)&irq_set->data;
+
+    *pfd = irqfd.resamplefd;
+
+    ret = ioctl(vdev->fd, VFIO_DEVICE_SET_IRQS, irq_set);
+    g_free(irq_set);
+    if (ret) {
+        error_report("vfio: Error: Failed to setup INTx unmask fd: %m\n");
+        goto fail_vfio;
+    }
+
+    /* Let'em rip */
+    vfio_unmask_intx(vdev);
+
+    vdev->intx.kvm_accel = true;
+
+    DPRINTF("%s(%04x:%02x:%02x.%x) KVM INTx accel enabled\n",
+            __func__, vdev->host.domain, vdev->host.bus,
+            vdev->host.slot, vdev->host.function);
+
+    return;
+
+fail_vfio:
+    irqfd.flags = KVM_IRQFD_FLAG_DEASSIGN;
+    kvm_vm_ioctl(kvm_state, KVM_IRQFD, &irqfd);
+fail_irqfd:
+    event_notifier_cleanup(&vdev->intx.unmask);
+fail:
+    qemu_set_fd_handler(irqfd.fd, vfio_intx_interrupt, NULL, vdev);
+    vfio_unmask_intx(vdev);
+#endif
+}
+
+static void vfio_disable_intx_kvm(VFIODevice *vdev)
+{
+#ifdef CONFIG_KVM
+    struct kvm_irqfd irqfd = {
+        .fd = event_notifier_get_fd(&vdev->intx.interrupt),
+        .gsi = vdev->intx.route.irq,
+        .flags = KVM_IRQFD_FLAG_DEASSIGN,
+    };
+
+    if (!vdev->intx.kvm_accel) {
+        return;
+    }
+
+    /*
+     * Get to a known state, hardware masked, QEMU ready to accept new
+     * interrupts, QEMU IRQ de-asserted.
+     */
+    vfio_mask_intx(vdev);
+    vdev->intx.pending = false;
+    qemu_set_irq(vdev->pdev.irq[vdev->intx.pin], 0);
+
+    /* Tell KVM to stop listening for an INTx irqfd */
+    if (kvm_vm_ioctl(kvm_state, KVM_IRQFD, &irqfd)) {
+        error_report("vfio: Error: Failed to disable INTx irqfd: %m\n");
+    }
+
+    /* We only need to close the eventfd for VFIO to cleanup the kernel side */
+    event_notifier_cleanup(&vdev->intx.unmask);
+
+    /* QEMU starts listening for interrupt events. */
+    qemu_set_fd_handler(irqfd.fd, vfio_intx_interrupt, NULL, vdev);
+
+    vdev->intx.kvm_accel = false;
+
+    /* If we've missed an event, let it re-fire through QEMU */
+    vfio_unmask_intx(vdev);
+
+    DPRINTF("%s(%04x:%02x:%02x.%x) KVM INTx accel disabled\n",
+            __func__, vdev->host.domain, vdev->host.bus,
+            vdev->host.slot, vdev->host.function);
+#endif
+}
+
+static void vfio_update_irq(PCIDevice *pdev)
+{
+    VFIODevice *vdev = DO_UPCAST(VFIODevice, pdev, pdev);
+    PCIINTxRoute route;
+
+    if (vdev->interrupt != VFIO_INT_INTx) {
+        return;
+    }
+
+    route = pci_device_route_intx_to_irq(&vdev->pdev, vdev->intx.pin);
+
+    if (!pci_intx_route_changed(&vdev->intx.route, &route)) {
+        return; /* Nothing changed */
+    }
+
+    DPRINTF("%s(%04x:%02x:%02x.%x) IRQ moved %d -> %d\n", __func__,
+            vdev->host.domain, vdev->host.bus, vdev->host.slot,
+            vdev->host.function, vdev->intx.route.irq, route.irq);
+
+    vfio_disable_intx_kvm(vdev);
+
+    vdev->intx.route = route;
+
+    if (route.mode != PCI_INTX_ENABLED) {
+        return;
+    }
+
+    vfio_enable_intx_kvm(vdev);
+
+    /* Re-enable the interrupt in cased we missed an EOI */
+    vfio_eoi(vdev);
+}
+
+static int vfio_enable_intx(VFIODevice *vdev)
+{
+    uint8_t pin = vfio_pci_read_config(&vdev->pdev, PCI_INTERRUPT_PIN, 1);
+    int ret, argsz;
+    struct vfio_irq_set *irq_set;
+    int32_t *pfd;
+
+    if (!pin) {
+        return 0;
+    }
+
+    vfio_disable_interrupts(vdev);
+
+    vdev->intx.pin = pin - 1; /* Pin A (1) -> irq[0] */
+
+#ifdef CONFIG_KVM
+    /*
+     * Only conditional to avoid generating error messages on platforms
+     * where we won't actually use the result anyway.
+     */
+    if (kvm_check_extension(kvm_state, KVM_CAP_IRQFD_RESAMPLE)) {
+        vdev->intx.route = pci_device_route_intx_to_irq(&vdev->pdev,
+                                                        vdev->intx.pin);
+    }
+#endif
+
+    ret = event_notifier_init(&vdev->intx.interrupt, 0);
+    if (ret) {
+        error_report("vfio: Error: event_notifier_init failed\n");
+        return ret;
+    }
+
+    argsz = sizeof(*irq_set) + sizeof(*pfd);
+
+    irq_set = g_malloc0(argsz);
+    irq_set->argsz = argsz;
+    irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_TRIGGER;
+    irq_set->index = VFIO_PCI_INTX_IRQ_INDEX;
+    irq_set->start = 0;
+    irq_set->count = 1;
+    pfd = (int32_t *)&irq_set->data;
+
+    *pfd = event_notifier_get_fd(&vdev->intx.interrupt);
+    qemu_set_fd_handler(*pfd, vfio_intx_interrupt, NULL, vdev);
+
+    ret = ioctl(vdev->fd, VFIO_DEVICE_SET_IRQS, irq_set);
+    g_free(irq_set);
+    if (ret) {
+        error_report("vfio: Error: Failed to setup INTx fd: %m\n");
+        qemu_set_fd_handler(*pfd, NULL, NULL, vdev);
+        event_notifier_cleanup(&vdev->intx.interrupt);
+        return -errno;
+    }
+
+    vfio_enable_intx_kvm(vdev);
+
+    vdev->interrupt = VFIO_INT_INTx;
+
+    DPRINTF("%s(%04x:%02x:%02x.%x)\n", __func__, vdev->host.domain,
+            vdev->host.bus, vdev->host.slot, vdev->host.function);
+
+    return 0;
+}
+
+static void vfio_disable_intx(VFIODevice *vdev)
+{
+    int fd;
+
+    qemu_del_timer(vdev->intx.mmap_timer);
+    vfio_disable_intx_kvm(vdev);
+    vfio_disable_irqindex(vdev, VFIO_PCI_INTX_IRQ_INDEX);
+    vdev->intx.pending = false;
+    qemu_set_irq(vdev->pdev.irq[vdev->intx.pin], 0);
+    vfio_mmap_set_enabled(vdev, true);
+
+    fd = event_notifier_get_fd(&vdev->intx.interrupt);
+    qemu_set_fd_handler(fd, NULL, NULL, vdev);
+    event_notifier_cleanup(&vdev->intx.interrupt);
+
+    vdev->interrupt = VFIO_INT_NONE;
+
+    DPRINTF("%s(%04x:%02x:%02x.%x)\n", __func__, vdev->host.domain,
+            vdev->host.bus, vdev->host.slot, vdev->host.function);
+}
+
+/*
+ * MSI/X
+ */
+static void vfio_msi_interrupt(void *opaque)
+{
+    VFIOMSIVector *vector = opaque;
+    VFIODevice *vdev = vector->vdev;
+    int nr = vector - vdev->msi_vectors;
+
+    if (!event_notifier_test_and_clear(&vector->interrupt)) {
+        return;
+    }
+
+    DPRINTF("%s(%04x:%02x:%02x.%x) vector %d\n", __func__,
+            vdev->host.domain, vdev->host.bus, vdev->host.slot,
+            vdev->host.function, nr);
+
+    if (vdev->interrupt == VFIO_INT_MSIX) {
+        msix_notify(&vdev->pdev, nr);
+    } else if (vdev->interrupt == VFIO_INT_MSI) {
+        msi_notify(&vdev->pdev, nr);
+    } else {
+        error_report("vfio: MSI interrupt receieved, but not enabled?\n");
+    }
+}
+
+static int vfio_enable_vectors(VFIODevice *vdev, bool msix)
+{
+    struct vfio_irq_set *irq_set;
+    int ret = 0, i, argsz;
+    int32_t *fds;
+
+    argsz = sizeof(*irq_set) + (vdev->nr_vectors * sizeof(*fds));
+
+    irq_set = g_malloc0(argsz);
+    irq_set->argsz = argsz;
+    irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_TRIGGER;
+    irq_set->index = msix ? VFIO_PCI_MSIX_IRQ_INDEX : VFIO_PCI_MSI_IRQ_INDEX;
+    irq_set->start = 0;
+    irq_set->count = vdev->nr_vectors;
+    fds = (int32_t *)&irq_set->data;
+
+    for (i = 0; i < vdev->nr_vectors; i++) {
+        if (!vdev->msi_vectors[i].use) {
+            fds[i] = -1;
+            continue;
+        }
+
+        fds[i] = event_notifier_get_fd(&vdev->msi_vectors[i].interrupt);
+    }
+
+    ret = ioctl(vdev->fd, VFIO_DEVICE_SET_IRQS, irq_set);
+
+    g_free(irq_set);
+
+    return ret;
+}
+
+static int vfio_msix_vector_use(PCIDevice *pdev,
+                                unsigned int nr, MSIMessage msg)
+{
+    VFIODevice *vdev = DO_UPCAST(VFIODevice, pdev, pdev);
+    VFIOMSIVector *vector;
+    int ret;
+
+    DPRINTF("%s(%04x:%02x:%02x.%x) vector %d used\n", __func__,
+            vdev->host.domain, vdev->host.bus, vdev->host.slot,
+            vdev->host.function, nr);
+
+    vector = &vdev->msi_vectors[nr];
+    vector->vdev = vdev;
+    vector->use = true;
+
+    msix_vector_use(pdev, nr);
+
+    if (event_notifier_init(&vector->interrupt, 0)) {
+        error_report("vfio: Error: event_notifier_init failed\n");
+    }
+
+    /*
+     * Attempt to enable route through KVM irqchip,
+     * default to userspace handling if unavailable.
+     */
+    vector->virq = kvm_irqchip_add_msi_route(kvm_state, msg);
+    if (vector->virq < 0 ||
+        kvm_irqchip_add_irqfd_notifier(kvm_state, &vector->interrupt,
+                                       vector->virq) < 0) {
+        if (vector->virq >= 0) {
+            kvm_irqchip_release_virq(kvm_state, vector->virq);
+            vector->virq = -1;
+        }
+        qemu_set_fd_handler(event_notifier_get_fd(&vector->interrupt),
+                            vfio_msi_interrupt, NULL, vector);
+    }
+
+    /*
+     * We don't want to have the host allocate all possible MSI vectors
+     * for a device if they're not in use, so we shutdown and incrementally
+     * increase them as needed.
+     */
+    if (vdev->nr_vectors < nr + 1) {
+        vfio_disable_irqindex(vdev, VFIO_PCI_MSIX_IRQ_INDEX);
+        vdev->nr_vectors = nr + 1;
+        ret = vfio_enable_vectors(vdev, true);
+        if (ret) {
+            error_report("vfio: failed to enable vectors, %d\n", ret);
+        }
+    } else {
+        int argsz;
+        struct vfio_irq_set *irq_set;
+        int32_t *pfd;
+
+        argsz = sizeof(*irq_set) + sizeof(*pfd);
+
+        irq_set = g_malloc0(argsz);
+        irq_set->argsz = argsz;
+        irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD |
+                         VFIO_IRQ_SET_ACTION_TRIGGER;
+        irq_set->index = VFIO_PCI_MSIX_IRQ_INDEX;
+        irq_set->start = nr;
+        irq_set->count = 1;
+        pfd = (int32_t *)&irq_set->data;
+
+        *pfd = event_notifier_get_fd(&vector->interrupt);
+
+        ret = ioctl(vdev->fd, VFIO_DEVICE_SET_IRQS, irq_set);
+        g_free(irq_set);
+        if (ret) {
+            error_report("vfio: failed to modify vector, %d\n", ret);
+        }
+    }
+
+    return 0;
+}
+
+static void vfio_msix_vector_release(PCIDevice *pdev, unsigned int nr)
+{
+    VFIODevice *vdev = DO_UPCAST(VFIODevice, pdev, pdev);
+    VFIOMSIVector *vector = &vdev->msi_vectors[nr];
+    int argsz;
+    struct vfio_irq_set *irq_set;
+    int32_t *pfd;
+
+    DPRINTF("%s(%04x:%02x:%02x.%x) vector %d released\n", __func__,
+            vdev->host.domain, vdev->host.bus, vdev->host.slot,
+            vdev->host.function, nr);
+
+    /*
+     * XXX What's the right thing to do here?  This turns off the interrupt
+     * completely, but do we really just want to switch the interrupt to
+     * bouncing through userspace and let msix.c drop it?  Not sure.
+     */
+    msix_vector_unuse(pdev, nr);
+
+    argsz = sizeof(*irq_set) + sizeof(*pfd);
+
+    irq_set = g_malloc0(argsz);
+    irq_set->argsz = argsz;
+    irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD |
+                     VFIO_IRQ_SET_ACTION_TRIGGER;
+    irq_set->index = VFIO_PCI_MSIX_IRQ_INDEX;
+    irq_set->start = nr;
+    irq_set->count = 1;
+    pfd = (int32_t *)&irq_set->data;
+
+    *pfd = -1;
+
+    ioctl(vdev->fd, VFIO_DEVICE_SET_IRQS, irq_set);
+
+    g_free(irq_set);
+
+    if (vector->virq < 0) {
+        qemu_set_fd_handler(event_notifier_get_fd(&vector->interrupt),
+                            NULL, NULL, NULL);
+    } else {
+        kvm_irqchip_remove_irqfd_notifier(kvm_state, &vector->interrupt,
+                                          vector->virq);
+        kvm_irqchip_release_virq(kvm_state, vector->virq);
+        vector->virq = -1;
+    }
+
+    event_notifier_cleanup(&vector->interrupt);
+    vector->use = false;
+}
+
+static void vfio_enable_msix(VFIODevice *vdev)
+{
+    vfio_disable_interrupts(vdev);
+
+    vdev->msi_vectors = g_malloc0(vdev->msix->entries * sizeof(VFIOMSIVector));
+
+    vdev->interrupt = VFIO_INT_MSIX;
+
+    if (msix_set_vector_notifiers(&vdev->pdev, vfio_msix_vector_use,
+                                  vfio_msix_vector_release)) {
+        error_report("vfio: msix_set_vector_notifiers failed\n");
+    }
+
+    DPRINTF("%s(%04x:%02x:%02x.%x)\n", __func__, vdev->host.domain,
+            vdev->host.bus, vdev->host.slot, vdev->host.function);
+}
+
+static void vfio_enable_msi(VFIODevice *vdev)
+{
+    int ret, i;
+
+    vfio_disable_interrupts(vdev);
+
+    vdev->nr_vectors = msi_nr_vectors_allocated(&vdev->pdev);
+retry:
+    vdev->msi_vectors = g_malloc0(vdev->nr_vectors * sizeof(VFIOMSIVector));
+
+    for (i = 0; i < vdev->nr_vectors; i++) {
+        MSIMessage msg;
+        VFIOMSIVector *vector = &vdev->msi_vectors[i];
+
+        vector->vdev = vdev;
+        vector->use = true;
+
+        if (event_notifier_init(&vector->interrupt, 0)) {
+            error_report("vfio: Error: event_notifier_init failed\n");
+        }
+
+        msg = msi_get_message(&vdev->pdev, i);
+
+        /*
+         * Attempt to enable route through KVM irqchip,
+         * default to userspace handling if unavailable.
+         */
+        vector->virq = kvm_irqchip_add_msi_route(kvm_state, msg);
+        if (vector->virq < 0 ||
+            kvm_irqchip_add_irqfd_notifier(kvm_state, &vector->interrupt,
+                                           vector->virq) < 0) {
+            qemu_set_fd_handler(event_notifier_get_fd(&vector->interrupt),
+                                vfio_msi_interrupt, NULL, vector);
+        }
+    }
+
+    ret = vfio_enable_vectors(vdev, false);
+    if (ret) {
+        if (ret < 0) {
+            error_report("vfio: Error: Failed to setup MSI fds: %m\n");
+        } else if (ret != vdev->nr_vectors) {
+            error_report("vfio: Error: Failed to enable %d "
+                         "MSI vectors, retry with %d\n", vdev->nr_vectors, ret);
+        }
+
+        for (i = 0; i < vdev->nr_vectors; i++) {
+            VFIOMSIVector *vector = &vdev->msi_vectors[i];
+            if (vector->virq >= 0) {
+                kvm_irqchip_remove_irqfd_notifier(kvm_state, &vector->interrupt,
+                                                  vector->virq);
+                kvm_irqchip_release_virq(kvm_state, vector->virq);
+                vector->virq = -1;
+            } else {
+                qemu_set_fd_handler(event_notifier_get_fd(&vector->interrupt),
+                                    NULL, NULL, NULL);
+            }
+            event_notifier_cleanup(&vector->interrupt);
+        }
+
+        g_free(vdev->msi_vectors);
+
+        if (ret > 0 && ret != vdev->nr_vectors) {
+            vdev->nr_vectors = ret;
+            goto retry;
+        }
+        vdev->nr_vectors = 0;
+
+        return;
+    }
+
+    vdev->interrupt = VFIO_INT_MSI;
+
+    DPRINTF("%s(%04x:%02x:%02x.%x) Enabled %d MSI vectors\n", __func__,
+            vdev->host.domain, vdev->host.bus, vdev->host.slot,
+            vdev->host.function, vdev->nr_vectors);
+}
+
+static void vfio_disable_msi_common(VFIODevice *vdev)
+{
+    g_free(vdev->msi_vectors);
+    vdev->msi_vectors = NULL;
+    vdev->nr_vectors = 0;
+    vdev->interrupt = VFIO_INT_NONE;
+
+    vfio_enable_intx(vdev);
+}
+
+static void vfio_disable_msix(VFIODevice *vdev)
+{
+    msix_unset_vector_notifiers(&vdev->pdev);
+
+    if (vdev->nr_vectors) {
+        vfio_disable_irqindex(vdev, VFIO_PCI_MSIX_IRQ_INDEX);
+    }
+
+    vfio_disable_msi_common(vdev);
+
+    DPRINTF("%s(%04x:%02x:%02x.%x)\n", __func__, vdev->host.domain,
+            vdev->host.bus, vdev->host.slot, vdev->host.function);
+}
+
+static void vfio_disable_msi(VFIODevice *vdev)
+{
+    int i;
+
+    vfio_disable_irqindex(vdev, VFIO_PCI_MSI_IRQ_INDEX);
+
+    for (i = 0; i < vdev->nr_vectors; i++) {
+        VFIOMSIVector *vector = &vdev->msi_vectors[i];
+
+        if (!vector->use) {
+            continue;
+        }
+
+        if (vector->virq >= 0) {
+            kvm_irqchip_remove_irqfd_notifier(kvm_state,
+                                              &vector->interrupt, vector->virq);
+            kvm_irqchip_release_virq(kvm_state, vector->virq);
+            vector->virq = -1;
+        } else {
+            qemu_set_fd_handler(event_notifier_get_fd(&vector->interrupt),
+                                NULL, NULL, NULL);
+        }
+
+        event_notifier_cleanup(&vector->interrupt);
+    }
+
+    vfio_disable_msi_common(vdev);
+
+    DPRINTF("%s(%04x:%02x:%02x.%x)\n", __func__, vdev->host.domain,
+            vdev->host.bus, vdev->host.slot, vdev->host.function);
+}
+
+/*
+ * IO Port/MMIO - Beware of the endians, VFIO is always little endian
+ */
+static void vfio_bar_write(void *opaque, hwaddr addr,
+                           uint64_t data, unsigned size)
+{
+    VFIOBAR *bar = opaque;
+    union {
+        uint8_t byte;
+        uint16_t word;
+        uint32_t dword;
+        uint64_t qword;
+    } buf;
+
+    switch (size) {
+    case 1:
+        buf.byte = data;
+        break;
+    case 2:
+        buf.word = cpu_to_le16(data);
+        break;
+    case 4:
+        buf.dword = cpu_to_le32(data);
+        break;
+    default:
+        hw_error("vfio: unsupported write size, %d bytes\n", size);
+        break;
+    }
+
+    if (pwrite(bar->fd, &buf, size, bar->fd_offset + addr) != size) {
+        error_report("%s(,0x%"HWADDR_PRIx", 0x%"PRIx64", %d) failed: %m\n",
+                     __func__, addr, data, size);
+    }
+
+    DPRINTF("%s(BAR%d+0x%"HWADDR_PRIx", 0x%"PRIx64", %d)\n",
+            __func__, bar->nr, addr, data, size);
+
+    /*
+     * A read or write to a BAR always signals an INTx EOI.  This will
+     * do nothing if not pending (including not in INTx mode).  We assume
+     * that a BAR access is in response to an interrupt and that BAR
+     * accesses will service the interrupt.  Unfortunately, we don't know
+     * which access will service the interrupt, so we're potentially
+     * getting quite a few host interrupts per guest interrupt.
+     */
+    vfio_eoi(container_of(bar, VFIODevice, bars[bar->nr]));
+}
+
+static uint64_t vfio_bar_read(void *opaque,
+                              hwaddr addr, unsigned size)
+{
+    VFIOBAR *bar = opaque;
+    union {
+        uint8_t byte;
+        uint16_t word;
+        uint32_t dword;
+        uint64_t qword;
+    } buf;
+    uint64_t data = 0;
+
+    if (pread(bar->fd, &buf, size, bar->fd_offset + addr) != size) {
+        error_report("%s(,0x%"HWADDR_PRIx", %d) failed: %m\n",
+                     __func__, addr, size);
+        return (uint64_t)-1;
+    }
+
+    switch (size) {
+    case 1:
+        data = buf.byte;
+        break;
+    case 2:
+        data = le16_to_cpu(buf.word);
+        break;
+    case 4:
+        data = le32_to_cpu(buf.dword);
+        break;
+    default:
+        hw_error("vfio: unsupported read size, %d bytes\n", size);
+        break;
+    }
+
+    DPRINTF("%s(BAR%d+0x%"HWADDR_PRIx", %d) = 0x%"PRIx64"\n",
+            __func__, bar->nr, addr, size, data);
+
+    /* Same as write above */
+    vfio_eoi(container_of(bar, VFIODevice, bars[bar->nr]));
+
+    return data;
+}
+
+static const MemoryRegionOps vfio_bar_ops = {
+    .read = vfio_bar_read,
+    .write = vfio_bar_write,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+/*
+ * PCI config space
+ */
+static uint32_t vfio_pci_read_config(PCIDevice *pdev, uint32_t addr, int len)
+{
+    VFIODevice *vdev = DO_UPCAST(VFIODevice, pdev, pdev);
+    uint32_t val = 0;
+
+    /*
+     * We only need QEMU PCI config support for the ROM BAR, the MSI and MSIX
+     * capabilities, and the multifunction bit below.  We let VFIO handle
+     * virtualizing everything else.  Performance is not a concern here.
+     */
+    if (ranges_overlap(addr, len, PCI_ROM_ADDRESS, 4) ||
+        (pdev->cap_present & QEMU_PCI_CAP_MSIX &&
+         ranges_overlap(addr, len, pdev->msix_cap, MSIX_CAP_LENGTH)) ||
+        (pdev->cap_present & QEMU_PCI_CAP_MSI &&
+         ranges_overlap(addr, len, pdev->msi_cap, vdev->msi_cap_size))) {
+
+        val = pci_default_read_config(pdev, addr, len);
+    } else {
+        if (pread(vdev->fd, &val, len, vdev->config_offset + addr) != len) {
+            error_report("%s(%04x:%02x:%02x.%x, 0x%x, 0x%x) failed: %m\n",
+                         __func__, vdev->host.domain, vdev->host.bus,
+                         vdev->host.slot, vdev->host.function, addr, len);
+            return -errno;
+        }
+        val = le32_to_cpu(val);
+    }
+
+    /* Multifunction bit is virualized in QEMU */
+    if (unlikely(ranges_overlap(addr, len, PCI_HEADER_TYPE, 1))) {
+        uint32_t mask = PCI_HEADER_TYPE_MULTI_FUNCTION;
+
+        if (len == 4) {
+            mask <<= 16;
+        }
+
+        if (pdev->cap_present & QEMU_PCI_CAP_MULTIFUNCTION) {
+            val |= mask;
+        } else {
+            val &= ~mask;
+        }
+    }
+
+    DPRINTF("%s(%04x:%02x:%02x.%x, @0x%x, len=0x%x) %x\n", __func__,
+            vdev->host.domain, vdev->host.bus, vdev->host.slot,
+            vdev->host.function, addr, len, val);
+
+    return val;
+}
+
+static void vfio_pci_write_config(PCIDevice *pdev, uint32_t addr,
+                                  uint32_t val, int len)
+{
+    VFIODevice *vdev = DO_UPCAST(VFIODevice, pdev, pdev);
+    uint32_t val_le = cpu_to_le32(val);
+
+    DPRINTF("%s(%04x:%02x:%02x.%x, @0x%x, 0x%x, len=0x%x)\n", __func__,
+            vdev->host.domain, vdev->host.bus, vdev->host.slot,
+            vdev->host.function, addr, val, len);
+
+    /* Write everything to VFIO, let it filter out what we can't write */
+    if (pwrite(vdev->fd, &val_le, len, vdev->config_offset + addr) != len) {
+        error_report("%s(%04x:%02x:%02x.%x, 0x%x, 0x%x, 0x%x) failed: %m\n",
+                     __func__, vdev->host.domain, vdev->host.bus,
+                     vdev->host.slot, vdev->host.function, addr, val, len);
+    }
+
+    /* Write standard header bits to emulation */
+    if (addr < PCI_CONFIG_HEADER_SIZE) {
+        pci_default_write_config(pdev, addr, val, len);
+        return;
+    }
+
+    /* MSI/MSI-X Enabling/Disabling */
+    if (pdev->cap_present & QEMU_PCI_CAP_MSI &&
+        ranges_overlap(addr, len, pdev->msi_cap, vdev->msi_cap_size)) {
+        int is_enabled, was_enabled = msi_enabled(pdev);
+
+        pci_default_write_config(pdev, addr, val, len);
+
+        is_enabled = msi_enabled(pdev);
+
+        if (!was_enabled && is_enabled) {
+            vfio_enable_msi(vdev);
+        } else if (was_enabled && !is_enabled) {
+            vfio_disable_msi(vdev);
+        }
+    }
+
+    if (pdev->cap_present & QEMU_PCI_CAP_MSIX &&
+        ranges_overlap(addr, len, pdev->msix_cap, MSIX_CAP_LENGTH)) {
+        int is_enabled, was_enabled = msix_enabled(pdev);
+
+        pci_default_write_config(pdev, addr, val, len);
+
+        is_enabled = msix_enabled(pdev);
+
+        if (!was_enabled && is_enabled) {
+            vfio_enable_msix(vdev);
+        } else if (was_enabled && !is_enabled) {
+            vfio_disable_msix(vdev);
+        }
+    }
+}
+
+/*
+ * DMA - Mapping and unmapping for the "type1" IOMMU interface used on x86
+ */
+static int vfio_dma_unmap(VFIOContainer *container,
+                          hwaddr iova, ram_addr_t size)
+{
+    struct vfio_iommu_type1_dma_unmap unmap = {
+        .argsz = sizeof(unmap),
+        .flags = 0,
+        .iova = iova,
+        .size = size,
+    };
+
+    if (ioctl(container->fd, VFIO_IOMMU_UNMAP_DMA, &unmap)) {
+        DPRINTF("VFIO_UNMAP_DMA: %d\n", -errno);
+        return -errno;
+    }
+
+    return 0;
+}
+
+static int vfio_dma_map(VFIOContainer *container, hwaddr iova,
+                        ram_addr_t size, void *vaddr, bool readonly)
+{
+    struct vfio_iommu_type1_dma_map map = {
+        .argsz = sizeof(map),
+        .flags = VFIO_DMA_MAP_FLAG_READ,
+        .vaddr = (__u64)(uintptr_t)vaddr,
+        .iova = iova,
+        .size = size,
+    };
+
+    if (!readonly) {
+        map.flags |= VFIO_DMA_MAP_FLAG_WRITE;
+    }
+
+    /*
+     * Try the mapping, if it fails with EBUSY, unmap the region and try
+     * again.  This shouldn't be necessary, but we sometimes see it in
+     * the the VGA ROM space.
+     */
+    if (ioctl(container->fd, VFIO_IOMMU_MAP_DMA, &map) == 0 ||
+        (errno == EBUSY && vfio_dma_unmap(container, iova, size) == 0 &&
+         ioctl(container->fd, VFIO_IOMMU_MAP_DMA, &map) == 0)) {
+        return 0;
+    }
+
+    DPRINTF("VFIO_MAP_DMA: %d\n", -errno);
+    return -errno;
+}
+
+static bool vfio_listener_skipped_section(MemoryRegionSection *section)
+{
+    return !memory_region_is_ram(section->mr);
+}
+
+static void vfio_listener_region_add(MemoryListener *listener,
+                                     MemoryRegionSection *section)
+{
+    VFIOContainer *container = container_of(listener, VFIOContainer,
+                                            iommu_data.listener);
+    hwaddr iova, end;
+    void *vaddr;
+    int ret;
+
+    if (vfio_listener_skipped_section(section)) {
+        DPRINTF("vfio: SKIPPING region_add %"HWADDR_PRIx" - %"PRIx64"\n",
+                section->offset_within_address_space,
+                section->offset_within_address_space + section->size - 1);
+        return;
+    }
+
+    if (unlikely((section->offset_within_address_space & ~TARGET_PAGE_MASK) !=
+                 (section->offset_within_region & ~TARGET_PAGE_MASK))) {
+        error_report("%s received unaligned region\n", __func__);
+        return;
+    }
+
+    iova = TARGET_PAGE_ALIGN(section->offset_within_address_space);
+    end = (section->offset_within_address_space + section->size) &
+          TARGET_PAGE_MASK;
+
+    if (iova >= end) {
+        return;
+    }
+
+    vaddr = memory_region_get_ram_ptr(section->mr) +
+            section->offset_within_region +
+            (iova - section->offset_within_address_space);
+
+    DPRINTF("vfio: region_add %"HWADDR_PRIx" - %"HWADDR_PRIx" [%p]\n",
+            iova, end - 1, vaddr);
+
+    ret = vfio_dma_map(container, iova, end - iova, vaddr, section->readonly);
+    if (ret) {
+        error_report("vfio_dma_map(%p, 0x%"HWADDR_PRIx", "
+                     "0x%"HWADDR_PRIx", %p) = %d (%m)\n",
+                     container, iova, end - iova, vaddr, ret);
+    }
+}
+
+static void vfio_listener_region_del(MemoryListener *listener,
+                                     MemoryRegionSection *section)
+{
+    VFIOContainer *container = container_of(listener, VFIOContainer,
+                                            iommu_data.listener);
+    hwaddr iova, end;
+    int ret;
+
+    if (vfio_listener_skipped_section(section)) {
+        DPRINTF("vfio: SKIPPING region_del %"HWADDR_PRIx" - %"PRIx64"\n",
+                section->offset_within_address_space,
+                section->offset_within_address_space + section->size - 1);
+        return;
+    }
+
+    if (unlikely((section->offset_within_address_space & ~TARGET_PAGE_MASK) !=
+                 (section->offset_within_region & ~TARGET_PAGE_MASK))) {
+        error_report("%s received unaligned region\n", __func__);
+        return;
+    }
+
+    iova = TARGET_PAGE_ALIGN(section->offset_within_address_space);
+    end = (section->offset_within_address_space + section->size) &
+          TARGET_PAGE_MASK;
+
+    if (iova >= end) {
+        return;
+    }
+
+    DPRINTF("vfio: region_del %"HWADDR_PRIx" - %"HWADDR_PRIx"\n",
+            iova, end - 1);
+
+    ret = vfio_dma_unmap(container, iova, end - iova);
+    if (ret) {
+        error_report("vfio_dma_unmap(%p, 0x%"HWADDR_PRIx", "
+                     "0x%"HWADDR_PRIx") = %d (%m)\n",
+                     container, iova, end - iova, ret);
+    }
+}
+
+static MemoryListener vfio_memory_listener = {
+    .region_add = vfio_listener_region_add,
+    .region_del = vfio_listener_region_del,
+};
+
+static void vfio_listener_release(VFIOContainer *container)
+{
+    memory_listener_unregister(&container->iommu_data.listener);
+}
+
+/*
+ * Interrupt setup
+ */
+static void vfio_disable_interrupts(VFIODevice *vdev)
+{
+    switch (vdev->interrupt) {
+    case VFIO_INT_INTx:
+        vfio_disable_intx(vdev);
+        break;
+    case VFIO_INT_MSI:
+        vfio_disable_msi(vdev);
+        break;
+    case VFIO_INT_MSIX:
+        vfio_disable_msix(vdev);
+        break;
+    }
+}
+
+static int vfio_setup_msi(VFIODevice *vdev, int pos)
+{
+    uint16_t ctrl;
+    bool msi_64bit, msi_maskbit;
+    int ret, entries;
+
+    if (pread(vdev->fd, &ctrl, sizeof(ctrl),
+              vdev->config_offset + pos + PCI_CAP_FLAGS) != sizeof(ctrl)) {
+        return -errno;
+    }
+    ctrl = le16_to_cpu(ctrl);
+
+    msi_64bit = !!(ctrl & PCI_MSI_FLAGS_64BIT);
+    msi_maskbit = !!(ctrl & PCI_MSI_FLAGS_MASKBIT);
+    entries = 1 << ((ctrl & PCI_MSI_FLAGS_QMASK) >> 1);
+
+    DPRINTF("%04x:%02x:%02x.%x PCI MSI CAP @0x%x\n", vdev->host.domain,
+            vdev->host.bus, vdev->host.slot, vdev->host.function, pos);
+
+    ret = msi_init(&vdev->pdev, pos, entries, msi_64bit, msi_maskbit);
+    if (ret < 0) {
+        if (ret == -ENOTSUP) {
+            return 0;
+        }
+        error_report("vfio: msi_init failed\n");
+        return ret;
+    }
+    vdev->msi_cap_size = 0xa + (msi_maskbit ? 0xa : 0) + (msi_64bit ? 0x4 : 0);
+
+    return 0;
+}
+
+/*
+ * We don't have any control over how pci_add_capability() inserts
+ * capabilities into the chain.  In order to setup MSI-X we need a
+ * MemoryRegion for the BAR.  In order to setup the BAR and not
+ * attempt to mmap the MSI-X table area, which VFIO won't allow, we
+ * need to first look for where the MSI-X table lives.  So we
+ * unfortunately split MSI-X setup across two functions.
+ */
+static int vfio_early_setup_msix(VFIODevice *vdev)
+{
+    uint8_t pos;
+    uint16_t ctrl;
+    uint32_t table, pba;
+
+    pos = pci_find_capability(&vdev->pdev, PCI_CAP_ID_MSIX);
+    if (!pos) {
+        return 0;
+    }
+
+    if (pread(vdev->fd, &ctrl, sizeof(ctrl),
+              vdev->config_offset + pos + PCI_CAP_FLAGS) != sizeof(ctrl)) {
+        return -errno;
+    }
+
+    if (pread(vdev->fd, &table, sizeof(table),
+              vdev->config_offset + pos + PCI_MSIX_TABLE) != sizeof(table)) {
+        return -errno;
+    }
+
+    if (pread(vdev->fd, &pba, sizeof(pba),
+              vdev->config_offset + pos + PCI_MSIX_PBA) != sizeof(pba)) {
+        return -errno;
+    }
+
+    ctrl = le16_to_cpu(ctrl);
+    table = le32_to_cpu(table);
+    pba = le32_to_cpu(pba);
+
+    vdev->msix = g_malloc0(sizeof(*(vdev->msix)));
+    vdev->msix->table_bar = table & PCI_MSIX_FLAGS_BIRMASK;
+    vdev->msix->table_offset = table & ~PCI_MSIX_FLAGS_BIRMASK;
+    vdev->msix->pba_bar = pba & PCI_MSIX_FLAGS_BIRMASK;
+    vdev->msix->pba_offset = pba & ~PCI_MSIX_FLAGS_BIRMASK;
+    vdev->msix->entries = (ctrl & PCI_MSIX_FLAGS_QSIZE) + 1;
+
+    DPRINTF("%04x:%02x:%02x.%x "
+            "PCI MSI-X CAP @0x%x, BAR %d, offset 0x%x, entries %d\n",
+            vdev->host.domain, vdev->host.bus, vdev->host.slot,
+            vdev->host.function, pos, vdev->msix->table_bar,
+            vdev->msix->table_offset, vdev->msix->entries);
+
+    return 0;
+}
+
+static int vfio_setup_msix(VFIODevice *vdev, int pos)
+{
+    int ret;
+
+    ret = msix_init(&vdev->pdev, vdev->msix->entries,
+                    &vdev->bars[vdev->msix->table_bar].mem,
+                    vdev->msix->table_bar, vdev->msix->table_offset,
+                    &vdev->bars[vdev->msix->pba_bar].mem,
+                    vdev->msix->pba_bar, vdev->msix->pba_offset, pos);
+    if (ret < 0) {
+        if (ret == -ENOTSUP) {
+            return 0;
+        }
+        error_report("vfio: msix_init failed\n");
+        return ret;
+    }
+
+    return 0;
+}
+
+static void vfio_teardown_msi(VFIODevice *vdev)
+{
+    msi_uninit(&vdev->pdev);
+
+    if (vdev->msix) {
+        msix_uninit(&vdev->pdev, &vdev->bars[vdev->msix->table_bar].mem,
+                    &vdev->bars[vdev->msix->pba_bar].mem);
+    }
+}
+
+/*
+ * Resource setup
+ */
+static void vfio_mmap_set_enabled(VFIODevice *vdev, bool enabled)
+{
+    int i;
+
+    for (i = 0; i < PCI_ROM_SLOT; i++) {
+        VFIOBAR *bar = &vdev->bars[i];
+
+        if (!bar->size) {
+            continue;
+        }
+
+        memory_region_set_enabled(&bar->mmap_mem, enabled);
+        if (vdev->msix && vdev->msix->table_bar == i) {
+            memory_region_set_enabled(&vdev->msix->mmap_mem, enabled);
+        }
+    }
+}
+
+static void vfio_unmap_bar(VFIODevice *vdev, int nr)
+{
+    VFIOBAR *bar = &vdev->bars[nr];
+
+    if (!bar->size) {
+        return;
+    }
+
+    memory_region_del_subregion(&bar->mem, &bar->mmap_mem);
+    munmap(bar->mmap, memory_region_size(&bar->mmap_mem));
+
+    if (vdev->msix && vdev->msix->table_bar == nr) {
+        memory_region_del_subregion(&bar->mem, &vdev->msix->mmap_mem);
+        munmap(vdev->msix->mmap, memory_region_size(&vdev->msix->mmap_mem));
+    }
+
+    memory_region_destroy(&bar->mem);
+}
+
+static int vfio_mmap_bar(VFIOBAR *bar, MemoryRegion *mem, MemoryRegion *submem,
+                         void **map, size_t size, off_t offset,
+                         const char *name)
+{
+    int ret = 0;
+
+    if (size && bar->flags & VFIO_REGION_INFO_FLAG_MMAP) {
+        int prot = 0;
+
+        if (bar->flags & VFIO_REGION_INFO_FLAG_READ) {
+            prot |= PROT_READ;
+        }
+
+        if (bar->flags & VFIO_REGION_INFO_FLAG_WRITE) {
+            prot |= PROT_WRITE;
+        }
+
+        *map = mmap(NULL, size, prot, MAP_SHARED,
+                    bar->fd, bar->fd_offset + offset);
+        if (*map == MAP_FAILED) {
+            *map = NULL;
+            ret = -errno;
+            goto empty_region;
+        }
+
+        memory_region_init_ram_ptr(submem, name, size, *map);
+    } else {
+empty_region:
+        /* Create a zero sized sub-region to make cleanup easy. */
+        memory_region_init(submem, name, 0);
+    }
+
+    memory_region_add_subregion(mem, offset, submem);
+
+    return ret;
+}
+
+static void vfio_map_bar(VFIODevice *vdev, int nr)
+{
+    VFIOBAR *bar = &vdev->bars[nr];
+    unsigned size = bar->size;
+    char name[64];
+    uint32_t pci_bar;
+    uint8_t type;
+    int ret;
+
+    /* Skip both unimplemented BARs and the upper half of 64bit BARS. */
+    if (!size) {
+        return;
+    }
+
+    snprintf(name, sizeof(name), "VFIO %04x:%02x:%02x.%x BAR %d",
+             vdev->host.domain, vdev->host.bus, vdev->host.slot,
+             vdev->host.function, nr);
+
+    /* Determine what type of BAR this is for registration */
+    ret = pread(vdev->fd, &pci_bar, sizeof(pci_bar),
+                vdev->config_offset + PCI_BASE_ADDRESS_0 + (4 * nr));
+    if (ret != sizeof(pci_bar)) {
+        error_report("vfio: Failed to read BAR %d (%m)\n", nr);
+        return;
+    }
+
+    pci_bar = le32_to_cpu(pci_bar);
+    type = pci_bar & (pci_bar & PCI_BASE_ADDRESS_SPACE_IO ?
+           ~PCI_BASE_ADDRESS_IO_MASK : ~PCI_BASE_ADDRESS_MEM_MASK);
+
+    /* A "slow" read/write mapping underlies all BARs */
+    memory_region_init_io(&bar->mem, &vfio_bar_ops, bar, name, size);
+    pci_register_bar(&vdev->pdev, nr, type, &bar->mem);
+
+    /*
+     * We can't mmap areas overlapping the MSIX vector table, so we
+     * potentially insert a direct-mapped subregion before and after it.
+     */
+    if (vdev->msix && vdev->msix->table_bar == nr) {
+        size = vdev->msix->table_offset & TARGET_PAGE_MASK;
+    }
+
+    strncat(name, " mmap", sizeof(name) - strlen(name) - 1);
+    if (vfio_mmap_bar(bar, &bar->mem,
+                      &bar->mmap_mem, &bar->mmap, size, 0, name)) {
+        error_report("%s unsupported. Performance may be slow\n", name);
+    }
+
+    if (vdev->msix && vdev->msix->table_bar == nr) {
+        unsigned start;
+
+        start = TARGET_PAGE_ALIGN(vdev->msix->table_offset +
+                                  (vdev->msix->entries * PCI_MSIX_ENTRY_SIZE));
+
+        size = start < bar->size ? bar->size - start : 0;
+        strncat(name, " msix-hi", sizeof(name) - strlen(name) - 1);
+        /* VFIOMSIXInfo contains another MemoryRegion for this mapping */
+        if (vfio_mmap_bar(bar, &bar->mem, &vdev->msix->mmap_mem,
+                          &vdev->msix->mmap, size, start, name)) {
+            error_report("%s unsupported. Performance may be slow\n", name);
+        }
+    }
+}
+
+static void vfio_map_bars(VFIODevice *vdev)
+{
+    int i;
+
+    for (i = 0; i < PCI_ROM_SLOT; i++) {
+        vfio_map_bar(vdev, i);
+    }
+}
+
+static void vfio_unmap_bars(VFIODevice *vdev)
+{
+    int i;
+
+    for (i = 0; i < PCI_ROM_SLOT; i++) {
+        vfio_unmap_bar(vdev, i);
+    }
+}
+
+/*
+ * General setup
+ */
+static uint8_t vfio_std_cap_max_size(PCIDevice *pdev, uint8_t pos)
+{
+    uint8_t tmp, next = 0xff;
+
+    for (tmp = pdev->config[PCI_CAPABILITY_LIST]; tmp;
+         tmp = pdev->config[tmp + 1]) {
+        if (tmp > pos && tmp < next) {
+            next = tmp;
+        }
+    }
+
+    return next - pos;
+}
+
+static int vfio_add_std_cap(VFIODevice *vdev, uint8_t pos)
+{
+    PCIDevice *pdev = &vdev->pdev;
+    uint8_t cap_id, next, size;
+    int ret;
+
+    cap_id = pdev->config[pos];
+    next = pdev->config[pos + 1];
+
+    /*
+     * If it becomes important to configure capabilities to their actual
+     * size, use this as the default when it's something we don't recognize.
+     * Since QEMU doesn't actually handle many of the config accesses,
+     * exact size doesn't seem worthwhile.
+     */
+    size = vfio_std_cap_max_size(pdev, pos);
+
+    /*
+     * pci_add_capability always inserts the new capability at the head
+     * of the chain.  Therefore to end up with a chain that matches the
+     * physical device, we insert from the end by making this recursive.
+     * This is also why we pre-caclulate size above as cached config space
+     * will be changed as we unwind the stack.
+     */
+    if (next) {
+        ret = vfio_add_std_cap(vdev, next);
+        if (ret) {
+            return ret;
+        }
+    } else {
+        pdev->config[PCI_CAPABILITY_LIST] = 0; /* Begin the rebuild */
+    }
+
+    switch (cap_id) {
+    case PCI_CAP_ID_MSI:
+        ret = vfio_setup_msi(vdev, pos);
+        break;
+    case PCI_CAP_ID_MSIX:
+        ret = vfio_setup_msix(vdev, pos);
+        break;
+    default:
+        ret = pci_add_capability(pdev, cap_id, pos, size);
+        break;
+    }
+
+    if (ret < 0) {
+        error_report("vfio: %04x:%02x:%02x.%x Error adding PCI capability "
+                     "0x%x[0x%x]@0x%x: %d\n", vdev->host.domain,
+                     vdev->host.bus, vdev->host.slot, vdev->host.function,
+                     cap_id, size, pos, ret);
+        return ret;
+    }
+
+    return 0;
+}
+
+static int vfio_add_capabilities(VFIODevice *vdev)
+{
+    PCIDevice *pdev = &vdev->pdev;
+
+    if (!(pdev->config[PCI_STATUS] & PCI_STATUS_CAP_LIST) ||
+        !pdev->config[PCI_CAPABILITY_LIST]) {
+        return 0; /* Nothing to add */
+    }
+
+    return vfio_add_std_cap(vdev, pdev->config[PCI_CAPABILITY_LIST]);
+}
+
+static int vfio_load_rom(VFIODevice *vdev)
+{
+    uint64_t size = vdev->rom_size;
+    char name[32];
+    off_t off = 0, voff = vdev->rom_offset;
+    ssize_t bytes;
+    void *ptr;
+
+    /* If loading ROM from file, pci handles it */
+    if (vdev->pdev.romfile || !vdev->pdev.rom_bar || !size) {
+        return 0;
+    }
+
+    DPRINTF("%s(%04x:%02x:%02x.%x)\n", __func__, vdev->host.domain,
+            vdev->host.bus, vdev->host.slot, vdev->host.function);
+
+    snprintf(name, sizeof(name), "vfio[%04x:%02x:%02x.%x].rom",
+             vdev->host.domain, vdev->host.bus, vdev->host.slot,
+             vdev->host.function);
+    memory_region_init_ram(&vdev->pdev.rom, name, size);
+    ptr = memory_region_get_ram_ptr(&vdev->pdev.rom);
+    memset(ptr, 0xff, size);
+
+    while (size) {
+        bytes = pread(vdev->fd, ptr + off, size, voff + off);
+        if (bytes == 0) {
+            break; /* expect that we could get back less than the ROM BAR */
+        } else if (bytes > 0) {
+            off += bytes;
+            size -= bytes;
+        } else {
+            if (errno == EINTR || errno == EAGAIN) {
+                continue;
+            }
+            error_report("vfio: Error reading device ROM: %m\n");
+            memory_region_destroy(&vdev->pdev.rom);
+            return -errno;
+        }
+    }
+
+    pci_register_bar(&vdev->pdev, PCI_ROM_SLOT, 0, &vdev->pdev.rom);
+    vdev->pdev.has_rom = true;
+    return 0;
+}
+
+static int vfio_connect_container(VFIOGroup *group)
+{
+    VFIOContainer *container;
+    int ret, fd;
+
+    if (group->container) {
+        return 0;
+    }
+
+    QLIST_FOREACH(container, &container_list, next) {
+        if (!ioctl(group->fd, VFIO_GROUP_SET_CONTAINER, &container->fd)) {
+            group->container = container;
+            QLIST_INSERT_HEAD(&container->group_list, group, container_next);
+            return 0;
+        }
+    }
+
+    fd = qemu_open("/dev/vfio/vfio", O_RDWR);
+    if (fd < 0) {
+        error_report("vfio: failed to open /dev/vfio/vfio: %m\n");
+        return -errno;
+    }
+
+    ret = ioctl(fd, VFIO_GET_API_VERSION);
+    if (ret != VFIO_API_VERSION) {
+        error_report("vfio: supported vfio version: %d, "
+                     "reported version: %d\n", VFIO_API_VERSION, ret);
+        close(fd);
+        return -EINVAL;
+    }
+
+    container = g_malloc0(sizeof(*container));
+    container->fd = fd;
+
+    if (ioctl(fd, VFIO_CHECK_EXTENSION, VFIO_TYPE1_IOMMU)) {
+        ret = ioctl(group->fd, VFIO_GROUP_SET_CONTAINER, &fd);
+        if (ret) {
+            error_report("vfio: failed to set group container: %m\n");
+            g_free(container);
+            close(fd);
+            return -errno;
+        }
+
+        ret = ioctl(fd, VFIO_SET_IOMMU, VFIO_TYPE1_IOMMU);
+        if (ret) {
+            error_report("vfio: failed to set iommu for container: %m\n");
+            g_free(container);
+            close(fd);
+            return -errno;
+        }
+
+        container->iommu_data.listener = vfio_memory_listener;
+        container->iommu_data.release = vfio_listener_release;
+
+        memory_listener_register(&container->iommu_data.listener, &address_space_memory);
+    } else {
+        error_report("vfio: No available IOMMU models\n");
+        g_free(container);
+        close(fd);
+        return -EINVAL;
+    }
+
+    QLIST_INIT(&container->group_list);
+    QLIST_INSERT_HEAD(&container_list, container, next);
+
+    group->container = container;
+    QLIST_INSERT_HEAD(&container->group_list, group, container_next);
+
+    return 0;
+}
+
+static void vfio_disconnect_container(VFIOGroup *group)
+{
+    VFIOContainer *container = group->container;
+
+    if (ioctl(group->fd, VFIO_GROUP_UNSET_CONTAINER, &container->fd)) {
+        error_report("vfio: error disconnecting group %d from container\n",
+                     group->groupid);
+    }
+
+    QLIST_REMOVE(group, container_next);
+    group->container = NULL;
+
+    if (QLIST_EMPTY(&container->group_list)) {
+        if (container->iommu_data.release) {
+            container->iommu_data.release(container);
+        }
+        QLIST_REMOVE(container, next);
+        DPRINTF("vfio_disconnect_container: close container->fd\n");
+        close(container->fd);
+        g_free(container);
+    }
+}
+
+static VFIOGroup *vfio_get_group(int groupid)
+{
+    VFIOGroup *group;
+    char path[32];
+    struct vfio_group_status status = { .argsz = sizeof(status) };
+
+    QLIST_FOREACH(group, &group_list, next) {
+        if (group->groupid == groupid) {
+            return group;
+        }
+    }
+
+    group = g_malloc0(sizeof(*group));
+
+    snprintf(path, sizeof(path), "/dev/vfio/%d", groupid);
+    group->fd = qemu_open(path, O_RDWR);
+    if (group->fd < 0) {
+        error_report("vfio: error opening %s: %m\n", path);
+        g_free(group);
+        return NULL;
+    }
+
+    if (ioctl(group->fd, VFIO_GROUP_GET_STATUS, &status)) {
+        error_report("vfio: error getting group status: %m\n");
+        close(group->fd);
+        g_free(group);
+        return NULL;
+    }
+
+    if (!(status.flags & VFIO_GROUP_FLAGS_VIABLE)) {
+        error_report("vfio: error, group %d is not viable, please ensure "
+                     "all devices within the iommu_group are bound to their "
+                     "vfio bus driver.\n", groupid);
+        close(group->fd);
+        g_free(group);
+        return NULL;
+    }
+
+    group->groupid = groupid;
+    QLIST_INIT(&group->device_list);
+
+    if (vfio_connect_container(group)) {
+        error_report("vfio: failed to setup container for group %d\n", groupid);
+        close(group->fd);
+        g_free(group);
+        return NULL;
+    }
+
+    QLIST_INSERT_HEAD(&group_list, group, next);
+
+    return group;
+}
+
+static void vfio_put_group(VFIOGroup *group)
+{
+    if (!QLIST_EMPTY(&group->device_list)) {
+        return;
+    }
+
+    vfio_disconnect_container(group);
+    QLIST_REMOVE(group, next);
+    DPRINTF("vfio_put_group: close group->fd\n");
+    close(group->fd);
+    g_free(group);
+}
+
+static int vfio_get_device(VFIOGroup *group, const char *name, VFIODevice *vdev)
+{
+    struct vfio_device_info dev_info = { .argsz = sizeof(dev_info) };
+    struct vfio_region_info reg_info = { .argsz = sizeof(reg_info) };
+    int ret, i;
+
+    ret = ioctl(group->fd, VFIO_GROUP_GET_DEVICE_FD, name);
+    if (ret < 0) {
+        error_report("vfio: error getting device %s from group %d: %m\n",
+                     name, group->groupid);
+        error_report("Verify all devices in group %d are bound to vfio-pci "
+                     "or pci-stub and not already in use\n", group->groupid);
+        return ret;
+    }
+
+    vdev->fd = ret;
+    vdev->group = group;
+    QLIST_INSERT_HEAD(&group->device_list, vdev, next);
+
+    /* Sanity check device */
+    ret = ioctl(vdev->fd, VFIO_DEVICE_GET_INFO, &dev_info);
+    if (ret) {
+        error_report("vfio: error getting device info: %m\n");
+        goto error;
+    }
+
+    DPRINTF("Device %s flags: %u, regions: %u, irgs: %u\n", name,
+            dev_info.flags, dev_info.num_regions, dev_info.num_irqs);
+
+    if (!(dev_info.flags & VFIO_DEVICE_FLAGS_PCI)) {
+        error_report("vfio: Um, this isn't a PCI device\n");
+        goto error;
+    }
+
+    vdev->reset_works = !!(dev_info.flags & VFIO_DEVICE_FLAGS_RESET);
+    if (!vdev->reset_works) {
+        error_report("Warning, device %s does not support reset\n", name);
+    }
+
+    if (dev_info.num_regions != VFIO_PCI_NUM_REGIONS) {
+        error_report("vfio: unexpected number of io regions %u\n",
+                     dev_info.num_regions);
+        goto error;
+    }
+
+    if (dev_info.num_irqs != VFIO_PCI_NUM_IRQS) {
+        error_report("vfio: unexpected number of irqs %u\n", dev_info.num_irqs);
+        goto error;
+    }
+
+    for (i = VFIO_PCI_BAR0_REGION_INDEX; i < VFIO_PCI_ROM_REGION_INDEX; i++) {
+        reg_info.index = i;
+
+        ret = ioctl(vdev->fd, VFIO_DEVICE_GET_REGION_INFO, &reg_info);
+        if (ret) {
+            error_report("vfio: Error getting region %d info: %m\n", i);
+            goto error;
+        }
+
+        DPRINTF("Device %s region %d:\n", name, i);
+        DPRINTF("  size: 0x%lx, offset: 0x%lx, flags: 0x%lx\n",
+                (unsigned long)reg_info.size, (unsigned long)reg_info.offset,
+                (unsigned long)reg_info.flags);
+
+        vdev->bars[i].flags = reg_info.flags;
+        vdev->bars[i].size = reg_info.size;
+        vdev->bars[i].fd_offset = reg_info.offset;
+        vdev->bars[i].fd = vdev->fd;
+        vdev->bars[i].nr = i;
+    }
+
+    reg_info.index = VFIO_PCI_ROM_REGION_INDEX;
+
+    ret = ioctl(vdev->fd, VFIO_DEVICE_GET_REGION_INFO, &reg_info);
+    if (ret) {
+        error_report("vfio: Error getting ROM info: %m\n");
+        goto error;
+    }
+
+    DPRINTF("Device %s ROM:\n", name);
+    DPRINTF("  size: 0x%lx, offset: 0x%lx, flags: 0x%lx\n",
+            (unsigned long)reg_info.size, (unsigned long)reg_info.offset,
+            (unsigned long)reg_info.flags);
+
+    vdev->rom_size = reg_info.size;
+    vdev->rom_offset = reg_info.offset;
+
+    reg_info.index = VFIO_PCI_CONFIG_REGION_INDEX;
+
+    ret = ioctl(vdev->fd, VFIO_DEVICE_GET_REGION_INFO, &reg_info);
+    if (ret) {
+        error_report("vfio: Error getting config info: %m\n");
+        goto error;
+    }
+
+    DPRINTF("Device %s config:\n", name);
+    DPRINTF("  size: 0x%lx, offset: 0x%lx, flags: 0x%lx\n",
+            (unsigned long)reg_info.size, (unsigned long)reg_info.offset,
+            (unsigned long)reg_info.flags);
+
+    vdev->config_size = reg_info.size;
+    vdev->config_offset = reg_info.offset;
+
+error:
+    if (ret) {
+        QLIST_REMOVE(vdev, next);
+        vdev->group = NULL;
+        close(vdev->fd);
+    }
+    return ret;
+}
+
+static void vfio_put_device(VFIODevice *vdev)
+{
+    QLIST_REMOVE(vdev, next);
+    vdev->group = NULL;
+    DPRINTF("vfio_put_device: close vdev->fd\n");
+    close(vdev->fd);
+    if (vdev->msix) {
+        g_free(vdev->msix);
+        vdev->msix = NULL;
+    }
+}
+
+static int vfio_initfn(PCIDevice *pdev)
+{
+    VFIODevice *pvdev, *vdev = DO_UPCAST(VFIODevice, pdev, pdev);
+    VFIOGroup *group;
+    char path[PATH_MAX], iommu_group_path[PATH_MAX], *group_name;
+    ssize_t len;
+    struct stat st;
+    int groupid;
+    int ret;
+
+    /* Check that the host device exists */
+    snprintf(path, sizeof(path),
+             "/sys/bus/pci/devices/%04x:%02x:%02x.%01x/",
+             vdev->host.domain, vdev->host.bus, vdev->host.slot,
+             vdev->host.function);
+    if (stat(path, &st) < 0) {
+        error_report("vfio: error: no such host device: %s\n", path);
+        return -errno;
+    }
+
+    strncat(path, "iommu_group", sizeof(path) - strlen(path) - 1);
+
+    len = readlink(path, iommu_group_path, PATH_MAX);
+    if (len <= 0) {
+        error_report("vfio: error no iommu_group for device\n");
+        return -errno;
+    }
+
+    iommu_group_path[len] = 0;
+    group_name = basename(iommu_group_path);
+
+    if (sscanf(group_name, "%d", &groupid) != 1) {
+        error_report("vfio: error reading %s: %m\n", path);
+        return -errno;
+    }
+
+    DPRINTF("%s(%04x:%02x:%02x.%x) group %d\n", __func__, vdev->host.domain,
+            vdev->host.bus, vdev->host.slot, vdev->host.function, groupid);
+
+    group = vfio_get_group(groupid);
+    if (!group) {
+        error_report("vfio: failed to get group %d\n", groupid);
+        return -ENOENT;
+    }
+
+    snprintf(path, sizeof(path), "%04x:%02x:%02x.%01x",
+            vdev->host.domain, vdev->host.bus, vdev->host.slot,
+            vdev->host.function);
+
+    QLIST_FOREACH(pvdev, &group->device_list, next) {
+        if (pvdev->host.domain == vdev->host.domain &&
+            pvdev->host.bus == vdev->host.bus &&
+            pvdev->host.slot == vdev->host.slot &&
+            pvdev->host.function == vdev->host.function) {
+
+            error_report("vfio: error: device %s is already attached\n", path);
+            vfio_put_group(group);
+            return -EBUSY;
+        }
+    }
+
+    ret = vfio_get_device(group, path, vdev);
+    if (ret) {
+        error_report("vfio: failed to get device %s\n", path);
+        vfio_put_group(group);
+        return ret;
+    }
+
+    /* Get a copy of config space */
+    ret = pread(vdev->fd, vdev->pdev.config,
+                MIN(pci_config_size(&vdev->pdev), vdev->config_size),
+                vdev->config_offset);
+    if (ret < (int)MIN(pci_config_size(&vdev->pdev), vdev->config_size)) {
+        ret = ret < 0 ? -errno : -EFAULT;
+        error_report("vfio: Failed to read device config space\n");
+        goto out_put;
+    }
+
+    /*
+     * Clear host resource mapping info.  If we choose not to register a
+     * BAR, such as might be the case with the option ROM, we can get
+     * confusing, unwritable, residual addresses from the host here.
+     */
+    memset(&vdev->pdev.config[PCI_BASE_ADDRESS_0], 0, 24);
+    memset(&vdev->pdev.config[PCI_ROM_ADDRESS], 0, 4);
+
+    vfio_load_rom(vdev);
+
+    ret = vfio_early_setup_msix(vdev);
+    if (ret) {
+        goto out_put;
+    }
+
+    vfio_map_bars(vdev);
+
+    ret = vfio_add_capabilities(vdev);
+    if (ret) {
+        goto out_teardown;
+    }
+
+    if (vfio_pci_read_config(&vdev->pdev, PCI_INTERRUPT_PIN, 1)) {
+        vdev->intx.mmap_timer = qemu_new_timer_ms(vm_clock,
+                                                  vfio_intx_mmap_enable, vdev);
+        pci_device_set_intx_routing_notifier(&vdev->pdev, vfio_update_irq);
+        ret = vfio_enable_intx(vdev);
+        if (ret) {
+            goto out_teardown;
+        }
+    }
+
+    return 0;
+
+out_teardown:
+    pci_device_set_intx_routing_notifier(&vdev->pdev, NULL);
+    vfio_teardown_msi(vdev);
+    vfio_unmap_bars(vdev);
+out_put:
+    vfio_put_device(vdev);
+    vfio_put_group(group);
+    return ret;
+}
+
+static void vfio_exitfn(PCIDevice *pdev)
+{
+    VFIODevice *vdev = DO_UPCAST(VFIODevice, pdev, pdev);
+    VFIOGroup *group = vdev->group;
+
+    pci_device_set_intx_routing_notifier(&vdev->pdev, NULL);
+    vfio_disable_interrupts(vdev);
+    if (vdev->intx.mmap_timer) {
+        qemu_free_timer(vdev->intx.mmap_timer);
+    }
+    vfio_teardown_msi(vdev);
+    vfio_unmap_bars(vdev);
+    vfio_put_device(vdev);
+    vfio_put_group(group);
+}
+
+static void vfio_pci_reset(DeviceState *dev)
+{
+    PCIDevice *pdev = DO_UPCAST(PCIDevice, qdev, dev);
+    VFIODevice *vdev = DO_UPCAST(VFIODevice, pdev, pdev);
+    uint16_t cmd;
+
+    DPRINTF("%s(%04x:%02x:%02x.%x)\n", __func__, vdev->host.domain,
+            vdev->host.bus, vdev->host.slot, vdev->host.function);
+
+    vfio_disable_interrupts(vdev);
+
+    /*
+     * Stop any ongoing DMA by disconecting I/O, MMIO, and bus master.
+     * Also put INTx Disable in known state.
+     */
+    cmd = vfio_pci_read_config(pdev, PCI_COMMAND, 2);
+    cmd &= ~(PCI_COMMAND_IO | PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER |
+             PCI_COMMAND_INTX_DISABLE);
+    vfio_pci_write_config(pdev, PCI_COMMAND, cmd, 2);
+
+    if (vdev->reset_works) {
+        if (ioctl(vdev->fd, VFIO_DEVICE_RESET)) {
+            error_report("vfio: Error unable to reset physical device "
+                         "(%04x:%02x:%02x.%x): %m\n", vdev->host.domain,
+                         vdev->host.bus, vdev->host.slot, vdev->host.function);
+        }
+    }
+
+    vfio_enable_intx(vdev);
+}
+
+static Property vfio_pci_dev_properties[] = {
+    DEFINE_PROP_PCI_HOST_DEVADDR("host", VFIODevice, host),
+    DEFINE_PROP_UINT32("x-intx-mmap-timeout-ms", VFIODevice,
+                       intx.mmap_timeout, 1100),
+    /*
+     * TODO - support passed fds... is this necessary?
+     * DEFINE_PROP_STRING("vfiofd", VFIODevice, vfiofd_name),
+     * DEFINE_PROP_STRING("vfiogroupfd, VFIODevice, vfiogroupfd_name),
+     */
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static const VMStateDescription vfio_pci_vmstate = {
+    .name = "vfio-pci",
+    .unmigratable = 1,
+};
+
+static void vfio_pci_dev_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    PCIDeviceClass *pdc = PCI_DEVICE_CLASS(klass);
+
+    dc->reset = vfio_pci_reset;
+    dc->props = vfio_pci_dev_properties;
+    dc->vmsd = &vfio_pci_vmstate;
+    dc->desc = "VFIO-based PCI device assignment";
+    pdc->init = vfio_initfn;
+    pdc->exit = vfio_exitfn;
+    pdc->config_read = vfio_pci_read_config;
+    pdc->config_write = vfio_pci_write_config;
+}
+
+static const TypeInfo vfio_pci_dev_info = {
+    .name = "vfio-pci",
+    .parent = TYPE_PCI_DEVICE,
+    .instance_size = sizeof(VFIODevice),
+    .class_init = vfio_pci_dev_class_init,
+};
+
+static void register_vfio_pci_dev_type(void)
+{
+    type_register_static(&vfio_pci_dev_info);
+}
+
+type_init(register_vfio_pci_dev_type)
diff --git a/hw/vga-isa-mm.c b/hw/vga-isa-mm.c
index 44ae7d92c..8ef4320d0 100644
--- a/hw/vga-isa-mm.c
+++ b/hw/vga-isa-mm.c
@@ -36,7 +36,7 @@ typedef struct ISAVGAMMState {
 } ISAVGAMMState;
 
 /* Memory mapped interface */
-static uint32_t vga_mm_readb (void *opaque, target_phys_addr_t addr)
+static uint32_t vga_mm_readb (void *opaque, hwaddr addr)
 {
     ISAVGAMMState *s = opaque;
 
@@ -44,14 +44,14 @@ static uint32_t vga_mm_readb (void *opaque, target_phys_addr_t addr)
 }
 
 static void vga_mm_writeb (void *opaque,
-                           target_phys_addr_t addr, uint32_t value)
+                           hwaddr addr, uint32_t value)
 {
     ISAVGAMMState *s = opaque;
 
     vga_ioport_write(&s->vga, addr >> s->it_shift, value & 0xff);
 }
 
-static uint32_t vga_mm_readw (void *opaque, target_phys_addr_t addr)
+static uint32_t vga_mm_readw (void *opaque, hwaddr addr)
 {
     ISAVGAMMState *s = opaque;
 
@@ -59,14 +59,14 @@ static uint32_t vga_mm_readw (void *opaque, target_phys_addr_t addr)
 }
 
 static void vga_mm_writew (void *opaque,
-                           target_phys_addr_t addr, uint32_t value)
+                           hwaddr addr, uint32_t value)
 {
     ISAVGAMMState *s = opaque;
 
     vga_ioport_write(&s->vga, addr >> s->it_shift, value & 0xffff);
 }
 
-static uint32_t vga_mm_readl (void *opaque, target_phys_addr_t addr)
+static uint32_t vga_mm_readl (void *opaque, hwaddr addr)
 {
     ISAVGAMMState *s = opaque;
 
@@ -74,7 +74,7 @@ static uint32_t vga_mm_readl (void *opaque, target_phys_addr_t addr)
 }
 
 static void vga_mm_writel (void *opaque,
-                           target_phys_addr_t addr, uint32_t value)
+                           hwaddr addr, uint32_t value)
 {
     ISAVGAMMState *s = opaque;
 
@@ -97,8 +97,8 @@ static const MemoryRegionOps vga_mm_ctrl_ops = {
     .endianness = DEVICE_NATIVE_ENDIAN,
 };
 
-static void vga_mm_init(ISAVGAMMState *s, target_phys_addr_t vram_base,
-                        target_phys_addr_t ctrl_base, int it_shift,
+static void vga_mm_init(ISAVGAMMState *s, hwaddr vram_base,
+                        hwaddr ctrl_base, int it_shift,
                         MemoryRegion *address_space)
 {
     MemoryRegion *s_ioport_ctrl, *vga_io_memory;
@@ -107,6 +107,7 @@ static void vga_mm_init(ISAVGAMMState *s, target_phys_addr_t vram_base,
     s_ioport_ctrl = g_malloc(sizeof(*s_ioport_ctrl));
     memory_region_init_io(s_ioport_ctrl, &vga_mm_ctrl_ops, s,
                           "vga-mm-ctrl", 0x100000);
+    memory_region_set_flush_coalesced(s_ioport_ctrl);
 
     vga_io_memory = g_malloc(sizeof(*vga_io_memory));
     /* XXX: endianness? */
@@ -122,8 +123,8 @@ static void vga_mm_init(ISAVGAMMState *s, target_phys_addr_t vram_base,
     memory_region_set_coalescing(vga_io_memory);
 }
 
-int isa_vga_mm_init(target_phys_addr_t vram_base,
-                    target_phys_addr_t ctrl_base, int it_shift,
+int isa_vga_mm_init(hwaddr vram_base,
+                    hwaddr ctrl_base, int it_shift,
                     MemoryRegion *address_space)
 {
     ISAVGAMMState *s;
diff --git a/hw/vga-isa.c b/hw/vga-isa.c
index d2904737b..046602b3d 100644
--- a/hw/vga-isa.c
+++ b/hw/vga-isa.c
@@ -1,6 +1,8 @@
 /*
  * QEMU ISA VGA Emulator.
  *
+ * see docs/specs/standard-vga.txt for virtual hardware specs.
+ *
  * Copyright (c) 2003 Fabrice Bellard
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
diff --git a/hw/vga-pci.c b/hw/vga-pci.c
index 9abbada8f..947e35c76 100644
--- a/hw/vga-pci.c
+++ b/hw/vga-pci.c
@@ -1,6 +1,8 @@
 /*
  * QEMU PCI VGA Emulator.
  *
+ * see docs/specs/standard-vga.txt for virtual hardware specs.
+ *
  * Copyright (c) 2003 Fabrice Bellard
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
@@ -24,15 +26,28 @@
 #include "hw.h"
 #include "console.h"
 #include "pci.h"
-#include "vga-pci.h"
 #include "vga_int.h"
 #include "pixel_ops.h"
 #include "qemu-timer.h"
 #include "loader.h"
 
+#define PCI_VGA_IOPORT_OFFSET 0x400
+#define PCI_VGA_IOPORT_SIZE   (0x3e0 - 0x3c0)
+#define PCI_VGA_BOCHS_OFFSET  0x500
+#define PCI_VGA_BOCHS_SIZE    (0x0b * 2)
+#define PCI_VGA_MMIO_SIZE     0x1000
+
+enum vga_pci_flags {
+    PCI_VGA_FLAG_ENABLE_MMIO = 1,
+};
+
 typedef struct PCIVGAState {
     PCIDevice dev;
     VGACommonState vga;
+    uint32_t flags;
+    MemoryRegion mmio;
+    MemoryRegion ioport;
+    MemoryRegion bochs;
 } PCIVGAState;
 
 static const VMStateDescription vmstate_vga_pci = {
@@ -47,36 +62,126 @@ static const VMStateDescription vmstate_vga_pci = {
     }
 };
 
-static int pci_vga_initfn(PCIDevice *dev)
+static uint64_t pci_vga_ioport_read(void *ptr, hwaddr addr,
+                                    unsigned size)
 {
-     PCIVGAState *d = DO_UPCAST(PCIVGAState, dev, dev);
-     VGACommonState *s = &d->vga;
+    PCIVGAState *d = ptr;
+    uint64_t ret = 0;
+
+    switch (size) {
+    case 1:
+        ret = vga_ioport_read(&d->vga, addr);
+        break;
+    case 2:
+        ret  = vga_ioport_read(&d->vga, addr);
+        ret |= vga_ioport_read(&d->vga, addr+1) << 8;
+        break;
+    }
+    return ret;
+}
 
-     // vga + console init
-     vga_common_init(s);
-     vga_init(s, pci_address_space(dev), pci_address_space_io(dev), true);
+static void pci_vga_ioport_write(void *ptr, hwaddr addr,
+                                 uint64_t val, unsigned size)
+{
+    PCIVGAState *d = ptr;
 
-     s->ds = graphic_console_init(s->update, s->invalidate,
-                                  s->screen_dump, s->text_update, s);
+    switch (size) {
+    case 1:
+        vga_ioport_write(&d->vga, addr + 0x3c0, val);
+        break;
+    case 2:
+        /*
+         * Update bytes in little endian order.  Allows to update
+         * indexed registers with a single word write because the
+         * index byte is updated first.
+         */
+        vga_ioport_write(&d->vga, addr + 0x3c0, val & 0xff);
+        vga_ioport_write(&d->vga, addr + 0x3c1, (val >> 8) & 0xff);
+        break;
+    }
+}
 
-     /* XXX: VGA_RAM_SIZE must be a power of two */
-     pci_register_bar(&d->dev, 0, PCI_BASE_ADDRESS_MEM_PREFETCH, &s->vram);
+static const MemoryRegionOps pci_vga_ioport_ops = {
+    .read = pci_vga_ioport_read,
+    .write = pci_vga_ioport_write,
+    .valid.min_access_size = 1,
+    .valid.max_access_size = 4,
+    .impl.min_access_size = 1,
+    .impl.max_access_size = 2,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
 
-     if (!dev->rom_bar) {
-         /* compatibility with pc-0.13 and older */
-         vga_init_vbe(s, pci_address_space(dev));
-     }
+static uint64_t pci_vga_bochs_read(void *ptr, hwaddr addr,
+                                   unsigned size)
+{
+    PCIVGAState *d = ptr;
+    int index = addr >> 1;
 
-     return 0;
+    vbe_ioport_write_index(&d->vga, 0, index);
+    return vbe_ioport_read_data(&d->vga, 0);
 }
 
-DeviceState *pci_vga_init(PCIBus *bus)
+static void pci_vga_bochs_write(void *ptr, hwaddr addr,
+                                uint64_t val, unsigned size)
 {
-    return &pci_create_simple(bus, -1, "VGA")->qdev;
+    PCIVGAState *d = ptr;
+    int index = addr >> 1;
+
+    vbe_ioport_write_index(&d->vga, 0, index);
+    vbe_ioport_write_data(&d->vga, 0, val);
+}
+
+static const MemoryRegionOps pci_vga_bochs_ops = {
+    .read = pci_vga_bochs_read,
+    .write = pci_vga_bochs_write,
+    .valid.min_access_size = 1,
+    .valid.max_access_size = 4,
+    .impl.min_access_size = 2,
+    .impl.max_access_size = 2,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+static int pci_std_vga_initfn(PCIDevice *dev)
+{
+    PCIVGAState *d = DO_UPCAST(PCIVGAState, dev, dev);
+    VGACommonState *s = &d->vga;
+
+    /* vga + console init */
+    vga_common_init(s);
+    vga_init(s, pci_address_space(dev), pci_address_space_io(dev), true);
+
+    s->ds = graphic_console_init(s->update, s->invalidate,
+                                 s->screen_dump, s->text_update, s);
+
+    /* XXX: VGA_RAM_SIZE must be a power of two */
+    pci_register_bar(&d->dev, 0, PCI_BASE_ADDRESS_MEM_PREFETCH, &s->vram);
+
+    /* mmio bar for vga register access */
+    if (d->flags & (1 << PCI_VGA_FLAG_ENABLE_MMIO)) {
+        memory_region_init(&d->mmio, "vga.mmio", 4096);
+        memory_region_init_io(&d->ioport, &pci_vga_ioport_ops, d,
+                              "vga ioports remapped", PCI_VGA_IOPORT_SIZE);
+        memory_region_init_io(&d->bochs, &pci_vga_bochs_ops, d,
+                              "bochs dispi interface", PCI_VGA_BOCHS_SIZE);
+
+        memory_region_add_subregion(&d->mmio, PCI_VGA_IOPORT_OFFSET,
+                                    &d->ioport);
+        memory_region_add_subregion(&d->mmio, PCI_VGA_BOCHS_OFFSET,
+                                    &d->bochs);
+        pci_register_bar(&d->dev, 2, PCI_BASE_ADDRESS_SPACE_MEMORY, &d->mmio);
+    }
+
+    if (!dev->rom_bar) {
+        /* compatibility with pc-0.13 and older */
+        vga_init_vbe(s, pci_address_space(dev));
+    }
+
+    return 0;
 }
 
 static Property vga_pci_properties[] = {
     DEFINE_PROP_UINT32("vgamem_mb", PCIVGAState, vga.vram_size_mb, 16),
+    DEFINE_PROP_BIT("mmio", PCIVGAState, flags, PCI_VGA_FLAG_ENABLE_MMIO, true),
     DEFINE_PROP_END_OF_LIST(),
 };
 
@@ -86,7 +191,7 @@ static void vga_class_init(ObjectClass *klass, void *data)
     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
 
     k->no_hotplug = 1;
-    k->init = pci_vga_initfn;
+    k->init = pci_std_vga_initfn;
     k->romfile = "vgabios-stdvga.bin";
     k->vendor_id = PCI_VENDOR_ID_QEMU;
     k->device_id = PCI_DEVICE_ID_QEMU_VGA;
diff --git a/hw/vga-pci.h b/hw/vga-pci.h
deleted file mode 100644
index 49abf1309..000000000
--- a/hw/vga-pci.h
+++ /dev/null
@@ -1,12 +0,0 @@
-#ifndef VGA_PCI_H
-#define VGA_PCI_H
-
-#include "qemu-common.h"
-
-/* vga-pci.c */
-DeviceState *pci_vga_init(PCIBus *bus);
-
-/* cirrus_vga.c */
-DeviceState *pci_cirrus_vga_init(PCIBus *bus);
-
-#endif
diff --git a/hw/vga.c b/hw/vga.c
index f82ced8e6..2b0200a16 100644
--- a/hw/vga.c
+++ b/hw/vga.c
@@ -166,12 +166,13 @@ static uint32_t expand4[256];
 static uint16_t expand2[256];
 static uint8_t expand4to8[16];
 
-static void vga_screen_dump(void *opaque, const char *filename, bool cswitch);
+static void vga_screen_dump(void *opaque, const char *filename, bool cswitch,
+                            Error **errp);
 
 static void vga_update_memory_access(VGACommonState *s)
 {
     MemoryRegion *region, *old_region = s->chain4_alias;
-    target_phys_addr_t base, offset, size;
+    hwaddr base, offset, size;
 
     s->chain4_alias = NULL;
 
@@ -360,6 +361,8 @@ uint32_t vga_ioport_read(void *opaque, uint32_t addr)
     VGACommonState *s = opaque;
     int val, index;
 
+    qemu_flush_coalesced_mmio_buffer();
+
     if (vga_ioport_invalid(s, addr)) {
         val = 0xff;
     } else {
@@ -452,6 +455,8 @@ void vga_ioport_write(void *opaque, uint32_t addr, uint32_t val)
     VGACommonState *s = opaque;
     int index;
 
+    qemu_flush_coalesced_mmio_buffer();
+
     /* check port range access depending on color/monochrome mode */
     if (vga_ioport_invalid(s, addr)) {
         return;
@@ -577,7 +582,6 @@ void vga_ioport_write(void *opaque, uint32_t addr, uint32_t val)
     }
 }
 
-#ifdef CONFIG_BOCHS_VBE
 static uint32_t vbe_ioport_read_index(void *opaque, uint32_t addr)
 {
     VGACommonState *s = opaque;
@@ -586,7 +590,7 @@ static uint32_t vbe_ioport_read_index(void *opaque, uint32_t addr)
     return val;
 }
 
-static uint32_t vbe_ioport_read_data(void *opaque, uint32_t addr)
+uint32_t vbe_ioport_read_data(void *opaque, uint32_t addr)
 {
     VGACommonState *s = opaque;
     uint32_t val;
@@ -622,13 +626,13 @@ static uint32_t vbe_ioport_read_data(void *opaque, uint32_t addr)
     return val;
 }
 
-static void vbe_ioport_write_index(void *opaque, uint32_t addr, uint32_t val)
+void vbe_ioport_write_index(void *opaque, uint32_t addr, uint32_t val)
 {
     VGACommonState *s = opaque;
     s->vbe_index = val;
 }
 
-static void vbe_ioport_write_data(void *opaque, uint32_t addr, uint32_t val)
+void vbe_ioport_write_data(void *opaque, uint32_t addr, uint32_t val)
 {
     VGACommonState *s = opaque;
 
@@ -779,10 +783,9 @@ static void vbe_ioport_write_data(void *opaque, uint32_t addr, uint32_t val)
         }
     }
 }
-#endif
 
 /* called for accesses between 0xa0000 and 0xc0000 */
-uint32_t vga_mem_readb(VGACommonState *s, target_phys_addr_t addr)
+uint32_t vga_mem_readb(VGACommonState *s, hwaddr addr)
 {
     int memory_map_mode, plane;
     uint32_t ret;
@@ -839,7 +842,7 @@ uint32_t vga_mem_readb(VGACommonState *s, target_phys_addr_t addr)
 }
 
 /* called for accesses between 0xa0000 and 0xc0000 */
-void vga_mem_writeb(VGACommonState *s, target_phys_addr_t addr, uint32_t val)
+void vga_mem_writeb(VGACommonState *s, hwaddr addr, uint32_t val)
 {
     int memory_map_mode, plane, write_mode, b, func_select, mask;
     uint32_t write_mask, bit_mask, set_mask;
@@ -1124,14 +1127,12 @@ static void vga_get_offsets(VGACommonState *s,
                             uint32_t *pline_compare)
 {
     uint32_t start_addr, line_offset, line_compare;
-#ifdef CONFIG_BOCHS_VBE
+
     if (s->vbe_regs[VBE_DISPI_INDEX_ENABLE] & VBE_DISPI_ENABLED) {
         line_offset = s->vbe_line_offset;
         start_addr = s->vbe_start_addr;
         line_compare = 65535;
-    } else
-#endif
-    {
+    } else {
         /* compute line_offset in bytes */
         line_offset = s->cr[VGA_CRTC_OFFSET];
         line_offset <<= 3;
@@ -1345,6 +1346,7 @@ static void vga_draw_text(VGACommonState *s, int full_update)
         s->last_scr_width = width * cw;
         s->last_scr_height = height * cheight;
         qemu_console_resize(s->ds, s->last_scr_width, s->last_scr_height);
+        dpy_text_resize(s->ds, width, height);
         s->last_depth = 0;
         s->last_width = width;
         s->last_height = height;
@@ -1358,6 +1360,14 @@ static void vga_draw_text(VGACommonState *s, int full_update)
     palette = s->last_palette;
     x_incr = cw * ((ds_get_bits_per_pixel(s->ds) + 7) >> 3);
 
+    if (full_update) {
+        s->full_update_text = 1;
+    }
+    if (s->full_update_gfx) {
+        s->full_update_gfx = 0;
+        full_update |= 1;
+    }
+
     cursor_offset = ((s->cr[VGA_CRTC_CURSOR_HI] << 8) |
                      s->cr[VGA_CRTC_CURSOR_LO]) - s->start_addr;
     if (cursor_offset != s->cursor_offset ||
@@ -1455,8 +1465,8 @@ static void vga_draw_text(VGACommonState *s, int full_update)
             ch_attr_ptr++;
         }
         if (cx_max != -1) {
-            dpy_update(s->ds, cx_min * cw, cy * cheight,
-                       (cx_max - cx_min + 1) * cw, cheight);
+            dpy_gfx_update(s->ds, cx_min * cw, cy * cheight,
+                           (cx_max - cx_min + 1) * cw, cheight);
         }
         dest += linesize * cheight;
         line1 = line + cheight;
@@ -1567,12 +1577,10 @@ static vga_draw_line_func * const vga_draw_line_table[NB_DEPTHS * VGA_DRAW_LINE_
 static int vga_get_bpp(VGACommonState *s)
 {
     int ret;
-#ifdef CONFIG_BOCHS_VBE
+
     if (s->vbe_regs[VBE_DISPI_INDEX_ENABLE] & VBE_DISPI_ENABLED) {
         ret = s->vbe_regs[VBE_DISPI_INDEX_BPP];
-    } else
-#endif
-    {
+    } else {
         ret = 0;
     }
     return ret;
@@ -1582,13 +1590,10 @@ static void vga_get_resolution(VGACommonState *s, int *pwidth, int *pheight)
 {
     int width, height;
 
-#ifdef CONFIG_BOCHS_VBE
     if (s->vbe_regs[VBE_DISPI_INDEX_ENABLE] & VBE_DISPI_ENABLED) {
         width = s->vbe_regs[VBE_DISPI_INDEX_XRES];
         height = s->vbe_regs[VBE_DISPI_INDEX_YRES];
-    } else
-#endif
-    {
+    } else {
         width = (s->cr[VGA_CRTC_H_DISP] + 1) * 8;
         height = s->cr[VGA_CRTC_V_DISP_END] |
             ((s->cr[VGA_CRTC_OVERFLOW] & 0x02) << 7) |
@@ -1611,7 +1616,7 @@ void vga_invalidate_scanlines(VGACommonState *s, int y1, int y2)
     }
 }
 
-static void vga_sync_dirty_bitmap(VGACommonState *s)
+void vga_sync_dirty_bitmap(VGACommonState *s)
 {
     memory_region_sync_dirty_bitmap(&s->vram);
 }
@@ -1692,7 +1697,7 @@ static void vga_draw_graphic(VGACommonState *s, int full_update)
 #if defined(HOST_WORDS_BIGENDIAN) != defined(TARGET_WORDS_BIGENDIAN)
             s->ds->surface->pf = qemu_different_endianness_pixelformat(depth);
 #endif
-            dpy_resize(s->ds);
+            dpy_gfx_resize(s->ds);
         } else {
             qemu_console_resize(s->ds, disp_width, height);
         }
@@ -1704,9 +1709,14 @@ static void vga_draw_graphic(VGACommonState *s, int full_update)
         s->last_depth = depth;
         full_update = 1;
     } else if (is_buffer_shared(s->ds->surface) &&
-               (full_update || s->ds->surface->data != s->vram_ptr + (s->start_addr * 4))) {
-        s->ds->surface->data = s->vram_ptr + (s->start_addr * 4);
-        dpy_setdata(s->ds);
+               (full_update || ds_get_data(s->ds) != s->vram_ptr
+                + (s->start_addr * 4))) {
+        qemu_free_displaysurface(s->ds);
+        s->ds->surface = qemu_create_displaysurface_from(disp_width,
+                height, depth,
+                s->line_offset,
+                s->vram_ptr + (s->start_addr * 4));
+        dpy_gfx_setdata(s->ds);
     }
 
     s->rgb_to_pixel =
@@ -1811,8 +1821,8 @@ static void vga_draw_graphic(VGACommonState *s, int full_update)
         } else {
             if (y_start >= 0) {
                 /* flush to display */
-                dpy_update(s->ds, 0, y_start,
-                           disp_width, y - y_start);
+                dpy_gfx_update(s->ds, 0, y_start,
+                               disp_width, y - y_start);
                 y_start = -1;
             }
         }
@@ -1832,8 +1842,8 @@ static void vga_draw_graphic(VGACommonState *s, int full_update)
     }
     if (y_start >= 0) {
         /* flush to display */
-        dpy_update(s->ds, 0, y_start,
-                   disp_width, y - y_start);
+        dpy_gfx_update(s->ds, 0, y_start,
+                       disp_width, y - y_start);
     }
     /* reset modified pages */
     if (page_max >= page_min) {
@@ -1867,8 +1877,8 @@ static void vga_draw_blank(VGACommonState *s, int full_update)
         memset(d, val, w);
         d += ds_get_linesize(s->ds);
     }
-    dpy_update(s->ds, 0, 0,
-               s->last_scr_width, s->last_scr_height);
+    dpy_gfx_update(s->ds, 0, 0,
+                   s->last_scr_width, s->last_scr_height);
 }
 
 #define GMODE_TEXT     0
@@ -1943,14 +1953,12 @@ void vga_common_reset(VGACommonState *s)
     s->dac_8bit = 0;
     memset(s->palette, '\0', sizeof(s->palette));
     s->bank_offset = 0;
-#ifdef CONFIG_BOCHS_VBE
     s->vbe_index = 0;
     memset(s->vbe_regs, '\0', sizeof(s->vbe_regs));
     s->vbe_regs[VBE_DISPI_INDEX_ID] = VBE_DISPI_ID5;
     s->vbe_start_addr = 0;
     s->vbe_line_offset = 0;
     s->vbe_bank_mask = (s->vram_size >> 16) - 1;
-#endif
     memset(s->font_offsets, '\0', sizeof(s->font_offsets));
     s->graphic_mode = -1; /* force full update */
     s->shift_control = 0;
@@ -2058,9 +2066,9 @@ static void vga_update_text(void *opaque, console_ch_t *chardata)
             cw != s->last_cw || cheight != s->last_ch) {
             s->last_scr_width = width * cw;
             s->last_scr_height = height * cheight;
-            s->ds->surface->width = width;
-            s->ds->surface->height = height;
-            dpy_resize(s->ds);
+            qemu_console_resize(s->ds, s->last_scr_width, s->last_scr_height);
+            dpy_text_resize(s->ds, width, height);
+            s->last_depth = 0;
             s->last_width = width;
             s->last_height = height;
             s->last_ch = cheight;
@@ -2068,6 +2076,14 @@ static void vga_update_text(void *opaque, console_ch_t *chardata)
             full_update = 1;
         }
 
+        if (full_update) {
+            s->full_update_gfx = 1;
+        }
+        if (s->full_update_text) {
+            s->full_update_text = 0;
+            full_update |= 1;
+        }
+
         /* Update "hardware" cursor */
         cursor_offset = ((s->cr[VGA_CRTC_CURSOR_HI] << 8) |
                          s->cr[VGA_CRTC_CURSOR_LO]) - s->start_addr;
@@ -2076,11 +2092,11 @@ static void vga_update_text(void *opaque, console_ch_t *chardata)
             s->cr[VGA_CRTC_CURSOR_END] != s->cursor_end || full_update) {
             cursor_visible = !(s->cr[VGA_CRTC_CURSOR_START] & 0x20);
             if (cursor_visible && cursor_offset < size && cursor_offset >= 0)
-                dpy_cursor(s->ds,
-                           TEXTMODE_X(cursor_offset),
-                           TEXTMODE_Y(cursor_offset));
+                dpy_text_cursor(s->ds,
+                                TEXTMODE_X(cursor_offset),
+                                TEXTMODE_Y(cursor_offset));
             else
-                dpy_cursor(s->ds, -1, -1);
+                dpy_text_cursor(s->ds, -1, -1);
             s->cursor_offset = cursor_offset;
             s->cursor_start = s->cr[VGA_CRTC_CURSOR_START];
             s->cursor_end = s->cr[VGA_CRTC_CURSOR_END];
@@ -2093,7 +2109,7 @@ static void vga_update_text(void *opaque, console_ch_t *chardata)
             for (i = 0; i < size; src ++, dst ++, i ++)
                 console_write_ch(dst, VMEM2CHTYPE(le32_to_cpu(*src)));
 
-            dpy_update(s->ds, 0, 0, width, height);
+            dpy_text_update(s->ds, 0, 0, width, height);
         } else {
             c_max = 0;
 
@@ -2116,7 +2132,7 @@ static void vga_update_text(void *opaque, console_ch_t *chardata)
 
             if (c_min <= c_max) {
                 i = TEXTMODE_Y(c_min);
-                dpy_update(s->ds, 0, i, width, TEXTMODE_Y(c_max) - i + 1);
+                dpy_text_update(s->ds, 0, i, width, TEXTMODE_Y(c_max) - i + 1);
             }
         }
 
@@ -2141,10 +2157,8 @@ static void vga_update_text(void *opaque, console_ch_t *chardata)
     /* Display a message */
     s->last_width = 60;
     s->last_height = height = 3;
-    dpy_cursor(s->ds, -1, -1);
-    s->ds->surface->width = s->last_width;
-    s->ds->surface->height = height;
-    dpy_resize(s->ds);
+    dpy_text_cursor(s->ds, -1, -1);
+    dpy_text_resize(s->ds, s->last_width, height);
 
     for (dst = chardata, i = 0; i < s->last_width * height; i ++)
         console_write_ch(dst ++, ' ');
@@ -2155,10 +2169,10 @@ static void vga_update_text(void *opaque, console_ch_t *chardata)
     for (i = 0; i < size; i ++)
         console_write_ch(dst ++, 0x00200100 | msg_buffer[i]);
 
-    dpy_update(s->ds, 0, 0, s->last_width, height);
+    dpy_text_update(s->ds, 0, 0, s->last_width, height);
 }
 
-static uint64_t vga_mem_read(void *opaque, target_phys_addr_t addr,
+static uint64_t vga_mem_read(void *opaque, hwaddr addr,
                              unsigned size)
 {
     VGACommonState *s = opaque;
@@ -2166,7 +2180,7 @@ static uint64_t vga_mem_read(void *opaque, target_phys_addr_t addr,
     return vga_mem_readb(s, addr);
 }
 
-static void vga_mem_write(void *opaque, target_phys_addr_t addr,
+static void vga_mem_write(void *opaque, hwaddr addr,
                           uint64_t data, unsigned size)
 {
     VGACommonState *s = opaque;
@@ -2224,13 +2238,11 @@ const VMStateDescription vmstate_vga_common = {
 
         VMSTATE_INT32(bank_offset, VGACommonState),
         VMSTATE_UINT8_EQUAL(is_vbe_vmstate, VGACommonState),
-#ifdef CONFIG_BOCHS_VBE
         VMSTATE_UINT16(vbe_index, VGACommonState),
         VMSTATE_UINT16_ARRAY(vbe_regs, VGACommonState, VBE_DISPI_INDEX_NB),
         VMSTATE_UINT32(vbe_start_addr, VGACommonState),
         VMSTATE_UINT32(vbe_line_offset, VGACommonState),
         VMSTATE_UINT32(vbe_bank_mask, VGACommonState),
-#endif
         VMSTATE_END_OF_LIST()
     }
 };
@@ -2270,11 +2282,7 @@ void vga_common_init(VGACommonState *s)
     }
     s->vram_size_mb = s->vram_size >> 20;
 
-#ifdef CONFIG_BOCHS_VBE
     s->is_vbe_vmstate = 1;
-#else
-    s->is_vbe_vmstate = 0;
-#endif
     memory_region_init_ram(&s->vram, "vga.vram", s->vram_size);
     vmstate_register_ram_global(&s->vram);
     xen_register_framebuffer(&s->vram);
@@ -2309,17 +2317,14 @@ static const MemoryRegionPortio vga_portio_list[] = {
     PORTIO_END_OF_LIST(),
 };
 
-#ifdef CONFIG_BOCHS_VBE
 static const MemoryRegionPortio vbe_portio_list[] = {
     { 0, 1, 2, .read = vbe_ioport_read_index, .write = vbe_ioport_write_index },
 # ifdef TARGET_I386
     { 1, 1, 2, .read = vbe_ioport_read_data, .write = vbe_ioport_write_data },
-# else
-    { 2, 1, 2, .read = vbe_ioport_read_data, .write = vbe_ioport_write_data },
 # endif
+    { 2, 1, 2, .read = vbe_ioport_read_data, .write = vbe_ioport_write_data },
     PORTIO_END_OF_LIST(),
 };
-#endif /* CONFIG_BOCHS_VBE */
 
 /* Used by both ISA and PCI */
 MemoryRegion *vga_init_io(VGACommonState *s,
@@ -2329,14 +2334,12 @@ MemoryRegion *vga_init_io(VGACommonState *s,
     MemoryRegion *vga_mem;
 
     *vga_ports = vga_portio_list;
-    *vbe_ports = NULL;
-#ifdef CONFIG_BOCHS_VBE
     *vbe_ports = vbe_portio_list;
-#endif
 
     vga_mem = g_malloc(sizeof(*vga_mem));
     memory_region_init_io(vga_mem, &vga_mem_ops, s,
                           "vga-lowmem", 0x20000);
+    memory_region_set_flush_coalesced(vga_mem);
 
     return vga_mem;
 }
@@ -2373,7 +2376,6 @@ void vga_init(VGACommonState *s, MemoryRegion *address_space,
 
 void vga_init_vbe(VGACommonState *s, MemoryRegion *system_memory)
 {
-#ifdef CONFIG_BOCHS_VBE
     /* With pc-0.12 and below we map both the PCI BAR and the fixed VBE region,
      * so use an alias to avoid double-mapping the same region.
      */
@@ -2384,58 +2386,59 @@ void vga_init_vbe(VGACommonState *s, MemoryRegion *system_memory)
                                 VBE_DISPI_LFB_PHYSICAL_ADDRESS,
                                 &s->vram_vbe);
     s->vbe_mapped = 1;
-#endif 
 }
 /********************************************************/
 /* vga screen dump */
 
-int ppm_save(const char *filename, struct DisplaySurface *ds)
+void ppm_save(const char *filename, struct DisplaySurface *ds, Error **errp)
 {
+    int width = pixman_image_get_width(ds->image);
+    int height = pixman_image_get_height(ds->image);
     FILE *f;
-    uint8_t *d, *d1;
-    uint32_t v;
-    int y, x;
-    uint8_t r, g, b;
+    int y;
     int ret;
-    char *linebuf, *pbuf;
+    pixman_image_t *linebuf;
 
     trace_ppm_save(filename, ds);
     f = fopen(filename, "wb");
-    if (!f)
-        return -1;
-    fprintf(f, "P6\n%d %d\n%d\n",
-            ds->width, ds->height, 255);
-    linebuf = g_malloc(ds->width * 3);
-    d1 = ds->data;
-    for(y = 0; y < ds->height; y++) {
-        d = d1;
-        pbuf = linebuf;
-        for(x = 0; x < ds->width; x++) {
-            if (ds->pf.bits_per_pixel == 32)
-                v = *(uint32_t *)d;
-            else
-                v = (uint32_t) (*(uint16_t *)d);
-            /* Limited to 8 or fewer bits per channel: */
-            r = ((v >> ds->pf.rshift) & ds->pf.rmax) << (8 - ds->pf.rbits);
-            g = ((v >> ds->pf.gshift) & ds->pf.gmax) << (8 - ds->pf.gbits);
-            b = ((v >> ds->pf.bshift) & ds->pf.bmax) << (8 - ds->pf.bbits);
-            *pbuf++ = r;
-            *pbuf++ = g;
-            *pbuf++ = b;
-            d += ds->pf.bytes_per_pixel;
-        }
-        d1 += ds->linesize;
-        ret = fwrite(linebuf, 1, pbuf - linebuf, f);
+    if (!f) {
+        error_setg(errp, "failed to open file '%s': %s", filename,
+                   strerror(errno));
+        return;
+    }
+    ret = fprintf(f, "P6\n%d %d\n%d\n", width, height, 255);
+    if (ret < 0) {
+        linebuf = NULL;
+        goto write_err;
+    }
+    linebuf = qemu_pixman_linebuf_create(PIXMAN_BE_r8g8b8, width);
+    for (y = 0; y < height; y++) {
+        qemu_pixman_linebuf_fill(linebuf, ds->image, width, y);
+        clearerr(f);
+        ret = fwrite(pixman_image_get_data(linebuf), 1,
+                     pixman_image_get_stride(linebuf), f);
         (void)ret;
+        if (ferror(f)) {
+            goto write_err;
+        }
     }
-    g_free(linebuf);
+
+out:
+    qemu_pixman_image_unref(linebuf);
     fclose(f);
-    return 0;
+    return;
+
+write_err:
+    error_setg(errp, "failed to write to file '%s': %s", filename,
+               strerror(errno));
+    unlink(filename);
+    goto out;
 }
 
 /* save the vga display in a PPM image even if no display is
    available */
-static void vga_screen_dump(void *opaque, const char *filename, bool cswitch)
+static void vga_screen_dump(void *opaque, const char *filename, bool cswitch,
+                            Error **errp)
 {
     VGACommonState *s = opaque;
 
@@ -2443,5 +2446,5 @@ static void vga_screen_dump(void *opaque, const char *filename, bool cswitch)
         vga_invalidate_display(s);
     }
     vga_hw_update();
-    ppm_save(filename, s->ds->surface);
+    ppm_save(filename, s->ds->surface, errp);
 }
diff --git a/hw/vga_int.h b/hw/vga_int.h
index 893809368..bcb738d8c 100644
--- a/hw/vga_int.h
+++ b/hw/vga_int.h
@@ -23,14 +23,12 @@
  */
 
 #include <hw/hw.h>
+#include "error.h"
 #include "memory.h"
 
 #define ST01_V_RETRACE      0x08
 #define ST01_DISP_ENABLE    0x01
 
-/* bochs VBE support */
-#define CONFIG_BOCHS_VBE
-
 #define VBE_DISPI_MAX_XRES              16000
 #define VBE_DISPI_MAX_YRES              12000
 #define VBE_DISPI_MAX_BPP               32
@@ -64,21 +62,6 @@
 
 #define VBE_DISPI_LFB_PHYSICAL_ADDRESS  0xE0000000
 
-#ifdef CONFIG_BOCHS_VBE
-
-#define VGA_STATE_COMMON_BOCHS_VBE              \
-    uint16_t vbe_index;                         \
-    uint16_t vbe_regs[VBE_DISPI_INDEX_NB];      \
-    uint32_t vbe_start_addr;                    \
-    uint32_t vbe_line_offset;                   \
-    uint32_t vbe_bank_mask;			\
-    int vbe_mapped;
-#else
-
-#define VGA_STATE_COMMON_BOCHS_VBE
-
-#endif /* !CONFIG_BOCHS_VBE */
-
 #define CH_ATTR_SIZE (160 * 100)
 #define VGA_MAX_HEIGHT 2048
 
@@ -139,7 +122,13 @@ typedef struct VGACommonState {
     void (*get_resolution)(struct VGACommonState *s,
                         int *pwidth,
                         int *pheight);
-    VGA_STATE_COMMON_BOCHS_VBE
+    /* bochs vbe state */
+    uint16_t vbe_index;
+    uint16_t vbe_regs[VBE_DISPI_INDEX_NB];
+    uint32_t vbe_start_addr;
+    uint32_t vbe_line_offset;
+    uint32_t vbe_bank_mask;
+    int vbe_mapped;
     /* display refresh support */
     DisplayState *ds;
     uint32_t font_offsets[2];
@@ -165,6 +154,8 @@ typedef struct VGACommonState {
     vga_hw_invalidate_ptr invalidate;
     vga_hw_screen_dump_ptr screen_dump;
     vga_hw_text_update_ptr text_update;
+    bool full_update_text;
+    bool full_update_gfx;
     /* hardware mouse cursor support */
     uint32_t invalidated_y_table[VGA_MAX_HEIGHT / 32];
     void (*cursor_invalidate)(struct VGACommonState *s);
@@ -195,19 +186,24 @@ MemoryRegion *vga_init_io(VGACommonState *s,
                           const MemoryRegionPortio **vbe_ports);
 void vga_common_reset(VGACommonState *s);
 
+void vga_sync_dirty_bitmap(VGACommonState *s);
 void vga_dirty_log_start(VGACommonState *s);
 void vga_dirty_log_stop(VGACommonState *s);
 
 extern const VMStateDescription vmstate_vga_common;
 uint32_t vga_ioport_read(void *opaque, uint32_t addr);
 void vga_ioport_write(void *opaque, uint32_t addr, uint32_t val);
-uint32_t vga_mem_readb(VGACommonState *s, target_phys_addr_t addr);
-void vga_mem_writeb(VGACommonState *s, target_phys_addr_t addr, uint32_t val);
+uint32_t vga_mem_readb(VGACommonState *s, hwaddr addr);
+void vga_mem_writeb(VGACommonState *s, hwaddr addr, uint32_t val);
 void vga_invalidate_scanlines(VGACommonState *s, int y1, int y2);
-int ppm_save(const char *filename, struct DisplaySurface *ds);
+void ppm_save(const char *filename, struct DisplaySurface *ds, Error **errp);
 
 int vga_ioport_invalid(VGACommonState *s, uint32_t addr);
+
 void vga_init_vbe(VGACommonState *s, MemoryRegion *address_space);
+uint32_t vbe_ioport_read_data(void *opaque, uint32_t addr);
+void vbe_ioport_write_index(void *opaque, uint32_t addr, uint32_t val);
+void vbe_ioport_write_data(void *opaque, uint32_t addr, uint32_t val);
 
 extern const uint8_t sr_mask[8];
 extern const uint8_t gr_mask[16];
diff --git a/hw/vhost.c b/hw/vhost.c
index 0fd8da84e..16322a14f 100644
--- a/hw/vhost.c
+++ b/hw/vhost.c
@@ -65,8 +65,8 @@ static void vhost_dev_sync_region(struct vhost_dev *dev,
 
 static int vhost_sync_dirty_bitmap(struct vhost_dev *dev,
                                    MemoryRegionSection *section,
-                                   target_phys_addr_t start_addr,
-                                   target_phys_addr_t end_addr)
+                                   hwaddr start_addr,
+                                   hwaddr end_addr)
 {
     int i;
 
@@ -93,8 +93,8 @@ static void vhost_log_sync(MemoryListener *listener,
 {
     struct vhost_dev *dev = container_of(listener, struct vhost_dev,
                                          memory_listener);
-    target_phys_addr_t start_addr = section->offset_within_address_space;
-    target_phys_addr_t end_addr = start_addr + section->size;
+    hwaddr start_addr = section->offset_within_address_space;
+    hwaddr end_addr = start_addr + section->size;
 
     vhost_sync_dirty_bitmap(dev, section, start_addr, end_addr);
 }
@@ -296,7 +296,7 @@ static int vhost_verify_ring_mappings(struct vhost_dev *dev,
     int i;
     for (i = 0; i < dev->nvqs; ++i) {
         struct vhost_virtqueue *vq = dev->vqs + i;
-        target_phys_addr_t l;
+        hwaddr l;
         void *p;
 
         if (!ranges_overlap(start_addr, size, vq->ring_phys, vq->ring_size)) {
@@ -362,7 +362,7 @@ static void vhost_set_memory(MemoryListener *listener,
 {
     struct vhost_dev *dev = container_of(listener, struct vhost_dev,
                                          memory_listener);
-    target_phys_addr_t start_addr = section->offset_within_address_space;
+    hwaddr start_addr = section->offset_within_address_space;
     ram_addr_t size = section->size;
     bool log_dirty = memory_region_is_logging(section->mr);
     int s = offsetof(struct vhost_memory, regions) +
@@ -434,8 +434,7 @@ static void vhost_set_memory(MemoryListener *listener,
 
 static bool vhost_section(MemoryRegionSection *section)
 {
-    return section->address_space == get_system_memory()
-        && memory_region_is_ram(section->mr);
+    return memory_region_is_ram(section->mr);
 }
 
 static void vhost_begin(MemoryListener *listener)
@@ -618,7 +617,7 @@ static int vhost_virtqueue_init(struct vhost_dev *dev,
                                 struct vhost_virtqueue *vq,
                                 unsigned idx)
 {
-    target_phys_addr_t s, l, a;
+    hwaddr s, l, a;
     int r;
     struct vhost_vring_file file = {
         .index = idx,
@@ -747,14 +746,15 @@ static void vhost_eventfd_del(MemoryListener *listener,
 {
 }
 
-int vhost_dev_init(struct vhost_dev *hdev, int devfd, bool force)
+int vhost_dev_init(struct vhost_dev *hdev, int devfd, const char *devpath,
+                   bool force)
 {
     uint64_t features;
     int r;
     if (devfd >= 0) {
         hdev->control = devfd;
     } else {
-        hdev->control = open("/dev/vhost-net", O_RDWR);
+        hdev->control = open(devpath, O_RDWR);
         if (hdev->control < 0) {
             return -errno;
         }
@@ -792,7 +792,7 @@ int vhost_dev_init(struct vhost_dev *hdev, int devfd, bool force)
     hdev->log_size = 0;
     hdev->log_enabled = false;
     hdev->started = false;
-    memory_listener_register(&hdev->memory_listener, NULL);
+    memory_listener_register(&hdev->memory_listener, &address_space_memory);
     hdev->force = force;
     return 0;
 fail:
@@ -948,7 +948,7 @@ void vhost_dev_stop(struct vhost_dev *hdev, VirtIODevice *vdev)
     }
     for (i = 0; i < hdev->n_mem_sections; ++i) {
         vhost_sync_dirty_bitmap(hdev, &hdev->mem_sections[i],
-                                0, (target_phys_addr_t)~0x0ull);
+                                0, (hwaddr)~0x0ull);
     }
     r = vdev->binding->set_guest_notifiers(vdev->binding_opaque, false);
     if (r < 0) {
diff --git a/hw/vhost.h b/hw/vhost.h
index 80e64df86..0c47229f9 100644
--- a/hw/vhost.h
+++ b/hw/vhost.h
@@ -44,7 +44,8 @@ struct vhost_dev {
     bool force;
 };
 
-int vhost_dev_init(struct vhost_dev *hdev, int devfd, bool force);
+int vhost_dev_init(struct vhost_dev *hdev, int devfd, const char *devpath,
+                   bool force);
 void vhost_dev_cleanup(struct vhost_dev *hdev);
 bool vhost_dev_query(struct vhost_dev *hdev, VirtIODevice *vdev);
 int vhost_dev_start(struct vhost_dev *hdev, VirtIODevice *vdev);
diff --git a/hw/vhost_net.c b/hw/vhost_net.c
index ecaa22dfb..824160153 100644
--- a/hw/vhost_net.c
+++ b/hw/vhost_net.c
@@ -109,7 +109,7 @@ struct vhost_net *vhost_net_init(NetClientState *backend, int devfd,
         (1 << VHOST_NET_F_VIRTIO_NET_HDR);
     net->backend = r;
 
-    r = vhost_dev_init(&net->dev, devfd, force);
+    r = vhost_dev_init(&net->dev, devfd, "/dev/vhost-net", force);
     if (r < 0) {
         goto fail;
     }
@@ -150,10 +150,6 @@ int vhost_net_start(struct vhost_net *net,
     if (r < 0) {
         goto fail_notifiers;
     }
-    if (net->dev.acked_features & (1 << VIRTIO_NET_F_MRG_RXBUF)) {
-        tap_set_vnet_hdr_len(net->nc,
-                             sizeof(struct virtio_net_hdr_mrg_rxbuf));
-    }
 
     r = vhost_dev_start(&net->dev, dev);
     if (r < 0) {
@@ -179,9 +175,6 @@ fail:
     }
     net->nc->info->poll(net->nc, true);
     vhost_dev_stop(&net->dev, dev);
-    if (net->dev.acked_features & (1 << VIRTIO_NET_F_MRG_RXBUF)) {
-        tap_set_vnet_hdr_len(net->nc, sizeof(struct virtio_net_hdr));
-    }
 fail_start:
     vhost_dev_disable_notifiers(&net->dev, dev);
 fail_notifiers:
@@ -199,18 +192,12 @@ void vhost_net_stop(struct vhost_net *net,
     }
     net->nc->info->poll(net->nc, true);
     vhost_dev_stop(&net->dev, dev);
-    if (net->dev.acked_features & (1 << VIRTIO_NET_F_MRG_RXBUF)) {
-        tap_set_vnet_hdr_len(net->nc, sizeof(struct virtio_net_hdr));
-    }
     vhost_dev_disable_notifiers(&net->dev, dev);
 }
 
 void vhost_net_cleanup(struct vhost_net *net)
 {
     vhost_dev_cleanup(&net->dev);
-    if (net->dev.acked_features & (1 << VIRTIO_NET_F_MRG_RXBUF)) {
-        tap_set_vnet_hdr_len(net->nc, sizeof(struct virtio_net_hdr));
-    }
     g_free(net);
 }
 #else
diff --git a/hw/virtex_ml507.c b/hw/virtex_ml507.c
index 79bc0d10e..6ab8fee0c 100644
--- a/hw/virtex_ml507.c
+++ b/hw/virtex_ml507.c
@@ -24,7 +24,7 @@
 
 #include "sysbus.h"
 #include "hw.h"
-#include "pc.h"
+#include "serial.h"
 #include "net.h"
 #include "flash.h"
 #include "sysemu.h"
@@ -58,7 +58,7 @@ static struct boot_info
 /* Create reset TLB entries for BookE, spanning the 32bit addr space.  */
 static void mmubooke_create_initial_mapping(CPUPPCState *env,
                                      target_ulong va,
-                                     target_phys_addr_t pa)
+                                     hwaddr pa)
 {
     ppcemb_tlb_t *tlb = &env->tlb.tlbe[0];
 
@@ -134,10 +134,10 @@ static void main_cpu_reset(void *opaque)
 }
 
 #define BINARY_DEVICE_TREE_FILE "virtex-ml507.dtb"
-static int xilinx_load_device_tree(target_phys_addr_t addr,
+static int xilinx_load_device_tree(hwaddr addr,
                                       uint32_t ramsize,
-                                      target_phys_addr_t initrd_base,
-                                      target_phys_addr_t initrd_size,
+                                      hwaddr initrd_base,
+                                      hwaddr initrd_size,
                                       const char *kernel_cmdline)
 {
     char *path;
@@ -183,17 +183,17 @@ static int xilinx_load_device_tree(target_phys_addr_t addr,
     return fdt_size;
 }
 
-static void virtex_init(ram_addr_t ram_size,
-                        const char *boot_device,
-                        const char *kernel_filename,
-                        const char *kernel_cmdline,
-                        const char *initrd_filename, const char *cpu_model)
+static void virtex_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t ram_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
     MemoryRegion *address_space_mem = get_system_memory();
     DeviceState *dev;
     PowerPCCPU *cpu;
     CPUPPCState *env;
-    target_phys_addr_t ram_base = 0;
+    hwaddr ram_base = 0;
     DriveInfo *dinfo;
     MemoryRegion *phys_ram = g_new(MemoryRegion, 1);
     qemu_irq irq[32], *cpu_irq;
@@ -233,7 +233,7 @@ static void virtex_init(ram_addr_t ram_size,
 
     if (kernel_filename) {
         uint64_t entry, low, high;
-        target_phys_addr_t boot_offset;
+        hwaddr boot_offset;
 
         /* Boots a kernel elf binary.  */
         kernel_size = load_elf(kernel_filename, NULL, NULL,
diff --git a/hw/virtio-blk.c b/hw/virtio-blk.c
index 6f6d172fd..e25cc9647 100644
--- a/hw/virtio-blk.c
+++ b/hw/virtio-blk.c
@@ -64,31 +64,22 @@ static void virtio_blk_req_complete(VirtIOBlockReq *req, int status)
 }
 
 static int virtio_blk_handle_rw_error(VirtIOBlockReq *req, int error,
-    int is_read)
+    bool is_read)
 {
-    BlockErrorAction action = bdrv_get_on_error(req->dev->bs, is_read);
+    BlockErrorAction action = bdrv_get_error_action(req->dev->bs, is_read, error);
     VirtIOBlock *s = req->dev;
 
-    if (action == BLOCK_ERR_IGNORE) {
-        bdrv_emit_qmp_error_event(s->bs, BDRV_ACTION_IGNORE, is_read);
-        return 0;
-    }
-
-    if ((error == ENOSPC && action == BLOCK_ERR_STOP_ENOSPC)
-            || action == BLOCK_ERR_STOP_ANY) {
+    if (action == BDRV_ACTION_STOP) {
         req->next = s->rq;
         s->rq = req;
-        bdrv_emit_qmp_error_event(s->bs, BDRV_ACTION_STOP, is_read);
-        vm_stop(RUN_STATE_IO_ERROR);
-        bdrv_iostatus_set_err(s->bs, error);
-    } else {
+    } else if (action == BDRV_ACTION_REPORT) {
         virtio_blk_req_complete(req, VIRTIO_BLK_S_IOERR);
         bdrv_acct_done(s->bs, &req->acct);
         g_free(req);
-        bdrv_emit_qmp_error_event(s->bs, BDRV_ACTION_REPORT, is_read);
     }
 
-    return 1;
+    bdrv_error_action(s->bs, action, is_read, error);
+    return action != BDRV_ACTION_IGNORE;
 }
 
 static void virtio_blk_rw_complete(void *opaque, int ret)
@@ -98,7 +89,7 @@ static void virtio_blk_rw_complete(void *opaque, int ret)
     trace_virtio_blk_rw_complete(req, ret);
 
     if (ret) {
-        int is_read = !(ldl_p(&req->out->type) & VIRTIO_BLK_T_OUT);
+        bool is_read = !(ldl_p(&req->out->type) & VIRTIO_BLK_T_OUT);
         if (virtio_blk_handle_rw_error(req, -ret, is_read))
             return;
     }
diff --git a/hw/virtio-net.c b/hw/virtio-net.c
index b1998b27d..108ce07cf 100644
--- a/hw/virtio-net.c
+++ b/hw/virtio-net.c
@@ -41,6 +41,8 @@ typedef struct VirtIONet
     int32_t tx_burst;
     int tx_waiting;
     uint32_t has_vnet_hdr;
+    size_t host_hdr_len;
+    size_t guest_hdr_len;
     uint8_t has_ufo;
     struct {
         VirtQueueElement elem;
@@ -200,16 +202,19 @@ static void virtio_net_reset(VirtIODevice *vdev)
     memset(n->vlans, 0, MAX_VLAN >> 3);
 }
 
-static int peer_has_vnet_hdr(VirtIONet *n)
+static void peer_test_vnet_hdr(VirtIONet *n)
 {
     if (!n->nic->nc.peer)
-        return 0;
+        return;
 
     if (n->nic->nc.peer->info->type != NET_CLIENT_OPTIONS_KIND_TAP)
-        return 0;
+        return;
 
     n->has_vnet_hdr = tap_has_vnet_hdr(n->nic->nc.peer);
+}
 
+static int peer_has_vnet_hdr(VirtIONet *n)
+{
     return n->has_vnet_hdr;
 }
 
@@ -223,15 +228,27 @@ static int peer_has_ufo(VirtIONet *n)
     return n->has_ufo;
 }
 
+static void virtio_net_set_mrg_rx_bufs(VirtIONet *n, int mergeable_rx_bufs)
+{
+    n->mergeable_rx_bufs = mergeable_rx_bufs;
+
+    n->guest_hdr_len = n->mergeable_rx_bufs ?
+        sizeof(struct virtio_net_hdr_mrg_rxbuf) : sizeof(struct virtio_net_hdr);
+
+    if (peer_has_vnet_hdr(n) &&
+        tap_has_vnet_hdr_len(n->nic->nc.peer, n->guest_hdr_len)) {
+        tap_set_vnet_hdr_len(n->nic->nc.peer, n->guest_hdr_len);
+        n->host_hdr_len = n->guest_hdr_len;
+    }
+}
+
 static uint32_t virtio_net_get_features(VirtIODevice *vdev, uint32_t features)
 {
     VirtIONet *n = to_virtio_net(vdev);
 
     features |= (1 << VIRTIO_NET_F_MAC);
 
-    if (peer_has_vnet_hdr(n)) {
-        tap_using_vnet_hdr(n->nic->nc.peer, 1);
-    } else {
+    if (!peer_has_vnet_hdr(n)) {
         features &= ~(0x1 << VIRTIO_NET_F_CSUM);
         features &= ~(0x1 << VIRTIO_NET_F_HOST_TSO4);
         features &= ~(0x1 << VIRTIO_NET_F_HOST_TSO6);
@@ -277,7 +294,7 @@ static void virtio_net_set_features(VirtIODevice *vdev, uint32_t features)
 {
     VirtIONet *n = to_virtio_net(vdev);
 
-    n->mergeable_rx_bufs = !!(features & (1 << VIRTIO_NET_F_MRG_RXBUF));
+    virtio_net_set_mrg_rx_bufs(n, !!(features & (1 << VIRTIO_NET_F_MRG_RXBUF)));
 
     if (n->has_vnet_hdr) {
         tap_set_offload(n->nic->nc.peer,
@@ -447,10 +464,6 @@ static void virtio_net_handle_rx(VirtIODevice *vdev, VirtQueue *vq)
     VirtIONet *n = to_virtio_net(vdev);
 
     qemu_flush_queued_packets(&n->nic->nc);
-
-    /* We now have RX buffers, signal to the IO thread to break out of the
-     * select to re-poll the tap file descriptor */
-    qemu_notify_event();
 }
 
 static int virtio_net_can_receive(NetClientState *nc)
@@ -503,41 +516,34 @@ static int virtio_net_has_buffers(VirtIONet *n, int bufsize)
  * cache.
  */
 static void work_around_broken_dhclient(struct virtio_net_hdr *hdr,
-                                        const uint8_t *buf, size_t size)
+                                        uint8_t *buf, size_t size)
 {
     if ((hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) && /* missing csum */
         (size > 27 && size < 1500) && /* normal sized MTU */
         (buf[12] == 0x08 && buf[13] == 0x00) && /* ethertype == IPv4 */
         (buf[23] == 17) && /* ip.protocol == UDP */
         (buf[34] == 0 && buf[35] == 67)) { /* udp.srcport == bootps */
-        /* FIXME this cast is evil */
-        net_checksum_calculate((uint8_t *)buf, size);
+        net_checksum_calculate(buf, size);
         hdr->flags &= ~VIRTIO_NET_HDR_F_NEEDS_CSUM;
     }
 }
 
-static int receive_header(VirtIONet *n, struct iovec *iov, int iovcnt,
-                          const void *buf, size_t size, size_t hdr_len)
+static void receive_header(VirtIONet *n, const struct iovec *iov, int iov_cnt,
+                           const void *buf, size_t size)
 {
-    struct virtio_net_hdr *hdr = (struct virtio_net_hdr *)iov[0].iov_base;
-    int offset = 0;
-
-    hdr->flags = 0;
-    hdr->gso_type = VIRTIO_NET_HDR_GSO_NONE;
-
     if (n->has_vnet_hdr) {
-        memcpy(hdr, buf, sizeof(*hdr));
-        offset = sizeof(*hdr);
-        work_around_broken_dhclient(hdr, buf + offset, size - offset);
+        /* FIXME this cast is evil */
+        void *wbuf = (void *)buf;
+        work_around_broken_dhclient(wbuf, wbuf + n->host_hdr_len,
+                                    size - n->host_hdr_len);
+        iov_from_buf(iov, iov_cnt, 0, buf, sizeof(struct virtio_net_hdr));
+    } else {
+        struct virtio_net_hdr hdr = {
+            .flags = 0,
+            .gso_type = VIRTIO_NET_HDR_GSO_NONE
+        };
+        iov_from_buf(iov, iov_cnt, 0, &hdr, sizeof hdr);
     }
-
-    /* We only ever receive a struct virtio_net_hdr from the tapfd,
-     * but we may be passing along a larger header to the guest.
-     */
-    iov[0].iov_base += hdr_len;
-    iov[0].iov_len  -= hdr_len;
-
-    return offset;
 }
 
 static int receive_filter(VirtIONet *n, const uint8_t *buf, int size)
@@ -550,9 +556,7 @@ static int receive_filter(VirtIONet *n, const uint8_t *buf, int size)
     if (n->promisc)
         return 1;
 
-    if (n->has_vnet_hdr) {
-        ptr += sizeof(struct virtio_net_hdr);
-    }
+    ptr += n->host_hdr_len;
 
     if (!memcmp(&ptr[12], vlan, sizeof(vlan))) {
         int vid = be16_to_cpup((uint16_t *)(ptr + 14)) & 0xfff;
@@ -596,19 +600,16 @@ static int receive_filter(VirtIONet *n, const uint8_t *buf, int size)
 static ssize_t virtio_net_receive(NetClientState *nc, const uint8_t *buf, size_t size)
 {
     VirtIONet *n = DO_UPCAST(NICState, nc, nc)->opaque;
-    struct virtio_net_hdr_mrg_rxbuf *mhdr = NULL;
-    size_t guest_hdr_len, offset, i, host_hdr_len;
+    struct iovec mhdr_sg[VIRTQUEUE_MAX_SIZE];
+    struct virtio_net_hdr_mrg_rxbuf mhdr;
+    unsigned mhdr_cnt = 0;
+    size_t offset, i, guest_offset;
 
     if (!virtio_net_can_receive(&n->nic->nc))
         return -1;
 
     /* hdr_len refers to the header we supply to the guest */
-    guest_hdr_len = n->mergeable_rx_bufs ?
-        sizeof(struct virtio_net_hdr_mrg_rxbuf) : sizeof(struct virtio_net_hdr);
-
-
-    host_hdr_len = n->has_vnet_hdr ? sizeof(struct virtio_net_hdr) : 0;
-    if (!virtio_net_has_buffers(n, size + guest_hdr_len - host_hdr_len))
+    if (!virtio_net_has_buffers(n, size + n->guest_hdr_len - n->host_hdr_len))
         return 0;
 
     if (!receive_filter(n, buf, size))
@@ -619,7 +620,7 @@ static ssize_t virtio_net_receive(NetClientState *nc, const uint8_t *buf, size_t
     while (offset < size) {
         VirtQueueElement elem;
         int len, total;
-        struct iovec sg[VIRTQUEUE_MAX_SIZE];
+        const struct iovec *sg = elem.in_sg;
 
         total = 0;
 
@@ -630,7 +631,7 @@ static ssize_t virtio_net_receive(NetClientState *nc, const uint8_t *buf, size_t
                     "i %zd mergeable %d offset %zd, size %zd, "
                     "guest hdr len %zd, host hdr len %zd guest features 0x%x",
                     i, n->mergeable_rx_bufs, offset, size,
-                    guest_hdr_len, host_hdr_len, n->vdev.guest_features);
+                    n->guest_hdr_len, n->host_hdr_len, n->vdev.guest_features);
             exit(1);
         }
 
@@ -639,24 +640,25 @@ static ssize_t virtio_net_receive(NetClientState *nc, const uint8_t *buf, size_t
             exit(1);
         }
 
-        if (!n->mergeable_rx_bufs && elem.in_sg[0].iov_len != guest_hdr_len) {
-            error_report("virtio-net header not in first element");
-            exit(1);
-        }
-
-        memcpy(&sg, &elem.in_sg[0], sizeof(sg[0]) * elem.in_num);
-
         if (i == 0) {
-            if (n->mergeable_rx_bufs)
-                mhdr = (struct virtio_net_hdr_mrg_rxbuf *)sg[0].iov_base;
+            assert(offset == 0);
+            if (n->mergeable_rx_bufs) {
+                mhdr_cnt = iov_copy(mhdr_sg, ARRAY_SIZE(mhdr_sg),
+                                    sg, elem.in_num,
+                                    offsetof(typeof(mhdr), num_buffers),
+                                    sizeof(mhdr.num_buffers));
+            }
 
-            offset += receive_header(n, sg, elem.in_num,
-                                     buf + offset, size - offset, guest_hdr_len);
-            total += guest_hdr_len;
+            receive_header(n, sg, elem.in_num, buf, size);
+            offset = n->host_hdr_len;
+            total += n->guest_hdr_len;
+            guest_offset = n->guest_hdr_len;
+        } else {
+            guest_offset = 0;
         }
 
         /* copy in packet.  ugh */
-        len = iov_from_buf(sg, elem.in_num, 0,
+        len = iov_from_buf(sg, elem.in_num, guest_offset,
                            buf + offset, size - offset);
         total += len;
         offset += len;
@@ -669,7 +671,7 @@ static ssize_t virtio_net_receive(NetClientState *nc, const uint8_t *buf, size_t
                          "i %zd mergeable %d offset %zd, size %zd, "
                          "guest hdr len %zd, host hdr len %zd",
                          i, n->mergeable_rx_bufs,
-                         offset, size, guest_hdr_len, host_hdr_len);
+                         offset, size, n->guest_hdr_len, n->host_hdr_len);
 #endif
             return size;
         }
@@ -678,8 +680,11 @@ static ssize_t virtio_net_receive(NetClientState *nc, const uint8_t *buf, size_t
         virtqueue_fill(n->rx_vq, &elem, total, i++);
     }
 
-    if (mhdr) {
-        stw_p(&mhdr->num_buffers, i);
+    if (mhdr_cnt) {
+        stw_p(&mhdr.num_buffers, i);
+        iov_from_buf(mhdr_sg, mhdr_cnt,
+                     0,
+                     &mhdr.num_buffers, sizeof mhdr.num_buffers);
     }
 
     virtqueue_flush(n->rx_vq, i);
@@ -694,7 +699,7 @@ static void virtio_net_tx_complete(NetClientState *nc, ssize_t len)
 {
     VirtIONet *n = DO_UPCAST(NICState, nc, nc)->opaque;
 
-    virtqueue_push(n->tx_vq, &n->async_tx.elem, n->async_tx.len);
+    virtqueue_push(n->tx_vq, &n->async_tx.elem, 0);
     virtio_notify(&n->vdev, n->tx_vq);
 
     n->async_tx.elem.out_num = n->async_tx.len = 0;
@@ -720,33 +725,35 @@ static int32_t virtio_net_flush_tx(VirtIONet *n, VirtQueue *vq)
     }
 
     while (virtqueue_pop(vq, &elem)) {
-        ssize_t ret, len = 0;
+        ssize_t ret, len;
         unsigned int out_num = elem.out_num;
         struct iovec *out_sg = &elem.out_sg[0];
-        unsigned hdr_len;
-
-        /* hdr_len refers to the header received from the guest */
-        hdr_len = n->mergeable_rx_bufs ?
-            sizeof(struct virtio_net_hdr_mrg_rxbuf) :
-            sizeof(struct virtio_net_hdr);
+        struct iovec sg[VIRTQUEUE_MAX_SIZE];
 
-        if (out_num < 1 || out_sg->iov_len != hdr_len) {
+        if (out_num < 1) {
             error_report("virtio-net header not in first element");
             exit(1);
         }
 
-        /* ignore the header if GSO is not supported */
-        if (!n->has_vnet_hdr) {
-            out_num--;
-            out_sg++;
-            len += hdr_len;
-        } else if (n->mergeable_rx_bufs) {
-            /* tapfd expects a struct virtio_net_hdr */
-            hdr_len -= sizeof(struct virtio_net_hdr);
-            out_sg->iov_len -= hdr_len;
-            len += hdr_len;
+        /*
+         * If host wants to see the guest header as is, we can
+         * pass it on unchanged. Otherwise, copy just the parts
+         * that host is interested in.
+         */
+        assert(n->host_hdr_len <= n->guest_hdr_len);
+        if (n->host_hdr_len != n->guest_hdr_len) {
+            unsigned sg_num = iov_copy(sg, ARRAY_SIZE(sg),
+                                       out_sg, out_num,
+                                       0, n->host_hdr_len);
+            sg_num += iov_copy(sg + sg_num, ARRAY_SIZE(sg) - sg_num,
+                             out_sg, out_num,
+                             n->guest_hdr_len, -1);
+            out_num = sg_num;
+            out_sg = sg;
         }
 
+        len = n->guest_hdr_len;
+
         ret = qemu_sendv_packet_async(&n->nic->nc, out_sg, out_num,
                                       virtio_net_tx_complete);
         if (ret == 0) {
@@ -758,7 +765,7 @@ static int32_t virtio_net_flush_tx(VirtIONet *n, VirtQueue *vq)
 
         len += ret;
 
-        virtqueue_push(vq, &elem, len);
+        virtqueue_push(vq, &elem, 0);
         virtio_notify(&n->vdev, vq);
 
         if (++num_packets >= n->tx_burst) {
@@ -903,7 +910,8 @@ static int virtio_net_load(QEMUFile *f, void *opaque, int version_id)
 
     qemu_get_buffer(f, n->mac, ETH_ALEN);
     n->tx_waiting = qemu_get_be32(f);
-    n->mergeable_rx_bufs = qemu_get_be32(f);
+
+    virtio_net_set_mrg_rx_bufs(n, qemu_get_be32(f));
 
     if (version_id >= 3)
         n->status = qemu_get_be16(f);
@@ -925,7 +933,9 @@ static int virtio_net_load(QEMUFile *f, void *opaque, int version_id)
             qemu_get_buffer(f, n->mac_table.macs,
                             n->mac_table.in_use * ETH_ALEN);
         } else if (n->mac_table.in_use) {
-            qemu_fseek(f, n->mac_table.in_use * ETH_ALEN, SEEK_CUR);
+            uint8_t *buf = g_malloc0(n->mac_table.in_use);
+            qemu_get_buffer(f, buf, n->mac_table.in_use * ETH_ALEN);
+            g_free(buf);
             n->mac_table.multi_overflow = n->mac_table.uni_overflow = 1;
             n->mac_table.in_use = 0;
         }
@@ -941,7 +951,6 @@ static int virtio_net_load(QEMUFile *f, void *opaque, int version_id)
         }
 
         if (n->has_vnet_hdr) {
-            tap_using_vnet_hdr(n->nic->nc.peer, 1);
             tap_set_offload(n->nic->nc.peer,
                     (n->vdev.guest_features >> VIRTIO_NET_F_GUEST_CSUM) & 1,
                     (n->vdev.guest_features >> VIRTIO_NET_F_GUEST_TSO4) & 1,
@@ -977,6 +986,11 @@ static int virtio_net_load(QEMUFile *f, void *opaque, int version_id)
         }
     }
     n->mac_table.first_multi = i;
+
+    /* nc.link_down can't be migrated, so infer link_down according
+     * to link status bit in n->status */
+    n->nic->nc.link_down = (n->status & VIRTIO_NET_S_LINK_UP) == 0;
+
     return 0;
 }
 
@@ -1035,12 +1049,19 @@ VirtIODevice *virtio_net_init(DeviceState *dev, NICConf *conf,
     n->status = VIRTIO_NET_S_LINK_UP;
 
     n->nic = qemu_new_nic(&net_virtio_info, conf, object_get_typename(OBJECT(dev)), dev->id, n);
+    peer_test_vnet_hdr(n);
+    if (peer_has_vnet_hdr(n)) {
+        tap_using_vnet_hdr(n->nic->nc.peer, 1);
+        n->host_hdr_len = sizeof(struct virtio_net_hdr);
+    } else {
+        n->host_hdr_len = 0;
+    }
 
     qemu_format_nic_info_str(&n->nic->nc, conf->macaddr.a);
 
     n->tx_waiting = 0;
     n->tx_burst = net->txburst;
-    n->mergeable_rx_bufs = 0;
+    virtio_net_set_mrg_rx_bufs(n, 0);
     n->promisc = 1; /* for compatibility */
 
     n->mac_table.macs = g_malloc0(MAC_TABLE_ENTRIES * ETH_ALEN);
diff --git a/hw/virtio-pci.c b/hw/virtio-pci.c
index b3f0710f3..71f4fb5dc 100644
--- a/hw/virtio-pci.c
+++ b/hw/virtio-pci.c
@@ -255,7 +255,7 @@ static void virtio_ioport_write(void *opaque, uint32_t addr, uint32_t val)
 {
     VirtIOPCIProxy *proxy = opaque;
     VirtIODevice *vdev = proxy->vdev;
-    target_phys_addr_t pa;
+    hwaddr pa;
 
     switch (addr) {
     case VIRTIO_PCI_GUEST_FEATURES:
@@ -266,7 +266,7 @@ static void virtio_ioport_write(void *opaque, uint32_t addr, uint32_t val)
         virtio_set_features(vdev, val);
         break;
     case VIRTIO_PCI_QUEUE_PFN:
-        pa = (target_phys_addr_t)val << VIRTIO_PCI_QUEUE_ADDR_SHIFT;
+        pa = (hwaddr)val << VIRTIO_PCI_QUEUE_ADDR_SHIFT;
         if (pa == 0) {
             virtio_pci_stop_ioeventfd(proxy);
             virtio_reset(proxy->vdev);
@@ -374,79 +374,39 @@ static uint32_t virtio_ioport_read(VirtIOPCIProxy *proxy, uint32_t addr)
     return ret;
 }
 
-static uint32_t virtio_pci_config_readb(void *opaque, uint32_t addr)
-{
-    VirtIOPCIProxy *proxy = opaque;
-    uint32_t config = VIRTIO_PCI_CONFIG(&proxy->pci_dev);
-    if (addr < config)
-        return virtio_ioport_read(proxy, addr);
-    addr -= config;
-    return virtio_config_readb(proxy->vdev, addr);
-}
-
-static uint32_t virtio_pci_config_readw(void *opaque, uint32_t addr)
-{
-    VirtIOPCIProxy *proxy = opaque;
-    uint32_t config = VIRTIO_PCI_CONFIG(&proxy->pci_dev);
-    uint16_t val;
-    if (addr < config)
-        return virtio_ioport_read(proxy, addr);
-    addr -= config;
-    val = virtio_config_readw(proxy->vdev, addr);
-    if (virtio_is_big_endian()) {
-        /*
-         * virtio is odd, ioports are LE but config space is target native
-         * endian. However, in qemu, all PIO is LE, so we need to re-swap
-         * on BE targets
-         */
-        val = bswap16(val);
-    }
-    return val;
-}
-
-static uint32_t virtio_pci_config_readl(void *opaque, uint32_t addr)
-{
-    VirtIOPCIProxy *proxy = opaque;
-    uint32_t config = VIRTIO_PCI_CONFIG(&proxy->pci_dev);
-    uint32_t val;
-    if (addr < config)
-        return virtio_ioport_read(proxy, addr);
-    addr -= config;
-    val = virtio_config_readl(proxy->vdev, addr);
-    if (virtio_is_big_endian()) {
-        val = bswap32(val);
-    }
-    return val;
-}
-
-static void virtio_pci_config_writeb(void *opaque, uint32_t addr, uint32_t val)
+static uint64_t virtio_pci_config_read(void *opaque, hwaddr addr,
+                                       unsigned size)
 {
     VirtIOPCIProxy *proxy = opaque;
     uint32_t config = VIRTIO_PCI_CONFIG(&proxy->pci_dev);
+    uint64_t val = 0;
     if (addr < config) {
-        virtio_ioport_write(proxy, addr, val);
-        return;
+        return virtio_ioport_read(proxy, addr);
     }
     addr -= config;
-    virtio_config_writeb(proxy->vdev, addr, val);
-}
 
-static void virtio_pci_config_writew(void *opaque, uint32_t addr, uint32_t val)
-{
-    VirtIOPCIProxy *proxy = opaque;
-    uint32_t config = VIRTIO_PCI_CONFIG(&proxy->pci_dev);
-    if (addr < config) {
-        virtio_ioport_write(proxy, addr, val);
-        return;
-    }
-    addr -= config;
-    if (virtio_is_big_endian()) {
-        val = bswap16(val);
+    switch (size) {
+    case 1:
+        val = virtio_config_readb(proxy->vdev, addr);
+        break;
+    case 2:
+        val = virtio_config_readw(proxy->vdev, addr);
+        if (virtio_is_big_endian()) {
+            val = bswap16(val);
+        }
+        break;
+    case 4:
+        val = virtio_config_readl(proxy->vdev, addr);
+        if (virtio_is_big_endian()) {
+            val = bswap32(val);
+        }
+        break;
     }
-    virtio_config_writew(proxy->vdev, addr, val);
+    return val;
 }
 
-static void virtio_pci_config_writel(void *opaque, uint32_t addr, uint32_t val)
+static void virtio_pci_config_write(void *opaque, hwaddr addr,
+                                    uint64_t val, unsigned size)
 {
     VirtIOPCIProxy *proxy = opaque;
     uint32_t config = VIRTIO_PCI_CONFIG(&proxy->pci_dev);
@@ -455,24 +415,36 @@ static void virtio_pci_config_writel(void *opaque, uint32_t addr, uint32_t val)
         return;
     }
     addr -= config;
-    if (virtio_is_big_endian()) {
-        val = bswap32(val);
+    /*
+     * Virtio-PCI is odd. Ioports are LE but config space is target native
+     * endian.
+     */
+    switch (size) {
+    case 1:
+        virtio_config_writeb(proxy->vdev, addr, val);
+        break;
+    case 2:
+        if (virtio_is_big_endian()) {
+            val = bswap16(val);
+        }
+        virtio_config_writew(proxy->vdev, addr, val);
+        break;
+    case 4:
+        if (virtio_is_big_endian()) {
+            val = bswap32(val);
+        }
+        virtio_config_writel(proxy->vdev, addr, val);
+        break;
     }
-    virtio_config_writel(proxy->vdev, addr, val);
 }
 
-static const MemoryRegionPortio virtio_portio[] = {
-    { 0, 0x10000, 1, .write = virtio_pci_config_writeb, },
-    { 0, 0x10000, 2, .write = virtio_pci_config_writew, },
-    { 0, 0x10000, 4, .write = virtio_pci_config_writel, },
-    { 0, 0x10000, 1, .read = virtio_pci_config_readb, },
-    { 0, 0x10000, 2, .read = virtio_pci_config_readw, },
-    { 0, 0x10000, 4, .read = virtio_pci_config_readl, },
-    PORTIO_END_OF_LIST()
-};
-
 static const MemoryRegionOps virtio_pci_config_ops = {
-    .old_portio = virtio_portio,
+    .read = virtio_pci_config_read,
+    .write = virtio_pci_config_write,
+    .impl = {
+        .min_access_size = 1,
+        .max_access_size = 4,
+    },
     .endianness = DEVICE_LITTLE_ENDIAN,
 };
 
@@ -517,7 +489,7 @@ static int kvm_virtio_pci_vq_vector_use(VirtIOPCIProxy *proxy,
     }
     irqfd->users++;
 
-    ret = kvm_irqchip_add_irq_notifier(kvm_state, n, irqfd->virq);
+    ret = kvm_irqchip_add_irqfd_notifier(kvm_state, n, irqfd->virq);
     if (ret < 0) {
         if (--irqfd->users == 0) {
             kvm_irqchip_release_virq(kvm_state, irqfd->virq);
@@ -538,7 +510,7 @@ static void kvm_virtio_pci_vq_vector_release(VirtIOPCIProxy *proxy,
     VirtIOIRQFD *irqfd = &proxy->vector_irqfd[vector];
     int ret;
 
-    ret = kvm_irqchip_remove_irq_notifier(kvm_state, n, irqfd->virq);
+    ret = kvm_irqchip_remove_irqfd_notifier(kvm_state, n, irqfd->virq);
     assert(ret == 0);
 
     if (--irqfd->users == 0) {
@@ -880,6 +852,41 @@ static void virtio_balloon_exit_pci(PCIDevice *pci_dev)
     virtio_exit_pci(pci_dev);
 }
 
+static int virtio_rng_init_pci(PCIDevice *pci_dev)
+{
+    VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
+    VirtIODevice *vdev;
+
+    if (proxy->rng.rng == NULL) {
+        proxy->rng.default_backend = RNG_RANDOM(object_new(TYPE_RNG_RANDOM));
+
+        object_property_add_child(OBJECT(pci_dev),
+                                  "default-backend",
+                                  OBJECT(proxy->rng.default_backend),
+                                  NULL);
+
+        object_property_set_link(OBJECT(pci_dev),
+                                 OBJECT(proxy->rng.default_backend),
+                                 "rng", NULL);
+    }
+
+    vdev = virtio_rng_init(&pci_dev->qdev, &proxy->rng);
+    if (!vdev) {
+        return -1;
+    }
+    virtio_init_pci(proxy, vdev);
+    return 0;
+}
+
+static void virtio_rng_exit_pci(PCIDevice *pci_dev)
+{
+    VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
+
+    virtio_pci_stop_ioeventfd(proxy);
+    virtio_rng_exit(proxy->vdev);
+    virtio_exit_pci(pci_dev);
+}
+
 static Property virtio_blk_properties[] = {
     DEFINE_PROP_HEX32("class", VirtIOPCIProxy, class_code, 0),
     DEFINE_BLOCK_PROPERTIES(VirtIOPCIProxy, blk.conf),
@@ -1010,6 +1017,50 @@ static TypeInfo virtio_balloon_info = {
     .class_init    = virtio_balloon_class_init,
 };
 
+static void virtio_rng_initfn(Object *obj)
+{
+    PCIDevice *pci_dev = PCI_DEVICE(obj);
+    VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
+
+    object_property_add_link(obj, "rng", TYPE_RNG_BACKEND,
+                             (Object **)&proxy->rng.rng, NULL);
+}
+
+static Property virtio_rng_properties[] = {
+    DEFINE_VIRTIO_COMMON_FEATURES(VirtIOPCIProxy, host_features),
+    /* Set a default rate limit of 2^47 bytes per minute or roughly 2TB/s.  If
+       you have an entropy source capable of generating more entropy than this
+       and you can pass it through via virtio-rng, then hats off to you.  Until
+       then, this is unlimited for all practical purposes.
+    */
+    DEFINE_PROP_UINT64("max-bytes", VirtIOPCIProxy, rng.max_bytes, INT64_MAX),
+    DEFINE_PROP_UINT32("period", VirtIOPCIProxy, rng.period_ms, 1 << 16),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void virtio_rng_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
+
+    k->init = virtio_rng_init_pci;
+    k->exit = virtio_rng_exit_pci;
+    k->vendor_id = PCI_VENDOR_ID_REDHAT_QUMRANET;
+    k->device_id = PCI_DEVICE_ID_VIRTIO_RNG;
+    k->revision = VIRTIO_PCI_ABI_VERSION;
+    k->class_id = PCI_CLASS_OTHERS;
+    dc->reset = virtio_pci_reset;
+    dc->props = virtio_rng_properties;
+}
+
+static TypeInfo virtio_rng_info = {
+    .name          = "virtio-rng-pci",
+    .parent        = TYPE_PCI_DEVICE,
+    .instance_size = sizeof(VirtIOPCIProxy),
+    .instance_init = virtio_rng_initfn,
+    .class_init    = virtio_rng_class_init,
+};
+
 static int virtio_scsi_init_pci(PCIDevice *pci_dev)
 {
     VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev);
@@ -1074,6 +1125,7 @@ static void virtio_pci_register_types(void)
     type_register_static(&virtio_serial_info);
     type_register_static(&virtio_balloon_info);
     type_register_static(&virtio_scsi_info);
+    type_register_static(&virtio_rng_info);
 }
 
 type_init(virtio_pci_register_types)
diff --git a/hw/virtio-pci.h b/hw/virtio-pci.h
index ac9d522f3..b58d9a2d1 100644
--- a/hw/virtio-pci.h
+++ b/hw/virtio-pci.h
@@ -17,6 +17,7 @@
 
 #include "virtio-blk.h"
 #include "virtio-net.h"
+#include "virtio-rng.h"
 #include "virtio-serial.h"
 #include "virtio-scsi.h"
 
@@ -46,6 +47,7 @@ typedef struct {
     virtio_serial_conf serial;
     virtio_net_conf net;
     VirtIOSCSIConf scsi;
+    VirtIORNGConf rng;
     bool ioeventfd_disabled;
     bool ioeventfd_started;
     VirtIOIRQFD *vector_irqfd;
diff --git a/hw/virtio-rng.c b/hw/virtio-rng.c
new file mode 100644
index 000000000..a73ef8e33
--- /dev/null
+++ b/hw/virtio-rng.c
@@ -0,0 +1,205 @@
+/*
+ * A virtio device implementing a hardware random number generator.
+ *
+ * Copyright 2012 Red Hat, Inc.
+ * Copyright 2012 Amit Shah <amit.shah@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * (at your option) any later version.  See the COPYING file in the
+ * top-level directory.
+ */
+
+#include "iov.h"
+#include "qdev.h"
+#include "virtio.h"
+#include "virtio-rng.h"
+#include "qemu/rng.h"
+
+typedef struct VirtIORNG {
+    VirtIODevice vdev;
+
+    DeviceState *qdev;
+
+    /* Only one vq - guest puts buffer(s) on it when it needs entropy */
+    VirtQueue *vq;
+
+    VirtIORNGConf *conf;
+
+    RngBackend *rng;
+
+    /* We purposefully don't migrate this state.  The quota will reset on the
+     * destination as a result.  Rate limiting is host state, not guest state.
+     */
+    QEMUTimer *rate_limit_timer;
+    int64_t quota_remaining;
+} VirtIORNG;
+
+static bool is_guest_ready(VirtIORNG *vrng)
+{
+    if (virtio_queue_ready(vrng->vq)
+        && (vrng->vdev.status & VIRTIO_CONFIG_S_DRIVER_OK)) {
+        return true;
+    }
+    return false;
+}
+
+static size_t get_request_size(VirtQueue *vq, unsigned quota)
+{
+    unsigned int in, out;
+
+    virtqueue_get_avail_bytes(vq, &in, &out, quota, 0);
+    return in;
+}
+
+static void virtio_rng_process(VirtIORNG *vrng);
+
+/* Send data from a char device over to the guest */
+static void chr_read(void *opaque, const void *buf, size_t size)
+{
+    VirtIORNG *vrng = opaque;
+    VirtQueueElement elem;
+    size_t len;
+    int offset;
+
+    if (!is_guest_ready(vrng)) {
+        return;
+    }
+
+    vrng->quota_remaining -= size;
+
+    offset = 0;
+    while (offset < size) {
+        if (!virtqueue_pop(vrng->vq, &elem)) {
+            break;
+        }
+        len = iov_from_buf(elem.in_sg, elem.in_num,
+                           0, buf + offset, size - offset);
+        offset += len;
+
+        virtqueue_push(vrng->vq, &elem, len);
+    }
+    virtio_notify(&vrng->vdev, vrng->vq);
+}
+
+static void virtio_rng_process(VirtIORNG *vrng)
+{
+    size_t size;
+    unsigned quota;
+
+    if (!is_guest_ready(vrng)) {
+        return;
+    }
+
+    if (vrng->quota_remaining < 0) {
+        quota = 0;
+    } else {
+        quota = MIN((uint64_t)vrng->quota_remaining, (uint64_t)UINT32_MAX);
+    }
+    size = get_request_size(vrng->vq, quota);
+    size = MIN(vrng->quota_remaining, size);
+    if (size) {
+        rng_backend_request_entropy(vrng->rng, size, chr_read, vrng);
+    }
+}
+
+static void handle_input(VirtIODevice *vdev, VirtQueue *vq)
+{
+    VirtIORNG *vrng = DO_UPCAST(VirtIORNG, vdev, vdev);
+    virtio_rng_process(vrng);
+}
+
+static uint32_t get_features(VirtIODevice *vdev, uint32_t f)
+{
+    return f;
+}
+
+static void virtio_rng_save(QEMUFile *f, void *opaque)
+{
+    VirtIORNG *vrng = opaque;
+
+    virtio_save(&vrng->vdev, f);
+}
+
+static int virtio_rng_load(QEMUFile *f, void *opaque, int version_id)
+{
+    VirtIORNG *vrng = opaque;
+
+    if (version_id != 1) {
+        return -EINVAL;
+    }
+    virtio_load(&vrng->vdev, f);
+
+    /* We may have an element ready but couldn't process it due to a quota
+     * limit.  Make sure to try again after live migration when the quota may
+     * have been reset.
+     */
+    virtio_rng_process(vrng);
+
+    return 0;
+}
+
+static void check_rate_limit(void *opaque)
+{
+    VirtIORNG *s = opaque;
+
+    s->quota_remaining = s->conf->max_bytes;
+    virtio_rng_process(s);
+    qemu_mod_timer(s->rate_limit_timer,
+                   qemu_get_clock_ms(vm_clock) + s->conf->period_ms);
+}
+
+
+VirtIODevice *virtio_rng_init(DeviceState *dev, VirtIORNGConf *conf)
+{
+    VirtIORNG *vrng;
+    VirtIODevice *vdev;
+    Error *local_err = NULL;
+
+    vdev = virtio_common_init("virtio-rng", VIRTIO_ID_RNG, 0,
+                              sizeof(VirtIORNG));
+
+    vrng = DO_UPCAST(VirtIORNG, vdev, vdev);
+
+    vrng->rng = conf->rng;
+    if (vrng->rng == NULL) {
+        qerror_report(QERR_INVALID_PARAMETER_VALUE, "rng", "a valid object");
+        return NULL;
+    }
+
+    rng_backend_open(vrng->rng, &local_err);
+    if (local_err) {
+        qerror_report_err(local_err);
+        error_free(local_err);
+        return NULL;
+    }
+
+    vrng->vq = virtio_add_queue(vdev, 8, handle_input);
+    vrng->vdev.get_features = get_features;
+
+    vrng->qdev = dev;
+    vrng->conf = conf;
+
+    assert(vrng->conf->max_bytes <= INT64_MAX);
+    vrng->quota_remaining = vrng->conf->max_bytes;
+
+    vrng->rate_limit_timer = qemu_new_timer_ms(vm_clock,
+                                               check_rate_limit, vrng);
+
+    qemu_mod_timer(vrng->rate_limit_timer,
+                   qemu_get_clock_ms(vm_clock) + vrng->conf->period_ms);
+
+    register_savevm(dev, "virtio-rng", -1, 1, virtio_rng_save,
+                    virtio_rng_load, vrng);
+
+    return vdev;
+}
+
+void virtio_rng_exit(VirtIODevice *vdev)
+{
+    VirtIORNG *vrng = DO_UPCAST(VirtIORNG, vdev, vdev);
+
+    qemu_del_timer(vrng->rate_limit_timer);
+    qemu_free_timer(vrng->rate_limit_timer);
+    unregister_savevm(vrng->qdev, "virtio-rng", vrng);
+    virtio_cleanup(vdev);
+}
diff --git a/hw/virtio-rng.h b/hw/virtio-rng.h
new file mode 100644
index 000000000..f42d748eb
--- /dev/null
+++ b/hw/virtio-rng.h
@@ -0,0 +1,28 @@
+/*
+ * Virtio RNG Support
+ *
+ * Copyright Red Hat, Inc. 2012
+ * Copyright Amit Shah <amit.shah@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * (at your option) any later version.  See the COPYING file in the
+ * top-level directory.
+ */
+
+#ifndef _QEMU_VIRTIO_RNG_H
+#define _QEMU_VIRTIO_RNG_H
+
+#include "qemu/rng.h"
+#include "qemu/rng-random.h"
+
+/* The Virtio ID for the virtio rng device */
+#define VIRTIO_ID_RNG    4
+
+struct VirtIORNGConf {
+    RngBackend *rng;
+    uint64_t max_bytes;
+    uint32_t period_ms;
+    RndRandom *default_backend;
+};
+
+#endif
diff --git a/hw/virtio-scsi.c b/hw/virtio-scsi.c
index c1b47a8f4..bfe186050 100644
--- a/hw/virtio-scsi.c
+++ b/hw/virtio-scsi.c
@@ -202,9 +202,9 @@ static void virtio_scsi_bad_req(void)
 }
 
 static void qemu_sgl_init_external(QEMUSGList *qsgl, struct iovec *sg,
-                                   target_phys_addr_t *addr, int num)
+                                   hwaddr *addr, int num)
 {
-    memset(qsgl, 0, sizeof(*qsgl));
+    qemu_sglist_init(qsgl, num, &dma_context_memory);
     while (num--) {
         qemu_sglist_add(qsgl, *(addr++), (sg++)->iov_len);
     }
@@ -424,15 +424,17 @@ static void virtio_scsi_command_complete(SCSIRequest *r, uint32_t status,
                                          size_t resid)
 {
     VirtIOSCSIReq *req = r->hba_private;
+    uint32_t sense_len;
 
     req->resp.cmd->response = VIRTIO_SCSI_S_OK;
     req->resp.cmd->status = status;
     if (req->resp.cmd->status == GOOD) {
-        req->resp.cmd->resid = resid;
+        req->resp.cmd->resid = tswap32(resid);
     } else {
         req->resp.cmd->resid = 0;
-        req->resp.cmd->sense_len =
-            scsi_req_get_sense(r, req->resp.cmd->sense, VIRTIO_SCSI_SENSE_SIZE);
+        sense_len = scsi_req_get_sense(r, req->resp.cmd->sense,
+                                       VIRTIO_SCSI_SENSE_SIZE);
+        req->resp.cmd->sense_len = tswap32(sense_len);
     }
     virtio_scsi_complete_req(req);
 }
@@ -532,8 +534,8 @@ static void virtio_scsi_get_config(VirtIODevice *vdev,
     stl_raw(&scsiconf->event_info_size, sizeof(VirtIOSCSIEvent));
     stl_raw(&scsiconf->sense_size, s->sense_size);
     stl_raw(&scsiconf->cdb_size, s->cdb_size);
-    stl_raw(&scsiconf->max_channel, VIRTIO_SCSI_MAX_CHANNEL);
-    stl_raw(&scsiconf->max_target, VIRTIO_SCSI_MAX_TARGET);
+    stw_raw(&scsiconf->max_channel, VIRTIO_SCSI_MAX_CHANNEL);
+    stw_raw(&scsiconf->max_target, VIRTIO_SCSI_MAX_TARGET);
     stl_raw(&scsiconf->max_lun, VIRTIO_SCSI_MAX_LUN);
 }
 
@@ -596,6 +598,10 @@ static void virtio_scsi_push_event(VirtIOSCSI *s, SCSIDevice *dev,
     VirtIOSCSIEvent *evt;
     int in_size;
 
+    if (!(s->vdev.status & VIRTIO_CONFIG_S_DRIVER_OK)) {
+        return;
+    }
+
     if (!req) {
         s->events_dropped = true;
         return;
@@ -648,7 +654,6 @@ static void virtio_scsi_change(SCSIBus *bus, SCSIDevice *dev, SCSISense sense)
     VirtIOSCSI *s = container_of(bus, VirtIOSCSI, bus);
 
     if (((s->vdev.guest_features >> VIRTIO_SCSI_F_CHANGE) & 1) &&
-        (s->vdev.status & VIRTIO_CONFIG_S_DRIVER_OK) &&
         dev->type != TYPE_ROM) {
         virtio_scsi_push_event(s, dev, VIRTIO_SCSI_T_PARAM_CHANGE,
                                sense.asc | (sense.ascq << 8));
@@ -659,8 +664,7 @@ static void virtio_scsi_hotplug(SCSIBus *bus, SCSIDevice *dev)
 {
     VirtIOSCSI *s = container_of(bus, VirtIOSCSI, bus);
 
-    if (((s->vdev.guest_features >> VIRTIO_SCSI_F_HOTPLUG) & 1) &&
-        (s->vdev.status & VIRTIO_CONFIG_S_DRIVER_OK)) {
+    if ((s->vdev.guest_features >> VIRTIO_SCSI_F_HOTPLUG) & 1) {
         virtio_scsi_push_event(s, dev, VIRTIO_SCSI_T_TRANSPORT_RESET,
                                VIRTIO_SCSI_EVT_RESET_RESCAN);
     }
diff --git a/hw/virtio-serial-bus.c b/hw/virtio-serial-bus.c
index 82073f5dc..155da58dc 100644
--- a/hw/virtio-serial-bus.c
+++ b/hw/virtio-serial-bus.c
@@ -53,6 +53,15 @@ struct VirtIOSerial {
     uint32_t *ports_map;
 
     struct virtio_console_config config;
+
+    struct {
+        QEMUTimer *timer;
+        int nr_active_ports;
+        struct {
+            VirtIOSerialPort *port;
+            uint8_t host_connected;
+        } *connected;
+    } post_load;
 };
 
 static VirtIOSerialPort *find_port_by_id(VirtIOSerial *vser, uint32_t id)
@@ -287,6 +296,7 @@ ssize_t virtio_serial_write(VirtIOSerialPort *port, const uint8_t *buf,
 size_t virtio_serial_guest_ready(VirtIOSerialPort *port)
 {
     VirtQueue *vq = port->ivq;
+    unsigned int bytes;
 
     if (!virtio_queue_ready(vq) ||
         !(port->vser->vdev.status & VIRTIO_CONFIG_S_DRIVER_OK) ||
@@ -296,14 +306,8 @@ size_t virtio_serial_guest_ready(VirtIOSerialPort *port)
     if (use_multiport(port->vser) && !port->guest_connected) {
         return 0;
     }
-
-    if (virtqueue_avail_bytes(vq, 4096, 0)) {
-        return 4096;
-    }
-    if (virtqueue_avail_bytes(vq, 1, 0)) {
-        return 1;
-    }
-    return 0;
+    virtqueue_get_avail_bytes(vq, &bytes, NULL, 4096, 0);
+    return bytes;
 }
 
 static void flush_queued_data_bh(void *opaque)
@@ -631,6 +635,29 @@ static void virtio_serial_save(QEMUFile *f, void *opaque)
     }
 }
 
+static void virtio_serial_post_load_timer_cb(void *opaque)
+{
+    int i;
+    VirtIOSerial *s = opaque;
+    VirtIOSerialPort *port;
+    uint8_t host_connected;
+
+    for (i = 0 ; i < s->post_load.nr_active_ports; ++i) {
+        port = s->post_load.connected[i].port;
+        host_connected = s->post_load.connected[i].host_connected;
+        if (host_connected != port->host_connected) {
+            /*
+             * We have to let the guest know of the host connection
+             * status change
+             */
+            send_control_event(port, VIRTIO_CONSOLE_PORT_OPEN,
+                               port->host_connected);
+        }
+    }
+    g_free(s->post_load.connected);
+    s->post_load.connected = NULL;
+}
+
 static int virtio_serial_load(QEMUFile *f, void *opaque, int version_id)
 {
     VirtIOSerial *s = opaque;
@@ -678,10 +705,13 @@ static int virtio_serial_load(QEMUFile *f, void *opaque, int version_id)
 
     qemu_get_be32s(f, &nr_active_ports);
 
+    s->post_load.nr_active_ports = nr_active_ports;
+    s->post_load.connected =
+        g_malloc0(sizeof(*s->post_load.connected) * nr_active_ports);
+
     /* Items in struct VirtIOSerialPort */
     for (i = 0; i < nr_active_ports; i++) {
         uint32_t id;
-        bool host_connected;
 
         id = qemu_get_be32(f);
         port = find_port_by_id(s, id);
@@ -690,15 +720,8 @@ static int virtio_serial_load(QEMUFile *f, void *opaque, int version_id)
         }
 
         port->guest_connected = qemu_get_byte(f);
-        host_connected = qemu_get_byte(f);
-        if (host_connected != port->host_connected) {
-            /*
-             * We have to let the guest know of the host connection
-             * status change
-             */
-            send_control_event(port, VIRTIO_CONSOLE_PORT_OPEN,
-                               port->host_connected);
-        }
+        s->post_load.connected[i].port = port;
+        s->post_load.connected[i].host_connected = qemu_get_byte(f);
 
         if (version_id > 2) {
             uint32_t elem_popped;
@@ -723,6 +746,7 @@ static int virtio_serial_load(QEMUFile *f, void *opaque, int version_id)
             }
         }
     }
+    qemu_mod_timer(s->post_load.timer, 1);
     return 0;
 }
 
@@ -972,6 +996,9 @@ VirtIODevice *virtio_serial_init(DeviceState *dev, virtio_serial_conf *conf)
     register_savevm(dev, "virtio-console", -1, 3, virtio_serial_save,
                     virtio_serial_load, vser);
 
+    vser->post_load.timer = qemu_new_timer_ns(vm_clock,
+            virtio_serial_post_load_timer_cb, vser);
+
     return vdev;
 }
 
@@ -984,6 +1011,8 @@ void virtio_serial_exit(VirtIODevice *vdev)
     g_free(vser->ivqs);
     g_free(vser->ovqs);
     g_free(vser->ports_map);
+    g_free(vser->post_load.connected);
+    qemu_free_timer(vser->post_load.timer);
 
     virtio_cleanup(vdev);
 }
diff --git a/hw/virtio.c b/hw/virtio.c
index 209c76375..f40a8c557 100644
--- a/hw/virtio.c
+++ b/hw/virtio.c
@@ -53,15 +53,15 @@ typedef struct VRingUsed
 typedef struct VRing
 {
     unsigned int num;
-    target_phys_addr_t desc;
-    target_phys_addr_t avail;
-    target_phys_addr_t used;
+    hwaddr desc;
+    hwaddr avail;
+    hwaddr used;
 } VRing;
 
 struct VirtQueue
 {
     VRing vring;
-    target_phys_addr_t pa;
+    hwaddr pa;
     uint16_t last_avail_idx;
     /* Last used index value we have signalled on */
     uint16_t signalled_used;
@@ -84,7 +84,7 @@ struct VirtQueue
 /* virt queue functions */
 static void virtqueue_init(VirtQueue *vq)
 {
-    target_phys_addr_t pa = vq->pa;
+    hwaddr pa = vq->pa;
 
     vq->vring.desc = pa;
     vq->vring.avail = pa + vq->vring.num * sizeof(VRingDesc);
@@ -93,51 +93,51 @@ static void virtqueue_init(VirtQueue *vq)
                                  VIRTIO_PCI_VRING_ALIGN);
 }
 
-static inline uint64_t vring_desc_addr(target_phys_addr_t desc_pa, int i)
+static inline uint64_t vring_desc_addr(hwaddr desc_pa, int i)
 {
-    target_phys_addr_t pa;
+    hwaddr pa;
     pa = desc_pa + sizeof(VRingDesc) * i + offsetof(VRingDesc, addr);
     return ldq_phys(pa);
 }
 
-static inline uint32_t vring_desc_len(target_phys_addr_t desc_pa, int i)
+static inline uint32_t vring_desc_len(hwaddr desc_pa, int i)
 {
-    target_phys_addr_t pa;
+    hwaddr pa;
     pa = desc_pa + sizeof(VRingDesc) * i + offsetof(VRingDesc, len);
     return ldl_phys(pa);
 }
 
-static inline uint16_t vring_desc_flags(target_phys_addr_t desc_pa, int i)
+static inline uint16_t vring_desc_flags(hwaddr desc_pa, int i)
 {
-    target_phys_addr_t pa;
+    hwaddr pa;
     pa = desc_pa + sizeof(VRingDesc) * i + offsetof(VRingDesc, flags);
     return lduw_phys(pa);
 }
 
-static inline uint16_t vring_desc_next(target_phys_addr_t desc_pa, int i)
+static inline uint16_t vring_desc_next(hwaddr desc_pa, int i)
 {
-    target_phys_addr_t pa;
+    hwaddr pa;
     pa = desc_pa + sizeof(VRingDesc) * i + offsetof(VRingDesc, next);
     return lduw_phys(pa);
 }
 
 static inline uint16_t vring_avail_flags(VirtQueue *vq)
 {
-    target_phys_addr_t pa;
+    hwaddr pa;
     pa = vq->vring.avail + offsetof(VRingAvail, flags);
     return lduw_phys(pa);
 }
 
 static inline uint16_t vring_avail_idx(VirtQueue *vq)
 {
-    target_phys_addr_t pa;
+    hwaddr pa;
     pa = vq->vring.avail + offsetof(VRingAvail, idx);
     return lduw_phys(pa);
 }
 
 static inline uint16_t vring_avail_ring(VirtQueue *vq, int i)
 {
-    target_phys_addr_t pa;
+    hwaddr pa;
     pa = vq->vring.avail + offsetof(VRingAvail, ring[i]);
     return lduw_phys(pa);
 }
@@ -149,49 +149,49 @@ static inline uint16_t vring_used_event(VirtQueue *vq)
 
 static inline void vring_used_ring_id(VirtQueue *vq, int i, uint32_t val)
 {
-    target_phys_addr_t pa;
+    hwaddr pa;
     pa = vq->vring.used + offsetof(VRingUsed, ring[i].id);
     stl_phys(pa, val);
 }
 
 static inline void vring_used_ring_len(VirtQueue *vq, int i, uint32_t val)
 {
-    target_phys_addr_t pa;
+    hwaddr pa;
     pa = vq->vring.used + offsetof(VRingUsed, ring[i].len);
     stl_phys(pa, val);
 }
 
 static uint16_t vring_used_idx(VirtQueue *vq)
 {
-    target_phys_addr_t pa;
+    hwaddr pa;
     pa = vq->vring.used + offsetof(VRingUsed, idx);
     return lduw_phys(pa);
 }
 
 static inline void vring_used_idx_set(VirtQueue *vq, uint16_t val)
 {
-    target_phys_addr_t pa;
+    hwaddr pa;
     pa = vq->vring.used + offsetof(VRingUsed, idx);
     stw_phys(pa, val);
 }
 
 static inline void vring_used_flags_set_bit(VirtQueue *vq, int mask)
 {
-    target_phys_addr_t pa;
+    hwaddr pa;
     pa = vq->vring.used + offsetof(VRingUsed, flags);
     stw_phys(pa, lduw_phys(pa) | mask);
 }
 
 static inline void vring_used_flags_unset_bit(VirtQueue *vq, int mask)
 {
-    target_phys_addr_t pa;
+    hwaddr pa;
     pa = vq->vring.used + offsetof(VRingUsed, flags);
     stw_phys(pa, lduw_phys(pa) & ~mask);
 }
 
 static inline void vring_avail_event(VirtQueue *vq, uint16_t val)
 {
-    target_phys_addr_t pa;
+    hwaddr pa;
     if (!vq->notification) {
         return;
     }
@@ -241,7 +241,7 @@ void virtqueue_fill(VirtQueue *vq, const VirtQueueElement *elem,
                                   elem->in_sg[i].iov_len,
                                   1, size);
 
-        offset += elem->in_sg[i].iov_len;
+        offset += size;
     }
 
     for (i = 0; i < elem->out_num; i++)
@@ -313,7 +313,7 @@ static unsigned int virtqueue_get_head(VirtQueue *vq, unsigned int idx)
     return head;
 }
 
-static unsigned virtqueue_next_desc(target_phys_addr_t desc_pa,
+static unsigned virtqueue_next_desc(hwaddr desc_pa,
                                     unsigned int i, unsigned int max)
 {
     unsigned int next;
@@ -335,17 +335,19 @@ static unsigned virtqueue_next_desc(target_phys_addr_t desc_pa,
     return next;
 }
 
-int virtqueue_avail_bytes(VirtQueue *vq, int in_bytes, int out_bytes)
+void virtqueue_get_avail_bytes(VirtQueue *vq, unsigned int *in_bytes,
+                               unsigned int *out_bytes,
+                               unsigned max_in_bytes, unsigned max_out_bytes)
 {
     unsigned int idx;
-    int total_bufs, in_total, out_total;
+    unsigned int total_bufs, in_total, out_total;
 
     idx = vq->last_avail_idx;
 
     total_bufs = in_total = out_total = 0;
     while (virtqueue_num_heads(vq, idx)) {
         unsigned int max, num_bufs, indirect = 0;
-        target_phys_addr_t desc_pa;
+        hwaddr desc_pa;
         int i;
 
         max = vq->vring.num;
@@ -380,13 +382,12 @@ int virtqueue_avail_bytes(VirtQueue *vq, int in_bytes, int out_bytes)
             }
 
             if (vring_desc_flags(desc_pa, i) & VRING_DESC_F_WRITE) {
-                if (in_bytes > 0 &&
-                    (in_total += vring_desc_len(desc_pa, i)) >= in_bytes)
-                    return 1;
+                in_total += vring_desc_len(desc_pa, i);
             } else {
-                if (out_bytes > 0 &&
-                    (out_total += vring_desc_len(desc_pa, i)) >= out_bytes)
-                    return 1;
+                out_total += vring_desc_len(desc_pa, i);
+            }
+            if (in_total >= max_in_bytes && out_total >= max_out_bytes) {
+                goto done;
             }
         } while ((i = virtqueue_next_desc(desc_pa, i, max)) != max);
 
@@ -395,15 +396,29 @@ int virtqueue_avail_bytes(VirtQueue *vq, int in_bytes, int out_bytes)
         else
             total_bufs++;
     }
+done:
+    if (in_bytes) {
+        *in_bytes = in_total;
+    }
+    if (out_bytes) {
+        *out_bytes = out_total;
+    }
+}
 
-    return 0;
+int virtqueue_avail_bytes(VirtQueue *vq, unsigned int in_bytes,
+                          unsigned int out_bytes)
+{
+    unsigned int in_total, out_total;
+
+    virtqueue_get_avail_bytes(vq, &in_total, &out_total, in_bytes, out_bytes);
+    return in_bytes <= in_total && out_bytes <= out_total;
 }
 
-void virtqueue_map_sg(struct iovec *sg, target_phys_addr_t *addr,
+void virtqueue_map_sg(struct iovec *sg, hwaddr *addr,
     size_t num_sg, int is_write)
 {
     unsigned int i;
-    target_phys_addr_t len;
+    hwaddr len;
 
     for (i = 0; i < num_sg; i++) {
         len = sg[i].iov_len;
@@ -418,7 +433,7 @@ void virtqueue_map_sg(struct iovec *sg, target_phys_addr_t *addr,
 int virtqueue_pop(VirtQueue *vq, VirtQueueElement *elem)
 {
     unsigned int i, head, max;
-    target_phys_addr_t desc_pa = vq->vring.desc;
+    hwaddr desc_pa = vq->vring.desc;
 
     if (!virtqueue_num_heads(vq, vq->last_avail_idx))
         return 0;
@@ -617,13 +632,13 @@ void virtio_config_writel(VirtIODevice *vdev, uint32_t addr, uint32_t data)
         vdev->set_config(vdev, vdev->config);
 }
 
-void virtio_queue_set_addr(VirtIODevice *vdev, int n, target_phys_addr_t addr)
+void virtio_queue_set_addr(VirtIODevice *vdev, int n, hwaddr addr)
 {
     vdev->vq[n].pa = addr;
     virtqueue_init(&vdev->vq[n]);
 }
 
-target_phys_addr_t virtio_queue_get_addr(VirtIODevice *vdev, int n)
+hwaddr virtio_queue_get_addr(VirtIODevice *vdev, int n)
 {
     return vdev->vq[n].pa;
 }
@@ -926,44 +941,44 @@ void virtio_bind_device(VirtIODevice *vdev, const VirtIOBindings *binding,
     vdev->binding_opaque = opaque;
 }
 
-target_phys_addr_t virtio_queue_get_desc_addr(VirtIODevice *vdev, int n)
+hwaddr virtio_queue_get_desc_addr(VirtIODevice *vdev, int n)
 {
     return vdev->vq[n].vring.desc;
 }
 
-target_phys_addr_t virtio_queue_get_avail_addr(VirtIODevice *vdev, int n)
+hwaddr virtio_queue_get_avail_addr(VirtIODevice *vdev, int n)
 {
     return vdev->vq[n].vring.avail;
 }
 
-target_phys_addr_t virtio_queue_get_used_addr(VirtIODevice *vdev, int n)
+hwaddr virtio_queue_get_used_addr(VirtIODevice *vdev, int n)
 {
     return vdev->vq[n].vring.used;
 }
 
-target_phys_addr_t virtio_queue_get_ring_addr(VirtIODevice *vdev, int n)
+hwaddr virtio_queue_get_ring_addr(VirtIODevice *vdev, int n)
 {
     return vdev->vq[n].vring.desc;
 }
 
-target_phys_addr_t virtio_queue_get_desc_size(VirtIODevice *vdev, int n)
+hwaddr virtio_queue_get_desc_size(VirtIODevice *vdev, int n)
 {
     return sizeof(VRingDesc) * vdev->vq[n].vring.num;
 }
 
-target_phys_addr_t virtio_queue_get_avail_size(VirtIODevice *vdev, int n)
+hwaddr virtio_queue_get_avail_size(VirtIODevice *vdev, int n)
 {
     return offsetof(VRingAvail, ring) +
         sizeof(uint64_t) * vdev->vq[n].vring.num;
 }
 
-target_phys_addr_t virtio_queue_get_used_size(VirtIODevice *vdev, int n)
+hwaddr virtio_queue_get_used_size(VirtIODevice *vdev, int n)
 {
     return offsetof(VRingUsed, ring) +
         sizeof(VRingUsedElem) * vdev->vq[n].vring.num;
 }
 
-target_phys_addr_t virtio_queue_get_ring_size(VirtIODevice *vdev, int n)
+hwaddr virtio_queue_get_ring_size(VirtIODevice *vdev, int n)
 {
     return vdev->vq[n].vring.used - vdev->vq[n].vring.desc +
 	    virtio_queue_get_used_size(vdev, n);
diff --git a/hw/virtio.h b/hw/virtio.h
index 7a4f56452..7c17f7ba0 100644
--- a/hw/virtio.h
+++ b/hw/virtio.h
@@ -69,7 +69,7 @@
 
 struct VirtQueue;
 
-static inline target_phys_addr_t vring_align(target_phys_addr_t addr,
+static inline hwaddr vring_align(hwaddr addr,
                                              unsigned long align)
 {
     return (addr + align - 1) & ~(align - 1);
@@ -84,8 +84,8 @@ typedef struct VirtQueueElement
     unsigned int index;
     unsigned int out_num;
     unsigned int in_num;
-    target_phys_addr_t in_addr[VIRTQUEUE_MAX_SIZE];
-    target_phys_addr_t out_addr[VIRTQUEUE_MAX_SIZE];
+    hwaddr in_addr[VIRTQUEUE_MAX_SIZE];
+    hwaddr out_addr[VIRTQUEUE_MAX_SIZE];
     struct iovec in_sg[VIRTQUEUE_MAX_SIZE];
     struct iovec out_sg[VIRTQUEUE_MAX_SIZE];
 } VirtQueueElement;
@@ -144,10 +144,14 @@ void virtqueue_flush(VirtQueue *vq, unsigned int count);
 void virtqueue_fill(VirtQueue *vq, const VirtQueueElement *elem,
                     unsigned int len, unsigned int idx);
 
-void virtqueue_map_sg(struct iovec *sg, target_phys_addr_t *addr,
+void virtqueue_map_sg(struct iovec *sg, hwaddr *addr,
     size_t num_sg, int is_write);
 int virtqueue_pop(VirtQueue *vq, VirtQueueElement *elem);
-int virtqueue_avail_bytes(VirtQueue *vq, int in_bytes, int out_bytes);
+int virtqueue_avail_bytes(VirtQueue *vq, unsigned int in_bytes,
+                          unsigned int out_bytes);
+void virtqueue_get_avail_bytes(VirtQueue *vq, unsigned int *in_bytes,
+                               unsigned int *out_bytes,
+                               unsigned max_in_bytes, unsigned max_out_bytes);
 
 void virtio_notify(VirtIODevice *vdev, VirtQueue *vq);
 
@@ -175,8 +179,8 @@ uint32_t virtio_config_readl(VirtIODevice *vdev, uint32_t addr);
 void virtio_config_writeb(VirtIODevice *vdev, uint32_t addr, uint32_t data);
 void virtio_config_writew(VirtIODevice *vdev, uint32_t addr, uint32_t data);
 void virtio_config_writel(VirtIODevice *vdev, uint32_t addr, uint32_t data);
-void virtio_queue_set_addr(VirtIODevice *vdev, int n, target_phys_addr_t addr);
-target_phys_addr_t virtio_queue_get_addr(VirtIODevice *vdev, int n);
+void virtio_queue_set_addr(VirtIODevice *vdev, int n, hwaddr addr);
+hwaddr virtio_queue_get_addr(VirtIODevice *vdev, int n);
 int virtio_queue_get_num(VirtIODevice *vdev, int n);
 void virtio_queue_notify(VirtIODevice *vdev, int n);
 uint16_t virtio_queue_vector(VirtIODevice *vdev, int n);
@@ -200,6 +204,8 @@ VirtIODevice *virtio_serial_init(DeviceState *dev, virtio_serial_conf *serial);
 VirtIODevice *virtio_balloon_init(DeviceState *dev);
 typedef struct VirtIOSCSIConf VirtIOSCSIConf;
 VirtIODevice *virtio_scsi_init(DeviceState *dev, VirtIOSCSIConf *conf);
+typedef struct VirtIORNGConf VirtIORNGConf;
+VirtIODevice *virtio_rng_init(DeviceState *dev, VirtIORNGConf *conf);
 #ifdef CONFIG_LINUX
 VirtIODevice *virtio_9p_init(DeviceState *dev, V9fsConf *conf);
 #endif
@@ -210,6 +216,7 @@ void virtio_blk_exit(VirtIODevice *vdev);
 void virtio_serial_exit(VirtIODevice *vdev);
 void virtio_balloon_exit(VirtIODevice *vdev);
 void virtio_scsi_exit(VirtIODevice *vdev);
+void virtio_rng_exit(VirtIODevice *vdev);
 
 #define DEFINE_VIRTIO_COMMON_FEATURES(_state, _field) \
 	DEFINE_PROP_BIT("indirect_desc", _state, _field, \
@@ -217,14 +224,14 @@ void virtio_scsi_exit(VirtIODevice *vdev);
 	DEFINE_PROP_BIT("event_idx", _state, _field, \
 			VIRTIO_RING_F_EVENT_IDX, true)
 
-target_phys_addr_t virtio_queue_get_desc_addr(VirtIODevice *vdev, int n);
-target_phys_addr_t virtio_queue_get_avail_addr(VirtIODevice *vdev, int n);
-target_phys_addr_t virtio_queue_get_used_addr(VirtIODevice *vdev, int n);
-target_phys_addr_t virtio_queue_get_ring_addr(VirtIODevice *vdev, int n);
-target_phys_addr_t virtio_queue_get_desc_size(VirtIODevice *vdev, int n);
-target_phys_addr_t virtio_queue_get_avail_size(VirtIODevice *vdev, int n);
-target_phys_addr_t virtio_queue_get_used_size(VirtIODevice *vdev, int n);
-target_phys_addr_t virtio_queue_get_ring_size(VirtIODevice *vdev, int n);
+hwaddr virtio_queue_get_desc_addr(VirtIODevice *vdev, int n);
+hwaddr virtio_queue_get_avail_addr(VirtIODevice *vdev, int n);
+hwaddr virtio_queue_get_used_addr(VirtIODevice *vdev, int n);
+hwaddr virtio_queue_get_ring_addr(VirtIODevice *vdev, int n);
+hwaddr virtio_queue_get_desc_size(VirtIODevice *vdev, int n);
+hwaddr virtio_queue_get_avail_size(VirtIODevice *vdev, int n);
+hwaddr virtio_queue_get_used_size(VirtIODevice *vdev, int n);
+hwaddr virtio_queue_get_ring_size(VirtIODevice *vdev, int n);
 uint16_t virtio_queue_get_last_avail_idx(VirtIODevice *vdev, int n);
 void virtio_queue_set_last_avail_idx(VirtIODevice *vdev, int n, uint16_t idx);
 VirtQueue *virtio_get_queue(VirtIODevice *vdev, int n);
diff --git a/hw/vmport.c b/hw/vmport.c
index a4f52ee5b..3ab3a1405 100644
--- a/hw/vmport.c
+++ b/hw/vmport.c
@@ -54,7 +54,8 @@ void vmport_register(unsigned char command, IOPortReadFunc *func, void *opaque)
     port_state->opaque[command] = opaque;
 }
 
-static uint32_t vmport_ioport_read(void *opaque, uint32_t addr)
+static uint64_t vmport_ioport_read(void *opaque, hwaddr addr,
+                                   unsigned size)
 {
     VMPortState *s = opaque;
     CPUX86State *env = cpu_single_env;
@@ -81,11 +82,12 @@ static uint32_t vmport_ioport_read(void *opaque, uint32_t addr)
     return s->func[command](s->opaque[command], addr);
 }
 
-static void vmport_ioport_write(void *opaque, uint32_t addr, uint32_t val)
+static void vmport_ioport_write(void *opaque, hwaddr addr,
+                                uint64_t val, unsigned size)
 {
     CPUX86State *env = cpu_single_env;
 
-    env->regs[R_EAX] = vmport_ioport_read(opaque, addr);
+    env->regs[R_EAX] = vmport_ioport_read(opaque, addr, 4);
 }
 
 static uint32_t vmport_cmd_get_version(void *opaque, uint32_t addr)
@@ -121,13 +123,14 @@ void vmmouse_set_data(const uint32_t *data)
     env->regs[R_ESI] = data[4]; env->regs[R_EDI] = data[5];
 }
 
-static const MemoryRegionPortio vmport_portio[] = {
-    {0, 1, 4, .read = vmport_ioport_read, .write = vmport_ioport_write },
-    PORTIO_END_OF_LIST(),
-};
-
 static const MemoryRegionOps vmport_ops = {
-    .old_portio = vmport_portio
+    .read = vmport_ioport_read,
+    .write = vmport_ioport_write,
+    .impl = {
+        .min_access_size = 4,
+        .max_access_size = 4,
+    },
+    .endianness = DEVICE_LITTLE_ENDIAN,
 };
 
 static int vmport_initfn(ISADevice *dev)
diff --git a/hw/vmware_vga.c b/hw/vmware_vga.c
index f5e4f440d..834588daf 100644
--- a/hw/vmware_vga.c
+++ b/hw/vmware_vga.c
@@ -25,20 +25,19 @@
 #include "loader.h"
 #include "console.h"
 #include "pci.h"
-#include "vmware_vga.h"
 
 #undef VERBOSE
 #define HW_RECT_ACCEL
 #define HW_FILL_ACCEL
 #define HW_MOUSE_ACCEL
 
-# include "vga_int.h"
+#include "vga_int.h"
+
+/* See http://vmware-svga.sf.net/ for some documentation on VMWare SVGA */
 
 struct vmsvga_state_s {
     VGACommonState vga;
 
-    int width;
-    int height;
     int invalidated;
     int depth;
     int bypp;
@@ -62,7 +61,6 @@ struct vmsvga_state_s {
     uint32_t wgreen;
     uint32_t wblue;
     int syncing;
-    int fb_size;
 
     MemoryRegion fifo_ram;
     uint8_t *fifo_ptr;
@@ -80,7 +78,7 @@ struct vmsvga_state_s {
         } *cmd;
     };
 
-#define REDRAW_FIFO_LEN	512
+#define REDRAW_FIFO_LEN  512
     struct vmsvga_rect_s {
         int x, y, w, h;
     } redraw_fifo[REDRAW_FIFO_LEN];
@@ -93,31 +91,31 @@ struct pci_vmsvga_state_s {
     MemoryRegion io_bar;
 };
 
-#define SVGA_MAGIC		0x900000UL
-#define SVGA_MAKE_ID(ver)	(SVGA_MAGIC << 8 | (ver))
-#define SVGA_ID_0		SVGA_MAKE_ID(0)
-#define SVGA_ID_1		SVGA_MAKE_ID(1)
-#define SVGA_ID_2		SVGA_MAKE_ID(2)
+#define SVGA_MAGIC              0x900000UL
+#define SVGA_MAKE_ID(ver)       (SVGA_MAGIC << 8 | (ver))
+#define SVGA_ID_0               SVGA_MAKE_ID(0)
+#define SVGA_ID_1               SVGA_MAKE_ID(1)
+#define SVGA_ID_2               SVGA_MAKE_ID(2)
 
-#define SVGA_LEGACY_BASE_PORT	0x4560
-#define SVGA_INDEX_PORT		0x0
-#define SVGA_VALUE_PORT		0x1
-#define SVGA_BIOS_PORT		0x2
+#define SVGA_LEGACY_BASE_PORT   0x4560
+#define SVGA_INDEX_PORT         0x0
+#define SVGA_VALUE_PORT         0x1
+#define SVGA_BIOS_PORT          0x2
 
 #define SVGA_VERSION_2
 
 #ifdef SVGA_VERSION_2
-# define SVGA_ID		SVGA_ID_2
-# define SVGA_IO_BASE		SVGA_LEGACY_BASE_PORT
-# define SVGA_IO_MUL		1
-# define SVGA_FIFO_SIZE		0x10000
-# define SVGA_PCI_DEVICE_ID	PCI_DEVICE_ID_VMWARE_SVGA2
+# define SVGA_ID                SVGA_ID_2
+# define SVGA_IO_BASE           SVGA_LEGACY_BASE_PORT
+# define SVGA_IO_MUL            1
+# define SVGA_FIFO_SIZE         0x10000
+# define SVGA_PCI_DEVICE_ID     PCI_DEVICE_ID_VMWARE_SVGA2
 #else
-# define SVGA_ID		SVGA_ID_1
-# define SVGA_IO_BASE		SVGA_LEGACY_BASE_PORT
-# define SVGA_IO_MUL		4
-# define SVGA_FIFO_SIZE		0x10000
-# define SVGA_PCI_DEVICE_ID	PCI_DEVICE_ID_VMWARE_SVGA
+# define SVGA_ID                SVGA_ID_1
+# define SVGA_IO_BASE           SVGA_LEGACY_BASE_PORT
+# define SVGA_IO_MUL            4
+# define SVGA_FIFO_SIZE         0x10000
+# define SVGA_PCI_DEVICE_ID     PCI_DEVICE_ID_VMWARE_SVGA
 #endif
 
 enum {
@@ -129,7 +127,7 @@ enum {
     SVGA_REG_MAX_WIDTH = 4,
     SVGA_REG_MAX_HEIGHT = 5,
     SVGA_REG_DEPTH = 6,
-    SVGA_REG_BITS_PER_PIXEL = 7,	/* Current bpp in the guest */
+    SVGA_REG_BITS_PER_PIXEL = 7,        /* Current bpp in the guest */
     SVGA_REG_PSEUDOCOLOR = 8,
     SVGA_REG_RED_MASK = 9,
     SVGA_REG_GREEN_MASK = 10,
@@ -142,46 +140,46 @@ enum {
 
     /* ID 1 and 2 registers */
     SVGA_REG_CAPABILITIES = 17,
-    SVGA_REG_MEM_START = 18,		/* Memory for command FIFO */
+    SVGA_REG_MEM_START = 18,            /* Memory for command FIFO */
     SVGA_REG_MEM_SIZE = 19,
-    SVGA_REG_CONFIG_DONE = 20,		/* Set when memory area configured */
-    SVGA_REG_SYNC = 21,			/* Write to force synchronization */
-    SVGA_REG_BUSY = 22,			/* Read to check if sync is done */
-    SVGA_REG_GUEST_ID = 23,		/* Set guest OS identifier */
-    SVGA_REG_CURSOR_ID = 24,		/* ID of cursor */
-    SVGA_REG_CURSOR_X = 25,		/* Set cursor X position */
-    SVGA_REG_CURSOR_Y = 26,		/* Set cursor Y position */
-    SVGA_REG_CURSOR_ON = 27,		/* Turn cursor on/off */
-    SVGA_REG_HOST_BITS_PER_PIXEL = 28,	/* Current bpp in the host */
-    SVGA_REG_SCRATCH_SIZE = 29,		/* Number of scratch registers */
-    SVGA_REG_MEM_REGS = 30,		/* Number of FIFO registers */
-    SVGA_REG_NUM_DISPLAYS = 31,		/* Number of guest displays */
-    SVGA_REG_PITCHLOCK = 32,		/* Fixed pitch for all modes */
-
-    SVGA_PALETTE_BASE = 1024,		/* Base of SVGA color map */
+    SVGA_REG_CONFIG_DONE = 20,          /* Set when memory area configured */
+    SVGA_REG_SYNC = 21,                 /* Write to force synchronization */
+    SVGA_REG_BUSY = 22,                 /* Read to check if sync is done */
+    SVGA_REG_GUEST_ID = 23,             /* Set guest OS identifier */
+    SVGA_REG_CURSOR_ID = 24,            /* ID of cursor */
+    SVGA_REG_CURSOR_X = 25,             /* Set cursor X position */
+    SVGA_REG_CURSOR_Y = 26,             /* Set cursor Y position */
+    SVGA_REG_CURSOR_ON = 27,            /* Turn cursor on/off */
+    SVGA_REG_HOST_BITS_PER_PIXEL = 28,  /* Current bpp in the host */
+    SVGA_REG_SCRATCH_SIZE = 29,         /* Number of scratch registers */
+    SVGA_REG_MEM_REGS = 30,             /* Number of FIFO registers */
+    SVGA_REG_NUM_DISPLAYS = 31,         /* Number of guest displays */
+    SVGA_REG_PITCHLOCK = 32,            /* Fixed pitch for all modes */
+
+    SVGA_PALETTE_BASE = 1024,           /* Base of SVGA color map */
     SVGA_PALETTE_END  = SVGA_PALETTE_BASE + 767,
     SVGA_SCRATCH_BASE = SVGA_PALETTE_BASE + 768,
 };
 
-#define SVGA_CAP_NONE			0
-#define SVGA_CAP_RECT_FILL		(1 << 0)
-#define SVGA_CAP_RECT_COPY		(1 << 1)
-#define SVGA_CAP_RECT_PAT_FILL		(1 << 2)
-#define SVGA_CAP_LEGACY_OFFSCREEN	(1 << 3)
-#define SVGA_CAP_RASTER_OP		(1 << 4)
-#define SVGA_CAP_CURSOR			(1 << 5)
-#define SVGA_CAP_CURSOR_BYPASS		(1 << 6)
-#define SVGA_CAP_CURSOR_BYPASS_2	(1 << 7)
-#define SVGA_CAP_8BIT_EMULATION		(1 << 8)
-#define SVGA_CAP_ALPHA_CURSOR		(1 << 9)
-#define SVGA_CAP_GLYPH			(1 << 10)
-#define SVGA_CAP_GLYPH_CLIPPING		(1 << 11)
-#define SVGA_CAP_OFFSCREEN_1		(1 << 12)
-#define SVGA_CAP_ALPHA_BLEND		(1 << 13)
-#define SVGA_CAP_3D			(1 << 14)
-#define SVGA_CAP_EXTENDED_FIFO		(1 << 15)
-#define SVGA_CAP_MULTIMON		(1 << 16)
-#define SVGA_CAP_PITCHLOCK		(1 << 17)
+#define SVGA_CAP_NONE                   0
+#define SVGA_CAP_RECT_FILL              (1 << 0)
+#define SVGA_CAP_RECT_COPY              (1 << 1)
+#define SVGA_CAP_RECT_PAT_FILL          (1 << 2)
+#define SVGA_CAP_LEGACY_OFFSCREEN       (1 << 3)
+#define SVGA_CAP_RASTER_OP              (1 << 4)
+#define SVGA_CAP_CURSOR                 (1 << 5)
+#define SVGA_CAP_CURSOR_BYPASS          (1 << 6)
+#define SVGA_CAP_CURSOR_BYPASS_2        (1 << 7)
+#define SVGA_CAP_8BIT_EMULATION         (1 << 8)
+#define SVGA_CAP_ALPHA_CURSOR           (1 << 9)
+#define SVGA_CAP_GLYPH                  (1 << 10)
+#define SVGA_CAP_GLYPH_CLIPPING         (1 << 11)
+#define SVGA_CAP_OFFSCREEN_1            (1 << 12)
+#define SVGA_CAP_ALPHA_BLEND            (1 << 13)
+#define SVGA_CAP_3D                     (1 << 14)
+#define SVGA_CAP_EXTENDED_FIFO          (1 << 15)
+#define SVGA_CAP_MULTIMON               (1 << 16)
+#define SVGA_CAP_PITCHLOCK              (1 << 17)
 
 /*
  * FIFO offsets (seen as an array of 32-bit words)
@@ -191,7 +189,7 @@ enum {
      * The original defined FIFO offsets
      */
     SVGA_FIFO_MIN = 0,
-    SVGA_FIFO_MAX,	/* The distance from MIN to MAX must be at least 10K */
+    SVGA_FIFO_MAX,      /* The distance from MIN to MAX must be at least 10K */
     SVGA_FIFO_NEXT_CMD,
     SVGA_FIFO_STOP,
 
@@ -205,21 +203,21 @@ enum {
     SVGA_FIFO_PITCHLOCK,
 };
 
-#define SVGA_FIFO_CAP_NONE		0
-#define SVGA_FIFO_CAP_FENCE		(1 << 0)
-#define SVGA_FIFO_CAP_ACCELFRONT	(1 << 1)
-#define SVGA_FIFO_CAP_PITCHLOCK		(1 << 2)
+#define SVGA_FIFO_CAP_NONE              0
+#define SVGA_FIFO_CAP_FENCE             (1 << 0)
+#define SVGA_FIFO_CAP_ACCELFRONT        (1 << 1)
+#define SVGA_FIFO_CAP_PITCHLOCK         (1 << 2)
 
-#define SVGA_FIFO_FLAG_NONE		0
-#define SVGA_FIFO_FLAG_ACCELFRONT	(1 << 0)
+#define SVGA_FIFO_FLAG_NONE             0
+#define SVGA_FIFO_FLAG_ACCELFRONT       (1 << 0)
 
 /* These values can probably be changed arbitrarily.  */
-#define SVGA_SCRATCH_SIZE		0x8000
-#define SVGA_MAX_WIDTH			2360
-#define SVGA_MAX_HEIGHT			1770
+#define SVGA_SCRATCH_SIZE               0x8000
+#define SVGA_MAX_WIDTH                  2360
+#define SVGA_MAX_HEIGHT                 1770
 
 #ifdef VERBOSE
-# define GUEST_OS_BASE		0x5001
+# define GUEST_OS_BASE          0x5001
 static const char *vmsvga_guest_id[] = {
     [0x00] = "Dos",
     [0x01] = "Windows 3.1",
@@ -298,44 +296,37 @@ static inline void vmsvga_update_rect(struct vmsvga_state_s *s,
     uint8_t *src;
     uint8_t *dst;
 
-    if (x + w > s->width) {
+    if (x + w > ds_get_width(s->vga.ds)) {
         fprintf(stderr, "%s: update width too large x: %d, w: %d\n",
-                        __FUNCTION__, x, w);
-        x = MIN(x, s->width);
-        w = s->width - x;
+                __func__, x, w);
+        x = MIN(x, ds_get_width(s->vga.ds));
+        w = ds_get_width(s->vga.ds) - x;
     }
 
-    if (y + h > s->height) {
+    if (y + h > ds_get_height(s->vga.ds)) {
         fprintf(stderr, "%s: update height too large y: %d, h: %d\n",
-                        __FUNCTION__, y, h);
-        y = MIN(y, s->height);
-        h = s->height - y;
+                __func__, y, h);
+        y = MIN(y, ds_get_height(s->vga.ds));
+        h = ds_get_height(s->vga.ds) - y;
     }
 
-    line = h;
-    bypl = s->bypp * s->width;
-    width = s->bypp * w;
-    start = s->bypp * x + bypl * y;
+    bypl = ds_get_linesize(s->vga.ds);
+    width = ds_get_bytes_per_pixel(s->vga.ds) * w;
+    start = ds_get_bytes_per_pixel(s->vga.ds) * x + bypl * y;
     src = s->vga.vram_ptr + start;
     dst = ds_get_data(s->vga.ds) + start;
 
-    for (; line > 0; line --, src += bypl, dst += bypl)
+    for (line = h; line > 0; line--, src += bypl, dst += bypl) {
         memcpy(dst, src, width);
-
-    dpy_update(s->vga.ds, x, y, w, h);
-}
-
-static inline void vmsvga_update_screen(struct vmsvga_state_s *s)
-{
-    memcpy(ds_get_data(s->vga.ds), s->vga.vram_ptr,
-           s->bypp * s->width * s->height);
-    dpy_update(s->vga.ds, 0, 0, s->width, s->height);
+    }
+    dpy_gfx_update(s->vga.ds, x, y, w, h);
 }
 
 static inline void vmsvga_update_rect_delayed(struct vmsvga_state_s *s,
                 int x, int y, int w, int h)
 {
-    struct vmsvga_rect_s *rect = &s->redraw_fifo[s->redraw_fifo_last ++];
+    struct vmsvga_rect_s *rect = &s->redraw_fifo[s->redraw_fifo_last++];
+
     s->redraw_fifo_last &= REDRAW_FIFO_LEN - 1;
     rect->x = x;
     rect->y = y;
@@ -346,6 +337,7 @@ static inline void vmsvga_update_rect_delayed(struct vmsvga_state_s *s,
 static inline void vmsvga_update_rect_flush(struct vmsvga_state_s *s)
 {
     struct vmsvga_rect_s *rect;
+
     if (s->invalidated) {
         s->redraw_fifo_first = s->redraw_fifo_last;
         return;
@@ -353,7 +345,7 @@ static inline void vmsvga_update_rect_flush(struct vmsvga_state_s *s)
     /* Overlapping region updates can be optimised out here - if someone
      * knows a smart algorithm to do that, please share.  */
     while (s->redraw_fifo_first != s->redraw_fifo_last) {
-        rect = &s->redraw_fifo[s->redraw_fifo_first ++];
+        rect = &s->redraw_fifo[s->redraw_fifo_first++];
         s->redraw_fifo_first &= REDRAW_FIFO_LEN - 1;
         vmsvga_update_rect(s, rect->x, rect->y, rect->w, rect->h);
     }
@@ -364,20 +356,21 @@ static inline void vmsvga_copy_rect(struct vmsvga_state_s *s,
                 int x0, int y0, int x1, int y1, int w, int h)
 {
     uint8_t *vram = s->vga.vram_ptr;
-    int bypl = s->bypp * s->width;
-    int width = s->bypp * w;
+    int bypl = ds_get_linesize(s->vga.ds);
+    int bypp = ds_get_bytes_per_pixel(s->vga.ds);
+    int width = bypp * w;
     int line = h;
     uint8_t *ptr[2];
 
     if (y1 > y0) {
-        ptr[0] = vram + s->bypp * x0 + bypl * (y0 + h - 1);
-        ptr[1] = vram + s->bypp * x1 + bypl * (y1 + h - 1);
+        ptr[0] = vram + bypp * x0 + bypl * (y0 + h - 1);
+        ptr[1] = vram + bypp * x1 + bypl * (y1 + h - 1);
         for (; line > 0; line --, ptr[0] -= bypl, ptr[1] -= bypl) {
             memmove(ptr[1], ptr[0], width);
         }
     } else {
-        ptr[0] = vram + s->bypp * x0 + bypl * y0;
-        ptr[1] = vram + s->bypp * x1 + bypl * y1;
+        ptr[0] = vram + bypp * x0 + bypl * y0;
+        ptr[1] = vram + bypp * x1 + bypl * y1;
         for (; line > 0; line --, ptr[0] += bypl, ptr[1] += bypl) {
             memmove(ptr[1], ptr[0], width);
         }
@@ -391,13 +384,11 @@ static inline void vmsvga_copy_rect(struct vmsvga_state_s *s,
 static inline void vmsvga_fill_rect(struct vmsvga_state_s *s,
                 uint32_t c, int x, int y, int w, int h)
 {
-    uint8_t *vram = s->vga.vram_ptr;
-    int bypp = s->bypp;
-    int bypl = bypp * s->width;
-    int width = bypp * w;
+    int bypl = ds_get_linesize(s->vga.ds);
+    int width = ds_get_bytes_per_pixel(s->vga.ds) * w;
     int line = h;
     int column;
-    uint8_t *fst = vram + bypp * x + bypl * y;
+    uint8_t *fst;
     uint8_t *dst;
     uint8_t *src;
     uint8_t col[4];
@@ -407,12 +398,14 @@ static inline void vmsvga_fill_rect(struct vmsvga_state_s *s,
     col[2] = c >> 16;
     col[3] = c >> 24;
 
+    fst = s->vga.vram_ptr + ds_get_bytes_per_pixel(s->vga.ds) * x + bypl * y;
+
     if (line--) {
         dst = fst;
         src = col;
         for (column = width; column > 0; column--) {
             *(dst++) = *(src++);
-            if (src - col == bypp) {
+            if (src - col == ds_get_bytes_per_pixel(s->vga.ds)) {
                 src = col;
             }
         }
@@ -438,8 +431,8 @@ struct vmsvga_cursor_definition_s {
     uint32_t image[4096];
 };
 
-#define SVGA_BITMAP_SIZE(w, h)		((((w) + 31) >> 5) * (h))
-#define SVGA_PIXMAP_SIZE(w, h, bpp)	(((((w) * (bpp)) + 31) >> 5) * (h))
+#define SVGA_BITMAP_SIZE(w, h)          ((((w) + 31) >> 5) * (h))
+#define SVGA_PIXMAP_SIZE(w, h, bpp)     (((((w) * (bpp)) + 31) >> 5) * (h))
 
 #ifdef HW_MOUSE_ACCEL
 static inline void vmsvga_cursor_define(struct vmsvga_state_s *s,
@@ -453,16 +446,16 @@ static inline void vmsvga_cursor_define(struct vmsvga_state_s *s,
     qc->hot_y = c->hot_y;
     switch (c->bpp) {
     case 1:
-        cursor_set_mono(qc, 0xffffff, 0x000000, (void*)c->image,
-                        1, (void*)c->mask);
+        cursor_set_mono(qc, 0xffffff, 0x000000, (void *)c->image,
+                        1, (void *)c->mask);
 #ifdef DEBUG
         cursor_print_ascii_art(qc, "vmware/mono");
 #endif
         break;
     case 32:
         /* fill alpha channel from mask, set color to zero */
-        cursor_set_mono(qc, 0x000000, 0x000000, (void*)c->mask,
-                        1, (void*)c->mask);
+        cursor_set_mono(qc, 0x000000, 0x000000, (void *)c->mask,
+                        1, (void *)c->mask);
         /* add in rgb values */
         pixels = c->width * c->height;
         for (i = 0; i < pixels; i++) {
@@ -474,36 +467,40 @@ static inline void vmsvga_cursor_define(struct vmsvga_state_s *s,
         break;
     default:
         fprintf(stderr, "%s: unhandled bpp %d, using fallback cursor\n",
-                __FUNCTION__, c->bpp);
+                __func__, c->bpp);
         cursor_put(qc);
         qc = cursor_builtin_left_ptr();
     }
 
-    if (s->vga.ds->cursor_define)
-        s->vga.ds->cursor_define(qc);
+    dpy_cursor_define(s->vga.ds, qc);
     cursor_put(qc);
 }
 #endif
 
-#define CMD(f)	le32_to_cpu(s->cmd->f)
+#define CMD(f)  le32_to_cpu(s->cmd->f)
 
 static inline int vmsvga_fifo_length(struct vmsvga_state_s *s)
 {
     int num;
-    if (!s->config || !s->enable)
+
+    if (!s->config || !s->enable) {
         return 0;
+    }
     num = CMD(next_cmd) - CMD(stop);
-    if (num < 0)
+    if (num < 0) {
         num += CMD(max) - CMD(min);
+    }
     return num >> 2;
 }
 
 static inline uint32_t vmsvga_fifo_read_raw(struct vmsvga_state_s *s)
 {
     uint32_t cmd = s->fifo[CMD(stop) >> 2];
+
     s->cmd->stop = cpu_to_le32(CMD(stop) + 4);
-    if (CMD(stop) >= CMD(max))
+    if (CMD(stop) >= CMD(max)) {
         s->cmd->stop = s->cmd->min;
+    }
     return cmd;
 }
 
@@ -529,8 +526,9 @@ static void vmsvga_fifo_run(struct vmsvga_state_s *s)
         case SVGA_CMD_UPDATE:
         case SVGA_CMD_UPDATE_VERBOSE:
             len -= 5;
-            if (len < 0)
+            if (len < 0) {
                 goto rewind;
+            }
 
             x = vmsvga_fifo_read(s);
             y = vmsvga_fifo_read(s);
@@ -541,8 +539,9 @@ static void vmsvga_fifo_run(struct vmsvga_state_s *s)
 
         case SVGA_CMD_RECT_FILL:
             len -= 6;
-            if (len < 0)
+            if (len < 0) {
                 goto rewind;
+            }
 
             colour = vmsvga_fifo_read(s);
             x = vmsvga_fifo_read(s);
@@ -559,8 +558,9 @@ static void vmsvga_fifo_run(struct vmsvga_state_s *s)
 
         case SVGA_CMD_RECT_COPY:
             len -= 7;
-            if (len < 0)
+            if (len < 0) {
                 goto rewind;
+            }
 
             x = vmsvga_fifo_read(s);
             y = vmsvga_fifo_read(s);
@@ -578,8 +578,9 @@ static void vmsvga_fifo_run(struct vmsvga_state_s *s)
 
         case SVGA_CMD_DEFINE_CURSOR:
             len -= 8;
-            if (len < 0)
+            if (len < 0) {
                 goto rewind;
+            }
 
             cursor.id = vmsvga_fifo_read(s);
             cursor.hot_x = vmsvga_fifo_read(s);
@@ -591,17 +592,21 @@ static void vmsvga_fifo_run(struct vmsvga_state_s *s)
 
             args = SVGA_BITMAP_SIZE(x, y) + SVGA_PIXMAP_SIZE(x, y, cursor.bpp);
             if (SVGA_BITMAP_SIZE(x, y) > sizeof cursor.mask ||
-                SVGA_PIXMAP_SIZE(x, y, cursor.bpp) > sizeof cursor.image)
+                SVGA_PIXMAP_SIZE(x, y, cursor.bpp) > sizeof cursor.image) {
                     goto badcmd;
+            }
 
             len -= args;
-            if (len < 0)
+            if (len < 0) {
                 goto rewind;
+            }
 
-            for (args = 0; args < SVGA_BITMAP_SIZE(x, y); args ++)
+            for (args = 0; args < SVGA_BITMAP_SIZE(x, y); args++) {
                 cursor.mask[args] = vmsvga_fifo_read_raw(s);
-            for (args = 0; args < SVGA_PIXMAP_SIZE(x, y, cursor.bpp); args ++)
+            }
+            for (args = 0; args < SVGA_PIXMAP_SIZE(x, y, cursor.bpp); args++) {
                 cursor.image[args] = vmsvga_fifo_read_raw(s);
+            }
 #ifdef HW_MOUSE_ACCEL
             vmsvga_cursor_define(s, &cursor);
             break;
@@ -616,9 +621,9 @@ static void vmsvga_fifo_run(struct vmsvga_state_s *s)
          */
         case SVGA_CMD_DEFINE_ALPHA_CURSOR:
             len -= 6;
-            if (len < 0)
+            if (len < 0) {
                 goto rewind;
-
+            }
             vmsvga_fifo_read(s);
             vmsvga_fifo_read(s);
             vmsvga_fifo_read(s);
@@ -634,9 +639,9 @@ static void vmsvga_fifo_run(struct vmsvga_state_s *s)
             goto badcmd;
         case SVGA_CMD_DRAW_GLYPH_CLIPPED:
             len -= 4;
-            if (len < 0)
+            if (len < 0) {
                 goto rewind;
-
+            }
             vmsvga_fifo_read(s);
             vmsvga_fifo_read(s);
             args = 7 + (vmsvga_fifo_read(s) >> 2);
@@ -660,12 +665,14 @@ static void vmsvga_fifo_run(struct vmsvga_state_s *s)
             args = 0;
         badcmd:
             len -= args;
-            if (len < 0)
+            if (len < 0) {
                 goto rewind;
-            while (args --)
+            }
+            while (args--) {
                 vmsvga_fifo_read(s);
+            }
             printf("%s: Unknown command 0x%02x in SVGA command FIFO\n",
-                            __FUNCTION__, cmd);
+                   __func__, cmd);
             break;
 
         rewind:
@@ -680,12 +687,14 @@ static void vmsvga_fifo_run(struct vmsvga_state_s *s)
 static uint32_t vmsvga_index_read(void *opaque, uint32_t address)
 {
     struct vmsvga_state_s *s = opaque;
+
     return s->index;
 }
 
 static void vmsvga_index_write(void *opaque, uint32_t address, uint32_t index)
 {
     struct vmsvga_state_s *s = opaque;
+
     s->index = index;
 }
 
@@ -693,6 +702,7 @@ static uint32_t vmsvga_value_read(void *opaque, uint32_t address)
 {
     uint32_t caps;
     struct vmsvga_state_s *s = opaque;
+
     switch (s->index) {
     case SVGA_REG_ID:
         return s->svgaid;
@@ -701,10 +711,10 @@ static uint32_t vmsvga_value_read(void *opaque, uint32_t address)
         return s->enable;
 
     case SVGA_REG_WIDTH:
-        return s->width;
+        return ds_get_width(s->vga.ds);
 
     case SVGA_REG_HEIGHT:
-        return s->height;
+        return ds_get_height(s->vga.ds);
 
     case SVGA_REG_MAX_WIDTH:
         return SVGA_MAX_WIDTH;
@@ -723,13 +733,15 @@ static uint32_t vmsvga_value_read(void *opaque, uint32_t address)
 
     case SVGA_REG_RED_MASK:
         return s->wred;
+
     case SVGA_REG_GREEN_MASK:
         return s->wgreen;
+
     case SVGA_REG_BLUE_MASK:
         return s->wblue;
 
     case SVGA_REG_BYTES_PER_LINE:
-        return ((s->depth + 7) >> 3) * s->new_width;
+        return s->bypp * s->new_width;
 
     case SVGA_REG_FB_START: {
         struct pci_vmsvga_state_s *pci_vmsvga
@@ -741,10 +753,10 @@ static uint32_t vmsvga_value_read(void *opaque, uint32_t address)
         return 0x0;
 
     case SVGA_REG_VRAM_SIZE:
-        return s->vga.vram_size;
+        return s->vga.vram_size; /* No physical VRAM besides the framebuffer */
 
     case SVGA_REG_FB_SIZE:
-        return s->fb_size;
+        return s->vga.vram_size;
 
     case SVGA_REG_CAPABILITIES:
         caps = SVGA_CAP_NONE;
@@ -755,9 +767,10 @@ static uint32_t vmsvga_value_read(void *opaque, uint32_t address)
         caps |= SVGA_CAP_RECT_FILL;
 #endif
 #ifdef HW_MOUSE_ACCEL
-        if (s->vga.ds->mouse_set)
+        if (dpy_cursor_define_supported(s->vga.ds)) {
             caps |= SVGA_CAP_CURSOR | SVGA_CAP_CURSOR_BYPASS_2 |
                     SVGA_CAP_CURSOR_BYPASS;
+        }
 #endif
         return caps;
 
@@ -806,9 +819,10 @@ static uint32_t vmsvga_value_read(void *opaque, uint32_t address)
 
     default:
         if (s->index >= SVGA_SCRATCH_BASE &&
-                s->index < SVGA_SCRATCH_BASE + s->scratch_size)
+            s->index < SVGA_SCRATCH_BASE + s->scratch_size) {
             return s->scratch[s->index - SVGA_SCRATCH_BASE];
-        printf("%s: Bad register %02x\n", __FUNCTION__, s->index);
+        }
+        printf("%s: Bad register %02x\n", __func__, s->index);
     }
 
     return 0;
@@ -817,21 +831,19 @@ static uint32_t vmsvga_value_read(void *opaque, uint32_t address)
 static void vmsvga_value_write(void *opaque, uint32_t address, uint32_t value)
 {
     struct vmsvga_state_s *s = opaque;
+
     switch (s->index) {
     case SVGA_REG_ID:
-        if (value == SVGA_ID_2 || value == SVGA_ID_1 || value == SVGA_ID_0)
+        if (value == SVGA_ID_2 || value == SVGA_ID_1 || value == SVGA_ID_0) {
             s->svgaid = value;
+        }
         break;
 
     case SVGA_REG_ENABLE:
-        s->enable = value;
-        s->config &= !!value;
-        s->width = -1;
-        s->height = -1;
+        s->enable = !!value;
         s->invalidated = 1;
         s->vga.invalidate(&s->vga);
-        if (s->enable) {
-            s->fb_size = ((s->depth + 7) >> 3) * s->new_width * s->new_height;
+        if (s->enable && s->config) {
             vga_dirty_log_stop(&s->vga);
         } else {
             vga_dirty_log_start(&s->vga);
@@ -839,19 +851,26 @@ static void vmsvga_value_write(void *opaque, uint32_t address, uint32_t value)
         break;
 
     case SVGA_REG_WIDTH:
-        s->new_width = value;
-        s->invalidated = 1;
+        if (value <= SVGA_MAX_WIDTH) {
+            s->new_width = value;
+            s->invalidated = 1;
+        } else {
+            printf("%s: Bad width: %i\n", __func__, value);
+        }
         break;
 
     case SVGA_REG_HEIGHT:
-        s->new_height = value;
-        s->invalidated = 1;
+        if (value <= SVGA_MAX_HEIGHT) {
+            s->new_height = value;
+            s->invalidated = 1;
+        } else {
+            printf("%s: Bad height: %i\n", __func__, value);
+        }
         break;
 
-    case SVGA_REG_DEPTH:
     case SVGA_REG_BITS_PER_PIXEL:
         if (value != s->depth) {
-            printf("%s: Bad colour depth: %i bits\n", __FUNCTION__, value);
+            printf("%s: Bad bits per pixel: %i bits\n", __func__, value);
             s->config = 0;
         }
         break;
@@ -860,15 +879,19 @@ static void vmsvga_value_write(void *opaque, uint32_t address, uint32_t value)
         if (value) {
             s->fifo = (uint32_t *) s->fifo_ptr;
             /* Check range and alignment.  */
-            if ((CMD(min) | CMD(max) |
-                        CMD(next_cmd) | CMD(stop)) & 3)
+            if ((CMD(min) | CMD(max) | CMD(next_cmd) | CMD(stop)) & 3) {
                 break;
-            if (CMD(min) < (uint8_t *) s->cmd->fifo - (uint8_t *) s->fifo)
+            }
+            if (CMD(min) < (uint8_t *) s->cmd->fifo - (uint8_t *) s->fifo) {
                 break;
-            if (CMD(max) > SVGA_FIFO_SIZE)
+            }
+            if (CMD(max) > SVGA_FIFO_SIZE) {
                 break;
-            if (CMD(max) < CMD(min) + 10 * 1024)
+            }
+            if (CMD(max) < CMD(min) + 10 * 1024) {
                 break;
+            }
+            vga_dirty_log_stop(&s->vga);
         }
         s->config = !!value;
         break;
@@ -882,9 +905,10 @@ static void vmsvga_value_write(void *opaque, uint32_t address, uint32_t value)
         s->guest = value;
 #ifdef VERBOSE
         if (value >= GUEST_OS_BASE && value < GUEST_OS_BASE +
-                ARRAY_SIZE(vmsvga_guest_id))
-            printf("%s: guest runs %s.\n", __FUNCTION__,
-                            vmsvga_guest_id[value - GUEST_OS_BASE]);
+            ARRAY_SIZE(vmsvga_guest_id)) {
+            printf("%s: guest runs %s.\n", __func__,
+                   vmsvga_guest_id[value - GUEST_OS_BASE]);
+        }
 #endif
         break;
 
@@ -904,11 +928,13 @@ static void vmsvga_value_write(void *opaque, uint32_t address, uint32_t value)
         s->cursor.on |= (value == SVGA_CURSOR_ON_SHOW);
         s->cursor.on &= (value != SVGA_CURSOR_ON_HIDE);
 #ifdef HW_MOUSE_ACCEL
-        if (s->vga.ds->mouse_set && value <= SVGA_CURSOR_ON_SHOW)
-            s->vga.ds->mouse_set(s->cursor.x, s->cursor.y, s->cursor.on);
+        if (value <= SVGA_CURSOR_ON_SHOW) {
+            dpy_mouse_set(s->vga.ds, s->cursor.x, s->cursor.y, s->cursor.on);
+        }
 #endif
         break;
 
+    case SVGA_REG_DEPTH:
     case SVGA_REG_MEM_REGS:
     case SVGA_REG_NUM_DISPLAYS:
     case SVGA_REG_PITCHLOCK:
@@ -921,28 +947,26 @@ static void vmsvga_value_write(void *opaque, uint32_t address, uint32_t value)
             s->scratch[s->index - SVGA_SCRATCH_BASE] = value;
             break;
         }
-        printf("%s: Bad register %02x\n", __FUNCTION__, s->index);
+        printf("%s: Bad register %02x\n", __func__, s->index);
     }
 }
 
 static uint32_t vmsvga_bios_read(void *opaque, uint32_t address)
 {
-    printf("%s: what are we supposed to return?\n", __FUNCTION__);
+    printf("%s: what are we supposed to return?\n", __func__);
     return 0xcafe;
 }
 
 static void vmsvga_bios_write(void *opaque, uint32_t address, uint32_t data)
 {
-    printf("%s: what are we supposed to do with (%08x)?\n",
-                    __FUNCTION__, data);
+    printf("%s: what are we supposed to do with (%08x)?\n", __func__, data);
 }
 
-static inline void vmsvga_size(struct vmsvga_state_s *s)
+static inline void vmsvga_check_size(struct vmsvga_state_s *s)
 {
-    if (s->new_width != s->width || s->new_height != s->height) {
-        s->width = s->new_width;
-        s->height = s->new_height;
-        qemu_console_resize(s->vga.ds, s->width, s->height);
+    if (s->new_width != ds_get_width(s->vga.ds) ||
+        s->new_height != ds_get_height(s->vga.ds)) {
+        qemu_console_resize(s->vga.ds, s->new_width, s->new_height);
         s->invalidated = 1;
     }
 }
@@ -950,12 +974,14 @@ static inline void vmsvga_size(struct vmsvga_state_s *s)
 static void vmsvga_update_display(void *opaque)
 {
     struct vmsvga_state_s *s = opaque;
+    bool dirty = false;
+
     if (!s->enable) {
         s->vga.update(&s->vga);
         return;
     }
 
-    vmsvga_size(s);
+    vmsvga_check_size(s);
 
     vmsvga_fifo_run(s);
     vmsvga_update_rect_flush(s);
@@ -964,9 +990,23 @@ static void vmsvga_update_display(void *opaque)
      * Is it more efficient to look at vram VGA-dirty bits or wait
      * for the driver to issue SVGA_CMD_UPDATE?
      */
-    if (s->invalidated) {
+    if (memory_region_is_logging(&s->vga.vram)) {
+        vga_sync_dirty_bitmap(&s->vga);
+        dirty = memory_region_get_dirty(&s->vga.vram, 0,
+            ds_get_linesize(s->vga.ds) * ds_get_height(s->vga.ds),
+            DIRTY_MEMORY_VGA);
+    }
+    if (s->invalidated || dirty) {
         s->invalidated = 0;
-        vmsvga_update_screen(s);
+        memcpy(ds_get_data(s->vga.ds), s->vga.vram_ptr,
+               ds_get_linesize(s->vga.ds) * ds_get_height(s->vga.ds));
+        dpy_gfx_update(s->vga.ds, 0, 0,
+                   ds_get_width(s->vga.ds), ds_get_height(s->vga.ds));
+    }
+    if (dirty) {
+        memory_region_reset_dirty(&s->vga.vram, 0,
+            ds_get_linesize(s->vga.ds) * ds_get_height(s->vga.ds),
+            DIRTY_MEMORY_VGA);
     }
 }
 
@@ -979,8 +1019,6 @@ static void vmsvga_reset(DeviceState *dev)
     s->index = 0;
     s->enable = 0;
     s->config = 0;
-    s->width = -1;
-    s->height = -1;
     s->svgaid = SVGA_ID;
     s->cursor.on = 0;
     s->redraw_fifo_first = 0;
@@ -1003,18 +1041,23 @@ static void vmsvga_invalidate_display(void *opaque)
 
 /* save the vga display in a PPM image even if no display is
    available */
-static void vmsvga_screen_dump(void *opaque, const char *filename, bool cswitch)
+static void vmsvga_screen_dump(void *opaque, const char *filename, bool cswitch,
+                               Error **errp)
 {
     struct vmsvga_state_s *s = opaque;
     if (!s->enable) {
-        s->vga.screen_dump(&s->vga, filename, cswitch);
+        s->vga.screen_dump(&s->vga, filename, cswitch, errp);
         return;
     }
 
-    if (s->depth == 32) {
-        DisplaySurface *ds = qemu_create_displaysurface_from(s->width,
-                s->height, 32, ds_get_linesize(s->vga.ds), s->vga.vram_ptr);
-        ppm_save(filename, ds);
+    if (ds_get_bits_per_pixel(s->vga.ds) == 32) {
+        DisplaySurface *ds = qemu_create_displaysurface_from(
+                                 ds_get_width(s->vga.ds),
+                                 ds_get_height(s->vga.ds),
+                                 32,
+                                 ds_get_linesize(s->vga.ds),
+                                 s->vga.vram_ptr);
+        ppm_save(filename, ds, errp);
         g_free(ds);
     }
 }
@@ -1023,8 +1066,9 @@ static void vmsvga_text_update(void *opaque, console_ch_t *chardata)
 {
     struct vmsvga_state_s *s = opaque;
 
-    if (s->vga.text_update)
+    if (s->vga.text_update) {
         s->vga.text_update(&s->vga, chardata);
+    }
 }
 
 static int vmsvga_post_load(void *opaque, int version_id)
@@ -1032,9 +1076,9 @@ static int vmsvga_post_load(void *opaque, int version_id)
     struct vmsvga_state_s *s = opaque;
 
     s->invalidated = 1;
-    if (s->config)
+    if (s->config) {
         s->fifo = (uint32_t *) s->fifo_ptr;
-
+    }
     return 0;
 }
 
@@ -1044,7 +1088,7 @@ static const VMStateDescription vmstate_vmware_vga_internal = {
     .minimum_version_id = 0,
     .minimum_version_id_old = 0,
     .post_load = vmsvga_post_load,
-    .fields      = (VMStateField []) {
+    .fields      = (VMStateField[]) {
         VMSTATE_INT32_EQUAL(depth, struct vmsvga_state_s),
         VMSTATE_INT32(enable, struct vmsvga_state_s),
         VMSTATE_INT32(config, struct vmsvga_state_s),
@@ -1060,7 +1104,7 @@ static const VMStateDescription vmstate_vmware_vga_internal = {
         VMSTATE_UINT32(guest, struct vmsvga_state_s),
         VMSTATE_UINT32(svgaid, struct vmsvga_state_s),
         VMSTATE_INT32(syncing, struct vmsvga_state_s),
-        VMSTATE_INT32(fb_size, struct vmsvga_state_s),
+        VMSTATE_UNUSED(4), /* was fb_size */
         VMSTATE_END_OF_LIST()
     }
 };
@@ -1070,7 +1114,7 @@ static const VMStateDescription vmstate_vmware_vga = {
     .version_id = 0,
     .minimum_version_id = 0,
     .minimum_version_id_old = 0,
-    .fields      = (VMStateField []) {
+    .fields      = (VMStateField[]) {
         VMSTATE_PCI_DEVICE(card, struct pci_vmsvga_state_s),
         VMSTATE_STRUCT(chip, struct pci_vmsvga_state_s, 0,
                        vmstate_vmware_vga_internal, struct vmsvga_state_s),
@@ -1098,40 +1142,16 @@ static void vmsvga_init(struct vmsvga_state_s *s,
     vga_common_init(&s->vga);
     vga_init(&s->vga, address_space, io, true);
     vmstate_register(NULL, 0, &vmstate_vga_common, &s->vga);
-
+    /* Save some values here in case they are changed later.
+     * This is suspicious and needs more though why it is needed. */
     s->depth = ds_get_bits_per_pixel(s->vga.ds);
     s->bypp = ds_get_bytes_per_pixel(s->vga.ds);
-    switch (s->depth) {
-    case 8:
-        s->wred   = 0x00000007;
-        s->wgreen = 0x00000038;
-        s->wblue  = 0x000000c0;
-        break;
-    case 15:
-        s->wred   = 0x0000001f;
-        s->wgreen = 0x000003e0;
-        s->wblue  = 0x00007c00;
-        break;
-    case 16:
-        s->wred   = 0x0000001f;
-        s->wgreen = 0x000007e0;
-        s->wblue  = 0x0000f800;
-        break;
-    case 24:
-        s->wred   = 0x00ff0000;
-        s->wgreen = 0x0000ff00;
-        s->wblue  = 0x000000ff;
-        break;
-    case 32:
-        s->wred   = 0x00ff0000;
-        s->wgreen = 0x0000ff00;
-        s->wblue  = 0x000000ff;
-        break;
-    }
+    s->wred = ds_get_rmask(s->vga.ds);
+    s->wgreen = ds_get_gmask(s->vga.ds);
+    s->wblue = ds_get_bmask(s->vga.ds);
 }
 
-static uint64_t vmsvga_io_read(void *opaque, target_phys_addr_t addr,
-                               unsigned size)
+static uint64_t vmsvga_io_read(void *opaque, hwaddr addr, unsigned size)
 {
     struct vmsvga_state_s *s = opaque;
 
@@ -1143,7 +1163,7 @@ static uint64_t vmsvga_io_read(void *opaque, target_phys_addr_t addr,
     }
 }
 
-static void vmsvga_io_write(void *opaque, target_phys_addr_t addr,
+static void vmsvga_io_write(void *opaque, hwaddr addr,
                             uint64_t data, unsigned size)
 {
     struct vmsvga_state_s *s = opaque;
@@ -1175,22 +1195,20 @@ static int pci_vmsvga_initfn(PCIDevice *dev)
 {
     struct pci_vmsvga_state_s *s =
         DO_UPCAST(struct pci_vmsvga_state_s, card, dev);
-    MemoryRegion *iomem;
-
-    iomem = &s->chip.vga.vram;
 
-    s->card.config[PCI_CACHE_LINE_SIZE]	= 0x08;		/* Cache line size */
-    s->card.config[PCI_LATENCY_TIMER] = 0x40;		/* Latency timer */
-    s->card.config[PCI_INTERRUPT_LINE] = 0xff;		/* End */
+    s->card.config[PCI_CACHE_LINE_SIZE] = 0x08;         /* Cache line size */
+    s->card.config[PCI_LATENCY_TIMER] = 0x40;           /* Latency timer */
+    s->card.config[PCI_INTERRUPT_LINE] = 0xff;          /* End */
 
     memory_region_init_io(&s->io_bar, &vmsvga_io_ops, &s->chip,
                           "vmsvga-io", 0x10);
+    memory_region_set_flush_coalesced(&s->io_bar);
     pci_register_bar(&s->card, 0, PCI_BASE_ADDRESS_SPACE_IO, &s->io_bar);
 
-    vmsvga_init(&s->chip, pci_address_space(dev),
-                pci_address_space_io(dev));
+    vmsvga_init(&s->chip, pci_address_space(dev), pci_address_space_io(dev));
 
-    pci_register_bar(&s->card, 1, PCI_BASE_ADDRESS_MEM_PREFETCH, iomem);
+    pci_register_bar(&s->card, 1, PCI_BASE_ADDRESS_MEM_PREFETCH,
+                     &s->chip.vga.vram);
     pci_register_bar(&s->card, 2, PCI_BASE_ADDRESS_MEM_PREFETCH,
                      &s->chip.fifo_ram);
 
diff --git a/hw/vmware_vga.h b/hw/vmware_vga.h
deleted file mode 100644
index 000fbddc0..000000000
--- a/hw/vmware_vga.h
+++ /dev/null
@@ -1,15 +0,0 @@
-#ifndef QEMU_VMWARE_VGA_H
-#define QEMU_VMWARE_VGA_H
-
-#include "qemu-common.h"
-
-/* vmware_vga.c */
-static inline DeviceState *pci_vmsvga_init(PCIBus *bus)
-{
-    PCIDevice *dev;
-
-    dev = pci_create_simple(bus, -1, "vmware-svga");
-    return &dev->qdev;
-}
-
-#endif
diff --git a/hw/wdt_i6300esb.c b/hw/wdt_i6300esb.c
index 4a8347490..da15c7391 100644
--- a/hw/wdt_i6300esb.c
+++ b/hw/wdt_i6300esb.c
@@ -257,14 +257,14 @@ static uint32_t i6300esb_config_read(PCIDevice *dev, uint32_t addr, int len)
     }
 }
 
-static uint32_t i6300esb_mem_readb(void *vp, target_phys_addr_t addr)
+static uint32_t i6300esb_mem_readb(void *vp, hwaddr addr)
 {
     i6300esb_debug ("addr = %x\n", (int) addr);
 
     return 0;
 }
 
-static uint32_t i6300esb_mem_readw(void *vp, target_phys_addr_t addr)
+static uint32_t i6300esb_mem_readw(void *vp, hwaddr addr)
 {
     uint32_t data = 0;
     I6300State *d = vp;
@@ -282,14 +282,14 @@ static uint32_t i6300esb_mem_readw(void *vp, target_phys_addr_t addr)
     return data;
 }
 
-static uint32_t i6300esb_mem_readl(void *vp, target_phys_addr_t addr)
+static uint32_t i6300esb_mem_readl(void *vp, hwaddr addr)
 {
     i6300esb_debug("addr = %x\n", (int) addr);
 
     return 0;
 }
 
-static void i6300esb_mem_writeb(void *vp, target_phys_addr_t addr, uint32_t val)
+static void i6300esb_mem_writeb(void *vp, hwaddr addr, uint32_t val)
 {
     I6300State *d = vp;
 
@@ -301,7 +301,7 @@ static void i6300esb_mem_writeb(void *vp, target_phys_addr_t addr, uint32_t val)
         d->unlock_state = 2;
 }
 
-static void i6300esb_mem_writew(void *vp, target_phys_addr_t addr, uint32_t val)
+static void i6300esb_mem_writew(void *vp, hwaddr addr, uint32_t val)
 {
     I6300State *d = vp;
 
@@ -334,7 +334,7 @@ static void i6300esb_mem_writew(void *vp, target_phys_addr_t addr, uint32_t val)
     }
 }
 
-static void i6300esb_mem_writel(void *vp, target_phys_addr_t addr, uint32_t val)
+static void i6300esb_mem_writel(void *vp, hwaddr addr, uint32_t val)
 {
     I6300State *d = vp;
 
diff --git a/hw/wm8750.c b/hw/wm8750.c
index 11bcec341..44f138fd5 100644
--- a/hw/wm8750.c
+++ b/hw/wm8750.c
@@ -361,10 +361,10 @@ static int wm8750_tx(I2CSlave *i2c, uint8_t data)
     uint16_t value;
 
     if (s->i2c_len >= 2) {
-        printf("%s: long message (%i bytes)\n", __FUNCTION__, s->i2c_len);
 #ifdef VERBOSE
-        return 1;
+        printf("%s: long message (%i bytes)\n", __func__, s->i2c_len);
 #endif
+        return 1;
     }
     s->i2c_data[s->i2c_len ++] = data;
     if (s->i2c_len != 2)
diff --git a/hw/xen-host-pci-device.c b/hw/xen-host-pci-device.c
index e7ff680ef..743b37b99 100644
--- a/hw/xen-host-pci-device.c
+++ b/hw/xen-host-pci-device.c
@@ -47,13 +47,13 @@ static int xen_host_pci_sysfs_path(const XenHostPCIDevice *d,
 }
 
 
-/* This size should be enough to read the first 7 lines of a ressource file */
-#define XEN_HOST_PCI_RESSOURCE_BUFFER_SIZE 400
+/* This size should be enough to read the first 7 lines of a resource file */
+#define XEN_HOST_PCI_RESOURCE_BUFFER_SIZE 400
 static int xen_host_pci_get_resource(XenHostPCIDevice *d)
 {
     int i, rc, fd;
     char path[PATH_MAX];
-    char buf[XEN_HOST_PCI_RESSOURCE_BUFFER_SIZE];
+    char buf[XEN_HOST_PCI_RESOURCE_BUFFER_SIZE];
     unsigned long long start, end, flags, size;
     char *endptr, *s;
     uint8_t type;
diff --git a/hw/xen.h b/hw/xen.h
index e5926b7b8..e3cca7fb9 100644
--- a/hw/xen.h
+++ b/hw/xen.h
@@ -8,6 +8,7 @@
  */
 #include <inttypes.h>
 
+#include "hw/irq.h"
 #include "qemu-common.h"
 
 /* xen-machine.c */
@@ -48,6 +49,7 @@ void xenstore_store_pv_console_info(int i, struct CharDriverState *chr);
 struct MemoryRegion;
 void xen_ram_alloc(ram_addr_t ram_addr, ram_addr_t size,
                    struct MemoryRegion *mr);
+void xen_modified_memory(ram_addr_t start, ram_addr_t length);
 #endif
 
 struct MemoryRegion;
diff --git a/hw/xen_apic.c b/hw/xen_apic.c
index a9e101f31..fc4536651 100644
--- a/hw/xen_apic.c
+++ b/hw/xen_apic.c
@@ -13,13 +13,13 @@
 #include "hw/msi.h"
 #include "xen.h"
 
-static uint64_t xen_apic_mem_read(void *opaque, target_phys_addr_t addr,
+static uint64_t xen_apic_mem_read(void *opaque, hwaddr addr,
                                   unsigned size)
 {
     return ~(uint64_t)0;
 }
 
-static void xen_apic_mem_write(void *opaque, target_phys_addr_t addr,
+static void xen_apic_mem_write(void *opaque, hwaddr addr,
                                uint64_t data, unsigned size)
 {
     if (size != sizeof(uint32_t)) {
diff --git a/hw/xen_domainbuild.c b/hw/xen_domainbuild.c
index a6a12e593..db1497469 100644
--- a/hw/xen_domainbuild.c
+++ b/hw/xen_domainbuild.c
@@ -153,7 +153,6 @@ static void xen_domain_poll(void *opaque)
 
 quit:
     qemu_system_shutdown_request();
-    return;
 }
 
 static int xen_domain_watcher(void)
diff --git a/hw/xen_machine_pv.c b/hw/xen_machine_pv.c
index 4b72aa755..426470351 100644
--- a/hw/xen_machine_pv.c
+++ b/hw/xen_machine_pv.c
@@ -29,13 +29,12 @@
 #include "xen_domainbuild.h"
 #include "blockdev.h"
 
-static void xen_init_pv(ram_addr_t ram_size,
-			const char *boot_device,
-			const char *kernel_filename,
-			const char *kernel_cmdline,
-			const char *initrd_filename,
-			const char *cpu_model)
+static void xen_init_pv(QEMUMachineInitArgs *args)
 {
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
     X86CPU *cpu;
     CPUX86State *env;
     DriveInfo *dinfo;
diff --git a/hw/xen_nic.c b/hw/xen_nic.c
index 8b79bfb73..cf7d5591b 100644
--- a/hw/xen_nic.c
+++ b/hw/xen_nic.c
@@ -415,6 +415,7 @@ static void net_event(struct XenDevice *xendev)
 {
     struct XenNetDev *netdev = container_of(xendev, struct XenNetDev, xendev);
     net_tx_packets(netdev);
+    qemu_flush_queued_packets(&netdev->nic->nc);
 }
 
 static int net_free(struct XenDevice *xendev)
diff --git a/hw/xen_platform.c b/hw/xen_platform.c
index 0d6c2ff8c..a54e7a2cd 100644
--- a/hw/xen_platform.c
+++ b/hw/xen_platform.c
@@ -85,8 +85,10 @@ static void log_writeb(PCIXenPlatformState *s, char val)
 
 static void unplug_nic(PCIBus *b, PCIDevice *d, void *o)
 {
+    /* We have to ignore passthrough devices */
     if (pci_get_word(d->config + PCI_CLASS_DEVICE) ==
-            PCI_CLASS_NETWORK_ETHERNET) {
+            PCI_CLASS_NETWORK_ETHERNET
+            && strcmp(d->name, "xen-pci-passthrough") != 0) {
         qdev_free(&d->qdev);
     }
 }
@@ -98,8 +100,10 @@ static void pci_unplug_nics(PCIBus *bus)
 
 static void unplug_disks(PCIBus *b, PCIDevice *d, void *o)
 {
+    /* We have to ignore passthrough devices */
     if (pci_get_word(d->config + PCI_CLASS_DEVICE) ==
-            PCI_CLASS_STORAGE_IDE) {
+            PCI_CLASS_STORAGE_IDE
+            && strcmp(d->name, "xen-pci-passthrough") != 0) {
         qdev_unplug(&(d->qdev), NULL);
     }
 }
@@ -224,18 +228,46 @@ static void platform_fixed_ioport_reset(void *opaque)
     platform_fixed_ioport_writeb(s, 0, 0);
 }
 
-const MemoryRegionPortio xen_platform_ioport[] = {
-    { 0, 16, 4, .write = platform_fixed_ioport_writel, },
-    { 0, 16, 2, .write = platform_fixed_ioport_writew, },
-    { 0, 16, 1, .write = platform_fixed_ioport_writeb, },
-    { 0, 16, 2, .read = platform_fixed_ioport_readw, },
-    { 0, 16, 1, .read = platform_fixed_ioport_readb, },
-    PORTIO_END_OF_LIST()
-};
+static uint64_t platform_fixed_ioport_read(void *opaque,
+                                           hwaddr addr,
+                                           unsigned size)
+{
+    switch (size) {
+    case 1:
+        return platform_fixed_ioport_readb(opaque, addr);
+    case 2:
+        return platform_fixed_ioport_readw(opaque, addr);
+    default:
+        return -1;
+    }
+}
+
+static void platform_fixed_ioport_write(void *opaque, hwaddr addr,
+
+                                        uint64_t val, unsigned size)
+{
+    switch (size) {
+    case 1:
+        platform_fixed_ioport_writeb(opaque, addr, val);
+        break;
+    case 2:
+        platform_fixed_ioport_writew(opaque, addr, val);
+        break;
+    case 4:
+        platform_fixed_ioport_writel(opaque, addr, val);
+        break;
+    }
+}
+
 
 static const MemoryRegionOps platform_fixed_io_ops = {
-    .old_portio = xen_platform_ioport,
-    .endianness = DEVICE_NATIVE_ENDIAN,
+    .read = platform_fixed_ioport_read,
+    .write = platform_fixed_ioport_write,
+    .impl = {
+        .min_access_size = 1,
+        .max_access_size = 4,
+    },
+    .endianness = DEVICE_LITTLE_ENDIAN,
 };
 
 static void platform_fixed_ioport_init(PCIXenPlatformState* s)
@@ -288,7 +320,7 @@ static void platform_ioport_bar_setup(PCIXenPlatformState *d)
     memory_region_init_io(&d->bar, &xen_pci_io_ops, d, "xen-pci", 0x100);
 }
 
-static uint64_t platform_mmio_read(void *opaque, target_phys_addr_t addr,
+static uint64_t platform_mmio_read(void *opaque, hwaddr addr,
                                    unsigned size)
 {
     DPRINTF("Warning: attempted read from physical address "
@@ -297,7 +329,7 @@ static uint64_t platform_mmio_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void platform_mmio_write(void *opaque, target_phys_addr_t addr,
+static void platform_mmio_write(void *opaque, hwaddr addr,
                                 uint64_t val, unsigned size)
 {
     DPRINTF("Warning: attempted write of 0x%"PRIx64" to physical "
diff --git a/hw/xen_pt.c b/hw/xen_pt.c
index 307119a12..7a3846e64 100644
--- a/hw/xen_pt.c
+++ b/hw/xen_pt.c
@@ -59,6 +59,7 @@
 #include "xen_backend.h"
 #include "xen_pt.h"
 #include "range.h"
+#include "exec-memory.h"
 
 #define XEN_PT_NR_IRQS (256)
 static uint8_t xen_pt_mapped_machine_irq[XEN_PT_NR_IRQS] = {0};
@@ -362,7 +363,7 @@ out:
 
 /* register regions */
 
-static uint64_t xen_pt_bar_read(void *o, target_phys_addr_t addr,
+static uint64_t xen_pt_bar_read(void *o, hwaddr addr,
                                 unsigned size)
 {
     PCIDevice *d = o;
@@ -372,7 +373,7 @@ static uint64_t xen_pt_bar_read(void *o, target_phys_addr_t addr,
                addr);
     return 0;
 }
-static void xen_pt_bar_write(void *o, target_phys_addr_t addr, uint64_t val,
+static void xen_pt_bar_write(void *o, hwaddr addr, uint64_t val,
                              unsigned size)
 {
     PCIDevice *d = o;
@@ -410,14 +411,17 @@ static int xen_pt_register_regions(XenPCIPassthroughState *s)
             if (r->type & XEN_HOST_PCI_REGION_TYPE_PREFETCH) {
                 type |= PCI_BASE_ADDRESS_MEM_PREFETCH;
             }
+            if (r->type & XEN_HOST_PCI_REGION_TYPE_MEM_64) {
+                type |= PCI_BASE_ADDRESS_MEM_TYPE_64;
+            }
         }
 
         memory_region_init_io(&s->bar[i], &ops, &s->dev,
                               "xen-pci-pt-bar", r->size);
         pci_register_bar(&s->dev, i, type, &s->bar[i]);
 
-        XEN_PT_LOG(&s->dev, "IO region %i registered (size=0x%08"PRIx64
-                   " base_addr=0x%08"PRIx64" type: %#x)\n",
+        XEN_PT_LOG(&s->dev, "IO region %i registered (size=0x%lx"PRIx64
+                   " base_addr=0x%lx"PRIx64" type: %#x)\n",
                    i, r->size, r->base_addr, type);
     }
 
@@ -597,14 +601,6 @@ static void xen_pt_region_update(XenPCIPassthroughState *s,
     }
 }
 
-static void xen_pt_begin(MemoryListener *l)
-{
-}
-
-static void xen_pt_commit(MemoryListener *l)
-{
-}
-
 static void xen_pt_region_add(MemoryListener *l, MemoryRegionSection *sec)
 {
     XenPCIPassthroughState *s = container_of(l, XenPCIPassthroughState,
@@ -621,36 +617,31 @@ static void xen_pt_region_del(MemoryListener *l, MemoryRegionSection *sec)
     xen_pt_region_update(s, sec, false);
 }
 
-static void xen_pt_region_nop(MemoryListener *l, MemoryRegionSection *s)
+static void xen_pt_io_region_add(MemoryListener *l, MemoryRegionSection *sec)
 {
-}
+    XenPCIPassthroughState *s = container_of(l, XenPCIPassthroughState,
+                                             io_listener);
 
-static void xen_pt_log_fns(MemoryListener *l, MemoryRegionSection *s)
-{
+    xen_pt_region_update(s, sec, true);
 }
 
-static void xen_pt_log_global_fns(MemoryListener *l)
+static void xen_pt_io_region_del(MemoryListener *l, MemoryRegionSection *sec)
 {
-}
+    XenPCIPassthroughState *s = container_of(l, XenPCIPassthroughState,
+                                             io_listener);
 
-static void xen_pt_eventfd_fns(MemoryListener *l, MemoryRegionSection *s,
-                               bool match_data, uint64_t data, EventNotifier *n)
-{
+    xen_pt_region_update(s, sec, false);
 }
 
 static const MemoryListener xen_pt_memory_listener = {
-    .begin = xen_pt_begin,
-    .commit = xen_pt_commit,
     .region_add = xen_pt_region_add,
-    .region_nop = xen_pt_region_nop,
     .region_del = xen_pt_region_del,
-    .log_start = xen_pt_log_fns,
-    .log_stop = xen_pt_log_fns,
-    .log_sync = xen_pt_log_fns,
-    .log_global_start = xen_pt_log_global_fns,
-    .log_global_stop = xen_pt_log_global_fns,
-    .eventfd_add = xen_pt_eventfd_fns,
-    .eventfd_del = xen_pt_eventfd_fns,
+    .priority = 10,
+};
+
+static const MemoryListener xen_pt_io_listener = {
+    .region_add = xen_pt_io_region_add,
+    .region_del = xen_pt_io_region_del,
     .priority = 10,
 };
 
@@ -691,6 +682,7 @@ static int xen_pt_initfn(PCIDevice *d)
     }
 
     s->memory_listener = xen_pt_memory_listener;
+    s->io_listener = xen_pt_io_listener;
 
     /* Handle real device's MMIO/PIO BARs */
     xen_pt_register_regions(s);
@@ -757,7 +749,8 @@ static int xen_pt_initfn(PCIDevice *d)
     }
 
 out:
-    memory_listener_register(&s->memory_listener, NULL);
+    memory_listener_register(&s->memory_listener, &address_space_memory);
+    memory_listener_register(&s->io_listener, &address_space_io);
     XEN_PT_LOG(d, "Real physical device %02x:%02x.%d registered successfuly!\n",
                bus, slot, func);
 
@@ -812,6 +805,7 @@ static void xen_pt_unregister_device(PCIDevice *d)
 
     xen_pt_unregister_regions(s);
     memory_listener_unregister(&s->memory_listener);
+    memory_listener_unregister(&s->io_listener);
 
     xen_host_pci_device_put(&s->real_device);
 }
diff --git a/hw/xen_pt.h b/hw/xen_pt.h
index 41904ece9..f15e69a29 100644
--- a/hw/xen_pt.h
+++ b/hw/xen_pt.h
@@ -96,7 +96,7 @@ typedef struct XenPTRegion {
  * - do NOT use ALL F for init_val, otherwise the tbl will not be registered.
  */
 
-/* emulated register infomation */
+/* emulated register information */
 struct XenPTRegInfo {
     uint32_t offset;
     uint32_t size;
@@ -140,7 +140,7 @@ typedef int (*xen_pt_reg_size_init_fn)
     (XenPCIPassthroughState *, const XenPTRegGroupInfo *,
      uint32_t base_offset, uint8_t *size);
 
-/* emulated register group infomation */
+/* emulated register group information */
 struct XenPTRegGroupInfo {
     uint8_t grp_id;
     XenPTRegisterGroupType grp_type;
@@ -209,6 +209,7 @@ struct XenPCIPassthroughState {
     MemoryRegion rom;
 
     MemoryListener memory_listener;
+    MemoryListener io_listener;
 };
 
 int xen_pt_config_init(XenPCIPassthroughState *s);
diff --git a/hw/xen_pt_config_init.c b/hw/xen_pt_config_init.c
index 00eb3d997..0a5f82cb8 100644
--- a/hw/xen_pt_config_init.c
+++ b/hw/xen_pt_config_init.c
@@ -342,6 +342,23 @@ static int xen_pt_cmd_reg_write(XenPCIPassthroughState *s, XenPTReg *cfg_entry,
 #define XEN_PT_BAR_IO_RO_MASK     0x00000003  /* BAR ReadOnly mask(I/O) */
 #define XEN_PT_BAR_IO_EMU_MASK    0xFFFFFFFC  /* BAR emul mask(I/O) */
 
+static bool is_64bit_bar(PCIIORegion *r)
+{
+    return !!(r->type & PCI_BASE_ADDRESS_MEM_TYPE_64);
+}
+
+static uint64_t xen_pt_get_bar_size(PCIIORegion *r)
+{
+    if (is_64bit_bar(r)) {
+        uint64_t size64;
+        size64 = (r + 1)->size;
+        size64 <<= 32;
+        size64 += r->size;
+        return size64;
+    }
+    return r->size;
+}
+
 static XenPTBarFlag xen_pt_bar_reg_parse(XenPCIPassthroughState *s,
                                          XenPTRegInfo *reg)
 {
@@ -366,7 +383,7 @@ static XenPTBarFlag xen_pt_bar_reg_parse(XenPCIPassthroughState *s,
 
     /* check unused BAR */
     r = &d->io_regions[index];
-    if (r->size == 0) {
+    if (!xen_pt_get_bar_size(r)) {
         return XEN_PT_BAR_FLAG_UNUSED;
     }
 
@@ -481,7 +498,12 @@ static int xen_pt_bar_reg_write(XenPCIPassthroughState *s, XenPTReg *cfg_entry,
     switch (s->bases[index].bar_flag) {
     case XEN_PT_BAR_FLAG_MEM:
         bar_emu_mask = XEN_PT_BAR_MEM_EMU_MASK;
-        bar_ro_mask = XEN_PT_BAR_MEM_RO_MASK | (r_size - 1);
+        if (!r_size) {
+            /* low 32 bits mask for 64 bit bars */
+            bar_ro_mask = XEN_PT_BAR_ALLF;
+        } else {
+            bar_ro_mask = XEN_PT_BAR_MEM_RO_MASK | (r_size - 1);
+        }
         break;
     case XEN_PT_BAR_FLAG_IO:
         bar_emu_mask = XEN_PT_BAR_IO_EMU_MASK;
@@ -489,7 +511,7 @@ static int xen_pt_bar_reg_write(XenPCIPassthroughState *s, XenPTReg *cfg_entry,
         break;
     case XEN_PT_BAR_FLAG_UPPER:
         bar_emu_mask = XEN_PT_BAR_ALLF;
-        bar_ro_mask = 0;    /* all upper 32bit are R/W */
+        bar_ro_mask = r_size ? r_size - 1 : 0;
         break;
     default:
         break;
@@ -501,22 +523,13 @@ static int xen_pt_bar_reg_write(XenPCIPassthroughState *s, XenPTReg *cfg_entry,
 
     /* check whether we need to update the virtual region address or not */
     switch (s->bases[index].bar_flag) {
+    case XEN_PT_BAR_FLAG_UPPER:
     case XEN_PT_BAR_FLAG_MEM:
         /* nothing to do */
         break;
     case XEN_PT_BAR_FLAG_IO:
         /* nothing to do */
         break;
-    case XEN_PT_BAR_FLAG_UPPER:
-        if (cfg_entry->data) {
-            if (cfg_entry->data != (XEN_PT_BAR_ALLF & ~bar_ro_mask)) {
-                XEN_PT_WARN(d, "Guest attempt to set high MMIO Base Address. "
-                            "Ignore mapping. "
-                            "(offset: 0x%02x, high address: 0x%08x)\n",
-                            reg->offset, cfg_entry->data);
-            }
-        }
-        break;
     default:
         break;
     }
@@ -562,7 +575,7 @@ static int xen_pt_exp_rom_bar_reg_write(XenPCIPassthroughState *s,
     return 0;
 }
 
-/* Header Type0 reg static infomation table */
+/* Header Type0 reg static information table */
 static XenPTRegInfo xen_pt_emu_reg_header0[] = {
     /* Vendor ID reg */
     {
@@ -753,7 +766,7 @@ static XenPTRegInfo xen_pt_emu_reg_header0[] = {
  * Vital Product Data Capability
  */
 
-/* Vital Product Data Capability Structure reg static infomation table */
+/* Vital Product Data Capability Structure reg static information table */
 static XenPTRegInfo xen_pt_emu_reg_vpd[] = {
     {
         .offset     = PCI_CAP_LIST_NEXT,
@@ -775,7 +788,7 @@ static XenPTRegInfo xen_pt_emu_reg_vpd[] = {
  * Vendor Specific Capability
  */
 
-/* Vendor Specific Capability Structure reg static infomation table */
+/* Vendor Specific Capability Structure reg static information table */
 static XenPTRegInfo xen_pt_emu_reg_vendor[] = {
     {
         .offset     = PCI_CAP_LIST_NEXT,
@@ -866,7 +879,7 @@ static int xen_pt_linkctrl2_reg_init(XenPCIPassthroughState *s,
     return 0;
 }
 
-/* PCI Express Capability Structure reg static infomation table */
+/* PCI Express Capability Structure reg static information table */
 static XenPTRegInfo xen_pt_emu_reg_pcie[] = {
     /* Next Pointer reg */
     {
@@ -981,7 +994,7 @@ static int xen_pt_pmcsr_reg_write(XenPCIPassthroughState *s,
     return 0;
 }
 
-/* Power Management Capability reg static infomation table */
+/* Power Management Capability reg static information table */
 static XenPTRegInfo xen_pt_emu_reg_pm[] = {
     /* Next Pointer reg */
     {
@@ -1259,7 +1272,7 @@ static int xen_pt_msgdata_reg_write(XenPCIPassthroughState *s,
     return 0;
 }
 
-/* MSI Capability Structure reg static infomation table */
+/* MSI Capability Structure reg static information table */
 static XenPTRegInfo xen_pt_emu_reg_msi[] = {
     /* Next Pointer reg */
     {
@@ -1396,7 +1409,7 @@ static int xen_pt_msixctrl_reg_write(XenPCIPassthroughState *s,
     return 0;
 }
 
-/* MSI-X Capability Structure reg static infomation table */
+/* MSI-X Capability Structure reg static information table */
 static XenPTRegInfo xen_pt_emu_reg_msix[] = {
     /* Next Pointer reg */
     {
diff --git a/hw/xen_pt_msi.c b/hw/xen_pt_msi.c
index 2299cc777..680767229 100644
--- a/hw/xen_pt_msi.c
+++ b/hw/xen_pt_msi.c
@@ -427,7 +427,7 @@ static void set_entry_value(XenPTMSIXEntry *e, int offset, uint32_t val)
     }
 }
 
-static void pci_msix_write(void *opaque, target_phys_addr_t addr,
+static void pci_msix_write(void *opaque, hwaddr addr,
                            uint64_t val, unsigned size)
 {
     XenPCIPassthroughState *s = opaque;
@@ -475,7 +475,7 @@ static void pci_msix_write(void *opaque, target_phys_addr_t addr,
     }
 }
 
-static uint64_t pci_msix_read(void *opaque, target_phys_addr_t addr,
+static uint64_t pci_msix_read(void *opaque, hwaddr addr,
                               unsigned size)
 {
     XenPCIPassthroughState *s = opaque;
diff --git a/hw/xenfb.c b/hw/xenfb.c
index 338800a4d..442a63a32 100644
--- a/hw/xenfb.c
+++ b/hw/xenfb.c
@@ -648,7 +648,7 @@ static void xenfb_guest_copy(struct XenFB *xenfb, int x, int y, int w, int h)
         xen_be_printf(&xenfb->c.xendev, 0, "%s: oops: convert %d -> %d bpp?\n",
                       __FUNCTION__, xenfb->depth, bpp);
 
-    dpy_update(xenfb->c.ds, x, y, w, h);
+    dpy_gfx_update(xenfb->c.ds, x, y, w, h);
 }
 
 #ifdef XENFB_TYPE_REFRESH_PERIOD
@@ -717,7 +717,7 @@ static void xenfb_update(void *opaque)
 	if (xenfb_queue_full(xenfb))
 	    return;
 
-        for (l = xenfb->c.ds->listeners; l != NULL; l = l->next) {
+        QLIST_FOREACH(l, &xenfb->c.ds->listeners, next) {
             if (l->idle)
                 continue;
             idle = 0;
@@ -766,7 +766,7 @@ static void xenfb_update(void *opaque)
         xen_be_printf(&xenfb->c.xendev, 1, "update: resizing: %dx%d @ %d bpp%s\n",
                       xenfb->width, xenfb->height, xenfb->depth,
                       is_buffer_shared(xenfb->c.ds->surface) ? " (shared)" : "");
-        dpy_resize(xenfb->c.ds);
+        dpy_gfx_resize(xenfb->c.ds);
         xenfb->up_fullscreen = 1;
     }
 
diff --git a/hw/xgmac.c b/hw/xgmac.c
index a91ef608f..ec50c745d 100644
--- a/hw/xgmac.c
+++ b/hw/xgmac.c
@@ -252,7 +252,7 @@ static void enet_update_irq(struct XgmacState *s)
     qemu_set_irq(s->sbd_irq, !!stat);
 }
 
-static uint64_t enet_read(void *opaque, target_phys_addr_t addr, unsigned size)
+static uint64_t enet_read(void *opaque, hwaddr addr, unsigned size)
 {
     struct XgmacState *s = opaque;
     uint64_t r = 0;
@@ -271,7 +271,7 @@ static uint64_t enet_read(void *opaque, target_phys_addr_t addr, unsigned size)
     return r;
 }
 
-static void enet_write(void *opaque, target_phys_addr_t addr,
+static void enet_write(void *opaque, hwaddr addr,
                        uint64_t value, unsigned size)
 {
     struct XgmacState *s = opaque;
diff --git a/hw/xics.c b/hw/xics.c
index b674771dc..1da310653 100644
--- a/hw/xics.c
+++ b/hw/xics.c
@@ -108,13 +108,13 @@ static void icp_set_cppr(struct icp_state *icp, int server, uint8_t cppr)
     }
 }
 
-static void icp_set_mfrr(struct icp_state *icp, int nr, uint8_t mfrr)
+static void icp_set_mfrr(struct icp_state *icp, int server, uint8_t mfrr)
 {
-    struct icp_server_state *ss = icp->ss + nr;
+    struct icp_server_state *ss = icp->ss + server;
 
     ss->mfrr = mfrr;
     if (mfrr < CPPR(ss)) {
-        icp_check_ipi(icp, nr);
+        icp_check_ipi(icp, server);
     }
 }
 
@@ -165,11 +165,12 @@ struct ics_irq_state {
     int server;
     uint8_t priority;
     uint8_t saved_priority;
-    enum xics_irq_type type;
-    int asserted:1;
-    int sent:1;
-    int rejected:1;
-    int masked_pending:1;
+#define XICS_STATUS_ASSERTED           0x1
+#define XICS_STATUS_SENT               0x2
+#define XICS_STATUS_REJECTED           0x4
+#define XICS_STATUS_MASKED_PENDING     0x8
+    uint8_t status;
+    bool lsi;
 };
 
 struct ics_state {
@@ -191,8 +192,8 @@ static void resend_msi(struct ics_state *ics, int srcno)
     struct ics_irq_state *irq = ics->irqs + srcno;
 
     /* FIXME: filter by server#? */
-    if (irq->rejected) {
-        irq->rejected = 0;
+    if (irq->status & XICS_STATUS_REJECTED) {
+        irq->status &= ~XICS_STATUS_REJECTED;
         if (irq->priority != 0xff) {
             icp_irq(ics->icp, irq->server, srcno + ics->offset,
                     irq->priority);
@@ -204,8 +205,10 @@ static void resend_lsi(struct ics_state *ics, int srcno)
 {
     struct ics_irq_state *irq = ics->irqs + srcno;
 
-    if ((irq->priority != 0xff) && irq->asserted && !irq->sent) {
-        irq->sent = 1;
+    if ((irq->priority != 0xff)
+        && (irq->status & XICS_STATUS_ASSERTED)
+        && !(irq->status & XICS_STATUS_SENT)) {
+        irq->status |= XICS_STATUS_SENT;
         icp_irq(ics->icp, irq->server, srcno + ics->offset, irq->priority);
     }
 }
@@ -216,7 +219,7 @@ static void set_irq_msi(struct ics_state *ics, int srcno, int val)
 
     if (val) {
         if (irq->priority == 0xff) {
-            irq->masked_pending = 1;
+            irq->status |= XICS_STATUS_MASKED_PENDING;
             /* masked pending */ ;
         } else  {
             icp_irq(ics->icp, irq->server, srcno + ics->offset, irq->priority);
@@ -228,7 +231,11 @@ static void set_irq_lsi(struct ics_state *ics, int srcno, int val)
 {
     struct ics_irq_state *irq = ics->irqs + srcno;
 
-    irq->asserted = val;
+    if (val) {
+        irq->status |= XICS_STATUS_ASSERTED;
+    } else {
+        irq->status &= ~XICS_STATUS_ASSERTED;
+    }
     resend_lsi(ics, srcno);
 }
 
@@ -237,7 +244,7 @@ static void ics_set_irq(void *opaque, int srcno, int val)
     struct ics_state *ics = (struct ics_state *)opaque;
     struct ics_irq_state *irq = ics->irqs + srcno;
 
-    if (irq->type == XICS_LSI) {
+    if (irq->lsi) {
         set_irq_lsi(ics, srcno, val);
     } else {
         set_irq_msi(ics, srcno, val);
@@ -248,11 +255,12 @@ static void write_xive_msi(struct ics_state *ics, int srcno)
 {
     struct ics_irq_state *irq = ics->irqs + srcno;
 
-    if (!irq->masked_pending || (irq->priority == 0xff)) {
+    if (!(irq->status & XICS_STATUS_MASKED_PENDING)
+        || (irq->priority == 0xff)) {
         return;
     }
 
-    irq->masked_pending = 0;
+    irq->status &= ~XICS_STATUS_MASKED_PENDING;
     icp_irq(ics->icp, irq->server, srcno + ics->offset, irq->priority);
 }
 
@@ -262,15 +270,16 @@ static void write_xive_lsi(struct ics_state *ics, int srcno)
 }
 
 static void ics_write_xive(struct ics_state *ics, int nr, int server,
-                           uint8_t priority)
+                           uint8_t priority, uint8_t saved_priority)
 {
     int srcno = nr - ics->offset;
     struct ics_irq_state *irq = ics->irqs + srcno;
 
     irq->server = server;
     irq->priority = priority;
+    irq->saved_priority = saved_priority;
 
-    if (irq->type == XICS_LSI) {
+    if (irq->lsi) {
         write_xive_lsi(ics, srcno);
     } else {
         write_xive_msi(ics, srcno);
@@ -281,8 +290,8 @@ static void ics_reject(struct ics_state *ics, int nr)
 {
     struct ics_irq_state *irq = ics->irqs + nr - ics->offset;
 
-    irq->rejected = 1; /* Irrelevant but harmless for LSI */
-    irq->sent = 0; /* Irrelevant but harmless for MSI */
+    irq->status |= XICS_STATUS_REJECTED; /* Irrelevant but harmless for LSI */
+    irq->status &= ~XICS_STATUS_SENT; /* Irrelevant but harmless for MSI */
 }
 
 static void ics_resend(struct ics_state *ics)
@@ -293,7 +302,7 @@ static void ics_resend(struct ics_state *ics)
         struct ics_irq_state *irq = ics->irqs + i;
 
         /* FIXME: filter by server#? */
-        if (irq->type == XICS_LSI) {
+        if (irq->lsi) {
             resend_lsi(ics, i);
         } else {
             resend_msi(ics, i);
@@ -306,8 +315,8 @@ static void ics_eoi(struct ics_state *ics, int nr)
     int srcno = nr - ics->offset;
     struct ics_irq_state *irq = ics->irqs + srcno;
 
-    if (irq->type == XICS_LSI) {
-        irq->sent = 0;
+    if (irq->lsi) {
+        irq->status &= ~XICS_STATUS_SENT;
     }
 }
 
@@ -317,34 +326,31 @@ static void ics_eoi(struct ics_state *ics, int nr)
 
 qemu_irq xics_get_qirq(struct icp_state *icp, int irq)
 {
-    if ((irq < icp->ics->offset)
-        || (irq >= (icp->ics->offset + icp->ics->nr_irqs))) {
+    if (!ics_valid_irq(icp->ics, irq)) {
         return NULL;
     }
 
     return icp->ics->qirqs[irq - icp->ics->offset];
 }
 
-void xics_set_irq_type(struct icp_state *icp, int irq,
-                       enum xics_irq_type type)
+void xics_set_irq_type(struct icp_state *icp, int irq, bool lsi)
 {
-    assert((irq >= icp->ics->offset)
-           && (irq < (icp->ics->offset + icp->ics->nr_irqs)));
-    assert((type == XICS_MSI) || (type == XICS_LSI));
+    assert(ics_valid_irq(icp->ics, irq));
 
-    icp->ics->irqs[irq - icp->ics->offset].type = type;
+    icp->ics->irqs[irq - icp->ics->offset].lsi = lsi;
 }
 
-static target_ulong h_cppr(CPUPPCState *env, sPAPREnvironment *spapr,
+static target_ulong h_cppr(PowerPCCPU *cpu, sPAPREnvironment *spapr,
                            target_ulong opcode, target_ulong *args)
 {
+    CPUPPCState *env = &cpu->env;
     target_ulong cppr = args[0];
 
     icp_set_cppr(spapr->icp, env->cpu_index, cppr);
     return H_SUCCESS;
 }
 
-static target_ulong h_ipi(CPUPPCState *env, sPAPREnvironment *spapr,
+static target_ulong h_ipi(PowerPCCPU *cpu, sPAPREnvironment *spapr,
                           target_ulong opcode, target_ulong *args)
 {
     target_ulong server = args[0];
@@ -359,18 +365,20 @@ static target_ulong h_ipi(CPUPPCState *env, sPAPREnvironment *spapr,
 
 }
 
-static target_ulong h_xirr(CPUPPCState *env, sPAPREnvironment *spapr,
+static target_ulong h_xirr(PowerPCCPU *cpu, sPAPREnvironment *spapr,
                            target_ulong opcode, target_ulong *args)
 {
+    CPUPPCState *env = &cpu->env;
     uint32_t xirr = icp_accept(spapr->icp->ss + env->cpu_index);
 
     args[0] = xirr;
     return H_SUCCESS;
 }
 
-static target_ulong h_eoi(CPUPPCState *env, sPAPREnvironment *spapr,
+static target_ulong h_eoi(PowerPCCPU *cpu, sPAPREnvironment *spapr,
                           target_ulong opcode, target_ulong *args)
 {
+    CPUPPCState *env = &cpu->env;
     target_ulong xirr = args[0];
 
     icp_eoi(spapr->icp, env->cpu_index, xirr);
@@ -399,7 +407,7 @@ static void rtas_set_xive(sPAPREnvironment *spapr, uint32_t token,
         return;
     }
 
-    ics_write_xive(ics, nr, server, priority);
+    ics_write_xive(ics, nr, server, priority, priority);
 
     rtas_st(rets, 0, 0); /* Success */
 }
@@ -447,14 +455,8 @@ static void rtas_int_off(sPAPREnvironment *spapr, uint32_t token,
         return;
     }
 
-    /* This is a NOP for now, since the described PAPR semantics don't
-     * seem to gel with what Linux does */
-#if 0
-    struct ics_irq_state *irq = xics->irqs + (nr - xics->offset);
-
-    irq->saved_priority = irq->priority;
-    ics_write_xive_msi(xics, nr, irq->server, 0xff);
-#endif
+    ics_write_xive(ics, nr, ics->irqs[nr - ics->offset].server, 0xff,
+                   ics->irqs[nr - ics->offset].priority);
 
     rtas_st(rets, 0, 0); /* Success */
 }
@@ -478,22 +480,40 @@ static void rtas_int_on(sPAPREnvironment *spapr, uint32_t token,
         return;
     }
 
-    /* This is a NOP for now, since the described PAPR semantics don't
-     * seem to gel with what Linux does */
-#if 0
-    struct ics_irq_state *irq = xics->irqs + (nr - xics->offset);
-
-    ics_write_xive_msi(xics, nr, irq->server, irq->saved_priority);
-#endif
+    ics_write_xive(ics, nr, ics->irqs[nr - ics->offset].server,
+                   ics->irqs[nr - ics->offset].saved_priority,
+                   ics->irqs[nr - ics->offset].saved_priority);
 
     rtas_st(rets, 0, 0); /* Success */
 }
 
+static void xics_reset(void *opaque)
+{
+    struct icp_state *icp = (struct icp_state *)opaque;
+    struct ics_state *ics = icp->ics;
+    int i;
+
+    for (i = 0; i < icp->nr_servers; i++) {
+        icp->ss[i].xirr = 0;
+        icp->ss[i].pending_priority = 0;
+        icp->ss[i].mfrr = 0xff;
+        /* Make all outputs are deasserted */
+        qemu_set_irq(icp->ss[i].output, 0);
+    }
+
+    for (i = 0; i < ics->nr_irqs; i++) {
+        /* Reset everything *except* the type */
+        ics->irqs[i].server = 0;
+        ics->irqs[i].status = 0;
+        ics->irqs[i].priority = 0xff;
+        ics->irqs[i].saved_priority = 0xff;
+    }
+}
+
 struct icp_state *xics_system_init(int nr_irqs)
 {
     CPUPPCState *env;
     int max_server_num;
-    int i;
     struct icp_state *icp;
     struct ics_state *ics;
 
@@ -508,10 +528,6 @@ struct icp_state *xics_system_init(int nr_irqs)
     icp->nr_servers = max_server_num + 1;
     icp->ss = g_malloc0(icp->nr_servers*sizeof(struct icp_server_state));
 
-    for (i = 0; i < icp->nr_servers; i++) {
-        icp->ss[i].mfrr = 0xff;
-    }
-
     for (env = first_cpu; env != NULL; env = env->next_cpu) {
         struct icp_server_state *ss = &icp->ss[env->cpu_index];
 
@@ -539,11 +555,6 @@ struct icp_state *xics_system_init(int nr_irqs)
     icp->ics = ics;
     ics->icp = icp;
 
-    for (i = 0; i < nr_irqs; i++) {
-        ics->irqs[i].priority = 0xff;
-        ics->irqs[i].saved_priority = 0xff;
-    }
-
     ics->qirqs = qemu_allocate_irqs(ics_set_irq, ics, nr_irqs);
 
     spapr_register_hypercall(H_CPPR, h_cppr);
@@ -556,5 +567,7 @@ struct icp_state *xics_system_init(int nr_irqs)
     spapr_rtas_register("ibm,int-off", rtas_int_off);
     spapr_rtas_register("ibm,int-on", rtas_int_on);
 
+    qemu_register_reset(xics_reset, icp);
+
     return icp;
 }
diff --git a/hw/xics.h b/hw/xics.h
index 99b96ac85..681726869 100644
--- a/hw/xics.h
+++ b/hw/xics.h
@@ -31,14 +31,8 @@
 
 struct icp_state;
 
-enum xics_irq_type {
-    XICS_MSI,        /* Message-signalled (edge) interrupt */
-    XICS_LSI,        /* Level-signalled interrupt */
-};
-
 qemu_irq xics_get_qirq(struct icp_state *icp, int irq);
-void xics_set_irq_type(struct icp_state *icp, int irq,
-                       enum xics_irq_type type);
+void xics_set_irq_type(struct icp_state *icp, int irq, bool lsi);
 
 struct icp_state *xics_system_init(int nr_irqs);
 
diff --git a/hw/xilinx.h b/hw/xilinx.h
index 556c5aa9f..9323fd07c 100644
--- a/hw/xilinx.h
+++ b/hw/xilinx.h
@@ -3,7 +3,7 @@
 #include "net.h"
 
 static inline DeviceState *
-xilinx_intc_create(target_phys_addr_t base, qemu_irq irq, int kind_of_intr)
+xilinx_intc_create(hwaddr base, qemu_irq irq, int kind_of_intr)
 {
     DeviceState *dev;
 
@@ -17,13 +17,13 @@ xilinx_intc_create(target_phys_addr_t base, qemu_irq irq, int kind_of_intr)
 
 /* OPB Timer/Counter.  */
 static inline DeviceState *
-xilinx_timer_create(target_phys_addr_t base, qemu_irq irq, int oto, int freq)
+xilinx_timer_create(hwaddr base, qemu_irq irq, int oto, int freq)
 {
     DeviceState *dev;
 
-    dev = qdev_create(NULL, "xlnx,xps-timer");
+    dev = qdev_create(NULL, "xlnx.xps-timer");
     qdev_prop_set_uint32(dev, "one-timer-only", oto);
-    qdev_prop_set_uint32(dev, "frequency", freq);
+    qdev_prop_set_uint32(dev, "clock-frequency", freq);
     qdev_init_nofail(dev);
     sysbus_mmio_map(sysbus_from_qdev(dev), 0, base);
     sysbus_connect_irq(sysbus_from_qdev(dev), 0, irq);
@@ -32,7 +32,7 @@ xilinx_timer_create(target_phys_addr_t base, qemu_irq irq, int oto, int freq)
 
 /* XPS Ethernet Lite MAC.  */
 static inline DeviceState *
-xilinx_ethlite_create(NICInfo *nd, target_phys_addr_t base, qemu_irq irq,
+xilinx_ethlite_create(NICInfo *nd, hwaddr base, qemu_irq irq,
                       int txpingpong, int rxpingpong)
 {
     DeviceState *dev;
@@ -51,17 +51,21 @@ xilinx_ethlite_create(NICInfo *nd, target_phys_addr_t base, qemu_irq irq,
 
 static inline DeviceState *
 xilinx_axiethernet_create(NICInfo *nd, StreamSlave *peer,
-                          target_phys_addr_t base, qemu_irq irq,
+                          hwaddr base, qemu_irq irq,
                           int txmem, int rxmem)
 {
     DeviceState *dev;
+    Error *errp = NULL;
+
     qemu_check_nic_model(nd, "xlnx.axi-ethernet");
 
     dev = qdev_create(NULL, "xlnx.axi-ethernet");
     qdev_set_nic_properties(dev, nd);
     qdev_prop_set_uint32(dev, "rxmem", rxmem);
     qdev_prop_set_uint32(dev, "txmem", txmem);
-    object_property_set_link(OBJECT(dev), OBJECT(peer), "tx_dev", NULL);
+    object_property_set_link(OBJECT(dev), OBJECT(peer), "axistream-connected",
+                             &errp);
+    assert_no_error(errp);
     qdev_init_nofail(dev);
     sysbus_mmio_map(sysbus_from_qdev(dev), 0, base);
     sysbus_connect_irq(sysbus_from_qdev(dev), 0, irq);
@@ -71,11 +75,15 @@ xilinx_axiethernet_create(NICInfo *nd, StreamSlave *peer,
 
 static inline void
 xilinx_axiethernetdma_init(DeviceState *dev, StreamSlave *peer,
-                           target_phys_addr_t base, qemu_irq irq,
+                           hwaddr base, qemu_irq irq,
                            qemu_irq irq2, int freqhz)
 {
+    Error *errp = NULL;
+
     qdev_prop_set_uint32(dev, "freqhz", freqhz);
-    object_property_set_link(OBJECT(dev), OBJECT(peer), "tx_dev", NULL);
+    object_property_set_link(OBJECT(dev), OBJECT(peer), "axistream-connected",
+                             &errp);
+    assert_no_error(errp);
     qdev_init_nofail(dev);
 
     sysbus_mmio_map(sysbus_from_qdev(dev), 0, base);
diff --git a/hw/xilinx_axidma.c b/hw/xilinx_axidma.c
index 0e28c5173..4575da176 100644
--- a/hw/xilinx_axidma.c
+++ b/hw/xilinx_axidma.c
@@ -140,7 +140,7 @@ static void stream_reset(struct Stream *s)
 }
 
 /* Map an offset addr into a channel index.  */
-static inline int streamid_from_addr(target_phys_addr_t addr)
+static inline int streamid_from_addr(hwaddr addr)
 {
     int sid;
 
@@ -159,7 +159,7 @@ static void stream_desc_show(struct SDesc *d)
 }
 #endif
 
-static void stream_desc_load(struct Stream *s, target_phys_addr_t addr)
+static void stream_desc_load(struct Stream *s, hwaddr addr)
 {
     struct SDesc *d = &s->desc;
     int i;
@@ -176,7 +176,7 @@ static void stream_desc_load(struct Stream *s, target_phys_addr_t addr)
     }
 }
 
-static void stream_desc_store(struct Stream *s, target_phys_addr_t addr)
+static void stream_desc_store(struct Stream *s, hwaddr addr)
 {
     struct SDesc *d = &s->desc;
     int i;
@@ -364,7 +364,7 @@ axidma_push(StreamSlave *obj, unsigned char *buf, size_t len, uint32_t *app)
     stream_update_irq(s);
 }
 
-static uint64_t axidma_read(void *opaque, target_phys_addr_t addr,
+static uint64_t axidma_read(void *opaque, hwaddr addr,
                             unsigned size)
 {
     struct XilinxAXIDMA *d = opaque;
@@ -399,7 +399,7 @@ static uint64_t axidma_read(void *opaque, target_phys_addr_t addr,
 
 }
 
-static void axidma_write(void *opaque, target_phys_addr_t addr,
+static void axidma_write(void *opaque, hwaddr addr,
                          uint64_t value, unsigned size)
 {
     struct XilinxAXIDMA *d = opaque;
diff --git a/hw/xilinx_axienet.c b/hw/xilinx_axienet.c
index eec155d44..baae02bd6 100644
--- a/hw/xilinx_axienet.c
+++ b/hw/xilinx_axienet.c
@@ -412,7 +412,7 @@ static void enet_update_irq(struct XilinxAXIEnet *s)
     qemu_set_irq(s->irq, !!s->regs[R_IP]);
 }
 
-static uint64_t enet_read(void *opaque, target_phys_addr_t addr, unsigned size)
+static uint64_t enet_read(void *opaque, hwaddr addr, unsigned size)
 {
     struct XilinxAXIEnet *s = opaque;
     uint32_t r = 0;
@@ -503,7 +503,7 @@ static uint64_t enet_read(void *opaque, target_phys_addr_t addr, unsigned size)
     return r;
 }
 
-static void enet_write(void *opaque, target_phys_addr_t addr,
+static void enet_write(void *opaque, hwaddr addr,
                        uint64_t value, unsigned size)
 {
     struct XilinxAXIEnet *s = opaque;
diff --git a/hw/xilinx_ethlite.c b/hw/xilinx_ethlite.c
index 56ca620dd..13bd45613 100644
--- a/hw/xilinx_ethlite.c
+++ b/hw/xilinx_ethlite.c
@@ -72,7 +72,7 @@ static inline void eth_pulse_irq(struct xlx_ethlite *s)
 }
 
 static uint64_t
-eth_read(void *opaque, target_phys_addr_t addr, unsigned int size)
+eth_read(void *opaque, hwaddr addr, unsigned int size)
 {
     struct xlx_ethlite *s = opaque;
     uint32_t r = 0;
@@ -100,7 +100,7 @@ eth_read(void *opaque, target_phys_addr_t addr, unsigned int size)
 }
 
 static void
-eth_write(void *opaque, target_phys_addr_t addr,
+eth_write(void *opaque, hwaddr addr,
           uint64_t val64, unsigned int size)
 {
     struct xlx_ethlite *s = opaque;
diff --git a/hw/xilinx_intc.c b/hw/xilinx_intc.c
index 386fd3074..776507980 100644
--- a/hw/xilinx_intc.c
+++ b/hw/xilinx_intc.c
@@ -74,7 +74,7 @@ static void update_irq(struct xlx_pic *p)
 }
 
 static uint64_t
-pic_read(void *opaque, target_phys_addr_t addr, unsigned int size)
+pic_read(void *opaque, hwaddr addr, unsigned int size)
 {
     struct xlx_pic *p = opaque;
     uint32_t r = 0;
@@ -93,7 +93,7 @@ pic_read(void *opaque, target_phys_addr_t addr, unsigned int size)
 }
 
 static void
-pic_write(void *opaque, target_phys_addr_t addr,
+pic_write(void *opaque, hwaddr addr,
           uint64_t val64, unsigned int size)
 {
     struct xlx_pic *p = opaque;
diff --git a/hw/xilinx_spi.c b/hw/xilinx_spi.c
new file mode 100644
index 000000000..039027442
--- /dev/null
+++ b/hw/xilinx_spi.c
@@ -0,0 +1,385 @@
+/*
+ * QEMU model of the Xilinx SPI Controller
+ *
+ * Copyright (C) 2010 Edgar E. Iglesias.
+ * Copyright (C) 2012 Peter A. G. Crosthwaite <peter.crosthwaite@petalogix.com>
+ * Copyright (C) 2012 PetaLogix
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "sysbus.h"
+#include "sysemu.h"
+#include "qemu-log.h"
+#include "fifo.h"
+
+#include "ssi.h"
+
+#ifdef XILINX_SPI_ERR_DEBUG
+#define DB_PRINT(...) do { \
+    fprintf(stderr,  ": %s: ", __func__); \
+    fprintf(stderr, ## __VA_ARGS__); \
+    } while (0);
+#else
+    #define DB_PRINT(...)
+#endif
+
+#define R_DGIER     (0x1c / 4)
+#define R_DGIER_IE  (1 << 31)
+
+#define R_IPISR     (0x20 / 4)
+#define IRQ_DRR_NOT_EMPTY    (1 << (31 - 23))
+#define IRQ_DRR_OVERRUN      (1 << (31 - 26))
+#define IRQ_DRR_FULL         (1 << (31 - 27))
+#define IRQ_TX_FF_HALF_EMPTY (1 << 6)
+#define IRQ_DTR_UNDERRUN     (1 << 3)
+#define IRQ_DTR_EMPTY        (1 << (31 - 29))
+
+#define R_IPIER     (0x28 / 4)
+#define R_SRR       (0x40 / 4)
+#define R_SPICR     (0x60 / 4)
+#define R_SPICR_TXFF_RST     (1 << 5)
+#define R_SPICR_RXFF_RST     (1 << 6)
+#define R_SPICR_MTI          (1 << 8)
+
+#define R_SPISR     (0x64 / 4)
+#define SR_TX_FULL    (1 << 3)
+#define SR_TX_EMPTY   (1 << 2)
+#define SR_RX_FULL    (1 << 1)
+#define SR_RX_EMPTY   (1 << 0)
+
+#define R_SPIDTR    (0x68 / 4)
+#define R_SPIDRR    (0x6C / 4)
+#define R_SPISSR    (0x70 / 4)
+#define R_TX_FF_OCY (0x74 / 4)
+#define R_RX_FF_OCY (0x78 / 4)
+#define R_MAX       (0x7C / 4)
+
+#define FIFO_CAPACITY 256
+
+typedef struct XilinxSPI {
+    SysBusDevice busdev;
+    MemoryRegion mmio;
+
+    qemu_irq irq;
+    int irqline;
+
+    uint8_t num_cs;
+    qemu_irq *cs_lines;
+
+    SSIBus *spi;
+
+    Fifo8 rx_fifo;
+    Fifo8 tx_fifo;
+
+    uint32_t regs[R_MAX];
+} XilinxSPI;
+
+static void txfifo_reset(XilinxSPI *s)
+{
+    fifo8_reset(&s->tx_fifo);
+
+    s->regs[R_SPISR] &= ~SR_TX_FULL;
+    s->regs[R_SPISR] |= SR_TX_EMPTY;
+}
+
+static void rxfifo_reset(XilinxSPI *s)
+{
+    fifo8_reset(&s->rx_fifo);
+
+    s->regs[R_SPISR] |= SR_RX_EMPTY;
+    s->regs[R_SPISR] &= ~SR_RX_FULL;
+}
+
+static void xlx_spi_update_cs(XilinxSPI *s)
+{
+   int i;
+
+    for (i = 0; i < s->num_cs; ++i) {
+        qemu_set_irq(s->cs_lines[i], !(~s->regs[R_SPISSR] & 1 << i));
+    }
+}
+
+static void xlx_spi_update_irq(XilinxSPI *s)
+{
+    uint32_t pending;
+
+    s->regs[R_IPISR] |=
+            (!fifo8_is_empty(&s->rx_fifo) ? IRQ_DRR_NOT_EMPTY : 0) |
+            (fifo8_is_full(&s->rx_fifo) ? IRQ_DRR_FULL : 0);
+
+    pending = s->regs[R_IPISR] & s->regs[R_IPIER];
+
+    pending = pending && (s->regs[R_DGIER] & R_DGIER_IE);
+    pending = !!pending;
+
+    /* This call lies right in the data paths so don't call the
+       irq chain unless things really changed.  */
+    if (pending != s->irqline) {
+        s->irqline = pending;
+        DB_PRINT("irq_change of state %d ISR:%x IER:%X\n",
+                    pending, s->regs[R_IPISR], s->regs[R_IPIER]);
+        qemu_set_irq(s->irq, pending);
+    }
+
+}
+
+static void xlx_spi_do_reset(XilinxSPI *s)
+{
+    memset(s->regs, 0, sizeof s->regs);
+
+    rxfifo_reset(s);
+    txfifo_reset(s);
+
+    s->regs[R_SPISSR] = ~0;
+    xlx_spi_update_irq(s);
+    xlx_spi_update_cs(s);
+}
+
+static void xlx_spi_reset(DeviceState *d)
+{
+    xlx_spi_do_reset(DO_UPCAST(XilinxSPI, busdev.qdev, d));
+}
+
+static inline int spi_master_enabled(XilinxSPI *s)
+{
+    return !(s->regs[R_SPICR] & R_SPICR_MTI);
+}
+
+static void spi_flush_txfifo(XilinxSPI *s)
+{
+    uint32_t tx;
+    uint32_t rx;
+
+    while (!fifo8_is_empty(&s->tx_fifo)) {
+        tx = (uint32_t)fifo8_pop(&s->tx_fifo);
+        DB_PRINT("data tx:%x\n", tx);
+        rx = ssi_transfer(s->spi, tx);
+        DB_PRINT("data rx:%x\n", rx);
+        if (fifo8_is_full(&s->rx_fifo)) {
+            s->regs[R_IPISR] |= IRQ_DRR_OVERRUN;
+        } else {
+            fifo8_push(&s->rx_fifo, (uint8_t)rx);
+            if (fifo8_is_full(&s->rx_fifo)) {
+                s->regs[R_SPISR] |= SR_RX_FULL;
+                s->regs[R_IPISR] |= IRQ_DRR_FULL;
+            }
+        }
+
+        s->regs[R_SPISR] &= ~SR_RX_EMPTY;
+        s->regs[R_SPISR] &= ~SR_TX_FULL;
+        s->regs[R_SPISR] |= SR_TX_EMPTY;
+
+        s->regs[R_IPISR] |= IRQ_DTR_EMPTY;
+        s->regs[R_IPISR] |= IRQ_DRR_NOT_EMPTY;
+    }
+
+}
+
+static uint64_t
+spi_read(void *opaque, hwaddr addr, unsigned int size)
+{
+    XilinxSPI *s = opaque;
+    uint32_t r = 0;
+
+    addr >>= 2;
+    switch (addr) {
+    case R_SPIDRR:
+        if (fifo8_is_empty(&s->rx_fifo)) {
+            DB_PRINT("Read from empty FIFO!\n");
+            return 0xdeadbeef;
+        }
+
+        s->regs[R_SPISR] &= ~SR_RX_FULL;
+        r = fifo8_pop(&s->rx_fifo);
+        if (fifo8_is_empty(&s->rx_fifo)) {
+            s->regs[R_SPISR] |= SR_RX_EMPTY;
+        }
+        break;
+
+    case R_SPISR:
+        r = s->regs[addr];
+        break;
+
+    default:
+        if (addr < ARRAY_SIZE(s->regs)) {
+            r = s->regs[addr];
+        }
+        break;
+
+    }
+    DB_PRINT("addr=" TARGET_FMT_plx " = %x\n", addr * 4, r);
+    xlx_spi_update_irq(s);
+    return r;
+}
+
+static void
+spi_write(void *opaque, hwaddr addr,
+            uint64_t val64, unsigned int size)
+{
+    XilinxSPI *s = opaque;
+    uint32_t value = val64;
+
+    DB_PRINT("addr=" TARGET_FMT_plx " = %x\n", addr, value);
+    addr >>= 2;
+    switch (addr) {
+    case R_SRR:
+        if (value != 0xa) {
+            DB_PRINT("Invalid write to SRR %x\n", value);
+        } else {
+            xlx_spi_do_reset(s);
+        }
+        break;
+
+    case R_SPIDTR:
+        s->regs[R_SPISR] &= ~SR_TX_EMPTY;
+        fifo8_push(&s->tx_fifo, (uint8_t)value);
+        if (fifo8_is_full(&s->tx_fifo)) {
+            s->regs[R_SPISR] |= SR_TX_FULL;
+        }
+        if (!spi_master_enabled(s)) {
+            goto done;
+        } else {
+            DB_PRINT("DTR and master enabled\n");
+        }
+        spi_flush_txfifo(s);
+        break;
+
+    case R_SPISR:
+        DB_PRINT("Invalid write to SPISR %x\n", value);
+        break;
+
+    case R_IPISR:
+        /* Toggle the bits.  */
+        s->regs[addr] ^= value;
+        break;
+
+    /* Slave Select Register.  */
+    case R_SPISSR:
+        s->regs[addr] = value;
+        xlx_spi_update_cs(s);
+        break;
+
+    case R_SPICR:
+        /* FIXME: reset irq and sr state to empty queues.  */
+        if (value & R_SPICR_RXFF_RST) {
+            rxfifo_reset(s);
+        }
+
+        if (value & R_SPICR_TXFF_RST) {
+            txfifo_reset(s);
+        }
+        value &= ~(R_SPICR_RXFF_RST | R_SPICR_TXFF_RST);
+        s->regs[addr] = value;
+
+        if (!(value & R_SPICR_MTI)) {
+            spi_flush_txfifo(s);
+        }
+        break;
+
+    default:
+        if (addr < ARRAY_SIZE(s->regs)) {
+            s->regs[addr] = value;
+        }
+        break;
+    }
+
+done:
+    xlx_spi_update_irq(s);
+}
+
+static const MemoryRegionOps spi_ops = {
+    .read = spi_read,
+    .write = spi_write,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+    .valid = {
+        .min_access_size = 4,
+        .max_access_size = 4
+    }
+};
+
+static int xilinx_spi_init(SysBusDevice *dev)
+{
+    int i;
+    XilinxSPI *s = FROM_SYSBUS(typeof(*s), dev);
+
+    DB_PRINT("\n");
+
+    s->spi = ssi_create_bus(&dev->qdev, "spi");
+
+    sysbus_init_irq(dev, &s->irq);
+    s->cs_lines = g_new(qemu_irq, s->num_cs);
+    ssi_auto_connect_slaves(DEVICE(s), s->cs_lines, s->spi);
+    for (i = 0; i < s->num_cs; ++i) {
+        sysbus_init_irq(dev, &s->cs_lines[i]);
+    }
+
+    memory_region_init_io(&s->mmio, &spi_ops, s, "xilinx-spi", R_MAX * 4);
+    sysbus_init_mmio(dev, &s->mmio);
+
+    s->irqline = -1;
+
+    fifo8_create(&s->tx_fifo, FIFO_CAPACITY);
+    fifo8_create(&s->rx_fifo, FIFO_CAPACITY);
+
+    return 0;
+}
+
+static const VMStateDescription vmstate_xilinx_spi = {
+    .name = "xilinx_spi",
+    .version_id = 1,
+    .minimum_version_id = 1,
+    .minimum_version_id_old = 1,
+    .fields = (VMStateField[]) {
+        VMSTATE_FIFO8(tx_fifo, XilinxSPI),
+        VMSTATE_FIFO8(rx_fifo, XilinxSPI),
+        VMSTATE_UINT32_ARRAY(regs, XilinxSPI, R_MAX),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static Property xilinx_spi_properties[] = {
+    DEFINE_PROP_UINT8("num-ss-bits", XilinxSPI, num_cs, 1),
+    DEFINE_PROP_END_OF_LIST(),
+};
+
+static void xilinx_spi_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    SysBusDeviceClass *k = SYS_BUS_DEVICE_CLASS(klass);
+
+    k->init = xilinx_spi_init;
+    dc->reset = xlx_spi_reset;
+    dc->props = xilinx_spi_properties;
+    dc->vmsd = &vmstate_xilinx_spi;
+}
+
+static TypeInfo xilinx_spi_info = {
+    .name           = "xlnx.xps-spi",
+    .parent         = TYPE_SYS_BUS_DEVICE,
+    .instance_size  = sizeof(XilinxSPI),
+    .class_init     = xilinx_spi_class_init,
+};
+
+static void xilinx_spi_register_types(void)
+{
+    type_register_static(&xilinx_spi_info);
+}
+
+type_init(xilinx_spi_register_types)
diff --git a/hw/xilinx_spips.c b/hw/xilinx_spips.c
new file mode 100644
index 000000000..ee7656d7d
--- /dev/null
+++ b/hw/xilinx_spips.c
@@ -0,0 +1,575 @@
+/*
+ * QEMU model of the Xilinx Zynq SPI controller
+ *
+ * Copyright (c) 2012 Peter A. G. Crosthwaite
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include "sysbus.h"
+#include "sysemu.h"
+#include "ptimer.h"
+#include "qemu-log.h"
+#include "fifo.h"
+#include "ssi.h"
+#include "bitops.h"
+
+#ifdef XILINX_SPIPS_ERR_DEBUG
+#define DB_PRINT(...) do { \
+    fprintf(stderr,  ": %s: ", __func__); \
+    fprintf(stderr, ## __VA_ARGS__); \
+    } while (0);
+#else
+    #define DB_PRINT(...)
+#endif
+
+/* config register */
+#define R_CONFIG            (0x00 / 4)
+#define IFMODE              (1 << 31)
+#define ENDIAN              (1 << 26)
+#define MODEFAIL_GEN_EN     (1 << 17)
+#define MAN_START_COM       (1 << 16)
+#define MAN_START_EN        (1 << 15)
+#define MANUAL_CS           (1 << 14)
+#define CS                  (0xF << 10)
+#define CS_SHIFT            (10)
+#define PERI_SEL            (1 << 9)
+#define REF_CLK             (1 << 8)
+#define FIFO_WIDTH          (3 << 6)
+#define BAUD_RATE_DIV       (7 << 3)
+#define CLK_PH              (1 << 2)
+#define CLK_POL             (1 << 1)
+#define MODE_SEL            (1 << 0)
+
+/* interrupt mechanism */
+#define R_INTR_STATUS       (0x04 / 4)
+#define R_INTR_EN           (0x08 / 4)
+#define R_INTR_DIS          (0x0C / 4)
+#define R_INTR_MASK         (0x10 / 4)
+#define IXR_TX_FIFO_UNDERFLOW   (1 << 6)
+#define IXR_RX_FIFO_FULL        (1 << 5)
+#define IXR_RX_FIFO_NOT_EMPTY   (1 << 4)
+#define IXR_TX_FIFO_FULL        (1 << 3)
+#define IXR_TX_FIFO_NOT_FULL    (1 << 2)
+#define IXR_TX_FIFO_MODE_FAIL   (1 << 1)
+#define IXR_RX_FIFO_OVERFLOW    (1 << 0)
+#define IXR_ALL                 ((IXR_TX_FIFO_UNDERFLOW<<1)-1)
+
+#define R_EN                (0x14 / 4)
+#define R_DELAY             (0x18 / 4)
+#define R_TX_DATA           (0x1C / 4)
+#define R_RX_DATA           (0x20 / 4)
+#define R_SLAVE_IDLE_COUNT  (0x24 / 4)
+#define R_TX_THRES          (0x28 / 4)
+#define R_RX_THRES          (0x2C / 4)
+#define R_TXD1              (0x80 / 4)
+#define R_TXD2              (0x84 / 4)
+#define R_TXD3              (0x88 / 4)
+
+#define R_LQSPI_CFG         (0xa0 / 4)
+#define R_LQSPI_CFG_RESET       0x03A002EB
+#define LQSPI_CFG_LQ_MODE       (1 << 31)
+#define LQSPI_CFG_TWO_MEM       (1 << 30)
+#define LQSPI_CFG_SEP_BUS       (1 << 30)
+#define LQSPI_CFG_U_PAGE        (1 << 28)
+#define LQSPI_CFG_MODE_EN       (1 << 25)
+#define LQSPI_CFG_MODE_WIDTH    8
+#define LQSPI_CFG_MODE_SHIFT    16
+#define LQSPI_CFG_DUMMY_WIDTH   3
+#define LQSPI_CFG_DUMMY_SHIFT   8
+#define LQSPI_CFG_INST_CODE     0xFF
+
+#define R_LQSPI_STS         (0xA4 / 4)
+#define LQSPI_STS_WR_RECVD      (1 << 1)
+
+#define R_MOD_ID            (0xFC / 4)
+
+#define R_MAX (R_MOD_ID+1)
+
+/* size of TXRX FIFOs */
+#define RXFF_A          32
+#define TXFF_A          32
+
+/* 16MB per linear region */
+#define LQSPI_ADDRESS_BITS 24
+/* Bite off 4k chunks at a time */
+#define LQSPI_CACHE_SIZE 1024
+
+#define SNOOP_CHECKING 0xFF
+#define SNOOP_NONE 0xFE
+#define SNOOP_STRIPING 0
+
+typedef struct {
+    SysBusDevice busdev;
+    MemoryRegion iomem;
+    MemoryRegion mmlqspi;
+
+    qemu_irq irq;
+    int irqline;
+
+    uint8_t num_cs;
+    uint8_t num_busses;
+
+    uint8_t snoop_state;
+    qemu_irq *cs_lines;
+    SSIBus **spi;
+
+    Fifo8 rx_fifo;
+    Fifo8 tx_fifo;
+
+    uint8_t num_txrx_bytes;
+
+    uint32_t regs[R_MAX];
+
+    uint32_t lqspi_buf[LQSPI_CACHE_SIZE];
+    hwaddr lqspi_cached_addr;
+} XilinxSPIPS;
+
+static inline int num_effective_busses(XilinxSPIPS *s)
+{
+    return (s->regs[R_LQSPI_STS] & LQSPI_CFG_SEP_BUS &&
+            s->regs[R_LQSPI_STS] & LQSPI_CFG_TWO_MEM) ? s->num_busses : 1;
+}
+
+static void xilinx_spips_update_cs_lines(XilinxSPIPS *s)
+{
+    int i, j;
+    bool found = false;
+    int field = s->regs[R_CONFIG] >> CS_SHIFT;
+
+    for (i = 0; i < s->num_cs; i++) {
+        for (j = 0; j < num_effective_busses(s); j++) {
+            int upage = !!(s->regs[R_LQSPI_STS] & LQSPI_CFG_U_PAGE);
+            int cs_to_set = (j * s->num_cs + i + upage) %
+                                (s->num_cs * s->num_busses);
+
+            if (~field & (1 << i) && !found) {
+                DB_PRINT("selecting slave %d\n", i);
+                qemu_set_irq(s->cs_lines[cs_to_set], 0);
+            } else {
+                qemu_set_irq(s->cs_lines[cs_to_set], 1);
+            }
+        }
+        if (~field & (1 << i)) {
+            found = true;
+        }
+    }
+    if (!found) {
+        s->snoop_state = SNOOP_CHECKING;
+    }
+}
+
+static void xilinx_spips_update_ixr(XilinxSPIPS *s)
+{
+    /* These are set/cleared as they occur */
+    s->regs[R_INTR_STATUS] &= (IXR_TX_FIFO_UNDERFLOW | IXR_RX_FIFO_OVERFLOW |
+                                IXR_TX_FIFO_MODE_FAIL);
+    /* these are pure functions of fifo state, set them here */
+    s->regs[R_INTR_STATUS] |=
+        (fifo8_is_full(&s->rx_fifo) ? IXR_RX_FIFO_FULL : 0) |
+        (s->rx_fifo.num >= s->regs[R_RX_THRES] ? IXR_RX_FIFO_NOT_EMPTY : 0) |
+        (fifo8_is_full(&s->tx_fifo) ? IXR_TX_FIFO_FULL : 0) |
+        (s->tx_fifo.num < s->regs[R_TX_THRES] ? IXR_TX_FIFO_NOT_FULL : 0);
+    /* drive external interrupt pin */
+    int new_irqline = !!(s->regs[R_INTR_MASK] & s->regs[R_INTR_STATUS] &
+                                                                IXR_ALL);
+    if (new_irqline != s->irqline) {
+        s->irqline = new_irqline;
+        qemu_set_irq(s->irq, s->irqline);
+    }
+}
+
+static void xilinx_spips_reset(DeviceState *d)
+{
+    XilinxSPIPS *s = DO_UPCAST(XilinxSPIPS, busdev.qdev, d);
+
+    int i;
+    for (i = 0; i < R_MAX; i++) {
+        s->regs[i] = 0;
+    }
+
+    fifo8_reset(&s->rx_fifo);
+    fifo8_reset(&s->rx_fifo);
+    /* non zero resets */
+    s->regs[R_CONFIG] |= MODEFAIL_GEN_EN;
+    s->regs[R_SLAVE_IDLE_COUNT] = 0xFF;
+    s->regs[R_TX_THRES] = 1;
+    s->regs[R_RX_THRES] = 1;
+    /* FIXME: move magic number definition somewhere sensible */
+    s->regs[R_MOD_ID] = 0x01090106;
+    s->regs[R_LQSPI_CFG] = R_LQSPI_CFG_RESET;
+    s->snoop_state = SNOOP_CHECKING;
+    xilinx_spips_update_ixr(s);
+    xilinx_spips_update_cs_lines(s);
+}
+
+static void xilinx_spips_flush_txfifo(XilinxSPIPS *s)
+{
+    for (;;) {
+        int i;
+        uint8_t rx;
+        uint8_t tx = 0;
+
+        for (i = 0; i < num_effective_busses(s); ++i) {
+            if (!i || s->snoop_state == SNOOP_STRIPING) {
+                if (fifo8_is_empty(&s->tx_fifo)) {
+                    s->regs[R_INTR_STATUS] |= IXR_TX_FIFO_UNDERFLOW;
+                    xilinx_spips_update_ixr(s);
+                    return;
+                } else {
+                    tx = fifo8_pop(&s->tx_fifo);
+                }
+            }
+            rx = ssi_transfer(s->spi[i], (uint32_t)tx);
+            DB_PRINT("tx = %02x rx = %02x\n", tx, rx);
+            if (!i || s->snoop_state == SNOOP_STRIPING) {
+                if (fifo8_is_full(&s->rx_fifo)) {
+                    s->regs[R_INTR_STATUS] |= IXR_RX_FIFO_OVERFLOW;
+                    DB_PRINT("rx FIFO overflow");
+                } else {
+                    fifo8_push(&s->rx_fifo, (uint8_t)rx);
+                }
+            }
+        }
+
+        switch (s->snoop_state) {
+        case (SNOOP_CHECKING):
+            switch (tx) { /* new instruction code */
+            case 0x0b: /* dual/quad output read DOR/QOR */
+            case 0x6b:
+                s->snoop_state = 4;
+                break;
+            /* FIXME: these vary between vendor - set to spansion */
+            case 0xbb: /* high performance dual read DIOR */
+                s->snoop_state = 4;
+                break;
+            case 0xeb: /* high performance quad read QIOR */
+                s->snoop_state = 6;
+                break;
+            default:
+                s->snoop_state = SNOOP_NONE;
+            }
+            break;
+        case (SNOOP_STRIPING):
+        case (SNOOP_NONE):
+            break;
+        default:
+            s->snoop_state--;
+        }
+    }
+}
+
+static inline void rx_data_bytes(XilinxSPIPS *s, uint32_t *value, int max)
+{
+    int i;
+
+    *value = 0;
+    for (i = 0; i < max && !fifo8_is_empty(&s->rx_fifo); ++i) {
+        uint32_t next = fifo8_pop(&s->rx_fifo) & 0xFF;
+        *value |= next << 8 * (s->regs[R_CONFIG] & ENDIAN ? 3-i : i);
+    }
+}
+
+static uint64_t xilinx_spips_read(void *opaque, hwaddr addr,
+                                                        unsigned size)
+{
+    XilinxSPIPS *s = opaque;
+    uint32_t mask = ~0;
+    uint32_t ret;
+
+    addr >>= 2;
+    switch (addr) {
+    case R_CONFIG:
+        mask = 0x0002FFFF;
+        break;
+    case R_INTR_STATUS:
+    case R_INTR_MASK:
+        mask = IXR_ALL;
+        break;
+    case  R_EN:
+        mask = 0x1;
+        break;
+    case R_SLAVE_IDLE_COUNT:
+        mask = 0xFF;
+        break;
+    case R_MOD_ID:
+        mask = 0x01FFFFFF;
+        break;
+    case R_INTR_EN:
+    case R_INTR_DIS:
+    case R_TX_DATA:
+        mask = 0;
+        break;
+    case R_RX_DATA:
+        rx_data_bytes(s, &ret, s->num_txrx_bytes);
+        DB_PRINT("addr=" TARGET_FMT_plx " = %x\n", addr * 4, ret);
+        xilinx_spips_update_ixr(s);
+        return ret;
+    }
+    DB_PRINT("addr=" TARGET_FMT_plx " = %x\n", addr * 4, s->regs[addr] & mask);
+    return s->regs[addr] & mask;
+
+}
+
+static inline void tx_data_bytes(XilinxSPIPS *s, uint32_t value, int num)
+{
+    int i;
+    for (i = 0; i < num && !fifo8_is_full(&s->tx_fifo); ++i) {
+        if (s->regs[R_CONFIG] & ENDIAN) {
+            fifo8_push(&s->tx_fifo, (uint8_t)(value >> 24));
+            value <<= 8;
+        } else {
+            fifo8_push(&s->tx_fifo, (uint8_t)value);
+            value >>= 8;
+        }
+    }
+}
+
+static void xilinx_spips_write(void *opaque, hwaddr addr,
+                                        uint64_t value, unsigned size)
+{
+    int mask = ~0;
+    int man_start_com = 0;
+    XilinxSPIPS *s = opaque;
+
+    DB_PRINT("addr=" TARGET_FMT_plx " = %x\n", addr, (unsigned)value);
+    addr >>= 2;
+    switch (addr) {
+    case R_CONFIG:
+        mask = 0x0002FFFF;
+        if (value & MAN_START_COM) {
+            man_start_com = 1;
+        }
+        break;
+    case R_INTR_STATUS:
+        mask = IXR_ALL;
+        s->regs[R_INTR_STATUS] &= ~(mask & value);
+        goto no_reg_update;
+    case R_INTR_DIS:
+        mask = IXR_ALL;
+        s->regs[R_INTR_MASK] &= ~(mask & value);
+        goto no_reg_update;
+    case R_INTR_EN:
+        mask = IXR_ALL;
+        s->regs[R_INTR_MASK] |= mask & value;
+        goto no_reg_update;
+    case R_EN:
+        mask = 0x1;
+        break;
+    case R_SLAVE_IDLE_COUNT:
+        mask = 0xFF;
+        break;
+    case R_RX_DATA:
+    case R_INTR_MASK:
+    case R_MOD_ID:
+        mask = 0;
+        break;
+    case R_TX_DATA:
+        tx_data_bytes(s, (uint32_t)value, s->num_txrx_bytes);
+        goto no_reg_update;
+    case R_TXD1:
+        tx_data_bytes(s, (uint32_t)value, 1);
+        goto no_reg_update;
+    case R_TXD2:
+        tx_data_bytes(s, (uint32_t)value, 2);
+        goto no_reg_update;
+    case R_TXD3:
+        tx_data_bytes(s, (uint32_t)value, 3);
+        goto no_reg_update;
+    }
+    s->regs[addr] = (s->regs[addr] & ~mask) | (value & mask);
+no_reg_update:
+    if (man_start_com) {
+        xilinx_spips_flush_txfifo(s);
+    }
+    xilinx_spips_update_ixr(s);
+    xilinx_spips_update_cs_lines(s);
+}
+
+static const MemoryRegionOps spips_ops = {
+    .read = xilinx_spips_read,
+    .write = xilinx_spips_write,
+    .endianness = DEVICE_LITTLE_ENDIAN,
+};
+
+#define LQSPI_CACHE_SIZE 1024
+
+static uint64_t
+lqspi_read(void *opaque, hwaddr addr, unsigned int size)
+{
+    int i;
+    XilinxSPIPS *s = opaque;
+
+    if (addr >= s->lqspi_cached_addr &&
+            addr <= s->lqspi_cached_addr + LQSPI_CACHE_SIZE - 4) {
+        return s->lqspi_buf[(addr - s->lqspi_cached_addr) >> 2];
+    } else {
+        int flash_addr = (addr / num_effective_busses(s));
+        int slave = flash_addr >> LQSPI_ADDRESS_BITS;
+        int cache_entry = 0;
+
+        DB_PRINT("config reg status: %08x\n", s->regs[R_LQSPI_CFG]);
+
+        fifo8_reset(&s->tx_fifo);
+        fifo8_reset(&s->rx_fifo);
+
+        s->regs[R_CONFIG] &= ~CS;
+        s->regs[R_CONFIG] |= (~(1 << slave) << CS_SHIFT) & CS;
+        xilinx_spips_update_cs_lines(s);
+
+        /* instruction */
+        DB_PRINT("pushing read instruction: %02x\n",
+                 (uint8_t)(s->regs[R_LQSPI_CFG] & LQSPI_CFG_INST_CODE));
+        fifo8_push(&s->tx_fifo, s->regs[R_LQSPI_CFG] & LQSPI_CFG_INST_CODE);
+        /* read address */
+        DB_PRINT("pushing read address %06x\n", flash_addr);
+        fifo8_push(&s->tx_fifo, (uint8_t)(flash_addr >> 16));
+        fifo8_push(&s->tx_fifo, (uint8_t)(flash_addr >> 8));
+        fifo8_push(&s->tx_fifo, (uint8_t)flash_addr);
+        /* mode bits */
+        if (s->regs[R_LQSPI_CFG] & LQSPI_CFG_MODE_EN) {
+            fifo8_push(&s->tx_fifo, extract32(s->regs[R_LQSPI_CFG],
+                                              LQSPI_CFG_MODE_SHIFT,
+                                              LQSPI_CFG_MODE_WIDTH));
+        }
+        /* dummy bytes */
+        for (i = 0; i < (extract32(s->regs[R_LQSPI_CFG], LQSPI_CFG_DUMMY_SHIFT,
+                                   LQSPI_CFG_DUMMY_WIDTH)); ++i) {
+            DB_PRINT("pushing dummy byte\n");
+            fifo8_push(&s->tx_fifo, 0);
+        }
+        xilinx_spips_flush_txfifo(s);
+        fifo8_reset(&s->rx_fifo);
+
+        DB_PRINT("starting QSPI data read\n");
+
+        for (i = 0; i < LQSPI_CACHE_SIZE / 4; ++i) {
+            tx_data_bytes(s, 0, 4);
+            xilinx_spips_flush_txfifo(s);
+            rx_data_bytes(s, &s->lqspi_buf[cache_entry], 4);
+            cache_entry++;
+        }
+
+        s->regs[R_CONFIG] |= CS;
+        xilinx_spips_update_cs_lines(s);
+
+        s->lqspi_cached_addr = addr;
+        return lqspi_read(opaque, addr, size);
+    }
+}
+
+static const MemoryRegionOps lqspi_ops = {
+    .read = lqspi_read,
+    .endianness = DEVICE_NATIVE_ENDIAN,
+    .valid = {
+        .min_access_size = 4,
+        .max_access_size = 4
+    }
+};
+
+static int xilinx_spips_init(SysBusDevice *dev)
+{
+    XilinxSPIPS *s = FROM_SYSBUS(typeof(*s), dev);
+    int i;
+
+    DB_PRINT("inited device model\n");
+
+    s->spi = g_new(SSIBus *, s->num_busses);
+    for (i = 0; i < s->num_busses; ++i) {
+        char bus_name[16];
+        snprintf(bus_name, 16, "spi%d", i);
+        s->spi[i] = ssi_create_bus(&dev->qdev, bus_name);
+    }
+
+    s->cs_lines = g_new(qemu_irq, s->num_cs * s->num_busses);
+    ssi_auto_connect_slaves(DEVICE(s), s->cs_lines, s->spi[0]);
+    ssi_auto_connect_slaves(DEVICE(s), s->cs_lines, s->spi[1]);
+    sysbus_init_irq(dev, &s->irq);
+    for (i = 0; i < s->num_cs * s->num_busses; ++i) {
+        sysbus_init_irq(dev, &s->cs_lines[i]);
+    }
+
+    memory_region_init_io(&s->iomem, &spips_ops, s, "spi", R_MAX*4);
+    sysbus_init_mmio(dev, &s->iomem);
+
+    memory_region_init_io(&s->mmlqspi, &lqspi_ops, s, "lqspi",
+                          (1 << LQSPI_ADDRESS_BITS) * 2);
+    sysbus_init_mmio(dev, &s->mmlqspi);
+
+    s->irqline = -1;
+    s->lqspi_cached_addr = ~0ULL;
+
+    fifo8_create(&s->rx_fifo, RXFF_A);
+    fifo8_create(&s->tx_fifo, TXFF_A);
+
+    return 0;
+}
+
+static int xilinx_spips_post_load(void *opaque, int version_id)
+{
+    xilinx_spips_update_ixr((XilinxSPIPS *)opaque);
+    xilinx_spips_update_cs_lines((XilinxSPIPS *)opaque);
+    return 0;
+}
+
+static const VMStateDescription vmstate_xilinx_spips = {
+    .name = "xilinx_spips",
+    .version_id = 2,
+    .minimum_version_id = 2,
+    .minimum_version_id_old = 2,
+    .post_load = xilinx_spips_post_load,
+    .fields = (VMStateField[]) {
+        VMSTATE_FIFO8(tx_fifo, XilinxSPIPS),
+        VMSTATE_FIFO8(rx_fifo, XilinxSPIPS),
+        VMSTATE_UINT32_ARRAY(regs, XilinxSPIPS, R_MAX),
+        VMSTATE_UINT8(snoop_state, XilinxSPIPS),
+        VMSTATE_END_OF_LIST()
+    }
+};
+
+static Property xilinx_spips_properties[] = {
+    DEFINE_PROP_UINT8("num-busses", XilinxSPIPS, num_busses, 1),
+    DEFINE_PROP_UINT8("num-ss-bits", XilinxSPIPS, num_cs, 4),
+    DEFINE_PROP_UINT8("num-txrx-bytes", XilinxSPIPS, num_txrx_bytes, 1),
+    DEFINE_PROP_END_OF_LIST(),
+};
+static void xilinx_spips_class_init(ObjectClass *klass, void *data)
+{
+    DeviceClass *dc = DEVICE_CLASS(klass);
+    SysBusDeviceClass *sdc = SYS_BUS_DEVICE_CLASS(klass);
+
+    sdc->init = xilinx_spips_init;
+    dc->reset = xilinx_spips_reset;
+    dc->props = xilinx_spips_properties;
+    dc->vmsd = &vmstate_xilinx_spips;
+}
+
+static const TypeInfo xilinx_spips_info = {
+    .name  = "xilinx,spips",
+    .parent = TYPE_SYS_BUS_DEVICE,
+    .instance_size  = sizeof(XilinxSPIPS),
+    .class_init = xilinx_spips_class_init,
+};
+
+static void xilinx_spips_register_types(void)
+{
+    type_register_static(&xilinx_spips_info);
+}
+
+type_init(xilinx_spips_register_types)
diff --git a/hw/xilinx_timer.c b/hw/xilinx_timer.c
index b562bd065..2b01dc286 100644
--- a/hw/xilinx_timer.c
+++ b/hw/xilinx_timer.c
@@ -24,6 +24,7 @@
 
 #include "sysbus.h"
 #include "ptimer.h"
+#include "qemu-log.h"
 
 #define D(x)
 
@@ -71,7 +72,7 @@ static inline unsigned int num_timers(struct timerblock *t)
     return 2 - t->one_timer_only;
 }
 
-static inline unsigned int timer_from_addr(target_phys_addr_t addr)
+static inline unsigned int timer_from_addr(hwaddr addr)
 {
     /* Timers get a 4x32bit control reg area each.  */
     return addr >> 2;
@@ -92,7 +93,7 @@ static void timer_update_irq(struct timerblock *t)
 }
 
 static uint64_t
-timer_read(void *opaque, target_phys_addr_t addr, unsigned int size)
+timer_read(void *opaque, hwaddr addr, unsigned int size)
 {
     struct timerblock *t = opaque;
     struct xlx_timer *xt;
@@ -119,7 +120,7 @@ timer_read(void *opaque, target_phys_addr_t addr, unsigned int size)
             break;
 
     }
-    D(printf("%s timer=%d %x=%x\n", __func__, timer, addr * 4, r));
+    D(fprintf(stderr, "%s timer=%d %x=%x\n", __func__, timer, addr * 4, r));
     return r;
 }
 
@@ -127,7 +128,7 @@ static void timer_enable(struct xlx_timer *xt)
 {
     uint64_t count;
 
-    D(printf("%s timer=%d down=%d\n", __func__,
+    D(fprintf(stderr, "%s timer=%d down=%d\n", __func__,
               xt->nr, xt->regs[R_TCSR] & TCSR_UDT));
 
     ptimer_stop(xt->ptimer);
@@ -141,7 +142,7 @@ static void timer_enable(struct xlx_timer *xt)
 }
 
 static void
-timer_write(void *opaque, target_phys_addr_t addr,
+timer_write(void *opaque, hwaddr addr,
             uint64_t val64, unsigned int size)
 {
     struct timerblock *t = opaque;
@@ -152,7 +153,7 @@ timer_write(void *opaque, target_phys_addr_t addr,
     addr >>= 2;
     timer = timer_from_addr(addr);
     xt = &t->timers[timer];
-    D(printf("%s addr=%x val=%x (timer=%d off=%d)\n",
+    D(fprintf(stderr, "%s addr=%x val=%x (timer=%d off=%d)\n",
              __func__, addr * 4, value, timer, addr & 3));
     /* Further decoding to address a specific timers reg.  */
     addr &= 3;
@@ -189,7 +190,7 @@ static void timer_hit(void *opaque)
 {
     struct xlx_timer *xt = opaque;
     struct timerblock *t = xt->parent;
-    D(printf("%s %d\n", __func__, timer));
+    D(fprintf(stderr, "%s %d\n", __func__, xt->nr));
     xt->regs[R_TCSR] |= TCSR_TINT;
 
     if (xt->regs[R_TCSR] & TCSR_ARHT)
@@ -217,14 +218,15 @@ static int xilinx_timer_init(SysBusDevice *dev)
         ptimer_set_freq(xt->ptimer, t->freq_hz);
     }
 
-    memory_region_init_io(&t->mmio, &timer_ops, t, "xlnx,xps-timer",
+    memory_region_init_io(&t->mmio, &timer_ops, t, "xlnx.xps-timer",
                           R_MAX * 4 * num_timers(t));
     sysbus_init_mmio(dev, &t->mmio);
     return 0;
 }
 
 static Property xilinx_timer_properties[] = {
-    DEFINE_PROP_UINT32("frequency", struct timerblock, freq_hz,   62 * 1000000),
+    DEFINE_PROP_UINT32("clock-frequency", struct timerblock, freq_hz,
+                                                                62 * 1000000),
     DEFINE_PROP_UINT8("one-timer-only", struct timerblock, one_timer_only, 0),
     DEFINE_PROP_END_OF_LIST(),
 };
@@ -239,7 +241,7 @@ static void xilinx_timer_class_init(ObjectClass *klass, void *data)
 }
 
 static TypeInfo xilinx_timer_info = {
-    .name          = "xlnx,xps-timer",
+    .name          = "xlnx.xps-timer",
     .parent        = TYPE_SYS_BUS_DEVICE,
     .instance_size = sizeof(struct timerblock),
     .class_init    = xilinx_timer_class_init,
diff --git a/hw/xilinx_uartlite.c b/hw/xilinx_uartlite.c
index d0f32db2c..d20fc4124 100644
--- a/hw/xilinx_uartlite.c
+++ b/hw/xilinx_uartlite.c
@@ -84,7 +84,7 @@ static void uart_update_status(struct xlx_uartlite *s)
 }
 
 static uint64_t
-uart_read(void *opaque, target_phys_addr_t addr, unsigned int size)
+uart_read(void *opaque, hwaddr addr, unsigned int size)
 {
     struct xlx_uartlite *s = opaque;
     uint32_t r = 0;
@@ -109,7 +109,7 @@ uart_read(void *opaque, target_phys_addr_t addr, unsigned int size)
 }
 
 static void
-uart_write(void *opaque, target_phys_addr_t addr,
+uart_write(void *opaque, hwaddr addr,
            uint64_t val64, unsigned int size)
 {
     struct xlx_uartlite *s = opaque;
diff --git a/hw/xilinx_zynq.c b/hw/xilinx_zynq.c
index 7e6c27359..1f12a3d1a 100644
--- a/hw/xilinx_zynq.c
+++ b/hw/xilinx_zynq.c
@@ -24,6 +24,11 @@
 #include "flash.h"
 #include "blockdev.h"
 #include "loader.h"
+#include "ssi.h"
+
+#define NUM_SPI_FLASHES 4
+#define NUM_QSPI_FLASHES 2
+#define NUM_QSPI_BUSSES 2
 
 #define FLASH_SIZE (64 * 1024 * 1024)
 #define FLASH_SECTOR_SIZE (128 * 1024)
@@ -46,10 +51,54 @@ static void gem_init(NICInfo *nd, uint32_t base, qemu_irq irq)
     sysbus_connect_irq(s, 0, irq);
 }
 
-static void zynq_init(ram_addr_t ram_size, const char *boot_device,
-                        const char *kernel_filename, const char *kernel_cmdline,
-                        const char *initrd_filename, const char *cpu_model)
+static inline void zynq_init_spi_flashes(uint32_t base_addr, qemu_irq irq,
+                                         bool is_qspi)
+{
+    DeviceState *dev;
+    SysBusDevice *busdev;
+    SSIBus *spi;
+    int i, j;
+    int num_busses =  is_qspi ? NUM_QSPI_BUSSES : 1;
+    int num_ss = is_qspi ? NUM_QSPI_FLASHES : NUM_SPI_FLASHES;
+
+    dev = qdev_create(NULL, "xilinx,spips");
+    qdev_prop_set_uint8(dev, "num-txrx-bytes", is_qspi ? 4 : 1);
+    qdev_prop_set_uint8(dev, "num-ss-bits", num_ss);
+    qdev_prop_set_uint8(dev, "num-busses", num_busses);
+    qdev_init_nofail(dev);
+    busdev = sysbus_from_qdev(dev);
+    sysbus_mmio_map(busdev, 0, base_addr);
+    if (is_qspi) {
+        sysbus_mmio_map(busdev, 1, 0xFC000000);
+    }
+    sysbus_connect_irq(busdev, 0, irq);
+
+    for (i = 0; i < num_busses; ++i) {
+        char bus_name[16];
+        qemu_irq cs_line;
+
+        snprintf(bus_name, 16, "spi%d", i);
+        spi = (SSIBus *)qdev_get_child_bus(dev, bus_name);
+
+        for (j = 0; j < num_ss; ++j) {
+            dev = ssi_create_slave_no_init(spi, "m25p80");
+            qdev_prop_set_string(dev, "partname", "n25q128");
+            qdev_init_nofail(dev);
+
+            cs_line = qdev_get_gpio_in(dev, 0);
+            sysbus_connect_irq(busdev, i * num_ss + j + 1, cs_line);
+        }
+    }
+
+}
+
+static void zynq_init(QEMUMachineInitArgs *args)
 {
+    ram_addr_t ram_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
     ARMCPU *cpu;
     MemoryRegion *address_space_mem = get_system_memory();
     MemoryRegion *ext_ram = g_new(MemoryRegion, 1);
@@ -113,6 +162,13 @@ static void zynq_init(ram_addr_t ram_size, const char *boot_device,
         pic[n] = qdev_get_gpio_in(dev, n);
     }
 
+    zynq_init_spi_flashes(0xE0006000, pic[58-IRQ_OFFSET], false);
+    zynq_init_spi_flashes(0xE0007000, pic[81-IRQ_OFFSET], false);
+    zynq_init_spi_flashes(0xE000D000, pic[51-IRQ_OFFSET], true);
+
+    sysbus_create_simple("xlnx,ps7-usb", 0xE0002000, pic[53-IRQ_OFFSET]);
+    sysbus_create_simple("xlnx,ps7-usb", 0xE0003000, pic[75-IRQ_OFFSET]);
+
     sysbus_create_simple("cadence_uart", 0xE0000000, pic[59-IRQ_OFFSET]);
     sysbus_create_simple("cadence_uart", 0xE0001000, pic[82-IRQ_OFFSET]);
 
diff --git a/hw/xtensa_lx60.c b/hw/xtensa_lx60.c
index 3653f65b1..4c42edc4e 100644
--- a/hw/xtensa_lx60.c
+++ b/hw/xtensa_lx60.c
@@ -31,7 +31,8 @@
 #include "elf.h"
 #include "memory.h"
 #include "exec-memory.h"
-#include "pc.h"
+#include "serial.h"
+#include "net.h"
 #include "sysbus.h"
 #include "flash.h"
 #include "blockdev.h"
@@ -57,7 +58,7 @@ static void lx60_fpga_reset(void *opaque)
     s->switches = 0;
 }
 
-static uint64_t lx60_fpga_read(void *opaque, target_phys_addr_t addr,
+static uint64_t lx60_fpga_read(void *opaque, hwaddr addr,
         unsigned size)
 {
     Lx60FpgaState *s = opaque;
@@ -78,7 +79,7 @@ static uint64_t lx60_fpga_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void lx60_fpga_write(void *opaque, target_phys_addr_t addr,
+static void lx60_fpga_write(void *opaque, hwaddr addr,
         uint64_t val, unsigned size)
 {
     Lx60FpgaState *s = opaque;
@@ -103,7 +104,7 @@ static const MemoryRegionOps lx60_fpga_ops = {
 };
 
 static Lx60FpgaState *lx60_fpga_init(MemoryRegion *address_space,
-        target_phys_addr_t base)
+        hwaddr base)
 {
     Lx60FpgaState *s = g_malloc(sizeof(Lx60FpgaState));
 
@@ -116,9 +117,9 @@ static Lx60FpgaState *lx60_fpga_init(MemoryRegion *address_space,
 }
 
 static void lx60_net_init(MemoryRegion *address_space,
-        target_phys_addr_t base,
-        target_phys_addr_t descriptors,
-        target_phys_addr_t buffers,
+        hwaddr base,
+        hwaddr descriptors,
+        hwaddr buffers,
         qemu_irq irq, NICInfo *nd)
 {
     DeviceState *dev;
@@ -154,10 +155,7 @@ static void lx60_reset(void *opaque)
     cpu_reset(CPU(cpu));
 }
 
-static void lx_init(const LxBoardDesc *board,
-        ram_addr_t ram_size, const char *boot_device,
-        const char *kernel_filename, const char *kernel_cmdline,
-        const char *initrd_filename, const char *cpu_model)
+static void lx_init(const LxBoardDesc *board, QEMUMachineInitArgs *args)
 {
 #ifdef TARGET_WORDS_BIGENDIAN
     int be = 1;
@@ -170,6 +168,9 @@ static void lx_init(const LxBoardDesc *board,
     MemoryRegion *ram, *rom, *system_io;
     DriveInfo *dinfo;
     pflash_t *flash = NULL;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
     int n;
 
     if (!cpu_model) {
@@ -193,7 +194,7 @@ static void lx_init(const LxBoardDesc *board,
     }
 
     ram = g_malloc(sizeof(*ram));
-    memory_region_init_ram(ram, "lx60.dram", ram_size);
+    memory_region_init_ram(ram, "lx60.dram", args->ram_size);
     vmstate_register_ram_global(ram);
     memory_region_add_subregion(system_memory, 0, ram);
 
@@ -268,34 +269,24 @@ static void lx_init(const LxBoardDesc *board,
     }
 }
 
-static void xtensa_lx60_init(ram_addr_t ram_size,
-                     const char *boot_device,
-                     const char *kernel_filename, const char *kernel_cmdline,
-                     const char *initrd_filename, const char *cpu_model)
+static void xtensa_lx60_init(QEMUMachineInitArgs *args)
 {
     static const LxBoardDesc lx60_board = {
         .flash_size = 0x400000,
         .flash_sector_size = 0x10000,
         .sram_size = 0x20000,
     };
-    lx_init(&lx60_board, ram_size, boot_device,
-            kernel_filename, kernel_cmdline,
-            initrd_filename, cpu_model);
+    lx_init(&lx60_board, args);
 }
 
-static void xtensa_lx200_init(ram_addr_t ram_size,
-                     const char *boot_device,
-                     const char *kernel_filename, const char *kernel_cmdline,
-                     const char *initrd_filename, const char *cpu_model)
+static void xtensa_lx200_init(QEMUMachineInitArgs *args)
 {
     static const LxBoardDesc lx200_board = {
         .flash_size = 0x1000000,
         .flash_sector_size = 0x20000,
         .sram_size = 0x2000000,
     };
-    lx_init(&lx200_board, ram_size, boot_device,
-            kernel_filename, kernel_cmdline,
-            initrd_filename, cpu_model);
+    lx_init(&lx200_board, args);
 }
 
 static QEMUMachine xtensa_lx60_machine = {
diff --git a/hw/xtensa_pic.c b/hw/xtensa_pic.c
index 653ded682..1ec70cd96 100644
--- a/hw/xtensa_pic.c
+++ b/hw/xtensa_pic.c
@@ -125,12 +125,13 @@ void xtensa_rearm_ccompare_timer(CPUXtensaState *env)
 
 static void xtensa_ccompare_cb(void *opaque)
 {
-    CPUXtensaState *env = opaque;
+    XtensaCPU *cpu = opaque;
+    CPUXtensaState *env = &cpu->env;
 
     if (env->halted) {
         env->halt_clock = qemu_get_clock_ns(vm_clock);
         xtensa_advance_ccount(env, env->wake_ccount - env->sregs[CCOUNT]);
-        if (!cpu_has_work(env)) {
+        if (!cpu_has_work(CPU(cpu))) {
             env->sregs[CCOUNT] = env->wake_ccount + 1;
             xtensa_rearm_ccompare_timer(env);
         }
@@ -139,12 +140,14 @@ static void xtensa_ccompare_cb(void *opaque)
 
 void xtensa_irq_init(CPUXtensaState *env)
 {
+    XtensaCPU *cpu = xtensa_env_get_cpu(env);
+
     env->irq_inputs = (void **)qemu_allocate_irqs(
             xtensa_set_irq, env, env->config->ninterrupt);
     if (xtensa_option_enabled(env->config, XTENSA_OPTION_TIMER_INTERRUPT) &&
             env->config->nccompare > 0) {
         env->ccompare_timer =
-            qemu_new_timer_ns(vm_clock, &xtensa_ccompare_cb, env);
+            qemu_new_timer_ns(vm_clock, &xtensa_ccompare_cb, cpu);
     }
 }
 
diff --git a/hw/xtensa_sim.c b/hw/xtensa_sim.c
index 831460b7c..0d633e472 100644
--- a/hw/xtensa_sim.c
+++ b/hw/xtensa_sim.c
@@ -44,16 +44,20 @@ static void sim_reset(void *opaque)
     cpu_reset(CPU(cpu));
 }
 
-static void sim_init(ram_addr_t ram_size,
-        const char *boot_device,
-        const char *kernel_filename, const char *kernel_cmdline,
-        const char *initrd_filename, const char *cpu_model)
+static void xtensa_sim_init(QEMUMachineInitArgs *args)
 {
     XtensaCPU *cpu = NULL;
     CPUXtensaState *env = NULL;
     MemoryRegion *ram, *rom;
+    ram_addr_t ram_size = args->ram_size;
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
     int n;
 
+    if (!cpu_model) {
+        cpu_model = XTENSA_DEFAULT_CPU_MODEL;
+    }
+
     for (n = 0; n < smp_cpus; n++) {
         cpu = cpu_xtensa_init(cpu_model);
         if (cpu == NULL) {
@@ -96,18 +100,6 @@ static void sim_init(ram_addr_t ram_size,
     }
 }
 
-static void xtensa_sim_init(ram_addr_t ram_size,
-                     const char *boot_device,
-                     const char *kernel_filename, const char *kernel_cmdline,
-                     const char *initrd_filename, const char *cpu_model)
-{
-    if (!cpu_model) {
-        cpu_model = XTENSA_DEFAULT_CPU_MODEL;
-    }
-    sim_init(ram_size, boot_device, kernel_filename, kernel_cmdline,
-            initrd_filename, cpu_model);
-}
-
 static QEMUMachine xtensa_sim_machine = {
     .name = "sim",
     .desc = "sim machine (" XTENSA_DEFAULT_CPU_MODEL ")",
diff --git a/hw/z2.c b/hw/z2.c
index 289cee90c..f62b8067b 100644
--- a/hw/z2.c
+++ b/hw/z2.c
@@ -161,10 +161,11 @@ static int zipit_lcd_init(SSISlave *dev)
 
 static VMStateDescription vmstate_zipit_lcd_state = {
     .name = "zipit-lcd",
-    .version_id = 1,
-    .minimum_version_id = 1,
-    .minimum_version_id_old = 1,
+    .version_id = 2,
+    .minimum_version_id = 2,
+    .minimum_version_id_old = 2,
     .fields = (VMStateField[]) {
+        VMSTATE_SSI_SLAVE(ssidev, ZipitLCD),
         VMSTATE_INT32(selected, ZipitLCD),
         VMSTATE_INT32(enabled, ZipitLCD),
         VMSTATE_BUFFER(buf, ZipitLCD),
@@ -294,11 +295,12 @@ static TypeInfo aer915_info = {
     .class_init    = aer915_class_init,
 };
 
-static void z2_init(ram_addr_t ram_size,
-                const char *boot_device,
-                const char *kernel_filename, const char *kernel_cmdline,
-                const char *initrd_filename, const char *cpu_model)
+static void z2_init(QEMUMachineInitArgs *args)
 {
+    const char *cpu_model = args->cpu_model;
+    const char *kernel_filename = args->kernel_filename;
+    const char *kernel_cmdline = args->kernel_cmdline;
+    const char *initrd_filename = args->initrd_filename;
     MemoryRegion *address_space_mem = get_system_memory();
     uint32_t sector_len = 0x10000;
     PXA2xxState *mpu;
diff --git a/hw/zaurus.c b/hw/zaurus.c
index 72838ec44..d77b34ecc 100644
--- a/hw/zaurus.c
+++ b/hw/zaurus.c
@@ -68,7 +68,7 @@ static inline void scoop_gpio_handler_update(ScoopInfo *s) {
     s->prev_level = level;
 }
 
-static uint64_t scoop_read(void *opaque, target_phys_addr_t addr,
+static uint64_t scoop_read(void *opaque, hwaddr addr,
                            unsigned size)
 {
     ScoopInfo *s = (ScoopInfo *) opaque;
@@ -102,7 +102,7 @@ static uint64_t scoop_read(void *opaque, target_phys_addr_t addr,
     return 0;
 }
 
-static void scoop_write(void *opaque, target_phys_addr_t addr,
+static void scoop_write(void *opaque, hwaddr addr,
                         uint64_t value, unsigned size)
 {
     ScoopInfo *s = (ScoopInfo *) opaque;
@@ -285,7 +285,7 @@ static struct QEMU_PACKED sl_param_info {
     .phadadj		= 0x01,
 };
 
-void sl_bootparam_write(target_phys_addr_t ptr)
+void sl_bootparam_write(hwaddr ptr)
 {
     cpu_physical_memory_write(ptr, (void *)&zaurus_bootparam,
                               sizeof(struct sl_param_info));
diff --git a/hw/zynq_slcr.c b/hw/zynq_slcr.c
index 4f9757577..dde430619 100644
--- a/hw/zynq_slcr.c
+++ b/hw/zynq_slcr.c
@@ -91,7 +91,7 @@ typedef enum {
 typedef enum {
   PSS,
   DDDR,
-  DMAC,
+  DMAC = 3,
   USB,
   GEM,
   SDIO,
@@ -246,7 +246,7 @@ static void zynq_slcr_reset(DeviceState *d)
 }
 
 static inline uint32_t zynq_slcr_read_imp(void *opaque,
-    target_phys_addr_t offset)
+    hwaddr offset)
 {
     ZynqSLCRState *s = (ZynqSLCRState *)opaque;
 
@@ -329,7 +329,7 @@ static inline uint32_t zynq_slcr_read_imp(void *opaque,
     }
 }
 
-static uint64_t zynq_slcr_read(void *opaque, target_phys_addr_t offset,
+static uint64_t zynq_slcr_read(void *opaque, hwaddr offset,
     unsigned size)
 {
     uint32_t ret = zynq_slcr_read_imp(opaque, offset);
@@ -338,7 +338,7 @@ static uint64_t zynq_slcr_read(void *opaque, target_phys_addr_t offset,
     return ret;
 }
 
-static void zynq_slcr_write(void *opaque, target_phys_addr_t offset,
+static void zynq_slcr_write(void *opaque, hwaddr offset,
                           uint64_t val, unsigned size)
 {
     ZynqSLCRState *s = (ZynqSLCRState *)opaque;
diff --git a/hwaddr.h b/hwaddr.h
new file mode 100644
index 000000000..251cf9216
--- /dev/null
+++ b/hwaddr.h
@@ -0,0 +1,24 @@
+/* Define hwaddr if it exists.  */
+
+#ifndef HWADDR_H
+#define HWADDR_H
+
+#ifndef CONFIG_USER_ONLY
+
+#define HWADDR_BITS 64
+/* hwaddr is the type of a physical address (its size can
+   be different from 'target_ulong').  */
+
+typedef uint64_t hwaddr;
+#define HWADDR_MAX UINT64_MAX
+#define TARGET_FMT_plx "%016" PRIx64
+#define HWADDR_PRId PRId64
+#define HWADDR_PRIi PRIi64
+#define HWADDR_PRIo PRIo64
+#define HWADDR_PRIu PRIu64
+#define HWADDR_PRIx PRIx64
+#define HWADDR_PRIX PRIX64
+
+#endif
+
+#endif
diff --git a/include/qemu/cpu.h b/include/qemu/cpu.h
index ad706a6db..61b76982f 100644
--- a/include/qemu/cpu.h
+++ b/include/qemu/cpu.h
@@ -54,6 +54,9 @@ typedef struct CPUClass {
 
 /**
  * CPUState:
+ * @created: Indicates whether the CPU thread has been successfully created.
+ * @stop: Indicates a pending stop request.
+ * @stopped: Indicates the CPU has been artificially stopped.
  *
  * State of one CPU core or thread.
  */
@@ -66,7 +69,13 @@ struct CPUState {
 #ifdef _WIN32
     HANDLE hThread;
 #endif
+    int thread_id;
+    struct QemuCond *halt_cond;
+    struct qemu_work_item *queued_work_first, *queued_work_last;
     bool thread_kicked;
+    bool created;
+    bool stop;
+    bool stopped;
 
     /* TODO Move common fields from CPUArchState here. */
 };
@@ -78,5 +87,54 @@ struct CPUState {
  */
 void cpu_reset(CPUState *cpu);
 
+/**
+ * qemu_cpu_has_work:
+ * @cpu: The vCPU to check.
+ *
+ * Checks whether the CPU has work to do.
+ *
+ * Returns: %true if the CPU has work, %false otherwise.
+ */
+bool qemu_cpu_has_work(CPUState *cpu);
+
+/**
+ * qemu_cpu_is_self:
+ * @cpu: The vCPU to check against.
+ *
+ * Checks whether the caller is executing on the vCPU thread.
+ *
+ * Returns: %true if called from @cpu's thread, %false otherwise.
+ */
+bool qemu_cpu_is_self(CPUState *cpu);
+
+/**
+ * qemu_cpu_kick:
+ * @cpu: The vCPU to kick.
+ *
+ * Kicks @cpu's thread.
+ */
+void qemu_cpu_kick(CPUState *cpu);
+
+/**
+ * cpu_is_stopped:
+ * @cpu: The CPU to check.
+ *
+ * Checks whether the CPU is stopped.
+ *
+ * Returns: %true if run state is not running or if artificially stopped;
+ * %false otherwise.
+ */
+bool cpu_is_stopped(CPUState *cpu);
+
+/**
+ * run_on_cpu:
+ * @cpu: The vCPU to run on.
+ * @func: The function to be executed.
+ * @data: Data to pass to the function.
+ *
+ * Schedules the function @func for execution on the vCPU @cpu.
+ */
+void run_on_cpu(CPUState *cpu, void (*func)(void *data), void *data);
+
 
 #endif
diff --git a/include/qemu/object.h b/include/qemu/object.h
index cc75feed6..ed1f47f05 100644
--- a/include/qemu/object.h
+++ b/include/qemu/object.h
@@ -230,6 +230,23 @@ typedef struct ObjectProperty
 } ObjectProperty;
 
 /**
+ * ObjectUnparent:
+ * @obj: the object that is being removed from the composition tree
+ *
+ * Called when an object is being removed from the QOM composition tree.
+ * The function should remove any backlinks from children objects to @obj.
+ */
+typedef void (ObjectUnparent)(Object *obj);
+
+/**
+ * ObjectFree:
+ * @obj: the object being freed
+ *
+ * Called when an object's last reference is removed.
+ */
+typedef void (ObjectFree)(void *obj);
+
+/**
  * ObjectClass:
  *
  * The base for all classes.  The only thing that #ObjectClass contains is an
@@ -240,6 +257,8 @@ struct ObjectClass
     /*< private >*/
     Type type;
     GSList *interfaces;
+
+    ObjectUnparent *unparent;
 };
 
 /**
@@ -261,6 +280,7 @@ struct Object
 {
     /*< private >*/
     ObjectClass *class;
+    ObjectFree *free;
     QTAILQ_HEAD(, ObjectProperty) properties;
     uint32_t ref;
     Object *parent;
@@ -485,15 +505,6 @@ void object_initialize_with_type(void *data, Type type);
 void object_initialize(void *obj, const char *typename);
 
 /**
- * object_finalize:
- * @obj: The object to finalize.
- *
- * This function destroys and object without freeing the memory associated with
- * it.
- */
-void object_finalize(void *obj);
-
-/**
  * object_dynamic_cast:
  * @obj: The object to cast.
  * @typename: The @typename to cast to.
@@ -947,6 +958,22 @@ void object_property_add_str(Object *obj, const char *name,
                              struct Error **errp);
 
 /**
+ * object_property_add_bool:
+ * @obj: the object to add a property to
+ * @name: the name of the property
+ * @get: the getter or NULL if the property is write-only.
+ * @set: the setter or NULL if the property is read-only
+ * @errp: if an error occurs, a pointer to an area to store the error
+ *
+ * Add a bool property using getters/setters.  This function will add a
+ * property of type 'bool'.
+ */
+void object_property_add_bool(Object *obj, const char *name,
+                              bool (*get)(Object *, struct Error **),
+                              void (*set)(Object *, bool, struct Error **),
+                              struct Error **errp);
+
+/**
  * object_child_foreach:
  * @obj: the object whose children will be navigated
  * @fn: the iterator function to be called
diff --git a/include/qemu/ratelimit.h b/include/qemu/ratelimit.h
index c6ac28114..d1610f135 100644
--- a/include/qemu/ratelimit.h
+++ b/include/qemu/ratelimit.h
@@ -42,7 +42,7 @@ static inline void ratelimit_set_speed(RateLimit *limit, uint64_t speed,
                                        uint64_t slice_ns)
 {
     limit->slice_ns = slice_ns;
-    limit->slice_quota = ((double)speed * 1000000000ULL) / slice_ns;
+    limit->slice_quota = ((double)speed * slice_ns)/1000000000ULL;
 }
 
 #endif
diff --git a/include/qemu/rng-random.h b/include/qemu/rng-random.h
new file mode 100644
index 000000000..6249290cc
--- /dev/null
+++ b/include/qemu/rng-random.h
@@ -0,0 +1,22 @@
+/*
+ * QEMU Random Number Generator Backend
+ *
+ * Copyright IBM, Corp. 2012
+ *
+ * Authors:
+ *  Anthony Liguori   <aliguori@us.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+#ifndef QEMU_RNG_RANDOM_H
+#define QEMU_RNG_RANDOM_H
+
+#include "qemu/object.h"
+
+#define TYPE_RNG_RANDOM "rng-random"
+#define RNG_RANDOM(obj) OBJECT_CHECK(RndRandom, (obj), TYPE_RNG_RANDOM)
+
+typedef struct RndRandom RndRandom;
+
+#endif
diff --git a/include/qemu/rng.h b/include/qemu/rng.h
new file mode 100644
index 000000000..d094bf8d4
--- /dev/null
+++ b/include/qemu/rng.h
@@ -0,0 +1,93 @@
+/*
+ * QEMU Random Number Generator Backend
+ *
+ * Copyright IBM, Corp. 2012
+ *
+ * Authors:
+ *  Anthony Liguori   <aliguori@us.ibm.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#ifndef QEMU_RNG_H
+#define QEMU_RNG_H
+
+#include "qemu/object.h"
+#include "qemu-common.h"
+#include "error.h"
+
+#define TYPE_RNG_BACKEND "rng-backend"
+#define RNG_BACKEND(obj) \
+    OBJECT_CHECK(RngBackend, (obj), TYPE_RNG_BACKEND)
+#define RNG_BACKEND_GET_CLASS(obj) \
+    OBJECT_GET_CLASS(RngBackendClass, (obj), TYPE_RNG_BACKEND)
+#define RNG_BACKEND_CLASS(klass) \
+    OBJECT_CLASS_CHECK(RngBackendClass, (klass), TYPE_RNG_BACKEND)
+
+typedef struct RngBackendClass RngBackendClass;
+typedef struct RngBackend RngBackend;
+
+typedef void (EntropyReceiveFunc)(void *opaque,
+                                  const void *data,
+                                  size_t size);
+
+struct RngBackendClass
+{
+    ObjectClass parent_class;
+
+    void (*request_entropy)(RngBackend *s, size_t size,
+                            EntropyReceiveFunc *recieve_entropy, void *opaque);
+    void (*cancel_requests)(RngBackend *s);
+
+    void (*opened)(RngBackend *s, Error **errp);
+};
+
+struct RngBackend
+{
+    Object parent;
+
+    /*< protected >*/
+    bool opened;
+};
+
+/**
+ * rng_backend_request_entropy:
+ * @s: the backend to request entropy from
+ * @size: the number of bytes of data to request
+ * @receive_entropy: a function to be invoked when entropy is available
+ * @opaque: data that should be passed to @receive_entropy
+ *
+ * This function is used by the front-end to request entropy from an entropy
+ * source.  This function can be called multiple times before @receive_entropy
+ * is invoked with different values of @receive_entropy and @opaque.  The
+ * backend will queue each request and handle appropriately.
+ *
+ * The backend does not need to pass the full amount of data to @receive_entropy
+ * but will pass a value greater than 0.
+ */
+void rng_backend_request_entropy(RngBackend *s, size_t size,
+                                 EntropyReceiveFunc *receive_entropy,
+                                 void *opaque);
+
+/**
+ * rng_backend_cancel_requests:
+ * @s: the backend to cancel all pending requests in
+ *
+ * Cancels all pending requests submitted by @rng_backend_request_entropy.  This
+ * should be used by a device during reset or in preparation for live migration
+ * to stop tracking any request.
+ */
+void rng_backend_cancel_requests(RngBackend *s);
+
+/**
+ * rng_backend_open:
+ * @s: the backend to open
+ * @errp: a pointer to return the #Error object if an error occurs.
+ *
+ * This function will open the backend if it is not already open.  Calling this
+ * function on an already opened backend will not result in an error.
+ */
+void rng_backend_open(RngBackend *s, Error **errp);
+
+#endif
diff --git a/input.c b/input.c
index 6968b3178..25d3973e2 100644
--- a/input.c
+++ b/input.c
@@ -28,6 +28,7 @@
 #include "console.h"
 #include "error.h"
 #include "qmp-commands.h"
+#include "qapi-types.h"
 
 static QEMUPutKBDEvent *qemu_put_kbd_event;
 static void *qemu_put_kbd_event_opaque;
@@ -37,6 +38,257 @@ static QTAILQ_HEAD(, QEMUPutMouseEntry) mouse_handlers =
 static NotifierList mouse_mode_notifiers = 
     NOTIFIER_LIST_INITIALIZER(mouse_mode_notifiers);
 
+static const int key_defs[] = {
+    [Q_KEY_CODE_SHIFT] = 0x2a,
+    [Q_KEY_CODE_SHIFT_R] = 0x36,
+
+    [Q_KEY_CODE_ALT] = 0x38,
+    [Q_KEY_CODE_ALT_R] = 0xb8,
+    [Q_KEY_CODE_ALTGR] = 0x64,
+    [Q_KEY_CODE_ALTGR_R] = 0xe4,
+    [Q_KEY_CODE_CTRL] = 0x1d,
+    [Q_KEY_CODE_CTRL_R] = 0x9d,
+
+    [Q_KEY_CODE_MENU] = 0xdd,
+
+    [Q_KEY_CODE_ESC] = 0x01,
+
+    [Q_KEY_CODE_1] = 0x02,
+    [Q_KEY_CODE_2] = 0x03,
+    [Q_KEY_CODE_3] = 0x04,
+    [Q_KEY_CODE_4] = 0x05,
+    [Q_KEY_CODE_5] = 0x06,
+    [Q_KEY_CODE_6] = 0x07,
+    [Q_KEY_CODE_7] = 0x08,
+    [Q_KEY_CODE_8] = 0x09,
+    [Q_KEY_CODE_9] = 0x0a,
+    [Q_KEY_CODE_0] = 0x0b,
+    [Q_KEY_CODE_MINUS] = 0x0c,
+    [Q_KEY_CODE_EQUAL] = 0x0d,
+    [Q_KEY_CODE_BACKSPACE] = 0x0e,
+
+    [Q_KEY_CODE_TAB] = 0x0f,
+    [Q_KEY_CODE_Q] = 0x10,
+    [Q_KEY_CODE_W] = 0x11,
+    [Q_KEY_CODE_E] = 0x12,
+    [Q_KEY_CODE_R] = 0x13,
+    [Q_KEY_CODE_T] = 0x14,
+    [Q_KEY_CODE_Y] = 0x15,
+    [Q_KEY_CODE_U] = 0x16,
+    [Q_KEY_CODE_I] = 0x17,
+    [Q_KEY_CODE_O] = 0x18,
+    [Q_KEY_CODE_P] = 0x19,
+    [Q_KEY_CODE_BRACKET_LEFT] = 0x1a,
+    [Q_KEY_CODE_BRACKET_RIGHT] = 0x1b,
+    [Q_KEY_CODE_RET] = 0x1c,
+
+    [Q_KEY_CODE_A] = 0x1e,
+    [Q_KEY_CODE_S] = 0x1f,
+    [Q_KEY_CODE_D] = 0x20,
+    [Q_KEY_CODE_F] = 0x21,
+    [Q_KEY_CODE_G] = 0x22,
+    [Q_KEY_CODE_H] = 0x23,
+    [Q_KEY_CODE_J] = 0x24,
+    [Q_KEY_CODE_K] = 0x25,
+    [Q_KEY_CODE_L] = 0x26,
+    [Q_KEY_CODE_SEMICOLON] = 0x27,
+    [Q_KEY_CODE_APOSTROPHE] = 0x28,
+    [Q_KEY_CODE_GRAVE_ACCENT] = 0x29,
+
+    [Q_KEY_CODE_BACKSLASH] = 0x2b,
+    [Q_KEY_CODE_Z] = 0x2c,
+    [Q_KEY_CODE_X] = 0x2d,
+    [Q_KEY_CODE_C] = 0x2e,
+    [Q_KEY_CODE_V] = 0x2f,
+    [Q_KEY_CODE_B] = 0x30,
+    [Q_KEY_CODE_N] = 0x31,
+    [Q_KEY_CODE_M] = 0x32,
+    [Q_KEY_CODE_COMMA] = 0x33,
+    [Q_KEY_CODE_DOT] = 0x34,
+    [Q_KEY_CODE_SLASH] = 0x35,
+
+    [Q_KEY_CODE_ASTERISK] = 0x37,
+
+    [Q_KEY_CODE_SPC] = 0x39,
+    [Q_KEY_CODE_CAPS_LOCK] = 0x3a,
+    [Q_KEY_CODE_F1] = 0x3b,
+    [Q_KEY_CODE_F2] = 0x3c,
+    [Q_KEY_CODE_F3] = 0x3d,
+    [Q_KEY_CODE_F4] = 0x3e,
+    [Q_KEY_CODE_F5] = 0x3f,
+    [Q_KEY_CODE_F6] = 0x40,
+    [Q_KEY_CODE_F7] = 0x41,
+    [Q_KEY_CODE_F8] = 0x42,
+    [Q_KEY_CODE_F9] = 0x43,
+    [Q_KEY_CODE_F10] = 0x44,
+    [Q_KEY_CODE_NUM_LOCK] = 0x45,
+    [Q_KEY_CODE_SCROLL_LOCK] = 0x46,
+
+    [Q_KEY_CODE_KP_DIVIDE] = 0xb5,
+    [Q_KEY_CODE_KP_MULTIPLY] = 0x37,
+    [Q_KEY_CODE_KP_SUBTRACT] = 0x4a,
+    [Q_KEY_CODE_KP_ADD] = 0x4e,
+    [Q_KEY_CODE_KP_ENTER] = 0x9c,
+    [Q_KEY_CODE_KP_DECIMAL] = 0x53,
+    [Q_KEY_CODE_SYSRQ] = 0x54,
+
+    [Q_KEY_CODE_KP_0] = 0x52,
+    [Q_KEY_CODE_KP_1] = 0x4f,
+    [Q_KEY_CODE_KP_2] = 0x50,
+    [Q_KEY_CODE_KP_3] = 0x51,
+    [Q_KEY_CODE_KP_4] = 0x4b,
+    [Q_KEY_CODE_KP_5] = 0x4c,
+    [Q_KEY_CODE_KP_6] = 0x4d,
+    [Q_KEY_CODE_KP_7] = 0x47,
+    [Q_KEY_CODE_KP_8] = 0x48,
+    [Q_KEY_CODE_KP_9] = 0x49,
+
+    [Q_KEY_CODE_LESS] = 0x56,
+
+    [Q_KEY_CODE_F11] = 0x57,
+    [Q_KEY_CODE_F12] = 0x58,
+
+    [Q_KEY_CODE_PRINT] = 0xb7,
+
+    [Q_KEY_CODE_HOME] = 0xc7,
+    [Q_KEY_CODE_PGUP] = 0xc9,
+    [Q_KEY_CODE_PGDN] = 0xd1,
+    [Q_KEY_CODE_END] = 0xcf,
+
+    [Q_KEY_CODE_LEFT] = 0xcb,
+    [Q_KEY_CODE_UP] = 0xc8,
+    [Q_KEY_CODE_DOWN] = 0xd0,
+    [Q_KEY_CODE_RIGHT] = 0xcd,
+
+    [Q_KEY_CODE_INSERT] = 0xd2,
+    [Q_KEY_CODE_DELETE] = 0xd3,
+#ifdef NEED_CPU_H
+#if defined(TARGET_SPARC) && !defined(TARGET_SPARC64)
+    [Q_KEY_CODE_STOP] = 0xf0,
+    [Q_KEY_CODE_AGAIN] = 0xf1,
+    [Q_KEY_CODE_PROPS] = 0xf2,
+    [Q_KEY_CODE_UNDO] = 0xf3,
+    [Q_KEY_CODE_FRONT] = 0xf4,
+    [Q_KEY_CODE_COPY] = 0xf5,
+    [Q_KEY_CODE_OPEN] = 0xf6,
+    [Q_KEY_CODE_PASTE] = 0xf7,
+    [Q_KEY_CODE_FIND] = 0xf8,
+    [Q_KEY_CODE_CUT] = 0xf9,
+    [Q_KEY_CODE_LF] = 0xfa,
+    [Q_KEY_CODE_HELP] = 0xfb,
+    [Q_KEY_CODE_META_L] = 0xfc,
+    [Q_KEY_CODE_META_R] = 0xfd,
+    [Q_KEY_CODE_COMPOSE] = 0xfe,
+#endif
+#endif
+    [Q_KEY_CODE_MAX] = 0,
+};
+
+int index_from_key(const char *key)
+{
+    int i;
+
+    for (i = 0; QKeyCode_lookup[i] != NULL; i++) {
+        if (!strcmp(key, QKeyCode_lookup[i])) {
+            break;
+        }
+    }
+
+    /* Return Q_KEY_CODE_MAX if the key is invalid */
+    return i;
+}
+
+int index_from_keycode(int code)
+{
+    int i;
+
+    for (i = 0; i < Q_KEY_CODE_MAX; i++) {
+        if (key_defs[i] == code) {
+            break;
+        }
+    }
+
+    /* Return Q_KEY_CODE_MAX if the code is invalid */
+    return i;
+}
+
+static int *keycodes;
+static int keycodes_size;
+static QEMUTimer *key_timer;
+
+static int keycode_from_keyvalue(const KeyValue *value)
+{
+    if (value->kind == KEY_VALUE_KIND_QCODE) {
+        return key_defs[value->qcode];
+    } else {
+        assert(value->kind == KEY_VALUE_KIND_NUMBER);
+        return value->number;
+    }
+}
+
+static void free_keycodes(void)
+{
+    g_free(keycodes);
+    keycodes = NULL;
+    keycodes_size = 0;
+}
+
+static void release_keys(void *opaque)
+{
+    int i;
+
+    for (i = 0; i < keycodes_size; i++) {
+        if (keycodes[i] & 0x80) {
+            kbd_put_keycode(0xe0);
+        }
+        kbd_put_keycode(keycodes[i]| 0x80);
+    }
+
+    free_keycodes();
+}
+
+void qmp_send_key(KeyValueList *keys, bool has_hold_time, int64_t hold_time,
+                  Error **errp)
+{
+    int keycode;
+    KeyValueList *p;
+
+    if (!key_timer) {
+        key_timer = qemu_new_timer_ns(vm_clock, release_keys, NULL);
+    }
+
+    if (keycodes != NULL) {
+        qemu_del_timer(key_timer);
+        release_keys(NULL);
+    }
+
+    if (!has_hold_time) {
+        hold_time = 100;
+    }
+
+    for (p = keys; p != NULL; p = p->next) {
+        /* key down events */
+        keycode = keycode_from_keyvalue(p->value);
+        if (keycode < 0x01 || keycode > 0xff) {
+            error_setg(errp, "invalid hex keycode 0x%x\n", keycode);
+            free_keycodes();
+            return;
+        }
+
+        if (keycode & 0x80) {
+            kbd_put_keycode(0xe0);
+        }
+        kbd_put_keycode(keycode & 0x7f);
+
+        keycodes = g_realloc(keycodes, sizeof(int) * (keycodes_size + 1));
+        keycodes[keycodes_size++] = keycode;
+    }
+
+    /* delayed key up events */
+    qemu_mod_timer(key_timer, qemu_get_clock_ns(vm_clock) +
+                   muldiv64(get_ticks_per_sec(), hold_time, 1000));
+}
+
 void qemu_add_kbd_event_handler(QEMUPutKBDEvent *func, void *opaque)
 {
     qemu_put_kbd_event_opaque = opaque;
diff --git a/iohandler.c b/iohandler.c
index dea43552d..60460a6f8 100644
--- a/iohandler.c
+++ b/iohandler.c
@@ -26,6 +26,7 @@
 #include "qemu-common.h"
 #include "qemu-char.h"
 #include "qemu-queue.h"
+#include "qemu-aio.h"
 #include "main-loop.h"
 
 #ifndef _WIN32
@@ -56,6 +57,8 @@ int qemu_set_fd_handler2(int fd,
 {
     IOHandlerRecord *ioh;
 
+    assert(fd >= 0);
+
     if (!fd_read && !fd_write) {
         QLIST_FOREACH(ioh, &io_handlers, next) {
             if (ioh->fd == fd) {
diff --git a/iov.c b/iov.c
index 60705c73a..a81eedced 100644
--- a/iov.c
+++ b/iov.c
@@ -26,7 +26,7 @@
 # include <sys/socket.h>
 #endif
 
-size_t iov_from_buf(struct iovec *iov, unsigned int iov_cnt,
+size_t iov_from_buf(const struct iovec *iov, unsigned int iov_cnt,
                     size_t offset, const void *buf, size_t bytes)
 {
     size_t done;
@@ -228,3 +228,129 @@ void iov_hexdump(const struct iovec *iov, const unsigned int iov_cnt,
         fprintf(fp, "\n");
     }
 }
+
+unsigned iov_copy(struct iovec *dst_iov, unsigned int dst_iov_cnt,
+                 const struct iovec *iov, unsigned int iov_cnt,
+                 size_t offset, size_t bytes)
+{
+    size_t len;
+    unsigned int i, j;
+    for (i = 0, j = 0; i < iov_cnt && j < dst_iov_cnt && bytes; i++) {
+        if (offset >= iov[i].iov_len) {
+            offset -= iov[i].iov_len;
+            continue;
+        }
+        len = MIN(bytes, iov[i].iov_len - offset);
+
+        dst_iov[j].iov_base = iov[i].iov_base + offset;
+        dst_iov[j].iov_len = len;
+        j++;
+        bytes -= len;
+        offset = 0;
+    }
+    assert(offset == 0);
+    return j;
+}
+
+/* io vectors */
+
+void qemu_iovec_init(QEMUIOVector *qiov, int alloc_hint)
+{
+    qiov->iov = g_malloc(alloc_hint * sizeof(struct iovec));
+    qiov->niov = 0;
+    qiov->nalloc = alloc_hint;
+    qiov->size = 0;
+}
+
+void qemu_iovec_init_external(QEMUIOVector *qiov, struct iovec *iov, int niov)
+{
+    int i;
+
+    qiov->iov = iov;
+    qiov->niov = niov;
+    qiov->nalloc = -1;
+    qiov->size = 0;
+    for (i = 0; i < niov; i++)
+        qiov->size += iov[i].iov_len;
+}
+
+void qemu_iovec_add(QEMUIOVector *qiov, void *base, size_t len)
+{
+    assert(qiov->nalloc != -1);
+
+    if (qiov->niov == qiov->nalloc) {
+        qiov->nalloc = 2 * qiov->nalloc + 1;
+        qiov->iov = g_realloc(qiov->iov, qiov->nalloc * sizeof(struct iovec));
+    }
+    qiov->iov[qiov->niov].iov_base = base;
+    qiov->iov[qiov->niov].iov_len = len;
+    qiov->size += len;
+    ++qiov->niov;
+}
+
+/*
+ * Concatenates (partial) iovecs from src to the end of dst.
+ * It starts copying after skipping `soffset' bytes at the
+ * beginning of src and adds individual vectors from src to
+ * dst copies up to `sbytes' bytes total, or up to the end
+ * of src if it comes first.  This way, it is okay to specify
+ * very large value for `sbytes' to indicate "up to the end
+ * of src".
+ * Only vector pointers are processed, not the actual data buffers.
+ */
+void qemu_iovec_concat(QEMUIOVector *dst,
+                       QEMUIOVector *src, size_t soffset, size_t sbytes)
+{
+    int i;
+    size_t done;
+    struct iovec *siov = src->iov;
+    assert(dst->nalloc != -1);
+    assert(src->size >= soffset);
+    for (i = 0, done = 0; done < sbytes && i < src->niov; i++) {
+        if (soffset < siov[i].iov_len) {
+            size_t len = MIN(siov[i].iov_len - soffset, sbytes - done);
+            qemu_iovec_add(dst, siov[i].iov_base + soffset, len);
+            done += len;
+            soffset = 0;
+        } else {
+            soffset -= siov[i].iov_len;
+        }
+    }
+    /* return done; */
+}
+
+void qemu_iovec_destroy(QEMUIOVector *qiov)
+{
+    assert(qiov->nalloc != -1);
+
+    qemu_iovec_reset(qiov);
+    g_free(qiov->iov);
+    qiov->nalloc = 0;
+    qiov->iov = NULL;
+}
+
+void qemu_iovec_reset(QEMUIOVector *qiov)
+{
+    assert(qiov->nalloc != -1);
+
+    qiov->niov = 0;
+    qiov->size = 0;
+}
+
+size_t qemu_iovec_to_buf(QEMUIOVector *qiov, size_t offset,
+                         void *buf, size_t bytes)
+{
+    return iov_to_buf(qiov->iov, qiov->niov, offset, buf, bytes);
+}
+
+size_t qemu_iovec_from_buf(QEMUIOVector *qiov, size_t offset,
+                           const void *buf, size_t bytes)
+{
+    return iov_from_buf(qiov->iov, qiov->niov, offset, buf, bytes);
+}
+
+size_t qemu_iovec_memset(QEMUIOVector *qiov, size_t offset,
+                         int fillc, size_t bytes)
+{
+    return iov_memset(qiov->iov, qiov->niov, offset, fillc, bytes);
+}
diff --git a/iov.h b/iov.h
index 381f37a54..34c8ec9fa 100644
--- a/iov.h
+++ b/iov.h
@@ -36,7 +36,7 @@ size_t iov_size(const struct iovec *iov, const unsigned int iov_cnt);
  * such "large" value is -1 (sinice size_t is unsigned),
  * so specifying `-1' as `bytes' means 'up to the end of iovec'.
  */
-size_t iov_from_buf(struct iovec *iov, unsigned int iov_cnt,
+size_t iov_from_buf(const struct iovec *iov, unsigned int iov_cnt,
                     size_t offset, const void *buf, size_t bytes);
 size_t iov_to_buf(const struct iovec *iov, const unsigned int iov_cnt,
                   size_t offset, void *buf, size_t bytes);
@@ -86,3 +86,12 @@ ssize_t iov_send_recv(int sockfd, struct iovec *iov, unsigned iov_cnt,
  */
 void iov_hexdump(const struct iovec *iov, const unsigned int iov_cnt,
                  FILE *fp, const char *prefix, size_t limit);
+
+/*
+ * Partial copy of vector from iov to dst_iov (data is not copied).
+ * dst_iov overlaps iov at a specified offset.
+ * size of dst_iov is at most bytes. dst vector count is returned.
+ */
+unsigned iov_copy(struct iovec *dst_iov, unsigned int dst_iov_cnt,
+                 const struct iovec *iov, unsigned int iov_cnt,
+                 size_t offset, size_t bytes);
diff --git a/kvm-all.c b/kvm-all.c
index 34b02c1fb..8e9a8d8fd 100644
--- a/kvm-all.c
+++ b/kvm-all.c
@@ -39,6 +39,10 @@
 #include <sys/eventfd.h>
 #endif
 
+#ifdef CONFIG_VALGRIND_H
+#include <valgrind/memcheck.h>
+#endif
+
 /* KVM uses PAGE_SIZE in its definition of COALESCED_MMIO_MAX */
 #define PAGE_SIZE TARGET_PAGE_SIZE
 
@@ -56,7 +60,7 @@
 
 typedef struct KVMSlot
 {
-    target_phys_addr_t start_addr;
+    hwaddr start_addr;
     ram_addr_t memory_size;
     void *ram;
     int slot;
@@ -84,10 +88,11 @@ struct KVMState
     int pit_state2;
     int xsave, xcrs;
     int many_ioeventfds;
+    int intx_set_mask;
     /* The man page (and posix) say ioctl numbers are signed int, but
      * they're not.  Linux, glibc and *BSD all treat ioctl numbers as
      * unsigned, and treating them as signed here can break things */
-    unsigned irqchip_inject_ioctl;
+    unsigned irq_set_ioctl;
 #ifdef KVM_CAP_IRQ_ROUTING
     struct kvm_irq_routing *irq_routes;
     int nr_allocated_irq_routes;
@@ -126,8 +131,8 @@ static KVMSlot *kvm_alloc_slot(KVMState *s)
 }
 
 static KVMSlot *kvm_lookup_matching_slot(KVMState *s,
-                                         target_phys_addr_t start_addr,
-                                         target_phys_addr_t end_addr)
+                                         hwaddr start_addr,
+                                         hwaddr end_addr)
 {
     int i;
 
@@ -147,8 +152,8 @@ static KVMSlot *kvm_lookup_matching_slot(KVMState *s,
  * Find overlapping slot with lowest start address
  */
 static KVMSlot *kvm_lookup_overlapping_slot(KVMState *s,
-                                            target_phys_addr_t start_addr,
-                                            target_phys_addr_t end_addr)
+                                            hwaddr start_addr,
+                                            hwaddr end_addr)
 {
     KVMSlot *found = NULL;
     int i;
@@ -171,7 +176,7 @@ static KVMSlot *kvm_lookup_overlapping_slot(KVMState *s,
 }
 
 int kvm_physical_memory_addr_from_host(KVMState *s, void *ram,
-                                       target_phys_addr_t *phys_addr)
+                                       hwaddr *phys_addr)
 {
     int i;
 
@@ -288,7 +293,7 @@ static int kvm_slot_dirty_pages_log_change(KVMSlot *mem, bool log_dirty)
     return kvm_set_user_memory_region(s, mem);
 }
 
-static int kvm_dirty_pages_log_change(target_phys_addr_t phys_addr,
+static int kvm_dirty_pages_log_change(hwaddr phys_addr,
                                       ram_addr_t size, bool log_dirty)
 {
     KVMState *s = kvm_state;
@@ -297,7 +302,7 @@ static int kvm_dirty_pages_log_change(target_phys_addr_t phys_addr,
     if (mem == NULL)  {
         fprintf(stderr, "BUG: %s: invalid parameters " TARGET_FMT_plx "-"
                 TARGET_FMT_plx "\n", __func__, phys_addr,
-                (target_phys_addr_t)(phys_addr + size - 1));
+                (hwaddr)(phys_addr + size - 1));
         return -EINVAL;
     }
     return kvm_slot_dirty_pages_log_change(mem, log_dirty);
@@ -358,8 +363,8 @@ static int kvm_get_dirty_pages_log_range(MemoryRegionSection *section,
 {
     unsigned int i, j;
     unsigned long page_number, c;
-    target_phys_addr_t addr, addr1;
-    unsigned int len = ((section->size / TARGET_PAGE_SIZE) + HOST_LONG_BITS - 1) / HOST_LONG_BITS;
+    hwaddr addr, addr1;
+    unsigned int len = ((section->size / getpagesize()) + HOST_LONG_BITS - 1) / HOST_LONG_BITS;
     unsigned long hpratio = getpagesize() / TARGET_PAGE_SIZE;
 
     /*
@@ -401,8 +406,8 @@ static int kvm_physical_sync_dirty_bitmap(MemoryRegionSection *section)
     KVMDirtyLog d;
     KVMSlot *mem;
     int ret = 0;
-    target_phys_addr_t start_addr = section->offset_within_address_space;
-    target_phys_addr_t end_addr = start_addr + section->size;
+    hwaddr start_addr = section->offset_within_address_space;
+    hwaddr end_addr = start_addr + section->size;
 
     d.dirty_bitmap = NULL;
     while (start_addr < end_addr) {
@@ -449,9 +454,10 @@ static int kvm_physical_sync_dirty_bitmap(MemoryRegionSection *section)
     return ret;
 }
 
-int kvm_coalesce_mmio_region(target_phys_addr_t start, ram_addr_t size)
+static void kvm_coalesce_mmio_region(MemoryListener *listener,
+                                     MemoryRegionSection *secion,
+                                     hwaddr start, hwaddr size)
 {
-    int ret = -ENOSYS;
     KVMState *s = kvm_state;
 
     if (s->coalesced_mmio) {
@@ -461,15 +467,14 @@ int kvm_coalesce_mmio_region(target_phys_addr_t start, ram_addr_t size)
         zone.size = size;
         zone.pad = 0;
 
-        ret = kvm_vm_ioctl(s, KVM_REGISTER_COALESCED_MMIO, &zone);
+        (void)kvm_vm_ioctl(s, KVM_REGISTER_COALESCED_MMIO, &zone);
     }
-
-    return ret;
 }
 
-int kvm_uncoalesce_mmio_region(target_phys_addr_t start, ram_addr_t size)
+static void kvm_uncoalesce_mmio_region(MemoryListener *listener,
+                                       MemoryRegionSection *secion,
+                                       hwaddr start, hwaddr size)
 {
-    int ret = -ENOSYS;
     KVMState *s = kvm_state;
 
     if (s->coalesced_mmio) {
@@ -479,10 +484,8 @@ int kvm_uncoalesce_mmio_region(target_phys_addr_t start, ram_addr_t size)
         zone.size = size;
         zone.pad = 0;
 
-        ret = kvm_vm_ioctl(s, KVM_UNREGISTER_COALESCED_MMIO, &zone);
+        (void)kvm_vm_ioctl(s, KVM_UNREGISTER_COALESCED_MMIO, &zone);
     }
-
-    return ret;
 }
 
 int kvm_check_extension(KVMState *s, unsigned int extension)
@@ -553,7 +556,7 @@ static void kvm_set_phys_mem(MemoryRegionSection *section, bool add)
     int err;
     MemoryRegion *mr = section->mr;
     bool log_dirty = memory_region_is_logging(mr);
-    target_phys_addr_t start_addr = section->offset_within_address_space;
+    hwaddr start_addr = section->offset_within_address_space;
     ram_addr_t size = section->size;
     void *ram = NULL;
     unsigned delta;
@@ -698,14 +701,6 @@ static void kvm_set_phys_mem(MemoryRegionSection *section, bool add)
     }
 }
 
-static void kvm_begin(MemoryListener *listener)
-{
-}
-
-static void kvm_commit(MemoryListener *listener)
-{
-}
-
 static void kvm_region_add(MemoryListener *listener,
                            MemoryRegionSection *section)
 {
@@ -718,11 +713,6 @@ static void kvm_region_del(MemoryListener *listener,
     kvm_set_phys_mem(section, false);
 }
 
-static void kvm_region_nop(MemoryListener *listener,
-                           MemoryRegionSection *section)
-{
-}
-
 static void kvm_log_sync(MemoryListener *listener,
                          MemoryRegionSection *section)
 {
@@ -750,9 +740,12 @@ static void kvm_log_global_stop(struct MemoryListener *listener)
     assert(r >= 0);
 }
 
-static void kvm_mem_ioeventfd_add(MemoryRegionSection *section,
-                                  bool match_data, uint64_t data, int fd)
+static void kvm_mem_ioeventfd_add(MemoryListener *listener,
+                                  MemoryRegionSection *section,
+                                  bool match_data, uint64_t data,
+                                  EventNotifier *e)
 {
+    int fd = event_notifier_get_fd(e);
     int r;
 
     assert(match_data && section->size <= 8);
@@ -764,9 +757,12 @@ static void kvm_mem_ioeventfd_add(MemoryRegionSection *section,
     }
 }
 
-static void kvm_mem_ioeventfd_del(MemoryRegionSection *section,
-                                  bool match_data, uint64_t data, int fd)
+static void kvm_mem_ioeventfd_del(MemoryListener *listener,
+                                  MemoryRegionSection *section,
+                                  bool match_data, uint64_t data,
+                                  EventNotifier *e)
 {
+    int fd = event_notifier_get_fd(e);
     int r;
 
     r = kvm_set_ioeventfd_mmio(fd, section->offset_within_address_space,
@@ -776,9 +772,12 @@ static void kvm_mem_ioeventfd_del(MemoryRegionSection *section,
     }
 }
 
-static void kvm_io_ioeventfd_add(MemoryRegionSection *section,
-                                 bool match_data, uint64_t data, int fd)
+static void kvm_io_ioeventfd_add(MemoryListener *listener,
+                                 MemoryRegionSection *section,
+                                 bool match_data, uint64_t data,
+                                 EventNotifier *e)
 {
+    int fd = event_notifier_get_fd(e);
     int r;
 
     assert(match_data && section->size == 2);
@@ -790,10 +789,13 @@ static void kvm_io_ioeventfd_add(MemoryRegionSection *section,
     }
 }
 
-static void kvm_io_ioeventfd_del(MemoryRegionSection *section,
-                                 bool match_data, uint64_t data, int fd)
+static void kvm_io_ioeventfd_del(MemoryListener *listener,
+                                 MemoryRegionSection *section,
+                                 bool match_data, uint64_t data,
+                                 EventNotifier *e)
 
 {
+    int fd = event_notifier_get_fd(e);
     int r;
 
     r = kvm_set_ioeventfd_pio_word(fd, section->offset_within_address_space,
@@ -803,56 +805,35 @@ static void kvm_io_ioeventfd_del(MemoryRegionSection *section,
     }
 }
 
-static void kvm_eventfd_add(MemoryListener *listener,
-                            MemoryRegionSection *section,
-                            bool match_data, uint64_t data,
-                            EventNotifier *e)
-{
-    if (section->address_space == get_system_memory()) {
-        kvm_mem_ioeventfd_add(section, match_data, data,
-			      event_notifier_get_fd(e));
-    } else {
-        kvm_io_ioeventfd_add(section, match_data, data,
-			     event_notifier_get_fd(e));
-    }
-}
-
-static void kvm_eventfd_del(MemoryListener *listener,
-                            MemoryRegionSection *section,
-                            bool match_data, uint64_t data,
-                            EventNotifier *e)
-{
-    if (section->address_space == get_system_memory()) {
-        kvm_mem_ioeventfd_del(section, match_data, data,
-			      event_notifier_get_fd(e));
-    } else {
-        kvm_io_ioeventfd_del(section, match_data, data,
-			     event_notifier_get_fd(e));
-    }
-}
-
 static MemoryListener kvm_memory_listener = {
-    .begin = kvm_begin,
-    .commit = kvm_commit,
     .region_add = kvm_region_add,
     .region_del = kvm_region_del,
-    .region_nop = kvm_region_nop,
     .log_start = kvm_log_start,
     .log_stop = kvm_log_stop,
     .log_sync = kvm_log_sync,
     .log_global_start = kvm_log_global_start,
     .log_global_stop = kvm_log_global_stop,
-    .eventfd_add = kvm_eventfd_add,
-    .eventfd_del = kvm_eventfd_del,
+    .eventfd_add = kvm_mem_ioeventfd_add,
+    .eventfd_del = kvm_mem_ioeventfd_del,
+    .coalesced_mmio_add = kvm_coalesce_mmio_region,
+    .coalesced_mmio_del = kvm_uncoalesce_mmio_region,
+    .priority = 10,
+};
+
+static MemoryListener kvm_io_listener = {
+    .eventfd_add = kvm_io_ioeventfd_add,
+    .eventfd_del = kvm_io_ioeventfd_del,
     .priority = 10,
 };
 
 static void kvm_handle_interrupt(CPUArchState *env, int mask)
 {
+    CPUState *cpu = ENV_GET_CPU(env);
+
     env->interrupt_request |= mask;
 
-    if (!qemu_cpu_is_self(env)) {
-        qemu_cpu_kick(env);
+    if (!qemu_cpu_is_self(cpu)) {
+        qemu_cpu_kick(cpu);
     }
 }
 
@@ -865,13 +846,13 @@ int kvm_set_irq(KVMState *s, int irq, int level)
 
     event.level = level;
     event.irq = irq;
-    ret = kvm_vm_ioctl(s, s->irqchip_inject_ioctl, &event);
+    ret = kvm_vm_ioctl(s, s->irq_set_ioctl, &event);
     if (ret < 0) {
         perror("kvm_set_irq");
         abort();
     }
 
-    return (s->irqchip_inject_ioctl == KVM_IRQ_LINE) ? 1 : event.status;
+    return (s->irq_set_ioctl == KVM_IRQ_LINE) ? 1 : event.status;
 }
 
 #ifdef KVM_CAP_IRQ_ROUTING
@@ -959,6 +940,30 @@ static void kvm_add_routing_entry(KVMState *s,
     kvm_irqchip_commit_routes(s);
 }
 
+static int kvm_update_routing_entry(KVMState *s,
+                                    struct kvm_irq_routing_entry *new_entry)
+{
+    struct kvm_irq_routing_entry *entry;
+    int n;
+
+    for (n = 0; n < s->irq_routes->nr; n++) {
+        entry = &s->irq_routes->entries[n];
+        if (entry->gsi != new_entry->gsi) {
+            continue;
+        }
+
+        entry->type = new_entry->type;
+        entry->flags = new_entry->flags;
+        entry->u = new_entry->u;
+
+        kvm_irqchip_commit_routes(s);
+
+        return 0;
+    }
+
+    return -ESRCH;
+}
+
 void kvm_irqchip_add_irq_route(KVMState *s, int irq, int irqchip, int pin)
 {
     struct kvm_irq_routing_entry e;
@@ -1121,6 +1126,24 @@ int kvm_irqchip_add_msi_route(KVMState *s, MSIMessage msg)
     return virq;
 }
 
+int kvm_irqchip_update_msi_route(KVMState *s, int virq, MSIMessage msg)
+{
+    struct kvm_irq_routing_entry kroute;
+
+    if (!kvm_irqchip_in_kernel()) {
+        return -ENOSYS;
+    }
+
+    kroute.gsi = virq;
+    kroute.type = KVM_IRQ_ROUTING_MSI;
+    kroute.flags = 0;
+    kroute.u.msi.address_lo = (uint32_t)msg.address;
+    kroute.u.msi.address_hi = msg.address >> 32;
+    kroute.u.msi.data = msg.data;
+
+    return kvm_update_routing_entry(s, &kroute);
+}
+
 static int kvm_irqchip_assign_irqfd(KVMState *s, int fd, int virq, bool assign)
 {
     struct kvm_irqfd irqfd = {
@@ -1162,24 +1185,14 @@ static int kvm_irqchip_assign_irqfd(KVMState *s, int fd, int virq, bool assign)
 }
 #endif /* !KVM_CAP_IRQ_ROUTING */
 
-int kvm_irqchip_add_irqfd(KVMState *s, int fd, int virq)
-{
-    return kvm_irqchip_assign_irqfd(s, fd, virq, true);
-}
-
-int kvm_irqchip_add_irq_notifier(KVMState *s, EventNotifier *n, int virq)
+int kvm_irqchip_add_irqfd_notifier(KVMState *s, EventNotifier *n, int virq)
 {
-    return kvm_irqchip_add_irqfd(s, event_notifier_get_fd(n), virq);
+    return kvm_irqchip_assign_irqfd(s, event_notifier_get_fd(n), virq, true);
 }
 
-int kvm_irqchip_remove_irqfd(KVMState *s, int fd, int virq)
+int kvm_irqchip_remove_irqfd_notifier(KVMState *s, EventNotifier *n, int virq)
 {
-    return kvm_irqchip_assign_irqfd(s, fd, virq, false);
-}
-
-int kvm_irqchip_remove_irq_notifier(KVMState *s, EventNotifier *n, int virq)
-{
-    return kvm_irqchip_remove_irqfd(s, event_notifier_get_fd(n), virq);
+    return kvm_irqchip_assign_irqfd(s, event_notifier_get_fd(n), virq, false);
 }
 
 static int kvm_irqchip_create(KVMState *s)
@@ -1200,10 +1213,6 @@ static int kvm_irqchip_create(KVMState *s)
         return ret;
     }
 
-    s->irqchip_inject_ioctl = KVM_IRQ_LINE;
-    if (kvm_check_extension(s, KVM_CAP_IRQ_INJECT_STATUS)) {
-        s->irqchip_inject_ioctl = KVM_IRQ_LINE_STATUS;
-    }
     kvm_kernel_irqchip = true;
     /* If we have an in-kernel IRQ chip then we must have asynchronous
      * interrupt delivery (though the reverse is not necessarily true)
@@ -1350,6 +1359,13 @@ int kvm_init(void)
     s->direct_msi = (kvm_check_extension(s, KVM_CAP_SIGNAL_MSI) > 0);
 #endif
 
+    s->intx_set_mask = kvm_check_extension(s, KVM_CAP_PCI_2_3);
+
+    s->irq_set_ioctl = KVM_IRQ_LINE;
+    if (kvm_check_extension(s, KVM_CAP_IRQ_INJECT_STATUS)) {
+        s->irq_set_ioctl = KVM_IRQ_LINE_STATUS;
+    }
+
     ret = kvm_arch_init(s);
     if (ret < 0) {
         goto err;
@@ -1361,7 +1377,8 @@ int kvm_init(void)
     }
 
     kvm_state = s;
-    memory_listener_register(&kvm_memory_listener, NULL);
+    memory_listener_register(&kvm_memory_listener, &address_space_memory);
+    memory_listener_register(&kvm_io_listener, &address_space_io);
 
     s->many_ioeventfds = kvm_check_many_ioeventfds();
 
@@ -1370,13 +1387,11 @@ int kvm_init(void)
     return 0;
 
 err:
-    if (s) {
-        if (s->vmfd >= 0) {
-            close(s->vmfd);
-        }
-        if (s->fd != -1) {
-            close(s->fd);
-        }
+    if (s->vmfd >= 0) {
+        close(s->vmfd);
+    }
+    if (s->fd != -1) {
+        close(s->fd);
     }
     g_free(s);
 
@@ -1485,8 +1500,10 @@ static void do_kvm_cpu_synchronize_state(void *_env)
 
 void kvm_cpu_synchronize_state(CPUArchState *env)
 {
+    CPUState *cpu = ENV_GET_CPU(env);
+
     if (!env->kvm_vcpu_dirty) {
-        run_on_cpu(env, do_kvm_cpu_synchronize_state, env);
+        run_on_cpu(cpu, do_kvm_cpu_synchronize_state, env);
     }
 }
 
@@ -1537,8 +1554,6 @@ int kvm_cpu_exec(CPUArchState *env)
         qemu_mutex_lock_iothread();
         kvm_arch_post_run(env, run);
 
-        kvm_flush_coalesced_mmio_buffer();
-
         if (run_ret < 0) {
             if (run_ret == -EINTR || run_ret == -EAGAIN) {
                 DPRINTF("io window exit\n");
@@ -1704,6 +1719,11 @@ int kvm_has_gsi_routing(void)
 #endif
 }
 
+int kvm_has_intx_set_mask(void)
+{
+    return kvm_state->intx_set_mask;
+}
+
 void *kvm_vmalloc(ram_addr_t size)
 {
 #ifdef TARGET_S390X
@@ -1719,6 +1739,9 @@ void *kvm_vmalloc(ram_addr_t size)
 
 void kvm_setup_guest_memory(void *start, size_t size)
 {
+#ifdef CONFIG_VALGRIND_H
+    VALGRIND_MAKE_MEM_DEFINED(start, size);
+#endif
     if (!kvm_has_sync_mmu()) {
         int ret = qemu_madvise(start, size, QEMU_MADV_DONTFORK);
 
@@ -1766,6 +1789,7 @@ static void kvm_invoke_set_guest_debug(void *data)
 
 int kvm_update_guest_debug(CPUArchState *env, unsigned long reinject_trap)
 {
+    CPUState *cpu = ENV_GET_CPU(env);
     struct kvm_set_guest_debug_data data;
 
     data.dbg.control = reinject_trap;
@@ -1776,7 +1800,7 @@ int kvm_update_guest_debug(CPUArchState *env, unsigned long reinject_trap)
     kvm_arch_update_guest_debug(env, &data.dbg);
     data.env = env;
 
-    run_on_cpu(env, kvm_invoke_set_guest_debug, &data);
+    run_on_cpu(cpu, kvm_invoke_set_guest_debug, &data);
     return data.err;
 }
 
@@ -1881,6 +1905,8 @@ void kvm_remove_all_breakpoints(CPUArchState *current_env)
                 }
             }
         }
+        QTAILQ_REMOVE(&s->kvm_sw_breakpoints, bp, entry);
+        g_free(bp);
     }
     kvm_arch_remove_all_hw_breakpoints();
 
diff --git a/kvm-stub.c b/kvm-stub.c
index 94c9ea15b..a3455e220 100644
--- a/kvm-stub.c
+++ b/kvm-stub.c
@@ -29,16 +29,6 @@ int kvm_init_vcpu(CPUArchState *env)
     return -ENOSYS;
 }
 
-int kvm_coalesce_mmio_region(target_phys_addr_t start, ram_addr_t size)
-{
-    return -ENOSYS;
-}
-
-int kvm_uncoalesce_mmio_region(target_phys_addr_t start, ram_addr_t size)
-{
-    return -ENOSYS;
-}
-
 int kvm_init(void)
 {
     return -ENOSYS;
@@ -141,22 +131,12 @@ void kvm_irqchip_release_virq(KVMState *s, int virq)
 {
 }
 
-int kvm_irqchip_add_irqfd(KVMState *s, int fd, int virq)
-{
-    return -ENOSYS;
-}
-
-int kvm_irqchip_add_irq_notifier(KVMState *s, EventNotifier *n, int virq)
-{
-    return -ENOSYS;
-}
-
-int kvm_irqchip_remove_irqfd(KVMState *s, int fd, int virq)
+int kvm_irqchip_add_irqfd_notifier(KVMState *s, EventNotifier *n, int virq)
 {
     return -ENOSYS;
 }
 
-int kvm_irqchip_remove_irq_notifier(KVMState *s, EventNotifier *n, int virq)
+int kvm_irqchip_remove_irqfd_notifier(KVMState *s, EventNotifier *n, int virq)
 {
     return -ENOSYS;
 }
diff --git a/kvm.h b/kvm.h
index 5b8f58881..72d866a96 100644
--- a/kvm.h
+++ b/kvm.h
@@ -20,6 +20,7 @@
 
 #ifdef CONFIG_KVM
 #include <linux/kvm.h>
+#include <linux/kvm_para.h>
 #endif
 
 extern int kvm_allowed;
@@ -117,6 +118,7 @@ int kvm_has_xcrs(void);
 int kvm_has_pit_state2(void);
 int kvm_has_many_ioeventfds(void);
 int kvm_has_gsi_routing(void);
+int kvm_has_intx_set_mask(void);
 
 #ifdef NEED_CPU_H
 int kvm_init_vcpu(CPUArchState *env);
@@ -128,8 +130,6 @@ void *kvm_vmalloc(ram_addr_t size);
 void *kvm_arch_vmalloc(ram_addr_t size);
 void kvm_setup_guest_memory(void *start, size_t size);
 
-int kvm_coalesce_mmio_region(target_phys_addr_t start, ram_addr_t size);
-int kvm_uncoalesce_mmio_region(target_phys_addr_t start, ram_addr_t size);
 void kvm_flush_coalesced_mmio_buffer(void);
 #endif
 
@@ -260,7 +260,7 @@ static inline void cpu_synchronize_post_init(CPUArchState *env)
 
 #if !defined(CONFIG_USER_ONLY)
 int kvm_physical_memory_addr_from_host(KVMState *s, void *ram_addr,
-                                       target_phys_addr_t *phys_addr);
+                                       hwaddr *phys_addr);
 #endif
 
 #endif
@@ -270,10 +270,11 @@ int kvm_set_ioeventfd_mmio(int fd, uint32_t adr, uint32_t val, bool assign,
 int kvm_set_ioeventfd_pio_word(int fd, uint16_t adr, uint16_t val, bool assign);
 
 int kvm_irqchip_add_msi_route(KVMState *s, MSIMessage msg);
+int kvm_irqchip_update_msi_route(KVMState *s, int virq, MSIMessage msg);
 void kvm_irqchip_release_virq(KVMState *s, int virq);
 
-int kvm_irqchip_add_irqfd(KVMState *s, int fd, int virq);
-int kvm_irqchip_remove_irqfd(KVMState *s, int fd, int virq);
-int kvm_irqchip_add_irq_notifier(KVMState *s, EventNotifier *n, int virq);
-int kvm_irqchip_remove_irq_notifier(KVMState *s, EventNotifier *n, int virq);
+int kvm_irqchip_add_irqfd_notifier(KVMState *s, EventNotifier *n, int virq);
+int kvm_irqchip_remove_irqfd_notifier(KVMState *s, EventNotifier *n, int virq);
+void kvm_pc_gsi_handler(void *opaque, int n, int level);
+void kvm_pc_setup_irq_routing(bool pci_enabled);
 #endif
diff --git a/libcacard/Makefile b/libcacard/Makefile
index 63990b700..c26aac65c 100644
--- a/libcacard/Makefile
+++ b/libcacard/Makefile
@@ -1,19 +1,22 @@
 -include ../config-host.mak
--include $(SRC_PATH)/Makefile.objs
 -include $(SRC_PATH)/rules.mak
+-include $(SRC_PATH)/Makefile.objs
 
 libcacard_includedir=$(includedir)/cacard
 
 $(call set-vpath, $(SRC_PATH))
 
 # objects linked into a shared library, built with libtool with -fPIC if required
-QEMU_OBJS=$(oslib-obj-y) qemu-timer-common.o $(trace-obj-y)
+QEMU_OBJS=$(oslib-obj-y) qemu-timer-common.o $(trace-obj-y) $(stub-obj-y)
 QEMU_OBJS_LIB=$(patsubst %.o,%.lo,$(QEMU_OBJS))
 
 QEMU_CFLAGS+=-I../
 
 libcacard.lib-y=$(patsubst %.o,%.lo,$(libcacard-y))
 
+vscclient: $(libcacard-y) $(QEMU_OBJS) vscclient.o cutils.o
+	$(call quiet-command,$(CC) -o $@ $^ $(libcacard_libs) $(LIBS),"  LINK  $@")
+
 clean:
 	rm -f *.o */*.o *.d */*.d *.a */*.a *~ */*~ vscclient *.lo */*.lo .libs/* */.libs/* *.la */*.la *.pc
 	rm -Rf .libs */.libs
diff --git a/libcacard/vcard.c b/libcacard/vcard.c
index b02556ee0..539177bb4 100644
--- a/libcacard/vcard.c
+++ b/libcacard/vcard.c
@@ -200,7 +200,6 @@ vcard_free(VCard *vcard)
     }
     vcard_buffer_response_delete(vcard->vcard_buffer_response);
     g_free(vcard);
-    return;
 }
 
 void
diff --git a/libcacard/vcard_emul_nss.c b/libcacard/vcard_emul_nss.c
index 802cae3a2..5f565e0b4 100644
--- a/libcacard/vcard_emul_nss.c
+++ b/libcacard/vcard_emul_nss.c
@@ -168,7 +168,6 @@ vcard_emul_delete_key(VCardKey *key)
     if (key->slot) {
         PK11_FreeSlot(key->slot);
     }
-    return;
 }
 
 /*
@@ -418,7 +417,6 @@ vcard_emul_reset(VCard *card, VCardPower power)
     /* TODO: we may also need to send insertion/removal events? */
     slot = vcard_emul_card_get_slot(card);
     PK11_Logout(slot); /* NOTE: ignoring SECStatus return value */
-    return;
 }
 
 
@@ -535,7 +533,6 @@ vcard_emul_get_atr(VCard *card, unsigned char *atr, int *atr_len)
 
     memcpy(atr, nss_atr, len);
     *atr_len = len;
-    return;
 }
 
 /*
@@ -1169,8 +1166,7 @@ vcard_emul_options(const char *args)
             NEXT_TOKEN(vname)
             NEXT_TOKEN(type_params)
             type_params_length = MIN(type_params_length, sizeof(type_str)-1);
-            strncpy(type_str, type_params, type_params_length);
-            type_str[type_params_length] = 0;
+            pstrcpy(type_str, type_params_length, type_params);
             type = vcard_emul_type_from_string(type_str);
 
             NEXT_TOKEN(type_params)
diff --git a/libcacard/vreader.c b/libcacard/vreader.c
index ec126dfa4..96d2407e7 100644
--- a/libcacard/vreader.c
+++ b/libcacard/vreader.c
@@ -93,7 +93,6 @@ vreader_free(VReader *reader)
         reader->reader_private_free(reader->reader_private);
     }
     g_free(reader);
-    return;
 }
 
 static VCard *
diff --git a/linux-headers/asm-powerpc/kvm_para.h b/linux-headers/asm-powerpc/kvm_para.h
index c047a8400..5e04383a1 100644
--- a/linux-headers/asm-powerpc/kvm_para.h
+++ b/linux-headers/asm-powerpc/kvm_para.h
@@ -17,8 +17,8 @@
  * Authors: Hollis Blanchard <hollisb@us.ibm.com>
  */
 
-#ifndef __POWERPC_KVM_PARA_H__
-#define __POWERPC_KVM_PARA_H__
+#ifndef _UAPI__POWERPC_KVM_PARA_H__
+#define _UAPI__POWERPC_KVM_PARA_H__
 
 #include <linux/types.h>
 
@@ -87,4 +87,4 @@ struct kvm_vcpu_arch_shared {
 #define KVM_MAGIC_FEAT_MAS0_TO_SPRG7	(1 << 1)
 
 
-#endif /* __POWERPC_KVM_PARA_H__ */
+#endif /* _UAPI__POWERPC_KVM_PARA_H__ */
diff --git a/linux-headers/asm-s390/kvm_para.h b/linux-headers/asm-s390/kvm_para.h
index 870051f64..ff1f4e7b3 100644
--- a/linux-headers/asm-s390/kvm_para.h
+++ b/linux-headers/asm-s390/kvm_para.h
@@ -1,5 +1,5 @@
 /*
- * definition for paravirtual devices on s390
+ * User API definitions for paravirtual devices on s390
  *
  * Copyright IBM Corp. 2008
  *
@@ -9,9 +9,3 @@
  *
  *    Author(s): Christian Borntraeger <borntraeger@de.ibm.com>
  */
-
-#ifndef __S390_KVM_PARA_H
-#define __S390_KVM_PARA_H
-
-
-#endif /* __S390_KVM_PARA_H */
diff --git a/linux-headers/asm-x86/kvm.h b/linux-headers/asm-x86/kvm.h
index 246617efd..a65ec29e6 100644
--- a/linux-headers/asm-x86/kvm.h
+++ b/linux-headers/asm-x86/kvm.h
@@ -9,6 +9,22 @@
 #include <linux/types.h>
 #include <linux/ioctl.h>
 
+#define DE_VECTOR 0
+#define DB_VECTOR 1
+#define BP_VECTOR 3
+#define OF_VECTOR 4
+#define BR_VECTOR 5
+#define UD_VECTOR 6
+#define NM_VECTOR 7
+#define DF_VECTOR 8
+#define TS_VECTOR 10
+#define NP_VECTOR 11
+#define SS_VECTOR 12
+#define GP_VECTOR 13
+#define PF_VECTOR 14
+#define MF_VECTOR 16
+#define MC_VECTOR 18
+
 /* Select x86 specific features in <linux/kvm.h> */
 #define __KVM_HAVE_PIT
 #define __KVM_HAVE_IOAPIC
@@ -25,6 +41,7 @@
 #define __KVM_HAVE_DEBUGREGS
 #define __KVM_HAVE_XSAVE
 #define __KVM_HAVE_XCRS
+#define __KVM_HAVE_READONLY_MEM
 
 /* Architectural interrupt line count. */
 #define KVM_NR_INTERRUPTS 256
diff --git a/linux-headers/linux/kvm.h b/linux-headers/linux/kvm.h
index 4b9e575dd..81d2feb7a 100644
--- a/linux-headers/linux/kvm.h
+++ b/linux-headers/linux/kvm.h
@@ -101,9 +101,13 @@ struct kvm_userspace_memory_region {
 	__u64 userspace_addr; /* start of the userspace allocated memory */
 };
 
-/* for kvm_memory_region::flags */
-#define KVM_MEM_LOG_DIRTY_PAGES  1UL
-#define KVM_MEMSLOT_INVALID      (1UL << 1)
+/*
+ * The bit 0 ~ bit 15 of kvm_memory_region::flags are visible for userspace,
+ * other bits are reserved for kvm internal use which are defined in
+ * include/linux/kvm_host.h.
+ */
+#define KVM_MEM_LOG_DIRTY_PAGES	(1UL << 0)
+#define KVM_MEM_READONLY	(1UL << 1)
 
 /* for KVM_IRQ_LINE */
 struct kvm_irq_level {
@@ -618,6 +622,10 @@ struct kvm_ppc_smmu_info {
 #define KVM_CAP_PPC_GET_SMMU_INFO 78
 #define KVM_CAP_S390_COW 79
 #define KVM_CAP_PPC_ALLOC_HTAB 80
+#ifdef __KVM_HAVE_READONLY_MEM
+#define KVM_CAP_READONLY_MEM 81
+#endif
+#define KVM_CAP_IRQFD_RESAMPLE 82
 
 #ifdef KVM_CAP_IRQ_ROUTING
 
@@ -683,12 +691,21 @@ struct kvm_xen_hvm_config {
 #endif
 
 #define KVM_IRQFD_FLAG_DEASSIGN (1 << 0)
+/*
+ * Available with KVM_CAP_IRQFD_RESAMPLE
+ *
+ * KVM_IRQFD_FLAG_RESAMPLE indicates resamplefd is valid and specifies
+ * the irqfd to operate in resampling mode for level triggered interrupt
+ * emlation.  See Documentation/virtual/kvm/api.txt.
+ */
+#define KVM_IRQFD_FLAG_RESAMPLE (1 << 1)
 
 struct kvm_irqfd {
 	__u32 fd;
 	__u32 gsi;
 	__u32 flags;
-	__u8  pad[20];
+	__u32 resamplefd;
+	__u8  pad[16];
 };
 
 struct kvm_clock_data {
diff --git a/linux-headers/linux/kvm_para.h b/linux-headers/linux/kvm_para.h
index 7bdcf93c1..cea2c5c72 100644
--- a/linux-headers/linux/kvm_para.h
+++ b/linux-headers/linux/kvm_para.h
@@ -1,5 +1,5 @@
-#ifndef __LINUX_KVM_PARA_H
-#define __LINUX_KVM_PARA_H
+#ifndef _UAPI__LINUX_KVM_PARA_H
+#define _UAPI__LINUX_KVM_PARA_H
 
 /*
  * This header file provides a method for making a hypercall to the host
@@ -25,4 +25,4 @@
  */
 #include <asm/kvm_para.h>
 
-#endif /* __LINUX_KVM_PARA_H */
+#endif /* _UAPI__LINUX_KVM_PARA_H */
diff --git a/linux-headers/linux/vfio.h b/linux-headers/linux/vfio.h
new file mode 100644
index 000000000..4758d1bfc
--- /dev/null
+++ b/linux-headers/linux/vfio.h
@@ -0,0 +1,368 @@
+/*
+ * VFIO API definition
+ *
+ * Copyright (C) 2012 Red Hat, Inc.  All rights reserved.
+ *     Author: Alex Williamson <alex.williamson@redhat.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+#ifndef _UAPIVFIO_H
+#define _UAPIVFIO_H
+
+#include <linux/types.h>
+#include <linux/ioctl.h>
+
+#define VFIO_API_VERSION	0
+
+
+/* Kernel & User level defines for VFIO IOCTLs. */
+
+/* Extensions */
+
+#define VFIO_TYPE1_IOMMU		1
+
+/*
+ * The IOCTL interface is designed for extensibility by embedding the
+ * structure length (argsz) and flags into structures passed between
+ * kernel and userspace.  We therefore use the _IO() macro for these
+ * defines to avoid implicitly embedding a size into the ioctl request.
+ * As structure fields are added, argsz will increase to match and flag
+ * bits will be defined to indicate additional fields with valid data.
+ * It's *always* the caller's responsibility to indicate the size of
+ * the structure passed by setting argsz appropriately.
+ */
+
+#define VFIO_TYPE	(';')
+#define VFIO_BASE	100
+
+/* -------- IOCTLs for VFIO file descriptor (/dev/vfio/vfio) -------- */
+
+/**
+ * VFIO_GET_API_VERSION - _IO(VFIO_TYPE, VFIO_BASE + 0)
+ *
+ * Report the version of the VFIO API.  This allows us to bump the entire
+ * API version should we later need to add or change features in incompatible
+ * ways.
+ * Return: VFIO_API_VERSION
+ * Availability: Always
+ */
+#define VFIO_GET_API_VERSION		_IO(VFIO_TYPE, VFIO_BASE + 0)
+
+/**
+ * VFIO_CHECK_EXTENSION - _IOW(VFIO_TYPE, VFIO_BASE + 1, __u32)
+ *
+ * Check whether an extension is supported.
+ * Return: 0 if not supported, 1 (or some other positive integer) if supported.
+ * Availability: Always
+ */
+#define VFIO_CHECK_EXTENSION		_IO(VFIO_TYPE, VFIO_BASE + 1)
+
+/**
+ * VFIO_SET_IOMMU - _IOW(VFIO_TYPE, VFIO_BASE + 2, __s32)
+ *
+ * Set the iommu to the given type.  The type must be supported by an
+ * iommu driver as verified by calling CHECK_EXTENSION using the same
+ * type.  A group must be set to this file descriptor before this
+ * ioctl is available.  The IOMMU interfaces enabled by this call are
+ * specific to the value set.
+ * Return: 0 on success, -errno on failure
+ * Availability: When VFIO group attached
+ */
+#define VFIO_SET_IOMMU			_IO(VFIO_TYPE, VFIO_BASE + 2)
+
+/* -------- IOCTLs for GROUP file descriptors (/dev/vfio/$GROUP) -------- */
+
+/**
+ * VFIO_GROUP_GET_STATUS - _IOR(VFIO_TYPE, VFIO_BASE + 3,
+ *						struct vfio_group_status)
+ *
+ * Retrieve information about the group.  Fills in provided
+ * struct vfio_group_info.  Caller sets argsz.
+ * Return: 0 on succes, -errno on failure.
+ * Availability: Always
+ */
+struct vfio_group_status {
+	__u32	argsz;
+	__u32	flags;
+#define VFIO_GROUP_FLAGS_VIABLE		(1 << 0)
+#define VFIO_GROUP_FLAGS_CONTAINER_SET	(1 << 1)
+};
+#define VFIO_GROUP_GET_STATUS		_IO(VFIO_TYPE, VFIO_BASE + 3)
+
+/**
+ * VFIO_GROUP_SET_CONTAINER - _IOW(VFIO_TYPE, VFIO_BASE + 4, __s32)
+ *
+ * Set the container for the VFIO group to the open VFIO file
+ * descriptor provided.  Groups may only belong to a single
+ * container.  Containers may, at their discretion, support multiple
+ * groups.  Only when a container is set are all of the interfaces
+ * of the VFIO file descriptor and the VFIO group file descriptor
+ * available to the user.
+ * Return: 0 on success, -errno on failure.
+ * Availability: Always
+ */
+#define VFIO_GROUP_SET_CONTAINER	_IO(VFIO_TYPE, VFIO_BASE + 4)
+
+/**
+ * VFIO_GROUP_UNSET_CONTAINER - _IO(VFIO_TYPE, VFIO_BASE + 5)
+ *
+ * Remove the group from the attached container.  This is the
+ * opposite of the SET_CONTAINER call and returns the group to
+ * an initial state.  All device file descriptors must be released
+ * prior to calling this interface.  When removing the last group
+ * from a container, the IOMMU will be disabled and all state lost,
+ * effectively also returning the VFIO file descriptor to an initial
+ * state.
+ * Return: 0 on success, -errno on failure.
+ * Availability: When attached to container
+ */
+#define VFIO_GROUP_UNSET_CONTAINER	_IO(VFIO_TYPE, VFIO_BASE + 5)
+
+/**
+ * VFIO_GROUP_GET_DEVICE_FD - _IOW(VFIO_TYPE, VFIO_BASE + 6, char)
+ *
+ * Return a new file descriptor for the device object described by
+ * the provided string.  The string should match a device listed in
+ * the devices subdirectory of the IOMMU group sysfs entry.  The
+ * group containing the device must already be added to this context.
+ * Return: new file descriptor on success, -errno on failure.
+ * Availability: When attached to container
+ */
+#define VFIO_GROUP_GET_DEVICE_FD	_IO(VFIO_TYPE, VFIO_BASE + 6)
+
+/* --------------- IOCTLs for DEVICE file descriptors --------------- */
+
+/**
+ * VFIO_DEVICE_GET_INFO - _IOR(VFIO_TYPE, VFIO_BASE + 7,
+ *						struct vfio_device_info)
+ *
+ * Retrieve information about the device.  Fills in provided
+ * struct vfio_device_info.  Caller sets argsz.
+ * Return: 0 on success, -errno on failure.
+ */
+struct vfio_device_info {
+	__u32	argsz;
+	__u32	flags;
+#define VFIO_DEVICE_FLAGS_RESET	(1 << 0)	/* Device supports reset */
+#define VFIO_DEVICE_FLAGS_PCI	(1 << 1)	/* vfio-pci device */
+	__u32	num_regions;	/* Max region index + 1 */
+	__u32	num_irqs;	/* Max IRQ index + 1 */
+};
+#define VFIO_DEVICE_GET_INFO		_IO(VFIO_TYPE, VFIO_BASE + 7)
+
+/**
+ * VFIO_DEVICE_GET_REGION_INFO - _IOWR(VFIO_TYPE, VFIO_BASE + 8,
+ *				       struct vfio_region_info)
+ *
+ * Retrieve information about a device region.  Caller provides
+ * struct vfio_region_info with index value set.  Caller sets argsz.
+ * Implementation of region mapping is bus driver specific.  This is
+ * intended to describe MMIO, I/O port, as well as bus specific
+ * regions (ex. PCI config space).  Zero sized regions may be used
+ * to describe unimplemented regions (ex. unimplemented PCI BARs).
+ * Return: 0 on success, -errno on failure.
+ */
+struct vfio_region_info {
+	__u32	argsz;
+	__u32	flags;
+#define VFIO_REGION_INFO_FLAG_READ	(1 << 0) /* Region supports read */
+#define VFIO_REGION_INFO_FLAG_WRITE	(1 << 1) /* Region supports write */
+#define VFIO_REGION_INFO_FLAG_MMAP	(1 << 2) /* Region supports mmap */
+	__u32	index;		/* Region index */
+	__u32	resv;		/* Reserved for alignment */
+	__u64	size;		/* Region size (bytes) */
+	__u64	offset;		/* Region offset from start of device fd */
+};
+#define VFIO_DEVICE_GET_REGION_INFO	_IO(VFIO_TYPE, VFIO_BASE + 8)
+
+/**
+ * VFIO_DEVICE_GET_IRQ_INFO - _IOWR(VFIO_TYPE, VFIO_BASE + 9,
+ *				    struct vfio_irq_info)
+ *
+ * Retrieve information about a device IRQ.  Caller provides
+ * struct vfio_irq_info with index value set.  Caller sets argsz.
+ * Implementation of IRQ mapping is bus driver specific.  Indexes
+ * using multiple IRQs are primarily intended to support MSI-like
+ * interrupt blocks.  Zero count irq blocks may be used to describe
+ * unimplemented interrupt types.
+ *
+ * The EVENTFD flag indicates the interrupt index supports eventfd based
+ * signaling.
+ *
+ * The MASKABLE flags indicates the index supports MASK and UNMASK
+ * actions described below.
+ *
+ * AUTOMASKED indicates that after signaling, the interrupt line is
+ * automatically masked by VFIO and the user needs to unmask the line
+ * to receive new interrupts.  This is primarily intended to distinguish
+ * level triggered interrupts.
+ *
+ * The NORESIZE flag indicates that the interrupt lines within the index
+ * are setup as a set and new subindexes cannot be enabled without first
+ * disabling the entire index.  This is used for interrupts like PCI MSI
+ * and MSI-X where the driver may only use a subset of the available
+ * indexes, but VFIO needs to enable a specific number of vectors
+ * upfront.  In the case of MSI-X, where the user can enable MSI-X and
+ * then add and unmask vectors, it's up to userspace to make the decision
+ * whether to allocate the maximum supported number of vectors or tear
+ * down setup and incrementally increase the vectors as each is enabled.
+ */
+struct vfio_irq_info {
+	__u32	argsz;
+	__u32	flags;
+#define VFIO_IRQ_INFO_EVENTFD		(1 << 0)
+#define VFIO_IRQ_INFO_MASKABLE		(1 << 1)
+#define VFIO_IRQ_INFO_AUTOMASKED	(1 << 2)
+#define VFIO_IRQ_INFO_NORESIZE		(1 << 3)
+	__u32	index;		/* IRQ index */
+	__u32	count;		/* Number of IRQs within this index */
+};
+#define VFIO_DEVICE_GET_IRQ_INFO	_IO(VFIO_TYPE, VFIO_BASE + 9)
+
+/**
+ * VFIO_DEVICE_SET_IRQS - _IOW(VFIO_TYPE, VFIO_BASE + 10, struct vfio_irq_set)
+ *
+ * Set signaling, masking, and unmasking of interrupts.  Caller provides
+ * struct vfio_irq_set with all fields set.  'start' and 'count' indicate
+ * the range of subindexes being specified.
+ *
+ * The DATA flags specify the type of data provided.  If DATA_NONE, the
+ * operation performs the specified action immediately on the specified
+ * interrupt(s).  For example, to unmask AUTOMASKED interrupt [0,0]:
+ * flags = (DATA_NONE|ACTION_UNMASK), index = 0, start = 0, count = 1.
+ *
+ * DATA_BOOL allows sparse support for the same on arrays of interrupts.
+ * For example, to mask interrupts [0,1] and [0,3] (but not [0,2]):
+ * flags = (DATA_BOOL|ACTION_MASK), index = 0, start = 1, count = 3,
+ * data = {1,0,1}
+ *
+ * DATA_EVENTFD binds the specified ACTION to the provided __s32 eventfd.
+ * A value of -1 can be used to either de-assign interrupts if already
+ * assigned or skip un-assigned interrupts.  For example, to set an eventfd
+ * to be trigger for interrupts [0,0] and [0,2]:
+ * flags = (DATA_EVENTFD|ACTION_TRIGGER), index = 0, start = 0, count = 3,
+ * data = {fd1, -1, fd2}
+ * If index [0,1] is previously set, two count = 1 ioctls calls would be
+ * required to set [0,0] and [0,2] without changing [0,1].
+ *
+ * Once a signaling mechanism is set, DATA_BOOL or DATA_NONE can be used
+ * with ACTION_TRIGGER to perform kernel level interrupt loopback testing
+ * from userspace (ie. simulate hardware triggering).
+ *
+ * Setting of an event triggering mechanism to userspace for ACTION_TRIGGER
+ * enables the interrupt index for the device.  Individual subindex interrupts
+ * can be disabled using the -1 value for DATA_EVENTFD or the index can be
+ * disabled as a whole with: flags = (DATA_NONE|ACTION_TRIGGER), count = 0.
+ *
+ * Note that ACTION_[UN]MASK specify user->kernel signaling (irqfds) while
+ * ACTION_TRIGGER specifies kernel->user signaling.
+ */
+struct vfio_irq_set {
+	__u32	argsz;
+	__u32	flags;
+#define VFIO_IRQ_SET_DATA_NONE		(1 << 0) /* Data not present */
+#define VFIO_IRQ_SET_DATA_BOOL		(1 << 1) /* Data is bool (u8) */
+#define VFIO_IRQ_SET_DATA_EVENTFD	(1 << 2) /* Data is eventfd (s32) */
+#define VFIO_IRQ_SET_ACTION_MASK	(1 << 3) /* Mask interrupt */
+#define VFIO_IRQ_SET_ACTION_UNMASK	(1 << 4) /* Unmask interrupt */
+#define VFIO_IRQ_SET_ACTION_TRIGGER	(1 << 5) /* Trigger interrupt */
+	__u32	index;
+	__u32	start;
+	__u32	count;
+	__u8	data[];
+};
+#define VFIO_DEVICE_SET_IRQS		_IO(VFIO_TYPE, VFIO_BASE + 10)
+
+#define VFIO_IRQ_SET_DATA_TYPE_MASK	(VFIO_IRQ_SET_DATA_NONE | \
+					 VFIO_IRQ_SET_DATA_BOOL | \
+					 VFIO_IRQ_SET_DATA_EVENTFD)
+#define VFIO_IRQ_SET_ACTION_TYPE_MASK	(VFIO_IRQ_SET_ACTION_MASK | \
+					 VFIO_IRQ_SET_ACTION_UNMASK | \
+					 VFIO_IRQ_SET_ACTION_TRIGGER)
+/**
+ * VFIO_DEVICE_RESET - _IO(VFIO_TYPE, VFIO_BASE + 11)
+ *
+ * Reset a device.
+ */
+#define VFIO_DEVICE_RESET		_IO(VFIO_TYPE, VFIO_BASE + 11)
+
+/*
+ * The VFIO-PCI bus driver makes use of the following fixed region and
+ * IRQ index mapping.  Unimplemented regions return a size of zero.
+ * Unimplemented IRQ types return a count of zero.
+ */
+
+enum {
+	VFIO_PCI_BAR0_REGION_INDEX,
+	VFIO_PCI_BAR1_REGION_INDEX,
+	VFIO_PCI_BAR2_REGION_INDEX,
+	VFIO_PCI_BAR3_REGION_INDEX,
+	VFIO_PCI_BAR4_REGION_INDEX,
+	VFIO_PCI_BAR5_REGION_INDEX,
+	VFIO_PCI_ROM_REGION_INDEX,
+	VFIO_PCI_CONFIG_REGION_INDEX,
+	VFIO_PCI_NUM_REGIONS
+};
+
+enum {
+	VFIO_PCI_INTX_IRQ_INDEX,
+	VFIO_PCI_MSI_IRQ_INDEX,
+	VFIO_PCI_MSIX_IRQ_INDEX,
+	VFIO_PCI_NUM_IRQS
+};
+
+/* -------- API for Type1 VFIO IOMMU -------- */
+
+/**
+ * VFIO_IOMMU_GET_INFO - _IOR(VFIO_TYPE, VFIO_BASE + 12, struct vfio_iommu_info)
+ *
+ * Retrieve information about the IOMMU object. Fills in provided
+ * struct vfio_iommu_info. Caller sets argsz.
+ *
+ * XXX Should we do these by CHECK_EXTENSION too?
+ */
+struct vfio_iommu_type1_info {
+	__u32	argsz;
+	__u32	flags;
+#define VFIO_IOMMU_INFO_PGSIZES (1 << 0)	/* supported page sizes info */
+	__u64	iova_pgsizes;		/* Bitmap of supported page sizes */
+};
+
+#define VFIO_IOMMU_GET_INFO _IO(VFIO_TYPE, VFIO_BASE + 12)
+
+/**
+ * VFIO_IOMMU_MAP_DMA - _IOW(VFIO_TYPE, VFIO_BASE + 13, struct vfio_dma_map)
+ *
+ * Map process virtual addresses to IO virtual addresses using the
+ * provided struct vfio_dma_map. Caller sets argsz. READ &/ WRITE required.
+ */
+struct vfio_iommu_type1_dma_map {
+	__u32	argsz;
+	__u32	flags;
+#define VFIO_DMA_MAP_FLAG_READ (1 << 0)		/* readable from device */
+#define VFIO_DMA_MAP_FLAG_WRITE (1 << 1)	/* writable from device */
+	__u64	vaddr;				/* Process virtual address */
+	__u64	iova;				/* IO virtual address */
+	__u64	size;				/* Size of mapping (bytes) */
+};
+
+#define VFIO_IOMMU_MAP_DMA _IO(VFIO_TYPE, VFIO_BASE + 13)
+
+/**
+ * VFIO_IOMMU_UNMAP_DMA - _IOW(VFIO_TYPE, VFIO_BASE + 14, struct vfio_dma_unmap)
+ *
+ * Unmap IO virtual addresses using the provided struct vfio_dma_unmap.
+ * Caller sets argsz.
+ */
+struct vfio_iommu_type1_dma_unmap {
+	__u32	argsz;
+	__u32	flags;
+	__u64	iova;				/* IO virtual address */
+	__u64	size;				/* Size of mapping (bytes) */
+};
+
+#define VFIO_IOMMU_UNMAP_DMA _IO(VFIO_TYPE, VFIO_BASE + 14)
+
+#endif /* _UAPIVFIO_H */
diff --git a/linux-headers/linux/virtio_config.h b/linux-headers/linux/virtio_config.h
index 4f51d8f3a..b7cda390f 100644
--- a/linux-headers/linux/virtio_config.h
+++ b/linux-headers/linux/virtio_config.h
@@ -1,5 +1,5 @@
-#ifndef _LINUX_VIRTIO_CONFIG_H
-#define _LINUX_VIRTIO_CONFIG_H
+#ifndef _UAPI_LINUX_VIRTIO_CONFIG_H
+#define _UAPI_LINUX_VIRTIO_CONFIG_H
 /* This header, excluding the #ifdef __KERNEL__ part, is BSD licensed so
  * anyone can use the definitions to implement compatible drivers/servers.
  *
@@ -51,4 +51,4 @@
  * suppressed them? */
 #define VIRTIO_F_NOTIFY_ON_EMPTY	24
 
-#endif /* _LINUX_VIRTIO_CONFIG_H */
+#endif /* _UAPI_LINUX_VIRTIO_CONFIG_H */
diff --git a/linux-headers/linux/virtio_ring.h b/linux-headers/linux/virtio_ring.h
index 1b333e253..921694a08 100644
--- a/linux-headers/linux/virtio_ring.h
+++ b/linux-headers/linux/virtio_ring.h
@@ -1,5 +1,5 @@
-#ifndef _LINUX_VIRTIO_RING_H
-#define _LINUX_VIRTIO_RING_H
+#ifndef _UAPI_LINUX_VIRTIO_RING_H
+#define _UAPI_LINUX_VIRTIO_RING_H
 /* An interface for efficient virtio implementation, currently for use by KVM
  * and lguest, but hopefully others soon.  Do NOT change this since it will
  * break existing servers and clients.
@@ -160,4 +160,4 @@ static __inline__ int vring_need_event(__u16 event_idx, __u16 new_idx, __u16 old
 	return (__u16)(new_idx - event_idx - 1) < (__u16)(new_idx - old);
 }
 
-#endif /* _LINUX_VIRTIO_RING_H */
+#endif /* _UAPI_LINUX_VIRTIO_RING_H */
diff --git a/linux-user/alpha/target_signal.h b/linux-user/alpha/target_signal.h
index 94f15f612..d3822da60 100644
--- a/linux-user/alpha/target_signal.h
+++ b/linux-user/alpha/target_signal.h
@@ -6,9 +6,10 @@
 /* this struct defines a stack used during syscall handling */
 
 typedef struct target_sigaltstack {
-	abi_ulong ss_sp;
-	abi_long ss_flags;
-	abi_ulong ss_size;
+    abi_ulong ss_sp;
+    int32_t ss_flags;
+    int32_t dummy;
+    abi_ulong ss_size;
 } target_stack_t;
 
 
diff --git a/linux-user/elfload.c b/linux-user/elfload.c
index 819fdd515..1d8bcb4e7 100644
--- a/linux-user/elfload.c
+++ b/linux-user/elfload.c
@@ -2442,7 +2442,7 @@ static void fill_prstatus(struct target_elf_prstatus *prstatus,
 
 static int fill_psinfo(struct target_elf_prpsinfo *psinfo, const TaskState *ts)
 {
-    char *filename, *base_filename;
+    char *base_filename;
     unsigned int i, len;
 
     (void) memset(psinfo, 0, sizeof (*psinfo));
@@ -2464,13 +2464,15 @@ static int fill_psinfo(struct target_elf_prpsinfo *psinfo, const TaskState *ts)
     psinfo->pr_uid = getuid();
     psinfo->pr_gid = getgid();
 
-    filename = strdup(ts->bprm->filename);
-    base_filename = strdup(basename(filename));
+    base_filename = g_path_get_basename(ts->bprm->filename);
+    /*
+     * Using strncpy here is fine: at max-length,
+     * this field is not NUL-terminated.
+     */
     (void) strncpy(psinfo->pr_fname, base_filename,
                    sizeof(psinfo->pr_fname));
-    free(base_filename);
-    free(filename);
 
+    g_free(base_filename);
     bswap_psinfo(psinfo);
     return (0);
 }
diff --git a/linux-user/linuxload.c b/linux-user/linuxload.c
index b47025f08..381ab8914 100644
--- a/linux-user/linuxload.c
+++ b/linux-user/linuxload.c
@@ -140,8 +140,9 @@ int loader_exec(const char * filename, char ** argv, char ** envp,
     bprm->p = TARGET_PAGE_SIZE*MAX_ARG_PAGES-sizeof(unsigned int);
     memset(bprm->page, 0, sizeof(bprm->page));
     retval = open(filename, O_RDONLY);
-    if (retval < 0)
-        return retval;
+    if (retval < 0) {
+        return -errno;
+    }
     bprm->fd = retval;
     bprm->filename = (char *)filename;
     bprm->argc = count(argv);
@@ -165,8 +166,7 @@ int loader_exec(const char * filename, char ** argv, char ** envp,
             retval = load_flt_binary(bprm,regs,infop);
 #endif
         } else {
-            fprintf(stderr, "Unknown binary format\n");
-            return -1;
+            return -ENOEXEC;
         }
     }
 
diff --git a/linux-user/main.c b/linux-user/main.c
index 1a1c661ee..25e35cd3d 100644
--- a/linux-user/main.c
+++ b/linux-user/main.c
@@ -89,19 +89,6 @@ int cpu_get_pic_interrupt(CPUX86State *env)
 }
 #endif
 
-/* timers for rdtsc */
-
-#if 0
-
-static uint64_t emu_time;
-
-int64_t cpu_get_real_ticks(void)
-{
-    return emu_time++;
-}
-
-#endif
-
 #if defined(CONFIG_USE_NPTL)
 /***********************************************************/
 /* Helper routines for implementing atomic operations.  */
@@ -1127,6 +1114,11 @@ void cpu_loop (CPUSPARCState *env)
     while (1) {
         trapnr = cpu_sparc_exec (env);
 
+        /* Compute PSR before exposing state.  */
+        if (env->cc_op != CC_OP_FLAGS) {
+            cpu_get_psr(env);
+        }
+
         switch (trapnr) {
 #ifndef TARGET_SPARC64
         case 0x88:
@@ -2294,6 +2286,12 @@ done_syscall:
                 queue_signal(env, info.si_signo, &info);
             }
             break;
+        case EXCP_DSPDIS:
+            info.si_signo = TARGET_SIGILL;
+            info.si_errno = 0;
+            info.si_code = TARGET_ILL_ILLOPC;
+            queue_signal(env, info.si_signo, &info);
+            break;
         default:
             //        error:
             fprintf(stderr, "qemu: unhandled CPU exception 0x%x - aborting\n",
@@ -2535,6 +2533,7 @@ void cpu_loop(CPUMBState *env)
         case EXCP_BREAK:
             /* Return address is 4 bytes after the call.  */
             env->regs[14] += 4;
+            env->sregs[SR_PC] = env->regs[14];
             ret = do_syscall(env, 
                              env->regs[12], 
                              env->regs[5], 
@@ -2545,7 +2544,6 @@ void cpu_loop(CPUMBState *env)
                              env->regs[10],
                              0, 0);
             env->regs[3] = ret;
-            env->sregs[SR_PC] = env->regs[14];
             break;
         case EXCP_HW_EXCP:
             env->regs[17] = env->sregs[SR_PC] + 4;
@@ -3143,10 +3141,8 @@ static void handle_arg_cpu(const char *arg)
     cpu_model = strdup(arg);
     if (cpu_model == NULL || is_help_option(cpu_model)) {
         /* XXX: implement xxx_cpu_list for targets that still miss it */
-#if defined(cpu_list_id)
-        cpu_list_id(stdout, &fprintf, "");
-#elif defined(cpu_list)
-        cpu_list(stdout, &fprintf); /* deprecated */
+#if defined(cpu_list)
+        cpu_list(stdout, &fprintf);
 #endif
         exit(1);
     }
@@ -3584,7 +3580,7 @@ int main(int argc, char **argv, char **envp)
     ret = loader_exec(filename, target_argv, target_environ, regs,
         info, &bprm);
     if (ret != 0) {
-        printf("Error %d while loading %s\n", ret, filename);
+        printf("Error while loading %s: %s\n", filename, strerror(-ret));
         _exit(1);
     }
 
diff --git a/linux-user/qemu.h b/linux-user/qemu.h
index 69b27d714..5e53dca09 100644
--- a/linux-user/qemu.h
+++ b/linux-user/qemu.h
@@ -219,6 +219,9 @@ unsigned long init_guest_space(unsigned long host_start,
 
 #include "qemu-log.h"
 
+/* syscall.c */
+int host_to_target_waitstatus(int status);
+
 /* strace.c */
 void print_syscall(int num,
                    abi_long arg1, abi_long arg2, abi_long arg3,
@@ -289,46 +292,29 @@ static inline int access_ok(int type, abi_ulong addr, abi_ulong size)
  * struct has been locked - usually with lock_user_struct().
  */
 #define __put_user(x, hptr)\
-({\
+({ __typeof(*hptr) pu_ = (x);\
     switch(sizeof(*hptr)) {\
-    case 1:\
-        *(uint8_t *)(hptr) = (uint8_t)(typeof(*hptr))(x);\
-        break;\
-    case 2:\
-        *(uint16_t *)(hptr) = tswap16((uint16_t)(typeof(*hptr))(x));\
-        break;\
-    case 4:\
-        *(uint32_t *)(hptr) = tswap32((uint32_t)(typeof(*hptr))(x));\
-        break;\
-    case 8:\
-        *(uint64_t *)(hptr) = tswap64((typeof(*hptr))(x));\
-        break;\
-    default:\
-        abort();\
+    case 1: break;\
+    case 2: pu_ = tswap16(pu_); break; \
+    case 4: pu_ = tswap32(pu_); break; \
+    case 8: pu_ = tswap64(pu_); break; \
+    default: abort();\
     }\
+    memcpy(hptr, &pu_, sizeof(pu_)); \
     0;\
 })
 
 #define __get_user(x, hptr) \
-({\
+({ __typeof(*hptr) gu_; \
+    memcpy(&gu_, hptr, sizeof(gu_)); \
     switch(sizeof(*hptr)) {\
-    case 1:\
-        x = (typeof(*hptr))*(uint8_t *)(hptr);\
-        break;\
-    case 2:\
-        x = (typeof(*hptr))tswap16(*(uint16_t *)(hptr));\
-        break;\
-    case 4:\
-        x = (typeof(*hptr))tswap32(*(uint32_t *)(hptr));\
-        break;\
-    case 8:\
-        x = (typeof(*hptr))tswap64(*(uint64_t *)(hptr));\
-        break;\
-    default:\
-        /* avoid warning */\
-        x = 0;\
-        abort();\
+    case 1: break; \
+    case 2: gu_ = tswap16(gu_); break; \
+    case 4: gu_ = tswap32(gu_); break; \
+    case 8: gu_ = tswap64(gu_); break; \
+    default: abort();\
     }\
+    (x) = gu_; \
     0;\
 })
 
diff --git a/linux-user/signal.c b/linux-user/signal.c
index 78691473f..95e2ffa00 100644
--- a/linux-user/signal.c
+++ b/linux-user/signal.c
@@ -202,46 +202,67 @@ void target_to_host_old_sigset(sigset_t *sigset,
 static inline void host_to_target_siginfo_noswap(target_siginfo_t *tinfo,
                                                  const siginfo_t *info)
 {
-    int sig;
-    sig = host_to_target_signal(info->si_signo);
+    int sig = host_to_target_signal(info->si_signo);
     tinfo->si_signo = sig;
     tinfo->si_errno = 0;
     tinfo->si_code = info->si_code;
-    if (sig == SIGILL || sig == SIGFPE || sig == SIGSEGV ||
-        sig == SIGBUS || sig == SIGTRAP) {
-        /* should never come here, but who knows. The information for
-           the target is irrelevant */
+
+    if (sig == TARGET_SIGILL || sig == TARGET_SIGFPE || sig == TARGET_SIGSEGV
+        || sig == TARGET_SIGBUS || sig == TARGET_SIGTRAP) {
+        /* Should never come here, but who knows. The information for
+           the target is irrelevant.  */
         tinfo->_sifields._sigfault._addr = 0;
-    } else if (sig == SIGIO) {
+    } else if (sig == TARGET_SIGIO) {
+        tinfo->_sifields._sigpoll._band = info->si_band;
 	tinfo->_sifields._sigpoll._fd = info->si_fd;
+    } else if (sig == TARGET_SIGCHLD) {
+        tinfo->_sifields._sigchld._pid = info->si_pid;
+        tinfo->_sifields._sigchld._uid = info->si_uid;
+        tinfo->_sifields._sigchld._status
+            = host_to_target_waitstatus(info->si_status);
+        tinfo->_sifields._sigchld._utime = info->si_utime;
+        tinfo->_sifields._sigchld._stime = info->si_stime;
     } else if (sig >= TARGET_SIGRTMIN) {
         tinfo->_sifields._rt._pid = info->si_pid;
         tinfo->_sifields._rt._uid = info->si_uid;
         /* XXX: potential problem if 64 bit */
-        tinfo->_sifields._rt._sigval.sival_ptr =
-            (abi_ulong)(unsigned long)info->si_value.sival_ptr;
+        tinfo->_sifields._rt._sigval.sival_ptr
+            = (abi_ulong)(unsigned long)info->si_value.sival_ptr;
     }
 }
 
 static void tswap_siginfo(target_siginfo_t *tinfo,
                           const target_siginfo_t *info)
 {
-    int sig;
-    sig = info->si_signo;
+    int sig = info->si_signo;
     tinfo->si_signo = tswap32(sig);
     tinfo->si_errno = tswap32(info->si_errno);
     tinfo->si_code = tswap32(info->si_code);
-    if (sig == SIGILL || sig == SIGFPE || sig == SIGSEGV ||
-        sig == SIGBUS || sig == SIGTRAP) {
-        tinfo->_sifields._sigfault._addr =
-            tswapal(info->_sifields._sigfault._addr);
-    } else if (sig == SIGIO) {
-	tinfo->_sifields._sigpoll._fd = tswap32(info->_sifields._sigpoll._fd);
+
+    if (sig == TARGET_SIGILL || sig == TARGET_SIGFPE || sig == TARGET_SIGSEGV
+        || sig == TARGET_SIGBUS || sig == TARGET_SIGTRAP) {
+        tinfo->_sifields._sigfault._addr
+            = tswapal(info->_sifields._sigfault._addr);
+    } else if (sig == TARGET_SIGIO) {
+        tinfo->_sifields._sigpoll._band
+            = tswap32(info->_sifields._sigpoll._band);
+        tinfo->_sifields._sigpoll._fd = tswap32(info->_sifields._sigpoll._fd);
+    } else if (sig == TARGET_SIGCHLD) {
+        tinfo->_sifields._sigchld._pid
+            = tswap32(info->_sifields._sigchld._pid);
+        tinfo->_sifields._sigchld._uid
+            = tswap32(info->_sifields._sigchld._uid);
+        tinfo->_sifields._sigchld._status
+            = tswap32(info->_sifields._sigchld._status);
+        tinfo->_sifields._sigchld._utime
+            = tswapal(info->_sifields._sigchld._utime);
+        tinfo->_sifields._sigchld._stime
+            = tswapal(info->_sifields._sigchld._stime);
     } else if (sig >= TARGET_SIGRTMIN) {
         tinfo->_sifields._rt._pid = tswap32(info->_sifields._rt._pid);
         tinfo->_sifields._rt._uid = tswap32(info->_sifields._rt._uid);
-        tinfo->_sifields._rt._sigval.sival_ptr =
-            tswapal(info->_sifields._rt._sigval.sival_ptr);
+        tinfo->_sifields._rt._sigval.sival_ptr
+            = tswapal(info->_sifields._rt._sigval.sival_ptr);
     }
 }
 
@@ -2762,7 +2783,6 @@ static void setup_frame(int sig, struct target_sigaction * ka,
 give_sigsegv:
     unlock_user_struct(frame, frame_addr, 1);
     force_sig(TARGET_SIGSEGV/*, current*/);
-    return;
 }
 
 long do_sigreturn(CPUMIPSState *regs)
@@ -2871,7 +2891,6 @@ static void setup_rt_frame(int sig, struct target_sigaction *ka,
 give_sigsegv:
     unlock_user_struct(frame, frame_addr, 1);
     force_sig(TARGET_SIGSEGV/*, current*/);
-    return;
 }
 
 long do_rt_sigreturn(CPUMIPSState *env)
diff --git a/linux-user/syscall.c b/linux-user/syscall.c
index 6257a04d0..e4291ed77 100644
--- a/linux-user/syscall.c
+++ b/linux-user/syscall.c
@@ -587,12 +587,17 @@ extern int setfsgid(int);
 extern int setgroups(int, gid_t *);
 
 /* ARM EABI and MIPS expect 64bit types aligned even on pairs or registers */
-#ifdef TARGET_ARM 
+#ifdef TARGET_ARM
 static inline int regpairs_aligned(void *cpu_env) {
     return ((((CPUARMState *)cpu_env)->eabi) == 1) ;
 }
 #elif defined(TARGET_MIPS)
 static inline int regpairs_aligned(void *cpu_env) { return 1; }
+#elif defined(TARGET_PPC) && !defined(TARGET_PPC64)
+/* SysV AVI for PPC32 expects 64bit parameters to be passed on odd/even pairs
+ * of registers which translates to the same as ARM/MIPS, because we start with
+ * r3 as arg1 */
+static inline int regpairs_aligned(void *cpu_env) { return 1; }
 #else
 static inline int regpairs_aligned(void *cpu_env) { return 0; }
 #endif
@@ -1744,55 +1749,96 @@ static abi_long do_getsockopt(int sockfd, int level, int optname,
     return ret;
 }
 
-/* FIXME
- * lock_iovec()/unlock_iovec() have a return code of 0 for success where
- * other lock functions have a return code of 0 for failure.
- */
-static abi_long lock_iovec(int type, struct iovec *vec, abi_ulong target_addr,
-                           int count, int copy)
+static struct iovec *lock_iovec(int type, abi_ulong target_addr,
+                                int count, int copy)
 {
     struct target_iovec *target_vec;
-    abi_ulong base;
+    struct iovec *vec;
+    abi_ulong total_len, max_len;
     int i;
 
-    target_vec = lock_user(VERIFY_READ, target_addr, count * sizeof(struct target_iovec), 1);
-    if (!target_vec)
-        return -TARGET_EFAULT;
-    for(i = 0;i < count; i++) {
-        base = tswapal(target_vec[i].iov_base);
-        vec[i].iov_len = tswapal(target_vec[i].iov_len);
-        if (vec[i].iov_len != 0) {
-            vec[i].iov_base = lock_user(type, base, vec[i].iov_len, copy);
-            /* Don't check lock_user return value. We must call writev even
-               if a element has invalid base address. */
+    if (count == 0) {
+        errno = 0;
+        return NULL;
+    }
+    if (count > IOV_MAX) {
+        errno = EINVAL;
+        return NULL;
+    }
+
+    vec = calloc(count, sizeof(struct iovec));
+    if (vec == NULL) {
+        errno = ENOMEM;
+        return NULL;
+    }
+
+    target_vec = lock_user(VERIFY_READ, target_addr,
+                           count * sizeof(struct target_iovec), 1);
+    if (target_vec == NULL) {
+        errno = EFAULT;
+        goto fail2;
+    }
+
+    /* ??? If host page size > target page size, this will result in a
+       value larger than what we can actually support.  */
+    max_len = 0x7fffffff & TARGET_PAGE_MASK;
+    total_len = 0;
+
+    for (i = 0; i < count; i++) {
+        abi_ulong base = tswapal(target_vec[i].iov_base);
+        abi_long len = tswapal(target_vec[i].iov_len);
+
+        if (len < 0) {
+            errno = EINVAL;
+            goto fail;
+        } else if (len == 0) {
+            /* Zero length pointer is ignored.  */
+            vec[i].iov_base = 0;
         } else {
-            /* zero length pointer is ignored */
-            vec[i].iov_base = NULL;
+            vec[i].iov_base = lock_user(type, base, len, copy);
+            if (!vec[i].iov_base) {
+                errno = EFAULT;
+                goto fail;
+            }
+            if (len > max_len - total_len) {
+                len = max_len - total_len;
+            }
         }
+        vec[i].iov_len = len;
+        total_len += len;
     }
-    unlock_user (target_vec, target_addr, 0);
-    return 0;
+
+    unlock_user(target_vec, target_addr, 0);
+    return vec;
+
+ fail:
+    free(vec);
+ fail2:
+    unlock_user(target_vec, target_addr, 0);
+    return NULL;
 }
 
-static abi_long unlock_iovec(struct iovec *vec, abi_ulong target_addr,
-                             int count, int copy)
+static void unlock_iovec(struct iovec *vec, abi_ulong target_addr,
+                         int count, int copy)
 {
     struct target_iovec *target_vec;
-    abi_ulong base;
     int i;
 
-    target_vec = lock_user(VERIFY_READ, target_addr, count * sizeof(struct target_iovec), 1);
-    if (!target_vec)
-        return -TARGET_EFAULT;
-    for(i = 0;i < count; i++) {
-        if (target_vec[i].iov_base) {
-            base = tswapal(target_vec[i].iov_base);
+    target_vec = lock_user(VERIFY_READ, target_addr,
+                           count * sizeof(struct target_iovec), 1);
+    if (target_vec) {
+        for (i = 0; i < count; i++) {
+            abi_ulong base = tswapal(target_vec[i].iov_base);
+            abi_long len = tswapal(target_vec[i].iov_base);
+            if (len < 0) {
+                break;
+            }
             unlock_user(vec[i].iov_base, base, copy ? vec[i].iov_len : 0);
         }
+        unlock_user(target_vec, target_addr, 0);
     }
-    unlock_user (target_vec, target_addr, 0);
 
-    return 0;
+    free(vec);
 }
 
 /* do_socket() Must return target values and target errnos. */
@@ -1888,8 +1934,7 @@ static abi_long do_sendrecvmsg(int fd, abi_ulong target_msg,
         ret = target_to_host_sockaddr(msg.msg_name, tswapal(msgp->msg_name),
                                 msg.msg_namelen);
         if (ret) {
-            unlock_user_struct(msgp, target_msg, send ? 0 : 1);
-            return ret;
+            goto out2;
         }
     } else {
         msg.msg_name = NULL;
@@ -1900,9 +1945,13 @@ static abi_long do_sendrecvmsg(int fd, abi_ulong target_msg,
     msg.msg_flags = tswap32(msgp->msg_flags);
 
     count = tswapal(msgp->msg_iovlen);
-    vec = alloca(count * sizeof(struct iovec));
     target_vec = tswapal(msgp->msg_iov);
-    lock_iovec(send ? VERIFY_READ : VERIFY_WRITE, vec, target_vec, count, send);
+    vec = lock_iovec(send ? VERIFY_READ : VERIFY_WRITE,
+                     target_vec, count, send);
+    if (vec == NULL) {
+        ret = -host_to_target_errno(errno);
+        goto out2;
+    }
     msg.msg_iovlen = count;
     msg.msg_iov = vec;
 
@@ -1932,6 +1981,7 @@ static abi_long do_sendrecvmsg(int fd, abi_ulong target_msg,
 
 out:
     unlock_iovec(vec, target_vec, count, !send);
+out2:
     unlock_user_struct(msgp, target_msg, send ? 0 : 1);
     return ret;
 }
@@ -3628,9 +3678,7 @@ static abi_long do_ioctl_dm(const IOCTLEntry *ie, uint8_t *buf_temp, int fd,
         unlock_user(argptr, arg, target_size);
     }
 out:
-    if (big_buf) {
-        free(big_buf);
-    }
+    g_free(big_buf);
     return ret;
 }
 
@@ -4875,7 +4923,7 @@ static int do_futex(target_ulong uaddr, int op, int val, target_ulong timeout,
 
 /* Map host to target signal numbers for the wait family of syscalls.
    Assume all other status bits are the same.  */
-static int host_to_target_waitstatus(int status)
+int host_to_target_waitstatus(int status)
 {
     if (WIFSIGNALED(status)) {
         return host_to_target_signal(WTERMSIG(status)) | (status & ~0x7f);
@@ -4964,8 +5012,8 @@ static int open_self_maps(void *cpu_env, int fd)
 #if defined(TARGET_ARM) || defined(TARGET_M68K) || defined(TARGET_UNICORE32)
     dprintf(fd, "%08llx-%08llx rw-p %08llx 00:00 0          [stack]\n",
                 (unsigned long long)ts->info->stack_limit,
-                (unsigned long long)(ts->stack_base + (TARGET_PAGE_SIZE - 1))
-                                     & TARGET_PAGE_MASK,
+                (unsigned long long)(ts->info->start_stack +
+                                     (TARGET_PAGE_SIZE - 1)) & TARGET_PAGE_MASK,
                 (unsigned long long)0);
 #endif
 
@@ -6531,6 +6579,8 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
             __put_user(stfs.f_fsid.__val[0], &target_stfs->f_fsid.val[0]);
             __put_user(stfs.f_fsid.__val[1], &target_stfs->f_fsid.val[1]);
             __put_user(stfs.f_namelen, &target_stfs->f_namelen);
+            __put_user(stfs.f_frsize, &target_stfs->f_frsize);
+            memset(target_stfs->f_spare, 0, sizeof(target_stfs->f_spare));
             unlock_user_struct(target_stfs, arg2, 1);
         }
         break;
@@ -6559,6 +6609,8 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
             __put_user(stfs.f_fsid.__val[0], &target_stfs->f_fsid.val[0]);
             __put_user(stfs.f_fsid.__val[1], &target_stfs->f_fsid.val[1]);
             __put_user(stfs.f_namelen, &target_stfs->f_namelen);
+            __put_user(stfs.f_frsize, &target_stfs->f_frsize);
+            memset(target_stfs->f_spare, 0, sizeof(target_stfs->f_spare));
             unlock_user_struct(target_stfs, arg3, 1);
         }
         break;
@@ -6890,6 +6942,8 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
         ret = get_errno(do_fork(cpu_env, arg1, arg2, arg3, arg5, arg4));
 #elif defined(TARGET_CRIS)
         ret = get_errno(do_fork(cpu_env, arg2, arg1, arg3, arg4, arg5));
+#elif defined(TARGET_MICROBLAZE)
+        ret = get_errno(do_fork(cpu_env, arg1, arg2, arg4, arg6, arg5));
 #elif defined(TARGET_S390X)
         ret = get_errno(do_fork(cpu_env, arg2, arg1, arg3, arg5, arg4));
 #else
@@ -7186,26 +7240,24 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
         break;
     case TARGET_NR_readv:
         {
-            int count = arg3;
-            struct iovec *vec;
-
-            vec = alloca(count * sizeof(struct iovec));
-            if (lock_iovec(VERIFY_WRITE, vec, arg2, count, 0) < 0)
-                goto efault;
-            ret = get_errno(readv(arg1, vec, count));
-            unlock_iovec(vec, arg2, count, 1);
+            struct iovec *vec = lock_iovec(VERIFY_WRITE, arg2, arg3, 0);
+            if (vec != NULL) {
+                ret = get_errno(readv(arg1, vec, arg3));
+                unlock_iovec(vec, arg2, arg3, 1);
+            } else {
+                ret = -host_to_target_errno(errno);
+            }
         }
         break;
     case TARGET_NR_writev:
         {
-            int count = arg3;
-            struct iovec *vec;
-
-            vec = alloca(count * sizeof(struct iovec));
-            if (lock_iovec(VERIFY_READ, vec, arg2, count, 1) < 0)
-                goto efault;
-            ret = get_errno(writev(arg1, vec, count));
-            unlock_iovec(vec, arg2, count, 0);
+            struct iovec *vec = lock_iovec(VERIFY_READ, arg2, arg3, 1);
+            if (vec != NULL) {
+                ret = get_errno(writev(arg1, vec, arg3));
+                unlock_iovec(vec, arg2, arg3, 0);
+            } else {
+                ret = -host_to_target_errno(errno);
+            }
         }
         break;
     case TARGET_NR_getsid:
@@ -7417,12 +7469,20 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
 #endif
 #ifdef TARGET_NR_pread64
     case TARGET_NR_pread64:
+        if (regpairs_aligned(cpu_env)) {
+            arg4 = arg5;
+            arg5 = arg6;
+        }
         if (!(p = lock_user(VERIFY_WRITE, arg2, arg3, 0)))
             goto efault;
         ret = get_errno(pread64(arg1, p, arg3, target_offset64(arg4, arg5)));
         unlock_user(p, arg2, ret);
         break;
     case TARGET_NR_pwrite64:
+        if (regpairs_aligned(cpu_env)) {
+            arg4 = arg5;
+            arg5 = arg6;
+        }
         if (!(p = lock_user(VERIFY_READ, arg2, arg3, 1)))
             goto efault;
         ret = get_errno(pwrite64(arg1, p, arg3, target_offset64(arg4, arg5)));
@@ -8630,14 +8690,13 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
 #ifdef TARGET_NR_vmsplice
 	case TARGET_NR_vmsplice:
         {
-            int count = arg3;
-            struct iovec *vec;
-
-            vec = alloca(count * sizeof(struct iovec));
-            if (lock_iovec(VERIFY_READ, vec, arg2, count, 1) < 0)
-                goto efault;
-            ret = get_errno(vmsplice(arg1, vec, count, arg4));
-            unlock_iovec(vec, arg2, count, 0);
+            struct iovec *vec = lock_iovec(VERIFY_READ, arg2, arg3, 1);
+            if (vec != NULL) {
+                ret = get_errno(vmsplice(arg1, vec, arg3, arg4));
+                unlock_iovec(vec, arg2, arg3, 0);
+            } else {
+                ret = -host_to_target_errno(errno);
+            }
         }
         break;
 #endif
@@ -8824,6 +8883,19 @@ abi_long do_syscall(void *cpu_env, int num, abi_long arg1,
         break;
     }
 #endif
+#ifdef TARGET_NR_gethostname
+    case TARGET_NR_gethostname:
+    {
+        char *name = lock_user(VERIFY_WRITE, arg1, arg2, 0);
+        if (name) {
+            ret = get_errno(gethostname(name, arg2));
+            unlock_user(name, arg1, arg2);
+        } else {
+            ret = -TARGET_EFAULT;
+        }
+        break;
+    }
+#endif
     default:
     unimplemented:
         gemu_log("qemu: Unsupported syscall: %d\n", num);
diff --git a/main-loop.c b/main-loop.c
index eb3b6e625..c87624e62 100644
--- a/main-loop.c
+++ b/main-loop.c
@@ -26,75 +26,12 @@
 #include "qemu-timer.h"
 #include "slirp/slirp.h"
 #include "main-loop.h"
+#include "qemu-aio.h"
 
 #ifndef _WIN32
 
 #include "compatfd.h"
 
-static int io_thread_fd = -1;
-
-void qemu_notify_event(void)
-{
-    /* Write 8 bytes to be compatible with eventfd.  */
-    static const uint64_t val = 1;
-    ssize_t ret;
-
-    if (io_thread_fd == -1) {
-        return;
-    }
-    do {
-        ret = write(io_thread_fd, &val, sizeof(val));
-    } while (ret < 0 && errno == EINTR);
-
-    /* EAGAIN is fine, a read must be pending.  */
-    if (ret < 0 && errno != EAGAIN) {
-        fprintf(stderr, "qemu_notify_event: write() failed: %s\n",
-                strerror(errno));
-        exit(1);
-    }
-}
-
-static void qemu_event_read(void *opaque)
-{
-    int fd = (intptr_t)opaque;
-    ssize_t len;
-    char buffer[512];
-
-    /* Drain the notify pipe.  For eventfd, only 8 bytes will be read.  */
-    do {
-        len = read(fd, buffer, sizeof(buffer));
-    } while ((len == -1 && errno == EINTR) || len == sizeof(buffer));
-}
-
-static int qemu_event_init(void)
-{
-    int err;
-    int fds[2];
-
-    err = qemu_eventfd(fds);
-    if (err == -1) {
-        return -errno;
-    }
-    err = fcntl_setfl(fds[0], O_NONBLOCK);
-    if (err < 0) {
-        goto fail;
-    }
-    err = fcntl_setfl(fds[1], O_NONBLOCK);
-    if (err < 0) {
-        goto fail;
-    }
-    qemu_set_fd_handler2(fds[0], NULL, qemu_event_read, NULL,
-                         (void *)(intptr_t)fds[0]);
-
-    io_thread_fd = fds[1];
-    return 0;
-
-fail:
-    close(fds[0]);
-    close(fds[1]);
-    return err;
-}
-
 /* If we have signalfd, we mask out the signals we want to handle and then
  * use signalfd to listen for them.  We rely on whatever the current signal
  * handler is to dispatch the signals when we receive them.
@@ -164,57 +101,42 @@ static int qemu_signal_init(void)
 
 #else /* _WIN32 */
 
-static HANDLE qemu_event_handle = NULL;
-
-static void dummy_event_handler(void *opaque)
-{
-}
-
-static int qemu_event_init(void)
+static int qemu_signal_init(void)
 {
-    qemu_event_handle = CreateEvent(NULL, FALSE, FALSE, NULL);
-    if (!qemu_event_handle) {
-        fprintf(stderr, "Failed CreateEvent: %ld\n", GetLastError());
-        return -1;
-    }
-    qemu_add_wait_object(qemu_event_handle, dummy_event_handler, NULL);
     return 0;
 }
+#endif
+
+static AioContext *qemu_aio_context;
 
 void qemu_notify_event(void)
 {
-    if (!qemu_event_handle) {
+    if (!qemu_aio_context) {
         return;
     }
-    if (!SetEvent(qemu_event_handle)) {
-        fprintf(stderr, "qemu_notify_event: SetEvent failed: %ld\n",
-                GetLastError());
-        exit(1);
-    }
+    aio_notify(qemu_aio_context);
 }
 
-static int qemu_signal_init(void)
-{
-    return 0;
-}
-#endif
-
-int main_loop_init(void)
+int qemu_init_main_loop(void)
 {
     int ret;
+    GSource *src;
 
-    qemu_mutex_lock_iothread();
-    ret = qemu_signal_init();
-    if (ret) {
-        return ret;
+    init_clocks();
+    if (init_timer_alarm() < 0) {
+        fprintf(stderr, "could not initialize alarm timer\n");
+        exit(1);
     }
 
-    /* Note eventfd must be drained before signalfd handlers run */
-    ret = qemu_event_init();
+    ret = qemu_signal_init();
     if (ret) {
         return ret;
     }
 
+    qemu_aio_context = aio_context_new();
+    src = aio_get_g_source(qemu_aio_context);
+    g_source_attach(src, NULL);
+    g_source_unref(src);
     return 0;
 }
 
@@ -400,7 +322,8 @@ void qemu_del_wait_object(HANDLE handle, WaitObjectFunc *func, void *opaque)
 
 void qemu_fd_register(int fd)
 {
-    WSAEventSelect(fd, qemu_event_handle, FD_READ | FD_ACCEPT | FD_CLOSE |
+    WSAEventSelect(fd, event_notifier_get_handle(&qemu_aio_context->notifier),
+                   FD_READ | FD_ACCEPT | FD_CLOSE |
                    FD_CONNECT | FD_WRITE | FD_OOB);
 }
 
@@ -477,8 +400,6 @@ int main_loop_wait(int nonblocking)
 
     if (nonblocking) {
         timeout = 0;
-    } else {
-        qemu_bh_update_timeout(&timeout);
     }
 
     /* poll any events */
@@ -501,9 +422,41 @@ int main_loop_wait(int nonblocking)
 
     qemu_run_all_timers();
 
-    /* Check bottom-halves last in case any of the earlier events triggered
-       them.  */
-    qemu_bh_poll();
-
     return ret;
 }
+
+/* Functions to operate on the main QEMU AioContext.  */
+
+QEMUBH *qemu_bh_new(QEMUBHFunc *cb, void *opaque)
+{
+    return aio_bh_new(qemu_aio_context, cb, opaque);
+}
+
+void qemu_aio_flush(void)
+{
+    aio_flush(qemu_aio_context);
+}
+
+bool qemu_aio_wait(void)
+{
+    return aio_poll(qemu_aio_context, true);
+}
+
+#ifdef CONFIG_POSIX
+void qemu_aio_set_fd_handler(int fd,
+                             IOHandler *io_read,
+                             IOHandler *io_write,
+                             AioFlushHandler *io_flush,
+                             void *opaque)
+{
+    aio_set_fd_handler(qemu_aio_context, fd, io_read, io_write, io_flush,
+                       opaque);
+}
+#endif
+
+void qemu_aio_set_event_notifier(EventNotifier *notifier,
+                                 EventNotifierHandler *io_read,
+                                 AioFlushEventNotifierHandler *io_flush)
+{
+    aio_set_event_notifier(qemu_aio_context, notifier, io_read, io_flush);
+}
diff --git a/main-loop.h b/main-loop.h
index dce1cd9d7..326c74269 100644
--- a/main-loop.h
+++ b/main-loop.h
@@ -25,6 +25,8 @@
 #ifndef QEMU_MAIN_LOOP_H
 #define QEMU_MAIN_LOOP_H 1
 
+#include "qemu-aio.h"
+
 #define SIG_IPI SIGUSR1
 
 /**
@@ -43,16 +45,6 @@
 int qemu_init_main_loop(void);
 
 /**
- * main_loop_init: Initializes main loop
- *
- * Internal (but shared for compatibility reasons) initialization routine
- * for the main loop. This should not be used by applications directly,
- * use qemu_init_main_loop() instead.
- *
- */
-int main_loop_init(void);
-
-/**
  * main_loop_wait: Run one iteration of the main loop.
  *
  * If @nonblocking is true, poll for events, otherwise suspend until
@@ -173,7 +165,6 @@ void qemu_del_wait_object(HANDLE handle, WaitObjectFunc *func, void *opaque);
 
 typedef void IOReadHandler(void *opaque, const uint8_t *buf, int size);
 typedef int IOCanReadHandler(void *opaque);
-typedef void IOHandler(void *opaque);
 
 /**
  * qemu_set_fd_handler2: Register a file descriptor with the main loop
@@ -254,56 +245,6 @@ int qemu_set_fd_handler(int fd,
                         IOHandler *fd_write,
                         void *opaque);
 
-typedef struct QEMUBH QEMUBH;
-typedef void QEMUBHFunc(void *opaque);
-
-/**
- * qemu_bh_new: Allocate a new bottom half structure.
- *
- * Bottom halves are lightweight callbacks whose invocation is guaranteed
- * to be wait-free, thread-safe and signal-safe.  The #QEMUBH structure
- * is opaque and must be allocated prior to its use.
- */
-QEMUBH *qemu_bh_new(QEMUBHFunc *cb, void *opaque);
-
-/**
- * qemu_bh_schedule: Schedule a bottom half.
- *
- * Scheduling a bottom half interrupts the main loop and causes the
- * execution of the callback that was passed to qemu_bh_new.
- *
- * Bottom halves that are scheduled from a bottom half handler are instantly
- * invoked.  This can create an infinite loop if a bottom half handler
- * schedules itself.
- *
- * @bh: The bottom half to be scheduled.
- */
-void qemu_bh_schedule(QEMUBH *bh);
-
-/**
- * qemu_bh_cancel: Cancel execution of a bottom half.
- *
- * Canceling execution of a bottom half undoes the effect of calls to
- * qemu_bh_schedule without freeing its resources yet.  While cancellation
- * itself is also wait-free and thread-safe, it can of course race with the
- * loop that executes bottom halves unless you are holding the iothread
- * mutex.  This makes it mostly useless if you are not holding the mutex.
- *
- * @bh: The bottom half to be canceled.
- */
-void qemu_bh_cancel(QEMUBH *bh);
-
-/**
- *qemu_bh_delete: Cancel execution of a bottom half and free its resources.
- *
- * Deleting a bottom half frees the memory that was allocated for it by
- * qemu_bh_new.  It also implies canceling the bottom half if it was
- * scheduled.
- *
- * @bh: The bottom half to be deleted.
- */
-void qemu_bh_delete(QEMUBH *bh);
-
 #ifdef CONFIG_POSIX
 /**
  * qemu_add_child_watch: Register a child process for reaping.
@@ -359,8 +300,7 @@ void qemu_fd_register(int fd);
 void qemu_iohandler_fill(int *pnfds, fd_set *readfds, fd_set *writefds, fd_set *xfds);
 void qemu_iohandler_poll(fd_set *readfds, fd_set *writefds, fd_set *xfds, int rc);
 
+QEMUBH *qemu_bh_new(QEMUBHFunc *cb, void *opaque);
 void qemu_bh_schedule_idle(QEMUBH *bh);
-int qemu_bh_poll(void);
-void qemu_bh_update_timeout(uint32_t *timeout);
 
 #endif
diff --git a/exec-obsolete.h b/memory-internal.h
index c09925610..1da240039 100644
--- a/exec-obsolete.h
+++ b/memory-internal.h
@@ -16,14 +16,32 @@
  * The functions declared here will be removed soon.
  */
 
-#ifndef EXEC_OBSOLETE_H
-#define EXEC_OBSOLETE_H
-
-#ifndef WANT_EXEC_OBSOLETE
-#error Do not include exec-obsolete.h
-#endif
+#ifndef MEMORY_INTERNAL_H
+#define MEMORY_INTERNAL_H
 
 #ifndef CONFIG_USER_ONLY
+#include "hw/xen.h"
+
+typedef struct PhysPageEntry PhysPageEntry;
+
+struct PhysPageEntry {
+    uint16_t is_leaf : 1;
+     /* index into phys_sections (is_leaf) or phys_map_nodes (!is_leaf) */
+    uint16_t ptr : 15;
+};
+
+typedef struct AddressSpaceDispatch AddressSpaceDispatch;
+
+struct AddressSpaceDispatch {
+    /* This is a multi-level map on the physical address space.
+     * The bottom level has pointers to MemoryRegionSections.
+     */
+    PhysPageEntry phys_map;
+    MemoryListener listener;
+};
+
+void address_space_init_dispatch(AddressSpace *as);
+void address_space_destroy_dispatch(AddressSpace *as);
 
 ram_addr_t qemu_ram_alloc_from_ptr(ram_addr_t size, void *host,
                                    MemoryRegion *mr);
@@ -33,13 +51,9 @@ void qemu_ram_free_from_ptr(ram_addr_t addr);
 
 struct MemoryRegion;
 struct MemoryRegionSection;
-void cpu_register_physical_memory_log(struct MemoryRegionSection *section,
-                                      bool readonly);
-
-void qemu_register_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size);
-void qemu_unregister_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size);
 
-int cpu_physical_memory_set_dirty_tracking(int enable);
+void qemu_register_coalesced_mmio(hwaddr addr, ram_addr_t size);
+void qemu_unregister_coalesced_mmio(hwaddr addr, ram_addr_t size);
 
 #define VGA_DIRTY_FLAG       0x01
 #define CODE_DIRTY_FLAG      0x02
@@ -74,11 +88,6 @@ static inline int cpu_physical_memory_get_dirty(ram_addr_t start,
 static inline int cpu_physical_memory_set_dirty_flags(ram_addr_t addr,
                                                       int dirty_flags)
 {
-    if ((dirty_flags & MIGRATION_DIRTY_FLAG) &&
-        !cpu_physical_memory_get_dirty(addr, TARGET_PAGE_SIZE,
-                                       MIGRATION_DIRTY_FLAG)) {
-        ram_list.dirty_pages++;
-    }
     return ram_list.phys_dirty[addr >> TARGET_PAGE_BITS] |= dirty_flags;
 }
 
@@ -92,11 +101,6 @@ static inline int cpu_physical_memory_clear_dirty_flags(ram_addr_t addr,
 {
     int mask = ~dirty_flags;
 
-    if ((dirty_flags & MIGRATION_DIRTY_FLAG) &&
-        cpu_physical_memory_get_dirty(addr, TARGET_PAGE_SIZE,
-                                      MIGRATION_DIRTY_FLAG)) {
-        ram_list.dirty_pages--;
-    }
     return ram_list.phys_dirty[addr >> TARGET_PAGE_BITS] &= mask;
 }
 
@@ -111,6 +115,7 @@ static inline void cpu_physical_memory_set_dirty_range(ram_addr_t start,
     for (addr = start; addr < end; addr += TARGET_PAGE_SIZE) {
         cpu_physical_memory_set_dirty_flags(addr, dirty_flags);
     }
+    xen_modified_memory(addr, length);
 }
 
 static inline void cpu_physical_memory_mask_dirty_range(ram_addr_t start,
diff --git a/memory.c b/memory.c
index d528d1f7f..741985385 100644
--- a/memory.c
+++ b/memory.c
@@ -20,16 +20,18 @@
 #include "kvm.h"
 #include <assert.h>
 
-#define WANT_EXEC_OBSOLETE
-#include "exec-obsolete.h"
+#include "memory-internal.h"
 
-unsigned memory_region_transaction_depth = 0;
-static bool memory_region_update_pending = false;
+static unsigned memory_region_transaction_depth;
+static bool memory_region_update_pending;
 static bool global_dirty_log = false;
 
 static QTAILQ_HEAD(memory_listeners, MemoryListener) memory_listeners
     = QTAILQ_HEAD_INITIALIZER(memory_listeners);
 
+static QTAILQ_HEAD(, AddressSpace) address_spaces
+    = QTAILQ_HEAD_INITIALIZER(address_spaces);
+
 typedef struct AddrRange AddrRange;
 
 /*
@@ -98,13 +100,17 @@ static bool memory_listener_match(MemoryListener *listener,
         switch (_direction) {                                           \
         case Forward:                                                   \
             QTAILQ_FOREACH(_listener, &memory_listeners, link) {        \
-                _listener->_callback(_listener, ##_args);               \
+                if (_listener->_callback) {                             \
+                    _listener->_callback(_listener, ##_args);           \
+                }                                                       \
             }                                                           \
             break;                                                      \
         case Reverse:                                                   \
             QTAILQ_FOREACH_REVERSE(_listener, &memory_listeners,        \
                                    memory_listeners, link) {            \
-                _listener->_callback(_listener, ##_args);               \
+                if (_listener->_callback) {                             \
+                    _listener->_callback(_listener, ##_args);           \
+                }                                                       \
             }                                                           \
             break;                                                      \
         default:                                                        \
@@ -119,7 +125,8 @@ static bool memory_listener_match(MemoryListener *listener,
         switch (_direction) {                                           \
         case Forward:                                                   \
             QTAILQ_FOREACH(_listener, &memory_listeners, link) {        \
-                if (memory_listener_match(_listener, _section)) {       \
+                if (_listener->_callback                                \
+                    && memory_listener_match(_listener, _section)) {    \
                     _listener->_callback(_listener, _section, ##_args); \
                 }                                                       \
             }                                                           \
@@ -127,7 +134,8 @@ static bool memory_listener_match(MemoryListener *listener,
         case Reverse:                                                   \
             QTAILQ_FOREACH_REVERSE(_listener, &memory_listeners,        \
                                    memory_listeners, link) {            \
-                if (memory_listener_match(_listener, _section)) {       \
+                if (_listener->_callback                                \
+                    && memory_listener_match(_listener, _section)) {    \
                     _listener->_callback(_listener, _section, ##_args); \
                 }                                                       \
             }                                                           \
@@ -140,7 +148,7 @@ static bool memory_listener_match(MemoryListener *listener,
 #define MEMORY_LISTENER_UPDATE_REGION(fr, as, dir, callback)            \
     MEMORY_LISTENER_CALL(callback, dir, (&(MemoryRegionSection) {       \
         .mr = (fr)->mr,                                                 \
-        .address_space = (as)->root,                                    \
+        .address_space = (as),                                          \
         .offset_within_region = (fr)->offset_in_region,                 \
         .size = int128_get64((fr)->addr.size),                          \
         .offset_within_address_space = int128_get64((fr)->addr.start),  \
@@ -202,7 +210,7 @@ typedef struct FlatView FlatView;
 /* Range of memory in the global map.  Addresses are absolute. */
 struct FlatRange {
     MemoryRegion *mr;
-    target_phys_addr_t offset_in_region;
+    hwaddr offset_in_region;
     AddrRange addr;
     uint8_t dirty_log_mask;
     bool readable;
@@ -218,17 +226,8 @@ struct FlatView {
     unsigned nr_allocated;
 };
 
-typedef struct AddressSpace AddressSpace;
 typedef struct AddressSpaceOps AddressSpaceOps;
 
-/* A system address space - I/O, memory, etc. */
-struct AddressSpace {
-    MemoryRegion *root;
-    FlatView current_map;
-    int ioeventfd_nb;
-    MemoryRegionIoeventfd *ioeventfds;
-};
-
 #define FOR_EACH_FLAT_RANGE(var, view)          \
     for (var = (view)->ranges; var < (view)->ranges + (view)->nr; ++var)
 
@@ -302,7 +301,7 @@ static void flatview_simplify(FlatView *view)
 }
 
 static void memory_region_read_accessor(void *opaque,
-                                        target_phys_addr_t addr,
+                                        hwaddr addr,
                                         uint64_t *value,
                                         unsigned size,
                                         unsigned shift,
@@ -311,12 +310,15 @@ static void memory_region_read_accessor(void *opaque,
     MemoryRegion *mr = opaque;
     uint64_t tmp;
 
+    if (mr->flush_coalesced_mmio) {
+        qemu_flush_coalesced_mmio_buffer();
+    }
     tmp = mr->ops->read(mr->opaque, addr, size);
     *value |= (tmp & mask) << shift;
 }
 
 static void memory_region_write_accessor(void *opaque,
-                                         target_phys_addr_t addr,
+                                         hwaddr addr,
                                          uint64_t *value,
                                          unsigned size,
                                          unsigned shift,
@@ -325,17 +327,20 @@ static void memory_region_write_accessor(void *opaque,
     MemoryRegion *mr = opaque;
     uint64_t tmp;
 
+    if (mr->flush_coalesced_mmio) {
+        qemu_flush_coalesced_mmio_buffer();
+    }
     tmp = (*value >> shift) & mask;
     mr->ops->write(mr->opaque, addr, tmp, size);
 }
 
-static void access_with_adjusted_size(target_phys_addr_t addr,
+static void access_with_adjusted_size(hwaddr addr,
                                       uint64_t *value,
                                       unsigned size,
                                       unsigned access_size_min,
                                       unsigned access_size_max,
                                       void (*access)(void *opaque,
-                                                     target_phys_addr_t addr,
+                                                     hwaddr addr,
                                                      uint64_t *value,
                                                      unsigned size,
                                                      unsigned shift,
@@ -360,8 +365,6 @@ static void access_with_adjusted_size(target_phys_addr_t addr,
     }
 }
 
-static AddressSpace address_space_memory;
-
 static const MemoryRegionPortio *find_portio(MemoryRegion *mr, uint64_t offset,
                                              unsigned width, bool write)
 {
@@ -450,18 +453,17 @@ const IORangeOps memory_region_iorange_ops = {
     .destructor = memory_region_iorange_destructor,
 };
 
-static AddressSpace address_space_io;
-
 static AddressSpace *memory_region_to_address_space(MemoryRegion *mr)
 {
+    AddressSpace *as;
+
     while (mr->parent) {
         mr = mr->parent;
     }
-    if (mr == address_space_memory.root) {
-        return &address_space_memory;
-    }
-    if (mr == address_space_io.root) {
-        return &address_space_io;
+    QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) {
+        if (mr == as->root) {
+            return as;
+        }
     }
     abort();
 }
@@ -477,7 +479,7 @@ static void render_memory_region(FlatView *view,
 {
     MemoryRegion *subregion;
     unsigned i;
-    target_phys_addr_t offset_in_region;
+    hwaddr offset_in_region;
     Int128 remain;
     Int128 now;
     FlatRange fr;
@@ -538,12 +540,12 @@ static void render_memory_region(FlatView *view,
             offset_in_region += int128_get64(now);
             int128_subfrom(&remain, now);
         }
-        if (int128_eq(base, view->ranges[i].addr.start)) {
-            now = int128_min(remain, view->ranges[i].addr.size);
-            int128_addto(&base, now);
-            offset_in_region += int128_get64(now);
-            int128_subfrom(&remain, now);
-        }
+        now = int128_sub(int128_min(int128_add(base, remain),
+                                    addrrange_end(view->ranges[i].addr)),
+                         base);
+        int128_addto(&base, now);
+        offset_in_region += int128_get64(now);
+        int128_subfrom(&remain, now);
     }
     if (int128_nz(remain)) {
         fr.mr = mr;
@@ -563,8 +565,10 @@ static FlatView generate_memory_topology(MemoryRegion *mr)
 
     flatview_init(&view);
 
-    render_memory_region(&view, mr, int128_zero(),
-                         addrrange_make(int128_zero(), int128_2_64()), false);
+    if (mr) {
+        render_memory_region(&view, mr, int128_zero(),
+                             addrrange_make(int128_zero(), int128_2_64()), false);
+    }
     flatview_simplify(&view);
 
     return view;
@@ -592,7 +596,7 @@ static void address_space_add_del_ioeventfds(AddressSpace *as,
                                                   fds_new[inew]))) {
             fd = &fds_old[iold];
             section = (MemoryRegionSection) {
-                .address_space = as->root,
+                .address_space = as,
                 .offset_within_address_space = int128_get64(fd->addr.start),
                 .size = int128_get64(fd->addr.size),
             };
@@ -605,7 +609,7 @@ static void address_space_add_del_ioeventfds(AddressSpace *as,
                                                          fds_old[iold]))) {
             fd = &fds_new[inew];
             section = (MemoryRegionSection) {
-                .address_space = as->root,
+                .address_space = as,
                 .offset_within_address_space = int128_get64(fd->addr.start),
                 .size = int128_get64(fd->addr.size),
             };
@@ -627,7 +631,7 @@ static void address_space_update_ioeventfds(AddressSpace *as)
     AddrRange tmp;
     unsigned i;
 
-    FOR_EACH_FLAT_RANGE(fr, &as->current_map) {
+    FOR_EACH_FLAT_RANGE(fr, as->current_map) {
         for (i = 0; i < fr->mr->ioeventfd_nb; ++i) {
             tmp = addrrange_shift(fr->mr->ioeventfds[i].addr,
                                   int128_sub(fr->addr.start,
@@ -715,53 +719,38 @@ static void address_space_update_topology_pass(AddressSpace *as,
 
 static void address_space_update_topology(AddressSpace *as)
 {
-    FlatView old_view = as->current_map;
+    FlatView old_view = *as->current_map;
     FlatView new_view = generate_memory_topology(as->root);
 
     address_space_update_topology_pass(as, old_view, new_view, false);
     address_space_update_topology_pass(as, old_view, new_view, true);
 
-    as->current_map = new_view;
+    *as->current_map = new_view;
     flatview_destroy(&old_view);
     address_space_update_ioeventfds(as);
 }
 
-static void memory_region_update_topology(MemoryRegion *mr)
-{
-    if (memory_region_transaction_depth) {
-        memory_region_update_pending |= !mr || mr->enabled;
-        return;
-    }
-
-    if (mr && !mr->enabled) {
-        return;
-    }
-
-    MEMORY_LISTENER_CALL_GLOBAL(begin, Forward);
-
-    if (address_space_memory.root) {
-        address_space_update_topology(&address_space_memory);
-    }
-    if (address_space_io.root) {
-        address_space_update_topology(&address_space_io);
-    }
-
-    MEMORY_LISTENER_CALL_GLOBAL(commit, Forward);
-
-    memory_region_update_pending = false;
-}
-
 void memory_region_transaction_begin(void)
 {
+    qemu_flush_coalesced_mmio_buffer();
     ++memory_region_transaction_depth;
 }
 
 void memory_region_transaction_commit(void)
 {
+    AddressSpace *as;
+
     assert(memory_region_transaction_depth);
     --memory_region_transaction_depth;
     if (!memory_region_transaction_depth && memory_region_update_pending) {
-        memory_region_update_topology(NULL);
+        memory_region_update_pending = false;
+        MEMORY_LISTENER_CALL_GLOBAL(begin, Forward);
+
+        QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) {
+            address_space_update_topology(as);
+        }
+
+        MEMORY_LISTENER_CALL_GLOBAL(commit, Forward);
     }
 }
 
@@ -826,10 +815,11 @@ void memory_region_init(MemoryRegion *mr,
     mr->dirty_log_mask = 0;
     mr->ioeventfd_nb = 0;
     mr->ioeventfds = NULL;
+    mr->flush_coalesced_mmio = false;
 }
 
 static bool memory_region_access_valid(MemoryRegion *mr,
-                                       target_phys_addr_t addr,
+                                       hwaddr addr,
                                        unsigned size,
                                        bool is_write)
 {
@@ -855,7 +845,7 @@ static bool memory_region_access_valid(MemoryRegion *mr,
 }
 
 static uint64_t memory_region_dispatch_read1(MemoryRegion *mr,
-                                             target_phys_addr_t addr,
+                                             hwaddr addr,
                                              unsigned size)
 {
     uint64_t data = 0;
@@ -896,7 +886,7 @@ static void adjust_endianness(MemoryRegion *mr, uint64_t *data, unsigned size)
 }
 
 static uint64_t memory_region_dispatch_read(MemoryRegion *mr,
-                                            target_phys_addr_t addr,
+                                            hwaddr addr,
                                             unsigned size)
 {
     uint64_t ret;
@@ -907,7 +897,7 @@ static uint64_t memory_region_dispatch_read(MemoryRegion *mr,
 }
 
 static void memory_region_dispatch_write(MemoryRegion *mr,
-                                         target_phys_addr_t addr,
+                                         hwaddr addr,
                                          uint64_t data,
                                          unsigned size)
 {
@@ -969,7 +959,7 @@ void memory_region_init_ram_ptr(MemoryRegion *mr,
 void memory_region_init_alias(MemoryRegion *mr,
                               const char *name,
                               MemoryRegion *orig,
-                              target_phys_addr_t offset,
+                              hwaddr offset,
                               uint64_t size)
 {
     memory_region_init(mr, name, size);
@@ -992,7 +982,7 @@ void memory_region_init_rom_device(MemoryRegion *mr,
     mr->ram_addr = qemu_ram_alloc(size, mr);
 }
 
-static uint64_t invalid_read(void *opaque, target_phys_addr_t addr,
+static uint64_t invalid_read(void *opaque, hwaddr addr,
                              unsigned size)
 {
     MemoryRegion *mr = opaque;
@@ -1004,7 +994,7 @@ static uint64_t invalid_read(void *opaque, target_phys_addr_t addr,
     return -1U;
 }
 
-static void invalid_write(void *opaque, target_phys_addr_t addr, uint64_t data,
+static void invalid_write(void *opaque, hwaddr addr, uint64_t data,
                           unsigned size)
 {
     MemoryRegion *mr = opaque;
@@ -1031,6 +1021,7 @@ void memory_region_init_reservation(MemoryRegion *mr,
 void memory_region_destroy(MemoryRegion *mr)
 {
     assert(QTAILQ_EMPTY(&mr->subregions));
+    assert(memory_region_transaction_depth == 0);
     mr->destructor(mr);
     memory_region_clear_coalescing(mr);
     g_free((char *)mr->name);
@@ -1069,20 +1060,22 @@ void memory_region_set_log(MemoryRegion *mr, bool log, unsigned client)
 {
     uint8_t mask = 1 << client;
 
+    memory_region_transaction_begin();
     mr->dirty_log_mask = (mr->dirty_log_mask & ~mask) | (log * mask);
-    memory_region_update_topology(mr);
+    memory_region_update_pending |= mr->enabled;
+    memory_region_transaction_commit();
 }
 
-bool memory_region_get_dirty(MemoryRegion *mr, target_phys_addr_t addr,
-                             target_phys_addr_t size, unsigned client)
+bool memory_region_get_dirty(MemoryRegion *mr, hwaddr addr,
+                             hwaddr size, unsigned client)
 {
     assert(mr->terminates);
     return cpu_physical_memory_get_dirty(mr->ram_addr + addr, size,
                                          1 << client);
 }
 
-void memory_region_set_dirty(MemoryRegion *mr, target_phys_addr_t addr,
-                             target_phys_addr_t size)
+void memory_region_set_dirty(MemoryRegion *mr, hwaddr addr,
+                             hwaddr size)
 {
     assert(mr->terminates);
     return cpu_physical_memory_set_dirty_range(mr->ram_addr + addr, size, -1);
@@ -1090,12 +1083,14 @@ void memory_region_set_dirty(MemoryRegion *mr, target_phys_addr_t addr,
 
 void memory_region_sync_dirty_bitmap(MemoryRegion *mr)
 {
+    AddressSpace *as;
     FlatRange *fr;
 
-    FOR_EACH_FLAT_RANGE(fr, &address_space_memory.current_map) {
-        if (fr->mr == mr) {
-            MEMORY_LISTENER_UPDATE_REGION(fr, &address_space_memory,
-                                          Forward, log_sync);
+    QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) {
+        FOR_EACH_FLAT_RANGE(fr, as->current_map) {
+            if (fr->mr == mr) {
+                MEMORY_LISTENER_UPDATE_REGION(fr, as, Forward, log_sync);
+            }
         }
     }
 }
@@ -1103,21 +1098,25 @@ void memory_region_sync_dirty_bitmap(MemoryRegion *mr)
 void memory_region_set_readonly(MemoryRegion *mr, bool readonly)
 {
     if (mr->readonly != readonly) {
+        memory_region_transaction_begin();
         mr->readonly = readonly;
-        memory_region_update_topology(mr);
+        memory_region_update_pending |= mr->enabled;
+        memory_region_transaction_commit();
     }
 }
 
 void memory_region_rom_device_set_readable(MemoryRegion *mr, bool readable)
 {
     if (mr->readable != readable) {
+        memory_region_transaction_begin();
         mr->readable = readable;
-        memory_region_update_topology(mr);
+        memory_region_update_pending |= mr->enabled;
+        memory_region_transaction_commit();
     }
 }
 
-void memory_region_reset_dirty(MemoryRegion *mr, target_phys_addr_t addr,
-                               target_phys_addr_t size, unsigned client)
+void memory_region_reset_dirty(MemoryRegion *mr, hwaddr addr,
+                               hwaddr size, unsigned client)
 {
     assert(mr->terminates);
     cpu_physical_memory_reset_dirty(mr->ram_addr + addr,
@@ -1136,16 +1135,24 @@ void *memory_region_get_ram_ptr(MemoryRegion *mr)
     return qemu_get_ram_ptr(mr->ram_addr & TARGET_PAGE_MASK);
 }
 
-static void memory_region_update_coalesced_range(MemoryRegion *mr)
+static void memory_region_update_coalesced_range_as(MemoryRegion *mr, AddressSpace *as)
 {
     FlatRange *fr;
     CoalescedMemoryRange *cmr;
     AddrRange tmp;
+    MemoryRegionSection section;
 
-    FOR_EACH_FLAT_RANGE(fr, &address_space_memory.current_map) {
+    FOR_EACH_FLAT_RANGE(fr, as->current_map) {
         if (fr->mr == mr) {
-            qemu_unregister_coalesced_mmio(int128_get64(fr->addr.start),
-                                           int128_get64(fr->addr.size));
+            section = (MemoryRegionSection) {
+                .address_space = as,
+                .offset_within_address_space = int128_get64(fr->addr.start),
+                .size = int128_get64(fr->addr.size),
+            };
+
+            MEMORY_LISTENER_CALL(coalesced_mmio_del, Reverse, &section,
+                                 int128_get64(fr->addr.start),
+                                 int128_get64(fr->addr.size));
             QTAILQ_FOREACH(cmr, &mr->coalesced, link) {
                 tmp = addrrange_shift(cmr->addr,
                                       int128_sub(fr->addr.start,
@@ -1154,13 +1161,23 @@ static void memory_region_update_coalesced_range(MemoryRegion *mr)
                     continue;
                 }
                 tmp = addrrange_intersection(tmp, fr->addr);
-                qemu_register_coalesced_mmio(int128_get64(tmp.start),
-                                             int128_get64(tmp.size));
+                MEMORY_LISTENER_CALL(coalesced_mmio_add, Forward, &section,
+                                     int128_get64(tmp.start),
+                                     int128_get64(tmp.size));
             }
         }
     }
 }
 
+static void memory_region_update_coalesced_range(MemoryRegion *mr)
+{
+    AddressSpace *as;
+
+    QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) {
+        memory_region_update_coalesced_range_as(mr, as);
+    }
+}
+
 void memory_region_set_coalescing(MemoryRegion *mr)
 {
     memory_region_clear_coalescing(mr);
@@ -1168,7 +1185,7 @@ void memory_region_set_coalescing(MemoryRegion *mr)
 }
 
 void memory_region_add_coalescing(MemoryRegion *mr,
-                                  target_phys_addr_t offset,
+                                  hwaddr offset,
                                   uint64_t size)
 {
     CoalescedMemoryRange *cmr = g_malloc(sizeof(*cmr));
@@ -1176,12 +1193,16 @@ void memory_region_add_coalescing(MemoryRegion *mr,
     cmr->addr = addrrange_make(int128_make64(offset), int128_make64(size));
     QTAILQ_INSERT_TAIL(&mr->coalesced, cmr, link);
     memory_region_update_coalesced_range(mr);
+    memory_region_set_flush_coalesced(mr);
 }
 
 void memory_region_clear_coalescing(MemoryRegion *mr)
 {
     CoalescedMemoryRange *cmr;
 
+    qemu_flush_coalesced_mmio_buffer();
+    mr->flush_coalesced_mmio = false;
+
     while (!QTAILQ_EMPTY(&mr->coalesced)) {
         cmr = QTAILQ_FIRST(&mr->coalesced);
         QTAILQ_REMOVE(&mr->coalesced, cmr, link);
@@ -1190,8 +1211,21 @@ void memory_region_clear_coalescing(MemoryRegion *mr)
     memory_region_update_coalesced_range(mr);
 }
 
+void memory_region_set_flush_coalesced(MemoryRegion *mr)
+{
+    mr->flush_coalesced_mmio = true;
+}
+
+void memory_region_clear_flush_coalesced(MemoryRegion *mr)
+{
+    qemu_flush_coalesced_mmio_buffer();
+    if (QTAILQ_EMPTY(&mr->coalesced)) {
+        mr->flush_coalesced_mmio = false;
+    }
+}
+
 void memory_region_add_eventfd(MemoryRegion *mr,
-                               target_phys_addr_t addr,
+                               hwaddr addr,
                                unsigned size,
                                bool match_data,
                                uint64_t data,
@@ -1206,6 +1240,8 @@ void memory_region_add_eventfd(MemoryRegion *mr,
     };
     unsigned i;
 
+    adjust_endianness(mr, &mrfd.data, size);
+    memory_region_transaction_begin();
     for (i = 0; i < mr->ioeventfd_nb; ++i) {
         if (memory_region_ioeventfd_before(mrfd, mr->ioeventfds[i])) {
             break;
@@ -1217,11 +1253,12 @@ void memory_region_add_eventfd(MemoryRegion *mr,
     memmove(&mr->ioeventfds[i+1], &mr->ioeventfds[i],
             sizeof(*mr->ioeventfds) * (mr->ioeventfd_nb-1 - i));
     mr->ioeventfds[i] = mrfd;
-    memory_region_update_topology(mr);
+    memory_region_update_pending |= mr->enabled;
+    memory_region_transaction_commit();
 }
 
 void memory_region_del_eventfd(MemoryRegion *mr,
-                               target_phys_addr_t addr,
+                               hwaddr addr,
                                unsigned size,
                                bool match_data,
                                uint64_t data,
@@ -1236,6 +1273,8 @@ void memory_region_del_eventfd(MemoryRegion *mr,
     };
     unsigned i;
 
+    adjust_endianness(mr, &mrfd.data, size);
+    memory_region_transaction_begin();
     for (i = 0; i < mr->ioeventfd_nb; ++i) {
         if (memory_region_ioeventfd_equal(mrfd, mr->ioeventfds[i])) {
             break;
@@ -1247,15 +1286,18 @@ void memory_region_del_eventfd(MemoryRegion *mr,
     --mr->ioeventfd_nb;
     mr->ioeventfds = g_realloc(mr->ioeventfds,
                                   sizeof(*mr->ioeventfds)*mr->ioeventfd_nb + 1);
-    memory_region_update_topology(mr);
+    memory_region_update_pending |= mr->enabled;
+    memory_region_transaction_commit();
 }
 
 static void memory_region_add_subregion_common(MemoryRegion *mr,
-                                               target_phys_addr_t offset,
+                                               hwaddr offset,
                                                MemoryRegion *subregion)
 {
     MemoryRegion *other;
 
+    memory_region_transaction_begin();
+
     assert(!subregion->parent);
     subregion->parent = mr;
     subregion->addr = offset;
@@ -1288,12 +1330,13 @@ static void memory_region_add_subregion_common(MemoryRegion *mr,
     }
     QTAILQ_INSERT_TAIL(&mr->subregions, subregion, subregions_link);
 done:
-    memory_region_update_topology(mr);
+    memory_region_update_pending |= mr->enabled && subregion->enabled;
+    memory_region_transaction_commit();
 }
 
 
 void memory_region_add_subregion(MemoryRegion *mr,
-                                 target_phys_addr_t offset,
+                                 hwaddr offset,
                                  MemoryRegion *subregion)
 {
     subregion->may_overlap = false;
@@ -1302,7 +1345,7 @@ void memory_region_add_subregion(MemoryRegion *mr,
 }
 
 void memory_region_add_subregion_overlap(MemoryRegion *mr,
-                                         target_phys_addr_t offset,
+                                         hwaddr offset,
                                          MemoryRegion *subregion,
                                          unsigned priority)
 {
@@ -1314,10 +1357,12 @@ void memory_region_add_subregion_overlap(MemoryRegion *mr,
 void memory_region_del_subregion(MemoryRegion *mr,
                                  MemoryRegion *subregion)
 {
+    memory_region_transaction_begin();
     assert(subregion->parent == mr);
     subregion->parent = NULL;
     QTAILQ_REMOVE(&mr->subregions, subregion, subregions_link);
-    memory_region_update_topology(mr);
+    memory_region_update_pending |= mr->enabled && subregion->enabled;
+    memory_region_transaction_commit();
 }
 
 void memory_region_set_enabled(MemoryRegion *mr, bool enabled)
@@ -1325,11 +1370,13 @@ void memory_region_set_enabled(MemoryRegion *mr, bool enabled)
     if (enabled == mr->enabled) {
         return;
     }
+    memory_region_transaction_begin();
     mr->enabled = enabled;
-    memory_region_update_topology(NULL);
+    memory_region_update_pending = true;
+    memory_region_transaction_commit();
 }
 
-void memory_region_set_address(MemoryRegion *mr, target_phys_addr_t addr)
+void memory_region_set_address(MemoryRegion *mr, hwaddr addr)
 {
     MemoryRegion *parent = mr->parent;
     unsigned priority = mr->priority;
@@ -1350,18 +1397,18 @@ void memory_region_set_address(MemoryRegion *mr, target_phys_addr_t addr)
     memory_region_transaction_commit();
 }
 
-void memory_region_set_alias_offset(MemoryRegion *mr, target_phys_addr_t offset)
+void memory_region_set_alias_offset(MemoryRegion *mr, hwaddr offset)
 {
-    target_phys_addr_t old_offset = mr->alias_offset;
-
     assert(mr->alias);
-    mr->alias_offset = offset;
 
-    if (offset == old_offset || !mr->parent) {
+    if (offset == mr->alias_offset) {
         return;
     }
 
-    memory_region_update_topology(mr);
+    memory_region_transaction_begin();
+    mr->alias_offset = offset;
+    memory_region_update_pending |= mr->enabled;
+    memory_region_transaction_commit();
 }
 
 ram_addr_t memory_region_get_ram_addr(MemoryRegion *mr)
@@ -1384,12 +1431,12 @@ static int cmp_flatrange_addr(const void *addr_, const void *fr_)
 
 static FlatRange *address_space_lookup(AddressSpace *as, AddrRange addr)
 {
-    return bsearch(&addr, as->current_map.ranges, as->current_map.nr,
+    return bsearch(&addr, as->current_map->ranges, as->current_map->nr,
                    sizeof(FlatRange), cmp_flatrange_addr);
 }
 
 MemoryRegionSection memory_region_find(MemoryRegion *address_space,
-                                       target_phys_addr_t addr, uint64_t size)
+                                       hwaddr addr, uint64_t size)
 {
     AddressSpace *as = memory_region_to_address_space(address_space);
     AddrRange range = addrrange_make(int128_make64(addr),
@@ -1401,7 +1448,7 @@ MemoryRegionSection memory_region_find(MemoryRegion *address_space,
         return ret;
     }
 
-    while (fr > as->current_map.ranges
+    while (fr > as->current_map->ranges
            && addrrange_intersects(fr[-1].addr, range)) {
         --fr;
     }
@@ -1422,7 +1469,7 @@ void memory_global_sync_dirty_bitmap(MemoryRegion *address_space)
     AddressSpace *as = memory_region_to_address_space(address_space);
     FlatRange *fr;
 
-    FOR_EACH_FLAT_RANGE(fr, &as->current_map) {
+    FOR_EACH_FLAT_RANGE(fr, as->current_map) {
         MEMORY_LISTENER_UPDATE_REGION(fr, as, Forward, log_sync);
     }
 }
@@ -1445,29 +1492,35 @@ static void listener_add_address_space(MemoryListener *listener,
     FlatRange *fr;
 
     if (listener->address_space_filter
-        && listener->address_space_filter != as->root) {
+        && listener->address_space_filter != as) {
         return;
     }
 
     if (global_dirty_log) {
-        listener->log_global_start(listener);
+        if (listener->log_global_start) {
+            listener->log_global_start(listener);
+        }
     }
-    FOR_EACH_FLAT_RANGE(fr, &as->current_map) {
+
+    FOR_EACH_FLAT_RANGE(fr, as->current_map) {
         MemoryRegionSection section = {
             .mr = fr->mr,
-            .address_space = as->root,
+            .address_space = as,
             .offset_within_region = fr->offset_in_region,
             .size = int128_get64(fr->addr.size),
             .offset_within_address_space = int128_get64(fr->addr.start),
             .readonly = fr->readonly,
         };
-        listener->region_add(listener, &section);
+        if (listener->region_add) {
+            listener->region_add(listener, &section);
+        }
     }
 }
 
-void memory_listener_register(MemoryListener *listener, MemoryRegion *filter)
+void memory_listener_register(MemoryListener *listener, AddressSpace *filter)
 {
     MemoryListener *other = NULL;
+    AddressSpace *as;
 
     listener->address_space_filter = filter;
     if (QTAILQ_EMPTY(&memory_listeners)
@@ -1482,8 +1535,10 @@ void memory_listener_register(MemoryListener *listener, MemoryRegion *filter)
         }
         QTAILQ_INSERT_BEFORE(other, listener, link);
     }
-    listener_add_address_space(listener, &address_space_memory);
-    listener_add_address_space(listener, &address_space_io);
+
+    QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) {
+        listener_add_address_space(listener, as);
+    }
 }
 
 void memory_listener_unregister(MemoryListener *listener)
@@ -1491,24 +1546,36 @@ void memory_listener_unregister(MemoryListener *listener)
     QTAILQ_REMOVE(&memory_listeners, listener, link);
 }
 
-void set_system_memory_map(MemoryRegion *mr)
+void address_space_init(AddressSpace *as, MemoryRegion *root)
 {
-    address_space_memory.root = mr;
-    memory_region_update_topology(NULL);
+    memory_region_transaction_begin();
+    as->root = root;
+    as->current_map = g_new(FlatView, 1);
+    flatview_init(as->current_map);
+    QTAILQ_INSERT_TAIL(&address_spaces, as, address_spaces_link);
+    as->name = NULL;
+    memory_region_transaction_commit();
+    address_space_init_dispatch(as);
 }
 
-void set_system_io_map(MemoryRegion *mr)
+void address_space_destroy(AddressSpace *as)
 {
-    address_space_io.root = mr;
-    memory_region_update_topology(NULL);
+    /* Flush out anything from MemoryListeners listening in on this */
+    memory_region_transaction_begin();
+    as->root = NULL;
+    memory_region_transaction_commit();
+    QTAILQ_REMOVE(&address_spaces, as, address_spaces_link);
+    address_space_destroy_dispatch(as);
+    flatview_destroy(as->current_map);
+    g_free(as->current_map);
 }
 
-uint64_t io_mem_read(MemoryRegion *mr, target_phys_addr_t addr, unsigned size)
+uint64_t io_mem_read(MemoryRegion *mr, hwaddr addr, unsigned size)
 {
     return memory_region_dispatch_read(mr, addr, size);
 }
 
-void io_mem_write(MemoryRegion *mr, target_phys_addr_t addr,
+void io_mem_write(MemoryRegion *mr, hwaddr addr,
                   uint64_t val, unsigned size)
 {
     memory_region_dispatch_write(mr, addr, val, size);
@@ -1526,7 +1593,7 @@ typedef QTAILQ_HEAD(queue, MemoryRegionList) MemoryRegionListHead;
 
 static void mtree_print_mr(fprintf_function mon_printf, void *f,
                            const MemoryRegion *mr, unsigned int level,
-                           target_phys_addr_t base,
+                           hwaddr base,
                            MemoryRegionListHead *alias_print_queue)
 {
     MemoryRegionList *new_ml, *ml, *next_ml;
@@ -1534,7 +1601,7 @@ static void mtree_print_mr(fprintf_function mon_printf, void *f,
     const MemoryRegion *submr;
     unsigned int i;
 
-    if (!mr) {
+    if (!mr || !mr->enabled) {
         return;
     }
 
@@ -1564,7 +1631,7 @@ static void mtree_print_mr(fprintf_function mon_printf, void *f,
                    "-" TARGET_FMT_plx "\n",
                    base + mr->addr,
                    base + mr->addr
-                   + (target_phys_addr_t)int128_get64(mr->size) - 1,
+                   + (hwaddr)int128_get64(mr->size) - 1,
                    mr->priority,
                    mr->readable ? 'R' : '-',
                    !mr->readonly && !(mr->rom_device && mr->readable) ? 'W'
@@ -1573,13 +1640,13 @@ static void mtree_print_mr(fprintf_function mon_printf, void *f,
                    mr->alias->name,
                    mr->alias_offset,
                    mr->alias_offset
-                   + (target_phys_addr_t)int128_get64(mr->size) - 1);
+                   + (hwaddr)int128_get64(mr->size) - 1);
     } else {
         mon_printf(f,
                    TARGET_FMT_plx "-" TARGET_FMT_plx " (prio %d, %c%c): %s\n",
                    base + mr->addr,
                    base + mr->addr
-                   + (target_phys_addr_t)int128_get64(mr->size) - 1,
+                   + (hwaddr)int128_get64(mr->size) - 1,
                    mr->priority,
                    mr->readable ? 'R' : '-',
                    !mr->readonly && !(mr->rom_device && mr->readable) ? 'W'
@@ -1620,16 +1687,16 @@ void mtree_info(fprintf_function mon_printf, void *f)
 {
     MemoryRegionListHead ml_head;
     MemoryRegionList *ml, *ml2;
+    AddressSpace *as;
 
     QTAILQ_INIT(&ml_head);
 
-    mon_printf(f, "memory\n");
-    mtree_print_mr(mon_printf, f, address_space_memory.root, 0, 0, &ml_head);
-
-    if (address_space_io.root &&
-        !QTAILQ_EMPTY(&address_space_io.root->subregions)) {
-        mon_printf(f, "I/O\n");
-        mtree_print_mr(mon_printf, f, address_space_io.root, 0, 0, &ml_head);
+    QTAILQ_FOREACH(as, &address_spaces, address_spaces_link) {
+        if (!as->name) {
+            continue;
+        }
+        mon_printf(f, "%s\n", as->name);
+        mtree_print_mr(mon_printf, f, as->root, 0, 0, &ml_head);
     }
 
     mon_printf(f, "aliases\n");
diff --git a/memory.h b/memory.h
index bd1bbaeab..9462bfd3a 100644
--- a/memory.h
+++ b/memory.h
@@ -20,7 +20,7 @@
 #include <stdbool.h>
 #include "qemu-common.h"
 #include "cpu-common.h"
-#include "targphys.h"
+#include "hwaddr.h"
 #include "qemu-queue.h"
 #include "iorange.h"
 #include "ioport.h"
@@ -48,7 +48,7 @@ typedef struct MemoryRegionIORange MemoryRegionIORange;
 struct MemoryRegionIORange {
     IORange iorange;
     MemoryRegion *mr;
-    target_phys_addr_t offset;
+    hwaddr offset;
 };
 
 /*
@@ -58,12 +58,12 @@ struct MemoryRegionOps {
     /* Read from the memory region. @addr is relative to @mr; @size is
      * in bytes. */
     uint64_t (*read)(void *opaque,
-                     target_phys_addr_t addr,
+                     hwaddr addr,
                      unsigned size);
     /* Write to the memory region. @addr is relative to @mr; @size is
      * in bytes. */
     void (*write)(void *opaque,
-                  target_phys_addr_t addr,
+                  hwaddr addr,
                   uint64_t data,
                   unsigned size);
 
@@ -84,7 +84,7 @@ struct MemoryRegionOps {
          * by the device (and results in machine dependent behaviour such
          * as a machine check exception).
          */
-        bool (*accepts)(void *opaque, target_phys_addr_t addr,
+        bool (*accepts)(void *opaque, hwaddr addr,
                         unsigned size, bool is_write);
     } valid;
     /* Internal implementation constraints: */
@@ -122,7 +122,7 @@ struct MemoryRegion {
     void *opaque;
     MemoryRegion *parent;
     Int128 size;
-    target_phys_addr_t addr;
+    hwaddr addr;
     void (*destructor)(MemoryRegion *mr);
     ram_addr_t ram_addr;
     bool subpage;
@@ -133,8 +133,9 @@ struct MemoryRegion {
     bool enabled;
     bool rom_device;
     bool warning_printed; /* For reservations */
+    bool flush_coalesced_mmio;
     MemoryRegion *alias;
-    target_phys_addr_t alias_offset;
+    hwaddr alias_offset;
     unsigned priority;
     bool may_overlap;
     QTAILQ_HEAD(subregions, MemoryRegion) subregions;
@@ -156,6 +157,22 @@ struct MemoryRegionPortio {
 
 #define PORTIO_END_OF_LIST() { }
 
+typedef struct AddressSpace AddressSpace;
+
+/**
+ * AddressSpace: describes a mapping of addresses to #MemoryRegion objects
+ */
+struct AddressSpace {
+    /* All fields are private. */
+    const char *name;
+    MemoryRegion *root;
+    struct FlatView *current_map;
+    int ioeventfd_nb;
+    struct MemoryRegionIoeventfd *ioeventfds;
+    struct AddressSpaceDispatch *dispatch;
+    QTAILQ_ENTRY(AddressSpace) address_spaces_link;
+};
+
 typedef struct MemoryRegionSection MemoryRegionSection;
 
 /**
@@ -171,10 +188,10 @@ typedef struct MemoryRegionSection MemoryRegionSection;
  */
 struct MemoryRegionSection {
     MemoryRegion *mr;
-    MemoryRegion *address_space;
-    target_phys_addr_t offset_within_region;
+    AddressSpace *address_space;
+    hwaddr offset_within_region;
     uint64_t size;
-    target_phys_addr_t offset_within_address_space;
+    hwaddr offset_within_address_space;
     bool readonly;
 };
 
@@ -201,9 +218,13 @@ struct MemoryListener {
                         bool match_data, uint64_t data, EventNotifier *e);
     void (*eventfd_del)(MemoryListener *listener, MemoryRegionSection *section,
                         bool match_data, uint64_t data, EventNotifier *e);
+    void (*coalesced_mmio_add)(MemoryListener *listener, MemoryRegionSection *section,
+                               hwaddr addr, hwaddr len);
+    void (*coalesced_mmio_del)(MemoryListener *listener, MemoryRegionSection *section,
+                               hwaddr addr, hwaddr len);
     /* Lower = earlier (during add), later (during del) */
     unsigned priority;
-    MemoryRegion *address_space_filter;
+    AddressSpace *address_space_filter;
     QTAILQ_ENTRY(MemoryListener) link;
 };
 
@@ -252,9 +273,9 @@ void memory_region_init_ram(MemoryRegion *mr,
                             uint64_t size);
 
 /**
- * memory_region_init_ram:  Initialize RAM memory region from a user-provided.
- *                          pointer.  Accesses into the region will modify
- *                          memory directly.
+ * memory_region_init_ram_ptr:  Initialize RAM memory region from a
+ *                              user-provided pointer.  Accesses into the
+ *                              region will modify memory directly.
  *
  * @mr: the #MemoryRegion to be initialized.
  * @name: the name of the region.
@@ -280,7 +301,7 @@ void memory_region_init_ram_ptr(MemoryRegion *mr,
 void memory_region_init_alias(MemoryRegion *mr,
                               const char *name,
                               MemoryRegion *orig,
-                              target_phys_addr_t offset,
+                              hwaddr offset,
                               uint64_t size);
 
 /**
@@ -416,8 +437,8 @@ void memory_region_set_log(MemoryRegion *mr, bool log, unsigned client);
  * @client: the user of the logging information; %DIRTY_MEMORY_MIGRATION or
  *          %DIRTY_MEMORY_VGA.
  */
-bool memory_region_get_dirty(MemoryRegion *mr, target_phys_addr_t addr,
-                             target_phys_addr_t size, unsigned client);
+bool memory_region_get_dirty(MemoryRegion *mr, hwaddr addr,
+                             hwaddr size, unsigned client);
 
 /**
  * memory_region_set_dirty: Mark a range of bytes as dirty in a memory region.
@@ -429,8 +450,8 @@ bool memory_region_get_dirty(MemoryRegion *mr, target_phys_addr_t addr,
  * @addr: the address (relative to the start of the region) being dirtied.
  * @size: size of the range being dirtied.
  */
-void memory_region_set_dirty(MemoryRegion *mr, target_phys_addr_t addr,
-                             target_phys_addr_t size);
+void memory_region_set_dirty(MemoryRegion *mr, hwaddr addr,
+                             hwaddr size);
 
 /**
  * memory_region_sync_dirty_bitmap: Synchronize a region's dirty bitmap with
@@ -455,8 +476,8 @@ void memory_region_sync_dirty_bitmap(MemoryRegion *mr);
  * @client: the user of the logging information; %DIRTY_MEMORY_MIGRATION or
  *          %DIRTY_MEMORY_VGA.
  */
-void memory_region_reset_dirty(MemoryRegion *mr, target_phys_addr_t addr,
-                               target_phys_addr_t size, unsigned client);
+void memory_region_reset_dirty(MemoryRegion *mr, hwaddr addr,
+                               hwaddr size, unsigned client);
 
 /**
  * memory_region_set_readonly: Turn a memory region read-only (or read-write)
@@ -506,7 +527,7 @@ void memory_region_set_coalescing(MemoryRegion *mr);
  * @size: the size of the subrange to be coalesced.
  */
 void memory_region_add_coalescing(MemoryRegion *mr,
-                                  target_phys_addr_t offset,
+                                  hwaddr offset,
                                   uint64_t size);
 
 /**
@@ -521,6 +542,31 @@ void memory_region_add_coalescing(MemoryRegion *mr,
 void memory_region_clear_coalescing(MemoryRegion *mr);
 
 /**
+ * memory_region_set_flush_coalesced: Enforce memory coalescing flush before
+ *                                    accesses.
+ *
+ * Ensure that pending coalesced MMIO request are flushed before the memory
+ * region is accessed. This property is automatically enabled for all regions
+ * passed to memory_region_set_coalescing() and memory_region_add_coalescing().
+ *
+ * @mr: the memory region to be updated.
+ */
+void memory_region_set_flush_coalesced(MemoryRegion *mr);
+
+/**
+ * memory_region_clear_flush_coalesced: Disable memory coalescing flush before
+ *                                      accesses.
+ *
+ * Clear the automatic coalesced MMIO flushing enabled via
+ * memory_region_set_flush_coalesced. Note that this service has no effect on
+ * memory regions that have MMIO coalescing enabled for themselves. For them,
+ * automatic flushing will stop once coalescing is disabled.
+ *
+ * @mr: the memory region to be updated.
+ */
+void memory_region_clear_flush_coalesced(MemoryRegion *mr);
+
+/**
  * memory_region_add_eventfd: Request an eventfd to be triggered when a word
  *                            is written to a location.
  *
@@ -537,7 +583,7 @@ void memory_region_clear_coalescing(MemoryRegion *mr);
  * @fd: the eventfd to be triggered when @addr, @size, and @data all match.
  **/
 void memory_region_add_eventfd(MemoryRegion *mr,
-                               target_phys_addr_t addr,
+                               hwaddr addr,
                                unsigned size,
                                bool match_data,
                                uint64_t data,
@@ -557,7 +603,7 @@ void memory_region_add_eventfd(MemoryRegion *mr,
  * @fd: the eventfd to be triggered when @addr, @size, and @data all match.
  */
 void memory_region_del_eventfd(MemoryRegion *mr,
-                               target_phys_addr_t addr,
+                               hwaddr addr,
                                unsigned size,
                                bool match_data,
                                uint64_t data,
@@ -578,10 +624,11 @@ void memory_region_del_eventfd(MemoryRegion *mr,
  * @subregion: the subregion to be added.
  */
 void memory_region_add_subregion(MemoryRegion *mr,
-                                 target_phys_addr_t offset,
+                                 hwaddr offset,
                                  MemoryRegion *subregion);
 /**
- * memory_region_add_subregion: Add a subregion to a container, with overlap.
+ * memory_region_add_subregion_overlap: Add a subregion to a container
+ *                                      with overlap.
  *
  * Adds a subregion at @offset.  The subregion may overlap with other
  * subregions.  Conflicts are resolved by having a higher @priority hide a
@@ -597,7 +644,7 @@ void memory_region_add_subregion(MemoryRegion *mr,
  * @priority: used for resolving overlaps; highest priority wins.
  */
 void memory_region_add_subregion_overlap(MemoryRegion *mr,
-                                         target_phys_addr_t offset,
+                                         hwaddr offset,
                                          MemoryRegion *subregion,
                                          unsigned priority);
 
@@ -645,7 +692,7 @@ void memory_region_set_enabled(MemoryRegion *mr, bool enabled);
  * @mr: the region to be updated
  * @addr: new address, relative to parent region
  */
-void memory_region_set_address(MemoryRegion *mr, target_phys_addr_t addr);
+void memory_region_set_address(MemoryRegion *mr, hwaddr addr);
 
 /*
  * memory_region_set_alias_offset: dynamically update a memory alias's offset
@@ -657,7 +704,7 @@ void memory_region_set_address(MemoryRegion *mr, target_phys_addr_t addr);
  * @offset: the new offset into the target memory region
  */
 void memory_region_set_alias_offset(MemoryRegion *mr,
-                                    target_phys_addr_t offset);
+                                    hwaddr offset);
 
 /**
  * memory_region_find: locate a MemoryRegion in an address space
@@ -678,7 +725,7 @@ void memory_region_set_alias_offset(MemoryRegion *mr,
  * @size: size of the area to be searched
  */
 MemoryRegionSection memory_region_find(MemoryRegion *address_space,
-                                       target_phys_addr_t addr, uint64_t size);
+                                       hwaddr addr, uint64_t size);
 
 /**
  * memory_region_section_addr: get offset within MemoryRegionSection
@@ -688,9 +735,9 @@ MemoryRegionSection memory_region_find(MemoryRegion *address_space,
  * @section: the memory region section being queried
  * @addr: address in address space
  */
-static inline target_phys_addr_t
+static inline hwaddr
 memory_region_section_addr(MemoryRegionSection *section,
-                           target_phys_addr_t addr)
+                           hwaddr addr)
 {
     addr -= section->offset_within_address_space;
     addr += section->offset_within_region;
@@ -728,7 +775,7 @@ void memory_region_transaction_commit(void);
  * @listener: an object containing the callbacks to be called
  * @filter: if non-%NULL, only regions in this address space will be observed
  */
-void memory_listener_register(MemoryListener *listener, MemoryRegion *filter);
+void memory_listener_register(MemoryListener *listener, AddressSpace *filter);
 
 /**
  * memory_listener_unregister: undo the effect of memory_listener_register()
@@ -743,12 +790,93 @@ void memory_listener_unregister(MemoryListener *listener);
 void memory_global_dirty_log_start(void);
 
 /**
- * memory_global_dirty_log_stop: begin dirty logging for all regions
+ * memory_global_dirty_log_stop: end dirty logging for all regions
  */
 void memory_global_dirty_log_stop(void);
 
 void mtree_info(fprintf_function mon_printf, void *f);
 
+/**
+ * address_space_init: initializes an address space
+ *
+ * @as: an uninitialized #AddressSpace
+ * @root: a #MemoryRegion that routes addesses for the address space
+ */
+void address_space_init(AddressSpace *as, MemoryRegion *root);
+
+
+/**
+ * address_space_destroy: destroy an address space
+ *
+ * Releases all resources associated with an address space.  After an address space
+ * is destroyed, its root memory region (given by address_space_init()) may be destroyed
+ * as well.
+ *
+ * @as: address space to be destroyed
+ */
+void address_space_destroy(AddressSpace *as);
+
+/**
+ * address_space_rw: read from or write to an address space.
+ *
+ * @as: #AddressSpace to be accessed
+ * @addr: address within that address space
+ * @buf: buffer with the data transferred
+ * @is_write: indicates the transfer direction
+ */
+void address_space_rw(AddressSpace *as, hwaddr addr, uint8_t *buf,
+                      int len, bool is_write);
+
+/**
+ * address_space_write: write to address space.
+ *
+ * @as: #AddressSpace to be accessed
+ * @addr: address within that address space
+ * @buf: buffer with the data transferred
+ */
+void address_space_write(AddressSpace *as, hwaddr addr,
+                         const uint8_t *buf, int len);
+
+/**
+ * address_space_read: read from an address space.
+ *
+ * @as: #AddressSpace to be accessed
+ * @addr: address within that address space
+ * @buf: buffer with the data transferred
+ */
+void address_space_read(AddressSpace *as, hwaddr addr, uint8_t *buf, int len);
+
+/* address_space_map: map a physical memory region into a host virtual address
+ *
+ * May map a subset of the requested range, given by and returned in @plen.
+ * May return %NULL if resources needed to perform the mapping are exhausted.
+ * Use only for reads OR writes - not for read-modify-write operations.
+ * Use cpu_register_map_client() to know when retrying the map operation is
+ * likely to succeed.
+ *
+ * @as: #AddressSpace to be accessed
+ * @addr: address within that address space
+ * @plen: pointer to length of buffer; updated on return
+ * @is_write: indicates the transfer direction
+ */
+void *address_space_map(AddressSpace *as, hwaddr addr,
+                        hwaddr *plen, bool is_write);
+
+/* address_space_unmap: Unmaps a memory region previously mapped by address_space_map()
+ *
+ * Will also mark the memory as dirty if @is_write == %true.  @access_len gives
+ * the amount of memory that was actually read or written by the caller.
+ *
+ * @as: #AddressSpace used
+ * @addr: address within that address space
+ * @len: buffer length as returned by address_space_map()
+ * @access_len: amount of data actually transferred
+ * @is_write: indicates the transfer direction
+ */
+void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len,
+                         int is_write, hwaddr access_len);
+
+
 #endif
 
 #endif
diff --git a/memory_mapping.c b/memory_mapping.c
index 6f5a2e3f7..a82e190ca 100644
--- a/memory_mapping.c
+++ b/memory_mapping.c
@@ -30,8 +30,8 @@ static void memory_mapping_list_add_mapping_sorted(MemoryMappingList *list,
 }
 
 static void create_new_memory_mapping(MemoryMappingList *list,
-                                      target_phys_addr_t phys_addr,
-                                      target_phys_addr_t virt_addr,
+                                      hwaddr phys_addr,
+                                      hwaddr virt_addr,
                                       ram_addr_t length)
 {
     MemoryMapping *memory_mapping;
@@ -46,8 +46,8 @@ static void create_new_memory_mapping(MemoryMappingList *list,
 }
 
 static inline bool mapping_contiguous(MemoryMapping *map,
-                                      target_phys_addr_t phys_addr,
-                                      target_phys_addr_t virt_addr)
+                                      hwaddr phys_addr,
+                                      hwaddr virt_addr)
 {
     return phys_addr == map->phys_addr + map->length &&
            virt_addr == map->virt_addr + map->length;
@@ -58,7 +58,7 @@ static inline bool mapping_contiguous(MemoryMapping *map,
  * [phys_addr, phys_addr + length) have intersection?
  */
 static inline bool mapping_have_same_region(MemoryMapping *map,
-                                            target_phys_addr_t phys_addr,
+                                            hwaddr phys_addr,
                                             ram_addr_t length)
 {
     return !(phys_addr + length < map->phys_addr ||
@@ -71,8 +71,8 @@ static inline bool mapping_have_same_region(MemoryMapping *map,
  * intersection are the same?
  */
 static inline bool mapping_conflict(MemoryMapping *map,
-                                    target_phys_addr_t phys_addr,
-                                    target_phys_addr_t virt_addr)
+                                    hwaddr phys_addr,
+                                    hwaddr virt_addr)
 {
     return virt_addr - map->virt_addr != phys_addr - map->phys_addr;
 }
@@ -83,7 +83,7 @@ static inline bool mapping_conflict(MemoryMapping *map,
  * in the intersection are the same.
  */
 static inline void mapping_merge(MemoryMapping *map,
-                                 target_phys_addr_t virt_addr,
+                                 hwaddr virt_addr,
                                  ram_addr_t length)
 {
     if (virt_addr < map->virt_addr) {
@@ -98,8 +98,8 @@ static inline void mapping_merge(MemoryMapping *map,
 }
 
 void memory_mapping_list_add_merge_sorted(MemoryMappingList *list,
-                                          target_phys_addr_t phys_addr,
-                                          target_phys_addr_t virt_addr,
+                                          hwaddr phys_addr,
+                                          hwaddr virt_addr,
                                           ram_addr_t length)
 {
     MemoryMapping *memory_mapping, *last_mapping;
diff --git a/memory_mapping.h b/memory_mapping.h
index ef72b0aba..d5ba46c7e 100644
--- a/memory_mapping.h
+++ b/memory_mapping.h
@@ -18,7 +18,7 @@
 
 /* The physical and virtual address in the memory mapping are contiguous. */
 typedef struct MemoryMapping {
-    target_phys_addr_t phys_addr;
+    hwaddr phys_addr;
     target_ulong virt_addr;
     ram_addr_t length;
     QTAILQ_ENTRY(MemoryMapping) next;
@@ -39,8 +39,8 @@ bool cpu_paging_enabled(CPUArchState *env);
  * and is contiguous. The list is sorted by phys_addr.
  */
 void memory_mapping_list_add_merge_sorted(MemoryMappingList *list,
-                                          target_phys_addr_t phys_addr,
-                                          target_phys_addr_t virt_addr,
+                                          hwaddr phys_addr,
+                                          hwaddr virt_addr,
                                           ram_addr_t length);
 
 void memory_mapping_list_free(MemoryMappingList *list);
diff --git a/migration-exec.c b/migration-exec.c
index 6c97db973..2b6fcb426 100644
--- a/migration-exec.c
+++ b/migration-exec.c
@@ -48,34 +48,28 @@ static int exec_close(MigrationState *s)
 {
     int ret = 0;
     DPRINTF("exec_close\n");
-    if (s->opaque) {
-        ret = qemu_fclose(s->opaque);
-        s->opaque = NULL;
-        s->fd = -1;
-        if (ret >= 0 && !(WIFEXITED(ret) && WEXITSTATUS(ret) == 0)) {
-            /* close succeeded, but non-zero exit code: */
-            ret = -EIO; /* fake errno value */
-        }
+    ret = qemu_fclose(s->opaque);
+    s->opaque = NULL;
+    s->fd = -1;
+    if (ret >= 0 && !(WIFEXITED(ret) && WEXITSTATUS(ret) == 0)) {
+        /* close succeeded, but non-zero exit code: */
+        ret = -EIO; /* fake errno value */
     }
     return ret;
 }
 
-int exec_start_outgoing_migration(MigrationState *s, const char *command)
+void exec_start_outgoing_migration(MigrationState *s, const char *command, Error **errp)
 {
     FILE *f;
 
     f = popen(command, "w");
     if (f == NULL) {
-        DPRINTF("Unable to popen exec target\n");
-        goto err_after_popen;
+        error_setg_errno(errp, errno, "failed to popen the migration target");
+        return;
     }
 
     s->fd = fileno(f);
-    if (s->fd == -1) {
-        DPRINTF("Unable to retrieve file descriptor for popen'd handle\n");
-        goto err_after_open;
-    }
-
+    assert(s->fd != -1);
     socket_set_nonblock(s->fd);
 
     s->opaque = qemu_popen(f, "w");
@@ -85,36 +79,27 @@ int exec_start_outgoing_migration(MigrationState *s, const char *command)
     s->write = file_write;
 
     migrate_fd_connect(s);
-    return 0;
-
-err_after_open:
-    pclose(f);
-err_after_popen:
-    return -1;
 }
 
 static void exec_accept_incoming_migration(void *opaque)
 {
     QEMUFile *f = opaque;
 
+    qemu_set_fd_handler2(qemu_get_fd(f), NULL, NULL, NULL, NULL);
     process_incoming_migration(f);
-    qemu_set_fd_handler2(qemu_stdio_fd(f), NULL, NULL, NULL, NULL);
-    qemu_fclose(f);
 }
 
-int exec_start_incoming_migration(const char *command)
+void exec_start_incoming_migration(const char *command, Error **errp)
 {
     QEMUFile *f;
 
     DPRINTF("Attempting to start an incoming migration\n");
     f = qemu_popen_cmd(command, "r");
     if(f == NULL) {
-        DPRINTF("Unable to apply qemu wrapper to popen file\n");
-        return -errno;
+        error_setg_errno(errp, errno, "failed to popen the migration source");
+        return;
     }
 
-    qemu_set_fd_handler2(qemu_stdio_fd(f), NULL,
+    qemu_set_fd_handler2(qemu_get_fd(f), NULL,
 			 exec_accept_incoming_migration, NULL, f);
-
-    return 0;
 }
diff --git a/migration-fd.c b/migration-fd.c
index 50138edb3..5fe28e09f 100644
--- a/migration-fd.c
+++ b/migration-fd.c
@@ -48,67 +48,52 @@ static int fd_close(MigrationState *s)
     int ret;
 
     DPRINTF("fd_close\n");
-    if (s->fd != -1) {
-        ret = fstat(s->fd, &st);
-        if (ret == 0 && S_ISREG(st.st_mode)) {
-            /*
-             * If the file handle is a regular file make sure the
-             * data is flushed to disk before signaling success.
-             */
-            ret = fsync(s->fd);
-            if (ret != 0) {
-                ret = -errno;
-                perror("migration-fd: fsync");
-                return ret;
-            }
-        }
-        ret = close(s->fd);
-        s->fd = -1;
+    ret = fstat(s->fd, &st);
+    if (ret == 0 && S_ISREG(st.st_mode)) {
+        /*
+         * If the file handle is a regular file make sure the
+         * data is flushed to disk before signaling success.
+         */
+        ret = fsync(s->fd);
         if (ret != 0) {
             ret = -errno;
-            perror("migration-fd: close");
+            perror("migration-fd: fsync");
             return ret;
         }
     }
-    return 0;
+    ret = close(s->fd);
+    s->fd = -1;
+    if (ret != 0) {
+        ret = -errno;
+        perror("migration-fd: close");
+    }
+    return ret;
 }
 
-int fd_start_outgoing_migration(MigrationState *s, const char *fdname)
+void fd_start_outgoing_migration(MigrationState *s, const char *fdname, Error **errp)
 {
-    s->fd = monitor_get_fd(cur_mon, fdname);
+    s->fd = monitor_get_fd(cur_mon, fdname, errp);
     if (s->fd == -1) {
-        DPRINTF("fd_migration: invalid file descriptor identifier\n");
-        goto err_after_get_fd;
-    }
-
-    if (fcntl(s->fd, F_SETFL, O_NONBLOCK) == -1) {
-        DPRINTF("Unable to set nonblocking mode on file descriptor\n");
-        goto err_after_open;
+        return;
     }
 
+    fcntl(s->fd, F_SETFL, O_NONBLOCK);
     s->get_error = fd_errno;
     s->write = fd_write;
     s->close = fd_close;
 
     migrate_fd_connect(s);
-    return 0;
-
-err_after_open:
-    close(s->fd);
-err_after_get_fd:
-    return -1;
 }
 
 static void fd_accept_incoming_migration(void *opaque)
 {
     QEMUFile *f = opaque;
 
+    qemu_set_fd_handler2(qemu_get_fd(f), NULL, NULL, NULL, NULL);
     process_incoming_migration(f);
-    qemu_set_fd_handler2(qemu_stdio_fd(f), NULL, NULL, NULL, NULL);
-    qemu_fclose(f);
 }
 
-int fd_start_incoming_migration(const char *infd)
+void fd_start_incoming_migration(const char *infd, Error **errp)
 {
     int fd;
     QEMUFile *f;
@@ -118,11 +103,9 @@ int fd_start_incoming_migration(const char *infd)
     fd = strtol(infd, NULL, 0);
     f = qemu_fdopen(fd, "rb");
     if(f == NULL) {
-        DPRINTF("Unable to apply qemu wrapper to file descriptor\n");
-        return -errno;
+        error_setg_errno(errp, errno, "failed to open the source descriptor");
+        return;
     }
 
     qemu_set_fd_handler2(fd, NULL, fd_accept_incoming_migration, NULL, f);
-
-    return 0;
 }
diff --git a/migration-tcp.c b/migration-tcp.c
index ac891c38a..5e855fe72 100644
--- a/migration-tcp.c
+++ b/migration-tcp.c
@@ -44,64 +44,34 @@ static int tcp_close(MigrationState *s)
 {
     int r = 0;
     DPRINTF("tcp_close\n");
-    if (s->fd != -1) {
-        if (close(s->fd) < 0) {
-            r = -errno;
-        }
-        s->fd = -1;
+    if (closesocket(s->fd) < 0) {
+        r = -socket_error();
     }
     return r;
 }
 
-static void tcp_wait_for_connect(void *opaque)
+static void tcp_wait_for_connect(int fd, void *opaque)
 {
     MigrationState *s = opaque;
-    int val, ret;
-    socklen_t valsize = sizeof(val);
-
-    DPRINTF("connect completed\n");
-    do {
-        ret = getsockopt(s->fd, SOL_SOCKET, SO_ERROR, (void *) &val, &valsize);
-    } while (ret == -1 && (socket_error()) == EINTR);
 
-    if (ret < 0) {
+    if (fd < 0) {
+        DPRINTF("migrate connect error\n");
+        s->fd = -1;
         migrate_fd_error(s);
-        return;
-    }
-
-    qemu_set_fd_handler2(s->fd, NULL, NULL, NULL, NULL);
-
-    if (val == 0)
+    } else {
+        DPRINTF("migrate connect success\n");
+        s->fd = fd;
         migrate_fd_connect(s);
-    else {
-        DPRINTF("error connecting %d\n", val);
-        migrate_fd_error(s);
     }
 }
 
-int tcp_start_outgoing_migration(MigrationState *s, const char *host_port,
-                                 Error **errp)
+void tcp_start_outgoing_migration(MigrationState *s, const char *host_port, Error **errp)
 {
-    bool in_progress;
-
     s->get_error = socket_errno;
     s->write = socket_write;
     s->close = tcp_close;
 
-    s->fd = inet_connect(host_port, false, &in_progress, errp);
-    if (error_is_set(errp)) {
-        migrate_fd_error(s);
-        return -1;
-    }
-
-    if (in_progress) {
-        DPRINTF("connect in progress\n");
-        qemu_set_fd_handler2(s->fd, NULL, NULL, tcp_wait_for_connect, s);
-    } else {
-        migrate_fd_connect(s);
-    }
-
-    return 0;
+    s->fd = inet_nonblocking_connect(host_port, tcp_wait_for_connect, s, errp);
 }
 
 static void tcp_accept_incoming_migration(void *opaque)
@@ -115,12 +85,14 @@ static void tcp_accept_incoming_migration(void *opaque)
     do {
         c = qemu_accept(s, (struct sockaddr *)&addr, &addrlen);
     } while (c == -1 && socket_error() == EINTR);
+    qemu_set_fd_handler2(s, NULL, NULL, NULL, NULL);
+    closesocket(s);
 
     DPRINTF("accepted migration\n");
 
     if (c == -1) {
         fprintf(stderr, "could not accept migration connection\n");
-        goto out2;
+        goto out;
     }
 
     f = qemu_fopen_socket(c);
@@ -130,26 +102,21 @@ static void tcp_accept_incoming_migration(void *opaque)
     }
 
     process_incoming_migration(f);
-    qemu_fclose(f);
+    return;
+
 out:
-    close(c);
-out2:
-    qemu_set_fd_handler2(s, NULL, NULL, NULL, NULL);
-    close(s);
+    closesocket(c);
 }
 
-int tcp_start_incoming_migration(const char *host_port, Error **errp)
+void tcp_start_incoming_migration(const char *host_port, Error **errp)
 {
     int s;
 
     s = inet_listen(host_port, NULL, 256, SOCK_STREAM, 0, errp);
-
     if (s < 0) {
-        return -1;
+        return;
     }
 
     qemu_set_fd_handler2(s, NULL, tcp_accept_incoming_migration, NULL,
                          (void *)(intptr_t)s);
-
-    return 0;
 }
diff --git a/migration-unix.c b/migration-unix.c
index 169de8867..dba72b4a5 100644
--- a/migration-unix.c
+++ b/migration-unix.c
@@ -44,78 +44,34 @@ static int unix_close(MigrationState *s)
 {
     int r = 0;
     DPRINTF("unix_close\n");
-    if (s->fd != -1) {
-        if (close(s->fd) < 0) {
-            r = -errno;
-        }
-        s->fd = -1;
+    if (close(s->fd) < 0) {
+        r = -errno;
     }
     return r;
 }
 
-static void unix_wait_for_connect(void *opaque)
+static void unix_wait_for_connect(int fd, void *opaque)
 {
     MigrationState *s = opaque;
-    int val, ret;
-    socklen_t valsize = sizeof(val);
-
-    DPRINTF("connect completed\n");
-    do {
-        ret = getsockopt(s->fd, SOL_SOCKET, SO_ERROR, (void *) &val, &valsize);
-    } while (ret == -1 && errno == EINTR);
 
-    if (ret < 0) {
+    if (fd < 0) {
+        DPRINTF("migrate connect error\n");
+        s->fd = -1;
         migrate_fd_error(s);
-        return;
-    }
-
-    qemu_set_fd_handler2(s->fd, NULL, NULL, NULL, NULL);
-
-    if (val == 0)
+    } else {
+        DPRINTF("migrate connect success\n");
+        s->fd = fd;
         migrate_fd_connect(s);
-    else {
-        DPRINTF("error connecting %d\n", val);
-        migrate_fd_error(s);
     }
 }
 
-int unix_start_outgoing_migration(MigrationState *s, const char *path)
+void unix_start_outgoing_migration(MigrationState *s, const char *path, Error **errp)
 {
-    struct sockaddr_un addr;
-    int ret;
-
-    addr.sun_family = AF_UNIX;
-    snprintf(addr.sun_path, sizeof(addr.sun_path), "%s", path);
     s->get_error = unix_errno;
     s->write = unix_write;
     s->close = unix_close;
 
-    s->fd = qemu_socket(PF_UNIX, SOCK_STREAM, 0);
-    if (s->fd == -1) {
-        DPRINTF("Unable to open socket");
-        return -errno;
-    }
-
-    socket_set_nonblock(s->fd);
-
-    do {
-        ret = connect(s->fd, (struct sockaddr *)&addr, sizeof(addr));
-        if (ret == -1) {
-            ret = -errno;
-        }
-        if (ret == -EINPROGRESS || ret == -EWOULDBLOCK) {
-	    qemu_set_fd_handler2(s->fd, NULL, NULL, unix_wait_for_connect, s);
-            return 0;
-        }
-    } while (ret == -EINTR);
-
-    if (ret < 0) {
-        DPRINTF("connect failed\n");
-        migrate_fd_error(s);
-        return ret;
-    }
-    migrate_fd_connect(s);
-    return 0;
+    s->fd = unix_nonblocking_connect(path, unix_wait_for_connect, s, errp);
 }
 
 static void unix_accept_incoming_migration(void *opaque)
@@ -129,12 +85,14 @@ static void unix_accept_incoming_migration(void *opaque)
     do {
         c = qemu_accept(s, (struct sockaddr *)&addr, &addrlen);
     } while (c == -1 && errno == EINTR);
+    qemu_set_fd_handler2(s, NULL, NULL, NULL, NULL);
+    close(s);
 
     DPRINTF("accepted migration\n");
 
     if (c == -1) {
         fprintf(stderr, "could not accept migration connection\n");
-        goto out2;
+        goto out;
     }
 
     f = qemu_fopen_socket(c);
@@ -144,51 +102,21 @@ static void unix_accept_incoming_migration(void *opaque)
     }
 
     process_incoming_migration(f);
-    qemu_fclose(f);
+    return;
+
 out:
     close(c);
-out2:
-    qemu_set_fd_handler2(s, NULL, NULL, NULL, NULL);
-    close(s);
 }
 
-int unix_start_incoming_migration(const char *path)
+void unix_start_incoming_migration(const char *path, Error **errp)
 {
-    struct sockaddr_un addr;
     int s;
-    int ret;
-
-    DPRINTF("Attempting to start an incoming migration\n");
 
-    s = qemu_socket(PF_UNIX, SOCK_STREAM, 0);
-    if (s == -1) {
-        fprintf(stderr, "Could not open unix socket: %s\n", strerror(errno));
-        return -errno;
-    }
-
-    memset(&addr, 0, sizeof(addr));
-    addr.sun_family = AF_UNIX;
-    snprintf(addr.sun_path, sizeof(addr.sun_path), "%s", path);
-
-    unlink(addr.sun_path);
-    if (bind(s, (struct sockaddr *) &addr, sizeof(addr)) < 0) {
-        ret = -errno;
-        fprintf(stderr, "bind(unix:%s): %s\n", addr.sun_path, strerror(errno));
-        goto err;
-    }
-    if (listen(s, 1) == -1) {
-        fprintf(stderr, "listen(unix:%s): %s\n", addr.sun_path,
-                strerror(errno));
-        ret = -errno;
-        goto err;
+    s = unix_listen(path, NULL, 0, errp);
+    if (s < 0) {
+        return;
     }
 
     qemu_set_fd_handler2(s, NULL, unix_accept_incoming_migration, NULL,
                          (void *)(intptr_t)s);
-
-    return 0;
-
-err:
-    close(s);
-    return ret;
 }
diff --git a/migration.c b/migration.c
index 1edeec5dd..73ce170dd 100644
--- a/migration.c
+++ b/migration.c
@@ -53,7 +53,7 @@ static NotifierList migration_state_notifiers =
    migrations at once.  For now we don't need to add
    dynamic creation of migration */
 
-static MigrationState *migrate_get_current(void)
+MigrationState *migrate_get_current(void)
 {
     static MigrationState current_migration = {
         .state = MIG_STATE_SETUP,
@@ -64,31 +64,34 @@ static MigrationState *migrate_get_current(void)
     return &current_migration;
 }
 
-int qemu_start_incoming_migration(const char *uri, Error **errp)
+void qemu_start_incoming_migration(const char *uri, Error **errp)
 {
     const char *p;
-    int ret;
 
     if (strstart(uri, "tcp:", &p))
-        ret = tcp_start_incoming_migration(p, errp);
+        tcp_start_incoming_migration(p, errp);
 #if !defined(WIN32)
     else if (strstart(uri, "exec:", &p))
-        ret =  exec_start_incoming_migration(p);
+        exec_start_incoming_migration(p, errp);
     else if (strstart(uri, "unix:", &p))
-        ret = unix_start_incoming_migration(p);
+        unix_start_incoming_migration(p, errp);
     else if (strstart(uri, "fd:", &p))
-        ret = fd_start_incoming_migration(p);
+        fd_start_incoming_migration(p, errp);
 #endif
     else {
-        fprintf(stderr, "unknown migration protocol: %s\n", uri);
-        ret = -EPROTONOSUPPORT;
+        error_setg(errp, "unknown migration protocol: %s\n", uri);
     }
-    return ret;
 }
 
-void process_incoming_migration(QEMUFile *f)
+static void process_incoming_migration_co(void *opaque)
 {
-    if (qemu_loadvm_state(f) < 0) {
+    QEMUFile *f = opaque;
+    int ret;
+
+    ret = qemu_loadvm_state(f);
+    qemu_set_fd_handler(qemu_get_fd(f), NULL, NULL, NULL);
+    qemu_fclose(f);
+    if (ret < 0) {
         fprintf(stderr, "load of migration failed\n");
         exit(0);
     }
@@ -102,10 +105,27 @@ void process_incoming_migration(QEMUFile *f)
     if (autostart) {
         vm_start();
     } else {
-        runstate_set(RUN_STATE_PRELAUNCH);
+        runstate_set(RUN_STATE_PAUSED);
     }
 }
 
+static void enter_migration_coroutine(void *opaque)
+{
+    Coroutine *co = opaque;
+    qemu_coroutine_enter(co, NULL);
+}
+
+void process_incoming_migration(QEMUFile *f)
+{
+    Coroutine *co = qemu_coroutine_create(process_incoming_migration_co);
+    int fd = qemu_get_fd(f);
+
+    assert(fd != -1);
+    socket_set_nonblock(fd);
+    qemu_set_fd_handler(fd, enter_migration_coroutine, NULL, co);
+    qemu_coroutine_enter(co, f);
+}
+
 /* amount of nanoseconds we are willing to wait for migration to be down.
  * the choice of nanoseconds is because it is the maximum resolution that
  * get_clock() can achieve. It is an internal measure. All user-visible
@@ -169,6 +189,8 @@ MigrationInfo *qmp_query_migrate(Error **errp)
         info->has_total_time = true;
         info->total_time = qemu_get_clock_ms(rt_clock)
             - s->total_time;
+        info->has_expected_downtime = true;
+        info->expected_downtime = s->expected_downtime;
 
         info->has_ram = true;
         info->ram = g_malloc0(sizeof(*info->ram));
@@ -178,6 +200,8 @@ MigrationInfo *qmp_query_migrate(Error **errp)
         info->ram->duplicate = dup_mig_pages_transferred();
         info->ram->normal = norm_mig_pages_transferred();
         info->ram->normal_bytes = norm_mig_bytes_transferred();
+        info->ram->dirty_pages_rate = s->dirty_pages_rate;
+
 
         if (blk_mig_active()) {
             info->has_disk = true;
@@ -195,6 +219,8 @@ MigrationInfo *qmp_query_migrate(Error **errp)
         info->has_status = true;
         info->status = g_strdup("completed");
         info->total_time = s->total_time;
+        info->has_downtime = true;
+        info->downtime = s->downtime;
 
         info->has_ram = true;
         info->ram = g_malloc0(sizeof(*info->ram));
@@ -240,19 +266,13 @@ static int migrate_fd_cleanup(MigrationState *s)
 {
     int ret = 0;
 
-    qemu_set_fd_handler2(s->fd, NULL, NULL, NULL, NULL);
-
     if (s->file) {
         DPRINTF("closing file\n");
         ret = qemu_fclose(s->file);
         s->file = NULL;
     }
 
-    if (s->fd != -1) {
-        close(s->fd);
-        s->fd = -1;
-    }
-
+    migrate_fd_close(s);
     return ret;
 }
 
@@ -279,18 +299,18 @@ static void migrate_fd_completed(MigrationState *s)
 static void migrate_fd_put_notify(void *opaque)
 {
     MigrationState *s = opaque;
+    int ret;
 
     qemu_set_fd_handler2(s->fd, NULL, NULL, NULL, NULL);
-    qemu_file_put_notify(s->file);
-    if (s->file && qemu_file_get_error(s->file)) {
+    ret = qemu_file_put_notify(s->file);
+    if (ret) {
         migrate_fd_error(s);
     }
 }
 
-static ssize_t migrate_fd_put_buffer(void *opaque, const void *data,
-                                     size_t size)
+ssize_t migrate_fd_put_buffer(MigrationState *s, const void *data,
+                              size_t size)
 {
-    MigrationState *s = opaque;
     ssize_t ret;
 
     if (s->state != MIG_STATE_ACTIVE) {
@@ -311,9 +331,8 @@ static ssize_t migrate_fd_put_buffer(void *opaque, const void *data,
     return ret;
 }
 
-static void migrate_fd_put_ready(void *opaque)
+void migrate_fd_put_ready(MigrationState *s)
 {
-    MigrationState *s = opaque;
     int ret;
 
     if (s->state != MIG_STATE_ACTIVE) {
@@ -327,8 +346,10 @@ static void migrate_fd_put_ready(void *opaque)
         migrate_fd_error(s);
     } else if (ret == 1) {
         int old_vm_running = runstate_is_running();
+        int64_t start_time, end_time;
 
         DPRINTF("done iterating\n");
+        start_time = qemu_get_clock_ms(rt_clock);
         qemu_system_wakeup_request(QEMU_WAKEUP_REASON_OTHER);
         vm_stop_force_state(RUN_STATE_FINISH_MIGRATE);
 
@@ -337,7 +358,9 @@ static void migrate_fd_put_ready(void *opaque)
         } else {
             migrate_fd_completed(s);
         }
-        s->total_time = qemu_get_clock_ms(rt_clock) - s->total_time;
+        end_time = qemu_get_clock_ms(rt_clock);
+        s->total_time = end_time - s->total_time;
+        s->downtime = end_time - start_time;
         if (s->state != MIG_STATE_COMPLETED) {
             if (old_vm_running) {
                 vm_start();
@@ -360,14 +383,13 @@ static void migrate_fd_cancel(MigrationState *s)
     migrate_fd_cleanup(s);
 }
 
-static void migrate_fd_wait_for_unfreeze(void *opaque)
+int migrate_fd_wait_for_unfreeze(MigrationState *s)
 {
-    MigrationState *s = opaque;
     int ret;
 
     DPRINTF("wait for unfreeze\n");
     if (s->state != MIG_STATE_ACTIVE)
-        return;
+        return -EINVAL;
 
     do {
         fd_set wfds;
@@ -379,16 +401,20 @@ static void migrate_fd_wait_for_unfreeze(void *opaque)
     } while (ret == -1 && (s->get_error(s)) == EINTR);
 
     if (ret == -1) {
-        qemu_file_set_error(s->file, -s->get_error(s));
+        return -s->get_error(s);
     }
+    return 0;
 }
 
-static int migrate_fd_close(void *opaque)
+int migrate_fd_close(MigrationState *s)
 {
-    MigrationState *s = opaque;
-
-    qemu_set_fd_handler2(s->fd, NULL, NULL, NULL, NULL);
-    return s->close(s);
+    int rc = 0;
+    if (s->fd != -1) {
+        qemu_set_fd_handler2(s->fd, NULL, NULL, NULL, NULL);
+        rc = s->close(s);
+        s->fd = -1;
+    }
+    return rc;
 }
 
 void add_migration_state_change_notifier(Notifier *notify)
@@ -422,12 +448,7 @@ void migrate_fd_connect(MigrationState *s)
     int ret;
 
     s->state = MIG_STATE_ACTIVE;
-    s->file = qemu_fopen_ops_buffered(s,
-                                      s->bandwidth_limit,
-                                      migrate_fd_put_buffer,
-                                      migrate_fd_put_ready,
-                                      migrate_fd_wait_for_unfreeze,
-                                      migrate_fd_close);
+    s->file = qemu_fopen_ops_buffered(s);
 
     DPRINTF("beginning savevm\n");
     ret = qemu_savevm_state_begin(s->file, &s->params);
@@ -479,10 +500,10 @@ void qmp_migrate(const char *uri, bool has_blk, bool blk,
                  bool has_inc, bool inc, bool has_detach, bool detach,
                  Error **errp)
 {
+    Error *local_err = NULL;
     MigrationState *s = migrate_get_current();
     MigrationParams params;
     const char *p;
-    int ret;
 
     params.blk = blk;
     params.shared = inc;
@@ -504,26 +525,23 @@ void qmp_migrate(const char *uri, bool has_blk, bool blk,
     s = migrate_init(&params);
 
     if (strstart(uri, "tcp:", &p)) {
-        ret = tcp_start_outgoing_migration(s, p, errp);
+        tcp_start_outgoing_migration(s, p, &local_err);
 #if !defined(WIN32)
     } else if (strstart(uri, "exec:", &p)) {
-        ret = exec_start_outgoing_migration(s, p);
+        exec_start_outgoing_migration(s, p, &local_err);
     } else if (strstart(uri, "unix:", &p)) {
-        ret = unix_start_outgoing_migration(s, p);
+        unix_start_outgoing_migration(s, p, &local_err);
     } else if (strstart(uri, "fd:", &p)) {
-        ret = fd_start_outgoing_migration(s, p);
+        fd_start_outgoing_migration(s, p, &local_err);
 #endif
     } else {
         error_set(errp, QERR_INVALID_PARAMETER_VALUE, "uri", "a valid migration protocol");
         return;
     }
 
-    if (ret < 0) {
-        if (!error_is_set(errp)) {
-            DPRINTF("migration failed: %s\n", strerror(-ret));
-            /* FIXME: we should return meaningful errors */
-            error_set(errp, QERR_UNDEFINED_ERROR);
-        }
+    if (local_err) {
+        migrate_fd_error(s);
+        error_propagate(errp, local_err);
         return;
     }
 
diff --git a/migration.h b/migration.h
index a9852fcae..c3a23cc6c 100644
--- a/migration.h
+++ b/migration.h
@@ -40,13 +40,16 @@ struct MigrationState
     void *opaque;
     MigrationParams params;
     int64_t total_time;
+    int64_t downtime;
+    int64_t expected_downtime;
+    int64_t dirty_pages_rate;
     bool enabled_capabilities[MIGRATION_CAPABILITY_MAX];
     int64_t xbzrle_cache_size;
 };
 
 void process_incoming_migration(QEMUFile *f);
 
-int qemu_start_incoming_migration(const char *uri, Error **errp);
+void qemu_start_incoming_migration(const char *uri, Error **errp);
 
 uint64_t migrate_max_downtime(void);
 
@@ -54,32 +57,38 @@ void do_info_migrate_print(Monitor *mon, const QObject *data);
 
 void do_info_migrate(Monitor *mon, QObject **ret_data);
 
-int exec_start_incoming_migration(const char *host_port);
+void exec_start_incoming_migration(const char *host_port, Error **errp);
 
-int exec_start_outgoing_migration(MigrationState *s, const char *host_port);
+void exec_start_outgoing_migration(MigrationState *s, const char *host_port, Error **errp);
 
-int tcp_start_incoming_migration(const char *host_port, Error **errp);
+void tcp_start_incoming_migration(const char *host_port, Error **errp);
 
-int tcp_start_outgoing_migration(MigrationState *s, const char *host_port,
-                                 Error **errp);
+void tcp_start_outgoing_migration(MigrationState *s, const char *host_port, Error **errp);
 
-int unix_start_incoming_migration(const char *path);
+void unix_start_incoming_migration(const char *path, Error **errp);
 
-int unix_start_outgoing_migration(MigrationState *s, const char *path);
+void unix_start_outgoing_migration(MigrationState *s, const char *path, Error **errp);
 
-int fd_start_incoming_migration(const char *path);
+void fd_start_incoming_migration(const char *path, Error **errp);
 
-int fd_start_outgoing_migration(MigrationState *s, const char *fdname);
+void fd_start_outgoing_migration(MigrationState *s, const char *fdname, Error **errp);
 
 void migrate_fd_error(MigrationState *s);
 
 void migrate_fd_connect(MigrationState *s);
 
+ssize_t migrate_fd_put_buffer(MigrationState *s, const void *data,
+                              size_t size);
+void migrate_fd_put_ready(MigrationState *s);
+int migrate_fd_wait_for_unfreeze(MigrationState *s);
+int migrate_fd_close(MigrationState *s);
+
 void add_migration_state_change_notifier(Notifier *notify);
 void remove_migration_state_change_notifier(Notifier *notify);
 bool migration_is_active(MigrationState *);
 bool migration_has_finished(MigrationState *);
 bool migration_has_failed(MigrationState *);
+MigrationState *migrate_get_current(void);
 
 uint64_t ram_bytes_remaining(void);
 uint64_t ram_bytes_transferred(void);
diff --git a/monitor.c b/monitor.c
index b17b1bb84..c0e32d60c 100644
--- a/monitor.c
+++ b/monitor.c
@@ -450,11 +450,14 @@ static const char *monitor_event_names[] = {
     [QEVENT_SPICE_DISCONNECTED] = "SPICE_DISCONNECTED",
     [QEVENT_BLOCK_JOB_COMPLETED] = "BLOCK_JOB_COMPLETED",
     [QEVENT_BLOCK_JOB_CANCELLED] = "BLOCK_JOB_CANCELLED",
+    [QEVENT_BLOCK_JOB_ERROR] = "BLOCK_JOB_ERROR",
+    [QEVENT_BLOCK_JOB_READY] = "BLOCK_JOB_READY",
     [QEVENT_DEVICE_TRAY_MOVED] = "DEVICE_TRAY_MOVED",
     [QEVENT_SUSPEND] = "SUSPEND",
     [QEVENT_SUSPEND_DISK] = "SUSPEND_DISK",
     [QEVENT_WAKEUP] = "WAKEUP",
     [QEVENT_BALLOON_CHANGE] = "BALLOON_CHANGE",
+    [QEVENT_SPICE_MIGRATE_COMPLETED] = "SPICE_MIGRATE_COMPLETED",
 };
 QEMU_BUILD_BUG_ON(ARRAY_SIZE(monitor_event_names) != QEVENT_MAX)
 
@@ -896,13 +899,7 @@ static void do_info_registers(Monitor *mon)
 {
     CPUArchState *env;
     env = mon_get_cpu();
-#ifdef TARGET_I386
-    cpu_dump_state(env, (FILE *)mon, monitor_fprintf,
-                   X86_DUMP_FPU);
-#else
-    cpu_dump_state(env, (FILE *)mon, monitor_fprintf,
-                   0);
-#endif
+    cpu_dump_state(env, (FILE *)mon, monitor_fprintf, CPU_DUMP_FPU);
 }
 
 static void do_info_jit(Monitor *mon)
@@ -943,45 +940,6 @@ static void do_trace_print_events(Monitor *mon)
     trace_print_events((FILE *)mon, &monitor_fprintf);
 }
 
-static int add_graphics_client(Monitor *mon, const QDict *qdict, QObject **ret_data)
-{
-    const char *protocol  = qdict_get_str(qdict, "protocol");
-    const char *fdname = qdict_get_str(qdict, "fdname");
-    CharDriverState *s;
-
-    if (strcmp(protocol, "spice") == 0) {
-        int fd = monitor_get_fd(mon, fdname);
-        int skipauth = qdict_get_try_bool(qdict, "skipauth", 0);
-        int tls = qdict_get_try_bool(qdict, "tls", 0);
-        if (!using_spice) {
-            /* correct one? spice isn't a device ,,, */
-            qerror_report(QERR_DEVICE_NOT_ACTIVE, "spice");
-            return -1;
-        }
-        if (qemu_spice_display_add_client(fd, skipauth, tls) < 0) {
-            close(fd);
-        }
-        return 0;
-#ifdef CONFIG_VNC
-    } else if (strcmp(protocol, "vnc") == 0) {
-	int fd = monitor_get_fd(mon, fdname);
-        int skipauth = qdict_get_try_bool(qdict, "skipauth", 0);
-	vnc_display_add_client(NULL, fd, skipauth);
-	return 0;
-#endif
-    } else if ((s = qemu_chr_find(protocol)) != NULL) {
-	int fd = monitor_get_fd(mon, fdname);
-	if (qemu_chr_add_client(s, fd) < 0) {
-	    qerror_report(QERR_ADD_CLIENT_FAILED);
-	    return -1;
-	}
-	return 0;
-    }
-
-    qerror_report(QERR_INVALID_PARAMETER, "protocol");
-    return -1;
-}
-
 static int client_migrate_info(Monitor *mon, const QDict *qdict,
                                MonitorCompletion cb, void *opaque)
 {
@@ -1016,12 +974,6 @@ static int client_migrate_info(Monitor *mon, const QDict *qdict,
     return -1;
 }
 
-static int do_screen_dump(Monitor *mon, const QDict *qdict, QObject **ret_data)
-{
-    vga_hw_screen_dump(qdict_get_str(qdict, "filename"));
-    return 0;
-}
-
 static void do_logfile(Monitor *mon, const QDict *qdict)
 {
     cpu_set_log_filename(qdict_get_str(qdict, "filename"));
@@ -1108,7 +1060,7 @@ static void monitor_printc(Monitor *mon, int c)
 }
 
 static void memory_dump(Monitor *mon, int count, int format, int wsize,
-                        target_phys_addr_t addr, int is_physical)
+                        hwaddr addr, int is_physical)
 {
     CPUArchState *env;
     int l, line_size, i, max_digits, len;
@@ -1242,7 +1194,7 @@ static void do_physical_memory_dump(Monitor *mon, const QDict *qdict)
     int count = qdict_get_int(qdict, "count");
     int format = qdict_get_int(qdict, "format");
     int size = qdict_get_int(qdict, "size");
-    target_phys_addr_t addr = qdict_get_int(qdict, "addr");
+    hwaddr addr = qdict_get_int(qdict, "addr");
 
     memory_dump(mon, count, format, size, addr, 1);
 }
@@ -1250,21 +1202,21 @@ static void do_physical_memory_dump(Monitor *mon, const QDict *qdict)
 static void do_print(Monitor *mon, const QDict *qdict)
 {
     int format = qdict_get_int(qdict, "format");
-    target_phys_addr_t val = qdict_get_int(qdict, "val");
+    hwaddr val = qdict_get_int(qdict, "val");
 
     switch(format) {
     case 'o':
-        monitor_printf(mon, "%#" TARGET_PRIoPHYS, val);
+        monitor_printf(mon, "%#" HWADDR_PRIo, val);
         break;
     case 'x':
-        monitor_printf(mon, "%#" TARGET_PRIxPHYS, val);
+        monitor_printf(mon, "%#" HWADDR_PRIx, val);
         break;
     case 'u':
-        monitor_printf(mon, "%" TARGET_PRIuPHYS, val);
+        monitor_printf(mon, "%" HWADDR_PRIu, val);
         break;
     default:
     case 'd':
-        monitor_printf(mon, "%" TARGET_PRIdPHYS, val);
+        monitor_printf(mon, "%" HWADDR_PRId, val);
         break;
     case 'c':
         monitor_printc(mon, val);
@@ -1290,245 +1242,6 @@ static void do_sum(Monitor *mon, const QDict *qdict)
     monitor_printf(mon, "%05d\n", sum);
 }
 
-typedef struct {
-    int keycode;
-    const char *name;
-} KeyDef;
-
-static const KeyDef key_defs[] = {
-    { 0x2a, "shift" },
-    { 0x36, "shift_r" },
-
-    { 0x38, "alt" },
-    { 0xb8, "alt_r" },
-    { 0x64, "altgr" },
-    { 0xe4, "altgr_r" },
-    { 0x1d, "ctrl" },
-    { 0x9d, "ctrl_r" },
-
-    { 0xdd, "menu" },
-
-    { 0x01, "esc" },
-
-    { 0x02, "1" },
-    { 0x03, "2" },
-    { 0x04, "3" },
-    { 0x05, "4" },
-    { 0x06, "5" },
-    { 0x07, "6" },
-    { 0x08, "7" },
-    { 0x09, "8" },
-    { 0x0a, "9" },
-    { 0x0b, "0" },
-    { 0x0c, "minus" },
-    { 0x0d, "equal" },
-    { 0x0e, "backspace" },
-
-    { 0x0f, "tab" },
-    { 0x10, "q" },
-    { 0x11, "w" },
-    { 0x12, "e" },
-    { 0x13, "r" },
-    { 0x14, "t" },
-    { 0x15, "y" },
-    { 0x16, "u" },
-    { 0x17, "i" },
-    { 0x18, "o" },
-    { 0x19, "p" },
-    { 0x1a, "bracket_left" },
-    { 0x1b, "bracket_right" },
-    { 0x1c, "ret" },
-
-    { 0x1e, "a" },
-    { 0x1f, "s" },
-    { 0x20, "d" },
-    { 0x21, "f" },
-    { 0x22, "g" },
-    { 0x23, "h" },
-    { 0x24, "j" },
-    { 0x25, "k" },
-    { 0x26, "l" },
-    { 0x27, "semicolon" },
-    { 0x28, "apostrophe" },
-    { 0x29, "grave_accent" },
-
-    { 0x2b, "backslash" },
-    { 0x2c, "z" },
-    { 0x2d, "x" },
-    { 0x2e, "c" },
-    { 0x2f, "v" },
-    { 0x30, "b" },
-    { 0x31, "n" },
-    { 0x32, "m" },
-    { 0x33, "comma" },
-    { 0x34, "dot" },
-    { 0x35, "slash" },
-
-    { 0x37, "asterisk" },
-
-    { 0x39, "spc" },
-    { 0x3a, "caps_lock" },
-    { 0x3b, "f1" },
-    { 0x3c, "f2" },
-    { 0x3d, "f3" },
-    { 0x3e, "f4" },
-    { 0x3f, "f5" },
-    { 0x40, "f6" },
-    { 0x41, "f7" },
-    { 0x42, "f8" },
-    { 0x43, "f9" },
-    { 0x44, "f10" },
-    { 0x45, "num_lock" },
-    { 0x46, "scroll_lock" },
-
-    { 0xb5, "kp_divide" },
-    { 0x37, "kp_multiply" },
-    { 0x4a, "kp_subtract" },
-    { 0x4e, "kp_add" },
-    { 0x9c, "kp_enter" },
-    { 0x53, "kp_decimal" },
-    { 0x54, "sysrq" },
-
-    { 0x52, "kp_0" },
-    { 0x4f, "kp_1" },
-    { 0x50, "kp_2" },
-    { 0x51, "kp_3" },
-    { 0x4b, "kp_4" },
-    { 0x4c, "kp_5" },
-    { 0x4d, "kp_6" },
-    { 0x47, "kp_7" },
-    { 0x48, "kp_8" },
-    { 0x49, "kp_9" },
-
-    { 0x56, "<" },
-
-    { 0x57, "f11" },
-    { 0x58, "f12" },
-
-    { 0xb7, "print" },
-
-    { 0xc7, "home" },
-    { 0xc9, "pgup" },
-    { 0xd1, "pgdn" },
-    { 0xcf, "end" },
-
-    { 0xcb, "left" },
-    { 0xc8, "up" },
-    { 0xd0, "down" },
-    { 0xcd, "right" },
-
-    { 0xd2, "insert" },
-    { 0xd3, "delete" },
-#if defined(TARGET_SPARC) && !defined(TARGET_SPARC64)
-    { 0xf0, "stop" },
-    { 0xf1, "again" },
-    { 0xf2, "props" },
-    { 0xf3, "undo" },
-    { 0xf4, "front" },
-    { 0xf5, "copy" },
-    { 0xf6, "open" },
-    { 0xf7, "paste" },
-    { 0xf8, "find" },
-    { 0xf9, "cut" },
-    { 0xfa, "lf" },
-    { 0xfb, "help" },
-    { 0xfc, "meta_l" },
-    { 0xfd, "meta_r" },
-    { 0xfe, "compose" },
-#endif
-    { 0, NULL },
-};
-
-static int get_keycode(const char *key)
-{
-    const KeyDef *p;
-    char *endp;
-    int ret;
-
-    for(p = key_defs; p->name != NULL; p++) {
-        if (!strcmp(key, p->name))
-            return p->keycode;
-    }
-    if (strstart(key, "0x", NULL)) {
-        ret = strtoul(key, &endp, 0);
-        if (*endp == '\0' && ret >= 0x01 && ret <= 0xff)
-            return ret;
-    }
-    return -1;
-}
-
-#define MAX_KEYCODES 16
-static uint8_t keycodes[MAX_KEYCODES];
-static int nb_pending_keycodes;
-static QEMUTimer *key_timer;
-
-static void release_keys(void *opaque)
-{
-    int keycode;
-
-    while (nb_pending_keycodes > 0) {
-        nb_pending_keycodes--;
-        keycode = keycodes[nb_pending_keycodes];
-        if (keycode & 0x80)
-            kbd_put_keycode(0xe0);
-        kbd_put_keycode(keycode | 0x80);
-    }
-}
-
-static void do_sendkey(Monitor *mon, const QDict *qdict)
-{
-    char keyname_buf[16];
-    char *separator;
-    int keyname_len, keycode, i;
-    const char *string = qdict_get_str(qdict, "string");
-    int has_hold_time = qdict_haskey(qdict, "hold_time");
-    int hold_time = qdict_get_try_int(qdict, "hold_time", -1);
-
-    if (nb_pending_keycodes > 0) {
-        qemu_del_timer(key_timer);
-        release_keys(NULL);
-    }
-    if (!has_hold_time)
-        hold_time = 100;
-    i = 0;
-    while (1) {
-        separator = strchr(string, '-');
-        keyname_len = separator ? separator - string : strlen(string);
-        if (keyname_len > 0) {
-            pstrcpy(keyname_buf, sizeof(keyname_buf), string);
-            if (keyname_len > sizeof(keyname_buf) - 1) {
-                monitor_printf(mon, "invalid key: '%s...'\n", keyname_buf);
-                return;
-            }
-            if (i == MAX_KEYCODES) {
-                monitor_printf(mon, "too many keys\n");
-                return;
-            }
-            keyname_buf[keyname_len] = 0;
-            keycode = get_keycode(keyname_buf);
-            if (keycode < 0) {
-                monitor_printf(mon, "unknown key: '%s'\n", keyname_buf);
-                return;
-            }
-            keycodes[i++] = keycode;
-        }
-        if (!separator)
-            break;
-        string = separator + 1;
-    }
-    nb_pending_keycodes = i;
-    /* key down events */
-    for (i = 0; i < nb_pending_keycodes; i++) {
-        keycode = keycodes[i];
-        if (keycode & 0x80)
-            kbd_put_keycode(0xe0);
-        kbd_put_keycode(keycode & 0x7f);
-    }
-    /* delayed key up events */
-    qemu_mod_timer(key_timer, qemu_get_clock_ns(vm_clock) +
-                   muldiv64(get_ticks_per_sec(), hold_time, 1000));
-}
-
 static int mouse_button_state;
 
 static void do_mouse_move(Monitor *mon, const QDict *qdict)
@@ -1625,9 +1338,9 @@ static void do_boot_set(Monitor *mon, const QDict *qdict)
 }
 
 #if defined(TARGET_I386)
-static void print_pte(Monitor *mon, target_phys_addr_t addr,
-                      target_phys_addr_t pte,
-                      target_phys_addr_t mask)
+static void print_pte(Monitor *mon, hwaddr addr,
+                      hwaddr pte,
+                      hwaddr mask)
 {
 #ifdef TARGET_X86_64
     if (addr & (1ULL << 47)) {
@@ -1696,7 +1409,7 @@ static void tlb_info_pae32(Monitor *mon, CPUArchState *env)
                     if (pde & PG_PSE_MASK) {
                         /* 2M pages with PAE, CR4.PSE is ignored */
                         print_pte(mon, (l1 << 30 ) + (l2 << 21), pde,
-                                  ~((target_phys_addr_t)(1 << 20) - 1));
+                                  ~((hwaddr)(1 << 20) - 1));
                     } else {
                         pt_addr = pde & 0x3fffffffff000ULL;
                         for (l3 = 0; l3 < 512; l3++) {
@@ -1706,7 +1419,7 @@ static void tlb_info_pae32(Monitor *mon, CPUArchState *env)
                                 print_pte(mon, (l1 << 30 ) + (l2 << 21)
                                           + (l3 << 12),
                                           pte & ~PG_PSE_MASK,
-                                          ~(target_phys_addr_t)0xfff);
+                                          ~(hwaddr)0xfff);
                             }
                         }
                     }
@@ -1798,9 +1511,9 @@ static void tlb_info(Monitor *mon)
     }
 }
 
-static void mem_print(Monitor *mon, target_phys_addr_t *pstart,
+static void mem_print(Monitor *mon, hwaddr *pstart,
                       int *plast_prot,
-                      target_phys_addr_t end, int prot)
+                      hwaddr end, int prot)
 {
     int prot1;
     prot1 = *plast_prot;
@@ -1826,7 +1539,7 @@ static void mem_info_32(Monitor *mon, CPUArchState *env)
     unsigned int l1, l2;
     int prot, last_prot;
     uint32_t pgd, pde, pte;
-    target_phys_addr_t start, end;
+    hwaddr start, end;
 
     pgd = env->cr[3] & ~0xfff;
     last_prot = 0;
@@ -1859,7 +1572,7 @@ static void mem_info_32(Monitor *mon, CPUArchState *env)
         }
     }
     /* Flush last range */
-    mem_print(mon, &start, &last_prot, (target_phys_addr_t)1 << 32, 0);
+    mem_print(mon, &start, &last_prot, (hwaddr)1 << 32, 0);
 }
 
 static void mem_info_pae32(Monitor *mon, CPUArchState *env)
@@ -1868,7 +1581,7 @@ static void mem_info_pae32(Monitor *mon, CPUArchState *env)
     int prot, last_prot;
     uint64_t pdpe, pde, pte;
     uint64_t pdp_addr, pd_addr, pt_addr;
-    target_phys_addr_t start, end;
+    hwaddr start, end;
 
     pdp_addr = env->cr[3] & ~0x1f;
     last_prot = 0;
@@ -1914,7 +1627,7 @@ static void mem_info_pae32(Monitor *mon, CPUArchState *env)
         }
     }
     /* Flush last range */
-    mem_print(mon, &start, &last_prot, (target_phys_addr_t)1 << 32, 0);
+    mem_print(mon, &start, &last_prot, (hwaddr)1 << 32, 0);
 }
 
 
@@ -1993,7 +1706,7 @@ static void mem_info_64(Monitor *mon, CPUArchState *env)
         }
     }
     /* Flush last range */
-    mem_print(mon, &start, &last_prot, (target_phys_addr_t)1 << 48, 0);
+    mem_print(mon, &start, &last_prot, (hwaddr)1 << 48, 0);
 }
 #endif
 
@@ -2275,7 +1988,8 @@ static void do_acl_remove(Monitor *mon, const QDict *qdict)
 #if defined(TARGET_I386)
 static void do_inject_mce(Monitor *mon, const QDict *qdict)
 {
-    CPUArchState *cenv;
+    X86CPU *cpu;
+    CPUX86State *cenv;
     int cpu_index = qdict_get_int(qdict, "cpu_index");
     int bank = qdict_get_int(qdict, "bank");
     uint64_t status = qdict_get_int(qdict, "status");
@@ -2288,8 +2002,9 @@ static void do_inject_mce(Monitor *mon, const QDict *qdict)
         flags |= MCE_INJECT_BROADCAST;
     }
     for (cenv = first_cpu; cenv != NULL; cenv = cenv->next_cpu) {
+        cpu = x86_env_get_cpu(cenv);
         if (cenv->cpu_index == cpu_index) {
-            cpu_x86_inject_mce(mon, cenv, bank, status, mcg_status, addr, misc,
+            cpu_x86_inject_mce(mon, cpu, bank, status, mcg_status, addr, misc,
                                flags);
             break;
         }
@@ -2362,7 +2077,7 @@ static void do_loadvm(Monitor *mon, const QDict *qdict)
     }
 }
 
-int monitor_get_fd(Monitor *mon, const char *fdname)
+int monitor_get_fd(Monitor *mon, const char *fdname, Error **errp)
 {
     mon_fd_t *monfd;
 
@@ -2383,6 +2098,7 @@ int monitor_get_fd(Monitor *mon, const char *fdname)
         return fd;
     }
 
+    error_setg(errp, "File descriptor named '%s' has not been found", fdname);
     return -1;
 }
 
@@ -2392,8 +2108,9 @@ static void monitor_fdset_cleanup(MonFdset *mon_fdset)
     MonFdsetFd *mon_fdset_fd_next;
 
     QLIST_FOREACH_SAFE(mon_fdset_fd, &mon_fdset->fds, next, mon_fdset_fd_next) {
-        if (mon_fdset_fd->removed ||
-                (QLIST_EMPTY(&mon_fdset->dup_fds) && mon_refcount == 0)) {
+        if ((mon_fdset_fd->removed ||
+                (QLIST_EMPTY(&mon_fdset->dup_fds) && mon_refcount == 0)) &&
+                runstate_is_running()) {
             close(mon_fdset_fd->fd);
             g_free(mon_fdset_fd->opaque);
             QLIST_REMOVE(mon_fdset_fd, next);
@@ -2422,8 +2139,6 @@ AddfdInfo *qmp_add_fd(bool has_fdset_id, int64_t fdset_id, bool has_opaque,
 {
     int fd;
     Monitor *mon = cur_mon;
-    MonFdset *mon_fdset;
-    MonFdsetFd *mon_fdset_fd;
     AddfdInfo *fdinfo;
 
     fd = qemu_chr_fe_get_msgfd(mon->chr);
@@ -2432,57 +2147,11 @@ AddfdInfo *qmp_add_fd(bool has_fdset_id, int64_t fdset_id, bool has_opaque,
         goto error;
     }
 
-    if (has_fdset_id) {
-        QLIST_FOREACH(mon_fdset, &mon_fdsets, next) {
-            if (mon_fdset->id == fdset_id) {
-                break;
-            }
-        }
-        if (mon_fdset == NULL) {
-            error_set(errp, QERR_INVALID_PARAMETER_VALUE, "fdset-id",
-                      "an existing fdset-id");
-            goto error;
-        }
-    } else {
-        int64_t fdset_id_prev = -1;
-        MonFdset *mon_fdset_cur = QLIST_FIRST(&mon_fdsets);
-
-        /* Use first available fdset ID */
-        QLIST_FOREACH(mon_fdset, &mon_fdsets, next) {
-            mon_fdset_cur = mon_fdset;
-            if (fdset_id_prev == mon_fdset_cur->id - 1) {
-                fdset_id_prev = mon_fdset_cur->id;
-                continue;
-            }
-            break;
-        }
-
-        mon_fdset = g_malloc0(sizeof(*mon_fdset));
-        mon_fdset->id = fdset_id_prev + 1;
-
-        /* The fdset list is ordered by fdset ID */
-        if (mon_fdset->id == 0) {
-            QLIST_INSERT_HEAD(&mon_fdsets, mon_fdset, next);
-        } else if (mon_fdset->id < mon_fdset_cur->id) {
-            QLIST_INSERT_BEFORE(mon_fdset_cur, mon_fdset, next);
-        } else {
-            QLIST_INSERT_AFTER(mon_fdset_cur, mon_fdset, next);
-        }
-    }
-
-    mon_fdset_fd = g_malloc0(sizeof(*mon_fdset_fd));
-    mon_fdset_fd->fd = fd;
-    mon_fdset_fd->removed = false;
-    if (has_opaque) {
-        mon_fdset_fd->opaque = g_strdup(opaque);
+    fdinfo = monitor_fdset_add_fd(fd, has_fdset_id, fdset_id,
+                                  has_opaque, opaque, errp);
+    if (fdinfo) {
+        return fdinfo;
     }
-    QLIST_INSERT_HEAD(&mon_fdset->fds, mon_fdset_fd, next);
-
-    fdinfo = g_malloc0(sizeof(*fdinfo));
-    fdinfo->fdset_id = mon_fdset->id;
-    fdinfo->fd = mon_fdset_fd->fd;
-
-    return fdinfo;
 
 error:
     if (fd != -1) {
@@ -2568,6 +2237,87 @@ FdsetInfoList *qmp_query_fdsets(Error **errp)
     return fdset_list;
 }
 
+AddfdInfo *monitor_fdset_add_fd(int fd, bool has_fdset_id, int64_t fdset_id,
+                                bool has_opaque, const char *opaque,
+                                Error **errp)
+{
+    MonFdset *mon_fdset = NULL;
+    MonFdsetFd *mon_fdset_fd;
+    AddfdInfo *fdinfo;
+
+    if (has_fdset_id) {
+        QLIST_FOREACH(mon_fdset, &mon_fdsets, next) {
+            /* Break if match found or match impossible due to ordering by ID */
+            if (fdset_id <= mon_fdset->id) {
+                if (fdset_id < mon_fdset->id) {
+                    mon_fdset = NULL;
+                }
+                break;
+            }
+        }
+    }
+
+    if (mon_fdset == NULL) {
+        int64_t fdset_id_prev = -1;
+        MonFdset *mon_fdset_cur = QLIST_FIRST(&mon_fdsets);
+
+        if (has_fdset_id) {
+            if (fdset_id < 0) {
+                error_set(errp, QERR_INVALID_PARAMETER_VALUE, "fdset-id",
+                          "a non-negative value");
+                return NULL;
+            }
+            /* Use specified fdset ID */
+            QLIST_FOREACH(mon_fdset, &mon_fdsets, next) {
+                mon_fdset_cur = mon_fdset;
+                if (fdset_id < mon_fdset_cur->id) {
+                    break;
+                }
+            }
+        } else {
+            /* Use first available fdset ID */
+            QLIST_FOREACH(mon_fdset, &mon_fdsets, next) {
+                mon_fdset_cur = mon_fdset;
+                if (fdset_id_prev == mon_fdset_cur->id - 1) {
+                    fdset_id_prev = mon_fdset_cur->id;
+                    continue;
+                }
+                break;
+            }
+        }
+
+        mon_fdset = g_malloc0(sizeof(*mon_fdset));
+        if (has_fdset_id) {
+            mon_fdset->id = fdset_id;
+        } else {
+            mon_fdset->id = fdset_id_prev + 1;
+        }
+
+        /* The fdset list is ordered by fdset ID */
+        if (!mon_fdset_cur) {
+            QLIST_INSERT_HEAD(&mon_fdsets, mon_fdset, next);
+        } else if (mon_fdset->id < mon_fdset_cur->id) {
+            QLIST_INSERT_BEFORE(mon_fdset_cur, mon_fdset, next);
+        } else {
+            QLIST_INSERT_AFTER(mon_fdset_cur, mon_fdset, next);
+        }
+    }
+
+    mon_fdset_fd = g_malloc0(sizeof(*mon_fdset_fd));
+    mon_fdset_fd->fd = fd;
+    mon_fdset_fd->removed = false;
+    if (has_opaque) {
+        mon_fdset_fd->opaque = g_strdup(opaque);
+    }
+    QLIST_INSERT_HEAD(&mon_fdset->fds, mon_fdset_fd, next);
+
+    fdinfo = g_malloc0(sizeof(*fdinfo));
+    fdinfo->fdset_id = mon_fdset->id;
+    fdinfo->fd = mon_fdset_fd->fd;
+
+    return fdinfo;
+}
+
 int monitor_fdset_get_fd(int64_t fdset_id, int flags)
 {
 #ifndef _WIN32
@@ -2651,6 +2401,26 @@ int monitor_fdset_dup_fd_remove(int dup_fd)
     return monitor_fdset_dup_fd_find_remove(dup_fd, true);
 }
 
+int monitor_handle_fd_param(Monitor *mon, const char *fdname)
+{
+    int fd;
+    Error *local_err = NULL;
+
+    if (!qemu_isdigit(fdname[0]) && mon) {
+
+        fd = monitor_get_fd(mon, fdname, &local_err);
+        if (fd == -1) {
+            qerror_report_err(local_err);
+            error_free(local_err);
+            return -1;
+        }
+    } else {
+        fd = qemu_parse_fd(fdname);
+    }
+
+    return fd;
+}
+
 /* mon_cmds and info_cmds would be sorted at runtime */
 static mon_cmd_t mon_cmds[] = {
 #include "hmp-commands.h"
@@ -3485,11 +3255,7 @@ static int64_t expr_unary(Monitor *mon)
         break;
     default:
         errno = 0;
-#if TARGET_PHYS_ADDR_BITS > 32
         n = strtoull(pch, &p, 0);
-#else
-        n = strtoul(pch, &p, 0);
-#endif
         if (errno == ERANGE) {
             expr_error(mon, "number too large");
         }
@@ -4315,7 +4081,6 @@ static void monitor_find_completion(const char *cmdline)
     int nb_args, i, len;
     const char *ptype, *str;
     const mon_cmd_t *cmd;
-    const KeyDef *key;
 
     parse_cmdline(cmdline, &nb_args, args);
 #ifdef DEBUG_COMPLETION
@@ -4389,8 +4154,8 @@ static void monitor_find_completion(const char *cmdline)
                 if (sep)
                     str = sep + 1;
                 readline_set_completion_index(cur_mon->rs, strlen(str));
-                for(key = key_defs; key->name != NULL; key++) {
-                    cmd_completion(str, key->name);
+                for (i = 0; i < Q_KEY_CODE_MAX; i++) {
+                    cmd_completion(str, QKeyCode_lookup[i]);
                 }
             } else if (!strcmp(cmd->name, "help|?")) {
                 readline_set_completion_index(cur_mon->rs, strlen(str));
@@ -4926,7 +4691,6 @@ void monitor_init(CharDriverState *chr, int flags)
     Monitor *mon;
 
     if (is_first_init) {
-        key_timer = qemu_new_timer_ns(vm_clock, release_keys, NULL);
         monitor_protocol_event_init();
         is_first_init = 0;
     }
diff --git a/monitor.h b/monitor.h
index 47d556b9d..b4ef9559d 100644
--- a/monitor.h
+++ b/monitor.h
@@ -38,11 +38,14 @@ typedef enum MonitorEvent {
     QEVENT_SPICE_DISCONNECTED,
     QEVENT_BLOCK_JOB_COMPLETED,
     QEVENT_BLOCK_JOB_CANCELLED,
+    QEVENT_BLOCK_JOB_ERROR,
+    QEVENT_BLOCK_JOB_READY,
     QEVENT_DEVICE_TRAY_MOVED,
     QEVENT_SUSPEND,
     QEVENT_SUSPEND_DISK,
     QEVENT_WAKEUP,
     QEVENT_BALLOON_CHANGE,
+    QEVENT_SPICE_MIGRATE_COMPLETED,
 
     /* Add to 'monitor_event_names' array in monitor.c when
      * defining new events here */
@@ -65,7 +68,8 @@ int monitor_read_block_device_key(Monitor *mon, const char *device,
                                   BlockDriverCompletionFunc *completion_cb,
                                   void *opaque);
 
-int monitor_get_fd(Monitor *mon, const char *fdname);
+int monitor_get_fd(Monitor *mon, const char *fdname, Error **errp);
+int monitor_handle_fd_param(Monitor *mon, const char *fdname);
 
 void monitor_vprintf(Monitor *mon, const char *fmt, va_list ap)
     GCC_FMT_ATTR(2, 0);
@@ -87,6 +91,9 @@ int qmp_qom_set(Monitor *mon, const QDict *qdict, QObject **ret);
 
 int qmp_qom_get(Monitor *mon, const QDict *qdict, QObject **ret);
 
+AddfdInfo *monitor_fdset_add_fd(int fd, bool has_fdset_id, int64_t fdset_id,
+                                bool has_opaque, const char *opaque,
+                                Error **errp);
 int monitor_fdset_get_fd(int64_t fdset_id, int flags);
 int monitor_fdset_dup_fd_add(int64_t fdset_id, int dup_fd);
 int monitor_fdset_dup_fd_remove(int dup_fd);
diff --git a/nbd.c b/nbd.c
index 0dd60c5f4..01976e8e3 100644
--- a/nbd.c
+++ b/nbd.c
@@ -57,9 +57,12 @@
 
 /* This is all part of the "official" NBD API */
 
+#define NBD_REQUEST_SIZE        (4 + 4 + 8 + 8 + 4)
 #define NBD_REPLY_SIZE          (4 + 4 + 8)
 #define NBD_REQUEST_MAGIC       0x25609513
 #define NBD_REPLY_MAGIC         0x67446698
+#define NBD_OPTS_MAGIC          0x49484156454F5054LL
+#define NBD_CLIENT_MAGIC        0x0000420281861253LL
 
 #define NBD_SET_SOCK            _IO(0xab, 0)
 #define NBD_SET_BLKSIZE         _IO(0xab, 1)
@@ -75,6 +78,49 @@
 
 #define NBD_OPT_EXPORT_NAME     (1 << 0)
 
+/* Definitions for opaque data types */
+
+typedef struct NBDRequest NBDRequest;
+
+struct NBDRequest {
+    QSIMPLEQ_ENTRY(NBDRequest) entry;
+    NBDClient *client;
+    uint8_t *data;
+};
+
+struct NBDExport {
+    int refcount;
+    void (*close)(NBDExport *exp);
+
+    BlockDriverState *bs;
+    char *name;
+    off_t dev_offset;
+    off_t size;
+    uint32_t nbdflags;
+    QTAILQ_HEAD(, NBDClient) clients;
+    QSIMPLEQ_HEAD(, NBDRequest) requests;
+    QTAILQ_ENTRY(NBDExport) next;
+};
+
+static QTAILQ_HEAD(, NBDExport) exports = QTAILQ_HEAD_INITIALIZER(exports);
+
+struct NBDClient {
+    int refcount;
+    void (*close)(NBDClient *client);
+
+    NBDExport *exp;
+    int sock;
+
+    Coroutine *recv_coroutine;
+
+    CoMutex send_lock;
+    Coroutine *send_coroutine;
+
+    QTAILQ_ENTRY(NBDClient) next;
+    int nb_requests;
+    bool closing;
+};
+
 /* That's all folks */
 
 ssize_t nbd_wr_sync(int fd, void *buffer, size_t size, bool do_read)
@@ -162,7 +208,14 @@ int tcp_socket_outgoing(const char *address, uint16_t port)
 
 int tcp_socket_outgoing_spec(const char *address_and_port)
 {
-    return inet_connect(address_and_port, true, NULL, NULL);
+    Error *local_err = NULL;
+    int fd = inet_connect(address_and_port, &local_err);
+
+    if (local_err != NULL) {
+        qerror_report_err(local_err);
+        error_free(local_err);
+    }
+    return fd;
 }
 
 int tcp_socket_incoming(const char *address, uint16_t port)
@@ -174,29 +227,57 @@ int tcp_socket_incoming(const char *address, uint16_t port)
 
 int tcp_socket_incoming_spec(const char *address_and_port)
 {
-    char *ostr  = NULL;
-    int olen = 0;
-    return inet_listen(address_and_port, ostr, olen, SOCK_STREAM, 0, NULL);
+    Error *local_err = NULL;
+    int fd = inet_listen(address_and_port, NULL, 0, SOCK_STREAM, 0, &local_err);
+
+    if (local_err != NULL) {
+        qerror_report_err(local_err);
+        error_free(local_err);
+    }
+    return fd;
 }
 
 int unix_socket_incoming(const char *path)
 {
-    char *ostr = NULL;
-    int olen = 0;
+    Error *local_err = NULL;
+    int fd = unix_listen(path, NULL, 0, &local_err);
 
-    return unix_listen(path, ostr, olen);
+    if (local_err != NULL) {
+        qerror_report_err(local_err);
+        error_free(local_err);
+    }
+    return fd;
 }
 
 int unix_socket_outgoing(const char *path)
 {
-    return unix_connect(path);
+    Error *local_err = NULL;
+    int fd = unix_connect(path, &local_err);
+
+    if (local_err != NULL) {
+        qerror_report_err(local_err);
+        error_free(local_err);
+    }
+    return fd;
 }
 
-/* Basic flow
+/* Basic flow for negotiation
 
    Server         Client
-
    Negotiate
+
+   or
+
+   Server         Client
+   Negotiate #1
+                  Option
+   Negotiate #2
+
+   ----
+
+   followed by
+
+   Server         Client
                   Request
    Response
                   Request
@@ -204,36 +285,152 @@ int unix_socket_outgoing(const char *path)
                   ...
    ...
                   Request (type == 2)
+
 */
 
-static int nbd_send_negotiate(int csock, off_t size, uint32_t flags)
+static int nbd_receive_options(NBDClient *client)
 {
+    int csock = client->sock;
+    char name[256];
+    uint32_t tmp, length;
+    uint64_t magic;
+    int rc;
+
+    /* Client sends:
+        [ 0 ..   3]   reserved (0)
+        [ 4 ..  11]   NBD_OPTS_MAGIC
+        [12 ..  15]   NBD_OPT_EXPORT_NAME
+        [16 ..  19]   length
+        [20 ..  xx]   export name (length bytes)
+     */
+
+    rc = -EINVAL;
+    if (read_sync(csock, &tmp, sizeof(tmp)) != sizeof(tmp)) {
+        LOG("read failed");
+        goto fail;
+    }
+    TRACE("Checking reserved");
+    if (tmp != 0) {
+        LOG("Bad reserved received");
+        goto fail;
+    }
+
+    if (read_sync(csock, &magic, sizeof(magic)) != sizeof(magic)) {
+        LOG("read failed");
+        goto fail;
+    }
+    TRACE("Checking reserved");
+    if (magic != be64_to_cpu(NBD_OPTS_MAGIC)) {
+        LOG("Bad magic received");
+        goto fail;
+    }
+
+    if (read_sync(csock, &tmp, sizeof(tmp)) != sizeof(tmp)) {
+        LOG("read failed");
+        goto fail;
+    }
+    TRACE("Checking option");
+    if (tmp != be32_to_cpu(NBD_OPT_EXPORT_NAME)) {
+        LOG("Bad option received");
+        goto fail;
+    }
+
+    if (read_sync(csock, &length, sizeof(length)) != sizeof(length)) {
+        LOG("read failed");
+        goto fail;
+    }
+    TRACE("Checking length");
+    length = be32_to_cpu(length);
+    if (length > 255) {
+        LOG("Bad length received");
+        goto fail;
+    }
+    if (read_sync(csock, name, length) != length) {
+        LOG("read failed");
+        goto fail;
+    }
+    name[length] = '\0';
+
+    client->exp = nbd_export_find(name);
+    if (!client->exp) {
+        LOG("export not found");
+        goto fail;
+    }
+
+    QTAILQ_INSERT_TAIL(&client->exp->clients, client, next);
+    nbd_export_get(client->exp);
+
+    TRACE("Option negotiation succeeded.");
+    rc = 0;
+fail:
+    return rc;
+}
+
+static int nbd_send_negotiate(NBDClient *client)
+{
+    int csock = client->sock;
     char buf[8 + 8 + 8 + 128];
     int rc;
+    const int myflags = (NBD_FLAG_HAS_FLAGS | NBD_FLAG_SEND_TRIM |
+                         NBD_FLAG_SEND_FLUSH | NBD_FLAG_SEND_FUA);
 
-    /* Negotiate
-        [ 0 ..   7]   passwd   ("NBDMAGIC")
-        [ 8 ..  15]   magic    (0x00420281861253)
+    /* Negotiation header without options:
+        [ 0 ..   7]   passwd       ("NBDMAGIC")
+        [ 8 ..  15]   magic        (NBD_CLIENT_MAGIC)
         [16 ..  23]   size
-        [24 ..  27]   flags
-        [28 .. 151]   reserved (0)
+        [24 ..  25]   server flags (0)
+        [24 ..  27]   export flags
+        [28 .. 151]   reserved     (0)
+
+       Negotiation header with options, part 1:
+        [ 0 ..   7]   passwd       ("NBDMAGIC")
+        [ 8 ..  15]   magic        (NBD_OPTS_MAGIC)
+        [16 ..  17]   server flags (0)
+
+       part 2 (after options are sent):
+        [18 ..  25]   size
+        [26 ..  27]   export flags
+        [28 .. 151]   reserved     (0)
      */
 
     socket_set_block(csock);
     rc = -EINVAL;
 
     TRACE("Beginning negotiation.");
+    memset(buf, 0, sizeof(buf));
     memcpy(buf, "NBDMAGIC", 8);
-    cpu_to_be64w((uint64_t*)(buf + 8), 0x00420281861253LL);
-    cpu_to_be64w((uint64_t*)(buf + 16), size);
-    cpu_to_be32w((uint32_t*)(buf + 24),
-                 flags | NBD_FLAG_HAS_FLAGS | NBD_FLAG_SEND_TRIM |
-                 NBD_FLAG_SEND_FLUSH | NBD_FLAG_SEND_FUA);
-    memset(buf + 28, 0, 124);
-
-    if (write_sync(csock, buf, sizeof(buf)) != sizeof(buf)) {
-        LOG("write failed");
-        goto fail;
+    if (client->exp) {
+        assert ((client->exp->nbdflags & ~65535) == 0);
+        cpu_to_be64w((uint64_t*)(buf + 8), NBD_CLIENT_MAGIC);
+        cpu_to_be64w((uint64_t*)(buf + 16), client->exp->size);
+        cpu_to_be16w((uint16_t*)(buf + 26), client->exp->nbdflags | myflags);
+    } else {
+        cpu_to_be64w((uint64_t*)(buf + 8), NBD_OPTS_MAGIC);
+    }
+
+    if (client->exp) {
+        if (write_sync(csock, buf, sizeof(buf)) != sizeof(buf)) {
+            LOG("write failed");
+            goto fail;
+        }
+    } else {
+        if (write_sync(csock, buf, 18) != 18) {
+            LOG("write failed");
+            goto fail;
+        }
+        rc = nbd_receive_options(client);
+        if (rc < 0) {
+            LOG("option negotiation failed");
+            goto fail;
+        }
+
+        assert ((client->exp->nbdflags & ~65535) == 0);
+        cpu_to_be64w((uint64_t*)(buf + 18), client->exp->size);
+        cpu_to_be16w((uint16_t*)(buf + 26), client->exp->nbdflags | myflags);
+        if (write_sync(csock, buf + 18, sizeof(buf) - 18) != sizeof(buf) - 18) {
+            LOG("write failed");
+            goto fail;
+        }
     }
 
     TRACE("Negotiation succeeded.");
@@ -295,7 +492,7 @@ int nbd_receive_negotiate(int csock, const char *name, uint32_t *flags,
         uint32_t namesize;
 
         TRACE("Checking magic (opts_magic)");
-        if (magic != 0x49484156454F5054LL) {
+        if (magic != NBD_OPTS_MAGIC) {
             LOG("Bad magic received");
             goto fail;
         }
@@ -334,7 +531,7 @@ int nbd_receive_negotiate(int csock, const char *name, uint32_t *flags,
     } else {
         TRACE("Checking magic (cli_magic)");
 
-        if (magic != 0x00420281861253LL) {
+        if (magic != NBD_CLIENT_MAGIC) {
             LOG("Bad magic received");
             goto fail;
         }
@@ -399,24 +596,23 @@ int nbd_init(int fd, int csock, uint32_t flags, off_t size, size_t blocksize)
         return -serrno;
     }
 
-    if (flags & NBD_FLAG_READ_ONLY) {
-        int read_only = 1;
-        TRACE("Setting readonly attribute");
+    if (ioctl(fd, NBD_SET_FLAGS, flags) < 0) {
+        if (errno == ENOTTY) {
+            int read_only = (flags & NBD_FLAG_READ_ONLY) != 0;
+            TRACE("Setting readonly attribute");
 
-        if (ioctl(fd, BLKROSET, (unsigned long) &read_only) < 0) {
+            if (ioctl(fd, BLKROSET, (unsigned long) &read_only) < 0) {
+                int serrno = errno;
+                LOG("Failed setting read-only attribute");
+                return -serrno;
+            }
+        } else {
             int serrno = errno;
-            LOG("Failed setting read-only attribute");
+            LOG("Failed setting flags");
             return -serrno;
         }
     }
 
-    if (ioctl(fd, NBD_SET_FLAGS, flags) < 0
-        && errno != ENOTTY) {
-        int serrno = errno;
-        LOG("Failed setting flags");
-        return -serrno;
-    }
-
     TRACE("Negotiation ended");
 
     return 0;
@@ -477,7 +673,7 @@ int nbd_client(int fd)
 
 ssize_t nbd_send_request(int csock, struct nbd_request *request)
 {
-    uint8_t buf[4 + 4 + 8 + 8 + 4];
+    uint8_t buf[NBD_REQUEST_SIZE];
     ssize_t ret;
 
     cpu_to_be32w((uint32_t*)buf, NBD_REQUEST_MAGIC);
@@ -504,7 +700,7 @@ ssize_t nbd_send_request(int csock, struct nbd_request *request)
 
 static ssize_t nbd_receive_request(int csock, struct nbd_request *request)
 {
-    uint8_t buf[4 + 4 + 8 + 8 + 4];
+    uint8_t buf[NBD_REQUEST_SIZE];
     uint32_t magic;
     ssize_t ret;
 
@@ -582,7 +778,7 @@ ssize_t nbd_receive_reply(int csock, struct nbd_reply *reply)
 
 static ssize_t nbd_send_reply(int csock, struct nbd_reply *reply)
 {
-    uint8_t buf[4 + 4 + 8];
+    uint8_t buf[NBD_REPLY_SIZE];
     ssize_t ret;
 
     /* Reply
@@ -610,58 +806,47 @@ static ssize_t nbd_send_reply(int csock, struct nbd_reply *reply)
 
 #define MAX_NBD_REQUESTS 16
 
-typedef struct NBDRequest NBDRequest;
-
-struct NBDRequest {
-    QSIMPLEQ_ENTRY(NBDRequest) entry;
-    NBDClient *client;
-    uint8_t *data;
-};
-
-struct NBDExport {
-    BlockDriverState *bs;
-    off_t dev_offset;
-    off_t size;
-    uint32_t nbdflags;
-    QSIMPLEQ_HEAD(, NBDRequest) requests;
-};
-
-struct NBDClient {
-    int refcount;
-    void (*close)(NBDClient *client);
-
-    NBDExport *exp;
-    int sock;
-
-    Coroutine *recv_coroutine;
-
-    CoMutex send_lock;
-    Coroutine *send_coroutine;
-
-    int nb_requests;
-};
-
-static void nbd_client_get(NBDClient *client)
+void nbd_client_get(NBDClient *client)
 {
     client->refcount++;
 }
 
-static void nbd_client_put(NBDClient *client)
+void nbd_client_put(NBDClient *client)
 {
     if (--client->refcount == 0) {
+        /* The last reference should be dropped by client->close,
+         * which is called by nbd_client_close.
+         */
+        assert(client->closing);
+
+        qemu_set_fd_handler2(client->sock, NULL, NULL, NULL, NULL);
+        close(client->sock);
+        client->sock = -1;
+        if (client->exp) {
+            QTAILQ_REMOVE(&client->exp->clients, client, next);
+            nbd_export_put(client->exp);
+        }
         g_free(client);
     }
 }
 
-static void nbd_client_close(NBDClient *client)
+void nbd_client_close(NBDClient *client)
 {
-    qemu_set_fd_handler2(client->sock, NULL, NULL, NULL, NULL);
-    close(client->sock);
-    client->sock = -1;
+    if (client->closing) {
+        return;
+    }
+
+    client->closing = true;
+
+    /* Force requests to finish.  They will drop their own references,
+     * then we'll close the socket and free the NBDClient.
+     */
+    shutdown(client->sock, 2);
+
+    /* Also tell the client, so that they release their reference.  */
     if (client->close) {
         client->close(client);
     }
-    nbd_client_put(client);
 }
 
 static NBDRequest *nbd_request_get(NBDClient *client)
@@ -695,28 +880,109 @@ static void nbd_request_put(NBDRequest *req)
 }
 
 NBDExport *nbd_export_new(BlockDriverState *bs, off_t dev_offset,
-                          off_t size, uint32_t nbdflags)
+                          off_t size, uint32_t nbdflags,
+                          void (*close)(NBDExport *))
 {
     NBDExport *exp = g_malloc0(sizeof(NBDExport));
     QSIMPLEQ_INIT(&exp->requests);
+    exp->refcount = 1;
+    QTAILQ_INIT(&exp->clients);
     exp->bs = bs;
     exp->dev_offset = dev_offset;
     exp->nbdflags = nbdflags;
     exp->size = size == -1 ? bdrv_getlength(bs) : size;
+    exp->close = close;
     return exp;
 }
 
+NBDExport *nbd_export_find(const char *name)
+{
+    NBDExport *exp;
+    QTAILQ_FOREACH(exp, &exports, next) {
+        if (strcmp(name, exp->name) == 0) {
+            return exp;
+        }
+    }
+
+    return NULL;
+}
+
+void nbd_export_set_name(NBDExport *exp, const char *name)
+{
+    if (exp->name == name) {
+        return;
+    }
+
+    nbd_export_get(exp);
+    if (exp->name != NULL) {
+        g_free(exp->name);
+        exp->name = NULL;
+        QTAILQ_REMOVE(&exports, exp, next);
+        nbd_export_put(exp);
+    }
+    if (name != NULL) {
+        nbd_export_get(exp);
+        exp->name = g_strdup(name);
+        QTAILQ_INSERT_TAIL(&exports, exp, next);
+    }
+    nbd_export_put(exp);
+}
+
 void nbd_export_close(NBDExport *exp)
 {
-    while (!QSIMPLEQ_EMPTY(&exp->requests)) {
-        NBDRequest *first = QSIMPLEQ_FIRST(&exp->requests);
-        QSIMPLEQ_REMOVE_HEAD(&exp->requests, entry);
-        qemu_vfree(first->data);
-        g_free(first);
+    NBDClient *client, *next;
+
+    nbd_export_get(exp);
+    QTAILQ_FOREACH_SAFE(client, &exp->clients, next, next) {
+        nbd_client_close(client);
+    }
+    nbd_export_set_name(exp, NULL);
+    nbd_export_put(exp);
+}
+
+void nbd_export_get(NBDExport *exp)
+{
+    assert(exp->refcount > 0);
+    exp->refcount++;
+}
+
+void nbd_export_put(NBDExport *exp)
+{
+    assert(exp->refcount > 0);
+    if (exp->refcount == 1) {
+        nbd_export_close(exp);
     }
 
-    bdrv_close(exp->bs);
-    g_free(exp);
+    if (--exp->refcount == 0) {
+        assert(exp->name == NULL);
+
+        if (exp->close) {
+            exp->close(exp);
+        }
+
+        while (!QSIMPLEQ_EMPTY(&exp->requests)) {
+            NBDRequest *first = QSIMPLEQ_FIRST(&exp->requests);
+            QSIMPLEQ_REMOVE_HEAD(&exp->requests, entry);
+            qemu_vfree(first->data);
+            g_free(first);
+        }
+
+        g_free(exp);
+    }
+}
+
+BlockDriverState *nbd_export_get_blockdev(NBDExport *exp)
+{
+    return exp->bs;
+}
+
+void nbd_export_close_all(void)
+{
+    NBDExport *exp, *next;
+
+    QTAILQ_FOREACH_SAFE(exp, &exports, next, next) {
+        nbd_export_close(exp);
+    }
 }
 
 static int nbd_can_read(void *opaque);
@@ -805,14 +1071,18 @@ out:
 static void nbd_trip(void *opaque)
 {
     NBDClient *client = opaque;
-    NBDRequest *req = nbd_request_get(client);
     NBDExport *exp = client->exp;
+    NBDRequest *req;
     struct nbd_request request;
     struct nbd_reply reply;
     ssize_t ret;
 
     TRACE("Reading request.");
+    if (client->closing) {
+        return;
+    }
 
+    req = nbd_request_get(client);
     ret = nbd_co_receive_request(req, &request);
     if (ret == -EAGAIN) {
         goto done;
@@ -974,15 +1244,21 @@ NBDClient *nbd_client_new(NBDExport *exp, int csock,
                           void (*close)(NBDClient *))
 {
     NBDClient *client;
-    if (nbd_send_negotiate(csock, exp->size, exp->nbdflags) < 0) {
-        return NULL;
-    }
     client = g_malloc0(sizeof(NBDClient));
     client->refcount = 1;
     client->exp = exp;
     client->sock = csock;
+    if (nbd_send_negotiate(client) < 0) {
+        g_free(client);
+        return NULL;
+    }
     client->close = close;
     qemu_co_mutex_init(&client->send_lock);
     qemu_set_fd_handler2(csock, nbd_can_read, nbd_read, NULL, client);
+
+    if (exp) {
+        QTAILQ_INSERT_TAIL(&exp->clients, client, next);
+        nbd_export_get(exp);
+    }
     return client;
 }
diff --git a/nbd.h b/nbd.h
index 40d58d359..344f05b79 100644
--- a/nbd.h
+++ b/nbd.h
@@ -79,9 +79,22 @@ typedef struct NBDExport NBDExport;
 typedef struct NBDClient NBDClient;
 
 NBDExport *nbd_export_new(BlockDriverState *bs, off_t dev_offset,
-                          off_t size, uint32_t nbdflags);
+                          off_t size, uint32_t nbdflags,
+                          void (*close)(NBDExport *));
 void nbd_export_close(NBDExport *exp);
+void nbd_export_get(NBDExport *exp);
+void nbd_export_put(NBDExport *exp);
+
+BlockDriverState *nbd_export_get_blockdev(NBDExport *exp);
+
+NBDExport *nbd_export_find(const char *name);
+void nbd_export_set_name(NBDExport *exp, const char *name);
+void nbd_export_close_all(void);
+
 NBDClient *nbd_client_new(NBDExport *exp, int csock,
                           void (*close)(NBDClient *));
+void nbd_client_close(NBDClient *client);
+void nbd_client_get(NBDClient *client);
+void nbd_client_put(NBDClient *client);
 
 #endif
diff --git a/net.c b/net.c
index 60043ddec..e8ae13e28 100644
--- a/net.c
+++ b/net.c
@@ -21,17 +21,14 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#include "net.h"
-
 #include "config-host.h"
 
-#include "net/tap.h"
-#include "net/socket.h"
-#include "net/dump.h"
-#include "net/slirp.h"
-#include "net/vde.h"
+#include "net.h"
+#include "net/clients.h"
 #include "net/hub.h"
+#include "net/slirp.h"
 #include "net/util.h"
+
 #include "monitor.h"
 #include "qemu-common.h"
 #include "qemu_socket.h"
@@ -357,7 +354,12 @@ void qemu_flush_queued_packets(NetClientState *nc)
 {
     nc->receive_disabled = 0;
 
-    qemu_net_queue_flush(nc->send_queue);
+    if (qemu_net_queue_flush(nc->send_queue)) {
+        /* We emptied the queue successfully, signal to the IO thread to repoll
+         * the file descriptor (for tap, for example).
+         */
+        qemu_notify_event();
+    }
 }
 
 static ssize_t qemu_send_packet_async_with_flags(NetClientState *sender,
@@ -418,16 +420,27 @@ ssize_t qemu_deliver_packet_iov(NetClientState *sender,
                                 void *opaque)
 {
     NetClientState *nc = opaque;
+    int ret;
 
     if (nc->link_down) {
         return iov_size(iov, iovcnt);
     }
 
+    if (nc->receive_disabled) {
+        return 0;
+    }
+
     if (nc->info->receive_iov) {
-        return nc->info->receive_iov(nc, iov, iovcnt);
+        ret = nc->info->receive_iov(nc, iov, iovcnt);
     } else {
-        return nc_sendv_compat(nc, iov, iovcnt);
+        ret = nc_sendv_compat(nc, iov, iovcnt);
+    }
+
+    if (ret == 0) {
+        nc->receive_disabled = 1;
     }
+
+    return ret;
 }
 
 ssize_t qemu_sendv_packet_async(NetClientState *sender,
@@ -522,24 +535,6 @@ int qemu_find_nic_model(NICInfo *nd, const char * const *models,
     return -1;
 }
 
-int net_handle_fd_param(Monitor *mon, const char *param)
-{
-    int fd;
-
-    if (!qemu_isdigit(param[0]) && mon) {
-
-        fd = monitor_get_fd(mon, param);
-        if (fd == -1) {
-            error_report("No file descriptor named %s found", param);
-            return -1;
-        }
-    } else {
-        fd = qemu_parse_fd(param);
-    }
-
-    return fd;
-}
-
 static int net_init_nic(const NetClientOptions *opts, const char *name,
                         NetClientState *peer)
 {
@@ -832,6 +827,7 @@ exit_err:
 void qmp_netdev_del(const char *id, Error **errp)
 {
     NetClientState *nc;
+    QemuOpts *opts;
 
     nc = qemu_find_netdev(id);
     if (!nc) {
@@ -839,8 +835,14 @@ void qmp_netdev_del(const char *id, Error **errp)
         return;
     }
 
+    opts = qemu_opts_find(qemu_find_opts_err("netdev", NULL), id);
+    if (!opts) {
+        error_setg(errp, "Device '%s' is not a netdev", id);
+        return;
+    }
+
     qemu_del_net_client(nc);
-    qemu_opts_del(qemu_opts_find(qemu_find_opts_err("netdev", errp), id));
+    qemu_opts_del(opts);
 }
 
 void print_net_client(Monitor *mon, NetClientState *nc)
diff --git a/net.h b/net.h
index 29750567e..04fda1d6c 100644
--- a/net.h
+++ b/net.h
@@ -168,8 +168,6 @@ int qmp_netdev_add(Monitor *mon, const QDict *qdict, QObject **ret);
 
 void qdev_set_nic_properties(DeviceState *dev, NICInfo *nd);
 
-int net_handle_fd_param(Monitor *mon, const char *param);
-
 #define POLYNOMIAL 0x04c11db6
 unsigned compute_mcast_idx(const uint8_t *ep);
 
diff --git a/net/socket.h b/net/clients.h
index 3f8a09245..c58cc6087 100644
--- a/net/socket.h
+++ b/net/clients.h
@@ -21,13 +21,35 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#ifndef QEMU_NET_SOCKET_H
-#define QEMU_NET_SOCKET_H
+#ifndef QEMU_NET_CLIENTS_H
+#define QEMU_NET_CLIENTS_H
 
 #include "net.h"
 #include "qapi-types.h"
 
+int net_init_dump(const NetClientOptions *opts, const char *name,
+                  NetClientState *peer);
+
+#ifdef CONFIG_SLIRP
+int net_init_slirp(const NetClientOptions *opts, const char *name,
+                   NetClientState *peer);
+#endif
+
+int net_init_hubport(const NetClientOptions *opts, const char *name,
+                     NetClientState *peer);
+
 int net_init_socket(const NetClientOptions *opts, const char *name,
                     NetClientState *peer);
 
-#endif /* QEMU_NET_SOCKET_H */
+int net_init_tap(const NetClientOptions *opts, const char *name,
+                 NetClientState *peer);
+
+int net_init_bridge(const NetClientOptions *opts, const char *name,
+                    NetClientState *peer);
+
+#ifdef CONFIG_VDE
+int net_init_vde(const NetClientOptions *opts, const char *name,
+                 NetClientState *peer);
+#endif
+
+#endif /* QEMU_NET_CLIENTS_H */
diff --git a/net/dump.c b/net/dump.c
index 004231d48..e0a5d7464 100644
--- a/net/dump.c
+++ b/net/dump.c
@@ -22,7 +22,7 @@
  * THE SOFTWARE.
  */
 
-#include "dump.h"
+#include "clients.h"
 #include "qemu-common.h"
 #include "qemu-error.h"
 #include "qemu-log.h"
diff --git a/net/dump.h b/net/dump.h
deleted file mode 100644
index 33f152b46..000000000
--- a/net/dump.h
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * QEMU System Emulator
- *
- * Copyright (c) 2003-2008 Fabrice Bellard
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- */
-#ifndef QEMU_NET_DUMP_H
-#define QEMU_NET_DUMP_H
-
-#include "net.h"
-#include "qapi-types.h"
-
-int net_init_dump(const NetClientOptions *opts, const char *name,
-                  NetClientState *peer);
-
-#endif /* QEMU_NET_DUMP_H */
diff --git a/net/hub.c b/net/hub.c
index ac157e32e..be413012b 100644
--- a/net/hub.c
+++ b/net/hub.c
@@ -14,6 +14,7 @@
 
 #include "monitor.h"
 #include "net.h"
+#include "clients.h"
 #include "hub.h"
 #include "iov.h"
 
@@ -97,12 +98,12 @@ static int net_hub_port_can_receive(NetClientState *nc)
             continue;
         }
 
-        if (!qemu_can_send_packet(&port->nc)) {
-            return 0;
+        if (qemu_can_send_packet(&port->nc)) {
+            return 1;
         }
     }
 
-    return 1;
+    return 0;
 }
 
 static ssize_t net_hub_port_receive(NetClientState *nc,
diff --git a/net/hub.h b/net/hub.h
index 26a1ade1f..4cbfdb128 100644
--- a/net/hub.h
+++ b/net/hub.h
@@ -17,8 +17,6 @@
 
 #include "qemu-common.h"
 
-int net_init_hubport(const NetClientOptions *opts, const char *name,
-                     NetClientState *peer);
 NetClientState *net_hub_add_port(int hub_id, const char *name);
 NetClientState *net_hub_find_client_by_name(int hub_id, const char *name);
 void net_hub_info(Monitor *mon);
diff --git a/net/queue.c b/net/queue.c
index e8030aafe..254f28013 100644
--- a/net/queue.c
+++ b/net/queue.c
@@ -83,12 +83,12 @@ void qemu_del_net_queue(NetQueue *queue)
     g_free(queue);
 }
 
-static ssize_t qemu_net_queue_append(NetQueue *queue,
-                                     NetClientState *sender,
-                                     unsigned flags,
-                                     const uint8_t *buf,
-                                     size_t size,
-                                     NetPacketSent *sent_cb)
+static void qemu_net_queue_append(NetQueue *queue,
+                                  NetClientState *sender,
+                                  unsigned flags,
+                                  const uint8_t *buf,
+                                  size_t size,
+                                  NetPacketSent *sent_cb)
 {
     NetPacket *packet;
 
@@ -100,16 +100,14 @@ static ssize_t qemu_net_queue_append(NetQueue *queue,
     memcpy(packet->data, buf, size);
 
     QTAILQ_INSERT_TAIL(&queue->packets, packet, entry);
-
-    return size;
 }
 
-static ssize_t qemu_net_queue_append_iov(NetQueue *queue,
-                                         NetClientState *sender,
-                                         unsigned flags,
-                                         const struct iovec *iov,
-                                         int iovcnt,
-                                         NetPacketSent *sent_cb)
+static void qemu_net_queue_append_iov(NetQueue *queue,
+                                      NetClientState *sender,
+                                      unsigned flags,
+                                      const struct iovec *iov,
+                                      int iovcnt,
+                                      NetPacketSent *sent_cb)
 {
     NetPacket *packet;
     size_t max_len = 0;
@@ -133,8 +131,6 @@ static ssize_t qemu_net_queue_append_iov(NetQueue *queue,
     }
 
     QTAILQ_INSERT_TAIL(&queue->packets, packet, entry);
-
-    return packet->size;
 }
 
 static ssize_t qemu_net_queue_deliver(NetQueue *queue,
@@ -177,7 +173,8 @@ ssize_t qemu_net_queue_send(NetQueue *queue,
     ssize_t ret;
 
     if (queue->delivering || !qemu_can_send_packet(sender)) {
-        return qemu_net_queue_append(queue, sender, flags, data, size, sent_cb);
+        qemu_net_queue_append(queue, sender, flags, data, size, sent_cb);
+        return 0;
     }
 
     ret = qemu_net_queue_deliver(queue, sender, flags, data, size);
@@ -201,8 +198,8 @@ ssize_t qemu_net_queue_send_iov(NetQueue *queue,
     ssize_t ret;
 
     if (queue->delivering || !qemu_can_send_packet(sender)) {
-        return qemu_net_queue_append_iov(queue, sender, flags,
-                                         iov, iovcnt, sent_cb);
+        qemu_net_queue_append_iov(queue, sender, flags, iov, iovcnt, sent_cb);
+        return 0;
     }
 
     ret = qemu_net_queue_deliver_iov(queue, sender, flags, iov, iovcnt);
@@ -228,7 +225,7 @@ void qemu_net_queue_purge(NetQueue *queue, NetClientState *from)
     }
 }
 
-void qemu_net_queue_flush(NetQueue *queue)
+bool qemu_net_queue_flush(NetQueue *queue)
 {
     while (!QTAILQ_EMPTY(&queue->packets)) {
         NetPacket *packet;
@@ -244,7 +241,7 @@ void qemu_net_queue_flush(NetQueue *queue)
                                      packet->size);
         if (ret == 0) {
             QTAILQ_INSERT_HEAD(&queue->packets, packet, entry);
-            break;
+            return false;
         }
 
         if (packet->sent_cb) {
@@ -253,4 +250,5 @@ void qemu_net_queue_flush(NetQueue *queue)
 
         g_free(packet);
     }
+    return true;
 }
diff --git a/net/queue.h b/net/queue.h
index 9d44a9b3b..fc02b3391 100644
--- a/net/queue.h
+++ b/net/queue.h
@@ -53,6 +53,6 @@ ssize_t qemu_net_queue_send_iov(NetQueue *queue,
                                 NetPacketSent *sent_cb);
 
 void qemu_net_queue_purge(NetQueue *queue, NetClientState *from);
-void qemu_net_queue_flush(NetQueue *queue);
+bool qemu_net_queue_flush(NetQueue *queue);
 
 #endif /* QEMU_NET_QUEUE_H */
diff --git a/net/slirp.c b/net/slirp.c
index 8db66ea53..afb52c3af 100644
--- a/net/slirp.c
+++ b/net/slirp.c
@@ -30,7 +30,8 @@
 #include <sys/wait.h>
 #endif
 #include "net.h"
-#include "net/hub.h"
+#include "clients.h"
+#include "hub.h"
 #include "monitor.h"
 #include "qemu_socket.h"
 #include "slirp/libslirp.h"
@@ -135,7 +136,7 @@ static int net_slirp_init(NetClientState *peer, const char *model,
                           const char *vhostname, const char *tftp_export,
                           const char *bootfile, const char *vdhcp_start,
                           const char *vnameserver, const char *smb_export,
-                          const char *vsmbserver)
+                          const char *vsmbserver, const char **dnssearch)
 {
     /* default settings according to historic slirp */
     struct in_addr net  = { .s_addr = htonl(0x0a000200) }; /* 10.0.2.0 */
@@ -241,7 +242,7 @@ static int net_slirp_init(NetClientState *peer, const char *model,
     s = DO_UPCAST(SlirpState, nc, nc);
 
     s->slirp = slirp_init(restricted, net, mask, host, vhostname,
-                          tftp_export, bootfile, dhcp, dns, s);
+                          tftp_export, bootfile, dhcp, dns, dnssearch, s);
     QTAILQ_INSERT_TAIL(&slirp_stacks, s, entry);
 
     for (config = slirp_configs; config; config = config->next) {
@@ -698,6 +699,31 @@ net_init_slirp_configs(const StringList *fwd, int flags)
     }
 }
 
+static const char **slirp_dnssearch(const StringList *dnsname)
+{
+    const StringList *c = dnsname;
+    size_t i = 0, num_opts = 0;
+    const char **ret;
+
+    while (c) {
+        num_opts++;
+        c = c->next;
+    }
+
+    if (num_opts == 0) {
+        return NULL;
+    }
+
+    ret = g_malloc((num_opts + 1) * sizeof(*ret));
+    c = dnsname;
+    while (c) {
+        ret[i++] = c->value->str;
+        c = c->next;
+    }
+    ret[i] = NULL;
+    return ret;
+}
+
 int net_init_slirp(const NetClientOptions *opts, const char *name,
                    NetClientState *peer)
 {
@@ -705,6 +731,7 @@ int net_init_slirp(const NetClientOptions *opts, const char *name,
     char *vnet;
     int ret;
     const NetdevUserOptions *user;
+    const char **dnssearch;
 
     assert(opts->kind == NET_CLIENT_OPTIONS_KIND_USER);
     user = opts->user;
@@ -713,6 +740,8 @@ int net_init_slirp(const NetClientOptions *opts, const char *name,
            user->has_ip  ? g_strdup_printf("%s/24", user->ip) :
            NULL;
 
+    dnssearch = slirp_dnssearch(user->dnssearch);
+
     /* all optional fields are initialized to "all bits zero" */
 
     net_init_slirp_configs(user->hostfwd, SLIRP_CFG_HOSTFWD);
@@ -721,7 +750,7 @@ int net_init_slirp(const NetClientOptions *opts, const char *name,
     ret = net_slirp_init(peer, "user", name, user->q_restrict, vnet,
                          user->host, user->hostname, user->tftp,
                          user->bootfile, user->dhcpstart, user->dns, user->smb,
-                         user->smbserver);
+                         user->smbserver, dnssearch);
 
     while (slirp_configs) {
         config = slirp_configs;
@@ -730,6 +759,7 @@ int net_init_slirp(const NetClientOptions *opts, const char *name,
     }
 
     g_free(vnet);
+    g_free(dnssearch);
 
     return ret;
 }
diff --git a/net/slirp.h b/net/slirp.h
index 5f685c4fb..2ca09b65b 100644
--- a/net/slirp.h
+++ b/net/slirp.h
@@ -31,9 +31,6 @@
 
 #ifdef CONFIG_SLIRP
 
-int net_init_slirp(const NetClientOptions *opts, const char *name,
-                   NetClientState *peer);
-
 void net_slirp_hostfwd_add(Monitor *mon, const QDict *qdict);
 void net_slirp_hostfwd_remove(Monitor *mon, const QDict *qdict);
 
diff --git a/net/socket.c b/net/socket.c
index c172c249b..c01323d4b 100644
--- a/net/socket.c
+++ b/net/socket.c
@@ -21,17 +21,17 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#include "net/socket.h"
-
 #include "config-host.h"
 
 #include "net.h"
+#include "clients.h"
 #include "monitor.h"
 #include "qemu-char.h"
 #include "qemu-common.h"
 #include "qemu-error.h"
 #include "qemu-option.h"
 #include "qemu_socket.h"
+#include "iov.h"
 
 typedef struct NetSocketState {
     NetClientState nc;
@@ -40,29 +40,106 @@ typedef struct NetSocketState {
     int state; /* 0 = getting length, 1 = getting data */
     unsigned int index;
     unsigned int packet_len;
+    unsigned int send_index;      /* number of bytes sent (only SOCK_STREAM) */
     uint8_t buf[4096];
     struct sockaddr_in dgram_dst; /* contains inet host and port destination iff connectionless (SOCK_DGRAM) */
+    IOHandler *send_fn;           /* differs between SOCK_STREAM/SOCK_DGRAM */
+    bool read_poll;               /* waiting to receive data? */
+    bool write_poll;              /* waiting to transmit data? */
 } NetSocketState;
 
 static void net_socket_accept(void *opaque);
+static void net_socket_writable(void *opaque);
+
+/* Only read packets from socket when peer can receive them */
+static int net_socket_can_send(void *opaque)
+{
+    NetSocketState *s = opaque;
+
+    return qemu_can_send_packet(&s->nc);
+}
+
+static void net_socket_update_fd_handler(NetSocketState *s)
+{
+    qemu_set_fd_handler2(s->fd,
+                         s->read_poll  ? net_socket_can_send : NULL,
+                         s->read_poll  ? s->send_fn : NULL,
+                         s->write_poll ? net_socket_writable : NULL,
+                         s);
+}
+
+static void net_socket_read_poll(NetSocketState *s, bool enable)
+{
+    s->read_poll = enable;
+    net_socket_update_fd_handler(s);
+}
+
+static void net_socket_write_poll(NetSocketState *s, bool enable)
+{
+    s->write_poll = enable;
+    net_socket_update_fd_handler(s);
+}
+
+static void net_socket_writable(void *opaque)
+{
+    NetSocketState *s = opaque;
+
+    net_socket_write_poll(s, false);
+
+    qemu_flush_queued_packets(&s->nc);
+}
 
-/* XXX: we consider we can send the whole packet without blocking */
 static ssize_t net_socket_receive(NetClientState *nc, const uint8_t *buf, size_t size)
 {
     NetSocketState *s = DO_UPCAST(NetSocketState, nc, nc);
-    uint32_t len;
-    len = htonl(size);
-
-    send_all(s->fd, (const uint8_t *)&len, sizeof(len));
-    return send_all(s->fd, buf, size);
+    uint32_t len = htonl(size);
+    struct iovec iov[] = {
+        {
+            .iov_base = &len,
+            .iov_len  = sizeof(len),
+        }, {
+            .iov_base = (void *)buf,
+            .iov_len  = size,
+        },
+    };
+    size_t remaining;
+    ssize_t ret;
+
+    remaining = iov_size(iov, 2) - s->send_index;
+    ret = iov_send(s->fd, iov, 2, s->send_index, remaining);
+
+    if (ret == -1 && errno == EAGAIN) {
+        ret = 0; /* handled further down */
+    }
+    if (ret == -1) {
+        s->send_index = 0;
+        return -errno;
+    }
+    if (ret < (ssize_t)remaining) {
+        s->send_index += ret;
+        net_socket_write_poll(s, true);
+        return 0;
+    }
+    s->send_index = 0;
+    return size;
 }
 
 static ssize_t net_socket_receive_dgram(NetClientState *nc, const uint8_t *buf, size_t size)
 {
     NetSocketState *s = DO_UPCAST(NetSocketState, nc, nc);
+    ssize_t ret;
+
+    do {
+        ret = qemu_sendto(s->fd, buf, size, 0,
+                          (struct sockaddr *)&s->dgram_dst,
+                          sizeof(s->dgram_dst));
+    } while (ret == -1 && errno == EINTR);
 
-    return sendto(s->fd, (const void *)buf, size, 0,
-                  (struct sockaddr *)&s->dgram_dst, sizeof(s->dgram_dst));
+    if (ret == -1 && errno == EAGAIN) {
+        net_socket_write_poll(s, true);
+        return 0;
+    }
+    return ret;
 }
 
 static void net_socket_send(void *opaque)
@@ -81,7 +158,8 @@ static void net_socket_send(void *opaque)
     } else if (size == 0) {
         /* end of connection */
     eoc:
-        qemu_set_fd_handler(s->fd, NULL, NULL, NULL);
+        net_socket_read_poll(s, false);
+        net_socket_write_poll(s, false);
         if (s->listen_fd != -1) {
             qemu_set_fd_handler(s->listen_fd, net_socket_accept, NULL, s);
         }
@@ -152,7 +230,8 @@ static void net_socket_send_dgram(void *opaque)
         return;
     if (size == 0) {
         /* end of connection */
-        qemu_set_fd_handler(s->fd, NULL, NULL, NULL);
+        net_socket_read_poll(s, false);
+        net_socket_write_poll(s, false);
         return;
     }
     qemu_send_packet(&s->nc, s->buf, size);
@@ -243,7 +322,8 @@ static void net_socket_cleanup(NetClientState *nc)
 {
     NetSocketState *s = DO_UPCAST(NetSocketState, nc, nc);
     if (s->fd != -1) {
-        qemu_set_fd_handler(s->fd, NULL, NULL, NULL);
+        net_socket_read_poll(s, false);
+        net_socket_write_poll(s, false);
         close(s->fd);
         s->fd = -1;
     }
@@ -314,8 +394,8 @@ static NetSocketState *net_socket_fd_init_dgram(NetClientState *peer,
 
     s->fd = fd;
     s->listen_fd = -1;
-
-    qemu_set_fd_handler(s->fd, net_socket_send_dgram, NULL, s);
+    s->send_fn = net_socket_send_dgram;
+    net_socket_read_poll(s, true);
 
     /* mcast: save bound address as dst */
     if (is_connected) {
@@ -332,7 +412,8 @@ err:
 static void net_socket_connect(void *opaque)
 {
     NetSocketState *s = opaque;
-    qemu_set_fd_handler(s->fd, net_socket_send, NULL, s);
+    s->send_fn = net_socket_send;
+    net_socket_read_poll(s, true);
 }
 
 static NetClientInfo net_socket_info = {
@@ -629,7 +710,7 @@ int net_init_socket(const NetClientOptions *opts, const char *name,
     if (sock->has_fd) {
         int fd;
 
-        fd = net_handle_fd_param(cur_mon, sock->fd);
+        fd = monitor_handle_fd_param(cur_mon, sock->fd);
         if (fd == -1 || !net_socket_fd_init(peer, "socket", name, fd, 1)) {
             return -1;
         }
@@ -666,7 +747,7 @@ int net_init_socket(const NetClientOptions *opts, const char *name,
         error_report("localaddr= is mandatory with udp=");
         return -1;
     }
-    if (net_socket_udp_init(peer, "udp", name, sock->udp, sock->localaddr) ==
+    if (net_socket_udp_init(peer, "socket", name, sock->udp, sock->localaddr) ==
         -1) {
         return -1;
     }
diff --git a/net/tap-win32.c b/net/tap-win32.c
index c0ea954ca..8d2d32b1c 100644
--- a/net/tap-win32.c
+++ b/net/tap-win32.c
@@ -26,9 +26,10 @@
  *  distribution); if not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "net/tap.h"
+#include "tap.h"
 
 #include "qemu-common.h"
+#include "clients.h"            /* net_init_tap */
 #include "net.h"
 #include "sysemu.h"
 #include "qemu-error.h"
@@ -751,3 +752,13 @@ struct vhost_net *tap_get_vhost_net(NetClientState *nc)
 {
     return NULL;
 }
+
+int tap_has_vnet_hdr_len(NetClientState *nc, int len)
+{
+    return 0;
+}
+
+void tap_set_vnet_hdr_len(NetClientState *nc, int len)
+{
+    assert(0);
+}
diff --git a/net/tap.c b/net/tap.c
index 197152579..1abfd44bd 100644
--- a/net/tap.c
+++ b/net/tap.c
@@ -23,7 +23,7 @@
  * THE SOFTWARE.
  */
 
-#include "net/tap.h"
+#include "tap.h"
 
 #include "config-host.h"
 
@@ -34,6 +34,7 @@
 #include <net/if.h>
 
 #include "net.h"
+#include "clients.h"
 #include "monitor.h"
 #include "sysemu.h"
 #include "qemu-char.h"
@@ -340,6 +341,13 @@ static TAPState *net_tap_fd_init(NetClientState *peer,
     s->using_vnet_hdr = 0;
     s->has_ufo = tap_probe_has_ufo(s->fd);
     tap_set_offload(&s->nc, 0, 0, 0, 0, 0);
+    /*
+     * Make sure host header length is set correctly in tap:
+     * it might have been modified by another instance of qemu.
+     */
+    if (tap_probe_vnet_hdr_len(s->fd, s->host_vnet_hdr_len)) {
+        tap_fd_set_vnet_hdr_len(s->fd, s->host_vnet_hdr_len);
+    }
     tap_read_poll(s, 1);
     s->vhost_net = NULL;
     return s;
@@ -610,7 +618,7 @@ int net_init_tap(const NetClientOptions *opts, const char *name,
             return -1;
         }
 
-        fd = net_handle_fd_param(cur_mon, tap->fd);
+        fd = monitor_handle_fd_param(cur_mon, tap->fd);
         if (fd == -1) {
             return -1;
         }
@@ -686,7 +694,7 @@ int net_init_tap(const NetClientOptions *opts, const char *name,
         int vhostfd;
 
         if (tap->has_vhostfd) {
-            vhostfd = net_handle_fd_param(cur_mon, tap->vhostfd);
+            vhostfd = monitor_handle_fd_param(cur_mon, tap->vhostfd);
             if (vhostfd == -1) {
                 return -1;
             }
diff --git a/net/tap.h b/net/tap.h
index 0fb018c4b..d44d83ae7 100644
--- a/net/tap.h
+++ b/net/tap.h
@@ -32,9 +32,6 @@
 #define DEFAULT_NETWORK_SCRIPT "/etc/qemu-ifup"
 #define DEFAULT_NETWORK_DOWN_SCRIPT "/etc/qemu-ifdown"
 
-int net_init_tap(const NetClientOptions *opts, const char *name,
-                 NetClientState *peer);
-
 int tap_open(char *ifname, int ifname_size, int *vnet_hdr, int vnet_hdr_required);
 
 ssize_t tap_read_packet(int tapfd, uint8_t *buf, int maxlen);
@@ -58,7 +55,4 @@ int tap_get_fd(NetClientState *nc);
 struct vhost_net;
 struct vhost_net *tap_get_vhost_net(NetClientState *nc);
 
-int net_init_bridge(const NetClientOptions *opts, const char *name,
-                    NetClientState *peer);
-
 #endif /* QEMU_NET_TAP_H */
diff --git a/net/vde.c b/net/vde.c
index b91a6c799..275bda92c 100644
--- a/net/vde.c
+++ b/net/vde.c
@@ -21,13 +21,12 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
-#include "net/vde.h"
-
 #include "config-host.h"
 
 #include <libvdeplug.h>
 
 #include "net.h"
+#include "clients.h"
 #include "qemu-char.h"
 #include "qemu-common.h"
 #include "qemu-option.h"
diff --git a/net/vde.h b/net/vde.h
deleted file mode 100644
index 6ce669893..000000000
--- a/net/vde.h
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * QEMU System Emulator
- *
- * Copyright (c) 2003-2008 Fabrice Bellard
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
- * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- */
-#ifndef QEMU_NET_VDE_H
-#define QEMU_NET_VDE_H
-
-#include "qemu-common.h"
-#include "qapi-types.h"
-
-#ifdef CONFIG_VDE
-
-int net_init_vde(const NetClientOptions *opts, const char *name,
-                 NetClientState *peer);
-
-#endif /* CONFIG_VDE */
-
-#endif /* QEMU_NET_VDE_H */
diff --git a/os-posix.c b/os-posix.c
index 79fa2288e..488e48005 100644
--- a/os-posix.c
+++ b/os-posix.c
@@ -148,8 +148,7 @@ void os_set_proc_name(const char *s)
     char name[16];
     if (!s)
         return;
-    name[sizeof(name) - 1] = 0;
-    strncpy(name, s, sizeof(name));
+    pstrcpy(name, sizeof(name), s);
     /* Could rewrite argv[0] too, but that's a bit more complicated.
        This simple way is enough for `top'. */
     if (prctl(PR_SET_NAME, name)) {
@@ -194,7 +193,6 @@ void os_parse_cmd_args(int index, const char *optarg)
         break;
 #endif
     }
-    return;
 }
 
 static void change_process_uid(void)
@@ -360,3 +358,8 @@ int qemu_create_pidfile(const char *filename)
     /* keep pidfile open & locked forever */
     return 0;
 }
+
+bool is_daemonized(void)
+{
+    return daemonize;
+}
diff --git a/osdep.c b/osdep.c
index 3b25297a2..3a63d26e7 100644
--- a/osdep.c
+++ b/osdep.c
@@ -88,7 +88,6 @@ static int qemu_dup_flags(int fd, int flags)
     int ret;
     int serrno;
     int dup_flags;
-    int setfl_flags;
 
 #ifdef F_DUPFD_CLOEXEC
     ret = fcntl(fd, F_DUPFD_CLOEXEC, 0);
@@ -113,16 +112,7 @@ static int qemu_dup_flags(int fd, int flags)
     }
 
     /* Set/unset flags that we can with fcntl */
-    setfl_flags = O_APPEND | O_ASYNC | O_NONBLOCK;
-#ifdef O_NOATIME
-    setfl_flags |= O_NOATIME;
-#endif
-#ifdef O_DIRECT
-    setfl_flags |= O_DIRECT;
-#endif
-    dup_flags &= ~setfl_flags;
-    dup_flags |= (flags & setfl_flags);
-    if (fcntl(ret, F_SETFL, dup_flags) == -1) {
+    if (fcntl(ret, F_SETFL, flags) == -1) {
         goto fail;
     }
 
@@ -144,6 +134,11 @@ fail:
     errno = serrno;
     return -1;
 }
+
+static int qemu_parse_fdset(const char *param)
+{
+    return qemu_parse_fd(param);
+}
 #endif
 
 /*
@@ -404,3 +399,4 @@ bool fips_get_state(void)
 {
     return fips_enabled;
 }
+
diff --git a/osdep.h b/osdep.h
index cb213e029..87d3b9cfa 100644
--- a/osdep.h
+++ b/osdep.h
@@ -108,6 +108,11 @@ void qemu_vfree(void *ptr);
 #else
 #define QEMU_MADV_DONTDUMP QEMU_MADV_INVALID
 #endif
+#ifdef MADV_HUGEPAGE
+#define QEMU_MADV_HUGEPAGE MADV_HUGEPAGE
+#else
+#define QEMU_MADV_HUGEPAGE QEMU_MADV_INVALID
+#endif
 
 #elif defined(CONFIG_POSIX_MADVISE)
 
@@ -116,6 +121,7 @@ void qemu_vfree(void *ptr);
 #define QEMU_MADV_DONTFORK  QEMU_MADV_INVALID
 #define QEMU_MADV_MERGEABLE QEMU_MADV_INVALID
 #define QEMU_MADV_DONTDUMP QEMU_MADV_INVALID
+#define QEMU_MADV_HUGEPAGE  QEMU_MADV_INVALID
 
 #else /* no-op */
 
@@ -124,11 +130,15 @@ void qemu_vfree(void *ptr);
 #define QEMU_MADV_DONTFORK  QEMU_MADV_INVALID
 #define QEMU_MADV_MERGEABLE QEMU_MADV_INVALID
 #define QEMU_MADV_DONTDUMP QEMU_MADV_INVALID
+#define QEMU_MADV_HUGEPAGE  QEMU_MADV_INVALID
 
 #endif
 
 int qemu_madvise(void *addr, size_t len, int advice);
 
+int qemu_open(const char *name, int flags, ...);
+int qemu_close(int fd);
+
 #if defined(__HAIKU__) && defined(__i386__)
 #define FMT_pid "%ld"
 #elif defined(WIN64)
diff --git a/oslib-posix.c b/oslib-posix.c
index dbeb6272b..9db9c3d8a 100644
--- a/oslib-posix.c
+++ b/oslib-posix.c
@@ -61,9 +61,6 @@ static int running_on_valgrind = -1;
 #ifdef CONFIG_LINUX
 #include <sys/syscall.h>
 #endif
-#ifdef CONFIG_EVENTFD
-#include <sys/eventfd.h>
-#endif
 
 int qemu_get_thread_id(void)
 {
@@ -183,34 +180,6 @@ int qemu_pipe(int pipefd[2])
     return ret;
 }
 
-/*
- * Creates an eventfd that looks like a pipe and has EFD_CLOEXEC set.
- */
-int qemu_eventfd(int fds[2])
-{
-#ifdef CONFIG_EVENTFD
-    int ret;
-
-    ret = eventfd(0, 0);
-    if (ret >= 0) {
-        fds[0] = ret;
-        fds[1] = dup(ret);
-        if (fds[1] == -1) {
-            close(ret);
-            return -1;
-        }
-        qemu_set_cloexec(ret);
-        qemu_set_cloexec(fds[1]);
-        return 0;
-    }
-    if (errno != ENOSYS) {
-        return -1;
-    }
-#endif
-
-    return qemu_pipe(fds);
-}
-
 int qemu_utimens(const char *path, const struct timespec *times)
 {
     struct timeval tv[2], tv_now;
diff --git a/oslib-win32.c b/oslib-win32.c
index ffbc6d0c9..51b33e8b2 100644
--- a/oslib-win32.c
+++ b/oslib-win32.c
@@ -74,6 +74,30 @@ void qemu_vfree(void *ptr)
     VirtualFree(ptr, 0, MEM_RELEASE);
 }
 
+/* FIXME: add proper locking */
+struct tm *gmtime_r(const time_t *timep, struct tm *result)
+{
+    struct tm *p = gmtime(timep);
+    memset(result, 0, sizeof(*result));
+    if (p) {
+        *result = *p;
+        p = result;
+    }
+    return p;
+}
+
+/* FIXME: add proper locking */
+struct tm *localtime_r(const time_t *timep, struct tm *result)
+{
+    struct tm *p = localtime(timep);
+    memset(result, 0, sizeof(*result));
+    if (p) {
+        *result = *p;
+        p = result;
+    }
+    return p;
+}
+
 void socket_set_block(int fd)
 {
     unsigned long opt = 0;
diff --git a/pc-bios/acpi-dsdt.aml b/pc-bios/acpi-dsdt.aml
new file mode 100644
index 000000000..bb3dd83a5
--- /dev/null
+++ b/pc-bios/acpi-dsdt.aml
diff --git a/pc-bios/bios.bin b/pc-bios/bios.bin
index eac67cb05..dc9b57ddc 100644
--- a/pc-bios/bios.bin
+++ b/pc-bios/bios.bin
diff --git a/pc-bios/multiboot.bin b/pc-bios/multiboot.bin
index f74a6e142..7b3c1745a 100644
--- a/pc-bios/multiboot.bin
+++ b/pc-bios/multiboot.bin
diff --git a/pc-bios/optionrom/multiboot.S b/pc-bios/optionrom/multiboot.S
index f08222a3c..003bcfb49 100644
--- a/pc-bios/optionrom/multiboot.S
+++ b/pc-bios/optionrom/multiboot.S
@@ -75,6 +75,13 @@ run_multiboot:
 	shr		$4, %eax
 	mov		%ax, %fs
 
+	/* Account for the EBDA in the multiboot structure's e801
+	 * map.
+	 */
+	int		$0x12
+	cwtl
+	movl		%eax, %fs:4
+
 	/* ES = mmap_addr */
 	mov 		%fs:48, %eax
 	shr		$4, %eax
diff --git a/pflib.c b/pflib.c
deleted file mode 100644
index 987e11001..000000000
--- a/pflib.c
+++ /dev/null
@@ -1,215 +0,0 @@
-/*
- * PixelFormat conversion library.
- *
- * Author: Gerd Hoffmann <kraxel@redhat.com>
- *
- * This work is licensed under the terms of the GNU GPL, version 2.  See
- * the COPYING file in the top-level directory.
- *
- * Contributions after 2012-01-13 are licensed under the terms of the
- * GNU GPL, version 2 or (at your option) any later version.
- */
-#include "qemu-common.h"
-#include "console.h"
-#include "pflib.h"
-
-typedef struct QemuPixel QemuPixel;
-
-typedef void (*pf_convert)(QemuPfConv *conv,
-                           void *dst, void *src, uint32_t cnt);
-typedef void (*pf_convert_from)(PixelFormat *pf,
-                                QemuPixel *dst, void *src, uint32_t cnt);
-typedef void (*pf_convert_to)(PixelFormat *pf,
-                              void *dst, QemuPixel *src, uint32_t cnt);
-
-struct QemuPfConv {
-    pf_convert        convert;
-    PixelFormat       src;
-    PixelFormat       dst;
-
-    /* for copy_generic() */
-    pf_convert_from   conv_from;
-    pf_convert_to     conv_to;
-    QemuPixel         *conv_buf;
-    uint32_t          conv_cnt;
-};
-
-struct QemuPixel {
-    uint8_t red;
-    uint8_t green;
-    uint8_t blue;
-    uint8_t alpha;
-};
-
-/* ----------------------------------------------------------------------- */
-/* PixelFormat -> QemuPixel conversions                                    */
-
-static void conv_16_to_pixel(PixelFormat *pf,
-                             QemuPixel *dst, void *src, uint32_t cnt)
-{
-    uint16_t *src16 = src;
-
-    while (cnt > 0) {
-        dst->red   = ((*src16 & pf->rmask) >> pf->rshift) << (8 - pf->rbits);
-        dst->green = ((*src16 & pf->gmask) >> pf->gshift) << (8 - pf->gbits);
-        dst->blue  = ((*src16 & pf->bmask) >> pf->bshift) << (8 - pf->bbits);
-        dst->alpha = ((*src16 & pf->amask) >> pf->ashift) << (8 - pf->abits);
-        dst++, src16++, cnt--;
-    }
-}
-
-/* assumes pf->{r,g,b,a}bits == 8 */
-static void conv_32_to_pixel_fast(PixelFormat *pf,
-                                  QemuPixel *dst, void *src, uint32_t cnt)
-{
-    uint32_t *src32 = src;
-
-    while (cnt > 0) {
-        dst->red   = (*src32 & pf->rmask) >> pf->rshift;
-        dst->green = (*src32 & pf->gmask) >> pf->gshift;
-        dst->blue  = (*src32 & pf->bmask) >> pf->bshift;
-        dst->alpha = (*src32 & pf->amask) >> pf->ashift;
-        dst++, src32++, cnt--;
-    }
-}
-
-static void conv_32_to_pixel_generic(PixelFormat *pf,
-                                     QemuPixel *dst, void *src, uint32_t cnt)
-{
-    uint32_t *src32 = src;
-
-    while (cnt > 0) {
-        if (pf->rbits < 8) {
-            dst->red   = ((*src32 & pf->rmask) >> pf->rshift) << (8 - pf->rbits);
-        } else {
-            dst->red   = ((*src32 & pf->rmask) >> pf->rshift) >> (pf->rbits - 8);
-        }
-        if (pf->gbits < 8) {
-            dst->green = ((*src32 & pf->gmask) >> pf->gshift) << (8 - pf->gbits);
-        } else {
-            dst->green = ((*src32 & pf->gmask) >> pf->gshift) >> (pf->gbits - 8);
-        }
-        if (pf->bbits < 8) {
-            dst->blue  = ((*src32 & pf->bmask) >> pf->bshift) << (8 - pf->bbits);
-        } else {
-            dst->blue  = ((*src32 & pf->bmask) >> pf->bshift) >> (pf->bbits - 8);
-        }
-        if (pf->abits < 8) {
-            dst->alpha = ((*src32 & pf->amask) >> pf->ashift) << (8 - pf->abits);
-        } else {
-            dst->alpha = ((*src32 & pf->amask) >> pf->ashift) >> (pf->abits - 8);
-        }
-        dst++, src32++, cnt--;
-    }
-}
-
-/* ----------------------------------------------------------------------- */
-/* QemuPixel -> PixelFormat conversions                                    */
-
-static void conv_pixel_to_16(PixelFormat *pf,
-                             void *dst, QemuPixel *src, uint32_t cnt)
-{
-    uint16_t *dst16 = dst;
-
-    while (cnt > 0) {
-        *dst16  = ((uint16_t)src->red   >> (8 - pf->rbits)) << pf->rshift;
-        *dst16 |= ((uint16_t)src->green >> (8 - pf->gbits)) << pf->gshift;
-        *dst16 |= ((uint16_t)src->blue  >> (8 - pf->bbits)) << pf->bshift;
-        *dst16 |= ((uint16_t)src->alpha >> (8 - pf->abits)) << pf->ashift;
-        dst16++, src++, cnt--;
-    }
-}
-
-static void conv_pixel_to_32(PixelFormat *pf,
-                             void *dst, QemuPixel *src, uint32_t cnt)
-{
-    uint32_t *dst32 = dst;
-
-    while (cnt > 0) {
-        *dst32  = ((uint32_t)src->red   >> (8 - pf->rbits)) << pf->rshift;
-        *dst32 |= ((uint32_t)src->green >> (8 - pf->gbits)) << pf->gshift;
-        *dst32 |= ((uint32_t)src->blue  >> (8 - pf->bbits)) << pf->bshift;
-        *dst32 |= ((uint32_t)src->alpha >> (8 - pf->abits)) << pf->ashift;
-        dst32++, src++, cnt--;
-    }
-}
-
-/* ----------------------------------------------------------------------- */
-/* PixelFormat -> PixelFormat conversions                                  */
-
-static void convert_copy(QemuPfConv *conv, void *dst, void *src, uint32_t cnt)
-{
-    uint32_t bytes = cnt * conv->src.bytes_per_pixel;
-    memcpy(dst, src, bytes);
-}
-
-static void convert_generic(QemuPfConv *conv, void *dst, void *src, uint32_t cnt)
-{
-    if (conv->conv_cnt < cnt) {
-        conv->conv_cnt = cnt;
-        conv->conv_buf = g_realloc(conv->conv_buf, sizeof(QemuPixel) * conv->conv_cnt);
-    }
-    conv->conv_from(&conv->src, conv->conv_buf, src, cnt);
-    conv->conv_to(&conv->dst, dst, conv->conv_buf, cnt);
-}
-
-/* ----------------------------------------------------------------------- */
-/* public interface                                                        */
-
-QemuPfConv *qemu_pf_conv_get(PixelFormat *dst, PixelFormat *src)
-{
-    QemuPfConv *conv = g_malloc0(sizeof(QemuPfConv));
-
-    conv->src = *src;
-    conv->dst = *dst;
-
-    if (memcmp(&conv->src, &conv->dst, sizeof(PixelFormat)) == 0) {
-        /* formats identical, can simply copy */
-        conv->convert = convert_copy;
-    } else {
-        /* generic two-step conversion: src -> QemuPixel -> dst  */
-        switch (conv->src.bytes_per_pixel) {
-        case 2:
-            conv->conv_from = conv_16_to_pixel;
-            break;
-        case 4:
-            if (conv->src.rbits == 8 && conv->src.gbits == 8 && conv->src.bbits == 8) {
-                conv->conv_from = conv_32_to_pixel_fast;
-            } else {
-                conv->conv_from = conv_32_to_pixel_generic;
-            }
-            break;
-        default:
-            goto err;
-        }
-        switch (conv->dst.bytes_per_pixel) {
-        case 2:
-            conv->conv_to = conv_pixel_to_16;
-            break;
-        case 4:
-            conv->conv_to = conv_pixel_to_32;
-            break;
-        default:
-            goto err;
-        }
-        conv->convert = convert_generic;
-    }
-    return conv;
-
-err:
-    g_free(conv);
-    return NULL;
-}
-
-void qemu_pf_conv_run(QemuPfConv *conv, void *dst, void *src, uint32_t cnt)
-{
-    conv->convert(conv, dst, src, cnt);
-}
-
-void qemu_pf_conv_put(QemuPfConv *conv)
-{
-    if (conv) {
-        g_free(conv->conv_buf);
-        g_free(conv);
-    }
-}
diff --git a/pflib.h b/pflib.h
deleted file mode 100644
index b70c313ac..000000000
--- a/pflib.h
+++ /dev/null
@@ -1,20 +0,0 @@
-#ifndef __QEMU_PFLIB_H
-#define __QEMU_PFLIB_H
-
-/*
- * PixelFormat conversion library.
- *
- * Author: Gerd Hoffmann <kraxel@redhat.com>
- *
- * This work is licensed under the terms of the GNU GPL, version 2.  See
- * the COPYING file in the top-level directory.
- *
- */
-
-typedef struct QemuPfConv QemuPfConv;
-
-QemuPfConv *qemu_pf_conv_get(PixelFormat *dst, PixelFormat *src);
-void qemu_pf_conv_run(QemuPfConv *conv, void *dst, void *src, uint32_t cnt);
-void qemu_pf_conv_put(QemuPfConv *conv);
-
-#endif
diff --git a/pixman/.gitignore b/pixman/.gitignore
new file mode 100644
index 000000000..96d222bbe
--- /dev/null
+++ b/pixman/.gitignore
@@ -0,0 +1,54 @@
+Makefile
+Makefile.in
+.deps
+.libs
+.msg
+*.pc
+*.lo
+*.la
+*.a
+*.o
+*~
+aclocal.m4
+autom4te.cache
+compile
+config.guess
+config.log
+config.status
+config.sub
+configure
+depcomp
+install-sh
+libtool
+ltmain.sh
+missing
+stamp-h?
+config.h
+config.h.in
+.*.swp
+pixman/pixman-combine32.c
+pixman/pixman-combine32.h
+pixman/pixman-combine64.c
+pixman/pixman-combine64.h
+pixman/pixman-version.h
+test/alpha-test
+test/blitters-test
+test/clip-in
+test/clip-test
+test/composite
+test/composite-test
+test/convolution-test
+test/fetch-test
+test/gradient-test
+test/oob-test
+test/region-test
+test/scaling-test
+test/screen-test
+test/trap-crasher
+test/trap-test
+test/window-test
+*.pdb
+*.dll
+*.lib
+*.ilk
+*.obj
diff --git a/pixman/AUTHORS b/pixman/AUTHORS
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/pixman/AUTHORS
diff --git a/pixman/CODING_STYLE b/pixman/CODING_STYLE
new file mode 100644
index 000000000..126f1a981
--- /dev/null
+++ b/pixman/CODING_STYLE
@@ -0,0 +1,216 @@
+Pixman coding style.
+====================
+
+The pixman coding style is close to cairo's with one exception: braces
+go on their own line, rather than on the line of the if/while/for:
+
+	if (condition)
+	{
+	    do_something();
+	    do_something_else();
+	}
+
+not
+
+	if (condition) {
+	    do_something();
+	    do_something_else();
+        }
+
+
+
+Specific guidelines:
+
+
+Indentation
+===========
+
+Each new level is indented four spaces:
+
+	if (condition)
+	    do_something();
+
+This may be achieved with space characters or with a combination of
+tab characters and space characters. Tab characters are interpreted as
+
+	Advance to the next column which is a multiple of 8.
+
+
+Names
+=====
+
+In all names, words are separated with underscores. Do not use
+CamelCase for any names.
+
+Macros have ALL_CAPITAL_NAMES
+
+Type names are in lower case and end with "_t". For example
+pixman_image_t.
+
+Labels, functions and variables have lower case names.
+
+
+Braces
+======
+
+Braces always go on their own line:
+
+	if (condition)
+	{
+	    do_this ();
+	    do_that ();
+	}
+	else
+	{
+	    do_the_other ();
+	}
+
+Rules for braces and substatements of if/while/for/do:
+
+* If a substatement spans multiple lines, then there must be braces
+  around it.
+
+* If the condition of an if/while/for spans multiple lines, then 
+  braces must be used for the substatements.
+
+* If one substatement of an if statement has braces, then the other
+  must too.
+
+* Otherwise, don't add braces.
+
+
+Comments
+========
+
+For comments either like this:
+
+        /* One line comment */
+
+or like this:
+
+	/* This is a multi-line comment
+	 *
+         * It extends over multiple lines
+	 */
+
+Generally comments should say things that is clear from the code
+itself. If too many comments say obvious things, then people will just
+stop reading all comments, including the good ones.
+
+
+Whitespace
+==========
+
+* Put a single space after commas
+
+* Put spaces around arithmetic operators such a +, -, *, /:
+
+        y * stride + x
+
+        x / unit_x
+
+* Do not put spaces after the address-of operator, the * when used as
+  a pointer derefernce or the ! and ~ operators:
+
+     &foo;
+
+     ~0x00000000
+
+     !condition
+
+     *result = 100
+
+* Break up long lines (> ~80 characters) and use whitespace to align
+  things nicely. This is one way:
+
+  	 some_very_long_function name (
+	 	implementation, op, src, mask, dest, 
+		src_x, src_y, mask_x, mask_y, dest_x, dest_y,
+		width, height);
+
+  This is another:
+
+        some_very_long_function_name (implementation, op,
+                                      src, mask, dest,
+				      src_x, src_y,
+				      mask_x, mask_y,
+				      dest_x, dest_y,
+				      width, height);
+
+* Separate logically distinct chunks with a single newline. This
+  obviously applies between functions, but also applies within a
+  function or block or structure definition.
+
+* Use a newline after a block of variable declarations.
+
+* Use a single space before a left parenthesis, except where the
+  standard will not allow it, (eg. when defining a parameterized macro).
+
+* Don't eliminate newlines just because things would still fit on one
+  line. This breaks the expected visual structure of the code making
+  it much harder to read and understand:
+
+	if (condition) foo (); else bar ();	/* Yuck! */
+
+* Do eliminate trailing whitespace (space or tab characters) on any
+  line. Also, avoid putting initial or final blank lines into any
+  file, and never use multiple blank lines instead of a single blank
+  line.
+
+* Do enable the default git pre-commit hook that detect trailing
+  whitespace for you and help you to avoid corrupting cairo's tree
+  with it. Do that as follows:
+
+	chmod a+x .git/hooks/pre-commit
+
+* You might also find the git-stripspace utility helpful which acts as
+  a filter to remove trailing whitespace as well as initial, final,
+  and duplicate blank lines.
+
+
+Function Definitions
+====================
+
+Function definitions should take the following form:
+
+	void
+	my_function (argument)
+	{
+	    do_my_things ();
+	}
+
+If all the parameters to a function fit naturally on one line, format
+them that way. Otherwise, put one argument on each line, adding
+whitespace so that the parameter names are aligned with each other.
+
+I.e., do either this:
+
+        void
+        short_arguments (const char *str, int x, int y, int z)
+        {
+        }
+
+or this:
+
+	void
+	long_arguments (const char *char_star_arg,
+			int	    int_arg,
+			double	   *double_star_arg,
+			double	    double_arg)
+	{
+	}
+
+
+Mode lines
+==========
+
+Given the rules above, what is the best way to simplify one's life as
+a code monkey? Get your editor to do most of the tedious work of
+beautifying your code!
+
+As a reward for reading this far, here are some mode lines for the more
+popular editors:
+/*
+ * vim:sw=4:sts=4:ts=8:tw=78:fo=tcroq:cindent:cino=\:0,(0
+ * vim:isk=a-z,A-Z,48-57,_,.,-,>
+ */
diff --git a/pixman/COPYING b/pixman/COPYING
new file mode 100644
index 000000000..b0571e6a6
--- /dev/null
+++ b/pixman/COPYING
@@ -0,0 +1,40 @@
+The following is the 'standard copyright' agreed upon by most contributors,
+and is currently the canonical license, though a modification is currently
+under discussion.  Copyright holders of new code should use this license
+statement where possible, and append their name to this list.  
+
+Copyright 1987, 1988, 1989, 1998  The Open Group
+Copyright 1987, 1988, 1989 Digital Equipment Corporation
+Copyright 1999, 2004, 2008 Keith Packard
+Copyright 2000 SuSE, Inc.
+Copyright 2000 Keith Packard, member of The XFree86 Project, Inc.
+Copyright 2004, 2005, 2007, 2008 Red Hat, Inc.
+Copyright 2004 Nicholas Miell
+Copyright 2005 Lars Knoll & Zack Rusin, Trolltech
+Copyright 2005 Trolltech AS
+Copyright 2007 Luca Barbato
+Copyright 2008 Aaron Plattner, NVIDIA Corporation
+Copyright 2008 Rodrigo Kumpera
+Copyright 2008 André Tupinambá
+Copyright 2008 Mozilla Corporation
+Copyright 2008 Frederic Plourde
+Copyright 2009 Sun Microsystems, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a
+copy of this software and associated documentation files (the "Software"),
+to deal in the Software without restriction, including without limitation
+the rights to use, copy, modify, merge, publish, distribute, sublicense,
+and/or sell copies of the Software, and to permit persons to whom the
+Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice (including the next
+paragraph) shall be included in all copies or substantial portions of the
+Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
diff --git a/pixman/ChangeLog b/pixman/ChangeLog
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/pixman/ChangeLog
diff --git a/pixman/INSTALL b/pixman/INSTALL
new file mode 100644
index 000000000..5458714e1
--- /dev/null
+++ b/pixman/INSTALL
@@ -0,0 +1,234 @@
+Installation Instructions
+*************************
+
+Copyright (C) 1994, 1995, 1996, 1999, 2000, 2001, 2002, 2004, 2005,
+2006 Free Software Foundation, Inc.
+
+This file is free documentation; the Free Software Foundation gives
+unlimited permission to copy, distribute and modify it.
+
+Basic Installation
+==================
+
+Briefly, the shell commands `./configure; make; make install' should
+configure, build, and install this package.  The following
+more-detailed instructions are generic; see the `README' file for
+instructions specific to this package.
+
+   The `configure' shell script attempts to guess correct values for
+various system-dependent variables used during compilation.  It uses
+those values to create a `Makefile' in each directory of the package.
+It may also create one or more `.h' files containing system-dependent
+definitions.  Finally, it creates a shell script `config.status' that
+you can run in the future to recreate the current configuration, and a
+file `config.log' containing compiler output (useful mainly for
+debugging `configure').
+
+   It can also use an optional file (typically called `config.cache'
+and enabled with `--cache-file=config.cache' or simply `-C') that saves
+the results of its tests to speed up reconfiguring.  Caching is
+disabled by default to prevent problems with accidental use of stale
+cache files.
+
+   If you need to do unusual things to compile the package, please try
+to figure out how `configure' could check whether to do them, and mail
+diffs or instructions to the address given in the `README' so they can
+be considered for the next release.  If you are using the cache, and at
+some point `config.cache' contains results you don't want to keep, you
+may remove or edit it.
+
+   The file `configure.ac' (or `configure.in') is used to create
+`configure' by a program called `autoconf'.  You need `configure.ac' if
+you want to change it or regenerate `configure' using a newer version
+of `autoconf'.
+
+The simplest way to compile this package is:
+
+  1. `cd' to the directory containing the package's source code and type
+     `./configure' to configure the package for your system.
+
+     Running `configure' might take a while.  While running, it prints
+     some messages telling which features it is checking for.
+
+  2. Type `make' to compile the package.
+
+  3. Optionally, type `make check' to run any self-tests that come with
+     the package.
+
+  4. Type `make install' to install the programs and any data files and
+     documentation.
+
+  5. You can remove the program binaries and object files from the
+     source code directory by typing `make clean'.  To also remove the
+     files that `configure' created (so you can compile the package for
+     a different kind of computer), type `make distclean'.  There is
+     also a `make maintainer-clean' target, but that is intended mainly
+     for the package's developers.  If you use it, you may have to get
+     all sorts of other programs in order to regenerate files that came
+     with the distribution.
+
+Compilers and Options
+=====================
+
+Some systems require unusual options for compilation or linking that the
+`configure' script does not know about.  Run `./configure --help' for
+details on some of the pertinent environment variables.
+
+   You can give `configure' initial values for configuration parameters
+by setting variables in the command line or in the environment.  Here
+is an example:
+
+     ./configure CC=c99 CFLAGS=-g LIBS=-lposix
+
+   *Note Defining Variables::, for more details.
+
+Compiling For Multiple Architectures
+====================================
+
+You can compile the package for more than one kind of computer at the
+same time, by placing the object files for each architecture in their
+own directory.  To do this, you can use GNU `make'.  `cd' to the
+directory where you want the object files and executables to go and run
+the `configure' script.  `configure' automatically checks for the
+source code in the directory that `configure' is in and in `..'.
+
+   With a non-GNU `make', it is safer to compile the package for one
+architecture at a time in the source code directory.  After you have
+installed the package for one architecture, use `make distclean' before
+reconfiguring for another architecture.
+
+Installation Names
+==================
+
+By default, `make install' installs the package's commands under
+`/usr/local/bin', include files under `/usr/local/include', etc.  You
+can specify an installation prefix other than `/usr/local' by giving
+`configure' the option `--prefix=PREFIX'.
+
+   You can specify separate installation prefixes for
+architecture-specific files and architecture-independent files.  If you
+pass the option `--exec-prefix=PREFIX' to `configure', the package uses
+PREFIX as the prefix for installing programs and libraries.
+Documentation and other data files still use the regular prefix.
+
+   In addition, if you use an unusual directory layout you can give
+options like `--bindir=DIR' to specify different values for particular
+kinds of files.  Run `configure --help' for a list of the directories
+you can set and what kinds of files go in them.
+
+   If the package supports it, you can cause programs to be installed
+with an extra prefix or suffix on their names by giving `configure' the
+option `--program-prefix=PREFIX' or `--program-suffix=SUFFIX'.
+
+Optional Features
+=================
+
+Some packages pay attention to `--enable-FEATURE' options to
+`configure', where FEATURE indicates an optional part of the package.
+They may also pay attention to `--with-PACKAGE' options, where PACKAGE
+is something like `gnu-as' or `x' (for the X Window System).  The
+`README' should mention any `--enable-' and `--with-' options that the
+package recognizes.
+
+   For packages that use the X Window System, `configure' can usually
+find the X include and library files automatically, but if it doesn't,
+you can use the `configure' options `--x-includes=DIR' and
+`--x-libraries=DIR' to specify their locations.
+
+Specifying the System Type
+==========================
+
+There may be some features `configure' cannot figure out automatically,
+but needs to determine by the type of machine the package will run on.
+Usually, assuming the package is built to be run on the _same_
+architectures, `configure' can figure that out, but if it prints a
+message saying it cannot guess the machine type, give it the
+`--build=TYPE' option.  TYPE can either be a short name for the system
+type, such as `sun4', or a canonical name which has the form:
+
+     CPU-COMPANY-SYSTEM
+
+where SYSTEM can have one of these forms:
+
+     OS KERNEL-OS
+
+   See the file `config.sub' for the possible values of each field.  If
+`config.sub' isn't included in this package, then this package doesn't
+need to know the machine type.
+
+   If you are _building_ compiler tools for cross-compiling, you should
+use the option `--target=TYPE' to select the type of system they will
+produce code for.
+
+   If you want to _use_ a cross compiler, that generates code for a
+platform different from the build platform, you should specify the
+"host" platform (i.e., that on which the generated programs will
+eventually be run) with `--host=TYPE'.
+
+Sharing Defaults
+================
+
+If you want to set default values for `configure' scripts to share, you
+can create a site shell script called `config.site' that gives default
+values for variables like `CC', `cache_file', and `prefix'.
+`configure' looks for `PREFIX/share/config.site' if it exists, then
+`PREFIX/etc/config.site' if it exists.  Or, you can set the
+`CONFIG_SITE' environment variable to the location of the site script.
+A warning: not all `configure' scripts look for a site script.
+
+Defining Variables
+==================
+
+Variables not defined in a site shell script can be set in the
+environment passed to `configure'.  However, some packages may run
+configure again during the build, and the customized values of these
+variables may be lost.  In order to avoid this problem, you should set
+them in the `configure' command line, using `VAR=value'.  For example:
+
+     ./configure CC=/usr/local2/bin/gcc
+
+causes the specified `gcc' to be used as the C compiler (unless it is
+overridden in the site shell script).
+
+Unfortunately, this technique does not work for `CONFIG_SHELL' due to
+an Autoconf bug.  Until the bug is fixed you can use this workaround:
+
+     CONFIG_SHELL=/bin/bash /bin/bash ./configure CONFIG_SHELL=/bin/bash
+
+`configure' Invocation
+======================
+
+`configure' recognizes the following options to control how it operates.
+
+`--help'
+`-h'
+     Print a summary of the options to `configure', and exit.
+
+`--version'
+`-V'
+     Print the version of Autoconf used to generate the `configure'
+     script, and exit.
+
+`--cache-file=FILE'
+     Enable the cache: use and save the results of the tests in FILE,
+     traditionally `config.cache'.  FILE defaults to `/dev/null' to
+     disable caching.
+
+`--config-cache'
+`-C'
+     Alias for `--cache-file=config.cache'.
+
+`--quiet'
+`--silent'
+`-q'
+     Do not print messages saying which checks are being made.  To
+     suppress all normal output, redirect it to `/dev/null' (any error
+     messages will still be shown).
+
+`--srcdir=DIR'
+     Look for the package's source code in directory DIR.  Usually
+     `configure' can determine that directory automatically.
+
+`configure' also accepts some other, not widely useful, options.  Run
+`configure --help' for more details.
+
diff --git a/pixman/Makefile.am b/pixman/Makefile.am
new file mode 100644
index 000000000..63b08c1fb
--- /dev/null
+++ b/pixman/Makefile.am
@@ -0,0 +1,132 @@
+SUBDIRS = pixman test
+
+pkgconfigdir=$(libdir)/pkgconfig
+pkgconfig_DATA=pixman-1.pc
+
+$(pkgconfig_DATA): pixman-1.pc.in
+
+snapshot:
+	distdir="$(distdir)-`date '+%Y%m%d'`"; \
+	test -d "$(srcdir)/.git" && distdir=$$distdir-`cd "$(srcdir)" && git rev-parse HEAD | cut -c 1-6`; \
+	$(MAKE) $(AM_MAKEFLAGS) distdir="$$distdir" dist
+
+GPGKEY=6FF7C1A8
+USERNAME=$$USER
+RELEASE_OR_SNAPSHOT = $$(if test "x$(CAIRO_VERSION_MINOR)" = "x$$(echo "$(CAIRO_VERSION_MINOR)/2*2" | bc)" ; then echo release; else echo snapshot; fi)
+RELEASE_CAIRO_HOST =	$(USERNAME)@cairographics.org
+RELEASE_CAIRO_DIR =	/srv/cairo.freedesktop.org/www/releases
+RELEASE_CAIRO_URL = 	http://cairographics.org/releases
+RELEASE_XORG_URL =	http://xorg.freedesktop.org/archive/individual/lib
+RELEASE_XORG_HOST =	$(USERNAME)@xorg.freedesktop.org
+RELEASE_XORG_DIR =	/srv/xorg.freedesktop.org/archive/individual/lib
+RELEASE_ANNOUNCE_LIST = cairo-announce@cairographics.org, xorg-announce@lists.freedesktop.org
+
+tar_gz = $(PACKAGE)-$(VERSION).tar.gz
+tar_bz2 = $(PACKAGE)-$(VERSION).tar.bz2
+
+sha1_tgz = $(tar_gz).sha1
+md5_tgz = $(tar_gz).md5
+
+sha1_tbz2 = $(tar_bz2).sha1
+md5_tbz2 = $(tar_bz2).md5
+
+gpg_file = $(sha1_tgz).asc
+
+$(sha1_tgz): $(tar_gz)
+	sha1sum $^ > $@
+
+$(md5_tgz): $(tar_gz)
+	md5sum $^ > $@
+
+$(sha1_tbz2): $(tar_bz2)
+	sha1sum $^ > $@
+
+$(md5_tbz2): $(tar_bz2)
+	md5sum $^ > $@
+
+$(gpg_file): $(sha1_tgz)
+	@echo "Please enter your GPG password to sign the checksum."
+	gpg --armor --sign $^ 
+
+HASHFILES = $(sha1_tgz) $(sha1_tbz2) $(md5_tgz) $(md5_tbz2)
+
+release-verify-newer:
+	@echo -n "Checking that no $(VERSION) release already exists at $(RELEASE_XORG_HOST)..."
+	@ssh $(RELEASE_XORG_HOST) test ! -e $(RELEASE_XORG_DIR)/$(tar_gz) \
+		|| (echo "Ouch." && echo "Found: $(RELEASE_XORG_HOST):$(RELEASE_XORG_DIR)/$(tar_gz)" \
+		&& echo "Refusing to try to generate a new release of the same name." \
+		&& false)
+	@ssh $(RELEASE_CAIRO_HOST) test ! -e $(RELEASE_CAIRO_DIR)/$(tar_gz) \
+		|| (echo "Ouch." && echo "Found: $(RELEASE_CAIRO_HOST):$(RELEASE_CAIRO_DIR)/$(tar_gz)" \
+		&& echo "Refusing to try to generate a new release of the same name." \
+		&& false)
+	@echo "Good."
+
+release-remove-old:
+	$(RM) $(tar_gz) $(tar_bz2) $(HASHFILES) $(gpg_file)
+
+ensure-prev:
+	@if [[ "$(PREV)" == "" ]]; then							\
+		echo ""							          &&	\
+		echo "You must set the PREV variable on the make command line to" &&	\
+		echo "the last version."				  	  &&	\
+		echo ""								  &&	\
+		echo "For example:"						  &&	\
+		echo "      make PREV=0.7.3"				  	  &&	\
+		echo ""								  &&	\
+		false;									\
+	fi
+
+release-check: ensure-prev release-verify-newer release-remove-old distcheck
+
+release-tag:
+	git tag -u $(GPGKEY) -m "$(PACKAGE) $(VERSION) release" $(PACKAGE)-$(VERSION)
+
+release-upload: release-check $(tar_gz) $(tar_bz2) $(sha1_tgz) $(sha1_tbz2) $(md5_tgz) $(gpg_file)
+	mkdir -p releases
+	scp $(tar_gz) $(sha1_tgz) $(gpg_file) $(RELEASE_CAIRO_HOST):$(RELEASE_CAIRO_DIR)
+	scp $(tar_gz) $(tar_bz2) $(RELEASE_XORG_HOST):$(RELEASE_XORG_DIR)
+	ssh $(RELEASE_CAIRO_HOST) "rm -f $(RELEASE_CAIRO_DIR)/LATEST-$(PACKAGE)-[0-9]* && ln -s $(tar_gz) $(RELEASE_CAIRO_DIR)/LATEST-$(PACKAGE)-$(VERSION)"
+
+release-publish-message: $(HASHFILES) ensure-prev
+	@echo "Please follow the instructions in RELEASING to push stuff out and"
+	@echo "send out the announcement mails.  Here is the excerpt you need:"
+	@echo ""
+	@echo "Lists:  $(RELEASE_ANNOUNCE_LIST)"
+	@echo "Subject: [ANNOUNCE] $(PACKAGE) release $(VERSION) now available"
+	@echo "============================== CUT HERE =============================="
+	@echo "A new $(PACKAGE) release $(VERSION) is now available"
+	@echo ""
+	@echo "tar.gz:"
+	@echo "	$(RELEASE_CAIRO_URL)/$(tar_gz)"
+	@echo "	$(RELEASE_XORG_URL)/$(tar_gz)"
+	@echo ""
+	@echo "tar.bz2:"
+	@echo "	$(RELEASE_XORG_URL)/$(tar_bz2)"
+	@echo ""
+	@echo "Hashes:"
+	@echo -n "	MD5:  "
+	@cat $(md5_tgz)
+	@echo -n "	MD5:  "
+	@cat $(md5_tbz2)
+	@echo -n "	SHA1: "
+	@cat $(sha1_tgz)
+	@echo -n "	SHA1: "
+	@cat $(sha1_tbz2)
+	@echo ""
+	@echo "GPG signature:"
+	@echo "	$(RELEASE_CAIRO_URL)/$(gpg_file)"
+	@echo "	(signed by `git config --get user.name` <`git config --get user.email`>)"
+	@echo ""
+	@echo "Git:"
+	@echo "	git://git.freedesktop.org/git/pixman"
+	@echo "	tag: $(PACKAGE)-$(VERSION)"
+	@echo ""
+	@echo "Log:"
+	@git log --no-merges "$(PACKAGE)-$(PREV)".."$(PACKAGE)-$(VERSION)" | git shortlog | awk '{ printf "\t"; print ; }' | cut -b1-80
+	@echo "============================== CUT HERE =============================="
+	@echo ""
+
+release-publish: release-upload release-tag release-publish-message
+
+.PHONY: release-upload release-publish release-publish-message release-tag
diff --git a/pixman/NEWS b/pixman/NEWS
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/pixman/NEWS
diff --git a/pixman/README b/pixman/README
new file mode 100644
index 000000000..60dff4561
--- /dev/null
+++ b/pixman/README
@@ -0,0 +1,22 @@
+pixman is a library that provides low-level pixel manipulation
+features such as image compositing and trapezoid rasterization.
+
+Please submit bugs & patches to the libpixman bugzilla:
+
+        https://bugs.freedesktop.org/enter_bug.cgi?product=pixman
+
+All questions regarding this software should be directed to the pixman
+mailing list:
+
+        http://lists.freedesktop.org/mailman/listinfo/pixman
+
+The master development code repository can be found at:
+
+	git://anongit.freedesktop.org/git/pixman
+
+	http://gitweb.freedesktop.org/?p=pixman;a=summary
+
+For more information on the git code manager, see:
+
+	http://wiki.x.org/wiki/GitPage
+
diff --git a/pixman/RELEASING b/pixman/RELEASING
new file mode 100644
index 000000000..7ddcef49f
--- /dev/null
+++ b/pixman/RELEASING
@@ -0,0 +1,57 @@
+Here are the steps to follow to create a new pixman release:
+
+1) Ensure that there are no uncommitted changes or unpushed commits,
+   and that you are up to date with the latest commits in the central
+   repository. Here are a couple of useful commands:
+
+	git diff			(no output)
+	
+	git status			(should report "nothing to commit")
+
+	git log master...origin		(no output; note: *3* dots)
+
+2) Increment pixman_(major|minor|micro) in configure.ac according to
+   the directions in that file. Use git commit to record this.
+
+3) Make sure that new version works, including
+
+	- make distcheck passes
+
+	- the X server still works with the new pixman version
+	  installed
+
+	- the cairo test suite hasn't gained any new failures compared
+	  to last pixman version.
+
+4) Use "git commit" to record any changes made in step 3.
+
+5) Generate and publish the tar files by running 
+
+	make PREV=<last version> GPGKEY=<your gpg key id> release-publish
+
+   If your freedesktop user name is different from your local one,
+   then also set the variable USER to your freedesktop user name.
+
+6) Run 
+
+	make release-publish-message
+
+   to generate a draft release announcement. Edit it as appropriate and
+   send it to 
+
+	cairo-announce@cairographics.org
+
+	pixman@lists.freedesktop.org
+
+	xorg-announce@lists.freedesktop.org
+
+7) Increment pixman_micro to the next larger (odd) number in
+   configure.ac. Commit this change, and push all commits created
+   during this process using
+
+	git push
+	git push --tags
+
+   You must use "--tags" here; otherwise the new tag will not
+   be pushed out.
+
diff --git a/pixman/TODO b/pixman/TODO
new file mode 100644
index 000000000..4434ec7cb
--- /dev/null
+++ b/pixman/TODO
@@ -0,0 +1,271 @@
+  - Testing
+    - Test implementations against each other
+    - Test both with and without the operator strength reduction.
+      They shold be identical.
+
+  - SSE 2 issues:
+
+      - Use MM_HINT_NTA instead of MM_HINT_T0
+
+      - Use of fbCompositeOver_x888x8x8888sse2()
+
+  - Update the RLEASING file
+
+  - Things to keep in mind if breaking ABI:
+
+      - There should be a guard #ifndef I_AM_EITHER_CAIRO_OR_THE_X_SERVER
+
+      - X server will require 16.16 essentially forever. Can we get
+        the required precision by simply adding offset_x/y to the
+        relevant rendering API?
+
+      - Get rid of workaround for X server bug.
+
+      - pixman_image_set_indexed() should copy its argument, and X
+        should be ported over to use a pixman_image as the
+        representation of a Picture, rather than creating one on each
+        operation.
+
+      - We should get rid of pixman_set_static_pointers()
+
+      - We should get rid of the various trapezoid helper functions().
+        (They only exist because they are theoretically available to
+        drivers).
+
+      - 16 bit regions should be deleted
+
+      - There should only be one trap rasterization API.
+
+      - The PIXMAN_g8/c8/etc formats should use the A channel
+        to indicate the actual depth. That way PIXMAN_x4c4 and PIXMAN_c8
+	won't collide.
+
+  - Maybe bite the bullet and make configure.ac generate a pixman-types.h
+    file that can be included from pixman.h to avoid the #ifdef magic
+    in pixman.h
+
+  - Make pixman_region_point_in() survive a NULL box, then fix up
+    pixman-compose.c
+
+      - Possibly look into inlining the fetch functions
+
+  - There is a bug with source clipping demonstrated by clip-test in the
+    test directory. If we interprete source clipping as given in
+    destination coordinates, which is probably the only sane choice,
+    then the result should have two red bars down the sides.
+    
+  - Test suite
+
+  - Add a general way of dealing with architecture specific
+    fast-paths.  The current idea is to have each operation that can
+    be optimized is called through a function pointer that is
+    initially set to an initialization function that is responsible for
+    setting the function pointer to the appropriate fast-path.
+
+  - Go through things marked FIXME
+
+  - Add calls to prepare and finish access where necessary.  grep for
+    ACCESS_MEM, and make sure they are correctly wrapped in prepare
+    and finish.
+
+  - restore READ/WRITE in the fbcompose combiners since they sometimes
+    store directly to destination drawables.
+
+  - It probably makes sense to move the more strange X region API
+    into pixman as well, but guarded with PIXMAN_XORG_COMPATIBILITY
+
+  - Reinstate the FbBits typedef? At the moment we don't
+    even have the FbBits type; we just use uint32_t everywhere.
+
+    Keith says in bug 2335:
+
+        The 64-bit code in fb (pixman) is probably broken; it hasn't been
+        used in quite some time as PCI (and AGP) is 32-bits wide, so
+        doing things 64-bits at a time is a net loss.  To quickly fix
+        this, I suggest just using 32-bit datatypes by setting
+        IC_SHIFT to 5 for all machines.
+
+  - Consider optimizing the 8/16 bit solid fills in pixman-util.c by
+    storing more than one value at a time.
+
+  - Add an image cache to prevent excessive malloc/free. Note that pixman
+    needs to be thread safe when used from cairo.
+
+  - Moving to 24.8 coordinates. This is tricky because X is still
+    defined as 16.16 and will be basically forever. It's possible we
+    could do this by adding extra offset_x/y parameters to the
+    trapezoid calls. The X server could then just call the API with
+    (0, 0). Cairo would have to make sure that the delta *within* a
+    batch of trapezoids does not exceed 16 bit.
+
+  - Consider adding actual backends. Brain dump:
+
+    A backend is something that knows how to
+
+      - Create images
+      - Composite three images
+      - Rasterize trapezoids
+      - Do solid fills and blits
+
+    These operations are provided by a vtable that the backend will
+    create when it is initialized. Initial backends:
+
+      - VMX
+      - SSE2
+      - MMX
+      - Plain Old C
+
+    When the SIMD backends are initialized, they will be passed a
+    pointer to the Plain Old C backend that they can use for fallback
+    purposes.
+
+    Images would gain a vtable as well that would contain things like
+
+      - Read scanline
+      - Write scanline
+
+    (Or even read_patch/write_patch as suggested by Keith a while
+    back).
+
+    This could simplify the compositing code considerably.
+
+  - Review the pixman_format_code_t enum to make sure it will support
+    future formats. Some formats we will probably need:
+
+    	   ARGB/ABGR with 16/32/64 bit integer/floating channels
+	   YUV2,
+	   YV12
+
+    Also we may need the ability to distinguish between PICT_c8 and
+    PICT_x4c4. (This could be done by interpreting the A channel as
+    the depth for TYPE_COLOR and TYPE_GRAY formats).
+
+    A possibility may be to reserve the two top bits and make them
+    encode "number of places to shift the channel widths given" Since
+    these bits are 00 at the moment everything will continue to work,
+    but these additional widths will be allowed:
+
+    	     All even widths between 18-32
+	     All multiples of four widths between 33 and 64
+	     All multiples of eight between 64 and 128
+
+    This means things like r21g22b21 won't work - is that worth
+    worrying about? I don't think so. And of course the bpp field
+    can't handle a depth of over 256, so > 64 bit channels arent'
+    really all that useful.
+
+    We could reserve one extra bit to indicate floating point, but
+    we may also just add 
+
+       	   PIXMAN_TYPE_ARGB_FLOAT
+	   PIXMAN_TYPE_BGRA_FLOAT
+	   PIXMAN_TYPE_A_FLOAT
+    
+    image types. With five bits we can support up to 32 different
+    format types, which should be enough for everybody, even if we
+    decide to support all the various video formats here:
+
+    	        http://www.fourcc.org/yuv.php
+
+    It may make sense to have a PIXMAN_TYPE_YUV, and then use the
+    channel bits to specify the exact subtype.
+
+    Another possibility is to add 
+
+      	  PIXMAN_TYPE_ARGB_W
+	  PIXMAN_TYPE_ARGB_WW
+    
+    where the channel widths would get 16 and 32 added to them,
+    respectively.
+
+    What about color spaces such a linear vs. srGB etc.?
+
+
+done:
+
+- Use pixmanFillsse2 and pixmanBltsse2
+
+- Be consistent about calling sse2 sse2
+
+- Rename "SSE" to "MMX_EXTENSIONS". (Deleted mmx extensions).
+
+- Commented-out uses of fbCompositeCopyAreasse2()
+
+- Consider whether calling regions region16 is really such a great
+  idea. Vlad wants 32 bit regions for Cairo. This will break X server
+  ABI, but should otherwise be mostly harmless, though a
+  pixman_region_get_boxes16() may be useful.
+
+- Altivec signal issue (Company has fix, there is also a patch by
+  dwmw2 in rawhide).
+
+- Behdad's MMX issue - see list
+
+- SSE2 issues:
+    - Crashes in Mozilla because of unaligned stack. Possible fixes
+        - Make use of gcc 4.2 feature to align the stack
+        - Write some sort of trampoline that aligns the stack
+          before calling SSE functions.
+
+- Get rid of the switch-of-doom; replace it with a big table
+  describing the various fast paths.
+
+- Make source clipping optional.
+    - done: source clipping happens through an indirection.
+        still needs to make the indirection settable. (And call it
+        from X)
+
+- Run cairo test suite; fix bugs
+	- one bug in source-scale-clip
+
+ - Remove the warning suppression in the ACCESS_MEM macro and fix the
+    warnings that are real
+	- irrelevant now.
+
+- make the wrapper functions global instead of image specific
+	- this won't work since pixman is linked to both fb and wfb
+
+- Add non-mmx solid fill
+
+- Make sure the endian-ness macros are defined correctly.
+
+- The rectangles in a region probably shouldn't be returned const as
+  the X server will be changing them.
+
+- Right now we _always_ have a clip region, which is empty by default.
+  Why does this work at all? It probably doesn't. The server
+  distinguishes two cases, one where nothing is clipped (CT_NONE), and
+  one where there is a clip region (CT_REGION).
+
+- Default clip region should be the full image
+
+  - Test if pseudo color still works. It does, but it also shows that
+    copying a pixman_indexed_t on every composite operation is not
+    going to fly. So, for now set_indexed() does not copy the 
+    indexed table. 
+
+    Also just the malloc() to allocate a pixman image shows up pretty
+    high.
+
+    Options include
+
+      - Make all the setters not copy their arguments
+
+      - Possibly combined with going back to the stack allocated 
+        approach that we already use for regions.
+
+      - Keep a cached pixman_image_t around for every picture. It would
+        have to be kept uptodate every time something changes about the
+        picture.
+
+      - Break the X server ABI and simply have the relevant parameter
+        stored in the pixman image. This would have the additional benefits
+        that:
+
+          - We can get rid of the annoying repeat field which is duplicated
+            elsewhere.
+
+          - We can use pixman_color_t and pixman_gradient_stop_t
+            etc. instead of the types that are defined in
+            renderproto.h
+
diff --git a/pixman/autogen.sh b/pixman/autogen.sh
new file mode 100755
index 000000000..354f254e4
--- /dev/null
+++ b/pixman/autogen.sh
@@ -0,0 +1,12 @@
+#! /bin/sh
+
+srcdir=`dirname $0`
+test -z "$srcdir" && srcdir=.
+
+ORIGDIR=`pwd`
+cd $srcdir
+
+autoreconf -v --install || exit 1
+cd $ORIGDIR || exit $?
+
+$srcdir/configure "$@"
diff --git a/pixman/configure.ac b/pixman/configure.ac
new file mode 100644
index 000000000..c9269f4ec
--- /dev/null
+++ b/pixman/configure.ac
@@ -0,0 +1,608 @@
+dnl  Copyright 2005 Red Hat, Inc.
+dnl 
+dnl  Permission to use, copy, modify, distribute, and sell this software and its
+dnl  documentation for any purpose is hereby granted without fee, provided that
+dnl  the above copyright notice appear in all copies and that both that
+dnl  copyright notice and this permission notice appear in supporting
+dnl  documentation, and that the name of Red Hat not be used in
+dnl  advertising or publicity pertaining to distribution of the software without
+dnl  specific, written prior permission.  Red Hat makes no
+dnl  representations about the suitability of this software for any purpose.  It
+dnl  is provided "as is" without express or implied warranty.
+dnl 
+dnl  RED HAT DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
+dnl  INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO
+dnl  EVENT SHALL RED HAT BE LIABLE FOR ANY SPECIAL, INDIRECT OR
+dnl  CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE,
+dnl  DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
+dnl  TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+dnl  PERFORMANCE OF THIS SOFTWARE.
+dnl
+dnl Process this file with autoconf to create configure.
+
+AC_PREREQ([2.57])
+
+#   Pixman versioning scheme
+#
+#   - The version in git has an odd MICRO version number
+#
+#   - Released versions both development and stable have an even MICRO 
+#     version number
+#
+#   - Released development versions have an odd MINOR number
+#
+#   - Released stable versions have an even MINOR number
+#
+#   - Versions that break ABI must have a new MAJOR number
+#
+#   - If you break the ABI, then at least this must be done:
+#
+#        - increment MAJOR
+#
+#        - In the first development release where you break ABI, find
+#          all instances of "pixman-n" and change them to pixman-(n+1)
+#
+#          This needs to be done at least in 
+#                    configure.ac
+#                    all Makefile.am's
+#                    pixman-n.pc.in
+#
+#      This ensures that binary incompatible versions can be installed
+#      in parallel.  See http://www106.pair.com/rhp/parallel.html for
+#      more information
+#
+
+m4_define([pixman_major], 0)
+m4_define([pixman_minor], 18)
+m4_define([pixman_micro], 4)
+
+m4_define([pixman_version],[pixman_major.pixman_minor.pixman_micro])
+
+AC_INIT(pixman, pixman_version, "pixman@lists.freedesktop.org", pixman)
+AM_INIT_AUTOMAKE([foreign dist-bzip2])
+
+# Suppress verbose compile lines
+m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])])
+
+AM_CONFIG_HEADER(config.h)
+
+AC_CANONICAL_HOST
+
+test_CFLAGS=${CFLAGS+set} # We may override autoconf default CFLAGS.
+
+AC_PROG_CC
+AM_PROG_AS
+AC_PROG_LIBTOOL
+AC_CHECK_FUNCS([getisax])
+AC_C_BIGENDIAN
+AC_C_INLINE
+
+AC_CHECK_SIZEOF(long)
+
+# Checks for Sun Studio compilers
+AC_CHECK_DECL([__SUNPRO_C], [SUNCC="yes"], [SUNCC="no"])
+AC_CHECK_DECL([__amd64], [AMD64_ABI="yes"], [AMD64_ABI="no"])
+
+# Default CFLAGS to -O -g rather than just the -g from AC_PROG_CC
+# if we're using Sun Studio and neither the user nor a config.site
+# has set CFLAGS.
+if test $SUNCC = yes &&			\
+   test "$test_CFLAGS" == "" &&		\
+   test "$CFLAGS" = "-g"
+then
+  CFLAGS="-O -g"
+fi
+
+# 
+# We ignore pixman_major in the version here because the major version should
+# always be encoded in the actual library name. Ie., the soname is:
+#
+#      pixman-$(pixman_major).0.minor.micro
+#
+m4_define([lt_current], [pixman_minor])
+m4_define([lt_revision], [pixman_micro])
+m4_define([lt_age], [pixman_minor])
+
+LT_VERSION_INFO="lt_current:lt_revision:lt_age"
+
+PIXMAN_VERSION_MAJOR=pixman_major()
+AC_SUBST(PIXMAN_VERSION_MAJOR)
+PIXMAN_VERSION_MINOR=pixman_minor()
+AC_SUBST(PIXMAN_VERSION_MINOR)
+PIXMAN_VERSION_MICRO=pixman_micro()
+AC_SUBST(PIXMAN_VERSION_MICRO)
+
+AC_SUBST(LT_VERSION_INFO)
+
+# Check for dependencies
+#PKG_CHECK_MODULES(DEP, x11)
+
+changequote(,)dnl
+if test "x$GCC" = "xyes"; then
+
+  case " $CFLAGS " in
+  *[\ \	]-Wall[\ \	]*) ;;
+  *) CFLAGS="$CFLAGS -Wall" ;;
+  esac 
+
+  case " $CFLAGS " in
+  *[\ \	]-fno-strict-aliasing[\ \	]*) ;;
+  *) CFLAGS="$CFLAGS -fno-strict-aliasing" ;;
+  esac
+
+fi changequote([,])dnl
+
+AC_PATH_PROG(PERL, perl, no)
+if test "x$PERL" = xno; then
+    AC_MSG_ERROR([Perl is required to build pixman.])
+fi
+AC_SUBST(PERL)
+
+dnl =========================================================================
+dnl -fvisibility stuff
+
+have_gcc4=no
+AC_MSG_CHECKING(for -fvisibility)
+AC_COMPILE_IFELSE([
+#if defined(__GNUC__) && (__GNUC__ >= 4)
+#else
+error Need GCC 4.0 for visibility
+#endif
+int main () { return 0; } 
+], have_gcc4=yes)
+
+if test "x$have_gcc4" = "xyes"; then
+   CFLAGS="$CFLAGS -fvisibility=hidden"
+fi
+AC_MSG_RESULT($have_gcc4)
+
+have_sunstudio8=no
+AC_MSG_CHECKING([for -xldscope (Sun compilers)])
+AC_COMPILE_IFELSE([
+#if defined(__SUNPRO_C) && (__SUNPRO_C >= 0x550)
+#else
+error Need Sun Studio 8 for visibility
+#endif
+int main () { return 0; } 
+], have_sunstudio8=yes)
+
+if test "x$have_sunstudio8" = "xyes"; then
+   CFLAGS="$CFLAGS -xldscope=hidden"
+fi
+AC_MSG_RESULT($have_sunstudio8)
+
+dnl ===========================================================================
+dnl Check for MMX
+
+if test "x$MMX_CFLAGS" = "x" ; then
+   if test "x$SUNCC" = "xyes"; then
+      # Sun Studio doesn't have an -xarch=mmx flag, so we have to use sse
+      # but if we're building 64-bit, mmx & sse support is on by default and
+      # -xarch=sse throws an error instead
+      if test "$AMD64_ABI" = "no" ; then
+         MMX_CFLAGS="-xarch=sse"
+      fi
+   else
+      MMX_CFLAGS="-mmmx -Winline"
+   fi
+fi
+
+have_mmx_intrinsics=no
+AC_MSG_CHECKING(whether to use MMX intrinsics)
+xserver_save_CFLAGS=$CFLAGS
+CFLAGS="$MMX_CFLAGS $CFLAGS"
+AC_COMPILE_IFELSE([
+#if defined(__GNUC__) && (__GNUC__ < 3 || (__GNUC__ == 3 && __GNUC_MINOR__ < 4))
+error "Need GCC >= 3.4 for MMX intrinsics"
+#endif
+#include <mmintrin.h>
+int main () {
+    __m64 v = _mm_cvtsi32_si64 (1);
+    return _mm_cvtsi64_si32 (v);
+}], have_mmx_intrinsics=yes)
+CFLAGS=$xserver_save_CFLAGS
+
+AC_ARG_ENABLE(mmx,
+   [AC_HELP_STRING([--disable-mmx],
+                   [disable MMX fast paths])],
+   [enable_mmx=$enableval], [enable_mmx=auto])
+
+if test $enable_mmx = no ; then
+   have_mmx_intrinsics=disabled
+fi
+
+if test $have_mmx_intrinsics = yes ; then
+   AC_DEFINE(USE_MMX, 1, [use MMX compiler intrinsics])
+else
+   MMX_CFLAGS=
+fi
+
+AC_MSG_RESULT($have_mmx_intrinsics)
+if test $enable_mmx = yes && test $have_mmx_intrinsics = no ; then
+   AC_MSG_ERROR([MMX intrinsics not detected])
+fi
+
+AM_CONDITIONAL(USE_MMX, test $have_mmx_intrinsics = yes)
+
+dnl ===========================================================================
+dnl Check for SSE2
+
+if test "x$SSE2_CFLAGS" = "x" ; then
+   if test "x$SUNCC" = "xyes"; then
+      # SSE2 is enabled by default in the Sun Studio 64-bit environment
+      if test "$AMD64_ABI" = "no" ; then
+         SSE2_CFLAGS="-xarch=sse2"
+      fi
+   else
+      SSE2_CFLAGS="-mmmx -msse2 -Winline"
+   fi
+fi
+
+have_sse2_intrinsics=no
+AC_MSG_CHECKING(whether to use SSE2 intrinsics)
+xserver_save_CFLAGS=$CFLAGS
+CFLAGS="$SSE2_CFLAGS $CFLAGS"
+
+AC_COMPILE_IFELSE([
+#if defined(__GNUC__) && (__GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 2))
+#   if !defined(__amd64__) && !defined(__x86_64__)
+#      error "Need GCC >= 4.2 for SSE2 intrinsics on x86"
+#   endif
+#endif
+#include <mmintrin.h>
+#include <xmmintrin.h>
+#include <emmintrin.h>
+int main () {
+    __m128i a = _mm_set1_epi32 (0), b = _mm_set1_epi32 (0), c;
+	c = _mm_xor_si128 (a, b);
+    return 0;
+}], have_sse2_intrinsics=yes)
+CFLAGS=$xserver_save_CFLAGS
+
+AC_ARG_ENABLE(sse2,
+   [AC_HELP_STRING([--disable-sse2],
+                   [disable SSE2 fast paths])],
+   [enable_sse2=$enableval], [enable_sse2=auto])
+
+if test $enable_sse2 = no ; then
+   have_sse2_intrinsics=disabled
+fi
+
+if test $have_sse2_intrinsics = yes ; then
+   AC_DEFINE(USE_SSE2, 1, [use SSE2 compiler intrinsics])
+fi
+
+AC_MSG_RESULT($have_sse2_intrinsics)
+if test $enable_sse2 = yes && test $have_sse2_intrinsics = no ; then
+   AC_MSG_ERROR([SSE2 intrinsics not detected])
+fi
+
+AM_CONDITIONAL(USE_SSE2, test $have_sse2_intrinsics = yes)
+
+dnl ===========================================================================
+dnl Other special flags needed when building code using MMX or SSE instructions
+case $host_os in
+   solaris*)
+      # When building 32-bit binaries, apply a mapfile to ensure that the
+      # binaries aren't flagged as only able to run on MMX+SSE capable CPUs
+      # since they check at runtime before using those instructions.
+      # Not all linkers grok the mapfile format so we check for that first.
+      if test "$AMD64_ABI" = "no" ; then
+	 use_hwcap_mapfile=no
+	 AC_MSG_CHECKING(whether to use a hardware capability map file)
+	 hwcap_save_LDFLAGS="$LDFLAGS"
+	 HWCAP_LDFLAGS='-Wl,-M,$(srcdir)/solaris-hwcap.mapfile'
+	 LDFLAGS="$LDFLAGS -Wl,-M,pixman/solaris-hwcap.mapfile"
+	 AC_LINK_IFELSE([int main() { return 0; }],
+			use_hwcap_mapfile=yes,
+			HWCAP_LDFLAGS="")
+	 LDFLAGS="$hwcap_save_LDFLAGS"
+	 AC_MSG_RESULT($use_hwcap_mapfile)
+      fi
+      if test "x$MMX_LDFLAGS" = "x" ; then
+         MMX_LDFLAGS="$HWCAP_LDFLAGS"
+      fi
+      if test "x$SSE2_LDFLAGS" = "x" ; then
+	 SSE2_LDFLAGS="$HWCAP_LDFLAGS"
+      fi
+      ;;
+esac
+
+AC_SUBST(MMX_CFLAGS)
+AC_SUBST(MMX_LDFLAGS)
+AC_SUBST(SSE2_CFLAGS)
+AC_SUBST(SSE2_LDFLAGS)
+
+dnl ===========================================================================
+dnl Check for VMX/Altivec
+if test -n "`$CC -v 2>&1 | grep version | grep Apple`"; then
+    VMX_CFLAGS="-faltivec"
+else
+    VMX_CFLAGS="-maltivec -mabi=altivec"
+fi
+
+have_vmx_intrinsics=no
+AC_MSG_CHECKING(whether to use VMX/Altivec intrinsics)
+xserver_save_CFLAGS=$CFLAGS
+CFLAGS="$VMX_CFLAGS $CFLAGS"
+AC_COMPILE_IFELSE([
+#if defined(__GNUC__) && (__GNUC__ < 3 || (__GNUC__ == 3 && __GNUC_MINOR__ < 4))
+error "Need GCC >= 3.4 for sane altivec support"
+#endif
+#include <altivec.h>
+int main () {
+    vector unsigned int v = vec_splat_u32 (1);
+    v = vec_sub (v, v);
+    return 0;
+}], have_vmx_intrinsics=yes)
+CFLAGS=$xserver_save_CFLAGS
+
+AC_ARG_ENABLE(vmx,
+   [AC_HELP_STRING([--disable-vmx],
+                   [disable VMX fast paths])],
+   [enable_vmx=$enableval], [enable_vmx=auto])
+
+if test $enable_vmx = no ; then
+   have_vmx_intrinsics=disabled
+fi
+
+if test $have_vmx_intrinsics = yes ; then
+   AC_DEFINE(USE_VMX, 1, [use VMX compiler intrinsics])
+else
+   VMX_CFLAGS=
+fi
+
+AC_MSG_RESULT($have_vmx_intrinsics)
+if test $enable_vmx = yes && test $have_vmx_intrinsics = no ; then
+   AC_MSG_ERROR([VMX intrinsics not detected])
+fi
+
+AC_SUBST(VMX_CFLAGS)
+
+AM_CONDITIONAL(USE_VMX, test $have_vmx_intrinsics = yes)
+
+dnl ==========================================================================
+dnl Check if assembler is gas compatible and supports ARM SIMD instructions
+have_arm_simd=no
+AC_MSG_CHECKING(whether to use ARM SIMD assembler)
+xserver_save_CFLAGS=$CFLAGS
+CFLAGS="-x assembler-with-cpp $CFLAGS"
+AC_COMPILE_IFELSE([[
+.text
+.arch armv6
+.object_arch armv4
+.arm
+.altmacro
+#ifndef __ARM_EABI__
+#error EABI is required (to be sure that calling conventions are compatible)
+#endif
+pld [r0]
+uqadd8 r0, r0, r0]], have_arm_simd=yes)
+CFLAGS=$xserver_save_CFLAGS
+
+AC_ARG_ENABLE(arm-simd,
+   [AC_HELP_STRING([--disable-arm-simd],
+                   [disable ARM SIMD fast paths])],
+   [enable_arm_simd=$enableval], [enable_arm_simd=auto])
+
+if test $enable_arm_simd = no ; then
+   have_arm_simd=disabled
+fi
+
+if test $have_arm_simd = yes ; then
+   AC_DEFINE(USE_ARM_SIMD, 1, [use ARM SIMD assembly optimizations])
+fi
+
+AM_CONDITIONAL(USE_ARM_SIMD, test $have_arm_simd = yes)
+
+AC_MSG_RESULT($have_arm_simd)
+if test $enable_arm_simd = yes && test $have_arm_simd = no ; then
+   AC_MSG_ERROR([ARM SIMD intrinsics not detected])
+fi
+
+dnl ==========================================================================
+dnl Check if assembler is gas compatible and supports NEON instructions
+have_arm_neon=no
+AC_MSG_CHECKING(whether to use ARM NEON assembler)
+xserver_save_CFLAGS=$CFLAGS
+CFLAGS="-x assembler-with-cpp $CFLAGS"
+AC_COMPILE_IFELSE([[
+.text
+.fpu neon
+.arch armv7a
+.object_arch armv4
+.eabi_attribute 10, 0
+.arm
+.altmacro
+#ifndef __ARM_EABI__
+#error EABI is required (to be sure that calling conventions are compatible)
+#endif
+pld [r0]
+vmovn.u16 d0, q0]], have_arm_neon=yes)
+CFLAGS=$xserver_save_CFLAGS
+
+AC_ARG_ENABLE(arm-neon,
+   [AC_HELP_STRING([--disable-arm-neon],
+                   [disable ARM NEON fast paths])],
+   [enable_arm_neon=$enableval], [enable_arm_neon=auto])
+
+if test $enable_arm_neon = no ; then
+   have_arm_neon=disabled
+fi
+
+if test $have_arm_neon = yes ; then
+   AC_DEFINE(USE_ARM_NEON, 1, [use ARM NEON assembly optimizations])
+fi
+
+AM_CONDITIONAL(USE_ARM_NEON, test $have_arm_neon = yes)
+
+AC_MSG_RESULT($have_arm_neon)
+if test $enable_arm_neon = yes && test $have_arm_neon = no ; then
+   AC_MSG_ERROR([ARM NEON intrinsics not detected])
+fi
+
+dnl =========================================================================================
+dnl Check for GNU-style inline assembly support
+
+have_gcc_inline_asm=no
+AC_MSG_CHECKING(whether to use GNU-style inline assembler)
+AC_COMPILE_IFELSE([
+int main () {
+    /* Most modern architectures have a NOP instruction, so this is a fairly generic test. */
+	asm volatile ( "\tnop\n" : : : "cc", "memory" );
+    return 0;
+}], have_gcc_inline_asm=yes)
+
+AC_ARG_ENABLE(gcc-inline-asm,
+   [AC_HELP_STRING([--disable-gcc-inline-asm],
+                   [disable GNU-style inline assembler])],
+   [enable_gcc_inline_asm=$enableval], [enable_gcc_inline_asm=auto])
+
+if test $enable_gcc_inline_asm = no ; then
+   have_gcc_inline_asm=disabled
+fi
+
+if test $have_gcc_inline_asm = yes ; then
+   AC_DEFINE(USE_GCC_INLINE_ASM, 1, [use GNU-style inline assembler])
+fi
+
+AC_MSG_RESULT($have_gcc_inline_asm)
+if test $enable_gcc_inline_asm = yes && test $have_gcc_inline_asm = no ; then
+   AC_MSG_ERROR([GNU-style inline assembler not detected])
+fi
+
+AM_CONDITIONAL(USE_GCC_INLINE_ASM, test $have_gcc_inline_asm = yes)
+
+dnl ==============================================
+dnl Timers
+
+AC_ARG_ENABLE(timers,
+   [AC_HELP_STRING([--enable-timers],
+		   [enable TIMER_BEGIN and TIMER_END macros [default=no]])],
+   [enable_timers=$enableval], [enable_timers=no])
+
+if test $enable_timers = yes ; then 
+   AC_DEFINE(PIXMAN_TIMERS, 1, [enable TIMER_BEGIN/TIMER_END macros])
+fi
+AC_SUBST(PIXMAN_TIMERS)
+
+dnl ===================================
+dnl GTK+
+
+AC_ARG_ENABLE(gtk,
+   [AC_HELP_STRING([--enable-gtk],
+                   [enable tests using GTK+ [default=auto]])],
+   [enable_gtk=$enableval], [enable_gtk=auto])
+
+PKG_PROG_PKG_CONFIG
+if test $enable_gtk = auto ; then
+   PKG_CHECK_EXISTS([gtk+-2.0], [enable_gtk=yes], [enable_gtk=no])
+fi
+if test $enable_gtk = yes ; then
+   PKG_CHECK_MODULES(GTK, [gtk+-2.0])
+fi
+
+AM_CONDITIONAL(HAVE_GTK, [test "x$enable_gtk" = xyes])
+
+AC_SUBST(GTK_CFLAGS)
+AC_SUBST(GTK_LIBS)
+AC_SUBST(DEP_CFLAGS)
+AC_SUBST(DEP_LIBS)
+
+dnl =====================================
+dnl posix_memalign 
+
+AC_CHECK_FUNC(posix_memalign, have_posix_memalign=yes, have_posix_memalign=no)
+if test x$have_posix_memalign = xyes; then
+   AC_DEFINE(HAVE_POSIX_MEMALIGN, 1, [Whether we have posix_memalign()])
+fi
+
+dnl =====================================
+dnl Thread local storage
+
+support_for__thread=no
+
+AC_MSG_CHECKING(for __thread)
+AC_COMPILE_IFELSE([
+#ifdef __MINGW32__
+#error MinGW has broken __thread support
+#endif
+__thread int x ;
+int main () { return 0; }
+], support_for__thread=yes)
+
+if test $support_for__thread = yes; then 
+   AC_DEFINE([TOOLCHAIN_SUPPORTS__THREAD],[],[Whether the tool chain supports __thread])
+fi
+
+AC_MSG_RESULT($support_for__thread)
+
+dnl posix tls
+
+if test $support_for__thread = no; then
+
+support_for_pthread_setspecific=no
+   
+AC_MSG_CHECKING(for pthread_setspecific)
+
+save_LDFLAGS=$LDFLAGS
+
+LDFLAGS="-pthread"
+
+AC_LINK_IFELSE([
+#include <pthread.h>
+
+#include <stdlib.h>
+#include <pthread.h>
+
+static pthread_once_t once_control = PTHREAD_ONCE_INIT;
+static pthread_key_t key;
+
+static void
+make_key (void)
+{
+    pthread_key_create (&key, NULL);
+}
+
+int
+main ()
+{
+    void *value = NULL;
+    
+    if (pthread_once (&once_control, make_key) != 0)
+    {
+	value = NULL;
+    }
+    else
+    {
+	value = pthread_getspecific (key);
+	if (!value)
+	{
+	    value = malloc (100);
+	    pthread_setspecific (key, value);
+	}
+    }
+}
+], support_for_pthread_setspecific=yes);
+
+LDFLAGS=$save_LDFLAGS
+
+if test $support_for_pthread_setspecific = yes; then
+   PTHREAD_LDFLAGS="-pthread"
+   AC_DEFINE([HAVE_PTHREAD_SETSPECIFIC], [], [Whether pthread_setspecific() is supported])
+fi
+
+AC_MSG_RESULT($support_for_pthread_setspecific);
+
+fi
+
+AC_SUBST(TOOLCHAIN_SUPPORTS__THREAD)
+AC_SUBST(HAVE_PTHREAD_SETSPECIFIC)
+AC_SUBST(PTHREAD_LDFLAGS)
+
+AC_OUTPUT([pixman-1.pc
+           pixman-1-uninstalled.pc
+           Makefile
+	   pixman/Makefile
+	   pixman/pixman-version.h
+	   test/Makefile])
diff --git a/pixman/pixman-1-uninstalled.pc.in b/pixman/pixman-1-uninstalled.pc.in
new file mode 100644
index 000000000..e0347d010
--- /dev/null
+++ b/pixman/pixman-1-uninstalled.pc.in
@@ -0,0 +1,5 @@
+Name: Pixman
+Description: The pixman library (version 1)
+Version: @PACKAGE_VERSION@
+Cflags: -I${pc_top_builddir}/${pcfiledir}/pixman
+Libs: ${pc_top_builddir}/${pcfiledir}/pixman/libpixman-1.la
diff --git a/pixman/pixman-1.pc.in b/pixman/pixman-1.pc.in
new file mode 100644
index 000000000..936d95db0
--- /dev/null
+++ b/pixman/pixman-1.pc.in
@@ -0,0 +1,11 @@
+prefix=@prefix@
+exec_prefix=@exec_prefix@
+libdir=@libdir@
+includedir=@includedir@
+
+Name: Pixman
+Description: The pixman library (version 1)
+Version: @PACKAGE_VERSION@
+Cflags: -I${includedir}/pixman-1 @DEP_CFLAGS@
+Libs: -L${libdir} -lpixman-1 @DEP_LIBS@
+
diff --git a/pixman/pixman/Makefile.am b/pixman/pixman/Makefile.am
new file mode 100644
index 000000000..66ad7f005
--- /dev/null
+++ b/pixman/pixman/Makefile.am
@@ -0,0 +1,126 @@
+lib_LTLIBRARIES = libpixman-1.la
+libpixman_1_la_LDFLAGS = -version-info $(LT_VERSION_INFO) -no-undefined @PTHREAD_LDFLAGS@ 
+libpixman_1_la_LIBADD = @DEP_LIBS@ -lm
+libpixman_1_la_SOURCES =			\
+	pixman.h				\
+	pixman-accessor.h			\
+	pixman-access.c				\
+	pixman-access-accessors.c		\
+	pixman-cpu.c				\
+	pixman-gradient-walker.c		\
+	pixman-region16.c			\
+	pixman-region32.c			\
+	pixman-compiler.h			\
+	pixman-private.h			\
+	pixman-image.c				\
+	pixman-implementation.c			\
+	pixman-combine32.c			\
+	pixman-combine32.h			\
+	pixman-combine64.c			\
+	pixman-combine64.h			\
+	pixman-general.c			\
+	pixman.c				\
+	pixman-fast-path.c			\
+	pixman-solid-fill.c			\
+	pixman-conical-gradient.c		\
+	pixman-linear-gradient.c		\
+	pixman-radial-gradient.c		\
+	pixman-bits-image.c			\
+	pixman-utils.c				\
+	pixman-edge.c				\
+	pixman-edge-accessors.c			\
+	pixman-edge-imp.h			\
+	pixman-trap.c				\
+	pixman-timer.c				\
+	pixman-matrix.c
+
+libpixmanincludedir = $(includedir)/pixman-1/
+libpixmaninclude_HEADERS = pixman.h pixman-version.h
+noinst_LTLIBRARIES = 
+
+BUILT_SOURCES = pixman-combine32.h pixman-combine32.c pixman-combine64.h pixman-combine64.c
+
+pixman-combine32.c : pixman-combine.c.template pixman-combine32.h make-combine.pl
+	$(PERL) $(srcdir)/make-combine.pl 8 < $(srcdir)/pixman-combine.c.template > $@ || ($(RM) $@; exit 1)
+pixman-combine32.h : pixman-combine.h.template make-combine.pl
+	$(PERL) $(srcdir)/make-combine.pl 8 < $(srcdir)/pixman-combine.h.template > $@ || ($(RM) $@; exit 1)
+
+pixman-combine64.c : pixman-combine.c.template pixman-combine64.h make-combine.pl
+	$(PERL) $(srcdir)/make-combine.pl 16 < $(srcdir)/pixman-combine.c.template > $@ || ($(RM) $@; exit 1)
+pixman-combine64.h : pixman-combine.h.template make-combine.pl
+	$(PERL) $(srcdir)/make-combine.pl 16 < $(srcdir)/pixman-combine.h.template > $@ || ($(RM) $@; exit 1)
+
+EXTRA_DIST = Makefile.win32 pixman-combine.c.template make-combine.pl pixman-region.c \
+	pixman-combine.h.template solaris-hwcap.mapfile pixman-x64-mmx-emulation.h
+CLEANFILES = pixman-combine32.c pixman-combine64.c pixman-combine32.h pixman-combine64.h
+
+# mmx code
+if USE_MMX
+noinst_LTLIBRARIES += libpixman-mmx.la
+libpixman_mmx_la_SOURCES = \
+	pixman-mmx.c
+libpixman_mmx_la_CFLAGS = $(DEP_CFLAGS) $(MMX_CFLAGS)
+libpixman_mmx_la_LIBADD = $(DEP_LIBS)
+libpixman_1_la_LDFLAGS += $(MMX_LDFLAGS)
+libpixman_1_la_LIBADD += libpixman-mmx.la
+
+ASM_CFLAGS_mmx=$(MMX_CFLAGS)
+endif
+
+# vmx code
+if USE_VMX
+noinst_LTLIBRARIES += libpixman-vmx.la
+libpixman_vmx_la_SOURCES = \
+	pixman-vmx.c \
+	pixman-combine32.h
+libpixman_vmx_la_CFLAGS = $(DEP_CFLAGS) $(VMX_CFLAGS)
+libpixman_vmx_la_LIBADD = $(DEP_LIBS)
+libpixman_1_la_LIBADD += libpixman-vmx.la
+
+ASM_CFLAGS_vmx=$(VMX_CFLAGS)
+endif
+
+# sse2 code
+if USE_SSE2
+noinst_LTLIBRARIES += libpixman-sse2.la
+libpixman_sse2_la_SOURCES = \
+	pixman-sse2.c
+libpixman_sse2_la_CFLAGS = $(DEP_CFLAGS) $(SSE2_CFLAGS)
+libpixman_sse2_la_LIBADD = $(DEP_LIBS)
+libpixman_1_la_LDFLAGS += $(SSE2_LDFLAGS)
+libpixman_1_la_LIBADD += libpixman-sse2.la
+
+ASM_CFLAGS_sse2=$(SSE2_CFLAGS)
+endif
+
+# arm simd code
+if USE_ARM_SIMD
+noinst_LTLIBRARIES += libpixman-arm-simd.la
+libpixman_arm_simd_la_SOURCES = \
+	pixman-arm-simd.c	\
+	pixman-arm-common.h	\
+	pixman-arm-simd-asm.S
+libpixman_arm_simd_la_CFLAGS = $(DEP_CFLAGS)
+libpixman_arm_simd_la_LIBADD = $(DEP_LIBS)
+libpixman_1_la_LIBADD += libpixman-arm-simd.la
+
+ASM_CFLAGS_arm_simd=
+endif
+
+# arm neon code
+if USE_ARM_NEON
+noinst_LTLIBRARIES += libpixman-arm-neon.la
+libpixman_arm_neon_la_SOURCES = \
+        pixman-arm-neon.c	\
+        pixman-arm-common.h	\
+        pixman-arm-neon-asm.S	\
+        pixman-arm-neon-asm.h
+libpixman_arm_neon_la_CFLAGS = $(DEP_CFLAGS)
+libpixman_arm_neon_la_LIBADD = $(DEP_LIBS)
+libpixman_1_la_LIBADD += libpixman-arm-neon.la
+
+ASM_CFLAGS_arm_neon=
+endif
+
+.c.s : $(libpixmaninclude_HEADERS) $(BUILT_SOURCES)
+	$(CC) $(CFLAGS) $(ASM_CFLAGS_$(@:pixman-%.s=%)) $(ASM_CFLAGS_$(@:pixman-arm-%.s=arm_%)) -DHAVE_CONFIG_H -I$(srcdir) -I$(builddir) -I$(top_builddir) -S -o $@ $<
diff --git a/pixman/pixman/Makefile.win32 b/pixman/pixman/Makefile.win32
new file mode 100644
index 000000000..388bee23a
--- /dev/null
+++ b/pixman/pixman/Makefile.win32
@@ -0,0 +1,143 @@
+LIBRARY     = pixman-1
+
+CC   = cl
+LINK = link
+
+CFG_VAR = $(CFG)
+ifeq ($(CFG_VAR),)
+CFG_VAR=release
+endif
+
+MMX_VAR = $(MMX)
+ifeq ($(MMX_VAR),)
+MMX_VAR=on
+endif
+
+SSE2_VAR = $(SSE2)
+ifeq ($(SSE2_VAR),)
+SSE2_VAR=on
+endif
+
+CFLAGS     = -MD -nologo -D_CRT_SECURE_NO_DEPRECATE -D_CRT_NONSTDC_NO_DEPRECATE -I../pixman/src -I. -DPACKAGE=$(LIBRARY) -DPACKAGE_VERSION="" -DPACKAGE_BUGREPORT=""
+MMX_CFLAGS = -DUSE_MMX -w14710 -w14714
+SSE2_CFLAGS = -DUSE_SSE2
+
+# optimization flags
+ifeq ($(CFG_VAR),debug)
+CFLAGS += -Od -Zi
+else
+CFLAGS += -O2
+endif
+
+SOURCES =				\
+	pixman-image.c			\
+	pixman-access.c			\
+	pixman-access-accessors.c	\
+	pixman-region16.c		\
+	pixman-region32.c		\
+	pixman-combine32.c		\
+	pixman-combine64.c		\
+	pixman-utils.c			\
+	pixman-edge.c			\
+	pixman-edge-accessors.c		\
+	pixman-trap.c			\
+	pixman-timer.c			\
+	pixman-matrix.c			\
+	pixman-gradient-walker.c	\
+	pixman-linear-gradient.c	\
+	pixman-radial-gradient.c	\
+	pixman-bits-image.c		\
+	pixman.c			\
+	pixman-cpu.c			\
+	pixman-fast-path.c		\
+	pixman-implementation.c		\
+	pixman-solid-fill.c		\
+	pixman-general.c		\
+	$(NULL)
+
+# MMX compilation flags
+ifeq ($(MMX_VAR),on)
+CFLAGS += $(MMX_CFLAGS)
+SOURCES += pixman-mmx.c
+endif
+
+# SSE2 compilation flags
+ifeq ($(SSE2_VAR),on)
+CFLAGS += $(SSE2_CFLAGS)
+SOURCES += pixman-sse2.c
+endif
+
+OBJECTS     = $(patsubst %.c, $(CFG_VAR)/%.obj, $(SOURCES))
+
+# targets
+all: inform informMMX informSSE2 $(CFG_VAR)/$(LIBRARY).lib
+	@exit 0
+clean: inform clean_r 
+	@exit 0
+pixman: inform informMMX informSSE2 $(CFG_VAR)/$(LIBRARY).lib 
+	@exit 0
+
+inform:
+ifneq ($(CFG),release)
+ifneq ($(CFG),debug)
+ifneq ($(CFG),)
+	@echo "Invalid specified configuration option : "$(CFG)"."
+	@echo
+	@echo -n "Possible choices for configuration are "
+	@echo "'release' and 'debug'"
+	@echo ""
+	@exit 1
+endif
+	@echo "Using default RELEASE configuration... (use CFG=release or CFG=debug)"
+endif
+endif
+
+informMMX:
+ifneq ($(MMX),off)
+ifneq ($(MMX),on)
+ifneq ($(MMX),)
+	@echo "Invalid specified MMX option : "$(MMX_VAR)"."
+	@echo
+	@echo -n "Possible choices for MMX are 'on' or 'off'"
+	@echo ""
+	@exit 1
+endif
+	@echo "Setting MMX flag to default value 'on'... (use MMX=on or MMX=off)"
+endif
+endif
+
+informSSE2:
+ifneq ($(SSE2),off)
+ifneq ($(SSE2),on)
+ifneq ($(SSE2),)
+	@echo "Invalid specified SSE option : "$(SSE2)"."
+	@echo
+	@echo -n "Possible choices for SSE2 are 'on' or 'off'"
+	@echo ""
+	@exit 1
+endif
+	@echo "Setting SSE2 flag to default value 'on'... (use SSE2=on or SSE2=off)"
+endif
+endif
+
+# pixman compilation and linking
+$(CFG_VAR)/%.obj: %.c
+	@mkdir -p $(CFG_VAR)
+	@$(CC) -c $(CFLAGS) -Fo"$@" $<
+
+$(CFG_VAR)/$(LIBRARY).lib: $(OBJECTS)
+	lib -NOLOGO -OUT:$@ $(OBJECTS) || exit 0
+
+pixman-combine32.c: pixman-combine.c.template pixman-combine32.h make-combine.pl
+	perl ./make-combine.pl 8 < $< > $@ || ($(RM) $@; exit 1)
+pixman-combine32.h: pixman-combine.h.template make-combine.pl
+	perl ./make-combine.pl 8 < $< > $@ || ($(RM) $@; exit 1)
+
+pixman-combine64.c: pixman-combine.c.template pixman-combine64.h make-combine.pl
+	perl ./make-combine.pl 16 < $< > $@ || ($(RM) $@; exit 1)
+pixman-combine64.h: pixman-combine.h.template make-combine.pl
+	perl ./make-combine.pl 16 < $< > $@ || ($(RM) $@; exit 1)
+
+clean_r:
+	@rm -f $(CFG_VAR)/*.obj $(CFG_VAR)/*.lib $(CFG_VAR)/*.pdb $(CFG)/*.ilk || exit 0
+	@rm -f $(CFG)/*.obj $(CFG)/*.lib $(CFG)/*.pdb $(CFG)/*.ilk pixman-combine32.c pixman-combine64.c pixman-combine64.c pixman-combine64.h || exit 0
diff --git a/pixman/pixman/make-combine.pl b/pixman/pixman/make-combine.pl
new file mode 100644
index 000000000..210a5da12
--- /dev/null
+++ b/pixman/pixman/make-combine.pl
@@ -0,0 +1,86 @@
+$usage = "Usage: combine.pl { 8 | 16 } < pixman-combine.c.template";
+
+$#ARGV == 0 or die $usage;
+
+# Get the component size.
+$size = int($ARGV[0]);
+$size == 8 or $size == 16 or die $usage;
+
+$pixel_size = $size * 4;
+$half_pixel_size = $size * 2;
+
+sub mask {
+    my $str = shift;
+    my $suffix;
+    $suffix = "ULL" if $size > 8;
+
+    return "0x" . $str . $suffix;
+}
+
+# Generate mask strings.
+$nibbles = $size / 4;
+$mask = "f" x $nibbles;
+$zero_mask = "0" x $nibbles;
+$one_half = "8" . "0" x ($nibbles - 1);
+
+print "/* WARNING: This file is generated by combine.pl from combine.inc.\n";
+print "   Please edit one of those files rather than this one. */\n";
+print "\n";
+
+print "#line 1 \"pixman-combine.c.template\"\n";
+
+$mask_ = mask($mask);
+$one_half_ = mask($one_half);
+$g_mask = mask($mask . $zero_mask);
+$b_mask = mask($mask . $zero_mask x 2);
+$a_mask = mask($mask . $zero_mask x 3);
+$rb_mask = mask($mask . $zero_mask . $mask);
+$ag_mask = mask($mask . $zero_mask . $mask . $zero_mask);
+$rb_one_half = mask($one_half . $zero_mask . $one_half);
+$rb_mask_plus_one = mask("1" . $zero_mask x 2 . "1" .  $zero_mask);
+
+while (<STDIN>) {
+    # Mask and 1/2 value for a single component.
+    s/#define COMPONENT_SIZE\b/$& $size/;
+    s/#define MASK\b/$& $mask_/;
+    s/#define ONE_HALF\b/$& $one_half_/;
+
+    # Shifts and masks for green, blue, and alpha.
+    s/#define G_SHIFT\b/$& $size/;
+    s/#define R_SHIFT\b/$& $size * 2/;
+    s/#define A_SHIFT\b/$& $size * 3/;
+    s/#define G_MASK\b/$& $g_mask/;
+    s/#define R_MASK\b/$& $b_mask/;
+    s/#define A_MASK\b/$& $a_mask/;
+
+    # Special values for dealing with red + blue at the same time.
+    s/#define RB_MASK\b/$& $rb_mask/;
+    s/#define AG_MASK\b/$& $ag_mask/;
+    s/#define RB_ONE_HALF\b/$& $rb_one_half/;
+    s/#define RB_MASK_PLUS_ONE\b/$& $rb_mask_plus_one/;
+
+    # Add 32/64 suffix to combining function types.
+    s/\bCombineFunc\b/CombineFunc$pixel_size/;
+    s/\bFbComposeFunctions\b/FbComposeFunctions$pixel_size/;
+    s/combine_width/combine_$pixel_size/;
+    s/_pixman_setup_combiner_functions_width/_pixman_setup_combiner_functions_$pixel_size/;
+    s/UNc/UN$size/g;
+    s/ALPHA_c/ALPHA_$size/g;
+    s/RED_c/RED_$size/g;
+    s/GREEN_c/GREEN_$size/g;
+    s/BLUE_c/BLUE_$size/g;
+
+    # Convert comp*_t values into the appropriate real types.
+    s/comp1_t/uint${size}_t/g;
+    s/comp2_t/uint${half_pixel_size}_t/g;
+    s/comp4_t/uint${pixel_size}_t/g;
+
+    # Change the function table name for the 64-bit version.
+    s/pixman_composeFunctions/pixman_composeFunctions64/ if $size == 16;
+
+    # Change the header for the 64-bit version
+    s/pixman-combine.h/pixman-combine64.h/ if $size == 16;
+    s/pixman-combine.h/pixman-combine32.h/ if $size == 8;
+
+    print;
+}
diff --git a/pixman/pixman/pixman-access-accessors.c b/pixman/pixman/pixman-access-accessors.c
new file mode 100644
index 000000000..3263582f1
--- /dev/null
+++ b/pixman/pixman/pixman-access-accessors.c
@@ -0,0 +1,3 @@
+#define PIXMAN_FB_ACCESSORS
+
+#include "pixman-access.c"
diff --git a/pixman/pixman/pixman-access.c b/pixman/pixman/pixman-access.c
new file mode 100644
index 000000000..b65ef661d
--- /dev/null
+++ b/pixman/pixman/pixman-access.c
@@ -0,0 +1,2902 @@
+/*
+ *
+ * Copyright © 2000 Keith Packard, member of The XFree86 Project, Inc.
+ *             2005 Lars Knoll & Zack Rusin, Trolltech
+ *             2008 Aaron Plattner, NVIDIA Corporation
+ *
+ * Permission to use, copy, modify, distribute, and sell this software and its
+ * documentation for any purpose is hereby granted without fee, provided that
+ * the above copyright notice appear in all copies and that both that
+ * copyright notice and this permission notice appear in supporting
+ * documentation, and that the name of Keith Packard not be used in
+ * advertising or publicity pertaining to distribution of the software without
+ * specific, written prior permission.  Keith Packard makes no
+ * representations about the suitability of this software for any purpose.  It
+ * is provided "as is" without express or implied warranty.
+ *
+ * THE COPYRIGHT HOLDERS DISCLAIM ALL WARRANTIES WITH REGARD TO THIS
+ * SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
+ * FITNESS, IN NO EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
+ * AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
+ * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS
+ * SOFTWARE.
+ */
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <stdlib.h>
+#include <string.h>
+#include <assert.h>
+
+#include "pixman-private.h"
+#include "pixman-accessor.h"
+
+#define CONVERT_RGB24_TO_Y15(s)						\
+    (((((s) >> 16) & 0xff) * 153 +					\
+      (((s) >>  8) & 0xff) * 301 +					\
+      (((s)      ) & 0xff) * 58) >> 2)
+
+#define CONVERT_RGB24_TO_RGB15(s)                                       \
+    ((((s) >> 3) & 0x001f) |                                            \
+     (((s) >> 6) & 0x03e0) |                                            \
+     (((s) >> 9) & 0x7c00))
+
+#define RGB15_TO_ENTRY(mif,rgb15)					\
+    ((mif)->ent[rgb15])
+
+#define RGB24_TO_ENTRY(mif,rgb24)					\
+    RGB15_TO_ENTRY (mif,CONVERT_RGB24_TO_RGB15 (rgb24))
+
+#define RGB24_TO_ENTRY_Y(mif,rgb24)					\
+    ((mif)->ent[CONVERT_RGB24_TO_Y15 (rgb24)])
+
+/*
+ * YV12 setup and access macros
+ */
+
+#define YV12_SETUP(image)                                               \
+    bits_image_t *__bits_image = (bits_image_t *)image;                 \
+    uint32_t *bits = __bits_image->bits;                                \
+    int stride = __bits_image->rowstride;                               \
+    int offset0 = stride < 0 ?                                          \
+    ((-stride) >> 1) * ((__bits_image->height - 1) >> 1) - stride :	\
+    stride * __bits_image->height;					\
+    int offset1 = stride < 0 ?                                          \
+    offset0 + ((-stride) >> 1) * ((__bits_image->height) >> 1) :	\
+	offset0 + (offset0 >> 2)
+
+/* Note no trailing semicolon on the above macro; if it's there, then
+ * the typical usage of YV12_SETUP(image); will have an extra trailing ;
+ * that some compilers will interpret as a statement -- and then any further
+ * variable declarations will cause an error.
+ */
+
+#define YV12_Y(line)                                                    \
+    ((uint8_t *) ((bits) + (stride) * (line)))
+
+#define YV12_U(line)                                                    \
+    ((uint8_t *) ((bits) + offset1 +                                    \
+                  ((stride) >> 1) * ((line) >> 1)))
+
+#define YV12_V(line)                                                    \
+    ((uint8_t *) ((bits) + offset0 +                                    \
+                  ((stride) >> 1) * ((line) >> 1)))
+
+/********************************** Fetch ************************************/
+
+static void
+fetch_scanline_a8r8g8b8 (pixman_image_t *image,
+                         int             x,
+                         int             y,
+                         int             width,
+                         uint32_t *      buffer,
+                         const uint32_t *mask,
+                         uint32_t        mask_bits)
+{
+    const uint32_t *bits = image->bits.bits + y * image->bits.rowstride;
+    
+    MEMCPY_WRAPPED (image,
+                    buffer, (const uint32_t *)bits + x,
+                    width * sizeof(uint32_t));
+}
+
+static void
+fetch_scanline_x8r8g8b8 (pixman_image_t *image,
+                         int             x,
+                         int             y,
+                         int             width,
+                         uint32_t *      buffer,
+                         const uint32_t *mask,
+                         uint32_t        mask_bits)
+{
+    const uint32_t *bits = image->bits.bits + y * image->bits.rowstride;
+    const uint32_t *pixel = (const uint32_t *)bits + x;
+    const uint32_t *end = pixel + width;
+    
+    while (pixel < end)
+	*buffer++ = READ (image, pixel++) | 0xff000000;
+}
+
+static void
+fetch_scanline_a8b8g8r8 (pixman_image_t *image,
+                         int             x,
+                         int             y,
+                         int             width,
+                         uint32_t *      buffer,
+                         const uint32_t *mask,
+                         uint32_t        mask_bits)
+{
+    const uint32_t *bits = image->bits.bits + y * image->bits.rowstride;
+    const uint32_t *pixel = (uint32_t *)bits + x;
+    const uint32_t *end = pixel + width;
+    
+    while (pixel < end)
+    {
+	uint32_t p = READ (image, pixel++);
+	
+	*buffer++ = (p & 0xff00ff00)	|
+	    ((p >> 16) & 0xff)		|
+	    ((p & 0xff) << 16);
+    }
+}
+
+static void
+fetch_scanline_x8b8g8r8 (pixman_image_t *image,
+                         int             x,
+                         int             y,
+                         int             width,
+                         uint32_t *      buffer,
+                         const uint32_t *mask,
+                         uint32_t        mask_bits)
+{
+    const uint32_t *bits = image->bits.bits + y * image->bits.rowstride;
+    const uint32_t *pixel = (uint32_t *)bits + x;
+    const uint32_t *end = pixel + width;
+    
+    while (pixel < end)
+    {
+	uint32_t p = READ (image, pixel++);
+	
+	*buffer++ = 0xff000000		|
+	    (p & 0x0000ff00)		|
+	    ((p >> 16) & 0xff)		|
+	    ((p & 0xff) << 16);
+    }
+}
+
+static void
+fetch_scanline_b8g8r8a8 (pixman_image_t *image,
+                         int             x,
+                         int             y,
+                         int             width,
+                         uint32_t *      buffer,
+                         const uint32_t *mask,
+                         uint32_t        mask_bits)
+{
+    const uint32_t *bits = image->bits.bits + y * image->bits.rowstride;
+    const uint32_t *pixel = (uint32_t *)bits + x;
+    const uint32_t *end = pixel + width;
+
+    while (pixel < end)
+    {
+	uint32_t p = READ (image, pixel++);
+
+	*buffer++ = (((p & 0xff000000) >> 24)	|
+	             ((p & 0x00ff0000) >> 8)	|
+	             ((p & 0x0000ff00) << 8)	|
+	             ((p & 0x000000ff) << 24));
+    }
+}
+
+static void
+fetch_scanline_b8g8r8x8 (pixman_image_t *image,
+                         int             x,
+                         int             y,
+                         int             width,
+                         uint32_t *      buffer,
+                         const uint32_t *mask,
+                         uint32_t        mask_bits)
+{
+    const uint32_t *bits = image->bits.bits + y * image->bits.rowstride;
+    const uint32_t *pixel = (uint32_t *)bits + x;
+    const uint32_t *end = pixel + width;
+    
+    while (pixel < end)
+    {
+	uint32_t p = READ (image, pixel++);
+	
+	*buffer++ = (0xff000000 |
+	             ((p & 0xff000000) >> 24)	|
+	             ((p & 0x00ff0000) >> 8)	|
+	             ((p & 0x0000ff00) << 8));
+    }
+}
+
+/* Expects a uint64_t buffer */
+static void
+fetch_scanline_a2r10g10b10 (pixman_image_t *image,
+                            int             x,
+                            int             y,
+                            int             width,
+                            uint32_t *      b,
+                            const uint32_t *mask,
+                            uint32_t        mask_bits)
+{
+    const uint32_t *bits = image->bits.bits + y * image->bits.rowstride;
+    const uint32_t *pixel = bits + x;
+    const uint32_t *end = pixel + width;
+    uint64_t *buffer = (uint64_t *)b;
+
+    while (pixel < end)
+    {
+	uint32_t p = READ (image, pixel++);
+	uint64_t a = p >> 30;
+	uint64_t r = (p >> 20) & 0x3ff;
+	uint64_t g = (p >> 10) & 0x3ff;
+	uint64_t b = p & 0x3ff;
+
+	r = r << 6 | r >> 4;
+	g = g << 6 | g >> 4;
+	b = b << 6 | b >> 4;
+
+	a <<= 14;
+	a |= a >> 2;
+	a |= a >> 4;
+	a |= a >> 8;
+
+	*buffer++ = a << 48 | r << 32 | g << 16 | b;
+    }
+}
+
+/* Expects a uint64_t buffer */
+static void
+fetch_scanline_x2r10g10b10 (pixman_image_t *image,
+                            int             x,
+                            int             y,
+                            int             width,
+                            uint32_t *      b,
+                            const uint32_t *mask,
+                            uint32_t        mask_bits)
+{
+    const uint32_t *bits = image->bits.bits + y * image->bits.rowstride;
+    const uint32_t *pixel = (uint32_t *)bits + x;
+    const uint32_t *end = pixel + width;
+    uint64_t *buffer = (uint64_t *)b;
+    
+    while (pixel < end)
+    {
+	uint32_t p = READ (image, pixel++);
+	uint64_t r = (p >> 20) & 0x3ff;
+	uint64_t g = (p >> 10) & 0x3ff;
+	uint64_t b = p & 0x3ff;
+	
+	r = r << 6 | r >> 4;
+	g = g << 6 | g >> 4;
+	b = b << 6 | b >> 4;
+	
+	*buffer++ = 0xffffULL << 48 | r << 32 | g << 16 | b;
+    }
+}
+
+/* Expects a uint64_t buffer */
+static void
+fetch_scanline_a2b10g10r10 (pixman_image_t *image,
+                            int             x,
+                            int             y,
+                            int             width,
+                            uint32_t *      b,
+                            const uint32_t *mask,
+                            uint32_t        mask_bits)
+{
+    const uint32_t *bits = image->bits.bits + y * image->bits.rowstride;
+    const uint32_t *pixel = bits + x;
+    const uint32_t *end = pixel + width;
+    uint64_t *buffer = (uint64_t *)b;
+    
+    while (pixel < end)
+    {
+	uint32_t p = READ (image, pixel++);
+	uint64_t a = p >> 30;
+	uint64_t b = (p >> 20) & 0x3ff;
+	uint64_t g = (p >> 10) & 0x3ff;
+	uint64_t r = p & 0x3ff;
+	
+	r = r << 6 | r >> 4;
+	g = g << 6 | g >> 4;
+	b = b << 6 | b >> 4;
+	
+	a <<= 14;
+	a |= a >> 2;
+	a |= a >> 4;
+	a |= a >> 8;
+
+	*buffer++ = a << 48 | r << 32 | g << 16 | b;
+    }
+}
+
+/* Expects a uint64_t buffer */
+static void
+fetch_scanline_x2b10g10r10 (pixman_image_t *image,
+                            int             x,
+                            int             y,
+                            int             width,
+                            uint32_t *      b,
+                            const uint32_t *mask,
+                            uint32_t        mask_bits)
+{
+    const uint32_t *bits = image->bits.bits + y * image->bits.rowstride;
+    const uint32_t *pixel = (uint32_t *)bits + x;
+    const uint32_t *end = pixel + width;
+    uint64_t *buffer = (uint64_t *)b;
+    
+    while (pixel < end)
+    {
+	uint32_t p = READ (image, pixel++);
+	uint64_t b = (p >> 20) & 0x3ff;
+	uint64_t g = (p >> 10) & 0x3ff;
+	uint64_t r = p & 0x3ff;
+	
+	r = r << 6 | r >> 4;
+	g = g << 6 | g >> 4;
+	b = b << 6 | b >> 4;
+	
+	*buffer++ = 0xffffULL << 48 | r << 32 | g << 16 | b;
+    }
+}
+
+static void
+fetch_scanline_r8g8b8 (pixman_image_t *image,
+                       int             x,
+                       int             y,
+                       int             width,
+                       uint32_t *      buffer,
+                       const uint32_t *mask,
+                       uint32_t        mask_bits)
+{
+    const uint32_t *bits = image->bits.bits + y * image->bits.rowstride;
+    const uint8_t *pixel = (const uint8_t *)bits + 3 * x;
+    const uint8_t *end = pixel + 3 * width;
+    
+    while (pixel < end)
+    {
+	uint32_t b = 0xff000000;
+	
+#ifdef WORDS_BIGENDIAN
+	b |= (READ (image, pixel++) << 16);
+	b |= (READ (image, pixel++) << 8);
+	b |= (READ (image, pixel++));
+#else
+	b |= (READ (image, pixel++));
+	b |= (READ (image, pixel++) << 8);
+	b |= (READ (image, pixel++) << 16);
+#endif
+	
+	*buffer++ = b;
+    }
+}
+
+static void
+fetch_scanline_b8g8r8 (pixman_image_t *image,
+                       int             x,
+                       int             y,
+                       int             width,
+                       uint32_t *      buffer,
+                       const uint32_t *mask,
+                       uint32_t        mask_bits)
+{
+    const uint32_t *bits = image->bits.bits + y * image->bits.rowstride;
+    const uint8_t *pixel = (const uint8_t *)bits + 3 * x;
+    const uint8_t *end = pixel + 3 * width;
+    
+    while (pixel < end)
+    {
+	uint32_t b = 0xff000000;
+#ifdef WORDS_BIGENDIAN
+	b |= (READ (image, pixel++));
+	b |= (READ (image, pixel++) << 8);
+	b |= (READ (image, pixel++) << 16);
+#else
+	b |= (READ (image, pixel++) << 16);
+	b |= (READ (image, pixel++) << 8);
+	b |= (READ (image, pixel++));
+#endif
+	*buffer++ = b;
+    }
+}
+
+static void
+fetch_scanline_r5g6b5 (pixman_image_t *image,
+                       int             x,
+                       int             y,
+                       int             width,
+                       uint32_t *      buffer,
+                       const uint32_t *mask,
+                       uint32_t        mask_bits)
+{
+    const uint32_t *bits = image->bits.bits + y * image->bits.rowstride;
+    const uint16_t *pixel = (const uint16_t *)bits + x;
+    const uint16_t *end = pixel + width;
+    
+    while (pixel < end)
+    {
+	uint32_t p = READ (image, pixel++);
+	uint32_t r = (((p) << 3) & 0xf8) |
+	    (((p) << 5) & 0xfc00) |
+	    (((p) << 8) & 0xf80000);
+	
+	r |= (r >> 5) & 0x70007;
+	r |= (r >> 6) & 0x300;
+	
+	*buffer++ = 0xff000000 | r;
+    }
+}
+
+static void
+fetch_scanline_b5g6r5 (pixman_image_t *image,
+                       int             x,
+                       int             y,
+                       int             width,
+                       uint32_t *      buffer,
+                       const uint32_t *mask,
+                       uint32_t        mask_bits)
+{
+    const uint32_t *bits = image->bits.bits + y * image->bits.rowstride;
+    const uint16_t *pixel = (const uint16_t *)bits + x;
+    const uint16_t *end = pixel + width;
+    
+    while (pixel < end)
+    {
+	uint32_t p = READ (image, pixel++);
+	uint32_t r, g, b;
+	
+	b = ((p & 0xf800) | ((p & 0xe000) >> 5)) >> 8;
+	g = ((p & 0x07e0) | ((p & 0x0600) >> 6)) << 5;
+	r = ((p & 0x001c) | ((p & 0x001f) << 5)) << 14;
+	
+	*buffer++ = 0xff000000 | r | g | b;
+    }
+}
+
+static void
+fetch_scanline_a1r5g5b5 (pixman_image_t *image,
+                         int             x,
+                         int             y,
+                         int             width,
+                         uint32_t *      buffer,
+                         const uint32_t *mask,
+                         uint32_t        mask_bits)
+{
+    const uint32_t *bits = image->bits.bits + y * image->bits.rowstride;
+    const uint16_t *pixel = (const uint16_t *)bits + x;
+    const uint16_t *end = pixel + width;
+    
+    while (pixel < end)
+    {
+	uint32_t p = READ (image, pixel++);
+	uint32_t r, g, b, a;
+	
+	a = (uint32_t) ((uint8_t) (0 - ((p & 0x8000) >> 15))) << 24;
+	r = ((p & 0x7c00) | ((p & 0x7000) >> 5)) << 9;
+	g = ((p & 0x03e0) | ((p & 0x0380) >> 5)) << 6;
+	b = ((p & 0x001c) | ((p & 0x001f) << 5)) >> 2;
+	
+	*buffer++ = a | r | g | b;
+    }
+}
+
+static void
+fetch_scanline_x1r5g5b5 (pixman_image_t *image,
+                         int             x,
+                         int             y,
+                         int             width,
+                         uint32_t *      buffer,
+                         const uint32_t *mask,
+                         uint32_t        mask_bits)
+{
+    const uint32_t *bits = image->bits.bits + y * image->bits.rowstride;
+    const uint16_t *pixel = (const uint16_t *)bits + x;
+    const uint16_t *end = pixel + width;
+    
+    while (pixel < end)
+    {
+	uint32_t p = READ (image, pixel++);
+	uint32_t r, g, b;
+	
+	r = ((p & 0x7c00) | ((p & 0x7000) >> 5)) << 9;
+	g = ((p & 0x03e0) | ((p & 0x0380) >> 5)) << 6;
+	b = ((p & 0x001c) | ((p & 0x001f) << 5)) >> 2;
+	
+	*buffer++ = 0xff000000 | r | g | b;
+    }
+}
+
+static void
+fetch_scanline_a1b5g5r5 (pixman_image_t *image,
+                         int             x,
+                         int             y,
+                         int             width,
+                         uint32_t *      buffer,
+                         const uint32_t *mask,
+                         uint32_t        mask_bits)
+{
+    const uint32_t *bits = image->bits.bits + y * image->bits.rowstride;
+    const uint16_t *pixel = (const uint16_t *)bits + x;
+    const uint16_t *end = pixel + width;
+    uint32_t r, g, b, a;
+    
+    while (pixel < end)
+    {
+	uint32_t p = READ (image, pixel++);
+	
+	a = (uint32_t) ((uint8_t) (0 - ((p & 0x8000) >> 15))) << 24;
+	b = ((p & 0x7c00) | ((p & 0x7000) >> 5)) >> 7;
+	g = ((p & 0x03e0) | ((p & 0x0380) >> 5)) << 6;
+	r = ((p & 0x001c) | ((p & 0x001f) << 5)) << 14;
+	
+	*buffer++ = a | r | g | b;
+    }
+}
+
+static void
+fetch_scanline_x1b5g5r5 (pixman_image_t *image,
+                         int             x,
+                         int             y,
+                         int             width,
+                         uint32_t *      buffer,
+                         const uint32_t *mask,
+                         uint32_t        mask_bits)
+{
+    const uint32_t *bits = image->bits.bits + y * image->bits.rowstride;
+    const uint16_t *pixel = (const uint16_t *)bits + x;
+    const uint16_t *end = pixel + width;
+    
+    while (pixel < end)
+    {
+	uint32_t p = READ (image, pixel++);
+	uint32_t r, g, b;
+	
+	b = ((p & 0x7c00) | ((p & 0x7000) >> 5)) >> 7;
+	g = ((p & 0x03e0) | ((p & 0x0380) >> 5)) << 6;
+	r = ((p & 0x001c) | ((p & 0x001f) << 5)) << 14;
+	
+	*buffer++ = 0xff000000 | r | g | b;
+    }
+}
+
+static void
+fetch_scanline_a4r4g4b4 (pixman_image_t *image,
+                         int             x,
+                         int             y,
+                         int             width,
+                         uint32_t *      buffer,
+                         const uint32_t *mask,
+                         uint32_t        mask_bits)
+{
+    const uint32_t *bits = image->bits.bits + y * image->bits.rowstride;
+    const uint16_t *pixel = (const uint16_t *)bits + x;
+    const uint16_t *end = pixel + width;
+    
+    while (pixel < end)
+    {
+	uint32_t p = READ (image, pixel++);
+	uint32_t r, g, b, a;
+	
+	a = ((p & 0xf000) | ((p & 0xf000) >> 4)) << 16;
+	r = ((p & 0x0f00) | ((p & 0x0f00) >> 4)) << 12;
+	g = ((p & 0x00f0) | ((p & 0x00f0) >> 4)) << 8;
+	b = ((p & 0x000f) | ((p & 0x000f) << 4));
+	
+	*buffer++ = a | r | g | b;
+    }
+}
+
+static void
+fetch_scanline_x4r4g4b4 (pixman_image_t *image,
+                         int             x,
+                         int             y,
+                         int             width,
+                         uint32_t *      buffer,
+                         const uint32_t *mask,
+                         uint32_t        mask_bits)
+{
+    const uint32_t *bits = image->bits.bits + y * image->bits.rowstride;
+    const uint16_t *pixel = (const uint16_t *)bits + x;
+    const uint16_t *end = pixel + width;
+    
+    while (pixel < end)
+    {
+	uint32_t p = READ (image, pixel++);
+	uint32_t r, g, b;
+	
+	r = ((p & 0x0f00) | ((p & 0x0f00) >> 4)) << 12;
+	g = ((p & 0x00f0) | ((p & 0x00f0) >> 4)) << 8;
+	b = ((p & 0x000f) | ((p & 0x000f) << 4));
+	
+	*buffer++ = 0xff000000 | r | g | b;
+    }
+}
+
+static void
+fetch_scanline_a4b4g4r4 (pixman_image_t *image,
+                         int             x,
+                         int             y,
+                         int             width,
+                         uint32_t *      buffer,
+                         const uint32_t *mask,
+                         uint32_t        mask_bits)
+{
+    const uint32_t *bits = image->bits.bits + y * image->bits.rowstride;
+    const uint16_t *pixel = (const uint16_t *)bits + x;
+    const uint16_t *end = pixel + width;
+    
+    while (pixel < end)
+    {
+	uint32_t p = READ (image, pixel++);
+	uint32_t r, g, b, a;
+	
+	a = ((p & 0xf000) | ((p & 0xf000) >> 4)) << 16;
+	b = ((p & 0x0f00) | ((p & 0x0f00) >> 4)) >> 4;
+	g = ((p & 0x00f0) | ((p & 0x00f0) >> 4)) << 8;
+	r = ((p & 0x000f) | ((p & 0x000f) << 4)) << 16;
+	
+	*buffer++ = a | r | g | b;
+    }
+}
+
+static void
+fetch_scanline_x4b4g4r4 (pixman_image_t *image,
+                         int             x,
+                         int             y,
+                         int             width,
+                         uint32_t *      buffer,
+                         const uint32_t *mask,
+                         uint32_t        mask_bits)
+{
+    const uint32_t *bits = image->bits.bits + y * image->bits.rowstride;
+    const uint16_t *pixel = (const uint16_t *)bits + x;
+    const uint16_t *end = pixel + width;
+    
+    while (pixel < end)
+    {
+	uint32_t p = READ (image, pixel++);
+	uint32_t r, g, b;
+	
+	b = ((p & 0x0f00) | ((p & 0x0f00) >> 4)) >> 4;
+	g = ((p & 0x00f0) | ((p & 0x00f0) >> 4)) << 8;
+	r = ((p & 0x000f) | ((p & 0x000f) << 4)) << 16;
+	
+	*buffer++ = 0xff000000 | r | g | b;
+    }
+}
+
+static void
+fetch_scanline_a8 (pixman_image_t *image,
+                   int             x,
+                   int             y,
+                   int             width,
+                   uint32_t *      buffer,
+                   const uint32_t *mask,
+                   uint32_t        mask_bits)
+{
+    const uint32_t *bits = image->bits.bits + y * image->bits.rowstride;
+    const uint8_t *pixel = (const uint8_t *)bits + x;
+    const uint8_t *end = pixel + width;
+    
+    while (pixel < end)
+	*buffer++ = READ (image, pixel++) << 24;
+}
+
+static void
+fetch_scanline_r3g3b2 (pixman_image_t *image,
+                       int             x,
+                       int             y,
+                       int             width,
+                       uint32_t *      buffer,
+                       const uint32_t *mask,
+                       uint32_t        mask_bits)
+{
+    const uint32_t *bits = image->bits.bits + y * image->bits.rowstride;
+    const uint8_t *pixel = (const uint8_t *)bits + x;
+    const uint8_t *end = pixel + width;
+    
+    while (pixel < end)
+    {
+	uint32_t p = READ (image, pixel++);
+	uint32_t r, g, b;
+	
+	r = ((p & 0xe0) | ((p & 0xe0) >> 3) | ((p & 0xc0) >> 6)) << 16;
+	g = ((p & 0x1c) | ((p & 0x18) >> 3) | ((p & 0x1c) << 3)) << 8;
+	b = (((p & 0x03)     ) |
+	     ((p & 0x03) << 2) |
+	     ((p & 0x03) << 4) |
+	     ((p & 0x03) << 6));
+	
+	*buffer++ = 0xff000000 | r | g | b;
+    }
+}
+
+static void
+fetch_scanline_b2g3r3 (pixman_image_t *image,
+                       int             x,
+                       int             y,
+                       int             width,
+                       uint32_t *      buffer,
+                       const uint32_t *mask,
+                       uint32_t        mask_bits)
+{
+    const uint32_t *bits = image->bits.bits + y * image->bits.rowstride;
+    const uint8_t *pixel = (const uint8_t *)bits + x;
+    const uint8_t *end = pixel + width;
+
+    while (pixel < end)
+    {
+	uint32_t p = READ (image, pixel++);
+	uint32_t r, g, b;
+
+	b  = p & 0xc0;
+	b |= b >> 2;
+	b |= b >> 4;
+	b &= 0xff;
+
+	g  = (p & 0x38) << 10;
+	g |= g >> 3;
+	g |= g >> 6;
+	g &= 0xff00;
+
+	r  = (p & 0x7) << 21;
+	r |= r >> 3;
+	r |= r >> 6;
+	r &= 0xff0000;
+
+	*buffer++ = 0xff000000 | r | g | b;
+    }
+}
+
+static void
+fetch_scanline_a2r2g2b2 (pixman_image_t *image,
+                         int             x,
+                         int             y,
+                         int             width,
+                         uint32_t *      buffer,
+                         const uint32_t *mask,
+                         uint32_t        mask_bits)
+{
+    const uint32_t *bits = image->bits.bits + y * image->bits.rowstride;
+    const uint8_t *pixel = (const uint8_t *)bits + x;
+    const uint8_t *end = pixel + width;
+    
+    while (pixel < end)
+    {
+	uint32_t p = READ (image, pixel++);
+	uint32_t a, r, g, b;
+	
+	a = ((p & 0xc0) * 0x55) << 18;
+	r = ((p & 0x30) * 0x55) << 12;
+	g = ((p & 0x0c) * 0x55) << 6;
+	b = ((p & 0x03) * 0x55);
+	
+	*buffer++ = a | r | g | b;
+    }
+}
+
+static void
+fetch_scanline_a2b2g2r2 (pixman_image_t *image,
+                         int             x,
+                         int             y,
+                         int             width,
+                         uint32_t *      buffer,
+                         const uint32_t *mask,
+                         uint32_t        mask_bits)
+{
+    const uint32_t *bits = image->bits.bits + y * image->bits.rowstride;
+    const uint8_t *pixel = (const uint8_t *)bits + x;
+    const uint8_t *end = pixel + width;
+    
+    while (pixel < end)
+    {
+	uint32_t p = READ (image, pixel++);
+	uint32_t a, r, g, b;
+	
+	a = ((p & 0xc0) * 0x55) << 18;
+	b = ((p & 0x30) * 0x55) >> 4;
+	g = ((p & 0x0c) * 0x55) << 6;
+	r = ((p & 0x03) * 0x55) << 16;
+	
+	*buffer++ = a | r | g | b;
+    }
+}
+
+static void
+fetch_scanline_c8 (pixman_image_t *image,
+                   int             x,
+                   int             y,
+                   int             width,
+                   uint32_t *      buffer,
+                   const uint32_t *mask,
+                   uint32_t        mask_bits)
+{
+    const uint32_t *bits = image->bits.bits + y * image->bits.rowstride;
+    const pixman_indexed_t * indexed = image->bits.indexed;
+    const uint8_t *pixel = (const uint8_t *)bits + x;
+    const uint8_t *end = pixel + width;
+    
+    while (pixel < end)
+    {
+	uint32_t p = READ (image, pixel++);
+	
+	*buffer++ = indexed->rgba[p];
+    }
+}
+
+static void
+fetch_scanline_x4a4 (pixman_image_t *image,
+                     int             x,
+                     int             y,
+                     int             width,
+                     uint32_t *      buffer,
+                     const uint32_t *mask,
+                     uint32_t        mask_bits)
+{
+    const uint32_t *bits = image->bits.bits + y * image->bits.rowstride;
+    const uint8_t *pixel = (const uint8_t *)bits + x;
+    const uint8_t *end = pixel + width;
+   
+    while (pixel < end)
+    {
+	uint8_t p = READ (image, pixel++) & 0xf;
+
+	*buffer++ = (p | (p << 4)) << 24;
+    }
+}
+
+#define FETCH_8(img,l,o)    (READ (img, (((uint8_t *)(l)) + ((o) >> 3))))
+#ifdef WORDS_BIGENDIAN
+#define FETCH_4(img,l,o)						\
+    (((4 * (o)) & 4) ? (FETCH_8 (img,l, 4 * (o)) & 0xf) : (FETCH_8 (img,l,(4 * (o))) >> 4))
+#else
+#define FETCH_4(img,l,o)						\
+    (((4 * (o)) & 4) ? (FETCH_8 (img, l, 4 * (o)) >> 4) : (FETCH_8 (img, l, (4 * (o))) & 0xf))
+#endif
+
+static void
+fetch_scanline_a4 (pixman_image_t *image,
+                   int             x,
+                   int             y,
+                   int             width,
+                   uint32_t *      buffer,
+                   const uint32_t *mask,
+                   uint32_t        mask_bits)
+{
+    const uint32_t *bits = image->bits.bits + y * image->bits.rowstride;
+    int i;
+
+    for (i = 0; i < width; ++i)
+    {
+	uint32_t p = FETCH_4 (image, bits, i + x);
+
+	p |= p << 4;
+
+	*buffer++ = p << 24;
+    }
+}
+
+static void
+fetch_scanline_r1g2b1 (pixman_image_t *image,
+                       int             x,
+                       int             y,
+                       int             width,
+                       uint32_t *      buffer,
+                       const uint32_t *mask,
+                       uint32_t        mask_bits)
+{
+    const uint32_t *bits = image->bits.bits + y * image->bits.rowstride;
+    int i;
+    
+    for (i = 0; i < width; ++i)
+    {
+	uint32_t p = FETCH_4 (image, bits, i + x);
+	uint32_t r, g, b;
+	
+	r = ((p & 0x8) * 0xff) << 13;
+	g = ((p & 0x6) * 0x55) << 7;
+	b = ((p & 0x1) * 0xff);
+	
+	*buffer++ = 0xff000000 | r | g | b;
+    }
+}
+
+static void
+fetch_scanline_b1g2r1 (pixman_image_t *image,
+                       int             x,
+                       int             y,
+                       int             width,
+                       uint32_t *      buffer,
+                       const uint32_t *mask,
+                       uint32_t        mask_bits)
+{
+    const uint32_t *bits = image->bits.bits + y * image->bits.rowstride;
+    int i;
+    
+    for (i = 0; i < width; ++i)
+    {
+	uint32_t p = FETCH_4 (image, bits, i + x);
+	uint32_t r, g, b;
+	
+	b = ((p & 0x8) * 0xff) >> 3;
+	g = ((p & 0x6) * 0x55) << 7;
+	r = ((p & 0x1) * 0xff) << 16;
+
+	*buffer++ = 0xff000000 | r | g | b;
+    }
+}
+
+static void
+fetch_scanline_a1r1g1b1 (pixman_image_t *image,
+                         int             x,
+                         int             y,
+                         int             width,
+                         uint32_t *      buffer,
+                         const uint32_t *mask,
+                         uint32_t        mask_bits)
+{
+    uint32_t a, r, g, b;
+    const uint32_t *bits = image->bits.bits + y * image->bits.rowstride;
+    int i;
+
+    for (i = 0; i < width; ++i)
+    {
+	uint32_t p = FETCH_4 (image, bits, i + x);
+
+	a = ((p & 0x8) * 0xff) << 21;
+	r = ((p & 0x4) * 0xff) << 14;
+	g = ((p & 0x2) * 0xff) << 7;
+	b = ((p & 0x1) * 0xff);
+
+	*buffer++ = a | r | g | b;
+    }
+}
+
+static void
+fetch_scanline_a1b1g1r1 (pixman_image_t *image,
+                         int             x,
+                         int             y,
+                         int             width,
+                         uint32_t *      buffer,
+                         const uint32_t *mask,
+                         uint32_t        mask_bits)
+{
+    const uint32_t *bits = image->bits.bits + y * image->bits.rowstride;
+    int i;
+
+    for (i = 0; i < width; ++i)
+    {
+	uint32_t p = FETCH_4 (image, bits, i + x);
+	uint32_t a, r, g, b;
+
+	a = ((p & 0x8) * 0xff) << 21;
+	b = ((p & 0x4) * 0xff) >> 2;
+	g = ((p & 0x2) * 0xff) << 7;
+	r = ((p & 0x1) * 0xff) << 16;
+
+	*buffer++ = a | r | g | b;
+    }
+}
+
+static void
+fetch_scanline_c4 (pixman_image_t *image,
+                   int             x,
+                   int             y,
+                   int             width,
+                   uint32_t *      buffer,
+                   const uint32_t *mask,
+                   uint32_t        mask_bits)
+{
+    const uint32_t *bits = image->bits.bits + y * image->bits.rowstride;
+    const pixman_indexed_t * indexed = image->bits.indexed;
+    int i;
+    
+    for (i = 0; i < width; ++i)
+    {
+	uint32_t p = FETCH_4 (image, bits, i + x);
+	
+	*buffer++ = indexed->rgba[p];
+    }
+}
+
+static void
+fetch_scanline_a1 (pixman_image_t *image,
+                   int             x,
+                   int             y,
+                   int             width,
+                   uint32_t *      buffer,
+                   const uint32_t *mask,
+                   uint32_t        mask_bits)
+{
+    const uint32_t *bits = image->bits.bits + y * image->bits.rowstride;
+    int i;
+    
+    for (i = 0; i < width; ++i)
+    {
+	uint32_t p = READ (image, bits + ((i + x) >> 5));
+	uint32_t a;
+	
+#ifdef WORDS_BIGENDIAN
+	a = p >> (0x1f - ((i + x) & 0x1f));
+#else
+	a = p >> ((i + x) & 0x1f);
+#endif
+	a = a & 1;
+	a |= a << 1;
+	a |= a << 2;
+	a |= a << 4;
+	
+	*buffer++ = a << 24;
+    }
+}
+
+static void
+fetch_scanline_g1 (pixman_image_t *image,
+                   int             x,
+                   int             y,
+                   int             width,
+                   uint32_t *      buffer,
+                   const uint32_t *mask,
+                   uint32_t        mask_bits)
+{
+    const uint32_t *bits = image->bits.bits + y * image->bits.rowstride;
+    const pixman_indexed_t * indexed = image->bits.indexed;
+    int i;
+    
+    for (i = 0; i < width; ++i)
+    {
+	uint32_t p = READ (image, bits + ((i + x) >> 5));
+	uint32_t a;
+	
+#ifdef WORDS_BIGENDIAN
+	a = p >> (0x1f - ((i + x) & 0x1f));
+#else
+	a = p >> ((i + x) & 0x1f);
+#endif
+	a = a & 1;
+	
+	*buffer++ = indexed->rgba[a];
+    }
+}
+
+static void
+fetch_scanline_yuy2 (pixman_image_t *image,
+                     int             x,
+                     int             line,
+                     int             width,
+                     uint32_t *      buffer,
+                     const uint32_t *mask,
+                     uint32_t        mask_bits)
+{
+    const uint32_t *bits = image->bits.bits + image->bits.rowstride * line;
+    int i;
+    
+    for (i = 0; i < width; i++)
+    {
+	int16_t y, u, v;
+	int32_t r, g, b;
+	
+	y = ((uint8_t *) bits)[(x + i) << 1] - 16;
+	u = ((uint8_t *) bits)[(((x + i) << 1) & - 4) + 1] - 128;
+	v = ((uint8_t *) bits)[(((x + i) << 1) & - 4) + 3] - 128;
+	
+	/* R = 1.164(Y - 16) + 1.596(V - 128) */
+	r = 0x012b27 * y + 0x019a2e * v;
+	/* G = 1.164(Y - 16) - 0.813(V - 128) - 0.391(U - 128) */
+	g = 0x012b27 * y - 0x00d0f2 * v - 0x00647e * u;
+	/* B = 1.164(Y - 16) + 2.018(U - 128) */
+	b = 0x012b27 * y + 0x0206a2 * u;
+	
+	*buffer++ = 0xff000000 |
+	    (r >= 0 ? r < 0x1000000 ? r         & 0xff0000 : 0xff0000 : 0) |
+	    (g >= 0 ? g < 0x1000000 ? (g >> 8)  & 0x00ff00 : 0x00ff00 : 0) |
+	    (b >= 0 ? b < 0x1000000 ? (b >> 16) & 0x0000ff : 0x0000ff : 0);
+    }
+}
+
+static void
+fetch_scanline_yv12 (pixman_image_t *image,
+                     int             x,
+                     int             line,
+                     int             width,
+                     uint32_t *      buffer,
+                     const uint32_t *mask,
+                     uint32_t        mask_bits)
+{
+    YV12_SETUP (image);
+    uint8_t *y_line = YV12_Y (line);
+    uint8_t *u_line = YV12_U (line);
+    uint8_t *v_line = YV12_V (line);
+    int i;
+    
+    for (i = 0; i < width; i++)
+    {
+	int16_t y, u, v;
+	int32_t r, g, b;
+
+	y = y_line[x + i] - 16;
+	u = u_line[(x + i) >> 1] - 128;
+	v = v_line[(x + i) >> 1] - 128;
+
+	/* R = 1.164(Y - 16) + 1.596(V - 128) */
+	r = 0x012b27 * y + 0x019a2e * v;
+	/* G = 1.164(Y - 16) - 0.813(V - 128) - 0.391(U - 128) */
+	g = 0x012b27 * y - 0x00d0f2 * v - 0x00647e * u;
+	/* B = 1.164(Y - 16) + 2.018(U - 128) */
+	b = 0x012b27 * y + 0x0206a2 * u;
+
+	*buffer++ = 0xff000000 |
+	    (r >= 0 ? r < 0x1000000 ? r         & 0xff0000 : 0xff0000 : 0) |
+	    (g >= 0 ? g < 0x1000000 ? (g >> 8)  & 0x00ff00 : 0x00ff00 : 0) |
+	    (b >= 0 ? b < 0x1000000 ? (b >> 16) & 0x0000ff : 0x0000ff : 0);
+    }
+}
+
+/**************************** Pixel wise fetching *****************************/
+
+/* Despite the type, expects a uint64_t buffer */
+static uint64_t
+fetch_pixel_a2r10g10b10 (bits_image_t *image,
+			 int		  offset,
+			 int           line)
+{
+    uint32_t *bits = image->bits + line * image->rowstride;
+    uint32_t p = READ (image, bits + offset);
+    uint64_t a = p >> 30;
+    uint64_t r = (p >> 20) & 0x3ff;
+    uint64_t g = (p >> 10) & 0x3ff;
+    uint64_t b = p & 0x3ff;
+
+    r = r << 6 | r >> 4;
+    g = g << 6 | g >> 4;
+    b = b << 6 | b >> 4;
+
+    a <<= 14;
+    a |= a >> 2;
+    a |= a >> 4;
+    a |= a >> 8;
+
+    return a << 48 | r << 32 | g << 16 | b;
+}
+
+/* Despite the type, this function expects a uint64_t buffer */
+static uint64_t
+fetch_pixel_x2r10g10b10 (bits_image_t *image,
+			 int	   offset,
+			 int           line)
+{
+    uint32_t *bits = image->bits + line * image->rowstride;
+    uint32_t p = READ (image, bits + offset);
+    uint64_t r = (p >> 20) & 0x3ff;
+    uint64_t g = (p >> 10) & 0x3ff;
+    uint64_t b = p & 0x3ff;
+    
+    r = r << 6 | r >> 4;
+    g = g << 6 | g >> 4;
+    b = b << 6 | b >> 4;
+    
+    return 0xffffULL << 48 | r << 32 | g << 16 | b;
+}
+
+/* Despite the type, expects a uint64_t buffer */
+static uint64_t
+fetch_pixel_a2b10g10r10 (bits_image_t *image,
+			 int           offset,
+			 int           line)
+{
+    uint32_t *bits = image->bits + line * image->rowstride;
+    uint32_t p = READ (image, bits + offset);
+    uint64_t a = p >> 30;
+    uint64_t b = (p >> 20) & 0x3ff;
+    uint64_t g = (p >> 10) & 0x3ff;
+    uint64_t r = p & 0x3ff;
+    
+    r = r << 6 | r >> 4;
+    g = g << 6 | g >> 4;
+    b = b << 6 | b >> 4;
+    
+    a <<= 14;
+    a |= a >> 2;
+    a |= a >> 4;
+    a |= a >> 8;
+    
+    return a << 48 | r << 32 | g << 16 | b;
+}
+
+/* Despite the type, this function expects a uint64_t buffer */
+static uint64_t
+fetch_pixel_x2b10g10r10 (bits_image_t *image,
+			 int           offset,
+			 int           line)
+{
+    uint32_t *bits = image->bits + line * image->rowstride;
+    uint32_t p = READ (image, bits + offset);
+    uint64_t b = (p >> 20) & 0x3ff;
+    uint64_t g = (p >> 10) & 0x3ff;
+    uint64_t r = p & 0x3ff;
+    
+    r = r << 6 | r >> 4;
+    g = g << 6 | g >> 4;
+    b = b << 6 | b >> 4;
+    
+    return 0xffffULL << 48 | r << 32 | g << 16 | b;
+}
+
+static uint32_t
+fetch_pixel_a8r8g8b8 (bits_image_t *image,
+		      int           offset,
+		      int           line)
+{
+    uint32_t *bits = image->bits + line * image->rowstride;
+    return READ (image, (uint32_t *)bits + offset);
+}
+
+static uint32_t
+fetch_pixel_x8r8g8b8 (bits_image_t *image,
+		      int           offset,
+		      int           line)
+{
+    uint32_t *bits = image->bits + line * image->rowstride;
+
+    return READ (image, (uint32_t *)bits + offset) | 0xff000000;
+}
+
+static uint32_t
+fetch_pixel_a8b8g8r8 (bits_image_t *image,
+		      int           offset,
+		      int           line)
+{
+    uint32_t *bits = image->bits + line * image->rowstride;
+    uint32_t pixel = READ (image, (uint32_t *)bits + offset);
+    
+    return ((pixel & 0xff000000) |
+	    ((pixel >> 16) & 0xff) |
+	    (pixel & 0x0000ff00) |
+	    ((pixel & 0xff) << 16));
+}
+
+static uint32_t
+fetch_pixel_x8b8g8r8 (bits_image_t *image,
+		      int           offset,
+		      int           line)
+{
+    uint32_t *bits = image->bits + line * image->rowstride;
+    uint32_t pixel = READ (image, (uint32_t *)bits + offset);
+    
+    return ((0xff000000) |
+	    ((pixel >> 16) & 0xff) |
+	    (pixel & 0x0000ff00) |
+	    ((pixel & 0xff) << 16));
+}
+
+static uint32_t
+fetch_pixel_b8g8r8a8 (bits_image_t *image,
+		      int           offset,
+		      int           line)
+{
+    uint32_t *bits = image->bits + line * image->rowstride;
+    uint32_t pixel = READ (image, (uint32_t *)bits + offset);
+    
+    return ((pixel & 0xff000000) >> 24 |
+	    (pixel & 0x00ff0000) >> 8 |
+	    (pixel & 0x0000ff00) << 8 |
+	    (pixel & 0x000000ff) << 24);
+}
+
+static uint32_t
+fetch_pixel_b8g8r8x8 (bits_image_t *image,
+		      int           offset,
+		      int           line)
+{
+    uint32_t *bits = image->bits + line * image->rowstride;
+    uint32_t pixel = READ (image, (uint32_t *)bits + offset);
+    
+    return ((0xff000000) |
+	    (pixel & 0xff000000) >> 24 |
+	    (pixel & 0x00ff0000) >> 8 |
+	    (pixel & 0x0000ff00) << 8);
+}
+
+static uint32_t
+fetch_pixel_r8g8b8 (bits_image_t *image,
+		    int           offset,
+		    int           line)
+{
+    uint32_t *bits = image->bits + line * image->rowstride;
+    uint8_t   *pixel = ((uint8_t *) bits) + (offset * 3);
+    
+#ifdef WORDS_BIGENDIAN
+    return (0xff000000 |
+	    (READ (image, pixel + 0) << 16) |
+	    (READ (image, pixel + 1) << 8) |
+	    (READ (image, pixel + 2)));
+#else
+    return (0xff000000 |
+	    (READ (image, pixel + 2) << 16) |
+	    (READ (image, pixel + 1) << 8) |
+	    (READ (image, pixel + 0)));
+#endif
+}
+
+static uint32_t
+fetch_pixel_b8g8r8 (bits_image_t *image,
+		    int           offset,
+		    int           line)
+{
+    uint32_t *bits = image->bits + line * image->rowstride;
+    uint8_t   *pixel = ((uint8_t *) bits) + (offset * 3);
+#ifdef WORDS_BIGENDIAN
+    return (0xff000000 |
+	    (READ (image, pixel + 2) << 16) |
+	    (READ (image, pixel + 1) << 8) |
+	    (READ (image, pixel + 0)));
+#else
+    return (0xff000000 |
+	    (READ (image, pixel + 0) << 16) |
+	    (READ (image, pixel + 1) << 8) |
+	    (READ (image, pixel + 2)));
+#endif
+}
+
+static uint32_t
+fetch_pixel_r5g6b5 (bits_image_t *image,
+		    int           offset,
+		    int           line)
+{
+    uint32_t *bits = image->bits + line * image->rowstride;
+    uint32_t pixel = READ (image, (uint16_t *) bits + offset);
+    uint32_t r, g, b;
+    
+    r = ((pixel & 0xf800) | ((pixel & 0xe000) >> 5)) << 8;
+    g = ((pixel & 0x07e0) | ((pixel & 0x0600) >> 6)) << 5;
+    b = ((pixel & 0x001c) | ((pixel & 0x001f) << 5)) >> 2;
+    
+    return (0xff000000 | r | g | b);
+}
+
+static uint32_t
+fetch_pixel_b5g6r5 (bits_image_t *image,
+		    int           offset,
+		    int           line)
+{
+    uint32_t r, g, b;
+    uint32_t *bits = image->bits + line * image->rowstride;
+    uint32_t pixel = READ (image, (uint16_t *) bits + offset);
+    
+    b = ((pixel & 0xf800) | ((pixel & 0xe000) >> 5)) >> 8;
+    g = ((pixel & 0x07e0) | ((pixel & 0x0600) >> 6)) << 5;
+    r = ((pixel & 0x001c) | ((pixel & 0x001f) << 5)) << 14;
+    
+    return (0xff000000 | r | g | b);
+}
+
+static uint32_t
+fetch_pixel_a1r5g5b5 (bits_image_t *image,
+		      int           offset,
+		      int           line)
+{
+    uint32_t *bits = image->bits + line * image->rowstride;
+    uint32_t pixel = READ (image, (uint16_t *) bits + offset);
+    uint32_t a, r, g, b;
+    
+    a = (uint32_t) ((uint8_t) (0 - ((pixel & 0x8000) >> 15))) << 24;
+    r = ((pixel & 0x7c00) | ((pixel & 0x7000) >> 5)) << 9;
+    g = ((pixel & 0x03e0) | ((pixel & 0x0380) >> 5)) << 6;
+    b = ((pixel & 0x001c) | ((pixel & 0x001f) << 5)) >> 2;
+    
+    return (a | r | g | b);
+}
+
+static uint32_t
+fetch_pixel_x1r5g5b5 (bits_image_t *image,
+		      int           offset,
+		      int           line)
+{
+    uint32_t *bits = image->bits + line * image->rowstride;
+    uint32_t pixel = READ (image, (uint16_t *) bits + offset);
+    uint32_t r, g, b;
+    
+    r = ((pixel & 0x7c00) | ((pixel & 0x7000) >> 5)) << 9;
+    g = ((pixel & 0x03e0) | ((pixel & 0x0380) >> 5)) << 6;
+    b = ((pixel & 0x001c) | ((pixel & 0x001f) << 5)) >> 2;
+    
+    return (0xff000000 | r | g | b);
+}
+
+static uint32_t
+fetch_pixel_a1b5g5r5 (bits_image_t *image,
+		      int           offset,
+		      int           line)
+{
+    uint32_t *bits = image->bits + line * image->rowstride;
+    uint32_t pixel = READ (image, (uint16_t *) bits + offset);
+    uint32_t a, r, g, b;
+    
+    a = (uint32_t) ((uint8_t) (0 - ((pixel & 0x8000) >> 15))) << 24;
+    b = ((pixel & 0x7c00) | ((pixel & 0x7000) >> 5)) >> 7;
+    g = ((pixel & 0x03e0) | ((pixel & 0x0380) >> 5)) << 6;
+    r = ((pixel & 0x001c) | ((pixel & 0x001f) << 5)) << 14;
+    
+    return (a | r | g | b);
+}
+
+static uint32_t
+fetch_pixel_x1b5g5r5 (bits_image_t *image,
+		      int           offset,
+		      int           line)
+{
+    uint32_t *bits = image->bits + line * image->rowstride;
+    uint32_t pixel = READ (image, (uint16_t *) bits + offset);
+    uint32_t r, g, b;
+    
+    b = ((pixel & 0x7c00) | ((pixel & 0x7000) >> 5)) >> 7;
+    g = ((pixel & 0x03e0) | ((pixel & 0x0380) >> 5)) << 6;
+    r = ((pixel & 0x001c) | ((pixel & 0x001f) << 5)) << 14;
+    
+    return (0xff000000 | r | g | b);
+}
+
+static uint32_t
+fetch_pixel_a4r4g4b4 (bits_image_t *image,
+		      int           offset,
+		      int           line)
+{
+    uint32_t *bits = image->bits + line * image->rowstride;
+    uint32_t pixel = READ (image, (uint16_t *) bits + offset);
+    uint32_t a, r, g, b;
+    
+    a = ((pixel & 0xf000) | ((pixel & 0xf000) >> 4)) << 16;
+    r = ((pixel & 0x0f00) | ((pixel & 0x0f00) >> 4)) << 12;
+    g = ((pixel & 0x00f0) | ((pixel & 0x00f0) >> 4)) << 8;
+    b = ((pixel & 0x000f) | ((pixel & 0x000f) << 4));
+    
+    return (a | r | g | b);
+}
+
+static uint32_t
+fetch_pixel_x4r4g4b4 (bits_image_t *image,
+		      int           offset,
+		      int           line)
+{
+    uint32_t *bits = image->bits + line * image->rowstride;
+    uint32_t pixel = READ (image, (uint16_t *) bits + offset);
+    uint32_t r, g, b;
+    
+    r = ((pixel & 0x0f00) | ((pixel & 0x0f00) >> 4)) << 12;
+    g = ((pixel & 0x00f0) | ((pixel & 0x00f0) >> 4)) << 8;
+    b = ((pixel & 0x000f) | ((pixel & 0x000f) << 4));
+    
+    return (0xff000000 | r | g | b);
+}
+
+static uint32_t
+fetch_pixel_a4b4g4r4 (bits_image_t *image,
+		      int           offset,
+		      int           line)
+{
+    uint32_t *bits = image->bits + line * image->rowstride;
+    uint32_t pixel = READ (image, (uint16_t *) bits + offset);
+    uint32_t a, r, g, b;
+    
+    a = ((pixel & 0xf000) | ((pixel & 0xf000) >> 4)) << 16;
+    b = ((pixel & 0x0f00) | ((pixel & 0x0f00) >> 4)) >> 4;
+    g = ((pixel & 0x00f0) | ((pixel & 0x00f0) >> 4)) << 8;
+    r = ((pixel & 0x000f) | ((pixel & 0x000f) << 4)) << 16;
+    
+    return (a | r | g | b);
+}
+
+static uint32_t
+fetch_pixel_x4b4g4r4 (bits_image_t *image,
+		      int           offset,
+		      int           line)
+{
+    uint32_t *bits = image->bits + line * image->rowstride;
+    uint32_t pixel = READ (image, (uint16_t *) bits + offset);
+    uint32_t r, g, b;
+    
+    b = ((pixel & 0x0f00) | ((pixel & 0x0f00) >> 4)) >> 4;
+    g = ((pixel & 0x00f0) | ((pixel & 0x00f0) >> 4)) << 8;
+    r = ((pixel & 0x000f) | ((pixel & 0x000f) << 4)) << 16;
+    
+    return (0xff000000 | r | g | b);
+}
+
+static uint32_t
+fetch_pixel_a8 (bits_image_t *image,
+		int           offset,
+		int           line)
+{
+    uint32_t *bits = image->bits + line * image->rowstride;
+    uint32_t pixel = READ (image, (uint8_t *) bits + offset);
+    
+    return pixel << 24;
+}
+
+static uint32_t
+fetch_pixel_r3g3b2 (bits_image_t *image,
+		    int           offset,
+		    int           line)
+{
+    uint32_t *bits = image->bits + line * image->rowstride;
+    uint32_t pixel = READ (image, (uint8_t *) bits + offset);
+    uint32_t r, g, b;
+    
+    r = ((pixel & 0xe0) |
+	 ((pixel & 0xe0) >> 3) |
+	 ((pixel & 0xc0) >> 6)) << 16;
+    
+    g = ((pixel & 0x1c) |
+	 ((pixel & 0x18) >> 3) |
+	 ((pixel & 0x1c) << 3)) << 8;
+    
+    b = (((pixel & 0x03)     ) |
+	 ((pixel & 0x03) << 2) |
+	 ((pixel & 0x03) << 4) |
+	 ((pixel & 0x03) << 6));
+    
+    return (0xff000000 | r | g | b);
+}
+
+static uint32_t
+fetch_pixel_b2g3r3 (bits_image_t *image,
+		    int           offset,
+		    int           line)
+{
+    uint32_t *bits = image->bits + line * image->rowstride;
+    uint32_t p = READ (image, (uint8_t *) bits + offset);
+    uint32_t r, g, b;
+
+    b  = p & 0xc0;
+    b |= b >> 2;
+    b |= b >> 4;
+    b &= 0xff;
+
+    g  = (p & 0x38) << 10;
+    g |= g >> 3;
+    g |= g >> 6;
+    g &= 0xff00;
+
+    r  = (p & 0x7) << 21;
+    r |= r >> 3;
+    r |= r >> 6;
+    r &= 0xff0000;
+
+    return 0xff000000 | r | g | b;
+}
+
+static uint32_t
+fetch_pixel_a2r2g2b2 (bits_image_t *image,
+		      int           offset,
+		      int           line)
+{
+    uint32_t *bits = image->bits + line * image->rowstride;
+    uint32_t pixel = READ (image, (uint8_t *) bits + offset);
+    uint32_t a, r, g, b;
+    
+    a = ((pixel & 0xc0) * 0x55) << 18;
+    r = ((pixel & 0x30) * 0x55) << 12;
+    g = ((pixel & 0x0c) * 0x55) << 6;
+    b = ((pixel & 0x03) * 0x55);
+    
+    return a | r | g | b;
+}
+
+static uint32_t
+fetch_pixel_a2b2g2r2 (bits_image_t *image,
+		      int           offset,
+		      int           line)
+{
+    uint32_t *bits = image->bits + line * image->rowstride;
+    uint32_t pixel = READ (image, (uint8_t *) bits + offset);
+    uint32_t a, r, g, b;
+    
+    a = ((pixel & 0xc0) * 0x55) << 18;
+    b = ((pixel & 0x30) * 0x55) >> 4;
+    g = ((pixel & 0x0c) * 0x55) << 6;
+    r = ((pixel & 0x03) * 0x55) << 16;
+    
+    return a | r | g | b;
+}
+
+static uint32_t
+fetch_pixel_c8 (bits_image_t *image,
+		int           offset,
+		int           line)
+{
+    uint32_t *bits = image->bits + line * image->rowstride;
+    uint32_t pixel = READ (image, (uint8_t *) bits + offset);
+    const pixman_indexed_t * indexed = image->indexed;
+    
+    return indexed->rgba[pixel];
+}
+
+static uint32_t
+fetch_pixel_x4a4 (bits_image_t *image,
+		  int           offset,
+		  int           line)
+{
+    uint32_t *bits = image->bits + line * image->rowstride;
+    uint32_t pixel = READ (image, (uint8_t *) bits + offset);
+    
+    return ((pixel & 0xf) | ((pixel & 0xf) << 4)) << 24;
+}
+
+static uint32_t
+fetch_pixel_a4 (bits_image_t *image,
+		int           offset,
+		int           line)
+{
+    uint32_t *bits = image->bits + line * image->rowstride;
+    uint32_t pixel = FETCH_4 (image, bits, offset);
+    
+    pixel |= pixel << 4;
+    return pixel << 24;
+}
+
+static uint32_t
+fetch_pixel_r1g2b1 (bits_image_t *image,
+		    int           offset,
+		    int           line)
+{
+    uint32_t *bits = image->bits + line * image->rowstride;
+    uint32_t pixel = FETCH_4 (image, bits, offset);
+    uint32_t r, g, b;
+    
+    r = ((pixel & 0x8) * 0xff) << 13;
+    g = ((pixel & 0x6) * 0x55) << 7;
+    b = ((pixel & 0x1) * 0xff);
+    
+    return 0xff000000 | r | g | b;
+}
+
+static uint32_t
+fetch_pixel_b1g2r1 (bits_image_t *image,
+		    int           offset,
+		    int           line)
+{
+    uint32_t *bits = image->bits + line * image->rowstride;
+    uint32_t pixel = FETCH_4 (image, bits, offset);
+    uint32_t r, g, b;
+    
+    b = ((pixel & 0x8) * 0xff) >> 3;
+    g = ((pixel & 0x6) * 0x55) << 7;
+    r = ((pixel & 0x1) * 0xff) << 16;
+    
+    return 0xff000000 | r | g | b;
+}
+
+static uint32_t
+fetch_pixel_a1r1g1b1 (bits_image_t *image,
+		      int           offset,
+		      int           line)
+{
+    uint32_t *bits = image->bits + line * image->rowstride;
+    uint32_t pixel = FETCH_4 (image, bits, offset);
+    uint32_t a, r, g, b;
+
+    a = ((pixel & 0x8) * 0xff) << 21;
+    r = ((pixel & 0x4) * 0xff) << 14;
+    g = ((pixel & 0x2) * 0xff) << 7;
+    b = ((pixel & 0x1) * 0xff);
+
+    return a | r | g | b;
+}
+
+static uint32_t
+fetch_pixel_a1b1g1r1 (bits_image_t *image,
+		      int           offset,
+		      int           line)
+{
+    uint32_t *bits = image->bits + line * image->rowstride;
+    uint32_t pixel = FETCH_4 (image, bits, offset);
+    uint32_t a, r, g, b;
+
+    a = ((pixel & 0x8) * 0xff) << 21;
+    b = ((pixel & 0x4) * 0xff) >> 2;
+    g = ((pixel & 0x2) * 0xff) << 7;
+    r = ((pixel & 0x1) * 0xff) << 16;
+
+    return a | r | g | b;
+}
+
+static uint32_t
+fetch_pixel_c4 (bits_image_t *image,
+		int           offset,
+		int           line)
+{
+    uint32_t *bits = image->bits + line * image->rowstride;
+    uint32_t pixel = FETCH_4 (image, bits, offset);
+    const pixman_indexed_t * indexed = image->indexed;
+
+    return indexed->rgba[pixel];
+}
+
+static uint32_t
+fetch_pixel_a1 (bits_image_t *image,
+		int           offset,
+		int           line)
+{
+    uint32_t *bits = image->bits + line * image->rowstride;
+    uint32_t pixel = READ (image, bits + (offset >> 5));
+    uint32_t a;
+    
+#ifdef WORDS_BIGENDIAN
+    a = pixel >> (0x1f - (offset & 0x1f));
+#else
+    a = pixel >> (offset & 0x1f);
+#endif
+    a = a & 1;
+    a |= a << 1;
+    a |= a << 2;
+    a |= a << 4;
+    
+    return a << 24;
+}
+
+static uint32_t
+fetch_pixel_g1 (bits_image_t *image,
+		int           offset,
+		int           line)
+{
+    uint32_t *bits = image->bits + line * image->rowstride;
+    uint32_t pixel = READ (image, bits + (offset >> 5));
+    const pixman_indexed_t * indexed = image->indexed;
+    uint32_t a;
+    
+#ifdef WORDS_BIGENDIAN
+    a = pixel >> (0x1f - (offset & 0x1f));
+#else
+    a = pixel >> (offset & 0x1f);
+#endif
+    a = a & 1;
+    
+    return indexed->rgba[a];
+}
+
+static uint32_t
+fetch_pixel_yuy2 (bits_image_t *image,
+		  int           offset,
+		  int           line)
+{
+    const uint32_t *bits = image->bits + image->rowstride * line;
+    
+    int16_t y, u, v;
+    int32_t r, g, b;
+    
+    y = ((uint8_t *) bits)[offset << 1] - 16;
+    u = ((uint8_t *) bits)[((offset << 1) & - 4) + 1] - 128;
+    v = ((uint8_t *) bits)[((offset << 1) & - 4) + 3] - 128;
+    
+    /* R = 1.164(Y - 16) + 1.596(V - 128) */
+    r = 0x012b27 * y + 0x019a2e * v;
+    
+    /* G = 1.164(Y - 16) - 0.813(V - 128) - 0.391(U - 128) */
+    g = 0x012b27 * y - 0x00d0f2 * v - 0x00647e * u;
+    
+    /* B = 1.164(Y - 16) + 2.018(U - 128) */
+    b = 0x012b27 * y + 0x0206a2 * u;
+    
+    return 0xff000000 |
+	(r >= 0 ? r < 0x1000000 ? r         & 0xff0000 : 0xff0000 : 0) |
+	(g >= 0 ? g < 0x1000000 ? (g >> 8)  & 0x00ff00 : 0x00ff00 : 0) |
+	(b >= 0 ? b < 0x1000000 ? (b >> 16) & 0x0000ff : 0x0000ff : 0);
+}
+
+static uint32_t
+fetch_pixel_yv12 (bits_image_t *image,
+		  int           offset,
+		  int           line)
+{
+    YV12_SETUP (image);
+    int16_t y = YV12_Y (line)[offset] - 16;
+    int16_t u = YV12_U (line)[offset >> 1] - 128;
+    int16_t v = YV12_V (line)[offset >> 1] - 128;
+    int32_t r, g, b;
+    
+    /* R = 1.164(Y - 16) + 1.596(V - 128) */
+    r = 0x012b27 * y + 0x019a2e * v;
+    
+    /* G = 1.164(Y - 16) - 0.813(V - 128) - 0.391(U - 128) */
+    g = 0x012b27 * y - 0x00d0f2 * v - 0x00647e * u;
+    
+    /* B = 1.164(Y - 16) + 2.018(U - 128) */
+    b = 0x012b27 * y + 0x0206a2 * u;
+    
+    return 0xff000000 |
+	(r >= 0 ? r < 0x1000000 ? r         & 0xff0000 : 0xff0000 : 0) |
+	(g >= 0 ? g < 0x1000000 ? (g >> 8)  & 0x00ff00 : 0x00ff00 : 0) |
+	(b >= 0 ? b < 0x1000000 ? (b >> 16) & 0x0000ff : 0x0000ff : 0);
+}
+
+/*********************************** Store ************************************/
+
+#define SPLIT_A(v)              \
+    uint32_t a = ((v) >> 24),   \
+	r = ((v) >> 16) & 0xff, \
+	g = ((v) >> 8) & 0xff,  \
+	b = (v) & 0xff
+
+#define SPLIT(v)                     \
+    uint32_t r = ((v) >> 16) & 0xff, \
+	g = ((v) >> 8) & 0xff,       \
+	b = (v) & 0xff
+
+static void
+store_scanline_a2r10g10b10 (bits_image_t *  image,
+                            int             x,
+                            int             y,
+                            int             width,
+                            const uint32_t *v)
+{
+    uint32_t *bits = image->bits + image->rowstride * y;
+    uint32_t *pixel = bits + x;
+    uint64_t *values = (uint64_t *)v;
+    int i;
+    
+    for (i = 0; i < width; ++i)
+    {
+	WRITE (image, pixel++,
+	       ((values[i] >> 32) & 0xc0000000) |
+	       ((values[i] >> 18) & 0x3ff00000) |
+	       ((values[i] >> 12) & 0xffc00) | 
+	       ((values[i] >> 6) & 0x3ff));    
+    }
+}
+
+static void
+store_scanline_x2r10g10b10 (bits_image_t *  image,
+                            int             x,
+                            int             y,
+                            int             width,
+                            const uint32_t *v)
+{
+    uint32_t *bits = image->bits + image->rowstride * y;
+    uint64_t *values = (uint64_t *)v;
+    uint32_t *pixel = bits + x;
+    int i;
+    
+    for (i = 0; i < width; ++i)
+    {
+	WRITE (image, pixel++,
+	       ((values[i] >> 18) & 0x3ff00000) | 
+	       ((values[i] >> 12) & 0xffc00) |
+	       ((values[i] >> 6) & 0x3ff));
+    }
+}
+
+static void
+store_scanline_a2b10g10r10 (bits_image_t *  image,
+                            int             x,
+                            int             y,
+                            int             width,
+                            const uint32_t *v)
+{
+    uint32_t *bits = image->bits + image->rowstride * y;
+    uint32_t *pixel = bits + x;
+    uint64_t *values = (uint64_t *)v;
+    int i;
+    
+    for (i = 0; i < width; ++i)
+    {
+	WRITE (image, pixel++,
+	       ((values[i] >> 32) & 0xc0000000) |
+	       ((values[i] >> 38) & 0x3ff) |
+	       ((values[i] >> 12) & 0xffc00) |
+	       ((values[i] << 14) & 0x3ff00000));
+    }
+}
+
+static void
+store_scanline_x2b10g10r10 (bits_image_t *  image,
+                            int             x,
+                            int             y,
+                            int             width,
+                            const uint32_t *v)
+{
+    uint32_t *bits = image->bits + image->rowstride * y;
+    uint64_t *values = (uint64_t *)v;
+    uint32_t *pixel = bits + x;
+    int i;
+    
+    for (i = 0; i < width; ++i)
+    {
+	WRITE (image, pixel++,
+	       ((values[i] >> 38) & 0x3ff) |
+	       ((values[i] >> 12) & 0xffc00) |
+	       ((values[i] << 14) & 0x3ff00000));
+    }
+}
+
+static void
+store_scanline_a8r8g8b8 (bits_image_t *  image,
+                         int             x,
+                         int             y,
+                         int             width,
+                         const uint32_t *values)
+{
+    uint32_t *bits = image->bits + image->rowstride * y;
+    
+    MEMCPY_WRAPPED (image, ((uint32_t *)bits) + x, values,
+                    width * sizeof(uint32_t));
+}
+
+static void
+store_scanline_x8r8g8b8 (bits_image_t *  image,
+                         int             x,
+                         int             y,
+                         int             width,
+                         const uint32_t *values)
+{
+    uint32_t *bits = image->bits + image->rowstride * y;
+    uint32_t *pixel = (uint32_t *)bits + x;
+    int i;
+    
+    for (i = 0; i < width; ++i)
+	WRITE (image, pixel++, values[i] & 0xffffff);
+}
+
+static void
+store_scanline_a8b8g8r8 (bits_image_t *  image,
+                         int             x,
+                         int             y,
+                         int             width,
+                         const uint32_t *values)
+{
+    uint32_t *bits = image->bits + image->rowstride * y;
+    uint32_t *pixel = (uint32_t *)bits + x;
+    int i;
+    
+    for (i = 0; i < width; ++i)
+    {
+	WRITE (image, pixel++,
+	       (values[i] & 0xff00ff00)         |
+	       ((values[i] >> 16) & 0xff)       |
+	       ((values[i] & 0xff) << 16));
+    }
+}
+
+static void
+store_scanline_x8b8g8r8 (bits_image_t *  image,
+                         int             x,
+                         int             y,
+                         int             width,
+                         const uint32_t *values)
+{
+    uint32_t *bits = image->bits + image->rowstride * y;
+    uint32_t *pixel = (uint32_t *)bits + x;
+    int i;
+    
+    for (i = 0; i < width; ++i)
+    {
+	WRITE (image, pixel++,
+	       (values[i] & 0x0000ff00)         |
+	       ((values[i] >> 16) & 0xff)       |
+	       ((values[i] & 0xff) << 16));
+    }
+}
+
+static void
+store_scanline_b8g8r8a8 (bits_image_t *  image,
+                         int             x,
+                         int             y,
+                         int             width,
+                         const uint32_t *values)
+{
+    uint32_t *bits = image->bits + image->rowstride * y;
+    uint32_t *pixel = (uint32_t *)bits + x;
+    int i;
+    
+    for (i = 0; i < width; ++i)
+    {
+	WRITE (image, pixel++,
+	       ((values[i] >> 24) & 0x000000ff) |
+	       ((values[i] >>  8) & 0x0000ff00) |
+	       ((values[i] <<  8) & 0x00ff0000) |
+	       ((values[i] << 24) & 0xff000000));
+    }
+}
+
+static void
+store_scanline_b8g8r8x8 (bits_image_t *  image,
+                         int             x,
+                         int             y,
+                         int             width,
+                         const uint32_t *values)
+{
+    uint32_t *bits = image->bits + image->rowstride * y;
+    uint32_t *pixel = (uint32_t *)bits + x;
+    int i;
+    
+    for (i = 0; i < width; ++i)
+    {
+	WRITE (image, pixel++,
+	       ((values[i] >>  8) & 0x0000ff00) |
+	       ((values[i] <<  8) & 0x00ff0000) |
+	       ((values[i] << 24) & 0xff000000));
+    }
+}
+
+static void
+store_scanline_r8g8b8 (bits_image_t *  image,
+                       int             x,
+                       int             y,
+                       int             width,
+                       const uint32_t *values)
+{
+    uint32_t *bits = image->bits + image->rowstride * y;
+    uint8_t *pixel = ((uint8_t *) bits) + 3 * x;
+    int i;
+    
+    for (i = 0; i < width; ++i)
+    {
+	uint32_t val = values[i];
+	
+#ifdef WORDS_BIGENDIAN
+	WRITE (image, pixel++, (val & 0x00ff0000) >> 16);
+	WRITE (image, pixel++, (val & 0x0000ff00) >>  8);
+	WRITE (image, pixel++, (val & 0x000000ff) >>  0);
+#else
+	WRITE (image, pixel++, (val & 0x000000ff) >>  0);
+	WRITE (image, pixel++, (val & 0x0000ff00) >>  8);
+	WRITE (image, pixel++, (val & 0x00ff0000) >> 16);
+#endif
+    }
+}
+
+static void
+store_scanline_b8g8r8 (bits_image_t *  image,
+                       int             x,
+                       int             y,
+                       int             width,
+                       const uint32_t *values)
+{
+    uint32_t *bits = image->bits + image->rowstride * y;
+    uint8_t *pixel = ((uint8_t *) bits) + 3 * x;
+    int i;
+    
+    for (i = 0; i < width; ++i)
+    {
+	uint32_t val = values[i];
+	
+#ifdef WORDS_BIGENDIAN
+	WRITE (image, pixel++, (val & 0x000000ff) >>  0);
+	WRITE (image, pixel++, (val & 0x0000ff00) >>  8);
+	WRITE (image, pixel++, (val & 0x00ff0000) >> 16);
+#else
+	WRITE (image, pixel++, (val & 0x00ff0000) >> 16);
+	WRITE (image, pixel++, (val & 0x0000ff00) >>  8);
+	WRITE (image, pixel++, (val & 0x000000ff) >>  0);
+#endif
+    }
+}
+
+static void
+store_scanline_r5g6b5 (bits_image_t *  image,
+                       int             x,
+                       int             y,
+                       int             width,
+                       const uint32_t *values)
+{
+    uint32_t *bits = image->bits + image->rowstride * y;
+    uint16_t *pixel = ((uint16_t *) bits) + x;
+    int i;
+    
+    for (i = 0; i < width; ++i)
+    {
+	uint32_t s = values[i];
+	
+	WRITE (image, pixel++,
+	       ((s >> 3) & 0x001f) |
+	       ((s >> 5) & 0x07e0) |
+	       ((s >> 8) & 0xf800));
+    }
+}
+
+static void
+store_scanline_b5g6r5 (bits_image_t *  image,
+                       int             x,
+                       int             y,
+                       int             width,
+                       const uint32_t *values)
+{
+    uint32_t *bits = image->bits + image->rowstride * y;
+    uint16_t  *pixel = ((uint16_t *) bits) + x;
+    int i;
+    
+    for (i = 0; i < width; ++i)
+    {
+	SPLIT (values[i]);
+	
+	WRITE (image, pixel++,
+	       ((b << 8) & 0xf800) |
+	       ((g << 3) & 0x07e0) |
+	       ((r >> 3)         ));
+    }
+}
+
+static void
+store_scanline_a1r5g5b5 (bits_image_t *  image,
+                         int             x,
+                         int             y,
+                         int             width,
+                         const uint32_t *values)
+{
+    uint32_t *bits = image->bits + image->rowstride * y;
+    uint16_t  *pixel = ((uint16_t *) bits) + x;
+    int i;
+    
+    for (i = 0; i < width; ++i)
+    {
+	SPLIT_A (values[i]);
+	
+	WRITE (image, pixel++,
+	       ((a << 8) & 0x8000) |
+	       ((r << 7) & 0x7c00) |
+	       ((g << 2) & 0x03e0) |
+	       ((b >> 3)         ));
+    }
+}
+
+static void
+store_scanline_x1r5g5b5 (bits_image_t *  image,
+                         int             x,
+                         int             y,
+                         int             width,
+                         const uint32_t *values)
+{
+    uint32_t *bits = image->bits + image->rowstride * y;
+    uint16_t  *pixel = ((uint16_t *) bits) + x;
+    int i;
+    
+    for (i = 0; i < width; ++i)
+    {
+	SPLIT (values[i]);
+	
+	WRITE (image, pixel++,
+	       ((r << 7) & 0x7c00) |
+	       ((g << 2) & 0x03e0) |
+	       ((b >> 3)         ));
+    }
+}
+
+static void
+store_scanline_a1b5g5r5 (bits_image_t *  image,
+                         int             x,
+                         int             y,
+                         int             width,
+                         const uint32_t *values)
+{
+    uint32_t *bits = image->bits + image->rowstride * y;
+    uint16_t  *pixel = ((uint16_t *) bits) + x;
+    int i;
+    
+    for (i = 0; i < width; ++i)
+    {
+	SPLIT_A (values[i]);
+	
+	WRITE (image, pixel++,
+	       ((a << 8) & 0x8000) |
+	       ((b << 7) & 0x7c00) |
+	       ((g << 2) & 0x03e0) |
+	       ((r >> 3)         ));
+    }
+}
+
+static void
+store_scanline_x1b5g5r5 (bits_image_t *  image,
+                         int             x,
+                         int             y,
+                         int             width,
+                         const uint32_t *values)
+{
+    uint32_t *bits = image->bits + image->rowstride * y;
+    uint16_t  *pixel = ((uint16_t *) bits) + x;
+    int i;
+    
+    for (i = 0; i < width; ++i)
+    {
+	SPLIT (values[i]);
+	
+	WRITE (image, pixel++, ((b << 7) & 0x7c00) |
+	       ((g << 2) & 0x03e0) |
+	       ((r >> 3)         ));
+    }
+}
+
+static void
+store_scanline_a4r4g4b4 (bits_image_t *  image,
+                         int             x,
+                         int             y,
+                         int             width,
+                         const uint32_t *values)
+{
+    uint32_t *bits = image->bits + image->rowstride * y;
+    uint16_t  *pixel = ((uint16_t *) bits) + x;
+    int i;
+    
+    for (i = 0; i < width; ++i)
+    {
+	SPLIT_A (values[i]);
+	
+	WRITE (image, pixel++,
+	       ((a << 8) & 0xf000) |
+	       ((r << 4) & 0x0f00) |
+	       ((g     ) & 0x00f0) |
+	       ((b >> 4)         ));
+    }
+}
+
+static void
+store_scanline_x4r4g4b4 (bits_image_t *  image,
+                         int             x,
+                         int             y,
+                         int             width,
+                         const uint32_t *values)
+{
+    uint32_t *bits = image->bits + image->rowstride * y;
+    uint16_t  *pixel = ((uint16_t *) bits) + x;
+    int i;
+    
+    for (i = 0; i < width; ++i)
+    {
+	SPLIT (values[i]);
+	
+	WRITE (image, pixel++,
+	       ((r << 4) & 0x0f00) |
+	       ((g     ) & 0x00f0) |
+	       ((b >> 4)         ));
+    }
+}
+
+static void
+store_scanline_a4b4g4r4 (bits_image_t *  image,
+                         int             x,
+                         int             y,
+                         int             width,
+                         const uint32_t *values)
+{
+    uint32_t *bits = image->bits + image->rowstride * y;
+    uint16_t  *pixel = ((uint16_t *) bits) + x;
+    int i;
+    
+    for (i = 0; i < width; ++i)
+    {
+	SPLIT_A (values[i]);
+	WRITE (image, pixel++, ((a << 8) & 0xf000) |
+	       ((b << 4) & 0x0f00) |
+	       ((g     ) & 0x00f0) |
+	       ((r >> 4)         ));
+    }
+}
+
+static void
+store_scanline_x4b4g4r4 (bits_image_t *  image,
+                         int             x,
+                         int             y,
+                         int             width,
+                         const uint32_t *values)
+{
+    uint32_t *bits = image->bits + image->rowstride * y;
+    uint16_t  *pixel = ((uint16_t *) bits) + x;
+    int i;
+    
+    for (i = 0; i < width; ++i)
+    {
+	SPLIT (values[i]);
+	
+	WRITE (image, pixel++,
+	       ((b << 4) & 0x0f00) |
+	       ((g     ) & 0x00f0) |
+	       ((r >> 4)         ));
+    }
+}
+
+static void
+store_scanline_a8 (bits_image_t *  image,
+                   int             x,
+                   int             y,
+                   int             width,
+                   const uint32_t *values)
+{
+    uint32_t *bits = image->bits + image->rowstride * y;
+    uint8_t   *pixel = ((uint8_t *) bits) + x;
+    int i;
+    
+    for (i = 0; i < width; ++i)
+    {
+	WRITE (image, pixel++, values[i] >> 24);
+    }
+}
+
+static void
+store_scanline_r3g3b2 (bits_image_t *  image,
+                       int             x,
+                       int             y,
+                       int             width,
+                       const uint32_t *values)
+{
+    uint32_t *bits = image->bits + image->rowstride * y;
+    uint8_t   *pixel = ((uint8_t *) bits) + x;
+    int i;
+    
+    for (i = 0; i < width; ++i)
+    {
+	SPLIT (values[i]);
+	
+	WRITE (image, pixel++,
+	       ((r     ) & 0xe0) |
+	       ((g >> 3) & 0x1c) |
+	       ((b >> 6)       ));
+    }
+}
+
+static void
+store_scanline_b2g3r3 (bits_image_t *  image,
+                       int             x,
+                       int             y,
+                       int             width,
+                       const uint32_t *values)
+{
+    uint32_t *bits = image->bits + image->rowstride * y;
+    uint8_t   *pixel = ((uint8_t *) bits) + x;
+    int i;
+    
+    for (i = 0; i < width; ++i)
+    {
+	SPLIT (values[i]);
+	
+	WRITE (image, pixel++,
+	       ((b     ) & 0xc0) |
+	       ((g >> 2) & 0x38) |
+	       ((r >> 5)       ));
+    }
+}
+
+static void
+store_scanline_a2r2g2b2 (bits_image_t *  image,
+                         int             x,
+                         int             y,
+                         int             width,
+                         const uint32_t *values)
+{
+    uint32_t *bits = image->bits + image->rowstride * y;
+    uint8_t   *pixel = ((uint8_t *) bits) + x;
+    int i;
+    
+    for (i = 0; i < width; ++i)
+    {
+	SPLIT_A (values[i]);
+	
+	WRITE (image, pixel++,
+	       ((a     ) & 0xc0) |
+	       ((r >> 2) & 0x30) |
+	       ((g >> 4) & 0x0c) |
+	       ((b >> 6)       ));
+    }
+}
+
+static void
+store_scanline_a2b2g2r2 (bits_image_t *  image,
+                         int             x,
+                         int             y,
+                         int             width,
+                         const uint32_t *values)
+{
+    uint32_t *bits = image->bits + image->rowstride * y;
+    uint8_t   *pixel = ((uint8_t *) bits) + x;
+    int i;
+    
+    for (i = 0; i < width; ++i)
+    {
+	SPLIT_A (values[i]);
+	
+	*(pixel++) =
+	    ((a     ) & 0xc0) |
+	    ((b >> 2) & 0x30) |
+	    ((g >> 4) & 0x0c) |
+	    ((r >> 6)       );
+    }
+}
+
+static void
+store_scanline_c8 (bits_image_t *  image,
+                   int             x,
+                   int             y,
+                   int             width,
+                   const uint32_t *values)
+{
+    uint32_t *bits = image->bits + image->rowstride * y;
+    uint8_t *pixel = ((uint8_t *) bits) + x;
+    const pixman_indexed_t *indexed = image->indexed;
+    int i;
+    
+    for (i = 0; i < width; ++i)
+	WRITE (image, pixel++, RGB24_TO_ENTRY (indexed,values[i]));
+}
+
+static void
+store_scanline_x4a4 (bits_image_t *  image,
+                     int             x,
+                     int             y,
+                     int             width,
+                     const uint32_t *values)
+{
+    uint32_t *bits = image->bits + image->rowstride * y;
+    uint8_t   *pixel = ((uint8_t *) bits) + x;
+    int i;
+
+    for (i = 0; i < width; ++i)
+	WRITE (image, pixel++, values[i] >> 28);
+}
+
+#define STORE_8(img,l,o,v)  (WRITE (img, (uint8_t *)(l) + ((o) >> 3), (v)))
+#ifdef WORDS_BIGENDIAN
+
+#define STORE_4(img,l,o,v)						\
+    do									\
+    {									\
+	int bo = 4 * (o);						\
+	int v4 = (v) & 0x0f;						\
+									\
+	STORE_8 (img, l, bo, (						\
+		     bo & 4 ?						\
+		     (FETCH_8 (img, l, bo) & 0xf0) | (v4) :		\
+		     (FETCH_8 (img, l, bo) & 0x0f) | (v4 << 4)));	\
+    } while (0)
+#else
+
+#define STORE_4(img,l,o,v)						\
+    do									\
+    {									\
+	int bo = 4 * (o);						\
+	int v4 = (v) & 0x0f;						\
+									\
+	STORE_8 (img, l, bo, (						\
+		     bo & 4 ?						\
+		     (FETCH_8 (img, l, bo) & 0x0f) | (v4 << 4) :	\
+		     (FETCH_8 (img, l, bo) & 0xf0) | (v4)));		\
+    } while (0)
+#endif
+
+static void
+store_scanline_a4 (bits_image_t *  image,
+                   int             x,
+                   int             y,
+                   int             width,
+                   const uint32_t *values)
+{
+    uint32_t *bits = image->bits + image->rowstride * y;
+    int i;
+
+    for (i = 0; i < width; ++i)
+	STORE_4 (image, bits, i + x, values[i] >> 28);
+}
+
+static void
+store_scanline_r1g2b1 (bits_image_t *  image,
+                       int             x,
+                       int             y,
+                       int             width,
+                       const uint32_t *values)
+{
+    uint32_t *bits = image->bits + image->rowstride * y;
+    int i;
+
+    for (i = 0; i < width; ++i)
+    {
+	uint32_t pixel;
+
+	SPLIT (values[i]);
+	pixel = (((r >> 4) & 0x8) |
+	         ((g >> 5) & 0x6) |
+	         ((b >> 7)      ));
+	STORE_4 (image, bits, i + x, pixel);
+    }
+}
+
+static void
+store_scanline_b1g2r1 (bits_image_t *  image,
+                       int             x,
+                       int             y,
+                       int             width,
+                       const uint32_t *values)
+{
+    uint32_t *bits = image->bits + image->rowstride * y;
+    int i;
+
+    for (i = 0; i < width; ++i)
+    {
+	uint32_t pixel;
+
+	SPLIT (values[i]);
+	pixel = (((b >> 4) & 0x8) |
+	         ((g >> 5) & 0x6) |
+	         ((r >> 7)      ));
+	STORE_4 (image, bits, i + x, pixel);
+    }
+}
+
+static void
+store_scanline_a1r1g1b1 (bits_image_t *  image,
+                         int             x,
+                         int             y,
+                         int             width,
+                         const uint32_t *values)
+{
+    uint32_t *bits = image->bits + image->rowstride * y;
+    int i;
+
+    for (i = 0; i < width; ++i)
+    {
+	uint32_t pixel;
+
+	SPLIT_A (values[i]);
+	pixel = (((a >> 4) & 0x8) |
+	         ((r >> 5) & 0x4) |
+	         ((g >> 6) & 0x2) |
+	         ((b >> 7)      ));
+
+	STORE_4 (image, bits, i + x, pixel);
+    }
+}
+
+static void
+store_scanline_a1b1g1r1 (bits_image_t *  image,
+                         int             x,
+                         int             y,
+                         int             width,
+                         const uint32_t *values)
+{
+    uint32_t *bits = image->bits + image->rowstride * y;
+    int i;
+
+    for (i = 0; i < width; ++i)
+    {
+	uint32_t pixel;
+
+	SPLIT_A (values[i]);
+	pixel = (((a >> 4) & 0x8) |
+	         ((b >> 5) & 0x4) |
+	         ((g >> 6) & 0x2) |
+	         ((r >> 7)      ));
+
+	STORE_4 (image, bits, i + x, pixel);
+    }
+}
+
+static void
+store_scanline_c4 (bits_image_t *  image,
+                   int             x,
+                   int             y,
+                   int             width,
+                   const uint32_t *values)
+{
+    uint32_t *bits = image->bits + image->rowstride * y;
+    const pixman_indexed_t *indexed = image->indexed;
+    int i;
+    
+    for (i = 0; i < width; ++i)
+    {
+	uint32_t pixel;
+	
+	pixel = RGB24_TO_ENTRY (indexed, values[i]);
+	STORE_4 (image, bits, i + x, pixel);
+    }
+}
+
+static void
+store_scanline_a1 (bits_image_t *  image,
+                   int             x,
+                   int             y,
+                   int             width,
+                   const uint32_t *values)
+{
+    uint32_t *bits = image->bits + image->rowstride * y;
+    int i;
+    
+    for (i = 0; i < width; ++i)
+    {
+	uint32_t  *pixel = ((uint32_t *) bits) + ((i + x) >> 5);
+	uint32_t mask, v;
+	
+#ifdef WORDS_BIGENDIAN
+	mask = 1 << (0x1f - ((i + x) & 0x1f));
+#else
+	mask = 1 << ((i + x) & 0x1f);
+#endif
+	v = values[i] & 0x80000000 ? mask : 0;
+	
+	WRITE (image, pixel, (READ (image, pixel) & ~mask) | v);
+    }
+}
+
+static void
+store_scanline_g1 (bits_image_t *  image,
+                   int             x,
+                   int             y,
+                   int             width,
+                   const uint32_t *values)
+{
+    uint32_t *bits = image->bits + image->rowstride * y;
+    const pixman_indexed_t *indexed = image->indexed;
+    int i;
+    
+    for (i = 0; i < width; ++i)
+    {
+	uint32_t  *pixel = ((uint32_t *) bits) + ((i + x) >> 5);
+	uint32_t mask, v;
+	
+#ifdef WORDS_BIGENDIAN
+	mask = 1 << (0x1f - ((i + x) & 0x1f));
+#else
+	mask = 1 << ((i + x) & 0x1f);
+#endif
+	v = RGB24_TO_ENTRY_Y (indexed, values[i]) & 0x1 ? mask : 0;
+	
+	WRITE (image, pixel, (READ (image, pixel) & ~mask) | v);
+    }
+}
+
+/*
+ * Contracts a 64bpp image to 32bpp and then stores it using a regular 32-bit
+ * store proc. Despite the type, this function expects a uint64_t buffer.
+ */
+static void
+store_scanline_generic_64 (bits_image_t *  image,
+                           int             x,
+                           int             y,
+                           int             width,
+                           const uint32_t *values)
+{
+    uint32_t *argb8_pixels;
+    
+    assert (image->common.type == BITS);
+    
+    argb8_pixels = pixman_malloc_ab (width, sizeof(uint32_t));
+    if (!argb8_pixels)
+	return;
+    
+    /* Contract the scanline.  We could do this in place if values weren't
+     * const.
+     */
+    pixman_contract (argb8_pixels, (uint64_t *)values, width);
+    
+    image->store_scanline_raw_32 (image, x, y, width, argb8_pixels);
+    
+    free (argb8_pixels);
+}
+
+/* Despite the type, this function expects both buffer
+ * and mask to be uint64_t
+ */
+static void
+fetch_scanline_generic_64 (pixman_image_t *image,
+                           int             x,
+                           int             y,
+                           int             width,
+                           uint32_t *      buffer,
+                           const uint32_t *mask,
+                           uint32_t        mask_bits)
+{
+    /* Fetch the pixels into the first half of buffer and then expand them in
+     * place.
+     */
+    image->bits.fetch_scanline_raw_32 (image, x, y, width, buffer, NULL, 0);
+    
+    pixman_expand ((uint64_t *)buffer, buffer, image->bits.format, width);
+}
+
+/* Despite the type, this function expects a uint64_t *buffer */
+static uint64_t
+fetch_pixel_generic_64 (bits_image_t *image,
+			int	      offset,
+			int           line)
+{
+    uint32_t pixel32 = image->fetch_pixel_raw_32 (image, offset, line);
+    uint64_t result;
+    
+    pixman_expand ((uint64_t *)&result, &pixel32, image->format, 1);
+
+    return result;
+}
+
+/*
+ * XXX: The transformed fetch path only works at 32-bpp so far.  When all
+ * paths have wide versions, this can be removed.
+ *
+ * WARNING: This function loses precision!
+ */
+static uint32_t
+fetch_pixel_generic_lossy_32 (bits_image_t *image,
+			      int           offset,
+			      int           line)
+{
+    uint64_t pixel64 = image->fetch_pixel_raw_64 (image, offset, line);
+    uint32_t result;
+    
+    pixman_contract (&result, &pixel64, 1);
+
+    return result;
+}
+
+typedef struct
+{
+    pixman_format_code_t	format;
+    fetch_scanline_t		fetch_scanline_raw_32;
+    fetch_scanline_t		fetch_scanline_raw_64;
+    fetch_pixel_32_t		fetch_pixel_raw_32;
+    fetch_pixel_64_t		fetch_pixel_raw_64;
+    store_scanline_t		store_scanline_raw_32;
+    store_scanline_t		store_scanline_raw_64;
+} format_info_t;
+
+#define FORMAT_INFO(format) 						\
+    {									\
+	PIXMAN_ ## format,						\
+	    fetch_scanline_ ## format,					\
+	    fetch_scanline_generic_64,					\
+	    fetch_pixel_ ## format, fetch_pixel_generic_64,		\
+	    store_scanline_ ## format, store_scanline_generic_64	\
+    }
+
+static const format_info_t accessors[] =
+{
+/* 32 bpp formats */
+    FORMAT_INFO (a8r8g8b8),
+    FORMAT_INFO (x8r8g8b8),
+    FORMAT_INFO (a8b8g8r8),
+    FORMAT_INFO (x8b8g8r8),
+    FORMAT_INFO (b8g8r8a8),
+    FORMAT_INFO (b8g8r8x8),
+    
+/* 24bpp formats */
+    FORMAT_INFO (r8g8b8),
+    FORMAT_INFO (b8g8r8),
+    
+/* 16bpp formats */
+    FORMAT_INFO (r5g6b5),
+    FORMAT_INFO (b5g6r5),
+    
+    FORMAT_INFO (a1r5g5b5),
+    FORMAT_INFO (x1r5g5b5),
+    FORMAT_INFO (a1b5g5r5),
+    FORMAT_INFO (x1b5g5r5),
+    FORMAT_INFO (a4r4g4b4),
+    FORMAT_INFO (x4r4g4b4),
+    FORMAT_INFO (a4b4g4r4),
+    FORMAT_INFO (x4b4g4r4),
+    
+/* 8bpp formats */
+    FORMAT_INFO (a8),
+    FORMAT_INFO (r3g3b2),
+    FORMAT_INFO (b2g3r3),
+    FORMAT_INFO (a2r2g2b2),
+    FORMAT_INFO (a2b2g2r2),
+    
+    FORMAT_INFO (c8),
+    
+#define fetch_scanline_g8 fetch_scanline_c8
+#define fetch_pixel_g8 fetch_pixel_c8
+#define store_scanline_g8 store_scanline_c8
+    FORMAT_INFO (g8),
+    
+#define fetch_scanline_x4c4 fetch_scanline_c8
+#define fetch_pixel_x4c4 fetch_pixel_c8
+#define store_scanline_x4c4 store_scanline_c8
+    FORMAT_INFO (x4c4),
+    
+#define fetch_scanline_x4g4 fetch_scanline_c8
+#define fetch_pixel_x4g4 fetch_pixel_c8
+#define store_scanline_x4g4 store_scanline_c8
+    FORMAT_INFO (x4g4),
+    
+    FORMAT_INFO (x4a4),
+    
+/* 4bpp formats */
+    FORMAT_INFO (a4),
+    FORMAT_INFO (r1g2b1),
+    FORMAT_INFO (b1g2r1),
+    FORMAT_INFO (a1r1g1b1),
+    FORMAT_INFO (a1b1g1r1),
+    
+    FORMAT_INFO (c4),
+    
+#define fetch_scanline_g4 fetch_scanline_c4
+#define fetch_pixel_g4 fetch_pixel_c4
+#define store_scanline_g4 store_scanline_c4
+    FORMAT_INFO (g4),
+    
+/* 1bpp formats */
+    FORMAT_INFO (a1),
+    FORMAT_INFO (g1),
+    
+/* Wide formats */
+    
+    { PIXMAN_a2r10g10b10,
+      NULL, fetch_scanline_a2r10g10b10,
+      fetch_pixel_generic_lossy_32, fetch_pixel_a2r10g10b10,
+      NULL, store_scanline_a2r10g10b10 },
+    
+    { PIXMAN_x2r10g10b10,
+      NULL, fetch_scanline_x2r10g10b10,
+      fetch_pixel_generic_lossy_32, fetch_pixel_x2r10g10b10,
+      NULL, store_scanline_x2r10g10b10 },
+    
+    { PIXMAN_a2b10g10r10,
+      NULL, fetch_scanline_a2b10g10r10,
+      fetch_pixel_generic_lossy_32, fetch_pixel_a2b10g10r10,
+      NULL, store_scanline_a2b10g10r10 },
+    
+    { PIXMAN_x2b10g10r10,
+      NULL, fetch_scanline_x2b10g10r10,
+      fetch_pixel_generic_lossy_32, fetch_pixel_x2b10g10r10,
+      NULL, store_scanline_x2b10g10r10 },
+    
+/* YUV formats */
+    { PIXMAN_yuy2,
+      fetch_scanline_yuy2, fetch_scanline_generic_64,
+      fetch_pixel_yuy2, fetch_pixel_generic_64,
+      NULL, NULL },
+    
+    { PIXMAN_yv12,
+      fetch_scanline_yv12, fetch_scanline_generic_64,
+      fetch_pixel_yv12, fetch_pixel_generic_64,
+      NULL, NULL },
+    
+    { PIXMAN_null },
+};
+
+static void
+setup_accessors (bits_image_t *image)
+{
+    const format_info_t *info = accessors;
+    
+    while (info->format != PIXMAN_null)
+    {
+	if (info->format == image->format)
+	{
+	    image->fetch_scanline_raw_32 = info->fetch_scanline_raw_32;
+	    image->fetch_scanline_raw_64 = info->fetch_scanline_raw_64;
+	    image->fetch_pixel_raw_32 = info->fetch_pixel_raw_32;
+	    image->fetch_pixel_raw_64 = info->fetch_pixel_raw_64;
+	    image->store_scanline_raw_32 = info->store_scanline_raw_32;
+	    image->store_scanline_raw_64 = info->store_scanline_raw_64;
+	    
+	    return;
+	}
+	
+	info++;
+    }
+}
+
+#ifndef PIXMAN_FB_ACCESSORS
+void
+_pixman_bits_image_setup_raw_accessors_accessors (bits_image_t *image);
+
+void
+_pixman_bits_image_setup_raw_accessors (bits_image_t *image)
+{
+    if (image->read_func || image->write_func)
+	_pixman_bits_image_setup_raw_accessors_accessors (image);
+    else
+	setup_accessors (image);
+}
+
+#else
+
+void
+_pixman_bits_image_setup_raw_accessors_accessors (bits_image_t *image)
+{
+    setup_accessors (image);
+}
+
+#endif
diff --git a/pixman/pixman/pixman-accessor.h b/pixman/pixman/pixman-accessor.h
new file mode 100644
index 000000000..90c8ea7b7
--- /dev/null
+++ b/pixman/pixman/pixman-accessor.h
@@ -0,0 +1,40 @@
+#ifdef PIXMAN_FB_ACCESSORS
+
+#define ACCESS(sym) sym##_accessors
+
+#define READ(img, ptr)							\
+    (((bits_image_t *)(img))->read_func ((ptr), sizeof(*(ptr))))
+#define WRITE(img, ptr,val)						\
+    (((bits_image_t *)(img))->write_func ((ptr), (val), sizeof (*(ptr))))
+
+#define MEMCPY_WRAPPED(img, dst, src, size)				\
+    do {								\
+	size_t _i;							\
+	uint8_t *_dst = (uint8_t*)(dst), *_src = (uint8_t*)(src);	\
+	for(_i = 0; _i < size; _i++) {					\
+	    WRITE((img), _dst +_i, READ((img), _src + _i));		\
+	}								\
+    } while (0)
+
+#define MEMSET_WRAPPED(img, dst, val, size)				\
+    do {								\
+	size_t _i;							\
+	uint8_t *_dst = (uint8_t*)(dst);				\
+	for(_i = 0; _i < (size_t) size; _i++) {				\
+	    WRITE((img), _dst +_i, (val));				\
+	}								\
+    } while (0)
+
+#else
+
+#define ACCESS(sym) sym
+
+#define READ(img, ptr)		(*(ptr))
+#define WRITE(img, ptr, val)	(*(ptr) = (val))
+#define MEMCPY_WRAPPED(img, dst, src, size)				\
+    memcpy(dst, src, size)
+#define MEMSET_WRAPPED(img, dst, val, size)				\
+    memset(dst, val, size)
+
+#endif
+
diff --git a/pixman/pixman/pixman-arm-common.h b/pixman/pixman/pixman-arm-common.h
new file mode 100644
index 000000000..8d432b1de
--- /dev/null
+++ b/pixman/pixman/pixman-arm-common.h
@@ -0,0 +1,273 @@
+/*
+ * Copyright © 2010 Nokia Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ *
+ * Author:  Siarhei Siamashka (siarhei.siamashka@nokia.com)
+ */
+
+#ifndef PIXMAN_ARM_COMMON_H
+#define PIXMAN_ARM_COMMON_H
+
+/* Define some macros which can expand into proxy functions between
+ * ARM assembly optimized functions and the rest of pixman fast path API.
+ *
+ * All the low level ARM assembly functions have to use ARM EABI
+ * calling convention and take up to 8 arguments:
+ *    width, height, dst, dst_stride, src, src_stride, mask, mask_stride
+ *
+ * The arguments are ordered with the most important coming first (the
+ * first 4 arguments are passed to function in registers, the rest are
+ * on stack). The last arguments are optional, for example if the
+ * function is not using mask, then 'mask' and 'mask_stride' can be
+ * omitted when doing a function call.
+ *
+ * Arguments 'src' and 'mask' contain either a pointer to the top left
+ * pixel of the composited rectangle or a pixel color value depending
+ * on the function type. In the case of just a color value (solid source
+ * or mask), the corresponding stride argument is unused.
+ */
+
+#define PIXMAN_ARM_BIND_FAST_PATH_SRC_DST(cputype, name,                \
+                                          src_type, src_cnt,            \
+                                          dst_type, dst_cnt)            \
+void                                                                    \
+pixman_composite_##name##_asm_##cputype (int32_t   w,                   \
+                                         int32_t   h,                   \
+                                         dst_type *dst,                 \
+                                         int32_t   dst_stride,          \
+                                         src_type *src,                 \
+                                         int32_t   src_stride);         \
+                                                                        \
+static void                                                             \
+cputype##_composite_##name (pixman_implementation_t *imp,               \
+                            pixman_op_t              op,                \
+                            pixman_image_t *         src_image,         \
+                            pixman_image_t *         mask_image,        \
+                            pixman_image_t *         dst_image,         \
+                            int32_t                  src_x,             \
+                            int32_t                  src_y,             \
+                            int32_t                  mask_x,            \
+                            int32_t                  mask_y,            \
+                            int32_t                  dest_x,            \
+                            int32_t                  dest_y,            \
+                            int32_t                  width,             \
+                            int32_t                  height)            \
+{                                                                       \
+    dst_type *dst_line;                                                 \
+    src_type *src_line;                                                 \
+    int32_t dst_stride, src_stride;                                     \
+                                                                        \
+    PIXMAN_IMAGE_GET_LINE (src_image, src_x, src_y, src_type,           \
+                           src_stride, src_line, src_cnt);              \
+    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, dst_type,         \
+                           dst_stride, dst_line, dst_cnt);              \
+                                                                        \
+    pixman_composite_##name##_asm_##cputype (width, height,             \
+                                             dst_line, dst_stride,      \
+                                             src_line, src_stride);     \
+}
+
+#define PIXMAN_ARM_BIND_FAST_PATH_N_DST(cputype, name,                  \
+                                        dst_type, dst_cnt)              \
+void                                                                    \
+pixman_composite_##name##_asm_##cputype (int32_t    w,                  \
+                                         int32_t    h,                  \
+                                         dst_type  *dst,                \
+                                         int32_t    dst_stride,         \
+                                         uint32_t   src);               \
+                                                                        \
+static void                                                             \
+cputype##_composite_##name (pixman_implementation_t *imp,               \
+                            pixman_op_t              op,                \
+                            pixman_image_t *         src_image,         \
+                            pixman_image_t *         mask_image,        \
+                            pixman_image_t *         dst_image,         \
+                            int32_t                  src_x,             \
+                            int32_t                  src_y,             \
+                            int32_t                  mask_x,            \
+                            int32_t                  mask_y,            \
+                            int32_t                  dest_x,            \
+                            int32_t                  dest_y,            \
+                            int32_t                  width,             \
+                            int32_t                  height)            \
+{                                                                       \
+    dst_type  *dst_line;                                                \
+    int32_t    dst_stride;                                              \
+    uint32_t   src;                                                     \
+                                                                        \
+    src = _pixman_image_get_solid (src_image, dst_image->bits.format);  \
+                                                                        \
+    if (src == 0)                                                       \
+	return;                                                         \
+                                                                        \
+    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, dst_type,         \
+                           dst_stride, dst_line, dst_cnt);              \
+                                                                        \
+    pixman_composite_##name##_asm_##cputype (width, height,             \
+                                             dst_line, dst_stride,      \
+                                             src);                      \
+}
+
+#define PIXMAN_ARM_BIND_FAST_PATH_N_MASK_DST(cputype, name,             \
+                                             mask_type, mask_cnt,       \
+                                             dst_type, dst_cnt)         \
+void                                                                    \
+pixman_composite_##name##_asm_##cputype (int32_t    w,                  \
+                                         int32_t    h,                  \
+                                         dst_type  *dst,                \
+                                         int32_t    dst_stride,         \
+                                         uint32_t   src,                \
+                                         int32_t    unused,             \
+                                         mask_type *mask,               \
+                                         int32_t    mask_stride);       \
+                                                                        \
+static void                                                             \
+cputype##_composite_##name (pixman_implementation_t *imp,               \
+                            pixman_op_t              op,                \
+                            pixman_image_t *         src_image,         \
+                            pixman_image_t *         mask_image,        \
+                            pixman_image_t *         dst_image,         \
+                            int32_t                  src_x,             \
+                            int32_t                  src_y,             \
+                            int32_t                  mask_x,            \
+                            int32_t                  mask_y,            \
+                            int32_t                  dest_x,            \
+                            int32_t                  dest_y,            \
+                            int32_t                  width,             \
+                            int32_t                  height)            \
+{                                                                       \
+    dst_type  *dst_line;                                                \
+    mask_type *mask_line;                                               \
+    int32_t    dst_stride, mask_stride;                                 \
+    uint32_t   src;                                                     \
+                                                                        \
+    src = _pixman_image_get_solid (src_image, dst_image->bits.format);  \
+                                                                        \
+    if (src == 0)                                                       \
+	return;                                                         \
+                                                                        \
+    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, dst_type,         \
+                           dst_stride, dst_line, dst_cnt);              \
+    PIXMAN_IMAGE_GET_LINE (mask_image, mask_x, mask_y, mask_type,       \
+                           mask_stride, mask_line, mask_cnt);           \
+                                                                        \
+    pixman_composite_##name##_asm_##cputype (width, height,             \
+                                             dst_line, dst_stride,      \
+                                             src, 0,                    \
+                                             mask_line, mask_stride);   \
+}
+
+#define PIXMAN_ARM_BIND_FAST_PATH_SRC_N_DST(cputype, name,              \
+                                            src_type, src_cnt,          \
+                                            dst_type, dst_cnt)          \
+void                                                                    \
+pixman_composite_##name##_asm_##cputype (int32_t    w,                  \
+                                         int32_t    h,                  \
+                                         dst_type  *dst,                \
+                                         int32_t    dst_stride,         \
+                                         src_type  *src,                \
+                                         int32_t    src_stride,         \
+                                         uint32_t   mask);              \
+                                                                        \
+static void                                                             \
+cputype##_composite_##name (pixman_implementation_t *imp,               \
+                            pixman_op_t              op,                \
+                            pixman_image_t *         src_image,         \
+                            pixman_image_t *         mask_image,        \
+                            pixman_image_t *         dst_image,         \
+                            int32_t                  src_x,             \
+                            int32_t                  src_y,             \
+                            int32_t                  mask_x,            \
+                            int32_t                  mask_y,            \
+                            int32_t                  dest_x,            \
+                            int32_t                  dest_y,            \
+                            int32_t                  width,             \
+                            int32_t                  height)            \
+{                                                                       \
+    dst_type  *dst_line;                                                \
+    src_type  *src_line;                                                \
+    int32_t    dst_stride, src_stride;                                  \
+    uint32_t   mask;                                                    \
+                                                                        \
+    mask = _pixman_image_get_solid (mask_image, dst_image->bits.format);\
+                                                                        \
+    if (mask == 0)                                                      \
+	return;                                                         \
+                                                                        \
+    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, dst_type,         \
+                           dst_stride, dst_line, dst_cnt);              \
+    PIXMAN_IMAGE_GET_LINE (src_image, src_x, src_y, src_type,           \
+                           src_stride, src_line, src_cnt);              \
+                                                                        \
+    pixman_composite_##name##_asm_##cputype (width, height,             \
+                                             dst_line, dst_stride,      \
+                                             src_line, src_stride,      \
+                                             mask);                     \
+}
+
+#define PIXMAN_ARM_BIND_FAST_PATH_SRC_MASK_DST(cputype, name,           \
+                                               src_type, src_cnt,       \
+                                               mask_type, mask_cnt,     \
+                                               dst_type, dst_cnt)       \
+void                                                                    \
+pixman_composite_##name##_asm_##cputype (int32_t    w,                  \
+                                         int32_t    h,                  \
+                                         dst_type  *dst,                \
+                                         int32_t    dst_stride,         \
+                                         src_type  *src,                \
+                                         int32_t    src_stride,         \
+                                         mask_type *mask,               \
+                                         int32_t    mask_stride);       \
+                                                                        \
+static void                                                             \
+cputype##_composite_##name (pixman_implementation_t *imp,               \
+                            pixman_op_t              op,                \
+                            pixman_image_t *         src_image,         \
+                            pixman_image_t *         mask_image,        \
+                            pixman_image_t *         dst_image,         \
+                            int32_t                  src_x,             \
+                            int32_t                  src_y,             \
+                            int32_t                  mask_x,            \
+                            int32_t                  mask_y,            \
+                            int32_t                  dest_x,            \
+                            int32_t                  dest_y,            \
+                            int32_t                  width,             \
+                            int32_t                  height)            \
+{                                                                       \
+    dst_type  *dst_line;                                                \
+    src_type  *src_line;                                                \
+    mask_type *mask_line;                                               \
+    int32_t    dst_stride, src_stride, mask_stride;                     \
+                                                                        \
+    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, dst_type,         \
+                           dst_stride, dst_line, dst_cnt);              \
+    PIXMAN_IMAGE_GET_LINE (src_image, src_x, src_y, src_type,           \
+                           src_stride, src_line, src_cnt);              \
+    PIXMAN_IMAGE_GET_LINE (mask_image, mask_x, mask_y, mask_type,       \
+                           mask_stride, mask_line, mask_cnt);           \
+                                                                        \
+    pixman_composite_##name##_asm_##cputype (width, height,             \
+                                             dst_line, dst_stride,      \
+                                             src_line, src_stride,      \
+                                             mask_line, mask_stride);   \
+}
+
+#endif
diff --git a/pixman/pixman/pixman-arm-detect-win32.asm b/pixman/pixman/pixman-arm-detect-win32.asm
new file mode 100644
index 000000000..8f5d5eb2a
--- /dev/null
+++ b/pixman/pixman/pixman-arm-detect-win32.asm
@@ -0,0 +1,21 @@
+    area pixman_msvc, code, readonly

+

+    export  pixman_msvc_try_arm_simd_op

+

+pixman_msvc_try_arm_simd_op

+    ;; I don't think the msvc arm asm knows how to do SIMD insns

+    ;; uqadd8 r3,r3,r3

+    dcd 0xe6633f93

+    mov pc,lr

+    endp

+

+    export  pixman_msvc_try_arm_neon_op

+

+pixman_msvc_try_arm_neon_op

+    ;; I don't think the msvc arm asm knows how to do NEON insns

+    ;; veor d0,d0,d0

+    dcd 0xf3000110

+    mov pc,lr

+    endp

+

+    end

diff --git a/pixman/pixman/pixman-arm-neon-asm.S b/pixman/pixman/pixman-arm-neon-asm.S
new file mode 100644
index 000000000..51bc347bc
--- /dev/null
+++ b/pixman/pixman/pixman-arm-neon-asm.S
@@ -0,0 +1,1713 @@
+/*
+ * Copyright © 2009 Nokia Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ *
+ * Author:  Siarhei Siamashka (siarhei.siamashka@nokia.com)
+ */
+
+/*
+ * This file contains implementations of NEON optimized pixel processing
+ * functions. There is no full and detailed tutorial, but some functions
+ * (those which are exposing some new or interesting features) are
+ * extensively commented and can be used as examples.
+ *
+ * You may want to have a look at the comments for following functions:
+ *  - pixman_composite_over_8888_0565_asm_neon
+ *  - pixman_composite_over_n_8_0565_asm_neon
+ */
+
+/* Prevent the stack from becoming executable for no reason... */
+#if defined(__linux__) && defined(__ELF__)
+.section .note.GNU-stack,"",%progbits
+#endif
+
+    .text
+    .fpu neon
+    .arch armv7a
+    .object_arch armv4
+    .eabi_attribute 10, 0 /* suppress Tag_FP_arch */
+    .eabi_attribute 12, 0 /* suppress Tag_Advanced_SIMD_arch */
+    .arm
+    .altmacro
+
+#include "pixman-arm-neon-asm.h"
+
+/* Global configuration options and preferences */
+
+/*
+ * The code can optionally make use of unaligned memory accesses to improve
+ * performance of handling leading/trailing pixels for each scanline.
+ * Configuration variable RESPECT_STRICT_ALIGNMENT can be set to 0 for
+ * example in linux if unaligned memory accesses are not configured to
+ * generate.exceptions.
+ */
+.set RESPECT_STRICT_ALIGNMENT, 1
+
+/*
+ * Set default prefetch type. There is a choice between the following options:
+ *
+ * PREFETCH_TYPE_NONE (may be useful for the ARM cores where PLD is set to work
+ * as NOP to workaround some HW bugs or for whatever other reason)
+ *
+ * PREFETCH_TYPE_SIMPLE (may be useful for simple single-issue ARM cores where
+ * advanced prefetch intruduces heavy overhead)
+ *
+ * PREFETCH_TYPE_ADVANCED (useful for superscalar cores such as ARM Cortex-A8
+ * which can run ARM and NEON instructions simultaneously so that extra ARM
+ * instructions do not add (many) extra cycles, but improve prefetch efficiency)
+ *
+ * Note: some types of function can't support advanced prefetch and fallback
+ *       to simple one (those which handle 24bpp pixels)
+ */
+.set PREFETCH_TYPE_DEFAULT, PREFETCH_TYPE_ADVANCED
+
+/* Prefetch distance in pixels for simple prefetch */
+.set PREFETCH_DISTANCE_SIMPLE, 64
+
+/*
+ * Implementation of pixman_composite_over_8888_0565_asm_neon
+ *
+ * This function takes a8r8g8b8 source buffer, r5g6b5 destination buffer and
+ * performs OVER compositing operation. Function fast_composite_over_8888_0565
+ * from pixman-fast-path.c does the same in C and can be used as a reference.
+ *
+ * First we need to have some NEON assembly code which can do the actual
+ * operation on the pixels and provide it to the template macro.
+ *
+ * Template macro quite conveniently takes care of emitting all the necessary
+ * code for memory reading and writing (including quite tricky cases of
+ * handling unaligned leading/trailing pixels), so we only need to deal with
+ * the data in NEON registers.
+ *
+ * NEON registers allocation in general is recommented to be the following:
+ * d0,  d1,  d2,  d3  - contain loaded source pixel data
+ * d4,  d5,  d6,  d7  - contain loaded destination pixels (if they are needed)
+ * d24, d25, d26, d27 - contain loading mask pixel data (if mask is used)
+ * d28, d29, d30, d31 - place for storing the result (destination pixels)
+ *
+ * As can be seen above, four 64-bit NEON registers are used for keeping
+ * intermediate pixel data and up to 8 pixels can be processed in one step
+ * for 32bpp formats (16 pixels for 16bpp, 32 pixels for 8bpp).
+ *
+ * This particular function uses the following registers allocation:
+ * d0,  d1,  d2,  d3  - contain loaded source pixel data
+ * d4,  d5            - contain loaded destination pixels (they are needed)
+ * d28, d29           - place for storing the result (destination pixels)
+ */
+
+/*
+ * Step one. We need to have some code to do some arithmetics on pixel data.
+ * This is implemented as a pair of macros: '*_head' and '*_tail'. When used
+ * back-to-back, they take pixel data from {d0, d1, d2, d3} and {d4, d5},
+ * perform all the needed calculations and write the result to {d28, d29}.
+ * The rationale for having two macros and not just one will be explained
+ * later. In practice, any single monolitic function which does the work can
+ * be split into two parts in any arbitrary way without affecting correctness.
+ *
+ * There is one special trick here too. Common template macro can optionally
+ * make our life a bit easier by doing R, G, B, A color components
+ * deinterleaving for 32bpp pixel formats (and this feature is used in
+ * 'pixman_composite_over_8888_0565_asm_neon' function). So it means that
+ * instead of having 8 packed pixels in {d0, d1, d2, d3} registers, we
+ * actually use d0 register for blue channel (a vector of eight 8-bit
+ * values), d1 register for green, d2 for red and d3 for alpha. This
+ * simple conversion can be also done with a few NEON instructions:
+ *
+ * Packed to planar conversion:
+ *  vuzp.8 d0, d1
+ *  vuzp.8 d2, d3
+ *  vuzp.8 d1, d3
+ *  vuzp.8 d0, d2
+ *
+ * Planar to packed conversion:
+ *  vzip.8 d0, d2
+ *  vzip.8 d1, d3
+ *  vzip.8 d2, d3
+ *  vzip.8 d0, d1
+ *
+ * But pixel can be loaded directly in planar format using VLD4.8 NEON
+ * instruction. It is 1 cycle slower than VLD1.32, so this is not always
+ * desirable, that's why deinterleaving is optional.
+ *
+ * But anyway, here is the code:
+ */
+.macro pixman_composite_over_8888_0565_process_pixblock_head
+    /* convert 8 r5g6b5 pixel data from {d4, d5} to planar 8-bit format
+       and put data into d6 - red, d7 - green, d30 - blue */
+    vshrn.u16   d6, q2, #8
+    vshrn.u16   d7, q2, #3
+    vsli.u16    q2, q2, #5
+    vsri.u8     d6, d6, #5
+    vmvn.8      d3, d3      /* invert source alpha */
+    vsri.u8     d7, d7, #6
+    vshrn.u16   d30, q2, #2
+    /* now do alpha blending, storing results in 8-bit planar format
+       into d16 - red, d19 - green, d18 - blue */
+    vmull.u8    q10, d3, d6
+    vmull.u8    q11, d3, d7
+    vmull.u8    q12, d3, d30
+    vrshr.u16   q13, q10, #8
+    vrshr.u16   q3, q11, #8
+    vrshr.u16   q15, q12, #8
+    vraddhn.u16 d20, q10, q13
+    vraddhn.u16 d23, q11, q3
+    vraddhn.u16 d22, q12, q15
+.endm
+
+.macro pixman_composite_over_8888_0565_process_pixblock_tail
+    /* ... continue alpha blending */
+    vqadd.u8    d16, d2, d20
+    vqadd.u8    q9, q0, q11
+    /* convert the result to r5g6b5 and store it into {d28, d29} */
+    vshll.u8    q14, d16, #8
+    vshll.u8    q8, d19, #8
+    vshll.u8    q9, d18, #8
+    vsri.u16    q14, q8, #5
+    vsri.u16    q14, q9, #11
+.endm
+
+/*
+ * OK, now we got almost everything that we need. Using the above two
+ * macros, the work can be done right. But now we want to optimize
+ * it a bit. ARM Cortex-A8 is an in-order core, and benefits really
+ * a lot from good code scheduling and software pipelining.
+ *
+ * Let's construct some code, which will run in the core main loop.
+ * Some pseudo-code of the main loop will look like this:
+ *   head
+ *   while (...) {
+ *     tail
+ *     head
+ *   }
+ *   tail
+ *
+ * It may look a bit weird, but this setup allows to hide instruction
+ * latencies better and also utilize dual-issue capability more
+ * efficiently (make pairs of load-store and ALU instructions).
+ *
+ * So what we need now is a '*_tail_head' macro, which will be used
+ * in the core main loop. A trivial straightforward implementation
+ * of this macro would look like this:
+ *
+ *   pixman_composite_over_8888_0565_process_pixblock_tail
+ *   vst1.16     {d28, d29}, [DST_W, :128]!
+ *   vld1.16     {d4, d5}, [DST_R, :128]!
+ *   vld4.32     {d0, d1, d2, d3}, [SRC]!
+ *   pixman_composite_over_8888_0565_process_pixblock_head
+ *   cache_preload 8, 8
+ *
+ * Now it also got some VLD/VST instructions. We simply can't move from
+ * processing one block of pixels to the other one with just arithmetics.
+ * The previously processed data needs to be written to memory and new
+ * data needs to be fetched. Fortunately, this main loop does not deal
+ * with partial leading/trailing pixels and can load/store a full block
+ * of pixels in a bulk. Additionally, destination buffer is already
+ * 16 bytes aligned here (which is good for performance).
+ *
+ * New things here are DST_R, DST_W, SRC and MASK identifiers. These
+ * are the aliases for ARM registers which are used as pointers for
+ * accessing data. We maintain separate pointers for reading and writing
+ * destination buffer (DST_R and DST_W).
+ *
+ * Another new thing is 'cache_preload' macro. It is used for prefetching
+ * data into CPU L2 cache and improve performance when dealing with large
+ * images which are far larger than cache size. It uses one argument
+ * (actually two, but they need to be the same here) - number of pixels
+ * in a block. Looking into 'pixman-arm-neon-asm.h' can provide some
+ * details about this macro. Moreover, if good performance is needed
+ * the code from this macro needs to be copied into '*_tail_head' macro
+ * and mixed with the rest of code for optimal instructions scheduling.
+ * We are actually doing it below.
+ *
+ * Now after all the explanations, here is the optimized code.
+ * Different instruction streams (originaling from '*_head', '*_tail'
+ * and 'cache_preload' macro) use different indentation levels for
+ * better readability. Actually taking the code from one of these
+ * indentation levels and ignoring a few VLD/VST instructions would
+ * result in exactly the code from '*_head', '*_tail' or 'cache_preload'
+ * macro!
+ */
+
+#if 1
+
+.macro pixman_composite_over_8888_0565_process_pixblock_tail_head
+        vqadd.u8    d16, d2, d20
+    vld1.16     {d4, d5}, [DST_R, :128]!
+        vqadd.u8    q9, q0, q11
+    vshrn.u16   d6, q2, #8
+    vld4.8      {d0, d1, d2, d3}, [SRC]!
+    vshrn.u16   d7, q2, #3
+    vsli.u16    q2, q2, #5
+        vshll.u8    q14, d16, #8
+                                    PF add PF_X, PF_X, #8
+        vshll.u8    q8, d19, #8
+                                    PF tst PF_CTL, #0xF
+    vsri.u8     d6, d6, #5
+                                    PF addne PF_X, PF_X, #8
+    vmvn.8      d3, d3
+                                    PF subne PF_CTL, PF_CTL, #1
+    vsri.u8     d7, d7, #6
+    vshrn.u16   d30, q2, #2
+    vmull.u8    q10, d3, d6
+                                    PF pld, [PF_SRC, PF_X, lsl #src_bpp_shift]
+    vmull.u8    q11, d3, d7
+    vmull.u8    q12, d3, d30
+                                    PF pld, [PF_DST, PF_X, lsl #dst_bpp_shift]
+        vsri.u16    q14, q8, #5
+                                    PF cmp PF_X, ORIG_W
+        vshll.u8    q9, d18, #8
+    vrshr.u16   q13, q10, #8
+                                    PF subge PF_X, PF_X, ORIG_W
+    vrshr.u16   q3, q11, #8
+    vrshr.u16   q15, q12, #8
+                                    PF subges PF_CTL, PF_CTL, #0x10
+        vsri.u16    q14, q9, #11
+                                    PF ldrgeb DUMMY, [PF_SRC, SRC_STRIDE, lsl #src_bpp_shift]!
+    vraddhn.u16 d20, q10, q13
+    vraddhn.u16 d23, q11, q3
+                                    PF ldrgeb DUMMY, [PF_DST, DST_STRIDE, lsl #dst_bpp_shift]!
+    vraddhn.u16 d22, q12, q15
+        vst1.16     {d28, d29}, [DST_W, :128]!
+.endm
+
+#else
+
+/* If we did not care much about the performance, we would just use this... */
+.macro pixman_composite_over_8888_0565_process_pixblock_tail_head
+    pixman_composite_over_8888_0565_process_pixblock_tail
+    vst1.16     {d28, d29}, [DST_W, :128]!
+    vld1.16     {d4, d5}, [DST_R, :128]!
+    vld4.32     {d0, d1, d2, d3}, [SRC]!
+    pixman_composite_over_8888_0565_process_pixblock_head
+    cache_preload 8, 8
+.endm
+
+#endif
+
+/*
+ * And now the final part. We are using 'generate_composite_function' macro
+ * to put all the stuff together. We are specifying the name of the function
+ * which we want to get, number of bits per pixel for the source, mask and
+ * destination (0 if unused, like mask in this case). Next come some bit
+ * flags:
+ *   FLAG_DST_READWRITE      - tells that the destination buffer is both read
+ *                             and written, for write-only buffer we would use
+ *                             FLAG_DST_WRITEONLY flag instead
+ *   FLAG_DEINTERLEAVE_32BPP - tells that we prefer to work with planar data
+ *                             and separate color channels for 32bpp format.
+ * The next things are:
+ *  - the number of pixels processed per iteration (8 in this case, because
+ *    that's the maximum what can fit into four 64-bit NEON registers).
+ *  - prefetch distance, measured in pixel blocks. In this case it is 5 times
+ *    by 8 pixels. That would be 40 pixels, or up to 160 bytes. Optimal
+ *    prefetch distance can be selected by running some benchmarks.
+ *
+ * After that we specify some macros, these are 'default_init',
+ * 'default_cleanup' here which are empty (but it is possible to have custom
+ * init/cleanup macros to be able to save/restore some extra NEON registers
+ * like d8-d15 or do anything else) followed by
+ * 'pixman_composite_over_8888_0565_process_pixblock_head',
+ * 'pixman_composite_over_8888_0565_process_pixblock_tail' and
+ * 'pixman_composite_over_8888_0565_process_pixblock_tail_head'
+ * which we got implemented above.
+ *
+ * The last part is the NEON registers allocation scheme.
+ */
+generate_composite_function \
+    pixman_composite_over_8888_0565_asm_neon, 32, 0, 16, \
+    FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
+    8, /* number of pixels, processed in a single block */ \
+    5, /* prefetch distance */ \
+    default_init, \
+    default_cleanup, \
+    pixman_composite_over_8888_0565_process_pixblock_head, \
+    pixman_composite_over_8888_0565_process_pixblock_tail, \
+    pixman_composite_over_8888_0565_process_pixblock_tail_head, \
+    28, /* dst_w_basereg */ \
+    4,  /* dst_r_basereg */ \
+    0,  /* src_basereg   */ \
+    24  /* mask_basereg  */
+
+/******************************************************************************/
+
+.macro pixman_composite_over_n_0565_process_pixblock_head
+    /* convert 8 r5g6b5 pixel data from {d4, d5} to planar 8-bit format
+       and put data into d6 - red, d7 - green, d30 - blue */
+    vshrn.u16   d6, q2, #8
+    vshrn.u16   d7, q2, #3
+    vsli.u16    q2, q2, #5
+    vsri.u8     d6, d6, #5
+    vsri.u8     d7, d7, #6
+    vshrn.u16   d30, q2, #2
+    /* now do alpha blending, storing results in 8-bit planar format
+       into d16 - red, d19 - green, d18 - blue */
+    vmull.u8    q10, d3, d6
+    vmull.u8    q11, d3, d7
+    vmull.u8    q12, d3, d30
+    vrshr.u16   q13, q10, #8
+    vrshr.u16   q3, q11, #8
+    vrshr.u16   q15, q12, #8
+    vraddhn.u16 d20, q10, q13
+    vraddhn.u16 d23, q11, q3
+    vraddhn.u16 d22, q12, q15
+.endm
+
+.macro pixman_composite_over_n_0565_process_pixblock_tail
+    /* ... continue alpha blending */
+    vqadd.u8    d16, d2, d20
+    vqadd.u8    q9, q0, q11
+    /* convert the result to r5g6b5 and store it into {d28, d29} */
+    vshll.u8    q14, d16, #8
+    vshll.u8    q8, d19, #8
+    vshll.u8    q9, d18, #8
+    vsri.u16    q14, q8, #5
+    vsri.u16    q14, q9, #11
+.endm
+
+/* TODO: expand macros and do better instructions scheduling */
+.macro pixman_composite_over_n_0565_process_pixblock_tail_head
+    pixman_composite_over_n_0565_process_pixblock_tail
+    vld1.16     {d4, d5}, [DST_R, :128]!
+    vst1.16     {d28, d29}, [DST_W, :128]!
+    pixman_composite_over_n_0565_process_pixblock_head
+.endm
+
+.macro pixman_composite_over_n_0565_init
+    add         DUMMY, sp, #ARGS_STACK_OFFSET
+    vld1.32     {d3[0]}, [DUMMY]
+    vdup.8      d0, d3[0]
+    vdup.8      d1, d3[1]
+    vdup.8      d2, d3[2]
+    vdup.8      d3, d3[3]
+    vmvn.8      d3, d3      /* invert source alpha */
+.endm
+
+generate_composite_function \
+    pixman_composite_over_n_0565_asm_neon, 0, 0, 16, \
+    FLAG_DST_READWRITE, \
+    8, /* number of pixels, processed in a single block */ \
+    5, /* prefetch distance */ \
+    pixman_composite_over_n_0565_init, \
+    default_cleanup, \
+    pixman_composite_over_n_0565_process_pixblock_head, \
+    pixman_composite_over_n_0565_process_pixblock_tail, \
+    pixman_composite_over_n_0565_process_pixblock_tail_head, \
+    28, /* dst_w_basereg */ \
+    4,  /* dst_r_basereg */ \
+    0,  /* src_basereg   */ \
+    24  /* mask_basereg  */
+
+/******************************************************************************/
+
+.macro pixman_composite_src_8888_0565_process_pixblock_head
+    vshll.u8    q8, d1, #8
+    vshll.u8    q14, d2, #8
+    vshll.u8    q9, d0, #8
+.endm
+
+.macro pixman_composite_src_8888_0565_process_pixblock_tail
+    vsri.u16    q14, q8, #5
+    vsri.u16    q14, q9, #11
+.endm
+
+.macro pixman_composite_src_8888_0565_process_pixblock_tail_head
+        vsri.u16    q14, q8, #5
+                                    PF add PF_X, PF_X, #8
+                                    PF tst PF_CTL, #0xF
+    vld4.8      {d0, d1, d2, d3}, [SRC]!
+                                    PF addne PF_X, PF_X, #8
+                                    PF subne PF_CTL, PF_CTL, #1
+        vsri.u16    q14, q9, #11
+                                    PF cmp PF_X, ORIG_W
+                                    PF pld, [PF_SRC, PF_X, lsl #src_bpp_shift]
+    vshll.u8    q8, d1, #8
+        vst1.16     {d28, d29}, [DST_W, :128]!
+                                    PF subge PF_X, PF_X, ORIG_W
+                                    PF subges PF_CTL, PF_CTL, #0x10
+    vshll.u8    q14, d2, #8
+                                    PF ldrgeb DUMMY, [PF_SRC, SRC_STRIDE, lsl #src_bpp_shift]!
+    vshll.u8    q9, d0, #8
+.endm
+
+generate_composite_function \
+    pixman_composite_src_8888_0565_asm_neon, 32, 0, 16, \
+    FLAG_DST_WRITEONLY | FLAG_DEINTERLEAVE_32BPP, \
+    8, /* number of pixels, processed in a single block */ \
+    10, /* prefetch distance */ \
+    default_init, \
+    default_cleanup, \
+    pixman_composite_src_8888_0565_process_pixblock_head, \
+    pixman_composite_src_8888_0565_process_pixblock_tail, \
+    pixman_composite_src_8888_0565_process_pixblock_tail_head
+
+/******************************************************************************/
+
+.macro pixman_composite_src_0565_8888_process_pixblock_head
+    vshrn.u16   d30, q0, #8
+    vshrn.u16   d29, q0, #3
+    vsli.u16    q0, q0, #5
+    vmov.u8     d31, #255
+    vsri.u8     d30, d30, #5
+    vsri.u8     d29, d29, #6
+    vshrn.u16   d28, q0, #2
+.endm
+
+.macro pixman_composite_src_0565_8888_process_pixblock_tail
+.endm
+
+/* TODO: expand macros and do better instructions scheduling */
+.macro pixman_composite_src_0565_8888_process_pixblock_tail_head
+    pixman_composite_src_0565_8888_process_pixblock_tail
+    vst4.8     {d28, d29, d30, d31}, [DST_W, :128]!
+    vld1.16    {d0, d1}, [SRC]!
+    pixman_composite_src_0565_8888_process_pixblock_head
+    cache_preload 8, 8
+.endm
+
+generate_composite_function \
+    pixman_composite_src_0565_8888_asm_neon, 16, 0, 32, \
+    FLAG_DST_WRITEONLY | FLAG_DEINTERLEAVE_32BPP, \
+    8, /* number of pixels, processed in a single block */ \
+    10, /* prefetch distance */ \
+    default_init, \
+    default_cleanup, \
+    pixman_composite_src_0565_8888_process_pixblock_head, \
+    pixman_composite_src_0565_8888_process_pixblock_tail, \
+    pixman_composite_src_0565_8888_process_pixblock_tail_head
+
+/******************************************************************************/
+
+.macro pixman_composite_add_8000_8000_process_pixblock_head
+    vqadd.u8    q14, q0, q2
+    vqadd.u8    q15, q1, q3
+.endm
+
+.macro pixman_composite_add_8000_8000_process_pixblock_tail
+.endm
+
+.macro pixman_composite_add_8000_8000_process_pixblock_tail_head
+    vld1.8      {d0, d1, d2, d3}, [SRC]!
+                                    PF add PF_X, PF_X, #32
+                                    PF tst PF_CTL, #0xF
+    vld1.8      {d4, d5, d6, d7}, [DST_R, :128]!
+                                    PF addne PF_X, PF_X, #32
+                                    PF subne PF_CTL, PF_CTL, #1
+        vst1.8      {d28, d29, d30, d31}, [DST_W, :128]!
+                                    PF cmp PF_X, ORIG_W
+                                    PF pld, [PF_SRC, PF_X, lsl #src_bpp_shift]
+                                    PF pld, [PF_DST, PF_X, lsl #dst_bpp_shift]
+                                    PF subge PF_X, PF_X, ORIG_W
+                                    PF subges PF_CTL, PF_CTL, #0x10
+    vqadd.u8    q14, q0, q2
+                                    PF ldrgeb DUMMY, [PF_SRC, SRC_STRIDE, lsl #src_bpp_shift]!
+                                    PF ldrgeb DUMMY, [PF_DST, DST_STRIDE, lsl #dst_bpp_shift]!
+    vqadd.u8    q15, q1, q3
+.endm
+
+generate_composite_function \
+    pixman_composite_add_8000_8000_asm_neon, 8, 0, 8, \
+    FLAG_DST_READWRITE, \
+    32, /* number of pixels, processed in a single block */ \
+    10, /* prefetch distance */ \
+    default_init, \
+    default_cleanup, \
+    pixman_composite_add_8000_8000_process_pixblock_head, \
+    pixman_composite_add_8000_8000_process_pixblock_tail, \
+    pixman_composite_add_8000_8000_process_pixblock_tail_head
+
+/******************************************************************************/
+
+.macro pixman_composite_add_8888_8888_process_pixblock_tail_head
+    vld1.8      {d0, d1, d2, d3}, [SRC]!
+                                    PF add PF_X, PF_X, #8
+                                    PF tst PF_CTL, #0xF
+    vld1.8      {d4, d5, d6, d7}, [DST_R, :128]!
+                                    PF addne PF_X, PF_X, #8
+                                    PF subne PF_CTL, PF_CTL, #1
+        vst1.8      {d28, d29, d30, d31}, [DST_W, :128]!
+                                    PF cmp PF_X, ORIG_W
+                                    PF pld, [PF_SRC, PF_X, lsl #src_bpp_shift]
+                                    PF pld, [PF_DST, PF_X, lsl #dst_bpp_shift]
+                                    PF subge PF_X, PF_X, ORIG_W
+                                    PF subges PF_CTL, PF_CTL, #0x10
+    vqadd.u8    q14, q0, q2
+                                    PF ldrgeb DUMMY, [PF_SRC, SRC_STRIDE, lsl #src_bpp_shift]!
+                                    PF ldrgeb DUMMY, [PF_DST, DST_STRIDE, lsl #dst_bpp_shift]!
+    vqadd.u8    q15, q1, q3
+.endm
+
+generate_composite_function \
+    pixman_composite_add_8888_8888_asm_neon, 32, 0, 32, \
+    FLAG_DST_READWRITE, \
+    8, /* number of pixels, processed in a single block */ \
+    10, /* prefetch distance */ \
+    default_init, \
+    default_cleanup, \
+    pixman_composite_add_8000_8000_process_pixblock_head, \
+    pixman_composite_add_8000_8000_process_pixblock_tail, \
+    pixman_composite_add_8888_8888_process_pixblock_tail_head
+
+generate_composite_function_single_scanline \
+    pixman_composite_scanline_add_asm_neon, 32, 0, 32, \
+    FLAG_DST_READWRITE, \
+    8, /* number of pixels, processed in a single block */ \
+    default_init, \
+    default_cleanup, \
+    pixman_composite_add_8000_8000_process_pixblock_head, \
+    pixman_composite_add_8000_8000_process_pixblock_tail, \
+    pixman_composite_add_8888_8888_process_pixblock_tail_head
+
+/******************************************************************************/
+
+.macro pixman_composite_over_8888_8888_process_pixblock_head
+    vmvn.8      d24, d3  /* get inverted alpha */
+    /* do alpha blending */
+    vmull.u8    q8, d24, d4
+    vmull.u8    q9, d24, d5
+    vmull.u8    q10, d24, d6
+    vmull.u8    q11, d24, d7
+.endm
+
+.macro pixman_composite_over_8888_8888_process_pixblock_tail
+    vrshr.u16   q14, q8, #8
+    vrshr.u16   q15, q9, #8
+    vrshr.u16   q12, q10, #8
+    vrshr.u16   q13, q11, #8
+    vraddhn.u16 d28, q14, q8
+    vraddhn.u16 d29, q15, q9
+    vraddhn.u16 d30, q12, q10
+    vraddhn.u16 d31, q13, q11
+    vqadd.u8    q14, q0, q14
+    vqadd.u8    q15, q1, q15
+.endm
+
+.macro pixman_composite_over_8888_8888_process_pixblock_tail_head
+    vld4.8      {d4, d5, d6, d7}, [DST_R, :128]!
+        vrshr.u16   q14, q8, #8
+                                    PF add PF_X, PF_X, #8
+                                    PF tst PF_CTL, #0xF
+        vrshr.u16   q15, q9, #8
+        vrshr.u16   q12, q10, #8
+        vrshr.u16   q13, q11, #8
+                                    PF addne PF_X, PF_X, #8
+                                    PF subne PF_CTL, PF_CTL, #1
+        vraddhn.u16 d28, q14, q8
+        vraddhn.u16 d29, q15, q9
+                                    PF cmp PF_X, ORIG_W
+        vraddhn.u16 d30, q12, q10
+        vraddhn.u16 d31, q13, q11
+        vqadd.u8    q14, q0, q14
+        vqadd.u8    q15, q1, q15
+    vld4.8      {d0, d1, d2, d3}, [SRC]!
+                                    PF pld, [PF_SRC, PF_X, lsl #src_bpp_shift]
+    vmvn.8      d22, d3
+                                    PF pld, [PF_DST, PF_X, lsl #dst_bpp_shift]
+        vst4.8      {d28, d29, d30, d31}, [DST_W, :128]!
+                                    PF subge PF_X, PF_X, ORIG_W
+    vmull.u8    q8, d22, d4
+                                    PF subges PF_CTL, PF_CTL, #0x10
+    vmull.u8    q9, d22, d5
+                                    PF ldrgeb DUMMY, [PF_SRC, SRC_STRIDE, lsl #src_bpp_shift]!
+    vmull.u8    q10, d22, d6
+                                    PF ldrgeb DUMMY, [PF_DST, DST_STRIDE, lsl #dst_bpp_shift]!
+    vmull.u8    q11, d22, d7
+.endm
+
+generate_composite_function \
+    pixman_composite_over_8888_8888_asm_neon, 32, 0, 32, \
+    FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
+    8, /* number of pixels, processed in a single block */ \
+    5, /* prefetch distance */ \
+    default_init, \
+    default_cleanup, \
+    pixman_composite_over_8888_8888_process_pixblock_head, \
+    pixman_composite_over_8888_8888_process_pixblock_tail, \
+    pixman_composite_over_8888_8888_process_pixblock_tail_head
+
+generate_composite_function_single_scanline \
+    pixman_composite_scanline_over_asm_neon, 32, 0, 32, \
+    FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
+    8, /* number of pixels, processed in a single block */ \
+    default_init, \
+    default_cleanup, \
+    pixman_composite_over_8888_8888_process_pixblock_head, \
+    pixman_composite_over_8888_8888_process_pixblock_tail, \
+    pixman_composite_over_8888_8888_process_pixblock_tail_head
+
+/******************************************************************************/
+
+/* TODO: expand macros and do better instructions scheduling */
+.macro pixman_composite_over_n_8888_process_pixblock_tail_head
+    pixman_composite_over_8888_8888_process_pixblock_tail
+    vld4.8      {d4, d5, d6, d7}, [DST_R, :128]!
+    vst4.8      {d28, d29, d30, d31}, [DST_W, :128]!
+    pixman_composite_over_8888_8888_process_pixblock_head
+.endm
+
+.macro pixman_composite_over_n_8888_init
+    add         DUMMY, sp, #ARGS_STACK_OFFSET
+    vld1.32     {d3[0]}, [DUMMY]
+    vdup.8      d0, d3[0]
+    vdup.8      d1, d3[1]
+    vdup.8      d2, d3[2]
+    vdup.8      d3, d3[3]
+.endm
+
+generate_composite_function \
+    pixman_composite_over_n_8888_asm_neon, 0, 0, 32, \
+    FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
+    8, /* number of pixels, processed in a single block */ \
+    5, /* prefetch distance */ \
+    pixman_composite_over_n_8888_init, \
+    default_cleanup, \
+    pixman_composite_over_8888_8888_process_pixblock_head, \
+    pixman_composite_over_8888_8888_process_pixblock_tail, \
+    pixman_composite_over_n_8888_process_pixblock_tail_head
+
+/******************************************************************************/
+
+.macro pixman_composite_over_reverse_n_8888_process_pixblock_tail_head
+        vrshr.u16   q14, q8, #8
+                                    PF add PF_X, PF_X, #8
+                                    PF tst PF_CTL, #0xF
+        vrshr.u16   q15, q9, #8
+        vrshr.u16   q12, q10, #8
+        vrshr.u16   q13, q11, #8
+                                    PF addne PF_X, PF_X, #8
+                                    PF subne PF_CTL, PF_CTL, #1
+        vraddhn.u16 d28, q14, q8
+        vraddhn.u16 d29, q15, q9
+                                    PF cmp PF_X, ORIG_W
+        vraddhn.u16 d30, q12, q10
+        vraddhn.u16 d31, q13, q11
+        vqadd.u8    q14, q0, q14
+        vqadd.u8    q15, q1, q15
+    vld4.8      {d0, d1, d2, d3}, [DST_R, :128]!
+    vmvn.8      d22, d3
+                                    PF pld, [PF_DST, PF_X, lsl #dst_bpp_shift]
+        vst4.8      {d28, d29, d30, d31}, [DST_W, :128]!
+                                    PF subge PF_X, PF_X, ORIG_W
+    vmull.u8    q8, d22, d4
+                                    PF subges PF_CTL, PF_CTL, #0x10
+    vmull.u8    q9, d22, d5
+    vmull.u8    q10, d22, d6
+                                    PF ldrgeb DUMMY, [PF_DST, DST_STRIDE, lsl #dst_bpp_shift]!
+    vmull.u8    q11, d22, d7
+.endm
+
+.macro pixman_composite_over_reverse_n_8888_init
+    add         DUMMY, sp, #ARGS_STACK_OFFSET
+    vld1.32     {d7[0]}, [DUMMY]
+    vdup.8      d4, d7[0]
+    vdup.8      d5, d7[1]
+    vdup.8      d6, d7[2]
+    vdup.8      d7, d7[3]
+.endm
+
+generate_composite_function \
+    pixman_composite_over_reverse_n_8888_asm_neon, 0, 0, 32, \
+    FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
+    8, /* number of pixels, processed in a single block */ \
+    5, /* prefetch distance */ \
+    pixman_composite_over_reverse_n_8888_init, \
+    default_cleanup, \
+    pixman_composite_over_8888_8888_process_pixblock_head, \
+    pixman_composite_over_8888_8888_process_pixblock_tail, \
+    pixman_composite_over_reverse_n_8888_process_pixblock_tail_head, \
+    28, /* dst_w_basereg */ \
+    0,  /* dst_r_basereg */ \
+    4,  /* src_basereg   */ \
+    24  /* mask_basereg  */
+
+/******************************************************************************/
+
+.macro pixman_composite_over_n_8_0565_process_pixblock_head
+    /* in */
+    vmull.u8    q0, d24, d8
+    vmull.u8    q1, d24, d9
+    vmull.u8    q6, d24, d10
+    vmull.u8    q7, d24, d11
+    vrshr.u16   q10, q0, #8
+    vrshr.u16   q11, q1, #8
+    vrshr.u16   q12, q6, #8
+    vrshr.u16   q13, q7, #8
+    vraddhn.u16 d0, q0, q10
+    vraddhn.u16 d1, q1, q11
+    vraddhn.u16 d2, q6, q12
+    vraddhn.u16 d3, q7, q13
+
+    vshrn.u16   d6, q2, #8
+    vshrn.u16   d7, q2, #3
+    vsli.u16    q2, q2, #5
+    vsri.u8     d6, d6, #5
+    vmvn.8      d3, d3
+    vsri.u8     d7, d7, #6
+    vshrn.u16   d30, q2, #2
+    /* now do alpha blending */
+    vmull.u8    q10, d3, d6
+    vmull.u8    q11, d3, d7
+    vmull.u8    q12, d3, d30
+    vrshr.u16   q13, q10, #8
+    vrshr.u16   q3, q11, #8
+    vrshr.u16   q15, q12, #8
+    vraddhn.u16 d20, q10, q13
+    vraddhn.u16 d23, q11, q3
+    vraddhn.u16 d22, q12, q15
+.endm
+
+.macro pixman_composite_over_n_8_0565_process_pixblock_tail
+    vqadd.u8    d16, d2, d20
+    vqadd.u8    q9, q0, q11
+    /* convert to r5g6b5 */
+    vshll.u8    q14, d16, #8
+    vshll.u8    q8, d19, #8
+    vshll.u8    q9, d18, #8
+    vsri.u16    q14, q8, #5
+    vsri.u16    q14, q9, #11
+.endm
+
+/* TODO: expand macros and do better instructions scheduling */
+.macro pixman_composite_over_n_8_0565_process_pixblock_tail_head
+    pixman_composite_over_n_8_0565_process_pixblock_tail
+    vst1.16     {d28, d29}, [DST_W, :128]!
+    vld1.16     {d4, d5}, [DST_R, :128]!
+    vld1.8      {d24}, [MASK]!
+    cache_preload 8, 8
+    pixman_composite_over_n_8_0565_process_pixblock_head
+.endm
+
+/*
+ * This function needs a special initialization of solid mask.
+ * Solid source pixel data is fetched from stack at ARGS_STACK_OFFSET
+ * offset, split into color components and replicated in d8-d11
+ * registers. Additionally, this function needs all the NEON registers,
+ * so it has to save d8-d15 registers which are callee saved according
+ * to ABI. These registers are restored from 'cleanup' macro. All the
+ * other NEON registers are caller saved, so can be clobbered freely
+ * without introducing any problems.
+ */
+.macro pixman_composite_over_n_8_0565_init
+    add         DUMMY, sp, #ARGS_STACK_OFFSET
+    vpush       {d8-d15}
+    vld1.32     {d11[0]}, [DUMMY]
+    vdup.8      d8, d11[0]
+    vdup.8      d9, d11[1]
+    vdup.8      d10, d11[2]
+    vdup.8      d11, d11[3]
+.endm
+
+.macro pixman_composite_over_n_8_0565_cleanup
+    vpop        {d8-d15}
+.endm
+
+generate_composite_function \
+    pixman_composite_over_n_8_0565_asm_neon, 0, 8, 16, \
+    FLAG_DST_READWRITE, \
+    8, /* number of pixels, processed in a single block */ \
+    5, /* prefetch distance */ \
+    pixman_composite_over_n_8_0565_init, \
+    pixman_composite_over_n_8_0565_cleanup, \
+    pixman_composite_over_n_8_0565_process_pixblock_head, \
+    pixman_composite_over_n_8_0565_process_pixblock_tail, \
+    pixman_composite_over_n_8_0565_process_pixblock_tail_head
+
+/******************************************************************************/
+
+.macro pixman_composite_src_0565_0565_process_pixblock_head
+.endm
+
+.macro pixman_composite_src_0565_0565_process_pixblock_tail
+.endm
+
+.macro pixman_composite_src_0565_0565_process_pixblock_tail_head
+    vst1.16 {d0, d1, d2, d3}, [DST_W, :128]!
+    vld1.16 {d0, d1, d2, d3}, [SRC]!
+    cache_preload 16, 16
+.endm
+
+generate_composite_function \
+    pixman_composite_src_0565_0565_asm_neon, 16, 0, 16, \
+    FLAG_DST_WRITEONLY, \
+    16, /* number of pixels, processed in a single block */ \
+    10, /* prefetch distance */ \
+    default_init, \
+    default_cleanup, \
+    pixman_composite_src_0565_0565_process_pixblock_head, \
+    pixman_composite_src_0565_0565_process_pixblock_tail, \
+    pixman_composite_src_0565_0565_process_pixblock_tail_head, \
+    0, /* dst_w_basereg */ \
+    0, /* dst_r_basereg */ \
+    0, /* src_basereg   */ \
+    0  /* mask_basereg  */
+
+/******************************************************************************/
+
+.macro pixman_composite_src_n_8_process_pixblock_head
+.endm
+
+.macro pixman_composite_src_n_8_process_pixblock_tail
+.endm
+
+.macro pixman_composite_src_n_8_process_pixblock_tail_head
+    vst1.8  {d0, d1, d2, d3}, [DST_W, :128]!
+.endm
+
+.macro pixman_composite_src_n_8_init
+    add         DUMMY, sp, #ARGS_STACK_OFFSET
+    vld1.32     {d0[0]}, [DUMMY]
+    vsli.u64    d0, d0, #8
+    vsli.u64    d0, d0, #16
+    vsli.u64    d0, d0, #32
+    vorr        d1, d0, d0
+    vorr        q1, q0, q0
+.endm
+
+.macro pixman_composite_src_n_8_cleanup
+.endm
+
+generate_composite_function \
+    pixman_composite_src_n_8_asm_neon, 0, 0, 8, \
+    FLAG_DST_WRITEONLY, \
+    32, /* number of pixels, processed in a single block */ \
+    0,  /* prefetch distance */ \
+    pixman_composite_src_n_8_init, \
+    pixman_composite_src_n_8_cleanup, \
+    pixman_composite_src_n_8_process_pixblock_head, \
+    pixman_composite_src_n_8_process_pixblock_tail, \
+    pixman_composite_src_n_8_process_pixblock_tail_head, \
+    0, /* dst_w_basereg */ \
+    0, /* dst_r_basereg */ \
+    0, /* src_basereg   */ \
+    0  /* mask_basereg  */
+
+/******************************************************************************/
+
+.macro pixman_composite_src_n_0565_process_pixblock_head
+.endm
+
+.macro pixman_composite_src_n_0565_process_pixblock_tail
+.endm
+
+.macro pixman_composite_src_n_0565_process_pixblock_tail_head
+    vst1.16 {d0, d1, d2, d3}, [DST_W, :128]!
+.endm
+
+.macro pixman_composite_src_n_0565_init
+    add         DUMMY, sp, #ARGS_STACK_OFFSET
+    vld1.32     {d0[0]}, [DUMMY]
+    vsli.u64    d0, d0, #16
+    vsli.u64    d0, d0, #32
+    vorr        d1, d0, d0
+    vorr        q1, q0, q0
+.endm
+
+.macro pixman_composite_src_n_0565_cleanup
+.endm
+
+generate_composite_function \
+    pixman_composite_src_n_0565_asm_neon, 0, 0, 16, \
+    FLAG_DST_WRITEONLY, \
+    16, /* number of pixels, processed in a single block */ \
+    0,  /* prefetch distance */ \
+    pixman_composite_src_n_0565_init, \
+    pixman_composite_src_n_0565_cleanup, \
+    pixman_composite_src_n_0565_process_pixblock_head, \
+    pixman_composite_src_n_0565_process_pixblock_tail, \
+    pixman_composite_src_n_0565_process_pixblock_tail_head, \
+    0, /* dst_w_basereg */ \
+    0, /* dst_r_basereg */ \
+    0, /* src_basereg   */ \
+    0  /* mask_basereg  */
+
+/******************************************************************************/
+
+.macro pixman_composite_src_n_8888_process_pixblock_head
+.endm
+
+.macro pixman_composite_src_n_8888_process_pixblock_tail
+.endm
+
+.macro pixman_composite_src_n_8888_process_pixblock_tail_head
+    vst1.32 {d0, d1, d2, d3}, [DST_W, :128]!
+.endm
+
+.macro pixman_composite_src_n_8888_init
+    add         DUMMY, sp, #ARGS_STACK_OFFSET
+    vld1.32     {d0[0]}, [DUMMY]
+    vsli.u64    d0, d0, #32
+    vorr        d1, d0, d0
+    vorr        q1, q0, q0
+.endm
+
+.macro pixman_composite_src_n_8888_cleanup
+.endm
+
+generate_composite_function \
+    pixman_composite_src_n_8888_asm_neon, 0, 0, 32, \
+    FLAG_DST_WRITEONLY, \
+    8, /* number of pixels, processed in a single block */ \
+    0, /* prefetch distance */ \
+    pixman_composite_src_n_8888_init, \
+    pixman_composite_src_n_8888_cleanup, \
+    pixman_composite_src_n_8888_process_pixblock_head, \
+    pixman_composite_src_n_8888_process_pixblock_tail, \
+    pixman_composite_src_n_8888_process_pixblock_tail_head, \
+    0, /* dst_w_basereg */ \
+    0, /* dst_r_basereg */ \
+    0, /* src_basereg   */ \
+    0  /* mask_basereg  */
+
+/******************************************************************************/
+
+.macro pixman_composite_src_8888_8888_process_pixblock_head
+.endm
+
+.macro pixman_composite_src_8888_8888_process_pixblock_tail
+.endm
+
+.macro pixman_composite_src_8888_8888_process_pixblock_tail_head
+    vst1.32 {d0, d1, d2, d3}, [DST_W, :128]!
+    vld1.32 {d0, d1, d2, d3}, [SRC]!
+    cache_preload 8, 8
+.endm
+
+generate_composite_function \
+    pixman_composite_src_8888_8888_asm_neon, 32, 0, 32, \
+    FLAG_DST_WRITEONLY, \
+    8, /* number of pixels, processed in a single block */ \
+    10, /* prefetch distance */ \
+    default_init, \
+    default_cleanup, \
+    pixman_composite_src_8888_8888_process_pixblock_head, \
+    pixman_composite_src_8888_8888_process_pixblock_tail, \
+    pixman_composite_src_8888_8888_process_pixblock_tail_head, \
+    0, /* dst_w_basereg */ \
+    0, /* dst_r_basereg */ \
+    0, /* src_basereg   */ \
+    0  /* mask_basereg  */
+
+/******************************************************************************/
+
+.macro pixman_composite_src_x888_8888_process_pixblock_head
+    vorr     q0, q0, q2
+    vorr     q1, q1, q2
+.endm
+
+.macro pixman_composite_src_x888_8888_process_pixblock_tail
+.endm
+
+.macro pixman_composite_src_x888_8888_process_pixblock_tail_head
+    vst1.32 {d0, d1, d2, d3}, [DST_W, :128]!
+    vld1.32 {d0, d1, d2, d3}, [SRC]!
+    vorr     q0, q0, q2
+    vorr     q1, q1, q2
+    cache_preload 8, 8
+.endm
+
+.macro pixman_composite_src_x888_8888_init
+    vmov.u8  q2, #0xFF
+    vshl.u32 q2, q2, #24
+.endm
+
+generate_composite_function \
+    pixman_composite_src_x888_8888_asm_neon, 32, 0, 32, \
+    FLAG_DST_WRITEONLY, \
+    8, /* number of pixels, processed in a single block */ \
+    10, /* prefetch distance */ \
+    pixman_composite_src_x888_8888_init, \
+    default_cleanup, \
+    pixman_composite_src_x888_8888_process_pixblock_head, \
+    pixman_composite_src_x888_8888_process_pixblock_tail, \
+    pixman_composite_src_x888_8888_process_pixblock_tail_head, \
+    0, /* dst_w_basereg */ \
+    0, /* dst_r_basereg */ \
+    0, /* src_basereg   */ \
+    0  /* mask_basereg  */
+
+/******************************************************************************/
+
+.macro pixman_composite_over_n_8_8888_process_pixblock_head
+    /* expecting deinterleaved source data in {d8, d9, d10, d11} */
+    /* d8 - blue, d9 - green, d10 - red, d11 - alpha */
+    /* and destination data in {d4, d5, d6, d7} */
+    /* mask is in d24 (d25, d26, d27 are unused) */
+
+    /* in */
+    vmull.u8    q0, d24, d8
+    vmull.u8    q1, d24, d9
+    vmull.u8    q6, d24, d10
+    vmull.u8    q7, d24, d11
+    vrshr.u16   q10, q0, #8
+    vrshr.u16   q11, q1, #8
+    vrshr.u16   q12, q6, #8
+    vrshr.u16   q13, q7, #8
+    vraddhn.u16 d0, q0, q10
+    vraddhn.u16 d1, q1, q11
+    vraddhn.u16 d2, q6, q12
+    vraddhn.u16 d3, q7, q13
+    vmvn.8      d24, d3  /* get inverted alpha */
+    /* source:      d0 - blue, d1 - green, d2 - red, d3 - alpha */
+    /* destination: d4 - blue, d5 - green, d6 - red, d7 - alpha */
+    /* now do alpha blending */
+    vmull.u8    q8, d24, d4
+    vmull.u8    q9, d24, d5
+    vmull.u8    q10, d24, d6
+    vmull.u8    q11, d24, d7
+.endm
+
+.macro pixman_composite_over_n_8_8888_process_pixblock_tail
+    vrshr.u16   q14, q8, #8
+    vrshr.u16   q15, q9, #8
+    vrshr.u16   q12, q10, #8
+    vrshr.u16   q13, q11, #8
+    vraddhn.u16 d28, q14, q8
+    vraddhn.u16 d29, q15, q9
+    vraddhn.u16 d30, q12, q10
+    vraddhn.u16 d31, q13, q11
+    vqadd.u8    q14, q0, q14
+    vqadd.u8    q15, q1, q15
+.endm
+
+/* TODO: expand macros and do better instructions scheduling */
+.macro pixman_composite_over_n_8_8888_process_pixblock_tail_head
+    pixman_composite_over_n_8_8888_process_pixblock_tail
+    vst4.8      {d28, d29, d30, d31}, [DST_W, :128]!
+    vld4.8      {d4, d5, d6, d7}, [DST_R, :128]!
+    vld1.8      {d24}, [MASK]!
+    cache_preload 8, 8
+    pixman_composite_over_n_8_8888_process_pixblock_head
+.endm
+
+.macro pixman_composite_over_n_8_8888_init
+    add         DUMMY, sp, #ARGS_STACK_OFFSET
+    vpush       {d8-d15}
+    vld1.32     {d11[0]}, [DUMMY]
+    vdup.8      d8, d11[0]
+    vdup.8      d9, d11[1]
+    vdup.8      d10, d11[2]
+    vdup.8      d11, d11[3]
+.endm
+
+.macro pixman_composite_over_n_8_8888_cleanup
+    vpop        {d8-d15}
+.endm
+
+generate_composite_function \
+    pixman_composite_over_n_8_8888_asm_neon, 0, 8, 32, \
+    FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
+    8, /* number of pixels, processed in a single block */ \
+    5, /* prefetch distance */ \
+    pixman_composite_over_n_8_8888_init, \
+    pixman_composite_over_n_8_8888_cleanup, \
+    pixman_composite_over_n_8_8888_process_pixblock_head, \
+    pixman_composite_over_n_8_8888_process_pixblock_tail, \
+    pixman_composite_over_n_8_8888_process_pixblock_tail_head
+
+/******************************************************************************/
+
+.macro pixman_composite_over_n_8888_8888_ca_process_pixblock_head
+    /*
+     * 'combine_mask_ca' replacement
+     *
+     * input:  solid src (n) in {d8,  d9,  d10, d11}
+     *         dest in          {d4,  d5,  d6,  d7 }
+     *         mask in          {d24, d25, d26, d27}
+     * output: updated src in   {d0,  d1,  d2,  d3 }
+     *         updated mask in  {d24, d25, d26, d3 }
+     */
+    vmull.u8    q0,  d24, d8
+    vmull.u8    q1,  d25, d9
+    vmull.u8    q6,  d26, d10
+    vmull.u8    q7,  d27, d11
+    vmull.u8    q9,  d11, d25
+    vmull.u8    q12, d11, d24
+    vmull.u8    q13, d11, d26
+    vrshr.u16   q8,  q0,  #8
+    vrshr.u16   q10, q1,  #8
+    vrshr.u16   q11, q6,  #8
+    vraddhn.u16 d0,  q0,  q8
+    vraddhn.u16 d1,  q1,  q10
+    vraddhn.u16 d2,  q6,  q11
+    vrshr.u16   q11, q12, #8
+    vrshr.u16   q8,  q9,  #8
+    vrshr.u16   q6,  q13, #8
+    vrshr.u16   q10, q7,  #8
+    vraddhn.u16 d24, q12, q11
+    vraddhn.u16 d25, q9,  q8
+    vraddhn.u16 d26, q13, q6
+    vraddhn.u16 d3,  q7,  q10
+    /*
+     * 'combine_over_ca' replacement
+     *
+     * output: updated dest in {d28, d29, d30, d31}
+     */
+    vmvn.8      d24, d24
+    vmvn.8      d25, d25
+    vmull.u8    q8,  d24, d4
+    vmull.u8    q9,  d25, d5
+    vmvn.8      d26, d26
+    vmvn.8      d27, d3
+    vmull.u8    q10, d26, d6
+    vmull.u8    q11, d27, d7
+.endm
+
+.macro pixman_composite_over_n_8888_8888_ca_process_pixblock_tail
+    /* ... continue 'combine_over_ca' replacement */
+    vrshr.u16   q14, q8,  #8
+    vrshr.u16   q15, q9,  #8
+    vrshr.u16   q6,  q10, #8
+    vrshr.u16   q7,  q11, #8
+    vraddhn.u16 d28, q14, q8
+    vraddhn.u16 d29, q15, q9
+    vraddhn.u16 d30, q6,  q10
+    vraddhn.u16 d31, q7,  q11
+    vqadd.u8    q14, q0,  q14
+    vqadd.u8    q15, q1,  q15
+.endm
+
+.macro pixman_composite_over_n_8888_8888_ca_process_pixblock_tail_head
+        vrshr.u16   q14, q8, #8
+        vrshr.u16   q15, q9, #8
+    vld4.8      {d4, d5, d6, d7}, [DST_R, :128]!
+        vrshr.u16   q6, q10, #8
+        vrshr.u16   q7, q11, #8
+        vraddhn.u16 d28, q14, q8
+        vraddhn.u16 d29, q15, q9
+        vraddhn.u16 d30, q6, q10
+        vraddhn.u16 d31, q7, q11
+    vld4.8      {d24, d25, d26, d27}, [MASK]!
+        vqadd.u8    q14, q0, q14
+        vqadd.u8    q15, q1, q15
+    cache_preload 8, 8
+    pixman_composite_over_n_8888_8888_ca_process_pixblock_head
+    vst4.8      {d28, d29, d30, d31}, [DST_W, :128]!
+.endm
+
+.macro pixman_composite_over_n_8888_8888_ca_init
+    add         DUMMY, sp, #ARGS_STACK_OFFSET
+    vpush       {d8-d15}
+    vld1.32     {d11[0]}, [DUMMY]
+    vdup.8      d8, d11[0]
+    vdup.8      d9, d11[1]
+    vdup.8      d10, d11[2]
+    vdup.8      d11, d11[3]
+.endm
+
+.macro pixman_composite_over_n_8888_8888_ca_cleanup
+    vpop        {d8-d15}
+.endm
+
+generate_composite_function \
+    pixman_composite_over_n_8888_8888_ca_asm_neon, 0, 32, 32, \
+    FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
+    8, /* number of pixels, processed in a single block */ \
+    5, /* prefetch distance */ \
+    pixman_composite_over_n_8888_8888_ca_init, \
+    pixman_composite_over_n_8888_8888_ca_cleanup, \
+    pixman_composite_over_n_8888_8888_ca_process_pixblock_head, \
+    pixman_composite_over_n_8888_8888_ca_process_pixblock_tail, \
+    pixman_composite_over_n_8888_8888_ca_process_pixblock_tail_head
+
+/******************************************************************************/
+
+.macro pixman_composite_add_n_8_8_process_pixblock_head
+    /* expecting source data in {d8, d9, d10, d11} */
+    /* d8 - blue, d9 - green, d10 - red, d11 - alpha */
+    /* and destination data in {d4, d5, d6, d7} */
+    /* mask is in d24, d25, d26, d27 */
+    vmull.u8    q0, d24, d11
+    vmull.u8    q1, d25, d11
+    vmull.u8    q6, d26, d11
+    vmull.u8    q7, d27, d11
+    vrshr.u16   q10, q0, #8
+    vrshr.u16   q11, q1, #8
+    vrshr.u16   q12, q6, #8
+    vrshr.u16   q13, q7, #8
+    vraddhn.u16 d0, q0, q10
+    vraddhn.u16 d1, q1, q11
+    vraddhn.u16 d2, q6, q12
+    vraddhn.u16 d3, q7, q13
+    vqadd.u8    q14, q0, q2
+    vqadd.u8    q15, q1, q3
+.endm
+
+.macro pixman_composite_add_n_8_8_process_pixblock_tail
+.endm
+
+/* TODO: expand macros and do better instructions scheduling */
+.macro pixman_composite_add_n_8_8_process_pixblock_tail_head
+    pixman_composite_add_n_8_8_process_pixblock_tail
+    vst1.8      {d28, d29, d30, d31}, [DST_W, :128]!
+    vld1.8      {d4, d5, d6, d7}, [DST_R, :128]!
+    vld1.8      {d24, d25, d26, d27}, [MASK]!
+    cache_preload 32, 32
+    pixman_composite_add_n_8_8_process_pixblock_head
+.endm
+
+.macro pixman_composite_add_n_8_8_init
+    add         DUMMY, sp, #ARGS_STACK_OFFSET
+    vpush       {d8-d15}
+    vld1.32     {d11[0]}, [DUMMY]
+    vdup.8      d11, d11[3]
+.endm
+
+.macro pixman_composite_add_n_8_8_cleanup
+    vpop        {d8-d15}
+.endm
+
+generate_composite_function \
+    pixman_composite_add_n_8_8_asm_neon, 0, 8, 8, \
+    FLAG_DST_READWRITE, \
+    32, /* number of pixels, processed in a single block */ \
+    5, /* prefetch distance */ \
+    pixman_composite_add_n_8_8_init, \
+    pixman_composite_add_n_8_8_cleanup, \
+    pixman_composite_add_n_8_8_process_pixblock_head, \
+    pixman_composite_add_n_8_8_process_pixblock_tail, \
+    pixman_composite_add_n_8_8_process_pixblock_tail_head
+
+/******************************************************************************/
+
+.macro pixman_composite_add_8_8_8_process_pixblock_head
+    /* expecting source data in {d0, d1, d2, d3} */
+    /* destination data in {d4, d5, d6, d7} */
+    /* mask in {d24, d25, d26, d27} */
+    vmull.u8    q8, d24, d0
+    vmull.u8    q9, d25, d1
+    vmull.u8    q10, d26, d2
+    vmull.u8    q11, d27, d3
+    vrshr.u16   q0, q8, #8
+    vrshr.u16   q1, q9, #8
+    vrshr.u16   q12, q10, #8
+    vrshr.u16   q13, q11, #8
+    vraddhn.u16 d0, q0, q8
+    vraddhn.u16 d1, q1, q9
+    vraddhn.u16 d2, q12, q10
+    vraddhn.u16 d3, q13, q11
+    vqadd.u8    q14, q0, q2
+    vqadd.u8    q15, q1, q3
+.endm
+
+.macro pixman_composite_add_8_8_8_process_pixblock_tail
+.endm
+
+/* TODO: expand macros and do better instructions scheduling */
+.macro pixman_composite_add_8_8_8_process_pixblock_tail_head
+    pixman_composite_add_8_8_8_process_pixblock_tail
+    vst1.8      {d28, d29, d30, d31}, [DST_W, :128]!
+    vld1.8      {d4, d5, d6, d7}, [DST_R, :128]!
+    vld1.8      {d24, d25, d26, d27}, [MASK]!
+    vld1.8      {d0, d1, d2, d3}, [SRC]!
+    cache_preload 32, 32
+    pixman_composite_add_8_8_8_process_pixblock_head
+.endm
+
+.macro pixman_composite_add_8_8_8_init
+.endm
+
+.macro pixman_composite_add_8_8_8_cleanup
+.endm
+
+generate_composite_function \
+    pixman_composite_add_8_8_8_asm_neon, 8, 8, 8, \
+    FLAG_DST_READWRITE, \
+    32, /* number of pixels, processed in a single block */ \
+    5, /* prefetch distance */ \
+    pixman_composite_add_8_8_8_init, \
+    pixman_composite_add_8_8_8_cleanup, \
+    pixman_composite_add_8_8_8_process_pixblock_head, \
+    pixman_composite_add_8_8_8_process_pixblock_tail, \
+    pixman_composite_add_8_8_8_process_pixblock_tail_head
+
+/******************************************************************************/
+
+.macro pixman_composite_add_8888_8888_8888_process_pixblock_head
+    /* expecting source data in {d0, d1, d2, d3} */
+    /* destination data in {d4, d5, d6, d7} */
+    /* mask in {d24, d25, d26, d27} */
+    vmull.u8    q8, d27, d0
+    vmull.u8    q9, d27, d1
+    vmull.u8    q10, d27, d2
+    vmull.u8    q11, d27, d3
+    vrshr.u16   q0, q8, #8
+    vrshr.u16   q1, q9, #8
+    vrshr.u16   q12, q10, #8
+    vrshr.u16   q13, q11, #8
+    vraddhn.u16 d0, q0, q8
+    vraddhn.u16 d1, q1, q9
+    vraddhn.u16 d2, q12, q10
+    vraddhn.u16 d3, q13, q11
+    vqadd.u8    q14, q0, q2
+    vqadd.u8    q15, q1, q3
+.endm
+
+.macro pixman_composite_add_8888_8888_8888_process_pixblock_tail
+.endm
+
+/* TODO: expand macros and do better instructions scheduling */
+.macro pixman_composite_add_8888_8888_8888_process_pixblock_tail_head
+    pixman_composite_add_8888_8888_8888_process_pixblock_tail
+    vst4.8      {d28, d29, d30, d31}, [DST_W, :128]!
+    vld4.8      {d4, d5, d6, d7}, [DST_R, :128]!
+    vld4.8      {d24, d25, d26, d27}, [MASK]!
+    vld4.8      {d0, d1, d2, d3}, [SRC]!
+    cache_preload 8, 8
+    pixman_composite_add_8888_8888_8888_process_pixblock_head
+.endm
+
+generate_composite_function \
+    pixman_composite_add_8888_8888_8888_asm_neon, 32, 32, 32, \
+    FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
+    8, /* number of pixels, processed in a single block */ \
+    10, /* prefetch distance */ \
+    default_init, \
+    default_cleanup, \
+    pixman_composite_add_8888_8888_8888_process_pixblock_head, \
+    pixman_composite_add_8888_8888_8888_process_pixblock_tail, \
+    pixman_composite_add_8888_8888_8888_process_pixblock_tail_head
+
+generate_composite_function_single_scanline \
+    pixman_composite_scanline_add_mask_asm_neon, 32, 32, 32, \
+    FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
+    8, /* number of pixels, processed in a single block */ \
+    default_init, \
+    default_cleanup, \
+    pixman_composite_add_8888_8888_8888_process_pixblock_head, \
+    pixman_composite_add_8888_8888_8888_process_pixblock_tail, \
+    pixman_composite_add_8888_8888_8888_process_pixblock_tail_head
+
+/******************************************************************************/
+
+.macro pixman_composite_over_8888_n_8888_process_pixblock_head
+    /* expecting source data in {d0, d1, d2, d3} */
+    /* destination data in {d4, d5, d6, d7} */
+    /* solid mask is in d15 */
+
+    /* 'in' */
+    vmull.u8    q8, d15, d3
+    vmull.u8    q6, d15, d2
+    vmull.u8    q5, d15, d1
+    vmull.u8    q4, d15, d0
+    vrshr.u16   q13, q8, #8
+    vrshr.u16   q12, q6, #8
+    vrshr.u16   q11, q5, #8
+    vrshr.u16   q10, q4, #8
+    vraddhn.u16 d3, q8, q13
+    vraddhn.u16 d2, q6, q12
+    vraddhn.u16 d1, q5, q11
+    vraddhn.u16 d0, q4, q10
+    vmvn.8      d24, d3  /* get inverted alpha */
+    /* now do alpha blending */
+    vmull.u8    q8, d24, d4
+    vmull.u8    q9, d24, d5
+    vmull.u8    q10, d24, d6
+    vmull.u8    q11, d24, d7
+.endm
+
+.macro pixman_composite_over_8888_n_8888_process_pixblock_tail
+    vrshr.u16   q14, q8, #8
+    vrshr.u16   q15, q9, #8
+    vrshr.u16   q12, q10, #8
+    vrshr.u16   q13, q11, #8
+    vraddhn.u16 d28, q14, q8
+    vraddhn.u16 d29, q15, q9
+    vraddhn.u16 d30, q12, q10
+    vraddhn.u16 d31, q13, q11
+    vqadd.u8    q14, q0, q14
+    vqadd.u8    q15, q1, q15
+.endm
+
+/* TODO: expand macros and do better instructions scheduling */
+.macro pixman_composite_over_8888_n_8888_process_pixblock_tail_head
+    vld4.8     {d4, d5, d6, d7}, [DST_R, :128]!
+    pixman_composite_over_8888_n_8888_process_pixblock_tail
+    vld4.8     {d0, d1, d2, d3}, [SRC]!
+    cache_preload 8, 8
+    pixman_composite_over_8888_n_8888_process_pixblock_head
+    vst4.8     {d28, d29, d30, d31}, [DST_W, :128]!
+.endm
+
+.macro pixman_composite_over_8888_n_8888_init
+    add         DUMMY, sp, #48
+    vpush       {d8-d15}
+    vld1.32     {d15[0]}, [DUMMY]
+    vdup.8      d15, d15[3]
+.endm
+
+.macro pixman_composite_over_8888_n_8888_cleanup
+    vpop        {d8-d15}
+.endm
+
+generate_composite_function \
+    pixman_composite_over_8888_n_8888_asm_neon, 32, 0, 32, \
+    FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
+    8, /* number of pixels, processed in a single block */ \
+    5, /* prefetch distance */ \
+    pixman_composite_over_8888_n_8888_init, \
+    pixman_composite_over_8888_n_8888_cleanup, \
+    pixman_composite_over_8888_n_8888_process_pixblock_head, \
+    pixman_composite_over_8888_n_8888_process_pixblock_tail, \
+    pixman_composite_over_8888_n_8888_process_pixblock_tail_head
+
+/******************************************************************************/
+
+/* TODO: expand macros and do better instructions scheduling */
+.macro pixman_composite_over_8888_8888_8888_process_pixblock_tail_head
+    vld4.8     {d4, d5, d6, d7}, [DST_R, :128]!
+    pixman_composite_over_8888_n_8888_process_pixblock_tail
+    vld4.8     {d0, d1, d2, d3}, [SRC]!
+    cache_preload 8, 8
+    vld4.8     {d12, d13, d14, d15}, [MASK]!
+    pixman_composite_over_8888_n_8888_process_pixblock_head
+    vst4.8     {d28, d29, d30, d31}, [DST_W, :128]!
+.endm
+
+.macro pixman_composite_over_8888_8888_8888_init
+    vpush       {d8-d15}
+.endm
+
+.macro pixman_composite_over_8888_8888_8888_cleanup
+    vpop        {d8-d15}
+.endm
+
+generate_composite_function \
+    pixman_composite_over_8888_8888_8888_asm_neon, 32, 32, 32, \
+    FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
+    8, /* number of pixels, processed in a single block */ \
+    5, /* prefetch distance */ \
+    pixman_composite_over_8888_8888_8888_init, \
+    pixman_composite_over_8888_8888_8888_cleanup, \
+    pixman_composite_over_8888_n_8888_process_pixblock_head, \
+    pixman_composite_over_8888_n_8888_process_pixblock_tail, \
+    pixman_composite_over_8888_8888_8888_process_pixblock_tail_head \
+    28, /* dst_w_basereg */ \
+    4,  /* dst_r_basereg */ \
+    0,  /* src_basereg   */ \
+    12  /* mask_basereg  */
+
+generate_composite_function_single_scanline \
+    pixman_composite_scanline_over_mask_asm_neon, 32, 32, 32, \
+    FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
+    8, /* number of pixels, processed in a single block */ \
+    pixman_composite_over_8888_8888_8888_init, \
+    pixman_composite_over_8888_8888_8888_cleanup, \
+    pixman_composite_over_8888_n_8888_process_pixblock_head, \
+    pixman_composite_over_8888_n_8888_process_pixblock_tail, \
+    pixman_composite_over_8888_8888_8888_process_pixblock_tail_head \
+    28, /* dst_w_basereg */ \
+    4,  /* dst_r_basereg */ \
+    0,  /* src_basereg   */ \
+    12  /* mask_basereg  */
+
+/******************************************************************************/
+
+/* TODO: expand macros and do better instructions scheduling */
+.macro pixman_composite_over_8888_8_8888_process_pixblock_tail_head
+    vld4.8     {d4, d5, d6, d7}, [DST_R, :128]!
+    pixman_composite_over_8888_n_8888_process_pixblock_tail
+    vld4.8     {d0, d1, d2, d3}, [SRC]!
+    cache_preload 8, 8
+    vld1.8     {d15}, [MASK]!
+    pixman_composite_over_8888_n_8888_process_pixblock_head
+    vst4.8     {d28, d29, d30, d31}, [DST_W, :128]!
+.endm
+
+.macro pixman_composite_over_8888_8_8888_init
+    vpush       {d8-d15}
+.endm
+
+.macro pixman_composite_over_8888_8_8888_cleanup
+    vpop        {d8-d15}
+.endm
+
+generate_composite_function \
+    pixman_composite_over_8888_8_8888_asm_neon, 32, 8, 32, \
+    FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
+    8, /* number of pixels, processed in a single block */ \
+    5, /* prefetch distance */ \
+    pixman_composite_over_8888_8_8888_init, \
+    pixman_composite_over_8888_8_8888_cleanup, \
+    pixman_composite_over_8888_n_8888_process_pixblock_head, \
+    pixman_composite_over_8888_n_8888_process_pixblock_tail, \
+    pixman_composite_over_8888_8_8888_process_pixblock_tail_head \
+    28, /* dst_w_basereg */ \
+    4,  /* dst_r_basereg */ \
+    0,  /* src_basereg   */ \
+    15  /* mask_basereg  */
+
+/******************************************************************************/
+
+.macro pixman_composite_src_0888_0888_process_pixblock_head
+.endm
+
+.macro pixman_composite_src_0888_0888_process_pixblock_tail
+.endm
+
+.macro pixman_composite_src_0888_0888_process_pixblock_tail_head
+    vst3.8 {d0, d1, d2}, [DST_W]!
+    vld3.8 {d0, d1, d2}, [SRC]!
+    cache_preload 8, 8
+.endm
+
+generate_composite_function \
+    pixman_composite_src_0888_0888_asm_neon, 24, 0, 24, \
+    FLAG_DST_WRITEONLY, \
+    8, /* number of pixels, processed in a single block */ \
+    10, /* prefetch distance */ \
+    default_init, \
+    default_cleanup, \
+    pixman_composite_src_0888_0888_process_pixblock_head, \
+    pixman_composite_src_0888_0888_process_pixblock_tail, \
+    pixman_composite_src_0888_0888_process_pixblock_tail_head, \
+    0, /* dst_w_basereg */ \
+    0, /* dst_r_basereg */ \
+    0, /* src_basereg   */ \
+    0  /* mask_basereg  */
+
+/******************************************************************************/
+
+.macro pixman_composite_src_0888_8888_rev_process_pixblock_head
+    vswp   d0, d2
+.endm
+
+.macro pixman_composite_src_0888_8888_rev_process_pixblock_tail
+.endm
+
+.macro pixman_composite_src_0888_8888_rev_process_pixblock_tail_head
+    vst4.8 {d0, d1, d2, d3}, [DST_W]!
+    vld3.8 {d0, d1, d2}, [SRC]!
+    vswp   d0, d2
+    cache_preload 8, 8
+.endm
+
+.macro pixman_composite_src_0888_8888_rev_init
+    veor   d3, d3, d3
+.endm
+
+generate_composite_function \
+    pixman_composite_src_0888_8888_rev_asm_neon, 24, 0, 32, \
+    FLAG_DST_WRITEONLY | FLAG_DEINTERLEAVE_32BPP, \
+    8, /* number of pixels, processed in a single block */ \
+    10, /* prefetch distance */ \
+    pixman_composite_src_0888_8888_rev_init, \
+    default_cleanup, \
+    pixman_composite_src_0888_8888_rev_process_pixblock_head, \
+    pixman_composite_src_0888_8888_rev_process_pixblock_tail, \
+    pixman_composite_src_0888_8888_rev_process_pixblock_tail_head, \
+    0, /* dst_w_basereg */ \
+    0, /* dst_r_basereg */ \
+    0, /* src_basereg   */ \
+    0  /* mask_basereg  */
+
+/******************************************************************************/
+
+.macro pixman_composite_src_0888_0565_rev_process_pixblock_head
+    vshll.u8    q8, d1, #8
+    vshll.u8    q9, d2, #8
+.endm
+
+.macro pixman_composite_src_0888_0565_rev_process_pixblock_tail
+    vshll.u8    q14, d0, #8
+    vsri.u16    q14, q8, #5
+    vsri.u16    q14, q9, #11
+.endm
+
+.macro pixman_composite_src_0888_0565_rev_process_pixblock_tail_head
+        vshll.u8    q14, d0, #8
+    vld3.8 {d0, d1, d2}, [SRC]!
+        vsri.u16    q14, q8, #5
+        vsri.u16    q14, q9, #11
+    vshll.u8    q8, d1, #8
+        vst1.16 {d28, d29}, [DST_W, :128]!
+    vshll.u8    q9, d2, #8
+.endm
+
+generate_composite_function \
+    pixman_composite_src_0888_0565_rev_asm_neon, 24, 0, 16, \
+    FLAG_DST_WRITEONLY, \
+    8, /* number of pixels, processed in a single block */ \
+    10, /* prefetch distance */ \
+    default_init, \
+    default_cleanup, \
+    pixman_composite_src_0888_0565_rev_process_pixblock_head, \
+    pixman_composite_src_0888_0565_rev_process_pixblock_tail, \
+    pixman_composite_src_0888_0565_rev_process_pixblock_tail_head, \
+    28, /* dst_w_basereg */ \
+    0, /* dst_r_basereg */ \
+    0, /* src_basereg   */ \
+    0  /* mask_basereg  */
+
+/******************************************************************************/
+
+.macro pixman_composite_src_pixbuf_8888_process_pixblock_head
+    vmull.u8    q8, d3, d0
+    vmull.u8    q9, d3, d1
+    vmull.u8    q10, d3, d2
+.endm
+
+.macro pixman_composite_src_pixbuf_8888_process_pixblock_tail
+    vrshr.u16   q11, q8, #8
+    vswp        d3, d31
+    vrshr.u16   q12, q9, #8
+    vrshr.u16   q13, q10, #8
+    vraddhn.u16 d30, q11, q8
+    vraddhn.u16 d29, q12, q9
+    vraddhn.u16 d28, q13, q10
+.endm
+
+.macro pixman_composite_src_pixbuf_8888_process_pixblock_tail_head
+        vrshr.u16   q11, q8, #8
+        vswp        d3, d31
+        vrshr.u16   q12, q9, #8
+        vrshr.u16   q13, q10, #8
+    vld4.8 {d0, d1, d2, d3}, [SRC]!
+        vraddhn.u16 d30, q11, q8
+                                    PF add PF_X, PF_X, #8
+                                    PF tst PF_CTL, #0xF
+                                    PF addne PF_X, PF_X, #8
+                                    PF subne PF_CTL, PF_CTL, #1
+        vraddhn.u16 d29, q12, q9
+        vraddhn.u16 d28, q13, q10
+    vmull.u8    q8, d3, d0
+    vmull.u8    q9, d3, d1
+    vmull.u8    q10, d3, d2
+        vst4.8 {d28, d29, d30, d31}, [DST_W, :128]!
+                                    PF cmp PF_X, ORIG_W
+                                    PF pld, [PF_SRC, PF_X, lsl #src_bpp_shift]
+                                    PF subge PF_X, PF_X, ORIG_W
+                                    PF subges PF_CTL, PF_CTL, #0x10
+                                    PF ldrgeb DUMMY, [PF_SRC, SRC_STRIDE, lsl #src_bpp_shift]!
+.endm
+
+generate_composite_function \
+    pixman_composite_src_pixbuf_8888_asm_neon, 32, 0, 32, \
+    FLAG_DST_WRITEONLY | FLAG_DEINTERLEAVE_32BPP, \
+    8, /* number of pixels, processed in a single block */ \
+    10, /* prefetch distance */ \
+    default_init, \
+    default_cleanup, \
+    pixman_composite_src_pixbuf_8888_process_pixblock_head, \
+    pixman_composite_src_pixbuf_8888_process_pixblock_tail, \
+    pixman_composite_src_pixbuf_8888_process_pixblock_tail_head, \
+    28, /* dst_w_basereg */ \
+    0, /* dst_r_basereg */ \
+    0, /* src_basereg   */ \
+    0  /* mask_basereg  */
diff --git a/pixman/pixman/pixman-arm-neon-asm.h b/pixman/pixman/pixman-arm-neon-asm.h
new file mode 100644
index 000000000..56c3faebf
--- /dev/null
+++ b/pixman/pixman/pixman-arm-neon-asm.h
@@ -0,0 +1,906 @@
+/*
+ * Copyright © 2009 Nokia Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ *
+ * Author:  Siarhei Siamashka (siarhei.siamashka@nokia.com)
+ */
+
+/*
+ * This file contains a macro ('generate_composite_function') which can
+ * construct 2D image processing functions, based on a common template.
+ * Any combinations of source, destination and mask images with 8bpp,
+ * 16bpp, 24bpp, 32bpp color formats are supported.
+ *
+ * This macro takes care of:
+ *  - handling of leading and trailing unaligned pixels
+ *  - doing most of the work related to L2 cache preload
+ *  - encourages the use of software pipelining for better instructions
+ *    scheduling
+ *
+ * The user of this macro has to provide some configuration parameters
+ * (bit depths for the images, prefetch distance, etc.) and a set of
+ * macros, which should implement basic code chunks responsible for
+ * pixels processing. See 'pixman-arm-neon-asm.S' file for the usage
+ * examples.
+ *
+ * TODO:
+ *  - try overlapped pixel method (from Ian Rickards) when processing
+ *    exactly two blocks of pixels
+ *  - maybe add an option to do reverse scanline processing
+ */
+
+/*
+ * Bit flags for 'generate_composite_function' macro which are used
+ * to tune generated functions behavior.
+ */
+.set FLAG_DST_WRITEONLY,       0
+.set FLAG_DST_READWRITE,       1
+.set FLAG_DEINTERLEAVE_32BPP,  2
+
+/*
+ * Offset in stack where mask and source pointer/stride can be accessed
+ * from 'init' macro. This is useful for doing special handling for solid mask.
+ */
+.set ARGS_STACK_OFFSET,        40
+
+/*
+ * Constants for selecting preferable prefetch type.
+ */
+.set PREFETCH_TYPE_NONE,       0 /* No prefetch at all */
+.set PREFETCH_TYPE_SIMPLE,     1 /* A simple, fixed-distance-ahead prefetch */
+.set PREFETCH_TYPE_ADVANCED,   2 /* Advanced fine-grained prefetch */
+
+/*
+ * Definitions of supplementary pixld/pixst macros (for partial load/store of
+ * pixel data).
+ */
+
+.macro pixldst1 op, elem_size, reg1, mem_operand, abits
+.if abits > 0
+    op&.&elem_size {d&reg1}, [&mem_operand&, :&abits&]!
+.else
+    op&.&elem_size {d&reg1}, [&mem_operand&]!
+.endif
+.endm
+
+.macro pixldst2 op, elem_size, reg1, reg2, mem_operand, abits
+.if abits > 0
+    op&.&elem_size {d&reg1, d&reg2}, [&mem_operand&, :&abits&]!
+.else
+    op&.&elem_size {d&reg1, d&reg2}, [&mem_operand&]!
+.endif
+.endm
+
+.macro pixldst4 op, elem_size, reg1, reg2, reg3, reg4, mem_operand, abits
+.if abits > 0
+    op&.&elem_size {d&reg1, d&reg2, d&reg3, d&reg4}, [&mem_operand&, :&abits&]!
+.else
+    op&.&elem_size {d&reg1, d&reg2, d&reg3, d&reg4}, [&mem_operand&]!
+.endif
+.endm
+
+.macro pixldst0 op, elem_size, reg1, idx, mem_operand, abits
+    op&.&elem_size {d&reg1[idx]}, [&mem_operand&]!
+.endm
+
+.macro pixldst3 op, elem_size, reg1, reg2, reg3, mem_operand
+    op&.&elem_size {d&reg1, d&reg2, d&reg3}, [&mem_operand&]!
+.endm
+
+.macro pixldst30 op, elem_size, reg1, reg2, reg3, idx, mem_operand
+    op&.&elem_size {d&reg1[idx], d&reg2[idx], d&reg3[idx]}, [&mem_operand&]!
+.endm
+
+.macro pixldst numbytes, op, elem_size, basereg, mem_operand, abits
+.if numbytes == 32
+    pixldst4 op, elem_size, %(basereg+4), %(basereg+5), \
+                              %(basereg+6), %(basereg+7), mem_operand, abits
+.elseif numbytes == 16
+    pixldst2 op, elem_size, %(basereg+2), %(basereg+3), mem_operand, abits
+.elseif numbytes == 8
+    pixldst1 op, elem_size, %(basereg+1), mem_operand, abits
+.elseif numbytes == 4
+    .if !RESPECT_STRICT_ALIGNMENT || (elem_size == 32)
+        pixldst0 op, 32, %(basereg+0), 1, mem_operand, abits
+    .elseif elem_size == 16
+        pixldst0 op, 16, %(basereg+0), 2, mem_operand, abits
+        pixldst0 op, 16, %(basereg+0), 3, mem_operand, abits
+    .else
+        pixldst0 op, 8, %(basereg+0), 4, mem_operand, abits
+        pixldst0 op, 8, %(basereg+0), 5, mem_operand, abits
+        pixldst0 op, 8, %(basereg+0), 6, mem_operand, abits
+        pixldst0 op, 8, %(basereg+0), 7, mem_operand, abits
+    .endif
+.elseif numbytes == 2
+    .if !RESPECT_STRICT_ALIGNMENT || (elem_size == 16)
+        pixldst0 op, 16, %(basereg+0), 1, mem_operand, abits
+    .else
+        pixldst0 op, 8, %(basereg+0), 2, mem_operand, abits
+        pixldst0 op, 8, %(basereg+0), 3, mem_operand, abits
+    .endif
+.elseif numbytes == 1
+    pixldst0 op, 8, %(basereg+0), 1, mem_operand, abits
+.else
+    .error "unsupported size: numbytes"
+.endif
+.endm
+
+.macro pixld numpix, bpp, basereg, mem_operand, abits=0
+.if bpp > 0
+.if (bpp == 32) && (numpix == 8) && (DEINTERLEAVE_32BPP_ENABLED != 0)
+    pixldst4 vld4, 8, %(basereg+4), %(basereg+5), \
+                      %(basereg+6), %(basereg+7), mem_operand, abits
+.elseif (bpp == 24) && (numpix == 8)
+    pixldst3 vld3, 8, %(basereg+3), %(basereg+4), %(basereg+5), mem_operand
+.elseif (bpp == 24) && (numpix == 4)
+    pixldst30 vld3, 8, %(basereg+0), %(basereg+1), %(basereg+2), 4, mem_operand
+    pixldst30 vld3, 8, %(basereg+0), %(basereg+1), %(basereg+2), 5, mem_operand
+    pixldst30 vld3, 8, %(basereg+0), %(basereg+1), %(basereg+2), 6, mem_operand
+    pixldst30 vld3, 8, %(basereg+0), %(basereg+1), %(basereg+2), 7, mem_operand
+.elseif (bpp == 24) && (numpix == 2)
+    pixldst30 vld3, 8, %(basereg+0), %(basereg+1), %(basereg+2), 2, mem_operand
+    pixldst30 vld3, 8, %(basereg+0), %(basereg+1), %(basereg+2), 3, mem_operand
+.elseif (bpp == 24) && (numpix == 1)
+    pixldst30 vld3, 8, %(basereg+0), %(basereg+1), %(basereg+2), 1, mem_operand
+.else
+    pixldst %(numpix * bpp / 8), vld1, %(bpp), basereg, mem_operand, abits
+.endif
+.endif
+.endm
+
+.macro pixst numpix, bpp, basereg, mem_operand, abits=0
+.if bpp > 0
+.if (bpp == 32) && (numpix == 8) && (DEINTERLEAVE_32BPP_ENABLED != 0)
+    pixldst4 vst4, 8, %(basereg+4), %(basereg+5), \
+                      %(basereg+6), %(basereg+7), mem_operand, abits
+.elseif (bpp == 24) && (numpix == 8)
+    pixldst3 vst3, 8, %(basereg+3), %(basereg+4), %(basereg+5), mem_operand
+.elseif (bpp == 24) && (numpix == 4)
+    pixldst30 vst3, 8, %(basereg+0), %(basereg+1), %(basereg+2), 4, mem_operand
+    pixldst30 vst3, 8, %(basereg+0), %(basereg+1), %(basereg+2), 5, mem_operand
+    pixldst30 vst3, 8, %(basereg+0), %(basereg+1), %(basereg+2), 6, mem_operand
+    pixldst30 vst3, 8, %(basereg+0), %(basereg+1), %(basereg+2), 7, mem_operand
+.elseif (bpp == 24) && (numpix == 2)
+    pixldst30 vst3, 8, %(basereg+0), %(basereg+1), %(basereg+2), 2, mem_operand
+    pixldst30 vst3, 8, %(basereg+0), %(basereg+1), %(basereg+2), 3, mem_operand
+.elseif (bpp == 24) && (numpix == 1)
+    pixldst30 vst3, 8, %(basereg+0), %(basereg+1), %(basereg+2), 1, mem_operand
+.else
+    pixldst %(numpix * bpp / 8), vst1, %(bpp), basereg, mem_operand, abits
+.endif
+.endif
+.endm
+
+.macro pixld_a numpix, bpp, basereg, mem_operand
+.if (bpp * numpix) <= 128
+    pixld numpix, bpp, basereg, mem_operand, %(bpp * numpix)
+.else
+    pixld numpix, bpp, basereg, mem_operand, 128
+.endif
+.endm
+
+.macro pixst_a numpix, bpp, basereg, mem_operand
+.if (bpp * numpix) <= 128
+    pixst numpix, bpp, basereg, mem_operand, %(bpp * numpix)
+.else
+    pixst numpix, bpp, basereg, mem_operand, 128
+.endif
+.endm
+
+.macro vuzp8 reg1, reg2
+    vuzp.8 d&reg1, d&reg2
+.endm
+
+.macro vzip8 reg1, reg2
+    vzip.8 d&reg1, d&reg2
+.endm
+
+/* deinterleave B, G, R, A channels for eight 32bpp pixels in 4 registers */
+.macro pixdeinterleave bpp, basereg
+.if (bpp == 32) && (DEINTERLEAVE_32BPP_ENABLED != 0)
+    vuzp8 %(basereg+0), %(basereg+1)
+    vuzp8 %(basereg+2), %(basereg+3)
+    vuzp8 %(basereg+1), %(basereg+3)
+    vuzp8 %(basereg+0), %(basereg+2)
+.endif
+.endm
+
+/* interleave B, G, R, A channels for eight 32bpp pixels in 4 registers */
+.macro pixinterleave bpp, basereg
+.if (bpp == 32) && (DEINTERLEAVE_32BPP_ENABLED != 0)
+    vzip8 %(basereg+0), %(basereg+2)
+    vzip8 %(basereg+1), %(basereg+3)
+    vzip8 %(basereg+2), %(basereg+3)
+    vzip8 %(basereg+0), %(basereg+1)
+.endif
+.endm
+
+/*
+ * This is a macro for implementing cache preload. The main idea is that
+ * cache preload logic is mostly independent from the rest of pixels
+ * processing code. It starts at the top left pixel and moves forward
+ * across pixels and can jump across scanlines. Prefetch distance is
+ * handled in an 'incremental' way: it starts from 0 and advances to the
+ * optimal distance over time. After reaching optimal prefetch distance,
+ * it is kept constant. There are some checks which prevent prefetching
+ * unneeded pixel lines below the image (but it still can prefetch a bit
+ * more data on the right side of the image - not a big issue and may
+ * be actually helpful when rendering text glyphs). Additional trick is
+ * the use of LDR instruction for prefetch instead of PLD when moving to
+ * the next line, the point is that we have a high chance of getting TLB
+ * miss in this case, and PLD would be useless.
+ *
+ * This sounds like it may introduce a noticeable overhead (when working with
+ * fully cached data). But in reality, due to having a separate pipeline and
+ * instruction queue for NEON unit in ARM Cortex-A8, normal ARM code can
+ * execute simultaneously with NEON and be completely shadowed by it. Thus
+ * we get no performance overhead at all (*). This looks like a very nice
+ * feature of Cortex-A8, if used wisely. We don't have a hardware prefetcher,
+ * but still can implement some rather advanced prefetch logic in sofware
+ * for almost zero cost!
+ *
+ * (*) The overhead of the prefetcher is visible when running some trivial
+ * pixels processing like simple copy. Anyway, having prefetch is a must
+ * when working with the graphics data.
+ */
+.macro PF a, x:vararg
+.if (PREFETCH_TYPE_CURRENT == PREFETCH_TYPE_ADVANCED)
+    a x
+.endif
+.endm
+
+.macro cache_preload std_increment, boost_increment
+.if (src_bpp_shift >= 0) || (dst_r_bpp != 0) || (mask_bpp_shift >= 0)
+.if regs_shortage
+    PF ldr ORIG_W, [sp] /* If we are short on regs, ORIG_W is kept on stack */
+.endif
+.if std_increment != 0
+    PF add PF_X, PF_X, #std_increment
+.endif
+    PF tst PF_CTL, #0xF
+    PF addne PF_X, PF_X, #boost_increment
+    PF subne PF_CTL, PF_CTL, #1
+    PF cmp PF_X, ORIG_W
+.if src_bpp_shift >= 0
+    PF pld, [PF_SRC, PF_X, lsl #src_bpp_shift]
+.endif
+.if dst_r_bpp != 0
+    PF pld, [PF_DST, PF_X, lsl #dst_bpp_shift]
+.endif
+.if mask_bpp_shift >= 0
+    PF pld, [PF_MASK, PF_X, lsl #mask_bpp_shift]
+.endif
+    PF subge PF_X, PF_X, ORIG_W
+    PF subges PF_CTL, PF_CTL, #0x10
+.if src_bpp_shift >= 0
+    PF ldrgeb DUMMY, [PF_SRC, SRC_STRIDE, lsl #src_bpp_shift]!
+.endif
+.if dst_r_bpp != 0
+    PF ldrgeb DUMMY, [PF_DST, DST_STRIDE, lsl #dst_bpp_shift]!
+.endif
+.if mask_bpp_shift >= 0
+    PF ldrgeb DUMMY, [PF_MASK, MASK_STRIDE, lsl #mask_bpp_shift]!
+.endif
+.endif
+.endm
+
+.macro cache_preload_simple
+.if (PREFETCH_TYPE_CURRENT == PREFETCH_TYPE_SIMPLE)
+.if src_bpp > 0
+    pld [SRC, #(PREFETCH_DISTANCE_SIMPLE * src_bpp / 8)]
+.endif
+.if dst_r_bpp > 0
+    pld [DST_R, #(PREFETCH_DISTANCE_SIMPLE * dst_r_bpp / 8)]
+.endif
+.if mask_bpp > 0
+    pld [MASK, #(PREFETCH_DISTANCE_SIMPLE * mask_bpp / 8)]
+.endif
+.endif
+.endm
+
+/*
+ * Macro which is used to process leading pixels until destination
+ * pointer is properly aligned (at 16 bytes boundary). When destination
+ * buffer uses 16bpp format, this is unnecessary, or even pointless.
+ */
+.macro ensure_destination_ptr_alignment process_pixblock_head, \
+                                        process_pixblock_tail, \
+                                        process_pixblock_tail_head
+.if dst_w_bpp != 24
+    tst         DST_R, #0xF
+    beq         2f
+
+.irp lowbit, 1, 2, 4, 8, 16
+local skip1
+.if (dst_w_bpp <= (lowbit * 8)) && ((lowbit * 8) < (pixblock_size * dst_w_bpp))
+.if lowbit < 16 /* we don't need more than 16-byte alignment */
+    tst         DST_R, #lowbit
+    beq         1f
+.endif
+    pixld       (lowbit * 8 / dst_w_bpp), src_bpp, src_basereg, SRC
+    pixld       (lowbit * 8 / dst_w_bpp), mask_bpp, mask_basereg, MASK
+.if dst_r_bpp > 0
+    pixld_a     (lowbit * 8 / dst_r_bpp), dst_r_bpp, dst_r_basereg, DST_R
+.else
+    add         DST_R, DST_R, #lowbit
+.endif
+    PF add      PF_X, PF_X, #(lowbit * 8 / dst_w_bpp)
+    sub         W, W, #(lowbit * 8 / dst_w_bpp)
+1:
+.endif
+.endr
+    pixdeinterleave src_bpp, src_basereg
+    pixdeinterleave mask_bpp, mask_basereg
+    pixdeinterleave dst_r_bpp, dst_r_basereg
+
+    process_pixblock_head
+    cache_preload 0, pixblock_size
+    cache_preload_simple
+    process_pixblock_tail
+
+    pixinterleave dst_w_bpp, dst_w_basereg
+.irp lowbit, 1, 2, 4, 8, 16
+.if (dst_w_bpp <= (lowbit * 8)) && ((lowbit * 8) < (pixblock_size * dst_w_bpp))
+.if lowbit < 16 /* we don't need more than 16-byte alignment */
+    tst         DST_W, #lowbit
+    beq         1f
+.endif
+    pixst_a     (lowbit * 8 / dst_w_bpp), dst_w_bpp, dst_w_basereg, DST_W
+1:
+.endif
+.endr
+.endif
+2:
+.endm
+
+/*
+ * Special code for processing up to (pixblock_size - 1) remaining
+ * trailing pixels. As SIMD processing performs operation on
+ * pixblock_size pixels, anything smaller than this has to be loaded
+ * and stored in a special way. Loading and storing of pixel data is
+ * performed in such a way that we fill some 'slots' in the NEON
+ * registers (some slots naturally are unused), then perform compositing
+ * operation as usual. In the end, the data is taken from these 'slots'
+ * and saved to memory.
+ *
+ * cache_preload_flag - allows to suppress prefetch if
+ *                      set to 0
+ * dst_aligned_flag   - selects whether destination buffer
+ *                      is aligned
+ */
+.macro process_trailing_pixels cache_preload_flag, \
+                               dst_aligned_flag, \
+                               process_pixblock_head, \
+                               process_pixblock_tail, \
+                               process_pixblock_tail_head
+    tst         W, #(pixblock_size - 1)
+    beq         2f
+.irp chunk_size, 16, 8, 4, 2, 1
+.if pixblock_size > chunk_size
+    tst         W, #chunk_size
+    beq         1f
+    pixld       chunk_size, src_bpp, src_basereg, SRC
+    pixld       chunk_size, mask_bpp, mask_basereg, MASK
+.if dst_aligned_flag != 0
+    pixld_a     chunk_size, dst_r_bpp, dst_r_basereg, DST_R
+.else
+    pixld       chunk_size, dst_r_bpp, dst_r_basereg, DST_R
+.endif
+.if cache_preload_flag != 0
+    PF add      PF_X, PF_X, #chunk_size
+.endif
+1:
+.endif
+.endr
+    pixdeinterleave src_bpp, src_basereg
+    pixdeinterleave mask_bpp, mask_basereg
+    pixdeinterleave dst_r_bpp, dst_r_basereg
+
+    process_pixblock_head
+.if cache_preload_flag != 0
+    cache_preload 0, pixblock_size
+    cache_preload_simple
+.endif
+    process_pixblock_tail
+    pixinterleave dst_w_bpp, dst_w_basereg
+.irp chunk_size, 16, 8, 4, 2, 1
+.if pixblock_size > chunk_size
+    tst         W, #chunk_size
+    beq         1f
+.if dst_aligned_flag != 0
+    pixst_a     chunk_size, dst_w_bpp, dst_w_basereg, DST_W
+.else
+    pixst       chunk_size, dst_w_bpp, dst_w_basereg, DST_W
+.endif
+1:
+.endif
+.endr
+2:
+.endm
+
+/*
+ * Macro, which performs all the needed operations to switch to the next
+ * scanline and start the next loop iteration unless all the scanlines
+ * are already processed.
+ */
+.macro advance_to_next_scanline start_of_loop_label
+.if regs_shortage
+    ldrd        W, [sp] /* load W and H (width and height) from stack */
+.else
+    mov         W, ORIG_W
+.endif
+    add         DST_W, DST_W, DST_STRIDE, lsl #dst_bpp_shift
+.if src_bpp != 0
+    add         SRC, SRC, SRC_STRIDE, lsl #src_bpp_shift
+.endif
+.if mask_bpp != 0
+    add         MASK, MASK, MASK_STRIDE, lsl #mask_bpp_shift
+.endif
+.if (dst_w_bpp != 24)
+    sub         DST_W, DST_W, W, lsl #dst_bpp_shift
+.endif
+.if (src_bpp != 24) && (src_bpp != 0)
+    sub         SRC, SRC, W, lsl #src_bpp_shift
+.endif
+.if (mask_bpp != 24) && (mask_bpp != 0)
+    sub         MASK, MASK, W, lsl #mask_bpp_shift
+.endif
+    subs        H, H, #1
+    mov         DST_R, DST_W
+.if regs_shortage
+    str         H, [sp, #4] /* save updated height to stack */
+.endif
+    bge         start_of_loop_label
+.endm
+
+/*
+ * Registers are allocated in the following way by default:
+ * d0, d1, d2, d3     - reserved for loading source pixel data
+ * d4, d5, d6, d7     - reserved for loading destination pixel data
+ * d24, d25, d26, d27 - reserved for loading mask pixel data
+ * d28, d29, d30, d31 - final destination pixel data for writeback to memory
+ */
+.macro generate_composite_function fname, \
+                                   src_bpp_, \
+                                   mask_bpp_, \
+                                   dst_w_bpp_, \
+                                   flags, \
+                                   pixblock_size_, \
+                                   prefetch_distance, \
+                                   init, \
+                                   cleanup, \
+                                   process_pixblock_head, \
+                                   process_pixblock_tail, \
+                                   process_pixblock_tail_head, \
+                                   dst_w_basereg_ = 28, \
+                                   dst_r_basereg_ = 4, \
+                                   src_basereg_   = 0, \
+                                   mask_basereg_  = 24
+
+    .func fname
+    .global fname
+    /* For ELF format also set function visibility to hidden */
+#ifdef __ELF__
+    .hidden fname
+    .type fname, %function
+#endif
+fname:
+    push        {r4-r12, lr}        /* save all registers */
+
+/*
+ * Select prefetch type for this function. If prefetch distance is
+ * set to 0 or one of the color formats is 24bpp, SIMPLE prefetch
+ * has to be used instead of ADVANCED.
+ */
+    .set PREFETCH_TYPE_CURRENT, PREFETCH_TYPE_DEFAULT
+.if prefetch_distance == 0
+    .set PREFETCH_TYPE_CURRENT, PREFETCH_TYPE_NONE
+.elseif (PREFETCH_TYPE_CURRENT > PREFETCH_TYPE_SIMPLE) && \
+        ((src_bpp_ == 24) || (mask_bpp_ == 24) || (dst_w_bpp_ == 24))
+    .set PREFETCH_TYPE_CURRENT, PREFETCH_TYPE_SIMPLE
+.endif
+
+/*
+ * Make some macro arguments globally visible and accessible
+ * from other macros
+ */
+    .set src_bpp, src_bpp_
+    .set mask_bpp, mask_bpp_
+    .set dst_w_bpp, dst_w_bpp_
+    .set pixblock_size, pixblock_size_
+    .set dst_w_basereg, dst_w_basereg_
+    .set dst_r_basereg, dst_r_basereg_
+    .set src_basereg, src_basereg_
+    .set mask_basereg, mask_basereg_
+
+/*
+ * Assign symbolic names to registers
+ */
+    W           .req        r0      /* width (is updated during processing) */
+    H           .req        r1      /* height (is updated during processing) */
+    DST_W       .req        r2      /* destination buffer pointer for writes */
+    DST_STRIDE  .req        r3      /* destination image stride */
+    SRC         .req        r4      /* source buffer pointer */
+    SRC_STRIDE  .req        r5      /* source image stride */
+    DST_R       .req        r6      /* destination buffer pointer for reads */
+
+    MASK        .req        r7      /* mask pointer */
+    MASK_STRIDE .req        r8      /* mask stride */
+
+    PF_CTL      .req        r9      /* combined lines counter and prefetch */
+                                    /* distance increment counter */
+    PF_X        .req        r10     /* pixel index in a scanline for current */
+                                    /* pretetch position */
+    PF_SRC      .req        r11     /* pointer to source scanline start */
+                                    /* for prefetch purposes */
+    PF_DST      .req        r12     /* pointer to destination scanline start */
+                                    /* for prefetch purposes */
+    PF_MASK     .req        r14     /* pointer to mask scanline start */
+                                    /* for prefetch purposes */
+/*
+ * Check whether we have enough registers for all the local variables.
+ * If we don't have enough registers, original width and height are
+ * kept on top of stack (and 'regs_shortage' variable is set to indicate
+ * this for the rest of code). Even if there are enough registers, the
+ * allocation scheme may be a bit different depending on whether source
+ * or mask is not used.
+ */
+.if (PREFETCH_TYPE_CURRENT < PREFETCH_TYPE_ADVANCED)
+    ORIG_W      .req        r10     /* saved original width */
+    DUMMY       .req        r12     /* temporary register */
+    .set        regs_shortage, 0
+.elseif mask_bpp == 0
+    ORIG_W      .req        r7      /* saved original width */
+    DUMMY       .req        r8      /* temporary register */
+    .set        regs_shortage, 0
+.elseif src_bpp == 0
+    ORIG_W      .req        r4      /* saved original width */
+    DUMMY       .req        r5      /* temporary register */
+    .set        regs_shortage, 0
+.else
+    ORIG_W      .req        r1      /* saved original width */
+    DUMMY       .req        r1      /* temporary register */
+    .set        regs_shortage, 1
+.endif
+
+    .set mask_bpp_shift, -1
+.if src_bpp == 32
+    .set src_bpp_shift, 2
+.elseif src_bpp == 24
+    .set src_bpp_shift, 0
+.elseif src_bpp == 16
+    .set src_bpp_shift, 1
+.elseif src_bpp == 8
+    .set src_bpp_shift, 0
+.elseif src_bpp == 0
+    .set src_bpp_shift, -1
+.else
+    .error "requested src bpp (src_bpp) is not supported"
+.endif
+.if mask_bpp == 32
+    .set mask_bpp_shift, 2
+.elseif mask_bpp == 24
+    .set mask_bpp_shift, 0
+.elseif mask_bpp == 8
+    .set mask_bpp_shift, 0
+.elseif mask_bpp == 0
+    .set mask_bpp_shift, -1
+.else
+    .error "requested mask bpp (mask_bpp) is not supported"
+.endif
+.if dst_w_bpp == 32
+    .set dst_bpp_shift, 2
+.elseif dst_w_bpp == 24
+    .set dst_bpp_shift, 0
+.elseif dst_w_bpp == 16
+    .set dst_bpp_shift, 1
+.elseif dst_w_bpp == 8
+    .set dst_bpp_shift, 0
+.else
+    .error "requested dst bpp (dst_w_bpp) is not supported"
+.endif
+
+.if (((flags) & FLAG_DST_READWRITE) != 0)
+    .set dst_r_bpp, dst_w_bpp
+.else
+    .set dst_r_bpp, 0
+.endif
+.if (((flags) & FLAG_DEINTERLEAVE_32BPP) != 0)
+    .set DEINTERLEAVE_32BPP_ENABLED, 1
+.else
+    .set DEINTERLEAVE_32BPP_ENABLED, 0
+.endif
+
+.if prefetch_distance < 0 || prefetch_distance > 15
+    .error "invalid prefetch distance (prefetch_distance)"
+.endif
+
+.if src_bpp > 0
+    ldr         SRC, [sp, #40]
+.endif
+.if mask_bpp > 0
+    ldr         MASK, [sp, #48]
+.endif
+    PF mov      PF_X, #0
+.if src_bpp > 0
+    ldr         SRC_STRIDE, [sp, #44]
+.endif
+.if mask_bpp > 0
+    ldr         MASK_STRIDE, [sp, #52]
+.endif
+    mov         DST_R, DST_W
+
+.if src_bpp == 24
+    sub         SRC_STRIDE, SRC_STRIDE, W
+    sub         SRC_STRIDE, SRC_STRIDE, W, lsl #1
+.endif
+.if mask_bpp == 24
+    sub         MASK_STRIDE, MASK_STRIDE, W
+    sub         MASK_STRIDE, MASK_STRIDE, W, lsl #1
+.endif
+.if dst_w_bpp == 24
+    sub         DST_STRIDE, DST_STRIDE, W
+    sub         DST_STRIDE, DST_STRIDE, W, lsl #1
+.endif
+
+/*
+ * Setup advanced prefetcher initial state
+ */
+    PF mov      PF_SRC, SRC
+    PF mov      PF_DST, DST_R
+    PF mov      PF_MASK, MASK
+    /* PF_CTL = prefetch_distance | ((h - 1) << 4) */
+    PF mov      PF_CTL, H, lsl #4
+    PF add      PF_CTL, #(prefetch_distance - 0x10)
+
+    init
+.if regs_shortage
+    push        {r0, r1}
+.endif
+    subs        H, H, #1
+.if regs_shortage
+    str         H, [sp, #4] /* save updated height to stack */
+.else
+    mov         ORIG_W, W
+.endif
+    blt         9f
+    cmp         W, #(pixblock_size * 2)
+    blt         8f
+/*
+ * This is the start of the pipelined loop, which if optimized for
+ * long scanlines
+ */
+0:
+    ensure_destination_ptr_alignment process_pixblock_head, \
+                                     process_pixblock_tail, \
+                                     process_pixblock_tail_head
+
+    /* Implement "head (tail_head) ... (tail_head) tail" loop pattern */
+    pixld_a     pixblock_size, dst_r_bpp, \
+                (dst_r_basereg - pixblock_size * dst_r_bpp / 64), DST_R
+    pixld       pixblock_size, src_bpp, \
+                (src_basereg - pixblock_size * src_bpp / 64), SRC
+    pixld       pixblock_size, mask_bpp, \
+                (mask_basereg - pixblock_size * mask_bpp / 64), MASK
+    PF add      PF_X, PF_X, #pixblock_size
+    process_pixblock_head
+    cache_preload 0, pixblock_size
+    cache_preload_simple
+    subs        W, W, #(pixblock_size * 2)
+    blt         2f
+1:
+    process_pixblock_tail_head
+    cache_preload_simple
+    subs        W, W, #pixblock_size
+    bge         1b
+2:
+    process_pixblock_tail
+    pixst_a     pixblock_size, dst_w_bpp, \
+                (dst_w_basereg - pixblock_size * dst_w_bpp / 64), DST_W
+
+    /* Process the remaining trailing pixels in the scanline */
+    process_trailing_pixels 1, 1, \
+                            process_pixblock_head, \
+                            process_pixblock_tail, \
+                            process_pixblock_tail_head
+    advance_to_next_scanline 0b
+
+.if regs_shortage
+    pop         {r0, r1}
+.endif
+    cleanup
+    pop         {r4-r12, pc}  /* exit */
+/*
+ * This is the start of the loop, designed to process images with small width
+ * (less than pixblock_size * 2 pixels). In this case neither pipelining
+ * nor prefetch are used.
+ */
+8:
+    /* Process exactly pixblock_size pixels if needed */
+    tst         W, #pixblock_size
+    beq         1f
+    pixld       pixblock_size, dst_r_bpp, \
+                (dst_r_basereg - pixblock_size * dst_r_bpp / 64), DST_R
+    pixld       pixblock_size, src_bpp, \
+                (src_basereg - pixblock_size * src_bpp / 64), SRC
+    pixld       pixblock_size, mask_bpp, \
+                (mask_basereg - pixblock_size * mask_bpp / 64), MASK
+    process_pixblock_head
+    process_pixblock_tail
+    pixst       pixblock_size, dst_w_bpp, \
+                (dst_w_basereg - pixblock_size * dst_w_bpp / 64), DST_W
+1:
+    /* Process the remaining trailing pixels in the scanline */
+    process_trailing_pixels 0, 0, \
+                            process_pixblock_head, \
+                            process_pixblock_tail, \
+                            process_pixblock_tail_head
+    advance_to_next_scanline 8b
+9:
+.if regs_shortage
+    pop         {r0, r1}
+.endif
+    cleanup
+    pop         {r4-r12, pc}  /* exit */
+
+    .unreq      SRC
+    .unreq      MASK
+    .unreq      DST_R
+    .unreq      DST_W
+    .unreq      ORIG_W
+    .unreq      W
+    .unreq      H
+    .unreq      SRC_STRIDE
+    .unreq      DST_STRIDE
+    .unreq      MASK_STRIDE
+    .unreq      PF_CTL
+    .unreq      PF_X
+    .unreq      PF_SRC
+    .unreq      PF_DST
+    .unreq      PF_MASK
+    .unreq      DUMMY
+    .endfunc
+.endm
+
+/*
+ * A simplified variant of function generation template for a single
+ * scanline processing (for implementing pixman combine functions)
+ */
+.macro generate_composite_function_single_scanline fname, \
+                                                   src_bpp_, \
+                                                   mask_bpp_, \
+                                                   dst_w_bpp_, \
+                                                   flags, \
+                                                   pixblock_size_, \
+                                                   init, \
+                                                   cleanup, \
+                                                   process_pixblock_head, \
+                                                   process_pixblock_tail, \
+                                                   process_pixblock_tail_head, \
+                                                   dst_w_basereg_ = 28, \
+                                                   dst_r_basereg_ = 4, \
+                                                   src_basereg_   = 0, \
+                                                   mask_basereg_  = 24
+
+    .func fname
+    .global fname
+    /* For ELF format also set function visibility to hidden */
+#ifdef __ELF__
+    .hidden fname
+    .type fname, %function
+#endif
+fname:
+    .set PREFETCH_TYPE_CURRENT, PREFETCH_TYPE_NONE
+/*
+ * Make some macro arguments globally visible and accessible
+ * from other macros
+ */
+    .set src_bpp, src_bpp_
+    .set mask_bpp, mask_bpp_
+    .set dst_w_bpp, dst_w_bpp_
+    .set pixblock_size, pixblock_size_
+    .set dst_w_basereg, dst_w_basereg_
+    .set dst_r_basereg, dst_r_basereg_
+    .set src_basereg, src_basereg_
+    .set mask_basereg, mask_basereg_
+/*
+ * Assign symbolic names to registers
+ */
+    W           .req        r0      /* width (is updated during processing) */
+    DST_W       .req        r1      /* destination buffer pointer for writes */
+    SRC         .req        r2      /* source buffer pointer */
+    DST_R       .req        ip      /* destination buffer pointer for reads */
+    MASK        .req        r3      /* mask pointer */
+
+.if (((flags) & FLAG_DST_READWRITE) != 0)
+    .set dst_r_bpp, dst_w_bpp
+.else
+    .set dst_r_bpp, 0
+.endif
+.if (((flags) & FLAG_DEINTERLEAVE_32BPP) != 0)
+    .set DEINTERLEAVE_32BPP_ENABLED, 1
+.else
+    .set DEINTERLEAVE_32BPP_ENABLED, 0
+.endif
+
+    init
+    mov         DST_R, DST_W
+
+    cmp         W, #pixblock_size
+    blt         8f
+
+    ensure_destination_ptr_alignment process_pixblock_head, \
+                                     process_pixblock_tail, \
+                                     process_pixblock_tail_head
+
+    subs        W, W, #pixblock_size
+    blt         7f
+
+    /* Implement "head (tail_head) ... (tail_head) tail" loop pattern */
+    pixld_a     pixblock_size, dst_r_bpp, \
+                (dst_r_basereg - pixblock_size * dst_r_bpp / 64), DST_R
+    pixld       pixblock_size, src_bpp, \
+                (src_basereg - pixblock_size * src_bpp / 64), SRC
+    pixld       pixblock_size, mask_bpp, \
+                (mask_basereg - pixblock_size * mask_bpp / 64), MASK
+    process_pixblock_head
+    subs        W, W, #pixblock_size
+    blt         2f
+1:
+    process_pixblock_tail_head
+    subs        W, W, #pixblock_size
+    bge         1b
+2:
+    process_pixblock_tail
+    pixst_a     pixblock_size, dst_w_bpp, \
+                (dst_w_basereg - pixblock_size * dst_w_bpp / 64), DST_W
+7:
+    /* Process the remaining trailing pixels in the scanline (dst aligned) */
+    process_trailing_pixels 0, 1, \
+                            process_pixblock_head, \
+                            process_pixblock_tail, \
+                            process_pixblock_tail_head
+
+    cleanup
+    bx         lr  /* exit */
+8:
+    /* Process the remaining trailing pixels in the scanline (dst unaligned) */
+    process_trailing_pixels 0, 0, \
+                            process_pixblock_head, \
+                            process_pixblock_tail, \
+                            process_pixblock_tail_head
+
+    cleanup
+    bx          lr  /* exit */
+
+    .unreq      SRC
+    .unreq      MASK
+    .unreq      DST_R
+    .unreq      DST_W
+    .unreq      W
+    .endfunc
+.endm
+
+.macro default_init
+.endm
+
+.macro default_cleanup
+.endm
diff --git a/pixman/pixman/pixman-arm-neon.c b/pixman/pixman/pixman-arm-neon.c
new file mode 100644
index 000000000..6808b3658
--- /dev/null
+++ b/pixman/pixman/pixman-arm-neon.c
@@ -0,0 +1,352 @@
+/*
+ * Copyright © 2009 ARM Ltd, Movial Creative Technologies Oy
+ *
+ * Permission to use, copy, modify, distribute, and sell this software and its
+ * documentation for any purpose is hereby granted without fee, provided that
+ * the above copyright notice appear in all copies and that both that
+ * copyright notice and this permission notice appear in supporting
+ * documentation, and that the name of ARM Ltd not be used in
+ * advertising or publicity pertaining to distribution of the software without
+ * specific, written prior permission.  ARM Ltd makes no
+ * representations about the suitability of this software for any purpose.  It
+ * is provided "as is" without express or implied warranty.
+ *
+ * THE COPYRIGHT HOLDERS DISCLAIM ALL WARRANTIES WITH REGARD TO THIS
+ * SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
+ * FITNESS, IN NO EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
+ * AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
+ * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS
+ * SOFTWARE.
+ *
+ * Author:  Ian Rickards (ian.rickards@arm.com)
+ * Author:  Jonathan Morton (jonathan.morton@movial.com)
+ * Author:  Markku Vire (markku.vire@movial.com)
+ *
+ */
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <string.h>
+#include "pixman-private.h"
+#include "pixman-arm-common.h"
+
+PIXMAN_ARM_BIND_FAST_PATH_SRC_DST (neon, src_8888_8888,
+                                   uint32_t, 1, uint32_t, 1)
+PIXMAN_ARM_BIND_FAST_PATH_SRC_DST (neon, src_x888_8888,
+                                   uint32_t, 1, uint32_t, 1)
+PIXMAN_ARM_BIND_FAST_PATH_SRC_DST (neon, src_0565_0565,
+                                   uint16_t, 1, uint16_t, 1)
+PIXMAN_ARM_BIND_FAST_PATH_SRC_DST (neon, src_0888_0888,
+                                   uint8_t, 3, uint8_t, 3)
+PIXMAN_ARM_BIND_FAST_PATH_SRC_DST (neon, src_8888_0565,
+                                   uint32_t, 1, uint16_t, 1)
+PIXMAN_ARM_BIND_FAST_PATH_SRC_DST (neon, src_0565_8888,
+                                   uint16_t, 1, uint32_t, 1)
+PIXMAN_ARM_BIND_FAST_PATH_SRC_DST (neon, src_0888_8888_rev,
+                                   uint8_t, 3, uint32_t, 1)
+PIXMAN_ARM_BIND_FAST_PATH_SRC_DST (neon, src_0888_0565_rev,
+                                   uint8_t, 3, uint16_t, 1)
+PIXMAN_ARM_BIND_FAST_PATH_SRC_DST (neon, src_pixbuf_8888,
+                                   uint32_t, 1, uint32_t, 1)
+PIXMAN_ARM_BIND_FAST_PATH_SRC_DST (neon, add_8000_8000,
+                                   uint8_t, 1, uint8_t, 1)
+PIXMAN_ARM_BIND_FAST_PATH_SRC_DST (neon, add_8888_8888,
+                                   uint32_t, 1, uint32_t, 1)
+PIXMAN_ARM_BIND_FAST_PATH_SRC_DST (neon, over_8888_0565,
+                                   uint32_t, 1, uint16_t, 1)
+PIXMAN_ARM_BIND_FAST_PATH_SRC_DST (neon, over_8888_8888,
+                                   uint32_t, 1, uint32_t, 1)
+
+PIXMAN_ARM_BIND_FAST_PATH_N_DST (neon, over_n_0565,
+                                 uint16_t, 1)
+PIXMAN_ARM_BIND_FAST_PATH_N_DST (neon, over_n_8888,
+                                 uint32_t, 1)
+PIXMAN_ARM_BIND_FAST_PATH_N_DST (neon, over_reverse_n_8888,
+                                 uint32_t, 1)
+
+PIXMAN_ARM_BIND_FAST_PATH_N_MASK_DST (neon, over_n_8_0565,
+                                      uint8_t, 1, uint16_t, 1)
+PIXMAN_ARM_BIND_FAST_PATH_N_MASK_DST (neon, over_n_8_8888,
+                                      uint8_t, 1, uint32_t, 1)
+PIXMAN_ARM_BIND_FAST_PATH_N_MASK_DST (neon, over_n_8888_8888_ca,
+                                      uint32_t, 1, uint32_t, 1)
+PIXMAN_ARM_BIND_FAST_PATH_N_MASK_DST (neon, add_n_8_8,
+                                      uint8_t, 1, uint8_t, 1)
+
+PIXMAN_ARM_BIND_FAST_PATH_SRC_N_DST (neon, over_8888_n_8888,
+                                     uint32_t, 1, uint32_t, 1)
+
+PIXMAN_ARM_BIND_FAST_PATH_SRC_MASK_DST (neon, add_8_8_8,
+                                        uint8_t, 1, uint8_t, 1, uint8_t, 1)
+PIXMAN_ARM_BIND_FAST_PATH_SRC_MASK_DST (neon, add_8888_8888_8888,
+                                        uint32_t, 1, uint32_t, 1, uint32_t, 1)
+PIXMAN_ARM_BIND_FAST_PATH_SRC_MASK_DST (neon, over_8888_8_8888,
+                                        uint32_t, 1, uint8_t, 1, uint32_t, 1)
+PIXMAN_ARM_BIND_FAST_PATH_SRC_MASK_DST (neon, over_8888_8888_8888,
+                                        uint32_t, 1, uint32_t, 1, uint32_t, 1)
+
+void
+pixman_composite_src_n_8_asm_neon (int32_t   w,
+                                   int32_t   h,
+                                   uint8_t  *dst,
+                                   int32_t   dst_stride,
+                                   uint8_t   src);
+
+void
+pixman_composite_src_n_0565_asm_neon (int32_t   w,
+                                      int32_t   h,
+                                      uint16_t *dst,
+                                      int32_t   dst_stride,
+                                      uint16_t  src);
+
+void
+pixman_composite_src_n_8888_asm_neon (int32_t   w,
+                                      int32_t   h,
+                                      uint32_t *dst,
+                                      int32_t   dst_stride,
+                                      uint32_t  src);
+
+static pixman_bool_t
+pixman_fill_neon (uint32_t *bits,
+                  int       stride,
+                  int       bpp,
+                  int       x,
+                  int       y,
+                  int       width,
+                  int       height,
+                  uint32_t  _xor)
+{
+    /* stride is always multiple of 32bit units in pixman */
+    uint32_t byte_stride = stride * sizeof(uint32_t);
+
+    switch (bpp)
+    {
+    case 8:
+	pixman_composite_src_n_8_asm_neon (
+		width,
+		height,
+		(uint8_t *)(((char *) bits) + y * byte_stride + x),
+		byte_stride,
+		_xor & 0xff);
+	return TRUE;
+    case 16:
+	pixman_composite_src_n_0565_asm_neon (
+		width,
+		height,
+		(uint16_t *)(((char *) bits) + y * byte_stride + x * 2),
+		byte_stride / 2,
+		_xor & 0xffff);
+	return TRUE;
+    case 32:
+	pixman_composite_src_n_8888_asm_neon (
+		width,
+		height,
+		(uint32_t *)(((char *) bits) + y * byte_stride + x * 4),
+		byte_stride / 4,
+		_xor);
+	return TRUE;
+    default:
+	return FALSE;
+    }
+}
+
+static pixman_bool_t
+pixman_blt_neon (uint32_t *src_bits,
+                 uint32_t *dst_bits,
+                 int       src_stride,
+                 int       dst_stride,
+                 int       src_bpp,
+                 int       dst_bpp,
+                 int       src_x,
+                 int       src_y,
+                 int       dst_x,
+                 int       dst_y,
+                 int       width,
+                 int       height)
+{
+    if (src_bpp != dst_bpp)
+	return FALSE;
+
+    switch (src_bpp)
+    {
+    case 16:
+	pixman_composite_src_0565_0565_asm_neon (
+		width, height,
+		(uint16_t *)(((char *) dst_bits) +
+		dst_y * dst_stride * 4 + dst_x * 2), dst_stride * 2,
+		(uint16_t *)(((char *) src_bits) +
+		src_y * src_stride * 4 + src_x * 2), src_stride * 2);
+	return TRUE;
+    case 32:
+	pixman_composite_src_8888_8888_asm_neon (
+		width, height,
+		(uint32_t *)(((char *) dst_bits) +
+		dst_y * dst_stride * 4 + dst_x * 4), dst_stride,
+		(uint32_t *)(((char *) src_bits) +
+		src_y * src_stride * 4 + src_x * 4), src_stride);
+	return TRUE;
+    default:
+	return FALSE;
+    }
+}
+
+static const pixman_fast_path_t arm_neon_fast_paths[] =
+{
+    PIXMAN_STD_FAST_PATH (SRC,  r5g6b5,   null,     r5g6b5,   neon_composite_src_0565_0565),
+    PIXMAN_STD_FAST_PATH (SRC,  b5g6r5,   null,     b5g6r5,   neon_composite_src_0565_0565),
+    PIXMAN_STD_FAST_PATH (SRC,  a8r8g8b8, null,     r5g6b5,   neon_composite_src_8888_0565),
+    PIXMAN_STD_FAST_PATH (SRC,  x8r8g8b8, null,     r5g6b5,   neon_composite_src_8888_0565),
+    PIXMAN_STD_FAST_PATH (SRC,  a8b8g8r8, null,     b5g6r5,   neon_composite_src_8888_0565),
+    PIXMAN_STD_FAST_PATH (SRC,  x8b8g8r8, null,     b5g6r5,   neon_composite_src_8888_0565),
+    PIXMAN_STD_FAST_PATH (SRC,  r5g6b5,   null,     a8r8g8b8, neon_composite_src_0565_8888),
+    PIXMAN_STD_FAST_PATH (SRC,  r5g6b5,   null,     x8r8g8b8, neon_composite_src_0565_8888),
+    PIXMAN_STD_FAST_PATH (SRC,  b5g6r5,   null,     a8b8g8r8, neon_composite_src_0565_8888),
+    PIXMAN_STD_FAST_PATH (SRC,  b5g6r5,   null,     x8b8g8r8, neon_composite_src_0565_8888),
+    PIXMAN_STD_FAST_PATH (SRC,  a8r8g8b8, null,     x8r8g8b8, neon_composite_src_8888_8888),
+    PIXMAN_STD_FAST_PATH (SRC,  x8r8g8b8, null,     x8r8g8b8, neon_composite_src_8888_8888),
+    PIXMAN_STD_FAST_PATH (SRC,  a8b8g8r8, null,     x8b8g8r8, neon_composite_src_8888_8888),
+    PIXMAN_STD_FAST_PATH (SRC,  x8b8g8r8, null,     x8b8g8r8, neon_composite_src_8888_8888),
+    PIXMAN_STD_FAST_PATH (SRC,  a8r8g8b8, null,     a8r8g8b8, neon_composite_src_8888_8888),
+    PIXMAN_STD_FAST_PATH (SRC,  a8b8g8r8, null,     a8b8g8r8, neon_composite_src_8888_8888),
+    PIXMAN_STD_FAST_PATH (SRC,  x8r8g8b8, null,     a8r8g8b8, neon_composite_src_x888_8888),
+    PIXMAN_STD_FAST_PATH (SRC,  x8b8g8r8, null,     a8b8g8r8, neon_composite_src_x888_8888),
+    PIXMAN_STD_FAST_PATH (SRC,  r8g8b8,   null,     r8g8b8,   neon_composite_src_0888_0888),
+    PIXMAN_STD_FAST_PATH (SRC,  b8g8r8,   null,     x8r8g8b8, neon_composite_src_0888_8888_rev),
+    PIXMAN_STD_FAST_PATH (SRC,  b8g8r8,   null,     r5g6b5,   neon_composite_src_0888_0565_rev),
+    PIXMAN_STD_FAST_PATH (SRC,  pixbuf,   pixbuf,   a8r8g8b8, neon_composite_src_pixbuf_8888),
+    PIXMAN_STD_FAST_PATH (OVER, solid,    a8,       r5g6b5,   neon_composite_over_n_8_0565),
+    PIXMAN_STD_FAST_PATH (OVER, solid,    a8,       b5g6r5,   neon_composite_over_n_8_0565),
+    PIXMAN_STD_FAST_PATH (OVER, solid,    a8,       a8r8g8b8, neon_composite_over_n_8_8888),
+    PIXMAN_STD_FAST_PATH (OVER, solid,    a8,       x8r8g8b8, neon_composite_over_n_8_8888),
+    PIXMAN_STD_FAST_PATH (OVER, solid,    a8,       a8b8g8r8, neon_composite_over_n_8_8888),
+    PIXMAN_STD_FAST_PATH (OVER, solid,    a8,       x8b8g8r8, neon_composite_over_n_8_8888),
+    PIXMAN_STD_FAST_PATH (OVER, solid,    null,     r5g6b5,   neon_composite_over_n_0565),
+    PIXMAN_STD_FAST_PATH (OVER, solid,    null,     a8r8g8b8, neon_composite_over_n_8888),
+    PIXMAN_STD_FAST_PATH (OVER, solid,    null,     x8r8g8b8, neon_composite_over_n_8888),
+    PIXMAN_STD_FAST_PATH_CA (OVER, solid, a8r8g8b8, a8r8g8b8, neon_composite_over_n_8888_8888_ca),
+    PIXMAN_STD_FAST_PATH_CA (OVER, solid, a8r8g8b8, x8r8g8b8, neon_composite_over_n_8888_8888_ca),
+    PIXMAN_STD_FAST_PATH_CA (OVER, solid, a8b8g8r8, a8b8g8r8, neon_composite_over_n_8888_8888_ca),
+    PIXMAN_STD_FAST_PATH_CA (OVER, solid, a8b8g8r8, x8b8g8r8, neon_composite_over_n_8888_8888_ca),
+    PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, solid,    a8r8g8b8, neon_composite_over_8888_n_8888),
+    PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, solid,    x8r8g8b8, neon_composite_over_8888_n_8888),
+    PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, a8,       a8r8g8b8, neon_composite_over_8888_8_8888),
+    PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, a8,       x8r8g8b8, neon_composite_over_8888_8_8888),
+    PIXMAN_STD_FAST_PATH (OVER, a8b8g8r8, a8,       a8b8g8r8, neon_composite_over_8888_8_8888),
+    PIXMAN_STD_FAST_PATH (OVER, a8b8g8r8, a8,       x8b8g8r8, neon_composite_over_8888_8_8888),
+    PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, a8r8g8b8, a8r8g8b8, neon_composite_over_8888_8888_8888),
+    PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, null,     r5g6b5,   neon_composite_over_8888_0565),
+    PIXMAN_STD_FAST_PATH (OVER, a8b8g8r8, null,     b5g6r5,   neon_composite_over_8888_0565),
+    PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, null,     a8r8g8b8, neon_composite_over_8888_8888),
+    PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, null,     x8r8g8b8, neon_composite_over_8888_8888),
+    PIXMAN_STD_FAST_PATH (OVER, a8b8g8r8, null,     a8b8g8r8, neon_composite_over_8888_8888),
+    PIXMAN_STD_FAST_PATH (OVER, a8b8g8r8, null,     x8b8g8r8, neon_composite_over_8888_8888),
+    PIXMAN_STD_FAST_PATH (OVER, x8r8g8b8, null,     a8r8g8b8, neon_composite_src_x888_8888),
+    PIXMAN_STD_FAST_PATH (OVER, x8b8g8r8, null,     a8b8g8r8, neon_composite_src_x888_8888),
+    PIXMAN_STD_FAST_PATH (ADD,  solid,    a8,       a8,       neon_composite_add_n_8_8),
+    PIXMAN_STD_FAST_PATH (ADD,  a8,       a8,       a8,       neon_composite_add_8_8_8),
+    PIXMAN_STD_FAST_PATH (ADD,  a8r8g8b8, a8r8g8b8, a8r8g8b8, neon_composite_add_8888_8888_8888),
+    PIXMAN_STD_FAST_PATH (ADD,  a8,       null,     a8,       neon_composite_add_8000_8000),
+    PIXMAN_STD_FAST_PATH (ADD,  a8r8g8b8, null,     a8r8g8b8, neon_composite_add_8888_8888),
+    PIXMAN_STD_FAST_PATH (ADD,  a8b8g8r8, null,     a8b8g8r8, neon_composite_add_8888_8888),
+    PIXMAN_STD_FAST_PATH (OVER_REVERSE, solid, null, a8r8g8b8, neon_composite_over_reverse_n_8888),
+    PIXMAN_STD_FAST_PATH (OVER_REVERSE, solid, null, a8b8g8r8, neon_composite_over_reverse_n_8888),
+
+    { PIXMAN_OP_NONE },
+};
+
+static pixman_bool_t
+arm_neon_blt (pixman_implementation_t *imp,
+              uint32_t *               src_bits,
+              uint32_t *               dst_bits,
+              int                      src_stride,
+              int                      dst_stride,
+              int                      src_bpp,
+              int                      dst_bpp,
+              int                      src_x,
+              int                      src_y,
+              int                      dst_x,
+              int                      dst_y,
+              int                      width,
+              int                      height)
+{
+    if (!pixman_blt_neon (
+            src_bits, dst_bits, src_stride, dst_stride, src_bpp, dst_bpp,
+            src_x, src_y, dst_x, dst_y, width, height))
+
+    {
+	return _pixman_implementation_blt (
+	    imp->delegate,
+	    src_bits, dst_bits, src_stride, dst_stride, src_bpp, dst_bpp,
+	    src_x, src_y, dst_x, dst_y, width, height);
+    }
+
+    return TRUE;
+}
+
+static pixman_bool_t
+arm_neon_fill (pixman_implementation_t *imp,
+               uint32_t *               bits,
+               int                      stride,
+               int                      bpp,
+               int                      x,
+               int                      y,
+               int                      width,
+               int                      height,
+               uint32_t xor)
+{
+    if (pixman_fill_neon (bits, stride, bpp, x, y, width, height, xor))
+	return TRUE;
+
+    return _pixman_implementation_fill (
+	imp->delegate, bits, stride, bpp, x, y, width, height, xor);
+}
+
+#define BIND_COMBINE_U(name)                                             \
+void                                                                     \
+pixman_composite_scanline_##name##_mask_asm_neon (int32_t         w,     \
+                                                  const uint32_t *dst,   \
+                                                  const uint32_t *src,   \
+                                                  const uint32_t *mask); \
+                                                                         \
+void                                                                     \
+pixman_composite_scanline_##name##_asm_neon (int32_t         w,          \
+                                             const uint32_t *dst,        \
+                                             const uint32_t *src);       \
+                                                                         \
+static void                                                              \
+neon_combine_##name##_u (pixman_implementation_t *imp,                   \
+                         pixman_op_t              op,                    \
+                         uint32_t *               dest,                  \
+                         const uint32_t *         src,                   \
+                         const uint32_t *         mask,                  \
+                         int                      width)                 \
+{                                                                        \
+    if (mask)                                                            \
+	pixman_composite_scanline_##name##_mask_asm_neon (width, dest,   \
+	                                                  src, mask);    \
+    else                                                                 \
+	pixman_composite_scanline_##name##_asm_neon (width, dest, src);  \
+}
+
+BIND_COMBINE_U (over)
+BIND_COMBINE_U (add)
+
+pixman_implementation_t *
+_pixman_implementation_create_arm_neon (void)
+{
+    pixman_implementation_t *general = _pixman_implementation_create_fast_path ();
+    pixman_implementation_t *imp =
+	_pixman_implementation_create (general, arm_neon_fast_paths);
+
+    imp->combine_32[PIXMAN_OP_OVER] = neon_combine_over_u;
+    imp->combine_32[PIXMAN_OP_ADD] = neon_combine_add_u;
+
+    imp->blt = arm_neon_blt;
+    imp->fill = arm_neon_fill;
+
+    return imp;
+}
diff --git a/pixman/pixman/pixman-arm-simd-asm.S b/pixman/pixman/pixman-arm-simd-asm.S
new file mode 100644
index 000000000..1a1a0d641
--- /dev/null
+++ b/pixman/pixman/pixman-arm-simd-asm.S
@@ -0,0 +1,330 @@
+/*
+ * Copyright © 2008 Mozilla Corporation
+ *
+ * Permission to use, copy, modify, distribute, and sell this software and its
+ * documentation for any purpose is hereby granted without fee, provided that
+ * the above copyright notice appear in all copies and that both that
+ * copyright notice and this permission notice appear in supporting
+ * documentation, and that the name of Mozilla Corporation not be used in
+ * advertising or publicity pertaining to distribution of the software without
+ * specific, written prior permission.  Mozilla Corporation makes no
+ * representations about the suitability of this software for any purpose.  It
+ * is provided "as is" without express or implied warranty.
+ *
+ * THE COPYRIGHT HOLDERS DISCLAIM ALL WARRANTIES WITH REGARD TO THIS
+ * SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
+ * FITNESS, IN NO EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
+ * AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
+ * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS
+ * SOFTWARE.
+ *
+ * Author:  Jeff Muizelaar (jeff@infidigm.net)
+ *
+ */
+
+/* Prevent the stack from becoming executable */
+#if defined(__linux__) && defined(__ELF__)
+.section .note.GNU-stack,"",%progbits
+#endif
+
+	.text
+	.arch armv6
+	.object_arch armv4
+	.arm
+	.altmacro
+
+/* Supplementary macro for setting function attributes */
+.macro pixman_asm_function fname
+	.func fname
+	.global fname
+#ifdef __ELF__
+	.hidden fname
+	.type fname, %function
+#endif
+fname:
+.endm
+
+/*
+ * The code below was generated by gcc 4.3.4 from the commented out
+ * functions in 'pixman-arm-simd.c' file with the following optimization
+ * options: "-O3 -mcpu=arm1136jf-s -fomit-frame-pointer"
+ *
+ * TODO: replace gcc generated code with hand tuned versions because
+ * the code quality is not very good, introduce symbolic register
+ * aliases for better readability and maintainability.
+ */
+
+pixman_asm_function pixman_composite_add_8000_8000_asm_armv6
+	push	{r4, r5, r6, r7, r8, r9, r10, r11}
+	mov	r10, r1
+	sub	sp, sp, #4
+	subs	r10, r10, #1
+	mov	r11, r0
+	mov	r8, r2
+	str	r3, [sp]
+	ldr	r7, [sp, #36]
+	bcc	0f
+6:	cmp	r11, #0
+	beq	1f
+	orr	r3, r8, r7
+	tst	r3, #3
+	beq	2f
+	mov	r1, r8
+	mov	r0, r7
+	mov	r12, r11
+	b	3f
+5:	tst	r3, #3
+	beq	4f
+3:	ldrb	r2, [r0], #1
+	subs	r12, r12, #1
+	ldrb	r3, [r1]
+	uqadd8	r3, r2, r3
+	strb	r3, [r1], #1
+	orr	r3, r1, r0
+	bne	5b
+1:	ldr	r3, [sp]
+	add	r8, r8, r3
+	ldr	r3, [sp, #40]
+	add	r7, r7, r3
+10:	subs	r10, r10, #1
+	bcs	6b
+0:	add	sp, sp, #4
+	pop	{r4, r5, r6, r7, r8, r9, r10, r11}
+	bx	lr
+2:	mov	r12, r11
+	mov	r1, r8
+	mov	r0, r7
+4:	cmp	r12, #3
+	subgt	r6, r12, #4
+	movgt	r9, r12
+	lsrgt	r5, r6, #2
+	addgt	r3, r5, #1
+	movgt	r12, #0
+	lslgt	r4, r3, #2
+	ble	7f
+8:	ldr	r3, [r0, r12]
+	ldr	r2, [r1, r12]
+	uqadd8	r3, r3, r2
+	str	r3, [r1, r12]
+	add	r12, r12, #4
+	cmp	r12, r4
+	bne	8b
+	sub	r3, r9, #4
+	bic	r3, r3, #3
+	add	r3, r3, #4
+	subs	r12, r6, r5, lsl #2
+	add	r1, r1, r3
+	add	r0, r0, r3
+	beq	1b
+7:	mov	r4, #0
+9:	ldrb	r3, [r1, r4]
+	ldrb	r2, [r0, r4]
+	uqadd8	r3, r2, r3
+	strb	r3, [r1, r4]
+	add	r4, r4, #1
+	cmp	r4, r12
+	bne	9b
+	ldr	r3, [sp]
+	add	r8, r8, r3
+	ldr	r3, [sp, #40]
+	add	r7, r7, r3
+	b	10b
+.endfunc
+
+pixman_asm_function pixman_composite_over_8888_8888_asm_armv6
+	push	{r4, r5, r6, r7, r8, r9, r10, r11}
+	sub	sp, sp, #20
+	cmp	r1, #0
+	mov	r12, r2
+	str	r1, [sp, #12]
+	str	r0, [sp, #16]
+	ldr	r2, [sp, #52]
+	beq	0f
+	lsl	r3, r3, #2
+	str	r3, [sp]
+	ldr	r3, [sp, #56]
+	mov	r10, #0
+	lsl	r3, r3, #2
+	str	r3, [sp, #8]
+	mov	r11, r3
+	b	1f
+6:	ldr	r11, [sp, #8]
+1:	ldr	r9, [sp]
+	mov	r0, r12
+	add	r12, r12, r9
+	mov	r1, r2
+	str	r12, [sp, #4]
+	add	r2, r2, r11
+	ldr	r12, [sp, #16]
+	ldr	r3, =0x00800080
+	ldr	r9, =0xff00ff00
+	mov	r11, #255
+	cmp	r12, #0
+	beq	4f
+5:	ldr	r5, [r1], #4
+	ldr	r4, [r0]
+	sub	r8, r11, r5, lsr #24
+	uxtb16	r6, r4
+	uxtb16	r7, r4, ror #8
+	mla	r6, r6, r8, r3
+	mla	r7, r7, r8, r3
+	uxtab16	r6, r6, r6, ror #8
+	uxtab16	r7, r7, r7, ror #8
+	and	r7, r7, r9
+	uxtab16	r6, r7, r6, ror #8
+	uqadd8	r5, r6, r5
+	str	r5, [r0], #4
+	subs	r12, r12, #1
+	bne	5b
+4:	ldr	r3, [sp, #12]
+	add	r10, r10, #1
+	cmp	r10, r3
+	ldr	r12, [sp, #4]
+	bne	6b
+0:	add	sp, sp, #20
+	pop	{r4, r5, r6, r7, r8, r9, r10, r11}
+	bx	lr
+.endfunc
+
+pixman_asm_function pixman_composite_over_8888_n_8888_asm_armv6
+	push	{r4, r5, r6, r7, r8, r9, r10, r11}
+	sub	sp, sp, #28
+	cmp	r1, #0
+	str	r1, [sp, #12]
+	ldrb	r1, [sp, #71]
+	mov	r12, r2
+	str	r0, [sp, #16]
+	ldr	r2, [sp, #60]
+	str	r1, [sp, #24]
+	beq	0f
+	lsl	r3, r3, #2
+	str	r3, [sp, #20]
+	ldr	r3, [sp, #64]
+	mov	r10, #0
+	lsl	r3, r3, #2
+	str	r3, [sp, #8]
+	mov	r11, r3
+	b	1f
+5:	ldr	r11, [sp, #8]
+1:	ldr	r4, [sp, #20]
+	mov	r0, r12
+	mov	r1, r2
+	add	r12, r12, r4
+	add	r2, r2, r11
+	str	r12, [sp]
+	str	r2, [sp, #4]
+	ldr	r12, [sp, #16]
+	ldr	r2, =0x00800080
+	ldr	r3, [sp, #24]
+	mov	r11, #255
+	cmp	r12, #0
+	beq	3f
+4:	ldr	r5, [r1], #4
+	ldr	r4, [r0]
+	uxtb16	r6, r5
+	uxtb16	r7, r5, ror #8
+	mla	r6, r6, r3, r2
+	mla	r7, r7, r3, r2
+	uxtab16	r6, r6, r6, ror #8
+	uxtab16	r7, r7, r7, ror #8
+	uxtb16	r6, r6, ror #8
+	uxtb16	r7, r7, ror #8
+	orr	r5, r6, r7, lsl #8
+	uxtb16	r6, r4
+	uxtb16	r7, r4, ror #8
+	sub	r8, r11, r5, lsr #24
+	mla	r6, r6, r8, r2
+	mla	r7, r7, r8, r2
+	uxtab16	r6, r6, r6, ror #8
+	uxtab16	r7, r7, r7, ror #8
+	uxtb16	r6, r6, ror #8
+	uxtb16	r7, r7, ror #8
+	orr	r6, r6, r7, lsl #8
+	uqadd8	r5, r6, r5
+	str	r5, [r0], #4
+	subs	r12, r12, #1
+	bne	4b
+3:	ldr	r1, [sp, #12]
+	add	r10, r10, #1
+	cmp	r10, r1
+	ldr	r12, [sp]
+	ldr	r2, [sp, #4]
+	bne	5b
+0:	add	sp, sp, #28
+	pop	{r4, r5, r6, r7, r8, r9, r10, r11}
+	bx	lr
+.endfunc
+
+pixman_asm_function pixman_composite_over_n_8_8888_asm_armv6
+	push	{r4, r5, r6, r7, r8, r9, r10, r11}
+	sub	sp, sp, #28
+	cmp	r1, #0
+	ldr	r9, [sp, #60]
+	str	r1, [sp, #12]
+	bic	r1, r9, #-16777216
+	str	r1, [sp, #20]
+	mov	r12, r2
+	lsr	r1, r9, #8
+	ldr	r2, [sp, #20]
+	bic	r1, r1, #-16777216
+	bic	r2, r2, #65280
+	bic	r1, r1, #65280
+	str	r2, [sp, #20]
+	str	r0, [sp, #16]
+	str	r1, [sp, #4]
+	ldr	r2, [sp, #68]
+	beq	0f
+	lsl	r3, r3, #2
+	str	r3, [sp, #24]
+	mov	r0, #0
+	b	1f
+5:	ldr	r3, [sp, #24]
+1:	ldr	r4, [sp, #72]
+	mov	r10, r12
+	mov	r1, r2
+	add	r12, r12, r3
+	add	r2, r2, r4
+	str	r12, [sp, #8]
+	str	r2, [sp]
+	ldr	r12, [sp, #16]
+	ldr	r11, =0x00800080
+	ldr	r2, [sp, #4]
+	ldr	r3, [sp, #20]
+	cmp	r12, #0
+	beq	3f
+4:	ldrb	r5, [r1], #1
+	ldr	r4, [r10]
+	mla	r6, r3, r5, r11
+	mla	r7, r2, r5, r11
+	uxtab16	r6, r6, r6, ror #8
+	uxtab16	r7, r7, r7, ror #8
+	uxtb16	r6, r6, ror #8
+	uxtb16	r7, r7, ror #8
+	orr	r5, r6, r7, lsl #8
+	uxtb16	r6, r4
+	uxtb16	r7, r4, ror #8
+	mvn	r8, r5
+	lsr	r8, r8, #24
+	mla	r6, r6, r8, r11
+	mla	r7, r7, r8, r11
+	uxtab16	r6, r6, r6, ror #8
+	uxtab16	r7, r7, r7, ror #8
+	uxtb16	r6, r6, ror #8
+	uxtb16	r7, r7, ror #8
+	orr	r6, r6, r7, lsl #8
+	uqadd8	r5, r6, r5
+	str	r5, [r10], #4
+	subs	r12, r12, #1
+	bne	4b
+3:	ldr	r4, [sp, #12]
+	add	r0, r0, #1
+	cmp	r0, r4
+	ldr	r12, [sp, #8]
+	ldr	r2, [sp]
+	bne	5b
+0:	add	sp, sp, #28
+	pop	{r4, r5, r6, r7, r8, r9, r10, r11}
+	bx	lr
+.endfunc
diff --git a/pixman/pixman/pixman-arm-simd.c b/pixman/pixman/pixman-arm-simd.c
new file mode 100644
index 000000000..389c9e01a
--- /dev/null
+++ b/pixman/pixman/pixman-arm-simd.c
@@ -0,0 +1,417 @@
+/*
+ * Copyright © 2008 Mozilla Corporation
+ *
+ * Permission to use, copy, modify, distribute, and sell this software and its
+ * documentation for any purpose is hereby granted without fee, provided that
+ * the above copyright notice appear in all copies and that both that
+ * copyright notice and this permission notice appear in supporting
+ * documentation, and that the name of Mozilla Corporation not be used in
+ * advertising or publicity pertaining to distribution of the software without
+ * specific, written prior permission.  Mozilla Corporation makes no
+ * representations about the suitability of this software for any purpose.  It
+ * is provided "as is" without express or implied warranty.
+ *
+ * THE COPYRIGHT HOLDERS DISCLAIM ALL WARRANTIES WITH REGARD TO THIS
+ * SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
+ * FITNESS, IN NO EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
+ * AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
+ * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS
+ * SOFTWARE.
+ *
+ * Author:  Jeff Muizelaar (jeff@infidigm.net)
+ *
+ */
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include "pixman-private.h"
+#include "pixman-arm-common.h"
+
+#if 0 /* This code was moved to 'pixman-arm-simd-asm.S' */
+
+void
+pixman_composite_add_8000_8000_asm_armv6 (int32_t  width,
+                                          int32_t  height,
+                                          uint8_t *dst_line,
+                                          int32_t  dst_stride,
+                                          uint8_t *src_line,
+                                          int32_t  src_stride)
+{
+    uint8_t *dst, *src;
+    int32_t w;
+    uint8_t s, d;
+
+    while (height--)
+    {
+	dst = dst_line;
+	dst_line += dst_stride;
+	src = src_line;
+	src_line += src_stride;
+	w = width;
+
+	/* ensure both src and dst are properly aligned before doing 32 bit reads
+	 * we'll stay in this loop if src and dst have differing alignments
+	 */
+	while (w && (((unsigned long)dst & 3) || ((unsigned long)src & 3)))
+	{
+	    s = *src;
+	    d = *dst;
+	    asm ("uqadd8 %0, %1, %2" : "+r" (d) : "r" (s));
+	    *dst = d;
+
+	    dst++;
+	    src++;
+	    w--;
+	}
+
+	while (w >= 4)
+	{
+	    asm ("uqadd8 %0, %1, %2"
+		 : "=r" (*(uint32_t*)dst)
+		 : "r" (*(uint32_t*)src), "r" (*(uint32_t*)dst));
+	    dst += 4;
+	    src += 4;
+	    w -= 4;
+	}
+
+	while (w)
+	{
+	    s = *src;
+	    d = *dst;
+	    asm ("uqadd8 %0, %1, %2" : "+r" (d) : "r" (s));
+	    *dst = d;
+
+	    dst++;
+	    src++;
+	    w--;
+	}
+    }
+
+}
+
+void
+pixman_composite_over_8888_8888_asm_armv6 (int32_t   width,
+                                           int32_t   height,
+                                           uint32_t *dst_line,
+                                           int32_t   dst_stride,
+                                           uint32_t *src_line,
+                                           int32_t   src_stride)
+{
+    uint32_t    *dst;
+    uint32_t    *src;
+    int32_t w;
+    uint32_t component_half = 0x800080;
+    uint32_t upper_component_mask = 0xff00ff00;
+    uint32_t alpha_mask = 0xff;
+
+    while (height--)
+    {
+	dst = dst_line;
+	dst_line += dst_stride;
+	src = src_line;
+	src_line += src_stride;
+	w = width;
+
+/* #define inner_branch */
+	asm volatile (
+	    "cmp %[w], #0\n\t"
+	    "beq 2f\n\t"
+	    "1:\n\t"
+	    /* load src */
+	    "ldr r5, [%[src]], #4\n\t"
+#ifdef inner_branch
+	    /* We can avoid doing the multiplication in two cases: 0x0 or 0xff.
+	     * The 0x0 case also allows us to avoid doing an unecessary data
+	     * write which is more valuable so we only check for that
+	     */
+	    "cmp r5, #0\n\t"
+	    "beq 3f\n\t"
+
+	    /* = 255 - alpha */
+	    "sub r8, %[alpha_mask], r5, lsr #24\n\t"
+
+	    "ldr r4, [%[dest]] \n\t"
+
+#else
+	    "ldr r4, [%[dest]] \n\t"
+
+	    /* = 255 - alpha */
+	    "sub r8, %[alpha_mask], r5, lsr #24\n\t"
+#endif
+	    "uxtb16 r6, r4\n\t"
+	    "uxtb16 r7, r4, ror #8\n\t"
+
+	    /* multiply by 257 and divide by 65536 */
+	    "mla r6, r6, r8, %[component_half]\n\t"
+	    "mla r7, r7, r8, %[component_half]\n\t"
+
+	    "uxtab16 r6, r6, r6, ror #8\n\t"
+	    "uxtab16 r7, r7, r7, ror #8\n\t"
+
+	    /* recombine the 0xff00ff00 bytes of r6 and r7 */
+	    "and r7, r7, %[upper_component_mask]\n\t"
+	    "uxtab16 r6, r7, r6, ror #8\n\t"
+
+	    "uqadd8 r5, r6, r5\n\t"
+
+#ifdef inner_branch
+	    "3:\n\t"
+
+#endif
+	    "str r5, [%[dest]], #4\n\t"
+	    /* increment counter and jmp to top */
+	    "subs	%[w], %[w], #1\n\t"
+	    "bne	1b\n\t"
+	    "2:\n\t"
+	    : [w] "+r" (w), [dest] "+r" (dst), [src] "+r" (src)
+	    : [component_half] "r" (component_half), [upper_component_mask] "r" (upper_component_mask),
+	      [alpha_mask] "r" (alpha_mask)
+	    : "r4", "r5", "r6", "r7", "r8", "cc", "memory"
+	    );
+    }
+}
+
+void
+pixman_composite_over_8888_n_8888_asm_armv6 (int32_t   width,
+                                             int32_t   height,
+                                             uint32_t *dst_line,
+                                             int32_t   dst_stride,
+                                             uint32_t *src_line,
+                                             int32_t   src_stride,
+                                             uint32_t  mask)
+{
+    uint32_t *dst;
+    uint32_t *src;
+    int32_t w;
+    uint32_t component_half = 0x800080;
+    uint32_t alpha_mask = 0xff;
+
+    mask = (mask) >> 24;
+
+    while (height--)
+    {
+	dst = dst_line;
+	dst_line += dst_stride;
+	src = src_line;
+	src_line += src_stride;
+	w = width;
+
+/* #define inner_branch */
+	asm volatile (
+	    "cmp %[w], #0\n\t"
+	    "beq 2f\n\t"
+	    "1:\n\t"
+	    /* load src */
+	    "ldr r5, [%[src]], #4\n\t"
+#ifdef inner_branch
+	    /* We can avoid doing the multiplication in two cases: 0x0 or 0xff.
+	     * The 0x0 case also allows us to avoid doing an unecessary data
+	     * write which is more valuable so we only check for that
+	     */
+	    "cmp r5, #0\n\t"
+	    "beq 3f\n\t"
+
+#endif
+	    "ldr r4, [%[dest]] \n\t"
+
+	    "uxtb16 r6, r5\n\t"
+	    "uxtb16 r7, r5, ror #8\n\t"
+
+	    /* multiply by alpha (r8) then by 257 and divide by 65536 */
+	    "mla r6, r6, %[mask_alpha], %[component_half]\n\t"
+	    "mla r7, r7, %[mask_alpha], %[component_half]\n\t"
+
+	    "uxtab16 r6, r6, r6, ror #8\n\t"
+	    "uxtab16 r7, r7, r7, ror #8\n\t"
+
+	    "uxtb16 r6, r6, ror #8\n\t"
+	    "uxtb16 r7, r7, ror #8\n\t"
+
+	    /* recombine */
+	    "orr r5, r6, r7, lsl #8\n\t"
+
+	    "uxtb16 r6, r4\n\t"
+	    "uxtb16 r7, r4, ror #8\n\t"
+
+	    /* 255 - alpha */
+	    "sub r8, %[alpha_mask], r5, lsr #24\n\t"
+
+	    /* multiply by alpha (r8) then by 257 and divide by 65536 */
+	    "mla r6, r6, r8, %[component_half]\n\t"
+	    "mla r7, r7, r8, %[component_half]\n\t"
+
+	    "uxtab16 r6, r6, r6, ror #8\n\t"
+	    "uxtab16 r7, r7, r7, ror #8\n\t"
+
+	    "uxtb16 r6, r6, ror #8\n\t"
+	    "uxtb16 r7, r7, ror #8\n\t"
+
+	    /* recombine */
+	    "orr r6, r6, r7, lsl #8\n\t"
+
+	    "uqadd8 r5, r6, r5\n\t"
+
+#ifdef inner_branch
+	    "3:\n\t"
+
+#endif
+	    "str r5, [%[dest]], #4\n\t"
+	    /* increment counter and jmp to top */
+	    "subs	%[w], %[w], #1\n\t"
+	    "bne	1b\n\t"
+	    "2:\n\t"
+	    : [w] "+r" (w), [dest] "+r" (dst), [src] "+r" (src)
+	    : [component_half] "r" (component_half), [mask_alpha] "r" (mask),
+	      [alpha_mask] "r" (alpha_mask)
+	    : "r4", "r5", "r6", "r7", "r8", "r9", "cc", "memory"
+	    );
+    }
+}
+
+void
+pixman_composite_over_n_8_8888_asm_armv6 (int32_t   width,
+                                          int32_t   height,
+                                          uint32_t *dst_line,
+                                          int32_t   dst_stride,
+                                          uint32_t  src,
+                                          int32_t   unused,
+                                          uint8_t  *mask_line,
+                                          int32_t   mask_stride)
+{
+    uint32_t  srca;
+    uint32_t *dst;
+    uint8_t  *mask;
+    int32_t w;
+
+    srca = src >> 24;
+
+    uint32_t component_mask = 0xff00ff;
+    uint32_t component_half = 0x800080;
+
+    uint32_t src_hi = (src >> 8) & component_mask;
+    uint32_t src_lo = src & component_mask;
+
+    while (height--)
+    {
+	dst = dst_line;
+	dst_line += dst_stride;
+	mask = mask_line;
+	mask_line += mask_stride;
+	w = width;
+
+/* #define inner_branch */
+	asm volatile (
+	    "cmp %[w], #0\n\t"
+	    "beq 2f\n\t"
+	    "1:\n\t"
+	    /* load mask */
+	    "ldrb r5, [%[mask]], #1\n\t"
+#ifdef inner_branch
+	    /* We can avoid doing the multiplication in two cases: 0x0 or 0xff.
+	     * The 0x0 case also allows us to avoid doing an unecessary data
+	     * write which is more valuable so we only check for that
+	     */
+	    "cmp r5, #0\n\t"
+	    "beq 3f\n\t"
+
+#endif
+	    "ldr r4, [%[dest]] \n\t"
+
+	    /* multiply by alpha (r8) then by 257 and divide by 65536 */
+	    "mla r6, %[src_lo], r5, %[component_half]\n\t"
+	    "mla r7, %[src_hi], r5, %[component_half]\n\t"
+
+	    "uxtab16 r6, r6, r6, ror #8\n\t"
+	    "uxtab16 r7, r7, r7, ror #8\n\t"
+
+	    "uxtb16 r6, r6, ror #8\n\t"
+	    "uxtb16 r7, r7, ror #8\n\t"
+
+	    /* recombine */
+	    "orr r5, r6, r7, lsl #8\n\t"
+
+	    "uxtb16 r6, r4\n\t"
+	    "uxtb16 r7, r4, ror #8\n\t"
+
+	    /* we could simplify this to use 'sub' if we were
+	     * willing to give up a register for alpha_mask
+	     */
+	    "mvn r8, r5\n\t"
+	    "mov r8, r8, lsr #24\n\t"
+
+	    /* multiply by alpha (r8) then by 257 and divide by 65536 */
+	    "mla r6, r6, r8, %[component_half]\n\t"
+	    "mla r7, r7, r8, %[component_half]\n\t"
+
+	    "uxtab16 r6, r6, r6, ror #8\n\t"
+	    "uxtab16 r7, r7, r7, ror #8\n\t"
+
+	    "uxtb16 r6, r6, ror #8\n\t"
+	    "uxtb16 r7, r7, ror #8\n\t"
+
+	    /* recombine */
+	    "orr r6, r6, r7, lsl #8\n\t"
+
+	    "uqadd8 r5, r6, r5\n\t"
+
+#ifdef inner_branch
+	    "3:\n\t"
+
+#endif
+	    "str r5, [%[dest]], #4\n\t"
+	    /* increment counter and jmp to top */
+	    "subs	%[w], %[w], #1\n\t"
+	    "bne	1b\n\t"
+	    "2:\n\t"
+	    : [w] "+r" (w), [dest] "+r" (dst), [src] "+r" (src), [mask] "+r" (mask)
+	    : [component_half] "r" (component_half),
+	      [src_hi] "r" (src_hi), [src_lo] "r" (src_lo)
+	    : "r4", "r5", "r6", "r7", "r8", "cc", "memory");
+    }
+}
+
+#endif
+
+PIXMAN_ARM_BIND_FAST_PATH_SRC_DST (armv6, add_8000_8000,
+                                   uint8_t, 1, uint8_t, 1)
+PIXMAN_ARM_BIND_FAST_PATH_SRC_DST (armv6, over_8888_8888,
+                                   uint32_t, 1, uint32_t, 1)
+
+PIXMAN_ARM_BIND_FAST_PATH_SRC_N_DST (armv6, over_8888_n_8888,
+                                     uint32_t, 1, uint32_t, 1)
+
+PIXMAN_ARM_BIND_FAST_PATH_N_MASK_DST (armv6, over_n_8_8888,
+                                      uint8_t, 1, uint32_t, 1)
+
+static const pixman_fast_path_t arm_simd_fast_paths[] =
+{
+    PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, null, a8r8g8b8, armv6_composite_over_8888_8888),
+    PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, null, x8r8g8b8, armv6_composite_over_8888_8888),
+    PIXMAN_STD_FAST_PATH (OVER, a8b8g8r8, null, a8b8g8r8, armv6_composite_over_8888_8888),
+    PIXMAN_STD_FAST_PATH (OVER, a8b8g8r8, null, x8b8g8r8, armv6_composite_over_8888_8888),
+    PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, solid, a8r8g8b8, armv6_composite_over_8888_n_8888),
+    PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, solid, x8r8g8b8, armv6_composite_over_8888_n_8888),
+    PIXMAN_STD_FAST_PATH (OVER, a8b8g8r8, solid, a8b8g8r8, armv6_composite_over_8888_n_8888),
+    PIXMAN_STD_FAST_PATH (OVER, a8b8g8r8, solid, x8b8g8r8, armv6_composite_over_8888_n_8888),
+
+    PIXMAN_STD_FAST_PATH (ADD, a8, null, a8, armv6_composite_add_8000_8000),
+
+    PIXMAN_STD_FAST_PATH (OVER, solid, a8, a8r8g8b8, armv6_composite_over_n_8_8888),
+    PIXMAN_STD_FAST_PATH (OVER, solid, a8, x8r8g8b8, armv6_composite_over_n_8_8888),
+    PIXMAN_STD_FAST_PATH (OVER, solid, a8, a8b8g8r8, armv6_composite_over_n_8_8888),
+    PIXMAN_STD_FAST_PATH (OVER, solid, a8, x8b8g8r8, armv6_composite_over_n_8_8888),
+
+    { PIXMAN_OP_NONE },
+};
+
+pixman_implementation_t *
+_pixman_implementation_create_arm_simd (void)
+{
+    pixman_implementation_t *general = _pixman_implementation_create_fast_path ();
+    pixman_implementation_t *imp = _pixman_implementation_create (general, arm_simd_fast_paths);
+
+    return imp;
+}
diff --git a/pixman/pixman/pixman-bits-image.c b/pixman/pixman/pixman-bits-image.c
new file mode 100644
index 000000000..0225ae5aa
--- /dev/null
+++ b/pixman/pixman/pixman-bits-image.c
@@ -0,0 +1,1034 @@
+/*
+ * Copyright © 2000 Keith Packard, member of The XFree86 Project, Inc.
+ *             2005 Lars Knoll & Zack Rusin, Trolltech
+ *             2008 Aaron Plattner, NVIDIA Corporation
+ * Copyright © 2000 SuSE, Inc.
+ * Copyright © 2007, 2009 Red Hat, Inc.
+ * Copyright © 2008 André Tupinambá <andrelrt@gmail.com>
+ *
+ * Permission to use, copy, modify, distribute, and sell this software and its
+ * documentation for any purpose is hereby granted without fee, provided that
+ * the above copyright notice appear in all copies and that both that
+ * copyright notice and this permission notice appear in supporting
+ * documentation, and that the name of Keith Packard not be used in
+ * advertising or publicity pertaining to distribution of the software without
+ * specific, written prior permission.  Keith Packard makes no
+ * representations about the suitability of this software for any purpose.  It
+ * is provided "as is" without express or implied warranty.
+ *
+ * THE COPYRIGHT HOLDERS DISCLAIM ALL WARRANTIES WITH REGARD TO THIS
+ * SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
+ * FITNESS, IN NO EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
+ * AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
+ * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS
+ * SOFTWARE.
+ */
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include "pixman-private.h"
+#include "pixman-combine32.h"
+
+/* Store functions */
+
+static void
+bits_image_store_scanline_32 (bits_image_t *  image,
+                              int             x,
+                              int             y,
+                              int             width,
+                              const uint32_t *buffer)
+{
+    image->store_scanline_raw_32 (image, x, y, width, buffer);
+
+    if (image->common.alpha_map)
+    {
+	x -= image->common.alpha_origin_x;
+	y -= image->common.alpha_origin_y;
+
+	bits_image_store_scanline_32 (image->common.alpha_map, x, y, width, buffer);
+    }
+}
+
+static void
+bits_image_store_scanline_64 (bits_image_t *  image,
+                              int             x,
+                              int             y,
+                              int             width,
+                              const uint32_t *buffer)
+{
+    image->store_scanline_raw_64 (image, x, y, width, buffer);
+
+    if (image->common.alpha_map)
+    {
+	x -= image->common.alpha_origin_x;
+	y -= image->common.alpha_origin_y;
+
+	bits_image_store_scanline_64 (image->common.alpha_map, x, y, width, buffer);
+    }
+}
+
+void
+_pixman_image_store_scanline_32 (bits_image_t *  image,
+                                 int             x,
+                                 int             y,
+                                 int             width,
+                                 const uint32_t *buffer)
+{
+    image->store_scanline_32 (image, x, y, width, buffer);
+}
+
+void
+_pixman_image_store_scanline_64 (bits_image_t *  image,
+                                 int             x,
+                                 int             y,
+                                 int             width,
+                                 const uint32_t *buffer)
+{
+    image->store_scanline_64 (image, x, y, width, buffer);
+}
+
+/* Fetch functions */
+
+static uint32_t
+bits_image_fetch_pixel_alpha (bits_image_t *image, int x, int y)
+{
+    uint32_t pixel;
+    uint32_t pixel_a;
+
+    pixel = image->fetch_pixel_raw_32 (image, x, y);
+
+    assert (image->common.alpha_map);
+
+    x -= image->common.alpha_origin_x;
+    y -= image->common.alpha_origin_y;
+
+    if (x < 0 || x >= image->common.alpha_map->width ||
+	y < 0 || y >= image->common.alpha_map->height)
+    {
+	pixel_a = 0;
+    }
+    else
+    {
+	pixel_a = image->common.alpha_map->fetch_pixel_raw_32 (
+	    image->common.alpha_map, x, y);
+	pixel_a = ALPHA_8 (pixel_a);
+    }
+
+    pixel &= 0x00ffffff;
+    pixel |= (pixel_a << 24);
+
+    return pixel;
+}
+
+static force_inline uint32_t
+get_pixel (bits_image_t *image, int x, int y, pixman_bool_t check_bounds)
+{
+    if (check_bounds &&
+	(x < 0 || x >= image->width || y < 0 || y >= image->height))
+    {
+	return 0;
+    }
+
+    return image->fetch_pixel_32 (image, x, y);
+}
+
+static force_inline void
+repeat (pixman_repeat_t repeat, int size, int *coord)
+{
+    switch (repeat)
+    {
+    case PIXMAN_REPEAT_NORMAL:
+	*coord = MOD (*coord, size);
+	break;
+
+    case PIXMAN_REPEAT_PAD:
+	*coord = CLIP (*coord, 0, size - 1);
+	break;
+
+    case PIXMAN_REPEAT_REFLECT:
+	*coord = MOD (*coord, size * 2);
+
+	if (*coord >= size)
+	    *coord = size * 2 - *coord - 1;
+	break;
+
+    case PIXMAN_REPEAT_NONE:
+	break;
+
+    default:
+        break;
+    }
+}
+
+static force_inline uint32_t
+bits_image_fetch_pixel_nearest (bits_image_t   *image,
+				pixman_fixed_t  x,
+				pixman_fixed_t  y)
+{
+    int x0 = pixman_fixed_to_int (x - pixman_fixed_e);
+    int y0 = pixman_fixed_to_int (y - pixman_fixed_e);
+
+    if (image->common.repeat != PIXMAN_REPEAT_NONE)
+    {
+	repeat (image->common.repeat, image->width, &x0);
+	repeat (image->common.repeat, image->height, &y0);
+
+	return get_pixel (image, x0, y0, FALSE);
+    }
+    else
+    {
+	return get_pixel (image, x0, y0, TRUE);
+    }
+}
+
+#if SIZEOF_LONG > 4
+
+static force_inline uint32_t
+bilinear_interpolation (uint32_t tl, uint32_t tr,
+			uint32_t bl, uint32_t br,
+			int distx, int disty)
+{
+    uint64_t distxy, distxiy, distixy, distixiy;
+    uint64_t tl64, tr64, bl64, br64;
+    uint64_t f, r;
+
+    distxy = distx * disty;
+    distxiy = distx * (256 - disty);
+    distixy = (256 - distx) * disty;
+    distixiy = (256 - distx) * (256 - disty);
+
+    /* Alpha and Blue */
+    tl64 = tl & 0xff0000ff;
+    tr64 = tr & 0xff0000ff;
+    bl64 = bl & 0xff0000ff;
+    br64 = br & 0xff0000ff;
+
+    f = tl64 * distixiy + tr64 * distxiy + bl64 * distixy + br64 * distxy;
+    r = f & 0x0000ff0000ff0000ull;
+
+    /* Red and Green */
+    tl64 = tl;
+    tl64 = ((tl64 << 16) & 0x000000ff00000000ull) | (tl64 & 0x0000ff00ull);
+
+    tr64 = tr;
+    tr64 = ((tr64 << 16) & 0x000000ff00000000ull) | (tr64 & 0x0000ff00ull);
+
+    bl64 = bl;
+    bl64 = ((bl64 << 16) & 0x000000ff00000000ull) | (bl64 & 0x0000ff00ull);
+
+    br64 = br;
+    br64 = ((br64 << 16) & 0x000000ff00000000ull) | (br64 & 0x0000ff00ull);
+
+    f = tl64 * distixiy + tr64 * distxiy + bl64 * distixy + br64 * distxy;
+    r |= ((f >> 16) & 0x000000ff00000000ull) | (f & 0xff000000ull);
+
+    return (uint32_t)(r >> 16);
+}
+
+#else
+
+static force_inline uint32_t
+bilinear_interpolation (uint32_t tl, uint32_t tr,
+			uint32_t bl, uint32_t br,
+			int distx, int disty)
+{
+    int distxy, distxiy, distixy, distixiy;
+    uint32_t f, r;
+
+    distxy = distx * disty;
+    distxiy = (distx << 8) - distxy;	/* distx * (256 - disty) */
+    distixy = (disty << 8) - distxy;	/* disty * (256 - distx) */
+    distixiy =
+	256 * 256 - (disty << 8) -
+	(distx << 8) + distxy;		/* (256 - distx) * (256 - disty) */
+
+    /* Blue */
+    r = (tl & 0x000000ff) * distixiy + (tr & 0x000000ff) * distxiy
+      + (bl & 0x000000ff) * distixy  + (br & 0x000000ff) * distxy;
+
+    /* Green */
+    f = (tl & 0x0000ff00) * distixiy + (tr & 0x0000ff00) * distxiy
+      + (bl & 0x0000ff00) * distixy  + (br & 0x0000ff00) * distxy;
+    r |= f & 0xff000000;
+
+    tl >>= 16;
+    tr >>= 16;
+    bl >>= 16;
+    br >>= 16;
+    r >>= 16;
+
+    /* Red */
+    f = (tl & 0x000000ff) * distixiy + (tr & 0x000000ff) * distxiy
+      + (bl & 0x000000ff) * distixy  + (br & 0x000000ff) * distxy;
+    r |= f & 0x00ff0000;
+
+    /* Alpha */
+    f = (tl & 0x0000ff00) * distixiy + (tr & 0x0000ff00) * distxiy
+      + (bl & 0x0000ff00) * distixy  + (br & 0x0000ff00) * distxy;
+    r |= f & 0xff000000;
+
+    return r;
+}
+
+#endif
+
+static force_inline uint32_t
+bits_image_fetch_pixel_bilinear (bits_image_t   *image,
+				 pixman_fixed_t  x,
+				 pixman_fixed_t  y)
+{
+    pixman_repeat_t repeat_mode = image->common.repeat;
+    int width = image->width;
+    int height = image->height;
+    int x1, y1, x2, y2;
+    uint32_t tl, tr, bl, br;
+    int32_t distx, disty;
+
+    x1 = x - pixman_fixed_1 / 2;
+    y1 = y - pixman_fixed_1 / 2;
+
+    distx = (x1 >> 8) & 0xff;
+    disty = (y1 >> 8) & 0xff;
+
+    x1 = pixman_fixed_to_int (x1);
+    y1 = pixman_fixed_to_int (y1);
+    x2 = x1 + 1;
+    y2 = y1 + 1;
+
+    if (repeat_mode != PIXMAN_REPEAT_NONE)
+    {
+	repeat (repeat_mode, width, &x1);
+	repeat (repeat_mode, height, &y1);
+	repeat (repeat_mode, width, &x2);
+	repeat (repeat_mode, height, &y2);
+
+	tl = get_pixel (image, x1, y1, FALSE);
+	bl = get_pixel (image, x1, y2, FALSE);
+	tr = get_pixel (image, x2, y1, FALSE);
+	br = get_pixel (image, x2, y2, FALSE);
+    }
+    else
+    {
+	tl = get_pixel (image, x1, y1, TRUE);
+	tr = get_pixel (image, x2, y1, TRUE);
+	bl = get_pixel (image, x1, y2, TRUE);
+	br = get_pixel (image, x2, y2, TRUE);
+    }
+
+    return bilinear_interpolation (tl, tr, bl, br, distx, disty);
+}
+
+static void
+bits_image_fetch_bilinear_no_repeat_8888 (pixman_image_t * ima,
+					  int              offset,
+					  int              line,
+					  int              width,
+					  uint32_t *       buffer,
+					  const uint32_t * mask,
+					  uint32_t         mask_bits)
+{
+    bits_image_t *bits = &ima->bits;
+    pixman_fixed_t x_top, x_bottom, x;
+    pixman_fixed_t ux_top, ux_bottom, ux;
+    pixman_vector_t v;
+    uint32_t top_mask, bottom_mask;
+    uint32_t *top_row;
+    uint32_t *bottom_row;
+    uint32_t *end;
+    uint32_t zero[2] = { 0, 0 };
+    int y, y1, y2;
+    int disty;
+    int mask_inc;
+    int w;
+
+    /* reference point is the center of the pixel */
+    v.vector[0] = pixman_int_to_fixed (offset) + pixman_fixed_1 / 2;
+    v.vector[1] = pixman_int_to_fixed (line) + pixman_fixed_1 / 2;
+    v.vector[2] = pixman_fixed_1;
+
+    if (!pixman_transform_point_3d (bits->common.transform, &v))
+	return;
+
+    ux = ux_top = ux_bottom = bits->common.transform->matrix[0][0];
+    x = x_top = x_bottom = v.vector[0] - pixman_fixed_1/2;
+
+    y = v.vector[1] - pixman_fixed_1/2;
+    disty = (y >> 8) & 0xff;
+
+    /* Load the pointers to the first and second lines from the source
+     * image that bilinear code must read.
+     *
+     * The main trick in this code is about the check if any line are
+     * outside of the image;
+     *
+     * When I realize that a line (any one) is outside, I change
+     * the pointer to a dummy area with zeros. Once I change this, I
+     * must be sure the pointer will not change, so I set the
+     * variables to each pointer increments inside the loop.
+     */
+    y1 = pixman_fixed_to_int (y);
+    y2 = y1 + 1;
+
+    if (y1 < 0 || y1 >= bits->height)
+    {
+	top_row = zero;
+	x_top = 0;
+	ux_top = 0;
+    }
+    else
+    {
+	top_row = bits->bits + y1 * bits->rowstride;
+	x_top = x;
+	ux_top = ux;
+    }
+
+    if (y2 < 0 || y2 >= bits->height)
+    {
+	bottom_row = zero;
+	x_bottom = 0;
+	ux_bottom = 0;
+    }
+    else
+    {
+	bottom_row = bits->bits + y2 * bits->rowstride;
+	x_bottom = x;
+	ux_bottom = ux;
+    }
+
+    /* Instead of checking whether the operation uses the mast in
+     * each loop iteration, verify this only once and prepare the
+     * variables to make the code smaller inside the loop.
+     */
+    if (!mask)
+    {
+        mask_inc = 0;
+        mask_bits = 1;
+        mask = &mask_bits;
+    }
+    else
+    {
+        /* If have a mask, prepare the variables to check it */
+        mask_inc = 1;
+    }
+
+    /* If both are zero, then the whole thing is zero */
+    if (top_row == zero && bottom_row == zero)
+    {
+	memset (buffer, 0, width * sizeof (uint32_t));
+	return;
+    }
+    else if (bits->format == PIXMAN_x8r8g8b8)
+    {
+	if (top_row == zero)
+	{
+	    top_mask = 0;
+	    bottom_mask = 0xff000000;
+	}
+	else if (bottom_row == zero)
+	{
+	    top_mask = 0xff000000;
+	    bottom_mask = 0;
+	}
+	else
+	{
+	    top_mask = 0xff000000;
+	    bottom_mask = 0xff000000;
+	}
+    }
+    else
+    {
+	top_mask = 0;
+	bottom_mask = 0;
+    }
+
+    end = buffer + width;
+
+    /* Zero fill to the left of the image */
+    while (buffer < end && x < pixman_fixed_minus_1)
+    {
+	*buffer++ = 0;
+	x += ux;
+	x_top += ux_top;
+	x_bottom += ux_bottom;
+	mask += mask_inc;
+    }
+
+    /* Left edge
+     */
+    while (buffer < end && x < 0)
+    {
+	uint32_t tr, br;
+	int32_t distx;
+
+	tr = top_row[pixman_fixed_to_int (x_top) + 1] | top_mask;
+	br = bottom_row[pixman_fixed_to_int (x_bottom) + 1] | bottom_mask;
+
+	distx = (x >> 8) & 0xff;
+
+	*buffer++ = bilinear_interpolation (0, tr, 0, br, distx, disty);
+
+	x += ux;
+	x_top += ux_top;
+	x_bottom += ux_bottom;
+	mask += mask_inc;
+    }
+
+    /* Main part */
+    w = pixman_int_to_fixed (bits->width - 1);
+
+    while (buffer < end  &&  x < w)
+    {
+	if (*mask)
+	{
+	    uint32_t tl, tr, bl, br;
+	    int32_t distx;
+
+	    tl = top_row [pixman_fixed_to_int (x_top)] | top_mask;
+	    tr = top_row [pixman_fixed_to_int (x_top) + 1] | top_mask;
+	    bl = bottom_row [pixman_fixed_to_int (x_bottom)] | bottom_mask;
+	    br = bottom_row [pixman_fixed_to_int (x_bottom) + 1] | bottom_mask;
+
+	    distx = (x >> 8) & 0xff;
+
+	    *buffer = bilinear_interpolation (tl, tr, bl, br, distx, disty);
+	}
+
+	buffer++;
+	x += ux;
+	x_top += ux_top;
+	x_bottom += ux_bottom;
+	mask += mask_inc;
+    }
+
+    /* Right Edge */
+    w = pixman_int_to_fixed (bits->width);
+    while (buffer < end  &&  x < w)
+    {
+	if (*mask)
+	{
+	    uint32_t tl, bl;
+	    int32_t distx;
+
+	    tl = top_row [pixman_fixed_to_int (x_top)] | top_mask;
+	    bl = bottom_row [pixman_fixed_to_int (x_bottom)] | bottom_mask;
+
+	    distx = (x >> 8) & 0xff;
+
+	    *buffer = bilinear_interpolation (tl, 0, bl, 0, distx, disty);
+	}
+
+	buffer++;
+	x += ux;
+	x_top += ux_top;
+	x_bottom += ux_bottom;
+	mask += mask_inc;
+    }
+
+    /* Zero fill to the left of the image */
+    while (buffer < end)
+	*buffer++ = 0;
+}
+
+static force_inline uint32_t
+bits_image_fetch_pixel_convolution (bits_image_t   *image,
+				    pixman_fixed_t  x,
+				    pixman_fixed_t  y)
+{
+    pixman_fixed_t *params = image->common.filter_params;
+    int x_off = (params[0] - pixman_fixed_1) >> 1;
+    int y_off = (params[1] - pixman_fixed_1) >> 1;
+    int32_t cwidth = pixman_fixed_to_int (params[0]);
+    int32_t cheight = pixman_fixed_to_int (params[1]);
+    int32_t srtot, sgtot, sbtot, satot;
+    int32_t i, j, x1, x2, y1, y2;
+    pixman_repeat_t repeat_mode = image->common.repeat;
+    int width = image->width;
+    int height = image->height;
+
+    params += 2;
+
+    x1 = pixman_fixed_to_int (x - pixman_fixed_e - x_off);
+    y1 = pixman_fixed_to_int (y - pixman_fixed_e - y_off);
+    x2 = x1 + cwidth;
+    y2 = y1 + cheight;
+
+    srtot = sgtot = sbtot = satot = 0;
+
+    for (i = y1; i < y2; ++i)
+    {
+	for (j = x1; j < x2; ++j)
+	{
+	    int rx = j;
+	    int ry = i;
+
+	    pixman_fixed_t f = *params;
+
+	    if (f)
+	    {
+		uint32_t pixel;
+
+		if (repeat_mode != PIXMAN_REPEAT_NONE)
+		{
+		    repeat (repeat_mode, width, &rx);
+		    repeat (repeat_mode, height, &ry);
+
+		    pixel = get_pixel (image, rx, ry, FALSE);
+		}
+		else
+		{
+		    pixel = get_pixel (image, rx, ry, TRUE);
+		}
+
+		srtot += RED_8 (pixel) * f;
+		sgtot += GREEN_8 (pixel) * f;
+		sbtot += BLUE_8 (pixel) * f;
+		satot += ALPHA_8 (pixel) * f;
+	    }
+
+	    params++;
+	}
+    }
+
+    satot >>= 16;
+    srtot >>= 16;
+    sgtot >>= 16;
+    sbtot >>= 16;
+
+    satot = CLIP (satot, 0, 0xff);
+    srtot = CLIP (srtot, 0, 0xff);
+    sgtot = CLIP (sgtot, 0, 0xff);
+    sbtot = CLIP (sbtot, 0, 0xff);
+
+    return ((satot << 24) | (srtot << 16) | (sgtot <<  8) | (sbtot));
+}
+
+static force_inline uint32_t
+bits_image_fetch_pixel_filtered (bits_image_t *image,
+				 pixman_fixed_t x,
+				 pixman_fixed_t y)
+{
+    switch (image->common.filter)
+    {
+    case PIXMAN_FILTER_NEAREST:
+    case PIXMAN_FILTER_FAST:
+	return bits_image_fetch_pixel_nearest (image, x, y);
+	break;
+
+    case PIXMAN_FILTER_BILINEAR:
+    case PIXMAN_FILTER_GOOD:
+    case PIXMAN_FILTER_BEST:
+	return bits_image_fetch_pixel_bilinear (image, x, y);
+	break;
+
+    case PIXMAN_FILTER_CONVOLUTION:
+	return bits_image_fetch_pixel_convolution (image, x, y);
+	break;
+
+    default:
+        break;
+    }
+
+    return 0;
+}
+
+static void
+bits_image_fetch_transformed (pixman_image_t * image,
+                              int              offset,
+                              int              line,
+                              int              width,
+                              uint32_t *       buffer,
+                              const uint32_t * mask,
+                              uint32_t         mask_bits)
+{
+    pixman_fixed_t x, y, w;
+    pixman_fixed_t ux, uy, uw;
+    pixman_vector_t v;
+    int i;
+
+    /* reference point is the center of the pixel */
+    v.vector[0] = pixman_int_to_fixed (offset) + pixman_fixed_1 / 2;
+    v.vector[1] = pixman_int_to_fixed (line) + pixman_fixed_1 / 2;
+    v.vector[2] = pixman_fixed_1;
+
+    /* when using convolution filters or PIXMAN_REPEAT_PAD one
+     * might get here without a transform */
+    if (image->common.transform)
+    {
+	if (!pixman_transform_point_3d (image->common.transform, &v))
+	    return;
+
+	ux = image->common.transform->matrix[0][0];
+	uy = image->common.transform->matrix[1][0];
+	uw = image->common.transform->matrix[2][0];
+    }
+    else
+    {
+	ux = pixman_fixed_1;
+	uy = 0;
+	uw = 0;
+    }
+
+    x = v.vector[0];
+    y = v.vector[1];
+    w = v.vector[2];
+
+    if (w == pixman_fixed_1 && uw == 0) /* Affine */
+    {
+	for (i = 0; i < width; ++i)
+	{
+	    if (!mask || (mask[i] & mask_bits))
+	    {
+		buffer[i] =
+		    bits_image_fetch_pixel_filtered (&image->bits, x, y);
+	    }
+
+	    x += ux;
+	    y += uy;
+	}
+    }
+    else
+    {
+	for (i = 0; i < width; ++i)
+	{
+	    pixman_fixed_t x0, y0;
+
+	    if (!mask || (mask[i] & mask_bits))
+	    {
+		x0 = ((pixman_fixed_48_16_t)x << 16) / w;
+		y0 = ((pixman_fixed_48_16_t)y << 16) / w;
+
+		buffer[i] =
+		    bits_image_fetch_pixel_filtered (&image->bits, x0, y0);
+	    }
+
+	    x += ux;
+	    y += uy;
+	    w += uw;
+	}
+    }
+}
+
+static void
+bits_image_fetch_solid_32 (pixman_image_t * image,
+                           int              x,
+                           int              y,
+                           int              width,
+                           uint32_t *       buffer,
+                           const uint32_t * mask,
+                           uint32_t         mask_bits)
+{
+    uint32_t color;
+    uint32_t *end;
+
+    color = image->bits.fetch_pixel_raw_32 (&image->bits, 0, 0);
+
+    end = buffer + width;
+    while (buffer < end)
+	*(buffer++) = color;
+}
+
+static void
+bits_image_fetch_solid_64 (pixman_image_t * image,
+                           int              x,
+                           int              y,
+                           int              width,
+                           uint32_t *       b,
+                           const uint32_t * unused,
+                           uint32_t         unused2)
+{
+    uint64_t color;
+    uint64_t *buffer = (uint64_t *)b;
+    uint64_t *end;
+
+    color = image->bits.fetch_pixel_raw_64 (&image->bits, 0, 0);
+
+    end = buffer + width;
+    while (buffer < end)
+	*(buffer++) = color;
+}
+
+static void
+bits_image_fetch_untransformed_repeat_none (bits_image_t *image,
+                                            pixman_bool_t wide,
+                                            int           x,
+                                            int           y,
+                                            int           width,
+                                            uint32_t *    buffer)
+{
+    uint32_t w;
+
+    if (y < 0 || y >= image->height)
+    {
+	memset (buffer, 0, width * (wide? 8 : 4));
+	return;
+    }
+
+    if (x < 0)
+    {
+	w = MIN (width, -x);
+
+	memset (buffer, 0, w * (wide ? 8 : 4));
+
+	width -= w;
+	buffer += w * (wide? 2 : 1);
+	x += w;
+    }
+
+    if (x < image->width)
+    {
+	w = MIN (width, image->width - x);
+
+	if (wide)
+	    image->fetch_scanline_raw_64 ((pixman_image_t *)image, x, y, w, buffer, NULL, 0);
+	else
+	    image->fetch_scanline_raw_32 ((pixman_image_t *)image, x, y, w, buffer, NULL, 0);
+
+	width -= w;
+	buffer += w * (wide? 2 : 1);
+	x += w;
+    }
+
+    memset (buffer, 0, width * (wide ? 8 : 4));
+}
+
+static void
+bits_image_fetch_untransformed_repeat_normal (bits_image_t *image,
+                                              pixman_bool_t wide,
+                                              int           x,
+                                              int           y,
+                                              int           width,
+                                              uint32_t *    buffer)
+{
+    uint32_t w;
+
+    while (y < 0)
+	y += image->height;
+
+    while (y >= image->height)
+	y -= image->height;
+
+    while (width)
+    {
+	while (x < 0)
+	    x += image->width;
+	while (x >= image->width)
+	    x -= image->width;
+
+	w = MIN (width, image->width - x);
+
+	if (wide)
+	    image->fetch_scanline_raw_64 ((pixman_image_t *)image, x, y, w, buffer, NULL, 0);
+	else
+	    image->fetch_scanline_raw_32 ((pixman_image_t *)image, x, y, w, buffer, NULL, 0);
+
+	buffer += w * (wide? 2 : 1);
+	x += w;
+	width -= w;
+    }
+}
+
+static void
+bits_image_fetch_untransformed_32 (pixman_image_t * image,
+                                   int              x,
+                                   int              y,
+                                   int              width,
+                                   uint32_t *       buffer,
+                                   const uint32_t * mask,
+                                   uint32_t         mask_bits)
+{
+    if (image->common.repeat == PIXMAN_REPEAT_NONE)
+    {
+	bits_image_fetch_untransformed_repeat_none (
+	    &image->bits, FALSE, x, y, width, buffer);
+    }
+    else
+    {
+	bits_image_fetch_untransformed_repeat_normal (
+	    &image->bits, FALSE, x, y, width, buffer);
+    }
+}
+
+static void
+bits_image_fetch_untransformed_64 (pixman_image_t * image,
+                                   int              x,
+                                   int              y,
+                                   int              width,
+                                   uint32_t *       buffer,
+                                   const uint32_t * unused,
+                                   uint32_t         unused2)
+{
+    if (image->common.repeat == PIXMAN_REPEAT_NONE)
+    {
+	bits_image_fetch_untransformed_repeat_none (
+	    &image->bits, TRUE, x, y, width, buffer);
+    }
+    else
+    {
+	bits_image_fetch_untransformed_repeat_normal (
+	    &image->bits, TRUE, x, y, width, buffer);
+    }
+}
+
+static void
+bits_image_property_changed (pixman_image_t *image)
+{
+    bits_image_t *bits = (bits_image_t *)image;
+
+    _pixman_bits_image_setup_raw_accessors (bits);
+
+    image->bits.fetch_pixel_32 = image->bits.fetch_pixel_raw_32;
+
+    if (bits->common.alpha_map)
+    {
+	image->common.get_scanline_64 =
+	    _pixman_image_get_scanline_generic_64;
+	image->common.get_scanline_32 =
+	    bits_image_fetch_transformed;
+
+	image->bits.fetch_pixel_32 = bits_image_fetch_pixel_alpha;
+    }
+    else if ((bits->common.repeat != PIXMAN_REPEAT_NONE) &&
+             bits->width == 1 &&
+             bits->height == 1)
+    {
+	image->common.get_scanline_64 = bits_image_fetch_solid_64;
+	image->common.get_scanline_32 = bits_image_fetch_solid_32;
+    }
+    else if (!bits->common.transform &&
+             bits->common.filter != PIXMAN_FILTER_CONVOLUTION &&
+             (bits->common.repeat == PIXMAN_REPEAT_NONE ||
+              bits->common.repeat == PIXMAN_REPEAT_NORMAL))
+    {
+	image->common.get_scanline_64 = bits_image_fetch_untransformed_64;
+	image->common.get_scanline_32 = bits_image_fetch_untransformed_32;
+    }
+    else if (bits->common.transform					&&
+	     bits->common.transform->matrix[2][0] == 0			&&
+	     bits->common.transform->matrix[2][1] == 0			&&
+	     bits->common.transform->matrix[2][2] == pixman_fixed_1	&&
+	     bits->common.transform->matrix[0][0] > 0			&&
+	     bits->common.transform->matrix[1][0] == 0			&&
+	     !bits->read_func						&&
+	     (bits->common.filter == PIXMAN_FILTER_BILINEAR ||
+	      bits->common.filter == PIXMAN_FILTER_GOOD	    ||
+	      bits->common.filter == PIXMAN_FILTER_BEST)		&&
+	     bits->common.repeat == PIXMAN_REPEAT_NONE			&&
+	     (bits->format == PIXMAN_a8r8g8b8	||
+	      bits->format == PIXMAN_x8r8g8b8))
+    {
+	image->common.get_scanline_64 =
+	    _pixman_image_get_scanline_generic_64;
+	image->common.get_scanline_32 =
+	    bits_image_fetch_bilinear_no_repeat_8888;
+    }
+    else
+    {
+	image->common.get_scanline_64 =
+	    _pixman_image_get_scanline_generic_64;
+	image->common.get_scanline_32 =
+	    bits_image_fetch_transformed;
+    }
+
+    bits->store_scanline_64 = bits_image_store_scanline_64;
+    bits->store_scanline_32 = bits_image_store_scanline_32;
+}
+
+static uint32_t *
+create_bits (pixman_format_code_t format,
+             int                  width,
+             int                  height,
+             int *                rowstride_bytes)
+{
+    int stride;
+    int buf_size;
+    int bpp;
+
+    /* what follows is a long-winded way, avoiding any possibility of integer
+     * overflows, of saying:
+     * stride = ((width * bpp + 0x1f) >> 5) * sizeof (uint32_t);
+     */
+
+    bpp = PIXMAN_FORMAT_BPP (format);
+    if (pixman_multiply_overflows_int (width, bpp))
+	return NULL;
+
+    stride = width * bpp;
+    if (pixman_addition_overflows_int (stride, 0x1f))
+	return NULL;
+
+    stride += 0x1f;
+    stride >>= 5;
+
+    stride *= sizeof (uint32_t);
+
+    if (pixman_multiply_overflows_int (height, stride))
+	return NULL;
+
+    buf_size = height * stride;
+
+    if (rowstride_bytes)
+	*rowstride_bytes = stride;
+
+    return calloc (buf_size, 1);
+}
+
+PIXMAN_EXPORT pixman_image_t *
+pixman_image_create_bits (pixman_format_code_t format,
+                          int                  width,
+                          int                  height,
+                          uint32_t *           bits,
+                          int                  rowstride_bytes)
+{
+    pixman_image_t *image;
+    uint32_t *free_me = NULL;
+
+    /* must be a whole number of uint32_t's
+     */
+    return_val_if_fail (
+	bits == NULL || (rowstride_bytes % sizeof (uint32_t)) == 0, NULL);
+
+    return_val_if_fail (PIXMAN_FORMAT_BPP (format) >= PIXMAN_FORMAT_DEPTH (format), NULL);
+
+    if (!bits && width && height)
+    {
+	free_me = bits = create_bits (format, width, height, &rowstride_bytes);
+	if (!bits)
+	    return NULL;
+    }
+
+    image = _pixman_image_allocate ();
+
+    if (!image)
+    {
+	if (free_me)
+	    free (free_me);
+
+	return NULL;
+    }
+
+    image->type = BITS;
+    image->bits.format = format;
+    image->bits.width = width;
+    image->bits.height = height;
+    image->bits.bits = bits;
+    image->bits.free_me = free_me;
+    image->bits.read_func = NULL;
+    image->bits.write_func = NULL;
+
+    /* The rowstride is stored in number of uint32_t */
+    image->bits.rowstride = rowstride_bytes / (int) sizeof (uint32_t);
+
+    image->bits.indexed = NULL;
+
+    image->common.property_changed = bits_image_property_changed;
+
+    _pixman_image_reset_clip_region (image);
+
+    return image;
+}
diff --git a/pixman/pixman/pixman-combine.c.template b/pixman/pixman/pixman-combine.c.template
new file mode 100644
index 000000000..c129980a8
--- /dev/null
+++ b/pixman/pixman/pixman-combine.c.template
@@ -0,0 +1,2436 @@
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <math.h>
+#include <string.h>
+
+#include "pixman-private.h"
+
+#include "pixman-combine.h"
+
+/*** per channel helper functions ***/
+
+static void
+combine_mask_ca (comp4_t *src, comp4_t *mask)
+{
+    comp4_t a = *mask;
+
+    comp4_t x;
+    comp2_t xa;
+
+    if (!a)
+    {
+	*(src) = 0;
+	return;
+    }
+
+    x = *(src);
+    if (a == ~0)
+    {
+	x = x >> A_SHIFT;
+	x |= x << G_SHIFT;
+	x |= x << R_SHIFT;
+	*(mask) = x;
+	return;
+    }
+
+    xa = x >> A_SHIFT;
+    UNcx4_MUL_UNcx4 (x, a);
+    *(src) = x;
+    
+    UNcx4_MUL_UNc (a, xa);
+    *(mask) = a;
+}
+
+static void
+combine_mask_value_ca (comp4_t *src, const comp4_t *mask)
+{
+    comp4_t a = *mask;
+    comp4_t x;
+
+    if (!a)
+    {
+	*(src) = 0;
+	return;
+    }
+
+    if (a == ~0)
+	return;
+
+    x = *(src);
+    UNcx4_MUL_UNcx4 (x, a);
+    *(src) = x;
+}
+
+static void
+combine_mask_alpha_ca (const comp4_t *src, comp4_t *mask)
+{
+    comp4_t a = *(mask);
+    comp4_t x;
+
+    if (!a)
+	return;
+
+    x = *(src) >> A_SHIFT;
+    if (x == MASK)
+	return;
+
+    if (a == ~0)
+    {
+	x |= x << G_SHIFT;
+	x |= x << R_SHIFT;
+	*(mask) = x;
+	return;
+    }
+
+    UNcx4_MUL_UNc (a, x);
+    *(mask) = a;
+}
+
+/*
+ * There are two ways of handling alpha -- either as a single unified value or
+ * a separate value for each component, hence each macro must have two
+ * versions.  The unified alpha version has a 'U' at the end of the name,
+ * the component version has a 'C'.  Similarly, functions which deal with
+ * this difference will have two versions using the same convention.
+ */
+
+/*
+ * All of the composing functions
+ */
+
+static force_inline comp4_t
+combine_mask (const comp4_t *src, const comp4_t *mask, int i)
+{
+    comp4_t s, m;
+
+    if (mask)
+    {
+	m = *(mask + i) >> A_SHIFT;
+
+	if (!m)
+	    return 0;
+    }
+
+    s = *(src + i);
+
+    if (mask)
+	UNcx4_MUL_UNc (s, m);
+
+    return s;
+}
+
+static void
+combine_clear (pixman_implementation_t *imp,
+               pixman_op_t              op,
+               comp4_t *                dest,
+               const comp4_t *          src,
+               const comp4_t *          mask,
+               int                      width)
+{
+    memset (dest, 0, width * sizeof(comp4_t));
+}
+
+static void
+combine_src_u (pixman_implementation_t *imp,
+               pixman_op_t              op,
+               comp4_t *                dest,
+               const comp4_t *          src,
+               const comp4_t *          mask,
+               int                      width)
+{
+    int i;
+
+    if (!mask)
+	memcpy (dest, src, width * sizeof (comp4_t));
+    else
+    {
+	for (i = 0; i < width; ++i)
+	{
+	    comp4_t s = combine_mask (src, mask, i);
+
+	    *(dest + i) = s;
+	}
+    }
+}
+
+/* if the Src is opaque, call combine_src_u */
+static void
+combine_over_u (pixman_implementation_t *imp,
+                pixman_op_t              op,
+                comp4_t *                dest,
+                const comp4_t *          src,
+                const comp4_t *          mask,
+                int                      width)
+{
+    int i;
+
+    for (i = 0; i < width; ++i)
+    {
+	comp4_t s = combine_mask (src, mask, i);
+	comp4_t d = *(dest + i);
+	comp4_t ia = ALPHA_c (~s);
+
+	UNcx4_MUL_UNc_ADD_UNcx4 (d, ia, s);
+	*(dest + i) = d;
+    }
+}
+
+/* if the Dst is opaque, this is a noop */
+static void
+combine_over_reverse_u (pixman_implementation_t *imp,
+                        pixman_op_t              op,
+                        comp4_t *                dest,
+                        const comp4_t *          src,
+                        const comp4_t *          mask,
+                        int                      width)
+{
+    int i;
+
+    for (i = 0; i < width; ++i)
+    {
+	comp4_t s = combine_mask (src, mask, i);
+	comp4_t d = *(dest + i);
+	comp4_t ia = ALPHA_c (~*(dest + i));
+	UNcx4_MUL_UNc_ADD_UNcx4 (s, ia, d);
+	*(dest + i) = s;
+    }
+}
+
+/* if the Dst is opaque, call combine_src_u */
+static void
+combine_in_u (pixman_implementation_t *imp,
+              pixman_op_t              op,
+              comp4_t *                dest,
+              const comp4_t *          src,
+              const comp4_t *          mask,
+              int                      width)
+{
+    int i;
+
+    for (i = 0; i < width; ++i)
+    {
+	comp4_t s = combine_mask (src, mask, i);
+	comp4_t a = ALPHA_c (*(dest + i));
+	UNcx4_MUL_UNc (s, a);
+	*(dest + i) = s;
+    }
+}
+
+/* if the Src is opaque, this is a noop */
+static void
+combine_in_reverse_u (pixman_implementation_t *imp,
+                      pixman_op_t              op,
+                      comp4_t *                dest,
+                      const comp4_t *          src,
+                      const comp4_t *          mask,
+                      int                      width)
+{
+    int i;
+
+    for (i = 0; i < width; ++i)
+    {
+	comp4_t s = combine_mask (src, mask, i);
+	comp4_t d = *(dest + i);
+	comp4_t a = ALPHA_c (s);
+	UNcx4_MUL_UNc (d, a);
+	*(dest + i) = d;
+    }
+}
+
+/* if the Dst is opaque, call combine_clear */
+static void
+combine_out_u (pixman_implementation_t *imp,
+               pixman_op_t              op,
+               comp4_t *                dest,
+               const comp4_t *          src,
+               const comp4_t *          mask,
+               int                      width)
+{
+    int i;
+
+    for (i = 0; i < width; ++i)
+    {
+	comp4_t s = combine_mask (src, mask, i);
+	comp4_t a = ALPHA_c (~*(dest + i));
+	UNcx4_MUL_UNc (s, a);
+	*(dest + i) = s;
+    }
+}
+
+/* if the Src is opaque, call combine_clear */
+static void
+combine_out_reverse_u (pixman_implementation_t *imp,
+                       pixman_op_t              op,
+                       comp4_t *                dest,
+                       const comp4_t *          src,
+                       const comp4_t *          mask,
+                       int                      width)
+{
+    int i;
+
+    for (i = 0; i < width; ++i)
+    {
+	comp4_t s = combine_mask (src, mask, i);
+	comp4_t d = *(dest + i);
+	comp4_t a = ALPHA_c (~s);
+	UNcx4_MUL_UNc (d, a);
+	*(dest + i) = d;
+    }
+}
+
+/* if the Src is opaque, call combine_in_u */
+/* if the Dst is opaque, call combine_over_u */
+/* if both the Src and Dst are opaque, call combine_src_u */
+static void
+combine_atop_u (pixman_implementation_t *imp,
+                pixman_op_t              op,
+                comp4_t *                dest,
+                const comp4_t *          src,
+                const comp4_t *          mask,
+                int                      width)
+{
+    int i;
+
+    for (i = 0; i < width; ++i)
+    {
+	comp4_t s = combine_mask (src, mask, i);
+	comp4_t d = *(dest + i);
+	comp4_t dest_a = ALPHA_c (d);
+	comp4_t src_ia = ALPHA_c (~s);
+
+	UNcx4_MUL_UNc_ADD_UNcx4_MUL_UNc (s, dest_a, d, src_ia);
+	*(dest + i) = s;
+    }
+}
+
+/* if the Src is opaque, call combine_over_reverse_u */
+/* if the Dst is opaque, call combine_in_reverse_u */
+/* if both the Src and Dst are opaque, call combine_dst_u */
+static void
+combine_atop_reverse_u (pixman_implementation_t *imp,
+                        pixman_op_t              op,
+                        comp4_t *                dest,
+                        const comp4_t *          src,
+                        const comp4_t *          mask,
+                        int                      width)
+{
+    int i;
+
+    for (i = 0; i < width; ++i)
+    {
+	comp4_t s = combine_mask (src, mask, i);
+	comp4_t d = *(dest + i);
+	comp4_t src_a = ALPHA_c (s);
+	comp4_t dest_ia = ALPHA_c (~d);
+
+	UNcx4_MUL_UNc_ADD_UNcx4_MUL_UNc (s, dest_ia, d, src_a);
+	*(dest + i) = s;
+    }
+}
+
+/* if the Src is opaque, call combine_over_u */
+/* if the Dst is opaque, call combine_over_reverse_u */
+/* if both the Src and Dst are opaque, call combine_clear */
+static void
+combine_xor_u (pixman_implementation_t *imp,
+               pixman_op_t              op,
+               comp4_t *                dest,
+               const comp4_t *          src,
+               const comp4_t *          mask,
+               int                      width)
+{
+    int i;
+
+    for (i = 0; i < width; ++i)
+    {
+	comp4_t s = combine_mask (src, mask, i);
+	comp4_t d = *(dest + i);
+	comp4_t src_ia = ALPHA_c (~s);
+	comp4_t dest_ia = ALPHA_c (~d);
+
+	UNcx4_MUL_UNc_ADD_UNcx4_MUL_UNc (s, dest_ia, d, src_ia);
+	*(dest + i) = s;
+    }
+}
+
+static void
+combine_add_u (pixman_implementation_t *imp,
+               pixman_op_t              op,
+               comp4_t *                dest,
+               const comp4_t *          src,
+               const comp4_t *          mask,
+               int                      width)
+{
+    int i;
+
+    for (i = 0; i < width; ++i)
+    {
+	comp4_t s = combine_mask (src, mask, i);
+	comp4_t d = *(dest + i);
+	UNcx4_ADD_UNcx4 (d, s);
+	*(dest + i) = d;
+    }
+}
+
+/* if the Src is opaque, call combine_add_u */
+/* if the Dst is opaque, call combine_add_u */
+/* if both the Src and Dst are opaque, call combine_add_u */
+static void
+combine_saturate_u (pixman_implementation_t *imp,
+                    pixman_op_t              op,
+                    comp4_t *                dest,
+                    const comp4_t *          src,
+                    const comp4_t *          mask,
+                    int                      width)
+{
+    int i;
+
+    for (i = 0; i < width; ++i)
+    {
+	comp4_t s = combine_mask (src, mask, i);
+	comp4_t d = *(dest + i);
+	comp2_t sa, da;
+
+	sa = s >> A_SHIFT;
+	da = ~d >> A_SHIFT;
+	if (sa > da)
+	{
+	    sa = DIV_UNc (da, sa);
+	    UNcx4_MUL_UNc (s, sa);
+	}
+	;
+	UNcx4_ADD_UNcx4 (d, s);
+	*(dest + i) = d;
+    }
+}
+
+/*
+ * PDF blend modes:
+ * The following blend modes have been taken from the PDF ISO 32000
+ * specification, which at this point in time is available from
+ * http://www.adobe.com/devnet/acrobat/pdfs/PDF32000_2008.pdf
+ * The relevant chapters are 11.3.5 and 11.3.6.
+ * The formula for computing the final pixel color given in 11.3.6 is:
+ * αr × Cr = (1 – αs) × αb × Cb + (1 – αb) × αs × Cs + αb × αs × B(Cb, Cs)
+ * with B() being the blend function.
+ * Note that OVER is a special case of this operation, using B(Cb, Cs) = Cs
+ *
+ * These blend modes should match the SVG filter draft specification, as
+ * it has been designed to mirror ISO 32000. Note that at the current point
+ * no released draft exists that shows this, as the formulas have not been
+ * updated yet after the release of ISO 32000.
+ *
+ * The default implementation here uses the PDF_SEPARABLE_BLEND_MODE and
+ * PDF_NON_SEPARABLE_BLEND_MODE macros, which take the blend function as an
+ * argument. Note that this implementation operates on premultiplied colors,
+ * while the PDF specification does not. Therefore the code uses the formula
+ * ar.Cra = (1 – as) . Dca + (1 – ad) . Sca + B(Dca, ad, Sca, as)
+ */
+
+/*
+ * Multiply
+ * B(Dca, ad, Sca, as) = Dca.Sca
+ */
+
+static void
+combine_multiply_u (pixman_implementation_t *imp,
+                    pixman_op_t              op,
+                    comp4_t *                dest,
+                    const comp4_t *          src,
+                    const comp4_t *          mask,
+                    int                      width)
+{
+    int i;
+
+    for (i = 0; i < width; ++i)
+    {
+	comp4_t s = combine_mask (src, mask, i);
+	comp4_t d = *(dest + i);
+	comp4_t ss = s;
+	comp4_t src_ia = ALPHA_c (~s);
+	comp4_t dest_ia = ALPHA_c (~d);
+
+	UNcx4_MUL_UNc_ADD_UNcx4_MUL_UNc (ss, dest_ia, d, src_ia);
+	UNcx4_MUL_UNcx4 (d, s);
+	UNcx4_ADD_UNcx4 (d, ss);
+
+	*(dest + i) = d;
+    }
+}
+
+static void
+combine_multiply_ca (pixman_implementation_t *imp,
+                     pixman_op_t              op,
+                     comp4_t *                dest,
+                     const comp4_t *          src,
+                     const comp4_t *          mask,
+                     int                      width)
+{
+    int i;
+
+    for (i = 0; i < width; ++i)
+    {
+	comp4_t m = *(mask + i);
+	comp4_t s = *(src + i);
+	comp4_t d = *(dest + i);
+	comp4_t r = d;
+	comp4_t dest_ia = ALPHA_c (~d);
+
+	combine_mask_value_ca (&s, &m);
+
+	UNcx4_MUL_UNcx4_ADD_UNcx4_MUL_UNc (r, ~m, s, dest_ia);
+	UNcx4_MUL_UNcx4 (d, s);
+	UNcx4_ADD_UNcx4 (r, d);
+
+	*(dest + i) = r;
+    }
+}
+
+#define PDF_SEPARABLE_BLEND_MODE(name)					\
+    static void								\
+    combine_ ## name ## _u (pixman_implementation_t *imp,		\
+			    pixman_op_t              op,		\
+                            comp4_t *                dest,		\
+			    const comp4_t *          src,		\
+			    const comp4_t *          mask,		\
+			    int                      width)		\
+    {									\
+	int i;								\
+	for (i = 0; i < width; ++i) {					\
+	    comp4_t s = combine_mask (src, mask, i);			\
+	    comp4_t d = *(dest + i);					\
+	    comp1_t sa = ALPHA_c (s);					\
+	    comp1_t isa = ~sa;						\
+	    comp1_t da = ALPHA_c (d);					\
+	    comp1_t ida = ~da;						\
+	    comp4_t result;						\
+									\
+	    result = d;							\
+	    UNcx4_MUL_UNc_ADD_UNcx4_MUL_UNc (result, isa, s, ida);	\
+	    								\
+	    *(dest + i) = result +					\
+		(DIV_ONE_UNc (sa * da) << A_SHIFT) +			\
+		(blend_ ## name (RED_c (d), da, RED_c (s), sa) << R_SHIFT) + \
+		(blend_ ## name (GREEN_c (d), da, GREEN_c (s), sa) << G_SHIFT) + \
+		(blend_ ## name (BLUE_c (d), da, BLUE_c (s), sa));	\
+	}								\
+    }									\
+    									\
+    static void								\
+    combine_ ## name ## _ca (pixman_implementation_t *imp,		\
+			     pixman_op_t              op,		\
+                             comp4_t *                dest,		\
+			     const comp4_t *          src,		\
+			     const comp4_t *          mask,		\
+			     int                     width)		\
+    {									\
+	int i;								\
+	for (i = 0; i < width; ++i) {					\
+	    comp4_t m = *(mask + i);					\
+	    comp4_t s = *(src + i);					\
+	    comp4_t d = *(dest + i);					\
+	    comp1_t da = ALPHA_c (d);					\
+	    comp1_t ida = ~da;						\
+	    comp4_t result;						\
+            								\
+	    combine_mask_value_ca (&s, &m);				\
+            								\
+	    result = d;							\
+	    UNcx4_MUL_UNcx4_ADD_UNcx4_MUL_UNc (result, ~m, s, ida);     \
+            								\
+	    result +=							\
+	        (DIV_ONE_UNc (ALPHA_c (m) * da) << A_SHIFT) +		\
+	        (blend_ ## name (RED_c (d), da, RED_c (s), RED_c (m)) << R_SHIFT) + \
+	        (blend_ ## name (GREEN_c (d), da, GREEN_c (s), GREEN_c (m)) << G_SHIFT) + \
+	        (blend_ ## name (BLUE_c (d), da, BLUE_c (s), BLUE_c (m))); \
+	    								\
+	    *(dest + i) = result;					\
+	}								\
+    }
+
+/*
+ * Screen
+ * B(Dca, ad, Sca, as) = Dca.sa + Sca.da - Dca.Sca
+ */
+static inline comp4_t
+blend_screen (comp4_t dca, comp4_t da, comp4_t sca, comp4_t sa)
+{
+    return DIV_ONE_UNc (sca * da + dca * sa - sca * dca);
+}
+
+PDF_SEPARABLE_BLEND_MODE (screen)
+
+/*
+ * Overlay
+ * B(Dca, Da, Sca, Sa) =
+ *   if 2.Dca < Da
+ *     2.Sca.Dca
+ *   otherwise
+ *     Sa.Da - 2.(Da - Dca).(Sa - Sca)
+ */
+static inline comp4_t
+blend_overlay (comp4_t dca, comp4_t da, comp4_t sca, comp4_t sa)
+{
+    comp4_t rca;
+
+    if (2 * dca < da)
+	rca = 2 * sca * dca;
+    else
+	rca = sa * da - 2 * (da - dca) * (sa - sca);
+    return DIV_ONE_UNc (rca);
+}
+
+PDF_SEPARABLE_BLEND_MODE (overlay)
+
+/*
+ * Darken
+ * B(Dca, Da, Sca, Sa) = min (Sca.Da, Dca.Sa)
+ */
+static inline comp4_t
+blend_darken (comp4_t dca, comp4_t da, comp4_t sca, comp4_t sa)
+{
+    comp4_t s, d;
+
+    s = sca * da;
+    d = dca * sa;
+    return DIV_ONE_UNc (s > d ? d : s);
+}
+
+PDF_SEPARABLE_BLEND_MODE (darken)
+
+/*
+ * Lighten
+ * B(Dca, Da, Sca, Sa) = max (Sca.Da, Dca.Sa)
+ */
+static inline comp4_t
+blend_lighten (comp4_t dca, comp4_t da, comp4_t sca, comp4_t sa)
+{
+    comp4_t s, d;
+
+    s = sca * da;
+    d = dca * sa;
+    return DIV_ONE_UNc (s > d ? s : d);
+}
+
+PDF_SEPARABLE_BLEND_MODE (lighten)
+
+/*
+ * Color dodge
+ * B(Dca, Da, Sca, Sa) =
+ *   if Dca == 0
+ *     0
+ *   if Sca == Sa
+ *     Sa.Da
+ *   otherwise
+ *     Sa.Da. min (1, Dca / Da / (1 - Sca/Sa))
+ */
+static inline comp4_t
+blend_color_dodge (comp4_t dca, comp4_t da, comp4_t sca, comp4_t sa)
+{
+    if (sca >= sa)
+    {
+	return dca == 0 ? 0 : DIV_ONE_UNc (sa * da);
+    }
+    else
+    {
+	comp4_t rca = dca * sa / (sa - sca);
+	return DIV_ONE_UNc (sa * MIN (rca, da));
+    }
+}
+
+PDF_SEPARABLE_BLEND_MODE (color_dodge)
+
+/*
+ * Color burn
+ * B(Dca, Da, Sca, Sa) =
+ *   if Dca == Da
+ *     Sa.Da
+ *   if Sca == 0
+ *     0
+ *   otherwise
+ *     Sa.Da.(1 - min (1, (1 - Dca/Da).Sa / Sca))
+ */
+static inline comp4_t
+blend_color_burn (comp4_t dca, comp4_t da, comp4_t sca, comp4_t sa)
+{
+    if (sca == 0)
+    {
+	return dca < da ? 0 : DIV_ONE_UNc (sa * da);
+    }
+    else
+    {
+	comp4_t rca = (da - dca) * sa / sca;
+	return DIV_ONE_UNc (sa * (MAX (rca, da) - rca));
+    }
+}
+
+PDF_SEPARABLE_BLEND_MODE (color_burn)
+
+/*
+ * Hard light
+ * B(Dca, Da, Sca, Sa) =
+ *   if 2.Sca < Sa
+ *     2.Sca.Dca
+ *   otherwise
+ *     Sa.Da - 2.(Da - Dca).(Sa - Sca)
+ */
+static inline comp4_t
+blend_hard_light (comp4_t dca, comp4_t da, comp4_t sca, comp4_t sa)
+{
+    if (2 * sca < sa)
+	return DIV_ONE_UNc (2 * sca * dca);
+    else
+	return DIV_ONE_UNc (sa * da - 2 * (da - dca) * (sa - sca));
+}
+
+PDF_SEPARABLE_BLEND_MODE (hard_light)
+
+/*
+ * Soft light
+ * B(Dca, Da, Sca, Sa) =
+ *   if (2.Sca <= Sa)
+ *     Dca.(Sa - (1 - Dca/Da).(2.Sca - Sa))
+ *   otherwise if Dca.4 <= Da
+ *     Dca.(Sa + (2.Sca - Sa).((16.Dca/Da - 12).Dca/Da + 3)
+ *   otherwise
+ *     (Dca.Sa + (SQRT (Dca/Da).Da - Dca).(2.Sca - Sa))
+ */
+static inline comp4_t
+blend_soft_light (comp4_t dca_org,
+		  comp4_t da_org,
+		  comp4_t sca_org,
+		  comp4_t sa_org)
+{
+    double dca = dca_org * (1.0 / MASK);
+    double da = da_org * (1.0 / MASK);
+    double sca = sca_org * (1.0 / MASK);
+    double sa = sa_org * (1.0 / MASK);
+    double rca;
+
+    if (2 * sca < sa)
+    {
+	if (da == 0)
+	    rca = dca * sa;
+	else
+	    rca = dca * sa - dca * (da - dca) * (sa - 2 * sca) / da;
+    }
+    else if (da == 0)
+    {
+	rca = 0;
+    }
+    else if (4 * dca <= da)
+    {
+	rca = dca * sa +
+	    (2 * sca - sa) * dca * ((16 * dca / da - 12) * dca / da + 3);
+    }
+    else
+    {
+	rca = dca * sa + (sqrt (dca * da) - dca) * (2 * sca - sa);
+    }
+    return rca * MASK + 0.5;
+}
+
+PDF_SEPARABLE_BLEND_MODE (soft_light)
+
+/*
+ * Difference
+ * B(Dca, Da, Sca, Sa) = abs (Dca.Sa - Sca.Da)
+ */
+static inline comp4_t
+blend_difference (comp4_t dca, comp4_t da, comp4_t sca, comp4_t sa)
+{
+    comp4_t dcasa = dca * sa;
+    comp4_t scada = sca * da;
+
+    if (scada < dcasa)
+	return DIV_ONE_UNc (dcasa - scada);
+    else
+	return DIV_ONE_UNc (scada - dcasa);
+}
+
+PDF_SEPARABLE_BLEND_MODE (difference)
+
+/*
+ * Exclusion
+ * B(Dca, Da, Sca, Sa) = (Sca.Da + Dca.Sa - 2.Sca.Dca)
+ */
+
+/* This can be made faster by writing it directly and not using
+ * PDF_SEPARABLE_BLEND_MODE, but that's a performance optimization */
+
+static inline comp4_t
+blend_exclusion (comp4_t dca, comp4_t da, comp4_t sca, comp4_t sa)
+{
+    return DIV_ONE_UNc (sca * da + dca * sa - 2 * dca * sca);
+}
+
+PDF_SEPARABLE_BLEND_MODE (exclusion)
+
+#undef PDF_SEPARABLE_BLEND_MODE
+
+/*
+ * PDF nonseperable blend modes are implemented using the following functions
+ * to operate in Hsl space, with Cmax, Cmid, Cmin referring to the max, mid
+ * and min value of the red, green and blue components.
+ *
+ * LUM (C) = 0.3 × Cred + 0.59 × Cgreen + 0.11 × Cblue
+ *
+ * clip_color (C):
+ *   l = LUM (C)
+ *   min = Cmin
+ *   max = Cmax
+ *   if n < 0.0
+ *     C = l + ( ( ( C – l ) × l ) ⁄ ( l – min ) )
+ *   if x > 1.0
+ *     C = l + ( ( ( C – l ) × ( 1 – l ) ) ⁄ ( max – l ) )
+ *   return C
+ *
+ * set_lum (C, l):
+ *   d = l – LUM (C)
+ *   C += d
+ *   return clip_color (C)
+ *
+ * SAT (C) = CH_MAX (C) - CH_MIN (C)
+ *
+ * set_sat (C, s):
+ *  if Cmax > Cmin
+ *    Cmid = ( ( ( Cmid – Cmin ) × s ) ⁄ ( Cmax – Cmin ) )
+ *    Cmax = s
+ *  else
+ *    Cmid = Cmax = 0.0
+ *  Cmin = 0.0
+ *  return C
+ */
+
+/* For premultiplied colors, we need to know what happens when C is
+ * multiplied by a real number. LUM and SAT are linear:
+ *
+ *    LUM (r × C) = r × LUM (C)		SAT (r * C) = r * SAT (C)
+ *
+ * If we extend clip_color with an extra argument a and change
+ *
+ *        if x >= 1.0
+ *
+ * into
+ *
+ *        if x >= a
+ *
+ * then clip_color is also linear:
+ *
+ *    r * clip_color (C, a) = clip_color (r_c, ra);
+ *
+ * for positive r.
+ *
+ * Similarly, we can extend set_lum with an extra argument that is just passed
+ * on to clip_color:
+ *
+ *   r * set_lum ( C, l, a)
+ *
+ *   = r × clip_color ( C + l - LUM (C), a)
+ *
+ *   = clip_color ( r * C + r × l - r * LUM (C), r * a)
+ *
+ *   = set_lum ( r * C, r * l, r * a)
+ *
+ * Finally, set_sat:
+ *
+ *    r * set_sat (C, s) = set_sat (x * C, r * s)
+ *
+ * The above holds for all non-zero x, because they x'es in the fraction for
+ * C_mid cancel out. Specifically, it holds for x = r:
+ *
+ *    r * set_sat (C, s) = set_sat (r_c, rs)
+ *
+ */
+
+/* So, for the non-separable PDF blend modes, we have (using s, d for
+ * non-premultiplied colors, and S, D for premultiplied:
+ *
+ *   Color:
+ *
+ *     a_s * a_d * B(s, d)
+ *   = a_s * a_d * set_lum (S/a_s, LUM (D/a_d), 1)
+ *   = set_lum (S * a_d, a_s * LUM (D), a_s * a_d)
+ *
+ *
+ *   Luminosity:
+ *
+ *     a_s * a_d * B(s, d)
+ *   = a_s * a_d * set_lum (D/a_d, LUM(S/a_s), 1)
+ *   = set_lum (a_s * D, a_d * LUM(S), a_s * a_d)
+ *
+ *
+ *   Saturation:
+ *
+ *     a_s * a_d * B(s, d)
+ *   = a_s * a_d * set_lum (set_sat (D/a_d, SAT (S/a_s)), LUM (D/a_d), 1)
+ *   = set_lum (a_s * a_d * set_sat (D/a_d, SAT (S/a_s)),
+ *                                        a_s * LUM (D), a_s * a_d)
+ *   = set_lum (set_sat (a_s * D, a_d * SAT (S), a_s * LUM (D), a_s * a_d))
+ *
+ *   Hue:
+ *
+ *     a_s * a_d * B(s, d)
+ *   = a_s * a_d * set_lum (set_sat (S/a_s, SAT (D/a_d)), LUM (D/a_d), 1)
+ *   = a_s * a_d * set_lum (set_sat (a_d * S, a_s * SAT (D)),
+ *                                        a_s * LUM (D), a_s * a_d)
+ *
+ */
+
+#define CH_MIN(c) (c[0] < c[1] ? (c[0] < c[2] ? c[0] : c[2]) : (c[1] < c[2] ? c[1] : c[2]))
+#define CH_MAX(c) (c[0] > c[1] ? (c[0] > c[2] ? c[0] : c[2]) : (c[1] > c[2] ? c[1] : c[2]))
+#define LUM(c) ((c[0] * 30 + c[1] * 59 + c[2] * 11) / 100)
+#define SAT(c) (CH_MAX (c) - CH_MIN (c))
+
+#define PDF_NON_SEPARABLE_BLEND_MODE(name)				\
+    static void								\
+    combine_ ## name ## _u (pixman_implementation_t *imp,		\
+			    pixman_op_t op,				\
+                            comp4_t *dest,				\
+			    const comp4_t *src,				\
+			    const comp4_t *mask,			\
+			    int width)					\
+    {									\
+	int i;								\
+	for (i = 0; i < width; ++i)					\
+	{								\
+	    comp4_t s = combine_mask (src, mask, i);			\
+	    comp4_t d = *(dest + i);					\
+	    comp1_t sa = ALPHA_c (s);					\
+	    comp1_t isa = ~sa;						\
+	    comp1_t da = ALPHA_c (d);					\
+	    comp1_t ida = ~da;						\
+	    comp4_t result;						\
+	    comp4_t sc[3], dc[3], c[3];					\
+            								\
+	    result = d;							\
+	    UNcx4_MUL_UNc_ADD_UNcx4_MUL_UNc (result, isa, s, ida);	\
+	    dc[0] = RED_c (d);						\
+	    sc[0] = RED_c (s);						\
+	    dc[1] = GREEN_c (d);					\
+	    sc[1] = GREEN_c (s);					\
+	    dc[2] = BLUE_c (d);						\
+	    sc[2] = BLUE_c (s);						\
+	    blend_ ## name (c, dc, da, sc, sa);				\
+            								\
+	    *(dest + i) = result +					\
+		(DIV_ONE_UNc (sa * da) << A_SHIFT) +			\
+		(DIV_ONE_UNc (c[0]) << R_SHIFT) +			\
+		(DIV_ONE_UNc (c[1]) << G_SHIFT) +			\
+		(DIV_ONE_UNc (c[2]));					\
+	}								\
+    }
+
+static void
+set_lum (comp4_t dest[3], comp4_t src[3], comp4_t sa, comp4_t lum)
+{
+    double a, l, min, max;
+    double tmp[3];
+
+    a = sa * (1.0 / MASK);
+
+    l = lum * (1.0 / MASK);
+    tmp[0] = src[0] * (1.0 / MASK);
+    tmp[1] = src[1] * (1.0 / MASK);
+    tmp[2] = src[2] * (1.0 / MASK);
+
+    l = l - LUM (tmp);
+    tmp[0] += l;
+    tmp[1] += l;
+    tmp[2] += l;
+
+    /* clip_color */
+    l = LUM (tmp);
+    min = CH_MIN (tmp);
+    max = CH_MAX (tmp);
+
+    if (min < 0)
+    {
+	tmp[0] = l + (tmp[0] - l) * l / (l - min);
+	tmp[1] = l + (tmp[1] - l) * l / (l - min);
+	tmp[2] = l + (tmp[2] - l) * l / (l - min);
+    }
+    if (max > a)
+    {
+	tmp[0] = l + (tmp[0] - l) * (a - l) / (max - l);
+	tmp[1] = l + (tmp[1] - l) * (a - l) / (max - l);
+	tmp[2] = l + (tmp[2] - l) * (a - l) / (max - l);
+    }
+
+    dest[0] = tmp[0] * MASK + 0.5;
+    dest[1] = tmp[1] * MASK + 0.5;
+    dest[2] = tmp[2] * MASK + 0.5;
+}
+
+static void
+set_sat (comp4_t dest[3], comp4_t src[3], comp4_t sat)
+{
+    int id[3];
+    comp4_t min, max;
+
+    if (src[0] > src[1])
+    {
+	if (src[0] > src[2])
+	{
+	    id[0] = 0;
+	    if (src[1] > src[2])
+	    {
+		id[1] = 1;
+		id[2] = 2;
+	    }
+	    else
+	    {
+		id[1] = 2;
+		id[2] = 1;
+	    }
+	}
+	else
+	{
+	    id[0] = 2;
+	    id[1] = 0;
+	    id[2] = 1;
+	}
+    }
+    else
+    {
+	if (src[0] > src[2])
+	{
+	    id[0] = 1;
+	    id[1] = 0;
+	    id[2] = 2;
+	}
+	else
+	{
+	    id[2] = 0;
+	    if (src[1] > src[2])
+	    {
+		id[0] = 1;
+		id[1] = 2;
+	    }
+	    else
+	    {
+		id[0] = 2;
+		id[1] = 1;
+	    }
+	}
+    }
+
+    max = dest[id[0]];
+    min = dest[id[2]];
+    if (max > min)
+    {
+	dest[id[1]] = (dest[id[1]] - min) * sat / (max - min);
+	dest[id[0]] = sat;
+	dest[id[2]] = 0;
+    }
+    else
+    {
+	dest[0] = dest[1] = dest[2] = 0;
+    }
+}
+
+/*
+ * Hue:
+ * B(Cb, Cs) = set_lum (set_sat (Cs, SAT (Cb)), LUM (Cb))
+ */
+static inline void
+blend_hsl_hue (comp4_t c[3],
+               comp4_t dc[3],
+               comp4_t da,
+               comp4_t sc[3],
+               comp4_t sa)
+{
+    c[0] = sc[0] * da;
+    c[1] = sc[1] * da;
+    c[2] = sc[2] * da;
+    set_sat (c, c, SAT (dc) * sa);
+    set_lum (c, c, sa * da, LUM (dc) * sa);
+}
+
+PDF_NON_SEPARABLE_BLEND_MODE (hsl_hue)
+
+/*
+ * Saturation:
+ * B(Cb, Cs) = set_lum (set_sat (Cb, SAT (Cs)), LUM (Cb))
+ */
+static inline void
+blend_hsl_saturation (comp4_t c[3],
+                      comp4_t dc[3],
+                      comp4_t da,
+                      comp4_t sc[3],
+                      comp4_t sa)
+{
+    c[0] = dc[0] * sa;
+    c[1] = dc[1] * sa;
+    c[2] = dc[2] * sa;
+    set_sat (c, c, SAT (sc) * da);
+    set_lum (c, c, sa * da, LUM (dc) * sa);
+}
+
+PDF_NON_SEPARABLE_BLEND_MODE (hsl_saturation)
+
+/*
+ * Color:
+ * B(Cb, Cs) = set_lum (Cs, LUM (Cb))
+ */
+static inline void
+blend_hsl_color (comp4_t c[3],
+                 comp4_t dc[3],
+                 comp4_t da,
+                 comp4_t sc[3],
+                 comp4_t sa)
+{
+    c[0] = sc[0] * da;
+    c[1] = sc[1] * da;
+    c[2] = sc[2] * da;
+    set_lum (c, c, sa * da, LUM (dc) * sa);
+}
+
+PDF_NON_SEPARABLE_BLEND_MODE (hsl_color)
+
+/*
+ * Luminosity:
+ * B(Cb, Cs) = set_lum (Cb, LUM (Cs))
+ */
+static inline void
+blend_hsl_luminosity (comp4_t c[3],
+                      comp4_t dc[3],
+                      comp4_t da,
+                      comp4_t sc[3],
+                      comp4_t sa)
+{
+    c[0] = dc[0] * sa;
+    c[1] = dc[1] * sa;
+    c[2] = dc[2] * sa;
+    set_lum (c, c, sa * da, LUM (sc) * da);
+}
+
+PDF_NON_SEPARABLE_BLEND_MODE (hsl_luminosity)
+
+#undef SAT
+#undef LUM
+#undef CH_MAX
+#undef CH_MIN
+#undef PDF_NON_SEPARABLE_BLEND_MODE
+
+/* Overlay
+ *
+ * All of the disjoint composing functions
+ *
+ * The four entries in the first column indicate what source contributions
+ * come from each of the four areas of the picture -- areas covered by neither
+ * A nor B, areas covered only by A, areas covered only by B and finally
+ * areas covered by both A and B.
+ * 
+ * Disjoint			Conjoint
+ * Fa		Fb		Fa		Fb
+ * (0,0,0,0)	0		0		0		0
+ * (0,A,0,A)	1		0		1		0
+ * (0,0,B,B)	0		1		0		1
+ * (0,A,B,A)	1		min((1-a)/b,1)	1		max(1-a/b,0)
+ * (0,A,B,B)	min((1-b)/a,1)	1		max(1-b/a,0)	1
+ * (0,0,0,A)	max(1-(1-b)/a,0) 0		min(1,b/a)	0
+ * (0,0,0,B)	0		max(1-(1-a)/b,0) 0		min(a/b,1)
+ * (0,A,0,0)	min(1,(1-b)/a)	0		max(1-b/a,0)	0
+ * (0,0,B,0)	0		min(1,(1-a)/b)	0		max(1-a/b,0)
+ * (0,0,B,A)	max(1-(1-b)/a,0) min(1,(1-a)/b)	 min(1,b/a)	max(1-a/b,0)
+ * (0,A,0,B)	min(1,(1-b)/a)	max(1-(1-a)/b,0) max(1-b/a,0)	min(1,a/b)
+ * (0,A,B,0)	min(1,(1-b)/a)	min(1,(1-a)/b)	max(1-b/a,0)	max(1-a/b,0)
+ */
+
+#define COMBINE_A_OUT 1
+#define COMBINE_A_IN  2
+#define COMBINE_B_OUT 4
+#define COMBINE_B_IN  8
+
+#define COMBINE_CLEAR   0
+#define COMBINE_A       (COMBINE_A_OUT | COMBINE_A_IN)
+#define COMBINE_B       (COMBINE_B_OUT | COMBINE_B_IN)
+#define COMBINE_A_OVER  (COMBINE_A_OUT | COMBINE_B_OUT | COMBINE_A_IN)
+#define COMBINE_B_OVER  (COMBINE_A_OUT | COMBINE_B_OUT | COMBINE_B_IN)
+#define COMBINE_A_ATOP  (COMBINE_B_OUT | COMBINE_A_IN)
+#define COMBINE_B_ATOP  (COMBINE_A_OUT | COMBINE_B_IN)
+#define COMBINE_XOR     (COMBINE_A_OUT | COMBINE_B_OUT)
+
+/* portion covered by a but not b */
+static comp1_t
+combine_disjoint_out_part (comp1_t a, comp1_t b)
+{
+    /* min (1, (1-b) / a) */
+
+    b = ~b;                 /* 1 - b */
+    if (b >= a)             /* 1 - b >= a -> (1-b)/a >= 1 */
+	return MASK;        /* 1 */
+    return DIV_UNc (b, a);     /* (1-b) / a */
+}
+
+/* portion covered by both a and b */
+static comp1_t
+combine_disjoint_in_part (comp1_t a, comp1_t b)
+{
+    /* max (1-(1-b)/a,0) */
+    /*  = - min ((1-b)/a - 1, 0) */
+    /*  = 1 - min (1, (1-b)/a) */
+
+    b = ~b;                 /* 1 - b */
+    if (b >= a)             /* 1 - b >= a -> (1-b)/a >= 1 */
+	return 0;           /* 1 - 1 */
+    return ~DIV_UNc(b, a);    /* 1 - (1-b) / a */
+}
+
+/* portion covered by a but not b */
+static comp1_t
+combine_conjoint_out_part (comp1_t a, comp1_t b)
+{
+    /* max (1-b/a,0) */
+    /* = 1-min(b/a,1) */
+
+    /* min (1, (1-b) / a) */
+
+    if (b >= a)             /* b >= a -> b/a >= 1 */
+	return 0x00;        /* 0 */
+    return ~DIV_UNc(b, a);    /* 1 - b/a */
+}
+
+/* portion covered by both a and b */
+static comp1_t
+combine_conjoint_in_part (comp1_t a, comp1_t b)
+{
+    /* min (1,b/a) */
+
+    if (b >= a)             /* b >= a -> b/a >= 1 */
+	return MASK;        /* 1 */
+    return DIV_UNc (b, a);     /* b/a */
+}
+
+#define GET_COMP(v, i)   ((comp2_t) (comp1_t) ((v) >> i))
+
+#define ADD(x, y, i, t)							\
+    ((t) = GET_COMP (x, i) + GET_COMP (y, i),				\
+     (comp4_t) ((comp1_t) ((t) | (0 - ((t) >> G_SHIFT)))) << (i))
+
+#define GENERIC(x, y, i, ax, ay, t, u, v)				\
+    ((t) = (MUL_UNc (GET_COMP (y, i), ay, (u)) +			\
+            MUL_UNc (GET_COMP (x, i), ax, (v))),			\
+     (comp4_t) ((comp1_t) ((t) |					\
+                           (0 - ((t) >> G_SHIFT)))) << (i))
+
+static void
+combine_disjoint_general_u (comp4_t *      dest,
+                            const comp4_t *src,
+                            const comp4_t *mask,
+                            int            width,
+                            comp1_t        combine)
+{
+    int i;
+
+    for (i = 0; i < width; ++i)
+    {
+	comp4_t s = combine_mask (src, mask, i);
+	comp4_t d = *(dest + i);
+	comp4_t m, n, o, p;
+	comp2_t Fa, Fb, t, u, v;
+	comp1_t sa = s >> A_SHIFT;
+	comp1_t da = d >> A_SHIFT;
+
+	switch (combine & COMBINE_A)
+	{
+	default:
+	    Fa = 0;
+	    break;
+
+	case COMBINE_A_OUT:
+	    Fa = combine_disjoint_out_part (sa, da);
+	    break;
+
+	case COMBINE_A_IN:
+	    Fa = combine_disjoint_in_part (sa, da);
+	    break;
+
+	case COMBINE_A:
+	    Fa = MASK;
+	    break;
+	}
+
+	switch (combine & COMBINE_B)
+	{
+	default:
+	    Fb = 0;
+	    break;
+
+	case COMBINE_B_OUT:
+	    Fb = combine_disjoint_out_part (da, sa);
+	    break;
+
+	case COMBINE_B_IN:
+	    Fb = combine_disjoint_in_part (da, sa);
+	    break;
+
+	case COMBINE_B:
+	    Fb = MASK;
+	    break;
+	}
+	m = GENERIC (s, d, 0, Fa, Fb, t, u, v);
+	n = GENERIC (s, d, G_SHIFT, Fa, Fb, t, u, v);
+	o = GENERIC (s, d, R_SHIFT, Fa, Fb, t, u, v);
+	p = GENERIC (s, d, A_SHIFT, Fa, Fb, t, u, v);
+	s = m | n | o | p;
+	*(dest + i) = s;
+    }
+}
+
+static void
+combine_disjoint_over_u (pixman_implementation_t *imp,
+                         pixman_op_t              op,
+                         comp4_t *                dest,
+                         const comp4_t *          src,
+                         const comp4_t *          mask,
+                         int                      width)
+{
+    int i;
+
+    for (i = 0; i < width; ++i)
+    {
+	comp4_t s = combine_mask (src, mask, i);
+	comp2_t a = s >> A_SHIFT;
+
+	if (a != 0x00)
+	{
+	    if (a != MASK)
+	    {
+		comp4_t d = *(dest + i);
+		a = combine_disjoint_out_part (d >> A_SHIFT, a);
+		UNcx4_MUL_UNc_ADD_UNcx4 (d, a, s);
+		s = d;
+	    }
+
+	    *(dest + i) = s;
+	}
+    }
+}
+
+static void
+combine_disjoint_in_u (pixman_implementation_t *imp,
+                       pixman_op_t              op,
+                       comp4_t *                dest,
+                       const comp4_t *          src,
+                       const comp4_t *          mask,
+                       int                      width)
+{
+    combine_disjoint_general_u (dest, src, mask, width, COMBINE_A_IN);
+}
+
+static void
+combine_disjoint_in_reverse_u (pixman_implementation_t *imp,
+                               pixman_op_t              op,
+                               comp4_t *                dest,
+                               const comp4_t *          src,
+                               const comp4_t *          mask,
+                               int                      width)
+{
+    combine_disjoint_general_u (dest, src, mask, width, COMBINE_B_IN);
+}
+
+static void
+combine_disjoint_out_u (pixman_implementation_t *imp,
+                        pixman_op_t              op,
+                        comp4_t *                dest,
+                        const comp4_t *          src,
+                        const comp4_t *          mask,
+                        int                      width)
+{
+    combine_disjoint_general_u (dest, src, mask, width, COMBINE_A_OUT);
+}
+
+static void
+combine_disjoint_out_reverse_u (pixman_implementation_t *imp,
+                                pixman_op_t              op,
+                                comp4_t *                dest,
+                                const comp4_t *          src,
+                                const comp4_t *          mask,
+                                int                      width)
+{
+    combine_disjoint_general_u (dest, src, mask, width, COMBINE_B_OUT);
+}
+
+static void
+combine_disjoint_atop_u (pixman_implementation_t *imp,
+                         pixman_op_t              op,
+                         comp4_t *                dest,
+                         const comp4_t *          src,
+                         const comp4_t *          mask,
+                         int                      width)
+{
+    combine_disjoint_general_u (dest, src, mask, width, COMBINE_A_ATOP);
+}
+
+static void
+combine_disjoint_atop_reverse_u (pixman_implementation_t *imp,
+                                 pixman_op_t              op,
+                                 comp4_t *                dest,
+                                 const comp4_t *          src,
+                                 const comp4_t *          mask,
+                                 int                      width)
+{
+    combine_disjoint_general_u (dest, src, mask, width, COMBINE_B_ATOP);
+}
+
+static void
+combine_disjoint_xor_u (pixman_implementation_t *imp,
+                        pixman_op_t              op,
+                        comp4_t *                dest,
+                        const comp4_t *          src,
+                        const comp4_t *          mask,
+                        int                      width)
+{
+    combine_disjoint_general_u (dest, src, mask, width, COMBINE_XOR);
+}
+
+static void
+combine_conjoint_general_u (comp4_t *      dest,
+                            const comp4_t *src,
+                            const comp4_t *mask,
+                            int            width,
+                            comp1_t        combine)
+{
+    int i;
+
+    for (i = 0; i < width; ++i)
+    {
+	comp4_t s = combine_mask (src, mask, i);
+	comp4_t d = *(dest + i);
+	comp4_t m, n, o, p;
+	comp2_t Fa, Fb, t, u, v;
+	comp1_t sa = s >> A_SHIFT;
+	comp1_t da = d >> A_SHIFT;
+
+	switch (combine & COMBINE_A)
+	{
+	default:
+	    Fa = 0;
+	    break;
+
+	case COMBINE_A_OUT:
+	    Fa = combine_conjoint_out_part (sa, da);
+	    break;
+
+	case COMBINE_A_IN:
+	    Fa = combine_conjoint_in_part (sa, da);
+	    break;
+
+	case COMBINE_A:
+	    Fa = MASK;
+	    break;
+	}
+
+	switch (combine & COMBINE_B)
+	{
+	default:
+	    Fb = 0;
+	    break;
+
+	case COMBINE_B_OUT:
+	    Fb = combine_conjoint_out_part (da, sa);
+	    break;
+
+	case COMBINE_B_IN:
+	    Fb = combine_conjoint_in_part (da, sa);
+	    break;
+
+	case COMBINE_B:
+	    Fb = MASK;
+	    break;
+	}
+
+	m = GENERIC (s, d, 0, Fa, Fb, t, u, v);
+	n = GENERIC (s, d, G_SHIFT, Fa, Fb, t, u, v);
+	o = GENERIC (s, d, R_SHIFT, Fa, Fb, t, u, v);
+	p = GENERIC (s, d, A_SHIFT, Fa, Fb, t, u, v);
+
+	s = m | n | o | p;
+
+	*(dest + i) = s;
+    }
+}
+
+static void
+combine_conjoint_over_u (pixman_implementation_t *imp,
+                         pixman_op_t              op,
+                         comp4_t *                dest,
+                         const comp4_t *          src,
+                         const comp4_t *          mask,
+                         int                      width)
+{
+    combine_conjoint_general_u (dest, src, mask, width, COMBINE_A_OVER);
+}
+
+static void
+combine_conjoint_over_reverse_u (pixman_implementation_t *imp,
+                                 pixman_op_t              op,
+                                 comp4_t *                dest,
+                                 const comp4_t *          src,
+                                 const comp4_t *          mask,
+                                 int                      width)
+{
+    combine_conjoint_general_u (dest, src, mask, width, COMBINE_B_OVER);
+}
+
+static void
+combine_conjoint_in_u (pixman_implementation_t *imp,
+                       pixman_op_t              op,
+                       comp4_t *                dest,
+                       const comp4_t *          src,
+                       const comp4_t *          mask,
+                       int                      width)
+{
+    combine_conjoint_general_u (dest, src, mask, width, COMBINE_A_IN);
+}
+
+static void
+combine_conjoint_in_reverse_u (pixman_implementation_t *imp,
+                               pixman_op_t              op,
+                               comp4_t *                dest,
+                               const comp4_t *          src,
+                               const comp4_t *          mask,
+                               int                      width)
+{
+    combine_conjoint_general_u (dest, src, mask, width, COMBINE_B_IN);
+}
+
+static void
+combine_conjoint_out_u (pixman_implementation_t *imp,
+                        pixman_op_t              op,
+                        comp4_t *                dest,
+                        const comp4_t *          src,
+                        const comp4_t *          mask,
+                        int                      width)
+{
+    combine_conjoint_general_u (dest, src, mask, width, COMBINE_A_OUT);
+}
+
+static void
+combine_conjoint_out_reverse_u (pixman_implementation_t *imp,
+                                pixman_op_t              op,
+                                comp4_t *                dest,
+                                const comp4_t *          src,
+                                const comp4_t *          mask,
+                                int                      width)
+{
+    combine_conjoint_general_u (dest, src, mask, width, COMBINE_B_OUT);
+}
+
+static void
+combine_conjoint_atop_u (pixman_implementation_t *imp,
+                         pixman_op_t              op,
+                         comp4_t *                dest,
+                         const comp4_t *          src,
+                         const comp4_t *          mask,
+                         int                      width)
+{
+    combine_conjoint_general_u (dest, src, mask, width, COMBINE_A_ATOP);
+}
+
+static void
+combine_conjoint_atop_reverse_u (pixman_implementation_t *imp,
+                                 pixman_op_t              op,
+                                 comp4_t *                dest,
+                                 const comp4_t *          src,
+                                 const comp4_t *          mask,
+                                 int                      width)
+{
+    combine_conjoint_general_u (dest, src, mask, width, COMBINE_B_ATOP);
+}
+
+static void
+combine_conjoint_xor_u (pixman_implementation_t *imp,
+                        pixman_op_t              op,
+                        comp4_t *                dest,
+                        const comp4_t *          src,
+                        const comp4_t *          mask,
+                        int                      width)
+{
+    combine_conjoint_general_u (dest, src, mask, width, COMBINE_XOR);
+}
+
+/************************************************************************/
+/*********************** Per Channel functions **************************/
+/************************************************************************/
+
+static void
+combine_clear_ca (pixman_implementation_t *imp,
+                  pixman_op_t              op,
+                  comp4_t *                dest,
+                  const comp4_t *          src,
+                  const comp4_t *          mask,
+                  int                      width)
+{
+    memset (dest, 0, width * sizeof(comp4_t));
+}
+
+static void
+combine_src_ca (pixman_implementation_t *imp,
+                pixman_op_t              op,
+                comp4_t *                dest,
+                const comp4_t *          src,
+                const comp4_t *          mask,
+                int                      width)
+{
+    int i;
+
+    for (i = 0; i < width; ++i)
+    {
+	comp4_t s = *(src + i);
+	comp4_t m = *(mask + i);
+
+	combine_mask_value_ca (&s, &m);
+
+	*(dest + i) = s;
+    }
+}
+
+static void
+combine_over_ca (pixman_implementation_t *imp,
+                 pixman_op_t              op,
+                 comp4_t *                dest,
+                 const comp4_t *          src,
+                 const comp4_t *          mask,
+                 int                      width)
+{
+    int i;
+
+    for (i = 0; i < width; ++i)
+    {
+	comp4_t s = *(src + i);
+	comp4_t m = *(mask + i);
+	comp4_t a;
+
+	combine_mask_ca (&s, &m);
+
+	a = ~m;
+	if (a)
+	{
+	    comp4_t d = *(dest + i);
+	    UNcx4_MUL_UNcx4_ADD_UNcx4 (d, a, s);
+	    s = d;
+	}
+
+	*(dest + i) = s;
+    }
+}
+
+static void
+combine_over_reverse_ca (pixman_implementation_t *imp,
+                         pixman_op_t              op,
+                         comp4_t *                dest,
+                         const comp4_t *          src,
+                         const comp4_t *          mask,
+                         int                      width)
+{
+    int i;
+
+    for (i = 0; i < width; ++i)
+    {
+	comp4_t d = *(dest + i);
+	comp4_t a = ~d >> A_SHIFT;
+
+	if (a)
+	{
+	    comp4_t s = *(src + i);
+	    comp4_t m = *(mask + i);
+
+	    UNcx4_MUL_UNcx4 (s, m);
+	    UNcx4_MUL_UNc_ADD_UNcx4 (s, a, d);
+
+	    *(dest + i) = s;
+	}
+    }
+}
+
+static void
+combine_in_ca (pixman_implementation_t *imp,
+               pixman_op_t              op,
+               comp4_t *                dest,
+               const comp4_t *          src,
+               const comp4_t *          mask,
+               int                      width)
+{
+    int i;
+
+    for (i = 0; i < width; ++i)
+    {
+	comp4_t d = *(dest + i);
+	comp2_t a = d >> A_SHIFT;
+	comp4_t s = 0;
+
+	if (a)
+	{
+	    comp4_t m = *(mask + i);
+
+	    s = *(src + i);
+	    combine_mask_value_ca (&s, &m);
+
+	    if (a != MASK)
+		UNcx4_MUL_UNc (s, a);
+	}
+
+	*(dest + i) = s;
+    }
+}
+
+static void
+combine_in_reverse_ca (pixman_implementation_t *imp,
+                       pixman_op_t              op,
+                       comp4_t *                dest,
+                       const comp4_t *          src,
+                       const comp4_t *          mask,
+                       int                      width)
+{
+    int i;
+
+    for (i = 0; i < width; ++i)
+    {
+	comp4_t s = *(src + i);
+	comp4_t m = *(mask + i);
+	comp4_t a;
+
+	combine_mask_alpha_ca (&s, &m);
+
+	a = m;
+	if (a != ~0)
+	{
+	    comp4_t d = 0;
+
+	    if (a)
+	    {
+		d = *(dest + i);
+		UNcx4_MUL_UNcx4 (d, a);
+	    }
+
+	    *(dest + i) = d;
+	}
+    }
+}
+
+static void
+combine_out_ca (pixman_implementation_t *imp,
+                pixman_op_t              op,
+                comp4_t *                dest,
+                const comp4_t *          src,
+                const comp4_t *          mask,
+                int                      width)
+{
+    int i;
+
+    for (i = 0; i < width; ++i)
+    {
+	comp4_t d = *(dest + i);
+	comp2_t a = ~d >> A_SHIFT;
+	comp4_t s = 0;
+
+	if (a)
+	{
+	    comp4_t m = *(mask + i);
+
+	    s = *(src + i);
+	    combine_mask_value_ca (&s, &m);
+
+	    if (a != MASK)
+		UNcx4_MUL_UNc (s, a);
+	}
+
+	*(dest + i) = s;
+    }
+}
+
+static void
+combine_out_reverse_ca (pixman_implementation_t *imp,
+                        pixman_op_t              op,
+                        comp4_t *                dest,
+                        const comp4_t *          src,
+                        const comp4_t *          mask,
+                        int                      width)
+{
+    int i;
+
+    for (i = 0; i < width; ++i)
+    {
+	comp4_t s = *(src + i);
+	comp4_t m = *(mask + i);
+	comp4_t a;
+
+	combine_mask_alpha_ca (&s, &m);
+
+	a = ~m;
+	if (a != ~0)
+	{
+	    comp4_t d = 0;
+
+	    if (a)
+	    {
+		d = *(dest + i);
+		UNcx4_MUL_UNcx4 (d, a);
+	    }
+
+	    *(dest + i) = d;
+	}
+    }
+}
+
+static void
+combine_atop_ca (pixman_implementation_t *imp,
+                 pixman_op_t              op,
+                 comp4_t *                dest,
+                 const comp4_t *          src,
+                 const comp4_t *          mask,
+                 int                      width)
+{
+    int i;
+
+    for (i = 0; i < width; ++i)
+    {
+	comp4_t d = *(dest + i);
+	comp4_t s = *(src + i);
+	comp4_t m = *(mask + i);
+	comp4_t ad;
+	comp2_t as = d >> A_SHIFT;
+
+	combine_mask_ca (&s, &m);
+
+	ad = ~m;
+
+	UNcx4_MUL_UNcx4_ADD_UNcx4_MUL_UNc (d, ad, s, as);
+
+	*(dest + i) = d;
+    }
+}
+
+static void
+combine_atop_reverse_ca (pixman_implementation_t *imp,
+                         pixman_op_t              op,
+                         comp4_t *                dest,
+                         const comp4_t *          src,
+                         const comp4_t *          mask,
+                         int                      width)
+{
+    int i;
+
+    for (i = 0; i < width; ++i)
+    {
+	comp4_t d = *(dest + i);
+	comp4_t s = *(src + i);
+	comp4_t m = *(mask + i);
+	comp4_t ad;
+	comp2_t as = ~d >> A_SHIFT;
+
+	combine_mask_ca (&s, &m);
+
+	ad = m;
+
+	UNcx4_MUL_UNcx4_ADD_UNcx4_MUL_UNc (d, ad, s, as);
+
+	*(dest + i) = d;
+    }
+}
+
+static void
+combine_xor_ca (pixman_implementation_t *imp,
+                pixman_op_t              op,
+                comp4_t *                dest,
+                const comp4_t *          src,
+                const comp4_t *          mask,
+                int                      width)
+{
+    int i;
+
+    for (i = 0; i < width; ++i)
+    {
+	comp4_t d = *(dest + i);
+	comp4_t s = *(src + i);
+	comp4_t m = *(mask + i);
+	comp4_t ad;
+	comp2_t as = ~d >> A_SHIFT;
+
+	combine_mask_ca (&s, &m);
+
+	ad = ~m;
+
+	UNcx4_MUL_UNcx4_ADD_UNcx4_MUL_UNc (d, ad, s, as);
+
+	*(dest + i) = d;
+    }
+}
+
+static void
+combine_add_ca (pixman_implementation_t *imp,
+                pixman_op_t              op,
+                comp4_t *                dest,
+                const comp4_t *          src,
+                const comp4_t *          mask,
+                int                      width)
+{
+    int i;
+
+    for (i = 0; i < width; ++i)
+    {
+	comp4_t s = *(src + i);
+	comp4_t m = *(mask + i);
+	comp4_t d = *(dest + i);
+
+	combine_mask_value_ca (&s, &m);
+
+	UNcx4_ADD_UNcx4 (d, s);
+
+	*(dest + i) = d;
+    }
+}
+
+static void
+combine_saturate_ca (pixman_implementation_t *imp,
+                     pixman_op_t              op,
+                     comp4_t *                dest,
+                     const comp4_t *          src,
+                     const comp4_t *          mask,
+                     int                      width)
+{
+    int i;
+
+    for (i = 0; i < width; ++i)
+    {
+	comp4_t s, d;
+	comp2_t sa, sr, sg, sb, da;
+	comp2_t t, u, v;
+	comp4_t m, n, o, p;
+
+	d = *(dest + i);
+	s = *(src + i);
+	m = *(mask + i);
+
+	combine_mask_ca (&s, &m);
+
+	sa = (m >> A_SHIFT);
+	sr = (m >> R_SHIFT) & MASK;
+	sg = (m >> G_SHIFT) & MASK;
+	sb =  m             & MASK;
+	da = ~d >> A_SHIFT;
+
+	if (sb <= da)
+	    m = ADD (s, d, 0, t);
+	else
+	    m = GENERIC (s, d, 0, (da << G_SHIFT) / sb, MASK, t, u, v);
+
+	if (sg <= da)
+	    n = ADD (s, d, G_SHIFT, t);
+	else
+	    n = GENERIC (s, d, G_SHIFT, (da << G_SHIFT) / sg, MASK, t, u, v);
+
+	if (sr <= da)
+	    o = ADD (s, d, R_SHIFT, t);
+	else
+	    o = GENERIC (s, d, R_SHIFT, (da << G_SHIFT) / sr, MASK, t, u, v);
+
+	if (sa <= da)
+	    p = ADD (s, d, A_SHIFT, t);
+	else
+	    p = GENERIC (s, d, A_SHIFT, (da << G_SHIFT) / sa, MASK, t, u, v);
+
+	*(dest + i) = m | n | o | p;
+    }
+}
+
+static void
+combine_disjoint_general_ca (comp4_t *      dest,
+                             const comp4_t *src,
+                             const comp4_t *mask,
+                             int            width,
+                             comp1_t        combine)
+{
+    int i;
+
+    for (i = 0; i < width; ++i)
+    {
+	comp4_t s, d;
+	comp4_t m, n, o, p;
+	comp4_t Fa, Fb;
+	comp2_t t, u, v;
+	comp4_t sa;
+	comp1_t da;
+
+	s = *(src + i);
+	m = *(mask + i);
+	d = *(dest + i);
+	da = d >> A_SHIFT;
+
+	combine_mask_ca (&s, &m);
+
+	sa = m;
+
+	switch (combine & COMBINE_A)
+	{
+	default:
+	    Fa = 0;
+	    break;
+
+	case COMBINE_A_OUT:
+	    m = (comp4_t)combine_disjoint_out_part ((comp1_t) (sa >> 0), da);
+	    n = (comp4_t)combine_disjoint_out_part ((comp1_t) (sa >> G_SHIFT), da) << G_SHIFT;
+	    o = (comp4_t)combine_disjoint_out_part ((comp1_t) (sa >> R_SHIFT), da) << R_SHIFT;
+	    p = (comp4_t)combine_disjoint_out_part ((comp1_t) (sa >> A_SHIFT), da) << A_SHIFT;
+	    Fa = m | n | o | p;
+	    break;
+
+	case COMBINE_A_IN:
+	    m = (comp4_t)combine_disjoint_in_part ((comp1_t) (sa >> 0), da);
+	    n = (comp4_t)combine_disjoint_in_part ((comp1_t) (sa >> G_SHIFT), da) << G_SHIFT;
+	    o = (comp4_t)combine_disjoint_in_part ((comp1_t) (sa >> R_SHIFT), da) << R_SHIFT;
+	    p = (comp4_t)combine_disjoint_in_part ((comp1_t) (sa >> A_SHIFT), da) << A_SHIFT;
+	    Fa = m | n | o | p;
+	    break;
+
+	case COMBINE_A:
+	    Fa = ~0;
+	    break;
+	}
+
+	switch (combine & COMBINE_B)
+	{
+	default:
+	    Fb = 0;
+	    break;
+
+	case COMBINE_B_OUT:
+	    m = (comp4_t)combine_disjoint_out_part (da, (comp1_t) (sa >> 0));
+	    n = (comp4_t)combine_disjoint_out_part (da, (comp1_t) (sa >> G_SHIFT)) << G_SHIFT;
+	    o = (comp4_t)combine_disjoint_out_part (da, (comp1_t) (sa >> R_SHIFT)) << R_SHIFT;
+	    p = (comp4_t)combine_disjoint_out_part (da, (comp1_t) (sa >> A_SHIFT)) << A_SHIFT;
+	    Fb = m | n | o | p;
+	    break;
+
+	case COMBINE_B_IN:
+	    m = (comp4_t)combine_disjoint_in_part (da, (comp1_t) (sa >> 0));
+	    n = (comp4_t)combine_disjoint_in_part (da, (comp1_t) (sa >> G_SHIFT)) << G_SHIFT;
+	    o = (comp4_t)combine_disjoint_in_part (da, (comp1_t) (sa >> R_SHIFT)) << R_SHIFT;
+	    p = (comp4_t)combine_disjoint_in_part (da, (comp1_t) (sa >> A_SHIFT)) << A_SHIFT;
+	    Fb = m | n | o | p;
+	    break;
+
+	case COMBINE_B:
+	    Fb = ~0;
+	    break;
+	}
+	m = GENERIC (s, d, 0, GET_COMP (Fa, 0), GET_COMP (Fb, 0), t, u, v);
+	n = GENERIC (s, d, G_SHIFT, GET_COMP (Fa, G_SHIFT), GET_COMP (Fb, G_SHIFT), t, u, v);
+	o = GENERIC (s, d, R_SHIFT, GET_COMP (Fa, R_SHIFT), GET_COMP (Fb, R_SHIFT), t, u, v);
+	p = GENERIC (s, d, A_SHIFT, GET_COMP (Fa, A_SHIFT), GET_COMP (Fb, A_SHIFT), t, u, v);
+
+	s = m | n | o | p;
+
+	*(dest + i) = s;
+    }
+}
+
+static void
+combine_disjoint_over_ca (pixman_implementation_t *imp,
+                          pixman_op_t              op,
+                          comp4_t *                dest,
+                          const comp4_t *          src,
+                          const comp4_t *          mask,
+                          int                      width)
+{
+    combine_disjoint_general_ca (dest, src, mask, width, COMBINE_A_OVER);
+}
+
+static void
+combine_disjoint_in_ca (pixman_implementation_t *imp,
+                        pixman_op_t              op,
+                        comp4_t *                dest,
+                        const comp4_t *          src,
+                        const comp4_t *          mask,
+                        int                      width)
+{
+    combine_disjoint_general_ca (dest, src, mask, width, COMBINE_A_IN);
+}
+
+static void
+combine_disjoint_in_reverse_ca (pixman_implementation_t *imp,
+                                pixman_op_t              op,
+                                comp4_t *                dest,
+                                const comp4_t *          src,
+                                const comp4_t *          mask,
+                                int                      width)
+{
+    combine_disjoint_general_ca (dest, src, mask, width, COMBINE_B_IN);
+}
+
+static void
+combine_disjoint_out_ca (pixman_implementation_t *imp,
+                         pixman_op_t              op,
+                         comp4_t *                dest,
+                         const comp4_t *          src,
+                         const comp4_t *          mask,
+                         int                      width)
+{
+    combine_disjoint_general_ca (dest, src, mask, width, COMBINE_A_OUT);
+}
+
+static void
+combine_disjoint_out_reverse_ca (pixman_implementation_t *imp,
+                                 pixman_op_t              op,
+                                 comp4_t *                dest,
+                                 const comp4_t *          src,
+                                 const comp4_t *          mask,
+                                 int                      width)
+{
+    combine_disjoint_general_ca (dest, src, mask, width, COMBINE_B_OUT);
+}
+
+static void
+combine_disjoint_atop_ca (pixman_implementation_t *imp,
+                          pixman_op_t              op,
+                          comp4_t *                dest,
+                          const comp4_t *          src,
+                          const comp4_t *          mask,
+                          int                      width)
+{
+    combine_disjoint_general_ca (dest, src, mask, width, COMBINE_A_ATOP);
+}
+
+static void
+combine_disjoint_atop_reverse_ca (pixman_implementation_t *imp,
+                                  pixman_op_t              op,
+                                  comp4_t *                dest,
+                                  const comp4_t *          src,
+                                  const comp4_t *          mask,
+                                  int                      width)
+{
+    combine_disjoint_general_ca (dest, src, mask, width, COMBINE_B_ATOP);
+}
+
+static void
+combine_disjoint_xor_ca (pixman_implementation_t *imp,
+                         pixman_op_t              op,
+                         comp4_t *                dest,
+                         const comp4_t *          src,
+                         const comp4_t *          mask,
+                         int                      width)
+{
+    combine_disjoint_general_ca (dest, src, mask, width, COMBINE_XOR);
+}
+
+static void
+combine_conjoint_general_ca (comp4_t *      dest,
+                             const comp4_t *src,
+                             const comp4_t *mask,
+                             int            width,
+                             comp1_t        combine)
+{
+    int i;
+
+    for (i = 0; i < width; ++i)
+    {
+	comp4_t s, d;
+	comp4_t m, n, o, p;
+	comp4_t Fa, Fb;
+	comp2_t t, u, v;
+	comp4_t sa;
+	comp1_t da;
+
+	s = *(src + i);
+	m = *(mask + i);
+	d = *(dest + i);
+	da = d >> A_SHIFT;
+
+	combine_mask_ca (&s, &m);
+
+	sa = m;
+
+	switch (combine & COMBINE_A)
+	{
+	default:
+	    Fa = 0;
+	    break;
+
+	case COMBINE_A_OUT:
+	    m = (comp4_t)combine_conjoint_out_part ((comp1_t) (sa >> 0), da);
+	    n = (comp4_t)combine_conjoint_out_part ((comp1_t) (sa >> G_SHIFT), da) << G_SHIFT;
+	    o = (comp4_t)combine_conjoint_out_part ((comp1_t) (sa >> R_SHIFT), da) << R_SHIFT;
+	    p = (comp4_t)combine_conjoint_out_part ((comp1_t) (sa >> A_SHIFT), da) << A_SHIFT;
+	    Fa = m | n | o | p;
+	    break;
+
+	case COMBINE_A_IN:
+	    m = (comp4_t)combine_conjoint_in_part ((comp1_t) (sa >> 0), da);
+	    n = (comp4_t)combine_conjoint_in_part ((comp1_t) (sa >> G_SHIFT), da) << G_SHIFT;
+	    o = (comp4_t)combine_conjoint_in_part ((comp1_t) (sa >> R_SHIFT), da) << R_SHIFT;
+	    p = (comp4_t)combine_conjoint_in_part ((comp1_t) (sa >> A_SHIFT), da) << A_SHIFT;
+	    Fa = m | n | o | p;
+	    break;
+
+	case COMBINE_A:
+	    Fa = ~0;
+	    break;
+	}
+
+	switch (combine & COMBINE_B)
+	{
+	default:
+	    Fb = 0;
+	    break;
+
+	case COMBINE_B_OUT:
+	    m = (comp4_t)combine_conjoint_out_part (da, (comp1_t) (sa >> 0));
+	    n = (comp4_t)combine_conjoint_out_part (da, (comp1_t) (sa >> G_SHIFT)) << G_SHIFT;
+	    o = (comp4_t)combine_conjoint_out_part (da, (comp1_t) (sa >> R_SHIFT)) << R_SHIFT;
+	    p = (comp4_t)combine_conjoint_out_part (da, (comp1_t) (sa >> A_SHIFT)) << A_SHIFT;
+	    Fb = m | n | o | p;
+	    break;
+
+	case COMBINE_B_IN:
+	    m = (comp4_t)combine_conjoint_in_part (da, (comp1_t) (sa >> 0));
+	    n = (comp4_t)combine_conjoint_in_part (da, (comp1_t) (sa >> G_SHIFT)) << G_SHIFT;
+	    o = (comp4_t)combine_conjoint_in_part (da, (comp1_t) (sa >> R_SHIFT)) << R_SHIFT;
+	    p = (comp4_t)combine_conjoint_in_part (da, (comp1_t) (sa >> A_SHIFT)) << A_SHIFT;
+	    Fb = m | n | o | p;
+	    break;
+
+	case COMBINE_B:
+	    Fb = ~0;
+	    break;
+	}
+	m = GENERIC (s, d, 0, GET_COMP (Fa, 0), GET_COMP (Fb, 0), t, u, v);
+	n = GENERIC (s, d, G_SHIFT, GET_COMP (Fa, G_SHIFT), GET_COMP (Fb, G_SHIFT), t, u, v);
+	o = GENERIC (s, d, R_SHIFT, GET_COMP (Fa, R_SHIFT), GET_COMP (Fb, R_SHIFT), t, u, v);
+	p = GENERIC (s, d, A_SHIFT, GET_COMP (Fa, A_SHIFT), GET_COMP (Fb, A_SHIFT), t, u, v);
+
+	s = m | n | o | p;
+
+	*(dest + i) = s;
+    }
+}
+
+static void
+combine_conjoint_over_ca (pixman_implementation_t *imp,
+                          pixman_op_t              op,
+                          comp4_t *                dest,
+                          const comp4_t *          src,
+                          const comp4_t *          mask,
+                          int                      width)
+{
+    combine_conjoint_general_ca (dest, src, mask, width, COMBINE_A_OVER);
+}
+
+static void
+combine_conjoint_over_reverse_ca (pixman_implementation_t *imp,
+                                  pixman_op_t              op,
+                                  comp4_t *                dest,
+                                  const comp4_t *          src,
+                                  const comp4_t *          mask,
+                                  int                      width)
+{
+    combine_conjoint_general_ca (dest, src, mask, width, COMBINE_B_OVER);
+}
+
+static void
+combine_conjoint_in_ca (pixman_implementation_t *imp,
+                        pixman_op_t              op,
+                        comp4_t *                dest,
+                        const comp4_t *          src,
+                        const comp4_t *          mask,
+                        int                      width)
+{
+    combine_conjoint_general_ca (dest, src, mask, width, COMBINE_A_IN);
+}
+
+static void
+combine_conjoint_in_reverse_ca (pixman_implementation_t *imp,
+                                pixman_op_t              op,
+                                comp4_t *                dest,
+                                const comp4_t *          src,
+                                const comp4_t *          mask,
+                                int                      width)
+{
+    combine_conjoint_general_ca (dest, src, mask, width, COMBINE_B_IN);
+}
+
+static void
+combine_conjoint_out_ca (pixman_implementation_t *imp,
+                         pixman_op_t              op,
+                         comp4_t *                dest,
+                         const comp4_t *          src,
+                         const comp4_t *          mask,
+                         int                      width)
+{
+    combine_conjoint_general_ca (dest, src, mask, width, COMBINE_A_OUT);
+}
+
+static void
+combine_conjoint_out_reverse_ca (pixman_implementation_t *imp,
+                                 pixman_op_t              op,
+                                 comp4_t *                dest,
+                                 const comp4_t *          src,
+                                 const comp4_t *          mask,
+                                 int                      width)
+{
+    combine_conjoint_general_ca (dest, src, mask, width, COMBINE_B_OUT);
+}
+
+static void
+combine_conjoint_atop_ca (pixman_implementation_t *imp,
+                          pixman_op_t              op,
+                          comp4_t *                dest,
+                          const comp4_t *          src,
+                          const comp4_t *          mask,
+                          int                      width)
+{
+    combine_conjoint_general_ca (dest, src, mask, width, COMBINE_A_ATOP);
+}
+
+static void
+combine_conjoint_atop_reverse_ca (pixman_implementation_t *imp,
+                                  pixman_op_t              op,
+                                  comp4_t *                dest,
+                                  const comp4_t *          src,
+                                  const comp4_t *          mask,
+                                  int                      width)
+{
+    combine_conjoint_general_ca (dest, src, mask, width, COMBINE_B_ATOP);
+}
+
+static void
+combine_conjoint_xor_ca (pixman_implementation_t *imp,
+                         pixman_op_t              op,
+                         comp4_t *                dest,
+                         const comp4_t *          src,
+                         const comp4_t *          mask,
+                         int                      width)
+{
+    combine_conjoint_general_ca (dest, src, mask, width, COMBINE_XOR);
+}
+
+void
+_pixman_setup_combiner_functions_width (pixman_implementation_t *imp)
+{
+    /* Unified alpha */
+    imp->combine_width[PIXMAN_OP_CLEAR] = combine_clear;
+    imp->combine_width[PIXMAN_OP_SRC] = combine_src_u;
+    /* dest */
+    imp->combine_width[PIXMAN_OP_OVER] = combine_over_u;
+    imp->combine_width[PIXMAN_OP_OVER_REVERSE] = combine_over_reverse_u;
+    imp->combine_width[PIXMAN_OP_IN] = combine_in_u;
+    imp->combine_width[PIXMAN_OP_IN_REVERSE] = combine_in_reverse_u;
+    imp->combine_width[PIXMAN_OP_OUT] = combine_out_u;
+    imp->combine_width[PIXMAN_OP_OUT_REVERSE] = combine_out_reverse_u;
+    imp->combine_width[PIXMAN_OP_ATOP] = combine_atop_u;
+    imp->combine_width[PIXMAN_OP_ATOP_REVERSE] = combine_atop_reverse_u;
+    imp->combine_width[PIXMAN_OP_XOR] = combine_xor_u;
+    imp->combine_width[PIXMAN_OP_ADD] = combine_add_u;
+    imp->combine_width[PIXMAN_OP_SATURATE] = combine_saturate_u;
+
+    /* Disjoint, unified */
+    imp->combine_width[PIXMAN_OP_DISJOINT_CLEAR] = combine_clear;
+    imp->combine_width[PIXMAN_OP_DISJOINT_SRC] = combine_src_u;
+    /* dest */
+    imp->combine_width[PIXMAN_OP_DISJOINT_OVER] = combine_disjoint_over_u;
+    imp->combine_width[PIXMAN_OP_DISJOINT_OVER_REVERSE] = combine_saturate_u;
+    imp->combine_width[PIXMAN_OP_DISJOINT_IN] = combine_disjoint_in_u;
+    imp->combine_width[PIXMAN_OP_DISJOINT_IN_REVERSE] = combine_disjoint_in_reverse_u;
+    imp->combine_width[PIXMAN_OP_DISJOINT_OUT] = combine_disjoint_out_u;
+    imp->combine_width[PIXMAN_OP_DISJOINT_OUT_REVERSE] = combine_disjoint_out_reverse_u;
+    imp->combine_width[PIXMAN_OP_DISJOINT_ATOP] = combine_disjoint_atop_u;
+    imp->combine_width[PIXMAN_OP_DISJOINT_ATOP_REVERSE] = combine_disjoint_atop_reverse_u;
+    imp->combine_width[PIXMAN_OP_DISJOINT_XOR] = combine_disjoint_xor_u;
+
+    /* Conjoint, unified */
+    imp->combine_width[PIXMAN_OP_CONJOINT_CLEAR] = combine_clear;
+    imp->combine_width[PIXMAN_OP_CONJOINT_SRC] = combine_src_u;
+    /* dest */
+    imp->combine_width[PIXMAN_OP_CONJOINT_OVER] = combine_conjoint_over_u;
+    imp->combine_width[PIXMAN_OP_CONJOINT_OVER_REVERSE] = combine_conjoint_over_reverse_u;
+    imp->combine_width[PIXMAN_OP_CONJOINT_IN] = combine_conjoint_in_u;
+    imp->combine_width[PIXMAN_OP_CONJOINT_IN_REVERSE] = combine_conjoint_in_reverse_u;
+    imp->combine_width[PIXMAN_OP_CONJOINT_OUT] = combine_conjoint_out_u;
+    imp->combine_width[PIXMAN_OP_CONJOINT_OUT_REVERSE] = combine_conjoint_out_reverse_u;
+    imp->combine_width[PIXMAN_OP_CONJOINT_ATOP] = combine_conjoint_atop_u;
+    imp->combine_width[PIXMAN_OP_CONJOINT_ATOP_REVERSE] = combine_conjoint_atop_reverse_u;
+    imp->combine_width[PIXMAN_OP_CONJOINT_XOR] = combine_conjoint_xor_u;
+
+    imp->combine_width[PIXMAN_OP_MULTIPLY] = combine_multiply_u;
+    imp->combine_width[PIXMAN_OP_SCREEN] = combine_screen_u;
+    imp->combine_width[PIXMAN_OP_OVERLAY] = combine_overlay_u;
+    imp->combine_width[PIXMAN_OP_DARKEN] = combine_darken_u;
+    imp->combine_width[PIXMAN_OP_LIGHTEN] = combine_lighten_u;
+    imp->combine_width[PIXMAN_OP_COLOR_DODGE] = combine_color_dodge_u;
+    imp->combine_width[PIXMAN_OP_COLOR_BURN] = combine_color_burn_u;
+    imp->combine_width[PIXMAN_OP_HARD_LIGHT] = combine_hard_light_u;
+    imp->combine_width[PIXMAN_OP_SOFT_LIGHT] = combine_soft_light_u;
+    imp->combine_width[PIXMAN_OP_DIFFERENCE] = combine_difference_u;
+    imp->combine_width[PIXMAN_OP_EXCLUSION] = combine_exclusion_u;
+    imp->combine_width[PIXMAN_OP_HSL_HUE] = combine_hsl_hue_u;
+    imp->combine_width[PIXMAN_OP_HSL_SATURATION] = combine_hsl_saturation_u;
+    imp->combine_width[PIXMAN_OP_HSL_COLOR] = combine_hsl_color_u;
+    imp->combine_width[PIXMAN_OP_HSL_LUMINOSITY] = combine_hsl_luminosity_u;
+
+    /* Component alpha combiners */
+    imp->combine_width_ca[PIXMAN_OP_CLEAR] = combine_clear_ca;
+    imp->combine_width_ca[PIXMAN_OP_SRC] = combine_src_ca;
+    /* dest */
+    imp->combine_width_ca[PIXMAN_OP_OVER] = combine_over_ca;
+    imp->combine_width_ca[PIXMAN_OP_OVER_REVERSE] = combine_over_reverse_ca;
+    imp->combine_width_ca[PIXMAN_OP_IN] = combine_in_ca;
+    imp->combine_width_ca[PIXMAN_OP_IN_REVERSE] = combine_in_reverse_ca;
+    imp->combine_width_ca[PIXMAN_OP_OUT] = combine_out_ca;
+    imp->combine_width_ca[PIXMAN_OP_OUT_REVERSE] = combine_out_reverse_ca;
+    imp->combine_width_ca[PIXMAN_OP_ATOP] = combine_atop_ca;
+    imp->combine_width_ca[PIXMAN_OP_ATOP_REVERSE] = combine_atop_reverse_ca;
+    imp->combine_width_ca[PIXMAN_OP_XOR] = combine_xor_ca;
+    imp->combine_width_ca[PIXMAN_OP_ADD] = combine_add_ca;
+    imp->combine_width_ca[PIXMAN_OP_SATURATE] = combine_saturate_ca;
+
+    /* Disjoint CA */
+    imp->combine_width_ca[PIXMAN_OP_DISJOINT_CLEAR] = combine_clear_ca;
+    imp->combine_width_ca[PIXMAN_OP_DISJOINT_SRC] = combine_src_ca;
+    /* dest */
+    imp->combine_width_ca[PIXMAN_OP_DISJOINT_OVER] = combine_disjoint_over_ca;
+    imp->combine_width_ca[PIXMAN_OP_DISJOINT_OVER_REVERSE] = combine_saturate_ca;
+    imp->combine_width_ca[PIXMAN_OP_DISJOINT_IN] = combine_disjoint_in_ca;
+    imp->combine_width_ca[PIXMAN_OP_DISJOINT_IN_REVERSE] = combine_disjoint_in_reverse_ca;
+    imp->combine_width_ca[PIXMAN_OP_DISJOINT_OUT] = combine_disjoint_out_ca;
+    imp->combine_width_ca[PIXMAN_OP_DISJOINT_OUT_REVERSE] = combine_disjoint_out_reverse_ca;
+    imp->combine_width_ca[PIXMAN_OP_DISJOINT_ATOP] = combine_disjoint_atop_ca;
+    imp->combine_width_ca[PIXMAN_OP_DISJOINT_ATOP_REVERSE] = combine_disjoint_atop_reverse_ca;
+    imp->combine_width_ca[PIXMAN_OP_DISJOINT_XOR] = combine_disjoint_xor_ca;
+
+    /* Conjoint CA */
+    imp->combine_width_ca[PIXMAN_OP_CONJOINT_CLEAR] = combine_clear_ca;
+    imp->combine_width_ca[PIXMAN_OP_CONJOINT_SRC] = combine_src_ca;
+    /* dest */
+    imp->combine_width_ca[PIXMAN_OP_CONJOINT_OVER] = combine_conjoint_over_ca;
+    imp->combine_width_ca[PIXMAN_OP_CONJOINT_OVER_REVERSE] = combine_conjoint_over_reverse_ca;
+    imp->combine_width_ca[PIXMAN_OP_CONJOINT_IN] = combine_conjoint_in_ca;
+    imp->combine_width_ca[PIXMAN_OP_CONJOINT_IN_REVERSE] = combine_conjoint_in_reverse_ca;
+    imp->combine_width_ca[PIXMAN_OP_CONJOINT_OUT] = combine_conjoint_out_ca;
+    imp->combine_width_ca[PIXMAN_OP_CONJOINT_OUT_REVERSE] = combine_conjoint_out_reverse_ca;
+    imp->combine_width_ca[PIXMAN_OP_CONJOINT_ATOP] = combine_conjoint_atop_ca;
+    imp->combine_width_ca[PIXMAN_OP_CONJOINT_ATOP_REVERSE] = combine_conjoint_atop_reverse_ca;
+    imp->combine_width_ca[PIXMAN_OP_CONJOINT_XOR] = combine_conjoint_xor_ca;
+
+    imp->combine_width_ca[PIXMAN_OP_MULTIPLY] = combine_multiply_ca;
+    imp->combine_width_ca[PIXMAN_OP_SCREEN] = combine_screen_ca;
+    imp->combine_width_ca[PIXMAN_OP_OVERLAY] = combine_overlay_ca;
+    imp->combine_width_ca[PIXMAN_OP_DARKEN] = combine_darken_ca;
+    imp->combine_width_ca[PIXMAN_OP_LIGHTEN] = combine_lighten_ca;
+    imp->combine_width_ca[PIXMAN_OP_COLOR_DODGE] = combine_color_dodge_ca;
+    imp->combine_width_ca[PIXMAN_OP_COLOR_BURN] = combine_color_burn_ca;
+    imp->combine_width_ca[PIXMAN_OP_HARD_LIGHT] = combine_hard_light_ca;
+    imp->combine_width_ca[PIXMAN_OP_SOFT_LIGHT] = combine_soft_light_ca;
+    imp->combine_width_ca[PIXMAN_OP_DIFFERENCE] = combine_difference_ca;
+    imp->combine_width_ca[PIXMAN_OP_EXCLUSION] = combine_exclusion_ca;
+
+    /* It is not clear that these make sense, so leave them out for now */
+    imp->combine_width_ca[PIXMAN_OP_HSL_HUE] = NULL;
+    imp->combine_width_ca[PIXMAN_OP_HSL_SATURATION] = NULL;
+    imp->combine_width_ca[PIXMAN_OP_HSL_COLOR] = NULL;
+    imp->combine_width_ca[PIXMAN_OP_HSL_LUMINOSITY] = NULL;
+}
+
diff --git a/pixman/pixman/pixman-combine.h.template b/pixman/pixman/pixman-combine.h.template
new file mode 100644
index 000000000..2f6392f96
--- /dev/null
+++ b/pixman/pixman/pixman-combine.h.template
@@ -0,0 +1,226 @@
+
+#define COMPONENT_SIZE
+#define MASK
+#define ONE_HALF
+
+#define A_SHIFT
+#define R_SHIFT
+#define G_SHIFT
+#define A_MASK
+#define R_MASK
+#define G_MASK
+
+#define RB_MASK
+#define AG_MASK
+#define RB_ONE_HALF
+#define RB_MASK_PLUS_ONE
+
+#define ALPHA_c(x) ((x) >> A_SHIFT)
+#define RED_c(x) (((x) >> R_SHIFT) & MASK)
+#define GREEN_c(x) (((x) >> G_SHIFT) & MASK)
+#define BLUE_c(x) ((x) & MASK)
+
+/*
+ * Helper macros.
+ */
+
+#define MUL_UNc(a, b, t)						\
+    ((t) = (a) * (b) + ONE_HALF, ((((t) >> G_SHIFT ) + (t) ) >> G_SHIFT ))
+
+#define DIV_UNc(a, b)							\
+    (((comp2_t) (a) * MASK) / (b))
+
+#define ADD_UNc(x, y, t)				     \
+    ((t) = x + y,					     \
+     (comp4_t) (comp1_t) ((t) | (0 - ((t) >> G_SHIFT))))
+
+#define DIV_ONE_UNc(x)							\
+    (((x) + ONE_HALF + (((x) + ONE_HALF) >> G_SHIFT)) >> G_SHIFT)
+
+/*
+ * The methods below use some tricks to be able to do two color
+ * components at the same time.
+ */
+
+/*
+ * x_rb = (x_rb * a) / 255
+ */
+#define UNc_rb_MUL_UNc(x, a, t)						\
+    do									\
+    {									\
+	t  = ((x) & RB_MASK) * (a);					\
+	t += RB_ONE_HALF;						\
+	x = (t + ((t >> G_SHIFT) & RB_MASK)) >> G_SHIFT;		\
+	x &= RB_MASK;							\
+    } while (0)
+
+/*
+ * x_rb = min (x_rb + y_rb, 255)
+ */
+#define UNc_rb_ADD_UNc_rb(x, y, t)					\
+    do									\
+    {									\
+	t = ((x) + (y));						\
+	t |= RB_MASK_PLUS_ONE - ((t >> G_SHIFT) & RB_MASK);		\
+	x = (t & RB_MASK);						\
+    } while (0)
+
+/*
+ * x_rb = (x_rb * a_rb) / 255
+ */
+#define UNc_rb_MUL_UNc_rb(x, a, t)					\
+    do									\
+    {									\
+	t  = (x & MASK) * (a & MASK);					\
+	t |= (x & R_MASK) * ((a >> R_SHIFT) & MASK);			\
+	t += RB_ONE_HALF;						\
+	t = (t + ((t >> G_SHIFT) & RB_MASK)) >> G_SHIFT;		\
+	x = t & RB_MASK;						\
+    } while (0)
+
+/*
+ * x_c = (x_c * a) / 255
+ */
+#define UNcx4_MUL_UNc(x, a)						\
+    do									\
+    {									\
+	comp4_t r1, r2, t;						\
+									\
+	r1 = (x);							\
+	UNc_rb_MUL_UNc (r1, a, t);					\
+									\
+	r2 = (x) >> G_SHIFT;						\
+	UNc_rb_MUL_UNc (r2, a, t);					\
+									\
+	x = r1 | (r2 << G_SHIFT);					\
+    } while (0)
+
+/*
+ * x_c = (x_c * a) / 255 + y_c
+ */
+#define UNcx4_MUL_UNc_ADD_UNcx4(x, a, y)				\
+    do									\
+    {									\
+	comp4_t r1, r2, r3, t;						\
+									\
+	r1 = (x);							\
+	r2 = (y) & RB_MASK;						\
+	UNc_rb_MUL_UNc (r1, a, t);					\
+	UNc_rb_ADD_UNc_rb (r1, r2, t);					\
+									\
+	r2 = (x) >> G_SHIFT;						\
+	r3 = ((y) >> G_SHIFT) & RB_MASK;				\
+	UNc_rb_MUL_UNc (r2, a, t);					\
+	UNc_rb_ADD_UNc_rb (r2, r3, t);					\
+									\
+	x = r1 | (r2 << G_SHIFT);					\
+    } while (0)
+
+/*
+ * x_c = (x_c * a + y_c * b) / 255
+ */
+#define UNcx4_MUL_UNc_ADD_UNcx4_MUL_UNc(x, a, y, b)			\
+    do									\
+    {									\
+	comp4_t r1, r2, r3, t;						\
+									\
+	r1 = x;								\
+	r2 = y;								\
+	UNc_rb_MUL_UNc (r1, a, t);					\
+	UNc_rb_MUL_UNc (r2, b, t);					\
+	UNc_rb_ADD_UNc_rb (r1, r2, t);					\
+									\
+	r2 = (x >> G_SHIFT);						\
+	r3 = (y >> G_SHIFT);						\
+	UNc_rb_MUL_UNc (r2, a, t);					\
+	UNc_rb_MUL_UNc (r3, b, t);					\
+	UNc_rb_ADD_UNc_rb (r2, r3, t);					\
+									\
+	x = r1 | (r2 << G_SHIFT);					\
+    } while (0)
+
+/*
+ * x_c = (x_c * a_c) / 255
+ */
+#define UNcx4_MUL_UNcx4(x, a)						\
+    do									\
+    {									\
+	comp4_t r1, r2, r3, t;						\
+									\
+	r1 = x;								\
+	r2 = a;								\
+	UNc_rb_MUL_UNc_rb (r1, r2, t);					\
+									\
+	r2 = x >> G_SHIFT;						\
+	r3 = a >> G_SHIFT;						\
+	UNc_rb_MUL_UNc_rb (r2, r3, t);					\
+									\
+	x = r1 | (r2 << G_SHIFT);					\
+    } while (0)
+
+/*
+ * x_c = (x_c * a_c) / 255 + y_c
+ */
+#define UNcx4_MUL_UNcx4_ADD_UNcx4(x, a, y)				\
+    do									\
+    {									\
+	comp4_t r1, r2, r3, t;						\
+									\
+	r1 = x;								\
+	r2 = a;								\
+	UNc_rb_MUL_UNc_rb (r1, r2, t);					\
+	r2 = y & RB_MASK;						\
+	UNc_rb_ADD_UNc_rb (r1, r2, t);					\
+									\
+	r2 = (x >> G_SHIFT);						\
+	r3 = (a >> G_SHIFT);						\
+	UNc_rb_MUL_UNc_rb (r2, r3, t);					\
+	r3 = (y >> G_SHIFT) & RB_MASK;					\
+	UNc_rb_ADD_UNc_rb (r2, r3, t);					\
+									\
+	x = r1 | (r2 << G_SHIFT);					\
+    } while (0)
+
+/*
+ * x_c = (x_c * a_c + y_c * b) / 255
+ */
+#define UNcx4_MUL_UNcx4_ADD_UNcx4_MUL_UNc(x, a, y, b)			\
+    do									\
+    {									\
+	comp4_t r1, r2, r3, t;						\
+									\
+	r1 = x;								\
+	r2 = a;								\
+	UNc_rb_MUL_UNc_rb (r1, r2, t);					\
+	r2 = y;								\
+	UNc_rb_MUL_UNc (r2, b, t);					\
+	UNc_rb_ADD_UNc_rb (r1, r2, t);					\
+									\
+	r2 = x >> G_SHIFT;						\
+	r3 = a >> G_SHIFT;						\
+	UNc_rb_MUL_UNc_rb (r2, r3, t);					\
+	r3 = y >> G_SHIFT;						\
+	UNc_rb_MUL_UNc (r3, b, t);					\
+	UNc_rb_ADD_UNc_rb (r2, r3, t);					\
+									\
+	x = r1 | (r2 << G_SHIFT);					\
+    } while (0)
+
+/*
+   x_c = min(x_c + y_c, 255)
+ */
+#define UNcx4_ADD_UNcx4(x, y)						\
+    do									\
+    {									\
+	comp4_t r1, r2, r3, t;						\
+									\
+	r1 = x & RB_MASK;						\
+	r2 = y & RB_MASK;						\
+	UNc_rb_ADD_UNc_rb (r1, r2, t);					\
+									\
+	r2 = (x >> G_SHIFT) & RB_MASK;					\
+	r3 = (y >> G_SHIFT) & RB_MASK;					\
+	UNc_rb_ADD_UNc_rb (r2, r3, t);					\
+									\
+	x = r1 | (r2 << G_SHIFT);					\
+    } while (0)
diff --git a/pixman/pixman/pixman-compiler.h b/pixman/pixman/pixman-compiler.h
new file mode 100644
index 000000000..26f7071c9
--- /dev/null
+++ b/pixman/pixman/pixman-compiler.h
@@ -0,0 +1,203 @@
+/* Pixman uses some non-standard compiler features. This file ensures
+ * they exist
+ *
+ * The features are:
+ *
+ *    FUNC	     must be defined to expand to the current function
+ *    PIXMAN_EXPORT  should be defined to whatever is required to
+ *                   export functions from a shared library
+ *    limits	     limits for various types must be defined
+ *    inline         must be defined
+ *    force_inline   must be defined
+ */
+#if defined (__GNUC__)
+#  define FUNC     ((const char*) (__PRETTY_FUNCTION__))
+#elif defined (__sun) || (defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L)
+#  define FUNC     ((const char*) (__func__))
+#else
+#  define FUNC     ((const char*) ("???"))
+#endif
+
+#ifndef INT16_MIN
+# define INT16_MIN              (-32767-1)
+#endif
+
+#ifndef INT16_MAX
+# define INT16_MAX              (32767)
+#endif
+
+#ifndef INT32_MIN
+# define INT32_MIN              (-2147483647-1)
+#endif
+
+#ifndef INT32_MAX
+# define INT32_MAX              (2147483647)
+#endif
+
+#ifndef UINT32_MIN
+# define UINT32_MIN             (0)
+#endif
+
+#ifndef UINT32_MAX
+# define UINT32_MAX             (4294967295U)
+#endif
+
+#ifndef M_PI
+# define M_PI			3.14159265358979323846
+#endif
+
+#ifdef _MSC_VER
+/* 'inline' is available only in C++ in MSVC */
+#   define inline __inline
+#   define force_inline __forceinline
+#elif defined __GNUC__ || (defined(__SUNPRO_C) && (__SUNPRO_C >= 0x590))
+#   define inline __inline__
+#   define force_inline __inline__ __attribute__ ((__always_inline__))
+#else
+#   ifndef force_inline
+#      define force_inline inline
+#   endif
+#endif
+
+/* GCC visibility */
+#if defined(__GNUC__) && __GNUC__ >= 4
+#   define PIXMAN_EXPORT __attribute__ ((visibility("default")))
+/* Sun Studio 8 visibility */
+#elif defined(__SUNPRO_C) && (__SUNPRO_C >= 0x550)
+#   define PIXMAN_EXPORT __global
+#else
+#   define PIXMAN_EXPORT
+#endif
+
+/* TLS */
+#if defined(TOOLCHAIN_SUPPORTS__THREAD)
+
+#   define PIXMAN_DEFINE_THREAD_LOCAL(type, name)			\
+    static __thread type name
+#   define PIXMAN_GET_THREAD_LOCAL(name)				\
+    (&name)
+
+#elif defined(__MINGW32__) && !defined(__WIN64)
+
+/* We can't include <windows.h> as it causes carious clashes with
+ * identifiers in pixman, sigh. So just declare the functions we need
+ * here.
+ */
+extern __stdcall long InterlockedCompareExchange(long volatile *, long, long);
+#define InterlockedCompareExchangePointer(d,e,c)			\
+    (void *)InterlockedCompareExchange((long volatile *)(d),(long)(e),(long)(c))
+extern __stdcall int TlsAlloc (void);
+extern __stdcall void *TlsGetValue (unsigned);
+extern __stdcall int TlsSetValue (unsigned, void *);
+extern __stdcall void *CreateMutexA(void *, int, char *);
+extern __stdcall int CloseHandle(void *);
+extern __stdcall unsigned WaitForSingleObject (void *, unsigned);
+extern __stdcall int ReleaseMutex (void *);
+
+#   define PIXMAN_DEFINE_THREAD_LOCAL(type, name)			\
+    static volatile int tls_ ## name ## _initialized = 0;		\
+    static void *tls_ ## name ## _mutex = NULL;				\
+    static unsigned tls_ ## name ## _index;				\
+									\
+    static type *							\
+    tls_ ## name ## _alloc (void)					\
+    {									\
+        type *value = calloc (1, sizeof (type));			\
+        if (value)							\
+            TlsSetValue (tls_ ## name ## _index, value);		\
+        return value;							\
+    }									\
+									\
+    static force_inline type *						\
+    tls_ ## name ## _get (void)						\
+    {									\
+	type *value;							\
+	if (!tls_ ## name ## _initialized)				\
+	{								\
+	    if (!tls_ ## name ## _mutex)				\
+	    {								\
+		void *mutex = CreateMutexA (NULL, 0, NULL);		\
+		if (InterlockedCompareExchangePointer (			\
+			&tls_ ## name ## _mutex, mutex, NULL) != NULL)	\
+		{							\
+		    CloseHandle (mutex);				\
+		}							\
+	    }								\
+	    WaitForSingleObject (tls_ ## name ## _mutex, 0xFFFFFFFF);	\
+	    if (!tls_ ## name ## _initialized)				\
+	    {								\
+		tls_ ## name ## _index = TlsAlloc ();			\
+		tls_ ## name ## _initialized = 1;			\
+	    }								\
+	    ReleaseMutex (tls_ ## name ## _mutex);			\
+	}								\
+	if (tls_ ## name ## _index == 0xFFFFFFFF)			\
+	    return NULL;						\
+	value = TlsGetValue (tls_ ## name ## _index);			\
+	if (!value)							\
+	    value = tls_ ## name ## _alloc ();				\
+	return value;							\
+    }
+
+#   define PIXMAN_GET_THREAD_LOCAL(name)				\
+    tls_ ## name ## _get ()
+
+#elif defined(_MSC_VER)
+
+#   define PIXMAN_DEFINE_THREAD_LOCAL(type, name)			\
+    static __declspec(thread) type name
+#   define PIXMAN_GET_THREAD_LOCAL(name)				\
+    (&name)
+
+#elif defined(HAVE_PTHREAD_SETSPECIFIC)
+
+#include <pthread.h>
+
+#  define PIXMAN_DEFINE_THREAD_LOCAL(type, name)			\
+    static pthread_once_t tls_ ## name ## _once_control = PTHREAD_ONCE_INIT; \
+    static pthread_key_t tls_ ## name ## _key;				\
+									\
+    static void								\
+    tls_ ## name ## _destroy_value (void *value)			\
+    {									\
+	free (value);							\
+    }									\
+									\
+    static void								\
+    tls_ ## name ## _make_key (void)					\
+    {									\
+	pthread_key_create (&tls_ ## name ## _key,			\
+			    tls_ ## name ## _destroy_value);		\
+    }									\
+									\
+    static type *							\
+    tls_ ## name ## _alloc (void)					\
+    {									\
+	type *value = calloc (1, sizeof (type));			\
+	if (value)							\
+	    pthread_setspecific (tls_ ## name ## _key, value);		\
+	return value;							\
+    }									\
+									\
+    static force_inline type *						\
+    tls_ ## name ## _get (void)						\
+    {									\
+	type *value = NULL;						\
+	if (pthread_once (&tls_ ## name ## _once_control,		\
+			  tls_ ## name ## _make_key) == 0)		\
+	{								\
+	    value = pthread_getspecific (tls_ ## name ## _key);		\
+	    if (!value)							\
+		value = tls_ ## name ## _alloc ();			\
+	}								\
+	return value;							\
+    }
+
+#   define PIXMAN_GET_THREAD_LOCAL(name)				\
+    tls_ ## name ## _get ()
+
+#else
+
+#    error "Unknown thread local support for this system"
+
+#endif
diff --git a/pixman/pixman/pixman-conical-gradient.c b/pixman/pixman/pixman-conical-gradient.c
new file mode 100644
index 000000000..0341a8ebf
--- /dev/null
+++ b/pixman/pixman/pixman-conical-gradient.c
@@ -0,0 +1,184 @@
+/*
+ * Copyright © 2000 SuSE, Inc.
+ * Copyright © 2007 Red Hat, Inc.
+ * Copyright © 2000 Keith Packard, member of The XFree86 Project, Inc.
+ *             2005 Lars Knoll & Zack Rusin, Trolltech
+ *
+ * Permission to use, copy, modify, distribute, and sell this software and its
+ * documentation for any purpose is hereby granted without fee, provided that
+ * the above copyright notice appear in all copies and that both that
+ * copyright notice and this permission notice appear in supporting
+ * documentation, and that the name of Keith Packard not be used in
+ * advertising or publicity pertaining to distribution of the software without
+ * specific, written prior permission.  Keith Packard makes no
+ * representations about the suitability of this software for any purpose.  It
+ * is provided "as is" without express or implied warranty.
+ *
+ * THE COPYRIGHT HOLDERS DISCLAIM ALL WARRANTIES WITH REGARD TO THIS
+ * SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
+ * FITNESS, IN NO EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
+ * AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
+ * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS
+ * SOFTWARE.
+ */
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <stdlib.h>
+#include <math.h>
+#include "pixman-private.h"
+
+static void
+conical_gradient_get_scanline_32 (pixman_image_t *image,
+                                  int             x,
+                                  int             y,
+                                  int             width,
+                                  uint32_t *      buffer,
+                                  const uint32_t *mask,
+                                  uint32_t        mask_bits)
+{
+    source_image_t *source = (source_image_t *)image;
+    gradient_t *gradient = (gradient_t *)source;
+    conical_gradient_t *conical = (conical_gradient_t *)image;
+    uint32_t       *end = buffer + width;
+    pixman_gradient_walker_t walker;
+    pixman_bool_t affine = TRUE;
+    double cx = 1.;
+    double cy = 0.;
+    double cz = 0.;
+    double rx = x + 0.5;
+    double ry = y + 0.5;
+    double rz = 1.;
+    double a = (conical->angle * M_PI) / (180. * 65536);
+
+    _pixman_gradient_walker_init (&walker, gradient, source->common.repeat);
+
+    if (source->common.transform)
+    {
+	pixman_vector_t v;
+
+	/* reference point is the center of the pixel */
+	v.vector[0] = pixman_int_to_fixed (x) + pixman_fixed_1 / 2;
+	v.vector[1] = pixman_int_to_fixed (y) + pixman_fixed_1 / 2;
+	v.vector[2] = pixman_fixed_1;
+
+	if (!pixman_transform_point_3d (source->common.transform, &v))
+	    return;
+
+	cx = source->common.transform->matrix[0][0] / 65536.;
+	cy = source->common.transform->matrix[1][0] / 65536.;
+	cz = source->common.transform->matrix[2][0] / 65536.;
+	
+	rx = v.vector[0] / 65536.;
+	ry = v.vector[1] / 65536.;
+	rz = v.vector[2] / 65536.;
+	
+	affine =
+	    source->common.transform->matrix[2][0] == 0 &&
+	    v.vector[2] == pixman_fixed_1;
+    }
+
+    if (affine)
+    {
+	rx -= conical->center.x / 65536.;
+	ry -= conical->center.y / 65536.;
+
+	while (buffer < end)
+	{
+	    double angle;
+
+	    if (!mask || *mask++ & mask_bits)
+	    {
+		pixman_fixed_48_16_t t;
+
+		angle = atan2 (ry, rx) + a;
+		t     = (pixman_fixed_48_16_t) (angle * (65536. / (2 * M_PI)));
+
+		*buffer = _pixman_gradient_walker_pixel (&walker, t);
+	    }
+
+	    ++buffer;
+	    
+	    rx += cx;
+	    ry += cy;
+	}
+    }
+    else
+    {
+	while (buffer < end)
+	{
+	    double x, y;
+	    double angle;
+
+	    if (!mask || *mask++ & mask_bits)
+	    {
+		pixman_fixed_48_16_t t;
+
+		if (rz != 0)
+		{
+		    x = rx / rz;
+		    y = ry / rz;
+		}
+		else
+		{
+		    x = y = 0.;
+		}
+
+		x -= conical->center.x / 65536.;
+		y -= conical->center.y / 65536.;
+		
+		angle = atan2 (y, x) + a;
+		t     = (pixman_fixed_48_16_t) (angle * (65536. / (2 * M_PI)));
+
+		*buffer = _pixman_gradient_walker_pixel (&walker, t);
+	    }
+
+	    ++buffer;
+	    
+	    rx += cx;
+	    ry += cy;
+	    rz += cz;
+	}
+    }
+}
+
+static void
+conical_gradient_property_changed (pixman_image_t *image)
+{
+    image->common.get_scanline_32 = conical_gradient_get_scanline_32;
+    image->common.get_scanline_64 = _pixman_image_get_scanline_generic_64;
+}
+
+PIXMAN_EXPORT pixman_image_t *
+pixman_image_create_conical_gradient (pixman_point_fixed_t *        center,
+                                      pixman_fixed_t                angle,
+                                      const pixman_gradient_stop_t *stops,
+                                      int                           n_stops)
+{
+    pixman_image_t *image = _pixman_image_allocate ();
+    conical_gradient_t *conical;
+
+    if (!image)
+	return NULL;
+
+    conical = &image->conical;
+
+    if (!_pixman_init_gradient (&conical->common, stops, n_stops))
+    {
+	free (image);
+	return NULL;
+    }
+
+    image->type = CONICAL;
+    conical->center = *center;
+    conical->angle = angle;
+
+    image->common.property_changed = conical_gradient_property_changed;
+
+    return image;
+}
+
diff --git a/pixman/pixman/pixman-cpu.c b/pixman/pixman/pixman-cpu.c
new file mode 100644
index 000000000..e96b140bd
--- /dev/null
+++ b/pixman/pixman/pixman-cpu.c
@@ -0,0 +1,575 @@
+/*
+ * Copyright © 2000 SuSE, Inc.
+ * Copyright © 2007 Red Hat, Inc.
+ *
+ * Permission to use, copy, modify, distribute, and sell this software and its
+ * documentation for any purpose is hereby granted without fee, provided that
+ * the above copyright notice appear in all copies and that both that
+ * copyright notice and this permission notice appear in supporting
+ * documentation, and that the name of SuSE not be used in advertising or
+ * publicity pertaining to distribution of the software without specific,
+ * written prior permission.  SuSE makes no representations about the
+ * suitability of this software for any purpose.  It is provided "as is"
+ * without express or implied warranty.
+ *
+ * SuSE DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING ALL
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL SuSE
+ * BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
+ * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+ * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <string.h>
+
+#if defined(USE_ARM_SIMD) && defined(_MSC_VER)
+/* Needed for EXCEPTION_ILLEGAL_INSTRUCTION */
+#include <windows.h>
+#endif
+
+#include "pixman-private.h"
+
+#ifdef USE_VMX
+
+/* The CPU detection code needs to be in a file not compiled with
+ * "-maltivec -mabi=altivec", as gcc would try to save vector register
+ * across function calls causing SIGILL on cpus without Altivec/vmx.
+ */
+static pixman_bool_t initialized = FALSE;
+static volatile pixman_bool_t have_vmx = TRUE;
+
+#ifdef __APPLE__
+#include <sys/sysctl.h>
+
+static pixman_bool_t
+pixman_have_vmx (void)
+{
+    if (!initialized)
+    {
+	size_t length = sizeof(have_vmx);
+	int error =
+	    sysctlbyname ("hw.optional.altivec", &have_vmx, &length, NULL, 0);
+
+	if (error)
+	    have_vmx = FALSE;
+
+	initialized = TRUE;
+    }
+    return have_vmx;
+}
+
+#elif defined (__linux__)
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <stdio.h>
+#include <linux/auxvec.h>
+#include <asm/cputable.h>
+
+static pixman_bool_t
+pixman_have_vmx (void)
+{
+    if (!initialized)
+    {
+	char fname[64];
+	unsigned long buf[64];
+	ssize_t count = 0;
+	pid_t pid;
+	int fd, i;
+
+	pid = getpid ();
+	snprintf (fname, sizeof(fname) - 1, "/proc/%d/auxv", pid);
+
+	fd = open (fname, O_RDONLY);
+	if (fd >= 0)
+	{
+	    for (i = 0; i <= (count / sizeof(unsigned long)); i += 2)
+	    {
+		/* Read more if buf is empty... */
+		if (i == (count / sizeof(unsigned long)))
+		{
+		    count = read (fd, buf, sizeof(buf));
+		    if (count <= 0)
+			break;
+		    i = 0;
+		}
+
+		if (buf[i] == AT_HWCAP)
+		{
+		    have_vmx = !!(buf[i + 1] & PPC_FEATURE_HAS_ALTIVEC);
+		    initialized = TRUE;
+		    break;
+		}
+		else if (buf[i] == AT_NULL)
+		{
+		    break;
+		}
+	    }
+	    close (fd);
+	}
+    }
+    if (!initialized)
+    {
+	/* Something went wrong. Assume 'no' rather than playing
+	   fragile tricks with catching SIGILL. */
+	have_vmx = FALSE;
+	initialized = TRUE;
+    }
+
+    return have_vmx;
+}
+
+#else /* !__APPLE__ && !__linux__ */
+#include <signal.h>
+#include <setjmp.h>
+
+static jmp_buf jump_env;
+
+static void
+vmx_test (int        sig,
+	  siginfo_t *si,
+	  void *     unused)
+{
+    longjmp (jump_env, 1);
+}
+
+static pixman_bool_t
+pixman_have_vmx (void)
+{
+    struct sigaction sa, osa;
+    int jmp_result;
+
+    if (!initialized)
+    {
+	sa.sa_flags = SA_SIGINFO;
+	sigemptyset (&sa.sa_mask);
+	sa.sa_sigaction = vmx_test;
+	sigaction (SIGILL, &sa, &osa);
+	jmp_result = setjmp (jump_env);
+	if (jmp_result == 0)
+	{
+	    asm volatile ( "vor 0, 0, 0" );
+	}
+	sigaction (SIGILL, &osa, NULL);
+	have_vmx = (jmp_result == 0);
+	initialized = TRUE;
+    }
+    return have_vmx;
+}
+
+#endif /* __APPLE__ */
+#endif /* USE_VMX */
+
+#if defined(USE_ARM_SIMD) || defined(USE_ARM_NEON)
+
+#if defined(_MSC_VER)
+
+#if defined(USE_ARM_SIMD)
+extern int pixman_msvc_try_arm_simd_op ();
+
+pixman_bool_t
+pixman_have_arm_simd (void)
+{
+    static pixman_bool_t initialized = FALSE;
+    static pixman_bool_t have_arm_simd = FALSE;
+
+    if (!initialized)
+    {
+	__try {
+	    pixman_msvc_try_arm_simd_op ();
+	    have_arm_simd = TRUE;
+	} __except (GetExceptionCode () == EXCEPTION_ILLEGAL_INSTRUCTION) {
+	    have_arm_simd = FALSE;
+	}
+	initialized = TRUE;
+    }
+
+    return have_arm_simd;
+}
+
+#endif /* USE_ARM_SIMD */
+
+#if defined(USE_ARM_NEON)
+extern int pixman_msvc_try_arm_neon_op ();
+
+pixman_bool_t
+pixman_have_arm_neon (void)
+{
+    static pixman_bool_t initialized = FALSE;
+    static pixman_bool_t have_arm_neon = FALSE;
+
+    if (!initialized)
+    {
+	__try
+	{
+	    pixman_msvc_try_arm_neon_op ();
+	    have_arm_neon = TRUE;
+	}
+	__except (GetExceptionCode () == EXCEPTION_ILLEGAL_INSTRUCTION)
+	{
+	    have_arm_neon = FALSE;
+	}
+	initialized = TRUE;
+    }
+
+    return have_arm_neon;
+}
+
+#endif /* USE_ARM_NEON */
+
+#else /* linux ELF */
+
+#include <stdlib.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/mman.h>
+#include <fcntl.h>
+#include <string.h>
+#include <elf.h>
+
+static pixman_bool_t arm_has_v7 = FALSE;
+static pixman_bool_t arm_has_v6 = FALSE;
+static pixman_bool_t arm_has_vfp = FALSE;
+static pixman_bool_t arm_has_neon = FALSE;
+static pixman_bool_t arm_has_iwmmxt = FALSE;
+static pixman_bool_t arm_tests_initialized = FALSE;
+
+static void
+pixman_arm_read_auxv ()
+{
+    int fd;
+    Elf32_auxv_t aux;
+
+    fd = open ("/proc/self/auxv", O_RDONLY);
+    if (fd >= 0)
+    {
+	while (read (fd, &aux, sizeof(Elf32_auxv_t)) == sizeof(Elf32_auxv_t))
+	{
+	    if (aux.a_type == AT_HWCAP)
+	    {
+		uint32_t hwcap = aux.a_un.a_val;
+		/* hardcode these values to avoid depending on specific
+		 * versions of the hwcap header, e.g. HWCAP_NEON
+		 */
+		arm_has_vfp = (hwcap & 64) != 0;
+		arm_has_iwmmxt = (hwcap & 512) != 0;
+		/* this flag is only present on kernel 2.6.29 */
+		arm_has_neon = (hwcap & 4096) != 0;
+	    }
+	    else if (aux.a_type == AT_PLATFORM)
+	    {
+		const char *plat = (const char*) aux.a_un.a_val;
+		if (strncmp (plat, "v7l", 3) == 0)
+		{
+		    arm_has_v7 = TRUE;
+		    arm_has_v6 = TRUE;
+		}
+		else if (strncmp (plat, "v6l", 3) == 0)
+		{
+		    arm_has_v6 = TRUE;
+		}
+	    }
+	}
+	close (fd);
+    }
+
+    arm_tests_initialized = TRUE;
+}
+
+#if defined(USE_ARM_SIMD)
+pixman_bool_t
+pixman_have_arm_simd (void)
+{
+    if (!arm_tests_initialized)
+	pixman_arm_read_auxv ();
+
+    return arm_has_v6;
+}
+
+#endif /* USE_ARM_SIMD */
+
+#if defined(USE_ARM_NEON)
+pixman_bool_t
+pixman_have_arm_neon (void)
+{
+    if (!arm_tests_initialized)
+	pixman_arm_read_auxv ();
+
+    return arm_has_neon;
+}
+
+#endif /* USE_ARM_NEON */
+
+#endif /* linux */
+
+#endif /* USE_ARM_SIMD || USE_ARM_NEON */
+
+#if defined(USE_MMX) || defined(USE_SSE2)
+/* The CPU detection code needs to be in a file not compiled with
+ * "-mmmx -msse", as gcc would generate CMOV instructions otherwise
+ * that would lead to SIGILL instructions on old CPUs that don't have
+ * it.
+ */
+#if !defined(__amd64__) && !defined(__x86_64__) && !defined(_M_AMD64)
+
+#ifdef HAVE_GETISAX
+#include <sys/auxv.h>
+#endif
+
+typedef enum
+{
+    NO_FEATURES = 0,
+    MMX = 0x1,
+    MMX_EXTENSIONS = 0x2,
+    SSE = 0x6,
+    SSE2 = 0x8,
+    CMOV = 0x10
+} cpu_features_t;
+
+
+static unsigned int
+detect_cpu_features (void)
+{
+    unsigned int features = 0;
+    unsigned int result = 0;
+
+#ifdef HAVE_GETISAX
+    if (getisax (&result, 1))
+    {
+	if (result & AV_386_CMOV)
+	    features |= CMOV;
+	if (result & AV_386_MMX)
+	    features |= MMX;
+	if (result & AV_386_AMD_MMX)
+	    features |= MMX_EXTENSIONS;
+	if (result & AV_386_SSE)
+	    features |= SSE;
+	if (result & AV_386_SSE2)
+	    features |= SSE2;
+    }
+#else
+    char vendor[13];
+#ifdef _MSC_VER
+    int vendor0 = 0, vendor1, vendor2;
+#endif
+    vendor[0] = 0;
+    vendor[12] = 0;
+
+#ifdef __GNUC__
+    /* see p. 118 of amd64 instruction set manual Vol3 */
+    /* We need to be careful about the handling of %ebx and
+     * %esp here. We can't declare either one as clobbered
+     * since they are special registers (%ebx is the "PIC
+     * register" holding an offset to global data, %esp the
+     * stack pointer), so we need to make sure they have their
+     * original values when we access the output operands.
+     */
+    __asm__ (
+        "pushf\n"
+        "pop %%eax\n"
+        "mov %%eax, %%ecx\n"
+        "xor $0x00200000, %%eax\n"
+        "push %%eax\n"
+        "popf\n"
+        "pushf\n"
+        "pop %%eax\n"
+        "mov $0x0, %%edx\n"
+        "xor %%ecx, %%eax\n"
+        "jz 1f\n"
+
+        "mov $0x00000000, %%eax\n"
+        "push %%ebx\n"
+        "cpuid\n"
+        "mov %%ebx, %%eax\n"
+        "pop %%ebx\n"
+        "mov %%eax, %1\n"
+        "mov %%edx, %2\n"
+        "mov %%ecx, %3\n"
+        "mov $0x00000001, %%eax\n"
+        "push %%ebx\n"
+        "cpuid\n"
+        "pop %%ebx\n"
+        "1:\n"
+        "mov %%edx, %0\n"
+	: "=r" (result),
+        "=m" (vendor[0]),
+        "=m" (vendor[4]),
+        "=m" (vendor[8])
+	:
+	: "%eax", "%ecx", "%edx"
+        );
+
+#elif defined (_MSC_VER)
+
+    _asm {
+	pushfd
+	pop eax
+	mov ecx, eax
+	xor eax, 00200000h
+	push eax
+	popfd
+	pushfd
+	pop eax
+	mov edx, 0
+	xor eax, ecx
+	jz nocpuid
+
+	mov eax, 0
+	push ebx
+	cpuid
+	mov eax, ebx
+	pop ebx
+	mov vendor0, eax
+	mov vendor1, edx
+	mov vendor2, ecx
+	mov eax, 1
+	push ebx
+	cpuid
+	pop ebx
+    nocpuid:
+	mov result, edx
+    }
+    memmove (vendor + 0, &vendor0, 4);
+    memmove (vendor + 4, &vendor1, 4);
+    memmove (vendor + 8, &vendor2, 4);
+
+#else
+#   error unsupported compiler
+#endif
+
+    features = 0;
+    if (result)
+    {
+	/* result now contains the standard feature bits */
+	if (result & (1 << 15))
+	    features |= CMOV;
+	if (result & (1 << 23))
+	    features |= MMX;
+	if (result & (1 << 25))
+	    features |= SSE;
+	if (result & (1 << 26))
+	    features |= SSE2;
+	if ((features & MMX) && !(features & SSE) &&
+	    (strcmp (vendor, "AuthenticAMD") == 0 ||
+	     strcmp (vendor, "Geode by NSC") == 0))
+	{
+	    /* check for AMD MMX extensions */
+#ifdef __GNUC__
+	    __asm__ (
+	        "	push %%ebx\n"
+	        "	mov $0x80000000, %%eax\n"
+	        "	cpuid\n"
+	        "	xor %%edx, %%edx\n"
+	        "	cmp $0x1, %%eax\n"
+	        "	jge 2f\n"
+	        "	mov $0x80000001, %%eax\n"
+	        "	cpuid\n"
+	        "2:\n"
+	        "	pop %%ebx\n"
+	        "	mov %%edx, %0\n"
+		: "=r" (result)
+		:
+		: "%eax", "%ecx", "%edx"
+	        );
+#elif defined _MSC_VER
+	    _asm {
+		push ebx
+		mov eax, 80000000h
+		cpuid
+		xor edx, edx
+		cmp eax, 1
+		jge notamd
+		mov eax, 80000001h
+		cpuid
+	    notamd:
+		pop ebx
+		mov result, edx
+	    }
+#endif
+	    if (result & (1 << 22))
+		features |= MMX_EXTENSIONS;
+	}
+    }
+#endif /* HAVE_GETISAX */
+
+    return features;
+}
+
+static pixman_bool_t
+pixman_have_mmx (void)
+{
+    static pixman_bool_t initialized = FALSE;
+    static pixman_bool_t mmx_present;
+
+    if (!initialized)
+    {
+	unsigned int features = detect_cpu_features ();
+	mmx_present = (features & (MMX | MMX_EXTENSIONS)) == (MMX | MMX_EXTENSIONS);
+	initialized = TRUE;
+    }
+
+    return mmx_present;
+}
+
+#ifdef USE_SSE2
+static pixman_bool_t
+pixman_have_sse2 (void)
+{
+    static pixman_bool_t initialized = FALSE;
+    static pixman_bool_t sse2_present;
+
+    if (!initialized)
+    {
+	unsigned int features = detect_cpu_features ();
+	sse2_present = (features & (MMX | MMX_EXTENSIONS | SSE | SSE2)) == (MMX | MMX_EXTENSIONS | SSE | SSE2);
+	initialized = TRUE;
+    }
+
+    return sse2_present;
+}
+
+#endif
+
+#else /* __amd64__ */
+#ifdef USE_MMX
+#define pixman_have_mmx() TRUE
+#endif
+#ifdef USE_SSE2
+#define pixman_have_sse2() TRUE
+#endif
+#endif /* __amd64__ */
+#endif
+
+pixman_implementation_t *
+_pixman_choose_implementation (void)
+{
+#ifdef USE_SSE2
+    if (pixman_have_sse2 ())
+	return _pixman_implementation_create_sse2 ();
+#endif
+#ifdef USE_MMX
+    if (pixman_have_mmx ())
+	return _pixman_implementation_create_mmx ();
+#endif
+
+#ifdef USE_ARM_NEON
+    if (pixman_have_arm_neon ())
+	return _pixman_implementation_create_arm_neon ();
+#endif
+#ifdef USE_ARM_SIMD
+    if (pixman_have_arm_simd ())
+	return _pixman_implementation_create_arm_simd ();
+#endif
+#ifdef USE_VMX
+    if (pixman_have_vmx ())
+	return _pixman_implementation_create_vmx ();
+#endif
+
+    return _pixman_implementation_create_fast_path ();
+}
+
diff --git a/pixman/pixman/pixman-edge-accessors.c b/pixman/pixman/pixman-edge-accessors.c
new file mode 100644
index 000000000..ea3a31e2f
--- /dev/null
+++ b/pixman/pixman/pixman-edge-accessors.c
@@ -0,0 +1,4 @@
+
+#define PIXMAN_FB_ACCESSORS
+
+#include "pixman-edge.c"
diff --git a/pixman/pixman/pixman-edge-imp.h b/pixman/pixman/pixman-edge-imp.h
new file mode 100644
index 000000000..a4698eddb
--- /dev/null
+++ b/pixman/pixman/pixman-edge-imp.h
@@ -0,0 +1,182 @@
+/*
+ * Copyright © 2004 Keith Packard
+ *
+ * Permission to use, copy, modify, distribute, and sell this software and its
+ * documentation for any purpose is hereby granted without fee, provided that
+ * the above copyright notice appear in all copies and that both that
+ * copyright notice and this permission notice appear in supporting
+ * documentation, and that the name of Keith Packard not be used in
+ * advertising or publicity pertaining to distribution of the software without
+ * specific, written prior permission.  Keith Packard makes no
+ * representations about the suitability of this software for any purpose.  It
+ * is provided "as is" without express or implied warranty.
+ *
+ * KEITH PACKARD DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
+ * INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO
+ * EVENT SHALL KEITH PACKARD BE LIABLE FOR ANY SPECIAL, INDIRECT OR
+ * CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE,
+ * DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
+ * TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+ * PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#ifndef rasterize_span
+#endif
+
+static void
+RASTERIZE_EDGES (pixman_image_t  *image,
+		pixman_edge_t	*l,
+		pixman_edge_t	*r,
+		pixman_fixed_t		t,
+		pixman_fixed_t		b)
+{
+    pixman_fixed_t  y = t;
+    uint32_t  *line;
+    uint32_t *buf = (image)->bits.bits;
+    int stride = (image)->bits.rowstride;
+    int width = (image)->bits.width;
+
+    line = buf + pixman_fixed_to_int (y) * stride;
+
+    for (;;)
+    {
+	pixman_fixed_t	lx;
+	pixman_fixed_t      rx;
+	int	lxi;
+	int rxi;
+
+	lx = l->x;
+	rx = r->x;
+#if N_BITS == 1
+	/* For the non-antialiased case, round the coordinates up, in effect
+	 * sampling just slightly to the left of the pixel. This is so that
+	 * when the sample point lies exactly on the line, we round towards
+	 * north-west.
+	 *
+	 * (The AA case does a similar  adjustment in RENDER_SAMPLES_X)
+	 */
+	lx += X_FRAC_FIRST(1) - pixman_fixed_e;
+	rx += X_FRAC_FIRST(1) - pixman_fixed_e;
+#endif
+	/* clip X */
+	if (lx < 0)
+	    lx = 0;
+	if (pixman_fixed_to_int (rx) >= width)
+#if N_BITS == 1
+	    rx = pixman_int_to_fixed (width);
+#else
+	    /* Use the last pixel of the scanline, covered 100%.
+	     * We can't use the first pixel following the scanline,
+	     * because accessing it could result in a buffer overrun.
+	     */
+	    rx = pixman_int_to_fixed (width) - 1;
+#endif
+
+	/* Skip empty (or backwards) sections */
+	if (rx > lx)
+	{
+
+	    /* Find pixel bounds for span */
+	    lxi = pixman_fixed_to_int (lx);
+	    rxi = pixman_fixed_to_int (rx);
+
+#if N_BITS == 1
+	    {
+
+#define LEFT_MASK(x)							\
+		(((x) & 0x1f) ?						\
+		 SCREEN_SHIFT_RIGHT (0xffffffff, (x) & 0x1f) : 0)
+#define RIGHT_MASK(x)							\
+		(((32 - (x)) & 0x1f) ?					\
+		 SCREEN_SHIFT_LEFT (0xffffffff, (32 - (x)) & 0x1f) : 0)
+		
+#define MASK_BITS(x,w,l,n,r) {						\
+		    n = (w);						\
+		    r = RIGHT_MASK ((x) + n);				\
+		    l = LEFT_MASK (x);					\
+		    if (l) {						\
+			n -= 32 - ((x) & 0x1f);				\
+			if (n < 0) {					\
+			    n = 0;					\
+			    l &= r;					\
+			    r = 0;					\
+			}						\
+		    }							\
+		    n >>= 5;						\
+		}
+		
+		uint32_t  *a = line;
+		uint32_t  startmask;
+		uint32_t  endmask;
+		int	    nmiddle;
+		int	    width = rxi - lxi;
+		int	    x = lxi;
+		
+		a += x >> 5;
+		x &= 0x1f;
+		
+		MASK_BITS (x, width, startmask, nmiddle, endmask);
+
+		if (startmask) {
+		    WRITE(image, a, READ(image, a) | startmask);
+		    a++;
+		}
+		while (nmiddle--)
+		    WRITE(image, a++, 0xffffffff);
+		if (endmask)
+		    WRITE(image, a, READ(image, a) | endmask);
+	    }
+#else
+	    {
+		DEFINE_ALPHA(line,lxi);
+		int	    lxs;
+		int     rxs;
+
+		/* Sample coverage for edge pixels */
+		lxs = RENDER_SAMPLES_X (lx, N_BITS);
+		rxs = RENDER_SAMPLES_X (rx, N_BITS);
+
+		/* Add coverage across row */
+		if (lxi == rxi)
+		{
+		    ADD_ALPHA (rxs - lxs);
+		}
+		else
+		{
+		    int	xi;
+
+		    ADD_ALPHA (N_X_FRAC(N_BITS) - lxs);
+		    STEP_ALPHA;
+		    for (xi = lxi + 1; xi < rxi; xi++)
+		    {
+			ADD_ALPHA (N_X_FRAC(N_BITS));
+			STEP_ALPHA;
+		    }
+		    ADD_ALPHA (rxs);
+		}
+	    }
+#endif
+	}
+
+	if (y == b)
+	    break;
+
+#if N_BITS > 1
+	if (pixman_fixed_frac (y) != Y_FRAC_LAST(N_BITS))
+	{
+	    RENDER_EDGE_STEP_SMALL (l);
+	    RENDER_EDGE_STEP_SMALL (r);
+	    y += STEP_Y_SMALL(N_BITS);
+	}
+	else
+#endif
+	{
+	    RENDER_EDGE_STEP_BIG (l);
+	    RENDER_EDGE_STEP_BIG (r);
+	    y += STEP_Y_BIG(N_BITS);
+	    line += stride;
+	}
+    }
+}
+
+#undef rasterize_span
diff --git a/pixman/pixman/pixman-edge.c b/pixman/pixman/pixman-edge.c
new file mode 100644
index 000000000..8d498ab44
--- /dev/null
+++ b/pixman/pixman/pixman-edge.c
@@ -0,0 +1,384 @@
+/*
+ * Copyright © 2004 Keith Packard
+ *
+ * Permission to use, copy, modify, distribute, and sell this software and its
+ * documentation for any purpose is hereby granted without fee, provided that
+ * the above copyright notice appear in all copies and that both that
+ * copyright notice and this permission notice appear in supporting
+ * documentation, and that the name of Keith Packard not be used in
+ * advertising or publicity pertaining to distribution of the software without
+ * specific, written prior permission.  Keith Packard makes no
+ * representations about the suitability of this software for any purpose.  It
+ * is provided "as is" without express or implied warranty.
+ *
+ * KEITH PACKARD DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
+ * INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO
+ * EVENT SHALL KEITH PACKARD BE LIABLE FOR ANY SPECIAL, INDIRECT OR
+ * CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE,
+ * DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
+ * TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+ * PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <string.h>
+
+#include "pixman-private.h"
+#include "pixman-accessor.h"
+
+/*
+ * Step across a small sample grid gap
+ */
+#define RENDER_EDGE_STEP_SMALL(edge)					\
+    {									\
+	edge->x += edge->stepx_small;					\
+	edge->e += edge->dx_small;					\
+	if (edge->e > 0)						\
+	{								\
+	    edge->e -= edge->dy;					\
+	    edge->x += edge->signdx;					\
+	}								\
+    }
+
+/*
+ * Step across a large sample grid gap
+ */
+#define RENDER_EDGE_STEP_BIG(edge)					\
+    {									\
+	edge->x += edge->stepx_big;					\
+	edge->e += edge->dx_big;					\
+	if (edge->e > 0)						\
+	{								\
+	    edge->e -= edge->dy;					\
+	    edge->x += edge->signdx;					\
+	}								\
+    }
+
+#ifdef PIXMAN_FB_ACCESSORS
+#define PIXMAN_RASTERIZE_EDGES pixman_rasterize_edges_accessors
+#else
+#define PIXMAN_RASTERIZE_EDGES pixman_rasterize_edges_no_accessors
+#endif
+
+/*
+ * 4 bit alpha
+ */
+
+#define N_BITS  4
+#define RASTERIZE_EDGES rasterize_edges_4
+
+#ifndef WORDS_BIGENDIAN
+#define SHIFT_4(o)      ((o) << 2)
+#else
+#define SHIFT_4(o)      ((1 - (o)) << 2)
+#endif
+
+#define GET_4(x, o)      (((x) >> SHIFT_4 (o)) & 0xf)
+#define PUT_4(x, o, v)							\
+    (((x) & ~(0xf << SHIFT_4 (o))) | (((v) & 0xf) << SHIFT_4 (o)))
+
+#define DEFINE_ALPHA(line, x)						\
+    uint8_t   *__ap = (uint8_t *) line + ((x) >> 1);			\
+    int __ao = (x) & 1
+
+#define STEP_ALPHA      ((__ap += __ao), (__ao ^= 1))
+
+#define ADD_ALPHA(a)							\
+    {									\
+        uint8_t __o = READ (image, __ap);				\
+        uint8_t __a = (a) + GET_4 (__o, __ao);				\
+        WRITE (image, __ap, PUT_4 (__o, __ao, __a | (0 - ((__a) >> 4)))); \
+    }
+
+#include "pixman-edge-imp.h"
+
+#undef ADD_ALPHA
+#undef STEP_ALPHA
+#undef DEFINE_ALPHA
+#undef RASTERIZE_EDGES
+#undef N_BITS
+
+
+/*
+ * 1 bit alpha
+ */
+
+#define N_BITS 1
+#define RASTERIZE_EDGES rasterize_edges_1
+
+#include "pixman-edge-imp.h"
+
+#undef RASTERIZE_EDGES
+#undef N_BITS
+
+/*
+ * 8 bit alpha
+ */
+
+static force_inline uint8_t
+clip255 (int x)
+{
+    if (x > 255)
+	return 255;
+
+    return x;
+}
+
+#define ADD_SATURATE_8(buf, val, length)				\
+    do									\
+    {									\
+        int i__ = (length);						\
+        uint8_t *buf__ = (buf);						\
+        int val__ = (val);						\
+									\
+        while (i__--)							\
+        {								\
+            WRITE (image, (buf__), clip255 (READ (image, (buf__)) + (val__))); \
+            (buf__)++;							\
+	}								\
+    } while (0)
+
+/*
+ * We want to detect the case where we add the same value to a long
+ * span of pixels.  The triangles on the end are filled in while we
+ * count how many sub-pixel scanlines contribute to the middle section.
+ *
+ *                 +--------------------------+
+ *  fill_height =|   \                      /
+ *                     +------------------+
+ *                      |================|
+ *                   fill_start       fill_end
+ */
+static void
+rasterize_edges_8 (pixman_image_t *image,
+                   pixman_edge_t * l,
+                   pixman_edge_t * r,
+                   pixman_fixed_t  t,
+                   pixman_fixed_t  b)
+{
+    pixman_fixed_t y = t;
+    uint32_t  *line;
+    int fill_start = -1, fill_end = -1;
+    int fill_size = 0;
+    uint32_t *buf = (image)->bits.bits;
+    int stride = (image)->bits.rowstride;
+    int width = (image)->bits.width;
+
+    line = buf + pixman_fixed_to_int (y) * stride;
+
+    for (;;)
+    {
+        uint8_t *ap = (uint8_t *) line;
+        pixman_fixed_t lx, rx;
+        int lxi, rxi;
+
+        /* clip X */
+        lx = l->x;
+        if (lx < 0)
+	    lx = 0;
+
+        rx = r->x;
+
+        if (pixman_fixed_to_int (rx) >= width)
+	{
+	    /* Use the last pixel of the scanline, covered 100%.
+	     * We can't use the first pixel following the scanline,
+	     * because accessing it could result in a buffer overrun.
+	     */
+	    rx = pixman_int_to_fixed (width) - 1;
+	}
+
+        /* Skip empty (or backwards) sections */
+        if (rx > lx)
+        {
+            int lxs, rxs;
+
+            /* Find pixel bounds for span. */
+            lxi = pixman_fixed_to_int (lx);
+            rxi = pixman_fixed_to_int (rx);
+
+            /* Sample coverage for edge pixels */
+            lxs = RENDER_SAMPLES_X (lx, 8);
+            rxs = RENDER_SAMPLES_X (rx, 8);
+
+            /* Add coverage across row */
+            if (lxi == rxi)
+            {
+                WRITE (image, ap + lxi,
+		       clip255 (READ (image, ap + lxi) + rxs - lxs));
+	    }
+            else
+            {
+                WRITE (image, ap + lxi,
+		       clip255 (READ (image, ap + lxi) + N_X_FRAC (8) - lxs));
+
+                /* Move forward so that lxi/rxi is the pixel span */
+                lxi++;
+
+                /* Don't bother trying to optimize the fill unless
+		 * the span is longer than 4 pixels. */
+                if (rxi - lxi > 4)
+                {
+                    if (fill_start < 0)
+                    {
+                        fill_start = lxi;
+                        fill_end = rxi;
+                        fill_size++;
+		    }
+                    else
+                    {
+                        if (lxi >= fill_end || rxi < fill_start)
+                        {
+                            /* We're beyond what we saved, just fill it */
+                            ADD_SATURATE_8 (ap + fill_start,
+                                            fill_size * N_X_FRAC (8),
+                                            fill_end - fill_start);
+                            fill_start = lxi;
+                            fill_end = rxi;
+                            fill_size = 1;
+			}
+                        else
+                        {
+                            /* Update fill_start */
+                            if (lxi > fill_start)
+                            {
+                                ADD_SATURATE_8 (ap + fill_start,
+                                                fill_size * N_X_FRAC (8),
+                                                lxi - fill_start);
+                                fill_start = lxi;
+			    }
+                            else if (lxi < fill_start)
+                            {
+                                ADD_SATURATE_8 (ap + lxi, N_X_FRAC (8),
+                                                fill_start - lxi);
+			    }
+
+                            /* Update fill_end */
+                            if (rxi < fill_end)
+                            {
+                                ADD_SATURATE_8 (ap + rxi,
+                                                fill_size * N_X_FRAC (8),
+                                                fill_end - rxi);
+                                fill_end = rxi;
+			    }
+                            else if (fill_end < rxi)
+                            {
+                                ADD_SATURATE_8 (ap + fill_end,
+                                                N_X_FRAC (8),
+                                                rxi - fill_end);
+			    }
+                            fill_size++;
+			}
+		    }
+		}
+                else
+                {
+                    ADD_SATURATE_8 (ap + lxi, N_X_FRAC (8), rxi - lxi);
+		}
+
+                WRITE (image, ap + rxi, clip255 (READ (image, ap + rxi) + rxs));
+	    }
+	}
+
+        if (y == b)
+        {
+            /* We're done, make sure we clean up any remaining fill. */
+            if (fill_start != fill_end)
+            {
+                if (fill_size == N_Y_FRAC (8))
+                {
+                    MEMSET_WRAPPED (image, ap + fill_start,
+				    0xff, fill_end - fill_start);
+		}
+                else
+                {
+                    ADD_SATURATE_8 (ap + fill_start, fill_size * N_X_FRAC (8),
+                                    fill_end - fill_start);
+		}
+	    }
+            break;
+	}
+
+        if (pixman_fixed_frac (y) != Y_FRAC_LAST (8))
+        {
+            RENDER_EDGE_STEP_SMALL (l);
+            RENDER_EDGE_STEP_SMALL (r);
+            y += STEP_Y_SMALL (8);
+	}
+        else
+        {
+            RENDER_EDGE_STEP_BIG (l);
+            RENDER_EDGE_STEP_BIG (r);
+            y += STEP_Y_BIG (8);
+            if (fill_start != fill_end)
+            {
+                if (fill_size == N_Y_FRAC (8))
+                {
+                    MEMSET_WRAPPED (image, ap + fill_start,
+				    0xff, fill_end - fill_start);
+		}
+                else
+                {
+                    ADD_SATURATE_8 (ap + fill_start, fill_size * N_X_FRAC (8),
+                                    fill_end - fill_start);
+		}
+		
+                fill_start = fill_end = -1;
+                fill_size = 0;
+	    }
+	    
+            line += stride;
+	}
+    }
+}
+
+#ifndef PIXMAN_FB_ACCESSORS
+static
+#endif
+void
+PIXMAN_RASTERIZE_EDGES (pixman_image_t *image,
+                        pixman_edge_t * l,
+                        pixman_edge_t * r,
+                        pixman_fixed_t  t,
+                        pixman_fixed_t  b)
+{
+    switch (PIXMAN_FORMAT_BPP (image->bits.format))
+    {
+    case 1:
+	rasterize_edges_1 (image, l, r, t, b);
+	break;
+
+    case 4:
+	rasterize_edges_4 (image, l, r, t, b);
+	break;
+
+    case 8:
+	rasterize_edges_8 (image, l, r, t, b);
+	break;
+
+    default:
+        break;
+    }
+}
+
+#ifndef PIXMAN_FB_ACCESSORS
+
+PIXMAN_EXPORT void
+pixman_rasterize_edges (pixman_image_t *image,
+                        pixman_edge_t * l,
+                        pixman_edge_t * r,
+                        pixman_fixed_t  t,
+                        pixman_fixed_t  b)
+{
+    return_if_fail (image->type == BITS);
+    
+    if (image->bits.read_func || image->bits.write_func)
+	pixman_rasterize_edges_accessors (image, l, r, t, b);
+    else
+	pixman_rasterize_edges_no_accessors (image, l, r, t, b);
+}
+
+#endif
diff --git a/pixman/pixman/pixman-fast-path.c b/pixman/pixman/pixman-fast-path.c
new file mode 100644
index 000000000..bf5b298c8
--- /dev/null
+++ b/pixman/pixman/pixman-fast-path.c
@@ -0,0 +1,2000 @@
+/* -*- Mode: c; c-basic-offset: 4; tab-width: 8; indent-tabs-mode: t; -*- */
+/*
+ * Copyright © 2000 SuSE, Inc.
+ * Copyright © 2007 Red Hat, Inc.
+ *
+ * Permission to use, copy, modify, distribute, and sell this software and its
+ * documentation for any purpose is hereby granted without fee, provided that
+ * the above copyright notice appear in all copies and that both that
+ * copyright notice and this permission notice appear in supporting
+ * documentation, and that the name of SuSE not be used in advertising or
+ * publicity pertaining to distribution of the software without specific,
+ * written prior permission.  SuSE makes no representations about the
+ * suitability of this software for any purpose.  It is provided "as is"
+ * without express or implied warranty.
+ *
+ * SuSE DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING ALL
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL SuSE
+ * BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
+ * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+ * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ *
+ * Author:  Keith Packard, SuSE, Inc.
+ */
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+#include <string.h>
+#include <stdlib.h>
+#include "pixman-private.h"
+#include "pixman-combine32.h"
+
+static force_inline uint32_t
+fetch_24 (uint8_t *a)
+{
+    if (((unsigned long)a) & 1)
+    {
+#ifdef WORDS_BIGENDIAN
+	return (*a << 16) | (*(uint16_t *)(a + 1));
+#else
+	return *a | (*(uint16_t *)(a + 1) << 8);
+#endif
+    }
+    else
+    {
+#ifdef WORDS_BIGENDIAN
+	return (*(uint16_t *)a << 8) | *(a + 2);
+#else
+	return *(uint16_t *)a | (*(a + 2) << 16);
+#endif
+    }
+}
+
+static force_inline void
+store_24 (uint8_t *a,
+          uint32_t v)
+{
+    if (((unsigned long)a) & 1)
+    {
+#ifdef WORDS_BIGENDIAN
+	*a = (uint8_t) (v >> 16);
+	*(uint16_t *)(a + 1) = (uint16_t) (v);
+#else
+	*a = (uint8_t) (v);
+	*(uint16_t *)(a + 1) = (uint16_t) (v >> 8);
+#endif
+    }
+    else
+    {
+#ifdef WORDS_BIGENDIAN
+	*(uint16_t *)a = (uint16_t)(v >> 8);
+	*(a + 2) = (uint8_t)v;
+#else
+	*(uint16_t *)a = (uint16_t)v;
+	*(a + 2) = (uint8_t)(v >> 16);
+#endif
+    }
+}
+
+static force_inline uint32_t
+over (uint32_t src,
+      uint32_t dest)
+{
+    uint32_t a = ~src >> 24;
+
+    UN8x4_MUL_UN8_ADD_UN8x4 (dest, a, src);
+
+    return dest;
+}
+
+static uint32_t
+in (uint32_t x,
+    uint8_t  y)
+{
+    uint16_t a = y;
+
+    UN8x4_MUL_UN8 (x, a);
+
+    return x;
+}
+
+/*
+ * Naming convention:
+ *
+ *  op_src_mask_dest
+ */
+static void
+fast_composite_over_x888_8_8888 (pixman_implementation_t *imp,
+                                 pixman_op_t              op,
+                                 pixman_image_t *         src_image,
+                                 pixman_image_t *         mask_image,
+                                 pixman_image_t *         dst_image,
+                                 int32_t                  src_x,
+                                 int32_t                  src_y,
+                                 int32_t                  mask_x,
+                                 int32_t                  mask_y,
+                                 int32_t                  dest_x,
+                                 int32_t                  dest_y,
+                                 int32_t                  width,
+                                 int32_t                  height)
+{
+    uint32_t    *src, *src_line;
+    uint32_t    *dst, *dst_line;
+    uint8_t     *mask, *mask_line;
+    int src_stride, mask_stride, dst_stride;
+    uint8_t m;
+    uint32_t s, d;
+    int32_t w;
+
+    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
+    PIXMAN_IMAGE_GET_LINE (mask_image, mask_x, mask_y, uint8_t, mask_stride, mask_line, 1);
+    PIXMAN_IMAGE_GET_LINE (src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
+
+    while (height--)
+    {
+	src = src_line;
+	src_line += src_stride;
+	dst = dst_line;
+	dst_line += dst_stride;
+	mask = mask_line;
+	mask_line += mask_stride;
+
+	w = width;
+	while (w--)
+	{
+	    m = *mask++;
+	    if (m)
+	    {
+		s = *src | 0xff000000;
+
+		if (m == 0xff)
+		{
+		    *dst = s;
+		}
+		else
+		{
+		    d = in (s, m);
+		    *dst = over (d, *dst);
+		}
+	    }
+	    src++;
+	    dst++;
+	}
+    }
+}
+
+static void
+fast_composite_in_n_8_8 (pixman_implementation_t *imp,
+                         pixman_op_t              op,
+                         pixman_image_t *         src_image,
+                         pixman_image_t *         mask_image,
+                         pixman_image_t *         dest_image,
+                         int32_t                  src_x,
+                         int32_t                  src_y,
+                         int32_t                  mask_x,
+                         int32_t                  mask_y,
+                         int32_t                  dest_x,
+                         int32_t                  dest_y,
+                         int32_t                  width,
+                         int32_t                  height)
+{
+    uint32_t src, srca;
+    uint8_t     *dst_line, *dst;
+    uint8_t     *mask_line, *mask, m;
+    int dst_stride, mask_stride;
+    int32_t w;
+    uint16_t t;
+
+    src = _pixman_image_get_solid (src_image, dest_image->bits.format);
+
+    srca = src >> 24;
+
+    PIXMAN_IMAGE_GET_LINE (dest_image, dest_x, dest_y, uint8_t, dst_stride, dst_line, 1);
+    PIXMAN_IMAGE_GET_LINE (mask_image, mask_x, mask_y, uint8_t, mask_stride, mask_line, 1);
+
+    if (srca == 0xff)
+    {
+	while (height--)
+	{
+	    dst = dst_line;
+	    dst_line += dst_stride;
+	    mask = mask_line;
+	    mask_line += mask_stride;
+	    w = width;
+
+	    while (w--)
+	    {
+		m = *mask++;
+
+		if (m == 0)
+		    *dst = 0;
+		else if (m != 0xff)
+		    *dst = MUL_UN8 (m, *dst, t);
+
+		dst++;
+	    }
+	}
+    }
+    else
+    {
+	while (height--)
+	{
+	    dst = dst_line;
+	    dst_line += dst_stride;
+	    mask = mask_line;
+	    mask_line += mask_stride;
+	    w = width;
+
+	    while (w--)
+	    {
+		m = *mask++;
+		m = MUL_UN8 (m, srca, t);
+
+		if (m == 0)
+		    *dst = 0;
+		else if (m != 0xff)
+		    *dst = MUL_UN8 (m, *dst, t);
+
+		dst++;
+	    }
+	}
+    }
+}
+
+static void
+fast_composite_in_8_8 (pixman_implementation_t *imp,
+                       pixman_op_t              op,
+                       pixman_image_t *         src_image,
+                       pixman_image_t *         mask_image,
+                       pixman_image_t *         dest_image,
+                       int32_t                  src_x,
+                       int32_t                  src_y,
+                       int32_t                  mask_x,
+                       int32_t                  mask_y,
+                       int32_t                  dest_x,
+                       int32_t                  dest_y,
+                       int32_t                  width,
+                       int32_t                  height)
+{
+    uint8_t     *dst_line, *dst;
+    uint8_t     *src_line, *src;
+    int dst_stride, src_stride;
+    int32_t w;
+    uint8_t s;
+    uint16_t t;
+
+    PIXMAN_IMAGE_GET_LINE (src_image, src_x, src_y, uint8_t, src_stride, src_line, 1);
+    PIXMAN_IMAGE_GET_LINE (dest_image, dest_x, dest_y, uint8_t, dst_stride, dst_line, 1);
+
+    while (height--)
+    {
+	dst = dst_line;
+	dst_line += dst_stride;
+	src = src_line;
+	src_line += src_stride;
+	w = width;
+
+	while (w--)
+	{
+	    s = *src++;
+
+	    if (s == 0)
+		*dst = 0;
+	    else if (s != 0xff)
+		*dst = MUL_UN8 (s, *dst, t);
+
+	    dst++;
+	}
+    }
+}
+
+static void
+fast_composite_over_n_8_8888 (pixman_implementation_t *imp,
+                              pixman_op_t              op,
+                              pixman_image_t *         src_image,
+                              pixman_image_t *         mask_image,
+                              pixman_image_t *         dst_image,
+                              int32_t                  src_x,
+                              int32_t                  src_y,
+                              int32_t                  mask_x,
+                              int32_t                  mask_y,
+                              int32_t                  dest_x,
+                              int32_t                  dest_y,
+                              int32_t                  width,
+                              int32_t                  height)
+{
+    uint32_t src, srca;
+    uint32_t    *dst_line, *dst, d;
+    uint8_t     *mask_line, *mask, m;
+    int dst_stride, mask_stride;
+    int32_t w;
+
+    src = _pixman_image_get_solid (src_image, dst_image->bits.format);
+
+    srca = src >> 24;
+    if (src == 0)
+	return;
+
+    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
+    PIXMAN_IMAGE_GET_LINE (mask_image, mask_x, mask_y, uint8_t, mask_stride, mask_line, 1);
+
+    while (height--)
+    {
+	dst = dst_line;
+	dst_line += dst_stride;
+	mask = mask_line;
+	mask_line += mask_stride;
+	w = width;
+
+	while (w--)
+	{
+	    m = *mask++;
+	    if (m == 0xff)
+	    {
+		if (srca == 0xff)
+		    *dst = src;
+		else
+		    *dst = over (src, *dst);
+	    }
+	    else if (m)
+	    {
+		d = in (src, m);
+		*dst = over (d, *dst);
+	    }
+	    dst++;
+	}
+    }
+}
+
+static void
+fast_composite_add_n_8888_8888_ca (pixman_implementation_t *imp,
+				   pixman_op_t              op,
+				   pixman_image_t *         src_image,
+				   pixman_image_t *         mask_image,
+				   pixman_image_t *         dst_image,
+				   int32_t                  src_x,
+				   int32_t                  src_y,
+				   int32_t                  mask_x,
+				   int32_t                  mask_y,
+				   int32_t                  dest_x,
+				   int32_t                  dest_y,
+				   int32_t                  width,
+				   int32_t                  height)
+{
+    uint32_t src, srca, s;
+    uint32_t    *dst_line, *dst, d;
+    uint32_t    *mask_line, *mask, ma;
+    int dst_stride, mask_stride;
+    int32_t w;
+
+    src = _pixman_image_get_solid (src_image, dst_image->bits.format);
+
+    srca = src >> 24;
+    if (src == 0)
+	return;
+
+    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
+    PIXMAN_IMAGE_GET_LINE (mask_image, mask_x, mask_y, uint32_t, mask_stride, mask_line, 1);
+
+    while (height--)
+    {
+	dst = dst_line;
+	dst_line += dst_stride;
+	mask = mask_line;
+	mask_line += mask_stride;
+	w = width;
+
+	while (w--)
+	{
+	    ma = *mask++;
+
+	    if (ma)
+	    {
+		d = *dst;
+		s = src;
+
+		UN8x4_MUL_UN8x4_ADD_UN8x4 (s, ma, d);
+
+		*dst = s;
+	    }
+
+	    dst++;
+	}
+    }
+}
+
+static void
+fast_composite_over_n_8888_8888_ca (pixman_implementation_t *imp,
+                                    pixman_op_t              op,
+                                    pixman_image_t *         src_image,
+                                    pixman_image_t *         mask_image,
+                                    pixman_image_t *         dst_image,
+                                    int32_t                  src_x,
+                                    int32_t                  src_y,
+                                    int32_t                  mask_x,
+                                    int32_t                  mask_y,
+                                    int32_t                  dest_x,
+                                    int32_t                  dest_y,
+                                    int32_t                  width,
+                                    int32_t                  height)
+{
+    uint32_t src, srca, s;
+    uint32_t    *dst_line, *dst, d;
+    uint32_t    *mask_line, *mask, ma;
+    int dst_stride, mask_stride;
+    int32_t w;
+
+    src = _pixman_image_get_solid (src_image, dst_image->bits.format);
+
+    srca = src >> 24;
+    if (src == 0)
+	return;
+
+    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
+    PIXMAN_IMAGE_GET_LINE (mask_image, mask_x, mask_y, uint32_t, mask_stride, mask_line, 1);
+
+    while (height--)
+    {
+	dst = dst_line;
+	dst_line += dst_stride;
+	mask = mask_line;
+	mask_line += mask_stride;
+	w = width;
+
+	while (w--)
+	{
+	    ma = *mask++;
+	    if (ma == 0xffffffff)
+	    {
+		if (srca == 0xff)
+		    *dst = src;
+		else
+		    *dst = over (src, *dst);
+	    }
+	    else if (ma)
+	    {
+		d = *dst;
+		s = src;
+
+		UN8x4_MUL_UN8x4 (s, ma);
+		UN8x4_MUL_UN8 (ma, srca);
+		ma = ~ma;
+		UN8x4_MUL_UN8x4_ADD_UN8x4 (d, ma, s);
+
+		*dst = d;
+	    }
+
+	    dst++;
+	}
+    }
+}
+
+static void
+fast_composite_over_n_8_0888 (pixman_implementation_t *imp,
+                              pixman_op_t              op,
+                              pixman_image_t *         src_image,
+                              pixman_image_t *         mask_image,
+                              pixman_image_t *         dst_image,
+                              int32_t                  src_x,
+                              int32_t                  src_y,
+                              int32_t                  mask_x,
+                              int32_t                  mask_y,
+                              int32_t                  dest_x,
+                              int32_t                  dest_y,
+                              int32_t                  width,
+                              int32_t                  height)
+{
+    uint32_t src, srca;
+    uint8_t     *dst_line, *dst;
+    uint32_t d;
+    uint8_t     *mask_line, *mask, m;
+    int dst_stride, mask_stride;
+    int32_t w;
+
+    src = _pixman_image_get_solid (src_image, dst_image->bits.format);
+
+    srca = src >> 24;
+    if (src == 0)
+	return;
+
+    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, uint8_t, dst_stride, dst_line, 3);
+    PIXMAN_IMAGE_GET_LINE (mask_image, mask_x, mask_y, uint8_t, mask_stride, mask_line, 1);
+
+    while (height--)
+    {
+	dst = dst_line;
+	dst_line += dst_stride;
+	mask = mask_line;
+	mask_line += mask_stride;
+	w = width;
+
+	while (w--)
+	{
+	    m = *mask++;
+	    if (m == 0xff)
+	    {
+		if (srca == 0xff)
+		{
+		    d = src;
+		}
+		else
+		{
+		    d = fetch_24 (dst);
+		    d = over (src, d);
+		}
+		store_24 (dst, d);
+	    }
+	    else if (m)
+	    {
+		d = over (in (src, m), fetch_24 (dst));
+		store_24 (dst, d);
+	    }
+	    dst += 3;
+	}
+    }
+}
+
+static void
+fast_composite_over_n_8_0565 (pixman_implementation_t *imp,
+                              pixman_op_t              op,
+                              pixman_image_t *         src_image,
+                              pixman_image_t *         mask_image,
+                              pixman_image_t *         dst_image,
+                              int32_t                  src_x,
+                              int32_t                  src_y,
+                              int32_t                  mask_x,
+                              int32_t                  mask_y,
+                              int32_t                  dest_x,
+                              int32_t                  dest_y,
+                              int32_t                  width,
+                              int32_t                  height)
+{
+    uint32_t src, srca;
+    uint16_t    *dst_line, *dst;
+    uint32_t d;
+    uint8_t     *mask_line, *mask, m;
+    int dst_stride, mask_stride;
+    int32_t w;
+
+    src = _pixman_image_get_solid (src_image, dst_image->bits.format);
+
+    srca = src >> 24;
+    if (src == 0)
+	return;
+
+    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, uint16_t, dst_stride, dst_line, 1);
+    PIXMAN_IMAGE_GET_LINE (mask_image, mask_x, mask_y, uint8_t, mask_stride, mask_line, 1);
+
+    while (height--)
+    {
+	dst = dst_line;
+	dst_line += dst_stride;
+	mask = mask_line;
+	mask_line += mask_stride;
+	w = width;
+
+	while (w--)
+	{
+	    m = *mask++;
+	    if (m == 0xff)
+	    {
+		if (srca == 0xff)
+		{
+		    d = src;
+		}
+		else
+		{
+		    d = *dst;
+		    d = over (src, CONVERT_0565_TO_0888 (d));
+		}
+		*dst = CONVERT_8888_TO_0565 (d);
+	    }
+	    else if (m)
+	    {
+		d = *dst;
+		d = over (in (src, m), CONVERT_0565_TO_0888 (d));
+		*dst = CONVERT_8888_TO_0565 (d);
+	    }
+	    dst++;
+	}
+    }
+}
+
+static void
+fast_composite_over_n_8888_0565_ca (pixman_implementation_t *imp,
+                                    pixman_op_t              op,
+                                    pixman_image_t *         src_image,
+                                    pixman_image_t *         mask_image,
+                                    pixman_image_t *         dst_image,
+                                    int32_t                  src_x,
+                                    int32_t                  src_y,
+                                    int32_t                  mask_x,
+                                    int32_t                  mask_y,
+                                    int32_t                  dest_x,
+                                    int32_t                  dest_y,
+                                    int32_t                  width,
+                                    int32_t                  height)
+{
+    uint32_t  src, srca, s;
+    uint16_t  src16;
+    uint16_t *dst_line, *dst;
+    uint32_t  d;
+    uint32_t *mask_line, *mask, ma;
+    int dst_stride, mask_stride;
+    int32_t w;
+
+    src = _pixman_image_get_solid (src_image, dst_image->bits.format);
+
+    srca = src >> 24;
+    if (src == 0)
+	return;
+
+    src16 = CONVERT_8888_TO_0565 (src);
+
+    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, uint16_t, dst_stride, dst_line, 1);
+    PIXMAN_IMAGE_GET_LINE (mask_image, mask_x, mask_y, uint32_t, mask_stride, mask_line, 1);
+
+    while (height--)
+    {
+	dst = dst_line;
+	dst_line += dst_stride;
+	mask = mask_line;
+	mask_line += mask_stride;
+	w = width;
+
+	while (w--)
+	{
+	    ma = *mask++;
+	    if (ma == 0xffffffff)
+	    {
+		if (srca == 0xff)
+		{
+		    *dst = src16;
+		}
+		else
+		{
+		    d = *dst;
+		    d = over (src, CONVERT_0565_TO_0888 (d));
+		    *dst = CONVERT_8888_TO_0565 (d);
+		}
+	    }
+	    else if (ma)
+	    {
+		d = *dst;
+		d = CONVERT_0565_TO_0888 (d);
+
+		s = src;
+
+		UN8x4_MUL_UN8x4 (s, ma);
+		UN8x4_MUL_UN8 (ma, srca);
+		ma = ~ma;
+		UN8x4_MUL_UN8x4_ADD_UN8x4 (d, ma, s);
+
+		*dst = CONVERT_8888_TO_0565 (d);
+	    }
+	    dst++;
+	}
+    }
+}
+
+static void
+fast_composite_over_8888_8888 (pixman_implementation_t *imp,
+                               pixman_op_t              op,
+                               pixman_image_t *         src_image,
+                               pixman_image_t *         mask_image,
+                               pixman_image_t *         dst_image,
+                               int32_t                  src_x,
+                               int32_t                  src_y,
+                               int32_t                  mask_x,
+                               int32_t                  mask_y,
+                               int32_t                  dest_x,
+                               int32_t                  dest_y,
+                               int32_t                  width,
+                               int32_t                  height)
+{
+    uint32_t    *dst_line, *dst;
+    uint32_t    *src_line, *src, s;
+    int dst_stride, src_stride;
+    uint8_t a;
+    int32_t w;
+
+    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
+    PIXMAN_IMAGE_GET_LINE (src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
+
+    while (height--)
+    {
+	dst = dst_line;
+	dst_line += dst_stride;
+	src = src_line;
+	src_line += src_stride;
+	w = width;
+
+	while (w--)
+	{
+	    s = *src++;
+	    a = s >> 24;
+	    if (a == 0xff)
+		*dst = s;
+	    else if (s)
+		*dst = over (s, *dst);
+	    dst++;
+	}
+    }
+}
+
+#if 0
+static void
+fast_composite_over_8888_0888 (pixman_implementation_t *imp,
+			       pixman_op_t              op,
+			       pixman_image_t *         src_image,
+			       pixman_image_t *         mask_image,
+			       pixman_image_t *         dst_image,
+			       int32_t                  src_x,
+			       int32_t                  src_y,
+			       int32_t                  mask_x,
+			       int32_t                  mask_y,
+			       int32_t                  dest_x,
+			       int32_t                  dest_y,
+			       int32_t                  width,
+			       int32_t                  height)
+{
+    uint8_t     *dst_line, *dst;
+    uint32_t d;
+    uint32_t    *src_line, *src, s;
+    uint8_t a;
+    int dst_stride, src_stride;
+    int32_t w;
+
+    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, uint8_t, dst_stride, dst_line, 3);
+    PIXMAN_IMAGE_GET_LINE (src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
+
+    while (height--)
+    {
+	dst = dst_line;
+	dst_line += dst_stride;
+	src = src_line;
+	src_line += src_stride;
+	w = width;
+
+	while (w--)
+	{
+	    s = *src++;
+	    a = s >> 24;
+	    if (a)
+	    {
+		if (a == 0xff)
+		    d = s;
+		else
+		    d = over (s, fetch_24 (dst));
+
+		store_24 (dst, d);
+	    }
+	    dst += 3;
+	}
+    }
+}
+#endif
+
+static void
+fast_composite_over_8888_0565 (pixman_implementation_t *imp,
+                               pixman_op_t              op,
+                               pixman_image_t *         src_image,
+                               pixman_image_t *         mask_image,
+                               pixman_image_t *         dst_image,
+                               int32_t                  src_x,
+                               int32_t                  src_y,
+                               int32_t                  mask_x,
+                               int32_t                  mask_y,
+                               int32_t                  dest_x,
+                               int32_t                  dest_y,
+                               int32_t                  width,
+                               int32_t                  height)
+{
+    uint16_t    *dst_line, *dst;
+    uint32_t d;
+    uint32_t    *src_line, *src, s;
+    uint8_t a;
+    int dst_stride, src_stride;
+    int32_t w;
+
+    PIXMAN_IMAGE_GET_LINE (src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
+    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, uint16_t, dst_stride, dst_line, 1);
+
+    while (height--)
+    {
+	dst = dst_line;
+	dst_line += dst_stride;
+	src = src_line;
+	src_line += src_stride;
+	w = width;
+
+	while (w--)
+	{
+	    s = *src++;
+	    a = s >> 24;
+	    if (s)
+	    {
+		if (a == 0xff)
+		{
+		    d = s;
+		}
+		else
+		{
+		    d = *dst;
+		    d = over (s, CONVERT_0565_TO_0888 (d));
+		}
+		*dst = CONVERT_8888_TO_0565 (d);
+	    }
+	    dst++;
+	}
+    }
+}
+
+static void
+fast_composite_src_x888_0565 (pixman_implementation_t *imp,
+                              pixman_op_t              op,
+                              pixman_image_t *         src_image,
+                              pixman_image_t *         mask_image,
+                              pixman_image_t *         dst_image,
+                              int32_t                  src_x,
+                              int32_t                  src_y,
+                              int32_t                  mask_x,
+                              int32_t                  mask_y,
+                              int32_t                  dest_x,
+                              int32_t                  dest_y,
+                              int32_t                  width,
+                              int32_t                  height)
+{
+    uint16_t    *dst_line, *dst;
+    uint32_t    *src_line, *src, s;
+    int dst_stride, src_stride;
+    int32_t w;
+
+    PIXMAN_IMAGE_GET_LINE (src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
+    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, uint16_t, dst_stride, dst_line, 1);
+
+    while (height--)
+    {
+	dst = dst_line;
+	dst_line += dst_stride;
+	src = src_line;
+	src_line += src_stride;
+	w = width;
+
+	while (w--)
+	{
+	    s = *src++;
+	    *dst = CONVERT_8888_TO_0565 (s);
+	    dst++;
+	}
+    }
+}
+
+static void
+fast_composite_add_8000_8000 (pixman_implementation_t *imp,
+                              pixman_op_t              op,
+                              pixman_image_t *         src_image,
+                              pixman_image_t *         mask_image,
+                              pixman_image_t *         dst_image,
+                              int32_t                  src_x,
+                              int32_t                  src_y,
+                              int32_t                  mask_x,
+                              int32_t                  mask_y,
+                              int32_t                  dest_x,
+                              int32_t                  dest_y,
+                              int32_t                  width,
+                              int32_t                  height)
+{
+    uint8_t     *dst_line, *dst;
+    uint8_t     *src_line, *src;
+    int dst_stride, src_stride;
+    int32_t w;
+    uint8_t s, d;
+    uint16_t t;
+
+    PIXMAN_IMAGE_GET_LINE (src_image, src_x, src_y, uint8_t, src_stride, src_line, 1);
+    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, uint8_t, dst_stride, dst_line, 1);
+
+    while (height--)
+    {
+	dst = dst_line;
+	dst_line += dst_stride;
+	src = src_line;
+	src_line += src_stride;
+	w = width;
+
+	while (w--)
+	{
+	    s = *src++;
+	    if (s)
+	    {
+		if (s != 0xff)
+		{
+		    d = *dst;
+		    t = d + s;
+		    s = t | (0 - (t >> 8));
+		}
+		*dst = s;
+	    }
+	    dst++;
+	}
+    }
+}
+
+static void
+fast_composite_add_8888_8888 (pixman_implementation_t *imp,
+                              pixman_op_t              op,
+                              pixman_image_t *         src_image,
+                              pixman_image_t *         mask_image,
+                              pixman_image_t *         dst_image,
+                              int32_t                  src_x,
+                              int32_t                  src_y,
+                              int32_t                  mask_x,
+                              int32_t                  mask_y,
+                              int32_t                  dest_x,
+                              int32_t                  dest_y,
+                              int32_t                  width,
+                              int32_t                  height)
+{
+    uint32_t    *dst_line, *dst;
+    uint32_t    *src_line, *src;
+    int dst_stride, src_stride;
+    int32_t w;
+    uint32_t s, d;
+
+    PIXMAN_IMAGE_GET_LINE (src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
+    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
+
+    while (height--)
+    {
+	dst = dst_line;
+	dst_line += dst_stride;
+	src = src_line;
+	src_line += src_stride;
+	w = width;
+
+	while (w--)
+	{
+	    s = *src++;
+	    if (s)
+	    {
+		if (s != 0xffffffff)
+		{
+		    d = *dst;
+		    if (d)
+			UN8x4_ADD_UN8x4 (s, d);
+		}
+		*dst = s;
+	    }
+	    dst++;
+	}
+    }
+}
+
+static void
+fast_composite_add_n_8_8 (pixman_implementation_t *imp,
+			  pixman_op_t              op,
+			  pixman_image_t *         src_image,
+			  pixman_image_t *         mask_image,
+			  pixman_image_t *         dst_image,
+			  int32_t                  src_x,
+			  int32_t                  src_y,
+			  int32_t                  mask_x,
+			  int32_t                  mask_y,
+			  int32_t                  dest_x,
+			  int32_t                  dest_y,
+			  int32_t                  width,
+			  int32_t                  height)
+{
+    uint8_t     *dst_line, *dst;
+    uint8_t     *mask_line, *mask;
+    int dst_stride, mask_stride;
+    int32_t w;
+    uint32_t src;
+    uint8_t sa;
+
+    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, uint8_t, dst_stride, dst_line, 1);
+    PIXMAN_IMAGE_GET_LINE (mask_image, mask_x, mask_y, uint8_t, mask_stride, mask_line, 1);
+    src = _pixman_image_get_solid (src_image, dst_image->bits.format);
+    sa = (src >> 24);
+
+    while (height--)
+    {
+	dst = dst_line;
+	dst_line += dst_stride;
+	mask = mask_line;
+	mask_line += mask_stride;
+	w = width;
+
+	while (w--)
+	{
+	    uint16_t tmp;
+	    uint16_t a;
+	    uint32_t m, d;
+	    uint32_t r;
+
+	    a = *mask++;
+	    d = *dst;
+
+	    m = MUL_UN8 (sa, a, tmp);
+	    r = ADD_UN8 (m, d, tmp);
+
+	    *dst++ = r;
+	}
+    }
+}
+
+#ifdef WORDS_BIGENDIAN
+#define CREATE_BITMASK(n) (0x80000000 >> (n))
+#define UPDATE_BITMASK(n) ((n) >> 1)
+#else
+#define CREATE_BITMASK(n) (1 << (n))
+#define UPDATE_BITMASK(n) ((n) << 1)
+#endif
+
+#define TEST_BIT(p, n)					\
+    (*((p) + ((n) >> 5)) & CREATE_BITMASK ((n) & 31))
+#define SET_BIT(p, n)							\
+    do { *((p) + ((n) >> 5)) |= CREATE_BITMASK ((n) & 31); } while (0);
+
+static void
+fast_composite_add_1000_1000 (pixman_implementation_t *imp,
+                              pixman_op_t              op,
+                              pixman_image_t *         src_image,
+                              pixman_image_t *         mask_image,
+                              pixman_image_t *         dst_image,
+                              int32_t                  src_x,
+                              int32_t                  src_y,
+                              int32_t                  mask_x,
+                              int32_t                  mask_y,
+                              int32_t                  dest_x,
+                              int32_t                  dest_y,
+                              int32_t                  width,
+                              int32_t                  height)
+{
+    uint32_t     *dst_line, *dst;
+    uint32_t     *src_line, *src;
+    int           dst_stride, src_stride;
+    int32_t       w;
+
+    PIXMAN_IMAGE_GET_LINE (src_image, 0, src_y, uint32_t,
+                           src_stride, src_line, 1);
+    PIXMAN_IMAGE_GET_LINE (dst_image, 0, dest_y, uint32_t,
+                           dst_stride, dst_line, 1);
+
+    while (height--)
+    {
+	dst = dst_line;
+	dst_line += dst_stride;
+	src = src_line;
+	src_line += src_stride;
+	w = width;
+
+	while (w--)
+	{
+	    /*
+	     * TODO: improve performance by processing uint32_t data instead
+	     *       of individual bits
+	     */
+	    if (TEST_BIT (src, src_x + w))
+		SET_BIT (dst, dest_x + w);
+	}
+    }
+}
+
+static void
+fast_composite_over_n_1_8888 (pixman_implementation_t *imp,
+                              pixman_op_t              op,
+                              pixman_image_t *         src_image,
+                              pixman_image_t *         mask_image,
+                              pixman_image_t *         dst_image,
+                              int32_t                  src_x,
+                              int32_t                  src_y,
+                              int32_t                  mask_x,
+                              int32_t                  mask_y,
+                              int32_t                  dest_x,
+                              int32_t                  dest_y,
+                              int32_t                  width,
+                              int32_t                  height)
+{
+    uint32_t     src, srca;
+    uint32_t    *dst, *dst_line;
+    uint32_t    *mask, *mask_line;
+    int          mask_stride, dst_stride;
+    uint32_t     bitcache, bitmask;
+    int32_t      w;
+
+    if (width <= 0)
+	return;
+
+    src = _pixman_image_get_solid (src_image, dst_image->bits.format);
+    srca = src >> 24;
+    if (src == 0)
+	return;
+
+    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, uint32_t,
+                           dst_stride, dst_line, 1);
+    PIXMAN_IMAGE_GET_LINE (mask_image, 0, mask_y, uint32_t,
+                           mask_stride, mask_line, 1);
+    mask_line += mask_x >> 5;
+
+    if (srca == 0xff)
+    {
+	while (height--)
+	{
+	    dst = dst_line;
+	    dst_line += dst_stride;
+	    mask = mask_line;
+	    mask_line += mask_stride;
+	    w = width;
+
+	    bitcache = *mask++;
+	    bitmask = CREATE_BITMASK (mask_x & 31);
+
+	    while (w--)
+	    {
+		if (bitmask == 0)
+		{
+		    bitcache = *mask++;
+		    bitmask = CREATE_BITMASK (0);
+		}
+		if (bitcache & bitmask)
+		    *dst = src;
+		bitmask = UPDATE_BITMASK (bitmask);
+		dst++;
+	    }
+	}
+    }
+    else
+    {
+	while (height--)
+	{
+	    dst = dst_line;
+	    dst_line += dst_stride;
+	    mask = mask_line;
+	    mask_line += mask_stride;
+	    w = width;
+
+	    bitcache = *mask++;
+	    bitmask = CREATE_BITMASK (mask_x & 31);
+
+	    while (w--)
+	    {
+		if (bitmask == 0)
+		{
+		    bitcache = *mask++;
+		    bitmask = CREATE_BITMASK (0);
+		}
+		if (bitcache & bitmask)
+		    *dst = over (src, *dst);
+		bitmask = UPDATE_BITMASK (bitmask);
+		dst++;
+	    }
+	}
+    }
+}
+
+static void
+fast_composite_over_n_1_0565 (pixman_implementation_t *imp,
+                              pixman_op_t              op,
+                              pixman_image_t *         src_image,
+                              pixman_image_t *         mask_image,
+                              pixman_image_t *         dst_image,
+                              int32_t                  src_x,
+                              int32_t                  src_y,
+                              int32_t                  mask_x,
+                              int32_t                  mask_y,
+                              int32_t                  dest_x,
+                              int32_t                  dest_y,
+                              int32_t                  width,
+                              int32_t                  height)
+{
+    uint32_t     src, srca;
+    uint16_t    *dst, *dst_line;
+    uint32_t    *mask, *mask_line;
+    int          mask_stride, dst_stride;
+    uint32_t     bitcache, bitmask;
+    int32_t      w;
+    uint32_t     d;
+    uint16_t     src565;
+
+    if (width <= 0)
+	return;
+
+    src = _pixman_image_get_solid (src_image, dst_image->bits.format);
+    srca = src >> 24;
+    if (src == 0)
+	return;
+
+    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, uint16_t,
+                           dst_stride, dst_line, 1);
+    PIXMAN_IMAGE_GET_LINE (mask_image, 0, mask_y, uint32_t,
+                           mask_stride, mask_line, 1);
+    mask_line += mask_x >> 5;
+
+    if (srca == 0xff)
+    {
+	src565 = CONVERT_8888_TO_0565 (src);
+	while (height--)
+	{
+	    dst = dst_line;
+	    dst_line += dst_stride;
+	    mask = mask_line;
+	    mask_line += mask_stride;
+	    w = width;
+
+	    bitcache = *mask++;
+	    bitmask = CREATE_BITMASK (mask_x & 31);
+
+	    while (w--)
+	    {
+		if (bitmask == 0)
+		{
+		    bitcache = *mask++;
+		    bitmask = CREATE_BITMASK (0);
+		}
+		if (bitcache & bitmask)
+		    *dst = src565;
+		bitmask = UPDATE_BITMASK (bitmask);
+		dst++;
+	    }
+	}
+    }
+    else
+    {
+	while (height--)
+	{
+	    dst = dst_line;
+	    dst_line += dst_stride;
+	    mask = mask_line;
+	    mask_line += mask_stride;
+	    w = width;
+
+	    bitcache = *mask++;
+	    bitmask = CREATE_BITMASK (mask_x & 31);
+
+	    while (w--)
+	    {
+		if (bitmask == 0)
+		{
+		    bitcache = *mask++;
+		    bitmask = CREATE_BITMASK (0);
+		}
+		if (bitcache & bitmask)
+		{
+		    d = over (src, CONVERT_0565_TO_0888 (*dst));
+		    *dst = CONVERT_8888_TO_0565 (d);
+		}
+		bitmask = UPDATE_BITMASK (bitmask);
+		dst++;
+	    }
+	}
+    }
+}
+
+/*
+ * Simple bitblt
+ */
+
+static void
+fast_composite_solid_fill (pixman_implementation_t *imp,
+                           pixman_op_t              op,
+                           pixman_image_t *         src_image,
+                           pixman_image_t *         mask_image,
+                           pixman_image_t *         dst_image,
+                           int32_t                  src_x,
+                           int32_t                  src_y,
+                           int32_t                  mask_x,
+                           int32_t                  mask_y,
+                           int32_t                  dest_x,
+                           int32_t                  dest_y,
+                           int32_t                  width,
+                           int32_t                  height)
+{
+    uint32_t src;
+
+    src = _pixman_image_get_solid (src_image, dst_image->bits.format);
+
+    if (dst_image->bits.format == PIXMAN_a8)
+    {
+	src = src >> 24;
+    }
+    else if (dst_image->bits.format == PIXMAN_r5g6b5 ||
+             dst_image->bits.format == PIXMAN_b5g6r5)
+    {
+	src = CONVERT_8888_TO_0565 (src);
+    }
+
+    pixman_fill (dst_image->bits.bits, dst_image->bits.rowstride,
+                 PIXMAN_FORMAT_BPP (dst_image->bits.format),
+                 dest_x, dest_y,
+                 width, height,
+                 src);
+}
+
+static void
+fast_composite_src_8888_x888 (pixman_implementation_t *imp,
+                              pixman_op_t              op,
+                              pixman_image_t *         src_image,
+                              pixman_image_t *         mask_image,
+                              pixman_image_t *         dst_image,
+                              int32_t                  src_x,
+                              int32_t                  src_y,
+                              int32_t                  mask_x,
+                              int32_t                  mask_y,
+                              int32_t                  dest_x,
+                              int32_t                  dest_y,
+                              int32_t                  width,
+                              int32_t                  height)
+{
+    uint32_t    *dst;
+    uint32_t    *src;
+    int dst_stride, src_stride;
+    uint32_t n_bytes = width * sizeof (uint32_t);
+
+    PIXMAN_IMAGE_GET_LINE (src_image, src_x, src_y, uint32_t, src_stride, src, 1);
+    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, uint32_t, dst_stride, dst, 1);
+
+    while (height--)
+    {
+	memcpy (dst, src, n_bytes);
+
+	dst += dst_stride;
+	src += src_stride;
+    }
+}
+
+static force_inline pixman_bool_t
+repeat (pixman_repeat_t repeat, int *c, int size)
+{
+    if (repeat == PIXMAN_REPEAT_NONE)
+    {
+	if (*c < 0 || *c >= size)
+	    return FALSE;
+    }
+    else if (repeat == PIXMAN_REPEAT_NORMAL)
+    {
+	while (*c >= size)
+	    *c -= size;
+	while (*c < 0)
+	    *c += size;
+    }
+    else if (repeat == PIXMAN_REPEAT_PAD)
+    {
+	*c = CLIP (*c, 0, size - 1);
+    }
+    else /* REFLECT */
+    {
+	*c = MOD (*c, size * 2);
+	if (*c >= size)
+	    *c = size * 2 - *c - 1;
+    }
+    return TRUE;
+}
+
+/* A macroified version of specialized nearest scalers for some
+ * common 8888 and 565 formats. It supports SRC and OVER ops.
+ *
+ * There are two repeat versions, one that handles repeat normal,
+ * and one without repeat handling that only works if the src region
+ * used is completely covered by the pre-repeated source samples.
+ *
+ * The loops are unrolled to process two pixels per iteration for better
+ * performance on most CPU architectures (superscalar processors
+ * can issue several operations simultaneously, other processors can hide
+ * instructions latencies by pipelining operations). Unrolling more
+ * does not make much sense because the compiler will start running out
+ * of spare registers soon.
+ */
+
+#define GET_8888_ALPHA(s) ((s) >> 24)
+ /* This is not actually used since we don't have an OVER with
+    565 source, but it is needed to build. */
+#define GET_0565_ALPHA(s) 0xff
+
+#define FAST_NEAREST(scale_func_name, SRC_FORMAT, DST_FORMAT,					\
+		     src_type_t, dst_type_t, OP, do_repeat)					\
+static void											\
+fast_composite_scaled_nearest_ ## scale_func_name ## _ ## OP (pixman_implementation_t *imp,	\
+							      pixman_op_t              op,      \
+							      pixman_image_t *         src_image, \
+							      pixman_image_t *         mask_image, \
+							      pixman_image_t *         dst_image, \
+							      int32_t                  src_x,   \
+							      int32_t                  src_y,   \
+							      int32_t                  mask_x,  \
+							      int32_t                  mask_y,  \
+							      int32_t                  dst_x,   \
+							      int32_t                  dst_y,   \
+							      int32_t                  width,   \
+							      int32_t                  height)  \
+{												\
+    dst_type_t *dst_line;									\
+    src_type_t *src_first_line;									\
+    uint32_t   d;										\
+    src_type_t s1, s2;										\
+    uint8_t   a1, a2;										\
+    int       w;										\
+    int       x1, x2, y;									\
+    pixman_fixed_t orig_vx;									\
+    pixman_fixed_t max_vx, max_vy;								\
+    pixman_vector_t v;										\
+    pixman_fixed_t vx, vy;									\
+    pixman_fixed_t unit_x, unit_y;								\
+												\
+    src_type_t *src;										\
+    dst_type_t *dst;										\
+    int       src_stride, dst_stride;								\
+												\
+    if (PIXMAN_OP_ ## OP != PIXMAN_OP_SRC && PIXMAN_OP_ ## OP != PIXMAN_OP_OVER)		\
+	abort();										\
+												\
+    PIXMAN_IMAGE_GET_LINE (dst_image, dst_x, dst_y, dst_type_t, dst_stride, dst_line, 1);	\
+    /* pass in 0 instead of src_x and src_y because src_x and src_y need to be			\
+     * transformed from destination space to source space */					\
+    PIXMAN_IMAGE_GET_LINE (src_image, 0, 0, src_type_t, src_stride, src_first_line, 1);		\
+												\
+    /* reference point is the center of the pixel */						\
+    v.vector[0] = pixman_int_to_fixed (src_x) + pixman_fixed_1 / 2;				\
+    v.vector[1] = pixman_int_to_fixed (src_y) + pixman_fixed_1 / 2;				\
+    v.vector[2] = pixman_fixed_1;								\
+												\
+    if (!pixman_transform_point_3d (src_image->common.transform, &v))				\
+	return;											\
+												\
+    unit_x = src_image->common.transform->matrix[0][0];						\
+    unit_y = src_image->common.transform->matrix[1][1];						\
+												\
+    /* Round down to closest integer, ensuring that 0.5 rounds to 0, not 1 */			\
+    v.vector[0] -= pixman_fixed_e;								\
+    v.vector[1] -= pixman_fixed_e;								\
+												\
+    vx = v.vector[0];										\
+    vy = v.vector[1];										\
+												\
+    if (do_repeat)										\
+    {												\
+	/* Clamp repeating positions inside the actual samples */				\
+	max_vx = src_image->bits.width << 16;							\
+	max_vy = src_image->bits.height << 16;							\
+												\
+	repeat (PIXMAN_REPEAT_NORMAL, &vx, max_vx);						\
+	repeat (PIXMAN_REPEAT_NORMAL, &vy, max_vy);						\
+    }												\
+												\
+    orig_vx = vx;										\
+												\
+    while (--height >= 0)									\
+    {												\
+	dst = dst_line;										\
+	dst_line += dst_stride;									\
+												\
+	y = vy >> 16;										\
+	vy += unit_y;										\
+	if (do_repeat)										\
+	    repeat (PIXMAN_REPEAT_NORMAL, &vy, max_vy);						\
+												\
+	src = src_first_line + src_stride * y;							\
+												\
+	w = width;										\
+	vx = orig_vx;										\
+	while ((w -= 2) >= 0)									\
+	{											\
+	    x1 = vx >> 16;									\
+	    vx += unit_x;									\
+	    if (do_repeat)									\
+	    {											\
+		/* This works because we know that unit_x is positive */			\
+		while (vx >= max_vx)								\
+		    vx -= max_vx;								\
+	    }											\
+	    s1 = src[x1];									\
+												\
+	    x2 = vx >> 16;									\
+	    vx += unit_x;									\
+	    if (do_repeat)									\
+	    {											\
+		/* This works because we know that unit_x is positive */			\
+		while (vx >= max_vx)								\
+		    vx -= max_vx;								\
+	    }											\
+	    s2 = src[x2];									\
+												\
+	    if (PIXMAN_OP_ ## OP == PIXMAN_OP_OVER)						\
+	    {											\
+		a1 = GET_ ## SRC_FORMAT ## _ALPHA(s1);						\
+		a2 = GET_ ## SRC_FORMAT ## _ALPHA(s2);						\
+												\
+		if (a1 == 0xff)									\
+		{										\
+		    *dst = CONVERT_ ## SRC_FORMAT ## _TO_ ## DST_FORMAT (s1);			\
+		}										\
+		else if (s1)									\
+		{										\
+		    d = CONVERT_## DST_FORMAT ## _TO_8888 (*dst);				\
+		    a1 ^= 0xff;									\
+		    UN8x4_MUL_UN8_ADD_UN8x4 (d, a1, s1);					\
+		    *dst = CONVERT_8888_TO_ ## DST_FORMAT (d);					\
+		}										\
+		dst++;										\
+												\
+		if (a2 == 0xff)									\
+		{										\
+		    *dst = CONVERT_ ## SRC_FORMAT ## _TO_ ## DST_FORMAT (s2);			\
+		}										\
+		else if (s2)									\
+		{										\
+		    d = CONVERT_## DST_FORMAT ## _TO_8888 (*dst);				\
+		    a2 ^= 0xff;									\
+		    UN8x4_MUL_UN8_ADD_UN8x4 (d, a2, s2);					\
+		    *dst = CONVERT_8888_TO_ ## DST_FORMAT (d);					\
+		}										\
+		dst++;										\
+	    }											\
+	    else /* PIXMAN_OP_SRC */								\
+	    {											\
+		    *dst++ = CONVERT_ ## SRC_FORMAT ## _TO_ ## DST_FORMAT (s1);			\
+		    *dst++ = CONVERT_ ## SRC_FORMAT ## _TO_ ## DST_FORMAT (s2);			\
+	    }											\
+	}											\
+												\
+	if (w & 1)										\
+	{											\
+	    x1 = vx >> 16;									\
+	    vx += unit_x;									\
+	    if (do_repeat)									\
+	    {											\
+		/* This works because we know that unit_x is positive */			\
+		while (vx >= max_vx)								\
+		    vx -= max_vx;								\
+	    }											\
+	    s1 = src[x1];									\
+												\
+	    if (PIXMAN_OP_ ## OP == PIXMAN_OP_OVER)						\
+	    {											\
+		a1 = GET_ ## SRC_FORMAT ## _ALPHA(s1);						\
+												\
+		if (a1 == 0xff)									\
+		{										\
+		    *dst = CONVERT_ ## SRC_FORMAT ## _TO_ ## DST_FORMAT (s1);			\
+		}										\
+		else if (s1)									\
+		{										\
+		    d = CONVERT_## DST_FORMAT ## _TO_8888 (*dst);				\
+		    a1 ^= 0xff;									\
+		    UN8x4_MUL_UN8_ADD_UN8x4 (d, a1, s1);					\
+		    *dst = CONVERT_8888_TO_ ## DST_FORMAT (d);					\
+		}										\
+		dst++;										\
+	    }											\
+	    else /* PIXMAN_OP_SRC */								\
+	    {											\
+		    *dst++ = CONVERT_ ## SRC_FORMAT ## _TO_ ## DST_FORMAT (s1);			\
+	    }											\
+	}											\
+    }												\
+}
+
+FAST_NEAREST(x888_x888_none, 8888, 8888, uint32_t, uint32_t, SRC, /*repeat: */ 0);
+FAST_NEAREST(x888_x888_normal, 8888, 8888, uint32_t, uint32_t, SRC, /*repeat: */ 1);
+FAST_NEAREST(x888_x888_none, 8888, 8888, uint32_t, uint32_t, OVER, /*repeat: */ 0);
+FAST_NEAREST(x888_x888_normal, 8888, 8888, uint32_t, uint32_t, OVER, /*repeat: */ 1);
+FAST_NEAREST(x888_565_none, 8888, 0565, uint32_t, uint16_t, SRC, /*repeat: */ 0);
+FAST_NEAREST(x888_565_normal, 8888, 0565, uint32_t, uint16_t, SRC, /*repeat: */ 1);
+FAST_NEAREST(565_565_none, 0565, 0565, uint16_t, uint16_t, SRC, /*repeat: */ 0);
+FAST_NEAREST(565_565_normal, 0565, 0565, uint16_t, uint16_t, SRC, /*repeat: */ 1);
+FAST_NEAREST(8888_565_none, 8888, 0565, uint32_t, uint16_t, OVER, /*repeat: */ 0);
+FAST_NEAREST(8888_565_normal, 8888, 0565, uint32_t, uint16_t, OVER, /*repeat: */ 1);
+
+static force_inline uint32_t
+fetch_nearest (pixman_repeat_t src_repeat,
+	       pixman_format_code_t format,
+	       uint32_t *src, int x, int src_width)
+{
+    if (repeat (src_repeat, &x, src_width))
+    {
+	if (format == PIXMAN_x8r8g8b8)
+	    return *(src + x) | 0xff000000;
+	else
+	    return *(src + x);
+    }
+    else
+    {
+	return 0;
+    }
+}
+
+static force_inline void
+combine_over (uint32_t s, uint32_t *dst)
+{
+    if (s)
+    {
+	uint8_t ia = 0xff - (s >> 24);
+
+	if (ia)
+	    UN8x4_MUL_UN8_ADD_UN8x4 (*dst, ia, s);
+	else
+	    *dst = s;
+    }
+}
+
+static force_inline void
+combine_src (uint32_t s, uint32_t *dst)
+{
+    *dst = s;
+}
+
+static void
+fast_composite_scaled_nearest (pixman_implementation_t *imp,
+			       pixman_op_t              op,
+			       pixman_image_t *         src_image,
+			       pixman_image_t *         mask_image,
+			       pixman_image_t *         dst_image,
+			       int32_t                  src_x,
+			       int32_t                  src_y,
+			       int32_t                  mask_x,
+			       int32_t                  mask_y,
+			       int32_t                  dest_x,
+			       int32_t                  dest_y,
+			       int32_t                  width,
+			       int32_t                  height)
+{
+    uint32_t       *dst_line;
+    uint32_t       *src_line;
+    int             dst_stride, src_stride;
+    int		    src_width, src_height;
+    pixman_repeat_t src_repeat;
+    pixman_fixed_t unit_x, unit_y;
+    pixman_format_code_t src_format;
+    pixman_vector_t v;
+    pixman_fixed_t vy;
+
+    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
+    /* pass in 0 instead of src_x and src_y because src_x and src_y need to be
+     * transformed from destination space to source space
+     */
+    PIXMAN_IMAGE_GET_LINE (src_image, 0, 0, uint32_t, src_stride, src_line, 1);
+
+    /* reference point is the center of the pixel */
+    v.vector[0] = pixman_int_to_fixed (src_x) + pixman_fixed_1 / 2;
+    v.vector[1] = pixman_int_to_fixed (src_y) + pixman_fixed_1 / 2;
+    v.vector[2] = pixman_fixed_1;
+
+    if (!pixman_transform_point_3d (src_image->common.transform, &v))
+	return;
+
+    unit_x = src_image->common.transform->matrix[0][0];
+    unit_y = src_image->common.transform->matrix[1][1];
+
+    /* Round down to closest integer, ensuring that 0.5 rounds to 0, not 1 */
+    v.vector[0] -= pixman_fixed_e;
+    v.vector[1] -= pixman_fixed_e;
+
+    src_height = src_image->bits.height;
+    src_width = src_image->bits.width;
+    src_repeat = src_image->common.repeat;
+    src_format = src_image->bits.format;
+
+    vy = v.vector[1];
+    while (height--)
+    {
+        pixman_fixed_t vx = v.vector[0];
+	int y = pixman_fixed_to_int (vy);
+	uint32_t *dst = dst_line;
+
+	dst_line += dst_stride;
+
+        /* adjust the y location by a unit vector in the y direction
+         * this is equivalent to transforming y+1 of the destination point to source space */
+        vy += unit_y;
+
+	if (!repeat (src_repeat, &y, src_height))
+	{
+	    if (op == PIXMAN_OP_SRC)
+		memset (dst, 0, sizeof (*dst) * width);
+	}
+	else
+	{
+	    int w = width;
+
+	    uint32_t *src = src_line + y * src_stride;
+
+	    while (w >= 2)
+	    {
+		uint32_t s1, s2;
+		int x1, x2;
+
+		x1 = pixman_fixed_to_int (vx);
+		vx += unit_x;
+
+		x2 = pixman_fixed_to_int (vx);
+		vx += unit_x;
+
+		w -= 2;
+
+		s1 = fetch_nearest (src_repeat, src_format, src, x1, src_width);
+		s2 = fetch_nearest (src_repeat, src_format, src, x2, src_width);
+
+		if (op == PIXMAN_OP_OVER)
+		{
+		    combine_over (s1, dst++);
+		    combine_over (s2, dst++);
+		}
+		else
+		{
+		    combine_src (s1, dst++);
+		    combine_src (s2, dst++);
+		}
+	    }
+
+	    while (w--)
+	    {
+		uint32_t s;
+		int x;
+
+		x = pixman_fixed_to_int (vx);
+		vx += unit_x;
+
+		s = fetch_nearest (src_repeat, src_format, src, x, src_width);
+
+		if (op == PIXMAN_OP_OVER)
+		    combine_over (s, dst++);
+		else
+		    combine_src (s, dst++);
+	    }
+	}
+    }
+}
+
+static const pixman_fast_path_t c_fast_paths[] =
+{
+    PIXMAN_STD_FAST_PATH (OVER, solid, a8, r5g6b5, fast_composite_over_n_8_0565),
+    PIXMAN_STD_FAST_PATH (OVER, solid, a8, b5g6r5, fast_composite_over_n_8_0565),
+    PIXMAN_STD_FAST_PATH (OVER, solid, a8, r8g8b8, fast_composite_over_n_8_0888),
+    PIXMAN_STD_FAST_PATH (OVER, solid, a8, b8g8r8, fast_composite_over_n_8_0888),
+    PIXMAN_STD_FAST_PATH (OVER, solid, a8, a8r8g8b8, fast_composite_over_n_8_8888),
+    PIXMAN_STD_FAST_PATH (OVER, solid, a8, x8r8g8b8, fast_composite_over_n_8_8888),
+    PIXMAN_STD_FAST_PATH (OVER, solid, a8, a8b8g8r8, fast_composite_over_n_8_8888),
+    PIXMAN_STD_FAST_PATH (OVER, solid, a8, x8b8g8r8, fast_composite_over_n_8_8888),
+    PIXMAN_STD_FAST_PATH (OVER, solid, a1, a8r8g8b8, fast_composite_over_n_1_8888),
+    PIXMAN_STD_FAST_PATH (OVER, solid, a1, x8r8g8b8, fast_composite_over_n_1_8888),
+    PIXMAN_STD_FAST_PATH (OVER, solid, a1, a8b8g8r8, fast_composite_over_n_1_8888),
+    PIXMAN_STD_FAST_PATH (OVER, solid, a1, x8b8g8r8, fast_composite_over_n_1_8888),
+    PIXMAN_STD_FAST_PATH (OVER, solid, a1, r5g6b5,   fast_composite_over_n_1_0565),
+    PIXMAN_STD_FAST_PATH (OVER, solid, a1, b5g6r5,   fast_composite_over_n_1_0565),
+    PIXMAN_STD_FAST_PATH_CA (OVER, solid, a8r8g8b8, a8r8g8b8, fast_composite_over_n_8888_8888_ca),
+    PIXMAN_STD_FAST_PATH_CA (OVER, solid, a8r8g8b8, x8r8g8b8, fast_composite_over_n_8888_8888_ca),
+    PIXMAN_STD_FAST_PATH_CA (OVER, solid, a8r8g8b8, r5g6b5, fast_composite_over_n_8888_0565_ca),
+    PIXMAN_STD_FAST_PATH_CA (OVER, solid, a8b8g8r8, a8b8g8r8, fast_composite_over_n_8888_8888_ca),
+    PIXMAN_STD_FAST_PATH_CA (OVER, solid, a8b8g8r8, x8b8g8r8, fast_composite_over_n_8888_8888_ca),
+    PIXMAN_STD_FAST_PATH_CA (OVER, solid, a8b8g8r8, b5g6r5, fast_composite_over_n_8888_0565_ca),
+    PIXMAN_STD_FAST_PATH (OVER, x8r8g8b8, a8, x8r8g8b8, fast_composite_over_x888_8_8888),
+    PIXMAN_STD_FAST_PATH (OVER, x8r8g8b8, a8, a8r8g8b8, fast_composite_over_x888_8_8888),
+    PIXMAN_STD_FAST_PATH (OVER, x8b8g8r8, a8, x8b8g8r8, fast_composite_over_x888_8_8888),
+    PIXMAN_STD_FAST_PATH (OVER, x8b8g8r8, a8, a8b8g8r8, fast_composite_over_x888_8_8888),
+    PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, null, a8r8g8b8, fast_composite_over_8888_8888),
+    PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, null, x8r8g8b8, fast_composite_over_8888_8888),
+    PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, null, r5g6b5, fast_composite_over_8888_0565),
+    PIXMAN_STD_FAST_PATH (OVER, a8b8g8r8, null, a8b8g8r8, fast_composite_over_8888_8888),
+    PIXMAN_STD_FAST_PATH (OVER, a8b8g8r8, null, x8b8g8r8, fast_composite_over_8888_8888),
+    PIXMAN_STD_FAST_PATH (OVER, a8b8g8r8, null, b5g6r5, fast_composite_over_8888_0565),
+    PIXMAN_STD_FAST_PATH (ADD, a8r8g8b8, null, a8r8g8b8, fast_composite_add_8888_8888),
+    PIXMAN_STD_FAST_PATH (ADD, a8b8g8r8, null, a8b8g8r8, fast_composite_add_8888_8888),
+    PIXMAN_STD_FAST_PATH (ADD, a8, null, a8, fast_composite_add_8000_8000),
+    PIXMAN_STD_FAST_PATH (ADD, a1, null, a1, fast_composite_add_1000_1000),
+    PIXMAN_STD_FAST_PATH_CA (ADD, solid, a8r8g8b8, a8r8g8b8, fast_composite_add_n_8888_8888_ca),
+    PIXMAN_STD_FAST_PATH (ADD, solid, a8, a8, fast_composite_add_n_8_8),
+    PIXMAN_STD_FAST_PATH (SRC, solid, null, a8r8g8b8, fast_composite_solid_fill),
+    PIXMAN_STD_FAST_PATH (SRC, solid, null, x8r8g8b8, fast_composite_solid_fill),
+    PIXMAN_STD_FAST_PATH (SRC, solid, null, a8b8g8r8, fast_composite_solid_fill),
+    PIXMAN_STD_FAST_PATH (SRC, solid, null, x8b8g8r8, fast_composite_solid_fill),
+    PIXMAN_STD_FAST_PATH (SRC, solid, null, a8, fast_composite_solid_fill),
+    PIXMAN_STD_FAST_PATH (SRC, solid, null, r5g6b5, fast_composite_solid_fill),
+    PIXMAN_STD_FAST_PATH (SRC, a8r8g8b8, null, x8r8g8b8, fast_composite_src_8888_x888),
+    PIXMAN_STD_FAST_PATH (SRC, x8r8g8b8, null, x8r8g8b8, fast_composite_src_8888_x888),
+    PIXMAN_STD_FAST_PATH (SRC, a8b8g8r8, null, x8b8g8r8, fast_composite_src_8888_x888),
+    PIXMAN_STD_FAST_PATH (SRC, x8b8g8r8, null, x8b8g8r8, fast_composite_src_8888_x888),
+    PIXMAN_STD_FAST_PATH (SRC, a8r8g8b8, null, r5g6b5, fast_composite_src_x888_0565),
+    PIXMAN_STD_FAST_PATH (SRC, x8r8g8b8, null, r5g6b5, fast_composite_src_x888_0565),
+    PIXMAN_STD_FAST_PATH (SRC, a8b8g8r8, null, b5g6r5, fast_composite_src_x888_0565),
+    PIXMAN_STD_FAST_PATH (SRC, x8b8g8r8, null, b5g6r5, fast_composite_src_x888_0565),
+    PIXMAN_STD_FAST_PATH (IN, a8, null, a8, fast_composite_in_8_8),
+    PIXMAN_STD_FAST_PATH (IN, solid, a8, a8, fast_composite_in_n_8_8),
+
+#define SCALED_NEAREST_FLAGS						\
+    (FAST_PATH_SCALE_TRANSFORM	|					\
+     FAST_PATH_NO_ALPHA_MAP	|					\
+     FAST_PATH_NEAREST_FILTER	|					\
+     FAST_PATH_NO_ACCESSORS	|					\
+     FAST_PATH_NO_WIDE_FORMAT)
+
+#define HAS_NORMAL_REPEAT_FLAGS						\
+    (FAST_PATH_NO_REFLECT_REPEAT |					\
+     FAST_PATH_NO_PAD_REPEAT     |					\
+     FAST_PATH_NO_NONE_REPEAT)
+
+#define SIMPLE_NEAREST_FAST_PATH(op,s,d,func)				\
+    {   PIXMAN_OP_ ## op,						\
+	PIXMAN_ ## s,							\
+	SCALED_NEAREST_FLAGS | HAS_NORMAL_REPEAT_FLAGS | FAST_PATH_16BIT_SAFE | FAST_PATH_X_UNIT_POSITIVE, \
+	PIXMAN_null, 0,							\
+	PIXMAN_ ## d, FAST_PATH_STD_DEST_FLAGS,				\
+	fast_composite_scaled_nearest_ ## func ## _normal ## _ ## op,	\
+    },									\
+    {   PIXMAN_OP_ ## op,						\
+	PIXMAN_ ## s,							\
+	SCALED_NEAREST_FLAGS | FAST_PATH_SAMPLES_COVER_CLIP,		\
+	PIXMAN_null, 0,							\
+	PIXMAN_ ## d, FAST_PATH_STD_DEST_FLAGS,				\
+	fast_composite_scaled_nearest_ ## func ## _none ## _ ## op,	\
+    }
+    SIMPLE_NEAREST_FAST_PATH (SRC, x8r8g8b8, x8r8g8b8, x888_x888),
+    SIMPLE_NEAREST_FAST_PATH (SRC, a8r8g8b8, x8r8g8b8, x888_x888),
+    SIMPLE_NEAREST_FAST_PATH (SRC, x8b8g8r8, x8b8g8r8, x888_x888),
+    SIMPLE_NEAREST_FAST_PATH (SRC, a8b8g8r8, x8b8g8r8, x888_x888),
+
+    SIMPLE_NEAREST_FAST_PATH (SRC, a8r8g8b8, a8r8g8b8, x888_x888),
+    SIMPLE_NEAREST_FAST_PATH (SRC, a8b8g8r8, a8b8g8r8, x888_x888),
+
+    SIMPLE_NEAREST_FAST_PATH (SRC, x8r8g8b8, r5g6b5, x888_565),
+    SIMPLE_NEAREST_FAST_PATH (SRC, a8r8g8b8, r5g6b5, x888_565),
+
+    SIMPLE_NEAREST_FAST_PATH (SRC, r5g6b5, r5g6b5, 565_565),
+
+    SIMPLE_NEAREST_FAST_PATH (OVER, a8r8g8b8, x8r8g8b8, x888_x888),
+    SIMPLE_NEAREST_FAST_PATH (OVER, a8b8g8r8, x8b8g8r8, x888_x888),
+    SIMPLE_NEAREST_FAST_PATH (OVER, a8r8g8b8, a8r8g8b8, x888_x888),
+    SIMPLE_NEAREST_FAST_PATH (OVER, a8b8g8r8, a8b8g8r8, x888_x888),
+
+    SIMPLE_NEAREST_FAST_PATH (OVER, a8r8g8b8, r5g6b5, 8888_565),
+
+#define NEAREST_FAST_PATH(op,s,d)		\
+    {   PIXMAN_OP_ ## op,			\
+	PIXMAN_ ## s, SCALED_NEAREST_FLAGS,	\
+	PIXMAN_null, 0,				\
+	PIXMAN_ ## d, FAST_PATH_STD_DEST_FLAGS,	\
+	fast_composite_scaled_nearest,		\
+    }
+
+    NEAREST_FAST_PATH (SRC, x8r8g8b8, x8r8g8b8),
+    NEAREST_FAST_PATH (SRC, a8r8g8b8, x8r8g8b8),
+    NEAREST_FAST_PATH (SRC, x8b8g8r8, x8b8g8r8),
+    NEAREST_FAST_PATH (SRC, a8b8g8r8, x8b8g8r8),
+
+    NEAREST_FAST_PATH (SRC, x8r8g8b8, a8r8g8b8),
+    NEAREST_FAST_PATH (SRC, a8r8g8b8, a8r8g8b8),
+    NEAREST_FAST_PATH (SRC, x8b8g8r8, a8b8g8r8),
+    NEAREST_FAST_PATH (SRC, a8b8g8r8, a8b8g8r8),
+
+    NEAREST_FAST_PATH (OVER, x8r8g8b8, x8r8g8b8),
+    NEAREST_FAST_PATH (OVER, a8r8g8b8, x8r8g8b8),
+    NEAREST_FAST_PATH (OVER, x8b8g8r8, x8b8g8r8),
+    NEAREST_FAST_PATH (OVER, a8b8g8r8, x8b8g8r8),
+
+    NEAREST_FAST_PATH (OVER, x8r8g8b8, a8r8g8b8),
+    NEAREST_FAST_PATH (OVER, a8r8g8b8, a8r8g8b8),
+    NEAREST_FAST_PATH (OVER, x8b8g8r8, a8b8g8r8),
+    NEAREST_FAST_PATH (OVER, a8b8g8r8, a8b8g8r8),
+
+    {   PIXMAN_OP_NONE	},
+};
+
+static void
+pixman_fill8 (uint32_t *bits,
+              int       stride,
+              int       x,
+              int       y,
+              int       width,
+              int       height,
+              uint32_t xor)
+{
+    int byte_stride = stride * (int) sizeof (uint32_t);
+    uint8_t *dst = (uint8_t *) bits;
+    uint8_t v = xor & 0xff;
+    int i;
+
+    dst = dst + y * byte_stride + x;
+
+    while (height--)
+    {
+	for (i = 0; i < width; ++i)
+	    dst[i] = v;
+
+	dst += byte_stride;
+    }
+}
+
+static void
+pixman_fill16 (uint32_t *bits,
+               int       stride,
+               int       x,
+               int       y,
+               int       width,
+               int       height,
+               uint32_t xor)
+{
+    int short_stride =
+	(stride * (int)sizeof (uint32_t)) / (int)sizeof (uint16_t);
+    uint16_t *dst = (uint16_t *)bits;
+    uint16_t v = xor & 0xffff;
+    int i;
+
+    dst = dst + y * short_stride + x;
+
+    while (height--)
+    {
+	for (i = 0; i < width; ++i)
+	    dst[i] = v;
+
+	dst += short_stride;
+    }
+}
+
+static void
+pixman_fill32 (uint32_t *bits,
+               int       stride,
+               int       x,
+               int       y,
+               int       width,
+               int       height,
+               uint32_t  xor)
+{
+    int i;
+
+    bits = bits + y * stride + x;
+
+    while (height--)
+    {
+	for (i = 0; i < width; ++i)
+	    bits[i] = xor;
+
+	bits += stride;
+    }
+}
+
+static pixman_bool_t
+fast_path_fill (pixman_implementation_t *imp,
+                uint32_t *               bits,
+                int                      stride,
+                int                      bpp,
+                int                      x,
+                int                      y,
+                int                      width,
+                int                      height,
+                uint32_t		 xor)
+{
+    switch (bpp)
+    {
+    case 8:
+	pixman_fill8 (bits, stride, x, y, width, height, xor);
+	break;
+
+    case 16:
+	pixman_fill16 (bits, stride, x, y, width, height, xor);
+	break;
+
+    case 32:
+	pixman_fill32 (bits, stride, x, y, width, height, xor);
+	break;
+
+    default:
+	return _pixman_implementation_fill (
+	    imp->delegate, bits, stride, bpp, x, y, width, height, xor);
+	break;
+    }
+
+    return TRUE;
+}
+
+pixman_implementation_t *
+_pixman_implementation_create_fast_path (void)
+{
+    pixman_implementation_t *general = _pixman_implementation_create_general ();
+    pixman_implementation_t *imp = _pixman_implementation_create (general, c_fast_paths);
+
+    imp->fill = fast_path_fill;
+
+    return imp;
+}
diff --git a/pixman/pixman/pixman-general.c b/pixman/pixman/pixman-general.c
new file mode 100644
index 000000000..bddf79aae
--- /dev/null
+++ b/pixman/pixman/pixman-general.c
@@ -0,0 +1,320 @@
+/*
+ * Copyright © 2009 Red Hat, Inc.
+ * Copyright © 2000 SuSE, Inc.
+ * Copyright © 2007 Red Hat, Inc.
+ * Copyright © 2000 Keith Packard, member of The XFree86 Project, Inc.
+ *             2005 Lars Knoll & Zack Rusin, Trolltech
+ *             2008 Aaron Plattner, NVIDIA Corporation
+ *
+ * Permission to use, copy, modify, distribute, and sell this software and its
+ * documentation for any purpose is hereby granted without fee, provided that
+ * the above copyright notice appear in all copies and that both that
+ * copyright notice and this permission notice appear in supporting
+ * documentation, and that the name of Red Hat not be used in advertising or
+ * publicity pertaining to distribution of the software without specific,
+ * written prior permission.  Red Hat makes no representations about the
+ * suitability of this software for any purpose.  It is provided "as is"
+ * without express or implied warranty.
+ *
+ * THE COPYRIGHT HOLDERS DISCLAIM ALL WARRANTIES WITH REGARD TO THIS
+ * SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
+ * FITNESS, IN NO EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
+ * AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
+ * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS
+ * SOFTWARE.
+ */
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+#include <limits.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include "pixman-private.h"
+#include "pixman-combine32.h"
+#include "pixman-private.h"
+
+#define SCANLINE_BUFFER_LENGTH 8192
+
+static void
+general_composite_rect  (pixman_implementation_t *imp,
+                         pixman_op_t              op,
+                         pixman_image_t *         src,
+                         pixman_image_t *         mask,
+                         pixman_image_t *         dest,
+                         int32_t                  src_x,
+                         int32_t                  src_y,
+                         int32_t                  mask_x,
+                         int32_t                  mask_y,
+                         int32_t                  dest_x,
+                         int32_t                  dest_y,
+                         int32_t                  width,
+                         int32_t                  height)
+{
+    uint8_t stack_scanline_buffer[SCANLINE_BUFFER_LENGTH * 3];
+    const pixman_format_code_t src_format =
+	src->type == BITS ? src->bits.format : 0;
+    const pixman_format_code_t mask_format =
+	mask && mask->type == BITS ? mask->bits.format : 0;
+    const pixman_format_code_t dest_format =
+	dest->type == BITS ? dest->bits.format : 0;
+    const int src_wide = PIXMAN_FORMAT_IS_WIDE (src_format);
+    const int mask_wide = mask && PIXMAN_FORMAT_IS_WIDE (mask_format);
+    const int dest_wide = PIXMAN_FORMAT_IS_WIDE (dest_format);
+    const int wide = src_wide || mask_wide || dest_wide;
+    const int Bpp = wide ? 8 : 4;
+    uint8_t *scanline_buffer = stack_scanline_buffer;
+    uint8_t *src_buffer, *mask_buffer, *dest_buffer;
+    fetch_scanline_t fetch_src = NULL, fetch_mask = NULL, fetch_dest = NULL;
+    pixman_combine_32_func_t compose;
+    store_scanline_t store;
+    source_image_class_t src_class, mask_class;
+    pixman_bool_t component_alpha;
+    uint32_t *bits;
+    int32_t stride;
+    int i;
+
+    if (width * Bpp > SCANLINE_BUFFER_LENGTH)
+    {
+	scanline_buffer = pixman_malloc_abc (width, 3, Bpp);
+
+	if (!scanline_buffer)
+	    return;
+    }
+
+    src_buffer = scanline_buffer;
+    mask_buffer = src_buffer + width * Bpp;
+    dest_buffer = mask_buffer + width * Bpp;
+
+    src_class = _pixman_image_classify (src,
+                                        src_x, src_y,
+                                        width, height);
+
+    mask_class = SOURCE_IMAGE_CLASS_UNKNOWN;
+
+    if (mask)
+    {
+	mask_class = _pixman_image_classify (mask,
+	                                     src_x, src_y,
+	                                     width, height);
+    }
+
+    if (op == PIXMAN_OP_CLEAR)
+	fetch_src = NULL;
+    else if (wide)
+	fetch_src = _pixman_image_get_scanline_64;
+    else
+	fetch_src = _pixman_image_get_scanline_32;
+
+    if (!mask || op == PIXMAN_OP_CLEAR)
+	fetch_mask = NULL;
+    else if (wide)
+	fetch_mask = _pixman_image_get_scanline_64;
+    else
+	fetch_mask = _pixman_image_get_scanline_32;
+
+    if (op == PIXMAN_OP_CLEAR || op == PIXMAN_OP_SRC)
+	fetch_dest = NULL;
+    else if (wide)
+	fetch_dest = _pixman_image_get_scanline_64;
+    else
+	fetch_dest = _pixman_image_get_scanline_32;
+
+    if (wide)
+	store = _pixman_image_store_scanline_64;
+    else
+	store = _pixman_image_store_scanline_32;
+
+    /* Skip the store step and composite directly into the
+     * destination if the output format of the compose func matches
+     * the destination format.
+     *
+     * If the destination format is a8r8g8b8 then we can always do
+     * this. If it is x8r8g8b8, then we can only do it if the
+     * operator doesn't make use of destination alpha.
+     */
+    if ((dest->bits.format == PIXMAN_a8r8g8b8)	||
+	(dest->bits.format == PIXMAN_x8r8g8b8	&&
+	 (op == PIXMAN_OP_OVER		||
+	  op == PIXMAN_OP_ADD		||
+	  op == PIXMAN_OP_SRC		||
+	  op == PIXMAN_OP_CLEAR		||
+	  op == PIXMAN_OP_IN_REVERSE	||
+	  op == PIXMAN_OP_OUT_REVERSE	||
+	  op == PIXMAN_OP_DST)))
+    {
+	if (!wide &&
+	    !dest->common.alpha_map &&
+	    !dest->bits.write_func)
+	{
+	    store = NULL;
+	}
+    }
+
+    if (!store)
+    {
+	bits = dest->bits.bits;
+	stride = dest->bits.rowstride;
+    }
+    else
+    {
+	bits = NULL;
+	stride = 0;
+    }
+
+    component_alpha =
+        fetch_src                       &&
+        fetch_mask                      &&
+        mask                            &&
+        mask->common.type == BITS       &&
+        mask->common.component_alpha    &&
+        PIXMAN_FORMAT_RGB (mask->bits.format);
+
+    if (wide)
+    {
+	if (component_alpha)
+	    compose = (pixman_combine_32_func_t)_pixman_implementation_combine_64_ca;
+	else
+	    compose = (pixman_combine_32_func_t)_pixman_implementation_combine_64;
+    }
+    else
+    {
+	if (component_alpha)
+	    compose = _pixman_implementation_combine_32_ca;
+	else
+	    compose = _pixman_implementation_combine_32;
+    }
+
+    if (!compose)
+	return;
+
+    if (!fetch_mask)
+	mask_buffer = NULL;
+
+    for (i = 0; i < height; ++i)
+    {
+	/* fill first half of scanline with source */
+	if (fetch_src)
+	{
+	    if (fetch_mask)
+	    {
+		/* fetch mask before source so that fetching of
+		   source can be optimized */
+		fetch_mask (mask, mask_x, mask_y + i,
+		            width, (void *)mask_buffer, 0, 0);
+
+		if (mask_class == SOURCE_IMAGE_CLASS_HORIZONTAL)
+		    fetch_mask = NULL;
+	    }
+
+	    if (src_class == SOURCE_IMAGE_CLASS_HORIZONTAL)
+	    {
+		fetch_src (src, src_x, src_y + i,
+		           width, (void *)src_buffer, 0, 0);
+		fetch_src = NULL;
+	    }
+	    else
+	    {
+		fetch_src (src, src_x, src_y + i,
+		           width, (void *)src_buffer, (void *)mask_buffer,
+		           0xffffffff);
+	    }
+	}
+	else if (fetch_mask)
+	{
+	    fetch_mask (mask, mask_x, mask_y + i,
+	                width, (void *)mask_buffer, 0, 0);
+	}
+
+	if (store)
+	{
+	    /* fill dest into second half of scanline */
+	    if (fetch_dest)
+	    {
+		fetch_dest (dest, dest_x, dest_y + i,
+		            width, (void *)dest_buffer, 0, 0);
+	    }
+
+	    /* blend */
+	    compose (imp->toplevel, op,
+		     (void *)dest_buffer,
+		     (void *)src_buffer,
+		     (void *)mask_buffer,
+		     width);
+
+	    /* write back */
+	    store (&(dest->bits), dest_x, dest_y + i, width,
+	           (void *)dest_buffer);
+	}
+	else
+	{
+	    /* blend */
+	    compose (imp->toplevel, op,
+		     bits + (dest_y + i) * stride + dest_x,
+	             (void *)src_buffer, (void *)mask_buffer, width);
+	}
+    }
+
+    if (scanline_buffer != stack_scanline_buffer)
+	free (scanline_buffer);
+}
+
+static const pixman_fast_path_t general_fast_path[] =
+{
+    { PIXMAN_OP_any, PIXMAN_any, 0, PIXMAN_any,	0, PIXMAN_any, 0, general_composite_rect },
+    { PIXMAN_OP_NONE }
+};
+
+static pixman_bool_t
+general_blt (pixman_implementation_t *imp,
+             uint32_t *               src_bits,
+             uint32_t *               dst_bits,
+             int                      src_stride,
+             int                      dst_stride,
+             int                      src_bpp,
+             int                      dst_bpp,
+             int                      src_x,
+             int                      src_y,
+             int                      dst_x,
+             int                      dst_y,
+             int                      width,
+             int                      height)
+{
+    /* We can't blit unless we have sse2 or mmx */
+
+    return FALSE;
+}
+
+static pixman_bool_t
+general_fill (pixman_implementation_t *imp,
+              uint32_t *               bits,
+              int                      stride,
+              int                      bpp,
+              int                      x,
+              int                      y,
+              int                      width,
+              int                      height,
+              uint32_t xor)
+{
+    return FALSE;
+}
+
+pixman_implementation_t *
+_pixman_implementation_create_general (void)
+{
+    pixman_implementation_t *imp = _pixman_implementation_create (NULL, general_fast_path);
+
+    _pixman_setup_combiner_functions_32 (imp);
+    _pixman_setup_combiner_functions_64 (imp);
+
+    imp->blt = general_blt;
+    imp->fill = general_fill;
+
+    return imp;
+}
+
diff --git a/pixman/pixman/pixman-gradient-walker.c b/pixman/pixman/pixman-gradient-walker.c
new file mode 100644
index 000000000..dd666b412
--- /dev/null
+++ b/pixman/pixman/pixman-gradient-walker.c
@@ -0,0 +1,254 @@
+/*
+ *
+ * Copyright © 2000 Keith Packard, member of The XFree86 Project, Inc.
+ *             2005 Lars Knoll & Zack Rusin, Trolltech
+ *
+ * Permission to use, copy, modify, distribute, and sell this software and its
+ * documentation for any purpose is hereby granted without fee, provided that
+ * the above copyright notice appear in all copies and that both that
+ * copyright notice and this permission notice appear in supporting
+ * documentation, and that the name of Keith Packard not be used in
+ * advertising or publicity pertaining to distribution of the software without
+ * specific, written prior permission.  Keith Packard makes no
+ * representations about the suitability of this software for any purpose.  It
+ * is provided "as is" without express or implied warranty.
+ *
+ * THE COPYRIGHT HOLDERS DISCLAIM ALL WARRANTIES WITH REGARD TO THIS
+ * SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
+ * FITNESS, IN NO EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
+ * AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
+ * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS
+ * SOFTWARE.
+ */
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+#include "pixman-private.h"
+
+void
+_pixman_gradient_walker_init (pixman_gradient_walker_t *walker,
+                              gradient_t *              gradient,
+                              unsigned int              spread)
+{
+    walker->num_stops = gradient->n_stops;
+    walker->stops     = gradient->stops;
+    walker->left_x    = 0;
+    walker->right_x   = 0x10000;
+    walker->stepper   = 0;
+    walker->left_ag   = 0;
+    walker->left_rb   = 0;
+    walker->right_ag  = 0;
+    walker->right_rb  = 0;
+    walker->spread    = spread;
+
+    walker->need_reset = TRUE;
+}
+
+void
+_pixman_gradient_walker_reset (pixman_gradient_walker_t *walker,
+                               pixman_fixed_32_32_t      pos)
+{
+    int32_t x, left_x, right_x;
+    pixman_color_t          *left_c, *right_c;
+    int n, count = walker->num_stops;
+    pixman_gradient_stop_t *      stops = walker->stops;
+
+    static const pixman_color_t transparent_black = { 0, 0, 0, 0 };
+
+    switch (walker->spread)
+    {
+    case PIXMAN_REPEAT_NORMAL:
+	x = (int32_t)pos & 0xFFFF;
+	for (n = 0; n < count; n++)
+	    if (x < stops[n].x)
+		break;
+	if (n == 0)
+	{
+	    left_x =  stops[count - 1].x - 0x10000;
+	    left_c = &stops[count - 1].color;
+	}
+	else
+	{
+	    left_x =  stops[n - 1].x;
+	    left_c = &stops[n - 1].color;
+	}
+
+	if (n == count)
+	{
+	    right_x =  stops[0].x + 0x10000;
+	    right_c = &stops[0].color;
+	}
+	else
+	{
+	    right_x =  stops[n].x;
+	    right_c = &stops[n].color;
+	}
+	left_x  += (pos - x);
+	right_x += (pos - x);
+	break;
+
+    case PIXMAN_REPEAT_PAD:
+	for (n = 0; n < count; n++)
+	    if (pos < stops[n].x)
+		break;
+
+	if (n == 0)
+	{
+	    left_x =  INT32_MIN;
+	    left_c = &stops[0].color;
+	}
+	else
+	{
+	    left_x =  stops[n - 1].x;
+	    left_c = &stops[n - 1].color;
+	}
+
+	if (n == count)
+	{
+	    right_x =  INT32_MAX;
+	    right_c = &stops[n - 1].color;
+	}
+	else
+	{
+	    right_x =  stops[n].x;
+	    right_c = &stops[n].color;
+	}
+	break;
+
+    case PIXMAN_REPEAT_REFLECT:
+	x = (int32_t)pos & 0xFFFF;
+	if ((int32_t)pos & 0x10000)
+	    x = 0x10000 - x;
+	for (n = 0; n < count; n++)
+	    if (x < stops[n].x)
+		break;
+
+	if (n == 0)
+	{
+	    left_x =  -stops[0].x;
+	    left_c = &stops[0].color;
+	}
+	else
+	{
+	    left_x =  stops[n - 1].x;
+	    left_c = &stops[n - 1].color;
+	}
+
+	if (n == count)
+	{
+	    right_x = 0x20000 - stops[n - 1].x;
+	    right_c = &stops[n - 1].color;
+	}
+	else
+	{
+	    right_x =  stops[n].x;
+	    right_c = &stops[n].color;
+	}
+
+	if ((int32_t)pos & 0x10000)
+	{
+	    pixman_color_t  *tmp_c;
+	    int32_t tmp_x;
+
+	    tmp_x   = 0x10000 - right_x;
+	    right_x = 0x10000 - left_x;
+	    left_x  = tmp_x;
+
+	    tmp_c   = right_c;
+	    right_c = left_c;
+	    left_c  = tmp_c;
+
+	    x = 0x10000 - x;
+	}
+	left_x  += (pos - x);
+	right_x += (pos - x);
+	break;
+
+    default:  /* REPEAT_NONE */
+	for (n = 0; n < count; n++)
+	    if (pos < stops[n].x)
+		break;
+
+	if (n == 0)
+	{
+	    left_x  =  INT32_MIN;
+	    right_x =  stops[0].x;
+	    left_c  = right_c = (pixman_color_t*) &transparent_black;
+	}
+	else if (n == count)
+	{
+	    left_x  = stops[n - 1].x;
+	    right_x = INT32_MAX;
+	    left_c  = right_c = (pixman_color_t*) &transparent_black;
+	}
+	else
+	{
+	    left_x  =  stops[n - 1].x;
+	    right_x =  stops[n].x;
+	    left_c  = &stops[n - 1].color;
+	    right_c = &stops[n].color;
+	}
+    }
+
+    walker->left_x   = left_x;
+    walker->right_x  = right_x;
+    walker->left_ag  = ((left_c->alpha >> 8) << 16)   | (left_c->green >> 8);
+    walker->left_rb  = ((left_c->red & 0xff00) << 8)  | (left_c->blue >> 8);
+    walker->right_ag = ((right_c->alpha >> 8) << 16)  | (right_c->green >> 8);
+    walker->right_rb = ((right_c->red & 0xff00) << 8) | (right_c->blue >> 8);
+
+    if (walker->left_x == walker->right_x                ||
+        ( walker->left_ag == walker->right_ag &&
+          walker->left_rb == walker->right_rb )   )
+    {
+	walker->stepper = 0;
+    }
+    else
+    {
+	int32_t width = right_x - left_x;
+	walker->stepper = ((1 << 24) + width / 2) / width;
+    }
+
+    walker->need_reset = FALSE;
+}
+
+#define  PIXMAN_GRADIENT_WALKER_NEED_RESET(w, x)                         \
+    ( (w)->need_reset || (x) < (w)->left_x || (x) >= (w)->right_x)
+
+
+/* the following assumes that PIXMAN_GRADIENT_WALKER_NEED_RESET(w,x) is FALSE */
+uint32_t
+_pixman_gradient_walker_pixel (pixman_gradient_walker_t *walker,
+                               pixman_fixed_32_32_t      x)
+{
+    int dist, idist;
+    uint32_t t1, t2, a, color;
+
+    if (PIXMAN_GRADIENT_WALKER_NEED_RESET (walker, x))
+	_pixman_gradient_walker_reset (walker, x);
+
+    dist  = ((int)(x - walker->left_x) * walker->stepper) >> 16;
+    idist = 256 - dist;
+
+    /* combined INTERPOLATE and premultiply */
+    t1 = walker->left_rb * idist + walker->right_rb * dist;
+    t1 = (t1 >> 8) & 0xff00ff;
+
+    t2  = walker->left_ag * idist + walker->right_ag * dist;
+    t2 &= 0xff00ff00;
+
+    color = t2 & 0xff000000;
+    a     = t2 >> 24;
+
+    t1  = t1 * a + 0x800080;
+    t1  = (t1 + ((t1 >> 8) & 0xff00ff)) >> 8;
+
+    t2  = (t2 >> 8) * a + 0x800080;
+    t2  = (t2 + ((t2 >> 8) & 0xff00ff));
+
+    return (color | (t1 & 0xff00ff) | (t2 & 0xff00));
+}
+
diff --git a/pixman/pixman/pixman-image.c b/pixman/pixman/pixman-image.c
new file mode 100644
index 000000000..03a39db87
--- /dev/null
+++ b/pixman/pixman/pixman-image.c
@@ -0,0 +1,762 @@
+/*
+ * Copyright © 2000 SuSE, Inc.
+ * Copyright © 2007 Red Hat, Inc.
+ *
+ * Permission to use, copy, modify, distribute, and sell this software and its
+ * documentation for any purpose is hereby granted without fee, provided that
+ * the above copyright notice appear in all copies and that both that
+ * copyright notice and this permission notice appear in supporting
+ * documentation, and that the name of SuSE not be used in advertising or
+ * publicity pertaining to distribution of the software without specific,
+ * written prior permission.  SuSE makes no representations about the
+ * suitability of this software for any purpose.  It is provided "as is"
+ * without express or implied warranty.
+ *
+ * SuSE DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING ALL
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL SuSE
+ * BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
+ * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+ * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <assert.h>
+
+#include "pixman-private.h"
+#include "pixman-combine32.h"
+
+pixman_bool_t
+_pixman_init_gradient (gradient_t *                  gradient,
+                       const pixman_gradient_stop_t *stops,
+                       int                           n_stops)
+{
+    return_val_if_fail (n_stops > 0, FALSE);
+
+    gradient->stops = pixman_malloc_ab (n_stops, sizeof (pixman_gradient_stop_t));
+    if (!gradient->stops)
+	return FALSE;
+
+    memcpy (gradient->stops, stops, n_stops * sizeof (pixman_gradient_stop_t));
+
+    gradient->n_stops = n_stops;
+
+    gradient->stop_range = 0xffff;
+    gradient->common.class = SOURCE_IMAGE_CLASS_UNKNOWN;
+
+    return TRUE;
+}
+
+/*
+ * By default, just evaluate the image at 32bpp and expand.  Individual image
+ * types can plug in a better scanline getter if they want to. For example
+ * we  could produce smoother gradients by evaluating them at higher color
+ * depth, but that's a project for the future.
+ */
+void
+_pixman_image_get_scanline_generic_64 (pixman_image_t * image,
+                                       int              x,
+                                       int              y,
+                                       int              width,
+                                       uint32_t *       buffer,
+                                       const uint32_t * mask,
+                                       uint32_t         mask_bits)
+{
+    uint32_t *mask8 = NULL;
+
+    /* Contract the mask image, if one exists, so that the 32-bit fetch
+     * function can use it.
+     */
+    if (mask)
+    {
+	mask8 = pixman_malloc_ab (width, sizeof(uint32_t));
+	if (!mask8)
+	    return;
+
+	pixman_contract (mask8, (uint64_t *)mask, width);
+    }
+
+    /* Fetch the source image into the first half of buffer. */
+    _pixman_image_get_scanline_32 (image, x, y, width, (uint32_t*)buffer, mask8,
+                                   mask_bits);
+
+    /* Expand from 32bpp to 64bpp in place. */
+    pixman_expand ((uint64_t *)buffer, buffer, PIXMAN_a8r8g8b8, width);
+
+    free (mask8);
+}
+
+pixman_image_t *
+_pixman_image_allocate (void)
+{
+    pixman_image_t *image = malloc (sizeof (pixman_image_t));
+
+    if (image)
+    {
+	image_common_t *common = &image->common;
+
+	pixman_region32_init (&common->clip_region);
+
+	common->have_clip_region = FALSE;
+	common->clip_sources = FALSE;
+	common->transform = NULL;
+	common->repeat = PIXMAN_REPEAT_NONE;
+	common->filter = PIXMAN_FILTER_NEAREST;
+	common->filter_params = NULL;
+	common->n_filter_params = 0;
+	common->alpha_map = NULL;
+	common->component_alpha = FALSE;
+	common->ref_count = 1;
+	common->classify = NULL;
+	common->client_clip = FALSE;
+	common->destroy_func = NULL;
+	common->destroy_data = NULL;
+	common->dirty = TRUE;
+    }
+
+    return image;
+}
+
+source_image_class_t
+_pixman_image_classify (pixman_image_t *image,
+                        int             x,
+                        int             y,
+                        int             width,
+                        int             height)
+{
+    if (image->common.classify)
+	return image->common.classify (image, x, y, width, height);
+    else
+	return SOURCE_IMAGE_CLASS_UNKNOWN;
+}
+
+void
+_pixman_image_get_scanline_32 (pixman_image_t *image,
+                               int             x,
+                               int             y,
+                               int             width,
+                               uint32_t *      buffer,
+                               const uint32_t *mask,
+                               uint32_t        mask_bits)
+{
+    image->common.get_scanline_32 (image, x, y, width, buffer, mask, mask_bits);
+}
+
+/* Even thought the type of buffer is uint32_t *, the function actually expects
+ * a uint64_t *buffer.
+ */
+void
+_pixman_image_get_scanline_64 (pixman_image_t *image,
+                               int             x,
+                               int             y,
+                               int             width,
+                               uint32_t *      buffer,
+                               const uint32_t *unused,
+                               uint32_t        unused2)
+{
+    image->common.get_scanline_64 (image, x, y, width, buffer, unused, unused2);
+}
+
+static void
+image_property_changed (pixman_image_t *image)
+{
+    image->common.dirty = TRUE;
+}
+
+/* Ref Counting */
+PIXMAN_EXPORT pixman_image_t *
+pixman_image_ref (pixman_image_t *image)
+{
+    image->common.ref_count++;
+
+    return image;
+}
+
+/* returns TRUE when the image is freed */
+PIXMAN_EXPORT pixman_bool_t
+pixman_image_unref (pixman_image_t *image)
+{
+    image_common_t *common = (image_common_t *)image;
+
+    common->ref_count--;
+
+    if (common->ref_count == 0)
+    {
+	if (image->common.destroy_func)
+	    image->common.destroy_func (image, image->common.destroy_data);
+
+	pixman_region32_fini (&common->clip_region);
+
+	if (common->transform)
+	    free (common->transform);
+
+	if (common->filter_params)
+	    free (common->filter_params);
+
+	if (common->alpha_map)
+	    pixman_image_unref ((pixman_image_t *)common->alpha_map);
+
+	if (image->type == LINEAR ||
+	    image->type == RADIAL ||
+	    image->type == CONICAL)
+	{
+	    if (image->gradient.stops)
+		free (image->gradient.stops);
+	}
+
+	if (image->type == BITS && image->bits.free_me)
+	    free (image->bits.free_me);
+
+	free (image);
+
+	return TRUE;
+    }
+
+    return FALSE;
+}
+
+PIXMAN_EXPORT void
+pixman_image_set_destroy_function (pixman_image_t *            image,
+                                   pixman_image_destroy_func_t func,
+                                   void *                      data)
+{
+    image->common.destroy_func = func;
+    image->common.destroy_data = data;
+}
+
+PIXMAN_EXPORT void *
+pixman_image_get_destroy_data (pixman_image_t *image)
+{
+  return image->common.destroy_data;
+}
+
+void
+_pixman_image_reset_clip_region (pixman_image_t *image)
+{
+    image->common.have_clip_region = FALSE;
+}
+
+static pixman_bool_t out_of_bounds_workaround = TRUE;
+
+/* Old X servers rely on out-of-bounds accesses when they are asked
+ * to composite with a window as the source. They create a pixman image
+ * pointing to some bogus position in memory, but then they set a clip
+ * region to the position where the actual bits are.
+ *
+ * Due to a bug in old versions of pixman, where it would not clip
+ * against the image bounds when a clip region was set, this would
+ * actually work. So by default we allow certain out-of-bound access
+ * to happen unless explicitly disabled.
+ *
+ * Fixed X servers should call this function to disable the workaround.
+ */
+PIXMAN_EXPORT void
+pixman_disable_out_of_bounds_workaround (void)
+{
+    out_of_bounds_workaround = FALSE;
+}
+
+static pixman_bool_t
+source_image_needs_out_of_bounds_workaround (bits_image_t *image)
+{
+    if (image->common.clip_sources                      &&
+        image->common.repeat == PIXMAN_REPEAT_NONE      &&
+	image->common.have_clip_region			&&
+        out_of_bounds_workaround)
+    {
+	if (!image->common.client_clip)
+	{
+	    /* There is no client clip, so if the clip region extends beyond the
+	     * drawable geometry, it must be because the X server generated the
+	     * bogus clip region.
+	     */
+	    const pixman_box32_t *extents =
+		pixman_region32_extents (&image->common.clip_region);
+
+	    if (extents->x1 >= 0 && extents->x2 <= image->width &&
+		extents->y1 >= 0 && extents->y2 <= image->height)
+	    {
+		return FALSE;
+	    }
+	}
+
+	return TRUE;
+    }
+
+    return FALSE;
+}
+
+static void
+compute_image_info (pixman_image_t *image)
+{
+    pixman_format_code_t code;
+    uint32_t flags = 0;
+
+    /* Transform */
+    if (!image->common.transform)
+    {
+	flags |= (FAST_PATH_ID_TRANSFORM | FAST_PATH_X_UNIT_POSITIVE);
+    }
+    else
+    {
+	if (image->common.transform->matrix[0][1] == 0 &&
+	    image->common.transform->matrix[1][0] == 0 &&
+	    image->common.transform->matrix[2][0] == 0 &&
+	    image->common.transform->matrix[2][1] == 0 &&
+	    image->common.transform->matrix[2][2] == pixman_fixed_1)
+	{
+	    flags |= FAST_PATH_SCALE_TRANSFORM;
+	}
+
+	if (image->common.transform->matrix[0][0] > 0)
+	    flags |= FAST_PATH_X_UNIT_POSITIVE;
+    }
+
+    /* Alpha map */
+    if (!image->common.alpha_map)
+	flags |= FAST_PATH_NO_ALPHA_MAP;
+
+    /* Filter */
+    switch (image->common.filter)
+    {
+    case PIXMAN_FILTER_NEAREST:
+    case PIXMAN_FILTER_FAST:
+	flags |= (FAST_PATH_NEAREST_FILTER | FAST_PATH_NO_CONVOLUTION_FILTER);
+	break;
+
+    case PIXMAN_FILTER_CONVOLUTION:
+	break;
+
+    default:
+	flags |= FAST_PATH_NO_CONVOLUTION_FILTER;
+	break;
+    }
+
+    /* Repeat mode */
+    switch (image->common.repeat)
+    {
+    case PIXMAN_REPEAT_NONE:
+	flags |= FAST_PATH_NO_REFLECT_REPEAT | FAST_PATH_NO_PAD_REPEAT;
+	break;
+
+    case PIXMAN_REPEAT_REFLECT:
+	flags |= FAST_PATH_NO_PAD_REPEAT | FAST_PATH_NO_NONE_REPEAT;
+	break;
+
+    case PIXMAN_REPEAT_PAD:
+	flags |= FAST_PATH_NO_REFLECT_REPEAT | FAST_PATH_NO_NONE_REPEAT;
+	break;
+
+    default:
+	flags |= FAST_PATH_NO_REFLECT_REPEAT | FAST_PATH_NO_PAD_REPEAT | FAST_PATH_NO_NONE_REPEAT;
+	break;
+    }
+
+    /* Component alpha */
+    if (image->common.component_alpha)
+	flags |= FAST_PATH_COMPONENT_ALPHA;
+    else
+	flags |= FAST_PATH_UNIFIED_ALPHA;
+
+    flags |= (FAST_PATH_NO_ACCESSORS | FAST_PATH_NO_WIDE_FORMAT);
+
+    /* Type specific checks */
+    switch (image->type)
+    {
+    case SOLID:
+	code = PIXMAN_solid;
+
+	if (image->solid.color.alpha == 0xffff)
+	    flags |= FAST_PATH_IS_OPAQUE;
+	break;
+
+    case BITS:
+	if (image->bits.width == 1	&&
+	    image->bits.height == 1	&&
+	    image->common.repeat != PIXMAN_REPEAT_NONE)
+	{
+	    code = PIXMAN_solid;
+	}
+	else
+	{
+	    code = image->bits.format;
+
+	    if (!image->common.transform &&
+		image->common.repeat == PIXMAN_REPEAT_NORMAL)
+	    {
+		flags |= FAST_PATH_SIMPLE_REPEAT;
+	    }
+	}
+
+	if (image->common.repeat != PIXMAN_REPEAT_NONE				&&
+	    !PIXMAN_FORMAT_A (image->bits.format)				&&
+	    PIXMAN_FORMAT_TYPE (image->bits.format) != PIXMAN_TYPE_GRAY		&&
+	    PIXMAN_FORMAT_TYPE (image->bits.format) != PIXMAN_TYPE_COLOR)
+	{
+	    flags |= FAST_PATH_IS_OPAQUE;
+	}
+
+	if (source_image_needs_out_of_bounds_workaround (&image->bits))
+	    flags |= FAST_PATH_NEEDS_WORKAROUND;
+
+	if (image->bits.read_func || image->bits.write_func)
+	    flags &= ~FAST_PATH_NO_ACCESSORS;
+
+	if (PIXMAN_FORMAT_IS_WIDE (image->bits.format))
+	    flags &= ~FAST_PATH_NO_WIDE_FORMAT;
+	break;
+
+    case LINEAR:
+    case RADIAL:
+	code = PIXMAN_unknown;
+
+	if (image->common.repeat != PIXMAN_REPEAT_NONE)
+	{
+	    int i;
+
+	    flags |= FAST_PATH_IS_OPAQUE;
+	    for (i = 0; i < image->gradient.n_stops; ++i)
+	    {
+		if (image->gradient.stops[i].color.alpha != 0xffff)
+		{
+		    flags &= ~FAST_PATH_IS_OPAQUE;
+		    break;
+		}
+	    }
+	}
+	break;
+
+    default:
+	code = PIXMAN_unknown;
+	break;
+    }
+
+    /* Both alpha maps and convolution filters can introduce
+     * non-opaqueness in otherwise opaque images. Also
+     * an image with component alpha turned on is only opaque
+     * if all channels are opaque, so we simply turn it off
+     * unconditionally for those images.
+     */
+    if (image->common.alpha_map					||
+	image->common.filter == PIXMAN_FILTER_CONVOLUTION	||
+	image->common.component_alpha)
+    {
+	flags &= ~FAST_PATH_IS_OPAQUE;
+    }
+
+    image->common.flags = flags;
+    image->common.extended_format_code = code;
+}
+
+void
+_pixman_image_validate (pixman_image_t *image)
+{
+    if (image->common.dirty)
+    {
+	compute_image_info (image);
+
+	/* It is important that property_changed is
+	 * called *after* compute_image_info() because
+	 * property_changed() can make use of the flags
+	 * to set up accessors etc.
+	 */
+	image->common.property_changed (image);
+
+	image->common.dirty = FALSE;
+    }
+
+    if (image->common.alpha_map)
+	_pixman_image_validate ((pixman_image_t *)image->common.alpha_map);
+}
+
+PIXMAN_EXPORT pixman_bool_t
+pixman_image_set_clip_region32 (pixman_image_t *   image,
+                                pixman_region32_t *region)
+{
+    image_common_t *common = (image_common_t *)image;
+    pixman_bool_t result;
+
+    if (region)
+    {
+	if ((result = pixman_region32_copy (&common->clip_region, region)))
+	    image->common.have_clip_region = TRUE;
+    }
+    else
+    {
+	_pixman_image_reset_clip_region (image);
+
+	result = TRUE;
+    }
+
+    image_property_changed (image);
+
+    return result;
+}
+
+PIXMAN_EXPORT pixman_bool_t
+pixman_image_set_clip_region (pixman_image_t *   image,
+                              pixman_region16_t *region)
+{
+    image_common_t *common = (image_common_t *)image;
+    pixman_bool_t result;
+
+    if (region)
+    {
+	if ((result = pixman_region32_copy_from_region16 (&common->clip_region, region)))
+	    image->common.have_clip_region = TRUE;
+    }
+    else
+    {
+	_pixman_image_reset_clip_region (image);
+
+	result = TRUE;
+    }
+
+    image_property_changed (image);
+
+    return result;
+}
+
+PIXMAN_EXPORT void
+pixman_image_set_has_client_clip (pixman_image_t *image,
+                                  pixman_bool_t   client_clip)
+{
+    image->common.client_clip = client_clip;
+}
+
+PIXMAN_EXPORT pixman_bool_t
+pixman_image_set_transform (pixman_image_t *          image,
+                            const pixman_transform_t *transform)
+{
+    static const pixman_transform_t id =
+    {
+	{ { pixman_fixed_1, 0, 0 },
+	  { 0, pixman_fixed_1, 0 },
+	  { 0, 0, pixman_fixed_1 } }
+    };
+
+    image_common_t *common = (image_common_t *)image;
+    pixman_bool_t result;
+
+    if (common->transform == transform)
+	return TRUE;
+
+    if (memcmp (&id, transform, sizeof (pixman_transform_t)) == 0)
+    {
+	free (common->transform);
+	common->transform = NULL;
+	result = TRUE;
+
+	goto out;
+    }
+
+    if (common->transform == NULL)
+	common->transform = malloc (sizeof (pixman_transform_t));
+
+    if (common->transform == NULL)
+    {
+	result = FALSE;
+
+	goto out;
+    }
+
+    memcpy (common->transform, transform, sizeof(pixman_transform_t));
+
+    result = TRUE;
+
+out:
+    image_property_changed (image);
+
+    return result;
+}
+
+PIXMAN_EXPORT void
+pixman_image_set_repeat (pixman_image_t *image,
+                         pixman_repeat_t repeat)
+{
+    image->common.repeat = repeat;
+
+    image_property_changed (image);
+}
+
+PIXMAN_EXPORT pixman_bool_t
+pixman_image_set_filter (pixman_image_t *      image,
+                         pixman_filter_t       filter,
+                         const pixman_fixed_t *params,
+                         int                   n_params)
+{
+    image_common_t *common = (image_common_t *)image;
+    pixman_fixed_t *new_params;
+
+    if (params == common->filter_params && filter == common->filter)
+	return TRUE;
+
+    new_params = NULL;
+    if (params)
+    {
+	new_params = pixman_malloc_ab (n_params, sizeof (pixman_fixed_t));
+	if (!new_params)
+	    return FALSE;
+
+	memcpy (new_params,
+	        params, n_params * sizeof (pixman_fixed_t));
+    }
+
+    common->filter = filter;
+
+    if (common->filter_params)
+	free (common->filter_params);
+
+    common->filter_params = new_params;
+    common->n_filter_params = n_params;
+
+    image_property_changed (image);
+    return TRUE;
+}
+
+PIXMAN_EXPORT void
+pixman_image_set_source_clipping (pixman_image_t *image,
+                                  pixman_bool_t   clip_sources)
+{
+    image->common.clip_sources = clip_sources;
+
+    image_property_changed (image);
+}
+
+/* Unlike all the other property setters, this function does not
+ * copy the content of indexed. Doing this copying is simply
+ * way, way too expensive.
+ */
+PIXMAN_EXPORT void
+pixman_image_set_indexed (pixman_image_t *        image,
+                          const pixman_indexed_t *indexed)
+{
+    bits_image_t *bits = (bits_image_t *)image;
+
+    bits->indexed = indexed;
+
+    image_property_changed (image);
+}
+
+PIXMAN_EXPORT void
+pixman_image_set_alpha_map (pixman_image_t *image,
+                            pixman_image_t *alpha_map,
+                            int16_t         x,
+                            int16_t         y)
+{
+    image_common_t *common = (image_common_t *)image;
+
+    return_if_fail (!alpha_map || alpha_map->type == BITS);
+
+    if (common->alpha_map != (bits_image_t *)alpha_map)
+    {
+	if (common->alpha_map)
+	    pixman_image_unref ((pixman_image_t *)common->alpha_map);
+
+	if (alpha_map)
+	    common->alpha_map = (bits_image_t *)pixman_image_ref (alpha_map);
+	else
+	    common->alpha_map = NULL;
+    }
+
+    common->alpha_origin_x = x;
+    common->alpha_origin_y = y;
+
+    image_property_changed (image);
+}
+
+PIXMAN_EXPORT void
+pixman_image_set_component_alpha   (pixman_image_t *image,
+                                    pixman_bool_t   component_alpha)
+{
+    image->common.component_alpha = component_alpha;
+
+    image_property_changed (image);
+}
+
+PIXMAN_EXPORT void
+pixman_image_set_accessors (pixman_image_t *           image,
+                            pixman_read_memory_func_t  read_func,
+                            pixman_write_memory_func_t write_func)
+{
+    return_if_fail (image != NULL);
+
+    if (image->type == BITS)
+    {
+	image->bits.read_func = read_func;
+	image->bits.write_func = write_func;
+
+	image_property_changed (image);
+    }
+}
+
+PIXMAN_EXPORT uint32_t *
+pixman_image_get_data (pixman_image_t *image)
+{
+    if (image->type == BITS)
+	return image->bits.bits;
+
+    return NULL;
+}
+
+PIXMAN_EXPORT int
+pixman_image_get_width (pixman_image_t *image)
+{
+    if (image->type == BITS)
+	return image->bits.width;
+
+    return 0;
+}
+
+PIXMAN_EXPORT int
+pixman_image_get_height (pixman_image_t *image)
+{
+    if (image->type == BITS)
+	return image->bits.height;
+
+    return 0;
+}
+
+PIXMAN_EXPORT int
+pixman_image_get_stride (pixman_image_t *image)
+{
+    if (image->type == BITS)
+	return image->bits.rowstride * (int) sizeof (uint32_t);
+
+    return 0;
+}
+
+PIXMAN_EXPORT int
+pixman_image_get_depth (pixman_image_t *image)
+{
+    if (image->type == BITS)
+	return PIXMAN_FORMAT_DEPTH (image->bits.format);
+
+    return 0;
+}
+
+uint32_t
+_pixman_image_get_solid (pixman_image_t *     image,
+                         pixman_format_code_t format)
+{
+    uint32_t result;
+
+    _pixman_image_get_scanline_32 (image, 0, 0, 1, &result, NULL, 0);
+
+    /* If necessary, convert RGB <--> BGR. */
+    if (PIXMAN_FORMAT_TYPE (format) != PIXMAN_TYPE_ARGB)
+    {
+	result = (((result & 0xff000000) >>  0) |
+	          ((result & 0x00ff0000) >> 16) |
+	          ((result & 0x0000ff00) >>  0) |
+	          ((result & 0x000000ff) << 16));
+    }
+
+    return result;
+}
diff --git a/pixman/pixman/pixman-implementation.c b/pixman/pixman/pixman-implementation.c
new file mode 100644
index 000000000..bc3749ef5
--- /dev/null
+++ b/pixman/pixman/pixman-implementation.c
@@ -0,0 +1,227 @@
+/*
+ * Copyright © 2009 Red Hat, Inc.
+ *
+ * Permission to use, copy, modify, distribute, and sell this software and its
+ * documentation for any purpose is hereby granted without fee, provided that
+ * the above copyright notice appear in all copies and that both that
+ * copyright notice and this permission notice appear in supporting
+ * documentation, and that the name of Red Hat not be used in advertising or
+ * publicity pertaining to distribution of the software without specific,
+ * written prior permission.  Red Hat makes no representations about the
+ * suitability of this software for any purpose.  It is provided "as is"
+ * without express or implied warranty.
+ *
+ * THE COPYRIGHT HOLDERS DISCLAIM ALL WARRANTIES WITH REGARD TO THIS
+ * SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
+ * FITNESS, IN NO EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
+ * AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
+ * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS
+ * SOFTWARE.
+ */
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+#include <stdlib.h>
+#include "pixman-private.h"
+
+static void
+delegate_combine_32 (pixman_implementation_t * imp,
+                     pixman_op_t               op,
+                     uint32_t *                dest,
+                     const uint32_t *          src,
+                     const uint32_t *          mask,
+                     int                       width)
+{
+    _pixman_implementation_combine_32 (imp->delegate,
+                                       op, dest, src, mask, width);
+}
+
+static void
+delegate_combine_64 (pixman_implementation_t * imp,
+                     pixman_op_t               op,
+                     uint64_t *                dest,
+                     const uint64_t *          src,
+                     const uint64_t *          mask,
+                     int                       width)
+{
+    _pixman_implementation_combine_64 (imp->delegate,
+                                       op, dest, src, mask, width);
+}
+
+static void
+delegate_combine_32_ca (pixman_implementation_t * imp,
+                        pixman_op_t               op,
+                        uint32_t *                dest,
+                        const uint32_t *          src,
+                        const uint32_t *          mask,
+                        int                       width)
+{
+    _pixman_implementation_combine_32_ca (imp->delegate,
+                                          op, dest, src, mask, width);
+}
+
+static void
+delegate_combine_64_ca (pixman_implementation_t * imp,
+                        pixman_op_t               op,
+                        uint64_t *                dest,
+                        const uint64_t *          src,
+                        const uint64_t *          mask,
+                        int                       width)
+{
+    _pixman_implementation_combine_64_ca (imp->delegate,
+                                          op, dest, src, mask, width);
+}
+
+static pixman_bool_t
+delegate_blt (pixman_implementation_t * imp,
+              uint32_t *                src_bits,
+              uint32_t *                dst_bits,
+              int                       src_stride,
+              int                       dst_stride,
+              int                       src_bpp,
+              int                       dst_bpp,
+              int                       src_x,
+              int                       src_y,
+              int                       dst_x,
+              int                       dst_y,
+              int                       width,
+              int                       height)
+{
+    return _pixman_implementation_blt (
+	imp->delegate, src_bits, dst_bits, src_stride, dst_stride,
+	src_bpp, dst_bpp, src_x, src_y, dst_x, dst_y,
+	width, height);
+}
+
+static pixman_bool_t
+delegate_fill (pixman_implementation_t *imp,
+               uint32_t *               bits,
+               int                      stride,
+               int                      bpp,
+               int                      x,
+               int                      y,
+               int                      width,
+               int                      height,
+               uint32_t                 xor)
+{
+    return _pixman_implementation_fill (
+	imp->delegate, bits, stride, bpp, x, y, width, height, xor);
+}
+
+pixman_implementation_t *
+_pixman_implementation_create (pixman_implementation_t *delegate,
+			       const pixman_fast_path_t *fast_paths)
+{
+    pixman_implementation_t *imp = malloc (sizeof (pixman_implementation_t));
+    pixman_implementation_t *d;
+    int i;
+
+    if (!imp)
+	return NULL;
+
+    assert (fast_paths);
+
+    /* Make sure the whole delegate chain has the right toplevel */
+    imp->delegate = delegate;
+    for (d = imp; d != NULL; d = d->delegate)
+	d->toplevel = imp;
+
+    /* Fill out function pointers with ones that just delegate
+     */
+    imp->blt = delegate_blt;
+    imp->fill = delegate_fill;
+
+    for (i = 0; i < PIXMAN_N_OPERATORS; ++i)
+    {
+	imp->combine_32[i] = delegate_combine_32;
+	imp->combine_64[i] = delegate_combine_64;
+	imp->combine_32_ca[i] = delegate_combine_32_ca;
+	imp->combine_64_ca[i] = delegate_combine_64_ca;
+    }
+
+    imp->fast_paths = fast_paths;
+    
+    return imp;
+}
+
+void
+_pixman_implementation_combine_32 (pixman_implementation_t * imp,
+                                   pixman_op_t               op,
+                                   uint32_t *                dest,
+                                   const uint32_t *          src,
+                                   const uint32_t *          mask,
+                                   int                       width)
+{
+    (*imp->combine_32[op]) (imp, op, dest, src, mask, width);
+}
+
+void
+_pixman_implementation_combine_64 (pixman_implementation_t * imp,
+                                   pixman_op_t               op,
+                                   uint64_t *                dest,
+                                   const uint64_t *          src,
+                                   const uint64_t *          mask,
+                                   int                       width)
+{
+    (*imp->combine_64[op]) (imp, op, dest, src, mask, width);
+}
+
+void
+_pixman_implementation_combine_32_ca (pixman_implementation_t * imp,
+                                      pixman_op_t               op,
+                                      uint32_t *                dest,
+                                      const uint32_t *          src,
+                                      const uint32_t *          mask,
+                                      int                       width)
+{
+    (*imp->combine_32_ca[op]) (imp, op, dest, src, mask, width);
+}
+
+void
+_pixman_implementation_combine_64_ca (pixman_implementation_t * imp,
+                                      pixman_op_t               op,
+                                      uint64_t *                dest,
+                                      const uint64_t *          src,
+                                      const uint64_t *          mask,
+                                      int                       width)
+{
+    (*imp->combine_64_ca[op]) (imp, op, dest, src, mask, width);
+}
+
+pixman_bool_t
+_pixman_implementation_blt (pixman_implementation_t * imp,
+                            uint32_t *                src_bits,
+                            uint32_t *                dst_bits,
+                            int                       src_stride,
+                            int                       dst_stride,
+                            int                       src_bpp,
+                            int                       dst_bpp,
+                            int                       src_x,
+                            int                       src_y,
+                            int                       dst_x,
+                            int                       dst_y,
+                            int                       width,
+                            int                       height)
+{
+    return (*imp->blt) (imp, src_bits, dst_bits, src_stride, dst_stride,
+			src_bpp, dst_bpp, src_x, src_y, dst_x, dst_y,
+			width, height);
+}
+
+pixman_bool_t
+_pixman_implementation_fill (pixman_implementation_t *imp,
+                             uint32_t *               bits,
+                             int                      stride,
+                             int                      bpp,
+                             int                      x,
+                             int                      y,
+                             int                      width,
+                             int                      height,
+                             uint32_t                 xor)
+{
+    return (*imp->fill) (imp, bits, stride, bpp, x, y, width, height, xor);
+}
+
diff --git a/pixman/pixman/pixman-linear-gradient.c b/pixman/pixman/pixman-linear-gradient.c
new file mode 100644
index 000000000..d9409fe50
--- /dev/null
+++ b/pixman/pixman/pixman-linear-gradient.c
@@ -0,0 +1,294 @@
+/*
+ * Copyright © 2000 SuSE, Inc.
+ * Copyright © 2007 Red Hat, Inc.
+ * Copyright © 2000 Keith Packard, member of The XFree86 Project, Inc.
+ *             2005 Lars Knoll & Zack Rusin, Trolltech
+ *
+ * Permission to use, copy, modify, distribute, and sell this software and its
+ * documentation for any purpose is hereby granted without fee, provided that
+ * the above copyright notice appear in all copies and that both that
+ * copyright notice and this permission notice appear in supporting
+ * documentation, and that the name of Keith Packard not be used in
+ * advertising or publicity pertaining to distribution of the software without
+ * specific, written prior permission.  Keith Packard makes no
+ * representations about the suitability of this software for any purpose.  It
+ * is provided "as is" without express or implied warranty.
+ *
+ * THE COPYRIGHT HOLDERS DISCLAIM ALL WARRANTIES WITH REGARD TO THIS
+ * SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
+ * FITNESS, IN NO EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
+ * AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
+ * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS
+ * SOFTWARE.
+ */
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+#include <stdlib.h>
+#include "pixman-private.h"
+
+static source_image_class_t
+linear_gradient_classify (pixman_image_t *image,
+                          int             x,
+                          int             y,
+                          int             width,
+                          int             height)
+{
+    linear_gradient_t *linear = (linear_gradient_t *)image;
+    pixman_vector_t v;
+    pixman_fixed_32_32_t l;
+    pixman_fixed_48_16_t dx, dy, a, b, off;
+    pixman_fixed_48_16_t factors[4];
+    int i;
+
+    image->source.class = SOURCE_IMAGE_CLASS_UNKNOWN;
+
+    dx = linear->p2.x - linear->p1.x;
+    dy = linear->p2.y - linear->p1.y;
+
+    l = dx * dx + dy * dy;
+
+    if (l)
+    {
+	a = (dx << 32) / l;
+	b = (dy << 32) / l;
+    }
+    else
+    {
+	a = b = 0;
+    }
+
+    off = (-a * linear->p1.x
+           -b * linear->p1.y) >> 16;
+
+    for (i = 0; i < 3; i++)
+    {
+	v.vector[0] = pixman_int_to_fixed ((i % 2) * (width  - 1) + x);
+	v.vector[1] = pixman_int_to_fixed ((i / 2) * (height - 1) + y);
+	v.vector[2] = pixman_fixed_1;
+
+	if (image->common.transform)
+	{
+	    if (!pixman_transform_point_3d (image->common.transform, &v))
+	    {
+		image->source.class = SOURCE_IMAGE_CLASS_UNKNOWN;
+
+		return image->source.class;
+	    }
+	}
+
+	factors[i] = ((a * v.vector[0] + b * v.vector[1]) >> 16) + off;
+    }
+
+    if (factors[2] == factors[0])
+	image->source.class = SOURCE_IMAGE_CLASS_HORIZONTAL;
+    else if (factors[1] == factors[0])
+	image->source.class = SOURCE_IMAGE_CLASS_VERTICAL;
+
+    return image->source.class;
+}
+
+static void
+linear_gradient_get_scanline_32 (pixman_image_t *image,
+                                 int             x,
+                                 int             y,
+                                 int             width,
+                                 uint32_t *      buffer,
+                                 const uint32_t *mask,
+                                 uint32_t        mask_bits)
+{
+    pixman_vector_t v, unit;
+    pixman_fixed_32_32_t l;
+    pixman_fixed_48_16_t dx, dy, a, b, off;
+    gradient_t *gradient = (gradient_t *)image;
+    source_image_t *source = (source_image_t *)image;
+    linear_gradient_t *linear = (linear_gradient_t *)image;
+    uint32_t *end = buffer + width;
+    pixman_gradient_walker_t walker;
+
+    _pixman_gradient_walker_init (&walker, gradient, source->common.repeat);
+
+    /* reference point is the center of the pixel */
+    v.vector[0] = pixman_int_to_fixed (x) + pixman_fixed_1 / 2;
+    v.vector[1] = pixman_int_to_fixed (y) + pixman_fixed_1 / 2;
+    v.vector[2] = pixman_fixed_1;
+
+    if (source->common.transform)
+    {
+	if (!pixman_transform_point_3d (source->common.transform, &v))
+	    return;
+	
+	unit.vector[0] = source->common.transform->matrix[0][0];
+	unit.vector[1] = source->common.transform->matrix[1][0];
+	unit.vector[2] = source->common.transform->matrix[2][0];
+    }
+    else
+    {
+	unit.vector[0] = pixman_fixed_1;
+	unit.vector[1] = 0;
+	unit.vector[2] = 0;
+    }
+
+    dx = linear->p2.x - linear->p1.x;
+    dy = linear->p2.y - linear->p1.y;
+
+    l = dx * dx + dy * dy;
+
+    if (l != 0)
+    {
+	a = (dx << 32) / l;
+	b = (dy << 32) / l;
+	off = (-a * linear->p1.x
+	       -b * linear->p1.y) >> 16;
+    }
+
+    if (l == 0 || (unit.vector[2] == 0 && v.vector[2] == pixman_fixed_1))
+    {
+	pixman_fixed_48_16_t inc, t;
+
+	/* affine transformation only */
+	if (l == 0)
+	{
+	    t = 0;
+	    inc = 0;
+	}
+	else
+	{
+	    t = ((a * v.vector[0] + b * v.vector[1]) >> 16) + off;
+	    inc = (a * unit.vector[0] + b * unit.vector[1]) >> 16;
+	}
+
+	if (source->class == SOURCE_IMAGE_CLASS_VERTICAL)
+	{
+	    register uint32_t color;
+
+	    color = _pixman_gradient_walker_pixel (&walker, t);
+	    while (buffer < end)
+		*buffer++ = color;
+	}
+	else
+	{
+	    if (!mask)
+	    {
+		while (buffer < end)
+		{
+		    *buffer++ = _pixman_gradient_walker_pixel (&walker, t);
+		    
+		    t += inc;
+		}
+	    }
+	    else
+	    {
+		while (buffer < end)
+		{
+		    if (*mask++ & mask_bits)
+			*buffer = _pixman_gradient_walker_pixel (&walker, t);
+
+		    buffer++;
+		    t += inc;
+		}
+	    }
+	}
+    }
+    else
+    {
+	/* projective transformation */
+	pixman_fixed_48_16_t t;
+
+	if (source->class == SOURCE_IMAGE_CLASS_VERTICAL)
+	{
+	    register uint32_t color;
+
+	    if (v.vector[2] == 0)
+	    {
+		t = 0;
+	    }
+	    else
+	    {
+		pixman_fixed_48_16_t x, y;
+
+		x = ((pixman_fixed_48_16_t) v.vector[0] << 16) / v.vector[2];
+		y = ((pixman_fixed_48_16_t) v.vector[1] << 16) / v.vector[2];
+		t = ((a * x + b * y) >> 16) + off;
+	    }
+
+	    color = _pixman_gradient_walker_pixel (&walker, t);
+	    while (buffer < end)
+		*buffer++ = color;
+	}
+	else
+	{
+	    while (buffer < end)
+	    {
+		if (!mask || *mask++ & mask_bits)
+		{
+		    if (v.vector[2] == 0)
+		    {
+			t = 0;
+		    }
+		    else
+		    {
+			pixman_fixed_48_16_t x, y;
+			x = ((pixman_fixed_48_16_t)v.vector[0] << 16) / v.vector[2];
+			y = ((pixman_fixed_48_16_t)v.vector[1] << 16) / v.vector[2];
+			t = ((a * x + b * y) >> 16) + off;
+		    }
+		    
+		    *buffer = _pixman_gradient_walker_pixel (&walker, t);
+		}
+		
+		++buffer;
+		
+		v.vector[0] += unit.vector[0];
+		v.vector[1] += unit.vector[1];
+		v.vector[2] += unit.vector[2];
+	    }
+	}
+    }
+}
+
+static void
+linear_gradient_property_changed (pixman_image_t *image)
+{
+    image->common.get_scanline_32 = linear_gradient_get_scanline_32;
+    image->common.get_scanline_64 = _pixman_image_get_scanline_generic_64;
+}
+
+PIXMAN_EXPORT pixman_image_t *
+pixman_image_create_linear_gradient (pixman_point_fixed_t *        p1,
+                                     pixman_point_fixed_t *        p2,
+                                     const pixman_gradient_stop_t *stops,
+                                     int                           n_stops)
+{
+    pixman_image_t *image;
+    linear_gradient_t *linear;
+
+    return_val_if_fail (n_stops >= 2, NULL);
+
+    image = _pixman_image_allocate ();
+
+    if (!image)
+	return NULL;
+
+    linear = &image->linear;
+
+    if (!_pixman_init_gradient (&linear->common, stops, n_stops))
+    {
+	free (image);
+	return NULL;
+    }
+
+    linear->p1 = *p1;
+    linear->p2 = *p2;
+
+    image->type = LINEAR;
+    image->source.class = SOURCE_IMAGE_CLASS_UNKNOWN;
+    image->common.classify = linear_gradient_classify;
+    image->common.property_changed = linear_gradient_property_changed;
+
+    return image;
+}
+
diff --git a/pixman/pixman/pixman-matrix.c b/pixman/pixman/pixman-matrix.c
new file mode 100644
index 000000000..abdfa0525
--- /dev/null
+++ b/pixman/pixman/pixman-matrix.c
@@ -0,0 +1,768 @@
+/*
+ * Copyright © 2008 Keith Packard
+ *
+ * Permission to use, copy, modify, distribute, and sell this software and its
+ * documentation for any purpose is hereby granted without fee, provided that
+ * the above copyright notice appear in all copies and that both that copyright
+ * notice and this permission notice appear in supporting documentation, and
+ * that the name of the copyright holders not be used in advertising or
+ * publicity pertaining to distribution of the software without specific,
+ * written prior permission.  The copyright holders make no representations
+ * about the suitability of this software for any purpose.  It is provided "as
+ * is" without express or implied warranty.
+ *
+ * THE COPYRIGHT HOLDERS DISCLAIM ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
+ * INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO
+ * EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY SPECIAL, INDIRECT OR
+ * CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE,
+ * DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
+ * TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
+ * OF THIS SOFTWARE.
+ */
+
+/*
+ * Matrix interfaces
+ */
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <math.h>
+#include <string.h>
+#include "pixman-private.h"
+
+#define F(x)    pixman_int_to_fixed (x)
+
+PIXMAN_EXPORT void
+pixman_transform_init_identity (struct pixman_transform *matrix)
+{
+    int i;
+
+    memset (matrix, '\0', sizeof (struct pixman_transform));
+    for (i = 0; i < 3; i++)
+	matrix->matrix[i][i] = F (1);
+}
+
+typedef pixman_fixed_32_32_t pixman_fixed_34_30_t;
+
+PIXMAN_EXPORT pixman_bool_t
+pixman_transform_point_3d (const struct pixman_transform *transform,
+                           struct pixman_vector *         vector)
+{
+    struct pixman_vector result;
+    pixman_fixed_32_32_t partial;
+    pixman_fixed_48_16_t v;
+    int i, j;
+
+    for (j = 0; j < 3; j++)
+    {
+	v = 0;
+	for (i = 0; i < 3; i++)
+	{
+	    partial = ((pixman_fixed_48_16_t) transform->matrix[j][i] *
+	               (pixman_fixed_48_16_t) vector->vector[i]);
+	    v += partial >> 16;
+	}
+	
+	if (v > pixman_max_fixed_48_16 || v < pixman_min_fixed_48_16)
+	    return FALSE;
+	
+	result.vector[j] = (pixman_fixed_t) v;
+    }
+    
+    *vector = result;
+
+    if (!result.vector[2])
+	return FALSE;
+
+    return TRUE;
+}
+
+PIXMAN_EXPORT pixman_bool_t
+pixman_transform_point (const struct pixman_transform *transform,
+                        struct pixman_vector *         vector)
+{
+    pixman_fixed_32_32_t partial;
+    pixman_fixed_34_30_t v[3];
+    pixman_fixed_48_16_t quo;
+    int i, j;
+
+    for (j = 0; j < 3; j++)
+    {
+	v[j] = 0;
+	
+	for (i = 0; i < 3; i++)
+	{
+	    partial = ((pixman_fixed_32_32_t) transform->matrix[j][i] *
+	               (pixman_fixed_32_32_t) vector->vector[i]);
+	    v[j] += partial >> 2;
+	}
+    }
+    
+    if (!(v[2] >> 16))
+	return FALSE;
+
+    for (j = 0; j < 2; j++)
+    {
+	quo = v[j] / (v[2] >> 16);
+	if (quo > pixman_max_fixed_48_16 || quo < pixman_min_fixed_48_16)
+	    return FALSE;
+	vector->vector[j] = (pixman_fixed_t) quo;
+    }
+    
+    vector->vector[2] = pixman_fixed_1;
+    return TRUE;
+}
+
+PIXMAN_EXPORT pixman_bool_t
+pixman_transform_multiply (struct pixman_transform *      dst,
+                           const struct pixman_transform *l,
+                           const struct pixman_transform *r)
+{
+    struct pixman_transform d;
+    int dx, dy;
+    int o;
+
+    for (dy = 0; dy < 3; dy++)
+    {
+	for (dx = 0; dx < 3; dx++)
+	{
+	    pixman_fixed_48_16_t v;
+	    pixman_fixed_32_32_t partial;
+	    
+	    v = 0;
+	    for (o = 0; o < 3; o++)
+	    {
+		partial =
+		    (pixman_fixed_32_32_t) l->matrix[dy][o] *
+		    (pixman_fixed_32_32_t) r->matrix[o][dx];
+
+		v += partial >> 16;
+	    }
+
+	    if (v > pixman_max_fixed_48_16 || v < pixman_min_fixed_48_16)
+		return FALSE;
+	    
+	    d.matrix[dy][dx] = (pixman_fixed_t) v;
+	}
+    }
+
+    *dst = d;
+    return TRUE;
+}
+
+PIXMAN_EXPORT void
+pixman_transform_init_scale (struct pixman_transform *t,
+                             pixman_fixed_t           sx,
+                             pixman_fixed_t           sy)
+{
+    memset (t, '\0', sizeof (struct pixman_transform));
+
+    t->matrix[0][0] = sx;
+    t->matrix[1][1] = sy;
+    t->matrix[2][2] = F (1);
+}
+
+static pixman_fixed_t
+fixed_inverse (pixman_fixed_t x)
+{
+    return (pixman_fixed_t) ((((pixman_fixed_48_16_t) F (1)) * F (1)) / x);
+}
+
+PIXMAN_EXPORT pixman_bool_t
+pixman_transform_scale (struct pixman_transform *forward,
+                        struct pixman_transform *reverse,
+                        pixman_fixed_t           sx,
+                        pixman_fixed_t           sy)
+{
+    struct pixman_transform t;
+
+    if (sx == 0 || sy == 0)
+	return FALSE;
+
+    if (forward)
+    {
+	pixman_transform_init_scale (&t, sx, sy);
+	if (!pixman_transform_multiply (forward, &t, forward))
+	    return FALSE;
+    }
+    
+    if (reverse)
+    {
+	pixman_transform_init_scale (&t, fixed_inverse (sx),
+	                             fixed_inverse (sy));
+	if (!pixman_transform_multiply (reverse, reverse, &t))
+	    return FALSE;
+    }
+    
+    return TRUE;
+}
+
+PIXMAN_EXPORT void
+pixman_transform_init_rotate (struct pixman_transform *t,
+                              pixman_fixed_t           c,
+                              pixman_fixed_t           s)
+{
+    memset (t, '\0', sizeof (struct pixman_transform));
+
+    t->matrix[0][0] = c;
+    t->matrix[0][1] = -s;
+    t->matrix[1][0] = s;
+    t->matrix[1][1] = c;
+    t->matrix[2][2] = F (1);
+}
+
+PIXMAN_EXPORT pixman_bool_t
+pixman_transform_rotate (struct pixman_transform *forward,
+                         struct pixman_transform *reverse,
+                         pixman_fixed_t           c,
+                         pixman_fixed_t           s)
+{
+    struct pixman_transform t;
+
+    if (forward)
+    {
+	pixman_transform_init_rotate (&t, c, s);
+	if (!pixman_transform_multiply (forward, &t, forward))
+	    return FALSE;
+    }
+
+    if (reverse)
+    {
+	pixman_transform_init_rotate (&t, c, -s);
+	if (!pixman_transform_multiply (reverse, reverse, &t))
+	    return FALSE;
+    }
+    
+    return TRUE;
+}
+
+PIXMAN_EXPORT void
+pixman_transform_init_translate (struct pixman_transform *t,
+                                 pixman_fixed_t           tx,
+                                 pixman_fixed_t           ty)
+{
+    memset (t, '\0', sizeof (struct pixman_transform));
+
+    t->matrix[0][0] = F (1);
+    t->matrix[0][2] = tx;
+    t->matrix[1][1] = F (1);
+    t->matrix[1][2] = ty;
+    t->matrix[2][2] = F (1);
+}
+
+PIXMAN_EXPORT pixman_bool_t
+pixman_transform_translate (struct pixman_transform *forward,
+                            struct pixman_transform *reverse,
+                            pixman_fixed_t           tx,
+                            pixman_fixed_t           ty)
+{
+    struct pixman_transform t;
+
+    if (forward)
+    {
+	pixman_transform_init_translate (&t, tx, ty);
+
+	if (!pixman_transform_multiply (forward, &t, forward))
+	    return FALSE;
+    }
+
+    if (reverse)
+    {
+	pixman_transform_init_translate (&t, -tx, -ty);
+
+	if (!pixman_transform_multiply (reverse, reverse, &t))
+	    return FALSE;
+    }
+    return TRUE;
+}
+
+PIXMAN_EXPORT pixman_bool_t
+pixman_transform_bounds (const struct pixman_transform *matrix,
+                         struct pixman_box16 *          b)
+
+{
+    struct pixman_vector v[4];
+    int i;
+    int x1, y1, x2, y2;
+
+    v[0].vector[0] = F (b->x1);
+    v[0].vector[1] = F (b->y1);
+    v[0].vector[2] = F (1);
+
+    v[1].vector[0] = F (b->x2);
+    v[1].vector[1] = F (b->y1);
+    v[1].vector[2] = F (1);
+
+    v[2].vector[0] = F (b->x2);
+    v[2].vector[1] = F (b->y2);
+    v[2].vector[2] = F (1);
+
+    v[3].vector[0] = F (b->x1);
+    v[3].vector[1] = F (b->y2);
+    v[3].vector[2] = F (1);
+
+    for (i = 0; i < 4; i++)
+    {
+	if (!pixman_transform_point (matrix, &v[i]))
+	    return FALSE;
+
+	x1 = pixman_fixed_to_int (v[i].vector[0]);
+	y1 = pixman_fixed_to_int (v[i].vector[1]);
+	x2 = pixman_fixed_to_int (pixman_fixed_ceil (v[i].vector[0]));
+	y2 = pixman_fixed_to_int (pixman_fixed_ceil (v[i].vector[1]));
+
+	if (i == 0)
+	{
+	    b->x1 = x1;
+	    b->y1 = y1;
+	    b->x2 = x2;
+	    b->y2 = y2;
+	}
+	else
+	{
+	    if (x1 < b->x1) b->x1 = x1;
+	    if (y1 < b->y1) b->y1 = y1;
+	    if (x2 > b->x2) b->x2 = x2;
+	    if (y2 > b->y2) b->y2 = y2;
+	}
+    }
+
+    return TRUE;
+}
+
+PIXMAN_EXPORT pixman_bool_t
+pixman_transform_invert (struct pixman_transform *      dst,
+                         const struct pixman_transform *src)
+{
+    struct pixman_f_transform m, r;
+
+    pixman_f_transform_from_pixman_transform (&m, src);
+
+    if (!pixman_f_transform_invert (&r, &m))
+	return FALSE;
+
+    if (!pixman_transform_from_pixman_f_transform (dst, &r))
+	return FALSE;
+
+    return TRUE;
+}
+
+static pixman_bool_t
+within_epsilon (pixman_fixed_t a,
+                pixman_fixed_t b,
+                pixman_fixed_t epsilon)
+{
+    pixman_fixed_t t = a - b;
+
+    if (t < 0)
+	t = -t;
+
+    return t <= epsilon;
+}
+
+#define EPSILON (pixman_fixed_t) (2)
+
+#define IS_SAME(a, b) (within_epsilon (a, b, EPSILON))
+#define IS_ZERO(a)    (within_epsilon (a, 0, EPSILON))
+#define IS_ONE(a)     (within_epsilon (a, F (1), EPSILON))
+#define IS_UNIT(a)			    \
+    (within_epsilon (a, F (1), EPSILON) ||  \
+     within_epsilon (a, F (-1), EPSILON) || \
+     IS_ZERO (a))
+#define IS_INT(a)    (IS_ZERO (pixman_fixed_frac (a)))
+
+PIXMAN_EXPORT pixman_bool_t
+pixman_transform_is_identity (const struct pixman_transform *t)
+{
+    return (IS_SAME (t->matrix[0][0], t->matrix[1][1]) &&
+	    IS_SAME (t->matrix[0][0], t->matrix[2][2]) &&
+	    !IS_ZERO (t->matrix[0][0]) &&
+	    IS_ZERO (t->matrix[0][1]) &&
+	    IS_ZERO (t->matrix[0][2]) &&
+	    IS_ZERO (t->matrix[1][0]) &&
+	    IS_ZERO (t->matrix[1][2]) &&
+	    IS_ZERO (t->matrix[2][0]) &&
+	    IS_ZERO (t->matrix[2][1]));
+}
+
+PIXMAN_EXPORT pixman_bool_t
+pixman_transform_is_scale (const struct pixman_transform *t)
+{
+    return (!IS_ZERO (t->matrix[0][0]) &&
+            IS_ZERO (t->matrix[0][1]) &&
+            IS_ZERO (t->matrix[0][2]) &&
+
+            IS_ZERO (t->matrix[1][0]) &&
+            !IS_ZERO (t->matrix[1][1]) &&
+            IS_ZERO (t->matrix[1][2]) &&
+
+            IS_ZERO (t->matrix[2][0]) &&
+            IS_ZERO (t->matrix[2][1]) &&
+            !IS_ZERO (t->matrix[2][2]));
+}
+
+PIXMAN_EXPORT pixman_bool_t
+pixman_transform_is_int_translate (const struct pixman_transform *t)
+{
+    return (IS_ONE (t->matrix[0][0]) &&
+            IS_ZERO (t->matrix[0][1]) &&
+            IS_INT (t->matrix[0][2]) &&
+
+            IS_ZERO (t->matrix[1][0]) &&
+            IS_ONE (t->matrix[1][1]) &&
+            IS_INT (t->matrix[1][2]) &&
+
+            IS_ZERO (t->matrix[2][0]) &&
+            IS_ZERO (t->matrix[2][1]) &&
+            IS_ONE (t->matrix[2][2]));
+}
+
+PIXMAN_EXPORT pixman_bool_t
+pixman_transform_is_inverse (const struct pixman_transform *a,
+                             const struct pixman_transform *b)
+{
+    struct pixman_transform t;
+
+    pixman_transform_multiply (&t, a, b);
+
+    return pixman_transform_is_identity (&t);
+}
+
+PIXMAN_EXPORT void
+pixman_f_transform_from_pixman_transform (struct pixman_f_transform *    ft,
+                                          const struct pixman_transform *t)
+{
+    int i, j;
+
+    for (j = 0; j < 3; j++)
+    {
+	for (i = 0; i < 3; i++)
+	    ft->m[j][i] = pixman_fixed_to_double (t->matrix[j][i]);
+    }
+}
+
+PIXMAN_EXPORT pixman_bool_t
+pixman_transform_from_pixman_f_transform (struct pixman_transform *        t,
+                                          const struct pixman_f_transform *ft)
+{
+    int i, j;
+
+    for (j = 0; j < 3; j++)
+    {
+	for (i = 0; i < 3; i++)
+	{
+	    double d = ft->m[j][i];
+	    if (d < -32767.0 || d > 32767.0)
+		return FALSE;
+	    d = d * 65536.0 + 0.5;
+	    t->matrix[j][i] = (pixman_fixed_t) floor (d);
+	}
+    }
+    
+    return TRUE;
+}
+
+static const int a[3] = { 3, 3, 2 };
+static const int b[3] = { 2, 1, 1 };
+
+PIXMAN_EXPORT pixman_bool_t
+pixman_f_transform_invert (struct pixman_f_transform *      dst,
+                           const struct pixman_f_transform *src)
+{
+    double det;
+    int i, j;
+    static int a[3] = { 2, 2, 1 };
+    static int b[3] = { 1, 0, 0 };
+
+    det = 0;
+    for (i = 0; i < 3; i++)
+    {
+	double p;
+	int ai = a[i];
+	int bi = b[i];
+	p = src->m[i][0] * (src->m[ai][2] * src->m[bi][1] -
+	                    src->m[ai][1] * src->m[bi][2]);
+	if (i == 1)
+	    p = -p;
+	det += p;
+    }
+    
+    if (det == 0)
+	return FALSE;
+    
+    det = 1 / det;
+    for (j = 0; j < 3; j++)
+    {
+	for (i = 0; i < 3; i++)
+	{
+	    double p;
+	    int ai = a[i];
+	    int aj = a[j];
+	    int bi = b[i];
+	    int bj = b[j];
+
+	    p = (src->m[ai][aj] * src->m[bi][bj] -
+	         src->m[ai][bj] * src->m[bi][aj]);
+	    
+	    if (((i + j) & 1) != 0)
+		p = -p;
+	    
+	    dst->m[j][i] = det * p;
+	}
+    }
+
+    return TRUE;
+}
+
+PIXMAN_EXPORT pixman_bool_t
+pixman_f_transform_point (const struct pixman_f_transform *t,
+                          struct pixman_f_vector *         v)
+{
+    struct pixman_f_vector result;
+    int i, j;
+    double a;
+
+    for (j = 0; j < 3; j++)
+    {
+	a = 0;
+	for (i = 0; i < 3; i++)
+	    a += t->m[j][i] * v->v[i];
+	result.v[j] = a;
+    }
+    
+    if (!result.v[2])
+	return FALSE;
+
+    for (j = 0; j < 2; j++)
+	v->v[j] = result.v[j] / result.v[2];
+
+    v->v[2] = 1;
+
+    return TRUE;
+}
+
+PIXMAN_EXPORT void
+pixman_f_transform_point_3d (const struct pixman_f_transform *t,
+                             struct pixman_f_vector *         v)
+{
+    struct pixman_f_vector result;
+    int i, j;
+    double a;
+
+    for (j = 0; j < 3; j++)
+    {
+	a = 0;
+	for (i = 0; i < 3; i++)
+	    a += t->m[j][i] * v->v[i];
+	result.v[j] = a;
+    }
+    
+    *v = result;
+}
+
+PIXMAN_EXPORT void
+pixman_f_transform_multiply (struct pixman_f_transform *      dst,
+                             const struct pixman_f_transform *l,
+                             const struct pixman_f_transform *r)
+{
+    struct pixman_f_transform d;
+    int dx, dy;
+    int o;
+
+    for (dy = 0; dy < 3; dy++)
+    {
+	for (dx = 0; dx < 3; dx++)
+	{
+	    double v = 0;
+	    for (o = 0; o < 3; o++)
+		v += l->m[dy][o] * r->m[o][dx];
+	    d.m[dy][dx] = v;
+	}
+    }
+    
+    *dst = d;
+}
+
+PIXMAN_EXPORT void
+pixman_f_transform_init_scale (struct pixman_f_transform *t,
+                               double                     sx,
+                               double                     sy)
+{
+    t->m[0][0] = sx;
+    t->m[0][1] = 0;
+    t->m[0][2] = 0;
+    t->m[1][0] = 0;
+    t->m[1][1] = sy;
+    t->m[1][2] = 0;
+    t->m[2][0] = 0;
+    t->m[2][1] = 0;
+    t->m[2][2] = 1;
+}
+
+PIXMAN_EXPORT pixman_bool_t
+pixman_f_transform_scale (struct pixman_f_transform *forward,
+                          struct pixman_f_transform *reverse,
+                          double                     sx,
+                          double                     sy)
+{
+    struct pixman_f_transform t;
+
+    if (sx == 0 || sy == 0)
+	return FALSE;
+
+    if (forward)
+    {
+	pixman_f_transform_init_scale (&t, sx, sy);
+	pixman_f_transform_multiply (forward, &t, forward);
+    }
+    
+    if (reverse)
+    {
+	pixman_f_transform_init_scale (&t, 1 / sx, 1 / sy);
+	pixman_f_transform_multiply (reverse, reverse, &t);
+    }
+    
+    return TRUE;
+}
+
+PIXMAN_EXPORT void
+pixman_f_transform_init_rotate (struct pixman_f_transform *t,
+                                double                     c,
+                                double                     s)
+{
+    t->m[0][0] = c;
+    t->m[0][1] = -s;
+    t->m[0][2] = 0;
+    t->m[1][0] = s;
+    t->m[1][1] = c;
+    t->m[1][2] = 0;
+    t->m[2][0] = 0;
+    t->m[2][1] = 0;
+    t->m[2][2] = 1;
+}
+
+PIXMAN_EXPORT pixman_bool_t
+pixman_f_transform_rotate (struct pixman_f_transform *forward,
+                           struct pixman_f_transform *reverse,
+                           double                     c,
+                           double                     s)
+{
+    struct pixman_f_transform t;
+
+    if (forward)
+    {
+	pixman_f_transform_init_rotate (&t, c, s);
+	pixman_f_transform_multiply (forward, &t, forward);
+    }
+    
+    if (reverse)
+    {
+	pixman_f_transform_init_rotate (&t, c, -s);
+	pixman_f_transform_multiply (reverse, reverse, &t);
+    }
+
+    return TRUE;
+}
+
+PIXMAN_EXPORT void
+pixman_f_transform_init_translate (struct pixman_f_transform *t,
+                                   double                     tx,
+                                   double                     ty)
+{
+    t->m[0][0] = 1;
+    t->m[0][1] = 0;
+    t->m[0][2] = tx;
+    t->m[1][0] = 0;
+    t->m[1][1] = 1;
+    t->m[1][2] = ty;
+    t->m[2][0] = 0;
+    t->m[2][1] = 0;
+    t->m[2][2] = 1;
+}
+
+PIXMAN_EXPORT pixman_bool_t
+pixman_f_transform_translate (struct pixman_f_transform *forward,
+                              struct pixman_f_transform *reverse,
+                              double                     tx,
+                              double                     ty)
+{
+    struct pixman_f_transform t;
+
+    if (forward)
+    {
+	pixman_f_transform_init_translate (&t, tx, ty);
+	pixman_f_transform_multiply (forward, &t, forward);
+    }
+
+    if (reverse)
+    {
+	pixman_f_transform_init_translate (&t, -tx, -ty);
+	pixman_f_transform_multiply (reverse, reverse, &t);
+    }
+
+    return TRUE;
+}
+
+PIXMAN_EXPORT pixman_bool_t
+pixman_f_transform_bounds (const struct pixman_f_transform *t,
+                           struct pixman_box16 *            b)
+{
+    struct pixman_f_vector v[4];
+    int i;
+    int x1, y1, x2, y2;
+
+    v[0].v[0] = b->x1;
+    v[0].v[1] = b->y1;
+    v[0].v[2] = 1;
+    v[1].v[0] = b->x2;
+    v[1].v[1] = b->y1;
+    v[1].v[2] = 1;
+    v[2].v[0] = b->x2;
+    v[2].v[1] = b->y2;
+    v[2].v[2] = 1;
+    v[3].v[0] = b->x1;
+    v[3].v[1] = b->y2;
+    v[3].v[2] = 1;
+
+    for (i = 0; i < 4; i++)
+    {
+	if (!pixman_f_transform_point (t, &v[i]))
+	    return FALSE;
+
+	x1 = floor (v[i].v[0]);
+	y1 = floor (v[i].v[1]);
+	x2 = ceil (v[i].v[0]);
+	y2 = ceil (v[i].v[1]);
+
+	if (i == 0)
+	{
+	    b->x1 = x1;
+	    b->y1 = y1;
+	    b->x2 = x2;
+	    b->y2 = y2;
+	}
+	else
+	{
+	    if (x1 < b->x1) b->x1 = x1;
+	    if (y1 < b->y1) b->y1 = y1;
+	    if (x2 > b->x2) b->x2 = x2;
+	    if (y2 > b->y2) b->y2 = y2;
+	}
+    }
+
+    return TRUE;
+}
+
+PIXMAN_EXPORT void
+pixman_f_transform_init_identity (struct pixman_f_transform *t)
+{
+    int i, j;
+
+    for (j = 0; j < 3; j++)
+    {
+	for (i = 0; i < 3; i++)
+	    t->m[j][i] = i == j ? 1 : 0;
+    }
+}
diff --git a/pixman/pixman/pixman-mmx.c b/pixman/pixman/pixman-mmx.c
new file mode 100644
index 000000000..d51b40cc1
--- /dev/null
+++ b/pixman/pixman/pixman-mmx.c
@@ -0,0 +1,3378 @@
+/*
+ * Copyright © 2004, 2005 Red Hat, Inc.
+ * Copyright © 2004 Nicholas Miell
+ * Copyright © 2005 Trolltech AS
+ *
+ * Permission to use, copy, modify, distribute, and sell this software and its
+ * documentation for any purpose is hereby granted without fee, provided that
+ * the above copyright notice appear in all copies and that both that
+ * copyright notice and this permission notice appear in supporting
+ * documentation, and that the name of Red Hat not be used in advertising or
+ * publicity pertaining to distribution of the software without specific,
+ * written prior permission.  Red Hat makes no representations about the
+ * suitability of this software for any purpose.  It is provided "as is"
+ * without express or implied warranty.
+ *
+ * THE COPYRIGHT HOLDERS DISCLAIM ALL WARRANTIES WITH REGARD TO THIS
+ * SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
+ * FITNESS, IN NO EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
+ * AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
+ * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS
+ * SOFTWARE.
+ *
+ * Author:  Søren Sandmann (sandmann@redhat.com)
+ * Minor Improvements: Nicholas Miell (nmiell@gmail.com)
+ * MMX code paths for fbcompose.c by Lars Knoll (lars@trolltech.com)
+ *
+ * Based on work by Owen Taylor
+ */
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#ifdef USE_MMX
+
+#include <mmintrin.h>
+#include "pixman-private.h"
+#include "pixman-combine32.h"
+
+#define no_vERBOSE
+
+#ifdef VERBOSE
+#define CHECKPOINT() error_f ("at %s %d\n", __FUNCTION__, __LINE__)
+#else
+#define CHECKPOINT()
+#endif
+
+/* Notes about writing mmx code
+ *
+ * give memory operands as the second operand. If you give it as the
+ * first, gcc will first load it into a register, then use that
+ * register
+ *
+ *   ie. use
+ *
+ *         _mm_mullo_pi16 (x, mmx_constant);
+ *
+ *   not
+ *
+ *         _mm_mullo_pi16 (mmx_constant, x);
+ *
+ * Also try to minimize dependencies. i.e. when you need a value, try
+ * to calculate it from a value that was calculated as early as
+ * possible.
+ */
+
+/* --------------- MMX primitives ------------------------------------- */
+
+#ifdef __GNUC__
+typedef uint64_t mmxdatafield;
+#else
+typedef __m64 mmxdatafield;
+/* If __m64 is defined as a struct or union, define M64_MEMBER to be the
+   name of the member used to access the data */
+# ifdef _MSC_VER
+#  define M64_MEMBER m64_u64
+# elif defined(__SUNPRO_C)
+#  define M64_MEMBER l_
+# endif
+#endif
+
+typedef struct
+{
+    mmxdatafield mmx_4x00ff;
+    mmxdatafield mmx_4x0080;
+    mmxdatafield mmx_565_rgb;
+    mmxdatafield mmx_565_unpack_multiplier;
+    mmxdatafield mmx_565_r;
+    mmxdatafield mmx_565_g;
+    mmxdatafield mmx_565_b;
+    mmxdatafield mmx_mask_0;
+    mmxdatafield mmx_mask_1;
+    mmxdatafield mmx_mask_2;
+    mmxdatafield mmx_mask_3;
+    mmxdatafield mmx_full_alpha;
+    mmxdatafield mmx_ffff0000ffff0000;
+    mmxdatafield mmx_0000ffff00000000;
+    mmxdatafield mmx_000000000000ffff;
+} mmx_data_t;
+
+#if defined(_MSC_VER)
+# define MMXDATA_INIT(field, val) { val ## UI64 }
+#elif defined(M64_MEMBER)       /* __m64 is a struct, not an integral type */
+# define MMXDATA_INIT(field, val) field =   { val ## ULL }
+#else                           /* __m64 is an integral type */
+# define MMXDATA_INIT(field, val) field =   val ## ULL
+#endif
+
+static const mmx_data_t c =
+{
+    MMXDATA_INIT (.mmx_4x00ff,                   0x00ff00ff00ff00ff),
+    MMXDATA_INIT (.mmx_4x0080,                   0x0080008000800080),
+    MMXDATA_INIT (.mmx_565_rgb,                  0x000001f0003f001f),
+    MMXDATA_INIT (.mmx_565_unpack_multiplier,    0x0000008404100840),
+    MMXDATA_INIT (.mmx_565_r,                    0x000000f800000000),
+    MMXDATA_INIT (.mmx_565_g,                    0x0000000000fc0000),
+    MMXDATA_INIT (.mmx_565_b,                    0x00000000000000f8),
+    MMXDATA_INIT (.mmx_mask_0,                   0xffffffffffff0000),
+    MMXDATA_INIT (.mmx_mask_1,                   0xffffffff0000ffff),
+    MMXDATA_INIT (.mmx_mask_2,                   0xffff0000ffffffff),
+    MMXDATA_INIT (.mmx_mask_3,                   0x0000ffffffffffff),
+    MMXDATA_INIT (.mmx_full_alpha,               0x00ff000000000000),
+    MMXDATA_INIT (.mmx_ffff0000ffff0000,         0xffff0000ffff0000),
+    MMXDATA_INIT (.mmx_0000ffff00000000,         0x0000ffff00000000),
+    MMXDATA_INIT (.mmx_000000000000ffff,         0x000000000000ffff),
+};
+
+#ifdef __GNUC__
+#    ifdef __ICC
+#        define MC(x)  M64 (c.mmx_ ## x)
+#    else
+#        define MC(x) ((__m64)c.mmx_ ## x)
+#    endif
+#else
+#    define MC(x) c.mmx_ ## x
+#endif
+
+static force_inline __m64
+M64 (uint64_t x)
+{
+#ifdef __ICC
+    return _mm_cvtsi64_m64 (x);
+#elif defined M64_MEMBER        /* __m64 is a struct, not an integral type */
+    __m64 res;
+
+    res.M64_MEMBER = x;
+    return res;
+#else                           /* __m64 is an integral type */
+    return (__m64)x;
+#endif
+}
+
+static force_inline uint64_t
+UINT64 (__m64 x)
+{
+#ifdef __ICC
+    return _mm_cvtm64_si64 (x);
+#elif defined M64_MEMBER        /* __m64 is a struct, not an integral type */
+    uint64_t res = x.M64_MEMBER;
+    return res;
+#else                           /* __m64 is an integral type */
+    return (uint64_t)x;
+#endif
+}
+
+static force_inline __m64
+shift (__m64 v,
+       int   s)
+{
+    if (s > 0)
+	return _mm_slli_si64 (v, s);
+    else if (s < 0)
+	return _mm_srli_si64 (v, -s);
+    else
+	return v;
+}
+
+static force_inline __m64
+negate (__m64 mask)
+{
+    return _mm_xor_si64 (mask, MC (4x00ff));
+}
+
+static force_inline __m64
+pix_multiply (__m64 a, __m64 b)
+{
+    __m64 res;
+
+    res = _mm_mullo_pi16 (a, b);
+    res = _mm_adds_pu16 (res, MC (4x0080));
+    res = _mm_adds_pu16 (res, _mm_srli_pi16 (res, 8));
+    res = _mm_srli_pi16 (res, 8);
+
+    return res;
+}
+
+static force_inline __m64
+pix_add (__m64 a, __m64 b)
+{
+    return _mm_adds_pu8 (a, b);
+}
+
+static force_inline __m64
+expand_alpha (__m64 pixel)
+{
+    __m64 t1, t2;
+
+    t1 = shift (pixel, -48);
+    t2 = shift (t1, 16);
+    t1 = _mm_or_si64 (t1, t2);
+    t2 = shift (t1, 32);
+    t1 = _mm_or_si64 (t1, t2);
+
+    return t1;
+}
+
+static force_inline __m64
+expand_alpha_rev (__m64 pixel)
+{
+    __m64 t1, t2;
+
+    /* move alpha to low 16 bits and zero the rest */
+    t1 = shift (pixel,  48);
+    t1 = shift (t1, -48);
+
+    t2 = shift (t1, 16);
+    t1 = _mm_or_si64 (t1, t2);
+    t2 = shift (t1, 32);
+    t1 = _mm_or_si64 (t1, t2);
+
+    return t1;
+}
+
+static force_inline __m64
+invert_colors (__m64 pixel)
+{
+    __m64 x, y, z;
+
+    x = y = z = pixel;
+
+    x = _mm_and_si64 (x, MC (ffff0000ffff0000));
+    y = _mm_and_si64 (y, MC (000000000000ffff));
+    z = _mm_and_si64 (z, MC (0000ffff00000000));
+
+    y = shift (y, 32);
+    z = shift (z, -32);
+
+    x = _mm_or_si64 (x, y);
+    x = _mm_or_si64 (x, z);
+
+    return x;
+}
+
+static force_inline __m64
+over (__m64 src,
+      __m64 srca,
+      __m64 dest)
+{
+    return _mm_adds_pu8 (src, pix_multiply (dest, negate (srca)));
+}
+
+static force_inline __m64
+over_rev_non_pre (__m64 src, __m64 dest)
+{
+    __m64 srca = expand_alpha (src);
+    __m64 srcfaaa = _mm_or_si64 (srca, MC (full_alpha));
+
+    return over (pix_multiply (invert_colors (src), srcfaaa), srca, dest);
+}
+
+static force_inline __m64
+in (__m64 src, __m64 mask)
+{
+    return pix_multiply (src, mask);
+}
+
+static force_inline __m64
+in_over_full_src_alpha (__m64 src, __m64 mask, __m64 dest)
+{
+    src = _mm_or_si64 (src, MC (full_alpha));
+
+    return over (in (src, mask), mask, dest);
+}
+
+#ifndef _MSC_VER
+static force_inline __m64
+in_over (__m64 src, __m64 srca, __m64 mask, __m64 dest)
+{
+    return over (in (src, mask), pix_multiply (srca, mask), dest);
+}
+
+#else
+
+#define in_over(src, srca, mask, dest)					\
+    over (in (src, mask), pix_multiply (srca, mask), dest)
+
+#endif
+
+static force_inline __m64
+load8888 (uint32_t v)
+{
+    return _mm_unpacklo_pi8 (_mm_cvtsi32_si64 (v), _mm_setzero_si64 ());
+}
+
+static force_inline __m64
+pack8888 (__m64 lo, __m64 hi)
+{
+    return _mm_packs_pu16 (lo, hi);
+}
+
+static force_inline uint32_t
+store8888 (__m64 v)
+{
+    return _mm_cvtsi64_si32 (pack8888 (v, _mm_setzero_si64 ()));
+}
+
+/* Expand 16 bits positioned at @pos (0-3) of a mmx register into
+ *
+ *    00RR00GG00BB
+ *
+ * --- Expanding 565 in the low word ---
+ *
+ * m = (m << (32 - 3)) | (m << (16 - 5)) | m;
+ * m = m & (01f0003f001f);
+ * m = m * (008404100840);
+ * m = m >> 8;
+ *
+ * Note the trick here - the top word is shifted by another nibble to
+ * avoid it bumping into the middle word
+ */
+static force_inline __m64
+expand565 (__m64 pixel, int pos)
+{
+    __m64 p = pixel;
+    __m64 t1, t2;
+
+    /* move pixel to low 16 bit and zero the rest */
+    p = shift (shift (p, (3 - pos) * 16), -48);
+
+    t1 = shift (p, 36 - 11);
+    t2 = shift (p, 16 - 5);
+
+    p = _mm_or_si64 (t1, p);
+    p = _mm_or_si64 (t2, p);
+    p = _mm_and_si64 (p, MC (565_rgb));
+
+    pixel = _mm_mullo_pi16 (p, MC (565_unpack_multiplier));
+    return _mm_srli_pi16 (pixel, 8);
+}
+
+static force_inline __m64
+expand8888 (__m64 in, int pos)
+{
+    if (pos == 0)
+	return _mm_unpacklo_pi8 (in, _mm_setzero_si64 ());
+    else
+	return _mm_unpackhi_pi8 (in, _mm_setzero_si64 ());
+}
+
+static force_inline __m64
+expandx888 (__m64 in, int pos)
+{
+    return _mm_or_si64 (expand8888 (in, pos), MC (full_alpha));
+}
+
+static force_inline __m64
+pack_565 (__m64 pixel, __m64 target, int pos)
+{
+    __m64 p = pixel;
+    __m64 t = target;
+    __m64 r, g, b;
+
+    r = _mm_and_si64 (p, MC (565_r));
+    g = _mm_and_si64 (p, MC (565_g));
+    b = _mm_and_si64 (p, MC (565_b));
+
+    r = shift (r, -(32 - 8) + pos * 16);
+    g = shift (g, -(16 - 3) + pos * 16);
+    b = shift (b, -(0  + 3) + pos * 16);
+
+    if (pos == 0)
+	t = _mm_and_si64 (t, MC (mask_0));
+    else if (pos == 1)
+	t = _mm_and_si64 (t, MC (mask_1));
+    else if (pos == 2)
+	t = _mm_and_si64 (t, MC (mask_2));
+    else if (pos == 3)
+	t = _mm_and_si64 (t, MC (mask_3));
+
+    p = _mm_or_si64 (r, t);
+    p = _mm_or_si64 (g, p);
+
+    return _mm_or_si64 (b, p);
+}
+
+#ifndef _MSC_VER
+
+static force_inline __m64
+pix_add_mul (__m64 x, __m64 a, __m64 y, __m64 b)
+{
+    x = pix_multiply (x, a);
+    y = pix_multiply (y, b);
+
+    return pix_add (x, y);
+}
+
+#else
+
+#define pix_add_mul(x, a, y, b)	 \
+    ( x = pix_multiply (x, a),	 \
+      y = pix_multiply (y, a),	 \
+      pix_add (x, y) )
+
+#endif
+
+/* --------------- MMX code patch for fbcompose.c --------------------- */
+
+static force_inline uint32_t
+combine (const uint32_t *src, const uint32_t *mask)
+{
+    uint32_t ssrc = *src;
+
+    if (mask)
+    {
+	__m64 m = load8888 (*mask);
+	__m64 s = load8888 (ssrc);
+
+	m = expand_alpha (m);
+	s = pix_multiply (s, m);
+
+	ssrc = store8888 (s);
+    }
+
+    return ssrc;
+}
+
+static void
+mmx_combine_over_u (pixman_implementation_t *imp,
+                    pixman_op_t              op,
+                    uint32_t *               dest,
+                    const uint32_t *         src,
+                    const uint32_t *         mask,
+                    int                      width)
+{
+    const uint32_t *end = dest + width;
+
+    while (dest < end)
+    {
+	uint32_t ssrc = combine (src, mask);
+	uint32_t a = ssrc >> 24;
+
+	if (a == 0xff)
+	{
+	    *dest = ssrc;
+	}
+	else if (ssrc)
+	{
+	    __m64 s, sa;
+	    s = load8888 (ssrc);
+	    sa = expand_alpha (s);
+	    *dest = store8888 (over (s, sa, load8888 (*dest)));
+	}
+
+	++dest;
+	++src;
+	if (mask)
+	    ++mask;
+    }
+    _mm_empty ();
+}
+
+static void
+mmx_combine_over_reverse_u (pixman_implementation_t *imp,
+                            pixman_op_t              op,
+                            uint32_t *               dest,
+                            const uint32_t *         src,
+                            const uint32_t *         mask,
+                            int                      width)
+{
+    const uint32_t *end = dest + width;
+
+    while (dest < end)
+    {
+	__m64 d, da;
+	uint32_t s = combine (src, mask);
+
+	d = load8888 (*dest);
+	da = expand_alpha (d);
+	*dest = store8888 (over (d, da, load8888 (s)));
+
+	++dest;
+	++src;
+	if (mask)
+	    mask++;
+    }
+    _mm_empty ();
+}
+
+static void
+mmx_combine_in_u (pixman_implementation_t *imp,
+                  pixman_op_t              op,
+                  uint32_t *               dest,
+                  const uint32_t *         src,
+                  const uint32_t *         mask,
+                  int                      width)
+{
+    const uint32_t *end = dest + width;
+
+    while (dest < end)
+    {
+	__m64 x, a;
+
+	x = load8888 (combine (src, mask));
+	a = load8888 (*dest);
+	a = expand_alpha (a);
+	x = pix_multiply (x, a);
+
+	*dest = store8888 (x);
+
+	++dest;
+	++src;
+	if (mask)
+	    mask++;
+    }
+    _mm_empty ();
+}
+
+static void
+mmx_combine_in_reverse_u (pixman_implementation_t *imp,
+                          pixman_op_t              op,
+                          uint32_t *               dest,
+                          const uint32_t *         src,
+                          const uint32_t *         mask,
+                          int                      width)
+{
+    const uint32_t *end = dest + width;
+
+    while (dest < end)
+    {
+	__m64 x, a;
+
+	x = load8888 (*dest);
+	a = load8888 (combine (src, mask));
+	a = expand_alpha (a);
+	x = pix_multiply (x, a);
+	*dest = store8888 (x);
+
+	++dest;
+	++src;
+	if (mask)
+	    mask++;
+    }
+    _mm_empty ();
+}
+
+static void
+mmx_combine_out_u (pixman_implementation_t *imp,
+                   pixman_op_t              op,
+                   uint32_t *               dest,
+                   const uint32_t *         src,
+                   const uint32_t *         mask,
+                   int                      width)
+{
+    const uint32_t *end = dest + width;
+
+    while (dest < end)
+    {
+	__m64 x, a;
+
+	x = load8888 (combine (src, mask));
+	a = load8888 (*dest);
+	a = expand_alpha (a);
+	a = negate (a);
+	x = pix_multiply (x, a);
+	*dest = store8888 (x);
+
+	++dest;
+	++src;
+	if (mask)
+	    mask++;
+    }
+    _mm_empty ();
+}
+
+static void
+mmx_combine_out_reverse_u (pixman_implementation_t *imp,
+                           pixman_op_t              op,
+                           uint32_t *               dest,
+                           const uint32_t *         src,
+                           const uint32_t *         mask,
+                           int                      width)
+{
+    const uint32_t *end = dest + width;
+
+    while (dest < end)
+    {
+	__m64 x, a;
+
+	x = load8888 (*dest);
+	a = load8888 (combine (src, mask));
+	a = expand_alpha (a);
+	a = negate (a);
+	x = pix_multiply (x, a);
+
+	*dest = store8888 (x);
+
+	++dest;
+	++src;
+	if (mask)
+	    mask++;
+    }
+    _mm_empty ();
+}
+
+static void
+mmx_combine_atop_u (pixman_implementation_t *imp,
+                    pixman_op_t              op,
+                    uint32_t *               dest,
+                    const uint32_t *         src,
+                    const uint32_t *         mask,
+                    int                      width)
+{
+    const uint32_t *end = dest + width;
+
+    while (dest < end)
+    {
+	__m64 s, da, d, sia;
+
+	s = load8888 (combine (src, mask));
+	d = load8888 (*dest);
+	sia = expand_alpha (s);
+	sia = negate (sia);
+	da = expand_alpha (d);
+	s = pix_add_mul (s, da, d, sia);
+	*dest = store8888 (s);
+
+	++dest;
+	++src;
+	if (mask)
+	    mask++;
+    }
+    _mm_empty ();
+}
+
+static void
+mmx_combine_atop_reverse_u (pixman_implementation_t *imp,
+                            pixman_op_t              op,
+                            uint32_t *               dest,
+                            const uint32_t *         src,
+                            const uint32_t *         mask,
+                            int                      width)
+{
+    const uint32_t *end;
+
+    end = dest + width;
+
+    while (dest < end)
+    {
+	__m64 s, dia, d, sa;
+
+	s = load8888 (combine (src, mask));
+	d = load8888 (*dest);
+	sa = expand_alpha (s);
+	dia = expand_alpha (d);
+	dia = negate (dia);
+	s = pix_add_mul (s, dia, d, sa);
+	*dest = store8888 (s);
+
+	++dest;
+	++src;
+	if (mask)
+	    mask++;
+    }
+    _mm_empty ();
+}
+
+static void
+mmx_combine_xor_u (pixman_implementation_t *imp,
+                   pixman_op_t              op,
+                   uint32_t *               dest,
+                   const uint32_t *         src,
+                   const uint32_t *         mask,
+                   int                      width)
+{
+    const uint32_t *end = dest + width;
+
+    while (dest < end)
+    {
+	__m64 s, dia, d, sia;
+
+	s = load8888 (combine (src, mask));
+	d = load8888 (*dest);
+	sia = expand_alpha (s);
+	dia = expand_alpha (d);
+	sia = negate (sia);
+	dia = negate (dia);
+	s = pix_add_mul (s, dia, d, sia);
+	*dest = store8888 (s);
+
+	++dest;
+	++src;
+	if (mask)
+	    mask++;
+    }
+    _mm_empty ();
+}
+
+static void
+mmx_combine_add_u (pixman_implementation_t *imp,
+                   pixman_op_t              op,
+                   uint32_t *               dest,
+                   const uint32_t *         src,
+                   const uint32_t *         mask,
+                   int                      width)
+{
+    const uint32_t *end = dest + width;
+
+    while (dest < end)
+    {
+	__m64 s, d;
+
+	s = load8888 (combine (src, mask));
+	d = load8888 (*dest);
+	s = pix_add (s, d);
+	*dest = store8888 (s);
+
+	++dest;
+	++src;
+	if (mask)
+	    mask++;
+    }
+    _mm_empty ();
+}
+
+static void
+mmx_combine_saturate_u (pixman_implementation_t *imp,
+                        pixman_op_t              op,
+                        uint32_t *               dest,
+                        const uint32_t *         src,
+                        const uint32_t *         mask,
+                        int                      width)
+{
+    const uint32_t *end = dest + width;
+
+    while (dest < end)
+    {
+	uint32_t s = combine (src, mask);
+	uint32_t d = *dest;
+	__m64 ms = load8888 (s);
+	__m64 md = load8888 (d);
+	uint32_t sa = s >> 24;
+	uint32_t da = ~d >> 24;
+
+	if (sa > da)
+	{
+	    __m64 msa = load8888 (DIV_UN8 (da, sa) << 24);
+	    msa = expand_alpha (msa);
+	    ms = pix_multiply (ms, msa);
+	}
+
+	md = pix_add (md, ms);
+	*dest = store8888 (md);
+
+	++src;
+	++dest;
+	if (mask)
+	    mask++;
+    }
+    _mm_empty ();
+}
+
+static void
+mmx_combine_src_ca (pixman_implementation_t *imp,
+                    pixman_op_t              op,
+                    uint32_t *               dest,
+                    const uint32_t *         src,
+                    const uint32_t *         mask,
+                    int                      width)
+{
+    const uint32_t *end = src + width;
+
+    while (src < end)
+    {
+	__m64 a = load8888 (*mask);
+	__m64 s = load8888 (*src);
+
+	s = pix_multiply (s, a);
+	*dest = store8888 (s);
+
+	++src;
+	++mask;
+	++dest;
+    }
+    _mm_empty ();
+}
+
+static void
+mmx_combine_over_ca (pixman_implementation_t *imp,
+                     pixman_op_t              op,
+                     uint32_t *               dest,
+                     const uint32_t *         src,
+                     const uint32_t *         mask,
+                     int                      width)
+{
+    const uint32_t *end = src + width;
+
+    while (src < end)
+    {
+	__m64 a = load8888 (*mask);
+	__m64 s = load8888 (*src);
+	__m64 d = load8888 (*dest);
+	__m64 sa = expand_alpha (s);
+
+	*dest = store8888 (in_over (s, sa, a, d));
+
+	++src;
+	++dest;
+	++mask;
+    }
+    _mm_empty ();
+}
+
+static void
+mmx_combine_over_reverse_ca (pixman_implementation_t *imp,
+                             pixman_op_t              op,
+                             uint32_t *               dest,
+                             const uint32_t *         src,
+                             const uint32_t *         mask,
+                             int                      width)
+{
+    const uint32_t *end = src + width;
+
+    while (src < end)
+    {
+	__m64 a = load8888 (*mask);
+	__m64 s = load8888 (*src);
+	__m64 d = load8888 (*dest);
+	__m64 da = expand_alpha (d);
+
+	*dest = store8888 (over (d, da, in (s, a)));
+
+	++src;
+	++dest;
+	++mask;
+    }
+    _mm_empty ();
+}
+
+static void
+mmx_combine_in_ca (pixman_implementation_t *imp,
+                   pixman_op_t              op,
+                   uint32_t *               dest,
+                   const uint32_t *         src,
+                   const uint32_t *         mask,
+                   int                      width)
+{
+    const uint32_t *end = src + width;
+
+    while (src < end)
+    {
+	__m64 a = load8888 (*mask);
+	__m64 s = load8888 (*src);
+	__m64 d = load8888 (*dest);
+	__m64 da = expand_alpha (d);
+
+	s = pix_multiply (s, a);
+	s = pix_multiply (s, da);
+	*dest = store8888 (s);
+
+	++src;
+	++dest;
+	++mask;
+    }
+    _mm_empty ();
+}
+
+static void
+mmx_combine_in_reverse_ca (pixman_implementation_t *imp,
+                           pixman_op_t              op,
+                           uint32_t *               dest,
+                           const uint32_t *         src,
+                           const uint32_t *         mask,
+                           int                      width)
+{
+    const uint32_t *end = src + width;
+
+    while (src < end)
+    {
+	__m64 a = load8888 (*mask);
+	__m64 s = load8888 (*src);
+	__m64 d = load8888 (*dest);
+	__m64 sa = expand_alpha (s);
+
+	a = pix_multiply (a, sa);
+	d = pix_multiply (d, a);
+	*dest = store8888 (d);
+
+	++src;
+	++dest;
+	++mask;
+    }
+    _mm_empty ();
+}
+
+static void
+mmx_combine_out_ca (pixman_implementation_t *imp,
+                    pixman_op_t              op,
+                    uint32_t *               dest,
+                    const uint32_t *         src,
+                    const uint32_t *         mask,
+                    int                      width)
+{
+    const uint32_t *end = src + width;
+
+    while (src < end)
+    {
+	__m64 a = load8888 (*mask);
+	__m64 s = load8888 (*src);
+	__m64 d = load8888 (*dest);
+	__m64 da = expand_alpha (d);
+
+	da = negate (da);
+	s = pix_multiply (s, a);
+	s = pix_multiply (s, da);
+	*dest = store8888 (s);
+
+	++src;
+	++dest;
+	++mask;
+    }
+    _mm_empty ();
+}
+
+static void
+mmx_combine_out_reverse_ca (pixman_implementation_t *imp,
+                            pixman_op_t              op,
+                            uint32_t *               dest,
+                            const uint32_t *         src,
+                            const uint32_t *         mask,
+                            int                      width)
+{
+    const uint32_t *end = src + width;
+
+    while (src < end)
+    {
+	__m64 a = load8888 (*mask);
+	__m64 s = load8888 (*src);
+	__m64 d = load8888 (*dest);
+	__m64 sa = expand_alpha (s);
+
+	a = pix_multiply (a, sa);
+	a = negate (a);
+	d = pix_multiply (d, a);
+	*dest = store8888 (d);
+
+	++src;
+	++dest;
+	++mask;
+    }
+    _mm_empty ();
+}
+
+static void
+mmx_combine_atop_ca (pixman_implementation_t *imp,
+                     pixman_op_t              op,
+                     uint32_t *               dest,
+                     const uint32_t *         src,
+                     const uint32_t *         mask,
+                     int                      width)
+{
+    const uint32_t *end = src + width;
+
+    while (src < end)
+    {
+	__m64 a = load8888 (*mask);
+	__m64 s = load8888 (*src);
+	__m64 d = load8888 (*dest);
+	__m64 da = expand_alpha (d);
+	__m64 sa = expand_alpha (s);
+
+	s = pix_multiply (s, a);
+	a = pix_multiply (a, sa);
+	a = negate (a);
+	d = pix_add_mul (d, a, s, da);
+	*dest = store8888 (d);
+
+	++src;
+	++dest;
+	++mask;
+    }
+    _mm_empty ();
+}
+
+static void
+mmx_combine_atop_reverse_ca (pixman_implementation_t *imp,
+                             pixman_op_t              op,
+                             uint32_t *               dest,
+                             const uint32_t *         src,
+                             const uint32_t *         mask,
+                             int                      width)
+{
+    const uint32_t *end = src + width;
+
+    while (src < end)
+    {
+	__m64 a = load8888 (*mask);
+	__m64 s = load8888 (*src);
+	__m64 d = load8888 (*dest);
+	__m64 da = expand_alpha (d);
+	__m64 sa = expand_alpha (s);
+
+	s = pix_multiply (s, a);
+	a = pix_multiply (a, sa);
+	da = negate (da);
+	d = pix_add_mul (d, a, s, da);
+	*dest = store8888 (d);
+
+	++src;
+	++dest;
+	++mask;
+    }
+    _mm_empty ();
+}
+
+static void
+mmx_combine_xor_ca (pixman_implementation_t *imp,
+                    pixman_op_t              op,
+                    uint32_t *               dest,
+                    const uint32_t *         src,
+                    const uint32_t *         mask,
+                    int                      width)
+{
+    const uint32_t *end = src + width;
+
+    while (src < end)
+    {
+	__m64 a = load8888 (*mask);
+	__m64 s = load8888 (*src);
+	__m64 d = load8888 (*dest);
+	__m64 da = expand_alpha (d);
+	__m64 sa = expand_alpha (s);
+
+	s = pix_multiply (s, a);
+	a = pix_multiply (a, sa);
+	da = negate (da);
+	a = negate (a);
+	d = pix_add_mul (d, a, s, da);
+	*dest = store8888 (d);
+
+	++src;
+	++dest;
+	++mask;
+    }
+    _mm_empty ();
+}
+
+static void
+mmx_combine_add_ca (pixman_implementation_t *imp,
+                    pixman_op_t              op,
+                    uint32_t *               dest,
+                    const uint32_t *         src,
+                    const uint32_t *         mask,
+                    int                      width)
+{
+    const uint32_t *end = src + width;
+
+    while (src < end)
+    {
+	__m64 a = load8888 (*mask);
+	__m64 s = load8888 (*src);
+	__m64 d = load8888 (*dest);
+
+	s = pix_multiply (s, a);
+	d = pix_add (s, d);
+	*dest = store8888 (d);
+
+	++src;
+	++dest;
+	++mask;
+    }
+    _mm_empty ();
+}
+
+/* ------------- MMX code paths called from fbpict.c -------------------- */
+
+static void
+mmx_composite_over_n_8888 (pixman_implementation_t *imp,
+                           pixman_op_t              op,
+                           pixman_image_t *         src_image,
+                           pixman_image_t *         mask_image,
+                           pixman_image_t *         dst_image,
+                           int32_t                  src_x,
+                           int32_t                  src_y,
+                           int32_t                  mask_x,
+                           int32_t                  mask_y,
+                           int32_t                  dest_x,
+                           int32_t                  dest_y,
+                           int32_t                  width,
+                           int32_t                  height)
+{
+    uint32_t src;
+    uint32_t    *dst_line, *dst;
+    int32_t w;
+    int dst_stride;
+    __m64 vsrc, vsrca;
+
+    CHECKPOINT ();
+
+    src = _pixman_image_get_solid (src_image, dst_image->bits.format);
+
+    if (src == 0)
+	return;
+
+    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
+
+    vsrc = load8888 (src);
+    vsrca = expand_alpha (vsrc);
+
+    while (height--)
+    {
+	dst = dst_line;
+	dst_line += dst_stride;
+	w = width;
+
+	CHECKPOINT ();
+
+	while (w && (unsigned long)dst & 7)
+	{
+	    *dst = store8888 (over (vsrc, vsrca, load8888 (*dst)));
+
+	    w--;
+	    dst++;
+	}
+
+	while (w >= 2)
+	{
+	    __m64 vdest;
+	    __m64 dest0, dest1;
+
+	    vdest = *(__m64 *)dst;
+
+	    dest0 = over (vsrc, vsrca, expand8888 (vdest, 0));
+	    dest1 = over (vsrc, vsrca, expand8888 (vdest, 1));
+
+	    *(__m64 *)dst = pack8888 (dest0, dest1);
+
+	    dst += 2;
+	    w -= 2;
+	}
+
+	CHECKPOINT ();
+
+	while (w)
+	{
+	    *dst = store8888 (over (vsrc, vsrca, load8888 (*dst)));
+
+	    w--;
+	    dst++;
+	}
+    }
+
+    _mm_empty ();
+}
+
+static void
+mmx_composite_over_n_0565 (pixman_implementation_t *imp,
+                           pixman_op_t              op,
+                           pixman_image_t *         src_image,
+                           pixman_image_t *         mask_image,
+                           pixman_image_t *         dst_image,
+                           int32_t                  src_x,
+                           int32_t                  src_y,
+                           int32_t                  mask_x,
+                           int32_t                  mask_y,
+                           int32_t                  dest_x,
+                           int32_t                  dest_y,
+                           int32_t                  width,
+                           int32_t                  height)
+{
+    uint32_t src;
+    uint16_t    *dst_line, *dst;
+    int32_t w;
+    int dst_stride;
+    __m64 vsrc, vsrca;
+
+    CHECKPOINT ();
+
+    src = _pixman_image_get_solid (src_image, dst_image->bits.format);
+
+    if (src == 0)
+	return;
+
+    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, uint16_t, dst_stride, dst_line, 1);
+
+    vsrc = load8888 (src);
+    vsrca = expand_alpha (vsrc);
+
+    while (height--)
+    {
+	dst = dst_line;
+	dst_line += dst_stride;
+	w = width;
+
+	CHECKPOINT ();
+
+	while (w && (unsigned long)dst & 7)
+	{
+	    uint64_t d = *dst;
+	    __m64 vdest = expand565 (M64 (d), 0);
+
+	    vdest = pack_565 (over (vsrc, vsrca, vdest), vdest, 0);
+	    *dst = UINT64 (vdest);
+
+	    w--;
+	    dst++;
+	}
+
+	while (w >= 4)
+	{
+	    __m64 vdest;
+
+	    vdest = *(__m64 *)dst;
+
+	    vdest = pack_565 (over (vsrc, vsrca, expand565 (vdest, 0)), vdest, 0);
+	    vdest = pack_565 (over (vsrc, vsrca, expand565 (vdest, 1)), vdest, 1);
+	    vdest = pack_565 (over (vsrc, vsrca, expand565 (vdest, 2)), vdest, 2);
+	    vdest = pack_565 (over (vsrc, vsrca, expand565 (vdest, 3)), vdest, 3);
+
+	    *(__m64 *)dst = vdest;
+
+	    dst += 4;
+	    w -= 4;
+	}
+
+	CHECKPOINT ();
+
+	while (w)
+	{
+	    uint64_t d = *dst;
+	    __m64 vdest = expand565 (M64 (d), 0);
+
+	    vdest = pack_565 (over (vsrc, vsrca, vdest), vdest, 0);
+	    *dst = UINT64 (vdest);
+
+	    w--;
+	    dst++;
+	}
+    }
+
+    _mm_empty ();
+}
+
+static void
+mmx_composite_over_n_8888_8888_ca (pixman_implementation_t *imp,
+                                   pixman_op_t              op,
+                                   pixman_image_t *         src_image,
+                                   pixman_image_t *         mask_image,
+                                   pixman_image_t *         dst_image,
+                                   int32_t                  src_x,
+                                   int32_t                  src_y,
+                                   int32_t                  mask_x,
+                                   int32_t                  mask_y,
+                                   int32_t                  dest_x,
+                                   int32_t                  dest_y,
+                                   int32_t                  width,
+                                   int32_t                  height)
+{
+    uint32_t src, srca;
+    uint32_t    *dst_line;
+    uint32_t    *mask_line;
+    int dst_stride, mask_stride;
+    __m64 vsrc, vsrca;
+
+    CHECKPOINT ();
+
+    src = _pixman_image_get_solid (src_image, dst_image->bits.format);
+
+    srca = src >> 24;
+    if (src == 0)
+	return;
+
+    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
+    PIXMAN_IMAGE_GET_LINE (mask_image, mask_x, mask_y, uint32_t, mask_stride, mask_line, 1);
+
+    vsrc = load8888 (src);
+    vsrca = expand_alpha (vsrc);
+
+    while (height--)
+    {
+	int twidth = width;
+	uint32_t *p = (uint32_t *)mask_line;
+	uint32_t *q = (uint32_t *)dst_line;
+
+	while (twidth && (unsigned long)q & 7)
+	{
+	    uint32_t m = *(uint32_t *)p;
+
+	    if (m)
+	    {
+		__m64 vdest = load8888 (*q);
+		vdest = in_over (vsrc, vsrca, load8888 (m), vdest);
+		*q = store8888 (vdest);
+	    }
+
+	    twidth--;
+	    p++;
+	    q++;
+	}
+
+	while (twidth >= 2)
+	{
+	    uint32_t m0, m1;
+	    m0 = *p;
+	    m1 = *(p + 1);
+
+	    if (m0 | m1)
+	    {
+		__m64 dest0, dest1;
+		__m64 vdest = *(__m64 *)q;
+
+		dest0 = in_over (vsrc, vsrca, load8888 (m0),
+		                 expand8888 (vdest, 0));
+		dest1 = in_over (vsrc, vsrca, load8888 (m1),
+		                 expand8888 (vdest, 1));
+
+		*(__m64 *)q = pack8888 (dest0, dest1);
+	    }
+
+	    p += 2;
+	    q += 2;
+	    twidth -= 2;
+	}
+
+	while (twidth)
+	{
+	    uint32_t m = *(uint32_t *)p;
+
+	    if (m)
+	    {
+		__m64 vdest = load8888 (*q);
+		vdest = in_over (vsrc, vsrca, load8888 (m), vdest);
+		*q = store8888 (vdest);
+	    }
+
+	    twidth--;
+	    p++;
+	    q++;
+	}
+
+	dst_line += dst_stride;
+	mask_line += mask_stride;
+    }
+
+    _mm_empty ();
+}
+
+static void
+mmx_composite_over_8888_n_8888 (pixman_implementation_t *imp,
+                                pixman_op_t              op,
+                                pixman_image_t *         src_image,
+                                pixman_image_t *         mask_image,
+                                pixman_image_t *         dst_image,
+                                int32_t                  src_x,
+                                int32_t                  src_y,
+                                int32_t                  mask_x,
+                                int32_t                  mask_y,
+                                int32_t                  dest_x,
+                                int32_t                  dest_y,
+                                int32_t                  width,
+                                int32_t                  height)
+{
+    uint32_t    *dst_line, *dst;
+    uint32_t    *src_line, *src;
+    uint32_t mask;
+    __m64 vmask;
+    int dst_stride, src_stride;
+    int32_t w;
+    __m64 srca;
+
+    CHECKPOINT ();
+
+    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
+    PIXMAN_IMAGE_GET_LINE (src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
+
+    mask = _pixman_image_get_solid (mask_image, dst_image->bits.format);
+    mask &= 0xff000000;
+    mask = mask | mask >> 8 | mask >> 16 | mask >> 24;
+    vmask = load8888 (mask);
+    srca = MC (4x00ff);
+
+    while (height--)
+    {
+	dst = dst_line;
+	dst_line += dst_stride;
+	src = src_line;
+	src_line += src_stride;
+	w = width;
+
+	while (w && (unsigned long)dst & 7)
+	{
+	    __m64 s = load8888 (*src);
+	    __m64 d = load8888 (*dst);
+
+	    *dst = store8888 (in_over (s, expand_alpha (s), vmask, d));
+
+	    w--;
+	    dst++;
+	    src++;
+	}
+
+	while (w >= 2)
+	{
+	    __m64 vs = *(__m64 *)src;
+	    __m64 vd = *(__m64 *)dst;
+	    __m64 vsrc0 = expand8888 (vs, 0);
+	    __m64 vsrc1 = expand8888 (vs, 1);
+
+	    *(__m64 *)dst = pack8888 (
+	        in_over (vsrc0, expand_alpha (vsrc0), vmask, expand8888 (vd, 0)),
+	        in_over (vsrc1, expand_alpha (vsrc1), vmask, expand8888 (vd, 1)));
+
+	    w -= 2;
+	    dst += 2;
+	    src += 2;
+	}
+
+	while (w)
+	{
+	    __m64 s = load8888 (*src);
+	    __m64 d = load8888 (*dst);
+
+	    *dst = store8888 (in_over (s, expand_alpha (s), vmask, d));
+
+	    w--;
+	    dst++;
+	    src++;
+	}
+    }
+
+    _mm_empty ();
+}
+
+static void
+mmx_composite_over_x888_n_8888 (pixman_implementation_t *imp,
+                                pixman_op_t              op,
+                                pixman_image_t *         src_image,
+                                pixman_image_t *         mask_image,
+                                pixman_image_t *         dst_image,
+                                int32_t                  src_x,
+                                int32_t                  src_y,
+                                int32_t                  mask_x,
+                                int32_t                  mask_y,
+                                int32_t                  dest_x,
+                                int32_t                  dest_y,
+                                int32_t                  width,
+                                int32_t                  height)
+{
+    uint32_t *dst_line, *dst;
+    uint32_t *src_line, *src;
+    uint32_t mask;
+    __m64 vmask;
+    int dst_stride, src_stride;
+    int32_t w;
+    __m64 srca;
+
+    CHECKPOINT ();
+
+    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
+    PIXMAN_IMAGE_GET_LINE (src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
+    mask = _pixman_image_get_solid (mask_image, dst_image->bits.format);
+
+    mask &= 0xff000000;
+    mask = mask | mask >> 8 | mask >> 16 | mask >> 24;
+    vmask = load8888 (mask);
+    srca = MC (4x00ff);
+
+    while (height--)
+    {
+	dst = dst_line;
+	dst_line += dst_stride;
+	src = src_line;
+	src_line += src_stride;
+	w = width;
+
+	while (w && (unsigned long)dst & 7)
+	{
+	    __m64 s = load8888 (*src | 0xff000000);
+	    __m64 d = load8888 (*dst);
+
+	    *dst = store8888 (in_over (s, srca, vmask, d));
+
+	    w--;
+	    dst++;
+	    src++;
+	}
+
+	while (w >= 16)
+	{
+	    __m64 vd0 = *(__m64 *)(dst + 0);
+	    __m64 vd1 = *(__m64 *)(dst + 2);
+	    __m64 vd2 = *(__m64 *)(dst + 4);
+	    __m64 vd3 = *(__m64 *)(dst + 6);
+	    __m64 vd4 = *(__m64 *)(dst + 8);
+	    __m64 vd5 = *(__m64 *)(dst + 10);
+	    __m64 vd6 = *(__m64 *)(dst + 12);
+	    __m64 vd7 = *(__m64 *)(dst + 14);
+
+	    __m64 vs0 = *(__m64 *)(src + 0);
+	    __m64 vs1 = *(__m64 *)(src + 2);
+	    __m64 vs2 = *(__m64 *)(src + 4);
+	    __m64 vs3 = *(__m64 *)(src + 6);
+	    __m64 vs4 = *(__m64 *)(src + 8);
+	    __m64 vs5 = *(__m64 *)(src + 10);
+	    __m64 vs6 = *(__m64 *)(src + 12);
+	    __m64 vs7 = *(__m64 *)(src + 14);
+
+	    vd0 = pack8888 (
+	        in_over (expandx888 (vs0, 0), srca, vmask, expand8888 (vd0, 0)),
+	        in_over (expandx888 (vs0, 1), srca, vmask, expand8888 (vd0, 1)));
+
+	    vd1 = pack8888 (
+	        in_over (expandx888 (vs1, 0), srca, vmask, expand8888 (vd1, 0)),
+	        in_over (expandx888 (vs1, 1), srca, vmask, expand8888 (vd1, 1)));
+
+	    vd2 = pack8888 (
+	        in_over (expandx888 (vs2, 0), srca, vmask, expand8888 (vd2, 0)),
+	        in_over (expandx888 (vs2, 1), srca, vmask, expand8888 (vd2, 1)));
+
+	    vd3 = pack8888 (
+	        in_over (expandx888 (vs3, 0), srca, vmask, expand8888 (vd3, 0)),
+	        in_over (expandx888 (vs3, 1), srca, vmask, expand8888 (vd3, 1)));
+
+	    vd4 = pack8888 (
+	        in_over (expandx888 (vs4, 0), srca, vmask, expand8888 (vd4, 0)),
+	        in_over (expandx888 (vs4, 1), srca, vmask, expand8888 (vd4, 1)));
+
+	    vd5 = pack8888 (
+	        in_over (expandx888 (vs5, 0), srca, vmask, expand8888 (vd5, 0)),
+	        in_over (expandx888 (vs5, 1), srca, vmask, expand8888 (vd5, 1)));
+
+	    vd6 = pack8888 (
+	        in_over (expandx888 (vs6, 0), srca, vmask, expand8888 (vd6, 0)),
+	        in_over (expandx888 (vs6, 1), srca, vmask, expand8888 (vd6, 1)));
+
+	    vd7 = pack8888 (
+	        in_over (expandx888 (vs7, 0), srca, vmask, expand8888 (vd7, 0)),
+	        in_over (expandx888 (vs7, 1), srca, vmask, expand8888 (vd7, 1)));
+
+	    *(__m64 *)(dst + 0) = vd0;
+	    *(__m64 *)(dst + 2) = vd1;
+	    *(__m64 *)(dst + 4) = vd2;
+	    *(__m64 *)(dst + 6) = vd3;
+	    *(__m64 *)(dst + 8) = vd4;
+	    *(__m64 *)(dst + 10) = vd5;
+	    *(__m64 *)(dst + 12) = vd6;
+	    *(__m64 *)(dst + 14) = vd7;
+
+	    w -= 16;
+	    dst += 16;
+	    src += 16;
+	}
+
+	while (w)
+	{
+	    __m64 s = load8888 (*src | 0xff000000);
+	    __m64 d = load8888 (*dst);
+
+	    *dst = store8888 (in_over (s, srca, vmask, d));
+
+	    w--;
+	    dst++;
+	    src++;
+	}
+    }
+
+    _mm_empty ();
+}
+
+static void
+mmx_composite_over_8888_8888 (pixman_implementation_t *imp,
+                              pixman_op_t              op,
+                              pixman_image_t *         src_image,
+                              pixman_image_t *         mask_image,
+                              pixman_image_t *         dst_image,
+                              int32_t                  src_x,
+                              int32_t                  src_y,
+                              int32_t                  mask_x,
+                              int32_t                  mask_y,
+                              int32_t                  dest_x,
+                              int32_t                  dest_y,
+                              int32_t                  width,
+                              int32_t                  height)
+{
+    uint32_t *dst_line, *dst;
+    uint32_t *src_line, *src;
+    uint32_t s;
+    int dst_stride, src_stride;
+    uint8_t a;
+    int32_t w;
+
+    CHECKPOINT ();
+
+    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
+    PIXMAN_IMAGE_GET_LINE (src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
+
+    while (height--)
+    {
+	dst = dst_line;
+	dst_line += dst_stride;
+	src = src_line;
+	src_line += src_stride;
+	w = width;
+
+	while (w--)
+	{
+	    s = *src++;
+	    a = s >> 24;
+
+	    if (a == 0xff)
+	    {
+		*dst = s;
+	    }
+	    else if (s)
+	    {
+		__m64 ms, sa;
+		ms = load8888 (s);
+		sa = expand_alpha (ms);
+		*dst = store8888 (over (ms, sa, load8888 (*dst)));
+	    }
+
+	    dst++;
+	}
+    }
+    _mm_empty ();
+}
+
+static void
+mmx_composite_over_8888_0565 (pixman_implementation_t *imp,
+                              pixman_op_t              op,
+                              pixman_image_t *         src_image,
+                              pixman_image_t *         mask_image,
+                              pixman_image_t *         dst_image,
+                              int32_t                  src_x,
+                              int32_t                  src_y,
+                              int32_t                  mask_x,
+                              int32_t                  mask_y,
+                              int32_t                  dest_x,
+                              int32_t                  dest_y,
+                              int32_t                  width,
+                              int32_t                  height)
+{
+    uint16_t    *dst_line, *dst;
+    uint32_t    *src_line, *src;
+    int dst_stride, src_stride;
+    int32_t w;
+
+    CHECKPOINT ();
+
+    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, uint16_t, dst_stride, dst_line, 1);
+    PIXMAN_IMAGE_GET_LINE (src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
+
+#if 0
+    /* FIXME */
+    assert (src_image->drawable == mask_image->drawable);
+#endif
+
+    while (height--)
+    {
+	dst = dst_line;
+	dst_line += dst_stride;
+	src = src_line;
+	src_line += src_stride;
+	w = width;
+
+	CHECKPOINT ();
+
+	while (w && (unsigned long)dst & 7)
+	{
+	    __m64 vsrc = load8888 (*src);
+	    uint64_t d = *dst;
+	    __m64 vdest = expand565 (M64 (d), 0);
+
+	    vdest = pack_565 (
+		over (vsrc, expand_alpha (vsrc), vdest), vdest, 0);
+
+	    *dst = UINT64 (vdest);
+
+	    w--;
+	    dst++;
+	    src++;
+	}
+
+	CHECKPOINT ();
+
+	while (w >= 4)
+	{
+	    __m64 vsrc0, vsrc1, vsrc2, vsrc3;
+	    __m64 vdest;
+
+	    vsrc0 = load8888 (*(src + 0));
+	    vsrc1 = load8888 (*(src + 1));
+	    vsrc2 = load8888 (*(src + 2));
+	    vsrc3 = load8888 (*(src + 3));
+
+	    vdest = *(__m64 *)dst;
+
+	    vdest = pack_565 (over (vsrc0, expand_alpha (vsrc0), expand565 (vdest, 0)), vdest, 0);
+	    vdest = pack_565 (over (vsrc1, expand_alpha (vsrc1), expand565 (vdest, 1)), vdest, 1);
+	    vdest = pack_565 (over (vsrc2, expand_alpha (vsrc2), expand565 (vdest, 2)), vdest, 2);
+	    vdest = pack_565 (over (vsrc3, expand_alpha (vsrc3), expand565 (vdest, 3)), vdest, 3);
+
+	    *(__m64 *)dst = vdest;
+
+	    w -= 4;
+	    dst += 4;
+	    src += 4;
+	}
+
+	CHECKPOINT ();
+
+	while (w)
+	{
+	    __m64 vsrc = load8888 (*src);
+	    uint64_t d = *dst;
+	    __m64 vdest = expand565 (M64 (d), 0);
+
+	    vdest = pack_565 (over (vsrc, expand_alpha (vsrc), vdest), vdest, 0);
+
+	    *dst = UINT64 (vdest);
+
+	    w--;
+	    dst++;
+	    src++;
+	}
+    }
+
+    _mm_empty ();
+}
+
+static void
+mmx_composite_over_n_8_8888 (pixman_implementation_t *imp,
+                             pixman_op_t              op,
+                             pixman_image_t *         src_image,
+                             pixman_image_t *         mask_image,
+                             pixman_image_t *         dst_image,
+                             int32_t                  src_x,
+                             int32_t                  src_y,
+                             int32_t                  mask_x,
+                             int32_t                  mask_y,
+                             int32_t                  dest_x,
+                             int32_t                  dest_y,
+                             int32_t                  width,
+                             int32_t                  height)
+{
+    uint32_t src, srca;
+    uint32_t *dst_line, *dst;
+    uint8_t *mask_line, *mask;
+    int dst_stride, mask_stride;
+    int32_t w;
+    __m64 vsrc, vsrca;
+    uint64_t srcsrc;
+
+    CHECKPOINT ();
+
+    src = _pixman_image_get_solid (src_image, dst_image->bits.format);
+
+    srca = src >> 24;
+    if (src == 0)
+	return;
+
+    srcsrc = (uint64_t)src << 32 | src;
+
+    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
+    PIXMAN_IMAGE_GET_LINE (mask_image, mask_x, mask_y, uint8_t, mask_stride, mask_line, 1);
+
+    vsrc = load8888 (src);
+    vsrca = expand_alpha (vsrc);
+
+    while (height--)
+    {
+	dst = dst_line;
+	dst_line += dst_stride;
+	mask = mask_line;
+	mask_line += mask_stride;
+	w = width;
+
+	CHECKPOINT ();
+
+	while (w && (unsigned long)dst & 7)
+	{
+	    uint64_t m = *mask;
+
+	    if (m)
+	    {
+		__m64 vdest = in_over (vsrc, vsrca,
+				       expand_alpha_rev (M64 (m)),
+				       load8888 (*dst));
+
+		*dst = store8888 (vdest);
+	    }
+
+	    w--;
+	    mask++;
+	    dst++;
+	}
+
+	CHECKPOINT ();
+
+	while (w >= 2)
+	{
+	    uint64_t m0, m1;
+
+	    m0 = *mask;
+	    m1 = *(mask + 1);
+
+	    if (srca == 0xff && (m0 & m1) == 0xff)
+	    {
+		*(uint64_t *)dst = srcsrc;
+	    }
+	    else if (m0 | m1)
+	    {
+		__m64 vdest;
+		__m64 dest0, dest1;
+
+		vdest = *(__m64 *)dst;
+
+		dest0 = in_over (vsrc, vsrca, expand_alpha_rev (M64 (m0)),
+				 expand8888 (vdest, 0));
+		dest1 = in_over (vsrc, vsrca, expand_alpha_rev (M64 (m1)),
+				 expand8888 (vdest, 1));
+
+		*(__m64 *)dst = pack8888 (dest0, dest1);
+	    }
+
+	    mask += 2;
+	    dst += 2;
+	    w -= 2;
+	}
+
+	CHECKPOINT ();
+
+	while (w)
+	{
+	    uint64_t m = *mask;
+
+	    if (m)
+	    {
+		__m64 vdest = load8888 (*dst);
+
+		vdest = in_over (
+		    vsrc, vsrca, expand_alpha_rev (M64 (m)), vdest);
+		*dst = store8888 (vdest);
+	    }
+
+	    w--;
+	    mask++;
+	    dst++;
+	}
+    }
+
+    _mm_empty ();
+}
+
+pixman_bool_t
+pixman_fill_mmx (uint32_t *bits,
+                 int       stride,
+                 int       bpp,
+                 int       x,
+                 int       y,
+                 int       width,
+                 int       height,
+                 uint32_t xor)
+{
+    uint64_t fill;
+    __m64 vfill;
+    uint32_t byte_width;
+    uint8_t     *byte_line;
+
+#ifdef __GNUC__
+    __m64 v1, v2, v3, v4, v5, v6, v7;
+#endif
+
+    if (bpp != 16 && bpp != 32 && bpp != 8)
+	return FALSE;
+
+    if (bpp == 8)
+    {
+	stride = stride * (int) sizeof (uint32_t) / 1;
+	byte_line = (uint8_t *)(((uint8_t *)bits) + stride * y + x);
+	byte_width = width;
+	stride *= 1;
+        xor = (xor & 0xff) * 0x01010101;
+    }
+    else if (bpp == 16)
+    {
+	stride = stride * (int) sizeof (uint32_t) / 2;
+	byte_line = (uint8_t *)(((uint16_t *)bits) + stride * y + x);
+	byte_width = 2 * width;
+	stride *= 2;
+        xor = (xor & 0xffff) * 0x00010001;
+    }
+    else
+    {
+	stride = stride * (int) sizeof (uint32_t) / 4;
+	byte_line = (uint8_t *)(((uint32_t *)bits) + stride * y + x);
+	byte_width = 4 * width;
+	stride *= 4;
+    }
+
+    fill = ((uint64_t)xor << 32) | xor;
+    vfill = M64 (fill);
+
+#ifdef __GNUC__
+    __asm__ (
+        "movq		%7,	%0\n"
+        "movq		%7,	%1\n"
+        "movq		%7,	%2\n"
+        "movq		%7,	%3\n"
+        "movq		%7,	%4\n"
+        "movq		%7,	%5\n"
+        "movq		%7,	%6\n"
+	: "=y" (v1), "=y" (v2), "=y" (v3),
+	  "=y" (v4), "=y" (v5), "=y" (v6), "=y" (v7)
+	: "y" (vfill));
+#endif
+
+    while (height--)
+    {
+	int w;
+	uint8_t *d = byte_line;
+
+	byte_line += stride;
+	w = byte_width;
+
+	while (w >= 1 && ((unsigned long)d & 1))
+	{
+	    *(uint8_t *)d = (xor & 0xff);
+	    w--;
+	    d++;
+	}
+
+	while (w >= 2 && ((unsigned long)d & 3))
+	{
+	    *(uint16_t *)d = xor;
+	    w -= 2;
+	    d += 2;
+	}
+
+	while (w >= 4 && ((unsigned long)d & 7))
+	{
+	    *(uint32_t *)d = xor;
+
+	    w -= 4;
+	    d += 4;
+	}
+
+	while (w >= 64)
+	{
+#ifdef __GNUC__
+	    __asm__ (
+	        "movq	%1,	  (%0)\n"
+	        "movq	%2,	 8(%0)\n"
+	        "movq	%3,	16(%0)\n"
+	        "movq	%4,	24(%0)\n"
+	        "movq	%5,	32(%0)\n"
+	        "movq	%6,	40(%0)\n"
+	        "movq	%7,	48(%0)\n"
+	        "movq	%8,	56(%0)\n"
+		:
+		: "r" (d),
+		  "y" (vfill), "y" (v1), "y" (v2), "y" (v3),
+		  "y" (v4), "y" (v5), "y" (v6), "y" (v7)
+		: "memory");
+#else
+	    *(__m64*) (d +  0) = vfill;
+	    *(__m64*) (d +  8) = vfill;
+	    *(__m64*) (d + 16) = vfill;
+	    *(__m64*) (d + 24) = vfill;
+	    *(__m64*) (d + 32) = vfill;
+	    *(__m64*) (d + 40) = vfill;
+	    *(__m64*) (d + 48) = vfill;
+	    *(__m64*) (d + 56) = vfill;
+#endif
+	    w -= 64;
+	    d += 64;
+	}
+
+	while (w >= 4)
+	{
+	    *(uint32_t *)d = xor;
+
+	    w -= 4;
+	    d += 4;
+	}
+	while (w >= 2)
+	{
+	    *(uint16_t *)d = xor;
+	    w -= 2;
+	    d += 2;
+	}
+	while (w >= 1)
+	{
+	    *(uint8_t *)d = (xor & 0xff);
+	    w--;
+	    d++;
+	}
+
+    }
+
+    _mm_empty ();
+    return TRUE;
+}
+
+static void
+mmx_composite_src_n_8_8888 (pixman_implementation_t *imp,
+                            pixman_op_t              op,
+                            pixman_image_t *         src_image,
+                            pixman_image_t *         mask_image,
+                            pixman_image_t *         dst_image,
+                            int32_t                  src_x,
+                            int32_t                  src_y,
+                            int32_t                  mask_x,
+                            int32_t                  mask_y,
+                            int32_t                  dest_x,
+                            int32_t                  dest_y,
+                            int32_t                  width,
+                            int32_t                  height)
+{
+    uint32_t src, srca;
+    uint32_t    *dst_line, *dst;
+    uint8_t     *mask_line, *mask;
+    int dst_stride, mask_stride;
+    int32_t w;
+    __m64 vsrc, vsrca;
+    uint64_t srcsrc;
+
+    CHECKPOINT ();
+
+    src = _pixman_image_get_solid (src_image, dst_image->bits.format);
+
+    srca = src >> 24;
+    if (src == 0)
+    {
+	pixman_fill_mmx (dst_image->bits.bits, dst_image->bits.rowstride,
+			 PIXMAN_FORMAT_BPP (dst_image->bits.format),
+	                 dest_x, dest_y, width, height, 0);
+	return;
+    }
+
+    srcsrc = (uint64_t)src << 32 | src;
+
+    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
+    PIXMAN_IMAGE_GET_LINE (mask_image, mask_x, mask_y, uint8_t, mask_stride, mask_line, 1);
+
+    vsrc = load8888 (src);
+    vsrca = expand_alpha (vsrc);
+
+    while (height--)
+    {
+	dst = dst_line;
+	dst_line += dst_stride;
+	mask = mask_line;
+	mask_line += mask_stride;
+	w = width;
+
+	CHECKPOINT ();
+
+	while (w && (unsigned long)dst & 7)
+	{
+	    uint64_t m = *mask;
+
+	    if (m)
+	    {
+		__m64 vdest = in (vsrc, expand_alpha_rev (M64 (m)));
+
+		*dst = store8888 (vdest);
+	    }
+	    else
+	    {
+		*dst = 0;
+	    }
+
+	    w--;
+	    mask++;
+	    dst++;
+	}
+
+	CHECKPOINT ();
+
+	while (w >= 2)
+	{
+	    uint64_t m0, m1;
+	    m0 = *mask;
+	    m1 = *(mask + 1);
+
+	    if (srca == 0xff && (m0 & m1) == 0xff)
+	    {
+		*(uint64_t *)dst = srcsrc;
+	    }
+	    else if (m0 | m1)
+	    {
+		__m64 vdest;
+		__m64 dest0, dest1;
+
+		vdest = *(__m64 *)dst;
+
+		dest0 = in (vsrc, expand_alpha_rev (M64 (m0)));
+		dest1 = in (vsrc, expand_alpha_rev (M64 (m1)));
+
+		*(__m64 *)dst = pack8888 (dest0, dest1);
+	    }
+	    else
+	    {
+		*(uint64_t *)dst = 0;
+	    }
+
+	    mask += 2;
+	    dst += 2;
+	    w -= 2;
+	}
+
+	CHECKPOINT ();
+
+	while (w)
+	{
+	    uint64_t m = *mask;
+
+	    if (m)
+	    {
+		__m64 vdest = load8888 (*dst);
+
+		vdest = in (vsrc, expand_alpha_rev (M64 (m)));
+		*dst = store8888 (vdest);
+	    }
+	    else
+	    {
+		*dst = 0;
+	    }
+
+	    w--;
+	    mask++;
+	    dst++;
+	}
+    }
+
+    _mm_empty ();
+}
+
+static void
+mmx_composite_over_n_8_0565 (pixman_implementation_t *imp,
+                             pixman_op_t              op,
+                             pixman_image_t *         src_image,
+                             pixman_image_t *         mask_image,
+                             pixman_image_t *         dst_image,
+                             int32_t                  src_x,
+                             int32_t                  src_y,
+                             int32_t                  mask_x,
+                             int32_t                  mask_y,
+                             int32_t                  dest_x,
+                             int32_t                  dest_y,
+                             int32_t                  width,
+                             int32_t                  height)
+{
+    uint32_t src, srca;
+    uint16_t *dst_line, *dst;
+    uint8_t *mask_line, *mask;
+    int dst_stride, mask_stride;
+    int32_t w;
+    __m64 vsrc, vsrca, tmp;
+    uint64_t srcsrcsrcsrc, src16;
+
+    CHECKPOINT ();
+
+    src = _pixman_image_get_solid (src_image, dst_image->bits.format);
+
+    srca = src >> 24;
+    if (src == 0)
+	return;
+
+    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, uint16_t, dst_stride, dst_line, 1);
+    PIXMAN_IMAGE_GET_LINE (mask_image, mask_x, mask_y, uint8_t, mask_stride, mask_line, 1);
+
+    vsrc = load8888 (src);
+    vsrca = expand_alpha (vsrc);
+
+    tmp = pack_565 (vsrc, _mm_setzero_si64 (), 0);
+    src16 = UINT64 (tmp);
+
+    srcsrcsrcsrc =
+	(uint64_t)src16 << 48 | (uint64_t)src16 << 32 |
+	(uint64_t)src16 << 16 | (uint64_t)src16;
+
+    while (height--)
+    {
+	dst = dst_line;
+	dst_line += dst_stride;
+	mask = mask_line;
+	mask_line += mask_stride;
+	w = width;
+
+	CHECKPOINT ();
+
+	while (w && (unsigned long)dst & 7)
+	{
+	    uint64_t m = *mask;
+
+	    if (m)
+	    {
+		uint64_t d = *dst;
+		__m64 vd = M64 (d);
+		__m64 vdest = in_over (
+		    vsrc, vsrca, expand_alpha_rev (M64 (m)), expand565 (vd, 0));
+
+		vd = pack_565 (vdest, _mm_setzero_si64 (), 0);
+		*dst = UINT64 (vd);
+	    }
+
+	    w--;
+	    mask++;
+	    dst++;
+	}
+
+	CHECKPOINT ();
+
+	while (w >= 4)
+	{
+	    uint64_t m0, m1, m2, m3;
+	    m0 = *mask;
+	    m1 = *(mask + 1);
+	    m2 = *(mask + 2);
+	    m3 = *(mask + 3);
+
+	    if (srca == 0xff && (m0 & m1 & m2 & m3) == 0xff)
+	    {
+		*(uint64_t *)dst = srcsrcsrcsrc;
+	    }
+	    else if (m0 | m1 | m2 | m3)
+	    {
+		__m64 vdest;
+		__m64 vm0, vm1, vm2, vm3;
+
+		vdest = *(__m64 *)dst;
+
+		vm0 = M64 (m0);
+		vdest = pack_565 (in_over (vsrc, vsrca, expand_alpha_rev (vm0),
+					   expand565 (vdest, 0)), vdest, 0);
+		vm1 = M64 (m1);
+		vdest = pack_565 (in_over (vsrc, vsrca, expand_alpha_rev (vm1),
+					   expand565 (vdest, 1)), vdest, 1);
+		vm2 = M64 (m2);
+		vdest = pack_565 (in_over (vsrc, vsrca, expand_alpha_rev (vm2),
+					   expand565 (vdest, 2)), vdest, 2);
+		vm3 = M64 (m3);
+		vdest = pack_565 (in_over (vsrc, vsrca, expand_alpha_rev (vm3),
+					   expand565 (vdest, 3)), vdest, 3);
+
+		*(__m64 *)dst = vdest;
+	    }
+
+	    w -= 4;
+	    mask += 4;
+	    dst += 4;
+	}
+
+	CHECKPOINT ();
+
+	while (w)
+	{
+	    uint64_t m = *mask;
+
+	    if (m)
+	    {
+		uint64_t d = *dst;
+		__m64 vd = M64 (d);
+		__m64 vdest = in_over (vsrc, vsrca, expand_alpha_rev (M64 (m)),
+				       expand565 (vd, 0));
+		vd = pack_565 (vdest, _mm_setzero_si64 (), 0);
+		*dst = UINT64 (vd);
+	    }
+
+	    w--;
+	    mask++;
+	    dst++;
+	}
+    }
+
+    _mm_empty ();
+}
+
+static void
+mmx_composite_over_pixbuf_0565 (pixman_implementation_t *imp,
+                                pixman_op_t              op,
+                                pixman_image_t *         src_image,
+                                pixman_image_t *         mask_image,
+                                pixman_image_t *         dst_image,
+                                int32_t                  src_x,
+                                int32_t                  src_y,
+                                int32_t                  mask_x,
+                                int32_t                  mask_y,
+                                int32_t                  dest_x,
+                                int32_t                  dest_y,
+                                int32_t                  width,
+                                int32_t                  height)
+{
+    uint16_t    *dst_line, *dst;
+    uint32_t    *src_line, *src;
+    int dst_stride, src_stride;
+    int32_t w;
+
+    CHECKPOINT ();
+
+    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, uint16_t, dst_stride, dst_line, 1);
+    PIXMAN_IMAGE_GET_LINE (src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
+
+#if 0
+    /* FIXME */
+    assert (src_image->drawable == mask_image->drawable);
+#endif
+
+    while (height--)
+    {
+	dst = dst_line;
+	dst_line += dst_stride;
+	src = src_line;
+	src_line += src_stride;
+	w = width;
+
+	CHECKPOINT ();
+
+	while (w && (unsigned long)dst & 7)
+	{
+	    __m64 vsrc = load8888 (*src);
+	    uint64_t d = *dst;
+	    __m64 vdest = expand565 (M64 (d), 0);
+
+	    vdest = pack_565 (over_rev_non_pre (vsrc, vdest), vdest, 0);
+
+	    *dst = UINT64 (vdest);
+
+	    w--;
+	    dst++;
+	    src++;
+	}
+
+	CHECKPOINT ();
+
+	while (w >= 4)
+	{
+	    uint32_t s0, s1, s2, s3;
+	    unsigned char a0, a1, a2, a3;
+
+	    s0 = *src;
+	    s1 = *(src + 1);
+	    s2 = *(src + 2);
+	    s3 = *(src + 3);
+
+	    a0 = (s0 >> 24);
+	    a1 = (s1 >> 24);
+	    a2 = (s2 >> 24);
+	    a3 = (s3 >> 24);
+
+	    if ((a0 & a1 & a2 & a3) == 0xFF)
+	    {
+		__m64 vdest;
+		vdest = pack_565 (invert_colors (load8888 (s0)), _mm_setzero_si64 (), 0);
+		vdest = pack_565 (invert_colors (load8888 (s1)), vdest, 1);
+		vdest = pack_565 (invert_colors (load8888 (s2)), vdest, 2);
+		vdest = pack_565 (invert_colors (load8888 (s3)), vdest, 3);
+
+		*(__m64 *)dst = vdest;
+	    }
+	    else if (s0 | s1 | s2 | s3)
+	    {
+		__m64 vdest = *(__m64 *)dst;
+
+		vdest = pack_565 (over_rev_non_pre (load8888 (s0), expand565 (vdest, 0)), vdest, 0);
+		vdest = pack_565 (over_rev_non_pre (load8888 (s1), expand565 (vdest, 1)), vdest, 1);
+		vdest = pack_565 (over_rev_non_pre (load8888 (s2), expand565 (vdest, 2)), vdest, 2);
+		vdest = pack_565 (over_rev_non_pre (load8888 (s3), expand565 (vdest, 3)), vdest, 3);
+
+		*(__m64 *)dst = vdest;
+	    }
+
+	    w -= 4;
+	    dst += 4;
+	    src += 4;
+	}
+
+	CHECKPOINT ();
+
+	while (w)
+	{
+	    __m64 vsrc = load8888 (*src);
+	    uint64_t d = *dst;
+	    __m64 vdest = expand565 (M64 (d), 0);
+
+	    vdest = pack_565 (over_rev_non_pre (vsrc, vdest), vdest, 0);
+
+	    *dst = UINT64 (vdest);
+
+	    w--;
+	    dst++;
+	    src++;
+	}
+    }
+
+    _mm_empty ();
+}
+
+static void
+mmx_composite_over_pixbuf_8888 (pixman_implementation_t *imp,
+                                pixman_op_t              op,
+                                pixman_image_t *         src_image,
+                                pixman_image_t *         mask_image,
+                                pixman_image_t *         dst_image,
+                                int32_t                  src_x,
+                                int32_t                  src_y,
+                                int32_t                  mask_x,
+                                int32_t                  mask_y,
+                                int32_t                  dest_x,
+                                int32_t                  dest_y,
+                                int32_t                  width,
+                                int32_t                  height)
+{
+    uint32_t    *dst_line, *dst;
+    uint32_t    *src_line, *src;
+    int dst_stride, src_stride;
+    int32_t w;
+
+    CHECKPOINT ();
+
+    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
+    PIXMAN_IMAGE_GET_LINE (src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
+
+#if 0
+    /* FIXME */
+    assert (src_image->drawable == mask_image->drawable);
+#endif
+
+    while (height--)
+    {
+	dst = dst_line;
+	dst_line += dst_stride;
+	src = src_line;
+	src_line += src_stride;
+	w = width;
+
+	while (w && (unsigned long)dst & 7)
+	{
+	    __m64 s = load8888 (*src);
+	    __m64 d = load8888 (*dst);
+
+	    *dst = store8888 (over_rev_non_pre (s, d));
+
+	    w--;
+	    dst++;
+	    src++;
+	}
+
+	while (w >= 2)
+	{
+	    uint64_t s0, s1;
+	    unsigned char a0, a1;
+	    __m64 d0, d1;
+
+	    s0 = *src;
+	    s1 = *(src + 1);
+
+	    a0 = (s0 >> 24);
+	    a1 = (s1 >> 24);
+
+	    if ((a0 & a1) == 0xFF)
+	    {
+		d0 = invert_colors (load8888 (s0));
+		d1 = invert_colors (load8888 (s1));
+
+		*(__m64 *)dst = pack8888 (d0, d1);
+	    }
+	    else if (s0 | s1)
+	    {
+		__m64 vdest = *(__m64 *)dst;
+
+		d0 = over_rev_non_pre (load8888 (s0), expand8888 (vdest, 0));
+		d1 = over_rev_non_pre (load8888 (s1), expand8888 (vdest, 1));
+
+		*(__m64 *)dst = pack8888 (d0, d1);
+	    }
+
+	    w -= 2;
+	    dst += 2;
+	    src += 2;
+	}
+
+	while (w)
+	{
+	    __m64 s = load8888 (*src);
+	    __m64 d = load8888 (*dst);
+
+	    *dst = store8888 (over_rev_non_pre (s, d));
+
+	    w--;
+	    dst++;
+	    src++;
+	}
+    }
+
+    _mm_empty ();
+}
+
+static void
+mmx_composite_over_n_8888_0565_ca (pixman_implementation_t *imp,
+                                   pixman_op_t              op,
+                                   pixman_image_t *         src_image,
+                                   pixman_image_t *         mask_image,
+                                   pixman_image_t *         dst_image,
+                                   int32_t                  src_x,
+                                   int32_t                  src_y,
+                                   int32_t                  mask_x,
+                                   int32_t                  mask_y,
+                                   int32_t                  dest_x,
+                                   int32_t                  dest_y,
+                                   int32_t                  width,
+                                   int32_t                  height)
+{
+    uint32_t src, srca;
+    uint16_t    *dst_line;
+    uint32_t    *mask_line;
+    int dst_stride, mask_stride;
+    __m64 vsrc, vsrca;
+
+    CHECKPOINT ();
+
+    src = _pixman_image_get_solid (src_image, dst_image->bits.format);
+
+    srca = src >> 24;
+    if (src == 0)
+	return;
+
+    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, uint16_t, dst_stride, dst_line, 1);
+    PIXMAN_IMAGE_GET_LINE (mask_image, mask_x, mask_y, uint32_t, mask_stride, mask_line, 1);
+
+    vsrc = load8888 (src);
+    vsrca = expand_alpha (vsrc);
+
+    while (height--)
+    {
+	int twidth = width;
+	uint32_t *p = (uint32_t *)mask_line;
+	uint16_t *q = (uint16_t *)dst_line;
+
+	while (twidth && ((unsigned long)q & 7))
+	{
+	    uint32_t m = *(uint32_t *)p;
+
+	    if (m)
+	    {
+		uint64_t d = *q;
+		__m64 vdest = expand565 (M64 (d), 0);
+		vdest = pack_565 (in_over (vsrc, vsrca, load8888 (m), vdest), vdest, 0);
+		*q = UINT64 (vdest);
+	    }
+
+	    twidth--;
+	    p++;
+	    q++;
+	}
+
+	while (twidth >= 4)
+	{
+	    uint32_t m0, m1, m2, m3;
+
+	    m0 = *p;
+	    m1 = *(p + 1);
+	    m2 = *(p + 2);
+	    m3 = *(p + 3);
+
+	    if ((m0 | m1 | m2 | m3))
+	    {
+		__m64 vdest = *(__m64 *)q;
+
+		vdest = pack_565 (in_over (vsrc, vsrca, load8888 (m0), expand565 (vdest, 0)), vdest, 0);
+		vdest = pack_565 (in_over (vsrc, vsrca, load8888 (m1), expand565 (vdest, 1)), vdest, 1);
+		vdest = pack_565 (in_over (vsrc, vsrca, load8888 (m2), expand565 (vdest, 2)), vdest, 2);
+		vdest = pack_565 (in_over (vsrc, vsrca, load8888 (m3), expand565 (vdest, 3)), vdest, 3);
+
+		*(__m64 *)q = vdest;
+	    }
+	    twidth -= 4;
+	    p += 4;
+	    q += 4;
+	}
+
+	while (twidth)
+	{
+	    uint32_t m;
+
+	    m = *(uint32_t *)p;
+	    if (m)
+	    {
+		uint64_t d = *q;
+		__m64 vdest = expand565 (M64 (d), 0);
+		vdest = pack_565 (in_over (vsrc, vsrca, load8888 (m), vdest), vdest, 0);
+		*q = UINT64 (vdest);
+	    }
+
+	    twidth--;
+	    p++;
+	    q++;
+	}
+
+	mask_line += mask_stride;
+	dst_line += dst_stride;
+    }
+
+    _mm_empty ();
+}
+
+static void
+mmx_composite_in_n_8_8 (pixman_implementation_t *imp,
+                        pixman_op_t              op,
+                        pixman_image_t *         src_image,
+                        pixman_image_t *         mask_image,
+                        pixman_image_t *         dst_image,
+                        int32_t                  src_x,
+                        int32_t                  src_y,
+                        int32_t                  mask_x,
+                        int32_t                  mask_y,
+                        int32_t                  dest_x,
+                        int32_t                  dest_y,
+                        int32_t                  width,
+                        int32_t                  height)
+{
+    uint8_t *dst_line, *dst;
+    uint8_t *mask_line, *mask;
+    int dst_stride, mask_stride;
+    int32_t w;
+    uint32_t src;
+    uint8_t sa;
+    __m64 vsrc, vsrca;
+
+    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, uint8_t, dst_stride, dst_line, 1);
+    PIXMAN_IMAGE_GET_LINE (mask_image, mask_x, mask_y, uint8_t, mask_stride, mask_line, 1);
+
+    src = _pixman_image_get_solid (src_image, dst_image->bits.format);
+
+    sa = src >> 24;
+
+    vsrc = load8888 (src);
+    vsrca = expand_alpha (vsrc);
+
+    while (height--)
+    {
+	dst = dst_line;
+	dst_line += dst_stride;
+	mask = mask_line;
+	mask_line += mask_stride;
+	w = width;
+
+	if ((((unsigned long)dst_image & 3) == 0) &&
+	    (((unsigned long)src_image & 3) == 0))
+	{
+	    while (w >= 4)
+	    {
+		uint32_t m;
+		__m64 vmask;
+		__m64 vdest;
+
+		m = 0;
+
+		vmask = load8888 (*(uint32_t *)mask);
+		vdest = load8888 (*(uint32_t *)dst);
+
+		*(uint32_t *)dst = store8888 (in (in (vsrca, vmask), vdest));
+
+		dst += 4;
+		mask += 4;
+		w -= 4;
+	    }
+	}
+
+	while (w--)
+	{
+	    uint16_t tmp;
+	    uint8_t a;
+	    uint32_t m, d;
+
+	    a = *mask++;
+	    d = *dst;
+
+	    m = MUL_UN8 (sa, a, tmp);
+	    d = MUL_UN8 (m, d, tmp);
+
+	    *dst++ = d;
+	}
+    }
+
+    _mm_empty ();
+}
+
+static void
+mmx_composite_in_8_8 (pixman_implementation_t *imp,
+                      pixman_op_t              op,
+                      pixman_image_t *         src_image,
+                      pixman_image_t *         mask_image,
+                      pixman_image_t *         dst_image,
+                      int32_t                  src_x,
+                      int32_t                  src_y,
+                      int32_t                  mask_x,
+                      int32_t                  mask_y,
+                      int32_t                  dest_x,
+                      int32_t                  dest_y,
+                      int32_t                  width,
+                      int32_t                  height)
+{
+    uint8_t     *dst_line, *dst;
+    uint8_t     *src_line, *src;
+    int src_stride, dst_stride;
+    int32_t w;
+
+    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, uint8_t, dst_stride, dst_line, 1);
+    PIXMAN_IMAGE_GET_LINE (src_image, src_x, src_y, uint8_t, src_stride, src_line, 1);
+
+    while (height--)
+    {
+	dst = dst_line;
+	dst_line += dst_stride;
+	src = src_line;
+	src_line += src_stride;
+	w = width;
+
+	if ((((unsigned long)dst_image & 3) == 0) &&
+	    (((unsigned long)src_image & 3) == 0))
+	{
+	    while (w >= 4)
+	    {
+		uint32_t *s = (uint32_t *)src;
+		uint32_t *d = (uint32_t *)dst;
+
+		*d = store8888 (in (load8888 (*s), load8888 (*d)));
+
+		w -= 4;
+		dst += 4;
+		src += 4;
+	    }
+	}
+
+	while (w--)
+	{
+	    uint8_t s, d;
+	    uint16_t tmp;
+
+	    s = *src;
+	    d = *dst;
+
+	    *dst = MUL_UN8 (s, d, tmp);
+
+	    src++;
+	    dst++;
+	}
+    }
+
+    _mm_empty ();
+}
+
+static void
+mmx_composite_add_n_8_8 (pixman_implementation_t *imp,
+			 pixman_op_t              op,
+			 pixman_image_t *         src_image,
+			 pixman_image_t *         mask_image,
+			 pixman_image_t *         dst_image,
+			 int32_t                  src_x,
+			 int32_t                  src_y,
+			 int32_t                  mask_x,
+			 int32_t                  mask_y,
+			 int32_t                  dest_x,
+			 int32_t                  dest_y,
+			 int32_t                  width,
+			 int32_t                  height)
+{
+    uint8_t     *dst_line, *dst;
+    uint8_t     *mask_line, *mask;
+    int dst_stride, mask_stride;
+    int32_t w;
+    uint32_t src;
+    uint8_t sa;
+    __m64 vsrc, vsrca;
+
+    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, uint8_t, dst_stride, dst_line, 1);
+    PIXMAN_IMAGE_GET_LINE (mask_image, mask_x, mask_y, uint8_t, mask_stride, mask_line, 1);
+
+    src = _pixman_image_get_solid (src_image, dst_image->bits.format);
+
+    sa = src >> 24;
+
+    if (src == 0)
+	return;
+
+    vsrc = load8888 (src);
+    vsrca = expand_alpha (vsrc);
+
+    while (height--)
+    {
+	dst = dst_line;
+	dst_line += dst_stride;
+	mask = mask_line;
+	mask_line += mask_stride;
+	w = width;
+
+	if ((((unsigned long)mask_image & 3) == 0) &&
+	    (((unsigned long)dst_image  & 3) == 0))
+	{
+	    while (w >= 4)
+	    {
+		__m64 vmask = load8888 (*(uint32_t *)mask);
+		__m64 vdest = load8888 (*(uint32_t *)dst);
+
+		*(uint32_t *)dst = store8888 (_mm_adds_pu8 (in (vsrca, vmask), vdest));
+
+		w -= 4;
+		dst += 4;
+		mask += 4;
+	    }
+	}
+
+	while (w--)
+	{
+	    uint16_t tmp;
+	    uint16_t a;
+	    uint32_t m, d;
+	    uint32_t r;
+
+	    a = *mask++;
+	    d = *dst;
+
+	    m = MUL_UN8 (sa, a, tmp);
+	    r = ADD_UN8 (m, d, tmp);
+
+	    *dst++ = r;
+	}
+    }
+
+    _mm_empty ();
+}
+
+static void
+mmx_composite_add_8000_8000 (pixman_implementation_t *imp,
+                             pixman_op_t              op,
+                             pixman_image_t *         src_image,
+                             pixman_image_t *         mask_image,
+                             pixman_image_t *         dst_image,
+                             int32_t                  src_x,
+                             int32_t                  src_y,
+                             int32_t                  mask_x,
+                             int32_t                  mask_y,
+                             int32_t                  dest_x,
+                             int32_t                  dest_y,
+                             int32_t                  width,
+                             int32_t                  height)
+{
+    uint8_t *dst_line, *dst;
+    uint8_t *src_line, *src;
+    int dst_stride, src_stride;
+    int32_t w;
+    uint8_t s, d;
+    uint16_t t;
+
+    CHECKPOINT ();
+
+    PIXMAN_IMAGE_GET_LINE (src_image, src_x, src_y, uint8_t, src_stride, src_line, 1);
+    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, uint8_t, dst_stride, dst_line, 1);
+
+    while (height--)
+    {
+	dst = dst_line;
+	dst_line += dst_stride;
+	src = src_line;
+	src_line += src_stride;
+	w = width;
+
+	while (w && (unsigned long)dst & 7)
+	{
+	    s = *src;
+	    d = *dst;
+	    t = d + s;
+	    s = t | (0 - (t >> 8));
+	    *dst = s;
+
+	    dst++;
+	    src++;
+	    w--;
+	}
+
+	while (w >= 8)
+	{
+	    *(__m64*)dst = _mm_adds_pu8 (*(__m64*)src, *(__m64*)dst);
+	    dst += 8;
+	    src += 8;
+	    w -= 8;
+	}
+
+	while (w)
+	{
+	    s = *src;
+	    d = *dst;
+	    t = d + s;
+	    s = t | (0 - (t >> 8));
+	    *dst = s;
+
+	    dst++;
+	    src++;
+	    w--;
+	}
+    }
+
+    _mm_empty ();
+}
+
+static void
+mmx_composite_add_8888_8888 (pixman_implementation_t *imp,
+                             pixman_op_t              op,
+                             pixman_image_t *         src_image,
+                             pixman_image_t *         mask_image,
+                             pixman_image_t *         dst_image,
+                             int32_t                  src_x,
+                             int32_t                  src_y,
+                             int32_t                  mask_x,
+                             int32_t                  mask_y,
+                             int32_t                  dest_x,
+                             int32_t                  dest_y,
+                             int32_t                  width,
+                             int32_t                  height)
+{
+    __m64 dst64;
+    uint32_t    *dst_line, *dst;
+    uint32_t    *src_line, *src;
+    int dst_stride, src_stride;
+    int32_t w;
+
+    CHECKPOINT ();
+
+    PIXMAN_IMAGE_GET_LINE (src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
+    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
+
+    while (height--)
+    {
+	dst = dst_line;
+	dst_line += dst_stride;
+	src = src_line;
+	src_line += src_stride;
+	w = width;
+
+	while (w && (unsigned long)dst & 7)
+	{
+	    *dst = _mm_cvtsi64_si32 (_mm_adds_pu8 (_mm_cvtsi32_si64 (*src),
+	                                           _mm_cvtsi32_si64 (*dst)));
+	    dst++;
+	    src++;
+	    w--;
+	}
+
+	while (w >= 2)
+	{
+	    dst64 = _mm_adds_pu8 (*(__m64*)src, *(__m64*)dst);
+	    *(uint64_t*)dst = UINT64 (dst64);
+	    dst += 2;
+	    src += 2;
+	    w -= 2;
+	}
+
+	if (w)
+	{
+	    *dst = _mm_cvtsi64_si32 (_mm_adds_pu8 (_mm_cvtsi32_si64 (*src),
+	                                           _mm_cvtsi32_si64 (*dst)));
+
+	}
+    }
+
+    _mm_empty ();
+}
+
+static pixman_bool_t
+pixman_blt_mmx (uint32_t *src_bits,
+                uint32_t *dst_bits,
+                int       src_stride,
+                int       dst_stride,
+                int       src_bpp,
+                int       dst_bpp,
+                int       src_x,
+                int       src_y,
+                int       dst_x,
+                int       dst_y,
+                int       width,
+                int       height)
+{
+    uint8_t *   src_bytes;
+    uint8_t *   dst_bytes;
+    int byte_width;
+
+    if (src_bpp != dst_bpp)
+	return FALSE;
+
+    if (src_bpp == 16)
+    {
+	src_stride = src_stride * (int) sizeof (uint32_t) / 2;
+	dst_stride = dst_stride * (int) sizeof (uint32_t) / 2;
+	src_bytes = (uint8_t *)(((uint16_t *)src_bits) + src_stride * (src_y) + (src_x));
+	dst_bytes = (uint8_t *)(((uint16_t *)dst_bits) + dst_stride * (dst_y) + (dst_x));
+	byte_width = 2 * width;
+	src_stride *= 2;
+	dst_stride *= 2;
+    }
+    else if (src_bpp == 32)
+    {
+	src_stride = src_stride * (int) sizeof (uint32_t) / 4;
+	dst_stride = dst_stride * (int) sizeof (uint32_t) / 4;
+	src_bytes = (uint8_t *)(((uint32_t *)src_bits) + src_stride * (src_y) + (src_x));
+	dst_bytes = (uint8_t *)(((uint32_t *)dst_bits) + dst_stride * (dst_y) + (dst_x));
+	byte_width = 4 * width;
+	src_stride *= 4;
+	dst_stride *= 4;
+    }
+    else
+    {
+	return FALSE;
+    }
+
+    while (height--)
+    {
+	int w;
+	uint8_t *s = src_bytes;
+	uint8_t *d = dst_bytes;
+	src_bytes += src_stride;
+	dst_bytes += dst_stride;
+	w = byte_width;
+
+	while (w >= 2 && ((unsigned long)d & 3))
+	{
+	    *(uint16_t *)d = *(uint16_t *)s;
+	    w -= 2;
+	    s += 2;
+	    d += 2;
+	}
+
+	while (w >= 4 && ((unsigned long)d & 7))
+	{
+	    *(uint32_t *)d = *(uint32_t *)s;
+
+	    w -= 4;
+	    s += 4;
+	    d += 4;
+	}
+
+	while (w >= 64)
+	{
+#if defined (__GNUC__) || (defined(__SUNPRO_C) && (__SUNPRO_C >= 0x590))
+	    __asm__ (
+	        "movq	  (%1),	  %%mm0\n"
+	        "movq	 8(%1),	  %%mm1\n"
+	        "movq	16(%1),	  %%mm2\n"
+	        "movq	24(%1),	  %%mm3\n"
+	        "movq	32(%1),	  %%mm4\n"
+	        "movq	40(%1),	  %%mm5\n"
+	        "movq	48(%1),	  %%mm6\n"
+	        "movq	56(%1),	  %%mm7\n"
+
+	        "movq	%%mm0,	  (%0)\n"
+	        "movq	%%mm1,	 8(%0)\n"
+	        "movq	%%mm2,	16(%0)\n"
+	        "movq	%%mm3,	24(%0)\n"
+	        "movq	%%mm4,	32(%0)\n"
+	        "movq	%%mm5,	40(%0)\n"
+	        "movq	%%mm6,	48(%0)\n"
+	        "movq	%%mm7,	56(%0)\n"
+		:
+		: "r" (d), "r" (s)
+		: "memory",
+		  "%mm0", "%mm1", "%mm2", "%mm3",
+		  "%mm4", "%mm5", "%mm6", "%mm7");
+#else
+	    __m64 v0 = *(__m64 *)(s + 0);
+	    __m64 v1 = *(__m64 *)(s + 8);
+	    __m64 v2 = *(__m64 *)(s + 16);
+	    __m64 v3 = *(__m64 *)(s + 24);
+	    __m64 v4 = *(__m64 *)(s + 32);
+	    __m64 v5 = *(__m64 *)(s + 40);
+	    __m64 v6 = *(__m64 *)(s + 48);
+	    __m64 v7 = *(__m64 *)(s + 56);
+	    *(__m64 *)(d + 0)  = v0;
+	    *(__m64 *)(d + 8)  = v1;
+	    *(__m64 *)(d + 16) = v2;
+	    *(__m64 *)(d + 24) = v3;
+	    *(__m64 *)(d + 32) = v4;
+	    *(__m64 *)(d + 40) = v5;
+	    *(__m64 *)(d + 48) = v6;
+	    *(__m64 *)(d + 56) = v7;
+#endif
+
+	    w -= 64;
+	    s += 64;
+	    d += 64;
+	}
+	while (w >= 4)
+	{
+	    *(uint32_t *)d = *(uint32_t *)s;
+
+	    w -= 4;
+	    s += 4;
+	    d += 4;
+	}
+	if (w >= 2)
+	{
+	    *(uint16_t *)d = *(uint16_t *)s;
+	    w -= 2;
+	    s += 2;
+	    d += 2;
+	}
+    }
+
+    _mm_empty ();
+
+    return TRUE;
+}
+
+static void
+mmx_composite_copy_area (pixman_implementation_t *imp,
+                         pixman_op_t              op,
+                         pixman_image_t *         src_image,
+                         pixman_image_t *         mask_image,
+                         pixman_image_t *         dst_image,
+                         int32_t                  src_x,
+                         int32_t                  src_y,
+                         int32_t                  mask_x,
+                         int32_t                  mask_y,
+                         int32_t                  dest_x,
+                         int32_t                  dest_y,
+                         int32_t                  width,
+                         int32_t                  height)
+{
+    pixman_blt_mmx (src_image->bits.bits,
+                    dst_image->bits.bits,
+                    src_image->bits.rowstride,
+                    dst_image->bits.rowstride,
+                    PIXMAN_FORMAT_BPP (src_image->bits.format),
+                    PIXMAN_FORMAT_BPP (dst_image->bits.format),
+                    src_x, src_y, dest_x, dest_y, width, height);
+}
+
+#if 0
+static void
+mmx_composite_over_x888_8_8888 (pixman_implementation_t *imp,
+                                pixman_op_t              op,
+                                pixman_image_t *         src_image,
+                                pixman_image_t *         mask_image,
+                                pixman_image_t *         dst_image,
+                                int32_t                  src_x,
+                                int32_t                  src_y,
+                                int32_t                  mask_x,
+                                int32_t                  mask_y,
+                                int32_t                  dest_x,
+                                int32_t                  dest_y,
+                                int32_t                  width,
+                                int32_t                  height)
+{
+    uint32_t  *src, *src_line;
+    uint32_t  *dst, *dst_line;
+    uint8_t  *mask, *mask_line;
+    int src_stride, mask_stride, dst_stride;
+    int32_t w;
+
+    PIXMAN_IMAGE_GET_LINE (dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
+    PIXMAN_IMAGE_GET_LINE (mask_image, mask_x, mask_y, uint8_t, mask_stride, mask_line, 1);
+    PIXMAN_IMAGE_GET_LINE (src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
+
+    while (height--)
+    {
+	src = src_line;
+	src_line += src_stride;
+	dst = dst_line;
+	dst_line += dst_stride;
+	mask = mask_line;
+	mask_line += mask_stride;
+
+	w = width;
+
+	while (w--)
+	{
+	    uint64_t m = *mask;
+
+	    if (m)
+	    {
+		__m64 s = load8888 (*src | 0xff000000);
+
+		if (m == 0xff)
+		{
+		    *dst = store8888 (s);
+		}
+		else
+		{
+		    __m64 sa = expand_alpha (s);
+		    __m64 vm = expand_alpha_rev (M64 (m));
+		    __m64 vdest = in_over (s, sa, vm, load8888 (*dst));
+
+		    *dst = store8888 (vdest);
+		}
+	    }
+
+	    mask++;
+	    dst++;
+	    src++;
+	}
+    }
+
+    _mm_empty ();
+}
+#endif
+
+static const pixman_fast_path_t mmx_fast_paths[] =
+{
+    PIXMAN_STD_FAST_PATH    (OVER, solid,    a8,       r5g6b5,   mmx_composite_over_n_8_0565       ),
+    PIXMAN_STD_FAST_PATH    (OVER, solid,    a8,       b5g6r5,   mmx_composite_over_n_8_0565       ),
+    PIXMAN_STD_FAST_PATH    (OVER, solid,    a8,       a8r8g8b8, mmx_composite_over_n_8_8888       ),
+    PIXMAN_STD_FAST_PATH    (OVER, solid,    a8,       x8r8g8b8, mmx_composite_over_n_8_8888       ),
+    PIXMAN_STD_FAST_PATH    (OVER, solid,    a8,       a8b8g8r8, mmx_composite_over_n_8_8888       ),
+    PIXMAN_STD_FAST_PATH    (OVER, solid,    a8,       x8b8g8r8, mmx_composite_over_n_8_8888       ),
+    PIXMAN_STD_FAST_PATH_CA (OVER, solid,    a8r8g8b8, a8r8g8b8, mmx_composite_over_n_8888_8888_ca ),
+    PIXMAN_STD_FAST_PATH_CA (OVER, solid,    a8r8g8b8, x8r8g8b8, mmx_composite_over_n_8888_8888_ca ),
+    PIXMAN_STD_FAST_PATH_CA (OVER, solid,    a8r8g8b8, r5g6b5,   mmx_composite_over_n_8888_0565_ca ),
+    PIXMAN_STD_FAST_PATH_CA (OVER, solid,    a8b8g8r8, a8b8g8r8, mmx_composite_over_n_8888_8888_ca ),
+    PIXMAN_STD_FAST_PATH_CA (OVER, solid,    a8b8g8r8, x8b8g8r8, mmx_composite_over_n_8888_8888_ca ),
+    PIXMAN_STD_FAST_PATH_CA (OVER, solid,    a8b8g8r8, b5g6r5,   mmx_composite_over_n_8888_0565_ca ),
+    PIXMAN_STD_FAST_PATH    (OVER, pixbuf,   pixbuf,   a8r8g8b8, mmx_composite_over_pixbuf_8888    ),
+    PIXMAN_STD_FAST_PATH    (OVER, pixbuf,   pixbuf,   x8r8g8b8, mmx_composite_over_pixbuf_8888    ),
+    PIXMAN_STD_FAST_PATH    (OVER, pixbuf,   pixbuf,   r5g6b5,   mmx_composite_over_pixbuf_0565    ),
+    PIXMAN_STD_FAST_PATH    (OVER, rpixbuf,  rpixbuf,  a8b8g8r8, mmx_composite_over_pixbuf_8888    ),
+    PIXMAN_STD_FAST_PATH    (OVER, rpixbuf,  rpixbuf,  x8b8g8r8, mmx_composite_over_pixbuf_8888    ),
+    PIXMAN_STD_FAST_PATH    (OVER, rpixbuf,  rpixbuf,  b5g6r5,   mmx_composite_over_pixbuf_0565    ),
+    PIXMAN_STD_FAST_PATH    (OVER, x8r8g8b8, solid,    a8r8g8b8, mmx_composite_over_x888_n_8888    ),
+    PIXMAN_STD_FAST_PATH    (OVER, x8r8g8b8, solid,    x8r8g8b8, mmx_composite_over_x888_n_8888    ),
+    PIXMAN_STD_FAST_PATH    (OVER, x8b8g8r8, solid,    a8b8g8r8, mmx_composite_over_x888_n_8888    ),
+    PIXMAN_STD_FAST_PATH    (OVER, x8b8g8r8, solid,    x8b8g8r8, mmx_composite_over_x888_n_8888    ),
+    PIXMAN_STD_FAST_PATH    (OVER, a8r8g8b8, solid,    a8r8g8b8, mmx_composite_over_8888_n_8888    ),
+    PIXMAN_STD_FAST_PATH    (OVER, a8r8g8b8, solid,    x8r8g8b8, mmx_composite_over_8888_n_8888    ),
+    PIXMAN_STD_FAST_PATH    (OVER, a8b8g8r8, solid,    a8b8g8r8, mmx_composite_over_8888_n_8888    ),
+    PIXMAN_STD_FAST_PATH    (OVER, a8b8g8r8, solid,    x8b8g8r8, mmx_composite_over_8888_n_8888    ),
+#if 0
+    /* FIXME: This code is commented out since it's apparently
+     * not actually faster than the generic code.
+     */
+    PIXMAN_STD_FAST_PATH    (OVER, x8r8g8b8, a8,       x8r8g8b8, mmx_composite_over_x888_8_8888    ),
+    PIXMAN_STD_FAST_PATH    (OVER, x8r8g8b8, a8,       a8r8g8b8, mmx_composite_over_x888_8_8888    ),
+    PIXMAN_STD_FAST_PATH    (OVER, x8b8r8g8, a8,       x8b8g8r8, mmx_composite_over_x888_8_8888    ),
+    PIXMAN_STD_FAST_PATH    (OVER, x8b8r8g8, a8,       a8r8g8b8, mmx_composite_over_x888_8_8888    ),
+#endif
+    PIXMAN_STD_FAST_PATH    (OVER, solid,    null,     a8r8g8b8, mmx_composite_over_n_8888         ),
+    PIXMAN_STD_FAST_PATH    (OVER, solid,    null,     x8r8g8b8, mmx_composite_over_n_8888         ),
+    PIXMAN_STD_FAST_PATH    (OVER, solid,    null,     r5g6b5,   mmx_composite_over_n_0565         ),
+    PIXMAN_STD_FAST_PATH    (OVER, x8r8g8b8, null,     x8r8g8b8, mmx_composite_copy_area           ),
+    PIXMAN_STD_FAST_PATH    (OVER, x8b8g8r8, null,     x8b8g8r8, mmx_composite_copy_area           ),
+
+    PIXMAN_STD_FAST_PATH    (OVER, a8r8g8b8, null,     a8r8g8b8, mmx_composite_over_8888_8888      ),
+    PIXMAN_STD_FAST_PATH    (OVER, a8r8g8b8, null,     x8r8g8b8, mmx_composite_over_8888_8888      ),
+    PIXMAN_STD_FAST_PATH    (OVER, a8r8g8b8, null,     r5g6b5,   mmx_composite_over_8888_0565      ),
+    PIXMAN_STD_FAST_PATH    (OVER, a8b8g8r8, null,     a8b8g8r8, mmx_composite_over_8888_8888      ),
+    PIXMAN_STD_FAST_PATH    (OVER, a8b8g8r8, null,     x8b8g8r8, mmx_composite_over_8888_8888      ),
+    PIXMAN_STD_FAST_PATH    (OVER, a8b8g8r8, null,     b5g6r5,   mmx_composite_over_8888_0565      ),
+
+    PIXMAN_STD_FAST_PATH    (ADD,  a8r8g8b8, null,     a8r8g8b8, mmx_composite_add_8888_8888       ),
+    PIXMAN_STD_FAST_PATH    (ADD,  a8b8g8r8, null,     a8b8g8r8, mmx_composite_add_8888_8888       ),
+    PIXMAN_STD_FAST_PATH    (ADD,  a8,       null,     a8,       mmx_composite_add_8000_8000       ),
+    PIXMAN_STD_FAST_PATH    (ADD,  solid,    a8,       a8,       mmx_composite_add_n_8_8           ),
+
+    PIXMAN_STD_FAST_PATH    (SRC,  solid,    a8,       a8r8g8b8, mmx_composite_src_n_8_8888        ),
+    PIXMAN_STD_FAST_PATH    (SRC,  solid,    a8,       x8r8g8b8, mmx_composite_src_n_8_8888        ),
+    PIXMAN_STD_FAST_PATH    (SRC,  solid,    a8,       a8b8g8r8, mmx_composite_src_n_8_8888        ),
+    PIXMAN_STD_FAST_PATH    (SRC,  solid,    a8,       x8b8g8r8, mmx_composite_src_n_8_8888        ),
+    PIXMAN_STD_FAST_PATH    (SRC,  a8r8g8b8, null,     a8r8g8b8, mmx_composite_copy_area           ),
+    PIXMAN_STD_FAST_PATH    (SRC,  a8b8g8r8, null,     a8b8g8r8, mmx_composite_copy_area           ),
+    PIXMAN_STD_FAST_PATH    (SRC,  a8r8g8b8, null,     x8r8g8b8, mmx_composite_copy_area           ),
+    PIXMAN_STD_FAST_PATH    (SRC,  a8b8g8r8, null,     x8b8g8r8, mmx_composite_copy_area           ),
+    PIXMAN_STD_FAST_PATH    (SRC,  x8r8g8b8, null,     x8r8g8b8, mmx_composite_copy_area           ),
+    PIXMAN_STD_FAST_PATH    (SRC,  x8b8g8r8, null,     x8b8g8r8, mmx_composite_copy_area           ),
+    PIXMAN_STD_FAST_PATH    (SRC,  r5g6b5,   null,     r5g6b5,   mmx_composite_copy_area           ),
+    PIXMAN_STD_FAST_PATH    (SRC,  b5g6r5,   null,     b5g6r5,   mmx_composite_copy_area           ),
+
+    PIXMAN_STD_FAST_PATH    (IN,   a8,       null,     a8,       mmx_composite_in_8_8              ),
+    PIXMAN_STD_FAST_PATH    (IN,   solid,    a8,       a8,       mmx_composite_in_n_8_8            ),
+
+    { PIXMAN_OP_NONE },
+};
+
+static pixman_bool_t
+mmx_blt (pixman_implementation_t *imp,
+         uint32_t *               src_bits,
+         uint32_t *               dst_bits,
+         int                      src_stride,
+         int                      dst_stride,
+         int                      src_bpp,
+         int                      dst_bpp,
+         int                      src_x,
+         int                      src_y,
+         int                      dst_x,
+         int                      dst_y,
+         int                      width,
+         int                      height)
+{
+    if (!pixman_blt_mmx (
+            src_bits, dst_bits, src_stride, dst_stride, src_bpp, dst_bpp,
+            src_x, src_y, dst_x, dst_y, width, height))
+
+    {
+	return _pixman_implementation_blt (
+	    imp->delegate,
+	    src_bits, dst_bits, src_stride, dst_stride, src_bpp, dst_bpp,
+	    src_x, src_y, dst_x, dst_y, width, height);
+    }
+
+    return TRUE;
+}
+
+static pixman_bool_t
+mmx_fill (pixman_implementation_t *imp,
+          uint32_t *               bits,
+          int                      stride,
+          int                      bpp,
+          int                      x,
+          int                      y,
+          int                      width,
+          int                      height,
+          uint32_t xor)
+{
+    if (!pixman_fill_mmx (bits, stride, bpp, x, y, width, height, xor))
+    {
+	return _pixman_implementation_fill (
+	    imp->delegate, bits, stride, bpp, x, y, width, height, xor);
+    }
+
+    return TRUE;
+}
+
+pixman_implementation_t *
+_pixman_implementation_create_mmx (void)
+{
+    pixman_implementation_t *general = _pixman_implementation_create_fast_path ();
+    pixman_implementation_t *imp = _pixman_implementation_create (general, mmx_fast_paths);
+
+    imp->combine_32[PIXMAN_OP_OVER] = mmx_combine_over_u;
+    imp->combine_32[PIXMAN_OP_OVER_REVERSE] = mmx_combine_over_reverse_u;
+    imp->combine_32[PIXMAN_OP_IN] = mmx_combine_in_u;
+    imp->combine_32[PIXMAN_OP_IN_REVERSE] = mmx_combine_in_reverse_u;
+    imp->combine_32[PIXMAN_OP_OUT] = mmx_combine_out_u;
+    imp->combine_32[PIXMAN_OP_OUT_REVERSE] = mmx_combine_out_reverse_u;
+    imp->combine_32[PIXMAN_OP_ATOP] = mmx_combine_atop_u;
+    imp->combine_32[PIXMAN_OP_ATOP_REVERSE] = mmx_combine_atop_reverse_u;
+    imp->combine_32[PIXMAN_OP_XOR] = mmx_combine_xor_u;
+    imp->combine_32[PIXMAN_OP_ADD] = mmx_combine_add_u;
+    imp->combine_32[PIXMAN_OP_SATURATE] = mmx_combine_saturate_u;
+
+    imp->combine_32_ca[PIXMAN_OP_SRC] = mmx_combine_src_ca;
+    imp->combine_32_ca[PIXMAN_OP_OVER] = mmx_combine_over_ca;
+    imp->combine_32_ca[PIXMAN_OP_OVER_REVERSE] = mmx_combine_over_reverse_ca;
+    imp->combine_32_ca[PIXMAN_OP_IN] = mmx_combine_in_ca;
+    imp->combine_32_ca[PIXMAN_OP_IN_REVERSE] = mmx_combine_in_reverse_ca;
+    imp->combine_32_ca[PIXMAN_OP_OUT] = mmx_combine_out_ca;
+    imp->combine_32_ca[PIXMAN_OP_OUT_REVERSE] = mmx_combine_out_reverse_ca;
+    imp->combine_32_ca[PIXMAN_OP_ATOP] = mmx_combine_atop_ca;
+    imp->combine_32_ca[PIXMAN_OP_ATOP_REVERSE] = mmx_combine_atop_reverse_ca;
+    imp->combine_32_ca[PIXMAN_OP_XOR] = mmx_combine_xor_ca;
+    imp->combine_32_ca[PIXMAN_OP_ADD] = mmx_combine_add_ca;
+
+    imp->blt = mmx_blt;
+    imp->fill = mmx_fill;
+
+    return imp;
+}
+
+#endif /* USE_MMX */
diff --git a/pixman/pixman/pixman-private.h b/pixman/pixman/pixman-private.h
new file mode 100644
index 000000000..d5767af48
--- /dev/null
+++ b/pixman/pixman/pixman-private.h
@@ -0,0 +1,870 @@
+#ifndef PACKAGE
+#  error config.h must be included before pixman-private.h
+#endif
+
+#ifndef PIXMAN_PRIVATE_H
+#define PIXMAN_PRIVATE_H
+
+#define PIXMAN_DISABLE_DEPRECATED
+#define PIXMAN_USE_INTERNAL_API
+
+#include "pixman.h"
+#include <time.h>
+#include <assert.h>
+#include <stdio.h>
+#include <string.h>
+
+#include "pixman-compiler.h"
+
+/*
+ * Images
+ */
+typedef struct image_common image_common_t;
+typedef struct source_image source_image_t;
+typedef struct solid_fill solid_fill_t;
+typedef struct gradient gradient_t;
+typedef struct linear_gradient linear_gradient_t;
+typedef struct horizontal_gradient horizontal_gradient_t;
+typedef struct vertical_gradient vertical_gradient_t;
+typedef struct conical_gradient conical_gradient_t;
+typedef struct radial_gradient radial_gradient_t;
+typedef struct bits_image bits_image_t;
+typedef struct circle circle_t;
+
+typedef void (*fetch_scanline_t) (pixman_image_t *image,
+				  int             x,
+				  int             y,
+				  int             width,
+				  uint32_t       *buffer,
+				  const uint32_t *mask,
+				  uint32_t        mask_bits);
+
+typedef uint32_t (*fetch_pixel_32_t) (bits_image_t *image,
+				      int           x,
+				      int           y);
+
+typedef uint64_t (*fetch_pixel_64_t) (bits_image_t *image,
+				      int           x,
+				      int           y);
+
+typedef void (*store_scanline_t) (bits_image_t *  image,
+				  int             x,
+				  int             y,
+				  int             width,
+				  const uint32_t *values);
+
+typedef enum
+{
+    BITS,
+    LINEAR,
+    CONICAL,
+    RADIAL,
+    SOLID
+} image_type_t;
+
+typedef enum
+{
+    SOURCE_IMAGE_CLASS_UNKNOWN,
+    SOURCE_IMAGE_CLASS_HORIZONTAL,
+    SOURCE_IMAGE_CLASS_VERTICAL,
+} source_image_class_t;
+
+typedef source_image_class_t (*classify_func_t) (pixman_image_t *image,
+						int             x,
+						int             y,
+						int             width,
+						int             height);
+typedef void (*property_changed_func_t) (pixman_image_t *image);
+
+struct image_common
+{
+    image_type_t                type;
+    int32_t                     ref_count;
+    pixman_region32_t           clip_region;
+    pixman_bool_t               have_clip_region;   /* FALSE if there is no clip */
+    pixman_bool_t               client_clip;        /* Whether the source clip was
+						       set by a client */
+    pixman_bool_t               clip_sources;       /* Whether the clip applies when
+						     * the image is used as a source
+						     */
+    pixman_bool_t		dirty;
+    pixman_transform_t *        transform;
+    pixman_repeat_t             repeat;
+    pixman_filter_t             filter;
+    pixman_fixed_t *            filter_params;
+    int                         n_filter_params;
+    bits_image_t *              alpha_map;
+    int                         alpha_origin_x;
+    int                         alpha_origin_y;
+    pixman_bool_t               component_alpha;
+    classify_func_t             classify;
+    property_changed_func_t     property_changed;
+    fetch_scanline_t            get_scanline_32;
+    fetch_scanline_t            get_scanline_64;
+
+    pixman_image_destroy_func_t destroy_func;
+    void *                      destroy_data;
+
+    uint32_t			flags;
+    pixman_format_code_t	extended_format_code;
+};
+
+struct source_image
+{
+    image_common_t common;
+    source_image_class_t class;
+};
+
+struct solid_fill
+{
+    source_image_t common;
+    pixman_color_t color;
+    
+    uint32_t	   color_32;
+    uint64_t	   color_64;
+};
+
+struct gradient
+{
+    source_image_t          common;
+    int                     n_stops;
+    pixman_gradient_stop_t *stops;
+    int                     stop_range;
+};
+
+struct linear_gradient
+{
+    gradient_t           common;
+    pixman_point_fixed_t p1;
+    pixman_point_fixed_t p2;
+};
+
+struct circle
+{
+    pixman_fixed_t x;
+    pixman_fixed_t y;
+    pixman_fixed_t radius;
+};
+
+struct radial_gradient
+{
+    gradient_t common;
+
+    circle_t   c1;
+    circle_t   c2;
+    double     cdx;
+    double     cdy;
+    double     dr;
+    double     A;
+};
+
+struct conical_gradient
+{
+    gradient_t           common;
+    pixman_point_fixed_t center;
+    pixman_fixed_t       angle;
+};
+
+struct bits_image
+{
+    image_common_t             common;
+    pixman_format_code_t       format;
+    const pixman_indexed_t *   indexed;
+    int                        width;
+    int                        height;
+    uint32_t *                 bits;
+    uint32_t *                 free_me;
+    int                        rowstride;  /* in number of uint32_t's */
+
+    /* Fetch a pixel, disregarding alpha maps, transformations etc. */
+    fetch_pixel_32_t	       fetch_pixel_raw_32;
+    fetch_pixel_64_t	       fetch_pixel_raw_64;
+
+    /* Fetch a pixel, taking alpha maps into account */
+    fetch_pixel_32_t	       fetch_pixel_32;
+    fetch_pixel_64_t	       fetch_pixel_64;
+
+    /* Fetch raw scanlines, with no regard for transformations, alpha maps etc. */
+    fetch_scanline_t           fetch_scanline_raw_32;
+    fetch_scanline_t           fetch_scanline_raw_64;
+
+    /* Store scanlines with no regard for alpha maps */
+    store_scanline_t           store_scanline_raw_32;
+    store_scanline_t           store_scanline_raw_64;
+
+    /* Store a scanline, taking alpha maps into account */
+    store_scanline_t           store_scanline_32;
+    store_scanline_t           store_scanline_64;
+
+    /* Used for indirect access to the bits */
+    pixman_read_memory_func_t  read_func;
+    pixman_write_memory_func_t write_func;
+};
+
+union pixman_image
+{
+    image_type_t       type;
+    image_common_t     common;
+    bits_image_t       bits;
+    source_image_t     source;
+    gradient_t         gradient;
+    linear_gradient_t  linear;
+    conical_gradient_t conical;
+    radial_gradient_t  radial;
+    solid_fill_t       solid;
+};
+
+
+void
+_pixman_bits_image_setup_raw_accessors (bits_image_t *image);
+
+void
+_pixman_image_get_scanline_generic_64  (pixman_image_t *image,
+                                        int             x,
+                                        int             y,
+                                        int             width,
+                                        uint32_t *      buffer,
+                                        const uint32_t *mask,
+                                        uint32_t        mask_bits);
+
+source_image_class_t
+_pixman_image_classify (pixman_image_t *image,
+                        int             x,
+                        int             y,
+                        int             width,
+                        int             height);
+
+void
+_pixman_image_get_scanline_32 (pixman_image_t *image,
+                               int             x,
+                               int             y,
+                               int             width,
+                               uint32_t *      buffer,
+                               const uint32_t *mask,
+                               uint32_t        mask_bits);
+
+/* Even thought the type of buffer is uint32_t *, the function actually expects
+ * a uint64_t *buffer.
+ */
+void
+_pixman_image_get_scanline_64 (pixman_image_t *image,
+                               int             x,
+                               int             y,
+                               int             width,
+                               uint32_t *      buffer,
+                               const uint32_t *unused,
+                               uint32_t        unused2);
+
+void
+_pixman_image_store_scanline_32 (bits_image_t *  image,
+                                 int             x,
+                                 int             y,
+                                 int             width,
+                                 const uint32_t *buffer);
+
+/* Even though the type of buffer is uint32_t *, the function
+ * actually expects a uint64_t *buffer.
+ */
+void
+_pixman_image_store_scanline_64 (bits_image_t *  image,
+                                 int             x,
+                                 int             y,
+                                 int             width,
+                                 const uint32_t *buffer);
+
+pixman_image_t *
+_pixman_image_allocate (void);
+
+pixman_bool_t
+_pixman_init_gradient (gradient_t *                  gradient,
+                       const pixman_gradient_stop_t *stops,
+                       int                           n_stops);
+void
+_pixman_image_reset_clip_region (pixman_image_t *image);
+
+void
+_pixman_image_validate (pixman_image_t *image);
+
+uint32_t
+_pixman_image_get_solid (pixman_image_t *     image,
+                         pixman_format_code_t format);
+
+#define PIXMAN_IMAGE_GET_LINE(image, x, y, type, out_stride, line, mul)	\
+    do									\
+    {									\
+	uint32_t *__bits__;						\
+	int       __stride__;						\
+        								\
+	__bits__ = image->bits.bits;					\
+	__stride__ = image->bits.rowstride;				\
+	(out_stride) =							\
+	    __stride__ * (int) sizeof (uint32_t) / (int) sizeof (type);	\
+	(line) =							\
+	    ((type *) __bits__) + (out_stride) * (y) + (mul) * (x);	\
+    } while (0)
+
+/*
+ * Gradient walker
+ */
+typedef struct
+{
+    uint32_t                left_ag;
+    uint32_t                left_rb;
+    uint32_t                right_ag;
+    uint32_t                right_rb;
+    int32_t                 left_x;
+    int32_t                 right_x;
+    int32_t                 stepper;
+
+    pixman_gradient_stop_t *stops;
+    int                     num_stops;
+    unsigned int            spread;
+
+    int                     need_reset;
+} pixman_gradient_walker_t;
+
+void
+_pixman_gradient_walker_init (pixman_gradient_walker_t *walker,
+                              gradient_t *              gradient,
+                              unsigned int              spread);
+
+void
+_pixman_gradient_walker_reset (pixman_gradient_walker_t *walker,
+                               pixman_fixed_32_32_t      pos);
+
+uint32_t
+_pixman_gradient_walker_pixel (pixman_gradient_walker_t *walker,
+                               pixman_fixed_32_32_t      x);
+
+/*
+ * Edges
+ */
+
+#define MAX_ALPHA(n)    ((1 << (n)) - 1)
+#define N_Y_FRAC(n)     ((n) == 1 ? 1 : (1 << ((n) / 2)) - 1)
+#define N_X_FRAC(n)     ((n) == 1 ? 1 : (1 << ((n) / 2)) + 1)
+
+#define STEP_Y_SMALL(n) (pixman_fixed_1 / N_Y_FRAC (n))
+#define STEP_Y_BIG(n)   (pixman_fixed_1 - (N_Y_FRAC (n) - 1) * STEP_Y_SMALL (n))
+
+#define Y_FRAC_FIRST(n) (STEP_Y_BIG (n) / 2)
+#define Y_FRAC_LAST(n)  (Y_FRAC_FIRST (n) + (N_Y_FRAC (n) - 1) * STEP_Y_SMALL (n))
+
+#define STEP_X_SMALL(n) (pixman_fixed_1 / N_X_FRAC (n))
+#define STEP_X_BIG(n)   (pixman_fixed_1 - (N_X_FRAC (n) - 1) * STEP_X_SMALL (n))
+
+#define X_FRAC_FIRST(n) (STEP_X_BIG (n) / 2)
+#define X_FRAC_LAST(n)  (X_FRAC_FIRST (n) + (N_X_FRAC (n) - 1) * STEP_X_SMALL (n))
+
+#define RENDER_SAMPLES_X(x, n)						\
+    ((n) == 1? 0 : (pixman_fixed_frac (x) +				\
+		    X_FRAC_FIRST (n)) / STEP_X_SMALL (n))
+
+void
+pixman_rasterize_edges_accessors (pixman_image_t *image,
+                                  pixman_edge_t * l,
+                                  pixman_edge_t * r,
+                                  pixman_fixed_t  t,
+                                  pixman_fixed_t  b);
+
+/*
+ * Implementations
+ */
+typedef struct pixman_implementation_t pixman_implementation_t;
+
+typedef void (*pixman_combine_32_func_t) (pixman_implementation_t *imp,
+					  pixman_op_t              op,
+					  uint32_t *               dest,
+					  const uint32_t *         src,
+					  const uint32_t *         mask,
+					  int                      width);
+
+typedef void (*pixman_combine_64_func_t) (pixman_implementation_t *imp,
+					  pixman_op_t              op,
+					  uint64_t *               dest,
+					  const uint64_t *         src,
+					  const uint64_t *         mask,
+					  int                      width);
+
+typedef void (*pixman_composite_func_t) (pixman_implementation_t *imp,
+					 pixman_op_t              op,
+					 pixman_image_t *         src,
+					 pixman_image_t *         mask,
+					 pixman_image_t *         dest,
+					 int32_t                  src_x,
+					 int32_t                  src_y,
+					 int32_t                  mask_x,
+					 int32_t                  mask_y,
+					 int32_t                  dest_x,
+					 int32_t                  dest_y,
+					 int32_t                  width,
+					 int32_t                  height);
+typedef pixman_bool_t (*pixman_blt_func_t) (pixman_implementation_t *imp,
+					    uint32_t *               src_bits,
+					    uint32_t *               dst_bits,
+					    int                      src_stride,
+					    int                      dst_stride,
+					    int                      src_bpp,
+					    int                      dst_bpp,
+					    int                      src_x,
+					    int                      src_y,
+					    int                      dst_x,
+					    int                      dst_y,
+					    int                      width,
+					    int                      height);
+typedef pixman_bool_t (*pixman_fill_func_t) (pixman_implementation_t *imp,
+					     uint32_t *               bits,
+					     int                      stride,
+					     int                      bpp,
+					     int                      x,
+					     int                      y,
+					     int                      width,
+					     int                      height,
+					     uint32_t                 xor);
+
+void _pixman_setup_combiner_functions_32 (pixman_implementation_t *imp);
+void _pixman_setup_combiner_functions_64 (pixman_implementation_t *imp);
+
+typedef struct
+{
+    pixman_op_t             op;
+    pixman_format_code_t    src_format;
+    uint32_t		    src_flags;
+    pixman_format_code_t    mask_format;
+    uint32_t		    mask_flags;
+    pixman_format_code_t    dest_format;
+    uint32_t		    dest_flags;
+    pixman_composite_func_t func;
+} pixman_fast_path_t;
+
+struct pixman_implementation_t
+{
+    pixman_implementation_t *	toplevel;
+    pixman_implementation_t *	delegate;
+    const pixman_fast_path_t *	fast_paths;
+    
+    pixman_blt_func_t		blt;
+    pixman_fill_func_t		fill;
+
+    pixman_combine_32_func_t	combine_32[PIXMAN_N_OPERATORS];
+    pixman_combine_32_func_t	combine_32_ca[PIXMAN_N_OPERATORS];
+    pixman_combine_64_func_t	combine_64[PIXMAN_N_OPERATORS];
+    pixman_combine_64_func_t	combine_64_ca[PIXMAN_N_OPERATORS];
+};
+
+pixman_implementation_t *
+_pixman_implementation_create (pixman_implementation_t *delegate,
+			       const pixman_fast_path_t *fast_paths);
+
+void
+_pixman_implementation_combine_32 (pixman_implementation_t *imp,
+                                   pixman_op_t              op,
+                                   uint32_t *               dest,
+                                   const uint32_t *         src,
+                                   const uint32_t *         mask,
+                                   int                      width);
+void
+_pixman_implementation_combine_64 (pixman_implementation_t *imp,
+                                   pixman_op_t              op,
+                                   uint64_t *               dest,
+                                   const uint64_t *         src,
+                                   const uint64_t *         mask,
+                                   int                      width);
+void
+_pixman_implementation_combine_32_ca (pixman_implementation_t *imp,
+                                      pixman_op_t              op,
+                                      uint32_t *               dest,
+                                      const uint32_t *         src,
+                                      const uint32_t *         mask,
+                                      int                      width);
+void
+_pixman_implementation_combine_64_ca (pixman_implementation_t *imp,
+                                      pixman_op_t              op,
+                                      uint64_t *               dest,
+                                      const uint64_t *         src,
+                                      const uint64_t *         mask,
+                                      int                      width);
+
+pixman_bool_t
+_pixman_implementation_blt (pixman_implementation_t *imp,
+                            uint32_t *               src_bits,
+                            uint32_t *               dst_bits,
+                            int                      src_stride,
+                            int                      dst_stride,
+                            int                      src_bpp,
+                            int                      dst_bpp,
+                            int                      src_x,
+                            int                      src_y,
+                            int                      dst_x,
+                            int                      dst_y,
+                            int                      width,
+                            int                      height);
+
+pixman_bool_t
+_pixman_implementation_fill (pixman_implementation_t *imp,
+                             uint32_t *               bits,
+                             int                      stride,
+                             int                      bpp,
+                             int                      x,
+                             int                      y,
+                             int                      width,
+                             int                      height,
+                             uint32_t                 xor);
+
+/* Specific implementations */
+pixman_implementation_t *
+_pixman_implementation_create_general (void);
+
+pixman_implementation_t *
+_pixman_implementation_create_fast_path (void);
+
+#ifdef USE_MMX
+pixman_implementation_t *
+_pixman_implementation_create_mmx (void);
+#endif
+
+#ifdef USE_SSE2
+pixman_implementation_t *
+_pixman_implementation_create_sse2 (void);
+#endif
+
+#ifdef USE_ARM_SIMD
+pixman_implementation_t *
+_pixman_implementation_create_arm_simd (void);
+#endif
+
+#ifdef USE_ARM_NEON
+pixman_implementation_t *
+_pixman_implementation_create_arm_neon (void);
+#endif
+
+#ifdef USE_VMX
+pixman_implementation_t *
+_pixman_implementation_create_vmx (void);
+#endif
+
+pixman_implementation_t *
+_pixman_choose_implementation (void);
+
+
+
+/*
+ * Utilities
+ */
+
+/* These "formats" all have depth 0, so they
+ * will never clash with any real ones
+ */
+#define PIXMAN_null             PIXMAN_FORMAT (0, 0, 0, 0, 0, 0)
+#define PIXMAN_solid            PIXMAN_FORMAT (0, 1, 0, 0, 0, 0)
+#define PIXMAN_pixbuf		PIXMAN_FORMAT (0, 2, 0, 0, 0, 0)
+#define PIXMAN_rpixbuf		PIXMAN_FORMAT (0, 3, 0, 0, 0, 0)
+#define PIXMAN_unknown		PIXMAN_FORMAT (0, 4, 0, 0, 0, 0)
+#define PIXMAN_any		PIXMAN_FORMAT (0, 5, 0, 0, 0, 0)
+
+#define PIXMAN_OP_any		(PIXMAN_N_OPERATORS + 1)
+
+#define FAST_PATH_ID_TRANSFORM			(1 <<  0)
+#define FAST_PATH_NO_ALPHA_MAP			(1 <<  1)
+#define FAST_PATH_NO_CONVOLUTION_FILTER		(1 <<  2)
+#define FAST_PATH_NO_PAD_REPEAT			(1 <<  3)
+#define FAST_PATH_NO_REFLECT_REPEAT		(1 <<  4)
+#define FAST_PATH_NO_ACCESSORS			(1 <<  5)
+#define FAST_PATH_NO_WIDE_FORMAT		(1 <<  6)
+#define FAST_PATH_COVERS_CLIP			(1 <<  7)
+#define FAST_PATH_COMPONENT_ALPHA		(1 <<  8)
+#define FAST_PATH_UNIFIED_ALPHA			(1 <<  9)
+#define FAST_PATH_SCALE_TRANSFORM		(1 << 10)
+#define FAST_PATH_NEAREST_FILTER		(1 << 11)
+#define FAST_PATH_SIMPLE_REPEAT			(1 << 12)
+#define FAST_PATH_IS_OPAQUE			(1 << 13)
+#define FAST_PATH_NEEDS_WORKAROUND		(1 << 14)
+#define FAST_PATH_NO_NONE_REPEAT		(1 << 15)
+#define FAST_PATH_SAMPLES_COVER_CLIP		(1 << 16)
+#define FAST_PATH_16BIT_SAFE			(1 << 17)
+#define FAST_PATH_X_UNIT_POSITIVE		(1 << 18)
+
+#define _FAST_PATH_STANDARD_FLAGS					\
+    (FAST_PATH_ID_TRANSFORM		|				\
+     FAST_PATH_NO_ALPHA_MAP		|				\
+     FAST_PATH_NO_CONVOLUTION_FILTER	|				\
+     FAST_PATH_NO_PAD_REPEAT		|				\
+     FAST_PATH_NO_REFLECT_REPEAT	|				\
+     FAST_PATH_NO_ACCESSORS		|				\
+     FAST_PATH_NO_WIDE_FORMAT		|				\
+     FAST_PATH_COVERS_CLIP)
+
+#define FAST_PATH_STD_SRC_FLAGS						\
+    _FAST_PATH_STANDARD_FLAGS
+#define FAST_PATH_STD_MASK_U_FLAGS					\
+    (_FAST_PATH_STANDARD_FLAGS		|				\
+     FAST_PATH_UNIFIED_ALPHA)
+#define FAST_PATH_STD_MASK_CA_FLAGS					\
+    (_FAST_PATH_STANDARD_FLAGS		|				\
+     FAST_PATH_COMPONENT_ALPHA)
+#define FAST_PATH_STD_DEST_FLAGS					\
+    (FAST_PATH_NO_ACCESSORS		|				\
+     FAST_PATH_NO_WIDE_FORMAT)
+
+#define FAST_PATH(op, src, src_flags, mask, mask_flags, dest, dest_flags, func) \
+    PIXMAN_OP_ ## op,							\
+    PIXMAN_ ## src,							\
+    src_flags,							        \
+    PIXMAN_ ## mask,						        \
+    mask_flags,							        \
+    PIXMAN_ ## dest,	                                                \
+    dest_flags,							        \
+    func
+
+#define PIXMAN_STD_FAST_PATH(op, src, mask, dest, func)			\
+    { FAST_PATH (							\
+	  op,								\
+	  src, FAST_PATH_STD_SRC_FLAGS,					\
+	  mask, (PIXMAN_ ## mask) ? FAST_PATH_STD_MASK_U_FLAGS : 0,	\
+	  dest, FAST_PATH_STD_DEST_FLAGS,				\
+	  func) }
+
+#define PIXMAN_STD_FAST_PATH_CA(op, src, mask, dest, func)		\
+    { FAST_PATH (							\
+	  op,								\
+	  src, FAST_PATH_STD_SRC_FLAGS,					\
+	  mask, FAST_PATH_STD_MASK_CA_FLAGS,				\
+	  dest, FAST_PATH_STD_DEST_FLAGS,				\
+	  func) }
+
+/* Memory allocation helpers */
+void *
+pixman_malloc_ab (unsigned int n, unsigned int b);
+
+void *
+pixman_malloc_abc (unsigned int a, unsigned int b, unsigned int c);
+
+pixman_bool_t
+pixman_multiply_overflows_int (unsigned int a, unsigned int b);
+
+pixman_bool_t
+pixman_addition_overflows_int (unsigned int a, unsigned int b);
+
+/* Compositing utilities */
+void
+pixman_expand (uint64_t *           dst,
+               const uint32_t *     src,
+               pixman_format_code_t format,
+               int                  width);
+
+void
+pixman_contract (uint32_t *      dst,
+                 const uint64_t *src,
+                 int             width);
+
+
+/* Region Helpers */
+pixman_bool_t
+pixman_region32_copy_from_region16 (pixman_region32_t *dst,
+                                    pixman_region16_t *src);
+
+pixman_bool_t
+pixman_region16_copy_from_region32 (pixman_region16_t *dst,
+                                    pixman_region32_t *src);
+
+
+/* Misc macros */
+
+#ifndef FALSE
+#   define FALSE 0
+#endif
+
+#ifndef TRUE
+#   define TRUE 1
+#endif
+
+#ifndef MIN
+#  define MIN(a, b) ((a < b) ? a : b)
+#endif
+
+#ifndef MAX
+#  define MAX(a, b) ((a > b) ? a : b)
+#endif
+
+/* Integer division that rounds towards -infinity */
+#define DIV(a, b)					   \
+    ((((a) < 0) == ((b) < 0)) ? (a) / (b) :                \
+     ((a) - (b) + 1 - (((b) < 0) << 1)) / (b))
+
+/* Modulus that produces the remainder wrt. DIV */
+#define MOD(a, b) ((a) < 0 ? ((b) - ((-(a) - 1) % (b))) - 1 : (a) % (b))
+
+#define CLIP(v, low, high) ((v) < (low) ? (low) : ((v) > (high) ? (high) : (v)))
+
+/* Conversion between 8888 and 0565 */
+
+#define CONVERT_8888_TO_0565(s)						\
+    ((((s) >> 3) & 0x001f) |						\
+     (((s) >> 5) & 0x07e0) |						\
+     (((s) >> 8) & 0xf800))
+
+#define CONVERT_0565_TO_0888(s)						\
+    (((((s) << 3) & 0xf8) | (((s) >> 2) & 0x7)) |			\
+     ((((s) << 5) & 0xfc00) | (((s) >> 1) & 0x300)) |			\
+     ((((s) << 8) & 0xf80000) | (((s) << 3) & 0x70000)))
+
+#define CONVERT_0565_TO_8888(s) (CONVERT_0565_TO_0888(s) | 0xff000000)
+
+/* Trivial versions that are useful in macros */
+#define CONVERT_8888_TO_8888(s) (s)
+#define CONVERT_0565_TO_0565(s) (s)
+
+#define PIXMAN_FORMAT_IS_WIDE(f)					\
+    (PIXMAN_FORMAT_A (f) > 8 ||						\
+     PIXMAN_FORMAT_R (f) > 8 ||						\
+     PIXMAN_FORMAT_G (f) > 8 ||						\
+     PIXMAN_FORMAT_B (f) > 8)
+
+#ifdef WORDS_BIGENDIAN
+#   define SCREEN_SHIFT_LEFT(x,n)	((x) << (n))
+#   define SCREEN_SHIFT_RIGHT(x,n)	((x) >> (n))
+#else
+#   define SCREEN_SHIFT_LEFT(x,n)	((x) >> (n))
+#   define SCREEN_SHIFT_RIGHT(x,n)	((x) << (n))
+#endif
+
+/*
+ * Various debugging code
+ */
+
+#undef DEBUG
+
+/* Turn on debugging depending on what type of release this is
+ */
+#if (((PIXMAN_VERSION_MICRO % 2) == 0) && ((PIXMAN_VERSION_MINOR % 2) == 1))
+
+/* Debugging gets turned on for development releases because these
+ * are the things that end up in bleeding edge distributions such
+ * as Rawhide etc.
+ *
+ * For performance reasons we don't turn it on for stable releases or
+ * random git checkouts. (Random git checkouts are often used for
+ * performance work).
+ */
+
+#    define DEBUG
+
+#endif
+
+#ifdef DEBUG
+
+void
+_pixman_log_error (const char *function, const char *message);
+
+#define return_if_fail(expr)                                            \
+    do                                                                  \
+    {                                                                   \
+	if (!(expr))							\
+	{								\
+	    _pixman_log_error (FUNC, "The expression " # expr " was false"); \
+	    return;							\
+	}								\
+    }                                                                   \
+    while (0)
+
+#define return_val_if_fail(expr, retval)                                \
+    do                                                                  \
+    {                                                                   \
+	if (!(expr))                                                    \
+	{								\
+	    _pixman_log_error (FUNC, "The expression " # expr " was false"); \
+	    return (retval);						\
+	}								\
+    }                                                                   \
+    while (0)
+
+#define critical_if_fail(expr)						\
+    do									\
+    {									\
+	if (!(expr))							\
+	    _pixman_log_error (FUNC, "The expression " # expr " was false"); \
+    }									\
+    while (0)
+
+
+#else
+
+#define _pixman_log_error(f,m) do { } while (0)				\
+
+#define return_if_fail(expr)						\
+    do                                                                  \
+    {                                                                   \
+	if (!(expr))							\
+	    return;							\
+    }                                                                   \
+    while (0)
+
+#define return_val_if_fail(expr, retval)                                \
+    do                                                                  \
+    {                                                                   \
+	if (!(expr))							\
+	    return (retval);						\
+    }                                                                   \
+    while (0)
+
+#define critical_if_fail(expr)						\
+    do									\
+    {									\
+    }									\
+    while (0)
+#endif
+
+/*
+ * Timers
+ */
+
+#ifdef PIXMAN_TIMERS
+
+static inline uint64_t
+oil_profile_stamp_rdtsc (void)
+{
+    uint64_t ts;
+
+    __asm__ __volatile__ ("rdtsc\n" : "=A" (ts));
+    return ts;
+}
+
+#define OIL_STAMP oil_profile_stamp_rdtsc
+
+typedef struct pixman_timer_t pixman_timer_t;
+
+struct pixman_timer_t
+{
+    int             initialized;
+    const char *    name;
+    uint64_t        n_times;
+    uint64_t        total;
+    pixman_timer_t *next;
+};
+
+extern int timer_defined;
+
+void pixman_timer_register (pixman_timer_t *timer);
+
+#define TIMER_BEGIN(tname)                                              \
+    {                                                                   \
+	static pixman_timer_t timer ## tname;                           \
+	uint64_t              begin ## tname;                           \
+        								\
+	if (!timer ## tname.initialized)				\
+	{                                                               \
+	    timer ## tname.initialized = 1;				\
+	    timer ## tname.name = # tname;				\
+	    pixman_timer_register (&timer ## tname);			\
+	}                                                               \
+									\
+	timer ## tname.n_times++;					\
+	begin ## tname = OIL_STAMP ();
+
+#define TIMER_END(tname)                                                \
+    timer ## tname.total += OIL_STAMP () - begin ## tname;		\
+    }
+
+#endif /* PIXMAN_TIMERS */
+
+#endif /* PIXMAN_PRIVATE_H */
diff --git a/pixman/pixman/pixman-radial-gradient.c b/pixman/pixman/pixman-radial-gradient.c
new file mode 100644
index 000000000..022157b9b
--- /dev/null
+++ b/pixman/pixman/pixman-radial-gradient.c
@@ -0,0 +1,368 @@
+/*
+ *
+ * Copyright © 2000 Keith Packard, member of The XFree86 Project, Inc.
+ * Copyright © 2000 SuSE, Inc.
+ *             2005 Lars Knoll & Zack Rusin, Trolltech
+ * Copyright © 2007 Red Hat, Inc.
+ *
+ *
+ * Permission to use, copy, modify, distribute, and sell this software and its
+ * documentation for any purpose is hereby granted without fee, provided that
+ * the above copyright notice appear in all copies and that both that
+ * copyright notice and this permission notice appear in supporting
+ * documentation, and that the name of Keith Packard not be used in
+ * advertising or publicity pertaining to distribution of the software without
+ * specific, written prior permission.  Keith Packard makes no
+ * representations about the suitability of this software for any purpose.  It
+ * is provided "as is" without express or implied warranty.
+ *
+ * THE COPYRIGHT HOLDERS DISCLAIM ALL WARRANTIES WITH REGARD TO THIS
+ * SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
+ * FITNESS, IN NO EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
+ * AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
+ * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS
+ * SOFTWARE.
+ */
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+#include <stdlib.h>
+#include <math.h>
+#include "pixman-private.h"
+
+static void
+radial_gradient_get_scanline_32 (pixman_image_t *image,
+                                 int             x,
+                                 int             y,
+                                 int             width,
+                                 uint32_t *      buffer,
+                                 const uint32_t *mask,
+                                 uint32_t        mask_bits)
+{
+    /*
+     * In the radial gradient problem we are given two circles (c₁,r₁) and
+     * (c₂,r₂) that define the gradient itself. Then, for any point p, we
+     * must compute the value(s) of t within [0.0, 1.0] representing the
+     * circle(s) that would color the point.
+     *
+     * There are potentially two values of t since the point p can be
+     * colored by both sides of the circle, (which happens whenever one
+     * circle is not entirely contained within the other).
+     *
+     * If we solve for a value of t that is outside of [0.0, 1.0] then we
+     * use the extend mode (NONE, REPEAT, REFLECT, or PAD) to map to a
+     * value within [0.0, 1.0].
+     *
+     * Here is an illustration of the problem:
+     *
+     *              p₂
+     *           p  •
+     *           •   ╲
+     *        ·       ╲r₂
+     *  p₁ ·           ╲
+     *  •              θ╲
+     *   ╲             ╌╌•
+     *    ╲r₁        ·   c₂
+     *    θ╲    ·
+     *    ╌╌•
+     *      c₁
+     *
+     * Given (c₁,r₁), (c₂,r₂) and p, we must find an angle θ such that two
+     * points p₁ and p₂ on the two circles are collinear with p. Then, the
+     * desired value of t is the ratio of the length of p₁p to the length
+     * of p₁p₂.
+     *
+     * So, we have six unknown values: (p₁x, p₁y), (p₂x, p₂y), θ and t.
+     * We can also write six equations that constrain the problem:
+     *
+     * Point p₁ is a distance r₁ from c₁ at an angle of θ:
+     *
+     *	1. p₁x = c₁x + r₁·cos θ
+     *	2. p₁y = c₁y + r₁·sin θ
+     *
+     * Point p₂ is a distance r₂ from c₂ at an angle of θ:
+     *
+     *	3. p₂x = c₂x + r2·cos θ
+     *	4. p₂y = c₂y + r2·sin θ
+     *
+     * Point p lies at a fraction t along the line segment p₁p₂:
+     *
+     *	5. px = t·p₂x + (1-t)·p₁x
+     *	6. py = t·p₂y + (1-t)·p₁y
+     *
+     * To solve, first subtitute 1-4 into 5 and 6:
+     *
+     * px = t·(c₂x + r₂·cos θ) + (1-t)·(c₁x + r₁·cos θ)
+     * py = t·(c₂y + r₂·sin θ) + (1-t)·(c₁y + r₁·sin θ)
+     *
+     * Then solve each for cos θ and sin θ expressed as a function of t:
+     *
+     * cos θ = (-(c₂x - c₁x)·t + (px - c₁x)) / ((r₂-r₁)·t + r₁)
+     * sin θ = (-(c₂y - c₁y)·t + (py - c₁y)) / ((r₂-r₁)·t + r₁)
+     *
+     * To simplify this a bit, we define new variables for several of the
+     * common terms as shown below:
+     *
+     *              p₂
+     *           p  •
+     *           •   ╲
+     *        ·  ┆    ╲r₂
+     *  p₁ ·     ┆     ╲
+     *  •     pdy┆      ╲
+     *   ╲       ┆       •c₂
+     *    ╲r₁    ┆   ·   ┆
+     *     ╲    ·┆       ┆cdy
+     *      •╌╌╌╌┴╌╌╌╌╌╌╌┘
+     *    c₁  pdx   cdx
+     *
+     * cdx = (c₂x - c₁x)
+     * cdy = (c₂y - c₁y)
+     *  dr =  r₂-r₁
+     * pdx =  px - c₁x
+     * pdy =  py - c₁y
+     *
+     * Note that cdx, cdy, and dr do not depend on point p at all, so can
+     * be pre-computed for the entire gradient. The simplifed equations
+     * are now:
+     *
+     * cos θ = (-cdx·t + pdx) / (dr·t + r₁)
+     * sin θ = (-cdy·t + pdy) / (dr·t + r₁)
+     *
+     * Finally, to get a single function of t and eliminate the last
+     * unknown θ, we use the identity sin²θ + cos²θ = 1. First, square
+     * each equation, (we knew a quadratic was coming since it must be
+     * possible to obtain two solutions in some cases):
+     *
+     * cos²θ = (cdx²t² - 2·cdx·pdx·t + pdx²) / (dr²·t² + 2·r₁·dr·t + r₁²)
+     * sin²θ = (cdy²t² - 2·cdy·pdy·t + pdy²) / (dr²·t² + 2·r₁·dr·t + r₁²)
+     *
+     * Then add both together, set the result equal to 1, and express as a
+     * standard quadratic equation in t of the form At² + Bt + C = 0
+     *
+     * (cdx² + cdy² - dr²)·t² - 2·(cdx·pdx + cdy·pdy + r₁·dr)·t + (pdx² + pdy² - r₁²) = 0
+     *
+     * In other words:
+     *
+     * A = cdx² + cdy² - dr²
+     * B = -2·(pdx·cdx + pdy·cdy + r₁·dr)
+     * C = pdx² + pdy² - r₁²
+     *
+     * And again, notice that A does not depend on p, so can be
+     * precomputed. From here we just use the quadratic formula to solve
+     * for t:
+     *
+     * t = (-2·B ± ⎷(B² - 4·A·C)) / 2·A
+     */
+
+    gradient_t *gradient = (gradient_t *)image;
+    source_image_t *source = (source_image_t *)image;
+    radial_gradient_t *radial = (radial_gradient_t *)image;
+    uint32_t *end = buffer + width;
+    pixman_gradient_walker_t walker;
+    pixman_bool_t affine = TRUE;
+    double cx = 1.;
+    double cy = 0.;
+    double cz = 0.;
+    double rx = x + 0.5;
+    double ry = y + 0.5;
+    double rz = 1.;
+
+    _pixman_gradient_walker_init (&walker, gradient, source->common.repeat);
+
+    if (source->common.transform)
+    {
+	pixman_vector_t v;
+	/* reference point is the center of the pixel */
+	v.vector[0] = pixman_int_to_fixed (x) + pixman_fixed_1 / 2;
+	v.vector[1] = pixman_int_to_fixed (y) + pixman_fixed_1 / 2;
+	v.vector[2] = pixman_fixed_1;
+	
+	if (!pixman_transform_point_3d (source->common.transform, &v))
+	    return;
+
+	cx = source->common.transform->matrix[0][0] / 65536.;
+	cy = source->common.transform->matrix[1][0] / 65536.;
+	cz = source->common.transform->matrix[2][0] / 65536.;
+	
+	rx = v.vector[0] / 65536.;
+	ry = v.vector[1] / 65536.;
+	rz = v.vector[2] / 65536.;
+
+	affine =
+	    source->common.transform->matrix[2][0] == 0 &&
+	    v.vector[2] == pixman_fixed_1;
+    }
+
+    if (affine)
+    {
+	/* When computing t over a scanline, we notice that some expressions
+	 * are constant so we can compute them just once. Given:
+	 *
+	 * t = (-2·B ± ⎷(B² - 4·A·C)) / 2·A
+	 *
+	 * where
+	 *
+	 * A = cdx² + cdy² - dr² [precomputed as radial->A]
+	 * B = -2·(pdx·cdx + pdy·cdy + r₁·dr)
+	 * C = pdx² + pdy² - r₁²
+	 *
+	 * Since we have an affine transformation, we know that (pdx, pdy)
+	 * increase linearly with each pixel,
+	 *
+	 * pdx = pdx₀ + n·cx,
+	 * pdy = pdy₀ + n·cy,
+	 *
+	 * we can then express B in terms of an linear increment along
+	 * the scanline:
+	 *
+	 * B = B₀ + n·cB, with
+	 * B₀ = -2·(pdx₀·cdx + pdy₀·cdy + r₁·dr) and
+	 * cB = -2·(cx·cdx + cy·cdy)
+	 *
+	 * Thus we can replace the full evaluation of B per-pixel (4 multiplies,
+	 * 2 additions) with a single addition.
+	 */
+	double r1   = radial->c1.radius / 65536.;
+	double r1sq = r1 * r1;
+	double pdx  = rx - radial->c1.x / 65536.;
+	double pdy  = ry - radial->c1.y / 65536.;
+	double A = radial->A;
+	double invA = -65536. / (2. * A);
+	double A4 = -4. * A;
+	double B  = -2. * (pdx*radial->cdx + pdy*radial->cdy + r1*radial->dr);
+	double cB = -2. *  (cx*radial->cdx +  cy*radial->cdy);
+	pixman_bool_t invert = A * radial->dr < 0;
+
+	while (buffer < end)
+	{
+	    if (!mask || *mask++ & mask_bits)
+	    {
+		pixman_fixed_48_16_t t;
+		double det = B * B + A4 * (pdx * pdx + pdy * pdy - r1sq);
+		if (det <= 0.)
+		    t = (pixman_fixed_48_16_t) (B * invA);
+		else if (invert)
+		    t = (pixman_fixed_48_16_t) ((B + sqrt (det)) * invA);
+		else
+		    t = (pixman_fixed_48_16_t) ((B - sqrt (det)) * invA);
+
+		*buffer = _pixman_gradient_walker_pixel (&walker, t);
+	    }
+	    ++buffer;
+
+	    pdx += cx;
+	    pdy += cy;
+	    B += cB;
+	}
+    }
+    else
+    {
+	/* projective */
+	while (buffer < end)
+	{
+	    if (!mask || *mask++ & mask_bits)
+	    {
+		double pdx, pdy;
+		double B, C;
+		double det;
+		double c1x = radial->c1.x / 65536.0;
+		double c1y = radial->c1.y / 65536.0;
+		double r1  = radial->c1.radius / 65536.0;
+		pixman_fixed_48_16_t t;
+		double x, y;
+
+		if (rz != 0)
+		{
+		    x = rx / rz;
+		    y = ry / rz;
+		}
+		else
+		{
+		    x = y = 0.;
+		}
+
+		pdx = x - c1x;
+		pdy = y - c1y;
+
+		B = -2 * (pdx * radial->cdx +
+			  pdy * radial->cdy +
+			  r1 * radial->dr);
+		C = (pdx * pdx + pdy * pdy - r1 * r1);
+
+		det = (B * B) - (4 * radial->A * C);
+		if (det < 0.0)
+		    det = 0.0;
+
+		if (radial->A * radial->dr < 0)
+		    t = (pixman_fixed_48_16_t) ((-B - sqrt (det)) / (2.0 * radial->A) * 65536);
+		else
+		    t = (pixman_fixed_48_16_t) ((-B + sqrt (det)) / (2.0 * radial->A) * 65536);
+
+		*buffer = _pixman_gradient_walker_pixel (&walker, t);
+	    }
+	    
+	    ++buffer;
+
+	    rx += cx;
+	    ry += cy;
+	    rz += cz;
+	}
+    }
+}
+
+static void
+radial_gradient_property_changed (pixman_image_t *image)
+{
+    image->common.get_scanline_32 = radial_gradient_get_scanline_32;
+    image->common.get_scanline_64 = _pixman_image_get_scanline_generic_64;
+}
+
+PIXMAN_EXPORT pixman_image_t *
+pixman_image_create_radial_gradient (pixman_point_fixed_t *        inner,
+                                     pixman_point_fixed_t *        outer,
+                                     pixman_fixed_t                inner_radius,
+                                     pixman_fixed_t                outer_radius,
+                                     const pixman_gradient_stop_t *stops,
+                                     int                           n_stops)
+{
+    pixman_image_t *image;
+    radial_gradient_t *radial;
+
+    return_val_if_fail (n_stops >= 2, NULL);
+
+    image = _pixman_image_allocate ();
+
+    if (!image)
+	return NULL;
+
+    radial = &image->radial;
+
+    if (!_pixman_init_gradient (&radial->common, stops, n_stops))
+    {
+	free (image);
+	return NULL;
+    }
+
+    image->type = RADIAL;
+
+    radial->c1.x = inner->x;
+    radial->c1.y = inner->y;
+    radial->c1.radius = inner_radius;
+    radial->c2.x = outer->x;
+    radial->c2.y = outer->y;
+    radial->c2.radius = outer_radius;
+    radial->cdx = pixman_fixed_to_double (radial->c2.x - radial->c1.x);
+    radial->cdy = pixman_fixed_to_double (radial->c2.y - radial->c1.y);
+    radial->dr = pixman_fixed_to_double (radial->c2.radius - radial->c1.radius);
+    radial->A = (radial->cdx * radial->cdx +
+		 radial->cdy * radial->cdy -
+		 radial->dr  * radial->dr);
+
+    image->common.property_changed = radial_gradient_property_changed;
+
+    return image;
+}
+
diff --git a/pixman/pixman/pixman-region.c b/pixman/pixman/pixman-region.c
new file mode 100644
index 000000000..a6a400581
--- /dev/null
+++ b/pixman/pixman/pixman-region.c
@@ -0,0 +1,2751 @@
+/*
+ * Copyright 1987, 1988, 1989, 1998  The Open Group
+ * 
+ * Permission to use, copy, modify, distribute, and sell this software and its
+ * documentation for any purpose is hereby granted without fee, provided that
+ * the above copyright notice appear in all copies and that both that
+ * copyright notice and this permission notice appear in supporting
+ * documentation.
+ * 
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ * 
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+ * OPEN GROUP BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+ * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ * 
+ * Except as contained in this notice, the name of The Open Group shall not be
+ * used in advertising or otherwise to promote the sale, use or other dealings
+ * in this Software without prior written authorization from The Open Group.
+ * 
+ * Copyright 1987, 1988, 1989 by
+ * Digital Equipment Corporation, Maynard, Massachusetts.
+ * 
+ *                    All Rights Reserved
+ * 
+ * Permission to use, copy, modify, and distribute this software and its
+ * documentation for any purpose and without fee is hereby granted,
+ * provided that the above copyright notice appear in all copies and that
+ * both that copyright notice and this permission notice appear in
+ * supporting documentation, and that the name of Digital not be
+ * used in advertising or publicity pertaining to distribution of the
+ * software without specific, written prior permission.
+ * 
+ * DIGITAL DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING
+ * ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL
+ * DIGITAL BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR
+ * ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,
+ * WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION,
+ * ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS
+ * SOFTWARE.
+ *
+ * Copyright © 1998 Keith Packard
+ *
+ * Permission to use, copy, modify, distribute, and sell this software and its
+ * documentation for any purpose is hereby granted without fee, provided that
+ * the above copyright notice appear in all copies and that both that
+ * copyright notice and this permission notice appear in supporting
+ * documentation, and that the name of Keith Packard not be used in
+ * advertising or publicity pertaining to distribution of the software without
+ * specific, written prior permission.  Keith Packard makes no
+ * representations about the suitability of this software for any purpose.  It
+ * is provided "as is" without express or implied warranty.
+ *
+ * KEITH PACKARD DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
+ * INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO
+ * EVENT SHALL KEITH PACKARD BE LIABLE FOR ANY SPECIAL, INDIRECT OR
+ * CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE,
+ * DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
+ * TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+ * PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include <stdlib.h>
+#include <limits.h>
+#include <string.h>
+#include <stdio.h>
+#include "pixman-private.h"
+
+#define PIXREGION_NIL(reg) ((reg)->data && !(reg)->data->numRects)
+/* not a region */
+#define PIXREGION_NAR(reg)      ((reg)->data == pixman_broken_data)
+#define PIXREGION_NUMRECTS(reg) ((reg)->data ? (reg)->data->numRects : 1)
+#define PIXREGION_SIZE(reg) ((reg)->data ? (reg)->data->size : 0)
+#define PIXREGION_RECTS(reg) \
+    ((reg)->data ? (box_type_t *)((reg)->data + 1) \
+     : &(reg)->extents)
+#define PIXREGION_BOXPTR(reg) ((box_type_t *)((reg)->data + 1))
+#define PIXREGION_BOX(reg, i) (&PIXREGION_BOXPTR (reg)[i])
+#define PIXREGION_TOP(reg) PIXREGION_BOX (reg, (reg)->data->numRects)
+#define PIXREGION_END(reg) PIXREGION_BOX (reg, (reg)->data->numRects - 1)
+
+#define GOOD_RECT(rect) ((rect)->x1 < (rect)->x2 && (rect)->y1 < (rect)->y2)
+#define BAD_RECT(rect) ((rect)->x1 > (rect)->x2 || (rect)->y1 > (rect)->y2)
+
+#ifdef DEBUG
+
+#define GOOD(reg)							\
+    do									\
+    {									\
+	if (!PREFIX (_selfcheck (reg)))					\
+	    _pixman_log_error (FUNC, "Malformed region " # reg);	\
+    } while (0)
+
+#else
+
+#define GOOD(reg)
+
+#endif
+
+static const box_type_t PREFIX (_empty_box_) = { 0, 0, 0, 0 };
+static const region_data_type_t PREFIX (_empty_data_) = { 0, 0 };
+static const region_data_type_t PREFIX (_broken_data_) = { 0, 0 };
+
+static box_type_t *pixman_region_empty_box =
+    (box_type_t *)&PREFIX (_empty_box_);
+static region_data_type_t *pixman_region_empty_data =
+    (region_data_type_t *)&PREFIX (_empty_data_);
+static region_data_type_t *pixman_broken_data =
+    (region_data_type_t *)&PREFIX (_broken_data_);
+
+static pixman_bool_t
+pixman_break (region_type_t *region);
+
+/*
+ * The functions in this file implement the Region abstraction used extensively
+ * throughout the X11 sample server. A Region is simply a set of disjoint
+ * (non-overlapping) rectangles, plus an "extent" rectangle which is the
+ * smallest single rectangle that contains all the non-overlapping rectangles.
+ *
+ * A Region is implemented as a "y-x-banded" array of rectangles.  This array
+ * imposes two degrees of order.  First, all rectangles are sorted by top side
+ * y coordinate first (y1), and then by left side x coordinate (x1).
+ *
+ * Furthermore, the rectangles are grouped into "bands".  Each rectangle in a
+ * band has the same top y coordinate (y1), and each has the same bottom y
+ * coordinate (y2).  Thus all rectangles in a band differ only in their left
+ * and right side (x1 and x2).  Bands are implicit in the array of rectangles:
+ * there is no separate list of band start pointers.
+ *
+ * The y-x band representation does not minimize rectangles.  In particular,
+ * if a rectangle vertically crosses a band (the rectangle has scanlines in
+ * the y1 to y2 area spanned by the band), then the rectangle may be broken
+ * down into two or more smaller rectangles stacked one atop the other.
+ *
+ *  -----------				    -----------
+ *  |         |				    |         |		    band 0
+ *  |         |  --------		    -----------  --------
+ *  |         |  |      |  in y-x banded    |         |  |      |   band 1
+ *  |         |  |      |  form is	    |         |  |      |
+ *  -----------  |      |		    -----------  --------
+ *               |      |				 |      |   band 2
+ *               --------				 --------
+ *
+ * An added constraint on the rectangles is that they must cover as much
+ * horizontal area as possible: no two rectangles within a band are allowed
+ * to touch.
+ *
+ * Whenever possible, bands will be merged together to cover a greater vertical
+ * distance (and thus reduce the number of rectangles). Two bands can be merged
+ * only if the bottom of one touches the top of the other and they have
+ * rectangles in the same places (of the same width, of course).
+ *
+ * Adam de Boor wrote most of the original region code.  Joel McCormack
+ * substantially modified or rewrote most of the core arithmetic routines, and
+ * added pixman_region_validate in order to support several speed improvements
+ * to pixman_region_validate_tree.  Bob Scheifler changed the representation
+ * to be more compact when empty or a single rectangle, and did a bunch of
+ * gratuitous reformatting. Carl Worth did further gratuitous reformatting
+ * while re-merging the server and client region code into libpixregion.
+ * Soren Sandmann did even more gratuitous reformatting.
+ */
+
+/*  true iff two Boxes overlap */
+#define EXTENTCHECK(r1, r2)	   \
+    (!( ((r1)->x2 <= (r2)->x1)  || \
+        ((r1)->x1 >= (r2)->x2)  || \
+        ((r1)->y2 <= (r2)->y1)  || \
+        ((r1)->y1 >= (r2)->y2) ) )
+
+/* true iff (x,y) is in Box */
+#define INBOX(r, x, y)	\
+    ( ((r)->x2 >  x) && \
+      ((r)->x1 <= x) && \
+      ((r)->y2 >  y) && \
+      ((r)->y1 <= y) )
+
+/* true iff Box r1 contains Box r2 */
+#define SUBSUMES(r1, r2)	\
+    ( ((r1)->x1 <= (r2)->x1) && \
+      ((r1)->x2 >= (r2)->x2) && \
+      ((r1)->y1 <= (r2)->y1) && \
+      ((r1)->y2 >= (r2)->y2) )
+
+static size_t
+PIXREGION_SZOF (size_t n)
+{
+    size_t size = n * sizeof(box_type_t);
+    
+    if (n > UINT32_MAX / sizeof(box_type_t))
+	return 0;
+
+    if (sizeof(region_data_type_t) > UINT32_MAX - size)
+	return 0;
+
+    return size + sizeof(region_data_type_t);
+}
+
+static void *
+alloc_data (size_t n)
+{
+    size_t sz = PIXREGION_SZOF (n);
+
+    if (!sz)
+	return NULL;
+
+    return malloc (sz);
+}
+
+#define FREE_DATA(reg) if ((reg)->data && (reg)->data->size) free ((reg)->data)
+
+#define RECTALLOC_BAIL(region, n, bail)					\
+    do									\
+    {									\
+	if (!(region)->data ||						\
+	    (((region)->data->numRects + (n)) > (region)->data->size))	\
+	{								\
+	    if (!pixman_rect_alloc (region, n))				\
+		goto bail;						\
+	}								\
+    } while (0)
+
+#define RECTALLOC(region, n)						\
+    do									\
+    {									\
+	if (!(region)->data ||						\
+	    (((region)->data->numRects + (n)) > (region)->data->size))	\
+	{								\
+	    if (!pixman_rect_alloc (region, n)) {			\
+		return FALSE;						\
+	    }								\
+	}								\
+    } while (0)
+
+#define ADDRECT(next_rect, nx1, ny1, nx2, ny2)      \
+    do						    \
+    {						    \
+	next_rect->x1 = nx1;                        \
+	next_rect->y1 = ny1;                        \
+	next_rect->x2 = nx2;                        \
+	next_rect->y2 = ny2;                        \
+	next_rect++;                                \
+    }						    \
+    while (0)
+
+#define NEWRECT(region, next_rect, nx1, ny1, nx2, ny2)			\
+    do									\
+    {									\
+	if (!(region)->data ||						\
+	    ((region)->data->numRects == (region)->data->size))		\
+	{								\
+	    if (!pixman_rect_alloc (region, 1))				\
+		return FALSE;						\
+	    next_rect = PIXREGION_TOP (region);				\
+	}								\
+	ADDRECT (next_rect, nx1, ny1, nx2, ny2);			\
+	region->data->numRects++;					\
+	critical_if_fail (region->data->numRects <= region->data->size);		\
+    } while (0)
+
+#define DOWNSIZE(reg, numRects)						\
+    do									\
+    {									\
+	if (((numRects) < ((reg)->data->size >> 1)) &&			\
+	    ((reg)->data->size > 50))					\
+	{								\
+	    region_data_type_t * new_data;				\
+	    size_t data_size = PIXREGION_SZOF (numRects);		\
+									\
+	    if (!data_size)						\
+	    {								\
+		new_data = NULL;					\
+	    }								\
+	    else							\
+	    {								\
+		new_data = (region_data_type_t *)			\
+		    realloc ((reg)->data, data_size);			\
+	    }								\
+									\
+	    if (new_data)						\
+	    {								\
+		new_data->size = (numRects);				\
+		(reg)->data = new_data;					\
+	    }								\
+	}								\
+    } while (0)
+
+PIXMAN_EXPORT pixman_bool_t
+PREFIX (_equal) (region_type_t *reg1, region_type_t *reg2)
+{
+    int i;
+    box_type_t *rects1;
+    box_type_t *rects2;
+
+    if (reg1->extents.x1 != reg2->extents.x1)
+	return FALSE;
+    
+    if (reg1->extents.x2 != reg2->extents.x2)
+	return FALSE;
+    
+    if (reg1->extents.y1 != reg2->extents.y1)
+	return FALSE;
+    
+    if (reg1->extents.y2 != reg2->extents.y2)
+	return FALSE;
+    
+    if (PIXREGION_NUMRECTS (reg1) != PIXREGION_NUMRECTS (reg2))
+	return FALSE;
+
+    rects1 = PIXREGION_RECTS (reg1);
+    rects2 = PIXREGION_RECTS (reg2);
+    
+    for (i = 0; i != PIXREGION_NUMRECTS (reg1); i++)
+    {
+	if (rects1[i].x1 != rects2[i].x1)
+	    return FALSE;
+	
+	if (rects1[i].x2 != rects2[i].x2)
+	    return FALSE;
+	
+	if (rects1[i].y1 != rects2[i].y1)
+	    return FALSE;
+	
+	if (rects1[i].y2 != rects2[i].y2)
+	    return FALSE;
+    }
+
+    return TRUE;
+}
+
+int
+PREFIX (_print) (region_type_t *rgn)
+{
+    int num, size;
+    int i;
+    box_type_t * rects;
+
+    num = PIXREGION_NUMRECTS (rgn);
+    size = PIXREGION_SIZE (rgn);
+    rects = PIXREGION_RECTS (rgn);
+
+    fprintf (stderr, "num: %d size: %d\n", num, size);
+    fprintf (stderr, "extents: %d %d %d %d\n",
+             rgn->extents.x1,
+	     rgn->extents.y1,
+	     rgn->extents.x2,
+	     rgn->extents.y2);
+    
+    for (i = 0; i < num; i++)
+    {
+	fprintf (stderr, "%d %d %d %d \n",
+	         rects[i].x1, rects[i].y1, rects[i].x2, rects[i].y2);
+    }
+    
+    fprintf (stderr, "\n");
+
+    return(num);
+}
+
+
+PIXMAN_EXPORT void
+PREFIX (_init) (region_type_t *region)
+{
+    region->extents = *pixman_region_empty_box;
+    region->data = pixman_region_empty_data;
+}
+
+PIXMAN_EXPORT void
+PREFIX (_init_rect) (region_type_t *	region,
+                     int		x,
+		     int		y,
+		     unsigned int	width,
+		     unsigned int	height)
+{
+    region->extents.x1 = x;
+    region->extents.y1 = y;
+    region->extents.x2 = x + width;
+    region->extents.y2 = y + height;
+
+    if (!GOOD_RECT (&region->extents))
+    {
+        if (BAD_RECT (&region->extents))
+            _pixman_log_error (FUNC, "Invalid rectangle passed");
+        PREFIX (_init) (region);
+        return;
+    }
+
+    region->data = NULL;
+}
+
+PIXMAN_EXPORT void
+PREFIX (_init_with_extents) (region_type_t *region, box_type_t *extents)
+{
+    if (!GOOD_RECT (extents))
+    {
+        if (BAD_RECT (extents))
+            _pixman_log_error (FUNC, "Invalid rectangle passed");
+        PREFIX (_init) (region);
+        return;
+    }
+    region->extents = *extents;
+
+    region->data = NULL;
+}
+
+PIXMAN_EXPORT void
+PREFIX (_fini) (region_type_t *region)
+{
+    GOOD (region);
+    FREE_DATA (region);
+}
+
+PIXMAN_EXPORT int
+PREFIX (_n_rects) (region_type_t *region)
+{
+    return PIXREGION_NUMRECTS (region);
+}
+
+PIXMAN_EXPORT box_type_t *
+PREFIX (_rectangles) (region_type_t *region,
+                      int               *n_rects)
+{
+    if (n_rects)
+	*n_rects = PIXREGION_NUMRECTS (region);
+
+    return PIXREGION_RECTS (region);
+}
+
+static pixman_bool_t
+pixman_break (region_type_t *region)
+{
+    FREE_DATA (region);
+
+    region->extents = *pixman_region_empty_box;
+    region->data = pixman_broken_data;
+
+    return FALSE;
+}
+
+static pixman_bool_t
+pixman_rect_alloc (region_type_t * region,
+                   int             n)
+{
+    region_data_type_t *data;
+
+    if (!region->data)
+    {
+	n++;
+	region->data = alloc_data (n);
+
+	if (!region->data)
+	    return pixman_break (region);
+
+	region->data->numRects = 1;
+	*PIXREGION_BOXPTR (region) = region->extents;
+    }
+    else if (!region->data->size)
+    {
+	region->data = alloc_data (n);
+
+	if (!region->data)
+	    return pixman_break (region);
+
+	region->data->numRects = 0;
+    }
+    else
+    {
+	size_t data_size;
+
+	if (n == 1)
+	{
+	    n = region->data->numRects;
+	    if (n > 500) /* XXX pick numbers out of a hat */
+		n = 250;
+	}
+
+	n += region->data->numRects;
+	data_size = PIXREGION_SZOF (n);
+
+	if (!data_size)
+	{
+	    data = NULL;
+	}
+	else
+	{
+	    data = (region_data_type_t *)
+		realloc (region->data, PIXREGION_SZOF (n));
+	}
+	
+	if (!data)
+	    return pixman_break (region);
+	
+	region->data = data;
+    }
+    
+    region->data->size = n;
+
+    return TRUE;
+}
+
+PIXMAN_EXPORT pixman_bool_t
+PREFIX (_copy) (region_type_t *dst, region_type_t *src)
+{
+    GOOD (dst);
+    GOOD (src);
+
+    if (dst == src)
+	return TRUE;
+    
+    dst->extents = src->extents;
+
+    if (!src->data || !src->data->size)
+    {
+	FREE_DATA (dst);
+	dst->data = src->data;
+	return TRUE;
+    }
+    
+    if (!dst->data || (dst->data->size < src->data->numRects))
+    {
+	FREE_DATA (dst);
+
+	dst->data = alloc_data (src->data->numRects);
+
+	if (!dst->data)
+	    return pixman_break (dst);
+
+	dst->data->size = src->data->numRects;
+    }
+
+    dst->data->numRects = src->data->numRects;
+
+    memmove ((char *)PIXREGION_BOXPTR (dst), (char *)PIXREGION_BOXPTR (src),
+             dst->data->numRects * sizeof(box_type_t));
+
+    return TRUE;
+}
+
+/*======================================================================
+ *	    Generic Region Operator
+ *====================================================================*/
+
+/*-
+ *-----------------------------------------------------------------------
+ * pixman_coalesce --
+ *	Attempt to merge the boxes in the current band with those in the
+ *	previous one.  We are guaranteed that the current band extends to
+ *      the end of the rects array.  Used only by pixman_op.
+ *
+ * Results:
+ *	The new index for the previous band.
+ *
+ * Side Effects:
+ *	If coalescing takes place:
+ *	    - rectangles in the previous band will have their y2 fields
+ *	      altered.
+ *	    - region->data->numRects will be decreased.
+ *
+ *-----------------------------------------------------------------------
+ */
+static inline int
+pixman_coalesce (region_type_t * region,      /* Region to coalesce		 */
+		 int             prev_start,  /* Index of start of previous band */
+		 int             cur_start)   /* Index of start of current band  */
+{
+    box_type_t *prev_box;       /* Current box in previous band	     */
+    box_type_t *cur_box;        /* Current box in current band       */
+    int numRects;               /* Number rectangles in both bands   */
+    int y2;                     /* Bottom of current band	     */
+
+    /*
+     * Figure out how many rectangles are in the band.
+     */
+    numRects = cur_start - prev_start;
+    critical_if_fail (numRects == region->data->numRects - cur_start);
+
+    if (!numRects) return cur_start;
+
+    /*
+     * The bands may only be coalesced if the bottom of the previous
+     * matches the top scanline of the current.
+     */
+    prev_box = PIXREGION_BOX (region, prev_start);
+    cur_box = PIXREGION_BOX (region, cur_start);
+    if (prev_box->y2 != cur_box->y1) return cur_start;
+
+    /*
+     * Make sure the bands have boxes in the same places. This
+     * assumes that boxes have been added in such a way that they
+     * cover the most area possible. I.e. two boxes in a band must
+     * have some horizontal space between them.
+     */
+    y2 = cur_box->y2;
+
+    do
+    {
+	if ((prev_box->x1 != cur_box->x1) || (prev_box->x2 != cur_box->x2))
+	    return (cur_start);
+	
+	prev_box++;
+	cur_box++;
+	numRects--;
+    }
+    while (numRects);
+
+    /*
+     * The bands may be merged, so set the bottom y of each box
+     * in the previous band to the bottom y of the current band.
+     */
+    numRects = cur_start - prev_start;
+    region->data->numRects -= numRects;
+
+    do
+    {
+	prev_box--;
+	prev_box->y2 = y2;
+	numRects--;
+    }
+    while (numRects);
+
+    return prev_start;
+}
+
+/* Quicky macro to avoid trivial reject procedure calls to pixman_coalesce */
+
+#define COALESCE(new_reg, prev_band, cur_band)                          \
+    do									\
+    {									\
+	if (cur_band - prev_band == new_reg->data->numRects - cur_band)	\
+	    prev_band = pixman_coalesce (new_reg, prev_band, cur_band);	\
+	else								\
+	    prev_band = cur_band;					\
+    } while (0)
+
+/*-
+ *-----------------------------------------------------------------------
+ * pixman_region_append_non_o --
+ *	Handle a non-overlapping band for the union and subtract operations.
+ *      Just adds the (top/bottom-clipped) rectangles into the region.
+ *      Doesn't have to check for subsumption or anything.
+ *
+ * Results:
+ *	None.
+ *
+ * Side Effects:
+ *	region->data->numRects is incremented and the rectangles overwritten
+ *	with the rectangles we're passed.
+ *
+ *-----------------------------------------------------------------------
+ */
+static inline pixman_bool_t
+pixman_region_append_non_o (region_type_t * region,
+			    box_type_t *    r,
+			    box_type_t *    r_end,
+			    int             y1,
+			    int             y2)
+{
+    box_type_t *next_rect;
+    int new_rects;
+
+    new_rects = r_end - r;
+
+    critical_if_fail (y1 < y2);
+    critical_if_fail (new_rects != 0);
+
+    /* Make sure we have enough space for all rectangles to be added */
+    RECTALLOC (region, new_rects);
+    next_rect = PIXREGION_TOP (region);
+    region->data->numRects += new_rects;
+
+    do
+    {
+	critical_if_fail (r->x1 < r->x2);
+	ADDRECT (next_rect, r->x1, y1, r->x2, y2);
+	r++;
+    }
+    while (r != r_end);
+
+    return TRUE;
+}
+
+#define FIND_BAND(r, r_band_end, r_end, ry1)			     \
+    do								     \
+    {								     \
+	ry1 = r->y1;						     \
+	r_band_end = r + 1;					     \
+	while ((r_band_end != r_end) && (r_band_end->y1 == ry1)) {   \
+	    r_band_end++;					     \
+	}							     \
+    } while (0)
+
+#define APPEND_REGIONS(new_reg, r, r_end)				\
+    do									\
+    {									\
+	int new_rects;							\
+	if ((new_rects = r_end - r)) {					\
+	    RECTALLOC_BAIL (new_reg, new_rects, bail);			\
+	    memmove ((char *)PIXREGION_TOP (new_reg), (char *)r,	\
+		     new_rects * sizeof(box_type_t));			\
+	    new_reg->data->numRects += new_rects;			\
+	}								\
+    } while (0)
+
+/*-
+ *-----------------------------------------------------------------------
+ * pixman_op --
+ *	Apply an operation to two regions. Called by pixman_region_union, pixman_region_inverse,
+ *	pixman_region_subtract, pixman_region_intersect....  Both regions MUST have at least one
+ *      rectangle, and cannot be the same object.
+ *
+ * Results:
+ *	TRUE if successful.
+ *
+ * Side Effects:
+ *	The new region is overwritten.
+ *	overlap set to TRUE if overlap_func ever returns TRUE.
+ *
+ * Notes:
+ *	The idea behind this function is to view the two regions as sets.
+ *	Together they cover a rectangle of area that this function divides
+ *	into horizontal bands where points are covered only by one region
+ *	or by both. For the first case, the non_overlap_func is called with
+ *	each the band and the band's upper and lower extents. For the
+ *	second, the overlap_func is called to process the entire band. It
+ *	is responsible for clipping the rectangles in the band, though
+ *	this function provides the boundaries.
+ *	At the end of each band, the new region is coalesced, if possible,
+ *	to reduce the number of rectangles in the region.
+ *
+ *-----------------------------------------------------------------------
+ */
+
+typedef pixman_bool_t (*overlap_proc_ptr) (region_type_t *region,
+					   box_type_t *   r1,
+					   box_type_t *   r1_end,
+					   box_type_t *   r2,
+					   box_type_t *   r2_end,
+					   int            y1,
+					   int            y2,
+					   int *          overlap);
+
+static pixman_bool_t
+pixman_op (region_type_t *  new_reg,               /* Place to store result	    */
+	   region_type_t *  reg1,                  /* First region in operation     */
+	   region_type_t *  reg2,                  /* 2d region in operation        */
+	   overlap_proc_ptr overlap_func,          /* Function to call for over-
+						    * lapping bands		    */
+	   int              append_non1,           /* Append non-overlapping bands  
+						    * in region 1 ?
+						    */
+	   int              append_non2,           /* Append non-overlapping bands
+						    * in region 2 ?
+						    */
+	   int *            overlap)
+{
+    box_type_t *r1;                 /* Pointer into first region     */
+    box_type_t *r2;                 /* Pointer into 2d region	     */
+    box_type_t *r1_end;             /* End of 1st region	     */
+    box_type_t *r2_end;             /* End of 2d region		     */
+    int ybot;                       /* Bottom of intersection	     */
+    int ytop;                       /* Top of intersection	     */
+    region_data_type_t *old_data;   /* Old data for new_reg	     */
+    int prev_band;                  /* Index of start of
+				     * previous band in new_reg       */
+    int cur_band;                   /* Index of start of current
+				     * band in new_reg		     */
+    box_type_t * r1_band_end;       /* End of current band in r1     */
+    box_type_t * r2_band_end;       /* End of current band in r2     */
+    int top;                        /* Top of non-overlapping band   */
+    int bot;                        /* Bottom of non-overlapping band*/
+    int r1y1;                       /* Temps for r1->y1 and r2->y1   */
+    int r2y1;
+    int new_size;
+    int numRects;
+
+    /*
+     * Break any region computed from a broken region
+     */
+    if (PIXREGION_NAR (reg1) || PIXREGION_NAR (reg2))
+	return pixman_break (new_reg);
+
+    /*
+     * Initialization:
+     *	set r1, r2, r1_end and r2_end appropriately, save the rectangles
+     * of the destination region until the end in case it's one of
+     * the two source regions, then mark the "new" region empty, allocating
+     * another array of rectangles for it to use.
+     */
+
+    r1 = PIXREGION_RECTS (reg1);
+    new_size = PIXREGION_NUMRECTS (reg1);
+    r1_end = r1 + new_size;
+
+    numRects = PIXREGION_NUMRECTS (reg2);
+    r2 = PIXREGION_RECTS (reg2);
+    r2_end = r2 + numRects;
+    
+    critical_if_fail (r1 != r1_end);
+    critical_if_fail (r2 != r2_end);
+
+    old_data = (region_data_type_t *)NULL;
+
+    if (((new_reg == reg1) && (new_size > 1)) ||
+        ((new_reg == reg2) && (numRects > 1)))
+    {
+        old_data = new_reg->data;
+        new_reg->data = pixman_region_empty_data;
+    }
+
+    /* guess at new size */
+    if (numRects > new_size)
+	new_size = numRects;
+
+    new_size <<= 1;
+
+    if (!new_reg->data)
+	new_reg->data = pixman_region_empty_data;
+    else if (new_reg->data->size)
+	new_reg->data->numRects = 0;
+
+    if (new_size > new_reg->data->size)
+    {
+        if (!pixman_rect_alloc (new_reg, new_size))
+        {
+            if (old_data)
+		free (old_data);
+            return FALSE;
+	}
+    }
+
+    /*
+     * Initialize ybot.
+     * In the upcoming loop, ybot and ytop serve different functions depending
+     * on whether the band being handled is an overlapping or non-overlapping
+     * band.
+     *  In the case of a non-overlapping band (only one of the regions
+     * has points in the band), ybot is the bottom of the most recent
+     * intersection and thus clips the top of the rectangles in that band.
+     * ytop is the top of the next intersection between the two regions and
+     * serves to clip the bottom of the rectangles in the current band.
+     *	For an overlapping band (where the two regions intersect), ytop clips
+     * the top of the rectangles of both regions and ybot clips the bottoms.
+     */
+
+    ybot = MIN (r1->y1, r2->y1);
+
+    /*
+     * prev_band serves to mark the start of the previous band so rectangles
+     * can be coalesced into larger rectangles. qv. pixman_coalesce, above.
+     * In the beginning, there is no previous band, so prev_band == cur_band
+     * (cur_band is set later on, of course, but the first band will always
+     * start at index 0). prev_band and cur_band must be indices because of
+     * the possible expansion, and resultant moving, of the new region's
+     * array of rectangles.
+     */
+    prev_band = 0;
+
+    do
+    {
+        /*
+	 * This algorithm proceeds one source-band (as opposed to a
+	 * destination band, which is determined by where the two regions
+	 * intersect) at a time. r1_band_end and r2_band_end serve to mark the
+	 * rectangle after the last one in the current band for their
+	 * respective regions.
+	 */
+        critical_if_fail (r1 != r1_end);
+        critical_if_fail (r2 != r2_end);
+
+        FIND_BAND (r1, r1_band_end, r1_end, r1y1);
+        FIND_BAND (r2, r2_band_end, r2_end, r2y1);
+
+        /*
+	 * First handle the band that doesn't intersect, if any.
+	 *
+	 * Note that attention is restricted to one band in the
+	 * non-intersecting region at once, so if a region has n
+	 * bands between the current position and the next place it overlaps
+	 * the other, this entire loop will be passed through n times.
+	 */
+        if (r1y1 < r2y1)
+        {
+            if (append_non1)
+            {
+                top = MAX (r1y1, ybot);
+                bot = MIN (r1->y2, r2y1);
+                if (top != bot)
+                {
+                    cur_band = new_reg->data->numRects;
+                    if (!pixman_region_append_non_o (new_reg, r1, r1_band_end, top, bot))
+			goto bail;
+                    COALESCE (new_reg, prev_band, cur_band);
+		}
+	    }
+            ytop = r2y1;
+	}
+        else if (r2y1 < r1y1)
+        {
+            if (append_non2)
+            {
+                top = MAX (r2y1, ybot);
+                bot = MIN (r2->y2, r1y1);
+		
+                if (top != bot)
+                {
+                    cur_band = new_reg->data->numRects;
+
+                    if (!pixman_region_append_non_o (new_reg, r2, r2_band_end, top, bot))
+			goto bail;
+
+                    COALESCE (new_reg, prev_band, cur_band);
+		}
+	    }
+            ytop = r1y1;
+	}
+        else
+        {
+            ytop = r1y1;
+	}
+
+        /*
+	 * Now see if we've hit an intersecting band. The two bands only
+	 * intersect if ybot > ytop
+	 */
+        ybot = MIN (r1->y2, r2->y2);
+        if (ybot > ytop)
+        {
+            cur_band = new_reg->data->numRects;
+
+            if (!(*overlap_func)(new_reg,
+                                 r1, r1_band_end,
+                                 r2, r2_band_end,
+                                 ytop, ybot,
+                                 overlap))
+	    {
+		goto bail;
+	    }
+	    
+            COALESCE (new_reg, prev_band, cur_band);
+	}
+
+        /*
+	 * If we've finished with a band (y2 == ybot) we skip forward
+	 * in the region to the next band.
+	 */
+        if (r1->y2 == ybot)
+	    r1 = r1_band_end;
+
+        if (r2->y2 == ybot)
+	    r2 = r2_band_end;
+
+    }
+    while (r1 != r1_end && r2 != r2_end);
+
+    /*
+     * Deal with whichever region (if any) still has rectangles left.
+     *
+     * We only need to worry about banding and coalescing for the very first
+     * band left.  After that, we can just group all remaining boxes,
+     * regardless of how many bands, into one final append to the list.
+     */
+
+    if ((r1 != r1_end) && append_non1)
+    {
+        /* Do first non_overlap1Func call, which may be able to coalesce */
+        FIND_BAND (r1, r1_band_end, r1_end, r1y1);
+	
+        cur_band = new_reg->data->numRects;
+	
+        if (!pixman_region_append_non_o (new_reg,
+                                         r1, r1_band_end,
+                                         MAX (r1y1, ybot), r1->y2))
+	{
+	    goto bail;
+	}
+	
+        COALESCE (new_reg, prev_band, cur_band);
+
+        /* Just append the rest of the boxes  */
+        APPEND_REGIONS (new_reg, r1_band_end, r1_end);
+    }
+    else if ((r2 != r2_end) && append_non2)
+    {
+        /* Do first non_overlap2Func call, which may be able to coalesce */
+        FIND_BAND (r2, r2_band_end, r2_end, r2y1);
+
+	cur_band = new_reg->data->numRects;
+
+        if (!pixman_region_append_non_o (new_reg,
+                                         r2, r2_band_end,
+                                         MAX (r2y1, ybot), r2->y2))
+	{
+	    goto bail;
+	}
+
+        COALESCE (new_reg, prev_band, cur_band);
+
+        /* Append rest of boxes */
+        APPEND_REGIONS (new_reg, r2_band_end, r2_end);
+    }
+
+    if (old_data)
+	free (old_data);
+
+    if (!(numRects = new_reg->data->numRects))
+    {
+        FREE_DATA (new_reg);
+        new_reg->data = pixman_region_empty_data;
+    }
+    else if (numRects == 1)
+    {
+        new_reg->extents = *PIXREGION_BOXPTR (new_reg);
+        FREE_DATA (new_reg);
+        new_reg->data = (region_data_type_t *)NULL;
+    }
+    else
+    {
+        DOWNSIZE (new_reg, numRects);
+    }
+
+    return TRUE;
+
+bail:
+    if (old_data)
+	free (old_data);
+
+    return pixman_break (new_reg);
+}
+
+/*-
+ *-----------------------------------------------------------------------
+ * pixman_set_extents --
+ *	Reset the extents of a region to what they should be. Called by
+ *	pixman_region_subtract and pixman_region_intersect as they can't
+ *      figure it out along the way or do so easily, as pixman_region_union can.
+ *
+ * Results:
+ *	None.
+ *
+ * Side Effects:
+ *	The region's 'extents' structure is overwritten.
+ *
+ *-----------------------------------------------------------------------
+ */
+static void
+pixman_set_extents (region_type_t *region)
+{
+    box_type_t *box, *box_end;
+
+    if (!region->data)
+	return;
+
+    if (!region->data->size)
+    {
+        region->extents.x2 = region->extents.x1;
+        region->extents.y2 = region->extents.y1;
+        return;
+    }
+
+    box = PIXREGION_BOXPTR (region);
+    box_end = PIXREGION_END (region);
+
+    /*
+     * Since box is the first rectangle in the region, it must have the
+     * smallest y1 and since box_end is the last rectangle in the region,
+     * it must have the largest y2, because of banding. Initialize x1 and
+     * x2 from  box and box_end, resp., as good things to initialize them
+     * to...
+     */
+    region->extents.x1 = box->x1;
+    region->extents.y1 = box->y1;
+    region->extents.x2 = box_end->x2;
+    region->extents.y2 = box_end->y2;
+
+    critical_if_fail (region->extents.y1 < region->extents.y2);
+
+    while (box <= box_end)
+    {
+        if (box->x1 < region->extents.x1)
+	    region->extents.x1 = box->x1;
+        if (box->x2 > region->extents.x2)
+	    region->extents.x2 = box->x2;
+        box++;
+    }
+
+    critical_if_fail (region->extents.x1 < region->extents.x2);
+}
+
+/*======================================================================
+ *	    Region Intersection
+ *====================================================================*/
+/*-
+ *-----------------------------------------------------------------------
+ * pixman_region_intersect_o --
+ *	Handle an overlapping band for pixman_region_intersect.
+ *
+ * Results:
+ *	TRUE if successful.
+ *
+ * Side Effects:
+ *	Rectangles may be added to the region.
+ *
+ *-----------------------------------------------------------------------
+ */
+/*ARGSUSED*/
+static pixman_bool_t
+pixman_region_intersect_o (region_type_t *region,
+                           box_type_t *   r1,
+                           box_type_t *   r1_end,
+                           box_type_t *   r2,
+                           box_type_t *   r2_end,
+                           int            y1,
+                           int            y2,
+                           int *          overlap)
+{
+    int x1;
+    int x2;
+    box_type_t *        next_rect;
+
+    next_rect = PIXREGION_TOP (region);
+
+    critical_if_fail (y1 < y2);
+    critical_if_fail (r1 != r1_end && r2 != r2_end);
+
+    do
+    {
+        x1 = MAX (r1->x1, r2->x1);
+        x2 = MIN (r1->x2, r2->x2);
+
+        /*
+	 * If there's any overlap between the two rectangles, add that
+	 * overlap to the new region.
+	 */
+        if (x1 < x2)
+	    NEWRECT (region, next_rect, x1, y1, x2, y2);
+
+        /*
+	 * Advance the pointer(s) with the leftmost right side, since the next
+	 * rectangle on that list may still overlap the other region's
+	 * current rectangle.
+	 */
+        if (r1->x2 == x2)
+        {
+            r1++;
+	}
+        if (r2->x2 == x2)
+        {
+            r2++;
+	}
+    }
+    while ((r1 != r1_end) && (r2 != r2_end));
+
+    return TRUE;
+}
+
+PIXMAN_EXPORT pixman_bool_t
+PREFIX (_intersect) (region_type_t *     new_reg,
+                     region_type_t *        reg1,
+                     region_type_t *        reg2)
+{
+    GOOD (reg1);
+    GOOD (reg2);
+    GOOD (new_reg);
+
+    /* check for trivial reject */
+    if (PIXREGION_NIL (reg1) || PIXREGION_NIL (reg2) ||
+        !EXTENTCHECK (&reg1->extents, &reg2->extents))
+    {
+        /* Covers about 20% of all cases */
+        FREE_DATA (new_reg);
+        new_reg->extents.x2 = new_reg->extents.x1;
+        new_reg->extents.y2 = new_reg->extents.y1;
+        if (PIXREGION_NAR (reg1) || PIXREGION_NAR (reg2))
+        {
+            new_reg->data = pixman_broken_data;
+            return FALSE;
+	}
+        else
+	{
+	    new_reg->data = pixman_region_empty_data;
+	}
+    }
+    else if (!reg1->data && !reg2->data)
+    {
+        /* Covers about 80% of cases that aren't trivially rejected */
+        new_reg->extents.x1 = MAX (reg1->extents.x1, reg2->extents.x1);
+        new_reg->extents.y1 = MAX (reg1->extents.y1, reg2->extents.y1);
+        new_reg->extents.x2 = MIN (reg1->extents.x2, reg2->extents.x2);
+        new_reg->extents.y2 = MIN (reg1->extents.y2, reg2->extents.y2);
+
+        FREE_DATA (new_reg);
+
+	new_reg->data = (region_data_type_t *)NULL;
+    }
+    else if (!reg2->data && SUBSUMES (&reg2->extents, &reg1->extents))
+    {
+        return PREFIX (_copy) (new_reg, reg1);
+    }
+    else if (!reg1->data && SUBSUMES (&reg1->extents, &reg2->extents))
+    {
+        return PREFIX (_copy) (new_reg, reg2);
+    }
+    else if (reg1 == reg2)
+    {
+        return PREFIX (_copy) (new_reg, reg1);
+    }
+    else
+    {
+        /* General purpose intersection */
+        int overlap; /* result ignored */
+
+        if (!pixman_op (new_reg, reg1, reg2, pixman_region_intersect_o, FALSE, FALSE,
+                        &overlap))
+	{
+	    return FALSE;
+	}
+	
+        pixman_set_extents (new_reg);
+    }
+
+    GOOD (new_reg);
+    return(TRUE);
+}
+
+#define MERGERECT(r)							\
+    do									\
+    {									\
+        if (r->x1 <= x2)						\
+	{								\
+            /* Merge with current rectangle */				\
+            if (r->x1 < x2)						\
+		*overlap = TRUE;					\
+									\
+            if (x2 < r->x2)						\
+		x2 = r->x2;						\
+	}								\
+	else								\
+	{								\
+            /* Add current rectangle, start new one */			\
+            NEWRECT (region, next_rect, x1, y1, x2, y2);		\
+            x1 = r->x1;							\
+            x2 = r->x2;							\
+	}								\
+        r++;								\
+    } while (0)
+
+/*======================================================================
+ *	    Region Union
+ *====================================================================*/
+
+/*-
+ *-----------------------------------------------------------------------
+ * pixman_region_union_o --
+ *	Handle an overlapping band for the union operation. Picks the
+ *	left-most rectangle each time and merges it into the region.
+ *
+ * Results:
+ *	TRUE if successful.
+ *
+ * Side Effects:
+ *	region is overwritten.
+ *	overlap is set to TRUE if any boxes overlap.
+ *
+ *-----------------------------------------------------------------------
+ */
+static pixman_bool_t
+pixman_region_union_o (region_type_t *region,
+		       box_type_t *   r1,
+		       box_type_t *   r1_end,
+		       box_type_t *   r2,
+		       box_type_t *   r2_end,
+		       int            y1,
+		       int            y2,
+		       int *          overlap)
+{
+    box_type_t *next_rect;
+    int x1;            /* left and right side of current union */
+    int x2;
+
+    critical_if_fail (y1 < y2);
+    critical_if_fail (r1 != r1_end && r2 != r2_end);
+
+    next_rect = PIXREGION_TOP (region);
+
+    /* Start off current rectangle */
+    if (r1->x1 < r2->x1)
+    {
+        x1 = r1->x1;
+        x2 = r1->x2;
+        r1++;
+    }
+    else
+    {
+        x1 = r2->x1;
+        x2 = r2->x2;
+        r2++;
+    }
+    while (r1 != r1_end && r2 != r2_end)
+    {
+        if (r1->x1 < r2->x1)
+	    MERGERECT (r1);
+	else
+	    MERGERECT (r2);
+    }
+
+    /* Finish off whoever (if any) is left */
+    if (r1 != r1_end)
+    {
+        do
+        {
+            MERGERECT (r1);
+	}
+        while (r1 != r1_end);
+    }
+    else if (r2 != r2_end)
+    {
+        do
+        {
+            MERGERECT (r2);
+	}
+        while (r2 != r2_end);
+    }
+
+    /* Add current rectangle */
+    NEWRECT (region, next_rect, x1, y1, x2, y2);
+
+    return TRUE;
+}
+
+/* Convenience function for performing union of region with a
+ * single rectangle
+ */
+PIXMAN_EXPORT pixman_bool_t
+PREFIX (_union_rect) (region_type_t *dest,
+                      region_type_t *source,
+                      int            x,
+		      int            y,
+                      unsigned int   width,
+		      unsigned int   height)
+{
+    region_type_t region;
+
+    region.extents.x1 = x;
+    region.extents.y1 = y;
+    region.extents.x2 = x + width;
+    region.extents.y2 = y + height;
+
+    if (!GOOD_RECT (&region.extents))
+    {
+        if (BAD_RECT (&region.extents))
+            _pixman_log_error (FUNC, "Invalid rectangle passed");
+	return PREFIX (_copy) (dest, source);
+    }
+
+    region.data = NULL;
+
+    return PREFIX (_union) (dest, source, &region);
+}
+
+PIXMAN_EXPORT pixman_bool_t
+PREFIX (_union) (region_type_t *new_reg,
+                 region_type_t *reg1,
+                 region_type_t *reg2)
+{
+    int overlap; /* result ignored */
+
+    /* Return TRUE if some overlap
+     * between reg1, reg2
+     */
+    GOOD (reg1);
+    GOOD (reg2);
+    GOOD (new_reg);
+
+    /*  checks all the simple cases */
+
+    /*
+     * Region 1 and 2 are the same
+     */
+    if (reg1 == reg2)
+        return PREFIX (_copy) (new_reg, reg1);
+
+    /*
+     * Region 1 is empty
+     */
+    if (PIXREGION_NIL (reg1))
+    {
+        if (PIXREGION_NAR (reg1))
+	    return pixman_break (new_reg);
+
+        if (new_reg != reg2)
+	    return PREFIX (_copy) (new_reg, reg2);
+
+	return TRUE;
+    }
+
+    /*
+     * Region 2 is empty
+     */
+    if (PIXREGION_NIL (reg2))
+    {
+        if (PIXREGION_NAR (reg2))
+	    return pixman_break (new_reg);
+
+	if (new_reg != reg1)
+	    return PREFIX (_copy) (new_reg, reg1);
+
+	return TRUE;
+    }
+
+    /*
+     * Region 1 completely subsumes region 2
+     */
+    if (!reg1->data && SUBSUMES (&reg1->extents, &reg2->extents))
+    {
+        if (new_reg != reg1)
+	    return PREFIX (_copy) (new_reg, reg1);
+
+	return TRUE;
+    }
+
+    /*
+     * Region 2 completely subsumes region 1
+     */
+    if (!reg2->data && SUBSUMES (&reg2->extents, &reg1->extents))
+    {
+        if (new_reg != reg2)
+	    return PREFIX (_copy) (new_reg, reg2);
+
+	return TRUE;
+    }
+
+    if (!pixman_op (new_reg, reg1, reg2, pixman_region_union_o, TRUE, TRUE, &overlap))
+	return FALSE;
+
+    new_reg->extents.x1 = MIN (reg1->extents.x1, reg2->extents.x1);
+    new_reg->extents.y1 = MIN (reg1->extents.y1, reg2->extents.y1);
+    new_reg->extents.x2 = MAX (reg1->extents.x2, reg2->extents.x2);
+    new_reg->extents.y2 = MAX (reg1->extents.y2, reg2->extents.y2);
+    
+    GOOD (new_reg);
+
+    return TRUE;
+}
+
+/*======================================================================
+ *	    Batch Rectangle Union
+ *====================================================================*/
+
+#define EXCHANGE_RECTS(a, b)	\
+    {                           \
+        box_type_t t;		\
+        t = rects[a];           \
+        rects[a] = rects[b];    \
+        rects[b] = t;           \
+    }
+
+static void
+quick_sort_rects (
+    box_type_t rects[],
+    int        numRects)
+{
+    int y1;
+    int x1;
+    int i, j;
+    box_type_t *r;
+
+    /* Always called with numRects > 1 */
+
+    do
+    {
+        if (numRects == 2)
+        {
+            if (rects[0].y1 > rects[1].y1 ||
+                (rects[0].y1 == rects[1].y1 && rects[0].x1 > rects[1].x1))
+	    {
+		EXCHANGE_RECTS (0, 1);
+	    }
+
+            return;
+	}
+
+        /* Choose partition element, stick in location 0 */
+        EXCHANGE_RECTS (0, numRects >> 1);
+        y1 = rects[0].y1;
+        x1 = rects[0].x1;
+
+        /* Partition array */
+        i = 0;
+        j = numRects;
+
+        do
+        {
+            r = &(rects[i]);
+            do
+            {
+                r++;
+                i++;
+	    }
+
+            while (i != numRects && (r->y1 < y1 || (r->y1 == y1 && r->x1 < x1)))
+		;
+
+	    r = &(rects[j]);
+            do
+            {
+                r--;
+                j--;
+	    }
+            while (y1 < r->y1 || (y1 == r->y1 && x1 < r->x1));
+	    
+            if (i < j)
+		EXCHANGE_RECTS (i, j);
+	}
+        while (i < j);
+
+        /* Move partition element back to middle */
+        EXCHANGE_RECTS (0, j);
+
+        /* Recurse */
+        if (numRects - j - 1 > 1)
+	    quick_sort_rects (&rects[j + 1], numRects - j - 1);
+
+        numRects = j;
+    }
+    while (numRects > 1);
+}
+
+/*-
+ *-----------------------------------------------------------------------
+ * pixman_region_validate --
+ *
+ *      Take a ``region'' which is a non-y-x-banded random collection of
+ *      rectangles, and compute a nice region which is the union of all the
+ *      rectangles.
+ *
+ * Results:
+ *	TRUE if successful.
+ *
+ * Side Effects:
+ *      The passed-in ``region'' may be modified.
+ *	overlap set to TRUE if any retangles overlapped,
+ *      else FALSE;
+ *
+ * Strategy:
+ *      Step 1. Sort the rectangles into ascending order with primary key y1
+ *		and secondary key x1.
+ *
+ *      Step 2. Split the rectangles into the minimum number of proper y-x
+ *		banded regions.  This may require horizontally merging
+ *		rectangles, and vertically coalescing bands.  With any luck,
+ *		this step in an identity transformation (ala the Box widget),
+ *		or a coalescing into 1 box (ala Menus).
+ *
+ *	Step 3. Merge the separate regions down to a single region by calling
+ *		pixman_region_union.  Maximize the work each pixman_region_union call does by using
+ *		a binary merge.
+ *
+ *-----------------------------------------------------------------------
+ */
+
+static pixman_bool_t
+validate (region_type_t * badreg,
+          int *           overlap)
+{
+    /* Descriptor for regions under construction  in Step 2. */
+    typedef struct
+    {
+        region_type_t reg;
+        int prev_band;
+        int cur_band;
+    } region_info_t;
+
+    region_info_t stack_regions[64];
+
+    int numRects;                   /* Original numRects for badreg	    */
+    region_info_t *ri;              /* Array of current regions		    */
+    int num_ri;                     /* Number of entries used in ri	    */
+    int size_ri;                    /* Number of entries available in ri    */
+    int i;                          /* Index into rects			    */
+    int j;                          /* Index into ri			    */
+    region_info_t *rit;             /* &ri[j]				    */
+    region_type_t *reg;             /* ri[j].reg			    */
+    box_type_t *box;                /* Current box in rects		    */
+    box_type_t *ri_box;             /* Last box in ri[j].reg		    */
+    region_type_t *hreg;            /* ri[j_half].reg			    */
+    pixman_bool_t ret = TRUE;
+
+    *overlap = FALSE;
+    if (!badreg->data)
+    {
+        GOOD (badreg);
+        return TRUE;
+    }
+    
+    numRects = badreg->data->numRects;
+    if (!numRects)
+    {
+        if (PIXREGION_NAR (badreg))
+	    return FALSE;
+        GOOD (badreg);
+        return TRUE;
+    }
+    
+    if (badreg->extents.x1 < badreg->extents.x2)
+    {
+        if ((numRects) == 1)
+        {
+            FREE_DATA (badreg);
+            badreg->data = (region_data_type_t *) NULL;
+	}
+        else
+        {
+            DOWNSIZE (badreg, numRects);
+	}
+
+        GOOD (badreg);
+
+	return TRUE;
+    }
+
+    /* Step 1: Sort the rects array into ascending (y1, x1) order */
+    quick_sort_rects (PIXREGION_BOXPTR (badreg), numRects);
+
+    /* Step 2: Scatter the sorted array into the minimum number of regions */
+
+    /* Set up the first region to be the first rectangle in badreg */
+    /* Note that step 2 code will never overflow the ri[0].reg rects array */
+    ri = stack_regions;
+    size_ri = sizeof (stack_regions) / sizeof (stack_regions[0]);
+    num_ri = 1;
+    ri[0].prev_band = 0;
+    ri[0].cur_band = 0;
+    ri[0].reg = *badreg;
+    box = PIXREGION_BOXPTR (&ri[0].reg);
+    ri[0].reg.extents = *box;
+    ri[0].reg.data->numRects = 1;
+    badreg->extents = *pixman_region_empty_box;
+    badreg->data = pixman_region_empty_data;
+
+    /* Now scatter rectangles into the minimum set of valid regions.  If the
+     * next rectangle to be added to a region would force an existing rectangle
+     * in the region to be split up in order to maintain y-x banding, just
+     * forget it.  Try the next region.  If it doesn't fit cleanly into any
+     * region, make a new one.
+     */
+
+    for (i = numRects; --i > 0;)
+    {
+        box++;
+        /* Look for a region to append box to */
+        for (j = num_ri, rit = ri; --j >= 0; rit++)
+        {
+            reg = &rit->reg;
+            ri_box = PIXREGION_END (reg);
+
+            if (box->y1 == ri_box->y1 && box->y2 == ri_box->y2)
+            {
+                /* box is in same band as ri_box.  Merge or append it */
+                if (box->x1 <= ri_box->x2)
+                {
+                    /* Merge it with ri_box */
+                    if (box->x1 < ri_box->x2)
+			*overlap = TRUE;
+
+                    if (box->x2 > ri_box->x2)
+			ri_box->x2 = box->x2;
+		}
+                else
+                {
+                    RECTALLOC_BAIL (reg, 1, bail);
+                    *PIXREGION_TOP (reg) = *box;
+                    reg->data->numRects++;
+		}
+		
+                goto next_rect;   /* So sue me */
+	    }
+            else if (box->y1 >= ri_box->y2)
+            {
+                /* Put box into new band */
+                if (reg->extents.x2 < ri_box->x2)
+		    reg->extents.x2 = ri_box->x2;
+		
+                if (reg->extents.x1 > box->x1)
+		    reg->extents.x1 = box->x1;
+		
+                COALESCE (reg, rit->prev_band, rit->cur_band);
+                rit->cur_band = reg->data->numRects;
+                RECTALLOC_BAIL (reg, 1, bail);
+                *PIXREGION_TOP (reg) = *box;
+                reg->data->numRects++;
+
+                goto next_rect;
+	    }
+            /* Well, this region was inappropriate.  Try the next one. */
+	} /* for j */
+
+        /* Uh-oh.  No regions were appropriate.  Create a new one. */
+        if (size_ri == num_ri)
+        {
+            size_t data_size;
+
+            /* Oops, allocate space for new region information */
+            size_ri <<= 1;
+
+            data_size = size_ri * sizeof(region_info_t);
+            if (data_size / size_ri != sizeof(region_info_t))
+		goto bail;
+
+            if (ri == stack_regions)
+            {
+                rit = malloc (data_size);
+                if (!rit)
+		    goto bail;
+                memcpy (rit, ri, num_ri * sizeof (region_info_t));
+	    }
+            else
+            {
+                rit = (region_info_t *) realloc (ri, data_size);
+                if (!rit)
+		    goto bail;
+	    }
+            ri = rit;
+            rit = &ri[num_ri];
+	}
+        num_ri++;
+        rit->prev_band = 0;
+        rit->cur_band = 0;
+        rit->reg.extents = *box;
+        rit->reg.data = (region_data_type_t *)NULL;
+
+	/* MUST force allocation */
+        if (!pixman_rect_alloc (&rit->reg, (i + num_ri) / num_ri))
+	    goto bail;
+	
+    next_rect: ;
+    } /* for i */
+
+    /* Make a final pass over each region in order to COALESCE and set
+     * extents.x2 and extents.y2
+     */
+    for (j = num_ri, rit = ri; --j >= 0; rit++)
+    {
+        reg = &rit->reg;
+        ri_box = PIXREGION_END (reg);
+        reg->extents.y2 = ri_box->y2;
+
+        if (reg->extents.x2 < ri_box->x2)
+	    reg->extents.x2 = ri_box->x2;
+	
+        COALESCE (reg, rit->prev_band, rit->cur_band);
+
+	if (reg->data->numRects == 1) /* keep unions happy below */
+        {
+            FREE_DATA (reg);
+            reg->data = (region_data_type_t *)NULL;
+	}
+    }
+
+    /* Step 3: Union all regions into a single region */
+    while (num_ri > 1)
+    {
+        int half = num_ri / 2;
+        for (j = num_ri & 1; j < (half + (num_ri & 1)); j++)
+        {
+            reg = &ri[j].reg;
+            hreg = &ri[j + half].reg;
+
+            if (!pixman_op (reg, reg, hreg, pixman_region_union_o, TRUE, TRUE, overlap))
+		ret = FALSE;
+
+            if (hreg->extents.x1 < reg->extents.x1)
+		reg->extents.x1 = hreg->extents.x1;
+
+            if (hreg->extents.y1 < reg->extents.y1)
+		reg->extents.y1 = hreg->extents.y1;
+
+            if (hreg->extents.x2 > reg->extents.x2)
+		reg->extents.x2 = hreg->extents.x2;
+
+            if (hreg->extents.y2 > reg->extents.y2)
+		reg->extents.y2 = hreg->extents.y2;
+
+            FREE_DATA (hreg);
+	}
+
+        num_ri -= half;
+
+	if (!ret)
+	    goto bail;
+    }
+
+    *badreg = ri[0].reg;
+
+    if (ri != stack_regions)
+	free (ri);
+
+    GOOD (badreg);
+    return ret;
+
+bail:
+    for (i = 0; i < num_ri; i++)
+	FREE_DATA (&ri[i].reg);
+
+    if (ri != stack_regions)
+	free (ri);
+
+    return pixman_break (badreg);
+}
+
+/*======================================================================
+ *                Region Subtraction
+ *====================================================================*/
+
+/*-
+ *-----------------------------------------------------------------------
+ * pixman_region_subtract_o --
+ *	Overlapping band subtraction. x1 is the left-most point not yet
+ *	checked.
+ *
+ * Results:
+ *	TRUE if successful.
+ *
+ * Side Effects:
+ *	region may have rectangles added to it.
+ *
+ *-----------------------------------------------------------------------
+ */
+/*ARGSUSED*/
+static pixman_bool_t
+pixman_region_subtract_o (region_type_t * region,
+			  box_type_t *    r1,
+			  box_type_t *    r1_end,
+			  box_type_t *    r2,
+			  box_type_t *    r2_end,
+			  int             y1,
+			  int             y2,
+			  int *           overlap)
+{
+    box_type_t *        next_rect;
+    int x1;
+
+    x1 = r1->x1;
+
+    critical_if_fail (y1 < y2);
+    critical_if_fail (r1 != r1_end && r2 != r2_end);
+
+    next_rect = PIXREGION_TOP (region);
+
+    do
+    {
+        if (r2->x2 <= x1)
+        {
+            /*
+	     * Subtrahend entirely to left of minuend: go to next subtrahend.
+	     */
+            r2++;
+	}
+        else if (r2->x1 <= x1)
+        {
+            /*
+	     * Subtrahend preceeds minuend: nuke left edge of minuend.
+	     */
+            x1 = r2->x2;
+            if (x1 >= r1->x2)
+            {
+                /*
+		 * Minuend completely covered: advance to next minuend and
+		 * reset left fence to edge of new minuend.
+		 */
+                r1++;
+                if (r1 != r1_end)
+		    x1 = r1->x1;
+	    }
+            else
+            {
+                /*
+		 * Subtrahend now used up since it doesn't extend beyond
+		 * minuend
+		 */
+                r2++;
+	    }
+	}
+        else if (r2->x1 < r1->x2)
+        {
+            /*
+	     * Left part of subtrahend covers part of minuend: add uncovered
+	     * part of minuend to region and skip to next subtrahend.
+	     */
+            critical_if_fail (x1 < r2->x1);
+            NEWRECT (region, next_rect, x1, y1, r2->x1, y2);
+
+            x1 = r2->x2;
+            if (x1 >= r1->x2)
+            {
+                /*
+		 * Minuend used up: advance to new...
+		 */
+                r1++;
+                if (r1 != r1_end)
+		    x1 = r1->x1;
+	    }
+            else
+            {
+                /*
+		 * Subtrahend used up
+		 */
+                r2++;
+	    }
+	}
+        else
+        {
+            /*
+	     * Minuend used up: add any remaining piece before advancing.
+	     */
+            if (r1->x2 > x1)
+		NEWRECT (region, next_rect, x1, y1, r1->x2, y2);
+
+            r1++;
+
+	    if (r1 != r1_end)
+		x1 = r1->x1;
+	}
+    }
+    while ((r1 != r1_end) && (r2 != r2_end));
+
+    /*
+     * Add remaining minuend rectangles to region.
+     */
+    while (r1 != r1_end)
+    {
+        critical_if_fail (x1 < r1->x2);
+
+        NEWRECT (region, next_rect, x1, y1, r1->x2, y2);
+
+        r1++;
+        if (r1 != r1_end)
+	    x1 = r1->x1;
+    }
+    return TRUE;
+}
+
+/*-
+ *-----------------------------------------------------------------------
+ * pixman_region_subtract --
+ *	Subtract reg_s from reg_m and leave the result in reg_d.
+ *	S stands for subtrahend, M for minuend and D for difference.
+ *
+ * Results:
+ *	TRUE if successful.
+ *
+ * Side Effects:
+ *	reg_d is overwritten.
+ *
+ *-----------------------------------------------------------------------
+ */
+PIXMAN_EXPORT pixman_bool_t
+PREFIX (_subtract) (region_type_t *reg_d,
+                    region_type_t *reg_m,
+                    region_type_t *reg_s)
+{
+    int overlap; /* result ignored */
+
+    GOOD (reg_m);
+    GOOD (reg_s);
+    GOOD (reg_d);
+    
+    /* check for trivial rejects */
+    if (PIXREGION_NIL (reg_m) || PIXREGION_NIL (reg_s) ||
+        !EXTENTCHECK (&reg_m->extents, &reg_s->extents))
+    {
+        if (PIXREGION_NAR (reg_s))
+	    return pixman_break (reg_d);
+	
+        return PREFIX (_copy) (reg_d, reg_m);
+    }
+    else if (reg_m == reg_s)
+    {
+        FREE_DATA (reg_d);
+        reg_d->extents.x2 = reg_d->extents.x1;
+        reg_d->extents.y2 = reg_d->extents.y1;
+        reg_d->data = pixman_region_empty_data;
+
+        return TRUE;
+    }
+
+    /* Add those rectangles in region 1 that aren't in region 2,
+       do yucky substraction for overlaps, and
+       just throw away rectangles in region 2 that aren't in region 1 */
+    if (!pixman_op (reg_d, reg_m, reg_s, pixman_region_subtract_o, TRUE, FALSE, &overlap))
+	return FALSE;
+
+    /*
+     * Can't alter reg_d's extents before we call pixman_op because
+     * it might be one of the source regions and pixman_op depends
+     * on the extents of those regions being unaltered. Besides, this
+     * way there's no checking against rectangles that will be nuked
+     * due to coalescing, so we have to examine fewer rectangles.
+     */
+    pixman_set_extents (reg_d);
+    GOOD (reg_d);
+    return TRUE;
+}
+
+/*======================================================================
+ *	    Region Inversion
+ *====================================================================*/
+
+/*-
+ *-----------------------------------------------------------------------
+ * pixman_region_inverse --
+ *	Take a region and a box and return a region that is everything
+ *	in the box but not in the region. The careful reader will note
+ *	that this is the same as subtracting the region from the box...
+ *
+ * Results:
+ *	TRUE.
+ *
+ * Side Effects:
+ *	new_reg is overwritten.
+ *
+ *-----------------------------------------------------------------------
+ */
+pixman_bool_t
+PIXMAN_EXPORT PREFIX (_inverse) (region_type_t *new_reg,  /* Destination region */
+                                 region_type_t *reg1,     /* Region to invert */
+                                 box_type_t *   inv_rect) /* Bounding box for inversion */
+{
+    region_type_t inv_reg; /* Quick and dirty region made from the
+			    * bounding box */
+    int overlap;           /* result ignored */
+
+    GOOD (reg1);
+    GOOD (new_reg);
+    
+    /* check for trivial rejects */
+    if (PIXREGION_NIL (reg1) || !EXTENTCHECK (inv_rect, &reg1->extents))
+    {
+        if (PIXREGION_NAR (reg1))
+	    return pixman_break (new_reg);
+	
+        new_reg->extents = *inv_rect;
+        FREE_DATA (new_reg);
+        new_reg->data = (region_data_type_t *)NULL;
+	
+        return TRUE;
+    }
+
+    /* Add those rectangles in region 1 that aren't in region 2,
+     * do yucky substraction for overlaps, and
+     * just throw away rectangles in region 2 that aren't in region 1
+     */
+    inv_reg.extents = *inv_rect;
+    inv_reg.data = (region_data_type_t *)NULL;
+    if (!pixman_op (new_reg, &inv_reg, reg1, pixman_region_subtract_o, TRUE, FALSE, &overlap))
+	return FALSE;
+
+    /*
+     * Can't alter new_reg's extents before we call pixman_op because
+     * it might be one of the source regions and pixman_op depends
+     * on the extents of those regions being unaltered. Besides, this
+     * way there's no checking against rectangles that will be nuked
+     * due to coalescing, so we have to examine fewer rectangles.
+     */
+    pixman_set_extents (new_reg);
+    GOOD (new_reg);
+    return TRUE;
+}
+
+/*
+ *   rect_in(region, rect)
+ *   This routine takes a pointer to a region and a pointer to a box
+ *   and determines if the box is outside/inside/partly inside the region.
+ *
+ *   The idea is to travel through the list of rectangles trying to cover the
+ *   passed box with them. Anytime a piece of the rectangle isn't covered
+ *   by a band of rectangles, part_out is set TRUE. Any time a rectangle in
+ *   the region covers part of the box, part_in is set TRUE. The process ends
+ *   when either the box has been completely covered (we reached a band that
+ *   doesn't overlap the box, part_in is TRUE and part_out is false), the
+ *   box has been partially covered (part_in == part_out == TRUE -- because of
+ *   the banding, the first time this is true we know the box is only
+ *   partially in the region) or is outside the region (we reached a band
+ *   that doesn't overlap the box at all and part_in is false)
+ */
+
+pixman_region_overlap_t
+PIXMAN_EXPORT PREFIX (_contains_rectangle) (region_type_t *  region,
+                                            box_type_t *     prect)
+{
+    box_type_t *     pbox;
+    box_type_t *     pbox_end;
+    int part_in, part_out;
+    int numRects;
+    int x, y;
+
+    GOOD (region);
+
+    numRects = PIXREGION_NUMRECTS (region);
+
+    /* useful optimization */
+    if (!numRects || !EXTENTCHECK (&region->extents, prect))
+	return(PIXMAN_REGION_OUT);
+
+    if (numRects == 1)
+    {
+        /* We know that it must be PIXMAN_REGION_IN or PIXMAN_REGION_PART */
+        if (SUBSUMES (&region->extents, prect))
+	    return(PIXMAN_REGION_IN);
+        else
+	    return(PIXMAN_REGION_PART);
+    }
+
+    part_out = FALSE;
+    part_in = FALSE;
+
+    /* (x,y) starts at upper left of rect, moving to the right and down */
+    x = prect->x1;
+    y = prect->y1;
+
+    /* can stop when both part_out and part_in are TRUE, or we reach prect->y2 */
+    for (pbox = PIXREGION_BOXPTR (region), pbox_end = pbox + numRects;
+         pbox != pbox_end;
+         pbox++)
+    {
+
+        if (pbox->y2 <= y)
+	    continue;   /* getting up to speed or skipping remainder of band */
+
+        if (pbox->y1 > y)
+        {
+            part_out = TRUE;     /* missed part of rectangle above */
+            if (part_in || (pbox->y1 >= prect->y2))
+		break;
+            y = pbox->y1;       /* x guaranteed to be == prect->x1 */
+	}
+
+        if (pbox->x2 <= x)
+	    continue;           /* not far enough over yet */
+
+        if (pbox->x1 > x)
+        {
+            part_out = TRUE;     /* missed part of rectangle to left */
+            if (part_in)
+		break;
+	}
+
+        if (pbox->x1 < prect->x2)
+        {
+            part_in = TRUE;      /* definitely overlap */
+            if (part_out)
+		break;
+	}
+
+        if (pbox->x2 >= prect->x2)
+        {
+            y = pbox->y2;       /* finished with this band */
+            if (y >= prect->y2)
+		break;
+            x = prect->x1;      /* reset x out to left again */
+	}
+        else
+        {
+            /*
+	     * Because boxes in a band are maximal width, if the first box
+	     * to overlap the rectangle doesn't completely cover it in that
+	     * band, the rectangle must be partially out, since some of it
+	     * will be uncovered in that band. part_in will have been set true
+	     * by now...
+	     */
+            part_out = TRUE;
+            break;
+	}
+    }
+
+    if (part_in)
+    {
+        if (y < prect->y2)
+	    return PIXMAN_REGION_PART;
+        else
+	    return PIXMAN_REGION_IN;
+    }
+    else
+    {
+        return PIXMAN_REGION_OUT;
+    }
+}
+
+/* PREFIX(_translate) (region, x, y)
+ * translates in place
+ */
+
+PIXMAN_EXPORT void
+PREFIX (_translate) (region_type_t *region, int x, int y)
+{
+    int x1, x2, y1, y2;
+    int nbox;
+    box_type_t * pbox;
+
+    GOOD (region);
+    region->extents.x1 = x1 = region->extents.x1 + x;
+    region->extents.y1 = y1 = region->extents.y1 + y;
+    region->extents.x2 = x2 = region->extents.x2 + x;
+    region->extents.y2 = y2 = region->extents.y2 + y;
+    
+    if (((x1 - SHRT_MIN) | (y1 - SHRT_MIN) | (SHRT_MAX - x2) | (SHRT_MAX - y2)) >= 0)
+    {
+        if (region->data && (nbox = region->data->numRects))
+        {
+            for (pbox = PIXREGION_BOXPTR (region); nbox--; pbox++)
+            {
+                pbox->x1 += x;
+                pbox->y1 += y;
+                pbox->x2 += x;
+                pbox->y2 += y;
+	    }
+	}
+        return;
+    }
+
+    if (((x2 - SHRT_MIN) | (y2 - SHRT_MIN) | (SHRT_MAX - x1) | (SHRT_MAX - y1)) <= 0)
+    {
+        region->extents.x2 = region->extents.x1;
+        region->extents.y2 = region->extents.y1;
+        FREE_DATA (region);
+        region->data = pixman_region_empty_data;
+        return;
+    }
+
+    if (x1 < SHRT_MIN)
+	region->extents.x1 = SHRT_MIN;
+    else if (x2 > SHRT_MAX)
+	region->extents.x2 = SHRT_MAX;
+
+    if (y1 < SHRT_MIN)
+	region->extents.y1 = SHRT_MIN;
+    else if (y2 > SHRT_MAX)
+	region->extents.y2 = SHRT_MAX;
+
+    if (region->data && (nbox = region->data->numRects))
+    {
+        box_type_t * pbox_out;
+
+        for (pbox_out = pbox = PIXREGION_BOXPTR (region); nbox--; pbox++)
+        {
+            pbox_out->x1 = x1 = pbox->x1 + x;
+            pbox_out->y1 = y1 = pbox->y1 + y;
+            pbox_out->x2 = x2 = pbox->x2 + x;
+            pbox_out->y2 = y2 = pbox->y2 + y;
+
+            if (((x2 - SHRT_MIN) | (y2 - SHRT_MIN) |
+                 (SHRT_MAX - x1) | (SHRT_MAX - y1)) <= 0)
+            {
+                region->data->numRects--;
+                continue;
+	    }
+
+            if (x1 < SHRT_MIN)
+		pbox_out->x1 = SHRT_MIN;
+            else if (x2 > SHRT_MAX)
+		pbox_out->x2 = SHRT_MAX;
+
+            if (y1 < SHRT_MIN)
+		pbox_out->y1 = SHRT_MIN;
+            else if (y2 > SHRT_MAX)
+		pbox_out->y2 = SHRT_MAX;
+
+            pbox_out++;
+	}
+
+        if (pbox_out != pbox)
+        {
+            if (region->data->numRects == 1)
+            {
+                region->extents = *PIXREGION_BOXPTR (region);
+                FREE_DATA (region);
+                region->data = (region_data_type_t *)NULL;
+	    }
+            else
+	    {
+		pixman_set_extents (region);
+	    }
+	}
+    }
+
+    GOOD (region);
+}
+
+PIXMAN_EXPORT void
+PREFIX (_reset) (region_type_t *region, box_type_t *box)
+{
+    GOOD (region);
+
+    critical_if_fail (GOOD_RECT (box));
+
+    region->extents = *box;
+
+    FREE_DATA (region);
+
+    region->data = NULL;
+}
+
+/* box is "return" value */
+PIXMAN_EXPORT int
+PREFIX (_contains_point) (region_type_t * region,
+                          int x, int y,
+                          box_type_t * box)
+{
+    box_type_t *pbox, *pbox_end;
+    int numRects;
+
+    GOOD (region);
+    numRects = PIXREGION_NUMRECTS (region);
+
+    if (!numRects || !INBOX (&region->extents, x, y))
+	return(FALSE);
+
+    if (numRects == 1)
+    {
+        if (box)
+	    *box = region->extents;
+
+        return(TRUE);
+    }
+
+    for (pbox = PIXREGION_BOXPTR (region), pbox_end = pbox + numRects;
+	 pbox != pbox_end;
+	 pbox++)
+    {
+        if (y >= pbox->y2)
+	    continue;           /* not there yet */
+
+        if ((y < pbox->y1) || (x < pbox->x1))
+	    break;              /* missed it */
+
+        if (x >= pbox->x2)
+	    continue;           /* not there yet */
+
+        if (box)
+	    *box = *pbox;
+
+        return(TRUE);
+    }
+
+    return(FALSE);
+}
+
+PIXMAN_EXPORT int
+PREFIX (_not_empty) (region_type_t * region)
+{
+    GOOD (region);
+
+    return(!PIXREGION_NIL (region));
+}
+
+PIXMAN_EXPORT box_type_t *
+PREFIX (_extents) (region_type_t * region)
+{
+    GOOD (region);
+
+    return(&region->extents);
+}
+
+/*
+ * Clip a list of scanlines to a region.  The caller has allocated the
+ * space.  FSorted is non-zero if the scanline origins are in ascending order.
+ *
+ * returns the number of new, clipped scanlines.
+ */
+
+PIXMAN_EXPORT pixman_bool_t
+PREFIX (_selfcheck) (region_type_t *reg)
+{
+    int i, numRects;
+
+    if ((reg->extents.x1 > reg->extents.x2) ||
+        (reg->extents.y1 > reg->extents.y2))
+    {
+	return FALSE;
+    }
+
+    numRects = PIXREGION_NUMRECTS (reg);
+    if (!numRects)
+    {
+	return ((reg->extents.x1 == reg->extents.x2) &&
+	        (reg->extents.y1 == reg->extents.y2) &&
+	        (reg->data->size || (reg->data == pixman_region_empty_data)));
+    }
+    else if (numRects == 1)
+    {
+	return (!reg->data);
+    }
+    else
+    {
+        box_type_t * pbox_p, * pbox_n;
+        box_type_t box;
+
+        pbox_p = PIXREGION_RECTS (reg);
+        box = *pbox_p;
+        box.y2 = pbox_p[numRects - 1].y2;
+        pbox_n = pbox_p + 1;
+
+        for (i = numRects; --i > 0; pbox_p++, pbox_n++)
+        {
+            if ((pbox_n->x1 >= pbox_n->x2) ||
+                (pbox_n->y1 >= pbox_n->y2))
+	    {
+		return FALSE;
+	    }
+
+            if (pbox_n->x1 < box.x1)
+		box.x1 = pbox_n->x1;
+	    
+            if (pbox_n->x2 > box.x2)
+		box.x2 = pbox_n->x2;
+	    
+            if ((pbox_n->y1 < pbox_p->y1) ||
+                ((pbox_n->y1 == pbox_p->y1) &&
+                 ((pbox_n->x1 < pbox_p->x2) || (pbox_n->y2 != pbox_p->y2))))
+	    {
+		return FALSE;
+	    }
+	}
+
+        return ((box.x1 == reg->extents.x1) &&
+                (box.x2 == reg->extents.x2) &&
+                (box.y1 == reg->extents.y1) &&
+                (box.y2 == reg->extents.y2));
+    }
+}
+
+PIXMAN_EXPORT pixman_bool_t
+PREFIX (_init_rects) (region_type_t *region,
+                      const box_type_t *boxes, int count)
+{
+    box_type_t *rects;
+    int displacement;
+    int i;
+
+    /* if it's 1, then we just want to set the extents, so call
+     * the existing method. */
+    if (count == 1)
+    {
+        PREFIX (_init_rect) (region,
+                             boxes[0].x1,
+                             boxes[0].y1,
+                             boxes[0].x2 - boxes[0].x1,
+                             boxes[0].y2 - boxes[0].y1);
+        return TRUE;
+    }
+
+    PREFIX (_init) (region);
+
+    /* if it's 0, don't call pixman_rect_alloc -- 0 rectangles is
+     * a special case, and causing pixman_rect_alloc would cause
+     * us to leak memory (because the 0-rect case should be the
+     * static pixman_region_empty_data data).
+     */
+    if (count == 0)
+	return TRUE;
+
+    if (!pixman_rect_alloc (region, count))
+	return FALSE;
+
+    rects = PIXREGION_RECTS (region);
+
+    /* Copy in the rects */
+    memcpy (rects, boxes, sizeof(box_type_t) * count);
+    region->data->numRects = count;
+
+    /* Eliminate empty and malformed rectangles */
+    displacement = 0;
+
+    for (i = 0; i < count; ++i)
+    {
+        box_type_t *box = &rects[i];
+
+        if (box->x1 >= box->x2 || box->y1 >= box->y2)
+	    displacement++;
+        else if (displacement)
+	    rects[i - displacement] = rects[i];
+    }
+
+    region->data->numRects -= displacement;
+
+    /* If eliminating empty rectangles caused there
+     * to be only 0 or 1 rectangles, deal with that.
+     */
+    if (region->data->numRects == 0)
+    {
+        FREE_DATA (region);
+        PREFIX (_init) (region);
+
+        return TRUE;
+    }
+
+    if (region->data->numRects == 1)
+    {
+        region->extents = rects[0];
+
+        FREE_DATA (region);
+        region->data = NULL;
+
+        GOOD (region);
+
+        return TRUE;
+    }
+
+    /* Validate */
+    region->extents.x1 = region->extents.x2 = 0;
+
+    return validate (region, &i);
+}
+
+#define READ(_ptr) (*(_ptr))
+
+static inline box_type_t *
+bitmap_addrect (region_type_t *reg,
+                box_type_t *r,
+                box_type_t **first_rect,
+                int rx1, int ry1,
+                int rx2, int ry2)
+{
+    if ((rx1 < rx2) && (ry1 < ry2) &&
+	(!(reg->data->numRects &&
+	   ((r-1)->y1 == ry1) && ((r-1)->y2 == ry2) &&
+	   ((r-1)->x1 <= rx1) && ((r-1)->x2 >= rx2))))
+    {
+	if (!reg->data ||
+	    reg->data->numRects == reg->data->size)
+	{
+	    if (!pixman_rect_alloc (reg, 1))
+		return NULL;
+	    *first_rect = PIXREGION_BOXPTR(reg);
+	    r = *first_rect + reg->data->numRects;
+	}
+	r->x1 = rx1;
+	r->y1 = ry1;
+	r->x2 = rx2;
+	r->y2 = ry2;
+	reg->data->numRects++;
+	if (r->x1 < reg->extents.x1)
+	    reg->extents.x1 = r->x1;
+	if (r->x2 > reg->extents.x2)
+	    reg->extents.x2 = r->x2;
+	r++;
+    }
+    return r;
+}
+
+/* Convert bitmap clip mask into clipping region.
+ * First, goes through each line and makes boxes by noting the transitions
+ * from 0 to 1 and 1 to 0.
+ * Then it coalesces the current line with the previous if they have boxes
+ * at the same X coordinates.
+ * Stride is in number of uint32_t per line.
+ */
+PIXMAN_EXPORT void
+PREFIX (_init_from_image) (region_type_t *region,
+                           pixman_image_t *image)
+{
+    uint32_t mask0 = 0xffffffff & ~SCREEN_SHIFT_RIGHT(0xffffffff, 1);
+    box_type_t *first_rect, *rects, *prect_line_start;
+    box_type_t *old_rect, *new_rect;
+    uint32_t *pw, w, *pw_line, *pw_line_end;
+    int	irect_prev_start, irect_line_start;
+    int	h, base, rx1 = 0, crects;
+    int	ib;
+    pixman_bool_t in_box, same;
+    int width, height, stride;
+
+    PREFIX(_init) (region);
+
+    return_if_fail (image->type == BITS);
+    return_if_fail (image->bits.format == PIXMAN_a1);
+
+    pw_line = pixman_image_get_data (image);
+    width = pixman_image_get_width (image);
+    height = pixman_image_get_height (image);
+    stride = pixman_image_get_stride (image) / 4;
+
+    first_rect = PIXREGION_BOXPTR(region);
+    rects = first_rect;
+
+    region->extents.x1 = width - 1;
+    region->extents.x2 = 0;
+    irect_prev_start = -1;
+    for (h = 0; h < height; h++)
+    {
+        pw = pw_line;
+        pw_line += stride;
+        irect_line_start = rects - first_rect;
+
+        /* If the Screen left most bit of the word is set, we're starting in
+         * a box */
+        if (READ(pw) & mask0)
+        {
+            in_box = TRUE;
+            rx1 = 0;
+        }
+        else
+        {
+            in_box = FALSE;
+        }
+
+        /* Process all words which are fully in the pixmap */
+        pw_line_end = pw + (width >> 5);
+        for (base = 0; pw < pw_line_end; base += 32)
+        {
+            w = READ(pw++);
+            if (in_box)
+            {
+                if (!~w)
+                    continue;
+            }
+            else
+            {
+                if (!w)
+                    continue;
+            }
+            for (ib = 0; ib < 32; ib++)
+            {
+                /* If the Screen left most bit of the word is set, we're
+                 * starting a box */
+                if (w & mask0)
+                {
+                    if (!in_box)
+                    {
+                        rx1 = base + ib;
+                        /* start new box */
+                        in_box = TRUE;
+                    }
+                }
+                else
+                {
+                    if (in_box)
+                    {
+                        /* end box */
+                        rects = bitmap_addrect (region, rects, &first_rect,
+                                                rx1, h, base + ib, h + 1);
+                        if (rects == NULL)
+                            goto error;
+                        in_box = FALSE;
+                    }
+                }
+                /* Shift the word VISUALLY left one. */
+                w = SCREEN_SHIFT_LEFT(w, 1);
+            }
+        }
+
+        if (width & 31)
+        {
+            /* Process final partial word on line */
+             w = READ(pw++);
+            for (ib = 0; ib < (width & 31); ib++)
+            {
+                /* If the Screen left most bit of the word is set, we're
+                 * starting a box */
+                if (w & mask0)
+                {
+                    if (!in_box)
+                    {
+                        rx1 = base + ib;
+                        /* start new box */
+                        in_box = TRUE;
+                    }
+                }
+                else
+                {
+                    if (in_box)
+                    {
+                        /* end box */
+                        rects = bitmap_addrect(region, rects, &first_rect,
+					       rx1, h, base + ib, h + 1);
+			if (rects == NULL)
+			    goto error;
+                        in_box = FALSE;
+                    }
+                }
+                /* Shift the word VISUALLY left one. */
+                w = SCREEN_SHIFT_LEFT(w, 1);
+            }
+        }
+        /* If scanline ended with last bit set, end the box */
+        if (in_box)
+        {
+            rects = bitmap_addrect(region, rects, &first_rect,
+				   rx1, h, base + (width & 31), h + 1);
+	    if (rects == NULL)
+		goto error;
+        }
+        /* if all rectangles on this line have the same x-coords as
+         * those on the previous line, then add 1 to all the previous  y2s and
+         * throw away all the rectangles from this line
+         */
+        same = FALSE;
+        if (irect_prev_start != -1)
+        {
+            crects = irect_line_start - irect_prev_start;
+            if (crects != 0 &&
+                crects == ((rects - first_rect) - irect_line_start))
+            {
+                old_rect = first_rect + irect_prev_start;
+                new_rect = prect_line_start = first_rect + irect_line_start;
+                same = TRUE;
+                while (old_rect < prect_line_start)
+                {
+                    if ((old_rect->x1 != new_rect->x1) ||
+                        (old_rect->x2 != new_rect->x2))
+                    {
+                          same = FALSE;
+                          break;
+                    }
+                    old_rect++;
+                    new_rect++;
+                }
+                if (same)
+                {
+                    old_rect = first_rect + irect_prev_start;
+                    while (old_rect < prect_line_start)
+                    {
+                        old_rect->y2 += 1;
+                        old_rect++;
+                    }
+                    rects -= crects;
+                    region->data->numRects -= crects;
+                }
+            }
+        }
+        if(!same)
+            irect_prev_start = irect_line_start;
+    }
+    if (!region->data->numRects)
+    {
+        region->extents.x1 = region->extents.x2 = 0;
+    }
+    else
+    {
+        region->extents.y1 = PIXREGION_BOXPTR(region)->y1;
+        region->extents.y2 = PIXREGION_END(region)->y2;
+        if (region->data->numRects == 1)
+        {
+            free (region->data);
+            region->data = NULL;
+        }
+    }
+
+ error:
+    return;
+}
diff --git a/pixman/pixman/pixman-region16.c b/pixman/pixman/pixman-region16.c
new file mode 100644
index 000000000..46f5e26ea
--- /dev/null
+++ b/pixman/pixman/pixman-region16.c
@@ -0,0 +1,63 @@
+/*
+ * Copyright © 2008 Red Hat, Inc.
+ *
+ * Permission to use, copy, modify, distribute, and sell this software
+ * and its documentation for any purpose is hereby granted without
+ * fee, provided that the above copyright notice appear in all copies
+ * and that both that copyright notice and this permission notice
+ * appear in supporting documentation, and that the name of
+ * Red Hat, Inc. not be used in advertising or publicity pertaining to
+ * distribution of the software without specific, written prior
+ * permission. Red Hat, Inc. makes no representations about the
+ * suitability of this software for any purpose.  It is provided "as
+ * is" without express or implied warranty.
+ *
+ * RED HAT, INC. DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS
+ * SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
+ * FITNESS, IN NO EVENT SHALL RED HAT, INC. BE LIABLE FOR ANY SPECIAL,
+ * INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
+ * RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
+ * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR
+ * IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ *
+ * Author: Soren Sandmann <sandmann@redhat.com>
+ */
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#undef PIXMAN_DISABLE_DEPRECATED
+
+#include "pixman-private.h"
+
+#include <stdlib.h>
+
+typedef pixman_box16_t		box_type_t;
+typedef pixman_region16_data_t	region_data_type_t;
+typedef pixman_region16_t	region_type_t;
+
+typedef struct {
+    int x, y;
+} point_type_t;
+
+#define PREFIX(x) pixman_region##x
+
+#include "pixman-region.c"
+
+/* This function exists only to make it possible to preserve the X ABI -
+ * it should go away at first opportunity.
+ *
+ * The problem is that the X ABI exports the three structs and has used
+ * them through macros. So the X server calls this function with
+ * the addresses of those structs which makes the existing code continue to
+ * work.
+ */
+PIXMAN_EXPORT void
+pixman_region_set_static_pointers (pixman_box16_t *empty_box,
+				   pixman_region16_data_t *empty_data,
+				   pixman_region16_data_t *broken_data)
+{
+    pixman_region_empty_box = empty_box;
+    pixman_region_empty_data = empty_data;
+    pixman_broken_data = broken_data;
+}
diff --git a/pixman/pixman/pixman-region32.c b/pixman/pixman/pixman-region32.c
new file mode 100644
index 000000000..aeee86cf9
--- /dev/null
+++ b/pixman/pixman/pixman-region32.c
@@ -0,0 +1,43 @@
+/*
+ * Copyright © 2008 Red Hat, Inc.
+ *
+ * Permission to use, copy, modify, distribute, and sell this software
+ * and its documentation for any purpose is hereby granted without
+ * fee, provided that the above copyright notice appear in all copies
+ * and that both that copyright notice and this permission notice
+ * appear in supporting documentation, and that the name of
+ * Red Hat, Inc. not be used in advertising or publicity pertaining to
+ * distribution of the software without specific, written prior
+ * permission. Red Hat, Inc. makes no representations about the
+ * suitability of this software for any purpose.  It is provided "as
+ * is" without express or implied warranty.
+ *
+ * RED HAT, INC. DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS
+ * SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
+ * FITNESS, IN NO EVENT SHALL RED HAT, INC. BE LIABLE FOR ANY SPECIAL,
+ * INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
+ * RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
+ * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR
+ * IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ *
+ * Author: Soren Sandmann <sandmann@redhat.com>
+ */
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include "pixman-private.h"
+
+#include <stdlib.h>
+
+typedef pixman_box32_t		box_type_t;
+typedef pixman_region32_data_t	region_data_type_t;
+typedef pixman_region32_t	region_type_t;
+
+typedef struct {
+    int x, y;
+} point_type_t;
+
+#define PREFIX(x) pixman_region32##x
+
+#include "pixman-region.c"
diff --git a/pixman/pixman/pixman-solid-fill.c b/pixman/pixman/pixman-solid-fill.c
new file mode 100644
index 000000000..48c999a0e
--- /dev/null
+++ b/pixman/pixman/pixman-solid-fill.c
@@ -0,0 +1,120 @@
+/*
+ * Copyright © 2000 SuSE, Inc.
+ * Copyright © 2007, 2009 Red Hat, Inc.
+ * Copyright © 2009 Soren Sandmann
+ *
+ * Permission to use, copy, modify, distribute, and sell this software and its
+ * documentation for any purpose is hereby granted without fee, provided that
+ * the above copyright notice appear in all copies and that both that
+ * copyright notice and this permission notice appear in supporting
+ * documentation, and that the name of SuSE not be used in advertising or
+ * publicity pertaining to distribution of the software without specific,
+ * written prior permission.  SuSE makes no representations about the
+ * suitability of this software for any purpose.  It is provided "as is"
+ * without express or implied warranty.
+ *
+ * SuSE DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING ALL
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL SuSE
+ * BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
+ * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+ * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+#include "pixman-private.h"
+
+static void
+solid_fill_get_scanline_32 (pixman_image_t *image,
+                            int             x,
+                            int             y,
+                            int             width,
+                            uint32_t *      buffer,
+                            const uint32_t *mask,
+                            uint32_t        mask_bits)
+{
+    uint32_t *end = buffer + width;
+    uint32_t color = image->solid.color_32;
+
+    while (buffer < end)
+	*(buffer++) = color;
+
+    return;
+}
+
+static void
+solid_fill_get_scanline_64 (pixman_image_t *image,
+			    int             x,
+			    int             y,
+			    int             width,
+			    uint32_t *      buffer,
+			    const uint32_t *mask,
+			    uint32_t        mask_bits)
+{
+    uint64_t *b = (uint64_t *)buffer;
+    uint64_t *e = b + width;
+    uint64_t color = image->solid.color_64;
+
+    while (b < e)
+	*(b++) = color;
+}
+
+static source_image_class_t
+solid_fill_classify (pixman_image_t *image,
+                     int             x,
+                     int             y,
+                     int             width,
+                     int             height)
+{
+    return (image->source.class = SOURCE_IMAGE_CLASS_HORIZONTAL);
+}
+
+static void
+solid_fill_property_changed (pixman_image_t *image)
+{
+    image->common.get_scanline_32 = solid_fill_get_scanline_32;
+    image->common.get_scanline_64 = solid_fill_get_scanline_64;
+}
+
+static uint32_t
+color_to_uint32 (const pixman_color_t *color)
+{
+    return
+        (color->alpha >> 8 << 24) |
+        (color->red >> 8 << 16) |
+        (color->green & 0xff00) |
+        (color->blue >> 8);
+}
+
+static uint64_t
+color_to_uint64 (const pixman_color_t *color)
+{
+    return
+        ((uint64_t)color->alpha << 48) |
+        ((uint64_t)color->red << 32) |
+        ((uint64_t)color->green << 16) |
+        ((uint64_t)color->blue);
+}
+
+PIXMAN_EXPORT pixman_image_t *
+pixman_image_create_solid_fill (pixman_color_t *color)
+{
+    pixman_image_t *img = _pixman_image_allocate ();
+
+    if (!img)
+	return NULL;
+
+    img->type = SOLID;
+    img->solid.color = *color;
+    img->solid.color_32 = color_to_uint32 (color);
+    img->solid.color_64 = color_to_uint64 (color);
+
+    img->source.class = SOURCE_IMAGE_CLASS_UNKNOWN;
+    img->common.classify = solid_fill_classify;
+    img->common.property_changed = solid_fill_property_changed;
+
+    return img;
+}
+
diff --git a/pixman/pixman/pixman-sse2.c b/pixman/pixman/pixman-sse2.c
new file mode 100644
index 000000000..946e7ba37
--- /dev/null
+++ b/pixman/pixman/pixman-sse2.c
@@ -0,0 +1,5972 @@
+/*
+ * Copyright © 2008 Rodrigo Kumpera
+ * Copyright © 2008 André Tupinambá
+ *
+ * Permission to use, copy, modify, distribute, and sell this software and its
+ * documentation for any purpose is hereby granted without fee, provided that
+ * the above copyright notice appear in all copies and that both that
+ * copyright notice and this permission notice appear in supporting
+ * documentation, and that the name of Red Hat not be used in advertising or
+ * publicity pertaining to distribution of the software without specific,
+ * written prior permission.  Red Hat makes no representations about the
+ * suitability of this software for any purpose.  It is provided "as is"
+ * without express or implied warranty.
+ *
+ * THE COPYRIGHT HOLDERS DISCLAIM ALL WARRANTIES WITH REGARD TO THIS
+ * SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
+ * FITNESS, IN NO EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
+ * AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
+ * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS
+ * SOFTWARE.
+ *
+ * Author:  Rodrigo Kumpera (kumpera@gmail.com)
+ *          André Tupinambá (andrelrt@gmail.com)
+ *
+ * Based on work by Owen Taylor and Søren Sandmann
+ */
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <mmintrin.h>
+#include <xmmintrin.h> /* for _mm_shuffle_pi16 and _MM_SHUFFLE */
+#include <emmintrin.h> /* for SSE2 intrinsics */
+#include "pixman-private.h"
+#include "pixman-combine32.h"
+
+#if defined(_MSC_VER) && defined(_M_AMD64)
+/* Windows 64 doesn't allow MMX to be used, so
+ * the pixman-x64-mmx-emulation.h file contains
+ * implementations of those MMX intrinsics that
+ * are used in the SSE2 implementation.
+ */
+#   include "pixman-x64-mmx-emulation.h"
+#endif
+
+#ifdef USE_SSE2
+
+/* --------------------------------------------------------------------
+ * Locals
+ */
+
+static __m64 mask_x0080;
+static __m64 mask_x00ff;
+static __m64 mask_x0101;
+static __m64 mask_x_alpha;
+
+static __m64 mask_x565_rgb;
+static __m64 mask_x565_unpack;
+
+static __m128i mask_0080;
+static __m128i mask_00ff;
+static __m128i mask_0101;
+static __m128i mask_ffff;
+static __m128i mask_ff000000;
+static __m128i mask_alpha;
+
+static __m128i mask_565_r;
+static __m128i mask_565_g1, mask_565_g2;
+static __m128i mask_565_b;
+static __m128i mask_red;
+static __m128i mask_green;
+static __m128i mask_blue;
+
+static __m128i mask_565_fix_rb;
+static __m128i mask_565_fix_g;
+
+/* ----------------------------------------------------------------------
+ * SSE2 Inlines
+ */
+static force_inline __m128i
+unpack_32_1x128 (uint32_t data)
+{
+    return _mm_unpacklo_epi8 (_mm_cvtsi32_si128 (data), _mm_setzero_si128 ());
+}
+
+static force_inline void
+unpack_128_2x128 (__m128i data, __m128i* data_lo, __m128i* data_hi)
+{
+    *data_lo = _mm_unpacklo_epi8 (data, _mm_setzero_si128 ());
+    *data_hi = _mm_unpackhi_epi8 (data, _mm_setzero_si128 ());
+}
+
+static force_inline __m128i
+unpack_565_to_8888 (__m128i lo)
+{
+    __m128i r, g, b, rb, t;
+
+    r = _mm_and_si128 (_mm_slli_epi32 (lo, 8), mask_red);
+    g = _mm_and_si128 (_mm_slli_epi32 (lo, 5), mask_green);
+    b = _mm_and_si128 (_mm_slli_epi32 (lo, 3), mask_blue);
+
+    rb = _mm_or_si128 (r, b);
+    t  = _mm_and_si128 (rb, mask_565_fix_rb);
+    t  = _mm_srli_epi32 (t, 5);
+    rb = _mm_or_si128 (rb, t);
+
+    t  = _mm_and_si128 (g, mask_565_fix_g);
+    t  = _mm_srli_epi32 (t, 6);
+    g  = _mm_or_si128 (g, t);
+
+    return _mm_or_si128 (rb, g);
+}
+
+static force_inline void
+unpack_565_128_4x128 (__m128i  data,
+                      __m128i* data0,
+                      __m128i* data1,
+                      __m128i* data2,
+                      __m128i* data3)
+{
+    __m128i lo, hi;
+
+    lo = _mm_unpacklo_epi16 (data, _mm_setzero_si128 ());
+    hi = _mm_unpackhi_epi16 (data, _mm_setzero_si128 ());
+
+    lo = unpack_565_to_8888 (lo);
+    hi = unpack_565_to_8888 (hi);
+
+    unpack_128_2x128 (lo, data0, data1);
+    unpack_128_2x128 (hi, data2, data3);
+}
+
+static force_inline uint16_t
+pack_565_32_16 (uint32_t pixel)
+{
+    return (uint16_t) (((pixel >> 8) & 0xf800) |
+		       ((pixel >> 5) & 0x07e0) |
+		       ((pixel >> 3) & 0x001f));
+}
+
+static force_inline __m128i
+pack_2x128_128 (__m128i lo, __m128i hi)
+{
+    return _mm_packus_epi16 (lo, hi);
+}
+
+static force_inline __m128i
+pack_565_2x128_128 (__m128i lo, __m128i hi)
+{
+    __m128i data;
+    __m128i r, g1, g2, b;
+
+    data = pack_2x128_128 (lo, hi);
+
+    r  = _mm_and_si128 (data, mask_565_r);
+    g1 = _mm_and_si128 (_mm_slli_epi32 (data, 3), mask_565_g1);
+    g2 = _mm_and_si128 (_mm_srli_epi32 (data, 5), mask_565_g2);
+    b  = _mm_and_si128 (_mm_srli_epi32 (data, 3), mask_565_b);
+
+    return _mm_or_si128 (_mm_or_si128 (_mm_or_si128 (r, g1), g2), b);
+}
+
+static force_inline __m128i
+pack_565_4x128_128 (__m128i* xmm0, __m128i* xmm1, __m128i* xmm2, __m128i* xmm3)
+{
+    return _mm_packus_epi16 (pack_565_2x128_128 (*xmm0, *xmm1),
+			     pack_565_2x128_128 (*xmm2, *xmm3));
+}
+
+static force_inline int
+is_opaque (__m128i x)
+{
+    __m128i ffs = _mm_cmpeq_epi8 (x, x);
+
+    return (_mm_movemask_epi8 (_mm_cmpeq_epi8 (x, ffs)) & 0x8888) == 0x8888;
+}
+
+static force_inline int
+is_zero (__m128i x)
+{
+    return _mm_movemask_epi8 (
+	_mm_cmpeq_epi8 (x, _mm_setzero_si128 ())) == 0xffff;
+}
+
+static force_inline int
+is_transparent (__m128i x)
+{
+    return (_mm_movemask_epi8 (
+		_mm_cmpeq_epi8 (x, _mm_setzero_si128 ())) & 0x8888) == 0x8888;
+}
+
+static force_inline __m128i
+expand_pixel_32_1x128 (uint32_t data)
+{
+    return _mm_shuffle_epi32 (unpack_32_1x128 (data), _MM_SHUFFLE (1, 0, 1, 0));
+}
+
+static force_inline __m128i
+expand_alpha_1x128 (__m128i data)
+{
+    return _mm_shufflehi_epi16 (_mm_shufflelo_epi16 (data,
+						     _MM_SHUFFLE (3, 3, 3, 3)),
+				_MM_SHUFFLE (3, 3, 3, 3));
+}
+
+static force_inline void
+expand_alpha_2x128 (__m128i  data_lo,
+                    __m128i  data_hi,
+                    __m128i* alpha_lo,
+                    __m128i* alpha_hi)
+{
+    __m128i lo, hi;
+
+    lo = _mm_shufflelo_epi16 (data_lo, _MM_SHUFFLE (3, 3, 3, 3));
+    hi = _mm_shufflelo_epi16 (data_hi, _MM_SHUFFLE (3, 3, 3, 3));
+
+    *alpha_lo = _mm_shufflehi_epi16 (lo, _MM_SHUFFLE (3, 3, 3, 3));
+    *alpha_hi = _mm_shufflehi_epi16 (hi, _MM_SHUFFLE (3, 3, 3, 3));
+}
+
+static force_inline void
+expand_alpha_rev_2x128 (__m128i  data_lo,
+                        __m128i  data_hi,
+                        __m128i* alpha_lo,
+                        __m128i* alpha_hi)
+{
+    __m128i lo, hi;
+
+    lo = _mm_shufflelo_epi16 (data_lo, _MM_SHUFFLE (0, 0, 0, 0));
+    hi = _mm_shufflelo_epi16 (data_hi, _MM_SHUFFLE (0, 0, 0, 0));
+    *alpha_lo = _mm_shufflehi_epi16 (lo, _MM_SHUFFLE (0, 0, 0, 0));
+    *alpha_hi = _mm_shufflehi_epi16 (hi, _MM_SHUFFLE (0, 0, 0, 0));
+}
+
+static force_inline void
+pix_multiply_2x128 (__m128i* data_lo,
+                    __m128i* data_hi,
+                    __m128i* alpha_lo,
+                    __m128i* alpha_hi,
+                    __m128i* ret_lo,
+                    __m128i* ret_hi)
+{
+    __m128i lo, hi;
+
+    lo = _mm_mullo_epi16 (*data_lo, *alpha_lo);
+    hi = _mm_mullo_epi16 (*data_hi, *alpha_hi);
+    lo = _mm_adds_epu16 (lo, mask_0080);
+    hi = _mm_adds_epu16 (hi, mask_0080);
+    *ret_lo = _mm_mulhi_epu16 (lo, mask_0101);
+    *ret_hi = _mm_mulhi_epu16 (hi, mask_0101);
+}
+
+static force_inline void
+pix_add_multiply_2x128 (__m128i* src_lo,
+                        __m128i* src_hi,
+                        __m128i* alpha_dst_lo,
+                        __m128i* alpha_dst_hi,
+                        __m128i* dst_lo,
+                        __m128i* dst_hi,
+                        __m128i* alpha_src_lo,
+                        __m128i* alpha_src_hi,
+                        __m128i* ret_lo,
+                        __m128i* ret_hi)
+{
+    __m128i t1_lo, t1_hi;
+    __m128i t2_lo, t2_hi;
+
+    pix_multiply_2x128 (src_lo, src_hi, alpha_dst_lo, alpha_dst_hi, &t1_lo, &t1_hi);
+    pix_multiply_2x128 (dst_lo, dst_hi, alpha_src_lo, alpha_src_hi, &t2_lo, &t2_hi);
+
+    *ret_lo = _mm_adds_epu8 (t1_lo, t2_lo);
+    *ret_hi = _mm_adds_epu8 (t1_hi, t2_hi);
+}
+
+static force_inline void
+negate_2x128 (__m128i  data_lo,
+              __m128i  data_hi,
+              __m128i* neg_lo,
+              __m128i* neg_hi)
+{
+    *neg_lo = _mm_xor_si128 (data_lo, mask_00ff);
+    *neg_hi = _mm_xor_si128 (data_hi, mask_00ff);
+}
+
+static force_inline void
+invert_colors_2x128 (__m128i  data_lo,
+                     __m128i  data_hi,
+                     __m128i* inv_lo,
+                     __m128i* inv_hi)
+{
+    __m128i lo, hi;
+
+    lo = _mm_shufflelo_epi16 (data_lo, _MM_SHUFFLE (3, 0, 1, 2));
+    hi = _mm_shufflelo_epi16 (data_hi, _MM_SHUFFLE (3, 0, 1, 2));
+    *inv_lo = _mm_shufflehi_epi16 (lo, _MM_SHUFFLE (3, 0, 1, 2));
+    *inv_hi = _mm_shufflehi_epi16 (hi, _MM_SHUFFLE (3, 0, 1, 2));
+}
+
+static force_inline void
+over_2x128 (__m128i* src_lo,
+            __m128i* src_hi,
+            __m128i* alpha_lo,
+            __m128i* alpha_hi,
+            __m128i* dst_lo,
+            __m128i* dst_hi)
+{
+    __m128i t1, t2;
+
+    negate_2x128 (*alpha_lo, *alpha_hi, &t1, &t2);
+
+    pix_multiply_2x128 (dst_lo, dst_hi, &t1, &t2, dst_lo, dst_hi);
+
+    *dst_lo = _mm_adds_epu8 (*src_lo, *dst_lo);
+    *dst_hi = _mm_adds_epu8 (*src_hi, *dst_hi);
+}
+
+static force_inline void
+over_rev_non_pre_2x128 (__m128i  src_lo,
+                        __m128i  src_hi,
+                        __m128i* dst_lo,
+                        __m128i* dst_hi)
+{
+    __m128i lo, hi;
+    __m128i alpha_lo, alpha_hi;
+
+    expand_alpha_2x128 (src_lo, src_hi, &alpha_lo, &alpha_hi);
+
+    lo = _mm_or_si128 (alpha_lo, mask_alpha);
+    hi = _mm_or_si128 (alpha_hi, mask_alpha);
+
+    invert_colors_2x128 (src_lo, src_hi, &src_lo, &src_hi);
+
+    pix_multiply_2x128 (&src_lo, &src_hi, &lo, &hi, &lo, &hi);
+
+    over_2x128 (&lo, &hi, &alpha_lo, &alpha_hi, dst_lo, dst_hi);
+}
+
+static force_inline void
+in_over_2x128 (__m128i* src_lo,
+               __m128i* src_hi,
+               __m128i* alpha_lo,
+               __m128i* alpha_hi,
+               __m128i* mask_lo,
+               __m128i* mask_hi,
+               __m128i* dst_lo,
+               __m128i* dst_hi)
+{
+    __m128i s_lo, s_hi;
+    __m128i a_lo, a_hi;
+
+    pix_multiply_2x128 (src_lo,   src_hi, mask_lo, mask_hi, &s_lo, &s_hi);
+    pix_multiply_2x128 (alpha_lo, alpha_hi, mask_lo, mask_hi, &a_lo, &a_hi);
+
+    over_2x128 (&s_lo, &s_hi, &a_lo, &a_hi, dst_lo, dst_hi);
+}
+
+static force_inline void
+cache_prefetch (__m128i* addr)
+{
+    _mm_prefetch ((void const*)addr, _MM_HINT_T0);
+}
+
+static force_inline void
+cache_prefetch_next (__m128i* addr)
+{
+    _mm_prefetch ((void const *)(addr + 4), _MM_HINT_T0); /* 64 bytes ahead */
+}
+
+/* prefetching NULL is very slow on some systems. don't do that. */
+
+static force_inline void
+maybe_prefetch (__m128i* addr)
+{
+    if (addr)
+	cache_prefetch (addr);
+}
+
+static force_inline void
+maybe_prefetch_next (__m128i* addr)
+{
+    if (addr)
+	cache_prefetch_next (addr);
+}
+
+/* load 4 pixels from a 16-byte boundary aligned address */
+static force_inline __m128i
+load_128_aligned (__m128i* src)
+{
+    return _mm_load_si128 (src);
+}
+
+/* load 4 pixels from a unaligned address */
+static force_inline __m128i
+load_128_unaligned (const __m128i* src)
+{
+    return _mm_loadu_si128 (src);
+}
+
+/* save 4 pixels using Write Combining memory on a 16-byte
+ * boundary aligned address
+ */
+static force_inline void
+save_128_write_combining (__m128i* dst,
+                          __m128i  data)
+{
+    _mm_stream_si128 (dst, data);
+}
+
+/* save 4 pixels on a 16-byte boundary aligned address */
+static force_inline void
+save_128_aligned (__m128i* dst,
+                  __m128i  data)
+{
+    _mm_store_si128 (dst, data);
+}
+
+/* save 4 pixels on a unaligned address */
+static force_inline void
+save_128_unaligned (__m128i* dst,
+                    __m128i  data)
+{
+    _mm_storeu_si128 (dst, data);
+}
+
+/* ------------------------------------------------------------------
+ * MMX inlines
+ */
+
+static force_inline __m64
+load_32_1x64 (uint32_t data)
+{
+    return _mm_cvtsi32_si64 (data);
+}
+
+static force_inline __m64
+unpack_32_1x64 (uint32_t data)
+{
+    return _mm_unpacklo_pi8 (load_32_1x64 (data), _mm_setzero_si64 ());
+}
+
+static force_inline __m64
+expand_alpha_1x64 (__m64 data)
+{
+    return _mm_shuffle_pi16 (data, _MM_SHUFFLE (3, 3, 3, 3));
+}
+
+static force_inline __m64
+expand_alpha_rev_1x64 (__m64 data)
+{
+    return _mm_shuffle_pi16 (data, _MM_SHUFFLE (0, 0, 0, 0));
+}
+
+static force_inline __m64
+expand_pixel_8_1x64 (uint8_t data)
+{
+    return _mm_shuffle_pi16 (
+	unpack_32_1x64 ((uint32_t)data), _MM_SHUFFLE (0, 0, 0, 0));
+}
+
+static force_inline __m64
+pix_multiply_1x64 (__m64 data,
+                   __m64 alpha)
+{
+    return _mm_mulhi_pu16 (_mm_adds_pu16 (_mm_mullo_pi16 (data, alpha),
+                                          mask_x0080),
+                           mask_x0101);
+}
+
+static force_inline __m64
+pix_add_multiply_1x64 (__m64* src,
+                       __m64* alpha_dst,
+                       __m64* dst,
+                       __m64* alpha_src)
+{
+    __m64 t1 = pix_multiply_1x64 (*src, *alpha_dst);
+    __m64 t2 = pix_multiply_1x64 (*dst, *alpha_src);
+
+    return _mm_adds_pu8 (t1, t2);
+}
+
+static force_inline __m64
+negate_1x64 (__m64 data)
+{
+    return _mm_xor_si64 (data, mask_x00ff);
+}
+
+static force_inline __m64
+invert_colors_1x64 (__m64 data)
+{
+    return _mm_shuffle_pi16 (data, _MM_SHUFFLE (3, 0, 1, 2));
+}
+
+static force_inline __m64
+over_1x64 (__m64 src, __m64 alpha, __m64 dst)
+{
+    return _mm_adds_pu8 (src, pix_multiply_1x64 (dst, negate_1x64 (alpha)));
+}
+
+static force_inline __m64
+in_over_1x64 (__m64* src, __m64* alpha, __m64* mask, __m64* dst)
+{
+    return over_1x64 (pix_multiply_1x64 (*src, *mask),
+                      pix_multiply_1x64 (*alpha, *mask),
+                      *dst);
+}
+
+static force_inline __m64
+over_rev_non_pre_1x64 (__m64 src, __m64 dst)
+{
+    __m64 alpha = expand_alpha_1x64 (src);
+
+    return over_1x64 (pix_multiply_1x64 (invert_colors_1x64 (src),
+                                         _mm_or_si64 (alpha, mask_x_alpha)),
+                      alpha,
+                      dst);
+}
+
+static force_inline uint32_t
+pack_1x64_32 (__m64 data)
+{
+    return _mm_cvtsi64_si32 (_mm_packs_pu16 (data, _mm_setzero_si64 ()));
+}
+
+/* Expand 16 bits positioned at @pos (0-3) of a mmx register into
+ *
+ *    00RR00GG00BB
+ *
+ * --- Expanding 565 in the low word ---
+ *
+ * m = (m << (32 - 3)) | (m << (16 - 5)) | m;
+ * m = m & (01f0003f001f);
+ * m = m * (008404100840);
+ * m = m >> 8;
+ *
+ * Note the trick here - the top word is shifted by another nibble to
+ * avoid it bumping into the middle word
+ */
+static force_inline __m64
+expand565_16_1x64 (uint16_t pixel)
+{
+    __m64 p;
+    __m64 t1, t2;
+
+    p = _mm_cvtsi32_si64 ((uint32_t) pixel);
+
+    t1 = _mm_slli_si64 (p, 36 - 11);
+    t2 = _mm_slli_si64 (p, 16 - 5);
+
+    p = _mm_or_si64 (t1, p);
+    p = _mm_or_si64 (t2, p);
+    p = _mm_and_si64 (p, mask_x565_rgb);
+    p = _mm_mullo_pi16 (p, mask_x565_unpack);
+
+    return _mm_srli_pi16 (p, 8);
+}
+
+/* ----------------------------------------------------------------------------
+ * Compose Core transformations
+ */
+static force_inline uint32_t
+core_combine_over_u_pixel_sse2 (uint32_t src, uint32_t dst)
+{
+    uint8_t a;
+    __m64 ms;
+
+    a = src >> 24;
+
+    if (a == 0xff)
+    {
+	return src;
+    }
+    else if (src)
+    {
+	ms = unpack_32_1x64 (src);
+	return pack_1x64_32 (
+	    over_1x64 (ms, expand_alpha_1x64 (ms), unpack_32_1x64 (dst)));
+    }
+
+    return dst;
+}
+
+static force_inline uint32_t
+combine1 (const uint32_t *ps, const uint32_t *pm)
+{
+    uint32_t s = *ps;
+
+    if (pm)
+    {
+	__m64 ms, mm;
+
+	mm = unpack_32_1x64 (*pm);
+	mm = expand_alpha_1x64 (mm);
+
+	ms = unpack_32_1x64 (s);
+	ms = pix_multiply_1x64 (ms, mm);
+
+	s = pack_1x64_32 (ms);
+    }
+
+    return s;
+}
+
+static force_inline __m128i
+combine4 (const __m128i *ps, const __m128i *pm)
+{
+    __m128i xmm_src_lo, xmm_src_hi;
+    __m128i xmm_msk_lo, xmm_msk_hi;
+    __m128i s;
+
+    if (pm)
+    {
+	xmm_msk_lo = load_128_unaligned (pm);
+
+	if (is_transparent (xmm_msk_lo))
+	    return _mm_setzero_si128 ();
+    }
+
+    s = load_128_unaligned (ps);
+
+    if (pm)
+    {
+	unpack_128_2x128 (s, &xmm_src_lo, &xmm_src_hi);
+	unpack_128_2x128 (xmm_msk_lo, &xmm_msk_lo, &xmm_msk_hi);
+
+	expand_alpha_2x128 (xmm_msk_lo, xmm_msk_hi, &xmm_msk_lo, &xmm_msk_hi);
+
+	pix_multiply_2x128 (&xmm_src_lo, &xmm_src_hi,
+			    &xmm_msk_lo, &xmm_msk_hi,
+			    &xmm_src_lo, &xmm_src_hi);
+
+	s = pack_2x128_128 (xmm_src_lo, xmm_src_hi);
+    }
+
+    return s;
+}
+
+static force_inline void
+core_combine_over_u_sse2 (uint32_t*       pd,
+                          const uint32_t* ps,
+                          const uint32_t* pm,
+                          int             w)
+{
+    uint32_t s, d;
+
+    __m128i xmm_dst_lo, xmm_dst_hi;
+    __m128i xmm_src_lo, xmm_src_hi;
+    __m128i xmm_alpha_lo, xmm_alpha_hi;
+
+    /* call prefetch hint to optimize cache load*/
+    cache_prefetch ((__m128i*)ps);
+    cache_prefetch ((__m128i*)pd);
+    maybe_prefetch ((__m128i*)pm);
+
+    /* Align dst on a 16-byte boundary */
+    while (w && ((unsigned long)pd & 15))
+    {
+	d = *pd;
+	s = combine1 (ps, pm);
+
+	*pd++ = core_combine_over_u_pixel_sse2 (s, d);
+	ps++;
+	if (pm)
+	    pm++;
+	w--;
+    }
+
+    /* call prefetch hint to optimize cache load*/
+    cache_prefetch ((__m128i*)ps);
+    cache_prefetch ((__m128i*)pd);
+    maybe_prefetch ((__m128i*)pm);
+
+    while (w >= 4)
+    {
+	/* fill cache line with next memory */
+	cache_prefetch_next ((__m128i*)ps);
+	cache_prefetch_next ((__m128i*)pd);
+	maybe_prefetch_next ((__m128i*)pm);
+
+	/* I'm loading unaligned because I'm not sure about
+	 * the address alignment.
+	 */
+	xmm_src_hi = combine4 ((__m128i*)ps, (__m128i*)pm);
+
+	if (is_opaque (xmm_src_hi))
+	{
+	    save_128_aligned ((__m128i*)pd, xmm_src_hi);
+	}
+	else if (!is_zero (xmm_src_hi))
+	{
+	    xmm_dst_hi = load_128_aligned ((__m128i*) pd);
+
+	    unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
+	    unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
+
+	    expand_alpha_2x128 (
+		xmm_src_lo, xmm_src_hi, &xmm_alpha_lo, &xmm_alpha_hi);
+
+	    over_2x128 (&xmm_src_lo, &xmm_src_hi,
+			&xmm_alpha_lo, &xmm_alpha_hi,
+			&xmm_dst_lo, &xmm_dst_hi);
+
+	    /* rebuid the 4 pixel data and save*/
+	    save_128_aligned ((__m128i*)pd,
+			      pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
+	}
+
+	w -= 4;
+	ps += 4;
+	pd += 4;
+	if (pm)
+	    pm += 4;
+    }
+
+    while (w)
+    {
+	d = *pd;
+	s = combine1 (ps, pm);
+
+	*pd++ = core_combine_over_u_pixel_sse2 (s, d);
+	ps++;
+	if (pm)
+	    pm++;
+
+	w--;
+    }
+}
+
+static force_inline void
+core_combine_over_reverse_u_sse2 (uint32_t*       pd,
+                                  const uint32_t* ps,
+                                  const uint32_t* pm,
+                                  int             w)
+{
+    uint32_t s, d;
+
+    __m128i xmm_dst_lo, xmm_dst_hi;
+    __m128i xmm_src_lo, xmm_src_hi;
+    __m128i xmm_alpha_lo, xmm_alpha_hi;
+
+    /* call prefetch hint to optimize cache load*/
+    cache_prefetch ((__m128i*)ps);
+    cache_prefetch ((__m128i*)pd);
+    maybe_prefetch ((__m128i*)pm);
+
+    /* Align dst on a 16-byte boundary */
+    while (w &&
+           ((unsigned long)pd & 15))
+    {
+	d = *pd;
+	s = combine1 (ps, pm);
+
+	*pd++ = core_combine_over_u_pixel_sse2 (d, s);
+	w--;
+	ps++;
+	if (pm)
+	    pm++;
+    }
+
+    /* call prefetch hint to optimize cache load*/
+    cache_prefetch ((__m128i*)ps);
+    cache_prefetch ((__m128i*)pd);
+    maybe_prefetch ((__m128i*)pm);
+
+    while (w >= 4)
+    {
+	/* fill cache line with next memory */
+	cache_prefetch_next ((__m128i*)ps);
+	cache_prefetch_next ((__m128i*)pd);
+	maybe_prefetch_next ((__m128i*)pm);
+
+	/* I'm loading unaligned because I'm not sure
+	 * about the address alignment.
+	 */
+	xmm_src_hi = combine4 ((__m128i*)ps, (__m128i*)pm);
+	xmm_dst_hi = load_128_aligned ((__m128i*) pd);
+
+	unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
+	unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
+
+	expand_alpha_2x128 (xmm_dst_lo, xmm_dst_hi,
+			    &xmm_alpha_lo, &xmm_alpha_hi);
+
+	over_2x128 (&xmm_dst_lo, &xmm_dst_hi,
+		    &xmm_alpha_lo, &xmm_alpha_hi,
+		    &xmm_src_lo, &xmm_src_hi);
+
+	/* rebuid the 4 pixel data and save*/
+	save_128_aligned ((__m128i*)pd,
+			  pack_2x128_128 (xmm_src_lo, xmm_src_hi));
+
+	w -= 4;
+	ps += 4;
+	pd += 4;
+
+	if (pm)
+	    pm += 4;
+    }
+
+    while (w)
+    {
+	d = *pd;
+	s = combine1 (ps, pm);
+
+	*pd++ = core_combine_over_u_pixel_sse2 (d, s);
+	ps++;
+	w--;
+	if (pm)
+	    pm++;
+    }
+}
+
+static force_inline uint32_t
+core_combine_in_u_pixelsse2 (uint32_t src, uint32_t dst)
+{
+    uint32_t maska = src >> 24;
+
+    if (maska == 0)
+    {
+	return 0;
+    }
+    else if (maska != 0xff)
+    {
+	return pack_1x64_32 (
+	    pix_multiply_1x64 (unpack_32_1x64 (dst),
+			       expand_alpha_1x64 (unpack_32_1x64 (src))));
+    }
+
+    return dst;
+}
+
+static force_inline void
+core_combine_in_u_sse2 (uint32_t*       pd,
+                        const uint32_t* ps,
+                        const uint32_t* pm,
+                        int             w)
+{
+    uint32_t s, d;
+
+    __m128i xmm_src_lo, xmm_src_hi;
+    __m128i xmm_dst_lo, xmm_dst_hi;
+
+    /* call prefetch hint to optimize cache load*/
+    cache_prefetch ((__m128i*)ps);
+    cache_prefetch ((__m128i*)pd);
+    maybe_prefetch ((__m128i*)pm);
+
+    while (w && ((unsigned long) pd & 15))
+    {
+	s = combine1 (ps, pm);
+	d = *pd;
+
+	*pd++ = core_combine_in_u_pixelsse2 (d, s);
+	w--;
+	ps++;
+	if (pm)
+	    pm++;
+    }
+
+    /* call prefetch hint to optimize cache load*/
+    cache_prefetch ((__m128i*)ps);
+    cache_prefetch ((__m128i*)pd);
+    maybe_prefetch ((__m128i*)pm);
+
+    while (w >= 4)
+    {
+	/* fill cache line with next memory */
+	cache_prefetch_next ((__m128i*)ps);
+	cache_prefetch_next ((__m128i*)pd);
+	maybe_prefetch_next ((__m128i*)pm);
+
+	xmm_dst_hi = load_128_aligned ((__m128i*) pd);
+	xmm_src_hi = combine4 ((__m128i*) ps, (__m128i*) pm);
+
+	unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
+	expand_alpha_2x128 (xmm_dst_lo, xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
+
+	unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
+	pix_multiply_2x128 (&xmm_src_lo, &xmm_src_hi,
+			    &xmm_dst_lo, &xmm_dst_hi,
+			    &xmm_dst_lo, &xmm_dst_hi);
+
+	save_128_aligned ((__m128i*)pd,
+			  pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
+
+	ps += 4;
+	pd += 4;
+	w -= 4;
+	if (pm)
+	    pm += 4;
+    }
+
+    while (w)
+    {
+	s = combine1 (ps, pm);
+	d = *pd;
+
+	*pd++ = core_combine_in_u_pixelsse2 (d, s);
+	w--;
+	ps++;
+	if (pm)
+	    pm++;
+    }
+}
+
+static force_inline void
+core_combine_reverse_in_u_sse2 (uint32_t*       pd,
+                                const uint32_t* ps,
+                                const uint32_t *pm,
+                                int             w)
+{
+    uint32_t s, d;
+
+    __m128i xmm_src_lo, xmm_src_hi;
+    __m128i xmm_dst_lo, xmm_dst_hi;
+
+    /* call prefetch hint to optimize cache load*/
+    cache_prefetch ((__m128i*)ps);
+    cache_prefetch ((__m128i*)pd);
+    maybe_prefetch ((__m128i*)pm);
+
+    while (w && ((unsigned long) pd & 15))
+    {
+	s = combine1 (ps, pm);
+	d = *pd;
+
+	*pd++ = core_combine_in_u_pixelsse2 (s, d);
+	ps++;
+	w--;
+	if (pm)
+	    pm++;
+    }
+
+    /* call prefetch hint to optimize cache load*/
+    cache_prefetch ((__m128i*)ps);
+    cache_prefetch ((__m128i*)pd);
+    maybe_prefetch ((__m128i*)pm);
+
+    while (w >= 4)
+    {
+	/* fill cache line with next memory */
+	cache_prefetch_next ((__m128i*)ps);
+	cache_prefetch_next ((__m128i*)pd);
+	maybe_prefetch_next ((__m128i*)pm);
+
+	xmm_dst_hi = load_128_aligned ((__m128i*) pd);
+	xmm_src_hi = combine4 ((__m128i*) ps, (__m128i*)pm);
+
+	unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
+	expand_alpha_2x128 (xmm_src_lo, xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
+
+	unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
+	pix_multiply_2x128 (&xmm_dst_lo, &xmm_dst_hi,
+			    &xmm_src_lo, &xmm_src_hi,
+			    &xmm_dst_lo, &xmm_dst_hi);
+
+	save_128_aligned (
+	    (__m128i*)pd, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
+
+	ps += 4;
+	pd += 4;
+	w -= 4;
+	if (pm)
+	    pm += 4;
+    }
+
+    while (w)
+    {
+	s = combine1 (ps, pm);
+	d = *pd;
+
+	*pd++ = core_combine_in_u_pixelsse2 (s, d);
+	w--;
+	ps++;
+	if (pm)
+	    pm++;
+    }
+}
+
+static force_inline void
+core_combine_reverse_out_u_sse2 (uint32_t*       pd,
+                                 const uint32_t* ps,
+                                 const uint32_t* pm,
+                                 int             w)
+{
+    /* call prefetch hint to optimize cache load*/
+    cache_prefetch ((__m128i*)ps);
+    cache_prefetch ((__m128i*)pd);
+    maybe_prefetch ((__m128i*)pm);
+
+    while (w && ((unsigned long) pd & 15))
+    {
+	uint32_t s = combine1 (ps, pm);
+	uint32_t d = *pd;
+
+	*pd++ = pack_1x64_32 (
+	    pix_multiply_1x64 (
+		unpack_32_1x64 (d), negate_1x64 (
+		    expand_alpha_1x64 (unpack_32_1x64 (s)))));
+
+	if (pm)
+	    pm++;
+	ps++;
+	w--;
+    }
+
+    /* call prefetch hint to optimize cache load*/
+    cache_prefetch ((__m128i*)ps);
+    cache_prefetch ((__m128i*)pd);
+    maybe_prefetch ((__m128i*)pm);
+
+    while (w >= 4)
+    {
+	__m128i xmm_src_lo, xmm_src_hi;
+	__m128i xmm_dst_lo, xmm_dst_hi;
+
+	/* fill cache line with next memory */
+	cache_prefetch_next ((__m128i*)ps);
+	cache_prefetch_next ((__m128i*)pd);
+	maybe_prefetch_next ((__m128i*)pm);
+
+	xmm_src_hi = combine4 ((__m128i*)ps, (__m128i*)pm);
+	xmm_dst_hi = load_128_aligned ((__m128i*) pd);
+
+	unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
+	unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
+
+	expand_alpha_2x128 (xmm_src_lo, xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
+	negate_2x128       (xmm_src_lo, xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
+
+	pix_multiply_2x128 (&xmm_dst_lo, &xmm_dst_hi,
+			    &xmm_src_lo, &xmm_src_hi,
+			    &xmm_dst_lo, &xmm_dst_hi);
+
+	save_128_aligned (
+	    (__m128i*)pd, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
+
+	ps += 4;
+	pd += 4;
+	if (pm)
+	    pm += 4;
+
+	w -= 4;
+    }
+
+    while (w)
+    {
+	uint32_t s = combine1 (ps, pm);
+	uint32_t d = *pd;
+
+	*pd++ = pack_1x64_32 (
+	    pix_multiply_1x64 (
+		unpack_32_1x64 (d), negate_1x64 (
+		    expand_alpha_1x64 (unpack_32_1x64 (s)))));
+	ps++;
+	if (pm)
+	    pm++;
+	w--;
+    }
+}
+
+static force_inline void
+core_combine_out_u_sse2 (uint32_t*       pd,
+                         const uint32_t* ps,
+                         const uint32_t* pm,
+                         int             w)
+{
+    /* call prefetch hint to optimize cache load*/
+    cache_prefetch ((__m128i*)ps);
+    cache_prefetch ((__m128i*)pd);
+    maybe_prefetch ((__m128i*)pm);
+
+    while (w && ((unsigned long) pd & 15))
+    {
+	uint32_t s = combine1 (ps, pm);
+	uint32_t d = *pd;
+
+	*pd++ = pack_1x64_32 (
+	    pix_multiply_1x64 (
+		unpack_32_1x64 (s), negate_1x64 (
+		    expand_alpha_1x64 (unpack_32_1x64 (d)))));
+	w--;
+	ps++;
+	if (pm)
+	    pm++;
+    }
+
+    /* call prefetch hint to optimize cache load*/
+    cache_prefetch ((__m128i*)ps);
+    cache_prefetch ((__m128i*)pd);
+    maybe_prefetch ((__m128i*)pm);
+
+    while (w >= 4)
+    {
+	__m128i xmm_src_lo, xmm_src_hi;
+	__m128i xmm_dst_lo, xmm_dst_hi;
+
+	/* fill cache line with next memory */
+	cache_prefetch_next ((__m128i*)ps);
+	cache_prefetch_next ((__m128i*)pd);
+	maybe_prefetch_next ((__m128i*)pm);
+
+	xmm_src_hi = combine4 ((__m128i*) ps, (__m128i*)pm);
+	xmm_dst_hi = load_128_aligned ((__m128i*) pd);
+
+	unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
+	unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
+
+	expand_alpha_2x128 (xmm_dst_lo, xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
+	negate_2x128       (xmm_dst_lo, xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
+
+	pix_multiply_2x128 (&xmm_src_lo, &xmm_src_hi,
+			    &xmm_dst_lo, &xmm_dst_hi,
+			    &xmm_dst_lo, &xmm_dst_hi);
+
+	save_128_aligned (
+	    (__m128i*)pd, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
+
+	ps += 4;
+	pd += 4;
+	w -= 4;
+	if (pm)
+	    pm += 4;
+    }
+
+    while (w)
+    {
+	uint32_t s = combine1 (ps, pm);
+	uint32_t d = *pd;
+
+	*pd++ = pack_1x64_32 (
+	    pix_multiply_1x64 (
+		unpack_32_1x64 (s), negate_1x64 (
+		    expand_alpha_1x64 (unpack_32_1x64 (d)))));
+	w--;
+	ps++;
+	if (pm)
+	    pm++;
+    }
+}
+
+static force_inline uint32_t
+core_combine_atop_u_pixel_sse2 (uint32_t src,
+                                uint32_t dst)
+{
+    __m64 s = unpack_32_1x64 (src);
+    __m64 d = unpack_32_1x64 (dst);
+
+    __m64 sa = negate_1x64 (expand_alpha_1x64 (s));
+    __m64 da = expand_alpha_1x64 (d);
+
+    return pack_1x64_32 (pix_add_multiply_1x64 (&s, &da, &d, &sa));
+}
+
+static force_inline void
+core_combine_atop_u_sse2 (uint32_t*       pd,
+                          const uint32_t* ps,
+                          const uint32_t* pm,
+                          int             w)
+{
+    uint32_t s, d;
+
+    __m128i xmm_src_lo, xmm_src_hi;
+    __m128i xmm_dst_lo, xmm_dst_hi;
+    __m128i xmm_alpha_src_lo, xmm_alpha_src_hi;
+    __m128i xmm_alpha_dst_lo, xmm_alpha_dst_hi;
+
+    /* call prefetch hint to optimize cache load*/
+    cache_prefetch ((__m128i*)ps);
+    cache_prefetch ((__m128i*)pd);
+    maybe_prefetch ((__m128i*)pm);
+
+    while (w && ((unsigned long) pd & 15))
+    {
+	s = combine1 (ps, pm);
+	d = *pd;
+
+	*pd++ = core_combine_atop_u_pixel_sse2 (s, d);
+	w--;
+	ps++;
+	if (pm)
+	    pm++;
+    }
+
+    /* call prefetch hint to optimize cache load*/
+    cache_prefetch ((__m128i*)ps);
+    cache_prefetch ((__m128i*)pd);
+    maybe_prefetch ((__m128i*)pm);
+
+    while (w >= 4)
+    {
+	/* fill cache line with next memory */
+	cache_prefetch_next ((__m128i*)ps);
+	cache_prefetch_next ((__m128i*)pd);
+	maybe_prefetch_next ((__m128i*)pm);
+
+	xmm_src_hi = combine4 ((__m128i*)ps, (__m128i*)pm);
+	xmm_dst_hi = load_128_aligned ((__m128i*) pd);
+
+	unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
+	unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
+
+	expand_alpha_2x128 (xmm_src_lo, xmm_src_hi,
+			    &xmm_alpha_src_lo, &xmm_alpha_src_hi);
+	expand_alpha_2x128 (xmm_dst_lo, xmm_dst_hi,
+			    &xmm_alpha_dst_lo, &xmm_alpha_dst_hi);
+
+	negate_2x128 (xmm_alpha_src_lo, xmm_alpha_src_hi,
+		      &xmm_alpha_src_lo, &xmm_alpha_src_hi);
+
+	pix_add_multiply_2x128 (
+	    &xmm_src_lo, &xmm_src_hi, &xmm_alpha_dst_lo, &xmm_alpha_dst_hi,
+	    &xmm_dst_lo, &xmm_dst_hi, &xmm_alpha_src_lo, &xmm_alpha_src_hi,
+	    &xmm_dst_lo, &xmm_dst_hi);
+
+	save_128_aligned (
+	    (__m128i*)pd, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
+
+	ps += 4;
+	pd += 4;
+	w -= 4;
+	if (pm)
+	    pm += 4;
+    }
+
+    while (w)
+    {
+	s = combine1 (ps, pm);
+	d = *pd;
+
+	*pd++ = core_combine_atop_u_pixel_sse2 (s, d);
+	w--;
+	ps++;
+	if (pm)
+	    pm++;
+    }
+}
+
+static force_inline uint32_t
+core_combine_reverse_atop_u_pixel_sse2 (uint32_t src,
+                                        uint32_t dst)
+{
+    __m64 s = unpack_32_1x64 (src);
+    __m64 d = unpack_32_1x64 (dst);
+
+    __m64 sa = expand_alpha_1x64 (s);
+    __m64 da = negate_1x64 (expand_alpha_1x64 (d));
+
+    return pack_1x64_32 (pix_add_multiply_1x64 (&s, &da, &d, &sa));
+}
+
+static force_inline void
+core_combine_reverse_atop_u_sse2 (uint32_t*       pd,
+                                  const uint32_t* ps,
+                                  const uint32_t* pm,
+                                  int             w)
+{
+    uint32_t s, d;
+
+    __m128i xmm_src_lo, xmm_src_hi;
+    __m128i xmm_dst_lo, xmm_dst_hi;
+    __m128i xmm_alpha_src_lo, xmm_alpha_src_hi;
+    __m128i xmm_alpha_dst_lo, xmm_alpha_dst_hi;
+
+    /* call prefetch hint to optimize cache load*/
+    cache_prefetch ((__m128i*)ps);
+    cache_prefetch ((__m128i*)pd);
+    maybe_prefetch ((__m128i*)pm);
+
+    while (w && ((unsigned long) pd & 15))
+    {
+	s = combine1 (ps, pm);
+	d = *pd;
+
+	*pd++ = core_combine_reverse_atop_u_pixel_sse2 (s, d);
+	ps++;
+	w--;
+	if (pm)
+	    pm++;
+    }
+
+    /* call prefetch hint to optimize cache load*/
+    cache_prefetch ((__m128i*)ps);
+    cache_prefetch ((__m128i*)pd);
+    maybe_prefetch ((__m128i*)pm);
+
+    while (w >= 4)
+    {
+	/* fill cache line with next memory */
+	cache_prefetch_next ((__m128i*)ps);
+	cache_prefetch_next ((__m128i*)pd);
+	maybe_prefetch_next ((__m128i*)pm);
+
+	xmm_src_hi = combine4 ((__m128i*)ps, (__m128i*)pm);
+	xmm_dst_hi = load_128_aligned ((__m128i*) pd);
+
+	unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
+	unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
+
+	expand_alpha_2x128 (xmm_src_lo, xmm_src_hi,
+			    &xmm_alpha_src_lo, &xmm_alpha_src_hi);
+	expand_alpha_2x128 (xmm_dst_lo, xmm_dst_hi,
+			    &xmm_alpha_dst_lo, &xmm_alpha_dst_hi);
+
+	negate_2x128 (xmm_alpha_dst_lo, xmm_alpha_dst_hi,
+		      &xmm_alpha_dst_lo, &xmm_alpha_dst_hi);
+
+	pix_add_multiply_2x128 (
+	    &xmm_src_lo, &xmm_src_hi, &xmm_alpha_dst_lo, &xmm_alpha_dst_hi,
+	    &xmm_dst_lo, &xmm_dst_hi, &xmm_alpha_src_lo, &xmm_alpha_src_hi,
+	    &xmm_dst_lo, &xmm_dst_hi);
+
+	save_128_aligned (
+	    (__m128i*)pd, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
+
+	ps += 4;
+	pd += 4;
+	w -= 4;
+	if (pm)
+	    pm += 4;
+    }
+
+    while (w)
+    {
+	s = combine1 (ps, pm);
+	d = *pd;
+
+	*pd++ = core_combine_reverse_atop_u_pixel_sse2 (s, d);
+	ps++;
+	w--;
+	if (pm)
+	    pm++;
+    }
+}
+
+static force_inline uint32_t
+core_combine_xor_u_pixel_sse2 (uint32_t src,
+                               uint32_t dst)
+{
+    __m64 s = unpack_32_1x64 (src);
+    __m64 d = unpack_32_1x64 (dst);
+
+    __m64 neg_d = negate_1x64 (expand_alpha_1x64 (d));
+    __m64 neg_s = negate_1x64 (expand_alpha_1x64 (s));
+
+    return pack_1x64_32 (pix_add_multiply_1x64 (&s, &neg_d, &d, &neg_s));
+}
+
+static force_inline void
+core_combine_xor_u_sse2 (uint32_t*       dst,
+                         const uint32_t* src,
+                         const uint32_t *mask,
+                         int             width)
+{
+    int w = width;
+    uint32_t s, d;
+    uint32_t* pd = dst;
+    const uint32_t* ps = src;
+    const uint32_t* pm = mask;
+
+    __m128i xmm_src, xmm_src_lo, xmm_src_hi;
+    __m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
+    __m128i xmm_alpha_src_lo, xmm_alpha_src_hi;
+    __m128i xmm_alpha_dst_lo, xmm_alpha_dst_hi;
+
+    /* call prefetch hint to optimize cache load*/
+    cache_prefetch ((__m128i*)ps);
+    cache_prefetch ((__m128i*)pd);
+    maybe_prefetch ((__m128i*)pm);
+
+    while (w && ((unsigned long) pd & 15))
+    {
+	s = combine1 (ps, pm);
+	d = *pd;
+
+	*pd++ = core_combine_xor_u_pixel_sse2 (s, d);
+	w--;
+	ps++;
+	if (pm)
+	    pm++;
+    }
+
+    /* call prefetch hint to optimize cache load*/
+    cache_prefetch ((__m128i*)ps);
+    cache_prefetch ((__m128i*)pd);
+    maybe_prefetch ((__m128i*)pm);
+
+    while (w >= 4)
+    {
+	/* fill cache line with next memory */
+	cache_prefetch_next ((__m128i*)ps);
+	cache_prefetch_next ((__m128i*)pd);
+	maybe_prefetch_next ((__m128i*)pm);
+
+	xmm_src = combine4 ((__m128i*) ps, (__m128i*) pm);
+	xmm_dst = load_128_aligned ((__m128i*) pd);
+
+	unpack_128_2x128 (xmm_src, &xmm_src_lo, &xmm_src_hi);
+	unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
+
+	expand_alpha_2x128 (xmm_src_lo, xmm_src_hi,
+			    &xmm_alpha_src_lo, &xmm_alpha_src_hi);
+	expand_alpha_2x128 (xmm_dst_lo, xmm_dst_hi,
+			    &xmm_alpha_dst_lo, &xmm_alpha_dst_hi);
+
+	negate_2x128 (xmm_alpha_src_lo, xmm_alpha_src_hi,
+		      &xmm_alpha_src_lo, &xmm_alpha_src_hi);
+	negate_2x128 (xmm_alpha_dst_lo, xmm_alpha_dst_hi,
+		      &xmm_alpha_dst_lo, &xmm_alpha_dst_hi);
+
+	pix_add_multiply_2x128 (
+	    &xmm_src_lo, &xmm_src_hi, &xmm_alpha_dst_lo, &xmm_alpha_dst_hi,
+	    &xmm_dst_lo, &xmm_dst_hi, &xmm_alpha_src_lo, &xmm_alpha_src_hi,
+	    &xmm_dst_lo, &xmm_dst_hi);
+
+	save_128_aligned (
+	    (__m128i*)pd, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
+
+	ps += 4;
+	pd += 4;
+	w -= 4;
+	if (pm)
+	    pm += 4;
+    }
+
+    while (w)
+    {
+	s = combine1 (ps, pm);
+	d = *pd;
+
+	*pd++ = core_combine_xor_u_pixel_sse2 (s, d);
+	w--;
+	ps++;
+	if (pm)
+	    pm++;
+    }
+}
+
+static force_inline void
+core_combine_add_u_sse2 (uint32_t*       dst,
+                         const uint32_t* src,
+                         const uint32_t* mask,
+                         int             width)
+{
+    int w = width;
+    uint32_t s, d;
+    uint32_t* pd = dst;
+    const uint32_t* ps = src;
+    const uint32_t* pm = mask;
+
+    /* call prefetch hint to optimize cache load*/
+    cache_prefetch ((__m128i*)ps);
+    cache_prefetch ((__m128i*)pd);
+    maybe_prefetch ((__m128i*)pm);
+
+    while (w && (unsigned long)pd & 15)
+    {
+	s = combine1 (ps, pm);
+	d = *pd;
+
+	ps++;
+	if (pm)
+	    pm++;
+	*pd++ = _mm_cvtsi64_si32 (
+	    _mm_adds_pu8 (_mm_cvtsi32_si64 (s), _mm_cvtsi32_si64 (d)));
+	w--;
+    }
+
+    /* call prefetch hint to optimize cache load*/
+    cache_prefetch ((__m128i*)ps);
+    cache_prefetch ((__m128i*)pd);
+    maybe_prefetch ((__m128i*)pm);
+
+    while (w >= 4)
+    {
+	__m128i s;
+
+	/* fill cache line with next memory */
+	cache_prefetch_next ((__m128i*)ps);
+	cache_prefetch_next ((__m128i*)pd);
+	maybe_prefetch_next ((__m128i*)pm);
+
+	s = combine4 ((__m128i*)ps, (__m128i*)pm);
+
+	save_128_aligned (
+	    (__m128i*)pd, _mm_adds_epu8 (s, load_128_aligned  ((__m128i*)pd)));
+
+	pd += 4;
+	ps += 4;
+	if (pm)
+	    pm += 4;
+	w -= 4;
+    }
+
+    while (w--)
+    {
+	s = combine1 (ps, pm);
+	d = *pd;
+
+	ps++;
+	*pd++ = _mm_cvtsi64_si32 (
+	    _mm_adds_pu8 (_mm_cvtsi32_si64 (s), _mm_cvtsi32_si64 (d)));
+	if (pm)
+	    pm++;
+    }
+}
+
+static force_inline uint32_t
+core_combine_saturate_u_pixel_sse2 (uint32_t src,
+                                    uint32_t dst)
+{
+    __m64 ms = unpack_32_1x64 (src);
+    __m64 md = unpack_32_1x64 (dst);
+    uint32_t sa = src >> 24;
+    uint32_t da = ~dst >> 24;
+
+    if (sa > da)
+    {
+	ms = pix_multiply_1x64 (
+	    ms, expand_alpha_1x64 (unpack_32_1x64 (DIV_UN8 (da, sa) << 24)));
+    }
+
+    return pack_1x64_32 (_mm_adds_pu16 (md, ms));
+}
+
+static force_inline void
+core_combine_saturate_u_sse2 (uint32_t *      pd,
+                              const uint32_t *ps,
+                              const uint32_t *pm,
+                              int             w)
+{
+    uint32_t s, d;
+
+    uint32_t pack_cmp;
+    __m128i xmm_src, xmm_dst;
+
+    /* call prefetch hint to optimize cache load*/
+    cache_prefetch ((__m128i*)ps);
+    cache_prefetch ((__m128i*)pd);
+    maybe_prefetch ((__m128i*)pm);
+
+    while (w && (unsigned long)pd & 15)
+    {
+	s = combine1 (ps, pm);
+	d = *pd;
+
+	*pd++ = core_combine_saturate_u_pixel_sse2 (s, d);
+	w--;
+	ps++;
+	if (pm)
+	    pm++;
+    }
+
+    /* call prefetch hint to optimize cache load*/
+    cache_prefetch ((__m128i*)ps);
+    cache_prefetch ((__m128i*)pd);
+    maybe_prefetch ((__m128i*)pm);
+
+    while (w >= 4)
+    {
+	/* fill cache line with next memory */
+	cache_prefetch_next ((__m128i*)ps);
+	cache_prefetch_next ((__m128i*)pd);
+	maybe_prefetch_next ((__m128i*)pm);
+
+	xmm_dst = load_128_aligned  ((__m128i*)pd);
+	xmm_src = combine4 ((__m128i*)ps, (__m128i*)pm);
+
+	pack_cmp = _mm_movemask_epi8 (
+	    _mm_cmpgt_epi32 (
+		_mm_srli_epi32 (xmm_src, 24),
+		_mm_srli_epi32 (_mm_xor_si128 (xmm_dst, mask_ff000000), 24)));
+
+	/* if some alpha src is grater than respective ~alpha dst */
+	if (pack_cmp)
+	{
+	    s = combine1 (ps++, pm);
+	    d = *pd;
+	    *pd++ = core_combine_saturate_u_pixel_sse2 (s, d);
+	    if (pm)
+		pm++;
+
+	    s = combine1 (ps++, pm);
+	    d = *pd;
+	    *pd++ = core_combine_saturate_u_pixel_sse2 (s, d);
+	    if (pm)
+		pm++;
+
+	    s = combine1 (ps++, pm);
+	    d = *pd;
+	    *pd++ = core_combine_saturate_u_pixel_sse2 (s, d);
+	    if (pm)
+		pm++;
+
+	    s = combine1 (ps++, pm);
+	    d = *pd;
+	    *pd++ = core_combine_saturate_u_pixel_sse2 (s, d);
+	    if (pm)
+		pm++;
+	}
+	else
+	{
+	    save_128_aligned ((__m128i*)pd, _mm_adds_epu8 (xmm_dst, xmm_src));
+
+	    pd += 4;
+	    ps += 4;
+	    if (pm)
+		pm += 4;
+	}
+
+	w -= 4;
+    }
+
+    while (w--)
+    {
+	s = combine1 (ps, pm);
+	d = *pd;
+
+	*pd++ = core_combine_saturate_u_pixel_sse2 (s, d);
+	ps++;
+	if (pm)
+	    pm++;
+    }
+}
+
+static force_inline void
+core_combine_src_ca_sse2 (uint32_t*       pd,
+                          const uint32_t* ps,
+                          const uint32_t *pm,
+                          int             w)
+{
+    uint32_t s, m;
+
+    __m128i xmm_src_lo, xmm_src_hi;
+    __m128i xmm_mask_lo, xmm_mask_hi;
+    __m128i xmm_dst_lo, xmm_dst_hi;
+
+    /* call prefetch hint to optimize cache load*/
+    cache_prefetch ((__m128i*)ps);
+    cache_prefetch ((__m128i*)pd);
+    cache_prefetch ((__m128i*)pm);
+
+    while (w && (unsigned long)pd & 15)
+    {
+	s = *ps++;
+	m = *pm++;
+	*pd++ = pack_1x64_32 (
+	    pix_multiply_1x64 (unpack_32_1x64 (s), unpack_32_1x64 (m)));
+	w--;
+    }
+
+    /* call prefetch hint to optimize cache load*/
+    cache_prefetch ((__m128i*)ps);
+    cache_prefetch ((__m128i*)pd);
+    cache_prefetch ((__m128i*)pm);
+
+    while (w >= 4)
+    {
+	/* fill cache line with next memory */
+	cache_prefetch_next ((__m128i*)ps);
+	cache_prefetch_next ((__m128i*)pd);
+	cache_prefetch_next ((__m128i*)pm);
+
+	xmm_src_hi = load_128_unaligned ((__m128i*)ps);
+	xmm_mask_hi = load_128_unaligned ((__m128i*)pm);
+
+	unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
+	unpack_128_2x128 (xmm_mask_hi, &xmm_mask_lo, &xmm_mask_hi);
+
+	pix_multiply_2x128 (&xmm_src_lo, &xmm_src_hi,
+			    &xmm_mask_lo, &xmm_mask_hi,
+			    &xmm_dst_lo, &xmm_dst_hi);
+
+	save_128_aligned (
+	    (__m128i*)pd, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
+
+	ps += 4;
+	pd += 4;
+	pm += 4;
+	w -= 4;
+    }
+
+    while (w)
+    {
+	s = *ps++;
+	m = *pm++;
+	*pd++ = pack_1x64_32 (
+	    pix_multiply_1x64 (unpack_32_1x64 (s), unpack_32_1x64 (m)));
+	w--;
+    }
+}
+
+static force_inline uint32_t
+core_combine_over_ca_pixel_sse2 (uint32_t src,
+                                 uint32_t mask,
+                                 uint32_t dst)
+{
+    __m64 s = unpack_32_1x64 (src);
+    __m64 expAlpha = expand_alpha_1x64 (s);
+    __m64 unpk_mask = unpack_32_1x64 (mask);
+    __m64 unpk_dst  = unpack_32_1x64 (dst);
+
+    return pack_1x64_32 (in_over_1x64 (&s, &expAlpha, &unpk_mask, &unpk_dst));
+}
+
+static force_inline void
+core_combine_over_ca_sse2 (uint32_t*       pd,
+                           const uint32_t* ps,
+                           const uint32_t *pm,
+                           int             w)
+{
+    uint32_t s, m, d;
+
+    __m128i xmm_alpha_lo, xmm_alpha_hi;
+    __m128i xmm_src_lo, xmm_src_hi;
+    __m128i xmm_dst_lo, xmm_dst_hi;
+    __m128i xmm_mask_lo, xmm_mask_hi;
+
+    /* call prefetch hint to optimize cache load*/
+    cache_prefetch ((__m128i*)ps);
+    cache_prefetch ((__m128i*)pd);
+    cache_prefetch ((__m128i*)pm);
+
+    while (w && (unsigned long)pd & 15)
+    {
+	s = *ps++;
+	m = *pm++;
+	d = *pd;
+
+	*pd++ = core_combine_over_ca_pixel_sse2 (s, m, d);
+	w--;
+    }
+
+    /* call prefetch hint to optimize cache load*/
+    cache_prefetch ((__m128i*)ps);
+    cache_prefetch ((__m128i*)pd);
+    cache_prefetch ((__m128i*)pm);
+
+    while (w >= 4)
+    {
+	/* fill cache line with next memory */
+	cache_prefetch_next ((__m128i*)ps);
+	cache_prefetch_next ((__m128i*)pd);
+	cache_prefetch_next ((__m128i*)pm);
+
+	xmm_dst_hi = load_128_aligned ((__m128i*)pd);
+	xmm_src_hi = load_128_unaligned ((__m128i*)ps);
+	xmm_mask_hi = load_128_unaligned ((__m128i*)pm);
+
+	unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
+	unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
+	unpack_128_2x128 (xmm_mask_hi, &xmm_mask_lo, &xmm_mask_hi);
+
+	expand_alpha_2x128 (xmm_src_lo, xmm_src_hi,
+			    &xmm_alpha_lo, &xmm_alpha_hi);
+
+	in_over_2x128 (&xmm_src_lo, &xmm_src_hi,
+		       &xmm_alpha_lo, &xmm_alpha_hi,
+		       &xmm_mask_lo, &xmm_mask_hi,
+		       &xmm_dst_lo, &xmm_dst_hi);
+
+	save_128_aligned (
+	    (__m128i*)pd, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
+
+	ps += 4;
+	pd += 4;
+	pm += 4;
+	w -= 4;
+    }
+
+    while (w)
+    {
+	s = *ps++;
+	m = *pm++;
+	d = *pd;
+
+	*pd++ = core_combine_over_ca_pixel_sse2 (s, m, d);
+	w--;
+    }
+}
+
+static force_inline uint32_t
+core_combine_over_reverse_ca_pixel_sse2 (uint32_t src,
+                                         uint32_t mask,
+                                         uint32_t dst)
+{
+    __m64 d = unpack_32_1x64 (dst);
+
+    return pack_1x64_32 (
+	over_1x64 (d, expand_alpha_1x64 (d),
+		   pix_multiply_1x64 (unpack_32_1x64 (src),
+				      unpack_32_1x64 (mask))));
+}
+
+static force_inline void
+core_combine_over_reverse_ca_sse2 (uint32_t*       pd,
+                                   const uint32_t* ps,
+                                   const uint32_t *pm,
+                                   int             w)
+{
+    uint32_t s, m, d;
+
+    __m128i xmm_alpha_lo, xmm_alpha_hi;
+    __m128i xmm_src_lo, xmm_src_hi;
+    __m128i xmm_dst_lo, xmm_dst_hi;
+    __m128i xmm_mask_lo, xmm_mask_hi;
+
+    /* call prefetch hint to optimize cache load*/
+    cache_prefetch ((__m128i*)ps);
+    cache_prefetch ((__m128i*)pd);
+    cache_prefetch ((__m128i*)pm);
+
+    while (w && (unsigned long)pd & 15)
+    {
+	s = *ps++;
+	m = *pm++;
+	d = *pd;
+
+	*pd++ = core_combine_over_reverse_ca_pixel_sse2 (s, m, d);
+	w--;
+    }
+
+    /* call prefetch hint to optimize cache load*/
+    cache_prefetch ((__m128i*)ps);
+    cache_prefetch ((__m128i*)pd);
+    cache_prefetch ((__m128i*)pm);
+
+    while (w >= 4)
+    {
+	/* fill cache line with next memory */
+	cache_prefetch_next ((__m128i*)ps);
+	cache_prefetch_next ((__m128i*)pd);
+	cache_prefetch_next ((__m128i*)pm);
+
+	xmm_dst_hi = load_128_aligned ((__m128i*)pd);
+	xmm_src_hi = load_128_unaligned ((__m128i*)ps);
+	xmm_mask_hi = load_128_unaligned ((__m128i*)pm);
+
+	unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
+	unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
+	unpack_128_2x128 (xmm_mask_hi, &xmm_mask_lo, &xmm_mask_hi);
+
+	expand_alpha_2x128 (xmm_dst_lo, xmm_dst_hi,
+			    &xmm_alpha_lo, &xmm_alpha_hi);
+	pix_multiply_2x128 (&xmm_src_lo, &xmm_src_hi,
+			    &xmm_mask_lo, &xmm_mask_hi,
+			    &xmm_mask_lo, &xmm_mask_hi);
+
+	over_2x128 (&xmm_dst_lo, &xmm_dst_hi,
+		    &xmm_alpha_lo, &xmm_alpha_hi,
+		    &xmm_mask_lo, &xmm_mask_hi);
+
+	save_128_aligned (
+	    (__m128i*)pd, pack_2x128_128 (xmm_mask_lo, xmm_mask_hi));
+
+	ps += 4;
+	pd += 4;
+	pm += 4;
+	w -= 4;
+    }
+
+    while (w)
+    {
+	s = *ps++;
+	m = *pm++;
+	d = *pd;
+
+	*pd++ = core_combine_over_reverse_ca_pixel_sse2 (s, m, d);
+	w--;
+    }
+}
+
+static force_inline void
+core_combine_in_ca_sse2 (uint32_t *      pd,
+                         const uint32_t *ps,
+                         const uint32_t *pm,
+                         int             w)
+{
+    uint32_t s, m, d;
+
+    __m128i xmm_alpha_lo, xmm_alpha_hi;
+    __m128i xmm_src_lo, xmm_src_hi;
+    __m128i xmm_dst_lo, xmm_dst_hi;
+    __m128i xmm_mask_lo, xmm_mask_hi;
+
+    /* call prefetch hint to optimize cache load*/
+    cache_prefetch ((__m128i*)ps);
+    cache_prefetch ((__m128i*)pd);
+    cache_prefetch ((__m128i*)pm);
+
+    while (w && (unsigned long)pd & 15)
+    {
+	s = *ps++;
+	m = *pm++;
+	d = *pd;
+
+	*pd++ = pack_1x64_32 (
+	    pix_multiply_1x64 (
+		pix_multiply_1x64 (unpack_32_1x64 (s), unpack_32_1x64 (m)),
+		expand_alpha_1x64 (unpack_32_1x64 (d))));
+
+	w--;
+    }
+
+    /* call prefetch hint to optimize cache load*/
+    cache_prefetch ((__m128i*)ps);
+    cache_prefetch ((__m128i*)pd);
+    cache_prefetch ((__m128i*)pm);
+
+    while (w >= 4)
+    {
+	/* fill cache line with next memory */
+	cache_prefetch_next ((__m128i*)ps);
+	cache_prefetch_next ((__m128i*)pd);
+	cache_prefetch_next ((__m128i*)pm);
+
+	xmm_dst_hi = load_128_aligned ((__m128i*)pd);
+	xmm_src_hi = load_128_unaligned ((__m128i*)ps);
+	xmm_mask_hi = load_128_unaligned ((__m128i*)pm);
+
+	unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
+	unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
+	unpack_128_2x128 (xmm_mask_hi, &xmm_mask_lo, &xmm_mask_hi);
+
+	expand_alpha_2x128 (xmm_dst_lo, xmm_dst_hi,
+			    &xmm_alpha_lo, &xmm_alpha_hi);
+
+	pix_multiply_2x128 (&xmm_src_lo, &xmm_src_hi,
+			    &xmm_mask_lo, &xmm_mask_hi,
+			    &xmm_dst_lo, &xmm_dst_hi);
+
+	pix_multiply_2x128 (&xmm_dst_lo, &xmm_dst_hi,
+			    &xmm_alpha_lo, &xmm_alpha_hi,
+			    &xmm_dst_lo, &xmm_dst_hi);
+
+	save_128_aligned (
+	    (__m128i*)pd, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
+
+	ps += 4;
+	pd += 4;
+	pm += 4;
+	w -= 4;
+    }
+
+    while (w)
+    {
+	s = *ps++;
+	m = *pm++;
+	d = *pd;
+
+	*pd++ = pack_1x64_32 (
+	    pix_multiply_1x64 (
+		pix_multiply_1x64 (
+		    unpack_32_1x64 (s), unpack_32_1x64 (m)),
+		expand_alpha_1x64 (unpack_32_1x64 (d))));
+
+	w--;
+    }
+}
+
+static force_inline void
+core_combine_in_reverse_ca_sse2 (uint32_t *      pd,
+                                 const uint32_t *ps,
+                                 const uint32_t *pm,
+                                 int             w)
+{
+    uint32_t s, m, d;
+
+    __m128i xmm_alpha_lo, xmm_alpha_hi;
+    __m128i xmm_src_lo, xmm_src_hi;
+    __m128i xmm_dst_lo, xmm_dst_hi;
+    __m128i xmm_mask_lo, xmm_mask_hi;
+
+    /* call prefetch hint to optimize cache load*/
+    cache_prefetch ((__m128i*)ps);
+    cache_prefetch ((__m128i*)pd);
+    cache_prefetch ((__m128i*)pm);
+
+    while (w && (unsigned long)pd & 15)
+    {
+	s = *ps++;
+	m = *pm++;
+	d = *pd;
+
+	*pd++ = pack_1x64_32 (
+	    pix_multiply_1x64 (
+		unpack_32_1x64 (d),
+		pix_multiply_1x64 (unpack_32_1x64 (m),
+				   expand_alpha_1x64 (unpack_32_1x64 (s)))));
+	w--;
+    }
+
+    /* call prefetch hint to optimize cache load*/
+    cache_prefetch ((__m128i*)ps);
+    cache_prefetch ((__m128i*)pd);
+    cache_prefetch ((__m128i*)pm);
+
+    while (w >= 4)
+    {
+	/* fill cache line with next memory */
+	cache_prefetch_next ((__m128i*)ps);
+	cache_prefetch_next ((__m128i*)pd);
+	cache_prefetch_next ((__m128i*)pm);
+
+	xmm_dst_hi = load_128_aligned ((__m128i*)pd);
+	xmm_src_hi = load_128_unaligned ((__m128i*)ps);
+	xmm_mask_hi = load_128_unaligned ((__m128i*)pm);
+
+	unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
+	unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
+	unpack_128_2x128 (xmm_mask_hi, &xmm_mask_lo, &xmm_mask_hi);
+
+	expand_alpha_2x128 (xmm_src_lo, xmm_src_hi,
+			    &xmm_alpha_lo, &xmm_alpha_hi);
+	pix_multiply_2x128 (&xmm_mask_lo, &xmm_mask_hi,
+			    &xmm_alpha_lo, &xmm_alpha_hi,
+			    &xmm_alpha_lo, &xmm_alpha_hi);
+
+	pix_multiply_2x128 (&xmm_dst_lo, &xmm_dst_hi,
+			    &xmm_alpha_lo, &xmm_alpha_hi,
+			    &xmm_dst_lo, &xmm_dst_hi);
+
+	save_128_aligned (
+	    (__m128i*)pd, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
+
+	ps += 4;
+	pd += 4;
+	pm += 4;
+	w -= 4;
+    }
+
+    while (w)
+    {
+	s = *ps++;
+	m = *pm++;
+	d = *pd;
+
+	*pd++ = pack_1x64_32 (
+	    pix_multiply_1x64 (
+		unpack_32_1x64 (d),
+		pix_multiply_1x64 (unpack_32_1x64 (m),
+				   expand_alpha_1x64 (unpack_32_1x64 (s)))));
+	w--;
+    }
+}
+
+static force_inline void
+core_combine_out_ca_sse2 (uint32_t *      pd,
+                          const uint32_t *ps,
+                          const uint32_t *pm,
+                          int             w)
+{
+    uint32_t s, m, d;
+
+    __m128i xmm_alpha_lo, xmm_alpha_hi;
+    __m128i xmm_src_lo, xmm_src_hi;
+    __m128i xmm_dst_lo, xmm_dst_hi;
+    __m128i xmm_mask_lo, xmm_mask_hi;
+
+    /* call prefetch hint to optimize cache load*/
+    cache_prefetch ((__m128i*)ps);
+    cache_prefetch ((__m128i*)pd);
+    cache_prefetch ((__m128i*)pm);
+
+    while (w && (unsigned long)pd & 15)
+    {
+	s = *ps++;
+	m = *pm++;
+	d = *pd;
+
+	*pd++ = pack_1x64_32 (
+	    pix_multiply_1x64 (
+		pix_multiply_1x64 (
+		    unpack_32_1x64 (s), unpack_32_1x64 (m)),
+		negate_1x64 (expand_alpha_1x64 (unpack_32_1x64 (d)))));
+	w--;
+    }
+
+    /* call prefetch hint to optimize cache load*/
+    cache_prefetch ((__m128i*)ps);
+    cache_prefetch ((__m128i*)pd);
+    cache_prefetch ((__m128i*)pm);
+
+    while (w >= 4)
+    {
+	/* fill cache line with next memory */
+	cache_prefetch_next ((__m128i*)ps);
+	cache_prefetch_next ((__m128i*)pd);
+	cache_prefetch_next ((__m128i*)pm);
+
+	xmm_dst_hi = load_128_aligned ((__m128i*)pd);
+	xmm_src_hi = load_128_unaligned ((__m128i*)ps);
+	xmm_mask_hi = load_128_unaligned ((__m128i*)pm);
+
+	unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
+	unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
+	unpack_128_2x128 (xmm_mask_hi, &xmm_mask_lo, &xmm_mask_hi);
+
+	expand_alpha_2x128 (xmm_dst_lo, xmm_dst_hi,
+			    &xmm_alpha_lo, &xmm_alpha_hi);
+	negate_2x128 (xmm_alpha_lo, xmm_alpha_hi,
+		      &xmm_alpha_lo, &xmm_alpha_hi);
+
+	pix_multiply_2x128 (&xmm_src_lo, &xmm_src_hi,
+			    &xmm_mask_lo, &xmm_mask_hi,
+			    &xmm_dst_lo, &xmm_dst_hi);
+	pix_multiply_2x128 (&xmm_dst_lo, &xmm_dst_hi,
+			    &xmm_alpha_lo, &xmm_alpha_hi,
+			    &xmm_dst_lo, &xmm_dst_hi);
+
+	save_128_aligned (
+	    (__m128i*)pd, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
+
+	ps += 4;
+	pd += 4;
+	pm += 4;
+	w -= 4;
+    }
+
+    while (w)
+    {
+	s = *ps++;
+	m = *pm++;
+	d = *pd;
+
+	*pd++ = pack_1x64_32 (
+	    pix_multiply_1x64 (
+		pix_multiply_1x64 (
+		    unpack_32_1x64 (s), unpack_32_1x64 (m)),
+		negate_1x64 (expand_alpha_1x64 (unpack_32_1x64 (d)))));
+
+	w--;
+    }
+}
+
+static force_inline void
+core_combine_out_reverse_ca_sse2 (uint32_t *      pd,
+                                  const uint32_t *ps,
+                                  const uint32_t *pm,
+                                  int             w)
+{
+    uint32_t s, m, d;
+
+    __m128i xmm_alpha_lo, xmm_alpha_hi;
+    __m128i xmm_src_lo, xmm_src_hi;
+    __m128i xmm_dst_lo, xmm_dst_hi;
+    __m128i xmm_mask_lo, xmm_mask_hi;
+
+    /* call prefetch hint to optimize cache load*/
+    cache_prefetch ((__m128i*)ps);
+    cache_prefetch ((__m128i*)pd);
+    cache_prefetch ((__m128i*)pm);
+
+    while (w && (unsigned long)pd & 15)
+    {
+	s = *ps++;
+	m = *pm++;
+	d = *pd;
+
+	*pd++ = pack_1x64_32 (
+	    pix_multiply_1x64 (
+		unpack_32_1x64 (d),
+		negate_1x64 (pix_multiply_1x64 (
+				 unpack_32_1x64 (m),
+				 expand_alpha_1x64 (unpack_32_1x64 (s))))));
+	w--;
+    }
+
+    /* call prefetch hint to optimize cache load*/
+    cache_prefetch ((__m128i*)ps);
+    cache_prefetch ((__m128i*)pd);
+    cache_prefetch ((__m128i*)pm);
+
+    while (w >= 4)
+    {
+	/* fill cache line with next memory */
+	cache_prefetch_next ((__m128i*)ps);
+	cache_prefetch_next ((__m128i*)pd);
+	cache_prefetch_next ((__m128i*)pm);
+
+	xmm_dst_hi = load_128_aligned ((__m128i*)pd);
+	xmm_src_hi = load_128_unaligned ((__m128i*)ps);
+	xmm_mask_hi = load_128_unaligned ((__m128i*)pm);
+
+	unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
+	unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
+	unpack_128_2x128 (xmm_mask_hi, &xmm_mask_lo, &xmm_mask_hi);
+
+	expand_alpha_2x128 (xmm_src_lo, xmm_src_hi,
+			    &xmm_alpha_lo, &xmm_alpha_hi);
+
+	pix_multiply_2x128 (&xmm_mask_lo, &xmm_mask_hi,
+			    &xmm_alpha_lo, &xmm_alpha_hi,
+			    &xmm_mask_lo, &xmm_mask_hi);
+
+	negate_2x128 (xmm_mask_lo, xmm_mask_hi,
+		      &xmm_mask_lo, &xmm_mask_hi);
+
+	pix_multiply_2x128 (&xmm_dst_lo, &xmm_dst_hi,
+			    &xmm_mask_lo, &xmm_mask_hi,
+			    &xmm_dst_lo, &xmm_dst_hi);
+
+	save_128_aligned (
+	    (__m128i*)pd, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
+
+	ps += 4;
+	pd += 4;
+	pm += 4;
+	w -= 4;
+    }
+
+    while (w)
+    {
+	s = *ps++;
+	m = *pm++;
+	d = *pd;
+
+	*pd++ = pack_1x64_32 (
+	    pix_multiply_1x64 (
+		unpack_32_1x64 (d),
+		negate_1x64 (pix_multiply_1x64 (
+				 unpack_32_1x64 (m),
+				 expand_alpha_1x64 (unpack_32_1x64 (s))))));
+	w--;
+    }
+}
+
+static force_inline uint32_t
+core_combine_atop_ca_pixel_sse2 (uint32_t src,
+                                 uint32_t mask,
+                                 uint32_t dst)
+{
+    __m64 m = unpack_32_1x64 (mask);
+    __m64 s = unpack_32_1x64 (src);
+    __m64 d = unpack_32_1x64 (dst);
+    __m64 sa = expand_alpha_1x64 (s);
+    __m64 da = expand_alpha_1x64 (d);
+
+    s = pix_multiply_1x64 (s, m);
+    m = negate_1x64 (pix_multiply_1x64 (m, sa));
+
+    return pack_1x64_32 (pix_add_multiply_1x64 (&d, &m, &s, &da));
+}
+
+static force_inline void
+core_combine_atop_ca_sse2 (uint32_t *      pd,
+                           const uint32_t *ps,
+                           const uint32_t *pm,
+                           int             w)
+{
+    uint32_t s, m, d;
+
+    __m128i xmm_src_lo, xmm_src_hi;
+    __m128i xmm_dst_lo, xmm_dst_hi;
+    __m128i xmm_alpha_src_lo, xmm_alpha_src_hi;
+    __m128i xmm_alpha_dst_lo, xmm_alpha_dst_hi;
+    __m128i xmm_mask_lo, xmm_mask_hi;
+
+    /* call prefetch hint to optimize cache load*/
+    cache_prefetch ((__m128i*)ps);
+    cache_prefetch ((__m128i*)pd);
+    cache_prefetch ((__m128i*)pm);
+
+    while (w && (unsigned long)pd & 15)
+    {
+	s = *ps++;
+	m = *pm++;
+	d = *pd;
+
+	*pd++ = core_combine_atop_ca_pixel_sse2 (s, m, d);
+	w--;
+    }
+
+    /* call prefetch hint to optimize cache load*/
+    cache_prefetch ((__m128i*)ps);
+    cache_prefetch ((__m128i*)pd);
+    cache_prefetch ((__m128i*)pm);
+
+    while (w >= 4)
+    {
+	/* fill cache line with next memory */
+	cache_prefetch_next ((__m128i*)ps);
+	cache_prefetch_next ((__m128i*)pd);
+	cache_prefetch_next ((__m128i*)pm);
+
+	xmm_dst_hi = load_128_aligned ((__m128i*)pd);
+	xmm_src_hi = load_128_unaligned ((__m128i*)ps);
+	xmm_mask_hi = load_128_unaligned ((__m128i*)pm);
+
+	unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
+	unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
+	unpack_128_2x128 (xmm_mask_hi, &xmm_mask_lo, &xmm_mask_hi);
+
+	expand_alpha_2x128 (xmm_src_lo, xmm_src_hi,
+			    &xmm_alpha_src_lo, &xmm_alpha_src_hi);
+	expand_alpha_2x128 (xmm_dst_lo, xmm_dst_hi,
+			    &xmm_alpha_dst_lo, &xmm_alpha_dst_hi);
+
+	pix_multiply_2x128 (&xmm_src_lo, &xmm_src_hi,
+			    &xmm_mask_lo, &xmm_mask_hi,
+			    &xmm_src_lo, &xmm_src_hi);
+	pix_multiply_2x128 (&xmm_mask_lo, &xmm_mask_hi,
+			    &xmm_alpha_src_lo, &xmm_alpha_src_hi,
+			    &xmm_mask_lo, &xmm_mask_hi);
+
+	negate_2x128 (xmm_mask_lo, xmm_mask_hi, &xmm_mask_lo, &xmm_mask_hi);
+
+	pix_add_multiply_2x128 (
+	    &xmm_dst_lo, &xmm_dst_hi, &xmm_mask_lo, &xmm_mask_hi,
+	    &xmm_src_lo, &xmm_src_hi, &xmm_alpha_dst_lo, &xmm_alpha_dst_hi,
+	    &xmm_dst_lo, &xmm_dst_hi);
+
+	save_128_aligned (
+	    (__m128i*)pd, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
+
+	ps += 4;
+	pd += 4;
+	pm += 4;
+	w -= 4;
+    }
+
+    while (w)
+    {
+	s = *ps++;
+	m = *pm++;
+	d = *pd;
+
+	*pd++ = core_combine_atop_ca_pixel_sse2 (s, m, d);
+	w--;
+    }
+}
+
+static force_inline uint32_t
+core_combine_reverse_atop_ca_pixel_sse2 (uint32_t src,
+                                         uint32_t mask,
+                                         uint32_t dst)
+{
+    __m64 m = unpack_32_1x64 (mask);
+    __m64 s = unpack_32_1x64 (src);
+    __m64 d = unpack_32_1x64 (dst);
+
+    __m64 da = negate_1x64 (expand_alpha_1x64 (d));
+    __m64 sa = expand_alpha_1x64 (s);
+
+    s = pix_multiply_1x64 (s, m);
+    m = pix_multiply_1x64 (m, sa);
+
+    return pack_1x64_32 (pix_add_multiply_1x64 (&d, &m, &s, &da));
+}
+
+static force_inline void
+core_combine_reverse_atop_ca_sse2 (uint32_t *      pd,
+                                   const uint32_t *ps,
+                                   const uint32_t *pm,
+                                   int             w)
+{
+    uint32_t s, m, d;
+
+    __m128i xmm_src_lo, xmm_src_hi;
+    __m128i xmm_dst_lo, xmm_dst_hi;
+    __m128i xmm_alpha_src_lo, xmm_alpha_src_hi;
+    __m128i xmm_alpha_dst_lo, xmm_alpha_dst_hi;
+    __m128i xmm_mask_lo, xmm_mask_hi;
+
+    /* call prefetch hint to optimize cache load*/
+    cache_prefetch ((__m128i*)ps);
+    cache_prefetch ((__m128i*)pd);
+    cache_prefetch ((__m128i*)pm);
+
+    while (w && (unsigned long)pd & 15)
+    {
+	s = *ps++;
+	m = *pm++;
+	d = *pd;
+
+	*pd++ = core_combine_reverse_atop_ca_pixel_sse2 (s, m, d);
+	w--;
+    }
+
+    /* call prefetch hint to optimize cache load*/
+    cache_prefetch ((__m128i*)ps);
+    cache_prefetch ((__m128i*)pd);
+    cache_prefetch ((__m128i*)pm);
+
+    while (w >= 4)
+    {
+	/* fill cache line with next memory */
+	cache_prefetch_next ((__m128i*)ps);
+	cache_prefetch_next ((__m128i*)pd);
+	cache_prefetch_next ((__m128i*)pm);
+
+	xmm_dst_hi = load_128_aligned ((__m128i*)pd);
+	xmm_src_hi = load_128_unaligned ((__m128i*)ps);
+	xmm_mask_hi = load_128_unaligned ((__m128i*)pm);
+
+	unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
+	unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
+	unpack_128_2x128 (xmm_mask_hi, &xmm_mask_lo, &xmm_mask_hi);
+
+	expand_alpha_2x128 (xmm_src_lo, xmm_src_hi,
+			    &xmm_alpha_src_lo, &xmm_alpha_src_hi);
+	expand_alpha_2x128 (xmm_dst_lo, xmm_dst_hi,
+			    &xmm_alpha_dst_lo, &xmm_alpha_dst_hi);
+
+	pix_multiply_2x128 (&xmm_src_lo, &xmm_src_hi,
+			    &xmm_mask_lo, &xmm_mask_hi,
+			    &xmm_src_lo, &xmm_src_hi);
+	pix_multiply_2x128 (&xmm_mask_lo, &xmm_mask_hi,
+			    &xmm_alpha_src_lo, &xmm_alpha_src_hi,
+			    &xmm_mask_lo, &xmm_mask_hi);
+
+	negate_2x128 (xmm_alpha_dst_lo, xmm_alpha_dst_hi,
+		      &xmm_alpha_dst_lo, &xmm_alpha_dst_hi);
+
+	pix_add_multiply_2x128 (
+	    &xmm_dst_lo, &xmm_dst_hi, &xmm_mask_lo, &xmm_mask_hi,
+	    &xmm_src_lo, &xmm_src_hi, &xmm_alpha_dst_lo, &xmm_alpha_dst_hi,
+	    &xmm_dst_lo, &xmm_dst_hi);
+
+	save_128_aligned (
+	    (__m128i*)pd, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
+
+	ps += 4;
+	pd += 4;
+	pm += 4;
+	w -= 4;
+    }
+
+    while (w)
+    {
+	s = *ps++;
+	m = *pm++;
+	d = *pd;
+
+	*pd++ = core_combine_reverse_atop_ca_pixel_sse2 (s, m, d);
+	w--;
+    }
+}
+
+static force_inline uint32_t
+core_combine_xor_ca_pixel_sse2 (uint32_t src,
+                                uint32_t mask,
+                                uint32_t dst)
+{
+    __m64 a = unpack_32_1x64 (mask);
+    __m64 s = unpack_32_1x64 (src);
+    __m64 d = unpack_32_1x64 (dst);
+
+    __m64 alpha_dst = negate_1x64 (pix_multiply_1x64 (
+				       a, expand_alpha_1x64 (s)));
+    __m64 dest      = pix_multiply_1x64 (s, a);
+    __m64 alpha_src = negate_1x64 (expand_alpha_1x64 (d));
+
+    return pack_1x64_32 (pix_add_multiply_1x64 (&d,
+                                                &alpha_dst,
+                                                &dest,
+                                                &alpha_src));
+}
+
+static force_inline void
+core_combine_xor_ca_sse2 (uint32_t *      pd,
+                          const uint32_t *ps,
+                          const uint32_t *pm,
+                          int             w)
+{
+    uint32_t s, m, d;
+
+    __m128i xmm_src_lo, xmm_src_hi;
+    __m128i xmm_dst_lo, xmm_dst_hi;
+    __m128i xmm_alpha_src_lo, xmm_alpha_src_hi;
+    __m128i xmm_alpha_dst_lo, xmm_alpha_dst_hi;
+    __m128i xmm_mask_lo, xmm_mask_hi;
+
+    /* call prefetch hint to optimize cache load*/
+    cache_prefetch ((__m128i*)ps);
+    cache_prefetch ((__m128i*)pd);
+    cache_prefetch ((__m128i*)pm);
+
+    while (w && (unsigned long)pd & 15)
+    {
+	s = *ps++;
+	m = *pm++;
+	d = *pd;
+
+	*pd++ = core_combine_xor_ca_pixel_sse2 (s, m, d);
+	w--;
+    }
+
+    /* call prefetch hint to optimize cache load*/
+    cache_prefetch ((__m128i*)ps);
+    cache_prefetch ((__m128i*)pd);
+    cache_prefetch ((__m128i*)pm);
+
+    while (w >= 4)
+    {
+	/* fill cache line with next memory */
+	cache_prefetch_next ((__m128i*)ps);
+	cache_prefetch_next ((__m128i*)pd);
+	cache_prefetch_next ((__m128i*)pm);
+
+	xmm_dst_hi = load_128_aligned ((__m128i*)pd);
+	xmm_src_hi = load_128_unaligned ((__m128i*)ps);
+	xmm_mask_hi = load_128_unaligned ((__m128i*)pm);
+
+	unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
+	unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
+	unpack_128_2x128 (xmm_mask_hi, &xmm_mask_lo, &xmm_mask_hi);
+
+	expand_alpha_2x128 (xmm_src_lo, xmm_src_hi,
+			    &xmm_alpha_src_lo, &xmm_alpha_src_hi);
+	expand_alpha_2x128 (xmm_dst_lo, xmm_dst_hi,
+			    &xmm_alpha_dst_lo, &xmm_alpha_dst_hi);
+
+	pix_multiply_2x128 (&xmm_src_lo, &xmm_src_hi,
+			    &xmm_mask_lo, &xmm_mask_hi,
+			    &xmm_src_lo, &xmm_src_hi);
+	pix_multiply_2x128 (&xmm_mask_lo, &xmm_mask_hi,
+			    &xmm_alpha_src_lo, &xmm_alpha_src_hi,
+			    &xmm_mask_lo, &xmm_mask_hi);
+
+	negate_2x128 (xmm_alpha_dst_lo, xmm_alpha_dst_hi,
+		      &xmm_alpha_dst_lo, &xmm_alpha_dst_hi);
+	negate_2x128 (xmm_mask_lo, xmm_mask_hi,
+		      &xmm_mask_lo, &xmm_mask_hi);
+
+	pix_add_multiply_2x128 (
+	    &xmm_dst_lo, &xmm_dst_hi, &xmm_mask_lo, &xmm_mask_hi,
+	    &xmm_src_lo, &xmm_src_hi, &xmm_alpha_dst_lo, &xmm_alpha_dst_hi,
+	    &xmm_dst_lo, &xmm_dst_hi);
+
+	save_128_aligned (
+	    (__m128i*)pd, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
+
+	ps += 4;
+	pd += 4;
+	pm += 4;
+	w -= 4;
+    }
+
+    while (w)
+    {
+	s = *ps++;
+	m = *pm++;
+	d = *pd;
+
+	*pd++ = core_combine_xor_ca_pixel_sse2 (s, m, d);
+	w--;
+    }
+}
+
+static force_inline void
+core_combine_add_ca_sse2 (uint32_t *      pd,
+                          const uint32_t *ps,
+                          const uint32_t *pm,
+                          int             w)
+{
+    uint32_t s, m, d;
+
+    __m128i xmm_src_lo, xmm_src_hi;
+    __m128i xmm_dst_lo, xmm_dst_hi;
+    __m128i xmm_mask_lo, xmm_mask_hi;
+
+    /* call prefetch hint to optimize cache load*/
+    cache_prefetch ((__m128i*)ps);
+    cache_prefetch ((__m128i*)pd);
+    cache_prefetch ((__m128i*)pm);
+
+    while (w && (unsigned long)pd & 15)
+    {
+	s = *ps++;
+	m = *pm++;
+	d = *pd;
+
+	*pd++ = pack_1x64_32 (
+	    _mm_adds_pu8 (pix_multiply_1x64 (unpack_32_1x64 (s),
+					     unpack_32_1x64 (m)),
+			  unpack_32_1x64 (d)));
+	w--;
+    }
+
+    /* call prefetch hint to optimize cache load*/
+    cache_prefetch ((__m128i*)ps);
+    cache_prefetch ((__m128i*)pd);
+    cache_prefetch ((__m128i*)pm);
+
+    while (w >= 4)
+    {
+	/* fill cache line with next memory */
+	cache_prefetch_next ((__m128i*)ps);
+	cache_prefetch_next ((__m128i*)pd);
+	cache_prefetch_next ((__m128i*)pm);
+
+	xmm_src_hi = load_128_unaligned ((__m128i*)ps);
+	xmm_mask_hi = load_128_unaligned ((__m128i*)pm);
+	xmm_dst_hi = load_128_aligned ((__m128i*)pd);
+
+	unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
+	unpack_128_2x128 (xmm_mask_hi, &xmm_mask_lo, &xmm_mask_hi);
+	unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
+
+	pix_multiply_2x128 (&xmm_src_lo, &xmm_src_hi,
+			    &xmm_mask_lo, &xmm_mask_hi,
+			    &xmm_src_lo, &xmm_src_hi);
+
+	save_128_aligned (
+	    (__m128i*)pd, pack_2x128_128 (
+		_mm_adds_epu8 (xmm_src_lo, xmm_dst_lo),
+		_mm_adds_epu8 (xmm_src_hi, xmm_dst_hi)));
+
+	ps += 4;
+	pd += 4;
+	pm += 4;
+	w -= 4;
+    }
+
+    while (w)
+    {
+	s = *ps++;
+	m = *pm++;
+	d = *pd;
+
+	*pd++ = pack_1x64_32 (
+	    _mm_adds_pu8 (pix_multiply_1x64 (unpack_32_1x64 (s),
+					     unpack_32_1x64 (m)),
+			  unpack_32_1x64 (d)));
+	w--;
+    }
+}
+
+/* ---------------------------------------------------
+ * fb_compose_setup_sSE2
+ */
+static force_inline __m64
+create_mask_16_64 (uint16_t mask)
+{
+    return _mm_set1_pi16 (mask);
+}
+
+static force_inline __m128i
+create_mask_16_128 (uint16_t mask)
+{
+    return _mm_set1_epi16 (mask);
+}
+
+static force_inline __m64
+create_mask_2x32_64 (uint32_t mask0,
+                     uint32_t mask1)
+{
+    return _mm_set_pi32 (mask0, mask1);
+}
+
+/* Work around a code generation bug in Sun Studio 12. */
+#if defined(__SUNPRO_C) && (__SUNPRO_C >= 0x590)
+# define create_mask_2x32_128(mask0, mask1)				\
+    (_mm_set_epi32 ((mask0), (mask1), (mask0), (mask1)))
+#else
+static force_inline __m128i
+create_mask_2x32_128 (uint32_t mask0,
+                      uint32_t mask1)
+{
+    return _mm_set_epi32 (mask0, mask1, mask0, mask1);
+}
+#endif
+
+/* SSE2 code patch for fbcompose.c */
+
+static void
+sse2_combine_over_u (pixman_implementation_t *imp,
+                     pixman_op_t              op,
+                     uint32_t *               dst,
+                     const uint32_t *         src,
+                     const uint32_t *         mask,
+                     int                      width)
+{
+    core_combine_over_u_sse2 (dst, src, mask, width);
+    _mm_empty ();
+}
+
+static void
+sse2_combine_over_reverse_u (pixman_implementation_t *imp,
+                             pixman_op_t              op,
+                             uint32_t *               dst,
+                             const uint32_t *         src,
+                             const uint32_t *         mask,
+                             int                      width)
+{
+    core_combine_over_reverse_u_sse2 (dst, src, mask, width);
+    _mm_empty ();
+}
+
+static void
+sse2_combine_in_u (pixman_implementation_t *imp,
+                   pixman_op_t              op,
+                   uint32_t *               dst,
+                   const uint32_t *         src,
+                   const uint32_t *         mask,
+                   int                      width)
+{
+    core_combine_in_u_sse2 (dst, src, mask, width);
+    _mm_empty ();
+}
+
+static void
+sse2_combine_in_reverse_u (pixman_implementation_t *imp,
+                           pixman_op_t              op,
+                           uint32_t *               dst,
+                           const uint32_t *         src,
+                           const uint32_t *         mask,
+                           int                      width)
+{
+    core_combine_reverse_in_u_sse2 (dst, src, mask, width);
+    _mm_empty ();
+}
+
+static void
+sse2_combine_out_u (pixman_implementation_t *imp,
+                    pixman_op_t              op,
+                    uint32_t *               dst,
+                    const uint32_t *         src,
+                    const uint32_t *         mask,
+                    int                      width)
+{
+    core_combine_out_u_sse2 (dst, src, mask, width);
+    _mm_empty ();
+}
+
+static void
+sse2_combine_out_reverse_u (pixman_implementation_t *imp,
+                            pixman_op_t              op,
+                            uint32_t *               dst,
+                            const uint32_t *         src,
+                            const uint32_t *         mask,
+                            int                      width)
+{
+    core_combine_reverse_out_u_sse2 (dst, src, mask, width);
+    _mm_empty ();
+}
+
+static void
+sse2_combine_atop_u (pixman_implementation_t *imp,
+                     pixman_op_t              op,
+                     uint32_t *               dst,
+                     const uint32_t *         src,
+                     const uint32_t *         mask,
+                     int                      width)
+{
+    core_combine_atop_u_sse2 (dst, src, mask, width);
+    _mm_empty ();
+}
+
+static void
+sse2_combine_atop_reverse_u (pixman_implementation_t *imp,
+                             pixman_op_t              op,
+                             uint32_t *               dst,
+                             const uint32_t *         src,
+                             const uint32_t *         mask,
+                             int                      width)
+{
+    core_combine_reverse_atop_u_sse2 (dst, src, mask, width);
+    _mm_empty ();
+}
+
+static void
+sse2_combine_xor_u (pixman_implementation_t *imp,
+                    pixman_op_t              op,
+                    uint32_t *               dst,
+                    const uint32_t *         src,
+                    const uint32_t *         mask,
+                    int                      width)
+{
+    core_combine_xor_u_sse2 (dst, src, mask, width);
+    _mm_empty ();
+}
+
+static void
+sse2_combine_add_u (pixman_implementation_t *imp,
+                    pixman_op_t              op,
+                    uint32_t *               dst,
+                    const uint32_t *         src,
+                    const uint32_t *         mask,
+                    int                      width)
+{
+    core_combine_add_u_sse2 (dst, src, mask, width);
+    _mm_empty ();
+}
+
+static void
+sse2_combine_saturate_u (pixman_implementation_t *imp,
+                         pixman_op_t              op,
+                         uint32_t *               dst,
+                         const uint32_t *         src,
+                         const uint32_t *         mask,
+                         int                      width)
+{
+    core_combine_saturate_u_sse2 (dst, src, mask, width);
+    _mm_empty ();
+}
+
+static void
+sse2_combine_src_ca (pixman_implementation_t *imp,
+                     pixman_op_t              op,
+                     uint32_t *               dst,
+                     const uint32_t *         src,
+                     const uint32_t *         mask,
+                     int                      width)
+{
+    core_combine_src_ca_sse2 (dst, src, mask, width);
+    _mm_empty ();
+}
+
+static void
+sse2_combine_over_ca (pixman_implementation_t *imp,
+                      pixman_op_t              op,
+                      uint32_t *               dst,
+                      const uint32_t *         src,
+                      const uint32_t *         mask,
+                      int                      width)
+{
+    core_combine_over_ca_sse2 (dst, src, mask, width);
+    _mm_empty ();
+}
+
+static void
+sse2_combine_over_reverse_ca (pixman_implementation_t *imp,
+                              pixman_op_t              op,
+                              uint32_t *               dst,
+                              const uint32_t *         src,
+                              const uint32_t *         mask,
+                              int                      width)
+{
+    core_combine_over_reverse_ca_sse2 (dst, src, mask, width);
+    _mm_empty ();
+}
+
+static void
+sse2_combine_in_ca (pixman_implementation_t *imp,
+                    pixman_op_t              op,
+                    uint32_t *               dst,
+                    const uint32_t *         src,
+                    const uint32_t *         mask,
+                    int                      width)
+{
+    core_combine_in_ca_sse2 (dst, src, mask, width);
+    _mm_empty ();
+}
+
+static void
+sse2_combine_in_reverse_ca (pixman_implementation_t *imp,
+                            pixman_op_t              op,
+                            uint32_t *               dst,
+                            const uint32_t *         src,
+                            const uint32_t *         mask,
+                            int                      width)
+{
+    core_combine_in_reverse_ca_sse2 (dst, src, mask, width);
+    _mm_empty ();
+}
+
+static void
+sse2_combine_out_ca (pixman_implementation_t *imp,
+                     pixman_op_t              op,
+                     uint32_t *               dst,
+                     const uint32_t *         src,
+                     const uint32_t *         mask,
+                     int                      width)
+{
+    core_combine_out_ca_sse2 (dst, src, mask, width);
+    _mm_empty ();
+}
+
+static void
+sse2_combine_out_reverse_ca (pixman_implementation_t *imp,
+                             pixman_op_t              op,
+                             uint32_t *               dst,
+                             const uint32_t *         src,
+                             const uint32_t *         mask,
+                             int                      width)
+{
+    core_combine_out_reverse_ca_sse2 (dst, src, mask, width);
+    _mm_empty ();
+}
+
+static void
+sse2_combine_atop_ca (pixman_implementation_t *imp,
+                      pixman_op_t              op,
+                      uint32_t *               dst,
+                      const uint32_t *         src,
+                      const uint32_t *         mask,
+                      int                      width)
+{
+    core_combine_atop_ca_sse2 (dst, src, mask, width);
+    _mm_empty ();
+}
+
+static void
+sse2_combine_atop_reverse_ca (pixman_implementation_t *imp,
+                              pixman_op_t              op,
+                              uint32_t *               dst,
+                              const uint32_t *         src,
+                              const uint32_t *         mask,
+                              int                      width)
+{
+    core_combine_reverse_atop_ca_sse2 (dst, src, mask, width);
+    _mm_empty ();
+}
+
+static void
+sse2_combine_xor_ca (pixman_implementation_t *imp,
+                     pixman_op_t              op,
+                     uint32_t *               dst,
+                     const uint32_t *         src,
+                     const uint32_t *         mask,
+                     int                      width)
+{
+    core_combine_xor_ca_sse2 (dst, src, mask, width);
+    _mm_empty ();
+}
+
+static void
+sse2_combine_add_ca (pixman_implementation_t *imp,
+                     pixman_op_t              op,
+                     uint32_t *               dst,
+                     const uint32_t *         src,
+                     const uint32_t *         mask,
+                     int                      width)
+{
+    core_combine_add_ca_sse2 (dst, src, mask, width);
+    _mm_empty ();
+}
+
+/* -------------------------------------------------------------------
+ * composite_over_n_8888
+ */
+
+static void
+sse2_composite_over_n_8888 (pixman_implementation_t *imp,
+                            pixman_op_t              op,
+                            pixman_image_t *         src_image,
+                            pixman_image_t *         mask_image,
+                            pixman_image_t *         dst_image,
+                            int32_t                  src_x,
+                            int32_t                  src_y,
+                            int32_t                  mask_x,
+                            int32_t                  mask_y,
+                            int32_t                  dest_x,
+                            int32_t                  dest_y,
+                            int32_t                  width,
+                            int32_t                  height)
+{
+    uint32_t src;
+    uint32_t    *dst_line, *dst, d;
+    int32_t w;
+    int dst_stride;
+    __m128i xmm_src, xmm_alpha;
+    __m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
+
+    src = _pixman_image_get_solid (src_image, dst_image->bits.format);
+
+    if (src == 0)
+	return;
+
+    PIXMAN_IMAGE_GET_LINE (
+	dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
+
+    xmm_src = expand_pixel_32_1x128 (src);
+    xmm_alpha = expand_alpha_1x128 (xmm_src);
+
+    while (height--)
+    {
+	dst = dst_line;
+
+	/* call prefetch hint to optimize cache load*/
+	cache_prefetch ((__m128i*)dst);
+
+	dst_line += dst_stride;
+	w = width;
+
+	while (w && (unsigned long)dst & 15)
+	{
+	    d = *dst;
+	    *dst++ = pack_1x64_32 (over_1x64 (_mm_movepi64_pi64 (xmm_src),
+	                                      _mm_movepi64_pi64 (xmm_alpha),
+	                                      unpack_32_1x64 (d)));
+	    w--;
+	}
+
+	cache_prefetch ((__m128i*)dst);
+
+	while (w >= 4)
+	{
+	    /* fill cache line with next memory */
+	    cache_prefetch_next ((__m128i*)dst);
+
+	    xmm_dst = load_128_aligned ((__m128i*)dst);
+
+	    unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
+
+	    over_2x128 (&xmm_src, &xmm_src,
+			&xmm_alpha, &xmm_alpha,
+			&xmm_dst_lo, &xmm_dst_hi);
+
+	    /* rebuid the 4 pixel data and save*/
+	    save_128_aligned (
+		(__m128i*)dst, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
+
+	    w -= 4;
+	    dst += 4;
+	}
+
+	while (w)
+	{
+	    d = *dst;
+	    *dst++ = pack_1x64_32 (over_1x64 (_mm_movepi64_pi64 (xmm_src),
+	                                      _mm_movepi64_pi64 (xmm_alpha),
+	                                      unpack_32_1x64 (d)));
+	    w--;
+	}
+
+    }
+    _mm_empty ();
+}
+
+/* ---------------------------------------------------------------------
+ * composite_over_n_0565
+ */
+static void
+sse2_composite_over_n_0565 (pixman_implementation_t *imp,
+                            pixman_op_t              op,
+                            pixman_image_t *         src_image,
+                            pixman_image_t *         mask_image,
+                            pixman_image_t *         dst_image,
+                            int32_t                  src_x,
+                            int32_t                  src_y,
+                            int32_t                  mask_x,
+                            int32_t                  mask_y,
+                            int32_t                  dest_x,
+                            int32_t                  dest_y,
+                            int32_t                  width,
+                            int32_t                  height)
+{
+    uint32_t src;
+    uint16_t    *dst_line, *dst, d;
+    int32_t w;
+    int dst_stride;
+    __m128i xmm_src, xmm_alpha;
+    __m128i xmm_dst, xmm_dst0, xmm_dst1, xmm_dst2, xmm_dst3;
+
+    src = _pixman_image_get_solid (src_image, dst_image->bits.format);
+
+    if (src == 0)
+	return;
+
+    PIXMAN_IMAGE_GET_LINE (
+	dst_image, dest_x, dest_y, uint16_t, dst_stride, dst_line, 1);
+
+    xmm_src = expand_pixel_32_1x128 (src);
+    xmm_alpha = expand_alpha_1x128 (xmm_src);
+
+    while (height--)
+    {
+	dst = dst_line;
+
+	/* call prefetch hint to optimize cache load*/
+	cache_prefetch ((__m128i*)dst);
+
+	dst_line += dst_stride;
+	w = width;
+
+	while (w && (unsigned long)dst & 15)
+	{
+	    d = *dst;
+
+	    *dst++ = pack_565_32_16 (
+		pack_1x64_32 (over_1x64 (_mm_movepi64_pi64 (xmm_src),
+					 _mm_movepi64_pi64 (xmm_alpha),
+					 expand565_16_1x64 (d))));
+	    w--;
+	}
+
+	/* call prefetch hint to optimize cache load*/
+	cache_prefetch ((__m128i*)dst);
+
+	while (w >= 8)
+	{
+	    /* fill cache line with next memory */
+	    cache_prefetch_next ((__m128i*)dst);
+
+	    xmm_dst = load_128_aligned ((__m128i*)dst);
+
+	    unpack_565_128_4x128 (xmm_dst,
+				  &xmm_dst0, &xmm_dst1, &xmm_dst2, &xmm_dst3);
+
+	    over_2x128 (&xmm_src, &xmm_src,
+			&xmm_alpha, &xmm_alpha,
+			&xmm_dst0, &xmm_dst1);
+	    over_2x128 (&xmm_src, &xmm_src,
+			&xmm_alpha, &xmm_alpha,
+			&xmm_dst2, &xmm_dst3);
+
+	    xmm_dst = pack_565_4x128_128 (
+		&xmm_dst0, &xmm_dst1, &xmm_dst2, &xmm_dst3);
+
+	    save_128_aligned ((__m128i*)dst, xmm_dst);
+
+	    dst += 8;
+	    w -= 8;
+	}
+
+	while (w--)
+	{
+	    d = *dst;
+	    *dst++ = pack_565_32_16 (
+		pack_1x64_32 (over_1x64 (_mm_movepi64_pi64 (xmm_src),
+					 _mm_movepi64_pi64 (xmm_alpha),
+					 expand565_16_1x64 (d))));
+	}
+    }
+
+    _mm_empty ();
+}
+
+/* ------------------------------
+ * composite_add_n_8888_8888_ca
+ */
+static void
+sse2_composite_add_n_8888_8888_ca (pixman_implementation_t *imp,
+				   pixman_op_t              op,
+				   pixman_image_t *         src_image,
+				   pixman_image_t *         mask_image,
+				   pixman_image_t *         dst_image,
+				   int32_t                  src_x,
+				   int32_t                  src_y,
+				   int32_t                  mask_x,
+				   int32_t                  mask_y,
+				   int32_t                  dest_x,
+				   int32_t                  dest_y,
+				   int32_t                  width,
+				   int32_t                  height)
+{
+    uint32_t src, srca;
+    uint32_t    *dst_line, d;
+    uint32_t    *mask_line, m;
+    uint32_t pack_cmp;
+    int dst_stride, mask_stride;
+
+    __m128i xmm_src, xmm_alpha;
+    __m128i xmm_dst;
+    __m128i xmm_mask, xmm_mask_lo, xmm_mask_hi;
+
+    __m64 mmx_src, mmx_alpha, mmx_mask, mmx_dest;
+
+    src = _pixman_image_get_solid (src_image, dst_image->bits.format);
+    srca = src >> 24;
+
+    if (src == 0)
+	return;
+
+    PIXMAN_IMAGE_GET_LINE (
+	dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
+    PIXMAN_IMAGE_GET_LINE (
+	mask_image, mask_x, mask_y, uint32_t, mask_stride, mask_line, 1);
+
+    xmm_src = _mm_unpacklo_epi8 (
+	create_mask_2x32_128 (src, src), _mm_setzero_si128 ());
+    xmm_alpha = expand_alpha_1x128 (xmm_src);
+    mmx_src   = _mm_movepi64_pi64 (xmm_src);
+    mmx_alpha = _mm_movepi64_pi64 (xmm_alpha);
+
+    while (height--)
+    {
+	int w = width;
+	const uint32_t *pm = (uint32_t *)mask_line;
+	uint32_t *pd = (uint32_t *)dst_line;
+
+	dst_line += dst_stride;
+	mask_line += mask_stride;
+
+	/* call prefetch hint to optimize cache load*/
+	cache_prefetch ((__m128i*)pd);
+	cache_prefetch ((__m128i*)pm);
+
+	while (w && (unsigned long)pd & 15)
+	{
+	    m = *pm++;
+
+	    if (m)
+	    {
+		d = *pd;
+
+		mmx_mask = unpack_32_1x64 (m);
+		mmx_dest = unpack_32_1x64 (d);
+
+		*pd = pack_1x64_32 (
+		    _mm_adds_pu8 (pix_multiply_1x64 (mmx_mask, mmx_src), mmx_dest));
+	    }
+
+	    pd++;
+	    w--;
+	}
+
+	/* call prefetch hint to optimize cache load*/
+	cache_prefetch ((__m128i*)pd);
+	cache_prefetch ((__m128i*)pm);
+
+	while (w >= 4)
+	{
+	    /* fill cache line with next memory */
+	    cache_prefetch_next ((__m128i*)pd);
+	    cache_prefetch_next ((__m128i*)pm);
+
+	    xmm_mask = load_128_unaligned ((__m128i*)pm);
+
+	    pack_cmp =
+		_mm_movemask_epi8 (
+		    _mm_cmpeq_epi32 (xmm_mask, _mm_setzero_si128 ()));
+
+	    /* if all bits in mask are zero, pack_cmp are equal to 0xffff */
+	    if (pack_cmp != 0xffff)
+	    {
+		xmm_dst = load_128_aligned ((__m128i*)pd);
+
+		unpack_128_2x128 (xmm_mask, &xmm_mask_lo, &xmm_mask_hi);
+
+		pix_multiply_2x128 (&xmm_src, &xmm_src,
+				    &xmm_mask_lo, &xmm_mask_hi,
+				    &xmm_mask_lo, &xmm_mask_hi);
+		xmm_mask_hi = pack_2x128_128 (xmm_mask_lo, xmm_mask_hi);
+
+		save_128_aligned (
+		    (__m128i*)pd, _mm_adds_epu8 (xmm_mask_hi, xmm_dst));
+	    }
+
+	    pd += 4;
+	    pm += 4;
+	    w -= 4;
+	}
+
+	while (w)
+	{
+	    m = *pm++;
+
+	    if (m)
+	    {
+		d = *pd;
+
+		mmx_mask = unpack_32_1x64 (m);
+		mmx_dest = unpack_32_1x64 (d);
+
+		*pd = pack_1x64_32 (
+		    _mm_adds_pu8 (pix_multiply_1x64 (mmx_mask, mmx_src), mmx_dest));
+	    }
+
+	    pd++;
+	    w--;
+	}
+    }
+
+    _mm_empty ();
+}
+
+/* ---------------------------------------------------------------------------
+ * composite_over_n_8888_8888_ca
+ */
+
+static void
+sse2_composite_over_n_8888_8888_ca (pixman_implementation_t *imp,
+                                    pixman_op_t              op,
+                                    pixman_image_t *         src_image,
+                                    pixman_image_t *         mask_image,
+                                    pixman_image_t *         dst_image,
+                                    int32_t                  src_x,
+                                    int32_t                  src_y,
+                                    int32_t                  mask_x,
+                                    int32_t                  mask_y,
+                                    int32_t                  dest_x,
+                                    int32_t                  dest_y,
+                                    int32_t                  width,
+                                    int32_t                  height)
+{
+    uint32_t src;
+    uint32_t    *dst_line, d;
+    uint32_t    *mask_line, m;
+    uint32_t pack_cmp;
+    int dst_stride, mask_stride;
+
+    __m128i xmm_src, xmm_alpha;
+    __m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
+    __m128i xmm_mask, xmm_mask_lo, xmm_mask_hi;
+
+    __m64 mmx_src, mmx_alpha, mmx_mask, mmx_dest;
+
+    src = _pixman_image_get_solid (src_image, dst_image->bits.format);
+
+    if (src == 0)
+	return;
+
+    PIXMAN_IMAGE_GET_LINE (
+	dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
+    PIXMAN_IMAGE_GET_LINE (
+	mask_image, mask_x, mask_y, uint32_t, mask_stride, mask_line, 1);
+
+    xmm_src = _mm_unpacklo_epi8 (
+	create_mask_2x32_128 (src, src), _mm_setzero_si128 ());
+    xmm_alpha = expand_alpha_1x128 (xmm_src);
+    mmx_src   = _mm_movepi64_pi64 (xmm_src);
+    mmx_alpha = _mm_movepi64_pi64 (xmm_alpha);
+
+    while (height--)
+    {
+	int w = width;
+	const uint32_t *pm = (uint32_t *)mask_line;
+	uint32_t *pd = (uint32_t *)dst_line;
+
+	dst_line += dst_stride;
+	mask_line += mask_stride;
+
+	/* call prefetch hint to optimize cache load*/
+	cache_prefetch ((__m128i*)pd);
+	cache_prefetch ((__m128i*)pm);
+
+	while (w && (unsigned long)pd & 15)
+	{
+	    m = *pm++;
+
+	    if (m)
+	    {
+		d = *pd;
+		mmx_mask = unpack_32_1x64 (m);
+		mmx_dest = unpack_32_1x64 (d);
+
+		*pd = pack_1x64_32 (in_over_1x64 (&mmx_src,
+		                                  &mmx_alpha,
+		                                  &mmx_mask,
+		                                  &mmx_dest));
+	    }
+
+	    pd++;
+	    w--;
+	}
+
+	/* call prefetch hint to optimize cache load*/
+	cache_prefetch ((__m128i*)pd);
+	cache_prefetch ((__m128i*)pm);
+
+	while (w >= 4)
+	{
+	    /* fill cache line with next memory */
+	    cache_prefetch_next ((__m128i*)pd);
+	    cache_prefetch_next ((__m128i*)pm);
+
+	    xmm_mask = load_128_unaligned ((__m128i*)pm);
+
+	    pack_cmp =
+		_mm_movemask_epi8 (
+		    _mm_cmpeq_epi32 (xmm_mask, _mm_setzero_si128 ()));
+
+	    /* if all bits in mask are zero, pack_cmp are equal to 0xffff */
+	    if (pack_cmp != 0xffff)
+	    {
+		xmm_dst = load_128_aligned ((__m128i*)pd);
+
+		unpack_128_2x128 (xmm_mask, &xmm_mask_lo, &xmm_mask_hi);
+		unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
+
+		in_over_2x128 (&xmm_src, &xmm_src,
+			       &xmm_alpha, &xmm_alpha,
+			       &xmm_mask_lo, &xmm_mask_hi,
+			       &xmm_dst_lo, &xmm_dst_hi);
+
+		save_128_aligned (
+		    (__m128i*)pd, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
+	    }
+
+	    pd += 4;
+	    pm += 4;
+	    w -= 4;
+	}
+
+	while (w)
+	{
+	    m = *pm++;
+
+	    if (m)
+	    {
+		d = *pd;
+		mmx_mask = unpack_32_1x64 (m);
+		mmx_dest = unpack_32_1x64 (d);
+
+		*pd = pack_1x64_32 (
+		    in_over_1x64 (&mmx_src, &mmx_alpha, &mmx_mask, &mmx_dest));
+	    }
+
+	    pd++;
+	    w--;
+	}
+    }
+
+    _mm_empty ();
+}
+
+/*---------------------------------------------------------------------
+ * composite_over_8888_n_8888
+ */
+
+static void
+sse2_composite_over_8888_n_8888 (pixman_implementation_t *imp,
+                                 pixman_op_t              op,
+                                 pixman_image_t *         src_image,
+                                 pixman_image_t *         mask_image,
+                                 pixman_image_t *         dst_image,
+                                 int32_t                  src_x,
+                                 int32_t                  src_y,
+                                 int32_t                  mask_x,
+                                 int32_t                  mask_y,
+                                 int32_t                  dest_x,
+                                 int32_t                  dest_y,
+                                 int32_t                  width,
+                                 int32_t                  height)
+{
+    uint32_t    *dst_line, *dst;
+    uint32_t    *src_line, *src;
+    uint32_t mask;
+    int32_t w;
+    int dst_stride, src_stride;
+
+    __m128i xmm_mask;
+    __m128i xmm_src, xmm_src_lo, xmm_src_hi;
+    __m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
+    __m128i xmm_alpha_lo, xmm_alpha_hi;
+
+    PIXMAN_IMAGE_GET_LINE (
+	dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
+    PIXMAN_IMAGE_GET_LINE (
+	src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
+
+    mask = _pixman_image_get_solid (mask_image, PIXMAN_a8r8g8b8);
+
+    xmm_mask = create_mask_16_128 (mask >> 24);
+
+    while (height--)
+    {
+	dst = dst_line;
+	dst_line += dst_stride;
+	src = src_line;
+	src_line += src_stride;
+	w = width;
+
+	/* call prefetch hint to optimize cache load*/
+	cache_prefetch ((__m128i*)dst);
+	cache_prefetch ((__m128i*)src);
+
+	while (w && (unsigned long)dst & 15)
+	{
+	    uint32_t s = *src++;
+	    uint32_t d = *dst;
+
+	    __m64 ms = unpack_32_1x64 (s);
+	    __m64 alpha    = expand_alpha_1x64 (ms);
+	    __m64 dest     = _mm_movepi64_pi64 (xmm_mask);
+	    __m64 alpha_dst = unpack_32_1x64 (d);
+
+	    *dst++ = pack_1x64_32 (
+		in_over_1x64 (&ms, &alpha, &dest, &alpha_dst));
+
+	    w--;
+	}
+
+	/* call prefetch hint to optimize cache load*/
+	cache_prefetch ((__m128i*)dst);
+	cache_prefetch ((__m128i*)src);
+
+	while (w >= 4)
+	{
+	    /* fill cache line with next memory */
+	    cache_prefetch_next ((__m128i*)dst);
+	    cache_prefetch_next ((__m128i*)src);
+
+	    xmm_src = load_128_unaligned ((__m128i*)src);
+	    xmm_dst = load_128_aligned ((__m128i*)dst);
+
+	    unpack_128_2x128 (xmm_src, &xmm_src_lo, &xmm_src_hi);
+	    unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
+	    expand_alpha_2x128 (xmm_src_lo, xmm_src_hi,
+				&xmm_alpha_lo, &xmm_alpha_hi);
+
+	    in_over_2x128 (&xmm_src_lo, &xmm_src_hi,
+			   &xmm_alpha_lo, &xmm_alpha_hi,
+			   &xmm_mask, &xmm_mask,
+			   &xmm_dst_lo, &xmm_dst_hi);
+
+	    save_128_aligned (
+		(__m128i*)dst, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
+
+	    dst += 4;
+	    src += 4;
+	    w -= 4;
+	}
+
+	while (w)
+	{
+	    uint32_t s = *src++;
+	    uint32_t d = *dst;
+
+	    __m64 ms = unpack_32_1x64 (s);
+	    __m64 alpha = expand_alpha_1x64 (ms);
+	    __m64 mask  = _mm_movepi64_pi64 (xmm_mask);
+	    __m64 dest  = unpack_32_1x64 (d);
+
+	    *dst++ = pack_1x64_32 (
+		in_over_1x64 (&ms, &alpha, &mask, &dest));
+
+	    w--;
+	}
+    }
+
+    _mm_empty ();
+}
+
+/* ---------------------------------------------------------------------
+ * composite_over_x888_n_8888
+ */
+static void
+sse2_composite_over_x888_n_8888 (pixman_implementation_t *imp,
+                                 pixman_op_t              op,
+                                 pixman_image_t *         src_image,
+                                 pixman_image_t *         mask_image,
+                                 pixman_image_t *         dst_image,
+                                 int32_t                  src_x,
+                                 int32_t                  src_y,
+                                 int32_t                  mask_x,
+                                 int32_t                  mask_y,
+                                 int32_t                  dest_x,
+                                 int32_t                  dest_y,
+                                 int32_t                  width,
+                                 int32_t                  height)
+{
+    uint32_t    *dst_line, *dst;
+    uint32_t    *src_line, *src;
+    uint32_t mask;
+    int dst_stride, src_stride;
+    int32_t w;
+
+    __m128i xmm_mask, xmm_alpha;
+    __m128i xmm_src, xmm_src_lo, xmm_src_hi;
+    __m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
+
+    PIXMAN_IMAGE_GET_LINE (
+	dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
+    PIXMAN_IMAGE_GET_LINE (
+	src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
+
+    mask = _pixman_image_get_solid (mask_image, PIXMAN_a8r8g8b8);
+
+    xmm_mask = create_mask_16_128 (mask >> 24);
+    xmm_alpha = mask_00ff;
+
+    while (height--)
+    {
+	dst = dst_line;
+	dst_line += dst_stride;
+	src = src_line;
+	src_line += src_stride;
+	w = width;
+
+	/* call prefetch hint to optimize cache load*/
+	cache_prefetch ((__m128i*)dst);
+	cache_prefetch ((__m128i*)src);
+
+	while (w && (unsigned long)dst & 15)
+	{
+	    uint32_t s = (*src++) | 0xff000000;
+	    uint32_t d = *dst;
+
+	    __m64 src   = unpack_32_1x64 (s);
+	    __m64 alpha = _mm_movepi64_pi64 (xmm_alpha);
+	    __m64 mask  = _mm_movepi64_pi64 (xmm_mask);
+	    __m64 dest  = unpack_32_1x64 (d);
+
+	    *dst++ = pack_1x64_32 (
+		in_over_1x64 (&src, &alpha, &mask, &dest));
+
+	    w--;
+	}
+
+	/* call prefetch hint to optimize cache load*/
+	cache_prefetch ((__m128i*)dst);
+	cache_prefetch ((__m128i*)src);
+
+	while (w >= 4)
+	{
+	    /* fill cache line with next memory */
+	    cache_prefetch_next ((__m128i*)dst);
+	    cache_prefetch_next ((__m128i*)src);
+
+	    xmm_src = _mm_or_si128 (
+		load_128_unaligned ((__m128i*)src), mask_ff000000);
+	    xmm_dst = load_128_aligned ((__m128i*)dst);
+
+	    unpack_128_2x128 (xmm_src, &xmm_src_lo, &xmm_src_hi);
+	    unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
+
+	    in_over_2x128 (&xmm_src_lo, &xmm_src_hi,
+			   &xmm_alpha, &xmm_alpha,
+			   &xmm_mask, &xmm_mask,
+			   &xmm_dst_lo, &xmm_dst_hi);
+
+	    save_128_aligned (
+		(__m128i*)dst, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
+
+	    dst += 4;
+	    src += 4;
+	    w -= 4;
+
+	}
+
+	while (w)
+	{
+	    uint32_t s = (*src++) | 0xff000000;
+	    uint32_t d = *dst;
+
+	    __m64 src  = unpack_32_1x64 (s);
+	    __m64 alpha = _mm_movepi64_pi64 (xmm_alpha);
+	    __m64 mask  = _mm_movepi64_pi64 (xmm_mask);
+	    __m64 dest  = unpack_32_1x64 (d);
+
+	    *dst++ = pack_1x64_32 (
+		in_over_1x64 (&src, &alpha, &mask, &dest));
+
+	    w--;
+	}
+    }
+
+    _mm_empty ();
+}
+
+/* --------------------------------------------------------------------
+ * composite_over_8888_8888
+ */
+static void
+sse2_composite_over_8888_8888 (pixman_implementation_t *imp,
+                               pixman_op_t              op,
+                               pixman_image_t *         src_image,
+                               pixman_image_t *         mask_image,
+                               pixman_image_t *         dst_image,
+                               int32_t                  src_x,
+                               int32_t                  src_y,
+                               int32_t                  mask_x,
+                               int32_t                  mask_y,
+                               int32_t                  dest_x,
+                               int32_t                  dest_y,
+                               int32_t                  width,
+                               int32_t                  height)
+{
+    int dst_stride, src_stride;
+    uint32_t    *dst_line, *dst;
+    uint32_t    *src_line, *src;
+
+    PIXMAN_IMAGE_GET_LINE (
+	dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
+    PIXMAN_IMAGE_GET_LINE (
+	src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
+
+    dst = dst_line;
+    src = src_line;
+
+    while (height--)
+    {
+	core_combine_over_u_sse2 (dst, src, NULL, width);
+
+	dst += dst_stride;
+	src += src_stride;
+    }
+    _mm_empty ();
+}
+
+/* ------------------------------------------------------------------
+ * composite_over_8888_0565
+ */
+static force_inline uint16_t
+composite_over_8888_0565pixel (uint32_t src, uint16_t dst)
+{
+    __m64 ms;
+
+    ms = unpack_32_1x64 (src);
+    return pack_565_32_16 (
+	pack_1x64_32 (
+	    over_1x64 (
+		ms, expand_alpha_1x64 (ms), expand565_16_1x64 (dst))));
+}
+
+static void
+sse2_composite_over_8888_0565 (pixman_implementation_t *imp,
+                               pixman_op_t              op,
+                               pixman_image_t *         src_image,
+                               pixman_image_t *         mask_image,
+                               pixman_image_t *         dst_image,
+                               int32_t                  src_x,
+                               int32_t                  src_y,
+                               int32_t                  mask_x,
+                               int32_t                  mask_y,
+                               int32_t                  dest_x,
+                               int32_t                  dest_y,
+                               int32_t                  width,
+                               int32_t                  height)
+{
+    uint16_t    *dst_line, *dst, d;
+    uint32_t    *src_line, *src, s;
+    int dst_stride, src_stride;
+    int32_t w;
+
+    __m128i xmm_alpha_lo, xmm_alpha_hi;
+    __m128i xmm_src, xmm_src_lo, xmm_src_hi;
+    __m128i xmm_dst, xmm_dst0, xmm_dst1, xmm_dst2, xmm_dst3;
+
+    PIXMAN_IMAGE_GET_LINE (
+	dst_image, dest_x, dest_y, uint16_t, dst_stride, dst_line, 1);
+    PIXMAN_IMAGE_GET_LINE (
+	src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
+
+#if 0
+    /* FIXME
+     *
+     * I copy the code from MMX one and keep the fixme.
+     * If it's a problem there, probably is a problem here.
+     */
+    assert (src_image->drawable == mask_image->drawable);
+#endif
+
+    while (height--)
+    {
+	dst = dst_line;
+	src = src_line;
+
+	/* call prefetch hint to optimize cache load*/
+	cache_prefetch ((__m128i*)src);
+	cache_prefetch ((__m128i*)dst);
+
+	dst_line += dst_stride;
+	src_line += src_stride;
+	w = width;
+
+	/* Align dst on a 16-byte boundary */
+	while (w &&
+	       ((unsigned long)dst & 15))
+	{
+	    s = *src++;
+	    d = *dst;
+
+	    *dst++ = composite_over_8888_0565pixel (s, d);
+	    w--;
+	}
+
+	/* call prefetch hint to optimize cache load*/
+	cache_prefetch ((__m128i*)src);
+	cache_prefetch ((__m128i*)dst);
+
+	/* It's a 8 pixel loop */
+	while (w >= 8)
+	{
+	    /* fill cache line with next memory */
+	    cache_prefetch_next ((__m128i*)src);
+	    cache_prefetch_next ((__m128i*)dst);
+
+	    /* I'm loading unaligned because I'm not sure
+	     * about the address alignment.
+	     */
+	    xmm_src = load_128_unaligned ((__m128i*) src);
+	    xmm_dst = load_128_aligned ((__m128i*) dst);
+
+	    /* Unpacking */
+	    unpack_128_2x128 (xmm_src, &xmm_src_lo, &xmm_src_hi);
+	    unpack_565_128_4x128 (xmm_dst,
+				  &xmm_dst0, &xmm_dst1, &xmm_dst2, &xmm_dst3);
+	    expand_alpha_2x128 (xmm_src_lo, xmm_src_hi,
+				&xmm_alpha_lo, &xmm_alpha_hi);
+
+	    /* I'm loading next 4 pixels from memory
+	     * before to optimze the memory read.
+	     */
+	    xmm_src = load_128_unaligned ((__m128i*) (src + 4));
+
+	    over_2x128 (&xmm_src_lo, &xmm_src_hi,
+			&xmm_alpha_lo, &xmm_alpha_hi,
+			&xmm_dst0, &xmm_dst1);
+
+	    /* Unpacking */
+	    unpack_128_2x128 (xmm_src, &xmm_src_lo, &xmm_src_hi);
+	    expand_alpha_2x128 (xmm_src_lo, xmm_src_hi,
+				&xmm_alpha_lo, &xmm_alpha_hi);
+
+	    over_2x128 (&xmm_src_lo, &xmm_src_hi,
+			&xmm_alpha_lo, &xmm_alpha_hi,
+			&xmm_dst2, &xmm_dst3);
+
+	    save_128_aligned (
+		(__m128i*)dst, pack_565_4x128_128 (
+		    &xmm_dst0, &xmm_dst1, &xmm_dst2, &xmm_dst3));
+
+	    w -= 8;
+	    dst += 8;
+	    src += 8;
+	}
+
+	while (w--)
+	{
+	    s = *src++;
+	    d = *dst;
+
+	    *dst++ = composite_over_8888_0565pixel (s, d);
+	}
+    }
+
+    _mm_empty ();
+}
+
+/* -----------------------------------------------------------------
+ * composite_over_n_8_8888
+ */
+
+static void
+sse2_composite_over_n_8_8888 (pixman_implementation_t *imp,
+                              pixman_op_t              op,
+                              pixman_image_t *         src_image,
+                              pixman_image_t *         mask_image,
+                              pixman_image_t *         dst_image,
+                              int32_t                  src_x,
+                              int32_t                  src_y,
+                              int32_t                  mask_x,
+                              int32_t                  mask_y,
+                              int32_t                  dest_x,
+                              int32_t                  dest_y,
+                              int32_t                  width,
+                              int32_t                  height)
+{
+    uint32_t src, srca;
+    uint32_t *dst_line, *dst;
+    uint8_t *mask_line, *mask;
+    int dst_stride, mask_stride;
+    int32_t w;
+    uint32_t m, d;
+
+    __m128i xmm_src, xmm_alpha, xmm_def;
+    __m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
+    __m128i xmm_mask, xmm_mask_lo, xmm_mask_hi;
+
+    __m64 mmx_src, mmx_alpha, mmx_mask, mmx_dest;
+
+    src = _pixman_image_get_solid (src_image, dst_image->bits.format);
+
+    srca = src >> 24;
+    if (src == 0)
+	return;
+
+    PIXMAN_IMAGE_GET_LINE (
+	dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
+    PIXMAN_IMAGE_GET_LINE (
+	mask_image, mask_x, mask_y, uint8_t, mask_stride, mask_line, 1);
+
+    xmm_def = create_mask_2x32_128 (src, src);
+    xmm_src = expand_pixel_32_1x128 (src);
+    xmm_alpha = expand_alpha_1x128 (xmm_src);
+    mmx_src   = _mm_movepi64_pi64 (xmm_src);
+    mmx_alpha = _mm_movepi64_pi64 (xmm_alpha);
+
+    while (height--)
+    {
+	dst = dst_line;
+	dst_line += dst_stride;
+	mask = mask_line;
+	mask_line += mask_stride;
+	w = width;
+
+	/* call prefetch hint to optimize cache load*/
+	cache_prefetch ((__m128i*)mask);
+	cache_prefetch ((__m128i*)dst);
+
+	while (w && (unsigned long)dst & 15)
+	{
+	    uint8_t m = *mask++;
+
+	    if (m)
+	    {
+		d = *dst;
+		mmx_mask = expand_pixel_8_1x64 (m);
+		mmx_dest = unpack_32_1x64 (d);
+
+		*dst = pack_1x64_32 (in_over_1x64 (&mmx_src,
+		                                   &mmx_alpha,
+		                                   &mmx_mask,
+		                                   &mmx_dest));
+	    }
+
+	    w--;
+	    dst++;
+	}
+
+	/* call prefetch hint to optimize cache load*/
+	cache_prefetch ((__m128i*)mask);
+	cache_prefetch ((__m128i*)dst);
+
+	while (w >= 4)
+	{
+	    /* fill cache line with next memory */
+	    cache_prefetch_next ((__m128i*)mask);
+	    cache_prefetch_next ((__m128i*)dst);
+
+	    m = *((uint32_t*)mask);
+
+	    if (srca == 0xff && m == 0xffffffff)
+	    {
+		save_128_aligned ((__m128i*)dst, xmm_def);
+	    }
+	    else if (m)
+	    {
+		xmm_dst = load_128_aligned ((__m128i*) dst);
+		xmm_mask = unpack_32_1x128 (m);
+		xmm_mask = _mm_unpacklo_epi8 (xmm_mask, _mm_setzero_si128 ());
+
+		/* Unpacking */
+		unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
+		unpack_128_2x128 (xmm_mask, &xmm_mask_lo, &xmm_mask_hi);
+
+		expand_alpha_rev_2x128 (xmm_mask_lo, xmm_mask_hi,
+					&xmm_mask_lo, &xmm_mask_hi);
+
+		in_over_2x128 (&xmm_src, &xmm_src,
+			       &xmm_alpha, &xmm_alpha,
+			       &xmm_mask_lo, &xmm_mask_hi,
+			       &xmm_dst_lo, &xmm_dst_hi);
+
+		save_128_aligned (
+		    (__m128i*)dst, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
+	    }
+
+	    w -= 4;
+	    dst += 4;
+	    mask += 4;
+	}
+
+	while (w)
+	{
+	    uint8_t m = *mask++;
+
+	    if (m)
+	    {
+		d = *dst;
+		mmx_mask = expand_pixel_8_1x64 (m);
+		mmx_dest = unpack_32_1x64 (d);
+
+		*dst = pack_1x64_32 (in_over_1x64 (&mmx_src,
+		                                   &mmx_alpha,
+		                                   &mmx_mask,
+		                                   &mmx_dest));
+	    }
+
+	    w--;
+	    dst++;
+	}
+    }
+
+    _mm_empty ();
+}
+
+/* ----------------------------------------------------------------
+ * composite_over_n_8_8888
+ */
+
+pixman_bool_t
+pixman_fill_sse2 (uint32_t *bits,
+                  int       stride,
+                  int       bpp,
+                  int       x,
+                  int       y,
+                  int       width,
+                  int       height,
+                  uint32_t  data)
+{
+    uint32_t byte_width;
+    uint8_t         *byte_line;
+
+    __m128i xmm_def;
+
+    if (bpp != 16 && bpp != 32)
+	return FALSE;
+
+    if (bpp == 16)
+    {
+	stride = stride * (int) sizeof (uint32_t) / 2;
+	byte_line = (uint8_t *)(((uint16_t *)bits) + stride * y + x);
+	byte_width = 2 * width;
+	stride *= 2;
+        data = (data & 0xffff) * 0x00010001;
+    }
+    else
+    {
+	stride = stride * (int) sizeof (uint32_t) / 4;
+	byte_line = (uint8_t *)(((uint32_t *)bits) + stride * y + x);
+	byte_width = 4 * width;
+	stride *= 4;
+    }
+
+    cache_prefetch ((__m128i*)byte_line);
+    xmm_def = create_mask_2x32_128 (data, data);
+
+    while (height--)
+    {
+	int w;
+	uint8_t *d = byte_line;
+	byte_line += stride;
+	w = byte_width;
+
+
+	cache_prefetch_next ((__m128i*)d);
+
+	while (w >= 2 && ((unsigned long)d & 3))
+	{
+	    *(uint16_t *)d = data;
+	    w -= 2;
+	    d += 2;
+	}
+
+	while (w >= 4 && ((unsigned long)d & 15))
+	{
+	    *(uint32_t *)d = data;
+
+	    w -= 4;
+	    d += 4;
+	}
+
+	cache_prefetch_next ((__m128i*)d);
+
+	while (w >= 128)
+	{
+	    cache_prefetch (((__m128i*)d) + 12);
+
+	    save_128_aligned ((__m128i*)(d),     xmm_def);
+	    save_128_aligned ((__m128i*)(d + 16),  xmm_def);
+	    save_128_aligned ((__m128i*)(d + 32),  xmm_def);
+	    save_128_aligned ((__m128i*)(d + 48),  xmm_def);
+	    save_128_aligned ((__m128i*)(d + 64),  xmm_def);
+	    save_128_aligned ((__m128i*)(d + 80),  xmm_def);
+	    save_128_aligned ((__m128i*)(d + 96),  xmm_def);
+	    save_128_aligned ((__m128i*)(d + 112), xmm_def);
+
+	    d += 128;
+	    w -= 128;
+	}
+
+	if (w >= 64)
+	{
+	    cache_prefetch (((__m128i*)d) + 8);
+
+	    save_128_aligned ((__m128i*)(d),     xmm_def);
+	    save_128_aligned ((__m128i*)(d + 16),  xmm_def);
+	    save_128_aligned ((__m128i*)(d + 32),  xmm_def);
+	    save_128_aligned ((__m128i*)(d + 48),  xmm_def);
+
+	    d += 64;
+	    w -= 64;
+	}
+
+	cache_prefetch_next ((__m128i*)d);
+
+	if (w >= 32)
+	{
+	    save_128_aligned ((__m128i*)(d),     xmm_def);
+	    save_128_aligned ((__m128i*)(d + 16),  xmm_def);
+
+	    d += 32;
+	    w -= 32;
+	}
+
+	if (w >= 16)
+	{
+	    save_128_aligned ((__m128i*)(d),     xmm_def);
+
+	    d += 16;
+	    w -= 16;
+	}
+
+	cache_prefetch_next ((__m128i*)d);
+
+	while (w >= 4)
+	{
+	    *(uint32_t *)d = data;
+
+	    w -= 4;
+	    d += 4;
+	}
+
+	if (w >= 2)
+	{
+	    *(uint16_t *)d = data;
+	    w -= 2;
+	    d += 2;
+	}
+    }
+
+    _mm_empty ();
+    return TRUE;
+}
+
+static void
+sse2_composite_src_n_8_8888 (pixman_implementation_t *imp,
+                             pixman_op_t              op,
+                             pixman_image_t *         src_image,
+                             pixman_image_t *         mask_image,
+                             pixman_image_t *         dst_image,
+                             int32_t                  src_x,
+                             int32_t                  src_y,
+                             int32_t                  mask_x,
+                             int32_t                  mask_y,
+                             int32_t                  dest_x,
+                             int32_t                  dest_y,
+                             int32_t                  width,
+                             int32_t                  height)
+{
+    uint32_t src, srca;
+    uint32_t    *dst_line, *dst;
+    uint8_t     *mask_line, *mask;
+    int dst_stride, mask_stride;
+    int32_t w;
+    uint32_t m;
+
+    __m128i xmm_src, xmm_def;
+    __m128i xmm_mask, xmm_mask_lo, xmm_mask_hi;
+
+    src = _pixman_image_get_solid (src_image, dst_image->bits.format);
+
+    srca = src >> 24;
+    if (src == 0)
+    {
+	pixman_fill_sse2 (dst_image->bits.bits, dst_image->bits.rowstride,
+	                  PIXMAN_FORMAT_BPP (dst_image->bits.format),
+	                  dest_x, dest_y, width, height, 0);
+	return;
+    }
+
+    PIXMAN_IMAGE_GET_LINE (
+	dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
+    PIXMAN_IMAGE_GET_LINE (
+	mask_image, mask_x, mask_y, uint8_t, mask_stride, mask_line, 1);
+
+    xmm_def = create_mask_2x32_128 (src, src);
+    xmm_src = expand_pixel_32_1x128 (src);
+
+    while (height--)
+    {
+	dst = dst_line;
+	dst_line += dst_stride;
+	mask = mask_line;
+	mask_line += mask_stride;
+	w = width;
+
+	/* call prefetch hint to optimize cache load*/
+	cache_prefetch ((__m128i*)mask);
+	cache_prefetch ((__m128i*)dst);
+
+	while (w && (unsigned long)dst & 15)
+	{
+	    uint8_t m = *mask++;
+
+	    if (m)
+	    {
+		*dst = pack_1x64_32 (
+		    pix_multiply_1x64 (
+			_mm_movepi64_pi64 (xmm_src), expand_pixel_8_1x64 (m)));
+	    }
+	    else
+	    {
+		*dst = 0;
+	    }
+
+	    w--;
+	    dst++;
+	}
+
+	/* call prefetch hint to optimize cache load*/
+	cache_prefetch ((__m128i*)mask);
+	cache_prefetch ((__m128i*)dst);
+
+	while (w >= 4)
+	{
+	    /* fill cache line with next memory */
+	    cache_prefetch_next ((__m128i*)mask);
+	    cache_prefetch_next ((__m128i*)dst);
+
+	    m = *((uint32_t*)mask);
+
+	    if (srca == 0xff && m == 0xffffffff)
+	    {
+		save_128_aligned ((__m128i*)dst, xmm_def);
+	    }
+	    else if (m)
+	    {
+		xmm_mask = unpack_32_1x128 (m);
+		xmm_mask = _mm_unpacklo_epi8 (xmm_mask, _mm_setzero_si128 ());
+
+		/* Unpacking */
+		unpack_128_2x128 (xmm_mask, &xmm_mask_lo, &xmm_mask_hi);
+
+		expand_alpha_rev_2x128 (xmm_mask_lo, xmm_mask_hi,
+					&xmm_mask_lo, &xmm_mask_hi);
+
+		pix_multiply_2x128 (&xmm_src, &xmm_src,
+				    &xmm_mask_lo, &xmm_mask_hi,
+				    &xmm_mask_lo, &xmm_mask_hi);
+
+		save_128_aligned (
+		    (__m128i*)dst, pack_2x128_128 (xmm_mask_lo, xmm_mask_hi));
+	    }
+	    else
+	    {
+		save_128_aligned ((__m128i*)dst, _mm_setzero_si128 ());
+	    }
+
+	    w -= 4;
+	    dst += 4;
+	    mask += 4;
+	}
+
+	while (w)
+	{
+	    uint8_t m = *mask++;
+
+	    if (m)
+	    {
+		*dst = pack_1x64_32 (
+		    pix_multiply_1x64 (
+			_mm_movepi64_pi64 (xmm_src), expand_pixel_8_1x64 (m)));
+	    }
+	    else
+	    {
+		*dst = 0;
+	    }
+
+	    w--;
+	    dst++;
+	}
+    }
+
+    _mm_empty ();
+}
+
+/*-----------------------------------------------------------------------
+ * composite_over_n_8_0565
+ */
+
+static void
+sse2_composite_over_n_8_0565 (pixman_implementation_t *imp,
+                              pixman_op_t              op,
+                              pixman_image_t *         src_image,
+                              pixman_image_t *         mask_image,
+                              pixman_image_t *         dst_image,
+                              int32_t                  src_x,
+                              int32_t                  src_y,
+                              int32_t                  mask_x,
+                              int32_t                  mask_y,
+                              int32_t                  dest_x,
+                              int32_t                  dest_y,
+                              int32_t                  width,
+                              int32_t                  height)
+{
+    uint32_t src, srca;
+    uint16_t    *dst_line, *dst, d;
+    uint8_t     *mask_line, *mask;
+    int dst_stride, mask_stride;
+    int32_t w;
+    uint32_t m;
+    __m64 mmx_src, mmx_alpha, mmx_mask, mmx_dest;
+
+    __m128i xmm_src, xmm_alpha;
+    __m128i xmm_mask, xmm_mask_lo, xmm_mask_hi;
+    __m128i xmm_dst, xmm_dst0, xmm_dst1, xmm_dst2, xmm_dst3;
+
+    src = _pixman_image_get_solid (src_image, dst_image->bits.format);
+
+    srca = src >> 24;
+    if (src == 0)
+	return;
+
+    PIXMAN_IMAGE_GET_LINE (
+	dst_image, dest_x, dest_y, uint16_t, dst_stride, dst_line, 1);
+    PIXMAN_IMAGE_GET_LINE (
+	mask_image, mask_x, mask_y, uint8_t, mask_stride, mask_line, 1);
+
+    xmm_src = expand_pixel_32_1x128 (src);
+    xmm_alpha = expand_alpha_1x128 (xmm_src);
+    mmx_src = _mm_movepi64_pi64 (xmm_src);
+    mmx_alpha = _mm_movepi64_pi64 (xmm_alpha);
+
+    while (height--)
+    {
+	dst = dst_line;
+	dst_line += dst_stride;
+	mask = mask_line;
+	mask_line += mask_stride;
+	w = width;
+
+	/* call prefetch hint to optimize cache load*/
+	cache_prefetch ((__m128i*)mask);
+	cache_prefetch ((__m128i*)dst);
+
+	while (w && (unsigned long)dst & 15)
+	{
+	    m = *mask++;
+
+	    if (m)
+	    {
+		d = *dst;
+		mmx_mask = expand_alpha_rev_1x64 (unpack_32_1x64 (m));
+		mmx_dest = expand565_16_1x64 (d);
+
+		*dst = pack_565_32_16 (
+		    pack_1x64_32 (
+			in_over_1x64 (
+			    &mmx_src, &mmx_alpha, &mmx_mask, &mmx_dest)));
+	    }
+
+	    w--;
+	    dst++;
+	}
+
+	/* call prefetch hint to optimize cache load*/
+	cache_prefetch ((__m128i*)mask);
+	cache_prefetch ((__m128i*)dst);
+
+	while (w >= 8)
+	{
+	    /* fill cache line with next memory */
+	    cache_prefetch_next ((__m128i*)mask);
+	    cache_prefetch_next ((__m128i*)dst);
+
+	    xmm_dst = load_128_aligned ((__m128i*) dst);
+	    unpack_565_128_4x128 (xmm_dst,
+				  &xmm_dst0, &xmm_dst1, &xmm_dst2, &xmm_dst3);
+
+	    m = *((uint32_t*)mask);
+	    mask += 4;
+
+	    if (m)
+	    {
+		xmm_mask = unpack_32_1x128 (m);
+		xmm_mask = _mm_unpacklo_epi8 (xmm_mask, _mm_setzero_si128 ());
+
+		/* Unpacking */
+		unpack_128_2x128 (xmm_mask, &xmm_mask_lo, &xmm_mask_hi);
+
+		expand_alpha_rev_2x128 (xmm_mask_lo, xmm_mask_hi,
+					&xmm_mask_lo, &xmm_mask_hi);
+
+		in_over_2x128 (&xmm_src, &xmm_src,
+			       &xmm_alpha, &xmm_alpha,
+			       &xmm_mask_lo, &xmm_mask_hi,
+			       &xmm_dst0, &xmm_dst1);
+	    }
+
+	    m = *((uint32_t*)mask);
+	    mask += 4;
+
+	    if (m)
+	    {
+		xmm_mask = unpack_32_1x128 (m);
+		xmm_mask = _mm_unpacklo_epi8 (xmm_mask, _mm_setzero_si128 ());
+
+		/* Unpacking */
+		unpack_128_2x128 (xmm_mask, &xmm_mask_lo, &xmm_mask_hi);
+
+		expand_alpha_rev_2x128 (xmm_mask_lo, xmm_mask_hi,
+					&xmm_mask_lo, &xmm_mask_hi);
+		in_over_2x128 (&xmm_src, &xmm_src,
+			       &xmm_alpha, &xmm_alpha,
+			       &xmm_mask_lo, &xmm_mask_hi,
+			       &xmm_dst2, &xmm_dst3);
+	    }
+
+	    save_128_aligned (
+		(__m128i*)dst, pack_565_4x128_128 (
+		    &xmm_dst0, &xmm_dst1, &xmm_dst2, &xmm_dst3));
+
+	    w -= 8;
+	    dst += 8;
+	}
+
+	while (w)
+	{
+	    m = *mask++;
+
+	    if (m)
+	    {
+		d = *dst;
+		mmx_mask = expand_alpha_rev_1x64 (unpack_32_1x64 (m));
+		mmx_dest = expand565_16_1x64 (d);
+
+		*dst = pack_565_32_16 (
+		    pack_1x64_32 (
+			in_over_1x64 (
+			    &mmx_src, &mmx_alpha, &mmx_mask, &mmx_dest)));
+	    }
+
+	    w--;
+	    dst++;
+	}
+    }
+
+    _mm_empty ();
+}
+
+/* -----------------------------------------------------------------------
+ * composite_over_pixbuf_0565
+ */
+
+static void
+sse2_composite_over_pixbuf_0565 (pixman_implementation_t *imp,
+                                 pixman_op_t              op,
+                                 pixman_image_t *         src_image,
+                                 pixman_image_t *         mask_image,
+                                 pixman_image_t *         dst_image,
+                                 int32_t                  src_x,
+                                 int32_t                  src_y,
+                                 int32_t                  mask_x,
+                                 int32_t                  mask_y,
+                                 int32_t                  dest_x,
+                                 int32_t                  dest_y,
+                                 int32_t                  width,
+                                 int32_t                  height)
+{
+    uint16_t    *dst_line, *dst, d;
+    uint32_t    *src_line, *src, s;
+    int dst_stride, src_stride;
+    int32_t w;
+    uint32_t opaque, zero;
+
+    __m64 ms;
+    __m128i xmm_src, xmm_src_lo, xmm_src_hi;
+    __m128i xmm_dst, xmm_dst0, xmm_dst1, xmm_dst2, xmm_dst3;
+
+    PIXMAN_IMAGE_GET_LINE (
+	dst_image, dest_x, dest_y, uint16_t, dst_stride, dst_line, 1);
+    PIXMAN_IMAGE_GET_LINE (
+	src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
+
+#if 0
+    /* FIXME
+     *
+     * I copy the code from MMX one and keep the fixme.
+     * If it's a problem there, probably is a problem here.
+     */
+    assert (src_image->drawable == mask_image->drawable);
+#endif
+
+    while (height--)
+    {
+	dst = dst_line;
+	dst_line += dst_stride;
+	src = src_line;
+	src_line += src_stride;
+	w = width;
+
+	/* call prefetch hint to optimize cache load*/
+	cache_prefetch ((__m128i*)src);
+	cache_prefetch ((__m128i*)dst);
+
+	while (w && (unsigned long)dst & 15)
+	{
+	    s = *src++;
+	    d = *dst;
+
+	    ms = unpack_32_1x64 (s);
+
+	    *dst++ = pack_565_32_16 (
+		pack_1x64_32 (
+		    over_rev_non_pre_1x64 (ms, expand565_16_1x64 (d))));
+	    w--;
+	}
+
+	/* call prefetch hint to optimize cache load*/
+	cache_prefetch ((__m128i*)src);
+	cache_prefetch ((__m128i*)dst);
+
+	while (w >= 8)
+	{
+	    /* fill cache line with next memory */
+	    cache_prefetch_next ((__m128i*)src);
+	    cache_prefetch_next ((__m128i*)dst);
+
+	    /* First round */
+	    xmm_src = load_128_unaligned ((__m128i*)src);
+	    xmm_dst = load_128_aligned  ((__m128i*)dst);
+
+	    opaque = is_opaque (xmm_src);
+	    zero = is_zero (xmm_src);
+
+	    unpack_565_128_4x128 (xmm_dst,
+				  &xmm_dst0, &xmm_dst1, &xmm_dst2, &xmm_dst3);
+	    unpack_128_2x128 (xmm_src, &xmm_src_lo, &xmm_src_hi);
+
+	    /* preload next round*/
+	    xmm_src = load_128_unaligned ((__m128i*)(src + 4));
+
+	    if (opaque)
+	    {
+		invert_colors_2x128 (xmm_src_lo, xmm_src_hi,
+				     &xmm_dst0, &xmm_dst1);
+	    }
+	    else if (!zero)
+	    {
+		over_rev_non_pre_2x128 (xmm_src_lo, xmm_src_hi,
+					&xmm_dst0, &xmm_dst1);
+	    }
+
+	    /* Second round */
+	    opaque = is_opaque (xmm_src);
+	    zero = is_zero (xmm_src);
+
+	    unpack_128_2x128 (xmm_src, &xmm_src_lo, &xmm_src_hi);
+
+	    if (opaque)
+	    {
+		invert_colors_2x128 (xmm_src_lo, xmm_src_hi,
+				     &xmm_dst2, &xmm_dst3);
+	    }
+	    else if (!zero)
+	    {
+		over_rev_non_pre_2x128 (xmm_src_lo, xmm_src_hi,
+					&xmm_dst2, &xmm_dst3);
+	    }
+
+	    save_128_aligned (
+		(__m128i*)dst, pack_565_4x128_128 (
+		    &xmm_dst0, &xmm_dst1, &xmm_dst2, &xmm_dst3));
+
+	    w -= 8;
+	    src += 8;
+	    dst += 8;
+	}
+
+	while (w)
+	{
+	    s = *src++;
+	    d = *dst;
+
+	    ms = unpack_32_1x64 (s);
+
+	    *dst++ = pack_565_32_16 (
+		pack_1x64_32 (
+		    over_rev_non_pre_1x64 (ms, expand565_16_1x64 (d))));
+	    w--;
+	}
+    }
+
+    _mm_empty ();
+}
+
+/* -------------------------------------------------------------------------
+ * composite_over_pixbuf_8888
+ */
+
+static void
+sse2_composite_over_pixbuf_8888 (pixman_implementation_t *imp,
+                                 pixman_op_t              op,
+                                 pixman_image_t *         src_image,
+                                 pixman_image_t *         mask_image,
+                                 pixman_image_t *         dst_image,
+                                 int32_t                  src_x,
+                                 int32_t                  src_y,
+                                 int32_t                  mask_x,
+                                 int32_t                  mask_y,
+                                 int32_t                  dest_x,
+                                 int32_t                  dest_y,
+                                 int32_t                  width,
+                                 int32_t                  height)
+{
+    uint32_t    *dst_line, *dst, d;
+    uint32_t    *src_line, *src, s;
+    int dst_stride, src_stride;
+    int32_t w;
+    uint32_t opaque, zero;
+
+    __m128i xmm_src_lo, xmm_src_hi;
+    __m128i xmm_dst_lo, xmm_dst_hi;
+
+    PIXMAN_IMAGE_GET_LINE (
+	dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
+    PIXMAN_IMAGE_GET_LINE (
+	src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
+
+#if 0
+    /* FIXME
+     *
+     * I copy the code from MMX one and keep the fixme.
+     * If it's a problem there, probably is a problem here.
+     */
+    assert (src_image->drawable == mask_image->drawable);
+#endif
+
+    while (height--)
+    {
+	dst = dst_line;
+	dst_line += dst_stride;
+	src = src_line;
+	src_line += src_stride;
+	w = width;
+
+	/* call prefetch hint to optimize cache load*/
+	cache_prefetch ((__m128i*)src);
+	cache_prefetch ((__m128i*)dst);
+
+	while (w && (unsigned long)dst & 15)
+	{
+	    s = *src++;
+	    d = *dst;
+
+	    *dst++ = pack_1x64_32 (
+		over_rev_non_pre_1x64 (
+		    unpack_32_1x64 (s), unpack_32_1x64 (d)));
+
+	    w--;
+	}
+
+	/* call prefetch hint to optimize cache load*/
+	cache_prefetch ((__m128i*)src);
+	cache_prefetch ((__m128i*)dst);
+
+	while (w >= 4)
+	{
+	    /* fill cache line with next memory */
+	    cache_prefetch_next ((__m128i*)src);
+	    cache_prefetch_next ((__m128i*)dst);
+
+	    xmm_src_hi = load_128_unaligned ((__m128i*)src);
+
+	    opaque = is_opaque (xmm_src_hi);
+	    zero = is_zero (xmm_src_hi);
+
+	    unpack_128_2x128 (xmm_src_hi, &xmm_src_lo, &xmm_src_hi);
+
+	    if (opaque)
+	    {
+		invert_colors_2x128 (xmm_src_lo, xmm_src_hi,
+				     &xmm_dst_lo, &xmm_dst_hi);
+
+		save_128_aligned (
+		    (__m128i*)dst, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
+	    }
+	    else if (!zero)
+	    {
+		xmm_dst_hi = load_128_aligned  ((__m128i*)dst);
+
+		unpack_128_2x128 (xmm_dst_hi, &xmm_dst_lo, &xmm_dst_hi);
+
+		over_rev_non_pre_2x128 (xmm_src_lo, xmm_src_hi,
+					&xmm_dst_lo, &xmm_dst_hi);
+
+		save_128_aligned (
+		    (__m128i*)dst, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
+	    }
+
+	    w -= 4;
+	    dst += 4;
+	    src += 4;
+	}
+
+	while (w)
+	{
+	    s = *src++;
+	    d = *dst;
+
+	    *dst++ = pack_1x64_32 (
+		over_rev_non_pre_1x64 (
+		    unpack_32_1x64 (s), unpack_32_1x64 (d)));
+
+	    w--;
+	}
+    }
+
+    _mm_empty ();
+}
+
+/* -------------------------------------------------------------------------------------------------
+ * composite_over_n_8888_0565_ca
+ */
+
+static void
+sse2_composite_over_n_8888_0565_ca (pixman_implementation_t *imp,
+                                    pixman_op_t              op,
+                                    pixman_image_t *         src_image,
+                                    pixman_image_t *         mask_image,
+                                    pixman_image_t *         dst_image,
+                                    int32_t                  src_x,
+                                    int32_t                  src_y,
+                                    int32_t                  mask_x,
+                                    int32_t                  mask_y,
+                                    int32_t                  dest_x,
+                                    int32_t                  dest_y,
+                                    int32_t                  width,
+                                    int32_t                  height)
+{
+    uint32_t src;
+    uint16_t    *dst_line, *dst, d;
+    uint32_t    *mask_line, *mask, m;
+    int dst_stride, mask_stride;
+    int w;
+    uint32_t pack_cmp;
+
+    __m128i xmm_src, xmm_alpha;
+    __m128i xmm_mask, xmm_mask_lo, xmm_mask_hi;
+    __m128i xmm_dst, xmm_dst0, xmm_dst1, xmm_dst2, xmm_dst3;
+
+    __m64 mmx_src, mmx_alpha, mmx_mask, mmx_dest;
+
+    src = _pixman_image_get_solid (src_image, dst_image->bits.format);
+
+    if (src == 0)
+	return;
+
+    PIXMAN_IMAGE_GET_LINE (
+	dst_image, dest_x, dest_y, uint16_t, dst_stride, dst_line, 1);
+    PIXMAN_IMAGE_GET_LINE (
+	mask_image, mask_x, mask_y, uint32_t, mask_stride, mask_line, 1);
+
+    xmm_src = expand_pixel_32_1x128 (src);
+    xmm_alpha = expand_alpha_1x128 (xmm_src);
+    mmx_src = _mm_movepi64_pi64 (xmm_src);
+    mmx_alpha = _mm_movepi64_pi64 (xmm_alpha);
+
+    while (height--)
+    {
+	w = width;
+	mask = mask_line;
+	dst = dst_line;
+	mask_line += mask_stride;
+	dst_line += dst_stride;
+
+	/* call prefetch hint to optimize cache load*/
+	cache_prefetch ((__m128i*)mask);
+	cache_prefetch ((__m128i*)dst);
+
+	while (w && ((unsigned long)dst & 15))
+	{
+	    m = *(uint32_t *) mask;
+
+	    if (m)
+	    {
+		d = *dst;
+		mmx_mask = unpack_32_1x64 (m);
+		mmx_dest = expand565_16_1x64 (d);
+
+		*dst = pack_565_32_16 (
+		    pack_1x64_32 (
+			in_over_1x64 (
+			    &mmx_src, &mmx_alpha, &mmx_mask, &mmx_dest)));
+	    }
+
+	    w--;
+	    dst++;
+	    mask++;
+	}
+
+	/* call prefetch hint to optimize cache load*/
+	cache_prefetch ((__m128i*)mask);
+	cache_prefetch ((__m128i*)dst);
+
+	while (w >= 8)
+	{
+	    /* fill cache line with next memory */
+	    cache_prefetch_next ((__m128i*)mask);
+	    cache_prefetch_next ((__m128i*)dst);
+
+	    /* First round */
+	    xmm_mask = load_128_unaligned ((__m128i*)mask);
+	    xmm_dst = load_128_aligned ((__m128i*)dst);
+
+	    pack_cmp = _mm_movemask_epi8 (
+		_mm_cmpeq_epi32 (xmm_mask, _mm_setzero_si128 ()));
+
+	    unpack_565_128_4x128 (xmm_dst,
+				  &xmm_dst0, &xmm_dst1, &xmm_dst2, &xmm_dst3);
+	    unpack_128_2x128 (xmm_mask, &xmm_mask_lo, &xmm_mask_hi);
+
+	    /* preload next round */
+	    xmm_mask = load_128_unaligned ((__m128i*)(mask + 4));
+
+	    /* preload next round */
+	    if (pack_cmp != 0xffff)
+	    {
+		in_over_2x128 (&xmm_src, &xmm_src,
+			       &xmm_alpha, &xmm_alpha,
+			       &xmm_mask_lo, &xmm_mask_hi,
+			       &xmm_dst0, &xmm_dst1);
+	    }
+
+	    /* Second round */
+	    pack_cmp = _mm_movemask_epi8 (
+		_mm_cmpeq_epi32 (xmm_mask, _mm_setzero_si128 ()));
+
+	    unpack_128_2x128 (xmm_mask, &xmm_mask_lo, &xmm_mask_hi);
+
+	    if (pack_cmp != 0xffff)
+	    {
+		in_over_2x128 (&xmm_src, &xmm_src,
+			       &xmm_alpha, &xmm_alpha,
+			       &xmm_mask_lo, &xmm_mask_hi,
+			       &xmm_dst2, &xmm_dst3);
+	    }
+
+	    save_128_aligned (
+		(__m128i*)dst, pack_565_4x128_128 (
+		    &xmm_dst0, &xmm_dst1, &xmm_dst2, &xmm_dst3));
+
+	    w -= 8;
+	    dst += 8;
+	    mask += 8;
+	}
+
+	while (w)
+	{
+	    m = *(uint32_t *) mask;
+
+	    if (m)
+	    {
+		d = *dst;
+		mmx_mask = unpack_32_1x64 (m);
+		mmx_dest = expand565_16_1x64 (d);
+
+		*dst = pack_565_32_16 (
+		    pack_1x64_32 (
+			in_over_1x64 (
+			    &mmx_src, &mmx_alpha, &mmx_mask, &mmx_dest)));
+	    }
+
+	    w--;
+	    dst++;
+	    mask++;
+	}
+    }
+
+    _mm_empty ();
+}
+
+/* -----------------------------------------------------------------------
+ * composite_in_n_8_8
+ */
+
+static void
+sse2_composite_in_n_8_8 (pixman_implementation_t *imp,
+                         pixman_op_t              op,
+                         pixman_image_t *         src_image,
+                         pixman_image_t *         mask_image,
+                         pixman_image_t *         dst_image,
+                         int32_t                  src_x,
+                         int32_t                  src_y,
+                         int32_t                  mask_x,
+                         int32_t                  mask_y,
+                         int32_t                  dest_x,
+                         int32_t                  dest_y,
+                         int32_t                  width,
+                         int32_t                  height)
+{
+    uint8_t     *dst_line, *dst;
+    uint8_t     *mask_line, *mask;
+    int dst_stride, mask_stride;
+    uint32_t d, m;
+    uint32_t src;
+    uint8_t sa;
+    int32_t w;
+
+    __m128i xmm_alpha;
+    __m128i xmm_mask, xmm_mask_lo, xmm_mask_hi;
+    __m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
+
+    PIXMAN_IMAGE_GET_LINE (
+	dst_image, dest_x, dest_y, uint8_t, dst_stride, dst_line, 1);
+    PIXMAN_IMAGE_GET_LINE (
+	mask_image, mask_x, mask_y, uint8_t, mask_stride, mask_line, 1);
+
+    src = _pixman_image_get_solid (src_image, dst_image->bits.format);
+
+    sa = src >> 24;
+
+    xmm_alpha = expand_alpha_1x128 (expand_pixel_32_1x128 (src));
+
+    while (height--)
+    {
+	dst = dst_line;
+	dst_line += dst_stride;
+	mask = mask_line;
+	mask_line += mask_stride;
+	w = width;
+
+	/* call prefetch hint to optimize cache load*/
+	cache_prefetch ((__m128i*)mask);
+	cache_prefetch ((__m128i*)dst);
+
+	while (w && ((unsigned long)dst & 15))
+	{
+	    m = (uint32_t) *mask++;
+	    d = (uint32_t) *dst;
+
+	    *dst++ = (uint8_t) pack_1x64_32 (
+		pix_multiply_1x64 (
+		    pix_multiply_1x64 (_mm_movepi64_pi64 (xmm_alpha),
+				       unpack_32_1x64 (m)),
+		    unpack_32_1x64 (d)));
+	    w--;
+	}
+
+	/* call prefetch hint to optimize cache load*/
+	cache_prefetch ((__m128i*)mask);
+	cache_prefetch ((__m128i*)dst);
+
+	while (w >= 16)
+	{
+	    /* fill cache line with next memory */
+	    cache_prefetch_next ((__m128i*)mask);
+	    cache_prefetch_next ((__m128i*)dst);
+
+	    xmm_mask = load_128_unaligned ((__m128i*)mask);
+	    xmm_dst = load_128_aligned ((__m128i*)dst);
+
+	    unpack_128_2x128 (xmm_mask, &xmm_mask_lo, &xmm_mask_hi);
+	    unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
+
+	    pix_multiply_2x128 (&xmm_alpha, &xmm_alpha,
+				&xmm_mask_lo, &xmm_mask_hi,
+				&xmm_mask_lo, &xmm_mask_hi);
+
+	    pix_multiply_2x128 (&xmm_mask_lo, &xmm_mask_hi,
+				&xmm_dst_lo, &xmm_dst_hi,
+				&xmm_dst_lo, &xmm_dst_hi);
+
+	    save_128_aligned (
+		(__m128i*)dst, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
+
+	    mask += 16;
+	    dst += 16;
+	    w -= 16;
+	}
+
+	while (w)
+	{
+	    m = (uint32_t) *mask++;
+	    d = (uint32_t) *dst;
+
+	    *dst++ = (uint8_t) pack_1x64_32 (
+		pix_multiply_1x64 (
+		    pix_multiply_1x64 (
+			_mm_movepi64_pi64 (xmm_alpha), unpack_32_1x64 (m)),
+		    unpack_32_1x64 (d)));
+	    w--;
+	}
+    }
+
+    _mm_empty ();
+}
+
+/* ---------------------------------------------------------------------------
+ * composite_in_8_8
+ */
+
+static void
+sse2_composite_in_8_8 (pixman_implementation_t *imp,
+                       pixman_op_t              op,
+                       pixman_image_t *         src_image,
+                       pixman_image_t *         mask_image,
+                       pixman_image_t *         dst_image,
+                       int32_t                  src_x,
+                       int32_t                  src_y,
+                       int32_t                  mask_x,
+                       int32_t                  mask_y,
+                       int32_t                  dest_x,
+                       int32_t                  dest_y,
+                       int32_t                  width,
+                       int32_t                  height)
+{
+    uint8_t     *dst_line, *dst;
+    uint8_t     *src_line, *src;
+    int src_stride, dst_stride;
+    int32_t w;
+    uint32_t s, d;
+
+    __m128i xmm_src, xmm_src_lo, xmm_src_hi;
+    __m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
+
+    PIXMAN_IMAGE_GET_LINE (
+	dst_image, dest_x, dest_y, uint8_t, dst_stride, dst_line, 1);
+    PIXMAN_IMAGE_GET_LINE (
+	src_image, src_x, src_y, uint8_t, src_stride, src_line, 1);
+
+    while (height--)
+    {
+	dst = dst_line;
+	dst_line += dst_stride;
+	src = src_line;
+	src_line += src_stride;
+	w = width;
+
+	/* call prefetch hint to optimize cache load*/
+	cache_prefetch ((__m128i*)src);
+	cache_prefetch ((__m128i*)dst);
+
+	while (w && ((unsigned long)dst & 15))
+	{
+	    s = (uint32_t) *src++;
+	    d = (uint32_t) *dst;
+
+	    *dst++ = (uint8_t) pack_1x64_32 (
+		pix_multiply_1x64 (
+		    unpack_32_1x64 (s), unpack_32_1x64 (d)));
+	    w--;
+	}
+
+	/* call prefetch hint to optimize cache load*/
+	cache_prefetch ((__m128i*)src);
+	cache_prefetch ((__m128i*)dst);
+
+	while (w >= 16)
+	{
+	    /* fill cache line with next memory */
+	    cache_prefetch_next ((__m128i*)src);
+	    cache_prefetch_next ((__m128i*)dst);
+
+	    xmm_src = load_128_unaligned ((__m128i*)src);
+	    xmm_dst = load_128_aligned ((__m128i*)dst);
+
+	    unpack_128_2x128 (xmm_src, &xmm_src_lo, &xmm_src_hi);
+	    unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
+
+	    pix_multiply_2x128 (&xmm_src_lo, &xmm_src_hi,
+				&xmm_dst_lo, &xmm_dst_hi,
+				&xmm_dst_lo, &xmm_dst_hi);
+
+	    save_128_aligned (
+		(__m128i*)dst, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
+
+	    src += 16;
+	    dst += 16;
+	    w -= 16;
+	}
+
+	while (w)
+	{
+	    s = (uint32_t) *src++;
+	    d = (uint32_t) *dst;
+
+	    *dst++ = (uint8_t) pack_1x64_32 (
+		pix_multiply_1x64 (unpack_32_1x64 (s), unpack_32_1x64 (d)));
+	    w--;
+	}
+    }
+
+    _mm_empty ();
+}
+
+/* -------------------------------------------------------------------------
+ * composite_add_n_8_8
+ */
+
+static void
+sse2_composite_add_n_8_8 (pixman_implementation_t *imp,
+			  pixman_op_t              op,
+			  pixman_image_t *         src_image,
+			  pixman_image_t *         mask_image,
+			  pixman_image_t *         dst_image,
+			  int32_t                  src_x,
+			  int32_t                  src_y,
+			  int32_t                  mask_x,
+			  int32_t                  mask_y,
+			  int32_t                  dest_x,
+			  int32_t                  dest_y,
+			  int32_t                  width,
+			  int32_t                  height)
+{
+    uint8_t     *dst_line, *dst;
+    uint8_t     *mask_line, *mask;
+    int dst_stride, mask_stride;
+    int32_t w;
+    uint32_t src;
+    uint8_t sa;
+    uint32_t m, d;
+
+    __m128i xmm_alpha;
+    __m128i xmm_mask, xmm_mask_lo, xmm_mask_hi;
+    __m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
+
+    PIXMAN_IMAGE_GET_LINE (
+	dst_image, dest_x, dest_y, uint8_t, dst_stride, dst_line, 1);
+    PIXMAN_IMAGE_GET_LINE (
+	mask_image, mask_x, mask_y, uint8_t, mask_stride, mask_line, 1);
+
+    src = _pixman_image_get_solid (src_image, dst_image->bits.format);
+
+    sa = src >> 24;
+
+    xmm_alpha = expand_alpha_1x128 (expand_pixel_32_1x128 (src));
+
+    while (height--)
+    {
+	dst = dst_line;
+	dst_line += dst_stride;
+	mask = mask_line;
+	mask_line += mask_stride;
+	w = width;
+
+	/* call prefetch hint to optimize cache load*/
+	cache_prefetch ((__m128i*)mask);
+	cache_prefetch ((__m128i*)dst);
+
+	while (w && ((unsigned long)dst & 15))
+	{
+	    m = (uint32_t) *mask++;
+	    d = (uint32_t) *dst;
+
+	    *dst++ = (uint8_t) pack_1x64_32 (
+		_mm_adds_pu16 (
+		    pix_multiply_1x64 (
+			_mm_movepi64_pi64 (xmm_alpha), unpack_32_1x64 (m)),
+		    unpack_32_1x64 (d)));
+	    w--;
+	}
+
+	/* call prefetch hint to optimize cache load*/
+	cache_prefetch ((__m128i*)mask);
+	cache_prefetch ((__m128i*)dst);
+
+	while (w >= 16)
+	{
+	    /* fill cache line with next memory */
+	    cache_prefetch_next ((__m128i*)mask);
+	    cache_prefetch_next ((__m128i*)dst);
+
+	    xmm_mask = load_128_unaligned ((__m128i*)mask);
+	    xmm_dst = load_128_aligned ((__m128i*)dst);
+
+	    unpack_128_2x128 (xmm_mask, &xmm_mask_lo, &xmm_mask_hi);
+	    unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
+
+	    pix_multiply_2x128 (&xmm_alpha, &xmm_alpha,
+				&xmm_mask_lo, &xmm_mask_hi,
+				&xmm_mask_lo, &xmm_mask_hi);
+
+	    xmm_dst_lo = _mm_adds_epu16 (xmm_mask_lo, xmm_dst_lo);
+	    xmm_dst_hi = _mm_adds_epu16 (xmm_mask_hi, xmm_dst_hi);
+
+	    save_128_aligned (
+		(__m128i*)dst, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
+
+	    mask += 16;
+	    dst += 16;
+	    w -= 16;
+	}
+
+	while (w)
+	{
+	    m = (uint32_t) *mask++;
+	    d = (uint32_t) *dst;
+
+	    *dst++ = (uint8_t) pack_1x64_32 (
+		_mm_adds_pu16 (
+		    pix_multiply_1x64 (
+			_mm_movepi64_pi64 (xmm_alpha), unpack_32_1x64 (m)),
+		    unpack_32_1x64 (d)));
+
+	    w--;
+	}
+    }
+
+    _mm_empty ();
+}
+
+/* ----------------------------------------------------------------------
+ * composite_add_8000_8000
+ */
+
+static void
+sse2_composite_add_8000_8000 (pixman_implementation_t *imp,
+                              pixman_op_t              op,
+                              pixman_image_t *         src_image,
+                              pixman_image_t *         mask_image,
+                              pixman_image_t *         dst_image,
+                              int32_t                  src_x,
+                              int32_t                  src_y,
+                              int32_t                  mask_x,
+                              int32_t                  mask_y,
+                              int32_t                  dest_x,
+                              int32_t                  dest_y,
+                              int32_t                  width,
+                              int32_t                  height)
+{
+    uint8_t     *dst_line, *dst;
+    uint8_t     *src_line, *src;
+    int dst_stride, src_stride;
+    int32_t w;
+    uint16_t t;
+
+    PIXMAN_IMAGE_GET_LINE (
+	src_image, src_x, src_y, uint8_t, src_stride, src_line, 1);
+    PIXMAN_IMAGE_GET_LINE (
+	dst_image, dest_x, dest_y, uint8_t, dst_stride, dst_line, 1);
+
+    while (height--)
+    {
+	dst = dst_line;
+	src = src_line;
+
+	/* call prefetch hint to optimize cache load*/
+	cache_prefetch ((__m128i*)src);
+	cache_prefetch ((__m128i*)dst);
+
+	dst_line += dst_stride;
+	src_line += src_stride;
+	w = width;
+
+	/* Small head */
+	while (w && (unsigned long)dst & 3)
+	{
+	    t = (*dst) + (*src++);
+	    *dst++ = t | (0 - (t >> 8));
+	    w--;
+	}
+
+	core_combine_add_u_sse2 ((uint32_t*)dst, (uint32_t*)src, NULL, w >> 2);
+
+	/* Small tail */
+	dst += w & 0xfffc;
+	src += w & 0xfffc;
+
+	w &= 3;
+
+	while (w)
+	{
+	    t = (*dst) + (*src++);
+	    *dst++ = t | (0 - (t >> 8));
+	    w--;
+	}
+    }
+
+    _mm_empty ();
+}
+
+/* ---------------------------------------------------------------------
+ * composite_add_8888_8888
+ */
+static void
+sse2_composite_add_8888_8888 (pixman_implementation_t *imp,
+                              pixman_op_t              op,
+                              pixman_image_t *         src_image,
+                              pixman_image_t *         mask_image,
+                              pixman_image_t *         dst_image,
+                              int32_t                  src_x,
+                              int32_t                  src_y,
+                              int32_t                  mask_x,
+                              int32_t                  mask_y,
+                              int32_t                  dest_x,
+                              int32_t                  dest_y,
+                              int32_t                  width,
+                              int32_t                  height)
+{
+    uint32_t    *dst_line, *dst;
+    uint32_t    *src_line, *src;
+    int dst_stride, src_stride;
+
+    PIXMAN_IMAGE_GET_LINE (
+	src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
+    PIXMAN_IMAGE_GET_LINE (
+	dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
+
+    while (height--)
+    {
+	dst = dst_line;
+	dst_line += dst_stride;
+	src = src_line;
+	src_line += src_stride;
+
+	core_combine_add_u_sse2 (dst, src, NULL, width);
+    }
+
+    _mm_empty ();
+}
+
+/* -------------------------------------------------------------------------------------------------
+ * sse2_composite_copy_area
+ */
+
+static pixman_bool_t
+pixman_blt_sse2 (uint32_t *src_bits,
+                 uint32_t *dst_bits,
+                 int       src_stride,
+                 int       dst_stride,
+                 int       src_bpp,
+                 int       dst_bpp,
+                 int       src_x,
+                 int       src_y,
+                 int       dst_x,
+                 int       dst_y,
+                 int       width,
+                 int       height)
+{
+    uint8_t *   src_bytes;
+    uint8_t *   dst_bytes;
+    int byte_width;
+
+    if (src_bpp != dst_bpp)
+	return FALSE;
+
+    if (src_bpp == 16)
+    {
+	src_stride = src_stride * (int) sizeof (uint32_t) / 2;
+	dst_stride = dst_stride * (int) sizeof (uint32_t) / 2;
+	src_bytes =(uint8_t *)(((uint16_t *)src_bits) + src_stride * (src_y) + (src_x));
+	dst_bytes = (uint8_t *)(((uint16_t *)dst_bits) + dst_stride * (dst_y) + (dst_x));
+	byte_width = 2 * width;
+	src_stride *= 2;
+	dst_stride *= 2;
+    }
+    else if (src_bpp == 32)
+    {
+	src_stride = src_stride * (int) sizeof (uint32_t) / 4;
+	dst_stride = dst_stride * (int) sizeof (uint32_t) / 4;
+	src_bytes = (uint8_t *)(((uint32_t *)src_bits) + src_stride * (src_y) + (src_x));
+	dst_bytes = (uint8_t *)(((uint32_t *)dst_bits) + dst_stride * (dst_y) + (dst_x));
+	byte_width = 4 * width;
+	src_stride *= 4;
+	dst_stride *= 4;
+    }
+    else
+    {
+	return FALSE;
+    }
+
+    cache_prefetch ((__m128i*)src_bytes);
+    cache_prefetch ((__m128i*)dst_bytes);
+
+    while (height--)
+    {
+	int w;
+	uint8_t *s = src_bytes;
+	uint8_t *d = dst_bytes;
+	src_bytes += src_stride;
+	dst_bytes += dst_stride;
+	w = byte_width;
+
+	cache_prefetch_next ((__m128i*)s);
+	cache_prefetch_next ((__m128i*)d);
+
+	while (w >= 2 && ((unsigned long)d & 3))
+	{
+	    *(uint16_t *)d = *(uint16_t *)s;
+	    w -= 2;
+	    s += 2;
+	    d += 2;
+	}
+
+	while (w >= 4 && ((unsigned long)d & 15))
+	{
+	    *(uint32_t *)d = *(uint32_t *)s;
+
+	    w -= 4;
+	    s += 4;
+	    d += 4;
+	}
+
+	cache_prefetch_next ((__m128i*)s);
+	cache_prefetch_next ((__m128i*)d);
+
+	while (w >= 64)
+	{
+	    __m128i xmm0, xmm1, xmm2, xmm3;
+
+	    /* 128 bytes ahead */
+	    cache_prefetch (((__m128i*)s) + 8);
+	    cache_prefetch (((__m128i*)d) + 8);
+
+	    xmm0 = load_128_unaligned ((__m128i*)(s));
+	    xmm1 = load_128_unaligned ((__m128i*)(s + 16));
+	    xmm2 = load_128_unaligned ((__m128i*)(s + 32));
+	    xmm3 = load_128_unaligned ((__m128i*)(s + 48));
+
+	    save_128_aligned ((__m128i*)(d),    xmm0);
+	    save_128_aligned ((__m128i*)(d + 16), xmm1);
+	    save_128_aligned ((__m128i*)(d + 32), xmm2);
+	    save_128_aligned ((__m128i*)(d + 48), xmm3);
+
+	    s += 64;
+	    d += 64;
+	    w -= 64;
+	}
+
+	cache_prefetch_next ((__m128i*)s);
+	cache_prefetch_next ((__m128i*)d);
+
+	while (w >= 16)
+	{
+	    save_128_aligned ((__m128i*)d, load_128_unaligned ((__m128i*)s) );
+
+	    w -= 16;
+	    d += 16;
+	    s += 16;
+	}
+
+	cache_prefetch_next ((__m128i*)s);
+	cache_prefetch_next ((__m128i*)d);
+
+	while (w >= 4)
+	{
+	    *(uint32_t *)d = *(uint32_t *)s;
+
+	    w -= 4;
+	    s += 4;
+	    d += 4;
+	}
+
+	if (w >= 2)
+	{
+	    *(uint16_t *)d = *(uint16_t *)s;
+	    w -= 2;
+	    s += 2;
+	    d += 2;
+	}
+    }
+
+    _mm_empty ();
+
+    return TRUE;
+}
+
+static void
+sse2_composite_copy_area (pixman_implementation_t *imp,
+                          pixman_op_t              op,
+                          pixman_image_t *         src_image,
+                          pixman_image_t *         mask_image,
+                          pixman_image_t *         dst_image,
+                          int32_t                  src_x,
+                          int32_t                  src_y,
+                          int32_t                  mask_x,
+                          int32_t                  mask_y,
+                          int32_t                  dest_x,
+                          int32_t                  dest_y,
+                          int32_t                  width,
+                          int32_t                  height)
+{
+    pixman_blt_sse2 (src_image->bits.bits,
+                     dst_image->bits.bits,
+                     src_image->bits.rowstride,
+                     dst_image->bits.rowstride,
+                     PIXMAN_FORMAT_BPP (src_image->bits.format),
+                     PIXMAN_FORMAT_BPP (dst_image->bits.format),
+                     src_x, src_y, dest_x, dest_y, width, height);
+}
+
+static void
+sse2_composite_over_x888_8_8888 (pixman_implementation_t *imp,
+                                 pixman_op_t              op,
+                                 pixman_image_t *         src_image,
+                                 pixman_image_t *         mask_image,
+                                 pixman_image_t *         dst_image,
+                                 int32_t                  src_x,
+                                 int32_t                  src_y,
+                                 int32_t                  mask_x,
+                                 int32_t                  mask_y,
+                                 int32_t                  dest_x,
+                                 int32_t                  dest_y,
+                                 int32_t                  width,
+                                 int32_t                  height)
+{
+    uint32_t    *src, *src_line, s;
+    uint32_t    *dst, *dst_line, d;
+    uint8_t         *mask, *mask_line;
+    uint32_t m;
+    int src_stride, mask_stride, dst_stride;
+    int32_t w;
+    __m64 ms;
+
+    __m128i xmm_src, xmm_src_lo, xmm_src_hi;
+    __m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
+    __m128i xmm_mask, xmm_mask_lo, xmm_mask_hi;
+
+    PIXMAN_IMAGE_GET_LINE (
+	dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
+    PIXMAN_IMAGE_GET_LINE (
+	mask_image, mask_x, mask_y, uint8_t, mask_stride, mask_line, 1);
+    PIXMAN_IMAGE_GET_LINE (
+	src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
+
+    while (height--)
+    {
+        src = src_line;
+        src_line += src_stride;
+        dst = dst_line;
+        dst_line += dst_stride;
+        mask = mask_line;
+        mask_line += mask_stride;
+
+        w = width;
+
+        /* call prefetch hint to optimize cache load*/
+        cache_prefetch ((__m128i*)src);
+        cache_prefetch ((__m128i*)dst);
+        cache_prefetch ((__m128i*)mask);
+
+        while (w && (unsigned long)dst & 15)
+        {
+            s = 0xff000000 | *src++;
+            m = (uint32_t) *mask++;
+            d = *dst;
+            ms = unpack_32_1x64 (s);
+
+            if (m != 0xff)
+            {
+		__m64 ma = expand_alpha_rev_1x64 (unpack_32_1x64 (m));
+		__m64 md = unpack_32_1x64 (d);
+
+                ms = in_over_1x64 (&ms, &mask_x00ff, &ma, &md);
+            }
+
+            *dst++ = pack_1x64_32 (ms);
+            w--;
+        }
+
+        /* call prefetch hint to optimize cache load*/
+        cache_prefetch ((__m128i*)src);
+        cache_prefetch ((__m128i*)dst);
+        cache_prefetch ((__m128i*)mask);
+
+        while (w >= 4)
+        {
+            /* fill cache line with next memory */
+            cache_prefetch_next ((__m128i*)src);
+            cache_prefetch_next ((__m128i*)dst);
+            cache_prefetch_next ((__m128i*)mask);
+
+            m = *(uint32_t*) mask;
+            xmm_src = _mm_or_si128 (load_128_unaligned ((__m128i*)src), mask_ff000000);
+
+            if (m == 0xffffffff)
+            {
+                save_128_aligned ((__m128i*)dst, xmm_src);
+            }
+            else
+            {
+                xmm_dst = load_128_aligned ((__m128i*)dst);
+
+                xmm_mask = _mm_unpacklo_epi16 (unpack_32_1x128 (m), _mm_setzero_si128());
+
+                unpack_128_2x128 (xmm_src, &xmm_src_lo, &xmm_src_hi);
+                unpack_128_2x128 (xmm_mask, &xmm_mask_lo, &xmm_mask_hi);
+                unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
+
+                expand_alpha_rev_2x128 (xmm_mask_lo, xmm_mask_hi, &xmm_mask_lo, &xmm_mask_hi);
+
+                in_over_2x128 (&xmm_src_lo, &xmm_src_hi, &mask_00ff, &mask_00ff, &xmm_mask_lo, &xmm_mask_hi, &xmm_dst_lo, &xmm_dst_hi);
+
+                save_128_aligned ((__m128i*)dst, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
+            }
+
+            src += 4;
+            dst += 4;
+            mask += 4;
+            w -= 4;
+        }
+
+        while (w)
+        {
+            m = (uint32_t) *mask++;
+
+            if (m)
+            {
+                s = 0xff000000 | *src;
+
+                if (m == 0xff)
+                {
+                    *dst = s;
+                }
+                else
+                {
+		    __m64 ma, md, ms;
+
+                    d = *dst;
+
+		    ma = expand_alpha_rev_1x64 (unpack_32_1x64 (m));
+		    md = unpack_32_1x64 (d);
+		    ms = unpack_32_1x64 (s);
+
+                    *dst = pack_1x64_32 (in_over_1x64 (&ms, &mask_x00ff, &ma, &md));
+                }
+
+            }
+
+            src++;
+            dst++;
+            w--;
+        }
+    }
+
+    _mm_empty ();
+}
+
+static void
+sse2_composite_over_8888_8_8888 (pixman_implementation_t *imp,
+                                 pixman_op_t              op,
+                                 pixman_image_t *         src_image,
+                                 pixman_image_t *         mask_image,
+                                 pixman_image_t *         dst_image,
+                                 int32_t                  src_x,
+                                 int32_t                  src_y,
+                                 int32_t                  mask_x,
+                                 int32_t                  mask_y,
+                                 int32_t                  dest_x,
+                                 int32_t                  dest_y,
+                                 int32_t                  width,
+                                 int32_t                  height)
+{
+    uint32_t    *src, *src_line, s;
+    uint32_t    *dst, *dst_line, d;
+    uint8_t         *mask, *mask_line;
+    uint32_t m;
+    int src_stride, mask_stride, dst_stride;
+    int32_t w;
+
+    __m128i xmm_src, xmm_src_lo, xmm_src_hi, xmm_srca_lo, xmm_srca_hi;
+    __m128i xmm_dst, xmm_dst_lo, xmm_dst_hi;
+    __m128i xmm_mask, xmm_mask_lo, xmm_mask_hi;
+
+    PIXMAN_IMAGE_GET_LINE (
+	dst_image, dest_x, dest_y, uint32_t, dst_stride, dst_line, 1);
+    PIXMAN_IMAGE_GET_LINE (
+	mask_image, mask_x, mask_y, uint8_t, mask_stride, mask_line, 1);
+    PIXMAN_IMAGE_GET_LINE (
+	src_image, src_x, src_y, uint32_t, src_stride, src_line, 1);
+
+    while (height--)
+    {
+        src = src_line;
+        src_line += src_stride;
+        dst = dst_line;
+        dst_line += dst_stride;
+        mask = mask_line;
+        mask_line += mask_stride;
+
+        w = width;
+
+        /* call prefetch hint to optimize cache load*/
+        cache_prefetch ((__m128i *)src);
+        cache_prefetch ((__m128i *)dst);
+        cache_prefetch ((__m128i *)mask);
+
+        while (w && (unsigned long)dst & 15)
+        {
+	    uint32_t sa;
+
+            s = *src++;
+            m = (uint32_t) *mask++;
+            d = *dst;
+
+	    sa = s >> 24;
+
+	    if (m)
+	    {
+		if (sa == 0xff && m == 0xff)
+		{
+		    *dst = s;
+		}
+		else
+		{
+		    __m64 ms, md, ma, msa;
+
+		    ma = expand_alpha_rev_1x64 (load_32_1x64 (m));
+		    ms = unpack_32_1x64 (s);
+		    md = unpack_32_1x64 (d);
+
+		    msa = expand_alpha_rev_1x64 (load_32_1x64 (sa));
+
+		    *dst = pack_1x64_32 (in_over_1x64 (&ms, &msa, &ma, &md));
+		}
+	    }
+
+	    dst++;
+            w--;
+        }
+
+        /* call prefetch hint to optimize cache load*/
+        cache_prefetch ((__m128i *)src);
+        cache_prefetch ((__m128i *)dst);
+        cache_prefetch ((__m128i *)mask);
+
+        while (w >= 4)
+        {
+            /* fill cache line with next memory */
+            cache_prefetch_next ((__m128i *)src);
+            cache_prefetch_next ((__m128i *)dst);
+            cache_prefetch_next ((__m128i *)mask);
+
+            m = *(uint32_t *) mask;
+
+	    if (m)
+	    {
+		xmm_src = load_128_unaligned ((__m128i*)src);
+
+		if (m == 0xffffffff && is_opaque (xmm_src))
+		{
+		    save_128_aligned ((__m128i *)dst, xmm_src);
+		}
+		else
+		{
+		    xmm_dst = load_128_aligned ((__m128i *)dst);
+
+		    xmm_mask = _mm_unpacklo_epi16 (unpack_32_1x128 (m), _mm_setzero_si128());
+
+		    unpack_128_2x128 (xmm_src, &xmm_src_lo, &xmm_src_hi);
+		    unpack_128_2x128 (xmm_mask, &xmm_mask_lo, &xmm_mask_hi);
+		    unpack_128_2x128 (xmm_dst, &xmm_dst_lo, &xmm_dst_hi);
+
+		    expand_alpha_2x128 (xmm_src_lo, xmm_src_hi, &xmm_srca_lo, &xmm_srca_hi);
+		    expand_alpha_rev_2x128 (xmm_mask_lo, xmm_mask_hi, &xmm_mask_lo, &xmm_mask_hi);
+
+		    in_over_2x128 (&xmm_src_lo, &xmm_src_hi, &xmm_srca_lo, &xmm_srca_hi,
+				   &xmm_mask_lo, &xmm_mask_hi, &xmm_dst_lo, &xmm_dst_hi);
+
+		    save_128_aligned ((__m128i*)dst, pack_2x128_128 (xmm_dst_lo, xmm_dst_hi));
+		}
+	    }
+
+            src += 4;
+            dst += 4;
+            mask += 4;
+            w -= 4;
+        }
+
+        while (w)
+        {
+	    uint32_t sa;
+
+            s = *src++;
+            m = (uint32_t) *mask++;
+            d = *dst;
+
+	    sa = s >> 24;
+
+	    if (m)
+	    {
+		if (sa == 0xff && m == 0xff)
+		{
+		    *dst = s;
+		}
+		else
+		{
+		    __m64 ms, md, ma, msa;
+
+		    ma = expand_alpha_rev_1x64 (load_32_1x64 (m));
+		    ms = unpack_32_1x64 (s);
+		    md = unpack_32_1x64 (d);
+
+		    msa = expand_alpha_rev_1x64 (load_32_1x64 (sa));
+
+		    *dst = pack_1x64_32 (in_over_1x64 (&ms, &msa, &ma, &md));
+		}
+	    }
+
+	    dst++;
+            w--;
+        }
+    }
+
+    _mm_empty ();
+}
+
+static const pixman_fast_path_t sse2_fast_paths[] =
+{
+    /* PIXMAN_OP_OVER */
+    PIXMAN_STD_FAST_PATH (OVER, solid, a8, r5g6b5, sse2_composite_over_n_8_0565),
+    PIXMAN_STD_FAST_PATH (OVER, solid, a8, b5g6r5, sse2_composite_over_n_8_0565),
+    PIXMAN_STD_FAST_PATH (OVER, solid, null, a8r8g8b8, sse2_composite_over_n_8888),
+    PIXMAN_STD_FAST_PATH (OVER, solid, null, x8r8g8b8, sse2_composite_over_n_8888),
+    PIXMAN_STD_FAST_PATH (OVER, solid, null, r5g6b5, sse2_composite_over_n_0565),
+    PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, null, a8r8g8b8, sse2_composite_over_8888_8888),
+    PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, null, x8r8g8b8, sse2_composite_over_8888_8888),
+    PIXMAN_STD_FAST_PATH (OVER, a8b8g8r8, null, a8b8g8r8, sse2_composite_over_8888_8888),
+    PIXMAN_STD_FAST_PATH (OVER, a8b8g8r8, null, x8b8g8r8, sse2_composite_over_8888_8888),
+    PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, null, r5g6b5, sse2_composite_over_8888_0565),
+    PIXMAN_STD_FAST_PATH (OVER, a8b8g8r8, null, b5g6r5, sse2_composite_over_8888_0565),
+    PIXMAN_STD_FAST_PATH (OVER, solid, a8, a8r8g8b8, sse2_composite_over_n_8_8888),
+    PIXMAN_STD_FAST_PATH (OVER, solid, a8, x8r8g8b8, sse2_composite_over_n_8_8888),
+    PIXMAN_STD_FAST_PATH (OVER, solid, a8, a8b8g8r8, sse2_composite_over_n_8_8888),
+    PIXMAN_STD_FAST_PATH (OVER, solid, a8, x8b8g8r8, sse2_composite_over_n_8_8888),
+    PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, a8, x8r8g8b8, sse2_composite_over_8888_8_8888),
+    PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, a8, a8r8g8b8, sse2_composite_over_8888_8_8888),
+    PIXMAN_STD_FAST_PATH (OVER, a8b8g8r8, a8, x8b8g8r8, sse2_composite_over_8888_8_8888),
+    PIXMAN_STD_FAST_PATH (OVER, a8b8g8r8, a8, a8b8g8r8, sse2_composite_over_8888_8_8888),
+    PIXMAN_STD_FAST_PATH (OVER, x8r8g8b8, a8, x8r8g8b8, sse2_composite_over_x888_8_8888),
+    PIXMAN_STD_FAST_PATH (OVER, x8r8g8b8, a8, a8r8g8b8, sse2_composite_over_x888_8_8888),
+    PIXMAN_STD_FAST_PATH (OVER, x8b8g8r8, a8, x8b8g8r8, sse2_composite_over_x888_8_8888),
+    PIXMAN_STD_FAST_PATH (OVER, x8b8g8r8, a8, a8b8g8r8, sse2_composite_over_x888_8_8888),
+    PIXMAN_STD_FAST_PATH (OVER, x8r8g8b8, solid, a8r8g8b8, sse2_composite_over_x888_n_8888),
+    PIXMAN_STD_FAST_PATH (OVER, x8r8g8b8, solid, x8r8g8b8, sse2_composite_over_x888_n_8888),
+    PIXMAN_STD_FAST_PATH (OVER, x8b8g8r8, solid, a8b8g8r8, sse2_composite_over_x888_n_8888),
+    PIXMAN_STD_FAST_PATH (OVER, x8b8g8r8, solid, x8b8g8r8, sse2_composite_over_x888_n_8888),
+    PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, solid, a8r8g8b8, sse2_composite_over_8888_n_8888),
+    PIXMAN_STD_FAST_PATH (OVER, a8r8g8b8, solid, x8r8g8b8, sse2_composite_over_8888_n_8888),
+    PIXMAN_STD_FAST_PATH (OVER, a8b8g8r8, solid, a8b8g8r8, sse2_composite_over_8888_n_8888),
+    PIXMAN_STD_FAST_PATH (OVER, a8b8g8r8, solid, x8b8g8r8, sse2_composite_over_8888_n_8888),
+    PIXMAN_STD_FAST_PATH_CA (OVER, solid, a8r8g8b8, a8r8g8b8, sse2_composite_over_n_8888_8888_ca),
+    PIXMAN_STD_FAST_PATH_CA (OVER, solid, a8r8g8b8, x8r8g8b8, sse2_composite_over_n_8888_8888_ca),
+    PIXMAN_STD_FAST_PATH_CA (OVER, solid, a8b8g8r8, a8b8g8r8, sse2_composite_over_n_8888_8888_ca),
+    PIXMAN_STD_FAST_PATH_CA (OVER, solid, a8b8g8r8, x8b8g8r8, sse2_composite_over_n_8888_8888_ca),
+    PIXMAN_STD_FAST_PATH_CA (OVER, solid, a8r8g8b8, r5g6b5, sse2_composite_over_n_8888_0565_ca),
+    PIXMAN_STD_FAST_PATH_CA (OVER, solid, a8b8g8r8, b5g6r5, sse2_composite_over_n_8888_0565_ca),
+    PIXMAN_STD_FAST_PATH (OVER, pixbuf, pixbuf, a8r8g8b8, sse2_composite_over_pixbuf_8888),
+    PIXMAN_STD_FAST_PATH (OVER, pixbuf, pixbuf, x8r8g8b8, sse2_composite_over_pixbuf_8888),
+    PIXMAN_STD_FAST_PATH (OVER, rpixbuf, rpixbuf, a8b8g8r8, sse2_composite_over_pixbuf_8888),
+    PIXMAN_STD_FAST_PATH (OVER, rpixbuf, rpixbuf, x8b8g8r8, sse2_composite_over_pixbuf_8888),
+    PIXMAN_STD_FAST_PATH (OVER, pixbuf, pixbuf, r5g6b5, sse2_composite_over_pixbuf_0565),
+    PIXMAN_STD_FAST_PATH (OVER, rpixbuf, rpixbuf, b5g6r5, sse2_composite_over_pixbuf_0565),
+    PIXMAN_STD_FAST_PATH (OVER, x8r8g8b8, null, x8r8g8b8, sse2_composite_copy_area),
+    PIXMAN_STD_FAST_PATH (OVER, x8b8g8r8, null, x8b8g8r8, sse2_composite_copy_area),
+
+    /* PIXMAN_OP_ADD */
+    PIXMAN_STD_FAST_PATH_CA (ADD, solid, a8r8g8b8, a8r8g8b8, sse2_composite_add_n_8888_8888_ca),
+    PIXMAN_STD_FAST_PATH (ADD, a8, null, a8, sse2_composite_add_8000_8000),
+    PIXMAN_STD_FAST_PATH (ADD, a8r8g8b8, null, a8r8g8b8, sse2_composite_add_8888_8888),
+    PIXMAN_STD_FAST_PATH (ADD, a8b8g8r8, null, a8b8g8r8, sse2_composite_add_8888_8888),
+    PIXMAN_STD_FAST_PATH (ADD, solid, a8, a8, sse2_composite_add_n_8_8),
+
+    /* PIXMAN_OP_SRC */
+    PIXMAN_STD_FAST_PATH (SRC, solid, a8, a8r8g8b8, sse2_composite_src_n_8_8888),
+    PIXMAN_STD_FAST_PATH (SRC, solid, a8, x8r8g8b8, sse2_composite_src_n_8_8888),
+    PIXMAN_STD_FAST_PATH (SRC, solid, a8, a8b8g8r8, sse2_composite_src_n_8_8888),
+    PIXMAN_STD_FAST_PATH (SRC, solid, a8, x8b8g8r8, sse2_composite_src_n_8_8888),
+    PIXMAN_STD_FAST_PATH (SRC, a8r8g8b8, null, a8r8g8b8, sse2_composite_copy_area),
+    PIXMAN_STD_FAST_PATH (SRC, a8b8g8r8, null, a8b8g8r8, sse2_composite_copy_area),
+    PIXMAN_STD_FAST_PATH (SRC, a8r8g8b8, null, x8r8g8b8, sse2_composite_copy_area),
+    PIXMAN_STD_FAST_PATH (SRC, a8b8g8r8, null, x8b8g8r8, sse2_composite_copy_area),
+    PIXMAN_STD_FAST_PATH (SRC, x8r8g8b8, null, x8r8g8b8, sse2_composite_copy_area),
+    PIXMAN_STD_FAST_PATH (SRC, x8b8g8r8, null, x8b8g8r8, sse2_composite_copy_area),
+    PIXMAN_STD_FAST_PATH (SRC, r5g6b5, null, r5g6b5, sse2_composite_copy_area),
+    PIXMAN_STD_FAST_PATH (SRC, b5g6r5, null, b5g6r5, sse2_composite_copy_area),
+
+    /* PIXMAN_OP_IN */
+    PIXMAN_STD_FAST_PATH (IN, a8, null, a8, sse2_composite_in_8_8),
+    PIXMAN_STD_FAST_PATH (IN, solid, a8, a8, sse2_composite_in_n_8_8),
+
+    { PIXMAN_OP_NONE },
+};
+
+static pixman_bool_t
+sse2_blt (pixman_implementation_t *imp,
+          uint32_t *               src_bits,
+          uint32_t *               dst_bits,
+          int                      src_stride,
+          int                      dst_stride,
+          int                      src_bpp,
+          int                      dst_bpp,
+          int                      src_x,
+          int                      src_y,
+          int                      dst_x,
+          int                      dst_y,
+          int                      width,
+          int                      height)
+{
+    if (!pixman_blt_sse2 (
+            src_bits, dst_bits, src_stride, dst_stride, src_bpp, dst_bpp,
+            src_x, src_y, dst_x, dst_y, width, height))
+
+    {
+	return _pixman_implementation_blt (
+	    imp->delegate,
+	    src_bits, dst_bits, src_stride, dst_stride, src_bpp, dst_bpp,
+	    src_x, src_y, dst_x, dst_y, width, height);
+    }
+
+    return TRUE;
+}
+
+#if defined(__GNUC__) && !defined(__x86_64__) && !defined(__amd64__)
+__attribute__((__force_align_arg_pointer__))
+#endif
+static pixman_bool_t
+sse2_fill (pixman_implementation_t *imp,
+           uint32_t *               bits,
+           int                      stride,
+           int                      bpp,
+           int                      x,
+           int                      y,
+           int                      width,
+           int                      height,
+           uint32_t xor)
+{
+    if (!pixman_fill_sse2 (bits, stride, bpp, x, y, width, height, xor))
+    {
+	return _pixman_implementation_fill (
+	    imp->delegate, bits, stride, bpp, x, y, width, height, xor);
+    }
+
+    return TRUE;
+}
+
+#if defined(__GNUC__) && !defined(__x86_64__) && !defined(__amd64__)
+__attribute__((__force_align_arg_pointer__))
+#endif
+pixman_implementation_t *
+_pixman_implementation_create_sse2 (void)
+{
+#ifdef USE_MMX
+    pixman_implementation_t *fallback = _pixman_implementation_create_mmx ();
+#else
+    pixman_implementation_t *fallback = _pixman_implementation_create_fast_path ();
+#endif
+    pixman_implementation_t *imp = _pixman_implementation_create (fallback, sse2_fast_paths);
+
+    /* SSE2 constants */
+    mask_565_r  = create_mask_2x32_128 (0x00f80000, 0x00f80000);
+    mask_565_g1 = create_mask_2x32_128 (0x00070000, 0x00070000);
+    mask_565_g2 = create_mask_2x32_128 (0x000000e0, 0x000000e0);
+    mask_565_b  = create_mask_2x32_128 (0x0000001f, 0x0000001f);
+    mask_red   = create_mask_2x32_128 (0x00f80000, 0x00f80000);
+    mask_green = create_mask_2x32_128 (0x0000fc00, 0x0000fc00);
+    mask_blue  = create_mask_2x32_128 (0x000000f8, 0x000000f8);
+    mask_565_fix_rb = create_mask_2x32_128 (0x00e000e0, 0x00e000e0);
+    mask_565_fix_g = create_mask_2x32_128  (0x0000c000, 0x0000c000);
+    mask_0080 = create_mask_16_128 (0x0080);
+    mask_00ff = create_mask_16_128 (0x00ff);
+    mask_0101 = create_mask_16_128 (0x0101);
+    mask_ffff = create_mask_16_128 (0xffff);
+    mask_ff000000 = create_mask_2x32_128 (0xff000000, 0xff000000);
+    mask_alpha = create_mask_2x32_128 (0x00ff0000, 0x00000000);
+
+    /* MMX constants */
+    mask_x565_rgb = create_mask_2x32_64 (0x000001f0, 0x003f001f);
+    mask_x565_unpack = create_mask_2x32_64 (0x00000084, 0x04100840);
+
+    mask_x0080 = create_mask_16_64 (0x0080);
+    mask_x00ff = create_mask_16_64 (0x00ff);
+    mask_x0101 = create_mask_16_64 (0x0101);
+    mask_x_alpha = create_mask_2x32_64 (0x00ff0000, 0x00000000);
+
+    _mm_empty ();
+
+    /* Set up function pointers */
+
+    /* SSE code patch for fbcompose.c */
+    imp->combine_32[PIXMAN_OP_OVER] = sse2_combine_over_u;
+    imp->combine_32[PIXMAN_OP_OVER_REVERSE] = sse2_combine_over_reverse_u;
+    imp->combine_32[PIXMAN_OP_IN] = sse2_combine_in_u;
+    imp->combine_32[PIXMAN_OP_IN_REVERSE] = sse2_combine_in_reverse_u;
+    imp->combine_32[PIXMAN_OP_OUT] = sse2_combine_out_u;
+    imp->combine_32[PIXMAN_OP_OUT_REVERSE] = sse2_combine_out_reverse_u;
+    imp->combine_32[PIXMAN_OP_ATOP] = sse2_combine_atop_u;
+    imp->combine_32[PIXMAN_OP_ATOP_REVERSE] = sse2_combine_atop_reverse_u;
+    imp->combine_32[PIXMAN_OP_XOR] = sse2_combine_xor_u;
+    imp->combine_32[PIXMAN_OP_ADD] = sse2_combine_add_u;
+
+    imp->combine_32[PIXMAN_OP_SATURATE] = sse2_combine_saturate_u;
+
+    imp->combine_32_ca[PIXMAN_OP_SRC] = sse2_combine_src_ca;
+    imp->combine_32_ca[PIXMAN_OP_OVER] = sse2_combine_over_ca;
+    imp->combine_32_ca[PIXMAN_OP_OVER_REVERSE] = sse2_combine_over_reverse_ca;
+    imp->combine_32_ca[PIXMAN_OP_IN] = sse2_combine_in_ca;
+    imp->combine_32_ca[PIXMAN_OP_IN_REVERSE] = sse2_combine_in_reverse_ca;
+    imp->combine_32_ca[PIXMAN_OP_OUT] = sse2_combine_out_ca;
+    imp->combine_32_ca[PIXMAN_OP_OUT_REVERSE] = sse2_combine_out_reverse_ca;
+    imp->combine_32_ca[PIXMAN_OP_ATOP] = sse2_combine_atop_ca;
+    imp->combine_32_ca[PIXMAN_OP_ATOP_REVERSE] = sse2_combine_atop_reverse_ca;
+    imp->combine_32_ca[PIXMAN_OP_XOR] = sse2_combine_xor_ca;
+    imp->combine_32_ca[PIXMAN_OP_ADD] = sse2_combine_add_ca;
+
+    imp->blt = sse2_blt;
+    imp->fill = sse2_fill;
+
+    return imp;
+}
+
+#endif /* USE_SSE2 */
diff --git a/pixman/pixman/pixman-timer.c b/pixman/pixman/pixman-timer.c
new file mode 100644
index 000000000..f5ae18e89
--- /dev/null
+++ b/pixman/pixman/pixman-timer.c
@@ -0,0 +1,66 @@
+/*
+ * Copyright © 2007 Red Hat, Inc.
+ *
+ * Permission to use, copy, modify, distribute, and sell this software and its
+ * documentation for any purpose is hereby granted without fee, provided that
+ * the above copyright notice appear in all copies and that both that
+ * copyright notice and this permission notice appear in supporting
+ * documentation, and that the name of Red Hat not be used in advertising or
+ * publicity pertaining to distribution of the software without specific,
+ * written prior permission.  Red Hat makes no representations about the
+ * suitability of this software for any purpose.  It is provided "as is"
+ * without express or implied warranty.
+ *
+ * RED HAT DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING ALL
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL RED HAT
+ * BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
+ * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+ * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <stdlib.h>
+#include <stdio.h>
+#include "pixman-private.h"
+
+#ifdef PIXMAN_TIMERS
+
+static pixman_timer_t *timers;
+
+static void
+dump_timers (void)
+{
+    pixman_timer_t *timer;
+
+    for (timer = timers; timer != NULL; timer = timer->next)
+    {
+	printf ("%s:   total: %llu     n: %llu      avg: %f\n",
+	        timer->name,
+	        timer->total,
+	        timer->n_times,
+	        timer->total / (double)timer->n_times);
+    }
+}
+
+void
+pixman_timer_register (pixman_timer_t *timer)
+{
+    static int initialized;
+
+    int atexit (void (*function)(void));
+
+    if (!initialized)
+    {
+	atexit (dump_timers);
+	initialized = 1;
+    }
+
+    timer->next = timers;
+    timers = timer;
+}
+
+#endif
diff --git a/pixman/pixman/pixman-trap.c b/pixman/pixman/pixman-trap.c
new file mode 100644
index 000000000..8353992c5
--- /dev/null
+++ b/pixman/pixman/pixman-trap.c
@@ -0,0 +1,392 @@
+/*
+ * Copyright © 2004 Keith Packard
+ *
+ * Permission to use, copy, modify, distribute, and sell this software and its
+ * documentation for any purpose is hereby granted without fee, provided that
+ * the above copyright notice appear in all copies and that both that
+ * copyright notice and this permission notice appear in supporting
+ * documentation, and that the name of Keith Packard not be used in
+ * advertising or publicity pertaining to distribution of the software without
+ * specific, written prior permission.  Keith Packard makes no
+ * representations about the suitability of this software for any purpose.  It
+ * is provided "as is" without express or implied warranty.
+ *
+ * KEITH PACKARD DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
+ * INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO
+ * EVENT SHALL KEITH PACKARD BE LIABLE FOR ANY SPECIAL, INDIRECT OR
+ * CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE,
+ * DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
+ * TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+ * PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+
+#include <stdio.h>
+#include "pixman-private.h"
+
+/*
+ * Compute the smallest value greater than or equal to y which is on a
+ * grid row.
+ */
+
+PIXMAN_EXPORT pixman_fixed_t
+pixman_sample_ceil_y (pixman_fixed_t y, int n)
+{
+    pixman_fixed_t f = pixman_fixed_frac (y);
+    pixman_fixed_t i = pixman_fixed_floor (y);
+
+    f = DIV (f - Y_FRAC_FIRST (n) + (STEP_Y_SMALL (n) - pixman_fixed_e), STEP_Y_SMALL (n)) * STEP_Y_SMALL (n) +
+	Y_FRAC_FIRST (n);
+    
+    if (f > Y_FRAC_LAST (n))
+    {
+	if (pixman_fixed_to_int (i) == 0x7fff)
+	{
+	    f = 0xffff; /* saturate */
+	}
+	else
+	{
+	    f = Y_FRAC_FIRST (n);
+	    i += pixman_fixed_1;
+	}
+    }
+    return (i | f);
+}
+
+/*
+ * Compute the largest value strictly less than y which is on a
+ * grid row.
+ */
+PIXMAN_EXPORT pixman_fixed_t
+pixman_sample_floor_y (pixman_fixed_t y,
+                       int            n)
+{
+    pixman_fixed_t f = pixman_fixed_frac (y);
+    pixman_fixed_t i = pixman_fixed_floor (y);
+
+    f = DIV (f - pixman_fixed_e - Y_FRAC_FIRST (n), STEP_Y_SMALL (n)) * STEP_Y_SMALL (n) +
+	Y_FRAC_FIRST (n);
+
+    if (f < Y_FRAC_FIRST (n))
+    {
+	if (pixman_fixed_to_int (i) == 0x8000)
+	{
+	    f = 0; /* saturate */
+	}
+	else
+	{
+	    f = Y_FRAC_LAST (n);
+	    i -= pixman_fixed_1;
+	}
+    }
+    return (i | f);
+}
+
+/*
+ * Step an edge by any amount (including negative values)
+ */
+PIXMAN_EXPORT void
+pixman_edge_step (pixman_edge_t *e,
+                  int            n)
+{
+    pixman_fixed_48_16_t ne;
+
+    e->x += n * e->stepx;
+
+    ne = e->e + n * (pixman_fixed_48_16_t) e->dx;
+
+    if (n >= 0)
+    {
+	if (ne > 0)
+	{
+	    int nx = (ne + e->dy - 1) / e->dy;
+	    e->e = ne - nx * (pixman_fixed_48_16_t) e->dy;
+	    e->x += nx * e->signdx;
+	}
+    }
+    else
+    {
+	if (ne <= -e->dy)
+	{
+	    int nx = (-ne) / e->dy;
+	    e->e = ne + nx * (pixman_fixed_48_16_t) e->dy;
+	    e->x -= nx * e->signdx;
+	}
+    }
+}
+
+/*
+ * A private routine to initialize the multi-step
+ * elements of an edge structure
+ */
+static void
+_pixman_edge_multi_init (pixman_edge_t * e,
+                         int             n,
+                         pixman_fixed_t *stepx_p,
+                         pixman_fixed_t *dx_p)
+{
+    pixman_fixed_t stepx;
+    pixman_fixed_48_16_t ne;
+
+    ne = n * (pixman_fixed_48_16_t) e->dx;
+    stepx = n * e->stepx;
+
+    if (ne > 0)
+    {
+	int nx = ne / e->dy;
+	ne -= nx * e->dy;
+	stepx += nx * e->signdx;
+    }
+
+    *dx_p = ne;
+    *stepx_p = stepx;
+}
+
+/*
+ * Initialize one edge structure given the line endpoints and a
+ * starting y value
+ */
+PIXMAN_EXPORT void
+pixman_edge_init (pixman_edge_t *e,
+                  int            n,
+                  pixman_fixed_t y_start,
+                  pixman_fixed_t x_top,
+                  pixman_fixed_t y_top,
+                  pixman_fixed_t x_bot,
+                  pixman_fixed_t y_bot)
+{
+    pixman_fixed_t dx, dy;
+
+    e->x = x_top;
+    e->e = 0;
+    dx = x_bot - x_top;
+    dy = y_bot - y_top;
+    e->dy = dy;
+    e->dx = 0;
+
+    if (dy)
+    {
+	if (dx >= 0)
+	{
+	    e->signdx = 1;
+	    e->stepx = dx / dy;
+	    e->dx = dx % dy;
+	    e->e = -dy;
+	}
+	else
+	{
+	    e->signdx = -1;
+	    e->stepx = -(-dx / dy);
+	    e->dx = -dx % dy;
+	    e->e = 0;
+	}
+
+	_pixman_edge_multi_init (e, STEP_Y_SMALL (n),
+				 &e->stepx_small, &e->dx_small);
+
+	_pixman_edge_multi_init (e, STEP_Y_BIG (n),
+				 &e->stepx_big, &e->dx_big);
+    }
+    pixman_edge_step (e, y_start - y_top);
+}
+
+/*
+ * Initialize one edge structure given a line, starting y value
+ * and a pixel offset for the line
+ */
+PIXMAN_EXPORT void
+pixman_line_fixed_edge_init (pixman_edge_t *            e,
+                             int                        n,
+                             pixman_fixed_t             y,
+                             const pixman_line_fixed_t *line,
+                             int                        x_off,
+                             int                        y_off)
+{
+    pixman_fixed_t x_off_fixed = pixman_int_to_fixed (x_off);
+    pixman_fixed_t y_off_fixed = pixman_int_to_fixed (y_off);
+    const pixman_point_fixed_t *top, *bot;
+
+    if (line->p1.y <= line->p2.y)
+    {
+	top = &line->p1;
+	bot = &line->p2;
+    }
+    else
+    {
+	top = &line->p2;
+	bot = &line->p1;
+    }
+    
+    pixman_edge_init (e, n, y,
+                      top->x + x_off_fixed,
+                      top->y + y_off_fixed,
+                      bot->x + x_off_fixed,
+                      bot->y + y_off_fixed);
+}
+
+PIXMAN_EXPORT void
+pixman_add_traps (pixman_image_t * image,
+                  int16_t          x_off,
+                  int16_t          y_off,
+                  int              ntrap,
+                  pixman_trap_t *  traps)
+{
+    int bpp;
+    int width;
+    int height;
+
+    pixman_fixed_t x_off_fixed;
+    pixman_fixed_t y_off_fixed;
+    pixman_edge_t l, r;
+    pixman_fixed_t t, b;
+
+    _pixman_image_validate (image);
+    
+    width = image->bits.width;
+    height = image->bits.height;
+    bpp = PIXMAN_FORMAT_BPP (image->bits.format);
+
+    x_off_fixed = pixman_int_to_fixed (x_off);
+    y_off_fixed = pixman_int_to_fixed (y_off);
+
+    while (ntrap--)
+    {
+	t = traps->top.y + y_off_fixed;
+	if (t < 0)
+	    t = 0;
+	t = pixman_sample_ceil_y (t, bpp);
+
+	b = traps->bot.y + y_off_fixed;
+	if (pixman_fixed_to_int (b) >= height)
+	    b = pixman_int_to_fixed (height) - 1;
+	b = pixman_sample_floor_y (b, bpp);
+
+	if (b >= t)
+	{
+	    /* initialize edge walkers */
+	    pixman_edge_init (&l, bpp, t,
+	                      traps->top.l + x_off_fixed,
+	                      traps->top.y + y_off_fixed,
+	                      traps->bot.l + x_off_fixed,
+	                      traps->bot.y + y_off_fixed);
+
+	    pixman_edge_init (&r, bpp, t,
+	                      traps->top.r + x_off_fixed,
+	                      traps->top.y + y_off_fixed,
+	                      traps->bot.r + x_off_fixed,
+	                      traps->bot.y + y_off_fixed);
+
+	    pixman_rasterize_edges (image, &l, &r, t, b);
+	}
+
+	traps++;
+    }
+}
+
+#if 0
+static void
+dump_image (pixman_image_t *image,
+            const char *    title)
+{
+    int i, j;
+
+    if (!image->type == BITS)
+	printf ("%s is not a regular image\n", title);
+
+    if (!image->bits.format == PIXMAN_a8)
+	printf ("%s is not an alpha mask\n", title);
+
+    printf ("\n\n\n%s: \n", title);
+
+    for (i = 0; i < image->bits.height; ++i)
+    {
+	uint8_t *line =
+	    (uint8_t *)&(image->bits.bits[i * image->bits.rowstride]);
+
+	for (j = 0; j < image->bits.width; ++j)
+	    printf ("%c", line[j] ? '#' : ' ');
+
+	printf ("\n");
+    }
+}
+#endif
+
+PIXMAN_EXPORT void
+pixman_add_trapezoids (pixman_image_t *          image,
+                       int16_t                   x_off,
+                       int                       y_off,
+                       int                       ntraps,
+                       const pixman_trapezoid_t *traps)
+{
+    int i;
+
+#if 0
+    dump_image (image, "before");
+#endif
+
+    for (i = 0; i < ntraps; ++i)
+    {
+	const pixman_trapezoid_t *trap = &(traps[i]);
+
+	if (!pixman_trapezoid_valid (trap))
+	    continue;
+
+	pixman_rasterize_trapezoid (image, trap, x_off, y_off);
+    }
+
+#if 0
+    dump_image (image, "after");
+#endif
+}
+
+PIXMAN_EXPORT void
+pixman_rasterize_trapezoid (pixman_image_t *          image,
+                            const pixman_trapezoid_t *trap,
+                            int                       x_off,
+                            int                       y_off)
+{
+    int bpp;
+    int width;
+    int height;
+
+    pixman_fixed_t x_off_fixed;
+    pixman_fixed_t y_off_fixed;
+    pixman_edge_t l, r;
+    pixman_fixed_t t, b;
+
+    return_if_fail (image->type == BITS);
+
+    _pixman_image_validate (image);
+    
+    if (!pixman_trapezoid_valid (trap))
+	return;
+
+    width = image->bits.width;
+    height = image->bits.height;
+    bpp = PIXMAN_FORMAT_BPP (image->bits.format);
+
+    x_off_fixed = pixman_int_to_fixed (x_off);
+    y_off_fixed = pixman_int_to_fixed (y_off);
+
+    t = trap->top + y_off_fixed;
+    if (t < 0)
+	t = 0;
+    t = pixman_sample_ceil_y (t, bpp);
+
+    b = trap->bottom + y_off_fixed;
+    if (pixman_fixed_to_int (b) >= height)
+	b = pixman_int_to_fixed (height) - 1;
+    b = pixman_sample_floor_y (b, bpp);
+    
+    if (b >= t)
+    {
+	/* initialize edge walkers */
+	pixman_line_fixed_edge_init (&l, bpp, t, &trap->left, x_off, y_off);
+	pixman_line_fixed_edge_init (&r, bpp, t, &trap->right, x_off, y_off);
+
+	pixman_rasterize_edges (image, &l, &r, t, b);
+    }
+}
diff --git a/pixman/pixman/pixman-utils.c b/pixman/pixman/pixman-utils.c
new file mode 100644
index 000000000..3ef88b753
--- /dev/null
+++ b/pixman/pixman/pixman-utils.c
@@ -0,0 +1,258 @@
+/*
+ * Copyright © 2000 SuSE, Inc.
+ * Copyright © 1999 Keith Packard
+ *
+ * Permission to use, copy, modify, distribute, and sell this software and its
+ * documentation for any purpose is hereby granted without fee, provided that
+ * the above copyright notice appear in all copies and that both that
+ * copyright notice and this permission notice appear in supporting
+ * documentation, and that the name of SuSE not be used in advertising or
+ * publicity pertaining to distribution of the software without specific,
+ * written prior permission.  SuSE makes no representations about the
+ * suitability of this software for any purpose.  It is provided "as is"
+ * without express or implied warranty.
+ *
+ * SuSE DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING ALL
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL SuSE
+ * BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
+ * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+ * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ *
+ * Author:  Keith Packard, SuSE, Inc.
+ */
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "pixman-private.h"
+
+pixman_bool_t
+pixman_multiply_overflows_int (unsigned int a,
+                               unsigned int b)
+{
+    return a >= INT32_MAX / b;
+}
+
+pixman_bool_t
+pixman_addition_overflows_int (unsigned int a,
+                               unsigned int b)
+{
+    return a > INT32_MAX - b;
+}
+
+void *
+pixman_malloc_ab (unsigned int a,
+                  unsigned int b)
+{
+    if (a >= INT32_MAX / b)
+	return NULL;
+
+    return malloc (a * b);
+}
+
+void *
+pixman_malloc_abc (unsigned int a,
+                   unsigned int b,
+                   unsigned int c)
+{
+    if (a >= INT32_MAX / b)
+	return NULL;
+    else if (a * b >= INT32_MAX / c)
+	return NULL;
+    else
+	return malloc (a * b * c);
+}
+
+/*
+ * Helper routine to expand a color component from 0 < n <= 8 bits to 16
+ * bits by replication.
+ */
+static inline uint64_t
+expand16 (const uint8_t val, int nbits)
+{
+    /* Start out with the high bit of val in the high bit of result. */
+    uint16_t result = (uint16_t)val << (16 - nbits);
+
+    if (nbits == 0)
+	return 0;
+
+    /* Copy the bits in result, doubling the number of bits each time, until
+     * we fill all 16 bits.
+     */
+    while (nbits < 16)
+    {
+	result |= result >> nbits;
+	nbits *= 2;
+    }
+
+    return result;
+}
+
+/*
+ * This function expands images from ARGB8 format to ARGB16.  To preserve
+ * precision, it needs to know the original source format.  For example, if the
+ * source was PIXMAN_x1r5g5b5 and the red component contained bits 12345, then
+ * the expanded value is 12345123.  To correctly expand this to 16 bits, it
+ * should be 1234512345123451 and not 1234512312345123.
+ */
+void
+pixman_expand (uint64_t *           dst,
+               const uint32_t *     src,
+               pixman_format_code_t format,
+               int                  width)
+{
+    /*
+     * Determine the sizes of each component and the masks and shifts
+     * required to extract them from the source pixel.
+     */
+    const int a_size = PIXMAN_FORMAT_A (format),
+              r_size = PIXMAN_FORMAT_R (format),
+              g_size = PIXMAN_FORMAT_G (format),
+              b_size = PIXMAN_FORMAT_B (format);
+    const int a_shift = 32 - a_size,
+              r_shift = 24 - r_size,
+              g_shift = 16 - g_size,
+              b_shift =  8 - b_size;
+    const uint8_t a_mask = ~(~0 << a_size),
+                  r_mask = ~(~0 << r_size),
+                  g_mask = ~(~0 << g_size),
+                  b_mask = ~(~0 << b_size);
+    int i;
+
+    /* Start at the end so that we can do the expansion in place
+     * when src == dst
+     */
+    for (i = width - 1; i >= 0; i--)
+    {
+	const uint32_t pixel = src[i];
+	const uint8_t a = (pixel >> a_shift) & a_mask,
+	              r = (pixel >> r_shift) & r_mask,
+	              g = (pixel >> g_shift) & g_mask,
+	              b = (pixel >> b_shift) & b_mask;
+	const uint64_t a16 = a_size ? expand16 (a, a_size) : 0xffff,
+	               r16 = expand16 (r, r_size),
+	               g16 = expand16 (g, g_size),
+	               b16 = expand16 (b, b_size);
+
+	dst[i] = a16 << 48 | r16 << 32 | g16 << 16 | b16;
+    }
+}
+
+/*
+ * Contracting is easier than expanding.  We just need to truncate the
+ * components.
+ */
+void
+pixman_contract (uint32_t *      dst,
+                 const uint64_t *src,
+                 int             width)
+{
+    int i;
+
+    /* Start at the beginning so that we can do the contraction in
+     * place when src == dst
+     */
+    for (i = 0; i < width; i++)
+    {
+	const uint8_t a = src[i] >> 56,
+	              r = src[i] >> 40,
+	              g = src[i] >> 24,
+	              b = src[i] >> 8;
+
+	dst[i] = a << 24 | r << 16 | g << 8 | b;
+    }
+}
+
+#define N_TMP_BOXES (16)
+
+pixman_bool_t
+pixman_region16_copy_from_region32 (pixman_region16_t *dst,
+                                    pixman_region32_t *src)
+{
+    int n_boxes, i;
+    pixman_box32_t *boxes32;
+    pixman_box16_t *boxes16;
+    pixman_bool_t retval;
+
+    boxes32 = pixman_region32_rectangles (src, &n_boxes);
+
+    boxes16 = pixman_malloc_ab (n_boxes, sizeof (pixman_box16_t));
+
+    if (!boxes16)
+	return FALSE;
+
+    for (i = 0; i < n_boxes; ++i)
+    {
+	boxes16[i].x1 = boxes32[i].x1;
+	boxes16[i].y1 = boxes32[i].y1;
+	boxes16[i].x2 = boxes32[i].x2;
+	boxes16[i].y2 = boxes32[i].y2;
+    }
+
+    pixman_region_fini (dst);
+    retval = pixman_region_init_rects (dst, boxes16, n_boxes);
+    free (boxes16);
+    return retval;
+}
+
+pixman_bool_t
+pixman_region32_copy_from_region16 (pixman_region32_t *dst,
+                                    pixman_region16_t *src)
+{
+    int n_boxes, i;
+    pixman_box16_t *boxes16;
+    pixman_box32_t *boxes32;
+    pixman_box32_t tmp_boxes[N_TMP_BOXES];
+    pixman_bool_t retval;
+
+    boxes16 = pixman_region_rectangles (src, &n_boxes);
+
+    if (n_boxes > N_TMP_BOXES)
+	boxes32 = pixman_malloc_ab (n_boxes, sizeof (pixman_box32_t));
+    else
+	boxes32 = tmp_boxes;
+
+    if (!boxes32)
+	return FALSE;
+
+    for (i = 0; i < n_boxes; ++i)
+    {
+	boxes32[i].x1 = boxes16[i].x1;
+	boxes32[i].y1 = boxes16[i].y1;
+	boxes32[i].x2 = boxes16[i].x2;
+	boxes32[i].y2 = boxes16[i].y2;
+    }
+
+    pixman_region32_fini (dst);
+    retval = pixman_region32_init_rects (dst, boxes32, n_boxes);
+
+    if (boxes32 != tmp_boxes)
+	free (boxes32);
+
+    return retval;
+}
+
+#ifdef DEBUG
+
+void
+_pixman_log_error (const char *function, const char *message)
+{
+    static int n_messages = 0;
+
+    if (n_messages < 10)
+    {
+	fprintf (stderr,
+		 "*** BUG ***\n"
+		 "In %s: %s\n"
+		 "Set a breakpoint on '_pixman_log_error' to debug\n\n",
+                 function, message);
+
+	n_messages++;
+    }
+}
+
+#endif
diff --git a/pixman/pixman/pixman-version.h.in b/pixman/pixman/pixman-version.h.in
new file mode 100644
index 000000000..256b2e6f1
--- /dev/null
+++ b/pixman/pixman/pixman-version.h.in
@@ -0,0 +1,50 @@
+/*
+ * Copyright © 2008 Red Hat, Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use, copy,
+ * modify, merge, publish, distribute, sublicense, and/or sell copies
+ * of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ * Author: Carl D. Worth <cworth@cworth.org>
+ */
+
+#ifndef PIXMAN_VERSION_H__
+#define PIXMAN_VERSION_H__
+
+#ifndef PIXMAN_H__
+#  error pixman-version.h should only be included by pixman.h
+#endif
+
+#define PIXMAN_VERSION_MAJOR @PIXMAN_VERSION_MAJOR@
+#define PIXMAN_VERSION_MINOR @PIXMAN_VERSION_MINOR@
+#define PIXMAN_VERSION_MICRO @PIXMAN_VERSION_MICRO@
+
+#define PIXMAN_VERSION_STRING "@PIXMAN_VERSION_MAJOR@.@PIXMAN_VERSION_MINOR@.@PIXMAN_VERSION_MICRO@"
+
+#define PIXMAN_VERSION_ENCODE(major, minor, micro) (	\
+	  ((major) * 10000)				\
+	+ ((minor) *   100)				\
+	+ ((micro) *     1))
+
+#define PIXMAN_VERSION PIXMAN_VERSION_ENCODE(	\
+	PIXMAN_VERSION_MAJOR,			\
+	PIXMAN_VERSION_MINOR,			\
+	PIXMAN_VERSION_MICRO)
+
+#endif /* PIXMAN_VERSION_H__ */
diff --git a/pixman/pixman/pixman-vmx.c b/pixman/pixman/pixman-vmx.c
new file mode 100644
index 000000000..e811cf733
--- /dev/null
+++ b/pixman/pixman/pixman-vmx.c
@@ -0,0 +1,1648 @@
+/*
+ * Copyright © 2007 Luca Barbato
+ *
+ * Permission to use, copy, modify, distribute, and sell this software and its
+ * documentation for any purpose is hereby granted without fee, provided that
+ * the above copyright notice appear in all copies and that both that
+ * copyright notice and this permission notice appear in supporting
+ * documentation, and that the name of Luca Barbato not be used in advertising or
+ * publicity pertaining to distribution of the software without specific,
+ * written prior permission.  Luca Barbato makes no representations about the
+ * suitability of this software for any purpose.  It is provided "as is"
+ * without express or implied warranty.
+ *
+ * THE COPYRIGHT HOLDERS DISCLAIM ALL WARRANTIES WITH REGARD TO THIS
+ * SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
+ * FITNESS, IN NO EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
+ * AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
+ * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS
+ * SOFTWARE.
+ *
+ * Author:  Luca Barbato (lu_zero@gentoo.org)
+ *
+ * Based on fbmmx.c by Owen Taylor, Søren Sandmann and Nicholas Miell
+ */
+
+#include <config.h>
+#include "pixman-private.h"
+#include "pixman-combine32.h"
+#include <altivec.h>
+
+#define AVV(x...) {x}
+
+static force_inline vector unsigned int
+splat_alpha (vector unsigned int pix)
+{
+    return vec_perm (pix, pix,
+		     (vector unsigned char)AVV (
+			 0x00, 0x00, 0x00, 0x00, 0x04, 0x04, 0x04, 0x04,
+			 0x08, 0x08, 0x08, 0x08, 0x0C, 0x0C, 0x0C, 0x0C));
+}
+
+static force_inline vector unsigned int
+pix_multiply (vector unsigned int p, vector unsigned int a)
+{
+    vector unsigned short hi, lo, mod;
+
+    /* unpack to short */
+    hi = (vector unsigned short)
+	vec_mergeh ((vector unsigned char)AVV (0),
+		    (vector unsigned char)p);
+
+    mod = (vector unsigned short)
+	vec_mergeh ((vector unsigned char)AVV (0),
+		    (vector unsigned char)a);
+
+    hi = vec_mladd (hi, mod, (vector unsigned short)
+                    AVV (0x0080, 0x0080, 0x0080, 0x0080,
+                         0x0080, 0x0080, 0x0080, 0x0080));
+
+    hi = vec_adds (hi, vec_sr (hi, vec_splat_u16 (8)));
+
+    hi = vec_sr (hi, vec_splat_u16 (8));
+
+    /* unpack to short */
+    lo = (vector unsigned short)
+	vec_mergel ((vector unsigned char)AVV (0),
+		    (vector unsigned char)p);
+    mod = (vector unsigned short)
+	vec_mergel ((vector unsigned char)AVV (0),
+		    (vector unsigned char)a);
+
+    lo = vec_mladd (lo, mod, (vector unsigned short)
+                    AVV (0x0080, 0x0080, 0x0080, 0x0080,
+                         0x0080, 0x0080, 0x0080, 0x0080));
+
+    lo = vec_adds (lo, vec_sr (lo, vec_splat_u16 (8)));
+
+    lo = vec_sr (lo, vec_splat_u16 (8));
+
+    return (vector unsigned int)vec_packsu (hi, lo);
+}
+
+static force_inline vector unsigned int
+pix_add (vector unsigned int a, vector unsigned int b)
+{
+    return (vector unsigned int)vec_adds ((vector unsigned char)a,
+                                          (vector unsigned char)b);
+}
+
+static force_inline vector unsigned int
+pix_add_mul (vector unsigned int x,
+             vector unsigned int a,
+             vector unsigned int y,
+             vector unsigned int b)
+{
+    vector unsigned int t1, t2;
+
+    t1 = pix_multiply (x, a);
+    t2 = pix_multiply (y, b);
+
+    return pix_add (t1, t2);
+}
+
+static force_inline vector unsigned int
+negate (vector unsigned int src)
+{
+    return vec_nor (src, src);
+}
+
+/* dest*~srca + src */
+static force_inline vector unsigned int
+over (vector unsigned int src,
+      vector unsigned int srca,
+      vector unsigned int dest)
+{
+    vector unsigned char tmp = (vector unsigned char)
+	pix_multiply (dest, negate (srca));
+
+    tmp = vec_adds ((vector unsigned char)src, tmp);
+    return (vector unsigned int)tmp;
+}
+
+/* in == pix_multiply */
+#define in_over(src, srca, mask, dest)					\
+    over (pix_multiply (src, mask),					\
+          pix_multiply (srca, mask), dest)
+
+
+#define COMPUTE_SHIFT_MASK(source)					\
+    source ## _mask = vec_lvsl (0, source);
+
+#define COMPUTE_SHIFT_MASKS(dest, source)				\
+    dest ## _mask = vec_lvsl (0, dest);					\
+    source ## _mask = vec_lvsl (0, source);				\
+    store_mask = vec_lvsr (0, dest);
+
+#define COMPUTE_SHIFT_MASKC(dest, source, mask)				\
+    mask ## _mask = vec_lvsl (0, mask);					\
+    dest ## _mask = vec_lvsl (0, dest);					\
+    source ## _mask = vec_lvsl (0, source);				\
+    store_mask = vec_lvsr (0, dest);
+
+/* notice you have to declare temp vars...
+ * Note: tmp3 and tmp4 must remain untouched!
+ */
+
+#define LOAD_VECTORS(dest, source)			  \
+    tmp1 = (typeof(tmp1))vec_ld (0, source);		  \
+    tmp2 = (typeof(tmp2))vec_ld (15, source);		  \
+    tmp3 = (typeof(tmp3))vec_ld (0, dest);		  \
+    v ## source = (typeof(v ## source))			  \
+	vec_perm (tmp1, tmp2, source ## _mask);		  \
+    tmp4 = (typeof(tmp4))vec_ld (15, dest);		  \
+    v ## dest = (typeof(v ## dest))			  \
+	vec_perm (tmp3, tmp4, dest ## _mask);
+
+#define LOAD_VECTORSC(dest, source, mask)		  \
+    tmp1 = (typeof(tmp1))vec_ld (0, source);		  \
+    tmp2 = (typeof(tmp2))vec_ld (15, source);		  \
+    tmp3 = (typeof(tmp3))vec_ld (0, dest);		  \
+    v ## source = (typeof(v ## source))			  \
+	vec_perm (tmp1, tmp2, source ## _mask);		  \
+    tmp4 = (typeof(tmp4))vec_ld (15, dest);		  \
+    tmp1 = (typeof(tmp1))vec_ld (0, mask);		  \
+    v ## dest = (typeof(v ## dest))			  \
+	vec_perm (tmp3, tmp4, dest ## _mask);		  \
+    tmp2 = (typeof(tmp2))vec_ld (15, mask);		  \
+    v ## mask = (typeof(v ## mask))			  \
+	vec_perm (tmp1, tmp2, mask ## _mask);
+
+#define LOAD_VECTORSM(dest, source, mask)				\
+    LOAD_VECTORSC (dest, source, mask)					\
+    v ## source = pix_multiply (v ## source,				\
+                                splat_alpha (v ## mask));
+
+#define STORE_VECTOR(dest)						\
+    edges = vec_perm (tmp4, tmp3, dest ## _mask);			\
+    tmp3 = vec_perm ((vector unsigned char)v ## dest, edges, store_mask); \
+    tmp1 = vec_perm (edges, (vector unsigned char)v ## dest, store_mask); \
+    vec_st ((vector unsigned int) tmp3, 15, dest);			\
+    vec_st ((vector unsigned int) tmp1, 0, dest);
+
+static void
+vmx_combine_over_u_no_mask (uint32_t *      dest,
+                            const uint32_t *src,
+                            int             width)
+{
+    int i;
+    vector unsigned int vdest, vsrc;
+    vector unsigned char tmp1, tmp2, tmp3, tmp4, edges,
+	dest_mask, src_mask, store_mask;
+
+    COMPUTE_SHIFT_MASKS (dest, src);
+
+    /* printf ("%s\n",__PRETTY_FUNCTION__); */
+    for (i = width / 4; i > 0; i--)
+    {
+
+	LOAD_VECTORS (dest, src);
+
+	vdest = over (vsrc, splat_alpha (vsrc), vdest);
+
+	STORE_VECTOR (dest);
+
+	src += 4;
+	dest += 4;
+    }
+
+    for (i = width % 4; --i >= 0;)
+    {
+	uint32_t s = src[i];
+	uint32_t d = dest[i];
+	uint32_t ia = ALPHA_8 (~s);
+
+	UN8x4_MUL_UN8_ADD_UN8x4 (d, ia, s);
+
+	dest[i] = d;
+    }
+}
+
+static void
+vmx_combine_over_u_mask (uint32_t *      dest,
+                         const uint32_t *src,
+                         const uint32_t *mask,
+                         int             width)
+{
+    int i;
+    vector unsigned int vdest, vsrc, vmask;
+    vector unsigned char tmp1, tmp2, tmp3, tmp4, edges,
+	dest_mask, src_mask, mask_mask, store_mask;
+
+    COMPUTE_SHIFT_MASKC (dest, src, mask);
+
+    /* printf ("%s\n",__PRETTY_FUNCTION__); */
+    for (i = width / 4; i > 0; i--)
+    {
+	LOAD_VECTORSM (dest, src, mask);
+
+	vdest = over (vsrc, splat_alpha (vsrc), vdest);
+
+	STORE_VECTOR (dest);
+
+	src += 4;
+	dest += 4;
+	mask += 4;
+    }
+
+    for (i = width % 4; --i >= 0;)
+    {
+	uint32_t m = ALPHA_8 (mask[i]);
+	uint32_t s = src[i];
+	uint32_t d = dest[i];
+	uint32_t ia;
+
+	UN8x4_MUL_UN8 (s, m);
+
+	ia = ALPHA_8 (~s);
+
+	UN8x4_MUL_UN8_ADD_UN8x4 (d, ia, s);
+	dest[i] = d;
+    }
+}
+
+static void
+vmx_combine_over_u (pixman_implementation_t *imp,
+                    pixman_op_t              op,
+                    uint32_t *               dest,
+                    const uint32_t *         src,
+                    const uint32_t *         mask,
+                    int                      width)
+{
+    if (mask)
+	vmx_combine_over_u_mask (dest, src, mask, width);
+    else
+	vmx_combine_over_u_no_mask (dest, src, width);
+}
+
+static void
+vmx_combine_over_reverse_u_no_mask (uint32_t *      dest,
+                                    const uint32_t *src,
+                                    int             width)
+{
+    int i;
+    vector unsigned int vdest, vsrc;
+    vector unsigned char tmp1, tmp2, tmp3, tmp4, edges,
+	dest_mask, src_mask, store_mask;
+
+    COMPUTE_SHIFT_MASKS (dest, src);
+
+    /* printf ("%s\n",__PRETTY_FUNCTION__); */
+    for (i = width / 4; i > 0; i--)
+    {
+
+	LOAD_VECTORS (dest, src);
+
+	vdest = over (vdest, splat_alpha (vdest), vsrc);
+
+	STORE_VECTOR (dest);
+
+	src += 4;
+	dest += 4;
+    }
+
+    for (i = width % 4; --i >= 0;)
+    {
+	uint32_t s = src[i];
+	uint32_t d = dest[i];
+	uint32_t ia = ALPHA_8 (~dest[i]);
+
+	UN8x4_MUL_UN8_ADD_UN8x4 (s, ia, d);
+	dest[i] = s;
+    }
+}
+
+static void
+vmx_combine_over_reverse_u_mask (uint32_t *      dest,
+                                 const uint32_t *src,
+                                 const uint32_t *mask,
+                                 int             width)
+{
+    int i;
+    vector unsigned int vdest, vsrc, vmask;
+    vector unsigned char tmp1, tmp2, tmp3, tmp4, edges,
+	dest_mask, src_mask, mask_mask, store_mask;
+
+    COMPUTE_SHIFT_MASKC (dest, src, mask);
+
+    /* printf ("%s\n",__PRETTY_FUNCTION__); */
+    for (i = width / 4; i > 0; i--)
+    {
+
+	LOAD_VECTORSM (dest, src, mask);
+
+	vdest = over (vdest, splat_alpha (vdest), vsrc);
+
+	STORE_VECTOR (dest);
+
+	src += 4;
+	dest += 4;
+	mask += 4;
+    }
+
+    for (i = width % 4; --i >= 0;)
+    {
+	uint32_t m = ALPHA_8 (mask[i]);
+	uint32_t s = src[i];
+	uint32_t d = dest[i];
+	uint32_t ia = ALPHA_8 (~dest[i]);
+
+	UN8x4_MUL_UN8 (s, m);
+
+	UN8x4_MUL_UN8_ADD_UN8x4 (s, ia, d);
+	dest[i] = s;
+    }
+}
+
+static void
+vmx_combine_over_reverse_u (pixman_implementation_t *imp,
+                            pixman_op_t              op,
+                            uint32_t *               dest,
+                            const uint32_t *         src,
+                            const uint32_t *         mask,
+                            int                      width)
+{
+    if (mask)
+	vmx_combine_over_reverse_u_mask (dest, src, mask, width);
+    else
+	vmx_combine_over_reverse_u_no_mask (dest, src, width);
+}
+
+static void
+vmx_combine_in_u_no_mask (uint32_t *      dest,
+                          const uint32_t *src,
+                          int             width)
+{
+    int i;
+    vector unsigned int vdest, vsrc;
+    vector unsigned char tmp1, tmp2, tmp3, tmp4, edges,
+	dest_mask, src_mask, store_mask;
+
+    COMPUTE_SHIFT_MASKS (dest, src);
+
+    /* printf ("%s\n",__PRETTY_FUNCTION__); */
+    for (i = width / 4; i > 0; i--)
+    {
+	LOAD_VECTORS (dest, src);
+
+	vdest = pix_multiply (vsrc, splat_alpha (vdest));
+
+	STORE_VECTOR (dest);
+
+	src += 4;
+	dest += 4;
+    }
+
+    for (i = width % 4; --i >= 0;)
+    {
+	uint32_t s = src[i];
+	uint32_t a = ALPHA_8 (dest[i]);
+
+	UN8x4_MUL_UN8 (s, a);
+	dest[i] = s;
+    }
+}
+
+static void
+vmx_combine_in_u_mask (uint32_t *      dest,
+                       const uint32_t *src,
+                       const uint32_t *mask,
+                       int             width)
+{
+    int i;
+    vector unsigned int vdest, vsrc, vmask;
+    vector unsigned char tmp1, tmp2, tmp3, tmp4, edges,
+	dest_mask, src_mask, mask_mask, store_mask;
+
+    COMPUTE_SHIFT_MASKC (dest, src, mask);
+
+    /* printf ("%s\n",__PRETTY_FUNCTION__); */
+    for (i = width / 4; i > 0; i--)
+    {
+	LOAD_VECTORSM (dest, src, mask);
+
+	vdest = pix_multiply (vsrc, splat_alpha (vdest));
+
+	STORE_VECTOR (dest);
+
+	src += 4;
+	dest += 4;
+	mask += 4;
+    }
+
+    for (i = width % 4; --i >= 0;)
+    {
+	uint32_t m = ALPHA_8 (mask[i]);
+	uint32_t s = src[i];
+	uint32_t a = ALPHA_8 (dest[i]);
+
+	UN8x4_MUL_UN8 (s, m);
+	UN8x4_MUL_UN8 (s, a);
+
+	dest[i] = s;
+    }
+}
+
+static void
+vmx_combine_in_u (pixman_implementation_t *imp,
+                  pixman_op_t              op,
+                  uint32_t *               dest,
+                  const uint32_t *         src,
+                  const uint32_t *         mask,
+                  int                      width)
+{
+    if (mask)
+	vmx_combine_in_u_mask (dest, src, mask, width);
+    else
+	vmx_combine_in_u_no_mask (dest, src, width);
+}
+
+static void
+vmx_combine_in_reverse_u_no_mask (uint32_t *      dest,
+                                  const uint32_t *src,
+                                  int             width)
+{
+    int i;
+    vector unsigned int vdest, vsrc;
+    vector unsigned char tmp1, tmp2, tmp3, tmp4, edges,
+	dest_mask, src_mask, store_mask;
+
+    COMPUTE_SHIFT_MASKS (dest, src);
+
+    /* printf ("%s\n",__PRETTY_FUNCTION__); */
+    for (i = width / 4; i > 0; i--)
+    {
+	LOAD_VECTORS (dest, src);
+
+	vdest = pix_multiply (vdest, splat_alpha (vsrc));
+
+	STORE_VECTOR (dest);
+
+	src += 4;
+	dest += 4;
+    }
+
+    for (i = width % 4; --i >= 0;)
+    {
+	uint32_t d = dest[i];
+	uint32_t a = ALPHA_8 (src[i]);
+
+	UN8x4_MUL_UN8 (d, a);
+
+	dest[i] = d;
+    }
+}
+
+static void
+vmx_combine_in_reverse_u_mask (uint32_t *      dest,
+                               const uint32_t *src,
+                               const uint32_t *mask,
+                               int             width)
+{
+    int i;
+    vector unsigned int vdest, vsrc, vmask;
+    vector unsigned char tmp1, tmp2, tmp3, tmp4, edges,
+	dest_mask, src_mask, mask_mask, store_mask;
+
+    COMPUTE_SHIFT_MASKC (dest, src, mask);
+
+    /* printf ("%s\n",__PRETTY_FUNCTION__); */
+    for (i = width / 4; i > 0; i--)
+    {
+	LOAD_VECTORSM (dest, src, mask);
+
+	vdest = pix_multiply (vdest, splat_alpha (vsrc));
+
+	STORE_VECTOR (dest);
+
+	src += 4;
+	dest += 4;
+	mask += 4;
+    }
+
+    for (i = width % 4; --i >= 0;)
+    {
+	uint32_t m = ALPHA_8 (mask[i]);
+	uint32_t d = dest[i];
+	uint32_t a = src[i];
+
+	UN8x4_MUL_UN8 (a, m);
+	a = ALPHA_8 (a);
+	UN8x4_MUL_UN8 (d, a);
+
+	dest[i] = d;
+    }
+}
+
+static void
+vmx_combine_in_reverse_u (pixman_implementation_t *imp,
+                          pixman_op_t              op,
+                          uint32_t *               dest,
+                          const uint32_t *         src,
+                          const uint32_t *         mask,
+                          int                      width)
+{
+    if (mask)
+	vmx_combine_in_reverse_u_mask (dest, src, mask, width);
+    else
+	vmx_combine_in_reverse_u_no_mask (dest, src, width);
+}
+
+static void
+vmx_combine_out_u_no_mask (uint32_t *      dest,
+                           const uint32_t *src,
+                           int             width)
+{
+    int i;
+    vector unsigned int vdest, vsrc;
+    vector unsigned char tmp1, tmp2, tmp3, tmp4, edges,
+	dest_mask, src_mask, store_mask;
+
+    COMPUTE_SHIFT_MASKS (dest, src);
+
+    /* printf ("%s\n",__PRETTY_FUNCTION__); */
+    for (i = width / 4; i > 0; i--)
+    {
+	LOAD_VECTORS (dest, src);
+
+	vdest = pix_multiply (vsrc, splat_alpha (negate (vdest)));
+
+	STORE_VECTOR (dest);
+
+	src += 4;
+	dest += 4;
+    }
+
+    for (i = width % 4; --i >= 0;)
+    {
+	uint32_t s = src[i];
+	uint32_t a = ALPHA_8 (~dest[i]);
+
+	UN8x4_MUL_UN8 (s, a);
+
+	dest[i] = s;
+    }
+}
+
+static void
+vmx_combine_out_u_mask (uint32_t *      dest,
+                        const uint32_t *src,
+                        const uint32_t *mask,
+                        int             width)
+{
+    int i;
+    vector unsigned int vdest, vsrc, vmask;
+    vector unsigned char tmp1, tmp2, tmp3, tmp4, edges,
+	dest_mask, src_mask, mask_mask, store_mask;
+
+    COMPUTE_SHIFT_MASKC (dest, src, mask);
+
+    /* printf ("%s\n",__PRETTY_FUNCTION__); */
+    for (i = width / 4; i > 0; i--)
+    {
+	LOAD_VECTORSM (dest, src, mask);
+
+	vdest = pix_multiply (vsrc, splat_alpha (negate (vdest)));
+
+	STORE_VECTOR (dest);
+
+	src += 4;
+	dest += 4;
+	mask += 4;
+    }
+
+    for (i = width % 4; --i >= 0;)
+    {
+	uint32_t m = ALPHA_8 (mask[i]);
+	uint32_t s = src[i];
+	uint32_t a = ALPHA_8 (~dest[i]);
+
+	UN8x4_MUL_UN8 (s, m);
+	UN8x4_MUL_UN8 (s, a);
+
+	dest[i] = s;
+    }
+}
+
+static void
+vmx_combine_out_u (pixman_implementation_t *imp,
+                   pixman_op_t              op,
+                   uint32_t *               dest,
+                   const uint32_t *         src,
+                   const uint32_t *         mask,
+                   int                      width)
+{
+    if (mask)
+	vmx_combine_out_u_mask (dest, src, mask, width);
+    else
+	vmx_combine_out_u_no_mask (dest, src, width);
+}
+
+static void
+vmx_combine_out_reverse_u_no_mask (uint32_t *      dest,
+                                   const uint32_t *src,
+                                   int             width)
+{
+    int i;
+    vector unsigned int vdest, vsrc;
+    vector unsigned char tmp1, tmp2, tmp3, tmp4, edges,
+	dest_mask, src_mask, store_mask;
+
+    COMPUTE_SHIFT_MASKS (dest, src);
+
+    /* printf ("%s\n",__PRETTY_FUNCTION__); */
+    for (i = width / 4; i > 0; i--)
+    {
+
+	LOAD_VECTORS (dest, src);
+
+	vdest = pix_multiply (vdest, splat_alpha (negate (vsrc)));
+
+	STORE_VECTOR (dest);
+
+	src += 4;
+	dest += 4;
+    }
+
+    for (i = width % 4; --i >= 0;)
+    {
+	uint32_t d = dest[i];
+	uint32_t a = ALPHA_8 (~src[i]);
+
+	UN8x4_MUL_UN8 (d, a);
+
+	dest[i] = d;
+    }
+}
+
+static void
+vmx_combine_out_reverse_u_mask (uint32_t *      dest,
+                                const uint32_t *src,
+                                const uint32_t *mask,
+                                int             width)
+{
+    int i;
+    vector unsigned int vdest, vsrc, vmask;
+    vector unsigned char tmp1, tmp2, tmp3, tmp4, edges,
+	dest_mask, src_mask, mask_mask, store_mask;
+
+    COMPUTE_SHIFT_MASKC (dest, src, mask);
+
+    /* printf ("%s\n",__PRETTY_FUNCTION__); */
+    for (i = width / 4; i > 0; i--)
+    {
+	LOAD_VECTORSM (dest, src, mask);
+
+	vdest = pix_multiply (vdest, splat_alpha (negate (vsrc)));
+
+	STORE_VECTOR (dest);
+
+	src += 4;
+	dest += 4;
+	mask += 4;
+    }
+
+    for (i = width % 4; --i >= 0;)
+    {
+	uint32_t m = ALPHA_8 (mask[i]);
+	uint32_t d = dest[i];
+	uint32_t a = src[i];
+
+	UN8x4_MUL_UN8 (a, m);
+	a = ALPHA_8 (~a);
+	UN8x4_MUL_UN8 (d, a);
+
+	dest[i] = d;
+    }
+}
+
+static void
+vmx_combine_out_reverse_u (pixman_implementation_t *imp,
+                           pixman_op_t              op,
+                           uint32_t *               dest,
+                           const uint32_t *         src,
+                           const uint32_t *         mask,
+                           int                      width)
+{
+    if (mask)
+	vmx_combine_out_reverse_u_mask (dest, src, mask, width);
+    else
+	vmx_combine_out_reverse_u_no_mask (dest, src, width);
+}
+
+static void
+vmx_combine_atop_u_no_mask (uint32_t *      dest,
+                            const uint32_t *src,
+                            int             width)
+{
+    int i;
+    vector unsigned int vdest, vsrc;
+    vector unsigned char tmp1, tmp2, tmp3, tmp4, edges,
+	dest_mask, src_mask, store_mask;
+
+    COMPUTE_SHIFT_MASKS (dest, src);
+
+    /* printf ("%s\n",__PRETTY_FUNCTION__); */
+    for (i = width / 4; i > 0; i--)
+    {
+	LOAD_VECTORS (dest, src);
+
+	vdest = pix_add_mul (vsrc, splat_alpha (vdest),
+			     vdest, splat_alpha (negate (vsrc)));
+
+	STORE_VECTOR (dest);
+
+	src += 4;
+	dest += 4;
+    }
+
+    for (i = width % 4; --i >= 0;)
+    {
+	uint32_t s = src[i];
+	uint32_t d = dest[i];
+	uint32_t dest_a = ALPHA_8 (d);
+	uint32_t src_ia = ALPHA_8 (~s);
+
+	UN8x4_MUL_UN8_ADD_UN8x4_MUL_UN8 (s, dest_a, d, src_ia);
+
+	dest[i] = s;
+    }
+}
+
+static void
+vmx_combine_atop_u_mask (uint32_t *      dest,
+                         const uint32_t *src,
+                         const uint32_t *mask,
+                         int             width)
+{
+    int i;
+    vector unsigned int vdest, vsrc, vmask;
+    vector unsigned char tmp1, tmp2, tmp3, tmp4, edges,
+	dest_mask, src_mask, mask_mask, store_mask;
+
+    COMPUTE_SHIFT_MASKC (dest, src, mask);
+
+    /* printf ("%s\n",__PRETTY_FUNCTION__); */
+    for (i = width / 4; i > 0; i--)
+    {
+	LOAD_VECTORSM (dest, src, mask);
+
+	vdest = pix_add_mul (vsrc, splat_alpha (vdest),
+			     vdest, splat_alpha (negate (vsrc)));
+
+	STORE_VECTOR (dest);
+
+	src += 4;
+	dest += 4;
+	mask += 4;
+    }
+
+    for (i = width % 4; --i >= 0;)
+    {
+	uint32_t m = ALPHA_8 (mask[i]);
+	uint32_t s = src[i];
+	uint32_t d = dest[i];
+	uint32_t dest_a = ALPHA_8 (d);
+	uint32_t src_ia;
+
+	UN8x4_MUL_UN8 (s, m);
+
+	src_ia = ALPHA_8 (~s);
+
+	UN8x4_MUL_UN8_ADD_UN8x4_MUL_UN8 (s, dest_a, d, src_ia);
+
+	dest[i] = s;
+    }
+}
+
+static void
+vmx_combine_atop_u (pixman_implementation_t *imp,
+                    pixman_op_t              op,
+                    uint32_t *               dest,
+                    const uint32_t *         src,
+                    const uint32_t *         mask,
+                    int                      width)
+{
+    if (mask)
+	vmx_combine_atop_u_mask (dest, src, mask, width);
+    else
+	vmx_combine_atop_u_no_mask (dest, src, width);
+}
+
+static void
+vmx_combine_atop_reverse_u_no_mask (uint32_t *      dest,
+                                    const uint32_t *src,
+                                    int             width)
+{
+    int i;
+    vector unsigned int vdest, vsrc;
+    vector unsigned char tmp1, tmp2, tmp3, tmp4, edges,
+	dest_mask, src_mask, store_mask;
+
+    COMPUTE_SHIFT_MASKS (dest, src);
+
+    /* printf ("%s\n",__PRETTY_FUNCTION__); */
+    for (i = width / 4; i > 0; i--)
+    {
+	LOAD_VECTORS (dest, src);
+
+	vdest = pix_add_mul (vdest, splat_alpha (vsrc),
+			     vsrc, splat_alpha (negate (vdest)));
+
+	STORE_VECTOR (dest);
+
+	src += 4;
+	dest += 4;
+    }
+
+    for (i = width % 4; --i >= 0;)
+    {
+	uint32_t s = src[i];
+	uint32_t d = dest[i];
+	uint32_t src_a = ALPHA_8 (s);
+	uint32_t dest_ia = ALPHA_8 (~d);
+
+	UN8x4_MUL_UN8_ADD_UN8x4_MUL_UN8 (s, dest_ia, d, src_a);
+
+	dest[i] = s;
+    }
+}
+
+static void
+vmx_combine_atop_reverse_u_mask (uint32_t *      dest,
+                                 const uint32_t *src,
+                                 const uint32_t *mask,
+                                 int             width)
+{
+    int i;
+    vector unsigned int vdest, vsrc, vmask;
+    vector unsigned char tmp1, tmp2, tmp3, tmp4, edges,
+	dest_mask, src_mask, mask_mask, store_mask;
+
+    COMPUTE_SHIFT_MASKC (dest, src, mask);
+
+    /* printf ("%s\n",__PRETTY_FUNCTION__); */
+    for (i = width / 4; i > 0; i--)
+    {
+	LOAD_VECTORSM (dest, src, mask);
+
+	vdest = pix_add_mul (vdest, splat_alpha (vsrc),
+			     vsrc, splat_alpha (negate (vdest)));
+
+	STORE_VECTOR (dest);
+
+	src += 4;
+	dest += 4;
+	mask += 4;
+    }
+
+    for (i = width % 4; --i >= 0;)
+    {
+	uint32_t m = ALPHA_8 (mask[i]);
+	uint32_t s = src[i];
+	uint32_t d = dest[i];
+	uint32_t src_a;
+	uint32_t dest_ia = ALPHA_8 (~d);
+
+	UN8x4_MUL_UN8 (s, m);
+
+	src_a = ALPHA_8 (s);
+
+	UN8x4_MUL_UN8_ADD_UN8x4_MUL_UN8 (s, dest_ia, d, src_a);
+
+	dest[i] = s;
+    }
+}
+
+static void
+vmx_combine_atop_reverse_u (pixman_implementation_t *imp,
+                            pixman_op_t              op,
+                            uint32_t *               dest,
+                            const uint32_t *         src,
+                            const uint32_t *         mask,
+                            int                      width)
+{
+    if (mask)
+	vmx_combine_atop_reverse_u_mask (dest, src, mask, width);
+    else
+	vmx_combine_atop_reverse_u_no_mask (dest, src, width);
+}
+
+static void
+vmx_combine_xor_u_no_mask (uint32_t *      dest,
+                           const uint32_t *src,
+                           int             width)
+{
+    int i;
+    vector unsigned int vdest, vsrc;
+    vector unsigned char tmp1, tmp2, tmp3, tmp4, edges,
+	dest_mask, src_mask, store_mask;
+
+    COMPUTE_SHIFT_MASKS (dest, src);
+
+    /* printf ("%s\n",__PRETTY_FUNCTION__); */
+    for (i = width / 4; i > 0; i--)
+    {
+	LOAD_VECTORS (dest, src);
+
+	vdest = pix_add_mul (vsrc, splat_alpha (negate (vdest)),
+			     vdest, splat_alpha (negate (vsrc)));
+
+	STORE_VECTOR (dest);
+
+	src += 4;
+	dest += 4;
+    }
+
+    for (i = width % 4; --i >= 0;)
+    {
+	uint32_t s = src[i];
+	uint32_t d = dest[i];
+	uint32_t src_ia = ALPHA_8 (~s);
+	uint32_t dest_ia = ALPHA_8 (~d);
+
+	UN8x4_MUL_UN8_ADD_UN8x4_MUL_UN8 (s, dest_ia, d, src_ia);
+
+	dest[i] = s;
+    }
+}
+
+static void
+vmx_combine_xor_u_mask (uint32_t *      dest,
+                        const uint32_t *src,
+                        const uint32_t *mask,
+                        int             width)
+{
+    int i;
+    vector unsigned int vdest, vsrc, vmask;
+    vector unsigned char tmp1, tmp2, tmp3, tmp4, edges,
+	dest_mask, src_mask, mask_mask, store_mask;
+
+    COMPUTE_SHIFT_MASKC (dest, src, mask);
+
+    /* printf ("%s\n",__PRETTY_FUNCTION__); */
+    for (i = width / 4; i > 0; i--)
+    {
+	LOAD_VECTORSM (dest, src, mask);
+
+	vdest = pix_add_mul (vsrc, splat_alpha (negate (vdest)),
+			     vdest, splat_alpha (negate (vsrc)));
+
+	STORE_VECTOR (dest);
+
+	src += 4;
+	dest += 4;
+	mask += 4;
+    }
+
+    for (i = width % 4; --i >= 0;)
+    {
+	uint32_t m = ALPHA_8 (mask[i]);
+	uint32_t s = src[i];
+	uint32_t d = dest[i];
+	uint32_t src_ia;
+	uint32_t dest_ia = ALPHA_8 (~d);
+
+	UN8x4_MUL_UN8 (s, m);
+
+	src_ia = ALPHA_8 (~s);
+
+	UN8x4_MUL_UN8_ADD_UN8x4_MUL_UN8 (s, dest_ia, d, src_ia);
+
+	dest[i] = s;
+    }
+}
+
+static void
+vmx_combine_xor_u (pixman_implementation_t *imp,
+                   pixman_op_t              op,
+                   uint32_t *               dest,
+                   const uint32_t *         src,
+                   const uint32_t *         mask,
+                   int                      width)
+{
+    if (mask)
+	vmx_combine_xor_u_mask (dest, src, mask, width);
+    else
+	vmx_combine_xor_u_no_mask (dest, src, width);
+}
+
+static void
+vmx_combine_add_u_no_mask (uint32_t *      dest,
+                           const uint32_t *src,
+                           int             width)
+{
+    int i;
+    vector unsigned int vdest, vsrc;
+    vector unsigned char tmp1, tmp2, tmp3, tmp4, edges,
+	dest_mask, src_mask, store_mask;
+
+    COMPUTE_SHIFT_MASKS (dest, src);
+    /* printf ("%s\n",__PRETTY_FUNCTION__); */
+    for (i = width / 4; i > 0; i--)
+    {
+	LOAD_VECTORS (dest, src);
+
+	vdest = pix_add (vsrc, vdest);
+
+	STORE_VECTOR (dest);
+
+	src += 4;
+	dest += 4;
+    }
+
+    for (i = width % 4; --i >= 0;)
+    {
+	uint32_t s = src[i];
+	uint32_t d = dest[i];
+
+	UN8x4_ADD_UN8x4 (d, s);
+
+	dest[i] = d;
+    }
+}
+
+static void
+vmx_combine_add_u_mask (uint32_t *      dest,
+                        const uint32_t *src,
+                        const uint32_t *mask,
+                        int             width)
+{
+    int i;
+    vector unsigned int vdest, vsrc, vmask;
+    vector unsigned char tmp1, tmp2, tmp3, tmp4, edges,
+	dest_mask, src_mask, mask_mask, store_mask;
+
+    COMPUTE_SHIFT_MASKC (dest, src, mask);
+
+    /* printf ("%s\n",__PRETTY_FUNCTION__); */
+    for (i = width / 4; i > 0; i--)
+    {
+	LOAD_VECTORSM (dest, src, mask);
+
+	vdest = pix_add (vsrc, vdest);
+
+	STORE_VECTOR (dest);
+
+	src += 4;
+	dest += 4;
+	mask += 4;
+    }
+
+    for (i = width % 4; --i >= 0;)
+    {
+	uint32_t m = ALPHA_8 (mask[i]);
+	uint32_t s = src[i];
+	uint32_t d = dest[i];
+
+	UN8x4_MUL_UN8 (s, m);
+	UN8x4_ADD_UN8x4 (d, s);
+
+	dest[i] = d;
+    }
+}
+
+static void
+vmx_combine_add_u (pixman_implementation_t *imp,
+                   pixman_op_t              op,
+                   uint32_t *               dest,
+                   const uint32_t *         src,
+                   const uint32_t *         mask,
+                   int                      width)
+{
+    if (mask)
+	vmx_combine_add_u_mask (dest, src, mask, width);
+    else
+	vmx_combine_add_u_no_mask (dest, src, width);
+}
+
+static void
+vmx_combine_src_ca (pixman_implementation_t *imp,
+                    pixman_op_t              op,
+                    uint32_t *               dest,
+                    const uint32_t *         src,
+                    const uint32_t *         mask,
+                    int                      width)
+{
+    int i;
+    vector unsigned int vdest, vsrc, vmask;
+    vector unsigned char tmp1, tmp2, tmp3, tmp4, edges,
+	dest_mask, mask_mask, src_mask, store_mask;
+
+    COMPUTE_SHIFT_MASKC (dest, src, mask);
+
+    /* printf ("%s\n",__PRETTY_FUNCTION__); */
+    for (i = width / 4; i > 0; i--)
+    {
+	LOAD_VECTORSC (dest, src, mask);
+
+	vdest = pix_multiply (vsrc, vmask);
+
+	STORE_VECTOR (dest);
+
+	mask += 4;
+	src += 4;
+	dest += 4;
+    }
+
+    for (i = width % 4; --i >= 0;)
+    {
+	uint32_t a = mask[i];
+	uint32_t s = src[i];
+
+	UN8x4_MUL_UN8x4 (s, a);
+
+	dest[i] = s;
+    }
+}
+
+static void
+vmx_combine_over_ca (pixman_implementation_t *imp,
+                     pixman_op_t              op,
+                     uint32_t *               dest,
+                     const uint32_t *         src,
+                     const uint32_t *         mask,
+                     int                      width)
+{
+    int i;
+    vector unsigned int vdest, vsrc, vmask;
+    vector unsigned char tmp1, tmp2, tmp3, tmp4, edges,
+	dest_mask, mask_mask, src_mask, store_mask;
+
+    COMPUTE_SHIFT_MASKC (dest, src, mask);
+
+    /* printf ("%s\n",__PRETTY_FUNCTION__); */
+    for (i = width / 4; i > 0; i--)
+    {
+	LOAD_VECTORSC (dest, src, mask);
+
+	vdest = in_over (vsrc, splat_alpha (vsrc), vmask, vdest);
+
+	STORE_VECTOR (dest);
+
+	mask += 4;
+	src += 4;
+	dest += 4;
+    }
+
+    for (i = width % 4; --i >= 0;)
+    {
+	uint32_t a = mask[i];
+	uint32_t s = src[i];
+	uint32_t d = dest[i];
+	uint32_t sa = ALPHA_8 (s);
+
+	UN8x4_MUL_UN8x4 (s, a);
+	UN8x4_MUL_UN8 (a, sa);
+	UN8x4_MUL_UN8x4_ADD_UN8x4 (d, ~a, s);
+
+	dest[i] = d;
+    }
+}
+
+static void
+vmx_combine_over_reverse_ca (pixman_implementation_t *imp,
+                             pixman_op_t              op,
+                             uint32_t *               dest,
+                             const uint32_t *         src,
+                             const uint32_t *         mask,
+                             int                      width)
+{
+    int i;
+    vector unsigned int vdest, vsrc, vmask;
+    vector unsigned char tmp1, tmp2, tmp3, tmp4, edges,
+	dest_mask, mask_mask, src_mask, store_mask;
+
+    COMPUTE_SHIFT_MASKC (dest, src, mask);
+
+    /* printf("%s\n",__PRETTY_FUNCTION__); */
+    for (i = width / 4; i > 0; i--)
+    {
+	LOAD_VECTORSC (dest, src, mask);
+
+	vdest = over (vdest, splat_alpha (vdest), pix_multiply (vsrc, vmask));
+
+	STORE_VECTOR (dest);
+
+	mask += 4;
+	src += 4;
+	dest += 4;
+    }
+
+    for (i = width % 4; --i >= 0;)
+    {
+	uint32_t a = mask[i];
+	uint32_t s = src[i];
+	uint32_t d = dest[i];
+	uint32_t ida = ALPHA_8 (~d);
+
+	UN8x4_MUL_UN8x4 (s, a);
+	UN8x4_MUL_UN8_ADD_UN8x4 (s, ida, d);
+
+	dest[i] = s;
+    }
+}
+
+static void
+vmx_combine_in_ca (pixman_implementation_t *imp,
+                   pixman_op_t              op,
+                   uint32_t *               dest,
+                   const uint32_t *         src,
+                   const uint32_t *         mask,
+                   int                      width)
+{
+    int i;
+    vector unsigned int vdest, vsrc, vmask;
+    vector unsigned char tmp1, tmp2, tmp3, tmp4, edges,
+	dest_mask, mask_mask, src_mask, store_mask;
+
+    COMPUTE_SHIFT_MASKC (dest, src, mask);
+
+    /* printf ("%s\n",__PRETTY_FUNCTION__); */
+    for (i = width / 4; i > 0; i--)
+    {
+	LOAD_VECTORSC (dest, src, mask);
+
+	vdest = pix_multiply (pix_multiply (vsrc, vmask), splat_alpha (vdest));
+
+	STORE_VECTOR (dest);
+
+	src += 4;
+	dest += 4;
+	mask += 4;
+    }
+
+    for (i = width % 4; --i >= 0;)
+    {
+	uint32_t a = mask[i];
+	uint32_t s = src[i];
+	uint32_t da = ALPHA_8 (dest[i]);
+
+	UN8x4_MUL_UN8x4 (s, a);
+	UN8x4_MUL_UN8 (s, da);
+
+	dest[i] = s;
+    }
+}
+
+static void
+vmx_combine_in_reverse_ca (pixman_implementation_t *imp,
+                           pixman_op_t              op,
+                           uint32_t *               dest,
+                           const uint32_t *         src,
+                           const uint32_t *         mask,
+                           int                      width)
+{
+    int i;
+    vector unsigned int vdest, vsrc, vmask;
+    vector unsigned char tmp1, tmp2, tmp3, tmp4, edges,
+	dest_mask, mask_mask, src_mask, store_mask;
+
+    COMPUTE_SHIFT_MASKC (dest, src, mask);
+
+    /* printf ("%s\n",__PRETTY_FUNCTION__); */
+    for (i = width / 4; i > 0; i--)
+    {
+
+	LOAD_VECTORSC (dest, src, mask);
+
+	vdest = pix_multiply (vdest, pix_multiply (vmask, splat_alpha (vsrc)));
+
+	STORE_VECTOR (dest);
+
+	src += 4;
+	dest += 4;
+	mask += 4;
+    }
+
+    for (i = width % 4; --i >= 0;)
+    {
+	uint32_t a = mask[i];
+	uint32_t d = dest[i];
+	uint32_t sa = ALPHA_8 (src[i]);
+
+	UN8x4_MUL_UN8 (a, sa);
+	UN8x4_MUL_UN8x4 (d, a);
+
+	dest[i] = d;
+    }
+}
+
+static void
+vmx_combine_out_ca (pixman_implementation_t *imp,
+                    pixman_op_t              op,
+                    uint32_t *               dest,
+                    const uint32_t *         src,
+                    const uint32_t *         mask,
+                    int                      width)
+{
+    int i;
+    vector unsigned int vdest, vsrc, vmask;
+    vector unsigned char tmp1, tmp2, tmp3, tmp4, edges,
+	dest_mask, mask_mask, src_mask, store_mask;
+
+    COMPUTE_SHIFT_MASKC (dest, src, mask);
+
+    /* printf ("%s\n",__PRETTY_FUNCTION__); */
+    for (i = width / 4; i > 0; i--)
+    {
+	LOAD_VECTORSC (dest, src, mask);
+
+	vdest = pix_multiply (
+	    pix_multiply (vsrc, vmask), splat_alpha (negate (vdest)));
+
+	STORE_VECTOR (dest);
+
+	src += 4;
+	dest += 4;
+	mask += 4;
+    }
+
+    for (i = width % 4; --i >= 0;)
+    {
+	uint32_t a = mask[i];
+	uint32_t s = src[i];
+	uint32_t d = dest[i];
+	uint32_t da = ALPHA_8 (~d);
+
+	UN8x4_MUL_UN8x4 (s, a);
+	UN8x4_MUL_UN8 (s, da);
+
+	dest[i] = s;
+    }
+}
+
+static void
+vmx_combine_out_reverse_ca (pixman_implementation_t *imp,
+                            pixman_op_t              op,
+                            uint32_t *               dest,
+                            const uint32_t *         src,
+                            const uint32_t *         mask,
+                            int                      width)
+{
+    int i;
+    vector unsigned int vdest, vsrc, vmask;
+    vector unsigned char tmp1, tmp2, tmp3, tmp4, edges,
+	dest_mask, mask_mask, src_mask, store_mask;
+
+    COMPUTE_SHIFT_MASKC (dest, src, mask);
+
+    /* printf ("%s\n",__PRETTY_FUNCTION__); */
+    for (i = width / 4; i > 0; i--)
+    {
+	LOAD_VECTORSC (dest, src, mask);
+
+	vdest = pix_multiply (
+	    vdest, negate (pix_multiply (vmask, splat_alpha (vsrc))));
+
+	STORE_VECTOR (dest);
+
+	src += 4;
+	dest += 4;
+	mask += 4;
+    }
+
+    for (i = width % 4; --i >= 0;)
+    {
+	uint32_t a = mask[i];
+	uint32_t s = src[i];
+	uint32_t d = dest[i];
+	uint32_t sa = ALPHA_8 (s);
+
+	UN8x4_MUL_UN8 (a, sa);
+	UN8x4_MUL_UN8x4 (d, ~a);
+
+	dest[i] = d;
+    }
+}
+
+static void
+vmx_combine_atop_ca (pixman_implementation_t *imp,
+                     pixman_op_t              op,
+                     uint32_t *               dest,
+                     const uint32_t *         src,
+                     const uint32_t *         mask,
+                     int                      width)
+{
+    int i;
+    vector unsigned int vdest, vsrc, vmask, vsrca;
+    vector unsigned char tmp1, tmp2, tmp3, tmp4, edges,
+	dest_mask, mask_mask, src_mask, store_mask;
+
+    COMPUTE_SHIFT_MASKC (dest, src, mask);
+
+    /* printf ("%s\n",__PRETTY_FUNCTION__); */
+    for (i = width / 4; i > 0; i--)
+    {
+	LOAD_VECTORSC (dest, src, mask);
+
+	vsrca = splat_alpha (vsrc);
+
+	vsrc = pix_multiply (vsrc, vmask);
+	vmask = pix_multiply (vmask, vsrca);
+
+	vdest = pix_add_mul (vsrc, splat_alpha (vdest),
+			     negate (vmask), vdest);
+
+	STORE_VECTOR (dest);
+
+	src += 4;
+	dest += 4;
+	mask += 4;
+    }
+
+    for (i = width % 4; --i >= 0;)
+    {
+	uint32_t a = mask[i];
+	uint32_t s = src[i];
+	uint32_t d = dest[i];
+	uint32_t sa = ALPHA_8 (s);
+	uint32_t da = ALPHA_8 (d);
+
+	UN8x4_MUL_UN8x4 (s, a);
+	UN8x4_MUL_UN8 (a, sa);
+	UN8x4_MUL_UN8x4_ADD_UN8x4_MUL_UN8 (d, ~a, s, da);
+
+	dest[i] = d;
+    }
+}
+
+static void
+vmx_combine_atop_reverse_ca (pixman_implementation_t *imp,
+                             pixman_op_t              op,
+                             uint32_t *               dest,
+                             const uint32_t *         src,
+                             const uint32_t *         mask,
+                             int                      width)
+{
+    int i;
+    vector unsigned int vdest, vsrc, vmask;
+    vector unsigned char tmp1, tmp2, tmp3, tmp4, edges,
+	dest_mask, mask_mask, src_mask, store_mask;
+
+    COMPUTE_SHIFT_MASKC (dest, src, mask);
+
+    /* printf ("%s\n",__PRETTY_FUNCTION__); */
+    for (i = width / 4; i > 0; i--)
+    {
+	LOAD_VECTORSC (dest, src, mask);
+
+	vdest = pix_add_mul (vdest,
+			     pix_multiply (vmask, splat_alpha (vsrc)),
+			     pix_multiply (vsrc, vmask),
+			     negate (splat_alpha (vdest)));
+
+	STORE_VECTOR (dest);
+
+	src += 4;
+	dest += 4;
+	mask += 4;
+    }
+
+    for (i = width % 4; --i >= 0;)
+    {
+	uint32_t a = mask[i];
+	uint32_t s = src[i];
+	uint32_t d = dest[i];
+	uint32_t sa = ALPHA_8 (s);
+	uint32_t da = ALPHA_8 (~d);
+
+	UN8x4_MUL_UN8x4 (s, a);
+	UN8x4_MUL_UN8 (a, sa);
+	UN8x4_MUL_UN8x4_ADD_UN8x4_MUL_UN8 (d, a, s, da);
+
+	dest[i] = d;
+    }
+}
+
+static void
+vmx_combine_xor_ca (pixman_implementation_t *imp,
+                    pixman_op_t              op,
+                    uint32_t *               dest,
+                    const uint32_t *         src,
+                    const uint32_t *         mask,
+                    int                      width)
+{
+    int i;
+    vector unsigned int vdest, vsrc, vmask;
+    vector unsigned char tmp1, tmp2, tmp3, tmp4, edges,
+	dest_mask, mask_mask, src_mask, store_mask;
+
+    COMPUTE_SHIFT_MASKC (dest, src, mask);
+
+    /* printf ("%s\n",__PRETTY_FUNCTION__); */
+    for (i = width / 4; i > 0; i--)
+    {
+	LOAD_VECTORSC (dest, src, mask);
+
+	vdest = pix_add_mul (vdest,
+			     negate (pix_multiply (vmask, splat_alpha (vsrc))),
+			     pix_multiply (vsrc, vmask),
+			     negate (splat_alpha (vdest)));
+
+	STORE_VECTOR (dest);
+
+	src += 4;
+	dest += 4;
+	mask += 4;
+    }
+
+    for (i = width % 4; --i >= 0;)
+    {
+	uint32_t a = mask[i];
+	uint32_t s = src[i];
+	uint32_t d = dest[i];
+	uint32_t sa = ALPHA_8 (s);
+	uint32_t da = ALPHA_8 (~d);
+
+	UN8x4_MUL_UN8x4 (s, a);
+	UN8x4_MUL_UN8 (a, sa);
+	UN8x4_MUL_UN8x4_ADD_UN8x4_MUL_UN8 (d, ~a, s, da);
+
+	dest[i] = d;
+    }
+}
+
+static void
+vmx_combine_add_ca (pixman_implementation_t *imp,
+                    pixman_op_t              op,
+                    uint32_t *               dest,
+                    const uint32_t *         src,
+                    const uint32_t *         mask,
+                    int                      width)
+{
+    int i;
+    vector unsigned int vdest, vsrc, vmask;
+    vector unsigned char tmp1, tmp2, tmp3, tmp4, edges,
+	dest_mask, mask_mask, src_mask, store_mask;
+
+    COMPUTE_SHIFT_MASKC (dest, src, mask);
+
+    /* printf ("%s\n",__PRETTY_FUNCTION__); */
+    for (i = width / 4; i > 0; i--)
+    {
+	LOAD_VECTORSC (dest, src, mask);
+
+	vdest = pix_add (pix_multiply (vsrc, vmask), vdest);
+
+	STORE_VECTOR (dest);
+
+	src += 4;
+	dest += 4;
+	mask += 4;
+    }
+
+    for (i = width % 4; --i >= 0;)
+    {
+	uint32_t a = mask[i];
+	uint32_t s = src[i];
+	uint32_t d = dest[i];
+
+	UN8x4_MUL_UN8x4 (s, a);
+	UN8x4_ADD_UN8x4 (s, d);
+
+	dest[i] = s;
+    }
+}
+
+static const pixman_fast_path_t vmx_fast_paths[] =
+{
+    {   PIXMAN_OP_NONE	},
+};
+
+pixman_implementation_t *
+_pixman_implementation_create_vmx (void)
+{
+    pixman_implementation_t *fast = _pixman_implementation_create_fast_path ();
+    pixman_implementation_t *imp = _pixman_implementation_create (fast, vmx_fast_paths);
+
+    /* Set up function pointers */
+
+    imp->combine_32[PIXMAN_OP_OVER] = vmx_combine_over_u;
+    imp->combine_32[PIXMAN_OP_OVER_REVERSE] = vmx_combine_over_reverse_u;
+    imp->combine_32[PIXMAN_OP_IN] = vmx_combine_in_u;
+    imp->combine_32[PIXMAN_OP_IN_REVERSE] = vmx_combine_in_reverse_u;
+    imp->combine_32[PIXMAN_OP_OUT] = vmx_combine_out_u;
+    imp->combine_32[PIXMAN_OP_OUT_REVERSE] = vmx_combine_out_reverse_u;
+    imp->combine_32[PIXMAN_OP_ATOP] = vmx_combine_atop_u;
+    imp->combine_32[PIXMAN_OP_ATOP_REVERSE] = vmx_combine_atop_reverse_u;
+    imp->combine_32[PIXMAN_OP_XOR] = vmx_combine_xor_u;
+
+    imp->combine_32[PIXMAN_OP_ADD] = vmx_combine_add_u;
+
+    imp->combine_32_ca[PIXMAN_OP_SRC] = vmx_combine_src_ca;
+    imp->combine_32_ca[PIXMAN_OP_OVER] = vmx_combine_over_ca;
+    imp->combine_32_ca[PIXMAN_OP_OVER_REVERSE] = vmx_combine_over_reverse_ca;
+    imp->combine_32_ca[PIXMAN_OP_IN] = vmx_combine_in_ca;
+    imp->combine_32_ca[PIXMAN_OP_IN_REVERSE] = vmx_combine_in_reverse_ca;
+    imp->combine_32_ca[PIXMAN_OP_OUT] = vmx_combine_out_ca;
+    imp->combine_32_ca[PIXMAN_OP_OUT_REVERSE] = vmx_combine_out_reverse_ca;
+    imp->combine_32_ca[PIXMAN_OP_ATOP] = vmx_combine_atop_ca;
+    imp->combine_32_ca[PIXMAN_OP_ATOP_REVERSE] = vmx_combine_atop_reverse_ca;
+    imp->combine_32_ca[PIXMAN_OP_XOR] = vmx_combine_xor_ca;
+    imp->combine_32_ca[PIXMAN_OP_ADD] = vmx_combine_add_ca;
+
+    return imp;
+}
diff --git a/pixman/pixman/pixman-x64-mmx-emulation.h b/pixman/pixman/pixman-x64-mmx-emulation.h
new file mode 100644
index 000000000..378019cf2
--- /dev/null
+++ b/pixman/pixman/pixman-x64-mmx-emulation.h
@@ -0,0 +1,263 @@
+#ifndef MMX_X64_H_INCLUDED
+#define MMX_X64_H_INCLUDED
+
+/* Implementation of x64 MMX substitition functions, before
+ * pixman is reimplemented not to use __m64 type on Visual C++
+ *
+ * Copyright (C)2009 by George Yohng
+ * Released in public domain.
+ */
+
+#include <intrin.h>
+
+#define M64C(a) (*(const __m64 *)(&a))
+#define M64U(a) (*(const unsigned long long *)(&a))
+
+__inline __m64
+_m_from_int (int a)
+{
+    long long i64 = a;
+
+    return M64C (i64);
+}
+
+__inline __m64
+_mm_setzero_si64 ()
+{
+    long long i64 = 0;
+
+    return M64C (i64);
+}
+
+__inline __m64
+_mm_set_pi32 (int i1,   int i0)
+{
+    unsigned long long i64 = ((unsigned)i0) + (((unsigned long long)(unsigned)i1) << 32);
+
+    return M64C (i64);
+}
+
+__inline void
+_m_empty ()
+{
+}
+
+__inline __m64
+_mm_set1_pi16 (short w)
+{
+    unsigned long long i64 = ((unsigned long long)(unsigned short)(w)) * 0x0001000100010001ULL;
+
+    return M64C (i64);
+}
+
+__inline int
+_m_to_int (__m64 m)
+{
+    return m.m64_i32[0];
+}
+
+__inline __m64
+_mm_movepi64_pi64 (__m128i a)
+{
+    return M64C (a.m128i_i64[0]);
+}
+
+__inline __m64
+_m_pand (__m64 a, __m64 b)
+{
+    unsigned long long i64 = M64U (a) & M64U (b);
+
+    return M64C (i64);
+}
+
+__inline __m64
+_m_por (__m64 a, __m64 b)
+{
+    unsigned long long i64 = M64U (a) | M64U (b);
+
+    return M64C (i64);
+}
+
+__inline __m64
+_m_pxor (__m64 a, __m64 b)
+{
+    unsigned long long i64 = M64U (a) ^ M64U (b);
+
+    return M64C (i64);
+}
+
+__inline __m64
+_m_pmulhuw (__m64 a, __m64 b)        /* unoptimized */
+{
+    unsigned short d[4] =
+    {
+	(unsigned short)((((unsigned)a.m64_u16[0]) * b.m64_u16[0]) >> 16),
+	(unsigned short)((((unsigned)a.m64_u16[1]) * b.m64_u16[1]) >> 16),
+	(unsigned short)((((unsigned)a.m64_u16[2]) * b.m64_u16[2]) >> 16),
+	(unsigned short)((((unsigned)a.m64_u16[3]) * b.m64_u16[3]) >> 16)
+    };
+
+    return M64C (d[0]);
+}
+
+__inline __m64
+_m_pmullw2 (__m64 a, __m64 b)        /* unoptimized */
+{
+    unsigned short d[4] =
+    {
+	(unsigned short)((((unsigned)a.m64_u16[0]) * b.m64_u16[0])),
+	(unsigned short)((((unsigned)a.m64_u16[1]) * b.m64_u16[1])),
+	(unsigned short)((((unsigned)a.m64_u16[2]) * b.m64_u16[2])),
+	(unsigned short)((((unsigned)a.m64_u16[3]) * b.m64_u16[3]))
+    };
+
+    return M64C (d[0]);
+}
+
+__inline __m64
+_m_pmullw (__m64 a, __m64 b)        /* unoptimized */
+{
+    unsigned long long x =
+	((unsigned long long)(unsigned short)((((unsigned)a.m64_u16[0]) * b.m64_u16[0])))  +
+	(((unsigned long long)(unsigned short)((((unsigned)a.m64_u16[1]) * b.m64_u16[1]))) << 16)  +
+	(((unsigned long long)(unsigned short)((((unsigned)a.m64_u16[2]) * b.m64_u16[2]))) << 32)  +
+	(((unsigned long long)(unsigned short)((((unsigned)a.m64_u16[3]) * b.m64_u16[3]))) << 48);
+
+    return M64C (x);
+}
+
+__inline __m64
+_m_paddusb (__m64 a, __m64 b)        /* unoptimized */
+{
+    unsigned long long x = (M64U (a) & 0x00FF00FF00FF00FFULL) +
+                           (M64U (b) & 0x00FF00FF00FF00FFULL);
+
+    unsigned long long y = ((M64U (a) >> 8) & 0x00FF00FF00FF00FFULL) +
+                           ((M64U (b) >> 8) & 0x00FF00FF00FF00FFULL);
+
+    x |= ((x & 0xFF00FF00FF00FF00ULL) >> 8) * 0xFF;
+    y |= ((y & 0xFF00FF00FF00FF00ULL) >> 8) * 0xFF;
+
+    x = (x & 0x00FF00FF00FF00FFULL) | ((y & 0x00FF00FF00FF00FFULL) << 8);
+
+    return M64C (x);
+}
+
+__inline __m64
+_m_paddusw (__m64 a, __m64 b)        /* unoptimized */
+{
+    unsigned long long x = (M64U (a) & 0x0000FFFF0000FFFFULL) +
+                           (M64U (b) & 0x0000FFFF0000FFFFULL);
+
+    unsigned long long y = ((M64U (a) >> 16) & 0x0000FFFF0000FFFFULL) +
+                           ((M64U (b) >> 16) & 0x0000FFFF0000FFFFULL);
+
+    x |= ((x & 0xFFFF0000FFFF0000) >> 16) * 0xFFFF;
+    y |= ((y & 0xFFFF0000FFFF0000) >> 16) * 0xFFFF;
+
+    x = (x & 0x0000FFFF0000FFFFULL) | ((y & 0x0000FFFF0000FFFFULL) << 16);
+
+    return M64C (x);
+}
+
+__inline __m64
+_m_pshufw (__m64 a, int n)         /* unoptimized */
+{
+    unsigned short d[4] =
+    {
+	a.m64_u16[n & 3],
+	a.m64_u16[(n >> 2) & 3],
+	a.m64_u16[(n >> 4) & 3],
+	a.m64_u16[(n >> 6) & 3]
+    };
+
+    return M64C (d[0]);
+}
+
+__inline unsigned char
+sat16 (unsigned short d)
+{
+    if (d > 0xFF) return 0xFF;
+    else return d & 0xFF;
+}
+
+__inline __m64
+_m_packuswb (__m64 m1, __m64 m2)          /* unoptimized */
+{
+    unsigned char d[8] =
+    {
+	sat16 (m1.m64_u16[0]),
+	sat16 (m1.m64_u16[1]),
+	sat16 (m1.m64_u16[2]),
+	sat16 (m1.m64_u16[3]),
+	sat16 (m2.m64_u16[0]),
+	sat16 (m2.m64_u16[1]),
+	sat16 (m2.m64_u16[2]),
+	sat16 (m2.m64_u16[3])
+    };
+
+    return M64C (d[0]);
+}
+
+__inline __m64 _m_punpcklbw (__m64 m1, __m64 m2)          /* unoptimized */
+{
+    unsigned char d[8] =
+    {
+	m1.m64_u8[0],
+	m2.m64_u8[0],
+	m1.m64_u8[1],
+	m2.m64_u8[1],
+	m1.m64_u8[2],
+	m2.m64_u8[2],
+	m1.m64_u8[3],
+	m2.m64_u8[3],
+    };
+
+    return M64C (d[0]);
+}
+
+__inline __m64 _m_punpckhbw (__m64 m1, __m64 m2)          /* unoptimized */
+{
+    unsigned char d[8] =
+    {
+	m1.m64_u8[4],
+	m2.m64_u8[4],
+	m1.m64_u8[5],
+	m2.m64_u8[5],
+	m1.m64_u8[6],
+	m2.m64_u8[6],
+	m1.m64_u8[7],
+	m2.m64_u8[7],
+    };
+
+    return M64C (d[0]);
+}
+
+__inline __m64 _m_psrlwi (__m64 a, int n)       /* unoptimized */
+{
+    unsigned short d[4] =
+    {
+	a.m64_u16[0] >> n,
+	a.m64_u16[1] >> n,
+	a.m64_u16[2] >> n,
+	a.m64_u16[3] >> n
+    };
+
+    return M64C (d[0]);
+}
+
+__inline __m64 _m_psrlqi (__m64 m, int n)
+{
+    unsigned long long x = M64U (m) >> n;
+
+    return M64C (x);
+}
+
+__inline __m64 _m_psllqi (__m64 m, int n)
+{
+    unsigned long long x = M64U (m) << n;
+
+    return M64C (x);
+}
+
+#endif /* MMX_X64_H_INCLUDED */
diff --git a/pixman/pixman/pixman.c b/pixman/pixman/pixman.c
new file mode 100644
index 000000000..548242ba0
--- /dev/null
+++ b/pixman/pixman/pixman.c
@@ -0,0 +1,1221 @@
+/* -*- Mode: c; c-basic-offset: 4; tab-width: 8; indent-tabs-mode: t; -*- */
+/*
+ * Copyright © 2000 SuSE, Inc.
+ * Copyright © 2007 Red Hat, Inc.
+ *
+ * Permission to use, copy, modify, distribute, and sell this software and its
+ * documentation for any purpose is hereby granted without fee, provided that
+ * the above copyright notice appear in all copies and that both that
+ * copyright notice and this permission notice appear in supporting
+ * documentation, and that the name of SuSE not be used in advertising or
+ * publicity pertaining to distribution of the software without specific,
+ * written prior permission.  SuSE makes no representations about the
+ * suitability of this software for any purpose.  It is provided "as is"
+ * without express or implied warranty.
+ *
+ * SuSE DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING ALL
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL SuSE
+ * BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
+ * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+ * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ *
+ * Author:  Keith Packard, SuSE, Inc.
+ */
+
+#ifdef HAVE_CONFIG_H
+#include <config.h>
+#endif
+#include "pixman-private.h"
+
+#include <stdlib.h>
+
+static pixman_implementation_t *imp;
+
+typedef struct operator_info_t operator_info_t;
+
+struct operator_info_t
+{
+    uint8_t	opaque_info[4];
+};
+
+#define PACK(neither, src, dest, both)			\
+    {{	    (uint8_t)PIXMAN_OP_ ## neither,		\
+	    (uint8_t)PIXMAN_OP_ ## src,			\
+	    (uint8_t)PIXMAN_OP_ ## dest,		\
+	    (uint8_t)PIXMAN_OP_ ## both		}}
+
+static const operator_info_t operator_table[] =
+{
+    /*    Neither Opaque         Src Opaque             Dst Opaque             Both Opaque */
+    PACK (CLEAR,                 CLEAR,                 CLEAR,                 CLEAR),
+    PACK (SRC,                   SRC,                   SRC,                   SRC),
+    PACK (DST,                   DST,                   DST,                   DST),
+    PACK (OVER,                  SRC,                   OVER,                  SRC),
+    PACK (OVER_REVERSE,          OVER_REVERSE,          DST,                   DST),
+    PACK (IN,                    IN,                    SRC,                   SRC),
+    PACK (IN_REVERSE,            DST,                   IN_REVERSE,            DST),
+    PACK (OUT,                   OUT,                   CLEAR,                 CLEAR),
+    PACK (OUT_REVERSE,           CLEAR,                 OUT_REVERSE,           CLEAR),
+    PACK (ATOP,                  IN,                    OVER,                  SRC),
+    PACK (ATOP_REVERSE,          OVER_REVERSE,          IN_REVERSE,            DST),
+    PACK (XOR,                   OUT,                   OUT_REVERSE,           CLEAR),
+    PACK (ADD,                   ADD,                   ADD,                   ADD),
+    PACK (SATURATE,              OVER_REVERSE,          DST,                   DST),
+
+    {{ 0 /* 0x0e */ }},
+    {{ 0 /* 0x0f */ }},
+
+    PACK (CLEAR,                 CLEAR,                 CLEAR,                 CLEAR),
+    PACK (SRC,                   SRC,                   SRC,                   SRC),
+    PACK (DST,                   DST,                   DST,                   DST),
+    PACK (DISJOINT_OVER,         DISJOINT_OVER,         DISJOINT_OVER,         DISJOINT_OVER),
+    PACK (DISJOINT_OVER_REVERSE, DISJOINT_OVER_REVERSE, DISJOINT_OVER_REVERSE, DISJOINT_OVER_REVERSE),
+    PACK (DISJOINT_IN,           DISJOINT_IN,           DISJOINT_IN,           DISJOINT_IN),
+    PACK (DISJOINT_IN_REVERSE,   DISJOINT_IN_REVERSE,   DISJOINT_IN_REVERSE,   DISJOINT_IN_REVERSE),
+    PACK (DISJOINT_OUT,          DISJOINT_OUT,          DISJOINT_OUT,          DISJOINT_OUT),
+    PACK (DISJOINT_OUT_REVERSE,  DISJOINT_OUT_REVERSE,  DISJOINT_OUT_REVERSE,  DISJOINT_OUT_REVERSE),
+    PACK (DISJOINT_ATOP,         DISJOINT_ATOP,         DISJOINT_ATOP,         DISJOINT_ATOP),
+    PACK (DISJOINT_ATOP_REVERSE, DISJOINT_ATOP_REVERSE, DISJOINT_ATOP_REVERSE, DISJOINT_ATOP_REVERSE),
+    PACK (DISJOINT_XOR,          DISJOINT_XOR,          DISJOINT_XOR,          DISJOINT_XOR),
+
+    {{ 0 /* 0x1c */ }},
+    {{ 0 /* 0x1d */ }},
+    {{ 0 /* 0x1e */ }},
+    {{ 0 /* 0x1f */ }},
+
+    PACK (CLEAR,                 CLEAR,                 CLEAR,                 CLEAR),
+    PACK (SRC,                   SRC,                   SRC,                   SRC),
+    PACK (DST,                   DST,                   DST,                   DST),
+    PACK (CONJOINT_OVER,         CONJOINT_OVER,         CONJOINT_OVER,         CONJOINT_OVER),
+    PACK (CONJOINT_OVER_REVERSE, CONJOINT_OVER_REVERSE, CONJOINT_OVER_REVERSE, CONJOINT_OVER_REVERSE),
+    PACK (CONJOINT_IN,           CONJOINT_IN,           CONJOINT_IN,           CONJOINT_IN),
+    PACK (CONJOINT_IN_REVERSE,   CONJOINT_IN_REVERSE,   CONJOINT_IN_REVERSE,   CONJOINT_IN_REVERSE),
+    PACK (CONJOINT_OUT,          CONJOINT_OUT,          CONJOINT_OUT,          CONJOINT_OUT),
+    PACK (CONJOINT_OUT_REVERSE,  CONJOINT_OUT_REVERSE,  CONJOINT_OUT_REVERSE,  CONJOINT_OUT_REVERSE),
+    PACK (CONJOINT_ATOP,         CONJOINT_ATOP,         CONJOINT_ATOP,         CONJOINT_ATOP),
+    PACK (CONJOINT_ATOP_REVERSE, CONJOINT_ATOP_REVERSE, CONJOINT_ATOP_REVERSE, CONJOINT_ATOP_REVERSE),
+    PACK (CONJOINT_XOR,          CONJOINT_XOR,          CONJOINT_XOR,          CONJOINT_XOR),
+
+    {{ 0 /* 0x2c */ }},
+    {{ 0 /* 0x2d */ }},
+    {{ 0 /* 0x2e */ }},
+    {{ 0 /* 0x2f */ }},
+
+    PACK (MULTIPLY,              MULTIPLY,              MULTIPLY,              MULTIPLY),
+    PACK (SCREEN,                SCREEN,                SCREEN,                SCREEN),
+    PACK (OVERLAY,               OVERLAY,               OVERLAY,               OVERLAY),
+    PACK (DARKEN,                DARKEN,                DARKEN,                DARKEN),
+    PACK (LIGHTEN,               LIGHTEN,               LIGHTEN,               LIGHTEN),
+    PACK (COLOR_DODGE,           COLOR_DODGE,           COLOR_DODGE,           COLOR_DODGE),
+    PACK (COLOR_BURN,            COLOR_BURN,            COLOR_BURN,            COLOR_BURN),
+    PACK (HARD_LIGHT,            HARD_LIGHT,            HARD_LIGHT,            HARD_LIGHT),
+    PACK (SOFT_LIGHT,            SOFT_LIGHT,            SOFT_LIGHT,            SOFT_LIGHT),
+    PACK (DIFFERENCE,            DIFFERENCE,            DIFFERENCE,            DIFFERENCE),
+    PACK (EXCLUSION,             EXCLUSION,             EXCLUSION,             EXCLUSION),
+    PACK (HSL_HUE,               HSL_HUE,               HSL_HUE,               HSL_HUE),
+    PACK (HSL_SATURATION,        HSL_SATURATION,        HSL_SATURATION,        HSL_SATURATION),
+    PACK (HSL_COLOR,             HSL_COLOR,             HSL_COLOR,             HSL_COLOR),
+    PACK (HSL_LUMINOSITY,        HSL_LUMINOSITY,        HSL_LUMINOSITY,        HSL_LUMINOSITY),
+};
+
+/*
+ * Optimize the current operator based on opacity of source or destination
+ * The output operator should be mathematically equivalent to the source.
+ */
+static pixman_op_t
+optimize_operator (pixman_op_t     op,
+		   uint32_t        src_flags,
+		   uint32_t        mask_flags,
+		   uint32_t        dst_flags)
+{
+    pixman_bool_t is_source_opaque, is_dest_opaque;
+    int opaqueness;
+
+    is_source_opaque = ((src_flags & mask_flags) & FAST_PATH_IS_OPAQUE) != 0;
+    is_dest_opaque = (dst_flags & FAST_PATH_IS_OPAQUE) != 0;
+
+    opaqueness = ((is_dest_opaque << 1) | is_source_opaque);
+
+    return operator_table[op].opaque_info[opaqueness];
+}
+
+static void
+apply_workaround (pixman_image_t *image,
+		  int32_t *       x,
+		  int32_t *       y,
+		  uint32_t **     save_bits,
+		  int *           save_dx,
+		  int *           save_dy)
+{
+    if (image && (image->common.flags & FAST_PATH_NEEDS_WORKAROUND))
+    {
+	/* Some X servers generate images that point to the
+	 * wrong place in memory, but then set the clip region
+	 * to point to the right place. Because of an old bug
+	 * in pixman, this would actually work.
+	 *
+	 * Here we try and undo the damage
+	 */
+	int bpp = PIXMAN_FORMAT_BPP (image->bits.format) / 8;
+	pixman_box32_t *extents;
+	uint8_t *t;
+	int dx, dy;
+	
+	extents = pixman_region32_extents (&(image->common.clip_region));
+	dx = extents->x1;
+	dy = extents->y1;
+	
+	*save_bits = image->bits.bits;
+	
+	*x -= dx;
+	*y -= dy;
+	pixman_region32_translate (&(image->common.clip_region), -dx, -dy);
+	
+	t = (uint8_t *)image->bits.bits;
+	t += dy * image->bits.rowstride * 4 + dx * bpp;
+	image->bits.bits = (uint32_t *)t;
+	
+	*save_dx = dx;
+	*save_dy = dy;
+    }
+}
+
+static void
+unapply_workaround (pixman_image_t *image, uint32_t *bits, int dx, int dy)
+{
+    if (image && (image->common.flags & FAST_PATH_NEEDS_WORKAROUND))
+    {
+	image->bits.bits = bits;
+	pixman_region32_translate (&image->common.clip_region, dx, dy);
+    }
+}
+
+/*
+ * Computing composite region
+ */
+static inline pixman_bool_t
+clip_general_image (pixman_region32_t * region,
+                    pixman_region32_t * clip,
+                    int                 dx,
+                    int                 dy)
+{
+    if (pixman_region32_n_rects (region) == 1 &&
+        pixman_region32_n_rects (clip) == 1)
+    {
+	pixman_box32_t *  rbox = pixman_region32_rectangles (region, NULL);
+	pixman_box32_t *  cbox = pixman_region32_rectangles (clip, NULL);
+	int v;
+
+	if (rbox->x1 < (v = cbox->x1 + dx))
+	    rbox->x1 = v;
+	if (rbox->x2 > (v = cbox->x2 + dx))
+	    rbox->x2 = v;
+	if (rbox->y1 < (v = cbox->y1 + dy))
+	    rbox->y1 = v;
+	if (rbox->y2 > (v = cbox->y2 + dy))
+	    rbox->y2 = v;
+	if (rbox->x1 >= rbox->x2 || rbox->y1 >= rbox->y2)
+	{
+	    pixman_region32_init (region);
+	    return FALSE;
+	}
+    }
+    else if (!pixman_region32_not_empty (clip))
+    {
+	return FALSE;
+    }
+    else
+    {
+	if (dx || dy)
+	    pixman_region32_translate (region, -dx, -dy);
+
+	if (!pixman_region32_intersect (region, region, clip))
+	    return FALSE;
+
+	if (dx || dy)
+	    pixman_region32_translate (region, dx, dy);
+    }
+
+    return pixman_region32_not_empty (region);
+}
+
+static inline pixman_bool_t
+clip_source_image (pixman_region32_t * region,
+                   pixman_image_t *    image,
+                   int                 dx,
+                   int                 dy)
+{
+    /* Source clips are ignored, unless they are explicitly turned on
+     * and the clip in question was set by an X client. (Because if
+     * the clip was not set by a client, then it is a hierarchy
+     * clip and those should always be ignored for sources).
+     */
+    if (!image->common.clip_sources || !image->common.client_clip)
+	return TRUE;
+
+    return clip_general_image (region,
+                               &image->common.clip_region,
+                               dx, dy);
+}
+
+/*
+ * returns FALSE if the final region is empty.  Indistinguishable from
+ * an allocation failure, but rendering ignores those anyways.
+ */
+static pixman_bool_t
+pixman_compute_composite_region32 (pixman_region32_t * region,
+                                   pixman_image_t *    src_image,
+                                   pixman_image_t *    mask_image,
+                                   pixman_image_t *    dst_image,
+                                   int32_t             src_x,
+                                   int32_t             src_y,
+                                   int32_t             mask_x,
+                                   int32_t             mask_y,
+                                   int32_t             dest_x,
+                                   int32_t             dest_y,
+                                   int32_t             width,
+                                   int32_t             height)
+{
+    region->extents.x1 = dest_x;
+    region->extents.x2 = dest_x + width;
+    region->extents.y1 = dest_y;
+    region->extents.y2 = dest_y + height;
+
+    region->extents.x1 = MAX (region->extents.x1, 0);
+    region->extents.y1 = MAX (region->extents.y1, 0);
+    region->extents.x2 = MIN (region->extents.x2, dst_image->bits.width);
+    region->extents.y2 = MIN (region->extents.y2, dst_image->bits.height);
+
+    region->data = 0;
+
+    /* Check for empty operation */
+    if (region->extents.x1 >= region->extents.x2 ||
+        region->extents.y1 >= region->extents.y2)
+    {
+	pixman_region32_init (region);
+	return FALSE;
+    }
+
+    if (dst_image->common.have_clip_region)
+    {
+	if (!clip_general_image (region, &dst_image->common.clip_region, 0, 0))
+	{
+	    pixman_region32_fini (region);
+	    return FALSE;
+	}
+    }
+
+    if (dst_image->common.alpha_map && dst_image->common.alpha_map->common.have_clip_region)
+    {
+	if (!clip_general_image (region, &dst_image->common.alpha_map->common.clip_region,
+	                         -dst_image->common.alpha_origin_x,
+	                         -dst_image->common.alpha_origin_y))
+	{
+	    pixman_region32_fini (region);
+	    return FALSE;
+	}
+    }
+
+    /* clip against src */
+    if (src_image->common.have_clip_region)
+    {
+	if (!clip_source_image (region, src_image, dest_x - src_x, dest_y - src_y))
+	{
+	    pixman_region32_fini (region);
+	    return FALSE;
+	}
+    }
+    if (src_image->common.alpha_map && src_image->common.alpha_map->common.have_clip_region)
+    {
+	if (!clip_source_image (region, (pixman_image_t *)src_image->common.alpha_map,
+	                        dest_x - (src_x - src_image->common.alpha_origin_x),
+	                        dest_y - (src_y - src_image->common.alpha_origin_y)))
+	{
+	    pixman_region32_fini (region);
+	    return FALSE;
+	}
+    }
+    /* clip against mask */
+    if (mask_image && mask_image->common.have_clip_region)
+    {
+	if (!clip_source_image (region, mask_image, dest_x - mask_x, dest_y - mask_y))
+	{
+	    pixman_region32_fini (region);
+	    return FALSE;
+	}
+	if (mask_image->common.alpha_map && mask_image->common.alpha_map->common.have_clip_region)
+	{
+	    if (!clip_source_image (region, (pixman_image_t *)mask_image->common.alpha_map,
+	                            dest_x - (mask_x - mask_image->common.alpha_origin_x),
+	                            dest_y - (mask_y - mask_image->common.alpha_origin_y)))
+	    {
+		pixman_region32_fini (region);
+		return FALSE;
+	    }
+	}
+    }
+
+    return TRUE;
+}
+
+static void
+walk_region_internal (pixman_implementation_t *imp,
+                      pixman_op_t              op,
+                      pixman_image_t *         src_image,
+                      pixman_image_t *         mask_image,
+                      pixman_image_t *         dst_image,
+                      int32_t                  src_x,
+                      int32_t                  src_y,
+                      int32_t                  mask_x,
+                      int32_t                  mask_y,
+                      int32_t                  dest_x,
+                      int32_t                  dest_y,
+                      int32_t                  width,
+                      int32_t                  height,
+                      pixman_bool_t            src_repeat,
+                      pixman_bool_t            mask_repeat,
+                      pixman_region32_t *      region,
+                      pixman_composite_func_t  composite_rect)
+{
+    int w, h, w_this, h_this;
+    int x_msk, y_msk, x_src, y_src, x_dst, y_dst;
+    int src_dy = src_y - dest_y;
+    int src_dx = src_x - dest_x;
+    int mask_dy = mask_y - dest_y;
+    int mask_dx = mask_x - dest_x;
+    const pixman_box32_t *pbox;
+    int n;
+
+    pbox = pixman_region32_rectangles (region, &n);
+
+    /* Fast path for non-repeating sources */
+    if (!src_repeat && !mask_repeat)
+    {
+       while (n--)
+       {
+           (*composite_rect) (imp, op,
+                              src_image, mask_image, dst_image,
+                              pbox->x1 + src_dx,
+                              pbox->y1 + src_dy,
+                              pbox->x1 + mask_dx,
+                              pbox->y1 + mask_dy,
+                              pbox->x1,
+                              pbox->y1,
+                              pbox->x2 - pbox->x1,
+                              pbox->y2 - pbox->y1);
+           
+           pbox++;
+       }
+
+       return;
+    }
+    
+    while (n--)
+    {
+	h = pbox->y2 - pbox->y1;
+	y_src = pbox->y1 + src_dy;
+	y_msk = pbox->y1 + mask_dy;
+	y_dst = pbox->y1;
+
+	while (h)
+	{
+	    h_this = h;
+	    w = pbox->x2 - pbox->x1;
+	    x_src = pbox->x1 + src_dx;
+	    x_msk = pbox->x1 + mask_dx;
+	    x_dst = pbox->x1;
+
+	    if (mask_repeat)
+	    {
+		y_msk = MOD (y_msk, mask_image->bits.height);
+		if (h_this > mask_image->bits.height - y_msk)
+		    h_this = mask_image->bits.height - y_msk;
+	    }
+
+	    if (src_repeat)
+	    {
+		y_src = MOD (y_src, src_image->bits.height);
+		if (h_this > src_image->bits.height - y_src)
+		    h_this = src_image->bits.height - y_src;
+	    }
+
+	    while (w)
+	    {
+		w_this = w;
+
+		if (mask_repeat)
+		{
+		    x_msk = MOD (x_msk, mask_image->bits.width);
+		    if (w_this > mask_image->bits.width - x_msk)
+			w_this = mask_image->bits.width - x_msk;
+		}
+
+		if (src_repeat)
+		{
+		    x_src = MOD (x_src, src_image->bits.width);
+		    if (w_this > src_image->bits.width - x_src)
+			w_this = src_image->bits.width - x_src;
+		}
+
+		(*composite_rect) (imp, op,
+				   src_image, mask_image, dst_image,
+				   x_src, y_src, x_msk, y_msk, x_dst, y_dst,
+				   w_this, h_this);
+		w -= w_this;
+
+		x_src += w_this;
+		x_msk += w_this;
+		x_dst += w_this;
+	    }
+
+	    h -= h_this;
+	    y_src += h_this;
+	    y_msk += h_this;
+	    y_dst += h_this;
+	}
+
+	pbox++;
+    }
+}
+
+#define IS_16BIT(x) (((x) >= INT16_MIN) && ((x) <= INT16_MAX))
+
+static force_inline uint32_t
+compute_src_extents_flags (pixman_image_t *image,
+			   pixman_box32_t *extents,
+			   int             x,
+			   int             y)
+{
+    pixman_box16_t extents16;
+    uint32_t flags;
+
+    flags = FAST_PATH_COVERS_CLIP;
+
+    if (image->common.type != BITS)
+	return flags;
+
+    if (image->common.repeat == PIXMAN_REPEAT_NONE &&
+	(x > extents->x1 || y > extents->y1 ||
+	 x + image->bits.width < extents->x2 ||
+	 y + image->bits.height < extents->y2))
+    {
+	flags &= ~FAST_PATH_COVERS_CLIP;
+    }
+
+    if (IS_16BIT (extents->x1 - x) &&
+	IS_16BIT (extents->y1 - y) &&
+	IS_16BIT (extents->x2 - x) &&
+	IS_16BIT (extents->y2 - y))
+    {
+	extents16.x1 = extents->x1 - x;
+	extents16.y1 = extents->y1 - y;
+	extents16.x2 = extents->x2 - x;
+	extents16.y2 = extents->y2 - y;
+
+	if (!image->common.transform ||
+	    pixman_transform_bounds (image->common.transform, &extents16))
+	{
+	    if (extents16.x1 >= 0  && extents16.y1 >= 0 &&
+		extents16.x2 <= image->bits.width &&
+		extents16.y2 <= image->bits.height)
+	    {
+		flags |= FAST_PATH_SAMPLES_COVER_CLIP;
+	    }
+	}
+    }
+
+    if (IS_16BIT (extents->x1 - x - 1) &&
+	IS_16BIT (extents->y1 - y - 1) &&
+	IS_16BIT (extents->x2 - x + 1) &&
+	IS_16BIT (extents->y2 - y + 1))
+    {
+	extents16.x1 = extents->x1 - x - 1;
+	extents16.y1 = extents->y1 - y - 1;
+	extents16.x2 = extents->x2 - x + 1;
+	extents16.y2 = extents->y2 - y + 1;
+
+	if (/* src space expanded by one in dest space fits in 16 bit */
+	    (!image->common.transform ||
+	     pixman_transform_bounds (image->common.transform, &extents16)) &&
+	    /* And src image size can be used as 16.16 fixed point */
+	    image->bits.width < 0x7fff &&
+	    image->bits.height < 0x7fff)
+	{
+	    /* Then we're "16bit safe" */
+	    flags |= FAST_PATH_16BIT_SAFE;
+	}
+    }
+
+    return flags;
+}
+
+#define N_CACHED_FAST_PATHS 8
+
+typedef struct
+{
+    pixman_fast_path_t cache [N_CACHED_FAST_PATHS];
+} cache_t;
+
+PIXMAN_DEFINE_THREAD_LOCAL (cache_t, fast_path_cache);
+
+static void
+do_composite (pixman_implementation_t *imp,
+	      pixman_op_t	       op,
+	      pixman_image_t	      *src,
+	      pixman_image_t	      *mask,
+	      pixman_image_t	      *dest,
+	      int		       src_x,
+	      int		       src_y,
+	      int		       mask_x,
+	      int		       mask_y,
+	      int		       dest_x,
+	      int		       dest_y,
+	      int		       width,
+	      int		       height)
+{
+    pixman_format_code_t src_format, mask_format, dest_format;
+    uint32_t src_flags, mask_flags, dest_flags;
+    pixman_region32_t region;
+    pixman_box32_t *extents;
+    uint32_t *src_bits;
+    int src_dx, src_dy;
+    uint32_t *mask_bits;
+    int mask_dx, mask_dy;
+    uint32_t *dest_bits;
+    int dest_dx, dest_dy;
+    pixman_bool_t need_workaround;
+    const pixman_fast_path_t *info;
+    cache_t *cache;
+    int i;
+
+    src_format = src->common.extended_format_code;
+    src_flags = src->common.flags;
+
+    if (mask)
+    {
+	mask_format = mask->common.extended_format_code;
+	mask_flags = mask->common.flags;
+    }
+    else
+    {
+	mask_format = PIXMAN_null;
+	mask_flags = FAST_PATH_IS_OPAQUE;
+    }
+
+    dest_format = dest->common.extended_format_code;
+    dest_flags = dest->common.flags;
+
+    /* Check for pixbufs */
+    if ((mask_format == PIXMAN_a8r8g8b8 || mask_format == PIXMAN_a8b8g8r8) &&
+	(src->type == BITS && src->bits.bits == mask->bits.bits)	   &&
+	(src->common.repeat == mask->common.repeat)			   &&
+	(src_x == mask_x && src_y == mask_y))
+    {
+	if (src_format == PIXMAN_x8b8g8r8)
+	    src_format = mask_format = PIXMAN_pixbuf;
+	else if (src_format == PIXMAN_x8r8g8b8)
+	    src_format = mask_format = PIXMAN_rpixbuf;
+    }
+
+    /* Check for workaround */
+    need_workaround = (src_flags | mask_flags | dest_flags) & FAST_PATH_NEEDS_WORKAROUND;
+
+    if (need_workaround)
+    {
+	apply_workaround (src, &src_x, &src_y, &src_bits, &src_dx, &src_dy);
+	apply_workaround (mask, &mask_x, &mask_y, &mask_bits, &mask_dx, &mask_dy);
+	apply_workaround (dest, &dest_x, &dest_y, &dest_bits, &dest_dx, &dest_dy);
+    }
+
+    pixman_region32_init (&region);
+    
+    if (!pixman_compute_composite_region32 (
+	    &region, src, mask, dest,
+	    src_x, src_y, mask_x, mask_y, dest_x, dest_y, width, height))
+    {
+	goto out;
+    }
+    
+    extents = pixman_region32_extents (&region);
+    
+    src_flags |= compute_src_extents_flags (src, extents, dest_x - src_x, dest_y - src_y);
+
+    if (mask)
+	mask_flags |= compute_src_extents_flags (mask, extents, dest_x - mask_x, dest_y - mask_y);
+
+    /*
+     * Check if we can replace our operator by a simpler one
+     * if the src or dest are opaque. The output operator should be
+     * mathematically equivalent to the source.
+     */
+    op = optimize_operator (op, src_flags, mask_flags, dest_flags);
+    if (op == PIXMAN_OP_DST)
+	goto out;
+
+    /* Check cache for fast paths */
+    cache = PIXMAN_GET_THREAD_LOCAL (fast_path_cache);
+
+    for (i = 0; i < N_CACHED_FAST_PATHS; ++i)
+    {
+	info = &(cache->cache[i]);
+
+	/* Note that we check for equality here, not whether
+	 * the cached fast path matches. This is to prevent
+	 * us from selecting an overly general fast path
+	 * when a more specific one would work.
+	 */
+	if (info->op == op			&&
+	    info->src_format == src_format	&&
+	    info->mask_format == mask_format	&&
+	    info->dest_format == dest_format	&&
+	    info->src_flags == src_flags	&&
+	    info->mask_flags == mask_flags	&&
+	    info->dest_flags == dest_flags	&&
+	    info->func)
+	{
+	    goto found;
+	}
+    }
+
+    while (imp)
+    {
+	info = imp->fast_paths;
+
+	while (info->op != PIXMAN_OP_NONE)
+	{
+	    if ((info->op == op || info->op == PIXMAN_OP_any)		&&
+		/* Formats */
+		((info->src_format == src_format) ||
+		 (info->src_format == PIXMAN_any))			&&
+		((info->mask_format == mask_format) ||
+		 (info->mask_format == PIXMAN_any))			&&
+		((info->dest_format == dest_format) ||
+		 (info->dest_format == PIXMAN_any))			&&
+		/* Flags */
+		(info->src_flags & src_flags) == info->src_flags	&&
+		(info->mask_flags & mask_flags) == info->mask_flags	&&
+		(info->dest_flags & dest_flags) == info->dest_flags)
+	    {
+		/* Set i to the last spot in the cache so that the
+		 * move-to-front code below will work
+		 */
+		i = N_CACHED_FAST_PATHS - 1;
+
+		goto found;
+	    }
+
+	    ++info;
+	}
+
+	imp = imp->delegate;
+    }
+
+    /* We didn't find a compositing routine. This should not happen, but if
+     * it somehow does, just exit rather than crash.
+     */
+    goto out;
+
+found:
+    walk_region_internal (imp, op,
+			  src, mask, dest,
+			  src_x, src_y, mask_x, mask_y,
+			  dest_x, dest_y,
+			  width, height,
+			  (src_flags & FAST_PATH_SIMPLE_REPEAT),
+			  (mask_flags & FAST_PATH_SIMPLE_REPEAT),
+			  &region, info->func);
+
+    if (i)
+    {
+	/* Make a copy of info->func, because info->func may change when
+	 * we update the cache.
+	 */
+	pixman_composite_func_t func = info->func;
+	
+	while (i--)
+	    cache->cache[i + 1] = cache->cache[i];
+
+	cache->cache[0].op = op;
+	cache->cache[0].src_format = src_format;
+	cache->cache[0].src_flags = src_flags;
+	cache->cache[0].mask_format = mask_format;
+	cache->cache[0].mask_flags = mask_flags;
+	cache->cache[0].dest_format = dest_format;
+	cache->cache[0].dest_flags = dest_flags;
+	cache->cache[0].func = func;
+    }
+
+out:
+    if (need_workaround)
+    {
+	unapply_workaround (src, src_bits, src_dx, src_dy);
+	unapply_workaround (mask, mask_bits, mask_dx, mask_dy);
+	unapply_workaround (dest, dest_bits, dest_dx, dest_dy);
+    }
+
+    pixman_region32_fini (&region);
+}
+
+PIXMAN_EXPORT void
+pixman_image_composite (pixman_op_t      op,
+                        pixman_image_t * src,
+                        pixman_image_t * mask,
+                        pixman_image_t * dest,
+                        int16_t          src_x,
+                        int16_t          src_y,
+                        int16_t          mask_x,
+                        int16_t          mask_y,
+                        int16_t          dest_x,
+                        int16_t          dest_y,
+                        uint16_t         width,
+                        uint16_t         height)
+{
+    pixman_image_composite32 (op, src, mask, dest, src_x, src_y, 
+                              mask_x, mask_y, dest_x, dest_y, width, height);
+}
+
+/*
+ * Work around GCC bug causing crashes in Mozilla with SSE2
+ *
+ * When using -msse, gcc generates movdqa instructions assuming that
+ * the stack is 16 byte aligned. Unfortunately some applications, such
+ * as Mozilla and Mono, end up aligning the stack to 4 bytes, which
+ * causes the movdqa instructions to fail.
+ *
+ * The __force_align_arg_pointer__ makes gcc generate a prologue that
+ * realigns the stack pointer to 16 bytes.
+ *
+ * On x86-64 this is not necessary because the standard ABI already
+ * calls for a 16 byte aligned stack.
+ *
+ * See https://bugs.freedesktop.org/show_bug.cgi?id=15693
+ */
+#if defined (USE_SSE2) && defined(__GNUC__) && !defined(__x86_64__) && !defined(__amd64__)
+__attribute__((__force_align_arg_pointer__))
+#endif
+PIXMAN_EXPORT void
+pixman_image_composite32 (pixman_op_t      op,
+                          pixman_image_t * src,
+                          pixman_image_t * mask,
+                          pixman_image_t * dest,
+                          int32_t          src_x,
+                          int32_t          src_y,
+                          int32_t          mask_x,
+                          int32_t          mask_y,
+                          int32_t          dest_x,
+                          int32_t          dest_y,
+                          int32_t          width,
+                          int32_t          height)
+{
+    _pixman_image_validate (src);
+    if (mask)
+	_pixman_image_validate (mask);
+    _pixman_image_validate (dest);
+
+    if (!imp)
+	imp = _pixman_choose_implementation ();
+
+    do_composite (imp, op,
+		  src, mask, dest,
+		  src_x, src_y,
+		  mask_x, mask_y,
+		  dest_x, dest_y,
+		  width, height);
+}
+
+PIXMAN_EXPORT pixman_bool_t
+pixman_blt (uint32_t *src_bits,
+            uint32_t *dst_bits,
+            int       src_stride,
+            int       dst_stride,
+            int       src_bpp,
+            int       dst_bpp,
+            int       src_x,
+            int       src_y,
+            int       dst_x,
+            int       dst_y,
+            int       width,
+            int       height)
+{
+    if (!imp)
+	imp = _pixman_choose_implementation ();
+
+    return _pixman_implementation_blt (imp, src_bits, dst_bits, src_stride, dst_stride,
+                                       src_bpp, dst_bpp,
+                                       src_x, src_y,
+                                       dst_x, dst_y,
+                                       width, height);
+}
+
+PIXMAN_EXPORT pixman_bool_t
+pixman_fill (uint32_t *bits,
+             int       stride,
+             int       bpp,
+             int       x,
+             int       y,
+             int       width,
+             int       height,
+             uint32_t xor)
+{
+    if (!imp)
+	imp = _pixman_choose_implementation ();
+
+    return _pixman_implementation_fill (imp, bits, stride, bpp, x, y, width, height, xor);
+}
+
+static uint32_t
+color_to_uint32 (const pixman_color_t *color)
+{
+    return
+        (color->alpha >> 8 << 24) |
+        (color->red >> 8 << 16) |
+        (color->green & 0xff00) |
+        (color->blue >> 8);
+}
+
+static pixman_bool_t
+color_to_pixel (pixman_color_t *     color,
+                uint32_t *           pixel,
+                pixman_format_code_t format)
+{
+    uint32_t c = color_to_uint32 (color);
+
+    if (!(format == PIXMAN_a8r8g8b8     ||
+          format == PIXMAN_x8r8g8b8     ||
+          format == PIXMAN_a8b8g8r8     ||
+          format == PIXMAN_x8b8g8r8     ||
+          format == PIXMAN_b8g8r8a8     ||
+          format == PIXMAN_b8g8r8x8     ||
+          format == PIXMAN_r5g6b5       ||
+          format == PIXMAN_b5g6r5       ||
+          format == PIXMAN_a8))
+    {
+	return FALSE;
+    }
+
+    if (PIXMAN_FORMAT_TYPE (format) == PIXMAN_TYPE_ABGR)
+    {
+	c = ((c & 0xff000000) >>  0) |
+	    ((c & 0x00ff0000) >> 16) |
+	    ((c & 0x0000ff00) >>  0) |
+	    ((c & 0x000000ff) << 16);
+    }
+    if (PIXMAN_FORMAT_TYPE (format) == PIXMAN_TYPE_BGRA)
+    {
+	c = ((c & 0xff000000) >> 24) |
+	    ((c & 0x00ff0000) >>  8) |
+	    ((c & 0x0000ff00) <<  8) |
+	    ((c & 0x000000ff) << 24);
+    }
+
+    if (format == PIXMAN_a8)
+	c = c >> 24;
+    else if (format == PIXMAN_r5g6b5 ||
+             format == PIXMAN_b5g6r5)
+	c = CONVERT_8888_TO_0565 (c);
+
+#if 0
+    printf ("color: %x %x %x %x\n", color->alpha, color->red, color->green, color->blue);
+    printf ("pixel: %x\n", c);
+#endif
+
+    *pixel = c;
+    return TRUE;
+}
+
+PIXMAN_EXPORT pixman_bool_t
+pixman_image_fill_rectangles (pixman_op_t                 op,
+                              pixman_image_t *            dest,
+                              pixman_color_t *            color,
+                              int                         n_rects,
+                              const pixman_rectangle16_t *rects)
+{
+    pixman_box32_t stack_boxes[6];
+    pixman_box32_t *boxes;
+    pixman_bool_t result;
+    int i;
+
+    if (n_rects > 6)
+    {
+        boxes = pixman_malloc_ab (sizeof (pixman_box32_t), n_rects);
+        if (boxes == NULL)
+            return FALSE;
+    }
+    else
+    {
+        boxes = stack_boxes;
+    }
+
+    for (i = 0; i < n_rects; ++i)
+    {
+        boxes[i].x1 = rects[i].x;
+        boxes[i].y1 = rects[i].y;
+        boxes[i].x2 = boxes[i].x1 + rects[i].width;
+        boxes[i].y2 = boxes[i].y1 + rects[i].height;
+    }
+
+    result = pixman_image_fill_boxes (op, dest, color, n_rects, boxes);
+
+    if (boxes != stack_boxes)
+        free (boxes);
+    
+    return result;
+}
+
+PIXMAN_EXPORT pixman_bool_t
+pixman_image_fill_boxes (pixman_op_t           op,
+                         pixman_image_t *      dest,
+                         pixman_color_t *      color,
+                         int                   n_boxes,
+                         const pixman_box32_t *boxes)
+{
+    pixman_image_t *solid;
+    pixman_color_t c;
+    int i;
+
+    _pixman_image_validate (dest);
+    
+    if (color->alpha == 0xffff)
+    {
+        if (op == PIXMAN_OP_OVER)
+            op = PIXMAN_OP_SRC;
+    }
+
+    if (op == PIXMAN_OP_CLEAR)
+    {
+        c.red = 0;
+        c.green = 0;
+        c.blue = 0;
+        c.alpha = 0;
+
+        color = &c;
+
+        op = PIXMAN_OP_SRC;
+    }
+
+    if (op == PIXMAN_OP_SRC)
+    {
+        uint32_t pixel;
+
+        if (color_to_pixel (color, &pixel, dest->bits.format))
+        {
+            pixman_region32_t fill_region;
+            int n_rects, j;
+            pixman_box32_t *rects;
+
+            if (!pixman_region32_init_rects (&fill_region, boxes, n_boxes))
+                return FALSE;
+
+            if (dest->common.have_clip_region)
+            {
+                if (!pixman_region32_intersect (&fill_region,
+                                                &fill_region,
+                                                &dest->common.clip_region))
+                    return FALSE;
+            }
+
+            rects = pixman_region32_rectangles (&fill_region, &n_rects);
+            for (j = 0; j < n_rects; ++j)
+            {
+                const pixman_box32_t *rect = &(rects[j]);
+                pixman_fill (dest->bits.bits, dest->bits.rowstride, PIXMAN_FORMAT_BPP (dest->bits.format),
+                             rect->x1, rect->y1, rect->x2 - rect->x1, rect->y2 - rect->y1,
+                             pixel);
+            }
+
+            pixman_region32_fini (&fill_region);
+            return TRUE;
+        }
+    }
+
+    solid = pixman_image_create_solid_fill (color);
+    if (!solid)
+        return FALSE;
+
+    for (i = 0; i < n_boxes; ++i)
+    {
+        const pixman_box32_t *box = &(boxes[i]);
+
+        pixman_image_composite32 (op, solid, NULL, dest,
+                                  0, 0, 0, 0,
+                                  box->x1, box->y1,
+                                  box->x2 - box->x1, box->y2 - box->y1);
+    }
+
+    pixman_image_unref (solid);
+
+    return TRUE;
+}
+
+/**
+ * pixman_version:
+ *
+ * Returns the version of the pixman library encoded in a single
+ * integer as per %PIXMAN_VERSION_ENCODE. The encoding ensures that
+ * later versions compare greater than earlier versions.
+ *
+ * A run-time comparison to check that pixman's version is greater than
+ * or equal to version X.Y.Z could be performed as follows:
+ *
+ * <informalexample><programlisting>
+ * if (pixman_version() >= PIXMAN_VERSION_ENCODE(X,Y,Z)) {...}
+ * </programlisting></informalexample>
+ *
+ * See also pixman_version_string() as well as the compile-time
+ * equivalents %PIXMAN_VERSION and %PIXMAN_VERSION_STRING.
+ *
+ * Return value: the encoded version.
+ **/
+PIXMAN_EXPORT int
+pixman_version (void)
+{
+    return PIXMAN_VERSION;
+}
+
+/**
+ * pixman_version_string:
+ *
+ * Returns the version of the pixman library as a human-readable string
+ * of the form "X.Y.Z".
+ *
+ * See also pixman_version() as well as the compile-time equivalents
+ * %PIXMAN_VERSION_STRING and %PIXMAN_VERSION.
+ *
+ * Return value: a string containing the version.
+ **/
+PIXMAN_EXPORT const char*
+pixman_version_string (void)
+{
+    return PIXMAN_VERSION_STRING;
+}
+
+/**
+ * pixman_format_supported_source:
+ * @format: A pixman_format_code_t format
+ *
+ * Return value: whether the provided format code is a supported
+ * format for a pixman surface used as a source in
+ * rendering.
+ *
+ * Currently, all pixman_format_code_t values are supported.
+ **/
+PIXMAN_EXPORT pixman_bool_t
+pixman_format_supported_source (pixman_format_code_t format)
+{
+    switch (format)
+    {
+    /* 32 bpp formats */
+    case PIXMAN_a2b10g10r10:
+    case PIXMAN_x2b10g10r10:
+    case PIXMAN_a2r10g10b10:
+    case PIXMAN_x2r10g10b10:
+    case PIXMAN_a8r8g8b8:
+    case PIXMAN_x8r8g8b8:
+    case PIXMAN_a8b8g8r8:
+    case PIXMAN_x8b8g8r8:
+    case PIXMAN_b8g8r8a8:
+    case PIXMAN_b8g8r8x8:
+    case PIXMAN_r8g8b8:
+    case PIXMAN_b8g8r8:
+    case PIXMAN_r5g6b5:
+    case PIXMAN_b5g6r5:
+    /* 16 bpp formats */
+    case PIXMAN_a1r5g5b5:
+    case PIXMAN_x1r5g5b5:
+    case PIXMAN_a1b5g5r5:
+    case PIXMAN_x1b5g5r5:
+    case PIXMAN_a4r4g4b4:
+    case PIXMAN_x4r4g4b4:
+    case PIXMAN_a4b4g4r4:
+    case PIXMAN_x4b4g4r4:
+    /* 8bpp formats */
+    case PIXMAN_a8:
+    case PIXMAN_r3g3b2:
+    case PIXMAN_b2g3r3:
+    case PIXMAN_a2r2g2b2:
+    case PIXMAN_a2b2g2r2:
+    case PIXMAN_c8:
+    case PIXMAN_g8:
+    case PIXMAN_x4a4:
+    /* Collides with PIXMAN_c8
+       case PIXMAN_x4c4:
+     */
+    /* Collides with PIXMAN_g8
+       case PIXMAN_x4g4:
+     */
+    /* 4bpp formats */
+    case PIXMAN_a4:
+    case PIXMAN_r1g2b1:
+    case PIXMAN_b1g2r1:
+    case PIXMAN_a1r1g1b1:
+    case PIXMAN_a1b1g1r1:
+    case PIXMAN_c4:
+    case PIXMAN_g4:
+    /* 1bpp formats */
+    case PIXMAN_a1:
+    case PIXMAN_g1:
+    /* YUV formats */
+    case PIXMAN_yuy2:
+    case PIXMAN_yv12:
+	return TRUE;
+
+    default:
+	return FALSE;
+    }
+}
+
+/**
+ * pixman_format_supported_destination:
+ * @format: A pixman_format_code_t format
+ *
+ * Return value: whether the provided format code is a supported
+ * format for a pixman surface used as a destination in
+ * rendering.
+ *
+ * Currently, all pixman_format_code_t values are supported
+ * except for the YUV formats.
+ **/
+PIXMAN_EXPORT pixman_bool_t
+pixman_format_supported_destination (pixman_format_code_t format)
+{
+    /* YUV formats cannot be written to at the moment */
+    if (format == PIXMAN_yuy2 || format == PIXMAN_yv12)
+	return FALSE;
+
+    return pixman_format_supported_source (format);
+}
+
+PIXMAN_EXPORT pixman_bool_t
+pixman_compute_composite_region (pixman_region16_t * region,
+                                 pixman_image_t *    src_image,
+                                 pixman_image_t *    mask_image,
+                                 pixman_image_t *    dst_image,
+                                 int16_t             src_x,
+                                 int16_t             src_y,
+                                 int16_t             mask_x,
+                                 int16_t             mask_y,
+                                 int16_t             dest_x,
+                                 int16_t             dest_y,
+                                 uint16_t            width,
+                                 uint16_t            height)
+{
+    pixman_region32_t r32;
+    pixman_bool_t retval;
+
+    pixman_region32_init (&r32);
+
+    retval = pixman_compute_composite_region32 (
+	&r32, src_image, mask_image, dst_image,
+	src_x, src_y, mask_x, mask_y, dest_x, dest_y,
+	width, height);
+
+    if (retval)
+    {
+	if (!pixman_region16_copy_from_region32 (region, &r32))
+	    retval = FALSE;
+    }
+
+    pixman_region32_fini (&r32);
+    return retval;
+}
diff --git a/pixman/pixman/pixman.h b/pixman/pixman/pixman.h
new file mode 100644
index 000000000..964d04ab9
--- /dev/null
+++ b/pixman/pixman/pixman.h
@@ -0,0 +1,936 @@
+/***********************************************************
+
+Copyright 1987, 1998  The Open Group
+
+Permission to use, copy, modify, distribute, and sell this software and its
+documentation for any purpose is hereby granted without fee, provided that
+the above copyright notice appear in all copies and that both that
+copyright notice and this permission notice appear in supporting
+documentation.
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+OPEN GROUP BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
+AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+Except as contained in this notice, the name of The Open Group shall not be
+used in advertising or otherwise to promote the sale, use or other dealings
+in this Software without prior written authorization from The Open Group.
+
+Copyright 1987 by Digital Equipment Corporation, Maynard, Massachusetts.
+
+                        All Rights Reserved
+
+Permission to use, copy, modify, and distribute this software and its
+documentation for any purpose and without fee is hereby granted,
+provided that the above copyright notice appear in all copies and that
+both that copyright notice and this permission notice appear in
+supporting documentation, and that the name of Digital not be
+used in advertising or publicity pertaining to distribution of the
+software without specific, written prior permission.
+
+DIGITAL DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING
+ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL
+DIGITAL BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR
+ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,
+WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION,
+ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS
+SOFTWARE.
+
+******************************************************************/
+/*
+ * Copyright © 1998, 2004 Keith Packard
+ * Copyright   2007 Red Hat, Inc.
+ *
+ * Permission to use, copy, modify, distribute, and sell this software and its
+ * documentation for any purpose is hereby granted without fee, provided that
+ * the above copyright notice appear in all copies and that both that
+ * copyright notice and this permission notice appear in supporting
+ * documentation, and that the name of Keith Packard not be used in
+ * advertising or publicity pertaining to distribution of the software without
+ * specific, written prior permission.  Keith Packard makes no
+ * representations about the suitability of this software for any purpose.  It
+ * is provided "as is" without express or implied warranty.
+ *
+ * KEITH PACKARD DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
+ * INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO
+ * EVENT SHALL KEITH PACKARD BE LIABLE FOR ANY SPECIAL, INDIRECT OR
+ * CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE,
+ * DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
+ * TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+ * PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#ifndef PIXMAN_H__
+#define PIXMAN_H__
+
+#include <pixman-version.h>
+
+#ifdef  __cplusplus
+#define PIXMAN_BEGIN_DECLS extern "C" {
+#define PIXMAN_END_DECLS }
+#else
+#define PIXMAN_BEGIN_DECLS
+#define PIXMAN_END_DECLS
+#endif
+
+PIXMAN_BEGIN_DECLS
+
+/*
+ * Standard integers
+ */
+
+#if !defined (PIXMAN_DONT_DEFINE_STDINT)
+
+#if defined (_SVR4) || defined (SVR4) || defined (__OpenBSD__) || defined (_sgi) || defined (__sun) || defined (sun) || defined (__digital__) || defined (__HP_cc)
+#  include <inttypes.h>
+/* VS 2010 (_MSC_VER 1600) has stdint.h */
+#elif defined (_MSC_VER) && _MSC_VER < 1600
+typedef __int8 int8_t;
+typedef unsigned __int8 uint8_t;
+typedef __int16 int16_t;
+typedef unsigned __int16 uint16_t;
+typedef __int32 int32_t;
+typedef unsigned __int32 uint32_t;
+typedef __int64 int64_t;
+typedef unsigned __int64 uint64_t;
+#elif defined (_AIX)
+#  include <sys/inttypes.h>
+#else
+#  include <stdint.h>
+#endif
+
+#endif
+
+/*
+ * Boolean
+ */
+typedef int pixman_bool_t;
+
+/*
+ * Fixpoint numbers
+ */
+typedef int64_t			pixman_fixed_32_32_t;
+typedef pixman_fixed_32_32_t	pixman_fixed_48_16_t;
+typedef uint32_t		pixman_fixed_1_31_t;
+typedef uint32_t		pixman_fixed_1_16_t;
+typedef int32_t			pixman_fixed_16_16_t;
+typedef pixman_fixed_16_16_t	pixman_fixed_t;
+
+#define pixman_fixed_e			((pixman_fixed_t) 1)
+#define pixman_fixed_1			(pixman_int_to_fixed(1))
+#define pixman_fixed_1_minus_e		(pixman_fixed_1 - pixman_fixed_e)
+#define pixman_fixed_minus_1		(pixman_int_to_fixed(-1))
+#define pixman_fixed_to_int(f)		((int) ((f) >> 16))
+#define pixman_int_to_fixed(i)		((pixman_fixed_t) ((i) << 16))
+#define pixman_fixed_to_double(f)	(double) ((f) / (double) pixman_fixed_1)
+#define pixman_double_to_fixed(d)	((pixman_fixed_t) ((d) * 65536.0))
+#define pixman_fixed_frac(f)		((f) & pixman_fixed_1_minus_e)
+#define pixman_fixed_floor(f)		((f) & ~pixman_fixed_1_minus_e)
+#define pixman_fixed_ceil(f)		pixman_fixed_floor ((f) + pixman_fixed_1_minus_e)
+#define pixman_fixed_fraction(f)	((f) & pixman_fixed_1_minus_e)
+#define pixman_fixed_mod_2(f)		((f) & (pixman_fixed1 | pixman_fixed_1_minus_e))
+#define pixman_max_fixed_48_16		((pixman_fixed_48_16_t) 0x7fffffff)
+#define pixman_min_fixed_48_16		(-((pixman_fixed_48_16_t) 1 << 31))
+
+/*
+ * Misc structs
+ */
+typedef struct pixman_color pixman_color_t;
+typedef struct pixman_point_fixed pixman_point_fixed_t;
+typedef struct pixman_line_fixed pixman_line_fixed_t;
+typedef struct pixman_vector pixman_vector_t;
+typedef struct pixman_transform pixman_transform_t;
+
+struct pixman_color
+{
+    uint16_t	red;
+    uint16_t    green;
+    uint16_t    blue;
+    uint16_t    alpha;
+};
+
+struct pixman_point_fixed
+{
+    pixman_fixed_t	x;
+    pixman_fixed_t	y;
+};
+
+struct pixman_line_fixed
+{
+    pixman_point_fixed_t	p1, p2;
+};
+
+/*
+ * Fixed point matrices
+ */
+
+struct pixman_vector
+{
+    pixman_fixed_t	vector[3];
+};
+
+struct pixman_transform
+{
+    pixman_fixed_t	matrix[3][3];
+};
+
+/* forward declaration (sorry) */
+struct pixman_box16;
+typedef  union pixman_image		pixman_image_t;
+
+void          pixman_transform_init_identity    (struct pixman_transform       *matrix);
+pixman_bool_t pixman_transform_point_3d         (const struct pixman_transform *transform,
+						 struct pixman_vector          *vector);
+pixman_bool_t pixman_transform_point            (const struct pixman_transform *transform,
+						 struct pixman_vector          *vector);
+pixman_bool_t pixman_transform_multiply         (struct pixman_transform       *dst,
+						 const struct pixman_transform *l,
+						 const struct pixman_transform *r);
+void          pixman_transform_init_scale       (struct pixman_transform       *t,
+						 pixman_fixed_t                 sx,
+						 pixman_fixed_t                 sy);
+pixman_bool_t pixman_transform_scale            (struct pixman_transform       *forward,
+						 struct pixman_transform       *reverse,
+						 pixman_fixed_t                 sx,
+						 pixman_fixed_t                 sy);
+void          pixman_transform_init_rotate      (struct pixman_transform       *t,
+						 pixman_fixed_t                 cos,
+						 pixman_fixed_t                 sin);
+pixman_bool_t pixman_transform_rotate           (struct pixman_transform       *forward,
+						 struct pixman_transform       *reverse,
+						 pixman_fixed_t                 c,
+						 pixman_fixed_t                 s);
+void          pixman_transform_init_translate   (struct pixman_transform       *t,
+						 pixman_fixed_t                 tx,
+						 pixman_fixed_t                 ty);
+pixman_bool_t pixman_transform_translate        (struct pixman_transform       *forward,
+						 struct pixman_transform       *reverse,
+						 pixman_fixed_t                 tx,
+						 pixman_fixed_t                 ty);
+pixman_bool_t pixman_transform_bounds           (const struct pixman_transform *matrix,
+						 struct pixman_box16           *b);
+pixman_bool_t pixman_transform_invert           (struct pixman_transform       *dst,
+						 const struct pixman_transform *src);
+pixman_bool_t pixman_transform_is_identity      (const struct pixman_transform *t);
+pixman_bool_t pixman_transform_is_scale         (const struct pixman_transform *t);
+pixman_bool_t pixman_transform_is_int_translate (const struct pixman_transform *t);
+pixman_bool_t pixman_transform_is_inverse       (const struct pixman_transform *a,
+						 const struct pixman_transform *b);
+
+/*
+ * Floating point matrices
+ */
+struct pixman_f_vector
+{
+    double  v[3];
+};
+
+struct pixman_f_transform
+{
+    double  m[3][3];
+};
+
+pixman_bool_t pixman_transform_from_pixman_f_transform (struct pixman_transform         *t,
+							const struct pixman_f_transform *ft);
+void          pixman_f_transform_from_pixman_transform (struct pixman_f_transform       *ft,
+							const struct pixman_transform   *t);
+pixman_bool_t pixman_f_transform_invert                (struct pixman_f_transform       *dst,
+							const struct pixman_f_transform *src);
+pixman_bool_t pixman_f_transform_point                 (const struct pixman_f_transform *t,
+							struct pixman_f_vector          *v);
+void          pixman_f_transform_point_3d              (const struct pixman_f_transform *t,
+							struct pixman_f_vector          *v);
+void          pixman_f_transform_multiply              (struct pixman_f_transform       *dst,
+							const struct pixman_f_transform *l,
+							const struct pixman_f_transform *r);
+void          pixman_f_transform_init_scale            (struct pixman_f_transform       *t,
+							double                           sx,
+							double                           sy);
+pixman_bool_t pixman_f_transform_scale                 (struct pixman_f_transform       *forward,
+							struct pixman_f_transform       *reverse,
+							double                           sx,
+							double                           sy);
+void          pixman_f_transform_init_rotate           (struct pixman_f_transform       *t,
+							double                           cos,
+							double                           sin);
+pixman_bool_t pixman_f_transform_rotate                (struct pixman_f_transform       *forward,
+							struct pixman_f_transform       *reverse,
+							double                           c,
+							double                           s);
+void          pixman_f_transform_init_translate        (struct pixman_f_transform       *t,
+							double                           tx,
+							double                           ty);
+pixman_bool_t pixman_f_transform_translate             (struct pixman_f_transform       *forward,
+							struct pixman_f_transform       *reverse,
+							double                           tx,
+							double                           ty);
+pixman_bool_t pixman_f_transform_bounds                (const struct pixman_f_transform *t,
+							struct pixman_box16             *b);
+void          pixman_f_transform_init_identity         (struct pixman_f_transform       *t);
+
+typedef enum
+{
+    PIXMAN_REPEAT_NONE,
+    PIXMAN_REPEAT_NORMAL,
+    PIXMAN_REPEAT_PAD,
+    PIXMAN_REPEAT_REFLECT
+} pixman_repeat_t;
+
+typedef enum
+{
+    PIXMAN_FILTER_FAST,
+    PIXMAN_FILTER_GOOD,
+    PIXMAN_FILTER_BEST,
+    PIXMAN_FILTER_NEAREST,
+    PIXMAN_FILTER_BILINEAR,
+    PIXMAN_FILTER_CONVOLUTION
+} pixman_filter_t;
+
+typedef enum
+{
+    PIXMAN_OP_CLEAR			= 0x00,
+    PIXMAN_OP_SRC			= 0x01,
+    PIXMAN_OP_DST			= 0x02,
+    PIXMAN_OP_OVER			= 0x03,
+    PIXMAN_OP_OVER_REVERSE		= 0x04,
+    PIXMAN_OP_IN			= 0x05,
+    PIXMAN_OP_IN_REVERSE		= 0x06,
+    PIXMAN_OP_OUT			= 0x07,
+    PIXMAN_OP_OUT_REVERSE		= 0x08,
+    PIXMAN_OP_ATOP			= 0x09,
+    PIXMAN_OP_ATOP_REVERSE		= 0x0a,
+    PIXMAN_OP_XOR			= 0x0b,
+    PIXMAN_OP_ADD			= 0x0c,
+    PIXMAN_OP_SATURATE			= 0x0d,
+
+    PIXMAN_OP_DISJOINT_CLEAR		= 0x10,
+    PIXMAN_OP_DISJOINT_SRC		= 0x11,
+    PIXMAN_OP_DISJOINT_DST		= 0x12,
+    PIXMAN_OP_DISJOINT_OVER		= 0x13,
+    PIXMAN_OP_DISJOINT_OVER_REVERSE	= 0x14,
+    PIXMAN_OP_DISJOINT_IN		= 0x15,
+    PIXMAN_OP_DISJOINT_IN_REVERSE	= 0x16,
+    PIXMAN_OP_DISJOINT_OUT		= 0x17,
+    PIXMAN_OP_DISJOINT_OUT_REVERSE	= 0x18,
+    PIXMAN_OP_DISJOINT_ATOP		= 0x19,
+    PIXMAN_OP_DISJOINT_ATOP_REVERSE	= 0x1a,
+    PIXMAN_OP_DISJOINT_XOR		= 0x1b,
+
+    PIXMAN_OP_CONJOINT_CLEAR		= 0x20,
+    PIXMAN_OP_CONJOINT_SRC		= 0x21,
+    PIXMAN_OP_CONJOINT_DST		= 0x22,
+    PIXMAN_OP_CONJOINT_OVER		= 0x23,
+    PIXMAN_OP_CONJOINT_OVER_REVERSE	= 0x24,
+    PIXMAN_OP_CONJOINT_IN		= 0x25,
+    PIXMAN_OP_CONJOINT_IN_REVERSE	= 0x26,
+    PIXMAN_OP_CONJOINT_OUT		= 0x27,
+    PIXMAN_OP_CONJOINT_OUT_REVERSE	= 0x28,
+    PIXMAN_OP_CONJOINT_ATOP		= 0x29,
+    PIXMAN_OP_CONJOINT_ATOP_REVERSE	= 0x2a,
+    PIXMAN_OP_CONJOINT_XOR		= 0x2b,
+
+    PIXMAN_OP_MULTIPLY                  = 0x30,
+    PIXMAN_OP_SCREEN                    = 0x31,
+    PIXMAN_OP_OVERLAY                   = 0x32,
+    PIXMAN_OP_DARKEN                    = 0x33,
+    PIXMAN_OP_LIGHTEN                   = 0x34,
+    PIXMAN_OP_COLOR_DODGE               = 0x35,
+    PIXMAN_OP_COLOR_BURN                = 0x36,
+    PIXMAN_OP_HARD_LIGHT                = 0x37,
+    PIXMAN_OP_SOFT_LIGHT                = 0x38,
+    PIXMAN_OP_DIFFERENCE                = 0x39,
+    PIXMAN_OP_EXCLUSION                 = 0x3a,
+    PIXMAN_OP_HSL_HUE			= 0x3b,
+    PIXMAN_OP_HSL_SATURATION		= 0x3c,
+    PIXMAN_OP_HSL_COLOR			= 0x3d,
+    PIXMAN_OP_HSL_LUMINOSITY		= 0x3e
+
+#ifdef PIXMAN_USE_INTERNAL_API
+    ,
+    PIXMAN_N_OPERATORS,
+    PIXMAN_OP_NONE = PIXMAN_N_OPERATORS
+#endif
+} pixman_op_t;
+
+/*
+ * Regions
+ */
+typedef struct pixman_region16_data	pixman_region16_data_t;
+typedef struct pixman_box16		pixman_box16_t;
+typedef struct pixman_rectangle16	pixman_rectangle16_t;
+typedef struct pixman_region16		pixman_region16_t;
+
+struct pixman_region16_data {
+    long		size;
+    long		numRects;
+/*  pixman_box16_t	rects[size];   in memory but not explicitly declared */
+};
+
+struct pixman_rectangle16
+{
+    int16_t	x, y;
+    uint16_t	width, height;
+};
+
+struct pixman_box16
+{
+    int16_t x1, y1, x2, y2;
+};
+
+struct pixman_region16
+{
+    pixman_box16_t          extents;
+    pixman_region16_data_t *data;
+};
+
+typedef enum
+{
+    PIXMAN_REGION_OUT,
+    PIXMAN_REGION_IN,
+    PIXMAN_REGION_PART
+} pixman_region_overlap_t;
+
+/* This function exists only to make it possible to preserve
+ * the X ABI - it should go away at first opportunity.
+ */
+void pixman_region_set_static_pointers (pixman_box16_t         *empty_box,
+					pixman_region16_data_t *empty_data,
+					pixman_region16_data_t *broken_data);
+
+/* creation/destruction */
+void                    pixman_region_init               (pixman_region16_t *region);
+void                    pixman_region_init_rect          (pixman_region16_t *region,
+							  int                x,
+							  int                y,
+							  unsigned int       width,
+							  unsigned int       height);
+pixman_bool_t           pixman_region_init_rects         (pixman_region16_t *region,
+							  const pixman_box16_t *boxes,
+							  int                count);
+void                    pixman_region_init_with_extents  (pixman_region16_t *region,
+							  pixman_box16_t    *extents);
+void                    pixman_region_init_from_image    (pixman_region16_t *region,
+							  pixman_image_t    *image);
+void                    pixman_region_fini               (pixman_region16_t *region);
+
+
+
+/* manipulation */
+void                    pixman_region_translate          (pixman_region16_t *region,
+							  int                x,
+							  int                y);
+pixman_bool_t           pixman_region_copy               (pixman_region16_t *dest,
+							  pixman_region16_t *source);
+pixman_bool_t           pixman_region_intersect          (pixman_region16_t *new_reg,
+							  pixman_region16_t *reg1,
+							  pixman_region16_t *reg2);
+pixman_bool_t           pixman_region_union              (pixman_region16_t *new_reg,
+							  pixman_region16_t *reg1,
+							  pixman_region16_t *reg2);
+pixman_bool_t           pixman_region_union_rect         (pixman_region16_t *dest,
+							  pixman_region16_t *source,
+							  int                x,
+							  int                y,
+							  unsigned int       width,
+							  unsigned int       height);
+pixman_bool_t           pixman_region_subtract           (pixman_region16_t *reg_d,
+							  pixman_region16_t *reg_m,
+							  pixman_region16_t *reg_s);
+pixman_bool_t           pixman_region_inverse            (pixman_region16_t *new_reg,
+							  pixman_region16_t *reg1,
+							  pixman_box16_t    *inv_rect);
+pixman_bool_t           pixman_region_contains_point     (pixman_region16_t *region,
+							  int                x,
+							  int                y,
+							  pixman_box16_t    *box);
+pixman_region_overlap_t pixman_region_contains_rectangle (pixman_region16_t *region,
+							  pixman_box16_t    *prect);
+pixman_bool_t           pixman_region_not_empty          (pixman_region16_t *region);
+pixman_box16_t *        pixman_region_extents            (pixman_region16_t *region);
+int                     pixman_region_n_rects            (pixman_region16_t *region);
+pixman_box16_t *        pixman_region_rectangles         (pixman_region16_t *region,
+							  int               *n_rects);
+pixman_bool_t           pixman_region_equal              (pixman_region16_t *region1,
+							  pixman_region16_t *region2);
+pixman_bool_t           pixman_region_selfcheck          (pixman_region16_t *region);
+void                    pixman_region_reset              (pixman_region16_t *region,
+							  pixman_box16_t    *box);
+/*
+ * 32 bit regions
+ */
+typedef struct pixman_region32_data	pixman_region32_data_t;
+typedef struct pixman_box32		pixman_box32_t;
+typedef struct pixman_rectangle32	pixman_rectangle32_t;
+typedef struct pixman_region32		pixman_region32_t;
+
+struct pixman_region32_data {
+    long		size;
+    long		numRects;
+/*  pixman_box32_t	rects[size];   in memory but not explicitly declared */
+};
+
+struct pixman_rectangle32
+{
+    int32_t x, y;
+    uint32_t width, height;
+};
+
+struct pixman_box32
+{
+    int32_t x1, y1, x2, y2;
+};
+
+struct pixman_region32
+{
+    pixman_box32_t          extents;
+    pixman_region32_data_t  *data;
+};
+
+/* creation/destruction */
+void                    pixman_region32_init               (pixman_region32_t *region);
+void                    pixman_region32_init_rect          (pixman_region32_t *region,
+							    int                x,
+							    int                y,
+							    unsigned int       width,
+							    unsigned int       height);
+pixman_bool_t           pixman_region32_init_rects         (pixman_region32_t *region,
+							    const pixman_box32_t *boxes,
+							    int                count);
+void                    pixman_region32_init_with_extents  (pixman_region32_t *region,
+							    pixman_box32_t    *extents);
+void                    pixman_region32_init_from_image    (pixman_region32_t *region,
+							    pixman_image_t    *image);
+void                    pixman_region32_fini               (pixman_region32_t *region);
+
+
+/* manipulation */
+void                    pixman_region32_translate          (pixman_region32_t *region,
+							    int                x,
+							    int                y);
+pixman_bool_t           pixman_region32_copy               (pixman_region32_t *dest,
+							    pixman_region32_t *source);
+pixman_bool_t           pixman_region32_intersect          (pixman_region32_t *new_reg,
+							    pixman_region32_t *reg1,
+							    pixman_region32_t *reg2);
+pixman_bool_t           pixman_region32_union              (pixman_region32_t *new_reg,
+							    pixman_region32_t *reg1,
+							    pixman_region32_t *reg2);
+pixman_bool_t           pixman_region32_union_rect         (pixman_region32_t *dest,
+							    pixman_region32_t *source,
+							    int                x,
+							    int                y,
+							    unsigned int       width,
+							    unsigned int       height);
+pixman_bool_t           pixman_region32_subtract           (pixman_region32_t *reg_d,
+							    pixman_region32_t *reg_m,
+							    pixman_region32_t *reg_s);
+pixman_bool_t           pixman_region32_inverse            (pixman_region32_t *new_reg,
+							    pixman_region32_t *reg1,
+							    pixman_box32_t    *inv_rect);
+pixman_bool_t           pixman_region32_contains_point     (pixman_region32_t *region,
+							    int                x,
+							    int                y,
+							    pixman_box32_t    *box);
+pixman_region_overlap_t pixman_region32_contains_rectangle (pixman_region32_t *region,
+							    pixman_box32_t    *prect);
+pixman_bool_t           pixman_region32_not_empty          (pixman_region32_t *region);
+pixman_box32_t *        pixman_region32_extents            (pixman_region32_t *region);
+int                     pixman_region32_n_rects            (pixman_region32_t *region);
+pixman_box32_t *        pixman_region32_rectangles         (pixman_region32_t *region,
+							    int               *n_rects);
+pixman_bool_t           pixman_region32_equal              (pixman_region32_t *region1,
+							    pixman_region32_t *region2);
+pixman_bool_t           pixman_region32_selfcheck          (pixman_region32_t *region);
+void                    pixman_region32_reset              (pixman_region32_t *region,
+							    pixman_box32_t    *box);
+
+
+/* Copy / Fill / Misc */
+pixman_bool_t pixman_blt                (uint32_t           *src_bits,
+					 uint32_t           *dst_bits,
+					 int                 src_stride,
+					 int                 dst_stride,
+					 int                 src_bpp,
+					 int                 dst_bpp,
+					 int                 src_x,
+					 int                 src_y,
+					 int                 dst_x,
+					 int                 dst_y,
+					 int                 width,
+					 int                 height);
+pixman_bool_t pixman_fill               (uint32_t           *bits,
+					 int                 stride,
+					 int                 bpp,
+					 int                 x,
+					 int                 y,
+					 int                 width,
+					 int                 height,
+					 uint32_t            _xor);
+
+int           pixman_version            (void);
+const char*   pixman_version_string     (void);
+
+/*
+ * Images
+ */
+typedef struct pixman_indexed		pixman_indexed_t;
+typedef struct pixman_gradient_stop	pixman_gradient_stop_t;
+
+typedef uint32_t (* pixman_read_memory_func_t) (const void *src, int size);
+typedef void     (* pixman_write_memory_func_t) (void *dst, uint32_t value, int size);
+
+typedef void     (* pixman_image_destroy_func_t) (pixman_image_t *image, void *data);
+
+struct pixman_gradient_stop {
+    pixman_fixed_t x;
+    pixman_color_t color;
+};
+
+#define PIXMAN_MAX_INDEXED  256 /* XXX depth must be <= 8 */
+
+#if PIXMAN_MAX_INDEXED <= 256
+typedef uint8_t pixman_index_type;
+#endif
+
+struct pixman_indexed
+{
+    pixman_bool_t       color;
+    uint32_t		rgba[PIXMAN_MAX_INDEXED];
+    pixman_index_type	ent[32768];
+};
+
+/*
+ * While the protocol is generous in format support, the
+ * sample implementation allows only packed RGB and GBR
+ * representations for data to simplify software rendering,
+ */
+#define PIXMAN_FORMAT(bpp,type,a,r,g,b)	(((bpp) << 24) |  \
+					 ((type) << 16) | \
+					 ((a) << 12) |	  \
+					 ((r) << 8) |	  \
+					 ((g) << 4) |	  \
+					 ((b)))
+
+#define PIXMAN_FORMAT_BPP(f)	(((f) >> 24)       )
+#define PIXMAN_FORMAT_TYPE(f)	(((f) >> 16) & 0xff)
+#define PIXMAN_FORMAT_A(f)	(((f) >> 12) & 0x0f)
+#define PIXMAN_FORMAT_R(f)	(((f) >>  8) & 0x0f)
+#define PIXMAN_FORMAT_G(f)	(((f) >>  4) & 0x0f)
+#define PIXMAN_FORMAT_B(f)	(((f)      ) & 0x0f)
+#define PIXMAN_FORMAT_RGB(f)	(((f)      ) & 0xfff)
+#define PIXMAN_FORMAT_VIS(f)	(((f)      ) & 0xffff)
+#define PIXMAN_FORMAT_DEPTH(f)	(PIXMAN_FORMAT_A(f) +	\
+				 PIXMAN_FORMAT_R(f) +	\
+				 PIXMAN_FORMAT_G(f) +	\
+				 PIXMAN_FORMAT_B(f))
+
+#define PIXMAN_TYPE_OTHER	0
+#define PIXMAN_TYPE_A		1
+#define PIXMAN_TYPE_ARGB	2
+#define PIXMAN_TYPE_ABGR	3
+#define PIXMAN_TYPE_COLOR	4
+#define PIXMAN_TYPE_GRAY	5
+#define PIXMAN_TYPE_YUY2	6
+#define PIXMAN_TYPE_YV12	7
+#define PIXMAN_TYPE_BGRA	8
+
+#define PIXMAN_FORMAT_COLOR(f)				\
+	(PIXMAN_FORMAT_TYPE(f) == PIXMAN_TYPE_ARGB ||	\
+	 PIXMAN_FORMAT_TYPE(f) == PIXMAN_TYPE_ABGR ||	\
+	 PIXMAN_FORMAT_TYPE(f) == PIXMAN_TYPE_BGRA)
+
+/* 32bpp formats */
+typedef enum {
+    PIXMAN_a8r8g8b8 =	 PIXMAN_FORMAT(32,PIXMAN_TYPE_ARGB,8,8,8,8),
+    PIXMAN_x8r8g8b8 =	 PIXMAN_FORMAT(32,PIXMAN_TYPE_ARGB,0,8,8,8),
+    PIXMAN_a8b8g8r8 =	 PIXMAN_FORMAT(32,PIXMAN_TYPE_ABGR,8,8,8,8),
+    PIXMAN_x8b8g8r8 =	 PIXMAN_FORMAT(32,PIXMAN_TYPE_ABGR,0,8,8,8),
+    PIXMAN_b8g8r8a8 =	 PIXMAN_FORMAT(32,PIXMAN_TYPE_BGRA,8,8,8,8),
+    PIXMAN_b8g8r8x8 =	 PIXMAN_FORMAT(32,PIXMAN_TYPE_BGRA,0,8,8,8),
+    PIXMAN_x2r10g10b10 = PIXMAN_FORMAT(32,PIXMAN_TYPE_ARGB,0,10,10,10),
+    PIXMAN_a2r10g10b10 = PIXMAN_FORMAT(32,PIXMAN_TYPE_ARGB,2,10,10,10),
+    PIXMAN_x2b10g10r10 = PIXMAN_FORMAT(32,PIXMAN_TYPE_ABGR,0,10,10,10),
+    PIXMAN_a2b10g10r10 = PIXMAN_FORMAT(32,PIXMAN_TYPE_ABGR,2,10,10,10),
+
+/* 24bpp formats */
+    PIXMAN_r8g8b8 =	 PIXMAN_FORMAT(24,PIXMAN_TYPE_ARGB,0,8,8,8),
+    PIXMAN_b8g8r8 =	 PIXMAN_FORMAT(24,PIXMAN_TYPE_ABGR,0,8,8,8),
+
+/* 16bpp formats */
+    PIXMAN_r5g6b5 =	 PIXMAN_FORMAT(16,PIXMAN_TYPE_ARGB,0,5,6,5),
+    PIXMAN_b5g6r5 =	 PIXMAN_FORMAT(16,PIXMAN_TYPE_ABGR,0,5,6,5),
+
+    PIXMAN_a1r5g5b5 =	 PIXMAN_FORMAT(16,PIXMAN_TYPE_ARGB,1,5,5,5),
+    PIXMAN_x1r5g5b5 =	 PIXMAN_FORMAT(16,PIXMAN_TYPE_ARGB,0,5,5,5),
+    PIXMAN_a1b5g5r5 =	 PIXMAN_FORMAT(16,PIXMAN_TYPE_ABGR,1,5,5,5),
+    PIXMAN_x1b5g5r5 =	 PIXMAN_FORMAT(16,PIXMAN_TYPE_ABGR,0,5,5,5),
+    PIXMAN_a4r4g4b4 =	 PIXMAN_FORMAT(16,PIXMAN_TYPE_ARGB,4,4,4,4),
+    PIXMAN_x4r4g4b4 =	 PIXMAN_FORMAT(16,PIXMAN_TYPE_ARGB,0,4,4,4),
+    PIXMAN_a4b4g4r4 =	 PIXMAN_FORMAT(16,PIXMAN_TYPE_ABGR,4,4,4,4),
+    PIXMAN_x4b4g4r4 =	 PIXMAN_FORMAT(16,PIXMAN_TYPE_ABGR,0,4,4,4),
+
+/* 8bpp formats */
+    PIXMAN_a8 =		 PIXMAN_FORMAT(8,PIXMAN_TYPE_A,8,0,0,0),
+    PIXMAN_r3g3b2 =	 PIXMAN_FORMAT(8,PIXMAN_TYPE_ARGB,0,3,3,2),
+    PIXMAN_b2g3r3 =	 PIXMAN_FORMAT(8,PIXMAN_TYPE_ABGR,0,3,3,2),
+    PIXMAN_a2r2g2b2 =	 PIXMAN_FORMAT(8,PIXMAN_TYPE_ARGB,2,2,2,2),
+    PIXMAN_a2b2g2r2 =	 PIXMAN_FORMAT(8,PIXMAN_TYPE_ABGR,2,2,2,2),
+
+    PIXMAN_c8 =		 PIXMAN_FORMAT(8,PIXMAN_TYPE_COLOR,0,0,0,0),
+    PIXMAN_g8 =		 PIXMAN_FORMAT(8,PIXMAN_TYPE_GRAY,0,0,0,0),
+
+    PIXMAN_x4a4 =	 PIXMAN_FORMAT(8,PIXMAN_TYPE_A,4,0,0,0),
+
+    PIXMAN_x4c4 =	 PIXMAN_FORMAT(8,PIXMAN_TYPE_COLOR,0,0,0,0),
+    PIXMAN_x4g4 =	 PIXMAN_FORMAT(8,PIXMAN_TYPE_GRAY,0,0,0,0),
+
+/* 4bpp formats */
+    PIXMAN_a4 =		 PIXMAN_FORMAT(4,PIXMAN_TYPE_A,4,0,0,0),
+    PIXMAN_r1g2b1 =	 PIXMAN_FORMAT(4,PIXMAN_TYPE_ARGB,0,1,2,1),
+    PIXMAN_b1g2r1 =	 PIXMAN_FORMAT(4,PIXMAN_TYPE_ABGR,0,1,2,1),
+    PIXMAN_a1r1g1b1 =	 PIXMAN_FORMAT(4,PIXMAN_TYPE_ARGB,1,1,1,1),
+    PIXMAN_a1b1g1r1 =	 PIXMAN_FORMAT(4,PIXMAN_TYPE_ABGR,1,1,1,1),
+
+    PIXMAN_c4 =		 PIXMAN_FORMAT(4,PIXMAN_TYPE_COLOR,0,0,0,0),
+    PIXMAN_g4 =		 PIXMAN_FORMAT(4,PIXMAN_TYPE_GRAY,0,0,0,0),
+
+/* 1bpp formats */
+    PIXMAN_a1 =		 PIXMAN_FORMAT(1,PIXMAN_TYPE_A,1,0,0,0),
+
+    PIXMAN_g1 =		 PIXMAN_FORMAT(1,PIXMAN_TYPE_GRAY,0,0,0,0),
+
+/* YUV formats */
+    PIXMAN_yuy2 =	 PIXMAN_FORMAT(16,PIXMAN_TYPE_YUY2,0,0,0,0),
+    PIXMAN_yv12 =	 PIXMAN_FORMAT(12,PIXMAN_TYPE_YV12,0,0,0,0)
+} pixman_format_code_t;
+
+/* Querying supported format values. */
+pixman_bool_t pixman_format_supported_destination (pixman_format_code_t format);
+pixman_bool_t pixman_format_supported_source      (pixman_format_code_t format);
+
+/* Constructors */
+pixman_image_t *pixman_image_create_solid_fill       (pixman_color_t               *color);
+pixman_image_t *pixman_image_create_linear_gradient  (pixman_point_fixed_t         *p1,
+						      pixman_point_fixed_t         *p2,
+						      const pixman_gradient_stop_t *stops,
+						      int                           n_stops);
+pixman_image_t *pixman_image_create_radial_gradient  (pixman_point_fixed_t         *inner,
+						      pixman_point_fixed_t         *outer,
+						      pixman_fixed_t                inner_radius,
+						      pixman_fixed_t                outer_radius,
+						      const pixman_gradient_stop_t *stops,
+						      int                           n_stops);
+pixman_image_t *pixman_image_create_conical_gradient (pixman_point_fixed_t         *center,
+						      pixman_fixed_t                angle,
+						      const pixman_gradient_stop_t *stops,
+						      int                           n_stops);
+pixman_image_t *pixman_image_create_bits             (pixman_format_code_t          format,
+						      int                           width,
+						      int                           height,
+						      uint32_t                     *bits,
+						      int                           rowstride_bytes);
+
+/* Destructor */
+pixman_image_t *pixman_image_ref                     (pixman_image_t               *image);
+pixman_bool_t   pixman_image_unref                   (pixman_image_t               *image);
+
+void		pixman_image_set_destroy_function    (pixman_image_t		   *image,
+						      pixman_image_destroy_func_t   function,
+						      void			   *data);
+void *		pixman_image_get_destroy_data        (pixman_image_t		   *image);
+
+/* Set properties */
+pixman_bool_t   pixman_image_set_clip_region         (pixman_image_t               *image,
+						      pixman_region16_t            *region);
+pixman_bool_t   pixman_image_set_clip_region32       (pixman_image_t               *image,
+						      pixman_region32_t            *region);
+void		pixman_image_set_has_client_clip     (pixman_image_t               *image,
+						      pixman_bool_t		    clien_clip);
+pixman_bool_t   pixman_image_set_transform           (pixman_image_t               *image,
+						      const pixman_transform_t     *transform);
+void            pixman_image_set_repeat              (pixman_image_t               *image,
+						      pixman_repeat_t               repeat);
+pixman_bool_t   pixman_image_set_filter              (pixman_image_t               *image,
+						      pixman_filter_t               filter,
+						      const pixman_fixed_t         *filter_params,
+						      int                           n_filter_params);
+void		pixman_image_set_source_clipping     (pixman_image_t		   *image,
+						      pixman_bool_t                 source_clipping);
+void            pixman_image_set_alpha_map           (pixman_image_t               *image,
+						      pixman_image_t               *alpha_map,
+						      int16_t                       x,
+						      int16_t                       y);
+void            pixman_image_set_component_alpha     (pixman_image_t               *image,
+						      pixman_bool_t                 component_alpha);
+void		pixman_image_set_accessors	     (pixman_image_t		   *image,
+						      pixman_read_memory_func_t	    read_func,
+						      pixman_write_memory_func_t    write_func);
+void		pixman_image_set_indexed	     (pixman_image_t		   *image,
+						      const pixman_indexed_t	   *indexed);
+uint32_t       *pixman_image_get_data                (pixman_image_t               *image);
+int		pixman_image_get_width               (pixman_image_t               *image);
+int             pixman_image_get_height              (pixman_image_t               *image);
+int		pixman_image_get_stride              (pixman_image_t               *image); /* in bytes */
+int		pixman_image_get_depth               (pixman_image_t		   *image);
+pixman_bool_t	pixman_image_fill_rectangles	     (pixman_op_t		    op,
+						      pixman_image_t		   *image,
+						      pixman_color_t		   *color,
+						      int			    n_rects,
+						      const pixman_rectangle16_t   *rects);
+pixman_bool_t   pixman_image_fill_boxes              (pixman_op_t                   op,
+                                                      pixman_image_t               *dest,
+                                                      pixman_color_t               *color,
+                                                      int                           n_boxes,
+                                                      const pixman_box32_t         *boxes);
+
+/* Composite */
+pixman_bool_t pixman_compute_composite_region (pixman_region16_t *region,
+					       pixman_image_t    *src_image,
+					       pixman_image_t    *mask_image,
+					       pixman_image_t    *dst_image,
+					       int16_t            src_x,
+					       int16_t            src_y,
+					       int16_t            mask_x,
+					       int16_t            mask_y,
+					       int16_t            dest_x,
+					       int16_t            dest_y,
+					       uint16_t           width,
+					       uint16_t           height);
+void          pixman_image_composite          (pixman_op_t        op,
+					       pixman_image_t    *src,
+					       pixman_image_t    *mask,
+					       pixman_image_t    *dest,
+					       int16_t            src_x,
+					       int16_t            src_y,
+					       int16_t            mask_x,
+					       int16_t            mask_y,
+					       int16_t            dest_x,
+					       int16_t            dest_y,
+					       uint16_t           width,
+					       uint16_t           height);
+void          pixman_image_composite32        (pixman_op_t        op,
+					       pixman_image_t    *src,
+					       pixman_image_t    *mask,
+					       pixman_image_t    *dest,
+					       int32_t            src_x,
+					       int32_t            src_y,
+					       int32_t            mask_x,
+					       int32_t            mask_y,
+					       int32_t            dest_x,
+					       int32_t            dest_y,
+					       int32_t            width,
+					       int32_t            height);
+
+/* Old X servers rely on out-of-bounds accesses when they are asked
+ * to composite with a window as the source. They create a pixman image
+ * pointing to some bogus position in memory, but then they set a clip
+ * region to the position where the actual bits are.
+ *
+ * Due to a bug in old versions of pixman, where it would not clip
+ * against the image bounds when a clip region was set, this would
+ * actually work. So by default we allow certain out-of-bound access
+ * to happen unless explicitly disabled.
+ *
+ * Fixed X servers should call this function to disable the workaround.
+ */
+void          pixman_disable_out_of_bounds_workaround (void);
+
+/*
+ * Trapezoids
+ */
+typedef struct pixman_edge pixman_edge_t;
+typedef struct pixman_trapezoid pixman_trapezoid_t;
+typedef struct pixman_trap pixman_trap_t;
+typedef struct pixman_span_fix pixman_span_fix_t;
+
+/*
+ * An edge structure.  This represents a single polygon edge
+ * and can be quickly stepped across small or large gaps in the
+ * sample grid
+ */
+struct pixman_edge
+{
+    pixman_fixed_t	x;
+    pixman_fixed_t	e;
+    pixman_fixed_t	stepx;
+    pixman_fixed_t	signdx;
+    pixman_fixed_t	dy;
+    pixman_fixed_t	dx;
+
+    pixman_fixed_t	stepx_small;
+    pixman_fixed_t	stepx_big;
+    pixman_fixed_t	dx_small;
+    pixman_fixed_t	dx_big;
+};
+
+struct pixman_trapezoid
+{
+    pixman_fixed_t	top, bottom;
+    pixman_line_fixed_t	left, right;
+};
+
+
+/* whether 't' is a well defined not obviously empty trapezoid */
+#define pixman_trapezoid_valid(t)				   \
+    ((t)->left.p1.y != (t)->left.p2.y &&			   \
+     (t)->right.p1.y != (t)->right.p2.y &&			   \
+     (int) ((t)->bottom - (t)->top) > 0)
+
+struct pixman_span_fix
+{
+    pixman_fixed_t	l, r, y;
+};
+
+struct pixman_trap
+{
+    pixman_span_fix_t	top, bot;
+};
+
+pixman_fixed_t pixman_sample_ceil_y        (pixman_fixed_t             y,
+					    int                        bpp);
+pixman_fixed_t pixman_sample_floor_y       (pixman_fixed_t             y,
+					    int                        bpp);
+void           pixman_edge_step            (pixman_edge_t             *e,
+					    int                        n);
+void           pixman_edge_init            (pixman_edge_t             *e,
+					    int                        bpp,
+					    pixman_fixed_t             y_start,
+					    pixman_fixed_t             x_top,
+					    pixman_fixed_t             y_top,
+					    pixman_fixed_t             x_bot,
+					    pixman_fixed_t             y_bot);
+void           pixman_line_fixed_edge_init (pixman_edge_t             *e,
+					    int                        bpp,
+					    pixman_fixed_t             y,
+					    const pixman_line_fixed_t *line,
+					    int                        x_off,
+					    int                        y_off);
+void           pixman_rasterize_edges      (pixman_image_t            *image,
+					    pixman_edge_t             *l,
+					    pixman_edge_t             *r,
+					    pixman_fixed_t             t,
+					    pixman_fixed_t             b);
+void           pixman_add_traps            (pixman_image_t            *image,
+					    int16_t                    x_off,
+					    int16_t                    y_off,
+					    int                        ntrap,
+					    pixman_trap_t             *traps);
+void           pixman_add_trapezoids       (pixman_image_t            *image,
+					    int16_t                    x_off,
+					    int                        y_off,
+					    int                        ntraps,
+					    const pixman_trapezoid_t  *traps);
+void           pixman_rasterize_trapezoid  (pixman_image_t            *image,
+					    const pixman_trapezoid_t  *trap,
+					    int                        x_off,
+					    int                        y_off);
+
+PIXMAN_END_DECLS
+
+#endif /* PIXMAN_H__ */
diff --git a/pixman/pixman/refactor b/pixman/pixman/refactor
new file mode 100644
index 000000000..52fceab17
--- /dev/null
+++ b/pixman/pixman/refactor
@@ -0,0 +1,478 @@
+Roadmap
+
+- Move all the fetchers etc. into pixman-image to make pixman-compose.c
+  less intimidating.
+
+  DONE
+
+- Make combiners for unified alpha take a mask argument. That way
+  we won't need two separate paths for unified vs component in the
+  general compositing code.
+
+  DONE, except that the Altivec code needs to be updated. Luca is
+  looking into that.
+
+- Delete separate 'unified alpha' path
+ 
+  DONE
+
+- Split images into their own files
+
+  DONE
+
+- Split the gradient walker code out into its own file
+
+  DONE
+
+- Add scanline getters per image
+
+  DONE
+
+- Generic 64 bit fetcher 
+
+  DONE
+
+- Split fast path tables into their respective architecture dependent
+  files.
+
+See "Render Algorithm" below for rationale
+
+Images will eventually have these virtual functions:
+
+       get_scanline()
+       get_scanline_wide()
+       get_pixel()
+       get_pixel_wide()
+       get_untransformed_pixel()
+       get_untransformed_pixel_wide()
+       get_unfiltered_pixel()
+       get_unfiltered_pixel_wide()
+
+       store_scanline()
+       store_scanline_wide()
+
+1.
+
+Initially we will just have get_scanline() and get_scanline_wide();
+these will be based on the ones in pixman-compose. Hopefully this will
+reduce the complexity in pixman_composite_rect_general().
+
+Note that there is access considerations - the compose function is
+being compiled twice.
+
+
+2.
+
+Split image types into their own source files. Export noop virtual
+reinit() call.  Call this whenever a property of the image changes.
+
+
+3. 
+
+Split the get_scanline() call into smaller functions that are
+initialized by the reinit() call.
+
+The Render Algorithm:
+	(first repeat, then filter, then transform, then clip)
+
+Starting from a destination pixel (x, y), do
+
+	1 x = x - xDst + xSrc
+	  y = y - yDst + ySrc
+
+	2 reject pixel that is outside the clip
+
+	This treats clipping as something that happens after
+	transformation, which I think is correct for client clips. For
+	hierarchy clips it is wrong, but who really cares? Without
+	GraphicsExposes hierarchy clips are basically irrelevant. Yes,
+	you could imagine cases where the pixels of a subwindow of a
+	redirected, transformed window should be treated as
+	transparent. I don't really care
+
+	Basically, I think the render spec should say that pixels that
+	are unavailable due to the hierarcy have undefined content,
+	and that GraphicsExposes are not generated. Ie., basically
+	that using non-redirected windows as sources is fail. This is
+	at least consistent with the current implementation and we can
+	update the spec later if someone makes it work.
+
+	The implication for render is that it should stop passing the
+	hierarchy clip to pixman. In pixman, if a souce image has a
+	clip it should be used in computing the composite region and
+	nowhere else, regardless of what "has_client_clip" says. The
+	default should be for there to not be any clip.
+
+	I would really like to get rid of the client clip as well for
+	source images, but unfortunately there is at least one
+	application in the wild that uses them.
+
+	3 Transform pixel: (x, y) = T(x, y)
+
+	4 Call p = GetUntransformedPixel (x, y)
+
+	5 If the image has an alpha map, then
+
+		Call GetUntransformedPixel (x, y) on the alpha map
+		
+		add resulting alpha channel to p
+
+	   return p
+
+	Where GetUnTransformedPixel is:
+
+	6 switch (filter)
+	  {
+	  case NEAREST:
+		return GetUnfilteredPixel (x, y);
+		break;
+
+	  case BILINEAR:
+		return GetUnfilteredPixel (...) // 4 times 
+		break;
+
+	  case CONVOLUTION:
+		return GetUnfilteredPixel (...) // as many times as necessary.
+		break;
+	  }
+
+	Where GetUnfilteredPixel (x, y) is
+
+	7 switch (repeat)
+	   {
+	   case REPEAT_NORMAL:
+	   case REPEAT_PAD:
+	   case REPEAT_REFLECT:
+		// adjust x, y as appropriate
+		break;
+
+	   case REPEAT_NONE:
+	        if (x, y) is outside image bounds
+		     return 0;
+		break;
+	   }
+
+	   return GetRawPixel(x, y)
+
+	Where GetRawPixel (x, y) is
+
+	8 Compute the pixel in question, depending on image type.
+
+For gradients, repeat has a totally different meaning, so
+UnfilteredPixel() and RawPixel() must be the same function so that
+gradients can do their own repeat algorithm.
+
+So, the GetRawPixel
+
+	for bits must deal with repeats
+	for gradients must deal with repeats (differently)
+	for solids, should ignore repeats.
+
+	for polygons, when we add them, either ignore repeats or do
+	something similar to bits (in which case, we may want an extra
+	layer of indirection to modify the coordinates).
+
+It is then possible to build things like "get scanline" or "get tile" on
+top of this. In the simplest case, just repeatedly calling GetPixel()
+would work, but specialized get_scanline()s or get_tile()s could be
+plugged in for common cases. 
+
+By not plugging anything in for images with access functions, we only
+have to compile the pixel functions twice, not the scanline functions.
+
+And we can get rid of fetchers for the bizarre formats that no one
+uses. Such as b2g3r3 etc. r1g2b1? Seriously? It is also worth
+considering a generic format based pixel fetcher for these edge cases.
+
+Since the actual routines depend on the image attributes, the images
+must be notified when those change and update their function pointers
+appropriately. So there should probably be a virtual function called
+(* reinit) or something like that.
+
+There will also be wide fetchers for both pixels and lines. The line
+fetcher will just call the wide pixel fetcher. The wide pixel fetcher
+will just call expand, except for 10 bit formats.
+
+Rendering pipeline:
+
+Drawable:
+	0. if (picture has alpha map)
+		0.1. Position alpha map according to the alpha_x/alpha_y
+	        0.2. Where the two drawables intersect, the alpha channel
+		     Replace the alpha channel of source with the one
+		     from the alpha map. Replacement only takes place
+		     in the intersection of the two drawables' geometries.
+	1. Repeat the drawable according to the repeat attribute
+	2. Reconstruct a continuous image according to the filter
+	3. Transform according to the transform attribute
+	4. Position image such that src_x, src_y is over dst_x, dst_y
+	5. Sample once per destination pixel 
+	6. Clip. If a pixel is not within the source clip, then no
+	   compositing takes place at that pixel. (Ie., it's *not*
+	   treated as 0).
+
+	Sampling a drawable: 
+
+	- If the channel does not have an alpha channel, the pixels in it
+	  are treated as opaque.
+
+	Note on reconstruction:
+
+	- The top left pixel has coordinates (0.5, 0.5) and pixels are
+	  spaced 1 apart.
+
+Gradient:
+	1. Unless gradient type is conical, repeat the underlying (0, 1)
+		gradient according to the repeat attribute
+	2. Integrate the gradient across the plane according to type.
+	3. Transform according to transform attribute
+	4. Position gradient 
+	5. Sample once per destination pixel.
+ 	6. Clip
+
+Solid Fill:
+	1. Repeat has no effect
+	2. Image is already continuous and defined for the entire plane
+	3. Transform has no effect
+	4. Positioning has no effect
+	5. Sample once per destination pixel.
+	6. Clip
+
+Polygon:
+	1. Repeat has no effect
+	2. Image is already continuous and defined on the whole plane
+	3. Transform according to transform attribute
+	4. Position image
+	5. Supersample 15x17 per destination pixel.
+	6. Clip
+
+Possibly interesting additions:
+	- More general transformations, such as warping, or general
+	  shading.
+
+	- Shader image where a function is called to generate the
+          pixel (ie., uploading assembly code).
+
+	- Resampling kernels
+
+	  In principle the polygon image uses a 15x17 box filter for
+	  resampling. If we allow general resampling filters, then we
+	  get all the various antialiasing types for free. 
+
+	  Bilinear downsampling looks terrible and could be much 
+	  improved by a resampling filter. NEAREST reconstruction
+	  combined with a box resampling filter is what GdkPixbuf
+	  does, I believe.
+
+	  Useful for high frequency gradients as well.
+
+	  (Note that the difference between a reconstruction and a
+	  resampling filter is mainly where in the pipeline they
+	  occur. High quality resampling should use a correctly
+	  oriented kernel so it should happen after transformation.
+
+	  An implementation can transform the resampling kernel and
+	  convolve it with the reconstruction if it so desires, but it
+	  will need to deal with the fact that the resampling kernel
+	  will not necessarily be pixel aligned.
+
+	  "Output kernels"
+
+	  One could imagine doing the resampling after compositing,
+	  ie., for each destination pixel sample each source image 16
+	  times, then composite those subpixels individually, then
+	  finally apply a kernel.
+
+	  However, this is effectively the same as full screen
+	  antialiasing, which is a simpler way to think about it. So
+	  resampling kernels may make sense for individual images, but
+	  not as a post-compositing step.
+	  
+	  Fullscreen AA is inefficient without chained compositing
+	  though. Consider an (image scaled up to oversample size IN
+	  some polygon) scaled down to screen size. With the current
+	  implementation, there will be a huge temporary. With chained
+	  compositing, the whole thing ends up being equivalent to the
+	  output kernel from above.
+
+	- Color space conversion
+
+	  The complete model here is that each surface has a color
+	  space associated with it and that the compositing operation
+	  also has one associated with it. Note also that gradients
+	  should have associcated colorspaces.
+
+	- Dithering
+
+	  If people dither something that is already dithered, it will
+	  look terrible, but don't do that, then. (Dithering happens
+	  after resampling if at all - what is the relationship
+	  with color spaces? Presumably dithering should happen in linear
+	  intensity space).
+
+	- Floating point surfaces, 16, 32 and possibly 64 bit per
+	  channel.
+
+	Maybe crack:
+
+	- Glyph polygons
+
+	  If glyphs could be given as polygons, they could be
+	  positioned and rasterized more accurately. The glyph
+	  structure would need subpixel positioning though.
+
+	- Luminance vs. coverage for the alpha channel
+
+	  Whether the alpha channel should be interpreted as luminance
+          modulation or as coverage (intensity modulation). This is a
+          bit of a departure from the rendering model though. It could
+	  also be considered whether it should be possible to have 
+	  both channels in the same drawable.
+
+	- Alternative for component alpha
+
+	  - Set component-alpha on the output image.
+
+	    - This means each of the components are sampled
+	      independently and composited in the corresponding
+	      channel only.
+
+	  - Have 3 x oversampled mask
+
+	  - Scale it down by 3 horizontally, with [ 1/3, 1/3, 1/3 ]
+            resampling filter. 
+
+	    Is this equivalent to just using a component alpha mask?
+
+	Incompatible changes:
+
+	- Gradients could be specified with premultiplied colors. (You
+	  can use a mask to get things like gradients from solid red to
+	  transparent red.
+
+Refactoring pixman
+
+The pixman code is not particularly nice to put it mildly. Among the
+issues are
+
+- inconsistent naming style (fb vs Fb, camelCase vs
+  underscore_naming). Sometimes there is even inconsistency *within*
+  one name.
+
+      fetchProc32 ACCESS(pixman_fetchProcForPicture32)
+
+  may be one of the uglies names ever created.
+
+  coding style: 
+  	 use the one from cairo except that pixman uses this brace style:
+	 
+		while (blah)
+		{
+		}
+
+	Format do while like this:
+
+	       do 
+	       {
+
+	       } 
+	       while (...);
+
+- PIXMAN_COMPOSITE_RECT_GENERAL() is horribly complex
+
+- switch case logic in pixman-access.c
+
+  Instead it would be better to just store function pointers in the
+  image objects themselves,
+
+  	get_pixel()
+	get_scanline()
+
+- Much of the scanline fetching code is for formats that no one 
+  ever uses. a2r2g2b2 anyone?
+
+  It would probably be worthwhile having a generic fetcher for any
+  pixman format whatsoever.
+
+- Code related to particular image types should be split into individual
+  files.
+
+	pixman-bits-image.c
+	pixman-linear-gradient-image.c
+	pixman-radial-gradient-image.c
+	pixman-solid-image.c
+
+- Fast path code should be split into files based on architecture:
+
+       pixman-mmx-fastpath.c
+       pixman-sse2-fastpath.c
+       pixman-c-fastpath.c
+
+       etc.
+
+  Each of these files should then export a fastpath table, which would
+  be declared in pixman-private.h. This should allow us to get rid
+  of the pixman-mmx.h files.
+
+  The fast path table should describe each fast path. Ie there should
+  be bitfields indicating what things the fast path can handle, rather than
+  like now where it is only allowed to take one format per src/mask/dest. Ie., 
+
+  { 
+    FAST_a8r8g8b8 | FAST_x8r8g8b8,
+    FAST_null,
+    FAST_x8r8g8b8,
+    FAST_repeat_normal | FAST_repeat_none,
+    the_fast_path
+  }
+
+There should then be *one* file that implements pixman_image_composite(). 
+This should do this:
+
+     optimize_operator();
+
+     convert 1x1 repeat to solid (actually this should be done at
+     image creation time).
+     
+     is there a useful fastpath?
+
+There should be a file called pixman-cpu.c that contains all the
+architecture specific stuff to detect what CPU features we have.
+
+Issues that must be kept in mind:
+
+       - we need accessor code to be preserved
+
+       - maybe there should be a "store_scanline" too?
+
+         Is this sufficient?
+
+	 We should preserve the optimization where the
+	 compositing happens directly in the destination
+	 whenever possible.
+
+	- It should be possible to create GPU samplers from the
+	  images.
+
+The "horizontal" classification should be a bit in the image, the
+"vertical" classification should just happen inside the gradient
+file. Note though that
+
+      (a) these will change if the tranformation/repeat changes.
+
+      (b) at the moment the optimization for linear gradients
+          takes the source rectangle into account. Presumably
+	  this is to also optimize the case where the gradient
+	  is close enough to horizontal?
+
+Who is responsible for repeats? In principle it should be the scanline
+fetch. Right now NORMAL repeats are handled by walk_composite_region()
+while other repeats are handled by the scanline code.
+
+
+(Random note on filtering: do you filter before or after
+transformation?  Hardware is going to filter after transformation;
+this is also what pixman does currently). It's not completely clear
+what filtering *after* transformation means. One thing that might look
+good would be to do *supersampling*, ie., compute multiple subpixels
+per destination pixel, then average them together.
diff --git a/pixman/pixman/solaris-hwcap.mapfile b/pixman/pixman/solaris-hwcap.mapfile
new file mode 100644
index 000000000..3605ca79f
--- /dev/null
+++ b/pixman/pixman/solaris-hwcap.mapfile
@@ -0,0 +1,30 @@
+###############################################################################
+#
+# Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice (including the next
+# paragraph) shall be included in all copies or substantial portions of the
+# Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+# DEALINGS IN THE SOFTWARE.
+#
+###############################################################################
+#
+# Override the linker's detection of CMOV/MMX/SSE instructions so this
+# library isn't flagged as only usable on CPU's with those ISA's, since it
+# checks at runtime for availability before calling them
+
+hwcap_1 = V0x0 FPU OVERRIDE;
diff --git a/pixman/test/Makefile.am b/pixman/test/Makefile.am
new file mode 100644
index 000000000..841ff8d7d
--- /dev/null
+++ b/pixman/test/Makefile.am
@@ -0,0 +1,84 @@
+TEST_LDADD = $(top_builddir)/pixman/libpixman-1.la
+INCLUDES = -I$(top_srcdir)/pixman -I$(top_builddir)/pixman
+
+TESTPROGRAMS =			\
+	a1-trap-test		\
+	region-test		\
+	fetch-test		\
+	oob-test		\
+	window-test		\
+	trap-crasher		\
+	alphamap		\
+	blitters-test		\
+	scaling-test		\
+	composite
+
+a1_trap_test_LDADD = $(TEST_LDADD)
+fetch_test_LDADD = $(TEST_LDADD)
+composite_LDADD = $(TEST_LDADD)
+trap_crasher_LDADD = $(TEST_LDADD)
+oob_test_LDADD = $(TEST_LDADD)
+window_test_LDADD = $(TEST_LDADD)
+
+region_test_LDADD = $(TEST_LDADD)
+region_test_SOURCES = region-test.c utils.c utils.h
+
+blitters_test_LDADD = $(TEST_LDADD)
+blitters_test_SOURCES = blitters-test.c utils.c utils.h
+
+scaling_test_LDADD = $(TEST_LDADD)
+scaling_test_SOURCES = scaling-test.c utils.c utils.h
+
+alphamap_LDADD = $(TEST_LDADD)
+alphamap_SOURCES = alphamap.c utils.c utils.h
+
+# GTK using test programs
+
+if HAVE_GTK
+
+GTK_LDADD = $(TEST_LDADD) $(GTK_LIBS)
+GTK_UTILS = gtk-utils.c gtk-utils.h
+
+TESTPROGRAMS_GTK =		\
+	clip-test		\
+	clip-in			\
+	composite-test		\
+	gradient-test		\
+	alpha-test		\
+	screen-test		\
+	convolution-test	\
+	trap-test		\
+	alphamap
+
+INCLUDES += $(GTK_CFLAGS)
+
+gradient_test_LDADD = $(GTK_LDADD)
+gradient_test_SOURCES = gradient-test.c $(GTK_UTILS)
+
+alpha_test_LDADD = $(GTK_LDADD)
+alpha_test_SOURCES = alpha-test.c $(GTK_UTILS)
+
+composite_test_LDADD = $(GTK_LDADD)
+composite_test_SOURCES = composite-test.c $(GTK_UTILS)
+
+clip_test_LDADD = $(GTK_LDADD)
+clip_test_SOURCES = clip-test.c $(GTK_UTILS)
+
+clip_in_LDADD = $(GTK_LDADD)
+clip_in_SOURCES = clip-in.c $(GTK_UTILS)
+
+trap_test_LDADD = $(GTK_LDADD)
+trap_test_SOURCES = trap-test.c $(GTK_UTILS)
+
+screen_test_LDADD = $(GTK_LDADD)
+screen_test_SOURCES = screen-test.c $(GTK_UTILS)
+
+convolution_test_LDADD = $(GTK_LDADD)
+convolution_test_SOURCES = convolution-test.c $(GTK_UTILS)
+
+endif
+
+noinst_PROGRAMS = $(TESTPROGRAMS) $(TESTPROGRAMS_GTK)
+
+TESTS = $(TESTPROGRAMS)
+
diff --git a/pixman/test/a1-trap-test.c b/pixman/test/a1-trap-test.c
new file mode 100644
index 000000000..6163e7c61
--- /dev/null
+++ b/pixman/test/a1-trap-test.c
@@ -0,0 +1,50 @@
+#include <assert.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include "pixman.h"
+
+int
+main (int argc, char **argv)
+{
+#define WIDTH 20
+#define HEIGHT 20
+
+    pixman_image_t *src_img;
+    pixman_image_t *mask_img;
+    pixman_image_t *dest_img;
+    pixman_trap_t trap;
+    pixman_color_t red = { 0xffff, 0x0000, 0x0000, 0xffff };
+    uint32_t *bits = malloc (WIDTH * HEIGHT * 4);
+    uint32_t *mbits = malloc (WIDTH * HEIGHT);
+
+    memset (mbits, 0, WIDTH * HEIGHT);
+    memset (bits, 0xff, WIDTH * HEIGHT * 4);
+    
+    trap.top.l = pixman_double_to_fixed (0.5);
+    trap.top.r = pixman_double_to_fixed (1.5);
+    trap.top.y = pixman_double_to_fixed (0.5);
+
+    trap.bot.l = pixman_double_to_fixed (0.5);
+    trap.bot.r = pixman_double_to_fixed (1.5);
+    trap.bot.y = pixman_double_to_fixed (1.5);
+
+    mask_img = pixman_image_create_bits (
+	PIXMAN_a1, WIDTH, HEIGHT, mbits, WIDTH);
+    src_img = pixman_image_create_solid_fill (&red);
+    dest_img = pixman_image_create_bits (
+	PIXMAN_a8r8g8b8, WIDTH, HEIGHT, bits, WIDTH * 4);
+    
+    pixman_add_traps (mask_img, 0, 0, 1, &trap);
+
+    pixman_image_composite (PIXMAN_OP_OVER,
+			    src_img, mask_img, dest_img,
+			    0, 0, 0, 0, 0, 0, WIDTH, HEIGHT);
+
+    assert (bits[0] == 0xffff0000);
+    assert (bits[1] == 0xffffffff);
+    assert (bits[1 * WIDTH + 0] == 0xffffffff);
+    assert (bits[1 * WIDTH + 1] == 0xffffffff);
+    
+    return 0;
+}
diff --git a/pixman/test/alpha-test.c b/pixman/test/alpha-test.c
new file mode 100644
index 000000000..92c208142
--- /dev/null
+++ b/pixman/test/alpha-test.c
@@ -0,0 +1,117 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include "pixman.h"
+#include "gtk-utils.h"
+
+int
+main (int argc, char **argv)
+{
+#define WIDTH 400
+#define HEIGHT 200
+    
+    uint32_t *alpha = malloc (WIDTH * HEIGHT * 4);
+    uint32_t *dest = malloc (WIDTH * HEIGHT * 4);
+    uint32_t *src = malloc (WIDTH * HEIGHT * 4);
+    pixman_image_t *grad_img;
+    pixman_image_t *alpha_img;
+    pixman_image_t *dest_img;
+    pixman_image_t *src_img;
+    int i;
+    pixman_gradient_stop_t stops[2] =
+	{
+	    { pixman_int_to_fixed (0), { 0x0000, 0x0000, 0x0000, 0x0000 } },
+	    { pixman_int_to_fixed (1), { 0xffff, 0x0000, 0x1111, 0xffff } }
+	};
+    pixman_point_fixed_t p1 = { pixman_double_to_fixed (0), 0 };
+    pixman_point_fixed_t p2 = { pixman_double_to_fixed (WIDTH),
+				pixman_int_to_fixed (0) };
+#if 0
+    pixman_transform_t trans = {
+	{ { pixman_double_to_fixed (2), pixman_double_to_fixed (0.5), pixman_double_to_fixed (-100), },
+	  { pixman_double_to_fixed (0), pixman_double_to_fixed (3), pixman_double_to_fixed (0), },
+	  { pixman_double_to_fixed (0), pixman_double_to_fixed (0.000), pixman_double_to_fixed (1.0) } 
+	}
+    };
+#else
+    pixman_transform_t trans = {
+	{ { pixman_fixed_1, 0, 0 },
+	  { 0, pixman_fixed_1, 0 },
+	  { 0, 0, pixman_fixed_1 } }
+    };
+#endif
+
+    pixman_point_fixed_t c_inner;
+    pixman_point_fixed_t c_outer;
+    pixman_fixed_t r_inner;
+    pixman_fixed_t r_outer;
+    
+    for (i = 0; i < WIDTH * HEIGHT; ++i)
+	alpha[i] = 0x4f00004f; /* pale blue */
+    
+    alpha_img = pixman_image_create_bits (PIXMAN_a8r8g8b8,
+					 WIDTH, HEIGHT, 
+					  alpha,
+					 WIDTH * 4);
+
+    for (i = 0; i < WIDTH * HEIGHT; ++i)
+	dest[i] = 0xffffff00;		/* yellow */
+    
+    dest_img = pixman_image_create_bits (PIXMAN_a8r8g8b8,
+					 WIDTH, HEIGHT, 
+					 dest,
+					 WIDTH * 4);
+
+    for (i = 0; i < WIDTH * HEIGHT; ++i)
+	src[i] = 0xffff0000;
+
+    src_img = pixman_image_create_bits (PIXMAN_a8r8g8b8,
+					WIDTH, HEIGHT,
+					src,
+					WIDTH * 4);
+    
+    c_inner.x = pixman_double_to_fixed (50.0);
+    c_inner.y = pixman_double_to_fixed (50.0);
+    c_outer.x = pixman_double_to_fixed (50.0);
+    c_outer.y = pixman_double_to_fixed (50.0);
+    r_inner = 0;
+    r_outer = pixman_double_to_fixed (50.0);
+    
+#if 0
+    grad_img = pixman_image_create_conical_gradient (&c_inner, r_inner,
+						    stops, 2);
+#endif
+#if 0
+    grad_img = pixman_image_create_conical_gradient (&c_inner, r_inner,
+						    stops, 2);
+    grad_img = pixman_image_create_linear_gradient (&c_inner, &c_outer,
+						   r_inner, r_outer,
+						   stops, 2);
+#endif
+    
+    grad_img = pixman_image_create_linear_gradient  (&p1, &p2,
+						    stops, 2);
+
+    pixman_image_set_transform (grad_img, &trans);
+    pixman_image_set_repeat (grad_img, PIXMAN_REPEAT_PAD);
+    
+    pixman_image_composite (PIXMAN_OP_OVER, grad_img, NULL, alpha_img,
+			    0, 0, 0, 0, 0, 0, 10 * WIDTH, HEIGHT);
+
+    pixman_image_set_alpha_map (src_img, alpha_img, 10, 10);
+    
+    pixman_image_composite (PIXMAN_OP_OVER, src_img, NULL, dest_img,
+			    0, 0, 0, 0, 0, 0, 10 * WIDTH, HEIGHT);
+    
+    printf ("0, 0: %x\n", dest[0]);
+    printf ("10, 10: %x\n", dest[10 * 10 + 10]);
+    printf ("w, h: %x\n", dest[(HEIGHT - 1) * 100 + (WIDTH - 1)]);
+    
+    show_image (dest_img);
+
+    pixman_image_unref (src_img);
+    pixman_image_unref (grad_img);
+    pixman_image_unref (alpha_img);
+    free (dest);
+    
+    return 0;
+}
diff --git a/pixman/test/alphamap.c b/pixman/test/alphamap.c
new file mode 100644
index 000000000..e6a25efcb
--- /dev/null
+++ b/pixman/test/alphamap.c
@@ -0,0 +1,49 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include "utils.h"
+
+#define WIDTH 400
+#define HEIGHT 200
+
+int
+main (int argc, char **argv)
+{
+    uint8_t *alpha = make_random_bytes (WIDTH * HEIGHT);
+    uint32_t *src = (uint32_t *)make_random_bytes (WIDTH * HEIGHT * 4);
+    uint32_t *dest = (uint32_t *)make_random_bytes (WIDTH * HEIGHT * 4);
+    int i;
+
+    pixman_image_t *a = pixman_image_create_bits (PIXMAN_a8, WIDTH, HEIGHT, (uint32_t *)alpha, WIDTH);
+    pixman_image_t *d = pixman_image_create_bits (PIXMAN_a8r8g8b8, WIDTH, HEIGHT, dest, WIDTH * 4);
+
+    for (i = 0; i < 2; ++i)
+    {
+	pixman_format_code_t sformat = (i == 0)? PIXMAN_a8r8g8b8 : PIXMAN_a2r10g10b10;
+	pixman_image_t *s = pixman_image_create_bits (sformat, WIDTH, HEIGHT, src, WIDTH * 4);
+	int j, k;
+
+	pixman_image_set_alpha_map (s, a, 0, 0);
+
+	pixman_image_composite (PIXMAN_OP_SRC, s, NULL, d, 0, 0, 0, 0, 0, 0, WIDTH, HEIGHT);
+
+	for (j = 0; j < HEIGHT; ++j)
+	{
+	    for (k = 0; k < WIDTH; ++k)
+	    {
+		uint8_t ap = ((uint8_t *)alpha)[j * WIDTH + k];
+		uint32_t dap = (dest[j * WIDTH + k] >> 24);
+		uint32_t sap = (src[j * WIDTH + k] >> 24);
+
+		if (ap != dap)
+		{
+		    printf ("Wrong alpha value at (%d, %d). Should be %d; got %d (src was %d)\n", k, j, ap, dap, sap);
+		    return 1;
+		}
+	    }
+	}
+
+	pixman_image_unref (s);
+    }
+
+    return 0;
+}
diff --git a/pixman/test/blitters-test-bisect.rb b/pixman/test/blitters-test-bisect.rb
new file mode 100644
index 000000000..62ff782e7
--- /dev/null
+++ b/pixman/test/blitters-test-bisect.rb
@@ -0,0 +1,43 @@
+#!/usr/bin/env ruby
+
+if not ARGV[0] or not ARGV[1] then
+    printf("Please provide two 'blitters-test' static binaries in the command line.\n\n")
+    printf("The first should be linked with a correct reference pixman library.\n")
+    printf("The second binary should be linked with a pixman library which needs to be tested.\n")
+    exit(0)
+end
+
+def test_range(min, max)
+    if `#{ARGV[0]} #{min} #{max} 2>/dev/null` == `#{ARGV[1]} #{min} #{max} 2>/dev/null` then
+        return
+    end
+    while max != min + 1 do
+        avg = ((min + max) / 2).to_i
+        res1 = `#{ARGV[0]} #{min} #{avg} 2>/dev/null`
+        res2 = `#{ARGV[1]} #{min} #{avg} 2>/dev/null`
+        if res1 != res2 then
+            max = avg
+        else
+            min = avg
+        end
+    end
+    return max
+end
+
+base = 1
+while true do
+    # run infinitely, processing 100000 test cases per iteration
+    printf("running tests %d-%d\n", base, base + 100000 - 1);
+    res = test_range(base, base + 100000 - 1)
+    if res then
+        printf("-- ref --\n")
+        printf("%s\n", `#{ARGV[0]} -#{res}`)
+        printf("-- new --\n")
+        printf("%s\n", `#{ARGV[1]} -#{res}`)
+
+        printf("\nFailed test %d, you can reproduce the problematic conditions by running\n", res)
+        printf("#{ARGV[1]} -%d\n", res)
+        exit(1)
+    end
+    base += 100000
+end
diff --git a/pixman/test/blitters-test.c b/pixman/test/blitters-test.c
new file mode 100644
index 000000000..1ebf6d9ca
--- /dev/null
+++ b/pixman/test/blitters-test.c
@@ -0,0 +1,497 @@
+/*
+ * Test program, which stresses the use of different color formats and
+ * compositing operations.
+ *
+ * Just run it without any command line arguments, and it will report either
+ *   "blitters test passed" - everything is ok
+ *   "blitters test failed!" - there is some problem
+ *
+ * In the case of failure, finding the problem involves the following steps:
+ * 1. Get the reference 'blitters-test' binary. It makes sense to disable all
+ *    the cpu specific optimizations in pixman and also configure it with
+ *    '--disable-shared' option. Those who are paranoid can also tweak the
+ *    sources to disable all fastpath functions. The resulting binary
+ *    can be renamed to something like 'blitters-test.ref'.
+ * 2. Compile the buggy binary (also with the '--disable-shared' option).
+ * 3. Run 'ruby blitters-test-bisect.rb ./blitters-test.ref ./blitters-test'
+ * 4. Look at the information about failed case (destination buffer content
+ *    will be shown) and try to figure out what is wrong. Loading
+ *    test program in gdb, specifying failed test number in the command
+ *    line with '-' character prepended and setting breakpoint on
+ *    'pixman_image_composite' function can provide detailed information
+ *    about function arguments
+ */
+#include <assert.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <config.h>
+#include "utils.h"
+
+static pixman_indexed_t palette;
+
+static void *
+aligned_malloc (size_t align, size_t size)
+{
+    void *result;
+
+#ifdef HAVE_POSIX_MEMALIGN
+    if (posix_memalign (&result, align, size) != 0)
+      result = NULL;
+#else
+    result = malloc (size);
+#endif
+
+    return result;
+}
+
+/* Create random image for testing purposes */
+static pixman_image_t *
+create_random_image (pixman_format_code_t *allowed_formats,
+		     int                   max_width,
+		     int                   max_height,
+		     int                   max_extra_stride,
+		     pixman_format_code_t *used_fmt)
+{
+    int n = 0, i, width, height, stride;
+    pixman_format_code_t fmt;
+    uint32_t *buf;
+    pixman_image_t *img;
+
+    while (allowed_formats[n] != -1)
+	n++;
+    fmt = allowed_formats[lcg_rand_n (n)];
+
+    width = lcg_rand_n (max_width) + 1;
+    height = lcg_rand_n (max_height) + 1;
+    stride = (width * PIXMAN_FORMAT_BPP (fmt) + 7) / 8 +
+	lcg_rand_n (max_extra_stride + 1);
+    stride = (stride + 3) & ~3;
+
+    /* do the allocation */
+    buf = aligned_malloc (64, stride * height);
+
+    /* initialize image with random data */
+    for (i = 0; i < stride * height; i++)
+    {
+	/* generation is biased to having more 0 or 255 bytes as
+	 * they are more likely to be special-cased in code
+	 */
+	*((uint8_t *)buf + i) = lcg_rand_n (4) ? lcg_rand_n (256) :
+	    (lcg_rand_n (2) ? 0 : 255);
+    }
+
+    img = pixman_image_create_bits (fmt, width, height, buf, stride);
+
+    if (PIXMAN_FORMAT_TYPE (fmt) == PIXMAN_TYPE_COLOR	||
+	PIXMAN_FORMAT_TYPE (fmt) == PIXMAN_TYPE_GRAY)
+    {
+	pixman_image_set_indexed (img, &palette);
+    }
+
+    image_endian_swap (img, PIXMAN_FORMAT_BPP (fmt));
+
+    if (used_fmt) *used_fmt = fmt;
+    return img;
+}
+
+/* Free random image, and optionally update crc32 based on its data */
+static uint32_t
+free_random_image (uint32_t initcrc,
+		   pixman_image_t *img,
+		   pixman_format_code_t fmt)
+{
+    uint32_t crc32 = 0;
+    int stride = pixman_image_get_stride (img);
+    uint32_t *data = pixman_image_get_data (img);
+    int height = pixman_image_get_height (img);
+
+    if (fmt != -1)
+    {
+	/* mask unused 'x' part */
+	if (PIXMAN_FORMAT_BPP (fmt) - PIXMAN_FORMAT_DEPTH (fmt) &&
+	    PIXMAN_FORMAT_DEPTH (fmt) != 0)
+	{
+	    int i;
+	    uint32_t *data = pixman_image_get_data (img);
+	    uint32_t mask = (1 << PIXMAN_FORMAT_DEPTH (fmt)) - 1;
+
+	    if (PIXMAN_FORMAT_TYPE (fmt) == PIXMAN_TYPE_BGRA)
+		mask <<= (PIXMAN_FORMAT_BPP (fmt) - PIXMAN_FORMAT_DEPTH (fmt));
+
+	    for (i = 0; i < 32; i++)
+		mask |= mask << (i * PIXMAN_FORMAT_BPP (fmt));
+
+	    for (i = 0; i < stride * height / 4; i++)
+		data[i] &= mask;
+	}
+
+	/* swap endiannes in order to provide identical results on both big
+	 * and litte endian systems
+	 */
+	image_endian_swap (img, PIXMAN_FORMAT_BPP (fmt));
+	crc32 = compute_crc32 (initcrc, data, stride * height);
+    }
+
+    pixman_image_unref (img);
+    free (data);
+
+    return crc32;
+}
+
+static pixman_op_t op_list[] = {
+    PIXMAN_OP_SRC,
+    PIXMAN_OP_OVER,
+    PIXMAN_OP_ADD,
+    PIXMAN_OP_CLEAR,
+    PIXMAN_OP_SRC,
+    PIXMAN_OP_DST,
+    PIXMAN_OP_OVER,
+    PIXMAN_OP_OVER_REVERSE,
+    PIXMAN_OP_IN,
+    PIXMAN_OP_IN_REVERSE,
+    PIXMAN_OP_OUT,
+    PIXMAN_OP_OUT_REVERSE,
+    PIXMAN_OP_ATOP,
+    PIXMAN_OP_ATOP_REVERSE,
+    PIXMAN_OP_XOR,
+    PIXMAN_OP_ADD,
+    PIXMAN_OP_SATURATE,
+    PIXMAN_OP_DISJOINT_CLEAR,
+    PIXMAN_OP_DISJOINT_SRC,
+    PIXMAN_OP_DISJOINT_DST,
+    PIXMAN_OP_DISJOINT_OVER,
+    PIXMAN_OP_DISJOINT_OVER_REVERSE,
+    PIXMAN_OP_DISJOINT_IN,
+    PIXMAN_OP_DISJOINT_IN_REVERSE,
+    PIXMAN_OP_DISJOINT_OUT,
+    PIXMAN_OP_DISJOINT_OUT_REVERSE,
+    PIXMAN_OP_DISJOINT_ATOP,
+    PIXMAN_OP_DISJOINT_ATOP_REVERSE,
+    PIXMAN_OP_DISJOINT_XOR,
+    PIXMAN_OP_CONJOINT_CLEAR,
+    PIXMAN_OP_CONJOINT_SRC,
+    PIXMAN_OP_CONJOINT_DST,
+    PIXMAN_OP_CONJOINT_OVER,
+    PIXMAN_OP_CONJOINT_OVER_REVERSE,
+    PIXMAN_OP_CONJOINT_IN,
+    PIXMAN_OP_CONJOINT_IN_REVERSE,
+    PIXMAN_OP_CONJOINT_OUT,
+    PIXMAN_OP_CONJOINT_OUT_REVERSE,
+    PIXMAN_OP_CONJOINT_ATOP,
+    PIXMAN_OP_CONJOINT_ATOP_REVERSE,
+    PIXMAN_OP_CONJOINT_XOR,
+    PIXMAN_OP_MULTIPLY,
+    PIXMAN_OP_SCREEN,
+    PIXMAN_OP_OVERLAY,
+    PIXMAN_OP_DARKEN,
+    PIXMAN_OP_LIGHTEN,
+    PIXMAN_OP_COLOR_DODGE,
+    PIXMAN_OP_COLOR_BURN,
+    PIXMAN_OP_HARD_LIGHT,
+    PIXMAN_OP_DIFFERENCE,
+    PIXMAN_OP_EXCLUSION,
+#if 0 /* these use floating point math and are not always bitexact on different platforms */
+    PIXMAN_OP_SOFT_LIGHT,
+    PIXMAN_OP_HSL_HUE,
+    PIXMAN_OP_HSL_SATURATION,
+    PIXMAN_OP_HSL_COLOR,
+    PIXMAN_OP_HSL_LUMINOSITY,
+#endif
+};
+
+static pixman_format_code_t img_fmt_list[] = {
+    PIXMAN_a8r8g8b8,
+    PIXMAN_x8r8g8b8,
+    PIXMAN_r5g6b5,
+    PIXMAN_r3g3b2,
+    PIXMAN_a8,
+    PIXMAN_a8b8g8r8,
+    PIXMAN_x8b8g8r8,
+    PIXMAN_b8g8r8a8,
+    PIXMAN_b8g8r8x8,
+    PIXMAN_r8g8b8,
+    PIXMAN_b8g8r8,
+    PIXMAN_r5g6b5,
+    PIXMAN_b5g6r5,
+    PIXMAN_x2r10g10b10,
+    PIXMAN_a2r10g10b10,
+    PIXMAN_x2b10g10r10,
+    PIXMAN_a2b10g10r10,
+    PIXMAN_a1r5g5b5,
+    PIXMAN_x1r5g5b5,
+    PIXMAN_a1b5g5r5,
+    PIXMAN_x1b5g5r5,
+    PIXMAN_a4r4g4b4,
+    PIXMAN_x4r4g4b4,
+    PIXMAN_a4b4g4r4,
+    PIXMAN_x4b4g4r4,
+    PIXMAN_a8,
+    PIXMAN_r3g3b2,
+    PIXMAN_b2g3r3,
+    PIXMAN_a2r2g2b2,
+    PIXMAN_a2b2g2r2,
+    PIXMAN_c8,
+    PIXMAN_g8,
+    PIXMAN_x4c4,
+    PIXMAN_x4g4,
+    PIXMAN_c4,
+    PIXMAN_g4,
+    PIXMAN_g1,
+    PIXMAN_x4a4,
+    PIXMAN_a4,
+    PIXMAN_r1g2b1,
+    PIXMAN_b1g2r1,
+    PIXMAN_a1r1g1b1,
+    PIXMAN_a1b1g1r1,
+    PIXMAN_a1,
+    -1
+};
+
+static pixman_format_code_t mask_fmt_list[] = {
+    PIXMAN_a8r8g8b8,
+    PIXMAN_a8,
+    PIXMAN_a4,
+    PIXMAN_a1,
+    -1
+};
+
+
+/*
+ * Composite operation with pseudorandom images
+ */
+uint32_t
+test_composite (uint32_t initcrc, int testnum, int verbose)
+{
+    int i;
+    pixman_image_t *src_img = NULL;
+    pixman_image_t *dst_img = NULL;
+    pixman_image_t *mask_img = NULL;
+    int src_width, src_height;
+    int dst_width, dst_height;
+    int src_stride, dst_stride;
+    int src_x, src_y;
+    int dst_x, dst_y;
+    int mask_x, mask_y;
+    int w, h;
+    int op;
+    pixman_format_code_t src_fmt, dst_fmt, mask_fmt;
+    uint32_t *dstbuf, *srcbuf, *maskbuf;
+    uint32_t crc32;
+    int max_width, max_height, max_extra_stride;
+
+    max_width = max_height = 24 + testnum / 10000;
+    max_extra_stride = 4 + testnum / 1000000;
+
+    if (max_width > 256)
+	max_width = 256;
+
+    if (max_height > 16)
+	max_height = 16;
+
+    if (max_extra_stride > 8)
+	max_extra_stride = 8;
+
+    lcg_srand (testnum);
+
+    op = op_list[lcg_rand_n (sizeof (op_list) / sizeof (op_list[0]))];
+
+    if (lcg_rand_n (8))
+    {
+	/* normal image */
+	src_img = create_random_image (img_fmt_list, max_width, max_height,
+				       max_extra_stride, &src_fmt);
+    }
+    else
+    {
+	/* solid case */
+	src_img = create_random_image (img_fmt_list, 1, 1,
+				       max_extra_stride, &src_fmt);
+
+	pixman_image_set_repeat (src_img, PIXMAN_REPEAT_NORMAL);
+    }
+
+    dst_img = create_random_image (img_fmt_list, max_width, max_height,
+				   max_extra_stride, &dst_fmt);
+
+    src_width = pixman_image_get_width (src_img);
+    src_height = pixman_image_get_height (src_img);
+    src_stride = pixman_image_get_stride (src_img);
+
+    dst_width = pixman_image_get_width (dst_img);
+    dst_height = pixman_image_get_height (dst_img);
+    dst_stride = pixman_image_get_stride (dst_img);
+
+    dstbuf = pixman_image_get_data (dst_img);
+    srcbuf = pixman_image_get_data (src_img);
+
+    src_x = lcg_rand_n (src_width);
+    src_y = lcg_rand_n (src_height);
+    dst_x = lcg_rand_n (dst_width);
+    dst_y = lcg_rand_n (dst_height);
+
+    mask_img = NULL;
+    mask_fmt = -1;
+    mask_x = 0;
+    mask_y = 0;
+    maskbuf = NULL;
+
+    if ((src_fmt == PIXMAN_x8r8g8b8 || src_fmt == PIXMAN_x8b8g8r8) &&
+	(lcg_rand_n (4) == 0))
+    {
+	/* PIXBUF */
+	mask_fmt = lcg_rand_n (2) ? PIXMAN_a8r8g8b8 : PIXMAN_a8b8g8r8;
+	mask_img = pixman_image_create_bits (mask_fmt,
+	                                     src_width,
+	                                     src_height,
+	                                     srcbuf,
+	                                     src_stride);
+	mask_x = src_x;
+	mask_y = src_y;
+	maskbuf = srcbuf;
+    }
+    else if (lcg_rand_n (2))
+    {
+	if (lcg_rand_n (2))
+	{
+	    mask_img = create_random_image (mask_fmt_list, max_width, max_height,
+					   max_extra_stride, &mask_fmt);
+	}
+	else
+	{
+	    /* solid case */
+	    mask_img = create_random_image (mask_fmt_list, 1, 1,
+					   max_extra_stride, &mask_fmt);
+	    pixman_image_set_repeat (mask_img, PIXMAN_REPEAT_NORMAL);
+	}
+
+	if (lcg_rand_n (2))
+	    pixman_image_set_component_alpha (mask_img, 1);
+
+	mask_x = lcg_rand_n (pixman_image_get_width (mask_img));
+	mask_y = lcg_rand_n (pixman_image_get_height (mask_img));
+    }
+
+
+    w = lcg_rand_n (dst_width - dst_x + 1);
+    h = lcg_rand_n (dst_height - dst_y + 1);
+
+    if (verbose)
+    {
+	printf ("op=%d, src_fmt=%08X, dst_fmt=%08X, mask_fmt=%08X\n",
+	    op, src_fmt, dst_fmt, mask_fmt);
+	printf ("src_width=%d, src_height=%d, dst_width=%d, dst_height=%d\n",
+	    src_width, src_height, dst_width, dst_height);
+	printf ("src_x=%d, src_y=%d, dst_x=%d, dst_y=%d\n",
+	    src_x, src_y, dst_x, dst_y);
+	printf ("src_stride=%d, dst_stride=%d\n",
+	    src_stride, dst_stride);
+	printf ("w=%d, h=%d\n", w, h);
+    }
+
+    pixman_image_composite (op, src_img, mask_img, dst_img,
+			    src_x, src_y, mask_x, mask_y, dst_x, dst_y, w, h);
+
+    if (verbose)
+    {
+	int j;
+
+	printf ("---\n");
+	for (i = 0; i < dst_height; i++)
+	{
+	    for (j = 0; j < dst_stride; j++)
+	    {
+		if (j == (dst_width * PIXMAN_FORMAT_BPP (dst_fmt) + 7) / 8)
+		    printf ("| ");
+
+		printf ("%02X ", *((uint8_t *)dstbuf + i * dst_stride + j));
+	    }
+	    printf ("\n");
+	}
+	printf ("---\n");
+    }
+
+    free_random_image (initcrc, src_img, -1);
+    crc32 = free_random_image (initcrc, dst_img, dst_fmt);
+
+    if (mask_img)
+    {
+	if (srcbuf == maskbuf)
+	    pixman_image_unref(mask_img);
+	else
+	    free_random_image (initcrc, mask_img, -1);
+    }
+
+
+    return crc32;
+}
+
+static void
+initialize_palette (void)
+{
+    int i;
+
+    for (i = 0; i < PIXMAN_MAX_INDEXED; ++i)
+	palette.rgba[i] = lcg_rand ();
+
+    for (i = 0; i < 32768; ++i)
+	palette.ent[i] = lcg_rand() & 0xff;
+}
+
+int
+main (int argc, char *argv[])
+{
+    int i, n1 = 1, n2 = 0;
+    uint32_t crc = 0;
+    int verbose = getenv ("VERBOSE") != NULL;
+
+    initialize_palette();
+
+    if (argc >= 3)
+    {
+	n1 = atoi (argv[1]);
+	n2 = atoi (argv[2]);
+    }
+    else if (argc >= 2)
+    {
+	n2 = atoi (argv[1]);
+    }
+    else
+    {
+	n1 = 1;
+	n2 = 2000000;
+    }
+
+    if (n2 < 0)
+    {
+	crc = test_composite (0, abs (n2), 1);
+	printf ("crc32=%08X\n", crc);
+    }
+    else
+    {
+	for (i = n1; i <= n2; i++)
+	{
+	    crc = test_composite (crc, i, 0);
+
+	    if (verbose)
+		printf ("%d: %08X\n", i, crc);
+	}
+	printf ("crc32=%08X\n", crc);
+
+	if (n2 == 2000000)
+	{
+	    /* Predefined value for running with all the fastpath functions
+	       disabled. It needs to be updated every time when changes are
+	       introduced to this program or behavior of pixman changes! */
+	    if (crc == 0xBBACC28D)
+	    {
+		printf ("blitters test passed\n");
+	    }
+	    else
+	    {
+		printf ("blitters test failed!\n");
+		return 1;
+	    }
+	}
+    }
+    return 0;
+}
diff --git a/pixman/test/clip-in.c b/pixman/test/clip-in.c
new file mode 100644
index 000000000..51579811f
--- /dev/null
+++ b/pixman/test/clip-in.c
@@ -0,0 +1,50 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include "pixman.h"
+#include "gtk-utils.h"
+
+/* This test demonstrates that clipping is done totally different depending
+ * on whether the source is transformed or not.
+ */
+int
+main (int argc, char **argv)
+{
+#define WIDTH 200
+#define HEIGHT 200
+
+#define SMALL 25
+    
+    uint32_t *sbits = malloc (SMALL * SMALL * 4);
+    uint32_t *bits = malloc (WIDTH * HEIGHT * 4);
+    pixman_transform_t trans = {
+    {
+	{ pixman_double_to_fixed (1.0), pixman_double_to_fixed (0), pixman_double_to_fixed (-0.1), },
+	{ pixman_double_to_fixed (0), pixman_double_to_fixed (1), pixman_double_to_fixed (-0.1), },
+	{ pixman_double_to_fixed (0), pixman_double_to_fixed (0), pixman_double_to_fixed (1.0) }
+    } };
+	  
+    pixman_image_t *src_img = pixman_image_create_bits (PIXMAN_a8r8g8b8, SMALL, SMALL, sbits, 4 * SMALL);
+    pixman_image_t *dest_img = pixman_image_create_bits (PIXMAN_a8r8g8b8, WIDTH, HEIGHT, bits, 4 * WIDTH);
+
+    memset (bits, 0xff, WIDTH * HEIGHT * 4);
+    memset (sbits, 0x00, SMALL * SMALL * 4);
+
+    pixman_image_composite (PIXMAN_OP_IN,
+			    src_img, NULL, dest_img,
+			    0, 0, 0, 0, SMALL, SMALL, 200, 200);
+    
+    pixman_image_set_transform (src_img, &trans);
+    
+    pixman_image_composite (PIXMAN_OP_IN,
+			    src_img, NULL, dest_img,
+			    0, 0, 0, 0, SMALL * 2, SMALL * 2, 200, 200);
+    
+    show_image (dest_img);
+    
+    pixman_image_unref (src_img);
+    pixman_image_unref (dest_img);
+    free (bits);
+    
+    return 0;
+}
diff --git a/pixman/test/clip-test.c b/pixman/test/clip-test.c
new file mode 100644
index 000000000..aa0df4482
--- /dev/null
+++ b/pixman/test/clip-test.c
@@ -0,0 +1,97 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include "pixman.h"
+#include "gtk-utils.h"
+
+#define WIDTH 200
+#define HEIGHT 200
+    
+static pixman_image_t *
+create_solid_bits (uint32_t pixel)
+{
+    uint32_t *pixels = malloc (WIDTH * HEIGHT * 4);
+    int i;
+    
+    for (i = 0; i < WIDTH * HEIGHT; ++i)
+	pixels[i] = pixel;
+
+    return pixman_image_create_bits (PIXMAN_a8r8g8b8,
+				     WIDTH, HEIGHT, 
+				     pixels,
+				     WIDTH * 4);
+}
+
+int
+main (int argc, char **argv)
+{
+    pixman_image_t *gradient_img;
+    pixman_image_t *src_img, *dst_img;
+    pixman_gradient_stop_t stops[2] =
+	{
+	    { pixman_int_to_fixed (0), { 0xffff, 0x0000, 0x0000, 0xffff } },
+	    { pixman_int_to_fixed (1), { 0xffff, 0xffff, 0x0000, 0xffff } }
+	};
+#if 0
+    pixman_point_fixed_t p1 = { 0, 0 };
+    pixman_point_fixed_t p2 = { pixman_int_to_fixed (WIDTH),
+				pixman_int_to_fixed (HEIGHT) };
+#endif
+    pixman_point_fixed_t c_inner;
+    pixman_point_fixed_t c_outer;
+    pixman_fixed_t r_inner;
+    pixman_fixed_t r_outer;
+    pixman_region32_t clip_region;
+    pixman_transform_t trans = {
+	{ { pixman_double_to_fixed (1.3), pixman_double_to_fixed (0), pixman_double_to_fixed (-0.5), },
+	  { pixman_double_to_fixed (0), pixman_double_to_fixed (1), pixman_double_to_fixed (-0.5), },
+	  { pixman_double_to_fixed (0), pixman_double_to_fixed (0), pixman_double_to_fixed (1.0) } 
+	}
+    };
+    
+    src_img = create_solid_bits (0xff0000ff);
+    
+    c_inner.x = pixman_double_to_fixed (100.0);
+    c_inner.y = pixman_double_to_fixed (100.0);
+    c_outer.x = pixman_double_to_fixed (100.0);
+    c_outer.y = pixman_double_to_fixed (100.0);
+    r_inner = 0;
+    r_outer = pixman_double_to_fixed (100.0);
+    
+    gradient_img = pixman_image_create_radial_gradient (&c_inner, &c_outer,
+							r_inner, r_outer,
+							stops, 2);
+
+#if 0
+    gradient_img = pixman_image_create_linear_gradient  (&p1, &p2,
+							 stops, 2);
+    
+#endif
+
+    pixman_image_composite (PIXMAN_OP_OVER, gradient_img, NULL, src_img,
+			    0, 0, 0, 0, 0, 0, WIDTH, HEIGHT);
+    
+    pixman_region32_init_rect (&clip_region, 50, 0, 100, 200);
+    pixman_image_set_clip_region32 (src_img, &clip_region);
+    pixman_image_set_source_clipping (src_img, TRUE);
+    pixman_image_set_has_client_clip (src_img, TRUE);
+    pixman_image_set_transform (src_img, &trans);
+    pixman_image_set_repeat (src_img, PIXMAN_REPEAT_NORMAL);
+    
+    dst_img = create_solid_bits (0xffff0000);
+    pixman_image_composite (PIXMAN_OP_OVER, src_img, NULL, dst_img,
+			    0, 0, 0, 0, 0, 0, WIDTH, HEIGHT);
+    
+
+#if 0
+    printf ("0, 0: %x\n", src[0]);
+    printf ("10, 10: %x\n", src[10 * 10 + 10]);
+    printf ("w, h: %x\n", src[(HEIGHT - 1) * 100 + (WIDTH - 1)]);
+#endif
+    
+    show_image (dst_img);
+    
+    pixman_image_unref (gradient_img);
+    pixman_image_unref (src_img);
+    
+    return 0;
+}
diff --git a/pixman/test/composite-test.c b/pixman/test/composite-test.c
new file mode 100644
index 000000000..5401abfdf
--- /dev/null
+++ b/pixman/test/composite-test.c
@@ -0,0 +1,192 @@
+#include <gtk/gtk.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include "pixman.h"
+#include "gtk-utils.h"
+
+#define WIDTH	60
+#define HEIGHT	60
+
+typedef struct {
+    const char *name;
+    pixman_op_t op;
+} operator_t;
+
+static const operator_t operators[] = {
+    { "CLEAR",		PIXMAN_OP_CLEAR },
+    { "SRC",		PIXMAN_OP_SRC },
+    { "DST",		PIXMAN_OP_DST },
+    { "OVER",		PIXMAN_OP_OVER },
+    { "OVER_REVERSE",	PIXMAN_OP_OVER_REVERSE },
+    { "IN",		PIXMAN_OP_IN },
+    { "IN_REVERSE",	PIXMAN_OP_IN_REVERSE },
+    { "OUT",		PIXMAN_OP_OUT },
+    { "OUT_REVERSE",	PIXMAN_OP_OUT_REVERSE },
+    { "ATOP",		PIXMAN_OP_ATOP },
+    { "ATOP_REVERSE",	PIXMAN_OP_ATOP_REVERSE },
+    { "XOR",		PIXMAN_OP_XOR },
+    { "ADD",		PIXMAN_OP_ADD },
+    { "SATURATE",	PIXMAN_OP_SATURATE },
+
+    { "MULTIPLY",	PIXMAN_OP_MULTIPLY },
+    { "SCREEN",		PIXMAN_OP_SCREEN },
+    { "OVERLAY",	PIXMAN_OP_OVERLAY },
+    { "DARKEN",		PIXMAN_OP_DARKEN },
+    { "LIGHTEN",	PIXMAN_OP_LIGHTEN },
+    { "COLOR_DODGE",	PIXMAN_OP_COLOR_DODGE },
+    { "COLOR_BURN",	PIXMAN_OP_COLOR_BURN },
+    { "HARD_LIGHT",	PIXMAN_OP_HARD_LIGHT },
+    { "SOFT_LIGHT",	PIXMAN_OP_SOFT_LIGHT },
+    { "DIFFERENCE",	PIXMAN_OP_DIFFERENCE },
+    { "EXCLUSION",	PIXMAN_OP_EXCLUSION },
+    { "HSL_HUE",	PIXMAN_OP_HSL_HUE },
+    { "HSL_SATURATION",	PIXMAN_OP_HSL_SATURATION },
+    { "HSL_COLOR",	PIXMAN_OP_HSL_COLOR },
+    { "HSL_LUMINOSITY",	PIXMAN_OP_HSL_LUMINOSITY },
+};
+
+static uint32_t
+reader (const void *src, int size)
+{
+    switch (size)
+    {
+    case 1:
+	return *(uint8_t *)src;
+    case 2:
+	return *(uint16_t *)src;
+    case 4:
+	return *(uint32_t *)src;
+    default:
+	g_assert_not_reached();
+    }
+}
+
+static void
+writer (void *src, uint32_t value, int size)
+{
+    switch (size)
+    {
+    case 1:
+	*(uint8_t *)src = value;
+	break;
+
+    case 2:
+	*(uint16_t *)src = value;
+	break;
+
+    case 4:
+	*(uint32_t *)src = value;
+	break;
+
+    default:
+        break;
+    }
+}
+
+int
+main (int argc, char **argv)
+{
+#define d2f pixman_double_to_fixed
+    
+    GtkWidget *window, *swindow;
+    GtkWidget *table;
+    uint32_t *dest = malloc (WIDTH * HEIGHT * 4);
+    uint32_t *src = malloc (WIDTH * HEIGHT * 4);
+    pixman_image_t *src_img;
+    pixman_image_t *dest_img;
+    pixman_point_fixed_t p1 = { -10 << 0, 0 };
+    pixman_point_fixed_t p2 = { WIDTH << 16, (HEIGHT - 10) << 16 };
+    uint16_t full = 0xcfff;
+    uint16_t low  = 0x5000;
+    uint16_t alpha = 0xffff;
+    pixman_gradient_stop_t stops[6] =
+    {
+	{ d2f (0.0), { full, low, low, alpha } },
+	{ d2f (0.25), { full, full, low, alpha } },
+	{ d2f (0.4), { low, full, low, alpha } },
+	{ d2f (0.5), { low, full, full, alpha } },
+	{ d2f (0.8), { low, low, full, alpha } },
+	{ d2f (1.0), { full, low, full, alpha } },
+    };
+	
+	    
+    int i;
+
+    gtk_init (&argc, &argv);
+
+    window = gtk_window_new (GTK_WINDOW_TOPLEVEL);
+
+    gtk_window_set_default_size (GTK_WINDOW (window), 800, 600);
+    
+    g_signal_connect (window, "delete-event",
+		      G_CALLBACK (gtk_main_quit),
+		      NULL);
+    table = gtk_table_new (G_N_ELEMENTS (operators) / 6, 6, TRUE);
+
+    src_img = pixman_image_create_linear_gradient (&p1, &p2, stops,
+						   sizeof (stops) / sizeof (stops[0]));
+
+    pixman_image_set_repeat (src_img, PIXMAN_REPEAT_PAD);
+    
+    dest_img = pixman_image_create_bits (PIXMAN_a8r8g8b8,
+					 WIDTH, HEIGHT,
+					 dest,
+					 WIDTH * 4);
+    pixman_image_set_accessors (dest_img, reader, writer);
+
+    for (i = 0; i < G_N_ELEMENTS (operators); ++i)
+    {
+	GtkWidget *image;
+	GdkPixbuf *pixbuf;
+	GtkWidget *vbox;
+	GtkWidget *label;
+	int j, k;
+
+	vbox = gtk_vbox_new (FALSE, 0);
+
+	label = gtk_label_new (operators[i].name);
+	gtk_box_pack_start (GTK_BOX (vbox), label, FALSE, FALSE, 6);
+	gtk_widget_show (label);
+
+	for (j = 0; j < HEIGHT; ++j)
+	{
+	    for (k = 0; k < WIDTH; ++k)
+		dest[j * WIDTH + k] = 0x7f6f6f00;
+	}
+	pixman_image_composite (operators[i].op, src_img, NULL, dest_img,
+				0, 0, 0, 0, 0, 0, WIDTH, HEIGHT);
+	pixbuf = pixbuf_from_argb32 (pixman_image_get_data (dest_img), TRUE,
+				     WIDTH, HEIGHT, WIDTH * 4);
+	image = gtk_image_new_from_pixbuf (pixbuf);
+	gtk_box_pack_start (GTK_BOX (vbox), image, FALSE, FALSE, 0);
+	gtk_widget_show (image);
+
+	gtk_table_attach_defaults (GTK_TABLE (table), vbox,
+				   i % 6, (i % 6) + 1, i / 6, (i / 6) + 1);
+	gtk_widget_show (vbox);
+
+	g_object_unref (pixbuf);
+    }
+
+    pixman_image_unref (src_img);
+    free (src);
+    pixman_image_unref (dest_img);
+    free (dest);
+
+    swindow = gtk_scrolled_window_new (NULL, NULL);
+    gtk_scrolled_window_set_policy (GTK_SCROLLED_WINDOW (swindow),
+				    GTK_POLICY_AUTOMATIC,
+				    GTK_POLICY_AUTOMATIC);
+    
+    gtk_scrolled_window_add_with_viewport (GTK_SCROLLED_WINDOW (swindow), table);
+    gtk_widget_show (table);
+
+    gtk_container_add (GTK_CONTAINER (window), swindow);
+    gtk_widget_show (swindow);
+
+    gtk_widget_show (window);
+
+    gtk_main ();
+
+    return 0;
+}
diff --git a/pixman/test/composite.c b/pixman/test/composite.c
new file mode 100644
index 000000000..0624cd37d
--- /dev/null
+++ b/pixman/test/composite.c
@@ -0,0 +1,901 @@
+/*
+ * Copyright © 2005 Eric Anholt
+ * Copyright © 2009 Chris Wilson
+ *
+ * Permission to use, copy, modify, distribute, and sell this software and its
+ * documentation for any purpose is hereby granted without fee, provided that
+ * the above copyright notice appear in all copies and that both that
+ * copyright notice and this permission notice appear in supporting
+ * documentation, and that the name of Eric Anholt not be used in
+ * advertising or publicity pertaining to distribution of the software without
+ * specific, written prior permission.  Eric Anholt makes no
+ * representations about the suitability of this software for any purpose.  It
+ * is provided "as is" without express or implied warranty.
+ *
+ * ERIC ANHOLT DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
+ * INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO
+ * EVENT SHALL ERIC ANHOLT BE LIABLE FOR ANY SPECIAL, INDIRECT OR
+ * CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE,
+ * DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
+ * TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+ * PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include <pixman.h>
+#include <stdio.h>
+#include <stdlib.h> /* abort() */
+#include <math.h>
+#include <config.h>
+
+#define FALSE 0
+#define TRUE !FALSE
+
+#define ARRAY_LENGTH(A) ((int) (sizeof (A) / sizeof ((A) [0])))
+#define min(a,b) ((a) <= (b) ? (a) : (b))
+#define max(a,b) ((a) >= (b) ? (a) : (b))
+
+typedef struct color_t color_t;
+typedef struct format_t format_t;
+typedef struct image_t image_t;
+typedef struct operator_t operator_t;
+
+struct color_t
+{
+    double r, g, b, a;
+};
+
+struct format_t
+{
+    pixman_format_code_t format;
+    const char *name;
+};
+
+static color_t colors[] =
+{
+    /* these are premultiplied in main() */
+    { 1.0, 1.0, 1.0, 1.0 },
+    { 1.0, 0.0, 0.0, 1.0 },
+    { 0.0, 1.0, 0.0, 1.0 },
+    { 0.0, 0.0, 1.0, 1.0 },
+    { 0.0, 0.0, 0.0, 1.0 },
+    { 0.5, 0.0, 0.0, 0.5 },
+};
+
+static uint16_t
+_color_double_to_short (double d)
+{
+    uint32_t i;
+
+    i = (uint32_t) (d * 65536);
+    i -= (i >> 16);
+
+    return i;
+}
+
+static void
+compute_pixman_color (const color_t *color,
+		      pixman_color_t *out)
+{
+    out->red   = _color_double_to_short (color->r);
+    out->green = _color_double_to_short (color->g);
+    out->blue  = _color_double_to_short (color->b);
+    out->alpha = _color_double_to_short (color->a);
+}
+
+static const format_t formats[] =
+{
+#define P(x) { PIXMAN_##x, #x }
+    P(a8),
+
+    /* 32bpp formats */
+    P(a8r8g8b8),
+    P(x8r8g8b8),
+    P(a8b8g8r8),
+    P(x8b8g8r8),
+    P(b8g8r8a8),
+    P(b8g8r8x8),
+
+    /* XXX: and here the errors begin! */
+#if 0
+    P(x2r10g10b10),
+    P(a2r10g10b10),
+    P(x2b10g10r10),
+    P(a2b10g10r10),
+
+    /* 24bpp formats */
+    P(r8g8b8),
+    P(b8g8r8),
+
+    /* 16bpp formats */
+    P(r5g6b5),
+    P(b5g6r5),
+
+    P(a1r5g5b5),
+    P(x1r5g5b5),
+    P(a1b5g5r5),
+    P(x1b5g5r5),
+    P(a4r4g4b4),
+    P(x4r4g4b4),
+    P(a4b4g4r4),
+    P(x4b4g4r4),
+
+    /* 8bpp formats */
+    P(a8),
+    P(r3g3b2),
+    P(b2g3r3),
+    P(a2r2g2b2),
+    P(a2b2g2r2),
+
+    P(x4a4),
+
+    /* 4bpp formats */
+    P(a4),
+    P(r1g2b1),
+    P(b1g2r1),
+    P(a1r1g1b1),
+    P(a1b1g1r1),
+
+    /* 1bpp formats */
+    P(a1)
+#endif
+#undef P
+};
+
+struct image_t
+{
+    pixman_image_t *image;
+    const format_t *format;
+    const color_t *color;
+    pixman_repeat_t repeat;
+    int size;
+};
+
+struct operator_t
+{
+    pixman_op_t op;
+    const char *name;
+};
+
+static const operator_t operators[] =
+{
+#define P(x) { PIXMAN_OP_##x, #x }
+    P(CLEAR),
+    P(SRC),
+    P(DST),
+    P(OVER),
+    P(OVER_REVERSE),
+    P(IN),
+    P(IN_REVERSE),
+    P(OUT),
+    P(OUT_REVERSE),
+    P(ATOP),
+    P(ATOP_REVERSE),
+    P(XOR),
+    P(ADD),
+    P(SATURATE),
+
+    P(DISJOINT_CLEAR),
+    P(DISJOINT_SRC),
+    P(DISJOINT_DST),
+    P(DISJOINT_OVER),
+    P(DISJOINT_OVER_REVERSE),
+    P(DISJOINT_IN),
+    P(DISJOINT_IN_REVERSE),
+    P(DISJOINT_OUT),
+    P(DISJOINT_OUT_REVERSE),
+    P(DISJOINT_ATOP),
+    P(DISJOINT_ATOP_REVERSE),
+    P(DISJOINT_XOR),
+
+    P(CONJOINT_CLEAR),
+    P(CONJOINT_SRC),
+    P(CONJOINT_DST),
+    P(CONJOINT_OVER),
+    P(CONJOINT_OVER_REVERSE),
+    P(CONJOINT_IN),
+    P(CONJOINT_IN_REVERSE),
+    P(CONJOINT_OUT),
+    P(CONJOINT_OUT_REVERSE),
+    P(CONJOINT_ATOP),
+    P(CONJOINT_ATOP_REVERSE),
+    P(CONJOINT_XOR),
+#undef P
+};
+
+static double
+calc_op (pixman_op_t op, double src, double dst, double srca, double dsta)
+{
+#define mult_chan(src, dst, Fa, Fb) min ((src) * (Fa) + (dst) * (Fb), 1.0)
+
+    double Fa, Fb;
+
+    switch (op)
+    {
+    case PIXMAN_OP_CLEAR:
+    case PIXMAN_OP_DISJOINT_CLEAR:
+    case PIXMAN_OP_CONJOINT_CLEAR:
+	return mult_chan (src, dst, 0.0, 0.0);
+
+    case PIXMAN_OP_SRC:
+    case PIXMAN_OP_DISJOINT_SRC:
+    case PIXMAN_OP_CONJOINT_SRC:
+	return mult_chan (src, dst, 1.0, 0.0);
+
+    case PIXMAN_OP_DST:
+    case PIXMAN_OP_DISJOINT_DST:
+    case PIXMAN_OP_CONJOINT_DST:
+	return mult_chan (src, dst, 0.0, 1.0);
+
+    case PIXMAN_OP_OVER:
+	return mult_chan (src, dst, 1.0, 1.0 - srca);
+
+    case PIXMAN_OP_OVER_REVERSE:
+	return mult_chan (src, dst, 1.0 - dsta, 1.0);
+
+    case PIXMAN_OP_IN:
+	return mult_chan (src, dst, dsta, 0.0);
+
+    case PIXMAN_OP_IN_REVERSE:
+	return mult_chan (src, dst, 0.0, srca);
+
+    case PIXMAN_OP_OUT:
+	return mult_chan (src, dst, 1.0 - dsta, 0.0);
+
+    case PIXMAN_OP_OUT_REVERSE:
+	return mult_chan (src, dst, 0.0, 1.0 - srca);
+
+    case PIXMAN_OP_ATOP:
+	return mult_chan (src, dst, dsta, 1.0 - srca);
+
+    case PIXMAN_OP_ATOP_REVERSE:
+	return mult_chan (src, dst, 1.0 - dsta,  srca);
+
+    case PIXMAN_OP_XOR:
+	return mult_chan (src, dst, 1.0 - dsta, 1.0 - srca);
+
+    case PIXMAN_OP_ADD:
+	return mult_chan (src, dst, 1.0, 1.0);
+
+    case PIXMAN_OP_SATURATE:
+    case PIXMAN_OP_DISJOINT_OVER_REVERSE:
+	if (srca == 0.0)
+	    Fa = 1.0;
+	else
+	    Fa = min (1.0, (1.0 - dsta) / srca);
+	return mult_chan (src, dst, Fa, 1.0);
+
+    case PIXMAN_OP_DISJOINT_OVER:
+	if (dsta == 0.0)
+	    Fb = 1.0;
+	else
+	    Fb = min (1.0, (1.0 - srca) / dsta);
+	return mult_chan (src, dst, 1.0, Fb);
+
+    case PIXMAN_OP_DISJOINT_IN:
+	if (srca == 0.0)
+	    Fa = 0.0;
+	else
+	    Fa = max (0.0, 1.0 - (1.0 - dsta) / srca);
+	return mult_chan (src, dst, Fa, 0.0);
+
+    case PIXMAN_OP_DISJOINT_IN_REVERSE:
+	if (dsta == 0.0)
+	    Fb = 0.0;
+	else
+	    Fb = max (0.0, 1.0 - (1.0 - srca) / dsta);
+	return mult_chan (src, dst, 0.0, Fb);
+
+    case PIXMAN_OP_DISJOINT_OUT:
+	if (srca == 0.0)
+	    Fa = 1.0;
+	else
+	    Fa = min (1.0, (1.0 - dsta) / srca);
+	return mult_chan (src, dst, Fa, 0.0);
+
+    case PIXMAN_OP_DISJOINT_OUT_REVERSE:
+	if (dsta == 0.0)
+	    Fb = 1.0;
+	else
+	    Fb = min (1.0, (1.0 - srca) / dsta);
+	return mult_chan (src, dst, 0.0, Fb);
+
+    case PIXMAN_OP_DISJOINT_ATOP:
+	if (srca == 0.0)
+	    Fa = 0.0;
+	else
+	    Fa = max (0.0, 1.0 - (1.0 - dsta) / srca);
+	if (dsta == 0.0)
+	    Fb = 1.0;
+	else
+	    Fb = min (1.0, (1.0 - srca) / dsta);
+	return mult_chan (src, dst, Fa, Fb);
+
+    case PIXMAN_OP_DISJOINT_ATOP_REVERSE:
+	if (srca == 0.0)
+	    Fa = 1.0;
+	else
+	    Fa = min (1.0, (1.0 - dsta) / srca);
+	if (dsta == 0.0)
+	    Fb = 0.0;
+	else
+	    Fb = max (0.0, 1.0 - (1.0 - srca) / dsta);
+	return mult_chan (src, dst, Fa, Fb);
+
+    case PIXMAN_OP_DISJOINT_XOR:
+	if (srca == 0.0)
+	    Fa = 1.0;
+	else
+	    Fa = min (1.0, (1.0 - dsta) / srca);
+	if (dsta == 0.0)
+	    Fb = 1.0;
+	else
+	    Fb = min (1.0, (1.0 - srca) / dsta);
+	return mult_chan (src, dst, Fa, Fb);
+
+    case PIXMAN_OP_CONJOINT_OVER:
+	if (dsta == 0.0)
+	    Fb = 0.0;
+	else
+	    Fb = max (0.0, 1.0 - srca / dsta);
+	return mult_chan (src, dst, 1.0, Fb);
+
+    case PIXMAN_OP_CONJOINT_OVER_REVERSE:
+	if (srca == 0.0)
+	    Fa = 0.0;
+	else
+	    Fa = max (0.0, 1.0 - dsta / srca);
+	return mult_chan (src, dst, Fa, 1.0);
+
+    case PIXMAN_OP_CONJOINT_IN:
+	if (srca == 0.0)
+	    Fa = 1.0;
+	else
+	    Fa = min (1.0, dsta / srca);
+	return mult_chan (src, dst, Fa, 0.0);
+
+    case PIXMAN_OP_CONJOINT_IN_REVERSE:
+	if (dsta == 0.0)
+	    Fb = 1.0;
+	else
+	    Fb = min (1.0, srca / dsta);
+	return mult_chan (src, dst, 0.0, Fb);
+
+    case PIXMAN_OP_CONJOINT_OUT:
+	if (srca == 0.0)
+	    Fa = 0.0;
+	else
+	    Fa = max (0.0, 1.0 - dsta / srca);
+	return mult_chan (src, dst, Fa, 0.0);
+
+    case PIXMAN_OP_CONJOINT_OUT_REVERSE:
+	if (dsta == 0.0)
+	    Fb = 0.0;
+	else
+	    Fb = max (0.0, 1.0 - srca / dsta);
+	return mult_chan (src, dst, 0.0, Fb);
+
+    case PIXMAN_OP_CONJOINT_ATOP:
+	if (srca == 0.0)
+	    Fa = 1.0;
+	else
+	    Fa = min (1.0, dsta / srca);
+	if (dsta == 0.0)
+	    Fb = 0.0;
+	else
+	    Fb = max (0.0, 1.0 - srca / dsta);
+	return mult_chan (src, dst, Fa, Fb);
+
+    case PIXMAN_OP_CONJOINT_ATOP_REVERSE:
+	if (srca == 0.0)
+	    Fa = 0.0;
+	else
+	    Fa = max (0.0, 1.0 - dsta / srca);
+	if (dsta == 0.0)
+	    Fb = 1.0;
+	else
+	    Fb = min (1.0, srca / dsta);
+	return mult_chan (src, dst, Fa, Fb);
+
+    case PIXMAN_OP_CONJOINT_XOR:
+	if (srca == 0.0)
+	    Fa = 0.0;
+	else
+	    Fa = max (0.0, 1.0 - dsta / srca);
+	if (dsta == 0.0)
+	    Fb = 0.0;
+	else
+	    Fb = max (0.0, 1.0 - srca / dsta);
+	return mult_chan (src, dst, Fa, Fb);
+
+    case PIXMAN_OP_MULTIPLY:
+    case PIXMAN_OP_SCREEN:
+    case PIXMAN_OP_OVERLAY:
+    case PIXMAN_OP_DARKEN:
+    case PIXMAN_OP_LIGHTEN:
+    case PIXMAN_OP_COLOR_DODGE:
+    case PIXMAN_OP_COLOR_BURN:
+    case PIXMAN_OP_HARD_LIGHT:
+    case PIXMAN_OP_SOFT_LIGHT:
+    case PIXMAN_OP_DIFFERENCE:
+    case PIXMAN_OP_EXCLUSION:
+    case PIXMAN_OP_HSL_HUE:
+    case PIXMAN_OP_HSL_SATURATION:
+    case PIXMAN_OP_HSL_COLOR:
+    case PIXMAN_OP_HSL_LUMINOSITY:
+    default:
+	abort();
+    }
+#undef mult_chan
+}
+
+static void
+do_composite (pixman_op_t op,
+	      const color_t *src,
+	      const color_t *mask,
+	      const color_t *dst,
+	      color_t *result,
+	      pixman_bool_t component_alpha)
+{
+    color_t srcval, srcalpha;
+
+    if (mask == NULL)
+    {
+	srcval = *src;
+
+	srcalpha.r = src->a;
+	srcalpha.g = src->a;
+	srcalpha.b = src->a;
+	srcalpha.a = src->a;
+    }
+    else if (component_alpha)
+    {
+	srcval.r = src->r * mask->r;
+	srcval.g = src->g * mask->g;
+	srcval.b = src->b * mask->b;
+	srcval.a = src->a * mask->a;
+
+	srcalpha.r = src->a * mask->r;
+	srcalpha.g = src->a * mask->g;
+	srcalpha.b = src->a * mask->b;
+	srcalpha.a = src->a * mask->a;
+    }
+    else
+    {
+	srcval.r = src->r * mask->a;
+	srcval.g = src->g * mask->a;
+	srcval.b = src->b * mask->a;
+	srcval.a = src->a * mask->a;
+
+	srcalpha.r = src->a * mask->a;
+	srcalpha.g = src->a * mask->a;
+	srcalpha.b = src->a * mask->a;
+	srcalpha.a = src->a * mask->a;
+    }
+
+    result->r = calc_op (op, srcval.r, dst->r, srcalpha.r, dst->a);
+    result->g = calc_op (op, srcval.g, dst->g, srcalpha.g, dst->a);
+    result->b = calc_op (op, srcval.b, dst->b, srcalpha.b, dst->a);
+    result->a = calc_op (op, srcval.a, dst->a, srcalpha.a, dst->a);
+}
+
+static void
+color_correct (pixman_format_code_t format,
+	       color_t *color)
+{
+#define round_pix(pix, mask) \
+    ((int)((pix) * (mask) + .5) / (double) (mask))
+
+    if (PIXMAN_FORMAT_R (format) == 0)
+    {
+	color->r = 0.0;
+	color->g = 0.0;
+	color->b = 0.0;
+    }
+    else
+    {
+	color->r = round_pix (color->r, PIXMAN_FORMAT_R (format));
+	color->g = round_pix (color->g, PIXMAN_FORMAT_G (format));
+	color->b = round_pix (color->b, PIXMAN_FORMAT_B (format));
+    }
+
+    if (PIXMAN_FORMAT_A (format) == 0)
+	color->a = 1.0;
+    else
+	color->a = round_pix (color->a, PIXMAN_FORMAT_A (format));
+
+#undef round_pix
+}
+
+static void
+get_pixel (pixman_image_t *image,
+	   pixman_format_code_t format,
+	   color_t *color)
+{
+#define MASK(N) ((1UL << (N))-1)
+
+    unsigned long rs, gs, bs, as;
+    int a, r, g, b;
+    unsigned long val;
+
+    val = *(unsigned long *) pixman_image_get_data (image);
+#ifdef WORDS_BIGENDIAN
+    val >>= 8 * sizeof(val) - PIXMAN_FORMAT_BPP (format);
+#endif
+
+    /* Number of bits in each channel */
+    a = PIXMAN_FORMAT_A (format);
+    r = PIXMAN_FORMAT_R (format);
+    g = PIXMAN_FORMAT_G (format);
+    b = PIXMAN_FORMAT_B (format);
+
+    switch (PIXMAN_FORMAT_TYPE (format))
+    {
+    case PIXMAN_TYPE_ARGB:
+        bs = 0;
+        gs = b + bs;
+        rs = g + gs;
+        as = r + rs;
+	break;
+
+    case PIXMAN_TYPE_ABGR:
+        rs = 0;
+        gs = r + rs;
+        bs = g + gs;
+        as = b + bs;
+	break;
+
+    case PIXMAN_TYPE_BGRA:
+        as = 0;
+	rs = PIXMAN_FORMAT_BPP (format) - (b + g + r);
+        gs = r + rs;
+        bs = g + gs;
+	break;
+
+    case PIXMAN_TYPE_A:
+        as = 0;
+        rs = 0;
+        gs = 0;
+        bs = 0;
+	break;
+
+    case PIXMAN_TYPE_OTHER:
+    case PIXMAN_TYPE_COLOR:
+    case PIXMAN_TYPE_GRAY:
+    case PIXMAN_TYPE_YUY2:
+    case PIXMAN_TYPE_YV12:
+    default:
+	abort ();
+        as = 0;
+        rs = 0;
+        gs = 0;
+        bs = 0;
+	break;
+    }
+
+    if (MASK (a) != 0)
+	color->a = ((val >> as) & MASK (a)) / (double) MASK (a);
+    else
+	color->a = 1.0;
+
+    if (MASK (r) != 0)
+    {
+	color->r = ((val >> rs) & MASK (r)) / (double) MASK (r);
+	color->g = ((val >> gs) & MASK (g)) / (double) MASK (g);
+	color->b = ((val >> bs) & MASK (b)) / (double) MASK (b);
+    }
+    else
+    {
+	color->r = 0.0;
+	color->g = 0.0;
+	color->b = 0.0;
+    }
+
+#undef MASK
+}
+
+static double
+eval_diff (color_t *expected, color_t *test)
+{
+    double rscale, gscale, bscale, ascale;
+    double rdiff, gdiff, bdiff, adiff;
+
+    /* XXX: Need to be provided mask shifts so we can produce useful error
+     * values.
+     */
+    rscale = 1.0 * (1 << 5);
+    gscale = 1.0 * (1 << 6);
+    bscale = 1.0 * (1 << 5);
+    ascale = 1.0 * 32;
+
+    rdiff = fabs (test->r - expected->r) * rscale;
+    bdiff = fabs (test->g - expected->g) * gscale;
+    gdiff = fabs (test->b - expected->b) * bscale;
+    adiff = fabs (test->a - expected->a) * ascale;
+
+    return max (max (max (rdiff, gdiff), bdiff), adiff);
+}
+
+static char *
+describe_image (image_t *info, char *buf, int buflen)
+{
+    if (info->size)
+    {
+	snprintf (buf, buflen, "%s %dx%d%s",
+		  info->format->name,
+		  info->size, info->size,
+		  info->repeat ? "R" :"");
+    }
+    else
+    {
+	snprintf (buf, buflen, "solid");
+    }
+
+    return buf;
+}
+
+/* Test a composite of a given operation, source, mask, and destination
+ * picture.
+ * Fills the window, and samples from the 0,0 pixel corner.
+ */
+static pixman_bool_t
+composite_test (image_t *dst,
+		const operator_t *op,
+		image_t *src,
+		image_t *mask,
+		pixman_bool_t component_alpha)
+{
+    pixman_color_t fill;
+    pixman_rectangle16_t rect;
+    color_t expected, result, tdst, tsrc, tmsk;
+    double diff;
+    pixman_bool_t success = TRUE;
+
+    compute_pixman_color (dst->color, &fill);
+    rect.x = rect.y = 0;
+    rect.width = rect.height = dst->size;
+    pixman_image_fill_rectangles (PIXMAN_OP_SRC, dst->image,
+				  &fill, 1, &rect);
+
+    if (mask != NULL)
+    {
+	pixman_image_set_component_alpha (mask->image, component_alpha);
+	pixman_image_composite (op->op, src->image, mask->image, dst->image,
+				0, 0,
+				0, 0,
+				0, 0,
+				dst->size, dst->size);
+
+	tmsk = *mask->color;
+	if (mask->size)
+	{
+	    color_correct (mask->format->format, &tmsk);
+
+	    if (component_alpha &&
+		PIXMAN_FORMAT_R (mask->format->format) == 0)
+	    {
+		/* Ax component-alpha masks expand alpha into
+		 * all color channels.
+		 */
+		tmsk.r = tmsk.g = tmsk.b = tmsk.a;
+	    }
+	}
+    }
+    else
+    {
+	pixman_image_composite (op->op, src->image, NULL, dst->image,
+				0, 0,
+				0, 0,
+				0, 0,
+				dst->size, dst->size);
+    }
+    get_pixel (dst->image, dst->format->format, &result);
+
+    tdst = *dst->color;
+    color_correct (dst->format->format, &tdst);
+    tsrc = *src->color;
+    if (src->size)
+	color_correct (src->format->format, &tsrc);
+    do_composite (op->op, &tsrc, mask ? &tmsk : NULL, &tdst,
+		  &expected, component_alpha);
+    color_correct (dst->format->format, &expected);
+
+    diff = eval_diff (&expected, &result);
+    if (diff > 3.0)
+    {
+	char buf[40];
+
+	snprintf (buf, sizeof (buf),
+		  "%s %scomposite",
+		  op->name,
+		  component_alpha ? "CA " : "");
+
+	printf ("%s test error of %.4f --\n"
+		"           R    G    B    A\n"
+		"got:       %.2f %.2f %.2f %.2f [%08lx]\n"
+		"expected:  %.2f %.2f %.2f %.2f\n",
+		buf, diff,
+		result.r, result.g, result.b, result.a,
+		*(unsigned long *) pixman_image_get_data (dst->image),
+		expected.r, expected.g, expected.b, expected.a);
+	
+	if (mask != NULL)
+	{
+	    printf ("src color: %.2f %.2f %.2f %.2f\n"
+		    "msk color: %.2f %.2f %.2f %.2f\n"
+		    "dst color: %.2f %.2f %.2f %.2f\n",
+		    src->color->r, src->color->g,
+		    src->color->b, src->color->a,
+		    mask->color->r, mask->color->g,
+		    mask->color->b, mask->color->a,
+		    dst->color->r, dst->color->g,
+		    dst->color->b, dst->color->a);
+	    printf ("src: %s, ", describe_image (src, buf, sizeof (buf)));
+	    printf ("mask: %s, ", describe_image (mask, buf, sizeof (buf)));
+	    printf ("dst: %s\n\n", describe_image (dst, buf, sizeof (buf)));
+	}
+	else
+	{
+	    printf ("src color: %.2f %.2f %.2f %.2f\n"
+		    "dst color: %.2f %.2f %.2f %.2f\n",
+		    src->color->r, src->color->g,
+		    src->color->b, src->color->a,
+		    dst->color->r, dst->color->g,
+		    dst->color->b, dst->color->a);
+	    printf ("src: %s, ", describe_image (src, buf, sizeof (buf)));
+	    printf ("dst: %s\n\n", describe_image (dst, buf, sizeof (buf)));
+	}
+
+	success = FALSE;
+    }
+
+    return success;
+}
+
+#define REPEAT 0x01000000
+#define FLAGS  0xff000000
+
+static void
+image_init (image_t *info,
+	    int color,
+	    int format,
+	    int size)
+{
+    pixman_color_t fill;
+
+    info->color = &colors[color];
+    compute_pixman_color (info->color, &fill);
+
+    info->format = &formats[format];
+    info->size = size & ~FLAGS;
+    info->repeat = PIXMAN_REPEAT_NONE;
+
+    if (info->size)
+    {
+	pixman_rectangle16_t rect;
+
+	info->image = pixman_image_create_bits (info->format->format,
+						info->size, info->size,
+						NULL, 0);
+
+	rect.x = rect.y = 0;
+	rect.width = rect.height = info->size;
+	pixman_image_fill_rectangles (PIXMAN_OP_SRC, info->image, &fill,
+				      1, &rect);
+
+	if (size & REPEAT)
+	{
+	    pixman_image_set_repeat (info->image, PIXMAN_REPEAT_NORMAL);
+	    info->repeat = PIXMAN_REPEAT_NORMAL;
+	}
+    }
+    else
+    {
+	info->image = pixman_image_create_solid_fill (&fill);
+    }
+}
+
+static void
+image_fini (image_t *info)
+{
+    pixman_image_unref (info->image);
+}
+
+int
+main (void)
+{
+    pixman_bool_t ok, group_ok = TRUE, ca;
+    int i, d, m, s;
+    int tests_passed = 0, tests_total = 0;
+    int sizes[] = { 1, 1 | REPEAT, 10 };
+    int num_tests;
+
+    for (i = 0; i < ARRAY_LENGTH (colors); i++)
+    {
+	colors[i].r *= colors[i].a;
+	colors[i].g *= colors[i].a;
+	colors[i].b *= colors[i].a;
+    }
+
+    num_tests = ARRAY_LENGTH (colors) * ARRAY_LENGTH (formats);
+
+    for (d = 0; d < num_tests; d++)
+    {
+	image_t dst;
+
+	image_init (
+	    &dst, d / ARRAY_LENGTH (formats), d % ARRAY_LENGTH (formats), 1);
+
+
+	for (s = -ARRAY_LENGTH (colors);
+	     s < ARRAY_LENGTH (sizes) * num_tests;
+	     s++)
+	{
+	    image_t src;
+
+	    if (s < 0)
+	    {
+		image_init (&src, -s - 1, 0, 0);
+	    }
+	    else
+	    {
+		image_init (&src,
+			    s / ARRAY_LENGTH (sizes) / ARRAY_LENGTH (formats),
+			    s / ARRAY_LENGTH (sizes) % ARRAY_LENGTH (formats),
+			    sizes[s % ARRAY_LENGTH (sizes)]);
+	    }
+
+	    for (m = -ARRAY_LENGTH (colors);
+		 m < ARRAY_LENGTH (sizes) * num_tests;
+		 m++)
+	    {
+		image_t mask;
+
+		if (m < 0)
+		{
+		    image_init (&mask, -m - 1, 0, 0);
+		}
+		else
+		{
+		    image_init (
+			&mask,
+			m / ARRAY_LENGTH (sizes) / ARRAY_LENGTH (formats),
+			m / ARRAY_LENGTH (sizes) % ARRAY_LENGTH (formats),
+			sizes[m % ARRAY_LENGTH (sizes)]);
+		}
+
+		for (ca = -1; ca <= 1; ca++)
+		{
+		    for (i = 0; i < ARRAY_LENGTH (operators); i++)
+		    {
+			const operator_t *op = &operators[i];
+
+			switch (ca)
+			{
+			case -1:
+			    ok = composite_test (&dst, op, &src, NULL, FALSE);
+			    break;
+			case 0:
+			    ok = composite_test (&dst, op, &src, &mask, FALSE);
+			    break;
+			case 1:
+			    ok = composite_test (&dst, op, &src, &mask,
+						 mask.size? TRUE : FALSE);
+			    break;
+                        default:
+                            break;
+			}
+			group_ok = group_ok && ok;
+			tests_passed += ok;
+			tests_total++;
+		    }
+		}
+
+		image_fini (&mask);
+	    }
+	    image_fini (&src);
+	}
+	image_fini (&dst);
+    }
+
+    return group_ok == FALSE;
+}
diff --git a/pixman/test/convolution-test.c b/pixman/test/convolution-test.c
new file mode 100644
index 000000000..da284af7b
--- /dev/null
+++ b/pixman/test/convolution-test.c
@@ -0,0 +1,47 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include "pixman.h"
+#include "gtk-utils.h"
+
+int
+main (int argc, char **argv)
+{
+#define WIDTH 200
+#define HEIGHT 200
+
+#define d2f pixman_double_to_fixed
+    
+    uint32_t *src = malloc (WIDTH * HEIGHT * 4);
+    uint32_t *mask = malloc (WIDTH * HEIGHT * 4);
+    uint32_t *dest = malloc (WIDTH * HEIGHT * 4);
+    pixman_fixed_t convolution[] =
+    {
+	d2f (3), d2f (3),
+	d2f (0.5), d2f (0.5), d2f (0.5),
+	d2f (0.5), d2f (0.5), d2f (0.5),
+	d2f (0.5), d2f (0.5), d2f (0.5),
+    };
+    pixman_image_t *simg, *mimg, *dimg;
+
+    int i;
+
+    for (i = 0; i < WIDTH * HEIGHT; ++i)
+    {
+	src[i] = 0x7f007f00;
+	mask[i] = (i % 256) * 0x01000000;
+	dest[i] = 0;
+    }
+
+    simg = pixman_image_create_bits (PIXMAN_a8r8g8b8, WIDTH, HEIGHT, src, WIDTH * 4);
+    mimg = pixman_image_create_bits (PIXMAN_a8r8g8b8, WIDTH, HEIGHT, mask, WIDTH * 4);
+    dimg = pixman_image_create_bits (PIXMAN_a8r8g8b8, WIDTH, HEIGHT, dest, WIDTH * 4);
+
+    pixman_image_set_filter (mimg, PIXMAN_FILTER_CONVOLUTION,
+			     convolution, 11);
+
+    pixman_image_composite (PIXMAN_OP_OVER, simg, mimg, dimg, 0, 0, 0, 0, 0, 0, WIDTH, HEIGHT);
+
+    show_image (dimg);
+    
+    return 0;
+}
diff --git a/pixman/test/fetch-test.c b/pixman/test/fetch-test.c
new file mode 100644
index 000000000..2ca16ddbf
--- /dev/null
+++ b/pixman/test/fetch-test.c
@@ -0,0 +1,201 @@
+#include <assert.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include "pixman.h"
+#include <config.h>
+
+#define SIZE 1024
+
+static pixman_indexed_t mono_palette =
+{
+    .rgba = { 0x00000000, 0x00ffffff },
+};
+
+
+typedef struct {
+    pixman_format_code_t format;
+    int width, height;
+    int stride;
+    uint32_t src[SIZE];
+    uint32_t dst[SIZE];
+    pixman_indexed_t *indexed;
+} testcase_t;
+
+static testcase_t testcases[] =
+{
+    {
+	.format = PIXMAN_a8r8g8b8,
+	.width = 2, .height = 2,
+	.stride = 8,
+	.src = { 0x00112233, 0x44556677,
+	         0x8899aabb, 0xccddeeff },
+	.dst = { 0x00112233, 0x44556677,
+	         0x8899aabb, 0xccddeeff },
+	.indexed = NULL,
+    },
+    {
+	.format = PIXMAN_g1,
+	.width = 8, .height = 2,
+	.stride = 4,
+#ifdef WORDS_BIGENDIAN
+	.src =
+	{
+	    0xaa000000,
+	    0x55000000
+	},
+#else
+	.src =
+	{
+	    0x00000055,
+	    0x000000aa
+	},
+#endif
+	.dst =
+	{
+	    0x00ffffff, 0x00000000, 0x00ffffff, 0x00000000, 0x00ffffff, 0x00000000, 0x00ffffff, 0x00000000,
+	    0x00000000, 0x00ffffff, 0x00000000, 0x00ffffff, 0x00000000, 0x00ffffff, 0x00000000, 0x00ffffff
+	},
+	.indexed = &mono_palette,
+    },
+#if 0
+    {
+	.format = PIXMAN_g8,
+	.width = 4, .height = 2,
+	.stride = 4,
+	.src = { 0x01234567,
+	         0x89abcdef },
+	.dst = { 0x00010101, 0x00232323, 0x00454545, 0x00676767,
+	         0x00898989, 0x00ababab, 0x00cdcdcd, 0x00efefef, },
+    },
+#endif
+    /* FIXME: make this work on big endian */
+    {
+	.format = PIXMAN_yv12,
+	.width = 8, .height = 2,
+	.stride = 8,
+#ifdef WORDS_BIGENDIAN
+	.src =
+	{
+	    0x00ff00ff, 0x00ff00ff,
+	    0xff00ff00, 0xff00ff00,
+	    0x80ff8000,
+	    0x800080ff
+	},
+#else
+	.src =
+	{
+	    0xff00ff00, 0xff00ff00,
+	    0x00ff00ff, 0x00ff00ff,
+	    0x0080ff80,
+	    0xff800080
+	},
+#endif
+	.dst =
+	{
+	    0xff000000, 0xffffffff, 0xffb80000, 0xffffe113,
+	    0xff000000, 0xffffffff, 0xff0023ee, 0xff4affff,
+	    0xffffffff, 0xff000000, 0xffffe113, 0xffb80000,
+	    0xffffffff, 0xff000000, 0xff4affff, 0xff0023ee,
+	},
+    },
+};
+
+int n_test_cases = sizeof(testcases)/sizeof(testcases[0]);
+
+
+static uint32_t
+reader (const void *src, int size)
+{
+    switch (size)
+    {
+    case 1:
+	return *(uint8_t *)src;
+    case 2:
+	return *(uint16_t *)src;
+    case 4:
+	return *(uint32_t *)src;
+    default:
+	assert(0);
+    }
+}
+
+
+static void
+writer (void *src, uint32_t value, int size)
+{
+    switch (size)
+    {
+    case 1:
+	*(uint8_t *)src = value;
+	break;
+    case 2:
+	*(uint16_t *)src = value;
+	break;
+    case 4:
+	*(uint32_t *)src = value;
+	break;
+    default:
+	assert(0);
+    }
+}
+
+
+int
+main (int argc, char **argv)
+{
+    uint32_t dst[SIZE];
+    pixman_image_t *src_img;
+    pixman_image_t *dst_img;
+    int i, j, x, y;
+    int ret = 0;
+
+    for (i = 0; i < n_test_cases; ++i)
+    {
+	for (j = 0; j < 2; ++j)
+	{
+	    src_img = pixman_image_create_bits (testcases[i].format,
+						testcases[i].width,
+						testcases[i].height,
+						testcases[i].src,
+						testcases[i].stride);
+	    pixman_image_set_indexed(src_img, testcases[i].indexed);
+
+	    dst_img = pixman_image_create_bits (PIXMAN_a8r8g8b8,
+						testcases[i].width,
+						testcases[i].height,
+						dst,
+						testcases[i].width*4);
+
+	    if (j)
+	    {
+		pixman_image_set_accessors (src_img, reader, writer);
+		pixman_image_set_accessors (dst_img, reader, writer);
+	    }
+
+	    pixman_image_composite (PIXMAN_OP_SRC, src_img, NULL, dst_img,
+				    0, 0, 0, 0, 0, 0, testcases[i].width, testcases[i].height);
+
+	    pixman_image_unref (src_img);
+	    pixman_image_unref (dst_img);
+
+	    for (y = 0; y < testcases[i].height; ++y)
+	    {
+		for (x = 0; x < testcases[i].width; ++x)
+		{
+		    int offset = y * testcases[i].width + x;
+
+		    if (dst[offset] != testcases[i].dst[offset])
+		    {
+			printf ("test %i%c: pixel mismatch at (x=%d,y=%d): %08x expected, %08x obtained\n",
+			        i + 1, 'a' + j,
+			        x, y,
+			        testcases[i].dst[offset], dst[offset]);
+			ret = 1;
+		    }
+		}
+	    }
+	}
+    }
+
+    return ret;
+}
diff --git a/pixman/test/gradient-test.c b/pixman/test/gradient-test.c
new file mode 100644
index 000000000..fc84844b0
--- /dev/null
+++ b/pixman/test/gradient-test.c
@@ -0,0 +1,89 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include "pixman.h"
+#include "gtk-utils.h"
+
+int
+main (int argc, char **argv)
+{
+#define WIDTH 400
+#define HEIGHT 200
+    
+    uint32_t *dest = malloc (WIDTH * HEIGHT * 4);
+    pixman_image_t *src_img;
+    pixman_image_t *dest_img;
+    int i;
+    pixman_gradient_stop_t stops[2] =
+	{
+	    { pixman_int_to_fixed (0), { 0xffff, 0xeeee, 0xeeee, 0xeeee } },
+	    { pixman_int_to_fixed (1), { 0xffff, 0x1111, 0x1111, 0x1111 } }
+	};
+    pixman_point_fixed_t p1 = { pixman_double_to_fixed (0), 0 };
+    pixman_point_fixed_t p2 = { pixman_double_to_fixed (WIDTH / 8.),
+				pixman_int_to_fixed (0) };
+#if 0
+    pixman_transform_t trans = {
+	{ { pixman_double_to_fixed (2), pixman_double_to_fixed (0.5), pixman_double_to_fixed (-100), },
+	  { pixman_double_to_fixed (0), pixman_double_to_fixed (3), pixman_double_to_fixed (0), },
+	  { pixman_double_to_fixed (0), pixman_double_to_fixed (0.000), pixman_double_to_fixed (1.0) } 
+	}
+    };
+#else
+    pixman_transform_t trans = {
+	{ { pixman_fixed_1, 0, 0 },
+	  { 0, pixman_fixed_1, 0 },
+	  { 0, 0, pixman_fixed_1 } }
+    };
+#endif
+
+    pixman_point_fixed_t c_inner;
+    pixman_point_fixed_t c_outer;
+    pixman_fixed_t r_inner;
+    pixman_fixed_t r_outer;
+    
+    for (i = 0; i < WIDTH * HEIGHT; ++i)
+	dest[i] = 0x4f00004f; /* pale blue */
+    
+    dest_img = pixman_image_create_bits (PIXMAN_a8r8g8b8,
+					 WIDTH, HEIGHT, 
+					 dest,
+					 WIDTH * 4);
+
+    c_inner.x = pixman_double_to_fixed (50.0);
+    c_inner.y = pixman_double_to_fixed (50.0);
+    c_outer.x = pixman_double_to_fixed (50.0);
+    c_outer.y = pixman_double_to_fixed (50.0);
+    r_inner = 0;
+    r_outer = pixman_double_to_fixed (50.0);
+    
+    src_img = pixman_image_create_conical_gradient (&c_inner, r_inner,
+						    stops, 2);
+#if 0
+    src_img = pixman_image_create_conical_gradient (&c_inner, r_inner,
+						    stops, 2);
+    src_img = pixman_image_create_linear_gradient (&c_inner, &c_outer,
+						   r_inner, r_outer,
+						   stops, 2);
+#endif
+    
+    src_img = pixman_image_create_linear_gradient  (&p1, &p2,
+						    stops, 2);
+    
+    pixman_image_set_transform (src_img, &trans);
+    pixman_image_set_repeat (src_img, PIXMAN_REPEAT_PAD);
+    
+    pixman_image_composite (PIXMAN_OP_OVER, src_img, NULL, dest_img,
+			    0, 0, 0, 0, 0, 0, 10 * WIDTH, HEIGHT);
+    
+    printf ("0, 0: %x\n", dest[0]);
+    printf ("10, 10: %x\n", dest[10 * 10 + 10]);
+    printf ("w, h: %x\n", dest[(HEIGHT - 1) * 100 + (WIDTH - 1)]);
+    
+    show_image (dest_img);
+    
+    pixman_image_unref (src_img);
+    pixman_image_unref (dest_img);
+    free (dest);
+    
+    return 0;
+}
diff --git a/pixman/test/gtk-utils.c b/pixman/test/gtk-utils.c
new file mode 100644
index 000000000..751a164c0
--- /dev/null
+++ b/pixman/test/gtk-utils.c
@@ -0,0 +1,113 @@
+#include <gtk/gtk.h>
+#include <config.h>
+#include "pixman-private.h"	/* For image->bits.format
+				 * FIXME: there should probably be public API for this
+				 */
+#include "gtk-utils.h"
+
+GdkPixbuf *
+pixbuf_from_argb32 (uint32_t *bits,
+		    gboolean has_alpha,
+		    int width,
+		    int height,
+		    int stride)
+{
+    GdkPixbuf *pixbuf = gdk_pixbuf_new (GDK_COLORSPACE_RGB, TRUE,
+					8, width, height);
+    int p_stride = gdk_pixbuf_get_rowstride (pixbuf);
+    guint32 *p_bits = (guint32 *)gdk_pixbuf_get_pixels (pixbuf);
+    int w, h;
+    
+    for (h = 0; h < height; ++h)
+    {
+	for (w = 0; w < width; ++w)
+	{
+	    uint32_t argb = bits[h * (stride / 4) + w];
+	    guint r, g, b, a;
+	    char *pb = (char *)p_bits;
+
+	    pb += h * p_stride + w * 4;
+
+	    r = (argb & 0x00ff0000) >> 16;
+	    g = (argb & 0x0000ff00) >> 8;
+	    b = (argb & 0x000000ff) >> 0;
+	    a = has_alpha? (argb & 0xff000000) >> 24 : 0xff;
+
+	    if (a)
+	    {
+		r = (r * 255) / a;
+		g = (g * 255) / a;
+		b = (b * 255) / a;
+	    }
+
+	    if (r > 255) r = 255;
+	    if (g > 255) g = 255;
+	    if (b > 255) b = 255;
+	    
+	    pb[0] = r;
+	    pb[1] = g;
+	    pb[2] = b;
+	    pb[3] = a;
+	}
+    }
+    
+    return pixbuf;
+}
+
+
+static gboolean
+on_expose (GtkWidget *widget, GdkEventExpose *expose, gpointer data)
+{
+    GdkPixbuf *pixbuf = data;
+    
+    gdk_draw_pixbuf (widget->window, NULL,
+		     pixbuf, 0, 0, 0, 0,
+		     gdk_pixbuf_get_width (pixbuf),
+		     gdk_pixbuf_get_height (pixbuf),
+		     GDK_RGB_DITHER_NONE,
+		     0, 0);
+    
+    return TRUE;
+}
+
+void
+show_image (pixman_image_t *image)
+{
+    GtkWidget *window;
+    GdkPixbuf *pixbuf;
+    int width, height, stride;
+    int argc;
+    char **argv;
+    char *arg0 = g_strdup ("pixman-test-program");
+    gboolean has_alpha;
+    pixman_format_code_t format;
+
+    argc = 1;
+    argv = (char **)&arg0;
+
+    gtk_init (&argc, &argv);
+    
+    window = gtk_window_new (GTK_WINDOW_TOPLEVEL);
+    width = pixman_image_get_width (image);
+    height = pixman_image_get_height (image);
+    stride = pixman_image_get_stride (image);
+
+    format = image->bits.format;
+    
+    if (format == PIXMAN_a8r8g8b8)
+	has_alpha = TRUE;
+    else if (format == PIXMAN_x8r8g8b8)
+	has_alpha = FALSE;
+    else
+	g_error ("Can't deal with this format: %x\n", format);
+    
+    pixbuf = pixbuf_from_argb32 (pixman_image_get_data (image), has_alpha,
+				 width, height, stride);
+    
+    g_signal_connect (window, "expose_event", G_CALLBACK (on_expose), pixbuf);
+    g_signal_connect (window, "delete_event", G_CALLBACK (gtk_main_quit), NULL);
+    
+    gtk_widget_show (window);
+    
+    gtk_main ();
+}
diff --git a/pixman/test/gtk-utils.h b/pixman/test/gtk-utils.h
new file mode 100644
index 000000000..2cb13bcf0
--- /dev/null
+++ b/pixman/test/gtk-utils.h
@@ -0,0 +1,13 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <glib.h>
+#include <gtk/gtk.h>
+#include "pixman.h"
+
+void show_image (pixman_image_t *image);
+
+GdkPixbuf *pixbuf_from_argb32 (uint32_t *bits,
+		               gboolean has_alpha,
+                               int width,
+                               int height,
+                               int stride);
diff --git a/pixman/test/oob-test.c b/pixman/test/oob-test.c
new file mode 100644
index 000000000..4f9e5a244
--- /dev/null
+++ b/pixman/test/oob-test.c
@@ -0,0 +1,101 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include "pixman.h"
+
+typedef struct
+{
+    int				width;
+    int				height;
+    int				stride;
+    pixman_format_code_t	format;
+    
+} image_info_t;
+
+typedef struct
+{
+    pixman_op_t		op;
+    
+    image_info_t	src;
+    image_info_t	dest;
+
+    int			src_x;
+    int			src_y;
+    int			dest_x;
+    int			dest_y;
+    int			width;
+    int			height;
+} composite_info_t;
+
+const composite_info_t info[] =
+{
+    {
+	PIXMAN_OP_SRC,
+	{  3, 6, 16, PIXMAN_a8r8g8b8 },
+	{  5, 7, 20, PIXMAN_x8r8g8b8 },
+	1, 8,
+	1, -1,
+	1, 8
+    },
+    {
+	PIXMAN_OP_SRC,
+	{ 7, 5, 36, PIXMAN_a8r8g8b8 },
+	{ 6, 5, 28, PIXMAN_x8r8g8b8 },
+	8, 5,
+	5, 3,
+	1, 2
+    },
+    {
+	PIXMAN_OP_OVER,
+	{ 10, 10, 40, PIXMAN_a2b10g10r10 },
+	{ 10, 10, 40, PIXMAN_a2b10g10r10 },
+	0, 0,
+	0, 0,
+	10, 10
+    },
+    {
+	PIXMAN_OP_OVER,
+	{ 10, 10, 40, PIXMAN_x2b10g10r10 },
+	{ 10, 10, 40, PIXMAN_x2b10g10r10 },
+	0, 0,
+	0, 0,
+	10, 10
+    },
+};
+
+static pixman_image_t *
+make_image (const image_info_t *info)
+{
+    char *data = malloc (info->stride * info->height);
+    int i;
+
+    for (i = 0; i < info->height * info->stride; ++i)
+	data[i] = (i % 255) ^ (((i % 16) << 4) | (i & 0xf0));
+
+    return pixman_image_create_bits (info->format, info->width, info->height, (uint32_t *)data, info->stride);
+}
+    
+static void
+test_composite (const composite_info_t *info)
+{
+    pixman_image_t *src = make_image (&info->src);
+    pixman_image_t *dest = make_image (&info->dest);
+
+    pixman_image_composite (PIXMAN_OP_SRC, src, NULL, dest,
+			    info->src_x, info->src_y,
+			    0, 0,
+			    info->dest_x, info->dest_y,
+			    info->width, info->height);
+}
+
+
+
+int
+main (int argc, char **argv)
+{
+    int i;
+
+    for (i = 0; i < sizeof (info) / sizeof (info[0]); ++i)
+	test_composite (&info[i]);
+    
+    return 0;
+}
diff --git a/pixman/test/region-test.c b/pixman/test/region-test.c
new file mode 100644
index 000000000..9d5a41eb9
--- /dev/null
+++ b/pixman/test/region-test.c
@@ -0,0 +1,123 @@
+#include <assert.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include "utils.h"
+
+int
+main ()
+{
+    pixman_region32_t r1;
+    pixman_region32_t r2;
+    pixman_region32_t r3;
+    pixman_box32_t boxes[] = {
+	{ 10, 10, 20, 20 },
+	{ 30, 30, 30, 40 },
+	{ 50, 45, 60, 44 },
+    };
+    pixman_box32_t boxes2[] = {
+	{ 2, 6, 7, 6 },
+	{ 4, 1, 6, 7 },
+    };
+    pixman_box32_t boxes3[] = {
+	{ 2, 6, 7, 6 },
+	{ 4, 1, 6, 1 },
+    };
+    int i, j;
+    pixman_box32_t *b;
+    pixman_image_t *image, *fill;
+    pixman_color_t white = {
+	0xffff,
+	0xffff,
+	0xffff,
+	0xffff
+    };
+
+    /* This used to go into an infinite loop before pixman-region.c
+     * was fixed to not use explict "short" variables
+     */
+    pixman_region32_init_rect (&r1, 0, 0, 20, 64000);
+    pixman_region32_init_rect (&r2, 0, 0, 20, 64000);
+    pixman_region32_init_rect (&r3, 0, 0, 20, 64000);
+
+    pixman_region32_subtract (&r1, &r2, &r3);
+
+
+    /* This would produce a region containing an empty
+     * rectangle in it. Such regions are considered malformed,
+     * but using an empty rectangle for initialization should
+     * work.
+     */
+    pixman_region32_init_rects (&r1, boxes, 3);
+
+    b = pixman_region32_rectangles (&r1, &i);
+
+    assert (i == 1);
+    
+    while (i--)
+    {
+	assert (b[i].x1 < b[i].x2);
+	assert (b[i].y1 < b[i].y2);
+    }
+
+    /* This would produce a rectangle containing the bounding box
+     * of the two rectangles. The correct result is to eliminate
+     * the broken rectangle.
+     */
+    pixman_region32_init_rects (&r1, boxes2, 2);
+
+    b = pixman_region32_rectangles (&r1, &i);
+
+    assert (i == 1);
+
+    assert (b[0].x1 == 4);
+    assert (b[0].y1 == 1);
+    assert (b[0].x2 == 6);
+    assert (b[0].y2 == 7);
+
+    /* This should produce an empty region */
+    pixman_region32_init_rects (&r1, boxes3, 2);
+
+    b = pixman_region32_rectangles (&r1, &i);
+
+    assert (i == 0);
+
+    fill = pixman_image_create_solid_fill (&white);
+    for (i = 0; i < 100; i++)
+    {
+	int image_size = 128;
+
+	pixman_region32_init (&r1);
+
+	/* Add some random rectangles */
+	for (j = 0; j < 64; j++)
+	    pixman_region32_union_rect (&r1, &r1,
+					lcg_rand_n (image_size),
+					lcg_rand_n (image_size),
+					lcg_rand_n (25),
+					lcg_rand_n (25));
+
+	/* Clip to image size */
+	pixman_region32_init_rect (&r2, 0, 0, image_size, image_size);
+	pixman_region32_intersect (&r1, &r1, &r2);
+	pixman_region32_fini (&r2);
+
+	/* render region to a1 mask */
+	image = pixman_image_create_bits (PIXMAN_a1, image_size, image_size, NULL, 0);
+	pixman_image_set_clip_region32 (image, &r1);
+	pixman_image_composite32 (PIXMAN_OP_SRC,
+				  fill, NULL, image,
+				  0, 0, 0, 0, 0, 0,
+				  image_size, image_size);
+	pixman_region32_init_from_image (&r2, image);
+
+	pixman_image_unref (image);
+
+	assert (pixman_region32_equal (&r1, &r2));
+	pixman_region32_fini (&r1);
+	pixman_region32_fini (&r2);
+
+    }
+    pixman_image_unref (fill);
+
+    return 0;
+}
diff --git a/pixman/test/scaling-test-bisect.rb b/pixman/test/scaling-test-bisect.rb
new file mode 100644
index 000000000..ab3426244
--- /dev/null
+++ b/pixman/test/scaling-test-bisect.rb
@@ -0,0 +1,38 @@
+#!/usr/bin/env ruby
+
+if not ARGV[0] or not ARGV[1] then
+    printf("Please provide two 'scaling-test' static binaries in the command line.\n\n")
+    printf("The first should be linked with the correct reference pixman library.\n")
+    printf("The second binrary should be linked with the pixman library to be tested.\n")
+    exit(0)
+end
+
+$MAX = 3000000
+$MIN = 1
+$AVG = 0
+
+if `#{ARGV[0]} #{$MAX} 2>/dev/null` == `#{ARGV[1]} #{$MAX} 2>/dev/null` then
+    printf("test ok\n")
+    exit(0)
+end
+
+printf("test failed, bisecting...\n")
+
+while $MAX != $MIN + 1 do
+    $AVG = (($MIN + $MAX) / 2).to_i
+    res1 = `#{ARGV[0]} #{$AVG} 2>/dev/null`
+    res2 = `#{ARGV[1]} #{$AVG} 2>/dev/null`
+    if res1 != res2 then
+        $MAX = $AVG
+    else
+        $MIN = $AVG
+    end
+end
+
+printf("-- ref --\n")
+printf("%s\n", `#{ARGV[0]} -#{$MAX}`)
+printf("-- new --\n")
+printf("%s\n", `#{ARGV[1]} -#{$MAX}`)
+
+printf("\nFailed test number is %d, you can reproduce the problematic conditions\n", $MAX)
+printf("by running 'scaling-test -%d'\n", $MAX)
diff --git a/pixman/test/scaling-test.c b/pixman/test/scaling-test.c
new file mode 100644
index 000000000..29772906d
--- /dev/null
+++ b/pixman/test/scaling-test.c
@@ -0,0 +1,291 @@
+/*
+ * Test program, which can detect problems with nearest neighbout scaling
+ * implementation. Also SRC and OVER opetations tested for 16bpp and 32bpp
+ * images.
+ *
+ * Just run it without any command line arguments, and it will report either
+ *   "scaling test passed" - everything is ok
+ *   "scaling test failed!" - there is some problem
+ *
+ * In the case of failure, finding the problem involves the following steps:
+ * 1. Get the reference 'scaling-test' binary. It makes sense to disable all
+ *    the cpu specific optimizations in pixman and also configure it with
+ *    '--disable-shared' option. Those who are paranoid can also tweak the
+ *    sources to disable all fastpath functions. The resulting binary
+ *    can be renamed to something like 'scaling-test.ref'.
+ * 2. Compile the buggy binary (also with the '--disable-shared' option).
+ * 3. Run 'ruby scaling-test-bisect.rb ./scaling-test.ref ./scaling-test'
+ * 4. Look at the information about failed case (destination buffer content
+ *    will be shown) and try to figure out what is wrong. It is possible
+ *    to use debugging print to stderr in pixman to get more information,
+ *    this does not interfere with the testing script.
+ */
+#include <assert.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include "utils.h"
+
+#define MAX_SRC_WIDTH  10
+#define MAX_SRC_HEIGHT 10
+#define MAX_DST_WIDTH  10
+#define MAX_DST_HEIGHT 10
+#define MAX_STRIDE     4
+
+/*
+ * Composite operation with pseudorandom images
+ */
+uint32_t
+test_composite (uint32_t initcrc,
+		int      testnum,
+		int      verbose)
+{
+    int                i;
+    pixman_image_t *   src_img;
+    pixman_image_t *   dst_img;
+    pixman_transform_t transform;
+    pixman_region16_t  clip;
+    int                src_width, src_height;
+    int                dst_width, dst_height;
+    int                src_stride, dst_stride;
+    int                src_x, src_y;
+    int                dst_x, dst_y;
+    int                src_bpp;
+    int                dst_bpp;
+    int                w, h;
+    int                scale_x = 32768, scale_y = 32768;
+    int                op;
+    int                repeat = 0;
+    int                src_fmt, dst_fmt;
+    uint32_t *         srcbuf;
+    uint32_t *         dstbuf;
+    uint32_t           crc32;
+
+    lcg_srand (testnum);
+
+    src_bpp = (lcg_rand_n (2) == 0) ? 2 : 4;
+    dst_bpp = (lcg_rand_n (2) == 0) ? 2 : 4;
+    op = (lcg_rand_n (2) == 0) ? PIXMAN_OP_SRC : PIXMAN_OP_OVER;
+
+    src_width = lcg_rand_n (MAX_SRC_WIDTH) + 1;
+    src_height = lcg_rand_n (MAX_SRC_HEIGHT) + 1;
+    dst_width = lcg_rand_n (MAX_DST_WIDTH) + 1;
+    dst_height = lcg_rand_n (MAX_DST_HEIGHT) + 1;
+    src_stride = src_width * src_bpp + lcg_rand_n (MAX_STRIDE) * src_bpp;
+    dst_stride = dst_width * dst_bpp + lcg_rand_n (MAX_STRIDE) * dst_bpp;
+
+    if (src_stride & 3)
+	src_stride += 2;
+
+    if (dst_stride & 3)
+	dst_stride += 2;
+
+    src_x = -(src_width / 4) + lcg_rand_n (src_width * 3 / 2);
+    src_y = -(src_height / 4) + lcg_rand_n (src_height * 3 / 2);
+    dst_x = -(dst_width / 4) + lcg_rand_n (dst_width * 3 / 2);
+    dst_y = -(dst_height / 4) + lcg_rand_n (dst_height * 3 / 2);
+    w = lcg_rand_n (dst_width * 3 / 2 - dst_x);
+    h = lcg_rand_n (dst_height * 3 / 2 - dst_y);
+
+    srcbuf = (uint32_t *)malloc (src_stride * src_height);
+    dstbuf = (uint32_t *)malloc (dst_stride * dst_height);
+
+    for (i = 0; i < src_stride * src_height; i++)
+	*((uint8_t *)srcbuf + i) = lcg_rand_n (256);
+
+    for (i = 0; i < dst_stride * dst_height; i++)
+	*((uint8_t *)dstbuf + i) = lcg_rand_n (256);
+
+    src_fmt = src_bpp == 4 ? (lcg_rand_n (2) == 0 ?
+                              PIXMAN_a8r8g8b8 : PIXMAN_x8r8g8b8) : PIXMAN_r5g6b5;
+
+    dst_fmt = dst_bpp == 4 ? (lcg_rand_n (2) == 0 ?
+                              PIXMAN_a8r8g8b8 : PIXMAN_x8r8g8b8) : PIXMAN_r5g6b5;
+
+    src_img = pixman_image_create_bits (
+        src_fmt, src_width, src_height, srcbuf, src_stride);
+
+    dst_img = pixman_image_create_bits (
+        dst_fmt, dst_width, dst_height, dstbuf, dst_stride);
+
+    image_endian_swap (src_img, src_bpp * 8);
+    image_endian_swap (dst_img, dst_bpp * 8);
+
+    if (lcg_rand_n (8) > 0)
+    {
+	scale_x = 32768 + lcg_rand_n (65536);
+	scale_y = 32768 + lcg_rand_n (65536);
+	pixman_transform_init_scale (&transform, scale_x, scale_y);
+	pixman_image_set_transform (src_img, &transform);
+    }
+
+    switch (lcg_rand_n (4))
+    {
+    case 0:
+	repeat = PIXMAN_REPEAT_NONE;
+	break;
+
+    case 1:
+	repeat = PIXMAN_REPEAT_NORMAL;
+	break;
+
+    case 2:
+	repeat = PIXMAN_REPEAT_PAD;
+	break;
+
+    case 3:
+	repeat = PIXMAN_REPEAT_REFLECT;
+	break;
+
+    default:
+        break;
+    }
+    pixman_image_set_repeat (src_img, repeat);
+
+    if (lcg_rand_n (2))
+	pixman_image_set_filter (src_img, PIXMAN_FILTER_NEAREST, NULL, 0);
+    else
+	pixman_image_set_filter (src_img, PIXMAN_FILTER_BILINEAR, NULL, 0);
+
+    if (verbose)
+    {
+	printf ("src_fmt=%08X, dst_fmt=%08X\n", src_fmt, dst_fmt);
+	printf ("op=%d, scale_x=%d, scale_y=%d, repeat=%d\n",
+	        op, scale_x, scale_y, repeat);
+	printf ("src_width=%d, src_height=%d, dst_width=%d, dst_height=%d\n",
+	        src_width, src_height, dst_width, dst_height);
+	printf ("src_x=%d, src_y=%d, dst_x=%d, dst_y=%d\n",
+	        src_x, src_y, dst_x, dst_y);
+	printf ("w=%d, h=%d\n", w, h);
+    }
+
+    if (lcg_rand_n (8) == 0)
+    {
+	pixman_box16_t clip_boxes[2];
+	int            n = lcg_rand_n (2) + 1;
+
+	for (i = 0; i < n; i++)
+	{
+	    clip_boxes[i].x1 = lcg_rand_n (src_width);
+	    clip_boxes[i].y1 = lcg_rand_n (src_height);
+	    clip_boxes[i].x2 =
+		clip_boxes[i].x1 + lcg_rand_n (src_width - clip_boxes[i].x1);
+	    clip_boxes[i].y2 =
+		clip_boxes[i].y1 + lcg_rand_n (src_height - clip_boxes[i].y1);
+
+	    if (verbose)
+	    {
+		printf ("source clip box: [%d,%d-%d,%d]\n",
+		        clip_boxes[i].x1, clip_boxes[i].y1,
+		        clip_boxes[i].x2, clip_boxes[i].y2);
+	    }
+	}
+
+	pixman_region_init_rects (&clip, clip_boxes, n);
+	pixman_image_set_clip_region (src_img, &clip);
+	pixman_image_set_source_clipping (src_img, 1);
+	pixman_region_fini (&clip);
+    }
+
+    if (lcg_rand_n (8) == 0)
+    {
+	pixman_box16_t clip_boxes[2];
+	int            n = lcg_rand_n (2) + 1;
+	for (i = 0; i < n; i++)
+	{
+	    clip_boxes[i].x1 = lcg_rand_n (dst_width);
+	    clip_boxes[i].y1 = lcg_rand_n (dst_height);
+	    clip_boxes[i].x2 =
+		clip_boxes[i].x1 + lcg_rand_n (dst_width - clip_boxes[i].x1);
+	    clip_boxes[i].y2 =
+		clip_boxes[i].y1 + lcg_rand_n (dst_height - clip_boxes[i].y1);
+
+	    if (verbose)
+	    {
+		printf ("destination clip box: [%d,%d-%d,%d]\n",
+		        clip_boxes[i].x1, clip_boxes[i].y1,
+		        clip_boxes[i].x2, clip_boxes[i].y2);
+	    }
+	}
+	pixman_region_init_rects (&clip, clip_boxes, n);
+	pixman_image_set_clip_region (dst_img, &clip);
+	pixman_region_fini (&clip);
+    }
+
+    pixman_image_composite (op, src_img, NULL, dst_img,
+                            src_x, src_y, 0, 0, dst_x, dst_y, w, h);
+
+    if (dst_fmt == PIXMAN_x8r8g8b8)
+    {
+	/* ignore unused part */
+	for (i = 0; i < dst_stride * dst_height / 4; i++)
+	    dstbuf[i] &= 0xFFFFFF;
+    }
+
+    image_endian_swap (dst_img, dst_bpp * 8);
+
+    if (verbose)
+    {
+	int j;
+	
+	for (i = 0; i < dst_height; i++)
+	{
+	    for (j = 0; j < dst_stride; j++)
+		printf ("%02X ", *((uint8_t *)dstbuf + i * dst_stride + j));
+
+	    printf ("\n");
+	}
+    }
+
+    pixman_image_unref (src_img);
+    pixman_image_unref (dst_img);
+
+    crc32 = compute_crc32 (initcrc, dstbuf, dst_stride * dst_height);
+    free (srcbuf);
+    free (dstbuf);
+    return crc32;
+}
+
+int
+main (int   argc, char *argv[])
+{
+    int      i, n = 0;
+    uint32_t crc = 0;
+
+    pixman_disable_out_of_bounds_workaround ();
+
+    if (argc >= 2)
+	n = atoi (argv[1]);
+
+    if (n == 0) n = 3000000;
+
+    if (n < 0)
+    {
+	crc = test_composite (0, -n, 1);
+	printf ("crc32=%08X\n", crc);
+    }
+    else
+    {
+	for (i = 1; i <= n; i++)
+	    crc = test_composite (crc, i, 0);
+
+	printf ("crc32=%08X\n", crc);
+
+	if (n == 3000000)
+	{
+	    /* predefined value for running with all the fastpath functions disabled  */
+	    /* it needs to be updated every time changes are introduced to this program! */
+
+	    if (crc == 0x2168ACD1)
+	    {
+		printf ("scaling test passed\n");
+	    }
+	    else
+	    {
+		printf ("scaling test failed!\n");
+		return 1;
+	    }
+	}
+    }
+
+    return 0;
+}
diff --git a/pixman/test/screen-test.c b/pixman/test/screen-test.c
new file mode 100644
index 000000000..e69dba3de
--- /dev/null
+++ b/pixman/test/screen-test.c
@@ -0,0 +1,44 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include "pixman.h"
+#include "gtk-utils.h"
+
+int
+main (int argc, char **argv)
+{
+#define WIDTH 40
+#define HEIGHT 40
+    
+    uint32_t *src1 = malloc (WIDTH * HEIGHT * 4);
+    uint32_t *src2 = malloc (WIDTH * HEIGHT * 4);
+    uint32_t *src3 = malloc (WIDTH * HEIGHT * 4);
+    uint32_t *dest = malloc (3 * WIDTH * 2 * HEIGHT * 4);
+    pixman_image_t *simg1, *simg2, *simg3, *dimg;
+
+    int i;
+
+    for (i = 0; i < WIDTH * HEIGHT; ++i)
+    {
+	src1[i] = 0x7ff00000;
+	src2[i] = 0x7f00ff00;
+	src3[i] = 0x7f0000ff;
+    }
+
+    for (i = 0; i < 3 * WIDTH * 2 * HEIGHT; ++i)
+    {
+	dest[i] = 0x0;
+    }
+
+    simg1 = pixman_image_create_bits (PIXMAN_a8r8g8b8, WIDTH, HEIGHT, src1, WIDTH * 4);
+    simg2 = pixman_image_create_bits (PIXMAN_a8r8g8b8, WIDTH, HEIGHT, src2, WIDTH * 4);
+    simg3 = pixman_image_create_bits (PIXMAN_a8r8g8b8, WIDTH, HEIGHT, src3, WIDTH * 4);
+    dimg  = pixman_image_create_bits (PIXMAN_a8r8g8b8, 3 * WIDTH, 2 * HEIGHT, dest, 3 * WIDTH * 4);
+
+    pixman_image_composite (PIXMAN_OP_SCREEN, simg1, NULL, dimg, 0, 0, 0, 0, WIDTH, HEIGHT / 4, WIDTH, HEIGHT);
+    pixman_image_composite (PIXMAN_OP_SCREEN, simg2, NULL, dimg, 0, 0, 0, 0, (WIDTH/2), HEIGHT / 4 + HEIGHT / 2, WIDTH, HEIGHT);
+    pixman_image_composite (PIXMAN_OP_SCREEN, simg3, NULL, dimg, 0, 0, 0, 0, (4 * WIDTH) / 3, HEIGHT, WIDTH, HEIGHT);
+
+    show_image (dimg);
+    
+    return 0;
+}
diff --git a/pixman/test/trap-crasher.c b/pixman/test/trap-crasher.c
new file mode 100644
index 000000000..42b82f674
--- /dev/null
+++ b/pixman/test/trap-crasher.c
@@ -0,0 +1,27 @@
+#include <stdlib.h>
+#include <pixman.h>
+
+int
+main()
+{
+    pixman_image_t *dst;
+    pixman_trapezoid_t traps[1] = {
+	{
+	    .top = 2147483646,
+	    .bottom = 2147483647,
+	    .left = {
+		.p1 = { .x = 0, .y = 0 },
+		.p2 = { .x = 0, .y = 2147483647 }
+	    },
+	    .right = {
+		.p1 = { .x = 65536, .y = 0 },
+		.p2 = { .x = 0, .y = 2147483647 }
+	    }
+	},
+    };
+    
+    dst = pixman_image_create_bits (PIXMAN_a8, 1, 1, NULL, -1);
+    
+    pixman_add_trapezoids (dst, 0, 0, sizeof (traps)/sizeof (traps[0]), traps);
+    return (0);
+}
diff --git a/pixman/test/trap-test.c b/pixman/test/trap-test.c
new file mode 100644
index 000000000..19295e7a5
--- /dev/null
+++ b/pixman/test/trap-test.c
@@ -0,0 +1,49 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include "pixman.h"
+#include "gtk-utils.h"
+
+int
+main (int argc, char **argv)
+{
+#define WIDTH 200
+#define HEIGHT 200
+
+    pixman_image_t *src_img;
+    pixman_image_t *mask_img;
+    pixman_image_t *dest_img;
+    pixman_trap_t trap;
+    pixman_color_t white = { 0x0000, 0xffff, 0x0000, 0xffff };
+    uint32_t *bits = malloc (WIDTH * HEIGHT * 4);
+    uint32_t *mbits = malloc (WIDTH * HEIGHT);
+
+    memset (mbits, 0, WIDTH * HEIGHT);
+    memset (bits, 0xff, WIDTH * HEIGHT * 4);
+    
+    trap.top.l = pixman_int_to_fixed (50) + 0x8000;
+    trap.top.r = pixman_int_to_fixed (150) + 0x8000;
+    trap.top.y = pixman_int_to_fixed (30);
+
+    trap.bot.l = pixman_int_to_fixed (50) + 0x8000;
+    trap.bot.r = pixman_int_to_fixed (150) + 0x8000;
+    trap.bot.y = pixman_int_to_fixed (150);
+
+    mask_img = pixman_image_create_bits (PIXMAN_a8, WIDTH, HEIGHT, mbits, WIDTH);
+    src_img = pixman_image_create_solid_fill (&white);
+    dest_img = pixman_image_create_bits (PIXMAN_a8r8g8b8, WIDTH, HEIGHT, bits, WIDTH * 4);
+    
+    pixman_add_traps (mask_img, 0, 0, 1, &trap);
+
+    pixman_image_composite (PIXMAN_OP_OVER,
+			    src_img, mask_img, dest_img,
+			    0, 0, 0, 0, 0, 0, WIDTH, HEIGHT);
+    
+    show_image (dest_img);
+    
+    pixman_image_unref (src_img);
+    pixman_image_unref (dest_img);
+    free (bits);
+    
+    return 0;
+}
diff --git a/pixman/test/utils.c b/pixman/test/utils.c
new file mode 100644
index 000000000..58cd100e2
--- /dev/null
+++ b/pixman/test/utils.c
@@ -0,0 +1,208 @@
+#include "utils.h"
+
+/* Random number seed
+ */
+
+uint32_t lcg_seed;
+
+/*----------------------------------------------------------------------------*\
+ *  CRC-32 version 2.0.0 by Craig Bruce, 2006-04-29.
+ *
+ *  This program generates the CRC-32 values for the files named in the
+ *  command-line arguments.  These are the same CRC-32 values used by GZIP,
+ *  PKZIP, and ZMODEM.  The Crc32_ComputeBuf () can also be detached and
+ *  used independently.
+ *
+ *  THIS PROGRAM IS PUBLIC-DOMAIN SOFTWARE.
+ *
+ *  Based on the byte-oriented implementation "File Verification Using CRC"
+ *  by Mark R. Nelson in Dr. Dobb's Journal, May 1992, pp. 64-67.
+ *
+ *  v1.0.0: original release.
+ *  v1.0.1: fixed printf formats.
+ *  v1.0.2: fixed something else.
+ *  v1.0.3: replaced CRC constant table by generator function.
+ *  v1.0.4: reformatted code, made ANSI C.  1994-12-05.
+ *  v2.0.0: rewrote to use memory buffer & static table, 2006-04-29.
+\*----------------------------------------------------------------------------*/
+
+/*----------------------------------------------------------------------------*\
+ *  NAME:
+ *     Crc32_ComputeBuf () - computes the CRC-32 value of a memory buffer
+ *  DESCRIPTION:
+ *     Computes or accumulates the CRC-32 value for a memory buffer.
+ *     The 'inCrc32' gives a previously accumulated CRC-32 value to allow
+ *     a CRC to be generated for multiple sequential buffer-fuls of data.
+ *     The 'inCrc32' for the first buffer must be zero.
+ *  ARGUMENTS:
+ *     inCrc32 - accumulated CRC-32 value, must be 0 on first call
+ *     buf     - buffer to compute CRC-32 value for
+ *     bufLen  - number of bytes in buffer
+ *  RETURNS:
+ *     crc32 - computed CRC-32 value
+ *  ERRORS:
+ *     (no errors are possible)
+\*----------------------------------------------------------------------------*/
+
+uint32_t
+compute_crc32 (uint32_t    in_crc32,
+	       const void *buf,
+	       size_t      buf_len)
+{
+    static const uint32_t crc_table[256] = {
+	0x00000000, 0x77073096, 0xEE0E612C, 0x990951BA, 0x076DC419, 0x706AF48F,
+	0xE963A535, 0x9E6495A3, 0x0EDB8832, 0x79DCB8A4, 0xE0D5E91E, 0x97D2D988,
+	0x09B64C2B, 0x7EB17CBD,	0xE7B82D07, 0x90BF1D91, 0x1DB71064, 0x6AB020F2,
+	0xF3B97148, 0x84BE41DE, 0x1ADAD47D, 0x6DDDE4EB, 0xF4D4B551, 0x83D385C7,
+	0x136C9856, 0x646BA8C0, 0xFD62F97A, 0x8A65C9EC,	0x14015C4F, 0x63066CD9,
+	0xFA0F3D63, 0x8D080DF5, 0x3B6E20C8, 0x4C69105E, 0xD56041E4, 0xA2677172,
+	0x3C03E4D1, 0x4B04D447, 0xD20D85FD, 0xA50AB56B, 0x35B5A8FA, 0x42B2986C,
+	0xDBBBC9D6, 0xACBCF940, 0x32D86CE3, 0x45DF5C75, 0xDCD60DCF, 0xABD13D59,
+	0x26D930AC, 0x51DE003A, 0xC8D75180, 0xBFD06116, 0x21B4F4B5, 0x56B3C423,
+	0xCFBA9599, 0xB8BDA50F, 0x2802B89E, 0x5F058808, 0xC60CD9B2, 0xB10BE924,
+	0x2F6F7C87, 0x58684C11, 0xC1611DAB, 0xB6662D3D, 0x76DC4190, 0x01DB7106,
+	0x98D220BC, 0xEFD5102A, 0x71B18589, 0x06B6B51F, 0x9FBFE4A5, 0xE8B8D433,
+	0x7807C9A2, 0x0F00F934, 0x9609A88E, 0xE10E9818, 0x7F6A0DBB, 0x086D3D2D,
+	0x91646C97, 0xE6635C01, 0x6B6B51F4, 0x1C6C6162, 0x856530D8, 0xF262004E,
+	0x6C0695ED, 0x1B01A57B, 0x8208F4C1, 0xF50FC457, 0x65B0D9C6, 0x12B7E950,
+	0x8BBEB8EA, 0xFCB9887C, 0x62DD1DDF, 0x15DA2D49, 0x8CD37CF3, 0xFBD44C65,
+	0x4DB26158, 0x3AB551CE, 0xA3BC0074, 0xD4BB30E2, 0x4ADFA541, 0x3DD895D7,
+	0xA4D1C46D, 0xD3D6F4FB, 0x4369E96A, 0x346ED9FC, 0xAD678846, 0xDA60B8D0,
+	0x44042D73, 0x33031DE5, 0xAA0A4C5F, 0xDD0D7CC9, 0x5005713C, 0x270241AA,
+	0xBE0B1010, 0xC90C2086, 0x5768B525, 0x206F85B3, 0xB966D409, 0xCE61E49F,
+	0x5EDEF90E, 0x29D9C998, 0xB0D09822, 0xC7D7A8B4, 0x59B33D17, 0x2EB40D81,
+	0xB7BD5C3B, 0xC0BA6CAD, 0xEDB88320, 0x9ABFB3B6, 0x03B6E20C, 0x74B1D29A,
+	0xEAD54739, 0x9DD277AF, 0x04DB2615, 0x73DC1683, 0xE3630B12, 0x94643B84,
+	0x0D6D6A3E, 0x7A6A5AA8, 0xE40ECF0B, 0x9309FF9D, 0x0A00AE27, 0x7D079EB1,
+	0xF00F9344, 0x8708A3D2, 0x1E01F268, 0x6906C2FE, 0xF762575D, 0x806567CB,
+	0x196C3671, 0x6E6B06E7, 0xFED41B76, 0x89D32BE0, 0x10DA7A5A, 0x67DD4ACC,
+	0xF9B9DF6F, 0x8EBEEFF9, 0x17B7BE43, 0x60B08ED5, 0xD6D6A3E8, 0xA1D1937E,
+	0x38D8C2C4, 0x4FDFF252, 0xD1BB67F1, 0xA6BC5767, 0x3FB506DD, 0x48B2364B,
+	0xD80D2BDA, 0xAF0A1B4C, 0x36034AF6, 0x41047A60, 0xDF60EFC3, 0xA867DF55,
+	0x316E8EEF, 0x4669BE79, 0xCB61B38C, 0xBC66831A, 0x256FD2A0, 0x5268E236,
+	0xCC0C7795, 0xBB0B4703, 0x220216B9, 0x5505262F, 0xC5BA3BBE, 0xB2BD0B28,
+	0x2BB45A92, 0x5CB36A04, 0xC2D7FFA7, 0xB5D0CF31, 0x2CD99E8B, 0x5BDEAE1D,
+	0x9B64C2B0, 0xEC63F226, 0x756AA39C, 0x026D930A, 0x9C0906A9, 0xEB0E363F,
+	0x72076785, 0x05005713, 0x95BF4A82, 0xE2B87A14, 0x7BB12BAE, 0x0CB61B38,
+	0x92D28E9B, 0xE5D5BE0D, 0x7CDCEFB7, 0x0BDBDF21, 0x86D3D2D4, 0xF1D4E242,
+	0x68DDB3F8, 0x1FDA836E, 0x81BE16CD, 0xF6B9265B, 0x6FB077E1, 0x18B74777,
+	0x88085AE6, 0xFF0F6A70, 0x66063BCA, 0x11010B5C, 0x8F659EFF, 0xF862AE69,
+	0x616BFFD3, 0x166CCF45, 0xA00AE278, 0xD70DD2EE, 0x4E048354, 0x3903B3C2,
+	0xA7672661, 0xD06016F7, 0x4969474D, 0x3E6E77DB, 0xAED16A4A, 0xD9D65ADC,
+	0x40DF0B66, 0x37D83BF0, 0xA9BCAE53, 0xDEBB9EC5, 0x47B2CF7F, 0x30B5FFE9,
+	0xBDBDF21C, 0xCABAC28A, 0x53B39330, 0x24B4A3A6, 0xBAD03605, 0xCDD70693,
+	0x54DE5729, 0x23D967BF, 0xB3667A2E, 0xC4614AB8, 0x5D681B02, 0x2A6F2B94,
+	0xB40BBE37, 0xC30C8EA1, 0x5A05DF1B, 0x2D02EF8D
+    };
+
+    uint32_t              crc32;
+    unsigned char *       byte_buf;
+    size_t                i;
+
+    /* accumulate crc32 for buffer */
+    crc32 = in_crc32 ^ 0xFFFFFFFF;
+    byte_buf = (unsigned char*) buf;
+
+    for (i = 0; i < buf_len; i++)
+	crc32 = (crc32 >> 8) ^ crc_table[(crc32 ^ byte_buf[i]) & 0xFF];
+
+    return (crc32 ^ 0xFFFFFFFF);
+}
+
+/* perform endian conversion of pixel data
+ */
+void
+image_endian_swap (pixman_image_t *img, int bpp)
+{
+    int stride = pixman_image_get_stride (img);
+    uint32_t *data = pixman_image_get_data (img);
+    int height = pixman_image_get_height (img);
+    int i, j;
+
+    /* swap bytes only on big endian systems */
+    volatile uint16_t endian_check_var = 0x1234;
+    if (*(volatile uint8_t *)&endian_check_var != 0x12)
+	return;
+
+    for (i = 0; i < height; i++)
+    {
+	uint8_t *line_data = (uint8_t *)data + stride * i;
+	/* swap bytes only for 16, 24 and 32 bpp for now */
+	switch (bpp)
+	{
+	case 1:
+	    for (j = 0; j < stride; j++)
+	    {
+		line_data[j] =
+		    ((line_data[j] & 0x80) >> 7) |
+		    ((line_data[j] & 0x40) >> 5) |
+		    ((line_data[j] & 0x20) >> 3) |
+		    ((line_data[j] & 0x10) >> 1) |
+		    ((line_data[j] & 0x08) << 1) |
+		    ((line_data[j] & 0x04) << 3) |
+		    ((line_data[j] & 0x02) << 5) |
+		    ((line_data[j] & 0x01) << 7);
+	    }
+	    break;
+	case 4:
+	    for (j = 0; j < stride; j++)
+	    {
+		line_data[j] = (line_data[j] >> 4) | (line_data[j] << 4);
+	    }
+	    break;
+	case 16:
+	    for (j = 0; j + 2 <= stride; j += 2)
+	    {
+		char t1 = line_data[j + 0];
+		char t2 = line_data[j + 1];
+
+		line_data[j + 1] = t1;
+		line_data[j + 0] = t2;
+	    }
+	    break;
+	case 24:
+	    for (j = 0; j + 3 <= stride; j += 3)
+	    {
+		char t1 = line_data[j + 0];
+		char t2 = line_data[j + 1];
+		char t3 = line_data[j + 2];
+
+		line_data[j + 2] = t1;
+		line_data[j + 1] = t2;
+		line_data[j + 0] = t3;
+	    }
+	    break;
+	case 32:
+	    for (j = 0; j + 4 <= stride; j += 4)
+	    {
+		char t1 = line_data[j + 0];
+		char t2 = line_data[j + 1];
+		char t3 = line_data[j + 2];
+		char t4 = line_data[j + 3];
+
+		line_data[j + 3] = t1;
+		line_data[j + 2] = t2;
+		line_data[j + 1] = t3;
+		line_data[j + 0] = t4;
+	    }
+	    break;
+	default:
+	    break;
+	}
+    }
+}
+
+uint8_t *
+make_random_bytes (int n_bytes)
+{
+    uint8_t *bytes = malloc (n_bytes);
+    int i;
+
+    if (!bytes)
+	return NULL;
+
+    for (i = 0; i < n_bytes; ++i)
+	bytes[i] = lcg_rand () & 0xff;
+
+    return bytes;
+}
diff --git a/pixman/test/utils.h b/pixman/test/utils.h
new file mode 100644
index 000000000..fb1ccec48
--- /dev/null
+++ b/pixman/test/utils.h
@@ -0,0 +1,45 @@
+#include <stdlib.h>
+#include <config.h>
+#include "pixman-private.h" /* For 'inline' definition */
+
+/* A primitive pseudorandom number generator,
+ * taken from POSIX.1-2001 example
+ */
+
+extern uint32_t lcg_seed;
+
+static inline uint32_t
+lcg_rand (void)
+{
+    lcg_seed = lcg_seed * 1103515245 + 12345;
+    return ((uint32_t)(lcg_seed / 65536) % 32768);
+}
+
+static inline void
+lcg_srand (uint32_t seed)
+{
+    lcg_seed = seed;
+}
+
+static inline uint32_t
+lcg_rand_n (int max)
+{
+    return lcg_rand () % max;
+}
+
+
+/* CRC 32 computation
+ */
+uint32_t
+compute_crc32 (uint32_t    in_crc32,
+	       const void *buf,
+	       size_t      buf_len);
+
+/* perform endian conversion of pixel data
+ */
+void
+image_endian_swap (pixman_image_t *img, int bpp);
+
+/* Generate n_bytes random bytes in malloced memory */
+uint8_t *
+make_random_bytes (int n_bytes);
diff --git a/pixman/test/window-test.c b/pixman/test/window-test.c
new file mode 100644
index 000000000..919fc16ed
--- /dev/null
+++ b/pixman/test/window-test.c
@@ -0,0 +1,173 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <config.h>
+#include "pixman-private.h"
+#include "pixman.h"
+
+#define FALSE 0
+#define TRUE 1
+
+/* Randomly decide between 32 and 16 bit
+ *
+ * Allocate bits with random width, stride and height
+ *
+ * Then make up some random offset (dx, dy)
+ *
+ * Then make an image with those values.
+ *
+ * Do this for both source and destination
+ *
+ * Composite them together using OVER.
+ *
+ * The bits in the source and the destination should have
+ * recognizable colors so that the result can be verified.
+ *
+ * Ie., walk the bits and verify that they have been composited.
+ */
+
+static int
+get_rand (int bound)
+{
+    return rand () % bound;
+}
+
+static pixman_image_t *
+make_image (int width, int height, pixman_bool_t src, int *rx, int *ry)
+{
+    pixman_format_code_t format;
+    pixman_image_t *image;
+    pixman_region32_t region;
+    uint8_t *bits;
+    int stride;
+    int bpp;
+    int dx, dy;
+    int i, j;
+
+    if (src)
+	format = PIXMAN_a8r8g8b8;
+    else
+	format = PIXMAN_r5g6b5;
+
+    bpp = PIXMAN_FORMAT_BPP (format) / 8;
+
+    stride = width + get_rand (width);
+    stride += (stride & 1);		/* Make it an even number */
+
+    bits = malloc (height * stride * bpp);
+
+    for (j = 0; j < height; ++j)
+    {
+	for (i = 0; i < width; ++i)
+	{
+	    uint8_t *pixel = bits + (stride * j + i) * bpp;
+
+	    if (src)
+		*(uint32_t *)pixel = 0x7f00007f;
+	    else
+		*(uint16_t *)pixel = 0xf100;
+	}
+    }
+
+    dx = dy = 0;
+
+    dx = get_rand (500);
+    dy = get_rand (500);
+
+    if (!src)
+    {
+	/* Now simulate the bogus X server translations */
+	bits -= (dy * stride + dx) * bpp;
+    }
+
+    image = pixman_image_create_bits (
+	format, width, height, (uint32_t *)bits, stride * bpp);
+
+    if (!src)
+    {
+	/* And add the bogus clip region */
+	pixman_region32_init_rect (&region, dx, dy, dx + width, dy + height);
+
+	pixman_image_set_clip_region32 (image, &region);
+    }
+
+    pixman_image_set_source_clipping (image, TRUE);
+
+    if (src)
+    {
+	pixman_transform_t trans;
+
+	pixman_transform_init_identity (&trans);
+
+	pixman_transform_translate (&trans,
+				    NULL,
+				    - pixman_int_to_fixed (width / 2),
+				    - pixman_int_to_fixed (height / 2));
+
+	pixman_transform_scale (&trans,
+				NULL,
+				pixman_double_to_fixed (0.5),
+				pixman_double_to_fixed (0.5));
+
+	pixman_transform_translate (&trans,
+				    NULL,
+				    pixman_int_to_fixed (width / 2),
+				    pixman_int_to_fixed (height / 2));
+
+	pixman_image_set_transform (image, &trans);
+	pixman_image_set_filter (image, PIXMAN_FILTER_BILINEAR, NULL, 0);
+	pixman_image_set_repeat (image, PIXMAN_REPEAT_PAD);
+    }
+
+    if (!src)
+    {
+	*rx = dx;
+	*ry = dy;
+    }
+    else
+    {
+	*rx = *ry = 0;
+    }
+
+    return image;
+}
+
+int
+main ()
+{
+    pixman_image_t *src, *dest;
+    int src_x, src_y, dest_x, dest_y;
+    int i, j;
+    int width = get_rand (499) + 1;
+    int height = get_rand (499) + 1;
+
+    src = make_image (width, height, TRUE, &src_x, &src_y);
+    dest = make_image (width, height, FALSE, &dest_x, &dest_y);
+
+    pixman_image_composite (
+	PIXMAN_OP_OVER, src, NULL, dest,
+	src_x, src_y,
+	-1, -1,
+	dest_x, dest_y,
+	width, height);
+
+    for (i = 0; i < height; ++i)
+    {
+	for (j = 0; j < width; ++j)
+	{
+	    uint8_t *bits = (uint8_t *)dest->bits.bits;
+	    int bpp = PIXMAN_FORMAT_BPP (dest->bits.format) / 8;
+	    int stride = dest->bits.rowstride * 4;
+
+	    uint8_t *pixel =
+		bits + (i + dest_y) * stride + (j + dest_x) * bpp;
+
+	    if (*(uint16_t *)pixel != 0x788f)
+	    {
+		printf ("bad pixel %x\n", *(uint16_t *)pixel);
+		assert (*(uint16_t *)pixel == 0x788f);
+	    }
+	}
+    }
+
+    return 0;
+}
diff --git a/posix-aio-compat.c b/posix-aio-compat.c
deleted file mode 100644
index 96e4daf50..000000000
--- a/posix-aio-compat.c
+++ /dev/null
@@ -1,679 +0,0 @@
-/*
- * QEMU posix-aio emulation
- *
- * Copyright IBM, Corp. 2008
- *
- * Authors:
- *  Anthony Liguori   <aliguori@us.ibm.com>
- *
- * This work is licensed under the terms of the GNU GPL, version 2.  See
- * the COPYING file in the top-level directory.
- *
- * Contributions after 2012-01-13 are licensed under the terms of the
- * GNU GPL, version 2 or (at your option) any later version.
- */
-
-#include <sys/ioctl.h>
-#include <sys/types.h>
-#include <pthread.h>
-#include <unistd.h>
-#include <errno.h>
-#include <time.h>
-#include <string.h>
-#include <stdlib.h>
-#include <stdio.h>
-
-#include "qemu-queue.h"
-#include "osdep.h"
-#include "sysemu.h"
-#include "qemu-common.h"
-#include "trace.h"
-#include "block_int.h"
-#include "iov.h"
-
-#include "block/raw-posix-aio.h"
-
-static void do_spawn_thread(void);
-
-struct qemu_paiocb {
-    BlockDriverAIOCB common;
-    int aio_fildes;
-    union {
-        struct iovec *aio_iov;
-        void *aio_ioctl_buf;
-    };
-    int aio_niov;
-    size_t aio_nbytes;
-#define aio_ioctl_cmd   aio_nbytes /* for QEMU_AIO_IOCTL */
-    off_t aio_offset;
-
-    QTAILQ_ENTRY(qemu_paiocb) node;
-    int aio_type;
-    ssize_t ret;
-    int active;
-    struct qemu_paiocb *next;
-};
-
-typedef struct PosixAioState {
-    int rfd, wfd;
-    struct qemu_paiocb *first_aio;
-} PosixAioState;
-
-
-static pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER;
-static pthread_cond_t cond = PTHREAD_COND_INITIALIZER;
-static pthread_t thread_id;
-static pthread_attr_t attr;
-static int max_threads = 64;
-static int cur_threads = 0;
-static int idle_threads = 0;
-static int new_threads = 0;     /* backlog of threads we need to create */
-static int pending_threads = 0; /* threads created but not running yet */
-static QEMUBH *new_thread_bh;
-static QTAILQ_HEAD(, qemu_paiocb) request_list;
-
-#ifdef CONFIG_PREADV
-static int preadv_present = 1;
-#else
-static int preadv_present = 0;
-#endif
-
-static void die2(int err, const char *what)
-{
-    fprintf(stderr, "%s failed: %s\n", what, strerror(err));
-    abort();
-}
-
-static void die(const char *what)
-{
-    die2(errno, what);
-}
-
-static void mutex_lock(pthread_mutex_t *mutex)
-{
-    int ret = pthread_mutex_lock(mutex);
-    if (ret) die2(ret, "pthread_mutex_lock");
-}
-
-static void mutex_unlock(pthread_mutex_t *mutex)
-{
-    int ret = pthread_mutex_unlock(mutex);
-    if (ret) die2(ret, "pthread_mutex_unlock");
-}
-
-static int cond_timedwait(pthread_cond_t *cond, pthread_mutex_t *mutex,
-                           struct timespec *ts)
-{
-    int ret = pthread_cond_timedwait(cond, mutex, ts);
-    if (ret && ret != ETIMEDOUT) die2(ret, "pthread_cond_timedwait");
-    return ret;
-}
-
-static void cond_signal(pthread_cond_t *cond)
-{
-    int ret = pthread_cond_signal(cond);
-    if (ret) die2(ret, "pthread_cond_signal");
-}
-
-static void thread_create(pthread_t *thread, pthread_attr_t *attr,
-                          void *(*start_routine)(void*), void *arg)
-{
-    int ret = pthread_create(thread, attr, start_routine, arg);
-    if (ret) die2(ret, "pthread_create");
-}
-
-static ssize_t handle_aiocb_ioctl(struct qemu_paiocb *aiocb)
-{
-    int ret;
-
-    ret = ioctl(aiocb->aio_fildes, aiocb->aio_ioctl_cmd, aiocb->aio_ioctl_buf);
-    if (ret == -1)
-        return -errno;
-
-    /*
-     * This looks weird, but the aio code only considers a request
-     * successful if it has written the full number of bytes.
-     *
-     * Now we overload aio_nbytes as aio_ioctl_cmd for the ioctl command,
-     * so in fact we return the ioctl command here to make posix_aio_read()
-     * happy..
-     */
-    return aiocb->aio_nbytes;
-}
-
-static ssize_t handle_aiocb_flush(struct qemu_paiocb *aiocb)
-{
-    int ret;
-
-    ret = qemu_fdatasync(aiocb->aio_fildes);
-    if (ret == -1)
-        return -errno;
-    return 0;
-}
-
-#ifdef CONFIG_PREADV
-
-static ssize_t
-qemu_preadv(int fd, const struct iovec *iov, int nr_iov, off_t offset)
-{
-    return preadv(fd, iov, nr_iov, offset);
-}
-
-static ssize_t
-qemu_pwritev(int fd, const struct iovec *iov, int nr_iov, off_t offset)
-{
-    return pwritev(fd, iov, nr_iov, offset);
-}
-
-#else
-
-static ssize_t
-qemu_preadv(int fd, const struct iovec *iov, int nr_iov, off_t offset)
-{
-    return -ENOSYS;
-}
-
-static ssize_t
-qemu_pwritev(int fd, const struct iovec *iov, int nr_iov, off_t offset)
-{
-    return -ENOSYS;
-}
-
-#endif
-
-static ssize_t handle_aiocb_rw_vector(struct qemu_paiocb *aiocb)
-{
-    ssize_t len;
-
-    do {
-        if (aiocb->aio_type & QEMU_AIO_WRITE)
-            len = qemu_pwritev(aiocb->aio_fildes,
-                               aiocb->aio_iov,
-                               aiocb->aio_niov,
-                               aiocb->aio_offset);
-         else
-            len = qemu_preadv(aiocb->aio_fildes,
-                              aiocb->aio_iov,
-                              aiocb->aio_niov,
-                              aiocb->aio_offset);
-    } while (len == -1 && errno == EINTR);
-
-    if (len == -1)
-        return -errno;
-    return len;
-}
-
-/*
- * Read/writes the data to/from a given linear buffer.
- *
- * Returns the number of bytes handles or -errno in case of an error. Short
- * reads are only returned if the end of the file is reached.
- */
-static ssize_t handle_aiocb_rw_linear(struct qemu_paiocb *aiocb, char *buf)
-{
-    ssize_t offset = 0;
-    ssize_t len;
-
-    while (offset < aiocb->aio_nbytes) {
-         if (aiocb->aio_type & QEMU_AIO_WRITE)
-             len = pwrite(aiocb->aio_fildes,
-                          (const char *)buf + offset,
-                          aiocb->aio_nbytes - offset,
-                          aiocb->aio_offset + offset);
-         else
-             len = pread(aiocb->aio_fildes,
-                         buf + offset,
-                         aiocb->aio_nbytes - offset,
-                         aiocb->aio_offset + offset);
-
-         if (len == -1 && errno == EINTR)
-             continue;
-         else if (len == -1) {
-             offset = -errno;
-             break;
-         } else if (len == 0)
-             break;
-
-         offset += len;
-    }
-
-    return offset;
-}
-
-static ssize_t handle_aiocb_rw(struct qemu_paiocb *aiocb)
-{
-    ssize_t nbytes;
-    char *buf;
-
-    if (!(aiocb->aio_type & QEMU_AIO_MISALIGNED)) {
-        /*
-         * If there is just a single buffer, and it is properly aligned
-         * we can just use plain pread/pwrite without any problems.
-         */
-        if (aiocb->aio_niov == 1)
-             return handle_aiocb_rw_linear(aiocb, aiocb->aio_iov->iov_base);
-
-        /*
-         * We have more than one iovec, and all are properly aligned.
-         *
-         * Try preadv/pwritev first and fall back to linearizing the
-         * buffer if it's not supported.
-         */
-        if (preadv_present) {
-            nbytes = handle_aiocb_rw_vector(aiocb);
-            if (nbytes == aiocb->aio_nbytes)
-                return nbytes;
-            if (nbytes < 0 && nbytes != -ENOSYS)
-                return nbytes;
-            preadv_present = 0;
-        }
-
-        /*
-         * XXX(hch): short read/write.  no easy way to handle the reminder
-         * using these interfaces.  For now retry using plain
-         * pread/pwrite?
-         */
-    }
-
-    /*
-     * Ok, we have to do it the hard way, copy all segments into
-     * a single aligned buffer.
-     */
-    buf = qemu_blockalign(aiocb->common.bs, aiocb->aio_nbytes);
-    if (aiocb->aio_type & QEMU_AIO_WRITE) {
-        char *p = buf;
-        int i;
-
-        for (i = 0; i < aiocb->aio_niov; ++i) {
-            memcpy(p, aiocb->aio_iov[i].iov_base, aiocb->aio_iov[i].iov_len);
-            p += aiocb->aio_iov[i].iov_len;
-        }
-    }
-
-    nbytes = handle_aiocb_rw_linear(aiocb, buf);
-    if (!(aiocb->aio_type & QEMU_AIO_WRITE)) {
-        char *p = buf;
-        size_t count = aiocb->aio_nbytes, copy;
-        int i;
-
-        for (i = 0; i < aiocb->aio_niov && count; ++i) {
-            copy = count;
-            if (copy > aiocb->aio_iov[i].iov_len)
-                copy = aiocb->aio_iov[i].iov_len;
-            memcpy(aiocb->aio_iov[i].iov_base, p, copy);
-            p     += copy;
-            count -= copy;
-        }
-    }
-    qemu_vfree(buf);
-
-    return nbytes;
-}
-
-static void posix_aio_notify_event(void);
-
-static void *aio_thread(void *unused)
-{
-    mutex_lock(&lock);
-    pending_threads--;
-    mutex_unlock(&lock);
-    do_spawn_thread();
-
-    while (1) {
-        struct qemu_paiocb *aiocb;
-        ssize_t ret = 0;
-        qemu_timeval tv;
-        struct timespec ts;
-
-        qemu_gettimeofday(&tv);
-        ts.tv_sec = tv.tv_sec + 10;
-        ts.tv_nsec = 0;
-
-        mutex_lock(&lock);
-
-        while (QTAILQ_EMPTY(&request_list) &&
-               !(ret == ETIMEDOUT)) {
-            idle_threads++;
-            ret = cond_timedwait(&cond, &lock, &ts);
-            idle_threads--;
-        }
-
-        if (QTAILQ_EMPTY(&request_list))
-            break;
-
-        aiocb = QTAILQ_FIRST(&request_list);
-        QTAILQ_REMOVE(&request_list, aiocb, node);
-        aiocb->active = 1;
-        mutex_unlock(&lock);
-
-        switch (aiocb->aio_type & QEMU_AIO_TYPE_MASK) {
-        case QEMU_AIO_READ:
-            ret = handle_aiocb_rw(aiocb);
-            if (ret >= 0 && ret < aiocb->aio_nbytes && aiocb->common.bs->growable) {
-                /* A short read means that we have reached EOF. Pad the buffer
-                 * with zeros for bytes after EOF. */
-                iov_memset(aiocb->aio_iov, aiocb->aio_niov, ret,
-                           0, aiocb->aio_nbytes - ret);
-
-                ret = aiocb->aio_nbytes;
-            }
-            break;
-        case QEMU_AIO_WRITE:
-            ret = handle_aiocb_rw(aiocb);
-            break;
-        case QEMU_AIO_FLUSH:
-            ret = handle_aiocb_flush(aiocb);
-            break;
-        case QEMU_AIO_IOCTL:
-            ret = handle_aiocb_ioctl(aiocb);
-            break;
-        default:
-            fprintf(stderr, "invalid aio request (0x%x)\n", aiocb->aio_type);
-            ret = -EINVAL;
-            break;
-        }
-
-        mutex_lock(&lock);
-        aiocb->ret = ret;
-        mutex_unlock(&lock);
-
-        posix_aio_notify_event();
-    }
-
-    cur_threads--;
-    mutex_unlock(&lock);
-
-    return NULL;
-}
-
-static void do_spawn_thread(void)
-{
-    sigset_t set, oldset;
-
-    mutex_lock(&lock);
-    if (!new_threads) {
-        mutex_unlock(&lock);
-        return;
-    }
-
-    new_threads--;
-    pending_threads++;
-
-    mutex_unlock(&lock);
-
-    /* block all signals */
-    if (sigfillset(&set)) die("sigfillset");
-    if (sigprocmask(SIG_SETMASK, &set, &oldset)) die("sigprocmask");
-
-    thread_create(&thread_id, &attr, aio_thread, NULL);
-
-    if (sigprocmask(SIG_SETMASK, &oldset, NULL)) die("sigprocmask restore");
-}
-
-static void spawn_thread_bh_fn(void *opaque)
-{
-    do_spawn_thread();
-}
-
-static void spawn_thread(void)
-{
-    cur_threads++;
-    new_threads++;
-    /* If there are threads being created, they will spawn new workers, so
-     * we don't spend time creating many threads in a loop holding a mutex or
-     * starving the current vcpu.
-     *
-     * If there are no idle threads, ask the main thread to create one, so we
-     * inherit the correct affinity instead of the vcpu affinity.
-     */
-    if (!pending_threads) {
-        qemu_bh_schedule(new_thread_bh);
-    }
-}
-
-static void qemu_paio_submit(struct qemu_paiocb *aiocb)
-{
-    aiocb->ret = -EINPROGRESS;
-    aiocb->active = 0;
-    mutex_lock(&lock);
-    if (idle_threads == 0 && cur_threads < max_threads)
-        spawn_thread();
-    QTAILQ_INSERT_TAIL(&request_list, aiocb, node);
-    mutex_unlock(&lock);
-    cond_signal(&cond);
-}
-
-static ssize_t qemu_paio_return(struct qemu_paiocb *aiocb)
-{
-    ssize_t ret;
-
-    mutex_lock(&lock);
-    ret = aiocb->ret;
-    mutex_unlock(&lock);
-
-    return ret;
-}
-
-static int qemu_paio_error(struct qemu_paiocb *aiocb)
-{
-    ssize_t ret = qemu_paio_return(aiocb);
-
-    if (ret < 0)
-        ret = -ret;
-    else
-        ret = 0;
-
-    return ret;
-}
-
-static void posix_aio_read(void *opaque)
-{
-    PosixAioState *s = opaque;
-    struct qemu_paiocb *acb, **pacb;
-    int ret;
-    ssize_t len;
-
-    /* read all bytes from signal pipe */
-    for (;;) {
-        char bytes[16];
-
-        len = read(s->rfd, bytes, sizeof(bytes));
-        if (len == -1 && errno == EINTR)
-            continue; /* try again */
-        if (len == sizeof(bytes))
-            continue; /* more to read */
-        break;
-    }
-
-    for(;;) {
-        pacb = &s->first_aio;
-        for(;;) {
-            acb = *pacb;
-            if (!acb)
-                return;
-
-            ret = qemu_paio_error(acb);
-            if (ret == ECANCELED) {
-                /* remove the request */
-                *pacb = acb->next;
-                qemu_aio_release(acb);
-            } else if (ret != EINPROGRESS) {
-                /* end of aio */
-                if (ret == 0) {
-                    ret = qemu_paio_return(acb);
-                    if (ret == acb->aio_nbytes)
-                        ret = 0;
-                    else
-                        ret = -EINVAL;
-                } else {
-                    ret = -ret;
-                }
-
-                trace_paio_complete(acb, acb->common.opaque, ret);
-
-                /* remove the request */
-                *pacb = acb->next;
-                /* call the callback */
-                acb->common.cb(acb->common.opaque, ret);
-                qemu_aio_release(acb);
-                break;
-            } else {
-                pacb = &acb->next;
-            }
-        }
-    }
-}
-
-static int posix_aio_flush(void *opaque)
-{
-    PosixAioState *s = opaque;
-    return !!s->first_aio;
-}
-
-static PosixAioState *posix_aio_state;
-
-static void posix_aio_notify_event(void)
-{
-    char byte = 0;
-    ssize_t ret;
-
-    ret = write(posix_aio_state->wfd, &byte, sizeof(byte));
-    if (ret < 0 && errno != EAGAIN)
-        die("write()");
-}
-
-static void paio_remove(struct qemu_paiocb *acb)
-{
-    struct qemu_paiocb **pacb;
-
-    /* remove the callback from the queue */
-    pacb = &posix_aio_state->first_aio;
-    for(;;) {
-        if (*pacb == NULL) {
-            fprintf(stderr, "paio_remove: aio request not found!\n");
-            break;
-        } else if (*pacb == acb) {
-            *pacb = acb->next;
-            qemu_aio_release(acb);
-            break;
-        }
-        pacb = &(*pacb)->next;
-    }
-}
-
-static void paio_cancel(BlockDriverAIOCB *blockacb)
-{
-    struct qemu_paiocb *acb = (struct qemu_paiocb *)blockacb;
-    int active = 0;
-
-    trace_paio_cancel(acb, acb->common.opaque);
-
-    mutex_lock(&lock);
-    if (!acb->active) {
-        QTAILQ_REMOVE(&request_list, acb, node);
-        acb->ret = -ECANCELED;
-    } else if (acb->ret == -EINPROGRESS) {
-        active = 1;
-    }
-    mutex_unlock(&lock);
-
-    if (active) {
-        /* fail safe: if the aio could not be canceled, we wait for
-           it */
-        while (qemu_paio_error(acb) == EINPROGRESS)
-            ;
-    }
-
-    paio_remove(acb);
-}
-
-static AIOPool raw_aio_pool = {
-    .aiocb_size         = sizeof(struct qemu_paiocb),
-    .cancel             = paio_cancel,
-};
-
-BlockDriverAIOCB *paio_submit(BlockDriverState *bs, int fd,
-        int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
-        BlockDriverCompletionFunc *cb, void *opaque, int type)
-{
-    struct qemu_paiocb *acb;
-
-    acb = qemu_aio_get(&raw_aio_pool, bs, cb, opaque);
-    acb->aio_type = type;
-    acb->aio_fildes = fd;
-
-    if (qiov) {
-        acb->aio_iov = qiov->iov;
-        acb->aio_niov = qiov->niov;
-    }
-    acb->aio_nbytes = nb_sectors * 512;
-    acb->aio_offset = sector_num * 512;
-
-    acb->next = posix_aio_state->first_aio;
-    posix_aio_state->first_aio = acb;
-
-    trace_paio_submit(acb, opaque, sector_num, nb_sectors, type);
-    qemu_paio_submit(acb);
-    return &acb->common;
-}
-
-BlockDriverAIOCB *paio_ioctl(BlockDriverState *bs, int fd,
-        unsigned long int req, void *buf,
-        BlockDriverCompletionFunc *cb, void *opaque)
-{
-    struct qemu_paiocb *acb;
-
-    acb = qemu_aio_get(&raw_aio_pool, bs, cb, opaque);
-    acb->aio_type = QEMU_AIO_IOCTL;
-    acb->aio_fildes = fd;
-    acb->aio_offset = 0;
-    acb->aio_ioctl_buf = buf;
-    acb->aio_ioctl_cmd = req;
-
-    acb->next = posix_aio_state->first_aio;
-    posix_aio_state->first_aio = acb;
-
-    qemu_paio_submit(acb);
-    return &acb->common;
-}
-
-int paio_init(void)
-{
-    PosixAioState *s;
-    int fds[2];
-    int ret;
-
-    if (posix_aio_state)
-        return 0;
-
-    s = g_malloc(sizeof(PosixAioState));
-
-    s->first_aio = NULL;
-    if (qemu_pipe(fds) == -1) {
-        fprintf(stderr, "failed to create pipe\n");
-        g_free(s);
-        return -1;
-    }
-
-    s->rfd = fds[0];
-    s->wfd = fds[1];
-
-    fcntl(s->rfd, F_SETFL, O_NONBLOCK);
-    fcntl(s->wfd, F_SETFL, O_NONBLOCK);
-
-    qemu_aio_set_fd_handler(s->rfd, posix_aio_read, NULL, posix_aio_flush, s);
-
-    ret = pthread_attr_init(&attr);
-    if (ret)
-        die2(ret, "pthread_attr_init");
-
-    ret = pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED);
-    if (ret)
-        die2(ret, "pthread_attr_setdetachstate");
-
-    QTAILQ_INIT(&request_list);
-    new_thread_bh = qemu_bh_new(spawn_thread_bh_fn, NULL);
-
-    posix_aio_state = s;
-    return 0;
-}
diff --git a/qapi-schema-guest.json b/qapi-schema-guest.json
index d955cf11f..ed0eb698c 100644
--- a/qapi-schema-guest.json
+++ b/qapi-schema-guest.json
@@ -293,7 +293,7 @@
 ##
 # @GuestFsFreezeStatus
 #
-# An enumation of filesystem freeze states
+# An enumeration of filesystem freeze states
 #
 # @thawed: filesystems thawed/unfrozen
 #
diff --git a/qapi-schema.json b/qapi-schema.json
index bd8ad7449..5dfa05239 100644
--- a/qapi-schema.json
+++ b/qapi-schema.json
@@ -22,15 +22,36 @@
 # @KVMMissingCap: the requested operation can't be fulfilled because a
 #                 required KVM capability is missing
 #
-# @MigrationExpected: the requested operation can't be fulfilled because a
-#                     migration process is expected
-#
 # Since: 1.2
 ##
 { 'enum': 'ErrorClass',
   'data': [ 'GenericError', 'CommandNotFound', 'DeviceEncrypted',
-            'DeviceNotActive', 'DeviceNotFound', 'KVMMissingCap',
-            'MigrationExpected' ] }
+            'DeviceNotActive', 'DeviceNotFound', 'KVMMissingCap' ] }
+
+##
+# @add_client
+#
+# Allow client connections for VNC, Spice and socket based
+# character devices to be passed in to QEMU via SCM_RIGHTS.
+#
+# @protocol: protocol name. Valid names are "vnc", "spice" or the
+#            name of a character device (eg. from -chardev id=XXXX)
+#
+# @fdname: file descriptor name previously passed via 'getfd' command
+#
+# @skipauth: #optional whether to skip authentication. Only applies
+#            to "vnc" and "spice" protocols
+#
+# @tls: #optional whether to perform TLS. Only applies to the "spice"
+#       protocol
+#
+# Returns: nothing on success.
+#
+# Since: 0.14.0
+##
+{ 'command': 'add_client',
+  'data': { 'protocol': 'str', 'fdname': 'str', '*skipauth': 'bool',
+            '*tls': 'bool' } }
 
 ##
 # @NameInfo:
@@ -118,13 +139,17 @@
 ##
 # @RunState
 #
-# An enumation of VM run states.
+# An enumeration of VM run states.
 #
 # @debug: QEMU is running on a debugger
 #
 # @finish-migrate: guest is paused to finish the migration process
 #
-# @inmigrate: guest is paused waiting for an incoming migration
+# @inmigrate: guest is paused waiting for an incoming migration.  Note
+# that this state does not tell whether the machine will start at the
+# end of the migration.  This depends on the command-line -S option and
+# any invocation of 'stop' or 'cont' that has happened since QEMU was
+# started.
 #
 # @internal-error: An internal error that prevents further guest execution
 # has occurred
@@ -156,6 +181,70 @@
             'running', 'save-vm', 'shutdown', 'suspended', 'watchdog' ] }
 
 ##
+# @SnapshotInfo
+#
+# @id: unique snapshot id
+#
+# @name: user chosen name
+#
+# @vm-state-size: size of the VM state
+#
+# @date-sec: UTC date of the snapshot in seconds
+#
+# @date-nsec: fractional part in nano seconds to be used with date-sec
+#
+# @vm-clock-sec: VM clock relative to boot in seconds
+#
+# @vm-clock-nsec: fractional part in nano seconds to be used with vm-clock-sec
+#
+# Since: 1.3
+#
+##
+
+{ 'type': 'SnapshotInfo',
+  'data': { 'id': 'str', 'name': 'str', 'vm-state-size': 'int',
+            'date-sec': 'int', 'date-nsec': 'int',
+            'vm-clock-sec': 'int', 'vm-clock-nsec': 'int' } }
+
+##
+# @ImageInfo:
+#
+# Information about a QEMU image file
+#
+# @filename: name of the image file
+#
+# @format: format of the image file
+#
+# @virtual-size: maximum capacity in bytes of the image
+#
+# @actual-size: #optional actual size on disk in bytes of the image
+#
+# @dirty-flag: #optional true if image is not cleanly closed
+#
+# @cluster-size: #optional size of a cluster in bytes
+#
+# @encrypted: #optional true if the image is encrypted
+#
+# @backing-filename: #optional name of the backing file
+#
+# @full-backing-filename: #optional full path of the backing file
+#
+# @backing-filename-format: #optional the format of the backing file
+#
+# @snapshots: #optional list of VM snapshots
+#
+# Since: 1.3
+#
+##
+
+{ 'type': 'ImageInfo',
+  'data': {'filename': 'str', 'format': 'str', '*dirty-flag': 'bool',
+           '*actual-size': 'int', 'virtual-size': 'int',
+           '*cluster-size': 'int', '*encrypted': 'bool',
+           '*backing-filename': 'str', '*full-backing-filename': 'str',
+           '*backing-filename-format': 'str', '*snapshots': ['SnapshotInfo'] } }
+
+##
 # @StatusInfo:
 #
 # Information about VCPU run state
@@ -294,13 +383,17 @@
 #
 # @normal : number of normal pages (since 1.2)
 #
-# @normal-bytes : number of normal bytes sent (since 1.2)
+# @normal-bytes: number of normal bytes sent (since 1.2)
+#
+# @dirty-pages-rate: number of pages dirtied by second by the
+#        guest (since 1.3)
 #
 # Since: 0.14.0
 ##
 { 'type': 'MigrationStats',
   'data': {'transferred': 'int', 'remaining': 'int', 'total': 'int' ,
-           'duplicate': 'int', 'normal': 'int', 'normal-bytes': 'int' } }
+           'duplicate': 'int', 'normal': 'int', 'normal-bytes': 'int',
+           'dirty-pages-rate' : 'int' } }
 
 ##
 # @XBZRLECacheStats
@@ -349,13 +442,23 @@
 #        If migration has ended, it returns the total migration
 #        time. (since 1.2)
 #
+# @downtime: #optional only present when migration finishes correctly
+#        total downtime in milliseconds for the guest.
+#        (since 1.3)
+#
+# @expected-downtime: #optional only present while migration is active
+#        expected downtime in milliseconds for the guest in last walk
+#        of the dirty bitmap. (since 1.3)
+#
 # Since: 0.14.0
 ##
 { 'type': 'MigrationInfo',
   'data': {'*status': 'str', '*ram': 'MigrationStats',
            '*disk': 'MigrationStats',
            '*xbzrle-cache': 'XBZRLECacheStats',
-           '*total-time': 'int'} }
+           '*total-time': 'int',
+           '*expected-downtime': 'int',
+           '*downtime': 'int'} }
 
 ##
 # @query-migrate
@@ -558,6 +661,18 @@
 { 'enum': 'BlockDeviceIoStatus', 'data': [ 'ok', 'failed', 'nospace' ] }
 
 ##
+# @BlockDirtyInfo:
+#
+# Block dirty bitmap information.
+#
+# @count: number of dirty bytes according to the dirty bitmap
+#
+# Since: 1.3
+##
+{ 'type': 'BlockDirtyInfo',
+  'data': {'count': 'int'} }
+
+##
 # @BlockInfo:
 #
 # Block device information.  This structure describes a virtual device and
@@ -576,6 +691,9 @@
 # @tray_open: #optional True if the device has a tray and it is open
 #             (only present if removable is true)
 #
+# @dirty: #optional dirty bitmap information (only present if the dirty
+#         bitmap is enabled)
+#
 # @io-status: #optional @BlockDeviceIoStatus. Only present if the device
 #             supports it and the VM is configured to stop on errors
 #
@@ -587,7 +705,8 @@
 { 'type': 'BlockInfo',
   'data': {'device': 'str', 'type': 'str', 'removable': 'bool',
            'locked': 'bool', '*inserted': 'BlockDeviceInfo',
-           '*tray_open': 'bool', '*io-status': 'BlockDeviceIoStatus'} }
+           '*tray_open': 'bool', '*io-status': 'BlockDeviceIoStatus',
+           '*dirty': 'BlockDirtyInfo' } }
 
 ##
 # @query-block:
@@ -785,7 +904,7 @@
 ##
 # @SpiceQueryMouseMode
 #
-# An enumation of Spice mouse states.
+# An enumeration of Spice mouse states.
 #
 # @client: Mouse cursor position is determined by the client.
 #
@@ -808,6 +927,9 @@
 #
 # @enabled: true if the SPICE server is enabled, false otherwise
 #
+# @migrated: true if the last guest migration completed and spice
+#            migration had completed as well. false otherwise.
+#
 # @host: #optional The hostname the SPICE server is bound to.  This depends on
 #        the name resolution on the host and may be an IP address.
 #
@@ -833,7 +955,7 @@
 # Since: 0.14.0
 ##
 { 'type': 'SpiceInfo',
-  'data': {'enabled': 'bool', '*host': 'str', '*port': 'int',
+  'data': {'enabled': 'bool', 'migrated': 'bool', '*host': 'str', '*port': 'int',
            '*tls-port': 'int', '*auth': 'str', '*compiled-version': 'str',
            'mouse-mode': 'SpiceQueryMouseMode', '*channels': ['SpiceChannel']} }
 
@@ -1021,6 +1143,46 @@
 { 'command': 'query-pci', 'returns': ['PciInfo'] }
 
 ##
+# @BlockdevOnError:
+#
+# An enumeration of possible behaviors for errors on I/O operations.
+# The exact meaning depends on whether the I/O was initiated by a guest
+# or by a block job
+#
+# @report: for guest operations, report the error to the guest;
+#          for jobs, cancel the job
+#
+# @ignore: ignore the error, only report a QMP event (BLOCK_IO_ERROR
+#          or BLOCK_JOB_ERROR)
+#
+# @enospc: same as @stop on ENOSPC, same as @report otherwise.
+#
+# @stop: for guest operations, stop the virtual machine;
+#        for jobs, pause the job
+#
+# Since: 1.3
+##
+{ 'enum': 'BlockdevOnError',
+  'data': ['report', 'ignore', 'enospc', 'stop'] }
+
+##
+# @MirrorSyncMode:
+#
+# An enumeration of possible behaviors for the initial synchronization
+# phase of storage mirroring.
+#
+# @top: copies data in the topmost image to the destination
+#
+# @full: copies data from all images to the destination
+#
+# @none: only copy data written from now on
+#
+# Since: 1.3
+##
+{ 'enum': 'MirrorSyncMode',
+  'data': ['top', 'full', 'none'] }
+
+##
 # @BlockJobInfo:
 #
 # Information about a long-running block device operation.
@@ -1031,15 +1193,24 @@
 #
 # @len: the maximum progress value
 #
+# @busy: false if the job is known to be in a quiescent state, with
+#        no pending I/O.  Since 1.3.
+#
+# @paused: whether the job is paused or, if @busy is true, will
+#          pause itself as soon as possible.  Since 1.3.
+#
 # @offset: the current progress value
 #
 # @speed: the rate limit, bytes per second
 #
+# @io-status: the status of the job (since 1.3)
+#
 # Since: 1.1
 ##
 { 'type': 'BlockJobInfo',
   'data': {'type': 'str', 'device': 'str', 'len': 'int',
-           'offset': 'int', 'speed': 'int'} }
+           'offset': 'int', 'busy': 'bool', 'paused': 'bool', 'speed': 'int',
+           'io-status': 'BlockDeviceIoStatus'} }
 
 ##
 # @query-block-jobs:
@@ -1072,7 +1243,9 @@
 # Since:  0.14.0
 #
 # Notes:  This function will succeed even if the guest is already in the stopped
-#         state
+#         state.  In "inmigrate" state, it will ensure that the guest
+#         remains paused once migration finishes, as if the -S option was
+#         passed on the command line.
 ##
 { 'command': 'stop' }
 
@@ -1161,11 +1334,14 @@
 # Since:  0.14.0
 #
 # Returns:  If successful, nothing
-#           If the QEMU is waiting for an incoming migration, MigrationExpected
 #           If QEMU was started with an encrypted block device and a key has
 #              not yet been set, DeviceEncrypted.
 #
-# Notes:  This command will succeed if the guest is currently running.
+# Notes:  This command will succeed if the guest is currently running.  It
+#         will also succeed if the guest is in the "inmigrate" state; in
+#         this case, the effect of the command is to make sure the guest
+#         starts once migration finishes, removing the effect of the -S
+#         command line option if it was passed.
 ##
 { 'command': 'cont' }
 
@@ -1307,7 +1483,7 @@
 # @format: #optional the format of the snapshot image, default is 'qcow2'.
 #
 # @mode: #optional whether and how QEMU should create a new image, default is
-# 'absolute-paths'.
+#        'absolute-paths'.
 ##
 { 'type': 'BlockdevSnapshot',
   'data': { 'device': 'str', 'snapshot-file': 'str', '*format': 'str',
@@ -1361,7 +1537,7 @@
 # @format: #optional the format of the snapshot image, default is 'qcow2'.
 #
 # @mode: #optional whether and how QEMU should create a new image, default is
-# 'absolute-paths'.
+#        'absolute-paths'.
 #
 # Returns: nothing on success
 #          If @device is not a valid block device, DeviceNotFound
@@ -1401,6 +1577,83 @@
   'returns': 'str' }
 
 ##
+# @block-commit
+#
+# Live commit of data from overlay image nodes into backing nodes - i.e.,
+# writes data between 'top' and 'base' into 'base'.
+#
+# @device:  the name of the device
+#
+# @base:   #optional The file name of the backing image to write data into.
+#                    If not specified, this is the deepest backing image
+#
+# @top:              The file name of the backing image within the image chain,
+#                    which contains the topmost data to be committed down.
+#                    Note, the active layer as 'top' is currently unsupported.
+#
+#                    If top == base, that is an error.
+#
+#
+# @speed:  #optional the maximum speed, in bytes per second
+#
+# Returns: Nothing on success
+#          If commit or stream is already active on this device, DeviceInUse
+#          If @device does not exist, DeviceNotFound
+#          If image commit is not supported by this device, NotSupported
+#          If @base or @top is invalid, a generic error is returned
+#          If @top is the active layer, or omitted, a generic error is returned
+#          If @speed is invalid, InvalidParameter
+#
+# Since: 1.3
+#
+##
+{ 'command': 'block-commit',
+  'data': { 'device': 'str', '*base': 'str', 'top': 'str',
+            '*speed': 'int' } }
+
+##
+# @drive-mirror
+#
+# Start mirroring a block device's writes to a new destination.
+#
+# @device:  the name of the device whose writes should be mirrored.
+#
+# @target: the target of the new image. If the file exists, or if it
+#          is a device, the existing file/device will be used as the new
+#          destination.  If it does not exist, a new file will be created.
+#
+# @format: #optional the format of the new destination, default is to
+#          probe if @mode is 'existing', else the format of the source
+#
+# @mode: #optional whether and how QEMU should create a new image, default is
+#        'absolute-paths'.
+#
+# @speed:  #optional the maximum speed, in bytes per second
+#
+# @sync: what parts of the disk image should be copied to the destination
+#        (all the disk, only the sectors allocated in the topmost image, or
+#        only new I/O).
+#
+# @on-source-error: #optional the action to take on an error on the source,
+#                   default 'report'.  'stop' and 'enospc' can only be used
+#                   if the block device supports io-status (see BlockInfo).
+#
+# @on-target-error: #optional the action to take on an error on the target,
+#                   default 'report' (no limitations, since this applies to
+#                   a different block device than @device).
+#
+# Returns: nothing on success
+#          If @device is not a valid block device, DeviceNotFound
+#
+# Since 1.3
+##
+{ 'command': 'drive-mirror',
+  'data': { 'device': 'str', 'target': 'str', '*format': 'str',
+            'sync': 'MirrorSyncMode', '*mode': 'NewImageMode',
+            '*speed': 'int', '*on-source-error': 'BlockdevOnError',
+            '*on-target-error': 'BlockdevOnError' } }
+
+##
 # @migrate_cancel
 #
 # Cancel the current executing migration process.
@@ -1736,13 +1989,18 @@
 #
 # @speed:  #optional the maximum speed, in bytes per second
 #
+# @on-error: #optional the action to take on an error (default report).
+#            'stop' and 'enospc' can only be used if the block device
+#            supports io-status (see BlockInfo).  Since 1.3.
+#
 # Returns: Nothing on success
 #          If @device does not exist, DeviceNotFound
 #
 # Since: 1.1
 ##
-{ 'command': 'block-stream', 'data': { 'device': 'str', '*base': 'str',
-                                       '*speed': 'int' } }
+{ 'command': 'block-stream',
+  'data': { 'device': 'str', '*base': 'str', '*speed': 'int',
+            '*on-error': 'BlockdevOnError' } }
 
 ##
 # @block-job-set-speed:
@@ -1786,12 +2044,84 @@
 #
 # @device: the device name
 #
+# @force: #optional whether to allow cancellation of a paused job (default
+#         false).  Since 1.3.
+#
 # Returns: Nothing on success
 #          If no background operation is active on this device, DeviceNotActive
 #
 # Since: 1.1
 ##
-{ 'command': 'block-job-cancel', 'data': { 'device': 'str' } }
+{ 'command': 'block-job-cancel', 'data': { 'device': 'str', '*force': 'bool' } }
+
+##
+# @block-job-pause:
+#
+# Pause an active background block operation.
+#
+# This command returns immediately after marking the active background block
+# operation for pausing.  It is an error to call this command if no
+# operation is in progress.  Pausing an already paused job has no cumulative
+# effect; a single block-job-resume command will resume the job.
+#
+# The operation will pause as soon as possible.  No event is emitted when
+# the operation is actually paused.  Cancelling a paused job automatically
+# resumes it.
+#
+# @device: the device name
+#
+# Returns: Nothing on success
+#          If no background operation is active on this device, DeviceNotActive
+#
+# Since: 1.3
+##
+{ 'command': 'block-job-pause', 'data': { 'device': 'str' } }
+
+##
+# @block-job-resume:
+#
+# Resume an active background block operation.
+#
+# This command returns immediately after resuming a paused background block
+# operation.  It is an error to call this command if no operation is in
+# progress.  Resuming an already running job is not an error.
+#
+# This command also clears the error status of the job.
+#
+# @device: the device name
+#
+# Returns: Nothing on success
+#          If no background operation is active on this device, DeviceNotActive
+#
+# Since: 1.3
+##
+{ 'command': 'block-job-resume', 'data': { 'device': 'str' } }
+
+##
+# @block-job-complete:
+#
+# Manually trigger completion of an active background block operation.  This
+# is supported for drive mirroring, where it also switches the device to
+# write to the target path only.  The ability to complete is signaled with
+# a BLOCK_JOB_READY event.
+#
+# This command completes an active background block operation synchronously.
+# The ordering of this command's return with the BLOCK_JOB_COMPLETED event
+# is not defined.  Note that if an I/O error occurs during the processing of
+# this command: 1) the command itself will fail; 2) the error will be processed
+# according to the rerror/werror arguments that were specified when starting
+# the operation.
+#
+# A cancelled or paused job cannot be completed.
+#
+# @device: the device name
+#
+# Returns: Nothing on success
+#          If no background operation is active on this device, DeviceNotActive
+#
+# Since: 1.3
+##
+{ 'command': 'block-job-complete', 'data': { 'device': 'str' } }
 
 ##
 # @ObjectTypeInfo:
@@ -1889,6 +2219,19 @@
 { 'command': 'xen-save-devices-state', 'data': {'filename': 'str'} }
 
 ##
+# @xen-set-global-dirty-log
+#
+# Enable or disable the global dirty log mode.
+#
+# @enable: true to enable, false to disable.
+#
+# Returns: nothing
+#
+# Since: 1.3
+##
+{ 'command': 'xen-set-global-dirty-log', 'data': { 'enable': 'bool' } }
+
+##
 # @device_del:
 #
 # Remove a device from a guest
@@ -1915,26 +2258,33 @@
 # supported on i386 and x86_64.
 #
 # @paging: if true, do paging to get guest's memory mapping. This allows
-# using gdb to process the core file. However, setting @paging to false
-# may be desirable because of two reasons:
+#          using gdb to process the core file.
 #
-#   1. The guest may be in a catastrophic state or can have corrupted
-#      memory, which cannot be trusted
-#   2. The guest can be in real-mode even if paging is enabled. For example,
-#      the guest uses ACPI to sleep, and ACPI sleep state goes in real-mode
+#          IMPORTANT: this option can make QEMU allocate several gigabytes
+#                     of RAM. This can happen for a large guest, or a
+#                     malicious guest pretending to be large.
+#
+#          Also, paging=true has the following limitations:
+#
+#             1. The guest may be in a catastrophic state or can have corrupted
+#                memory, which cannot be trusted
+#             2. The guest can be in real-mode even if paging is enabled. For
+#                example, the guest uses ACPI to sleep, and ACPI sleep state
+#                goes in real-mode
 #
 # @protocol: the filename or file descriptor of the vmcore. The supported
-# protocols are:
+#            protocols are:
 #
-#   1. file: the protocol starts with "file:", and the following string is
-#      the file's path.
-#   2. fd: the protocol starts with "fd:", and the following string is the
-#      fd's name.
+#            1. file: the protocol starts with "file:", and the following
+#               string is the file's path.
+#            2. fd: the protocol starts with "fd:", and the following string
+#               is the fd's name.
 #
 # @begin: #optional if specified, the starting physical address.
 #
 # @length: #optional if specified, the memory size, in bytes. If you don't
-# want to dump all guest's memory, please specify the start @begin and @length
+#          want to dump all guest's memory, please specify the start @begin
+#          and @length
 #
 # Returns: nothing on success
 #
@@ -1943,6 +2293,7 @@
 { 'command': 'dump-guest-memory',
   'data': { 'paging': 'bool', 'protocol': 'str', '*begin': 'int',
             '*length': 'int' } }
+
 ##
 # @netdev_add:
 #
@@ -2053,6 +2404,9 @@
 #
 # @dns: #optional guest-visible address of the virtual nameserver
 #
+# @dnssearch: #optional list of DNS suffixes to search, passed as DHCP option
+#             to the guest
+#
 # @smb: #optional root directory of the built-in SMB server
 #
 # @smbserver: #optional IP address of the built-in SMB server
@@ -2075,6 +2429,7 @@
     '*bootfile':  'str',
     '*dhcpstart': 'str',
     '*dns':       'str',
+    '*dnssearch': ['String'],
     '*smb':       'str',
     '*smbserver': 'str',
     '*hostfwd':   ['String'],
@@ -2275,6 +2630,59 @@
     'opts': 'NetClientOptions' } }
 
 ##
+# @InetSocketAddress
+#
+# Captures a socket address or address range in the Internet namespace.
+#
+# @host: host part of the address
+#
+# @port: port part of the address, or lowest port if @to is present
+#
+# @to: highest port to try
+#
+# @ipv4: whether to accept IPv4 addresses, default try both IPv4 and IPv6
+#        #optional
+#
+# @ipv6: whether to accept IPv6 addresses, default try both IPv4 and IPv6
+#        #optional
+#
+# Since 1.3
+##
+{ 'type': 'InetSocketAddress',
+  'data': {
+    'host': 'str',
+    'port': 'str',
+    '*to': 'uint16',
+    '*ipv4': 'bool',
+    '*ipv6': 'bool' } }
+
+##
+# @UnixSocketAddress
+#
+# Captures a socket address in the local ("Unix socket") namespace.
+#
+# @path: filesystem path to use
+#
+# Since 1.3
+##
+{ 'type': 'UnixSocketAddress',
+  'data': {
+    'path': 'str' } }
+
+##
+# @SocketAddress
+#
+# Captures the address of a socket, which could also be a named file descriptor
+#
+# Since 1.3
+##
+{ 'union': 'SocketAddress',
+  'data': {
+    'inet': 'InetSocketAddress',
+    'unix': 'UnixSocketAddress',
+    'fd': 'String' } }
+
+##
 # @getfd:
 #
 # Receive a file descriptor via SCM rights and assign it a name
@@ -2381,7 +2789,7 @@
 #
 # Returns: @AddfdInfo on success
 #          If file descriptor was not received, FdNotSupplied
-#          If @fdset-id does not exist, InvalidParameterValue
+#          If @fdset-id is a negative value, InvalidParameterValue
 #
 # Notes: The list of fd sets is shared by all monitor connections.
 #
@@ -2493,3 +2901,119 @@
 # Since: 1.2.0
 ##
 { 'command': 'query-target', 'returns': 'TargetInfo' }
+
+##
+# @QKeyCode:
+#
+# An enumeration of key name.
+#
+# This is used by the send-key command.
+#
+# Since: 1.3.0
+##
+{ 'enum': 'QKeyCode',
+  'data': [ 'shift', 'shift_r', 'alt', 'alt_r', 'altgr', 'altgr_r', 'ctrl',
+            'ctrl_r', 'menu', 'esc', '1', '2', '3', '4', '5', '6', '7', '8',
+            '9', '0', 'minus', 'equal', 'backspace', 'tab', 'q', 'w', 'e',
+            'r', 't', 'y', 'u', 'i', 'o', 'p', 'bracket_left', 'bracket_right',
+            'ret', 'a', 's', 'd', 'f', 'g', 'h', 'j', 'k', 'l', 'semicolon',
+            'apostrophe', 'grave_accent', 'backslash', 'z', 'x', 'c', 'v', 'b',
+            'n', 'm', 'comma', 'dot', 'slash', 'asterisk', 'spc', 'caps_lock',
+            'f1', 'f2', 'f3', 'f4', 'f5', 'f6', 'f7', 'f8', 'f9', 'f10',
+            'num_lock', 'scroll_lock', 'kp_divide', 'kp_multiply',
+            'kp_subtract', 'kp_add', 'kp_enter', 'kp_decimal', 'sysrq', 'kp_0',
+            'kp_1', 'kp_2', 'kp_3', 'kp_4', 'kp_5', 'kp_6', 'kp_7', 'kp_8',
+            'kp_9', 'less', 'f11', 'f12', 'print', 'home', 'pgup', 'pgdn', 'end',
+            'left', 'up', 'down', 'right', 'insert', 'delete', 'stop', 'again',
+            'props', 'undo', 'front', 'copy', 'open', 'paste', 'find', 'cut',
+             'lf', 'help', 'meta_l', 'meta_r', 'compose' ] }
+
+##
+# @KeyValue
+#
+# Represents a keyboard key.
+#
+# Since: 1.3.0
+##
+{ 'union': 'KeyValue',
+  'data': {
+    'number': 'int',
+    'qcode': 'QKeyCode' } }
+
+##
+# @send-key:
+#
+# Send keys to guest.
+#
+# @keys: An array of @KeyValue elements. All @KeyValues in this array are
+#        simultaneously sent to the guest. A @KeyValue.number value is sent
+#        directly to the guest, while @KeyValue.qcode must be a valid
+#        @QKeyCode value
+#
+# @hold-time: #optional time to delay key up events, milliseconds. Defaults
+#             to 100
+#
+# Returns: Nothing on success
+#          If key is unknown or redundant, InvalidParameter
+#
+# Since: 1.3.0
+#
+##
+{ 'command': 'send-key',
+  'data': { 'keys': ['KeyValue'], '*hold-time': 'int' } }
+
+##
+# @screendump:
+#
+# Write a PPM of the VGA screen to a file.
+#
+# @filename: the path of a new PPM file to store the image
+#
+# Returns: Nothing on success
+#
+# Since: 0.14.0
+##
+{ 'command': 'screendump', 'data': {'filename': 'str'} }
+
+##
+# @nbd-server-start:
+#
+# Start an NBD server listening on the given host and port.  Block
+# devices can then be exported using @nbd-server-add.  The NBD
+# server will present them as named exports; for example, another
+# QEMU instance could refer to them as "nbd:HOST:PORT:exportname=NAME".
+#
+# @addr: Address on which to listen.
+#
+# Returns: error if the server is already running.
+#
+# Since: 1.3.0
+##
+{ 'command': 'nbd-server-start',
+  'data': { 'addr': 'SocketAddress' } }
+
+##
+# @nbd-server-add:
+#
+# Export a device to QEMU's embedded NBD server.
+#
+# @device: Block device to be exported
+#
+# @writable: Whether clients should be able to write to the device via the
+#     NBD connection (default false). #optional
+#
+# Returns: error if the device is already marked for export.
+#
+# Since: 1.3.0
+##
+{ 'command': 'nbd-server-add', 'data': {'device': 'str', '*writable': 'bool'} }
+
+##
+# @nbd-server-stop:
+#
+# Stop QEMU's embedded NBD server, and unregister all devices previously
+# added via @nbd-server-add.
+#
+# Since: 1.3.0
+##
+{ 'command': 'nbd-server-stop' }
diff --git a/qapi/Makefile.objs b/qapi/Makefile.objs
index 5f5846e76..f9bd3b991 100644
--- a/qapi/Makefile.objs
+++ b/qapi/Makefile.objs
@@ -1,3 +1,5 @@
 qapi-obj-y = qapi-visit-core.o qapi-dealloc-visitor.o qmp-input-visitor.o
 qapi-obj-y += qmp-output-visitor.o qmp-registry.o qmp-dispatch.o
-qapi-obj-y += string-input-visitor.o string-output-visitor.o opts-visitor.o
+qapi-obj-y += string-input-visitor.o string-output-visitor.o
+
+common-obj-y += opts-visitor.o
diff --git a/qapi/qapi-dealloc-visitor.c b/qapi/qapi-dealloc-visitor.c
index a15452373..75214e7da 100644
--- a/qapi/qapi-dealloc-visitor.c
+++ b/qapi/qapi-dealloc-visitor.c
@@ -132,6 +132,11 @@ static void qapi_dealloc_type_number(Visitor *v, double *obj, const char *name,
 {
 }
 
+static void qapi_dealloc_type_size(Visitor *v, uint64_t *obj, const char *name,
+                                   Error **errp)
+{
+}
+
 static void qapi_dealloc_type_enum(Visitor *v, int *obj, const char *strings[],
                                    const char *kind, const char *name,
                                    Error **errp)
@@ -164,6 +169,7 @@ QapiDeallocVisitor *qapi_dealloc_visitor_new(void)
     v->visitor.type_bool = qapi_dealloc_type_bool;
     v->visitor.type_str = qapi_dealloc_type_str;
     v->visitor.type_number = qapi_dealloc_type_number;
+    v->visitor.type_size = qapi_dealloc_type_size;
 
     QTAILQ_INIT(&v->stack);
 
diff --git a/qemu-aio.h b/qemu-aio.h
index bfdd35f02..3889fe97a 100644
--- a/qemu-aio.h
+++ b/qemu-aio.h
@@ -15,43 +15,184 @@
 #define QEMU_AIO_H
 
 #include "qemu-common.h"
-#include "qemu-char.h"
+#include "qemu-queue.h"
+#include "event_notifier.h"
 
 typedef struct BlockDriverAIOCB BlockDriverAIOCB;
 typedef void BlockDriverCompletionFunc(void *opaque, int ret);
 
-typedef struct AIOPool {
+typedef struct AIOCBInfo {
     void (*cancel)(BlockDriverAIOCB *acb);
-    int aiocb_size;
-    BlockDriverAIOCB *free_aiocb;
-} AIOPool;
+    size_t aiocb_size;
+} AIOCBInfo;
 
 struct BlockDriverAIOCB {
-    AIOPool *pool;
+    const AIOCBInfo *aiocb_info;
     BlockDriverState *bs;
     BlockDriverCompletionFunc *cb;
     void *opaque;
-    BlockDriverAIOCB *next;
 };
 
-void *qemu_aio_get(AIOPool *pool, BlockDriverState *bs,
+void *qemu_aio_get(const AIOCBInfo *aiocb_info, BlockDriverState *bs,
                    BlockDriverCompletionFunc *cb, void *opaque);
 void qemu_aio_release(void *p);
 
+typedef struct AioHandler AioHandler;
+typedef void QEMUBHFunc(void *opaque);
+typedef void IOHandler(void *opaque);
+
+typedef struct AioContext {
+    GSource source;
+
+    /* The list of registered AIO handlers */
+    QLIST_HEAD(, AioHandler) aio_handlers;
+
+    /* This is a simple lock used to protect the aio_handlers list.
+     * Specifically, it's used to ensure that no callbacks are removed while
+     * we're walking and dispatching callbacks.
+     */
+    int walking_handlers;
+
+    /* Anchor of the list of Bottom Halves belonging to the context */
+    struct QEMUBH *first_bh;
+
+    /* A simple lock used to protect the first_bh list, and ensure that
+     * no callbacks are removed while we're walking and dispatching callbacks.
+     */
+    int walking_bh;
+
+    /* Used for aio_notify.  */
+    EventNotifier notifier;
+} AioContext;
+
 /* Returns 1 if there are still outstanding AIO requests; 0 otherwise */
-typedef int (AioFlushHandler)(void *opaque);
+typedef int (AioFlushEventNotifierHandler)(EventNotifier *e);
+
+/**
+ * aio_context_new: Allocate a new AioContext.
+ *
+ * AioContext provide a mini event-loop that can be waited on synchronously.
+ * They also provide bottom halves, a service to execute a piece of code
+ * as soon as possible.
+ */
+AioContext *aio_context_new(void);
+
+/**
+ * aio_context_ref:
+ * @ctx: The AioContext to operate on.
+ *
+ * Add a reference to an AioContext.
+ */
+void aio_context_ref(AioContext *ctx);
+
+/**
+ * aio_context_unref:
+ * @ctx: The AioContext to operate on.
+ *
+ * Drop a reference to an AioContext.
+ */
+void aio_context_unref(AioContext *ctx);
+
+/**
+ * aio_bh_new: Allocate a new bottom half structure.
+ *
+ * Bottom halves are lightweight callbacks whose invocation is guaranteed
+ * to be wait-free, thread-safe and signal-safe.  The #QEMUBH structure
+ * is opaque and must be allocated prior to its use.
+ */
+QEMUBH *aio_bh_new(AioContext *ctx, QEMUBHFunc *cb, void *opaque);
+
+/**
+ * aio_notify: Force processing of pending events.
+ *
+ * Similar to signaling a condition variable, aio_notify forces
+ * aio_wait to exit, so that the next call will re-examine pending events.
+ * The caller of aio_notify will usually call aio_wait again very soon,
+ * or go through another iteration of the GLib main loop.  Hence, aio_notify
+ * also has the side effect of recalculating the sets of file descriptors
+ * that the main loop waits for.
+ *
+ * Calling aio_notify is rarely necessary, because for example scheduling
+ * a bottom half calls it already.
+ */
+void aio_notify(AioContext *ctx);
+
+/**
+ * aio_bh_poll: Poll bottom halves for an AioContext.
+ *
+ * These are internal functions used by the QEMU main loop.
+ */
+int aio_bh_poll(AioContext *ctx);
+
+/**
+ * qemu_bh_schedule: Schedule a bottom half.
+ *
+ * Scheduling a bottom half interrupts the main loop and causes the
+ * execution of the callback that was passed to qemu_bh_new.
+ *
+ * Bottom halves that are scheduled from a bottom half handler are instantly
+ * invoked.  This can create an infinite loop if a bottom half handler
+ * schedules itself.
+ *
+ * @bh: The bottom half to be scheduled.
+ */
+void qemu_bh_schedule(QEMUBH *bh);
+
+/**
+ * qemu_bh_cancel: Cancel execution of a bottom half.
+ *
+ * Canceling execution of a bottom half undoes the effect of calls to
+ * qemu_bh_schedule without freeing its resources yet.  While cancellation
+ * itself is also wait-free and thread-safe, it can of course race with the
+ * loop that executes bottom halves unless you are holding the iothread
+ * mutex.  This makes it mostly useless if you are not holding the mutex.
+ *
+ * @bh: The bottom half to be canceled.
+ */
+void qemu_bh_cancel(QEMUBH *bh);
+
+/**
+ *qemu_bh_delete: Cancel execution of a bottom half and free its resources.
+ *
+ * Deleting a bottom half frees the memory that was allocated for it by
+ * qemu_bh_new.  It also implies canceling the bottom half if it was
+ * scheduled.
+ *
+ * @bh: The bottom half to be deleted.
+ */
+void qemu_bh_delete(QEMUBH *bh);
 
 /* Flush any pending AIO operation. This function will block until all
  * outstanding AIO operations have been completed or cancelled. */
-void qemu_aio_flush(void);
+void aio_flush(AioContext *ctx);
 
-/* Wait for a single AIO completion to occur.  This function will wait
- * until a single AIO event has completed and it will ensure something
- * has moved before returning. This can issue new pending aio as
- * result of executing I/O completion or bh callbacks.
+/* Return whether there are any pending callbacks from the GSource
+ * attached to the AioContext.
  *
- * Return whether there is still any pending AIO operation.  */
-bool qemu_aio_wait(void);
+ * This is used internally in the implementation of the GSource.
+ */
+bool aio_pending(AioContext *ctx);
+
+/* Progress in completing AIO work to occur.  This can issue new pending
+ * aio as a result of executing I/O completion or bh callbacks.
+ *
+ * If there is no pending AIO operation or completion (bottom half),
+ * return false.  If there are pending bottom halves, return true.
+ *
+ * If there are no pending bottom halves, but there are pending AIO
+ * operations, it may not be possible to make any progress without
+ * blocking.  If @blocking is true, this function will wait until one
+ * or more AIO events have completed, to ensure something has moved
+ * before returning.
+ *
+ * If @blocking is false, this function will also return false if the
+ * function cannot make any progress without blocking.
+ */
+bool aio_poll(AioContext *ctx, bool blocking);
+
+#ifdef CONFIG_POSIX
+/* Returns 1 if there are still outstanding AIO requests; 0 otherwise */
+typedef int (AioFlushHandler)(void *opaque);
 
 /* Register a file descriptor and associated callbacks.  Behaves very similarly
  * to qemu_set_fd_handler2.  Unlike qemu_set_fd_handler2, these callbacks will
@@ -60,10 +201,45 @@ bool qemu_aio_wait(void);
  * Code that invokes AIO completion functions should rely on this function
  * instead of qemu_set_fd_handler[2].
  */
-int qemu_aio_set_fd_handler(int fd,
-                            IOHandler *io_read,
-                            IOHandler *io_write,
-                            AioFlushHandler *io_flush,
-                            void *opaque);
+void aio_set_fd_handler(AioContext *ctx,
+                        int fd,
+                        IOHandler *io_read,
+                        IOHandler *io_write,
+                        AioFlushHandler *io_flush,
+                        void *opaque);
+#endif
+
+/* Register an event notifier and associated callbacks.  Behaves very similarly
+ * to event_notifier_set_handler.  Unlike event_notifier_set_handler, these callbacks
+ * will be invoked when using either qemu_aio_wait() or qemu_aio_flush().
+ *
+ * Code that invokes AIO completion functions should rely on this function
+ * instead of event_notifier_set_handler.
+ */
+void aio_set_event_notifier(AioContext *ctx,
+                            EventNotifier *notifier,
+                            EventNotifierHandler *io_read,
+                            AioFlushEventNotifierHandler *io_flush);
+
+/* Return a GSource that lets the main loop poll the file descriptors attached
+ * to this AioContext.
+ */
+GSource *aio_get_g_source(AioContext *ctx);
+
+/* Functions to operate on the main QEMU AioContext.  */
+
+void qemu_aio_flush(void);
+bool qemu_aio_wait(void);
+void qemu_aio_set_event_notifier(EventNotifier *notifier,
+                                 EventNotifierHandler *io_read,
+                                 AioFlushEventNotifierHandler *io_flush);
+
+#ifdef CONFIG_POSIX
+void qemu_aio_set_fd_handler(int fd,
+                             IOHandler *io_read,
+                             IOHandler *io_write,
+                             AioFlushHandler *io_flush,
+                             void *opaque);
+#endif
 
 #endif
diff --git a/qemu-barrier.h b/qemu-barrier.h
index 7e1119781..faa83d265 100644
--- a/qemu-barrier.h
+++ b/qemu-barrier.h
@@ -6,6 +6,8 @@
 
 #if defined(__i386__)
 
+#include "compiler.h"        /* QEMU_GNUC_PREREQ */
+
 /*
  * Because of the strongly ordered x86 storage model, wmb() and rmb() are nops
  * on x86(well, a compiler barrier only).  Well, at least as long as
@@ -19,7 +21,7 @@
  * mfence on 32 bit as well, e.g. if built with -march=pentium-m.
  * However, on i386, there seem to be known bugs as recently as 4.3.
  * */
-#if defined(__GNUC__) && __GNUC__ >= 4 && __GNUC_MINOR__ >= 4
+#if QEMU_GNUC_PREREQ(4, 4)
 #define smp_mb() __sync_synchronize()
 #else
 #define smp_mb() asm volatile("lock; addl $0,0(%%esp) " ::: "memory")
diff --git a/qemu-char.c b/qemu-char.c
index 398baf1e0..242b79990 100644
--- a/qemu-char.c
+++ b/qemu-char.c
@@ -123,19 +123,20 @@ void qemu_chr_be_event(CharDriverState *s, int event)
     s->chr_event(s->handler_opaque, event);
 }
 
-static void qemu_chr_generic_open_bh(void *opaque)
+static void qemu_chr_fire_open_event(void *opaque)
 {
     CharDriverState *s = opaque;
     qemu_chr_be_event(s, CHR_EVENT_OPENED);
-    qemu_bh_delete(s->bh);
-    s->bh = NULL;
+    qemu_free_timer(s->open_timer);
+    s->open_timer = NULL;
 }
 
 void qemu_chr_generic_open(CharDriverState *s)
 {
-    if (s->bh == NULL) {
-	s->bh = qemu_bh_new(qemu_chr_generic_open_bh, s);
-	qemu_bh_schedule(s->bh);
+    if (s->open_timer == NULL) {
+        s->open_timer = qemu_new_timer_ms(rt_clock,
+                                          qemu_chr_fire_open_event, s);
+        qemu_mod_timer(s->open_timer, qemu_get_clock_ms(rt_clock) - 1);
     }
 }
 
@@ -2097,14 +2098,14 @@ static CharDriverState *qemu_chr_open_udp(QemuOpts *opts)
 {
     CharDriverState *chr = NULL;
     NetCharDriver *s = NULL;
+    Error *local_err = NULL;
     int fd = -1;
 
     chr = g_malloc0(sizeof(CharDriverState));
     s = g_malloc0(sizeof(NetCharDriver));
 
-    fd = inet_dgram_opts(opts);
+    fd = inet_dgram_opts(opts, &local_err);
     if (fd < 0) {
-        fprintf(stderr, "inet_dgram_opts failed\n");
         goto return_err;
     }
 
@@ -2118,6 +2119,10 @@ static CharDriverState *qemu_chr_open_udp(QemuOpts *opts)
     return chr;
 
 return_err:
+    if (local_err) {
+        qerror_report_err(local_err);
+        error_free(local_err);
+    }
     g_free(chr);
     g_free(s);
     if (fd >= 0) {
@@ -2141,17 +2146,14 @@ typedef struct {
 
 static void tcp_chr_accept(void *opaque);
 
-static void tcp_chr_connect(void *opaque);
-
 static int tcp_chr_write(CharDriverState *chr, const uint8_t *buf, int len)
 {
     TCPCharDriver *s = chr->opaque;
     if (s->connected) {
         return send_all(s->fd, buf, len);
     } else {
-        /* (Re-)connect for unconnected writing */
-        tcp_chr_connect(chr);
-        return 0;
+        /* XXX: indicate an error ? */
+        return len;
     }
 }
 
@@ -2332,8 +2334,10 @@ static void tcp_chr_connect(void *opaque)
     TCPCharDriver *s = chr->opaque;
 
     s->connected = 1;
-    qemu_set_fd_handler2(s->fd, tcp_chr_read_poll,
-                         tcp_chr_read, NULL, chr);
+    if (s->fd >= 0) {
+        qemu_set_fd_handler2(s->fd, tcp_chr_read_poll,
+                             tcp_chr_read, NULL, chr);
+    }
     qemu_chr_generic_open(chr);
 }
 
@@ -2429,6 +2433,7 @@ static CharDriverState *qemu_chr_open_socket(QemuOpts *opts)
 {
     CharDriverState *chr = NULL;
     TCPCharDriver *s = NULL;
+    Error *local_err = NULL;
     int fd = -1;
     int is_listen;
     int is_waitconnect;
@@ -2449,15 +2454,15 @@ static CharDriverState *qemu_chr_open_socket(QemuOpts *opts)
 
     if (is_unix) {
         if (is_listen) {
-            fd = unix_listen_opts(opts);
+            fd = unix_listen_opts(opts, &local_err);
         } else {
-            fd = unix_connect_opts(opts);
+            fd = unix_connect_opts(opts, &local_err, NULL, NULL);
         }
     } else {
         if (is_listen) {
-            fd = inet_listen_opts(opts, 0, NULL);
+            fd = inet_listen_opts(opts, 0, &local_err);
         } else {
-            fd = inet_connect_opts(opts, NULL, NULL);
+            fd = inet_connect_opts(opts, &local_err, NULL, NULL);
         }
     }
     if (fd < 0) {
@@ -2518,8 +2523,13 @@ static CharDriverState *qemu_chr_open_socket(QemuOpts *opts)
     return chr;
 
  fail:
-    if (fd >= 0)
+    if (local_err) {
+        qerror_report_err(local_err);
+        error_free(local_err);
+    }
+    if (fd >= 0) {
         closesocket(fd);
+    }
     g_free(s);
     g_free(chr);
     return NULL;
diff --git a/qemu-char.h b/qemu-char.h
index 486644b3b..a121e04cd 100644
--- a/qemu-char.h
+++ b/qemu-char.h
@@ -5,6 +5,7 @@
 #include "qemu-queue.h"
 #include "qemu-option.h"
 #include "qemu-config.h"
+#include "qemu-aio.h"
 #include "qobject.h"
 #include "qstring.h"
 #include "main-loop.h"
@@ -69,7 +70,7 @@ struct CharDriverState {
     void (*chr_guest_open)(struct CharDriverState *chr);
     void (*chr_guest_close)(struct CharDriverState *chr);
     void *opaque;
-    QEMUBH *bh;
+    QEMUTimer *open_timer;
     char *label;
     char *filename;
     int opened;
diff --git a/qemu-common.h b/qemu-common.h
index e5c2bcd20..cef264cc8 100644
--- a/qemu-common.h
+++ b/qemu-common.h
@@ -1,5 +1,14 @@
 
-/* Common header file that is included by all of qemu.  */
+/* Common header file that is included by all of QEMU.
+ *
+ * This file is supposed to be included only by .c files. No header file should
+ * depend on qemu-common.h, as this would easily lead to circular header
+ * dependencies.
+ *
+ * If a header file uses a definition from qemu-common.h, that definition
+ * must be moved to a separate header file, and the header that uses it
+ * must include that header.
+ */
 #ifndef QEMU_COMMON_H
 #define QEMU_COMMON_H
 
@@ -14,6 +23,7 @@
 
 typedef struct QEMUTimer QEMUTimer;
 typedef struct QEMUFile QEMUFile;
+typedef struct QEMUBH QEMUBH;
 typedef struct DeviceState DeviceState;
 
 struct Monitor;
@@ -167,7 +177,6 @@ int qemu_fls(int i);
 int qemu_fdatasync(int fd);
 int fcntl_setfl(int fd, int flag);
 int qemu_parse_fd(const char *param);
-int qemu_parse_fdset(const char *param);
 
 /*
  * strtosz() suffixes used to specify the default treatment of an
@@ -208,8 +217,6 @@ const char *path(const char *pathname);
 
 void *qemu_oom_check(void *ptr);
 
-int qemu_open(const char *name, int flags, ...);
-int qemu_close(int fd);
 ssize_t qemu_write_full(int fd, const void *buf, size_t count)
     QEMU_WARN_UNUSED_RESULT;
 ssize_t qemu_send_full(int fd, const void *buf, size_t count, int flags)
@@ -218,14 +225,26 @@ ssize_t qemu_recv_full(int fd, void *buf, size_t count, int flags)
     QEMU_WARN_UNUSED_RESULT;
 
 #ifndef _WIN32
-int qemu_eventfd(int pipefd[2]);
 int qemu_pipe(int pipefd[2]);
 #endif
 
 #ifdef _WIN32
+/* MinGW needs type casts for the 'buf' and 'optval' arguments. */
+#define qemu_getsockopt(sockfd, level, optname, optval, optlen) \
+    getsockopt(sockfd, level, optname, (void *)optval, optlen)
+#define qemu_setsockopt(sockfd, level, optname, optval, optlen) \
+    setsockopt(sockfd, level, optname, (const void *)optval, optlen)
 #define qemu_recv(sockfd, buf, len, flags) recv(sockfd, (void *)buf, len, flags)
+#define qemu_sendto(sockfd, buf, len, flags, destaddr, addrlen) \
+    sendto(sockfd, (const void *)buf, len, flags, destaddr, addrlen)
 #else
+#define qemu_getsockopt(sockfd, level, optname, optval, optlen) \
+    getsockopt(sockfd, level, optname, optval, optlen)
+#define qemu_setsockopt(sockfd, level, optname, optval, optlen) \
+    setsockopt(sockfd, level, optname, optval, optlen)
 #define qemu_recv(sockfd, buf, len, flags) recv(sockfd, buf, len, flags)
+#define qemu_sendto(sockfd, buf, len, flags, destaddr, addrlen) \
+    sendto(sockfd, buf, len, flags, destaddr, addrlen)
 #endif
 
 /* Error handling.  */
@@ -249,10 +268,8 @@ typedef struct DriveInfo DriveInfo;
 typedef struct DisplayState DisplayState;
 typedef struct DisplayChangeListener DisplayChangeListener;
 typedef struct DisplaySurface DisplaySurface;
-typedef struct DisplayAllocator DisplayAllocator;
 typedef struct PixelFormat PixelFormat;
-typedef struct TextConsole TextConsole;
-typedef TextConsole QEMUConsole;
+typedef struct QemuConsole QemuConsole;
 typedef struct CharDriverState CharDriverState;
 typedef struct MACAddr MACAddr;
 typedef struct NetClientState NetClientState;
@@ -273,7 +290,6 @@ typedef struct PCIEPort PCIEPort;
 typedef struct PCIESlot PCIESlot;
 typedef struct MSIMessage MSIMessage;
 typedef struct SerialState SerialState;
-typedef struct IRQState *qemu_irq;
 typedef struct PCMCIACardState PCMCIACardState;
 typedef struct MouseTransformInfo MouseTransformInfo;
 typedef struct uWireSlave uWireSlave;
@@ -311,9 +327,7 @@ void cpu_save(QEMUFile *f, void *opaque);
 int cpu_load(QEMUFile *f, void *opaque, int version_id);
 
 /* Unblock cpu */
-void qemu_cpu_kick(void *env);
 void qemu_cpu_kick_self(void);
-int qemu_cpu_is_self(void *env);
 
 /* work queue */
 struct qemu_work_item {
diff --git a/qemu-config.c b/qemu-config.c
index c05ffbc44..10d1ba417 100644
--- a/qemu-config.c
+++ b/qemu-config.c
@@ -114,6 +114,10 @@ static QemuOptsList qemu_drive_opts = {
             .name = "copy-on-read",
             .type = QEMU_OPT_BOOL,
             .help = "copy read data from backing file into image file",
+        },{
+            .name = "boot",
+            .type = QEMU_OPT_BOOL,
+            .help = "(deprecated, ignored)",
         },
         { /* end of list */ }
     },
@@ -537,6 +541,9 @@ QemuOptsList qemu_spice_opts = {
         },{
             .name = "playback-compression",
             .type = QEMU_OPT_BOOL,
+        }, {
+            .name = "seamless-migration",
+            .type = QEMU_OPT_BOOL,
         },
         { /* end of list */ }
     },
@@ -612,6 +619,14 @@ static QemuOptsList qemu_machine_opts = {
             .name = "dump-guest-core",
             .type = QEMU_OPT_BOOL,
             .help = "Include guest memory in  a core dump",
+        }, {
+            .name = "mem-merge",
+            .type = QEMU_OPT_BOOL,
+            .help = "enable/disable memory merge support",
+        },{
+            .name = "usb",
+            .type = QEMU_OPT_BOOL,
+            .help = "Set on/off to enable/disable usb",
         },
         { /* End of list */ }
     },
@@ -638,11 +653,44 @@ QemuOptsList qemu_boot_opts = {
         }, {
             .name = "splash-time",
             .type = QEMU_OPT_STRING,
+        }, {
+            .name = "reboot-timeout",
+            .type = QEMU_OPT_STRING,
         },
         { /*End of list */ }
     },
 };
 
+static QemuOptsList qemu_add_fd_opts = {
+    .name = "add-fd",
+    .head = QTAILQ_HEAD_INITIALIZER(qemu_add_fd_opts.head),
+    .desc = {
+        {
+            .name = "fd",
+            .type = QEMU_OPT_NUMBER,
+            .help = "file descriptor of which a duplicate is added to fd set",
+        },{
+            .name = "set",
+            .type = QEMU_OPT_NUMBER,
+            .help = "ID of the fd set to add fd to",
+        },{
+            .name = "opaque",
+            .type = QEMU_OPT_STRING,
+            .help = "free-form string used to describe fd",
+        },
+        { /* end of list */ }
+    },
+};
+
+static QemuOptsList qemu_object_opts = {
+    .name = "object",
+    .implied_opt_name = "qom-type",
+    .head = QTAILQ_HEAD_INITIALIZER(qemu_object_opts.head),
+    .desc = {
+        { }
+    },
+};
+
 static QemuOptsList *vm_config_groups[32] = {
     &qemu_drive_opts,
     &qemu_chardev_opts,
@@ -659,6 +707,8 @@ static QemuOptsList *vm_config_groups[32] = {
     &qemu_boot_opts,
     &qemu_iscsi_opts,
     &qemu_sandbox_opts,
+    &qemu_add_fd_opts,
+    &qemu_object_opts,
     NULL,
 };
 
diff --git a/qemu-config.h b/qemu-config.h
index 5557562c3..812c4c5b1 100644
--- a/qemu-config.h
+++ b/qemu-config.h
@@ -1,6 +1,8 @@
 #ifndef QEMU_CONFIG_H
 #define QEMU_CONFIG_H
 
+#include <stdio.h>
+#include "qemu-option.h"
 #include "error.h"
 
 extern QemuOptsList qemu_fsdev_opts;
diff --git a/qemu-coroutine-lock.c b/qemu-coroutine-lock.c
index 26ad76bf5..9dda3f86c 100644
--- a/qemu-coroutine-lock.c
+++ b/qemu-coroutine-lock.c
@@ -26,7 +26,7 @@
 #include "qemu-coroutine.h"
 #include "qemu-coroutine-int.h"
 #include "qemu-queue.h"
-#include "main-loop.h"
+#include "qemu-aio.h"
 #include "trace.h"
 
 static QTAILQ_HEAD(, Coroutine) unlock_bh_queue =
diff --git a/qemu-doc.texi b/qemu-doc.texi
index 35cabbcb9..6d7f50d83 100644
--- a/qemu-doc.texi
+++ b/qemu-doc.texi
@@ -416,11 +416,13 @@ snapshots.
 * vm_snapshots::              VM snapshots
 * qemu_img_invocation::       qemu-img Invocation
 * qemu_nbd_invocation::       qemu-nbd Invocation
+* disk_images_formats::       Disk image file formats
 * host_drives::               Using host drives
 * disk_images_fat_images::    Virtual FAT disk images
 * disk_images_nbd::           NBD access
 * disk_images_sheepdog::      Sheepdog disk images
 * disk_images_iscsi::         iSCSI LUNs
+* disk_images_gluster::       GlusterFS disk images
 @end menu
 
 @node disk_images_quickstart
@@ -506,6 +508,172 @@ state is not saved or restored properly (in particular USB).
 
 @include qemu-nbd.texi
 
+@node disk_images_formats
+@subsection Disk image file formats
+
+QEMU supports many image file formats that can be used with VMs as well as with
+any of the tools (like @code{qemu-img}). This includes the preferred formats
+raw and qcow2 as well as formats that are supported for compatibility with
+older QEMU versions or other hypervisors.
+
+Depending on the image format, different options can be passed to
+@code{qemu-img create} and @code{qemu-img convert} using the @code{-o} option.
+This section describes each format and the options that are supported for it.
+
+@table @option
+@item raw
+
+Raw disk image format. This format has the advantage of
+being simple and easily exportable to all other emulators. If your
+file system supports @emph{holes} (for example in ext2 or ext3 on
+Linux or NTFS on Windows), then only the written sectors will reserve
+space. Use @code{qemu-img info} to know the real size used by the
+image or @code{ls -ls} on Unix/Linux.
+
+@item qcow2
+QEMU image format, the most versatile format. Use it to have smaller
+images (useful if your filesystem does not supports holes, for example
+on Windows), optional AES encryption, zlib based compression and
+support of multiple VM snapshots.
+
+Supported options:
+@table @code
+@item compat
+Determines the qcow2 version to use. @code{compat=0.10} uses the traditional
+image format that can be read by any QEMU since 0.10 (this is the default).
+@code{compat=1.1} enables image format extensions that only QEMU 1.1 and
+newer understand. Amongst others, this includes zero clusters, which allow
+efficient copy-on-read for sparse images.
+
+@item backing_file
+File name of a base image (see @option{create} subcommand)
+@item backing_fmt
+Image format of the base image
+@item encryption
+If this option is set to @code{on}, the image is encrypted.
+
+Encryption uses the AES format which is very secure (128 bit keys). Use
+a long password (16 characters) to get maximum protection.
+
+@item cluster_size
+Changes the qcow2 cluster size (must be between 512 and 2M). Smaller cluster
+sizes can improve the image file size whereas larger cluster sizes generally
+provide better performance.
+
+@item preallocation
+Preallocation mode (allowed values: off, metadata). An image with preallocated
+metadata is initially larger but can improve performance when the image needs
+to grow.
+
+@item lazy_refcounts
+If this option is set to @code{on}, reference count updates are postponed with
+the goal of avoiding metadata I/O and improving performance. This is
+particularly interesting with @option{cache=writethrough} which doesn't batch
+metadata updates. The tradeoff is that after a host crash, the reference count
+tables must be rebuilt, i.e. on the next open an (automatic) @code{qemu-img
+check -r all} is required, which may take some time.
+
+This option can only be enabled if @code{compat=1.1} is specified.
+
+@end table
+
+@item qed
+Old QEMU image format with support for backing files and compact image files
+(when your filesystem or transport medium does not support holes).
+
+When converting QED images to qcow2, you might want to consider using the
+@code{lazy_refcounts=on} option to get a more QED-like behaviour.
+
+Supported options:
+@table @code
+@item backing_file
+File name of a base image (see @option{create} subcommand).
+@item backing_fmt
+Image file format of backing file (optional).  Useful if the format cannot be
+autodetected because it has no header, like some vhd/vpc files.
+@item cluster_size
+Changes the cluster size (must be power-of-2 between 4K and 64K). Smaller
+cluster sizes can improve the image file size whereas larger cluster sizes
+generally provide better performance.
+@item table_size
+Changes the number of clusters per L1/L2 table (must be power-of-2 between 1
+and 16).  There is normally no need to change this value but this option can be
+used for performance benchmarking.
+@end table
+
+@item qcow
+Old QEMU image format with support for backing files, compact image files,
+encryption and compression.
+
+Supported options:
+@table @code
+@item backing_file
+File name of a base image (see @option{create} subcommand)
+@item encryption
+If this option is set to @code{on}, the image is encrypted.
+@end table
+
+@item cow
+User Mode Linux Copy On Write image format. It is supported only for
+compatibility with previous versions.
+Supported options:
+@table @code
+@item backing_file
+File name of a base image (see @option{create} subcommand)
+@end table
+
+@item vdi
+VirtualBox 1.1 compatible image format.
+Supported options:
+@table @code
+@item static
+If this option is set to @code{on}, the image is created with metadata
+preallocation.
+@end table
+
+@item vmdk
+VMware 3 and 4 compatible image format.
+
+Supported options:
+@table @code
+@item backing_file
+File name of a base image (see @option{create} subcommand).
+@item compat6
+Create a VMDK version 6 image (instead of version 4)
+@item subformat
+Specifies which VMDK subformat to use. Valid options are
+@code{monolithicSparse} (default),
+@code{monolithicFlat},
+@code{twoGbMaxExtentSparse},
+@code{twoGbMaxExtentFlat} and
+@code{streamOptimized}.
+@end table
+
+@item vpc
+VirtualPC compatible image format (VHD).
+Supported options:
+@table @code
+@item subformat
+Specifies which VHD subformat to use. Valid options are
+@code{dynamic} (default) and @code{fixed}.
+@end table
+@end table
+
+@subsubsection Read-only formats
+More disk image file formats are supported in a read-only mode.
+@table @option
+@item bochs
+Bochs images of @code{growing} type.
+@item cloop
+Linux Compressed Loop image, useful only to reuse directly compressed
+CD-ROM images present for example in the Knoppix CD-ROMs.
+@item dmg
+Apple disk image.
+@item parallels
+Parallels disk image format.
+@end table
+
+
 @node host_drives
 @subsection Using host drives
 
@@ -610,14 +778,14 @@ QEMU can access directly to block device exported using the Network Block Device
 protocol.
 
 @example
-qemu-system-i386 linux.img -hdb nbd:my_nbd_server.mydomain.org:1024
+qemu-system-i386 linux.img -hdb nbd://my_nbd_server.mydomain.org:1024/
 @end example
 
 If the NBD server is located on the same host, you can use an unix socket instead
 of an inet socket:
 
 @example
-qemu-system-i386 linux.img -hdb nbd:unix:/tmp/my_socket
+qemu-system-i386 linux.img -hdb nbd+unix://?socket=/tmp/my_socket
 @end example
 
 In this case, the block device must be exported using qemu-nbd:
@@ -631,17 +799,26 @@ The use of qemu-nbd allows to share a disk between several guests:
 qemu-nbd --socket=/tmp/my_socket --share=2 my_disk.qcow2
 @end example
 
+@noindent
 and then you can use it with two guests:
 @example
-qemu-system-i386 linux1.img -hdb nbd:unix:/tmp/my_socket
-qemu-system-i386 linux2.img -hdb nbd:unix:/tmp/my_socket
+qemu-system-i386 linux1.img -hdb nbd+unix://?socket=/tmp/my_socket
+qemu-system-i386 linux2.img -hdb nbd+unix://?socket=/tmp/my_socket
 @end example
 
-If the nbd-server uses named exports (since NBD 2.9.18), you must use the
-"exportname" option:
+If the nbd-server uses named exports (supported since NBD 2.9.18, or with QEMU's
+own embedded NBD server), you must specify an export name in the URI:
 @example
-qemu-system-i386 -cdrom nbd:localhost:exportname=debian-500-ppc-netinst
-qemu-system-i386 -cdrom nbd:localhost:exportname=openSUSE-11.1-ppc-netinst
+qemu-system-i386 -cdrom nbd://localhost/debian-500-ppc-netinst
+qemu-system-i386 -cdrom nbd://localhost/openSUSE-11.1-ppc-netinst
+@end example
+
+The URI syntax for NBD is supported since QEMU 1.3.  An alternative syntax is
+also available.  Here are some example of the older syntax:
+@example
+qemu-system-i386 linux.img -hdb nbd:my_nbd_server.mydomain.org:1024
+qemu-system-i386 linux2.img -hdb nbd:unix:/tmp/my_socket
+qemu-system-i386 -cdrom nbd:localhost:10809:exportname=debian-500-ppc-netinst
 @end example
 
 @node disk_images_sheepdog
@@ -805,7 +982,55 @@ qemu-system-i386 -iscsi initiator-name=iqn.qemu.test:my-initiator \
     -cdrom iscsi://127.0.0.1/iqn.qemu.test/2
 @end example
 
+@node disk_images_gluster
+@subsection GlusterFS disk images
+
+GlusterFS is an user space distributed file system.
+
+You can boot from the GlusterFS disk image with the command:
+@example
+qemu-system-x86_64 -drive file=gluster[+@var{transport}]://[@var{server}[:@var{port}]]/@var{volname}/@var{image}[?socket=...]
+@end example
+
+@var{gluster} is the protocol.
+
+@var{transport} specifies the transport type used to connect to gluster
+management daemon (glusterd). Valid transport types are
+tcp, unix and rdma. If a transport type isn't specified, then tcp
+type is assumed.
+
+@var{server} specifies the server where the volume file specification for
+the given volume resides. This can be either hostname, ipv4 address
+or ipv6 address. ipv6 address needs to be within square brackets [ ].
+If transport type is unix, then @var{server} field should not be specifed.
+Instead @var{socket} field needs to be populated with the path to unix domain
+socket.
+
+@var{port} is the port number on which glusterd is listening. This is optional
+and if not specified, QEMU will send 0 which will make gluster to use the
+default port. If the transport type is unix, then @var{port} should not be
+specified.
 
+@var{volname} is the name of the gluster volume which contains the disk image.
+
+@var{image} is the path to the actual disk image that resides on gluster volume.
+
+You can create a GlusterFS disk image with the command:
+@example
+qemu-img create gluster://@var{server}/@var{volname}/@var{image} @var{size}
+@end example
+
+Examples
+@example
+qemu-system-x86_64 -drive file=gluster://1.2.3.4/testvol/a.img
+qemu-system-x86_64 -drive file=gluster+tcp://1.2.3.4/testvol/a.img
+qemu-system-x86_64 -drive file=gluster+tcp://1.2.3.4:24007/testvol/dir/a.img
+qemu-system-x86_64 -drive file=gluster+tcp://[1:2:3:4:5:6:7:8]/testvol/dir/a.img
+qemu-system-x86_64 -drive file=gluster+tcp://[1:2:3:4:5:6:7:8]:24007/testvol/dir/a.img
+qemu-system-x86_64 -drive file=gluster+tcp://server.domain.com:24007/testvol/dir/a.img
+qemu-system-x86_64 -drive file=gluster+unix:///testvol/dir/a.img?socket=/tmp/glusterd.socket
+qemu-system-x86_64 -drive file=gluster+rdma://1.2.3.4:24007/testvol/a.img
+@end example
 
 @node pcsys_network
 @section Network emulation
diff --git a/qemu-file.h b/qemu-file.h
index 31b83f6bb..d64bdbb19 100644
--- a/qemu-file.h
+++ b/qemu-file.h
@@ -47,6 +47,10 @@ typedef int (QEMUFileGetBufferFunc)(void *opaque, uint8_t *buf,
  */
 typedef int (QEMUFileCloseFunc)(void *opaque);
 
+/* Called to return the OS file descriptor associated to the QEMUFile.
+ */
+typedef int (QEMUFileGetFD)(void *opaque);
+
 /* Called to determine if the file has exceeded its bandwidth allocation.  The
  * bandwidth capping is a soft limit, not a hard limit.
  */
@@ -59,19 +63,23 @@ typedef int (QEMUFileRateLimit)(void *opaque);
 typedef int64_t (QEMUFileSetRateLimit)(void *opaque, int64_t new_rate);
 typedef int64_t (QEMUFileGetRateLimit)(void *opaque);
 
-QEMUFile *qemu_fopen_ops(void *opaque, QEMUFilePutBufferFunc *put_buffer,
-                         QEMUFileGetBufferFunc *get_buffer,
-                         QEMUFileCloseFunc *close,
-                         QEMUFileRateLimit *rate_limit,
-                         QEMUFileSetRateLimit *set_rate_limit,
-                         QEMUFileGetRateLimit *get_rate_limit);
+typedef struct QEMUFileOps {
+    QEMUFilePutBufferFunc *put_buffer;
+    QEMUFileGetBufferFunc *get_buffer;
+    QEMUFileCloseFunc *close;
+    QEMUFileGetFD *get_fd;
+    QEMUFileRateLimit *rate_limit;
+    QEMUFileSetRateLimit *set_rate_limit;
+    QEMUFileGetRateLimit *get_rate_limit;
+} QEMUFileOps;
+
+QEMUFile *qemu_fopen_ops(void *opaque, const QEMUFileOps *ops);
 QEMUFile *qemu_fopen(const char *filename, const char *mode);
 QEMUFile *qemu_fdopen(int fd, const char *mode);
 QEMUFile *qemu_fopen_socket(int fd);
 QEMUFile *qemu_popen(FILE *popen_file, const char *mode);
 QEMUFile *qemu_popen_cmd(const char *command, const char *mode);
-int qemu_stdio_fd(QEMUFile *f);
-void qemu_fflush(QEMUFile *f);
+int qemu_get_fd(QEMUFile *f);
 int qemu_fclose(QEMUFile *f);
 void qemu_put_buffer(QEMUFile *f, const uint8_t *buf, int size);
 void qemu_put_byte(QEMUFile *f, int v);
@@ -104,12 +112,11 @@ int qemu_file_rate_limit(QEMUFile *f);
 int64_t qemu_file_set_rate_limit(QEMUFile *f, int64_t new_rate);
 int64_t qemu_file_get_rate_limit(QEMUFile *f);
 int qemu_file_get_error(QEMUFile *f);
-void qemu_file_set_error(QEMUFile *f, int error);
 
 /* Try to send any outstanding data.  This function is useful when output is
  * halted due to rate limiting or EAGAIN errors occur as it can be used to
  * resume output. */
-void qemu_file_put_notify(QEMUFile *f);
+int qemu_file_put_notify(QEMUFile *f);
 
 static inline void qemu_put_be64s(QEMUFile *f, const uint64_t *pv)
 {
@@ -231,8 +238,4 @@ static inline void qemu_get_sbe64s(QEMUFile *f, int64_t *pv)
 {
     qemu_get_be64s(f, (uint64_t *)pv);
 }
-
-int64_t qemu_ftell(QEMUFile *f);
-int64_t qemu_fseek(QEMUFile *f, int64_t pos, int whence);
-
 #endif
diff --git a/qemu-ga.c b/qemu-ga.c
index 762307988..9b59a5246 100644
--- a/qemu-ga.c
+++ b/qemu-ga.c
@@ -40,8 +40,8 @@
 #else
 #define QGA_VIRTIO_PATH_DEFAULT "\\\\.\\Global\\org.qemu.guest_agent.0"
 #endif
-#define QGA_PIDFILE_DEFAULT "/var/run/qemu-ga.pid"
-#define QGA_STATEDIR_DEFAULT "/tmp"
+#define QGA_STATEDIR_DEFAULT CONFIG_QEMU_LOCALSTATEDIR "/run"
+#define QGA_PIDFILE_DEFAULT QGA_STATEDIR_DEFAULT "/qemu-ga.pid"
 #define QGA_SENTINEL_BYTE 0xFF
 
 struct GAState {
@@ -114,12 +114,10 @@ static gboolean register_signal_handlers(void)
     ret = sigaction(SIGINT, &sigact, NULL);
     if (ret == -1) {
         g_error("error configuring signal handler: %s", strerror(errno));
-        return false;
     }
     ret = sigaction(SIGTERM, &sigact, NULL);
     if (ret == -1) {
         g_error("error configuring signal handler: %s", strerror(errno));
-        return false;
     }
 
     return true;
@@ -257,7 +255,7 @@ static bool ga_open_pidfile(const char *pidfile)
         g_critical("Failed to truncate pid file");
         goto fail;
     }
-    sprintf(pidstr, "%d", getpid());
+    snprintf(pidstr, sizeof(pidstr), "%d\n", getpid());
     if (write(pidfd, pidstr, strlen(pidstr)) != strlen(pidstr)) {
         g_critical("Failed to write pid file");
         goto fail;
diff --git a/qemu-img-cmds.hx b/qemu-img-cmds.hx
index 39419a031..a18136302 100644
--- a/qemu-img-cmds.hx
+++ b/qemu-img-cmds.hx
@@ -34,9 +34,9 @@ STEXI
 ETEXI
 
 DEF("info", img_info,
-    "info [-f fmt] filename")
+    "info [-f fmt] [--output=ofmt] [--backing-chain] filename")
 STEXI
-@item info [-f @var{fmt}] @var{filename}
+@item info [-f @var{fmt}] [--output=@var{ofmt}] [--backing-chain] @var{filename}
 ETEXI
 
 DEF("snapshot", img_snapshot,
diff --git a/qemu-img.c b/qemu-img.c
index b41e670a6..e29e01b72 100644
--- a/qemu-img.c
+++ b/qemu-img.c
@@ -21,12 +21,16 @@
  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  * THE SOFTWARE.
  */
+#include "qapi-visit.h"
+#include "qapi/qmp-output-visitor.h"
+#include "qjson.h"
 #include "qemu-common.h"
 #include "qemu-option.h"
 #include "qemu-error.h"
 #include "osdep.h"
 #include "sysemu.h"
 #include "block_int.h"
+#include <getopt.h>
 #include <stdio.h>
 
 #ifdef _WIN32
@@ -84,12 +88,13 @@ static void help(void)
            "  '-p' show progress of command (only certain commands)\n"
            "  '-S' indicates the consecutive number of bytes that must contain only zeros\n"
            "       for qemu-img to create a sparse image during conversion\n"
+           "  '--output' takes the format in which the output must be done (human or json)\n"
            "\n"
            "Parameters to check subcommand:\n"
            "  '-r' tries to repair any inconsistencies that are found during the check.\n"
            "       '-r leaks' repairs only cluster leaks, whereas '-r all' fixes all\n"
            "       kinds of errors, with a higher risk of choosing the wrong fix or\n"
-           "       hiding corruption that has already occured.\n"
+           "       hiding corruption that has already occurred.\n"
            "\n"
            "Parameters to snapshot subcommand:\n"
            "  'snapshot' is the name of the snapshot to create, apply or delete\n"
@@ -221,7 +226,8 @@ static int print_block_option_help(const char *filename, const char *fmt)
 
 static BlockDriverState *bdrv_new_open(const char *filename,
                                        const char *fmt,
-                                       int flags)
+                                       int flags,
+                                       bool require_io)
 {
     BlockDriverState *bs;
     BlockDriver *drv;
@@ -246,7 +252,7 @@ static BlockDriverState *bdrv_new_open(const char *filename,
         goto fail;
     }
 
-    if (bdrv_is_encrypted(bs)) {
+    if (bdrv_is_encrypted(bs) && require_io) {
         printf("Disk image '%s' is encrypted.\n", filename);
         if (read_password(password, sizeof(password)) < 0) {
             error_report("No password given");
@@ -413,7 +419,7 @@ static int img_check(int argc, char **argv)
     }
     filename = argv[optind++];
 
-    bs = bdrv_new_open(filename, fmt, flags);
+    bs = bdrv_new_open(filename, fmt, flags, true);
     if (!bs) {
         return 1;
     }
@@ -520,7 +526,7 @@ static int img_commit(int argc, char **argv)
         return -1;
     }
 
-    bs = bdrv_new_open(filename, fmt, flags);
+    bs = bdrv_new_open(filename, fmt, flags, true);
     if (!bs) {
         return 1;
     }
@@ -668,7 +674,7 @@ static int img_convert(int argc, char **argv)
     QEMUOptionParameter *out_baseimg_param;
     char *options = NULL;
     const char *snapshot_name = NULL;
-    float local_progress;
+    float local_progress = 0;
     int min_sparse = 8; /* Need at least 4k of zeros for sparse detection */
 
     fmt = NULL;
@@ -762,7 +768,7 @@ static int img_convert(int argc, char **argv)
 
     total_sectors = 0;
     for (bs_i = 0; bs_i < bs_n; bs_i++) {
-        bs[bs_i] = bdrv_new_open(argv[optind + bs_i], fmt, BDRV_O_FLAGS);
+        bs[bs_i] = bdrv_new_open(argv[optind + bs_i], fmt, BDRV_O_FLAGS, true);
         if (!bs[bs_i]) {
             error_report("Could not open '%s'", argv[optind + bs_i]);
             ret = -1;
@@ -881,7 +887,7 @@ static int img_convert(int argc, char **argv)
         return -1;
     }
 
-    out_bs = bdrv_new_open(out_filename, out_fmt, flags);
+    out_bs = bdrv_new_open(out_filename, out_fmt, flags, true);
     if (!out_bs) {
         ret = -1;
         goto out;
@@ -908,8 +914,10 @@ static int img_convert(int argc, char **argv)
         sector_num = 0;
 
         nb_sectors = total_sectors;
-        local_progress = (float)100 /
-            (nb_sectors / MIN(nb_sectors, cluster_sectors));
+        if (nb_sectors != 0) {
+            local_progress = (float)100 /
+                (nb_sectors / MIN(nb_sectors, cluster_sectors));
+        }
 
         for(;;) {
             int64_t bs_num;
@@ -980,8 +988,10 @@ static int img_convert(int argc, char **argv)
 
         sector_num = 0; // total number of sectors converted so far
         nb_sectors = total_sectors - sector_num;
-        local_progress = (float)100 /
-            (nb_sectors / MIN(nb_sectors, IO_BUF_SIZE / 512));
+        if (nb_sectors != 0) {
+            local_progress = (float)100 /
+                (nb_sectors / MIN(nb_sectors, IO_BUF_SIZE / 512));
+        }
 
         for(;;) {
             nb_sectors = total_sectors - sector_num;
@@ -1102,21 +1112,312 @@ static void dump_snapshots(BlockDriverState *bs)
     g_free(sn_tab);
 }
 
-static int img_info(int argc, char **argv)
+static void dump_json_image_info_list(ImageInfoList *list)
+{
+    Error *errp = NULL;
+    QString *str;
+    QmpOutputVisitor *ov = qmp_output_visitor_new();
+    QObject *obj;
+    visit_type_ImageInfoList(qmp_output_get_visitor(ov),
+                             &list, NULL, &errp);
+    obj = qmp_output_get_qobject(ov);
+    str = qobject_to_json_pretty(obj);
+    assert(str != NULL);
+    printf("%s\n", qstring_get_str(str));
+    qobject_decref(obj);
+    qmp_output_visitor_cleanup(ov);
+    QDECREF(str);
+}
+
+static void collect_snapshots(BlockDriverState *bs , ImageInfo *info)
+{
+    int i, sn_count;
+    QEMUSnapshotInfo *sn_tab = NULL;
+    SnapshotInfoList *info_list, *cur_item = NULL;
+    sn_count = bdrv_snapshot_list(bs, &sn_tab);
+
+    for (i = 0; i < sn_count; i++) {
+        info->has_snapshots = true;
+        info_list = g_new0(SnapshotInfoList, 1);
+
+        info_list->value                = g_new0(SnapshotInfo, 1);
+        info_list->value->id            = g_strdup(sn_tab[i].id_str);
+        info_list->value->name          = g_strdup(sn_tab[i].name);
+        info_list->value->vm_state_size = sn_tab[i].vm_state_size;
+        info_list->value->date_sec      = sn_tab[i].date_sec;
+        info_list->value->date_nsec     = sn_tab[i].date_nsec;
+        info_list->value->vm_clock_sec  = sn_tab[i].vm_clock_nsec / 1000000000;
+        info_list->value->vm_clock_nsec = sn_tab[i].vm_clock_nsec % 1000000000;
+
+        /* XXX: waiting for the qapi to support qemu-queue.h types */
+        if (!cur_item) {
+            info->snapshots = cur_item = info_list;
+        } else {
+            cur_item->next = info_list;
+            cur_item = info_list;
+        }
+
+    }
+
+    g_free(sn_tab);
+}
+
+static void dump_json_image_info(ImageInfo *info)
+{
+    Error *errp = NULL;
+    QString *str;
+    QmpOutputVisitor *ov = qmp_output_visitor_new();
+    QObject *obj;
+    visit_type_ImageInfo(qmp_output_get_visitor(ov),
+                         &info, NULL, &errp);
+    obj = qmp_output_get_qobject(ov);
+    str = qobject_to_json_pretty(obj);
+    assert(str != NULL);
+    printf("%s\n", qstring_get_str(str));
+    qobject_decref(obj);
+    qmp_output_visitor_cleanup(ov);
+    QDECREF(str);
+}
+
+static void collect_image_info(BlockDriverState *bs,
+                   ImageInfo *info,
+                   const char *filename,
+                   const char *fmt)
 {
-    int c;
-    const char *filename, *fmt;
-    BlockDriverState *bs;
-    char size_buf[128], dsize_buf[128];
     uint64_t total_sectors;
-    int64_t allocated_size;
     char backing_filename[1024];
     char backing_filename2[1024];
     BlockDriverInfo bdi;
 
+    bdrv_get_geometry(bs, &total_sectors);
+
+    info->filename        = g_strdup(filename);
+    info->format          = g_strdup(bdrv_get_format_name(bs));
+    info->virtual_size    = total_sectors * 512;
+    info->actual_size     = bdrv_get_allocated_file_size(bs);
+    info->has_actual_size = info->actual_size >= 0;
+    if (bdrv_is_encrypted(bs)) {
+        info->encrypted = true;
+        info->has_encrypted = true;
+    }
+    if (bdrv_get_info(bs, &bdi) >= 0) {
+        if (bdi.cluster_size != 0) {
+            info->cluster_size = bdi.cluster_size;
+            info->has_cluster_size = true;
+        }
+        info->dirty_flag = bdi.is_dirty;
+        info->has_dirty_flag = true;
+    }
+    bdrv_get_backing_filename(bs, backing_filename, sizeof(backing_filename));
+    if (backing_filename[0] != '\0') {
+        info->backing_filename = g_strdup(backing_filename);
+        info->has_backing_filename = true;
+        bdrv_get_full_backing_filename(bs, backing_filename2,
+                                       sizeof(backing_filename2));
+
+        if (strcmp(backing_filename, backing_filename2) != 0) {
+            info->full_backing_filename =
+                        g_strdup(backing_filename2);
+            info->has_full_backing_filename = true;
+        }
+
+        if (bs->backing_format[0]) {
+            info->backing_filename_format = g_strdup(bs->backing_format);
+            info->has_backing_filename_format = true;
+        }
+    }
+}
+
+static void dump_human_image_info(ImageInfo *info)
+{
+    char size_buf[128], dsize_buf[128];
+    if (!info->has_actual_size) {
+        snprintf(dsize_buf, sizeof(dsize_buf), "unavailable");
+    } else {
+        get_human_readable_size(dsize_buf, sizeof(dsize_buf),
+                                info->actual_size);
+    }
+    get_human_readable_size(size_buf, sizeof(size_buf), info->virtual_size);
+    printf("image: %s\n"
+           "file format: %s\n"
+           "virtual size: %s (%" PRId64 " bytes)\n"
+           "disk size: %s\n",
+           info->filename, info->format, size_buf,
+           info->virtual_size,
+           dsize_buf);
+
+    if (info->has_encrypted && info->encrypted) {
+        printf("encrypted: yes\n");
+    }
+
+    if (info->has_cluster_size) {
+        printf("cluster_size: %" PRId64 "\n", info->cluster_size);
+    }
+
+    if (info->has_dirty_flag && info->dirty_flag) {
+        printf("cleanly shut down: no\n");
+    }
+
+    if (info->has_backing_filename) {
+        printf("backing file: %s", info->backing_filename);
+        if (info->has_full_backing_filename) {
+            printf(" (actual path: %s)", info->full_backing_filename);
+        }
+        putchar('\n');
+        if (info->has_backing_filename_format) {
+            printf("backing file format: %s\n", info->backing_filename_format);
+        }
+    }
+
+    if (info->has_snapshots) {
+        SnapshotInfoList *elem;
+        char buf[256];
+
+        printf("Snapshot list:\n");
+        printf("%s\n", bdrv_snapshot_dump(buf, sizeof(buf), NULL));
+
+        /* Ideally bdrv_snapshot_dump() would operate on SnapshotInfoList but
+         * we convert to the block layer's native QEMUSnapshotInfo for now.
+         */
+        for (elem = info->snapshots; elem; elem = elem->next) {
+            QEMUSnapshotInfo sn = {
+                .vm_state_size = elem->value->vm_state_size,
+                .date_sec = elem->value->date_sec,
+                .date_nsec = elem->value->date_nsec,
+                .vm_clock_nsec = elem->value->vm_clock_sec * 1000000000ULL +
+                                 elem->value->vm_clock_nsec,
+            };
+
+            pstrcpy(sn.id_str, sizeof(sn.id_str), elem->value->id);
+            pstrcpy(sn.name, sizeof(sn.name), elem->value->name);
+            printf("%s\n", bdrv_snapshot_dump(buf, sizeof(buf), &sn));
+        }
+    }
+}
+
+static void dump_human_image_info_list(ImageInfoList *list)
+{
+    ImageInfoList *elem;
+    bool delim = false;
+
+    for (elem = list; elem; elem = elem->next) {
+        if (delim) {
+            printf("\n");
+        }
+        delim = true;
+
+        dump_human_image_info(elem->value);
+    }
+}
+
+static gboolean str_equal_func(gconstpointer a, gconstpointer b)
+{
+    return strcmp(a, b) == 0;
+}
+
+/**
+ * Open an image file chain and return an ImageInfoList
+ *
+ * @filename: topmost image filename
+ * @fmt: topmost image format (may be NULL to autodetect)
+ * @chain: true  - enumerate entire backing file chain
+ *         false - only topmost image file
+ *
+ * Returns a list of ImageInfo objects or NULL if there was an error opening an
+ * image file.  If there was an error a message will have been printed to
+ * stderr.
+ */
+static ImageInfoList *collect_image_info_list(const char *filename,
+                                              const char *fmt,
+                                              bool chain)
+{
+    ImageInfoList *head = NULL;
+    ImageInfoList **last = &head;
+    GHashTable *filenames;
+
+    filenames = g_hash_table_new_full(g_str_hash, str_equal_func, NULL, NULL);
+
+    while (filename) {
+        BlockDriverState *bs;
+        ImageInfo *info;
+        ImageInfoList *elem;
+
+        if (g_hash_table_lookup_extended(filenames, filename, NULL, NULL)) {
+            error_report("Backing file '%s' creates an infinite loop.",
+                         filename);
+            goto err;
+        }
+        g_hash_table_insert(filenames, (gpointer)filename, NULL);
+
+        bs = bdrv_new_open(filename, fmt, BDRV_O_FLAGS | BDRV_O_NO_BACKING,
+                           false);
+        if (!bs) {
+            goto err;
+        }
+
+        info = g_new0(ImageInfo, 1);
+        collect_image_info(bs, info, filename, fmt);
+        collect_snapshots(bs, info);
+
+        elem = g_new0(ImageInfoList, 1);
+        elem->value = info;
+        *last = elem;
+        last = &elem->next;
+
+        bdrv_delete(bs);
+
+        filename = fmt = NULL;
+        if (chain) {
+            if (info->has_full_backing_filename) {
+                filename = info->full_backing_filename;
+            } else if (info->has_backing_filename) {
+                filename = info->backing_filename;
+            }
+            if (info->has_backing_filename_format) {
+                fmt = info->backing_filename_format;
+            }
+        }
+    }
+    g_hash_table_destroy(filenames);
+    return head;
+
+err:
+    qapi_free_ImageInfoList(head);
+    g_hash_table_destroy(filenames);
+    return NULL;
+}
+
+enum {
+    OPTION_OUTPUT = 256,
+    OPTION_BACKING_CHAIN = 257,
+};
+
+typedef enum OutputFormat {
+    OFORMAT_JSON,
+    OFORMAT_HUMAN,
+} OutputFormat;
+
+static int img_info(int argc, char **argv)
+{
+    int c;
+    OutputFormat output_format = OFORMAT_HUMAN;
+    bool chain = false;
+    const char *filename, *fmt, *output;
+    ImageInfoList *list;
+
     fmt = NULL;
+    output = NULL;
     for(;;) {
-        c = getopt(argc, argv, "f:h");
+        int option_index = 0;
+        static const struct option long_options[] = {
+            {"help", no_argument, 0, 'h'},
+            {"format", required_argument, 0, 'f'},
+            {"output", required_argument, 0, OPTION_OUTPUT},
+            {"backing-chain", no_argument, 0, OPTION_BACKING_CHAIN},
+            {0, 0, 0, 0}
+        };
+        c = getopt_long(argc, argv, "f:h",
+                        long_options, &option_index);
         if (c == -1) {
             break;
         }
@@ -1128,6 +1429,12 @@ static int img_info(int argc, char **argv)
         case 'f':
             fmt = optarg;
             break;
+        case OPTION_OUTPUT:
+            output = optarg;
+            break;
+        case OPTION_BACKING_CHAIN:
+            chain = true;
+            break;
         }
     }
     if (optind >= argc) {
@@ -1135,49 +1442,34 @@ static int img_info(int argc, char **argv)
     }
     filename = argv[optind++];
 
-    bs = bdrv_new_open(filename, fmt, BDRV_O_FLAGS | BDRV_O_NO_BACKING);
-    if (!bs) {
+    if (output && !strcmp(output, "json")) {
+        output_format = OFORMAT_JSON;
+    } else if (output && !strcmp(output, "human")) {
+        output_format = OFORMAT_HUMAN;
+    } else if (output) {
+        error_report("--output must be used with human or json as argument.");
         return 1;
     }
-    bdrv_get_geometry(bs, &total_sectors);
-    get_human_readable_size(size_buf, sizeof(size_buf), total_sectors * 512);
-    allocated_size = bdrv_get_allocated_file_size(bs);
-    if (allocated_size < 0) {
-        snprintf(dsize_buf, sizeof(dsize_buf), "unavailable");
-    } else {
-        get_human_readable_size(dsize_buf, sizeof(dsize_buf),
-                                allocated_size);
-    }
-    printf("image: %s\n"
-           "file format: %s\n"
-           "virtual size: %s (%" PRId64 " bytes)\n"
-           "disk size: %s\n",
-           filename, bdrv_get_format_name(bs), size_buf,
-           (total_sectors * 512),
-           dsize_buf);
-    if (bdrv_is_encrypted(bs)) {
-        printf("encrypted: yes\n");
-    }
-    if (bdrv_get_info(bs, &bdi) >= 0) {
-        if (bdi.cluster_size != 0) {
-            printf("cluster_size: %d\n", bdi.cluster_size);
-        }
-        if (bdi.is_dirty) {
-            printf("cleanly shut down: no\n");
-        }
+
+    list = collect_image_info_list(filename, fmt, chain);
+    if (!list) {
+        return 1;
     }
-    bdrv_get_backing_filename(bs, backing_filename, sizeof(backing_filename));
-    if (backing_filename[0] != '\0') {
-        bdrv_get_full_backing_filename(bs, backing_filename2,
-                                       sizeof(backing_filename2));
-        printf("backing file: %s", backing_filename);
-        if (strcmp(backing_filename, backing_filename2) != 0) {
-            printf(" (actual path: %s)", backing_filename2);
+
+    switch (output_format) {
+    case OFORMAT_HUMAN:
+        dump_human_image_info_list(list);
+        break;
+    case OFORMAT_JSON:
+        if (chain) {
+            dump_json_image_info_list(list);
+        } else {
+            dump_json_image_info(list->value);
         }
-        putchar('\n');
+        break;
     }
-    dump_snapshots(bs);
-    bdrv_delete(bs);
+
+    qapi_free_ImageInfoList(list);
     return 0;
 }
 
@@ -1248,7 +1540,7 @@ static int img_snapshot(int argc, char **argv)
     filename = argv[optind++];
 
     /* Open the image */
-    bs = bdrv_new_open(filename, NULL, bdrv_oflags);
+    bs = bdrv_new_open(filename, NULL, bdrv_oflags, true);
     if (!bs) {
         return 1;
     }
@@ -1366,7 +1658,7 @@ static int img_rebase(int argc, char **argv)
      * Ignore the old backing file for unsafe rebase in case we want to correct
      * the reference to a renamed or moved backing file.
      */
-    bs = bdrv_new_open(filename, fmt, flags);
+    bs = bdrv_new_open(filename, fmt, flags, true);
     if (!bs) {
         return 1;
     }
@@ -1409,13 +1701,15 @@ static int img_rebase(int argc, char **argv)
             error_report("Could not open old backing file '%s'", backing_name);
             goto out;
         }
-
-        bs_new_backing = bdrv_new("new_backing");
-        ret = bdrv_open(bs_new_backing, out_baseimg, BDRV_O_FLAGS,
+        if (out_baseimg[0]) {
+            bs_new_backing = bdrv_new("new_backing");
+            ret = bdrv_open(bs_new_backing, out_baseimg, BDRV_O_FLAGS,
                         new_backing_drv);
-        if (ret) {
-            error_report("Could not open new backing file '%s'", out_baseimg);
-            goto out;
+            if (ret) {
+                error_report("Could not open new backing file '%s'",
+                             out_baseimg);
+                goto out;
+            }
         }
     }
 
@@ -1431,22 +1725,27 @@ static int img_rebase(int argc, char **argv)
     if (!unsafe) {
         uint64_t num_sectors;
         uint64_t old_backing_num_sectors;
-        uint64_t new_backing_num_sectors;
+        uint64_t new_backing_num_sectors = 0;
         uint64_t sector;
         int n;
         uint8_t * buf_old;
         uint8_t * buf_new;
-        float local_progress;
+        float local_progress = 0;
 
         buf_old = qemu_blockalign(bs, IO_BUF_SIZE);
         buf_new = qemu_blockalign(bs, IO_BUF_SIZE);
 
         bdrv_get_geometry(bs, &num_sectors);
         bdrv_get_geometry(bs_old_backing, &old_backing_num_sectors);
-        bdrv_get_geometry(bs_new_backing, &new_backing_num_sectors);
+        if (bs_new_backing) {
+            bdrv_get_geometry(bs_new_backing, &new_backing_num_sectors);
+        }
+
+        if (num_sectors != 0) {
+            local_progress = (float)100 /
+                (num_sectors / MIN(num_sectors, IO_BUF_SIZE / 512));
+        }
 
-        local_progress = (float)100 /
-            (num_sectors / MIN(num_sectors, IO_BUF_SIZE / 512));
         for (sector = 0; sector < num_sectors; sector += n) {
 
             /* How many sectors can we handle with the next read? */
@@ -1480,7 +1779,7 @@ static int img_rebase(int argc, char **argv)
                 }
             }
 
-            if (sector >= new_backing_num_sectors) {
+            if (sector >= new_backing_num_sectors || !bs_new_backing) {
                 memset(buf_new, 0, n * BDRV_SECTOR_SIZE);
             } else {
                 if (sector + n > new_backing_num_sectors) {
@@ -1526,7 +1825,12 @@ static int img_rebase(int argc, char **argv)
      * backing file are overwritten in the COW file now, so the visible content
      * doesn't change when we switch the backing file.
      */
-    ret = bdrv_change_backing_file(bs, out_baseimg, out_basefmt);
+    if (out_baseimg && *out_baseimg) {
+        ret = bdrv_change_backing_file(bs, out_baseimg, out_basefmt);
+    } else {
+        ret = bdrv_change_backing_file(bs, NULL, NULL);
+    }
+
     if (ret == -ENOSPC) {
         error_report("Could not change the backing file to '%s': No "
                      "space left in the file header", out_baseimg);
@@ -1639,7 +1943,7 @@ static int img_resize(int argc, char **argv)
     n = qemu_opt_get_size(param, BLOCK_OPT_SIZE, 0);
     qemu_opts_del(param);
 
-    bs = bdrv_new_open(filename, fmt, BDRV_O_FLAGS | BDRV_O_RDWR);
+    bs = bdrv_new_open(filename, fmt, BDRV_O_FLAGS | BDRV_O_RDWR, true);
     if (!bs) {
         ret = -1;
         goto out;
@@ -1697,14 +2001,13 @@ int main(int argc, char **argv)
 
     error_set_progname(argv[0]);
 
+    qemu_init_main_loop();
     bdrv_init();
     if (argc < 2)
         help();
     cmdname = argv[1];
     argc--; argv++;
 
-    qemu_init_main_loop();
-
     /* find the command */
     for(cmd = img_cmds; cmd->name != NULL; cmd++) {
         if (!strcmp(cmdname, cmd->name)) {
diff --git a/qemu-img.texi b/qemu-img.texi
index 6b42e35fe..00fca8da8 100644
--- a/qemu-img.texi
+++ b/qemu-img.texi
@@ -28,6 +28,10 @@ Command parameters:
 is the disk image format. It is guessed automatically in most cases. See below
 for a description of the supported disk formats.
 
+@item --backing-chain
+will enumerate information about backing files in a disk image chain. Refer
+below for further description.
+
 @item size
 is the disk image size in bytes. Optional suffixes @code{k} or @code{K}
 (kilobyte, 1024) @code{M} (megabyte, 1024k) and @code{G} (gigabyte, 1024M)
@@ -87,7 +91,7 @@ Perform a consistency check on the disk image @var{filename}.
 If @code{-r} is specified, qemu-img tries to repair any inconsistencies found
 during the check. @code{-r leaks} repairs only cluster leaks, whereas
 @code{-r all} fixes all kinds of errors, with a higher risk of choosing the
-wrong fix or hiding corruption that has already occured.
+wrong fix or hiding corruption that has already occurred.
 
 Only the formats @code{qcow2}, @code{qed} and @code{vdi} support
 consistency checks.
@@ -129,12 +133,28 @@ created as a copy on write image of the specified base image; the
 @var{backing_file} should have the same content as the input's base image,
 however the path, image format, etc may differ.
 
-@item info [-f @var{fmt}] @var{filename}
+@item info [-f @var{fmt}] [--output=@var{ofmt}] [--backing-chain] @var{filename}
 
 Give information about the disk image @var{filename}. Use it in
 particular to know the size reserved on disk which can be different
 from the displayed size. If VM snapshots are stored in the disk image,
-they are displayed too.
+they are displayed too. The command can output in the format @var{ofmt}
+which is either @code{human} or @code{json}.
+
+If a disk image has a backing file chain, information about each disk image in
+the chain can be recursively enumerated by using the option @code{--backing-chain}.
+
+For instance, if you have an image chain like:
+
+@example
+base.qcow2 <- snap1.qcow2 <- snap2.qcow2
+@end example
+
+To enumerate information about each disk image in the above chain, starting from top to base, do:
+
+@example
+qemu-img info --backing-chain snap2.qcow2
+@end example
 
 @item snapshot [-l | -a @var{snapshot} | -c @var{snapshot} | -d @var{snapshot} ] @var{filename}
 
@@ -147,7 +167,9 @@ Changes the backing file of an image. Only the formats @code{qcow2} and
 
 The backing file is changed to @var{backing_file} and (if the image format of
 @var{filename} supports this) the backing file format is changed to
-@var{backing_fmt}.
+@var{backing_fmt}. If @var{backing_file} is specified as ``'' (the empty
+string), then the image is rebased onto no backing file (i.e. it will exist
+independently of any backing file).
 
 There are two different modes in which @code{rebase} can operate:
 @table @option
@@ -204,7 +226,10 @@ After using this command to grow a disk image, you must use file system and
 partitioning tools inside the VM to actually begin using the new space on the
 device.
 @end table
+@c man end
 
+@ignore
+@c man begin NOTES
 Supported image file formats:
 
 @table @option
@@ -225,6 +250,13 @@ support of multiple VM snapshots.
 
 Supported options:
 @table @code
+@item compat
+Determines the qcow2 version to use. @code{compat=0.10} uses the traditional
+image format that can be read by any QEMU since 0.10 (this is the default).
+@code{compat=1.1} enables image format extensions that only QEMU 1.1 and
+newer understand. Amongst others, this includes zero clusters, which allow
+efficient copy-on-read for sparse images.
+
 @item backing_file
 File name of a base image (see @option{create} subcommand)
 @item backing_fmt
@@ -245,73 +277,33 @@ Preallocation mode (allowed values: off, metadata). An image with preallocated
 metadata is initially larger but can improve performance when the image needs
 to grow.
 
-@end table
+@item lazy_refcounts
+If this option is set to @code{on}, reference count updates are postponed with
+the goal of avoiding metadata I/O and improving performance. This is
+particularly interesting with @option{cache=writethrough} which doesn't batch
+metadata updates. The tradeoff is that after a host crash, the reference count
+tables must be rebuilt, i.e. on the next open an (automatic) @code{qemu-img
+check -r all} is required, which may take some time.
 
-@item qed
-Image format with support for backing files and compact image files (when your
-filesystem or transport medium does not support holes).  Good performance due
-to less metadata than the more featureful qcow2 format, especially with
-cache=writethrough or cache=directsync.  Consider using qcow2 which will soon
-have a similar optimization and is most actively developed.
+This option can only be enabled if @code{compat=1.1} is specified.
 
-Supported options:
-@table @code
-@item backing_file
-File name of a base image (see @option{create} subcommand).
-@item backing_fmt
-Image file format of backing file (optional).  Useful if the format cannot be
-autodetected because it has no header, like some vhd/vpc files.
-@item cluster_size
-Changes the cluster size (must be power-of-2 between 4K and 64K). Smaller
-cluster sizes can improve the image file size whereas larger cluster sizes
-generally provide better performance.
-@item table_size
-Changes the number of clusters per L1/L2 table (must be power-of-2 between 1
-and 16).  There is normally no need to change this value but this option can be
-used for performance benchmarking.
 @end table
 
-@item qcow
-Old QEMU image format. Left for compatibility.
+@item Other
+QEMU also supports various other image file formats for compatibility with
+older QEMU versions or other hypervisors, including VMDK, VDI, VHD (vpc), qcow1
+and QED. For a full list of supported formats see @code{qemu-img --help}.
+For a more detailed description of these formats, see the QEMU Emulation User
+Documentation.
 
-Supported options:
-@table @code
-@item backing_file
-File name of a base image (see @option{create} subcommand)
-@item encryption
-If this option is set to @code{on}, the image is encrypted.
-@end table
-
-@item cow
-User Mode Linux Copy On Write image format. Used to be the only growable
-image format in QEMU. It is supported only for compatibility with
-previous versions. It does not work on win32.
-@item vdi
-VirtualBox 1.1 compatible image format.
-@item vmdk
-VMware 3 and 4 compatible image format.
-
-Supported options:
-@table @code
-@item backing_fmt
-Image format of the base image
-@item compat6
-Create a VMDK version 6 image (instead of version 4)
-@end table
-
-@item vpc
-VirtualPC compatible image format (VHD).
-
-@item cloop
-Linux Compressed Loop image, useful only to reuse directly compressed
-CD-ROM images present for example in the Knoppix CD-ROMs.
+The main purpose of the block drivers for these formats is image conversion.
+For running VMs, it is recommended to convert the disk images to either raw or
+qcow2 in order to achieve good performance.
 @end table
 
 
 @c man end
 
-@ignore
-
 @setfilename qemu-img
 @settitle QEMU disk image utility
 
diff --git a/qemu-io.c b/qemu-io.c
index d0f4fb70c..92cdb2ab9 100644
--- a/qemu-io.c
+++ b/qemu-io.c
@@ -1362,7 +1362,7 @@ static int aio_write_f(int argc, char **argv)
 
 static int aio_flush_f(int argc, char **argv)
 {
-    qemu_aio_flush();
+    bdrv_drain_all();
     return 0;
 }
 
@@ -1892,9 +1892,8 @@ int main(int argc, char **argv)
         exit(1);
     }
 
-    bdrv_init();
-
     qemu_init_main_loop();
+    bdrv_init();
 
     /* initialize commands */
     quit_init();
diff --git a/qemu-log.c b/qemu-log.c
index 396aafdf6..a4c3d1f2e 100644
--- a/qemu-log.c
+++ b/qemu-log.c
@@ -116,6 +116,9 @@ const CPULogItem cpu_log_items[] = {
       "show all i/o ports accesses" },
     { LOG_UNIMP, "unimp",
       "log unimplemented functionality" },
+    { LOG_GUEST_ERROR, "guest_errors",
+      "log when the guest OS does something invalid (eg accessing a\n"
+      "non-existent register)" },
     { 0, NULL, NULL },
 };
 
diff --git a/qemu-log.h b/qemu-log.h
index 5ccecf30a..344eca3f1 100644
--- a/qemu-log.h
+++ b/qemu-log.h
@@ -35,6 +35,7 @@ static inline bool qemu_log_enabled(void)
 #define CPU_LOG_TB_CPU     (1 << 8)
 #define CPU_LOG_RESET      (1 << 9)
 #define LOG_UNIMP          (1 << 10)
+#define LOG_GUEST_ERROR    (1 << 11)
 
 /* Returns true if a bit is set in the current loglevel mask
  */
@@ -83,10 +84,10 @@ static inline void log_cpu_state_mask(int mask, CPUArchState *env1, int flags)
 }
 
 /* disas() and target_disas() to qemu_logfile: */
-static inline void log_target_disas(target_ulong start, target_ulong len,
-                                    int flags)
+static inline void log_target_disas(CPUArchState *env, target_ulong start,
+                                    target_ulong len, int flags)
 {
-    target_disas(qemu_logfile, start, len, flags);
+    target_disas(qemu_logfile, env, start, len, flags);
 }
 
 static inline void log_disas(void *code, unsigned long size)
diff --git a/qemu-nbd.c b/qemu-nbd.c
index 1c1cf6a46..80f08d846 100644
--- a/qemu-nbd.c
+++ b/qemu-nbd.c
@@ -41,8 +41,8 @@ static NBDExport *exp;
 static int verbose;
 static char *srcpath;
 static char *sockpath;
-static bool sigterm_reported;
-static bool nbd_started;
+static int persistent = 0;
+static enum { RUNNING, TERMINATE, TERMINATING, TERMINATED } state;
 static int shared = 1;
 static int nb_fds;
 
@@ -186,7 +186,7 @@ static int find_partition(BlockDriverState *bs, int partition,
 
 static void termsig_handler(int signum)
 {
-    sigterm_reported = true;
+    state = TERMINATE;
     qemu_notify_event();
 }
 
@@ -269,10 +269,20 @@ static int nbd_can_accept(void *opaque)
     return nb_fds < shared;
 }
 
+static void nbd_export_closed(NBDExport *exp)
+{
+    assert(state == TERMINATING);
+    state = TERMINATED;
+}
+
 static void nbd_client_closed(NBDClient *client)
 {
     nb_fds--;
+    if (nb_fds == 0 && !persistent && state == RUNNING) {
+        state = TERMINATE;
+    }
     qemu_notify_event();
+    nbd_client_put(client);
 }
 
 static void nbd_accept(void *opaque)
@@ -282,7 +292,11 @@ static void nbd_accept(void *opaque)
     socklen_t addr_len = sizeof(addr);
 
     int fd = accept(server_fd, (struct sockaddr *)&addr, &addr_len);
-    nbd_started = true;
+    if (state >= TERMINATE) {
+        close(fd);
+        return;
+    }
+
     if (fd >= 0 && nbd_client_new(exp, fd, nbd_client_closed)) {
         nb_fds++;
     }
@@ -329,7 +343,6 @@ int main(int argc, char **argv)
     int partition = -1;
     int ret;
     int fd;
-    int persistent = 0;
     bool seen_cache = false;
 #ifdef CONFIG_LINUX_AIO
     bool seen_aio = false;
@@ -526,6 +539,7 @@ int main(int argc, char **argv)
         snprintf(sockpath, 128, SOCKET_PATH, basename(device));
     }
 
+    qemu_init_main_loop();
     bdrv_init();
     atexit(bdrv_close_all);
 
@@ -546,7 +560,7 @@ int main(int argc, char **argv)
         }
     }
 
-    exp = nbd_export_new(bs, dev_offset, fd_size, nbdflags);
+    exp = nbd_export_new(bs, dev_offset, fd_size, nbdflags, nbd_export_closed);
 
     if (sockpath) {
         fd = unix_socket_incoming(sockpath);
@@ -571,7 +585,6 @@ int main(int argc, char **argv)
         memset(&client_thread, 0, sizeof(client_thread));
     }
 
-    qemu_init_main_loop();
     qemu_set_fd_handler2(fd, nbd_can_accept, nbd_accept, NULL,
                          (void *)(uintptr_t)fd);
 
@@ -581,11 +594,18 @@ int main(int argc, char **argv)
         err(EXIT_FAILURE, "Could not chdir to root directory");
     }
 
+    state = RUNNING;
     do {
         main_loop_wait(false);
-    } while (!sigterm_reported && (persistent || !nbd_started || nb_fds > 0));
+        if (state == TERMINATE) {
+            state = TERMINATING;
+            nbd_export_close(exp);
+            nbd_export_put(exp);
+            exp = NULL;
+        }
+    } while (state != TERMINATED);
 
-    nbd_export_close(exp);
+    bdrv_close(bs);
     if (sockpath) {
         unlink(sockpath);
     }
diff --git a/qemu-options.hx b/qemu-options.hx
index 3c411c427..de43b1b48 100644
--- a/qemu-options.hx
+++ b/qemu-options.hx
@@ -6,10 +6,6 @@ HXCOMM construct option structures, enums and help message for specified
 HXCOMM architectures.
 HXCOMM HXCOMM can be used for comments, discarded from both texi and C
 
-HXCOMM TODO : when we are able to change -help output without breaking
-HXCOMM libvirt we should update the help options which refer to -cpu ?,
-HXCOMM -driver ?, etc to use the preferred -cpu help etc instead.
-
 DEFHEADING(Standard options:)
 STEXI
 @table @option
@@ -33,17 +29,18 @@ ETEXI
 
 DEF("machine", HAS_ARG, QEMU_OPTION_machine, \
     "-machine [type=]name[,prop[=value][,...]]\n"
-    "                selects emulated machine (-machine ? for list)\n"
+    "                selects emulated machine ('-machine help' for list)\n"
     "                property accel=accel1[:accel2[:...]] selects accelerator\n"
     "                supported accelerators are kvm, xen, tcg (default: tcg)\n"
     "                kernel_irqchip=on|off controls accelerated irqchip support\n"
     "                kvm_shadow_mem=size of KVM shadow MMU\n"
-    "                dump-guest-core=on|off include guest memory in a core dump (default=on)\n",
+    "                dump-guest-core=on|off include guest memory in a core dump (default=on)\n"
+    "                mem-merge=on|off controls memory merge support (default: on)\n",
     QEMU_ARCH_ALL)
 STEXI
 @item -machine [type=]@var{name}[,prop=@var{value}[,...]]
 @findex -machine
-Select the emulated machine by @var{name}. Use @code{-machine ?} to list
+Select the emulated machine by @var{name}. Use @code{-machine help} to list
 available machines. Supported machine properties are:
 @table @option
 @item accel=@var{accels1}[:@var{accels2}[:...]]
@@ -57,6 +54,10 @@ Enables in-kernel irqchip support for the chosen accelerator when available.
 Defines the size of the KVM shadow MMU.
 @item dump-guest-core=on|off
 Include guest memory in a core dump. The default is on.
+@item mem-merge=on|off
+Enables or disables memory merge support. This feature, when supported by
+the host, de-duplicates identical memory pages among VMs instances
+(enabled by default).
 @end table
 ETEXI
 
@@ -64,11 +65,11 @@ HXCOMM Deprecated by -machine
 DEF("M", HAS_ARG, QEMU_OPTION_M, "", QEMU_ARCH_ALL)
 
 DEF("cpu", HAS_ARG, QEMU_OPTION_cpu,
-    "-cpu cpu        select CPU (-cpu ? for list)\n", QEMU_ARCH_ALL)
+    "-cpu cpu        select CPU ('-cpu help' for list)\n", QEMU_ARCH_ALL)
 STEXI
 @item -cpu @var{model}
 @findex -cpu
-Select CPU model (-cpu ? for list and additional feature selection)
+Select CPU model (@code{-cpu help} for list and additional feature selection)
 ETEXI
 
 DEF("smp", HAS_ARG, QEMU_OPTION_smp,
@@ -205,33 +206,33 @@ Open drive @option{file} as read-only. Guest write attempts will fail.
 file sectors into the image file.
 @end table
 
-By default, writethrough caching is used for all block device.  This means that
-the host page cache will be used to read and write data but write notification
-will be sent to the guest only when the data has been reported as written by
-the storage subsystem.
+By default, the @option{cache=writeback} mode is used. It will report data
+writes as completed as soon as the data is present in the host page cache.
+This is safe as long as your guest OS makes sure to correctly flush disk caches
+where needed. If your guest OS does not handle volatile disk write caches
+correctly and your host crashes or loses power, then the guest may experience
+data corruption.
 
-Writeback caching will report data writes as completed as soon as the data is
-present in the host page cache.  This is safe as long as you trust your host.
-If your host crashes or loses power, then the guest may experience data
-corruption.
+For such guests, you should consider using @option{cache=writethrough}. This
+means that the host page cache will be used to read and write data, but write
+notification will be sent to the guest only after QEMU has made sure to flush
+each write to the disk. Be aware that this has a major impact on performance.
 
 The host page cache can be avoided entirely with @option{cache=none}.  This will
-attempt to do disk IO directly to the guests memory.  QEMU may still perform
-an internal copy of the data.
+attempt to do disk IO directly to the guest's memory.  QEMU may still perform
+an internal copy of the data. Note that this is considered a writeback mode and
+the guest OS must handle the disk write cache correctly in order to avoid data
+corruption on host crashes.
 
 The host page cache can be avoided while only sending write notifications to
-the guest when the data has been reported as written by the storage subsystem
-using @option{cache=directsync}.
-
-Some block drivers perform badly with @option{cache=writethrough}, most notably,
-qcow2.  If performance is more important than correctness,
-@option{cache=writeback} should be used with qcow2.
+the guest when the data has been flushed to the disk using
+@option{cache=directsync}.
 
 In case you don't care about data integrity over host failures, use
-cache=unsafe. This option tells QEMU that it never needs to write any data
-to the disk but can instead keeps things in cache. If anything goes wrong,
+@option{cache=unsafe}. This option tells QEMU that it never needs to write any
+data to the disk but can instead keep things in cache. If anything goes wrong,
 like your host losing power, the disk storage getting disconnected accidentally,
-etc. you're image will most probably be rendered unusable.   When using
+etc. your image will most probably be rendered unusable.   When using
 the @option{-snapshot} option, unsafe caching is always used.
 
 Copy-on-read avoids accessing the same backing file sectors repeatedly and is
@@ -252,6 +253,14 @@ qemu-system-i386 -drive file=file,index=2,media=disk
 qemu-system-i386 -drive file=file,index=3,media=disk
 @end example
 
+You can open an image using pre-opened file descriptors from an fd set:
+@example
+qemu-system-i386
+-add-fd fd=3,set=2,opaque="rdwr:/path/to/file"
+-add-fd fd=4,set=2,opaque="rdonly:/path/to/file"
+-drive file=/dev/fdset/2,index=0,media=disk
+@end example
+
 You can connect a CDROM to the slave of ide0:
 @example
 qemu-system-i386 -drive file=file,if=ide,index=1,media=cdrom
@@ -284,6 +293,34 @@ qemu-system-i386 -hda a -hdb b
 @end example
 ETEXI
 
+DEF("add-fd", HAS_ARG, QEMU_OPTION_add_fd,
+    "-add-fd fd=fd,set=set[,opaque=opaque]\n"
+    "                Add 'fd' to fd 'set'\n", QEMU_ARCH_ALL)
+STEXI
+@item -add-fd fd=@var{fd},set=@var{set}[,opaque=@var{opaque}]
+@findex -add-fd
+
+Add a file descriptor to an fd set.  Valid options are:
+
+@table @option
+@item fd=@var{fd}
+This option defines the file descriptor of which a duplicate is added to fd set.
+The file descriptor cannot be stdin, stdout, or stderr.
+@item set=@var{set}
+This option defines the ID of the fd set to add the file descriptor to.
+@item opaque=@var{opaque}
+This option defines a free-form string that can be used to describe @var{fd}.
+@end table
+
+You can open an image using pre-opened file descriptors from an fd set:
+@example
+qemu-system-i386
+-add-fd fd=3,set=2,opaque="rdwr:/path/to/file"
+-add-fd fd=4,set=2,opaque="rdonly:/path/to/file"
+-drive file=/dev/fdset/2,index=0,media=disk
+@end example
+ETEXI
+
 DEF("set", HAS_ARG, QEMU_OPTION_set,
     "-set group.id.arg=value\n"
     "                set <arg> parameter for item <id> of type <group>\n"
@@ -339,13 +376,14 @@ ETEXI
 
 DEF("boot", HAS_ARG, QEMU_OPTION_boot,
     "-boot [order=drives][,once=drives][,menu=on|off]\n"
-    "      [,splash=sp_name][,splash-time=sp_time]\n"
+    "      [,splash=sp_name][,splash-time=sp_time][,reboot-timeout=rb_time]\n"
     "                'drives': floppy (a), hard disk (c), CD-ROM (d), network (n)\n"
     "                'sp_name': the file's name that would be passed to bios as logo picture, if menu=on\n"
-    "                'sp_time': the period that splash picture last if menu=on, unit is ms\n",
+    "                'sp_time': the period that splash picture last if menu=on, unit is ms\n"
+    "                'rb_timeout': the timeout before guest reboot when boot failed, unit is ms\n",
     QEMU_ARCH_ALL)
 STEXI
-@item -boot [order=@var{drives}][,once=@var{drives}][,menu=on|off][,splash=@var{sp_name}][,splash-time=@var{sp_time}]
+@item -boot [order=@var{drives}][,once=@var{drives}][,menu=on|off][,splash=@var{sp_name}][,splash-time=@var{sp_time}][,reboot-timeout=@var{rb_timeout}]
 @findex -boot
 Specify boot order @var{drives} as a string of drive letters. Valid
 drive letters depend on the target achitecture. The x86 PC uses: a, b
@@ -364,6 +402,11 @@ limitation: The splash file could be a jpeg file or a BMP file in 24 BPP
 format(true color). The resolution should be supported by the SVGA mode, so
 the recommended is 320x240, 640x480, 800x640.
 
+A timeout could be passed to bios, guest will pause for @var{rb_timeout} ms
+when boot failed, then reboot. If @var{rb_timeout} is '-1', guest will not
+reboot, qemu passes '-1' to bios by default. Currently Seabios for X86
+system support it.
+
 @example
 # try to boot from network first, then from hard disk
 qemu-system-i386 -boot order=nc
@@ -452,12 +495,12 @@ ETEXI
 DEF("soundhw", HAS_ARG, QEMU_OPTION_soundhw,
     "-soundhw c1,... enable audio support\n"
     "                and only specified sound cards (comma separated list)\n"
-    "                use -soundhw ? to get the list of supported cards\n"
-    "                use -soundhw all to enable all of them\n", QEMU_ARCH_ALL)
+    "                use '-soundhw help' to get the list of supported cards\n"
+    "                use '-soundhw all' to enable all of them\n", QEMU_ARCH_ALL)
 STEXI
 @item -soundhw @var{card1}[,@var{card2},...] or -soundhw all
 @findex -soundhw
-Enable audio and selected sound hardware. Use ? to print all
+Enable audio and selected sound hardware. Use 'help' to print all
 available sound hardware.
 
 @example
@@ -466,7 +509,7 @@ qemu-system-i386 -soundhw es1370 disk.img
 qemu-system-i386 -soundhw ac97 disk.img
 qemu-system-i386 -soundhw hda disk.img
 qemu-system-i386 -soundhw all disk.img
-qemu-system-i386 -soundhw ?
+qemu-system-i386 -soundhw help
 @end example
 
 Note that Linux's i810_audio OSS kernel (for AC97) module might
@@ -555,16 +598,16 @@ DEF("device", HAS_ARG, QEMU_OPTION_device,
     "-device driver[,prop[=value][,...]]\n"
     "                add device (based on driver)\n"
     "                prop=value,... sets driver properties\n"
-    "                use -device ? to print all possible drivers\n"
-    "                use -device driver,? to print all possible properties\n",
+    "                use '-device help' to print all possible drivers\n"
+    "                use '-device driver,help' to print all possible properties\n",
     QEMU_ARCH_ALL)
 STEXI
 @item -device @var{driver}[,@var{prop}[=@var{value}][,...]]
 @findex -device
 Add device @var{driver}.  @var{prop}=@var{value} sets driver
 properties.  Valid properties depend on the driver.  To get help on
-possible drivers and properties, use @code{-device ?} and
-@code{-device @var{driver},?}.
+possible drivers and properties, use @code{-device help} and
+@code{-device @var{driver},help}.
 ETEXI
 
 DEFHEADING()
@@ -838,7 +881,23 @@ Enable SDL.
 ETEXI
 
 DEF("spice", HAS_ARG, QEMU_OPTION_spice,
-    "-spice <args>   enable spice\n", QEMU_ARCH_ALL)
+    "-spice [port=port][,tls-port=secured-port][,x509-dir=<dir>]\n"
+    "       [,x509-key-file=<file>][,x509-key-password=<file>]\n"
+    "       [,x509-cert-file=<file>][,x509-cacert-file=<file>]\n"
+    "       [,x509-dh-key-file=<file>][,addr=addr][,ipv4|ipv6]\n"
+    "       [,tls-ciphers=<list>]\n"
+    "       [,tls-channel=[main|display|cursor|inputs|record|playback]]\n"
+    "       [,plaintext-channel=[main|display|cursor|inputs|record|playback]]\n"
+    "       [,sasl][,password=<secret>][,disable-ticketing]\n"
+    "       [,image-compression=[auto_glz|auto_lz|quic|glz|lz|off]]\n"
+    "       [,jpeg-wan-compression=[auto|never|always]]\n"
+    "       [,zlib-glz-wan-compression=[auto|never|always]]\n"
+    "       [,streaming-video=[off|all|filter]][,disable-copy-paste]\n"
+    "       [,agent-mouse=[on|off]][,playback-compression=[on|off]]\n"
+    "       [,seamless-migration=[on|off]]\n"
+    "   enable spice\n"
+    "   at least one of {port, tls-port} is mandatory\n",
+    QEMU_ARCH_ALL)
 STEXI
 @item -spice @var{option}[,@var{option}[,...]]
 @findex -spice
@@ -920,6 +979,9 @@ Enable/disable passing mouse events via vdagent.  Default is on.
 @item playback-compression=[on|off]
 Enable/disable audio stream compression (using celt 0.5.1).  Default is on.
 
+@item seamless-migration=[on|off]
+Enable/disable spice seamless migration. Default is off.
+
 @end table
 ETEXI
 
@@ -1256,8 +1318,8 @@ DEF("net", HAS_ARG, QEMU_OPTION_net,
     "                create a new Network Interface Card and connect it to VLAN 'n'\n"
 #ifdef CONFIG_SLIRP
     "-net user[,vlan=n][,name=str][,net=addr[/mask]][,host=addr][,restrict=on|off]\n"
-    "         [,hostname=host][,dhcpstart=addr][,dns=addr][,tftp=dir][,bootfile=f]\n"
-    "         [,hostfwd=rule][,guestfwd=rule]"
+    "         [,hostname=host][,dhcpstart=addr][,dns=addr][,dnssearch=domain][,tftp=dir]\n"
+    "         [,bootfile=f][,hostfwd=rule][,guestfwd=rule]"
 #ifndef _WIN32
                                              "[,smb=dir[,smbserver=addr]]\n"
 #endif
@@ -1335,9 +1397,10 @@ Valid values for @var{type} are
 @code{virtio}, @code{i82551}, @code{i82557b}, @code{i82559er},
 @code{ne2k_pci}, @code{ne2k_isa}, @code{pcnet}, @code{rtl8139},
 @code{e1000}, @code{smc91c111}, @code{lance} and @code{mcf_fec}.
-Not all devices are supported on all targets.  Use -net nic,model=?
+Not all devices are supported on all targets.  Use @code{-net nic,model=help}
 for a list of available devices for your target.
 
+@item -netdev user,id=@var{id}[,@var{option}][,@var{option}][,...]
 @item -net user[,@var{option}][,@var{option}][,...]
 Use the user mode network stack which requires no administrator
 privilege to run. Valid options are:
@@ -1346,6 +1409,7 @@ privilege to run. Valid options are:
 @item vlan=@var{n}
 Connect user mode stack to VLAN @var{n} (@var{n} = 0 is the default).
 
+@item id=@var{id}
 @item name=@var{name}
 Assign symbolic name for use in monitor commands.
 
@@ -1364,7 +1428,7 @@ able to contact the host and no guest IP packets will be routed over the host
 to the outside. This option does not affect any explicitly set forwarding rules.
 
 @item hostname=@var{name}
-Specifies the client hostname reported by the builtin DHCP server.
+Specifies the client hostname reported by the built-in DHCP server.
 
 @item dhcpstart=@var{addr}
 Specify the first of the 16 IPs the built-in DHCP server can assign. Default
@@ -1375,6 +1439,18 @@ Specify the guest-visible address of the virtual nameserver. The address must
 be different from the host address. Default is the 3rd IP in the guest network,
 i.e. x.x.x.3.
 
+@item dnssearch=@var{domain}
+Provides an entry for the domain-search list sent by the built-in
+DHCP server. More than one domain suffix can be transmitted by specifying
+this option multiple times. If supported, this will cause the guest to
+automatically try to append the given domain suffix(es) in case a domain name
+can not be resolved.
+
+Example:
+@example
+qemu -net user,dnssearch=mgmt.example.org,dnssearch=example.org [...]
+@end example
+
 @item tftp=@var{dir}
 When using the user mode network stack, activate a built-in TFTP
 server. The files in @var{dir} will be exposed as the root of a TFTP server.
@@ -1471,6 +1547,7 @@ processed and applied to -net user. Mixing them with the new configuration
 syntax gives undefined results. Their use for new applications is discouraged
 as they will be removed from future versions.
 
+@item -netdev tap,id=@var{id}[,fd=@var{h}][,ifname=@var{name}][,script=@var{file}][,downscript=@var{dfile}][,helper=@var{helper}]
 @item -net tap[,vlan=@var{n}][,name=@var{name}][,fd=@var{h}][,ifname=@var{name}][,script=@var{file}][,downscript=@var{dfile}][,helper=@var{helper}]
 Connect the host TAP network interface @var{name} to VLAN @var{n}.
 
@@ -1510,6 +1587,7 @@ qemu-system-i386 linux.img \
                  -net nic -net tap,"helper=/usr/local/libexec/qemu-bridge-helper"
 @end example
 
+@item -netdev bridge,id=@var{id}[,br=@var{bridge}][,helper=@var{helper}]
 @item -net bridge[,vlan=@var{n}][,name=@var{name}][,br=@var{bridge}][,helper=@var{helper}]
 Connect a host TAP network interface to a host bridge device.
 
@@ -1532,6 +1610,7 @@ qemu-system-i386 linux.img -net bridge -net nic,model=virtio
 qemu-system-i386 linux.img -net bridge,br=qemubr0 -net nic,model=virtio
 @end example
 
+@item -netdev socket,id=@var{id}[,fd=@var{h}][,listen=[@var{host}]:@var{port}][,connect=@var{host}:@var{port}]
 @item -net socket[,vlan=@var{n}][,name=@var{name}][,fd=@var{h}] [,listen=[@var{host}]:@var{port}][,connect=@var{host}:@var{port}]
 
 Connect the VLAN @var{n} to a remote VLAN in another QEMU virtual
@@ -1554,6 +1633,7 @@ qemu-system-i386 linux.img \
                  -net socket,connect=127.0.0.1:1234
 @end example
 
+@item -netdev socket,id=@var{id}[,fd=@var{h}][,mcast=@var{maddr}:@var{port}[,localaddr=@var{addr}]]
 @item -net socket[,vlan=@var{n}][,name=@var{name}][,fd=@var{h}][,mcast=@var{maddr}:@var{port}[,localaddr=@var{addr}]]
 
 Create a VLAN @var{n} shared with another QEMU virtual
@@ -1605,6 +1685,7 @@ qemu-system-i386 linux.img \
                  -net socket,mcast=239.192.168.1:1102,localaddr=1.2.3.4
 @end example
 
+@item -netdev vde,id=@var{id}[,sock=@var{socketpath}][,port=@var{n}][,group=@var{groupname}][,mode=@var{octalmode}]
 @item -net vde[,vlan=@var{n}][,name=@var{name}][,sock=@var{socketpath}] [,port=@var{n}][,group=@var{groupname}][,mode=@var{octalmode}]
 Connect VLAN @var{n} to PORT @var{n} of a vde switch running on host and
 listening for incoming connections on @var{socketpath}. Use GROUP @var{groupname}
@@ -1985,6 +2066,23 @@ qemu-system-i386 --drive file=sheepdog:192.0.2.1:30000:MyVirtualMachine
 
 See also @url{http://http://www.osrg.net/sheepdog/}.
 
+@item GlusterFS
+GlusterFS is an user space distributed file system.
+QEMU supports the use of GlusterFS volumes for hosting VM disk images using
+TCP, Unix Domain Sockets and RDMA transport protocols.
+
+Syntax for specifying a VM disk image on GlusterFS volume is
+@example
+gluster[+transport]://[server[:port]]/volname/image[?socket=...]
+@end example
+
+
+Example
+@example
+qemu-system-x86_84 --drive file=gluster://192.0.2.1/testvol/a.img
+@end example
+
+See also @url{http://www.gluster.org}.
 @end table
 ETEXI
 
@@ -2361,7 +2459,7 @@ Shorthand for -gdb tcp::1234, i.e. open a gdbserver on TCP port 1234
 ETEXI
 
 DEF("d", HAS_ARG, QEMU_OPTION_d, \
-    "-d item1,...    output log to /tmp/qemu.log (use -d ? for a list of log items)\n",
+    "-d item1,...    output log to /tmp/qemu.log (use '-d help' for a list of log items)\n",
     QEMU_ARCH_ALL)
 STEXI
 @item -d
@@ -2496,13 +2594,13 @@ ETEXI
 
 DEF("clock", HAS_ARG, QEMU_OPTION_clock, \
     "-clock          force the use of the given methods for timer alarm.\n" \
-    "                To see what timers are available use -clock ?\n",
+    "                To see what timers are available use '-clock help'\n",
     QEMU_ARCH_ALL)
 STEXI
 @item -clock @var{method}
 @findex -clock
 Force the use of the given methods for timer alarm. To see what timers
-are available use -clock ?.
+are available use @code{-clock help}.
 ETEXI
 
 HXCOMM Options deprecated by -rtc
@@ -2571,7 +2669,7 @@ watchdog with a single timer, or @code{i6300esb} (Intel 6300ESB I/O
 controller hub) which is a much more featureful PCI-based dual-timer
 watchdog.  Choose a model for which your guest has drivers.
 
-Use @code{-watchdog ?} to list available hardware models.  Only one
+Use @code{-watchdog help} to list available hardware models.  Only one
 watchdog can be enabled for a guest.
 ETEXI
 
@@ -2819,6 +2917,30 @@ STEXI
 Enable FIPS 140-2 compliance mode.
 ETEXI
 
+HXCOMM Deprecated by -machine accel=tcg property
+DEF("no-kvm", 0, QEMU_OPTION_no_kvm, "", QEMU_ARCH_I386)
+
+HXCOMM Deprecated by kvm-pit driver properties
+DEF("no-kvm-pit-reinjection", 0, QEMU_OPTION_no_kvm_pit_reinjection,
+    "", QEMU_ARCH_I386)
+
+HXCOMM Deprecated (ignored)
+DEF("no-kvm-pit", 0, QEMU_OPTION_no_kvm_pit, "", QEMU_ARCH_I386)
+
+HXCOMM Deprecated by -machine kernel_irqchip=on|off property
+DEF("no-kvm-irqchip", 0, QEMU_OPTION_no_kvm_irqchip, "", QEMU_ARCH_I386)
+
+HXCOMM Deprecated (ignored)
+DEF("tdf", 0, QEMU_OPTION_tdf,"", QEMU_ARCH_ALL)
+
+DEF("object", HAS_ARG, QEMU_OPTION_object,
+    "-object TYPENAME[,PROP1=VALUE1,...]\n"
+    "                create an new object of type TYPENAME setting properties\n"
+    "                in the order they are specified.  Note that the 'id'\n"
+    "                property must be set.  These objects are placed in the\n"
+    "                '/objects' path.\n",
+    QEMU_ARCH_ALL)
+
 HXCOMM This is the last statement. Insert new options before this line!
 STEXI
 @end table
diff --git a/qemu-os-posix.h b/qemu-os-posix.h
index 8e1149d96..7f198e475 100644
--- a/qemu-os-posix.h
+++ b/qemu-os-posix.h
@@ -46,4 +46,6 @@ typedef struct timeval qemu_timeval;
 typedef struct timespec qemu_timespec;
 int qemu_utimens(const char *path, const qemu_timespec *times);
 
+bool is_daemonized(void);
+
 #endif
diff --git a/qemu-os-win32.h b/qemu-os-win32.h
index 753679b19..d0e9234d2 100644
--- a/qemu-os-win32.h
+++ b/qemu-os-win32.h
@@ -28,7 +28,6 @@
 
 #include <windows.h>
 #include <winsock2.h>
-#include "main-loop.h"
 
 /* Workaround for older versions of MinGW. */
 #ifndef ECONNREFUSED
@@ -68,6 +67,12 @@
 /* Declaration of ffs() is missing in MinGW's strings.h. */
 int ffs(int i);
 
+/* Missing POSIX functions. Don't use MinGW-w64 macros. */
+#undef gmtime_r
+struct tm *gmtime_r(const time_t *timep, struct tm *result);
+#undef localtime_r
+struct tm *localtime_r(const time_t *timep, struct tm *result);
+
 static inline void os_setup_signal_handling(void) {}
 static inline void os_daemonize(void) {}
 static inline void os_setup_post(void) {}
@@ -86,4 +91,9 @@ typedef struct {
 } qemu_timeval;
 int qemu_gettimeofday(qemu_timeval *tp);
 
+static inline bool is_daemonized(void)
+{
+    return false;
+}
+
 #endif
diff --git a/qemu-pixman.c b/qemu-pixman.c
new file mode 100644
index 000000000..e46e1804f
--- /dev/null
+++ b/qemu-pixman.c
@@ -0,0 +1,80 @@
+/*
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#include "qemu-pixman.h"
+
+int qemu_pixman_get_type(int rshift, int gshift, int bshift)
+{
+    int type = PIXMAN_TYPE_OTHER;
+
+    if (rshift > gshift && gshift > bshift) {
+        if (bshift == 0) {
+            type = PIXMAN_TYPE_ARGB;
+        } else {
+#if PIXMAN_VERSION >= PIXMAN_VERSION_ENCODE(0, 21, 8)
+            type = PIXMAN_TYPE_RGBA;
+#endif
+        }
+    } else if (rshift < gshift && gshift < bshift) {
+        if (rshift == 0) {
+            type = PIXMAN_TYPE_ABGR;
+        } else {
+#if PIXMAN_VERSION >= PIXMAN_VERSION_ENCODE(0, 21, 8)
+            type = PIXMAN_TYPE_BGRA;
+#endif
+        }
+    }
+    return type;
+}
+
+pixman_format_code_t qemu_pixman_get_format(PixelFormat *pf)
+{
+    pixman_format_code_t format;
+    int type;
+
+    type = qemu_pixman_get_type(pf->rshift, pf->gshift, pf->bshift);
+    format = PIXMAN_FORMAT(pf->bits_per_pixel, type,
+                           pf->abits, pf->rbits, pf->gbits, pf->bbits);
+    if (!pixman_format_supported_source(format)) {
+        return 0;
+    }
+    return format;
+}
+
+pixman_image_t *qemu_pixman_linebuf_create(pixman_format_code_t format,
+                                           int width)
+{
+    pixman_image_t *image = pixman_image_create_bits(format, width, 1, NULL, 0);
+    assert(image != NULL);
+    return image;
+}
+
+void qemu_pixman_linebuf_fill(pixman_image_t *linebuf, pixman_image_t *fb,
+                              int width, int y)
+{
+    pixman_image_composite(PIXMAN_OP_SRC, fb, NULL, linebuf,
+                           0, y, 0, 0, 0, 0, width, 1);
+}
+
+pixman_image_t *qemu_pixman_mirror_create(pixman_format_code_t format,
+                                          pixman_image_t *image)
+{
+    pixman_image_t *mirror;
+
+    mirror = pixman_image_create_bits(format,
+                                      pixman_image_get_width(image),
+                                      pixman_image_get_height(image),
+                                      NULL,
+                                      pixman_image_get_stride(image));
+    return mirror;
+}
+
+void qemu_pixman_image_unref(pixman_image_t *image)
+{
+    if (image == NULL) {
+        return;
+    }
+    pixman_image_unref(image);
+}
diff --git a/qemu-pixman.h b/qemu-pixman.h
new file mode 100644
index 000000000..bee55eb7d
--- /dev/null
+++ b/qemu-pixman.h
@@ -0,0 +1,39 @@
+/*
+ * This work is licensed under the terms of the GNU GPL, version 2 or later.
+ * See the COPYING file in the top-level directory.
+ */
+
+#ifndef QEMU_PIXMAN_H
+#define QEMU_PIXMAN_H
+
+#include <pixman.h>
+
+#include "console.h"
+
+/*
+ * pixman image formats are defined to be native endian,
+ * that means host byte order on qemu.  So we go define
+ * fixed formats here for cases where it is needed, like
+ * feeding libjpeg / libpng and writing screenshots.
+ */
+
+#ifdef HOST_WORDS_BIGENDIAN
+# define PIXMAN_BE_r8g8b8     PIXMAN_r8g8b8
+#else
+# define PIXMAN_BE_r8g8b8     PIXMAN_b8g8r8
+#endif
+
+/* -------------------------------------------------------------------- */
+
+int qemu_pixman_get_type(int rshift, int gshift, int bshift);
+pixman_format_code_t qemu_pixman_get_format(PixelFormat *pf);
+
+pixman_image_t *qemu_pixman_linebuf_create(pixman_format_code_t format,
+                                           int width);
+void qemu_pixman_linebuf_fill(pixman_image_t *linebuf, pixman_image_t *fb,
+                              int width, int y);
+pixman_image_t *qemu_pixman_mirror_create(pixman_format_code_t format,
+                                          pixman_image_t *image);
+void qemu_pixman_image_unref(pixman_image_t *image);
+
+#endif /* QEMU_PIXMAN_H */
diff --git a/qemu-seccomp.c b/qemu-seccomp.c
index 64329a3c0..2a71d6fee 100644
--- a/qemu-seccomp.c
+++ b/qemu-seccomp.c
@@ -26,8 +26,12 @@ static const struct QemuSeccompSyscall seccomp_whitelist[] = {
     { SCMP_SYS(timer_gettime), 254 },
     { SCMP_SYS(futex), 253 },
     { SCMP_SYS(select), 252 },
+#if defined(__x86_64__)
     { SCMP_SYS(recvfrom), 251 },
     { SCMP_SYS(sendto), 250 },
+#elif defined(__i386__)
+    { SCMP_SYS(socketcall), 250 },
+#endif
     { SCMP_SYS(read), 249 },
     { SCMP_SYS(brk), 248 },
     { SCMP_SYS(clone), 247 },
@@ -36,15 +40,30 @@ static const struct QemuSeccompSyscall seccomp_whitelist[] = {
     { SCMP_SYS(execve), 245 },
     { SCMP_SYS(open), 245 },
     { SCMP_SYS(ioctl), 245 },
+#if defined(__x86_64__)
+    { SCMP_SYS(socket), 245 },
+    { SCMP_SYS(setsockopt), 245 },
     { SCMP_SYS(recvmsg), 245 },
     { SCMP_SYS(sendmsg), 245 },
     { SCMP_SYS(accept), 245 },
     { SCMP_SYS(connect), 245 },
+    { SCMP_SYS(socketpair), 245 },
+    { SCMP_SYS(bind), 245 },
+    { SCMP_SYS(listen), 245 },
+    { SCMP_SYS(semget), 245 },
+#elif defined(__i386__)
+    { SCMP_SYS(ipc), 245 },
+#endif
     { SCMP_SYS(gettimeofday), 245 },
     { SCMP_SYS(readlink), 245 },
     { SCMP_SYS(access), 245 },
     { SCMP_SYS(prctl), 245 },
     { SCMP_SYS(signalfd), 245 },
+    { SCMP_SYS(getrlimit), 245 },
+    { SCMP_SYS(set_tid_address), 245 },
+    { SCMP_SYS(statfs), 245 },
+    { SCMP_SYS(unlink), 245 },
+    { SCMP_SYS(wait4), 245 },
 #if defined(__i386__)
     { SCMP_SYS(fcntl64), 245 },
     { SCMP_SYS(fstat64), 245 },
@@ -56,30 +75,33 @@ static const struct QemuSeccompSyscall seccomp_whitelist[] = {
     { SCMP_SYS(sigreturn), 245 },
     { SCMP_SYS(_newselect), 245 },
     { SCMP_SYS(_llseek), 245 },
-    { SCMP_SYS(mmap2), 245},
+    { SCMP_SYS(mmap2), 245 },
     { SCMP_SYS(sigprocmask), 245 },
-#elif defined(__x86_64__)
-    { SCMP_SYS(sched_getparam), 245},
-    { SCMP_SYS(sched_getscheduler), 245},
-    { SCMP_SYS(fstat), 245},
-    { SCMP_SYS(clock_getres), 245},
-    { SCMP_SYS(sched_get_priority_min), 245},
-    { SCMP_SYS(sched_get_priority_max), 245},
-    { SCMP_SYS(stat), 245},
-    { SCMP_SYS(socket), 245},
-    { SCMP_SYS(setsockopt), 245},
-    { SCMP_SYS(uname), 245},
-    { SCMP_SYS(semget), 245},
 #endif
+    { SCMP_SYS(sched_getparam), 245 },
+    { SCMP_SYS(sched_getscheduler), 245 },
+    { SCMP_SYS(fstat), 245 },
+    { SCMP_SYS(clock_getres), 245 },
+    { SCMP_SYS(sched_get_priority_min), 245 },
+    { SCMP_SYS(sched_get_priority_max), 245 },
+    { SCMP_SYS(stat), 245 },
+    { SCMP_SYS(uname), 245 },
     { SCMP_SYS(eventfd2), 245 },
     { SCMP_SYS(dup), 245 },
+    { SCMP_SYS(dup2), 245 },
+    { SCMP_SYS(dup3), 245 },
     { SCMP_SYS(gettid), 245 },
+    { SCMP_SYS(getgid), 245 },
+    { SCMP_SYS(getegid), 245 },
+    { SCMP_SYS(getuid), 245 },
+    { SCMP_SYS(geteuid), 245 },
     { SCMP_SYS(timer_create), 245 },
     { SCMP_SYS(exit), 245 },
     { SCMP_SYS(clock_gettime), 245 },
     { SCMP_SYS(time), 245 },
     { SCMP_SYS(restart_syscall), 245 },
     { SCMP_SYS(pwrite64), 245 },
+    { SCMP_SYS(nanosleep), 245 },
     { SCMP_SYS(chown), 245 },
     { SCMP_SYS(openat), 245 },
     { SCMP_SYS(getdents), 245 },
@@ -93,8 +115,6 @@ static const struct QemuSeccompSyscall seccomp_whitelist[] = {
     { SCMP_SYS(lseek), 245 },
     { SCMP_SYS(pselect6), 245 },
     { SCMP_SYS(fork), 245 },
-    { SCMP_SYS(bind), 245 },
-    { SCMP_SYS(listen), 245 },
     { SCMP_SYS(eventfd), 245 },
     { SCMP_SYS(rt_sigprocmask), 245 },
     { SCMP_SYS(write), 244 },
@@ -104,10 +124,112 @@ static const struct QemuSeccompSyscall seccomp_whitelist[] = {
     { SCMP_SYS(pipe2), 242 },
     { SCMP_SYS(munmap), 242 },
     { SCMP_SYS(mremap), 242 },
+    { SCMP_SYS(fdatasync), 242 },
+    { SCMP_SYS(close), 242 },
+    { SCMP_SYS(rt_sigpending), 242 },
+    { SCMP_SYS(rt_sigtimedwait), 242 },
+    { SCMP_SYS(readv), 242 },
+    { SCMP_SYS(writev), 242 },
+    { SCMP_SYS(preadv), 242 },
+    { SCMP_SYS(pwritev), 242 },
+    { SCMP_SYS(setrlimit), 242 },
+    { SCMP_SYS(ftruncate), 242 },
+    { SCMP_SYS(lstat), 242 },
+    { SCMP_SYS(pipe), 242 },
+    { SCMP_SYS(umask), 242 },
+    { SCMP_SYS(chdir), 242 },
+    { SCMP_SYS(setitimer), 242 },
+    { SCMP_SYS(setsid), 242 },
+    { SCMP_SYS(poll), 242 },
+    { SCMP_SYS(epoll_create), 242 },
+    { SCMP_SYS(epoll_ctl), 242 },
+    { SCMP_SYS(epoll_wait), 242 },
+#if defined(__i386__)
+    { SCMP_SYS(waitpid), 242 },
+#elif defined(__x86_64__)
     { SCMP_SYS(getsockname), 242 },
     { SCMP_SYS(getpeername), 242 },
-    { SCMP_SYS(fdatasync), 242 },
-    { SCMP_SYS(close), 242 }
+    { SCMP_SYS(accept4), 242 },
+    { SCMP_SYS(newfstatat), 241 },
+    { SCMP_SYS(shutdown), 241 },
+    { SCMP_SYS(getsockopt), 241 },
+    { SCMP_SYS(semctl), 241 },
+    { SCMP_SYS(semop), 241 },
+    { SCMP_SYS(semtimedop), 241 },
+    { SCMP_SYS(epoll_ctl_old), 241 },
+    { SCMP_SYS(epoll_wait_old), 241 },
+#endif
+    { SCMP_SYS(epoll_pwait), 241 },
+    { SCMP_SYS(epoll_create1), 241 },
+    { SCMP_SYS(ppoll), 241 },
+    { SCMP_SYS(creat), 241 },
+    { SCMP_SYS(link), 241 },
+    { SCMP_SYS(getpid), 241 },
+    { SCMP_SYS(getppid), 241 },
+    { SCMP_SYS(getpgrp), 241 },
+    { SCMP_SYS(getpgid), 241 },
+    { SCMP_SYS(getsid), 241 },
+    { SCMP_SYS(getdents64), 241 },
+    { SCMP_SYS(getresuid), 241 },
+    { SCMP_SYS(getresgid), 241 },
+    { SCMP_SYS(getgroups), 241 },
+#if defined(__i386__)
+    { SCMP_SYS(getresuid32), 241 },
+    { SCMP_SYS(getresgid32), 241 },
+    { SCMP_SYS(getgroups32), 241 },
+    { SCMP_SYS(signal), 241 },
+    { SCMP_SYS(sigaction), 241 },
+    { SCMP_SYS(sigsuspend), 241 },
+    { SCMP_SYS(sigpending), 241 },
+    { SCMP_SYS(truncate64), 241 },
+    { SCMP_SYS(ftruncate64), 241 },
+    { SCMP_SYS(fchown32), 241 },
+    { SCMP_SYS(chown32), 241 },
+    { SCMP_SYS(lchown32), 241 },
+    { SCMP_SYS(statfs64), 241 },
+    { SCMP_SYS(fstatfs64), 241 },
+    { SCMP_SYS(fstatat64), 241 },
+    { SCMP_SYS(lstat64), 241 },
+    { SCMP_SYS(sendfile64), 241 },
+    { SCMP_SYS(ugetrlimit), 241 },
+#endif
+    { SCMP_SYS(alarm), 241 },
+    { SCMP_SYS(rt_sigsuspend), 241 },
+    { SCMP_SYS(rt_sigqueueinfo), 241 },
+    { SCMP_SYS(rt_tgsigqueueinfo), 241 },
+    { SCMP_SYS(sigaltstack), 241 },
+    { SCMP_SYS(signalfd4), 241 },
+    { SCMP_SYS(truncate), 241 },
+    { SCMP_SYS(fchown), 241 },
+    { SCMP_SYS(lchown), 241 },
+    { SCMP_SYS(fchownat), 241 },
+    { SCMP_SYS(fstatfs), 241 },
+    { SCMP_SYS(sendfile), 241 },
+    { SCMP_SYS(getitimer), 241 },
+    { SCMP_SYS(syncfs), 241 },
+    { SCMP_SYS(fsync), 241 },
+    { SCMP_SYS(fchdir), 241 },
+    { SCMP_SYS(flock), 241 },
+    { SCMP_SYS(msync), 241 },
+    { SCMP_SYS(sched_setparam), 241 },
+    { SCMP_SYS(sched_setscheduler), 241 },
+    { SCMP_SYS(sched_yield), 241 },
+    { SCMP_SYS(sched_rr_get_interval), 241 },
+    { SCMP_SYS(sched_setaffinity), 241 },
+    { SCMP_SYS(sched_getaffinity), 241 },
+    { SCMP_SYS(readahead), 241 },
+    { SCMP_SYS(timer_getoverrun), 241 },
+    { SCMP_SYS(unlinkat), 241 },
+    { SCMP_SYS(readlinkat), 241 },
+    { SCMP_SYS(faccessat), 241 },
+    { SCMP_SYS(get_robust_list), 241 },
+    { SCMP_SYS(splice), 241 },
+    { SCMP_SYS(vmsplice), 241 },
+    { SCMP_SYS(getcpu), 241 },
+    { SCMP_SYS(sendmmsg), 241 },
+    { SCMP_SYS(recvmmsg), 241 },
+    { SCMP_SYS(prlimit64), 241 },
+    { SCMP_SYS(waitid), 241 }
 };
 
 int seccomp_start(void)
diff --git a/qemu-sockets.c b/qemu-sockets.c
index 361d890da..d314cf1d1 100644
--- a/qemu-sockets.c
+++ b/qemu-sockets.c
@@ -22,8 +22,10 @@
 #include <errno.h>
 #include <unistd.h>
 
+#include "monitor.h"
 #include "qemu_socket.h"
 #include "qemu-common.h" /* for qemu_isdigit */
+#include "main-loop.h"
 
 #ifndef AI_ADDRCONFIG
 # define AI_ADDRCONFIG 0
@@ -54,9 +56,6 @@ static QemuOptsList dummy_opts = {
         },{
             .name = "ipv6",
             .type = QEMU_OPT_BOOL,
-        },{
-            .name = "block",
-            .type = QEMU_OPT_BOOL,
         },
         { /* end if list */ }
     },
@@ -122,8 +121,7 @@ int inet_listen_opts(QemuOpts *opts, int port_offset, Error **errp)
 
     if ((qemu_opt_get(opts, "host") == NULL) ||
         (qemu_opt_get(opts, "port") == NULL)) {
-        fprintf(stderr, "%s: host and/or port not specified\n", __FUNCTION__);
-        error_set(errp, QERR_SOCKET_CREATE_FAILED);
+        error_setg(errp, "host and/or port not specified");
         return -1;
     }
     pstrcpy(port, sizeof(port), qemu_opt_get(opts, "port"));
@@ -140,9 +138,8 @@ int inet_listen_opts(QemuOpts *opts, int port_offset, Error **errp)
         snprintf(port, sizeof(port), "%d", atoi(port) + port_offset);
     rc = getaddrinfo(strlen(addr) ? addr : NULL, port, &ai, &res);
     if (rc != 0) {
-        fprintf(stderr,"getaddrinfo(%s,%s): %s\n", addr, port,
-                gai_strerror(rc));
-        error_set(errp, QERR_SOCKET_CREATE_FAILED);
+        error_setg(errp, "address resolution failed for %s:%s: %s", addr, port,
+                   gai_strerror(rc));
         return -1;
     }
 
@@ -153,10 +150,8 @@ int inet_listen_opts(QemuOpts *opts, int port_offset, Error **errp)
 		        NI_NUMERICHOST | NI_NUMERICSERV);
         slisten = qemu_socket(e->ai_family, e->ai_socktype, e->ai_protocol);
         if (slisten < 0) {
-            fprintf(stderr,"%s: socket(%s): %s\n", __FUNCTION__,
-                    inet_strfamily(e->ai_family), strerror(errno));
             if (!e->ai_next) {
-                error_set(errp, QERR_SOCKET_CREATE_FAILED);
+                error_set_errno(errp, errno, QERR_SOCKET_CREATE_FAILED);
             }
             continue;
         }
@@ -178,24 +173,19 @@ int inet_listen_opts(QemuOpts *opts, int port_offset, Error **errp)
                 goto listen;
             }
             if (p == port_max) {
-                fprintf(stderr,"%s: bind(%s,%s,%d): %s\n", __FUNCTION__,
-                        inet_strfamily(e->ai_family), uaddr, inet_getport(e),
-                        strerror(errno));
                 if (!e->ai_next) {
-                    error_set(errp, QERR_SOCKET_BIND_FAILED);
+                    error_set_errno(errp, errno, QERR_SOCKET_BIND_FAILED);
                 }
             }
         }
         closesocket(slisten);
     }
-    fprintf(stderr, "%s: FAILED\n", __FUNCTION__);
     freeaddrinfo(res);
     return -1;
 
 listen:
     if (listen(slisten,1) != 0) {
-        error_set(errp, QERR_SOCKET_LISTEN_FAILED);
-        perror("listen");
+        error_set_errno(errp, errno, QERR_SOCKET_LISTEN_FAILED);
         closesocket(slisten);
         freeaddrinfo(res);
         return -1;
@@ -209,104 +199,204 @@ listen:
     return slisten;
 }
 
-int inet_connect_opts(QemuOpts *opts, bool *in_progress, Error **errp)
+#ifdef _WIN32
+#define QEMU_SOCKET_RC_INPROGRESS(rc) \
+    ((rc) == -EINPROGRESS || (rc) == -EWOULDBLOCK || (rc) == -WSAEALREADY)
+#else
+#define QEMU_SOCKET_RC_INPROGRESS(rc) \
+    ((rc) == -EINPROGRESS)
+#endif
+
+/* Struct to store connect state for non blocking connect */
+typedef struct ConnectState {
+    int fd;
+    struct addrinfo *addr_list;
+    struct addrinfo *current_addr;
+    NonBlockingConnectHandler *callback;
+    void *opaque;
+} ConnectState;
+
+static int inet_connect_addr(struct addrinfo *addr, bool *in_progress,
+                             ConnectState *connect_state, Error **errp);
+
+static void wait_for_connect(void *opaque)
 {
-    struct addrinfo ai,*res,*e;
+    ConnectState *s = opaque;
+    int val = 0, rc = 0;
+    socklen_t valsize = sizeof(val);
+    bool in_progress;
+
+    qemu_set_fd_handler2(s->fd, NULL, NULL, NULL, NULL);
+
+    do {
+        rc = getsockopt(s->fd, SOL_SOCKET, SO_ERROR, (void *) &val, &valsize);
+    } while (rc == -1 && socket_error() == EINTR);
+
+    /* update rc to contain error */
+    if (!rc && val) {
+        rc = -1;
+    }
+
+    /* connect error */
+    if (rc < 0) {
+        closesocket(s->fd);
+        s->fd = rc;
+    }
+
+    /* try to connect to the next address on the list */
+    if (s->current_addr) {
+        while (s->current_addr->ai_next != NULL && s->fd < 0) {
+            s->current_addr = s->current_addr->ai_next;
+            s->fd = inet_connect_addr(s->current_addr, &in_progress, s, NULL);
+            /* connect in progress */
+            if (in_progress) {
+                return;
+            }
+        }
+
+        freeaddrinfo(s->addr_list);
+    }
+
+    if (s->callback) {
+        s->callback(s->fd, s->opaque);
+    }
+    g_free(s);
+}
+
+static int inet_connect_addr(struct addrinfo *addr, bool *in_progress,
+                             ConnectState *connect_state, Error **errp)
+{
+    int sock, rc;
+
+    *in_progress = false;
+
+    sock = qemu_socket(addr->ai_family, addr->ai_socktype, addr->ai_protocol);
+    if (sock < 0) {
+        error_set_errno(errp, errno, QERR_SOCKET_CREATE_FAILED);
+        return -1;
+    }
+    qemu_setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on));
+    if (connect_state != NULL) {
+        socket_set_nonblock(sock);
+    }
+    /* connect to peer */
+    do {
+        rc = 0;
+        if (connect(sock, addr->ai_addr, addr->ai_addrlen) < 0) {
+            rc = -socket_error();
+        }
+    } while (rc == -EINTR);
+
+    if (connect_state != NULL && QEMU_SOCKET_RC_INPROGRESS(rc)) {
+        connect_state->fd = sock;
+        qemu_set_fd_handler2(sock, NULL, NULL, wait_for_connect,
+                             connect_state);
+        *in_progress = true;
+    } else if (rc < 0) {
+        error_set_errno(errp, errno, QERR_SOCKET_CONNECT_FAILED);
+        closesocket(sock);
+        return -1;
+    }
+    return sock;
+}
+
+static struct addrinfo *inet_parse_connect_opts(QemuOpts *opts, Error **errp)
+{
+    struct addrinfo ai, *res;
+    int rc;
     const char *addr;
     const char *port;
-    char uaddr[INET6_ADDRSTRLEN+1];
-    char uport[33];
-    int sock,rc;
-    bool block;
 
-    memset(&ai,0, sizeof(ai));
+    memset(&ai, 0, sizeof(ai));
+
     ai.ai_flags = AI_CANONNAME | AI_ADDRCONFIG;
     ai.ai_family = PF_UNSPEC;
     ai.ai_socktype = SOCK_STREAM;
 
-    if (in_progress) {
-        *in_progress = false;
-    }
-
     addr = qemu_opt_get(opts, "host");
     port = qemu_opt_get(opts, "port");
-    block = qemu_opt_get_bool(opts, "block", 0);
     if (addr == NULL || port == NULL) {
-        fprintf(stderr, "inet_connect: host and/or port not specified\n");
-        error_set(errp, QERR_SOCKET_CREATE_FAILED);
-        return -1;
+        error_setg(errp, "host and/or port not specified");
+        return NULL;
     }
 
-    if (qemu_opt_get_bool(opts, "ipv4", 0))
+    if (qemu_opt_get_bool(opts, "ipv4", 0)) {
         ai.ai_family = PF_INET;
-    if (qemu_opt_get_bool(opts, "ipv6", 0))
+    }
+    if (qemu_opt_get_bool(opts, "ipv6", 0)) {
         ai.ai_family = PF_INET6;
+    }
 
     /* lookup */
-    if (0 != (rc = getaddrinfo(addr, port, &ai, &res))) {
-        fprintf(stderr,"getaddrinfo(%s,%s): %s\n", addr, port,
-                gai_strerror(rc));
-        error_set(errp, QERR_SOCKET_CREATE_FAILED);
-	return -1;
+    rc = getaddrinfo(addr, port, &ai, &res);
+    if (rc != 0) {
+        error_setg(errp, "address resolution failed for %s:%s: %s", addr, port,
+                   gai_strerror(rc));
+        return NULL;
+    }
+    return res;
+}
+
+/**
+ * Create a socket and connect it to an address.
+ *
+ * @opts: QEMU options, recognized parameters strings "host" and "port",
+ *        bools "ipv4" and "ipv6".
+ * @errp: set on error
+ * @callback: callback function for non-blocking connect
+ * @opaque: opaque for callback function
+ *
+ * Returns: -1 on error, file descriptor on success.
+ *
+ * If @callback is non-null, the connect is non-blocking.  If this
+ * function succeeds, callback will be called when the connection
+ * completes, with the file descriptor on success, or -1 on error.
+ */
+int inet_connect_opts(QemuOpts *opts, Error **errp,
+                      NonBlockingConnectHandler *callback, void *opaque)
+{
+    struct addrinfo *res, *e;
+    int sock = -1;
+    bool in_progress;
+    ConnectState *connect_state = NULL;
+
+    res = inet_parse_connect_opts(opts, errp);
+    if (!res) {
+        return -1;
+    }
+
+    if (callback != NULL) {
+        connect_state = g_malloc0(sizeof(*connect_state));
+        connect_state->addr_list = res;
+        connect_state->callback = callback;
+        connect_state->opaque = opaque;
     }
 
     for (e = res; e != NULL; e = e->ai_next) {
-        if (getnameinfo((struct sockaddr*)e->ai_addr,e->ai_addrlen,
-                            uaddr,INET6_ADDRSTRLEN,uport,32,
-                            NI_NUMERICHOST | NI_NUMERICSERV) != 0) {
-            fprintf(stderr,"%s: getnameinfo: oops\n", __FUNCTION__);
-            continue;
-        }
-        sock = qemu_socket(e->ai_family, e->ai_socktype, e->ai_protocol);
-        if (sock < 0) {
-            fprintf(stderr,"%s: socket(%s): %s\n", __FUNCTION__,
-            inet_strfamily(e->ai_family), strerror(errno));
-            continue;
+        if (connect_state != NULL) {
+            connect_state->current_addr = e;
         }
-        setsockopt(sock,SOL_SOCKET,SO_REUSEADDR,(void*)&on,sizeof(on));
-        if (!block) {
-            socket_set_nonblock(sock);
-        }
-        /* connect to peer */
-        do {
-            rc = 0;
-            if (connect(sock, e->ai_addr, e->ai_addrlen) < 0) {
-                rc = -socket_error();
-            }
-        } while (rc == -EINTR);
-
-  #ifdef _WIN32
-        if (!block && (rc == -EINPROGRESS || rc == -EWOULDBLOCK
-                       || rc == -WSAEALREADY)) {
-  #else
-        if (!block && (rc == -EINPROGRESS)) {
-  #endif
-            if (in_progress) {
-                *in_progress = true;
+        sock = inet_connect_addr(e, &in_progress, connect_state, errp);
+        if (in_progress) {
+            return sock;
+        } else if (sock >= 0) {
+            /* non blocking socket immediate success, call callback */
+            if (callback != NULL) {
+                callback(sock, opaque);
             }
-        } else if (rc < 0) {
-            if (NULL == e->ai_next)
-                fprintf(stderr, "%s: connect(%s,%s,%s,%s): %s\n", __FUNCTION__,
-                        inet_strfamily(e->ai_family),
-                        e->ai_canonname, uaddr, uport, strerror(errno));
-            closesocket(sock);
-            continue;
+            break;
         }
-        freeaddrinfo(res);
-        return sock;
     }
-    error_set(errp, QERR_SOCKET_CONNECT_FAILED);
+    g_free(connect_state);
     freeaddrinfo(res);
-    return -1;
+    return sock;
 }
 
-int inet_dgram_opts(QemuOpts *opts)
+int inet_dgram_opts(QemuOpts *opts, Error **errp)
 {
     struct addrinfo ai, *peer = NULL, *local = NULL;
     const char *addr;
     const char *port;
-    char uaddr[INET6_ADDRSTRLEN+1];
-    char uport[33];
     int sock = -1, rc;
 
     /* lookup peer addr */
@@ -321,7 +411,7 @@ int inet_dgram_opts(QemuOpts *opts)
         addr = "localhost";
     }
     if (port == NULL || strlen(port) == 0) {
-        fprintf(stderr, "inet_dgram: port not specified\n");
+        error_setg(errp, "remote port not specified");
         return -1;
     }
 
@@ -331,8 +421,8 @@ int inet_dgram_opts(QemuOpts *opts)
         ai.ai_family = PF_INET6;
 
     if (0 != (rc = getaddrinfo(addr, port, &ai, &peer))) {
-        fprintf(stderr,"getaddrinfo(%s,%s): %s\n", addr, port,
-                gai_strerror(rc));
+        error_setg(errp, "address resolution failed for %s:%s: %s", addr, port,
+                   gai_strerror(rc));
 	return -1;
     }
 
@@ -351,44 +441,28 @@ int inet_dgram_opts(QemuOpts *opts)
         port = "0";
 
     if (0 != (rc = getaddrinfo(addr, port, &ai, &local))) {
-        fprintf(stderr,"getaddrinfo(%s,%s): %s\n", addr, port,
-                gai_strerror(rc));
-        return -1;
+        error_setg(errp, "address resolution failed for %s:%s: %s", addr, port,
+                   gai_strerror(rc));
+        goto err;
     }
 
     /* create socket */
     sock = qemu_socket(peer->ai_family, peer->ai_socktype, peer->ai_protocol);
     if (sock < 0) {
-        fprintf(stderr,"%s: socket(%s): %s\n", __FUNCTION__,
-                inet_strfamily(peer->ai_family), strerror(errno));
+        error_set_errno(errp, errno, QERR_SOCKET_CREATE_FAILED);
         goto err;
     }
     setsockopt(sock,SOL_SOCKET,SO_REUSEADDR,(void*)&on,sizeof(on));
 
     /* bind socket */
-    if (getnameinfo((struct sockaddr*)local->ai_addr,local->ai_addrlen,
-                    uaddr,INET6_ADDRSTRLEN,uport,32,
-                    NI_NUMERICHOST | NI_NUMERICSERV) != 0) {
-        fprintf(stderr, "%s: getnameinfo: oops\n", __FUNCTION__);
-        goto err;
-    }
     if (bind(sock, local->ai_addr, local->ai_addrlen) < 0) {
-        fprintf(stderr,"%s: bind(%s,%s,%d): OK\n", __FUNCTION__,
-                inet_strfamily(local->ai_family), uaddr, inet_getport(local));
+        error_set_errno(errp, errno, QERR_SOCKET_BIND_FAILED);
         goto err;
     }
 
     /* connect to peer */
-    if (getnameinfo((struct sockaddr*)peer->ai_addr, peer->ai_addrlen,
-                    uaddr, INET6_ADDRSTRLEN, uport, 32,
-                    NI_NUMERICHOST | NI_NUMERICSERV) != 0) {
-        fprintf(stderr, "%s: getnameinfo: oops\n", __FUNCTION__);
-        goto err;
-    }
     if (connect(sock,peer->ai_addr,peer->ai_addrlen) < 0) {
-        fprintf(stderr, "%s: connect(%s,%s,%s,%s): %s\n", __FUNCTION__,
-                inet_strfamily(peer->ai_family),
-                peer->ai_canonname, uaddr, uport, strerror(errno));
+        error_set_errno(errp, errno, QERR_SOCKET_CONNECT_FAILED);
         goto err;
     }
 
@@ -407,59 +481,92 @@ err:
 }
 
 /* compatibility wrapper */
-static int inet_parse(QemuOpts *opts, const char *str)
+static InetSocketAddress *inet_parse(const char *str, Error **errp)
 {
+    InetSocketAddress *addr;
     const char *optstr, *h;
-    char addr[64];
+    char host[64];
     char port[33];
+    int to;
     int pos;
 
+    addr = g_new0(InetSocketAddress, 1);
+
     /* parse address */
     if (str[0] == ':') {
         /* no host given */
-        addr[0] = '\0';
-        if (1 != sscanf(str,":%32[^,]%n",port,&pos)) {
-            fprintf(stderr, "%s: portonly parse error (%s)\n",
-                    __FUNCTION__, str);
-            return -1;
+        host[0] = '\0';
+        if (1 != sscanf(str, ":%32[^,]%n", port, &pos)) {
+            error_setg(errp, "error parsing port in address '%s'", str);
+            goto fail;
         }
     } else if (str[0] == '[') {
         /* IPv6 addr */
-        if (2 != sscanf(str,"[%64[^]]]:%32[^,]%n",addr,port,&pos)) {
-            fprintf(stderr, "%s: ipv6 parse error (%s)\n",
-                    __FUNCTION__, str);
-            return -1;
+        if (2 != sscanf(str, "[%64[^]]]:%32[^,]%n", host, port, &pos)) {
+            error_setg(errp, "error parsing IPv6 address '%s'", str);
+            goto fail;
         }
-        qemu_opt_set(opts, "ipv6", "on");
+        addr->ipv6 = addr->has_ipv6 = true;
     } else if (qemu_isdigit(str[0])) {
         /* IPv4 addr */
-        if (2 != sscanf(str,"%64[0-9.]:%32[^,]%n",addr,port,&pos)) {
-            fprintf(stderr, "%s: ipv4 parse error (%s)\n",
-                    __FUNCTION__, str);
-            return -1;
+        if (2 != sscanf(str, "%64[0-9.]:%32[^,]%n", host, port, &pos)) {
+            error_setg(errp, "error parsing IPv4 address '%s'", str);
+            goto fail;
         }
-        qemu_opt_set(opts, "ipv4", "on");
+        addr->ipv4 = addr->has_ipv4 = true;
     } else {
         /* hostname */
-        if (2 != sscanf(str,"%64[^:]:%32[^,]%n",addr,port,&pos)) {
-            fprintf(stderr, "%s: hostname parse error (%s)\n",
-                    __FUNCTION__, str);
-            return -1;
+        if (2 != sscanf(str, "%64[^:]:%32[^,]%n", host, port, &pos)) {
+            error_setg(errp, "error parsing address '%s'", str);
+            goto fail;
         }
     }
-    qemu_opt_set(opts, "host", addr);
-    qemu_opt_set(opts, "port", port);
+
+    addr->host = g_strdup(host);
+    addr->port = g_strdup(port);
 
     /* parse options */
     optstr = str + pos;
     h = strstr(optstr, ",to=");
-    if (h)
-        qemu_opt_set(opts, "to", h+4);
-    if (strstr(optstr, ",ipv4"))
-        qemu_opt_set(opts, "ipv4", "on");
-    if (strstr(optstr, ",ipv6"))
-        qemu_opt_set(opts, "ipv6", "on");
-    return 0;
+    if (h) {
+        h += 4;
+        if (sscanf(h, "%d%n", &to, &pos) != 1 ||
+            (h[pos] != '\0' && h[pos] != ',')) {
+            error_setg(errp, "error parsing to= argument");
+            goto fail;
+        }
+        addr->has_to = true;
+        addr->to = to;
+    }
+    if (strstr(optstr, ",ipv4")) {
+        addr->ipv4 = addr->has_ipv4 = true;
+    }
+    if (strstr(optstr, ",ipv6")) {
+        addr->ipv6 = addr->has_ipv6 = true;
+    }
+    return addr;
+
+fail:
+    qapi_free_InetSocketAddress(addr);
+    return NULL;
+}
+
+static void inet_addr_to_opts(QemuOpts *opts, InetSocketAddress *addr)
+{
+    bool ipv4 = addr->ipv4 || !addr->has_ipv4;
+    bool ipv6 = addr->ipv6 || !addr->has_ipv6;
+
+    if (!ipv4 || !ipv6) {
+        qemu_opt_set_bool(opts, "ipv4", ipv4);
+        qemu_opt_set_bool(opts, "ipv6", ipv6);
+    }
+    if (addr->has_to) {
+        char to[20];
+        snprintf(to, sizeof(to), "%d", addr->to);
+        qemu_opt_set(opts, "to", to);
+    }
+    qemu_opt_set(opts, "host", addr->host);
+    qemu_opt_set(opts, "port", addr->port);
 }
 
 int inet_listen(const char *str, char *ostr, int olen,
@@ -468,9 +575,13 @@ int inet_listen(const char *str, char *ostr, int olen,
     QemuOpts *opts;
     char *optstr;
     int sock = -1;
+    InetSocketAddress *addr;
 
-    opts = qemu_opts_create(&dummy_opts, NULL, 0, NULL);
-    if (inet_parse(opts, str) == 0) {
+    addr = inet_parse(str, errp);
+    if (addr != NULL) {
+        opts = qemu_opts_create(&dummy_opts, NULL, 0, NULL);
+        inet_addr_to_opts(opts, addr);
+        qapi_free_InetSocketAddress(addr);
         sock = inet_listen_opts(opts, port_offset, errp);
         if (sock != -1 && ostr) {
             optstr = strchr(str, ',');
@@ -486,34 +597,73 @@ int inet_listen(const char *str, char *ostr, int olen,
                          optstr ? optstr : "");
             }
         }
-    } else {
-        error_set(errp, QERR_SOCKET_CREATE_FAILED);
+        qemu_opts_del(opts);
     }
-    qemu_opts_del(opts);
     return sock;
 }
 
-int inet_connect(const char *str, bool block, bool *in_progress, Error **errp)
+/**
+ * Create a blocking socket and connect it to an address.
+ *
+ * @str: address string
+ * @errp: set in case of an error
+ *
+ * Returns -1 in case of error, file descriptor on success
+ **/
+int inet_connect(const char *str, Error **errp)
 {
     QemuOpts *opts;
     int sock = -1;
+    InetSocketAddress *addr;
 
-    opts = qemu_opts_create(&dummy_opts, NULL, 0, NULL);
-    if (inet_parse(opts, str) == 0) {
-        if (block) {
-            qemu_opt_set(opts, "block", "on");
-        }
-        sock = inet_connect_opts(opts, in_progress, errp);
-    } else {
-        error_set(errp, QERR_SOCKET_CREATE_FAILED);
+    addr = inet_parse(str, errp);
+    if (addr != NULL) {
+        opts = qemu_opts_create(&dummy_opts, NULL, 0, NULL);
+        inet_addr_to_opts(opts, addr);
+        qapi_free_InetSocketAddress(addr);
+        sock = inet_connect_opts(opts, errp, NULL, NULL);
+        qemu_opts_del(opts);
+    }
+    return sock;
+}
+
+/**
+ * Create a non-blocking socket and connect it to an address.
+ * Calls the callback function with fd in case of success or -1 in case of
+ * error.
+ *
+ * @str: address string
+ * @callback: callback function that is called when connect completes,
+ *            cannot be NULL.
+ * @opaque: opaque for callback function
+ * @errp: set in case of an error
+ *
+ * Returns: -1 on immediate error, file descriptor on success.
+ **/
+int inet_nonblocking_connect(const char *str,
+                             NonBlockingConnectHandler *callback,
+                             void *opaque, Error **errp)
+{
+    QemuOpts *opts;
+    int sock = -1;
+    InetSocketAddress *addr;
+
+    g_assert(callback != NULL);
+
+    addr = inet_parse(str, errp);
+    if (addr != NULL) {
+        opts = qemu_opts_create(&dummy_opts, NULL, 0, NULL);
+        inet_addr_to_opts(opts, addr);
+        qapi_free_InetSocketAddress(addr);
+        sock = inet_connect_opts(opts, errp, callback, opaque);
+        qemu_opts_del(opts);
     }
-    qemu_opts_del(opts);
     return sock;
 }
 
 #ifndef _WIN32
 
-int unix_listen_opts(QemuOpts *opts)
+int unix_listen_opts(QemuOpts *opts, Error **errp)
 {
     struct sockaddr_un un;
     const char *path = qemu_opt_get(opts, "path");
@@ -521,7 +671,7 @@ int unix_listen_opts(QemuOpts *opts)
 
     sock = qemu_socket(PF_UNIX, SOCK_STREAM, 0);
     if (sock < 0) {
-        perror("socket(unix)");
+        error_set_errno(errp, errno, QERR_SOCKET_CREATE_FAILED);
         return -1;
     }
 
@@ -546,11 +696,11 @@ int unix_listen_opts(QemuOpts *opts)
 
     unlink(un.sun_path);
     if (bind(sock, (struct sockaddr*) &un, sizeof(un)) < 0) {
-        fprintf(stderr, "bind(unix:%s): %s\n", un.sun_path, strerror(errno));
+        error_set_errno(errp, errno, QERR_SOCKET_BIND_FAILED);
         goto err;
     }
     if (listen(sock, 1) < 0) {
-        fprintf(stderr, "listen(unix:%s): %s\n", un.sun_path, strerror(errno));
+        error_set_errno(errp, errno, QERR_SOCKET_LISTEN_FAILED);
         goto err;
     }
 
@@ -561,37 +711,85 @@ err:
     return -1;
 }
 
-int unix_connect_opts(QemuOpts *opts)
+int unix_connect_opts(QemuOpts *opts, Error **errp,
+                      NonBlockingConnectHandler *callback, void *opaque)
 {
     struct sockaddr_un un;
     const char *path = qemu_opt_get(opts, "path");
-    int sock;
+    ConnectState *connect_state = NULL;
+    int sock, rc;
 
     if (NULL == path) {
-        fprintf(stderr, "unix connect: no path specified\n");
+        error_setg(errp, "unix connect: no path specified\n");
         return -1;
     }
 
     sock = qemu_socket(PF_UNIX, SOCK_STREAM, 0);
     if (sock < 0) {
-        perror("socket(unix)");
+        error_set_errno(errp, errno, QERR_SOCKET_CREATE_FAILED);
         return -1;
     }
+    if (callback != NULL) {
+        connect_state = g_malloc0(sizeof(*connect_state));
+        connect_state->callback = callback;
+        connect_state->opaque = opaque;
+        socket_set_nonblock(sock);
+    }
 
     memset(&un, 0, sizeof(un));
     un.sun_family = AF_UNIX;
     snprintf(un.sun_path, sizeof(un.sun_path), "%s", path);
-    if (connect(sock, (struct sockaddr*) &un, sizeof(un)) < 0) {
-        fprintf(stderr, "connect(unix:%s): %s\n", path, strerror(errno));
+
+    /* connect to peer */
+    do {
+        rc = 0;
+        if (connect(sock, (struct sockaddr *) &un, sizeof(un)) < 0) {
+            rc = -socket_error();
+        }
+    } while (rc == -EINTR);
+
+    if (connect_state != NULL && QEMU_SOCKET_RC_INPROGRESS(rc)) {
+        connect_state->fd = sock;
+        qemu_set_fd_handler2(sock, NULL, NULL, wait_for_connect,
+                             connect_state);
+        return sock;
+    } else if (rc >= 0) {
+        /* non blocking socket immediate success, call callback */
+        if (callback != NULL) {
+            callback(sock, opaque);
+        }
+    }
+
+    if (rc < 0) {
+        error_set_errno(errp, -rc, QERR_SOCKET_CONNECT_FAILED);
         close(sock);
-	return -1;
+        sock = -1;
     }
 
+    g_free(connect_state);
     return sock;
 }
 
+#else
+
+int unix_listen_opts(QemuOpts *opts, Error **errp)
+{
+    error_setg(errp, "unix sockets are not available on windows");
+    errno = ENOTSUP;
+    return -1;
+}
+
+int unix_connect_opts(QemuOpts *opts, Error **errp,
+                      NonBlockingConnectHandler *callback, void *opaque)
+{
+    error_setg(errp, "unix sockets are not available on windows");
+    errno = ENOTSUP;
+    return -1;
+}
+#endif
+
 /* compatibility wrapper */
-int unix_listen(const char *str, char *ostr, int olen)
+int unix_listen(const char *str, char *ostr, int olen, Error **errp)
 {
     QemuOpts *opts;
     char *path, *optstr;
@@ -612,7 +810,7 @@ int unix_listen(const char *str, char *ostr, int olen)
         qemu_opt_set(opts, "path", str);
     }
 
-    sock = unix_listen_opts(opts);
+    sock = unix_listen_opts(opts, errp);
 
     if (sock != -1 && ostr)
         snprintf(ostr, olen, "%s%s", qemu_opt_get(opts, "path"), optstr ? optstr : "");
@@ -620,49 +818,132 @@ int unix_listen(const char *str, char *ostr, int olen)
     return sock;
 }
 
-int unix_connect(const char *path)
+int unix_connect(const char *path, Error **errp)
 {
     QemuOpts *opts;
     int sock;
 
     opts = qemu_opts_create(&dummy_opts, NULL, 0, NULL);
     qemu_opt_set(opts, "path", path);
-    sock = unix_connect_opts(opts);
+    sock = unix_connect_opts(opts, errp, NULL, NULL);
     qemu_opts_del(opts);
     return sock;
 }
 
-#else
 
-int unix_listen_opts(QemuOpts *opts)
+int unix_nonblocking_connect(const char *path,
+                             NonBlockingConnectHandler *callback,
+                             void *opaque, Error **errp)
 {
-    fprintf(stderr, "unix sockets are not available on windows\n");
-    errno = ENOTSUP;
-    return -1;
+    QemuOpts *opts;
+    int sock = -1;
+
+    g_assert(callback != NULL);
+
+    opts = qemu_opts_create(&dummy_opts, NULL, 0, NULL);
+    qemu_opt_set(opts, "path", path);
+    sock = unix_connect_opts(opts, errp, callback, opaque);
+    qemu_opts_del(opts);
+    return sock;
 }
 
-int unix_connect_opts(QemuOpts *opts)
+SocketAddress *socket_parse(const char *str, Error **errp)
 {
-    fprintf(stderr, "unix sockets are not available on windows\n");
-    errno = ENOTSUP;
-    return -1;
+    SocketAddress *addr = NULL;
+
+    addr = g_new(SocketAddress, 1);
+    if (strstart(str, "unix:", NULL)) {
+        if (str[5] == '\0') {
+            error_setg(errp, "invalid Unix socket address\n");
+            goto fail;
+        } else {
+            addr->kind = SOCKET_ADDRESS_KIND_UNIX;
+            addr->q_unix = g_new(UnixSocketAddress, 1);
+            addr->q_unix->path = g_strdup(str + 5);
+        }
+    } else if (strstart(str, "fd:", NULL)) {
+        if (str[3] == '\0') {
+            error_setg(errp, "invalid file descriptor address\n");
+            goto fail;
+        } else {
+            addr->kind = SOCKET_ADDRESS_KIND_FD;
+            addr->fd = g_new(String, 1);
+            addr->fd->str = g_strdup(str + 3);
+        }
+    } else {
+        addr->kind = SOCKET_ADDRESS_KIND_INET;
+        addr->inet = g_new(InetSocketAddress, 1);
+        addr->inet = inet_parse(str, errp);
+        if (addr->inet == NULL) {
+            goto fail;
+        }
+    }
+    return addr;
+
+fail:
+    qapi_free_SocketAddress(addr);
+    return NULL;
 }
 
-int unix_listen(const char *path, char *ostr, int olen)
+int socket_connect(SocketAddress *addr, Error **errp,
+                   NonBlockingConnectHandler *callback, void *opaque)
 {
-    fprintf(stderr, "unix sockets are not available on windows\n");
-    errno = ENOTSUP;
-    return -1;
+    QemuOpts *opts;
+    int fd;
+
+    opts = qemu_opts_create(&dummy_opts, NULL, 0, NULL);
+    switch (addr->kind) {
+    case SOCKET_ADDRESS_KIND_INET:
+        inet_addr_to_opts(opts, addr->inet);
+        fd = inet_connect_opts(opts, errp, callback, opaque);
+        break;
+
+    case SOCKET_ADDRESS_KIND_UNIX:
+        qemu_opt_set(opts, "path", addr->q_unix->path);
+        fd = unix_connect_opts(opts, errp, callback, opaque);
+        break;
+
+    case SOCKET_ADDRESS_KIND_FD:
+        fd = monitor_get_fd(cur_mon, addr->fd->str, errp);
+        if (callback) {
+            callback(fd, opaque);
+        }
+        break;
+
+    default:
+        abort();
+    }
+    qemu_opts_del(opts);
+    return fd;
 }
 
-int unix_connect(const char *path)
+int socket_listen(SocketAddress *addr, Error **errp)
 {
-    fprintf(stderr, "unix sockets are not available on windows\n");
-    errno = ENOTSUP;
-    return -1;
-}
+    QemuOpts *opts;
+    int fd;
 
-#endif
+    opts = qemu_opts_create(&dummy_opts, NULL, 0, NULL);
+    switch (addr->kind) {
+    case SOCKET_ADDRESS_KIND_INET:
+        inet_addr_to_opts(opts, addr->inet);
+        fd = inet_listen_opts(opts, 0, errp);
+        break;
+
+    case SOCKET_ADDRESS_KIND_UNIX:
+        qemu_opt_set(opts, "path", addr->q_unix->path);
+        fd = unix_listen_opts(opts, errp);
+        break;
+
+    case SOCKET_ADDRESS_KIND_FD:
+        fd = monitor_get_fd(cur_mon, addr->fd->str, errp);
+        break;
+
+    default:
+        abort();
+    }
+    qemu_opts_del(opts);
+    return fd;
+}
 
 #ifdef _WIN32
 static void socket_cleanup(void)
diff --git a/qemu-tech.texi b/qemu-tech.texi
index d73dda8e3..8aefa743a 100644
--- a/qemu-tech.texi
+++ b/qemu-tech.texi
@@ -262,16 +262,16 @@ Current QEMU limitations:
 
 @item Core Xtensa ISA emulation, including most options: code density,
 loop, extended L32R, 16- and 32-bit multiplication, 32-bit division,
-MAC16, miscellaneous operations, boolean, multiprocessor synchronization,
+MAC16, miscellaneous operations, boolean, FP coprocessor, coprocessor
+context, debug, multiprocessor synchronization,
 conditional store, exceptions, relocatable vectors, unaligned exception,
 interrupts (including high priority and timer), hardware alignment,
 region protection, region translation, MMU, windowed registers, thread
 pointer, processor ID.
 
-@item Not implemented options: FP coprocessor, coprocessor context,
-data/instruction cache (including cache prefetch and locking), XLMI,
-processor interface, debug. Also options not covered by the core ISA
-(e.g. FLIX, wide branches) are not implemented.
+@item Not implemented options: data/instruction cache (including cache
+prefetch and locking), XLMI, processor interface. Also options not
+covered by the core ISA (e.g. FLIX, wide branches) are not implemented.
 
 @item Can run most Xtensa Linux binaries.
 
diff --git a/qemu-thread-posix.c b/qemu-thread-posix.c
index 8fbabdac3..4ef9c7b3f 100644
--- a/qemu-thread-posix.c
+++ b/qemu-thread-posix.c
@@ -17,6 +17,9 @@
 #include <signal.h>
 #include <stdint.h>
 #include <string.h>
+#include <limits.h>
+#include <unistd.h>
+#include <sys/time.h>
 #include "qemu-thread.h"
 
 static void error_exit(int err, const char *msg)
@@ -115,6 +118,155 @@ void qemu_cond_wait(QemuCond *cond, QemuMutex *mutex)
         error_exit(err, __func__);
 }
 
+void qemu_sem_init(QemuSemaphore *sem, int init)
+{
+    int rc;
+
+#if defined(__OpenBSD__) || defined(__APPLE__) || defined(__NetBSD__)
+    rc = pthread_mutex_init(&sem->lock, NULL);
+    if (rc != 0) {
+        error_exit(rc, __func__);
+    }
+    rc = pthread_cond_init(&sem->cond, NULL);
+    if (rc != 0) {
+        error_exit(rc, __func__);
+    }
+    if (init < 0) {
+        error_exit(EINVAL, __func__);
+    }
+    sem->count = init;
+#else
+    rc = sem_init(&sem->sem, 0, init);
+    if (rc < 0) {
+        error_exit(errno, __func__);
+    }
+#endif
+}
+
+void qemu_sem_destroy(QemuSemaphore *sem)
+{
+    int rc;
+
+#if defined(__OpenBSD__) || defined(__APPLE__) || defined(__NetBSD__)
+    rc = pthread_cond_destroy(&sem->cond);
+    if (rc < 0) {
+        error_exit(rc, __func__);
+    }
+    rc = pthread_mutex_destroy(&sem->lock);
+    if (rc < 0) {
+        error_exit(rc, __func__);
+    }
+#else
+    rc = sem_destroy(&sem->sem);
+    if (rc < 0) {
+        error_exit(errno, __func__);
+    }
+#endif
+}
+
+void qemu_sem_post(QemuSemaphore *sem)
+{
+    int rc;
+
+#if defined(__OpenBSD__) || defined(__APPLE__) || defined(__NetBSD__)
+    pthread_mutex_lock(&sem->lock);
+    if (sem->count == INT_MAX) {
+        rc = EINVAL;
+    } else if (sem->count++ < 0) {
+        rc = pthread_cond_signal(&sem->cond);
+    } else {
+        rc = 0;
+    }
+    pthread_mutex_unlock(&sem->lock);
+    if (rc != 0) {
+        error_exit(rc, __func__);
+    }
+#else
+    rc = sem_post(&sem->sem);
+    if (rc < 0) {
+        error_exit(errno, __func__);
+    }
+#endif
+}
+
+static void compute_abs_deadline(struct timespec *ts, int ms)
+{
+    struct timeval tv;
+    gettimeofday(&tv, NULL);
+    ts->tv_nsec = tv.tv_usec * 1000 + (ms % 1000) * 1000000;
+    ts->tv_sec = tv.tv_sec + ms / 1000;
+    if (ts->tv_nsec >= 1000000000) {
+        ts->tv_sec++;
+        ts->tv_nsec -= 1000000000;
+    }
+}
+
+int qemu_sem_timedwait(QemuSemaphore *sem, int ms)
+{
+    int rc;
+    struct timespec ts;
+
+#if defined(__OpenBSD__) || defined(__APPLE__) || defined(__NetBSD__)
+    compute_abs_deadline(&ts, ms);
+    pthread_mutex_lock(&sem->lock);
+    --sem->count;
+    while (sem->count < 0) {
+        rc = pthread_cond_timedwait(&sem->cond, &sem->lock, &ts);
+        if (rc == ETIMEDOUT) {
+            break;
+        }
+        if (rc != 0) {
+            error_exit(rc, __func__);
+        }
+    }
+    pthread_mutex_unlock(&sem->lock);
+    return (rc == ETIMEDOUT ? -1 : 0);
+#else
+    if (ms <= 0) {
+        /* This is cheaper than sem_timedwait.  */
+        do {
+            rc = sem_trywait(&sem->sem);
+        } while (rc == -1 && errno == EINTR);
+        if (rc == -1 && errno == EAGAIN) {
+            return -1;
+        }
+    } else {
+        compute_abs_deadline(&ts, ms);
+        do {
+            rc = sem_timedwait(&sem->sem, &ts);
+        } while (rc == -1 && errno == EINTR);
+        if (rc == -1 && errno == ETIMEDOUT) {
+            return -1;
+        }
+    }
+    if (rc < 0) {
+        error_exit(errno, __func__);
+    }
+    return 0;
+#endif
+}
+
+void qemu_sem_wait(QemuSemaphore *sem)
+{
+#if defined(__OpenBSD__) || defined(__APPLE__) || defined(__NetBSD__)
+    pthread_mutex_lock(&sem->lock);
+    --sem->count;
+    while (sem->count < 0) {
+        pthread_cond_wait(&sem->cond, &sem->lock);
+    }
+    pthread_mutex_unlock(&sem->lock);
+#else
+    int rc;
+
+    do {
+        rc = sem_wait(&sem->sem);
+    } while (rc == -1 && errno == EINTR);
+    if (rc < 0) {
+        error_exit(errno, __func__);
+    }
+#endif
+}
+
 void qemu_thread_create(QemuThread *thread,
                        void *(*start_routine)(void*),
                        void *arg, int mode)
diff --git a/qemu-thread-posix.h b/qemu-thread-posix.h
index ee4618e62..380bae209 100644
--- a/qemu-thread-posix.h
+++ b/qemu-thread-posix.h
@@ -1,6 +1,7 @@
 #ifndef __QEMU_THREAD_POSIX_H
 #define __QEMU_THREAD_POSIX_H 1
 #include "pthread.h"
+#include <semaphore.h>
 
 struct QemuMutex {
     pthread_mutex_t lock;
@@ -10,6 +11,16 @@ struct QemuCond {
     pthread_cond_t cond;
 };
 
+struct QemuSemaphore {
+#if defined(__OpenBSD__) || defined(__APPLE__) || defined(__NetBSD__)
+    pthread_mutex_t lock;
+    pthread_cond_t cond;
+    int count;
+#else
+    sem_t sem;
+#endif
+};
+
 struct QemuThread {
     pthread_t thread;
 };
diff --git a/qemu-thread-win32.c b/qemu-thread-win32.c
index 177b398cc..4b3db60f5 100644
--- a/qemu-thread-win32.c
+++ b/qemu-thread-win32.c
@@ -192,6 +192,41 @@ void qemu_cond_wait(QemuCond *cond, QemuMutex *mutex)
     qemu_mutex_lock(mutex);
 }
 
+void qemu_sem_init(QemuSemaphore *sem, int init)
+{
+    /* Manual reset.  */
+    sem->sema = CreateSemaphore(NULL, init, LONG_MAX, NULL);
+}
+
+void qemu_sem_destroy(QemuSemaphore *sem)
+{
+    CloseHandle(sem->sema);
+}
+
+void qemu_sem_post(QemuSemaphore *sem)
+{
+    ReleaseSemaphore(sem->sema, 1, NULL);
+}
+
+int qemu_sem_timedwait(QemuSemaphore *sem, int ms)
+{
+    int rc = WaitForSingleObject(sem->sema, ms);
+    if (rc == WAIT_OBJECT_0) {
+        return 0;
+    }
+    if (rc != WAIT_TIMEOUT) {
+        error_exit(GetLastError(), __func__);
+    }
+    return -1;
+}
+
+void qemu_sem_wait(QemuSemaphore *sem)
+{
+    if (WaitForSingleObject(sem->sema, INFINITE) != WAIT_OBJECT_0) {
+        error_exit(GetLastError(), __func__);
+    }
+}
+
 struct QemuThreadData {
     /* Passed to win32_start_routine.  */
     void             *(*start_routine)(void *);
diff --git a/qemu-thread-win32.h b/qemu-thread-win32.h
index b9d1be847..13adb958f 100644
--- a/qemu-thread-win32.h
+++ b/qemu-thread-win32.h
@@ -13,6 +13,10 @@ struct QemuCond {
     HANDLE continue_event;
 };
 
+struct QemuSemaphore {
+    HANDLE sema;
+};
+
 typedef struct QemuThreadData QemuThreadData;
 struct QemuThread {
     QemuThreadData *data;
diff --git a/qemu-thread.h b/qemu-thread.h
index 05fdaaf50..3ee2f6b1f 100644
--- a/qemu-thread.h
+++ b/qemu-thread.h
@@ -6,6 +6,7 @@
 
 typedef struct QemuMutex QemuMutex;
 typedef struct QemuCond QemuCond;
+typedef struct QemuSemaphore QemuSemaphore;
 typedef struct QemuThread QemuThread;
 
 #ifdef _WIN32
@@ -38,6 +39,12 @@ void qemu_cond_signal(QemuCond *cond);
 void qemu_cond_broadcast(QemuCond *cond);
 void qemu_cond_wait(QemuCond *cond, QemuMutex *mutex);
 
+void qemu_sem_init(QemuSemaphore *sem, int init);
+void qemu_sem_post(QemuSemaphore *sem);
+void qemu_sem_wait(QemuSemaphore *sem);
+int qemu_sem_timedwait(QemuSemaphore *sem, int ms);
+void qemu_sem_destroy(QemuSemaphore *sem);
+
 void qemu_thread_create(QemuThread *thread,
                         void *(*start_routine)(void *),
                         void *arg, int mode);
diff --git a/qemu-timer.c b/qemu-timer.c
index c7a1551a3..0d2bb9428 100644
--- a/qemu-timer.c
+++ b/qemu-timer.c
@@ -30,6 +30,9 @@
 #include "hw/hw.h"
 
 #include "qemu-timer.h"
+#ifdef CONFIG_POSIX
+#include <pthread.h>
+#endif
 
 #ifdef _WIN32
 #include <mmsystem.h>
@@ -372,21 +375,20 @@ bool qemu_timer_expired(QEMUTimer *timer_head, int64_t current_time)
 
 void qemu_run_timers(QEMUClock *clock)
 {
-    QEMUTimer **ptimer_head, *ts;
+    QEMUTimer *ts;
     int64_t current_time;
    
     if (!clock->enabled)
         return;
 
     current_time = qemu_get_clock_ns(clock);
-    ptimer_head = &clock->active_timers;
     for(;;) {
-        ts = *ptimer_head;
+        ts = clock->active_timers;
         if (!qemu_timer_expired_ns(ts, current_time)) {
             break;
         }
         /* remove timer from the list before calling the callback */
-        *ptimer_head = ts->next;
+        clock->active_timers = ts->next;
         ts->next = NULL;
 
         /* run the callback (the timer list can be modified) */
@@ -431,9 +433,11 @@ void qemu_unregister_clock_reset_notifier(QEMUClock *clock, Notifier *notifier)
 
 void init_clocks(void)
 {
-    rt_clock = qemu_new_clock(QEMU_CLOCK_REALTIME);
-    vm_clock = qemu_new_clock(QEMU_CLOCK_VIRTUAL);
-    host_clock = qemu_new_clock(QEMU_CLOCK_HOST);
+    if (!rt_clock) {
+        rt_clock = qemu_new_clock(QEMU_CLOCK_REALTIME);
+        vm_clock = qemu_new_clock(QEMU_CLOCK_VIRTUAL);
+        host_clock = qemu_new_clock(QEMU_CLOCK_HOST);
+    }
 }
 
 uint64_t qemu_timer_expire_time_ns(QEMUTimer *ts)
@@ -495,12 +499,12 @@ static int dynticks_start_timer(struct qemu_alarm_timer *t)
     memset(&ev, 0, sizeof(ev));
     ev.sigev_value.sival_int = 0;
     ev.sigev_notify = SIGEV_SIGNAL;
-#ifdef SIGEV_THREAD_ID
+#ifdef CONFIG_SIGEV_THREAD_ID
     if (qemu_signalfd_available()) {
         ev.sigev_notify = SIGEV_THREAD_ID;
         ev._sigev_un._tid = qemu_get_thread_id();
     }
-#endif /* SIGEV_THREAD_ID */
+#endif /* CONFIG_SIGEV_THREAD_ID */
     ev.sigev_signo = SIGALRM;
 
     if (timer_create(CLOCK_REALTIME, &ev, &host_timer)) {
@@ -741,11 +745,28 @@ static void quit_timers(void)
     t->stop(t);
 }
 
+#ifdef CONFIG_POSIX
+static void reinit_timers(void)
+{
+    struct qemu_alarm_timer *t = alarm_timer;
+    t->stop(t);
+    if (t->start(t)) {
+        fprintf(stderr, "Internal timer error: aborting\n");
+        exit(1);
+    }
+    qemu_rearm_alarm_timer(t);
+}
+#endif /* CONFIG_POSIX */
+
 int init_timer_alarm(void)
 {
     struct qemu_alarm_timer *t = NULL;
     int i, err = -1;
 
+    if (alarm_timer) {
+        return 0;
+    }
+
     for (i = 0; alarm_timers[i].name; i++) {
         t = &alarm_timers[i];
 
@@ -760,6 +781,9 @@ int init_timer_alarm(void)
     }
 
     atexit(quit_timers);
+#ifdef CONFIG_POSIX
+    pthread_atfork(NULL, NULL, reinit_timers);
+#endif
     alarm_timer = t;
     return 0;
 
diff --git a/qemu-timer.h b/qemu-timer.h
index f8af595f1..da7e97cd5 100644
--- a/qemu-timer.h
+++ b/qemu-timer.h
@@ -218,7 +218,7 @@ static inline int64_t cpu_get_real_ticks(void)
     return val;
 }
 
-#elif defined(__sparc_v8plus__) || defined(__sparc_v8plusa__) || defined(__sparc_v9__)
+#elif defined(__sparc__)
 
 static inline int64_t cpu_get_real_ticks (void)
 {
@@ -227,6 +227,8 @@ static inline int64_t cpu_get_real_ticks (void)
     asm volatile("rd %%tick,%0" : "=r"(rval));
     return rval;
 #else
+    /* We need an %o or %g register for this.  For recent enough gcc
+       there is an "h" constraint for that.  Don't bother with that.  */
     union {
         uint64_t i64;
         struct {
@@ -234,8 +236,8 @@ static inline int64_t cpu_get_real_ticks (void)
             uint32_t low;
         }       i32;
     } rval;
-    asm volatile("rd %%tick,%1; srlx %1,32,%0"
-                 : "=r"(rval.i32.high), "=r"(rval.i32.low));
+    asm volatile("rd %%tick,%%g1; srlx %%g1,32,%0; mov %%g1,%1"
+                 : "=r"(rval.i32.high), "=r"(rval.i32.low) : : "g1");
     return rval.i64;
 #endif
 }
diff --git a/qemu-tool.c b/qemu-tool.c
index 18205baba..b46631e42 100644
--- a/qemu-tool.c
+++ b/qemu-tool.c
@@ -19,6 +19,7 @@
 #include "qemu-log.h"
 #include "migration.h"
 #include "main-loop.h"
+#include "sysemu.h"
 #include "qemu_socket.h"
 #include "slirp/libslirp.h"
 
@@ -37,6 +38,11 @@ const char *qemu_get_vm_name(void)
 
 Monitor *cur_mon;
 
+void vm_stop(RunState state)
+{
+    abort();
+}
+
 int monitor_cur_is_qmp(void)
 {
     return 0;
@@ -62,29 +68,9 @@ void monitor_protocol_event(MonitorEvent event, QObject *data)
 {
 }
 
-int monitor_fdset_get_fd(int64_t fdset_id, int flags)
-{
-    return -1;
-}
-
-int monitor_fdset_dup_fd_add(int64_t fdset_id, int dup_fd)
-{
-    return -1;
-}
-
-int monitor_fdset_dup_fd_remove(int dup_fd)
-{
-    return -1;
-}
-
-int monitor_fdset_dup_fd_find(int dup_fd)
-{
-    return -1;
-}
-
 int64_t cpu_get_clock(void)
 {
-    return qemu_get_clock_ns(rt_clock);
+    return get_clock_realtime();
 }
 
 int64_t cpu_get_icount(void)
@@ -106,13 +92,6 @@ void qemu_clock_warp(QEMUClock *clock)
 {
 }
 
-int qemu_init_main_loop(void)
-{
-    init_clocks();
-    init_timer_alarm();
-    return main_loop_init();
-}
-
 void slirp_update_timeout(uint32_t *timeout)
 {
 }
diff --git a/qemu-user.c b/qemu-user.c
index 13fb9ae77..08ccb0fe8 100644
--- a/qemu-user.c
+++ b/qemu-user.c
@@ -35,23 +35,3 @@ void monitor_vprintf(Monitor *mon, const char *fmt, va_list ap)
 void monitor_set_error(Monitor *mon, QError *qerror)
 {
 }
-
-int monitor_fdset_get_fd(int64_t fdset_id, int flags)
-{
-    return -1;
-}
-
-int monitor_fdset_dup_fd_add(int64_t fdset_id, int dup_fd)
-{
-    return -1;
-}
-
-int monitor_fdset_dup_fd_remove(int dup_fd)
-{
-    return -1;
-}
-
-int monitor_fdset_dup_fd_find(int dup_fd)
-{
-    return -1;
-}
diff --git a/qemu_socket.h b/qemu_socket.h
index 30ae6af8b..02490ad06 100644
--- a/qemu_socket.h
+++ b/qemu_socket.h
@@ -38,19 +38,37 @@ void socket_set_block(int fd);
 void socket_set_nonblock(int fd);
 int send_all(int fd, const void *buf, int len1);
 
-/* New, ipv6-ready socket helper functions, see qemu-sockets.c */
+/* callback function for nonblocking connect
+ * valid fd on success, negative error code on failure
+ */
+typedef void NonBlockingConnectHandler(int fd, void *opaque);
+
 int inet_listen_opts(QemuOpts *opts, int port_offset, Error **errp);
 int inet_listen(const char *str, char *ostr, int olen,
                 int socktype, int port_offset, Error **errp);
-int inet_connect_opts(QemuOpts *opts, bool *in_progress, Error **errp);
-int inet_connect(const char *str, bool block, bool *in_progress, Error **errp);
-int inet_dgram_opts(QemuOpts *opts);
+int inet_connect_opts(QemuOpts *opts, Error **errp,
+                      NonBlockingConnectHandler *callback, void *opaque);
+int inet_connect(const char *str, Error **errp);
+int inet_nonblocking_connect(const char *str,
+                             NonBlockingConnectHandler *callback,
+                             void *opaque, Error **errp);
+
+int inet_dgram_opts(QemuOpts *opts, Error **errp);
 const char *inet_strfamily(int family);
 
-int unix_listen_opts(QemuOpts *opts);
-int unix_listen(const char *path, char *ostr, int olen);
-int unix_connect_opts(QemuOpts *opts);
-int unix_connect(const char *path);
+int unix_listen_opts(QemuOpts *opts, Error **errp);
+int unix_listen(const char *path, char *ostr, int olen, Error **errp);
+int unix_connect_opts(QemuOpts *opts, Error **errp,
+                      NonBlockingConnectHandler *callback, void *opaque);
+int unix_connect(const char *path, Error **errp);
+int unix_nonblocking_connect(const char *str,
+                             NonBlockingConnectHandler *callback,
+                             void *opaque, Error **errp);
+
+SocketAddress *socket_parse(const char *str, Error **errp);
+int socket_connect(SocketAddress *addr, Error **errp,
+                   NonBlockingConnectHandler *callback, void *opaque);
+int socket_listen(SocketAddress *addr, Error **errp);
 
 /* Old, ipv4 only bits.  Don't use for new code. */
 int parse_host_port(struct sockaddr_in *saddr, const char *str);
diff --git a/qerror.h b/qerror.h
index d0a76a4f7..8db4309ae 100644
--- a/qerror.h
+++ b/qerror.h
@@ -48,6 +48,15 @@ void assert_no_error(Error *err);
 #define QERR_BASE_NOT_FOUND \
     ERROR_CLASS_GENERIC_ERROR, "Base '%s' not found"
 
+#define QERR_BLOCK_JOB_NOT_ACTIVE \
+    ERROR_CLASS_DEVICE_NOT_ACTIVE, "No active block job on device '%s'"
+
+#define QERR_BLOCK_JOB_PAUSED \
+    ERROR_CLASS_GENERIC_ERROR, "The block job for device '%s' is currently paused"
+
+#define QERR_BLOCK_JOB_NOT_READY \
+    ERROR_CLASS_GENERIC_ERROR, "The active block job for device '%s' cannot be completed"
+
 #define QERR_BLOCK_FORMAT_FEATURE_NOT_SUPPORTED \
     ERROR_CLASS_GENERIC_ERROR, "Block format '%s' used by device '%s' does not support feature '%s'"
 
@@ -159,9 +168,6 @@ void assert_no_error(Error *err);
 #define QERR_MIGRATION_NOT_SUPPORTED \
     ERROR_CLASS_GENERIC_ERROR, "State blocked by non-migratable device '%s'"
 
-#define QERR_MIGRATION_EXPECTED \
-    ERROR_CLASS_MIGRATION_EXPECTED, "An incoming migration is expected before this command can be executed"
-
 #define QERR_MISSING_PARAMETER \
     ERROR_CLASS_GENERIC_ERROR, "Parameter '%s' is missing"
 
@@ -231,9 +237,6 @@ void assert_no_error(Error *err);
 #define QERR_VIRTFS_FEATURE_BLOCKS_MIGRATION \
     ERROR_CLASS_GENERIC_ERROR, "Migration is disabled when VirtFS export path '%s' is mounted in the guest using mount_tag '%s'"
 
-#define QERR_VNC_SERVER_FAILED \
-    ERROR_CLASS_GENERIC_ERROR, "Could not start VNC server on %s"
-
 #define QERR_SOCKET_CONNECT_FAILED \
     ERROR_CLASS_GENERIC_ERROR, "Failed to connect to socket"
 
diff --git a/qga/channel-posix.c b/qga/channel-posix.c
index 57eea06c4..d152827bc 100644
--- a/qga/channel-posix.c
+++ b/qga/channel-posix.c
@@ -181,9 +181,11 @@ static gboolean ga_channel_open(GAChannel *c, const gchar *path, GAChannelMethod
         break;
     }
     case GA_CHANNEL_UNIX_LISTEN: {
-        int fd = unix_listen(path, NULL, strlen(path));
-        if (fd == -1) {
-            g_critical("error opening path: %s", strerror(errno));
+        Error *local_err = NULL;
+        int fd = unix_listen(path, NULL, strlen(path), &local_err);
+        if (local_err != NULL) {
+            g_critical("%s", error_get_pretty(local_err));
+            error_free(local_err);
             return false;
         }
         ga_channel_listen_add(c, fd, true);
diff --git a/qga/commands-posix.c b/qga/commands-posix.c
index ce9042123..726930a90 100644
--- a/qga/commands-posix.c
+++ b/qga/commands-posix.c
@@ -828,7 +828,7 @@ GuestNetworkInterfaceList *qmp_guest_network_get_interfaces(Error **errp)
             }
 
             memset(&ifr, 0, sizeof(ifr));
-            strncpy(ifr.ifr_name,  info->value->name, IF_NAMESIZE);
+            pstrcpy(ifr.ifr_name, IF_NAMESIZE, info->value->name);
             if (ioctl(sock, SIOCGIFHWADDR, &ifr) == -1) {
                 snprintf(err_msg, sizeof(err_msg),
                          "failed to get MAC address of %s: %s",
@@ -988,8 +988,6 @@ int64_t qmp_guest_fsfreeze_thaw(Error **err)
 void qmp_guest_fstrim(bool has_minimum, int64_t minimum, Error **err)
 {
     error_set(err, QERR_UNSUPPORTED);
-
-    return;
 }
 #endif
 
diff --git a/qga/commands-win32.c b/qga/commands-win32.c
index 54bc5462e..5bd8fb27f 100644
--- a/qga/commands-win32.c
+++ b/qga/commands-win32.c
@@ -180,8 +180,6 @@ int64_t qmp_guest_fsfreeze_thaw(Error **err)
 void qmp_guest_fstrim(bool has_minimum, int64_t minimum, Error **err)
 {
     error_set(err, QERR_UNSUPPORTED);
-
-    return;
 }
 
 typedef enum {
diff --git a/qmp-commands.hx b/qmp-commands.hx
index 3745a2119..5c692d0cb 100644
--- a/qmp-commands.hx
+++ b/qmp-commands.hx
@@ -146,10 +146,7 @@ EQMP
     {
         .name       = "screendump",
         .args_type  = "filename:F",
-        .params     = "filename",
-        .help       = "save screen into PPM image 'filename'",
-        .user_print = monitor_user_noop,
-        .mhandler.cmd_new = do_screen_dump,
+        .mhandler.cmd_new = qmp_marshal_input_screendump,
     },
 
 SQMP
@@ -335,6 +332,34 @@ Example:
 EQMP
 
     {
+        .name       = "send-key",
+        .args_type  = "keys:O,hold-time:i?",
+        .mhandler.cmd_new = qmp_marshal_input_send_key,
+    },
+
+SQMP
+send-key
+----------
+
+Send keys to VM.
+
+Arguments:
+
+keys array:
+    - "key": key sequence (a json-array of key enum values)
+
+- hold-time: time to delay key up events, milliseconds. Defaults to 100
+             (json-int, optional)
+
+Example:
+
+-> { "execute": "send-key",
+     "arguments": { 'keys': [ 'ctrl', 'alt', 'delete' ] } }
+<- { "return": {} }
+
+EQMP
+
+    {
         .name       = "cpu",
         .args_type  = "index:i",
         .mhandler.cmd_new = qmp_marshal_input_cpu,
@@ -468,6 +493,30 @@ Example:
 EQMP
 
     {
+        .name       = "xen-set-global-dirty-log",
+        .args_type  = "enable:b",
+        .mhandler.cmd_new = qmp_marshal_input_xen_set_global_dirty_log,
+    },
+
+SQMP
+xen-set-global-dirty-log
+-------
+
+Enable or disable the global dirty log mode.
+
+Arguments:
+
+- "enable": Enable it or disable it.
+
+Example:
+
+-> { "execute": "xen-set-global-dirty-log",
+     "arguments": { "enable": true } }
+<- { "return": {} }
+
+EQMP
+
+    {
         .name       = "migrate",
         .args_type  = "detach:-d,blk:-b,inc:-i,uri:s",
         .mhandler.cmd_new = qmp_marshal_input_migrate,
@@ -762,11 +811,17 @@ EQMP
 
     {
         .name       = "block-stream",
-        .args_type  = "device:B,base:s?,speed:o?",
+        .args_type  = "device:B,base:s?,speed:o?,on-error:s?",
         .mhandler.cmd_new = qmp_marshal_input_block_stream,
     },
 
     {
+        .name       = "block-commit",
+        .args_type  = "device:B,base:s?,top:s,speed:o?",
+        .mhandler.cmd_new = qmp_marshal_input_block_commit,
+    },
+
+    {
         .name       = "block-job-set-speed",
         .args_type  = "device:B,speed:o",
         .mhandler.cmd_new = qmp_marshal_input_block_job_set_speed,
@@ -774,10 +829,25 @@ EQMP
 
     {
         .name       = "block-job-cancel",
-        .args_type  = "device:B",
+        .args_type  = "device:B,force:b?",
         .mhandler.cmd_new = qmp_marshal_input_block_job_cancel,
     },
     {
+        .name       = "block-job-pause",
+        .args_type  = "device:B",
+        .mhandler.cmd_new = qmp_marshal_input_block_job_pause,
+    },
+    {
+        .name       = "block-job-resume",
+        .args_type  = "device:B",
+        .mhandler.cmd_new = qmp_marshal_input_block_job_resume,
+    },
+    {
+        .name       = "block-job-complete",
+        .args_type  = "device:B",
+        .mhandler.cmd_new = qmp_marshal_input_block_job_complete,
+    },
+    {
         .name       = "transaction",
         .args_type  = "actions:q",
         .mhandler.cmd_new = qmp_marshal_input_transaction,
@@ -866,6 +936,54 @@ Example:
 EQMP
 
     {
+        .name       = "drive-mirror",
+        .args_type  = "sync:s,device:B,target:s,speed:i?,mode:s?,format:s?,"
+                      "on-source-error:s?,on-target-error:s?",
+        .mhandler.cmd_new = qmp_marshal_input_drive_mirror,
+    },
+
+SQMP
+drive-mirror
+------------
+
+Start mirroring a block device's writes to a new destination. target
+specifies the target of the new image. If the file exists, or if it is
+a device, it will be used as the new destination for writes. If it does not
+exist, a new file will be created. format specifies the format of the
+mirror image, default is to probe if mode='existing', else the format
+of the source.
+
+Arguments:
+
+- "device": device name to operate on (json-string)
+- "target": name of new image file (json-string)
+- "format": format of new image (json-string, optional)
+- "mode": how an image file should be created into the target
+  file/device (NewImageMode, optional, default 'absolute-paths')
+- "speed": maximum speed of the streaming job, in bytes per second
+  (json-int)
+- "sync": what parts of the disk image should be copied to the destination;
+  possibilities include "full" for all the disk, "top" for only the sectors
+  allocated in the topmost image, or "none" to only replicate new I/O
+  (MirrorSyncMode).
+- "on-source-error": the action to take on an error on the source
+  (BlockdevOnError, default 'report')
+- "on-target-error": the action to take on an error on the target
+  (BlockdevOnError, default 'report')
+
+
+
+Example:
+
+-> { "execute": "drive-mirror", "arguments": { "device": "ide-hd0",
+                                               "target": "/some/place/my-image",
+                                               "sync": "full",
+                                               "format": "qcow2" } }
+<- { "return": {} }
+
+EQMP
+
+    {
         .name       = "balloon",
         .args_type  = "value:M",
         .mhandler.cmd_new = qmp_marshal_input_balloon,
@@ -1206,10 +1324,7 @@ EQMP
     {
         .name       = "add_client",
         .args_type  = "protocol:s,fdname:s,skipauth:b?,tls:b?",
-        .params     = "protocol fdname skipauth tls",
-        .help       = "add a graphics client",
-        .user_print = monitor_user_noop,
-        .mhandler.cmd_new = add_graphics_client,
+        .mhandler.cmd_new = qmp_marshal_input_add_client,
     },
 
 SQMP
@@ -2242,6 +2357,11 @@ The main json-object contains the following:
 - "total-time": total amount of ms since migration started.  If
                 migration has ended, it returns the total migration
 		 time (json-int)
+- "downtime": only present when migration has finished correctly
+              total amount in ms for downtime that happened (json-int)
+- "expected-downtime": only present while migration is active
+                total amount in ms for downtime that was calculated on
+		the last bitmap round (json-int)
 - "ram": only present if "status" is "active", it is a json-object with the
   following RAM information (in bytes):
          - "transferred": amount transferred (json-int)
@@ -2279,6 +2399,7 @@ Examples:
           "remaining":123,
           "total":246,
           "total-time":12345,
+          "downtime":12345,
           "duplicate":123,
           "normal":123,
           "normal-bytes":123456
@@ -2302,6 +2423,7 @@ Examples:
             "remaining":123,
             "total":246,
             "total-time":12345,
+            "expected-downtime":12345,
             "duplicate":123,
             "normal":123,
             "normal-bytes":123456
@@ -2320,6 +2442,7 @@ Examples:
             "remaining":1053304,
             "transferred":3720,
             "total-time":12345,
+            "expected-downtime":12345,
             "duplicate":123,
             "normal":123,
             "normal-bytes":123456
@@ -2344,6 +2467,7 @@ Examples:
             "remaining":1053304,
             "transferred":3720,
             "total-time":12345,
+            "expected-downtime":12345,
             "duplicate":10,
             "normal":3333,
             "normal-bytes":3412992
@@ -2481,6 +2605,22 @@ EQMP
     },
 
     {
+        .name       = "nbd-server-start",
+        .args_type  = "addr:q",
+        .mhandler.cmd_new = qmp_marshal_input_nbd_server_start,
+    },
+    {
+        .name       = "nbd-server-add",
+        .args_type  = "device:B,writable:b?",
+        .mhandler.cmd_new = qmp_marshal_input_nbd_server_add,
+    },
+    {
+        .name       = "nbd-server-stop",
+        .args_type  = "",
+        .mhandler.cmd_new = qmp_marshal_input_nbd_server_stop,
+    },
+
+    {
         .name       = "change-vnc-password",
         .args_type  = "password:s",
         .mhandler.cmd_new = qmp_marshal_input_change_vnc_password,
diff --git a/qmp.c b/qmp.c
index 84639220d..e3a7f0b21 100644
--- a/qmp.c
+++ b/qmp.c
@@ -85,7 +85,11 @@ void qmp_quit(Error **err)
 
 void qmp_stop(Error **errp)
 {
-    vm_stop(RUN_STATE_PAUSED);
+    if (runstate_check(RUN_STATE_INMIGRATE)) {
+        autostart = 0;
+    } else {
+        vm_stop(RUN_STATE_PAUSED);
+    }
 }
 
 void qmp_system_reset(Error **errp)
@@ -144,10 +148,7 @@ void qmp_cont(Error **errp)
 {
     Error *local_err = NULL;
 
-    if (runstate_check(RUN_STATE_INMIGRATE)) {
-        error_set(errp, QERR_MIGRATION_EXPECTED);
-        return;
-    } else if (runstate_check(RUN_STATE_INTERNAL_ERROR) ||
+    if (runstate_check(RUN_STATE_INTERNAL_ERROR) ||
                runstate_check(RUN_STATE_SHUTDOWN)) {
         error_set(errp, QERR_RESET_REQUIRED);
         return;
@@ -162,7 +163,11 @@ void qmp_cont(Error **errp)
         return;
     }
 
-    vm_start();
+    if (runstate_check(RUN_STATE_INMIGRATE)) {
+        autostart = 1;
+    } else {
+        vm_start();
+    }
 }
 
 void qmp_system_wakeup(Error **errp)
@@ -349,11 +354,9 @@ void qmp_change_vnc_password(const char *password, Error **errp)
     }
 }
 
-static void qmp_change_vnc_listen(const char *target, Error **err)
+static void qmp_change_vnc_listen(const char *target, Error **errp)
 {
-    if (vnc_display_open(NULL, target) < 0) {
-        error_set(err, QERR_VNC_SERVER_FAILED, target);
-    }
+    vnc_display_open(NULL, target, errp);
 }
 
 static void qmp_change_vnc(const char *target, bool has_arg, const char *arg,
@@ -468,14 +471,51 @@ DevicePropertyInfoList *qmp_device_list_properties(const char *typename,
     return prop_list;
 }
 
-CpuDefinitionInfoList GCC_WEAK *arch_query_cpu_definitions(Error **errp)
-{
-    error_set(errp, QERR_NOT_SUPPORTED);
-    return NULL;
-}
-
 CpuDefinitionInfoList *qmp_query_cpu_definitions(Error **errp)
 {
     return arch_query_cpu_definitions(errp);
 }
 
+void qmp_add_client(const char *protocol, const char *fdname,
+                    bool has_skipauth, bool skipauth, bool has_tls, bool tls,
+                    Error **errp)
+{
+    CharDriverState *s;
+    int fd;
+
+    fd = monitor_get_fd(cur_mon, fdname, errp);
+    if (fd < 0) {
+        return;
+    }
+
+    if (strcmp(protocol, "spice") == 0) {
+        if (!using_spice) {
+            error_set(errp, QERR_DEVICE_NOT_ACTIVE, "spice");
+            close(fd);
+            return;
+        }
+        skipauth = has_skipauth ? skipauth : false;
+        tls = has_tls ? tls : false;
+        if (qemu_spice_display_add_client(fd, skipauth, tls) < 0) {
+            error_setg(errp, "spice failed to add client");
+            close(fd);
+        }
+        return;
+#ifdef CONFIG_VNC
+    } else if (strcmp(protocol, "vnc") == 0) {
+        skipauth = has_skipauth ? skipauth : false;
+        vnc_display_add_client(NULL, fd, skipauth);
+        return;
+#endif
+    } else if ((s = qemu_chr_find(protocol)) != NULL) {
+        if (qemu_chr_add_client(s, fd) < 0) {
+            error_setg(errp, "failed to add client");
+            close(fd);
+            return;
+        }
+        return;
+    }
+
+    error_setg(errp, "protocol '%s' is invalid", protocol);
+    close(fd);
+}
diff --git a/qobject.h b/qobject.h
index d42386dde..9124649ed 100644
--- a/qobject.h
+++ b/qobject.h
@@ -71,7 +71,7 @@ typedef struct QObject {
 
 /* High-level interface for qobject_decref() */
 #define QDECREF(obj)              \
-    qobject_decref(QOBJECT(obj))
+    qobject_decref(obj ? QOBJECT(obj) : NULL)
 
 /* Initialize an object to default values */
 #define QOBJECT_INIT(obj, qtype_type)   \
diff --git a/qom/object.c b/qom/object.c
index e3e924263..0739aa294 100644
--- a/qom/object.c
+++ b/qom/object.c
@@ -307,6 +307,7 @@ void object_initialize_with_type(void *data, TypeImpl *type)
 
     memset(obj, 0, type->instance_size);
     obj->class = type->class;
+    object_ref(obj);
     QTAILQ_INIT(&obj->properties);
     object_init_with_type(obj, type);
 }
@@ -362,6 +363,9 @@ void object_unparent(Object *obj)
     if (obj->parent) {
         object_property_del_child(obj->parent, obj, NULL);
     }
+    if (obj->class->unparent) {
+        (obj->class->unparent)(obj);
+    }
 }
 
 static void object_deinit(Object *obj, TypeImpl *type)
@@ -375,7 +379,7 @@ static void object_deinit(Object *obj, TypeImpl *type)
     }
 }
 
-void object_finalize(void *data)
+static void object_finalize(void *data)
 {
     Object *obj = data;
     TypeImpl *ti = obj->class->type;
@@ -384,6 +388,9 @@ void object_finalize(void *data)
     object_property_del_all(obj);
 
     g_assert(obj->ref == 0);
+    if (obj->free) {
+        obj->free(obj);
+    }
 }
 
 Object *object_new_with_type(Type type)
@@ -395,7 +402,7 @@ Object *object_new_with_type(Type type)
 
     obj = g_malloc(type->instance_size);
     object_initialize_with_type(obj, type);
-    object_ref(obj);
+    obj->free = g_free;
 
     return obj;
 }
@@ -412,12 +419,11 @@ void object_delete(Object *obj)
     object_unparent(obj);
     g_assert(obj->ref == 1);
     object_unref(obj);
-    g_free(obj);
 }
 
 Object *object_dynamic_cast(Object *obj, const char *typename)
 {
-    if (object_class_dynamic_cast(object_get_class(obj), typename)) {
+    if (obj && object_class_dynamic_cast(object_get_class(obj), typename)) {
         return obj;
     }
 
@@ -430,7 +436,7 @@ Object *object_dynamic_cast_assert(Object *obj, const char *typename)
 
     inst = object_dynamic_cast(obj, typename);
 
-    if (!inst) {
+    if (!inst && obj) {
         fprintf(stderr, "Object %p is not an instance of type %s\n",
                 obj, typename);
         abort();
@@ -1183,6 +1189,62 @@ void object_property_add_str(Object *obj, const char *name,
                         prop, errp);
 }
 
+typedef struct BoolProperty
+{
+    bool (*get)(Object *, Error **);
+    void (*set)(Object *, bool, Error **);
+} BoolProperty;
+
+static void property_get_bool(Object *obj, Visitor *v, void *opaque,
+                              const char *name, Error **errp)
+{
+    BoolProperty *prop = opaque;
+    bool value;
+
+    value = prop->get(obj, errp);
+    visit_type_bool(v, &value, name, errp);
+}
+
+static void property_set_bool(Object *obj, Visitor *v, void *opaque,
+                              const char *name, Error **errp)
+{
+    BoolProperty *prop = opaque;
+    bool value;
+    Error *local_err = NULL;
+
+    visit_type_bool(v, &value, name, &local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        return;
+    }
+
+    prop->set(obj, value, errp);
+}
+
+static void property_release_bool(Object *obj, const char *name,
+                                  void *opaque)
+{
+    BoolProperty *prop = opaque;
+    g_free(prop);
+}
+
+void object_property_add_bool(Object *obj, const char *name,
+                              bool (*get)(Object *, Error **),
+                              void (*set)(Object *, bool, Error **),
+                              Error **errp)
+{
+    BoolProperty *prop = g_malloc0(sizeof(*prop));
+
+    prop->get = get;
+    prop->set = set;
+
+    object_property_add(obj, name, "bool",
+                        get ? property_get_bool : NULL,
+                        set ? property_set_bool : NULL,
+                        property_release_bool,
+                        prop, errp);
+}
+
 static char *qdev_get_type(Object *obj, Error **errp)
 {
     return g_strdup(object_get_typename(obj));
diff --git a/roms/Makefile b/roms/Makefile
index feb9c2b14..5e645bc7d 100644
--- a/roms/Makefile
+++ b/roms/Makefile
@@ -12,6 +12,7 @@ bios: config.seabios
 	sh configure-seabios.sh $<
 	make -C seabios out/bios.bin
 	cp seabios/out/bios.bin ../pc-bios/bios.bin
+	cp seabios/out/*dsdt.aml ../pc-bios/
 
 seavgabios: $(patsubst %,seavgabios-%,$(vgabios_variants))
 
diff --git a/roms/seabios/Makefile b/roms/seabios/Makefile
index 849824d28..b0e203113 100644
--- a/roms/seabios/Makefile
+++ b/roms/seabios/Makefile
@@ -13,7 +13,7 @@ SRCBOTH=misc.c stacks.c pmm.c output.c util.c block.c floppy.c ata.c mouse.c \
     pnpbios.c pirtable.c vgahooks.c ramdisk.c pcibios.c blockcmd.c \
     usb.c usb-uhci.c usb-ohci.c usb-ehci.c usb-hid.c usb-msc.c \
     virtio-ring.c virtio-pci.c virtio-blk.c virtio-scsi.c apm.c ahci.c \
-    usb-uas.c lsi-scsi.c
+    usb-uas.c lsi-scsi.c esp-scsi.c megasas.c
 SRC16=$(SRCBOTH) system.c disk.c font.c
 SRC32FLAT=$(SRCBOTH) post.c shadow.c memmap.c coreboot.c boot.c \
     acpi.c smm.c mptable.c smbios.c pciinit.c optionroms.c mtrr.c \
@@ -75,6 +75,7 @@ all: $(target-y)
 
 # Make definitions
 .PHONY : all clean distclean FORCE
+.DELETE_ON_ERROR:
 
 vpath %.c src vgasrc
 vpath %.S src vgasrc
@@ -221,7 +222,7 @@ $(OUT)vgabios.bin: $(OUT)vgabios.bin.raw tools/buildrom.py
 
 ################ DSDT build rules
 
-iasl-option=$(shell if "$(1)" "$(2)" -h > /dev/null 2>&1 \
+iasl-option=$(shell if test -z "`$(1) $(2) 2>&1 > /dev/null`" \
     ; then echo "$(2)"; else echo "$(3)"; fi ;)
 
 $(OUT)%.hex: src/%.dsl ./tools/acpi_extract_preprocess.py ./tools/acpi_extract.py
@@ -232,7 +233,7 @@ $(OUT)%.hex: src/%.dsl ./tools/acpi_extract_preprocess.py ./tools/acpi_extract.p
 	$(Q)$(PYTHON) ./tools/acpi_extract.py $(OUT)$*.lst > $(OUT)$*.off
 	$(Q)cat $(OUT)$*.off > $@
 
-$(OUT)ccode32flat.o: $(OUT)acpi-dsdt.hex $(OUT)ssdt-proc.hex $(OUT)ssdt-pcihp.hex
+$(OUT)ccode32flat.o: $(OUT)acpi-dsdt.hex $(OUT)ssdt-proc.hex $(OUT)ssdt-pcihp.hex $(OUT)ssdt-susp.hex
 
 ################ Kconfig rules
 
diff --git a/roms/seabios/src/Kconfig b/roms/seabios/src/Kconfig
index bc343ee81..0b112edd8 100644
--- a/roms/seabios/src/Kconfig
+++ b/roms/seabios/src/Kconfig
@@ -4,14 +4,24 @@ mainmenu "SeaBIOS Configuration"
 
 menu "General Features"
 
+choice
+    prompt "Build Target"
+    default QEMU
+
     config COREBOOT
         bool "Build for coreboot"
-        default n
         help
             Configure as a coreboot payload.
 
+    config QEMU
+        bool "Build for QEMU"
+        help
+            Configure as QEMU bios.
+
+endchoice
+
     config XEN
-        depends on !COREBOOT
+        depends on QEMU
         bool "Build for Xen HVM"
         default y
         help
@@ -108,23 +118,35 @@ menu "Hardware support"
         help
             Support for AHCI disk code.
     config VIRTIO_BLK
-        depends on DRIVES && !COREBOOT
+        depends on DRIVES && QEMU
         bool "virtio-blk controllers"
         default y
         help
             Support boot from virtio-blk storage.
     config VIRTIO_SCSI
-        depends on DRIVES && !COREBOOT
+        depends on DRIVES && QEMU
         bool "virtio-scsi controllers"
         default y
         help
             Support boot from virtio-scsi storage.
+    config ESP_SCSI
+        depends on DRIVES && QEMU
+        bool "AMD PCscsi controllers"
+        default y
+        help
+            Support boot from AMD PCscsi storage.
     config LSI_SCSI
-        depends on DRIVES && !COREBOOT
+        depends on DRIVES && QEMU
         bool "lsi53c895a scsi controllers"
         default y
         help
             Support boot from qemu-emulated lsi53c895a scsi storage.
+    config MEGASAS
+        depends on DRIVES
+        bool "LSI MegaRAID SAS controllers"
+        default y
+        help
+            Support boot from LSI MegaRAID SAS scsi storage.
     config FLOPPY
         depends on DRIVES
         bool "Floppy controller"
@@ -205,17 +227,23 @@ menu "Hardware support"
             Support parallel ports. This also enables int 17 parallel port calls.
 
     config USE_SMM
-        depends on !COREBOOT
+        depends on QEMU
         bool "System Management Mode (SMM)"
         default y
         help
             Support System Management Mode (on emulators).
     config MTRR_INIT
-        depends on !COREBOOT
+        depends on QEMU
         bool "Initialize MTRRs"
         default y
         help
             Initialize the Memory Type Range Registers (on emulators).
+    config PMTIMER
+        depends on QEMU
+        bool "Use ACPI timer"
+        default y
+        help
+            Use the ACPI timer instead of the TSC for timekeeping (on qemu).
 endmenu
 
 menu "BIOS interfaces"
@@ -315,7 +343,7 @@ menu "BIOS interfaces"
 endmenu
 
 menu "BIOS Tables"
-    depends on !COREBOOT
+    depends on QEMU
     config PIRTABLE
         bool "PIR table"
         default y
@@ -365,7 +393,7 @@ menu "Debugging"
             Base port for serial - generally 0x3f8, 0x2f8, 0x3e8, or 0x2e8.
 
     config DEBUG_IO
-        depends on !COREBOOT && DEBUG_LEVEL != 0
+        depends on QEMU && DEBUG_LEVEL != 0
         bool "Special IO port debugging"
         default y
         help
diff --git a/roms/seabios/src/acpi-dsdt.dsl b/roms/seabios/src/acpi-dsdt.dsl
index 72dc7d8aa..711302ece 100644
--- a/roms/seabios/src/acpi-dsdt.dsl
+++ b/roms/seabios/src/acpi-dsdt.dsl
@@ -86,7 +86,10 @@ DefinitionBlock (
 #define prt_slot3(nr) prt_slot(nr, LNKC, LNKD, LNKA, LNKB)
                prt_slot0(0x0000),
                /* Device 1 is power mgmt device, and can only use irq 9 */
-               prt_slot(0x0001, LNKS, LNKB, LNKC, LNKD),
+               Package() { 0x1ffff, 0,    0, 9 },
+               Package() { 0x1ffff, 1, LNKB, 0 },
+               Package() { 0x1ffff, 2, LNKC, 0 },
+               Package() { 0x1ffff, 3, LNKD, 0 },
                prt_slot2(0x0002),
                prt_slot3(0x0003),
                prt_slot0(0x0004),
@@ -266,15 +269,11 @@ DefinitionBlock (
                     Return (0x0F)
             }
             Name(_CRS, ResourceTemplate() {
-                DWordMemory(
-                    ResourceConsumer, PosDecode, MinFixed, MaxFixed,
-                    NonCacheable, ReadWrite,
-                    0x00000000,
-                    0xFED00000,
-                    0xFED003FF,
-                    0x00000000,
-                    0x00000400 /* 1K memory: FED00000 - FED003FF */
-                )
+                IRQNoFlags () {2, 8}
+                Memory32Fixed (ReadOnly,
+                    0xFED00000,         // Address Base
+                    0x00000400,         // Address Length
+                    )
             })
         }
     }
@@ -556,7 +555,6 @@ DefinitionBlock (
                     PCNT(Local0, 3)
                 }
             }
-            Return(One)
         }
 
     }
@@ -651,17 +649,6 @@ DefinitionBlock (
             Method (_CRS, 0, NotSerialized) { Return (IQCR(PRQ3)) }
             Method (_SRS, 1, NotSerialized) { SETIRQ(PRQ3, Arg0) }
         }
-        Device(LNKS) {
-            Name(_HID, EISAID("PNP0C0F"))     // PCI interrupt link
-            Name(_UID, 5)
-            Name(_PRS, ResourceTemplate() {
-                Interrupt (, Level, ActiveHigh, Shared)
-                    { 9 }
-            })
-            Method (_STA, 0, NotSerialized) { Return (IQST(PRQ0)) }
-            Method (_DIS, 0, NotSerialized) { DISIRQ(PRQ0) }
-            Method (_CRS, 0, NotSerialized) { Return (IQCR(PRQ0)) }
-        }
     }
 
 /****************************************************************
@@ -737,7 +724,6 @@ DefinitionBlock (
                 }
                 Increment(Local0)
             }
-            Return(One)
         }
     }
 
@@ -751,54 +737,40 @@ DefinitionBlock (
         Name(_HID, "ACPI0006")
 
         Method(_L00) {
-            Return(0x01)
         }
         Method(_E01) {
             // PCI hotplug event
-            Return(\_SB.PCI0.PCNF())
+            \_SB.PCI0.PCNF()
         }
         Method(_E02) {
             // CPU hotplug event
-            Return(\_SB.PRSC())
+            \_SB.PRSC()
         }
         Method(_L03) {
-            Return(0x01)
         }
         Method(_L04) {
-            Return(0x01)
         }
         Method(_L05) {
-            Return(0x01)
         }
         Method(_L06) {
-            Return(0x01)
         }
         Method(_L07) {
-            Return(0x01)
         }
         Method(_L08) {
-            Return(0x01)
         }
         Method(_L09) {
-            Return(0x01)
         }
         Method(_L0A) {
-            Return(0x01)
         }
         Method(_L0B) {
-            Return(0x01)
         }
         Method(_L0C) {
-            Return(0x01)
         }
         Method(_L0D) {
-            Return(0x01)
         }
         Method(_L0E) {
-            Return(0x01)
         }
         Method(_L0F) {
-            Return(0x01)
         }
     }
 }
diff --git a/roms/seabios/src/acpi.c b/roms/seabios/src/acpi.c
index 39b71720e..6d239fa7f 100644
--- a/roms/seabios/src/acpi.c
+++ b/roms/seabios/src/acpi.c
@@ -7,11 +7,12 @@
 
 #include "acpi.h" // struct rsdp_descriptor
 #include "util.h" // memcpy
+#include "byteorder.h" // cpu_to_le16
 #include "pci.h" // pci_find_init_device
 #include "pci_ids.h" // PCI_VENDOR_ID_INTEL
 #include "pci_regs.h" // PCI_INTERRUPT_LINE
 #include "ioport.h" // inl
-#include "paravirt.h"
+#include "paravirt.h" // qemu_cfg_irq0_override
 
 /****************************************************/
 /* ACPI tables init */
@@ -400,34 +401,115 @@ encodeLen(u8 *ssdt_ptr, int length, int bytes)
 #include "ssdt-proc.hex"
 
 /* 0x5B 0x83 ProcessorOp PkgLength NameString ProcID */
-#define SD_OFFSET_CPUHEX (*ssdt_proc_name - *ssdt_proc_start + 2)
-#define SD_OFFSET_CPUID1 (*ssdt_proc_name - *ssdt_proc_start + 4)
-#define SD_OFFSET_CPUID2 (*ssdt_proc_id - *ssdt_proc_start)
-#define SD_SIZEOF (*ssdt_proc_end - *ssdt_proc_start)
-#define SD_PROC (ssdp_proc_aml + *ssdt_proc_start)
+#define PROC_OFFSET_CPUHEX (*ssdt_proc_name - *ssdt_proc_start + 2)
+#define PROC_OFFSET_CPUID1 (*ssdt_proc_name - *ssdt_proc_start + 4)
+#define PROC_OFFSET_CPUID2 (*ssdt_proc_id - *ssdt_proc_start)
+#define PROC_SIZEOF (*ssdt_proc_end - *ssdt_proc_start)
+#define PROC_AML (ssdp_proc_aml + *ssdt_proc_start)
+
+/* 0x5B 0x82 DeviceOp PkgLength NameString */
+#define PCIHP_OFFSET_HEX (*ssdt_pcihp_name - *ssdt_pcihp_start + 1)
+#define PCIHP_OFFSET_ID (*ssdt_pcihp_id - *ssdt_pcihp_start)
+#define PCIHP_OFFSET_ADR (*ssdt_pcihp_adr - *ssdt_pcihp_start)
+#define PCIHP_OFFSET_EJ0 (*ssdt_pcihp_ej0 - *ssdt_pcihp_start)
+#define PCIHP_SIZEOF (*ssdt_pcihp_end - *ssdt_pcihp_start)
+#define PCIHP_AML (ssdp_pcihp_aml + *ssdt_pcihp_start)
+#define PCI_SLOTS 32
 
 #define SSDT_SIGNATURE 0x54445353 // SSDT
+#define SSDT_HEADER_LENGTH 36
+
+#include "ssdt-susp.hex"
+#include "ssdt-pcihp.hex"
+
+#define PCI_RMV_BASE 0xae0c
+
+static u8*
+build_notify(u8 *ssdt_ptr, const char *name, int skip, int count,
+             const char *target, int ofs)
+{
+    count -= skip;
+
+    *(ssdt_ptr++) = 0x14; // MethodOp
+    ssdt_ptr = encodeLen(ssdt_ptr, 2+5+(12*count), 2);
+    memcpy(ssdt_ptr, name, 4);
+    ssdt_ptr += 4;
+    *(ssdt_ptr++) = 0x02; // MethodOp
+
+    int i;
+    for (i = skip; count-- > 0; i++) {
+        *(ssdt_ptr++) = 0xA0; // IfOp
+        ssdt_ptr = encodeLen(ssdt_ptr, 11, 1);
+        *(ssdt_ptr++) = 0x93; // LEqualOp
+        *(ssdt_ptr++) = 0x68; // Arg0Op
+        *(ssdt_ptr++) = 0x0A; // BytePrefix
+        *(ssdt_ptr++) = i;
+        *(ssdt_ptr++) = 0x86; // NotifyOp
+        memcpy(ssdt_ptr, target, 4);
+        ssdt_ptr[ofs] = getHex(i >> 4);
+        ssdt_ptr[ofs + 1] = getHex(i);
+        ssdt_ptr += 4;
+        *(ssdt_ptr++) = 0x69; // Arg1Op
+    }
+    return ssdt_ptr;
+}
+
+static void patch_pcihp(int slot, u8 *ssdt_ptr, u32 eject)
+{
+    ssdt_ptr[PCIHP_OFFSET_HEX] = getHex(slot >> 4);
+    ssdt_ptr[PCIHP_OFFSET_HEX+1] = getHex(slot);
+    ssdt_ptr[PCIHP_OFFSET_ID] = slot;
+    ssdt_ptr[PCIHP_OFFSET_ADR + 2] = slot;
+
+    /* Runtime patching of EJ0: to disable hotplug for a slot,
+     * replace the method name: _EJ0 by EJ0_. */
+    /* Sanity check */
+    if (memcmp(ssdt_ptr + PCIHP_OFFSET_EJ0, "_EJ0", 4)) {
+        warn_internalerror();
+    }
+    if (!eject) {
+        memcpy(ssdt_ptr + PCIHP_OFFSET_EJ0, "EJ0_", 4);
+    }
+}
 
 static void*
 build_ssdt(void)
 {
     int acpi_cpus = MaxCountCPUs > 0xff ? 0xff : MaxCountCPUs;
-    // length = ScopeOp + procs + NTYF method + CPON package
-    int length = ((1+3+4)
-                  + (acpi_cpus * SD_SIZEOF)
-                  + (1+2+5+(12*acpi_cpus))
-                  + (6+2+1+(1*acpi_cpus))
-                  + 17);
-    u8 *ssdt = malloc_high(sizeof(struct acpi_table_header) + length);
+    int length = (sizeof(ssdp_susp_aml)                     // _S3_ / _S4_ / _S5_
+                  + (1+3+4)                                 // Scope(_SB_)
+                  + (acpi_cpus * PROC_SIZEOF)               // procs
+                  + (1+2+5+(12*acpi_cpus))                  // NTFY
+                  + (6+2+1+(1*acpi_cpus))                   // CPON
+                  + 17                                      // BDAT
+                  + (1+3+4)                                 // Scope(PCI0)
+                  + ((PCI_SLOTS - 1) * PCIHP_SIZEOF)        // slots
+                  + (1+2+5+(12*(PCI_SLOTS - 1))));          // PCNT
+    u8 *ssdt = malloc_high(length);
     if (! ssdt) {
         warn_noalloc();
         return NULL;
     }
-    u8 *ssdt_ptr = ssdt + sizeof(struct acpi_table_header);
+    u8 *ssdt_ptr = ssdt;
+
+    // Copy header and encode fwcfg values in the S3_ / S4_ / S5_ packages
+    int sys_state_size;
+    char *sys_states = romfile_loadfile("etc/system-states", &sys_state_size);
+    if (!sys_states || sys_state_size != 6)
+        sys_states = (char[]){128, 0, 0, 129, 128, 128};
+
+    memcpy(ssdt_ptr, ssdp_susp_aml, sizeof(ssdp_susp_aml));
+    if (!(sys_states[3] & 128))
+        ssdt_ptr[acpi_s3_name[0]] = 'X';
+    if (!(sys_states[4] & 128))
+        ssdt_ptr[acpi_s4_name[0]] = 'X';
+    else
+        ssdt_ptr[acpi_s4_pkg[0] + 1] = ssdt[acpi_s4_pkg[0] + 3] = sys_states[4] & 127;
+    ssdt_ptr += sizeof(ssdp_susp_aml);
 
     // build Scope(_SB_) header
     *(ssdt_ptr++) = 0x10; // ScopeOp
-    ssdt_ptr = encodeLen(ssdt_ptr, length-1, 3);
+    ssdt_ptr = encodeLen(ssdt_ptr, length - (ssdt_ptr - ssdt), 3);
     *(ssdt_ptr++) = '_';
     *(ssdt_ptr++) = 'S';
     *(ssdt_ptr++) = 'B';
@@ -436,37 +518,17 @@ build_ssdt(void)
     // build Processor object for each processor
     int i;
     for (i=0; i<acpi_cpus; i++) {
-        memcpy(ssdt_ptr, SD_PROC, SD_SIZEOF);
-        ssdt_ptr[SD_OFFSET_CPUHEX] = getHex(i >> 4);
-        ssdt_ptr[SD_OFFSET_CPUHEX+1] = getHex(i);
-        ssdt_ptr[SD_OFFSET_CPUID1] = i;
-        ssdt_ptr[SD_OFFSET_CPUID2] = i;
-        ssdt_ptr += SD_SIZEOF;
+        memcpy(ssdt_ptr, PROC_AML, PROC_SIZEOF);
+        ssdt_ptr[PROC_OFFSET_CPUHEX] = getHex(i >> 4);
+        ssdt_ptr[PROC_OFFSET_CPUHEX+1] = getHex(i);
+        ssdt_ptr[PROC_OFFSET_CPUID1] = i;
+        ssdt_ptr[PROC_OFFSET_CPUID2] = i;
+        ssdt_ptr += PROC_SIZEOF;
     }
 
     // build "Method(NTFY, 2) {If (LEqual(Arg0, 0x00)) {Notify(CP00, Arg1)} ...}"
     // Arg0 = Processor ID = APIC ID
-    *(ssdt_ptr++) = 0x14; // MethodOp
-    ssdt_ptr = encodeLen(ssdt_ptr, 2+5+(12*acpi_cpus), 2);
-    *(ssdt_ptr++) = 'N';
-    *(ssdt_ptr++) = 'T';
-    *(ssdt_ptr++) = 'F';
-    *(ssdt_ptr++) = 'Y';
-    *(ssdt_ptr++) = 0x02;
-    for (i=0; i<acpi_cpus; i++) {
-        *(ssdt_ptr++) = 0xA0; // IfOp
-        ssdt_ptr = encodeLen(ssdt_ptr, 11, 1);
-        *(ssdt_ptr++) = 0x93; // LEqualOp
-        *(ssdt_ptr++) = 0x68; // Arg0Op
-        *(ssdt_ptr++) = 0x0A; // BytePrefix
-        *(ssdt_ptr++) = i;
-        *(ssdt_ptr++) = 0x86; // NotifyOp
-        *(ssdt_ptr++) = 'C';
-        *(ssdt_ptr++) = 'P';
-        *(ssdt_ptr++) = getHex(i >> 4);
-        *(ssdt_ptr++) = getHex(i);
-        *(ssdt_ptr++) = 0x69; // Arg1Op
-    }
+    ssdt_ptr = build_notify(ssdt_ptr, "NTFY", 0, acpi_cpus, "CP00", 2);
 
     // build "Name(CPON, Package() { One, One, ..., Zero, Zero, ... })"
     *(ssdt_ptr++) = 0x08; // NameOp
@@ -505,66 +567,28 @@ build_ssdt(void)
     *(u32*)ssdt_ptr = sizeof(struct bfld);
     ssdt_ptr += 4;
 
-    build_header((void*)ssdt, SSDT_SIGNATURE, ssdt_ptr - ssdt, 1);
-
-    //hexdump(ssdt, ssdt_ptr - ssdt);
-
-    return ssdt;
-}
-
-#include "ssdt-pcihp.hex"
-
-#define PCI_RMV_BASE 0xae0c
-
-extern void link_time_assertion(void);
-
-static void* build_pcihp(void)
-{
-    char *sys_states;
-    int sys_state_size;
-    u32 rmvc_pcrm;
-    int i;
-
-    u8 *ssdt = malloc_high(sizeof ssdp_pcihp_aml);
-    if (!ssdt) {
-        warn_noalloc();
-        return NULL;
+    // build Scope(PCI0) opcode
+    *(ssdt_ptr++) = 0x10; // ScopeOp
+    ssdt_ptr = encodeLen(ssdt_ptr, length - (ssdt_ptr - ssdt), 3);
+    *(ssdt_ptr++) = 'P';
+    *(ssdt_ptr++) = 'C';
+    *(ssdt_ptr++) = 'I';
+    *(ssdt_ptr++) = '0';
+
+    // build Device object for each slot
+    u32 rmvc_pcrm = inl(PCI_RMV_BASE);
+    for (i=1; i<PCI_SLOTS; i++) {
+        u32 eject = rmvc_pcrm & (0x1 << i);
+        memcpy(ssdt_ptr, PCIHP_AML, PCIHP_SIZEOF);
+        patch_pcihp(i, ssdt_ptr, eject != 0);
+        ssdt_ptr += PCIHP_SIZEOF;
     }
-    memcpy(ssdt, ssdp_pcihp_aml, sizeof ssdp_pcihp_aml);
 
-    /* Runtime patching of EJ0: to disable hotplug for a slot,
-     * replace the method name: _EJ0 by EJ0_. */
-    if (ARRAY_SIZE(aml_ej0_name) != ARRAY_SIZE(aml_adr_dword)) {
-        link_time_assertion();
-    }
+    ssdt_ptr = build_notify(ssdt_ptr, "PCNT", 1, PCI_SLOTS, "S00_", 1);
 
-    rmvc_pcrm = inl(PCI_RMV_BASE);
-    for (i = 0; i < ARRAY_SIZE(aml_ej0_name); ++i) {
-        /* Slot is in byte 2 in _ADR */
-        u8 slot = ssdp_pcihp_aml[aml_adr_dword[i] + 2] & 0x1F;
-        /* Sanity check */
-        if (memcmp(ssdp_pcihp_aml + aml_ej0_name[i], "_EJ0", 4)) {
-            warn_internalerror();
-            free(ssdt);
-            return NULL;
-        }
-        if (!(rmvc_pcrm & (0x1 << slot))) {
-            memcpy(ssdt + aml_ej0_name[i], "EJ0_", 4);
-        }
-    }
-
-    sys_states = romfile_loadfile("etc/system-states", &sys_state_size);
-    if (!sys_states || sys_state_size != 6)
-        sys_states = (char[]){128, 0, 0, 129, 128, 128};
+    build_header((void*)ssdt, SSDT_SIGNATURE, ssdt_ptr - ssdt, 1);
 
-    if (!(sys_states[3] & 128))
-        ssdt[acpi_s3_name[0]] = 'X';
-    if (!(sys_states[4] & 128))
-        ssdt[acpi_s4_name[0]] = 'X';
-    else
-        ssdt[acpi_s4_pkg[0] + 1] = ssdt[acpi_s4_pkg[0] + 3] = sys_states[4] & 127;
-    ((struct acpi_table_header*)ssdt)->checksum = 0;
-    ((struct acpi_table_header*)ssdt)->checksum -= checksum(ssdt, sizeof(ssdp_pcihp_aml));
+    //hexdump(ssdt, ssdt_ptr - ssdt);
 
     return ssdt;
 }
@@ -750,7 +774,6 @@ acpi_bios_init(void)
     ACPI_INIT_TABLE(build_madt());
     ACPI_INIT_TABLE(build_hpet());
     ACPI_INIT_TABLE(build_srat());
-    ACPI_INIT_TABLE(build_pcihp());
 
     u16 i, external_tables = qemu_cfg_acpi_additional_tables();
 
diff --git a/roms/seabios/src/ata.c b/roms/seabios/src/ata.c
index 7ff5f86c5..f604b3726 100644
--- a/roms/seabios/src/ata.c
+++ b/roms/seabios/src/ata.c
@@ -8,6 +8,7 @@
 #include "types.h" // u8
 #include "ioport.h" // inb
 #include "util.h" // dprintf
+#include "byteorder.h" // be16_to_cpu
 #include "cmos.h" // inb_cmos
 #include "pic.h" // enable_hwirq
 #include "biosvar.h" // GET_GLOBAL
@@ -696,7 +697,7 @@ ata_extract_model(char *model, u32 size, u16 *buffer)
     // Read model name
     int i;
     for (i=0; i<size/2; i++)
-        *(u16*)&model[i*2] = ntohs(buffer[27+i]);
+        *(u16*)&model[i*2] = be16_to_cpu(buffer[27+i]);
     model[size] = 0x00;
     nullTrailingSpace(model);
     return model;
diff --git a/roms/seabios/src/block.c b/roms/seabios/src/block.c
index 1c200ccaf..cfe5c33c4 100644
--- a/roms/seabios/src/block.c
+++ b/roms/seabios/src/block.c
@@ -335,6 +335,8 @@ process_op(struct disk_op_s *op)
     case DTYPE_UAS:
     case DTYPE_VIRTIO_SCSI:
     case DTYPE_LSI_SCSI:
+    case DTYPE_ESP_SCSI:
+    case DTYPE_MEGASAS:
         return process_scsi_op(op);
     default:
         op->count = 0;
diff --git a/roms/seabios/src/blockcmd.c b/roms/seabios/src/blockcmd.c
index 43656500e..81b191be6 100644
--- a/roms/seabios/src/blockcmd.c
+++ b/roms/seabios/src/blockcmd.c
@@ -6,7 +6,8 @@
 // This file may be distributed under the terms of the GNU LGPLv3 license.
 
 #include "biosvar.h" // GET_GLOBAL
-#include "util.h" // htonl
+#include "util.h" // dprintf
+#include "byteorder.h" // be32_to_cpu
 #include "disk.h" // struct disk_op_s
 #include "blockcmd.h" // struct cdb_request_sense
 #include "ata.h" // atapi_cmd_data
@@ -15,6 +16,8 @@
 #include "usb-uas.h" // usb_cmd_data
 #include "virtio-scsi.h" // virtio_scsi_cmd_data
 #include "lsi-scsi.h" // lsi_scsi_cmd_data
+#include "esp-scsi.h" // esp_scsi_cmd_data
+#include "megasas.h" // megasas_cmd_data
 #include "boot.h" // boot_add_hd
 
 // Route command to low-level handler.
@@ -35,6 +38,10 @@ cdb_cmd_data(struct disk_op_s *op, void *cdbcmd, u16 blocksize)
         return virtio_scsi_cmd_data(op, cdbcmd, blocksize);
     case DTYPE_LSI_SCSI:
         return lsi_scsi_cmd_data(op, cdbcmd, blocksize);
+    case DTYPE_ESP_SCSI:
+        return esp_scsi_cmd_data(op, cdbcmd, blocksize);
+    case DTYPE_MEGASAS:
+        return megasas_cmd_data(op, cdbcmd, blocksize);
     default:
         op->count = 0;
         return DISK_RET_EPARAM;
@@ -141,12 +148,12 @@ scsi_init_drive(struct drive_s *drive, const char *s, int prio)
     // READ CAPACITY returns the address of the last block.
     // We do not bother with READ CAPACITY(16) because BIOS does not support
     // 64-bit LBA anyway.
-    drive->blksize = ntohl(capdata.blksize);
+    drive->blksize = be32_to_cpu(capdata.blksize);
     if (drive->blksize != DISK_SECTOR_SIZE) {
         dprintf(1, "%s: unsupported block size %d\n", s, drive->blksize);
         return -1;
     }
-    drive->sectors = (u64)ntohl(capdata.sectors) + 1;
+    drive->sectors = (u64)be32_to_cpu(capdata.sectors) + 1;
     dprintf(1, "%s blksize=%d sectors=%d\n"
             , s, drive->blksize, (unsigned)drive->sectors);
 
@@ -240,7 +247,7 @@ cdb_mode_sense_geom(struct disk_op_s *op, struct cdbres_mode_sense_geom *data)
     cmd.command = CDB_CMD_MODE_SENSE;
     cmd.flags = 8; /* DBD */
     cmd.page = MODE_PAGE_HD_GEOMETRY;
-    cmd.count = htons(sizeof(*data));
+    cmd.count = cpu_to_be16(sizeof(*data));
     op->count = 1;
     op->buf_fl = data;
     return cdb_cmd_data(op, &cmd, sizeof(*data));
@@ -253,8 +260,8 @@ cdb_read(struct disk_op_s *op)
     struct cdb_rwdata_10 cmd;
     memset(&cmd, 0, sizeof(cmd));
     cmd.command = CDB_CMD_READ_10;
-    cmd.lba = htonl(op->lba);
-    cmd.count = htons(op->count);
+    cmd.lba = cpu_to_be32(op->lba);
+    cmd.count = cpu_to_be16(op->count);
     return cdb_cmd_data(op, &cmd, GET_GLOBAL(op->drive_g->blksize));
 }
 
@@ -265,7 +272,7 @@ cdb_write(struct disk_op_s *op)
     struct cdb_rwdata_10 cmd;
     memset(&cmd, 0, sizeof(cmd));
     cmd.command = CDB_CMD_WRITE_10;
-    cmd.lba = htonl(op->lba);
-    cmd.count = htons(op->count);
+    cmd.lba = cpu_to_be32(op->lba);
+    cmd.count = cpu_to_be16(op->count);
     return cdb_cmd_data(op, &cmd, GET_GLOBAL(op->drive_g->blksize));
 }
diff --git a/roms/seabios/src/bootsplash.c b/roms/seabios/src/bootsplash.c
index a85e2b2ec..78023a594 100644
--- a/roms/seabios/src/bootsplash.c
+++ b/roms/seabios/src/bootsplash.c
@@ -11,7 +11,8 @@
 #include "util.h" // dprintf
 #include "jpeg.h" // splash
 #include "vbe.h" // struct vbe_info
-#include "bmp.h"
+#include "bmp.h" // bmp_alloc
+
 
 /****************************************************************
  * Helper functions
@@ -63,7 +64,7 @@ find_videomode(struct vbe_info *vesa_info, struct vbe_mode_info *mode_info
         struct bregs br;
         memset(&br, 0, sizeof(br));
         br.ax = 0x4f01;
-        br.cx = (1 << 14) | videomode;
+        br.cx = videomode;
         br.di = FLATPTR_TO_OFFSET(mode_info);
         br.es = FLATPTR_TO_SEG(mode_info);
         call16_int10(&br);
@@ -216,7 +217,7 @@ enable_bootsplash(void)
     dprintf(5, "Switching to graphics mode\n");
     memset(&br, 0, sizeof(br));
     br.ax = 0x4f02;
-    br.bx = (1 << 14) | videomode;
+    br.bx = videomode | VBE_MODE_LINEAR_FRAME_BUFFER;
     call16_int10(&br);
     if (br.ax != 0x4f) {
         dprintf(1, "set_mode failed.\n");
diff --git a/roms/seabios/src/byteorder.h b/roms/seabios/src/byteorder.h
new file mode 100644
index 000000000..5a8a64a74
--- /dev/null
+++ b/roms/seabios/src/byteorder.h
@@ -0,0 +1,69 @@
+#ifndef __BYTEORDER_H
+#define __BYTEORDER_H
+
+static inline u16 __swab16_constant(u16 val) {
+    return (val<<8) | (val>>8);
+}
+static inline u32 __swab32_constant(u32 val) {
+    return (val<<24) | ((val&0xff00)<<8) | ((val&0xff0000)>>8) | (val>>24);
+}
+static inline u64 __swab64_constant(u64 val) {
+    return ((u64)__swab32_constant(val) << 32) | __swab32_constant(val>>32);
+}
+static inline u32 __swab32(u32 val) {
+    asm("bswapl %0" : "+r"(val));
+    return val;
+}
+static inline u64 __swab64(u64 val) {
+    union u64_u32_u i, o;
+    i.val = val;
+    o.lo = __swab32(i.hi);
+    o.hi = __swab32(i.lo);
+    return o.val;
+}
+
+#define swab16(x) __swab16_constant(x)
+#define swab32(x) (__builtin_constant_p((u32)(x)) \
+                   ? __swab32_constant(x) : __swab32(x))
+#define swab64(x) (__builtin_constant_p((u64)(x)) \
+                   ? __swab64_constant(x) : __swab64(x))
+
+static inline u16 cpu_to_le16(u16 x) {
+    return x;
+}
+static inline u32 cpu_to_le32(u32 x) {
+    return x;
+}
+static inline u64 cpu_to_le64(u64 x) {
+    return x;
+}
+static inline u16 le16_to_cpu(u16 x) {
+    return x;
+}
+static inline u32 le32_to_cpu(u32 x) {
+    return x;
+}
+static inline u32 le64_to_cpu(u64 x) {
+    return x;
+}
+
+static inline u16 cpu_to_be16(u16 x) {
+    return swab16(x);
+}
+static inline u32 cpu_to_be32(u32 x) {
+    return swab32(x);
+}
+static inline u64 cpu_to_be64(u64 x) {
+    return swab64(x);
+}
+static inline u16 be16_to_cpu(u16 x) {
+    return swab16(x);
+}
+static inline u32 be32_to_cpu(u32 x) {
+    return swab32(x);
+}
+static inline u32 be64_to_cpu(u64 x) {
+    return swab64(x);
+}
+
+#endif // byteorder.h
diff --git a/roms/seabios/src/clock.c b/roms/seabios/src/clock.c
index 69e9f178b..71b913e59 100644
--- a/roms/seabios/src/clock.c
+++ b/roms/seabios/src/clock.c
@@ -129,11 +129,42 @@ emulate_tsc(void)
     return ret;
 }
 
+u16 pmtimer_ioport VAR16VISIBLE;
+u32 pmtimer_wraps VARLOW;
+u32 pmtimer_last VARLOW;
+
+void pmtimer_init(u16 ioport, u32 khz)
+{
+    if (!CONFIG_PMTIMER)
+        return;
+    dprintf(1, "Using pmtimer, ioport 0x%x, freq %d kHz\n", ioport, khz);
+    SET_GLOBAL(pmtimer_ioport, ioport);
+    SET_GLOBAL(cpu_khz, khz);
+}
+
+static u64 pmtimer_get(void)
+{
+    u16 ioport = GET_GLOBAL(pmtimer_ioport);
+    u32 wraps = GET_LOW(pmtimer_wraps);
+    u32 pmtimer = inl(ioport) & 0xffffff;
+
+    if (pmtimer < GET_LOW(pmtimer_last)) {
+        wraps++;
+        SET_LOW(pmtimer_wraps, wraps);
+    }
+    SET_LOW(pmtimer_last, pmtimer);
+
+    dprintf(9, "pmtimer: %u:%u\n", wraps, pmtimer);
+    return (u64)wraps << 24 | pmtimer;
+}
+
 static u64
 get_tsc(void)
 {
     if (unlikely(GET_GLOBAL(no_tsc)))
         return emulate_tsc();
+    if (CONFIG_PMTIMER && GET_GLOBAL(pmtimer_ioport))
+        return pmtimer_get();
     return rdtscll();
 }
 
diff --git a/roms/seabios/src/coreboot.c b/roms/seabios/src/coreboot.c
index 55590cef6..e4767f116 100644
--- a/roms/seabios/src/coreboot.c
+++ b/roms/seabios/src/coreboot.c
@@ -6,6 +6,7 @@
 
 #include "memmap.h" // add_e820
 #include "util.h" // dprintf
+#include "byteorder.h" // be32_to_cpu
 #include "lzmadecode.h" // LzmaDecode
 #include "smbios.h" // smbios_init
 #include "boot.h" // boot_add_cbfs
@@ -329,17 +330,17 @@ coreboot_cbfs_setup(void)
         return;
 
     struct cbfs_header *hdr = *(void **)CBFS_HEADPTR_ADDR;
-    if (hdr->magic != htonl(CBFS_HEADER_MAGIC)) {
+    if (hdr->magic != cpu_to_be32(CBFS_HEADER_MAGIC)) {
         dprintf(1, "Unable to find CBFS (ptr=%p; got %x not %x)\n"
-                , hdr, hdr->magic, htonl(CBFS_HEADER_MAGIC));
+                , hdr, hdr->magic, cpu_to_be32(CBFS_HEADER_MAGIC));
         return;
     }
     dprintf(1, "Found CBFS header at %p\n", hdr);
 
-    struct cbfs_file *cfile = (void *)(0 - ntohl(hdr->romsize)
-                                       + ntohl(hdr->offset));
+    struct cbfs_file *cfile = (void *)(0 - be32_to_cpu(hdr->romsize)
+                                       + be32_to_cpu(hdr->offset));
     for (;;) {
-        if (cfile < (struct cbfs_file *)(0xFFFFFFFF - ntohl(hdr->romsize)))
+        if (cfile < (struct cbfs_file *)(0xFFFFFFFF - be32_to_cpu(hdr->romsize)))
             break;
         u64 magic = cfile->magic;
         if (magic != CBFS_FILE_MAGIC)
@@ -352,10 +353,10 @@ coreboot_cbfs_setup(void)
         memset(file, 0, sizeof(*file));
         strtcpy(file->name, cfile->filename, sizeof(file->name));
         dprintf(3, "Found CBFS file: %s\n", file->name);
-        file->size = file->rawsize = ntohl(cfile->len);
+        file->size = file->rawsize = be32_to_cpu(cfile->len);
         file->id = (u32)cfile;
         file->copy = cbfs_copyfile;
-        file->data = (void*)cfile + ntohl(cfile->offset);
+        file->data = (void*)cfile + be32_to_cpu(cfile->offset);
         int len = strlen(file->name);
         if (len > 5 && strcmp(&file->name[len-5], ".lzma") == 0) {
             // Using compression.
@@ -365,7 +366,7 @@ coreboot_cbfs_setup(void)
         }
         romfile_add(file);
 
-        cfile = (void*)ALIGN((u32)file->data + file->size, ntohl(hdr->align));
+        cfile = (void*)ALIGN((u32)file->data + file->size, be32_to_cpu(hdr->align));
     }
 }
 
@@ -394,13 +395,13 @@ cbfs_run_payload(struct cbfs_file *file)
     if (!CONFIG_COREBOOT || !CONFIG_COREBOOT_FLASH || !file)
         return;
     dprintf(1, "Run %s\n", file->filename);
-    struct cbfs_payload *pay = (void*)file + ntohl(file->offset);
+    struct cbfs_payload *pay = (void*)file + be32_to_cpu(file->offset);
     struct cbfs_payload_segment *seg = pay->segments;
     for (;;) {
-        void *src = (void*)pay + ntohl(seg->offset);
-        void *dest = (void*)ntohl((u32)seg->load_addr);
-        u32 src_len = ntohl(seg->len);
-        u32 dest_len = ntohl(seg->mem_len);
+        void *src = (void*)pay + be32_to_cpu(seg->offset);
+        void *dest = (void*)(u32)be64_to_cpu(seg->load_addr);
+        u32 src_len = be32_to_cpu(seg->len);
+        u32 dest_len = be32_to_cpu(seg->mem_len);
         switch (seg->type) {
         case PAYLOAD_SEGMENT_BSS:
             dprintf(3, "BSS segment %d@%p\n", dest_len, dest);
@@ -415,12 +416,12 @@ cbfs_run_payload(struct cbfs_file *file)
         default:
             dprintf(3, "Segment %x %d@%p -> %d@%p\n"
                     , seg->type, src_len, src, dest_len, dest);
-            if (seg->compression == htonl(CBFS_COMPRESS_NONE)) {
+            if (seg->compression == cpu_to_be32(CBFS_COMPRESS_NONE)) {
                 if (src_len > dest_len)
                     src_len = dest_len;
                 memcpy(dest, src, src_len);
             } else if (CONFIG_LZMA
-                       && seg->compression == htonl(CBFS_COMPRESS_LZMA)) {
+                       && seg->compression == cpu_to_be32(CBFS_COMPRESS_LZMA)) {
                 int ret = ulzma(dest, dest_len, src, src_len);
                 if (ret < 0)
                     return;
diff --git a/roms/seabios/src/disk.c b/roms/seabios/src/disk.c
index 8e1d3ec56..0291fe3e0 100644
--- a/roms/seabios/src/disk.c
+++ b/roms/seabios/src/disk.c
@@ -615,7 +615,7 @@ disk_1348(struct bregs *regs, struct drive_s *drive_g)
         u8 sum = checksum_far(SEG_LOW, &DefaultDPTE, 15);
         SET_LOW(DefaultDPTE.checksum, -sum);
     } else {
-        SET_FARVAR(seg, param_far->dpte.segoff, 0);
+        SET_FARVAR(seg, param_far->dpte.segoff, 0xffffffff);
         bdf = GET_GLOBAL(drive_g->cntl_id);
     }
 
diff --git a/roms/seabios/src/disk.h b/roms/seabios/src/disk.h
index 2b2511f90..21debec0f 100644
--- a/roms/seabios/src/disk.h
+++ b/roms/seabios/src/disk.h
@@ -234,6 +234,8 @@ struct drive_s {
 #define DTYPE_USB          0x0a
 #define DTYPE_UAS          0x0b
 #define DTYPE_LSI_SCSI     0x0c
+#define DTYPE_ESP_SCSI     0x0d
+#define DTYPE_MEGASAS      0x0e
 
 #define MAXDESCSIZE 80
 
diff --git a/roms/seabios/src/esp-scsi.c b/roms/seabios/src/esp-scsi.c
new file mode 100644
index 000000000..c43e55b3b
--- /dev/null
+++ b/roms/seabios/src/esp-scsi.c
@@ -0,0 +1,233 @@
+// AMD PCscsi boot support.
+//
+// Copyright (C) 2012 Red Hat Inc.
+//
+// Authors:
+//  Paolo Bonzini <pbonzini@redhat.com>
+//
+// based on lsi-scsi.c which is written by:
+//  Gerd Hoffman <kraxel@redhat.com>
+//
+// This file may be distributed under the terms of the GNU LGPLv3 license.
+
+#include "util.h" // dprintf
+#include "pci.h" // foreachpci
+#include "config.h" // CONFIG_*
+#include "biosvar.h" // GET_GLOBAL
+#include "pci_ids.h" // PCI_DEVICE_ID
+#include "pci_regs.h" // PCI_VENDOR_ID
+#include "boot.h" // bootprio_find_scsi_device
+#include "blockcmd.h" // scsi_init_drive
+#include "disk.h"
+
+#define ESP_TCLO      0x00
+#define ESP_TCMID     0x04
+#define ESP_FIFO      0x08
+#define ESP_CMD       0x0c
+#define ESP_WBUSID    0x10
+#define ESP_TCHI      0x38
+
+#define ESP_RSTAT     0x10
+#define ESP_RINTR     0x14
+#define ESP_RFLAGS    0x1c
+
+#define ESP_DMA_CMD   0x40
+#define ESP_DMA_STC   0x44
+#define ESP_DMA_SPA   0x48
+#define ESP_DMA_WBC   0x4c
+#define ESP_DMA_WAC   0x50
+#define ESP_DMA_STAT  0x54
+#define ESP_DMA_SMDLA 0x58
+#define ESP_DMA_WMAC  0x58c
+
+#define ESP_CMD_DMA      0x80
+#define ESP_CMD_RESET    0x02
+#define ESP_CMD_TI       0x10
+#define ESP_CMD_ICCS     0x11
+#define ESP_CMD_SELATN   0x42
+
+#define ESP_STAT_DI      0x01
+#define ESP_STAT_CD      0x02
+#define ESP_STAT_MSG     0x04
+#define ESP_STAT_TC      0x10
+
+#define ESP_INTR_DC      0x20
+
+struct esp_lun_s {
+    struct drive_s drive;
+    struct pci_device *pci;
+    u32 iobase;
+    u8 target;
+    u8 lun;
+};
+
+static void
+esp_scsi_dma(u32 iobase, u32 buf, u32 len, int read)
+{
+    outb(len         & 0xff, iobase + ESP_TCLO);
+    outb((len >> 8)  & 0xff, iobase + ESP_TCMID);
+    outb((len >> 16) & 0xff, iobase + ESP_TCHI);
+    outl(buf,                iobase + ESP_DMA_SPA);
+    outl(len,                iobase + ESP_DMA_STC);
+    outb(read ? 0x83 : 0x03, iobase + ESP_DMA_CMD);
+}
+
+static int
+esp_scsi_cmd(struct esp_lun_s *llun, struct disk_op_s *op,
+             u8 *cdbcmd, u16 target, u16 lun, u16 blocksize)
+{
+    u32 iobase = GET_GLOBAL(llun->iobase);
+    int i, state;
+    u8 status;
+
+    outb(target, iobase + ESP_WBUSID);
+
+    /*
+     * We need to pass the LUN at the beginning of the command, and the FIFO
+     * is only 16 bytes, so we cannot support 16-byte CDBs.  The alternative
+     * would be to use DMA for the 17-byte command too, which is quite
+     * overkill.
+     */
+    outb(lun, iobase + ESP_FIFO);
+    cdbcmd[1] &= 0x1f;
+    cdbcmd[1] |= lun << 5;
+    for (i = 0; i < 12; i++)
+        outb(cdbcmd[i], iobase + ESP_FIFO);
+    outb(ESP_CMD_SELATN, iobase + ESP_CMD);
+
+    for (state = 0;;) {
+        u8 stat = inb(iobase + ESP_RSTAT);
+
+        /* Detect disconnected device.  */
+        if (state == 0 && (inb(iobase + ESP_RINTR) & ESP_INTR_DC)) {
+            return DISK_RET_ENOTREADY;
+        }
+
+        /* HBA reads command, clears CD, sets TC -> do DMA if needed.  */
+        if (state == 0 && (stat & ESP_STAT_TC)) {
+            state++;
+            if (op->count && blocksize) {
+                /* Data phase.  */
+                u32 count = (u32)op->count * blocksize;
+                esp_scsi_dma(iobase, (u32)op->buf_fl, count,
+                             cdb_is_read(cdbcmd, blocksize));
+                outb(ESP_CMD_TI | ESP_CMD_DMA, iobase + ESP_CMD);
+                continue;
+            }
+        }
+
+	/* At end of DMA TC is set again -> complete command.  */
+        if (state == 1 && (stat & ESP_STAT_TC)) {
+            state++;
+            outb(ESP_CMD_ICCS, iobase + ESP_CMD);
+            continue;
+        }
+
+	/* Finally read data from the message in phase.  */
+        if (state == 2 && (stat & ESP_STAT_MSG)) {
+            state++;
+            status = inb(iobase + ESP_FIFO);
+            inb(iobase + ESP_FIFO);
+            break;
+        }
+        usleep(5);
+    }
+
+    if (status == 0) {
+        return DISK_RET_SUCCESS;
+    }
+
+    return DISK_RET_EBADTRACK;
+}
+
+int
+esp_scsi_cmd_data(struct disk_op_s *op, void *cdbcmd, u16 blocksize)
+{
+    if (!CONFIG_ESP_SCSI)
+        return DISK_RET_EBADTRACK;
+
+    struct esp_lun_s *llun =
+        container_of(op->drive_g, struct esp_lun_s, drive);
+
+    return esp_scsi_cmd(llun, op, cdbcmd,
+                        GET_GLOBAL(llun->target), GET_GLOBAL(llun->lun),
+                        blocksize);
+}
+
+static int
+esp_scsi_add_lun(struct pci_device *pci, u32 iobase, u8 target, u8 lun)
+{
+    struct esp_lun_s *llun = malloc_fseg(sizeof(*llun));
+    if (!llun) {
+        warn_noalloc();
+        return -1;
+    }
+    memset(llun, 0, sizeof(*llun));
+    llun->drive.type = DTYPE_ESP_SCSI;
+    llun->drive.cntl_id = pci->bdf;
+    llun->pci = pci;
+    llun->target = target;
+    llun->lun = lun;
+    llun->iobase = iobase;
+
+    char *name = znprintf(16, "esp %02x:%02x.%x %d:%d",
+                          pci_bdf_to_bus(pci->bdf), pci_bdf_to_dev(pci->bdf),
+                          pci_bdf_to_fn(pci->bdf), target, lun);
+    int prio = bootprio_find_scsi_device(pci, target, lun);
+    int ret = scsi_init_drive(&llun->drive, name, prio);
+    free(name);
+    if (ret)
+        goto fail;
+    return 0;
+
+fail:
+    free(llun);
+    return -1;
+}
+
+static void
+esp_scsi_scan_target(struct pci_device *pci, u32 iobase, u8 target)
+{
+    esp_scsi_add_lun(pci, iobase, target, 0);
+}
+
+static void
+init_esp_scsi(struct pci_device *pci)
+{
+    u16 bdf = pci->bdf;
+    u32 iobase = pci_config_readl(pci->bdf, PCI_BASE_ADDRESS_0)
+        & PCI_BASE_ADDRESS_IO_MASK;
+
+    dprintf(1, "found esp at %02x:%02x.%x, io @ %x\n",
+            pci_bdf_to_bus(bdf), pci_bdf_to_dev(bdf),
+            pci_bdf_to_fn(bdf), iobase);
+
+    pci_config_maskw(bdf, PCI_COMMAND, 0, PCI_COMMAND_MASTER);
+
+    // reset
+    outb(ESP_CMD_RESET, iobase + ESP_CMD);
+
+    int i;
+    for (i = 0; i <= 7; i++)
+        esp_scsi_scan_target(pci, iobase, i);
+
+    return;
+}
+
+void
+esp_scsi_setup(void)
+{
+    ASSERT32FLAT();
+    if (!CONFIG_ESP_SCSI)
+        return;
+
+    dprintf(3, "init esp\n");
+
+    struct pci_device *pci;
+    foreachpci(pci) {
+        if (pci->vendor != PCI_VENDOR_ID_AMD
+            || pci->device != PCI_DEVICE_ID_AMD_SCSI)
+            continue;
+        init_esp_scsi(pci);
+    }
+}
diff --git a/roms/seabios/src/esp-scsi.h b/roms/seabios/src/esp-scsi.h
new file mode 100644
index 000000000..dc555f395
--- /dev/null
+++ b/roms/seabios/src/esp-scsi.h
@@ -0,0 +1,8 @@
+#ifndef __ESP_SCSI_H
+#define __ESP_SCSI_H
+
+struct disk_op_s;
+int esp_scsi_cmd_data(struct disk_op_s *op, void *cdbcmd, u16 blocksize);
+void esp_scsi_setup(void);
+
+#endif /* __ESP_SCSI_H */
diff --git a/roms/seabios/src/farptr.h b/roms/seabios/src/farptr.h
index 3a85c6b83..5b6c5c109 100644
--- a/roms/seabios/src/farptr.h
+++ b/roms/seabios/src/farptr.h
@@ -26,8 +26,8 @@ extern u16 __segment_FS, __segment_GS;
 #define READ64_SEG(prefix, SEG, value, var) do {                \
         union u64_u32_u __value;                                \
         union u64_u32_u *__r64_ptr = (union u64_u32_u *)&(var); \
-        READ32_SEG(prefix, SEG, __value.hi, __r64_ptr->hi);     \
         READ32_SEG(prefix, SEG, __value.lo, __r64_ptr->lo);     \
+        READ32_SEG(prefix, SEG, __value.hi, __r64_ptr->hi);     \
         *(u64*)&(value) = __value.val;                          \
     } while (0)
 #define WRITE8_SEG(prefix, SEG, var, value)                     \
@@ -44,8 +44,8 @@ extern u16 __segment_FS, __segment_GS;
         union u64_u32_u *__w64_ptr = (union u64_u32_u *)&(var); \
         typeof(var) __value_tmp = (value);                      \
         __value.val = *(u64*)&__value_tmp;                      \
-        WRITE32_SEG(prefix, SEG, __w64_ptr->hi, __value.hi);    \
         WRITE32_SEG(prefix, SEG, __w64_ptr->lo, __value.lo);    \
+        WRITE32_SEG(prefix, SEG, __w64_ptr->hi, __value.hi);    \
     } while (0)
 
 // Macros for automatically choosing the appropriate memory size
diff --git a/roms/seabios/src/lsi-scsi.c b/roms/seabios/src/lsi-scsi.c
index b70b3a5a6..f8d715b83 100644
--- a/roms/seabios/src/lsi-scsi.c
+++ b/roms/seabios/src/lsi-scsi.c
@@ -177,6 +177,8 @@ init_lsi_scsi(struct pci_device *pci)
     u32 iobase = pci_config_readl(pci->bdf, PCI_BASE_ADDRESS_0)
         & PCI_BASE_ADDRESS_IO_MASK;
 
+    pci_config_maskw(bdf, PCI_COMMAND, 0, PCI_COMMAND_MASTER);
+
     dprintf(1, "found lsi53c895a at %02x:%02x.%x, io @ %x\n",
             pci_bdf_to_bus(bdf), pci_bdf_to_dev(bdf),
             pci_bdf_to_fn(bdf), iobase);
diff --git a/roms/seabios/src/megasas.c b/roms/seabios/src/megasas.c
new file mode 100644
index 000000000..f710cd7a8
--- /dev/null
+++ b/roms/seabios/src/megasas.c
@@ -0,0 +1,399 @@
+// MegaRAID SAS boot support.
+//
+// Copyright (C) 2012 Hannes Reinecke, SUSE Linux Products GmbH
+//
+// Authors:
+//  Hannes Reinecke <hare@suse.de>
+//
+// based on virtio-scsi.c which is written by:
+//  Paolo Bonzini <pbonzini@redhat.com>
+//
+// This file may be distributed under the terms of the GNU LGPLv3 license.
+
+#include "util.h" // dprintf
+#include "pci.h" // foreachpci
+#include "config.h" // CONFIG_*
+#include "biosvar.h" // GET_GLOBAL
+#include "pci_ids.h" // PCI_DEVICE_ID_XXX
+#include "pci_regs.h" // PCI_VENDOR_ID
+#include "boot.h" // bootprio_find_scsi_device
+#include "blockcmd.h" // scsi_init_drive
+#include "disk.h"
+
+#define MFI_DB 0x0 // Doorbell
+#define MFI_OMSG0 0x18 // Outbound message 0
+#define MFI_IDB 0x20 // Inbound doorbell
+#define MFI_ODB 0x2c // Outbound doorbell
+#define MFI_IQP 0x40 // Inbound queue port
+#define MFI_OSP0 0xb0 // Outbound scratch pad0
+#define MFI_IQPL 0xc0 // Inbound queue port (low bytes)
+#define MFI_IQPH 0xc4 // Inbound queue port (high bytes)
+
+#define MFI_STATE_MASK                0xf0000000
+#define MFI_STATE_WAIT_HANDSHAKE      0x60000000
+#define MFI_STATE_BOOT_MESSAGE_PENDING 0x90000000
+#define MFI_STATE_READY               0xb0000000
+#define MFI_STATE_OPERATIONAL         0xc0000000
+#define MFI_STATE_FAULT               0xf0000000
+
+/* MFI Commands */
+typedef enum {
+    MFI_CMD_INIT = 0x00,
+    MFI_CMD_LD_READ,
+    MFI_CMD_LD_WRITE,
+    MFI_CMD_LD_SCSI_IO,
+    MFI_CMD_PD_SCSI_IO,
+    MFI_CMD_DCMD,
+    MFI_CMD_ABORT,
+    MFI_CMD_SMP,
+    MFI_CMD_STP
+} mfi_cmd_t;
+
+struct megasas_cmd_frame {
+    u8 cmd;             /*00h */
+    u8 sense_len;       /*01h */
+    u8 cmd_status;      /*02h */
+    u8 scsi_status;     /*03h */
+
+    u8 target_id;       /*04h */
+    u8 lun;             /*05h */
+    u8 cdb_len;         /*06h */
+    u8 sge_count;       /*07h */
+
+    u32 context;        /*08h */
+    u32 context_64;     /*0Ch */
+
+    u16 flags;          /*10h */
+    u16 timeout;        /*12h */
+    u32 data_xfer_len;   /*14h */
+
+    union {
+        struct {
+            u32 opcode;       /*18h */
+            u8 mbox[12];      /*1Ch */
+            u32 sgl_addr;     /*28h */
+            u32 sgl_len;      /*32h */
+            u32 pad;          /*34h */
+        } dcmd;
+        struct {
+            u32 sense_buf_lo; /*18h */
+            u32 sense_buf_hi; /*1Ch */
+            u8 cdb[16];       /*20h */
+            u32 sgl_addr;     /*30h */
+            u32 sgl_len;      /*34h */
+        } pthru;
+        struct {
+            u8 pad[22];       /*18h */
+        } gen;
+    };
+} __attribute__ ((packed));
+
+struct mfi_ld_list_s {
+    u32     count;
+    u32     reserved_0;
+    struct {
+        u8          target;
+        u8          lun;
+        u16         seq;
+        u8          state;
+        u8          reserved_1[3];
+        u64         size;
+    } lds[64];
+} __attribute__ ((packed));
+
+#define MEGASAS_POLL_TIMEOUT 60000 // 60 seconds polling timeout
+
+struct megasas_lun_s {
+    struct drive_s drive;
+    struct pci_device *pci;
+    struct megasas_cmd_frame *frame;
+    u32 iobase;
+    u8 target;
+    u8 lun;
+};
+
+static int megasas_fire_cmd(u16 pci_id, u32 ioaddr,
+                            struct megasas_cmd_frame *frame)
+{
+    u32 frame_addr = (u32)frame;
+    int frame_count = 1;
+    u8 cmd_state;
+    u64 end;
+
+    dprintf(2, "Frame 0x%x\n", frame_addr);
+    if (pci_id == PCI_DEVICE_ID_LSI_SAS2004 ||
+        pci_id == PCI_DEVICE_ID_LSI_SAS2008) {
+        outl(0, ioaddr + MFI_IQPH);
+        outl(frame_addr | frame_count << 1 | 1, ioaddr + MFI_IQPL);
+    } else if (pci_id == PCI_DEVICE_ID_DELL_PERC5 ||
+               pci_id == PCI_DEVICE_ID_LSI_SAS1064R ||
+               pci_id == PCI_DEVICE_ID_LSI_VERDE_ZCR) {
+        outl(frame_addr >> 3 | frame_count, ioaddr + MFI_IQP);
+    } else {
+        outl(frame_addr | frame_count << 1 | 1, ioaddr + MFI_IQP);
+    }
+
+    end = calc_future_tsc(MEGASAS_POLL_TIMEOUT);
+    do {
+        for (;;) {
+            cmd_state = GET_LOWFLAT(frame->cmd_status);
+            if (cmd_state != 0xff)
+                break;
+            if (check_tsc(end)) {
+                warn_timeout();
+                return -1;
+            }
+            yield();
+        }
+    } while (cmd_state == 0xff);
+
+    if (cmd_state == 0 || cmd_state == 0x2d)
+        return 0;
+    dprintf(1, "ERROR: Frame 0x%x, status 0x%x\n", frame_addr, cmd_state);
+    return -1;
+}
+
+int
+megasas_cmd_data(struct disk_op_s *op, void *cdbcmd, u16 blocksize)
+{
+    struct megasas_lun_s *mlun =
+        container_of(op->drive_g, struct megasas_lun_s, drive);
+    u8 *cdb = cdbcmd;
+    struct megasas_cmd_frame *frame = GET_GLOBAL(mlun->frame);
+    u16 pci_id = GET_GLOBAL(mlun->pci->device);
+    int i;
+
+    if (!CONFIG_MEGASAS)
+        return DISK_RET_EBADTRACK;
+
+    memset_fl(frame, 0, sizeof(*frame));
+    SET_LOWFLAT(frame->cmd, MFI_CMD_LD_SCSI_IO);
+    SET_LOWFLAT(frame->cmd_status, 0xFF);
+    SET_LOWFLAT(frame->target_id, GET_GLOBAL(mlun->target));
+    SET_LOWFLAT(frame->lun, GET_GLOBAL(mlun->lun));
+    SET_LOWFLAT(frame->flags, 0x0001);
+    SET_LOWFLAT(frame->data_xfer_len, op->count * blocksize);
+    SET_LOWFLAT(frame->cdb_len, 16);
+
+    for (i = 0; i < 16; i++) {
+        SET_LOWFLAT(frame->pthru.cdb[i], cdb[i]);
+    }
+    dprintf(2, "pthru cmd 0x%x count %d bs %d\n",
+            cdb[0], op->count, blocksize);
+
+    if (op->count) {
+        SET_LOWFLAT(frame->pthru.sgl_addr, (u32)op->buf_fl);
+        SET_LOWFLAT(frame->pthru.sgl_len, op->count * blocksize);
+        SET_LOWFLAT(frame->sge_count, 1);
+    }
+    SET_LOWFLAT(frame->context, (u32)frame);
+
+    if (megasas_fire_cmd(pci_id, GET_GLOBAL(mlun->iobase), frame) == 0)
+        return DISK_RET_SUCCESS;
+
+    dprintf(2, "pthru cmd 0x%x failed\n", cdb[0]);
+    return DISK_RET_EBADTRACK;
+}
+
+static int
+megasas_add_lun(struct pci_device *pci, u32 iobase, u8 target, u8 lun)
+{
+    struct megasas_lun_s *mlun = malloc_fseg(sizeof(*mlun));
+    char *name;
+    int prio, ret = 0;
+
+    if (!mlun) {
+        warn_noalloc();
+        return -1;
+    }
+    memset(mlun, 0, sizeof(*mlun));
+    mlun->drive.type = DTYPE_MEGASAS;
+    mlun->drive.cntl_id = pci->bdf;
+    mlun->pci = pci;
+    mlun->target = target;
+    mlun->lun = lun;
+    mlun->iobase = iobase;
+    mlun->frame = memalign_low(256, sizeof(struct megasas_cmd_frame));
+    if (!mlun->frame) {
+        warn_noalloc();
+        free(mlun);
+        return -1;
+    }
+    name = znprintf(36, "MegaRAID SAS (PCI %02x:%02x.%x) LD %d:%d",
+                    pci_bdf_to_bus(pci->bdf), pci_bdf_to_dev(pci->bdf),
+                    pci_bdf_to_fn(pci->bdf), target, lun);
+    prio = bootprio_find_scsi_device(pci, target, lun);
+    ret = scsi_init_drive(&mlun->drive, name, prio);
+    free(name);
+    if (ret) {
+        free(mlun->frame);
+        free(mlun);
+        ret = -1;
+    }
+
+    return ret;
+}
+
+static void megasas_scan_target(struct pci_device *pci, u32 iobase)
+{
+    struct mfi_ld_list_s ld_list;
+    struct megasas_cmd_frame *frame = memalign_tmp(256, sizeof(*frame));
+    int i;
+
+    memset(&ld_list, 0, sizeof(ld_list));
+    memset_fl(frame, 0, sizeof(*frame));
+
+    frame->cmd = MFI_CMD_DCMD;
+    frame->cmd_status = 0xFF;
+    frame->sge_count = 1;
+    frame->flags = 0x0011;
+    frame->data_xfer_len = sizeof(ld_list);
+    frame->dcmd.opcode = 0x03010000;
+    frame->dcmd.sgl_addr = (u32)MAKE_FLATPTR(GET_SEG(SS), &ld_list);
+    frame->dcmd.sgl_len = sizeof(ld_list);
+    frame->context = (u32)frame;
+
+    if (megasas_fire_cmd(pci->device, iobase, frame) == 0) {
+        dprintf(2, "%d LD found\n", ld_list.count);
+        for (i = 0; i < ld_list.count; i++) {
+            dprintf(2, "LD %d:%d state 0x%x\n",
+                    ld_list.lds[i].target, ld_list.lds[i].lun,
+                    ld_list.lds[i].state);
+            if (ld_list.lds[i].state != 0) {
+                megasas_add_lun(pci, iobase,
+                                ld_list.lds[i].target, ld_list.lds[i].lun);
+            }
+        }
+    }
+}
+
+static int megasas_transition_to_ready(struct pci_device *pci, u32 ioaddr)
+{
+    u32 fw_state = 0, new_state, mfi_flags = 0;
+    u64 end;
+
+    if (pci->device == PCI_DEVICE_ID_LSI_SAS1064R ||
+        pci->device == PCI_DEVICE_ID_DELL_PERC5)
+        new_state = inl(ioaddr + MFI_OMSG0) & MFI_STATE_MASK;
+    else
+        new_state = inl(ioaddr + MFI_OSP0) & MFI_STATE_MASK;
+
+    while (fw_state != new_state) {
+        switch (new_state) {
+        case MFI_STATE_FAULT:
+            dprintf(1, "ERROR: fw in fault state\n");
+            return -1;
+            break;
+        case MFI_STATE_WAIT_HANDSHAKE:
+            mfi_flags = 0x08;
+            /* fallthrough */
+        case MFI_STATE_BOOT_MESSAGE_PENDING:
+            mfi_flags |= 0x10;
+            if (pci->device == PCI_DEVICE_ID_LSI_SAS2004 ||
+                pci->device == PCI_DEVICE_ID_LSI_SAS2008 ||
+                pci->device == PCI_DEVICE_ID_LSI_SAS2208 ||
+                pci->device == PCI_DEVICE_ID_LSI_SAS3108) {
+                outl(ioaddr + MFI_DB, mfi_flags);
+            } else {
+                outl(ioaddr + MFI_IDB, mfi_flags);
+            }
+            break;
+        case MFI_STATE_OPERATIONAL:
+            mfi_flags = 0x07;
+            if (pci->device == PCI_DEVICE_ID_LSI_SAS2004 ||
+                pci->device == PCI_DEVICE_ID_LSI_SAS2008 ||
+                pci->device == PCI_DEVICE_ID_LSI_SAS2208 ||
+                pci->device == PCI_DEVICE_ID_LSI_SAS3108) {
+                outl(ioaddr + MFI_DB, mfi_flags);
+                if (pci->device == PCI_DEVICE_ID_LSI_SAS2208 ||
+                    pci->device == PCI_DEVICE_ID_LSI_SAS3108) {
+                    int j = 0;
+                    u32 doorbell;
+
+                    while (j < MEGASAS_POLL_TIMEOUT) {
+                        doorbell = inl(ioaddr + MFI_DB) & 1;
+                        if (!doorbell)
+                            break;
+                        msleep(20);
+                        j++;
+                    }
+                }
+            } else {
+                outw(ioaddr + MFI_IDB, mfi_flags);
+            }
+            break;
+        case MFI_STATE_READY:
+            dprintf(2, "MegaRAID SAS fw ready\n");
+            return 0;
+        }
+        // The current state should not last longer than poll timeout
+        end = calc_future_tsc(MEGASAS_POLL_TIMEOUT);
+        for (;;) {
+            if (check_tsc(end)) {
+                break;
+            }
+            yield();
+            fw_state = new_state;
+            if (pci->device == PCI_DEVICE_ID_LSI_SAS1064R ||
+                pci->device == PCI_DEVICE_ID_DELL_PERC5)
+                new_state = inl(ioaddr + MFI_OMSG0) & MFI_STATE_MASK;
+            else
+                new_state = inl(ioaddr + MFI_OSP0) & MFI_STATE_MASK;
+            if (new_state != fw_state) {
+                break;
+            }
+        }
+    }
+    dprintf(1, "ERROR: fw in state %x\n", new_state & MFI_STATE_MASK);
+    return -1;
+}
+
+static void
+init_megasas(struct pci_device *pci)
+{
+    u16 bdf = pci->bdf;
+    u32 iobase = pci_config_readl(pci->bdf, PCI_BASE_ADDRESS_2)
+        & PCI_BASE_ADDRESS_IO_MASK;
+
+    dprintf(1, "found MegaRAID SAS at %02x:%02x.%x, io @ %x\n",
+            pci_bdf_to_bus(bdf), pci_bdf_to_dev(bdf),
+            pci_bdf_to_fn(bdf), iobase);
+
+    pci_config_maskw(pci->bdf, PCI_COMMAND, 0,
+                     PCI_COMMAND_IO | PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER);
+    // reset
+    if (megasas_transition_to_ready(pci, iobase) == 0)
+        megasas_scan_target(pci, iobase);
+
+    return;
+}
+
+void
+megasas_setup(void)
+{
+    ASSERT32FLAT();
+    if (!CONFIG_MEGASAS)
+        return;
+
+    dprintf(3, "init megasas\n");
+
+    struct pci_device *pci;
+    foreachpci(pci) {
+        if (pci->vendor != PCI_VENDOR_ID_LSI_LOGIC &&
+            pci->vendor != PCI_VENDOR_ID_DELL)
+            continue;
+        if (pci->device != PCI_DEVICE_ID_LSI_SAS1064R ||
+            pci->device != PCI_DEVICE_ID_LSI_SAS1078 ||
+            pci->device != PCI_DEVICE_ID_LSI_SAS1078DE ||
+            pci->device != PCI_DEVICE_ID_LSI_SAS2108 ||
+            pci->device != PCI_DEVICE_ID_LSI_SAS2108E ||
+            pci->device != PCI_DEVICE_ID_LSI_SAS2004 ||
+            pci->device != PCI_DEVICE_ID_LSI_SAS2008 ||
+            pci->device != PCI_DEVICE_ID_LSI_VERDE_ZCR ||
+            pci->device != PCI_DEVICE_ID_DELL_PERC5 ||
+            pci->device != PCI_DEVICE_ID_LSI_SAS2208 ||
+            pci->device != PCI_DEVICE_ID_LSI_SAS3108)
+            continue;
+        init_megasas(pci);
+    }
+}
diff --git a/roms/seabios/src/megasas.h b/roms/seabios/src/megasas.h
new file mode 100644
index 000000000..124042e1c
--- /dev/null
+++ b/roms/seabios/src/megasas.h
@@ -0,0 +1,8 @@
+#ifndef __MEGASAS_H
+#define __MEGASAS_H
+
+struct disk_op_s;
+int megasas_cmd_data(struct disk_op_s *op, void *cdbcmd, u16 blocksize);
+void megasas_setup(void);
+
+#endif /* __MEGASAS_H */
diff --git a/roms/seabios/src/paravirt.c b/roms/seabios/src/paravirt.c
index 2a98d53ed..4b5c4417d 100644
--- a/roms/seabios/src/paravirt.c
+++ b/roms/seabios/src/paravirt.c
@@ -8,7 +8,8 @@
 // This file may be distributed under the terms of the GNU LGPLv3 license.
 
 #include "config.h" // CONFIG_COREBOOT
-#include "util.h" // ntoh[ls]
+#include "util.h" // dprintf
+#include "byteorder.h" // be32_to_cpu
 #include "ioport.h" // outw
 #include "paravirt.h" // qemu_cfg_port_probe
 #include "smbios.h" // struct smbios_structure_header
@@ -327,7 +328,7 @@ void qemu_cfg_romfile_setup(void)
 
     u32 count;
     qemu_cfg_read_entry(&count, QEMU_CFG_FILE_DIR, sizeof(count));
-    count = ntohl(count);
+    count = be32_to_cpu(count);
     u32 e;
     for (e = 0; e < count; e++) {
         struct QemuCfgFile qfile;
@@ -339,8 +340,8 @@ void qemu_cfg_romfile_setup(void)
         }
         memset(file, 0, sizeof(*file));
         strtcpy(file->name, qfile.name, sizeof(file->name));
-        file->size = ntohl(qfile.size);
-        file->id = ntohs(qfile.select);
+        file->size = be32_to_cpu(qfile.size);
+        file->id = be16_to_cpu(qfile.select);
         file->copy = qemu_cfg_read_file;
         romfile_add(file);
         dprintf(3, "Found fw_cfg file: %s (size=%d)\n", file->name, file->size);
diff --git a/roms/seabios/src/pci_ids.h b/roms/seabios/src/pci_ids.h
index 4b59585bc..665e94553 100644
--- a/roms/seabios/src/pci_ids.h
+++ b/roms/seabios/src/pci_ids.h
@@ -199,6 +199,7 @@
 #define PCI_DEVICE_ID_LSI_63C815	0x1000
 #define PCI_DEVICE_ID_LSI_SAS1064	0x0050
 #define PCI_DEVICE_ID_LSI_SAS1064R	0x0411
+#define PCI_DEVICE_ID_LSI_VERDE_ZCR	0x0413
 #define PCI_DEVICE_ID_LSI_SAS1066	0x005E
 #define PCI_DEVICE_ID_LSI_SAS1068	0x0054
 #define PCI_DEVICE_ID_LSI_SAS1064A	0x005C
@@ -206,6 +207,13 @@
 #define PCI_DEVICE_ID_LSI_SAS1066E	0x005A
 #define PCI_DEVICE_ID_LSI_SAS1068E	0x0058
 #define PCI_DEVICE_ID_LSI_SAS1078	0x0060
+#define PCI_DEVICE_ID_LSI_SAS1078DE	0x007C
+#define PCI_DEVICE_ID_LSI_SAS2108E	0x0078
+#define PCI_DEVICE_ID_LSI_SAS2108	0x0079
+#define PCI_DEVICE_ID_LSI_SAS2208	0x005B
+#define PCI_DEVICE_ID_LSI_SAS3108	0x005D
+#define PCI_DEVICE_ID_LSI_SAS2004	0x0071
+#define PCI_DEVICE_ID_LSI_SAS2008	0x0073
 
 #define PCI_VENDOR_ID_ATI		0x1002
 /* Mach64 */
diff --git a/roms/seabios/src/pciinit.c b/roms/seabios/src/pciinit.c
index 68f302a2f..0e87ab0ba 100644
--- a/roms/seabios/src/pciinit.c
+++ b/roms/seabios/src/pciinit.c
@@ -119,16 +119,6 @@ static void piix_isa_bridge_init(struct pci_device *pci, void *arg)
     dprintf(1, "PIIX3/PIIX4 init: elcr=%02x %02x\n", elcr[0], elcr[1]);
 }
 
-static const struct pci_device_id pci_isa_bridge_tbl[] = {
-    /* PIIX3/PIIX4 PCI to ISA bridge */
-    PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82371SB_0,
-               piix_isa_bridge_init),
-    PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82371AB_0,
-               piix_isa_bridge_init),
-
-    PCI_DEVICE_END
-};
-
 static void storage_ide_init(struct pci_device *pci, void *arg)
 {
     /* IDE: we map it as in ISA mode */
@@ -158,27 +148,8 @@ static void apple_macio_init(struct pci_device *pci, void *arg)
     pci_set_io_region_addr(pci, 0, 0x80800000, 0);
 }
 
-static const struct pci_device_id pci_class_tbl[] = {
-    /* STORAGE IDE */
-    PCI_DEVICE_CLASS(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82371SB_1,
-                     PCI_CLASS_STORAGE_IDE, piix_ide_init),
-    PCI_DEVICE_CLASS(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82371AB,
-                     PCI_CLASS_STORAGE_IDE, piix_ide_init),
-    PCI_DEVICE_CLASS(PCI_ANY_ID, PCI_ANY_ID, PCI_CLASS_STORAGE_IDE,
-                     storage_ide_init),
-
-    /* PIC, IBM, MIPC & MPIC2 */
-    PCI_DEVICE_CLASS(PCI_VENDOR_ID_IBM, 0x0046, PCI_CLASS_SYSTEM_PIC,
-                     pic_ibm_init),
-    PCI_DEVICE_CLASS(PCI_VENDOR_ID_IBM, 0xFFFF, PCI_CLASS_SYSTEM_PIC,
-                     pic_ibm_init),
-
-    /* 0xff00 */
-    PCI_DEVICE_CLASS(PCI_VENDOR_ID_APPLE, 0x0017, 0xff00, apple_macio_init),
-    PCI_DEVICE_CLASS(PCI_VENDOR_ID_APPLE, 0x0022, 0xff00, apple_macio_init),
-
-    PCI_DEVICE_END,
-};
+/* PM Timer ticks per second (HZ) */
+#define PM_TIMER_FREQUENCY  3579545
 
 /* PIIX4 Power Management device (for ACPI) */
 static void piix4_pm_init(struct pci_device *pci, void *arg)
@@ -191,13 +162,39 @@ static void piix4_pm_init(struct pci_device *pci, void *arg)
     pci_config_writeb(bdf, 0x80, 0x01); /* enable PM io space */
     pci_config_writel(bdf, 0x90, PORT_SMB_BASE | 1);
     pci_config_writeb(bdf, 0xd2, 0x09); /* enable SMBus io space */
+
+    pmtimer_init(PORT_ACPI_PM_BASE + 0x08, PM_TIMER_FREQUENCY / 1000);
 }
 
 static const struct pci_device_id pci_device_tbl[] = {
+    /* PIIX3/PIIX4 PCI to ISA bridge */
+    PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82371SB_0,
+               piix_isa_bridge_init),
+    PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82371AB_0,
+               piix_isa_bridge_init),
+
+    /* STORAGE IDE */
+    PCI_DEVICE_CLASS(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82371SB_1,
+                     PCI_CLASS_STORAGE_IDE, piix_ide_init),
+    PCI_DEVICE_CLASS(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82371AB,
+                     PCI_CLASS_STORAGE_IDE, piix_ide_init),
+    PCI_DEVICE_CLASS(PCI_ANY_ID, PCI_ANY_ID, PCI_CLASS_STORAGE_IDE,
+                     storage_ide_init),
+
+    /* PIC, IBM, MIPC & MPIC2 */
+    PCI_DEVICE_CLASS(PCI_VENDOR_ID_IBM, 0x0046, PCI_CLASS_SYSTEM_PIC,
+                     pic_ibm_init),
+    PCI_DEVICE_CLASS(PCI_VENDOR_ID_IBM, 0xFFFF, PCI_CLASS_SYSTEM_PIC,
+                     pic_ibm_init),
+
     /* PIIX4 Power Management device (for ACPI) */
     PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82371AB_3,
                piix4_pm_init),
 
+    /* 0xff00 */
+    PCI_DEVICE_CLASS(PCI_VENDOR_ID_APPLE, 0x0017, 0xff00, apple_macio_init),
+    PCI_DEVICE_CLASS(PCI_VENDOR_ID_APPLE, 0x0022, 0xff00, apple_macio_init),
+
     PCI_DEVICE_END,
 };
 
@@ -208,17 +205,15 @@ static void pci_bios_init_device(struct pci_device *pci)
             , pci_bdf_to_bus(bdf), pci_bdf_to_dev(bdf), pci_bdf_to_fn(bdf)
             , pci->vendor, pci->device);
 
-    pci_init_device(pci_class_tbl, pci, NULL);
-
-    /* enable memory mappings */
-    pci_config_maskw(bdf, PCI_COMMAND, 0, PCI_COMMAND_IO | PCI_COMMAND_MEMORY);
-
     /* map the interrupt */
     int pin = pci_config_readb(bdf, PCI_INTERRUPT_PIN);
     if (pin != 0)
         pci_config_writeb(bdf, PCI_INTERRUPT_LINE, pci_slot_get_irq(pci, pin));
 
     pci_init_device(pci_device_tbl, pci, NULL);
+
+    /* enable memory mappings */
+    pci_config_maskw(bdf, PCI_COMMAND, 0, PCI_COMMAND_IO | PCI_COMMAND_MEMORY);
 }
 
 static void pci_bios_init_devices(void)
@@ -227,9 +222,32 @@ static void pci_bios_init_devices(void)
     foreachpci(pci) {
         pci_bios_init_device(pci);
     }
+}
+
+
+/****************************************************************
+ * Platform device initialization
+ ****************************************************************/
+
+void i440fx_mem_addr_init(struct pci_device *dev, void *arg)
+{
+    if (RamSize <= 0x80000000)
+        pcimem_start = 0x80000000;
+    else if (RamSize <= 0xc0000000)
+        pcimem_start = 0xc0000000;
+}
 
+static const struct pci_device_id pci_platform_tbl[] = {
+    PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_82441,
+               i440fx_mem_addr_init),
+    PCI_DEVICE_END
+};
+
+static void pci_bios_init_platform(void)
+{
+    struct pci_device *pci;
     foreachpci(pci) {
-        pci_init_device(pci_isa_bridge_tbl, pci, NULL);
+        pci_init_device(pci_platform_tbl, pci, NULL);
     }
 }
 
@@ -592,8 +610,6 @@ static void pci_region_map_entries(struct pci_bus *busses, struct pci_region *r)
 
 static void pci_bios_map_devices(struct pci_bus *busses)
 {
-    pcimem_start = RamSize;
-
     if (pci_bios_init_root_regions(busses)) {
         struct pci_region r64_mem, r64_pref;
         r64_mem.list = NULL;
@@ -656,6 +672,9 @@ pci_setup(void)
     dprintf(1, "=== PCI device probing ===\n");
     pci_probe_devices();
 
+    pcimem_start = RamSize;
+    pci_bios_init_platform();
+
     dprintf(1, "=== PCI new allocation pass #1 ===\n");
     struct pci_bus *busses = malloc_tmp(sizeof(*busses) * (MaxPCIBus + 1));
     if (!busses) {
diff --git a/roms/seabios/src/post.c b/roms/seabios/src/post.c
index 0f31b4c35..0133f753e 100644
--- a/roms/seabios/src/post.c
+++ b/roms/seabios/src/post.c
@@ -28,7 +28,8 @@
 #include "virtio-blk.h" // virtio_blk_setup
 #include "virtio-scsi.h" // virtio_scsi_setup
 #include "lsi-scsi.h" // lsi_scsi_setup
-
+#include "esp-scsi.h" // esp_scsi_setup
+#include "megasas.h" // megasas_setup
 
 /****************************************************************
  * BIOS init
@@ -196,6 +197,8 @@ init_hw(void)
     virtio_blk_setup();
     virtio_scsi_setup();
     lsi_scsi_setup();
+    esp_scsi_setup();
+    megasas_setup();
 }
 
 // Begin the boot process by invoking an int0x19 in 16bit mode.
diff --git a/roms/seabios/src/smp.c b/roms/seabios/src/smp.c
index 3c36f8cf1..4975412f1 100644
--- a/roms/seabios/src/smp.c
+++ b/roms/seabios/src/smp.c
@@ -77,7 +77,7 @@ ASM16(
 
 int apic_id_is_present(u8 apic_id)
 {
-    return FoundAPICIDs[apic_id/32] & (1 << (apic_id % 32));
+    return !!(FoundAPICIDs[apic_id/32] & (1ul << (apic_id % 32)));
 }
 
 // find and initialize the CPUs by launching a SIPI to them
diff --git a/roms/seabios/src/ssdt-pcihp.dsl b/roms/seabios/src/ssdt-pcihp.dsl
index 12555e23e..cd66b831b 100644
--- a/roms/seabios/src/ssdt-pcihp.dsl
+++ b/roms/seabios/src/ssdt-pcihp.dsl
@@ -12,123 +12,23 @@ DefinitionBlock ("ssdt-pcihp.aml", "SSDT", 0x01, "BXPC", "BXSSDTPCIHP", 0x1)
     External (\_SB.PCI0.PCEJ, MethodObj)
 
     Scope(\_SB.PCI0) {
+
         /* Bulk generated PCI hotplug devices */
+        ACPI_EXTRACT_DEVICE_START ssdt_pcihp_start
+        ACPI_EXTRACT_DEVICE_END ssdt_pcihp_end
+        ACPI_EXTRACT_DEVICE_STRING ssdt_pcihp_name
+
         // Method _EJ0 can be patched by BIOS to EJ0_
         // at runtime, if the slot is detected to not support hotplug.
         // Extract the offset of the address dword and the
         // _EJ0 name to allow this patching.
-#define hotplug_slot(slot)                              \
-        Device (S##slot) {                              \
-           ACPI_EXTRACT_NAME_DWORD_CONST aml_adr_dword  \
-           Name (_ADR, 0x##slot##0000)                  \
-           ACPI_EXTRACT_METHOD_STRING aml_ej0_name      \
-           Method (_EJ0, 1) { Return(PCEJ(0x##slot)) }  \
-           Name (_SUN, 0x##slot)                        \
-        }
-
-        hotplug_slot(01)
-        hotplug_slot(02)
-        hotplug_slot(03)
-        hotplug_slot(04)
-        hotplug_slot(05)
-        hotplug_slot(06)
-        hotplug_slot(07)
-        hotplug_slot(08)
-        hotplug_slot(09)
-        hotplug_slot(0a)
-        hotplug_slot(0b)
-        hotplug_slot(0c)
-        hotplug_slot(0d)
-        hotplug_slot(0e)
-        hotplug_slot(0f)
-        hotplug_slot(10)
-        hotplug_slot(11)
-        hotplug_slot(12)
-        hotplug_slot(13)
-        hotplug_slot(14)
-        hotplug_slot(15)
-        hotplug_slot(16)
-        hotplug_slot(17)
-        hotplug_slot(18)
-        hotplug_slot(19)
-        hotplug_slot(1a)
-        hotplug_slot(1b)
-        hotplug_slot(1c)
-        hotplug_slot(1d)
-        hotplug_slot(1e)
-        hotplug_slot(1f)
-
-#define gen_pci_hotplug(slot)   \
-            If (LEqual(Arg0, 0x##slot)) { Notify(S##slot, Arg1) }
-
-        Method(PCNT, 2) {
-            gen_pci_hotplug(01)
-            gen_pci_hotplug(02)
-            gen_pci_hotplug(03)
-            gen_pci_hotplug(04)
-            gen_pci_hotplug(05)
-            gen_pci_hotplug(06)
-            gen_pci_hotplug(07)
-            gen_pci_hotplug(08)
-            gen_pci_hotplug(09)
-            gen_pci_hotplug(0a)
-            gen_pci_hotplug(0b)
-            gen_pci_hotplug(0c)
-            gen_pci_hotplug(0d)
-            gen_pci_hotplug(0e)
-            gen_pci_hotplug(0f)
-            gen_pci_hotplug(10)
-            gen_pci_hotplug(11)
-            gen_pci_hotplug(12)
-            gen_pci_hotplug(13)
-            gen_pci_hotplug(14)
-            gen_pci_hotplug(15)
-            gen_pci_hotplug(16)
-            gen_pci_hotplug(17)
-            gen_pci_hotplug(18)
-            gen_pci_hotplug(19)
-            gen_pci_hotplug(1a)
-            gen_pci_hotplug(1b)
-            gen_pci_hotplug(1c)
-            gen_pci_hotplug(1d)
-            gen_pci_hotplug(1e)
-            gen_pci_hotplug(1f)
+        Device (SAA) {
+           ACPI_EXTRACT_NAME_BYTE_CONST ssdt_pcihp_id
+           Name (_SUN, 0xAA)
+           ACPI_EXTRACT_NAME_DWORD_CONST ssdt_pcihp_adr
+           Name (_ADR, 0xAA0000)
+           ACPI_EXTRACT_METHOD_STRING ssdt_pcihp_ej0
+           Method (_EJ0, 1) { Return(PCEJ(_SUN)) }
         }
     }
-
-    Scope(\) {
-/****************************************************************
- * Suspend
- ****************************************************************/
-
-    /*
-     * S3 (suspend-to-ram), S4 (suspend-to-disk) and S5 (power-off) type codes:
-     * must match piix4 emulation.
-     */
-
-        ACPI_EXTRACT_NAME_STRING acpi_s3_name
-        Name (_S3, Package (0x04)
-        {
-            One,  /* PM1a_CNT.SLP_TYP */
-            One,  /* PM1b_CNT.SLP_TYP */
-            Zero,  /* reserved */
-            Zero   /* reserved */
-        })
-        ACPI_EXTRACT_NAME_STRING acpi_s4_name
-        ACPI_EXTRACT_PKG_START acpi_s4_pkg
-        Name (_S4, Package (0x04)
-        {
-            0x2,  /* PM1a_CNT.SLP_TYP */
-            0x2,  /* PM1b_CNT.SLP_TYP */
-            Zero,  /* reserved */
-            Zero   /* reserved */
-        })
-        Name (_S5, Package (0x04)
-        {
-            Zero,  /* PM1a_CNT.SLP_TYP */
-            Zero,  /* PM1b_CNT.SLP_TYP */
-            Zero,  /* reserved */
-            Zero   /* reserved */
-        })
-    }
 }
diff --git a/roms/seabios/src/ssdt-susp.dsl b/roms/seabios/src/ssdt-susp.dsl
new file mode 100644
index 000000000..0b3fa082e
--- /dev/null
+++ b/roms/seabios/src/ssdt-susp.dsl
@@ -0,0 +1,41 @@
+ACPI_EXTRACT_ALL_CODE ssdp_susp_aml
+
+DefinitionBlock ("ssdt-susp.aml", "SSDT", 0x01, "BXPC", "BXSSDTSUSP", 0x1)
+{
+
+/****************************************************************
+ * Suspend
+ ****************************************************************/
+
+    Scope(\) {
+    /*
+     * S3 (suspend-to-ram), S4 (suspend-to-disk) and S5 (power-off) type codes:
+     * must match piix4 emulation.
+     */
+
+        ACPI_EXTRACT_NAME_STRING acpi_s3_name
+        Name (_S3, Package (0x04)
+        {
+            One,  /* PM1a_CNT.SLP_TYP */
+            One,  /* PM1b_CNT.SLP_TYP */
+            Zero,  /* reserved */
+            Zero   /* reserved */
+        })
+        ACPI_EXTRACT_NAME_STRING acpi_s4_name
+        ACPI_EXTRACT_PKG_START acpi_s4_pkg
+        Name (_S4, Package (0x04)
+        {
+            0x2,  /* PM1a_CNT.SLP_TYP */
+            0x2,  /* PM1b_CNT.SLP_TYP */
+            Zero,  /* reserved */
+            Zero   /* reserved */
+        })
+        Name (_S5, Package (0x04)
+        {
+            Zero,  /* PM1a_CNT.SLP_TYP */
+            Zero,  /* PM1b_CNT.SLP_TYP */
+            Zero,  /* reserved */
+            Zero   /* reserved */
+        })
+    }
+}
diff --git a/roms/seabios/src/types.h b/roms/seabios/src/types.h
index b10f3b312..24b078e83 100644
--- a/roms/seabios/src/types.h
+++ b/roms/seabios/src/types.h
@@ -17,7 +17,7 @@ typedef signed long long s64;
 typedef u32 size_t;
 
 union u64_u32_u {
-    struct { u32 hi, lo; };
+    struct { u32 lo, hi; };
     u64 val;
 };
 
diff --git a/roms/seabios/src/util.h b/roms/seabios/src/util.h
index 89e928c5f..7723bb17c 100644
--- a/roms/seabios/src/util.h
+++ b/roms/seabios/src/util.h
@@ -104,36 +104,6 @@ static inline u32 __fls(u32 word)
     return word;
 }
 
-static inline u16 __htons_constant(u16 val) {
-    return (val<<8) | (val>>8);
-}
-static inline u32 __htonl_constant(u32 val) {
-    return (val<<24) | ((val&0xff00)<<8) | ((val&0xff0000)>>8) | (val>>24);
-}
-static inline u32 __htonl(u32 val) {
-    asm("bswapl %0" : "+r"(val));
-    return val;
-}
-#define htonl(x) (__builtin_constant_p((u32)(x)) ? __htonl_constant(x) : __htonl(x))
-#define ntohl(x) htonl(x)
-#define htons(x) __htons_constant(x)
-#define ntohs(x) htons(x)
-
-static inline u16 cpu_to_le16(u16 x)
-{
-    return x;
-}
-
-static inline u32 cpu_to_le32(u32 x)
-{
-    return x;
-}
-
-static inline u32 le32_to_cpu(u32 x)
-{
-    return x;
-}
-
 static inline u32 getesp(void) {
     u32 esp;
     asm("movl %%esp, %0" : "=rm"(esp));
@@ -312,6 +282,7 @@ void lpt_setup(void);
 // clock.c
 #define PIT_TICK_RATE 1193180   // Underlying HZ of PIT
 #define PIT_TICK_INTERVAL 65536 // Default interval for 18.2Hz timer
+void pmtimer_init(u16 ioport, u32 khz);
 int check_tsc(u64 end);
 void timer_setup(void);
 void ndelay(u32 count);
diff --git a/roms/seabios/tools/acpi_extract.py b/roms/seabios/tools/acpi_extract.py
index 167a322a3..329567885 100755
--- a/roms/seabios/tools/acpi_extract.py
+++ b/roms/seabios/tools/acpi_extract.py
@@ -93,7 +93,7 @@ def aml_pkglen(offset):
     pkglenbytes = aml_pkglen_bytes(offset)
     pkglen = aml[offset] & 0x3F
     # If multibyte, first nibble only uses bits 0-3
-    if ((pkglenbytes > 0) and (pkglen & 0x30)):
+    if ((pkglenbytes > 1) and (pkglen & 0x30)):
         die("PkgLen bytes 0x%x but first nibble 0x%x expected 0x0X" %
             (pkglen, pkglen))
     offset += 1
@@ -164,6 +164,28 @@ def aml_name_word_const(offset):
 def aml_name_byte_const(offset):
     return aml_data_byte_const(aml_name_string(offset) + 4)
 
+def aml_device_start(offset):
+    #0x5B 0x82 DeviceOp PkgLength NameString
+    if ((aml[offset] != 0x5B) or (aml[offset + 1] != 0x82)):
+        die( "Name offset 0x%x: expected 0x5B 0x82 actual 0x%x 0x%x" %
+             (offset, aml[offset], aml[offset + 1]));
+    return offset
+
+def aml_device_string(offset):
+    #0x5B 0x82 DeviceOp PkgLength NameString
+    start = aml_device_start(offset)
+    offset += 2
+    pkglenbytes = aml_pkglen_bytes(offset)
+    offset += pkglenbytes
+    return offset
+
+def aml_device_end(offset):
+    start = aml_device_start(offset)
+    offset += 2
+    pkglenbytes = aml_pkglen_bytes(offset)
+    pkglen = aml_pkglen(offset)
+    return offset + pkglen
+
 def aml_processor_start(offset):
     #0x5B 0x83 ProcessorOp PkgLength NameString ProcID
     if ((aml[offset] != 0x5B) or (aml[offset + 1] != 0x83)):
@@ -271,6 +293,12 @@ for i in range(len(asl)):
         offset = aml_name_string(offset)
     elif (directive == "ACPI_EXTRACT_METHOD_STRING"):
         offset = aml_method_string(offset)
+    elif (directive == "ACPI_EXTRACT_DEVICE_START"):
+        offset = aml_device_start(offset)
+    elif (directive == "ACPI_EXTRACT_DEVICE_STRING"):
+        offset = aml_device_string(offset)
+    elif (directive == "ACPI_EXTRACT_DEVICE_END"):
+        offset = aml_device_end(offset)
     elif (directive == "ACPI_EXTRACT_PROCESSOR_START"):
         offset = aml_processor_start(offset)
     elif (directive == "ACPI_EXTRACT_PROCESSOR_STRING"):
diff --git a/roms/seabios/vgasrc/Kconfig b/roms/seabios/vgasrc/Kconfig
index 6b22c3928..089a44790 100644
--- a/roms/seabios/vgasrc/Kconfig
+++ b/roms/seabios/vgasrc/Kconfig
@@ -43,6 +43,27 @@ menu "VGA ROM"
                 Build support for Geode LX vga.
     endchoice
 
+    choice
+        depends on VGA_GEODEGX2 || VGA_GEODELX
+        prompt "Output Mode"
+        default VGA_OUTPUT_CRT
+
+        config VGA_OUTPUT_CRT
+            bool "CRT"
+            help
+                Use CRT for output.
+
+        config VGA_OUTPUT_PANEL
+            bool "Flat Panel"
+            help
+                Use flat panel for output.
+
+        config VGA_OUTPUT_CRT_PANEL
+            bool "CRT and Flat Panel"
+            help
+                Use CRT and flat panel for output.
+    endchoice
+
     config BUILD_VGABIOS
         bool
         default !NO_VGABIOS
diff --git a/roms/seabios/vgasrc/bochsvga.c b/roms/seabios/vgasrc/bochsvga.c
index 2a8aeb186..938dba07c 100644
--- a/roms/seabios/vgasrc/bochsvga.c
+++ b/roms/seabios/vgasrc/bochsvga.c
@@ -364,6 +364,7 @@ bochsvga_init(void)
     SET_VGA(VBE_framebuffer, lfb_addr);
     u32 totalmem = dispi_read(VBE_DISPI_INDEX_VIDEO_MEMORY_64K) * 64 * 1024;
     SET_VGA(VBE_total_memory, totalmem);
+    SET_VGA(VBE_win_granularity, 64);
     SET_VGA(VBE_capabilities, VBE_CAPABILITY_8BIT_DAC);
 
     dprintf(1, "VBE DISPI: lfb_addr=%x, size %d MB\n",
diff --git a/roms/seabios/vgasrc/geodevga.c b/roms/seabios/vgasrc/geodevga.c
index 5c6caf045..5b42e00ea 100644
--- a/roms/seabios/vgasrc/geodevga.c
+++ b/roms/seabios/vgasrc/geodevga.c
@@ -20,7 +20,7 @@
 * MSR and High Mem access through VSA Virtual Register
 ****************************************************************/
 
-static union u64_u32_u geode_msrRead(u32 msrAddr)
+static u64 geode_msr_read(u32 msrAddr)
 {
     union u64_u32_u val;
     asm __volatile__ (
@@ -33,11 +33,14 @@ static union u64_u32_u geode_msrRead(u32 msrAddr)
         : "c"(msrAddr)
         : "cc"
     );
-    return val;
+    return val.val;
 }
 
-static void geode_msrWrite(u32 msrAddr,u32 andhi, u32 andlo, u32 orhi, u32 orlo)
+static void geode_msr_mask(u32 msrAddr, u64 off, u64 on)
 {
+    union u64_u32_u uand, uor;
+    uand.val = ~off;
+    uor.val = on;
     asm __volatile__ (
         "push   %%eax                   \n"
         "movw   $0x0AC1C, %%dx          \n"
@@ -47,12 +50,12 @@ static void geode_msrWrite(u32 msrAddr,u32 andhi, u32 andlo, u32 orhi, u32 orlo)
         "pop    %%eax                   \n"
         "outw   %%ax, %%dx              \n"
         :
-        : "c"(msrAddr), "S" (andhi), "D" (andlo), "b" (orhi), "a" (orlo)
+        : "c"(msrAddr), "S" (uand.hi), "D" (uand.lo), "b" (uor.hi), "a" (uor.lo)
         : "%edx","cc"
     );
 }
 
-static u32 geode_memRead(u32 addr)
+static u32 geode_mem_read(u32 addr)
 {
     u32 val;
     asm __volatile__ (
@@ -69,7 +72,7 @@ static u32 geode_memRead(u32 addr)
     return val;
 }
 
-static void geode_memWrite(u32 addr, u32 and, u32 or )
+static void geode_mem_mask(u32 addr, u32 off, u32 or)
 {
     asm __volatile__ (
         "movw   $0x0AC1C, %%dx          \n"
@@ -78,167 +81,159 @@ static void geode_memWrite(u32 addr, u32 and, u32 or )
         "addb   $2, %%dl                \n"
         "outw   %%ax, %%dx              \n"
         :
-        : "b"(addr), "S" (and), "D" (or)
+        : "b"(addr), "S" (~off), "D" (or)
         : "%eax","cc"
     );
 }
 
-static int legacyio_check(void)
-{
-    int ret=0;
-    union u64_u32_u val;
+#define VP_FP_START     0x400
 
-    if (CONFIG_VGA_GEODEGX2)
-        val=geode_msrRead(GLIU0_P2D_BM_4);
-    else
-        val=geode_msrRead(MSR_GLIU0_BASE4);
-    if (val.lo != 0x0A0fffe0)
-        ret|=1;
-
-    val=geode_msrRead(GLIU0_IOD_BM_0);
-    if (val.lo != 0x3c0ffff0)
-        ret|=2;
+static u32 GeodeFB VAR16;
+static u32 GeodeDC VAR16;
+static u32 GeodeVP VAR16;
 
-    val=geode_msrRead(GLIU0_IOD_BM_1);
-    if (val.lo != 0x3d0ffff0)
-        ret|=4;
-
-    return ret;
+static u32 geode_dc_read(int reg)
+{
+    u32 val = geode_mem_read(GET_GLOBAL(GeodeDC) + reg);
+    dprintf(4, "%s(0x%08x) = 0x%08x\n"
+            , __func__, GET_GLOBAL(GeodeDC) + reg, val);
+    return val;
 }
 
-/****************************************************************
-* Extened CRTC Register functions
-****************************************************************/
-static void crtce_lock(void)
+static void geode_dc_write(int reg, u32 val)
 {
-    stdvga_crtc_write(VGAREG_VGA_CRTC_ADDRESS, EXTENDED_REGISTER_LOCK
-                      , CRTCE_LOCK);
+    dprintf(4, "%s(0x%08x, 0x%08x)\n"
+            , __func__, GET_GLOBAL(GeodeDC) + reg, val);
+    geode_mem_mask(GET_GLOBAL(GeodeDC) + reg, ~0, val);
 }
 
-static void crtce_unlock(void)
+static void geode_dc_mask(int reg, u32 off, u32 on)
 {
-    stdvga_crtc_write(VGAREG_VGA_CRTC_ADDRESS, EXTENDED_REGISTER_LOCK
-                      , CRTCE_UNLOCK);
+    dprintf(4, "%s(0x%08x, 0x%08x, 0x%08x)\n"
+            , __func__, GET_GLOBAL(GeodeDC) + reg, off, on);
+    geode_mem_mask(GET_GLOBAL(GeodeDC) + reg, off, on);
 }
 
-static u8 crtce_read(u8 reg)
+static u32 geode_vp_read(int reg)
 {
-    crtce_unlock();
-    u8 val = stdvga_crtc_read(VGAREG_VGA_CRTC_ADDRESS, reg);
-    crtce_lock();
+    u32 val = geode_mem_read(GET_GLOBAL(GeodeVP) + reg);
+    dprintf(4, "%s(0x%08x) = 0x%08x\n"
+            , __func__, GET_GLOBAL(GeodeVP) + reg, val);
     return val;
 }
 
-static void crtce_write(u8 reg, u8 val)
+static void geode_vp_write(int reg, u32 val)
 {
-    crtce_unlock();
-    stdvga_crtc_write(VGAREG_VGA_CRTC_ADDRESS, reg, val);
-    crtce_lock();
+    dprintf(4, "%s(0x%08x, 0x%08x)\n"
+            , __func__, GET_GLOBAL(GeodeVP) + reg, val);
+    geode_mem_mask(GET_GLOBAL(GeodeVP) + reg, ~0, val);
 }
 
-/****************************************************************
-* Display Controller Functions
-****************************************************************/
-static u32 dc_read(u16 seg, u32 reg)
+static void geode_vp_mask(int reg, u32 off, u32 on)
 {
-    u32 val, *dest_far = (void*)reg;
-    val = GET_FARVAR(seg,*dest_far);
-    return val;
+    dprintf(4, "%s(0x%08x, 0x%08x, 0x%08x)\n"
+            , __func__, GET_GLOBAL(GeodeVP) + reg, off, on);
+    geode_mem_mask(GET_GLOBAL(GeodeVP) + reg, off, on);
 }
 
-static void dc_write(u16 seg, u32 reg, u32 val)
+static u32 geode_fp_read(int reg)
 {
-    u32 *dest_far = (void*)reg;
-    SET_FARVAR(seg,*dest_far,val);
+    u32 val = geode_mem_read(GET_GLOBAL(GeodeVP) + VP_FP_START + reg);
+    dprintf(4, "%s(0x%08x) = 0x%08x\n"
+            , __func__, GET_GLOBAL(GeodeVP) + reg, val);
+    return val;
 }
 
-static void dc_set(u16 seg, u32 reg, u32 and, u32 or)
+static void geode_fp_write(int reg, u32 val)
 {
-    u32 val = dc_read(seg,reg);
-    val &=and;
-    val |=or;
-    dc_write(seg,reg,val);
+    dprintf(4, "%s(0x%08x, 0x%08x)\n"
+            , __func__, GET_GLOBAL(GeodeVP) + VP_FP_START + reg, val);
+    geode_mem_mask(GET_GLOBAL(GeodeVP) + reg, ~0, val);
 }
 
-static void dc_unlock(u16 seg)
-{
-    dc_write(seg,DC_UNLOCK,DC_LOCK_UNLOCK);
-}
+/****************************************************************
+ * Helper functions
+ ****************************************************************/
 
-static void dc_lock(u16 seg)
+static int legacyio_check(void)
 {
-    dc_write(seg,DC_UNLOCK,DC_LOCK_LOCK);
-}
+    int ret=0;
+    u64 val;
 
-static u16 dc_map(u16 seg)
-{
-    u8 reg;
-
-    reg = crtce_read(EXTENDED_MODE_CONTROL);
-    reg &= 0xf9;
-    switch (seg) {
-    case SEG_GRAPH:
-        reg |= 0x02;
-        break;
-    case SEG_MTEXT:
-        reg |= 0x04;
-        break;
-    case SEG_CTEXT:
-        reg |= 0x06;
-        break;
-    default:
-        seg=0;
-        break;
-    }
+    if (CONFIG_VGA_GEODEGX2)
+        val = geode_msr_read(GLIU0_P2D_BM_4);
+    else
+        val = geode_msr_read(MSR_GLIU0_BASE4);
+    if ((val & 0xffffffff) != 0x0A0fffe0)
+        ret|=1;
 
-    crtce_write(EXTENDED_MODE_CONTROL,reg);
-    return seg;
+    val = geode_msr_read(GLIU0_IOD_BM_0);
+    if ((val & 0xffffffff) != 0x3c0ffff0)
+        ret|=2;
+
+    val = geode_msr_read(GLIU0_IOD_BM_1);
+    if ((val & 0xffffffff) != 0x3d0ffff0)
+        ret|=4;
+
+    return ret;
 }
 
-static void dc_unmap(void)
+static u32 framebuffer_size(void)
 {
-    dc_map(0);
+    /* We use the P2D_R0 msr to read out the number of pages.
+     * One page has a size of 4k
+     *
+     * Bit      Name    Description
+     * 39:20    PMAX    Physical Memory Address Max
+     * 19:0     PMIX    Physical Memory Address Min
+     *
+     */
+    u64 msr = geode_msr_read(GLIU0_P2D_RO);
+
+    u32 pmax = (msr >> 20) & 0x000fffff;
+    u32 pmin = msr & 0x000fffff;
+
+    u32 val = pmax - pmin;
+    val += 1;
+
+    /* The page size is 4k */
+    return (val << 12);
 }
 
-
 /****************************************************************
 * Init Functions
 ****************************************************************/
 
 /* Set up the dc (display controller) portion of the geodelx
-*  The dc provides hardware support for VGA graphics
-*  for features not accessible from the VGA registers,
-*  the dc's pci bar can be mapped to a vga memory segment
+*  The dc provides hardware support for VGA graphics.
 */
-static int dc_setup(void)
+static void dc_setup(void)
 {
-    u32 fb, dc_fb;
-    u16 seg;
-
     dprintf(2, "DC_SETUP\n");
 
-    seg = dc_map(SEG_GRAPH);
-    dc_unlock(seg);
+    geode_dc_write(DC_UNLOCK, DC_LOCK_UNLOCK);
 
     /* zero memory config */
-    dc_write(seg,DC_FB_ST_OFFSET,0x0);
-    dc_write(seg,DC_CB_ST_OFFSET,0x0);
-    dc_write(seg,DC_CURS_ST_OFFSET,0x0);
+    geode_dc_write(DC_FB_ST_OFFSET, 0x0);
+    geode_dc_write(DC_CB_ST_OFFSET, 0x0);
+    geode_dc_write(DC_CURS_ST_OFFSET, 0x0);
 
     /* read fb-bar from pci, then point dc to the fb base */
-    dc_fb = dc_read(seg,DC_GLIU0_MEM_OFFSET);
-    fb = pci_config_readl(GET_GLOBAL(VgaBDF), PCI_BASE_ADDRESS_0);
-    if (fb!=dc_fb) {
-        dc_write(seg,DC_GLIU0_MEM_OFFSET,fb);
-    }
+    u32 fb = GET_GLOBAL(GeodeFB);
+    if (geode_dc_read(DC_GLIU0_MEM_OFFSET) != fb)
+        geode_dc_write(DC_GLIU0_MEM_OFFSET, fb);
 
-    dc_set(seg,DC_DISPLAY_CFG,DC_CFG_MSK,DC_GDEN+DC_TRUP);
-    dc_set(seg,DC_GENERAL_CFG,0,DC_VGAE);
+    geode_dc_mask(DC_DISPLAY_CFG, ~DC_CFG_MSK, DC_DISPLAY_CFG_GDEN|DC_DISPLAY_CFG_TRUP);
+    geode_dc_write(DC_GENERAL_CFG, DC_DISPLAY_CFG_VGAE);
 
-    dc_lock(seg);
-    dc_unmap();
+    geode_dc_write(DC_UNLOCK, DC_LOCK_LOCK);
 
-    return 0;
+    u32 fb_size = framebuffer_size(); // in byte
+    dprintf(1, "%d KB of video memory at 0x%08x\n", fb_size / 1024, fb);
+
+    /* update VBE variables */
+    SET_VGA(VBE_framebuffer, fb);
+    SET_VGA(VBE_total_memory, fb_size / 1024 / 64); // number of 64K blocks
 }
 
 /* Setup the vp (video processor) portion of the geodelx
@@ -247,38 +242,75 @@ static int dc_setup(void)
 *  The High Mem Access virtual register is used to  configure the
 *   pci mmio bar from 16bit friendly io space.
 */
-int vp_setup(void)
+static void vp_setup(void)
 {
-    u32 reg,vp;
+    u32 msr_addr;
+    u64 msr;
 
     dprintf(2,"VP_SETUP\n");
+
     /* set output to crt and RGB/YUV */
     if (CONFIG_VGA_GEODEGX2)
-        geode_msrWrite(VP_MSR_CONFIG_GX2, ~0, ~0xf8, 0, 0);
+        msr_addr = VP_MSR_CONFIG_GX2;
     else
-        geode_msrWrite(VP_MSR_CONFIG_LX, ~0, ~0xf8, 0, 0);
+        msr_addr = VP_MSR_CONFIG_LX;
+
+    /* set output mode (RGB/YUV) */
+    msr = geode_msr_read(msr_addr);
+    msr &= ~VP_MSR_CONFIG_FMT;         // mask out FMT (bits 5:3)
+
+    if (CONFIG_VGA_OUTPUT_PANEL || CONFIG_VGA_OUTPUT_CRT_PANEL) {
+        msr |= VP_MSR_CONFIG_FMT_FP;   // flat panel
+
+        if (CONFIG_VGA_OUTPUT_CRT_PANEL) {
+            msr |= VP_MSR_CONFIG_FPC;  // simultaneous Flat Panel and CRT
+            dprintf(1, "output: simultaneous Flat Panel and CRT\n");
+        } else {
+            msr &= ~VP_MSR_CONFIG_FPC; // no simultaneous Flat Panel and CRT
+            dprintf(1, "ouput: flat panel\n");
+        }
+    } else {
+        msr |= VP_MSR_CONFIG_FMT_CRT;  // CRT only
+       dprintf(1, "output: CRT\n");
+    }
+    geode_msr_mask(msr_addr, ~msr, msr);
 
-    /* get vp register base from pci */
-    vp = pci_config_readl(GET_GLOBAL(VgaBDF), PCI_BASE_ADDRESS_3);
 
     /* Set mmio registers
     * there may be some timing issues here, the reads seem
     * to slow things down enough work reliably
     */
 
-    reg = geode_memRead(vp+VP_MISC);
+    u32 reg = geode_vp_read(VP_MISC);
     dprintf(1,"VP_SETUP VP_MISC=0x%08x\n",reg);
-    geode_memWrite(vp+VP_MISC,0,VP_BYP_BOTH);
-    reg = geode_memRead(vp+VP_MISC);
+    geode_vp_write(VP_MISC, VP_DCFG_BYP_BOTH);
+    reg = geode_vp_read(VP_MISC);
     dprintf(1,"VP_SETUP VP_MISC=0x%08x\n",reg);
 
-    reg = geode_memRead(vp+VP_DCFG);
+    reg = geode_vp_read(VP_DCFG);
     dprintf(1,"VP_SETUP VP_DCFG=0x%08x\n",reg);
-    geode_memWrite(vp+VP_DCFG, ~0,VP_CRT_EN+VP_HSYNC_EN+VP_VSYNC_EN+VP_DAC_BL_EN+VP_CRT_SKEW);
-    reg = geode_memRead(vp+VP_DCFG);
+    geode_vp_mask(VP_DCFG, 0, VP_DCFG_CRT_EN|VP_DCFG_HSYNC_EN|VP_DCFG_VSYNC_EN|VP_DCFG_DAC_BL_EN|VP_DCFG_CRT_SKEW);
+    reg = geode_vp_read(VP_DCFG);
     dprintf(1,"VP_SETUP VP_DCFG=0x%08x\n",reg);
 
-    return 0;
+    /* setup flat panel */
+    if (CONFIG_VGA_OUTPUT_PANEL || CONFIG_VGA_OUTPUT_CRT_PANEL) {
+        dprintf(1, "Setting up flat panel\n");
+        /* write timing register */
+        geode_fp_write(FP_PT1, 0x0);
+        geode_fp_write(FP_PT2, FP_PT2_SCRC);
+
+        /* set pad select for TFT/LVDS */
+        msr  = VP_MSR_PADSEL_TFT_SEL_HIGH;
+        msr  = msr << 32;
+        msr |= VP_MSR_PADSEL_TFT_SEL_LOW;
+        geode_msr_mask(VP_MSR_PADSEL, ~msr, msr);
+
+        /* turn the panel on (if it isn't already) */
+        reg = geode_fp_read(FP_PM);
+        reg |= FP_PM_P;
+        geode_fp_write(FP_PM, reg);
+    }
 }
 
 static u8 geode_crtc_01[] VAR16 = {
@@ -366,8 +398,18 @@ int geodevga_init(void)
     if (GET_GLOBAL(VgaBDF) < 0)
         // Device should be at 00:01.1
         SET_VGA(VgaBDF, pci_to_bdf(0, 1, 1));
-    ret |= vp_setup();
-    ret |= dc_setup();
 
-    return ret;
+    // setup geode struct which is used for register access
+    SET_VGA(GeodeFB, pci_config_readl(GET_GLOBAL(VgaBDF), PCI_BASE_ADDRESS_0));
+    SET_VGA(GeodeDC, pci_config_readl(GET_GLOBAL(VgaBDF), PCI_BASE_ADDRESS_2));
+    SET_VGA(GeodeVP, pci_config_readl(GET_GLOBAL(VgaBDF), PCI_BASE_ADDRESS_3));
+
+    dprintf(1, "fb addr: 0x%08x\n", GET_GLOBAL(GeodeFB));
+    dprintf(1, "dc addr: 0x%08x\n", GET_GLOBAL(GeodeDC));
+    dprintf(1, "vp addr: 0x%08x\n", GET_GLOBAL(GeodeVP));
+
+    vp_setup();
+    dc_setup();
+
+    return 0;
 }
diff --git a/roms/seabios/vgasrc/geodevga.h b/roms/seabios/vgasrc/geodevga.h
index fd7ce432a..5993b23b4 100644
--- a/roms/seabios/vgasrc/geodevga.h
+++ b/roms/seabios/vgasrc/geodevga.h
@@ -13,13 +13,6 @@
 #define VRC_DATA                        0xAC1E  // Data register
 #define VR_UNLOCK                       0xFC53  // Virtual register unlock code
 
-#define EXTENDED_REGISTER_LOCK          0x30
-#define EXTENDED_MODE_CONTROL           0x43
-#define EXTENDED_START_ADDR             0x44
-
-#define CRTCE_UNLOCK                    0x4c
-#define CRTCE_LOCK                      0xff
-
 // Graphics-specific registers:
 #define OEM_BAR0                        0x50
 #define OEM_BAR1                        0x54
@@ -33,11 +26,23 @@
 #define MSR_GLIU0                       (1 << 28)
 #define MSR_GLIU0_BASE4                 (MSR_GLIU0 + 0x23)  /* LX */
 #define GLIU0_P2D_BM_4                  (MSR_GLIU0 + 0x24)  /* GX2 */
+#define GLIU0_P2D_RO			(MSR_GLIU0 + 0x29)
 #define GLIU0_IOD_BM_0                  (MSR_GLIU0 + 0xE0)
 #define GLIU0_IOD_BM_1                  (MSR_GLIU0 + 0xE1)
 #define DC_SPARE                        0x80000011
 #define VP_MSR_CONFIG_GX2               0xc0002001  /* GX2 */
 #define VP_MSR_CONFIG_LX                0x48002001  /* LX */
+#define VP_MSR_PADSEL                   0x48002011
+
+#define VP_MSR_PADSEL_TFT_SEL_LOW       0xDFFFFFFF
+#define VP_MSR_PADSEL_TFT_SEL_HIGH      0x0000003F
+
+/* VP_MSR_CONFIG bits */
+#define VP_MSR_CONFIG_FMT_CRT           (0)
+#define VP_MSR_CONFIG_FMT_FP            (1 << 3)
+#define VP_MSR_CONFIG_FPC               (1 << 15)
+#define VP_MSR_CONFIG_FMT               ((1 << 3) | (1 << 4) | (1 << 5))
+
 
 /* DC REG OFFSET */
 #define DC_UNLOCK                       0x0
@@ -53,19 +58,28 @@
 #define VP_DCFG                         0x8
 #define VP_MISC                         0x50
 
+/* FP REG OFFSET */
+#define FP_PT1                          0x00
+#define FP_PT2                          0x08
+#define FP_PM                           0x10
+
 
 /* DC bits */
-#define DC_VGAE                         (1 << 7)
-#define DC_GDEN                         (1 << 3)
-#define DC_TRUP                         (1 << 6)
+#define DC_DISPLAY_CFG_VGAE             (1 << 7)
+#define DC_DISPLAY_CFG_GDEN             (1 << 3)
+#define DC_DISPLAY_CFG_TRUP             (1 << 6)
 
 /* VP bits */
-#define VP_CRT_EN                       (1 << 0)
-#define VP_HSYNC_EN                     (1 << 1)
-#define VP_VSYNC_EN                     (1 << 2)
-#define VP_DAC_BL_EN                    (1 << 3)
-#define VP_CRT_SKEW                     (1 << 16)
-#define VP_BYP_BOTH                     (1 << 0)
+#define VP_DCFG_CRT_EN                  (1 << 0)
+#define VP_DCFG_HSYNC_EN                (1 << 1)
+#define VP_DCFG_VSYNC_EN                (1 << 2)
+#define VP_DCFG_DAC_BL_EN               (1 << 3)
+#define VP_DCFG_CRT_SKEW                (1 << 16)
+#define VP_DCFG_BYP_BOTH                (1 << 0)
+
+/* FP bits */
+#define FP_PM_P                         (1 << 24)       /* panel power ctl */
+#define FP_PT2_SCRC                     (1 << 27)       /* panel shift clock retrace activity ctl  */
 
 /* Mask */
 #define DC_CFG_MSK                      0xf000a6
diff --git a/roms/seabios/vgasrc/stdvgamodes.c b/roms/seabios/vgasrc/stdvgamodes.c
index 5497da8f6..1756adef0 100644
--- a/roms/seabios/vgasrc/stdvgamodes.c
+++ b/roms/seabios/vgasrc/stdvgamodes.c
@@ -334,6 +334,16 @@ stdvga_find_mode(int mode)
 void
 stdvga_list_modes(u16 seg, u16 *dest, u16 *last)
 {
+    int i;
+    for (i = 0; i < ARRAY_SIZE(vga_modes); i++) {
+        struct stdvga_mode_s *stdmode_g = &vga_modes[i];
+        u16 mode = GET_GLOBAL(stdmode_g->mode);
+        if (mode == 0xffff)
+            continue;
+        SET_FARVAR(seg, *dest, mode);
+        dest++;
+    }
+
     SET_FARVAR(seg, *dest, 0xffff);
 }
 
diff --git a/roms/seabios/vgasrc/vbe.c b/roms/seabios/vgasrc/vbe.c
index 227a244a1..d962333bd 100644
--- a/roms/seabios/vgasrc/vbe.c
+++ b/roms/seabios/vgasrc/vbe.c
@@ -17,7 +17,7 @@
 u32 VBE_total_memory VAR16 = 256 * 1024;
 u32 VBE_capabilities VAR16;
 u32 VBE_framebuffer VAR16;
-u16 VBE_win_granularity VAR16 = 64;
+u16 VBE_win_granularity VAR16;
 
 static void
 vbe_104f00(struct bregs *regs)
@@ -74,7 +74,7 @@ vbe_104f01(struct bregs *regs)
 
     dprintf(1, "VBE mode info request: %x\n", mode);
 
-    struct vgamode_s *vmode_g = vgahw_find_mode(mode);
+    struct vgamode_s *vmode_g = vgahw_find_mode(mode & ~MF_VBEFLAGS);
     if (! vmode_g) {
         dprintf(1, "VBE mode %x not found\n", mode);
         regs->ax = 0x014f;
@@ -82,27 +82,22 @@ vbe_104f01(struct bregs *regs)
     }
 
     memset_far(seg, info, 0, sizeof(*info));
-    u16 mode_attr = VBE_MODE_ATTRIBUTE_SUPPORTED |
-                    VBE_MODE_ATTRIBUTE_EXTENDED_INFORMATION_AVAILABLE |
-                    VBE_MODE_ATTRIBUTE_COLOR_MODE |
-                    VBE_MODE_ATTRIBUTE_GRAPHICS_MODE |
-                    VBE_MODE_ATTRIBUTE_NOT_VGA_COMPATIBLE;
-    u32 framebuffer = GET_GLOBAL(VBE_framebuffer);
-    if (framebuffer)
-        mode_attr |= VBE_MODE_ATTRIBUTE_LINEAR_FRAME_BUFFER_MODE;
-    SET_FARVAR(seg, info->mode_attributes, mode_attr);
+
+    // Basic information about video controller.
+    u32 win_granularity = GET_GLOBAL(VBE_win_granularity);
     SET_FARVAR(seg, info->winA_attributes,
-               VBE_WINDOW_ATTRIBUTE_RELOCATABLE |
+               (win_granularity ? VBE_WINDOW_ATTRIBUTE_RELOCATABLE : 0) |
                VBE_WINDOW_ATTRIBUTE_READABLE |
                VBE_WINDOW_ATTRIBUTE_WRITEABLE);
     SET_FARVAR(seg, info->winB_attributes, 0);
-    SET_FARVAR(seg, info->win_granularity, GET_GLOBAL(VBE_win_granularity));
+    SET_FARVAR(seg, info->win_granularity, win_granularity);
     SET_FARVAR(seg, info->win_size, 64); /* Bank size 64K */
     SET_FARVAR(seg, info->winA_seg, GET_GLOBAL(vmode_g->sstart));
     SET_FARVAR(seg, info->winB_seg, 0x0);
     extern void entry_104f05(void);
     SET_FARVAR(seg, info->win_func_ptr
                , SEGOFF(get_global_seg(), (u32)entry_104f05));
+    // Basic information about mode.
     int width = GET_GLOBAL(vmode_g->width);
     int height = GET_GLOBAL(vmode_g->height);
     int linesize = DIV_ROUND_UP(width * vga_bpp(vmode_g), 8);
@@ -112,18 +107,50 @@ vbe_104f01(struct bregs *regs)
     SET_FARVAR(seg, info->xcharsize, GET_GLOBAL(vmode_g->cwidth));
     SET_FARVAR(seg, info->ycharsize, GET_GLOBAL(vmode_g->cheight));
     int depth = GET_GLOBAL(vmode_g->depth);
-    int planes = (depth == 4) ? 4 : 1;
-    SET_FARVAR(seg, info->planes, planes);
     SET_FARVAR(seg, info->bits_per_pixel, depth);
-    SET_FARVAR(seg, info->banks, 1);
-    SET_FARVAR(seg, info->mem_model, GET_GLOBAL(vmode_g->memmodel));
-    SET_FARVAR(seg, info->bank_size, 0);
-    u32 pages = GET_GLOBAL(VBE_total_memory) / ALIGN(height * linesize, 64*1024);
-    SET_FARVAR(seg, info->pages, (pages / planes) - 1);
+    u8 memmodel = GET_GLOBAL(vmode_g->memmodel);
+    SET_FARVAR(seg, info->mem_model, memmodel);
     SET_FARVAR(seg, info->reserved0, 1);
 
-    u8 r_size, r_pos, g_size, g_pos, b_size, b_pos, a_size, a_pos;
+    // Mode specific info.
+    u16 mode_attr = VBE_MODE_ATTRIBUTE_SUPPORTED |
+                    VBE_MODE_ATTRIBUTE_EXTENDED_INFORMATION_AVAILABLE |
+                    VBE_MODE_ATTRIBUTE_COLOR_MODE |
+                    VBE_MODE_ATTRIBUTE_GRAPHICS_MODE |
+                    VBE_MODE_ATTRIBUTE_NOT_VGA_COMPATIBLE;
+    u32 framebuffer = 0;
+    int planes = 1, banks = 1;
+    u32 pages = GET_GLOBAL(VBE_total_memory) / ALIGN(height * linesize, 64*1024);
+    switch (memmodel) {
+    case MM_TEXT:
+        mode_attr &= ~VBE_MODE_ATTRIBUTE_GRAPHICS_MODE;
+        mode_attr |= VBE_MODE_ATTRIBUTE_TTY_BIOS_SUPPORT;
+        if (GET_GLOBAL(vmode_g->sstart) == SEG_MTEXT)
+            mode_attr &= ~VBE_MODE_ATTRIBUTE_COLOR_MODE;
+        pages = 1;
+        break;
+    case MM_CGA:
+        pages = 1;
+        banks = 2;
+        SET_FARVAR(seg, info->bank_size, 8);
+        break;
+    case MM_PLANAR:
+        planes = 4;
+        pages /= 4;
+        break;
+    default:
+        framebuffer = GET_GLOBAL(VBE_framebuffer);
+        if (framebuffer)
+            mode_attr |= VBE_MODE_ATTRIBUTE_LINEAR_FRAME_BUFFER_MODE;
+        break;
+    }
+    SET_FARVAR(seg, info->mode_attributes, mode_attr);
+    SET_FARVAR(seg, info->planes, planes);
+    SET_FARVAR(seg, info->pages, pages - 1);
+    SET_FARVAR(seg, info->banks, banks);
 
+    // Pixel color breakdown
+    u8 r_size, r_pos, g_size, g_pos, b_size, b_pos, a_size, a_pos;
     switch (depth) {
     case 15: r_size = 5; r_pos = 10; g_size = 5; g_pos = 5;
              b_size = 5; b_pos = 0; a_size = 1; a_pos = 15; break;
@@ -132,11 +159,13 @@ vbe_104f01(struct bregs *regs)
     case 24: r_size = 8; r_pos = 16; g_size = 8; g_pos = 8;
              b_size = 8; b_pos = 0; a_size = 0; a_pos = 0; break;
     case 32: r_size = 8; r_pos = 16; g_size = 8; g_pos = 8;
-             b_size = 8; b_pos = 0; a_size = 8; a_pos = 24; break;
+             b_size = 8; b_pos = 0; a_size = 8; a_pos = 24;
+             SET_FARVAR(seg, info->directcolor_info,
+                        VBE_DIRECTCOLOR_RESERVED_BITS_AVAILABLE);
+             break;
     default: r_size = 0; r_pos = 0; g_size = 0; g_pos = 0;
              b_size = 0; b_pos = 0; a_size = 0; a_pos = 0; break;
     }
-
     SET_FARVAR(seg, info->red_size, r_size);
     SET_FARVAR(seg, info->red_pos, r_pos);
     SET_FARVAR(seg, info->green_size, g_size);
@@ -146,31 +175,23 @@ vbe_104f01(struct bregs *regs)
     SET_FARVAR(seg, info->alpha_size, a_size);
     SET_FARVAR(seg, info->alpha_pos, a_pos);
 
-    if (depth == 32)
-        SET_FARVAR(seg, info->directcolor_info,
-                   VBE_DIRECTCOLOR_RESERVED_BITS_AVAILABLE);
-    else
-        SET_FARVAR(seg, info->directcolor_info, 0);
-
-    if (depth > 4)
-        SET_FARVAR(seg, info->phys_base, GET_GLOBAL(VBE_framebuffer));
-    else
-        SET_FARVAR(seg, info->phys_base, 0);
-
-    SET_FARVAR(seg, info->reserved1, 0);
-    SET_FARVAR(seg, info->reserved2, 0);
-    SET_FARVAR(seg, info->linear_bytes_per_scanline, linesize);
-    SET_FARVAR(seg, info->bank_pages, 0);
-    SET_FARVAR(seg, info->linear_pages, 0);
-    SET_FARVAR(seg, info->linear_red_size, r_size);
-    SET_FARVAR(seg, info->linear_red_pos, r_pos);
-    SET_FARVAR(seg, info->linear_green_size, g_size);
-    SET_FARVAR(seg, info->linear_green_pos, g_pos);
-    SET_FARVAR(seg, info->linear_blue_size, b_size);
-    SET_FARVAR(seg, info->linear_blue_pos, b_pos);
-    SET_FARVAR(seg, info->linear_alpha_size, a_size);
-    SET_FARVAR(seg, info->linear_alpha_pos, a_pos);
-    SET_FARVAR(seg, info->pixclock_max, 0);
+    // Linear framebuffer info.
+    if (framebuffer) {
+        SET_FARVAR(seg, info->phys_base, framebuffer);
+
+        SET_FARVAR(seg, info->reserved1, 0);
+        SET_FARVAR(seg, info->reserved2, 0);
+        SET_FARVAR(seg, info->linear_bytes_per_scanline, linesize);
+        SET_FARVAR(seg, info->linear_pages, 0);
+        SET_FARVAR(seg, info->linear_red_size, r_size);
+        SET_FARVAR(seg, info->linear_red_pos, r_pos);
+        SET_FARVAR(seg, info->linear_green_size, g_size);
+        SET_FARVAR(seg, info->linear_green_pos, g_pos);
+        SET_FARVAR(seg, info->linear_blue_size, b_size);
+        SET_FARVAR(seg, info->linear_blue_pos, b_pos);
+        SET_FARVAR(seg, info->linear_alpha_size, a_size);
+        SET_FARVAR(seg, info->linear_alpha_pos, a_pos);
+    }
 
     regs->ax = 0x004f;
 }
diff --git a/roms/seabios/vgasrc/vgafb.c b/roms/seabios/vgasrc/vgafb.c
index 233f3d5fb..3b2f367fb 100644
--- a/roms/seabios/vgasrc/vgafb.c
+++ b/roms/seabios/vgasrc/vgafb.c
@@ -8,6 +8,7 @@
 #include "vgabios.h" // vgafb_scroll
 #include "biosvar.h" // GET_BDA
 #include "util.h" // memset_far
+#include "byteorder.h" // cpu_to_be16
 #include "stdvga.h" // stdvga_planar4_plane
 
 
@@ -272,7 +273,7 @@ write_gfx_char_cga(struct vgamode_s *vmode_g
             for (j = 0; j < 8; j++)
                 if (fontline & (1<<j))
                     pixels |= (ca.attr & 0x03) << (j*2);
-            pixels = htons(pixels);
+            pixels = cpu_to_be16(pixels);
             if (ca.attr & 0x80)
                 pixels ^= GET_FARVAR(SEG_GRAPH, *(u16*)dest_far);
             SET_FARVAR(SEG_CTEXT, *(u16*)dest_far, pixels);
diff --git a/rules.mak b/rules.mak
index 1b173aa98..d0b04e44f 100644
--- a/rules.mak
+++ b/rules.mak
@@ -31,7 +31,7 @@ endif
 %.o: %.m
 	$(call quiet-command,$(OBJCC) $(QEMU_INCLUDES) $(QEMU_CFLAGS) $(QEMU_DGFLAGS) $(CFLAGS) -c -o $@ $<,"  OBJC  $(TARGET_DIR)$@")
 
-LINK = $(call quiet-command,$(CC) $(QEMU_CFLAGS) $(CFLAGS) $(LDFLAGS) -o $@ $(sort $(1)) $(LIBS),"  LINK  $(TARGET_DIR)$@")
+LINK = $(call quiet-command,$(CC) $(QEMU_CFLAGS) $(CFLAGS) $(LDFLAGS) -o $@ $(sort $(filter %.o, $1)) $(filter-out %.o, $1) $(LIBS),"  LINK  $(TARGET_DIR)$@")
 
 %$(EXESUF): %.o
 	$(call LINK,$^)
diff --git a/savevm.c b/savevm.c
index c7fe28314..5d04d5968 100644
--- a/savevm.c
+++ b/savevm.c
@@ -86,6 +86,7 @@
 #include "memory.h"
 #include "qmp-commands.h"
 #include "trace.h"
+#include "bitops.h"
 
 #define SELF_ANNOUNCE_ROUNDS 5
 
@@ -162,12 +163,7 @@ void qemu_announce_self(void)
 #define IO_BUF_SIZE 32768
 
 struct QEMUFile {
-    QEMUFilePutBufferFunc *put_buffer;
-    QEMUFileGetBufferFunc *get_buffer;
-    QEMUFileCloseFunc *close;
-    QEMUFileRateLimit *rate_limit;
-    QEMUFileSetRateLimit *set_rate_limit;
-    QEMUFileGetRateLimit *get_rate_limit;
+    const QEMUFileOps *ops;
     void *opaque;
     int is_write;
 
@@ -192,28 +188,52 @@ typedef struct QEMUFileSocket
     QEMUFile *file;
 } QEMUFileSocket;
 
+static int socket_get_fd(void *opaque)
+{
+    QEMUFileSocket *s = opaque;
+
+    return s->fd;
+}
+
 static int socket_get_buffer(void *opaque, uint8_t *buf, int64_t pos, int size)
 {
     QEMUFileSocket *s = opaque;
     ssize_t len;
 
-    do {
+    for (;;) {
         len = qemu_recv(s->fd, buf, size, 0);
-    } while (len == -1 && socket_error() == EINTR);
+        if (len != -1) {
+            break;
+        }
+        if (socket_error() == EAGAIN) {
+            assert(qemu_in_coroutine());
+            qemu_coroutine_yield();
+        } else if (socket_error() != EINTR) {
+            break;
+        }
+    }
 
-    if (len == -1)
+    if (len == -1) {
         len = -socket_error();
-
+    }
     return len;
 }
 
 static int socket_close(void *opaque)
 {
     QEMUFileSocket *s = opaque;
+    closesocket(s->fd);
     g_free(s);
     return 0;
 }
 
+static int stdio_get_fd(void *opaque)
+{
+    QEMUFileStdio *s = opaque;
+
+    return fileno(s->stdio_file);
+}
+
 static int stdio_put_buffer(void *opaque, const uint8_t *buf, int64_t pos, int size)
 {
     QEMUFileStdio *s = opaque;
@@ -226,10 +246,19 @@ static int stdio_get_buffer(void *opaque, uint8_t *buf, int64_t pos, int size)
     FILE *fp = s->stdio_file;
     int bytes;
 
-    do {
+    for (;;) {
         clearerr(fp);
         bytes = fread(buf, 1, size, fp);
-    } while ((bytes == 0) && ferror(fp) && (errno == EINTR));
+        if (bytes != 0 || !ferror(fp)) {
+            break;
+        }
+        if (errno == EAGAIN) {
+            assert(qemu_in_coroutine());
+            qemu_coroutine_yield();
+        } else if (errno != EINTR) {
+            break;
+        }
+    }
     return bytes;
 }
 
@@ -256,6 +285,18 @@ static int stdio_fclose(void *opaque)
     return ret;
 }
 
+static const QEMUFileOps stdio_pipe_read_ops = {
+    .get_fd =     stdio_get_fd,
+    .get_buffer = stdio_get_buffer,
+    .close =      stdio_pclose
+};
+
+static const QEMUFileOps stdio_pipe_write_ops = {
+    .get_fd =     stdio_get_fd,
+    .put_buffer = stdio_put_buffer,
+    .close =      stdio_pclose
+};
+
 QEMUFile *qemu_popen(FILE *stdio_file, const char *mode)
 {
     QEMUFileStdio *s;
@@ -270,11 +311,9 @@ QEMUFile *qemu_popen(FILE *stdio_file, const char *mode)
     s->stdio_file = stdio_file;
 
     if(mode[0] == 'r') {
-        s->file = qemu_fopen_ops(s, NULL, stdio_get_buffer, stdio_pclose, 
-				 NULL, NULL, NULL);
+        s->file = qemu_fopen_ops(s, &stdio_pipe_read_ops);
     } else {
-        s->file = qemu_fopen_ops(s, stdio_put_buffer, NULL, stdio_pclose, 
-				 NULL, NULL, NULL);
+        s->file = qemu_fopen_ops(s, &stdio_pipe_write_ops);
     }
     return s->file;
 }
@@ -291,16 +330,17 @@ QEMUFile *qemu_popen_cmd(const char *command, const char *mode)
     return qemu_popen(popen_file, mode);
 }
 
-int qemu_stdio_fd(QEMUFile *f)
-{
-    QEMUFileStdio *p;
-    int fd;
-
-    p = (QEMUFileStdio *)f->opaque;
-    fd = fileno(p->stdio_file);
+static const QEMUFileOps stdio_file_read_ops = {
+    .get_fd =     stdio_get_fd,
+    .get_buffer = stdio_get_buffer,
+    .close =      stdio_fclose
+};
 
-    return fd;
-}
+static const QEMUFileOps stdio_file_write_ops = {
+    .get_fd =     stdio_get_fd,
+    .put_buffer = stdio_put_buffer,
+    .close =      stdio_fclose
+};
 
 QEMUFile *qemu_fdopen(int fd, const char *mode)
 {
@@ -319,11 +359,9 @@ QEMUFile *qemu_fdopen(int fd, const char *mode)
         goto fail;
 
     if(mode[0] == 'r') {
-        s->file = qemu_fopen_ops(s, NULL, stdio_get_buffer, stdio_fclose, 
-				 NULL, NULL, NULL);
+        s->file = qemu_fopen_ops(s, &stdio_file_read_ops);
     } else {
-        s->file = qemu_fopen_ops(s, stdio_put_buffer, NULL, stdio_fclose, 
-				 NULL, NULL, NULL);
+        s->file = qemu_fopen_ops(s, &stdio_file_write_ops);
     }
     return s->file;
 
@@ -332,31 +370,21 @@ fail:
     return NULL;
 }
 
+static const QEMUFileOps socket_read_ops = {
+    .get_fd =     socket_get_fd,
+    .get_buffer = socket_get_buffer,
+    .close =      socket_close
+};
+
 QEMUFile *qemu_fopen_socket(int fd)
 {
     QEMUFileSocket *s = g_malloc0(sizeof(QEMUFileSocket));
 
     s->fd = fd;
-    s->file = qemu_fopen_ops(s, NULL, socket_get_buffer, socket_close, 
-			     NULL, NULL, NULL);
+    s->file = qemu_fopen_ops(s, &socket_read_ops);
     return s->file;
 }
 
-static int file_put_buffer(void *opaque, const uint8_t *buf,
-                            int64_t pos, int size)
-{
-    QEMUFileStdio *s = opaque;
-    fseek(s->stdio_file, pos, SEEK_SET);
-    return fwrite(buf, 1, size, s->stdio_file);
-}
-
-static int file_get_buffer(void *opaque, uint8_t *buf, int64_t pos, int size)
-{
-    QEMUFileStdio *s = opaque;
-    fseek(s->stdio_file, pos, SEEK_SET);
-    return fread(buf, 1, size, s->stdio_file);
-}
-
 QEMUFile *qemu_fopen(const char *filename, const char *mode)
 {
     QEMUFileStdio *s;
@@ -375,11 +403,9 @@ QEMUFile *qemu_fopen(const char *filename, const char *mode)
         goto fail;
     
     if(mode[0] == 'w') {
-        s->file = qemu_fopen_ops(s, file_put_buffer, NULL, stdio_fclose, 
-				 NULL, NULL, NULL);
+        s->file = qemu_fopen_ops(s, &stdio_file_write_ops);
     } else {
-        s->file = qemu_fopen_ops(s, NULL, file_get_buffer, stdio_fclose, 
-			       NULL, NULL, NULL);
+        s->file = qemu_fopen_ops(s, &stdio_file_read_ops);
     }
     return s->file;
 fail:
@@ -404,32 +430,31 @@ static int bdrv_fclose(void *opaque)
     return bdrv_flush(opaque);
 }
 
+static const QEMUFileOps bdrv_read_ops = {
+    .get_buffer = block_get_buffer,
+    .close =      bdrv_fclose
+};
+
+static const QEMUFileOps bdrv_write_ops = {
+    .put_buffer = block_put_buffer,
+    .close =      bdrv_fclose
+};
+
 static QEMUFile *qemu_fopen_bdrv(BlockDriverState *bs, int is_writable)
 {
     if (is_writable)
-        return qemu_fopen_ops(bs, block_put_buffer, NULL, bdrv_fclose, 
-			      NULL, NULL, NULL);
-    return qemu_fopen_ops(bs, NULL, block_get_buffer, bdrv_fclose, NULL, NULL, NULL);
+        return qemu_fopen_ops(bs, &bdrv_write_ops);
+    return qemu_fopen_ops(bs, &bdrv_read_ops);
 }
 
-QEMUFile *qemu_fopen_ops(void *opaque, QEMUFilePutBufferFunc *put_buffer,
-                         QEMUFileGetBufferFunc *get_buffer,
-                         QEMUFileCloseFunc *close,
-                         QEMUFileRateLimit *rate_limit,
-                         QEMUFileSetRateLimit *set_rate_limit,
-                         QEMUFileGetRateLimit *get_rate_limit)
+QEMUFile *qemu_fopen_ops(void *opaque, const QEMUFileOps *ops)
 {
     QEMUFile *f;
 
     f = g_malloc0(sizeof(QEMUFile));
 
     f->opaque = opaque;
-    f->put_buffer = put_buffer;
-    f->get_buffer = get_buffer;
-    f->close = close;
-    f->rate_limit = rate_limit;
-    f->set_rate_limit = set_rate_limit;
-    f->get_rate_limit = get_rate_limit;
+    f->ops = ops;
     f->is_write = 0;
 
     return f;
@@ -440,42 +465,29 @@ int qemu_file_get_error(QEMUFile *f)
     return f->last_error;
 }
 
-void qemu_file_set_error(QEMUFile *f, int ret)
+static void qemu_file_set_error(QEMUFile *f, int ret)
 {
     f->last_error = ret;
 }
 
-/** Sets last_error conditionally
- *
- * Sets last_error only if ret is negative _and_ no error
- * was set before.
- */
-static void qemu_file_set_if_error(QEMUFile *f, int ret)
-{
-    if (ret < 0 && !f->last_error) {
-        qemu_file_set_error(f, ret);
-    }
-}
-
 /** Flushes QEMUFile buffer
  *
- * In case of error, last_error is set.
  */
-void qemu_fflush(QEMUFile *f)
+static int qemu_fflush(QEMUFile *f)
 {
-    if (!f->put_buffer)
-        return;
+    int ret = 0;
 
-    if (f->is_write && f->buf_index > 0) {
-        int len;
+    if (!f->ops->put_buffer)
+        return 0;
 
-        len = f->put_buffer(f->opaque, f->buf, f->buf_offset, f->buf_index);
-        if (len > 0)
+    if (f->is_write && f->buf_index > 0) {
+        ret = f->ops->put_buffer(f->opaque, f->buf, f->buf_offset, f->buf_index);
+        if (ret >= 0) {
             f->buf_offset += f->buf_index;
-        else
-            qemu_file_set_error(f, -EINVAL);
+        }
         f->buf_index = 0;
     }
+    return ret;
 }
 
 static void qemu_fill_buffer(QEMUFile *f)
@@ -483,7 +495,7 @@ static void qemu_fill_buffer(QEMUFile *f)
     int len;
     int pending;
 
-    if (!f->get_buffer)
+    if (!f->ops->get_buffer)
         return;
 
     if (f->is_write)
@@ -496,31 +508,23 @@ static void qemu_fill_buffer(QEMUFile *f)
     f->buf_index = 0;
     f->buf_size = pending;
 
-    len = f->get_buffer(f->opaque, f->buf + pending, f->buf_offset,
+    len = f->ops->get_buffer(f->opaque, f->buf + pending, f->buf_offset,
                         IO_BUF_SIZE - pending);
     if (len > 0) {
         f->buf_size += len;
         f->buf_offset += len;
     } else if (len == 0) {
-        f->last_error = -EIO;
+        qemu_file_set_error(f, -EIO);
     } else if (len != -EAGAIN)
         qemu_file_set_error(f, len);
 }
 
-/** Calls close function and set last_error if needed
- *
- * Internal function. qemu_fflush() must be called before this.
- *
- * Returns f->close() return value, or 0 if close function is not set.
- */
-static int qemu_fclose_internal(QEMUFile *f)
+int qemu_get_fd(QEMUFile *f)
 {
-    int ret = 0;
-    if (f->close) {
-        ret = f->close(f->opaque);
-        qemu_file_set_if_error(f, ret);
+    if (f->ops->get_fd) {
+        return f->ops->get_fd(f->opaque);
     }
-    return ret;
+    return -1;
 }
 
 /** Closes the file
@@ -534,8 +538,14 @@ static int qemu_fclose_internal(QEMUFile *f)
 int qemu_fclose(QEMUFile *f)
 {
     int ret;
-    qemu_fflush(f);
-    ret = qemu_fclose_internal(f);
+    ret = qemu_fflush(f);
+
+    if (f->ops->close) {
+        int ret2 = f->ops->close(f->opaque);
+        if (ret >= 0) {
+            ret = ret2;
+        }
+    }
     /* If any error was spotted before closing, we should report it
      * instead of the close() return value.
      */
@@ -546,22 +556,26 @@ int qemu_fclose(QEMUFile *f)
     return ret;
 }
 
-void qemu_file_put_notify(QEMUFile *f)
+int qemu_file_put_notify(QEMUFile *f)
 {
-    f->put_buffer(f->opaque, NULL, 0, 0);
+    return f->ops->put_buffer(f->opaque, NULL, 0, 0);
 }
 
 void qemu_put_buffer(QEMUFile *f, const uint8_t *buf, int size)
 {
     int l;
 
-    if (!f->last_error && f->is_write == 0 && f->buf_index > 0) {
+    if (f->last_error) {
+        return;
+    }
+
+    if (f->is_write == 0 && f->buf_index > 0) {
         fprintf(stderr,
                 "Attempted to write to buffer while read buffer is not empty\n");
         abort();
     }
 
-    while (!f->last_error && size > 0) {
+    while (size > 0) {
         l = IO_BUF_SIZE - f->buf_index;
         if (l > size)
             l = size;
@@ -570,14 +584,23 @@ void qemu_put_buffer(QEMUFile *f, const uint8_t *buf, int size)
         f->buf_index += l;
         buf += l;
         size -= l;
-        if (f->buf_index >= IO_BUF_SIZE)
-            qemu_fflush(f);
+        if (f->buf_index >= IO_BUF_SIZE) {
+            int ret = qemu_fflush(f);
+            if (ret < 0) {
+                qemu_file_set_error(f, ret);
+                break;
+            }
+        }
     }
 }
 
 void qemu_put_byte(QEMUFile *f, int v)
 {
-    if (!f->last_error && f->is_write == 0 && f->buf_index > 0) {
+    if (f->last_error) {
+        return;
+    }
+
+    if (f->is_write == 0 && f->buf_index > 0) {
         fprintf(stderr,
                 "Attempted to write to buffer while read buffer is not empty\n");
         abort();
@@ -585,8 +608,12 @@ void qemu_put_byte(QEMUFile *f, int v)
 
     f->buf[f->buf_index++] = v;
     f->is_write = 1;
-    if (f->buf_index >= IO_BUF_SIZE)
-        qemu_fflush(f);
+    if (f->buf_index >= IO_BUF_SIZE) {
+        int ret = qemu_fflush(f);
+        if (ret < 0) {
+            qemu_file_set_error(f, ret);
+        }
+    }
 }
 
 static void qemu_file_skip(QEMUFile *f, int size)
@@ -671,44 +698,23 @@ int qemu_get_byte(QEMUFile *f)
     return result;
 }
 
-int64_t qemu_ftell(QEMUFile *f)
+static int64_t qemu_ftell(QEMUFile *f)
 {
     return f->buf_offset - f->buf_size + f->buf_index;
 }
 
-int64_t qemu_fseek(QEMUFile *f, int64_t pos, int whence)
-{
-    if (whence == SEEK_SET) {
-        /* nothing to do */
-    } else if (whence == SEEK_CUR) {
-        pos += qemu_ftell(f);
-    } else {
-        /* SEEK_END not supported */
-        return -1;
-    }
-    if (f->put_buffer) {
-        qemu_fflush(f);
-        f->buf_offset = pos;
-    } else {
-        f->buf_offset = pos;
-        f->buf_index = 0;
-        f->buf_size = 0;
-    }
-    return pos;
-}
-
 int qemu_file_rate_limit(QEMUFile *f)
 {
-    if (f->rate_limit)
-        return f->rate_limit(f->opaque);
+    if (f->ops->rate_limit)
+        return f->ops->rate_limit(f->opaque);
 
     return 0;
 }
 
 int64_t qemu_file_get_rate_limit(QEMUFile *f)
 {
-    if (f->get_rate_limit)
-        return f->get_rate_limit(f->opaque);
+    if (f->ops->get_rate_limit)
+        return f->ops->get_rate_limit(f->opaque);
 
     return 0;
 }
@@ -717,8 +723,8 @@ int64_t qemu_file_set_rate_limit(QEMUFile *f, int64_t new_rate)
 {
     /* any failed or completed migration keeps its state to allow probing of
      * migration data, but has no associated file anymore */
-    if (f && f->set_rate_limit)
-        return f->set_rate_limit(f->opaque, new_rate);
+    if (f && f->ops->set_rate_limit)
+        return f->ops->set_rate_limit(f->opaque, new_rate);
 
     return 0;
 }
@@ -1159,6 +1165,46 @@ const VMStateInfo vmstate_info_unused_buffer = {
     .put  = put_unused_buffer,
 };
 
+/* bitmaps (as defined by bitmap.h). Note that size here is the size
+ * of the bitmap in bits. The on-the-wire format of a bitmap is 64
+ * bit words with the bits in big endian order. The in-memory format
+ * is an array of 'unsigned long', which may be either 32 or 64 bits.
+ */
+/* This is the number of 64 bit words sent over the wire */
+#define BITS_TO_U64S(nr) DIV_ROUND_UP(nr, 64)
+static int get_bitmap(QEMUFile *f, void *pv, size_t size)
+{
+    unsigned long *bmp = pv;
+    int i, idx = 0;
+    for (i = 0; i < BITS_TO_U64S(size); i++) {
+        uint64_t w = qemu_get_be64(f);
+        bmp[idx++] = w;
+        if (sizeof(unsigned long) == 4 && idx < BITS_TO_LONGS(size)) {
+            bmp[idx++] = w >> 32;
+        }
+    }
+    return 0;
+}
+
+static void put_bitmap(QEMUFile *f, void *pv, size_t size)
+{
+    unsigned long *bmp = pv;
+    int i, idx = 0;
+    for (i = 0; i < BITS_TO_U64S(size); i++) {
+        uint64_t w = bmp[idx++];
+        if (sizeof(unsigned long) == 4 && idx < BITS_TO_LONGS(size)) {
+            w |= ((uint64_t)bmp[idx++]) << 32;
+        }
+        qemu_put_be64(f, w);
+    }
+}
+
+const VMStateInfo vmstate_info_bitmap = {
+    .name = "bitmap",
+    .get = get_bitmap,
+    .put = put_bitmap,
+};
+
 typedef struct CompatEntry {
     char idstr[256];
     int instance_id;
@@ -2201,7 +2247,6 @@ void qmp_xen_save_devices_state(const char *filename, Error **errp)
  the_end:
     if (saved_vm_running)
         vm_start();
-    return;
 }
 
 int load_vmstate(const char *name)
diff --git a/scripts/checkpatch.pl b/scripts/checkpatch.pl
index b98dc6cad..ec0aa4cd9 100755
--- a/scripts/checkpatch.pl
+++ b/scripts/checkpatch.pl
@@ -97,6 +97,9 @@ my $dbg_values = 0;
 my $dbg_possible = 0;
 my $dbg_type = 0;
 my $dbg_attr = 0;
+my $dbg_adv_dcs = 0;
+my $dbg_adv_checking = 0;
+my $dbg_adv_apw = 0;
 for my $key (keys %debug) {
 	## no critic
 	eval "\${dbg_$key} = '$debug{$key}';";
@@ -2486,8 +2489,11 @@ sub process {
 		if ($line =~ /(^.*)\bif\b/ && $line !~ /\#\s*if/) {
 			my ($level, $endln, @chunks) =
 				ctx_statement_full($linenr, $realcnt, 1);
-			#print "chunks<$#chunks> linenr<$linenr> endln<$endln> level<$level>\n";
-			#print "APW: <<$chunks[1][0]>><<$chunks[1][1]>>\n";
+                        if ($dbg_adv_apw) {
+                            print "APW: chunks<$#chunks> linenr<$linenr> endln<$endln> level<$level>\n";
+                            print "APW: <<$chunks[1][0]>><<$chunks[1][1]>>\n"
+                                if $#chunks >= 1;
+                        }
 			if ($#chunks >= 0 && $level == 0) {
 				my $allowed = 0;
 				my $seen = 0;
@@ -2512,18 +2518,22 @@ sub process {
 
 					$seen++ if ($block =~ /^\s*{/);
 
-					#print "cond<$cond> block<$block> allowed<$allowed>\n";
+                                        print "APW: cond<$cond> block<$block> allowed<$allowed>\n"
+                                            if $dbg_adv_apw;
 					if (statement_lines($cond) > 1) {
-						#print "APW: ALLOWED: cond<$cond>\n";
-						$allowed = 1;
+                                            print "APW: ALLOWED: cond<$cond>\n"
+                                                if $dbg_adv_apw;
+                                            $allowed = 1;
 					}
 					if ($block =~/\b(?:if|for|while)\b/) {
-						#print "APW: ALLOWED: block<$block>\n";
-						$allowed = 1;
+                                            print "APW: ALLOWED: block<$block>\n"
+                                                if $dbg_adv_apw;
+                                            $allowed = 1;
 					}
 					if (statement_block_size($block) > 1) {
-						#print "APW: ALLOWED: lines block<$block>\n";
-						$allowed = 1;
+                                            print "APW: ALLOWED: lines block<$block>\n"
+                                                if $dbg_adv_apw;
+                                            $allowed = 1;
 					}
 				}
 				if ($seen != ($#chunks + 1)) {
@@ -2537,32 +2547,41 @@ sub process {
 					$line !~ /\#\s*else/) {
 			my $allowed = 0;
 
-			# Check the pre-context.
-			if (substr($line, 0, $-[0]) =~ /(\}\s*)$/) {
-				#print "APW: ALLOWED: pre<$1>\n";
-				$allowed = 1;
-			}
+                        # Check the pre-context.
+                        if (substr($line, 0, $-[0]) =~ /(\}\s*)$/) {
+                            my $pre = $1;
+
+                            if ($line !~ /else/) {
+                                print "APW: ALLOWED: pre<$pre> line<$line>\n"
+                                    if $dbg_adv_apw;
+                                $allowed = 1;
+                            }
+                        }
 
 			my ($level, $endln, @chunks) =
 				ctx_statement_full($linenr, $realcnt, $-[0]);
 
 			# Check the condition.
 			my ($cond, $block) = @{$chunks[0]};
-			#print "CHECKING<$linenr> cond<$cond> block<$block>\n";
+                        print "CHECKING<$linenr> cond<$cond> block<$block>\n"
+                            if $dbg_adv_checking;
 			if (defined $cond) {
 				substr($block, 0, length($cond), '');
 			}
 			if (statement_lines($cond) > 1) {
-				#print "APW: ALLOWED: cond<$cond>\n";
-				$allowed = 1;
+                            print "APW: ALLOWED: cond<$cond>\n"
+                                if $dbg_adv_apw;
+                            $allowed = 1;
 			}
 			if ($block =~/\b(?:if|for|while)\b/) {
-				#print "APW: ALLOWED: block<$block>\n";
-				$allowed = 1;
+                            print "APW: ALLOWED: block<$block>\n"
+                                if $dbg_adv_apw;
+                            $allowed = 1;
 			}
 			if (statement_block_size($block) > 1) {
-				#print "APW: ALLOWED: lines block<$block>\n";
-				$allowed = 1;
+                            print "APW: ALLOWED: lines block<$block>\n"
+                                if $dbg_adv_apw;
+                            $allowed = 1;
 			}
 			# Check the post-context.
 			if (defined $chunks[1]) {
@@ -2571,10 +2590,13 @@ sub process {
 					substr($block, 0, length($cond), '');
 				}
 				if ($block =~ /^\s*\{/) {
-					#print "APW: ALLOWED: chunk-1 block<$block>\n";
-					$allowed = 1;
+                                    print "APW: ALLOWED: chunk-1 block<$block>\n"
+                                        if $dbg_adv_apw;
+                                    $allowed = 1;
 				}
 			}
+                        print "DCS: level=$level block<$block> allowed=$allowed\n"
+                            if $dbg_adv_dcs;
 			if ($level == 0 && $block !~ /^\s*\{/ && !$allowed) {
 				my $herectx = $here . "\n";;
 				my $cnt = statement_rawlines($block);
diff --git a/scripts/kvm/kvm_stat b/scripts/kvm/kvm_stat
index e8d68f05c..762544b19 100755
--- a/scripts/kvm/kvm_stat
+++ b/scripts/kvm/kvm_stat
@@ -170,6 +170,12 @@ vendor_exit_reasons = {
     'IBM/S390': s390_exit_reasons,
 }
 
+syscall_numbers = {
+    'IBM/S390': 331,
+}
+
+sc_perf_evt_open = 298
+
 exit_reasons = None
 
 for line in file('/proc/cpuinfo').readlines():
@@ -177,7 +183,8 @@ for line in file('/proc/cpuinfo').readlines():
         for flag in line.split():
             if flag in vendor_exit_reasons:
                 exit_reasons = vendor_exit_reasons[flag]
-
+            if flag in syscall_numbers:
+                sc_perf_evt_open = syscall_numbers[flag]
 filters = {
     'kvm_exit': ('exit_reason', exit_reasons)
 }
@@ -206,7 +213,7 @@ class perf_event_attr(ctypes.Structure):
                 ('bp_len', ctypes.c_uint64),
                 ]
 def _perf_event_open(attr, pid, cpu, group_fd, flags):
-    return syscall(298, ctypes.pointer(attr), ctypes.c_int(pid),
+    return syscall(sc_perf_evt_open, ctypes.pointer(attr), ctypes.c_int(pid),
                    ctypes.c_int(cpu), ctypes.c_int(group_fd),
                    ctypes.c_long(flags))
 
diff --git a/scripts/qapi-types.py b/scripts/qapi-types.py
index cf601ae2d..6bc239187 100644
--- a/scripts/qapi-types.py
+++ b/scripts/qapi-types.py
@@ -28,6 +28,16 @@ typedef struct %(name)sList
 ''',
                  name=name)
 
+def generate_fwd_enum_struct(name, members):
+    return mcgen('''
+typedef struct %(name)sList
+{
+    %(name)s value;
+    struct %(name)sList *next;
+} %(name)sList;
+''',
+                 name=name)
+
 def generate_struct(structname, fieldname, members):
     ret = mcgen('''
 struct %(name)s
@@ -81,9 +91,9 @@ const char *%(name)s_lookup[] = {
 
 def generate_enum_name(name):
     if name.isupper():
-        return c_fun(name)
+        return c_fun(name, False)
     new_name = ''
-    for c in c_fun(name):
+    for c in c_fun(name, False):
         if c.isupper():
             new_name += '_'
         new_name += c
@@ -263,7 +273,8 @@ fdecl.write(mcgen('''
 #ifndef %(guard)s
 #define %(guard)s
 
-#include "qemu-common.h"
+#include <stdbool.h>
+#include <stdint.h>
 
 ''',
                   guard=guardname(h_file)))
@@ -276,7 +287,8 @@ for expr in exprs:
     if expr.has_key('type'):
         ret += generate_fwd_struct(expr['type'], expr['data'])
     elif expr.has_key('enum'):
-        ret += generate_enum(expr['enum'], expr['data'])
+        ret += generate_enum(expr['enum'], expr['data']) + "\n"
+        ret += generate_fwd_enum_struct(expr['enum'], expr['data'])
         fdef.write(generate_enum_lookup(expr['enum'], expr['data']))
     elif expr.has_key('union'):
         ret += generate_fwd_struct(expr['union'], expr['data']) + "\n"
@@ -300,6 +312,9 @@ for expr in exprs:
         fdef.write(generate_type_cleanup(expr['union'] + "List") + "\n")
         ret += generate_type_cleanup_decl(expr['union'])
         fdef.write(generate_type_cleanup(expr['union']) + "\n")
+    elif expr.has_key('enum'):
+        ret += generate_type_cleanup_decl(expr['enum'] + "List")
+        fdef.write(generate_type_cleanup(expr['enum'] + "List") + "\n")
     else:
         continue
     fdecl.write(ret)
diff --git a/scripts/qapi-visit.py b/scripts/qapi-visit.py
index 04ef7c41a..a360de719 100644
--- a/scripts/qapi-visit.py
+++ b/scripts/qapi-visit.py
@@ -157,7 +157,7 @@ void visit_type_%(name)s(Visitor *m, %(name)s ** obj, const char *name, Error **
     if (!error_is_set(errp)) {
         visit_start_struct(m, (void **)obj, "%(name)s", name, sizeof(%(name)s), &err);
         if (!err) {
-            if (!obj || *obj) {
+            if (obj && *obj) {
                 visit_type_%(name)sKind(m, &(*obj)->kind, "type", &err);
                 if (!err) {
                     switch ((*obj)->kind) {
@@ -173,7 +173,7 @@ void visit_type_%(name)s(Visitor *m, %(name)s ** obj, const char *name, Error **
                 break;
 ''',
                 abbrev = de_camel_case(name).upper(),
-                enum = c_fun(de_camel_case(key)).upper(),
+                enum = c_fun(de_camel_case(key),False).upper(),
                 c_type=members[key],
                 c_name=c_fun(key))
 
@@ -217,6 +217,16 @@ void visit_type_%(name)sList(Visitor *m, %(name)sList ** obj, const char *name,
 
     return ret
 
+def generate_enum_declaration(name, members, genlist=True):
+    ret = ""
+    if genlist:
+        ret += mcgen('''
+void visit_type_%(name)sList(Visitor *m, %(name)sList ** obj, const char *name, Error **errp);
+''',
+                     name=name)
+
+    return ret
+
 def generate_decl_enum(name, members, genlist=True):
     return mcgen('''
 
@@ -335,10 +345,12 @@ for expr in exprs:
         ret += generate_declaration(expr['union'], expr['data'])
         fdecl.write(ret)
     elif expr.has_key('enum'):
-        ret = generate_visit_enum(expr['enum'], expr['data'])
+        ret = generate_visit_list(expr['enum'], expr['data'])
+        ret += generate_visit_enum(expr['enum'], expr['data'])
         fdef.write(ret)
 
         ret = generate_decl_enum(expr['enum'], expr['data'])
+        ret += generate_enum_declaration(expr['enum'], expr['data'])
         fdecl.write(ret)
 
 fdecl.write('''
diff --git a/scripts/qapi.py b/scripts/qapi.py
index 122b4cb6d..afc5f32ae 100644
--- a/scripts/qapi.py
+++ b/scripts/qapi.py
@@ -141,7 +141,7 @@ def camel_case(name):
             new_name += ch.lower()
     return new_name
 
-def c_var(name):
+def c_var(name, protect=True):
     # ANSI X3J11/88-090, 3.1.1
     c89_words = set(['auto', 'break', 'case', 'char', 'const', 'continue',
                      'default', 'do', 'double', 'else', 'enum', 'extern', 'float',
@@ -156,12 +156,14 @@ def c_var(name):
     # GCC http://gcc.gnu.org/onlinedocs/gcc-4.7.1/gcc/C-Extensions.html
     # excluding _.*
     gcc_words = set(['asm', 'typeof'])
-    if name in c89_words | c99_words | c11_words | gcc_words:
+    # namespace pollution:
+    polluted_words = set(['unix'])
+    if protect and (name in c89_words | c99_words | c11_words | gcc_words | polluted_words):
         return "q_" + name
     return name.replace('-', '_').lstrip("*")
 
-def c_fun(name):
-    return c_var(name).replace('.', '_')
+def c_fun(name, protect=True):
+    return c_var(name, protect).replace('.', '_')
 
 def c_list_type(name):
     return '%sList' % name
diff --git a/scripts/tracetool/backend/dtrace.py b/scripts/tracetool/backend/dtrace.py
index 9cab75cde..23c43e277 100644
--- a/scripts/tracetool/backend/dtrace.py
+++ b/scripts/tracetool/backend/dtrace.py
@@ -73,6 +73,15 @@ def d(events):
         '};')
 
 
+# Technically 'self' is not used by systemtap yet, but
+# they recommended we keep it in the reserved list anyway
+RESERVED_WORDS = (
+    'break', 'catch', 'continue', 'delete', 'else', 'for',
+    'foreach', 'function', 'global', 'if', 'in', 'limit',
+    'long', 'next', 'probe', 'return', 'self', 'string',
+    'try', 'while'
+    )
+
 def stap(events):
     for e in events:
         # Define prototype for probe arguments
@@ -87,7 +96,7 @@ def stap(events):
         if len(e.args) > 0:
             for name in e.args.names():
                 # Append underscore to reserved keywords
-                if name in ('limit', 'in', 'next', 'self'):
+                if name in RESERVED_WORDS:
                     name += '_'
                 out('  %s = $arg%d;' % (name, i))
                 i += 1
diff --git a/scripts/update-linux-headers.sh b/scripts/update-linux-headers.sh
index a639c5bff..4c7b566fd 100755
--- a/scripts/update-linux-headers.sh
+++ b/scripts/update-linux-headers.sh
@@ -28,7 +28,22 @@ if [ -z "$output" ]; then
     output="$PWD"
 fi
 
-for arch in x86 powerpc s390; do
+# This will pick up non-directories too (eg "Kconfig") but we will
+# ignore them in the next loop.
+ARCHLIST=$(cd "$linux/arch" && echo *)
+
+for arch in $ARCHLIST; do
+    # Discard anything which isn't a KVM-supporting architecture
+    if ! [ -e "$linux/arch/$arch/include/asm/kvm.h" ] &&
+        ! [ -e "$linux/arch/$arch/include/uapi/asm/kvm.h" ] ; then
+        continue
+    fi
+
+    # Blacklist architectures which have KVM headers but are actually dead
+    if [ "$arch" = "ia64" ]; then
+        continue
+    fi
+
     make -C "$linux" INSTALL_HDR_PATH="$tmpdir" SRCARCH=$arch headers_install
 
     rm -rf "$output/linux-headers/asm-$arch"
@@ -43,7 +58,7 @@ done
 
 rm -rf "$output/linux-headers/linux"
 mkdir -p "$output/linux-headers/linux"
-for header in kvm.h kvm_para.h vhost.h virtio_config.h virtio_ring.h; do
+for header in kvm.h kvm_para.h vfio.h vhost.h virtio_config.h virtio_ring.h; do
     cp "$tmpdir/include/linux/$header" "$output/linux-headers/linux"
 done
 rm -rf "$output/linux-headers/asm-generic"
diff --git a/slirp/Makefile.objs b/slirp/Makefile.objs
index bb43d3c08..2daa9dc58 100644
--- a/slirp/Makefile.objs
+++ b/slirp/Makefile.objs
@@ -1,3 +1,3 @@
-common-obj-y = cksum.o if.o ip_icmp.o ip_input.o ip_output.o
+common-obj-y = cksum.o if.o ip_icmp.o ip_input.o ip_output.o dnssearch.o
 common-obj-y += slirp.o mbuf.o misc.o sbuf.o socket.o tcp_input.o tcp_output.o
 common-obj-y += tcp_subr.o tcp_timer.o udp.o bootp.o tftp.o arp_table.o
diff --git a/slirp/arp_table.c b/slirp/arp_table.c
index 5d7b8acd1..bf698c1ac 100644
--- a/slirp/arp_table.c
+++ b/slirp/arp_table.c
@@ -38,7 +38,9 @@ void arp_table_add(Slirp *slirp, uint32_t ip_addr, uint8_t ethaddr[ETH_ALEN])
                 ethaddr[3], ethaddr[4], ethaddr[5]));
 
     /* Check 0.0.0.0/8 invalid source-only addresses */
-    assert((ip_addr & htonl(~(0xf << 28))) != 0);
+    if ((ip_addr & htonl(~(0xf << 28))) == 0) {
+        return;
+    }
 
     if (ip_addr == 0xffffffff || ip_addr == broadcast_addr) {
         /* Do not register broadcast addresses */
diff --git a/slirp/bootp.c b/slirp/bootp.c
index 64eac7d10..b7db9fa33 100644
--- a/slirp/bootp.c
+++ b/slirp/bootp.c
@@ -287,6 +287,18 @@ static void bootp_reply(Slirp *slirp, const struct bootp_t *bp)
             memcpy(q, slirp->client_hostname, val);
             q += val;
         }
+
+        if (slirp->vdnssearch) {
+            size_t spaceleft = sizeof(rbp->bp_vend) - (q - rbp->bp_vend);
+            val = slirp->vdnssearch_len;
+            if (val + 1 > spaceleft) {
+                g_warning("DHCP packet size exceeded, "
+                    "omitting domain-search option.");
+            } else {
+                memcpy(q, slirp->vdnssearch, val);
+                q += val;
+            }
+        }
     } else {
         static const char nak_msg[] = "requested address not available";
 
diff --git a/slirp/dnssearch.c b/slirp/dnssearch.c
new file mode 100644
index 000000000..4c9064ecb
--- /dev/null
+++ b/slirp/dnssearch.c
@@ -0,0 +1,314 @@
+/*
+ * Domain search option for DHCP (RFC 3397)
+ *
+ * Copyright (c) 2012 Klaus Stengel
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+#include <glib.h>
+#include "slirp.h"
+
+static const uint8_t RFC3397_OPT_DOMAIN_SEARCH = 119;
+static const uint8_t MAX_OPT_LEN = 255;
+static const uint8_t OPT_HEADER_LEN = 2;
+static const uint8_t REFERENCE_LEN = 2;
+
+struct compact_domain;
+
+typedef struct compact_domain {
+    struct compact_domain *self;
+    struct compact_domain *refdom;
+    uint8_t *labels;
+    size_t len;
+    size_t common_octets;
+} CompactDomain;
+
+static size_t
+domain_suffix_diffoff(const CompactDomain *a, const CompactDomain *b)
+{
+    size_t la = a->len, lb = b->len;
+    uint8_t *da = a->labels + la, *db = b->labels + lb;
+    size_t i, lm = (la < lb) ? la : lb;
+
+    for (i = 0; i < lm; i++) {
+        da--; db--;
+        if (*da != *db) {
+            break;
+        }
+    }
+    return i;
+}
+
+static int domain_suffix_ord(const void *cva, const void *cvb)
+{
+    const CompactDomain *a = cva, *b = cvb;
+    size_t la = a->len, lb = b->len;
+    size_t doff = domain_suffix_diffoff(a, b);
+    uint8_t ca = a->labels[la - doff];
+    uint8_t cb = b->labels[lb - doff];
+
+    if (ca < cb) {
+        return -1;
+    }
+    if (ca > cb) {
+        return 1;
+    }
+    if (la < lb) {
+        return -1;
+    }
+    if (la > lb) {
+        return 1;
+    }
+    return 0;
+}
+
+static size_t domain_common_label(CompactDomain *a, CompactDomain *b)
+{
+    size_t res, doff = domain_suffix_diffoff(a, b);
+    uint8_t *first_eq_pos = a->labels + (a->len - doff);
+    uint8_t *label = a->labels;
+
+    while (*label && label < first_eq_pos) {
+        label += *label + 1;
+    }
+    res = a->len - (label - a->labels);
+    /* only report if it can help to reduce the packet size */
+    return (res > REFERENCE_LEN) ? res : 0;
+}
+
+static void domain_fixup_order(CompactDomain *cd, size_t n)
+{
+    size_t i;
+
+    for (i = 0; i < n; i++) {
+        CompactDomain *cur = cd + i, *next = cd[i].self;
+
+        while (!cur->common_octets) {
+            CompactDomain *tmp = next->self; /* backup target value */
+
+            next->self = cur;
+            cur->common_octets++;
+
+            cur = next;
+            next = tmp;
+        }
+    }
+}
+
+static void domain_mklabels(CompactDomain *cd, const char *input)
+{
+    uint8_t *len_marker = cd->labels;
+    uint8_t *output = len_marker; /* pre-incremented */
+    const char *in = input;
+    char cur_chr;
+    size_t len = 0;
+
+    if (cd->len == 0) {
+        goto fail;
+    }
+    cd->len++;
+
+    do {
+        cur_chr = *in++;
+        if (cur_chr == '.' || cur_chr == '\0') {
+            len = output - len_marker;
+            if ((len == 0 && cur_chr == '.') || len >= 64) {
+                goto fail;
+            }
+            *len_marker = len;
+
+            output++;
+            len_marker = output;
+        } else {
+            output++;
+            *output = cur_chr;
+        }
+    } while (cur_chr != '\0');
+
+    /* ensure proper zero-termination */
+    if (len != 0) {
+        *len_marker = 0;
+        cd->len++;
+    }
+    return;
+
+fail:
+    g_warning("failed to parse domain name '%s'\n", input);
+    cd->len = 0;
+}
+
+static void
+domain_mkxrefs(CompactDomain *doms, CompactDomain *last, size_t depth)
+{
+    CompactDomain *i = doms, *target = doms;
+
+    do {
+        if (i->labels < target->labels) {
+            target = i;
+        }
+    } while (i++ != last);
+
+    for (i = doms; i != last; i++) {
+        CompactDomain *group_last;
+        size_t next_depth;
+
+        if (i->common_octets == depth) {
+            continue;
+        }
+
+        next_depth = -1;
+        for (group_last = i; group_last != last; group_last++) {
+            size_t co = group_last->common_octets;
+            if (co <= depth) {
+                break;
+            }
+            if (co < next_depth) {
+                next_depth = co;
+            }
+        }
+        domain_mkxrefs(i, group_last, next_depth);
+
+        i = group_last;
+        if (i == last) {
+            break;
+        }
+    }
+
+    if (depth == 0) {
+        return;
+    }
+
+    i = doms;
+    do {
+        if (i != target && i->refdom == NULL) {
+            i->refdom = target;
+            i->common_octets = depth;
+        }
+    } while (i++ != last);
+}
+
+static size_t domain_compactify(CompactDomain *domains, size_t n)
+{
+    uint8_t *start = domains->self->labels, *outptr = start;
+    size_t i;
+
+    for (i = 0; i < n; i++) {
+        CompactDomain *cd = domains[i].self;
+        CompactDomain *rd = cd->refdom;
+
+        if (rd != NULL) {
+            size_t moff = (rd->labels - start)
+                    + (rd->len - cd->common_octets);
+            if (moff < 0x3FFFu) {
+                cd->len -= cd->common_octets - 2;
+                cd->labels[cd->len - 1] = moff & 0xFFu;
+                cd->labels[cd->len - 2] = 0xC0u | (moff >> 8);
+            }
+        }
+
+        if (cd->labels != outptr) {
+            memmove(outptr, cd->labels, cd->len);
+            cd->labels = outptr;
+        }
+        outptr += cd->len;
+    }
+    return outptr - start;
+}
+
+int translate_dnssearch(Slirp *s, const char **names)
+{
+    size_t blocks, bsrc_start, bsrc_end, bdst_start;
+    size_t i, num_domains, memreq = 0;
+    uint8_t *result = NULL, *outptr;
+    CompactDomain *domains = NULL;
+    const char **nameptr = names;
+
+    while (*nameptr != NULL) {
+        nameptr++;
+    }
+
+    num_domains = nameptr - names;
+    if (num_domains == 0) {
+        return -2;
+    }
+
+    domains = g_malloc(num_domains * sizeof(*domains));
+
+    for (i = 0; i < num_domains; i++) {
+        size_t nlen = strlen(names[i]);
+        memreq += nlen + 2; /* 1 zero octet + 1 label length octet */
+        domains[i].self = domains + i;
+        domains[i].len = nlen;
+        domains[i].common_octets = 0;
+        domains[i].refdom = NULL;
+    }
+
+    /* reserve extra 2 header bytes for each 255 bytes of output */
+    memreq += ((memreq + MAX_OPT_LEN - 1) / MAX_OPT_LEN) * OPT_HEADER_LEN;
+    result = g_malloc(memreq * sizeof(*result));
+
+    outptr = result;
+    for (i = 0; i < num_domains; i++) {
+        domains[i].labels = outptr;
+        domain_mklabels(domains + i, names[i]);
+        outptr += domains[i].len;
+    }
+
+    if (outptr == result) {
+        g_free(domains);
+        g_free(result);
+        return -1;
+    }
+
+    qsort(domains, num_domains, sizeof(*domains), domain_suffix_ord);
+    domain_fixup_order(domains, num_domains);
+
+    for (i = 1; i < num_domains; i++) {
+        size_t cl = domain_common_label(domains + i - 1, domains + i);
+        domains[i - 1].common_octets = cl;
+    }
+
+    domain_mkxrefs(domains, domains + num_domains - 1, 0);
+    memreq = domain_compactify(domains, num_domains);
+
+    blocks = (memreq + MAX_OPT_LEN - 1) / MAX_OPT_LEN;
+    bsrc_end = memreq;
+    bsrc_start = (blocks - 1) * MAX_OPT_LEN;
+    bdst_start = bsrc_start + blocks * OPT_HEADER_LEN;
+    memreq += blocks * OPT_HEADER_LEN;
+
+    while (blocks--) {
+        size_t len = bsrc_end - bsrc_start;
+        memmove(result + bdst_start, result + bsrc_start, len);
+        result[bdst_start - 2] = RFC3397_OPT_DOMAIN_SEARCH;
+        result[bdst_start - 1] = len;
+        bsrc_end = bsrc_start;
+        bsrc_start -= MAX_OPT_LEN;
+        bdst_start -= MAX_OPT_LEN + OPT_HEADER_LEN;
+    }
+
+    g_free(domains);
+    s->vdnssearch = result;
+    s->vdnssearch_len = memreq;
+    return 0;
+}
diff --git a/slirp/ip_icmp.h b/slirp/ip_icmp.h
index 1a1af91e1..be4426b8e 100644
--- a/slirp/ip_icmp.h
+++ b/slirp/ip_icmp.h
@@ -92,8 +92,8 @@ struct icmp {
 
 /*
  * Lower bounds on packet lengths for various types.
- * For the error advice packets must first insure that the
- * packet is large enought to contain the returned ip header.
+ * For the error advice packets must first ensure that the
+ * packet is large enough to contain the returned ip header.
  * Only then can we do the check to see if 64 bits of packet
  * data have been returned, since we need to check the returned
  * ip header length.
diff --git a/slirp/ip_input.c b/slirp/ip_input.c
index ce24faf16..6f4cff8fd 100644
--- a/slirp/ip_input.c
+++ b/slirp/ip_input.c
@@ -213,7 +213,6 @@ ip_input(struct mbuf *m)
 	return;
 bad:
 	m_free(m);
-	return;
 }
 
 #define iptofrag(P) ((struct ipasfrag *)(((char*)(P)) - sizeof(struct qlink)))
diff --git a/slirp/libslirp.h b/slirp/libslirp.h
index 9b471b505..49609c2ad 100644
--- a/slirp/libslirp.h
+++ b/slirp/libslirp.h
@@ -12,7 +12,8 @@ Slirp *slirp_init(int restricted, struct in_addr vnetwork,
                   struct in_addr vnetmask, struct in_addr vhost,
                   const char *vhostname, const char *tftp_path,
                   const char *bootfile, struct in_addr vdhcp_start,
-                  struct in_addr vnameserver, void *opaque);
+                  struct in_addr vnameserver, const char **vdnssearch,
+                  void *opaque);
 void slirp_cleanup(Slirp *slirp);
 
 void slirp_update_timeout(uint32_t *timeout);
diff --git a/slirp/misc.c b/slirp/misc.c
index 0bee86433..664532a66 100644
--- a/slirp/misc.c
+++ b/slirp/misc.c
@@ -253,20 +253,6 @@ void lprint(const char *format, ...)
     va_end(args);
 }
 
-void
-u_sleep(int usec)
-{
-	struct timeval t;
-	fd_set fdset;
-
-	FD_ZERO(&fdset);
-
-	t.tv_sec = 0;
-	t.tv_usec = usec * 1000;
-
-	select(0, &fdset, &fdset, &fdset, &t);
-}
-
 void slirp_connection_info(Slirp *slirp, Monitor *mon)
 {
     const char * const tcpstates[] = {
diff --git a/slirp/misc.h b/slirp/misc.h
index ed40a103c..cc36aeb95 100644
--- a/slirp/misc.h
+++ b/slirp/misc.h
@@ -64,7 +64,6 @@ void snooze_hup(int);
 void snooze(void);
 void relay(int);
 void add_emu(char *);
-void u_sleep(int);
 void fd_nonblock(int);
 void fd_block(int);
 int rsh_exec(struct socket *, struct socket *, char *, char *, char *);
diff --git a/slirp/slirp.c b/slirp/slirp.c
index 38e0a2193..3395d509a 100644
--- a/slirp/slirp.c
+++ b/slirp/slirp.c
@@ -203,7 +203,8 @@ Slirp *slirp_init(int restricted, struct in_addr vnetwork,
                   struct in_addr vnetmask, struct in_addr vhost,
                   const char *vhostname, const char *tftp_path,
                   const char *bootfile, struct in_addr vdhcp_start,
-                  struct in_addr vnameserver, void *opaque)
+                  struct in_addr vnameserver, const char **vdnssearch,
+                  void *opaque)
 {
     Slirp *slirp = g_malloc0(sizeof(Slirp));
 
@@ -233,6 +234,10 @@ Slirp *slirp_init(int restricted, struct in_addr vnetwork,
     slirp->vdhcp_startaddr = vdhcp_start;
     slirp->vnameserver_addr = vnameserver;
 
+    if (vdnssearch) {
+        translate_dnssearch(slirp, vdnssearch);
+    }
+
     slirp->opaque = opaque;
 
     register_savevm(NULL, "slirp", 0, 3,
@@ -252,6 +257,7 @@ void slirp_cleanup(Slirp *slirp)
     ip_cleanup(slirp);
     m_cleanup(slirp);
 
+    g_free(slirp->vdnssearch);
     g_free(slirp->tftp_prefix);
     g_free(slirp->bootp_filename);
     g_free(slirp);
diff --git a/slirp/slirp.h b/slirp/slirp.h
index f2c5eca89..0107b07e6 100644
--- a/slirp/slirp.h
+++ b/slirp/slirp.h
@@ -235,6 +235,8 @@ struct Slirp {
     /* bootp/dhcp states */
     BOOTPClient bootp_clients[NB_BOOTP_CLIENTS];
     char *bootp_filename;
+    size_t vdnssearch_len;
+    uint8_t *vdnssearch;
 
     /* tcp states */
     struct socket tcb;
@@ -294,6 +296,9 @@ void lprint(const char *, ...) GCC_FMT_ATTR(1, 2);
 #define SO_OPTIONS DO_KEEPALIVE
 #define TCP_MAXIDLE (TCPTV_KEEPCNT * TCPTV_KEEPINTVL)
 
+/* dnssearch.c */
+int translate_dnssearch(Slirp *s, const char ** names);
+
 /* cksum.c */
 int cksum(struct mbuf *m, int len);
 
diff --git a/slirp/tcp_input.c b/slirp/tcp_input.c
index 942aaf44f..6440eae7f 100644
--- a/slirp/tcp_input.c
+++ b/slirp/tcp_input.c
@@ -1281,8 +1281,6 @@ drop:
 	 * Drop space held by incoming segment and return.
 	 */
 	m_free(m);
-
-	return;
 }
 
 static void
diff --git a/slirp/tcp_subr.c b/slirp/tcp_subr.c
index 025b37436..1542e4361 100644
--- a/slirp/tcp_subr.c
+++ b/slirp/tcp_subr.c
@@ -114,9 +114,9 @@ tcp_respond(struct tcpcb *tp, struct tcpiphdr *ti, struct mbuf *m,
 	int win = 0;
 
 	DEBUG_CALL("tcp_respond");
-	DEBUG_ARG("tp = %lx", (long)tp);
-	DEBUG_ARG("ti = %lx", (long)ti);
-	DEBUG_ARG("m = %lx", (long)m);
+	DEBUG_ARG("tp = %p", tp);
+	DEBUG_ARG("ti = %p", ti);
+	DEBUG_ARG("m = %p", m);
 	DEBUG_ARG("ack = %u", ack);
 	DEBUG_ARG("seq = %u", seq);
 	DEBUG_ARG("flags = %x", flags);
@@ -124,7 +124,7 @@ tcp_respond(struct tcpcb *tp, struct tcpiphdr *ti, struct mbuf *m,
 	if (tp)
 		win = sbspace(&tp->t_socket->so_rcv);
         if (m == NULL) {
-		if ((m = m_get(tp->t_socket->slirp)) == NULL)
+		if (!tp || (m = m_get(tp->t_socket->slirp)) == NULL)
 			return;
 		tlen = 0;
 		m->m_data += IF_MAXLINKHDR;
diff --git a/slirp/tftp.c b/slirp/tftp.c
index b78765f3a..1a79c45cf 100644
--- a/slirp/tftp.c
+++ b/slirp/tftp.c
@@ -37,6 +37,10 @@ static inline void tftp_session_update(struct tftp_session *spt)
 
 static void tftp_session_terminate(struct tftp_session *spt)
 {
+    if (spt->fd >= 0) {
+        close(spt->fd);
+        spt->fd = -1;
+    }
     g_free(spt->filename);
     spt->slirp = NULL;
 }
@@ -54,7 +58,7 @@ static int tftp_session_allocate(Slirp *slirp, struct tftp_t *tp)
 
     /* sessions time out after 5 inactive seconds */
     if ((int)(curtime - spt->timestamp) > 5000) {
-        g_free(spt->filename);
+        tftp_session_terminate(spt);
         goto found;
     }
   }
@@ -64,6 +68,7 @@ static int tftp_session_allocate(Slirp *slirp, struct tftp_t *tp)
  found:
   memset(spt, 0, sizeof(*spt));
   memcpy(&spt->client_ip, &tp->ip.ip_src, sizeof(spt->client_ip));
+  spt->fd = -1;
   spt->client_port = tp->udp.uh_sport;
   spt->slirp = slirp;
 
@@ -92,37 +97,36 @@ static int tftp_session_find(Slirp *slirp, struct tftp_t *tp)
   return -1;
 }
 
-static int tftp_read_data(struct tftp_session *spt, uint16_t block_nr,
+static int tftp_read_data(struct tftp_session *spt, uint32_t block_nr,
                           uint8_t *buf, int len)
 {
-  int fd;
-  int bytes_read = 0;
+    int bytes_read = 0;
 
-  fd = open(spt->filename, O_RDONLY | O_BINARY);
+    if (spt->fd < 0) {
+        spt->fd = open(spt->filename, O_RDONLY | O_BINARY);
+    }
 
-  if (fd < 0) {
-    return -1;
-  }
+    if (spt->fd < 0) {
+        return -1;
+    }
 
-  if (len) {
-    lseek(fd, block_nr * 512, SEEK_SET);
+    if (len) {
+        lseek(spt->fd, block_nr * 512, SEEK_SET);
 
-    bytes_read = read(fd, buf, len);
-  }
-
-  close(fd);
+        bytes_read = read(spt->fd, buf, len);
+    }
 
-  return bytes_read;
+    return bytes_read;
 }
 
 static int tftp_send_oack(struct tftp_session *spt,
-                          const char *key, uint32_t value,
+                          const char *keys[], uint32_t values[], int nb,
                           struct tftp_t *recv_tp)
 {
     struct sockaddr_in saddr, daddr;
     struct mbuf *m;
     struct tftp_t *tp;
-    int n = 0;
+    int i, n = 0;
 
     m = m_get(spt->slirp);
 
@@ -136,10 +140,12 @@ static int tftp_send_oack(struct tftp_session *spt,
     m->m_data += sizeof(struct udpiphdr);
 
     tp->tp_op = htons(TFTP_OACK);
-    n += snprintf(tp->x.tp_buf + n, sizeof(tp->x.tp_buf) - n, "%s",
-                  key) + 1;
-    n += snprintf(tp->x.tp_buf + n, sizeof(tp->x.tp_buf) - n, "%u",
-                  value) + 1;
+    for (i = 0; i < nb; i++) {
+        n += snprintf(tp->x.tp_buf + n, sizeof(tp->x.tp_buf) - n, "%s",
+                      keys[i]) + 1;
+        n += snprintf(tp->x.tp_buf + n, sizeof(tp->x.tp_buf) - n, "%u",
+                      values[i]) + 1;
+    }
 
     saddr.sin_addr = recv_tp->ip.ip_dst;
     saddr.sin_port = recv_tp->udp.uh_dport;
@@ -193,23 +199,18 @@ out:
   tftp_session_terminate(spt);
 }
 
-static int tftp_send_data(struct tftp_session *spt,
-                          uint16_t block_nr,
-			  struct tftp_t *recv_tp)
+static void tftp_send_next_block(struct tftp_session *spt,
+                                 struct tftp_t *recv_tp)
 {
   struct sockaddr_in saddr, daddr;
   struct mbuf *m;
   struct tftp_t *tp;
   int nobytes;
 
-  if (block_nr < 1) {
-    return -1;
-  }
-
   m = m_get(spt->slirp);
 
   if (!m) {
-    return -1;
+    return;
   }
 
   memset(m->m_data, 0, m->m_size);
@@ -219,7 +220,7 @@ static int tftp_send_data(struct tftp_session *spt,
   m->m_data += sizeof(struct udpiphdr);
 
   tp->tp_op = htons(TFTP_DATA);
-  tp->x.tp_data.tp_block_nr = htons(block_nr);
+  tp->x.tp_data.tp_block_nr = htons((spt->block_nr + 1) & 0xffff);
 
   saddr.sin_addr = recv_tp->ip.ip_dst;
   saddr.sin_port = recv_tp->udp.uh_dport;
@@ -227,7 +228,7 @@ static int tftp_send_data(struct tftp_session *spt,
   daddr.sin_addr = spt->client_ip;
   daddr.sin_port = spt->client_port;
 
-  nobytes = tftp_read_data(spt, block_nr - 1, tp->x.tp_data.tp_buf, 512);
+  nobytes = tftp_read_data(spt, spt->block_nr, tp->x.tp_data.tp_buf, 512);
 
   if (nobytes < 0) {
     m_free(m);
@@ -236,7 +237,7 @@ static int tftp_send_data(struct tftp_session *spt,
 
     tftp_send_error(spt, 1, "File not found", tp);
 
-    return -1;
+    return;
   }
 
   m->m_len = sizeof(struct tftp_t) - (512 - nobytes) -
@@ -251,7 +252,7 @@ static int tftp_send_data(struct tftp_session *spt,
     tftp_session_terminate(spt);
   }
 
-  return 0;
+  spt->block_nr++;
 }
 
 static void tftp_handle_rrq(Slirp *slirp, struct tftp_t *tp, int pktlen)
@@ -260,6 +261,9 @@ static void tftp_handle_rrq(Slirp *slirp, struct tftp_t *tp, int pktlen)
   int s, k;
   size_t prefix_len;
   char *req_fname;
+  const char *option_name[2];
+  uint32_t option_value[2];
+  int nb_options = 0;
 
   /* check if a session already exists and if so terminate it */
   s = tftp_session_find(slirp, tp);
@@ -337,7 +341,7 @@ static void tftp_handle_rrq(Slirp *slirp, struct tftp_t *tp, int pktlen)
       return;
   }
 
-  while (k < pktlen) {
+  while (k < pktlen && nb_options < ARRAY_SIZE(option_name)) {
       const char *key, *value;
 
       key = &tp->x.tp_buf[k];
@@ -364,12 +368,32 @@ static void tftp_handle_rrq(Slirp *slirp, struct tftp_t *tp, int pktlen)
 	      }
 	  }
 
-	  tftp_send_oack(spt, "tsize", tsize, tp);
-	  return;
+          option_name[nb_options] = "tsize";
+          option_value[nb_options] = tsize;
+          nb_options++;
+      } else if (strcasecmp(key, "blksize") == 0) {
+          int blksize = atoi(value);
+
+          /* If blksize option is bigger than what we will
+           * emit, accept the option with our packet size.
+           * Otherwise, simply do as we didn't see the option.
+           */
+          if (blksize >= 512) {
+              option_name[nb_options] = "blksize";
+              option_value[nb_options] = 512;
+              nb_options++;
+          }
       }
   }
 
-  tftp_send_data(spt, 1, tp);
+  if (nb_options > 0) {
+      assert(nb_options <= ARRAY_SIZE(option_name));
+      tftp_send_oack(spt, option_name, option_value, nb_options, tp);
+      return;
+  }
+
+  spt->block_nr = 0;
+  tftp_send_next_block(spt, tp);
 }
 
 static void tftp_handle_ack(Slirp *slirp, struct tftp_t *tp, int pktlen)
@@ -382,11 +406,7 @@ static void tftp_handle_ack(Slirp *slirp, struct tftp_t *tp, int pktlen)
     return;
   }
 
-  if (tftp_send_data(&slirp->tftp_sessions[s],
-		     ntohs(tp->x.tp_data.tp_block_nr) + 1,
-		     tp) < 0) {
-    return;
-  }
+  tftp_send_next_block(&slirp->tftp_sessions[s], tp);
 }
 
 static void tftp_handle_error(Slirp *slirp, struct tftp_t *tp, int pktlen)
diff --git a/slirp/tftp.h b/slirp/tftp.h
index 72e5e91be..51704e487 100644
--- a/slirp/tftp.h
+++ b/slirp/tftp.h
@@ -33,9 +33,11 @@ struct tftp_t {
 struct tftp_session {
     Slirp *slirp;
     char *filename;
+    int fd;
 
     struct in_addr client_ip;
     uint16_t client_port;
+    uint32_t block_nr;
 
     int timestamp;
 };
diff --git a/slirp/udp.c b/slirp/udp.c
index ced509656..9286cb7d3 100644
--- a/slirp/udp.c
+++ b/slirp/udp.c
@@ -231,7 +231,6 @@ udp_input(register struct mbuf *m, int iphlen)
 	return;
 bad:
 	m_free(m);
-	return;
 }
 
 int udp_output2(struct socket *so, struct mbuf *m,
diff --git a/softmmu_defs.h b/softmmu_defs.h
index 8d59f9d4f..1f25e33ce 100644
--- a/softmmu_defs.h
+++ b/softmmu_defs.h
@@ -9,25 +9,6 @@
 #ifndef SOFTMMU_DEFS_H
 #define SOFTMMU_DEFS_H
 
-#ifndef CONFIG_TCG_PASS_AREG0
-uint8_t __ldb_mmu(target_ulong addr, int mmu_idx);
-void __stb_mmu(target_ulong addr, uint8_t val, int mmu_idx);
-uint16_t __ldw_mmu(target_ulong addr, int mmu_idx);
-void __stw_mmu(target_ulong addr, uint16_t val, int mmu_idx);
-uint32_t __ldl_mmu(target_ulong addr, int mmu_idx);
-void __stl_mmu(target_ulong addr, uint32_t val, int mmu_idx);
-uint64_t __ldq_mmu(target_ulong addr, int mmu_idx);
-void __stq_mmu(target_ulong addr, uint64_t val, int mmu_idx);
-
-uint8_t __ldb_cmmu(target_ulong addr, int mmu_idx);
-void __stb_cmmu(target_ulong addr, uint8_t val, int mmu_idx);
-uint16_t __ldw_cmmu(target_ulong addr, int mmu_idx);
-void __stw_cmmu(target_ulong addr, uint16_t val, int mmu_idx);
-uint32_t __ldl_cmmu(target_ulong addr, int mmu_idx);
-void __stl_cmmu(target_ulong addr, uint32_t val, int mmu_idx);
-uint64_t __ldq_cmmu(target_ulong addr, int mmu_idx);
-void __stq_cmmu(target_ulong addr, uint64_t val, int mmu_idx);
-#else
 uint8_t helper_ldb_mmu(CPUArchState *env, target_ulong addr, int mmu_idx);
 void helper_stb_mmu(CPUArchState *env, target_ulong addr, uint8_t val,
                     int mmu_idx);
@@ -54,5 +35,3 @@ uint64_t helper_ldq_cmmu(CPUArchState *env, target_ulong addr, int mmu_idx);
 void helper_stq_cmmu(CPUArchState *env, target_ulong addr, uint64_t val,
                      int mmu_idx);
 #endif
-
-#endif
diff --git a/softmmu_header.h b/softmmu_header.h
index cf1aa38fc..d8d9c81b0 100644
--- a/softmmu_header.h
+++ b/softmmu_header.h
@@ -78,23 +78,10 @@
 #define ADDR_READ addr_read
 #endif
 
-#ifndef CONFIG_TCG_PASS_AREG0
-#define ENV_PARAM
-#define ENV_VAR
-#define CPU_PREFIX
-#define HELPER_PREFIX __
-#else
-#define ENV_PARAM CPUArchState *env,
-#define ENV_VAR env,
-#define CPU_PREFIX cpu_
-#define HELPER_PREFIX helper_
-#endif
-
 /* generic load/store macros */
 
 static inline RES_TYPE
-glue(glue(glue(CPU_PREFIX, ld), USUFFIX), MEMSUFFIX)(ENV_PARAM
-                                                     target_ulong ptr)
+glue(glue(cpu_ld, USUFFIX), MEMSUFFIX)(CPUArchState *env, target_ulong ptr)
 {
     int page_index;
     RES_TYPE res;
@@ -106,9 +93,7 @@ glue(glue(glue(CPU_PREFIX, ld), USUFFIX), MEMSUFFIX)(ENV_PARAM
     mmu_idx = CPU_MMU_INDEX;
     if (unlikely(env->tlb_table[mmu_idx][page_index].ADDR_READ !=
                  (addr & (TARGET_PAGE_MASK | (DATA_SIZE - 1))))) {
-        res = glue(glue(glue(HELPER_PREFIX, ld), SUFFIX), MMUSUFFIX)(ENV_VAR
-                                                                     addr,
-                                                                     mmu_idx);
+        res = glue(glue(helper_ld, SUFFIX), MMUSUFFIX)(env, addr, mmu_idx);
     } else {
         uintptr_t hostaddr = addr + env->tlb_table[mmu_idx][page_index].addend;
         res = glue(glue(ld, USUFFIX), _raw)(hostaddr);
@@ -118,8 +103,7 @@ glue(glue(glue(CPU_PREFIX, ld), USUFFIX), MEMSUFFIX)(ENV_PARAM
 
 #if DATA_SIZE <= 2
 static inline int
-glue(glue(glue(CPU_PREFIX, lds), SUFFIX), MEMSUFFIX)(ENV_PARAM
-                                                     target_ulong ptr)
+glue(glue(cpu_lds, SUFFIX), MEMSUFFIX)(CPUArchState *env, target_ulong ptr)
 {
     int res, page_index;
     target_ulong addr;
@@ -130,8 +114,8 @@ glue(glue(glue(CPU_PREFIX, lds), SUFFIX), MEMSUFFIX)(ENV_PARAM
     mmu_idx = CPU_MMU_INDEX;
     if (unlikely(env->tlb_table[mmu_idx][page_index].ADDR_READ !=
                  (addr & (TARGET_PAGE_MASK | (DATA_SIZE - 1))))) {
-        res = (DATA_STYPE)glue(glue(glue(HELPER_PREFIX, ld), SUFFIX),
-                               MMUSUFFIX)(ENV_VAR addr, mmu_idx);
+        res = (DATA_STYPE)glue(glue(helper_ld, SUFFIX),
+                               MMUSUFFIX)(env, addr, mmu_idx);
     } else {
         uintptr_t hostaddr = addr + env->tlb_table[mmu_idx][page_index].addend;
         res = glue(glue(lds, SUFFIX), _raw)(hostaddr);
@@ -145,8 +129,8 @@ glue(glue(glue(CPU_PREFIX, lds), SUFFIX), MEMSUFFIX)(ENV_PARAM
 /* generic store macro */
 
 static inline void
-glue(glue(glue(CPU_PREFIX, st), SUFFIX), MEMSUFFIX)(ENV_PARAM target_ulong ptr,
-                                                    RES_TYPE v)
+glue(glue(cpu_st, SUFFIX), MEMSUFFIX)(CPUArchState *env, target_ulong ptr,
+                                      RES_TYPE v)
 {
     int page_index;
     target_ulong addr;
@@ -157,8 +141,7 @@ glue(glue(glue(CPU_PREFIX, st), SUFFIX), MEMSUFFIX)(ENV_PARAM target_ulong ptr,
     mmu_idx = CPU_MMU_INDEX;
     if (unlikely(env->tlb_table[mmu_idx][page_index].addr_write !=
                  (addr & (TARGET_PAGE_MASK | (DATA_SIZE - 1))))) {
-        glue(glue(glue(HELPER_PREFIX, st), SUFFIX), MMUSUFFIX)(ENV_VAR addr, v,
-                                                               mmu_idx);
+        glue(glue(helper_st, SUFFIX), MMUSUFFIX)(env, addr, v, mmu_idx);
     } else {
         uintptr_t hostaddr = addr + env->tlb_table[mmu_idx][page_index].addend;
         glue(glue(st, SUFFIX), _raw)(hostaddr, v);
@@ -170,52 +153,50 @@ glue(glue(glue(CPU_PREFIX, st), SUFFIX), MEMSUFFIX)(ENV_PARAM target_ulong ptr,
 #if ACCESS_TYPE != (NB_MMU_MODES + 1)
 
 #if DATA_SIZE == 8
-static inline float64 glue(glue(CPU_PREFIX, ldfq), MEMSUFFIX)(ENV_PARAM
-                                                              target_ulong ptr)
+static inline float64 glue(cpu_ldfq, MEMSUFFIX)(CPUArchState *env,
+                                                target_ulong ptr)
 {
     union {
         float64 d;
         uint64_t i;
     } u;
-    u.i = glue(glue(CPU_PREFIX, ldq), MEMSUFFIX)(ENV_VAR ptr);
+    u.i = glue(cpu_ldq, MEMSUFFIX)(env, ptr);
     return u.d;
 }
 
-static inline void glue(glue(CPU_PREFIX, stfq), MEMSUFFIX)(ENV_PARAM
-                                                           target_ulong ptr,
-                                                           float64 v)
+static inline void glue(cpu_stfq, MEMSUFFIX)(CPUArchState *env,
+                                             target_ulong ptr, float64 v)
 {
     union {
         float64 d;
         uint64_t i;
     } u;
     u.d = v;
-    glue(glue(CPU_PREFIX, stq), MEMSUFFIX)(ENV_VAR ptr, u.i);
+    glue(cpu_stq, MEMSUFFIX)(env, ptr, u.i);
 }
 #endif /* DATA_SIZE == 8 */
 
 #if DATA_SIZE == 4
-static inline float32 glue(glue(CPU_PREFIX, ldfl), MEMSUFFIX)(ENV_PARAM
-                                                              target_ulong ptr)
+static inline float32 glue(cpu_ldfl, MEMSUFFIX)(CPUArchState *env,
+                                                target_ulong ptr)
 {
     union {
         float32 f;
         uint32_t i;
     } u;
-    u.i = glue(glue(CPU_PREFIX, ldl), MEMSUFFIX)(ENV_VAR ptr);
+    u.i = glue(cpu_ldl, MEMSUFFIX)(env, ptr);
     return u.f;
 }
 
-static inline void glue(glue(CPU_PREFIX, stfl), MEMSUFFIX)(ENV_PARAM
-                                                           target_ulong ptr,
-                                                           float32 v)
+static inline void glue(cpu_stfl, MEMSUFFIX)(CPUArchState *env,
+                                             target_ulong ptr, float32 v)
 {
     union {
         float32 f;
         uint32_t i;
     } u;
     u.f = v;
-    glue(glue(CPU_PREFIX, stl), MEMSUFFIX)(ENV_VAR ptr, u.i);
+    glue(cpu_stl, MEMSUFFIX)(env, ptr, u.i);
 }
 #endif /* DATA_SIZE == 4 */
 
@@ -230,7 +211,3 @@ static inline void glue(glue(CPU_PREFIX, stfl), MEMSUFFIX)(ENV_PARAM
 #undef CPU_MMU_INDEX
 #undef MMUSUFFIX
 #undef ADDR_READ
-#undef ENV_PARAM
-#undef ENV_VAR
-#undef CPU_PREFIX
-#undef HELPER_PREFIX
diff --git a/softmmu_template.h b/softmmu_template.h
index b8bd70032..ce30d8bdd 100644
--- a/softmmu_template.h
+++ b/softmmu_template.h
@@ -54,24 +54,12 @@
 #define ADDR_READ addr_read
 #endif
 
-#ifndef CONFIG_TCG_PASS_AREG0
-#define ENV_PARAM
-#define ENV_VAR
-#define CPU_PREFIX
-#define HELPER_PREFIX __
-#else
-#define ENV_PARAM CPUArchState *env,
-#define ENV_VAR env,
-#define CPU_PREFIX cpu_
-#define HELPER_PREFIX helper_
-#endif
-
-static DATA_TYPE glue(glue(slow_ld, SUFFIX), MMUSUFFIX)(ENV_PARAM
+static DATA_TYPE glue(glue(slow_ld, SUFFIX), MMUSUFFIX)(CPUArchState *env,
                                                         target_ulong addr,
                                                         int mmu_idx,
                                                         uintptr_t retaddr);
-static inline DATA_TYPE glue(io_read, SUFFIX)(ENV_PARAM
-                                              target_phys_addr_t physaddr,
+static inline DATA_TYPE glue(io_read, SUFFIX)(CPUArchState *env,
+                                              hwaddr physaddr,
                                               target_ulong addr,
                                               uintptr_t retaddr)
 {
@@ -104,14 +92,13 @@ static inline DATA_TYPE glue(io_read, SUFFIX)(ENV_PARAM
 
 /* handle all cases except unaligned access which span two pages */
 DATA_TYPE
-glue(glue(glue(HELPER_PREFIX, ld), SUFFIX), MMUSUFFIX)(ENV_PARAM
-                                                       target_ulong addr,
-                                                       int mmu_idx)
+glue(glue(helper_ld, SUFFIX), MMUSUFFIX)(CPUArchState *env, target_ulong addr,
+                                         int mmu_idx)
 {
     DATA_TYPE res;
     int index;
     target_ulong tlb_addr;
-    target_phys_addr_t ioaddr;
+    hwaddr ioaddr;
     uintptr_t retaddr;
 
     /* test if there is match for unaligned or IO access */
@@ -124,25 +111,25 @@ glue(glue(glue(HELPER_PREFIX, ld), SUFFIX), MMUSUFFIX)(ENV_PARAM
             /* IO access */
             if ((addr & (DATA_SIZE - 1)) != 0)
                 goto do_unaligned_access;
-            retaddr = GETPC();
+            retaddr = GETPC_EXT();
             ioaddr = env->iotlb[mmu_idx][index];
-            res = glue(io_read, SUFFIX)(ENV_VAR ioaddr, addr, retaddr);
+            res = glue(io_read, SUFFIX)(env, ioaddr, addr, retaddr);
         } else if (((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1) >= TARGET_PAGE_SIZE) {
             /* slow unaligned access (it spans two pages or IO) */
         do_unaligned_access:
-            retaddr = GETPC();
+            retaddr = GETPC_EXT();
 #ifdef ALIGNED_ONLY
-            do_unaligned_access(ENV_VAR addr, READ_ACCESS_TYPE, mmu_idx, retaddr);
+            do_unaligned_access(env, addr, READ_ACCESS_TYPE, mmu_idx, retaddr);
 #endif
-            res = glue(glue(slow_ld, SUFFIX), MMUSUFFIX)(ENV_VAR addr,
+            res = glue(glue(slow_ld, SUFFIX), MMUSUFFIX)(env, addr,
                                                          mmu_idx, retaddr);
         } else {
             /* unaligned/aligned access in the same page */
             uintptr_t addend;
 #ifdef ALIGNED_ONLY
             if ((addr & (DATA_SIZE - 1)) != 0) {
-                retaddr = GETPC();
-                do_unaligned_access(ENV_VAR addr, READ_ACCESS_TYPE, mmu_idx, retaddr);
+                retaddr = GETPC_EXT();
+                do_unaligned_access(env, addr, READ_ACCESS_TYPE, mmu_idx, retaddr);
             }
 #endif
             addend = env->tlb_table[mmu_idx][index].addend;
@@ -151,10 +138,10 @@ glue(glue(glue(HELPER_PREFIX, ld), SUFFIX), MMUSUFFIX)(ENV_PARAM
         }
     } else {
         /* the page is not in the TLB : fill it */
-        retaddr = GETPC();
+        retaddr = GETPC_EXT();
 #ifdef ALIGNED_ONLY
         if ((addr & (DATA_SIZE - 1)) != 0)
-            do_unaligned_access(ENV_VAR addr, READ_ACCESS_TYPE, mmu_idx, retaddr);
+            do_unaligned_access(env, addr, READ_ACCESS_TYPE, mmu_idx, retaddr);
 #endif
         tlb_fill(env, addr, READ_ACCESS_TYPE, mmu_idx, retaddr);
         goto redo;
@@ -164,14 +151,14 @@ glue(glue(glue(HELPER_PREFIX, ld), SUFFIX), MMUSUFFIX)(ENV_PARAM
 
 /* handle all unaligned cases */
 static DATA_TYPE
-glue(glue(slow_ld, SUFFIX), MMUSUFFIX)(ENV_PARAM
+glue(glue(slow_ld, SUFFIX), MMUSUFFIX)(CPUArchState *env,
                                        target_ulong addr,
                                        int mmu_idx,
                                        uintptr_t retaddr)
 {
     DATA_TYPE res, res1, res2;
     int index, shift;
-    target_phys_addr_t ioaddr;
+    hwaddr ioaddr;
     target_ulong tlb_addr, addr1, addr2;
 
     index = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
@@ -183,15 +170,15 @@ glue(glue(slow_ld, SUFFIX), MMUSUFFIX)(ENV_PARAM
             if ((addr & (DATA_SIZE - 1)) != 0)
                 goto do_unaligned_access;
             ioaddr = env->iotlb[mmu_idx][index];
-            res = glue(io_read, SUFFIX)(ENV_VAR ioaddr, addr, retaddr);
+            res = glue(io_read, SUFFIX)(env, ioaddr, addr, retaddr);
         } else if (((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1) >= TARGET_PAGE_SIZE) {
         do_unaligned_access:
             /* slow unaligned access (it spans two pages) */
             addr1 = addr & ~(DATA_SIZE - 1);
             addr2 = addr1 + DATA_SIZE;
-            res1 = glue(glue(slow_ld, SUFFIX), MMUSUFFIX)(ENV_VAR addr1,
+            res1 = glue(glue(slow_ld, SUFFIX), MMUSUFFIX)(env, addr1,
                                                           mmu_idx, retaddr);
-            res2 = glue(glue(slow_ld, SUFFIX), MMUSUFFIX)(ENV_VAR addr2,
+            res2 = glue(glue(slow_ld, SUFFIX), MMUSUFFIX)(env, addr2,
                                                           mmu_idx, retaddr);
             shift = (addr & (DATA_SIZE - 1)) * 8;
 #ifdef TARGET_WORDS_BIGENDIAN
@@ -216,14 +203,14 @@ glue(glue(slow_ld, SUFFIX), MMUSUFFIX)(ENV_PARAM
 
 #ifndef SOFTMMU_CODE_ACCESS
 
-static void glue(glue(slow_st, SUFFIX), MMUSUFFIX)(ENV_PARAM
+static void glue(glue(slow_st, SUFFIX), MMUSUFFIX)(CPUArchState *env,
                                                    target_ulong addr,
                                                    DATA_TYPE val,
                                                    int mmu_idx,
                                                    uintptr_t retaddr);
 
-static inline void glue(io_write, SUFFIX)(ENV_PARAM
-                                          target_phys_addr_t physaddr,
+static inline void glue(io_write, SUFFIX)(CPUArchState *env,
+                                          hwaddr physaddr,
                                           DATA_TYPE val,
                                           target_ulong addr,
                                           uintptr_t retaddr)
@@ -253,12 +240,11 @@ static inline void glue(io_write, SUFFIX)(ENV_PARAM
 #endif /* SHIFT > 2 */
 }
 
-void glue(glue(glue(HELPER_PREFIX, st), SUFFIX), MMUSUFFIX)(ENV_PARAM
-                                                            target_ulong addr,
-                                                            DATA_TYPE val,
-                                                            int mmu_idx)
+void glue(glue(helper_st, SUFFIX), MMUSUFFIX)(CPUArchState *env,
+                                              target_ulong addr, DATA_TYPE val,
+                                              int mmu_idx)
 {
-    target_phys_addr_t ioaddr;
+    hwaddr ioaddr;
     target_ulong tlb_addr;
     uintptr_t retaddr;
     int index;
@@ -271,24 +257,24 @@ void glue(glue(glue(HELPER_PREFIX, st), SUFFIX), MMUSUFFIX)(ENV_PARAM
             /* IO access */
             if ((addr & (DATA_SIZE - 1)) != 0)
                 goto do_unaligned_access;
-            retaddr = GETPC();
+            retaddr = GETPC_EXT();
             ioaddr = env->iotlb[mmu_idx][index];
-            glue(io_write, SUFFIX)(ENV_VAR ioaddr, val, addr, retaddr);
+            glue(io_write, SUFFIX)(env, ioaddr, val, addr, retaddr);
         } else if (((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1) >= TARGET_PAGE_SIZE) {
         do_unaligned_access:
-            retaddr = GETPC();
+            retaddr = GETPC_EXT();
 #ifdef ALIGNED_ONLY
-            do_unaligned_access(ENV_VAR addr, 1, mmu_idx, retaddr);
+            do_unaligned_access(env, addr, 1, mmu_idx, retaddr);
 #endif
-            glue(glue(slow_st, SUFFIX), MMUSUFFIX)(ENV_VAR addr, val,
+            glue(glue(slow_st, SUFFIX), MMUSUFFIX)(env, addr, val,
                                                    mmu_idx, retaddr);
         } else {
             /* aligned/unaligned access in the same page */
             uintptr_t addend;
 #ifdef ALIGNED_ONLY
             if ((addr & (DATA_SIZE - 1)) != 0) {
-                retaddr = GETPC();
-                do_unaligned_access(ENV_VAR addr, 1, mmu_idx, retaddr);
+                retaddr = GETPC_EXT();
+                do_unaligned_access(env, addr, 1, mmu_idx, retaddr);
             }
 #endif
             addend = env->tlb_table[mmu_idx][index].addend;
@@ -297,10 +283,10 @@ void glue(glue(glue(HELPER_PREFIX, st), SUFFIX), MMUSUFFIX)(ENV_PARAM
         }
     } else {
         /* the page is not in the TLB : fill it */
-        retaddr = GETPC();
+        retaddr = GETPC_EXT();
 #ifdef ALIGNED_ONLY
         if ((addr & (DATA_SIZE - 1)) != 0)
-            do_unaligned_access(ENV_VAR addr, 1, mmu_idx, retaddr);
+            do_unaligned_access(env, addr, 1, mmu_idx, retaddr);
 #endif
         tlb_fill(env, addr, 1, mmu_idx, retaddr);
         goto redo;
@@ -308,13 +294,13 @@ void glue(glue(glue(HELPER_PREFIX, st), SUFFIX), MMUSUFFIX)(ENV_PARAM
 }
 
 /* handles all unaligned cases */
-static void glue(glue(slow_st, SUFFIX), MMUSUFFIX)(ENV_PARAM
+static void glue(glue(slow_st, SUFFIX), MMUSUFFIX)(CPUArchState *env,
                                                    target_ulong addr,
                                                    DATA_TYPE val,
                                                    int mmu_idx,
                                                    uintptr_t retaddr)
 {
-    target_phys_addr_t ioaddr;
+    hwaddr ioaddr;
     target_ulong tlb_addr;
     int index, i;
 
@@ -327,7 +313,7 @@ static void glue(glue(slow_st, SUFFIX), MMUSUFFIX)(ENV_PARAM
             if ((addr & (DATA_SIZE - 1)) != 0)
                 goto do_unaligned_access;
             ioaddr = env->iotlb[mmu_idx][index];
-            glue(io_write, SUFFIX)(ENV_VAR ioaddr, val, addr, retaddr);
+            glue(io_write, SUFFIX)(env, ioaddr, val, addr, retaddr);
         } else if (((addr & ~TARGET_PAGE_MASK) + DATA_SIZE - 1) >= TARGET_PAGE_SIZE) {
         do_unaligned_access:
             /* XXX: not efficient, but simple */
@@ -335,11 +321,11 @@ static void glue(glue(slow_st, SUFFIX), MMUSUFFIX)(ENV_PARAM
              * previous page from the TLB cache.  */
             for(i = DATA_SIZE - 1; i >= 0; i--) {
 #ifdef TARGET_WORDS_BIGENDIAN
-                glue(slow_stb, MMUSUFFIX)(ENV_VAR addr + i,
+                glue(slow_stb, MMUSUFFIX)(env, addr + i,
                                           val >> (((DATA_SIZE - 1) * 8) - (i * 8)),
                                           mmu_idx, retaddr);
 #else
-                glue(slow_stb, MMUSUFFIX)(ENV_VAR addr + i,
+                glue(slow_stb, MMUSUFFIX)(env, addr + i,
                                           val >> (i * 8),
                                           mmu_idx, retaddr);
 #endif
@@ -366,7 +352,3 @@ static void glue(glue(slow_st, SUFFIX), MMUSUFFIX)(ENV_PARAM
 #undef USUFFIX
 #undef DATA_SIZE
 #undef ADDR_READ
-#undef ENV_PARAM
-#undef ENV_VAR
-#undef CPU_PREFIX
-#undef HELPER_PREFIX
diff --git a/sparc-dis.c b/sparc-dis.c
index cdd337a73..1d017faaa 100644
--- a/sparc-dis.c
+++ b/sparc-dis.c
@@ -3270,6 +3270,6 @@ print_insn_sparc (bfd_vma memaddr, disassemble_info *info)
     }
 
   info->insn_type = dis_noninsn;        /* Mark as non-valid instruction.  */
-  (*info->fprintf_func) (stream, _("unknown"));
+  (*info->fprintf_func) (stream, ".long %#08lx", insn);
   return sizeof (buffer);
 }
diff --git a/stubs/Makefile.objs b/stubs/Makefile.objs
new file mode 100644
index 000000000..035b29a1f
--- /dev/null
+++ b/stubs/Makefile.objs
@@ -0,0 +1,8 @@
+stub-obj-y += arch-query-cpu-def.o
+stub-obj-y += fdset-add-fd.o
+stub-obj-y += fdset-find-fd.o
+stub-obj-y += fdset-get-fd.o
+stub-obj-y += fdset-remove-fd.o
+stub-obj-y += get-fd.o
+stub-obj-y += set-fd-handler.o
+stub-obj-$(CONFIG_WIN32) += fd-register.o
diff --git a/stubs/arch-query-cpu-def.c b/stubs/arch-query-cpu-def.c
new file mode 100644
index 000000000..47b524628
--- /dev/null
+++ b/stubs/arch-query-cpu-def.c
@@ -0,0 +1,9 @@
+#include "qemu-common.h"
+#include "arch_init.h"
+#include "qerror.h"
+
+CpuDefinitionInfoList *arch_query_cpu_definitions(Error **errp)
+{
+    error_set(errp, QERR_NOT_SUPPORTED);
+    return NULL;
+}
diff --git a/stubs/fd-register.c b/stubs/fd-register.c
new file mode 100644
index 000000000..813b6dd7c
--- /dev/null
+++ b/stubs/fd-register.c
@@ -0,0 +1,6 @@
+#include "qemu-common.h"
+#include "main-loop.h"
+
+void qemu_fd_register(int fd)
+{
+}
diff --git a/stubs/fdset-add-fd.c b/stubs/fdset-add-fd.c
new file mode 100644
index 000000000..09fe2a839
--- /dev/null
+++ b/stubs/fdset-add-fd.c
@@ -0,0 +1,7 @@
+#include "qemu-common.h"
+#include "monitor.h"
+
+int monitor_fdset_dup_fd_add(int64_t fdset_id, int dup_fd)
+{
+    return -1;
+}
diff --git a/stubs/fdset-find-fd.c b/stubs/fdset-find-fd.c
new file mode 100644
index 000000000..f82baa066
--- /dev/null
+++ b/stubs/fdset-find-fd.c
@@ -0,0 +1,7 @@
+#include "qemu-common.h"
+#include "monitor.h"
+
+int monitor_fdset_dup_fd_find(int dup_fd)
+{
+    return -1;
+}
diff --git a/stubs/fdset-get-fd.c b/stubs/fdset-get-fd.c
new file mode 100644
index 000000000..4106cf90f
--- /dev/null
+++ b/stubs/fdset-get-fd.c
@@ -0,0 +1,7 @@
+#include "qemu-common.h"
+#include "monitor.h"
+
+int monitor_fdset_get_fd(int64_t fdset_id, int flags)
+{
+    return -1;
+}
diff --git a/stubs/fdset-remove-fd.c b/stubs/fdset-remove-fd.c
new file mode 100644
index 000000000..861b31247
--- /dev/null
+++ b/stubs/fdset-remove-fd.c
@@ -0,0 +1,7 @@
+#include "qemu-common.h"
+#include "monitor.h"
+
+int monitor_fdset_dup_fd_remove(int dupfd)
+{
+    return -1;
+}
diff --git a/stubs/get-fd.c b/stubs/get-fd.c
new file mode 100644
index 000000000..3561ab60e
--- /dev/null
+++ b/stubs/get-fd.c
@@ -0,0 +1,8 @@
+#include "qemu-common.h"
+#include "monitor.h"
+
+int monitor_get_fd(Monitor *mon, const char *name, Error **errp)
+{
+    error_setg(errp, "only QEMU supports file descriptor passing");
+    return -1;
+}
diff --git a/stubs/set-fd-handler.c b/stubs/set-fd-handler.c
new file mode 100644
index 000000000..4807b5dc2
--- /dev/null
+++ b/stubs/set-fd-handler.c
@@ -0,0 +1,11 @@
+#include "qemu-common.h"
+#include "main-loop.h"
+
+int qemu_set_fd_handler2(int fd,
+                         IOCanReadHandler *fd_read_poll,
+                         IOHandler *fd_read,
+                         IOHandler *fd_write,
+                         void *opaque)
+{
+    abort();
+}
diff --git a/sysconfigs/target/cpus-x86_64.conf b/sysconfigs/target/cpus-x86_64.conf
deleted file mode 100644
index cee0ea9e5..000000000
--- a/sysconfigs/target/cpus-x86_64.conf
+++ /dev/null
@@ -1,128 +0,0 @@
-# x86 CPU MODELS
-
-[cpudef]
-   name = "Conroe"
-   level = "2"
-   vendor = "GenuineIntel"
-   family = "6"
-   model = "2"
-   stepping = "3"
-   feature_edx = "sse2 sse fxsr mmx clflush pse36 pat cmov mca pge mtrr sep apic cx8 mce pae msr tsc pse de fpu"
-   feature_ecx = "ssse3 sse3"
-   extfeature_edx = "i64 xd syscall"
-   extfeature_ecx = "lahf_lm"
-   xlevel = "0x8000000A"
-   model_id = "Intel Celeron_4x0 (Conroe/Merom Class Core 2)"
-
-[cpudef]
-   name = "Penryn"
-   level = "2"
-   vendor = "GenuineIntel"
-   family = "6"
-   model = "2"
-   stepping = "3"
-   feature_edx = "sse2 sse fxsr mmx clflush pse36 pat cmov mca pge mtrr sep apic cx8 mce pae msr tsc pse de fpu"
-   feature_ecx = "sse4.1 cx16 ssse3 sse3"
-   extfeature_edx = "i64 xd syscall"
-   extfeature_ecx = "lahf_lm"
-   xlevel = "0x8000000A"
-   model_id = "Intel Core 2 Duo P9xxx (Penryn Class Core 2)"
-
-[cpudef]
-   name = "Nehalem"
-   level = "2"
-   vendor = "GenuineIntel"
-   family = "6"
-   model = "2"
-   stepping = "3"
-   feature_edx = "sse2 sse fxsr mmx clflush pse36 pat cmov mca pge mtrr sep apic cx8 mce pae msr tsc pse de fpu"
-   feature_ecx = "popcnt sse4.2 sse4.1 cx16 ssse3 sse3"
-   extfeature_edx = "i64 syscall xd"
-   extfeature_ecx = "lahf_lm"
-   xlevel = "0x8000000A"
-   model_id = "Intel Core i7 9xx (Nehalem Class Core i7)"
-
-[cpudef]
-   name = "Westmere"
-   level = "11"
-   vendor = "GenuineIntel"
-   family = "6"
-   model = "44"
-   stepping = "1"
-   feature_edx = "sse2 sse fxsr mmx clflush pse36 pat cmov mca pge mtrr sep apic cx8 mce pae msr tsc pse de fpu"
-   feature_ecx = "aes popcnt sse4.2 sse4.1 cx16 ssse3 sse3"
-   extfeature_edx = "i64 syscall xd"
-   extfeature_ecx = "lahf_lm"
-   xlevel = "0x8000000A"
-   model_id = "Westmere E56xx/L56xx/X56xx (Nehalem-C)"
-
-[cpudef]
-   name = "SandyBridge"
-   level = "0xd"
-   vendor = "GenuineIntel"
-   family = "6"
-   model = "42"
-   stepping = "1"
-   feature_edx = " sse2 sse fxsr mmx clflush pse36 pat cmov mca pge mtrr sep apic cx8 mce pae msr tsc pse de fpu"
-   feature_ecx = "avx xsave aes tsc-deadline popcnt x2apic sse4.2 sse4.1 cx16 ssse3 pclmulqdq sse3"
-   extfeature_edx = "i64 rdtscp nx syscall "
-   extfeature_ecx = "lahf_lm"
-   xlevel = "0x8000000A"
-   model_id = "Intel Xeon E312xx (Sandy Bridge)"
-
-[cpudef]
-   name = "Opteron_G1"
-   level = "5"
-   vendor = "AuthenticAMD"
-   family = "15"
-   model = "6"
-   stepping = "1"
-   feature_edx = "sse2 sse fxsr mmx clflush pse36 pat cmov mca pge mtrr sep apic cx8 mce pae msr tsc pse de fpu"
-   feature_ecx = "sse3"
-   extfeature_edx = "lm fxsr mmx nx pse36 pat cmov mca pge mtrr syscall apic cx8 mce pae msr tsc pse de fpu"
-   extfeature_ecx = " "
-   xlevel = "0x80000008"
-   model_id = "AMD Opteron 240 (Gen 1 Class Opteron)"
-
-[cpudef]
-   name = "Opteron_G2"
-   level = "5"
-   vendor = "AuthenticAMD"
-   family = "15"
-   model = "6"
-   stepping = "1"
-   feature_edx = "sse2 sse fxsr mmx clflush pse36 pat cmov mca pge mtrr sep apic cx8 mce pae msr tsc pse de fpu"
-   feature_ecx = "cx16 sse3"
-   extfeature_edx = "lm rdtscp fxsr mmx nx pse36 pat cmov mca pge mtrr syscall apic cx8 mce pae msr tsc pse de fpu"
-   extfeature_ecx = "svm lahf_lm"
-   xlevel = "0x80000008"
-   model_id = "AMD Opteron 22xx (Gen 2 Class Opteron)"
-
-[cpudef]
-   name = "Opteron_G3"
-   level = "5"
-   vendor = "AuthenticAMD"
-   family = "15"
-   model = "6"
-   stepping = "1"
-   feature_edx = "sse2 sse fxsr mmx clflush pse36 pat cmov mca pge mtrr sep apic cx8 mce pae msr tsc pse de fpu"
-   feature_ecx = "popcnt cx16 monitor sse3"
-   extfeature_edx = "lm rdtscp fxsr mmx nx pse36 pat cmov mca pge mtrr syscall apic cx8 mce pae msr tsc pse de fpu"
-   extfeature_ecx = "misalignsse sse4a abm svm lahf_lm"
-   xlevel = "0x80000008"
-   model_id = "AMD Opteron 23xx (Gen 3 Class Opteron)"
-
-[cpudef]
-   name = "Opteron_G4"
-   level = "0xd"
-   vendor = "AuthenticAMD"
-   family = "21"
-   model = "1"
-   stepping = "2"
-   feature_edx = "sse2 sse fxsr mmx clflush pse36 pat cmov mca pge mtrr sep apic cx8 mce pae msr tsc pse de fpu"
-   feature_ecx = "avx xsave aes popcnt sse4.2 sse4.1 cx16 ssse3 pclmulqdq sse3"
-   extfeature_edx = "lm rdtscp pdpe1gb fxsr mmx nx pse36 pat cmov mca pge mtrr syscall apic cx8 mce pae msr tsc pse de fpu"
-   extfeature_ecx = " fma4 xop 3dnowprefetch misalignsse sse4a abm svm lahf_lm"
-   xlevel = "0x8000001A"
-   model_id = "AMD Opteron 62xx class CPU"
-
diff --git a/sysemu.h b/sysemu.h
index 65552acee..f5ac66406 100644
--- a/sysemu.h
+++ b/sysemu.h
@@ -19,7 +19,6 @@ extern uint8_t qemu_uuid[];
 int qemu_uuid_parse(const char *str, uint8_t *uuid);
 #define UUID_FMT "%02hhx%02hhx%02hhx%02hhx-%02hhx%02hhx-%02hhx%02hhx-%02hhx%02hhx-%02hhx%02hhx%02hhx%02hhx%02hhx%02hhx"
 
-void runstate_init(void);
 bool runstate_check(RunState state);
 void runstate_set(RunState new_state);
 int runstate_is_running(void);
@@ -52,16 +51,12 @@ void qemu_system_wakeup_enable(WakeupReason reason, bool enabled);
 void qemu_register_wakeup_notifier(Notifier *notifier);
 void qemu_system_shutdown_request(void);
 void qemu_system_powerdown_request(void);
+void qemu_register_powerdown_notifier(Notifier *notifier);
 void qemu_system_debug_request(void);
 void qemu_system_vmstop_request(RunState reason);
 int qemu_shutdown_requested_get(void);
 int qemu_reset_requested_get(void);
-int qemu_shutdown_requested(void);
-int qemu_reset_requested(void);
-int qemu_powerdown_requested(void);
 void qemu_system_killed(int signal, pid_t pid);
-void qemu_kill_report(void);
-extern qemu_irq qemu_system_powerdown;
 void qemu_devices_reset(void);
 void qemu_system_reset(bool report);
 
@@ -105,10 +100,7 @@ typedef enum {
 } VGAInterfaceType;
 
 extern int vga_interface_type;
-#define cirrus_vga_enabled (vga_interface_type == VGA_CIRRUS)
-#define std_vga_enabled (vga_interface_type == VGA_STD)
 #define xenfb_enabled (vga_interface_type == VGA_XENFB)
-#define vmsvga_enabled (vga_interface_type == VGA_VMWARE)
 #define qxl_enabled (vga_interface_type == VGA_QXL)
 
 extern int graphic_width;
@@ -119,7 +111,6 @@ extern const char *keyboard_layout;
 extern int win2k_install_hack;
 extern int alt_grab;
 extern int ctrl_grab;
-extern int usb_enabled;
 extern int smp_cpus;
 extern int max_cpus;
 extern int cursor_hide;
@@ -189,4 +180,7 @@ void register_devices(void);
 void add_boot_device_path(int32_t bootindex, DeviceState *dev,
                           const char *suffix);
 char *get_boot_devices_list(uint32_t *size);
+
+bool usb_enabled(bool default_usb);
+
 #endif
diff --git a/target-alpha/cpu.c b/target-alpha/cpu.c
index 62d2a669a..11a19ebc8 100644
--- a/target-alpha/cpu.c
+++ b/target-alpha/cpu.c
@@ -19,7 +19,7 @@
  * <http://www.gnu.org/licenses/lgpl-2.1.html>
  */
 
-#include "cpu-qom.h"
+#include "cpu.h"
 #include "qemu-common.h"
 
 
diff --git a/target-alpha/cpu.h b/target-alpha/cpu.h
index 5689760ce..34221fb18 100644
--- a/target-alpha/cpu.h
+++ b/target-alpha/cpu.h
@@ -454,7 +454,7 @@ void cpu_alpha_store_fpcr (CPUAlphaState *env, uint64_t val);
 #ifndef CONFIG_USER_ONLY
 void swap_shadow_regs(CPUAlphaState *env);
 QEMU_NORETURN void cpu_unassigned_access(CPUAlphaState *env1,
-                                         target_phys_addr_t addr, int is_write,
+                                         hwaddr addr, int is_write,
                                          int is_exec, int unused, int size);
 #endif
 
@@ -510,8 +510,10 @@ static inline void cpu_set_tls(CPUAlphaState *env, target_ulong newtls)
 }
 #endif
 
-static inline bool cpu_has_work(CPUAlphaState *env)
+static inline bool cpu_has_work(CPUState *cpu)
 {
+    CPUAlphaState *env = &ALPHA_CPU(cpu)->env;
+
     /* Here we are checking to see if the CPU should wake up from HALT.
        We will have gotten into this state only for WTINT from PALmode.  */
     /* ??? I'm not sure how the IPL state works with WTINT to keep a CPU
diff --git a/target-alpha/helper.c b/target-alpha/helper.c
index 81d4763b4..d9d7f75b5 100644
--- a/target-alpha/helper.c
+++ b/target-alpha/helper.c
@@ -315,7 +315,7 @@ static int get_physical_address(CPUAlphaState *env, target_ulong addr,
     return ret;
 }
 
-target_phys_addr_t cpu_get_phys_page_debug(CPUAlphaState *env, target_ulong addr)
+hwaddr cpu_get_phys_page_debug(CPUAlphaState *env, target_ulong addr)
 {
     target_ulong phys;
     int prot, fail;
diff --git a/target-alpha/helper.h b/target-alpha/helper.h
index a184def3e..dd55f89aa 100644
--- a/target-alpha/helper.h
+++ b/target-alpha/helper.h
@@ -1,102 +1,102 @@
 #include "def-helper.h"
 
 DEF_HELPER_3(excp, noreturn, env, int, int)
-DEF_HELPER_FLAGS_1(load_pcc, TCG_CALL_CONST | TCG_CALL_PURE, i64, env)
-
-DEF_HELPER_3(addqv, i64, env, i64, i64)
-DEF_HELPER_3(addlv, i64, env, i64, i64)
-DEF_HELPER_3(subqv, i64, env, i64, i64)
-DEF_HELPER_3(sublv, i64, env, i64, i64)
-DEF_HELPER_3(mullv, i64, env, i64, i64)
-DEF_HELPER_3(mulqv, i64, env, i64, i64)
-DEF_HELPER_FLAGS_2(umulh, TCG_CALL_CONST | TCG_CALL_PURE, i64, i64, i64)
-
-DEF_HELPER_FLAGS_1(ctpop, TCG_CALL_CONST | TCG_CALL_PURE, i64, i64)
-DEF_HELPER_FLAGS_1(ctlz, TCG_CALL_CONST | TCG_CALL_PURE, i64, i64)
-DEF_HELPER_FLAGS_1(cttz, TCG_CALL_CONST | TCG_CALL_PURE, i64, i64)
-
-DEF_HELPER_FLAGS_2(zap, TCG_CALL_CONST | TCG_CALL_PURE, i64, i64, i64)
-DEF_HELPER_FLAGS_2(zapnot, TCG_CALL_CONST | TCG_CALL_PURE, i64, i64, i64)
-
-DEF_HELPER_FLAGS_2(cmpbge, TCG_CALL_CONST | TCG_CALL_PURE, i64, i64, i64)
-
-DEF_HELPER_FLAGS_2(minub8, TCG_CALL_CONST | TCG_CALL_PURE, i64, i64, i64)
-DEF_HELPER_FLAGS_2(minsb8, TCG_CALL_CONST | TCG_CALL_PURE, i64, i64, i64)
-DEF_HELPER_FLAGS_2(minuw4, TCG_CALL_CONST | TCG_CALL_PURE, i64, i64, i64)
-DEF_HELPER_FLAGS_2(minsw4, TCG_CALL_CONST | TCG_CALL_PURE, i64, i64, i64)
-DEF_HELPER_FLAGS_2(maxub8, TCG_CALL_CONST | TCG_CALL_PURE, i64, i64, i64)
-DEF_HELPER_FLAGS_2(maxsb8, TCG_CALL_CONST | TCG_CALL_PURE, i64, i64, i64)
-DEF_HELPER_FLAGS_2(maxuw4, TCG_CALL_CONST | TCG_CALL_PURE, i64, i64, i64)
-DEF_HELPER_FLAGS_2(maxsw4, TCG_CALL_CONST | TCG_CALL_PURE, i64, i64, i64)
-DEF_HELPER_FLAGS_2(perr, TCG_CALL_CONST | TCG_CALL_PURE, i64, i64, i64)
-DEF_HELPER_FLAGS_1(pklb, TCG_CALL_CONST | TCG_CALL_PURE, i64, i64)
-DEF_HELPER_FLAGS_1(pkwb, TCG_CALL_CONST | TCG_CALL_PURE, i64, i64)
-DEF_HELPER_FLAGS_1(unpkbl, TCG_CALL_CONST | TCG_CALL_PURE, i64, i64)
-DEF_HELPER_FLAGS_1(unpkbw, TCG_CALL_CONST | TCG_CALL_PURE, i64, i64)
-
-DEF_HELPER_FLAGS_1(load_fpcr, TCG_CALL_CONST | TCG_CALL_PURE, i64, env)
-DEF_HELPER_FLAGS_2(store_fpcr, TCG_CALL_CONST, void, env, i64)
-
-DEF_HELPER_FLAGS_1(f_to_memory, TCG_CALL_CONST | TCG_CALL_PURE, i32, i64)
-DEF_HELPER_FLAGS_1(memory_to_f, TCG_CALL_CONST | TCG_CALL_PURE, i64, i32)
-DEF_HELPER_FLAGS_3(addf, TCG_CALL_CONST, i64, env, i64, i64)
-DEF_HELPER_FLAGS_3(subf, TCG_CALL_CONST, i64, env, i64, i64)
-DEF_HELPER_FLAGS_3(mulf, TCG_CALL_CONST, i64, env, i64, i64)
-DEF_HELPER_FLAGS_3(divf, TCG_CALL_CONST, i64, env, i64, i64)
-DEF_HELPER_FLAGS_2(sqrtf, TCG_CALL_CONST, i64, env, i64)
-
-DEF_HELPER_FLAGS_1(g_to_memory, TCG_CALL_CONST | TCG_CALL_PURE, i64, i64)
-DEF_HELPER_FLAGS_1(memory_to_g, TCG_CALL_CONST | TCG_CALL_PURE, i64, i64)
-DEF_HELPER_FLAGS_3(addg, TCG_CALL_CONST, i64, env, i64, i64)
-DEF_HELPER_FLAGS_3(subg, TCG_CALL_CONST, i64, env, i64, i64)
-DEF_HELPER_FLAGS_3(mulg, TCG_CALL_CONST, i64, env, i64, i64)
-DEF_HELPER_FLAGS_3(divg, TCG_CALL_CONST, i64, env, i64, i64)
-DEF_HELPER_FLAGS_2(sqrtg, TCG_CALL_CONST, i64, env, i64)
-
-DEF_HELPER_FLAGS_1(s_to_memory, TCG_CALL_CONST | TCG_CALL_PURE, i32, i64)
-DEF_HELPER_FLAGS_1(memory_to_s, TCG_CALL_CONST | TCG_CALL_PURE, i64, i32)
-DEF_HELPER_FLAGS_3(adds, TCG_CALL_CONST, i64, env, i64, i64)
-DEF_HELPER_FLAGS_3(subs, TCG_CALL_CONST, i64, env, i64, i64)
-DEF_HELPER_FLAGS_3(muls, TCG_CALL_CONST, i64, env, i64, i64)
-DEF_HELPER_FLAGS_3(divs, TCG_CALL_CONST, i64, env, i64, i64)
-DEF_HELPER_FLAGS_2(sqrts, TCG_CALL_CONST, i64, env, i64)
-
-DEF_HELPER_FLAGS_3(addt, TCG_CALL_CONST, i64, env, i64, i64)
-DEF_HELPER_FLAGS_3(subt, TCG_CALL_CONST, i64, env, i64, i64)
-DEF_HELPER_FLAGS_3(mult, TCG_CALL_CONST, i64, env, i64, i64)
-DEF_HELPER_FLAGS_3(divt, TCG_CALL_CONST, i64, env, i64, i64)
-DEF_HELPER_FLAGS_2(sqrtt, TCG_CALL_CONST, i64, env, i64)
-
-DEF_HELPER_FLAGS_3(cmptun, TCG_CALL_CONST, i64, env, i64, i64)
-DEF_HELPER_FLAGS_3(cmpteq, TCG_CALL_CONST, i64, env, i64, i64)
-DEF_HELPER_FLAGS_3(cmptle, TCG_CALL_CONST, i64, env, i64, i64)
-DEF_HELPER_FLAGS_3(cmptlt, TCG_CALL_CONST, i64, env, i64, i64)
-DEF_HELPER_FLAGS_3(cmpgeq, TCG_CALL_CONST, i64, env, i64, i64)
-DEF_HELPER_FLAGS_3(cmpgle, TCG_CALL_CONST, i64, env, i64, i64)
-DEF_HELPER_FLAGS_3(cmpglt, TCG_CALL_CONST, i64, env, i64, i64)
-
-DEF_HELPER_FLAGS_2(cvtts, TCG_CALL_CONST, i64, env, i64)
-DEF_HELPER_FLAGS_2(cvtst, TCG_CALL_CONST, i64, env, i64)
-DEF_HELPER_FLAGS_2(cvtqs, TCG_CALL_CONST, i64, env, i64)
-DEF_HELPER_FLAGS_2(cvtqt, TCG_CALL_CONST, i64, env, i64)
-DEF_HELPER_FLAGS_2(cvtqf, TCG_CALL_CONST, i64, env, i64)
-DEF_HELPER_FLAGS_2(cvtgf, TCG_CALL_CONST, i64, env, i64)
-DEF_HELPER_FLAGS_2(cvtgq, TCG_CALL_CONST, i64, env, i64)
-DEF_HELPER_FLAGS_2(cvtqg, TCG_CALL_CONST, i64, env, i64)
-
-DEF_HELPER_FLAGS_2(cvttq, TCG_CALL_CONST, i64, env, i64)
-DEF_HELPER_FLAGS_2(cvttq_c, TCG_CALL_CONST, i64, env, i64)
-DEF_HELPER_FLAGS_2(cvttq_svic, TCG_CALL_CONST, i64, env, i64)
-
-DEF_HELPER_FLAGS_2(setroundmode, TCG_CALL_CONST, void, env, i32)
-DEF_HELPER_FLAGS_2(setflushzero, TCG_CALL_CONST, void, env, i32)
-DEF_HELPER_FLAGS_1(fp_exc_clear, TCG_CALL_CONST, void, env)
-DEF_HELPER_FLAGS_1(fp_exc_get, TCG_CALL_CONST | TCG_CALL_PURE, i32, env)
-DEF_HELPER_3(fp_exc_raise, void, env, i32, i32)
-DEF_HELPER_3(fp_exc_raise_s, void, env, i32, i32)
-
-DEF_HELPER_2(ieee_input, void, env, i64)
-DEF_HELPER_2(ieee_input_cmp, void, env, i64)
+DEF_HELPER_FLAGS_1(load_pcc, TCG_CALL_NO_RWG_SE, i64, env)
+
+DEF_HELPER_FLAGS_3(addqv, TCG_CALL_NO_WG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_3(addlv, TCG_CALL_NO_WG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_3(subqv, TCG_CALL_NO_WG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_3(sublv, TCG_CALL_NO_WG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_3(mullv, TCG_CALL_NO_WG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_3(mulqv, TCG_CALL_NO_WG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_2(umulh, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+
+DEF_HELPER_FLAGS_1(ctpop, TCG_CALL_NO_RWG_SE, i64, i64)
+DEF_HELPER_FLAGS_1(ctlz, TCG_CALL_NO_RWG_SE, i64, i64)
+DEF_HELPER_FLAGS_1(cttz, TCG_CALL_NO_RWG_SE, i64, i64)
+
+DEF_HELPER_FLAGS_2(zap, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(zapnot, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+
+DEF_HELPER_FLAGS_2(cmpbge, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+
+DEF_HELPER_FLAGS_2(minub8, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(minsb8, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(minuw4, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(minsw4, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(maxub8, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(maxsb8, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(maxuw4, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(maxsw4, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(perr, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_1(pklb, TCG_CALL_NO_RWG_SE, i64, i64)
+DEF_HELPER_FLAGS_1(pkwb, TCG_CALL_NO_RWG_SE, i64, i64)
+DEF_HELPER_FLAGS_1(unpkbl, TCG_CALL_NO_RWG_SE, i64, i64)
+DEF_HELPER_FLAGS_1(unpkbw, TCG_CALL_NO_RWG_SE, i64, i64)
+
+DEF_HELPER_FLAGS_1(load_fpcr, TCG_CALL_NO_RWG_SE, i64, env)
+DEF_HELPER_FLAGS_2(store_fpcr, TCG_CALL_NO_RWG, void, env, i64)
+
+DEF_HELPER_FLAGS_1(f_to_memory, TCG_CALL_NO_RWG_SE, i32, i64)
+DEF_HELPER_FLAGS_1(memory_to_f, TCG_CALL_NO_RWG_SE, i64, i32)
+DEF_HELPER_FLAGS_3(addf, TCG_CALL_NO_RWG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_3(subf, TCG_CALL_NO_RWG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_3(mulf, TCG_CALL_NO_RWG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_3(divf, TCG_CALL_NO_RWG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_2(sqrtf, TCG_CALL_NO_RWG, i64, env, i64)
+
+DEF_HELPER_FLAGS_1(g_to_memory, TCG_CALL_NO_RWG_SE, i64, i64)
+DEF_HELPER_FLAGS_1(memory_to_g, TCG_CALL_NO_RWG_SE, i64, i64)
+DEF_HELPER_FLAGS_3(addg, TCG_CALL_NO_RWG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_3(subg, TCG_CALL_NO_RWG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_3(mulg, TCG_CALL_NO_RWG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_3(divg, TCG_CALL_NO_RWG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_2(sqrtg, TCG_CALL_NO_RWG, i64, env, i64)
+
+DEF_HELPER_FLAGS_1(s_to_memory, TCG_CALL_NO_RWG_SE, i32, i64)
+DEF_HELPER_FLAGS_1(memory_to_s, TCG_CALL_NO_RWG_SE, i64, i32)
+DEF_HELPER_FLAGS_3(adds, TCG_CALL_NO_RWG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_3(subs, TCG_CALL_NO_RWG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_3(muls, TCG_CALL_NO_RWG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_3(divs, TCG_CALL_NO_RWG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_2(sqrts, TCG_CALL_NO_RWG, i64, env, i64)
+
+DEF_HELPER_FLAGS_3(addt, TCG_CALL_NO_RWG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_3(subt, TCG_CALL_NO_RWG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_3(mult, TCG_CALL_NO_RWG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_3(divt, TCG_CALL_NO_RWG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_2(sqrtt, TCG_CALL_NO_RWG, i64, env, i64)
+
+DEF_HELPER_FLAGS_3(cmptun, TCG_CALL_NO_RWG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_3(cmpteq, TCG_CALL_NO_RWG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_3(cmptle, TCG_CALL_NO_RWG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_3(cmptlt, TCG_CALL_NO_RWG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_3(cmpgeq, TCG_CALL_NO_RWG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_3(cmpgle, TCG_CALL_NO_RWG, i64, env, i64, i64)
+DEF_HELPER_FLAGS_3(cmpglt, TCG_CALL_NO_RWG, i64, env, i64, i64)
+
+DEF_HELPER_FLAGS_2(cvtts, TCG_CALL_NO_RWG, i64, env, i64)
+DEF_HELPER_FLAGS_2(cvtst, TCG_CALL_NO_RWG, i64, env, i64)
+DEF_HELPER_FLAGS_2(cvtqs, TCG_CALL_NO_RWG, i64, env, i64)
+DEF_HELPER_FLAGS_2(cvtqt, TCG_CALL_NO_RWG, i64, env, i64)
+DEF_HELPER_FLAGS_2(cvtqf, TCG_CALL_NO_RWG, i64, env, i64)
+DEF_HELPER_FLAGS_2(cvtgf, TCG_CALL_NO_RWG, i64, env, i64)
+DEF_HELPER_FLAGS_2(cvtgq, TCG_CALL_NO_RWG, i64, env, i64)
+DEF_HELPER_FLAGS_2(cvtqg, TCG_CALL_NO_RWG, i64, env, i64)
+
+DEF_HELPER_FLAGS_2(cvttq, TCG_CALL_NO_RWG, i64, env, i64)
+DEF_HELPER_FLAGS_2(cvttq_c, TCG_CALL_NO_RWG, i64, env, i64)
+DEF_HELPER_FLAGS_2(cvttq_svic, TCG_CALL_NO_RWG, i64, env, i64)
+
+DEF_HELPER_FLAGS_2(setroundmode, TCG_CALL_NO_RWG, void, env, i32)
+DEF_HELPER_FLAGS_2(setflushzero, TCG_CALL_NO_RWG, void, env, i32)
+DEF_HELPER_FLAGS_1(fp_exc_clear, TCG_CALL_NO_RWG, void, env)
+DEF_HELPER_FLAGS_1(fp_exc_get, TCG_CALL_NO_RWG_SE, i32, env)
+DEF_HELPER_FLAGS_3(fp_exc_raise, TCG_CALL_NO_WG, void, env, i32, i32)
+DEF_HELPER_FLAGS_3(fp_exc_raise_s, TCG_CALL_NO_WG, void, env, i32, i32)
+
+DEF_HELPER_FLAGS_2(ieee_input, TCG_CALL_NO_WG, void, env, i64)
+DEF_HELPER_FLAGS_2(ieee_input_cmp, TCG_CALL_NO_WG, void, env, i64)
 
 #if !defined (CONFIG_USER_ONLY)
 DEF_HELPER_2(hw_ret, void, env, i64)
@@ -110,13 +110,13 @@ DEF_HELPER_2(stq_phys, void, i64, i64)
 DEF_HELPER_3(stl_c_phys, i64, env, i64, i64)
 DEF_HELPER_3(stq_c_phys, i64, env, i64, i64)
 
-DEF_HELPER_FLAGS_1(tbia, TCG_CALL_CONST, void, env)
-DEF_HELPER_FLAGS_2(tbis, TCG_CALL_CONST, void, env, i64)
+DEF_HELPER_FLAGS_1(tbia, TCG_CALL_NO_RWG, void, env)
+DEF_HELPER_FLAGS_2(tbis, TCG_CALL_NO_RWG, void, env, i64)
 
 DEF_HELPER_1(halt, void, i64);
 
-DEF_HELPER_FLAGS_0(get_time, TCG_CALL_CONST, i64)
-DEF_HELPER_FLAGS_2(set_alarm, TCG_CALL_CONST, void, env, i64)
+DEF_HELPER_FLAGS_0(get_time, TCG_CALL_NO_RWG, i64)
+DEF_HELPER_FLAGS_2(set_alarm, TCG_CALL_NO_RWG, void, env, i64)
 #endif
 
 #include "def-helper.h"
diff --git a/target-alpha/mem_helper.c b/target-alpha/mem_helper.c
index 87cada48f..617836cc6 100644
--- a/target-alpha/mem_helper.c
+++ b/target-alpha/mem_helper.c
@@ -107,7 +107,7 @@ static void do_unaligned_access(CPUAlphaState *env, target_ulong addr,
     cpu_loop_exit(env);
 }
 
-void cpu_unassigned_access(CPUAlphaState *env, target_phys_addr_t addr,
+void cpu_unassigned_access(CPUAlphaState *env, hwaddr addr,
                            int is_write, int is_exec, int unused, int size)
 {
     env->trap_arg0 = addr;
diff --git a/target-alpha/translate.c b/target-alpha/translate.c
index 12de6a3fb..4045f788e 100644
--- a/target-alpha/translate.c
+++ b/target-alpha/translate.c
@@ -426,27 +426,15 @@ static ExitStatus gen_bcond_internal(DisasContext *ctx, TCGCond cond,
 
         return EXIT_GOTO_TB;
     } else {
-        int lab_over = gen_new_label();
-
-        /* ??? Consider using either
-             movi pc, next
-             addi tmp, pc, disp
-             movcond pc, cond, 0, tmp, pc
-           or
-             setcond tmp, cond, 0
-             movi pc, next
-             neg tmp, tmp
-             andi tmp, tmp, disp
-             add pc, pc, tmp
-           The current diamond subgraph surely isn't efficient.  */
+        TCGv_i64 z = tcg_const_i64(0);
+        TCGv_i64 d = tcg_const_i64(dest);
+        TCGv_i64 p = tcg_const_i64(ctx->pc);
 
-        tcg_gen_brcondi_i64(cond, cmp, 0, lab_true);
-        tcg_gen_movi_i64(cpu_pc, ctx->pc);
-        tcg_gen_br(lab_over);
-        gen_set_label(lab_true);
-        tcg_gen_movi_i64(cpu_pc, dest);
-        gen_set_label(lab_over);
+        tcg_gen_movcond_i64(cond, cpu_pc, cmp, z, d, p);
 
+        tcg_temp_free_i64(z);
+        tcg_temp_free_i64(d);
+        tcg_temp_free_i64(p);
         return EXIT_PC_UPDATED;
     }
 }
@@ -521,61 +509,67 @@ static ExitStatus gen_fbcond(DisasContext *ctx, TCGCond cond, int ra,
 static void gen_cmov(TCGCond cond, int ra, int rb, int rc,
                      int islit, uint8_t lit, int mask)
 {
-    TCGCond inv_cond = tcg_invert_cond(cond);
-    int l1;
+    TCGv_i64 c1, z, v1;
 
-    if (unlikely(rc == 31))
+    if (unlikely(rc == 31)) {
         return;
+    }
 
-    l1 = gen_new_label();
-
-    if (ra != 31) {
-        if (mask) {
-            TCGv tmp = tcg_temp_new();
-            tcg_gen_andi_i64(tmp, cpu_ir[ra], 1);
-            tcg_gen_brcondi_i64(inv_cond, tmp, 0, l1);
-            tcg_temp_free(tmp);
-        } else
-            tcg_gen_brcondi_i64(inv_cond, cpu_ir[ra], 0, l1);
-    } else {
+    if (ra == 31) {
         /* Very uncommon case - Do not bother to optimize.  */
-        TCGv tmp = tcg_const_i64(0);
-        tcg_gen_brcondi_i64(inv_cond, tmp, 0, l1);
-        tcg_temp_free(tmp);
+        c1 = tcg_const_i64(0);
+    } else if (mask) {
+        c1 = tcg_const_i64(1);
+        tcg_gen_and_i64(c1, c1, cpu_ir[ra]);
+    } else {
+        c1 = cpu_ir[ra];
     }
+    if (islit) {
+        v1 = tcg_const_i64(lit);
+    } else {
+        v1 = cpu_ir[rb];
+    }
+    z = tcg_const_i64(0);
 
-    if (islit)
-        tcg_gen_movi_i64(cpu_ir[rc], lit);
-    else
-        tcg_gen_mov_i64(cpu_ir[rc], cpu_ir[rb]);
-    gen_set_label(l1);
+    tcg_gen_movcond_i64(cond, cpu_ir[rc], c1, z, v1, cpu_ir[rc]);
+
+    tcg_temp_free_i64(z);
+    if (ra == 31 || mask) {
+        tcg_temp_free_i64(c1);
+    }
+    if (islit) {
+        tcg_temp_free_i64(v1);
+    }
 }
 
 static void gen_fcmov(TCGCond cond, int ra, int rb, int rc)
 {
-    TCGv cmp_tmp;
-    int l1;
+    TCGv_i64 c1, z, v1;
 
     if (unlikely(rc == 31)) {
         return;
     }
 
-    cmp_tmp = tcg_temp_new();
+    c1 = tcg_temp_new_i64();
     if (unlikely(ra == 31)) {
-        tcg_gen_movi_i64(cmp_tmp, 0);
+        tcg_gen_movi_i64(c1, 0);
+    } else {
+        gen_fold_mzero(cond, c1, cpu_fir[ra]);
+    }
+    if (rb == 31) {
+        v1 = tcg_const_i64(0);
     } else {
-        gen_fold_mzero(cond, cmp_tmp, cpu_fir[ra]);
+        v1 = cpu_fir[rb];
     }
+    z = tcg_const_i64(0);
 
-    l1 = gen_new_label();
-    tcg_gen_brcondi_i64(tcg_invert_cond(cond), cmp_tmp, 0, l1);
-    tcg_temp_free(cmp_tmp);
+    tcg_gen_movcond_i64(cond, cpu_fir[rc], c1, z, v1, cpu_fir[rc]);
 
-    if (rb != 31)
-        tcg_gen_mov_i64(cpu_fir[rc], cpu_fir[rb]);
-    else
-        tcg_gen_movi_i64(cpu_fir[rc], 0);
-    gen_set_label(l1);
+    tcg_temp_free_i64(z);
+    tcg_temp_free_i64(c1);
+    if (rb == 31) {
+        tcg_temp_free_i64(v1);
+    }
 }
 
 #define QUAL_RM_N       0x080   /* Round mode nearest even */
@@ -3379,7 +3373,7 @@ static inline void gen_intermediate_code_internal(CPUAlphaState *env,
     int max_insns;
 
     pc_start = tb->pc;
-    gen_opc_end = gen_opc_buf + OPC_MAX_SIZE;
+    gen_opc_end = tcg_ctx.gen_opc_buf + OPC_MAX_SIZE;
 
     ctx.tb = tb;
     ctx.env = env;
@@ -3412,7 +3406,7 @@ static inline void gen_intermediate_code_internal(CPUAlphaState *env,
             }
         }
         if (search_pc) {
-            j = gen_opc_ptr - gen_opc_buf;
+            j = tcg_ctx.gen_opc_ptr - tcg_ctx.gen_opc_buf;
             if (lj < j) {
                 lj++;
                 while (lj < j)
@@ -3427,7 +3421,7 @@ static inline void gen_intermediate_code_internal(CPUAlphaState *env,
         insn = cpu_ldl_code(env, ctx.pc);
         num_insns++;
 
-	if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP))) {
+	if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
             tcg_gen_debug_insn_start(ctx.pc);
         }
 
@@ -3438,7 +3432,7 @@ static inline void gen_intermediate_code_internal(CPUAlphaState *env,
            or exhaust instruction count, stop generation.  */
         if (ret == NO_EXIT
             && ((ctx.pc & (TARGET_PAGE_SIZE - 1)) == 0
-                || gen_opc_ptr >= gen_opc_end
+                || tcg_ctx.gen_opc_ptr >= gen_opc_end
                 || num_insns >= max_insns
                 || singlestep
                 || env->singlestep_enabled)) {
@@ -3469,9 +3463,9 @@ static inline void gen_intermediate_code_internal(CPUAlphaState *env,
     }
 
     gen_icount_end(tb, num_insns);
-    *gen_opc_ptr = INDEX_op_end;
+    *tcg_ctx.gen_opc_ptr = INDEX_op_end;
     if (search_pc) {
-        j = gen_opc_ptr - gen_opc_buf;
+        j = tcg_ctx.gen_opc_ptr - tcg_ctx.gen_opc_buf;
         lj++;
         while (lj <= j)
             gen_opc_instr_start[lj++] = 0;
@@ -3483,7 +3477,7 @@ static inline void gen_intermediate_code_internal(CPUAlphaState *env,
 #ifdef DEBUG_DISAS
     if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
         qemu_log("IN: %s\n", lookup_symbol(pc_start));
-        log_target_disas(pc_start, ctx.pc - pc_start, 1);
+        log_target_disas(env, pc_start, ctx.pc - pc_start, 1);
         qemu_log("\n");
     }
 #endif
@@ -3549,6 +3543,7 @@ CPUAlphaState * cpu_alpha_init (const char *cpu_model)
     }
     env->implver = implver;
     env->amask = amask;
+    env->cpu_model_str = cpu_model;
 
     qemu_init_vcpu(env);
     return env;
diff --git a/target-arm/Makefile.objs b/target-arm/Makefile.objs
index f447c4fdf..b6f1a9ebf 100644
--- a/target-arm/Makefile.objs
+++ b/target-arm/Makefile.objs
@@ -2,5 +2,3 @@ obj-y += arm-semi.o
 obj-$(CONFIG_SOFTMMU) += machine.o
 obj-y += translate.o op_helper.o helper.o cpu.o
 obj-y += neon_helper.o iwmmxt_helper.o
-
-$(obj)/op_helper.o: QEMU_CFLAGS += $(HELPER_CFLAGS)
diff --git a/target-arm/arm-semi.c b/target-arm/arm-semi.c
index 73bde584a..7743d6701 100644
--- a/target-arm/arm-semi.c
+++ b/target-arm/arm-semi.c
@@ -166,17 +166,20 @@ static void arm_semi_flen_cb(CPUARMState *env, target_ulong ret, target_ulong er
 #endif
 }
 
-#define ARG(n)					\
-({						\
-    target_ulong __arg;				\
-    /* FIXME - handle get_user() failure */	\
-    get_user_ual(__arg, args + (n) * 4);	\
-    __arg;					\
-})
+/* Read the input value from the argument block; fail the semihosting
+ * call if the memory read fails.
+ */
+#define GET_ARG(n) do {                                 \
+    if (get_user_ual(arg ## n, args + (n) * 4)) {       \
+        return (uint32_t)-1;                            \
+    }                                                   \
+} while (0)
+
 #define SET_ARG(n, val) put_user_ual(val, args + (n) * 4)
 uint32_t do_arm_semihosting(CPUARMState *env)
 {
     target_ulong args;
+    target_ulong arg0, arg1, arg2, arg3;
     char * s;
     int nr;
     uint32_t ret;
@@ -191,33 +194,39 @@ uint32_t do_arm_semihosting(CPUARMState *env)
     args = env->regs[1];
     switch (nr) {
     case TARGET_SYS_OPEN:
-        if (!(s = lock_user_string(ARG(0))))
+        GET_ARG(0);
+        GET_ARG(1);
+        GET_ARG(2);
+        s = lock_user_string(arg0);
+        if (!s) {
             /* FIXME - should this error code be -TARGET_EFAULT ? */
             return (uint32_t)-1;
-        if (ARG(1) >= 12) {
-            unlock_user(s, ARG(0), 0);
+        }
+        if (arg1 >= 12) {
+            unlock_user(s, arg0, 0);
             return (uint32_t)-1;
         }
         if (strcmp(s, ":tt") == 0) {
-            int result_fileno = ARG(1) < 4 ? STDIN_FILENO : STDOUT_FILENO;
-            unlock_user(s, ARG(0), 0);
+            int result_fileno = arg1 < 4 ? STDIN_FILENO : STDOUT_FILENO;
+            unlock_user(s, arg0, 0);
             return result_fileno;
         }
         if (use_gdb_syscalls()) {
-            gdb_do_syscall(arm_semi_cb, "open,%s,%x,1a4", ARG(0),
-			   (int)ARG(2)+1, gdb_open_modeflags[ARG(1)]);
+            gdb_do_syscall(arm_semi_cb, "open,%s,%x,1a4", arg0,
+                           (int)arg2+1, gdb_open_modeflags[arg1]);
             ret = env->regs[0];
         } else {
-            ret = set_swi_errno(ts, open(s, open_modeflags[ARG(1)], 0644));
+            ret = set_swi_errno(ts, open(s, open_modeflags[arg1], 0644));
         }
-        unlock_user(s, ARG(0), 0);
+        unlock_user(s, arg0, 0);
         return ret;
     case TARGET_SYS_CLOSE:
+        GET_ARG(0);
         if (use_gdb_syscalls()) {
-            gdb_do_syscall(arm_semi_cb, "close,%x", ARG(0));
+            gdb_do_syscall(arm_semi_cb, "close,%x", arg0);
             return env->regs[0];
         } else {
-            return set_swi_errno(ts, close(ARG(0)));
+            return set_swi_errno(ts, close(arg0));
         }
     case TARGET_SYS_WRITEC:
         {
@@ -248,35 +257,45 @@ uint32_t do_arm_semihosting(CPUARMState *env)
         unlock_user(s, args, 0);
         return ret;
     case TARGET_SYS_WRITE:
-        len = ARG(2);
+        GET_ARG(0);
+        GET_ARG(1);
+        GET_ARG(2);
+        len = arg2;
         if (use_gdb_syscalls()) {
             arm_semi_syscall_len = len;
-            gdb_do_syscall(arm_semi_cb, "write,%x,%x,%x", ARG(0), ARG(1), len);
+            gdb_do_syscall(arm_semi_cb, "write,%x,%x,%x", arg0, arg1, len);
             return env->regs[0];
         } else {
-            if (!(s = lock_user(VERIFY_READ, ARG(1), len, 1)))
+            s = lock_user(VERIFY_READ, arg1, len, 1);
+            if (!s) {
                 /* FIXME - should this error code be -TARGET_EFAULT ? */
                 return (uint32_t)-1;
-            ret = set_swi_errno(ts, write(ARG(0), s, len));
-            unlock_user(s, ARG(1), 0);
+            }
+            ret = set_swi_errno(ts, write(arg0, s, len));
+            unlock_user(s, arg1, 0);
             if (ret == (uint32_t)-1)
                 return -1;
             return len - ret;
         }
     case TARGET_SYS_READ:
-        len = ARG(2);
+        GET_ARG(0);
+        GET_ARG(1);
+        GET_ARG(2);
+        len = arg2;
         if (use_gdb_syscalls()) {
             arm_semi_syscall_len = len;
-            gdb_do_syscall(arm_semi_cb, "read,%x,%x,%x", ARG(0), ARG(1), len);
+            gdb_do_syscall(arm_semi_cb, "read,%x,%x,%x", arg0, arg1, len);
             return env->regs[0];
         } else {
-            if (!(s = lock_user(VERIFY_WRITE, ARG(1), len, 0)))
+            s = lock_user(VERIFY_WRITE, arg1, len, 0);
+            if (!s) {
                 /* FIXME - should this error code be -TARGET_EFAULT ? */
                 return (uint32_t)-1;
-            do
-              ret = set_swi_errno(ts, read(ARG(0), s, len));
-            while (ret == -1 && errno == EINTR);
-            unlock_user(s, ARG(1), len);
+            }
+            do {
+                ret = set_swi_errno(ts, read(arg0, s, len));
+            } while (ret == -1 && errno == EINTR);
+            unlock_user(s, arg1, len);
             if (ret == (uint32_t)-1)
                 return -1;
             return len - ret;
@@ -285,30 +304,34 @@ uint32_t do_arm_semihosting(CPUARMState *env)
        /* XXX: Read from debug console. Not implemented.  */
         return 0;
     case TARGET_SYS_ISTTY:
+        GET_ARG(0);
         if (use_gdb_syscalls()) {
-            gdb_do_syscall(arm_semi_cb, "isatty,%x", ARG(0));
+            gdb_do_syscall(arm_semi_cb, "isatty,%x", arg0);
             return env->regs[0];
         } else {
-            return isatty(ARG(0));
+            return isatty(arg0);
         }
     case TARGET_SYS_SEEK:
+        GET_ARG(0);
+        GET_ARG(1);
         if (use_gdb_syscalls()) {
-            gdb_do_syscall(arm_semi_cb, "lseek,%x,%x,0", ARG(0), ARG(1));
+            gdb_do_syscall(arm_semi_cb, "lseek,%x,%x,0", arg0, arg1);
             return env->regs[0];
         } else {
-            ret = set_swi_errno(ts, lseek(ARG(0), ARG(1), SEEK_SET));
+            ret = set_swi_errno(ts, lseek(arg0, arg1, SEEK_SET));
             if (ret == (uint32_t)-1)
               return -1;
             return 0;
         }
     case TARGET_SYS_FLEN:
+        GET_ARG(0);
         if (use_gdb_syscalls()) {
             gdb_do_syscall(arm_semi_flen_cb, "fstat,%x,%x",
-			   ARG(0), env->regs[13]-64);
+                           arg0, env->regs[13]-64);
             return env->regs[0];
         } else {
             struct stat buf;
-            ret = set_swi_errno(ts, fstat(ARG(0), &buf));
+            ret = set_swi_errno(ts, fstat(arg0, &buf));
             if (ret == (uint32_t)-1)
                 return -1;
             return buf.st_size;
@@ -317,35 +340,43 @@ uint32_t do_arm_semihosting(CPUARMState *env)
         /* XXX: Not implemented.  */
         return -1;
     case TARGET_SYS_REMOVE:
+        GET_ARG(0);
+        GET_ARG(1);
         if (use_gdb_syscalls()) {
-            gdb_do_syscall(arm_semi_cb, "unlink,%s", ARG(0), (int)ARG(1)+1);
+            gdb_do_syscall(arm_semi_cb, "unlink,%s", arg0, (int)arg1+1);
             ret = env->regs[0];
         } else {
-            if (!(s = lock_user_string(ARG(0))))
+            s = lock_user_string(arg0);
+            if (!s) {
                 /* FIXME - should this error code be -TARGET_EFAULT ? */
                 return (uint32_t)-1;
+            }
             ret =  set_swi_errno(ts, remove(s));
-            unlock_user(s, ARG(0), 0);
+            unlock_user(s, arg0, 0);
         }
         return ret;
     case TARGET_SYS_RENAME:
+        GET_ARG(0);
+        GET_ARG(1);
+        GET_ARG(2);
+        GET_ARG(3);
         if (use_gdb_syscalls()) {
             gdb_do_syscall(arm_semi_cb, "rename,%s,%s",
-                           ARG(0), (int)ARG(1)+1, ARG(2), (int)ARG(3)+1);
+                           arg0, (int)arg1+1, arg2, (int)arg3+1);
             return env->regs[0];
         } else {
             char *s2;
-            s = lock_user_string(ARG(0));
-            s2 = lock_user_string(ARG(2));
+            s = lock_user_string(arg0);
+            s2 = lock_user_string(arg2);
             if (!s || !s2)
                 /* FIXME - should this error code be -TARGET_EFAULT ? */
                 ret = (uint32_t)-1;
             else
                 ret = set_swi_errno(ts, rename(s, s2));
             if (s2)
-                unlock_user(s2, ARG(2), 0);
+                unlock_user(s2, arg2, 0);
             if (s)
-                unlock_user(s, ARG(0), 0);
+                unlock_user(s, arg0, 0);
             return ret;
         }
     case TARGET_SYS_CLOCK:
@@ -353,15 +384,19 @@ uint32_t do_arm_semihosting(CPUARMState *env)
     case TARGET_SYS_TIME:
         return set_swi_errno(ts, time(NULL));
     case TARGET_SYS_SYSTEM:
+        GET_ARG(0);
+        GET_ARG(1);
         if (use_gdb_syscalls()) {
-            gdb_do_syscall(arm_semi_cb, "system,%s", ARG(0), (int)ARG(1)+1);
+            gdb_do_syscall(arm_semi_cb, "system,%s", arg0, (int)arg1+1);
             return env->regs[0];
         } else {
-            if (!(s = lock_user_string(ARG(0))))
+            s = lock_user_string(arg0);
+            if (!s) {
                 /* FIXME - should this error code be -TARGET_EFAULT ? */
                 return (uint32_t)-1;
+            }
             ret = set_swi_errno(ts, system(s));
-            unlock_user(s, ARG(0), 0);
+            unlock_user(s, arg0, 0);
             return ret;
         }
     case TARGET_SYS_ERRNO:
@@ -375,22 +410,24 @@ uint32_t do_arm_semihosting(CPUARMState *env)
             /* Build a command-line from the original argv.
              *
              * The inputs are:
-             *     * ARG(0), pointer to a buffer of at least the size
-             *               specified in ARG(1).
-             *     * ARG(1), size of the buffer pointed to by ARG(0) in
+             *     * arg0, pointer to a buffer of at least the size
+             *               specified in arg1.
+             *     * arg1, size of the buffer pointed to by arg0 in
              *               bytes.
              *
              * The outputs are:
-             *     * ARG(0), pointer to null-terminated string of the
+             *     * arg0, pointer to null-terminated string of the
              *               command line.
-             *     * ARG(1), length of the string pointed to by ARG(0).
+             *     * arg1, length of the string pointed to by arg0.
              */
 
             char *output_buffer;
-            size_t input_size = ARG(1);
+            size_t input_size;
             size_t output_size;
             int status = 0;
-
+            GET_ARG(0);
+            GET_ARG(1);
+            input_size = arg1;
             /* Compute the size of the output string.  */
 #if !defined(CONFIG_USER_ONLY)
             output_size = strlen(ts->boot_info->kernel_filename)
@@ -414,10 +451,13 @@ uint32_t do_arm_semihosting(CPUARMState *env)
             }
 
             /* Adjust the command-line length.  */
-            SET_ARG(1, output_size - 1);
+            if (SET_ARG(1, output_size - 1)) {
+                /* Couldn't write back to argument block */
+                return -1;
+            }
 
             /* Lock the buffer on the ARM side.  */
-            output_buffer = lock_user(VERIFY_WRITE, ARG(0), output_size, 0);
+            output_buffer = lock_user(VERIFY_WRITE, arg0, output_size, 0);
             if (!output_buffer) {
                 return -1;
             }
@@ -449,7 +489,7 @@ uint32_t do_arm_semihosting(CPUARMState *env)
         out:
 #endif
             /* Unlock the buffer on the ARM side.  */
-            unlock_user(output_buffer, ARG(0), output_size);
+            unlock_user(output_buffer, arg0, output_size);
 
             return status;
         }
@@ -457,6 +497,7 @@ uint32_t do_arm_semihosting(CPUARMState *env)
         {
             uint32_t *ptr;
             uint32_t limit;
+            GET_ARG(0);
 
 #ifdef CONFIG_USER_ONLY
             /* Some C libraries assume the heap immediately follows .bss, so
@@ -477,25 +518,29 @@ uint32_t do_arm_semihosting(CPUARMState *env)
                 ts->heap_limit = limit;
             }
 
-            if (!(ptr = lock_user(VERIFY_WRITE, ARG(0), 16, 0)))
+            ptr = lock_user(VERIFY_WRITE, arg0, 16, 0);
+            if (!ptr) {
                 /* FIXME - should this error code be -TARGET_EFAULT ? */
                 return (uint32_t)-1;
+            }
             ptr[0] = tswap32(ts->heap_base);
             ptr[1] = tswap32(ts->heap_limit);
             ptr[2] = tswap32(ts->stack_base);
             ptr[3] = tswap32(0); /* Stack limit.  */
-            unlock_user(ptr, ARG(0), 16);
+            unlock_user(ptr, arg0, 16);
 #else
             limit = ram_size;
-            if (!(ptr = lock_user(VERIFY_WRITE, ARG(0), 16, 0)))
+            ptr = lock_user(VERIFY_WRITE, arg0, 16, 0);
+            if (!ptr) {
                 /* FIXME - should this error code be -TARGET_EFAULT ? */
                 return (uint32_t)-1;
+            }
             /* TODO: Make this use the limit of the loaded application.  */
             ptr[0] = tswap32(limit / 2);
             ptr[1] = tswap32(limit);
             ptr[2] = tswap32(limit); /* Stack base */
             ptr[3] = tswap32(0); /* Stack limit.  */
-            unlock_user(ptr, ARG(0), 16);
+            unlock_user(ptr, arg0, 16);
 #endif
             return 0;
         }
diff --git a/target-arm/cpu.h b/target-arm/cpu.h
index d7f93d98f..e4ff918fa 100644
--- a/target-arm/cpu.h
+++ b/target-arm/cpu.h
@@ -423,8 +423,6 @@ void armv7m_nvic_complete_irq(void *opaque, int irq);
     (((cp) << 16) | ((is64) << 15) | ((crn) << 11) |    \
      ((crm) << 7) | ((opc1) << 3) | (opc2))
 
-#define DECODE_CPREG_CRN(enc) (((enc) >> 7) & 0xf)
-
 /* ARMCPRegInfo type field bits. If the SPECIAL bit is set this is a
  * special-behaviour cp reg and bits [15..8] indicate what behaviour
  * it has. Otherwise it is a simple cp reg, where CONST indicates that
@@ -720,8 +718,10 @@ static inline void cpu_get_tb_cpu_state(CPUARMState *env, target_ulong *pc,
     }
 }
 
-static inline bool cpu_has_work(CPUARMState *env)
+static inline bool cpu_has_work(CPUState *cpu)
 {
+    CPUARMState *env = &ARM_CPU(cpu)->env;
+
     return env->interrupt_request &
         (CPU_INTERRUPT_FIQ | CPU_INTERRUPT_HARD | CPU_INTERRUPT_EXITTB);
 }
@@ -734,9 +734,10 @@ static inline void cpu_pc_from_tb(CPUARMState *env, TranslationBlock *tb)
 }
 
 /* Load an instruction and return it in the standard little-endian order */
-static inline uint32_t arm_ldl_code(uint32_t addr, bool do_swap)
+static inline uint32_t arm_ldl_code(CPUARMState *env, uint32_t addr,
+                                    bool do_swap)
 {
-    uint32_t insn = ldl_code(addr);
+    uint32_t insn = cpu_ldl_code(env, addr);
     if (do_swap) {
         return bswap32(insn);
     }
@@ -744,9 +745,10 @@ static inline uint32_t arm_ldl_code(uint32_t addr, bool do_swap)
 }
 
 /* Ditto, for a halfword (Thumb) instruction */
-static inline uint16_t arm_lduw_code(uint32_t addr, bool do_swap)
+static inline uint16_t arm_lduw_code(CPUARMState *env, uint32_t addr,
+                                     bool do_swap)
 {
-    uint16_t insn = lduw_code(addr);
+    uint16_t insn = cpu_lduw_code(env, addr);
     if (do_swap) {
         return bswap16(insn);
     }
diff --git a/target-arm/helper.c b/target-arm/helper.c
index dceaa95c8..ab8b73493 100644
--- a/target-arm/helper.c
+++ b/target-arm/helper.c
@@ -8,7 +8,7 @@
 #ifndef CONFIG_USER_ONLY
 static inline int get_phys_addr(CPUARMState *env, uint32_t address,
                                 int access_type, int is_user,
-                                target_phys_addr_t *phys_ptr, int *prot,
+                                hwaddr *phys_ptr, int *prot,
                                 target_ulong *page_size);
 #endif
 
@@ -517,7 +517,7 @@ static inline bool extended_addresses_enabled(CPUARMState *env)
 
 static int ats_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value)
 {
-    target_phys_addr_t phys_addr;
+    hwaddr phys_addr;
     target_ulong page_size;
     int prot;
     int ret, is_user = ri->opc2 & 2;
@@ -645,7 +645,7 @@ static int pmsav5_insn_ap_read(CPUARMState *env, const ARMCPRegInfo *ri,
 static int arm946_prbs_read(CPUARMState *env, const ARMCPRegInfo *ri,
                             uint64_t *value)
 {
-    if (ri->crm > 8) {
+    if (ri->crm >= 8) {
         return EXCP_UDEF;
     }
     *value = env->cp15.c6_region[ri->crm];
@@ -655,7 +655,7 @@ static int arm946_prbs_read(CPUARMState *env, const ARMCPRegInfo *ri,
 static int arm946_prbs_write(CPUARMState *env, const ARMCPRegInfo *ri,
                              uint64_t value)
 {
-    if (ri->crm > 8) {
+    if (ri->crm >= 8) {
         return EXCP_UDEF;
     }
     env->cp15.c6_region[ri->crm] = value;
@@ -1562,11 +1562,6 @@ uint32_t HELPER(rbit)(uint32_t x)
     return x;
 }
 
-uint32_t HELPER(abs)(uint32_t x)
-{
-    return ((int32_t)x < 0) ? -x : x;
-}
-
 #if defined(CONFIG_USER_ONLY)
 
 void do_interrupt (CPUARMState *env)
@@ -1756,7 +1751,7 @@ static void do_interrupt_v7m(CPUARMState *env)
     case EXCP_BKPT:
         if (semihosting_enabled) {
             int nr;
-            nr = arm_lduw_code(env->regs[15], env->bswap_code) & 0xff;
+            nr = arm_lduw_code(env, env->regs[15], env->bswap_code) & 0xff;
             if (nr == 0xab) {
                 env->regs[15] += 2;
                 env->regs[0] = do_arm_semihosting(env);
@@ -1828,9 +1823,10 @@ void do_interrupt(CPUARMState *env)
         if (semihosting_enabled) {
             /* Check for semihosting interrupt.  */
             if (env->thumb) {
-                mask = arm_lduw_code(env->regs[15] - 2, env->bswap_code) & 0xff;
+                mask = arm_lduw_code(env, env->regs[15] - 2, env->bswap_code)
+                    & 0xff;
             } else {
-                mask = arm_ldl_code(env->regs[15] - 4, env->bswap_code)
+                mask = arm_ldl_code(env, env->regs[15] - 4, env->bswap_code)
                     & 0xffffff;
             }
             /* Only intercept calls from privileged modes, to provide some
@@ -1851,7 +1847,7 @@ void do_interrupt(CPUARMState *env)
     case EXCP_BKPT:
         /* See if this is a semihosting syscall.  */
         if (env->thumb && semihosting_enabled) {
-            mask = arm_lduw_code(env->regs[15], env->bswap_code) & 0xff;
+            mask = arm_lduw_code(env, env->regs[15], env->bswap_code) & 0xff;
             if (mask == 0xab
                   && (env->uncached_cpsr & CPSR_M) != ARM_CPU_MODE_USR) {
                 env->regs[15] += 2;
@@ -1979,7 +1975,7 @@ static uint32_t get_level1_table_address(CPUARMState *env, uint32_t address)
 }
 
 static int get_phys_addr_v5(CPUARMState *env, uint32_t address, int access_type,
-                            int is_user, target_phys_addr_t *phys_ptr,
+                            int is_user, hwaddr *phys_ptr,
                             int *prot, target_ulong *page_size)
 {
     int code;
@@ -1989,7 +1985,7 @@ static int get_phys_addr_v5(CPUARMState *env, uint32_t address, int access_type,
     int ap;
     int domain;
     int domain_prot;
-    target_phys_addr_t phys_addr;
+    hwaddr phys_addr;
 
     /* Pagetable walk.  */
     /* Lookup l1 descriptor.  */
@@ -2074,7 +2070,7 @@ do_fault:
 }
 
 static int get_phys_addr_v6(CPUARMState *env, uint32_t address, int access_type,
-                            int is_user, target_phys_addr_t *phys_ptr,
+                            int is_user, hwaddr *phys_ptr,
                             int *prot, target_ulong *page_size)
 {
     int code;
@@ -2086,7 +2082,7 @@ static int get_phys_addr_v6(CPUARMState *env, uint32_t address, int access_type,
     int ap;
     int domain = 0;
     int domain_prot;
-    target_phys_addr_t phys_addr;
+    hwaddr phys_addr;
 
     /* Pagetable walk.  */
     /* Lookup l1 descriptor.  */
@@ -2196,7 +2192,7 @@ typedef enum {
 
 static int get_phys_addr_lpae(CPUARMState *env, uint32_t address,
                               int access_type, int is_user,
-                              target_phys_addr_t *phys_ptr, int *prot,
+                              hwaddr *phys_ptr, int *prot,
                               target_ulong *page_size_ptr)
 {
     /* Read an LPAE long-descriptor translation table. */
@@ -2207,7 +2203,7 @@ static int get_phys_addr_lpae(CPUARMState *env, uint32_t address,
     uint64_t ttbr;
     int ttbr_select;
     int n;
-    target_phys_addr_t descaddr;
+    hwaddr descaddr;
     uint32_t tableattrs;
     target_ulong page_size;
     uint32_t attrs;
@@ -2365,7 +2361,7 @@ do_fault:
 
 static int get_phys_addr_mpu(CPUARMState *env, uint32_t address,
                              int access_type, int is_user,
-                             target_phys_addr_t *phys_ptr, int *prot)
+                             hwaddr *phys_ptr, int *prot)
 {
     int n;
     uint32_t mask;
@@ -2449,7 +2445,7 @@ static int get_phys_addr_mpu(CPUARMState *env, uint32_t address,
  */
 static inline int get_phys_addr(CPUARMState *env, uint32_t address,
                                 int access_type, int is_user,
-                                target_phys_addr_t *phys_ptr, int *prot,
+                                hwaddr *phys_ptr, int *prot,
                                 target_ulong *page_size)
 {
     /* Fast Context Switch Extension.  */
@@ -2481,7 +2477,7 @@ static inline int get_phys_addr(CPUARMState *env, uint32_t address,
 int cpu_arm_handle_mmu_fault (CPUARMState *env, target_ulong address,
                               int access_type, int mmu_idx)
 {
-    target_phys_addr_t phys_addr;
+    hwaddr phys_addr;
     target_ulong page_size;
     int prot;
     int ret, is_user;
@@ -2491,7 +2487,7 @@ int cpu_arm_handle_mmu_fault (CPUARMState *env, target_ulong address,
                         &page_size);
     if (ret == 0) {
         /* Map a single [sub]page.  */
-        phys_addr &= ~(target_phys_addr_t)0x3ff;
+        phys_addr &= ~(hwaddr)0x3ff;
         address &= ~(uint32_t)0x3ff;
         tlb_set_page (env, address, phys_addr, prot, mmu_idx, page_size);
         return 0;
@@ -2511,9 +2507,9 @@ int cpu_arm_handle_mmu_fault (CPUARMState *env, target_ulong address,
     return 1;
 }
 
-target_phys_addr_t cpu_get_phys_page_debug(CPUARMState *env, target_ulong addr)
+hwaddr cpu_get_phys_page_debug(CPUARMState *env, target_ulong addr)
 {
-    target_phys_addr_t phys_addr;
+    hwaddr phys_addr;
     target_ulong page_size;
     int prot;
     int ret;
diff --git a/target-arm/helper.h b/target-arm/helper.h
index 21e9cfe05..3d23ceb25 100644
--- a/target-arm/helper.h
+++ b/target-arm/helper.h
@@ -1,19 +1,18 @@
 #include "def-helper.h"
 
-DEF_HELPER_1(clz, i32, i32)
-DEF_HELPER_1(sxtb16, i32, i32)
-DEF_HELPER_1(uxtb16, i32, i32)
-
-DEF_HELPER_2(add_setq, i32, i32, i32)
-DEF_HELPER_2(add_saturate, i32, i32, i32)
-DEF_HELPER_2(sub_saturate, i32, i32, i32)
-DEF_HELPER_2(add_usaturate, i32, i32, i32)
-DEF_HELPER_2(sub_usaturate, i32, i32, i32)
-DEF_HELPER_1(double_saturate, i32, s32)
-DEF_HELPER_2(sdiv, s32, s32, s32)
-DEF_HELPER_2(udiv, i32, i32, i32)
-DEF_HELPER_1(rbit, i32, i32)
-DEF_HELPER_1(abs, i32, i32)
+DEF_HELPER_FLAGS_1(clz, TCG_CALL_NO_RWG_SE, i32, i32)
+DEF_HELPER_FLAGS_1(sxtb16, TCG_CALL_NO_RWG_SE, i32, i32)
+DEF_HELPER_FLAGS_1(uxtb16, TCG_CALL_NO_RWG_SE, i32, i32)
+
+DEF_HELPER_3(add_setq, i32, env, i32, i32)
+DEF_HELPER_3(add_saturate, i32, env, i32, i32)
+DEF_HELPER_3(sub_saturate, i32, env, i32, i32)
+DEF_HELPER_3(add_usaturate, i32, env, i32, i32)
+DEF_HELPER_3(sub_usaturate, i32, env, i32, i32)
+DEF_HELPER_2(double_saturate, i32, env, s32)
+DEF_HELPER_FLAGS_2(sdiv, TCG_CALL_NO_RWG_SE, s32, s32, s32)
+DEF_HELPER_FLAGS_2(udiv, TCG_CALL_NO_RWG_SE, i32, i32, i32)
+DEF_HELPER_FLAGS_1(rbit, TCG_CALL_NO_RWG_SE, i32, i32)
 
 #define PAS_OP(pfx)  \
     DEF_HELPER_3(pfx ## add8, i32, i32, i32, ptr) \
@@ -40,21 +39,22 @@ PAS_OP(uq)
 PAS_OP(uh)
 #undef PAS_OP
 
-DEF_HELPER_2(ssat, i32, i32, i32)
-DEF_HELPER_2(usat, i32, i32, i32)
-DEF_HELPER_2(ssat16, i32, i32, i32)
-DEF_HELPER_2(usat16, i32, i32, i32)
+DEF_HELPER_3(ssat, i32, env, i32, i32)
+DEF_HELPER_3(usat, i32, env, i32, i32)
+DEF_HELPER_3(ssat16, i32, env, i32, i32)
+DEF_HELPER_3(usat16, i32, env, i32, i32)
 
-DEF_HELPER_2(usad8, i32, i32, i32)
+DEF_HELPER_FLAGS_2(usad8, TCG_CALL_NO_RWG_SE, i32, i32, i32)
 
 DEF_HELPER_1(logicq_cc, i32, i64)
 
-DEF_HELPER_3(sel_flags, i32, i32, i32, i32)
-DEF_HELPER_1(exception, void, i32)
-DEF_HELPER_0(wfi, void)
+DEF_HELPER_FLAGS_3(sel_flags, TCG_CALL_NO_RWG_SE,
+                   i32, i32, i32, i32)
+DEF_HELPER_2(exception, void, env, i32)
+DEF_HELPER_1(wfi, void, env)
 
-DEF_HELPER_2(cpsr_write, void, i32, i32)
-DEF_HELPER_0(cpsr_read, i32)
+DEF_HELPER_3(cpsr_write, void, env, i32, i32)
+DEF_HELPER_1(cpsr_read, i32, env)
 
 DEF_HELPER_3(v7m_msr, void, env, i32, i32)
 DEF_HELPER_2(v7m_mrs, i32, env, i32)
@@ -67,8 +67,8 @@ DEF_HELPER_2(get_cp_reg64, i64, env, ptr)
 DEF_HELPER_2(get_r13_banked, i32, env, i32)
 DEF_HELPER_3(set_r13_banked, void, env, i32, i32)
 
-DEF_HELPER_1(get_user_reg, i32, i32)
-DEF_HELPER_2(set_user_reg, void, i32, i32)
+DEF_HELPER_2(get_user_reg, i32, env, i32)
+DEF_HELPER_3(set_user_reg, void, env, i32, i32)
 
 DEF_HELPER_1(vfp_get_fpscr, i32, env)
 DEF_HELPER_2(vfp_set_fpscr, void, env, i32)
@@ -140,20 +140,15 @@ DEF_HELPER_2(recpe_f32, f32, f32, env)
 DEF_HELPER_2(rsqrte_f32, f32, f32, env)
 DEF_HELPER_2(recpe_u32, i32, i32, env)
 DEF_HELPER_2(rsqrte_u32, i32, i32, env)
-DEF_HELPER_4(neon_tbl, i32, i32, i32, i32, i32)
+DEF_HELPER_5(neon_tbl, i32, env, i32, i32, i32, i32)
 
-DEF_HELPER_2(add_cc, i32, i32, i32)
-DEF_HELPER_2(adc_cc, i32, i32, i32)
-DEF_HELPER_2(sub_cc, i32, i32, i32)
-DEF_HELPER_2(sbc_cc, i32, i32, i32)
+DEF_HELPER_3(adc_cc, i32, env, i32, i32)
+DEF_HELPER_3(sbc_cc, i32, env, i32, i32)
 
-DEF_HELPER_2(shl, i32, i32, i32)
-DEF_HELPER_2(shr, i32, i32, i32)
-DEF_HELPER_2(sar, i32, i32, i32)
-DEF_HELPER_2(shl_cc, i32, i32, i32)
-DEF_HELPER_2(shr_cc, i32, i32, i32)
-DEF_HELPER_2(sar_cc, i32, i32, i32)
-DEF_HELPER_2(ror_cc, i32, i32, i32)
+DEF_HELPER_3(shl_cc, i32, env, i32, i32)
+DEF_HELPER_3(shr_cc, i32, env, i32, i32)
+DEF_HELPER_3(sar_cc, i32, env, i32, i32)
+DEF_HELPER_3(ror_cc, i32, env, i32, i32)
 
 /* neon_helper.c */
 DEF_HELPER_3(neon_qadd_u8, i32, env, i32, i32)
@@ -343,7 +338,6 @@ DEF_HELPER_2(neon_mull_s16, i64, i32, i32)
 
 DEF_HELPER_1(neon_negl_u16, i64, i64)
 DEF_HELPER_1(neon_negl_u32, i64, i64)
-DEF_HELPER_1(neon_negl_u64, i64, i64)
 
 DEF_HELPER_2(neon_qabs_s8, i32, env, i32)
 DEF_HELPER_2(neon_qabs_s16, i32, env, i32)
diff --git a/target-arm/neon_helper.c b/target-arm/neon_helper.c
index 8bb5129d6..89280b6ee 100644
--- a/target-arm/neon_helper.c
+++ b/target-arm/neon_helper.c
@@ -788,7 +788,6 @@ uint64_t HELPER(neon_qshlu_s64)(CPUARMState *env, uint64_t valop, uint64_t shift
     return helper_neon_qshl_u64(env, valop, shiftop);
 }
 
-/* FIXME: This is wrong.  */
 #define NEON_FN(dest, src1, src2) do { \
     int8_t tmp; \
     tmp = (int8_t)src2; \
@@ -1665,12 +1664,6 @@ uint64_t HELPER(neon_negl_u32)(uint64_t x)
     return low | ((uint64_t)high << 32);
 }
 
-/* FIXME:  There should be a native op for this.  */
-uint64_t HELPER(neon_negl_u64)(uint64_t x)
-{
-    return -x;
-}
-
 /* Saturating sign manipulation.  */
 /* ??? Make these use NEON_VOP1 */
 #define DO_QABS8(x) do { \
diff --git a/target-arm/op_helper.c b/target-arm/op_helper.c
index d77bfab77..6e3ab90e3 100644
--- a/target-arm/op_helper.c
+++ b/target-arm/op_helper.c
@@ -17,19 +17,18 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 #include "cpu.h"
-#include "dyngen-exec.h"
 #include "helper.h"
 
 #define SIGNBIT (uint32_t)0x80000000
 #define SIGNBIT64 ((uint64_t)1 << 63)
 
-static void raise_exception(int tt)
+static void raise_exception(CPUARMState *env, int tt)
 {
     env->exception_index = tt;
     cpu_loop_exit(env);
 }
 
-uint32_t HELPER(neon_tbl)(uint32_t ireg, uint32_t def,
+uint32_t HELPER(neon_tbl)(CPUARMState *env, uint32_t ireg, uint32_t def,
                           uint32_t rn, uint32_t maxindex)
 {
     uint32_t val;
@@ -72,16 +71,12 @@ uint32_t HELPER(neon_tbl)(uint32_t ireg, uint32_t def,
 /* try to fill the TLB and return an exception if error. If retaddr is
    NULL, it means that the function was called in C code (i.e. not
    from generated code or from helper.c) */
-/* XXX: fix it to restore all registers */
-void tlb_fill(CPUARMState *env1, target_ulong addr, int is_write, int mmu_idx,
+void tlb_fill(CPUARMState *env, target_ulong addr, int is_write, int mmu_idx,
               uintptr_t retaddr)
 {
     TranslationBlock *tb;
-    CPUARMState *saved_env;
     int ret;
 
-    saved_env = env;
-    env = env1;
     ret = cpu_arm_handle_mmu_fault(env, addr, is_write, mmu_idx);
     if (unlikely(ret)) {
         if (retaddr) {
@@ -93,15 +88,12 @@ void tlb_fill(CPUARMState *env1, target_ulong addr, int is_write, int mmu_idx,
                 cpu_restore_state(tb, env, retaddr);
             }
         }
-        raise_exception(env->exception_index);
+        raise_exception(env, env->exception_index);
     }
-    env = saved_env;
 }
 #endif
 
-/* FIXME: Pass an explicit pointer to QF to CPUARMState, and move saturating
-   instructions into helper.c  */
-uint32_t HELPER(add_setq)(uint32_t a, uint32_t b)
+uint32_t HELPER(add_setq)(CPUARMState *env, uint32_t a, uint32_t b)
 {
     uint32_t res = a + b;
     if (((res ^ a) & SIGNBIT) && !((a ^ b) & SIGNBIT))
@@ -109,7 +101,7 @@ uint32_t HELPER(add_setq)(uint32_t a, uint32_t b)
     return res;
 }
 
-uint32_t HELPER(add_saturate)(uint32_t a, uint32_t b)
+uint32_t HELPER(add_saturate)(CPUARMState *env, uint32_t a, uint32_t b)
 {
     uint32_t res = a + b;
     if (((res ^ a) & SIGNBIT) && !((a ^ b) & SIGNBIT)) {
@@ -119,7 +111,7 @@ uint32_t HELPER(add_saturate)(uint32_t a, uint32_t b)
     return res;
 }
 
-uint32_t HELPER(sub_saturate)(uint32_t a, uint32_t b)
+uint32_t HELPER(sub_saturate)(CPUARMState *env, uint32_t a, uint32_t b)
 {
     uint32_t res = a - b;
     if (((res ^ a) & SIGNBIT) && ((a ^ b) & SIGNBIT)) {
@@ -129,7 +121,7 @@ uint32_t HELPER(sub_saturate)(uint32_t a, uint32_t b)
     return res;
 }
 
-uint32_t HELPER(double_saturate)(int32_t val)
+uint32_t HELPER(double_saturate)(CPUARMState *env, int32_t val)
 {
     uint32_t res;
     if (val >= 0x40000000) {
@@ -144,7 +136,7 @@ uint32_t HELPER(double_saturate)(int32_t val)
     return res;
 }
 
-uint32_t HELPER(add_usaturate)(uint32_t a, uint32_t b)
+uint32_t HELPER(add_usaturate)(CPUARMState *env, uint32_t a, uint32_t b)
 {
     uint32_t res = a + b;
     if (res < a) {
@@ -154,7 +146,7 @@ uint32_t HELPER(add_usaturate)(uint32_t a, uint32_t b)
     return res;
 }
 
-uint32_t HELPER(sub_usaturate)(uint32_t a, uint32_t b)
+uint32_t HELPER(sub_usaturate)(CPUARMState *env, uint32_t a, uint32_t b)
 {
     uint32_t res = a - b;
     if (res > a) {
@@ -165,7 +157,7 @@ uint32_t HELPER(sub_usaturate)(uint32_t a, uint32_t b)
 }
 
 /* Signed saturation.  */
-static inline uint32_t do_ssat(int32_t val, int shift)
+static inline uint32_t do_ssat(CPUARMState *env, int32_t val, int shift)
 {
     int32_t top;
     uint32_t mask;
@@ -183,7 +175,7 @@ static inline uint32_t do_ssat(int32_t val, int shift)
 }
 
 /* Unsigned saturation.  */
-static inline uint32_t do_usat(int32_t val, int shift)
+static inline uint32_t do_usat(CPUARMState *env, int32_t val, int shift)
 {
     uint32_t max;
 
@@ -199,62 +191,62 @@ static inline uint32_t do_usat(int32_t val, int shift)
 }
 
 /* Signed saturate.  */
-uint32_t HELPER(ssat)(uint32_t x, uint32_t shift)
+uint32_t HELPER(ssat)(CPUARMState *env, uint32_t x, uint32_t shift)
 {
-    return do_ssat(x, shift);
+    return do_ssat(env, x, shift);
 }
 
 /* Dual halfword signed saturate.  */
-uint32_t HELPER(ssat16)(uint32_t x, uint32_t shift)
+uint32_t HELPER(ssat16)(CPUARMState *env, uint32_t x, uint32_t shift)
 {
     uint32_t res;
 
-    res = (uint16_t)do_ssat((int16_t)x, shift);
-    res |= do_ssat(((int32_t)x) >> 16, shift) << 16;
+    res = (uint16_t)do_ssat(env, (int16_t)x, shift);
+    res |= do_ssat(env, ((int32_t)x) >> 16, shift) << 16;
     return res;
 }
 
 /* Unsigned saturate.  */
-uint32_t HELPER(usat)(uint32_t x, uint32_t shift)
+uint32_t HELPER(usat)(CPUARMState *env, uint32_t x, uint32_t shift)
 {
-    return do_usat(x, shift);
+    return do_usat(env, x, shift);
 }
 
 /* Dual halfword unsigned saturate.  */
-uint32_t HELPER(usat16)(uint32_t x, uint32_t shift)
+uint32_t HELPER(usat16)(CPUARMState *env, uint32_t x, uint32_t shift)
 {
     uint32_t res;
 
-    res = (uint16_t)do_usat((int16_t)x, shift);
-    res |= do_usat(((int32_t)x) >> 16, shift) << 16;
+    res = (uint16_t)do_usat(env, (int16_t)x, shift);
+    res |= do_usat(env, ((int32_t)x) >> 16, shift) << 16;
     return res;
 }
 
-void HELPER(wfi)(void)
+void HELPER(wfi)(CPUARMState *env)
 {
     env->exception_index = EXCP_HLT;
     env->halted = 1;
     cpu_loop_exit(env);
 }
 
-void HELPER(exception)(uint32_t excp)
+void HELPER(exception)(CPUARMState *env, uint32_t excp)
 {
     env->exception_index = excp;
     cpu_loop_exit(env);
 }
 
-uint32_t HELPER(cpsr_read)(void)
+uint32_t HELPER(cpsr_read)(CPUARMState *env)
 {
     return cpsr_read(env) & ~CPSR_EXEC;
 }
 
-void HELPER(cpsr_write)(uint32_t val, uint32_t mask)
+void HELPER(cpsr_write)(CPUARMState *env, uint32_t val, uint32_t mask)
 {
     cpsr_write(env, val, mask);
 }
 
 /* Access to user mode registers from privileged modes.  */
-uint32_t HELPER(get_user_reg)(uint32_t regno)
+uint32_t HELPER(get_user_reg)(CPUARMState *env, uint32_t regno)
 {
     uint32_t val;
 
@@ -271,7 +263,7 @@ uint32_t HELPER(get_user_reg)(uint32_t regno)
     return val;
 }
 
-void HELPER(set_user_reg)(uint32_t regno, uint32_t val)
+void HELPER(set_user_reg)(CPUARMState *env, uint32_t regno, uint32_t val)
 {
     if (regno == 13) {
         env->banked_r13[0] = val;
@@ -290,7 +282,7 @@ void HELPER(set_cp_reg)(CPUARMState *env, void *rip, uint32_t value)
     const ARMCPRegInfo *ri = rip;
     int excp = ri->writefn(env, ri, value);
     if (excp) {
-        raise_exception(excp);
+        raise_exception(env, excp);
     }
 }
 
@@ -300,7 +292,7 @@ uint32_t HELPER(get_cp_reg)(CPUARMState *env, void *rip)
     uint64_t value;
     int excp = ri->readfn(env, ri, &value);
     if (excp) {
-        raise_exception(excp);
+        raise_exception(env, excp);
     }
     return value;
 }
@@ -310,7 +302,7 @@ void HELPER(set_cp_reg64)(CPUARMState *env, void *rip, uint64_t value)
     const ARMCPRegInfo *ri = rip;
     int excp = ri->writefn(env, ri, value);
     if (excp) {
-        raise_exception(excp);
+        raise_exception(env, excp);
     }
 }
 
@@ -320,7 +312,7 @@ uint64_t HELPER(get_cp_reg64)(CPUARMState *env, void *rip)
     uint64_t value;
     int excp = ri->readfn(env, ri, &value);
     if (excp) {
-        raise_exception(excp);
+        raise_exception(env, excp);
     }
     return value;
 }
@@ -329,17 +321,7 @@ uint64_t HELPER(get_cp_reg64)(CPUARMState *env, void *rip)
    The only way to do that in TCG is a conditional branch, which clobbers
    all our temporaries.  For now implement these as helper functions.  */
 
-uint32_t HELPER (add_cc)(uint32_t a, uint32_t b)
-{
-    uint32_t result;
-    result = a + b;
-    env->NF = env->ZF = result;
-    env->CF = result < a;
-    env->VF = (a ^ b ^ -1) & (a ^ result);
-    return result;
-}
-
-uint32_t HELPER(adc_cc)(uint32_t a, uint32_t b)
+uint32_t HELPER(adc_cc)(CPUARMState *env, uint32_t a, uint32_t b)
 {
     uint32_t result;
     if (!env->CF) {
@@ -354,17 +336,7 @@ uint32_t HELPER(adc_cc)(uint32_t a, uint32_t b)
     return result;
 }
 
-uint32_t HELPER(sub_cc)(uint32_t a, uint32_t b)
-{
-    uint32_t result;
-    result = a - b;
-    env->NF = env->ZF = result;
-    env->CF = a >= b;
-    env->VF = (a ^ b) & (a ^ result);
-    return result;
-}
-
-uint32_t HELPER(sbc_cc)(uint32_t a, uint32_t b)
+uint32_t HELPER(sbc_cc)(CPUARMState *env, uint32_t a, uint32_t b)
 {
     uint32_t result;
     if (!env->CF) {
@@ -381,31 +353,7 @@ uint32_t HELPER(sbc_cc)(uint32_t a, uint32_t b)
 
 /* Similarly for variable shift instructions.  */
 
-uint32_t HELPER(shl)(uint32_t x, uint32_t i)
-{
-    int shift = i & 0xff;
-    if (shift >= 32)
-        return 0;
-    return x << shift;
-}
-
-uint32_t HELPER(shr)(uint32_t x, uint32_t i)
-{
-    int shift = i & 0xff;
-    if (shift >= 32)
-        return 0;
-    return (uint32_t)x >> shift;
-}
-
-uint32_t HELPER(sar)(uint32_t x, uint32_t i)
-{
-    int shift = i & 0xff;
-    if (shift >= 32)
-        shift = 31;
-    return (int32_t)x >> shift;
-}
-
-uint32_t HELPER(shl_cc)(uint32_t x, uint32_t i)
+uint32_t HELPER(shl_cc)(CPUARMState *env, uint32_t x, uint32_t i)
 {
     int shift = i & 0xff;
     if (shift >= 32) {
@@ -421,7 +369,7 @@ uint32_t HELPER(shl_cc)(uint32_t x, uint32_t i)
     return x;
 }
 
-uint32_t HELPER(shr_cc)(uint32_t x, uint32_t i)
+uint32_t HELPER(shr_cc)(CPUARMState *env, uint32_t x, uint32_t i)
 {
     int shift = i & 0xff;
     if (shift >= 32) {
@@ -437,7 +385,7 @@ uint32_t HELPER(shr_cc)(uint32_t x, uint32_t i)
     return x;
 }
 
-uint32_t HELPER(sar_cc)(uint32_t x, uint32_t i)
+uint32_t HELPER(sar_cc)(CPUARMState *env, uint32_t x, uint32_t i)
 {
     int shift = i & 0xff;
     if (shift >= 32) {
@@ -450,7 +398,7 @@ uint32_t HELPER(sar_cc)(uint32_t x, uint32_t i)
     return x;
 }
 
-uint32_t HELPER(ror_cc)(uint32_t x, uint32_t i)
+uint32_t HELPER(ror_cc)(CPUARMState *env, uint32_t x, uint32_t i)
 {
     int shift1, shift;
     shift1 = i & 0xff;
diff --git a/target-arm/translate.c b/target-arm/translate.c
index edef79a2c..c42110ab0 100644
--- a/target-arm/translate.c
+++ b/target-arm/translate.c
@@ -85,6 +85,7 @@ static TCGv_ptr cpu_env;
 /* We reuse the same 64-bit temporaries for efficiency.  */
 static TCGv_i64 cpu_V0, cpu_V1, cpu_M0;
 static TCGv_i32 cpu_R[16];
+static TCGv_i32 cpu_CF, cpu_NF, cpu_VF, cpu_ZF;
 static TCGv_i32 cpu_exclusive_addr;
 static TCGv_i32 cpu_exclusive_val;
 static TCGv_i32 cpu_exclusive_high;
@@ -115,6 +116,11 @@ void arm_translate_init(void)
                                           offsetof(CPUARMState, regs[i]),
                                           regnames[i]);
     }
+    cpu_CF = tcg_global_mem_new_i32(TCG_AREG0, offsetof(CPUARMState, CF), "CF");
+    cpu_NF = tcg_global_mem_new_i32(TCG_AREG0, offsetof(CPUARMState, NF), "NF");
+    cpu_VF = tcg_global_mem_new_i32(TCG_AREG0, offsetof(CPUARMState, VF), "VF");
+    cpu_ZF = tcg_global_mem_new_i32(TCG_AREG0, offsetof(CPUARMState, ZF), "ZF");
+
     cpu_exclusive_addr = tcg_global_mem_new_i32(TCG_AREG0,
         offsetof(CPUARMState, exclusive_addr), "exclusive_addr");
     cpu_exclusive_val = tcg_global_mem_new_i32(TCG_AREG0,
@@ -199,7 +205,7 @@ static void store_reg(DisasContext *s, int reg, TCGv var)
 static inline void gen_set_cpsr(TCGv var, uint32_t mask)
 {
     TCGv tmp_mask = tcg_const_i32(mask);
-    gen_helper_cpsr_write(var, tmp_mask);
+    gen_helper_cpsr_write(cpu_env, var, tmp_mask);
     tcg_temp_free_i32(tmp_mask);
 }
 /* Set NZCV flags from the high 4 bits of var.  */
@@ -209,7 +215,7 @@ static void gen_exception(int excp)
 {
     TCGv tmp = tcg_temp_new_i32();
     tcg_gen_movi_i32(tmp, excp);
-    gen_helper_exception(tmp);
+    gen_helper_exception(cpu_env, tmp);
     tcg_temp_free_i32(tmp);
 }
 
@@ -271,15 +277,6 @@ static void gen_sbfx(TCGv var, int shift, int width)
     }
 }
 
-/* Bitfield insertion.  Insert val into base.  Clobbers base and val.  */
-static void gen_bfi(TCGv dest, TCGv base, TCGv val, int shift, uint32_t mask)
-{
-    tcg_gen_andi_i32(val, val, mask);
-    tcg_gen_shli_i32(val, val, shift);
-    tcg_gen_andi_i32(base, base, ~(mask << shift));
-    tcg_gen_or_i32(dest, base, val);
-}
-
 /* Return (b << 32) + a. Mark inputs as dead */
 static TCGv_i64 gen_addq_msw(TCGv_i64 a, TCGv b)
 {
@@ -369,70 +366,122 @@ static void gen_add16(TCGv t0, TCGv t1)
     tcg_temp_free_i32(t1);
 }
 
-#define gen_set_CF(var) tcg_gen_st_i32(var, cpu_env, offsetof(CPUARMState, CF))
-
 /* Set CF to the top bit of var.  */
 static void gen_set_CF_bit31(TCGv var)
 {
-    TCGv tmp = tcg_temp_new_i32();
-    tcg_gen_shri_i32(tmp, var, 31);
-    gen_set_CF(tmp);
-    tcg_temp_free_i32(tmp);
+    tcg_gen_shri_i32(cpu_CF, var, 31);
 }
 
 /* Set N and Z flags from var.  */
 static inline void gen_logic_CC(TCGv var)
 {
-    tcg_gen_st_i32(var, cpu_env, offsetof(CPUARMState, NF));
-    tcg_gen_st_i32(var, cpu_env, offsetof(CPUARMState, ZF));
+    tcg_gen_mov_i32(cpu_NF, var);
+    tcg_gen_mov_i32(cpu_ZF, var);
 }
 
 /* T0 += T1 + CF.  */
 static void gen_adc(TCGv t0, TCGv t1)
 {
-    TCGv tmp;
     tcg_gen_add_i32(t0, t0, t1);
-    tmp = load_cpu_field(CF);
-    tcg_gen_add_i32(t0, t0, tmp);
-    tcg_temp_free_i32(tmp);
+    tcg_gen_add_i32(t0, t0, cpu_CF);
 }
 
 /* dest = T0 + T1 + CF. */
 static void gen_add_carry(TCGv dest, TCGv t0, TCGv t1)
 {
-    TCGv tmp;
     tcg_gen_add_i32(dest, t0, t1);
-    tmp = load_cpu_field(CF);
-    tcg_gen_add_i32(dest, dest, tmp);
-    tcg_temp_free_i32(tmp);
+    tcg_gen_add_i32(dest, dest, cpu_CF);
 }
 
 /* dest = T0 - T1 + CF - 1.  */
 static void gen_sub_carry(TCGv dest, TCGv t0, TCGv t1)
 {
-    TCGv tmp;
     tcg_gen_sub_i32(dest, t0, t1);
-    tmp = load_cpu_field(CF);
-    tcg_gen_add_i32(dest, dest, tmp);
+    tcg_gen_add_i32(dest, dest, cpu_CF);
     tcg_gen_subi_i32(dest, dest, 1);
+}
+
+/* dest = T0 + T1. Compute C, N, V and Z flags */
+static void gen_add_CC(TCGv dest, TCGv t0, TCGv t1)
+{
+    TCGv tmp;
+    tcg_gen_add_i32(cpu_NF, t0, t1);
+    tcg_gen_mov_i32(cpu_ZF, cpu_NF);
+    tcg_gen_setcond_i32(TCG_COND_LTU, cpu_CF, cpu_NF, t0);
+    tcg_gen_xor_i32(cpu_VF, cpu_NF, t0);
+    tmp = tcg_temp_new_i32();
+    tcg_gen_xor_i32(tmp, t0, t1);
+    tcg_gen_andc_i32(cpu_VF, cpu_VF, tmp);
     tcg_temp_free_i32(tmp);
+    tcg_gen_mov_i32(dest, cpu_NF);
 }
 
-/* FIXME:  Implement this natively.  */
-#define tcg_gen_abs_i32(t0, t1) gen_helper_abs(t0, t1)
+/* dest = T0 - T1. Compute C, N, V and Z flags */
+static void gen_sub_CC(TCGv dest, TCGv t0, TCGv t1)
+{
+    TCGv tmp;
+    tcg_gen_sub_i32(cpu_NF, t0, t1);
+    tcg_gen_mov_i32(cpu_ZF, cpu_NF);
+    tcg_gen_setcond_i32(TCG_COND_GEU, cpu_CF, t0, t1);
+    tcg_gen_xor_i32(cpu_VF, cpu_NF, t0);
+    tmp = tcg_temp_new_i32();
+    tcg_gen_xor_i32(tmp, t0, t1);
+    tcg_gen_and_i32(cpu_VF, cpu_VF, tmp);
+    tcg_temp_free_i32(tmp);
+    tcg_gen_mov_i32(dest, cpu_NF);
+}
 
-static void shifter_out_im(TCGv var, int shift)
+#define GEN_SHIFT(name)                                               \
+static void gen_##name(TCGv dest, TCGv t0, TCGv t1)                   \
+{                                                                     \
+    TCGv tmp1, tmp2, tmp3;                                            \
+    tmp1 = tcg_temp_new_i32();                                        \
+    tcg_gen_andi_i32(tmp1, t1, 0xff);                                 \
+    tmp2 = tcg_const_i32(0);                                          \
+    tmp3 = tcg_const_i32(0x1f);                                       \
+    tcg_gen_movcond_i32(TCG_COND_GTU, tmp2, tmp1, tmp3, tmp2, t0);    \
+    tcg_temp_free_i32(tmp3);                                          \
+    tcg_gen_andi_i32(tmp1, tmp1, 0x1f);                               \
+    tcg_gen_##name##_i32(dest, tmp2, tmp1);                           \
+    tcg_temp_free_i32(tmp2);                                          \
+    tcg_temp_free_i32(tmp1);                                          \
+}
+GEN_SHIFT(shl)
+GEN_SHIFT(shr)
+#undef GEN_SHIFT
+
+static void gen_sar(TCGv dest, TCGv t0, TCGv t1)
+{
+    TCGv tmp1, tmp2;
+    tmp1 = tcg_temp_new_i32();
+    tcg_gen_andi_i32(tmp1, t1, 0xff);
+    tmp2 = tcg_const_i32(0x1f);
+    tcg_gen_movcond_i32(TCG_COND_GTU, tmp1, tmp1, tmp2, tmp2, tmp1);
+    tcg_temp_free_i32(tmp2);
+    tcg_gen_sar_i32(dest, t0, tmp1);
+    tcg_temp_free_i32(tmp1);
+}
+
+static void tcg_gen_abs_i32(TCGv dest, TCGv src)
 {
+    TCGv c0 = tcg_const_i32(0);
     TCGv tmp = tcg_temp_new_i32();
+    tcg_gen_neg_i32(tmp, src);
+    tcg_gen_movcond_i32(TCG_COND_GT, dest, src, c0, src, tmp);
+    tcg_temp_free_i32(c0);
+    tcg_temp_free_i32(tmp);
+}
+
+static void shifter_out_im(TCGv var, int shift)
+{
     if (shift == 0) {
-        tcg_gen_andi_i32(tmp, var, 1);
+        tcg_gen_andi_i32(cpu_CF, var, 1);
     } else {
-        tcg_gen_shri_i32(tmp, var, shift);
-        if (shift != 31)
-            tcg_gen_andi_i32(tmp, tmp, 1);
+        tcg_gen_shri_i32(cpu_CF, var, shift);
+        if (shift != 31) {
+            tcg_gen_andi_i32(cpu_CF, cpu_CF, 1);
+        }
     }
-    gen_set_CF(tmp);
-    tcg_temp_free_i32(tmp);
 }
 
 /* Shift by immediate.  Includes special handling for shift == 0.  */
@@ -449,8 +498,7 @@ static inline void gen_arm_shift_im(TCGv var, int shiftop, int shift, int flags)
     case 1: /* LSR */
         if (shift == 0) {
             if (flags) {
-                tcg_gen_shri_i32(var, var, 31);
-                gen_set_CF(var);
+                tcg_gen_shri_i32(cpu_CF, var, 31);
             }
             tcg_gen_movi_i32(var, 0);
         } else {
@@ -474,11 +522,11 @@ static inline void gen_arm_shift_im(TCGv var, int shiftop, int shift, int flags)
                 shifter_out_im(var, shift - 1);
             tcg_gen_rotri_i32(var, var, shift); break;
         } else {
-            TCGv tmp = load_cpu_field(CF);
+            TCGv tmp = tcg_temp_new_i32();
+            tcg_gen_shli_i32(tmp, cpu_CF, 31);
             if (flags)
                 shifter_out_im(var, 0);
             tcg_gen_shri_i32(var, var, 1);
-            tcg_gen_shli_i32(tmp, tmp, 31);
             tcg_gen_or_i32(var, var, tmp);
             tcg_temp_free_i32(tmp);
         }
@@ -490,16 +538,22 @@ static inline void gen_arm_shift_reg(TCGv var, int shiftop,
 {
     if (flags) {
         switch (shiftop) {
-        case 0: gen_helper_shl_cc(var, var, shift); break;
-        case 1: gen_helper_shr_cc(var, var, shift); break;
-        case 2: gen_helper_sar_cc(var, var, shift); break;
-        case 3: gen_helper_ror_cc(var, var, shift); break;
+        case 0: gen_helper_shl_cc(var, cpu_env, var, shift); break;
+        case 1: gen_helper_shr_cc(var, cpu_env, var, shift); break;
+        case 2: gen_helper_sar_cc(var, cpu_env, var, shift); break;
+        case 3: gen_helper_ror_cc(var, cpu_env, var, shift); break;
         }
     } else {
         switch (shiftop) {
-        case 0: gen_helper_shl(var, var, shift); break;
-        case 1: gen_helper_shr(var, var, shift); break;
-        case 2: gen_helper_sar(var, var, shift); break;
+        case 0:
+            gen_shl(var, var, shift);
+            break;
+        case 1:
+            gen_shr(var, var, shift);
+            break;
+        case 2:
+            gen_sar(var, var, shift);
+            break;
         case 3: tcg_gen_andi_i32(shift, shift, 0x1f);
                 tcg_gen_rotr_i32(var, var, shift); break;
         }
@@ -603,99 +657,75 @@ static void gen_thumb2_parallel_addsub(int op1, int op2, TCGv a, TCGv b)
 static void gen_test_cc(int cc, int label)
 {
     TCGv tmp;
-    TCGv tmp2;
     int inv;
 
     switch (cc) {
     case 0: /* eq: Z */
-        tmp = load_cpu_field(ZF);
-        tcg_gen_brcondi_i32(TCG_COND_EQ, tmp, 0, label);
+        tcg_gen_brcondi_i32(TCG_COND_EQ, cpu_ZF, 0, label);
         break;
     case 1: /* ne: !Z */
-        tmp = load_cpu_field(ZF);
-        tcg_gen_brcondi_i32(TCG_COND_NE, tmp, 0, label);
+        tcg_gen_brcondi_i32(TCG_COND_NE, cpu_ZF, 0, label);
         break;
     case 2: /* cs: C */
-        tmp = load_cpu_field(CF);
-        tcg_gen_brcondi_i32(TCG_COND_NE, tmp, 0, label);
+        tcg_gen_brcondi_i32(TCG_COND_NE, cpu_CF, 0, label);
         break;
     case 3: /* cc: !C */
-        tmp = load_cpu_field(CF);
-        tcg_gen_brcondi_i32(TCG_COND_EQ, tmp, 0, label);
+        tcg_gen_brcondi_i32(TCG_COND_EQ, cpu_CF, 0, label);
         break;
     case 4: /* mi: N */
-        tmp = load_cpu_field(NF);
-        tcg_gen_brcondi_i32(TCG_COND_LT, tmp, 0, label);
+        tcg_gen_brcondi_i32(TCG_COND_LT, cpu_NF, 0, label);
         break;
     case 5: /* pl: !N */
-        tmp = load_cpu_field(NF);
-        tcg_gen_brcondi_i32(TCG_COND_GE, tmp, 0, label);
+        tcg_gen_brcondi_i32(TCG_COND_GE, cpu_NF, 0, label);
         break;
     case 6: /* vs: V */
-        tmp = load_cpu_field(VF);
-        tcg_gen_brcondi_i32(TCG_COND_LT, tmp, 0, label);
+        tcg_gen_brcondi_i32(TCG_COND_LT, cpu_VF, 0, label);
         break;
     case 7: /* vc: !V */
-        tmp = load_cpu_field(VF);
-        tcg_gen_brcondi_i32(TCG_COND_GE, tmp, 0, label);
+        tcg_gen_brcondi_i32(TCG_COND_GE, cpu_VF, 0, label);
         break;
     case 8: /* hi: C && !Z */
         inv = gen_new_label();
-        tmp = load_cpu_field(CF);
-        tcg_gen_brcondi_i32(TCG_COND_EQ, tmp, 0, inv);
-        tcg_temp_free_i32(tmp);
-        tmp = load_cpu_field(ZF);
-        tcg_gen_brcondi_i32(TCG_COND_NE, tmp, 0, label);
+        tcg_gen_brcondi_i32(TCG_COND_EQ, cpu_CF, 0, inv);
+        tcg_gen_brcondi_i32(TCG_COND_NE, cpu_ZF, 0, label);
         gen_set_label(inv);
         break;
     case 9: /* ls: !C || Z */
-        tmp = load_cpu_field(CF);
-        tcg_gen_brcondi_i32(TCG_COND_EQ, tmp, 0, label);
-        tcg_temp_free_i32(tmp);
-        tmp = load_cpu_field(ZF);
-        tcg_gen_brcondi_i32(TCG_COND_EQ, tmp, 0, label);
+        tcg_gen_brcondi_i32(TCG_COND_EQ, cpu_CF, 0, label);
+        tcg_gen_brcondi_i32(TCG_COND_EQ, cpu_ZF, 0, label);
         break;
     case 10: /* ge: N == V -> N ^ V == 0 */
-        tmp = load_cpu_field(VF);
-        tmp2 = load_cpu_field(NF);
-        tcg_gen_xor_i32(tmp, tmp, tmp2);
-        tcg_temp_free_i32(tmp2);
+        tmp = tcg_temp_new_i32();
+        tcg_gen_xor_i32(tmp, cpu_VF, cpu_NF);
         tcg_gen_brcondi_i32(TCG_COND_GE, tmp, 0, label);
+        tcg_temp_free_i32(tmp);
         break;
     case 11: /* lt: N != V -> N ^ V != 0 */
-        tmp = load_cpu_field(VF);
-        tmp2 = load_cpu_field(NF);
-        tcg_gen_xor_i32(tmp, tmp, tmp2);
-        tcg_temp_free_i32(tmp2);
+        tmp = tcg_temp_new_i32();
+        tcg_gen_xor_i32(tmp, cpu_VF, cpu_NF);
         tcg_gen_brcondi_i32(TCG_COND_LT, tmp, 0, label);
+        tcg_temp_free_i32(tmp);
         break;
     case 12: /* gt: !Z && N == V */
         inv = gen_new_label();
-        tmp = load_cpu_field(ZF);
-        tcg_gen_brcondi_i32(TCG_COND_EQ, tmp, 0, inv);
-        tcg_temp_free_i32(tmp);
-        tmp = load_cpu_field(VF);
-        tmp2 = load_cpu_field(NF);
-        tcg_gen_xor_i32(tmp, tmp, tmp2);
-        tcg_temp_free_i32(tmp2);
+        tcg_gen_brcondi_i32(TCG_COND_EQ, cpu_ZF, 0, inv);
+        tmp = tcg_temp_new_i32();
+        tcg_gen_xor_i32(tmp, cpu_VF, cpu_NF);
         tcg_gen_brcondi_i32(TCG_COND_GE, tmp, 0, label);
+        tcg_temp_free_i32(tmp);
         gen_set_label(inv);
         break;
     case 13: /* le: Z || N != V */
-        tmp = load_cpu_field(ZF);
-        tcg_gen_brcondi_i32(TCG_COND_EQ, tmp, 0, label);
-        tcg_temp_free_i32(tmp);
-        tmp = load_cpu_field(VF);
-        tmp2 = load_cpu_field(NF);
-        tcg_gen_xor_i32(tmp, tmp, tmp2);
-        tcg_temp_free_i32(tmp2);
+        tcg_gen_brcondi_i32(TCG_COND_EQ, cpu_ZF, 0, label);
+        tmp = tcg_temp_new_i32();
+        tcg_gen_xor_i32(tmp, cpu_VF, cpu_NF);
         tcg_gen_brcondi_i32(TCG_COND_LT, tmp, 0, label);
+        tcg_temp_free_i32(tmp);
         break;
     default:
         fprintf(stderr, "Bad condition code 0x%x\n", cc);
         abort();
     }
-    tcg_temp_free_i32(tmp);
 }
 
 static const uint8_t table_logic_cc[16] = {
@@ -2628,12 +2658,12 @@ static int disas_vfp_insn(CPUARMState * env, DisasContext *s, uint32_t insn)
                         switch (size) {
                         case 0:
                             tmp2 = neon_load_reg(rn, pass);
-                            gen_bfi(tmp, tmp2, tmp, offset, 0xff);
+                            tcg_gen_deposit_i32(tmp, tmp2, tmp, offset, 8);
                             tcg_temp_free_i32(tmp2);
                             break;
                         case 1:
                             tmp2 = neon_load_reg(rn, pass);
-                            gen_bfi(tmp, tmp2, tmp, offset, 0xffff);
+                            tcg_gen_deposit_i32(tmp, tmp2, tmp, offset, 16);
                             tcg_temp_free_i32(tmp2);
                             break;
                         case 2:
@@ -3989,7 +4019,8 @@ static int disas_neon_ls_insn(CPUARMState * env, DisasContext *s, uint32_t insn)
                     }
                     if (size != 2) {
                         tmp2 = neon_load_reg(rd, pass);
-                        gen_bfi(tmp, tmp2, tmp, shift, size ? 0xffff : 0xff);
+                        tcg_gen_deposit_i32(tmp, tmp2, tmp,
+                                            shift, size ? 16 : 8);
                         tcg_temp_free_i32(tmp2);
                     }
                     neon_store_reg(rd, pass, tmp);
@@ -4160,7 +4191,9 @@ static inline void gen_neon_negl(TCGv_i64 var, int size)
     switch (size) {
     case 0: gen_helper_neon_negl_u16(var, var); break;
     case 1: gen_helper_neon_negl_u32(var, var); break;
-    case 2: gen_helper_neon_negl_u64(var, var); break;
+    case 2:
+        tcg_gen_neg_i64(var, var);
+        break;
     default: abort();
     }
 }
@@ -6121,7 +6154,7 @@ static int disas_neon_data_insn(CPUARMState * env, DisasContext *s, uint32_t ins
                 tmp2 = neon_load_reg(rm, 0);
                 tmp4 = tcg_const_i32(rn);
                 tmp5 = tcg_const_i32(n);
-                gen_helper_neon_tbl(tmp2, tmp2, tmp, tmp4, tmp5);
+                gen_helper_neon_tbl(tmp2, cpu_env, tmp2, tmp, tmp4, tmp5);
                 tcg_temp_free_i32(tmp);
                 if (insn & (1 << 6)) {
                     tmp = neon_load_reg(rd, 1);
@@ -6130,7 +6163,7 @@ static int disas_neon_data_insn(CPUARMState * env, DisasContext *s, uint32_t ins
                     tcg_gen_movi_i32(tmp, 0);
                 }
                 tmp3 = neon_load_reg(rm, 1);
-                gen_helper_neon_tbl(tmp3, tmp3, tmp, tmp4, tmp5);
+                gen_helper_neon_tbl(tmp3, cpu_env, tmp3, tmp, tmp4, tmp5);
                 tcg_temp_free_i32(tmp5);
                 tcg_temp_free_i32(tmp4);
                 neon_store_reg(rd, 0, tmp2);
@@ -6534,7 +6567,7 @@ static void disas_arm_insn(CPUARMState * env, DisasContext *s)
     TCGv addr;
     TCGv_i64 tmp64;
 
-    insn = arm_ldl_code(s->pc, s->bswap_code);
+    insn = arm_ldl_code(env, s->pc, s->bswap_code);
     s->pc += 4;
 
     /* M variants do not implement ARM mode.  */
@@ -6818,7 +6851,7 @@ static void disas_arm_insn(CPUARMState * env, DisasContext *s)
                     tmp = load_cpu_field(spsr);
                 } else {
                     tmp = tcg_temp_new_i32();
-                    gen_helper_cpsr_read(tmp);
+                    gen_helper_cpsr_read(tmp, cpu_env);
                 }
                 store_reg(s, rd, tmp);
             }
@@ -6869,11 +6902,11 @@ static void disas_arm_insn(CPUARMState * env, DisasContext *s)
             tmp = load_reg(s, rm);
             tmp2 = load_reg(s, rn);
             if (op1 & 2)
-                gen_helper_double_saturate(tmp2, tmp2);
+                gen_helper_double_saturate(tmp2, cpu_env, tmp2);
             if (op1 & 1)
-                gen_helper_sub_saturate(tmp, tmp, tmp2);
+                gen_helper_sub_saturate(tmp, cpu_env, tmp, tmp2);
             else
-                gen_helper_add_saturate(tmp, tmp, tmp2);
+                gen_helper_add_saturate(tmp, cpu_env, tmp, tmp2);
             tcg_temp_free_i32(tmp2);
             store_reg(s, rd, tmp);
             break;
@@ -6911,7 +6944,7 @@ static void disas_arm_insn(CPUARMState * env, DisasContext *s)
                 tcg_temp_free_i64(tmp64);
                 if ((sh & 2) == 0) {
                     tmp2 = load_reg(s, rn);
-                    gen_helper_add_setq(tmp, tmp, tmp2);
+                    gen_helper_add_setq(tmp, cpu_env, tmp, tmp2);
                     tcg_temp_free_i32(tmp2);
                 }
                 store_reg(s, rd, tmp);
@@ -6931,7 +6964,7 @@ static void disas_arm_insn(CPUARMState * env, DisasContext *s)
                 } else {
                     if (op1 == 0) {
                         tmp2 = load_reg(s, rn);
-                        gen_helper_add_setq(tmp, tmp, tmp2);
+                        gen_helper_add_setq(tmp, cpu_env, tmp, tmp2);
                         tcg_temp_free_i32(tmp2);
                     }
                     store_reg(s, rd, tmp);
@@ -7005,11 +7038,11 @@ static void disas_arm_insn(CPUARMState * env, DisasContext *s)
                 if (IS_USER(s)) {
                     goto illegal_op;
                 }
-                gen_helper_sub_cc(tmp, tmp, tmp2);
+                gen_sub_CC(tmp, tmp, tmp2);
                 gen_exception_return(s, tmp);
             } else {
                 if (set_cc) {
-                    gen_helper_sub_cc(tmp, tmp, tmp2);
+                    gen_sub_CC(tmp, tmp, tmp2);
                 } else {
                     tcg_gen_sub_i32(tmp, tmp, tmp2);
                 }
@@ -7018,7 +7051,7 @@ static void disas_arm_insn(CPUARMState * env, DisasContext *s)
             break;
         case 0x03:
             if (set_cc) {
-                gen_helper_sub_cc(tmp, tmp2, tmp);
+                gen_sub_CC(tmp, tmp2, tmp);
             } else {
                 tcg_gen_sub_i32(tmp, tmp2, tmp);
             }
@@ -7026,7 +7059,7 @@ static void disas_arm_insn(CPUARMState * env, DisasContext *s)
             break;
         case 0x04:
             if (set_cc) {
-                gen_helper_add_cc(tmp, tmp, tmp2);
+                gen_add_CC(tmp, tmp, tmp2);
             } else {
                 tcg_gen_add_i32(tmp, tmp, tmp2);
             }
@@ -7034,7 +7067,7 @@ static void disas_arm_insn(CPUARMState * env, DisasContext *s)
             break;
         case 0x05:
             if (set_cc) {
-                gen_helper_adc_cc(tmp, tmp, tmp2);
+                gen_helper_adc_cc(tmp, cpu_env, tmp, tmp2);
             } else {
                 gen_add_carry(tmp, tmp, tmp2);
             }
@@ -7042,7 +7075,7 @@ static void disas_arm_insn(CPUARMState * env, DisasContext *s)
             break;
         case 0x06:
             if (set_cc) {
-                gen_helper_sbc_cc(tmp, tmp, tmp2);
+                gen_helper_sbc_cc(tmp, cpu_env, tmp, tmp2);
             } else {
                 gen_sub_carry(tmp, tmp, tmp2);
             }
@@ -7050,7 +7083,7 @@ static void disas_arm_insn(CPUARMState * env, DisasContext *s)
             break;
         case 0x07:
             if (set_cc) {
-                gen_helper_sbc_cc(tmp, tmp2, tmp);
+                gen_helper_sbc_cc(tmp, cpu_env, tmp2, tmp);
             } else {
                 gen_sub_carry(tmp, tmp2, tmp);
             }
@@ -7072,13 +7105,13 @@ static void disas_arm_insn(CPUARMState * env, DisasContext *s)
             break;
         case 0x0a:
             if (set_cc) {
-                gen_helper_sub_cc(tmp, tmp, tmp2);
+                gen_sub_CC(tmp, tmp, tmp2);
             }
             tcg_temp_free_i32(tmp);
             break;
         case 0x0b:
             if (set_cc) {
-                gen_helper_add_cc(tmp, tmp, tmp2);
+                gen_add_CC(tmp, tmp, tmp2);
             }
             tcg_temp_free_i32(tmp);
             break;
@@ -7395,9 +7428,9 @@ static void disas_arm_insn(CPUARMState * env, DisasContext *s)
                         sh = (insn >> 16) & 0x1f;
                         tmp2 = tcg_const_i32(sh);
                         if (insn & (1 << 22))
-                          gen_helper_usat(tmp, tmp, tmp2);
+                          gen_helper_usat(tmp, cpu_env, tmp, tmp2);
                         else
-                          gen_helper_ssat(tmp, tmp, tmp2);
+                          gen_helper_ssat(tmp, cpu_env, tmp, tmp2);
                         tcg_temp_free_i32(tmp2);
                         store_reg(s, rd, tmp);
                     } else if ((insn & 0x00300fe0) == 0x00200f20) {
@@ -7406,9 +7439,9 @@ static void disas_arm_insn(CPUARMState * env, DisasContext *s)
                         sh = (insn >> 16) & 0x1f;
                         tmp2 = tcg_const_i32(sh);
                         if (insn & (1 << 22))
-                          gen_helper_usat16(tmp, tmp, tmp2);
+                          gen_helper_usat16(tmp, cpu_env, tmp, tmp2);
                         else
-                          gen_helper_ssat16(tmp, tmp, tmp2);
+                          gen_helper_ssat16(tmp, cpu_env, tmp, tmp2);
                         tcg_temp_free_i32(tmp2);
                         store_reg(s, rd, tmp);
                     } else if ((insn & 0x00700fe0) == 0x00000fa0) {
@@ -7518,7 +7551,7 @@ static void disas_arm_insn(CPUARMState * env, DisasContext *s)
                              * however it may overflow considered as a signed
                              * operation, in which case we must set the Q flag.
                              */
-                            gen_helper_add_setq(tmp, tmp, tmp2);
+                            gen_helper_add_setq(tmp, cpu_env, tmp, tmp2);
                         }
                         tcg_temp_free_i32(tmp2);
                         if (insn & (1 << 22)) {
@@ -7534,7 +7567,7 @@ static void disas_arm_insn(CPUARMState * env, DisasContext *s)
                             if (rd != 15)
                               {
                                 tmp2 = load_reg(s, rd);
-                                gen_helper_add_setq(tmp, tmp, tmp2);
+                                gen_helper_add_setq(tmp, cpu_env, tmp, tmp2);
                                 tcg_temp_free_i32(tmp2);
                               }
                             store_reg(s, rn, tmp);
@@ -7593,7 +7626,7 @@ static void disas_arm_insn(CPUARMState * env, DisasContext *s)
                         }
                         if (i != 32) {
                             tmp2 = load_reg(s, rd);
-                            gen_bfi(tmp, tmp2, tmp, shift, (1u << i) - 1);
+                            tcg_gen_deposit_i32(tmp, tmp2, tmp, shift, i);
                             tcg_temp_free_i32(tmp2);
                         }
                         store_reg(s, rd, tmp);
@@ -7719,7 +7752,7 @@ static void disas_arm_insn(CPUARMState * env, DisasContext *s)
                             tmp = gen_ld32(addr, IS_USER(s));
                             if (user) {
                                 tmp2 = tcg_const_i32(i);
-                                gen_helper_set_user_reg(tmp2, tmp);
+                                gen_helper_set_user_reg(cpu_env, tmp2, tmp);
                                 tcg_temp_free_i32(tmp2);
                                 tcg_temp_free_i32(tmp);
                             } else if (i == rn) {
@@ -7738,7 +7771,7 @@ static void disas_arm_insn(CPUARMState * env, DisasContext *s)
                             } else if (user) {
                                 tmp = tcg_temp_new_i32();
                                 tmp2 = tcg_const_i32(i);
-                                gen_helper_get_user_reg(tmp, tmp2);
+                                gen_helper_get_user_reg(tmp, cpu_env, tmp2);
                                 tcg_temp_free_i32(tmp2);
                             } else {
                                 tmp = load_reg(s, i);
@@ -7865,31 +7898,31 @@ gen_thumb2_data_op(DisasContext *s, int op, int conds, uint32_t shifter_out, TCG
         break;
     case 8: /* add */
         if (conds)
-            gen_helper_add_cc(t0, t0, t1);
+            gen_add_CC(t0, t0, t1);
         else
             tcg_gen_add_i32(t0, t0, t1);
         break;
     case 10: /* adc */
         if (conds)
-            gen_helper_adc_cc(t0, t0, t1);
+            gen_helper_adc_cc(t0, cpu_env, t0, t1);
         else
             gen_adc(t0, t1);
         break;
     case 11: /* sbc */
         if (conds)
-            gen_helper_sbc_cc(t0, t0, t1);
+            gen_helper_sbc_cc(t0, cpu_env, t0, t1);
         else
             gen_sub_carry(t0, t0, t1);
         break;
     case 13: /* sub */
         if (conds)
-            gen_helper_sub_cc(t0, t0, t1);
+            gen_sub_CC(t0, t0, t1);
         else
             tcg_gen_sub_i32(t0, t0, t1);
         break;
     case 14: /* rsb */
         if (conds)
-            gen_helper_sub_cc(t0, t1, t0);
+            gen_sub_CC(t0, t1, t0);
         else
             tcg_gen_sub_i32(t0, t1, t0);
         break;
@@ -7962,7 +7995,7 @@ static int disas_thumb2_insn(CPUARMState *env, DisasContext *s, uint16_t insn_hw
         /* Fall through to 32-bit decode.  */
     }
 
-    insn = arm_lduw_code(s->pc, s->bswap_code);
+    insn = arm_lduw_code(env, s->pc, s->bswap_code);
     s->pc += 2;
     insn |= (uint32_t)insn_hw1 << 16;
 
@@ -8111,7 +8144,7 @@ static int disas_thumb2_insn(CPUARMState *env, DisasContext *s, uint16_t insn_hw
                     gen_st32(tmp, addr, 0);
                     tcg_gen_addi_i32(addr, addr, 4);
                     tmp = tcg_temp_new_i32();
-                    gen_helper_cpsr_read(tmp);
+                    gen_helper_cpsr_read(tmp, cpu_env);
                     gen_st32(tmp, addr, 0);
                     if (insn & (1 << 21)) {
                         if ((insn & (1 << 24)) == 0) {
@@ -8293,11 +8326,11 @@ static int disas_thumb2_insn(CPUARMState *env, DisasContext *s, uint16_t insn_hw
                 tmp = load_reg(s, rn);
                 tmp2 = load_reg(s, rm);
                 if (op & 1)
-                    gen_helper_double_saturate(tmp, tmp);
+                    gen_helper_double_saturate(tmp, cpu_env, tmp);
                 if (op & 2)
-                    gen_helper_sub_saturate(tmp, tmp2, tmp);
+                    gen_helper_sub_saturate(tmp, cpu_env, tmp2, tmp);
                 else
-                    gen_helper_add_saturate(tmp, tmp, tmp2);
+                    gen_helper_add_saturate(tmp, cpu_env, tmp, tmp2);
                 tcg_temp_free_i32(tmp2);
             } else {
                 tmp = load_reg(s, rn);
@@ -8353,7 +8386,7 @@ static int disas_thumb2_insn(CPUARMState *env, DisasContext *s, uint16_t insn_hw
                 tcg_temp_free_i32(tmp2);
                 if (rs != 15) {
                     tmp2 = load_reg(s, rs);
-                    gen_helper_add_setq(tmp, tmp, tmp2);
+                    gen_helper_add_setq(tmp, cpu_env, tmp, tmp2);
                     tcg_temp_free_i32(tmp2);
                 }
                 break;
@@ -8370,13 +8403,13 @@ static int disas_thumb2_insn(CPUARMState *env, DisasContext *s, uint16_t insn_hw
                      * however it may overflow considered as a signed
                      * operation, in which case we must set the Q flag.
                      */
-                    gen_helper_add_setq(tmp, tmp, tmp2);
+                    gen_helper_add_setq(tmp, cpu_env, tmp, tmp2);
                 }
                 tcg_temp_free_i32(tmp2);
                 if (rs != 15)
                   {
                     tmp2 = load_reg(s, rs);
-                    gen_helper_add_setq(tmp, tmp, tmp2);
+                    gen_helper_add_setq(tmp, cpu_env, tmp, tmp2);
                     tcg_temp_free_i32(tmp2);
                   }
                 break;
@@ -8393,7 +8426,7 @@ static int disas_thumb2_insn(CPUARMState *env, DisasContext *s, uint16_t insn_hw
                 if (rs != 15)
                   {
                     tmp2 = load_reg(s, rs);
-                    gen_helper_add_setq(tmp, tmp, tmp2);
+                    gen_helper_add_setq(tmp, cpu_env, tmp, tmp2);
                     tcg_temp_free_i32(tmp2);
                   }
                 break;
@@ -8632,7 +8665,7 @@ static int disas_thumb2_insn(CPUARMState *env, DisasContext *s, uint16_t insn_hw
                             gen_helper_v7m_mrs(tmp, cpu_env, addr);
                             tcg_temp_free_i32(addr);
                         } else {
-                            gen_helper_cpsr_read(tmp);
+                            gen_helper_cpsr_read(tmp, cpu_env);
                         }
                         store_reg(s, rd, tmp);
                         break;
@@ -8704,7 +8737,7 @@ static int disas_thumb2_insn(CPUARMState *env, DisasContext *s, uint16_t insn_hw
                         imm = imm + 1 - shift;
                         if (imm != 32) {
                             tmp2 = load_reg(s, rd);
-                            gen_bfi(tmp, tmp2, tmp, shift, (1u << imm) - 1);
+                            tcg_gen_deposit_i32(tmp, tmp2, tmp, shift, imm);
                             tcg_temp_free_i32(tmp2);
                         }
                         break;
@@ -8721,15 +8754,15 @@ static int disas_thumb2_insn(CPUARMState *env, DisasContext *s, uint16_t insn_hw
                         if (op & 4) {
                             /* Unsigned.  */
                             if ((op & 1) && shift == 0)
-                                gen_helper_usat16(tmp, tmp, tmp2);
+                                gen_helper_usat16(tmp, cpu_env, tmp, tmp2);
                             else
-                                gen_helper_usat(tmp, tmp, tmp2);
+                                gen_helper_usat(tmp, cpu_env, tmp, tmp2);
                         } else {
                             /* Signed.  */
                             if ((op & 1) && shift == 0)
-                                gen_helper_ssat16(tmp, tmp, tmp2);
+                                gen_helper_ssat16(tmp, cpu_env, tmp, tmp2);
                             else
-                                gen_helper_ssat(tmp, tmp, tmp2);
+                                gen_helper_ssat(tmp, cpu_env, tmp, tmp2);
                         }
                         tcg_temp_free_i32(tmp2);
                         break;
@@ -8992,7 +9025,7 @@ static void disas_thumb_insn(CPUARMState *env, DisasContext *s)
         }
     }
 
-    insn = arm_lduw_code(s->pc, s->bswap_code);
+    insn = arm_lduw_code(env, s->pc, s->bswap_code);
     s->pc += 2;
 
     switch (insn >> 12) {
@@ -9017,12 +9050,12 @@ static void disas_thumb_insn(CPUARMState *env, DisasContext *s)
                 if (s->condexec_mask)
                     tcg_gen_sub_i32(tmp, tmp, tmp2);
                 else
-                    gen_helper_sub_cc(tmp, tmp, tmp2);
+                    gen_sub_CC(tmp, tmp, tmp2);
             } else {
                 if (s->condexec_mask)
                     tcg_gen_add_i32(tmp, tmp, tmp2);
                 else
-                    gen_helper_add_cc(tmp, tmp, tmp2);
+                    gen_add_CC(tmp, tmp, tmp2);
             }
             tcg_temp_free_i32(tmp2);
             store_reg(s, rd, tmp);
@@ -9053,7 +9086,7 @@ static void disas_thumb_insn(CPUARMState *env, DisasContext *s)
             tcg_gen_movi_i32(tmp2, insn & 0xff);
             switch (op) {
             case 1: /* cmp */
-                gen_helper_sub_cc(tmp, tmp, tmp2);
+                gen_sub_CC(tmp, tmp, tmp2);
                 tcg_temp_free_i32(tmp);
                 tcg_temp_free_i32(tmp2);
                 break;
@@ -9061,7 +9094,7 @@ static void disas_thumb_insn(CPUARMState *env, DisasContext *s)
                 if (s->condexec_mask)
                     tcg_gen_add_i32(tmp, tmp, tmp2);
                 else
-                    gen_helper_add_cc(tmp, tmp, tmp2);
+                    gen_add_CC(tmp, tmp, tmp2);
                 tcg_temp_free_i32(tmp2);
                 store_reg(s, rd, tmp);
                 break;
@@ -9069,7 +9102,7 @@ static void disas_thumb_insn(CPUARMState *env, DisasContext *s)
                 if (s->condexec_mask)
                     tcg_gen_sub_i32(tmp, tmp, tmp2);
                 else
-                    gen_helper_sub_cc(tmp, tmp, tmp2);
+                    gen_sub_CC(tmp, tmp, tmp2);
                 tcg_temp_free_i32(tmp2);
                 store_reg(s, rd, tmp);
                 break;
@@ -9105,7 +9138,7 @@ static void disas_thumb_insn(CPUARMState *env, DisasContext *s)
             case 1: /* cmp */
                 tmp = load_reg(s, rd);
                 tmp2 = load_reg(s, rm);
-                gen_helper_sub_cc(tmp, tmp, tmp2);
+                gen_sub_CC(tmp, tmp, tmp2);
                 tcg_temp_free_i32(tmp2);
                 tcg_temp_free_i32(tmp);
                 break;
@@ -9166,25 +9199,25 @@ static void disas_thumb_insn(CPUARMState *env, DisasContext *s)
             break;
         case 0x2: /* lsl */
             if (s->condexec_mask) {
-                gen_helper_shl(tmp2, tmp2, tmp);
+                gen_shl(tmp2, tmp2, tmp);
             } else {
-                gen_helper_shl_cc(tmp2, tmp2, tmp);
+                gen_helper_shl_cc(tmp2, cpu_env, tmp2, tmp);
                 gen_logic_CC(tmp2);
             }
             break;
         case 0x3: /* lsr */
             if (s->condexec_mask) {
-                gen_helper_shr(tmp2, tmp2, tmp);
+                gen_shr(tmp2, tmp2, tmp);
             } else {
-                gen_helper_shr_cc(tmp2, tmp2, tmp);
+                gen_helper_shr_cc(tmp2, cpu_env, tmp2, tmp);
                 gen_logic_CC(tmp2);
             }
             break;
         case 0x4: /* asr */
             if (s->condexec_mask) {
-                gen_helper_sar(tmp2, tmp2, tmp);
+                gen_sar(tmp2, tmp2, tmp);
             } else {
-                gen_helper_sar_cc(tmp2, tmp2, tmp);
+                gen_helper_sar_cc(tmp2, cpu_env, tmp2, tmp);
                 gen_logic_CC(tmp2);
             }
             break;
@@ -9192,20 +9225,20 @@ static void disas_thumb_insn(CPUARMState *env, DisasContext *s)
             if (s->condexec_mask)
                 gen_adc(tmp, tmp2);
             else
-                gen_helper_adc_cc(tmp, tmp, tmp2);
+                gen_helper_adc_cc(tmp, cpu_env, tmp, tmp2);
             break;
         case 0x6: /* sbc */
             if (s->condexec_mask)
                 gen_sub_carry(tmp, tmp, tmp2);
             else
-                gen_helper_sbc_cc(tmp, tmp, tmp2);
+                gen_helper_sbc_cc(tmp, cpu_env, tmp, tmp2);
             break;
         case 0x7: /* ror */
             if (s->condexec_mask) {
                 tcg_gen_andi_i32(tmp, tmp, 0x1f);
                 tcg_gen_rotr_i32(tmp2, tmp2, tmp);
             } else {
-                gen_helper_ror_cc(tmp2, tmp2, tmp);
+                gen_helper_ror_cc(tmp2, cpu_env, tmp2, tmp);
                 gen_logic_CC(tmp2);
             }
             break;
@@ -9218,14 +9251,14 @@ static void disas_thumb_insn(CPUARMState *env, DisasContext *s)
             if (s->condexec_mask)
                 tcg_gen_neg_i32(tmp, tmp2);
             else
-                gen_helper_sub_cc(tmp, tmp, tmp2);
+                gen_sub_CC(tmp, tmp, tmp2);
             break;
         case 0xa: /* cmp */
-            gen_helper_sub_cc(tmp, tmp, tmp2);
+            gen_sub_CC(tmp, tmp, tmp2);
             rd = 16;
             break;
         case 0xb: /* cmn */
-            gen_helper_add_cc(tmp, tmp, tmp2);
+            gen_add_CC(tmp, tmp, tmp2);
             rd = 16;
             break;
         case 0xc: /* orr */
@@ -9694,7 +9727,7 @@ static inline void gen_intermediate_code_internal(CPUARMState *env,
 
     dc->tb = tb;
 
-    gen_opc_end = gen_opc_buf + OPC_MAX_SIZE;
+    gen_opc_end = tcg_ctx.gen_opc_buf + OPC_MAX_SIZE;
 
     dc->is_jmp = DISAS_NEXT;
     dc->pc = pc_start;
@@ -9801,7 +9834,7 @@ static inline void gen_intermediate_code_internal(CPUARMState *env,
             }
         }
         if (search_pc) {
-            j = gen_opc_ptr - gen_opc_buf;
+            j = tcg_ctx.gen_opc_ptr - tcg_ctx.gen_opc_buf;
             if (lj < j) {
                 lj++;
                 while (lj < j)
@@ -9816,7 +9849,7 @@ static inline void gen_intermediate_code_internal(CPUARMState *env,
         if (num_insns + 1 == max_insns && (tb->cflags & CF_LAST_IO))
             gen_io_start();
 
-        if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP))) {
+        if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
             tcg_gen_debug_insn_start(dc->pc);
         }
 
@@ -9848,7 +9881,7 @@ static inline void gen_intermediate_code_internal(CPUARMState *env,
          * Also stop translation when a page boundary is reached.  This
          * ensures prefetch aborts occur at the right place.  */
         num_insns ++;
-    } while (!dc->is_jmp && gen_opc_ptr < gen_opc_end &&
+    } while (!dc->is_jmp && tcg_ctx.gen_opc_ptr < gen_opc_end &&
              !env->singlestep_enabled &&
              !singlestep &&
              dc->pc < next_page_start &&
@@ -9913,7 +9946,7 @@ static inline void gen_intermediate_code_internal(CPUARMState *env,
             /* nothing more to generate */
             break;
         case DISAS_WFI:
-            gen_helper_wfi();
+            gen_helper_wfi(cpu_env);
             break;
         case DISAS_SWI:
             gen_exception(EXCP_SWI);
@@ -9929,19 +9962,19 @@ static inline void gen_intermediate_code_internal(CPUARMState *env,
 
 done_generating:
     gen_icount_end(tb, num_insns);
-    *gen_opc_ptr = INDEX_op_end;
+    *tcg_ctx.gen_opc_ptr = INDEX_op_end;
 
 #ifdef DEBUG_DISAS
     if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
         qemu_log("----------------\n");
         qemu_log("IN: %s\n", lookup_symbol(pc_start));
-        log_target_disas(pc_start, dc->pc - pc_start,
+        log_target_disas(env, pc_start, dc->pc - pc_start,
                          dc->thumb | (dc->bswap_code << 1));
         qemu_log("\n");
     }
 #endif
     if (search_pc) {
-        j = gen_opc_ptr - gen_opc_buf;
+        j = tcg_ctx.gen_opc_ptr - tcg_ctx.gen_opc_buf;
         lj++;
         while (lj <= j)
             gen_opc_instr_start[lj++] = 0;
@@ -9970,19 +10003,6 @@ void cpu_dump_state(CPUARMState *env, FILE *f, fprintf_function cpu_fprintf,
                     int flags)
 {
     int i;
-#if 0
-    union {
-        uint32_t i;
-        float s;
-    } s0, s1;
-    CPU_DoubleU d;
-    /* ??? This assumes float64 and double have the same layout.
-       Oh well, it's only debug dumps.  */
-    union {
-        float64 f64;
-        double d;
-    } d0;
-#endif
     uint32_t psr;
 
     for(i=0;i<16;i++) {
@@ -10002,20 +10022,23 @@ void cpu_dump_state(CPUARMState *env, FILE *f, fprintf_function cpu_fprintf,
                 psr & CPSR_T ? 'T' : 'A',
                 cpu_mode_names[psr & 0xf], (psr & 0x10) ? 32 : 26);
 
-#if 0
-    for (i = 0; i < 16; i++) {
-        d.d = env->vfp.regs[i];
-        s0.i = d.l.lower;
-        s1.i = d.l.upper;
-        d0.f64 = d.d;
-        cpu_fprintf(f, "s%02d=%08x(%8g) s%02d=%08x(%8g) d%02d=%08x%08x(%8g)\n",
-                    i * 2, (int)s0.i, s0.s,
-                    i * 2 + 1, (int)s1.i, s1.s,
-                    i, (int)(uint32_t)d.l.upper, (int)(uint32_t)d.l.lower,
-                    d0.d);
+    if (flags & CPU_DUMP_FPU) {
+        int numvfpregs = 0;
+        if (arm_feature(env, ARM_FEATURE_VFP)) {
+            numvfpregs += 16;
+        }
+        if (arm_feature(env, ARM_FEATURE_VFP3)) {
+            numvfpregs += 16;
+        }
+        for (i = 0; i < numvfpregs; i++) {
+            uint64_t v = float64_val(env->vfp.regs[i]);
+            cpu_fprintf(f, "s%02d=%08x s%02d=%08x d%02d=%016" PRIx64 "\n",
+                        i * 2, (uint32_t)v,
+                        i * 2 + 1, (uint32_t)(v >> 32),
+                        i, v);
+        }
+        cpu_fprintf(f, "FPSCR: %08x\n", (int)env->vfp.xregs[ARM_VFP_FPSCR]);
     }
-    cpu_fprintf(f, "FPSCR: %08x\n", (int)env->vfp.xregs[ARM_VFP_FPSCR]);
-#endif
 }
 
 void restore_state_to_opc(CPUARMState *env, TranslationBlock *tb, int pc_pos)
diff --git a/target-cris/Makefile.objs b/target-cris/Makefile.objs
index 4b09e8c6b..afb87bcc8 100644
--- a/target-cris/Makefile.objs
+++ b/target-cris/Makefile.objs
@@ -1,4 +1,2 @@
 obj-y += translate.o op_helper.o helper.o cpu.o
 obj-$(CONFIG_SOFTMMU) += mmu.o machine.o
-
-$(obj)/op_helper.o: QEMU_CFLAGS += $(HELPER_CFLAGS)
diff --git a/target-cris/cpu.h b/target-cris/cpu.h
index 4f4df6d9b..2c27506d0 100644
--- a/target-cris/cpu.h
+++ b/target-cris/cpu.h
@@ -285,8 +285,10 @@ static inline void cpu_get_tb_cpu_state(CPUCRISState *env, target_ulong *pc,
 #define cpu_list cris_cpu_list
 void cris_cpu_list(FILE *f, fprintf_function cpu_fprintf);
 
-static inline bool cpu_has_work(CPUCRISState *env)
+static inline bool cpu_has_work(CPUState *cpu)
 {
+    CPUCRISState *env = &CRIS_CPU(cpu)->env;
+
     return env->interrupt_request & (CPU_INTERRUPT_HARD | CPU_INTERRUPT_NMI);
 }
 
diff --git a/target-cris/helper.c b/target-cris/helper.c
index bfbc29ec6..324fe052f 100644
--- a/target-cris/helper.c
+++ b/target-cris/helper.c
@@ -151,7 +151,7 @@ static void do_interruptv10(CPUCRISState *env)
 	}
 
 	/* Now that we are in kernel mode, load the handlers address.  */
-	env->pc = ldl_code(env->pregs[PR_EBP] + ex_vec * 4);
+        env->pc = cpu_ldl_code(env, env->pregs[PR_EBP] + ex_vec * 4);
 	env->locked_irq = 1;
 	env->pregs[PR_CCS] |= F_FLAG_V10; /* set F.  */
 
@@ -233,7 +233,7 @@ void do_interrupt(CPUCRISState *env)
 	/* Now that we are in kernel mode, load the handlers address.
 	   This load may not fault, real hw leaves that behaviour as
 	   undefined.  */
-	env->pc = ldl_code(env->pregs[PR_EBP] + ex_vec * 4);
+        env->pc = cpu_ldl_code(env, env->pregs[PR_EBP] + ex_vec * 4);
 
 	/* Clear the excption_index to avoid spurios hw_aborts for recursive
 	   bus faults.  */
@@ -246,7 +246,7 @@ void do_interrupt(CPUCRISState *env)
 		   env->pregs[PR_ERP]);
 }
 
-target_phys_addr_t cpu_get_phys_page_debug(CPUCRISState * env, target_ulong addr)
+hwaddr cpu_get_phys_page_debug(CPUCRISState * env, target_ulong addr)
 {
 	uint32_t phy = addr;
 	struct cris_mmu_result res;
diff --git a/target-cris/helper.h b/target-cris/helper.h
index 093063a14..fe12083a1 100644
--- a/target-cris/helper.h
+++ b/target-cris/helper.h
@@ -1,26 +1,29 @@
 #include "def-helper.h"
 
-DEF_HELPER_1(raise_exception, void, i32)
-DEF_HELPER_1(tlb_flush_pid, void, i32)
-DEF_HELPER_1(spc_write, void, i32)
+DEF_HELPER_2(raise_exception, void, env, i32)
+DEF_HELPER_2(tlb_flush_pid, void, env, i32)
+DEF_HELPER_2(spc_write, void, env, i32)
 DEF_HELPER_3(dump, void, i32, i32, i32)
-DEF_HELPER_0(rfe, void);
-DEF_HELPER_0(rfn, void);
+DEF_HELPER_1(rfe, void, env);
+DEF_HELPER_1(rfn, void, env);
 
-DEF_HELPER_2(movl_sreg_reg, void, i32, i32)
-DEF_HELPER_2(movl_reg_sreg, void, i32, i32)
+DEF_HELPER_3(movl_sreg_reg, void, env, i32, i32)
+DEF_HELPER_3(movl_reg_sreg, void, env, i32, i32)
 
-DEF_HELPER_FLAGS_1(lz, TCG_CALL_PURE, i32, i32);
-DEF_HELPER_FLAGS_3(btst, TCG_CALL_PURE, i32, i32, i32, i32);
+DEF_HELPER_FLAGS_1(lz, TCG_CALL_NO_SE, i32, i32);
+DEF_HELPER_FLAGS_4(btst, TCG_CALL_NO_SE, i32, env, i32, i32, i32);
 
-DEF_HELPER_FLAGS_3(evaluate_flags_muls, TCG_CALL_PURE, i32, i32, i32, i32)
-DEF_HELPER_FLAGS_3(evaluate_flags_mulu, TCG_CALL_PURE, i32, i32, i32, i32)
-DEF_HELPER_FLAGS_4(evaluate_flags_mcp, TCG_CALL_PURE, i32, i32, i32, i32, i32)
-DEF_HELPER_FLAGS_4(evaluate_flags_alu_4, TCG_CALL_PURE, i32, i32, i32, i32, i32)
-DEF_HELPER_FLAGS_4(evaluate_flags_sub_4, TCG_CALL_PURE, i32, i32, i32, i32, i32)
-DEF_HELPER_FLAGS_2(evaluate_flags_move_4, TCG_CALL_PURE, i32, i32, i32)
-DEF_HELPER_FLAGS_2(evaluate_flags_move_2, TCG_CALL_PURE, i32, i32, i32)
-DEF_HELPER_0(evaluate_flags, void)
-DEF_HELPER_0(top_evaluate_flags, void)
+DEF_HELPER_FLAGS_4(evaluate_flags_muls, TCG_CALL_NO_SE, i32, env, i32, i32, i32)
+DEF_HELPER_FLAGS_4(evaluate_flags_mulu, TCG_CALL_NO_SE, i32, env, i32, i32, i32)
+DEF_HELPER_FLAGS_5(evaluate_flags_mcp, TCG_CALL_NO_SE, i32, env,
+                                                      i32, i32, i32, i32)
+DEF_HELPER_FLAGS_5(evaluate_flags_alu_4, TCG_CALL_NO_SE, i32, env,
+                                                        i32, i32, i32, i32)
+DEF_HELPER_FLAGS_5(evaluate_flags_sub_4, TCG_CALL_NO_SE, i32, env,
+                                                        i32, i32, i32, i32)
+DEF_HELPER_FLAGS_3(evaluate_flags_move_4, TCG_CALL_NO_SE, i32, env, i32, i32)
+DEF_HELPER_FLAGS_3(evaluate_flags_move_2, TCG_CALL_NO_SE, i32, env, i32, i32)
+DEF_HELPER_1(evaluate_flags, void, env)
+DEF_HELPER_1(top_evaluate_flags, void, env)
 
 #include "def-helper.h"
diff --git a/target-cris/op_helper.c b/target-cris/op_helper.c
index ac7c98c8e..a7468d41c 100644
--- a/target-cris/op_helper.c
+++ b/target-cris/op_helper.c
@@ -19,7 +19,6 @@
  */
 
 #include "cpu.h"
-#include "dyngen-exec.h"
 #include "mmu.h"
 #include "helper.h"
 #include "host-utils.h"
@@ -55,17 +54,12 @@
 /* Try to fill the TLB and return an exception if error. If retaddr is
    NULL, it means that the function was called in C code (i.e. not
    from generated code or from helper.c) */
-/* XXX: fix it to restore all registers */
-void tlb_fill(CPUCRISState *env1, target_ulong addr, int is_write, int mmu_idx,
+void tlb_fill(CPUCRISState *env, target_ulong addr, int is_write, int mmu_idx,
               uintptr_t retaddr)
 {
     TranslationBlock *tb;
-    CPUCRISState *saved_env;
     int ret;
 
-    saved_env = env;
-    env = env1;
-
     D_LOG("%s pc=%x tpc=%x ra=%p\n", __func__,
           env->pc, env->debug1, (void *)retaddr);
     ret = cpu_cris_handle_mmu_fault(env, addr, is_write, mmu_idx);
@@ -79,23 +73,22 @@ void tlb_fill(CPUCRISState *env1, target_ulong addr, int is_write, int mmu_idx,
                 cpu_restore_state(tb, env, retaddr);
 
 		/* Evaluate flags after retranslation.  */
-                helper_top_evaluate_flags();
+                helper_top_evaluate_flags(env);
             }
         }
         cpu_loop_exit(env);
     }
-    env = saved_env;
 }
 
 #endif
 
-void helper_raise_exception(uint32_t index)
+void helper_raise_exception(CPUCRISState *env, uint32_t index)
 {
 	env->exception_index = index;
         cpu_loop_exit(env);
 }
 
-void helper_tlb_flush_pid(uint32_t pid)
+void helper_tlb_flush_pid(CPUCRISState *env, uint32_t pid)
 {
 #if !defined(CONFIG_USER_ONLY)
 	pid &= 0xff;
@@ -104,7 +97,7 @@ void helper_tlb_flush_pid(uint32_t pid)
 #endif
 }
 
-void helper_spc_write(uint32_t new_spc)
+void helper_spc_write(CPUCRISState *env, uint32_t new_spc)
 {
 #if !defined(CONFIG_USER_ONLY)
 	tlb_flush_page(env, env->pregs[PR_SPC]);
@@ -121,7 +114,7 @@ void helper_dump(uint32_t a0, uint32_t a1, uint32_t a2)
 #define EXTRACT_FIELD(src, start, end) \
 	    (((src) >> start) & ((1 << (end - start + 1)) - 1))
 
-void helper_movl_sreg_reg (uint32_t sreg, uint32_t reg)
+void helper_movl_sreg_reg(CPUCRISState *env, uint32_t sreg, uint32_t reg)
 {
 	uint32_t srs;
 	srs = env->pregs[PR_SRS];
@@ -171,7 +164,7 @@ void helper_movl_sreg_reg (uint32_t sreg, uint32_t reg)
 #endif
 }
 
-void helper_movl_reg_sreg (uint32_t reg, uint32_t sreg)
+void helper_movl_reg_sreg(CPUCRISState *env, uint32_t reg, uint32_t sreg)
 {
 	uint32_t srs;
 	env->pregs[PR_SRS] &= 3;
@@ -216,7 +209,7 @@ static void cris_ccs_rshift(CPUCRISState *env)
 	env->pregs[PR_CCS] = ccs;
 }
 
-void helper_rfe(void)
+void helper_rfe(CPUCRISState *env)
 {
 	int rflag = env->pregs[PR_CCS] & R_FLAG;
 
@@ -232,7 +225,7 @@ void helper_rfe(void)
 		env->pregs[PR_CCS] |= P_FLAG;
 }
 
-void helper_rfn(void)
+void helper_rfn(CPUCRISState *env)
 {
 	int rflag = env->pregs[PR_CCS] & R_FLAG;
 
@@ -256,7 +249,7 @@ uint32_t helper_lz(uint32_t t0)
 	return clz32(t0);
 }
 
-uint32_t helper_btst(uint32_t t0, uint32_t t1, uint32_t ccs)
+uint32_t helper_btst(CPUCRISState *env, uint32_t t0, uint32_t t1, uint32_t ccs)
 {
 	/* FIXME: clean this up.  */
 
@@ -284,7 +277,8 @@ uint32_t helper_btst(uint32_t t0, uint32_t t1, uint32_t ccs)
 	return ccs;
 }
 
-static inline uint32_t evaluate_flags_writeback(uint32_t flags, uint32_t ccs)
+static inline uint32_t evaluate_flags_writeback(CPUCRISState *env,
+                                                uint32_t flags, uint32_t ccs)
 {
 	unsigned int x, z, mask;
 
@@ -303,7 +297,8 @@ static inline uint32_t evaluate_flags_writeback(uint32_t flags, uint32_t ccs)
 	return ccs;
 }
 
-uint32_t helper_evaluate_flags_muls(uint32_t ccs, uint32_t res, uint32_t mof)
+uint32_t helper_evaluate_flags_muls(CPUCRISState *env,
+                                    uint32_t ccs, uint32_t res, uint32_t mof)
 {
 	uint32_t flags = 0;
 	int64_t tmp;
@@ -321,10 +316,11 @@ uint32_t helper_evaluate_flags_muls(uint32_t ccs, uint32_t res, uint32_t mof)
 	if ((dneg && mof != -1)
 	    || (!dneg && mof != 0))
 		flags |= V_FLAG;
-	return evaluate_flags_writeback(flags, ccs);
+        return evaluate_flags_writeback(env, flags, ccs);
 }
 
-uint32_t helper_evaluate_flags_mulu(uint32_t ccs, uint32_t res, uint32_t mof)
+uint32_t helper_evaluate_flags_mulu(CPUCRISState *env,
+                                    uint32_t ccs, uint32_t res, uint32_t mof)
 {
 	uint32_t flags = 0;
 	uint64_t tmp;
@@ -339,10 +335,10 @@ uint32_t helper_evaluate_flags_mulu(uint32_t ccs, uint32_t res, uint32_t mof)
 	if (mof)
 		flags |= V_FLAG;
 
-	return evaluate_flags_writeback(flags, ccs);
+        return evaluate_flags_writeback(env, flags, ccs);
 }
 
-uint32_t helper_evaluate_flags_mcp(uint32_t ccs,
+uint32_t helper_evaluate_flags_mcp(CPUCRISState *env, uint32_t ccs,
 				   uint32_t src, uint32_t dst, uint32_t res)
 {
 	uint32_t flags = 0;
@@ -368,10 +364,10 @@ uint32_t helper_evaluate_flags_mcp(uint32_t ccs,
 			flags |= R_FLAG;
 	}
 
-	return evaluate_flags_writeback(flags, ccs);
+        return evaluate_flags_writeback(env, flags, ccs);
 }
 
-uint32_t helper_evaluate_flags_alu_4(uint32_t ccs,
+uint32_t helper_evaluate_flags_alu_4(CPUCRISState *env, uint32_t ccs,
 				     uint32_t src, uint32_t dst, uint32_t res)
 {
 	uint32_t flags = 0;
@@ -397,10 +393,10 @@ uint32_t helper_evaluate_flags_alu_4(uint32_t ccs,
 			flags |= C_FLAG;
 	}
 
-	return evaluate_flags_writeback(flags, ccs);
+        return evaluate_flags_writeback(env, flags, ccs);
 }
 
-uint32_t helper_evaluate_flags_sub_4(uint32_t ccs,
+uint32_t helper_evaluate_flags_sub_4(CPUCRISState *env, uint32_t ccs,
 				     uint32_t src, uint32_t dst, uint32_t res)
 {
 	uint32_t flags = 0;
@@ -427,10 +423,11 @@ uint32_t helper_evaluate_flags_sub_4(uint32_t ccs,
 	}
 
 	flags ^= C_FLAG;
-	return evaluate_flags_writeback(flags, ccs);
+        return evaluate_flags_writeback(env, flags, ccs);
 }
 
-uint32_t helper_evaluate_flags_move_4(uint32_t ccs, uint32_t res)
+uint32_t helper_evaluate_flags_move_4(CPUCRISState *env,
+                                      uint32_t ccs, uint32_t res)
 {
 	uint32_t flags = 0;
 
@@ -439,9 +436,10 @@ uint32_t helper_evaluate_flags_move_4(uint32_t ccs, uint32_t res)
 	else if (res == 0L)
 		flags |= Z_FLAG;
 
-	return evaluate_flags_writeback(flags, ccs);
+        return evaluate_flags_writeback(env, flags, ccs);
 }
-uint32_t helper_evaluate_flags_move_2(uint32_t ccs, uint32_t res)
+uint32_t helper_evaluate_flags_move_2(CPUCRISState *env,
+                                      uint32_t ccs, uint32_t res)
 {
 	uint32_t flags = 0;
 
@@ -450,12 +448,12 @@ uint32_t helper_evaluate_flags_move_2(uint32_t ccs, uint32_t res)
 	else if (res == 0)
 		flags |= Z_FLAG;
 
-	return evaluate_flags_writeback(flags, ccs);
+        return evaluate_flags_writeback(env, flags, ccs);
 }
 
 /* TODO: This is expensive. We could split things up and only evaluate part of
    CCR on a need to know basis. For now, we simply re-evaluate everything.  */
-void  helper_evaluate_flags(void)
+void helper_evaluate_flags(CPUCRISState *env)
 {
 	uint32_t src, dst, res;
 	uint32_t flags = 0;
@@ -571,25 +569,26 @@ void  helper_evaluate_flags(void)
 	if (env->cc_op == CC_OP_SUB || env->cc_op == CC_OP_CMP)
 		flags ^= C_FLAG;
 
-	env->pregs[PR_CCS] = evaluate_flags_writeback(flags, env->pregs[PR_CCS]);
+        env->pregs[PR_CCS] = evaluate_flags_writeback(env, flags,
+                                                      env->pregs[PR_CCS]);
 }
 
-void helper_top_evaluate_flags(void)
+void helper_top_evaluate_flags(CPUCRISState *env)
 {
 	switch (env->cc_op)
 	{
 		case CC_OP_MCP:
-			env->pregs[PR_CCS] = helper_evaluate_flags_mcp(
+                        env->pregs[PR_CCS] = helper_evaluate_flags_mcp(env,
 					env->pregs[PR_CCS], env->cc_src,
 					env->cc_dest, env->cc_result);
 			break;
 		case CC_OP_MULS:
-			env->pregs[PR_CCS] = helper_evaluate_flags_muls(
+                        env->pregs[PR_CCS] = helper_evaluate_flags_muls(env,
 					env->pregs[PR_CCS], env->cc_result,
 					env->pregs[PR_MOF]);
 			break;
 		case CC_OP_MULU:
-			env->pregs[PR_CCS] = helper_evaluate_flags_mulu(
+                        env->pregs[PR_CCS] = helper_evaluate_flags_mulu(env,
 					env->pregs[PR_CCS], env->cc_result,
 					env->pregs[PR_MOF]);
 			break;
@@ -604,18 +603,18 @@ void helper_top_evaluate_flags(void)
 		{
 			case 4:
 				env->pregs[PR_CCS] =
-					helper_evaluate_flags_move_4(
+                                        helper_evaluate_flags_move_4(env,
 							env->pregs[PR_CCS],
 							env->cc_result);
 				break;
 			case 2:
 				env->pregs[PR_CCS] =
-					helper_evaluate_flags_move_2(
+                                        helper_evaluate_flags_move_2(env,
 							env->pregs[PR_CCS],
 							env->cc_result);
 				break;
 			default:
-				helper_evaluate_flags();
+                                helper_evaluate_flags(env);
 				break;
 		}
 		break;
@@ -626,12 +625,12 @@ void helper_top_evaluate_flags(void)
 		case CC_OP_CMP:
 			if (env->cc_size == 4)
 				env->pregs[PR_CCS] =
-					helper_evaluate_flags_sub_4(
+                                        helper_evaluate_flags_sub_4(env,
 						env->pregs[PR_CCS],
 						env->cc_src, env->cc_dest,
 						env->cc_result);
 			else
-				helper_evaluate_flags();
+                                helper_evaluate_flags(env);
 			break;
 		default:
 		{
@@ -639,13 +638,13 @@ void helper_top_evaluate_flags(void)
 			{
 			case 4:
 				env->pregs[PR_CCS] =
-					helper_evaluate_flags_alu_4(
+                                        helper_evaluate_flags_alu_4(env,
 						env->pregs[PR_CCS],
 						env->cc_src, env->cc_dest,
 						env->cc_result);
 				break;
 			default:
-				helper_evaluate_flags();
+                                helper_evaluate_flags(env);
 				break;
 			}
 		}
diff --git a/target-cris/translate.c b/target-cris/translate.c
index 1ad9ec788..0b0e86dbd 100644
--- a/target-cris/translate.c
+++ b/target-cris/translate.c
@@ -74,89 +74,89 @@ static TCGv env_pc;
 
 /* This is the state at translation time.  */
 typedef struct DisasContext {
-	CPUCRISState *env;
-	target_ulong pc, ppc;
-
-	/* Decoder.  */
-	unsigned int (*decoder)(struct DisasContext *dc);
-	uint32_t ir;
-	uint32_t opcode;
-	unsigned int op1;
-	unsigned int op2;
-	unsigned int zsize, zzsize;
-	unsigned int mode;
-	unsigned int postinc;
-
-	unsigned int size;
-	unsigned int src;
-	unsigned int dst;
-	unsigned int cond;
-
-	int update_cc;
-	int cc_op;
-	int cc_size;
-	uint32_t cc_mask;
-
-	int cc_size_uptodate; /* -1 invalid or last written value.  */
-
-	int cc_x_uptodate;  /* 1 - ccs, 2 - known | X_FLAG. 0 not uptodate.  */
-	int flags_uptodate; /* Wether or not $ccs is uptodate.  */
-	int flagx_known; /* Wether or not flags_x has the x flag known at
-			    translation time.  */
-	int flags_x;
-
-	int clear_x; /* Clear x after this insn?  */
-	int clear_prefix; /* Clear prefix after this insn?  */
-	int clear_locked_irq; /* Clear the irq lockout.  */
-	int cpustate_changed;
-	unsigned int tb_flags; /* tb dependent flags.  */
-	int is_jmp;
+    CPUCRISState *env;
+    target_ulong pc, ppc;
+
+    /* Decoder.  */
+        unsigned int (*decoder)(CPUCRISState *env, struct DisasContext *dc);
+    uint32_t ir;
+    uint32_t opcode;
+    unsigned int op1;
+    unsigned int op2;
+    unsigned int zsize, zzsize;
+    unsigned int mode;
+    unsigned int postinc;
+
+    unsigned int size;
+    unsigned int src;
+    unsigned int dst;
+    unsigned int cond;
+
+    int update_cc;
+    int cc_op;
+    int cc_size;
+    uint32_t cc_mask;
+
+    int cc_size_uptodate; /* -1 invalid or last written value.  */
+
+    int cc_x_uptodate;  /* 1 - ccs, 2 - known | X_FLAG. 0 not uptodate.  */
+    int flags_uptodate; /* Wether or not $ccs is uptodate.  */
+    int flagx_known; /* Wether or not flags_x has the x flag known at
+                translation time.  */
+    int flags_x;
+
+    int clear_x; /* Clear x after this insn?  */
+    int clear_prefix; /* Clear prefix after this insn?  */
+    int clear_locked_irq; /* Clear the irq lockout.  */
+    int cpustate_changed;
+    unsigned int tb_flags; /* tb dependent flags.  */
+    int is_jmp;
 
 #define JMP_NOJMP     0
 #define JMP_DIRECT    1
 #define JMP_DIRECT_CC 2
 #define JMP_INDIRECT  3
-	int jmp; /* 0=nojmp, 1=direct, 2=indirect.  */ 
-	uint32_t jmp_pc;
+    int jmp; /* 0=nojmp, 1=direct, 2=indirect.  */
+    uint32_t jmp_pc;
 
-	int delayed_branch;
+    int delayed_branch;
 
-	struct TranslationBlock *tb;
-	int singlestep_enabled;
+    struct TranslationBlock *tb;
+    int singlestep_enabled;
 } DisasContext;
 
 static void gen_BUG(DisasContext *dc, const char *file, int line)
 {
-	printf ("BUG: pc=%x %s %d\n", dc->pc, file, line);
-	qemu_log("BUG: pc=%x %s %d\n", dc->pc, file, line);
-	cpu_abort(dc->env, "%s:%d\n", file, line);
+    printf("BUG: pc=%x %s %d\n", dc->pc, file, line);
+    qemu_log("BUG: pc=%x %s %d\n", dc->pc, file, line);
+    cpu_abort(dc->env, "%s:%d\n", file, line);
 }
 
 static const char *regnames[] =
 {
-	"$r0", "$r1", "$r2", "$r3",
-	"$r4", "$r5", "$r6", "$r7",
-	"$r8", "$r9", "$r10", "$r11",
-	"$r12", "$r13", "$sp", "$acr",
+    "$r0", "$r1", "$r2", "$r3",
+    "$r4", "$r5", "$r6", "$r7",
+    "$r8", "$r9", "$r10", "$r11",
+    "$r12", "$r13", "$sp", "$acr",
 };
 static const char *pregnames[] =
 {
-	"$bz", "$vr", "$pid", "$srs",
-	"$wz", "$exs", "$eda", "$mof",
-	"$dz", "$ebp", "$erp", "$srp",
-	"$nrp", "$ccs", "$usp", "$spc",
+    "$bz", "$vr", "$pid", "$srs",
+    "$wz", "$exs", "$eda", "$mof",
+    "$dz", "$ebp", "$erp", "$srp",
+    "$nrp", "$ccs", "$usp", "$spc",
 };
 
 /* We need this table to handle preg-moves with implicit width.  */
 static int preg_sizes[] = {
-	1, /* bz.  */
-	1, /* vr.  */
-	4, /* pid.  */
-	1, /* srs.  */
-	2, /* wz.  */
-	4, 4, 4,
-	4, 4, 4, 4,
-	4, 4, 4, 4,
+    1, /* bz.  */
+    1, /* vr.  */
+    4, /* pid.  */
+    1, /* srs.  */
+    2, /* wz.  */
+    4, 4, 4,
+    4, 4, 4, 4,
+    4, 4, 4, 4,
 };
 
 #define t_gen_mov_TN_env(tn, member) \
@@ -166,358 +166,368 @@ static int preg_sizes[] = {
 
 static inline void t_gen_mov_TN_reg(TCGv tn, int r)
 {
-	if (r < 0 || r > 15)
-		fprintf(stderr, "wrong register read $r%d\n", r);
-	tcg_gen_mov_tl(tn, cpu_R[r]);
+    if (r < 0 || r > 15) {
+        fprintf(stderr, "wrong register read $r%d\n", r);
+    }
+    tcg_gen_mov_tl(tn, cpu_R[r]);
 }
 static inline void t_gen_mov_reg_TN(int r, TCGv tn)
 {
-	if (r < 0 || r > 15)
-		fprintf(stderr, "wrong register write $r%d\n", r);
-	tcg_gen_mov_tl(cpu_R[r], tn);
+    if (r < 0 || r > 15) {
+        fprintf(stderr, "wrong register write $r%d\n", r);
+    }
+    tcg_gen_mov_tl(cpu_R[r], tn);
 }
 
 static inline void _t_gen_mov_TN_env(TCGv tn, int offset)
 {
-	if (offset > sizeof (CPUCRISState))
-		fprintf(stderr, "wrong load from env from off=%d\n", offset);
-	tcg_gen_ld_tl(tn, cpu_env, offset);
+    if (offset > sizeof(CPUCRISState)) {
+        fprintf(stderr, "wrong load from env from off=%d\n", offset);
+    }
+    tcg_gen_ld_tl(tn, cpu_env, offset);
 }
 static inline void _t_gen_mov_env_TN(int offset, TCGv tn)
 {
-	if (offset > sizeof (CPUCRISState))
-		fprintf(stderr, "wrong store to env at off=%d\n", offset);
-	tcg_gen_st_tl(tn, cpu_env, offset);
+    if (offset > sizeof(CPUCRISState)) {
+        fprintf(stderr, "wrong store to env at off=%d\n", offset);
+    }
+    tcg_gen_st_tl(tn, cpu_env, offset);
 }
 
 static inline void t_gen_mov_TN_preg(TCGv tn, int r)
 {
-	if (r < 0 || r > 15)
-		fprintf(stderr, "wrong register read $p%d\n", r);
-	if (r == PR_BZ || r == PR_WZ || r == PR_DZ)
-		tcg_gen_mov_tl(tn, tcg_const_tl(0));
-	else if (r == PR_VR)
-		tcg_gen_mov_tl(tn, tcg_const_tl(32));
-	else
-		tcg_gen_mov_tl(tn, cpu_PR[r]);
+    if (r < 0 || r > 15) {
+        fprintf(stderr, "wrong register read $p%d\n", r);
+    }
+    if (r == PR_BZ || r == PR_WZ || r == PR_DZ) {
+        tcg_gen_mov_tl(tn, tcg_const_tl(0));
+    } else if (r == PR_VR) {
+        tcg_gen_mov_tl(tn, tcg_const_tl(32));
+    } else {
+        tcg_gen_mov_tl(tn, cpu_PR[r]);
+    }
 }
 static inline void t_gen_mov_preg_TN(DisasContext *dc, int r, TCGv tn)
 {
-	if (r < 0 || r > 15)
-		fprintf(stderr, "wrong register write $p%d\n", r);
-	if (r == PR_BZ || r == PR_WZ || r == PR_DZ)
-		return;
-	else if (r == PR_SRS)
-		tcg_gen_andi_tl(cpu_PR[r], tn, 3);
-	else {
-		if (r == PR_PID) 
-			gen_helper_tlb_flush_pid(tn);
-		if (dc->tb_flags & S_FLAG && r == PR_SPC) 
-			gen_helper_spc_write(tn);
-		else if (r == PR_CCS)
-			dc->cpustate_changed = 1;
-		tcg_gen_mov_tl(cpu_PR[r], tn);
-	}
+    if (r < 0 || r > 15) {
+        fprintf(stderr, "wrong register write $p%d\n", r);
+    }
+    if (r == PR_BZ || r == PR_WZ || r == PR_DZ) {
+        return;
+    } else if (r == PR_SRS) {
+        tcg_gen_andi_tl(cpu_PR[r], tn, 3);
+    } else {
+        if (r == PR_PID) {
+            gen_helper_tlb_flush_pid(cpu_env, tn);
+        }
+        if (dc->tb_flags & S_FLAG && r == PR_SPC) {
+            gen_helper_spc_write(cpu_env, tn);
+        } else if (r == PR_CCS) {
+            dc->cpustate_changed = 1;
+        }
+        tcg_gen_mov_tl(cpu_PR[r], tn);
+    }
 }
 
 /* Sign extend at translation time.  */
 static int sign_extend(unsigned int val, unsigned int width)
 {
-	int sval;
-
-	/* LSL.  */
-	val <<= 31 - width;
-	sval = val;
-	/* ASR.  */
-	sval >>= 31 - width;
-	return sval;
-}
-
-static int cris_fetch(DisasContext *dc, uint32_t addr,
-		      unsigned int size, unsigned int sign)
-{
-	int r;
-
-	switch (size) {
-		case 4:
-		{
-			r = ldl_code(addr);
-			break;
-		}
-		case 2:
-		{
-			if (sign) {
-				r = ldsw_code(addr);
-			} else {
-				r = lduw_code(addr);
-			}
-			break;
-		}
-		case 1:
-		{
-			if (sign) {
-				r = ldsb_code(addr);
-			} else {
-				r = ldub_code(addr);
-			}
-			break;
-		}
-		default:
-			cpu_abort(dc->env, "Invalid fetch size %d\n", size);
-			break;
-	}
-	return r;
+    int sval;
+
+    /* LSL.  */
+    val <<= 31 - width;
+    sval = val;
+    /* ASR.  */
+    sval >>= 31 - width;
+    return sval;
+}
+
+static int cris_fetch(CPUCRISState *env, DisasContext *dc, uint32_t addr,
+              unsigned int size, unsigned int sign)
+{
+    int r;
+
+    switch (size) {
+    case 4:
+    {
+        r = cpu_ldl_code(env, addr);
+        break;
+    }
+    case 2:
+    {
+        if (sign) {
+            r = cpu_ldsw_code(env, addr);
+        } else {
+            r = cpu_lduw_code(env, addr);
+        }
+        break;
+    }
+    case 1:
+    {
+        if (sign) {
+            r = cpu_ldsb_code(env, addr);
+        } else {
+            r = cpu_ldub_code(env, addr);
+        }
+        break;
+    }
+    default:
+        cpu_abort(dc->env, "Invalid fetch size %d\n", size);
+        break;
+    }
+    return r;
 }
 
 static void cris_lock_irq(DisasContext *dc)
 {
-	dc->clear_locked_irq = 0;
-	t_gen_mov_env_TN(locked_irq, tcg_const_tl(1));
+    dc->clear_locked_irq = 0;
+    t_gen_mov_env_TN(locked_irq, tcg_const_tl(1));
 }
 
 static inline void t_gen_raise_exception(uint32_t index)
 {
         TCGv_i32 tmp = tcg_const_i32(index);
-	gen_helper_raise_exception(tmp);
+        gen_helper_raise_exception(cpu_env, tmp);
         tcg_temp_free_i32(tmp);
 }
 
 static void t_gen_lsl(TCGv d, TCGv a, TCGv b)
 {
-	TCGv t0, t_31;
+    TCGv t0, t_31;
 
-	t0 = tcg_temp_new();
-	t_31 = tcg_const_tl(31);
-	tcg_gen_shl_tl(d, a, b);
+    t0 = tcg_temp_new();
+    t_31 = tcg_const_tl(31);
+    tcg_gen_shl_tl(d, a, b);
 
-	tcg_gen_sub_tl(t0, t_31, b);
-	tcg_gen_sar_tl(t0, t0, t_31);
-	tcg_gen_and_tl(t0, t0, d);
-	tcg_gen_xor_tl(d, d, t0);
-	tcg_temp_free(t0);
-	tcg_temp_free(t_31);
+    tcg_gen_sub_tl(t0, t_31, b);
+    tcg_gen_sar_tl(t0, t0, t_31);
+    tcg_gen_and_tl(t0, t0, d);
+    tcg_gen_xor_tl(d, d, t0);
+    tcg_temp_free(t0);
+    tcg_temp_free(t_31);
 }
 
 static void t_gen_lsr(TCGv d, TCGv a, TCGv b)
 {
-	TCGv t0, t_31;
+    TCGv t0, t_31;
 
-	t0 = tcg_temp_new();
-	t_31 = tcg_temp_new();
-	tcg_gen_shr_tl(d, a, b);
+    t0 = tcg_temp_new();
+    t_31 = tcg_temp_new();
+    tcg_gen_shr_tl(d, a, b);
 
-	tcg_gen_movi_tl(t_31, 31);
-	tcg_gen_sub_tl(t0, t_31, b);
-	tcg_gen_sar_tl(t0, t0, t_31);
-	tcg_gen_and_tl(t0, t0, d);
-	tcg_gen_xor_tl(d, d, t0);
-	tcg_temp_free(t0);
-	tcg_temp_free(t_31);
+    tcg_gen_movi_tl(t_31, 31);
+    tcg_gen_sub_tl(t0, t_31, b);
+    tcg_gen_sar_tl(t0, t0, t_31);
+    tcg_gen_and_tl(t0, t0, d);
+    tcg_gen_xor_tl(d, d, t0);
+    tcg_temp_free(t0);
+    tcg_temp_free(t_31);
 }
 
 static void t_gen_asr(TCGv d, TCGv a, TCGv b)
 {
-	TCGv t0, t_31;
+    TCGv t0, t_31;
 
-	t0 = tcg_temp_new();
-	t_31 = tcg_temp_new();
-	tcg_gen_sar_tl(d, a, b);
+    t0 = tcg_temp_new();
+    t_31 = tcg_temp_new();
+    tcg_gen_sar_tl(d, a, b);
 
-	tcg_gen_movi_tl(t_31, 31);
-	tcg_gen_sub_tl(t0, t_31, b);
-	tcg_gen_sar_tl(t0, t0, t_31);
-	tcg_gen_or_tl(d, d, t0);
-	tcg_temp_free(t0);
-	tcg_temp_free(t_31);
+    tcg_gen_movi_tl(t_31, 31);
+    tcg_gen_sub_tl(t0, t_31, b);
+    tcg_gen_sar_tl(t0, t0, t_31);
+    tcg_gen_or_tl(d, d, t0);
+    tcg_temp_free(t0);
+    tcg_temp_free(t_31);
 }
 
 /* 64-bit signed mul, lower result in d and upper in d2.  */
 static void t_gen_muls(TCGv d, TCGv d2, TCGv a, TCGv b)
 {
-	TCGv_i64 t0, t1;
+    TCGv_i64 t0, t1;
 
-	t0 = tcg_temp_new_i64();
-	t1 = tcg_temp_new_i64();
+    t0 = tcg_temp_new_i64();
+    t1 = tcg_temp_new_i64();
 
-	tcg_gen_ext_i32_i64(t0, a);
-	tcg_gen_ext_i32_i64(t1, b);
-	tcg_gen_mul_i64(t0, t0, t1);
+    tcg_gen_ext_i32_i64(t0, a);
+    tcg_gen_ext_i32_i64(t1, b);
+    tcg_gen_mul_i64(t0, t0, t1);
 
-	tcg_gen_trunc_i64_i32(d, t0);
-	tcg_gen_shri_i64(t0, t0, 32);
-	tcg_gen_trunc_i64_i32(d2, t0);
+    tcg_gen_trunc_i64_i32(d, t0);
+    tcg_gen_shri_i64(t0, t0, 32);
+    tcg_gen_trunc_i64_i32(d2, t0);
 
-	tcg_temp_free_i64(t0);
-	tcg_temp_free_i64(t1);
+    tcg_temp_free_i64(t0);
+    tcg_temp_free_i64(t1);
 }
 
 /* 64-bit unsigned muls, lower result in d and upper in d2.  */
 static void t_gen_mulu(TCGv d, TCGv d2, TCGv a, TCGv b)
 {
-	TCGv_i64 t0, t1;
+    TCGv_i64 t0, t1;
 
-	t0 = tcg_temp_new_i64();
-	t1 = tcg_temp_new_i64();
+    t0 = tcg_temp_new_i64();
+    t1 = tcg_temp_new_i64();
 
-	tcg_gen_extu_i32_i64(t0, a);
-	tcg_gen_extu_i32_i64(t1, b);
-	tcg_gen_mul_i64(t0, t0, t1);
+    tcg_gen_extu_i32_i64(t0, a);
+    tcg_gen_extu_i32_i64(t1, b);
+    tcg_gen_mul_i64(t0, t0, t1);
 
-	tcg_gen_trunc_i64_i32(d, t0);
-	tcg_gen_shri_i64(t0, t0, 32);
-	tcg_gen_trunc_i64_i32(d2, t0);
+    tcg_gen_trunc_i64_i32(d, t0);
+    tcg_gen_shri_i64(t0, t0, 32);
+    tcg_gen_trunc_i64_i32(d2, t0);
 
-	tcg_temp_free_i64(t0);
-	tcg_temp_free_i64(t1);
+    tcg_temp_free_i64(t0);
+    tcg_temp_free_i64(t1);
 }
 
 static void t_gen_cris_dstep(TCGv d, TCGv a, TCGv b)
 {
-	int l1;
+    int l1;
 
-	l1 = gen_new_label();
+    l1 = gen_new_label();
 
-	/* 
-	 * d <<= 1
-	 * if (d >= s)
-	 *    d -= s;
-	 */
-	tcg_gen_shli_tl(d, a, 1);
-	tcg_gen_brcond_tl(TCG_COND_LTU, d, b, l1);
-	tcg_gen_sub_tl(d, d, b);
-	gen_set_label(l1);
+    /*
+     * d <<= 1
+     * if (d >= s)
+     *    d -= s;
+     */
+    tcg_gen_shli_tl(d, a, 1);
+    tcg_gen_brcond_tl(TCG_COND_LTU, d, b, l1);
+    tcg_gen_sub_tl(d, d, b);
+    gen_set_label(l1);
 }
 
 static void t_gen_cris_mstep(TCGv d, TCGv a, TCGv b, TCGv ccs)
 {
-	TCGv t;
+    TCGv t;
 
-	/* 
-	 * d <<= 1
-	 * if (n)
-	 *    d += s;
-	 */
-	t = tcg_temp_new();
-	tcg_gen_shli_tl(d, a, 1);
-	tcg_gen_shli_tl(t, ccs, 31 - 3);
-	tcg_gen_sari_tl(t, t, 31);
-	tcg_gen_and_tl(t, t, b);
-	tcg_gen_add_tl(d, d, t);
-	tcg_temp_free(t);
+    /*
+     * d <<= 1
+     * if (n)
+     *    d += s;
+     */
+    t = tcg_temp_new();
+    tcg_gen_shli_tl(d, a, 1);
+    tcg_gen_shli_tl(t, ccs, 31 - 3);
+    tcg_gen_sari_tl(t, t, 31);
+    tcg_gen_and_tl(t, t, b);
+    tcg_gen_add_tl(d, d, t);
+    tcg_temp_free(t);
 }
 
 /* Extended arithmetics on CRIS.  */
 static inline void t_gen_add_flag(TCGv d, int flag)
 {
-	TCGv c;
+    TCGv c;
 
-	c = tcg_temp_new();
-	t_gen_mov_TN_preg(c, PR_CCS);
-	/* Propagate carry into d.  */
-	tcg_gen_andi_tl(c, c, 1 << flag);
-	if (flag)
-		tcg_gen_shri_tl(c, c, flag);
-	tcg_gen_add_tl(d, d, c);
-	tcg_temp_free(c);
+    c = tcg_temp_new();
+    t_gen_mov_TN_preg(c, PR_CCS);
+    /* Propagate carry into d.  */
+    tcg_gen_andi_tl(c, c, 1 << flag);
+    if (flag) {
+        tcg_gen_shri_tl(c, c, flag);
+    }
+    tcg_gen_add_tl(d, d, c);
+    tcg_temp_free(c);
 }
 
 static inline void t_gen_addx_carry(DisasContext *dc, TCGv d)
 {
-	if (dc->flagx_known) {
-		if (dc->flags_x) {
-			TCGv c;
+    if (dc->flagx_known) {
+        if (dc->flags_x) {
+            TCGv c;
             
-			c = tcg_temp_new();
-			t_gen_mov_TN_preg(c, PR_CCS);
-			/* C flag is already at bit 0.  */
-			tcg_gen_andi_tl(c, c, C_FLAG);
-			tcg_gen_add_tl(d, d, c);
-			tcg_temp_free(c);
-		}
-	} else {
-		TCGv x, c;
-
-		x = tcg_temp_new();
-		c = tcg_temp_new();
-		t_gen_mov_TN_preg(x, PR_CCS);
-		tcg_gen_mov_tl(c, x);
-
-		/* Propagate carry into d if X is set. Branch free.  */
-		tcg_gen_andi_tl(c, c, C_FLAG);
-		tcg_gen_andi_tl(x, x, X_FLAG);
-		tcg_gen_shri_tl(x, x, 4);
-
-		tcg_gen_and_tl(x, x, c);
-		tcg_gen_add_tl(d, d, x);        
-		tcg_temp_free(x);
-		tcg_temp_free(c);
-	}
+            c = tcg_temp_new();
+            t_gen_mov_TN_preg(c, PR_CCS);
+            /* C flag is already at bit 0.  */
+            tcg_gen_andi_tl(c, c, C_FLAG);
+            tcg_gen_add_tl(d, d, c);
+            tcg_temp_free(c);
+        }
+    } else {
+        TCGv x, c;
+
+        x = tcg_temp_new();
+        c = tcg_temp_new();
+        t_gen_mov_TN_preg(x, PR_CCS);
+        tcg_gen_mov_tl(c, x);
+
+        /* Propagate carry into d if X is set. Branch free.  */
+        tcg_gen_andi_tl(c, c, C_FLAG);
+        tcg_gen_andi_tl(x, x, X_FLAG);
+        tcg_gen_shri_tl(x, x, 4);
+
+        tcg_gen_and_tl(x, x, c);
+        tcg_gen_add_tl(d, d, x);
+        tcg_temp_free(x);
+        tcg_temp_free(c);
+    }
 }
 
 static inline void t_gen_subx_carry(DisasContext *dc, TCGv d)
 {
-	if (dc->flagx_known) {
-		if (dc->flags_x) {
-			TCGv c;
+    if (dc->flagx_known) {
+        if (dc->flags_x) {
+            TCGv c;
             
-			c = tcg_temp_new();
-			t_gen_mov_TN_preg(c, PR_CCS);
-			/* C flag is already at bit 0.  */
-			tcg_gen_andi_tl(c, c, C_FLAG);
-			tcg_gen_sub_tl(d, d, c);
-			tcg_temp_free(c);
-		}
-	} else {
-		TCGv x, c;
-
-		x = tcg_temp_new();
-		c = tcg_temp_new();
-		t_gen_mov_TN_preg(x, PR_CCS);
-		tcg_gen_mov_tl(c, x);
-
-		/* Propagate carry into d if X is set. Branch free.  */
-		tcg_gen_andi_tl(c, c, C_FLAG);
-		tcg_gen_andi_tl(x, x, X_FLAG);
-		tcg_gen_shri_tl(x, x, 4);
-
-		tcg_gen_and_tl(x, x, c);
-		tcg_gen_sub_tl(d, d, x);
-		tcg_temp_free(x);
-		tcg_temp_free(c);
-	}
+            c = tcg_temp_new();
+            t_gen_mov_TN_preg(c, PR_CCS);
+            /* C flag is already at bit 0.  */
+            tcg_gen_andi_tl(c, c, C_FLAG);
+            tcg_gen_sub_tl(d, d, c);
+            tcg_temp_free(c);
+        }
+    } else {
+        TCGv x, c;
+
+        x = tcg_temp_new();
+        c = tcg_temp_new();
+        t_gen_mov_TN_preg(x, PR_CCS);
+        tcg_gen_mov_tl(c, x);
+
+        /* Propagate carry into d if X is set. Branch free.  */
+        tcg_gen_andi_tl(c, c, C_FLAG);
+        tcg_gen_andi_tl(x, x, X_FLAG);
+        tcg_gen_shri_tl(x, x, 4);
+
+        tcg_gen_and_tl(x, x, c);
+        tcg_gen_sub_tl(d, d, x);
+        tcg_temp_free(x);
+        tcg_temp_free(c);
+    }
 }
 
 /* Swap the two bytes within each half word of the s operand.
    T0 = ((T0 << 8) & 0xff00ff00) | ((T0 >> 8) & 0x00ff00ff)  */
 static inline void t_gen_swapb(TCGv d, TCGv s)
 {
-	TCGv t, org_s;
+    TCGv t, org_s;
 
-	t = tcg_temp_new();
-	org_s = tcg_temp_new();
+    t = tcg_temp_new();
+    org_s = tcg_temp_new();
 
-	/* d and s may refer to the same object.  */
-	tcg_gen_mov_tl(org_s, s);
-	tcg_gen_shli_tl(t, org_s, 8);
-	tcg_gen_andi_tl(d, t, 0xff00ff00);
-	tcg_gen_shri_tl(t, org_s, 8);
-	tcg_gen_andi_tl(t, t, 0x00ff00ff);
-	tcg_gen_or_tl(d, d, t);
-	tcg_temp_free(t);
-	tcg_temp_free(org_s);
+    /* d and s may refer to the same object.  */
+    tcg_gen_mov_tl(org_s, s);
+    tcg_gen_shli_tl(t, org_s, 8);
+    tcg_gen_andi_tl(d, t, 0xff00ff00);
+    tcg_gen_shri_tl(t, org_s, 8);
+    tcg_gen_andi_tl(t, t, 0x00ff00ff);
+    tcg_gen_or_tl(d, d, t);
+    tcg_temp_free(t);
+    tcg_temp_free(org_s);
 }
 
 /* Swap the halfwords of the s operand.  */
 static inline void t_gen_swapw(TCGv d, TCGv s)
 {
-	TCGv t;
-	/* d and s refer the same object.  */
-	t = tcg_temp_new();
-	tcg_gen_mov_tl(t, s);
-	tcg_gen_shli_tl(d, t, 16);
-	tcg_gen_shri_tl(t, t, 16);
-	tcg_gen_or_tl(d, d, t);
-	tcg_temp_free(t);
+    TCGv t;
+    /* d and s refer the same object.  */
+    t = tcg_temp_new();
+    tcg_gen_mov_tl(t, s);
+    tcg_gen_shli_tl(d, t, 16);
+    tcg_gen_shri_tl(t, t, 16);
+    tcg_gen_or_tl(d, d, t);
+    tcg_temp_free(t);
 }
 
 /* Reverse the within each byte.
@@ -532,607 +542,611 @@ static inline void t_gen_swapw(TCGv d, TCGv s)
  */
 static inline void t_gen_swapr(TCGv d, TCGv s)
 {
-	struct {
-		int shift; /* LSL when positive, LSR when negative.  */
-		uint32_t mask;
-	} bitrev [] = {
-		{7, 0x80808080},
-		{5, 0x40404040},
-		{3, 0x20202020},
-		{1, 0x10101010},
-		{-1, 0x08080808},
-		{-3, 0x04040404},
-		{-5, 0x02020202},
-		{-7, 0x01010101}
-	};
-	int i;
-	TCGv t, org_s;
-
-	/* d and s refer the same object.  */
-	t = tcg_temp_new();
-	org_s = tcg_temp_new();
-	tcg_gen_mov_tl(org_s, s);
-
-	tcg_gen_shli_tl(t, org_s,  bitrev[0].shift);
-	tcg_gen_andi_tl(d, t,  bitrev[0].mask);
-	for (i = 1; i < ARRAY_SIZE(bitrev); i++) {
-		if (bitrev[i].shift >= 0) {
-			tcg_gen_shli_tl(t, org_s,  bitrev[i].shift);
-		} else {
-			tcg_gen_shri_tl(t, org_s,  -bitrev[i].shift);
-		}
-		tcg_gen_andi_tl(t, t,  bitrev[i].mask);
-		tcg_gen_or_tl(d, d, t);
-	}
-	tcg_temp_free(t);
-	tcg_temp_free(org_s);
+    struct {
+        int shift; /* LSL when positive, LSR when negative.  */
+        uint32_t mask;
+    } bitrev[] = {
+        {7, 0x80808080},
+        {5, 0x40404040},
+        {3, 0x20202020},
+        {1, 0x10101010},
+        {-1, 0x08080808},
+        {-3, 0x04040404},
+        {-5, 0x02020202},
+        {-7, 0x01010101}
+    };
+    int i;
+    TCGv t, org_s;
+
+    /* d and s refer the same object.  */
+    t = tcg_temp_new();
+    org_s = tcg_temp_new();
+    tcg_gen_mov_tl(org_s, s);
+
+    tcg_gen_shli_tl(t, org_s,  bitrev[0].shift);
+    tcg_gen_andi_tl(d, t,  bitrev[0].mask);
+    for (i = 1; i < ARRAY_SIZE(bitrev); i++) {
+        if (bitrev[i].shift >= 0) {
+            tcg_gen_shli_tl(t, org_s,  bitrev[i].shift);
+        } else {
+            tcg_gen_shri_tl(t, org_s,  -bitrev[i].shift);
+        }
+        tcg_gen_andi_tl(t, t,  bitrev[i].mask);
+        tcg_gen_or_tl(d, d, t);
+    }
+    tcg_temp_free(t);
+    tcg_temp_free(org_s);
 }
 
 static void t_gen_cc_jmp(TCGv pc_true, TCGv pc_false)
 {
-	int l1;
+    int l1;
 
-	l1 = gen_new_label();
+    l1 = gen_new_label();
 
-	/* Conditional jmp.  */
-	tcg_gen_mov_tl(env_pc, pc_false);
-	tcg_gen_brcondi_tl(TCG_COND_EQ, env_btaken, 0, l1);
-	tcg_gen_mov_tl(env_pc, pc_true);
-	gen_set_label(l1);
+    /* Conditional jmp.  */
+    tcg_gen_mov_tl(env_pc, pc_false);
+    tcg_gen_brcondi_tl(TCG_COND_EQ, env_btaken, 0, l1);
+    tcg_gen_mov_tl(env_pc, pc_true);
+    gen_set_label(l1);
 }
 
 static void gen_goto_tb(DisasContext *dc, int n, target_ulong dest)
 {
-	TranslationBlock *tb;
-	tb = dc->tb;
-	if ((tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK)) {
-		tcg_gen_goto_tb(n);
-		tcg_gen_movi_tl(env_pc, dest);
+    TranslationBlock *tb;
+    tb = dc->tb;
+    if ((tb->pc & TARGET_PAGE_MASK) == (dest & TARGET_PAGE_MASK)) {
+        tcg_gen_goto_tb(n);
+        tcg_gen_movi_tl(env_pc, dest);
                 tcg_gen_exit_tb((tcg_target_long)tb + n);
-	} else {
-		tcg_gen_movi_tl(env_pc, dest);
-		tcg_gen_exit_tb(0);
-	}
+    } else {
+        tcg_gen_movi_tl(env_pc, dest);
+        tcg_gen_exit_tb(0);
+    }
 }
 
 static inline void cris_clear_x_flag(DisasContext *dc)
 {
-	if (dc->flagx_known && dc->flags_x)
-		dc->flags_uptodate = 0;
+    if (dc->flagx_known && dc->flags_x) {
+        dc->flags_uptodate = 0;
+    }
 
-	dc->flagx_known = 1;
-	dc->flags_x = 0;
+    dc->flagx_known = 1;
+    dc->flags_x = 0;
 }
 
 static void cris_flush_cc_state(DisasContext *dc)
 {
-	if (dc->cc_size_uptodate != dc->cc_size) {
-		tcg_gen_movi_tl(cc_size, dc->cc_size);
-		dc->cc_size_uptodate = dc->cc_size;
-	}
-	tcg_gen_movi_tl(cc_op, dc->cc_op);
-	tcg_gen_movi_tl(cc_mask, dc->cc_mask);
+    if (dc->cc_size_uptodate != dc->cc_size) {
+        tcg_gen_movi_tl(cc_size, dc->cc_size);
+        dc->cc_size_uptodate = dc->cc_size;
+    }
+    tcg_gen_movi_tl(cc_op, dc->cc_op);
+    tcg_gen_movi_tl(cc_mask, dc->cc_mask);
 }
 
 static void cris_evaluate_flags(DisasContext *dc)
 {
-	if (dc->flags_uptodate)
-		return;
-
-	cris_flush_cc_state(dc);
-
-	switch (dc->cc_op)
-	{
-	case CC_OP_MCP:
-		gen_helper_evaluate_flags_mcp(cpu_PR[PR_CCS],
-					cpu_PR[PR_CCS], cc_src,
-					cc_dest, cc_result);
-		break;
-	case CC_OP_MULS:
-		gen_helper_evaluate_flags_muls(cpu_PR[PR_CCS],
-					cpu_PR[PR_CCS], cc_result,
-					cpu_PR[PR_MOF]);
-		break;
-	case CC_OP_MULU:
-		gen_helper_evaluate_flags_mulu(cpu_PR[PR_CCS],
-					cpu_PR[PR_CCS], cc_result,
-					cpu_PR[PR_MOF]);
-		break;
-	case CC_OP_MOVE:
-	case CC_OP_AND:
-	case CC_OP_OR:
-	case CC_OP_XOR:
-	case CC_OP_ASR:
-	case CC_OP_LSR:
-	case CC_OP_LSL:
-		switch (dc->cc_size)
-		{
-		case 4:
-			gen_helper_evaluate_flags_move_4(cpu_PR[PR_CCS],
-						cpu_PR[PR_CCS], cc_result);
-			break;
-		case 2:
-			gen_helper_evaluate_flags_move_2(cpu_PR[PR_CCS],
-						cpu_PR[PR_CCS], cc_result);
-			break;
-		default:
-			gen_helper_evaluate_flags();
-			break;
-		}
-		break;
-	case CC_OP_FLAGS:
-		/* live.  */
-		break;
-	case CC_OP_SUB:
-	case CC_OP_CMP:
-		if (dc->cc_size == 4)
-			gen_helper_evaluate_flags_sub_4(cpu_PR[PR_CCS],
-				cpu_PR[PR_CCS], cc_src, cc_dest, cc_result);
-		else
-			gen_helper_evaluate_flags();
-
-		break;
-	default:
-		switch (dc->cc_size)
-		{
-			case 4:
-			gen_helper_evaluate_flags_alu_4(cpu_PR[PR_CCS],
-				cpu_PR[PR_CCS], cc_src, cc_dest, cc_result);
-				break;
-			default:
-				gen_helper_evaluate_flags();
-				break;
-		}
-		break;
-	}
-
-	if (dc->flagx_known) {
-		if (dc->flags_x)
-			tcg_gen_ori_tl(cpu_PR[PR_CCS], 
-				       cpu_PR[PR_CCS], X_FLAG);
-		else if (dc->cc_op == CC_OP_FLAGS)
-			tcg_gen_andi_tl(cpu_PR[PR_CCS], 
-					cpu_PR[PR_CCS], ~X_FLAG);
+    if (dc->flags_uptodate) {
+        return;
+    }
+
+    cris_flush_cc_state(dc);
+
+    switch (dc->cc_op) {
+    case CC_OP_MCP:
+        gen_helper_evaluate_flags_mcp(cpu_PR[PR_CCS], cpu_env,
+                cpu_PR[PR_CCS], cc_src,
+                cc_dest, cc_result);
+        break;
+    case CC_OP_MULS:
+        gen_helper_evaluate_flags_muls(cpu_PR[PR_CCS], cpu_env,
+                cpu_PR[PR_CCS], cc_result,
+                cpu_PR[PR_MOF]);
+        break;
+    case CC_OP_MULU:
+        gen_helper_evaluate_flags_mulu(cpu_PR[PR_CCS], cpu_env,
+                cpu_PR[PR_CCS], cc_result,
+                cpu_PR[PR_MOF]);
+        break;
+    case CC_OP_MOVE:
+    case CC_OP_AND:
+    case CC_OP_OR:
+    case CC_OP_XOR:
+    case CC_OP_ASR:
+    case CC_OP_LSR:
+    case CC_OP_LSL:
+        switch (dc->cc_size) {
+        case 4:
+            gen_helper_evaluate_flags_move_4(cpu_PR[PR_CCS],
+                    cpu_env, cpu_PR[PR_CCS], cc_result);
+            break;
+        case 2:
+            gen_helper_evaluate_flags_move_2(cpu_PR[PR_CCS],
+                    cpu_env, cpu_PR[PR_CCS], cc_result);
+            break;
+        default:
+            gen_helper_evaluate_flags(cpu_env);
+            break;
         }
-	dc->flags_uptodate = 1;
+        break;
+    case CC_OP_FLAGS:
+        /* live.  */
+        break;
+    case CC_OP_SUB:
+    case CC_OP_CMP:
+        if (dc->cc_size == 4) {
+            gen_helper_evaluate_flags_sub_4(cpu_PR[PR_CCS], cpu_env,
+                    cpu_PR[PR_CCS], cc_src, cc_dest, cc_result);
+        } else {
+            gen_helper_evaluate_flags(cpu_env);
+        }
+
+        break;
+    default:
+        switch (dc->cc_size) {
+        case 4:
+            gen_helper_evaluate_flags_alu_4(cpu_PR[PR_CCS], cpu_env,
+                    cpu_PR[PR_CCS], cc_src, cc_dest, cc_result);
+            break;
+        default:
+            gen_helper_evaluate_flags(cpu_env);
+            break;
+        }
+        break;
+    }
+
+    if (dc->flagx_known) {
+        if (dc->flags_x) {
+            tcg_gen_ori_tl(cpu_PR[PR_CCS], cpu_PR[PR_CCS], X_FLAG);
+        } else if (dc->cc_op == CC_OP_FLAGS) {
+            tcg_gen_andi_tl(cpu_PR[PR_CCS], cpu_PR[PR_CCS], ~X_FLAG);
+        }
+    }
+    dc->flags_uptodate = 1;
 }
 
 static void cris_cc_mask(DisasContext *dc, unsigned int mask)
 {
-	uint32_t ovl;
+    uint32_t ovl;
 
-	if (!mask) {
-		dc->update_cc = 0;
-		return;
-	}	
+    if (!mask) {
+        dc->update_cc = 0;
+        return;
+    }
 
-	/* Check if we need to evaluate the condition codes due to 
-	   CC overlaying.  */
-	ovl = (dc->cc_mask ^ mask) & ~mask;
-	if (ovl) {
-		/* TODO: optimize this case. It trigs all the time.  */
-		cris_evaluate_flags (dc);
-	}
-	dc->cc_mask = mask;
-	dc->update_cc = 1;
+    /* Check if we need to evaluate the condition codes due to
+       CC overlaying.  */
+    ovl = (dc->cc_mask ^ mask) & ~mask;
+    if (ovl) {
+        /* TODO: optimize this case. It trigs all the time.  */
+        cris_evaluate_flags(dc);
+    }
+    dc->cc_mask = mask;
+    dc->update_cc = 1;
 }
 
 static void cris_update_cc_op(DisasContext *dc, int op, int size)
 {
-	dc->cc_op = op;
-	dc->cc_size = size;
-	dc->flags_uptodate = 0;
+    dc->cc_op = op;
+    dc->cc_size = size;
+    dc->flags_uptodate = 0;
 }
 
 static inline void cris_update_cc_x(DisasContext *dc)
 {
-	/* Save the x flag state at the time of the cc snapshot.  */
-	if (dc->flagx_known) {
-		if (dc->cc_x_uptodate == (2 | dc->flags_x))
-			return;
-		tcg_gen_movi_tl(cc_x, dc->flags_x);
-		dc->cc_x_uptodate = 2 | dc->flags_x;
-	}
-	else {
-		tcg_gen_andi_tl(cc_x, cpu_PR[PR_CCS], X_FLAG);
-		dc->cc_x_uptodate = 1;
-	}
+    /* Save the x flag state at the time of the cc snapshot.  */
+    if (dc->flagx_known) {
+        if (dc->cc_x_uptodate == (2 | dc->flags_x)) {
+            return;
+        }
+        tcg_gen_movi_tl(cc_x, dc->flags_x);
+        dc->cc_x_uptodate = 2 | dc->flags_x;
+    } else {
+        tcg_gen_andi_tl(cc_x, cpu_PR[PR_CCS], X_FLAG);
+        dc->cc_x_uptodate = 1;
+    }
 }
 
 /* Update cc prior to executing ALU op. Needs source operands untouched.  */
 static void cris_pre_alu_update_cc(DisasContext *dc, int op, 
-				   TCGv dst, TCGv src, int size)
-{
-	if (dc->update_cc) {
-		cris_update_cc_op(dc, op, size);
-		tcg_gen_mov_tl(cc_src, src);
-
-		if (op != CC_OP_MOVE
-		    && op != CC_OP_AND
-		    && op != CC_OP_OR
-		    && op != CC_OP_XOR
-		    && op != CC_OP_ASR
-		    && op != CC_OP_LSR
-		    && op != CC_OP_LSL)
-			tcg_gen_mov_tl(cc_dest, dst);
+                   TCGv dst, TCGv src, int size)
+{
+    if (dc->update_cc) {
+        cris_update_cc_op(dc, op, size);
+        tcg_gen_mov_tl(cc_src, src);
+
+        if (op != CC_OP_MOVE
+            && op != CC_OP_AND
+            && op != CC_OP_OR
+            && op != CC_OP_XOR
+            && op != CC_OP_ASR
+            && op != CC_OP_LSR
+            && op != CC_OP_LSL) {
+            tcg_gen_mov_tl(cc_dest, dst);
+        }
 
-		cris_update_cc_x(dc);
-	}
+        cris_update_cc_x(dc);
+    }
 }
 
 /* Update cc after executing ALU op. needs the result.  */
 static inline void cris_update_result(DisasContext *dc, TCGv res)
 {
-	if (dc->update_cc)
-		tcg_gen_mov_tl(cc_result, res);
+    if (dc->update_cc) {
+        tcg_gen_mov_tl(cc_result, res);
+    }
 }
 
 /* Returns one if the write back stage should execute.  */
 static void cris_alu_op_exec(DisasContext *dc, int op, 
-			       TCGv dst, TCGv a, TCGv b, int size)
-{
-	/* Emit the ALU insns.  */
-	switch (op)
-	{
-		case CC_OP_ADD:
-			tcg_gen_add_tl(dst, a, b);
-			/* Extended arithmetics.  */
-			t_gen_addx_carry(dc, dst);
-			break;
-		case CC_OP_ADDC:
-			tcg_gen_add_tl(dst, a, b);
-			t_gen_add_flag(dst, 0); /* C_FLAG.  */
-			break;
-		case CC_OP_MCP:
-			tcg_gen_add_tl(dst, a, b);
-			t_gen_add_flag(dst, 8); /* R_FLAG.  */
-			break;
-		case CC_OP_SUB:
-			tcg_gen_sub_tl(dst, a, b);
-			/* Extended arithmetics.  */
-			t_gen_subx_carry(dc, dst);
-			break;
-		case CC_OP_MOVE:
-			tcg_gen_mov_tl(dst, b);
-			break;
-		case CC_OP_OR:
-			tcg_gen_or_tl(dst, a, b);
-			break;
-		case CC_OP_AND:
-			tcg_gen_and_tl(dst, a, b);
-			break;
-		case CC_OP_XOR:
-			tcg_gen_xor_tl(dst, a, b);
-			break;
-		case CC_OP_LSL:
-			t_gen_lsl(dst, a, b);
-			break;
-		case CC_OP_LSR:
-			t_gen_lsr(dst, a, b);
-			break;
-		case CC_OP_ASR:
-			t_gen_asr(dst, a, b);
-			break;
-		case CC_OP_NEG:
-			tcg_gen_neg_tl(dst, b);
-			/* Extended arithmetics.  */
-			t_gen_subx_carry(dc, dst);
-			break;
-		case CC_OP_LZ:
-			gen_helper_lz(dst, b);
-			break;
-		case CC_OP_MULS:
-			t_gen_muls(dst, cpu_PR[PR_MOF], a, b);
-			break;
-		case CC_OP_MULU:
-			t_gen_mulu(dst, cpu_PR[PR_MOF], a, b);
-			break;
-		case CC_OP_DSTEP:
-			t_gen_cris_dstep(dst, a, b);
-			break;
-		case CC_OP_MSTEP:
-			t_gen_cris_mstep(dst, a, b, cpu_PR[PR_CCS]);
-			break;
-		case CC_OP_BOUND:
-		{
-			int l1;
-			l1 = gen_new_label();
-			tcg_gen_mov_tl(dst, a);
-			tcg_gen_brcond_tl(TCG_COND_LEU, a, b, l1);
-			tcg_gen_mov_tl(dst, b);
-			gen_set_label(l1);
-		}
-		break;
-		case CC_OP_CMP:
-			tcg_gen_sub_tl(dst, a, b);
-			/* Extended arithmetics.  */
-			t_gen_subx_carry(dc, dst);
-			break;
-		default:
-			qemu_log("illegal ALU op.\n");
-			BUG();
-			break;
-	}
-
-	if (size == 1)
-		tcg_gen_andi_tl(dst, dst, 0xff);
-	else if (size == 2)
-		tcg_gen_andi_tl(dst, dst, 0xffff);
+                   TCGv dst, TCGv a, TCGv b, int size)
+{
+    /* Emit the ALU insns.  */
+    switch (op) {
+    case CC_OP_ADD:
+        tcg_gen_add_tl(dst, a, b);
+        /* Extended arithmetics.  */
+        t_gen_addx_carry(dc, dst);
+        break;
+    case CC_OP_ADDC:
+        tcg_gen_add_tl(dst, a, b);
+        t_gen_add_flag(dst, 0); /* C_FLAG.  */
+        break;
+    case CC_OP_MCP:
+        tcg_gen_add_tl(dst, a, b);
+        t_gen_add_flag(dst, 8); /* R_FLAG.  */
+        break;
+    case CC_OP_SUB:
+        tcg_gen_sub_tl(dst, a, b);
+        /* Extended arithmetics.  */
+        t_gen_subx_carry(dc, dst);
+        break;
+    case CC_OP_MOVE:
+        tcg_gen_mov_tl(dst, b);
+        break;
+    case CC_OP_OR:
+        tcg_gen_or_tl(dst, a, b);
+        break;
+    case CC_OP_AND:
+        tcg_gen_and_tl(dst, a, b);
+        break;
+    case CC_OP_XOR:
+        tcg_gen_xor_tl(dst, a, b);
+        break;
+    case CC_OP_LSL:
+        t_gen_lsl(dst, a, b);
+        break;
+    case CC_OP_LSR:
+        t_gen_lsr(dst, a, b);
+        break;
+    case CC_OP_ASR:
+        t_gen_asr(dst, a, b);
+        break;
+    case CC_OP_NEG:
+        tcg_gen_neg_tl(dst, b);
+        /* Extended arithmetics.  */
+        t_gen_subx_carry(dc, dst);
+        break;
+    case CC_OP_LZ:
+        gen_helper_lz(dst, b);
+        break;
+    case CC_OP_MULS:
+        t_gen_muls(dst, cpu_PR[PR_MOF], a, b);
+        break;
+    case CC_OP_MULU:
+        t_gen_mulu(dst, cpu_PR[PR_MOF], a, b);
+        break;
+    case CC_OP_DSTEP:
+        t_gen_cris_dstep(dst, a, b);
+        break;
+    case CC_OP_MSTEP:
+        t_gen_cris_mstep(dst, a, b, cpu_PR[PR_CCS]);
+        break;
+    case CC_OP_BOUND:
+    {
+        int l1;
+        l1 = gen_new_label();
+        tcg_gen_mov_tl(dst, a);
+        tcg_gen_brcond_tl(TCG_COND_LEU, a, b, l1);
+        tcg_gen_mov_tl(dst, b);
+        gen_set_label(l1);
+    }
+        break;
+    case CC_OP_CMP:
+        tcg_gen_sub_tl(dst, a, b);
+        /* Extended arithmetics.  */
+        t_gen_subx_carry(dc, dst);
+        break;
+    default:
+        qemu_log("illegal ALU op.\n");
+        BUG();
+        break;
+    }
+
+    if (size == 1) {
+        tcg_gen_andi_tl(dst, dst, 0xff);
+    } else if (size == 2) {
+        tcg_gen_andi_tl(dst, dst, 0xffff);
+    }
 }
 
 static void cris_alu(DisasContext *dc, int op,
-			       TCGv d, TCGv op_a, TCGv op_b, int size)
+                   TCGv d, TCGv op_a, TCGv op_b, int size)
 {
-	TCGv tmp;
-	int writeback;
+    TCGv tmp;
+    int writeback;
 
-	writeback = 1;
+    writeback = 1;
 
-	if (op == CC_OP_CMP) {
-		tmp = tcg_temp_new();
-		writeback = 0;
-	} else if (size == 4) {
-		tmp = d;
-		writeback = 0;
-	} else
-		tmp = tcg_temp_new();
+    if (op == CC_OP_CMP) {
+        tmp = tcg_temp_new();
+        writeback = 0;
+    } else if (size == 4) {
+        tmp = d;
+        writeback = 0;
+    } else {
+        tmp = tcg_temp_new();
+    }
 
 
-	cris_pre_alu_update_cc(dc, op, op_a, op_b, size);
-	cris_alu_op_exec(dc, op, tmp, op_a, op_b, size);
-	cris_update_result(dc, tmp);
+    cris_pre_alu_update_cc(dc, op, op_a, op_b, size);
+    cris_alu_op_exec(dc, op, tmp, op_a, op_b, size);
+    cris_update_result(dc, tmp);
 
-	/* Writeback.  */
-	if (writeback) {
-		if (size == 1)
-			tcg_gen_andi_tl(d, d, ~0xff);
-		else
-			tcg_gen_andi_tl(d, d, ~0xffff);
-		tcg_gen_or_tl(d, d, tmp);
-	}
-	if (!TCGV_EQUAL(tmp, d))
-		tcg_temp_free(tmp);
+    /* Writeback.  */
+    if (writeback) {
+        if (size == 1) {
+            tcg_gen_andi_tl(d, d, ~0xff);
+        } else {
+            tcg_gen_andi_tl(d, d, ~0xffff);
+        }
+        tcg_gen_or_tl(d, d, tmp);
+    }
+    if (!TCGV_EQUAL(tmp, d)) {
+        tcg_temp_free(tmp);
+    }
 }
 
 static int arith_cc(DisasContext *dc)
 {
-	if (dc->update_cc) {
-		switch (dc->cc_op) {
-			case CC_OP_ADDC: return 1;
-			case CC_OP_ADD: return 1;
-			case CC_OP_SUB: return 1;
-			case CC_OP_DSTEP: return 1;
-			case CC_OP_LSL: return 1;
-			case CC_OP_LSR: return 1;
-			case CC_OP_ASR: return 1;
-			case CC_OP_CMP: return 1;
-			case CC_OP_NEG: return 1;
-			case CC_OP_OR: return 1;
-			case CC_OP_AND: return 1;
-			case CC_OP_XOR: return 1;
-			case CC_OP_MULU: return 1;
-			case CC_OP_MULS: return 1;
-			default:
-				return 0;
-		}
-	}
-	return 0;
+    if (dc->update_cc) {
+        switch (dc->cc_op) {
+        case CC_OP_ADDC: return 1;
+        case CC_OP_ADD: return 1;
+        case CC_OP_SUB: return 1;
+        case CC_OP_DSTEP: return 1;
+        case CC_OP_LSL: return 1;
+        case CC_OP_LSR: return 1;
+        case CC_OP_ASR: return 1;
+        case CC_OP_CMP: return 1;
+        case CC_OP_NEG: return 1;
+        case CC_OP_OR: return 1;
+        case CC_OP_AND: return 1;
+        case CC_OP_XOR: return 1;
+        case CC_OP_MULU: return 1;
+        case CC_OP_MULS: return 1;
+        default:
+            return 0;
+        }
+    }
+    return 0;
 }
 
 static void gen_tst_cc (DisasContext *dc, TCGv cc, int cond)
 {
-	int arith_opt, move_opt;
-
-	/* TODO: optimize more condition codes.  */
-
-	/*
-	 * If the flags are live, we've gotta look into the bits of CCS.
-	 * Otherwise, if we just did an arithmetic operation we try to
-	 * evaluate the condition code faster.
-	 *
-	 * When this function is done, T0 should be non-zero if the condition
-	 * code is true.
-	 */
-	arith_opt = arith_cc(dc) && !dc->flags_uptodate;
-	move_opt = (dc->cc_op == CC_OP_MOVE);
-	switch (cond) {
-		case CC_EQ:
-			if ((arith_opt || move_opt)
-			    && dc->cc_x_uptodate != (2 | X_FLAG)) {
-				tcg_gen_setcond_tl(TCG_COND_EQ, cc,
-						   cc_result, tcg_const_tl(0));
-			}
-			else {
-				cris_evaluate_flags(dc);
-				tcg_gen_andi_tl(cc, 
-						cpu_PR[PR_CCS], Z_FLAG);
-			}
-			break;
-		case CC_NE:
-			if ((arith_opt || move_opt)
-			    && dc->cc_x_uptodate != (2 | X_FLAG)) {
-				tcg_gen_mov_tl(cc, cc_result);
-	 		} else {
-				cris_evaluate_flags(dc);
-				tcg_gen_xori_tl(cc, cpu_PR[PR_CCS],
-						Z_FLAG);
-				tcg_gen_andi_tl(cc, cc, Z_FLAG);
-			}
-			break;
-		case CC_CS:
-			cris_evaluate_flags(dc);
-			tcg_gen_andi_tl(cc, cpu_PR[PR_CCS], C_FLAG);
-			break;
-		case CC_CC:
-			cris_evaluate_flags(dc);
-			tcg_gen_xori_tl(cc, cpu_PR[PR_CCS], C_FLAG);
-			tcg_gen_andi_tl(cc, cc, C_FLAG);
-			break;
-		case CC_VS:
-			cris_evaluate_flags(dc);
-			tcg_gen_andi_tl(cc, cpu_PR[PR_CCS], V_FLAG);
-			break;
-		case CC_VC:
-			cris_evaluate_flags(dc);
-			tcg_gen_xori_tl(cc, cpu_PR[PR_CCS],
-					V_FLAG);
-			tcg_gen_andi_tl(cc, cc, V_FLAG);
-			break;
-		case CC_PL:
-			if (arith_opt || move_opt) {
-				int bits = 31;
-
-				if (dc->cc_size == 1)
-					bits = 7;
-				else if (dc->cc_size == 2)
-					bits = 15;	
-
-				tcg_gen_shri_tl(cc, cc_result, bits);
-				tcg_gen_xori_tl(cc, cc, 1);
-			} else {
-				cris_evaluate_flags(dc);
-				tcg_gen_xori_tl(cc, cpu_PR[PR_CCS],
-						N_FLAG);
-				tcg_gen_andi_tl(cc, cc, N_FLAG);
-			}
-			break;
-		case CC_MI:
-			if (arith_opt || move_opt) {
-				int bits = 31;
-
-				if (dc->cc_size == 1)
-					bits = 7;
-				else if (dc->cc_size == 2)
-					bits = 15;	
-
-				tcg_gen_shri_tl(cc, cc_result, bits);
-				tcg_gen_andi_tl(cc, cc, 1);
-			}
-			else {
-				cris_evaluate_flags(dc);
-				tcg_gen_andi_tl(cc, cpu_PR[PR_CCS],
-						N_FLAG);
-			}
-			break;
-		case CC_LS:
-			cris_evaluate_flags(dc);
-			tcg_gen_andi_tl(cc, cpu_PR[PR_CCS],
-					C_FLAG | Z_FLAG);
-			break;
-		case CC_HI:
-			cris_evaluate_flags(dc);
-			{
-				TCGv tmp;
-
-				tmp = tcg_temp_new();
-				tcg_gen_xori_tl(tmp, cpu_PR[PR_CCS],
-						C_FLAG | Z_FLAG);
-				/* Overlay the C flag on top of the Z.  */
-				tcg_gen_shli_tl(cc, tmp, 2);
-				tcg_gen_and_tl(cc, tmp, cc);
-				tcg_gen_andi_tl(cc, cc, Z_FLAG);
-
-				tcg_temp_free(tmp);
-			}
-			break;
-		case CC_GE:
-			cris_evaluate_flags(dc);
-			/* Overlay the V flag on top of the N.  */
-			tcg_gen_shli_tl(cc, cpu_PR[PR_CCS], 2);
-			tcg_gen_xor_tl(cc,
-				       cpu_PR[PR_CCS], cc);
-			tcg_gen_andi_tl(cc, cc, N_FLAG);
-			tcg_gen_xori_tl(cc, cc, N_FLAG);
-			break;
-		case CC_LT:
-			cris_evaluate_flags(dc);
-			/* Overlay the V flag on top of the N.  */
-			tcg_gen_shli_tl(cc, cpu_PR[PR_CCS], 2);
-			tcg_gen_xor_tl(cc,
-				       cpu_PR[PR_CCS], cc);
-			tcg_gen_andi_tl(cc, cc, N_FLAG);
-			break;
-		case CC_GT:
-			cris_evaluate_flags(dc);
-			{
-				TCGv n, z;
-
-				n = tcg_temp_new();
-				z = tcg_temp_new();
-
-				/* To avoid a shift we overlay everything on
-				   the V flag.  */
-				tcg_gen_shri_tl(n, cpu_PR[PR_CCS], 2);
-				tcg_gen_shri_tl(z, cpu_PR[PR_CCS], 1);
-				/* invert Z.  */
-				tcg_gen_xori_tl(z, z, 2);
-
-				tcg_gen_xor_tl(n, n, cpu_PR[PR_CCS]);
-				tcg_gen_xori_tl(n, n, 2);
-				tcg_gen_and_tl(cc, z, n);
-				tcg_gen_andi_tl(cc, cc, 2);
-
-				tcg_temp_free(n);
-				tcg_temp_free(z);
-			}
-			break;
-		case CC_LE:
-			cris_evaluate_flags(dc);
-			{
-				TCGv n, z;
-
-				n = tcg_temp_new();
-				z = tcg_temp_new();
-
-				/* To avoid a shift we overlay everything on
-				   the V flag.  */
-				tcg_gen_shri_tl(n, cpu_PR[PR_CCS], 2);
-				tcg_gen_shri_tl(z, cpu_PR[PR_CCS], 1);
-
-				tcg_gen_xor_tl(n, n, cpu_PR[PR_CCS]);
-				tcg_gen_or_tl(cc, z, n);
-				tcg_gen_andi_tl(cc, cc, 2);
-
-				tcg_temp_free(n);
-				tcg_temp_free(z);
-			}
-			break;
-		case CC_P:
-			cris_evaluate_flags(dc);
-			tcg_gen_andi_tl(cc, cpu_PR[PR_CCS], P_FLAG);
-			break;
-		case CC_A:
-			tcg_gen_movi_tl(cc, 1);
-			break;
-		default:
-			BUG();
-			break;
-	};
+    int arith_opt, move_opt;
+
+    /* TODO: optimize more condition codes.  */
+
+    /*
+     * If the flags are live, we've gotta look into the bits of CCS.
+     * Otherwise, if we just did an arithmetic operation we try to
+     * evaluate the condition code faster.
+     *
+     * When this function is done, T0 should be non-zero if the condition
+     * code is true.
+     */
+    arith_opt = arith_cc(dc) && !dc->flags_uptodate;
+    move_opt = (dc->cc_op == CC_OP_MOVE);
+    switch (cond) {
+    case CC_EQ:
+        if ((arith_opt || move_opt)
+                && dc->cc_x_uptodate != (2 | X_FLAG)) {
+            tcg_gen_setcond_tl(TCG_COND_EQ, cc,
+                    cc_result, tcg_const_tl(0));
+        } else {
+            cris_evaluate_flags(dc);
+            tcg_gen_andi_tl(cc,
+                    cpu_PR[PR_CCS], Z_FLAG);
+        }
+        break;
+    case CC_NE:
+        if ((arith_opt || move_opt)
+                && dc->cc_x_uptodate != (2 | X_FLAG)) {
+            tcg_gen_mov_tl(cc, cc_result);
+        } else {
+            cris_evaluate_flags(dc);
+            tcg_gen_xori_tl(cc, cpu_PR[PR_CCS],
+                    Z_FLAG);
+            tcg_gen_andi_tl(cc, cc, Z_FLAG);
+        }
+        break;
+    case CC_CS:
+        cris_evaluate_flags(dc);
+        tcg_gen_andi_tl(cc, cpu_PR[PR_CCS], C_FLAG);
+        break;
+    case CC_CC:
+        cris_evaluate_flags(dc);
+        tcg_gen_xori_tl(cc, cpu_PR[PR_CCS], C_FLAG);
+        tcg_gen_andi_tl(cc, cc, C_FLAG);
+        break;
+    case CC_VS:
+        cris_evaluate_flags(dc);
+        tcg_gen_andi_tl(cc, cpu_PR[PR_CCS], V_FLAG);
+        break;
+    case CC_VC:
+        cris_evaluate_flags(dc);
+        tcg_gen_xori_tl(cc, cpu_PR[PR_CCS],
+                V_FLAG);
+        tcg_gen_andi_tl(cc, cc, V_FLAG);
+        break;
+    case CC_PL:
+        if (arith_opt || move_opt) {
+            int bits = 31;
+
+            if (dc->cc_size == 1) {
+                bits = 7;
+            } else if (dc->cc_size == 2) {
+                bits = 15;
+            }
+
+            tcg_gen_shri_tl(cc, cc_result, bits);
+            tcg_gen_xori_tl(cc, cc, 1);
+        } else {
+            cris_evaluate_flags(dc);
+            tcg_gen_xori_tl(cc, cpu_PR[PR_CCS],
+                    N_FLAG);
+            tcg_gen_andi_tl(cc, cc, N_FLAG);
+        }
+        break;
+    case CC_MI:
+        if (arith_opt || move_opt) {
+            int bits = 31;
+
+            if (dc->cc_size == 1) {
+                bits = 7;
+            } else if (dc->cc_size == 2) {
+                bits = 15;
+            }
+
+            tcg_gen_shri_tl(cc, cc_result, bits);
+            tcg_gen_andi_tl(cc, cc, 1);
+        } else {
+            cris_evaluate_flags(dc);
+            tcg_gen_andi_tl(cc, cpu_PR[PR_CCS],
+                    N_FLAG);
+        }
+        break;
+    case CC_LS:
+        cris_evaluate_flags(dc);
+        tcg_gen_andi_tl(cc, cpu_PR[PR_CCS],
+                C_FLAG | Z_FLAG);
+        break;
+    case CC_HI:
+        cris_evaluate_flags(dc);
+        {
+            TCGv tmp;
+
+            tmp = tcg_temp_new();
+            tcg_gen_xori_tl(tmp, cpu_PR[PR_CCS],
+                    C_FLAG | Z_FLAG);
+            /* Overlay the C flag on top of the Z.  */
+            tcg_gen_shli_tl(cc, tmp, 2);
+            tcg_gen_and_tl(cc, tmp, cc);
+            tcg_gen_andi_tl(cc, cc, Z_FLAG);
+
+            tcg_temp_free(tmp);
+        }
+        break;
+    case CC_GE:
+        cris_evaluate_flags(dc);
+        /* Overlay the V flag on top of the N.  */
+        tcg_gen_shli_tl(cc, cpu_PR[PR_CCS], 2);
+        tcg_gen_xor_tl(cc,
+                cpu_PR[PR_CCS], cc);
+        tcg_gen_andi_tl(cc, cc, N_FLAG);
+        tcg_gen_xori_tl(cc, cc, N_FLAG);
+        break;
+    case CC_LT:
+        cris_evaluate_flags(dc);
+        /* Overlay the V flag on top of the N.  */
+        tcg_gen_shli_tl(cc, cpu_PR[PR_CCS], 2);
+        tcg_gen_xor_tl(cc,
+                cpu_PR[PR_CCS], cc);
+        tcg_gen_andi_tl(cc, cc, N_FLAG);
+        break;
+    case CC_GT:
+        cris_evaluate_flags(dc);
+        {
+            TCGv n, z;
+
+            n = tcg_temp_new();
+            z = tcg_temp_new();
+
+            /* To avoid a shift we overlay everything on
+                   the V flag.  */
+            tcg_gen_shri_tl(n, cpu_PR[PR_CCS], 2);
+            tcg_gen_shri_tl(z, cpu_PR[PR_CCS], 1);
+            /* invert Z.  */
+            tcg_gen_xori_tl(z, z, 2);
+
+            tcg_gen_xor_tl(n, n, cpu_PR[PR_CCS]);
+            tcg_gen_xori_tl(n, n, 2);
+            tcg_gen_and_tl(cc, z, n);
+            tcg_gen_andi_tl(cc, cc, 2);
+
+            tcg_temp_free(n);
+            tcg_temp_free(z);
+        }
+        break;
+    case CC_LE:
+        cris_evaluate_flags(dc);
+        {
+            TCGv n, z;
+
+            n = tcg_temp_new();
+            z = tcg_temp_new();
+
+            /* To avoid a shift we overlay everything on
+                   the V flag.  */
+            tcg_gen_shri_tl(n, cpu_PR[PR_CCS], 2);
+            tcg_gen_shri_tl(z, cpu_PR[PR_CCS], 1);
+
+            tcg_gen_xor_tl(n, n, cpu_PR[PR_CCS]);
+            tcg_gen_or_tl(cc, z, n);
+            tcg_gen_andi_tl(cc, cc, 2);
+
+            tcg_temp_free(n);
+            tcg_temp_free(z);
+        }
+        break;
+    case CC_P:
+        cris_evaluate_flags(dc);
+        tcg_gen_andi_tl(cc, cpu_PR[PR_CCS], P_FLAG);
+        break;
+    case CC_A:
+        tcg_gen_movi_tl(cc, 1);
+        break;
+    default:
+        BUG();
+        break;
+    };
 }
 
 static void cris_store_direct_jmp(DisasContext *dc)
 {
-	/* Store the direct jmp state into the cpu-state.  */
-	if (dc->jmp == JMP_DIRECT || dc->jmp == JMP_DIRECT_CC) {
-		if (dc->jmp == JMP_DIRECT) {
-			tcg_gen_movi_tl(env_btaken, 1);
-		}
-		tcg_gen_movi_tl(env_btarget, dc->jmp_pc);
-		dc->jmp = JMP_INDIRECT;
-	}
+    /* Store the direct jmp state into the cpu-state.  */
+    if (dc->jmp == JMP_DIRECT || dc->jmp == JMP_DIRECT_CC) {
+        if (dc->jmp == JMP_DIRECT) {
+            tcg_gen_movi_tl(env_btaken, 1);
+        }
+        tcg_gen_movi_tl(env_btarget, dc->jmp_pc);
+        dc->jmp = JMP_INDIRECT;
+    }
 }
 
 static void cris_prepare_cc_branch (DisasContext *dc, 
-				    int offset, int cond)
+                    int offset, int cond)
 {
-	/* This helps us re-schedule the micro-code to insns in delay-slots
-	   before the actual jump.  */
-	dc->delayed_branch = 2;
-	dc->jmp = JMP_DIRECT_CC;
-	dc->jmp_pc = dc->pc + offset;
+    /* This helps us re-schedule the micro-code to insns in delay-slots
+       before the actual jump.  */
+    dc->delayed_branch = 2;
+    dc->jmp = JMP_DIRECT_CC;
+    dc->jmp_pc = dc->pc + offset;
 
-	gen_tst_cc (dc, env_btaken, cond);
-	tcg_gen_movi_tl(env_btarget, dc->jmp_pc);
+    gen_tst_cc(dc, env_btaken, cond);
+    tcg_gen_movi_tl(env_btarget, dc->jmp_pc);
 }
 
 
@@ -1140,1994 +1154,2013 @@ static void cris_prepare_cc_branch (DisasContext *dc,
    when the dest addr is constant to allow tb chaining.  */
 static inline void cris_prepare_jmp (DisasContext *dc, unsigned int type)
 {
-	/* This helps us re-schedule the micro-code to insns in delay-slots
-	   before the actual jump.  */
-	dc->delayed_branch = 2;
-	dc->jmp = type;
-	if (type == JMP_INDIRECT) {
-		tcg_gen_movi_tl(env_btaken, 1);
-	}
+    /* This helps us re-schedule the micro-code to insns in delay-slots
+       before the actual jump.  */
+    dc->delayed_branch = 2;
+    dc->jmp = type;
+    if (type == JMP_INDIRECT) {
+        tcg_gen_movi_tl(env_btaken, 1);
+    }
 }
 
 static void gen_load64(DisasContext *dc, TCGv_i64 dst, TCGv addr)
 {
-	int mem_index = cpu_mmu_index(dc->env);
+    int mem_index = cpu_mmu_index(dc->env);
 
-	/* If we get a fault on a delayslot we must keep the jmp state in
-	   the cpu-state to be able to re-execute the jmp.  */
-	if (dc->delayed_branch == 1)
-		cris_store_direct_jmp(dc);
+    /* If we get a fault on a delayslot we must keep the jmp state in
+       the cpu-state to be able to re-execute the jmp.  */
+    if (dc->delayed_branch == 1) {
+        cris_store_direct_jmp(dc);
+    }
 
-        tcg_gen_qemu_ld64(dst, addr, mem_index);
+    tcg_gen_qemu_ld64(dst, addr, mem_index);
 }
 
 static void gen_load(DisasContext *dc, TCGv dst, TCGv addr, 
-		     unsigned int size, int sign)
-{
-	int mem_index = cpu_mmu_index(dc->env);
-
-	/* If we get a fault on a delayslot we must keep the jmp state in
-	   the cpu-state to be able to re-execute the jmp.  */
-	if (dc->delayed_branch == 1)
-		cris_store_direct_jmp(dc);
-
-	if (size == 1) {
-		if (sign)
-			tcg_gen_qemu_ld8s(dst, addr, mem_index);
-		else
-			tcg_gen_qemu_ld8u(dst, addr, mem_index);
-	}
-	else if (size == 2) {
-		if (sign)
-			tcg_gen_qemu_ld16s(dst, addr, mem_index);
-		else
-			tcg_gen_qemu_ld16u(dst, addr, mem_index);
-	}
-	else if (size == 4) {
-		tcg_gen_qemu_ld32u(dst, addr, mem_index);
-	}
-	else {
-		abort();
-	}
+             unsigned int size, int sign)
+{
+    int mem_index = cpu_mmu_index(dc->env);
+
+    /* If we get a fault on a delayslot we must keep the jmp state in
+       the cpu-state to be able to re-execute the jmp.  */
+    if (dc->delayed_branch == 1) {
+        cris_store_direct_jmp(dc);
+    }
+
+    if (size == 1) {
+        if (sign) {
+            tcg_gen_qemu_ld8s(dst, addr, mem_index);
+        } else {
+            tcg_gen_qemu_ld8u(dst, addr, mem_index);
+        }
+    } else if (size == 2) {
+        if (sign) {
+            tcg_gen_qemu_ld16s(dst, addr, mem_index);
+        } else {
+            tcg_gen_qemu_ld16u(dst, addr, mem_index);
+        }
+    } else if (size == 4) {
+        tcg_gen_qemu_ld32u(dst, addr, mem_index);
+    } else {
+        abort();
+    }
 }
 
 static void gen_store (DisasContext *dc, TCGv addr, TCGv val,
-		       unsigned int size)
+               unsigned int size)
 {
-	int mem_index = cpu_mmu_index(dc->env);
+    int mem_index = cpu_mmu_index(dc->env);
 
-	/* If we get a fault on a delayslot we must keep the jmp state in
-	   the cpu-state to be able to re-execute the jmp.  */
-	if (dc->delayed_branch == 1)
- 		cris_store_direct_jmp(dc);
+    /* If we get a fault on a delayslot we must keep the jmp state in
+       the cpu-state to be able to re-execute the jmp.  */
+    if (dc->delayed_branch == 1) {
+        cris_store_direct_jmp(dc);
+    }
 
 
-	/* Conditional writes. We only support the kind were X and P are known
-	   at translation time.  */
-	if (dc->flagx_known && dc->flags_x && (dc->tb_flags & P_FLAG)) {
-		dc->postinc = 0;
-		cris_evaluate_flags(dc);
-		tcg_gen_ori_tl(cpu_PR[PR_CCS], cpu_PR[PR_CCS], C_FLAG);
-		return;
-	}
+    /* Conditional writes. We only support the kind were X and P are known
+       at translation time.  */
+    if (dc->flagx_known && dc->flags_x && (dc->tb_flags & P_FLAG)) {
+        dc->postinc = 0;
+        cris_evaluate_flags(dc);
+        tcg_gen_ori_tl(cpu_PR[PR_CCS], cpu_PR[PR_CCS], C_FLAG);
+        return;
+    }
 
-	if (size == 1)
-		tcg_gen_qemu_st8(val, addr, mem_index);
-	else if (size == 2)
-		tcg_gen_qemu_st16(val, addr, mem_index);
-	else
-		tcg_gen_qemu_st32(val, addr, mem_index);
+    if (size == 1) {
+        tcg_gen_qemu_st8(val, addr, mem_index);
+    } else if (size == 2) {
+        tcg_gen_qemu_st16(val, addr, mem_index);
+    } else {
+        tcg_gen_qemu_st32(val, addr, mem_index);
+    }
 
-	if (dc->flagx_known && dc->flags_x) {
-		cris_evaluate_flags(dc);
-		tcg_gen_andi_tl(cpu_PR[PR_CCS], cpu_PR[PR_CCS], ~C_FLAG);
-	}
+    if (dc->flagx_known && dc->flags_x) {
+        cris_evaluate_flags(dc);
+        tcg_gen_andi_tl(cpu_PR[PR_CCS], cpu_PR[PR_CCS], ~C_FLAG);
+    }
 }
 
 static inline void t_gen_sext(TCGv d, TCGv s, int size)
 {
-	if (size == 1)
-		tcg_gen_ext8s_i32(d, s);
-	else if (size == 2)
-		tcg_gen_ext16s_i32(d, s);
-	else if(!TCGV_EQUAL(d, s))
-		tcg_gen_mov_tl(d, s);
+    if (size == 1) {
+        tcg_gen_ext8s_i32(d, s);
+    } else if (size == 2) {
+        tcg_gen_ext16s_i32(d, s);
+    } else if (!TCGV_EQUAL(d, s)) {
+        tcg_gen_mov_tl(d, s);
+    }
 }
 
 static inline void t_gen_zext(TCGv d, TCGv s, int size)
 {
-	if (size == 1)
-		tcg_gen_ext8u_i32(d, s);
-	else if (size == 2)
-		tcg_gen_ext16u_i32(d, s);
-	else if (!TCGV_EQUAL(d, s))
-		tcg_gen_mov_tl(d, s);
+    if (size == 1) {
+        tcg_gen_ext8u_i32(d, s);
+    } else if (size == 2) {
+        tcg_gen_ext16u_i32(d, s);
+    } else if (!TCGV_EQUAL(d, s)) {
+        tcg_gen_mov_tl(d, s);
+    }
 }
 
 #if DISAS_CRIS
 static char memsize_char(int size)
 {
-	switch (size)
-	{
-		case 1: return 'b';  break;
-		case 2: return 'w';  break;
-		case 4: return 'd';  break;
-		default:
-			return 'x';
-			break;
-	}
+    switch (size) {
+    case 1: return 'b';  break;
+    case 2: return 'w';  break;
+    case 4: return 'd';  break;
+    default:
+        return 'x';
+        break;
+    }
 }
 #endif
 
 static inline unsigned int memsize_z(DisasContext *dc)
 {
-	return dc->zsize + 1;
+    return dc->zsize + 1;
 }
 
 static inline unsigned int memsize_zz(DisasContext *dc)
 {
-	switch (dc->zzsize)
-	{
-		case 0: return 1;
-		case 1: return 2;
-		default:
-			return 4;
-	}
+    switch (dc->zzsize) {
+    case 0: return 1;
+    case 1: return 2;
+    default:
+        return 4;
+    }
 }
 
 static inline void do_postinc (DisasContext *dc, int size)
 {
-	if (dc->postinc)
-		tcg_gen_addi_tl(cpu_R[dc->op1], cpu_R[dc->op1], size);
+    if (dc->postinc) {
+        tcg_gen_addi_tl(cpu_R[dc->op1], cpu_R[dc->op1], size);
+    }
 }
 
 static inline void dec_prep_move_r(DisasContext *dc, int rs, int rd,
-				   int size, int s_ext, TCGv dst)
+                   int size, int s_ext, TCGv dst)
 {
-	if (s_ext)
-		t_gen_sext(dst, cpu_R[rs], size);
-	else
-		t_gen_zext(dst, cpu_R[rs], size);
+    if (s_ext) {
+        t_gen_sext(dst, cpu_R[rs], size);
+    } else {
+        t_gen_zext(dst, cpu_R[rs], size);
+    }
 }
 
 /* Prepare T0 and T1 for a register alu operation.
    s_ext decides if the operand1 should be sign-extended or zero-extended when
    needed.  */
 static void dec_prep_alu_r(DisasContext *dc, int rs, int rd,
-			  int size, int s_ext, TCGv dst, TCGv src)
+              int size, int s_ext, TCGv dst, TCGv src)
 {
-	dec_prep_move_r(dc, rs, rd, size, s_ext, src);
+    dec_prep_move_r(dc, rs, rd, size, s_ext, src);
 
-	if (s_ext)
-		t_gen_sext(dst, cpu_R[rd], size);
-	else
-		t_gen_zext(dst, cpu_R[rd], size);
+    if (s_ext) {
+        t_gen_sext(dst, cpu_R[rd], size);
+    } else {
+        t_gen_zext(dst, cpu_R[rd], size);
+    }
 }
 
-static int dec_prep_move_m(DisasContext *dc, int s_ext, int memsize,
-			   TCGv dst)
+static int dec_prep_move_m(CPUCRISState *env, DisasContext *dc,
+                           int s_ext, int memsize, TCGv dst)
 {
-	unsigned int rs;
-	uint32_t imm;
-	int is_imm;
-	int insn_len = 2;
+    unsigned int rs;
+    uint32_t imm;
+    int is_imm;
+    int insn_len = 2;
 
-	rs = dc->op1;
-	is_imm = rs == 15 && dc->postinc;
+    rs = dc->op1;
+    is_imm = rs == 15 && dc->postinc;
 
-	/* Load [$rs] onto T1.  */
-	if (is_imm) {
-		insn_len = 2 + memsize;
-		if (memsize == 1)
-			insn_len++;
+    /* Load [$rs] onto T1.  */
+    if (is_imm) {
+        insn_len = 2 + memsize;
+        if (memsize == 1) {
+            insn_len++;
+        }
 
-		imm = cris_fetch(dc, dc->pc + 2, memsize, s_ext);
-		tcg_gen_movi_tl(dst, imm);
-		dc->postinc = 0;
-	} else {
-		cris_flush_cc_state(dc);
-		gen_load(dc, dst, cpu_R[rs], memsize, 0);
-		if (s_ext)
-			t_gen_sext(dst, dst, memsize);
-		else
-			t_gen_zext(dst, dst, memsize);
-	}
-	return insn_len;
+        imm = cris_fetch(env, dc, dc->pc + 2, memsize, s_ext);
+        tcg_gen_movi_tl(dst, imm);
+        dc->postinc = 0;
+    } else {
+        cris_flush_cc_state(dc);
+        gen_load(dc, dst, cpu_R[rs], memsize, 0);
+        if (s_ext) {
+            t_gen_sext(dst, dst, memsize);
+        } else {
+            t_gen_zext(dst, dst, memsize);
+        }
+    }
+    return insn_len;
 }
 
 /* Prepare T0 and T1 for a memory + alu operation.
    s_ext decides if the operand1 should be sign-extended or zero-extended when
    needed.  */
-static int dec_prep_alu_m(DisasContext *dc, int s_ext, int memsize,
-			  TCGv dst, TCGv src)
+static int dec_prep_alu_m(CPUCRISState *env, DisasContext *dc,
+                          int s_ext, int memsize, TCGv dst, TCGv src)
 {
-	int insn_len;
+    int insn_len;
 
-	insn_len = dec_prep_move_m(dc, s_ext, memsize, src);
-	tcg_gen_mov_tl(dst, cpu_R[dc->op2]);
-	return insn_len;
+    insn_len = dec_prep_move_m(env, dc, s_ext, memsize, src);
+    tcg_gen_mov_tl(dst, cpu_R[dc->op2]);
+    return insn_len;
 }
 
 #if DISAS_CRIS
 static const char *cc_name(int cc)
 {
-	static const char *cc_names[16] = {
-		"cc", "cs", "ne", "eq", "vc", "vs", "pl", "mi",
-		"ls", "hi", "ge", "lt", "gt", "le", "a", "p"
-	};
-	assert(cc < 16);
-	return cc_names[cc];
+    static const char *cc_names[16] = {
+        "cc", "cs", "ne", "eq", "vc", "vs", "pl", "mi",
+        "ls", "hi", "ge", "lt", "gt", "le", "a", "p"
+    };
+    assert(cc < 16);
+    return cc_names[cc];
 }
 #endif
 
 /* Start of insn decoders.  */
 
-static int dec_bccq(DisasContext *dc)
+static int dec_bccq(CPUCRISState *env, DisasContext *dc)
 {
-	int32_t offset;
-	int sign;
-	uint32_t cond = dc->op2;
+    int32_t offset;
+    int sign;
+    uint32_t cond = dc->op2;
 
-	offset = EXTRACT_FIELD (dc->ir, 1, 7);
-	sign = EXTRACT_FIELD(dc->ir, 0, 0);
+    offset = EXTRACT_FIELD(dc->ir, 1, 7);
+    sign = EXTRACT_FIELD(dc->ir, 0, 0);
 
-	offset *= 2;
-	offset |= sign << 8;
-	offset = sign_extend(offset, 8);
+    offset *= 2;
+    offset |= sign << 8;
+    offset = sign_extend(offset, 8);
 
-	LOG_DIS("b%s %x\n", cc_name(cond), dc->pc + offset);
+    LOG_DIS("b%s %x\n", cc_name(cond), dc->pc + offset);
 
-	/* op2 holds the condition-code.  */
-	cris_cc_mask(dc, 0);
-	cris_prepare_cc_branch (dc, offset, cond);
-	return 2;
+    /* op2 holds the condition-code.  */
+    cris_cc_mask(dc, 0);
+    cris_prepare_cc_branch(dc, offset, cond);
+    return 2;
 }
-static int dec_addoq(DisasContext *dc)
+static int dec_addoq(CPUCRISState *env, DisasContext *dc)
 {
-	int32_t imm;
+    int32_t imm;
 
-	dc->op1 = EXTRACT_FIELD(dc->ir, 0, 7);
-	imm = sign_extend(dc->op1, 7);
+    dc->op1 = EXTRACT_FIELD(dc->ir, 0, 7);
+    imm = sign_extend(dc->op1, 7);
 
-	LOG_DIS("addoq %d, $r%u\n", imm, dc->op2);
-	cris_cc_mask(dc, 0);
-	/* Fetch register operand,  */
-	tcg_gen_addi_tl(cpu_R[R_ACR], cpu_R[dc->op2], imm);
+    LOG_DIS("addoq %d, $r%u\n", imm, dc->op2);
+    cris_cc_mask(dc, 0);
+    /* Fetch register operand,  */
+    tcg_gen_addi_tl(cpu_R[R_ACR], cpu_R[dc->op2], imm);
 
-	return 2;
+    return 2;
 }
-static int dec_addq(DisasContext *dc)
+static int dec_addq(CPUCRISState *env, DisasContext *dc)
 {
-	LOG_DIS("addq %u, $r%u\n", dc->op1, dc->op2);
+    LOG_DIS("addq %u, $r%u\n", dc->op1, dc->op2);
 
-	dc->op1 = EXTRACT_FIELD(dc->ir, 0, 5);
+    dc->op1 = EXTRACT_FIELD(dc->ir, 0, 5);
 
-	cris_cc_mask(dc, CC_MASK_NZVC);
+    cris_cc_mask(dc, CC_MASK_NZVC);
 
-	cris_alu(dc, CC_OP_ADD,
-		    cpu_R[dc->op2], cpu_R[dc->op2], tcg_const_tl(dc->op1), 4);
-	return 2;
+    cris_alu(dc, CC_OP_ADD,
+            cpu_R[dc->op2], cpu_R[dc->op2], tcg_const_tl(dc->op1), 4);
+    return 2;
 }
-static int dec_moveq(DisasContext *dc)
+static int dec_moveq(CPUCRISState *env, DisasContext *dc)
 {
-	uint32_t imm;
+    uint32_t imm;
 
-	dc->op1 = EXTRACT_FIELD(dc->ir, 0, 5);
-	imm = sign_extend(dc->op1, 5);
-	LOG_DIS("moveq %d, $r%u\n", imm, dc->op2);
+    dc->op1 = EXTRACT_FIELD(dc->ir, 0, 5);
+    imm = sign_extend(dc->op1, 5);
+    LOG_DIS("moveq %d, $r%u\n", imm, dc->op2);
 
-	tcg_gen_movi_tl(cpu_R[dc->op2], imm);
-	return 2;
+    tcg_gen_movi_tl(cpu_R[dc->op2], imm);
+    return 2;
 }
-static int dec_subq(DisasContext *dc)
+static int dec_subq(CPUCRISState *env, DisasContext *dc)
 {
-	dc->op1 = EXTRACT_FIELD(dc->ir, 0, 5);
+    dc->op1 = EXTRACT_FIELD(dc->ir, 0, 5);
 
-	LOG_DIS("subq %u, $r%u\n", dc->op1, dc->op2);
+    LOG_DIS("subq %u, $r%u\n", dc->op1, dc->op2);
 
-	cris_cc_mask(dc, CC_MASK_NZVC);
-	cris_alu(dc, CC_OP_SUB,
-		    cpu_R[dc->op2], cpu_R[dc->op2], tcg_const_tl(dc->op1), 4);
-	return 2;
+    cris_cc_mask(dc, CC_MASK_NZVC);
+    cris_alu(dc, CC_OP_SUB,
+            cpu_R[dc->op2], cpu_R[dc->op2], tcg_const_tl(dc->op1), 4);
+    return 2;
 }
-static int dec_cmpq(DisasContext *dc)
+static int dec_cmpq(CPUCRISState *env, DisasContext *dc)
 {
-	uint32_t imm;
-	dc->op1 = EXTRACT_FIELD(dc->ir, 0, 5);
-	imm = sign_extend(dc->op1, 5);
+    uint32_t imm;
+    dc->op1 = EXTRACT_FIELD(dc->ir, 0, 5);
+    imm = sign_extend(dc->op1, 5);
 
-	LOG_DIS("cmpq %d, $r%d\n", imm, dc->op2);
-	cris_cc_mask(dc, CC_MASK_NZVC);
+    LOG_DIS("cmpq %d, $r%d\n", imm, dc->op2);
+    cris_cc_mask(dc, CC_MASK_NZVC);
 
-	cris_alu(dc, CC_OP_CMP,
-		    cpu_R[dc->op2], cpu_R[dc->op2], tcg_const_tl(imm), 4);
-	return 2;
+    cris_alu(dc, CC_OP_CMP,
+            cpu_R[dc->op2], cpu_R[dc->op2], tcg_const_tl(imm), 4);
+    return 2;
 }
-static int dec_andq(DisasContext *dc)
+static int dec_andq(CPUCRISState *env, DisasContext *dc)
 {
-	uint32_t imm;
-	dc->op1 = EXTRACT_FIELD(dc->ir, 0, 5);
-	imm = sign_extend(dc->op1, 5);
+    uint32_t imm;
+    dc->op1 = EXTRACT_FIELD(dc->ir, 0, 5);
+    imm = sign_extend(dc->op1, 5);
 
-	LOG_DIS("andq %d, $r%d\n", imm, dc->op2);
-	cris_cc_mask(dc, CC_MASK_NZ);
+    LOG_DIS("andq %d, $r%d\n", imm, dc->op2);
+    cris_cc_mask(dc, CC_MASK_NZ);
 
-	cris_alu(dc, CC_OP_AND,
-		    cpu_R[dc->op2], cpu_R[dc->op2], tcg_const_tl(imm), 4);
-	return 2;
+    cris_alu(dc, CC_OP_AND,
+            cpu_R[dc->op2], cpu_R[dc->op2], tcg_const_tl(imm), 4);
+    return 2;
 }
-static int dec_orq(DisasContext *dc)
+static int dec_orq(CPUCRISState *env, DisasContext *dc)
 {
-	uint32_t imm;
-	dc->op1 = EXTRACT_FIELD(dc->ir, 0, 5);
-	imm = sign_extend(dc->op1, 5);
-	LOG_DIS("orq %d, $r%d\n", imm, dc->op2);
-	cris_cc_mask(dc, CC_MASK_NZ);
+    uint32_t imm;
+    dc->op1 = EXTRACT_FIELD(dc->ir, 0, 5);
+    imm = sign_extend(dc->op1, 5);
+    LOG_DIS("orq %d, $r%d\n", imm, dc->op2);
+    cris_cc_mask(dc, CC_MASK_NZ);
 
-	cris_alu(dc, CC_OP_OR,
-		    cpu_R[dc->op2], cpu_R[dc->op2], tcg_const_tl(imm), 4);
-	return 2;
+    cris_alu(dc, CC_OP_OR,
+            cpu_R[dc->op2], cpu_R[dc->op2], tcg_const_tl(imm), 4);
+    return 2;
 }
-static int dec_btstq(DisasContext *dc)
+static int dec_btstq(CPUCRISState *env, DisasContext *dc)
 {
-	dc->op1 = EXTRACT_FIELD(dc->ir, 0, 4);
-	LOG_DIS("btstq %u, $r%d\n", dc->op1, dc->op2);
+    dc->op1 = EXTRACT_FIELD(dc->ir, 0, 4);
+    LOG_DIS("btstq %u, $r%d\n", dc->op1, dc->op2);
 
-	cris_cc_mask(dc, CC_MASK_NZ);
-	cris_evaluate_flags(dc);
-	gen_helper_btst(cpu_PR[PR_CCS], cpu_R[dc->op2],
-			tcg_const_tl(dc->op1), cpu_PR[PR_CCS]);
-	cris_alu(dc, CC_OP_MOVE,
-		 cpu_R[dc->op2], cpu_R[dc->op2], cpu_R[dc->op2], 4);
-	cris_update_cc_op(dc, CC_OP_FLAGS, 4);
-	dc->flags_uptodate = 1;
-	return 2;
+    cris_cc_mask(dc, CC_MASK_NZ);
+    cris_evaluate_flags(dc);
+        gen_helper_btst(cpu_PR[PR_CCS], cpu_env, cpu_R[dc->op2],
+            tcg_const_tl(dc->op1), cpu_PR[PR_CCS]);
+    cris_alu(dc, CC_OP_MOVE,
+         cpu_R[dc->op2], cpu_R[dc->op2], cpu_R[dc->op2], 4);
+    cris_update_cc_op(dc, CC_OP_FLAGS, 4);
+    dc->flags_uptodate = 1;
+    return 2;
 }
-static int dec_asrq(DisasContext *dc)
+static int dec_asrq(CPUCRISState *env, DisasContext *dc)
 {
-	dc->op1 = EXTRACT_FIELD(dc->ir, 0, 4);
-	LOG_DIS("asrq %u, $r%d\n", dc->op1, dc->op2);
-	cris_cc_mask(dc, CC_MASK_NZ);
+    dc->op1 = EXTRACT_FIELD(dc->ir, 0, 4);
+    LOG_DIS("asrq %u, $r%d\n", dc->op1, dc->op2);
+    cris_cc_mask(dc, CC_MASK_NZ);
 
-	tcg_gen_sari_tl(cpu_R[dc->op2], cpu_R[dc->op2], dc->op1);
-	cris_alu(dc, CC_OP_MOVE,
-		    cpu_R[dc->op2],
-		    cpu_R[dc->op2], cpu_R[dc->op2], 4);
-	return 2;
+    tcg_gen_sari_tl(cpu_R[dc->op2], cpu_R[dc->op2], dc->op1);
+    cris_alu(dc, CC_OP_MOVE,
+            cpu_R[dc->op2],
+            cpu_R[dc->op2], cpu_R[dc->op2], 4);
+    return 2;
 }
-static int dec_lslq(DisasContext *dc)
+static int dec_lslq(CPUCRISState *env, DisasContext *dc)
 {
-	dc->op1 = EXTRACT_FIELD(dc->ir, 0, 4);
-	LOG_DIS("lslq %u, $r%d\n", dc->op1, dc->op2);
+    dc->op1 = EXTRACT_FIELD(dc->ir, 0, 4);
+    LOG_DIS("lslq %u, $r%d\n", dc->op1, dc->op2);
 
-	cris_cc_mask(dc, CC_MASK_NZ);
+    cris_cc_mask(dc, CC_MASK_NZ);
 
-	tcg_gen_shli_tl(cpu_R[dc->op2], cpu_R[dc->op2], dc->op1);
+    tcg_gen_shli_tl(cpu_R[dc->op2], cpu_R[dc->op2], dc->op1);
 
-	cris_alu(dc, CC_OP_MOVE,
-		    cpu_R[dc->op2],
-		    cpu_R[dc->op2], cpu_R[dc->op2], 4);
-	return 2;
+    cris_alu(dc, CC_OP_MOVE,
+            cpu_R[dc->op2],
+            cpu_R[dc->op2], cpu_R[dc->op2], 4);
+    return 2;
 }
-static int dec_lsrq(DisasContext *dc)
+static int dec_lsrq(CPUCRISState *env, DisasContext *dc)
 {
-	dc->op1 = EXTRACT_FIELD(dc->ir, 0, 4);
-	LOG_DIS("lsrq %u, $r%d\n", dc->op1, dc->op2);
+    dc->op1 = EXTRACT_FIELD(dc->ir, 0, 4);
+    LOG_DIS("lsrq %u, $r%d\n", dc->op1, dc->op2);
 
-	cris_cc_mask(dc, CC_MASK_NZ);
+    cris_cc_mask(dc, CC_MASK_NZ);
 
-	tcg_gen_shri_tl(cpu_R[dc->op2], cpu_R[dc->op2], dc->op1);
-	cris_alu(dc, CC_OP_MOVE,
-		    cpu_R[dc->op2],
-		    cpu_R[dc->op2], cpu_R[dc->op2], 4);
-	return 2;
+    tcg_gen_shri_tl(cpu_R[dc->op2], cpu_R[dc->op2], dc->op1);
+    cris_alu(dc, CC_OP_MOVE,
+            cpu_R[dc->op2],
+            cpu_R[dc->op2], cpu_R[dc->op2], 4);
+    return 2;
 }
 
-static int dec_move_r(DisasContext *dc)
+static int dec_move_r(CPUCRISState *env, DisasContext *dc)
 {
-	int size = memsize_zz(dc);
+    int size = memsize_zz(dc);
 
-	LOG_DIS("move.%c $r%u, $r%u\n",
-		    memsize_char(size), dc->op1, dc->op2);
+    LOG_DIS("move.%c $r%u, $r%u\n",
+            memsize_char(size), dc->op1, dc->op2);
 
-	cris_cc_mask(dc, CC_MASK_NZ);
-	if (size == 4) {
-		dec_prep_move_r(dc, dc->op1, dc->op2, size, 0, cpu_R[dc->op2]);
-		cris_cc_mask(dc, CC_MASK_NZ);
-		cris_update_cc_op(dc, CC_OP_MOVE, 4);
-		cris_update_cc_x(dc);
-		cris_update_result(dc, cpu_R[dc->op2]);
-	}
-	else {
-		TCGv t0;
+    cris_cc_mask(dc, CC_MASK_NZ);
+    if (size == 4) {
+        dec_prep_move_r(dc, dc->op1, dc->op2, size, 0, cpu_R[dc->op2]);
+        cris_cc_mask(dc, CC_MASK_NZ);
+        cris_update_cc_op(dc, CC_OP_MOVE, 4);
+        cris_update_cc_x(dc);
+        cris_update_result(dc, cpu_R[dc->op2]);
+    } else {
+        TCGv t0;
 
-		t0 = tcg_temp_new();
-		dec_prep_move_r(dc, dc->op1, dc->op2, size, 0, t0);
-		cris_alu(dc, CC_OP_MOVE,
-			 cpu_R[dc->op2],
-			 cpu_R[dc->op2], t0, size);
-		tcg_temp_free(t0);
-	}
-	return 2;
+        t0 = tcg_temp_new();
+        dec_prep_move_r(dc, dc->op1, dc->op2, size, 0, t0);
+        cris_alu(dc, CC_OP_MOVE,
+             cpu_R[dc->op2],
+             cpu_R[dc->op2], t0, size);
+        tcg_temp_free(t0);
+    }
+    return 2;
 }
 
-static int dec_scc_r(DisasContext *dc)
+static int dec_scc_r(CPUCRISState *env, DisasContext *dc)
 {
-	int cond = dc->op2;
+    int cond = dc->op2;
 
-	LOG_DIS("s%s $r%u\n",
-		    cc_name(cond), dc->op1);
+    LOG_DIS("s%s $r%u\n",
+            cc_name(cond), dc->op1);
 
-	if (cond != CC_A)
-	{
-		int l1;
+    if (cond != CC_A) {
+        int l1;
 
-		gen_tst_cc (dc, cpu_R[dc->op1], cond);
-		l1 = gen_new_label();
-		tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_R[dc->op1], 0, l1);
-		tcg_gen_movi_tl(cpu_R[dc->op1], 1);
-		gen_set_label(l1);
-	}
-	else
-		tcg_gen_movi_tl(cpu_R[dc->op1], 1);
+        gen_tst_cc(dc, cpu_R[dc->op1], cond);
+        l1 = gen_new_label();
+        tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_R[dc->op1], 0, l1);
+        tcg_gen_movi_tl(cpu_R[dc->op1], 1);
+        gen_set_label(l1);
+    } else {
+        tcg_gen_movi_tl(cpu_R[dc->op1], 1);
+    }
 
-	cris_cc_mask(dc, 0);
-	return 2;
+    cris_cc_mask(dc, 0);
+    return 2;
 }
 
 static inline void cris_alu_alloc_temps(DisasContext *dc, int size, TCGv *t)
 {
-	if (size == 4) {
-		t[0] = cpu_R[dc->op2];
-		t[1] = cpu_R[dc->op1];
-	} else {
-		t[0] = tcg_temp_new();
-		t[1] = tcg_temp_new();
-	}
+    if (size == 4) {
+        t[0] = cpu_R[dc->op2];
+        t[1] = cpu_R[dc->op1];
+    } else {
+        t[0] = tcg_temp_new();
+        t[1] = tcg_temp_new();
+    }
 }
 
 static inline void cris_alu_free_temps(DisasContext *dc, int size, TCGv *t)
 {
-	if (size != 4) {
-		tcg_temp_free(t[0]);
-		tcg_temp_free(t[1]);
-	}
+    if (size != 4) {
+        tcg_temp_free(t[0]);
+        tcg_temp_free(t[1]);
+    }
 }
 
-static int dec_and_r(DisasContext *dc)
+static int dec_and_r(CPUCRISState *env, DisasContext *dc)
 {
-	TCGv t[2];
-	int size = memsize_zz(dc);
+    TCGv t[2];
+    int size = memsize_zz(dc);
 
-	LOG_DIS("and.%c $r%u, $r%u\n",
-		    memsize_char(size), dc->op1, dc->op2);
+    LOG_DIS("and.%c $r%u, $r%u\n",
+            memsize_char(size), dc->op1, dc->op2);
 
-	cris_cc_mask(dc, CC_MASK_NZ);
+    cris_cc_mask(dc, CC_MASK_NZ);
 
-	cris_alu_alloc_temps(dc, size, t);
-	dec_prep_alu_r(dc, dc->op1, dc->op2, size, 0, t[0], t[1]);
-	cris_alu(dc, CC_OP_AND, cpu_R[dc->op2], t[0], t[1], size);
-	cris_alu_free_temps(dc, size, t);
-	return 2;
+    cris_alu_alloc_temps(dc, size, t);
+    dec_prep_alu_r(dc, dc->op1, dc->op2, size, 0, t[0], t[1]);
+    cris_alu(dc, CC_OP_AND, cpu_R[dc->op2], t[0], t[1], size);
+    cris_alu_free_temps(dc, size, t);
+    return 2;
 }
 
-static int dec_lz_r(DisasContext *dc)
+static int dec_lz_r(CPUCRISState *env, DisasContext *dc)
 {
-	TCGv t0;
-	LOG_DIS("lz $r%u, $r%u\n",
-		    dc->op1, dc->op2);
-	cris_cc_mask(dc, CC_MASK_NZ);
-	t0 = tcg_temp_new();
-	dec_prep_alu_r(dc, dc->op1, dc->op2, 4, 0, cpu_R[dc->op2], t0);
-	cris_alu(dc, CC_OP_LZ, cpu_R[dc->op2], cpu_R[dc->op2], t0, 4);
-	tcg_temp_free(t0);
-	return 2;
+    TCGv t0;
+    LOG_DIS("lz $r%u, $r%u\n",
+            dc->op1, dc->op2);
+    cris_cc_mask(dc, CC_MASK_NZ);
+    t0 = tcg_temp_new();
+    dec_prep_alu_r(dc, dc->op1, dc->op2, 4, 0, cpu_R[dc->op2], t0);
+    cris_alu(dc, CC_OP_LZ, cpu_R[dc->op2], cpu_R[dc->op2], t0, 4);
+    tcg_temp_free(t0);
+    return 2;
 }
 
-static int dec_lsl_r(DisasContext *dc)
+static int dec_lsl_r(CPUCRISState *env, DisasContext *dc)
 {
-	TCGv t[2];
-	int size = memsize_zz(dc);
+    TCGv t[2];
+    int size = memsize_zz(dc);
 
-	LOG_DIS("lsl.%c $r%u, $r%u\n",
-		    memsize_char(size), dc->op1, dc->op2);
+    LOG_DIS("lsl.%c $r%u, $r%u\n",
+            memsize_char(size), dc->op1, dc->op2);
 
-	cris_cc_mask(dc, CC_MASK_NZ);
-	cris_alu_alloc_temps(dc, size, t);
-	dec_prep_alu_r(dc, dc->op1, dc->op2, size, 0, t[0], t[1]);
-	tcg_gen_andi_tl(t[1], t[1], 63);
-	cris_alu(dc, CC_OP_LSL, cpu_R[dc->op2], t[0], t[1], size);
-	cris_alu_alloc_temps(dc, size, t);
-	return 2;
+    cris_cc_mask(dc, CC_MASK_NZ);
+    cris_alu_alloc_temps(dc, size, t);
+    dec_prep_alu_r(dc, dc->op1, dc->op2, size, 0, t[0], t[1]);
+    tcg_gen_andi_tl(t[1], t[1], 63);
+    cris_alu(dc, CC_OP_LSL, cpu_R[dc->op2], t[0], t[1], size);
+    cris_alu_alloc_temps(dc, size, t);
+    return 2;
 }
 
-static int dec_lsr_r(DisasContext *dc)
+static int dec_lsr_r(CPUCRISState *env, DisasContext *dc)
 {
-	TCGv t[2];
-	int size = memsize_zz(dc);
+    TCGv t[2];
+    int size = memsize_zz(dc);
 
-	LOG_DIS("lsr.%c $r%u, $r%u\n",
-		    memsize_char(size), dc->op1, dc->op2);
+    LOG_DIS("lsr.%c $r%u, $r%u\n",
+            memsize_char(size), dc->op1, dc->op2);
 
-	cris_cc_mask(dc, CC_MASK_NZ);
-	cris_alu_alloc_temps(dc, size, t);
-	dec_prep_alu_r(dc, dc->op1, dc->op2, size, 0, t[0], t[1]);
-	tcg_gen_andi_tl(t[1], t[1], 63);
-	cris_alu(dc, CC_OP_LSR, cpu_R[dc->op2], t[0], t[1], size);
-	cris_alu_free_temps(dc, size, t);
-	return 2;
+    cris_cc_mask(dc, CC_MASK_NZ);
+    cris_alu_alloc_temps(dc, size, t);
+    dec_prep_alu_r(dc, dc->op1, dc->op2, size, 0, t[0], t[1]);
+    tcg_gen_andi_tl(t[1], t[1], 63);
+    cris_alu(dc, CC_OP_LSR, cpu_R[dc->op2], t[0], t[1], size);
+    cris_alu_free_temps(dc, size, t);
+    return 2;
 }
 
-static int dec_asr_r(DisasContext *dc)
+static int dec_asr_r(CPUCRISState *env, DisasContext *dc)
 {
-	TCGv t[2];
-	int size = memsize_zz(dc);
+    TCGv t[2];
+    int size = memsize_zz(dc);
 
-	LOG_DIS("asr.%c $r%u, $r%u\n",
-		    memsize_char(size), dc->op1, dc->op2);
+    LOG_DIS("asr.%c $r%u, $r%u\n",
+            memsize_char(size), dc->op1, dc->op2);
 
-	cris_cc_mask(dc, CC_MASK_NZ);
-	cris_alu_alloc_temps(dc, size, t);
-	dec_prep_alu_r(dc, dc->op1, dc->op2, size, 1, t[0], t[1]);
-	tcg_gen_andi_tl(t[1], t[1], 63);
-	cris_alu(dc, CC_OP_ASR, cpu_R[dc->op2], t[0], t[1], size);
-	cris_alu_free_temps(dc, size, t);
-	return 2;
+    cris_cc_mask(dc, CC_MASK_NZ);
+    cris_alu_alloc_temps(dc, size, t);
+    dec_prep_alu_r(dc, dc->op1, dc->op2, size, 1, t[0], t[1]);
+    tcg_gen_andi_tl(t[1], t[1], 63);
+    cris_alu(dc, CC_OP_ASR, cpu_R[dc->op2], t[0], t[1], size);
+    cris_alu_free_temps(dc, size, t);
+    return 2;
 }
 
-static int dec_muls_r(DisasContext *dc)
+static int dec_muls_r(CPUCRISState *env, DisasContext *dc)
 {
-	TCGv t[2];
-	int size = memsize_zz(dc);
+    TCGv t[2];
+    int size = memsize_zz(dc);
 
-	LOG_DIS("muls.%c $r%u, $r%u\n",
-		    memsize_char(size), dc->op1, dc->op2);
-	cris_cc_mask(dc, CC_MASK_NZV);
-	cris_alu_alloc_temps(dc, size, t);
-	dec_prep_alu_r(dc, dc->op1, dc->op2, size, 1, t[0], t[1]);
+    LOG_DIS("muls.%c $r%u, $r%u\n",
+            memsize_char(size), dc->op1, dc->op2);
+    cris_cc_mask(dc, CC_MASK_NZV);
+    cris_alu_alloc_temps(dc, size, t);
+    dec_prep_alu_r(dc, dc->op1, dc->op2, size, 1, t[0], t[1]);
 
-	cris_alu(dc, CC_OP_MULS, cpu_R[dc->op2], t[0], t[1], 4);
-	cris_alu_free_temps(dc, size, t);
-	return 2;
+    cris_alu(dc, CC_OP_MULS, cpu_R[dc->op2], t[0], t[1], 4);
+    cris_alu_free_temps(dc, size, t);
+    return 2;
 }
 
-static int dec_mulu_r(DisasContext *dc)
+static int dec_mulu_r(CPUCRISState *env, DisasContext *dc)
 {
-	TCGv t[2];
-	int size = memsize_zz(dc);
+    TCGv t[2];
+    int size = memsize_zz(dc);
 
-	LOG_DIS("mulu.%c $r%u, $r%u\n",
-		    memsize_char(size), dc->op1, dc->op2);
-	cris_cc_mask(dc, CC_MASK_NZV);
-	cris_alu_alloc_temps(dc, size, t);
-	dec_prep_alu_r(dc, dc->op1, dc->op2, size, 0, t[0], t[1]);
+    LOG_DIS("mulu.%c $r%u, $r%u\n",
+            memsize_char(size), dc->op1, dc->op2);
+    cris_cc_mask(dc, CC_MASK_NZV);
+    cris_alu_alloc_temps(dc, size, t);
+    dec_prep_alu_r(dc, dc->op1, dc->op2, size, 0, t[0], t[1]);
 
-	cris_alu(dc, CC_OP_MULU, cpu_R[dc->op2], t[0], t[1], 4);
-	cris_alu_alloc_temps(dc, size, t);
-	return 2;
+    cris_alu(dc, CC_OP_MULU, cpu_R[dc->op2], t[0], t[1], 4);
+    cris_alu_alloc_temps(dc, size, t);
+    return 2;
 }
 
 
-static int dec_dstep_r(DisasContext *dc)
+static int dec_dstep_r(CPUCRISState *env, DisasContext *dc)
 {
-	LOG_DIS("dstep $r%u, $r%u\n", dc->op1, dc->op2);
-	cris_cc_mask(dc, CC_MASK_NZ);
-	cris_alu(dc, CC_OP_DSTEP,
-		    cpu_R[dc->op2], cpu_R[dc->op2], cpu_R[dc->op1], 4);
-	return 2;
+    LOG_DIS("dstep $r%u, $r%u\n", dc->op1, dc->op2);
+    cris_cc_mask(dc, CC_MASK_NZ);
+    cris_alu(dc, CC_OP_DSTEP,
+            cpu_R[dc->op2], cpu_R[dc->op2], cpu_R[dc->op1], 4);
+    return 2;
 }
 
-static int dec_xor_r(DisasContext *dc)
+static int dec_xor_r(CPUCRISState *env, DisasContext *dc)
 {
-	TCGv t[2];
-	int size = memsize_zz(dc);
-	LOG_DIS("xor.%c $r%u, $r%u\n",
-		    memsize_char(size), dc->op1, dc->op2);
-	BUG_ON(size != 4); /* xor is dword.  */
-	cris_cc_mask(dc, CC_MASK_NZ);
-	cris_alu_alloc_temps(dc, size, t);
-	dec_prep_alu_r(dc, dc->op1, dc->op2, size, 0, t[0], t[1]);
+    TCGv t[2];
+    int size = memsize_zz(dc);
+    LOG_DIS("xor.%c $r%u, $r%u\n",
+            memsize_char(size), dc->op1, dc->op2);
+    BUG_ON(size != 4); /* xor is dword.  */
+    cris_cc_mask(dc, CC_MASK_NZ);
+    cris_alu_alloc_temps(dc, size, t);
+    dec_prep_alu_r(dc, dc->op1, dc->op2, size, 0, t[0], t[1]);
 
-	cris_alu(dc, CC_OP_XOR, cpu_R[dc->op2], t[0], t[1], 4);
-	cris_alu_free_temps(dc, size, t);
-	return 2;
+    cris_alu(dc, CC_OP_XOR, cpu_R[dc->op2], t[0], t[1], 4);
+    cris_alu_free_temps(dc, size, t);
+    return 2;
 }
 
-static int dec_bound_r(DisasContext *dc)
+static int dec_bound_r(CPUCRISState *env, DisasContext *dc)
 {
-	TCGv l0;
-	int size = memsize_zz(dc);
-	LOG_DIS("bound.%c $r%u, $r%u\n",
-		    memsize_char(size), dc->op1, dc->op2);
-	cris_cc_mask(dc, CC_MASK_NZ);
-	l0 = tcg_temp_local_new();
-	dec_prep_move_r(dc, dc->op1, dc->op2, size, 0, l0);
-	cris_alu(dc, CC_OP_BOUND, cpu_R[dc->op2], cpu_R[dc->op2], l0, 4);
-	tcg_temp_free(l0);
-	return 2;
+    TCGv l0;
+    int size = memsize_zz(dc);
+    LOG_DIS("bound.%c $r%u, $r%u\n",
+            memsize_char(size), dc->op1, dc->op2);
+    cris_cc_mask(dc, CC_MASK_NZ);
+    l0 = tcg_temp_local_new();
+    dec_prep_move_r(dc, dc->op1, dc->op2, size, 0, l0);
+    cris_alu(dc, CC_OP_BOUND, cpu_R[dc->op2], cpu_R[dc->op2], l0, 4);
+    tcg_temp_free(l0);
+    return 2;
 }
 
-static int dec_cmp_r(DisasContext *dc)
+static int dec_cmp_r(CPUCRISState *env, DisasContext *dc)
 {
-	TCGv t[2];
-	int size = memsize_zz(dc);
-	LOG_DIS("cmp.%c $r%u, $r%u\n",
-		    memsize_char(size), dc->op1, dc->op2);
-	cris_cc_mask(dc, CC_MASK_NZVC);
-	cris_alu_alloc_temps(dc, size, t);
-	dec_prep_alu_r(dc, dc->op1, dc->op2, size, 0, t[0], t[1]);
+    TCGv t[2];
+    int size = memsize_zz(dc);
+    LOG_DIS("cmp.%c $r%u, $r%u\n",
+            memsize_char(size), dc->op1, dc->op2);
+    cris_cc_mask(dc, CC_MASK_NZVC);
+    cris_alu_alloc_temps(dc, size, t);
+    dec_prep_alu_r(dc, dc->op1, dc->op2, size, 0, t[0], t[1]);
 
-	cris_alu(dc, CC_OP_CMP, cpu_R[dc->op2], t[0], t[1], size);
-	cris_alu_free_temps(dc, size, t);
-	return 2;
+    cris_alu(dc, CC_OP_CMP, cpu_R[dc->op2], t[0], t[1], size);
+    cris_alu_free_temps(dc, size, t);
+    return 2;
 }
 
-static int dec_abs_r(DisasContext *dc)
+static int dec_abs_r(CPUCRISState *env, DisasContext *dc)
 {
-	TCGv t0;
+    TCGv t0;
 
-	LOG_DIS("abs $r%u, $r%u\n",
-		    dc->op1, dc->op2);
-	cris_cc_mask(dc, CC_MASK_NZ);
+    LOG_DIS("abs $r%u, $r%u\n",
+            dc->op1, dc->op2);
+    cris_cc_mask(dc, CC_MASK_NZ);
 
-	t0 = tcg_temp_new();
-	tcg_gen_sari_tl(t0, cpu_R[dc->op1], 31);
-	tcg_gen_xor_tl(cpu_R[dc->op2], cpu_R[dc->op1], t0);
-	tcg_gen_sub_tl(cpu_R[dc->op2], cpu_R[dc->op2], t0);
-	tcg_temp_free(t0);
+    t0 = tcg_temp_new();
+    tcg_gen_sari_tl(t0, cpu_R[dc->op1], 31);
+    tcg_gen_xor_tl(cpu_R[dc->op2], cpu_R[dc->op1], t0);
+    tcg_gen_sub_tl(cpu_R[dc->op2], cpu_R[dc->op2], t0);
+    tcg_temp_free(t0);
 
-	cris_alu(dc, CC_OP_MOVE,
-		    cpu_R[dc->op2], cpu_R[dc->op2], cpu_R[dc->op2], 4);
-	return 2;
+    cris_alu(dc, CC_OP_MOVE,
+            cpu_R[dc->op2], cpu_R[dc->op2], cpu_R[dc->op2], 4);
+    return 2;
 }
 
-static int dec_add_r(DisasContext *dc)
+static int dec_add_r(CPUCRISState *env, DisasContext *dc)
 {
-	TCGv t[2];
-	int size = memsize_zz(dc);
-	LOG_DIS("add.%c $r%u, $r%u\n",
-		    memsize_char(size), dc->op1, dc->op2);
-	cris_cc_mask(dc, CC_MASK_NZVC);
-	cris_alu_alloc_temps(dc, size, t);
-	dec_prep_alu_r(dc, dc->op1, dc->op2, size, 0, t[0], t[1]);
+    TCGv t[2];
+    int size = memsize_zz(dc);
+    LOG_DIS("add.%c $r%u, $r%u\n",
+            memsize_char(size), dc->op1, dc->op2);
+    cris_cc_mask(dc, CC_MASK_NZVC);
+    cris_alu_alloc_temps(dc, size, t);
+    dec_prep_alu_r(dc, dc->op1, dc->op2, size, 0, t[0], t[1]);
 
-	cris_alu(dc, CC_OP_ADD, cpu_R[dc->op2], t[0], t[1], size);
-	cris_alu_free_temps(dc, size, t);
-	return 2;
+    cris_alu(dc, CC_OP_ADD, cpu_R[dc->op2], t[0], t[1], size);
+    cris_alu_free_temps(dc, size, t);
+    return 2;
 }
 
-static int dec_addc_r(DisasContext *dc)
+static int dec_addc_r(CPUCRISState *env, DisasContext *dc)
 {
-	LOG_DIS("addc $r%u, $r%u\n",
-		    dc->op1, dc->op2);
-	cris_evaluate_flags(dc);
-	/* Set for this insn.  */
-	dc->flagx_known = 1;
-	dc->flags_x = X_FLAG;
+    LOG_DIS("addc $r%u, $r%u\n",
+            dc->op1, dc->op2);
+    cris_evaluate_flags(dc);
+    /* Set for this insn.  */
+    dc->flagx_known = 1;
+    dc->flags_x = X_FLAG;
 
-	cris_cc_mask(dc, CC_MASK_NZVC);
-	cris_alu(dc, CC_OP_ADDC,
-		 cpu_R[dc->op2], cpu_R[dc->op2], cpu_R[dc->op1], 4);
-	return 2;
+    cris_cc_mask(dc, CC_MASK_NZVC);
+    cris_alu(dc, CC_OP_ADDC,
+         cpu_R[dc->op2], cpu_R[dc->op2], cpu_R[dc->op1], 4);
+    return 2;
 }
 
-static int dec_mcp_r(DisasContext *dc)
+static int dec_mcp_r(CPUCRISState *env, DisasContext *dc)
 {
-	LOG_DIS("mcp $p%u, $r%u\n",
-		     dc->op2, dc->op1);
-	cris_evaluate_flags(dc);
-	cris_cc_mask(dc, CC_MASK_RNZV);
-	cris_alu(dc, CC_OP_MCP,
-		    cpu_R[dc->op1], cpu_R[dc->op1], cpu_PR[dc->op2], 4);
-	return 2;
+    LOG_DIS("mcp $p%u, $r%u\n",
+             dc->op2, dc->op1);
+    cris_evaluate_flags(dc);
+    cris_cc_mask(dc, CC_MASK_RNZV);
+    cris_alu(dc, CC_OP_MCP,
+            cpu_R[dc->op1], cpu_R[dc->op1], cpu_PR[dc->op2], 4);
+    return 2;
 }
 
 #if DISAS_CRIS
 static char * swapmode_name(int mode, char *modename) {
-	int i = 0;
-	if (mode & 8)
-		modename[i++] = 'n';
-	if (mode & 4)
-		modename[i++] = 'w';
-	if (mode & 2)
-		modename[i++] = 'b';
-	if (mode & 1)
-		modename[i++] = 'r';
-	modename[i++] = 0;
-	return modename;
+    int i = 0;
+    if (mode & 8) {
+        modename[i++] = 'n';
+    }
+    if (mode & 4) {
+        modename[i++] = 'w';
+    }
+    if (mode & 2) {
+        modename[i++] = 'b';
+    }
+    if (mode & 1) {
+        modename[i++] = 'r';
+    }
+    modename[i++] = 0;
+    return modename;
 }
 #endif
 
-static int dec_swap_r(DisasContext *dc)
+static int dec_swap_r(CPUCRISState *env, DisasContext *dc)
 {
-	TCGv t0;
+    TCGv t0;
 #if DISAS_CRIS
-	char modename[4];
+    char modename[4];
 #endif
-	LOG_DIS("swap%s $r%u\n",
-		     swapmode_name(dc->op2, modename), dc->op1);
-
-	cris_cc_mask(dc, CC_MASK_NZ);
-	t0 = tcg_temp_new();
-	t_gen_mov_TN_reg(t0, dc->op1);
-	if (dc->op2 & 8)
-		tcg_gen_not_tl(t0, t0);
-	if (dc->op2 & 4)
-		t_gen_swapw(t0, t0);
-	if (dc->op2 & 2)
-		t_gen_swapb(t0, t0);
-	if (dc->op2 & 1)
-		t_gen_swapr(t0, t0);
-	cris_alu(dc, CC_OP_MOVE,
-		    cpu_R[dc->op1], cpu_R[dc->op1], t0, 4);
-	tcg_temp_free(t0);
-	return 2;
-}
-
-static int dec_or_r(DisasContext *dc)
-{
-	TCGv t[2];
-	int size = memsize_zz(dc);
-	LOG_DIS("or.%c $r%u, $r%u\n",
-		    memsize_char(size), dc->op1, dc->op2);
-	cris_cc_mask(dc, CC_MASK_NZ);
-	cris_alu_alloc_temps(dc, size, t);
-	dec_prep_alu_r(dc, dc->op1, dc->op2, size, 0, t[0], t[1]);
-	cris_alu(dc, CC_OP_OR, cpu_R[dc->op2], t[0], t[1], size);
-	cris_alu_free_temps(dc, size, t);
-	return 2;
-}
-
-static int dec_addi_r(DisasContext *dc)
-{
-	TCGv t0;
-	LOG_DIS("addi.%c $r%u, $r%u\n",
-		    memsize_char(memsize_zz(dc)), dc->op2, dc->op1);
-	cris_cc_mask(dc, 0);
-	t0 = tcg_temp_new();
-	tcg_gen_shl_tl(t0, cpu_R[dc->op2], tcg_const_tl(dc->zzsize));
-	tcg_gen_add_tl(cpu_R[dc->op1], cpu_R[dc->op1], t0);
-	tcg_temp_free(t0);
-	return 2;
-}
-
-static int dec_addi_acr(DisasContext *dc)
-{
-	TCGv t0;
-	LOG_DIS("addi.%c $r%u, $r%u, $acr\n",
-		  memsize_char(memsize_zz(dc)), dc->op2, dc->op1);
-	cris_cc_mask(dc, 0);
-	t0 = tcg_temp_new();
-	tcg_gen_shl_tl(t0, cpu_R[dc->op2], tcg_const_tl(dc->zzsize));
-	tcg_gen_add_tl(cpu_R[R_ACR], cpu_R[dc->op1], t0);
-	tcg_temp_free(t0);
-	return 2;
-}
-
-static int dec_neg_r(DisasContext *dc)
-{
-	TCGv t[2];
-	int size = memsize_zz(dc);
-	LOG_DIS("neg.%c $r%u, $r%u\n",
-		    memsize_char(size), dc->op1, dc->op2);
-	cris_cc_mask(dc, CC_MASK_NZVC);
-	cris_alu_alloc_temps(dc, size, t);
-	dec_prep_alu_r(dc, dc->op1, dc->op2, size, 0, t[0], t[1]);
-
-	cris_alu(dc, CC_OP_NEG, cpu_R[dc->op2], t[0], t[1], size);
-	cris_alu_free_temps(dc, size, t);
-	return 2;
-}
-
-static int dec_btst_r(DisasContext *dc)
-{
-	LOG_DIS("btst $r%u, $r%u\n",
-		    dc->op1, dc->op2);
-	cris_cc_mask(dc, CC_MASK_NZ);
-	cris_evaluate_flags(dc);
-	gen_helper_btst(cpu_PR[PR_CCS], cpu_R[dc->op2],
-			cpu_R[dc->op1], cpu_PR[PR_CCS]);
-	cris_alu(dc, CC_OP_MOVE, cpu_R[dc->op2],
-		 cpu_R[dc->op2], cpu_R[dc->op2], 4);
-	cris_update_cc_op(dc, CC_OP_FLAGS, 4);
-	dc->flags_uptodate = 1;
-	return 2;
-}
-
-static int dec_sub_r(DisasContext *dc)
-{
-	TCGv t[2];
-	int size = memsize_zz(dc);
-	LOG_DIS("sub.%c $r%u, $r%u\n",
-		    memsize_char(size), dc->op1, dc->op2);
-	cris_cc_mask(dc, CC_MASK_NZVC);
-	cris_alu_alloc_temps(dc, size, t);
-	dec_prep_alu_r(dc, dc->op1, dc->op2, size, 0, t[0], t[1]);
-	cris_alu(dc, CC_OP_SUB, cpu_R[dc->op2], t[0], t[1], size);
-	cris_alu_free_temps(dc, size, t);
-	return 2;
+    LOG_DIS("swap%s $r%u\n",
+             swapmode_name(dc->op2, modename), dc->op1);
+
+    cris_cc_mask(dc, CC_MASK_NZ);
+    t0 = tcg_temp_new();
+    t_gen_mov_TN_reg(t0, dc->op1);
+    if (dc->op2 & 8) {
+        tcg_gen_not_tl(t0, t0);
+    }
+    if (dc->op2 & 4) {
+        t_gen_swapw(t0, t0);
+    }
+    if (dc->op2 & 2) {
+        t_gen_swapb(t0, t0);
+    }
+    if (dc->op2 & 1) {
+        t_gen_swapr(t0, t0);
+    }
+    cris_alu(dc, CC_OP_MOVE, cpu_R[dc->op1], cpu_R[dc->op1], t0, 4);
+    tcg_temp_free(t0);
+    return 2;
+}
+
+static int dec_or_r(CPUCRISState *env, DisasContext *dc)
+{
+    TCGv t[2];
+    int size = memsize_zz(dc);
+    LOG_DIS("or.%c $r%u, $r%u\n",
+            memsize_char(size), dc->op1, dc->op2);
+    cris_cc_mask(dc, CC_MASK_NZ);
+    cris_alu_alloc_temps(dc, size, t);
+    dec_prep_alu_r(dc, dc->op1, dc->op2, size, 0, t[0], t[1]);
+    cris_alu(dc, CC_OP_OR, cpu_R[dc->op2], t[0], t[1], size);
+    cris_alu_free_temps(dc, size, t);
+    return 2;
+}
+
+static int dec_addi_r(CPUCRISState *env, DisasContext *dc)
+{
+    TCGv t0;
+    LOG_DIS("addi.%c $r%u, $r%u\n",
+            memsize_char(memsize_zz(dc)), dc->op2, dc->op1);
+    cris_cc_mask(dc, 0);
+    t0 = tcg_temp_new();
+    tcg_gen_shl_tl(t0, cpu_R[dc->op2], tcg_const_tl(dc->zzsize));
+    tcg_gen_add_tl(cpu_R[dc->op1], cpu_R[dc->op1], t0);
+    tcg_temp_free(t0);
+    return 2;
+}
+
+static int dec_addi_acr(CPUCRISState *env, DisasContext *dc)
+{
+    TCGv t0;
+    LOG_DIS("addi.%c $r%u, $r%u, $acr\n",
+          memsize_char(memsize_zz(dc)), dc->op2, dc->op1);
+    cris_cc_mask(dc, 0);
+    t0 = tcg_temp_new();
+    tcg_gen_shl_tl(t0, cpu_R[dc->op2], tcg_const_tl(dc->zzsize));
+    tcg_gen_add_tl(cpu_R[R_ACR], cpu_R[dc->op1], t0);
+    tcg_temp_free(t0);
+    return 2;
+}
+
+static int dec_neg_r(CPUCRISState *env, DisasContext *dc)
+{
+    TCGv t[2];
+    int size = memsize_zz(dc);
+    LOG_DIS("neg.%c $r%u, $r%u\n",
+            memsize_char(size), dc->op1, dc->op2);
+    cris_cc_mask(dc, CC_MASK_NZVC);
+    cris_alu_alloc_temps(dc, size, t);
+    dec_prep_alu_r(dc, dc->op1, dc->op2, size, 0, t[0], t[1]);
+
+    cris_alu(dc, CC_OP_NEG, cpu_R[dc->op2], t[0], t[1], size);
+    cris_alu_free_temps(dc, size, t);
+    return 2;
+}
+
+static int dec_btst_r(CPUCRISState *env, DisasContext *dc)
+{
+    LOG_DIS("btst $r%u, $r%u\n",
+            dc->op1, dc->op2);
+    cris_cc_mask(dc, CC_MASK_NZ);
+    cris_evaluate_flags(dc);
+        gen_helper_btst(cpu_PR[PR_CCS], cpu_env, cpu_R[dc->op2],
+            cpu_R[dc->op1], cpu_PR[PR_CCS]);
+    cris_alu(dc, CC_OP_MOVE, cpu_R[dc->op2],
+         cpu_R[dc->op2], cpu_R[dc->op2], 4);
+    cris_update_cc_op(dc, CC_OP_FLAGS, 4);
+    dc->flags_uptodate = 1;
+    return 2;
+}
+
+static int dec_sub_r(CPUCRISState *env, DisasContext *dc)
+{
+    TCGv t[2];
+    int size = memsize_zz(dc);
+    LOG_DIS("sub.%c $r%u, $r%u\n",
+            memsize_char(size), dc->op1, dc->op2);
+    cris_cc_mask(dc, CC_MASK_NZVC);
+    cris_alu_alloc_temps(dc, size, t);
+    dec_prep_alu_r(dc, dc->op1, dc->op2, size, 0, t[0], t[1]);
+    cris_alu(dc, CC_OP_SUB, cpu_R[dc->op2], t[0], t[1], size);
+    cris_alu_free_temps(dc, size, t);
+    return 2;
 }
 
 /* Zero extension. From size to dword.  */
-static int dec_movu_r(DisasContext *dc)
+static int dec_movu_r(CPUCRISState *env, DisasContext *dc)
 {
-	TCGv t0;
-	int size = memsize_z(dc);
-	LOG_DIS("movu.%c $r%u, $r%u\n",
-		    memsize_char(size),
-		    dc->op1, dc->op2);
+    TCGv t0;
+    int size = memsize_z(dc);
+    LOG_DIS("movu.%c $r%u, $r%u\n",
+            memsize_char(size),
+            dc->op1, dc->op2);
 
-	cris_cc_mask(dc, CC_MASK_NZ);
-	t0 = tcg_temp_new();
-	dec_prep_move_r(dc, dc->op1, dc->op2, size, 0, t0);
-	cris_alu(dc, CC_OP_MOVE, cpu_R[dc->op2], cpu_R[dc->op2], t0, 4);
-	tcg_temp_free(t0);
-	return 2;
+    cris_cc_mask(dc, CC_MASK_NZ);
+    t0 = tcg_temp_new();
+    dec_prep_move_r(dc, dc->op1, dc->op2, size, 0, t0);
+    cris_alu(dc, CC_OP_MOVE, cpu_R[dc->op2], cpu_R[dc->op2], t0, 4);
+    tcg_temp_free(t0);
+    return 2;
 }
 
 /* Sign extension. From size to dword.  */
-static int dec_movs_r(DisasContext *dc)
+static int dec_movs_r(CPUCRISState *env, DisasContext *dc)
 {
-	TCGv t0;
-	int size = memsize_z(dc);
-	LOG_DIS("movs.%c $r%u, $r%u\n",
-		    memsize_char(size),
-		    dc->op1, dc->op2);
+    TCGv t0;
+    int size = memsize_z(dc);
+    LOG_DIS("movs.%c $r%u, $r%u\n",
+            memsize_char(size),
+            dc->op1, dc->op2);
 
-	cris_cc_mask(dc, CC_MASK_NZ);
-	t0 = tcg_temp_new();
-	/* Size can only be qi or hi.  */
-	t_gen_sext(t0, cpu_R[dc->op1], size);
-	cris_alu(dc, CC_OP_MOVE,
-		    cpu_R[dc->op2], cpu_R[dc->op1], t0, 4);
-	tcg_temp_free(t0);
-	return 2;
+    cris_cc_mask(dc, CC_MASK_NZ);
+    t0 = tcg_temp_new();
+    /* Size can only be qi or hi.  */
+    t_gen_sext(t0, cpu_R[dc->op1], size);
+    cris_alu(dc, CC_OP_MOVE,
+            cpu_R[dc->op2], cpu_R[dc->op1], t0, 4);
+    tcg_temp_free(t0);
+    return 2;
 }
 
 /* zero extension. From size to dword.  */
-static int dec_addu_r(DisasContext *dc)
+static int dec_addu_r(CPUCRISState *env, DisasContext *dc)
 {
-	TCGv t0;
-	int size = memsize_z(dc);
-	LOG_DIS("addu.%c $r%u, $r%u\n",
-		    memsize_char(size),
-		    dc->op1, dc->op2);
+    TCGv t0;
+    int size = memsize_z(dc);
+    LOG_DIS("addu.%c $r%u, $r%u\n",
+            memsize_char(size),
+            dc->op1, dc->op2);
 
-	cris_cc_mask(dc, CC_MASK_NZVC);
-	t0 = tcg_temp_new();
-	/* Size can only be qi or hi.  */
-	t_gen_zext(t0, cpu_R[dc->op1], size);
-	cris_alu(dc, CC_OP_ADD,
-		    cpu_R[dc->op2], cpu_R[dc->op2], t0, 4);
-	tcg_temp_free(t0);
-	return 2;
+    cris_cc_mask(dc, CC_MASK_NZVC);
+    t0 = tcg_temp_new();
+    /* Size can only be qi or hi.  */
+    t_gen_zext(t0, cpu_R[dc->op1], size);
+    cris_alu(dc, CC_OP_ADD, cpu_R[dc->op2], cpu_R[dc->op2], t0, 4);
+    tcg_temp_free(t0);
+    return 2;
 }
 
 /* Sign extension. From size to dword.  */
-static int dec_adds_r(DisasContext *dc)
+static int dec_adds_r(CPUCRISState *env, DisasContext *dc)
 {
-	TCGv t0;
-	int size = memsize_z(dc);
-	LOG_DIS("adds.%c $r%u, $r%u\n",
-		    memsize_char(size),
-		    dc->op1, dc->op2);
+    TCGv t0;
+    int size = memsize_z(dc);
+    LOG_DIS("adds.%c $r%u, $r%u\n",
+            memsize_char(size),
+            dc->op1, dc->op2);
 
-	cris_cc_mask(dc, CC_MASK_NZVC);
-	t0 = tcg_temp_new();
-	/* Size can only be qi or hi.  */
-	t_gen_sext(t0, cpu_R[dc->op1], size);
-	cris_alu(dc, CC_OP_ADD,
-		    cpu_R[dc->op2], cpu_R[dc->op2], t0, 4);
-	tcg_temp_free(t0);
-	return 2;
+    cris_cc_mask(dc, CC_MASK_NZVC);
+    t0 = tcg_temp_new();
+    /* Size can only be qi or hi.  */
+    t_gen_sext(t0, cpu_R[dc->op1], size);
+    cris_alu(dc, CC_OP_ADD,
+            cpu_R[dc->op2], cpu_R[dc->op2], t0, 4);
+    tcg_temp_free(t0);
+    return 2;
 }
 
 /* Zero extension. From size to dword.  */
-static int dec_subu_r(DisasContext *dc)
+static int dec_subu_r(CPUCRISState *env, DisasContext *dc)
 {
-	TCGv t0;
-	int size = memsize_z(dc);
-	LOG_DIS("subu.%c $r%u, $r%u\n",
-		    memsize_char(size),
-		    dc->op1, dc->op2);
+    TCGv t0;
+    int size = memsize_z(dc);
+    LOG_DIS("subu.%c $r%u, $r%u\n",
+            memsize_char(size),
+            dc->op1, dc->op2);
 
-	cris_cc_mask(dc, CC_MASK_NZVC);
-	t0 = tcg_temp_new();
-	/* Size can only be qi or hi.  */
-	t_gen_zext(t0, cpu_R[dc->op1], size);
-	cris_alu(dc, CC_OP_SUB,
-		    cpu_R[dc->op2], cpu_R[dc->op2], t0, 4);
-	tcg_temp_free(t0);
-	return 2;
+    cris_cc_mask(dc, CC_MASK_NZVC);
+    t0 = tcg_temp_new();
+    /* Size can only be qi or hi.  */
+    t_gen_zext(t0, cpu_R[dc->op1], size);
+    cris_alu(dc, CC_OP_SUB,
+            cpu_R[dc->op2], cpu_R[dc->op2], t0, 4);
+    tcg_temp_free(t0);
+    return 2;
 }
 
 /* Sign extension. From size to dword.  */
-static int dec_subs_r(DisasContext *dc)
-{
-	TCGv t0;
-	int size = memsize_z(dc);
-	LOG_DIS("subs.%c $r%u, $r%u\n",
-		    memsize_char(size),
-		    dc->op1, dc->op2);
-
-	cris_cc_mask(dc, CC_MASK_NZVC);
-	t0 = tcg_temp_new();
-	/* Size can only be qi or hi.  */
-	t_gen_sext(t0, cpu_R[dc->op1], size);
-	cris_alu(dc, CC_OP_SUB,
-		    cpu_R[dc->op2], cpu_R[dc->op2], t0, 4);
-	tcg_temp_free(t0);
-	return 2;
-}
-
-static int dec_setclrf(DisasContext *dc)
-{
-	uint32_t flags;
-	int set = (~dc->opcode >> 2) & 1;
-
-
-	flags = (EXTRACT_FIELD(dc->ir, 12, 15) << 4)
-		| EXTRACT_FIELD(dc->ir, 0, 3);
-	if (set && flags == 0) {
-		LOG_DIS("nop\n");
-		return 2;
-	} else if (!set && (flags & 0x20)) {
-		LOG_DIS("di\n");
-	}
-	else {
-		LOG_DIS("%sf %x\n",
-			     set ? "set" : "clr",
-			    flags);
-	}
-
-	/* User space is not allowed to touch these. Silently ignore.  */
-	if (dc->tb_flags & U_FLAG) {
-		flags &= ~(S_FLAG | I_FLAG | U_FLAG);
-	}
-
-	if (flags & X_FLAG) {
-		dc->flagx_known = 1;
-		if (set)
-			dc->flags_x = X_FLAG;
-		else
-			dc->flags_x = 0;
-	}
-
-	/* Break the TB if any of the SPI flag changes.  */
-	if (flags & (P_FLAG | S_FLAG)) {
-		tcg_gen_movi_tl(env_pc, dc->pc + 2);
-		dc->is_jmp = DISAS_UPDATE;
-		dc->cpustate_changed = 1;
-	}
-
-	/* For the I flag, only act on posedge.  */
-	if ((flags & I_FLAG)) {
-		tcg_gen_movi_tl(env_pc, dc->pc + 2);
-		dc->is_jmp = DISAS_UPDATE;
-		dc->cpustate_changed = 1;
-	}
-
-
-	/* Simply decode the flags.  */
-	cris_evaluate_flags (dc);
-	cris_update_cc_op(dc, CC_OP_FLAGS, 4);
-	cris_update_cc_x(dc);
-	tcg_gen_movi_tl(cc_op, dc->cc_op);
-
-	if (set) {
-		if (!(dc->tb_flags & U_FLAG) && (flags & U_FLAG)) {
-			/* Enter user mode.  */
-			t_gen_mov_env_TN(ksp, cpu_R[R_SP]);
-			tcg_gen_mov_tl(cpu_R[R_SP], cpu_PR[PR_USP]);
-			dc->cpustate_changed = 1;
-		}
-		tcg_gen_ori_tl(cpu_PR[PR_CCS], cpu_PR[PR_CCS], flags);
-	}
-	else
-		tcg_gen_andi_tl(cpu_PR[PR_CCS], cpu_PR[PR_CCS], ~flags);
-
-	dc->flags_uptodate = 1;
-	dc->clear_x = 0;
-	return 2;
-}
-
-static int dec_move_rs(DisasContext *dc)
-{
-	LOG_DIS("move $r%u, $s%u\n", dc->op1, dc->op2);
-	cris_cc_mask(dc, 0);
-	gen_helper_movl_sreg_reg(tcg_const_tl(dc->op2), tcg_const_tl(dc->op1));
-	return 2;
-}
-static int dec_move_sr(DisasContext *dc)
-{
-	LOG_DIS("move $s%u, $r%u\n", dc->op2, dc->op1);
-	cris_cc_mask(dc, 0);
-	gen_helper_movl_reg_sreg(tcg_const_tl(dc->op1), tcg_const_tl(dc->op2));
-	return 2;
-}
-
-static int dec_move_rp(DisasContext *dc)
-{
-	TCGv t[2];
-	LOG_DIS("move $r%u, $p%u\n", dc->op1, dc->op2);
-	cris_cc_mask(dc, 0);
-
-	t[0] = tcg_temp_new();
-	if (dc->op2 == PR_CCS) {
-		cris_evaluate_flags(dc);
-		t_gen_mov_TN_reg(t[0], dc->op1);
-		if (dc->tb_flags & U_FLAG) {
-			t[1] = tcg_temp_new();
-			/* User space is not allowed to touch all flags.  */
-			tcg_gen_andi_tl(t[0], t[0], 0x39f);
-			tcg_gen_andi_tl(t[1], cpu_PR[PR_CCS], ~0x39f);
-			tcg_gen_or_tl(t[0], t[1], t[0]);
-			tcg_temp_free(t[1]);
-		}
-	}
-	else
-		t_gen_mov_TN_reg(t[0], dc->op1);
-
-	t_gen_mov_preg_TN(dc, dc->op2, t[0]);
-	if (dc->op2 == PR_CCS) {
-		cris_update_cc_op(dc, CC_OP_FLAGS, 4);
-		dc->flags_uptodate = 1;
-	}
-	tcg_temp_free(t[0]);
-	return 2;
-}
-static int dec_move_pr(DisasContext *dc)
-{
-	TCGv t0;
-	LOG_DIS("move $p%u, $r%u\n", dc->op2, dc->op1);
-	cris_cc_mask(dc, 0);
-
-	if (dc->op2 == PR_CCS)
-		cris_evaluate_flags(dc);
-
-        if (dc->op2 == PR_DZ) {
-		tcg_gen_movi_tl(cpu_R[dc->op1], 0);
+static int dec_subs_r(CPUCRISState *env, DisasContext *dc)
+{
+    TCGv t0;
+    int size = memsize_z(dc);
+    LOG_DIS("subs.%c $r%u, $r%u\n",
+            memsize_char(size),
+            dc->op1, dc->op2);
+
+    cris_cc_mask(dc, CC_MASK_NZVC);
+    t0 = tcg_temp_new();
+    /* Size can only be qi or hi.  */
+    t_gen_sext(t0, cpu_R[dc->op1], size);
+    cris_alu(dc, CC_OP_SUB,
+            cpu_R[dc->op2], cpu_R[dc->op2], t0, 4);
+    tcg_temp_free(t0);
+    return 2;
+}
+
+static int dec_setclrf(CPUCRISState *env, DisasContext *dc)
+{
+    uint32_t flags;
+    int set = (~dc->opcode >> 2) & 1;
+
+
+    flags = (EXTRACT_FIELD(dc->ir, 12, 15) << 4)
+        | EXTRACT_FIELD(dc->ir, 0, 3);
+    if (set && flags == 0) {
+        LOG_DIS("nop\n");
+        return 2;
+    } else if (!set && (flags & 0x20)) {
+        LOG_DIS("di\n");
+    } else {
+        LOG_DIS("%sf %x\n", set ? "set" : "clr", flags);
+    }
+
+    /* User space is not allowed to touch these. Silently ignore.  */
+    if (dc->tb_flags & U_FLAG) {
+        flags &= ~(S_FLAG | I_FLAG | U_FLAG);
+    }
+
+    if (flags & X_FLAG) {
+        dc->flagx_known = 1;
+        if (set) {
+            dc->flags_x = X_FLAG;
         } else {
-		t0 = tcg_temp_new();
-		t_gen_mov_TN_preg(t0, dc->op2);
-		cris_alu(dc, CC_OP_MOVE,
-			 cpu_R[dc->op1], cpu_R[dc->op1], t0,
-			 preg_sizes[dc->op2]);
-		tcg_temp_free(t0);
-	}
-	return 2;
-}
-
-static int dec_move_mr(DisasContext *dc)
-{
-	int memsize = memsize_zz(dc);
-	int insn_len;
-	LOG_DIS("move.%c [$r%u%s, $r%u\n",
-		    memsize_char(memsize),
-		    dc->op1, dc->postinc ? "+]" : "]",
-		    dc->op2);
-
-	if (memsize == 4) {
-		insn_len = dec_prep_move_m(dc, 0, 4, cpu_R[dc->op2]);
-		cris_cc_mask(dc, CC_MASK_NZ);
-		cris_update_cc_op(dc, CC_OP_MOVE, 4);
-		cris_update_cc_x(dc);
-		cris_update_result(dc, cpu_R[dc->op2]);
-	}
-	else {
-		TCGv t0;
-
-		t0 = tcg_temp_new();
-		insn_len = dec_prep_move_m(dc, 0, memsize, t0);
-		cris_cc_mask(dc, CC_MASK_NZ);
-		cris_alu(dc, CC_OP_MOVE,
-			    cpu_R[dc->op2], cpu_R[dc->op2], t0, memsize);
-		tcg_temp_free(t0);
-	}
-	do_postinc(dc, memsize);
-	return insn_len;
+            dc->flags_x = 0;
+        }
+    }
+
+    /* Break the TB if any of the SPI flag changes.  */
+    if (flags & (P_FLAG | S_FLAG)) {
+        tcg_gen_movi_tl(env_pc, dc->pc + 2);
+        dc->is_jmp = DISAS_UPDATE;
+        dc->cpustate_changed = 1;
+    }
+
+    /* For the I flag, only act on posedge.  */
+    if ((flags & I_FLAG)) {
+        tcg_gen_movi_tl(env_pc, dc->pc + 2);
+        dc->is_jmp = DISAS_UPDATE;
+        dc->cpustate_changed = 1;
+    }
+
+
+    /* Simply decode the flags.  */
+    cris_evaluate_flags(dc);
+    cris_update_cc_op(dc, CC_OP_FLAGS, 4);
+    cris_update_cc_x(dc);
+    tcg_gen_movi_tl(cc_op, dc->cc_op);
+
+    if (set) {
+        if (!(dc->tb_flags & U_FLAG) && (flags & U_FLAG)) {
+            /* Enter user mode.  */
+            t_gen_mov_env_TN(ksp, cpu_R[R_SP]);
+            tcg_gen_mov_tl(cpu_R[R_SP], cpu_PR[PR_USP]);
+            dc->cpustate_changed = 1;
+        }
+        tcg_gen_ori_tl(cpu_PR[PR_CCS], cpu_PR[PR_CCS], flags);
+    } else {
+        tcg_gen_andi_tl(cpu_PR[PR_CCS], cpu_PR[PR_CCS], ~flags);
+    }
+
+    dc->flags_uptodate = 1;
+    dc->clear_x = 0;
+    return 2;
+}
+
+static int dec_move_rs(CPUCRISState *env, DisasContext *dc)
+{
+    LOG_DIS("move $r%u, $s%u\n", dc->op1, dc->op2);
+    cris_cc_mask(dc, 0);
+        gen_helper_movl_sreg_reg(cpu_env, tcg_const_tl(dc->op2),
+                                 tcg_const_tl(dc->op1));
+    return 2;
+}
+static int dec_move_sr(CPUCRISState *env, DisasContext *dc)
+{
+    LOG_DIS("move $s%u, $r%u\n", dc->op2, dc->op1);
+    cris_cc_mask(dc, 0);
+        gen_helper_movl_reg_sreg(cpu_env, tcg_const_tl(dc->op1),
+                                 tcg_const_tl(dc->op2));
+    return 2;
+}
+
+static int dec_move_rp(CPUCRISState *env, DisasContext *dc)
+{
+    TCGv t[2];
+    LOG_DIS("move $r%u, $p%u\n", dc->op1, dc->op2);
+    cris_cc_mask(dc, 0);
+
+    t[0] = tcg_temp_new();
+    if (dc->op2 == PR_CCS) {
+        cris_evaluate_flags(dc);
+        t_gen_mov_TN_reg(t[0], dc->op1);
+        if (dc->tb_flags & U_FLAG) {
+            t[1] = tcg_temp_new();
+            /* User space is not allowed to touch all flags.  */
+            tcg_gen_andi_tl(t[0], t[0], 0x39f);
+            tcg_gen_andi_tl(t[1], cpu_PR[PR_CCS], ~0x39f);
+            tcg_gen_or_tl(t[0], t[1], t[0]);
+            tcg_temp_free(t[1]);
+        }
+    } else {
+        t_gen_mov_TN_reg(t[0], dc->op1);
+    }
+
+    t_gen_mov_preg_TN(dc, dc->op2, t[0]);
+    if (dc->op2 == PR_CCS) {
+        cris_update_cc_op(dc, CC_OP_FLAGS, 4);
+        dc->flags_uptodate = 1;
+    }
+    tcg_temp_free(t[0]);
+    return 2;
+}
+static int dec_move_pr(CPUCRISState *env, DisasContext *dc)
+{
+    TCGv t0;
+    LOG_DIS("move $p%u, $r%u\n", dc->op2, dc->op1);
+    cris_cc_mask(dc, 0);
+
+    if (dc->op2 == PR_CCS) {
+        cris_evaluate_flags(dc);
+    }
+
+    if (dc->op2 == PR_DZ) {
+        tcg_gen_movi_tl(cpu_R[dc->op1], 0);
+    } else {
+        t0 = tcg_temp_new();
+        t_gen_mov_TN_preg(t0, dc->op2);
+        cris_alu(dc, CC_OP_MOVE,
+                cpu_R[dc->op1], cpu_R[dc->op1], t0,
+                preg_sizes[dc->op2]);
+        tcg_temp_free(t0);
+    }
+    return 2;
+}
+
+static int dec_move_mr(CPUCRISState *env, DisasContext *dc)
+{
+    int memsize = memsize_zz(dc);
+    int insn_len;
+    LOG_DIS("move.%c [$r%u%s, $r%u\n",
+            memsize_char(memsize),
+            dc->op1, dc->postinc ? "+]" : "]",
+                    dc->op2);
+
+    if (memsize == 4) {
+        insn_len = dec_prep_move_m(env, dc, 0, 4, cpu_R[dc->op2]);
+        cris_cc_mask(dc, CC_MASK_NZ);
+        cris_update_cc_op(dc, CC_OP_MOVE, 4);
+        cris_update_cc_x(dc);
+        cris_update_result(dc, cpu_R[dc->op2]);
+    } else {
+        TCGv t0;
+
+        t0 = tcg_temp_new();
+        insn_len = dec_prep_move_m(env, dc, 0, memsize, t0);
+        cris_cc_mask(dc, CC_MASK_NZ);
+        cris_alu(dc, CC_OP_MOVE,
+                cpu_R[dc->op2], cpu_R[dc->op2], t0, memsize);
+        tcg_temp_free(t0);
+    }
+    do_postinc(dc, memsize);
+    return insn_len;
 }
 
 static inline void cris_alu_m_alloc_temps(TCGv *t)
 {
-	t[0] = tcg_temp_new();
-	t[1] = tcg_temp_new();
+    t[0] = tcg_temp_new();
+    t[1] = tcg_temp_new();
 }
 
 static inline void cris_alu_m_free_temps(TCGv *t)
 {
-	tcg_temp_free(t[0]);
-	tcg_temp_free(t[1]);
-}
-
-static int dec_movs_m(DisasContext *dc)
-{
-	TCGv t[2];
-	int memsize = memsize_z(dc);
-	int insn_len;
-	LOG_DIS("movs.%c [$r%u%s, $r%u\n",
-		    memsize_char(memsize),
-		    dc->op1, dc->postinc ? "+]" : "]",
-		    dc->op2);
-
-	cris_alu_m_alloc_temps(t);
-	/* sign extend.  */
-	insn_len = dec_prep_alu_m(dc, 1, memsize, t[0], t[1]);
-	cris_cc_mask(dc, CC_MASK_NZ);
-	cris_alu(dc, CC_OP_MOVE,
-		    cpu_R[dc->op2], cpu_R[dc->op2], t[1], 4);
-	do_postinc(dc, memsize);
-	cris_alu_m_free_temps(t);
-	return insn_len;
-}
-
-static int dec_addu_m(DisasContext *dc)
-{
-	TCGv t[2];
-	int memsize = memsize_z(dc);
-	int insn_len;
-	LOG_DIS("addu.%c [$r%u%s, $r%u\n",
-		    memsize_char(memsize),
-		    dc->op1, dc->postinc ? "+]" : "]",
-		    dc->op2);
-
-	cris_alu_m_alloc_temps(t);
-	/* sign extend.  */
-	insn_len = dec_prep_alu_m(dc, 0, memsize, t[0], t[1]);
-	cris_cc_mask(dc, CC_MASK_NZVC);
-	cris_alu(dc, CC_OP_ADD,
-		    cpu_R[dc->op2], cpu_R[dc->op2], t[1], 4);
-	do_postinc(dc, memsize);
-	cris_alu_m_free_temps(t);
-	return insn_len;
-}
-
-static int dec_adds_m(DisasContext *dc)
-{
-	TCGv t[2];
-	int memsize = memsize_z(dc);
-	int insn_len;
-	LOG_DIS("adds.%c [$r%u%s, $r%u\n",
-		    memsize_char(memsize),
-		    dc->op1, dc->postinc ? "+]" : "]",
-		    dc->op2);
-
-	cris_alu_m_alloc_temps(t);
-	/* sign extend.  */
-	insn_len = dec_prep_alu_m(dc, 1, memsize, t[0], t[1]);
-	cris_cc_mask(dc, CC_MASK_NZVC);
-	cris_alu(dc, CC_OP_ADD, cpu_R[dc->op2], cpu_R[dc->op2], t[1], 4);
-	do_postinc(dc, memsize);
-	cris_alu_m_free_temps(t);
-	return insn_len;
-}
-
-static int dec_subu_m(DisasContext *dc)
-{
-	TCGv t[2];
-	int memsize = memsize_z(dc);
-	int insn_len;
-	LOG_DIS("subu.%c [$r%u%s, $r%u\n",
-		    memsize_char(memsize),
-		    dc->op1, dc->postinc ? "+]" : "]",
-		    dc->op2);
-
-	cris_alu_m_alloc_temps(t);
-	/* sign extend.  */
-	insn_len = dec_prep_alu_m(dc, 0, memsize, t[0], t[1]);
-	cris_cc_mask(dc, CC_MASK_NZVC);
-	cris_alu(dc, CC_OP_SUB, cpu_R[dc->op2], cpu_R[dc->op2], t[1], 4);
-	do_postinc(dc, memsize);
-	cris_alu_m_free_temps(t);
-	return insn_len;
-}
-
-static int dec_subs_m(DisasContext *dc)
-{
-	TCGv t[2];
-	int memsize = memsize_z(dc);
-	int insn_len;
-	LOG_DIS("subs.%c [$r%u%s, $r%u\n",
-		    memsize_char(memsize),
-		    dc->op1, dc->postinc ? "+]" : "]",
-		    dc->op2);
-
-	cris_alu_m_alloc_temps(t);
-	/* sign extend.  */
-	insn_len = dec_prep_alu_m(dc, 1, memsize, t[0], t[1]);
-	cris_cc_mask(dc, CC_MASK_NZVC);
-	cris_alu(dc, CC_OP_SUB, cpu_R[dc->op2], cpu_R[dc->op2], t[1], 4);
-	do_postinc(dc, memsize);
-	cris_alu_m_free_temps(t);
-	return insn_len;
-}
-
-static int dec_movu_m(DisasContext *dc)
-{
-	TCGv t[2];
-	int memsize = memsize_z(dc);
-	int insn_len;
-
-	LOG_DIS("movu.%c [$r%u%s, $r%u\n",
-		    memsize_char(memsize),
-		    dc->op1, dc->postinc ? "+]" : "]",
-		    dc->op2);
-
-	cris_alu_m_alloc_temps(t);
-	insn_len = dec_prep_alu_m(dc, 0, memsize, t[0], t[1]);
-	cris_cc_mask(dc, CC_MASK_NZ);
-	cris_alu(dc, CC_OP_MOVE, cpu_R[dc->op2], cpu_R[dc->op2], t[1], 4);
-	do_postinc(dc, memsize);
-	cris_alu_m_free_temps(t);
-	return insn_len;
-}
-
-static int dec_cmpu_m(DisasContext *dc)
-{
-	TCGv t[2];
-	int memsize = memsize_z(dc);
-	int insn_len;
-	LOG_DIS("cmpu.%c [$r%u%s, $r%u\n",
-		    memsize_char(memsize),
-		    dc->op1, dc->postinc ? "+]" : "]",
-		    dc->op2);
-
-	cris_alu_m_alloc_temps(t);
-	insn_len = dec_prep_alu_m(dc, 0, memsize, t[0], t[1]);
-	cris_cc_mask(dc, CC_MASK_NZVC);
-	cris_alu(dc, CC_OP_CMP, cpu_R[dc->op2], cpu_R[dc->op2], t[1], 4);
-	do_postinc(dc, memsize);
-	cris_alu_m_free_temps(t);
-	return insn_len;
-}
-
-static int dec_cmps_m(DisasContext *dc)
-{
-	TCGv t[2];
-	int memsize = memsize_z(dc);
-	int insn_len;
-	LOG_DIS("cmps.%c [$r%u%s, $r%u\n",
-		    memsize_char(memsize),
-		    dc->op1, dc->postinc ? "+]" : "]",
-		    dc->op2);
-
-	cris_alu_m_alloc_temps(t);
-	insn_len = dec_prep_alu_m(dc, 1, memsize, t[0], t[1]);
-	cris_cc_mask(dc, CC_MASK_NZVC);
-	cris_alu(dc, CC_OP_CMP,
-		    cpu_R[dc->op2], cpu_R[dc->op2], t[1],
-		    memsize_zz(dc));
-	do_postinc(dc, memsize);
-	cris_alu_m_free_temps(t);
-	return insn_len;
-}
-
-static int dec_cmp_m(DisasContext *dc)
-{
-	TCGv t[2];
-	int memsize = memsize_zz(dc);
-	int insn_len;
-	LOG_DIS("cmp.%c [$r%u%s, $r%u\n",
-		    memsize_char(memsize),
-		    dc->op1, dc->postinc ? "+]" : "]",
-		    dc->op2);
-
-	cris_alu_m_alloc_temps(t);
-	insn_len = dec_prep_alu_m(dc, 0, memsize, t[0], t[1]);
-	cris_cc_mask(dc, CC_MASK_NZVC);
-	cris_alu(dc, CC_OP_CMP,
-		    cpu_R[dc->op2], cpu_R[dc->op2], t[1],
-		    memsize_zz(dc));
-	do_postinc(dc, memsize);
-	cris_alu_m_free_temps(t);
-	return insn_len;
-}
-
-static int dec_test_m(DisasContext *dc)
-{
-	TCGv t[2];
-	int memsize = memsize_zz(dc);
-	int insn_len;
-	LOG_DIS("test.%c [$r%u%s] op2=%x\n",
-		    memsize_char(memsize),
-		    dc->op1, dc->postinc ? "+]" : "]",
-		    dc->op2);
-
-	cris_evaluate_flags(dc);
-
-	cris_alu_m_alloc_temps(t);
-	insn_len = dec_prep_alu_m(dc, 0, memsize, t[0], t[1]);
-	cris_cc_mask(dc, CC_MASK_NZ);
-	tcg_gen_andi_tl(cpu_PR[PR_CCS], cpu_PR[PR_CCS], ~3);
-
-	cris_alu(dc, CC_OP_CMP,
-		 cpu_R[dc->op2], t[1], tcg_const_tl(0), memsize_zz(dc));
-	do_postinc(dc, memsize);
-	cris_alu_m_free_temps(t);
-	return insn_len;
-}
-
-static int dec_and_m(DisasContext *dc)
-{
-	TCGv t[2];
-	int memsize = memsize_zz(dc);
-	int insn_len;
-	LOG_DIS("and.%c [$r%u%s, $r%u\n",
-		    memsize_char(memsize),
-		    dc->op1, dc->postinc ? "+]" : "]",
-		    dc->op2);
-
-	cris_alu_m_alloc_temps(t);
-	insn_len = dec_prep_alu_m(dc, 0, memsize, t[0], t[1]);
-	cris_cc_mask(dc, CC_MASK_NZ);
-	cris_alu(dc, CC_OP_AND, cpu_R[dc->op2], t[0], t[1], memsize_zz(dc));
-	do_postinc(dc, memsize);
-	cris_alu_m_free_temps(t);
-	return insn_len;
-}
-
-static int dec_add_m(DisasContext *dc)
-{
-	TCGv t[2];
-	int memsize = memsize_zz(dc);
-	int insn_len;
-	LOG_DIS("add.%c [$r%u%s, $r%u\n",
-		    memsize_char(memsize),
-		    dc->op1, dc->postinc ? "+]" : "]",
-		    dc->op2);
-
-	cris_alu_m_alloc_temps(t);
-	insn_len = dec_prep_alu_m(dc, 0, memsize, t[0], t[1]);
-	cris_cc_mask(dc, CC_MASK_NZVC);
-	cris_alu(dc, CC_OP_ADD,
-		 cpu_R[dc->op2], t[0], t[1], memsize_zz(dc));
-	do_postinc(dc, memsize);
-	cris_alu_m_free_temps(t);
-	return insn_len;
-}
-
-static int dec_addo_m(DisasContext *dc)
-{
-	TCGv t[2];
-	int memsize = memsize_zz(dc);
-	int insn_len;
-	LOG_DIS("add.%c [$r%u%s, $r%u\n",
-		    memsize_char(memsize),
-		    dc->op1, dc->postinc ? "+]" : "]",
-		    dc->op2);
-
-	cris_alu_m_alloc_temps(t);
-	insn_len = dec_prep_alu_m(dc, 1, memsize, t[0], t[1]);
-	cris_cc_mask(dc, 0);
-	cris_alu(dc, CC_OP_ADD, cpu_R[R_ACR], t[0], t[1], 4);
-	do_postinc(dc, memsize);
-	cris_alu_m_free_temps(t);
-	return insn_len;
-}
-
-static int dec_bound_m(DisasContext *dc)
-{
-	TCGv l[2];
-	int memsize = memsize_zz(dc);
-	int insn_len;
-	LOG_DIS("bound.%c [$r%u%s, $r%u\n",
-		    memsize_char(memsize),
-		    dc->op1, dc->postinc ? "+]" : "]",
-		    dc->op2);
-
-	l[0] = tcg_temp_local_new();
-	l[1] = tcg_temp_local_new();
-	insn_len = dec_prep_alu_m(dc, 0, memsize, l[0], l[1]);
-	cris_cc_mask(dc, CC_MASK_NZ);
-	cris_alu(dc, CC_OP_BOUND, cpu_R[dc->op2], l[0], l[1], 4);
-	do_postinc(dc, memsize);
-	tcg_temp_free(l[0]);
-	tcg_temp_free(l[1]);
-	return insn_len;
-}
-
-static int dec_addc_mr(DisasContext *dc)
-{
-	TCGv t[2];
-	int insn_len = 2;
-	LOG_DIS("addc [$r%u%s, $r%u\n",
-		    dc->op1, dc->postinc ? "+]" : "]",
-		    dc->op2);
-
-	cris_evaluate_flags(dc);
-
-	/* Set for this insn.  */
-	dc->flagx_known = 1;
-	dc->flags_x = X_FLAG;
-
-	cris_alu_m_alloc_temps(t);
-	insn_len = dec_prep_alu_m(dc, 0, 4, t[0], t[1]);
-	cris_cc_mask(dc, CC_MASK_NZVC);
-	cris_alu(dc, CC_OP_ADDC, cpu_R[dc->op2], t[0], t[1], 4);
-	do_postinc(dc, 4);
-	cris_alu_m_free_temps(t);
-	return insn_len;
-}
-
-static int dec_sub_m(DisasContext *dc)
-{
-	TCGv t[2];
-	int memsize = memsize_zz(dc);
-	int insn_len;
-	LOG_DIS("sub.%c [$r%u%s, $r%u ir=%x zz=%x\n",
-		    memsize_char(memsize),
-		    dc->op1, dc->postinc ? "+]" : "]",
-		    dc->op2, dc->ir, dc->zzsize);
-
-	cris_alu_m_alloc_temps(t);
-	insn_len = dec_prep_alu_m(dc, 0, memsize, t[0], t[1]);
-	cris_cc_mask(dc, CC_MASK_NZVC);
-	cris_alu(dc, CC_OP_SUB, cpu_R[dc->op2], t[0], t[1], memsize);
-	do_postinc(dc, memsize);
-	cris_alu_m_free_temps(t);
-	return insn_len;
-}
-
-static int dec_or_m(DisasContext *dc)
-{
-	TCGv t[2];
-	int memsize = memsize_zz(dc);
-	int insn_len;
-	LOG_DIS("or.%c [$r%u%s, $r%u pc=%x\n",
-		    memsize_char(memsize),
-		    dc->op1, dc->postinc ? "+]" : "]",
-		    dc->op2, dc->pc);
-
-	cris_alu_m_alloc_temps(t);
-	insn_len = dec_prep_alu_m(dc, 0, memsize, t[0], t[1]);
-	cris_cc_mask(dc, CC_MASK_NZ);
-	cris_alu(dc, CC_OP_OR,
-		    cpu_R[dc->op2], t[0], t[1], memsize_zz(dc));
-	do_postinc(dc, memsize);
-	cris_alu_m_free_temps(t);
-	return insn_len;
-}
-
-static int dec_move_mp(DisasContext *dc)
-{
-	TCGv t[2];
-	int memsize = memsize_zz(dc);
-	int insn_len = 2;
-
-	LOG_DIS("move.%c [$r%u%s, $p%u\n",
-		    memsize_char(memsize),
-		    dc->op1,
-		    dc->postinc ? "+]" : "]",
-		    dc->op2);
-
-	cris_alu_m_alloc_temps(t);
-	insn_len = dec_prep_alu_m(dc, 0, memsize, t[0], t[1]);
-	cris_cc_mask(dc, 0);
-	if (dc->op2 == PR_CCS) {
-		cris_evaluate_flags(dc);
-		if (dc->tb_flags & U_FLAG) {
-			/* User space is not allowed to touch all flags.  */
-			tcg_gen_andi_tl(t[1], t[1], 0x39f);
-			tcg_gen_andi_tl(t[0], cpu_PR[PR_CCS], ~0x39f);
-			tcg_gen_or_tl(t[1], t[0], t[1]);
-		}
-	}
-
-	t_gen_mov_preg_TN(dc, dc->op2, t[1]);
-
-	do_postinc(dc, memsize);
-	cris_alu_m_free_temps(t);
-	return insn_len;
-}
-
-static int dec_move_pm(DisasContext *dc)
-{
-	TCGv t0;
-	int memsize;
-
-	memsize = preg_sizes[dc->op2];
-
-	LOG_DIS("move.%c $p%u, [$r%u%s\n",
-		     memsize_char(memsize), 
-		     dc->op2, dc->op1, dc->postinc ? "+]" : "]");
-
-	/* prepare store. Address in T0, value in T1.  */
-	if (dc->op2 == PR_CCS)
-		cris_evaluate_flags(dc);
-	t0 = tcg_temp_new();
-	t_gen_mov_TN_preg(t0, dc->op2);
-	cris_flush_cc_state(dc);
-	gen_store(dc, cpu_R[dc->op1], t0, memsize);
-	tcg_temp_free(t0);
-
-	cris_cc_mask(dc, 0);
-	if (dc->postinc)
-		tcg_gen_addi_tl(cpu_R[dc->op1], cpu_R[dc->op1], memsize);
-	return 2;
-}
-
-static int dec_movem_mr(DisasContext *dc)
-{
-	TCGv_i64 tmp[16];
-        TCGv tmp32;
-	TCGv addr;
-	int i;
-	int nr = dc->op2 + 1;
-
-	LOG_DIS("movem [$r%u%s, $r%u\n", dc->op1,
-		    dc->postinc ? "+]" : "]", dc->op2);
-
-	addr = tcg_temp_new();
-	/* There are probably better ways of doing this.  */
-	cris_flush_cc_state(dc);
-	for (i = 0; i < (nr >> 1); i++) {
-		tmp[i] = tcg_temp_new_i64();
-		tcg_gen_addi_tl(addr, cpu_R[dc->op1], i * 8);
-		gen_load64(dc, tmp[i], addr);
-	}
-	if (nr & 1) {
-		tmp32 = tcg_temp_new_i32();
-		tcg_gen_addi_tl(addr, cpu_R[dc->op1], i * 8);
-		gen_load(dc, tmp32, addr, 4, 0);
-	} else
-		TCGV_UNUSED(tmp32);
-	tcg_temp_free(addr);
-
-	for (i = 0; i < (nr >> 1); i++) {
-		tcg_gen_trunc_i64_i32(cpu_R[i * 2], tmp[i]);
-		tcg_gen_shri_i64(tmp[i], tmp[i], 32);
-		tcg_gen_trunc_i64_i32(cpu_R[i * 2 + 1], tmp[i]);
-		tcg_temp_free_i64(tmp[i]);
-	}
-	if (nr & 1) {
-		tcg_gen_mov_tl(cpu_R[dc->op2], tmp32);
-		tcg_temp_free(tmp32);
-	}
+    tcg_temp_free(t[0]);
+    tcg_temp_free(t[1]);
+}
+
+static int dec_movs_m(CPUCRISState *env, DisasContext *dc)
+{
+    TCGv t[2];
+    int memsize = memsize_z(dc);
+    int insn_len;
+    LOG_DIS("movs.%c [$r%u%s, $r%u\n",
+            memsize_char(memsize),
+            dc->op1, dc->postinc ? "+]" : "]",
+            dc->op2);
+
+    cris_alu_m_alloc_temps(t);
+    /* sign extend.  */
+        insn_len = dec_prep_alu_m(env, dc, 1, memsize, t[0], t[1]);
+    cris_cc_mask(dc, CC_MASK_NZ);
+    cris_alu(dc, CC_OP_MOVE,
+            cpu_R[dc->op2], cpu_R[dc->op2], t[1], 4);
+    do_postinc(dc, memsize);
+    cris_alu_m_free_temps(t);
+    return insn_len;
+}
+
+static int dec_addu_m(CPUCRISState *env, DisasContext *dc)
+{
+    TCGv t[2];
+    int memsize = memsize_z(dc);
+    int insn_len;
+    LOG_DIS("addu.%c [$r%u%s, $r%u\n",
+            memsize_char(memsize),
+            dc->op1, dc->postinc ? "+]" : "]",
+            dc->op2);
+
+    cris_alu_m_alloc_temps(t);
+    /* sign extend.  */
+        insn_len = dec_prep_alu_m(env, dc, 0, memsize, t[0], t[1]);
+    cris_cc_mask(dc, CC_MASK_NZVC);
+    cris_alu(dc, CC_OP_ADD,
+            cpu_R[dc->op2], cpu_R[dc->op2], t[1], 4);
+    do_postinc(dc, memsize);
+    cris_alu_m_free_temps(t);
+    return insn_len;
+}
+
+static int dec_adds_m(CPUCRISState *env, DisasContext *dc)
+{
+    TCGv t[2];
+    int memsize = memsize_z(dc);
+    int insn_len;
+    LOG_DIS("adds.%c [$r%u%s, $r%u\n",
+            memsize_char(memsize),
+            dc->op1, dc->postinc ? "+]" : "]",
+            dc->op2);
+
+    cris_alu_m_alloc_temps(t);
+    /* sign extend.  */
+        insn_len = dec_prep_alu_m(env, dc, 1, memsize, t[0], t[1]);
+    cris_cc_mask(dc, CC_MASK_NZVC);
+    cris_alu(dc, CC_OP_ADD, cpu_R[dc->op2], cpu_R[dc->op2], t[1], 4);
+    do_postinc(dc, memsize);
+    cris_alu_m_free_temps(t);
+    return insn_len;
+}
+
+static int dec_subu_m(CPUCRISState *env, DisasContext *dc)
+{
+    TCGv t[2];
+    int memsize = memsize_z(dc);
+    int insn_len;
+    LOG_DIS("subu.%c [$r%u%s, $r%u\n",
+            memsize_char(memsize),
+            dc->op1, dc->postinc ? "+]" : "]",
+            dc->op2);
+
+    cris_alu_m_alloc_temps(t);
+    /* sign extend.  */
+        insn_len = dec_prep_alu_m(env, dc, 0, memsize, t[0], t[1]);
+    cris_cc_mask(dc, CC_MASK_NZVC);
+    cris_alu(dc, CC_OP_SUB, cpu_R[dc->op2], cpu_R[dc->op2], t[1], 4);
+    do_postinc(dc, memsize);
+    cris_alu_m_free_temps(t);
+    return insn_len;
+}
+
+static int dec_subs_m(CPUCRISState *env, DisasContext *dc)
+{
+    TCGv t[2];
+    int memsize = memsize_z(dc);
+    int insn_len;
+    LOG_DIS("subs.%c [$r%u%s, $r%u\n",
+            memsize_char(memsize),
+            dc->op1, dc->postinc ? "+]" : "]",
+            dc->op2);
+
+    cris_alu_m_alloc_temps(t);
+    /* sign extend.  */
+        insn_len = dec_prep_alu_m(env, dc, 1, memsize, t[0], t[1]);
+    cris_cc_mask(dc, CC_MASK_NZVC);
+    cris_alu(dc, CC_OP_SUB, cpu_R[dc->op2], cpu_R[dc->op2], t[1], 4);
+    do_postinc(dc, memsize);
+    cris_alu_m_free_temps(t);
+    return insn_len;
+}
+
+static int dec_movu_m(CPUCRISState *env, DisasContext *dc)
+{
+    TCGv t[2];
+    int memsize = memsize_z(dc);
+    int insn_len;
+
+    LOG_DIS("movu.%c [$r%u%s, $r%u\n",
+            memsize_char(memsize),
+            dc->op1, dc->postinc ? "+]" : "]",
+            dc->op2);
+
+    cris_alu_m_alloc_temps(t);
+        insn_len = dec_prep_alu_m(env, dc, 0, memsize, t[0], t[1]);
+    cris_cc_mask(dc, CC_MASK_NZ);
+    cris_alu(dc, CC_OP_MOVE, cpu_R[dc->op2], cpu_R[dc->op2], t[1], 4);
+    do_postinc(dc, memsize);
+    cris_alu_m_free_temps(t);
+    return insn_len;
+}
+
+static int dec_cmpu_m(CPUCRISState *env, DisasContext *dc)
+{
+    TCGv t[2];
+    int memsize = memsize_z(dc);
+    int insn_len;
+    LOG_DIS("cmpu.%c [$r%u%s, $r%u\n",
+            memsize_char(memsize),
+            dc->op1, dc->postinc ? "+]" : "]",
+            dc->op2);
+
+    cris_alu_m_alloc_temps(t);
+        insn_len = dec_prep_alu_m(env, dc, 0, memsize, t[0], t[1]);
+    cris_cc_mask(dc, CC_MASK_NZVC);
+    cris_alu(dc, CC_OP_CMP, cpu_R[dc->op2], cpu_R[dc->op2], t[1], 4);
+    do_postinc(dc, memsize);
+    cris_alu_m_free_temps(t);
+    return insn_len;
+}
+
+static int dec_cmps_m(CPUCRISState *env, DisasContext *dc)
+{
+    TCGv t[2];
+    int memsize = memsize_z(dc);
+    int insn_len;
+    LOG_DIS("cmps.%c [$r%u%s, $r%u\n",
+            memsize_char(memsize),
+            dc->op1, dc->postinc ? "+]" : "]",
+            dc->op2);
+
+    cris_alu_m_alloc_temps(t);
+        insn_len = dec_prep_alu_m(env, dc, 1, memsize, t[0], t[1]);
+    cris_cc_mask(dc, CC_MASK_NZVC);
+    cris_alu(dc, CC_OP_CMP,
+            cpu_R[dc->op2], cpu_R[dc->op2], t[1],
+            memsize_zz(dc));
+    do_postinc(dc, memsize);
+    cris_alu_m_free_temps(t);
+    return insn_len;
+}
+
+static int dec_cmp_m(CPUCRISState *env, DisasContext *dc)
+{
+    TCGv t[2];
+    int memsize = memsize_zz(dc);
+    int insn_len;
+    LOG_DIS("cmp.%c [$r%u%s, $r%u\n",
+            memsize_char(memsize),
+            dc->op1, dc->postinc ? "+]" : "]",
+            dc->op2);
+
+    cris_alu_m_alloc_temps(t);
+        insn_len = dec_prep_alu_m(env, dc, 0, memsize, t[0], t[1]);
+    cris_cc_mask(dc, CC_MASK_NZVC);
+    cris_alu(dc, CC_OP_CMP,
+            cpu_R[dc->op2], cpu_R[dc->op2], t[1],
+            memsize_zz(dc));
+    do_postinc(dc, memsize);
+    cris_alu_m_free_temps(t);
+    return insn_len;
+}
+
+static int dec_test_m(CPUCRISState *env, DisasContext *dc)
+{
+    TCGv t[2];
+    int memsize = memsize_zz(dc);
+    int insn_len;
+    LOG_DIS("test.%c [$r%u%s] op2=%x\n",
+            memsize_char(memsize),
+            dc->op1, dc->postinc ? "+]" : "]",
+            dc->op2);
+
+    cris_evaluate_flags(dc);
+
+    cris_alu_m_alloc_temps(t);
+        insn_len = dec_prep_alu_m(env, dc, 0, memsize, t[0], t[1]);
+    cris_cc_mask(dc, CC_MASK_NZ);
+    tcg_gen_andi_tl(cpu_PR[PR_CCS], cpu_PR[PR_CCS], ~3);
+
+    cris_alu(dc, CC_OP_CMP,
+         cpu_R[dc->op2], t[1], tcg_const_tl(0), memsize_zz(dc));
+    do_postinc(dc, memsize);
+    cris_alu_m_free_temps(t);
+    return insn_len;
+}
+
+static int dec_and_m(CPUCRISState *env, DisasContext *dc)
+{
+    TCGv t[2];
+    int memsize = memsize_zz(dc);
+    int insn_len;
+    LOG_DIS("and.%c [$r%u%s, $r%u\n",
+            memsize_char(memsize),
+            dc->op1, dc->postinc ? "+]" : "]",
+            dc->op2);
+
+    cris_alu_m_alloc_temps(t);
+        insn_len = dec_prep_alu_m(env, dc, 0, memsize, t[0], t[1]);
+    cris_cc_mask(dc, CC_MASK_NZ);
+    cris_alu(dc, CC_OP_AND, cpu_R[dc->op2], t[0], t[1], memsize_zz(dc));
+    do_postinc(dc, memsize);
+    cris_alu_m_free_temps(t);
+    return insn_len;
+}
+
+static int dec_add_m(CPUCRISState *env, DisasContext *dc)
+{
+    TCGv t[2];
+    int memsize = memsize_zz(dc);
+    int insn_len;
+    LOG_DIS("add.%c [$r%u%s, $r%u\n",
+            memsize_char(memsize),
+            dc->op1, dc->postinc ? "+]" : "]",
+            dc->op2);
+
+    cris_alu_m_alloc_temps(t);
+        insn_len = dec_prep_alu_m(env, dc, 0, memsize, t[0], t[1]);
+    cris_cc_mask(dc, CC_MASK_NZVC);
+    cris_alu(dc, CC_OP_ADD,
+         cpu_R[dc->op2], t[0], t[1], memsize_zz(dc));
+    do_postinc(dc, memsize);
+    cris_alu_m_free_temps(t);
+    return insn_len;
+}
+
+static int dec_addo_m(CPUCRISState *env, DisasContext *dc)
+{
+    TCGv t[2];
+    int memsize = memsize_zz(dc);
+    int insn_len;
+    LOG_DIS("add.%c [$r%u%s, $r%u\n",
+            memsize_char(memsize),
+            dc->op1, dc->postinc ? "+]" : "]",
+            dc->op2);
+
+    cris_alu_m_alloc_temps(t);
+        insn_len = dec_prep_alu_m(env, dc, 1, memsize, t[0], t[1]);
+    cris_cc_mask(dc, 0);
+    cris_alu(dc, CC_OP_ADD, cpu_R[R_ACR], t[0], t[1], 4);
+    do_postinc(dc, memsize);
+    cris_alu_m_free_temps(t);
+    return insn_len;
+}
+
+static int dec_bound_m(CPUCRISState *env, DisasContext *dc)
+{
+    TCGv l[2];
+    int memsize = memsize_zz(dc);
+    int insn_len;
+    LOG_DIS("bound.%c [$r%u%s, $r%u\n",
+            memsize_char(memsize),
+            dc->op1, dc->postinc ? "+]" : "]",
+            dc->op2);
+
+    l[0] = tcg_temp_local_new();
+    l[1] = tcg_temp_local_new();
+        insn_len = dec_prep_alu_m(env, dc, 0, memsize, l[0], l[1]);
+    cris_cc_mask(dc, CC_MASK_NZ);
+    cris_alu(dc, CC_OP_BOUND, cpu_R[dc->op2], l[0], l[1], 4);
+    do_postinc(dc, memsize);
+    tcg_temp_free(l[0]);
+    tcg_temp_free(l[1]);
+    return insn_len;
+}
+
+static int dec_addc_mr(CPUCRISState *env, DisasContext *dc)
+{
+    TCGv t[2];
+    int insn_len = 2;
+    LOG_DIS("addc [$r%u%s, $r%u\n",
+            dc->op1, dc->postinc ? "+]" : "]",
+            dc->op2);
+
+    cris_evaluate_flags(dc);
+
+    /* Set for this insn.  */
+    dc->flagx_known = 1;
+    dc->flags_x = X_FLAG;
+
+    cris_alu_m_alloc_temps(t);
+        insn_len = dec_prep_alu_m(env, dc, 0, 4, t[0], t[1]);
+    cris_cc_mask(dc, CC_MASK_NZVC);
+    cris_alu(dc, CC_OP_ADDC, cpu_R[dc->op2], t[0], t[1], 4);
+    do_postinc(dc, 4);
+    cris_alu_m_free_temps(t);
+    return insn_len;
+}
+
+static int dec_sub_m(CPUCRISState *env, DisasContext *dc)
+{
+    TCGv t[2];
+    int memsize = memsize_zz(dc);
+    int insn_len;
+    LOG_DIS("sub.%c [$r%u%s, $r%u ir=%x zz=%x\n",
+            memsize_char(memsize),
+            dc->op1, dc->postinc ? "+]" : "]",
+            dc->op2, dc->ir, dc->zzsize);
+
+    cris_alu_m_alloc_temps(t);
+        insn_len = dec_prep_alu_m(env, dc, 0, memsize, t[0], t[1]);
+    cris_cc_mask(dc, CC_MASK_NZVC);
+    cris_alu(dc, CC_OP_SUB, cpu_R[dc->op2], t[0], t[1], memsize);
+    do_postinc(dc, memsize);
+    cris_alu_m_free_temps(t);
+    return insn_len;
+}
+
+static int dec_or_m(CPUCRISState *env, DisasContext *dc)
+{
+    TCGv t[2];
+    int memsize = memsize_zz(dc);
+    int insn_len;
+    LOG_DIS("or.%c [$r%u%s, $r%u pc=%x\n",
+            memsize_char(memsize),
+            dc->op1, dc->postinc ? "+]" : "]",
+            dc->op2, dc->pc);
+
+    cris_alu_m_alloc_temps(t);
+        insn_len = dec_prep_alu_m(env, dc, 0, memsize, t[0], t[1]);
+    cris_cc_mask(dc, CC_MASK_NZ);
+    cris_alu(dc, CC_OP_OR,
+            cpu_R[dc->op2], t[0], t[1], memsize_zz(dc));
+    do_postinc(dc, memsize);
+    cris_alu_m_free_temps(t);
+    return insn_len;
+}
+
+static int dec_move_mp(CPUCRISState *env, DisasContext *dc)
+{
+    TCGv t[2];
+    int memsize = memsize_zz(dc);
+    int insn_len = 2;
+
+    LOG_DIS("move.%c [$r%u%s, $p%u\n",
+            memsize_char(memsize),
+            dc->op1,
+            dc->postinc ? "+]" : "]",
+            dc->op2);
+
+    cris_alu_m_alloc_temps(t);
+        insn_len = dec_prep_alu_m(env, dc, 0, memsize, t[0], t[1]);
+    cris_cc_mask(dc, 0);
+    if (dc->op2 == PR_CCS) {
+        cris_evaluate_flags(dc);
+        if (dc->tb_flags & U_FLAG) {
+            /* User space is not allowed to touch all flags.  */
+            tcg_gen_andi_tl(t[1], t[1], 0x39f);
+            tcg_gen_andi_tl(t[0], cpu_PR[PR_CCS], ~0x39f);
+            tcg_gen_or_tl(t[1], t[0], t[1]);
+        }
+    }
+
+    t_gen_mov_preg_TN(dc, dc->op2, t[1]);
+
+    do_postinc(dc, memsize);
+    cris_alu_m_free_temps(t);
+    return insn_len;
+}
+
+static int dec_move_pm(CPUCRISState *env, DisasContext *dc)
+{
+    TCGv t0;
+    int memsize;
+
+    memsize = preg_sizes[dc->op2];
+
+    LOG_DIS("move.%c $p%u, [$r%u%s\n",
+            memsize_char(memsize),
+            dc->op2, dc->op1, dc->postinc ? "+]" : "]");
+
+    /* prepare store. Address in T0, value in T1.  */
+    if (dc->op2 == PR_CCS) {
+        cris_evaluate_flags(dc);
+    }
+    t0 = tcg_temp_new();
+    t_gen_mov_TN_preg(t0, dc->op2);
+    cris_flush_cc_state(dc);
+    gen_store(dc, cpu_R[dc->op1], t0, memsize);
+    tcg_temp_free(t0);
+
+    cris_cc_mask(dc, 0);
+    if (dc->postinc) {
+        tcg_gen_addi_tl(cpu_R[dc->op1], cpu_R[dc->op1], memsize);
+    }
+    return 2;
+}
+
+static int dec_movem_mr(CPUCRISState *env, DisasContext *dc)
+{
+    TCGv_i64 tmp[16];
+    TCGv tmp32;
+    TCGv addr;
+    int i;
+    int nr = dc->op2 + 1;
+
+    LOG_DIS("movem [$r%u%s, $r%u\n", dc->op1,
+            dc->postinc ? "+]" : "]", dc->op2);
+
+    addr = tcg_temp_new();
+    /* There are probably better ways of doing this.  */
+    cris_flush_cc_state(dc);
+    for (i = 0; i < (nr >> 1); i++) {
+        tmp[i] = tcg_temp_new_i64();
+        tcg_gen_addi_tl(addr, cpu_R[dc->op1], i * 8);
+        gen_load64(dc, tmp[i], addr);
+    }
+    if (nr & 1) {
+        tmp32 = tcg_temp_new_i32();
+        tcg_gen_addi_tl(addr, cpu_R[dc->op1], i * 8);
+        gen_load(dc, tmp32, addr, 4, 0);
+    } else {
+        TCGV_UNUSED(tmp32);
+    }
+    tcg_temp_free(addr);
 
-	/* writeback the updated pointer value.  */
-	if (dc->postinc)
-		tcg_gen_addi_tl(cpu_R[dc->op1], cpu_R[dc->op1], nr * 4);
+    for (i = 0; i < (nr >> 1); i++) {
+        tcg_gen_trunc_i64_i32(cpu_R[i * 2], tmp[i]);
+        tcg_gen_shri_i64(tmp[i], tmp[i], 32);
+        tcg_gen_trunc_i64_i32(cpu_R[i * 2 + 1], tmp[i]);
+        tcg_temp_free_i64(tmp[i]);
+    }
+    if (nr & 1) {
+        tcg_gen_mov_tl(cpu_R[dc->op2], tmp32);
+        tcg_temp_free(tmp32);
+    }
 
-	/* gen_load might want to evaluate the previous insns flags.  */
-	cris_cc_mask(dc, 0);
-	return 2;
-}
-
-static int dec_movem_rm(DisasContext *dc)
-{
-	TCGv tmp;
-	TCGv addr;
-	int i;
+    /* writeback the updated pointer value.  */
+    if (dc->postinc) {
+        tcg_gen_addi_tl(cpu_R[dc->op1], cpu_R[dc->op1], nr * 4);
+    }
 
-	LOG_DIS("movem $r%u, [$r%u%s\n", dc->op2, dc->op1,
-		     dc->postinc ? "+]" : "]");
+    /* gen_load might want to evaluate the previous insns flags.  */
+    cris_cc_mask(dc, 0);
+    return 2;
+}
 
-	cris_flush_cc_state(dc);
+static int dec_movem_rm(CPUCRISState *env, DisasContext *dc)
+{
+    TCGv tmp;
+    TCGv addr;
+    int i;
+
+    LOG_DIS("movem $r%u, [$r%u%s\n", dc->op2, dc->op1,
+            dc->postinc ? "+]" : "]");
 
-	tmp = tcg_temp_new();
-	addr = tcg_temp_new();
-	tcg_gen_movi_tl(tmp, 4);
-	tcg_gen_mov_tl(addr, cpu_R[dc->op1]);
-	for (i = 0; i <= dc->op2; i++) {
-		/* Displace addr.  */
-		/* Perform the store.  */
-		gen_store(dc, addr, cpu_R[i], 4);
-		tcg_gen_add_tl(addr, addr, tmp);
-	}
-	if (dc->postinc)
-		tcg_gen_mov_tl(cpu_R[dc->op1], addr);
-	cris_cc_mask(dc, 0);
-	tcg_temp_free(tmp);
-	tcg_temp_free(addr);
-	return 2;
+    cris_flush_cc_state(dc);
+
+    tmp = tcg_temp_new();
+    addr = tcg_temp_new();
+    tcg_gen_movi_tl(tmp, 4);
+    tcg_gen_mov_tl(addr, cpu_R[dc->op1]);
+    for (i = 0; i <= dc->op2; i++) {
+        /* Displace addr.  */
+        /* Perform the store.  */
+        gen_store(dc, addr, cpu_R[i], 4);
+        tcg_gen_add_tl(addr, addr, tmp);
+    }
+    if (dc->postinc) {
+        tcg_gen_mov_tl(cpu_R[dc->op1], addr);
+    }
+    cris_cc_mask(dc, 0);
+    tcg_temp_free(tmp);
+    tcg_temp_free(addr);
+    return 2;
 }
 
-static int dec_move_rm(DisasContext *dc)
+static int dec_move_rm(CPUCRISState *env, DisasContext *dc)
 {
-	int memsize;
+    int memsize;
+
+    memsize = memsize_zz(dc);
 
-	memsize = memsize_zz(dc);
-
-	LOG_DIS("move.%c $r%u, [$r%u]\n",
-		     memsize_char(memsize), dc->op2, dc->op1);
+    LOG_DIS("move.%c $r%u, [$r%u]\n",
+            memsize_char(memsize), dc->op2, dc->op1);
 
-	/* prepare store.  */
-	cris_flush_cc_state(dc);
-	gen_store(dc, cpu_R[dc->op1], cpu_R[dc->op2], memsize);
+    /* prepare store.  */
+    cris_flush_cc_state(dc);
+    gen_store(dc, cpu_R[dc->op1], cpu_R[dc->op2], memsize);
 
-	if (dc->postinc)
-		tcg_gen_addi_tl(cpu_R[dc->op1], cpu_R[dc->op1], memsize);
-	cris_cc_mask(dc, 0);
-	return 2;
+    if (dc->postinc) {
+        tcg_gen_addi_tl(cpu_R[dc->op1], cpu_R[dc->op1], memsize);
+    }
+    cris_cc_mask(dc, 0);
+    return 2;
 }
 
-static int dec_lapcq(DisasContext *dc)
+static int dec_lapcq(CPUCRISState *env, DisasContext *dc)
 {
-	LOG_DIS("lapcq %x, $r%u\n",
-		    dc->pc + dc->op1*2, dc->op2);
-	cris_cc_mask(dc, 0);
-	tcg_gen_movi_tl(cpu_R[dc->op2], dc->pc + dc->op1 * 2);
-	return 2;
-}
+    LOG_DIS("lapcq %x, $r%u\n",
+            dc->pc + dc->op1*2, dc->op2);
+    cris_cc_mask(dc, 0);
+    tcg_gen_movi_tl(cpu_R[dc->op2], dc->pc + dc->op1 * 2);
+    return 2;
+}
 
-static int dec_lapc_im(DisasContext *dc)
+static int dec_lapc_im(CPUCRISState *env, DisasContext *dc)
 {
-	unsigned int rd;
-	int32_t imm;
-	int32_t pc;
+    unsigned int rd;
+    int32_t imm;
+    int32_t pc;
 
-	rd = dc->op2;
+    rd = dc->op2;
 
-	cris_cc_mask(dc, 0);
-	imm = cris_fetch(dc, dc->pc + 2, 4, 0);
-	LOG_DIS("lapc 0x%x, $r%u\n", imm + dc->pc, dc->op2);
+    cris_cc_mask(dc, 0);
+    imm = cris_fetch(env, dc, dc->pc + 2, 4, 0);
+    LOG_DIS("lapc 0x%x, $r%u\n", imm + dc->pc, dc->op2);
 
-	pc = dc->pc;
-	pc += imm;
-	tcg_gen_movi_tl(cpu_R[rd], pc);
-	return 6;
+    pc = dc->pc;
+    pc += imm;
+    tcg_gen_movi_tl(cpu_R[rd], pc);
+    return 6;
 }
 
 /* Jump to special reg.  */
-static int dec_jump_p(DisasContext *dc)
+static int dec_jump_p(CPUCRISState *env, DisasContext *dc)
 {
-	LOG_DIS("jump $p%u\n", dc->op2);
+    LOG_DIS("jump $p%u\n", dc->op2);
 
-	if (dc->op2 == PR_CCS)
-		cris_evaluate_flags(dc);
-	t_gen_mov_TN_preg(env_btarget, dc->op2);
-	/* rete will often have low bit set to indicate delayslot.  */
-	tcg_gen_andi_tl(env_btarget, env_btarget, ~1);
-	cris_cc_mask(dc, 0);
-	cris_prepare_jmp(dc, JMP_INDIRECT);
-	return 2;
+    if (dc->op2 == PR_CCS) {
+        cris_evaluate_flags(dc);
+    }
+    t_gen_mov_TN_preg(env_btarget, dc->op2);
+    /* rete will often have low bit set to indicate delayslot.  */
+    tcg_gen_andi_tl(env_btarget, env_btarget, ~1);
+    cris_cc_mask(dc, 0);
+    cris_prepare_jmp(dc, JMP_INDIRECT);
+    return 2;
 }
 
 /* Jump and save.  */
-static int dec_jas_r(DisasContext *dc)
-{
-	LOG_DIS("jas $r%u, $p%u\n", dc->op1, dc->op2);
-	cris_cc_mask(dc, 0);
-	/* Store the return address in Pd.  */
-	tcg_gen_mov_tl(env_btarget, cpu_R[dc->op1]);
-	if (dc->op2 > 15)
-		abort();
-	t_gen_mov_preg_TN(dc, dc->op2, tcg_const_tl(dc->pc + 4));
+static int dec_jas_r(CPUCRISState *env, DisasContext *dc)
+{
+    LOG_DIS("jas $r%u, $p%u\n", dc->op1, dc->op2);
+    cris_cc_mask(dc, 0);
+    /* Store the return address in Pd.  */
+    tcg_gen_mov_tl(env_btarget, cpu_R[dc->op1]);
+    if (dc->op2 > 15) {
+        abort();
+    }
+    t_gen_mov_preg_TN(dc, dc->op2, tcg_const_tl(dc->pc + 4));
 
-	cris_prepare_jmp(dc, JMP_INDIRECT);
-	return 2;
+    cris_prepare_jmp(dc, JMP_INDIRECT);
+    return 2;
 }
 
-static int dec_jas_im(DisasContext *dc)
+static int dec_jas_im(CPUCRISState *env, DisasContext *dc)
 {
-	uint32_t imm;
+    uint32_t imm;
 
-	imm = cris_fetch(dc, dc->pc + 2, 4, 0);
+    imm = cris_fetch(env, dc, dc->pc + 2, 4, 0);
 
-	LOG_DIS("jas 0x%x\n", imm);
-	cris_cc_mask(dc, 0);
-	/* Store the return address in Pd.  */
-	t_gen_mov_preg_TN(dc, dc->op2, tcg_const_tl(dc->pc + 8));
+    LOG_DIS("jas 0x%x\n", imm);
+    cris_cc_mask(dc, 0);
+    /* Store the return address in Pd.  */
+    t_gen_mov_preg_TN(dc, dc->op2, tcg_const_tl(dc->pc + 8));
 
-	dc->jmp_pc = imm;
-	cris_prepare_jmp(dc, JMP_DIRECT);
-	return 6;
+    dc->jmp_pc = imm;
+    cris_prepare_jmp(dc, JMP_DIRECT);
+    return 6;
 }
 
-static int dec_jasc_im(DisasContext *dc)
+static int dec_jasc_im(CPUCRISState *env, DisasContext *dc)
 {
-	uint32_t imm;
+    uint32_t imm;
 
-	imm = cris_fetch(dc, dc->pc + 2, 4, 0);
+    imm = cris_fetch(env, dc, dc->pc + 2, 4, 0);
 
-	LOG_DIS("jasc 0x%x\n", imm);
-	cris_cc_mask(dc, 0);
-	/* Store the return address in Pd.  */
-	t_gen_mov_preg_TN(dc, dc->op2, tcg_const_tl(dc->pc + 8 + 4));
+    LOG_DIS("jasc 0x%x\n", imm);
+    cris_cc_mask(dc, 0);
+    /* Store the return address in Pd.  */
+    t_gen_mov_preg_TN(dc, dc->op2, tcg_const_tl(dc->pc + 8 + 4));
 
-	dc->jmp_pc = imm;
-	cris_prepare_jmp(dc, JMP_DIRECT);
-	return 6;
+    dc->jmp_pc = imm;
+    cris_prepare_jmp(dc, JMP_DIRECT);
+    return 6;
 }
 
-static int dec_jasc_r(DisasContext *dc)
+static int dec_jasc_r(CPUCRISState *env, DisasContext *dc)
 {
-	LOG_DIS("jasc_r $r%u, $p%u\n", dc->op1, dc->op2);
-	cris_cc_mask(dc, 0);
-	/* Store the return address in Pd.  */
-	tcg_gen_mov_tl(env_btarget, cpu_R[dc->op1]);
-	t_gen_mov_preg_TN(dc, dc->op2, tcg_const_tl(dc->pc + 4 + 4));
-	cris_prepare_jmp(dc, JMP_INDIRECT);
-	return 2;
+    LOG_DIS("jasc_r $r%u, $p%u\n", dc->op1, dc->op2);
+    cris_cc_mask(dc, 0);
+    /* Store the return address in Pd.  */
+    tcg_gen_mov_tl(env_btarget, cpu_R[dc->op1]);
+    t_gen_mov_preg_TN(dc, dc->op2, tcg_const_tl(dc->pc + 4 + 4));
+    cris_prepare_jmp(dc, JMP_INDIRECT);
+    return 2;
 }
 
-static int dec_bcc_im(DisasContext *dc)
+static int dec_bcc_im(CPUCRISState *env, DisasContext *dc)
 {
-	int32_t offset;
-	uint32_t cond = dc->op2;
+    int32_t offset;
+    uint32_t cond = dc->op2;
 
-	offset = cris_fetch(dc, dc->pc + 2, 2, 1);
+    offset = cris_fetch(env, dc, dc->pc + 2, 2, 1);
 
-	LOG_DIS("b%s %d pc=%x dst=%x\n",
-		    cc_name(cond), offset,
-		    dc->pc, dc->pc + offset);
+    LOG_DIS("b%s %d pc=%x dst=%x\n",
+            cc_name(cond), offset,
+            dc->pc, dc->pc + offset);
 
-	cris_cc_mask(dc, 0);
-	/* op2 holds the condition-code.  */
-	cris_prepare_cc_branch (dc, offset, cond);
-	return 4;
+    cris_cc_mask(dc, 0);
+    /* op2 holds the condition-code.  */
+    cris_prepare_cc_branch(dc, offset, cond);
+    return 4;
 }
 
-static int dec_bas_im(DisasContext *dc)
+static int dec_bas_im(CPUCRISState *env, DisasContext *dc)
 {
-	int32_t simm;
+    int32_t simm;
 
+    simm = cris_fetch(env, dc, dc->pc + 2, 4, 0);
 
-	simm = cris_fetch(dc, dc->pc + 2, 4, 0);
+    LOG_DIS("bas 0x%x, $p%u\n", dc->pc + simm, dc->op2);
+    cris_cc_mask(dc, 0);
+    /* Store the return address in Pd.  */
+    t_gen_mov_preg_TN(dc, dc->op2, tcg_const_tl(dc->pc + 8));
 
-	LOG_DIS("bas 0x%x, $p%u\n", dc->pc + simm, dc->op2);
-	cris_cc_mask(dc, 0);
-	/* Store the return address in Pd.  */
-	t_gen_mov_preg_TN(dc, dc->op2, tcg_const_tl(dc->pc + 8));
-
-	dc->jmp_pc = dc->pc + simm;
-	cris_prepare_jmp(dc, JMP_DIRECT);
-	return 6;
+    dc->jmp_pc = dc->pc + simm;
+    cris_prepare_jmp(dc, JMP_DIRECT);
+    return 6;
 }
 
-static int dec_basc_im(DisasContext *dc)
+static int dec_basc_im(CPUCRISState *env, DisasContext *dc)
 {
-	int32_t simm;
-	simm = cris_fetch(dc, dc->pc + 2, 4, 0);
+    int32_t simm;
+    simm = cris_fetch(env, dc, dc->pc + 2, 4, 0);
 
-	LOG_DIS("basc 0x%x, $p%u\n", dc->pc + simm, dc->op2);
-	cris_cc_mask(dc, 0);
-	/* Store the return address in Pd.  */
-	t_gen_mov_preg_TN(dc, dc->op2, tcg_const_tl(dc->pc + 12));
+    LOG_DIS("basc 0x%x, $p%u\n", dc->pc + simm, dc->op2);
+    cris_cc_mask(dc, 0);
+    /* Store the return address in Pd.  */
+    t_gen_mov_preg_TN(dc, dc->op2, tcg_const_tl(dc->pc + 12));
 
-	dc->jmp_pc = dc->pc + simm;
-	cris_prepare_jmp(dc, JMP_DIRECT);
-	return 6;
+    dc->jmp_pc = dc->pc + simm;
+    cris_prepare_jmp(dc, JMP_DIRECT);
+    return 6;
 }
 
-static int dec_rfe_etc(DisasContext *dc)
+static int dec_rfe_etc(CPUCRISState *env, DisasContext *dc)
 {
-	cris_cc_mask(dc, 0);
-
-	if (dc->op2 == 15) {
-		t_gen_mov_env_TN(halted, tcg_const_tl(1));
-		tcg_gen_movi_tl(env_pc, dc->pc + 2);
-		t_gen_raise_exception(EXCP_HLT);
-		return 2;
-	}
+    cris_cc_mask(dc, 0);
 
-	switch (dc->op2 & 7) {
-		case 2:
-			/* rfe.  */
-			LOG_DIS("rfe\n");
-			cris_evaluate_flags(dc);
-			gen_helper_rfe();
-			dc->is_jmp = DISAS_UPDATE;
-			break;
-		case 5:
-			/* rfn.  */
-			LOG_DIS("rfn\n");
-			cris_evaluate_flags(dc);
-			gen_helper_rfn();
-			dc->is_jmp = DISAS_UPDATE;
-			break;
-		case 6:
-			LOG_DIS("break %d\n", dc->op1);
-			cris_evaluate_flags (dc);
-			/* break.  */
-			tcg_gen_movi_tl(env_pc, dc->pc + 2);
+    if (dc->op2 == 15) {
+        t_gen_mov_env_TN(halted, tcg_const_tl(1));
+        tcg_gen_movi_tl(env_pc, dc->pc + 2);
+        t_gen_raise_exception(EXCP_HLT);
+        return 2;
+    }
 
-			/* Breaks start at 16 in the exception vector.  */
-			t_gen_mov_env_TN(trap_vector, 
-					 tcg_const_tl(dc->op1 + 16));
-			t_gen_raise_exception(EXCP_BREAK);
-			dc->is_jmp = DISAS_UPDATE;
-			break;
-		default:
-			printf ("op2=%x\n", dc->op2);
-			BUG();
-			break;
+    switch (dc->op2 & 7) {
+    case 2:
+        /* rfe.  */
+        LOG_DIS("rfe\n");
+        cris_evaluate_flags(dc);
+        gen_helper_rfe(cpu_env);
+        dc->is_jmp = DISAS_UPDATE;
+        break;
+    case 5:
+        /* rfn.  */
+        LOG_DIS("rfn\n");
+        cris_evaluate_flags(dc);
+        gen_helper_rfn(cpu_env);
+        dc->is_jmp = DISAS_UPDATE;
+        break;
+    case 6:
+        LOG_DIS("break %d\n", dc->op1);
+        cris_evaluate_flags(dc);
+        /* break.  */
+        tcg_gen_movi_tl(env_pc, dc->pc + 2);
+
+        /* Breaks start at 16 in the exception vector.  */
+        t_gen_mov_env_TN(trap_vector,
+                tcg_const_tl(dc->op1 + 16));
+        t_gen_raise_exception(EXCP_BREAK);
+        dc->is_jmp = DISAS_UPDATE;
+        break;
+    default:
+        printf("op2=%x\n", dc->op2);
+        BUG();
+        break;
 
-	}
-	return 2;
+    }
+    return 2;
 }
 
-static int dec_ftag_fidx_d_m(DisasContext *dc)
+static int dec_ftag_fidx_d_m(CPUCRISState *env, DisasContext *dc)
 {
-	return 2;
+    return 2;
 }
 
-static int dec_ftag_fidx_i_m(DisasContext *dc)
+static int dec_ftag_fidx_i_m(CPUCRISState *env, DisasContext *dc)
 {
-	return 2;
+    return 2;
 }
 
-static int dec_null(DisasContext *dc)
+static int dec_null(CPUCRISState *env, DisasContext *dc)
 {
-	printf ("unknown insn pc=%x opc=%x op1=%x op2=%x\n",
-		dc->pc, dc->opcode, dc->op1, dc->op2);
-	fflush(NULL);
-	BUG();
-	return 2;
+    printf("unknown insn pc=%x opc=%x op1=%x op2=%x\n",
+        dc->pc, dc->opcode, dc->op1, dc->op2);
+    fflush(NULL);
+    BUG();
+    return 2;
 }
 
 static struct decoder_info {
-	struct {
-		uint32_t bits;
-		uint32_t mask;
-	};
-	int (*dec)(DisasContext *dc);
+    struct {
+        uint32_t bits;
+        uint32_t mask;
+    };
+    int (*dec)(CPUCRISState *env, DisasContext *dc);
 } decinfo[] = {
-	/* Order matters here.  */
-	{DEC_MOVEQ, dec_moveq},
-	{DEC_BTSTQ, dec_btstq},
-	{DEC_CMPQ, dec_cmpq},
-	{DEC_ADDOQ, dec_addoq},
-	{DEC_ADDQ, dec_addq},
-	{DEC_SUBQ, dec_subq},
-	{DEC_ANDQ, dec_andq},
-	{DEC_ORQ, dec_orq},
-	{DEC_ASRQ, dec_asrq},
-	{DEC_LSLQ, dec_lslq},
-	{DEC_LSRQ, dec_lsrq},
-	{DEC_BCCQ, dec_bccq},
-
-	{DEC_BCC_IM, dec_bcc_im},
-	{DEC_JAS_IM, dec_jas_im},
-	{DEC_JAS_R, dec_jas_r},
-	{DEC_JASC_IM, dec_jasc_im},
-	{DEC_JASC_R, dec_jasc_r},
-	{DEC_BAS_IM, dec_bas_im},
-	{DEC_BASC_IM, dec_basc_im},
-	{DEC_JUMP_P, dec_jump_p},
-	{DEC_LAPC_IM, dec_lapc_im},
-	{DEC_LAPCQ, dec_lapcq},
-
-	{DEC_RFE_ETC, dec_rfe_etc},
-	{DEC_ADDC_MR, dec_addc_mr},
-
-	{DEC_MOVE_MP, dec_move_mp},
-	{DEC_MOVE_PM, dec_move_pm},
-	{DEC_MOVEM_MR, dec_movem_mr},
-	{DEC_MOVEM_RM, dec_movem_rm},
-	{DEC_MOVE_PR, dec_move_pr},
-	{DEC_SCC_R, dec_scc_r},
-	{DEC_SETF, dec_setclrf},
-	{DEC_CLEARF, dec_setclrf},
-
-	{DEC_MOVE_SR, dec_move_sr},
-	{DEC_MOVE_RP, dec_move_rp},
-	{DEC_SWAP_R, dec_swap_r},
-	{DEC_ABS_R, dec_abs_r},
-	{DEC_LZ_R, dec_lz_r},
-	{DEC_MOVE_RS, dec_move_rs},
-	{DEC_BTST_R, dec_btst_r},
-	{DEC_ADDC_R, dec_addc_r},
-
-	{DEC_DSTEP_R, dec_dstep_r},
-	{DEC_XOR_R, dec_xor_r},
-	{DEC_MCP_R, dec_mcp_r},
-	{DEC_CMP_R, dec_cmp_r},
-
-	{DEC_ADDI_R, dec_addi_r},
-	{DEC_ADDI_ACR, dec_addi_acr},
-
-	{DEC_ADD_R, dec_add_r},
-	{DEC_SUB_R, dec_sub_r},
-
-	{DEC_ADDU_R, dec_addu_r},
-	{DEC_ADDS_R, dec_adds_r},
-	{DEC_SUBU_R, dec_subu_r},
-	{DEC_SUBS_R, dec_subs_r},
-	{DEC_LSL_R, dec_lsl_r},
-
-	{DEC_AND_R, dec_and_r},
-	{DEC_OR_R, dec_or_r},
-	{DEC_BOUND_R, dec_bound_r},
-	{DEC_ASR_R, dec_asr_r},
-	{DEC_LSR_R, dec_lsr_r},
-
-	{DEC_MOVU_R, dec_movu_r},
-	{DEC_MOVS_R, dec_movs_r},
-	{DEC_NEG_R, dec_neg_r},
-	{DEC_MOVE_R, dec_move_r},
-
-	{DEC_FTAG_FIDX_I_M, dec_ftag_fidx_i_m},
-	{DEC_FTAG_FIDX_D_M, dec_ftag_fidx_d_m},
-
-	{DEC_MULS_R, dec_muls_r},
-	{DEC_MULU_R, dec_mulu_r},
-
-	{DEC_ADDU_M, dec_addu_m},
-	{DEC_ADDS_M, dec_adds_m},
-	{DEC_SUBU_M, dec_subu_m},
-	{DEC_SUBS_M, dec_subs_m},
-
-	{DEC_CMPU_M, dec_cmpu_m},
-	{DEC_CMPS_M, dec_cmps_m},
-	{DEC_MOVU_M, dec_movu_m},
-	{DEC_MOVS_M, dec_movs_m},
-
-	{DEC_CMP_M, dec_cmp_m},
-	{DEC_ADDO_M, dec_addo_m},
-	{DEC_BOUND_M, dec_bound_m},
-	{DEC_ADD_M, dec_add_m},
-	{DEC_SUB_M, dec_sub_m},
-	{DEC_AND_M, dec_and_m},
-	{DEC_OR_M, dec_or_m},
-	{DEC_MOVE_RM, dec_move_rm},
-	{DEC_TEST_M, dec_test_m},
-	{DEC_MOVE_MR, dec_move_mr},
-
-	{{0, 0}, dec_null}
+    /* Order matters here.  */
+    {DEC_MOVEQ, dec_moveq},
+    {DEC_BTSTQ, dec_btstq},
+    {DEC_CMPQ, dec_cmpq},
+    {DEC_ADDOQ, dec_addoq},
+    {DEC_ADDQ, dec_addq},
+    {DEC_SUBQ, dec_subq},
+    {DEC_ANDQ, dec_andq},
+    {DEC_ORQ, dec_orq},
+    {DEC_ASRQ, dec_asrq},
+    {DEC_LSLQ, dec_lslq},
+    {DEC_LSRQ, dec_lsrq},
+    {DEC_BCCQ, dec_bccq},
+
+    {DEC_BCC_IM, dec_bcc_im},
+    {DEC_JAS_IM, dec_jas_im},
+    {DEC_JAS_R, dec_jas_r},
+    {DEC_JASC_IM, dec_jasc_im},
+    {DEC_JASC_R, dec_jasc_r},
+    {DEC_BAS_IM, dec_bas_im},
+    {DEC_BASC_IM, dec_basc_im},
+    {DEC_JUMP_P, dec_jump_p},
+    {DEC_LAPC_IM, dec_lapc_im},
+    {DEC_LAPCQ, dec_lapcq},
+
+    {DEC_RFE_ETC, dec_rfe_etc},
+    {DEC_ADDC_MR, dec_addc_mr},
+
+    {DEC_MOVE_MP, dec_move_mp},
+    {DEC_MOVE_PM, dec_move_pm},
+    {DEC_MOVEM_MR, dec_movem_mr},
+    {DEC_MOVEM_RM, dec_movem_rm},
+    {DEC_MOVE_PR, dec_move_pr},
+    {DEC_SCC_R, dec_scc_r},
+    {DEC_SETF, dec_setclrf},
+    {DEC_CLEARF, dec_setclrf},
+
+    {DEC_MOVE_SR, dec_move_sr},
+    {DEC_MOVE_RP, dec_move_rp},
+    {DEC_SWAP_R, dec_swap_r},
+    {DEC_ABS_R, dec_abs_r},
+    {DEC_LZ_R, dec_lz_r},
+    {DEC_MOVE_RS, dec_move_rs},
+    {DEC_BTST_R, dec_btst_r},
+    {DEC_ADDC_R, dec_addc_r},
+
+    {DEC_DSTEP_R, dec_dstep_r},
+    {DEC_XOR_R, dec_xor_r},
+    {DEC_MCP_R, dec_mcp_r},
+    {DEC_CMP_R, dec_cmp_r},
+
+    {DEC_ADDI_R, dec_addi_r},
+    {DEC_ADDI_ACR, dec_addi_acr},
+
+    {DEC_ADD_R, dec_add_r},
+    {DEC_SUB_R, dec_sub_r},
+
+    {DEC_ADDU_R, dec_addu_r},
+    {DEC_ADDS_R, dec_adds_r},
+    {DEC_SUBU_R, dec_subu_r},
+    {DEC_SUBS_R, dec_subs_r},
+    {DEC_LSL_R, dec_lsl_r},
+
+    {DEC_AND_R, dec_and_r},
+    {DEC_OR_R, dec_or_r},
+    {DEC_BOUND_R, dec_bound_r},
+    {DEC_ASR_R, dec_asr_r},
+    {DEC_LSR_R, dec_lsr_r},
+
+    {DEC_MOVU_R, dec_movu_r},
+    {DEC_MOVS_R, dec_movs_r},
+    {DEC_NEG_R, dec_neg_r},
+    {DEC_MOVE_R, dec_move_r},
+
+    {DEC_FTAG_FIDX_I_M, dec_ftag_fidx_i_m},
+    {DEC_FTAG_FIDX_D_M, dec_ftag_fidx_d_m},
+
+    {DEC_MULS_R, dec_muls_r},
+    {DEC_MULU_R, dec_mulu_r},
+
+    {DEC_ADDU_M, dec_addu_m},
+    {DEC_ADDS_M, dec_adds_m},
+    {DEC_SUBU_M, dec_subu_m},
+    {DEC_SUBS_M, dec_subs_m},
+
+    {DEC_CMPU_M, dec_cmpu_m},
+    {DEC_CMPS_M, dec_cmps_m},
+    {DEC_MOVU_M, dec_movu_m},
+    {DEC_MOVS_M, dec_movs_m},
+
+    {DEC_CMP_M, dec_cmp_m},
+    {DEC_ADDO_M, dec_addo_m},
+    {DEC_BOUND_M, dec_bound_m},
+    {DEC_ADD_M, dec_add_m},
+    {DEC_SUB_M, dec_sub_m},
+    {DEC_AND_M, dec_and_m},
+    {DEC_OR_M, dec_or_m},
+    {DEC_MOVE_RM, dec_move_rm},
+    {DEC_TEST_M, dec_test_m},
+    {DEC_MOVE_MR, dec_move_mr},
+
+    {{0, 0}, dec_null}
 };
 
-static unsigned int crisv32_decoder(DisasContext *dc)
+static unsigned int crisv32_decoder(CPUCRISState *env, DisasContext *dc)
 {
-	int insn_len = 2;
-	int i;
-
-	if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP)))
-		tcg_gen_debug_insn_start(dc->pc);
-
-	/* Load a halfword onto the instruction register.  */
-	dc->ir = cris_fetch(dc, dc->pc, 2, 0);
+    int insn_len = 2;
+    int i;
 
-	/* Now decode it.  */
-	dc->opcode   = EXTRACT_FIELD(dc->ir, 4, 11);
-	dc->op1      = EXTRACT_FIELD(dc->ir, 0, 3);
-	dc->op2      = EXTRACT_FIELD(dc->ir, 12, 15);
-	dc->zsize    = EXTRACT_FIELD(dc->ir, 4, 4);
-	dc->zzsize   = EXTRACT_FIELD(dc->ir, 4, 5);
-	dc->postinc  = EXTRACT_FIELD(dc->ir, 10, 10);
+    if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
+        tcg_gen_debug_insn_start(dc->pc);
+        }
 
-	/* Large switch for all insns.  */
-	for (i = 0; i < ARRAY_SIZE(decinfo); i++) {
-		if ((dc->opcode & decinfo[i].mask) == decinfo[i].bits)
-		{
-			insn_len = decinfo[i].dec(dc);
-			break;
-		}
-	}
+    /* Load a halfword onto the instruction register.  */
+        dc->ir = cris_fetch(env, dc, dc->pc, 2, 0);
+
+    /* Now decode it.  */
+    dc->opcode   = EXTRACT_FIELD(dc->ir, 4, 11);
+    dc->op1      = EXTRACT_FIELD(dc->ir, 0, 3);
+    dc->op2      = EXTRACT_FIELD(dc->ir, 12, 15);
+    dc->zsize    = EXTRACT_FIELD(dc->ir, 4, 4);
+    dc->zzsize   = EXTRACT_FIELD(dc->ir, 4, 5);
+    dc->postinc  = EXTRACT_FIELD(dc->ir, 10, 10);
+
+    /* Large switch for all insns.  */
+    for (i = 0; i < ARRAY_SIZE(decinfo); i++) {
+        if ((dc->opcode & decinfo[i].mask) == decinfo[i].bits) {
+            insn_len = decinfo[i].dec(env, dc);
+            break;
+        }
+    }
 
 #if !defined(CONFIG_USER_ONLY)
-	/* Single-stepping ?  */
-	if (dc->tb_flags & S_FLAG) {
-		int l1;
-
-		l1 = gen_new_label();
-		tcg_gen_brcondi_tl(TCG_COND_NE, cpu_PR[PR_SPC], dc->pc, l1);
-		/* We treat SPC as a break with an odd trap vector.  */
-		cris_evaluate_flags (dc);
-		t_gen_mov_env_TN(trap_vector, tcg_const_tl(3));
-		tcg_gen_movi_tl(env_pc, dc->pc + insn_len);
-		tcg_gen_movi_tl(cpu_PR[PR_SPC], dc->pc + insn_len);
-		t_gen_raise_exception(EXCP_BREAK);
-		gen_set_label(l1);
-	}
+    /* Single-stepping ?  */
+    if (dc->tb_flags & S_FLAG) {
+        int l1;
+
+        l1 = gen_new_label();
+        tcg_gen_brcondi_tl(TCG_COND_NE, cpu_PR[PR_SPC], dc->pc, l1);
+        /* We treat SPC as a break with an odd trap vector.  */
+        cris_evaluate_flags(dc);
+        t_gen_mov_env_TN(trap_vector, tcg_const_tl(3));
+        tcg_gen_movi_tl(env_pc, dc->pc + insn_len);
+        tcg_gen_movi_tl(cpu_PR[PR_SPC], dc->pc + insn_len);
+        t_gen_raise_exception(EXCP_BREAK);
+        gen_set_label(l1);
+    }
 #endif
-	return insn_len;
+    return insn_len;
 }
 
 static void check_breakpoint(CPUCRISState *env, DisasContext *dc)
 {
-	CPUBreakpoint *bp;
+    CPUBreakpoint *bp;
 
-	if (unlikely(!QTAILQ_EMPTY(&env->breakpoints))) {
-		QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
-			if (bp->pc == dc->pc) {
-				cris_evaluate_flags (dc);
-				tcg_gen_movi_tl(env_pc, dc->pc);
-				t_gen_raise_exception(EXCP_DEBUG);
-				dc->is_jmp = DISAS_UPDATE;
-			}
-		}
-	}
+    if (unlikely(!QTAILQ_EMPTY(&env->breakpoints))) {
+        QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
+            if (bp->pc == dc->pc) {
+                cris_evaluate_flags(dc);
+                tcg_gen_movi_tl(env_pc, dc->pc);
+                t_gen_raise_exception(EXCP_DEBUG);
+                dc->is_jmp = DISAS_UPDATE;
+            }
+        }
+    }
 }
 
 #include "translate_v10.c"
@@ -3171,250 +3204,256 @@ static void
 gen_intermediate_code_internal(CPUCRISState *env, TranslationBlock *tb,
                                int search_pc)
 {
-	uint16_t *gen_opc_end;
-   	uint32_t pc_start;
-	unsigned int insn_len;
-	int j, lj;
-	struct DisasContext ctx;
-	struct DisasContext *dc = &ctx;
-	uint32_t next_page_start;
-	target_ulong npc;
-        int num_insns;
-        int max_insns;
-
-	qemu_log_try_set_file(stderr);
-
-	if (env->pregs[PR_VR] == 32) {
-		dc->decoder = crisv32_decoder;
-		dc->clear_locked_irq = 0;
-	} else {
-		dc->decoder = crisv10_decoder;
-		dc->clear_locked_irq = 1;
-	}
-
-	/* Odd PC indicates that branch is rexecuting due to exception in the
-	 * delayslot, like in real hw.
-	 */
-	pc_start = tb->pc & ~1;
-	dc->env = env;
-	dc->tb = tb;
-
-	gen_opc_end = gen_opc_buf + OPC_MAX_SIZE;
-
-	dc->is_jmp = DISAS_NEXT;
-	dc->ppc = pc_start;
-	dc->pc = pc_start;
-	dc->singlestep_enabled = env->singlestep_enabled;
-	dc->flags_uptodate = 1;
-	dc->flagx_known = 1;
-	dc->flags_x = tb->flags & X_FLAG;
-	dc->cc_x_uptodate = 0;
-	dc->cc_mask = 0;
-	dc->update_cc = 0;
-	dc->clear_prefix = 0;
-
-	cris_update_cc_op(dc, CC_OP_FLAGS, 4);
-	dc->cc_size_uptodate = -1;
-
-	/* Decode TB flags.  */
-	dc->tb_flags = tb->flags & (S_FLAG | P_FLAG | U_FLAG \
-					| X_FLAG | PFIX_FLAG);
-	dc->delayed_branch = !!(tb->flags & 7);
-	if (dc->delayed_branch)
-		dc->jmp = JMP_INDIRECT;
-	else
-		dc->jmp = JMP_NOJMP;
-
-	dc->cpustate_changed = 0;
-
-	if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
-		qemu_log(
-                        "srch=%d pc=%x %x flg=%" PRIx64 " bt=%x ds=%u ccs=%x\n"
-			"pid=%x usp=%x\n"
-			"%x.%x.%x.%x\n"
-			"%x.%x.%x.%x\n"
-			"%x.%x.%x.%x\n"
-			"%x.%x.%x.%x\n",
-			search_pc, dc->pc, dc->ppc,
-                        (uint64_t)tb->flags,
-			env->btarget, (unsigned)tb->flags & 7,
-			env->pregs[PR_CCS], 
-			env->pregs[PR_PID], env->pregs[PR_USP],
-			env->regs[0], env->regs[1], env->regs[2], env->regs[3],
-			env->regs[4], env->regs[5], env->regs[6], env->regs[7],
-			env->regs[8], env->regs[9],
-			env->regs[10], env->regs[11],
-			env->regs[12], env->regs[13],
-			env->regs[14], env->regs[15]);
-		qemu_log("--------------\n");
-		qemu_log("IN: %s\n", lookup_symbol(pc_start));
-	}
-
-	next_page_start = (pc_start & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE;
-	lj = -1;
-        num_insns = 0;
-        max_insns = tb->cflags & CF_COUNT_MASK;
-        if (max_insns == 0)
-            max_insns = CF_COUNT_MASK;
-
-        gen_icount_start();
-	do
-	{
-		check_breakpoint(env, dc);
-
-		if (search_pc) {
-			j = gen_opc_ptr - gen_opc_buf;
-			if (lj < j) {
-				lj++;
-				while (lj < j)
-					gen_opc_instr_start[lj++] = 0;
-			}
-			if (dc->delayed_branch == 1)
-				gen_opc_pc[lj] = dc->ppc | 1;
-			else
-				gen_opc_pc[lj] = dc->pc;
-			gen_opc_instr_start[lj] = 1;
-                        gen_opc_icount[lj] = num_insns;
-		}
-
-		/* Pretty disas.  */
-		LOG_DIS("%8.8x:\t", dc->pc);
-
-                if (num_insns + 1 == max_insns && (tb->cflags & CF_LAST_IO))
-                    gen_io_start();
-		dc->clear_x = 1;
-
-		insn_len = dc->decoder(dc);
-		dc->ppc = dc->pc;
-		dc->pc += insn_len;
-		if (dc->clear_x)
-			cris_clear_x_flag(dc);
-
-                num_insns++;
-		/* Check for delayed branches here. If we do it before
-		   actually generating any host code, the simulator will just
-		   loop doing nothing for on this program location.  */
-		if (dc->delayed_branch) {
-			dc->delayed_branch--;
-			if (dc->delayed_branch == 0)
-			{
-				if (tb->flags & 7)
-					t_gen_mov_env_TN(dslot, 
-						tcg_const_tl(0));
-				if (dc->cpustate_changed || !dc->flagx_known
-				    || (dc->flags_x != (tb->flags & X_FLAG))) {
-					cris_store_direct_jmp(dc);
-				}
-
-				if (dc->clear_locked_irq) {
-					dc->clear_locked_irq = 0;
-					t_gen_mov_env_TN(locked_irq,
-							 tcg_const_tl(0));
-				}
-
-				if (dc->jmp == JMP_DIRECT_CC) {
-					int l1;
-
-					l1 = gen_new_label();
-					cris_evaluate_flags(dc);
-
-					/* Conditional jmp.  */
-					tcg_gen_brcondi_tl(TCG_COND_EQ,
-							   env_btaken, 0, l1);
-					gen_goto_tb(dc, 1, dc->jmp_pc);
-					gen_set_label(l1);
-					gen_goto_tb(dc, 0, dc->pc);
-					dc->is_jmp = DISAS_TB_JUMP;
-					dc->jmp = JMP_NOJMP;
-				} else if (dc->jmp == JMP_DIRECT) {
-					cris_evaluate_flags(dc);
-					gen_goto_tb(dc, 0, dc->jmp_pc);
-					dc->is_jmp = DISAS_TB_JUMP;
-					dc->jmp = JMP_NOJMP;
-				} else {
-					t_gen_cc_jmp(env_btarget, 
-						     tcg_const_tl(dc->pc));
-					dc->is_jmp = DISAS_JUMP;
-				}
-				break;
-			}
-		}
-
-		/* If we are rexecuting a branch due to exceptions on
-		   delay slots dont break.  */
-		if (!(tb->pc & 1) && env->singlestep_enabled)
-			break;
-	} while (!dc->is_jmp && !dc->cpustate_changed
-		 && gen_opc_ptr < gen_opc_end
-                 && !singlestep
-		 && (dc->pc < next_page_start)
-                 && num_insns < max_insns);
-
-	if (dc->clear_locked_irq)
-		t_gen_mov_env_TN(locked_irq, tcg_const_tl(0));
-
-	npc = dc->pc;
+    uint16_t *gen_opc_end;
+    uint32_t pc_start;
+    unsigned int insn_len;
+    int j, lj;
+    struct DisasContext ctx;
+    struct DisasContext *dc = &ctx;
+    uint32_t next_page_start;
+    target_ulong npc;
+    int num_insns;
+    int max_insns;
+
+    qemu_log_try_set_file(stderr);
+
+    if (env->pregs[PR_VR] == 32) {
+        dc->decoder = crisv32_decoder;
+        dc->clear_locked_irq = 0;
+    } else {
+        dc->decoder = crisv10_decoder;
+        dc->clear_locked_irq = 1;
+    }
+
+    /* Odd PC indicates that branch is rexecuting due to exception in the
+     * delayslot, like in real hw.
+     */
+    pc_start = tb->pc & ~1;
+    dc->env = env;
+    dc->tb = tb;
+
+    gen_opc_end = tcg_ctx.gen_opc_buf + OPC_MAX_SIZE;
+
+    dc->is_jmp = DISAS_NEXT;
+    dc->ppc = pc_start;
+    dc->pc = pc_start;
+    dc->singlestep_enabled = env->singlestep_enabled;
+    dc->flags_uptodate = 1;
+    dc->flagx_known = 1;
+    dc->flags_x = tb->flags & X_FLAG;
+    dc->cc_x_uptodate = 0;
+    dc->cc_mask = 0;
+    dc->update_cc = 0;
+    dc->clear_prefix = 0;
+
+    cris_update_cc_op(dc, CC_OP_FLAGS, 4);
+    dc->cc_size_uptodate = -1;
+
+    /* Decode TB flags.  */
+    dc->tb_flags = tb->flags & (S_FLAG | P_FLAG | U_FLAG \
+            | X_FLAG | PFIX_FLAG);
+    dc->delayed_branch = !!(tb->flags & 7);
+    if (dc->delayed_branch) {
+        dc->jmp = JMP_INDIRECT;
+    } else {
+        dc->jmp = JMP_NOJMP;
+    }
+
+    dc->cpustate_changed = 0;
+
+    if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
+        qemu_log(
+                "srch=%d pc=%x %x flg=%" PRIx64 " bt=%x ds=%u ccs=%x\n"
+                "pid=%x usp=%x\n"
+                "%x.%x.%x.%x\n"
+                "%x.%x.%x.%x\n"
+                "%x.%x.%x.%x\n"
+                "%x.%x.%x.%x\n",
+                search_pc, dc->pc, dc->ppc,
+                (uint64_t)tb->flags,
+                env->btarget, (unsigned)tb->flags & 7,
+                env->pregs[PR_CCS],
+                env->pregs[PR_PID], env->pregs[PR_USP],
+                env->regs[0], env->regs[1], env->regs[2], env->regs[3],
+                env->regs[4], env->regs[5], env->regs[6], env->regs[7],
+                env->regs[8], env->regs[9],
+                env->regs[10], env->regs[11],
+                env->regs[12], env->regs[13],
+                env->regs[14], env->regs[15]);
+        qemu_log("--------------\n");
+        qemu_log("IN: %s\n", lookup_symbol(pc_start));
+    }
+
+    next_page_start = (pc_start & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE;
+    lj = -1;
+    num_insns = 0;
+    max_insns = tb->cflags & CF_COUNT_MASK;
+    if (max_insns == 0) {
+        max_insns = CF_COUNT_MASK;
+    }
+
+    gen_icount_start();
+    do {
+        check_breakpoint(env, dc);
+
+        if (search_pc) {
+            j = tcg_ctx.gen_opc_ptr - tcg_ctx.gen_opc_buf;
+            if (lj < j) {
+                lj++;
+                while (lj < j) {
+                    gen_opc_instr_start[lj++] = 0;
+                }
+            }
+            if (dc->delayed_branch == 1) {
+                gen_opc_pc[lj] = dc->ppc | 1;
+            } else {
+                gen_opc_pc[lj] = dc->pc;
+            }
+            gen_opc_instr_start[lj] = 1;
+            gen_opc_icount[lj] = num_insns;
+        }
+
+        /* Pretty disas.  */
+        LOG_DIS("%8.8x:\t", dc->pc);
+
+        if (num_insns + 1 == max_insns && (tb->cflags & CF_LAST_IO)) {
+            gen_io_start();
+        }
+        dc->clear_x = 1;
+
+        insn_len = dc->decoder(env, dc);
+        dc->ppc = dc->pc;
+        dc->pc += insn_len;
+        if (dc->clear_x) {
+            cris_clear_x_flag(dc);
+        }
+
+        num_insns++;
+        /* Check for delayed branches here. If we do it before
+           actually generating any host code, the simulator will just
+           loop doing nothing for on this program location.  */
+        if (dc->delayed_branch) {
+            dc->delayed_branch--;
+            if (dc->delayed_branch == 0) {
+                if (tb->flags & 7) {
+                    t_gen_mov_env_TN(dslot, tcg_const_tl(0));
+                }
+                if (dc->cpustate_changed || !dc->flagx_known
+                    || (dc->flags_x != (tb->flags & X_FLAG))) {
+                    cris_store_direct_jmp(dc);
+                }
+
+                if (dc->clear_locked_irq) {
+                    dc->clear_locked_irq = 0;
+                    t_gen_mov_env_TN(locked_irq, tcg_const_tl(0));
+                }
+
+                if (dc->jmp == JMP_DIRECT_CC) {
+                    int l1;
+
+                    l1 = gen_new_label();
+                    cris_evaluate_flags(dc);
+
+                    /* Conditional jmp.  */
+                    tcg_gen_brcondi_tl(TCG_COND_EQ,
+                               env_btaken, 0, l1);
+                    gen_goto_tb(dc, 1, dc->jmp_pc);
+                    gen_set_label(l1);
+                    gen_goto_tb(dc, 0, dc->pc);
+                    dc->is_jmp = DISAS_TB_JUMP;
+                    dc->jmp = JMP_NOJMP;
+                } else if (dc->jmp == JMP_DIRECT) {
+                    cris_evaluate_flags(dc);
+                    gen_goto_tb(dc, 0, dc->jmp_pc);
+                    dc->is_jmp = DISAS_TB_JUMP;
+                    dc->jmp = JMP_NOJMP;
+                } else {
+                    t_gen_cc_jmp(env_btarget, tcg_const_tl(dc->pc));
+                    dc->is_jmp = DISAS_JUMP;
+                }
+                break;
+            }
+        }
+
+        /* If we are rexecuting a branch due to exceptions on
+           delay slots dont break.  */
+        if (!(tb->pc & 1) && env->singlestep_enabled) {
+            break;
+        }
+    } while (!dc->is_jmp && !dc->cpustate_changed
+            && tcg_ctx.gen_opc_ptr < gen_opc_end
+            && !singlestep
+            && (dc->pc < next_page_start)
+            && num_insns < max_insns);
+
+    if (dc->clear_locked_irq) {
+        t_gen_mov_env_TN(locked_irq, tcg_const_tl(0));
+    }
+
+    npc = dc->pc;
 
         if (tb->cflags & CF_LAST_IO)
             gen_io_end();
-	/* Force an update if the per-tb cpu state has changed.  */
-	if (dc->is_jmp == DISAS_NEXT
-	    && (dc->cpustate_changed || !dc->flagx_known 
-	    || (dc->flags_x != (tb->flags & X_FLAG)))) {
-		dc->is_jmp = DISAS_UPDATE;
-		tcg_gen_movi_tl(env_pc, npc);
-	}
-	/* Broken branch+delayslot sequence.  */
-	if (dc->delayed_branch == 1) {
-		/* Set env->dslot to the size of the branch insn.  */
-		t_gen_mov_env_TN(dslot, tcg_const_tl(dc->pc - dc->ppc));
-		cris_store_direct_jmp(dc);
-	}
-
-	cris_evaluate_flags (dc);
-
-	if (unlikely(env->singlestep_enabled)) {
-		if (dc->is_jmp == DISAS_NEXT)
-			tcg_gen_movi_tl(env_pc, npc);
-		t_gen_raise_exception(EXCP_DEBUG);
-	} else {
-		switch(dc->is_jmp) {
-			case DISAS_NEXT:
-				gen_goto_tb(dc, 1, npc);
-				break;
-			default:
-			case DISAS_JUMP:
-			case DISAS_UPDATE:
-				/* indicate that the hash table must be used
-				   to find the next TB */
-				tcg_gen_exit_tb(0);
-				break;
-			case DISAS_SWI:
-			case DISAS_TB_JUMP:
-				/* nothing more to generate */
-				break;
-		}
-	}
-        gen_icount_end(tb, num_insns);
-	*gen_opc_ptr = INDEX_op_end;
-	if (search_pc) {
-		j = gen_opc_ptr - gen_opc_buf;
-		lj++;
-		while (lj <= j)
-			gen_opc_instr_start[lj++] = 0;
-	} else {
-		tb->size = dc->pc - pc_start;
-                tb->icount = num_insns;
-	}
+    /* Force an update if the per-tb cpu state has changed.  */
+    if (dc->is_jmp == DISAS_NEXT
+        && (dc->cpustate_changed || !dc->flagx_known
+        || (dc->flags_x != (tb->flags & X_FLAG)))) {
+        dc->is_jmp = DISAS_UPDATE;
+        tcg_gen_movi_tl(env_pc, npc);
+    }
+    /* Broken branch+delayslot sequence.  */
+    if (dc->delayed_branch == 1) {
+        /* Set env->dslot to the size of the branch insn.  */
+        t_gen_mov_env_TN(dslot, tcg_const_tl(dc->pc - dc->ppc));
+        cris_store_direct_jmp(dc);
+    }
+
+    cris_evaluate_flags(dc);
+
+    if (unlikely(env->singlestep_enabled)) {
+        if (dc->is_jmp == DISAS_NEXT) {
+            tcg_gen_movi_tl(env_pc, npc);
+        }
+        t_gen_raise_exception(EXCP_DEBUG);
+    } else {
+        switch (dc->is_jmp) {
+        case DISAS_NEXT:
+            gen_goto_tb(dc, 1, npc);
+            break;
+        default:
+        case DISAS_JUMP:
+        case DISAS_UPDATE:
+            /* indicate that the hash table must be used
+                   to find the next TB */
+            tcg_gen_exit_tb(0);
+            break;
+        case DISAS_SWI:
+        case DISAS_TB_JUMP:
+            /* nothing more to generate */
+            break;
+        }
+    }
+    gen_icount_end(tb, num_insns);
+    *tcg_ctx.gen_opc_ptr = INDEX_op_end;
+    if (search_pc) {
+        j = tcg_ctx.gen_opc_ptr - tcg_ctx.gen_opc_buf;
+        lj++;
+        while (lj <= j) {
+            gen_opc_instr_start[lj++] = 0;
+        }
+    } else {
+        tb->size = dc->pc - pc_start;
+        tb->icount = num_insns;
+    }
 
 #ifdef DEBUG_DISAS
 #if !DISAS_CRIS
-	if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
-		log_target_disas(pc_start, dc->pc - pc_start,
+    if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
+        log_target_disas(env, pc_start, dc->pc - pc_start,
                                  dc->env->pregs[PR_VR]);
-		qemu_log("\nisize=%d osize=%td\n",
-			dc->pc - pc_start, gen_opc_ptr - gen_opc_buf);
-	}
+        qemu_log("\nisize=%d osize=%td\n",
+            dc->pc - pc_start, tcg_ctx.gen_opc_ptr - tcg_ctx.gen_opc_buf);
+    }
 #endif
 #endif
 }
@@ -3432,41 +3471,45 @@ void gen_intermediate_code_pc (CPUCRISState *env, struct TranslationBlock *tb)
 void cpu_dump_state (CPUCRISState *env, FILE *f, fprintf_function cpu_fprintf,
                      int flags)
 {
-	int i;
-	uint32_t srs;
-
-	if (!env || !f)
-		return;
-
-	cpu_fprintf(f, "PC=%x CCS=%x btaken=%d btarget=%x\n"
-		    "cc_op=%d cc_src=%d cc_dest=%d cc_result=%x cc_mask=%x\n",
-		    env->pc, env->pregs[PR_CCS], env->btaken, env->btarget,
-		    env->cc_op,
-		    env->cc_src, env->cc_dest, env->cc_result, env->cc_mask);
-
-
-	for (i = 0; i < 16; i++) {
-		cpu_fprintf(f, "%s=%8.8x ",regnames[i], env->regs[i]);
-		if ((i + 1) % 4 == 0)
-			cpu_fprintf(f, "\n");
-	}
-	cpu_fprintf(f, "\nspecial regs:\n");
-	for (i = 0; i < 16; i++) {
-		cpu_fprintf(f, "%s=%8.8x ", pregnames[i], env->pregs[i]);
-		if ((i + 1) % 4 == 0)
-			cpu_fprintf(f, "\n");
-	}
-	srs = env->pregs[PR_SRS];
-	cpu_fprintf(f, "\nsupport function regs bank %x:\n", srs);
-	if (srs < 256) {
-		for (i = 0; i < 16; i++) {
-			cpu_fprintf(f, "s%2.2d=%8.8x ",
-				    i, env->sregs[srs][i]);
-			if ((i + 1) % 4 == 0)
-				cpu_fprintf(f, "\n");
-		}
-	}
-	cpu_fprintf(f, "\n\n");
+    int i;
+    uint32_t srs;
+
+    if (!env || !f) {
+        return;
+    }
+
+    cpu_fprintf(f, "PC=%x CCS=%x btaken=%d btarget=%x\n"
+            "cc_op=%d cc_src=%d cc_dest=%d cc_result=%x cc_mask=%x\n",
+            env->pc, env->pregs[PR_CCS], env->btaken, env->btarget,
+            env->cc_op,
+            env->cc_src, env->cc_dest, env->cc_result, env->cc_mask);
+
+
+    for (i = 0; i < 16; i++) {
+        cpu_fprintf(f, "%s=%8.8x ", regnames[i], env->regs[i]);
+        if ((i + 1) % 4 == 0) {
+            cpu_fprintf(f, "\n");
+        }
+    }
+    cpu_fprintf(f, "\nspecial regs:\n");
+    for (i = 0; i < 16; i++) {
+        cpu_fprintf(f, "%s=%8.8x ", pregnames[i], env->pregs[i]);
+        if ((i + 1) % 4 == 0) {
+            cpu_fprintf(f, "\n");
+        }
+    }
+    srs = env->pregs[PR_SRS];
+    cpu_fprintf(f, "\nsupport function regs bank %x:\n", srs);
+    if (srs < ARRAY_SIZE(env->sregs)) {
+        for (i = 0; i < 16; i++) {
+            cpu_fprintf(f, "s%2.2d=%8.8x ",
+                    i, env->sregs[srs][i]);
+            if ((i + 1) % 4 == 0) {
+                cpu_fprintf(f, "\n");
+            }
+        }
+    }
+    cpu_fprintf(f, "\n\n");
 
 }
 
@@ -3475,11 +3518,11 @@ struct
     uint32_t vr;
     const char *name;
 } cris_cores[] = {
-	{8, "crisv8"},
-	{9, "crisv9"},
-	{10, "crisv10"},
-	{11, "crisv11"},
-	{32, "crisv32"},
+    {8, "crisv8"},
+    {9, "crisv9"},
+    {10, "crisv10"},
+    {11, "crisv11"},
+    {32, "crisv32"},
 };
 
 void cris_cpu_list(FILE *f, fprintf_function cpu_fprintf)
@@ -3578,5 +3621,5 @@ CRISCPU *cpu_cris_init(const char *cpu_model)
 
 void restore_state_to_opc(CPUCRISState *env, TranslationBlock *tb, int pc_pos)
 {
-	env->pc = gen_opc_pc[pc_pos];
+    env->pc = gen_opc_pc[pc_pos];
 }
diff --git a/target-cris/translate_v10.c b/target-cris/translate_v10.c
index 3629629d9..d2cca892e 100644
--- a/target-cris/translate_v10.c
+++ b/target-cris/translate_v10.c
@@ -164,8 +164,8 @@ static unsigned int crisv10_post_memaddr(DisasContext *dc, unsigned int size)
     return insn_len;
 }
 
-static int dec10_prep_move_m(DisasContext *dc, int s_ext, int memsize,
-                           TCGv dst)
+static int dec10_prep_move_m(CPUCRISState *env, DisasContext *dc,
+                             int s_ext, int memsize, TCGv dst)
 {
     unsigned int rs;
     uint32_t imm;
@@ -182,17 +182,17 @@ static int dec10_prep_move_m(DisasContext *dc, int s_ext, int memsize,
         if (memsize != 4) {
             if (s_ext) {
                 if (memsize == 1)
-                    imm = ldsb_code(dc->pc + 2);
+                    imm = cpu_ldsb_code(env, dc->pc + 2);
                 else
-                    imm = ldsw_code(dc->pc + 2);
+                    imm = cpu_ldsw_code(env, dc->pc + 2);
             } else {
                 if (memsize == 1)
-                    imm = ldub_code(dc->pc + 2);
+                    imm = cpu_ldub_code(env, dc->pc + 2);
                 else
-                    imm = lduw_code(dc->pc + 2);
+                    imm = cpu_lduw_code(env, dc->pc + 2);
             }
         } else
-            imm = ldl_code(dc->pc + 2);
+            imm = cpu_ldl_code(env, dc->pc + 2);
 
         tcg_gen_movi_tl(dst, imm);
 
@@ -289,7 +289,7 @@ static unsigned int dec10_quick_imm(DisasContext *dc)
             } else {
                 /* BTST */
                 cris_update_cc_op(dc, CC_OP_FLAGS, 4);
-                gen_helper_btst(cpu_PR[PR_CCS], cpu_R[dc->dst],
+                gen_helper_btst(cpu_PR[PR_CCS], cpu_env, cpu_R[dc->dst],
                            tcg_const_tl(imm), cpu_PR[PR_CCS]);
             }
             break;
@@ -723,7 +723,7 @@ static unsigned int dec10_reg(DisasContext *dc)
                 LOG_DIS("btst $r%d, $r%d sz=%d\n", dc->src, dc->dst, size);
                 cris_cc_mask(dc, CC_MASK_NZVC);
                 cris_update_cc_op(dc, CC_OP_FLAGS, 4);
-                gen_helper_btst(cpu_PR[PR_CCS], cpu_R[dc->dst],
+                gen_helper_btst(cpu_PR[PR_CCS], cpu_env, cpu_R[dc->dst],
                            cpu_R[dc->src], cpu_PR[PR_CCS]);
                 break;
             case CRISV10_REG_DSTEP:
@@ -752,7 +752,8 @@ static unsigned int dec10_reg(DisasContext *dc)
     return insn_len;
 }
 
-static unsigned int dec10_ind_move_m_r(DisasContext *dc, unsigned int size)
+static unsigned int dec10_ind_move_m_r(CPUCRISState *env, DisasContext *dc,
+                                       unsigned int size)
 {
     unsigned int insn_len = 2;
     TCGv t;
@@ -762,7 +763,7 @@ static unsigned int dec10_ind_move_m_r(DisasContext *dc, unsigned int size)
 
     cris_cc_mask(dc, CC_MASK_NZVC);
     t = tcg_temp_new();
-    insn_len += dec10_prep_move_m(dc, 0, size, t);
+    insn_len += dec10_prep_move_m(env, dc, 0, size, t);
     cris_alu(dc, CC_OP_MOVE, cpu_R[dc->dst], cpu_R[dc->dst], t, size);
     if (dc->dst == 15) {
         tcg_gen_mov_tl(env_btarget, cpu_R[dc->dst]);
@@ -789,7 +790,7 @@ static unsigned int dec10_ind_move_r_m(DisasContext *dc, unsigned int size)
     return insn_len;
 }
 
-static unsigned int dec10_ind_move_m_pr(DisasContext *dc)
+static unsigned int dec10_ind_move_m_pr(CPUCRISState *env, DisasContext *dc)
 {
     unsigned int insn_len = 2, rd = dc->dst;
     TCGv t, addr;
@@ -799,7 +800,7 @@ static unsigned int dec10_ind_move_m_pr(DisasContext *dc)
 
     addr = tcg_temp_new();
     t = tcg_temp_new();
-    insn_len += dec10_prep_move_m(dc, 0, 4, t);
+    insn_len += dec10_prep_move_m(env, dc, 0, 4, t);
     if (rd == 15) {
         tcg_gen_mov_tl(env_btarget, t);
         cris_prepare_jmp(dc, JMP_INDIRECT);
@@ -899,14 +900,15 @@ static void dec10_movem_m_r(DisasContext *dc)
     tcg_temp_free(t0);
 }
 
-static int dec10_ind_alu(DisasContext *dc, int op, unsigned int size)
+static int dec10_ind_alu(CPUCRISState *env, DisasContext *dc,
+                         int op, unsigned int size)
 {
     int insn_len = 0;
     int rd = dc->dst;
     TCGv t[2];
 
     cris_alu_m_alloc_temps(t);
-    insn_len += dec10_prep_move_m(dc, 0, size, t[0]);
+    insn_len += dec10_prep_move_m(env, dc, 0, size, t[0]);
     cris_alu(dc, op, cpu_R[dc->dst], cpu_R[rd], t[0], size);
     if (dc->dst == 15) {
         tcg_gen_mov_tl(env_btarget, cpu_R[dc->dst]);
@@ -920,14 +922,15 @@ static int dec10_ind_alu(DisasContext *dc, int op, unsigned int size)
     return insn_len;
 }
 
-static int dec10_ind_bound(DisasContext *dc, unsigned int size)
+static int dec10_ind_bound(CPUCRISState *env, DisasContext *dc,
+                           unsigned int size)
 {
     int insn_len = 0;
     int rd = dc->dst;
     TCGv t;
 
     t = tcg_temp_local_new();
-    insn_len += dec10_prep_move_m(dc, 0, size, t);
+    insn_len += dec10_prep_move_m(env, dc, 0, size, t);
     cris_alu(dc, CC_OP_BOUND, cpu_R[dc->dst], cpu_R[rd], t, 4);
     if (dc->dst == 15) {
         tcg_gen_mov_tl(env_btarget, cpu_R[dc->dst]);
@@ -940,7 +943,7 @@ static int dec10_ind_bound(DisasContext *dc, unsigned int size)
     return insn_len;
 }
 
-static int dec10_alux_m(DisasContext *dc, int op)
+static int dec10_alux_m(CPUCRISState *env, DisasContext *dc, int op)
 {
     unsigned int size = (dc->size & 1) ? 2 : 1;
     unsigned int sx = !!(dc->size & 2);
@@ -953,7 +956,7 @@ static int dec10_alux_m(DisasContext *dc, int op)
     t = tcg_temp_new();
 
     cris_cc_mask(dc, CC_MASK_NZVC);
-    insn_len += dec10_prep_move_m(dc, sx, size, t);
+    insn_len += dec10_prep_move_m(env, dc, sx, size, t);
     cris_alu(dc, op, cpu_R[dc->dst], cpu_R[rd], t, 4);
     if (dc->dst == 15) {
         tcg_gen_mov_tl(env_btarget, cpu_R[dc->dst]);
@@ -966,7 +969,7 @@ static int dec10_alux_m(DisasContext *dc, int op)
     return insn_len;
 }
 
-static int dec10_dip(DisasContext *dc)
+static int dec10_dip(CPUCRISState *env, DisasContext *dc)
 {
     int insn_len = 2;
     uint32_t imm;
@@ -974,7 +977,7 @@ static int dec10_dip(DisasContext *dc)
     LOG_DIS("dip pc=%x opcode=%d r%d r%d\n",
               dc->pc, dc->opcode, dc->src, dc->dst);
     if (dc->src == 15) {
-        imm = ldl_code(dc->pc + 2);
+        imm = cpu_ldl_code(env, dc->pc + 2);
         tcg_gen_movi_tl(cpu_PR[PR_PREFIX], imm);
         if (dc->postinc)
             insn_len += 4;
@@ -989,7 +992,7 @@ static int dec10_dip(DisasContext *dc)
     return insn_len;
 }
 
-static int dec10_bdap_m(DisasContext *dc, int size)
+static int dec10_bdap_m(CPUCRISState *env, DisasContext *dc, int size)
 {
     int insn_len = 2;
     int rd = dc->dst;
@@ -1014,13 +1017,13 @@ static int dec10_bdap_m(DisasContext *dc, int size)
     }
 #endif
     /* Now the rest of the modes are truly indirect.  */
-    insn_len += dec10_prep_move_m(dc, 1, size, cpu_PR[PR_PREFIX]);
+    insn_len += dec10_prep_move_m(env, dc, 1, size, cpu_PR[PR_PREFIX]);
     tcg_gen_add_tl(cpu_PR[PR_PREFIX], cpu_PR[PR_PREFIX], cpu_R[rd]);
     cris_set_prefix(dc);
     return insn_len;
 }
 
-static unsigned int dec10_ind(DisasContext *dc)
+static unsigned int dec10_ind(CPUCRISState *env, DisasContext *dc)
 {
     unsigned int insn_len = 2;
     unsigned int size = dec10_size(dc->size);
@@ -1031,7 +1034,7 @@ static unsigned int dec10_ind(DisasContext *dc)
     if (dc->size != 3) {
         switch (dc->opcode) {
             case CRISV10_IND_MOVE_M_R:
-                return dec10_ind_move_m_r(dc, size);
+                return dec10_ind_move_m_r(env, dc, size);
                 break;
             case CRISV10_IND_MOVE_R_M:
                 return dec10_ind_move_r_m(dc, size);
@@ -1039,7 +1042,7 @@ static unsigned int dec10_ind(DisasContext *dc)
             case CRISV10_IND_CMP:
                 LOG_DIS("cmp size=%d op=%d %d\n",  size, dc->src, dc->dst);
                 cris_cc_mask(dc, CC_MASK_NZVC);
-                insn_len += dec10_ind_alu(dc, CC_OP_CMP, size);
+                insn_len += dec10_ind_alu(env, dc, CC_OP_CMP, size);
                 break;
             case CRISV10_IND_TEST:
                 LOG_DIS("test size=%d op=%d %d\n",  size, dc->src, dc->dst);
@@ -1047,7 +1050,7 @@ static unsigned int dec10_ind(DisasContext *dc)
                 cris_evaluate_flags(dc);
                 cris_cc_mask(dc, CC_MASK_NZVC);
                 cris_alu_m_alloc_temps(t);
-                insn_len += dec10_prep_move_m(dc, 0, size, t[0]);
+                insn_len += dec10_prep_move_m(env, dc, 0, size, t[0]);
                 tcg_gen_andi_tl(cpu_PR[PR_CCS], cpu_PR[PR_CCS], ~3);
                 cris_alu(dc, CC_OP_CMP, cpu_R[dc->dst],
                          t[0], tcg_const_tl(0), size);
@@ -1056,39 +1059,39 @@ static unsigned int dec10_ind(DisasContext *dc)
             case CRISV10_IND_ADD:
                 LOG_DIS("add size=%d op=%d %d\n",  size, dc->src, dc->dst);
                 cris_cc_mask(dc, CC_MASK_NZVC);
-                insn_len += dec10_ind_alu(dc, CC_OP_ADD, size);
+                insn_len += dec10_ind_alu(env, dc, CC_OP_ADD, size);
                 break;
             case CRISV10_IND_SUB:
                 LOG_DIS("sub size=%d op=%d %d\n",  size, dc->src, dc->dst);
                 cris_cc_mask(dc, CC_MASK_NZVC);
-                insn_len += dec10_ind_alu(dc, CC_OP_SUB, size);
+                insn_len += dec10_ind_alu(env, dc, CC_OP_SUB, size);
                 break;
             case CRISV10_IND_BOUND:
                 LOG_DIS("bound size=%d op=%d %d\n",  size, dc->src, dc->dst);
                 cris_cc_mask(dc, CC_MASK_NZVC);
-                insn_len += dec10_ind_bound(dc, size);
+                insn_len += dec10_ind_bound(env, dc, size);
                 break;
             case CRISV10_IND_AND:
                 LOG_DIS("and size=%d op=%d %d\n",  size, dc->src, dc->dst);
                 cris_cc_mask(dc, CC_MASK_NZVC);
-                insn_len += dec10_ind_alu(dc, CC_OP_AND, size);
+                insn_len += dec10_ind_alu(env, dc, CC_OP_AND, size);
                 break;
             case CRISV10_IND_OR:
                 LOG_DIS("or size=%d op=%d %d\n",  size, dc->src, dc->dst);
                 cris_cc_mask(dc, CC_MASK_NZVC);
-                insn_len += dec10_ind_alu(dc, CC_OP_OR, size);
+                insn_len += dec10_ind_alu(env, dc, CC_OP_OR, size);
                 break;
             case CRISV10_IND_MOVX:
-                insn_len = dec10_alux_m(dc, CC_OP_MOVE);
+                insn_len = dec10_alux_m(env, dc, CC_OP_MOVE);
                 break;
             case CRISV10_IND_ADDX:
-                insn_len = dec10_alux_m(dc, CC_OP_ADD);
+                insn_len = dec10_alux_m(env, dc, CC_OP_ADD);
                 break;
             case CRISV10_IND_SUBX:
-                insn_len = dec10_alux_m(dc, CC_OP_SUB);
+                insn_len = dec10_alux_m(env, dc, CC_OP_SUB);
                 break;
             case CRISV10_IND_CMPX:
-                insn_len = dec10_alux_m(dc, CC_OP_CMP);
+                insn_len = dec10_alux_m(env, dc, CC_OP_CMP);
                 break;
             case CRISV10_IND_MUL:
                 /* This is a reg insn coded in the mem indir space.  */
@@ -1097,7 +1100,7 @@ static unsigned int dec10_ind(DisasContext *dc)
                 dec10_reg_mul(dc, size, dc->ir & (1 << 10));
                 break;
             case CRISV10_IND_BDAP_M:
-                insn_len = dec10_bdap_m(dc, size);
+                insn_len = dec10_bdap_m(env, dc, size);
                 break;
             default:
                 LOG_DIS("pc=%x var-ind.%d %d r%d r%d\n",
@@ -1110,7 +1113,7 @@ static unsigned int dec10_ind(DisasContext *dc)
 
     switch (dc->opcode) {
         case CRISV10_IND_MOVE_M_SPR:
-            insn_len = dec10_ind_move_m_pr(dc);
+            insn_len = dec10_ind_move_m_pr(env, dc);
             break;
         case CRISV10_IND_MOVE_SPR_M:
             insn_len = dec10_ind_move_pr_m(dc);
@@ -1119,7 +1122,7 @@ static unsigned int dec10_ind(DisasContext *dc)
             if (dc->src == 15) {
                 LOG_DIS("jump.%d %d r%d r%d direct\n", size,
                          dc->opcode, dc->src, dc->dst);
-                imm = ldl_code(dc->pc + 2);
+                imm = cpu_ldl_code(env, dc->pc + 2);
                 if (dc->mode == CRISV10_MODE_AUTOINC)
                     insn_len += size;
 
@@ -1168,24 +1171,24 @@ static unsigned int dec10_ind(DisasContext *dc)
             dc->delayed_branch--; /* v10 has no dslot here.  */
             break;
         case CRISV10_IND_MOVX:
-            insn_len = dec10_alux_m(dc, CC_OP_MOVE);
+            insn_len = dec10_alux_m(env, dc, CC_OP_MOVE);
             break;
         case CRISV10_IND_ADDX:
-            insn_len = dec10_alux_m(dc, CC_OP_ADD);
+            insn_len = dec10_alux_m(env, dc, CC_OP_ADD);
             break;
         case CRISV10_IND_SUBX:
-            insn_len = dec10_alux_m(dc, CC_OP_SUB);
+            insn_len = dec10_alux_m(env, dc, CC_OP_SUB);
             break;
         case CRISV10_IND_CMPX:
-            insn_len = dec10_alux_m(dc, CC_OP_CMP);
+            insn_len = dec10_alux_m(env, dc, CC_OP_CMP);
             break;
         case CRISV10_IND_DIP:
-            insn_len = dec10_dip(dc);
+            insn_len = dec10_dip(env, dc);
             break;
         case CRISV10_IND_BCC_M:
 
             cris_cc_mask(dc, 0);
-            imm = ldsw_code(dc->pc + 2);
+            imm = cpu_ldsw_code(env, dc->pc + 2);
             simm = (int16_t)imm;
             simm += 4;
 
@@ -1202,7 +1205,7 @@ static unsigned int dec10_ind(DisasContext *dc)
     return insn_len;
 }
 
-static unsigned int crisv10_decoder(DisasContext *dc)
+static unsigned int crisv10_decoder(CPUCRISState *env, DisasContext *dc)
 {
     unsigned int insn_len = 2;
 
@@ -1210,7 +1213,7 @@ static unsigned int crisv10_decoder(DisasContext *dc)
         tcg_gen_debug_insn_start(dc->pc);
 
     /* Load a halfword onto the instruction register.  */
-    dc->ir = lduw_code(dc->pc);
+    dc->ir = cpu_lduw_code(env, dc->pc);
 
     /* Now decode it.  */
     dc->opcode   = EXTRACT_FIELD(dc->ir, 6, 9);
@@ -1235,7 +1238,7 @@ static unsigned int crisv10_decoder(DisasContext *dc)
             break;
         case CRISV10_MODE_AUTOINC:
         case CRISV10_MODE_INDIRECT:
-            insn_len = dec10_ind(dc);
+            insn_len = dec10_ind(env, dc);
             break;
     }
 
diff --git a/target-i386/arch_memory_mapping.c b/target-i386/arch_memory_mapping.c
index 8e5a56a3a..41f9d1c3e 100644
--- a/target-i386/arch_memory_mapping.c
+++ b/target-i386/arch_memory_mapping.c
@@ -16,10 +16,10 @@
 #include "memory_mapping.h"
 
 /* PAE Paging or IA-32e Paging */
-static void walk_pte(MemoryMappingList *list, target_phys_addr_t pte_start_addr,
+static void walk_pte(MemoryMappingList *list, hwaddr pte_start_addr,
                      int32_t a20_mask, target_ulong start_line_addr)
 {
-    target_phys_addr_t pte_addr, start_paddr;
+    hwaddr pte_addr, start_paddr;
     uint64_t pte;
     target_ulong start_vaddr;
     int i;
@@ -46,10 +46,10 @@ static void walk_pte(MemoryMappingList *list, target_phys_addr_t pte_start_addr,
 
 /* 32-bit Paging */
 static void walk_pte2(MemoryMappingList *list,
-                      target_phys_addr_t pte_start_addr, int32_t a20_mask,
+                      hwaddr pte_start_addr, int32_t a20_mask,
                       target_ulong start_line_addr)
 {
-    target_phys_addr_t pte_addr, start_paddr;
+    hwaddr pte_addr, start_paddr;
     uint32_t pte;
     target_ulong start_vaddr;
     int i;
@@ -75,10 +75,10 @@ static void walk_pte2(MemoryMappingList *list,
 }
 
 /* PAE Paging or IA-32e Paging */
-static void walk_pde(MemoryMappingList *list, target_phys_addr_t pde_start_addr,
+static void walk_pde(MemoryMappingList *list, hwaddr pde_start_addr,
                      int32_t a20_mask, target_ulong start_line_addr)
 {
-    target_phys_addr_t pde_addr, pte_start_addr, start_paddr;
+    hwaddr pde_addr, pte_start_addr, start_paddr;
     uint64_t pde;
     target_ulong line_addr, start_vaddr;
     int i;
@@ -112,10 +112,10 @@ static void walk_pde(MemoryMappingList *list, target_phys_addr_t pde_start_addr,
 
 /* 32-bit Paging */
 static void walk_pde2(MemoryMappingList *list,
-                      target_phys_addr_t pde_start_addr, int32_t a20_mask,
+                      hwaddr pde_start_addr, int32_t a20_mask,
                       bool pse)
 {
-    target_phys_addr_t pde_addr, pte_start_addr, start_paddr;
+    hwaddr pde_addr, pte_start_addr, start_paddr;
     uint32_t pde;
     target_ulong line_addr, start_vaddr;
     int i;
@@ -149,9 +149,9 @@ static void walk_pde2(MemoryMappingList *list,
 
 /* PAE Paging */
 static void walk_pdpe2(MemoryMappingList *list,
-                       target_phys_addr_t pdpe_start_addr, int32_t a20_mask)
+                       hwaddr pdpe_start_addr, int32_t a20_mask)
 {
-    target_phys_addr_t pdpe_addr, pde_start_addr;
+    hwaddr pdpe_addr, pde_start_addr;
     uint64_t pdpe;
     target_ulong line_addr;
     int i;
@@ -173,10 +173,10 @@ static void walk_pdpe2(MemoryMappingList *list,
 #ifdef TARGET_X86_64
 /* IA-32e Paging */
 static void walk_pdpe(MemoryMappingList *list,
-                      target_phys_addr_t pdpe_start_addr, int32_t a20_mask,
+                      hwaddr pdpe_start_addr, int32_t a20_mask,
                       target_ulong start_line_addr)
 {
-    target_phys_addr_t pdpe_addr, pde_start_addr, start_paddr;
+    hwaddr pdpe_addr, pde_start_addr, start_paddr;
     uint64_t pdpe;
     target_ulong line_addr, start_vaddr;
     int i;
@@ -210,9 +210,9 @@ static void walk_pdpe(MemoryMappingList *list,
 
 /* IA-32e Paging */
 static void walk_pml4e(MemoryMappingList *list,
-                       target_phys_addr_t pml4e_start_addr, int32_t a20_mask)
+                       hwaddr pml4e_start_addr, int32_t a20_mask)
 {
-    target_phys_addr_t pml4e_addr, pdpe_start_addr;
+    hwaddr pml4e_addr, pdpe_start_addr;
     uint64_t pml4e;
     target_ulong line_addr;
     int i;
@@ -242,20 +242,20 @@ int cpu_get_memory_mapping(MemoryMappingList *list, CPUArchState *env)
     if (env->cr[4] & CR4_PAE_MASK) {
 #ifdef TARGET_X86_64
         if (env->hflags & HF_LMA_MASK) {
-            target_phys_addr_t pml4e_addr;
+            hwaddr pml4e_addr;
 
             pml4e_addr = (env->cr[3] & ~0xfff) & env->a20_mask;
             walk_pml4e(list, pml4e_addr, env->a20_mask);
         } else
 #endif
         {
-            target_phys_addr_t pdpe_addr;
+            hwaddr pdpe_addr;
 
             pdpe_addr = (env->cr[3] & ~0x1f) & env->a20_mask;
             walk_pdpe2(list, pdpe_addr, env->a20_mask);
         }
     } else {
-        target_phys_addr_t pde_addr;
+        hwaddr pde_addr;
         bool pse;
 
         pde_addr = (env->cr[3] & ~0xfff) & env->a20_mask;
diff --git a/target-i386/cc_helper.c b/target-i386/cc_helper.c
index 07892f904..9422003f2 100644
--- a/target-i386/cc_helper.c
+++ b/target-i386/cc_helper.c
@@ -353,6 +353,16 @@ void helper_sti(CPUX86State *env)
     env->eflags |= IF_MASK;
 }
 
+void helper_clac(CPUX86State *env)
+{
+    env->eflags &= ~AC_MASK;
+}
+
+void helper_stac(CPUX86State *env)
+{
+    env->eflags |= AC_MASK;
+}
+
 #if 0
 /* vm86plus instructions */
 void helper_cli_vm(CPUX86State *env)
diff --git a/target-i386/cpu.c b/target-i386/cpu.c
index 423e00905..c6c2ca03a 100644
--- a/target-i386/cpu.c
+++ b/target-i386/cpu.c
@@ -37,6 +37,13 @@
 #include <linux/kvm_para.h>
 #endif
 
+#include "sysemu.h"
+#ifndef CONFIG_USER_ONLY
+#include "hw/xen.h"
+#include "hw/sysbus.h"
+#include "hw/apic_internal.h"
+#endif
+
 /* feature flags taken from "Intel Processor Identification and the CPUID
  * Instruction" and AMD's "CPUID Specification".  In cases of disagreement
  * between feature naming conventions, aliases may be added.
@@ -59,34 +66,43 @@ static const char *ext_feature_name[] = {
     NULL, "pcid", "dca", "sse4.1|sse4_1",
     "sse4.2|sse4_2", "x2apic", "movbe", "popcnt",
     "tsc-deadline", "aes", "xsave", "osxsave",
-    "avx", NULL, NULL, "hypervisor",
+    "avx", "f16c", "rdrand", "hypervisor",
 };
+/* Feature names that are already defined on feature_name[] but are set on
+ * CPUID[8000_0001].EDX on AMD CPUs don't have their names on
+ * ext2_feature_name[]. They are copied automatically to cpuid_ext2_features
+ * if and only if CPU vendor is AMD.
+ */
 static const char *ext2_feature_name[] = {
-    "fpu", "vme", "de", "pse",
-    "tsc", "msr", "pae", "mce",
-    "cx8" /* AMD CMPXCHG8B */, "apic", NULL, "syscall",
-    "mtrr", "pge", "mca", "cmov",
-    "pat", "pse36", NULL, NULL /* Linux mp */,
-    "nx|xd", NULL, "mmxext", "mmx",
-    "fxsr", "fxsr_opt|ffxsr", "pdpe1gb" /* AMD Page1GB */, "rdtscp",
+    NULL /* fpu */, NULL /* vme */, NULL /* de */, NULL /* pse */,
+    NULL /* tsc */, NULL /* msr */, NULL /* pae */, NULL /* mce */,
+    NULL /* cx8 */ /* AMD CMPXCHG8B */, NULL /* apic */, NULL, "syscall",
+    NULL /* mtrr */, NULL /* pge */, NULL /* mca */, NULL /* cmov */,
+    NULL /* pat */, NULL /* pse36 */, NULL, NULL /* Linux mp */,
+    "nx|xd", NULL, "mmxext", NULL /* mmx */,
+    NULL /* fxsr */, "fxsr_opt|ffxsr", "pdpe1gb" /* AMD Page1GB */, "rdtscp",
     NULL, "lm|i64", "3dnowext", "3dnow",
 };
 static const char *ext3_feature_name[] = {
     "lahf_lm" /* AMD LahfSahf */, "cmp_legacy", "svm", "extapic" /* AMD ExtApicSpace */,
     "cr8legacy" /* AMD AltMovCr8 */, "abm", "sse4a", "misalignsse",
     "3dnowprefetch", "osvw", "ibs", "xop",
-    "skinit", "wdt", NULL, NULL,
-    "fma4", NULL, "cvt16", "nodeid_msr",
-    NULL, NULL, NULL, NULL,
-    NULL, NULL, NULL, NULL,
+    "skinit", "wdt", NULL, "lwp",
+    "fma4", "tce", NULL, "nodeid_msr",
+    NULL, "tbm", "topoext", "perfctr_core",
+    "perfctr_nb", NULL, NULL, NULL,
     NULL, NULL, NULL, NULL,
 };
 
 static const char *kvm_feature_name[] = {
-    "kvmclock", "kvm_nopiodelay", "kvm_mmu", "kvmclock", "kvm_asyncpf", NULL, "kvm_pv_eoi", NULL,
-    NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
-    NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
-    NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
+    "kvmclock", "kvm_nopiodelay", "kvm_mmu", "kvmclock",
+    "kvm_asyncpf", "kvm_steal_time", "kvm_pv_eoi", NULL,
+    NULL, NULL, NULL, NULL,
+    NULL, NULL, NULL, NULL,
+    NULL, NULL, NULL, NULL,
+    NULL, NULL, NULL, NULL,
+    NULL, NULL, NULL, NULL,
+    NULL, NULL, NULL, NULL,
 };
 
 static const char *svm_feature_name[] = {
@@ -100,6 +116,13 @@ static const char *svm_feature_name[] = {
     NULL, NULL, NULL, NULL,
 };
 
+static const char *cpuid_7_0_ebx_feature_name[] = {
+    "fsgsbase", NULL, NULL, "bmi1", "hle", "avx2", NULL, "smep",
+    "bmi2", "erms", "invpcid", "rtm", NULL, NULL, NULL, NULL,
+    NULL, NULL, "rdseed", "adx", "smap", NULL, NULL, NULL,
+    NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
+};
+
 /* collects per-function cpuid data
  */
 typedef struct model_features_t {
@@ -113,6 +136,25 @@ typedef struct model_features_t {
 int check_cpuid = 0;
 int enforce_cpuid = 0;
 
+#if defined(CONFIG_KVM)
+static uint32_t kvm_default_features = (1 << KVM_FEATURE_CLOCKSOURCE) |
+        (1 << KVM_FEATURE_NOP_IO_DELAY) |
+        (1 << KVM_FEATURE_MMU_OP) |
+        (1 << KVM_FEATURE_CLOCKSOURCE2) |
+        (1 << KVM_FEATURE_ASYNC_PF) |
+        (1 << KVM_FEATURE_STEAL_TIME) |
+        (1 << KVM_FEATURE_CLOCKSOURCE_STABLE_BIT);
+static const uint32_t kvm_pv_eoi_features = (0x1 << KVM_FEATURE_PV_EOI);
+#else
+static uint32_t kvm_default_features = 0;
+static const uint32_t kvm_pv_eoi_features = 0;
+#endif
+
+void enable_kvm_pv_eoi(void)
+{
+    kvm_default_features |= kvm_pv_eoi_features;
+}
+
 void host_cpuid(uint32_t function, uint32_t count,
                 uint32_t *eax, uint32_t *ebx, uint32_t *ecx, uint32_t *edx)
 {
@@ -215,14 +257,17 @@ static void add_flagname_to_bitmaps(const char *flagname, uint32_t *features,
                                     uint32_t *ext2_features,
                                     uint32_t *ext3_features,
                                     uint32_t *kvm_features,
-                                    uint32_t *svm_features)
+                                    uint32_t *svm_features,
+                                    uint32_t *cpuid_7_0_ebx_features)
 {
     if (!lookup_feature(features, flagname, NULL, feature_name) &&
         !lookup_feature(ext_features, flagname, NULL, ext_feature_name) &&
         !lookup_feature(ext2_features, flagname, NULL, ext2_feature_name) &&
         !lookup_feature(ext3_features, flagname, NULL, ext3_feature_name) &&
         !lookup_feature(kvm_features, flagname, NULL, kvm_feature_name) &&
-        !lookup_feature(svm_features, flagname, NULL, svm_feature_name))
+        !lookup_feature(svm_features, flagname, NULL, svm_feature_name) &&
+        !lookup_feature(cpuid_7_0_ebx_features, flagname, NULL,
+                        cpuid_7_0_ebx_feature_name))
             fprintf(stderr, "CPU feature %s not found\n", flagname);
 }
 
@@ -240,7 +285,6 @@ typedef struct x86_def_t {
     uint32_t xlevel;
     char model_id[48];
     int vendor_override;
-    uint32_t flags;
     /* Store the results of Centaur's CPUID instructions */
     uint32_t ext4_features;
     uint32_t xlevel2;
@@ -259,7 +303,6 @@ typedef struct x86_def_t {
           CPUID_MSR | CPUID_MCE | CPUID_CX8 | CPUID_PGE | CPUID_CMOV | \
           CPUID_PAT | CPUID_FXSR | CPUID_MMX | CPUID_SSE | CPUID_SSE2 | \
           CPUID_PAE | CPUID_SEP | CPUID_APIC)
-#define EXT2_FEATURE_MASK 0x0183F3FF
 
 #define TCG_FEATURES (CPUID_FP87 | CPUID_PSE | CPUID_TSC | CPUID_MSR | \
           CPUID_PAE | CPUID_MCE | CPUID_CX8 | CPUID_APIC | CPUID_SEP | \
@@ -272,12 +315,12 @@ typedef struct x86_def_t {
           /* missing:
           CPUID_VME, CPUID_DTS, CPUID_SS, CPUID_HT, CPUID_TM, CPUID_PBE */
 #define TCG_EXT_FEATURES (CPUID_EXT_SSE3 | CPUID_EXT_MONITOR | \
-          CPUID_EXT_CX16 | CPUID_EXT_POPCNT | \
+          CPUID_EXT_SSSE3 | CPUID_EXT_CX16 | CPUID_EXT_POPCNT | \
           CPUID_EXT_HYPERVISOR)
           /* missing:
           CPUID_EXT_DTES64, CPUID_EXT_DSCPL, CPUID_EXT_VMX, CPUID_EXT_EST,
           CPUID_EXT_TM2, CPUID_EXT_XTPR, CPUID_EXT_PDCM, CPUID_EXT_XSAVE */
-#define TCG_EXT2_FEATURES ((TCG_FEATURES & EXT2_FEATURE_MASK) | \
+#define TCG_EXT2_FEATURES ((TCG_FEATURES & CPUID_EXT2_AMD_ALIASES) | \
           CPUID_EXT2_NX | CPUID_EXT2_MMXEXT | CPUID_EXT2_RDTSCP | \
           CPUID_EXT2_3DNOW | CPUID_EXT2_3DNOWEXT)
           /* missing:
@@ -285,6 +328,7 @@ typedef struct x86_def_t {
 #define TCG_EXT3_FEATURES (CPUID_EXT3_LAHF_LM | CPUID_EXT3_SVM | \
           CPUID_EXT3_CR8LEG | CPUID_EXT3_ABM | CPUID_EXT3_SSE4A)
 #define TCG_SVM_FEATURES 0
+#define TCG_7_0_EBX_FEATURES (CPUID_7_0_EBX_SMEP | CPUID_7_0_EBX_SMAP)
 
 /* maintains list of cpu model definitions
  */
@@ -306,7 +350,7 @@ static x86_def_t builtin_x86_defs[] = {
             CPUID_MTRR | CPUID_CLFLUSH | CPUID_MCA |
             CPUID_PSE36,
         .ext_features = CPUID_EXT_SSE3 | CPUID_EXT_CX16 | CPUID_EXT_POPCNT,
-        .ext2_features = (PPRO_FEATURES & EXT2_FEATURE_MASK) |
+        .ext2_features = (PPRO_FEATURES & CPUID_EXT2_AMD_ALIASES) |
             CPUID_EXT2_LM | CPUID_EXT2_SYSCALL | CPUID_EXT2_NX,
         .ext3_features = CPUID_EXT3_LAHF_LM | CPUID_EXT3_SVM |
             CPUID_EXT3_ABM | CPUID_EXT3_SSE4A,
@@ -326,7 +370,7 @@ static x86_def_t builtin_x86_defs[] = {
             CPUID_PSE36 | CPUID_VME | CPUID_HT,
         .ext_features = CPUID_EXT_SSE3 | CPUID_EXT_MONITOR | CPUID_EXT_CX16 |
             CPUID_EXT_POPCNT,
-        .ext2_features = (PPRO_FEATURES & EXT2_FEATURE_MASK) |
+        .ext2_features = (PPRO_FEATURES & CPUID_EXT2_AMD_ALIASES) |
             CPUID_EXT2_LM | CPUID_EXT2_SYSCALL | CPUID_EXT2_NX |
             CPUID_EXT2_3DNOW | CPUID_EXT2_3DNOWEXT | CPUID_EXT2_MMXEXT |
             CPUID_EXT2_FFXSR | CPUID_EXT2_PDPE1GB | CPUID_EXT2_RDTSCP,
@@ -374,7 +418,7 @@ static x86_def_t builtin_x86_defs[] = {
         /* Missing: CPUID_EXT_POPCNT, CPUID_EXT_MONITOR */
         .ext_features = CPUID_EXT_SSE3 | CPUID_EXT_CX16,
         /* Missing: CPUID_EXT2_PDPE1GB, CPUID_EXT2_RDTSCP */
-        .ext2_features = (PPRO_FEATURES & EXT2_FEATURE_MASK) |
+        .ext2_features = (PPRO_FEATURES & CPUID_EXT2_AMD_ALIASES) |
             CPUID_EXT2_LM | CPUID_EXT2_SYSCALL | CPUID_EXT2_NX,
         /* Missing: CPUID_EXT3_LAHF_LM, CPUID_EXT3_CMP_LEG, CPUID_EXT3_EXTAPIC,
                     CPUID_EXT3_CR8LEG, CPUID_EXT3_ABM, CPUID_EXT3_SSE4A,
@@ -403,7 +447,7 @@ static x86_def_t builtin_x86_defs[] = {
         .features = PPRO_FEATURES |
             CPUID_MTRR | CPUID_CLFLUSH | CPUID_MCA | CPUID_PSE36,
         .ext_features = CPUID_EXT_SSE3,
-        .ext2_features = PPRO_FEATURES & EXT2_FEATURE_MASK,
+        .ext2_features = PPRO_FEATURES & CPUID_EXT2_AMD_ALIASES,
         .ext3_features = 0,
         .xlevel = 0x80000008,
         .model_id = "Common 32-bit KVM processor"
@@ -468,8 +512,10 @@ static x86_def_t builtin_x86_defs[] = {
         .family = 6,
         .model = 2,
         .stepping = 3,
-        .features = PPRO_FEATURES | CPUID_PSE36 | CPUID_VME | CPUID_MTRR | CPUID_MCA,
-        .ext2_features = (PPRO_FEATURES & EXT2_FEATURE_MASK) | CPUID_EXT2_MMXEXT | CPUID_EXT2_3DNOW | CPUID_EXT2_3DNOWEXT,
+        .features = PPRO_FEATURES | CPUID_PSE36 | CPUID_VME | CPUID_MTRR |
+            CPUID_MCA,
+        .ext2_features = (PPRO_FEATURES & CPUID_EXT2_AMD_ALIASES) |
+            CPUID_EXT2_MMXEXT | CPUID_EXT2_3DNOW | CPUID_EXT2_3DNOWEXT,
         .xlevel = 0x80000008,
     },
     {
@@ -485,13 +531,296 @@ static x86_def_t builtin_x86_defs[] = {
             /* Some CPUs got no CPUID_SEP */
         .ext_features = CPUID_EXT_SSE3 | CPUID_EXT_MONITOR | CPUID_EXT_SSSE3 |
             CPUID_EXT_DSCPL | CPUID_EXT_EST | CPUID_EXT_TM2 | CPUID_EXT_XTPR,
-        .ext2_features = (PPRO_FEATURES & EXT2_FEATURE_MASK) | CPUID_EXT2_NX,
+        .ext2_features = (PPRO_FEATURES & CPUID_EXT2_AMD_ALIASES) |
+            CPUID_EXT2_NX,
         .ext3_features = CPUID_EXT3_LAHF_LM,
         .xlevel = 0x8000000A,
         .model_id = "Intel(R) Atom(TM) CPU N270   @ 1.60GHz",
     },
+    {
+        .name = "Conroe",
+        .level = 2,
+        .vendor1 = CPUID_VENDOR_INTEL_1,
+        .vendor2 = CPUID_VENDOR_INTEL_2,
+        .vendor3 = CPUID_VENDOR_INTEL_3,
+        .family = 6,
+        .model = 2,
+        .stepping = 3,
+        .features = CPUID_SSE2 | CPUID_SSE | CPUID_FXSR | CPUID_MMX |
+             CPUID_CLFLUSH | CPUID_PSE36 | CPUID_PAT | CPUID_CMOV | CPUID_MCA |
+             CPUID_PGE | CPUID_MTRR | CPUID_SEP | CPUID_APIC | CPUID_CX8 |
+             CPUID_MCE | CPUID_PAE | CPUID_MSR | CPUID_TSC | CPUID_PSE |
+             CPUID_DE | CPUID_FP87,
+        .ext_features = CPUID_EXT_SSSE3 | CPUID_EXT_SSE3,
+        .ext2_features = CPUID_EXT2_LM | CPUID_EXT2_NX | CPUID_EXT2_SYSCALL,
+        .ext3_features = CPUID_EXT3_LAHF_LM,
+        .xlevel = 0x8000000A,
+        .model_id = "Intel Celeron_4x0 (Conroe/Merom Class Core 2)",
+    },
+    {
+        .name = "Penryn",
+        .level = 2,
+        .vendor1 = CPUID_VENDOR_INTEL_1,
+        .vendor2 = CPUID_VENDOR_INTEL_2,
+        .vendor3 = CPUID_VENDOR_INTEL_3,
+        .family = 6,
+        .model = 2,
+        .stepping = 3,
+        .features = CPUID_SSE2 | CPUID_SSE | CPUID_FXSR | CPUID_MMX |
+             CPUID_CLFLUSH | CPUID_PSE36 | CPUID_PAT | CPUID_CMOV | CPUID_MCA |
+             CPUID_PGE | CPUID_MTRR | CPUID_SEP | CPUID_APIC | CPUID_CX8 |
+             CPUID_MCE | CPUID_PAE | CPUID_MSR | CPUID_TSC | CPUID_PSE |
+             CPUID_DE | CPUID_FP87,
+        .ext_features = CPUID_EXT_SSE41 | CPUID_EXT_CX16 | CPUID_EXT_SSSE3 |
+             CPUID_EXT_SSE3,
+        .ext2_features = CPUID_EXT2_LM | CPUID_EXT2_NX | CPUID_EXT2_SYSCALL,
+        .ext3_features = CPUID_EXT3_LAHF_LM,
+        .xlevel = 0x8000000A,
+        .model_id = "Intel Core 2 Duo P9xxx (Penryn Class Core 2)",
+    },
+    {
+        .name = "Nehalem",
+        .level = 2,
+        .vendor1 = CPUID_VENDOR_INTEL_1,
+        .vendor2 = CPUID_VENDOR_INTEL_2,
+        .vendor3 = CPUID_VENDOR_INTEL_3,
+        .family = 6,
+        .model = 2,
+        .stepping = 3,
+        .features = CPUID_SSE2 | CPUID_SSE | CPUID_FXSR | CPUID_MMX |
+             CPUID_CLFLUSH | CPUID_PSE36 | CPUID_PAT | CPUID_CMOV | CPUID_MCA |
+             CPUID_PGE | CPUID_MTRR | CPUID_SEP | CPUID_APIC | CPUID_CX8 |
+             CPUID_MCE | CPUID_PAE | CPUID_MSR | CPUID_TSC | CPUID_PSE |
+             CPUID_DE | CPUID_FP87,
+        .ext_features = CPUID_EXT_POPCNT | CPUID_EXT_SSE42 | CPUID_EXT_SSE41 |
+             CPUID_EXT_CX16 | CPUID_EXT_SSSE3 | CPUID_EXT_SSE3,
+        .ext2_features = CPUID_EXT2_LM | CPUID_EXT2_SYSCALL | CPUID_EXT2_NX,
+        .ext3_features = CPUID_EXT3_LAHF_LM,
+        .xlevel = 0x8000000A,
+        .model_id = "Intel Core i7 9xx (Nehalem Class Core i7)",
+    },
+    {
+        .name = "Westmere",
+        .level = 11,
+        .vendor1 = CPUID_VENDOR_INTEL_1,
+        .vendor2 = CPUID_VENDOR_INTEL_2,
+        .vendor3 = CPUID_VENDOR_INTEL_3,
+        .family = 6,
+        .model = 44,
+        .stepping = 1,
+        .features = CPUID_SSE2 | CPUID_SSE | CPUID_FXSR | CPUID_MMX |
+             CPUID_CLFLUSH | CPUID_PSE36 | CPUID_PAT | CPUID_CMOV | CPUID_MCA |
+             CPUID_PGE | CPUID_MTRR | CPUID_SEP | CPUID_APIC | CPUID_CX8 |
+             CPUID_MCE | CPUID_PAE | CPUID_MSR | CPUID_TSC | CPUID_PSE |
+             CPUID_DE | CPUID_FP87,
+        .ext_features = CPUID_EXT_AES | CPUID_EXT_POPCNT | CPUID_EXT_SSE42 |
+             CPUID_EXT_SSE41 | CPUID_EXT_CX16 | CPUID_EXT_SSSE3 |
+             CPUID_EXT_SSE3,
+        .ext2_features = CPUID_EXT2_LM | CPUID_EXT2_SYSCALL | CPUID_EXT2_NX,
+        .ext3_features = CPUID_EXT3_LAHF_LM,
+        .xlevel = 0x8000000A,
+        .model_id = "Westmere E56xx/L56xx/X56xx (Nehalem-C)",
+    },
+    {
+        .name = "SandyBridge",
+        .level = 0xd,
+        .vendor1 = CPUID_VENDOR_INTEL_1,
+        .vendor2 = CPUID_VENDOR_INTEL_2,
+        .vendor3 = CPUID_VENDOR_INTEL_3,
+        .family = 6,
+        .model = 42,
+        .stepping = 1,
+        .features = CPUID_SSE2 | CPUID_SSE | CPUID_FXSR | CPUID_MMX |
+             CPUID_CLFLUSH | CPUID_PSE36 | CPUID_PAT | CPUID_CMOV | CPUID_MCA |
+             CPUID_PGE | CPUID_MTRR | CPUID_SEP | CPUID_APIC | CPUID_CX8 |
+             CPUID_MCE | CPUID_PAE | CPUID_MSR | CPUID_TSC | CPUID_PSE |
+             CPUID_DE | CPUID_FP87,
+        .ext_features = CPUID_EXT_AVX | CPUID_EXT_XSAVE | CPUID_EXT_AES |
+             CPUID_EXT_TSC_DEADLINE_TIMER | CPUID_EXT_POPCNT |
+             CPUID_EXT_X2APIC | CPUID_EXT_SSE42 | CPUID_EXT_SSE41 |
+             CPUID_EXT_CX16 | CPUID_EXT_SSSE3 | CPUID_EXT_PCLMULQDQ |
+             CPUID_EXT_SSE3,
+        .ext2_features = CPUID_EXT2_LM | CPUID_EXT2_RDTSCP | CPUID_EXT2_NX |
+             CPUID_EXT2_SYSCALL,
+        .ext3_features = CPUID_EXT3_LAHF_LM,
+        .xlevel = 0x8000000A,
+        .model_id = "Intel Xeon E312xx (Sandy Bridge)",
+    },
+    {
+        .name = "Haswell",
+        .level = 0xd,
+        .vendor1 = CPUID_VENDOR_INTEL_1,
+        .vendor2 = CPUID_VENDOR_INTEL_2,
+        .vendor3 = CPUID_VENDOR_INTEL_3,
+        .family = 6,
+        .model = 60,
+        .stepping = 1,
+        .features = CPUID_SSE2 | CPUID_SSE | CPUID_FXSR | CPUID_MMX |
+             CPUID_CLFLUSH | CPUID_PSE36 | CPUID_PAT | CPUID_CMOV | CPUID_MCA |
+             CPUID_PGE | CPUID_MTRR | CPUID_SEP | CPUID_APIC | CPUID_CX8 |
+             CPUID_MCE | CPUID_PAE | CPUID_MSR | CPUID_TSC | CPUID_PSE |
+             CPUID_DE | CPUID_FP87,
+        .ext_features = CPUID_EXT_AVX | CPUID_EXT_XSAVE | CPUID_EXT_AES |
+             CPUID_EXT_POPCNT | CPUID_EXT_X2APIC | CPUID_EXT_SSE42 |
+             CPUID_EXT_SSE41 | CPUID_EXT_CX16 | CPUID_EXT_SSSE3 |
+             CPUID_EXT_PCLMULQDQ | CPUID_EXT_SSE3 |
+             CPUID_EXT_TSC_DEADLINE_TIMER | CPUID_EXT_FMA | CPUID_EXT_MOVBE |
+             CPUID_EXT_PCID,
+        .ext2_features = CPUID_EXT2_LM | CPUID_EXT2_RDTSCP | CPUID_EXT2_NX |
+             CPUID_EXT2_SYSCALL,
+        .ext3_features = CPUID_EXT3_LAHF_LM,
+        .cpuid_7_0_ebx_features = CPUID_7_0_EBX_FSGSBASE | CPUID_7_0_EBX_BMI1 |
+            CPUID_7_0_EBX_HLE | CPUID_7_0_EBX_AVX2 | CPUID_7_0_EBX_SMEP |
+            CPUID_7_0_EBX_BMI2 | CPUID_7_0_EBX_ERMS | CPUID_7_0_EBX_INVPCID |
+            CPUID_7_0_EBX_RTM,
+        .xlevel = 0x8000000A,
+        .model_id = "Intel Core Processor (Haswell)",
+    },
+    {
+        .name = "Opteron_G1",
+        .level = 5,
+        .vendor1 = CPUID_VENDOR_AMD_1,
+        .vendor2 = CPUID_VENDOR_AMD_2,
+        .vendor3 = CPUID_VENDOR_AMD_3,
+        .family = 15,
+        .model = 6,
+        .stepping = 1,
+        .features = CPUID_SSE2 | CPUID_SSE | CPUID_FXSR | CPUID_MMX |
+             CPUID_CLFLUSH | CPUID_PSE36 | CPUID_PAT | CPUID_CMOV | CPUID_MCA |
+             CPUID_PGE | CPUID_MTRR | CPUID_SEP | CPUID_APIC | CPUID_CX8 |
+             CPUID_MCE | CPUID_PAE | CPUID_MSR | CPUID_TSC | CPUID_PSE |
+             CPUID_DE | CPUID_FP87,
+        .ext_features = CPUID_EXT_SSE3,
+        .ext2_features = CPUID_EXT2_LM | CPUID_EXT2_FXSR | CPUID_EXT2_MMX |
+             CPUID_EXT2_NX | CPUID_EXT2_PSE36 | CPUID_EXT2_PAT |
+             CPUID_EXT2_CMOV | CPUID_EXT2_MCA | CPUID_EXT2_PGE |
+             CPUID_EXT2_MTRR | CPUID_EXT2_SYSCALL | CPUID_EXT2_APIC |
+             CPUID_EXT2_CX8 | CPUID_EXT2_MCE | CPUID_EXT2_PAE | CPUID_EXT2_MSR |
+             CPUID_EXT2_TSC | CPUID_EXT2_PSE | CPUID_EXT2_DE | CPUID_EXT2_FPU,
+        .xlevel = 0x80000008,
+        .model_id = "AMD Opteron 240 (Gen 1 Class Opteron)",
+    },
+    {
+        .name = "Opteron_G2",
+        .level = 5,
+        .vendor1 = CPUID_VENDOR_AMD_1,
+        .vendor2 = CPUID_VENDOR_AMD_2,
+        .vendor3 = CPUID_VENDOR_AMD_3,
+        .family = 15,
+        .model = 6,
+        .stepping = 1,
+        .features = CPUID_SSE2 | CPUID_SSE | CPUID_FXSR | CPUID_MMX |
+             CPUID_CLFLUSH | CPUID_PSE36 | CPUID_PAT | CPUID_CMOV | CPUID_MCA |
+             CPUID_PGE | CPUID_MTRR | CPUID_SEP | CPUID_APIC | CPUID_CX8 |
+             CPUID_MCE | CPUID_PAE | CPUID_MSR | CPUID_TSC | CPUID_PSE |
+             CPUID_DE | CPUID_FP87,
+        .ext_features = CPUID_EXT_CX16 | CPUID_EXT_SSE3,
+        .ext2_features = CPUID_EXT2_LM | CPUID_EXT2_RDTSCP | CPUID_EXT2_FXSR |
+             CPUID_EXT2_MMX | CPUID_EXT2_NX | CPUID_EXT2_PSE36 |
+             CPUID_EXT2_PAT | CPUID_EXT2_CMOV | CPUID_EXT2_MCA |
+             CPUID_EXT2_PGE | CPUID_EXT2_MTRR | CPUID_EXT2_SYSCALL |
+             CPUID_EXT2_APIC | CPUID_EXT2_CX8 | CPUID_EXT2_MCE |
+             CPUID_EXT2_PAE | CPUID_EXT2_MSR | CPUID_EXT2_TSC | CPUID_EXT2_PSE |
+             CPUID_EXT2_DE | CPUID_EXT2_FPU,
+        .ext3_features = CPUID_EXT3_SVM | CPUID_EXT3_LAHF_LM,
+        .xlevel = 0x80000008,
+        .model_id = "AMD Opteron 22xx (Gen 2 Class Opteron)",
+    },
+    {
+        .name = "Opteron_G3",
+        .level = 5,
+        .vendor1 = CPUID_VENDOR_AMD_1,
+        .vendor2 = CPUID_VENDOR_AMD_2,
+        .vendor3 = CPUID_VENDOR_AMD_3,
+        .family = 15,
+        .model = 6,
+        .stepping = 1,
+        .features = CPUID_SSE2 | CPUID_SSE | CPUID_FXSR | CPUID_MMX |
+             CPUID_CLFLUSH | CPUID_PSE36 | CPUID_PAT | CPUID_CMOV | CPUID_MCA |
+             CPUID_PGE | CPUID_MTRR | CPUID_SEP | CPUID_APIC | CPUID_CX8 |
+             CPUID_MCE | CPUID_PAE | CPUID_MSR | CPUID_TSC | CPUID_PSE |
+             CPUID_DE | CPUID_FP87,
+        .ext_features = CPUID_EXT_POPCNT | CPUID_EXT_CX16 | CPUID_EXT_MONITOR |
+             CPUID_EXT_SSE3,
+        .ext2_features = CPUID_EXT2_LM | CPUID_EXT2_RDTSCP | CPUID_EXT2_FXSR |
+             CPUID_EXT2_MMX | CPUID_EXT2_NX | CPUID_EXT2_PSE36 |
+             CPUID_EXT2_PAT | CPUID_EXT2_CMOV | CPUID_EXT2_MCA |
+             CPUID_EXT2_PGE | CPUID_EXT2_MTRR | CPUID_EXT2_SYSCALL |
+             CPUID_EXT2_APIC | CPUID_EXT2_CX8 | CPUID_EXT2_MCE |
+             CPUID_EXT2_PAE | CPUID_EXT2_MSR | CPUID_EXT2_TSC | CPUID_EXT2_PSE |
+             CPUID_EXT2_DE | CPUID_EXT2_FPU,
+        .ext3_features = CPUID_EXT3_MISALIGNSSE | CPUID_EXT3_SSE4A |
+             CPUID_EXT3_ABM | CPUID_EXT3_SVM | CPUID_EXT3_LAHF_LM,
+        .xlevel = 0x80000008,
+        .model_id = "AMD Opteron 23xx (Gen 3 Class Opteron)",
+    },
+    {
+        .name = "Opteron_G4",
+        .level = 0xd,
+        .vendor1 = CPUID_VENDOR_AMD_1,
+        .vendor2 = CPUID_VENDOR_AMD_2,
+        .vendor3 = CPUID_VENDOR_AMD_3,
+        .family = 21,
+        .model = 1,
+        .stepping = 2,
+        .features = CPUID_SSE2 | CPUID_SSE | CPUID_FXSR | CPUID_MMX |
+             CPUID_CLFLUSH | CPUID_PSE36 | CPUID_PAT | CPUID_CMOV | CPUID_MCA |
+             CPUID_PGE | CPUID_MTRR | CPUID_SEP | CPUID_APIC | CPUID_CX8 |
+             CPUID_MCE | CPUID_PAE | CPUID_MSR | CPUID_TSC | CPUID_PSE |
+             CPUID_DE | CPUID_FP87,
+        .ext_features = CPUID_EXT_AVX | CPUID_EXT_XSAVE | CPUID_EXT_AES |
+             CPUID_EXT_POPCNT | CPUID_EXT_SSE42 | CPUID_EXT_SSE41 |
+             CPUID_EXT_CX16 | CPUID_EXT_SSSE3 | CPUID_EXT_PCLMULQDQ |
+             CPUID_EXT_SSE3,
+        .ext2_features = CPUID_EXT2_LM | CPUID_EXT2_RDTSCP |
+             CPUID_EXT2_PDPE1GB | CPUID_EXT2_FXSR | CPUID_EXT2_MMX |
+             CPUID_EXT2_NX | CPUID_EXT2_PSE36 | CPUID_EXT2_PAT |
+             CPUID_EXT2_CMOV | CPUID_EXT2_MCA | CPUID_EXT2_PGE |
+             CPUID_EXT2_MTRR | CPUID_EXT2_SYSCALL | CPUID_EXT2_APIC |
+             CPUID_EXT2_CX8 | CPUID_EXT2_MCE | CPUID_EXT2_PAE | CPUID_EXT2_MSR |
+             CPUID_EXT2_TSC | CPUID_EXT2_PSE | CPUID_EXT2_DE | CPUID_EXT2_FPU,
+        .ext3_features = CPUID_EXT3_FMA4 | CPUID_EXT3_XOP |
+             CPUID_EXT3_3DNOWPREFETCH | CPUID_EXT3_MISALIGNSSE |
+             CPUID_EXT3_SSE4A | CPUID_EXT3_ABM | CPUID_EXT3_SVM |
+             CPUID_EXT3_LAHF_LM,
+        .xlevel = 0x8000001A,
+        .model_id = "AMD Opteron 62xx class CPU",
+    },
+    {
+        .name = "Opteron_G5",
+        .level = 0xd,
+        .vendor1 = CPUID_VENDOR_AMD_1,
+        .vendor2 = CPUID_VENDOR_AMD_2,
+        .vendor3 = CPUID_VENDOR_AMD_3,
+        .family = 21,
+        .model = 2,
+        .stepping = 0,
+        .features = CPUID_SSE2 | CPUID_SSE | CPUID_FXSR | CPUID_MMX |
+             CPUID_CLFLUSH | CPUID_PSE36 | CPUID_PAT | CPUID_CMOV | CPUID_MCA |
+             CPUID_PGE | CPUID_MTRR | CPUID_SEP | CPUID_APIC | CPUID_CX8 |
+             CPUID_MCE | CPUID_PAE | CPUID_MSR | CPUID_TSC | CPUID_PSE |
+             CPUID_DE | CPUID_FP87,
+        .ext_features = CPUID_EXT_F16C | CPUID_EXT_AVX | CPUID_EXT_XSAVE |
+             CPUID_EXT_AES | CPUID_EXT_POPCNT | CPUID_EXT_SSE42 |
+             CPUID_EXT_SSE41 | CPUID_EXT_CX16 | CPUID_EXT_FMA |
+             CPUID_EXT_SSSE3 | CPUID_EXT_PCLMULQDQ | CPUID_EXT_SSE3,
+        .ext2_features = CPUID_EXT2_LM | CPUID_EXT2_RDTSCP |
+             CPUID_EXT2_PDPE1GB | CPUID_EXT2_FXSR | CPUID_EXT2_MMX |
+             CPUID_EXT2_NX | CPUID_EXT2_PSE36 | CPUID_EXT2_PAT |
+             CPUID_EXT2_CMOV | CPUID_EXT2_MCA | CPUID_EXT2_PGE |
+             CPUID_EXT2_MTRR | CPUID_EXT2_SYSCALL | CPUID_EXT2_APIC |
+             CPUID_EXT2_CX8 | CPUID_EXT2_MCE | CPUID_EXT2_PAE | CPUID_EXT2_MSR |
+             CPUID_EXT2_TSC | CPUID_EXT2_PSE | CPUID_EXT2_DE | CPUID_EXT2_FPU,
+        .ext3_features = CPUID_EXT3_TBM | CPUID_EXT3_FMA4 | CPUID_EXT3_XOP |
+             CPUID_EXT3_3DNOWPREFETCH | CPUID_EXT3_MISALIGNSSE |
+             CPUID_EXT3_SSE4A | CPUID_EXT3_ABM | CPUID_EXT3_SVM |
+             CPUID_EXT3_LAHF_LM,
+        .xlevel = 0x8000001A,
+        .model_id = "AMD Opteron 63xx class CPU",
+    },
 };
 
+#ifdef CONFIG_KVM
 static int cpu_x86_fill_model_id(char *str)
 {
     uint32_t eax = 0, ebx = 0, ecx = 0, edx = 0;
@@ -506,14 +835,23 @@ static int cpu_x86_fill_model_id(char *str)
     }
     return 0;
 }
+#endif
 
-static int cpu_x86_fill_host(x86_def_t *x86_cpu_def)
+/* Fill a x86_def_t struct with information about the host CPU, and
+ * the CPU features supported by the host hardware + host kernel
+ *
+ * This function may be called only if KVM is enabled.
+ */
+static void kvm_cpu_fill_host(x86_def_t *x86_cpu_def)
 {
+#ifdef CONFIG_KVM
+    KVMState *s = kvm_state;
     uint32_t eax = 0, ebx = 0, ecx = 0, edx = 0;
 
+    assert(kvm_enabled());
+
     x86_cpu_def->name = "host";
     host_cpuid(0x0, 0, &eax, &ebx, &ecx, &edx);
-    x86_cpu_def->level = eax;
     x86_cpu_def->vendor1 = ebx;
     x86_cpu_def->vendor2 = edx;
     x86_cpu_def->vendor3 = ecx;
@@ -522,21 +860,24 @@ static int cpu_x86_fill_host(x86_def_t *x86_cpu_def)
     x86_cpu_def->family = ((eax >> 8) & 0x0F) + ((eax >> 20) & 0xFF);
     x86_cpu_def->model = ((eax >> 4) & 0x0F) | ((eax & 0xF0000) >> 12);
     x86_cpu_def->stepping = eax & 0x0F;
-    x86_cpu_def->ext_features = ecx;
-    x86_cpu_def->features = edx;
 
-    if (kvm_enabled() && x86_cpu_def->level >= 7) {
-        x86_cpu_def->cpuid_7_0_ebx_features = kvm_arch_get_supported_cpuid(kvm_state, 0x7, 0, R_EBX);
+    x86_cpu_def->level = kvm_arch_get_supported_cpuid(s, 0x0, 0, R_EAX);
+    x86_cpu_def->features = kvm_arch_get_supported_cpuid(s, 0x1, 0, R_EDX);
+    x86_cpu_def->ext_features = kvm_arch_get_supported_cpuid(s, 0x1, 0, R_ECX);
+
+    if (x86_cpu_def->level >= 7) {
+        x86_cpu_def->cpuid_7_0_ebx_features =
+                    kvm_arch_get_supported_cpuid(s, 0x7, 0, R_EBX);
     } else {
         x86_cpu_def->cpuid_7_0_ebx_features = 0;
     }
 
-    host_cpuid(0x80000000, 0, &eax, &ebx, &ecx, &edx);
-    x86_cpu_def->xlevel = eax;
+    x86_cpu_def->xlevel = kvm_arch_get_supported_cpuid(s, 0x80000000, 0, R_EAX);
+    x86_cpu_def->ext2_features =
+                kvm_arch_get_supported_cpuid(s, 0x80000001, 0, R_EDX);
+    x86_cpu_def->ext3_features =
+                kvm_arch_get_supported_cpuid(s, 0x80000001, 0, R_ECX);
 
-    host_cpuid(0x80000001, 0, &eax, &ebx, &ecx, &edx);
-    x86_cpu_def->ext2_features = edx;
-    x86_cpu_def->ext3_features = ecx;
     cpu_x86_fill_model_id(x86_cpu_def->model_id);
     x86_cpu_def->vendor_override = 0;
 
@@ -545,11 +886,13 @@ static int cpu_x86_fill_host(x86_def_t *x86_cpu_def)
         x86_cpu_def->vendor2 == CPUID_VENDOR_VIA_2 &&
         x86_cpu_def->vendor3 == CPUID_VENDOR_VIA_3) {
         host_cpuid(0xC0000000, 0, &eax, &ebx, &ecx, &edx);
+        eax = kvm_arch_get_supported_cpuid(s, 0xC0000000, 0, R_EAX);
         if (eax >= 0xC0000001) {
             /* Support VIA max extended level */
             x86_cpu_def->xlevel2 = eax;
             host_cpuid(0xC0000001, 0, &eax, &ebx, &ecx, &edx);
-            x86_cpu_def->ext4_features = edx;
+            x86_cpu_def->ext4_features =
+                    kvm_arch_get_supported_cpuid(s, 0xC0000001, 0, R_EDX);
         }
     }
 
@@ -560,8 +903,7 @@ static int cpu_x86_fill_host(x86_def_t *x86_cpu_def)
      * unsupported ones later.
      */
     x86_cpu_def->svm_features = -1;
-
-    return 0;
+#endif /* CONFIG_KVM */
 }
 
 static int unavailable_host_feature(struct model_features_t *f, uint32_t mask)
@@ -582,8 +924,10 @@ static int unavailable_host_feature(struct model_features_t *f, uint32_t mask)
 /* best effort attempt to inform user requested cpu flags aren't making
  * their way to the guest.  Note: ft[].check_feat ideally should be
  * specified via a guest_def field to suppress report of extraneous flags.
+ *
+ * This function may be called only if KVM is enabled.
  */
-static int check_features_against_host(x86_def_t *guest_def)
+static int kvm_check_features_against_host(x86_def_t *guest_def)
 {
     x86_def_t host_def;
     uint32_t mask;
@@ -598,7 +942,9 @@ static int check_features_against_host(x86_def_t *guest_def)
         {&guest_def->ext3_features, &host_def.ext3_features,
             ~CPUID_EXT3_SVM, ext3_feature_name, 0x80000001}};
 
-    cpu_x86_fill_host(&host_def);
+    assert(kvm_enabled());
+
+    kvm_cpu_fill_host(&host_def);
     for (rv = 0, i = 0; i < ARRAY_SIZE(ft); ++i)
         for (mask = 1; mask; mask <<= 1)
             if (ft[i].check_feat & mask && *ft[i].guest_feat & mask &&
@@ -846,7 +1192,7 @@ static void x86_cpuid_set_tsc_freq(Object *obj, Visitor *v, void *opaque,
 {
     X86CPU *cpu = X86_CPU(obj);
     const int64_t min = 0;
-    const int64_t max = INT_MAX;
+    const int64_t max = INT64_MAX;
     int64_t value;
 
     visit_type_int(v, &value, name, errp);
@@ -872,39 +1218,29 @@ static int cpu_x86_find_by_name(x86_def_t *x86_cpu_def, const char *cpu_model)
     /* Features to be added*/
     uint32_t plus_features = 0, plus_ext_features = 0;
     uint32_t plus_ext2_features = 0, plus_ext3_features = 0;
-    uint32_t plus_kvm_features = 0, plus_svm_features = 0;
+    uint32_t plus_kvm_features = kvm_default_features, plus_svm_features = 0;
+    uint32_t plus_7_0_ebx_features = 0;
     /* Features to be removed */
     uint32_t minus_features = 0, minus_ext_features = 0;
     uint32_t minus_ext2_features = 0, minus_ext3_features = 0;
     uint32_t minus_kvm_features = 0, minus_svm_features = 0;
+    uint32_t minus_7_0_ebx_features = 0;
     uint32_t numvalue;
 
     for (def = x86_defs; def; def = def->next)
         if (name && !strcmp(name, def->name))
             break;
     if (kvm_enabled() && name && strcmp(name, "host") == 0) {
-        cpu_x86_fill_host(x86_cpu_def);
+        kvm_cpu_fill_host(x86_cpu_def);
     } else if (!def) {
         goto error;
     } else {
         memcpy(x86_cpu_def, def, sizeof(*def));
     }
 
-#if defined(CONFIG_KVM)
-    plus_kvm_features = (1 << KVM_FEATURE_CLOCKSOURCE) |
-        (1 << KVM_FEATURE_NOP_IO_DELAY) | 
-        (1 << KVM_FEATURE_MMU_OP) |
-        (1 << KVM_FEATURE_CLOCKSOURCE2) |
-        (1 << KVM_FEATURE_ASYNC_PF) | 
-        (1 << KVM_FEATURE_STEAL_TIME) |
-        (1 << KVM_FEATURE_CLOCKSOURCE_STABLE_BIT);
-#else
-    plus_kvm_features = 0;
-#endif
-
     add_flagname_to_bitmaps("hypervisor", &plus_features,
-        &plus_ext_features, &plus_ext2_features, &plus_ext3_features,
-        &plus_kvm_features, &plus_svm_features);
+            &plus_ext_features, &plus_ext2_features, &plus_ext3_features,
+            &plus_kvm_features, &plus_svm_features,  &plus_7_0_ebx_features);
 
     featurestr = strtok(NULL, ",");
 
@@ -914,12 +1250,12 @@ static int cpu_x86_find_by_name(x86_def_t *x86_cpu_def, const char *cpu_model)
             add_flagname_to_bitmaps(featurestr + 1, &plus_features,
                             &plus_ext_features, &plus_ext2_features,
                             &plus_ext3_features, &plus_kvm_features,
-                            &plus_svm_features);
+                            &plus_svm_features, &plus_7_0_ebx_features);
         } else if (featurestr[0] == '-') {
             add_flagname_to_bitmaps(featurestr + 1, &minus_features,
                             &minus_ext_features, &minus_ext2_features,
                             &minus_ext3_features, &minus_kvm_features,
-                            &minus_svm_features);
+                            &minus_svm_features, &minus_7_0_ebx_features);
         } else if ((val = strchr(featurestr, '='))) {
             *val = 0; val++;
             if (!strcmp(featurestr, "family")) {
@@ -1025,16 +1361,21 @@ static int cpu_x86_find_by_name(x86_def_t *x86_cpu_def, const char *cpu_model)
     x86_cpu_def->ext3_features |= plus_ext3_features;
     x86_cpu_def->kvm_features |= plus_kvm_features;
     x86_cpu_def->svm_features |= plus_svm_features;
+    x86_cpu_def->cpuid_7_0_ebx_features |= plus_7_0_ebx_features;
     x86_cpu_def->features &= ~minus_features;
     x86_cpu_def->ext_features &= ~minus_ext_features;
     x86_cpu_def->ext2_features &= ~minus_ext2_features;
     x86_cpu_def->ext3_features &= ~minus_ext3_features;
     x86_cpu_def->kvm_features &= ~minus_kvm_features;
     x86_cpu_def->svm_features &= ~minus_svm_features;
-    if (check_cpuid) {
-        if (check_features_against_host(x86_cpu_def) && enforce_cpuid)
+    x86_cpu_def->cpuid_7_0_ebx_features &= ~minus_7_0_ebx_features;
+    if (check_cpuid && kvm_enabled()) {
+        if (kvm_check_features_against_host(x86_cpu_def) && enforce_cpuid)
             goto error;
     }
+    if (x86_cpu_def->cpuid_7_0_ebx_features && x86_cpu_def->level < 7) {
+        x86_cpu_def->level = 7;
+    }
     g_free(s);
     return 0;
 
@@ -1073,70 +1414,28 @@ static void listflags(char *buf, int bufsize, uint32_t fbits,
         }
 }
 
-/* generate CPU information:
- * -?        list model names
- * -?model   list model names/IDs
- * -?dump    output all model (x86_def_t) data
- * -?cpuid   list all recognized cpuid flag names
- */
-void x86_cpu_list(FILE *f, fprintf_function cpu_fprintf, const char *optarg)
+/* generate CPU information. */
+void x86_cpu_list(FILE *f, fprintf_function cpu_fprintf)
 {
-    unsigned char model = !strcmp("?model", optarg);
-    unsigned char dump = !strcmp("?dump", optarg);
-    unsigned char cpuid = !strcmp("?cpuid", optarg);
     x86_def_t *def;
     char buf[256];
 
-    if (cpuid) {
-        (*cpu_fprintf)(f, "Recognized CPUID flags:\n");
-        listflags(buf, sizeof (buf), (uint32_t)~0, feature_name, 1);
-        (*cpu_fprintf)(f, "  f_edx: %s\n", buf);
-        listflags(buf, sizeof (buf), (uint32_t)~0, ext_feature_name, 1);
-        (*cpu_fprintf)(f, "  f_ecx: %s\n", buf);
-        listflags(buf, sizeof (buf), (uint32_t)~0, ext2_feature_name, 1);
-        (*cpu_fprintf)(f, "  extf_edx: %s\n", buf);
-        listflags(buf, sizeof (buf), (uint32_t)~0, ext3_feature_name, 1);
-        (*cpu_fprintf)(f, "  extf_ecx: %s\n", buf);
-        return;
-    }
     for (def = x86_defs; def; def = def->next) {
-        snprintf(buf, sizeof (buf), def->flags ? "[%s]": "%s", def->name);
-        if (model || dump) {
-            (*cpu_fprintf)(f, "x86 %16s  %-48s\n", buf, def->model_id);
-        } else {
-            (*cpu_fprintf)(f, "x86 %16s\n", buf);
-        }
-        if (dump) {
-            memcpy(buf, &def->vendor1, sizeof (def->vendor1));
-            memcpy(buf + 4, &def->vendor2, sizeof (def->vendor2));
-            memcpy(buf + 8, &def->vendor3, sizeof (def->vendor3));
-            buf[12] = '\0';
-            (*cpu_fprintf)(f,
-                "  family %d model %d stepping %d level %d xlevel 0x%x"
-                " vendor \"%s\"\n",
-                def->family, def->model, def->stepping, def->level,
-                def->xlevel, buf);
-            listflags(buf, sizeof (buf), def->features, feature_name, 0);
-            (*cpu_fprintf)(f, "  feature_edx %08x (%s)\n", def->features,
-                buf);
-            listflags(buf, sizeof (buf), def->ext_features, ext_feature_name,
-                0);
-            (*cpu_fprintf)(f, "  feature_ecx %08x (%s)\n", def->ext_features,
-                buf);
-            listflags(buf, sizeof (buf), def->ext2_features, ext2_feature_name,
-                0);
-            (*cpu_fprintf)(f, "  extfeature_edx %08x (%s)\n",
-                def->ext2_features, buf);
-            listflags(buf, sizeof (buf), def->ext3_features, ext3_feature_name,
-                0);
-            (*cpu_fprintf)(f, "  extfeature_ecx %08x (%s)\n",
-                def->ext3_features, buf);
-            (*cpu_fprintf)(f, "\n");
-        }
+        snprintf(buf, sizeof(buf), "%s", def->name);
+        (*cpu_fprintf)(f, "x86 %16s  %-48s\n", buf, def->model_id);
     }
     if (kvm_enabled()) {
         (*cpu_fprintf)(f, "x86 %16s\n", "[host]");
     }
+    (*cpu_fprintf)(f, "\nRecognized CPUID flags:\n");
+    listflags(buf, sizeof(buf), (uint32_t)~0, feature_name, 1);
+    (*cpu_fprintf)(f, "  %s\n", buf);
+    listflags(buf, sizeof(buf), (uint32_t)~0, ext_feature_name, 1);
+    (*cpu_fprintf)(f, "  %s\n", buf);
+    listflags(buf, sizeof(buf), (uint32_t)~0, ext2_feature_name, 1);
+    (*cpu_fprintf)(f, "  %s\n", buf);
+    listflags(buf, sizeof(buf), (uint32_t)~0, ext3_feature_name, 1);
+    (*cpu_fprintf)(f, "  %s\n", buf);
 }
 
 CpuDefinitionInfoList *arch_query_cpu_definitions(Error **errp)
@@ -1160,6 +1459,32 @@ CpuDefinitionInfoList *arch_query_cpu_definitions(Error **errp)
     return cpu_list;
 }
 
+#ifdef CONFIG_KVM
+static void filter_features_for_kvm(X86CPU *cpu)
+{
+    CPUX86State *env = &cpu->env;
+    KVMState *s = kvm_state;
+
+    env->cpuid_features &=
+        kvm_arch_get_supported_cpuid(s, 1, 0, R_EDX);
+    env->cpuid_ext_features &=
+        kvm_arch_get_supported_cpuid(s, 1, 0, R_ECX);
+    env->cpuid_ext2_features &=
+        kvm_arch_get_supported_cpuid(s, 0x80000001, 0, R_EDX);
+    env->cpuid_ext3_features &=
+        kvm_arch_get_supported_cpuid(s, 0x80000001, 0, R_ECX);
+    env->cpuid_svm_features  &=
+        kvm_arch_get_supported_cpuid(s, 0x8000000A, 0, R_EDX);
+    env->cpuid_7_0_ebx_features &=
+        kvm_arch_get_supported_cpuid(s, 7, 0, R_EBX);
+    env->cpuid_kvm_features &=
+        kvm_arch_get_supported_cpuid(s, KVM_CPUID_FEATURES, 0, R_EAX);
+    env->cpuid_ext4_features &=
+        kvm_arch_get_supported_cpuid(s, 0xC0000001, 0, R_EDX);
+
+}
+#endif
+
 int cpu_x86_register(X86CPU *cpu, const char *cpu_model)
 {
     CPUX86State *env = &cpu->env;
@@ -1192,10 +1517,21 @@ int cpu_x86_register(X86CPU *cpu, const char *cpu_model)
     env->cpuid_kvm_features = def->kvm_features;
     env->cpuid_svm_features = def->svm_features;
     env->cpuid_ext4_features = def->ext4_features;
-    env->cpuid_7_0_ebx = def->cpuid_7_0_ebx_features;
+    env->cpuid_7_0_ebx_features = def->cpuid_7_0_ebx_features;
     env->cpuid_xlevel2 = def->xlevel2;
     object_property_set_int(OBJECT(cpu), (int64_t)def->tsc_khz * 1000,
                             "tsc-frequency", &error);
+
+    /* On AMD CPUs, some CPUID[8000_0001].EDX bits must match the bits on
+     * CPUID[1].EDX.
+     */
+    if (env->cpuid_vendor1 == CPUID_VENDOR_AMD_1 &&
+            env->cpuid_vendor2 == CPUID_VENDOR_AMD_2 &&
+            env->cpuid_vendor3 == CPUID_VENDOR_AMD_3) {
+        env->cpuid_ext2_features &= ~CPUID_EXT2_AMD_ALIASES;
+        env->cpuid_ext2_features |= (def->features & CPUID_EXT2_AMD_ALIASES);
+    }
+
     if (!kvm_enabled()) {
         env->cpuid_features &= TCG_FEATURES;
         env->cpuid_ext_features &= TCG_EXT_FEATURES;
@@ -1206,9 +1542,14 @@ int cpu_x86_register(X86CPU *cpu, const char *cpu_model)
             );
         env->cpuid_ext3_features &= TCG_EXT3_FEATURES;
         env->cpuid_svm_features &= TCG_SVM_FEATURES;
+    } else {
+#ifdef CONFIG_KVM
+        filter_features_for_kvm(cpu);
+#endif
     }
     object_property_set_str(OBJECT(cpu), def->model_id, "model-id", &error);
-    if (error_is_set(&error)) {
+    if (error) {
+        fprintf(stderr, "%s\n", error_get_pretty(error));
         error_free(error);
         return -1;
     }
@@ -1216,109 +1557,6 @@ int cpu_x86_register(X86CPU *cpu, const char *cpu_model)
 }
 
 #if !defined(CONFIG_USER_ONLY)
-/* copy vendor id string to 32 bit register, nul pad as needed
- */
-static void cpyid(const char *s, uint32_t *id)
-{
-    char *d = (char *)id;
-    char i;
-
-    for (i = sizeof (*id); i--; )
-        *d++ = *s ? *s++ : '\0';
-}
-
-/* interpret radix and convert from string to arbitrary scalar,
- * otherwise flag failure
- */
-#define setscalar(pval, str, perr)                      \
-{                                                       \
-    char *pend;                                         \
-    unsigned long ul;                                   \
-                                                        \
-    ul = strtoul(str, &pend, 0);                        \
-    *str && !*pend ? (*pval = ul) : (*perr = 1);        \
-}
-
-/* map cpuid options to feature bits, otherwise return failure
- * (option tags in *str are delimited by whitespace)
- */
-static void setfeatures(uint32_t *pval, const char *str,
-    const char **featureset, int *perr)
-{
-    const char *p, *q;
-
-    for (q = p = str; *p || *q; q = p) {
-        while (iswhite(*p))
-            q = ++p;
-        while (*p && !iswhite(*p))
-            ++p;
-        if (!*q && !*p)
-            return;
-        if (!lookup_feature(pval, q, p, featureset)) {
-            fprintf(stderr, "error: feature \"%.*s\" not available in set\n",
-                (int)(p - q), q);
-            *perr = 1;
-            return;
-        }
-    }
-}
-
-/* map config file options to x86_def_t form
- */
-static int cpudef_setfield(const char *name, const char *str, void *opaque)
-{
-    x86_def_t *def = opaque;
-    int err = 0;
-
-    if (!strcmp(name, "name")) {
-        g_free((void *)def->name);
-        def->name = g_strdup(str);
-    } else if (!strcmp(name, "model_id")) {
-        strncpy(def->model_id, str, sizeof (def->model_id));
-    } else if (!strcmp(name, "level")) {
-        setscalar(&def->level, str, &err)
-    } else if (!strcmp(name, "vendor")) {
-        cpyid(&str[0], &def->vendor1);
-        cpyid(&str[4], &def->vendor2);
-        cpyid(&str[8], &def->vendor3);
-    } else if (!strcmp(name, "family")) {
-        setscalar(&def->family, str, &err)
-    } else if (!strcmp(name, "model")) {
-        setscalar(&def->model, str, &err)
-    } else if (!strcmp(name, "stepping")) {
-        setscalar(&def->stepping, str, &err)
-    } else if (!strcmp(name, "feature_edx")) {
-        setfeatures(&def->features, str, feature_name, &err);
-    } else if (!strcmp(name, "feature_ecx")) {
-        setfeatures(&def->ext_features, str, ext_feature_name, &err);
-    } else if (!strcmp(name, "extfeature_edx")) {
-        setfeatures(&def->ext2_features, str, ext2_feature_name, &err);
-    } else if (!strcmp(name, "extfeature_ecx")) {
-        setfeatures(&def->ext3_features, str, ext3_feature_name, &err);
-    } else if (!strcmp(name, "xlevel")) {
-        setscalar(&def->xlevel, str, &err)
-    } else {
-        fprintf(stderr, "error: unknown option [%s = %s]\n", name, str);
-        return (1);
-    }
-    if (err) {
-        fprintf(stderr, "error: bad option value [%s = %s]\n", name, str);
-        return (1);
-    }
-    return (0);
-}
-
-/* register config file entry as x86_def_t
- */
-static int cpudef_register(QemuOpts *opts, void *opaque)
-{
-    x86_def_t *def = g_malloc0(sizeof (x86_def_t));
-
-    qemu_opt_foreach(opts, cpudef_setfield, def, 1);
-    def->next = x86_defs;
-    x86_defs = def;
-    return (0);
-}
 
 void cpu_clear_apic_feature(CPUX86State *env)
 {
@@ -1327,8 +1565,7 @@ void cpu_clear_apic_feature(CPUX86State *env)
 
 #endif /* !CONFIG_USER_ONLY */
 
-/* register "cpudef" models defined in configuration file.  Here we first
- * preload any built-in definitions
+/* Initialize list of CPU models, filling some non-static fields if necessary
  */
 void x86_cpudef_setup(void)
 {
@@ -1336,24 +1573,23 @@ void x86_cpudef_setup(void)
     static const char *model_with_versions[] = { "qemu32", "qemu64", "athlon" };
 
     for (i = 0; i < ARRAY_SIZE(builtin_x86_defs); ++i) {
-        builtin_x86_defs[i].next = x86_defs;
-        builtin_x86_defs[i].flags = 1;
+        x86_def_t *def = &builtin_x86_defs[i];
+        def->next = x86_defs;
 
         /* Look for specific "cpudef" models that */
         /* have the QEMU version in .model_id */
         for (j = 0; j < ARRAY_SIZE(model_with_versions); j++) {
-            if (strcmp(model_with_versions[j], builtin_x86_defs[i].name) == 0) {
-                pstrcpy(builtin_x86_defs[i].model_id, sizeof(builtin_x86_defs[i].model_id), "QEMU Virtual CPU version ");
-                pstrcat(builtin_x86_defs[i].model_id, sizeof(builtin_x86_defs[i].model_id), qemu_get_version());
+            if (strcmp(model_with_versions[j], def->name) == 0) {
+                pstrcpy(def->model_id, sizeof(def->model_id),
+                        "QEMU Virtual CPU version ");
+                pstrcat(def->model_id, sizeof(def->model_id),
+                        qemu_get_version());
                 break;
             }
         }
 
-        x86_defs = &builtin_x86_defs[i];
+        x86_defs = def;
     }
-#if !defined(CONFIG_USER_ONLY)
-    qemu_opts_foreach(qemu_find_opts("cpudef"), cpudef_register, NULL, 0);
-#endif
 }
 
 static void get_cpuid_vendor(CPUX86State *env, uint32_t *ebx,
@@ -1474,7 +1710,7 @@ void cpu_x86_cpuid(CPUX86State *env, uint32_t index, uint32_t count,
         /* Structured Extended Feature Flags Enumeration Leaf */
         if (count == 0) {
             *eax = 0; /* Maximum ECX value for sub-leaves */
-            *ebx = env->cpuid_7_0_ebx; /* Feature flags */
+            *ebx = env->cpuid_7_0_ebx_features; /* Feature flags */
             *ecx = 0; /* Reserved */
             *edx = 0; /* Reserved */
         } else {
@@ -1653,7 +1889,7 @@ static void x86_cpu_reset(CPUState *s)
 
     if (qemu_loglevel_mask(CPU_LOG_RESET)) {
         qemu_log("CPU Reset (CPU %d)\n", env->cpu_index);
-        log_cpu_state(env, X86_DUMP_FPU | X86_DUMP_CCOP);
+        log_cpu_state(env, CPU_DUMP_FPU | CPU_DUMP_CCOP);
     }
 
     xcc->parent_reset(s);
@@ -1764,12 +2000,65 @@ static void mce_init(X86CPU *cpu)
     }
 }
 
+#define MSI_ADDR_BASE 0xfee00000
+
+#ifndef CONFIG_USER_ONLY
+static void x86_cpu_apic_init(X86CPU *cpu, Error **errp)
+{
+    static int apic_mapped;
+    CPUX86State *env = &cpu->env;
+    APICCommonState *apic;
+    const char *apic_type = "apic";
+
+    if (kvm_irqchip_in_kernel()) {
+        apic_type = "kvm-apic";
+    } else if (xen_enabled()) {
+        apic_type = "xen-apic";
+    }
+
+    env->apic_state = qdev_try_create(NULL, apic_type);
+    if (env->apic_state == NULL) {
+        error_setg(errp, "APIC device '%s' could not be created", apic_type);
+        return;
+    }
+
+    object_property_add_child(OBJECT(cpu), "apic",
+                              OBJECT(env->apic_state), NULL);
+    qdev_prop_set_uint8(env->apic_state, "id", env->cpuid_apic_id);
+    /* TODO: convert to link<> */
+    apic = APIC_COMMON(env->apic_state);
+    apic->cpu = cpu;
+
+    if (qdev_init(env->apic_state)) {
+        error_setg(errp, "APIC device '%s' could not be initialized",
+                   object_get_typename(OBJECT(env->apic_state)));
+        return;
+    }
+
+    /* XXX: mapping more APICs at the same memory location */
+    if (apic_mapped == 0) {
+        /* NOTE: the APIC is directly connected to the CPU - it is not
+           on the global memory bus. */
+        /* XXX: what if the base changes? */
+        sysbus_mmio_map(sysbus_from_qdev(env->apic_state), 0, MSI_ADDR_BASE);
+        apic_mapped = 1;
+    }
+}
+#endif
+
 void x86_cpu_realize(Object *obj, Error **errp)
 {
     X86CPU *cpu = X86_CPU(obj);
 
 #ifndef CONFIG_USER_ONLY
     qemu_register_reset(x86_cpu_machine_reset_cb, cpu);
+
+    if (cpu->env.cpuid_features & CPUID_APIC || smp_cpus > 1) {
+        x86_cpu_apic_init(cpu, errp);
+        if (error_is_set(errp)) {
+            return;
+        }
+    }
 #endif
 
     mce_init(cpu);
diff --git a/target-i386/cpu.h b/target-i386/cpu.h
index 0677502dc..90ef1ff1e 100644
--- a/target-i386/cpu.h
+++ b/target-i386/cpu.h
@@ -123,8 +123,8 @@
 
 /* hidden flags - used internally by qemu to represent additional cpu
    states. Only the CPL, INHIBIT_IRQ, SMM and SVMI are not
-   redundant. We avoid using the IOPL_MASK, TF_MASK and VM_MASK bit
-   position to ease oring with eflags. */
+   redundant. We avoid using the IOPL_MASK, TF_MASK, VM_MASK and AC_MASK
+   bit positions to ease oring with eflags. */
 /* current cpl */
 #define HF_CPL_SHIFT         0
 /* true if soft mmu is being used */
@@ -147,10 +147,12 @@
 #define HF_CS64_SHIFT       15 /* only used on x86_64: 64 bit code segment  */
 #define HF_RF_SHIFT         16 /* must be same as eflags */
 #define HF_VM_SHIFT         17 /* must be same as eflags */
+#define HF_AC_SHIFT         18 /* must be same as eflags */
 #define HF_SMM_SHIFT        19 /* CPU in SMM mode */
 #define HF_SVME_SHIFT       20 /* SVME enabled (copy of EFER.SVME) */
 #define HF_SVMI_SHIFT       21 /* SVM intercepts are active */
 #define HF_OSFXSR_SHIFT     22 /* CR4.OSFXSR */
+#define HF_SMAP_SHIFT       23 /* CR4.SMAP */
 
 #define HF_CPL_MASK          (3 << HF_CPL_SHIFT)
 #define HF_SOFTMMU_MASK      (1 << HF_SOFTMMU_SHIFT)
@@ -168,10 +170,12 @@
 #define HF_CS64_MASK         (1 << HF_CS64_SHIFT)
 #define HF_RF_MASK           (1 << HF_RF_SHIFT)
 #define HF_VM_MASK           (1 << HF_VM_SHIFT)
+#define HF_AC_MASK           (1 << HF_AC_SHIFT)
 #define HF_SMM_MASK          (1 << HF_SMM_SHIFT)
 #define HF_SVME_MASK         (1 << HF_SVME_SHIFT)
 #define HF_SVMI_MASK         (1 << HF_SVMI_SHIFT)
 #define HF_OSFXSR_MASK       (1 << HF_OSFXSR_SHIFT)
+#define HF_SMAP_MASK         (1 << HF_SMAP_SHIFT)
 
 /* hflags2 */
 
@@ -210,6 +214,13 @@
 #define CR4_OSFXSR_SHIFT 9
 #define CR4_OSFXSR_MASK (1 << CR4_OSFXSR_SHIFT)
 #define CR4_OSXMMEXCPT_MASK  (1 << 10)
+#define CR4_VMXE_MASK   (1 << 13)
+#define CR4_SMXE_MASK   (1 << 14)
+#define CR4_FSGSBASE_MASK (1 << 16)
+#define CR4_PCIDE_MASK  (1 << 17)
+#define CR4_OSXSAVE_MASK (1 << 18)
+#define CR4_SMEP_MASK   (1 << 20)
+#define CR4_SMAP_MASK   (1 << 21)
 
 #define DR6_BD          (1 << 13)
 #define DR6_BS          (1 << 14)
@@ -382,6 +393,7 @@
 #define CPUID_PBE (1 << 31)
 
 #define CPUID_EXT_SSE3     (1 << 0)
+#define CPUID_EXT_PCLMULQDQ (1 << 1)
 #define CPUID_EXT_DTES64   (1 << 2)
 #define CPUID_EXT_MONITOR  (1 << 3)
 #define CPUID_EXT_DSCPL    (1 << 4)
@@ -391,9 +403,11 @@
 #define CPUID_EXT_TM2      (1 << 8)
 #define CPUID_EXT_SSSE3    (1 << 9)
 #define CPUID_EXT_CID      (1 << 10)
+#define CPUID_EXT_FMA      (1 << 12)
 #define CPUID_EXT_CX16     (1 << 13)
 #define CPUID_EXT_XTPR     (1 << 14)
 #define CPUID_EXT_PDCM     (1 << 15)
+#define CPUID_EXT_PCID     (1 << 17)
 #define CPUID_EXT_DCA      (1 << 18)
 #define CPUID_EXT_SSE41    (1 << 19)
 #define CPUID_EXT_SSE42    (1 << 20)
@@ -401,14 +415,36 @@
 #define CPUID_EXT_MOVBE    (1 << 22)
 #define CPUID_EXT_POPCNT   (1 << 23)
 #define CPUID_EXT_TSC_DEADLINE_TIMER (1 << 24)
+#define CPUID_EXT_AES      (1 << 25)
 #define CPUID_EXT_XSAVE    (1 << 26)
 #define CPUID_EXT_OSXSAVE  (1 << 27)
+#define CPUID_EXT_AVX      (1 << 28)
+#define CPUID_EXT_F16C     (1 << 29)
+#define CPUID_EXT_RDRAND   (1 << 30)
 #define CPUID_EXT_HYPERVISOR  (1 << 31)
 
+#define CPUID_EXT2_FPU     (1 << 0)
+#define CPUID_EXT2_VME     (1 << 1)
+#define CPUID_EXT2_DE      (1 << 2)
+#define CPUID_EXT2_PSE     (1 << 3)
+#define CPUID_EXT2_TSC     (1 << 4)
+#define CPUID_EXT2_MSR     (1 << 5)
+#define CPUID_EXT2_PAE     (1 << 6)
+#define CPUID_EXT2_MCE     (1 << 7)
+#define CPUID_EXT2_CX8     (1 << 8)
+#define CPUID_EXT2_APIC    (1 << 9)
 #define CPUID_EXT2_SYSCALL (1 << 11)
+#define CPUID_EXT2_MTRR    (1 << 12)
+#define CPUID_EXT2_PGE     (1 << 13)
+#define CPUID_EXT2_MCA     (1 << 14)
+#define CPUID_EXT2_CMOV    (1 << 15)
+#define CPUID_EXT2_PAT     (1 << 16)
+#define CPUID_EXT2_PSE36   (1 << 17)
 #define CPUID_EXT2_MP      (1 << 19)
 #define CPUID_EXT2_NX      (1 << 20)
 #define CPUID_EXT2_MMXEXT  (1 << 22)
+#define CPUID_EXT2_MMX     (1 << 23)
+#define CPUID_EXT2_FXSR    (1 << 24)
 #define CPUID_EXT2_FFXSR   (1 << 25)
 #define CPUID_EXT2_PDPE1GB (1 << 26)
 #define CPUID_EXT2_RDTSCP  (1 << 27)
@@ -416,6 +452,17 @@
 #define CPUID_EXT2_3DNOWEXT (1 << 30)
 #define CPUID_EXT2_3DNOW   (1 << 31)
 
+/* CPUID[8000_0001].EDX bits that are aliase of CPUID[1].EDX bits on AMD CPUs */
+#define CPUID_EXT2_AMD_ALIASES (CPUID_EXT2_FPU | CPUID_EXT2_VME | \
+                                CPUID_EXT2_DE | CPUID_EXT2_PSE | \
+                                CPUID_EXT2_TSC | CPUID_EXT2_MSR | \
+                                CPUID_EXT2_PAE | CPUID_EXT2_MCE | \
+                                CPUID_EXT2_CX8 | CPUID_EXT2_APIC | \
+                                CPUID_EXT2_MTRR | CPUID_EXT2_PGE | \
+                                CPUID_EXT2_MCA | CPUID_EXT2_CMOV | \
+                                CPUID_EXT2_PAT | CPUID_EXT2_PSE36 | \
+                                CPUID_EXT2_MMX | CPUID_EXT2_FXSR)
+
 #define CPUID_EXT3_LAHF_LM (1 << 0)
 #define CPUID_EXT3_CMP_LEG (1 << 1)
 #define CPUID_EXT3_SVM     (1 << 2)
@@ -427,7 +474,17 @@
 #define CPUID_EXT3_3DNOWPREFETCH (1 << 8)
 #define CPUID_EXT3_OSVW    (1 << 9)
 #define CPUID_EXT3_IBS     (1 << 10)
+#define CPUID_EXT3_XOP     (1 << 11)
 #define CPUID_EXT3_SKINIT  (1 << 12)
+#define CPUID_EXT3_WDT     (1 << 13)
+#define CPUID_EXT3_LWP     (1 << 15)
+#define CPUID_EXT3_FMA4    (1 << 16)
+#define CPUID_EXT3_TCE     (1 << 17)
+#define CPUID_EXT3_NODEID  (1 << 19)
+#define CPUID_EXT3_TBM     (1 << 21)
+#define CPUID_EXT3_TOPOEXT (1 << 22)
+#define CPUID_EXT3_PERFCORE (1 << 23)
+#define CPUID_EXT3_PERFNB  (1 << 24)
 
 #define CPUID_SVM_NPT          (1 << 0)
 #define CPUID_SVM_LBRV         (1 << 1)
@@ -440,6 +497,19 @@
 #define CPUID_SVM_PAUSEFILTER  (1 << 10)
 #define CPUID_SVM_PFTHRESHOLD  (1 << 12)
 
+#define CPUID_7_0_EBX_FSGSBASE (1 << 0)
+#define CPUID_7_0_EBX_BMI1     (1 << 3)
+#define CPUID_7_0_EBX_HLE      (1 << 4)
+#define CPUID_7_0_EBX_AVX2     (1 << 5)
+#define CPUID_7_0_EBX_SMEP     (1 << 7)
+#define CPUID_7_0_EBX_BMI2     (1 << 8)
+#define CPUID_7_0_EBX_ERMS     (1 << 9)
+#define CPUID_7_0_EBX_INVPCID  (1 << 10)
+#define CPUID_7_0_EBX_RTM      (1 << 11)
+#define CPUID_7_0_EBX_RDSEED   (1 << 18)
+#define CPUID_7_0_EBX_ADX      (1 << 19)
+#define CPUID_7_0_EBX_SMAP     (1 << 20)
+
 #define CPUID_VENDOR_INTEL_1 0x756e6547 /* "Genu" */
 #define CPUID_VENDOR_INTEL_2 0x49656e69 /* "ineI" */
 #define CPUID_VENDOR_INTEL_3 0x6c65746e /* "ntel" */
@@ -615,7 +685,7 @@ typedef struct {
 #define CPU_NB_REGS CPU_NB_REGS32
 #endif
 
-#define NB_MMU_MODES 2
+#define NB_MMU_MODES 3
 
 typedef enum TPRAccess {
     TPR_ACCESS_READ,
@@ -745,7 +815,7 @@ typedef struct CPUX86State {
     uint32_t cpuid_xlevel2;
     uint32_t cpuid_ext4_features;
     /* Flags from CPUID[EAX=7,ECX=0].EBX */
-    uint32_t cpuid_7_0_ebx;
+    uint32_t cpuid_7_0_ebx_features;
 
     /* MTRRs */
     uint64_t mtrr_fixed[11];
@@ -792,7 +862,7 @@ typedef struct CPUX86State {
 
 X86CPU *cpu_x86_init(const char *cpu_model);
 int cpu_x86_exec(CPUX86State *s);
-void x86_cpu_list (FILE *f, fprintf_function cpu_fprintf, const char *optarg);
+void x86_cpu_list(FILE *f, fprintf_function cpu_fprintf);
 void x86_cpudef_setup(void);
 int cpu_x86_support_mca_broadcast(CPUX86State *env);
 
@@ -859,9 +929,11 @@ static inline void cpu_x86_load_seg_cache(CPUX86State *env,
     }
 }
 
-static inline void cpu_x86_load_seg_cache_sipi(CPUX86State *env,
+static inline void cpu_x86_load_seg_cache_sipi(X86CPU *cpu,
                                                int sipi_vector)
 {
+    CPUX86State *env = &cpu->env;
+
     env->eip = 0;
     cpu_x86_load_seg_cache(env, R_CS, sipi_vector << 8,
                            sipi_vector << 12,
@@ -947,10 +1019,6 @@ void cpu_x86_update_cr4(CPUX86State *env, uint32_t new_cr4);
 void cpu_smm_update(CPUX86State *env);
 uint64_t cpu_get_tsc(CPUX86State *env);
 
-/* used to debug */
-#define X86_DUMP_FPU  0x0001 /* dump FPU state too */
-#define X86_DUMP_CCOP 0x0002 /* dump qemu flag cache */
-
 #define TARGET_PAGE_BITS 12
 
 #ifdef TARGET_X86_64
@@ -976,7 +1044,7 @@ static inline CPUX86State *cpu_init(const char *cpu_model)
 #define cpu_exec cpu_x86_exec
 #define cpu_gen_code cpu_x86_gen_code
 #define cpu_signal_handler cpu_x86_signal_handler
-#define cpu_list_id x86_cpu_list
+#define cpu_list x86_cpu_list
 #define cpudef_setup	x86_cpudef_setup
 
 #define CPU_SAVE_VERSION 12
@@ -984,10 +1052,15 @@ static inline CPUX86State *cpu_init(const char *cpu_model)
 /* MMU modes definitions */
 #define MMU_MODE0_SUFFIX _kernel
 #define MMU_MODE1_SUFFIX _user
-#define MMU_USER_IDX 1
+#define MMU_MODE2_SUFFIX _ksmap /* Kernel with SMAP override */
+#define MMU_KERNEL_IDX  0
+#define MMU_USER_IDX    1
+#define MMU_KSMAP_IDX   2
 static inline int cpu_mmu_index (CPUX86State *env)
 {
-    return (env->hflags & HF_CPL_MASK) == 3 ? 1 : 0;
+    return (env->hflags & HF_CPL_MASK) == 3 ? MMU_USER_IDX :
+        ((env->hflags & HF_SMAP_MASK) && (env->eflags & AC_MASK))
+        ? MMU_KSMAP_IDX : MMU_KERNEL_IDX;
 }
 
 #undef EAX
@@ -1049,8 +1122,10 @@ static inline void cpu_clone_regs(CPUX86State *env, target_ulong newsp)
 #include "hw/apic.h"
 #endif
 
-static inline bool cpu_has_work(CPUX86State *env)
+static inline bool cpu_has_work(CPUState *cpu)
 {
+    CPUX86State *env = &X86_CPU(cpu)->env;
+
     return ((env->interrupt_request & (CPU_INTERRUPT_HARD |
                                        CPU_INTERRUPT_POLL)) &&
             (env->eflags & IF_MASK)) ||
@@ -1073,7 +1148,7 @@ static inline void cpu_get_tb_cpu_state(CPUX86State *env, target_ulong *pc,
     *cs_base = env->segs[R_CS].base;
     *pc = *cs_base + env->eip;
     *flags = env->hflags |
-        (env->eflags & (IOPL_MASK | TF_MASK | RF_MASK | VM_MASK));
+        (env->eflags & (IOPL_MASK | TF_MASK | RF_MASK | VM_MASK | AC_MASK));
 }
 
 void do_cpu_init(X86CPU *cpu);
@@ -1082,7 +1157,7 @@ void do_cpu_sipi(X86CPU *cpu);
 #define MCE_INJECT_BROADCAST    1
 #define MCE_INJECT_UNCOND_AO    2
 
-void cpu_x86_inject_mce(Monitor *mon, CPUX86State *cenv, int bank,
+void cpu_x86_inject_mce(Monitor *mon, X86CPU *cpu, int bank,
                         uint64_t status, uint64_t mcg_status, uint64_t addr,
                         uint64_t misc, int flags);
 
@@ -1139,4 +1214,6 @@ void do_smm_enter(CPUX86State *env1);
 
 void cpu_report_tpr_access(CPUX86State *env, TPRAccess access);
 
+void enable_kvm_pv_eoi(void);
+
 #endif /* CPU_I386_H */
diff --git a/target-i386/helper.c b/target-i386/helper.c
index 8a5da3d7c..bf206cfa9 100644
--- a/target-i386/helper.c
+++ b/target-i386/helper.c
@@ -284,7 +284,7 @@ void cpu_dump_state(CPUX86State *env, FILE *f, fprintf_function cpu_fprintf,
         cpu_fprintf(f, "\nDR6=" TARGET_FMT_lx " DR7=" TARGET_FMT_lx "\n",
                     env->dr[6], env->dr[7]);
     }
-    if (flags & X86_DUMP_CCOP) {
+    if (flags & CPU_DUMP_CCOP) {
         if ((unsigned)env->cc_op < CC_OP_NB)
             snprintf(cc_op_name, sizeof(cc_op_name), "%s", cc_op_str[env->cc_op]);
         else
@@ -303,7 +303,7 @@ void cpu_dump_state(CPUX86State *env, FILE *f, fprintf_function cpu_fprintf,
         }
     }
     cpu_fprintf(f, "EFER=%016" PRIx64 "\n", env->efer);
-    if (flags & X86_DUMP_FPU) {
+    if (flags & CPU_DUMP_FPU) {
         int fptag;
         fptag = 0;
         for(i = 0; i < 8; i++) {
@@ -443,17 +443,27 @@ void cpu_x86_update_cr4(CPUX86State *env, uint32_t new_cr4)
 #if defined(DEBUG_MMU)
     printf("CR4 update: CR4=%08x\n", (uint32_t)env->cr[4]);
 #endif
-    if ((new_cr4 & (CR4_PGE_MASK | CR4_PAE_MASK | CR4_PSE_MASK)) !=
-        (env->cr[4] & (CR4_PGE_MASK | CR4_PAE_MASK | CR4_PSE_MASK))) {
+    if ((new_cr4 ^ env->cr[4]) &
+        (CR4_PGE_MASK | CR4_PAE_MASK | CR4_PSE_MASK |
+         CR4_SMEP_MASK | CR4_SMAP_MASK)) {
         tlb_flush(env, 1);
     }
     /* SSE handling */
-    if (!(env->cpuid_features & CPUID_SSE))
+    if (!(env->cpuid_features & CPUID_SSE)) {
         new_cr4 &= ~CR4_OSFXSR_MASK;
-    if (new_cr4 & CR4_OSFXSR_MASK)
+    }
+    env->hflags &= ~HF_OSFXSR_MASK;
+    if (new_cr4 & CR4_OSFXSR_MASK) {
         env->hflags |= HF_OSFXSR_MASK;
-    else
-        env->hflags &= ~HF_OSFXSR_MASK;
+    }
+
+    if (!(env->cpuid_7_0_ebx_features & CPUID_7_0_EBX_SMAP)) {
+        new_cr4 &= ~CR4_SMAP_MASK;
+    }
+    env->hflags &= ~HF_SMAP_MASK;
+    if (new_cr4 & CR4_SMAP_MASK) {
+        env->hflags |= HF_SMAP_MASK;
+    }
 
     env->cr[4] = new_cr4;
 }
@@ -493,7 +503,7 @@ int cpu_x86_handle_mmu_fault(CPUX86State *env, target_ulong addr,
     uint64_t ptep, pte;
     target_ulong pde_addr, pte_addr;
     int error_code, is_dirty, prot, page_size, is_write, is_user;
-    target_phys_addr_t paddr;
+    hwaddr paddr;
     uint32_t page_offset;
     target_ulong vaddr, virt_addr;
 
@@ -591,17 +601,38 @@ int cpu_x86_handle_mmu_fault(CPUX86State *env, target_ulong addr,
             /* 2 MB page */
             page_size = 2048 * 1024;
             ptep ^= PG_NX_MASK;
-            if ((ptep & PG_NX_MASK) && is_write1 == 2)
+            if ((ptep & PG_NX_MASK) && is_write1 == 2) {
                 goto do_fault_protect;
-            if (is_user) {
-                if (!(ptep & PG_USER_MASK))
+            }
+            switch (mmu_idx) {
+            case MMU_USER_IDX:
+                if (!(ptep & PG_USER_MASK)) {
                     goto do_fault_protect;
-                if (is_write && !(ptep & PG_RW_MASK))
+                }
+                if (is_write && !(ptep & PG_RW_MASK)) {
                     goto do_fault_protect;
-            } else {
+                }
+                break;
+
+            case MMU_KERNEL_IDX:
+                if (is_write1 != 2 && (env->cr[4] & CR4_SMAP_MASK) &&
+                    (ptep & PG_USER_MASK)) {
+                    goto do_fault_protect;
+                }
+                /* fall through */
+            case MMU_KSMAP_IDX:
+                if (is_write1 == 2 && (env->cr[4] & CR4_SMEP_MASK) &&
+                    (ptep & PG_USER_MASK)) {
+                    goto do_fault_protect;
+                }
                 if ((env->cr[0] & CR0_WP_MASK) &&
-                    is_write && !(ptep & PG_RW_MASK))
+                    is_write && !(ptep & PG_RW_MASK)) {
                     goto do_fault_protect;
+                }
+                break;
+
+            default: /* cannot happen */
+                break;
             }
             is_dirty = is_write && !(pde & PG_DIRTY_MASK);
             if (!(pde & PG_ACCESSED_MASK) || is_dirty) {
@@ -635,15 +666,35 @@ int cpu_x86_handle_mmu_fault(CPUX86State *env, target_ulong addr,
             ptep ^= PG_NX_MASK;
             if ((ptep & PG_NX_MASK) && is_write1 == 2)
                 goto do_fault_protect;
-            if (is_user) {
-                if (!(ptep & PG_USER_MASK))
+            switch (mmu_idx) {
+            case MMU_USER_IDX:
+                if (!(ptep & PG_USER_MASK)) {
                     goto do_fault_protect;
-                if (is_write && !(ptep & PG_RW_MASK))
+                }
+                if (is_write && !(ptep & PG_RW_MASK)) {
                     goto do_fault_protect;
-            } else {
+                }
+                break;
+
+            case MMU_KERNEL_IDX:
+                if (is_write1 != 2 && (env->cr[4] & CR4_SMAP_MASK) &&
+                    (ptep & PG_USER_MASK)) {
+                    goto do_fault_protect;
+                }
+                /* fall through */
+            case MMU_KSMAP_IDX:
+                if (is_write1 == 2 && (env->cr[4] & CR4_SMEP_MASK) &&
+                    (ptep & PG_USER_MASK)) {
+                    goto do_fault_protect;
+                }
                 if ((env->cr[0] & CR0_WP_MASK) &&
-                    is_write && !(ptep & PG_RW_MASK))
+                    is_write && !(ptep & PG_RW_MASK)) {
                     goto do_fault_protect;
+                }
+                break;
+
+            default: /* cannot happen */
+                break;
             }
             is_dirty = is_write && !(pte & PG_DIRTY_MASK);
             if (!(pte & PG_ACCESSED_MASK) || is_dirty) {
@@ -670,15 +721,35 @@ int cpu_x86_handle_mmu_fault(CPUX86State *env, target_ulong addr,
         /* if PSE bit is set, then we use a 4MB page */
         if ((pde & PG_PSE_MASK) && (env->cr[4] & CR4_PSE_MASK)) {
             page_size = 4096 * 1024;
-            if (is_user) {
-                if (!(pde & PG_USER_MASK))
+            switch (mmu_idx) {
+            case MMU_USER_IDX:
+                if (!(pde & PG_USER_MASK)) {
                     goto do_fault_protect;
-                if (is_write && !(pde & PG_RW_MASK))
+                }
+                if (is_write && !(pde & PG_RW_MASK)) {
                     goto do_fault_protect;
-            } else {
+                }
+                break;
+
+            case MMU_KERNEL_IDX:
+                if (is_write1 != 2 && (env->cr[4] & CR4_SMAP_MASK) &&
+                    (pde & PG_USER_MASK)) {
+                    goto do_fault_protect;
+                }
+                /* fall through */
+            case MMU_KSMAP_IDX:
+                if (is_write1 == 2 && (env->cr[4] & CR4_SMEP_MASK) &&
+                    (pde & PG_USER_MASK)) {
+                    goto do_fault_protect;
+                }
                 if ((env->cr[0] & CR0_WP_MASK) &&
-                    is_write && !(pde & PG_RW_MASK))
+                    is_write && !(pde & PG_RW_MASK)) {
                     goto do_fault_protect;
+                }
+                break;
+
+            default: /* cannot happen */
+                break;
             }
             is_dirty = is_write && !(pde & PG_DIRTY_MASK);
             if (!(pde & PG_ACCESSED_MASK) || is_dirty) {
@@ -707,15 +778,35 @@ int cpu_x86_handle_mmu_fault(CPUX86State *env, target_ulong addr,
             }
             /* combine pde and pte user and rw protections */
             ptep = pte & pde;
-            if (is_user) {
-                if (!(ptep & PG_USER_MASK))
+            switch (mmu_idx) {
+            case MMU_USER_IDX:
+                if (!(ptep & PG_USER_MASK)) {
+                    goto do_fault_protect;
+                }
+                if (is_write && !(ptep & PG_RW_MASK)) {
+                    goto do_fault_protect;
+                }
+                break;
+
+            case MMU_KERNEL_IDX:
+                if (is_write1 != 2 && (env->cr[4] & CR4_SMAP_MASK) &&
+                    (ptep & PG_USER_MASK)) {
                     goto do_fault_protect;
-                if (is_write && !(ptep & PG_RW_MASK))
+                }
+                /* fall through */
+            case MMU_KSMAP_IDX:
+                if (is_write1 == 2 && (env->cr[4] & CR4_SMEP_MASK) &&
+                    (ptep & PG_USER_MASK)) {
                     goto do_fault_protect;
-            } else {
+                }
                 if ((env->cr[0] & CR0_WP_MASK) &&
-                    is_write && !(ptep & PG_RW_MASK))
+                    is_write && !(ptep & PG_RW_MASK)) {
                     goto do_fault_protect;
+                }
+                break;
+
+            default: /* cannot happen */
+                break;
             }
             is_dirty = is_write && !(pte & PG_DIRTY_MASK);
             if (!(pte & PG_ACCESSED_MASK) || is_dirty) {
@@ -762,8 +853,9 @@ int cpu_x86_handle_mmu_fault(CPUX86State *env, target_ulong addr,
     if (is_user)
         error_code |= PG_ERROR_U_MASK;
     if (is_write1 == 2 &&
-        (env->efer & MSR_EFER_NXE) &&
-        (env->cr[4] & CR4_PAE_MASK))
+        (((env->efer & MSR_EFER_NXE) &&
+          (env->cr[4] & CR4_PAE_MASK)) ||
+         (env->cr[4] & CR4_SMEP_MASK)))
         error_code |= PG_ERROR_I_D_MASK;
     if (env->intercept_exceptions & (1 << EXCP0E_PAGE)) {
         /* cr2 is not modified in case of exceptions */
@@ -777,11 +869,11 @@ int cpu_x86_handle_mmu_fault(CPUX86State *env, target_ulong addr,
     return 1;
 }
 
-target_phys_addr_t cpu_get_phys_page_debug(CPUX86State *env, target_ulong addr)
+hwaddr cpu_get_phys_page_debug(CPUX86State *env, target_ulong addr)
 {
     target_ulong pde_addr, pte_addr;
     uint64_t pte;
-    target_phys_addr_t paddr;
+    hwaddr paddr;
     uint32_t page_offset;
     int page_size;
 
@@ -1049,10 +1141,11 @@ static void do_inject_x86_mce(void *data)
     }
 }
 
-void cpu_x86_inject_mce(Monitor *mon, CPUX86State *cenv, int bank,
+void cpu_x86_inject_mce(Monitor *mon, X86CPU *cpu, int bank,
                         uint64_t status, uint64_t mcg_status, uint64_t addr,
                         uint64_t misc, int flags)
 {
+    CPUX86State *cenv = &cpu->env;
     MCEInjectionParams params = {
         .mon = mon,
         .env = cenv,
@@ -1084,7 +1177,7 @@ void cpu_x86_inject_mce(Monitor *mon, CPUX86State *cenv, int bank,
         return;
     }
 
-    run_on_cpu(cenv, do_inject_x86_mce, &params);
+    run_on_cpu(CPU(cpu), do_inject_x86_mce, &params);
     if (flags & MCE_INJECT_BROADCAST) {
         params.bank = 1;
         params.status = MCI_STATUS_VAL | MCI_STATUS_UC;
@@ -1096,7 +1189,7 @@ void cpu_x86_inject_mce(Monitor *mon, CPUX86State *cenv, int bank,
                 continue;
             }
             params.env = env;
-            run_on_cpu(cenv, do_inject_x86_mce, &params);
+            run_on_cpu(CPU(cpu), do_inject_x86_mce, &params);
         }
     }
 }
@@ -1151,6 +1244,7 @@ X86CPU *cpu_x86_init(const char *cpu_model)
 {
     X86CPU *cpu;
     CPUX86State *env;
+    Error *error = NULL;
 
     cpu = X86_CPU(object_new(TYPE_X86_CPU));
     env = &cpu->env;
@@ -1161,8 +1255,12 @@ X86CPU *cpu_x86_init(const char *cpu_model)
         return NULL;
     }
 
-    x86_cpu_realize(OBJECT(cpu), NULL);
-
+    x86_cpu_realize(OBJECT(cpu), &error);
+    if (error) {
+        error_free(error);
+        object_delete(OBJECT(cpu));
+        return NULL;
+    }
     return cpu;
 }
 
diff --git a/target-i386/helper.h b/target-i386/helper.h
index ab6af638e..970fcd98f 100644
--- a/target-i386/helper.h
+++ b/target-i386/helper.h
@@ -1,7 +1,7 @@
 #include "def-helper.h"
 
-DEF_HELPER_FLAGS_2(cc_compute_all, TCG_CALL_PURE, i32, env, int)
-DEF_HELPER_FLAGS_2(cc_compute_c, TCG_CALL_PURE, i32, env, int)
+DEF_HELPER_FLAGS_2(cc_compute_all, TCG_CALL_NO_SE, i32, env, int)
+DEF_HELPER_FLAGS_2(cc_compute_c, TCG_CALL_NO_SE, i32, env, int)
 
 DEF_HELPER_0(lock, void)
 DEF_HELPER_0(unlock, void)
@@ -67,6 +67,8 @@ DEF_HELPER_3(raise_interrupt, void, env, int, int)
 DEF_HELPER_2(raise_exception, void, env, int)
 DEF_HELPER_1(cli, void, env)
 DEF_HELPER_1(sti, void, env)
+DEF_HELPER_1(clac, void, env)
+DEF_HELPER_1(stac, void, env)
 DEF_HELPER_1(set_inhibit_irq, void, env)
 DEF_HELPER_1(reset_inhibit_irq, void, env)
 DEF_HELPER_3(boundw, void, env, tl, int)
diff --git a/target-i386/kvm.c b/target-i386/kvm.c
index ffc294ec3..f669281e1 100644
--- a/target-i386/kvm.c
+++ b/target-i386/kvm.c
@@ -31,6 +31,7 @@
 #include "hw/apic.h"
 #include "ioport.h"
 #include "hyperv.h"
+#include "hw/pci.h"
 
 //#define DEBUG_KVM
 
@@ -97,6 +98,19 @@ static struct kvm_cpuid2 *try_get_cpuid(KVMState *s, int max)
     return cpuid;
 }
 
+/* Run KVM_GET_SUPPORTED_CPUID ioctl(), allocating a buffer large enough
+ * for all entries.
+ */
+static struct kvm_cpuid2 *get_supported_cpuid(KVMState *s)
+{
+    struct kvm_cpuid2 *cpuid;
+    int max = 1;
+    while ((cpuid = try_get_cpuid(s, max)) == NULL) {
+        max *= 2;
+    }
+    return cpuid;
+}
+
 struct kvm_para_features {
     int cap;
     int feature;
@@ -122,60 +136,98 @@ static int get_para_features(KVMState *s)
 }
 
 
+/* Returns the value for a specific register on the cpuid entry
+ */
+static uint32_t cpuid_entry_get_reg(struct kvm_cpuid_entry2 *entry, int reg)
+{
+    uint32_t ret = 0;
+    switch (reg) {
+    case R_EAX:
+        ret = entry->eax;
+        break;
+    case R_EBX:
+        ret = entry->ebx;
+        break;
+    case R_ECX:
+        ret = entry->ecx;
+        break;
+    case R_EDX:
+        ret = entry->edx;
+        break;
+    }
+    return ret;
+}
+
+/* Find matching entry for function/index on kvm_cpuid2 struct
+ */
+static struct kvm_cpuid_entry2 *cpuid_find_entry(struct kvm_cpuid2 *cpuid,
+                                                 uint32_t function,
+                                                 uint32_t index)
+{
+    int i;
+    for (i = 0; i < cpuid->nent; ++i) {
+        if (cpuid->entries[i].function == function &&
+            cpuid->entries[i].index == index) {
+            return &cpuid->entries[i];
+        }
+    }
+    /* not found: */
+    return NULL;
+}
+
 uint32_t kvm_arch_get_supported_cpuid(KVMState *s, uint32_t function,
                                       uint32_t index, int reg)
 {
     struct kvm_cpuid2 *cpuid;
-    int i, max;
     uint32_t ret = 0;
     uint32_t cpuid_1_edx;
-    int has_kvm_features = 0;
+    bool found = false;
 
-    max = 1;
-    while ((cpuid = try_get_cpuid(s, max)) == NULL) {
-        max *= 2;
+    cpuid = get_supported_cpuid(s);
+
+    struct kvm_cpuid_entry2 *entry = cpuid_find_entry(cpuid, function, index);
+    if (entry) {
+        found = true;
+        ret = cpuid_entry_get_reg(entry, reg);
     }
 
-    for (i = 0; i < cpuid->nent; ++i) {
-        if (cpuid->entries[i].function == function &&
-            cpuid->entries[i].index == index) {
-            if (cpuid->entries[i].function == KVM_CPUID_FEATURES) {
-                has_kvm_features = 1;
-            }
-            switch (reg) {
-            case R_EAX:
-                ret = cpuid->entries[i].eax;
-                break;
-            case R_EBX:
-                ret = cpuid->entries[i].ebx;
-                break;
-            case R_ECX:
-                ret = cpuid->entries[i].ecx;
-                break;
-            case R_EDX:
-                ret = cpuid->entries[i].edx;
-                switch (function) {
-                case 1:
-                    /* KVM before 2.6.30 misreports the following features */
-                    ret |= CPUID_MTRR | CPUID_PAT | CPUID_MCE | CPUID_MCA;
-                    break;
-                case 0x80000001:
-                    /* On Intel, kvm returns cpuid according to the Intel spec,
-                     * so add missing bits according to the AMD spec:
-                     */
-                    cpuid_1_edx = kvm_arch_get_supported_cpuid(s, 1, 0, R_EDX);
-                    ret |= cpuid_1_edx & 0x183f7ff;
-                    break;
-                }
-                break;
-            }
+    /* Fixups for the data returned by KVM, below */
+
+    if (function == 1 && reg == R_EDX) {
+        /* KVM before 2.6.30 misreports the following features */
+        ret |= CPUID_MTRR | CPUID_PAT | CPUID_MCE | CPUID_MCA;
+    } else if (function == 1 && reg == R_ECX) {
+        /* We can set the hypervisor flag, even if KVM does not return it on
+         * GET_SUPPORTED_CPUID
+         */
+        ret |= CPUID_EXT_HYPERVISOR;
+        /* tsc-deadline flag is not returned by GET_SUPPORTED_CPUID, but it
+         * can be enabled if the kernel has KVM_CAP_TSC_DEADLINE_TIMER,
+         * and the irqchip is in the kernel.
+         */
+        if (kvm_irqchip_in_kernel() &&
+                kvm_check_extension(s, KVM_CAP_TSC_DEADLINE_TIMER)) {
+            ret |= CPUID_EXT_TSC_DEADLINE_TIMER;
         }
+
+        /* x2apic is reported by GET_SUPPORTED_CPUID, but it can't be enabled
+         * without the in-kernel irqchip
+         */
+        if (!kvm_irqchip_in_kernel()) {
+            ret &= ~CPUID_EXT_X2APIC;
+        }
+    } else if (function == 0x80000001 && reg == R_EDX) {
+        /* On Intel, kvm returns cpuid according to the Intel spec,
+         * so add missing bits according to the AMD spec:
+         */
+        cpuid_1_edx = kvm_arch_get_supported_cpuid(s, 1, 0, R_EDX);
+        ret |= cpuid_1_edx & CPUID_EXT2_AMD_ALIASES;
     }
 
     g_free(cpuid);
 
     /* fallback for older kernels */
-    if (!has_kvm_features && (function == KVM_CPUID_FEATURES)) {
+    if ((function == KVM_CPUID_FEATURES) && !found) {
         ret = get_para_features(s);
     }
 
@@ -228,8 +280,9 @@ static int kvm_get_mce_cap_supported(KVMState *s, uint64_t *mce_cap,
     return -ENOSYS;
 }
 
-static void kvm_mce_inject(CPUX86State *env, target_phys_addr_t paddr, int code)
+static void kvm_mce_inject(X86CPU *cpu, hwaddr paddr, int code)
 {
+    CPUX86State *env = &cpu->env;
     uint64_t status = MCI_STATUS_VAL | MCI_STATUS_UC | MCI_STATUS_EN |
                       MCI_STATUS_MISCV | MCI_STATUS_ADDRV | MCI_STATUS_S;
     uint64_t mcg_status = MCG_STATUS_MCIP;
@@ -241,7 +294,7 @@ static void kvm_mce_inject(CPUX86State *env, target_phys_addr_t paddr, int code)
         status |= 0xc0;
         mcg_status |= MCG_STATUS_RIPV;
     }
-    cpu_x86_inject_mce(NULL, env, 9, status, mcg_status, paddr,
+    cpu_x86_inject_mce(NULL, cpu, 9, status, mcg_status, paddr,
                        (MCM_ADDR_PHYS << 6) | 0xc,
                        cpu_x86_support_mca_broadcast(env) ?
                        MCE_INJECT_BROADCAST : 0);
@@ -255,8 +308,9 @@ static void hardware_memory_error(void)
 
 int kvm_arch_on_sigbus_vcpu(CPUX86State *env, int code, void *addr)
 {
+    X86CPU *cpu = x86_env_get_cpu(env);
     ram_addr_t ram_addr;
-    target_phys_addr_t paddr;
+    hwaddr paddr;
 
     if ((env->mcg_cap & MCG_SER_P) && addr
         && (code == BUS_MCEERR_AR || code == BUS_MCEERR_AO)) {
@@ -272,7 +326,7 @@ int kvm_arch_on_sigbus_vcpu(CPUX86State *env, int code, void *addr)
             }
         }
         kvm_hwpoison_page_add(ram_addr);
-        kvm_mce_inject(env, paddr, code);
+        kvm_mce_inject(cpu, paddr, code);
     } else {
         if (code == BUS_MCEERR_AO) {
             return 0;
@@ -289,7 +343,7 @@ int kvm_arch_on_sigbus(int code, void *addr)
 {
     if ((first_cpu->mcg_cap & MCG_SER_P) && addr && code == BUS_MCEERR_AO) {
         ram_addr_t ram_addr;
-        target_phys_addr_t paddr;
+        hwaddr paddr;
 
         /* Hope we are lucky for AO MCE */
         if (qemu_ram_addr_from_host(addr, &ram_addr) ||
@@ -300,7 +354,7 @@ int kvm_arch_on_sigbus(int code, void *addr)
             return 0;
         }
         kvm_hwpoison_page_add(ram_addr);
-        kvm_mce_inject(first_cpu, paddr, code);
+        kvm_mce_inject(x86_env_get_cpu(first_cpu), paddr, code);
     } else {
         if (code == BUS_MCEERR_AO) {
             return 0;
@@ -358,31 +412,12 @@ int kvm_arch_init_vcpu(CPUX86State *env)
         struct kvm_cpuid2 cpuid;
         struct kvm_cpuid_entry2 entries[100];
     } QEMU_PACKED cpuid_data;
-    KVMState *s = env->kvm_state;
     uint32_t limit, i, j, cpuid_i;
     uint32_t unused;
     struct kvm_cpuid_entry2 *c;
     uint32_t signature[3];
     int r;
 
-    env->cpuid_features &= kvm_arch_get_supported_cpuid(s, 1, 0, R_EDX);
-
-    i = env->cpuid_ext_features & CPUID_EXT_HYPERVISOR;
-    j = env->cpuid_ext_features & CPUID_EXT_TSC_DEADLINE_TIMER;
-    env->cpuid_ext_features &= kvm_arch_get_supported_cpuid(s, 1, 0, R_ECX);
-    env->cpuid_ext_features |= i;
-    if (j && kvm_irqchip_in_kernel() &&
-        kvm_check_extension(s, KVM_CAP_TSC_DEADLINE_TIMER)) {
-        env->cpuid_ext_features |= CPUID_EXT_TSC_DEADLINE_TIMER;
-    }
-
-    env->cpuid_ext2_features &= kvm_arch_get_supported_cpuid(s, 0x80000001,
-                                                             0, R_EDX);
-    env->cpuid_ext3_features &= kvm_arch_get_supported_cpuid(s, 0x80000001,
-                                                             0, R_ECX);
-    env->cpuid_svm_features  &= kvm_arch_get_supported_cpuid(s, 0x8000000A,
-                                                             0, R_EDX);
-
     cpuid_i = 0;
 
     /* Paravirtualization CPUIDs */
@@ -403,8 +438,7 @@ int kvm_arch_init_vcpu(CPUX86State *env)
     c = &cpuid_data.entries[cpuid_i++];
     memset(c, 0, sizeof(*c));
     c->function = KVM_CPUID_FEATURES;
-    c->eax = env->cpuid_kvm_features &
-        kvm_arch_get_supported_cpuid(s, KVM_CPUID_FEATURES, 0, R_EAX);
+    c->eax = env->cpuid_kvm_features;
 
     if (hyperv_enabled()) {
         memcpy(signature, "Hv#1\0\0\0\0\0\0\0\0", 12);
@@ -525,8 +559,6 @@ int kvm_arch_init_vcpu(CPUX86State *env)
 
     /* Call Centaur's CPUID instructions they are supported. */
     if (env->cpuid_xlevel2 > 0) {
-        env->cpuid_ext4_features &=
-            kvm_arch_get_supported_cpuid(s, 0xC0000001, 0, R_EDX);
         cpu_x86_cpuid(env, 0xC0000000, 0, &limit, &unused, &unused, &unused);
 
         for (i = 0xC0000000; i <= limit; i++) {
@@ -1364,8 +1396,9 @@ static int kvm_put_mp_state(CPUX86State *env)
     return kvm_vcpu_ioctl(env, KVM_SET_MP_STATE, &mp_state);
 }
 
-static int kvm_get_mp_state(CPUX86State *env)
+static int kvm_get_mp_state(X86CPU *cpu)
 {
+    CPUX86State *env = &cpu->env;
     struct kvm_mp_state mp_state;
     int ret;
 
@@ -1551,9 +1584,10 @@ static int kvm_get_debugregs(CPUX86State *env)
 
 int kvm_arch_put_registers(CPUX86State *env, int level)
 {
+    CPUState *cpu = ENV_GET_CPU(env);
     int ret;
 
-    assert(cpu_is_stopped(env) || qemu_cpu_is_self(env));
+    assert(cpu_is_stopped(cpu) || qemu_cpu_is_self(cpu));
 
     ret = kvm_getput_regs(env, 1);
     if (ret < 0) {
@@ -1608,9 +1642,10 @@ int kvm_arch_put_registers(CPUX86State *env, int level)
 
 int kvm_arch_get_registers(CPUX86State *env)
 {
+    X86CPU *cpu = x86_env_get_cpu(env);
     int ret;
 
-    assert(cpu_is_stopped(env) || qemu_cpu_is_self(env));
+    assert(cpu_is_stopped(CPU(cpu)) || qemu_cpu_is_self(CPU(cpu)));
 
     ret = kvm_getput_regs(env, 0);
     if (ret < 0) {
@@ -1632,7 +1667,7 @@ int kvm_arch_get_registers(CPUX86State *env)
     if (ret < 0) {
         return ret;
     }
-    ret = kvm_get_mp_state(env);
+    ret = kvm_get_mp_state(cpu);
     if (ret < 0) {
         return ret;
     }
@@ -1780,8 +1815,10 @@ int kvm_arch_process_async_events(CPUX86State *env)
     return env->halted;
 }
 
-static int kvm_handle_halt(CPUX86State *env)
+static int kvm_handle_halt(X86CPU *cpu)
 {
+    CPUX86State *env = &cpu->env;
+
     if (!((env->interrupt_request & CPU_INTERRUPT_HARD) &&
           (env->eflags & IF_MASK)) &&
         !(env->interrupt_request & CPU_INTERRUPT_NMI)) {
@@ -1908,14 +1945,15 @@ void kvm_arch_remove_all_hw_breakpoints(void)
 
 static CPUWatchpoint hw_watchpoint;
 
-static int kvm_handle_debug(struct kvm_debug_exit_arch *arch_info)
+static int kvm_handle_debug(CPUX86State *env,
+                            struct kvm_debug_exit_arch *arch_info)
 {
     int ret = 0;
     int n;
 
     if (arch_info->exception == 1) {
         if (arch_info->dr6 & (1 << 14)) {
-            if (cpu_single_env->singlestep_enabled) {
+            if (env->singlestep_enabled) {
                 ret = EXCP_DEBUG;
             }
         } else {
@@ -1927,13 +1965,13 @@ static int kvm_handle_debug(struct kvm_debug_exit_arch *arch_info)
                         break;
                     case 0x1:
                         ret = EXCP_DEBUG;
-                        cpu_single_env->watchpoint_hit = &hw_watchpoint;
+                        env->watchpoint_hit = &hw_watchpoint;
                         hw_watchpoint.vaddr = hw_breakpoint[n].addr;
                         hw_watchpoint.flags = BP_MEM_WRITE;
                         break;
                     case 0x3:
                         ret = EXCP_DEBUG;
-                        cpu_single_env->watchpoint_hit = &hw_watchpoint;
+                        env->watchpoint_hit = &hw_watchpoint;
                         hw_watchpoint.vaddr = hw_breakpoint[n].addr;
                         hw_watchpoint.flags = BP_MEM_ACCESS;
                         break;
@@ -1941,16 +1979,16 @@ static int kvm_handle_debug(struct kvm_debug_exit_arch *arch_info)
                 }
             }
         }
-    } else if (kvm_find_sw_breakpoint(cpu_single_env, arch_info->pc)) {
+    } else if (kvm_find_sw_breakpoint(env, arch_info->pc)) {
         ret = EXCP_DEBUG;
     }
     if (ret == 0) {
-        cpu_synchronize_state(cpu_single_env);
-        assert(cpu_single_env->exception_injected == -1);
+        cpu_synchronize_state(env);
+        assert(env->exception_injected == -1);
 
         /* pass to guest */
-        cpu_single_env->exception_injected = arch_info->exception;
-        cpu_single_env->has_error_code = 0;
+        env->exception_injected = arch_info->exception;
+        env->has_error_code = 0;
     }
 
     return ret;
@@ -1995,13 +2033,14 @@ static bool host_supports_vmx(void)
 
 int kvm_arch_handle_exit(CPUX86State *env, struct kvm_run *run)
 {
+    X86CPU *cpu = x86_env_get_cpu(env);
     uint64_t code;
     int ret;
 
     switch (run->exit_reason) {
     case KVM_EXIT_HLT:
         DPRINTF("handle_hlt\n");
-        ret = kvm_handle_halt(env);
+        ret = kvm_handle_halt(cpu);
         break;
     case KVM_EXIT_SET_TPR:
         ret = 0;
@@ -2033,7 +2072,7 @@ int kvm_arch_handle_exit(CPUX86State *env, struct kvm_run *run)
         break;
     case KVM_EXIT_DEBUG:
         DPRINTF("kvm_exit_debug\n");
-        ret = kvm_handle_debug(&run->debug.arch);
+        ret = kvm_handle_debug(env, &run->debug.arch);
         break;
     default:
         fprintf(stderr, "KVM: unknown exit reason %d\n", run->exit_reason);
@@ -2068,3 +2107,143 @@ void kvm_arch_init_irq_routing(KVMState *s)
     kvm_msi_via_irqfd_allowed = true;
     kvm_gsi_routing_allowed = true;
 }
+
+/* Classic KVM device assignment interface. Will remain x86 only. */
+int kvm_device_pci_assign(KVMState *s, PCIHostDeviceAddress *dev_addr,
+                          uint32_t flags, uint32_t *dev_id)
+{
+    struct kvm_assigned_pci_dev dev_data = {
+        .segnr = dev_addr->domain,
+        .busnr = dev_addr->bus,
+        .devfn = PCI_DEVFN(dev_addr->slot, dev_addr->function),
+        .flags = flags,
+    };
+    int ret;
+
+    dev_data.assigned_dev_id =
+        (dev_addr->domain << 16) | (dev_addr->bus << 8) | dev_data.devfn;
+
+    ret = kvm_vm_ioctl(s, KVM_ASSIGN_PCI_DEVICE, &dev_data);
+    if (ret < 0) {
+        return ret;
+    }
+
+    *dev_id = dev_data.assigned_dev_id;
+
+    return 0;
+}
+
+int kvm_device_pci_deassign(KVMState *s, uint32_t dev_id)
+{
+    struct kvm_assigned_pci_dev dev_data = {
+        .assigned_dev_id = dev_id,
+    };
+
+    return kvm_vm_ioctl(s, KVM_DEASSIGN_PCI_DEVICE, &dev_data);
+}
+
+static int kvm_assign_irq_internal(KVMState *s, uint32_t dev_id,
+                                   uint32_t irq_type, uint32_t guest_irq)
+{
+    struct kvm_assigned_irq assigned_irq = {
+        .assigned_dev_id = dev_id,
+        .guest_irq = guest_irq,
+        .flags = irq_type,
+    };
+
+    if (kvm_check_extension(s, KVM_CAP_ASSIGN_DEV_IRQ)) {
+        return kvm_vm_ioctl(s, KVM_ASSIGN_DEV_IRQ, &assigned_irq);
+    } else {
+        return kvm_vm_ioctl(s, KVM_ASSIGN_IRQ, &assigned_irq);
+    }
+}
+
+int kvm_device_intx_assign(KVMState *s, uint32_t dev_id, bool use_host_msi,
+                           uint32_t guest_irq)
+{
+    uint32_t irq_type = KVM_DEV_IRQ_GUEST_INTX |
+        (use_host_msi ? KVM_DEV_IRQ_HOST_MSI : KVM_DEV_IRQ_HOST_INTX);
+
+    return kvm_assign_irq_internal(s, dev_id, irq_type, guest_irq);
+}
+
+int kvm_device_intx_set_mask(KVMState *s, uint32_t dev_id, bool masked)
+{
+    struct kvm_assigned_pci_dev dev_data = {
+        .assigned_dev_id = dev_id,
+        .flags = masked ? KVM_DEV_ASSIGN_MASK_INTX : 0,
+    };
+
+    return kvm_vm_ioctl(s, KVM_ASSIGN_SET_INTX_MASK, &dev_data);
+}
+
+static int kvm_deassign_irq_internal(KVMState *s, uint32_t dev_id,
+                                     uint32_t type)
+{
+    struct kvm_assigned_irq assigned_irq = {
+        .assigned_dev_id = dev_id,
+        .flags = type,
+    };
+
+    return kvm_vm_ioctl(s, KVM_DEASSIGN_DEV_IRQ, &assigned_irq);
+}
+
+int kvm_device_intx_deassign(KVMState *s, uint32_t dev_id, bool use_host_msi)
+{
+    return kvm_deassign_irq_internal(s, dev_id, KVM_DEV_IRQ_GUEST_INTX |
+        (use_host_msi ? KVM_DEV_IRQ_HOST_MSI : KVM_DEV_IRQ_HOST_INTX));
+}
+
+int kvm_device_msi_assign(KVMState *s, uint32_t dev_id, int virq)
+{
+    return kvm_assign_irq_internal(s, dev_id, KVM_DEV_IRQ_HOST_MSI |
+                                              KVM_DEV_IRQ_GUEST_MSI, virq);
+}
+
+int kvm_device_msi_deassign(KVMState *s, uint32_t dev_id)
+{
+    return kvm_deassign_irq_internal(s, dev_id, KVM_DEV_IRQ_GUEST_MSI |
+                                                KVM_DEV_IRQ_HOST_MSI);
+}
+
+bool kvm_device_msix_supported(KVMState *s)
+{
+    /* The kernel lacks a corresponding KVM_CAP, so we probe by calling
+     * KVM_ASSIGN_SET_MSIX_NR with an invalid parameter. */
+    return kvm_vm_ioctl(s, KVM_ASSIGN_SET_MSIX_NR, NULL) == -EFAULT;
+}
+
+int kvm_device_msix_init_vectors(KVMState *s, uint32_t dev_id,
+                                 uint32_t nr_vectors)
+{
+    struct kvm_assigned_msix_nr msix_nr = {
+        .assigned_dev_id = dev_id,
+        .entry_nr = nr_vectors,
+    };
+
+    return kvm_vm_ioctl(s, KVM_ASSIGN_SET_MSIX_NR, &msix_nr);
+}
+
+int kvm_device_msix_set_vector(KVMState *s, uint32_t dev_id, uint32_t vector,
+                               int virq)
+{
+    struct kvm_assigned_msix_entry msix_entry = {
+        .assigned_dev_id = dev_id,
+        .gsi = virq,
+        .entry = vector,
+    };
+
+    return kvm_vm_ioctl(s, KVM_ASSIGN_SET_MSIX_ENTRY, &msix_entry);
+}
+
+int kvm_device_msix_assign(KVMState *s, uint32_t dev_id)
+{
+    return kvm_assign_irq_internal(s, dev_id, KVM_DEV_IRQ_HOST_MSIX |
+                                              KVM_DEV_IRQ_GUEST_MSIX, 0);
+}
+
+int kvm_device_msix_deassign(KVMState *s, uint32_t dev_id)
+{
+    return kvm_deassign_irq_internal(s, dev_id, KVM_DEV_IRQ_GUEST_MSIX |
+                                                KVM_DEV_IRQ_HOST_MSIX);
+}
diff --git a/target-i386/kvm_i386.h b/target-i386/kvm_i386.h
index b82bbf401..f6ab82f93 100644
--- a/target-i386/kvm_i386.h
+++ b/target-i386/kvm_i386.h
@@ -11,6 +11,28 @@
 #ifndef QEMU_KVM_I386_H
 #define QEMU_KVM_I386_H
 
+#include "kvm.h"
+
 bool kvm_allows_irq0_override(void);
 
+int kvm_device_pci_assign(KVMState *s, PCIHostDeviceAddress *dev_addr,
+                          uint32_t flags, uint32_t *dev_id);
+int kvm_device_pci_deassign(KVMState *s, uint32_t dev_id);
+
+int kvm_device_intx_assign(KVMState *s, uint32_t dev_id,
+                           bool use_host_msi, uint32_t guest_irq);
+int kvm_device_intx_set_mask(KVMState *s, uint32_t dev_id, bool masked);
+int kvm_device_intx_deassign(KVMState *s, uint32_t dev_id, bool use_host_msi);
+
+int kvm_device_msi_assign(KVMState *s, uint32_t dev_id, int virq);
+int kvm_device_msi_deassign(KVMState *s, uint32_t dev_id);
+
+bool kvm_device_msix_supported(KVMState *s);
+int kvm_device_msix_init_vectors(KVMState *s, uint32_t dev_id,
+                                 uint32_t nr_vectors);
+int kvm_device_msix_set_vector(KVMState *s, uint32_t dev_id, uint32_t vector,
+                               int virq);
+int kvm_device_msix_assign(KVMState *s, uint32_t dev_id);
+int kvm_device_msix_deassign(KVMState *s, uint32_t dev_id);
+
 #endif
diff --git a/target-i386/seg_helper.c b/target-i386/seg_helper.c
index 5fff8d59c..ff9337441 100644
--- a/target-i386/seg_helper.c
+++ b/target-i386/seg_helper.c
@@ -31,7 +31,7 @@
 #ifdef DEBUG_PCALL
 # define LOG_PCALL(...) qemu_log_mask(CPU_LOG_PCALL, ## __VA_ARGS__)
 # define LOG_PCALL_STATE(env)                                  \
-    log_cpu_state_mask(CPU_LOG_PCALL, (env), X86_DUMP_CCOP)
+    log_cpu_state_mask(CPU_LOG_PCALL, (env), CPU_DUMP_CCOP)
 #else
 # define LOG_PCALL(...) do { } while (0)
 # define LOG_PCALL_STATE(env) do { } while (0)
@@ -1177,7 +1177,7 @@ static void do_interrupt_all(CPUX86State *env, int intno, int is_int,
                 qemu_log(" EAX=" TARGET_FMT_lx, EAX);
             }
             qemu_log("\n");
-            log_cpu_state(env, X86_DUMP_CCOP);
+            log_cpu_state(env, CPU_DUMP_CCOP);
 #if 0
             {
                 int i;
diff --git a/target-i386/smm_helper.c b/target-i386/smm_helper.c
index 8b04eb2e0..eea2fe978 100644
--- a/target-i386/smm_helper.c
+++ b/target-i386/smm_helper.c
@@ -47,7 +47,7 @@ void do_smm_enter(CPUX86State *env)
     int i, offset;
 
     qemu_log_mask(CPU_LOG_INT, "SMM: enter\n");
-    log_cpu_state_mask(CPU_LOG_INT, env, X86_DUMP_CCOP);
+    log_cpu_state_mask(CPU_LOG_INT, env, CPU_DUMP_CCOP);
 
     env->hflags |= HF_SMM_MASK;
     cpu_smm_update(env);
@@ -295,7 +295,7 @@ void helper_rsm(CPUX86State *env)
     cpu_smm_update(env);
 
     qemu_log_mask(CPU_LOG_INT, "SMM: after RSM\n");
-    log_cpu_state_mask(CPU_LOG_INT, env, X86_DUMP_CCOP);
+    log_cpu_state_mask(CPU_LOG_INT, env, CPU_DUMP_CCOP);
 }
 
 #endif /* !CONFIG_USER_ONLY */
diff --git a/target-i386/svm_helper.c b/target-i386/svm_helper.c
index 4943c37fe..a238d95a5 100644
--- a/target-i386/svm_helper.c
+++ b/target-i386/svm_helper.c
@@ -85,7 +85,7 @@ void helper_svm_check_io(CPUX86State *env, uint32_t port, uint32_t param,
 }
 #else
 
-static inline void svm_save_seg(CPUX86State *env, target_phys_addr_t addr,
+static inline void svm_save_seg(CPUX86State *env, hwaddr addr,
                                 const SegmentCache *sc)
 {
     stw_phys(addr + offsetof(struct vmcb_seg, selector),
@@ -98,7 +98,7 @@ static inline void svm_save_seg(CPUX86State *env, target_phys_addr_t addr,
              ((sc->flags >> 8) & 0xff) | ((sc->flags >> 12) & 0x0f00));
 }
 
-static inline void svm_load_seg(CPUX86State *env, target_phys_addr_t addr,
+static inline void svm_load_seg(CPUX86State *env, hwaddr addr,
                                 SegmentCache *sc)
 {
     unsigned int flags;
@@ -110,7 +110,7 @@ static inline void svm_load_seg(CPUX86State *env, target_phys_addr_t addr,
     sc->flags = ((flags & 0xff) << 8) | ((flags & 0x0f00) << 12);
 }
 
-static inline void svm_load_seg_cache(CPUX86State *env, target_phys_addr_t addr,
+static inline void svm_load_seg_cache(CPUX86State *env, hwaddr addr,
                                       int seg_reg)
 {
     SegmentCache sc1, *sc = &sc1;
diff --git a/target-i386/translate.c b/target-i386/translate.c
index eb0cabcf1..8e676ba1a 100644
--- a/target-i386/translate.c
+++ b/target-i386/translate.c
@@ -107,6 +107,7 @@ typedef struct DisasContext {
     int cpuid_ext_features;
     int cpuid_ext2_features;
     int cpuid_ext3_features;
+    int cpuid_7_0_ebx_features;
 } DisasContext;
 
 static void gen_eob(DisasContext *s);
@@ -2017,7 +2018,8 @@ static void gen_shifti(DisasContext *s1, int op, int ot, int d, int c)
     }
 }
 
-static void gen_lea_modrm(DisasContext *s, int modrm, int *reg_ptr, int *offset_ptr)
+static void gen_lea_modrm(CPUX86State *env, DisasContext *s, int modrm,
+                          int *reg_ptr, int *offset_ptr)
 {
     target_long disp;
     int havesib;
@@ -2043,7 +2045,7 @@ static void gen_lea_modrm(DisasContext *s, int modrm, int *reg_ptr, int *offset_
 
         if (base == 4) {
             havesib = 1;
-            code = cpu_ldub_code(cpu_single_env, s->pc++);
+            code = cpu_ldub_code(env, s->pc++);
             scale = (code >> 6) & 3;
             index = ((code >> 3) & 7) | REX_X(s);
             base = (code & 7);
@@ -2054,7 +2056,7 @@ static void gen_lea_modrm(DisasContext *s, int modrm, int *reg_ptr, int *offset_
         case 0:
             if ((base & 7) == 5) {
                 base = -1;
-                disp = (int32_t)cpu_ldl_code(cpu_single_env, s->pc);
+                disp = (int32_t)cpu_ldl_code(env, s->pc);
                 s->pc += 4;
                 if (CODE64(s) && !havesib) {
                     disp += s->pc + s->rip_offset;
@@ -2064,11 +2066,11 @@ static void gen_lea_modrm(DisasContext *s, int modrm, int *reg_ptr, int *offset_
             }
             break;
         case 1:
-            disp = (int8_t)cpu_ldub_code(cpu_single_env, s->pc++);
+            disp = (int8_t)cpu_ldub_code(env, s->pc++);
             break;
         default:
         case 2:
-            disp = (int32_t)cpu_ldl_code(cpu_single_env, s->pc);
+            disp = (int32_t)cpu_ldl_code(env, s->pc);
             s->pc += 4;
             break;
         }
@@ -2131,7 +2133,7 @@ static void gen_lea_modrm(DisasContext *s, int modrm, int *reg_ptr, int *offset_
         switch (mod) {
         case 0:
             if (rm == 6) {
-                disp = cpu_lduw_code(cpu_single_env, s->pc);
+                disp = cpu_lduw_code(env, s->pc);
                 s->pc += 2;
                 gen_op_movl_A0_im(disp);
                 rm = 0; /* avoid SS override */
@@ -2141,11 +2143,11 @@ static void gen_lea_modrm(DisasContext *s, int modrm, int *reg_ptr, int *offset_
             }
             break;
         case 1:
-            disp = (int8_t)cpu_ldub_code(cpu_single_env, s->pc++);
+            disp = (int8_t)cpu_ldub_code(env, s->pc++);
             break;
         default:
         case 2:
-            disp = cpu_lduw_code(cpu_single_env, s->pc);
+            disp = cpu_lduw_code(env, s->pc);
             s->pc += 2;
             break;
         }
@@ -2201,7 +2203,7 @@ static void gen_lea_modrm(DisasContext *s, int modrm, int *reg_ptr, int *offset_
     *offset_ptr = disp;
 }
 
-static void gen_nop_modrm(DisasContext *s, int modrm)
+static void gen_nop_modrm(CPUX86State *env, DisasContext *s, int modrm)
 {
     int mod, rm, base, code;
 
@@ -2215,7 +2217,7 @@ static void gen_nop_modrm(DisasContext *s, int modrm)
         base = rm;
 
         if (base == 4) {
-            code = cpu_ldub_code(cpu_single_env, s->pc++);
+            code = cpu_ldub_code(env, s->pc++);
             base = (code & 7);
         }
 
@@ -2275,7 +2277,8 @@ static void gen_add_A0_ds_seg(DisasContext *s)
 
 /* generate modrm memory load or store of 'reg'. TMP0 is used if reg ==
    OR_TMP0 */
-static void gen_ldst_modrm(DisasContext *s, int modrm, int ot, int reg, int is_store)
+static void gen_ldst_modrm(CPUX86State *env, DisasContext *s, int modrm,
+                           int ot, int reg, int is_store)
 {
     int mod, rm, opreg, disp;
 
@@ -2292,7 +2295,7 @@ static void gen_ldst_modrm(DisasContext *s, int modrm, int ot, int reg, int is_s
                 gen_op_mov_reg_T0(ot, reg);
         }
     } else {
-        gen_lea_modrm(s, modrm, &opreg, &disp);
+        gen_lea_modrm(env, s, modrm, &opreg, &disp);
         if (is_store) {
             if (reg != OR_TMP0)
                 gen_op_mov_TN_reg(ot, 0, reg);
@@ -2305,22 +2308,22 @@ static void gen_ldst_modrm(DisasContext *s, int modrm, int ot, int reg, int is_s
     }
 }
 
-static inline uint32_t insn_get(DisasContext *s, int ot)
+static inline uint32_t insn_get(CPUX86State *env, DisasContext *s, int ot)
 {
     uint32_t ret;
 
     switch(ot) {
     case OT_BYTE:
-        ret = cpu_ldub_code(cpu_single_env, s->pc);
+        ret = cpu_ldub_code(env, s->pc);
         s->pc++;
         break;
     case OT_WORD:
-        ret = cpu_lduw_code(cpu_single_env, s->pc);
+        ret = cpu_lduw_code(env, s->pc);
         s->pc += 2;
         break;
     default:
     case OT_LONG:
-        ret = cpu_ldl_code(cpu_single_env, s->pc);
+        ret = cpu_ldl_code(env, s->pc);
         s->pc += 4;
         break;
     }
@@ -3166,7 +3169,8 @@ static const struct SSEOpHelper_eppi sse_op_table7[256] = {
     [0x63] = SSE42_OP(pcmpistri),
 };
 
-static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
+static void gen_sse(CPUX86State *env, DisasContext *s, int b,
+                    target_ulong pc_start, int rex_r)
 {
     int b1, op1_offset, op2_offset, is_xmm, val, ot;
     int modrm, mod, rm, reg, reg_addr, offset_addr;
@@ -3229,7 +3233,7 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
         gen_helper_enter_mmx(cpu_env);
     }
 
-    modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+    modrm = cpu_ldub_code(env, s->pc++);
     reg = ((modrm >> 3) & 7);
     if (is_xmm)
         reg |= rex_r;
@@ -3240,7 +3244,7 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
         case 0x0e7: /* movntq */
             if (mod == 3)
                 goto illegal_op;
-            gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+            gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
             gen_stq_env_A0(s->mem_index, offsetof(CPUX86State,fpregs[reg].mmx));
             break;
         case 0x1e7: /* movntdq */
@@ -3248,20 +3252,20 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
         case 0x12b: /* movntps */
             if (mod == 3)
                 goto illegal_op;
-            gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+            gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
             gen_sto_env_A0(s->mem_index, offsetof(CPUX86State,xmm_regs[reg]));
             break;
         case 0x3f0: /* lddqu */
             if (mod == 3)
                 goto illegal_op;
-            gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+            gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
             gen_ldo_env_A0(s->mem_index, offsetof(CPUX86State,xmm_regs[reg]));
             break;
         case 0x22b: /* movntss */
         case 0x32b: /* movntsd */
             if (mod == 3)
                 goto illegal_op;
-            gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+            gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
             if (b1 & 1) {
                 gen_stq_env_A0(s->mem_index, offsetof(CPUX86State,
                     xmm_regs[reg]));
@@ -3274,12 +3278,12 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
         case 0x6e: /* movd mm, ea */
 #ifdef TARGET_X86_64
             if (s->dflag == 2) {
-                gen_ldst_modrm(s, modrm, OT_QUAD, OR_TMP0, 0);
+                gen_ldst_modrm(env, s, modrm, OT_QUAD, OR_TMP0, 0);
                 tcg_gen_st_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,fpregs[reg].mmx));
             } else
 #endif
             {
-                gen_ldst_modrm(s, modrm, OT_LONG, OR_TMP0, 0);
+                gen_ldst_modrm(env, s, modrm, OT_LONG, OR_TMP0, 0);
                 tcg_gen_addi_ptr(cpu_ptr0, cpu_env, 
                                  offsetof(CPUX86State,fpregs[reg].mmx));
                 tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
@@ -3289,14 +3293,14 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
         case 0x16e: /* movd xmm, ea */
 #ifdef TARGET_X86_64
             if (s->dflag == 2) {
-                gen_ldst_modrm(s, modrm, OT_QUAD, OR_TMP0, 0);
+                gen_ldst_modrm(env, s, modrm, OT_QUAD, OR_TMP0, 0);
                 tcg_gen_addi_ptr(cpu_ptr0, cpu_env, 
                                  offsetof(CPUX86State,xmm_regs[reg]));
                 gen_helper_movq_mm_T0_xmm(cpu_ptr0, cpu_T[0]);
             } else
 #endif
             {
-                gen_ldst_modrm(s, modrm, OT_LONG, OR_TMP0, 0);
+                gen_ldst_modrm(env, s, modrm, OT_LONG, OR_TMP0, 0);
                 tcg_gen_addi_ptr(cpu_ptr0, cpu_env, 
                                  offsetof(CPUX86State,xmm_regs[reg]));
                 tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
@@ -3305,7 +3309,7 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
             break;
         case 0x6f: /* movq mm, ea */
             if (mod != 3) {
-                gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
                 gen_ldq_env_A0(s->mem_index, offsetof(CPUX86State,fpregs[reg].mmx));
             } else {
                 rm = (modrm & 7);
@@ -3322,7 +3326,7 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
         case 0x16f: /* movdqa xmm, ea */
         case 0x26f: /* movdqu xmm, ea */
             if (mod != 3) {
-                gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
                 gen_ldo_env_A0(s->mem_index, offsetof(CPUX86State,xmm_regs[reg]));
             } else {
                 rm = (modrm & 7) | REX_B(s);
@@ -3332,7 +3336,7 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
             break;
         case 0x210: /* movss xmm, ea */
             if (mod != 3) {
-                gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
                 gen_op_ld_T0_A0(OT_LONG + s->mem_index);
                 tcg_gen_st32_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,xmm_regs[reg].XMM_L(0)));
                 gen_op_movl_T0_0();
@@ -3347,7 +3351,7 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
             break;
         case 0x310: /* movsd xmm, ea */
             if (mod != 3) {
-                gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
                 gen_ldq_env_A0(s->mem_index, offsetof(CPUX86State,xmm_regs[reg].XMM_Q(0)));
                 gen_op_movl_T0_0();
                 tcg_gen_st32_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,xmm_regs[reg].XMM_L(2)));
@@ -3361,7 +3365,7 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
         case 0x012: /* movlps */
         case 0x112: /* movlpd */
             if (mod != 3) {
-                gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
                 gen_ldq_env_A0(s->mem_index, offsetof(CPUX86State,xmm_regs[reg].XMM_Q(0)));
             } else {
                 /* movhlps */
@@ -3372,7 +3376,7 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
             break;
         case 0x212: /* movsldup */
             if (mod != 3) {
-                gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
                 gen_ldo_env_A0(s->mem_index, offsetof(CPUX86State,xmm_regs[reg]));
             } else {
                 rm = (modrm & 7) | REX_B(s);
@@ -3388,7 +3392,7 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
             break;
         case 0x312: /* movddup */
             if (mod != 3) {
-                gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
                 gen_ldq_env_A0(s->mem_index, offsetof(CPUX86State,xmm_regs[reg].XMM_Q(0)));
             } else {
                 rm = (modrm & 7) | REX_B(s);
@@ -3401,7 +3405,7 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
         case 0x016: /* movhps */
         case 0x116: /* movhpd */
             if (mod != 3) {
-                gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
                 gen_ldq_env_A0(s->mem_index, offsetof(CPUX86State,xmm_regs[reg].XMM_Q(1)));
             } else {
                 /* movlhps */
@@ -3412,7 +3416,7 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
             break;
         case 0x216: /* movshdup */
             if (mod != 3) {
-                gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
                 gen_ldo_env_A0(s->mem_index, offsetof(CPUX86State,xmm_regs[reg]));
             } else {
                 rm = (modrm & 7) | REX_B(s);
@@ -3433,8 +3437,8 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
 
                 if (b1 == 1 && reg != 0)
                     goto illegal_op;
-                field_length = cpu_ldub_code(cpu_single_env, s->pc++) & 0x3F;
-                bit_index = cpu_ldub_code(cpu_single_env, s->pc++) & 0x3F;
+                field_length = cpu_ldub_code(env, s->pc++) & 0x3F;
+                bit_index = cpu_ldub_code(env, s->pc++) & 0x3F;
                 tcg_gen_addi_ptr(cpu_ptr0, cpu_env,
                     offsetof(CPUX86State,xmm_regs[reg]));
                 if (b1 == 1)
@@ -3452,13 +3456,13 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
             if (s->dflag == 2) {
                 tcg_gen_ld_i64(cpu_T[0], cpu_env, 
                                offsetof(CPUX86State,fpregs[reg].mmx));
-                gen_ldst_modrm(s, modrm, OT_QUAD, OR_TMP0, 1);
+                gen_ldst_modrm(env, s, modrm, OT_QUAD, OR_TMP0, 1);
             } else
 #endif
             {
                 tcg_gen_ld32u_tl(cpu_T[0], cpu_env, 
                                  offsetof(CPUX86State,fpregs[reg].mmx.MMX_L(0)));
-                gen_ldst_modrm(s, modrm, OT_LONG, OR_TMP0, 1);
+                gen_ldst_modrm(env, s, modrm, OT_LONG, OR_TMP0, 1);
             }
             break;
         case 0x17e: /* movd ea, xmm */
@@ -3466,18 +3470,18 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
             if (s->dflag == 2) {
                 tcg_gen_ld_i64(cpu_T[0], cpu_env, 
                                offsetof(CPUX86State,xmm_regs[reg].XMM_Q(0)));
-                gen_ldst_modrm(s, modrm, OT_QUAD, OR_TMP0, 1);
+                gen_ldst_modrm(env, s, modrm, OT_QUAD, OR_TMP0, 1);
             } else
 #endif
             {
                 tcg_gen_ld32u_tl(cpu_T[0], cpu_env, 
                                  offsetof(CPUX86State,xmm_regs[reg].XMM_L(0)));
-                gen_ldst_modrm(s, modrm, OT_LONG, OR_TMP0, 1);
+                gen_ldst_modrm(env, s, modrm, OT_LONG, OR_TMP0, 1);
             }
             break;
         case 0x27e: /* movq xmm, ea */
             if (mod != 3) {
-                gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
                 gen_ldq_env_A0(s->mem_index, offsetof(CPUX86State,xmm_regs[reg].XMM_Q(0)));
             } else {
                 rm = (modrm & 7) | REX_B(s);
@@ -3488,7 +3492,7 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
             break;
         case 0x7f: /* movq ea, mm */
             if (mod != 3) {
-                gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
                 gen_stq_env_A0(s->mem_index, offsetof(CPUX86State,fpregs[reg].mmx));
             } else {
                 rm = (modrm & 7);
@@ -3503,7 +3507,7 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
         case 0x17f: /* movdqa ea, xmm */
         case 0x27f: /* movdqu ea, xmm */
             if (mod != 3) {
-                gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
                 gen_sto_env_A0(s->mem_index, offsetof(CPUX86State,xmm_regs[reg]));
             } else {
                 rm = (modrm & 7) | REX_B(s);
@@ -3513,7 +3517,7 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
             break;
         case 0x211: /* movss ea, xmm */
             if (mod != 3) {
-                gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
                 tcg_gen_ld32u_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,xmm_regs[reg].XMM_L(0)));
                 gen_op_st_T0_A0(OT_LONG + s->mem_index);
             } else {
@@ -3524,7 +3528,7 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
             break;
         case 0x311: /* movsd ea, xmm */
             if (mod != 3) {
-                gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
                 gen_stq_env_A0(s->mem_index, offsetof(CPUX86State,xmm_regs[reg].XMM_Q(0)));
             } else {
                 rm = (modrm & 7) | REX_B(s);
@@ -3535,7 +3539,7 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
         case 0x013: /* movlps */
         case 0x113: /* movlpd */
             if (mod != 3) {
-                gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
                 gen_stq_env_A0(s->mem_index, offsetof(CPUX86State,xmm_regs[reg].XMM_Q(0)));
             } else {
                 goto illegal_op;
@@ -3544,7 +3548,7 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
         case 0x017: /* movhps */
         case 0x117: /* movhpd */
             if (mod != 3) {
-                gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
                 gen_stq_env_A0(s->mem_index, offsetof(CPUX86State,xmm_regs[reg].XMM_Q(1)));
             } else {
                 goto illegal_op;
@@ -3559,7 +3563,7 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
             if (b1 >= 2) {
 	        goto illegal_op;
             }
-            val = cpu_ldub_code(cpu_single_env, s->pc++);
+            val = cpu_ldub_code(env, s->pc++);
             if (is_xmm) {
                 gen_op_movl_T0_im(val);
                 tcg_gen_st32_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,xmm_t0.XMM_L(0)));
@@ -3609,7 +3613,7 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
         case 0x12a: /* cvtpi2pd */
             gen_helper_enter_mmx(cpu_env);
             if (mod != 3) {
-                gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
                 op2_offset = offsetof(CPUX86State,mmx_t0);
                 gen_ldq_env_A0(s->mem_index, op2_offset);
             } else {
@@ -3632,7 +3636,7 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
         case 0x22a: /* cvtsi2ss */
         case 0x32a: /* cvtsi2sd */
             ot = (s->dflag == 2) ? OT_QUAD : OT_LONG;
-            gen_ldst_modrm(s, modrm, ot, OR_TMP0, 0);
+            gen_ldst_modrm(env, s, modrm, ot, OR_TMP0, 0);
             op1_offset = offsetof(CPUX86State,xmm_regs[reg]);
             tcg_gen_addi_ptr(cpu_ptr0, cpu_env, op1_offset);
             if (ot == OT_LONG) {
@@ -3654,7 +3658,7 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
         case 0x12d: /* cvtpd2pi */
             gen_helper_enter_mmx(cpu_env);
             if (mod != 3) {
-                gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
                 op2_offset = offsetof(CPUX86State,xmm_t0);
                 gen_ldo_env_A0(s->mem_index, op2_offset);
             } else {
@@ -3685,7 +3689,7 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
         case 0x32d: /* cvtsd2si */
             ot = (s->dflag == 2) ? OT_QUAD : OT_LONG;
             if (mod != 3) {
-                gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
                 if ((b >> 8) & 1) {
                     gen_ldq_env_A0(s->mem_index, offsetof(CPUX86State,xmm_t0.XMM_Q(0)));
                 } else {
@@ -3717,8 +3721,8 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
         case 0xc4: /* pinsrw */
         case 0x1c4:
             s->rip_offset = 1;
-            gen_ldst_modrm(s, modrm, OT_WORD, OR_TMP0, 0);
-            val = cpu_ldub_code(cpu_single_env, s->pc++);
+            gen_ldst_modrm(env, s, modrm, OT_WORD, OR_TMP0, 0);
+            val = cpu_ldub_code(env, s->pc++);
             if (b1) {
                 val &= 7;
                 tcg_gen_st16_tl(cpu_T[0], cpu_env,
@@ -3734,7 +3738,7 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
             if (mod != 3)
                 goto illegal_op;
             ot = (s->dflag == 2) ? OT_QUAD : OT_LONG;
-            val = cpu_ldub_code(cpu_single_env, s->pc++);
+            val = cpu_ldub_code(env, s->pc++);
             if (b1) {
                 val &= 7;
                 rm = (modrm & 7) | REX_B(s);
@@ -3751,7 +3755,7 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
             break;
         case 0x1d6: /* movq ea, xmm */
             if (mod != 3) {
-                gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
                 gen_stq_env_A0(s->mem_index, offsetof(CPUX86State,xmm_regs[reg].XMM_Q(0)));
             } else {
                 rm = (modrm & 7) | REX_B(s);
@@ -3795,7 +3799,7 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
                 goto crc32;
         case 0x038:
             b = modrm;
-            modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+            modrm = cpu_ldub_code(env, s->pc++);
             rm = modrm & 7;
             reg = ((modrm >> 3) & 7) | rex_r;
             mod = (modrm >> 6) & 3;
@@ -3816,7 +3820,7 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
                     op2_offset = offsetof(CPUX86State,xmm_regs[rm | REX_B(s)]);
                 } else {
                     op2_offset = offsetof(CPUX86State,xmm_t0);
-                    gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                    gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
                     switch (b) {
                     case 0x20: case 0x30: /* pmovsxbw, pmovzxbw */
                     case 0x23: case 0x33: /* pmovsxwd, pmovzxwd */
@@ -3851,7 +3855,7 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
                     op2_offset = offsetof(CPUX86State,fpregs[rm].mmx);
                 } else {
                     op2_offset = offsetof(CPUX86State,mmx_t0);
-                    gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                    gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
                     gen_ldq_env_A0(s->mem_index, op2_offset);
                 }
             }
@@ -3869,7 +3873,7 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
         case 0x338: /* crc32 */
         crc32:
             b = modrm;
-            modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+            modrm = cpu_ldub_code(env, s->pc++);
             reg = ((modrm >> 3) & 7) | rex_r;
 
             if (b != 0xf0 && b != 0xf1)
@@ -3889,7 +3893,7 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
 
             gen_op_mov_TN_reg(OT_LONG, 0, reg);
             tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
-            gen_ldst_modrm(s, modrm, ot, OR_TMP0, 0);
+            gen_ldst_modrm(env, s, modrm, ot, OR_TMP0, 0);
             gen_helper_crc32(cpu_T[0], cpu_tmp2_i32,
                              cpu_T[0], tcg_const_i32(8 << ot));
 
@@ -3899,7 +3903,7 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
         case 0x03a:
         case 0x13a:
             b = modrm;
-            modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+            modrm = cpu_ldub_code(env, s->pc++);
             rm = modrm & 7;
             reg = ((modrm >> 3) & 7) | rex_r;
             mod = (modrm >> 6) & 3;
@@ -3918,9 +3922,9 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
                 ot = (s->dflag == 2) ? OT_QUAD : OT_LONG;
                 rm = (modrm & 7) | REX_B(s);
                 if (mod != 3)
-                    gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                    gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
                 reg = ((modrm >> 3) & 7) | rex_r;
-                val = cpu_ldub_code(cpu_single_env, s->pc++);
+                val = cpu_ldub_code(env, s->pc++);
                 switch (b) {
                 case 0x14: /* pextrb */
                     tcg_gen_ld8u_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,
@@ -4050,7 +4054,7 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
                     op2_offset = offsetof(CPUX86State,xmm_regs[rm | REX_B(s)]);
                 } else {
                     op2_offset = offsetof(CPUX86State,xmm_t0);
-                    gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                    gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
                     gen_ldo_env_A0(s->mem_index, op2_offset);
                 }
             } else {
@@ -4059,11 +4063,11 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
                     op2_offset = offsetof(CPUX86State,fpregs[rm].mmx);
                 } else {
                     op2_offset = offsetof(CPUX86State,mmx_t0);
-                    gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                    gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
                     gen_ldq_env_A0(s->mem_index, op2_offset);
                 }
             }
-            val = cpu_ldub_code(cpu_single_env, s->pc++);
+            val = cpu_ldub_code(env, s->pc++);
 
             if ((b & 0xfc) == 0x60) { /* pcmpXstrX */
                 s->cc_op = CC_OP_EFLAGS;
@@ -4094,7 +4098,7 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
         if (is_xmm) {
             op1_offset = offsetof(CPUX86State,xmm_regs[reg]);
             if (mod != 3) {
-                gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
                 op2_offset = offsetof(CPUX86State,xmm_t0);
                 if (b1 >= 2 && ((b >= 0x50 && b <= 0x5f && b != 0x5b) ||
                                 b == 0xc2)) {
@@ -4117,7 +4121,7 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
         } else {
             op1_offset = offsetof(CPUX86State,fpregs[reg].mmx);
             if (mod != 3) {
-                gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
                 op2_offset = offsetof(CPUX86State,mmx_t0);
                 gen_ldq_env_A0(s->mem_index, op2_offset);
             } else {
@@ -4129,7 +4133,7 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
         case 0x0f: /* 3DNow! data insns */
             if (!(s->cpuid_ext2_features & CPUID_EXT2_3DNOW))
                 goto illegal_op;
-            val = cpu_ldub_code(cpu_single_env, s->pc++);
+            val = cpu_ldub_code(env, s->pc++);
             sse_fn_epp = sse_op_table5[val];
             if (!sse_fn_epp) {
                 goto illegal_op;
@@ -4140,7 +4144,7 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
             break;
         case 0x70: /* pshufx insn */
         case 0xc6: /* pshufx insn */
-            val = cpu_ldub_code(cpu_single_env, s->pc++);
+            val = cpu_ldub_code(env, s->pc++);
             tcg_gen_addi_ptr(cpu_ptr0, cpu_env, op1_offset);
             tcg_gen_addi_ptr(cpu_ptr1, cpu_env, op2_offset);
             /* XXX: introduce a new table? */
@@ -4149,7 +4153,7 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
             break;
         case 0xc2:
             /* compare insns */
-            val = cpu_ldub_code(cpu_single_env, s->pc++);
+            val = cpu_ldub_code(env, s->pc++);
             if (val >= 8)
                 goto illegal_op;
             sse_fn_epp = sse_op_table4[val][b1];
@@ -4194,7 +4198,8 @@ static void gen_sse(DisasContext *s, int b, target_ulong pc_start, int rex_r)
 
 /* convert one instruction. s->is_jmp is set if the translation must
    be stopped. Return the next pc value */
-static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
+static target_ulong disas_insn(CPUX86State *env, DisasContext *s,
+                               target_ulong pc_start)
 {
     int b, prefixes, aflag, dflag;
     int shift, ot;
@@ -4202,8 +4207,9 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
     target_ulong next_eip, tval;
     int rex_w, rex_r;
 
-    if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP)))
+    if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
         tcg_gen_debug_insn_start(pc_start);
+    }
     s->pc = pc_start;
     prefixes = 0;
     aflag = s->code32;
@@ -4218,7 +4224,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
 #endif
     s->rip_offset = 0; /* for relative ip address */
  next_byte:
-    b = cpu_ldub_code(cpu_single_env, s->pc);
+    b = cpu_ldub_code(env, s->pc);
     s->pc++;
     /* check prefixes */
 #ifdef TARGET_X86_64
@@ -4333,7 +4339,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
     case 0x0f:
         /**************************/
         /* extended op code */
-        b = cpu_ldub_code(cpu_single_env, s->pc++) | 0x100;
+        b = cpu_ldub_code(env, s->pc++) | 0x100;
         goto reswitch;
 
         /**************************/
@@ -4358,12 +4364,12 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
 
             switch(f) {
             case 0: /* OP Ev, Gv */
-                modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+                modrm = cpu_ldub_code(env, s->pc++);
                 reg = ((modrm >> 3) & 7) | rex_r;
                 mod = (modrm >> 6) & 3;
                 rm = (modrm & 7) | REX_B(s);
                 if (mod != 3) {
-                    gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                    gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
                     opreg = OR_TMP0;
                 } else if (op == OP_XORL && rm == reg) {
                 xor_zero:
@@ -4380,12 +4386,12 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
                 gen_op(s, op, ot, opreg);
                 break;
             case 1: /* OP Gv, Ev */
-                modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+                modrm = cpu_ldub_code(env, s->pc++);
                 mod = (modrm >> 6) & 3;
                 reg = ((modrm >> 3) & 7) | rex_r;
                 rm = (modrm & 7) | REX_B(s);
                 if (mod != 3) {
-                    gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                    gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
                     gen_op_ld_T1_A0(ot + s->mem_index);
                 } else if (op == OP_XORL && rm == reg) {
                     goto xor_zero;
@@ -4395,7 +4401,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
                 gen_op(s, op, ot, reg);
                 break;
             case 2: /* OP A, Iv */
-                val = insn_get(s, ot);
+                val = insn_get(env, s, ot);
                 gen_op_movl_T1_im(val);
                 gen_op(s, op, ot, OR_EAX);
                 break;
@@ -4417,7 +4423,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
             else
                 ot = dflag + OT_WORD;
 
-            modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+            modrm = cpu_ldub_code(env, s->pc++);
             mod = (modrm >> 6) & 3;
             rm = (modrm & 7) | REX_B(s);
             op = (modrm >> 3) & 7;
@@ -4427,7 +4433,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
                     s->rip_offset = 1;
                 else
                     s->rip_offset = insn_const_size(ot);
-                gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
                 opreg = OR_TMP0;
             } else {
                 opreg = rm;
@@ -4438,10 +4444,10 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
             case 0x80:
             case 0x81:
             case 0x82:
-                val = insn_get(s, ot);
+                val = insn_get(env, s, ot);
                 break;
             case 0x83:
-                val = (int8_t)insn_get(s, OT_BYTE);
+                val = (int8_t)insn_get(env, s, OT_BYTE);
                 break;
             }
             gen_op_movl_T1_im(val);
@@ -4466,14 +4472,14 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         else
             ot = dflag + OT_WORD;
 
-        modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+        modrm = cpu_ldub_code(env, s->pc++);
         mod = (modrm >> 6) & 3;
         rm = (modrm & 7) | REX_B(s);
         op = (modrm >> 3) & 7;
         if (mod != 3) {
             if (op == 0)
                 s->rip_offset = insn_const_size(ot);
-            gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+            gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
             gen_op_ld_T0_A0(ot + s->mem_index);
         } else {
             gen_op_mov_TN_reg(ot, 0, rm);
@@ -4481,7 +4487,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
 
         switch(op) {
         case 0: /* test */
-            val = insn_get(s, ot);
+            val = insn_get(env, s, ot);
             gen_op_movl_T1_im(val);
             gen_op_testl_T0_T1_cc();
             s->cc_op = CC_OP_LOGICB + ot;
@@ -4698,7 +4704,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         else
             ot = dflag + OT_WORD;
 
-        modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+        modrm = cpu_ldub_code(env, s->pc++);
         mod = (modrm >> 6) & 3;
         rm = (modrm & 7) | REX_B(s);
         op = (modrm >> 3) & 7;
@@ -4717,7 +4723,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
             }
         }
         if (mod != 3) {
-            gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+            gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
             if (op >= 2 && op != 3 && op != 5)
                 gen_op_ld_T0_A0(ot + s->mem_index);
         } else {
@@ -4810,10 +4816,10 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         else
             ot = dflag + OT_WORD;
 
-        modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+        modrm = cpu_ldub_code(env, s->pc++);
         reg = ((modrm >> 3) & 7) | rex_r;
 
-        gen_ldst_modrm(s, modrm, ot, OR_TMP0, 0);
+        gen_ldst_modrm(env, s, modrm, ot, OR_TMP0, 0);
         gen_op_mov_TN_reg(ot, 1, reg);
         gen_op_testl_T0_T1_cc();
         s->cc_op = CC_OP_LOGICB + ot;
@@ -4825,7 +4831,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
             ot = OT_BYTE;
         else
             ot = dflag + OT_WORD;
-        val = insn_get(s, ot);
+        val = insn_get(env, s, ot);
 
         gen_op_mov_TN_reg(ot, 0, OR_EAX);
         gen_op_movl_T1_im(val);
@@ -4875,18 +4881,18 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
     case 0x69: /* imul Gv, Ev, I */
     case 0x6b:
         ot = dflag + OT_WORD;
-        modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+        modrm = cpu_ldub_code(env, s->pc++);
         reg = ((modrm >> 3) & 7) | rex_r;
         if (b == 0x69)
             s->rip_offset = insn_const_size(ot);
         else if (b == 0x6b)
             s->rip_offset = 1;
-        gen_ldst_modrm(s, modrm, ot, OR_TMP0, 0);
+        gen_ldst_modrm(env, s, modrm, ot, OR_TMP0, 0);
         if (b == 0x69) {
-            val = insn_get(s, ot);
+            val = insn_get(env, s, ot);
             gen_op_movl_T1_im(val);
         } else if (b == 0x6b) {
-            val = (int8_t)insn_get(s, OT_BYTE);
+            val = (int8_t)insn_get(env, s, OT_BYTE);
             gen_op_movl_T1_im(val);
         } else {
             gen_op_mov_TN_reg(ot, 1, reg);
@@ -4939,7 +4945,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
             ot = OT_BYTE;
         else
             ot = dflag + OT_WORD;
-        modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+        modrm = cpu_ldub_code(env, s->pc++);
         reg = ((modrm >> 3) & 7) | rex_r;
         mod = (modrm >> 6) & 3;
         if (mod == 3) {
@@ -4950,7 +4956,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
             gen_op_mov_reg_T1(ot, reg);
             gen_op_mov_reg_T0(ot, rm);
         } else {
-            gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+            gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
             gen_op_mov_TN_reg(ot, 0, reg);
             gen_op_ld_T1_A0(ot + s->mem_index);
             gen_op_addl_T0_T1();
@@ -4970,7 +4976,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
                 ot = OT_BYTE;
             else
                 ot = dflag + OT_WORD;
-            modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+            modrm = cpu_ldub_code(env, s->pc++);
             reg = ((modrm >> 3) & 7) | rex_r;
             mod = (modrm >> 6) & 3;
             t0 = tcg_temp_local_new();
@@ -4982,7 +4988,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
                 rm = (modrm & 7) | REX_B(s);
                 gen_op_mov_v_reg(ot, t0, rm);
             } else {
-                gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
                 tcg_gen_mov_tl(a0, cpu_A0);
                 gen_op_ld_v(ot + s->mem_index, t0, a0);
                 rm = 0; /* avoid warning */
@@ -5018,7 +5024,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         }
         break;
     case 0x1c7: /* cmpxchg8b */
-        modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+        modrm = cpu_ldub_code(env, s->pc++);
         mod = (modrm >> 6) & 3;
         if ((mod == 3) || ((modrm & 0x38) != 0x8))
             goto illegal_op;
@@ -5029,7 +5035,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
             gen_jmp_im(pc_start - s->cs_base);
             if (s->cc_op != CC_OP_DYNAMIC)
                 gen_op_set_cc_op(s->cc_op);
-            gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+            gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
             gen_helper_cmpxchg16b(cpu_env, cpu_A0);
         } else
 #endif        
@@ -5039,7 +5045,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
             gen_jmp_im(pc_start - s->cs_base);
             if (s->cc_op != CC_OP_DYNAMIC)
                 gen_op_set_cc_op(s->cc_op);
-            gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+            gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
             gen_helper_cmpxchg8b(cpu_env, cpu_A0);
         }
         s->cc_op = CC_OP_EFLAGS;
@@ -5080,9 +5086,9 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
             ot = dflag + OT_WORD;
         }
         if (b == 0x68)
-            val = insn_get(s, ot);
+            val = insn_get(env, s, ot);
         else
-            val = (int8_t)insn_get(s, OT_BYTE);
+            val = (int8_t)insn_get(env, s, OT_BYTE);
         gen_op_movl_T0_im(val);
         gen_push_T0(s);
         break;
@@ -5092,7 +5098,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         } else {
             ot = dflag + OT_WORD;
         }
-        modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+        modrm = cpu_ldub_code(env, s->pc++);
         mod = (modrm >> 6) & 3;
         gen_pop_T0(s);
         if (mod == 3) {
@@ -5103,7 +5109,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         } else {
             /* NOTE: order is important too for MMU exceptions */
             s->popl_esp_hack = 1 << ot;
-            gen_ldst_modrm(s, modrm, ot, OR_TMP0, 1);
+            gen_ldst_modrm(env, s, modrm, ot, OR_TMP0, 1);
             s->popl_esp_hack = 0;
             gen_pop_update(s);
         }
@@ -5111,9 +5117,9 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
     case 0xc8: /* enter */
         {
             int level;
-            val = cpu_lduw_code(cpu_single_env, s->pc);
+            val = cpu_lduw_code(env, s->pc);
             s->pc += 2;
-            level = cpu_ldub_code(cpu_single_env, s->pc++);
+            level = cpu_ldub_code(env, s->pc++);
             gen_enter(s, val, level);
         }
         break;
@@ -5193,11 +5199,11 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
             ot = OT_BYTE;
         else
             ot = dflag + OT_WORD;
-        modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+        modrm = cpu_ldub_code(env, s->pc++);
         reg = ((modrm >> 3) & 7) | rex_r;
 
         /* generate a generic store */
-        gen_ldst_modrm(s, modrm, ot, reg, 1);
+        gen_ldst_modrm(env, s, modrm, ot, reg, 1);
         break;
     case 0xc6:
     case 0xc7: /* mov Ev, Iv */
@@ -5205,13 +5211,13 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
             ot = OT_BYTE;
         else
             ot = dflag + OT_WORD;
-        modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+        modrm = cpu_ldub_code(env, s->pc++);
         mod = (modrm >> 6) & 3;
         if (mod != 3) {
             s->rip_offset = insn_const_size(ot);
-            gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+            gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
         }
-        val = insn_get(s, ot);
+        val = insn_get(env, s, ot);
         gen_op_movl_T0_im(val);
         if (mod != 3)
             gen_op_st_T0_A0(ot + s->mem_index);
@@ -5224,18 +5230,18 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
             ot = OT_BYTE;
         else
             ot = OT_WORD + dflag;
-        modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+        modrm = cpu_ldub_code(env, s->pc++);
         reg = ((modrm >> 3) & 7) | rex_r;
 
-        gen_ldst_modrm(s, modrm, ot, OR_TMP0, 0);
+        gen_ldst_modrm(env, s, modrm, ot, OR_TMP0, 0);
         gen_op_mov_reg_T0(ot, reg);
         break;
     case 0x8e: /* mov seg, Gv */
-        modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+        modrm = cpu_ldub_code(env, s->pc++);
         reg = (modrm >> 3) & 7;
         if (reg >= 6 || reg == R_CS)
             goto illegal_op;
-        gen_ldst_modrm(s, modrm, OT_WORD, OR_TMP0, 0);
+        gen_ldst_modrm(env, s, modrm, OT_WORD, OR_TMP0, 0);
         gen_movl_seg_T0(s, reg, pc_start - s->cs_base);
         if (reg == R_SS) {
             /* if reg == SS, inhibit interrupts/trace */
@@ -5251,7 +5257,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         }
         break;
     case 0x8c: /* mov Gv, seg */
-        modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+        modrm = cpu_ldub_code(env, s->pc++);
         reg = (modrm >> 3) & 7;
         mod = (modrm >> 6) & 3;
         if (reg >= 6)
@@ -5261,7 +5267,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
             ot = OT_WORD + dflag;
         else
             ot = OT_WORD;
-        gen_ldst_modrm(s, modrm, ot, OR_TMP0, 1);
+        gen_ldst_modrm(env, s, modrm, ot, OR_TMP0, 1);
         break;
 
     case 0x1b6: /* movzbS Gv, Eb */
@@ -5274,7 +5280,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
             d_ot = dflag + OT_WORD;
             /* ot is the size of source */
             ot = (b & 1) + OT_BYTE;
-            modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+            modrm = cpu_ldub_code(env, s->pc++);
             reg = ((modrm >> 3) & 7) | rex_r;
             mod = (modrm >> 6) & 3;
             rm = (modrm & 7) | REX_B(s);
@@ -5298,7 +5304,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
                 }
                 gen_op_mov_reg_T0(d_ot, reg);
             } else {
-                gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
                 if (b & 8) {
                     gen_op_lds_T0_A0(ot + s->mem_index);
                 } else {
@@ -5311,7 +5317,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
 
     case 0x8d: /* lea */
         ot = dflag + OT_WORD;
-        modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+        modrm = cpu_ldub_code(env, s->pc++);
         mod = (modrm >> 6) & 3;
         if (mod == 3)
             goto illegal_op;
@@ -5320,7 +5326,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         s->override = -1;
         val = s->addseg;
         s->addseg = 0;
-        gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+        gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
         s->addseg = val;
         gen_op_mov_reg_A0(ot - OT_WORD, reg);
         break;
@@ -5338,16 +5344,16 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
                 ot = dflag + OT_WORD;
 #ifdef TARGET_X86_64
             if (s->aflag == 2) {
-                offset_addr = cpu_ldq_code(cpu_single_env, s->pc);
+                offset_addr = cpu_ldq_code(env, s->pc);
                 s->pc += 8;
                 gen_op_movq_A0_im(offset_addr);
             } else
 #endif
             {
                 if (s->aflag) {
-                    offset_addr = insn_get(s, OT_LONG);
+                    offset_addr = insn_get(env, s, OT_LONG);
                 } else {
-                    offset_addr = insn_get(s, OT_WORD);
+                    offset_addr = insn_get(env, s, OT_WORD);
                 }
                 gen_op_movl_A0_im(offset_addr);
             }
@@ -5385,7 +5391,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         gen_op_mov_reg_T0(OT_BYTE, R_EAX);
         break;
     case 0xb0 ... 0xb7: /* mov R, Ib */
-        val = insn_get(s, OT_BYTE);
+        val = insn_get(env, s, OT_BYTE);
         gen_op_movl_T0_im(val);
         gen_op_mov_reg_T0(OT_BYTE, (b & 7) | REX_B(s));
         break;
@@ -5394,7 +5400,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         if (dflag == 2) {
             uint64_t tmp;
             /* 64 bit case */
-            tmp = cpu_ldq_code(cpu_single_env, s->pc);
+            tmp = cpu_ldq_code(env, s->pc);
             s->pc += 8;
             reg = (b & 7) | REX_B(s);
             gen_movtl_T0_im(tmp);
@@ -5403,7 +5409,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
 #endif
         {
             ot = dflag ? OT_LONG : OT_WORD;
-            val = insn_get(s, ot);
+            val = insn_get(env, s, ot);
             reg = (b & 7) | REX_B(s);
             gen_op_movl_T0_im(val);
             gen_op_mov_reg_T0(ot, reg);
@@ -5422,7 +5428,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
             ot = OT_BYTE;
         else
             ot = dflag + OT_WORD;
-        modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+        modrm = cpu_ldub_code(env, s->pc++);
         reg = ((modrm >> 3) & 7) | rex_r;
         mod = (modrm >> 6) & 3;
         if (mod == 3) {
@@ -5433,7 +5439,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
             gen_op_mov_reg_T0(ot, rm);
             gen_op_mov_reg_T1(ot, reg);
         } else {
-            gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+            gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
             gen_op_mov_TN_reg(ot, 0, reg);
             /* for xchg, lock is implicit */
             if (!(prefixes & PREFIX_LOCK))
@@ -5465,12 +5471,12 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         op = R_GS;
     do_lxx:
         ot = dflag ? OT_LONG : OT_WORD;
-        modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+        modrm = cpu_ldub_code(env, s->pc++);
         reg = ((modrm >> 3) & 7) | rex_r;
         mod = (modrm >> 6) & 3;
         if (mod == 3)
             goto illegal_op;
-        gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+        gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
         gen_op_ld_T1_A0(ot + s->mem_index);
         gen_add_A0_im(s, 1 << (ot - OT_WORD + 1));
         /* load the segment first to handle exceptions properly */
@@ -5497,7 +5503,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
             else
                 ot = dflag + OT_WORD;
 
-            modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+            modrm = cpu_ldub_code(env, s->pc++);
             mod = (modrm >> 6) & 3;
             op = (modrm >> 3) & 7;
 
@@ -5505,7 +5511,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
                 if (shift == 2) {
                     s->rip_offset = 1;
                 }
-                gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
                 opreg = OR_TMP0;
             } else {
                 opreg = (modrm & 7) | REX_B(s);
@@ -5516,7 +5522,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
                 gen_shift(s, op, ot, opreg, OR_ECX);
             } else {
                 if (shift == 2) {
-                    shift = cpu_ldub_code(cpu_single_env, s->pc++);
+                    shift = cpu_ldub_code(env, s->pc++);
                 }
                 gen_shifti(s, op, ot, opreg, shift);
             }
@@ -5550,12 +5556,12 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         shift = 0;
     do_shiftd:
         ot = dflag + OT_WORD;
-        modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+        modrm = cpu_ldub_code(env, s->pc++);
         mod = (modrm >> 6) & 3;
         rm = (modrm & 7) | REX_B(s);
         reg = ((modrm >> 3) & 7) | rex_r;
         if (mod != 3) {
-            gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+            gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
             opreg = OR_TMP0;
         } else {
             opreg = rm;
@@ -5563,7 +5569,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         gen_op_mov_TN_reg(ot, 1, reg);
 
         if (shift) {
-            val = cpu_ldub_code(cpu_single_env, s->pc++);
+            val = cpu_ldub_code(env, s->pc++);
             tcg_gen_movi_tl(cpu_T3, val);
         } else {
             tcg_gen_mov_tl(cpu_T3, cpu_regs[R_ECX]);
@@ -5580,13 +5586,13 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
             gen_exception(s, EXCP07_PREX, pc_start - s->cs_base);
             break;
         }
-        modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+        modrm = cpu_ldub_code(env, s->pc++);
         mod = (modrm >> 6) & 3;
         rm = modrm & 7;
         op = ((b & 7) << 3) | ((modrm >> 3) & 7);
         if (mod != 3) {
             /* memory op */
-            gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+            gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
             switch(op) {
             case 0x00 ... 0x07: /* fxxxs */
             case 0x10 ... 0x17: /* fixxxl */
@@ -6211,7 +6217,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
             ot = OT_BYTE;
         else
             ot = dflag ? OT_LONG : OT_WORD;
-        val = cpu_ldub_code(cpu_single_env, s->pc++);
+        val = cpu_ldub_code(env, s->pc++);
         gen_op_movl_T0_im(val);
         gen_check_io(s, ot, pc_start - s->cs_base,
                      SVM_IOIO_TYPE_MASK | svm_is_rep(prefixes));
@@ -6231,7 +6237,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
             ot = OT_BYTE;
         else
             ot = dflag ? OT_LONG : OT_WORD;
-        val = cpu_ldub_code(cpu_single_env, s->pc++);
+        val = cpu_ldub_code(env, s->pc++);
         gen_op_movl_T0_im(val);
         gen_check_io(s, ot, pc_start - s->cs_base,
                      svm_is_rep(prefixes));
@@ -6293,7 +6299,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         /************************/
         /* control */
     case 0xc2: /* ret im */
-        val = cpu_ldsw_code(cpu_single_env, s->pc);
+        val = cpu_ldsw_code(env, s->pc);
         s->pc += 2;
         gen_pop_T0(s);
         if (CODE64(s) && s->dflag)
@@ -6313,7 +6319,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         gen_eob(s);
         break;
     case 0xca: /* lret im */
-        val = cpu_ldsw_code(cpu_single_env, s->pc);
+        val = cpu_ldsw_code(env, s->pc);
         s->pc += 2;
     do_lret:
         if (s->pe && !s->vm86) {
@@ -6369,9 +6375,9 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
     case 0xe8: /* call im */
         {
             if (dflag)
-                tval = (int32_t)insn_get(s, OT_LONG);
+                tval = (int32_t)insn_get(env, s, OT_LONG);
             else
-                tval = (int16_t)insn_get(s, OT_WORD);
+                tval = (int16_t)insn_get(env, s, OT_WORD);
             next_eip = s->pc - s->cs_base;
             tval += next_eip;
             if (s->dflag == 0)
@@ -6390,8 +6396,8 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
             if (CODE64(s))
                 goto illegal_op;
             ot = dflag ? OT_LONG : OT_WORD;
-            offset = insn_get(s, ot);
-            selector = insn_get(s, OT_WORD);
+            offset = insn_get(env, s, ot);
+            selector = insn_get(env, s, OT_WORD);
 
             gen_op_movl_T0_im(selector);
             gen_op_movl_T1_imu(offset);
@@ -6399,9 +6405,9 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         goto do_lcall;
     case 0xe9: /* jmp im */
         if (dflag)
-            tval = (int32_t)insn_get(s, OT_LONG);
+            tval = (int32_t)insn_get(env, s, OT_LONG);
         else
-            tval = (int16_t)insn_get(s, OT_WORD);
+            tval = (int16_t)insn_get(env, s, OT_WORD);
         tval += s->pc - s->cs_base;
         if (s->dflag == 0)
             tval &= 0xffff;
@@ -6416,28 +6422,28 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
             if (CODE64(s))
                 goto illegal_op;
             ot = dflag ? OT_LONG : OT_WORD;
-            offset = insn_get(s, ot);
-            selector = insn_get(s, OT_WORD);
+            offset = insn_get(env, s, ot);
+            selector = insn_get(env, s, OT_WORD);
 
             gen_op_movl_T0_im(selector);
             gen_op_movl_T1_imu(offset);
         }
         goto do_ljmp;
     case 0xeb: /* jmp Jb */
-        tval = (int8_t)insn_get(s, OT_BYTE);
+        tval = (int8_t)insn_get(env, s, OT_BYTE);
         tval += s->pc - s->cs_base;
         if (s->dflag == 0)
             tval &= 0xffff;
         gen_jmp(s, tval);
         break;
     case 0x70 ... 0x7f: /* jcc Jb */
-        tval = (int8_t)insn_get(s, OT_BYTE);
+        tval = (int8_t)insn_get(env, s, OT_BYTE);
         goto do_jcc;
     case 0x180 ... 0x18f: /* jcc Jv */
         if (dflag) {
-            tval = (int32_t)insn_get(s, OT_LONG);
+            tval = (int32_t)insn_get(env, s, OT_LONG);
         } else {
-            tval = (int16_t)insn_get(s, OT_WORD);
+            tval = (int16_t)insn_get(env, s, OT_WORD);
         }
     do_jcc:
         next_eip = s->pc - s->cs_base;
@@ -6448,9 +6454,9 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         break;
 
     case 0x190 ... 0x19f: /* setcc Gv */
-        modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+        modrm = cpu_ldub_code(env, s->pc++);
         gen_setcc(s, b);
-        gen_ldst_modrm(s, modrm, OT_BYTE, OR_TMP0, 1);
+        gen_ldst_modrm(env, s, modrm, OT_BYTE, OR_TMP0, 1);
         break;
     case 0x140 ... 0x14f: /* cmov Gv, Ev */
         {
@@ -6458,12 +6464,12 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
             TCGv t0;
 
             ot = dflag + OT_WORD;
-            modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+            modrm = cpu_ldub_code(env, s->pc++);
             reg = ((modrm >> 3) & 7) | rex_r;
             mod = (modrm >> 6) & 3;
             t0 = tcg_temp_local_new();
             if (mod != 3) {
-                gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
                 gen_op_ld_v(ot + s->mem_index, t0, cpu_A0);
             } else {
                 rm = (modrm & 7) | REX_B(s);
@@ -6555,7 +6561,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
             }
             gen_pop_update(s);
             s->cc_op = CC_OP_EFLAGS;
-            /* abort translation because TF flag may change */
+            /* abort translation because TF/AC flag may change */
             gen_jmp_im(s->pc - s->cs_base);
             gen_eob(s);
         }
@@ -6616,19 +6622,19 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         /* bit operations */
     case 0x1ba: /* bt/bts/btr/btc Gv, im */
         ot = dflag + OT_WORD;
-        modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+        modrm = cpu_ldub_code(env, s->pc++);
         op = (modrm >> 3) & 7;
         mod = (modrm >> 6) & 3;
         rm = (modrm & 7) | REX_B(s);
         if (mod != 3) {
             s->rip_offset = 1;
-            gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+            gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
             gen_op_ld_T0_A0(ot + s->mem_index);
         } else {
             gen_op_mov_TN_reg(ot, 0, rm);
         }
         /* load shift */
-        val = cpu_ldub_code(cpu_single_env, s->pc++);
+        val = cpu_ldub_code(env, s->pc++);
         gen_op_movl_T1_im(val);
         if (op < 4)
             goto illegal_op;
@@ -6647,13 +6653,13 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         op = 3;
     do_btx:
         ot = dflag + OT_WORD;
-        modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+        modrm = cpu_ldub_code(env, s->pc++);
         reg = ((modrm >> 3) & 7) | rex_r;
         mod = (modrm >> 6) & 3;
         rm = (modrm & 7) | REX_B(s);
         gen_op_mov_TN_reg(OT_LONG, 1, reg);
         if (mod != 3) {
-            gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+            gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
             /* specific case: we need to add a displacement */
             gen_exts(ot, cpu_T[1]);
             tcg_gen_sari_tl(cpu_tmp0, cpu_T[1], 3 + ot);
@@ -6708,9 +6714,9 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
             TCGv t0;
 
             ot = dflag + OT_WORD;
-            modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+            modrm = cpu_ldub_code(env, s->pc++);
             reg = ((modrm >> 3) & 7) | rex_r;
-            gen_ldst_modrm(s,modrm, ot, OR_TMP0, 0);
+            gen_ldst_modrm(env, s,modrm, ot, OR_TMP0, 0);
             gen_extu(ot, cpu_T[0]);
             t0 = tcg_temp_local_new();
             tcg_gen_mov_tl(t0, cpu_T[0]);
@@ -6780,7 +6786,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
     case 0xd4: /* aam */
         if (CODE64(s))
             goto illegal_op;
-        val = cpu_ldub_code(cpu_single_env, s->pc++);
+        val = cpu_ldub_code(env, s->pc++);
         if (val == 0) {
             gen_exception(s, EXCP00_DIVZ, pc_start - s->cs_base);
         } else {
@@ -6791,7 +6797,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
     case 0xd5: /* aad */
         if (CODE64(s))
             goto illegal_op;
-        val = cpu_ldub_code(cpu_single_env, s->pc++);
+        val = cpu_ldub_code(env, s->pc++);
         gen_helper_aad(cpu_env, tcg_const_i32(val));
         s->cc_op = CC_OP_LOGICB;
         break;
@@ -6825,7 +6831,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         gen_interrupt(s, EXCP03_INT3, pc_start - s->cs_base, s->pc - s->cs_base);
         break;
     case 0xcd: /* int N */
-        val = cpu_ldub_code(cpu_single_env, s->pc++);
+        val = cpu_ldub_code(env, s->pc++);
         if (s->vm86 && s->iopl != 3) {
             gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
         } else {
@@ -6847,7 +6853,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         gen_debug(s, pc_start - s->cs_base);
 #else
         /* start debug */
-        tb_flush(cpu_single_env);
+        tb_flush(env);
         cpu_set_log(CPU_LOG_INT | CPU_LOG_TB_IN_ASM);
 #endif
         break;
@@ -6895,13 +6901,13 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         if (CODE64(s))
             goto illegal_op;
         ot = dflag ? OT_LONG : OT_WORD;
-        modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+        modrm = cpu_ldub_code(env, s->pc++);
         reg = (modrm >> 3) & 7;
         mod = (modrm >> 6) & 3;
         if (mod == 3)
             goto illegal_op;
         gen_op_mov_TN_reg(ot, 0, reg);
-        gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+        gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
         gen_jmp_im(pc_start - s->cs_base);
         tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
         if (ot == OT_WORD) {
@@ -6942,7 +6948,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         {
             int l1, l2, l3;
 
-            tval = (int8_t)insn_get(s, OT_BYTE);
+            tval = (int8_t)insn_get(env, s, OT_BYTE);
             next_eip = s->pc - s->cs_base;
             tval += next_eip;
             if (s->dflag == 0)
@@ -7022,7 +7028,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         break;
     case 0x134: /* sysenter */
         /* For Intel SYSENTER is valid on 64-bit */
-        if (CODE64(s) && cpu_single_env->cpuid_vendor1 != CPUID_VENDOR_INTEL_1)
+        if (CODE64(s) && env->cpuid_vendor1 != CPUID_VENDOR_INTEL_1)
             goto illegal_op;
         if (!s->pe) {
             gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
@@ -7035,7 +7041,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         break;
     case 0x135: /* sysexit */
         /* For Intel SYSEXIT is valid on 64-bit */
-        if (CODE64(s) && cpu_single_env->cpuid_vendor1 != CPUID_VENDOR_INTEL_1)
+        if (CODE64(s) && env->cpuid_vendor1 != CPUID_VENDOR_INTEL_1)
             goto illegal_op;
         if (!s->pe) {
             gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
@@ -7086,7 +7092,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         }
         break;
     case 0x100:
-        modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+        modrm = cpu_ldub_code(env, s->pc++);
         mod = (modrm >> 6) & 3;
         op = (modrm >> 3) & 7;
         switch(op) {
@@ -7098,7 +7104,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
             ot = OT_WORD;
             if (mod == 3)
                 ot += s->dflag;
-            gen_ldst_modrm(s, modrm, ot, OR_TMP0, 1);
+            gen_ldst_modrm(env, s, modrm, ot, OR_TMP0, 1);
             break;
         case 2: /* lldt */
             if (!s->pe || s->vm86)
@@ -7107,7 +7113,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
                 gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
             } else {
                 gen_svm_check_intercept(s, pc_start, SVM_EXIT_LDTR_WRITE);
-                gen_ldst_modrm(s, modrm, OT_WORD, OR_TMP0, 0);
+                gen_ldst_modrm(env, s, modrm, OT_WORD, OR_TMP0, 0);
                 gen_jmp_im(pc_start - s->cs_base);
                 tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
                 gen_helper_lldt(cpu_env, cpu_tmp2_i32);
@@ -7121,7 +7127,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
             ot = OT_WORD;
             if (mod == 3)
                 ot += s->dflag;
-            gen_ldst_modrm(s, modrm, ot, OR_TMP0, 1);
+            gen_ldst_modrm(env, s, modrm, ot, OR_TMP0, 1);
             break;
         case 3: /* ltr */
             if (!s->pe || s->vm86)
@@ -7130,7 +7136,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
                 gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
             } else {
                 gen_svm_check_intercept(s, pc_start, SVM_EXIT_TR_WRITE);
-                gen_ldst_modrm(s, modrm, OT_WORD, OR_TMP0, 0);
+                gen_ldst_modrm(env, s, modrm, OT_WORD, OR_TMP0, 0);
                 gen_jmp_im(pc_start - s->cs_base);
                 tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
                 gen_helper_ltr(cpu_env, cpu_tmp2_i32);
@@ -7140,7 +7146,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         case 5: /* verw */
             if (!s->pe || s->vm86)
                 goto illegal_op;
-            gen_ldst_modrm(s, modrm, OT_WORD, OR_TMP0, 0);
+            gen_ldst_modrm(env, s, modrm, OT_WORD, OR_TMP0, 0);
             if (s->cc_op != CC_OP_DYNAMIC)
                 gen_op_set_cc_op(s->cc_op);
             if (op == 4) {
@@ -7155,7 +7161,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         }
         break;
     case 0x101:
-        modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+        modrm = cpu_ldub_code(env, s->pc++);
         mod = (modrm >> 6) & 3;
         op = (modrm >> 3) & 7;
         rm = modrm & 7;
@@ -7164,7 +7170,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
             if (mod == 3)
                 goto illegal_op;
             gen_svm_check_intercept(s, pc_start, SVM_EXIT_GDTR_READ);
-            gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+            gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
             tcg_gen_ld32u_tl(cpu_T[0], cpu_env, offsetof(CPUX86State, gdt.limit));
             gen_op_st_T0_A0(OT_WORD + s->mem_index);
             gen_add_A0_im(s, 2);
@@ -7205,12 +7211,30 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
                     gen_helper_mwait(cpu_env, tcg_const_i32(s->pc - pc_start));
                     gen_eob(s);
                     break;
+                case 2: /* clac */
+                    if (!(s->cpuid_7_0_ebx_features & CPUID_7_0_EBX_SMAP) ||
+                        s->cpl != 0) {
+                        goto illegal_op;
+                    }
+                    gen_helper_clac(cpu_env);
+                    gen_jmp_im(s->pc - s->cs_base);
+                    gen_eob(s);
+                    break;
+                case 3: /* stac */
+                    if (!(s->cpuid_7_0_ebx_features & CPUID_7_0_EBX_SMAP) ||
+                        s->cpl != 0) {
+                        goto illegal_op;
+                    }
+                    gen_helper_stac(cpu_env);
+                    gen_jmp_im(s->pc - s->cs_base);
+                    gen_eob(s);
+                    break;
                 default:
                     goto illegal_op;
                 }
             } else { /* sidt */
                 gen_svm_check_intercept(s, pc_start, SVM_EXIT_IDTR_READ);
-                gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
                 tcg_gen_ld32u_tl(cpu_T[0], cpu_env, offsetof(CPUX86State, idt.limit));
                 gen_op_st_T0_A0(OT_WORD + s->mem_index);
                 gen_add_A0_im(s, 2);
@@ -7312,7 +7336,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
             } else {
                 gen_svm_check_intercept(s, pc_start,
                                         op==2 ? SVM_EXIT_GDTR_WRITE : SVM_EXIT_IDTR_WRITE);
-                gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
                 gen_op_ld_T1_A0(OT_WORD + s->mem_index);
                 gen_add_A0_im(s, 2);
                 gen_op_ld_T0_A0(CODE64(s) + OT_LONG + s->mem_index);
@@ -7334,14 +7358,14 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
 #else
             tcg_gen_ld32u_tl(cpu_T[0], cpu_env, offsetof(CPUX86State,cr[0]));
 #endif
-            gen_ldst_modrm(s, modrm, OT_WORD, OR_TMP0, 1);
+            gen_ldst_modrm(env, s, modrm, OT_WORD, OR_TMP0, 1);
             break;
         case 6: /* lmsw */
             if (s->cpl != 0) {
                 gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
             } else {
                 gen_svm_check_intercept(s, pc_start, SVM_EXIT_WRITE_CR0);
-                gen_ldst_modrm(s, modrm, OT_WORD, OR_TMP0, 0);
+                gen_ldst_modrm(env, s, modrm, OT_WORD, OR_TMP0, 0);
                 gen_helper_lmsw(cpu_env, cpu_T[0]);
                 gen_jmp_im(s->pc - s->cs_base);
                 gen_eob(s);
@@ -7355,7 +7379,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
                     if (s->cc_op != CC_OP_DYNAMIC)
                         gen_op_set_cc_op(s->cc_op);
                     gen_jmp_im(pc_start - s->cs_base);
-                    gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                    gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
                     gen_helper_invlpg(cpu_env, cpu_A0);
                     gen_jmp_im(s->pc - s->cs_base);
                     gen_eob(s);
@@ -7422,7 +7446,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
             /* d_ot is the size of destination */
             d_ot = dflag + OT_WORD;
 
-            modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+            modrm = cpu_ldub_code(env, s->pc++);
             reg = ((modrm >> 3) & 7) | rex_r;
             mod = (modrm >> 6) & 3;
             rm = (modrm & 7) | REX_B(s);
@@ -7434,7 +7458,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
                     tcg_gen_ext32s_tl(cpu_T[0], cpu_T[0]);
                 gen_op_mov_reg_T0(d_ot, reg);
             } else {
-                gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
                 if (d_ot == OT_QUAD) {
                     gen_op_lds_T0_A0(OT_LONG + s->mem_index);
                 } else {
@@ -7454,12 +7478,12 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
             t1 = tcg_temp_local_new();
             t2 = tcg_temp_local_new();
             ot = OT_WORD;
-            modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+            modrm = cpu_ldub_code(env, s->pc++);
             reg = (modrm >> 3) & 7;
             mod = (modrm >> 6) & 3;
             rm = modrm & 7;
             if (mod != 3) {
-                gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
                 gen_op_ld_v(ot + s->mem_index, t0, cpu_A0);
                 a0 = tcg_temp_local_new();
                 tcg_gen_mov_tl(a0, cpu_A0);
@@ -7502,9 +7526,9 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
             if (!s->pe || s->vm86)
                 goto illegal_op;
             ot = dflag ? OT_LONG : OT_WORD;
-            modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+            modrm = cpu_ldub_code(env, s->pc++);
             reg = ((modrm >> 3) & 7) | rex_r;
-            gen_ldst_modrm(s, modrm, OT_WORD, OR_TMP0, 0);
+            gen_ldst_modrm(env, s, modrm, OT_WORD, OR_TMP0, 0);
             t0 = tcg_temp_local_new();
             if (s->cc_op != CC_OP_DYNAMIC)
                 gen_op_set_cc_op(s->cc_op);
@@ -7523,7 +7547,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         }
         break;
     case 0x118:
-        modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+        modrm = cpu_ldub_code(env, s->pc++);
         mod = (modrm >> 6) & 3;
         op = (modrm >> 3) & 7;
         switch(op) {
@@ -7533,24 +7557,24 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         case 3: /* prefetchnt0 */
             if (mod == 3)
                 goto illegal_op;
-            gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+            gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
             /* nothing more to do */
             break;
         default: /* nop (multi byte) */
-            gen_nop_modrm(s, modrm);
+            gen_nop_modrm(env, s, modrm);
             break;
         }
         break;
     case 0x119 ... 0x11f: /* nop (multi byte) */
-        modrm = cpu_ldub_code(cpu_single_env, s->pc++);
-        gen_nop_modrm(s, modrm);
+        modrm = cpu_ldub_code(env, s->pc++);
+        gen_nop_modrm(env, s, modrm);
         break;
     case 0x120: /* mov reg, crN */
     case 0x122: /* mov crN, reg */
         if (s->cpl != 0) {
             gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
         } else {
-            modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+            modrm = cpu_ldub_code(env, s->pc++);
             /* Ignore the mod bits (assume (modrm&0xc0)==0xc0).
              * AMD documentation (24594.pdf) and testing of
              * intel 386 and 486 processors all show that the mod bits
@@ -7596,7 +7620,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         if (s->cpl != 0) {
             gen_exception(s, EXCP0D_GPF, pc_start - s->cs_base);
         } else {
-            modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+            modrm = cpu_ldub_code(env, s->pc++);
             /* Ignore the mod bits (assume (modrm&0xc0)==0xc0).
              * AMD documentation (24594.pdf) and testing of
              * intel 386 and 486 processors all show that the mod bits
@@ -7640,16 +7664,16 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         if (!(s->cpuid_features & CPUID_SSE2))
             goto illegal_op;
         ot = s->dflag == 2 ? OT_QUAD : OT_LONG;
-        modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+        modrm = cpu_ldub_code(env, s->pc++);
         mod = (modrm >> 6) & 3;
         if (mod == 3)
             goto illegal_op;
         reg = ((modrm >> 3) & 7) | rex_r;
         /* generate a generic store */
-        gen_ldst_modrm(s, modrm, ot, reg, 1);
+        gen_ldst_modrm(env, s, modrm, ot, reg, 1);
         break;
     case 0x1ae:
-        modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+        modrm = cpu_ldub_code(env, s->pc++);
         mod = (modrm >> 6) & 3;
         op = (modrm >> 3) & 7;
         switch(op) {
@@ -7661,7 +7685,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
                 gen_exception(s, EXCP07_PREX, pc_start - s->cs_base);
                 break;
             }
-            gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+            gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
             if (s->cc_op != CC_OP_DYNAMIC)
                 gen_op_set_cc_op(s->cc_op);
             gen_jmp_im(pc_start - s->cs_base);
@@ -7675,7 +7699,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
                 gen_exception(s, EXCP07_PREX, pc_start - s->cs_base);
                 break;
             }
-            gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+            gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
             if (s->cc_op != CC_OP_DYNAMIC)
                 gen_op_set_cc_op(s->cc_op);
             gen_jmp_im(pc_start - s->cs_base);
@@ -7691,7 +7715,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
             if ((s->flags & HF_EM_MASK) || !(s->flags & HF_OSFXSR_MASK) ||
                 mod == 3)
                 goto illegal_op;
-            gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+            gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
             if (op == 2) {
                 gen_op_ld_T0_A0(OT_LONG + s->mem_index);
                 tcg_gen_trunc_tl_i32(cpu_tmp2_i32, cpu_T[0]);
@@ -7716,7 +7740,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
                 /* clflush */
                 if (!(s->cpuid_features & CPUID_CLFLUSH))
                     goto illegal_op;
-                gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+                gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
             }
             break;
         default:
@@ -7724,11 +7748,11 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         }
         break;
     case 0x10d: /* 3DNow! prefetch(w) */
-        modrm = cpu_ldub_code(cpu_single_env, s->pc++);
+        modrm = cpu_ldub_code(env, s->pc++);
         mod = (modrm >> 6) & 3;
         if (mod == 3)
             goto illegal_op;
-        gen_lea_modrm(s, modrm, &reg_addr, &offset_addr);
+        gen_lea_modrm(env, s, modrm, &reg_addr, &offset_addr);
         /* ignore for now */
         break;
     case 0x1aa: /* rsm */
@@ -7747,8 +7771,8 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         if (!(s->cpuid_ext_features & CPUID_EXT_POPCNT))
             goto illegal_op;
 
-        modrm = cpu_ldub_code(cpu_single_env, s->pc++);
-        reg = ((modrm >> 3) & 7);
+        modrm = cpu_ldub_code(env, s->pc++);
+        reg = ((modrm >> 3) & 7) | rex_r;
 
         if (s->prefix & PREFIX_DATA)
             ot = OT_WORD;
@@ -7757,7 +7781,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
         else
             ot = OT_QUAD;
 
-        gen_ldst_modrm(s, modrm, ot, OR_TMP0, 0);
+        gen_ldst_modrm(env, s, modrm, ot, OR_TMP0, 0);
         gen_helper_popcnt(cpu_T[0], cpu_env, cpu_T[0], tcg_const_i32(ot));
         gen_op_mov_reg_T0(ot, reg);
 
@@ -7774,7 +7798,7 @@ static target_ulong disas_insn(DisasContext *s, target_ulong pc_start)
     case 0x1c2:
     case 0x1c4 ... 0x1c6:
     case 0x1d0 ... 0x1fe:
-        gen_sse(s, b, pc_start, rex_r);
+        gen_sse(env, s, b, pc_start, rex_r);
         break;
     default:
         goto illegal_op;
@@ -7900,15 +7924,13 @@ static inline void gen_intermediate_code_internal(CPUX86State *env,
     /* select memory access functions */
     dc->mem_index = 0;
     if (flags & HF_SOFTMMU_MASK) {
-        if (dc->cpl == 3)
-            dc->mem_index = 2 * 4;
-        else
-            dc->mem_index = 1 * 4;
+        dc->mem_index = (cpu_mmu_index(env) + 1) << 2;
     }
     dc->cpuid_features = env->cpuid_features;
     dc->cpuid_ext_features = env->cpuid_ext_features;
     dc->cpuid_ext2_features = env->cpuid_ext2_features;
     dc->cpuid_ext3_features = env->cpuid_ext3_features;
+    dc->cpuid_7_0_ebx_features = env->cpuid_7_0_ebx_features;
 #ifdef TARGET_X86_64
     dc->lma = (flags >> HF_LMA_SHIFT) & 1;
     dc->code64 = (flags >> HF_CS64_SHIFT) & 1;
@@ -7940,7 +7962,7 @@ static inline void gen_intermediate_code_internal(CPUX86State *env,
     cpu_ptr0 = tcg_temp_new_ptr();
     cpu_ptr1 = tcg_temp_new_ptr();
 
-    gen_opc_end = gen_opc_buf + OPC_MAX_SIZE;
+    gen_opc_end = tcg_ctx.gen_opc_buf + OPC_MAX_SIZE;
 
     dc->is_jmp = DISAS_NEXT;
     pc_ptr = pc_start;
@@ -7962,7 +7984,7 @@ static inline void gen_intermediate_code_internal(CPUX86State *env,
             }
         }
         if (search_pc) {
-            j = gen_opc_ptr - gen_opc_buf;
+            j = tcg_ctx.gen_opc_ptr - tcg_ctx.gen_opc_buf;
             if (lj < j) {
                 lj++;
                 while (lj < j)
@@ -7976,7 +7998,7 @@ static inline void gen_intermediate_code_internal(CPUX86State *env,
         if (num_insns + 1 == max_insns && (tb->cflags & CF_LAST_IO))
             gen_io_start();
 
-        pc_ptr = disas_insn(dc, pc_ptr);
+        pc_ptr = disas_insn(env, dc, pc_ptr);
         num_insns++;
         /* stop translation if indicated */
         if (dc->is_jmp)
@@ -7993,7 +8015,7 @@ static inline void gen_intermediate_code_internal(CPUX86State *env,
             break;
         }
         /* if too long translation, stop generation too */
-        if (gen_opc_ptr >= gen_opc_end ||
+        if (tcg_ctx.gen_opc_ptr >= gen_opc_end ||
             (pc_ptr - pc_start) >= (TARGET_PAGE_SIZE - 32) ||
             num_insns >= max_insns) {
             gen_jmp_im(pc_ptr - dc->cs_base);
@@ -8009,10 +8031,10 @@ static inline void gen_intermediate_code_internal(CPUX86State *env,
     if (tb->cflags & CF_LAST_IO)
         gen_io_end();
     gen_icount_end(tb, num_insns);
-    *gen_opc_ptr = INDEX_op_end;
+    *tcg_ctx.gen_opc_ptr = INDEX_op_end;
     /* we don't forget to fill the last values */
     if (search_pc) {
-        j = gen_opc_ptr - gen_opc_buf;
+        j = tcg_ctx.gen_opc_ptr - tcg_ctx.gen_opc_buf;
         lj++;
         while (lj <= j)
             gen_opc_instr_start[lj++] = 0;
@@ -8029,7 +8051,7 @@ static inline void gen_intermediate_code_internal(CPUX86State *env,
         else
 #endif
             disas_flags = !dc->code32;
-        log_target_disas(pc_start, pc_ptr - pc_start, disas_flags);
+        log_target_disas(env, pc_start, pc_ptr - pc_start, disas_flags);
         qemu_log("\n");
     }
 #endif
diff --git a/target-lm32/Makefile.objs b/target-lm32/Makefile.objs
index 2e0e093e1..ca20f2144 100644
--- a/target-lm32/Makefile.objs
+++ b/target-lm32/Makefile.objs
@@ -1,4 +1,2 @@
 obj-y += translate.o op_helper.o helper.o cpu.o
 obj-$(CONFIG_SOFTMMU) += machine.o
-
-$(obj)/op_helper.o: QEMU_CFLAGS += $(HELPER_CFLAGS)
diff --git a/target-lm32/cpu.h b/target-lm32/cpu.h
index da80469f5..7243b4f7c 100644
--- a/target-lm32/cpu.h
+++ b/target-lm32/cpu.h
@@ -253,8 +253,10 @@ static inline void cpu_get_tb_cpu_state(CPULM32State *env, target_ulong *pc,
     *flags = 0;
 }
 
-static inline bool cpu_has_work(CPULM32State *env)
+static inline bool cpu_has_work(CPUState *cpu)
 {
+    CPULM32State *env = &LM32_CPU(cpu)->env;
+
     return env->interrupt_request & CPU_INTERRUPT_HARD;
 }
 
diff --git a/target-lm32/helper.c b/target-lm32/helper.c
index 1ea477fea..0ed7cfda1 100644
--- a/target-lm32/helper.c
+++ b/target-lm32/helper.c
@@ -37,7 +37,7 @@ int cpu_lm32_handle_mmu_fault(CPULM32State *env, target_ulong address, int rw,
     return 0;
 }
 
-target_phys_addr_t cpu_get_phys_page_debug(CPULM32State *env, target_ulong addr)
+hwaddr cpu_get_phys_page_debug(CPULM32State *env, target_ulong addr)
 {
     return addr & TARGET_PAGE_MASK;
 }
diff --git a/target-lm32/helper.h b/target-lm32/helper.h
index 9d335efc4..07f567017 100644
--- a/target-lm32/helper.h
+++ b/target-lm32/helper.h
@@ -1,14 +1,14 @@
 #include "def-helper.h"
 
-DEF_HELPER_1(raise_exception, void, i32)
-DEF_HELPER_0(hlt, void)
-DEF_HELPER_1(wcsr_im, void, i32)
-DEF_HELPER_1(wcsr_ip, void, i32)
-DEF_HELPER_1(wcsr_jtx, void, i32)
-DEF_HELPER_1(wcsr_jrx, void, i32)
-DEF_HELPER_0(rcsr_im, i32)
-DEF_HELPER_0(rcsr_ip, i32)
-DEF_HELPER_0(rcsr_jtx, i32)
-DEF_HELPER_0(rcsr_jrx, i32)
+DEF_HELPER_2(raise_exception, void, env, i32)
+DEF_HELPER_1(hlt, void, env)
+DEF_HELPER_2(wcsr_im, void, env, i32)
+DEF_HELPER_2(wcsr_ip, void, env, i32)
+DEF_HELPER_2(wcsr_jtx, void, env, i32)
+DEF_HELPER_2(wcsr_jrx, void, env, i32)
+DEF_HELPER_1(rcsr_im, i32, env)
+DEF_HELPER_1(rcsr_ip, i32, env)
+DEF_HELPER_1(rcsr_jtx, i32, env)
+DEF_HELPER_1(rcsr_jrx, i32, env)
 
 #include "def-helper.h"
diff --git a/target-lm32/op_helper.c b/target-lm32/op_helper.c
index 51edc1a0e..7b91d8c31 100644
--- a/target-lm32/op_helper.c
+++ b/target-lm32/op_helper.c
@@ -1,6 +1,5 @@
 #include <assert.h>
 #include "cpu.h"
-#include "dyngen-exec.h"
 #include "helper.h"
 #include "host-utils.h"
 
@@ -18,55 +17,55 @@
 #define SHIFT 3
 #include "softmmu_template.h"
 
-void helper_raise_exception(uint32_t index)
+void helper_raise_exception(CPULM32State *env, uint32_t index)
 {
     env->exception_index = index;
     cpu_loop_exit(env);
 }
 
-void helper_hlt(void)
+void helper_hlt(CPULM32State *env)
 {
     env->halted = 1;
     env->exception_index = EXCP_HLT;
     cpu_loop_exit(env);
 }
 
-void helper_wcsr_im(uint32_t im)
+void helper_wcsr_im(CPULM32State *env, uint32_t im)
 {
     lm32_pic_set_im(env->pic_state, im);
 }
 
-void helper_wcsr_ip(uint32_t im)
+void helper_wcsr_ip(CPULM32State *env, uint32_t im)
 {
     lm32_pic_set_ip(env->pic_state, im);
 }
 
-void helper_wcsr_jtx(uint32_t jtx)
+void helper_wcsr_jtx(CPULM32State *env, uint32_t jtx)
 {
     lm32_juart_set_jtx(env->juart_state, jtx);
 }
 
-void helper_wcsr_jrx(uint32_t jrx)
+void helper_wcsr_jrx(CPULM32State *env, uint32_t jrx)
 {
     lm32_juart_set_jrx(env->juart_state, jrx);
 }
 
-uint32_t helper_rcsr_im(void)
+uint32_t helper_rcsr_im(CPULM32State *env)
 {
     return lm32_pic_get_im(env->pic_state);
 }
 
-uint32_t helper_rcsr_ip(void)
+uint32_t helper_rcsr_ip(CPULM32State *env)
 {
     return lm32_pic_get_ip(env->pic_state);
 }
 
-uint32_t helper_rcsr_jtx(void)
+uint32_t helper_rcsr_jtx(CPULM32State *env)
 {
     return lm32_juart_get_jtx(env->juart_state);
 }
 
-uint32_t helper_rcsr_jrx(void)
+uint32_t helper_rcsr_jrx(CPULM32State *env)
 {
     return lm32_juart_get_jrx(env->juart_state);
 }
@@ -74,17 +73,12 @@ uint32_t helper_rcsr_jrx(void)
 /* Try to fill the TLB and return an exception if error. If retaddr is
    NULL, it means that the function was called in C code (i.e. not
    from generated code or from helper.c) */
-/* XXX: fix it to restore all registers */
-void tlb_fill(CPULM32State *env1, target_ulong addr, int is_write, int mmu_idx,
+void tlb_fill(CPULM32State *env, target_ulong addr, int is_write, int mmu_idx,
               uintptr_t retaddr)
 {
     TranslationBlock *tb;
-    CPULM32State *saved_env;
     int ret;
 
-    saved_env = env;
-    env = env1;
-
     ret = cpu_lm32_handle_mmu_fault(env, addr, is_write, mmu_idx);
     if (unlikely(ret)) {
         if (retaddr) {
@@ -98,7 +92,6 @@ void tlb_fill(CPULM32State *env1, target_ulong addr, int is_write, int mmu_idx,
         }
         cpu_loop_exit(env);
     }
-    env = saved_env;
 }
 #endif
 
diff --git a/target-lm32/translate.c b/target-lm32/translate.c
index 872a2ba65..af986499f 100644
--- a/target-lm32/translate.c
+++ b/target-lm32/translate.c
@@ -116,7 +116,7 @@ static inline void t_gen_raise_exception(DisasContext *dc, uint32_t index)
 {
     TCGv_i32 tmp = tcg_const_i32(index);
 
-    gen_helper_raise_exception(tmp);
+    gen_helper_raise_exception(cpu_env, tmp);
     tcg_temp_free_i32(tmp);
 }
 
@@ -179,7 +179,7 @@ static void dec_and(DisasContext *dc)
     } else  {
         if (dc->r0 == 0 && dc->r1 == 0 && dc->r2 == 0) {
             tcg_gen_movi_tl(cpu_pc, dc->pc + 4);
-            gen_helper_hlt();
+            gen_helper_hlt(cpu_env);
         } else {
             tcg_gen_and_tl(cpu_R[dc->r2], cpu_R[dc->r0], cpu_R[dc->r1]);
         }
@@ -601,10 +601,10 @@ static void dec_rcsr(DisasContext *dc)
         tcg_gen_mov_tl(cpu_R[dc->r2], cpu_ie);
         break;
     case CSR_IM:
-        gen_helper_rcsr_im(cpu_R[dc->r2]);
+        gen_helper_rcsr_im(cpu_R[dc->r2], cpu_env);
         break;
     case CSR_IP:
-        gen_helper_rcsr_ip(cpu_R[dc->r2]);
+        gen_helper_rcsr_ip(cpu_R[dc->r2], cpu_env);
         break;
     case CSR_CC:
         tcg_gen_mov_tl(cpu_R[dc->r2], cpu_cc);
@@ -622,10 +622,10 @@ static void dec_rcsr(DisasContext *dc)
         tcg_gen_mov_tl(cpu_R[dc->r2], cpu_deba);
         break;
     case CSR_JTX:
-        gen_helper_rcsr_jtx(cpu_R[dc->r2]);
+        gen_helper_rcsr_jtx(cpu_R[dc->r2], cpu_env);
         break;
     case CSR_JRX:
-        gen_helper_rcsr_jrx(cpu_R[dc->r2]);
+        gen_helper_rcsr_jrx(cpu_R[dc->r2], cpu_env);
         break;
     case CSR_ICC:
     case CSR_DCC:
@@ -812,7 +812,7 @@ static void dec_wcsr(DisasContext *dc)
         if (use_icount) {
             gen_io_start();
         }
-        gen_helper_wcsr_im(cpu_R[dc->r1]);
+        gen_helper_wcsr_im(cpu_env, cpu_R[dc->r1]);
         tcg_gen_movi_tl(cpu_pc, dc->pc + 4);
         if (use_icount) {
             gen_io_end();
@@ -824,7 +824,7 @@ static void dec_wcsr(DisasContext *dc)
         if (use_icount) {
             gen_io_start();
         }
-        gen_helper_wcsr_ip(cpu_R[dc->r1]);
+        gen_helper_wcsr_ip(cpu_env, cpu_R[dc->r1]);
         tcg_gen_movi_tl(cpu_pc, dc->pc + 4);
         if (use_icount) {
             gen_io_end();
@@ -844,10 +844,10 @@ static void dec_wcsr(DisasContext *dc)
         tcg_gen_mov_tl(cpu_deba, cpu_R[dc->r1]);
         break;
     case CSR_JTX:
-        gen_helper_wcsr_jtx(cpu_R[dc->r1]);
+        gen_helper_wcsr_jtx(cpu_env, cpu_R[dc->r1]);
         break;
     case CSR_JRX:
-        gen_helper_wcsr_jrx(cpu_R[dc->r1]);
+        gen_helper_wcsr_jrx(cpu_env, cpu_R[dc->r1]);
         break;
     case CSR_DC:
         tcg_gen_mov_tl(cpu_dc, cpu_R[dc->r1]);
@@ -940,15 +940,13 @@ static const DecoderInfo decinfo[] = {
     dec_cmpne
 };
 
-static inline void decode(DisasContext *dc)
+static inline void decode(DisasContext *dc, uint32_t ir)
 {
-    uint32_t ir;
-
-    if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP))) {
+    if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
         tcg_gen_debug_insn_start(dc->pc);
     }
 
-    dc->ir = ir = ldl_code(dc->pc);
+    dc->ir = ir;
     LOG_DIS("%8.8x\t", dc->ir);
 
     /* try guessing 'empty' instruction memory, although it may be a valid
@@ -1020,7 +1018,7 @@ static void gen_intermediate_code_internal(CPULM32State *env,
     dc->env = env;
     dc->tb = tb;
 
-    gen_opc_end = gen_opc_buf + OPC_MAX_SIZE;
+    gen_opc_end = tcg_ctx.gen_opc_buf + OPC_MAX_SIZE;
 
     dc->is_jmp = DISAS_NEXT;
     dc->pc = pc_start;
@@ -1049,7 +1047,7 @@ static void gen_intermediate_code_internal(CPULM32State *env,
         check_breakpoint(env, dc);
 
         if (search_pc) {
-            j = gen_opc_ptr - gen_opc_buf;
+            j = tcg_ctx.gen_opc_ptr - tcg_ctx.gen_opc_buf;
             if (lj < j) {
                 lj++;
                 while (lj < j) {
@@ -1068,12 +1066,12 @@ static void gen_intermediate_code_internal(CPULM32State *env,
             gen_io_start();
         }
 
-        decode(dc);
+        decode(dc, cpu_ldl_code(env, dc->pc));
         dc->pc += 4;
         num_insns++;
 
     } while (!dc->is_jmp
-         && gen_opc_ptr < gen_opc_end
+         && tcg_ctx.gen_opc_ptr < gen_opc_end
          && !env->singlestep_enabled
          && !singlestep
          && (dc->pc < next_page_start)
@@ -1107,9 +1105,9 @@ static void gen_intermediate_code_internal(CPULM32State *env,
     }
 
     gen_icount_end(tb, num_insns);
-    *gen_opc_ptr = INDEX_op_end;
+    *tcg_ctx.gen_opc_ptr = INDEX_op_end;
     if (search_pc) {
-        j = gen_opc_ptr - gen_opc_buf;
+        j = tcg_ctx.gen_opc_ptr - tcg_ctx.gen_opc_buf;
         lj++;
         while (lj <= j) {
             gen_opc_instr_start[lj++] = 0;
@@ -1122,9 +1120,10 @@ static void gen_intermediate_code_internal(CPULM32State *env,
 #ifdef DEBUG_DISAS
     if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
         qemu_log("\n");
-        log_target_disas(pc_start, dc->pc - pc_start, 0);
+        log_target_disas(env, pc_start, dc->pc - pc_start, 0);
         qemu_log("\nisize=%d osize=%td\n",
-            dc->pc - pc_start, gen_opc_ptr - gen_opc_buf);
+            dc->pc - pc_start, tcg_ctx.gen_opc_ptr -
+            tcg_ctx.gen_opc_buf);
     }
 #endif
 }
diff --git a/target-m68k/Makefile.objs b/target-m68k/Makefile.objs
index cda60157f..7eccfab0e 100644
--- a/target-m68k/Makefile.objs
+++ b/target-m68k/Makefile.objs
@@ -1,5 +1,3 @@
 obj-y += m68k-semi.o
 obj-y += translate.o op_helper.o helper.o cpu.o
 obj-$(CONFIG_SOFTMMU) += machine.o
-
-$(obj)/op_helper.o: QEMU_CFLAGS += $(HELPER_CFLAGS)
diff --git a/target-m68k/cpu.h b/target-m68k/cpu.h
index 5e6ee5096..780e2c94e 100644
--- a/target-m68k/cpu.h
+++ b/target-m68k/cpu.h
@@ -257,8 +257,10 @@ static inline void cpu_get_tb_cpu_state(CPUM68KState *env, target_ulong *pc,
             | ((env->macsr >> 4) & 0xf);        /* Bits 0-3 */
 }
 
-static inline bool cpu_has_work(CPUM68KState *env)
+static inline bool cpu_has_work(CPUState *cpu)
 {
+    CPUM68KState *env = &M68K_CPU(cpu)->env;
+
     return env->interrupt_request & CPU_INTERRUPT_HARD;
 }
 
diff --git a/target-m68k/helper.c b/target-m68k/helper.c
index eac0053c5..a5d010047 100644
--- a/target-m68k/helper.c
+++ b/target-m68k/helper.c
@@ -289,7 +289,7 @@ int cpu_m68k_handle_mmu_fault (CPUM68KState *env, target_ulong address, int rw,
 /* MMU */
 
 /* TODO: This will need fixing once the MMU is implemented.  */
-target_phys_addr_t cpu_get_phys_page_debug(CPUM68KState *env, target_ulong addr)
+hwaddr cpu_get_phys_page_debug(CPUM68KState *env, target_ulong addr)
 {
     return addr;
 }
diff --git a/target-m68k/helpers.h b/target-m68k/helpers.h
index cb8a0c7d4..8112b44a5 100644
--- a/target-m68k/helpers.h
+++ b/target-m68k/helpers.h
@@ -49,6 +49,6 @@ DEF_HELPER_3(set_mac_exts, void, env, i32, i32)
 DEF_HELPER_3(set_mac_extu, void, env, i32, i32)
 
 DEF_HELPER_2(flush_flags, void, env, i32)
-DEF_HELPER_1(raise_exception, void, i32)
+DEF_HELPER_2(raise_exception, void, env, i32)
 
 #include "def-helper.h"
diff --git a/target-m68k/m68k-semi.c b/target-m68k/m68k-semi.c
index 3bb30cd1f..9f7a24cdb 100644
--- a/target-m68k/m68k-semi.c
+++ b/target-m68k/m68k-semi.c
@@ -133,37 +133,61 @@ static void translate_stat(CPUM68KState *env, target_ulong addr, struct stat *s)
     unlock_user(p, addr, sizeof(struct m68k_gdb_stat));
 }
 
+static void m68k_semi_return_u32(CPUM68KState *env, uint32_t ret, uint32_t err)
+{
+    target_ulong args = env->dregs[1];
+    if (put_user_u32(ret, args) ||
+        put_user_u32(err, args + 4)) {
+        /* The m68k semihosting ABI does not provide any way to report this
+         * error to the guest, so the best we can do is log it in qemu.
+         * It is always a guest error not to pass us a valid argument block.
+         */
+        qemu_log_mask(LOG_GUEST_ERROR, "m68k-semihosting: return value "
+                      "discarded because argument block not writable\n");
+    }
+}
+
+static void m68k_semi_return_u64(CPUM68KState *env, uint64_t ret, uint32_t err)
+{
+    target_ulong args = env->dregs[1];
+    if (put_user_u32(ret >> 32, args) ||
+        put_user_u32(ret, args + 4) ||
+        put_user_u32(err, args + 8)) {
+        /* No way to report this via m68k semihosting ABI; just log it */
+        qemu_log_mask(LOG_GUEST_ERROR, "m68k-semihosting: return value "
+                      "discarded because argument block not writable\n");
+    }
+}
+
 static int m68k_semi_is_fseek;
 
 static void m68k_semi_cb(CPUM68KState *env, target_ulong ret, target_ulong err)
 {
-    target_ulong args;
-
-    args = env->dregs[1];
     if (m68k_semi_is_fseek) {
         /* FIXME: We've already lost the high bits of the fseek
            return value.  */
-        /* FIXME - handle put_user() failure */
-        put_user_u32(0, args);
-        args += 4;
+        m68k_semi_return_u64(env, ret, err);
         m68k_semi_is_fseek = 0;
+    } else {
+        m68k_semi_return_u32(env, ret, err);
     }
-    /* FIXME - handle put_user() failure */
-    put_user_u32(ret, args);
-    put_user_u32(errno, args + 4);
 }
 
-#define ARG(n)					\
-({						\
-    target_ulong __arg;				\
-    /* FIXME - handle get_user() failure */	\
-    get_user_ual(__arg, args + (n) * 4);	\
-    __arg;					\
-})
-#define PARG(x) ((unsigned long)ARG(x))
+/* Read the input value from the argument block; fail the semihosting
+ * call if the memory read fails.
+ */
+#define GET_ARG(n) do {                                 \
+    if (get_user_ual(arg ## n, args + (n) * 4)) {       \
+        result = -1;                                    \
+        errno = EFAULT;                                 \
+        goto failed;                                    \
+    }                                                   \
+} while (0)
+
 void do_m68k_semihosting(CPUM68KState *env, int nr)
 {
     uint32_t args;
+    target_ulong arg0, arg1, arg2, arg3;
     void *p;
     void *q;
     uint32_t len;
@@ -175,27 +199,33 @@ void do_m68k_semihosting(CPUM68KState *env, int nr)
         gdb_exit(env, env->dregs[0]);
         exit(env->dregs[0]);
     case HOSTED_OPEN:
+        GET_ARG(0);
+        GET_ARG(1);
+        GET_ARG(2);
+        GET_ARG(3);
         if (use_gdb_syscalls()) {
-            gdb_do_syscall(m68k_semi_cb, "open,%s,%x,%x", ARG(0), (int)ARG(1),
-                           ARG(2), ARG(3));
+            gdb_do_syscall(m68k_semi_cb, "open,%s,%x,%x", arg0, (int)arg1,
+                           arg2, arg3);
             return;
         } else {
-            if (!(p = lock_user_string(ARG(0)))) {
+            p = lock_user_string(arg0);
+            if (!p) {
                 /* FIXME - check error code? */
                 result = -1;
             } else {
-                result = open(p, translate_openflags(ARG(2)), ARG(3));
-                unlock_user(p, ARG(0), 0);
+                result = open(p, translate_openflags(arg2), arg3);
+                unlock_user(p, arg0, 0);
             }
         }
         break;
     case HOSTED_CLOSE:
         {
             /* Ignore attempts to close stdin/out/err.  */
-            int fd = ARG(0);
+            GET_ARG(0);
+            int fd = arg0;
             if (fd > 2) {
                 if (use_gdb_syscalls()) {
-                    gdb_do_syscall(m68k_semi_cb, "close,%x", ARG(0));
+                    gdb_do_syscall(m68k_semi_cb, "close,%x", arg0);
                     return;
                 } else {
                     result = close(fd);
@@ -206,123 +236,147 @@ void do_m68k_semihosting(CPUM68KState *env, int nr)
             break;
         }
     case HOSTED_READ:
-        len = ARG(2);
+        GET_ARG(0);
+        GET_ARG(1);
+        GET_ARG(2);
+        len = arg2;
         if (use_gdb_syscalls()) {
             gdb_do_syscall(m68k_semi_cb, "read,%x,%x,%x",
-                           ARG(0), ARG(1), len);
+                           arg0, arg1, len);
             return;
         } else {
-            if (!(p = lock_user(VERIFY_WRITE, ARG(1), len, 0))) {
+            p = lock_user(VERIFY_WRITE, arg1, len, 0);
+            if (!p) {
                 /* FIXME - check error code? */
                 result = -1;
             } else {
-                result = read(ARG(0), p, len);
-                unlock_user(p, ARG(1), len);
+                result = read(arg0, p, len);
+                unlock_user(p, arg1, len);
             }
         }
         break;
     case HOSTED_WRITE:
-        len = ARG(2);
+        GET_ARG(0);
+        GET_ARG(1);
+        GET_ARG(2);
+        len = arg2;
         if (use_gdb_syscalls()) {
             gdb_do_syscall(m68k_semi_cb, "write,%x,%x,%x",
-                           ARG(0), ARG(1), len);
+                           arg0, arg1, len);
             return;
         } else {
-            if (!(p = lock_user(VERIFY_READ, ARG(1), len, 1))) {
+            p = lock_user(VERIFY_READ, arg1, len, 1);
+            if (!p) {
                 /* FIXME - check error code? */
                 result = -1;
             } else {
-                result = write(ARG(0), p, len);
-                unlock_user(p, ARG(0), 0);
+                result = write(arg0, p, len);
+                unlock_user(p, arg0, 0);
             }
         }
         break;
     case HOSTED_LSEEK:
         {
             uint64_t off;
-            off = (uint32_t)ARG(2) | ((uint64_t)ARG(1) << 32);
+            GET_ARG(0);
+            GET_ARG(1);
+            GET_ARG(2);
+            GET_ARG(3);
+            off = (uint32_t)arg2 | ((uint64_t)arg1 << 32);
             if (use_gdb_syscalls()) {
                 m68k_semi_is_fseek = 1;
                 gdb_do_syscall(m68k_semi_cb, "fseek,%x,%lx,%x",
-                               ARG(0), off, ARG(3));
+                               arg0, off, arg3);
             } else {
-                off = lseek(ARG(0), off, ARG(3));
-                /* FIXME - handle put_user() failure */
-                put_user_u32(off >> 32, args);
-                put_user_u32(off, args + 4);
-                put_user_u32(errno, args + 8);
+                off = lseek(arg0, off, arg3);
+                m68k_semi_return_u64(env, off, errno);
             }
             return;
         }
     case HOSTED_RENAME:
+        GET_ARG(0);
+        GET_ARG(1);
+        GET_ARG(2);
+        GET_ARG(3);
         if (use_gdb_syscalls()) {
             gdb_do_syscall(m68k_semi_cb, "rename,%s,%s",
-                           ARG(0), (int)ARG(1), ARG(2), (int)ARG(3));
+                           arg0, (int)arg1, arg2, (int)arg3);
             return;
         } else {
-            p = lock_user_string(ARG(0));
-            q = lock_user_string(ARG(2));
+            p = lock_user_string(arg0);
+            q = lock_user_string(arg2);
             if (!p || !q) {
                 /* FIXME - check error code? */
                 result = -1;
             } else {
                 result = rename(p, q);
             }
-            unlock_user(p, ARG(0), 0);
-            unlock_user(q, ARG(2), 0);
+            unlock_user(p, arg0, 0);
+            unlock_user(q, arg2, 0);
         }
         break;
     case HOSTED_UNLINK:
+        GET_ARG(0);
+        GET_ARG(1);
         if (use_gdb_syscalls()) {
             gdb_do_syscall(m68k_semi_cb, "unlink,%s",
-                           ARG(0), (int)ARG(1));
+                           arg0, (int)arg1);
             return;
         } else {
-            if (!(p = lock_user_string(ARG(0)))) {
+            p = lock_user_string(arg0);
+            if (!p) {
                 /* FIXME - check error code? */
                 result = -1;
             } else {
                 result = unlink(p);
-                unlock_user(p, ARG(0), 0);
+                unlock_user(p, arg0, 0);
             }
         }
         break;
     case HOSTED_STAT:
+        GET_ARG(0);
+        GET_ARG(1);
+        GET_ARG(2);
         if (use_gdb_syscalls()) {
             gdb_do_syscall(m68k_semi_cb, "stat,%s,%x",
-                           ARG(0), (int)ARG(1), ARG(2));
+                           arg0, (int)arg1, arg2);
             return;
         } else {
             struct stat s;
-            if (!(p = lock_user_string(ARG(0)))) {
+            p = lock_user_string(arg0);
+            if (!p) {
                 /* FIXME - check error code? */
                 result = -1;
             } else {
                 result = stat(p, &s);
-                unlock_user(p, ARG(0), 0);
+                unlock_user(p, arg0, 0);
             }
             if (result == 0) {
-                translate_stat(env, ARG(2), &s);
+                translate_stat(env, arg2, &s);
             }
         }
         break;
     case HOSTED_FSTAT:
+        GET_ARG(0);
+        GET_ARG(1);
         if (use_gdb_syscalls()) {
             gdb_do_syscall(m68k_semi_cb, "fstat,%x,%x",
-                           ARG(0), ARG(1));
+                           arg0, arg1);
             return;
         } else {
             struct stat s;
-            result = fstat(ARG(0), &s);
+            result = fstat(arg0, &s);
             if (result == 0) {
-                translate_stat(env, ARG(1), &s);
+                translate_stat(env, arg1, &s);
             }
         }
         break;
     case HOSTED_GETTIMEOFDAY:
+        GET_ARG(0);
+        GET_ARG(1);
         if (use_gdb_syscalls()) {
             gdb_do_syscall(m68k_semi_cb, "gettimeofday,%x,%x",
-                           ARG(0), ARG(1));
+                           arg0, arg1);
             return;
         } else {
             qemu_timeval tv;
@@ -330,37 +384,41 @@ void do_m68k_semihosting(CPUM68KState *env, int nr)
             result = qemu_gettimeofday(&tv);
             if (result != 0) {
                 if (!(p = lock_user(VERIFY_WRITE,
-                                    ARG(0), sizeof(struct gdb_timeval), 0))) {
+                                    arg0, sizeof(struct gdb_timeval), 0))) {
                     /* FIXME - check error code? */
                     result = -1;
                 } else {
                     p->tv_sec = cpu_to_be32(tv.tv_sec);
                     p->tv_usec = cpu_to_be64(tv.tv_usec);
-                    unlock_user(p, ARG(0), sizeof(struct gdb_timeval));
+                    unlock_user(p, arg0, sizeof(struct gdb_timeval));
                 }
             }
         }
         break;
     case HOSTED_ISATTY:
+        GET_ARG(0);
         if (use_gdb_syscalls()) {
-            gdb_do_syscall(m68k_semi_cb, "isatty,%x", ARG(0));
+            gdb_do_syscall(m68k_semi_cb, "isatty,%x", arg0);
             return;
         } else {
-            result = isatty(ARG(0));
+            result = isatty(arg0);
         }
         break;
     case HOSTED_SYSTEM:
+        GET_ARG(0);
+        GET_ARG(1);
         if (use_gdb_syscalls()) {
             gdb_do_syscall(m68k_semi_cb, "system,%s",
-                           ARG(0), (int)ARG(1));
+                           arg0, (int)arg1);
             return;
         } else {
-            if (!(p = lock_user_string(ARG(0)))) {
+            p = lock_user_string(arg0);
+            if (!p) {
                 /* FIXME - check error code? */
                 result = -1;
             } else {
                 result = system(p);
-                unlock_user(p, ARG(0), 0);
+                unlock_user(p, arg0, 0);
             }
         }
         break;
@@ -402,7 +460,6 @@ void do_m68k_semihosting(CPUM68KState *env, int nr)
         cpu_abort(env, "Unsupported semihosting syscall %d\n", nr);
         result = 0;
     }
-    /* FIXME - handle put_user() failure */
-    put_user_u32(result, args);
-    put_user_u32(errno, args + 4);
+failed:
+    m68k_semi_return_u32(env, result, errno);
 }
diff --git a/target-m68k/op_helper.c b/target-m68k/op_helper.c
index 1971a5789..aa005048e 100644
--- a/target-m68k/op_helper.c
+++ b/target-m68k/op_helper.c
@@ -17,17 +17,16 @@
  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
  */
 #include "cpu.h"
-#include "dyngen-exec.h"
 #include "helpers.h"
 
 #if defined(CONFIG_USER_ONLY)
 
-void do_interrupt(CPUM68KState *env1)
+void do_interrupt(CPUM68KState *env)
 {
-    env1->exception_index = -1;
+    env->exception_index = -1;
 }
 
-void do_interrupt_m68k_hardirq(CPUM68KState *env1)
+void do_interrupt_m68k_hardirq(CPUM68KState *env)
 {
 }
 
@@ -54,16 +53,12 @@ extern int semihosting_enabled;
 /* Try to fill the TLB and return an exception if error. If retaddr is
    NULL, it means that the function was called in C code (i.e. not
    from generated code or from helper.c) */
-/* XXX: fix it to restore all registers */
-void tlb_fill(CPUM68KState *env1, target_ulong addr, int is_write, int mmu_idx,
+void tlb_fill(CPUM68KState *env, target_ulong addr, int is_write, int mmu_idx,
               uintptr_t retaddr)
 {
     TranslationBlock *tb;
-    CPUM68KState *saved_env;
     int ret;
 
-    saved_env = env;
-    env = env1;
     ret = cpu_m68k_handle_mmu_fault(env, addr, is_write, mmu_idx);
     if (unlikely(ret)) {
         if (retaddr) {
@@ -77,24 +72,23 @@ void tlb_fill(CPUM68KState *env1, target_ulong addr, int is_write, int mmu_idx,
         }
         cpu_loop_exit(env);
     }
-    env = saved_env;
 }
 
-static void do_rte(void)
+static void do_rte(CPUM68KState *env)
 {
     uint32_t sp;
     uint32_t fmt;
 
     sp = env->aregs[7];
-    fmt = ldl_kernel(sp);
-    env->pc = ldl_kernel(sp + 4);
+    fmt = cpu_ldl_kernel(env, sp);
+    env->pc = cpu_ldl_kernel(env, sp + 4);
     sp |= (fmt >> 28) & 3;
     env->sr = fmt & 0xffff;
     m68k_switch_sp(env);
     env->aregs[7] = sp + 8;
 }
 
-static void do_interrupt_all(int is_hw)
+static void do_interrupt_all(CPUM68KState *env, int is_hw)
 {
     uint32_t sp;
     uint32_t fmt;
@@ -108,14 +102,14 @@ static void do_interrupt_all(int is_hw)
         switch (env->exception_index) {
         case EXCP_RTE:
             /* Return from an exception.  */
-            do_rte();
+            do_rte(env);
             return;
         case EXCP_HALT_INSN:
             if (semihosting_enabled
                     && (env->sr & SR_S) != 0
                     && (env->pc & 3) == 0
-                    && lduw_code(env->pc - 4) == 0x4e71
-                    && ldl_code(env->pc) == 0x4e7bf000) {
+                    && cpu_lduw_code(env, env->pc - 4) == 0x4e71
+                    && cpu_ldl_code(env, env->pc) == 0x4e7bf000) {
                 env->pc += 4;
                 do_m68k_semihosting(env, env->dregs[0]);
                 return;
@@ -151,44 +145,34 @@ static void do_interrupt_all(int is_hw)
     /* ??? This could cause MMU faults.  */
     sp &= ~3;
     sp -= 4;
-    stl_kernel(sp, retaddr);
+    cpu_stl_kernel(env, sp, retaddr);
     sp -= 4;
-    stl_kernel(sp, fmt);
+    cpu_stl_kernel(env, sp, fmt);
     env->aregs[7] = sp;
     /* Jump to vector.  */
-    env->pc = ldl_kernel(env->vbr + vector);
+    env->pc = cpu_ldl_kernel(env, env->vbr + vector);
 }
 
-void do_interrupt(CPUM68KState *env1)
+void do_interrupt(CPUM68KState *env)
 {
-    CPUM68KState *saved_env;
-
-    saved_env = env;
-    env = env1;
-    do_interrupt_all(0);
-    env = saved_env;
+    do_interrupt_all(env, 0);
 }
 
-void do_interrupt_m68k_hardirq(CPUM68KState *env1)
+void do_interrupt_m68k_hardirq(CPUM68KState *env)
 {
-    CPUM68KState *saved_env;
-
-    saved_env = env;
-    env = env1;
-    do_interrupt_all(1);
-    env = saved_env;
+    do_interrupt_all(env, 1);
 }
 #endif
 
-static void raise_exception(int tt)
+static void raise_exception(CPUM68KState *env, int tt)
 {
     env->exception_index = tt;
     cpu_loop_exit(env);
 }
 
-void HELPER(raise_exception)(uint32_t tt)
+void HELPER(raise_exception)(CPUM68KState *env, uint32_t tt)
 {
-    raise_exception(tt);
+    raise_exception(env, tt);
 }
 
 void HELPER(divu)(CPUM68KState *env, uint32_t word)
@@ -202,14 +186,12 @@ void HELPER(divu)(CPUM68KState *env, uint32_t word)
     num = env->div1;
     den = env->div2;
     /* ??? This needs to make sure the throwing location is accurate.  */
-    if (den == 0)
-        raise_exception(EXCP_DIV0);
+    if (den == 0) {
+        raise_exception(env, EXCP_DIV0);
+    }
     quot = num / den;
     rem = num % den;
     flags = 0;
-    /* Avoid using a PARAM1 of zero.  This breaks dyngen because it uses
-       the address of a symbol, and gcc knows symbols can't have address
-       zero.  */
     if (word && quot > 0xffff)
         flags |= CCF_V;
     if (quot == 0)
@@ -231,8 +213,9 @@ void HELPER(divs)(CPUM68KState *env, uint32_t word)
 
     num = env->div1;
     den = env->div2;
-    if (den == 0)
-        raise_exception(EXCP_DIV0);
+    if (den == 0) {
+        raise_exception(env, EXCP_DIV0);
+    }
     quot = num / den;
     rem = num % den;
     flags = 0;
diff --git a/target-m68k/translate.c b/target-m68k/translate.c
index 9fc1e3102..b13be4899 100644
--- a/target-m68k/translate.c
+++ b/target-m68k/translate.c
@@ -150,18 +150,24 @@ static void *gen_throws_exception;
 #define OS_SINGLE 4
 #define OS_DOUBLE 5
 
-typedef void (*disas_proc)(DisasContext *, uint16_t);
+typedef void (*disas_proc)(CPUM68KState *env, DisasContext *s, uint16_t insn);
 
 #ifdef DEBUG_DISPATCH
-#define DISAS_INSN(name) \
-  static void real_disas_##name (DisasContext *s, uint16_t insn); \
-  static void disas_##name (DisasContext *s, uint16_t insn) { \
-    qemu_log("Dispatch " #name "\n"); \
-    real_disas_##name(s, insn); } \
-  static void real_disas_##name (DisasContext *s, uint16_t insn)
+#define DISAS_INSN(name)                                                \
+    static void real_disas_##name(CPUM68KState *env, DisasContext *s,   \
+                                  uint16_t insn);                       \
+    static void disas_##name(CPUM68KState *env, DisasContext *s,        \
+                             uint16_t insn)                             \
+    {                                                                   \
+        qemu_log("Dispatch " #name "\n");                               \
+        real_disas_##name(s, env, insn);                                \
+    }                                                                   \
+    static void real_disas_##name(CPUM68KState *env, DisasContext *s,   \
+                                  uint16_t insn)
 #else
-#define DISAS_INSN(name) \
-  static void disas_##name (DisasContext *s, uint16_t insn)
+#define DISAS_INSN(name)                                                \
+    static void disas_##name(CPUM68KState *env, DisasContext *s,        \
+                             uint16_t insn)
 #endif
 
 /* Generate a load from the specified address.  Narrow values are
@@ -257,12 +263,12 @@ static TCGv gen_ldst(DisasContext *s, int opsize, TCGv addr, TCGv val,
 }
 
 /* Read a 32-bit immediate constant.  */
-static inline uint32_t read_im32(DisasContext *s)
+static inline uint32_t read_im32(CPUM68KState *env, DisasContext *s)
 {
     uint32_t im;
-    im = ((uint32_t)lduw_code(s->pc)) << 16;
+    im = ((uint32_t)cpu_lduw_code(env, s->pc)) << 16;
     s->pc += 2;
-    im |= lduw_code(s->pc);
+    im |= cpu_lduw_code(env, s->pc);
     s->pc += 2;
     return im;
 }
@@ -288,7 +294,8 @@ static TCGv gen_addr_index(uint16_t ext, TCGv tmp)
 
 /* Handle a base + index + displacement effective addresss.
    A NULL_QREG base means pc-relative.  */
-static TCGv gen_lea_indexed(DisasContext *s, int opsize, TCGv base)
+static TCGv gen_lea_indexed(CPUM68KState *env, DisasContext *s, int opsize,
+                            TCGv base)
 {
     uint32_t offset;
     uint16_t ext;
@@ -297,7 +304,7 @@ static TCGv gen_lea_indexed(DisasContext *s, int opsize, TCGv base)
     uint32_t bd, od;
 
     offset = s->pc;
-    ext = lduw_code(s->pc);
+    ext = cpu_lduw_code(env, s->pc);
     s->pc += 2;
 
     if ((ext & 0x800) == 0 && !m68k_feature(s->env, M68K_FEATURE_WORD_INDEX))
@@ -311,10 +318,10 @@ static TCGv gen_lea_indexed(DisasContext *s, int opsize, TCGv base)
         if ((ext & 0x30) > 0x10) {
             /* base displacement */
             if ((ext & 0x30) == 0x20) {
-                bd = (int16_t)lduw_code(s->pc);
+                bd = (int16_t)cpu_lduw_code(env, s->pc);
                 s->pc += 2;
             } else {
-                bd = read_im32(s);
+                bd = read_im32(env, s);
             }
         } else {
             bd = 0;
@@ -360,10 +367,10 @@ static TCGv gen_lea_indexed(DisasContext *s, int opsize, TCGv base)
             if ((ext & 3) > 1) {
                 /* outer displacement */
                 if ((ext & 3) == 2) {
-                    od = (int16_t)lduw_code(s->pc);
+                    od = (int16_t)cpu_lduw_code(env, s->pc);
                     s->pc += 2;
                 } else {
-                    od = read_im32(s);
+                    od = read_im32(env, s);
                 }
             } else {
                 od = 0;
@@ -492,7 +499,8 @@ static inline TCGv gen_extend(TCGv val, int opsize, int sign)
 
 /* Generate code for an "effective address".  Does not adjust the base
    register for autoincrement addressing modes.  */
-static TCGv gen_lea(DisasContext *s, uint16_t insn, int opsize)
+static TCGv gen_lea(CPUM68KState *env, DisasContext *s, uint16_t insn,
+                    int opsize)
 {
     TCGv reg;
     TCGv tmp;
@@ -514,29 +522,29 @@ static TCGv gen_lea(DisasContext *s, uint16_t insn, int opsize)
     case 5: /* Indirect displacement.  */
         reg = AREG(insn, 0);
         tmp = tcg_temp_new();
-        ext = lduw_code(s->pc);
+        ext = cpu_lduw_code(env, s->pc);
         s->pc += 2;
         tcg_gen_addi_i32(tmp, reg, (int16_t)ext);
         return tmp;
     case 6: /* Indirect index + displacement.  */
         reg = AREG(insn, 0);
-        return gen_lea_indexed(s, opsize, reg);
+        return gen_lea_indexed(env, s, opsize, reg);
     case 7: /* Other */
         switch (insn & 7) {
         case 0: /* Absolute short.  */
-            offset = ldsw_code(s->pc);
+            offset = cpu_ldsw_code(env, s->pc);
             s->pc += 2;
             return tcg_const_i32(offset);
         case 1: /* Absolute long.  */
-            offset = read_im32(s);
+            offset = read_im32(env, s);
             return tcg_const_i32(offset);
         case 2: /* pc displacement  */
             offset = s->pc;
-            offset += ldsw_code(s->pc);
+            offset += cpu_ldsw_code(env, s->pc);
             s->pc += 2;
             return tcg_const_i32(offset);
         case 3: /* pc index+displacement.  */
-            return gen_lea_indexed(s, opsize, NULL_QREG);
+            return gen_lea_indexed(env, s, opsize, NULL_QREG);
         case 4: /* Immediate.  */
         default:
             return NULL_QREG;
@@ -548,15 +556,16 @@ static TCGv gen_lea(DisasContext *s, uint16_t insn, int opsize)
 
 /* Helper function for gen_ea. Reuse the computed address between the
    for read/write operands.  */
-static inline TCGv gen_ea_once(DisasContext *s, uint16_t insn, int opsize,
-                              TCGv val, TCGv *addrp, ea_what what)
+static inline TCGv gen_ea_once(CPUM68KState *env, DisasContext *s,
+                               uint16_t insn, int opsize, TCGv val,
+                               TCGv *addrp, ea_what what)
 {
     TCGv tmp;
 
     if (addrp && what == EA_STORE) {
         tmp = *addrp;
     } else {
-        tmp = gen_lea(s, insn, opsize);
+        tmp = gen_lea(env, s, insn, opsize);
         if (IS_NULL_QREG(tmp))
             return tmp;
         if (addrp)
@@ -568,8 +577,8 @@ static inline TCGv gen_ea_once(DisasContext *s, uint16_t insn, int opsize,
 /* Generate code to load/store a value ito/from an EA.  If VAL > 0 this is
    a write otherwise it is a read (0 == sign extend, -1 == zero extend).
    ADDRP is non-null for readwrite operands.  */
-static TCGv gen_ea(DisasContext *s, uint16_t insn, int opsize, TCGv val,
-                   TCGv *addrp, ea_what what)
+static TCGv gen_ea(CPUM68KState *env, DisasContext *s, uint16_t insn,
+                   int opsize, TCGv val, TCGv *addrp, ea_what what)
 {
     TCGv reg;
     TCGv result;
@@ -609,7 +618,7 @@ static TCGv gen_ea(DisasContext *s, uint16_t insn, int opsize, TCGv val,
             if (addrp && what == EA_STORE) {
                 tmp = *addrp;
             } else {
-                tmp = gen_lea(s, insn, opsize);
+                tmp = gen_lea(env, s, insn, opsize);
                 if (IS_NULL_QREG(tmp))
                     return tmp;
                 if (addrp)
@@ -626,33 +635,35 @@ static TCGv gen_ea(DisasContext *s, uint16_t insn, int opsize, TCGv val,
         return result;
     case 5: /* Indirect displacement.  */
     case 6: /* Indirect index + displacement.  */
-        return gen_ea_once(s, insn, opsize, val, addrp, what);
+        return gen_ea_once(env, s, insn, opsize, val, addrp, what);
     case 7: /* Other */
         switch (insn & 7) {
         case 0: /* Absolute short.  */
         case 1: /* Absolute long.  */
         case 2: /* pc displacement  */
         case 3: /* pc index+displacement.  */
-            return gen_ea_once(s, insn, opsize, val, addrp, what);
+            return gen_ea_once(env, s, insn, opsize, val, addrp, what);
         case 4: /* Immediate.  */
             /* Sign extend values for consistency.  */
             switch (opsize) {
             case OS_BYTE:
-                if (what == EA_LOADS)
-                    offset = ldsb_code(s->pc + 1);
-                else
-                    offset = ldub_code(s->pc + 1);
+                if (what == EA_LOADS) {
+                    offset = cpu_ldsb_code(env, s->pc + 1);
+                } else {
+                    offset = cpu_ldub_code(env, s->pc + 1);
+                }
                 s->pc += 2;
                 break;
             case OS_WORD:
-                if (what == EA_LOADS)
-                    offset = ldsw_code(s->pc);
-                else
-                    offset = lduw_code(s->pc);
+                if (what == EA_LOADS) {
+                    offset = cpu_ldsw_code(env, s->pc);
+                } else {
+                    offset = cpu_lduw_code(env, s->pc);
+                }
                 s->pc += 2;
                 break;
             case OS_LONG:
-                offset = read_im32(s);
+                offset = read_im32(env, s);
                 break;
             default:
                 qemu_assert(0, "Bad immediate operand");
@@ -815,7 +826,7 @@ static void gen_exception(DisasContext *s, uint32_t where, int nr)
 {
     gen_flush_cc_op(s);
     gen_jmp_im(s, where);
-    gen_helper_raise_exception(tcg_const_i32(nr));
+    gen_helper_raise_exception(cpu_env, tcg_const_i32(nr));
 }
 
 static inline void gen_addr_fault(DisasContext *s)
@@ -823,20 +834,21 @@ static inline void gen_addr_fault(DisasContext *s)
     gen_exception(s, s->insn_pc, EXCP_ADDRESS);
 }
 
-#define SRC_EA(result, opsize, op_sign, addrp) do { \
-    result = gen_ea(s, insn, opsize, NULL_QREG, addrp, op_sign ? EA_LOADS : EA_LOADU); \
-    if (IS_NULL_QREG(result)) { \
-        gen_addr_fault(s); \
-        return; \
-    } \
+#define SRC_EA(env, result, opsize, op_sign, addrp) do {                \
+        result = gen_ea(env, s, insn, opsize, NULL_QREG, addrp,         \
+                        op_sign ? EA_LOADS : EA_LOADU);                 \
+        if (IS_NULL_QREG(result)) {                                     \
+            gen_addr_fault(s);                                          \
+            return;                                                     \
+        }                                                               \
     } while (0)
 
-#define DEST_EA(insn, opsize, val, addrp) do { \
-    TCGv ea_result = gen_ea(s, insn, opsize, val, addrp, EA_STORE); \
-    if (IS_NULL_QREG(ea_result)) { \
-        gen_addr_fault(s); \
-        return; \
-    } \
+#define DEST_EA(env, insn, opsize, val, addrp) do {                     \
+        TCGv ea_result = gen_ea(env, s, insn, opsize, val, addrp, EA_STORE); \
+        if (IS_NULL_QREG(ea_result)) {                                  \
+            gen_addr_fault(s);                                          \
+            return;                                                     \
+        }                                                               \
     } while (0)
 
 /* Generate a jump to an immediate address.  */
@@ -872,8 +884,7 @@ DISAS_INSN(undef_fpu)
 DISAS_INSN(undef)
 {
     gen_exception(s, s->pc - 2, EXCP_UNSUPPORTED);
-    cpu_abort(cpu_single_env, "Illegal instruction: %04x @ %08x",
-              insn, s->pc - 2);
+    cpu_abort(env, "Illegal instruction: %04x @ %08x", insn, s->pc - 2);
 }
 
 DISAS_INSN(mulw)
@@ -890,7 +901,7 @@ DISAS_INSN(mulw)
         tcg_gen_ext16s_i32(tmp, reg);
     else
         tcg_gen_ext16u_i32(tmp, reg);
-    SRC_EA(src, OS_WORD, sign, NULL);
+    SRC_EA(env, src, OS_WORD, sign, NULL);
     tcg_gen_mul_i32(tmp, tmp, src);
     tcg_gen_mov_i32(reg, tmp);
     /* Unlike m68k, coldfire always clears the overflow bit.  */
@@ -911,7 +922,7 @@ DISAS_INSN(divw)
     } else {
         tcg_gen_ext16u_i32(QREG_DIV1, reg);
     }
-    SRC_EA(src, OS_WORD, sign, NULL);
+    SRC_EA(env, src, OS_WORD, sign, NULL);
     tcg_gen_mov_i32(QREG_DIV2, src);
     if (sign) {
         gen_helper_divs(cpu_env, tcg_const_i32(1));
@@ -934,7 +945,7 @@ DISAS_INSN(divl)
     TCGv reg;
     uint16_t ext;
 
-    ext = lduw_code(s->pc);
+    ext = cpu_lduw_code(env, s->pc);
     s->pc += 2;
     if (ext & 0x87f8) {
         gen_exception(s, s->pc - 4, EXCP_UNSUPPORTED);
@@ -943,7 +954,7 @@ DISAS_INSN(divl)
     num = DREG(ext, 12);
     reg = DREG(ext, 0);
     tcg_gen_mov_i32(QREG_DIV1, num);
-    SRC_EA(den, OS_LONG, 0, NULL);
+    SRC_EA(env, den, OS_LONG, 0, NULL);
     tcg_gen_mov_i32(QREG_DIV2, den);
     if (ext & 0x0800) {
         gen_helper_divs(cpu_env, tcg_const_i32(0));
@@ -973,11 +984,11 @@ DISAS_INSN(addsub)
     reg = DREG(insn, 9);
     dest = tcg_temp_new();
     if (insn & 0x100) {
-        SRC_EA(tmp, OS_LONG, 0, &addr);
+        SRC_EA(env, tmp, OS_LONG, 0, &addr);
         src = reg;
     } else {
         tmp = reg;
-        SRC_EA(src, OS_LONG, 0, NULL);
+        SRC_EA(env, src, OS_LONG, 0, NULL);
     }
     if (add) {
         tcg_gen_add_i32(dest, tmp, src);
@@ -990,7 +1001,7 @@ DISAS_INSN(addsub)
     }
     gen_update_cc_add(dest, src);
     if (insn & 0x100) {
-        DEST_EA(insn, OS_LONG, dest, &addr);
+        DEST_EA(env, insn, OS_LONG, dest, &addr);
     } else {
         tcg_gen_mov_i32(reg, dest);
     }
@@ -1020,7 +1031,7 @@ DISAS_INSN(bitop_reg)
     else
         opsize = OS_LONG;
     op = (insn >> 6) & 3;
-    SRC_EA(src1, opsize, 0, op ? &addr: NULL);
+    SRC_EA(env, src1, opsize, 0, op ? &addr: NULL);
     src2 = DREG(insn, 9);
     dest = tcg_temp_new();
 
@@ -1055,7 +1066,7 @@ DISAS_INSN(bitop_reg)
         break;
     }
     if (op)
-        DEST_EA(insn, opsize, dest, &addr);
+        DEST_EA(env, insn, opsize, dest, &addr);
 }
 
 DISAS_INSN(sats)
@@ -1086,9 +1097,9 @@ DISAS_INSN(movem)
     TCGv tmp;
     int is_load;
 
-    mask = lduw_code(s->pc);
+    mask = cpu_lduw_code(env, s->pc);
     s->pc += 2;
-    tmp = gen_lea(s, insn, OS_LONG);
+    tmp = gen_lea(env, s, insn, OS_LONG);
     if (IS_NULL_QREG(tmp)) {
         gen_addr_fault(s);
         return;
@@ -1130,14 +1141,14 @@ DISAS_INSN(bitop_im)
         opsize = OS_LONG;
     op = (insn >> 6) & 3;
 
-    bitnum = lduw_code(s->pc);
+    bitnum = cpu_lduw_code(env, s->pc);
     s->pc += 2;
     if (bitnum & 0xff00) {
-        disas_undef(s, insn);
+        disas_undef(env, s, insn);
         return;
     }
 
-    SRC_EA(src1, opsize, 0, op ? &addr: NULL);
+    SRC_EA(env, src1, opsize, 0, op ? &addr: NULL);
 
     gen_flush_flags(s);
     if (opsize == OS_BYTE)
@@ -1172,7 +1183,7 @@ DISAS_INSN(bitop_im)
         default: /* btst */
             break;
         }
-        DEST_EA(insn, opsize, tmp, &addr);
+        DEST_EA(env, insn, opsize, tmp, &addr);
     }
 }
 
@@ -1185,8 +1196,8 @@ DISAS_INSN(arith_im)
     TCGv addr;
 
     op = (insn >> 9) & 7;
-    SRC_EA(src1, OS_LONG, 0, (op == 6) ? NULL : &addr);
-    im = read_im32(s);
+    SRC_EA(env, src1, OS_LONG, 0, (op == 6) ? NULL : &addr);
+    im = read_im32(env, s);
     dest = tcg_temp_new();
     switch (op) {
     case 0: /* ori */
@@ -1225,7 +1236,7 @@ DISAS_INSN(arith_im)
         abort();
     }
     if (op != 6) {
-        DEST_EA(insn, OS_LONG, dest, &addr);
+        DEST_EA(env, insn, OS_LONG, dest, &addr);
     }
 }
 
@@ -1257,7 +1268,7 @@ DISAS_INSN(move)
     default:
         abort();
     }
-    SRC_EA(src, opsize, 1, NULL);
+    SRC_EA(env, src, opsize, 1, NULL);
     op = (insn >> 6) & 7;
     if (op == 1) {
         /* movea */
@@ -1268,7 +1279,7 @@ DISAS_INSN(move)
         /* normal move */
         uint16_t dest_ea;
         dest_ea = ((insn >> 9) & 7) | (op << 3);
-        DEST_EA(dest_ea, opsize, src, NULL);
+        DEST_EA(env, dest_ea, opsize, src, NULL);
         /* This will be correct because loads sign extend.  */
         gen_logic_cc(s, src);
     }
@@ -1289,7 +1300,7 @@ DISAS_INSN(lea)
     TCGv tmp;
 
     reg = AREG(insn, 9);
-    tmp = gen_lea(s, insn, OS_LONG);
+    tmp = gen_lea(env, s, insn, OS_LONG);
     if (IS_NULL_QREG(tmp)) {
         gen_addr_fault(s);
         return;
@@ -1314,7 +1325,7 @@ DISAS_INSN(clr)
     default:
         abort();
     }
-    DEST_EA(insn, opsize, tcg_const_i32(0), NULL);
+    DEST_EA(env, insn, opsize, tcg_const_i32(0), NULL);
     gen_logic_cc(s, tcg_const_i32(0));
 }
 
@@ -1363,7 +1374,8 @@ static void gen_set_sr_im(DisasContext *s, uint16_t val, int ccr_only)
     }
 }
 
-static void gen_set_sr(DisasContext *s, uint16_t insn, int ccr_only)
+static void gen_set_sr(CPUM68KState *env, DisasContext *s, uint16_t insn,
+                       int ccr_only)
 {
     TCGv tmp;
     TCGv reg;
@@ -1383,17 +1395,17 @@ static void gen_set_sr(DisasContext *s, uint16_t insn, int ccr_only)
     else if ((insn & 0x3f) == 0x3c)
       {
         uint16_t val;
-        val = lduw_code(s->pc);
+        val = cpu_lduw_code(env, s->pc);
         s->pc += 2;
         gen_set_sr_im(s, val, ccr_only);
       }
     else
-        disas_undef(s, insn);
+        disas_undef(env, s, insn);
 }
 
 DISAS_INSN(move_to_ccr)
 {
-    gen_set_sr(s, insn, 1);
+    gen_set_sr(env, s, insn, 1);
 }
 
 DISAS_INSN(not)
@@ -1424,7 +1436,7 @@ DISAS_INSN(pea)
 {
     TCGv tmp;
 
-    tmp = gen_lea(s, insn, OS_LONG);
+    tmp = gen_lea(env, s, insn, OS_LONG);
     if (IS_NULL_QREG(tmp)) {
         gen_addr_fault(s);
         return;
@@ -1470,7 +1482,7 @@ DISAS_INSN(tst)
     default:
         abort();
     }
-    SRC_EA(tmp, opsize, 1, NULL);
+    SRC_EA(env, tmp, opsize, 1, NULL);
     gen_logic_cc(s, tmp);
 }
 
@@ -1492,10 +1504,10 @@ DISAS_INSN(tas)
     TCGv addr;
 
     dest = tcg_temp_new();
-    SRC_EA(src1, OS_BYTE, 1, &addr);
+    SRC_EA(env, src1, OS_BYTE, 1, &addr);
     gen_logic_cc(s, src1);
     tcg_gen_ori_i32(dest, src1, 0x80);
-    DEST_EA(insn, OS_BYTE, dest, &addr);
+    DEST_EA(env, insn, OS_BYTE, dest, &addr);
 }
 
 DISAS_INSN(mull)
@@ -1507,14 +1519,14 @@ DISAS_INSN(mull)
 
     /* The upper 32 bits of the product are discarded, so
        muls.l and mulu.l are functionally equivalent.  */
-    ext = lduw_code(s->pc);
+    ext = cpu_lduw_code(env, s->pc);
     s->pc += 2;
     if (ext & 0x87ff) {
         gen_exception(s, s->pc - 4, EXCP_UNSUPPORTED);
         return;
     }
     reg = DREG(ext, 12);
-    SRC_EA(src1, OS_LONG, 0, NULL);
+    SRC_EA(env, src1, OS_LONG, 0, NULL);
     dest = tcg_temp_new();
     tcg_gen_mul_i32(dest, src1, reg);
     tcg_gen_mov_i32(reg, dest);
@@ -1528,7 +1540,7 @@ DISAS_INSN(link)
     TCGv reg;
     TCGv tmp;
 
-    offset = ldsw_code(s->pc);
+    offset = cpu_ldsw_code(env, s->pc);
     s->pc += 2;
     reg = AREG(insn, 0);
     tmp = tcg_temp_new();
@@ -1572,7 +1584,7 @@ DISAS_INSN(jump)
 
     /* Load the target address first to ensure correct exception
        behavior.  */
-    tmp = gen_lea(s, insn, OS_LONG);
+    tmp = gen_lea(env, s, insn, OS_LONG);
     if (IS_NULL_QREG(tmp)) {
         gen_addr_fault(s);
         return;
@@ -1592,7 +1604,7 @@ DISAS_INSN(addsubq)
     int val;
     TCGv addr;
 
-    SRC_EA(src1, OS_LONG, 0, &addr);
+    SRC_EA(env, src1, OS_LONG, 0, &addr);
     val = (insn >> 9) & 7;
     if (val == 0)
         val = 8;
@@ -1619,7 +1631,7 @@ DISAS_INSN(addsubq)
         }
         gen_update_cc_add(dest, src2);
     }
-    DEST_EA(insn, OS_LONG, dest, &addr);
+    DEST_EA(env, insn, OS_LONG, dest, &addr);
 }
 
 DISAS_INSN(tpf)
@@ -1634,7 +1646,7 @@ DISAS_INSN(tpf)
     case 4: /* No extension words.  */
         break;
     default:
-        disas_undef(s, insn);
+        disas_undef(env, s, insn);
     }
 }
 
@@ -1649,10 +1661,10 @@ DISAS_INSN(branch)
     op = (insn >> 8) & 0xf;
     offset = (int8_t)insn;
     if (offset == 0) {
-        offset = ldsw_code(s->pc);
+        offset = cpu_ldsw_code(env, s->pc);
         s->pc += 2;
     } else if (offset == -1) {
-        offset = read_im32(s);
+        offset = read_im32(env, s);
     }
     if (op == 1) {
         /* bsr */
@@ -1691,7 +1703,7 @@ DISAS_INSN(mvzs)
         opsize = OS_WORD;
     else
         opsize = OS_BYTE;
-    SRC_EA(src, opsize, (insn & 0x80) == 0, NULL);
+    SRC_EA(env, src, opsize, (insn & 0x80) == 0, NULL);
     reg = DREG(insn, 9);
     tcg_gen_mov_i32(reg, src);
     gen_logic_cc(s, src);
@@ -1707,11 +1719,11 @@ DISAS_INSN(or)
     reg = DREG(insn, 9);
     dest = tcg_temp_new();
     if (insn & 0x100) {
-        SRC_EA(src, OS_LONG, 0, &addr);
+        SRC_EA(env, src, OS_LONG, 0, &addr);
         tcg_gen_or_i32(dest, src, reg);
-        DEST_EA(insn, OS_LONG, dest, &addr);
+        DEST_EA(env, insn, OS_LONG, dest, &addr);
     } else {
-        SRC_EA(src, OS_LONG, 0, NULL);
+        SRC_EA(env, src, OS_LONG, 0, NULL);
         tcg_gen_or_i32(dest, src, reg);
         tcg_gen_mov_i32(reg, dest);
     }
@@ -1723,7 +1735,7 @@ DISAS_INSN(suba)
     TCGv src;
     TCGv reg;
 
-    SRC_EA(src, OS_LONG, 0, NULL);
+    SRC_EA(env, src, OS_LONG, 0, NULL);
     reg = AREG(insn, 9);
     tcg_gen_sub_i32(reg, reg, src);
 }
@@ -1749,7 +1761,7 @@ DISAS_INSN(mov3q)
         val = -1;
     src = tcg_const_i32(val);
     gen_logic_cc(s, src);
-    DEST_EA(insn, OS_LONG, src, NULL);
+    DEST_EA(env, insn, OS_LONG, src, NULL);
 }
 
 DISAS_INSN(cmp)
@@ -1777,7 +1789,7 @@ DISAS_INSN(cmp)
     default:
         abort();
     }
-    SRC_EA(src, opsize, 1, NULL);
+    SRC_EA(env, src, opsize, 1, NULL);
     reg = DREG(insn, 9);
     dest = tcg_temp_new();
     tcg_gen_sub_i32(dest, reg, src);
@@ -1796,7 +1808,7 @@ DISAS_INSN(cmpa)
     } else {
         opsize = OS_WORD;
     }
-    SRC_EA(src, opsize, 1, NULL);
+    SRC_EA(env, src, opsize, 1, NULL);
     reg = AREG(insn, 9);
     dest = tcg_temp_new();
     tcg_gen_sub_i32(dest, reg, src);
@@ -1811,12 +1823,12 @@ DISAS_INSN(eor)
     TCGv dest;
     TCGv addr;
 
-    SRC_EA(src, OS_LONG, 0, &addr);
+    SRC_EA(env, src, OS_LONG, 0, &addr);
     reg = DREG(insn, 9);
     dest = tcg_temp_new();
     tcg_gen_xor_i32(dest, src, reg);
     gen_logic_cc(s, dest);
-    DEST_EA(insn, OS_LONG, dest, &addr);
+    DEST_EA(env, insn, OS_LONG, dest, &addr);
 }
 
 DISAS_INSN(and)
@@ -1829,11 +1841,11 @@ DISAS_INSN(and)
     reg = DREG(insn, 9);
     dest = tcg_temp_new();
     if (insn & 0x100) {
-        SRC_EA(src, OS_LONG, 0, &addr);
+        SRC_EA(env, src, OS_LONG, 0, &addr);
         tcg_gen_and_i32(dest, src, reg);
-        DEST_EA(insn, OS_LONG, dest, &addr);
+        DEST_EA(env, insn, OS_LONG, dest, &addr);
     } else {
-        SRC_EA(src, OS_LONG, 0, NULL);
+        SRC_EA(env, src, OS_LONG, 0, NULL);
         tcg_gen_and_i32(dest, src, reg);
         tcg_gen_mov_i32(reg, dest);
     }
@@ -1845,7 +1857,7 @@ DISAS_INSN(adda)
     TCGv src;
     TCGv reg;
 
-    SRC_EA(src, OS_LONG, 0, NULL);
+    SRC_EA(env, src, OS_LONG, 0, NULL);
     reg = AREG(insn, 9);
     tcg_gen_add_i32(reg, reg, src);
 }
@@ -1934,13 +1946,13 @@ DISAS_INSN(strldsr)
     uint32_t addr;
 
     addr = s->pc - 2;
-    ext = lduw_code(s->pc);
+    ext = cpu_lduw_code(env, s->pc);
     s->pc += 2;
     if (ext != 0x46FC) {
         gen_exception(s, addr, EXCP_UNSUPPORTED);
         return;
     }
-    ext = lduw_code(s->pc);
+    ext = cpu_lduw_code(env, s->pc);
     s->pc += 2;
     if (IS_USER(s) || (ext & SR_S) == 0) {
         gen_exception(s, addr, EXCP_PRIVILEGE);
@@ -1970,7 +1982,7 @@ DISAS_INSN(move_to_sr)
         gen_exception(s, s->pc - 2, EXCP_PRIVILEGE);
         return;
     }
-    gen_set_sr(s, insn, 0);
+    gen_set_sr(env, s, insn, 0);
     gen_lookup_tb(s);
 }
 
@@ -2008,7 +2020,7 @@ DISAS_INSN(stop)
         return;
     }
 
-    ext = lduw_code(s->pc);
+    ext = cpu_lduw_code(env, s->pc);
     s->pc += 2;
 
     gen_set_sr_im(s, ext, 0);
@@ -2035,7 +2047,7 @@ DISAS_INSN(movec)
         return;
     }
 
-    ext = lduw_code(s->pc);
+    ext = cpu_lduw_code(env, s->pc);
     s->pc += 2;
 
     if (ext & 0x8000) {
@@ -2100,7 +2112,7 @@ DISAS_INSN(fpu)
     int set_dest;
     int opsize;
 
-    ext = lduw_code(s->pc);
+    ext = cpu_lduw_code(env, s->pc);
     s->pc += 2;
     opmode = ext & 0x7f;
     switch ((ext >> 13) & 7) {
@@ -2136,7 +2148,7 @@ DISAS_INSN(fpu)
                 tcg_gen_addi_i32(tmp32, tmp32, -8);
                 break;
             case 5:
-                offset = ldsw_code(s->pc);
+                offset = cpu_ldsw_code(env, s->pc);
                 s->pc += 2;
                 tcg_gen_addi_i32(tmp32, tmp32, offset);
                 break;
@@ -2162,7 +2174,7 @@ DISAS_INSN(fpu)
         default:
             goto undef;
         }
-        DEST_EA(insn, opsize, tmp32, NULL);
+        DEST_EA(env, insn, opsize, tmp32, NULL);
         tcg_temp_free_i32(tmp32);
         return;
     case 4: /* fmove to control register.  */
@@ -2190,7 +2202,7 @@ DISAS_INSN(fpu)
                       (ext >> 10) & 7);
             goto undef;
         }
-        DEST_EA(insn, OS_LONG, tmp32, NULL);
+        DEST_EA(env, insn, OS_LONG, tmp32, NULL);
         break;
     case 6: /* fmovem */
     case 7:
@@ -2200,7 +2212,7 @@ DISAS_INSN(fpu)
             int i;
             if ((ext & 0x1f00) != 0x1000 || (ext & 0xff) == 0)
                 goto undef;
-            tmp32 = gen_lea(s, insn, OS_LONG);
+            tmp32 = gen_lea(env, s, insn, OS_LONG);
             if (IS_NULL_QREG(tmp32)) {
                 gen_addr_fault(s);
                 return;
@@ -2250,12 +2262,12 @@ DISAS_INSN(fpu)
                 tcg_gen_addi_i32(tmp32, tmp32, -8);
                 break;
             case 5:
-                offset = ldsw_code(s->pc);
+                offset = cpu_ldsw_code(env, s->pc);
                 s->pc += 2;
                 tcg_gen_addi_i32(tmp32, tmp32, offset);
                 break;
             case 7:
-                offset = ldsw_code(s->pc);
+                offset = cpu_ldsw_code(env, s->pc);
                 offset += s->pc - 2;
                 s->pc += 2;
                 tcg_gen_addi_i32(tmp32, tmp32, offset);
@@ -2275,7 +2287,7 @@ DISAS_INSN(fpu)
             }
             tcg_temp_free_i32(tmp32);
         } else {
-            SRC_EA(tmp32, opsize, 1, NULL);
+            SRC_EA(env, tmp32, opsize, 1, NULL);
             src = tcg_temp_new_i64();
             switch (opsize) {
             case OS_LONG:
@@ -2370,7 +2382,7 @@ DISAS_INSN(fpu)
 undef:
     /* FIXME: Is this right for offset addressing modes?  */
     s->pc -= 2;
-    disas_undef_fpu(s, insn);
+    disas_undef_fpu(env, s, insn);
 }
 
 DISAS_INSN(fbcc)
@@ -2381,10 +2393,10 @@ DISAS_INSN(fbcc)
     int l1;
 
     addr = s->pc;
-    offset = ldsw_code(s->pc);
+    offset = cpu_ldsw_code(env, s->pc);
     s->pc += 2;
     if (insn & (1 << 6)) {
-        offset = (offset << 16) | lduw_code(s->pc);
+        offset = (offset << 16) | cpu_lduw_code(env, s->pc);
         s->pc += 2;
     }
 
@@ -2506,18 +2518,18 @@ DISAS_INSN(mac)
         s->done_mac = 1;
     }
 
-    ext = lduw_code(s->pc);
+    ext = cpu_lduw_code(env, s->pc);
     s->pc += 2;
 
     acc = ((insn >> 7) & 1) | ((ext >> 3) & 2);
     dual = ((insn & 0x30) != 0 && (ext & 3) != 0);
     if (dual && !m68k_feature(s->env, M68K_FEATURE_CF_EMAC_B)) {
-        disas_undef(s, insn);
+        disas_undef(env, s, insn);
         return;
     }
     if (insn & 0x30) {
         /* MAC with load.  */
-        tmp = gen_lea(s, insn, OS_LONG);
+        tmp = gen_lea(env, s, insn, OS_LONG);
         addr = tcg_temp_new();
         tcg_gen_and_i32(addr, tmp, QREG_MAC_MASK);
         /* Load the value now to ensure correct exception behavior.
@@ -2731,7 +2743,7 @@ DISAS_INSN(to_mac)
     int accnum;
     accnum = (insn >> 9) & 3;
     acc = MACREG(accnum);
-    SRC_EA(val, OS_LONG, 0, NULL);
+    SRC_EA(env, val, OS_LONG, 0, NULL);
     if (s->env->macsr & MACSR_FI) {
         tcg_gen_ext_i32_i64(acc, val);
         tcg_gen_shli_i64(acc, acc, 8);
@@ -2748,7 +2760,7 @@ DISAS_INSN(to_mac)
 DISAS_INSN(to_macsr)
 {
     TCGv val;
-    SRC_EA(val, OS_LONG, 0, NULL);
+    SRC_EA(env, val, OS_LONG, 0, NULL);
     gen_helper_set_macsr(cpu_env, val);
     gen_lookup_tb(s);
 }
@@ -2756,7 +2768,7 @@ DISAS_INSN(to_macsr)
 DISAS_INSN(to_mask)
 {
     TCGv val;
-    SRC_EA(val, OS_LONG, 0, NULL);
+    SRC_EA(env, val, OS_LONG, 0, NULL);
     tcg_gen_ori_i32(QREG_MAC_MASK, val, 0xffff0000);
 }
 
@@ -2764,7 +2776,7 @@ DISAS_INSN(to_mext)
 {
     TCGv val;
     TCGv acc;
-    SRC_EA(val, OS_LONG, 0, NULL);
+    SRC_EA(env, val, OS_LONG, 0, NULL);
     acc = tcg_const_i32((insn & 0x400) ? 2 : 0);
     if (s->env->macsr & MACSR_FI)
         gen_helper_set_mac_extf(cpu_env, val, acc);
@@ -2941,10 +2953,14 @@ static void disas_m68k_insn(CPUM68KState * env, DisasContext *s)
 {
     uint16_t insn;
 
-    insn = lduw_code(s->pc);
+    if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
+        tcg_gen_debug_insn_start(s->pc);
+    }
+
+    insn = cpu_lduw_code(env, s->pc);
     s->pc += 2;
 
-    opcode_table[insn](s, insn);
+    opcode_table[insn](env, s, insn);
 }
 
 /* generate intermediate code for basic block 'tb'.  */
@@ -2966,7 +2982,7 @@ gen_intermediate_code_internal(CPUM68KState *env, TranslationBlock *tb,
 
     dc->tb = tb;
 
-    gen_opc_end = gen_opc_buf + OPC_MAX_SIZE;
+    gen_opc_end = tcg_ctx.gen_opc_buf + OPC_MAX_SIZE;
 
     dc->env = env;
     dc->is_jmp = DISAS_NEXT;
@@ -2999,7 +3015,7 @@ gen_intermediate_code_internal(CPUM68KState *env, TranslationBlock *tb,
                 break;
         }
         if (search_pc) {
-            j = gen_opc_ptr - gen_opc_buf;
+            j = tcg_ctx.gen_opc_ptr - tcg_ctx.gen_opc_buf;
             if (lj < j) {
                 lj++;
                 while (lj < j)
@@ -3014,7 +3030,7 @@ gen_intermediate_code_internal(CPUM68KState *env, TranslationBlock *tb,
         dc->insn_pc = dc->pc;
 	disas_m68k_insn(env, dc);
         num_insns++;
-    } while (!dc->is_jmp && gen_opc_ptr < gen_opc_end &&
+    } while (!dc->is_jmp && tcg_ctx.gen_opc_ptr < gen_opc_end &&
              !env->singlestep_enabled &&
              !singlestep &&
              (pc_offset) < (TARGET_PAGE_SIZE - 32) &&
@@ -3028,7 +3044,7 @@ gen_intermediate_code_internal(CPUM68KState *env, TranslationBlock *tb,
             gen_flush_cc_op(dc);
             tcg_gen_movi_i32(QREG_PC, dc->pc);
         }
-        gen_helper_raise_exception(tcg_const_i32(EXCP_DEBUG));
+        gen_helper_raise_exception(cpu_env, tcg_const_i32(EXCP_DEBUG));
     } else {
         switch(dc->is_jmp) {
         case DISAS_NEXT:
@@ -3048,18 +3064,18 @@ gen_intermediate_code_internal(CPUM68KState *env, TranslationBlock *tb,
         }
     }
     gen_icount_end(tb, num_insns);
-    *gen_opc_ptr = INDEX_op_end;
+    *tcg_ctx.gen_opc_ptr = INDEX_op_end;
 
 #ifdef DEBUG_DISAS
     if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
         qemu_log("----------------\n");
         qemu_log("IN: %s\n", lookup_symbol(pc_start));
-        log_target_disas(pc_start, dc->pc - pc_start, 0);
+        log_target_disas(env, pc_start, dc->pc - pc_start, 0);
         qemu_log("\n");
     }
 #endif
     if (search_pc) {
-        j = gen_opc_ptr - gen_opc_buf;
+        j = tcg_ctx.gen_opc_ptr - tcg_ctx.gen_opc_buf;
         lj++;
         while (lj <= j)
             gen_opc_instr_start[lj++] = 0;
diff --git a/target-microblaze/Makefile.objs b/target-microblaze/Makefile.objs
index 4b09e8c6b..afb87bcc8 100644
--- a/target-microblaze/Makefile.objs
+++ b/target-microblaze/Makefile.objs
@@ -1,4 +1,2 @@
 obj-y += translate.o op_helper.o helper.o cpu.o
 obj-$(CONFIG_SOFTMMU) += mmu.o machine.o
-
-$(obj)/op_helper.o: QEMU_CFLAGS += $(HELPER_CFLAGS)
diff --git a/target-microblaze/cpu.h b/target-microblaze/cpu.h
index 4968c244e..585bbd6db 100644
--- a/target-microblaze/cpu.h
+++ b/target-microblaze/cpu.h
@@ -345,6 +345,7 @@ static inline void cpu_clone_regs(CPUMBState *env, target_ulong newsp)
 
 static inline void cpu_set_tls(CPUMBState *env, target_ulong newtls)
 {
+    env->regs[21] = newtls;
 }
 
 static inline int cpu_interrupts_enabled(CPUMBState *env)
@@ -369,12 +370,14 @@ static inline void cpu_get_tb_cpu_state(CPUMBState *env, target_ulong *pc,
 }
 
 #if !defined(CONFIG_USER_ONLY)
-void cpu_unassigned_access(CPUMBState *env1, target_phys_addr_t addr,
+void cpu_unassigned_access(CPUMBState *env1, hwaddr addr,
                            int is_write, int is_exec, int is_asi, int size);
 #endif
 
-static inline bool cpu_has_work(CPUMBState *env)
+static inline bool cpu_has_work(CPUState *cpu)
 {
+    CPUMBState *env = &MICROBLAZE_CPU(cpu)->env;
+
     return env->interrupt_request & (CPU_INTERRUPT_HARD | CPU_INTERRUPT_NMI);
 }
 
diff --git a/target-microblaze/helper.c b/target-microblaze/helper.c
index 74fce2682..efaa123a1 100644
--- a/target-microblaze/helper.c
+++ b/target-microblaze/helper.c
@@ -258,7 +258,7 @@ void do_interrupt(CPUMBState *env)
     }
 }
 
-target_phys_addr_t cpu_get_phys_page_debug(CPUMBState * env, target_ulong addr)
+hwaddr cpu_get_phys_page_debug(CPUMBState * env, target_ulong addr)
 {
     target_ulong vaddr, paddr = 0;
     struct microblaze_mmu_lookup lu;
diff --git a/target-microblaze/helper.h b/target-microblaze/helper.h
index 9dcfb0f1d..a667122e9 100644
--- a/target-microblaze/helper.h
+++ b/target-microblaze/helper.h
@@ -1,39 +1,39 @@
 #include "def-helper.h"
 
-DEF_HELPER_1(raise_exception, void, i32)
-DEF_HELPER_0(debug, void)
-DEF_HELPER_FLAGS_3(carry, TCG_CALL_PURE | TCG_CALL_CONST, i32, i32, i32, i32)
+DEF_HELPER_2(raise_exception, void, env, i32)
+DEF_HELPER_1(debug, void, env)
+DEF_HELPER_FLAGS_3(carry, TCG_CALL_NO_RWG_SE, i32, i32, i32, i32)
 DEF_HELPER_2(cmp, i32, i32, i32)
 DEF_HELPER_2(cmpu, i32, i32, i32)
-DEF_HELPER_FLAGS_1(clz, TCG_CALL_PURE | TCG_CALL_CONST, i32, i32)
-
-DEF_HELPER_2(divs, i32, i32, i32)
-DEF_HELPER_2(divu, i32, i32, i32)
-
-DEF_HELPER_2(fadd, i32, i32, i32)
-DEF_HELPER_2(frsub, i32, i32, i32)
-DEF_HELPER_2(fmul, i32, i32, i32)
-DEF_HELPER_2(fdiv, i32, i32, i32)
-DEF_HELPER_1(flt, i32, i32)
-DEF_HELPER_1(fint, i32, i32)
-DEF_HELPER_1(fsqrt, i32, i32)
-
-DEF_HELPER_2(fcmp_un, i32, i32, i32)
-DEF_HELPER_2(fcmp_lt, i32, i32, i32)
-DEF_HELPER_2(fcmp_eq, i32, i32, i32)
-DEF_HELPER_2(fcmp_le, i32, i32, i32)
-DEF_HELPER_2(fcmp_gt, i32, i32, i32)
-DEF_HELPER_2(fcmp_ne, i32, i32, i32)
-DEF_HELPER_2(fcmp_ge, i32, i32, i32)
-
-DEF_HELPER_FLAGS_2(pcmpbf, TCG_CALL_PURE | TCG_CALL_CONST, i32, i32, i32)
+DEF_HELPER_FLAGS_1(clz, TCG_CALL_NO_RWG_SE, i32, i32)
+
+DEF_HELPER_3(divs, i32, env, i32, i32)
+DEF_HELPER_3(divu, i32, env, i32, i32)
+
+DEF_HELPER_3(fadd, i32, env, i32, i32)
+DEF_HELPER_3(frsub, i32, env, i32, i32)
+DEF_HELPER_3(fmul, i32, env, i32, i32)
+DEF_HELPER_3(fdiv, i32, env, i32, i32)
+DEF_HELPER_2(flt, i32, env, i32)
+DEF_HELPER_2(fint, i32, env, i32)
+DEF_HELPER_2(fsqrt, i32, env, i32)
+
+DEF_HELPER_3(fcmp_un, i32, env, i32, i32)
+DEF_HELPER_3(fcmp_lt, i32, env, i32, i32)
+DEF_HELPER_3(fcmp_eq, i32, env, i32, i32)
+DEF_HELPER_3(fcmp_le, i32, env, i32, i32)
+DEF_HELPER_3(fcmp_gt, i32, env, i32, i32)
+DEF_HELPER_3(fcmp_ne, i32, env, i32, i32)
+DEF_HELPER_3(fcmp_ge, i32, env, i32, i32)
+
+DEF_HELPER_FLAGS_2(pcmpbf, TCG_CALL_NO_RWG_SE, i32, i32, i32)
 #if !defined(CONFIG_USER_ONLY)
-DEF_HELPER_1(mmu_read, i32, i32)
-DEF_HELPER_2(mmu_write, void, i32, i32)
+DEF_HELPER_2(mmu_read, i32, env, i32)
+DEF_HELPER_3(mmu_write, void, env, i32, i32)
 #endif
 
-DEF_HELPER_4(memalign, void, i32, i32, i32, i32)
-DEF_HELPER_1(stackprot, void, i32)
+DEF_HELPER_5(memalign, void, env, i32, i32, i32, i32)
+DEF_HELPER_2(stackprot, void, env, i32)
 
 DEF_HELPER_2(get, i32, i32, i32)
 DEF_HELPER_3(put, void, i32, i32, i32)
diff --git a/target-microblaze/op_helper.c b/target-microblaze/op_helper.c
index 3b1f07243..210296b30 100644
--- a/target-microblaze/op_helper.c
+++ b/target-microblaze/op_helper.c
@@ -20,7 +20,6 @@
 
 #include <assert.h>
 #include "cpu.h"
-#include "dyngen-exec.h"
 #include "helper.h"
 #include "host-utils.h"
 
@@ -42,17 +41,12 @@
 /* Try to fill the TLB and return an exception if error. If retaddr is
    NULL, it means that the function was called in C code (i.e. not
    from generated code or from helper.c) */
-/* XXX: fix it to restore all registers */
-void tlb_fill(CPUMBState *env1, target_ulong addr, int is_write, int mmu_idx,
+void tlb_fill(CPUMBState *env, target_ulong addr, int is_write, int mmu_idx,
               uintptr_t retaddr)
 {
     TranslationBlock *tb;
-    CPUMBState *saved_env;
     int ret;
 
-    saved_env = env;
-    env = env1;
-
     ret = cpu_mb_handle_mmu_fault(env, addr, is_write, mmu_idx);
     if (unlikely(ret)) {
         if (retaddr) {
@@ -66,7 +60,6 @@ void tlb_fill(CPUMBState *env1, target_ulong addr, int is_write, int mmu_idx,
         }
         cpu_loop_exit(env);
     }
-    env = saved_env;
 }
 #endif
 
@@ -105,13 +98,13 @@ uint32_t helper_get(uint32_t id, uint32_t ctrl)
     return 0xdead0000 | id;
 }
 
-void helper_raise_exception(uint32_t index)
+void helper_raise_exception(CPUMBState *env, uint32_t index)
 {
     env->exception_index = index;
     cpu_loop_exit(env);
 }
 
-void helper_debug(void)
+void helper_debug(CPUMBState *env)
 {
     int i;
 
@@ -176,7 +169,7 @@ uint32_t helper_carry(uint32_t a, uint32_t b, uint32_t cf)
     return ncf;
 }
 
-static inline int div_prepare(uint32_t a, uint32_t b)
+static inline int div_prepare(CPUMBState *env, uint32_t a, uint32_t b)
 {
     if (b == 0) {
         env->sregs[SR_MSR] |= MSR_DZ;
@@ -184,7 +177,7 @@ static inline int div_prepare(uint32_t a, uint32_t b)
         if ((env->sregs[SR_MSR] & MSR_EE)
             && !(env->pvr.regs[2] & PVR2_DIV_ZERO_EXC_MASK)) {
             env->sregs[SR_ESR] = ESR_EC_DIVZERO;
-            helper_raise_exception(EXCP_HW_EXCP);
+            helper_raise_exception(env, EXCP_HW_EXCP);
         }
         return 0;
     }
@@ -192,28 +185,30 @@ static inline int div_prepare(uint32_t a, uint32_t b)
     return 1;
 }
 
-uint32_t helper_divs(uint32_t a, uint32_t b)
+uint32_t helper_divs(CPUMBState *env, uint32_t a, uint32_t b)
 {
-    if (!div_prepare(a, b))
+    if (!div_prepare(env, a, b)) {
         return 0;
+    }
     return (int32_t)a / (int32_t)b;
 }
 
-uint32_t helper_divu(uint32_t a, uint32_t b)
+uint32_t helper_divu(CPUMBState *env, uint32_t a, uint32_t b)
 {
-    if (!div_prepare(a, b))
+    if (!div_prepare(env, a, b)) {
         return 0;
+    }
     return a / b;
 }
 
 /* raise FPU exception.  */
-static void raise_fpu_exception(void)
+static void raise_fpu_exception(CPUMBState *env)
 {
     env->sregs[SR_ESR] = ESR_EC_FPU;
-    helper_raise_exception(EXCP_HW_EXCP);
+    helper_raise_exception(env, EXCP_HW_EXCP);
 }
 
-static void update_fpu_flags(int flags)
+static void update_fpu_flags(CPUMBState *env, int flags)
 {
     int raise = 0;
 
@@ -236,11 +231,11 @@ static void update_fpu_flags(int flags)
     if (raise
         && (env->pvr.regs[2] & PVR2_FPU_EXC_MASK)
         && (env->sregs[SR_MSR] & MSR_EE)) {
-        raise_fpu_exception();
+        raise_fpu_exception(env);
     }
 }
 
-uint32_t helper_fadd(uint32_t a, uint32_t b)
+uint32_t helper_fadd(CPUMBState *env, uint32_t a, uint32_t b)
 {
     CPU_FloatU fd, fa, fb;
     int flags;
@@ -251,11 +246,11 @@ uint32_t helper_fadd(uint32_t a, uint32_t b)
     fd.f = float32_add(fa.f, fb.f, &env->fp_status);
 
     flags = get_float_exception_flags(&env->fp_status);
-    update_fpu_flags(flags);
+    update_fpu_flags(env, flags);
     return fd.l;
 }
 
-uint32_t helper_frsub(uint32_t a, uint32_t b)
+uint32_t helper_frsub(CPUMBState *env, uint32_t a, uint32_t b)
 {
     CPU_FloatU fd, fa, fb;
     int flags;
@@ -265,11 +260,11 @@ uint32_t helper_frsub(uint32_t a, uint32_t b)
     fb.l = b;
     fd.f = float32_sub(fb.f, fa.f, &env->fp_status);
     flags = get_float_exception_flags(&env->fp_status);
-    update_fpu_flags(flags);
+    update_fpu_flags(env, flags);
     return fd.l;
 }
 
-uint32_t helper_fmul(uint32_t a, uint32_t b)
+uint32_t helper_fmul(CPUMBState *env, uint32_t a, uint32_t b)
 {
     CPU_FloatU fd, fa, fb;
     int flags;
@@ -279,12 +274,12 @@ uint32_t helper_fmul(uint32_t a, uint32_t b)
     fb.l = b;
     fd.f = float32_mul(fa.f, fb.f, &env->fp_status);
     flags = get_float_exception_flags(&env->fp_status);
-    update_fpu_flags(flags);
+    update_fpu_flags(env, flags);
 
     return fd.l;
 }
 
-uint32_t helper_fdiv(uint32_t a, uint32_t b)
+uint32_t helper_fdiv(CPUMBState *env, uint32_t a, uint32_t b)
 {
     CPU_FloatU fd, fa, fb;
     int flags;
@@ -294,12 +289,12 @@ uint32_t helper_fdiv(uint32_t a, uint32_t b)
     fb.l = b;
     fd.f = float32_div(fb.f, fa.f, &env->fp_status);
     flags = get_float_exception_flags(&env->fp_status);
-    update_fpu_flags(flags);
+    update_fpu_flags(env, flags);
 
     return fd.l;
 }
 
-uint32_t helper_fcmp_un(uint32_t a, uint32_t b)
+uint32_t helper_fcmp_un(CPUMBState *env, uint32_t a, uint32_t b)
 {
     CPU_FloatU fa, fb;
     uint32_t r = 0;
@@ -308,7 +303,7 @@ uint32_t helper_fcmp_un(uint32_t a, uint32_t b)
     fb.l = b;
 
     if (float32_is_signaling_nan(fa.f) || float32_is_signaling_nan(fb.f)) {
-        update_fpu_flags(float_flag_invalid);
+        update_fpu_flags(env, float_flag_invalid);
         r = 1;
     }
 
@@ -319,7 +314,7 @@ uint32_t helper_fcmp_un(uint32_t a, uint32_t b)
     return r;
 }
 
-uint32_t helper_fcmp_lt(uint32_t a, uint32_t b)
+uint32_t helper_fcmp_lt(CPUMBState *env, uint32_t a, uint32_t b)
 {
     CPU_FloatU fa, fb;
     int r;
@@ -330,12 +325,12 @@ uint32_t helper_fcmp_lt(uint32_t a, uint32_t b)
     fb.l = b;
     r = float32_lt(fb.f, fa.f, &env->fp_status);
     flags = get_float_exception_flags(&env->fp_status);
-    update_fpu_flags(flags & float_flag_invalid);
+    update_fpu_flags(env, flags & float_flag_invalid);
 
     return r;
 }
 
-uint32_t helper_fcmp_eq(uint32_t a, uint32_t b)
+uint32_t helper_fcmp_eq(CPUMBState *env, uint32_t a, uint32_t b)
 {
     CPU_FloatU fa, fb;
     int flags;
@@ -346,12 +341,12 @@ uint32_t helper_fcmp_eq(uint32_t a, uint32_t b)
     fb.l = b;
     r = float32_eq_quiet(fa.f, fb.f, &env->fp_status);
     flags = get_float_exception_flags(&env->fp_status);
-    update_fpu_flags(flags & float_flag_invalid);
+    update_fpu_flags(env, flags & float_flag_invalid);
 
     return r;
 }
 
-uint32_t helper_fcmp_le(uint32_t a, uint32_t b)
+uint32_t helper_fcmp_le(CPUMBState *env, uint32_t a, uint32_t b)
 {
     CPU_FloatU fa, fb;
     int flags;
@@ -362,13 +357,13 @@ uint32_t helper_fcmp_le(uint32_t a, uint32_t b)
     set_float_exception_flags(0, &env->fp_status);
     r = float32_le(fa.f, fb.f, &env->fp_status);
     flags = get_float_exception_flags(&env->fp_status);
-    update_fpu_flags(flags & float_flag_invalid);
+    update_fpu_flags(env, flags & float_flag_invalid);
 
 
     return r;
 }
 
-uint32_t helper_fcmp_gt(uint32_t a, uint32_t b)
+uint32_t helper_fcmp_gt(CPUMBState *env, uint32_t a, uint32_t b)
 {
     CPU_FloatU fa, fb;
     int flags, r;
@@ -378,11 +373,11 @@ uint32_t helper_fcmp_gt(uint32_t a, uint32_t b)
     set_float_exception_flags(0, &env->fp_status);
     r = float32_lt(fa.f, fb.f, &env->fp_status);
     flags = get_float_exception_flags(&env->fp_status);
-    update_fpu_flags(flags & float_flag_invalid);
+    update_fpu_flags(env, flags & float_flag_invalid);
     return r;
 }
 
-uint32_t helper_fcmp_ne(uint32_t a, uint32_t b)
+uint32_t helper_fcmp_ne(CPUMBState *env, uint32_t a, uint32_t b)
 {
     CPU_FloatU fa, fb;
     int flags, r;
@@ -392,12 +387,12 @@ uint32_t helper_fcmp_ne(uint32_t a, uint32_t b)
     set_float_exception_flags(0, &env->fp_status);
     r = !float32_eq_quiet(fa.f, fb.f, &env->fp_status);
     flags = get_float_exception_flags(&env->fp_status);
-    update_fpu_flags(flags & float_flag_invalid);
+    update_fpu_flags(env, flags & float_flag_invalid);
 
     return r;
 }
 
-uint32_t helper_fcmp_ge(uint32_t a, uint32_t b)
+uint32_t helper_fcmp_ge(CPUMBState *env, uint32_t a, uint32_t b)
 {
     CPU_FloatU fa, fb;
     int flags, r;
@@ -407,12 +402,12 @@ uint32_t helper_fcmp_ge(uint32_t a, uint32_t b)
     set_float_exception_flags(0, &env->fp_status);
     r = !float32_lt(fa.f, fb.f, &env->fp_status);
     flags = get_float_exception_flags(&env->fp_status);
-    update_fpu_flags(flags & float_flag_invalid);
+    update_fpu_flags(env, flags & float_flag_invalid);
 
     return r;
 }
 
-uint32_t helper_flt(uint32_t a)
+uint32_t helper_flt(CPUMBState *env, uint32_t a)
 {
     CPU_FloatU fd, fa;
 
@@ -421,7 +416,7 @@ uint32_t helper_flt(uint32_t a)
     return fd.l;
 }
 
-uint32_t helper_fint(uint32_t a)
+uint32_t helper_fint(CPUMBState *env, uint32_t a)
 {
     CPU_FloatU fa;
     uint32_t r;
@@ -431,12 +426,12 @@ uint32_t helper_fint(uint32_t a)
     fa.l = a;
     r = float32_to_int32(fa.f, &env->fp_status);
     flags = get_float_exception_flags(&env->fp_status);
-    update_fpu_flags(flags);
+    update_fpu_flags(env, flags);
 
     return r;
 }
 
-uint32_t helper_fsqrt(uint32_t a)
+uint32_t helper_fsqrt(CPUMBState *env, uint32_t a)
 {
     CPU_FloatU fd, fa;
     int flags;
@@ -445,7 +440,7 @@ uint32_t helper_fsqrt(uint32_t a)
     fa.l = a;
     fd.l = float32_sqrt(fa.f, &env->fp_status);
     flags = get_float_exception_flags(&env->fp_status);
-    update_fpu_flags(flags);
+    update_fpu_flags(env, flags);
 
     return fd.l;
 }
@@ -463,7 +458,8 @@ uint32_t helper_pcmpbf(uint32_t a, uint32_t b)
     return 0;
 }
 
-void helper_memalign(uint32_t addr, uint32_t dr, uint32_t wr, uint32_t mask)
+void helper_memalign(CPUMBState *env, uint32_t addr, uint32_t dr, uint32_t wr,
+                     uint32_t mask)
 {
     if (addr & mask) {
             qemu_log_mask(CPU_LOG_INT,
@@ -478,45 +474,39 @@ void helper_memalign(uint32_t addr, uint32_t dr, uint32_t wr, uint32_t mask)
             if (!(env->sregs[SR_MSR] & MSR_EE)) {
                 return;
             }
-            helper_raise_exception(EXCP_HW_EXCP);
+            helper_raise_exception(env, EXCP_HW_EXCP);
     }
 }
 
-void helper_stackprot(uint32_t addr)
+void helper_stackprot(CPUMBState *env, uint32_t addr)
 {
     if (addr < env->slr || addr > env->shr) {
             qemu_log("Stack protector violation at %x %x %x\n",
                      addr, env->slr, env->shr);
             env->sregs[SR_EAR] = addr;
             env->sregs[SR_ESR] = ESR_EC_STACKPROT;
-            helper_raise_exception(EXCP_HW_EXCP);
+            helper_raise_exception(env, EXCP_HW_EXCP);
     }
 }
 
 #if !defined(CONFIG_USER_ONLY)
 /* Writes/reads to the MMU's special regs end up here.  */
-uint32_t helper_mmu_read(uint32_t rn)
+uint32_t helper_mmu_read(CPUMBState *env, uint32_t rn)
 {
     return mmu_read(env, rn);
 }
 
-void helper_mmu_write(uint32_t rn, uint32_t v)
+void helper_mmu_write(CPUMBState *env, uint32_t rn, uint32_t v)
 {
     mmu_write(env, rn, v);
 }
 
-void cpu_unassigned_access(CPUMBState *env1, target_phys_addr_t addr,
+void cpu_unassigned_access(CPUMBState *env, hwaddr addr,
                            int is_write, int is_exec, int is_asi, int size)
 {
-    CPUMBState *saved_env;
-
-    saved_env = env;
-    env = env1;
-
     qemu_log_mask(CPU_LOG_INT, "Unassigned " TARGET_FMT_plx " wr=%d exe=%d\n",
              addr, is_write, is_exec);
     if (!(env->sregs[SR_MSR] & MSR_EE)) {
-        env = saved_env;
         return;
     }
 
@@ -524,14 +514,13 @@ void cpu_unassigned_access(CPUMBState *env1, target_phys_addr_t addr,
     if (is_exec) {
         if ((env->pvr.regs[2] & PVR2_IOPB_BUS_EXC_MASK)) {
             env->sregs[SR_ESR] = ESR_EC_INSN_BUS;
-            helper_raise_exception(EXCP_HW_EXCP);
+            helper_raise_exception(env, EXCP_HW_EXCP);
         }
     } else {
         if ((env->pvr.regs[2] & PVR2_DOPB_BUS_EXC_MASK)) {
             env->sregs[SR_ESR] = ESR_EC_DATA_BUS;
-            helper_raise_exception(EXCP_HW_EXCP);
+            helper_raise_exception(env, EXCP_HW_EXCP);
         }
     }
-    env = saved_env;
 }
 #endif
diff --git a/target-microblaze/translate.c b/target-microblaze/translate.c
index 7470149db..cce449495 100644
--- a/target-microblaze/translate.c
+++ b/target-microblaze/translate.c
@@ -126,7 +126,7 @@ static inline void t_gen_raise_exception(DisasContext *dc, uint32_t index)
 
     t_sync_flags(dc);
     tcg_gen_movi_tl(cpu_SR[SR_PC], dc->pc);
-    gen_helper_raise_exception(tmp);
+    gen_helper_raise_exception(cpu_env, tmp);
     tcg_temp_free_i32(tmp);
     dc->is_jmp = DISAS_UPDATE;
 }
@@ -503,9 +503,9 @@ static void dec_msr(DisasContext *dc)
         sr &= 7;
         LOG_DIS("m%ss sr%d r%d imm=%x\n", to ? "t" : "f", sr, dc->ra, dc->imm);
         if (to)
-            gen_helper_mmu_write(tcg_const_tl(sr), cpu_R[dc->ra]);
+            gen_helper_mmu_write(cpu_env, tcg_const_tl(sr), cpu_R[dc->ra]);
         else
-            gen_helper_mmu_read(cpu_R[dc->rd], tcg_const_tl(sr));
+            gen_helper_mmu_read(cpu_R[dc->rd], cpu_env, tcg_const_tl(sr));
         return;
     }
 #endif
@@ -704,9 +704,11 @@ static void dec_div(DisasContext *dc)
     }
 
     if (u)
-        gen_helper_divu(cpu_R[dc->rd], *(dec_alu_op_b(dc)), cpu_R[dc->ra]);
+        gen_helper_divu(cpu_R[dc->rd], cpu_env, *(dec_alu_op_b(dc)),
+                        cpu_R[dc->ra]);
     else
-        gen_helper_divs(cpu_R[dc->rd], *(dec_alu_op_b(dc)), cpu_R[dc->ra]);
+        gen_helper_divs(cpu_R[dc->rd], cpu_env, *(dec_alu_op_b(dc)),
+                        cpu_R[dc->ra]);
     if (!dc->rd)
         tcg_gen_movi_tl(cpu_R[dc->rd], 0);
 }
@@ -838,7 +840,7 @@ static void dec_bit(DisasContext *dc)
             LOG_DIS("swapb r%d r%d\n", dc->rd, dc->ra);
             tcg_gen_bswap32_i32(cpu_R[dc->rd], cpu_R[dc->ra]);
             break;
-        case 0x1e1:
+        case 0x1e2:
             /*swaph */
             LOG_DIS("swaph r%d r%d\n", dc->rd, dc->ra);
             tcg_gen_rotri_i32(cpu_R[dc->rd], cpu_R[dc->ra], 16);
@@ -912,7 +914,7 @@ static inline TCGv *compute_ldst_addr(DisasContext *dc, TCGv *t)
         tcg_gen_add_tl(*t, cpu_R[dc->ra], cpu_R[dc->rb]);
 
         if (stackprot) {
-            gen_helper_stackprot(*t);
+            gen_helper_stackprot(cpu_env, *t);
         }
         return t;
     }
@@ -930,7 +932,7 @@ static inline TCGv *compute_ldst_addr(DisasContext *dc, TCGv *t)
     }
 
     if (stackprot) {
-        gen_helper_stackprot(*t);
+        gen_helper_stackprot(cpu_env, *t);
     }
     return t;
 }
@@ -1056,7 +1058,7 @@ static void dec_load(DisasContext *dc)
         gen_load(dc, v, *addr, size);
 
         tcg_gen_movi_tl(cpu_SR[SR_PC], dc->pc);
-        gen_helper_memalign(*addr, tcg_const_tl(dc->rd),
+        gen_helper_memalign(cpu_env, *addr, tcg_const_tl(dc->rd),
                             tcg_const_tl(0), tcg_const_tl(size - 1));
         if (dc->rd) {
             if (rev) {
@@ -1218,7 +1220,7 @@ static void dec_store(DisasContext *dc)
          *        the alignment checks in between the probe and the mem
          *        access.
          */
-        gen_helper_memalign(*addr, tcg_const_tl(dc->rd),
+        gen_helper_memalign(cpu_env, *addr, tcg_const_tl(dc->rd),
                             tcg_const_tl(1), tcg_const_tl(size - 1));
     }
 
@@ -1493,49 +1495,53 @@ static void dec_fpu(DisasContext *dc)
 
     switch (fpu_insn) {
         case 0:
-            gen_helper_fadd(cpu_R[dc->rd], cpu_R[dc->ra], cpu_R[dc->rb]);
+            gen_helper_fadd(cpu_R[dc->rd], cpu_env, cpu_R[dc->ra],
+                            cpu_R[dc->rb]);
             break;
 
         case 1:
-            gen_helper_frsub(cpu_R[dc->rd], cpu_R[dc->ra], cpu_R[dc->rb]);
+            gen_helper_frsub(cpu_R[dc->rd], cpu_env, cpu_R[dc->ra],
+                             cpu_R[dc->rb]);
             break;
 
         case 2:
-            gen_helper_fmul(cpu_R[dc->rd], cpu_R[dc->ra], cpu_R[dc->rb]);
+            gen_helper_fmul(cpu_R[dc->rd], cpu_env, cpu_R[dc->ra],
+                            cpu_R[dc->rb]);
             break;
 
         case 3:
-            gen_helper_fdiv(cpu_R[dc->rd], cpu_R[dc->ra], cpu_R[dc->rb]);
+            gen_helper_fdiv(cpu_R[dc->rd], cpu_env, cpu_R[dc->ra],
+                            cpu_R[dc->rb]);
             break;
 
         case 4:
             switch ((dc->ir >> 4) & 7) {
                 case 0:
-                    gen_helper_fcmp_un(cpu_R[dc->rd],
+                    gen_helper_fcmp_un(cpu_R[dc->rd], cpu_env,
                                        cpu_R[dc->ra], cpu_R[dc->rb]);
                     break;
                 case 1:
-                    gen_helper_fcmp_lt(cpu_R[dc->rd],
+                    gen_helper_fcmp_lt(cpu_R[dc->rd], cpu_env,
                                        cpu_R[dc->ra], cpu_R[dc->rb]);
                     break;
                 case 2:
-                    gen_helper_fcmp_eq(cpu_R[dc->rd],
+                    gen_helper_fcmp_eq(cpu_R[dc->rd], cpu_env,
                                        cpu_R[dc->ra], cpu_R[dc->rb]);
                     break;
                 case 3:
-                    gen_helper_fcmp_le(cpu_R[dc->rd],
+                    gen_helper_fcmp_le(cpu_R[dc->rd], cpu_env,
                                        cpu_R[dc->ra], cpu_R[dc->rb]);
                     break;
                 case 4:
-                    gen_helper_fcmp_gt(cpu_R[dc->rd],
+                    gen_helper_fcmp_gt(cpu_R[dc->rd], cpu_env,
                                        cpu_R[dc->ra], cpu_R[dc->rb]);
                     break;
                 case 5:
-                    gen_helper_fcmp_ne(cpu_R[dc->rd],
+                    gen_helper_fcmp_ne(cpu_R[dc->rd], cpu_env,
                                        cpu_R[dc->ra], cpu_R[dc->rb]);
                     break;
                 case 6:
-                    gen_helper_fcmp_ge(cpu_R[dc->rd],
+                    gen_helper_fcmp_ge(cpu_R[dc->rd], cpu_env,
                                        cpu_R[dc->ra], cpu_R[dc->rb]);
                     break;
                 default:
@@ -1552,21 +1558,21 @@ static void dec_fpu(DisasContext *dc)
             if (!dec_check_fpuv2(dc)) {
                 return;
             }
-            gen_helper_flt(cpu_R[dc->rd], cpu_R[dc->ra]);
+            gen_helper_flt(cpu_R[dc->rd], cpu_env, cpu_R[dc->ra]);
             break;
 
         case 6:
             if (!dec_check_fpuv2(dc)) {
                 return;
             }
-            gen_helper_fint(cpu_R[dc->rd], cpu_R[dc->ra]);
+            gen_helper_fint(cpu_R[dc->rd], cpu_env, cpu_R[dc->ra]);
             break;
 
         case 7:
             if (!dec_check_fpuv2(dc)) {
                 return;
             }
-            gen_helper_fsqrt(cpu_R[dc->rd], cpu_R[dc->ra]);
+            gen_helper_fsqrt(cpu_R[dc->rd], cpu_env, cpu_R[dc->ra]);
             break;
 
         default:
@@ -1654,15 +1660,15 @@ static struct decoder_info {
     {{0, 0}, dec_null}
 };
 
-static inline void decode(DisasContext *dc)
+static inline void decode(DisasContext *dc, uint32_t ir)
 {
-    uint32_t ir;
     int i;
 
-    if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP)))
+    if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
         tcg_gen_debug_insn_start(dc->pc);
+    }
 
-    dc->ir = ir = ldl_code(dc->pc);
+    dc->ir = ir;
     LOG_DIS("%8.8x\t", dc->ir);
 
     if (dc->ir)
@@ -1735,7 +1741,7 @@ gen_intermediate_code_internal(CPUMBState *env, TranslationBlock *tb,
     dc->tb = tb;
     org_flags = dc->synced_flags = dc->tb_flags = tb->flags;
 
-    gen_opc_end = gen_opc_buf + OPC_MAX_SIZE;
+    gen_opc_end = tcg_ctx.gen_opc_buf + OPC_MAX_SIZE;
 
     dc->is_jmp = DISAS_NEXT;
     dc->jmp = 0;
@@ -1778,7 +1784,7 @@ gen_intermediate_code_internal(CPUMBState *env, TranslationBlock *tb,
         check_breakpoint(env, dc);
 
         if (search_pc) {
-            j = gen_opc_ptr - gen_opc_buf;
+            j = tcg_ctx.gen_opc_ptr - tcg_ctx.gen_opc_buf;
             if (lj < j) {
                 lj++;
                 while (lj < j)
@@ -1796,7 +1802,7 @@ gen_intermediate_code_internal(CPUMBState *env, TranslationBlock *tb,
             gen_io_start();
 
         dc->clear_imm = 1;
-	decode(dc);
+        decode(dc, cpu_ldl_code(env, dc->pc));
         if (dc->clear_imm)
             dc->tb_flags &= ~IMM_FLAG;
         dc->pc += 4;
@@ -1840,7 +1846,7 @@ gen_intermediate_code_internal(CPUMBState *env, TranslationBlock *tb,
         if (env->singlestep_enabled)
             break;
     } while (!dc->is_jmp && !dc->cpustate_changed
-         && gen_opc_ptr < gen_opc_end
+         && tcg_ctx.gen_opc_ptr < gen_opc_end
                  && !singlestep
          && (dc->pc < next_page_start)
                  && num_insns < max_insns);
@@ -1871,7 +1877,7 @@ gen_intermediate_code_internal(CPUMBState *env, TranslationBlock *tb,
         if (dc->is_jmp != DISAS_JUMP) {
             tcg_gen_movi_tl(cpu_SR[SR_PC], npc);
         }
-        gen_helper_raise_exception(tmp);
+        gen_helper_raise_exception(cpu_env, tmp);
         tcg_temp_free_i32(tmp);
     } else {
         switch(dc->is_jmp) {
@@ -1891,9 +1897,9 @@ gen_intermediate_code_internal(CPUMBState *env, TranslationBlock *tb,
         }
     }
     gen_icount_end(tb, num_insns);
-    *gen_opc_ptr = INDEX_op_end;
+    *tcg_ctx.gen_opc_ptr = INDEX_op_end;
     if (search_pc) {
-        j = gen_opc_ptr - gen_opc_buf;
+        j = tcg_ctx.gen_opc_ptr - tcg_ctx.gen_opc_buf;
         lj++;
         while (lj <= j)
             gen_opc_instr_start[lj++] = 0;
@@ -1907,10 +1913,11 @@ gen_intermediate_code_internal(CPUMBState *env, TranslationBlock *tb,
     if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
         qemu_log("\n");
 #if DISAS_GNU
-        log_target_disas(pc_start, dc->pc - pc_start, 0);
+        log_target_disas(env, pc_start, dc->pc - pc_start, 0);
 #endif
         qemu_log("\nisize=%d osize=%td\n",
-            dc->pc - pc_start, gen_opc_ptr - gen_opc_buf);
+            dc->pc - pc_start, tcg_ctx.gen_opc_ptr -
+            tcg_ctx.gen_opc_buf);
     }
 #endif
 #endif
diff --git a/target-mips/Makefile.objs b/target-mips/Makefile.objs
index 2e0e093e1..119c81651 100644
--- a/target-mips/Makefile.objs
+++ b/target-mips/Makefile.objs
@@ -1,4 +1,2 @@
-obj-y += translate.o op_helper.o helper.o cpu.o
+obj-y += translate.o dsp_helper.o op_helper.o lmi_helper.o helper.o cpu.o
 obj-$(CONFIG_SOFTMMU) += machine.o
-
-$(obj)/op_helper.o: QEMU_CFLAGS += $(HELPER_CFLAGS)
diff --git a/target-mips/TODO b/target-mips/TODO
index 2a3546f62..1d782d802 100644
--- a/target-mips/TODO
+++ b/target-mips/TODO
@@ -6,8 +6,7 @@ General
 - Unimplemented ASEs:
   - MDMX
   - SmartMIPS
-  - DSP r1
-  - DSP r2
+  - microMIPS DSP r1 & r2 encodings
 - MT ASE only partially implemented and not functional
 - Shadow register support only partially implemented,
   lacks set switching on interrupt/exception.
diff --git a/target-mips/cpu.h b/target-mips/cpu.h
index ce3467f14..aebb2d5b7 100644
--- a/target-mips/cpu.h
+++ b/target-mips/cpu.h
@@ -37,11 +37,11 @@ typedef struct CPUMIPSTLBContext CPUMIPSTLBContext;
 struct CPUMIPSTLBContext {
     uint32_t nb_tlb;
     uint32_t tlb_in_use;
-    int (*map_address) (struct CPUMIPSState *env, target_phys_addr_t *physical, int *prot, target_ulong address, int rw, int access_type);
-    void (*helper_tlbwi) (void);
-    void (*helper_tlbwr) (void);
-    void (*helper_tlbp) (void);
-    void (*helper_tlbr) (void);
+    int (*map_address) (struct CPUMIPSState *env, hwaddr *physical, int *prot, target_ulong address, int rw, int access_type);
+    void (*helper_tlbwi)(struct CPUMIPSState *env);
+    void (*helper_tlbwr)(struct CPUMIPSState *env);
+    void (*helper_tlbp)(struct CPUMIPSState *env);
+    void (*helper_tlbr)(struct CPUMIPSState *env);
     union {
         struct {
             r4k_tlb_t tlb[MIPS_TLB_MAX];
@@ -415,7 +415,7 @@ struct CPUMIPSState {
     int error_code;
     uint32_t hflags;    /* CPU State */
     /* TMASK defines different execution modes */
-#define MIPS_HFLAG_TMASK  0x007FF
+#define MIPS_HFLAG_TMASK  0xC07FF
 #define MIPS_HFLAG_MODE   0x00007 /* execution modes                    */
     /* The KSU flags must be the lowest bits in hflags. The flag order
        must be the same as defined for CP0 Status. This allows to use
@@ -453,6 +453,9 @@ struct CPUMIPSState {
 #define MIPS_HFLAG_BDS32  0x10000 /* branch requires 32-bit delay slot  */
 #define MIPS_HFLAG_BX     0x20000 /* branch exchanges execution mode    */
 #define MIPS_HFLAG_BMASK  (MIPS_HFLAG_BMASK_BASE | MIPS_HFLAG_BMASK_EXT)
+    /* MIPS DSP resources access. */
+#define MIPS_HFLAG_DSP   0x40000  /* Enable access to MIPS DSP resources. */
+#define MIPS_HFLAG_DSPR2 0x80000  /* Enable access to MIPS DSPR2 resources. */
     target_ulong btarget;        /* Jump / branch target               */
     target_ulong bcond;          /* Branch condition (if needed)       */
 
@@ -479,18 +482,18 @@ struct CPUMIPSState {
 #include "cpu-qom.h"
 
 #if !defined(CONFIG_USER_ONLY)
-int no_mmu_map_address (CPUMIPSState *env, target_phys_addr_t *physical, int *prot,
+int no_mmu_map_address (CPUMIPSState *env, hwaddr *physical, int *prot,
                         target_ulong address, int rw, int access_type);
-int fixed_mmu_map_address (CPUMIPSState *env, target_phys_addr_t *physical, int *prot,
+int fixed_mmu_map_address (CPUMIPSState *env, hwaddr *physical, int *prot,
                            target_ulong address, int rw, int access_type);
-int r4k_map_address (CPUMIPSState *env, target_phys_addr_t *physical, int *prot,
+int r4k_map_address (CPUMIPSState *env, hwaddr *physical, int *prot,
                      target_ulong address, int rw, int access_type);
-void r4k_helper_tlbwi (void);
-void r4k_helper_tlbwr (void);
-void r4k_helper_tlbp (void);
-void r4k_helper_tlbr (void);
+void r4k_helper_tlbwi(CPUMIPSState *env);
+void r4k_helper_tlbwr(CPUMIPSState *env);
+void r4k_helper_tlbp(CPUMIPSState *env);
+void r4k_helper_tlbr(CPUMIPSState *env);
 
-void cpu_unassigned_access(CPUMIPSState *env, target_phys_addr_t addr,
+void cpu_unassigned_access(CPUMIPSState *env, hwaddr addr,
                            int is_write, int is_exec, int unused, int size);
 #endif
 
@@ -610,8 +613,9 @@ enum {
     EXCP_MDMX,
     EXCP_C2E,
     EXCP_CACHE, /* 32 */
+    EXCP_DSPDIS,
 
-    EXCP_LAST = EXCP_CACHE,
+    EXCP_LAST = EXCP_DSPDIS,
 };
 /* Dummy exception for conditional stores.  */
 #define EXCP_SC 0x100
@@ -658,7 +662,7 @@ int cpu_mips_handle_mmu_fault (CPUMIPSState *env, target_ulong address, int rw,
 void do_interrupt (CPUMIPSState *env);
 #if !defined(CONFIG_USER_ONLY)
 void r4k_invalidate_tlb (CPUMIPSState *env, int idx, int use_extra);
-target_phys_addr_t cpu_mips_translate_address (CPUMIPSState *env, target_ulong address,
+hwaddr cpu_mips_translate_address (CPUMIPSState *env, target_ulong address,
 		                               int rw);
 #endif
 
@@ -706,16 +710,17 @@ static inline int mips_vpe_active(CPUMIPSState *env)
     return active;
 }
 
-static inline int cpu_has_work(CPUMIPSState *env)
+static inline bool cpu_has_work(CPUState *cpu)
 {
-    int has_work = 0;
+    CPUMIPSState *env = &MIPS_CPU(cpu)->env;
+    bool has_work = false;
 
     /* It is implementation dependent if non-enabled interrupts
        wake-up the CPU, however most of the implementations only
        check for interrupts that can be taken. */
     if ((env->interrupt_request & CPU_INTERRUPT_HARD) &&
         cpu_mips_hw_interrupts_pending(env)) {
-        has_work = 1;
+        has_work = true;
     }
 
     /* MIPS-MT has the ability to halt the CPU.  */
@@ -723,11 +728,11 @@ static inline int cpu_has_work(CPUMIPSState *env)
         /* The QEMU model will issue an _WAKE request whenever the CPUs
            should be woken up.  */
         if (env->interrupt_request & CPU_INTERRUPT_WAKE) {
-            has_work = 1;
+            has_work = true;
         }
 
         if (!mips_vpe_active(env)) {
-            has_work = 0;
+            has_work = false;
         }
     }
     return has_work;
@@ -742,4 +747,68 @@ static inline void cpu_pc_from_tb(CPUMIPSState *env, TranslationBlock *tb)
     env->hflags |= tb->flags & MIPS_HFLAG_BMASK;
 }
 
+static inline void compute_hflags(CPUMIPSState *env)
+{
+    env->hflags &= ~(MIPS_HFLAG_COP1X | MIPS_HFLAG_64 | MIPS_HFLAG_CP0 |
+                     MIPS_HFLAG_F64 | MIPS_HFLAG_FPU | MIPS_HFLAG_KSU |
+                     MIPS_HFLAG_UX);
+    if (!(env->CP0_Status & (1 << CP0St_EXL)) &&
+        !(env->CP0_Status & (1 << CP0St_ERL)) &&
+        !(env->hflags & MIPS_HFLAG_DM)) {
+        env->hflags |= (env->CP0_Status >> CP0St_KSU) & MIPS_HFLAG_KSU;
+    }
+#if defined(TARGET_MIPS64)
+    if (((env->hflags & MIPS_HFLAG_KSU) != MIPS_HFLAG_UM) ||
+        (env->CP0_Status & (1 << CP0St_PX)) ||
+        (env->CP0_Status & (1 << CP0St_UX))) {
+        env->hflags |= MIPS_HFLAG_64;
+    }
+    if (env->CP0_Status & (1 << CP0St_UX)) {
+        env->hflags |= MIPS_HFLAG_UX;
+    }
+#endif
+    if ((env->CP0_Status & (1 << CP0St_CU0)) ||
+        !(env->hflags & MIPS_HFLAG_KSU)) {
+        env->hflags |= MIPS_HFLAG_CP0;
+    }
+    if (env->CP0_Status & (1 << CP0St_CU1)) {
+        env->hflags |= MIPS_HFLAG_FPU;
+    }
+    if (env->CP0_Status & (1 << CP0St_FR)) {
+        env->hflags |= MIPS_HFLAG_F64;
+    }
+    if (env->insn_flags & ASE_DSPR2) {
+        /* Enables access MIPS DSP resources, now our cpu is DSP ASER2,
+           so enable to access DSPR2 resources. */
+        if (env->CP0_Status & (1 << CP0St_MX)) {
+            env->hflags |= MIPS_HFLAG_DSP | MIPS_HFLAG_DSPR2;
+        }
+
+    } else if (env->insn_flags & ASE_DSP) {
+        /* Enables access MIPS DSP resources, now our cpu is DSP ASE,
+           so enable to access DSP resources. */
+        if (env->CP0_Status & (1 << CP0St_MX)) {
+            env->hflags |= MIPS_HFLAG_DSP;
+        }
+
+    }
+    if (env->insn_flags & ISA_MIPS32R2) {
+        if (env->active_fpu.fcr0 & (1 << FCR0_F64)) {
+            env->hflags |= MIPS_HFLAG_COP1X;
+        }
+    } else if (env->insn_flags & ISA_MIPS32) {
+        if (env->hflags & MIPS_HFLAG_64) {
+            env->hflags |= MIPS_HFLAG_COP1X;
+        }
+    } else if (env->insn_flags & ISA_MIPS4) {
+        /* All supported MIPS IV CPUs use the XX (CU3) to enable
+           and disable the MIPS IV extensions to the MIPS III ISA.
+           Some other MIPS IV CPUs ignore the bit, so the check here
+           would be too restrictive for them.  */
+        if (env->CP0_Status & (1 << CP0St_CU3)) {
+            env->hflags |= MIPS_HFLAG_COP1X;
+        }
+    }
+}
+
 #endif /* !defined (__MIPS_CPU_H__) */
diff --git a/target-mips/dsp_helper.c b/target-mips/dsp_helper.c
new file mode 100644
index 000000000..e7949c22c
--- /dev/null
+++ b/target-mips/dsp_helper.c
@@ -0,0 +1,4033 @@
+/*
+ * MIPS ASE DSP Instruction emulation helpers for QEMU.
+ *
+ * Copyright (c) 2012  Jia Liu <proljc@gmail.com>
+ *                     Dongxue Zhang <elat.era@gmail.com>
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "cpu.h"
+#include "helper.h"
+
+/*** MIPS DSP internal functions begin ***/
+#define MIPSDSP_ABS(x) (((x) >= 0) ? x : -x)
+#define MIPSDSP_OVERFLOW(a, b, c, d) (!(!((a ^ b ^ -1) & (a ^ c) & d)))
+
+static inline void set_DSPControl_overflow_flag(uint32_t flag, int position,
+                                                CPUMIPSState *env)
+{
+    env->active_tc.DSPControl |= (target_ulong)flag << position;
+}
+
+static inline void set_DSPControl_carryflag(uint32_t flag, CPUMIPSState *env)
+{
+    env->active_tc.DSPControl |= (target_ulong)flag << 13;
+}
+
+static inline uint32_t get_DSPControl_carryflag(CPUMIPSState *env)
+{
+    return (env->active_tc.DSPControl >> 13) & 0x01;
+}
+
+static inline void set_DSPControl_24(uint32_t flag, int len, CPUMIPSState *env)
+{
+  uint32_t filter;
+
+  filter = ((0x01 << len) - 1) << 24;
+  filter = ~filter;
+
+  env->active_tc.DSPControl &= filter;
+  env->active_tc.DSPControl |= (target_ulong)flag << 24;
+}
+
+static inline uint32_t get_DSPControl_24(int len, CPUMIPSState *env)
+{
+  uint32_t filter;
+
+  filter = (0x01 << len) - 1;
+
+  return (env->active_tc.DSPControl >> 24) & filter;
+}
+
+static inline void set_DSPControl_pos(uint32_t pos, CPUMIPSState *env)
+{
+    target_ulong dspc;
+
+    dspc = env->active_tc.DSPControl;
+#ifndef TARGET_MIPS64
+    dspc = dspc & 0xFFFFFFC0;
+    dspc |= pos;
+#else
+    dspc = dspc & 0xFFFFFF80;
+    dspc |= pos;
+#endif
+    env->active_tc.DSPControl = dspc;
+}
+
+static inline uint32_t get_DSPControl_pos(CPUMIPSState *env)
+{
+    target_ulong dspc;
+    uint32_t pos;
+
+    dspc = env->active_tc.DSPControl;
+
+#ifndef TARGET_MIPS64
+    pos = dspc & 0x3F;
+#else
+    pos = dspc & 0x7F;
+#endif
+
+    return pos;
+}
+
+static inline void set_DSPControl_efi(uint32_t flag, CPUMIPSState *env)
+{
+    env->active_tc.DSPControl &= 0xFFFFBFFF;
+    env->active_tc.DSPControl |= (target_ulong)flag << 14;
+}
+
+#define DO_MIPS_SAT_ABS(size)                                          \
+static inline int##size##_t mipsdsp_sat_abs##size(int##size##_t a,         \
+                                                  CPUMIPSState *env)   \
+{                                                                      \
+    if (a == INT##size##_MIN) {                                        \
+        set_DSPControl_overflow_flag(1, 20, env);                      \
+        return INT##size##_MAX;                                        \
+    } else {                                                           \
+        return MIPSDSP_ABS(a);                                         \
+    }                                                                  \
+}
+DO_MIPS_SAT_ABS(8)
+DO_MIPS_SAT_ABS(16)
+DO_MIPS_SAT_ABS(32)
+#undef DO_MIPS_SAT_ABS
+
+/* get sum value */
+static inline int16_t mipsdsp_add_i16(int16_t a, int16_t b, CPUMIPSState *env)
+{
+    int16_t tempI;
+
+    tempI = a + b;
+
+    if (MIPSDSP_OVERFLOW(a, b, tempI, 0x8000)) {
+        set_DSPControl_overflow_flag(1, 20, env);
+    }
+
+    return tempI;
+}
+
+static inline int16_t mipsdsp_sat_add_i16(int16_t a, int16_t b,
+                                          CPUMIPSState *env)
+{
+    int16_t tempS;
+
+    tempS = a + b;
+
+    if (MIPSDSP_OVERFLOW(a, b, tempS, 0x8000)) {
+        if (a > 0) {
+            tempS = 0x7FFF;
+        } else {
+            tempS = 0x8000;
+        }
+        set_DSPControl_overflow_flag(1, 20, env);
+    }
+
+    return tempS;
+}
+
+static inline int32_t mipsdsp_sat_add_i32(int32_t a, int32_t b,
+                                          CPUMIPSState *env)
+{
+    int32_t tempI;
+
+    tempI = a + b;
+
+    if (MIPSDSP_OVERFLOW(a, b, tempI, 0x80000000)) {
+        if (a > 0) {
+            tempI = 0x7FFFFFFF;
+        } else {
+            tempI = 0x80000000;
+        }
+        set_DSPControl_overflow_flag(1, 20, env);
+    }
+
+    return tempI;
+}
+
+static inline uint8_t mipsdsp_add_u8(uint8_t a, uint8_t b, CPUMIPSState *env)
+{
+    uint16_t temp;
+
+    temp = (uint16_t)a + (uint16_t)b;
+
+    if (temp & 0x0100) {
+        set_DSPControl_overflow_flag(1, 20, env);
+    }
+
+    return temp & 0xFF;
+}
+
+static inline uint16_t mipsdsp_add_u16(uint16_t a, uint16_t b,
+                                       CPUMIPSState *env)
+{
+    uint32_t temp;
+
+    temp = (uint32_t)a + (uint32_t)b;
+
+    if (temp & 0x00010000) {
+        set_DSPControl_overflow_flag(1, 20, env);
+    }
+
+    return temp & 0xFFFF;
+}
+
+static inline uint8_t mipsdsp_sat_add_u8(uint8_t a, uint8_t b,
+                                         CPUMIPSState *env)
+{
+    uint8_t  result;
+    uint16_t temp;
+
+    temp = (uint16_t)a + (uint16_t)b;
+    result = temp & 0xFF;
+
+    if (0x0100 & temp) {
+        result = 0xFF;
+        set_DSPControl_overflow_flag(1, 20, env);
+    }
+
+    return result;
+}
+
+static inline uint16_t mipsdsp_sat_add_u16(uint16_t a, uint16_t b,
+                                           CPUMIPSState *env)
+{
+    uint16_t result;
+    uint32_t temp;
+
+    temp = (uint32_t)a + (uint32_t)b;
+    result = temp & 0xFFFF;
+
+    if (0x00010000 & temp) {
+        result = 0xFFFF;
+        set_DSPControl_overflow_flag(1, 20, env);
+    }
+
+    return result;
+}
+
+static inline int32_t mipsdsp_sat32_acc_q31(int32_t acc, int32_t a,
+                                            CPUMIPSState *env)
+{
+    int64_t temp;
+    int32_t temp32, temp31, result;
+    int64_t temp_sum;
+
+#ifndef TARGET_MIPS64
+    temp = ((uint64_t)env->active_tc.HI[acc] << 32) |
+           (uint64_t)env->active_tc.LO[acc];
+#else
+    temp = (uint64_t)env->active_tc.LO[acc];
+#endif
+
+    temp_sum = (int64_t)a + temp;
+
+    temp32 = (temp_sum >> 32) & 0x01;
+    temp31 = (temp_sum >> 31) & 0x01;
+    result = temp_sum & 0xFFFFFFFF;
+
+    /* FIXME
+       This sat function may wrong, because user manual wrote:
+       temp127..0 ← temp + ( (signA) || a31..0
+       if ( temp32 ≠ temp31 ) then
+           if ( temp32 = 0 ) then
+               temp31..0 ← 0x80000000
+           else
+                temp31..0 ← 0x7FFFFFFF
+           endif
+           DSPControlouflag:16+acc ← 1
+       endif
+     */
+    if (temp32 != temp31) {
+        if (temp32 == 0) {
+            result = 0x7FFFFFFF;
+        } else {
+            result = 0x80000000;
+        }
+        set_DSPControl_overflow_flag(1, 16 + acc, env);
+    }
+
+    return result;
+}
+
+/* a[0] is LO, a[1] is HI. */
+static inline void mipsdsp_sat64_acc_add_q63(int64_t *ret,
+                                             int32_t ac,
+                                             int64_t *a,
+                                             CPUMIPSState *env)
+{
+    bool temp64;
+
+    ret[0] = env->active_tc.LO[ac] + a[0];
+    ret[1] = env->active_tc.HI[ac] + a[1];
+
+    if (((uint64_t)ret[0] < (uint64_t)env->active_tc.LO[ac]) &&
+        ((uint64_t)ret[0] < (uint64_t)a[0])) {
+        ret[1] += 1;
+    }
+    temp64 = ret[1] & 1;
+    if (temp64 != ((ret[0] >> 63) & 0x01)) {
+        if (temp64) {
+            ret[0] = (0x01ull << 63);
+            ret[1] = ~0ull;
+        } else {
+            ret[0] = (0x01ull << 63) - 1;
+            ret[1] = 0x00;
+        }
+        set_DSPControl_overflow_flag(1, 16 + ac, env);
+    }
+}
+
+static inline void mipsdsp_sat64_acc_sub_q63(int64_t *ret,
+                                             int32_t ac,
+                                             int64_t *a,
+                                             CPUMIPSState *env)
+{
+    bool temp64;
+
+    ret[0] = env->active_tc.LO[ac] - a[0];
+    ret[1] = env->active_tc.HI[ac] - a[1];
+
+    if ((uint64_t)ret[0] > (uint64_t)env->active_tc.LO[ac]) {
+        ret[1] -= 1;
+    }
+    temp64 = ret[1] & 1;
+    if (temp64 != ((ret[0] >> 63) & 0x01)) {
+        if (temp64) {
+            ret[0] = (0x01ull << 63);
+            ret[1] = ~0ull;
+        } else {
+            ret[0] = (0x01ull << 63) - 1;
+            ret[1] = 0x00;
+        }
+        set_DSPControl_overflow_flag(1, 16 + ac, env);
+    }
+}
+
+static inline int32_t mipsdsp_mul_i16_i16(int16_t a, int16_t b,
+                                          CPUMIPSState *env)
+{
+    int32_t temp;
+
+    temp = (int32_t)a * (int32_t)b;
+
+    if ((temp > (int)0x7FFF) || (temp < (int)0xFFFF8000)) {
+        set_DSPControl_overflow_flag(1, 21, env);
+    }
+    temp &= 0x0000FFFF;
+
+    return temp;
+}
+
+static inline int32_t mipsdsp_mul_u16_u16(int32_t a, int32_t b)
+{
+    return a * b;
+}
+
+static inline int32_t mipsdsp_mul_i32_i32(int32_t a, int32_t b)
+{
+    return a * b;
+}
+
+static inline int32_t mipsdsp_sat16_mul_i16_i16(int16_t a, int16_t b,
+                                                CPUMIPSState *env)
+{
+    int32_t temp;
+
+    temp = (int32_t)a * (int32_t)b;
+
+    if (temp > (int)0x7FFF) {
+        temp = 0x00007FFF;
+        set_DSPControl_overflow_flag(1, 21, env);
+    } else if (temp < (int)0xffff8000) {
+        temp = 0xFFFF8000;
+        set_DSPControl_overflow_flag(1, 21, env);
+    }
+    temp &= 0x0000FFFF;
+
+    return temp;
+}
+
+static inline int32_t mipsdsp_mul_q15_q15_overflowflag21(uint16_t a, uint16_t b,
+                                                         CPUMIPSState *env)
+{
+    int32_t temp;
+
+    if ((a == 0x8000) && (b == 0x8000)) {
+        temp = 0x7FFFFFFF;
+        set_DSPControl_overflow_flag(1, 21, env);
+    } else {
+        temp = ((int32_t)(int16_t)a * (int32_t)(int16_t)b) << 1;
+    }
+
+    return temp;
+}
+
+/* right shift */
+static inline uint8_t mipsdsp_rshift_u8(uint8_t a, target_ulong mov)
+{
+    return a >> mov;
+}
+
+static inline uint16_t mipsdsp_rshift_u16(uint16_t a, target_ulong mov)
+{
+    return a >> mov;
+}
+
+static inline int8_t mipsdsp_rashift8(int8_t a, target_ulong mov)
+{
+    return a >> mov;
+}
+
+static inline int16_t mipsdsp_rashift16(int16_t a, target_ulong mov)
+{
+    return a >> mov;
+}
+
+static inline int32_t mipsdsp_rashift32(int32_t a, target_ulong mov)
+{
+    return a >> mov;
+}
+
+static inline int16_t mipsdsp_rshift1_add_q16(int16_t a, int16_t b)
+{
+    int32_t temp;
+
+    temp = (int32_t)a + (int32_t)b;
+
+    return (temp >> 1) & 0xFFFF;
+}
+
+/* round right shift */
+static inline int16_t mipsdsp_rrshift1_add_q16(int16_t a, int16_t b)
+{
+    int32_t temp;
+
+    temp = (int32_t)a + (int32_t)b;
+    temp += 1;
+
+    return (temp >> 1) & 0xFFFF;
+}
+
+static inline int32_t mipsdsp_rshift1_add_q32(int32_t a, int32_t b)
+{
+    int64_t temp;
+
+    temp = (int64_t)a + (int64_t)b;
+
+    return (temp >> 1) & 0xFFFFFFFF;
+}
+
+static inline int32_t mipsdsp_rrshift1_add_q32(int32_t a, int32_t b)
+{
+    int64_t temp;
+
+    temp = (int64_t)a + (int64_t)b;
+    temp += 1;
+
+    return (temp >> 1) & 0xFFFFFFFF;
+}
+
+static inline uint8_t mipsdsp_rshift1_add_u8(uint8_t a, uint8_t b)
+{
+    uint16_t temp;
+
+    temp = (uint16_t)a + (uint16_t)b;
+
+    return (temp >> 1) & 0x00FF;
+}
+
+static inline uint8_t mipsdsp_rrshift1_add_u8(uint8_t a, uint8_t b)
+{
+    uint16_t temp;
+
+    temp = (uint16_t)a + (uint16_t)b + 1;
+
+    return (temp >> 1) & 0x00FF;
+}
+
+static inline uint8_t mipsdsp_rshift1_sub_u8(uint8_t a, uint8_t b)
+{
+    uint16_t temp;
+
+    temp = (uint16_t)a - (uint16_t)b;
+
+    return (temp >> 1) & 0x00FF;
+}
+
+static inline uint8_t mipsdsp_rrshift1_sub_u8(uint8_t a, uint8_t b)
+{
+    uint16_t temp;
+
+    temp = (uint16_t)a - (uint16_t)b + 1;
+
+    return (temp >> 1) & 0x00FF;
+}
+
+static inline int64_t mipsdsp_rashift_short_acc(int32_t ac,
+                                                int32_t shift,
+                                                CPUMIPSState *env)
+{
+    int32_t sign, temp31;
+    int64_t temp, acc;
+
+    sign = (env->active_tc.HI[ac] >> 31) & 0x01;
+    acc = ((int64_t)env->active_tc.HI[ac] << 32) |
+          ((int64_t)env->active_tc.LO[ac] & 0xFFFFFFFF);
+    if (shift == 0) {
+        temp = acc;
+    } else {
+        if (sign == 0) {
+            temp = (((int64_t)0x01 << (32 - shift + 1)) - 1) & (acc >> shift);
+        } else {
+            temp = ((((int64_t)0x01 << (shift + 1)) - 1) << (32 - shift)) |
+                   (acc >> shift);
+        }
+    }
+
+    temp31 = (temp >> 31) & 0x01;
+    if (sign != temp31) {
+        set_DSPControl_overflow_flag(1, 23, env);
+    }
+
+    return temp;
+}
+
+/*  128 bits long. p[0] is LO, p[1] is HI. */
+static inline void mipsdsp_rndrashift_short_acc(int64_t *p,
+                                                int32_t ac,
+                                                int32_t shift,
+                                                CPUMIPSState *env)
+{
+    int64_t acc;
+
+    acc = ((int64_t)env->active_tc.HI[ac] << 32) |
+          ((int64_t)env->active_tc.LO[ac] & 0xFFFFFFFF);
+    if (shift == 0) {
+        p[0] = acc << 1;
+        p[1] = (acc >> 63) & 0x01;
+    } else {
+        p[0] = acc >> (shift - 1);
+        p[1] = 0;
+    }
+}
+
+/* 128 bits long. p[0] is LO, p[1] is HI */
+static inline void mipsdsp_rashift_acc(uint64_t *p,
+                                       uint32_t ac,
+                                       uint32_t shift,
+                                       CPUMIPSState *env)
+{
+    uint64_t tempB, tempA;
+
+    tempB = env->active_tc.HI[ac];
+    tempA = env->active_tc.LO[ac];
+    shift = shift & 0x1F;
+
+    if (shift == 0) {
+        p[1] = tempB;
+        p[0] = tempA;
+    } else {
+        p[0] = (tempB << (64 - shift)) | (tempA >> shift);
+        p[1] = (int64_t)tempB >> shift;
+    }
+}
+
+/* 128 bits long. p[0] is LO, p[1] is HI , p[2] is sign of HI.*/
+static inline void mipsdsp_rndrashift_acc(uint64_t *p,
+                                          uint32_t ac,
+                                          uint32_t shift,
+                                          CPUMIPSState *env)
+{
+    int64_t tempB, tempA;
+
+    tempB = env->active_tc.HI[ac];
+    tempA = env->active_tc.LO[ac];
+    shift = shift & 0x3F;
+
+    if (shift == 0) {
+        p[2] = tempB >> 63;
+        p[1] = (tempB << 1) | (tempA >> 63);
+        p[0] = tempA << 1;
+    } else {
+        p[0] = (tempB << (65 - shift)) | (tempA >> (shift - 1));
+        p[1] = (int64_t)tempB >> (shift - 1);
+        if (tempB >= 0) {
+            p[2] = 0x0;
+        } else {
+            p[2] = ~0ull;
+        }
+    }
+}
+
+static inline int32_t mipsdsp_mul_q15_q15(int32_t ac, uint16_t a, uint16_t b,
+                                          CPUMIPSState *env)
+{
+    int32_t temp;
+
+    if ((a == 0x8000) && (b == 0x8000)) {
+        temp = 0x7FFFFFFF;
+        set_DSPControl_overflow_flag(1, 16 + ac, env);
+    } else {
+        temp = ((uint32_t)a * (uint32_t)b) << 1;
+    }
+
+    return temp;
+}
+
+static inline int64_t mipsdsp_mul_q31_q31(int32_t ac, uint32_t a, uint32_t b,
+                                          CPUMIPSState *env)
+{
+    uint64_t temp;
+
+    if ((a == 0x80000000) && (b == 0x80000000)) {
+        temp = (0x01ull << 63) - 1;
+        set_DSPControl_overflow_flag(1, 16 + ac, env);
+    } else {
+        temp = ((uint64_t)a * (uint64_t)b) << 1;
+    }
+
+    return temp;
+}
+
+static inline uint16_t mipsdsp_mul_u8_u8(uint8_t a, uint8_t b)
+{
+    return (uint16_t)a * (uint16_t)b;
+}
+
+static inline uint16_t mipsdsp_mul_u8_u16(uint8_t a, uint16_t b,
+                                          CPUMIPSState *env)
+{
+    uint32_t tempI;
+
+    tempI = (uint32_t)a * (uint32_t)b;
+    if (tempI > 0x0000FFFF) {
+        tempI = 0x0000FFFF;
+        set_DSPControl_overflow_flag(1, 21, env);
+    }
+
+    return tempI & 0x0000FFFF;
+}
+
+static inline uint64_t mipsdsp_mul_u32_u32(uint32_t a, uint32_t b)
+{
+    return (uint64_t)a * (uint64_t)b;
+}
+
+static inline int16_t mipsdsp_rndq15_mul_q15_q15(uint16_t a, uint16_t b,
+                                                 CPUMIPSState *env)
+{
+    uint32_t temp;
+
+    if ((a == 0x8000) && (b == 0x8000)) {
+        temp = 0x7FFF0000;
+        set_DSPControl_overflow_flag(1, 21, env);
+    } else {
+        temp = (a * b) << 1;
+        temp = temp + 0x00008000;
+    }
+
+    return (temp & 0xFFFF0000) >> 16;
+}
+
+static inline int32_t mipsdsp_sat16_mul_q15_q15(uint16_t a, uint16_t b,
+                                                CPUMIPSState *env)
+{
+    int32_t temp;
+
+    if ((a == 0x8000) && (b == 0x8000)) {
+        temp = 0x7FFF0000;
+        set_DSPControl_overflow_flag(1, 21, env);
+    } else {
+        temp = ((uint32_t)a * (uint32_t)b);
+        temp = temp << 1;
+    }
+
+    return (temp >> 16) & 0x0000FFFF;
+}
+
+static inline uint16_t mipsdsp_trunc16_sat16_round(int32_t a,
+                                                   CPUMIPSState *env)
+{
+    int64_t temp;
+
+    temp = (int32_t)a + 0x00008000;
+
+    if (a > (int)0x7fff8000) {
+        temp = 0x7FFFFFFF;
+        set_DSPControl_overflow_flag(1, 22, env);
+    }
+
+    return (temp >> 16) & 0xFFFF;
+}
+
+static inline uint8_t mipsdsp_sat8_reduce_precision(uint16_t a,
+                                                    CPUMIPSState *env)
+{
+    uint16_t mag;
+    uint32_t sign;
+
+    sign = (a >> 15) & 0x01;
+    mag = a & 0x7FFF;
+
+    if (sign == 0) {
+        if (mag > 0x7F80) {
+            set_DSPControl_overflow_flag(1, 22, env);
+            return 0xFF;
+        } else {
+            return (mag >> 7) & 0xFFFF;
+        }
+    } else {
+        set_DSPControl_overflow_flag(1, 22, env);
+        return 0x00;
+    }
+}
+
+static inline uint8_t mipsdsp_lshift8(uint8_t a, uint8_t s, CPUMIPSState *env)
+{
+    uint8_t sign;
+    uint8_t discard;
+
+    if (s == 0) {
+        return a;
+    } else {
+        sign = (a >> 7) & 0x01;
+        if (sign != 0) {
+            discard = (((0x01 << (8 - s)) - 1) << s) |
+                      ((a >> (6 - (s - 1))) & ((0x01 << s) - 1));
+        } else {
+            discard = a >> (6 - (s - 1));
+        }
+
+        if (discard != 0x00) {
+            set_DSPControl_overflow_flag(1, 22, env);
+        }
+        return a << s;
+    }
+}
+
+static inline uint16_t mipsdsp_lshift16(uint16_t a, uint8_t s,
+                                        CPUMIPSState *env)
+{
+    uint8_t  sign;
+    uint16_t discard;
+
+    if (s == 0) {
+        return a;
+    } else {
+        sign = (a >> 15) & 0x01;
+        if (sign != 0) {
+            discard = (((0x01 << (16 - s)) - 1) << s) |
+                      ((a >> (14 - (s - 1))) & ((0x01 << s) - 1));
+        } else {
+            discard = a >> (14 - (s - 1));
+        }
+
+        if ((discard != 0x0000) && (discard != 0xFFFF)) {
+            set_DSPControl_overflow_flag(1, 22, env);
+        }
+        return a << s;
+    }
+}
+
+
+static inline uint32_t mipsdsp_lshift32(uint32_t a, uint8_t s,
+                                        CPUMIPSState *env)
+{
+    uint32_t discard;
+
+    if (s == 0) {
+        return a;
+    } else {
+        discard = (int32_t)a >> (31 - (s - 1));
+
+        if ((discard != 0x00000000) && (discard != 0xFFFFFFFF)) {
+            set_DSPControl_overflow_flag(1, 22, env);
+        }
+        return a << s;
+    }
+}
+
+static inline uint16_t mipsdsp_sat16_lshift(uint16_t a, uint8_t s,
+                                            CPUMIPSState *env)
+{
+    uint8_t  sign;
+    uint16_t discard;
+
+    if (s == 0) {
+        return a;
+    } else {
+        sign = (a >> 15) & 0x01;
+        if (sign != 0) {
+            discard = (((0x01 << (16 - s)) - 1) << s) |
+                      ((a >> (14 - (s - 1))) & ((0x01 << s) - 1));
+        } else {
+            discard = a >> (14 - (s - 1));
+        }
+
+        if ((discard != 0x0000) && (discard != 0xFFFF)) {
+            set_DSPControl_overflow_flag(1, 22, env);
+            return (sign == 0) ? 0x7FFF : 0x8000;
+        } else {
+            return a << s;
+        }
+    }
+}
+
+static inline uint32_t mipsdsp_sat32_lshift(uint32_t a, uint8_t s,
+                                            CPUMIPSState *env)
+{
+    uint8_t  sign;
+    uint32_t discard;
+
+    if (s == 0) {
+        return a;
+    } else {
+        sign = (a >> 31) & 0x01;
+        if (sign != 0) {
+            discard = (((0x01 << (32 - s)) - 1) << s) |
+                      ((a >> (30 - (s - 1))) & ((0x01 << s) - 1));
+        } else {
+            discard = a >> (30 - (s - 1));
+        }
+
+        if ((discard != 0x00000000) && (discard != 0xFFFFFFFF)) {
+            set_DSPControl_overflow_flag(1, 22, env);
+            return (sign == 0) ? 0x7FFFFFFF : 0x80000000;
+        } else {
+            return a << s;
+        }
+    }
+}
+
+static inline uint8_t mipsdsp_rnd8_rashift(uint8_t a, uint8_t s)
+{
+    uint32_t temp;
+
+    if (s == 0) {
+        temp = (uint32_t)a << 1;
+    } else {
+        temp = (int32_t)(int8_t)a >> (s - 1);
+    }
+
+    return (temp + 1) >> 1;
+}
+
+static inline uint16_t mipsdsp_rnd16_rashift(uint16_t a, uint8_t s)
+{
+    uint32_t temp;
+
+    if (s == 0) {
+        temp = (uint32_t)a << 1;
+    } else {
+        temp = (int32_t)(int16_t)a >> (s - 1);
+    }
+
+    return (temp + 1) >> 1;
+}
+
+static inline uint32_t mipsdsp_rnd32_rashift(uint32_t a, uint8_t s)
+{
+    int64_t temp;
+
+    if (s == 0) {
+        temp = (uint64_t)a << 1;
+    } else {
+        temp = (int64_t)(int32_t)a >> (s - 1);
+    }
+    temp += 1;
+
+    return (temp >> 1) & 0xFFFFFFFFull;
+}
+
+static inline uint16_t mipsdsp_sub_i16(int16_t a, int16_t b, CPUMIPSState *env)
+{
+    int16_t  temp;
+
+    temp = a - b;
+    if (MIPSDSP_OVERFLOW(a, -b, temp, 0x8000)) {
+        set_DSPControl_overflow_flag(1, 20, env);
+    }
+
+    return temp;
+}
+
+static inline uint16_t mipsdsp_sat16_sub(int16_t a, int16_t b,
+                                         CPUMIPSState *env)
+{
+    int16_t  temp;
+
+    temp = a - b;
+    if (MIPSDSP_OVERFLOW(a, -b, temp, 0x8000)) {
+        if (a > 0) {
+            temp = 0x7FFF;
+        } else {
+            temp = 0x8000;
+        }
+        set_DSPControl_overflow_flag(1, 20, env);
+    }
+
+    return temp;
+}
+
+static inline uint32_t mipsdsp_sat32_sub(int32_t a, int32_t b,
+                                         CPUMIPSState *env)
+{
+    int32_t  temp;
+
+    temp = a - b;
+    if (MIPSDSP_OVERFLOW(a, -b, temp, 0x80000000)) {
+        if (a > 0) {
+            temp = 0x7FFFFFFF;
+        } else {
+            temp = 0x80000000;
+        }
+        set_DSPControl_overflow_flag(1, 20, env);
+    }
+
+    return temp & 0xFFFFFFFFull;
+}
+
+static inline uint16_t mipsdsp_rshift1_sub_q16(int16_t a, int16_t b)
+{
+    int32_t  temp;
+
+    temp = (int32_t)a - (int32_t)b;
+
+    return (temp >> 1) & 0x0000FFFF;
+}
+
+static inline uint16_t mipsdsp_rrshift1_sub_q16(int16_t a, int16_t b)
+{
+    int32_t  temp;
+
+    temp = (int32_t)a - (int32_t)b;
+    temp += 1;
+
+    return (temp >> 1) & 0x0000FFFF;
+}
+
+static inline uint32_t mipsdsp_rshift1_sub_q32(int32_t a, int32_t b)
+{
+    int64_t  temp;
+
+    temp = (int64_t)a - (int64_t)b;
+
+    return (temp >> 1) & 0xFFFFFFFFull;
+}
+
+static inline uint32_t mipsdsp_rrshift1_sub_q32(int32_t a, int32_t b)
+{
+    int64_t  temp;
+
+    temp = (int64_t)a - (int64_t)b;
+    temp += 1;
+
+    return (temp >> 1) & 0xFFFFFFFFull;
+}
+
+static inline uint16_t mipsdsp_sub_u16_u16(uint16_t a, uint16_t b,
+                                           CPUMIPSState *env)
+{
+    uint8_t  temp16;
+    uint32_t temp;
+
+    temp = (uint32_t)a - (uint32_t)b;
+    temp16 = (temp >> 16) & 0x01;
+    if (temp16 == 1) {
+        set_DSPControl_overflow_flag(1, 20, env);
+    }
+    return temp & 0x0000FFFF;
+}
+
+static inline uint16_t mipsdsp_satu16_sub_u16_u16(uint16_t a, uint16_t b,
+                                                  CPUMIPSState *env)
+{
+    uint8_t  temp16;
+    uint32_t temp;
+
+    temp   = (uint32_t)a - (uint32_t)b;
+    temp16 = (temp >> 16) & 0x01;
+
+    if (temp16 == 1) {
+        temp = 0x0000;
+        set_DSPControl_overflow_flag(1, 20, env);
+    }
+
+    return temp & 0x0000FFFF;
+}
+
+static inline uint8_t mipsdsp_sub_u8(uint8_t a, uint8_t b, CPUMIPSState *env)
+{
+    uint8_t  temp8;
+    uint16_t temp;
+
+    temp = (uint16_t)a - (uint16_t)b;
+    temp8 = (temp >> 8) & 0x01;
+    if (temp8 == 1) {
+        set_DSPControl_overflow_flag(1, 20, env);
+    }
+
+    return temp & 0x00FF;
+}
+
+static inline uint8_t mipsdsp_satu8_sub(uint8_t a, uint8_t b, CPUMIPSState *env)
+{
+    uint8_t  temp8;
+    uint16_t temp;
+
+    temp = (uint16_t)a - (uint16_t)b;
+    temp8 = (temp >> 8) & 0x01;
+    if (temp8 == 1) {
+        temp = 0x00;
+        set_DSPControl_overflow_flag(1, 20, env);
+    }
+
+    return temp & 0x00FF;
+}
+
+static inline uint32_t mipsdsp_sub32(int32_t a, int32_t b, CPUMIPSState *env)
+{
+    int32_t temp;
+
+    temp = a - b;
+    if (MIPSDSP_OVERFLOW(a, -b, temp, 0x80000000)) {
+        set_DSPControl_overflow_flag(1, 20, env);
+    }
+
+    return temp;
+}
+
+static inline int32_t mipsdsp_add_i32(int32_t a, int32_t b, CPUMIPSState *env)
+{
+    int32_t temp;
+
+    temp = a + b;
+
+    if (MIPSDSP_OVERFLOW(a, b, temp, 0x80000000)) {
+        set_DSPControl_overflow_flag(1, 20, env);
+    }
+
+    return temp;
+}
+
+static inline int32_t mipsdsp_cmp_eq(int32_t a, int32_t b)
+{
+    return a == b;
+}
+
+static inline int32_t mipsdsp_cmp_le(int32_t a, int32_t b)
+{
+    return a <= b;
+}
+
+static inline int32_t mipsdsp_cmp_lt(int32_t a, int32_t b)
+{
+    return a < b;
+}
+
+static inline int32_t mipsdsp_cmpu_eq(uint32_t a, uint32_t b)
+{
+    return a == b;
+}
+
+static inline int32_t mipsdsp_cmpu_le(uint32_t a, uint32_t b)
+{
+    return a <= b;
+}
+
+static inline int32_t mipsdsp_cmpu_lt(uint32_t a, uint32_t b)
+{
+    return a < b;
+}
+/*** MIPS DSP internal functions end ***/
+
+#define MIPSDSP_LHI 0xFFFFFFFF00000000ull
+#define MIPSDSP_LLO 0x00000000FFFFFFFFull
+#define MIPSDSP_HI  0xFFFF0000
+#define MIPSDSP_LO  0x0000FFFF
+#define MIPSDSP_Q3  0xFF000000
+#define MIPSDSP_Q2  0x00FF0000
+#define MIPSDSP_Q1  0x0000FF00
+#define MIPSDSP_Q0  0x000000FF
+
+#define MIPSDSP_SPLIT32_8(num, a, b, c, d)  \
+    do {                                    \
+        a = (num >> 24) & MIPSDSP_Q0;       \
+        b = (num >> 16) & MIPSDSP_Q0;       \
+        c = (num >> 8) & MIPSDSP_Q0;        \
+        d = num & MIPSDSP_Q0;               \
+    } while (0)
+
+#define MIPSDSP_SPLIT32_16(num, a, b)       \
+    do {                                    \
+        a = (num >> 16) & MIPSDSP_LO;       \
+        b = num & MIPSDSP_LO;               \
+    } while (0)
+
+#define MIPSDSP_RETURN32(a)             ((target_long)(int32_t)a)
+#define MIPSDSP_RETURN32_8(a, b, c, d)  ((target_long)(int32_t) \
+                                         (((uint32_t)a << 24) | \
+                                         (((uint32_t)b << 16) | \
+                                         (((uint32_t)c << 8) |  \
+                                          ((uint32_t)d & 0xFF)))))
+#define MIPSDSP_RETURN32_16(a, b)       ((target_long)(int32_t) \
+                                         (((uint32_t)a << 16) | \
+                                          ((uint32_t)b & 0xFFFF)))
+
+#ifdef TARGET_MIPS64
+#define MIPSDSP_SPLIT64_16(num, a, b, c, d)  \
+    do {                                     \
+        a = (num >> 48) & MIPSDSP_LO;        \
+        b = (num >> 32) & MIPSDSP_LO;        \
+        c = (num >> 16) & MIPSDSP_LO;        \
+        d = num & MIPSDSP_LO;                \
+    } while (0)
+
+#define MIPSDSP_SPLIT64_32(num, a, b)       \
+    do {                                    \
+        a = (num >> 32) & MIPSDSP_LLO;      \
+        b = num & MIPSDSP_LLO;              \
+    } while (0)
+
+#define MIPSDSP_RETURN64_16(a, b, c, d) (((uint64_t)a << 48) | \
+                                         ((uint64_t)b << 32) | \
+                                         ((uint64_t)c << 16) | \
+                                         (uint64_t)d)
+#define MIPSDSP_RETURN64_32(a, b)       (((uint64_t)a << 32) | (uint64_t)b)
+#endif
+
+/** DSP Arithmetic Sub-class insns **/
+#define ARITH_PH(name, func)                                      \
+target_ulong helper_##name##_ph(target_ulong rs, target_ulong rt) \
+{                                                                 \
+    uint16_t  rsh, rsl, rth, rtl, temph, templ;                   \
+                                                                  \
+    MIPSDSP_SPLIT32_16(rs, rsh, rsl);                             \
+    MIPSDSP_SPLIT32_16(rt, rth, rtl);                             \
+                                                                  \
+    temph = mipsdsp_##func(rsh, rth);                             \
+    templ = mipsdsp_##func(rsl, rtl);                             \
+                                                                  \
+    return MIPSDSP_RETURN32_16(temph, templ);                     \
+}
+
+#define ARITH_PH_ENV(name, func)                                  \
+target_ulong helper_##name##_ph(target_ulong rs, target_ulong rt, \
+                                CPUMIPSState *env)                \
+{                                                                 \
+    uint16_t  rsh, rsl, rth, rtl, temph, templ;                   \
+                                                                  \
+    MIPSDSP_SPLIT32_16(rs, rsh, rsl);                             \
+    MIPSDSP_SPLIT32_16(rt, rth, rtl);                             \
+                                                                  \
+    temph = mipsdsp_##func(rsh, rth, env);                        \
+    templ = mipsdsp_##func(rsl, rtl, env);                        \
+                                                                  \
+    return MIPSDSP_RETURN32_16(temph, templ);                     \
+}
+
+
+ARITH_PH_ENV(addq, add_i16);
+ARITH_PH_ENV(addq_s, sat_add_i16);
+ARITH_PH_ENV(addu, add_u16);
+ARITH_PH_ENV(addu_s, sat_add_u16);
+
+ARITH_PH(addqh, rshift1_add_q16);
+ARITH_PH(addqh_r, rrshift1_add_q16);
+
+ARITH_PH_ENV(subq, sub_i16);
+ARITH_PH_ENV(subq_s, sat16_sub);
+ARITH_PH_ENV(subu, sub_u16_u16);
+ARITH_PH_ENV(subu_s, satu16_sub_u16_u16);
+
+ARITH_PH(subqh, rshift1_sub_q16);
+ARITH_PH(subqh_r, rrshift1_sub_q16);
+
+#undef ARITH_PH
+#undef ARITH_PH_ENV
+
+#ifdef TARGET_MIPS64
+#define ARITH_QH_ENV(name, func) \
+target_ulong helper_##name##_qh(target_ulong rs, target_ulong rt, \
+                                CPUMIPSState *env)           \
+{                                                            \
+    uint16_t rs3, rs2, rs1, rs0;                             \
+    uint16_t rt3, rt2, rt1, rt0;                             \
+    uint16_t tempD, tempC, tempB, tempA;                     \
+                                                             \
+    MIPSDSP_SPLIT64_16(rs, rs3, rs2, rs1, rs0);              \
+    MIPSDSP_SPLIT64_16(rt, rt3, rt2, rt1, rt0);              \
+                                                             \
+    tempD = mipsdsp_##func(rs3, rt3, env);                   \
+    tempC = mipsdsp_##func(rs2, rt2, env);                   \
+    tempB = mipsdsp_##func(rs1, rt1, env);                   \
+    tempA = mipsdsp_##func(rs0, rt0, env);                   \
+                                                             \
+    return MIPSDSP_RETURN64_16(tempD, tempC, tempB, tempA);  \
+}
+
+ARITH_QH_ENV(addq, add_i16);
+ARITH_QH_ENV(addq_s, sat_add_i16);
+ARITH_QH_ENV(addu, add_u16);
+ARITH_QH_ENV(addu_s, sat_add_u16);
+
+ARITH_QH_ENV(subq, sub_i16);
+ARITH_QH_ENV(subq_s, sat16_sub);
+ARITH_QH_ENV(subu, sub_u16_u16);
+ARITH_QH_ENV(subu_s, satu16_sub_u16_u16);
+
+#undef ARITH_QH_ENV
+
+#endif
+
+#define ARITH_W(name, func) \
+target_ulong helper_##name##_w(target_ulong rs, target_ulong rt) \
+{                                                                \
+    uint32_t rd;                                                 \
+    rd = mipsdsp_##func(rs, rt);                                 \
+    return MIPSDSP_RETURN32(rd);                                 \
+}
+
+#define ARITH_W_ENV(name, func) \
+target_ulong helper_##name##_w(target_ulong rs, target_ulong rt, \
+                               CPUMIPSState *env)                \
+{                                                                \
+    uint32_t rd;                                                 \
+    rd = mipsdsp_##func(rs, rt, env);                            \
+    return MIPSDSP_RETURN32(rd);                                 \
+}
+
+ARITH_W_ENV(addq_s, sat_add_i32);
+
+ARITH_W(addqh, rshift1_add_q32);
+ARITH_W(addqh_r, rrshift1_add_q32);
+
+ARITH_W_ENV(subq_s, sat32_sub);
+
+ARITH_W(subqh, rshift1_sub_q32);
+ARITH_W(subqh_r, rrshift1_sub_q32);
+
+#undef ARITH_W
+#undef ARITH_W_ENV
+
+target_ulong helper_absq_s_w(target_ulong rt, CPUMIPSState *env)
+{
+    uint32_t rd;
+
+    rd = mipsdsp_sat_abs32(rt, env);
+
+    return (target_ulong)rd;
+}
+
+
+#if defined(TARGET_MIPS64)
+
+#define ARITH_PW_ENV(name, func) \
+target_ulong helper_##name##_pw(target_ulong rs, target_ulong rt, \
+                                CPUMIPSState *env)                \
+{                                                                 \
+    uint32_t rs1, rs0;                                            \
+    uint32_t rt1, rt0;                                            \
+    uint32_t tempB, tempA;                                        \
+                                                                  \
+    MIPSDSP_SPLIT64_32(rs, rs1, rs0);                             \
+    MIPSDSP_SPLIT64_32(rt, rt1, rt0);                             \
+                                                                  \
+    tempB = mipsdsp_##func(rs1, rt1, env);                        \
+    tempA = mipsdsp_##func(rs0, rt0, env);                        \
+                                                                  \
+    return MIPSDSP_RETURN64_32(tempB, tempA);                     \
+}
+
+ARITH_PW_ENV(addq, add_i32);
+ARITH_PW_ENV(addq_s, sat_add_i32);
+ARITH_PW_ENV(subq, sub32);
+ARITH_PW_ENV(subq_s, sat32_sub);
+
+#undef ARITH_PW_ENV
+
+#endif
+
+#define ARITH_QB(name, func) \
+target_ulong helper_##name##_qb(target_ulong rs, target_ulong rt) \
+{                                                                 \
+    uint8_t  rs0, rs1, rs2, rs3;                                  \
+    uint8_t  rt0, rt1, rt2, rt3;                                  \
+    uint8_t  temp0, temp1, temp2, temp3;                          \
+                                                                  \
+    MIPSDSP_SPLIT32_8(rs, rs3, rs2, rs1, rs0);                    \
+    MIPSDSP_SPLIT32_8(rt, rt3, rt2, rt1, rt0);                    \
+                                                                  \
+    temp0 = mipsdsp_##func(rs0, rt0);                             \
+    temp1 = mipsdsp_##func(rs1, rt1);                             \
+    temp2 = mipsdsp_##func(rs2, rt2);                             \
+    temp3 = mipsdsp_##func(rs3, rt3);                             \
+                                                                  \
+    return MIPSDSP_RETURN32_8(temp3, temp2, temp1, temp0);        \
+}
+
+#define ARITH_QB_ENV(name, func) \
+target_ulong helper_##name##_qb(target_ulong rs, target_ulong rt, \
+                                CPUMIPSState *env)          \
+{                                                           \
+    uint8_t  rs0, rs1, rs2, rs3;                            \
+    uint8_t  rt0, rt1, rt2, rt3;                            \
+    uint8_t  temp0, temp1, temp2, temp3;                    \
+                                                            \
+    MIPSDSP_SPLIT32_8(rs, rs3, rs2, rs1, rs0);              \
+    MIPSDSP_SPLIT32_8(rt, rt3, rt2, rt1, rt0);              \
+                                                            \
+    temp0 = mipsdsp_##func(rs0, rt0, env);                  \
+    temp1 = mipsdsp_##func(rs1, rt1, env);                  \
+    temp2 = mipsdsp_##func(rs2, rt2, env);                  \
+    temp3 = mipsdsp_##func(rs3, rt3, env);                  \
+                                                            \
+    return MIPSDSP_RETURN32_8(temp3, temp2, temp1, temp0);  \
+}
+
+ARITH_QB(adduh, rshift1_add_u8);
+ARITH_QB(adduh_r, rrshift1_add_u8);
+
+ARITH_QB_ENV(addu, add_u8);
+ARITH_QB_ENV(addu_s, sat_add_u8);
+
+#undef ADDU_QB
+#undef ADDU_QB_ENV
+
+#if defined(TARGET_MIPS64)
+#define ARITH_OB(name, func) \
+target_ulong helper_##name##_ob(target_ulong rs, target_ulong rt) \
+{                                                                 \
+    int i;                                                        \
+    uint8_t rs_t[8], rt_t[8];                                     \
+    uint8_t temp[8];                                              \
+    uint64_t result;                                              \
+                                                                  \
+    result = 0;                                                   \
+                                                                  \
+    for (i = 0; i < 8; i++) {                                     \
+        rs_t[i] = (rs >> (8 * i)) & MIPSDSP_Q0;                   \
+        rt_t[i] = (rt >> (8 * i)) & MIPSDSP_Q0;                   \
+        temp[i] = mipsdsp_##func(rs_t[i], rt_t[i]);               \
+        result |= (uint64_t)temp[i] << (8 * i);                   \
+    }                                                             \
+                                                                  \
+    return result;                                                \
+}
+
+#define ARITH_OB_ENV(name, func) \
+target_ulong helper_##name##_ob(target_ulong rs, target_ulong rt, \
+                                CPUMIPSState *env)                \
+{                                                                 \
+    int i;                                                        \
+    uint8_t rs_t[8], rt_t[8];                                     \
+    uint8_t temp[8];                                              \
+    uint64_t result;                                              \
+                                                                  \
+    result = 0;                                                   \
+                                                                  \
+    for (i = 0; i < 8; i++) {                                     \
+        rs_t[i] = (rs >> (8 * i)) & MIPSDSP_Q0;                   \
+        rt_t[i] = (rt >> (8 * i)) & MIPSDSP_Q0;                   \
+        temp[i] = mipsdsp_##func(rs_t[i], rt_t[i], env);          \
+        result |= (uint64_t)temp[i] << (8 * i);                   \
+    }                                                             \
+                                                                  \
+    return result;                                                \
+}
+
+ARITH_OB_ENV(addu, add_u8);
+ARITH_OB_ENV(addu_s, sat_add_u8);
+
+ARITH_OB(adduh, rshift1_add_u8);
+ARITH_OB(adduh_r, rrshift1_add_u8);
+
+ARITH_OB_ENV(subu, sub_u8);
+ARITH_OB_ENV(subu_s, satu8_sub);
+
+ARITH_OB(subuh, rshift1_sub_u8);
+ARITH_OB(subuh_r, rrshift1_sub_u8);
+
+#undef ARITH_OB
+#undef ARITH_OB_ENV
+
+#endif
+
+#define SUBU_QB(name, func) \
+target_ulong helper_##name##_qb(target_ulong rs,               \
+                                target_ulong rt,               \
+                                CPUMIPSState *env)             \
+{                                                              \
+    uint8_t rs3, rs2, rs1, rs0;                                \
+    uint8_t rt3, rt2, rt1, rt0;                                \
+    uint8_t tempD, tempC, tempB, tempA;                        \
+                                                               \
+    MIPSDSP_SPLIT32_8(rs, rs3, rs2, rs1, rs0);                 \
+    MIPSDSP_SPLIT32_8(rt, rt3, rt2, rt1, rt0);                 \
+                                                               \
+    tempD = mipsdsp_##func(rs3, rt3, env);                     \
+    tempC = mipsdsp_##func(rs2, rt2, env);                     \
+    tempB = mipsdsp_##func(rs1, rt1, env);                     \
+    tempA = mipsdsp_##func(rs0, rt0, env);                     \
+                                                               \
+    return MIPSDSP_RETURN32_8(tempD, tempC, tempB, tempA);     \
+}
+
+SUBU_QB(subu, sub_u8);
+SUBU_QB(subu_s, satu8_sub);
+
+#undef SUBU_QB
+
+#define SUBUH_QB(name, var) \
+target_ulong helper_##name##_qb(target_ulong rs, target_ulong rt) \
+{                                                                 \
+    uint8_t rs3, rs2, rs1, rs0;                                   \
+    uint8_t rt3, rt2, rt1, rt0;                                   \
+    uint8_t tempD, tempC, tempB, tempA;                           \
+                                                                  \
+    MIPSDSP_SPLIT32_8(rs, rs3, rs2, rs1, rs0);                    \
+    MIPSDSP_SPLIT32_8(rt, rt3, rt2, rt1, rt0);                    \
+                                                                  \
+    tempD = ((uint16_t)rs3 - (uint16_t)rt3 + var) >> 1;           \
+    tempC = ((uint16_t)rs2 - (uint16_t)rt2 + var) >> 1;           \
+    tempB = ((uint16_t)rs1 - (uint16_t)rt1 + var) >> 1;           \
+    tempA = ((uint16_t)rs0 - (uint16_t)rt0 + var) >> 1;           \
+                                                                  \
+    return ((uint32_t)tempD << 24) | ((uint32_t)tempC << 16) |    \
+        ((uint32_t)tempB << 8) | ((uint32_t)tempA);               \
+}
+
+SUBUH_QB(subuh, 0);
+SUBUH_QB(subuh_r, 1);
+
+#undef SUBUH_QB
+
+target_ulong helper_addsc(target_ulong rs, target_ulong rt, CPUMIPSState *env)
+{
+    uint64_t temp, tempRs, tempRt;
+    int32_t flag;
+
+    tempRs = (uint64_t)rs & MIPSDSP_LLO;
+    tempRt = (uint64_t)rt & MIPSDSP_LLO;
+
+    temp = tempRs + tempRt;
+    flag = (temp & 0x0100000000ull) >> 32;
+    set_DSPControl_carryflag(flag, env);
+
+    return (target_long)(int32_t)(temp & MIPSDSP_LLO);
+}
+
+target_ulong helper_addwc(target_ulong rs, target_ulong rt, CPUMIPSState *env)
+{
+    uint32_t rd;
+    int32_t temp32, temp31;
+    int64_t tempL;
+
+    tempL = (int64_t)(int32_t)rs + (int64_t)(int32_t)rt +
+        get_DSPControl_carryflag(env);
+    temp31 = (tempL >> 31) & 0x01;
+    temp32 = (tempL >> 32) & 0x01;
+
+    if (temp31 != temp32) {
+        set_DSPControl_overflow_flag(1, 20, env);
+    }
+
+    rd = tempL & MIPSDSP_LLO;
+
+    return (target_long)(int32_t)rd;
+}
+
+target_ulong helper_modsub(target_ulong rs, target_ulong rt)
+{
+    int32_t decr;
+    uint16_t lastindex;
+    target_ulong rd;
+
+    decr = rt & MIPSDSP_Q0;
+    lastindex = (rt >> 8) & MIPSDSP_LO;
+
+    if ((rs & MIPSDSP_LLO) == 0x00000000) {
+        rd = (target_ulong)lastindex;
+    } else {
+        rd = rs - decr;
+    }
+
+    return rd;
+}
+
+target_ulong helper_raddu_w_qb(target_ulong rs)
+{
+    uint8_t  rs3, rs2, rs1, rs0;
+    uint16_t temp;
+
+    MIPSDSP_SPLIT32_8(rs, rs3, rs2, rs1, rs0);
+
+    temp = (uint16_t)rs3 + (uint16_t)rs2 + (uint16_t)rs1 + (uint16_t)rs0;
+
+    return (target_ulong)temp;
+}
+
+#if defined(TARGET_MIPS64)
+target_ulong helper_raddu_l_ob(target_ulong rs)
+{
+    int i;
+    uint16_t rs_t[8];
+    uint64_t temp;
+
+    temp = 0;
+
+    for (i = 0; i < 8; i++) {
+        rs_t[i] = (rs >> (8 * i)) & MIPSDSP_Q0;
+        temp += (uint64_t)rs_t[i];
+    }
+
+    return temp;
+}
+#endif
+
+target_ulong helper_absq_s_qb(target_ulong rt, CPUMIPSState *env)
+{
+    uint8_t tempD, tempC, tempB, tempA;
+
+    MIPSDSP_SPLIT32_8(rt, tempD, tempC, tempB, tempA);
+
+    tempD = mipsdsp_sat_abs8(tempD, env);
+    tempC = mipsdsp_sat_abs8(tempC, env);
+    tempB = mipsdsp_sat_abs8(tempB, env);
+    tempA = mipsdsp_sat_abs8(tempA, env);
+
+    return MIPSDSP_RETURN32_8(tempD, tempC, tempB, tempA);
+}
+
+target_ulong helper_absq_s_ph(target_ulong rt, CPUMIPSState *env)
+{
+    uint16_t tempB, tempA;
+
+    MIPSDSP_SPLIT32_16(rt, tempB, tempA);
+
+    tempB = mipsdsp_sat_abs16 (tempB, env);
+    tempA = mipsdsp_sat_abs16 (tempA, env);
+
+    return MIPSDSP_RETURN32_16(tempB, tempA);
+}
+
+#if defined(TARGET_MIPS64)
+target_ulong helper_absq_s_ob(target_ulong rt, CPUMIPSState *env)
+{
+    int i;
+    int8_t temp[8];
+    uint64_t result;
+
+    for (i = 0; i < 8; i++) {
+        temp[i] = (rt >> (8 * i)) & MIPSDSP_Q0;
+        temp[i] = mipsdsp_sat_abs8(temp[i], env);
+    }
+
+    for (i = 0; i < 8; i++) {
+        result = (uint64_t)(uint8_t)temp[i] << (8 * i);
+    }
+
+    return result;
+}
+
+target_ulong helper_absq_s_qh(target_ulong rt, CPUMIPSState *env)
+{
+    int16_t tempD, tempC, tempB, tempA;
+
+    MIPSDSP_SPLIT64_16(rt, tempD, tempC, tempB, tempA);
+
+    tempD = mipsdsp_sat_abs16(tempD, env);
+    tempC = mipsdsp_sat_abs16(tempC, env);
+    tempB = mipsdsp_sat_abs16(tempB, env);
+    tempA = mipsdsp_sat_abs16(tempA, env);
+
+    return MIPSDSP_RETURN64_16(tempD, tempC, tempB, tempA);
+}
+
+target_ulong helper_absq_s_pw(target_ulong rt, CPUMIPSState *env)
+{
+    int32_t tempB, tempA;
+
+    MIPSDSP_SPLIT64_32(rt, tempB, tempA);
+
+    tempB = mipsdsp_sat_abs32(tempB, env);
+    tempA = mipsdsp_sat_abs32(tempA, env);
+
+    return MIPSDSP_RETURN64_32(tempB, tempA);
+}
+#endif
+
+#define PRECR_QB_PH(name, a, b)\
+target_ulong helper_##name##_qb_ph(target_ulong rs, target_ulong rt) \
+{                                                                    \
+    uint8_t tempD, tempC, tempB, tempA;                              \
+                                                                     \
+    tempD = (rs >> a) & MIPSDSP_Q0;                                  \
+    tempC = (rs >> b) & MIPSDSP_Q0;                                  \
+    tempB = (rt >> a) & MIPSDSP_Q0;                                  \
+    tempA = (rt >> b) & MIPSDSP_Q0;                                  \
+                                                                     \
+    return MIPSDSP_RETURN32_8(tempD, tempC, tempB, tempA);           \
+}
+
+PRECR_QB_PH(precr, 16, 0);
+PRECR_QB_PH(precrq, 24, 8);
+
+#undef PRECR_QB_OH
+
+target_ulong helper_precr_sra_ph_w(uint32_t sa, target_ulong rs,
+                                   target_ulong rt)
+{
+    uint16_t tempB, tempA;
+
+    tempB = ((int32_t)rt >> sa) & MIPSDSP_LO;
+    tempA = ((int32_t)rs >> sa) & MIPSDSP_LO;
+
+    return MIPSDSP_RETURN32_16(tempB, tempA);
+}
+
+target_ulong helper_precr_sra_r_ph_w(uint32_t sa,
+                                     target_ulong rs, target_ulong rt)
+{
+    uint64_t tempB, tempA;
+
+    /* If sa = 0, then (sa - 1) = -1 will case shift error, so we need else. */
+    if (sa == 0) {
+        tempB = (rt & MIPSDSP_LO) << 1;
+        tempA = (rs & MIPSDSP_LO) << 1;
+    } else {
+        tempB = ((int32_t)rt >> (sa - 1)) + 1;
+        tempA = ((int32_t)rs >> (sa - 1)) + 1;
+    }
+    rt = (((tempB >> 1) & MIPSDSP_LO) << 16) | ((tempA >> 1) & MIPSDSP_LO);
+
+    return (target_long)(int32_t)rt;
+}
+
+target_ulong helper_precrq_ph_w(target_ulong rs, target_ulong rt)
+{
+    uint16_t tempB, tempA;
+
+    tempB = (rs & MIPSDSP_HI) >> 16;
+    tempA = (rt & MIPSDSP_HI) >> 16;
+
+    return MIPSDSP_RETURN32_16(tempB, tempA);
+}
+
+target_ulong helper_precrq_rs_ph_w(target_ulong rs, target_ulong rt,
+                                   CPUMIPSState *env)
+{
+    uint16_t tempB, tempA;
+
+    tempB = mipsdsp_trunc16_sat16_round(rs, env);
+    tempA = mipsdsp_trunc16_sat16_round(rt, env);
+
+    return MIPSDSP_RETURN32_16(tempB, tempA);
+}
+
+#if defined(TARGET_MIPS64)
+target_ulong helper_precr_ob_qh(target_ulong rs, target_ulong rt)
+{
+    uint8_t rs6, rs4, rs2, rs0;
+    uint8_t rt6, rt4, rt2, rt0;
+    uint64_t temp;
+
+    rs6 = (rs >> 48) & MIPSDSP_Q0;
+    rs4 = (rs >> 32) & MIPSDSP_Q0;
+    rs2 = (rs >> 16) & MIPSDSP_Q0;
+    rs0 = rs & MIPSDSP_Q0;
+    rt6 = (rt >> 48) & MIPSDSP_Q0;
+    rt4 = (rt >> 32) & MIPSDSP_Q0;
+    rt2 = (rt >> 16) & MIPSDSP_Q0;
+    rt0 = rt & MIPSDSP_Q0;
+
+    temp = ((uint64_t)rs6 << 56) | ((uint64_t)rs4 << 48) |
+           ((uint64_t)rs2 << 40) | ((uint64_t)rs0 << 32) |
+           ((uint64_t)rt6 << 24) | ((uint64_t)rt4 << 16) |
+           ((uint64_t)rt2 << 8) | (uint64_t)rt0;
+
+    return temp;
+}
+
+#define PRECR_QH_PW(name, var) \
+target_ulong helper_precr_##name##_qh_pw(target_ulong rs, target_ulong rt, \
+                                    uint32_t sa)                      \
+{                                                                     \
+    uint16_t rs3, rs2, rs1, rs0;                                      \
+    uint16_t rt3, rt2, rt1, rt0;                                      \
+    uint16_t tempD, tempC, tempB, tempA;                              \
+                                                                      \
+    MIPSDSP_SPLIT64_16(rs, rs3, rs2, rs1, rs0);                       \
+    MIPSDSP_SPLIT64_16(rt, rt3, rt2, rt1, rt0);                       \
+                                                                      \
+    /* When sa = 0, we use rt2, rt0, rs2, rs0;                        \
+     * when sa != 0, we use rt3, rt1, rs3, rs1. */                    \
+    if (sa == 0) {                                                    \
+        tempD = rt2 << var;                                           \
+        tempC = rt0 << var;                                           \
+        tempB = rs2 << var;                                           \
+        tempA = rs0 << var;                                           \
+    } else {                                                          \
+        tempD = (((int16_t)rt3 >> sa) + var) >> var;                  \
+        tempC = (((int16_t)rt1 >> sa) + var) >> var;                  \
+        tempB = (((int16_t)rs3 >> sa) + var) >> var;                  \
+        tempA = (((int16_t)rs1 >> sa) + var) >> var;                  \
+    }                                                                 \
+                                                                      \
+    return MIPSDSP_RETURN64_16(tempD, tempC, tempB, tempA);           \
+}
+
+PRECR_QH_PW(sra, 0);
+PRECR_QH_PW(sra_r, 1);
+
+#undef PRECR_QH_PW
+
+target_ulong helper_precrq_ob_qh(target_ulong rs, target_ulong rt)
+{
+    uint8_t rs6, rs4, rs2, rs0;
+    uint8_t rt6, rt4, rt2, rt0;
+    uint64_t temp;
+
+    rs6 = (rs >> 56) & MIPSDSP_Q0;
+    rs4 = (rs >> 40) & MIPSDSP_Q0;
+    rs2 = (rs >> 24) & MIPSDSP_Q0;
+    rs0 = (rs >> 8) & MIPSDSP_Q0;
+    rt6 = (rt >> 56) & MIPSDSP_Q0;
+    rt4 = (rt >> 40) & MIPSDSP_Q0;
+    rt2 = (rt >> 24) & MIPSDSP_Q0;
+    rt0 = (rt >> 8) & MIPSDSP_Q0;
+
+    temp = ((uint64_t)rs6 << 56) | ((uint64_t)rs4 << 48) |
+           ((uint64_t)rs2 << 40) | ((uint64_t)rs0 << 32) |
+           ((uint64_t)rt6 << 24) | ((uint64_t)rt4 << 16) |
+           ((uint64_t)rt2 << 8) | (uint64_t)rt0;
+
+    return temp;
+}
+
+target_ulong helper_precrq_qh_pw(target_ulong rs, target_ulong rt)
+{
+    uint16_t tempD, tempC, tempB, tempA;
+
+    tempD = (rs >> 48) & MIPSDSP_LO;
+    tempC = (rs >> 16) & MIPSDSP_LO;
+    tempB = (rt >> 48) & MIPSDSP_LO;
+    tempA = (rt >> 16) & MIPSDSP_LO;
+
+    return MIPSDSP_RETURN64_16(tempD, tempC, tempB, tempA);
+}
+
+target_ulong helper_precrq_rs_qh_pw(target_ulong rs, target_ulong rt,
+                                    CPUMIPSState *env)
+{
+    uint32_t rs2, rs0;
+    uint32_t rt2, rt0;
+    uint16_t tempD, tempC, tempB, tempA;
+
+    rs2 = (rs >> 32) & MIPSDSP_LLO;
+    rs0 = rs & MIPSDSP_LLO;
+    rt2 = (rt >> 32) & MIPSDSP_LLO;
+    rt0 = rt & MIPSDSP_LLO;
+
+    tempD = mipsdsp_trunc16_sat16_round(rs2, env);
+    tempC = mipsdsp_trunc16_sat16_round(rs0, env);
+    tempB = mipsdsp_trunc16_sat16_round(rt2, env);
+    tempA = mipsdsp_trunc16_sat16_round(rt0, env);
+
+    return MIPSDSP_RETURN64_16(tempD, tempC, tempB, tempA);
+}
+
+target_ulong helper_precrq_pw_l(target_ulong rs, target_ulong rt)
+{
+    uint32_t tempB, tempA;
+
+    tempB = (rs >> 32) & MIPSDSP_LLO;
+    tempA = (rt >> 32) & MIPSDSP_LLO;
+
+    return MIPSDSP_RETURN64_32(tempB, tempA);
+}
+#endif
+
+target_ulong helper_precrqu_s_qb_ph(target_ulong rs, target_ulong rt,
+                                    CPUMIPSState *env)
+{
+    uint8_t  tempD, tempC, tempB, tempA;
+    uint16_t rsh, rsl, rth, rtl;
+
+    rsh = (rs & MIPSDSP_HI) >> 16;
+    rsl =  rs & MIPSDSP_LO;
+    rth = (rt & MIPSDSP_HI) >> 16;
+    rtl =  rt & MIPSDSP_LO;
+
+    tempD = mipsdsp_sat8_reduce_precision(rsh, env);
+    tempC = mipsdsp_sat8_reduce_precision(rsl, env);
+    tempB = mipsdsp_sat8_reduce_precision(rth, env);
+    tempA = mipsdsp_sat8_reduce_precision(rtl, env);
+
+    return MIPSDSP_RETURN32_8(tempD, tempC, tempB, tempA);
+}
+
+#if defined(TARGET_MIPS64)
+target_ulong helper_precrqu_s_ob_qh(target_ulong rs, target_ulong rt,
+                                    CPUMIPSState *env)
+{
+    int i;
+    uint16_t rs3, rs2, rs1, rs0;
+    uint16_t rt3, rt2, rt1, rt0;
+    uint8_t temp[8];
+    uint64_t result;
+
+    result = 0;
+
+    MIPSDSP_SPLIT64_16(rs, rs3, rs2, rs1, rs0);
+    MIPSDSP_SPLIT64_16(rt, rt3, rt2, rt1, rt0);
+
+    temp[7] = mipsdsp_sat8_reduce_precision(rs3, env);
+    temp[6] = mipsdsp_sat8_reduce_precision(rs2, env);
+    temp[5] = mipsdsp_sat8_reduce_precision(rs1, env);
+    temp[4] = mipsdsp_sat8_reduce_precision(rs0, env);
+    temp[3] = mipsdsp_sat8_reduce_precision(rt3, env);
+    temp[2] = mipsdsp_sat8_reduce_precision(rt2, env);
+    temp[1] = mipsdsp_sat8_reduce_precision(rt1, env);
+    temp[0] = mipsdsp_sat8_reduce_precision(rt0, env);
+
+    for (i = 0; i < 8; i++) {
+        result |= (uint64_t)temp[i] << (8 * i);
+    }
+
+    return result;
+}
+
+#define PRECEQ_PW(name, a, b) \
+target_ulong helper_preceq_pw_##name(target_ulong rt) \
+{                                                       \
+    uint16_t tempB, tempA;                              \
+    uint32_t tempBI, tempAI;                            \
+                                                        \
+    tempB = (rt >> a) & MIPSDSP_LO;                     \
+    tempA = (rt >> b) & MIPSDSP_LO;                     \
+                                                        \
+    tempBI = (uint32_t)tempB << 16;                     \
+    tempAI = (uint32_t)tempA << 16;                     \
+                                                        \
+    return MIPSDSP_RETURN64_32(tempBI, tempAI);         \
+}
+
+PRECEQ_PW(qhl, 48, 32);
+PRECEQ_PW(qhr, 16, 0);
+PRECEQ_PW(qhla, 48, 16);
+PRECEQ_PW(qhra, 32, 0);
+
+#undef PRECEQ_PW
+
+#endif
+
+#define PRECEQU_PH(name, a, b) \
+target_ulong helper_precequ_ph_##name(target_ulong rt) \
+{                                                        \
+    uint16_t tempB, tempA;                               \
+                                                         \
+    tempB = (rt >> a) & MIPSDSP_Q0;                      \
+    tempA = (rt >> b) & MIPSDSP_Q0;                      \
+                                                         \
+    tempB = tempB << 7;                                  \
+    tempA = tempA << 7;                                  \
+                                                         \
+    return MIPSDSP_RETURN32_16(tempB, tempA);            \
+}
+
+PRECEQU_PH(qbl, 24, 16);
+PRECEQU_PH(qbr, 8, 0);
+PRECEQU_PH(qbla, 24, 8);
+PRECEQU_PH(qbra, 16, 0);
+
+#undef PRECEQU_PH
+
+#if defined(TARGET_MIPS64)
+#define PRECEQU_QH(name, a, b, c, d) \
+target_ulong helper_precequ_qh_##name(target_ulong rt)       \
+{                                                            \
+    uint16_t tempD, tempC, tempB, tempA;                     \
+                                                             \
+    tempD = (rt >> a) & MIPSDSP_Q0;                          \
+    tempC = (rt >> b) & MIPSDSP_Q0;                          \
+    tempB = (rt >> c) & MIPSDSP_Q0;                          \
+    tempA = (rt >> d) & MIPSDSP_Q0;                          \
+                                                             \
+    tempD = tempD << 7;                                      \
+    tempC = tempC << 7;                                      \
+    tempB = tempB << 7;                                      \
+    tempA = tempA << 7;                                      \
+                                                             \
+    return MIPSDSP_RETURN64_16(tempD, tempC, tempB, tempA);  \
+}
+
+PRECEQU_QH(obl, 56, 48, 40, 32);
+PRECEQU_QH(obr, 24, 16, 8, 0);
+PRECEQU_QH(obla, 56, 40, 24, 8);
+PRECEQU_QH(obra, 48, 32, 16, 0);
+
+#undef PRECEQU_QH
+
+#endif
+
+#define PRECEU_PH(name, a, b) \
+target_ulong helper_preceu_ph_##name(target_ulong rt) \
+{                                                     \
+    uint16_t tempB, tempA;                            \
+                                                      \
+    tempB = (rt >> a) & MIPSDSP_Q0;                   \
+    tempA = (rt >> b) & MIPSDSP_Q0;                   \
+                                                      \
+    return MIPSDSP_RETURN32_16(tempB, tempA);         \
+}
+
+PRECEU_PH(qbl, 24, 16);
+PRECEU_PH(qbr, 8, 0);
+PRECEU_PH(qbla, 24, 8);
+PRECEU_PH(qbra, 16, 0);
+
+#undef PRECEU_PH
+
+#if defined(TARGET_MIPS64)
+#define PRECEU_QH(name, a, b, c, d) \
+target_ulong helper_preceu_qh_##name(target_ulong rt)        \
+{                                                            \
+    uint16_t tempD, tempC, tempB, tempA;                     \
+                                                             \
+    tempD = (rt >> a) & MIPSDSP_Q0;                          \
+    tempC = (rt >> b) & MIPSDSP_Q0;                          \
+    tempB = (rt >> c) & MIPSDSP_Q0;                          \
+    tempA = (rt >> d) & MIPSDSP_Q0;                          \
+                                                             \
+    return MIPSDSP_RETURN64_16(tempD, tempC, tempB, tempA);  \
+}
+
+PRECEU_QH(obl, 56, 48, 40, 32);
+PRECEU_QH(obr, 24, 16, 8, 0);
+PRECEU_QH(obla, 56, 40, 24, 8);
+PRECEU_QH(obra, 48, 32, 16, 0);
+
+#undef PRECEU_QH
+
+#endif
+
+/** DSP GPR-Based Shift Sub-class insns **/
+#define SHIFT_QB(name, func) \
+target_ulong helper_##name##_qb(target_ulong sa, target_ulong rt) \
+{                                                                    \
+    uint8_t rt3, rt2, rt1, rt0;                                      \
+                                                                     \
+    sa = sa & 0x07;                                                  \
+                                                                     \
+    MIPSDSP_SPLIT32_8(rt, rt3, rt2, rt1, rt0);                       \
+                                                                     \
+    rt3 = mipsdsp_##func(rt3, sa);                                   \
+    rt2 = mipsdsp_##func(rt2, sa);                                   \
+    rt1 = mipsdsp_##func(rt1, sa);                                   \
+    rt0 = mipsdsp_##func(rt0, sa);                                   \
+                                                                     \
+    return MIPSDSP_RETURN32_8(rt3, rt2, rt1, rt0);                   \
+}
+
+#define SHIFT_QB_ENV(name, func) \
+target_ulong helper_##name##_qb(target_ulong sa, target_ulong rt,\
+                                CPUMIPSState *env) \
+{                                                                    \
+    uint8_t rt3, rt2, rt1, rt0;                                      \
+                                                                     \
+    sa = sa & 0x07;                                                  \
+                                                                     \
+    MIPSDSP_SPLIT32_8(rt, rt3, rt2, rt1, rt0);                       \
+                                                                     \
+    rt3 = mipsdsp_##func(rt3, sa, env);                              \
+    rt2 = mipsdsp_##func(rt2, sa, env);                              \
+    rt1 = mipsdsp_##func(rt1, sa, env);                              \
+    rt0 = mipsdsp_##func(rt0, sa, env);                              \
+                                                                     \
+    return MIPSDSP_RETURN32_8(rt3, rt2, rt1, rt0);                   \
+}
+
+SHIFT_QB_ENV(shll, lshift8);
+SHIFT_QB(shrl, rshift_u8);
+
+SHIFT_QB(shra, rashift8);
+SHIFT_QB(shra_r, rnd8_rashift);
+
+#undef SHIFT_QB
+#undef SHIFT_QB_ENV
+
+#if defined(TARGET_MIPS64)
+#define SHIFT_OB(name, func) \
+target_ulong helper_##name##_ob(target_ulong rt, target_ulong sa) \
+{                                                                        \
+    int i;                                                               \
+    uint8_t rt_t[8];                                                     \
+    uint64_t temp;                                                       \
+                                                                         \
+    sa = sa & 0x07;                                                      \
+    temp = 0;                                                            \
+                                                                         \
+    for (i = 0; i < 8; i++) {                                            \
+        rt_t[i] = (rt >> (8 * i)) & MIPSDSP_Q0;                          \
+        rt_t[i] = mipsdsp_##func(rt_t[i], sa);                           \
+        temp |= (uint64_t)rt_t[i] << (8 * i);                            \
+    }                                                                    \
+                                                                         \
+    return temp;                                                         \
+}
+
+#define SHIFT_OB_ENV(name, func) \
+target_ulong helper_##name##_ob(target_ulong rt, target_ulong sa, \
+                                CPUMIPSState *env)                       \
+{                                                                        \
+    int i;                                                               \
+    uint8_t rt_t[8];                                                     \
+    uint64_t temp;                                                       \
+                                                                         \
+    sa = sa & 0x07;                                                      \
+    temp = 0;                                                            \
+                                                                         \
+    for (i = 0; i < 8; i++) {                                            \
+        rt_t[i] = (rt >> (8 * i)) & MIPSDSP_Q0;                          \
+        rt_t[i] = mipsdsp_##func(rt_t[i], sa, env);                      \
+        temp |= (uint64_t)rt_t[i] << (8 * i);                            \
+    }                                                                    \
+                                                                         \
+    return temp;                                                         \
+}
+
+SHIFT_OB_ENV(shll, lshift8);
+SHIFT_OB(shrl, rshift_u8);
+
+SHIFT_OB(shra, rashift8);
+SHIFT_OB(shra_r, rnd8_rashift);
+
+#undef SHIFT_OB
+#undef SHIFT_OB_ENV
+
+#endif
+
+#define SHIFT_PH(name, func) \
+target_ulong helper_##name##_ph(target_ulong sa, target_ulong rt, \
+                                CPUMIPSState *env)                \
+{                                                                 \
+    uint16_t rth, rtl;                                            \
+                                                                  \
+    sa = sa & 0x0F;                                               \
+                                                                  \
+    MIPSDSP_SPLIT32_16(rt, rth, rtl);                             \
+                                                                  \
+    rth = mipsdsp_##func(rth, sa, env);                           \
+    rtl = mipsdsp_##func(rtl, sa, env);                           \
+                                                                  \
+    return MIPSDSP_RETURN32_16(rth, rtl);                         \
+}
+
+SHIFT_PH(shll, lshift16);
+SHIFT_PH(shll_s, sat16_lshift);
+
+#undef SHIFT_PH
+
+#if defined(TARGET_MIPS64)
+#define SHIFT_QH(name, func) \
+target_ulong helper_##name##_qh(target_ulong rt, target_ulong sa) \
+{                                                                 \
+    uint16_t rt3, rt2, rt1, rt0;                                  \
+                                                                  \
+    sa = sa & 0x0F;                                               \
+                                                                  \
+    MIPSDSP_SPLIT64_16(rt, rt3, rt2, rt1, rt0);                   \
+                                                                  \
+    rt3 = mipsdsp_##func(rt3, sa);                                \
+    rt2 = mipsdsp_##func(rt2, sa);                                \
+    rt1 = mipsdsp_##func(rt1, sa);                                \
+    rt0 = mipsdsp_##func(rt0, sa);                                \
+                                                                  \
+    return MIPSDSP_RETURN64_16(rt3, rt2, rt1, rt0);               \
+}
+
+#define SHIFT_QH_ENV(name, func) \
+target_ulong helper_##name##_qh(target_ulong rt, target_ulong sa, \
+                                CPUMIPSState *env)                \
+{                                                                 \
+    uint16_t rt3, rt2, rt1, rt0;                                  \
+                                                                  \
+    sa = sa & 0x0F;                                               \
+                                                                  \
+    MIPSDSP_SPLIT64_16(rt, rt3, rt2, rt1, rt0);                   \
+                                                                  \
+    rt3 = mipsdsp_##func(rt3, sa, env);                           \
+    rt2 = mipsdsp_##func(rt2, sa, env);                           \
+    rt1 = mipsdsp_##func(rt1, sa, env);                           \
+    rt0 = mipsdsp_##func(rt0, sa, env);                           \
+                                                                  \
+    return MIPSDSP_RETURN64_16(rt3, rt2, rt1, rt0);               \
+}
+
+SHIFT_QH_ENV(shll, lshift16);
+SHIFT_QH_ENV(shll_s, sat16_lshift);
+
+SHIFT_QH(shrl, rshift_u16);
+SHIFT_QH(shra, rashift16);
+SHIFT_QH(shra_r, rnd16_rashift);
+
+#undef SHIFT_QH
+#undef SHIFT_QH_ENV
+
+#endif
+
+#define SHIFT_W(name, func) \
+target_ulong helper_##name##_w(target_ulong sa, target_ulong rt) \
+{                                                                       \
+    uint32_t temp;                                                      \
+                                                                        \
+    sa = sa & 0x1F;                                                     \
+    temp = mipsdsp_##func(rt, sa);                                      \
+                                                                        \
+    return (target_long)(int32_t)temp;                                  \
+}
+
+#define SHIFT_W_ENV(name, func) \
+target_ulong helper_##name##_w(target_ulong sa, target_ulong rt, \
+                               CPUMIPSState *env) \
+{                                                                       \
+    uint32_t temp;                                                      \
+                                                                        \
+    sa = sa & 0x1F;                                                     \
+    temp = mipsdsp_##func(rt, sa, env);                                 \
+                                                                        \
+    return (target_long)(int32_t)temp;                                  \
+}
+
+SHIFT_W_ENV(shll_s, sat32_lshift);
+SHIFT_W(shra_r, rnd32_rashift);
+
+#undef SHIFT_W
+#undef SHIFT_W_ENV
+
+#if defined(TARGET_MIPS64)
+#define SHIFT_PW(name, func) \
+target_ulong helper_##name##_pw(target_ulong rt, target_ulong sa) \
+{                                                                 \
+    uint32_t rt1, rt0;                                            \
+                                                                  \
+    sa = sa & 0x1F;                                               \
+    MIPSDSP_SPLIT64_32(rt, rt1, rt0);                             \
+                                                                  \
+    rt1 = mipsdsp_##func(rt1, sa);                                \
+    rt0 = mipsdsp_##func(rt0, sa);                                \
+                                                                  \
+    return MIPSDSP_RETURN64_32(rt1, rt0);                         \
+}
+
+#define SHIFT_PW_ENV(name, func) \
+target_ulong helper_##name##_pw(target_ulong rt, target_ulong sa, \
+                                CPUMIPSState *env)                \
+{                                                                 \
+    uint32_t rt1, rt0;                                            \
+                                                                  \
+    sa = sa & 0x1F;                                               \
+    MIPSDSP_SPLIT64_32(rt, rt1, rt0);                             \
+                                                                  \
+    rt1 = mipsdsp_##func(rt1, sa, env);                           \
+    rt0 = mipsdsp_##func(rt0, sa, env);                           \
+                                                                  \
+    return MIPSDSP_RETURN64_32(rt1, rt0);                         \
+}
+
+SHIFT_PW_ENV(shll, lshift32);
+SHIFT_PW_ENV(shll_s, sat32_lshift);
+
+SHIFT_PW(shra, rashift32);
+SHIFT_PW(shra_r, rnd32_rashift);
+
+#undef SHIFT_PW
+#undef SHIFT_PW_ENV
+
+#endif
+
+#define SHIFT_PH(name, func) \
+target_ulong helper_##name##_ph(target_ulong sa, target_ulong rt) \
+{                                                                    \
+    uint16_t rth, rtl;                                               \
+                                                                     \
+    sa = sa & 0x0F;                                                  \
+                                                                     \
+    MIPSDSP_SPLIT32_16(rt, rth, rtl);                                \
+                                                                     \
+    rth = mipsdsp_##func(rth, sa);                                   \
+    rtl = mipsdsp_##func(rtl, sa);                                   \
+                                                                     \
+    return MIPSDSP_RETURN32_16(rth, rtl);                            \
+}
+
+SHIFT_PH(shrl, rshift_u16);
+SHIFT_PH(shra, rashift16);
+SHIFT_PH(shra_r, rnd16_rashift);
+
+#undef SHIFT_PH
+
+/** DSP Multiply Sub-class insns **/
+/* Return value made up by two 16bits value.
+ * FIXME give the macro a better name.
+ */
+#define MUL_RETURN32_16_PH(name, func, \
+                           rsmov1, rsmov2, rsfilter, \
+                           rtmov1, rtmov2, rtfilter) \
+target_ulong helper_##name(target_ulong rs, target_ulong rt, \
+                           CPUMIPSState *env)                \
+{                                                            \
+    uint16_t rsB, rsA, rtB, rtA;                             \
+                                                             \
+    rsB = (rs >> rsmov1) & rsfilter;                         \
+    rsA = (rs >> rsmov2) & rsfilter;                         \
+    rtB = (rt >> rtmov1) & rtfilter;                         \
+    rtA = (rt >> rtmov2) & rtfilter;                         \
+                                                             \
+    rsB = mipsdsp_##func(rsB, rtB, env);                     \
+    rsA = mipsdsp_##func(rsA, rtA, env);                     \
+                                                             \
+    return MIPSDSP_RETURN32_16(rsB, rsA);                    \
+}
+
+MUL_RETURN32_16_PH(muleu_s_ph_qbl, mul_u8_u16, \
+                      24, 16, MIPSDSP_Q0, \
+                      16, 0, MIPSDSP_LO);
+MUL_RETURN32_16_PH(muleu_s_ph_qbr, mul_u8_u16, \
+                      8, 0, MIPSDSP_Q0, \
+                      16, 0, MIPSDSP_LO);
+MUL_RETURN32_16_PH(mulq_rs_ph, rndq15_mul_q15_q15, \
+                      16, 0, MIPSDSP_LO, \
+                      16, 0, MIPSDSP_LO);
+MUL_RETURN32_16_PH(mul_ph, mul_i16_i16, \
+                      16, 0, MIPSDSP_LO, \
+                      16, 0, MIPSDSP_LO);
+MUL_RETURN32_16_PH(mul_s_ph, sat16_mul_i16_i16, \
+                      16, 0, MIPSDSP_LO, \
+                      16, 0, MIPSDSP_LO);
+MUL_RETURN32_16_PH(mulq_s_ph, sat16_mul_q15_q15, \
+                      16, 0, MIPSDSP_LO, \
+                      16, 0, MIPSDSP_LO);
+
+#undef MUL_RETURN32_16_PH
+
+#define MUL_RETURN32_32_ph(name, func, movbits) \
+target_ulong helper_##name(target_ulong rs, target_ulong rt, \
+                                  CPUMIPSState *env)         \
+{                                                            \
+    int16_t rsh, rth;                                        \
+    int32_t temp;                                            \
+                                                             \
+    rsh = (rs >> movbits) & MIPSDSP_LO;                      \
+    rth = (rt >> movbits) & MIPSDSP_LO;                      \
+    temp = mipsdsp_##func(rsh, rth, env);                    \
+                                                             \
+    return (target_long)(int32_t)temp;                       \
+}
+
+MUL_RETURN32_32_ph(muleq_s_w_phl, mul_q15_q15_overflowflag21, 16);
+MUL_RETURN32_32_ph(muleq_s_w_phr, mul_q15_q15_overflowflag21, 0);
+
+#undef MUL_RETURN32_32_ph
+
+#define MUL_VOID_PH(name, use_ac_env) \
+void helper_##name(uint32_t ac, target_ulong rs, target_ulong rt,        \
+                          CPUMIPSState *env)                             \
+{                                                                        \
+    int16_t rsh, rsl, rth, rtl;                                          \
+    int32_t tempB, tempA;                                                \
+    int64_t acc, dotp;                                                   \
+                                                                         \
+    MIPSDSP_SPLIT32_16(rs, rsh, rsl);                                    \
+    MIPSDSP_SPLIT32_16(rt, rth, rtl);                                    \
+                                                                         \
+    if (use_ac_env == 1) {                                               \
+        tempB = mipsdsp_mul_q15_q15(ac, rsh, rth, env);                  \
+        tempA = mipsdsp_mul_q15_q15(ac, rsl, rtl, env);                  \
+    } else {                                                             \
+        tempB = mipsdsp_mul_u16_u16(rsh, rth);                           \
+        tempA = mipsdsp_mul_u16_u16(rsl, rtl);                           \
+    }                                                                    \
+                                                                         \
+    dotp = (int64_t)tempB - (int64_t)tempA;                              \
+    acc = ((uint64_t)env->active_tc.HI[ac] << 32) |                      \
+          ((uint64_t)env->active_tc.LO[ac] & MIPSDSP_LLO);               \
+    dotp = dotp + acc;                                                   \
+    env->active_tc.HI[ac] = (target_long)(int32_t)                       \
+                            ((dotp & MIPSDSP_LHI) >> 32);                \
+    env->active_tc.LO[ac] = (target_long)(int32_t)(dotp & MIPSDSP_LLO);  \
+}
+
+MUL_VOID_PH(mulsaq_s_w_ph, 1);
+MUL_VOID_PH(mulsa_w_ph, 0);
+
+#undef MUL_VOID_PH
+
+#if defined(TARGET_MIPS64)
+#define MUL_RETURN64_16_QH(name, func, \
+                           rsmov1, rsmov2, rsmov3, rsmov4, rsfilter, \
+                           rtmov1, rtmov2, rtmov3, rtmov4, rtfilter) \
+target_ulong helper_##name(target_ulong rs, target_ulong rt,         \
+                           CPUMIPSState *env)                        \
+{                                                                    \
+    uint16_t rs3, rs2, rs1, rs0;                                     \
+    uint16_t rt3, rt2, rt1, rt0;                                     \
+    uint16_t tempD, tempC, tempB, tempA;                             \
+                                                                     \
+    rs3 = (rs >> rsmov1) & rsfilter;                                 \
+    rs2 = (rs >> rsmov2) & rsfilter;                                 \
+    rs1 = (rs >> rsmov3) & rsfilter;                                 \
+    rs0 = (rs >> rsmov4) & rsfilter;                                 \
+    rt3 = (rt >> rtmov1) & rtfilter;                                 \
+    rt2 = (rt >> rtmov2) & rtfilter;                                 \
+    rt1 = (rt >> rtmov3) & rtfilter;                                 \
+    rt0 = (rt >> rtmov4) & rtfilter;                                 \
+                                                                     \
+    tempD = mipsdsp_##func(rs3, rt3, env);                           \
+    tempC = mipsdsp_##func(rs2, rt2, env);                           \
+    tempB = mipsdsp_##func(rs1, rt1, env);                           \
+    tempA = mipsdsp_##func(rs0, rt0, env);                           \
+                                                                     \
+    return MIPSDSP_RETURN64_16(tempD, tempC, tempB, tempA);          \
+}
+
+MUL_RETURN64_16_QH(muleu_s_qh_obl, mul_u8_u16, \
+                   56, 48, 40, 32, MIPSDSP_Q0, \
+                   48, 32, 16, 0, MIPSDSP_LO);
+MUL_RETURN64_16_QH(muleu_s_qh_obr, mul_u8_u16, \
+                   24, 16, 8, 0, MIPSDSP_Q0, \
+                   48, 32, 16, 0, MIPSDSP_LO);
+MUL_RETURN64_16_QH(mulq_rs_qh, rndq15_mul_q15_q15, \
+                   48, 32, 16, 0, MIPSDSP_LO, \
+                   48, 32, 16, 0, MIPSDSP_LO);
+
+#undef MUL_RETURN64_16_QH
+
+#define MUL_RETURN64_32_QH(name, \
+                           rsmov1, rsmov2, \
+                           rtmov1, rtmov2) \
+target_ulong helper_##name(target_ulong rs, target_ulong rt, \
+                           CPUMIPSState *env)                \
+{                                                            \
+    uint16_t rsB, rsA;                                       \
+    uint16_t rtB, rtA;                                       \
+    uint32_t tempB, tempA;                                   \
+                                                             \
+    rsB = (rs >> rsmov1) & MIPSDSP_LO;                       \
+    rsA = (rs >> rsmov2) & MIPSDSP_LO;                       \
+    rtB = (rt >> rtmov1) & MIPSDSP_LO;                       \
+    rtA = (rt >> rtmov2) & MIPSDSP_LO;                       \
+                                                             \
+    tempB = mipsdsp_mul_q15_q15(5, rsB, rtB, env);           \
+    tempA = mipsdsp_mul_q15_q15(5, rsA, rtA, env);           \
+                                                             \
+    return ((uint64_t)tempB << 32) | (uint64_t)tempA;        \
+}
+
+MUL_RETURN64_32_QH(muleq_s_pw_qhl, 48, 32, 48, 32);
+MUL_RETURN64_32_QH(muleq_s_pw_qhr, 16, 0, 16, 0);
+
+#undef MUL_RETURN64_32_QH
+
+void helper_mulsaq_s_w_qh(target_ulong rs, target_ulong rt, uint32_t ac,
+                          CPUMIPSState *env)
+{
+    int16_t rs3, rs2, rs1, rs0;
+    int16_t rt3, rt2, rt1, rt0;
+    int32_t tempD, tempC, tempB, tempA;
+    int64_t acc[2];
+    int64_t temp[2];
+    int64_t temp_sum;
+
+    MIPSDSP_SPLIT64_16(rs, rs3, rs2, rs1, rs0);
+    MIPSDSP_SPLIT64_16(rt, rt3, rt2, rt1, rt0);
+
+    tempD = mipsdsp_mul_q15_q15(ac, rs3, rt3, env);
+    tempC = mipsdsp_mul_q15_q15(ac, rs2, rt2, env);
+    tempB = mipsdsp_mul_q15_q15(ac, rs1, rt1, env);
+    tempA = mipsdsp_mul_q15_q15(ac, rs0, rt0, env);
+
+    temp[0] = ((int32_t)tempD - (int32_t)tempC) +
+              ((int32_t)tempB - (int32_t)tempA);
+    temp[0] = (int64_t)(temp[0] << 30) >> 30;
+    if (((temp[0] >> 33) & 0x01) == 0) {
+        temp[1] = 0x00;
+    } else {
+        temp[1] = ~0ull;
+    }
+
+    acc[0] = env->active_tc.LO[ac];
+    acc[1] = env->active_tc.HI[ac];
+
+    temp_sum = acc[0] + temp[0];
+    if (((uint64_t)temp_sum < (uint64_t)acc[0]) &&
+       ((uint64_t)temp_sum < (uint64_t)temp[0])) {
+        acc[1] += 1;
+    }
+    acc[0] = temp_sum;
+    acc[1] += temp[1];
+
+    env->active_tc.HI[ac] = acc[1];
+    env->active_tc.LO[ac] = acc[0];
+}
+#endif
+
+#define DP_QB(name, func, is_add, rsmov1, rsmov2, rtmov1, rtmov2) \
+void helper_##name(uint32_t ac, target_ulong rs, target_ulong rt,        \
+                   CPUMIPSState *env)                                    \
+{                                                                        \
+    uint8_t rs3, rs2;                                                    \
+    uint8_t rt3, rt2;                                                    \
+    uint16_t tempB, tempA;                                               \
+    uint64_t tempC, dotp;                                                \
+                                                                         \
+    rs3 = (rs >> rsmov1) & MIPSDSP_Q0;                                   \
+    rs2 = (rs >> rsmov2) & MIPSDSP_Q0;                                   \
+    rt3 = (rt >> rtmov1) & MIPSDSP_Q0;                                   \
+    rt2 = (rt >> rtmov2) & MIPSDSP_Q0;                                   \
+    tempB = mipsdsp_##func(rs3, rt3);                                    \
+    tempA = mipsdsp_##func(rs2, rt2);                                    \
+    dotp = (int64_t)tempB + (int64_t)tempA;                              \
+    if (is_add) {                                                        \
+        tempC = (((uint64_t)env->active_tc.HI[ac] << 32) |               \
+                 ((uint64_t)env->active_tc.LO[ac] & MIPSDSP_LLO))        \
+            + dotp;                                                      \
+    } else {                                                             \
+        tempC = (((uint64_t)env->active_tc.HI[ac] << 32) |               \
+                 ((uint64_t)env->active_tc.LO[ac] & MIPSDSP_LLO))        \
+            - dotp;                                                      \
+    }                                                                    \
+                                                                         \
+    env->active_tc.HI[ac] = (target_long)(int32_t)                       \
+                            ((tempC & MIPSDSP_LHI) >> 32);               \
+    env->active_tc.LO[ac] = (target_long)(int32_t)(tempC & MIPSDSP_LLO); \
+}
+
+DP_QB(dpau_h_qbl, mul_u8_u8, 1, 24, 16, 24, 16);
+DP_QB(dpau_h_qbr, mul_u8_u8, 1, 8, 0, 8, 0);
+DP_QB(dpsu_h_qbl, mul_u8_u8, 0, 24, 16, 24, 16);
+DP_QB(dpsu_h_qbr, mul_u8_u8, 0, 8, 0, 8, 0);
+
+#undef DP_QB
+
+#if defined(TARGET_MIPS64)
+#define DP_OB(name, add_sub, \
+              rsmov1, rsmov2, rsmov3, rsmov4, \
+              rtmov1, rtmov2, rtmov3, rtmov4) \
+void helper_##name(target_ulong rs, target_ulong rt, uint32_t ac,       \
+                       CPUMIPSState *env)                               \
+{                                                                       \
+    uint8_t rsD, rsC, rsB, rsA;                                         \
+    uint8_t rtD, rtC, rtB, rtA;                                         \
+    uint16_t tempD, tempC, tempB, tempA;                                \
+    uint64_t temp[2];                                                   \
+    uint64_t acc[2];                                                    \
+    uint64_t temp_sum;                                                  \
+                                                                        \
+    temp[0] = 0;                                                        \
+    temp[1] = 0;                                                        \
+                                                                        \
+    rsD = (rs >> rsmov1) & MIPSDSP_Q0;                                  \
+    rsC = (rs >> rsmov2) & MIPSDSP_Q0;                                  \
+    rsB = (rs >> rsmov3) & MIPSDSP_Q0;                                  \
+    rsA = (rs >> rsmov4) & MIPSDSP_Q0;                                  \
+    rtD = (rt >> rtmov1) & MIPSDSP_Q0;                                  \
+    rtC = (rt >> rtmov2) & MIPSDSP_Q0;                                  \
+    rtB = (rt >> rtmov3) & MIPSDSP_Q0;                                  \
+    rtA = (rt >> rtmov4) & MIPSDSP_Q0;                                  \
+                                                                        \
+    tempD = mipsdsp_mul_u8_u8(rsD, rtD);                                \
+    tempC = mipsdsp_mul_u8_u8(rsC, rtC);                                \
+    tempB = mipsdsp_mul_u8_u8(rsB, rtB);                                \
+    tempA = mipsdsp_mul_u8_u8(rsA, rtA);                                \
+                                                                        \
+    temp[0] = (uint64_t)tempD + (uint64_t)tempC +                       \
+      (uint64_t)tempB + (uint64_t)tempA;                                \
+                                                                        \
+    acc[0] = env->active_tc.LO[ac];                                     \
+    acc[1] = env->active_tc.HI[ac];                                     \
+                                                                        \
+    if (add_sub) {                                                      \
+        temp_sum = acc[0] + temp[0];                                    \
+        if (((uint64_t)temp_sum < (uint64_t)acc[0]) &&                  \
+            ((uint64_t)temp_sum < (uint64_t)temp[0])) {                 \
+            acc[1] += 1;                                                \
+        }                                                               \
+        temp[0] = temp_sum;                                             \
+        temp[1] = acc[1] + temp[1];                                     \
+    } else {                                                            \
+        temp_sum = acc[0] - temp[0];                                    \
+        if ((uint64_t)temp_sum > (uint64_t)acc[0]) {                    \
+            acc[1] -= 1;                                                \
+        }                                                               \
+        temp[0] = temp_sum;                                             \
+        temp[1] = acc[1] - temp[1];                                     \
+    }                                                                   \
+                                                                        \
+    env->active_tc.HI[ac] = temp[1];                                    \
+    env->active_tc.LO[ac] = temp[0];                                    \
+}
+
+DP_OB(dpau_h_obl, 1, 56, 48, 40, 32, 56, 48, 40, 32);
+DP_OB(dpau_h_obr, 1, 24, 16, 8, 0, 24, 16, 8, 0);
+DP_OB(dpsu_h_obl, 0, 56, 48, 40, 32, 56, 48, 40, 32);
+DP_OB(dpsu_h_obr, 0, 24, 16, 8, 0, 24, 16, 8, 0);
+
+#undef DP_OB
+#endif
+
+#define DP_NOFUNC_PH(name, is_add, rsmov1, rsmov2, rtmov1, rtmov2)             \
+void helper_##name(uint32_t ac, target_ulong rs, target_ulong rt,              \
+                   CPUMIPSState *env)                                          \
+{                                                                              \
+    uint16_t rsB, rsA, rtB, rtA;                                               \
+    int32_t  tempA, tempB;                                                     \
+    int64_t  acc;                                                              \
+                                                                               \
+    rsB = (rs >> rsmov1) & MIPSDSP_LO;                                         \
+    rsA = (rs >> rsmov2) & MIPSDSP_LO;                                         \
+    rtB = (rt >> rtmov1) & MIPSDSP_LO;                                         \
+    rtA = (rt >> rtmov2) & MIPSDSP_LO;                                         \
+                                                                               \
+    tempB = (int32_t)rsB * (int32_t)rtB;                                       \
+    tempA = (int32_t)rsA * (int32_t)rtA;                                       \
+                                                                               \
+    acc = ((uint64_t)env->active_tc.HI[ac] << 32) |                            \
+          ((uint64_t)env->active_tc.LO[ac] & MIPSDSP_LLO);                     \
+                                                                               \
+    if (is_add) {                                                              \
+        acc = acc + ((int64_t)tempB + (int64_t)tempA);                         \
+    } else {                                                                   \
+        acc = acc - ((int64_t)tempB + (int64_t)tempA);                         \
+    }                                                                          \
+                                                                               \
+    env->active_tc.HI[ac] = (target_long)(int32_t)((acc & MIPSDSP_LHI) >> 32); \
+    env->active_tc.LO[ac] = (target_long)(int32_t)(acc & MIPSDSP_LLO);         \
+}
+
+DP_NOFUNC_PH(dpa_w_ph, 1, 16, 0, 16, 0);
+DP_NOFUNC_PH(dpax_w_ph, 1, 16, 0, 0, 16);
+DP_NOFUNC_PH(dps_w_ph, 0, 16, 0, 16, 0);
+DP_NOFUNC_PH(dpsx_w_ph, 0, 16, 0, 0, 16);
+#undef DP_NOFUNC_PH
+
+#define DP_HASFUNC_PH(name, is_add, rsmov1, rsmov2, rtmov1, rtmov2) \
+void helper_##name(uint32_t ac, target_ulong rs, target_ulong rt,   \
+                   CPUMIPSState *env)                      \
+{                                                          \
+    int16_t rsB, rsA, rtB, rtA;                            \
+    int32_t tempB, tempA;                                  \
+    int64_t acc, dotp;                                     \
+                                                           \
+    rsB = (rs >> rsmov1) & MIPSDSP_LO;                     \
+    rsA = (rs >> rsmov2) & MIPSDSP_LO;                     \
+    rtB = (rt >> rtmov1) & MIPSDSP_LO;                     \
+    rtA = (rt >> rtmov2) & MIPSDSP_LO;                     \
+                                                           \
+    tempB = mipsdsp_mul_q15_q15(ac, rsB, rtB, env);        \
+    tempA = mipsdsp_mul_q15_q15(ac, rsA, rtA, env);        \
+                                                           \
+    dotp = (int64_t)tempB + (int64_t)tempA;                \
+    acc = ((uint64_t)env->active_tc.HI[ac] << 32) |        \
+          ((uint64_t)env->active_tc.LO[ac] & MIPSDSP_LLO); \
+                                                           \
+    if (is_add) {                                          \
+        acc = acc + dotp;                                  \
+    } else {                                               \
+        acc = acc - dotp;                                  \
+    }                                                      \
+                                                           \
+    env->active_tc.HI[ac] = (target_long)(int32_t)         \
+        ((acc & MIPSDSP_LHI) >> 32);                       \
+    env->active_tc.LO[ac] = (target_long)(int32_t)         \
+        (acc & MIPSDSP_LLO);                               \
+}
+
+DP_HASFUNC_PH(dpaq_s_w_ph, 1, 16, 0, 16, 0);
+DP_HASFUNC_PH(dpaqx_s_w_ph, 1, 16, 0, 0, 16);
+DP_HASFUNC_PH(dpsq_s_w_ph, 0, 16, 0, 16, 0);
+DP_HASFUNC_PH(dpsqx_s_w_ph, 0, 16, 0, 0, 16);
+
+#undef DP_HASFUNC_PH
+
+#define DP_128OPERATION_PH(name, is_add) \
+void helper_##name(uint32_t ac, target_ulong rs, target_ulong rt, \
+                          CPUMIPSState *env)                             \
+{                                                                        \
+    int16_t rsh, rsl, rth, rtl;                                          \
+    int32_t tempB, tempA, tempC62_31, tempC63;                           \
+    int64_t acc, dotp, tempC;                                            \
+                                                                         \
+    MIPSDSP_SPLIT32_16(rs, rsh, rsl);                                    \
+    MIPSDSP_SPLIT32_16(rt, rth, rtl);                                    \
+                                                                         \
+    tempB = mipsdsp_mul_q15_q15(ac, rsh, rtl, env);                      \
+    tempA = mipsdsp_mul_q15_q15(ac, rsl, rth, env);                      \
+                                                                         \
+    dotp = (int64_t)tempB + (int64_t)tempA;                              \
+    acc = ((uint64_t)env->active_tc.HI[ac] << 32) |                      \
+          ((uint64_t)env->active_tc.LO[ac] & MIPSDSP_LLO);               \
+    if (is_add) {                                                        \
+        tempC = acc + dotp;                                              \
+    } else {                                                             \
+        tempC = acc - dotp;                                              \
+    }                                                                    \
+    tempC63 = (tempC >> 63) & 0x01;                                      \
+    tempC62_31 = (tempC >> 31) & 0xFFFFFFFF;                             \
+                                                                         \
+    if ((tempC63 == 0) && (tempC62_31 != 0x00000000)) {                  \
+        tempC = 0x7FFFFFFF;                                              \
+        set_DSPControl_overflow_flag(1, 16 + ac, env);                   \
+    }                                                                    \
+                                                                         \
+    if ((tempC63 == 1) && (tempC62_31 != 0xFFFFFFFF)) {                  \
+        tempC = (int64_t)(int32_t)0x80000000;                            \
+        set_DSPControl_overflow_flag(1, 16 + ac, env);                   \
+    }                                                                    \
+                                                                         \
+    env->active_tc.HI[ac] = (target_long)(int32_t)                       \
+        ((tempC & MIPSDSP_LHI) >> 32);                                   \
+    env->active_tc.LO[ac] = (target_long)(int32_t)                       \
+        (tempC & MIPSDSP_LLO);                                           \
+}
+
+DP_128OPERATION_PH(dpaqx_sa_w_ph, 1);
+DP_128OPERATION_PH(dpsqx_sa_w_ph, 0);
+
+#undef DP_128OPERATION_HP
+
+#if defined(TARGET_MIPS64)
+#define DP_QH(name, is_add, use_ac_env) \
+void helper_##name(target_ulong rs, target_ulong rt, uint32_t ac,    \
+                   CPUMIPSState *env)                                \
+{                                                                    \
+    int32_t rs3, rs2, rs1, rs0;                                      \
+    int32_t rt3, rt2, rt1, rt0;                                      \
+    int32_t tempD, tempC, tempB, tempA;                              \
+    int64_t acc[2];                                                  \
+    int64_t temp[2];                                                 \
+    int64_t temp_sum;                                                \
+                                                                     \
+    MIPSDSP_SPLIT64_16(rs, rs3, rs2, rs1, rs0);                      \
+    MIPSDSP_SPLIT64_16(rt, rt3, rt2, rt1, rt0);                      \
+                                                                     \
+    if (use_ac_env) {                                                \
+        tempD = mipsdsp_mul_q15_q15(ac, rs3, rt3, env);              \
+        tempC = mipsdsp_mul_q15_q15(ac, rs2, rt2, env);              \
+        tempB = mipsdsp_mul_q15_q15(ac, rs1, rt1, env);              \
+        tempA = mipsdsp_mul_q15_q15(ac, rs0, rt0, env);              \
+    } else {                                                         \
+        tempD = mipsdsp_mul_u16_u16(rs3, rt3);                       \
+        tempC = mipsdsp_mul_u16_u16(rs2, rt2);                       \
+        tempB = mipsdsp_mul_u16_u16(rs1, rt1);                       \
+        tempA = mipsdsp_mul_u16_u16(rs0, rt0);                       \
+    }                                                                \
+                                                                     \
+    temp[0] = (int64_t)tempD + (int64_t)tempC +                      \
+              (int64_t)tempB + (int64_t)tempA;                       \
+                                                                     \
+    if (temp[0] >= 0) {                                              \
+        temp[1] = 0;                                                 \
+    } else {                                                         \
+        temp[1] = ~0ull;                                             \
+    }                                                                \
+                                                                     \
+    acc[1] = env->active_tc.HI[ac];                                  \
+    acc[0] = env->active_tc.LO[ac];                                  \
+                                                                     \
+    if (is_add) {                                                    \
+        temp_sum = acc[0] + temp[0];                                 \
+        if (((uint64_t)temp_sum < (uint64_t)acc[0]) &&               \
+            ((uint64_t)temp_sum < (uint64_t)temp[0])) {              \
+            acc[1] = acc[1] + 1;                                     \
+        }                                                            \
+        temp[0] = temp_sum;                                          \
+        temp[1] = acc[1] + temp[1];                                  \
+    } else {                                                         \
+        temp_sum = acc[0] - temp[0];                                 \
+        if ((uint64_t)temp_sum > (uint64_t)acc[0]) {                 \
+            acc[1] = acc[1] - 1;                                     \
+        }                                                            \
+        temp[0] = temp_sum;                                          \
+        temp[1] = acc[1] - temp[1];                                  \
+    }                                                                \
+                                                                     \
+    env->active_tc.HI[ac] = temp[1];                                 \
+    env->active_tc.LO[ac] = temp[0];                                 \
+}
+
+DP_QH(dpa_w_qh, 1, 0);
+DP_QH(dpaq_s_w_qh, 1, 1);
+DP_QH(dps_w_qh, 0, 0);
+DP_QH(dpsq_s_w_qh, 0, 1);
+
+#undef DP_QH
+
+#endif
+
+#define DP_L_W(name, is_add) \
+void helper_##name(uint32_t ac, target_ulong rs, target_ulong rt,     \
+                   CPUMIPSState *env)                                 \
+{                                                                     \
+    int32_t temp63;                                                   \
+    int64_t dotp, acc;                                                \
+    uint64_t temp;                                                    \
+                                                                      \
+    dotp = mipsdsp_mul_q31_q31(ac, rs, rt, env);                      \
+    acc = ((uint64_t)env->active_tc.HI[ac] << 32) |                   \
+          ((uint64_t)env->active_tc.LO[ac] & MIPSDSP_LLO);            \
+    if (!is_add) {                                                    \
+        dotp = -dotp;                                                 \
+    }                                                                 \
+                                                                      \
+    temp = acc + dotp;                                                \
+    if (MIPSDSP_OVERFLOW((uint64_t)acc, (uint64_t)dotp, temp,         \
+                         (0x01ull << 63))) {                          \
+        temp63 = (temp >> 63) & 0x01;                                 \
+        if (temp63 == 1) {                                            \
+            temp = (0x01ull << 63) - 1;                               \
+        } else {                                                      \
+            temp = 0x01ull << 63;                                     \
+        }                                                             \
+                                                                      \
+        set_DSPControl_overflow_flag(1, 16 + ac, env);                \
+    }                                                                 \
+                                                                      \
+    env->active_tc.HI[ac] = (target_long)(int32_t)                    \
+        ((temp & MIPSDSP_LHI) >> 32);                                 \
+    env->active_tc.LO[ac] = (target_long)(int32_t)                    \
+        (temp & MIPSDSP_LLO);                                         \
+}
+
+DP_L_W(dpaq_sa_l_w, 1);
+DP_L_W(dpsq_sa_l_w, 0);
+
+#undef DP_L_W
+
+#if defined(TARGET_MIPS64)
+#define DP_L_PW(name, func) \
+void helper_##name(target_ulong rs, target_ulong rt, uint32_t ac, \
+                   CPUMIPSState *env)                             \
+{                                                                 \
+    int32_t rs1, rs0;                                             \
+    int32_t rt1, rt0;                                             \
+    int64_t tempB[2], tempA[2];                                   \
+    int64_t temp[2];                                              \
+    int64_t acc[2];                                               \
+    int64_t temp_sum;                                             \
+                                                                  \
+    temp[0] = 0;                                                  \
+    temp[1] = 0;                                                  \
+                                                                  \
+    MIPSDSP_SPLIT64_32(rs, rs1, rs0);                             \
+    MIPSDSP_SPLIT64_32(rt, rt1, rt0);                             \
+                                                                  \
+    tempB[0] = mipsdsp_mul_q31_q31(ac, rs1, rt1, env);            \
+    tempA[0] = mipsdsp_mul_q31_q31(ac, rs0, rt0, env);            \
+                                                                  \
+    if (tempB[0] >= 0) {                                          \
+        tempB[1] = 0x00;                                          \
+    } else {                                                      \
+        tempB[1] = ~0ull;                                         \
+    }                                                             \
+                                                                  \
+    if (tempA[0] >= 0) {                                          \
+        tempA[1] = 0x00;                                          \
+    } else {                                                      \
+        tempA[1] = ~0ull;                                         \
+    }                                                             \
+                                                                  \
+    temp_sum = tempB[0] + tempA[0];                               \
+    if (((uint64_t)temp_sum < (uint64_t)tempB[0]) &&              \
+        ((uint64_t)temp_sum < (uint64_t)tempA[0])) {              \
+        temp[1] += 1;                                             \
+    }                                                             \
+    temp[0] = temp_sum;                                           \
+    temp[1] += tempB[1] + tempA[1];                               \
+                                                                  \
+    mipsdsp_##func(acc, ac, temp, env);                           \
+                                                                  \
+    env->active_tc.HI[ac] = acc[1];                               \
+    env->active_tc.LO[ac] = acc[0];                               \
+}
+
+DP_L_PW(dpaq_sa_l_pw, sat64_acc_add_q63);
+DP_L_PW(dpsq_sa_l_pw, sat64_acc_sub_q63);
+
+#undef DP_L_PW
+
+void helper_mulsaq_s_l_pw(target_ulong rs, target_ulong rt, uint32_t ac,
+                          CPUMIPSState *env)
+{
+    int32_t rs1, rs0;
+    int32_t rt1, rt0;
+    int64_t tempB[2], tempA[2];
+    int64_t temp[2];
+    int64_t acc[2];
+    int64_t temp_sum;
+
+    rs1 = (rs >> 32) & MIPSDSP_LLO;
+    rs0 = rs & MIPSDSP_LLO;
+    rt1 = (rt >> 32) & MIPSDSP_LLO;
+    rt0 = rt & MIPSDSP_LLO;
+
+    tempB[0] = mipsdsp_mul_q31_q31(ac, rs1, rt1, env);
+    tempA[0] = mipsdsp_mul_q31_q31(ac, rs0, rt0, env);
+
+    if (tempB[0] >= 0) {
+        tempB[1] = 0x00;
+    } else {
+        tempB[1] = ~0ull;
+    }
+
+    if (tempA[0] >= 0) {
+        tempA[1] = 0x00;
+    } else {
+        tempA[1] = ~0ull;
+    }
+
+    acc[0] = env->active_tc.LO[ac];
+    acc[1] = env->active_tc.HI[ac];
+
+    temp_sum = tempB[0] - tempA[0];
+    if ((uint64_t)temp_sum > (uint64_t)tempB[0]) {
+        tempB[1] -= 1;
+    }
+    temp[0] = temp_sum;
+    temp[1] = tempB[1] - tempA[1];
+
+    if ((temp[1] & 0x01) == 0) {
+        temp[1] = 0x00;
+    } else {
+        temp[1] = ~0ull;
+    }
+
+    temp_sum = acc[0] + temp[0];
+    if (((uint64_t)temp_sum < (uint64_t)acc[0]) &&
+       ((uint64_t)temp_sum < (uint64_t)temp[0])) {
+        acc[1] += 1;
+    }
+    acc[0] = temp_sum;
+    acc[1] += temp[1];
+
+    env->active_tc.HI[ac] = acc[1];
+    env->active_tc.LO[ac] = acc[0];
+}
+#endif
+
+#define MAQ_S_W(name, mov) \
+void helper_##name(uint32_t ac, target_ulong rs, target_ulong rt, \
+                   CPUMIPSState *env)                             \
+{                                                                 \
+    int16_t rsh, rth;                                             \
+    int32_t tempA;                                                \
+    int64_t tempL, acc;                                           \
+                                                                  \
+    rsh = (rs >> mov) & MIPSDSP_LO;                               \
+    rth = (rt >> mov) & MIPSDSP_LO;                               \
+    tempA  = mipsdsp_mul_q15_q15(ac, rsh, rth, env);              \
+    acc = ((uint64_t)env->active_tc.HI[ac] << 32) |               \
+          ((uint64_t)env->active_tc.LO[ac] & MIPSDSP_LLO);        \
+    tempL  = (int64_t)tempA + acc;                                \
+    env->active_tc.HI[ac] = (target_long)(int32_t)                \
+        ((tempL & MIPSDSP_LHI) >> 32);                            \
+    env->active_tc.LO[ac] = (target_long)(int32_t)                \
+        (tempL & MIPSDSP_LLO);                                    \
+}
+
+MAQ_S_W(maq_s_w_phl, 16);
+MAQ_S_W(maq_s_w_phr, 0);
+
+#undef MAQ_S_W
+
+#define MAQ_SA_W(name, mov) \
+void helper_##name(uint32_t ac, target_ulong rs, target_ulong rt,        \
+                   CPUMIPSState *env)                                    \
+{                                                                        \
+    int16_t rsh, rth;                                                    \
+    int32_t tempA;                                                       \
+                                                                         \
+    rsh = (rs >> mov) & MIPSDSP_LO;                                      \
+    rth = (rt >> mov) & MIPSDSP_LO;                                      \
+    tempA = mipsdsp_mul_q15_q15(ac, rsh, rth, env);                      \
+    tempA = mipsdsp_sat32_acc_q31(ac, tempA, env);                       \
+                                                                         \
+    env->active_tc.HI[ac] = (target_long)(int32_t)(((int64_t)tempA &     \
+                                                    MIPSDSP_LHI) >> 32); \
+    env->active_tc.LO[ac] = (target_long)(int32_t)((int64_t)tempA &      \
+                                                   MIPSDSP_LLO);         \
+}
+
+MAQ_SA_W(maq_sa_w_phl, 16);
+MAQ_SA_W(maq_sa_w_phr, 0);
+
+#undef MAQ_SA_W
+
+#define MULQ_W(name, addvar) \
+target_ulong helper_##name(target_ulong rs, target_ulong rt,   \
+                           CPUMIPSState *env)                  \
+{                                                              \
+    uint32_t rs_t, rt_t;                                       \
+    int32_t tempI;                                             \
+    int64_t tempL;                                             \
+                                                               \
+    rs_t = rs & MIPSDSP_LLO;                                   \
+    rt_t = rt & MIPSDSP_LLO;                                   \
+                                                               \
+    if ((rs_t == 0x80000000) && (rt_t == 0x80000000)) {        \
+        tempL = 0x7FFFFFFF00000000ull;                         \
+        set_DSPControl_overflow_flag(1, 21, env);              \
+    } else {                                                   \
+        tempL  = ((int64_t)rs_t * (int64_t)rt_t) << 1;         \
+        tempL += addvar;                                       \
+    }                                                          \
+    tempI = (tempL & MIPSDSP_LHI) >> 32;                       \
+                                                               \
+    return (target_long)(int32_t)tempI;                        \
+}
+
+MULQ_W(mulq_s_w, 0);
+MULQ_W(mulq_rs_w, 0x80000000ull);
+
+#undef MULQ_W
+
+#if defined(TARGET_MIPS64)
+
+#define MAQ_S_W_QH(name, mov) \
+void helper_##name(target_ulong rs, target_ulong rt, uint32_t ac, \
+                   CPUMIPSState *env)                             \
+{                                                                 \
+    int16_t rs_t, rt_t;                                           \
+    int32_t temp_mul;                                             \
+    int64_t temp[2];                                              \
+    int64_t acc[2];                                               \
+    int64_t temp_sum;                                             \
+                                                                  \
+    temp[0] = 0;                                                  \
+    temp[1] = 0;                                                  \
+                                                                  \
+    rs_t = (rs >> mov) & MIPSDSP_LO;                              \
+    rt_t = (rt >> mov) & MIPSDSP_LO;                              \
+    temp_mul = mipsdsp_mul_q15_q15(ac, rs_t, rt_t, env);          \
+                                                                  \
+    temp[0] = (int64_t)temp_mul;                                  \
+    if (temp[0] >= 0) {                                           \
+        temp[1] = 0x00;                                           \
+    } else {                                                      \
+        temp[1] = ~0ull;                                          \
+    }                                                             \
+                                                                  \
+    acc[0] = env->active_tc.LO[ac];                               \
+    acc[1] = env->active_tc.HI[ac];                               \
+                                                                  \
+    temp_sum = acc[0] + temp[0];                                  \
+    if (((uint64_t)temp_sum < (uint64_t)acc[0]) &&                \
+        ((uint64_t)temp_sum < (uint64_t)temp[0])) {               \
+        acc[1] += 1;                                              \
+    }                                                             \
+    acc[0] = temp_sum;                                            \
+    acc[1] += temp[1];                                            \
+                                                                  \
+    env->active_tc.HI[ac] = acc[1];                               \
+    env->active_tc.LO[ac] = acc[0];                               \
+}
+
+MAQ_S_W_QH(maq_s_w_qhll, 48);
+MAQ_S_W_QH(maq_s_w_qhlr, 32);
+MAQ_S_W_QH(maq_s_w_qhrl, 16);
+MAQ_S_W_QH(maq_s_w_qhrr, 0);
+
+#undef MAQ_S_W_QH
+
+#define MAQ_SA_W(name, mov) \
+void helper_##name(target_ulong rs, target_ulong rt, uint32_t ac, \
+                   CPUMIPSState *env)                             \
+{                                                                 \
+    int16_t rs_t, rt_t;                                           \
+    int32_t temp;                                                 \
+    int64_t acc[2];                                               \
+                                                                  \
+    rs_t = (rs >> mov) & MIPSDSP_LO;                              \
+    rt_t = (rt >> mov) & MIPSDSP_LO;                              \
+    temp = mipsdsp_mul_q15_q15(ac, rs_t, rt_t, env);              \
+    temp = mipsdsp_sat32_acc_q31(ac, temp, env);                  \
+                                                                  \
+    acc[0] = (int64_t)(int32_t)temp;                              \
+    if (acc[0] >= 0) {                                            \
+        acc[1] = 0x00;                                            \
+    } else {                                                      \
+        acc[1] = ~0ull;                                           \
+    }                                                             \
+                                                                  \
+    env->active_tc.HI[ac] = acc[1];                               \
+    env->active_tc.LO[ac] = acc[0];                               \
+}
+
+MAQ_SA_W(maq_sa_w_qhll, 48);
+MAQ_SA_W(maq_sa_w_qhlr, 32);
+MAQ_SA_W(maq_sa_w_qhrl, 16);
+MAQ_SA_W(maq_sa_w_qhrr, 0);
+
+#undef MAQ_SA_W
+
+#define MAQ_S_L_PW(name, mov) \
+void helper_##name(target_ulong rs, target_ulong rt, uint32_t ac, \
+                   CPUMIPSState *env)                             \
+{                                                                 \
+    int32_t rs_t, rt_t;                                           \
+    int64_t temp[2];                                              \
+    int64_t acc[2];                                               \
+    int64_t temp_sum;                                             \
+                                                                  \
+    temp[0] = 0;                                                  \
+    temp[1] = 0;                                                  \
+                                                                  \
+    rs_t = (rs >> mov) & MIPSDSP_LLO;                             \
+    rt_t = (rt >> mov) & MIPSDSP_LLO;                             \
+                                                                  \
+    temp[0] = mipsdsp_mul_q31_q31(ac, rs_t, rt_t, env);           \
+    if (temp[0] >= 0) {                                           \
+        temp[1] = 0x00;                                           \
+    } else {                                                      \
+        temp[1] = ~0ull;                                          \
+    }                                                             \
+                                                                  \
+    acc[0] = env->active_tc.LO[ac];                               \
+    acc[1] = env->active_tc.HI[ac];                               \
+                                                                  \
+    temp_sum = acc[0] + temp[0];                                  \
+    if (((uint64_t)temp_sum < (uint64_t)acc[0]) &&                \
+        ((uint64_t)temp_sum < (uint64_t)temp[0])) {               \
+        acc[1] += 1;                                              \
+    }                                                             \
+    acc[0] = temp_sum;                                            \
+    acc[1] += temp[1];                                            \
+                                                                  \
+    env->active_tc.HI[ac] = acc[1];                               \
+    env->active_tc.LO[ac] = acc[0];                               \
+}
+
+MAQ_S_L_PW(maq_s_l_pwl, 32);
+MAQ_S_L_PW(maq_s_l_pwr, 0);
+
+#undef MAQ_S_L_PW
+
+#define DM_OPERATE(name, func, is_add, sigext) \
+void helper_##name(target_ulong rs, target_ulong rt, uint32_t ac,    \
+                  CPUMIPSState *env)                                 \
+{                                                                    \
+    int32_t rs1, rs0;                                                \
+    int32_t rt1, rt0;                                                \
+    int64_t tempBL[2], tempAL[2];                                    \
+    int64_t acc[2];                                                  \
+    int64_t temp[2];                                                 \
+    int64_t temp_sum;                                                \
+                                                                     \
+    temp[0] = 0x00;                                                  \
+    temp[1] = 0x00;                                                  \
+                                                                     \
+    MIPSDSP_SPLIT64_32(rs, rs1, rs0);                                \
+    MIPSDSP_SPLIT64_32(rt, rt1, rt0);                                \
+                                                                     \
+    if (sigext) {                                                    \
+        tempBL[0] = (int64_t)mipsdsp_##func(rs1, rt1);               \
+        tempAL[0] = (int64_t)mipsdsp_##func(rs0, rt0);               \
+                                                                     \
+        if (tempBL[0] >= 0) {                                        \
+            tempBL[1] = 0x0;                                         \
+        } else {                                                     \
+            tempBL[1] = ~0ull;                                       \
+        }                                                            \
+                                                                     \
+        if (tempAL[0] >= 0) {                                        \
+            tempAL[1] = 0x0;                                         \
+        } else {                                                     \
+            tempAL[1] = ~0ull;                                       \
+        }                                                            \
+    } else {                                                         \
+        tempBL[0] = mipsdsp_##func(rs1, rt1);                        \
+        tempAL[0] = mipsdsp_##func(rs0, rt0);                        \
+        tempBL[1] = 0;                                               \
+        tempAL[1] = 0;                                               \
+    }                                                                \
+                                                                     \
+    acc[1] = env->active_tc.HI[ac];                                  \
+    acc[0] = env->active_tc.LO[ac];                                  \
+                                                                     \
+    temp_sum = tempBL[0] + tempAL[0];                                \
+    if (((uint64_t)temp_sum < (uint64_t)tempBL[0]) &&                \
+        ((uint64_t)temp_sum < (uint64_t)tempAL[0])) {                \
+        temp[1] += 1;                                                \
+    }                                                                \
+    temp[0] = temp_sum;                                              \
+    temp[1] += tempBL[1] + tempAL[1];                                \
+                                                                     \
+    if (is_add) {                                                    \
+        temp_sum = acc[0] + temp[0];                                 \
+        if (((uint64_t)temp_sum < (uint64_t)acc[0]) &&               \
+            ((uint64_t)temp_sum < (uint64_t)temp[0])) {              \
+            acc[1] += 1;                                             \
+        }                                                            \
+        temp[0] = temp_sum;                                          \
+        temp[1] = acc[1] + temp[1];                                  \
+    } else {                                                         \
+        temp_sum = acc[0] - temp[0];                                 \
+        if ((uint64_t)temp_sum > (uint64_t)acc[0]) {                 \
+            acc[1] -= 1;                                             \
+        }                                                            \
+        temp[0] = temp_sum;                                          \
+        temp[1] = acc[1] - temp[1];                                  \
+    }                                                                \
+                                                                     \
+    env->active_tc.HI[ac] = temp[1];                                 \
+    env->active_tc.LO[ac] = temp[0];                                 \
+}
+
+DM_OPERATE(dmadd, mul_i32_i32, 1, 1);
+DM_OPERATE(dmaddu, mul_u32_u32, 1, 0);
+DM_OPERATE(dmsub, mul_i32_i32, 0, 1);
+DM_OPERATE(dmsubu, mul_u32_u32, 0, 0);
+#undef DM_OPERATE
+#endif
+
+/** DSP Bit/Manipulation Sub-class insns **/
+target_ulong helper_bitrev(target_ulong rt)
+{
+    int32_t temp;
+    uint32_t rd;
+    int i;
+
+    temp = rt & MIPSDSP_LO;
+    rd = 0;
+    for (i = 0; i < 16; i++) {
+        rd = (rd << 1) | (temp & 1);
+        temp = temp >> 1;
+    }
+
+    return (target_ulong)rd;
+}
+
+#define BIT_INSV(name, posfilter, sizefilter, ret_type)         \
+target_ulong helper_##name(CPUMIPSState *env, target_ulong rs,  \
+                           target_ulong rt)                     \
+{                                                               \
+    uint32_t pos, size, msb, lsb;                               \
+    target_ulong filter;                                        \
+    target_ulong temp, temprs, temprt;                          \
+    target_ulong dspc;                                          \
+                                                                \
+    dspc = env->active_tc.DSPControl;                           \
+                                                                \
+    pos  = dspc & posfilter;                                    \
+    size = (dspc >> 7) & sizefilter;                            \
+                                                                \
+    msb  = pos + size - 1;                                      \
+    lsb  = pos;                                                 \
+                                                                \
+    if (lsb > msb || (msb > TARGET_LONG_BITS)) {                \
+        return rt;                                              \
+    }                                                           \
+                                                                \
+    filter = ((int32_t)0x01 << size) - 1;                       \
+    filter = filter << pos;                                     \
+    temprs = rs & filter;                                       \
+    temprt = rt & ~filter;                                      \
+    temp = temprs | temprt;                                     \
+                                                                \
+    return (target_long)(ret_type)temp;                         \
+}
+
+BIT_INSV(insv, 0x1F, 0x1F, int32_t);
+#ifdef TARGET_MIPS64
+BIT_INSV(dinsv, 0x7F, 0x3F, target_long);
+#endif
+
+#undef BIT_INSV
+
+
+/** DSP Compare-Pick Sub-class insns **/
+#define CMP_HAS_RET(name, func, split_num, filter, bit_size) \
+target_ulong helper_##name(target_ulong rs, target_ulong rt) \
+{                                                       \
+    uint32_t rs_t, rt_t;                                \
+    uint8_t cc;                                         \
+    uint32_t temp = 0;                                  \
+    int i;                                              \
+                                                        \
+    for (i = 0; i < split_num; i++) {                   \
+        rs_t = (rs >> (bit_size * i)) & filter;         \
+        rt_t = (rt >> (bit_size * i)) & filter;         \
+        cc = mipsdsp_##func(rs_t, rt_t);                \
+        temp |= cc << i;                                \
+    }                                                   \
+                                                        \
+    return (target_ulong)temp;                          \
+}
+
+CMP_HAS_RET(cmpgu_eq_qb, cmpu_eq, 4, MIPSDSP_Q0, 8);
+CMP_HAS_RET(cmpgu_lt_qb, cmpu_lt, 4, MIPSDSP_Q0, 8);
+CMP_HAS_RET(cmpgu_le_qb, cmpu_le, 4, MIPSDSP_Q0, 8);
+
+#ifdef TARGET_MIPS64
+CMP_HAS_RET(cmpgu_eq_ob, cmpu_eq, 8, MIPSDSP_Q0, 8);
+CMP_HAS_RET(cmpgu_lt_ob, cmpu_lt, 8, MIPSDSP_Q0, 8);
+CMP_HAS_RET(cmpgu_le_ob, cmpu_le, 8, MIPSDSP_Q0, 8);
+#endif
+
+#undef CMP_HAS_RET
+
+
+#define CMP_NO_RET(name, func, split_num, filter, bit_size) \
+void helper_##name(target_ulong rs, target_ulong rt,        \
+                            CPUMIPSState *env)              \
+{                                                           \
+    int##bit_size##_t rs_t, rt_t;                           \
+    int##bit_size##_t flag = 0;                             \
+    int##bit_size##_t cc;                                   \
+    int i;                                                  \
+                                                            \
+    for (i = 0; i < split_num; i++) {                       \
+        rs_t = (rs >> (bit_size * i)) & filter;             \
+        rt_t = (rt >> (bit_size * i)) & filter;             \
+                                                            \
+        cc = mipsdsp_##func((int32_t)rs_t, (int32_t)rt_t);  \
+        flag |= cc << i;                                    \
+    }                                                       \
+                                                            \
+    set_DSPControl_24(flag, split_num, env);                \
+}
+
+CMP_NO_RET(cmpu_eq_qb, cmpu_eq, 4, MIPSDSP_Q0, 8);
+CMP_NO_RET(cmpu_lt_qb, cmpu_lt, 4, MIPSDSP_Q0, 8);
+CMP_NO_RET(cmpu_le_qb, cmpu_le, 4, MIPSDSP_Q0, 8);
+
+CMP_NO_RET(cmp_eq_ph, cmp_eq, 2, MIPSDSP_LO, 16);
+CMP_NO_RET(cmp_lt_ph, cmp_lt, 2, MIPSDSP_LO, 16);
+CMP_NO_RET(cmp_le_ph, cmp_le, 2, MIPSDSP_LO, 16);
+
+#ifdef TARGET_MIPS64
+CMP_NO_RET(cmpu_eq_ob, cmpu_eq, 8, MIPSDSP_Q0, 8);
+CMP_NO_RET(cmpu_lt_ob, cmpu_lt, 8, MIPSDSP_Q0, 8);
+CMP_NO_RET(cmpu_le_ob, cmpu_le, 8, MIPSDSP_Q0, 8);
+
+CMP_NO_RET(cmp_eq_qh, cmp_eq, 4, MIPSDSP_LO, 16);
+CMP_NO_RET(cmp_lt_qh, cmp_lt, 4, MIPSDSP_LO, 16);
+CMP_NO_RET(cmp_le_qh, cmp_le, 4, MIPSDSP_LO, 16);
+
+CMP_NO_RET(cmp_eq_pw, cmp_eq, 2, MIPSDSP_LLO, 32);
+CMP_NO_RET(cmp_lt_pw, cmp_lt, 2, MIPSDSP_LLO, 32);
+CMP_NO_RET(cmp_le_pw, cmp_le, 2, MIPSDSP_LLO, 32);
+#endif
+#undef CMP_NO_RET
+
+#if defined(TARGET_MIPS64)
+
+#define CMPGDU_OB(name) \
+target_ulong helper_cmpgdu_##name##_ob(target_ulong rs, target_ulong rt, \
+                                       CPUMIPSState *env)  \
+{                                                     \
+    int i;                                            \
+    uint8_t rs_t, rt_t;                               \
+    uint32_t cond;                                    \
+                                                      \
+    cond = 0;                                         \
+                                                      \
+    for (i = 0; i < 8; i++) {                         \
+        rs_t = (rs >> (8 * i)) & MIPSDSP_Q0;          \
+        rt_t = (rt >> (8 * i)) & MIPSDSP_Q0;          \
+                                                      \
+        if (mipsdsp_cmpu_##name(rs_t, rt_t)) {        \
+            cond |= 0x01 << i;                        \
+        }                                             \
+    }                                                 \
+                                                      \
+    set_DSPControl_24(cond, 8, env);                  \
+                                                      \
+    return (uint64_t)cond;                            \
+}
+
+CMPGDU_OB(eq)
+CMPGDU_OB(lt)
+CMPGDU_OB(le)
+#undef CMPGDU_OB
+#endif
+
+#define PICK_INSN(name, split_num, filter, bit_size, ret32bit) \
+target_ulong helper_##name(target_ulong rs, target_ulong rt,   \
+                            CPUMIPSState *env)                 \
+{                                                              \
+    uint32_t rs_t, rt_t;                                       \
+    uint32_t cc;                                               \
+    target_ulong dsp;                                          \
+    int i;                                                     \
+    target_ulong result = 0;                                   \
+                                                               \
+    dsp = env->active_tc.DSPControl;                           \
+    for (i = 0; i < split_num; i++) {                          \
+        rs_t = (rs >> (bit_size * i)) & filter;                \
+        rt_t = (rt >> (bit_size * i)) & filter;                \
+        cc = (dsp >> (24 + i)) & 0x01;                         \
+        cc = cc == 1 ? rs_t : rt_t;                            \
+                                                               \
+        result |= (target_ulong)cc << (bit_size * i);          \
+    }                                                          \
+                                                               \
+    if (ret32bit) {                                            \
+        result = (target_long)(int32_t)(result & MIPSDSP_LLO); \
+    }                                                          \
+                                                               \
+    return result;                                             \
+}
+
+PICK_INSN(pick_qb, 4, MIPSDSP_Q0, 8, 1);
+PICK_INSN(pick_ph, 2, MIPSDSP_LO, 16, 1);
+
+#ifdef TARGET_MIPS64
+PICK_INSN(pick_ob, 8, MIPSDSP_Q0, 8, 0);
+PICK_INSN(pick_qh, 4, MIPSDSP_LO, 16, 0);
+PICK_INSN(pick_pw, 2, MIPSDSP_LLO, 32, 0);
+#endif
+#undef PICK_INSN
+
+#define APPEND_INSN(name, ret_32) \
+target_ulong helper_##name(target_ulong rt, target_ulong rs, uint32_t sa) \
+{                                                                         \
+    target_ulong temp;                                                    \
+                                                                          \
+    if (ret_32) {                                                         \
+        temp = ((rt & MIPSDSP_LLO) << sa) |                               \
+               ((rs & MIPSDSP_LLO) & ((0x01 << sa) - 1));                 \
+        temp = (target_long)(int32_t)(temp & MIPSDSP_LLO);                \
+    } else {                                                              \
+        temp = (rt << sa) | (rs & ((0x01 << sa) - 1));                    \
+    }                                                                     \
+                                                                          \
+    return temp;                                                          \
+}
+
+APPEND_INSN(append, 1);
+#ifdef TARGET_MIPS64
+APPEND_INSN(dappend, 0);
+#endif
+#undef APPEND_INSN
+
+#define PREPEND_INSN(name, or_val, ret_32)                    \
+target_ulong helper_##name(target_ulong rs, target_ulong rt,  \
+                           uint32_t sa)                       \
+{                                                             \
+    sa |= or_val;                                             \
+                                                              \
+    if (1) {                                                  \
+        return (target_long)(int32_t)(uint32_t)               \
+            (((rs & MIPSDSP_LLO) << (32 - sa)) |              \
+             ((rt & MIPSDSP_LLO) >> sa));                     \
+    } else {                                                  \
+        return (rs << (64 - sa)) | (rt >> sa);                \
+    }                                                         \
+}
+
+PREPEND_INSN(prepend, 0, 1);
+#ifdef TARGET_MIPS64
+PREPEND_INSN(prependw, 0, 0);
+PREPEND_INSN(prependd, 0x20, 0);
+#endif
+#undef PREPEND_INSN
+
+#define BALIGN_INSN(name, filter, ret32) \
+target_ulong helper_##name(target_ulong rs, target_ulong rt, uint32_t bp) \
+{                                                                         \
+    bp = bp & 0x03;                                                       \
+                                                                          \
+    if ((bp & 1) == 0) {                                                  \
+        return rt;                                                        \
+    } else {                                                              \
+        if (ret32) {                                                      \
+            return (target_long)(int32_t)((rt << (8 * bp)) |              \
+                                          (rs >> (8 * (4 - bp))));        \
+        } else {                                                          \
+            return (rt << (8 * bp)) | (rs >> (8 * (8 - bp)));             \
+        }                                                                 \
+    }                                                                     \
+}
+
+BALIGN_INSN(balign, 0x03, 1);
+#if defined(TARGET_MIPS64)
+BALIGN_INSN(dbalign, 0x07, 0);
+#endif
+#undef BALIGN_INSN
+
+target_ulong helper_packrl_ph(target_ulong rs, target_ulong rt)
+{
+    uint32_t rsl, rth;
+
+    rsl =  rs & MIPSDSP_LO;
+    rth = (rt & MIPSDSP_HI) >> 16;
+
+    return (target_long)(int32_t)((rsl << 16) | rth);
+}
+
+#if defined(TARGET_MIPS64)
+target_ulong helper_packrl_pw(target_ulong rs, target_ulong rt)
+{
+    uint32_t rs0, rt1;
+
+    rs0 = rs & MIPSDSP_LLO;
+    rt1 = (rt >> 32) & MIPSDSP_LLO;
+
+    return ((uint64_t)rs0 << 32) | (uint64_t)rt1;
+}
+#endif
+
+/** DSP Accumulator and DSPControl Access Sub-class insns **/
+target_ulong helper_extr_w(target_ulong ac, target_ulong shift,
+                           CPUMIPSState *env)
+{
+    int32_t tempI;
+    int64_t tempDL[2];
+
+    shift = shift & 0x0F;
+
+    mipsdsp_rndrashift_short_acc(tempDL, ac, shift, env);
+    if ((tempDL[1] != 0 || (tempDL[0] & MIPSDSP_LHI) != 0) &&
+        (tempDL[1] != 1 || (tempDL[0] & MIPSDSP_LHI) != MIPSDSP_LHI)) {
+        set_DSPControl_overflow_flag(1, 23, env);
+    }
+
+    tempI = (tempDL[0] >> 1) & MIPSDSP_LLO;
+
+    tempDL[0] += 1;
+    if (tempDL[0] == 0) {
+        tempDL[1] += 1;
+    }
+
+    if ((!(tempDL[1] == 0 && (tempDL[0] & MIPSDSP_LHI) == 0x00)) &&
+        (!(tempDL[1] == 1 && (tempDL[0] & MIPSDSP_LHI) == MIPSDSP_LHI))) {
+        set_DSPControl_overflow_flag(1, 23, env);
+    }
+
+    return (target_long)tempI;
+}
+
+target_ulong helper_extr_r_w(target_ulong ac, target_ulong shift,
+                             CPUMIPSState *env)
+{
+    int64_t tempDL[2];
+
+    shift = shift & 0x0F;
+
+    mipsdsp_rndrashift_short_acc(tempDL, ac, shift, env);
+    if ((tempDL[1] != 0 || (tempDL[0] & MIPSDSP_LHI) != 0) &&
+        (tempDL[1] != 1 || (tempDL[0] & MIPSDSP_LHI) != MIPSDSP_LHI)) {
+        set_DSPControl_overflow_flag(1, 23, env);
+    }
+
+    tempDL[0] += 1;
+    if (tempDL[0] == 0) {
+        tempDL[1] += 1;
+    }
+
+    if ((tempDL[1] != 0 || (tempDL[0] & MIPSDSP_LHI) != 0) &&
+        (tempDL[1] != 1 && (tempDL[0] & MIPSDSP_LHI) != MIPSDSP_LHI)) {
+        set_DSPControl_overflow_flag(1, 23, env);
+    }
+
+    return (target_long)(int32_t)(tempDL[0] >> 1);
+}
+
+target_ulong helper_extr_rs_w(target_ulong ac, target_ulong shift,
+                              CPUMIPSState *env)
+{
+    int32_t tempI, temp64;
+    int64_t tempDL[2];
+
+    shift = shift & 0x0F;
+
+    mipsdsp_rndrashift_short_acc(tempDL, ac, shift, env);
+    if ((tempDL[1] != 0 || (tempDL[0] & MIPSDSP_LHI) != 0) &&
+        (tempDL[1] != 1 || (tempDL[0] & MIPSDSP_LHI) != MIPSDSP_LHI)) {
+        set_DSPControl_overflow_flag(1, 23, env);
+    }
+    tempDL[0] += 1;
+    if (tempDL[0] == 0) {
+        tempDL[1] += 1;
+    }
+    tempI = tempDL[0] >> 1;
+
+    if ((tempDL[1] != 0 || (tempDL[0] & MIPSDSP_LHI) != 0) &&
+        (tempDL[1] != 1 || (tempDL[0] & MIPSDSP_LHI) != MIPSDSP_LHI)) {
+        temp64 = tempDL[1];
+        if (temp64 == 0) {
+            tempI = 0x7FFFFFFF;
+        } else {
+            tempI = 0x80000000;
+        }
+        set_DSPControl_overflow_flag(1, 23, env);
+    }
+
+    return (target_long)tempI;
+}
+
+#if defined(TARGET_MIPS64)
+target_ulong helper_dextr_w(target_ulong ac, target_ulong shift,
+                            CPUMIPSState *env)
+{
+    uint64_t temp[3];
+
+    shift = shift & 0x3F;
+
+    mipsdsp_rndrashift_acc(temp, ac, shift, env);
+
+    return (int64_t)(int32_t)(temp[0] >> 1);
+}
+
+target_ulong helper_dextr_r_w(target_ulong ac, target_ulong shift,
+                              CPUMIPSState *env)
+{
+    uint64_t temp[3];
+    uint32_t temp128;
+
+    shift = shift & 0x3F;
+    mipsdsp_rndrashift_acc(temp, ac, shift, env);
+
+    temp[0] += 1;
+    if (temp[0] == 0) {
+        temp[1] += 1;
+        if (temp[1] == 0) {
+            temp[2] += 1;
+        }
+    }
+
+    temp128 = temp[2] & 0x01;
+
+    if ((temp128 != 0 || temp[1] != 0) &&
+       (temp128 != 1 || temp[1] != ~0ull)) {
+        set_DSPControl_overflow_flag(1, 23, env);
+    }
+
+    return (int64_t)(int32_t)(temp[0] >> 1);
+}
+
+target_ulong helper_dextr_rs_w(target_ulong ac, target_ulong shift,
+                               CPUMIPSState *env)
+{
+    uint64_t temp[3];
+    uint32_t temp128;
+
+    shift = shift & 0x3F;
+    mipsdsp_rndrashift_acc(temp, ac, shift, env);
+
+    temp[0] += 1;
+    if (temp[0] == 0) {
+        temp[1] += 1;
+        if (temp[1] == 0) {
+            temp[2] += 1;
+        }
+    }
+
+    temp128 = temp[2] & 0x01;
+
+    if ((temp128 != 0 || temp[1] != 0) &&
+       (temp128 != 1 || temp[1] != ~0ull)) {
+        if (temp128 == 0) {
+            temp[0] = 0x0FFFFFFFF;
+        } else {
+            temp[0] = 0x0100000000ULL;
+        }
+        set_DSPControl_overflow_flag(1, 23, env);
+    }
+
+    return (int64_t)(int32_t)(temp[0] >> 1);
+}
+
+target_ulong helper_dextr_l(target_ulong ac, target_ulong shift,
+                            CPUMIPSState *env)
+{
+    uint64_t temp[3];
+    target_ulong result;
+
+    shift = shift & 0x3F;
+
+    mipsdsp_rndrashift_acc(temp, ac, shift, env);
+    result = (temp[1] << 63) | (temp[0] >> 1);
+
+    return result;
+}
+
+target_ulong helper_dextr_r_l(target_ulong ac, target_ulong shift,
+                              CPUMIPSState *env)
+{
+    uint64_t temp[3];
+    uint32_t temp128;
+    target_ulong result;
+
+    shift = shift & 0x3F;
+    mipsdsp_rndrashift_acc(temp, ac, shift, env);
+
+    temp[0] += 1;
+    if (temp[0] == 0) {
+        temp[1] += 1;
+        if (temp[1] == 0) {
+            temp[2] += 1;
+        }
+    }
+
+    temp128 = temp[2] & 0x01;
+
+    if ((temp128 != 0 || temp[1] != 0) &&
+       (temp128 != 1 || temp[1] != ~0ull)) {
+        set_DSPControl_overflow_flag(1, 23, env);
+    }
+
+    result = (temp[1] << 63) | (temp[0] >> 1);
+
+    return result;
+}
+
+target_ulong helper_dextr_rs_l(target_ulong ac, target_ulong shift,
+                               CPUMIPSState *env)
+{
+    uint64_t temp[3];
+    uint32_t temp128;
+    target_ulong result;
+
+    shift = shift & 0x3F;
+    mipsdsp_rndrashift_acc(temp, ac, shift, env);
+
+    temp[0] += 1;
+    if (temp[0] == 0) {
+        temp[1] += 1;
+        if (temp[1] == 0) {
+            temp[2] += 1;
+        }
+    }
+
+    temp128 = temp[2] & 0x01;
+
+    if ((temp128 != 0 || temp[1] != 0) &&
+       (temp128 != 1 || temp[1] != ~0ull)) {
+        if (temp128 == 0) {
+            temp[1] &= ~0x00ull - 1;
+            temp[0] |= ~0x00ull - 1;
+        } else {
+            temp[1] |= 0x01;
+            temp[0] &= 0x01;
+        }
+        set_DSPControl_overflow_flag(1, 23, env);
+    }
+    result = (temp[1] << 63) | (temp[0] >> 1);
+
+    return result;
+}
+#endif
+
+target_ulong helper_extr_s_h(target_ulong ac, target_ulong shift,
+                             CPUMIPSState *env)
+{
+    int64_t temp;
+
+    shift = shift & 0x0F;
+
+    temp = mipsdsp_rashift_short_acc(ac, shift, env);
+    if (temp > (int64_t)0x7FFF) {
+        temp = 0x00007FFF;
+        set_DSPControl_overflow_flag(1, 23, env);
+    } else if (temp < (int64_t)0xFFFFFFFFFFFF8000ULL) {
+        temp = 0xFFFF8000;
+        set_DSPControl_overflow_flag(1, 23, env);
+    }
+
+    return (target_long)(int32_t)(temp & 0xFFFFFFFF);
+}
+
+
+#if defined(TARGET_MIPS64)
+target_ulong helper_dextr_s_h(target_ulong ac, target_ulong shift,
+                              CPUMIPSState *env)
+{
+    int64_t temp[2];
+    uint32_t temp127;
+
+    shift = shift & 0x1F;
+
+    mipsdsp_rashift_acc((uint64_t *)temp, ac, shift, env);
+
+    temp127 = (temp[1] >> 63) & 0x01;
+
+    if ((temp127 == 0) && (temp[1] > 0 || temp[0] > 32767)) {
+        temp[0] &= 0xFFFF0000;
+        temp[0] |= 0x00007FFF;
+        set_DSPControl_overflow_flag(1, 23, env);
+    } else if ((temp127 == 1) &&
+            (temp[1] < 0xFFFFFFFFFFFFFFFFll
+             || temp[0] < 0xFFFFFFFFFFFF1000ll)) {
+        temp[0] &= 0xFFFF0000;
+        temp[0] |= 0x00008000;
+        set_DSPControl_overflow_flag(1, 23, env);
+    }
+
+    return (int64_t)(int16_t)(temp[0] & MIPSDSP_LO);
+}
+
+#endif
+
+target_ulong helper_extp(target_ulong ac, target_ulong size, CPUMIPSState *env)
+{
+    int32_t start_pos;
+    int sub;
+    uint32_t temp;
+    uint64_t acc;
+
+    size = size & 0x1F;
+
+    temp = 0;
+    start_pos = get_DSPControl_pos(env);
+    sub = start_pos - (size + 1);
+    if (sub >= -1) {
+        acc = ((uint64_t)env->active_tc.HI[ac] << 32) |
+              ((uint64_t)env->active_tc.LO[ac] & MIPSDSP_LLO);
+        temp = (acc >> (start_pos - size)) &
+               (((uint32_t)0x01 << (size + 1)) - 1);
+        set_DSPControl_efi(0, env);
+    } else {
+        set_DSPControl_efi(1, env);
+    }
+
+    return (target_ulong)temp;
+}
+
+target_ulong helper_extpdp(target_ulong ac, target_ulong size,
+                           CPUMIPSState *env)
+{
+    int32_t start_pos;
+    int sub;
+    uint32_t temp;
+    uint64_t acc;
+
+    size = size & 0x1F;
+    temp = 0;
+    start_pos = get_DSPControl_pos(env);
+    sub = start_pos - (size + 1);
+    if (sub >= -1) {
+        acc  = ((uint64_t)env->active_tc.HI[ac] << 32) |
+               ((uint64_t)env->active_tc.LO[ac] & MIPSDSP_LLO);
+        temp = (acc >> (start_pos - size)) &
+               (((uint32_t)0x01 << (size + 1)) - 1);
+
+        set_DSPControl_pos(start_pos - (size + 1), env);
+        set_DSPControl_efi(0, env);
+    } else {
+        set_DSPControl_efi(1, env);
+    }
+
+    return (target_ulong)temp;
+}
+
+
+#if defined(TARGET_MIPS64)
+target_ulong helper_dextp(target_ulong ac, target_ulong size, CPUMIPSState *env)
+{
+    int start_pos;
+    int len;
+    int sub;
+    uint64_t tempB, tempA;
+    uint64_t temp;
+
+    temp = 0;
+
+    size = size & 0x3F;
+    start_pos = get_DSPControl_pos(env);
+    len = start_pos - size;
+    tempB = env->active_tc.HI[ac];
+    tempA = env->active_tc.LO[ac];
+
+    sub = start_pos - (size + 1);
+
+    if (sub >= -1) {
+        temp = (tempB << (64 - len)) | (tempA >> len);
+        temp = temp & ((0x01 << (size + 1)) - 1);
+        set_DSPControl_efi(0, env);
+    } else {
+        set_DSPControl_efi(1, env);
+    }
+
+    return temp;
+}
+
+target_ulong helper_dextpdp(target_ulong ac, target_ulong size,
+                            CPUMIPSState *env)
+{
+    int start_pos;
+    int len;
+    int sub;
+    uint64_t tempB, tempA;
+    uint64_t temp;
+
+    temp = 0;
+    size = size & 0x3F;
+    start_pos = get_DSPControl_pos(env);
+    len = start_pos - size;
+    tempB = env->active_tc.HI[ac];
+    tempA = env->active_tc.LO[ac];
+
+    sub = start_pos - (size + 1);
+
+    if (sub >= -1) {
+        temp = (tempB << (64 - len)) | (tempA >> len);
+        temp = temp & ((0x01 << (size + 1)) - 1);
+        set_DSPControl_pos(sub, env);
+        set_DSPControl_efi(0, env);
+    } else {
+        set_DSPControl_efi(1, env);
+    }
+
+    return temp;
+}
+
+#endif
+
+void helper_shilo(target_ulong ac, target_ulong rs, CPUMIPSState *env)
+{
+    int8_t  rs5_0;
+    uint64_t temp, acc;
+
+    rs5_0 = rs & 0x3F;
+    rs5_0 = (int8_t)(rs5_0 << 2) >> 2;
+    rs5_0 = MIPSDSP_ABS(rs5_0);
+    acc   = (((uint64_t)env->active_tc.HI[ac] << 32) & MIPSDSP_LHI) |
+            ((uint64_t)env->active_tc.LO[ac] & MIPSDSP_LLO);
+    if (rs5_0 == 0) {
+        temp = acc;
+    } else {
+        if (rs5_0 > 0) {
+            temp = acc >> rs5_0;
+        } else {
+            temp = acc << rs5_0;
+        }
+    }
+
+    env->active_tc.HI[ac] = (target_ulong)(int32_t)((temp & MIPSDSP_LHI) >> 32);
+    env->active_tc.LO[ac] = (target_ulong)(int32_t)(temp & MIPSDSP_LLO);
+}
+
+#if defined(TARGET_MIPS64)
+void helper_dshilo(target_ulong shift, target_ulong ac, CPUMIPSState *env)
+{
+    int8_t shift_t;
+    uint64_t tempB, tempA;
+
+    shift_t = (int8_t)(shift << 1) >> 1;
+
+    tempB = env->active_tc.HI[ac];
+    tempA = env->active_tc.LO[ac];
+
+    if (shift_t != 0) {
+        if (shift_t >= 0) {
+            tempA = (tempB << (64 - shift_t)) | (tempA >> shift_t);
+            tempB = tempB >> shift_t;
+        } else {
+            shift_t = -shift_t;
+            tempB = (tempB << shift_t) | (tempA >> (64 - shift_t));
+            tempA = tempA << shift_t;
+        }
+    }
+
+    env->active_tc.HI[ac] = tempB;
+    env->active_tc.LO[ac] = tempA;
+}
+
+#endif
+void helper_mthlip(target_ulong ac, target_ulong rs, CPUMIPSState *env)
+{
+    int32_t tempA, tempB, pos;
+
+    tempA = rs;
+    tempB = env->active_tc.LO[ac];
+    env->active_tc.HI[ac] = (target_long)tempB;
+    env->active_tc.LO[ac] = (target_long)tempA;
+    pos = get_DSPControl_pos(env);
+
+    if (pos > 32) {
+        return;
+    } else {
+        set_DSPControl_pos(pos + 32, env);
+    }
+}
+
+#if defined(TARGET_MIPS64)
+void helper_dmthlip(target_ulong rs, target_ulong ac, CPUMIPSState *env)
+{
+    uint8_t ac_t;
+    uint8_t pos;
+    uint64_t tempB, tempA;
+
+    ac_t = ac & 0x3;
+
+    tempA = rs;
+    tempB = env->active_tc.LO[ac_t];
+
+    env->active_tc.HI[ac_t] = tempB;
+    env->active_tc.LO[ac_t] = tempA;
+
+    pos = get_DSPControl_pos(env);
+
+    if (pos <= 64) {
+        pos = pos + 64;
+        set_DSPControl_pos(pos, env);
+    }
+}
+#endif
+
+void helper_wrdsp(target_ulong rs, target_ulong mask_num, CPUMIPSState *env)
+{
+    uint8_t  mask[6];
+    uint8_t  i;
+    uint32_t newbits, overwrite;
+    target_ulong dsp;
+
+    newbits   = 0x00;
+    overwrite = 0xFFFFFFFF;
+    dsp = env->active_tc.DSPControl;
+
+    for (i = 0; i < 6; i++) {
+        mask[i] = (mask_num >> i) & 0x01;
+    }
+
+    if (mask[0] == 1) {
+#if defined(TARGET_MIPS64)
+        overwrite &= 0xFFFFFF80;
+        newbits   &= 0xFFFFFF80;
+        newbits   |= 0x0000007F & rs;
+#else
+        overwrite &= 0xFFFFFFC0;
+        newbits   &= 0xFFFFFFC0;
+        newbits   |= 0x0000003F & rs;
+#endif
+    }
+
+    if (mask[1] == 1) {
+        overwrite &= 0xFFFFE07F;
+        newbits   &= 0xFFFFE07F;
+        newbits   |= 0x00001F80 & rs;
+    }
+
+    if (mask[2] == 1) {
+        overwrite &= 0xFFFFDFFF;
+        newbits   &= 0xFFFFDFFF;
+        newbits   |= 0x00002000 & rs;
+    }
+
+    if (mask[3] == 1) {
+        overwrite &= 0xFF00FFFF;
+        newbits   &= 0xFF00FFFF;
+        newbits   |= 0x00FF0000 & rs;
+    }
+
+    if (mask[4] == 1) {
+        overwrite &= 0x00FFFFFF;
+        newbits   &= 0x00FFFFFF;
+        newbits   |= 0xFF000000 & rs;
+    }
+
+    if (mask[5] == 1) {
+        overwrite &= 0xFFFFBFFF;
+        newbits   &= 0xFFFFBFFF;
+        newbits   |= 0x00004000 & rs;
+    }
+
+    dsp = dsp & overwrite;
+    dsp = dsp | newbits;
+    env->active_tc.DSPControl = dsp;
+}
+
+target_ulong helper_rddsp(target_ulong masknum, CPUMIPSState *env)
+{
+    uint8_t  mask[6];
+    uint32_t ruler, i;
+    target_ulong temp;
+    target_ulong dsp;
+
+    ruler = 0x01;
+    for (i = 0; i < 6; i++) {
+        mask[i] = (masknum & ruler) >> i ;
+        ruler = ruler << 1;
+    }
+
+    temp  = 0x00;
+    dsp = env->active_tc.DSPControl;
+
+    if (mask[0] == 1) {
+#if defined(TARGET_MIPS64)
+        temp |= dsp & 0x7F;
+#else
+        temp |= dsp & 0x3F;
+#endif
+    }
+
+    if (mask[1] == 1) {
+        temp |= dsp & 0x1F80;
+    }
+
+    if (mask[2] == 1) {
+        temp |= dsp & 0x2000;
+    }
+
+    if (mask[3] == 1) {
+        temp |= dsp & 0x00FF0000;
+    }
+
+    if (mask[4] == 1) {
+        temp |= dsp & 0xFF000000;
+    }
+
+    if (mask[5] == 1) {
+        temp |= dsp & 0x4000;
+    }
+
+    return temp;
+}
+
+
+#undef MIPSDSP_LHI
+#undef MIPSDSP_LLO
+#undef MIPSDSP_HI
+#undef MIPSDSP_LO
+#undef MIPSDSP_Q3
+#undef MIPSDSP_Q2
+#undef MIPSDSP_Q1
+#undef MIPSDSP_Q0
+
+#undef MIPSDSP_SPLIT32_8
+#undef MIPSDSP_SPLIT32_16
+
+#undef MIPSDSP_RETURN32
+#undef MIPSDSP_RETURN32_8
+#undef MIPSDSP_RETURN32_16
+
+#ifdef TARGET_MIPS64
+#undef MIPSDSP_SPLIT64_16
+#undef MIPSDSP_SPLIT64_32
+#undef MIPSDSP_RETURN64_16
+#undef MIPSDSP_RETURN64_32
+#endif
diff --git a/target-mips/helper.c b/target-mips/helper.c
index 4208bb20c..e877b8db7 100644
--- a/target-mips/helper.c
+++ b/target-mips/helper.c
@@ -36,7 +36,7 @@ enum {
 #if !defined(CONFIG_USER_ONLY)
 
 /* no MMU emulation */
-int no_mmu_map_address (CPUMIPSState *env, target_phys_addr_t *physical, int *prot,
+int no_mmu_map_address (CPUMIPSState *env, hwaddr *physical, int *prot,
                         target_ulong address, int rw, int access_type)
 {
     *physical = address;
@@ -45,7 +45,7 @@ int no_mmu_map_address (CPUMIPSState *env, target_phys_addr_t *physical, int *pr
 }
 
 /* fixed mapping MMU emulation */
-int fixed_mmu_map_address (CPUMIPSState *env, target_phys_addr_t *physical, int *prot,
+int fixed_mmu_map_address (CPUMIPSState *env, hwaddr *physical, int *prot,
                            target_ulong address, int rw, int access_type)
 {
     if (address <= (int32_t)0x7FFFFFFFUL) {
@@ -63,7 +63,7 @@ int fixed_mmu_map_address (CPUMIPSState *env, target_phys_addr_t *physical, int
 }
 
 /* MIPS32/MIPS64 R4000-style MMU emulation */
-int r4k_map_address (CPUMIPSState *env, target_phys_addr_t *physical, int *prot,
+int r4k_map_address (CPUMIPSState *env, hwaddr *physical, int *prot,
                      target_ulong address, int rw, int access_type)
 {
     uint8_t ASID = env->CP0_EntryHi & 0xFF;
@@ -99,7 +99,7 @@ int r4k_map_address (CPUMIPSState *env, target_phys_addr_t *physical, int *prot,
     return TLBRET_NOMATCH;
 }
 
-static int get_physical_address (CPUMIPSState *env, target_phys_addr_t *physical,
+static int get_physical_address (CPUMIPSState *env, hwaddr *physical,
                                 int *prot, target_ulong address,
                                 int rw, int access_type)
 {
@@ -254,9 +254,9 @@ static void raise_mmu_exception(CPUMIPSState *env, target_ulong address,
 }
 
 #if !defined(CONFIG_USER_ONLY)
-target_phys_addr_t cpu_get_phys_page_debug(CPUMIPSState *env, target_ulong addr)
+hwaddr cpu_get_phys_page_debug(CPUMIPSState *env, target_ulong addr)
 {
-    target_phys_addr_t phys_addr;
+    hwaddr phys_addr;
     int prot;
 
     if (get_physical_address(env, &phys_addr, &prot, addr, 0, ACCESS_INT) != 0)
@@ -269,7 +269,7 @@ int cpu_mips_handle_mmu_fault (CPUMIPSState *env, target_ulong address, int rw,
                                int mmu_idx)
 {
 #if !defined(CONFIG_USER_ONLY)
-    target_phys_addr_t physical;
+    hwaddr physical;
     int prot;
     int access_type;
 #endif
@@ -308,9 +308,9 @@ int cpu_mips_handle_mmu_fault (CPUMIPSState *env, target_ulong address, int rw,
 }
 
 #if !defined(CONFIG_USER_ONLY)
-target_phys_addr_t cpu_mips_translate_address(CPUMIPSState *env, target_ulong address, int rw)
+hwaddr cpu_mips_translate_address(CPUMIPSState *env, target_ulong address, int rw)
 {
-    target_phys_addr_t physical;
+    hwaddr physical;
     int prot;
     int access_type;
     int ret = 0;
@@ -592,6 +592,9 @@ void do_interrupt (CPUMIPSState *env)
     case EXCP_THREAD:
         cause = 25;
         goto set_EPC;
+    case EXCP_DSPDIS:
+        cause = 26;
+        goto set_EPC;
     case EXCP_CACHE:
         cause = 30;
         if (env->CP0_Status & (1 << CP0St_BEV)) {
diff --git a/target-mips/helper.h b/target-mips/helper.h
index 76fb451e7..acf9ebd75 100644
--- a/target-mips/helper.h
+++ b/target-mips/helper.h
@@ -1,222 +1,218 @@
 #include "def-helper.h"
 
-DEF_HELPER_2(raise_exception_err, noreturn, i32, int)
-DEF_HELPER_1(raise_exception, noreturn, i32)
+DEF_HELPER_3(raise_exception_err, noreturn, env, i32, int)
+DEF_HELPER_2(raise_exception, noreturn, env, i32)
 
 #ifdef TARGET_MIPS64
-DEF_HELPER_3(ldl, tl, tl, tl, int)
-DEF_HELPER_3(ldr, tl, tl, tl, int)
-DEF_HELPER_3(sdl, void, tl, tl, int)
-DEF_HELPER_3(sdr, void, tl, tl, int)
+DEF_HELPER_4(sdl, void, env, tl, tl, int)
+DEF_HELPER_4(sdr, void, env, tl, tl, int)
 #endif
-DEF_HELPER_3(lwl, tl, tl, tl, int)
-DEF_HELPER_3(lwr, tl, tl, tl, int)
-DEF_HELPER_3(swl, void, tl, tl, int)
-DEF_HELPER_3(swr, void, tl, tl, int)
+DEF_HELPER_4(swl, void, env, tl, tl, int)
+DEF_HELPER_4(swr, void, env, tl, tl, int)
 
 #ifndef CONFIG_USER_ONLY
-DEF_HELPER_2(ll, tl, tl, int)
-DEF_HELPER_3(sc, tl, tl, tl, int)
+DEF_HELPER_3(ll, tl, env, tl, int)
+DEF_HELPER_4(sc, tl, env, tl, tl, int)
 #ifdef TARGET_MIPS64
-DEF_HELPER_2(lld, tl, tl, int)
-DEF_HELPER_3(scd, tl, tl, tl, int)
+DEF_HELPER_3(lld, tl, env, tl, int)
+DEF_HELPER_4(scd, tl, env, tl, tl, int)
 #endif
 #endif
 
-DEF_HELPER_FLAGS_1(clo, TCG_CALL_CONST | TCG_CALL_PURE, tl, tl)
-DEF_HELPER_FLAGS_1(clz, TCG_CALL_CONST | TCG_CALL_PURE, tl, tl)
+DEF_HELPER_FLAGS_1(clo, TCG_CALL_NO_RWG_SE, tl, tl)
+DEF_HELPER_FLAGS_1(clz, TCG_CALL_NO_RWG_SE, tl, tl)
 #ifdef TARGET_MIPS64
-DEF_HELPER_FLAGS_1(dclo, TCG_CALL_CONST | TCG_CALL_PURE, tl, tl)
-DEF_HELPER_FLAGS_1(dclz, TCG_CALL_CONST | TCG_CALL_PURE, tl, tl)
-DEF_HELPER_2(dmult, void, tl, tl)
-DEF_HELPER_2(dmultu, void, tl, tl)
-#endif
-
-DEF_HELPER_2(muls, tl, tl, tl)
-DEF_HELPER_2(mulsu, tl, tl, tl)
-DEF_HELPER_2(macc, tl, tl, tl)
-DEF_HELPER_2(maccu, tl, tl, tl)
-DEF_HELPER_2(msac, tl, tl, tl)
-DEF_HELPER_2(msacu, tl, tl, tl)
-DEF_HELPER_2(mulhi, tl, tl, tl)
-DEF_HELPER_2(mulhiu, tl, tl, tl)
-DEF_HELPER_2(mulshi, tl, tl, tl)
-DEF_HELPER_2(mulshiu, tl, tl, tl)
-DEF_HELPER_2(macchi, tl, tl, tl)
-DEF_HELPER_2(macchiu, tl, tl, tl)
-DEF_HELPER_2(msachi, tl, tl, tl)
-DEF_HELPER_2(msachiu, tl, tl, tl)
+DEF_HELPER_FLAGS_1(dclo, TCG_CALL_NO_RWG_SE, tl, tl)
+DEF_HELPER_FLAGS_1(dclz, TCG_CALL_NO_RWG_SE, tl, tl)
+DEF_HELPER_3(dmult, void, env, tl, tl)
+DEF_HELPER_3(dmultu, void, env, tl, tl)
+#endif
+
+DEF_HELPER_3(muls, tl, env, tl, tl)
+DEF_HELPER_3(mulsu, tl, env, tl, tl)
+DEF_HELPER_3(macc, tl, env, tl, tl)
+DEF_HELPER_3(maccu, tl, env, tl, tl)
+DEF_HELPER_3(msac, tl, env, tl, tl)
+DEF_HELPER_3(msacu, tl, env, tl, tl)
+DEF_HELPER_3(mulhi, tl, env, tl, tl)
+DEF_HELPER_3(mulhiu, tl, env, tl, tl)
+DEF_HELPER_3(mulshi, tl, env, tl, tl)
+DEF_HELPER_3(mulshiu, tl, env, tl, tl)
+DEF_HELPER_3(macchi, tl, env, tl, tl)
+DEF_HELPER_3(macchiu, tl, env, tl, tl)
+DEF_HELPER_3(msachi, tl, env, tl, tl)
+DEF_HELPER_3(msachiu, tl, env, tl, tl)
 
 #ifndef CONFIG_USER_ONLY
 /* CP0 helpers */
-DEF_HELPER_0(mfc0_mvpcontrol, tl)
-DEF_HELPER_0(mfc0_mvpconf0, tl)
-DEF_HELPER_0(mfc0_mvpconf1, tl)
-DEF_HELPER_0(mftc0_vpecontrol, tl)
-DEF_HELPER_0(mftc0_vpeconf0, tl)
-DEF_HELPER_0(mfc0_random, tl)
-DEF_HELPER_0(mfc0_tcstatus, tl)
-DEF_HELPER_0(mftc0_tcstatus, tl)
-DEF_HELPER_0(mfc0_tcbind, tl)
-DEF_HELPER_0(mftc0_tcbind, tl)
-DEF_HELPER_0(mfc0_tcrestart, tl)
-DEF_HELPER_0(mftc0_tcrestart, tl)
-DEF_HELPER_0(mfc0_tchalt, tl)
-DEF_HELPER_0(mftc0_tchalt, tl)
-DEF_HELPER_0(mfc0_tccontext, tl)
-DEF_HELPER_0(mftc0_tccontext, tl)
-DEF_HELPER_0(mfc0_tcschedule, tl)
-DEF_HELPER_0(mftc0_tcschedule, tl)
-DEF_HELPER_0(mfc0_tcschefback, tl)
-DEF_HELPER_0(mftc0_tcschefback, tl)
-DEF_HELPER_0(mfc0_count, tl)
-DEF_HELPER_0(mftc0_entryhi, tl)
-DEF_HELPER_0(mftc0_status, tl)
-DEF_HELPER_0(mftc0_cause, tl)
-DEF_HELPER_0(mftc0_epc, tl)
-DEF_HELPER_0(mftc0_ebase, tl)
-DEF_HELPER_1(mftc0_configx, tl, tl)
-DEF_HELPER_0(mfc0_lladdr, tl)
-DEF_HELPER_1(mfc0_watchlo, tl, i32)
-DEF_HELPER_1(mfc0_watchhi, tl, i32)
-DEF_HELPER_0(mfc0_debug, tl)
-DEF_HELPER_0(mftc0_debug, tl)
+DEF_HELPER_1(mfc0_mvpcontrol, tl, env)
+DEF_HELPER_1(mfc0_mvpconf0, tl, env)
+DEF_HELPER_1(mfc0_mvpconf1, tl, env)
+DEF_HELPER_1(mftc0_vpecontrol, tl, env)
+DEF_HELPER_1(mftc0_vpeconf0, tl, env)
+DEF_HELPER_1(mfc0_random, tl, env)
+DEF_HELPER_1(mfc0_tcstatus, tl, env)
+DEF_HELPER_1(mftc0_tcstatus, tl, env)
+DEF_HELPER_1(mfc0_tcbind, tl, env)
+DEF_HELPER_1(mftc0_tcbind, tl, env)
+DEF_HELPER_1(mfc0_tcrestart, tl, env)
+DEF_HELPER_1(mftc0_tcrestart, tl, env)
+DEF_HELPER_1(mfc0_tchalt, tl, env)
+DEF_HELPER_1(mftc0_tchalt, tl, env)
+DEF_HELPER_1(mfc0_tccontext, tl, env)
+DEF_HELPER_1(mftc0_tccontext, tl, env)
+DEF_HELPER_1(mfc0_tcschedule, tl, env)
+DEF_HELPER_1(mftc0_tcschedule, tl, env)
+DEF_HELPER_1(mfc0_tcschefback, tl, env)
+DEF_HELPER_1(mftc0_tcschefback, tl, env)
+DEF_HELPER_1(mfc0_count, tl, env)
+DEF_HELPER_1(mftc0_entryhi, tl, env)
+DEF_HELPER_1(mftc0_status, tl, env)
+DEF_HELPER_1(mftc0_cause, tl, env)
+DEF_HELPER_1(mftc0_epc, tl, env)
+DEF_HELPER_1(mftc0_ebase, tl, env)
+DEF_HELPER_2(mftc0_configx, tl, env, tl)
+DEF_HELPER_1(mfc0_lladdr, tl, env)
+DEF_HELPER_2(mfc0_watchlo, tl, env, i32)
+DEF_HELPER_2(mfc0_watchhi, tl, env, i32)
+DEF_HELPER_1(mfc0_debug, tl, env)
+DEF_HELPER_1(mftc0_debug, tl, env)
 #ifdef TARGET_MIPS64
-DEF_HELPER_0(dmfc0_tcrestart, tl)
-DEF_HELPER_0(dmfc0_tchalt, tl)
-DEF_HELPER_0(dmfc0_tccontext, tl)
-DEF_HELPER_0(dmfc0_tcschedule, tl)
-DEF_HELPER_0(dmfc0_tcschefback, tl)
-DEF_HELPER_0(dmfc0_lladdr, tl)
-DEF_HELPER_1(dmfc0_watchlo, tl, i32)
+DEF_HELPER_1(dmfc0_tcrestart, tl, env)
+DEF_HELPER_1(dmfc0_tchalt, tl, env)
+DEF_HELPER_1(dmfc0_tccontext, tl, env)
+DEF_HELPER_1(dmfc0_tcschedule, tl, env)
+DEF_HELPER_1(dmfc0_tcschefback, tl, env)
+DEF_HELPER_1(dmfc0_lladdr, tl, env)
+DEF_HELPER_2(dmfc0_watchlo, tl, env, i32)
 #endif /* TARGET_MIPS64 */
 
-DEF_HELPER_1(mtc0_index, void, tl)
-DEF_HELPER_1(mtc0_mvpcontrol, void, tl)
-DEF_HELPER_1(mtc0_vpecontrol, void, tl)
-DEF_HELPER_1(mttc0_vpecontrol, void, tl)
-DEF_HELPER_1(mtc0_vpeconf0, void, tl)
-DEF_HELPER_1(mttc0_vpeconf0, void, tl)
-DEF_HELPER_1(mtc0_vpeconf1, void, tl)
-DEF_HELPER_1(mtc0_yqmask, void, tl)
-DEF_HELPER_1(mtc0_vpeopt, void, tl)
-DEF_HELPER_1(mtc0_entrylo0, void, tl)
-DEF_HELPER_1(mtc0_tcstatus, void, tl)
-DEF_HELPER_1(mttc0_tcstatus, void, tl)
-DEF_HELPER_1(mtc0_tcbind, void, tl)
-DEF_HELPER_1(mttc0_tcbind, void, tl)
-DEF_HELPER_1(mtc0_tcrestart, void, tl)
-DEF_HELPER_1(mttc0_tcrestart, void, tl)
-DEF_HELPER_1(mtc0_tchalt, void, tl)
-DEF_HELPER_1(mttc0_tchalt, void, tl)
-DEF_HELPER_1(mtc0_tccontext, void, tl)
-DEF_HELPER_1(mttc0_tccontext, void, tl)
-DEF_HELPER_1(mtc0_tcschedule, void, tl)
-DEF_HELPER_1(mttc0_tcschedule, void, tl)
-DEF_HELPER_1(mtc0_tcschefback, void, tl)
-DEF_HELPER_1(mttc0_tcschefback, void, tl)
-DEF_HELPER_1(mtc0_entrylo1, void, tl)
-DEF_HELPER_1(mtc0_context, void, tl)
-DEF_HELPER_1(mtc0_pagemask, void, tl)
-DEF_HELPER_1(mtc0_pagegrain, void, tl)
-DEF_HELPER_1(mtc0_wired, void, tl)
-DEF_HELPER_1(mtc0_srsconf0, void, tl)
-DEF_HELPER_1(mtc0_srsconf1, void, tl)
-DEF_HELPER_1(mtc0_srsconf2, void, tl)
-DEF_HELPER_1(mtc0_srsconf3, void, tl)
-DEF_HELPER_1(mtc0_srsconf4, void, tl)
-DEF_HELPER_1(mtc0_hwrena, void, tl)
-DEF_HELPER_1(mtc0_count, void, tl)
-DEF_HELPER_1(mtc0_entryhi, void, tl)
-DEF_HELPER_1(mttc0_entryhi, void, tl)
-DEF_HELPER_1(mtc0_compare, void, tl)
-DEF_HELPER_1(mtc0_status, void, tl)
-DEF_HELPER_1(mttc0_status, void, tl)
-DEF_HELPER_1(mtc0_intctl, void, tl)
-DEF_HELPER_1(mtc0_srsctl, void, tl)
-DEF_HELPER_1(mtc0_cause, void, tl)
-DEF_HELPER_1(mttc0_cause, void, tl)
-DEF_HELPER_1(mtc0_ebase, void, tl)
-DEF_HELPER_1(mttc0_ebase, void, tl)
-DEF_HELPER_1(mtc0_config0, void, tl)
-DEF_HELPER_1(mtc0_config2, void, tl)
-DEF_HELPER_1(mtc0_lladdr, void, tl)
-DEF_HELPER_2(mtc0_watchlo, void, tl, i32)
-DEF_HELPER_2(mtc0_watchhi, void, tl, i32)
-DEF_HELPER_1(mtc0_xcontext, void, tl)
-DEF_HELPER_1(mtc0_framemask, void, tl)
-DEF_HELPER_1(mtc0_debug, void, tl)
-DEF_HELPER_1(mttc0_debug, void, tl)
-DEF_HELPER_1(mtc0_performance0, void, tl)
-DEF_HELPER_1(mtc0_taglo, void, tl)
-DEF_HELPER_1(mtc0_datalo, void, tl)
-DEF_HELPER_1(mtc0_taghi, void, tl)
-DEF_HELPER_1(mtc0_datahi, void, tl)
+DEF_HELPER_2(mtc0_index, void, env, tl)
+DEF_HELPER_2(mtc0_mvpcontrol, void, env, tl)
+DEF_HELPER_2(mtc0_vpecontrol, void, env, tl)
+DEF_HELPER_2(mttc0_vpecontrol, void, env, tl)
+DEF_HELPER_2(mtc0_vpeconf0, void, env, tl)
+DEF_HELPER_2(mttc0_vpeconf0, void, env, tl)
+DEF_HELPER_2(mtc0_vpeconf1, void, env, tl)
+DEF_HELPER_2(mtc0_yqmask, void, env, tl)
+DEF_HELPER_2(mtc0_vpeopt, void, env, tl)
+DEF_HELPER_2(mtc0_entrylo0, void, env, tl)
+DEF_HELPER_2(mtc0_tcstatus, void, env, tl)
+DEF_HELPER_2(mttc0_tcstatus, void, env, tl)
+DEF_HELPER_2(mtc0_tcbind, void, env, tl)
+DEF_HELPER_2(mttc0_tcbind, void, env, tl)
+DEF_HELPER_2(mtc0_tcrestart, void, env, tl)
+DEF_HELPER_2(mttc0_tcrestart, void, env, tl)
+DEF_HELPER_2(mtc0_tchalt, void, env, tl)
+DEF_HELPER_2(mttc0_tchalt, void, env, tl)
+DEF_HELPER_2(mtc0_tccontext, void, env, tl)
+DEF_HELPER_2(mttc0_tccontext, void, env, tl)
+DEF_HELPER_2(mtc0_tcschedule, void, env, tl)
+DEF_HELPER_2(mttc0_tcschedule, void, env, tl)
+DEF_HELPER_2(mtc0_tcschefback, void, env, tl)
+DEF_HELPER_2(mttc0_tcschefback, void, env, tl)
+DEF_HELPER_2(mtc0_entrylo1, void, env, tl)
+DEF_HELPER_2(mtc0_context, void, env, tl)
+DEF_HELPER_2(mtc0_pagemask, void, env, tl)
+DEF_HELPER_2(mtc0_pagegrain, void, env, tl)
+DEF_HELPER_2(mtc0_wired, void, env, tl)
+DEF_HELPER_2(mtc0_srsconf0, void, env, tl)
+DEF_HELPER_2(mtc0_srsconf1, void, env, tl)
+DEF_HELPER_2(mtc0_srsconf2, void, env, tl)
+DEF_HELPER_2(mtc0_srsconf3, void, env, tl)
+DEF_HELPER_2(mtc0_srsconf4, void, env, tl)
+DEF_HELPER_2(mtc0_hwrena, void, env, tl)
+DEF_HELPER_2(mtc0_count, void, env, tl)
+DEF_HELPER_2(mtc0_entryhi, void, env, tl)
+DEF_HELPER_2(mttc0_entryhi, void, env, tl)
+DEF_HELPER_2(mtc0_compare, void, env, tl)
+DEF_HELPER_2(mtc0_status, void, env, tl)
+DEF_HELPER_2(mttc0_status, void, env, tl)
+DEF_HELPER_2(mtc0_intctl, void, env, tl)
+DEF_HELPER_2(mtc0_srsctl, void, env, tl)
+DEF_HELPER_2(mtc0_cause, void, env, tl)
+DEF_HELPER_2(mttc0_cause, void, env, tl)
+DEF_HELPER_2(mtc0_ebase, void, env, tl)
+DEF_HELPER_2(mttc0_ebase, void, env, tl)
+DEF_HELPER_2(mtc0_config0, void, env, tl)
+DEF_HELPER_2(mtc0_config2, void, env, tl)
+DEF_HELPER_2(mtc0_lladdr, void, env, tl)
+DEF_HELPER_3(mtc0_watchlo, void, env, tl, i32)
+DEF_HELPER_3(mtc0_watchhi, void, env, tl, i32)
+DEF_HELPER_2(mtc0_xcontext, void, env, tl)
+DEF_HELPER_2(mtc0_framemask, void, env, tl)
+DEF_HELPER_2(mtc0_debug, void, env, tl)
+DEF_HELPER_2(mttc0_debug, void, env, tl)
+DEF_HELPER_2(mtc0_performance0, void, env, tl)
+DEF_HELPER_2(mtc0_taglo, void, env, tl)
+DEF_HELPER_2(mtc0_datalo, void, env, tl)
+DEF_HELPER_2(mtc0_taghi, void, env, tl)
+DEF_HELPER_2(mtc0_datahi, void, env, tl)
 
 /* MIPS MT functions */
-DEF_HELPER_1(mftgpr, tl, i32);
-DEF_HELPER_1(mftlo, tl, i32)
-DEF_HELPER_1(mfthi, tl, i32)
-DEF_HELPER_1(mftacx, tl, i32)
-DEF_HELPER_0(mftdsp, tl)
-DEF_HELPER_2(mttgpr, void, tl, i32)
-DEF_HELPER_2(mttlo, void, tl, i32)
-DEF_HELPER_2(mtthi, void, tl, i32)
-DEF_HELPER_2(mttacx, void, tl, i32)
-DEF_HELPER_1(mttdsp, void, tl)
+DEF_HELPER_2(mftgpr, tl, env, i32);
+DEF_HELPER_2(mftlo, tl, env, i32)
+DEF_HELPER_2(mfthi, tl, env, i32)
+DEF_HELPER_2(mftacx, tl, env, i32)
+DEF_HELPER_1(mftdsp, tl, env)
+DEF_HELPER_3(mttgpr, void, env, tl, i32)
+DEF_HELPER_3(mttlo, void, env, tl, i32)
+DEF_HELPER_3(mtthi, void, env, tl, i32)
+DEF_HELPER_3(mttacx, void, env, tl, i32)
+DEF_HELPER_2(mttdsp, void, env, tl)
 DEF_HELPER_0(dmt, tl)
 DEF_HELPER_0(emt, tl)
-DEF_HELPER_0(dvpe, tl)
-DEF_HELPER_0(evpe, tl)
+DEF_HELPER_1(dvpe, tl, env)
+DEF_HELPER_1(evpe, tl, env)
 #endif /* !CONFIG_USER_ONLY */
 
 /* microMIPS functions */
-DEF_HELPER_3(lwm, void, tl, tl, i32);
-DEF_HELPER_3(swm, void, tl, tl, i32);
+DEF_HELPER_4(lwm, void, env, tl, tl, i32);
+DEF_HELPER_4(swm, void, env, tl, tl, i32);
 #ifdef TARGET_MIPS64
-DEF_HELPER_3(ldm, void, tl, tl, i32);
-DEF_HELPER_3(sdm, void, tl, tl, i32);
+DEF_HELPER_4(ldm, void, env, tl, tl, i32);
+DEF_HELPER_4(sdm, void, env, tl, tl, i32);
 #endif
 
 DEF_HELPER_2(fork, void, tl, tl)
-DEF_HELPER_1(yield, tl, tl)
+DEF_HELPER_2(yield, tl, env, tl)
 
 /* CP1 functions */
-DEF_HELPER_1(cfc1, tl, i32)
-DEF_HELPER_2(ctc1, void, tl, i32)
-
-DEF_HELPER_1(float_cvtd_s, i64, i32)
-DEF_HELPER_1(float_cvtd_w, i64, i32)
-DEF_HELPER_1(float_cvtd_l, i64, i64)
-DEF_HELPER_1(float_cvtl_d, i64, i64)
-DEF_HELPER_1(float_cvtl_s, i64, i32)
-DEF_HELPER_1(float_cvtps_pw, i64, i64)
-DEF_HELPER_1(float_cvtpw_ps, i64, i64)
-DEF_HELPER_1(float_cvts_d, i32, i64)
-DEF_HELPER_1(float_cvts_w, i32, i32)
-DEF_HELPER_1(float_cvts_l, i32, i64)
-DEF_HELPER_1(float_cvts_pl, i32, i32)
-DEF_HELPER_1(float_cvts_pu, i32, i32)
-DEF_HELPER_1(float_cvtw_s, i32, i32)
-DEF_HELPER_1(float_cvtw_d, i32, i64)
-
-DEF_HELPER_2(float_addr_ps, i64, i64, i64)
-DEF_HELPER_2(float_mulr_ps, i64, i64, i64)
+DEF_HELPER_2(cfc1, tl, env, i32)
+DEF_HELPER_3(ctc1, void, env, tl, i32)
 
-#define FOP_PROTO(op)                       \
-DEF_HELPER_1(float_ ## op ## l_s, i64, i32) \
-DEF_HELPER_1(float_ ## op ## l_d, i64, i64) \
-DEF_HELPER_1(float_ ## op ## w_s, i32, i32) \
-DEF_HELPER_1(float_ ## op ## w_d, i32, i64)
+DEF_HELPER_2(float_cvtd_s, i64, env, i32)
+DEF_HELPER_2(float_cvtd_w, i64, env, i32)
+DEF_HELPER_2(float_cvtd_l, i64, env, i64)
+DEF_HELPER_2(float_cvtl_d, i64, env, i64)
+DEF_HELPER_2(float_cvtl_s, i64, env, i32)
+DEF_HELPER_2(float_cvtps_pw, i64, env, i64)
+DEF_HELPER_2(float_cvtpw_ps, i64, env, i64)
+DEF_HELPER_2(float_cvts_d, i32, env, i64)
+DEF_HELPER_2(float_cvts_w, i32, env, i32)
+DEF_HELPER_2(float_cvts_l, i32, env, i64)
+DEF_HELPER_2(float_cvts_pl, i32, env, i32)
+DEF_HELPER_2(float_cvts_pu, i32, env, i32)
+DEF_HELPER_2(float_cvtw_s, i32, env, i32)
+DEF_HELPER_2(float_cvtw_d, i32, env, i64)
+
+DEF_HELPER_3(float_addr_ps, i64, env, i64, i64)
+DEF_HELPER_3(float_mulr_ps, i64, env, i64, i64)
+
+#define FOP_PROTO(op)                            \
+DEF_HELPER_2(float_ ## op ## l_s, i64, env, i32) \
+DEF_HELPER_2(float_ ## op ## l_d, i64, env, i64) \
+DEF_HELPER_2(float_ ## op ## w_s, i32, env, i32) \
+DEF_HELPER_2(float_ ## op ## w_d, i32, env, i64)
 FOP_PROTO(round)
 FOP_PROTO(trunc)
 FOP_PROTO(ceil)
 FOP_PROTO(floor)
 #undef FOP_PROTO
 
-#define FOP_PROTO(op)                       \
-DEF_HELPER_1(float_ ## op ## _s, i32, i32)  \
-DEF_HELPER_1(float_ ## op ## _d, i64, i64)
+#define FOP_PROTO(op)                            \
+DEF_HELPER_2(float_ ## op ## _s, i32, env, i32)  \
+DEF_HELPER_2(float_ ## op ## _d, i64, env, i64)
 FOP_PROTO(sqrt)
 FOP_PROTO(rsqrt)
 FOP_PROTO(recip)
@@ -228,14 +224,20 @@ DEF_HELPER_1(float_ ## op ## _d, i64, i64)  \
 DEF_HELPER_1(float_ ## op ## _ps, i64, i64)
 FOP_PROTO(abs)
 FOP_PROTO(chs)
+#undef FOP_PROTO
+
+#define FOP_PROTO(op)                            \
+DEF_HELPER_2(float_ ## op ## _s, i32, env, i32)  \
+DEF_HELPER_2(float_ ## op ## _d, i64, env, i64)  \
+DEF_HELPER_2(float_ ## op ## _ps, i64, env, i64)
 FOP_PROTO(recip1)
 FOP_PROTO(rsqrt1)
 #undef FOP_PROTO
 
-#define FOP_PROTO(op)                             \
-DEF_HELPER_2(float_ ## op ## _s, i32, i32, i32)   \
-DEF_HELPER_2(float_ ## op ## _d, i64, i64, i64)   \
-DEF_HELPER_2(float_ ## op ## _ps, i64, i64, i64)
+#define FOP_PROTO(op)                                  \
+DEF_HELPER_3(float_ ## op ## _s, i32, env, i32, i32)   \
+DEF_HELPER_3(float_ ## op ## _d, i64, env, i64, i64)   \
+DEF_HELPER_3(float_ ## op ## _ps, i64, env, i64, i64)
 FOP_PROTO(add)
 FOP_PROTO(sub)
 FOP_PROTO(mul)
@@ -244,23 +246,23 @@ FOP_PROTO(recip2)
 FOP_PROTO(rsqrt2)
 #undef FOP_PROTO
 
-#define FOP_PROTO(op)                                 \
-DEF_HELPER_3(float_ ## op ## _s, i32, i32, i32, i32)  \
-DEF_HELPER_3(float_ ## op ## _d, i64, i64, i64, i64)  \
-DEF_HELPER_3(float_ ## op ## _ps, i64, i64, i64, i64)
-FOP_PROTO(muladd)
-FOP_PROTO(mulsub)
-FOP_PROTO(nmuladd)
-FOP_PROTO(nmulsub)
+#define FOP_PROTO(op)                                      \
+DEF_HELPER_4(float_ ## op ## _s, i32, env, i32, i32, i32)  \
+DEF_HELPER_4(float_ ## op ## _d, i64, env, i64, i64, i64)  \
+DEF_HELPER_4(float_ ## op ## _ps, i64, env, i64, i64, i64)
+FOP_PROTO(madd)
+FOP_PROTO(msub)
+FOP_PROTO(nmadd)
+FOP_PROTO(nmsub)
 #undef FOP_PROTO
 
-#define FOP_PROTO(op)                               \
-DEF_HELPER_3(cmp_d_ ## op, void, i64, i64, int)     \
-DEF_HELPER_3(cmpabs_d_ ## op, void, i64, i64, int)  \
-DEF_HELPER_3(cmp_s_ ## op, void, i32, i32, int)     \
-DEF_HELPER_3(cmpabs_s_ ## op, void, i32, i32, int)  \
-DEF_HELPER_3(cmp_ps_ ## op, void, i64, i64, int)    \
-DEF_HELPER_3(cmpabs_ps_ ## op, void, i64, i64, int)
+#define FOP_PROTO(op)                                    \
+DEF_HELPER_4(cmp_d_ ## op, void, env, i64, i64, int)     \
+DEF_HELPER_4(cmpabs_d_ ## op, void, env, i64, i64, int)  \
+DEF_HELPER_4(cmp_s_ ## op, void, env, i32, i32, int)     \
+DEF_HELPER_4(cmpabs_s_ ## op, void, env, i32, i32, int)  \
+DEF_HELPER_4(cmp_ps_ ## op, void, env, i64, i64, int)    \
+DEF_HELPER_4(cmpabs_ps_ ## op, void, env, i64, i64, int)
 FOP_PROTO(f)
 FOP_PROTO(un)
 FOP_PROTO(eq)
@@ -281,20 +283,428 @@ FOP_PROTO(ngt)
 
 /* Special functions */
 #ifndef CONFIG_USER_ONLY
-DEF_HELPER_0(tlbwi, void)
-DEF_HELPER_0(tlbwr, void)
-DEF_HELPER_0(tlbp, void)
-DEF_HELPER_0(tlbr, void)
-DEF_HELPER_0(di, tl)
-DEF_HELPER_0(ei, tl)
-DEF_HELPER_0(eret, void)
-DEF_HELPER_0(deret, void)
+DEF_HELPER_1(tlbwi, void, env)
+DEF_HELPER_1(tlbwr, void, env)
+DEF_HELPER_1(tlbp, void, env)
+DEF_HELPER_1(tlbr, void, env)
+DEF_HELPER_1(di, tl, env)
+DEF_HELPER_1(ei, tl, env)
+DEF_HELPER_1(eret, void, env)
+DEF_HELPER_1(deret, void, env)
 #endif /* !CONFIG_USER_ONLY */
-DEF_HELPER_0(rdhwr_cpunum, tl)
-DEF_HELPER_0(rdhwr_synci_step, tl)
-DEF_HELPER_0(rdhwr_cc, tl)
-DEF_HELPER_0(rdhwr_ccres, tl)
-DEF_HELPER_1(pmon, void, int)
-DEF_HELPER_0(wait, void)
+DEF_HELPER_1(rdhwr_cpunum, tl, env)
+DEF_HELPER_1(rdhwr_synci_step, tl, env)
+DEF_HELPER_1(rdhwr_cc, tl, env)
+DEF_HELPER_1(rdhwr_ccres, tl, env)
+DEF_HELPER_2(pmon, void, env, int)
+DEF_HELPER_1(wait, void, env)
+
+/* Loongson multimedia functions.  */
+DEF_HELPER_FLAGS_2(paddsh, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(paddush, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(paddh, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(paddw, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(paddsb, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(paddusb, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(paddb, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+
+DEF_HELPER_FLAGS_2(psubsh, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(psubush, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(psubh, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(psubw, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(psubsb, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(psubusb, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(psubb, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+
+DEF_HELPER_FLAGS_2(pshufh, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(packsswh, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(packsshb, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(packushb, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+
+DEF_HELPER_FLAGS_2(punpcklhw, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(punpckhhw, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(punpcklbh, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(punpckhbh, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(punpcklwd, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(punpckhwd, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+
+DEF_HELPER_FLAGS_2(pavgh, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(pavgb, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(pmaxsh, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(pminsh, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(pmaxub, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(pminub, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+
+DEF_HELPER_FLAGS_2(pcmpeqw, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(pcmpgtw, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(pcmpeqh, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(pcmpgth, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(pcmpeqb, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(pcmpgtb, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+
+DEF_HELPER_FLAGS_2(psllw, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(psllh, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(psrlw, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(psrlh, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(psraw, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(psrah, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+
+DEF_HELPER_FLAGS_2(pmullh, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(pmulhh, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(pmulhuh, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(pmaddhw, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+
+DEF_HELPER_FLAGS_2(pasubub, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_1(biadd, TCG_CALL_NO_RWG_SE, i64, i64)
+DEF_HELPER_FLAGS_1(pmovmskb, TCG_CALL_NO_RWG_SE, i64, i64)
+
+/*** MIPS DSP ***/
+/* DSP Arithmetic Sub-class insns */
+DEF_HELPER_FLAGS_3(addq_ph, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_3(addq_s_ph, 0, tl, tl, tl, env)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_3(addq_qh, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_3(addq_s_qh, 0, tl, tl, tl, env)
+#endif
+DEF_HELPER_FLAGS_3(addq_s_w, 0, tl, tl, tl, env)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_3(addq_pw, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_3(addq_s_pw, 0, tl, tl, tl, env)
+#endif
+DEF_HELPER_FLAGS_3(addu_qb, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_3(addu_s_qb, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_2(adduh_qb, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_2(adduh_r_qb, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_3(addu_ph, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_3(addu_s_ph, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_2(addqh_ph, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_2(addqh_r_ph, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_2(addqh_w, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_2(addqh_r_w, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_3(addu_ob, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_3(addu_s_ob, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_2(adduh_ob, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_2(adduh_r_ob, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_3(addu_qh, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_3(addu_s_qh, 0, tl, tl, tl, env)
+#endif
+DEF_HELPER_FLAGS_3(subq_ph, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_3(subq_s_ph, 0, tl, tl, tl, env)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_3(subq_qh, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_3(subq_s_qh, 0, tl, tl, tl, env)
+#endif
+DEF_HELPER_FLAGS_3(subq_s_w, 0, tl, tl, tl, env)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_3(subq_pw, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_3(subq_s_pw, 0, tl, tl, tl, env)
+#endif
+DEF_HELPER_FLAGS_3(subu_qb, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_3(subu_s_qb, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_2(subuh_qb, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_2(subuh_r_qb, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_3(subu_ph, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_3(subu_s_ph, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_2(subqh_ph, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_2(subqh_r_ph, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_2(subqh_w, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_2(subqh_r_w, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_3(subu_ob, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_3(subu_s_ob, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_2(subuh_ob, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_2(subuh_r_ob, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_3(subu_qh, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_3(subu_s_qh, 0, tl, tl, tl, env)
+#endif
+DEF_HELPER_FLAGS_3(addsc, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_3(addwc, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_2(modsub, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_1(raddu_w_qb, TCG_CALL_NO_RWG_SE, tl, tl)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_1(raddu_l_ob, TCG_CALL_NO_RWG_SE, tl, tl)
+#endif
+DEF_HELPER_FLAGS_2(absq_s_qb, 0, tl, tl, env)
+DEF_HELPER_FLAGS_2(absq_s_ph, 0, tl, tl, env)
+DEF_HELPER_FLAGS_2(absq_s_w, 0, tl, tl, env)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_2(absq_s_ob, 0, tl, tl, env)
+DEF_HELPER_FLAGS_2(absq_s_qh, 0, tl, tl, env)
+DEF_HELPER_FLAGS_2(absq_s_pw, 0, tl, tl, env)
+#endif
+DEF_HELPER_FLAGS_2(precr_qb_ph, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_2(precrq_qb_ph, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_3(precr_sra_ph_w, TCG_CALL_NO_RWG_SE,
+                   tl, i32, tl, tl)
+DEF_HELPER_FLAGS_3(precr_sra_r_ph_w, TCG_CALL_NO_RWG_SE,
+                   tl, i32, tl, tl)
+DEF_HELPER_FLAGS_2(precrq_ph_w, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_3(precrq_rs_ph_w, 0, tl, tl, tl, env)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_2(precr_ob_qh, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_3(precr_sra_qh_pw,
+                   TCG_CALL_NO_RWG_SE, tl, tl, tl, i32)
+DEF_HELPER_FLAGS_3(precr_sra_r_qh_pw,
+                   TCG_CALL_NO_RWG_SE, tl, tl, tl, i32)
+DEF_HELPER_FLAGS_2(precrq_ob_qh, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_2(precrq_qh_pw, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_3(precrq_rs_qh_pw,
+                   TCG_CALL_NO_RWG_SE, tl, tl, tl, env)
+DEF_HELPER_FLAGS_2(precrq_pw_l, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+#endif
+DEF_HELPER_FLAGS_3(precrqu_s_qb_ph, 0, tl, tl, tl, env)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_3(precrqu_s_ob_qh,
+                   TCG_CALL_NO_RWG_SE, tl, tl, tl, env)
+
+DEF_HELPER_FLAGS_1(preceq_pw_qhl, TCG_CALL_NO_RWG_SE, tl, tl)
+DEF_HELPER_FLAGS_1(preceq_pw_qhr, TCG_CALL_NO_RWG_SE, tl, tl)
+DEF_HELPER_FLAGS_1(preceq_pw_qhla, TCG_CALL_NO_RWG_SE, tl, tl)
+DEF_HELPER_FLAGS_1(preceq_pw_qhra, TCG_CALL_NO_RWG_SE, tl, tl)
+#endif
+DEF_HELPER_FLAGS_1(precequ_ph_qbl, TCG_CALL_NO_RWG_SE, tl, tl)
+DEF_HELPER_FLAGS_1(precequ_ph_qbr, TCG_CALL_NO_RWG_SE, tl, tl)
+DEF_HELPER_FLAGS_1(precequ_ph_qbla, TCG_CALL_NO_RWG_SE, tl, tl)
+DEF_HELPER_FLAGS_1(precequ_ph_qbra, TCG_CALL_NO_RWG_SE, tl, tl)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_1(precequ_qh_obl, TCG_CALL_NO_RWG_SE, tl, tl)
+DEF_HELPER_FLAGS_1(precequ_qh_obr, TCG_CALL_NO_RWG_SE, tl, tl)
+DEF_HELPER_FLAGS_1(precequ_qh_obla, TCG_CALL_NO_RWG_SE, tl, tl)
+DEF_HELPER_FLAGS_1(precequ_qh_obra, TCG_CALL_NO_RWG_SE, tl, tl)
+#endif
+DEF_HELPER_FLAGS_1(preceu_ph_qbl, TCG_CALL_NO_RWG_SE, tl, tl)
+DEF_HELPER_FLAGS_1(preceu_ph_qbr, TCG_CALL_NO_RWG_SE, tl, tl)
+DEF_HELPER_FLAGS_1(preceu_ph_qbla, TCG_CALL_NO_RWG_SE, tl, tl)
+DEF_HELPER_FLAGS_1(preceu_ph_qbra, TCG_CALL_NO_RWG_SE, tl, tl)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_1(preceu_qh_obl, TCG_CALL_NO_RWG_SE, tl, tl)
+DEF_HELPER_FLAGS_1(preceu_qh_obr, TCG_CALL_NO_RWG_SE, tl, tl)
+DEF_HELPER_FLAGS_1(preceu_qh_obla, TCG_CALL_NO_RWG_SE, tl, tl)
+DEF_HELPER_FLAGS_1(preceu_qh_obra, TCG_CALL_NO_RWG_SE, tl, tl)
+#endif
+
+/* DSP GPR-Based Shift Sub-class insns */
+DEF_HELPER_FLAGS_3(shll_qb, 0, tl, tl, tl, env)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_3(shll_ob, 0, tl, tl, tl, env)
+#endif
+DEF_HELPER_FLAGS_3(shll_ph, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_3(shll_s_ph, 0, tl, tl, tl, env)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_3(shll_qh, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_3(shll_s_qh, 0, tl, tl, tl, env)
+#endif
+DEF_HELPER_FLAGS_3(shll_s_w, 0, tl, tl, tl, env)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_3(shll_pw, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_3(shll_s_pw, 0, tl, tl, tl, env)
+#endif
+DEF_HELPER_FLAGS_2(shrl_qb, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_2(shrl_ph, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_2(shrl_ob, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_2(shrl_qh, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+#endif
+DEF_HELPER_FLAGS_2(shra_qb, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_2(shra_r_qb, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_2(shra_ob, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_2(shra_r_ob, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+#endif
+DEF_HELPER_FLAGS_2(shra_ph, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_2(shra_r_ph, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_2(shra_r_w, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_2(shra_qh, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_2(shra_r_qh, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_2(shra_pw, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_2(shra_r_pw, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+#endif
+
+/* DSP Multiply Sub-class insns */
+DEF_HELPER_FLAGS_3(muleu_s_ph_qbl, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_3(muleu_s_ph_qbr, 0, tl, tl, tl, env)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_3(muleu_s_qh_obl, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_3(muleu_s_qh_obr, 0, tl, tl, tl, env)
+#endif
+DEF_HELPER_FLAGS_3(mulq_rs_ph, 0, tl, tl, tl, env)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_3(mulq_rs_qh, 0, tl, tl, tl, env)
+#endif
+DEF_HELPER_FLAGS_3(muleq_s_w_phl, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_3(muleq_s_w_phr, 0, tl, tl, tl, env)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_3(muleq_s_pw_qhl, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_3(muleq_s_pw_qhr, 0, tl, tl, tl, env)
+#endif
+DEF_HELPER_FLAGS_4(dpau_h_qbl, 0, void, i32, tl, tl, env)
+DEF_HELPER_FLAGS_4(dpau_h_qbr, 0, void, i32, tl, tl, env)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_4(dpau_h_obl, 0, void, tl, tl, i32, env)
+DEF_HELPER_FLAGS_4(dpau_h_obr, 0, void, tl, tl, i32, env)
+#endif
+DEF_HELPER_FLAGS_4(dpsu_h_qbl, 0, void, i32, tl, tl, env)
+DEF_HELPER_FLAGS_4(dpsu_h_qbr, 0, void, i32, tl, tl, env)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_4(dpsu_h_obl, 0, void, tl, tl, i32, env)
+DEF_HELPER_FLAGS_4(dpsu_h_obr, 0, void, tl, tl, i32, env)
+#endif
+DEF_HELPER_FLAGS_4(dpa_w_ph, 0, void, i32, tl, tl, env)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_4(dpa_w_qh, 0, void, tl, tl, i32, env)
+#endif
+DEF_HELPER_FLAGS_4(dpax_w_ph, 0, void, i32, tl, tl, env)
+DEF_HELPER_FLAGS_4(dpaq_s_w_ph, 0, void, i32, tl, tl, env)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_4(dpaq_s_w_qh, 0, void, tl, tl, i32, env)
+#endif
+DEF_HELPER_FLAGS_4(dpaqx_s_w_ph, 0, void, i32, tl, tl, env)
+DEF_HELPER_FLAGS_4(dpaqx_sa_w_ph, 0, void, i32, tl, tl, env)
+DEF_HELPER_FLAGS_4(dps_w_ph, 0, void, i32, tl, tl, env)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_4(dps_w_qh, 0, void, tl, tl, i32, env)
+#endif
+DEF_HELPER_FLAGS_4(dpsx_w_ph, 0, void, i32, tl, tl, env)
+DEF_HELPER_FLAGS_4(dpsq_s_w_ph, 0, void, i32, tl, tl, env)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_4(dpsq_s_w_qh, 0, void, tl, tl, i32, env)
+#endif
+DEF_HELPER_FLAGS_4(dpsqx_s_w_ph, 0, void, i32, tl, tl, env)
+DEF_HELPER_FLAGS_4(dpsqx_sa_w_ph, 0, void, i32, tl, tl, env)
+DEF_HELPER_FLAGS_4(mulsaq_s_w_ph, 0, void, i32, tl, tl, env)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_4(mulsaq_s_w_qh, 0, void, tl, tl, i32, env)
+#endif
+DEF_HELPER_FLAGS_4(dpaq_sa_l_w, 0, void, i32, tl, tl, env)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_4(dpaq_sa_l_pw, 0, void, tl, tl, i32, env)
+#endif
+DEF_HELPER_FLAGS_4(dpsq_sa_l_w, 0, void, i32, tl, tl, env)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_4(dpsq_sa_l_pw, 0, void, tl, tl, i32, env)
+DEF_HELPER_FLAGS_4(mulsaq_s_l_pw, 0, void, tl, tl, i32, env)
+#endif
+DEF_HELPER_FLAGS_4(maq_s_w_phl, 0, void, i32, tl, tl, env)
+DEF_HELPER_FLAGS_4(maq_s_w_phr, 0, void, i32, tl, tl, env)
+DEF_HELPER_FLAGS_4(maq_sa_w_phl, 0, void, i32, tl, tl, env)
+DEF_HELPER_FLAGS_4(maq_sa_w_phr, 0, void, i32, tl, tl, env)
+DEF_HELPER_FLAGS_3(mul_ph, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_3(mul_s_ph, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_3(mulq_s_ph, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_3(mulq_s_w, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_3(mulq_rs_w, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_4(mulsa_w_ph, 0, void, i32, tl, tl, env)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_4(maq_s_w_qhll, 0, void, tl, tl, i32, env)
+DEF_HELPER_FLAGS_4(maq_s_w_qhlr, 0, void, tl, tl, i32, env)
+DEF_HELPER_FLAGS_4(maq_s_w_qhrl, 0, void, tl, tl, i32, env)
+DEF_HELPER_FLAGS_4(maq_s_w_qhrr, 0, void, tl, tl, i32, env)
+DEF_HELPER_FLAGS_4(maq_sa_w_qhll, 0, void, tl, tl, i32, env)
+DEF_HELPER_FLAGS_4(maq_sa_w_qhlr, 0, void, tl, tl, i32, env)
+DEF_HELPER_FLAGS_4(maq_sa_w_qhrl, 0, void, tl, tl, i32, env)
+DEF_HELPER_FLAGS_4(maq_sa_w_qhrr, 0, void, tl, tl, i32, env)
+DEF_HELPER_FLAGS_4(maq_s_l_pwl, 0, void, tl, tl, i32, env)
+DEF_HELPER_FLAGS_4(maq_s_l_pwr, 0, void, tl, tl, i32, env)
+DEF_HELPER_FLAGS_4(dmadd, 0, void, tl, tl, i32, env)
+DEF_HELPER_FLAGS_4(dmaddu, 0, void, tl, tl, i32, env)
+DEF_HELPER_FLAGS_4(dmsub, 0, void, tl, tl, i32, env)
+DEF_HELPER_FLAGS_4(dmsubu, 0, void, tl, tl, i32, env)
+#endif
+
+/* DSP Bit/Manipulation Sub-class insns */
+DEF_HELPER_FLAGS_1(bitrev, TCG_CALL_NO_RWG_SE, tl, tl)
+DEF_HELPER_FLAGS_3(insv, 0, tl, env, tl, tl)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_3(dinsv, 0, tl, env, tl, tl);
+#endif
+
+/* DSP Compare-Pick Sub-class insns */
+DEF_HELPER_FLAGS_3(cmpu_eq_qb, 0, void, tl, tl, env)
+DEF_HELPER_FLAGS_3(cmpu_lt_qb, 0, void, tl, tl, env)
+DEF_HELPER_FLAGS_3(cmpu_le_qb, 0, void, tl, tl, env)
+DEF_HELPER_FLAGS_2(cmpgu_eq_qb, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_2(cmpgu_lt_qb, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_2(cmpgu_le_qb, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_3(cmp_eq_ph, 0, void, tl, tl, env)
+DEF_HELPER_FLAGS_3(cmp_lt_ph, 0, void, tl, tl, env)
+DEF_HELPER_FLAGS_3(cmp_le_ph, 0, void, tl, tl, env)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_3(cmpu_eq_ob, 0, void, tl, tl, env)
+DEF_HELPER_FLAGS_3(cmpu_lt_ob, 0, void, tl, tl, env)
+DEF_HELPER_FLAGS_3(cmpu_le_ob, 0, void, tl, tl, env)
+DEF_HELPER_FLAGS_3(cmpgdu_eq_ob, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_3(cmpgdu_lt_ob, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_3(cmpgdu_le_ob, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_2(cmpgu_eq_ob, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_2(cmpgu_lt_ob, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_2(cmpgu_le_ob, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+DEF_HELPER_FLAGS_3(cmp_eq_qh, 0, void, tl, tl, env)
+DEF_HELPER_FLAGS_3(cmp_lt_qh, 0, void, tl, tl, env)
+DEF_HELPER_FLAGS_3(cmp_le_qh, 0, void, tl, tl, env)
+DEF_HELPER_FLAGS_3(cmp_eq_pw, 0, void, tl, tl, env)
+DEF_HELPER_FLAGS_3(cmp_lt_pw, 0, void, tl, tl, env)
+DEF_HELPER_FLAGS_3(cmp_le_pw, 0, void, tl, tl, env)
+#endif
+DEF_HELPER_FLAGS_3(pick_qb, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_3(pick_ph, 0, tl, tl, tl, env)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_3(pick_ob, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_3(pick_qh, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_3(pick_pw, 0, tl, tl, tl, env)
+#endif
+DEF_HELPER_FLAGS_3(append, TCG_CALL_NO_RWG_SE, tl, tl, tl, i32)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_3(dappend, TCG_CALL_NO_RWG_SE, tl, tl, tl, i32)
+#endif
+DEF_HELPER_FLAGS_3(prepend, TCG_CALL_NO_RWG_SE, tl, tl, tl, i32)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_3(prependd, TCG_CALL_NO_RWG_SE, tl, tl, tl, i32)
+DEF_HELPER_FLAGS_3(prependw, TCG_CALL_NO_RWG_SE, tl, tl, tl, i32)
+#endif
+DEF_HELPER_FLAGS_3(balign, TCG_CALL_NO_RWG_SE, tl, tl, tl, i32)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_3(dbalign, TCG_CALL_NO_RWG_SE, tl, tl, tl, i32)
+#endif
+DEF_HELPER_FLAGS_2(packrl_ph, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_2(packrl_pw, TCG_CALL_NO_RWG_SE, tl, tl, tl)
+#endif
+
+/* DSP Accumulator and DSPControl Access Sub-class insns */
+DEF_HELPER_FLAGS_3(extr_w, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_3(extr_r_w, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_3(extr_rs_w, 0, tl, tl, tl, env)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_3(dextr_w, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_3(dextr_r_w, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_3(dextr_rs_w, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_3(dextr_l, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_3(dextr_r_l, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_3(dextr_rs_l, 0, tl, tl, tl, env)
+#endif
+DEF_HELPER_FLAGS_3(extr_s_h, 0, tl, tl, tl, env)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_3(dextr_s_h, 0, tl, tl, tl, env)
+#endif
+DEF_HELPER_FLAGS_3(extp, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_3(extpdp, 0, tl, tl, tl, env)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_3(dextp, 0, tl, tl, tl, env)
+DEF_HELPER_FLAGS_3(dextpdp, 0, tl, tl, tl, env)
+#endif
+DEF_HELPER_FLAGS_3(shilo, 0, void, tl, tl, env)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_3(dshilo, 0, void, tl, tl, env)
+#endif
+DEF_HELPER_FLAGS_3(mthlip, 0, void, tl, tl, env)
+#if defined(TARGET_MIPS64)
+DEF_HELPER_FLAGS_3(dmthlip, 0, void, tl, tl, env)
+#endif
+DEF_HELPER_FLAGS_3(wrdsp, 0, void, tl, tl, env)
+DEF_HELPER_FLAGS_2(rddsp, 0, tl, tl, env)
+
+
 
 #include "def-helper.h"
diff --git a/target-mips/lmi_helper.c b/target-mips/lmi_helper.c
new file mode 100644
index 000000000..1b2435351
--- /dev/null
+++ b/target-mips/lmi_helper.c
@@ -0,0 +1,744 @@
+/*
+ *  Loongson Multimedia Instruction emulation helpers for QEMU.
+ *
+ *  Copyright (c) 2011  Richard Henderson <rth@twiddle.net>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "cpu.h"
+#include "helper.h"
+
+/* If the byte ordering doesn't matter, i.e. all columns are treated
+   identically, then this union can be used directly.  If byte ordering
+   does matter, we generally ignore dumping to memory.  */
+typedef union {
+    uint8_t  ub[8];
+    int8_t   sb[8];
+    uint16_t uh[4];
+    int16_t  sh[4];
+    uint32_t uw[2];
+    int32_t  sw[2];
+    uint64_t d;
+} LMIValue;
+
+/* Some byte ordering issues can be mitigated by XORing in the following.  */
+#ifdef HOST_WORDS_BIGENDIAN
+# define BYTE_ORDER_XOR(N) N
+#else
+# define BYTE_ORDER_XOR(N) 0
+#endif
+
+#define SATSB(x)  (x < -0x80 ? -0x80 : x > 0x7f ? 0x7f : x)
+#define SATUB(x)  (x > 0xff ? 0xff : x)
+
+#define SATSH(x)  (x < -0x8000 ? -0x8000 : x > 0x7fff ? 0x7fff : x)
+#define SATUH(x)  (x > 0xffff ? 0xffff : x)
+
+#define SATSW(x) \
+    (x < -0x80000000ll ? -0x80000000ll : x > 0x7fffffff ? 0x7fffffff : x)
+#define SATUW(x)  (x > 0xffffffffull ? 0xffffffffull : x)
+
+uint64_t helper_paddsb(uint64_t fs, uint64_t ft)
+{
+    LMIValue vs, vt;
+    unsigned int i;
+
+    vs.d = fs;
+    vt.d = ft;
+    for (i = 0; i < 8; ++i) {
+        int r = vs.sb[i] + vt.sb[i];
+        vs.sb[i] = SATSB(r);
+    }
+    return vs.d;
+}
+
+uint64_t helper_paddusb(uint64_t fs, uint64_t ft)
+{
+    LMIValue vs, vt;
+    unsigned int i;
+
+    vs.d = fs;
+    vt.d = ft;
+    for (i = 0; i < 8; ++i) {
+        int r = vs.ub[i] + vt.ub[i];
+        vs.ub[i] = SATUB(r);
+    }
+    return vs.d;
+}
+
+uint64_t helper_paddsh(uint64_t fs, uint64_t ft)
+{
+    LMIValue vs, vt;
+    unsigned int i;
+
+    vs.d = fs;
+    vt.d = ft;
+    for (i = 0; i < 4; ++i) {
+        int r = vs.sh[i] + vt.sh[i];
+        vs.sh[i] = SATSH(r);
+    }
+    return vs.d;
+}
+
+uint64_t helper_paddush(uint64_t fs, uint64_t ft)
+{
+    LMIValue vs, vt;
+    unsigned int i;
+
+    vs.d = fs;
+    vt.d = ft;
+    for (i = 0; i < 4; ++i) {
+        int r = vs.uh[i] + vt.uh[i];
+        vs.uh[i] = SATUH(r);
+    }
+    return vs.d;
+}
+
+uint64_t helper_paddb(uint64_t fs, uint64_t ft)
+{
+    LMIValue vs, vt;
+    unsigned int i;
+
+    vs.d = fs;
+    vt.d = ft;
+    for (i = 0; i < 8; ++i) {
+        vs.ub[i] += vt.ub[i];
+    }
+    return vs.d;
+}
+
+uint64_t helper_paddh(uint64_t fs, uint64_t ft)
+{
+    LMIValue vs, vt;
+    unsigned int i;
+
+    vs.d = fs;
+    vt.d = ft;
+    for (i = 0; i < 4; ++i) {
+        vs.uh[i] += vt.uh[i];
+    }
+    return vs.d;
+}
+
+uint64_t helper_paddw(uint64_t fs, uint64_t ft)
+{
+    LMIValue vs, vt;
+    unsigned int i;
+
+    vs.d = fs;
+    vt.d = ft;
+    for (i = 0; i < 2; ++i) {
+        vs.uw[i] += vt.uw[i];
+    }
+    return vs.d;
+}
+
+uint64_t helper_psubsb(uint64_t fs, uint64_t ft)
+{
+    LMIValue vs, vt;
+    unsigned int i;
+
+    vs.d = fs;
+    vt.d = ft;
+    for (i = 0; i < 8; ++i) {
+        int r = vs.sb[i] - vt.sb[i];
+        vs.sb[i] = SATSB(r);
+    }
+    return vs.d;
+}
+
+uint64_t helper_psubusb(uint64_t fs, uint64_t ft)
+{
+    LMIValue vs, vt;
+    unsigned int i;
+
+    vs.d = fs;
+    vt.d = ft;
+    for (i = 0; i < 8; ++i) {
+        int r = vs.ub[i] - vt.ub[i];
+        vs.ub[i] = SATUB(r);
+    }
+    return vs.d;
+}
+
+uint64_t helper_psubsh(uint64_t fs, uint64_t ft)
+{
+    LMIValue vs, vt;
+    unsigned int i;
+
+    vs.d = fs;
+    vt.d = ft;
+    for (i = 0; i < 4; ++i) {
+        int r = vs.sh[i] - vt.sh[i];
+        vs.sh[i] = SATSH(r);
+    }
+    return vs.d;
+}
+
+uint64_t helper_psubush(uint64_t fs, uint64_t ft)
+{
+    LMIValue vs, vt;
+    unsigned int i;
+
+    vs.d = fs;
+    vt.d = ft;
+    for (i = 0; i < 4; ++i) {
+        int r = vs.uh[i] - vt.uh[i];
+        vs.uh[i] = SATUH(r);
+    }
+    return vs.d;
+}
+
+uint64_t helper_psubb(uint64_t fs, uint64_t ft)
+{
+    LMIValue vs, vt;
+    unsigned int i;
+
+    vs.d = fs;
+    vt.d = ft;
+    for (i = 0; i < 8; ++i) {
+        vs.ub[i] -= vt.ub[i];
+    }
+    return vs.d;
+}
+
+uint64_t helper_psubh(uint64_t fs, uint64_t ft)
+{
+    LMIValue vs, vt;
+    unsigned int i;
+
+    vs.d = fs;
+    vt.d = ft;
+    for (i = 0; i < 4; ++i) {
+        vs.uh[i] -= vt.uh[i];
+    }
+    return vs.d;
+}
+
+uint64_t helper_psubw(uint64_t fs, uint64_t ft)
+{
+    LMIValue vs, vt;
+    unsigned int i;
+
+    vs.d = fs;
+    vt.d = ft;
+    for (i = 0; i < 2; ++i) {
+        vs.uw[i] -= vt.uw[i];
+    }
+    return vs.d;
+}
+
+uint64_t helper_pshufh(uint64_t fs, uint64_t ft)
+{
+    unsigned host = BYTE_ORDER_XOR(3);
+    LMIValue vd, vs;
+    unsigned i;
+
+    vs.d = fs;
+    vd.d = 0;
+    for (i = 0; i < 4; i++, ft >>= 2) {
+        vd.uh[i ^ host] = vs.uh[(ft & 3) ^ host];
+    }
+    return vd.d;
+}
+
+uint64_t helper_packsswh(uint64_t fs, uint64_t ft)
+{
+    uint64_t fd = 0;
+    int64_t tmp;
+
+    tmp = (int32_t)(fs >> 0);
+    tmp = SATSH(tmp);
+    fd |= (tmp & 0xffff) << 0;
+
+    tmp = (int32_t)(fs >> 32);
+    tmp = SATSH(tmp);
+    fd |= (tmp & 0xffff) << 16;
+
+    tmp = (int32_t)(ft >> 0);
+    tmp = SATSH(tmp);
+    fd |= (tmp & 0xffff) << 32;
+
+    tmp = (int32_t)(ft >> 32);
+    tmp = SATSH(tmp);
+    fd |= (tmp & 0xffff) << 48;
+
+    return fd;
+}
+
+uint64_t helper_packsshb(uint64_t fs, uint64_t ft)
+{
+    uint64_t fd = 0;
+    unsigned int i;
+
+    for (i = 0; i < 4; ++i) {
+        int16_t tmp = fs >> (i * 16);
+        tmp = SATSB(tmp);
+        fd |= (uint64_t)(tmp & 0xff) << (i * 8);
+    }
+    for (i = 0; i < 4; ++i) {
+        int16_t tmp = ft >> (i * 16);
+        tmp = SATSB(tmp);
+        fd |= (uint64_t)(tmp & 0xff) << (i * 8 + 32);
+    }
+
+    return fd;
+}
+
+uint64_t helper_packushb(uint64_t fs, uint64_t ft)
+{
+    uint64_t fd = 0;
+    unsigned int i;
+
+    for (i = 0; i < 4; ++i) {
+        int16_t tmp = fs >> (i * 16);
+        tmp = SATUB(tmp);
+        fd |= (uint64_t)(tmp & 0xff) << (i * 8);
+    }
+    for (i = 0; i < 4; ++i) {
+        int16_t tmp = ft >> (i * 16);
+        tmp = SATUB(tmp);
+        fd |= (uint64_t)(tmp & 0xff) << (i * 8 + 32);
+    }
+
+    return fd;
+}
+
+uint64_t helper_punpcklwd(uint64_t fs, uint64_t ft)
+{
+    return (fs & 0xffffffff) | (ft << 32);
+}
+
+uint64_t helper_punpckhwd(uint64_t fs, uint64_t ft)
+{
+    return (fs >> 32) | (ft & ~0xffffffffull);
+}
+
+uint64_t helper_punpcklhw(uint64_t fs, uint64_t ft)
+{
+    unsigned host = BYTE_ORDER_XOR(3);
+    LMIValue vd, vs, vt;
+
+    vs.d = fs;
+    vt.d = ft;
+    vd.uh[0 ^ host] = vs.uh[0 ^ host];
+    vd.uh[1 ^ host] = vt.uh[0 ^ host];
+    vd.uh[2 ^ host] = vs.uh[1 ^ host];
+    vd.uh[3 ^ host] = vt.uh[1 ^ host];
+
+    return vd.d;
+}
+
+uint64_t helper_punpckhhw(uint64_t fs, uint64_t ft)
+{
+    unsigned host = BYTE_ORDER_XOR(3);
+    LMIValue vd, vs, vt;
+
+    vs.d = fs;
+    vt.d = ft;
+    vd.uh[0 ^ host] = vs.uh[2 ^ host];
+    vd.uh[1 ^ host] = vt.uh[2 ^ host];
+    vd.uh[2 ^ host] = vs.uh[3 ^ host];
+    vd.uh[3 ^ host] = vt.uh[3 ^ host];
+
+    return vd.d;
+}
+
+uint64_t helper_punpcklbh(uint64_t fs, uint64_t ft)
+{
+    unsigned host = BYTE_ORDER_XOR(7);
+    LMIValue vd, vs, vt;
+
+    vs.d = fs;
+    vt.d = ft;
+    vd.ub[0 ^ host] = vs.ub[0 ^ host];
+    vd.ub[1 ^ host] = vt.ub[0 ^ host];
+    vd.ub[2 ^ host] = vs.ub[1 ^ host];
+    vd.ub[3 ^ host] = vt.ub[1 ^ host];
+    vd.ub[4 ^ host] = vs.ub[2 ^ host];
+    vd.ub[5 ^ host] = vt.ub[2 ^ host];
+    vd.ub[6 ^ host] = vs.ub[3 ^ host];
+    vd.ub[7 ^ host] = vt.ub[3 ^ host];
+
+    return vd.d;
+}
+
+uint64_t helper_punpckhbh(uint64_t fs, uint64_t ft)
+{
+    unsigned host = BYTE_ORDER_XOR(7);
+    LMIValue vd, vs, vt;
+
+    vs.d = fs;
+    vt.d = ft;
+    vd.ub[0 ^ host] = vs.ub[4 ^ host];
+    vd.ub[1 ^ host] = vt.ub[4 ^ host];
+    vd.ub[2 ^ host] = vs.ub[5 ^ host];
+    vd.ub[3 ^ host] = vt.ub[5 ^ host];
+    vd.ub[4 ^ host] = vs.ub[6 ^ host];
+    vd.ub[5 ^ host] = vt.ub[6 ^ host];
+    vd.ub[6 ^ host] = vs.ub[7 ^ host];
+    vd.ub[7 ^ host] = vt.ub[7 ^ host];
+
+    return vd.d;
+}
+
+uint64_t helper_pavgh(uint64_t fs, uint64_t ft)
+{
+    LMIValue vs, vt;
+    unsigned i;
+
+    vs.d = fs;
+    vt.d = ft;
+    for (i = 0; i < 4; i++) {
+        vs.uh[i] = (vs.uh[i] + vt.uh[i] + 1) >> 1;
+    }
+    return vs.d;
+}
+
+uint64_t helper_pavgb(uint64_t fs, uint64_t ft)
+{
+    LMIValue vs, vt;
+    unsigned i;
+
+    vs.d = fs;
+    vt.d = ft;
+    for (i = 0; i < 8; i++) {
+        vs.ub[i] = (vs.ub[i] + vt.ub[i] + 1) >> 1;
+    }
+    return vs.d;
+}
+
+uint64_t helper_pmaxsh(uint64_t fs, uint64_t ft)
+{
+    LMIValue vs, vt;
+    unsigned i;
+
+    vs.d = fs;
+    vt.d = ft;
+    for (i = 0; i < 4; i++) {
+        vs.sh[i] = (vs.sh[i] >= vt.sh[i] ? vs.sh[i] : vt.sh[i]);
+    }
+    return vs.d;
+}
+
+uint64_t helper_pminsh(uint64_t fs, uint64_t ft)
+{
+    LMIValue vs, vt;
+    unsigned i;
+
+    vs.d = fs;
+    vt.d = ft;
+    for (i = 0; i < 4; i++) {
+        vs.sh[i] = (vs.sh[i] <= vt.sh[i] ? vs.sh[i] : vt.sh[i]);
+    }
+    return vs.d;
+}
+
+uint64_t helper_pmaxub(uint64_t fs, uint64_t ft)
+{
+    LMIValue vs, vt;
+    unsigned i;
+
+    vs.d = fs;
+    vt.d = ft;
+    for (i = 0; i < 4; i++) {
+        vs.ub[i] = (vs.ub[i] >= vt.ub[i] ? vs.ub[i] : vt.ub[i]);
+    }
+    return vs.d;
+}
+
+uint64_t helper_pminub(uint64_t fs, uint64_t ft)
+{
+    LMIValue vs, vt;
+    unsigned i;
+
+    vs.d = fs;
+    vt.d = ft;
+    for (i = 0; i < 4; i++) {
+        vs.ub[i] = (vs.ub[i] <= vt.ub[i] ? vs.ub[i] : vt.ub[i]);
+    }
+    return vs.d;
+}
+
+uint64_t helper_pcmpeqw(uint64_t fs, uint64_t ft)
+{
+    LMIValue vs, vt;
+    unsigned i;
+
+    vs.d = fs;
+    vt.d = ft;
+    for (i = 0; i < 2; i++) {
+        vs.uw[i] = -(vs.uw[i] == vt.uw[i]);
+    }
+    return vs.d;
+}
+
+uint64_t helper_pcmpgtw(uint64_t fs, uint64_t ft)
+{
+    LMIValue vs, vt;
+    unsigned i;
+
+    vs.d = fs;
+    vt.d = ft;
+    for (i = 0; i < 2; i++) {
+        vs.uw[i] = -(vs.uw[i] > vt.uw[i]);
+    }
+    return vs.d;
+}
+
+uint64_t helper_pcmpeqh(uint64_t fs, uint64_t ft)
+{
+    LMIValue vs, vt;
+    unsigned i;
+
+    vs.d = fs;
+    vt.d = ft;
+    for (i = 0; i < 4; i++) {
+        vs.uh[i] = -(vs.uh[i] == vt.uh[i]);
+    }
+    return vs.d;
+}
+
+uint64_t helper_pcmpgth(uint64_t fs, uint64_t ft)
+{
+    LMIValue vs, vt;
+    unsigned i;
+
+    vs.d = fs;
+    vt.d = ft;
+    for (i = 0; i < 4; i++) {
+        vs.uh[i] = -(vs.uh[i] > vt.uh[i]);
+    }
+    return vs.d;
+}
+
+uint64_t helper_pcmpeqb(uint64_t fs, uint64_t ft)
+{
+    LMIValue vs, vt;
+    unsigned i;
+
+    vs.d = fs;
+    vt.d = ft;
+    for (i = 0; i < 8; i++) {
+        vs.ub[i] = -(vs.ub[i] == vt.ub[i]);
+    }
+    return vs.d;
+}
+
+uint64_t helper_pcmpgtb(uint64_t fs, uint64_t ft)
+{
+    LMIValue vs, vt;
+    unsigned i;
+
+    vs.d = fs;
+    vt.d = ft;
+    for (i = 0; i < 8; i++) {
+        vs.ub[i] = -(vs.ub[i] > vt.ub[i]);
+    }
+    return vs.d;
+}
+
+uint64_t helper_psllw(uint64_t fs, uint64_t ft)
+{
+    LMIValue vs;
+    unsigned i;
+
+    ft &= 0x7f;
+    if (ft > 31) {
+        return 0;
+    }
+    vs.d = fs;
+    for (i = 0; i < 2; ++i) {
+        vs.uw[i] <<= ft;
+    }
+    return vs.d;
+}
+
+uint64_t helper_psrlw(uint64_t fs, uint64_t ft)
+{
+    LMIValue vs;
+    unsigned i;
+
+    ft &= 0x7f;
+    if (ft > 31) {
+        return 0;
+    }
+    vs.d = fs;
+    for (i = 0; i < 2; ++i) {
+        vs.uw[i] >>= ft;
+    }
+    return vs.d;
+}
+
+uint64_t helper_psraw(uint64_t fs, uint64_t ft)
+{
+    LMIValue vs;
+    unsigned i;
+
+    ft &= 0x7f;
+    if (ft > 31) {
+        ft = 31;
+    }
+    vs.d = fs;
+    for (i = 0; i < 2; ++i) {
+        vs.sw[i] >>= ft;
+    }
+    return vs.d;
+}
+
+uint64_t helper_psllh(uint64_t fs, uint64_t ft)
+{
+    LMIValue vs;
+    unsigned i;
+
+    ft &= 0x7f;
+    if (ft > 15) {
+        return 0;
+    }
+    vs.d = fs;
+    for (i = 0; i < 4; ++i) {
+        vs.uh[i] <<= ft;
+    }
+    return vs.d;
+}
+
+uint64_t helper_psrlh(uint64_t fs, uint64_t ft)
+{
+    LMIValue vs;
+    unsigned i;
+
+    ft &= 0x7f;
+    if (ft > 15) {
+        return 0;
+    }
+    vs.d = fs;
+    for (i = 0; i < 4; ++i) {
+        vs.uh[i] >>= ft;
+    }
+    return vs.d;
+}
+
+uint64_t helper_psrah(uint64_t fs, uint64_t ft)
+{
+    LMIValue vs;
+    unsigned i;
+
+    ft &= 0x7f;
+    if (ft > 15) {
+        ft = 15;
+    }
+    vs.d = fs;
+    for (i = 0; i < 4; ++i) {
+        vs.sh[i] >>= ft;
+    }
+    return vs.d;
+}
+
+uint64_t helper_pmullh(uint64_t fs, uint64_t ft)
+{
+    LMIValue vs, vt;
+    unsigned i;
+
+    vs.d = fs;
+    vt.d = ft;
+    for (i = 0; i < 4; ++i) {
+        vs.sh[i] *= vt.sh[i];
+    }
+    return vs.d;
+}
+
+uint64_t helper_pmulhh(uint64_t fs, uint64_t ft)
+{
+    LMIValue vs, vt;
+    unsigned i;
+
+    vs.d = fs;
+    vt.d = ft;
+    for (i = 0; i < 4; ++i) {
+        int32_t r = vs.sh[i] * vt.sh[i];
+        vs.sh[i] = r >> 16;
+    }
+    return vs.d;
+}
+
+uint64_t helper_pmulhuh(uint64_t fs, uint64_t ft)
+{
+    LMIValue vs, vt;
+    unsigned i;
+
+    vs.d = fs;
+    vt.d = ft;
+    for (i = 0; i < 4; ++i) {
+        uint32_t r = vs.uh[i] * vt.uh[i];
+        vs.uh[i] = r >> 16;
+    }
+    return vs.d;
+}
+
+uint64_t helper_pmaddhw(uint64_t fs, uint64_t ft)
+{
+    unsigned host = BYTE_ORDER_XOR(3);
+    LMIValue vs, vt;
+    uint32_t p0, p1;
+
+    vs.d = fs;
+    vt.d = ft;
+    p0  = vs.sh[0 ^ host] * vt.sh[0 ^ host];
+    p0 += vs.sh[1 ^ host] * vt.sh[1 ^ host];
+    p1  = vs.sh[2 ^ host] * vt.sh[2 ^ host];
+    p1 += vs.sh[3 ^ host] * vt.sh[3 ^ host];
+
+    return ((uint64_t)p1 << 32) | p0;
+}
+
+uint64_t helper_pasubub(uint64_t fs, uint64_t ft)
+{
+    LMIValue vs, vt;
+    unsigned i;
+
+    vs.d = fs;
+    vt.d = ft;
+    for (i = 0; i < 8; ++i) {
+        int r = vs.ub[i] - vt.ub[i];
+        vs.ub[i] = (r < 0 ? -r : r);
+    }
+    return vs.d;
+}
+
+uint64_t helper_biadd(uint64_t fs)
+{
+    unsigned i, fd;
+
+    for (i = fd = 0; i < 8; ++i) {
+        fd += (fs >> (i * 8)) & 0xff;
+    }
+    return fd & 0xffff;
+}
+
+uint64_t helper_pmovmskb(uint64_t fs)
+{
+    unsigned fd = 0;
+
+    fd |= ((fs >>  7) & 1) << 0;
+    fd |= ((fs >> 15) & 1) << 1;
+    fd |= ((fs >> 23) & 1) << 2;
+    fd |= ((fs >> 31) & 1) << 3;
+    fd |= ((fs >> 39) & 1) << 4;
+    fd |= ((fs >> 47) & 1) << 5;
+    fd |= ((fs >> 55) & 1) << 6;
+    fd |= ((fs >> 63) & 1) << 7;
+
+    return fd & 0xff;
+}
diff --git a/target-mips/op_helper.c b/target-mips/op_helper.c
index e5bc93e22..f45d494b1 100644
--- a/target-mips/op_helper.c
+++ b/target-mips/op_helper.c
@@ -18,8 +18,6 @@
  */
 #include <stdlib.h>
 #include "cpu.h"
-#include "dyngen-exec.h"
-
 #include "host-utils.h"
 
 #include "helper.h"
@@ -32,102 +30,71 @@
 static inline void cpu_mips_tlb_flush (CPUMIPSState *env, int flush_global);
 #endif
 
-static inline void compute_hflags(CPUMIPSState *env)
-{
-    env->hflags &= ~(MIPS_HFLAG_COP1X | MIPS_HFLAG_64 | MIPS_HFLAG_CP0 |
-                     MIPS_HFLAG_F64 | MIPS_HFLAG_FPU | MIPS_HFLAG_KSU |
-                     MIPS_HFLAG_UX);
-    if (!(env->CP0_Status & (1 << CP0St_EXL)) &&
-        !(env->CP0_Status & (1 << CP0St_ERL)) &&
-        !(env->hflags & MIPS_HFLAG_DM)) {
-        env->hflags |= (env->CP0_Status >> CP0St_KSU) & MIPS_HFLAG_KSU;
-    }
-#if defined(TARGET_MIPS64)
-    if (((env->hflags & MIPS_HFLAG_KSU) != MIPS_HFLAG_UM) ||
-        (env->CP0_Status & (1 << CP0St_PX)) ||
-        (env->CP0_Status & (1 << CP0St_UX))) {
-        env->hflags |= MIPS_HFLAG_64;
-    }
-    if (env->CP0_Status & (1 << CP0St_UX)) {
-        env->hflags |= MIPS_HFLAG_UX;
-    }
-#endif
-    if ((env->CP0_Status & (1 << CP0St_CU0)) ||
-        !(env->hflags & MIPS_HFLAG_KSU)) {
-        env->hflags |= MIPS_HFLAG_CP0;
-    }
-    if (env->CP0_Status & (1 << CP0St_CU1)) {
-        env->hflags |= MIPS_HFLAG_FPU;
-    }
-    if (env->CP0_Status & (1 << CP0St_FR)) {
-        env->hflags |= MIPS_HFLAG_F64;
-    }
-    if (env->insn_flags & ISA_MIPS32R2) {
-        if (env->active_fpu.fcr0 & (1 << FCR0_F64)) {
-            env->hflags |= MIPS_HFLAG_COP1X;
-        }
-    } else if (env->insn_flags & ISA_MIPS32) {
-        if (env->hflags & MIPS_HFLAG_64) {
-            env->hflags |= MIPS_HFLAG_COP1X;
-        }
-    } else if (env->insn_flags & ISA_MIPS4) {
-        /* All supported MIPS IV CPUs use the XX (CU3) to enable
-           and disable the MIPS IV extensions to the MIPS III ISA.
-           Some other MIPS IV CPUs ignore the bit, so the check here
-           would be too restrictive for them.  */
-        if (env->CP0_Status & (1 << CP0St_CU3)) {
-            env->hflags |= MIPS_HFLAG_COP1X;
-        }
-    }
-}
-
 /*****************************************************************************/
 /* Exceptions processing helpers */
 
-void helper_raise_exception_err (uint32_t exception, int error_code)
+static inline void QEMU_NORETURN do_raise_exception_err(CPUMIPSState *env,
+                                                        uint32_t exception,
+                                                        int error_code,
+                                                        uintptr_t pc)
 {
+    TranslationBlock *tb;
 #if 1
     if (exception < 0x100)
         qemu_log("%s: %d %d\n", __func__, exception, error_code);
 #endif
     env->exception_index = exception;
     env->error_code = error_code;
+
+    if (pc) {
+        /* now we have a real cpu fault */
+        tb = tb_find_pc(pc);
+        if (tb) {
+            /* the PC is inside the translated code. It means that we have
+               a virtual CPU fault */
+            cpu_restore_state(tb, env, pc);
+        }
+    }
+
     cpu_loop_exit(env);
 }
 
-void helper_raise_exception (uint32_t exception)
+static inline void QEMU_NORETURN do_raise_exception(CPUMIPSState *env,
+                                                    uint32_t exception,
+                                                    uintptr_t pc)
 {
-    helper_raise_exception_err(exception, 0);
+    do_raise_exception_err(env, exception, 0, pc);
 }
 
-#if !defined(CONFIG_USER_ONLY)
-static void do_restore_state(uintptr_t pc)
+void helper_raise_exception_err(CPUMIPSState *env, uint32_t exception,
+                                int error_code)
 {
-    TranslationBlock *tb;
+    do_raise_exception_err(env, exception, error_code, 0);
+}
 
-    tb = tb_find_pc (pc);
-    if (tb) {
-        cpu_restore_state(tb, env, pc);
-    }
+void helper_raise_exception(CPUMIPSState *env, uint32_t exception)
+{
+    do_raise_exception(env, exception, 0);
 }
-#endif
 
 #if defined(CONFIG_USER_ONLY)
 #define HELPER_LD(name, insn, type)                                     \
-static inline type do_##name(target_ulong addr, int mem_idx)            \
+static inline type do_##name(CPUMIPSState *env, target_ulong addr,      \
+                             int mem_idx)                               \
 {                                                                       \
     return (type) insn##_raw(addr);                                     \
 }
 #else
 #define HELPER_LD(name, insn, type)                                     \
-static inline type do_##name(target_ulong addr, int mem_idx)            \
+static inline type do_##name(CPUMIPSState *env, target_ulong addr,      \
+                             int mem_idx)                               \
 {                                                                       \
     switch (mem_idx)                                                    \
     {                                                                   \
-    case 0: return (type) insn##_kernel(addr); break;                   \
-    case 1: return (type) insn##_super(addr); break;                    \
+    case 0: return (type) cpu_##insn##_kernel(env, addr); break;        \
+    case 1: return (type) cpu_##insn##_super(env, addr); break;         \
     default:                                                            \
-    case 2: return (type) insn##_user(addr); break;                     \
+    case 2: return (type) cpu_##insn##_user(env, addr); break;          \
     }                                                                   \
 }
 #endif
@@ -140,20 +107,22 @@ HELPER_LD(ld, ldq, int64_t)
 
 #if defined(CONFIG_USER_ONLY)
 #define HELPER_ST(name, insn, type)                                     \
-static inline void do_##name(target_ulong addr, type val, int mem_idx)  \
+static inline void do_##name(CPUMIPSState *env, target_ulong addr,      \
+                             type val, int mem_idx)                     \
 {                                                                       \
     insn##_raw(addr, val);                                              \
 }
 #else
 #define HELPER_ST(name, insn, type)                                     \
-static inline void do_##name(target_ulong addr, type val, int mem_idx)  \
+static inline void do_##name(CPUMIPSState *env, target_ulong addr,      \
+                             type val, int mem_idx)                     \
 {                                                                       \
     switch (mem_idx)                                                    \
     {                                                                   \
-    case 0: insn##_kernel(addr, val); break;                            \
-    case 1: insn##_super(addr, val); break;                             \
+    case 0: cpu_##insn##_kernel(env, addr, val); break;                 \
+    case 1: cpu_##insn##_super(env, addr, val); break;                  \
     default:                                                            \
-    case 2: insn##_user(addr, val); break;                              \
+    case 2: cpu_##insn##_user(env, addr, val); break;                   \
     }                                                                   \
 }
 #endif
@@ -187,12 +156,12 @@ target_ulong helper_dclz (target_ulong arg1)
 #endif /* TARGET_MIPS64 */
 
 /* 64 bits arithmetic for 32 bits hosts */
-static inline uint64_t get_HILO (void)
+static inline uint64_t get_HILO(CPUMIPSState *env)
 {
     return ((uint64_t)(env->active_tc.HI[0]) << 32) | (uint32_t)env->active_tc.LO[0];
 }
 
-static inline target_ulong set_HIT0_LO(uint64_t HILO)
+static inline target_ulong set_HIT0_LO(CPUMIPSState *env, uint64_t HILO)
 {
     target_ulong tmp;
     env->active_tc.LO[0] = (int32_t)(HILO & 0xFFFFFFFF);
@@ -200,7 +169,7 @@ static inline target_ulong set_HIT0_LO(uint64_t HILO)
     return tmp;
 }
 
-static inline target_ulong set_HI_LOT0(uint64_t HILO)
+static inline target_ulong set_HI_LOT0(CPUMIPSState *env, uint64_t HILO)
 {
     target_ulong tmp = env->active_tc.LO[0] = (int32_t)(HILO & 0xFFFFFFFF);
     env->active_tc.HI[0] = (int32_t)(HILO >> 32);
@@ -208,91 +177,110 @@ static inline target_ulong set_HI_LOT0(uint64_t HILO)
 }
 
 /* Multiplication variants of the vr54xx. */
-target_ulong helper_muls (target_ulong arg1, target_ulong arg2)
+target_ulong helper_muls(CPUMIPSState *env, target_ulong arg1,
+                         target_ulong arg2)
 {
-    return set_HI_LOT0(0 - ((int64_t)(int32_t)arg1 * (int64_t)(int32_t)arg2));
+    return set_HI_LOT0(env, 0 - ((int64_t)(int32_t)arg1 *
+                                 (int64_t)(int32_t)arg2));
 }
 
-target_ulong helper_mulsu (target_ulong arg1, target_ulong arg2)
+target_ulong helper_mulsu(CPUMIPSState *env, target_ulong arg1,
+                          target_ulong arg2)
 {
-    return set_HI_LOT0(0 - (uint64_t)(uint32_t)arg1 * (uint64_t)(uint32_t)arg2);
+    return set_HI_LOT0(env, 0 - (uint64_t)(uint32_t)arg1 *
+                       (uint64_t)(uint32_t)arg2);
 }
 
-target_ulong helper_macc (target_ulong arg1, target_ulong arg2)
+target_ulong helper_macc(CPUMIPSState *env, target_ulong arg1,
+                         target_ulong arg2)
 {
-    return set_HI_LOT0((int64_t)get_HILO() + (int64_t)(int32_t)arg1 *
-                                             (int64_t)(int32_t)arg2);
+    return set_HI_LOT0(env, (int64_t)get_HILO(env) + (int64_t)(int32_t)arg1 *
+                       (int64_t)(int32_t)arg2);
 }
 
-target_ulong helper_macchi (target_ulong arg1, target_ulong arg2)
+target_ulong helper_macchi(CPUMIPSState *env, target_ulong arg1,
+                           target_ulong arg2)
 {
-    return set_HIT0_LO((int64_t)get_HILO() + (int64_t)(int32_t)arg1 *
-                                             (int64_t)(int32_t)arg2);
+    return set_HIT0_LO(env, (int64_t)get_HILO(env) + (int64_t)(int32_t)arg1 *
+                       (int64_t)(int32_t)arg2);
 }
 
-target_ulong helper_maccu (target_ulong arg1, target_ulong arg2)
+target_ulong helper_maccu(CPUMIPSState *env, target_ulong arg1,
+                          target_ulong arg2)
 {
-    return set_HI_LOT0((uint64_t)get_HILO() + (uint64_t)(uint32_t)arg1 *
-                                              (uint64_t)(uint32_t)arg2);
+    return set_HI_LOT0(env, (uint64_t)get_HILO(env) +
+                       (uint64_t)(uint32_t)arg1 * (uint64_t)(uint32_t)arg2);
 }
 
-target_ulong helper_macchiu (target_ulong arg1, target_ulong arg2)
+target_ulong helper_macchiu(CPUMIPSState *env, target_ulong arg1,
+                            target_ulong arg2)
 {
-    return set_HIT0_LO((uint64_t)get_HILO() + (uint64_t)(uint32_t)arg1 *
-                                              (uint64_t)(uint32_t)arg2);
+    return set_HIT0_LO(env, (uint64_t)get_HILO(env) +
+                       (uint64_t)(uint32_t)arg1 * (uint64_t)(uint32_t)arg2);
 }
 
-target_ulong helper_msac (target_ulong arg1, target_ulong arg2)
+target_ulong helper_msac(CPUMIPSState *env, target_ulong arg1,
+                         target_ulong arg2)
 {
-    return set_HI_LOT0((int64_t)get_HILO() - (int64_t)(int32_t)arg1 *
-                                             (int64_t)(int32_t)arg2);
+    return set_HI_LOT0(env, (int64_t)get_HILO(env) - (int64_t)(int32_t)arg1 *
+                       (int64_t)(int32_t)arg2);
 }
 
-target_ulong helper_msachi (target_ulong arg1, target_ulong arg2)
+target_ulong helper_msachi(CPUMIPSState *env, target_ulong arg1,
+                           target_ulong arg2)
 {
-    return set_HIT0_LO((int64_t)get_HILO() - (int64_t)(int32_t)arg1 *
-                                             (int64_t)(int32_t)arg2);
+    return set_HIT0_LO(env, (int64_t)get_HILO(env) - (int64_t)(int32_t)arg1 *
+                       (int64_t)(int32_t)arg2);
 }
 
-target_ulong helper_msacu (target_ulong arg1, target_ulong arg2)
+target_ulong helper_msacu(CPUMIPSState *env, target_ulong arg1,
+                          target_ulong arg2)
 {
-    return set_HI_LOT0((uint64_t)get_HILO() - (uint64_t)(uint32_t)arg1 *
-                                              (uint64_t)(uint32_t)arg2);
+    return set_HI_LOT0(env, (uint64_t)get_HILO(env) -
+                       (uint64_t)(uint32_t)arg1 * (uint64_t)(uint32_t)arg2);
 }
 
-target_ulong helper_msachiu (target_ulong arg1, target_ulong arg2)
+target_ulong helper_msachiu(CPUMIPSState *env, target_ulong arg1,
+                            target_ulong arg2)
 {
-    return set_HIT0_LO((uint64_t)get_HILO() - (uint64_t)(uint32_t)arg1 *
-                                              (uint64_t)(uint32_t)arg2);
+    return set_HIT0_LO(env, (uint64_t)get_HILO(env) -
+                       (uint64_t)(uint32_t)arg1 * (uint64_t)(uint32_t)arg2);
 }
 
-target_ulong helper_mulhi (target_ulong arg1, target_ulong arg2)
+target_ulong helper_mulhi(CPUMIPSState *env, target_ulong arg1,
+                          target_ulong arg2)
 {
-    return set_HIT0_LO((int64_t)(int32_t)arg1 * (int64_t)(int32_t)arg2);
+    return set_HIT0_LO(env, (int64_t)(int32_t)arg1 * (int64_t)(int32_t)arg2);
 }
 
-target_ulong helper_mulhiu (target_ulong arg1, target_ulong arg2)
+target_ulong helper_mulhiu(CPUMIPSState *env, target_ulong arg1,
+                           target_ulong arg2)
 {
-    return set_HIT0_LO((uint64_t)(uint32_t)arg1 * (uint64_t)(uint32_t)arg2);
+    return set_HIT0_LO(env, (uint64_t)(uint32_t)arg1 *
+                       (uint64_t)(uint32_t)arg2);
 }
 
-target_ulong helper_mulshi (target_ulong arg1, target_ulong arg2)
+target_ulong helper_mulshi(CPUMIPSState *env, target_ulong arg1,
+                           target_ulong arg2)
 {
-    return set_HIT0_LO(0 - (int64_t)(int32_t)arg1 * (int64_t)(int32_t)arg2);
+    return set_HIT0_LO(env, 0 - (int64_t)(int32_t)arg1 *
+                       (int64_t)(int32_t)arg2);
 }
 
-target_ulong helper_mulshiu (target_ulong arg1, target_ulong arg2)
+target_ulong helper_mulshiu(CPUMIPSState *env, target_ulong arg1,
+                            target_ulong arg2)
 {
-    return set_HIT0_LO(0 - (uint64_t)(uint32_t)arg1 * (uint64_t)(uint32_t)arg2);
+    return set_HIT0_LO(env, 0 - (uint64_t)(uint32_t)arg1 *
+                       (uint64_t)(uint32_t)arg2);
 }
 
 #ifdef TARGET_MIPS64
-void helper_dmult (target_ulong arg1, target_ulong arg2)
+void helper_dmult(CPUMIPSState *env, target_ulong arg1, target_ulong arg2)
 {
     muls64(&(env->active_tc.LO[0]), &(env->active_tc.HI[0]), arg1, arg2);
 }
 
-void helper_dmultu (target_ulong arg1, target_ulong arg2)
+void helper_dmultu(CPUMIPSState *env, target_ulong arg1, target_ulong arg2)
 {
     mulu64(&(env->active_tc.LO[0]), &(env->active_tc.HI[0]), arg1, arg2);
 }
@@ -300,9 +288,11 @@ void helper_dmultu (target_ulong arg1, target_ulong arg2)
 
 #ifndef CONFIG_USER_ONLY
 
-static inline target_phys_addr_t do_translate_address(target_ulong address, int rw)
+static inline hwaddr do_translate_address(CPUMIPSState *env,
+                                                      target_ulong address,
+                                                      int rw)
 {
-    target_phys_addr_t lladdr;
+    hwaddr lladdr;
 
     lladdr = cpu_mips_translate_address(env, address, rw);
 
@@ -314,10 +304,10 @@ static inline target_phys_addr_t do_translate_address(target_ulong address, int
 }
 
 #define HELPER_LD_ATOMIC(name, insn)                                          \
-target_ulong helper_##name(target_ulong arg, int mem_idx)                     \
+target_ulong helper_##name(CPUMIPSState *env, target_ulong arg, int mem_idx)  \
 {                                                                             \
-    env->lladdr = do_translate_address(arg, 0);                               \
-    env->llval = do_##insn(arg, mem_idx);                                     \
+    env->lladdr = do_translate_address(env, arg, 0);                          \
+    env->llval = do_##insn(env, arg, mem_idx);                                \
     return env->llval;                                                        \
 }
 HELPER_LD_ATOMIC(ll, lw)
@@ -327,18 +317,19 @@ HELPER_LD_ATOMIC(lld, ld)
 #undef HELPER_LD_ATOMIC
 
 #define HELPER_ST_ATOMIC(name, ld_insn, st_insn, almask)                      \
-target_ulong helper_##name(target_ulong arg1, target_ulong arg2, int mem_idx) \
+target_ulong helper_##name(CPUMIPSState *env, target_ulong arg1,              \
+                           target_ulong arg2, int mem_idx)                    \
 {                                                                             \
     target_long tmp;                                                          \
                                                                               \
     if (arg2 & almask) {                                                      \
         env->CP0_BadVAddr = arg2;                                             \
-        helper_raise_exception(EXCP_AdES);                                    \
+        helper_raise_exception(env, EXCP_AdES);                               \
     }                                                                         \
-    if (do_translate_address(arg2, 1) == env->lladdr) {                       \
-        tmp = do_##ld_insn(arg2, mem_idx);                                    \
+    if (do_translate_address(env, arg2, 1) == env->lladdr) {                  \
+        tmp = do_##ld_insn(env, arg2, mem_idx);                               \
         if (tmp == env->llval) {                                              \
-            do_##st_insn(arg2, arg1, mem_idx);                                \
+            do_##st_insn(env, arg2, arg1, mem_idx);                           \
             return 1;                                                         \
         }                                                                     \
     }                                                                         \
@@ -359,80 +350,34 @@ HELPER_ST_ATOMIC(scd, ld, sd, 0x7)
 #define GET_OFFSET(addr, offset) (addr - (offset))
 #endif
 
-target_ulong helper_lwl(target_ulong arg1, target_ulong arg2, int mem_idx)
-{
-    target_ulong tmp;
-
-    tmp = do_lbu(arg2, mem_idx);
-    arg1 = (arg1 & 0x00FFFFFF) | (tmp << 24);
-
-    if (GET_LMASK(arg2) <= 2) {
-        tmp = do_lbu(GET_OFFSET(arg2, 1), mem_idx);
-        arg1 = (arg1 & 0xFF00FFFF) | (tmp << 16);
-    }
-
-    if (GET_LMASK(arg2) <= 1) {
-        tmp = do_lbu(GET_OFFSET(arg2, 2), mem_idx);
-        arg1 = (arg1 & 0xFFFF00FF) | (tmp << 8);
-    }
-
-    if (GET_LMASK(arg2) == 0) {
-        tmp = do_lbu(GET_OFFSET(arg2, 3), mem_idx);
-        arg1 = (arg1 & 0xFFFFFF00) | tmp;
-    }
-    return (int32_t)arg1;
-}
-
-target_ulong helper_lwr(target_ulong arg1, target_ulong arg2, int mem_idx)
-{
-    target_ulong tmp;
-
-    tmp = do_lbu(arg2, mem_idx);
-    arg1 = (arg1 & 0xFFFFFF00) | tmp;
-
-    if (GET_LMASK(arg2) >= 1) {
-        tmp = do_lbu(GET_OFFSET(arg2, -1), mem_idx);
-        arg1 = (arg1 & 0xFFFF00FF) | (tmp << 8);
-    }
-
-    if (GET_LMASK(arg2) >= 2) {
-        tmp = do_lbu(GET_OFFSET(arg2, -2), mem_idx);
-        arg1 = (arg1 & 0xFF00FFFF) | (tmp << 16);
-    }
-
-    if (GET_LMASK(arg2) == 3) {
-        tmp = do_lbu(GET_OFFSET(arg2, -3), mem_idx);
-        arg1 = (arg1 & 0x00FFFFFF) | (tmp << 24);
-    }
-    return (int32_t)arg1;
-}
-
-void helper_swl(target_ulong arg1, target_ulong arg2, int mem_idx)
+void helper_swl(CPUMIPSState *env, target_ulong arg1, target_ulong arg2,
+                int mem_idx)
 {
-    do_sb(arg2, (uint8_t)(arg1 >> 24), mem_idx);
+    do_sb(env, arg2, (uint8_t)(arg1 >> 24), mem_idx);
 
     if (GET_LMASK(arg2) <= 2)
-        do_sb(GET_OFFSET(arg2, 1), (uint8_t)(arg1 >> 16), mem_idx);
+        do_sb(env, GET_OFFSET(arg2, 1), (uint8_t)(arg1 >> 16), mem_idx);
 
     if (GET_LMASK(arg2) <= 1)
-        do_sb(GET_OFFSET(arg2, 2), (uint8_t)(arg1 >> 8), mem_idx);
+        do_sb(env, GET_OFFSET(arg2, 2), (uint8_t)(arg1 >> 8), mem_idx);
 
     if (GET_LMASK(arg2) == 0)
-        do_sb(GET_OFFSET(arg2, 3), (uint8_t)arg1, mem_idx);
+        do_sb(env, GET_OFFSET(arg2, 3), (uint8_t)arg1, mem_idx);
 }
 
-void helper_swr(target_ulong arg1, target_ulong arg2, int mem_idx)
+void helper_swr(CPUMIPSState *env, target_ulong arg1, target_ulong arg2,
+                int mem_idx)
 {
-    do_sb(arg2, (uint8_t)arg1, mem_idx);
+    do_sb(env, arg2, (uint8_t)arg1, mem_idx);
 
     if (GET_LMASK(arg2) >= 1)
-        do_sb(GET_OFFSET(arg2, -1), (uint8_t)(arg1 >> 8), mem_idx);
+        do_sb(env, GET_OFFSET(arg2, -1), (uint8_t)(arg1 >> 8), mem_idx);
 
     if (GET_LMASK(arg2) >= 2)
-        do_sb(GET_OFFSET(arg2, -2), (uint8_t)(arg1 >> 16), mem_idx);
+        do_sb(env, GET_OFFSET(arg2, -2), (uint8_t)(arg1 >> 16), mem_idx);
 
     if (GET_LMASK(arg2) == 3)
-        do_sb(GET_OFFSET(arg2, -3), (uint8_t)(arg1 >> 24), mem_idx);
+        do_sb(env, GET_OFFSET(arg2, -3), (uint8_t)(arg1 >> 24), mem_idx);
 }
 
 #if defined(TARGET_MIPS64)
@@ -445,292 +390,155 @@ void helper_swr(target_ulong arg1, target_ulong arg2, int mem_idx)
 #define GET_LMASK64(v) (((v) & 7) ^ 7)
 #endif
 
-target_ulong helper_ldl(target_ulong arg1, target_ulong arg2, int mem_idx)
-{
-    uint64_t tmp;
-
-    tmp = do_lbu(arg2, mem_idx);
-    arg1 = (arg1 & 0x00FFFFFFFFFFFFFFULL) | (tmp << 56);
-
-    if (GET_LMASK64(arg2) <= 6) {
-        tmp = do_lbu(GET_OFFSET(arg2, 1), mem_idx);
-        arg1 = (arg1 & 0xFF00FFFFFFFFFFFFULL) | (tmp << 48);
-    }
-
-    if (GET_LMASK64(arg2) <= 5) {
-        tmp = do_lbu(GET_OFFSET(arg2, 2), mem_idx);
-        arg1 = (arg1 & 0xFFFF00FFFFFFFFFFULL) | (tmp << 40);
-    }
-
-    if (GET_LMASK64(arg2) <= 4) {
-        tmp = do_lbu(GET_OFFSET(arg2, 3), mem_idx);
-        arg1 = (arg1 & 0xFFFFFF00FFFFFFFFULL) | (tmp << 32);
-    }
-
-    if (GET_LMASK64(arg2) <= 3) {
-        tmp = do_lbu(GET_OFFSET(arg2, 4), mem_idx);
-        arg1 = (arg1 & 0xFFFFFFFF00FFFFFFULL) | (tmp << 24);
-    }
-
-    if (GET_LMASK64(arg2) <= 2) {
-        tmp = do_lbu(GET_OFFSET(arg2, 5), mem_idx);
-        arg1 = (arg1 & 0xFFFFFFFFFF00FFFFULL) | (tmp << 16);
-    }
-
-    if (GET_LMASK64(arg2) <= 1) {
-        tmp = do_lbu(GET_OFFSET(arg2, 6), mem_idx);
-        arg1 = (arg1 & 0xFFFFFFFFFFFF00FFULL) | (tmp << 8);
-    }
-
-    if (GET_LMASK64(arg2) == 0) {
-        tmp = do_lbu(GET_OFFSET(arg2, 7), mem_idx);
-        arg1 = (arg1 & 0xFFFFFFFFFFFFFF00ULL) | tmp;
-    }
-
-    return arg1;
-}
-
-target_ulong helper_ldr(target_ulong arg1, target_ulong arg2, int mem_idx)
-{
-    uint64_t tmp;
-
-    tmp = do_lbu(arg2, mem_idx);
-    arg1 = (arg1 & 0xFFFFFFFFFFFFFF00ULL) | tmp;
-
-    if (GET_LMASK64(arg2) >= 1) {
-        tmp = do_lbu(GET_OFFSET(arg2, -1), mem_idx);
-        arg1 = (arg1 & 0xFFFFFFFFFFFF00FFULL) | (tmp  << 8);
-    }
-
-    if (GET_LMASK64(arg2) >= 2) {
-        tmp = do_lbu(GET_OFFSET(arg2, -2), mem_idx);
-        arg1 = (arg1 & 0xFFFFFFFFFF00FFFFULL) | (tmp << 16);
-    }
-
-    if (GET_LMASK64(arg2) >= 3) {
-        tmp = do_lbu(GET_OFFSET(arg2, -3), mem_idx);
-        arg1 = (arg1 & 0xFFFFFFFF00FFFFFFULL) | (tmp << 24);
-    }
-
-    if (GET_LMASK64(arg2) >= 4) {
-        tmp = do_lbu(GET_OFFSET(arg2, -4), mem_idx);
-        arg1 = (arg1 & 0xFFFFFF00FFFFFFFFULL) | (tmp << 32);
-    }
-
-    if (GET_LMASK64(arg2) >= 5) {
-        tmp = do_lbu(GET_OFFSET(arg2, -5), mem_idx);
-        arg1 = (arg1 & 0xFFFF00FFFFFFFFFFULL) | (tmp << 40);
-    }
-
-    if (GET_LMASK64(arg2) >= 6) {
-        tmp = do_lbu(GET_OFFSET(arg2, -6), mem_idx);
-        arg1 = (arg1 & 0xFF00FFFFFFFFFFFFULL) | (tmp << 48);
-    }
-
-    if (GET_LMASK64(arg2) == 7) {
-        tmp = do_lbu(GET_OFFSET(arg2, -7), mem_idx);
-        arg1 = (arg1 & 0x00FFFFFFFFFFFFFFULL) | (tmp << 56);
-    }
-
-    return arg1;
-}
-
-void helper_sdl(target_ulong arg1, target_ulong arg2, int mem_idx)
+void helper_sdl(CPUMIPSState *env, target_ulong arg1, target_ulong arg2,
+                int mem_idx)
 {
-    do_sb(arg2, (uint8_t)(arg1 >> 56), mem_idx);
+    do_sb(env, arg2, (uint8_t)(arg1 >> 56), mem_idx);
 
     if (GET_LMASK64(arg2) <= 6)
-        do_sb(GET_OFFSET(arg2, 1), (uint8_t)(arg1 >> 48), mem_idx);
+        do_sb(env, GET_OFFSET(arg2, 1), (uint8_t)(arg1 >> 48), mem_idx);
 
     if (GET_LMASK64(arg2) <= 5)
-        do_sb(GET_OFFSET(arg2, 2), (uint8_t)(arg1 >> 40), mem_idx);
+        do_sb(env, GET_OFFSET(arg2, 2), (uint8_t)(arg1 >> 40), mem_idx);
 
     if (GET_LMASK64(arg2) <= 4)
-        do_sb(GET_OFFSET(arg2, 3), (uint8_t)(arg1 >> 32), mem_idx);
+        do_sb(env, GET_OFFSET(arg2, 3), (uint8_t)(arg1 >> 32), mem_idx);
 
     if (GET_LMASK64(arg2) <= 3)
-        do_sb(GET_OFFSET(arg2, 4), (uint8_t)(arg1 >> 24), mem_idx);
+        do_sb(env, GET_OFFSET(arg2, 4), (uint8_t)(arg1 >> 24), mem_idx);
 
     if (GET_LMASK64(arg2) <= 2)
-        do_sb(GET_OFFSET(arg2, 5), (uint8_t)(arg1 >> 16), mem_idx);
+        do_sb(env, GET_OFFSET(arg2, 5), (uint8_t)(arg1 >> 16), mem_idx);
 
     if (GET_LMASK64(arg2) <= 1)
-        do_sb(GET_OFFSET(arg2, 6), (uint8_t)(arg1 >> 8), mem_idx);
+        do_sb(env, GET_OFFSET(arg2, 6), (uint8_t)(arg1 >> 8), mem_idx);
 
     if (GET_LMASK64(arg2) <= 0)
-        do_sb(GET_OFFSET(arg2, 7), (uint8_t)arg1, mem_idx);
+        do_sb(env, GET_OFFSET(arg2, 7), (uint8_t)arg1, mem_idx);
 }
 
-void helper_sdr(target_ulong arg1, target_ulong arg2, int mem_idx)
+void helper_sdr(CPUMIPSState *env, target_ulong arg1, target_ulong arg2,
+                int mem_idx)
 {
-    do_sb(arg2, (uint8_t)arg1, mem_idx);
+    do_sb(env, arg2, (uint8_t)arg1, mem_idx);
 
     if (GET_LMASK64(arg2) >= 1)
-        do_sb(GET_OFFSET(arg2, -1), (uint8_t)(arg1 >> 8), mem_idx);
+        do_sb(env, GET_OFFSET(arg2, -1), (uint8_t)(arg1 >> 8), mem_idx);
 
     if (GET_LMASK64(arg2) >= 2)
-        do_sb(GET_OFFSET(arg2, -2), (uint8_t)(arg1 >> 16), mem_idx);
+        do_sb(env, GET_OFFSET(arg2, -2), (uint8_t)(arg1 >> 16), mem_idx);
 
     if (GET_LMASK64(arg2) >= 3)
-        do_sb(GET_OFFSET(arg2, -3), (uint8_t)(arg1 >> 24), mem_idx);
+        do_sb(env, GET_OFFSET(arg2, -3), (uint8_t)(arg1 >> 24), mem_idx);
 
     if (GET_LMASK64(arg2) >= 4)
-        do_sb(GET_OFFSET(arg2, -4), (uint8_t)(arg1 >> 32), mem_idx);
+        do_sb(env, GET_OFFSET(arg2, -4), (uint8_t)(arg1 >> 32), mem_idx);
 
     if (GET_LMASK64(arg2) >= 5)
-        do_sb(GET_OFFSET(arg2, -5), (uint8_t)(arg1 >> 40), mem_idx);
+        do_sb(env, GET_OFFSET(arg2, -5), (uint8_t)(arg1 >> 40), mem_idx);
 
     if (GET_LMASK64(arg2) >= 6)
-        do_sb(GET_OFFSET(arg2, -6), (uint8_t)(arg1 >> 48), mem_idx);
+        do_sb(env, GET_OFFSET(arg2, -6), (uint8_t)(arg1 >> 48), mem_idx);
 
     if (GET_LMASK64(arg2) == 7)
-        do_sb(GET_OFFSET(arg2, -7), (uint8_t)(arg1 >> 56), mem_idx);
+        do_sb(env, GET_OFFSET(arg2, -7), (uint8_t)(arg1 >> 56), mem_idx);
 }
 #endif /* TARGET_MIPS64 */
 
 static const int multiple_regs[] = { 16, 17, 18, 19, 20, 21, 22, 23, 30 };
 
-void helper_lwm (target_ulong addr, target_ulong reglist, uint32_t mem_idx)
+void helper_lwm(CPUMIPSState *env, target_ulong addr, target_ulong reglist,
+                uint32_t mem_idx)
 {
     target_ulong base_reglist = reglist & 0xf;
     target_ulong do_r31 = reglist & 0x10;
-#ifdef CONFIG_USER_ONLY
-#undef ldfun
-#define ldfun ldl_raw
-#else
-    uint32_t (*ldfun)(target_ulong);
-
-    switch (mem_idx)
-    {
-    case 0: ldfun = ldl_kernel; break;
-    case 1: ldfun = ldl_super; break;
-    default:
-    case 2: ldfun = ldl_user; break;
-    }
-#endif
 
     if (base_reglist > 0 && base_reglist <= ARRAY_SIZE (multiple_regs)) {
         target_ulong i;
 
         for (i = 0; i < base_reglist; i++) {
-            env->active_tc.gpr[multiple_regs[i]] = (target_long) ldfun(addr);
+            env->active_tc.gpr[multiple_regs[i]] =
+                (target_long)do_lw(env, addr, mem_idx);
             addr += 4;
         }
     }
 
     if (do_r31) {
-        env->active_tc.gpr[31] = (target_long) ldfun(addr);
+        env->active_tc.gpr[31] = (target_long)do_lw(env, addr, mem_idx);
     }
 }
 
-void helper_swm (target_ulong addr, target_ulong reglist, uint32_t mem_idx)
+void helper_swm(CPUMIPSState *env, target_ulong addr, target_ulong reglist,
+                uint32_t mem_idx)
 {
     target_ulong base_reglist = reglist & 0xf;
     target_ulong do_r31 = reglist & 0x10;
-#ifdef CONFIG_USER_ONLY
-#undef stfun
-#define stfun stl_raw
-#else
-    void (*stfun)(target_ulong, uint32_t);
-
-    switch (mem_idx)
-    {
-    case 0: stfun = stl_kernel; break;
-    case 1: stfun = stl_super; break;
-     default:
-    case 2: stfun = stl_user; break;
-    }
-#endif
 
     if (base_reglist > 0 && base_reglist <= ARRAY_SIZE (multiple_regs)) {
         target_ulong i;
 
         for (i = 0; i < base_reglist; i++) {
-            stfun(addr, env->active_tc.gpr[multiple_regs[i]]);
+            do_sw(env, addr, env->active_tc.gpr[multiple_regs[i]], mem_idx);
             addr += 4;
         }
     }
 
     if (do_r31) {
-        stfun(addr, env->active_tc.gpr[31]);
+        do_sw(env, addr, env->active_tc.gpr[31], mem_idx);
     }
 }
 
 #if defined(TARGET_MIPS64)
-void helper_ldm (target_ulong addr, target_ulong reglist, uint32_t mem_idx)
+void helper_ldm(CPUMIPSState *env, target_ulong addr, target_ulong reglist,
+                uint32_t mem_idx)
 {
     target_ulong base_reglist = reglist & 0xf;
     target_ulong do_r31 = reglist & 0x10;
-#ifdef CONFIG_USER_ONLY
-#undef ldfun
-#define ldfun ldq_raw
-#else
-    uint64_t (*ldfun)(target_ulong);
-
-    switch (mem_idx)
-    {
-    case 0: ldfun = ldq_kernel; break;
-    case 1: ldfun = ldq_super; break;
-    default:
-    case 2: ldfun = ldq_user; break;
-    }
-#endif
 
     if (base_reglist > 0 && base_reglist <= ARRAY_SIZE (multiple_regs)) {
         target_ulong i;
 
         for (i = 0; i < base_reglist; i++) {
-            env->active_tc.gpr[multiple_regs[i]] = ldfun(addr);
+            env->active_tc.gpr[multiple_regs[i]] = do_ld(env, addr, mem_idx);
             addr += 8;
         }
     }
 
     if (do_r31) {
-        env->active_tc.gpr[31] = ldfun(addr);
+        env->active_tc.gpr[31] = do_ld(env, addr, mem_idx);
     }
 }
 
-void helper_sdm (target_ulong addr, target_ulong reglist, uint32_t mem_idx)
+void helper_sdm(CPUMIPSState *env, target_ulong addr, target_ulong reglist,
+                uint32_t mem_idx)
 {
     target_ulong base_reglist = reglist & 0xf;
     target_ulong do_r31 = reglist & 0x10;
-#ifdef CONFIG_USER_ONLY
-#undef stfun
-#define stfun stq_raw
-#else
-    void (*stfun)(target_ulong, uint64_t);
-
-    switch (mem_idx)
-    {
-    case 0: stfun = stq_kernel; break;
-    case 1: stfun = stq_super; break;
-     default:
-    case 2: stfun = stq_user; break;
-    }
-#endif
 
     if (base_reglist > 0 && base_reglist <= ARRAY_SIZE (multiple_regs)) {
         target_ulong i;
 
         for (i = 0; i < base_reglist; i++) {
-            stfun(addr, env->active_tc.gpr[multiple_regs[i]]);
+            do_sd(env, addr, env->active_tc.gpr[multiple_regs[i]], mem_idx);
             addr += 8;
         }
     }
 
     if (do_r31) {
-        stfun(addr, env->active_tc.gpr[31]);
+        do_sd(env, addr, env->active_tc.gpr[31], mem_idx);
     }
 }
 #endif
 
 #ifndef CONFIG_USER_ONLY
 /* SMP helpers.  */
-static int mips_vpe_is_wfi(CPUMIPSState *c)
+static bool mips_vpe_is_wfi(MIPSCPU *c)
 {
+    CPUMIPSState *env = &c->env;
+
     /* If the VPE is halted but otherwise active, it means it's waiting for
        an interrupt.  */
-    return c->halted && mips_vpe_active(c);
+    return env->halted && mips_vpe_active(env);
 }
 
 static inline void mips_vpe_wake(CPUMIPSState *c)
@@ -741,27 +549,33 @@ static inline void mips_vpe_wake(CPUMIPSState *c)
     cpu_interrupt(c, CPU_INTERRUPT_WAKE);
 }
 
-static inline void mips_vpe_sleep(CPUMIPSState *c)
+static inline void mips_vpe_sleep(MIPSCPU *cpu)
 {
+    CPUMIPSState *c = &cpu->env;
+
     /* The VPE was shut off, really go to bed.
        Reset any old _WAKE requests.  */
     c->halted = 1;
     cpu_reset_interrupt(c, CPU_INTERRUPT_WAKE);
 }
 
-static inline void mips_tc_wake(CPUMIPSState *c, int tc)
+static inline void mips_tc_wake(MIPSCPU *cpu, int tc)
 {
+    CPUMIPSState *c = &cpu->env;
+
     /* FIXME: TC reschedule.  */
-    if (mips_vpe_active(c) && !mips_vpe_is_wfi(c)) {
+    if (mips_vpe_active(c) && !mips_vpe_is_wfi(cpu)) {
         mips_vpe_wake(c);
     }
 }
 
-static inline void mips_tc_sleep(CPUMIPSState *c, int tc)
+static inline void mips_tc_sleep(MIPSCPU *cpu, int tc)
 {
+    CPUMIPSState *c = &cpu->env;
+
     /* FIXME: TC reschedule.  */
     if (!mips_vpe_active(c)) {
-        mips_vpe_sleep(c);
+        mips_vpe_sleep(cpu);
     }
 }
 
@@ -772,7 +586,7 @@ static inline void mips_tc_sleep(CPUMIPSState *c, int tc)
    FIXME: This code assumes that all VPEs have the same number of TCs,
           which depends on runtime setup. Can probably be fixed by
           walking the list of CPUMIPSStates.  */
-static CPUMIPSState *mips_cpu_map_tc(int *tc)
+static CPUMIPSState *mips_cpu_map_tc(CPUMIPSState *env, int *tc)
 {
     CPUMIPSState *other;
     int vpe_idx, nr_threads = env->nr_threads;
@@ -799,7 +613,7 @@ static CPUMIPSState *mips_cpu_map_tc(int *tc)
    These helper call synchronizes the regs for a given cpu.  */
 
 /* Called for updates to CP0_Status.  */
-static void sync_c0_status(CPUMIPSState *cpu, int tc)
+static void sync_c0_status(CPUMIPSState *env, CPUMIPSState *cpu, int tc)
 {
     int32_t tcstatus, *tcst;
     uint32_t v = cpu->CP0_Status;
@@ -834,7 +648,8 @@ static void sync_c0_status(CPUMIPSState *cpu, int tc)
 }
 
 /* Called for updates to CP0_TCStatus.  */
-static void sync_c0_tcstatus(CPUMIPSState *cpu, int tc, target_ulong v)
+static void sync_c0_tcstatus(CPUMIPSState *cpu, int tc,
+                             target_ulong v)
 {
     uint32_t status;
     uint32_t tcu, tmx, tasid, tksu;
@@ -883,35 +698,35 @@ static void sync_c0_entryhi(CPUMIPSState *cpu, int tc)
 }
 
 /* CP0 helpers */
-target_ulong helper_mfc0_mvpcontrol (void)
+target_ulong helper_mfc0_mvpcontrol(CPUMIPSState *env)
 {
     return env->mvp->CP0_MVPControl;
 }
 
-target_ulong helper_mfc0_mvpconf0 (void)
+target_ulong helper_mfc0_mvpconf0(CPUMIPSState *env)
 {
     return env->mvp->CP0_MVPConf0;
 }
 
-target_ulong helper_mfc0_mvpconf1 (void)
+target_ulong helper_mfc0_mvpconf1(CPUMIPSState *env)
 {
     return env->mvp->CP0_MVPConf1;
 }
 
-target_ulong helper_mfc0_random (void)
+target_ulong helper_mfc0_random(CPUMIPSState *env)
 {
     return (int32_t)cpu_mips_get_random(env);
 }
 
-target_ulong helper_mfc0_tcstatus (void)
+target_ulong helper_mfc0_tcstatus(CPUMIPSState *env)
 {
     return env->active_tc.CP0_TCStatus;
 }
 
-target_ulong helper_mftc0_tcstatus(void)
+target_ulong helper_mftc0_tcstatus(CPUMIPSState *env)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
-    CPUMIPSState *other = mips_cpu_map_tc(&other_tc);
+    CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
 
     if (other_tc == other->current_tc)
         return other->active_tc.CP0_TCStatus;
@@ -919,15 +734,15 @@ target_ulong helper_mftc0_tcstatus(void)
         return other->tcs[other_tc].CP0_TCStatus;
 }
 
-target_ulong helper_mfc0_tcbind (void)
+target_ulong helper_mfc0_tcbind(CPUMIPSState *env)
 {
     return env->active_tc.CP0_TCBind;
 }
 
-target_ulong helper_mftc0_tcbind(void)
+target_ulong helper_mftc0_tcbind(CPUMIPSState *env)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
-    CPUMIPSState *other = mips_cpu_map_tc(&other_tc);
+    CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
 
     if (other_tc == other->current_tc)
         return other->active_tc.CP0_TCBind;
@@ -935,15 +750,15 @@ target_ulong helper_mftc0_tcbind(void)
         return other->tcs[other_tc].CP0_TCBind;
 }
 
-target_ulong helper_mfc0_tcrestart (void)
+target_ulong helper_mfc0_tcrestart(CPUMIPSState *env)
 {
     return env->active_tc.PC;
 }
 
-target_ulong helper_mftc0_tcrestart(void)
+target_ulong helper_mftc0_tcrestart(CPUMIPSState *env)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
-    CPUMIPSState *other = mips_cpu_map_tc(&other_tc);
+    CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
 
     if (other_tc == other->current_tc)
         return other->active_tc.PC;
@@ -951,15 +766,15 @@ target_ulong helper_mftc0_tcrestart(void)
         return other->tcs[other_tc].PC;
 }
 
-target_ulong helper_mfc0_tchalt (void)
+target_ulong helper_mfc0_tchalt(CPUMIPSState *env)
 {
     return env->active_tc.CP0_TCHalt;
 }
 
-target_ulong helper_mftc0_tchalt(void)
+target_ulong helper_mftc0_tchalt(CPUMIPSState *env)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
-    CPUMIPSState *other = mips_cpu_map_tc(&other_tc);
+    CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
 
     if (other_tc == other->current_tc)
         return other->active_tc.CP0_TCHalt;
@@ -967,15 +782,15 @@ target_ulong helper_mftc0_tchalt(void)
         return other->tcs[other_tc].CP0_TCHalt;
 }
 
-target_ulong helper_mfc0_tccontext (void)
+target_ulong helper_mfc0_tccontext(CPUMIPSState *env)
 {
     return env->active_tc.CP0_TCContext;
 }
 
-target_ulong helper_mftc0_tccontext(void)
+target_ulong helper_mftc0_tccontext(CPUMIPSState *env)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
-    CPUMIPSState *other = mips_cpu_map_tc(&other_tc);
+    CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
 
     if (other_tc == other->current_tc)
         return other->active_tc.CP0_TCContext;
@@ -983,15 +798,15 @@ target_ulong helper_mftc0_tccontext(void)
         return other->tcs[other_tc].CP0_TCContext;
 }
 
-target_ulong helper_mfc0_tcschedule (void)
+target_ulong helper_mfc0_tcschedule(CPUMIPSState *env)
 {
     return env->active_tc.CP0_TCSchedule;
 }
 
-target_ulong helper_mftc0_tcschedule(void)
+target_ulong helper_mftc0_tcschedule(CPUMIPSState *env)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
-    CPUMIPSState *other = mips_cpu_map_tc(&other_tc);
+    CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
 
     if (other_tc == other->current_tc)
         return other->active_tc.CP0_TCSchedule;
@@ -999,15 +814,15 @@ target_ulong helper_mftc0_tcschedule(void)
         return other->tcs[other_tc].CP0_TCSchedule;
 }
 
-target_ulong helper_mfc0_tcschefback (void)
+target_ulong helper_mfc0_tcschefback(CPUMIPSState *env)
 {
     return env->active_tc.CP0_TCScheFBack;
 }
 
-target_ulong helper_mftc0_tcschefback(void)
+target_ulong helper_mftc0_tcschefback(CPUMIPSState *env)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
-    CPUMIPSState *other = mips_cpu_map_tc(&other_tc);
+    CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
 
     if (other_tc == other->current_tc)
         return other->active_tc.CP0_TCScheFBack;
@@ -1015,24 +830,24 @@ target_ulong helper_mftc0_tcschefback(void)
         return other->tcs[other_tc].CP0_TCScheFBack;
 }
 
-target_ulong helper_mfc0_count (void)
+target_ulong helper_mfc0_count(CPUMIPSState *env)
 {
     return (int32_t)cpu_mips_get_count(env);
 }
 
-target_ulong helper_mftc0_entryhi(void)
+target_ulong helper_mftc0_entryhi(CPUMIPSState *env)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
-    CPUMIPSState *other = mips_cpu_map_tc(&other_tc);
+    CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
 
     return other->CP0_EntryHi;
 }
 
-target_ulong helper_mftc0_cause(void)
+target_ulong helper_mftc0_cause(CPUMIPSState *env)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
     int32_t tccause;
-    CPUMIPSState *other = mips_cpu_map_tc(&other_tc);
+    CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
 
     if (other_tc == other->current_tc) {
         tccause = other->CP0_Cause;
@@ -1043,30 +858,30 @@ target_ulong helper_mftc0_cause(void)
     return tccause;
 }
 
-target_ulong helper_mftc0_status(void)
+target_ulong helper_mftc0_status(CPUMIPSState *env)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
-    CPUMIPSState *other = mips_cpu_map_tc(&other_tc);
+    CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
 
     return other->CP0_Status;
 }
 
-target_ulong helper_mfc0_lladdr (void)
+target_ulong helper_mfc0_lladdr(CPUMIPSState *env)
 {
     return (int32_t)(env->lladdr >> env->CP0_LLAddr_shift);
 }
 
-target_ulong helper_mfc0_watchlo (uint32_t sel)
+target_ulong helper_mfc0_watchlo(CPUMIPSState *env, uint32_t sel)
 {
     return (int32_t)env->CP0_WatchLo[sel];
 }
 
-target_ulong helper_mfc0_watchhi (uint32_t sel)
+target_ulong helper_mfc0_watchhi(CPUMIPSState *env, uint32_t sel)
 {
     return env->CP0_WatchHi[sel];
 }
 
-target_ulong helper_mfc0_debug (void)
+target_ulong helper_mfc0_debug(CPUMIPSState *env)
 {
     target_ulong t0 = env->CP0_Debug;
     if (env->hflags & MIPS_HFLAG_DM)
@@ -1075,11 +890,11 @@ target_ulong helper_mfc0_debug (void)
     return t0;
 }
 
-target_ulong helper_mftc0_debug(void)
+target_ulong helper_mftc0_debug(CPUMIPSState *env)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
     int32_t tcstatus;
-    CPUMIPSState *other = mips_cpu_map_tc(&other_tc);
+    CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
 
     if (other_tc == other->current_tc)
         tcstatus = other->active_tc.CP0_Debug_tcstatus;
@@ -1092,43 +907,43 @@ target_ulong helper_mftc0_debug(void)
 }
 
 #if defined(TARGET_MIPS64)
-target_ulong helper_dmfc0_tcrestart (void)
+target_ulong helper_dmfc0_tcrestart(CPUMIPSState *env)
 {
     return env->active_tc.PC;
 }
 
-target_ulong helper_dmfc0_tchalt (void)
+target_ulong helper_dmfc0_tchalt(CPUMIPSState *env)
 {
     return env->active_tc.CP0_TCHalt;
 }
 
-target_ulong helper_dmfc0_tccontext (void)
+target_ulong helper_dmfc0_tccontext(CPUMIPSState *env)
 {
     return env->active_tc.CP0_TCContext;
 }
 
-target_ulong helper_dmfc0_tcschedule (void)
+target_ulong helper_dmfc0_tcschedule(CPUMIPSState *env)
 {
     return env->active_tc.CP0_TCSchedule;
 }
 
-target_ulong helper_dmfc0_tcschefback (void)
+target_ulong helper_dmfc0_tcschefback(CPUMIPSState *env)
 {
     return env->active_tc.CP0_TCScheFBack;
 }
 
-target_ulong helper_dmfc0_lladdr (void)
+target_ulong helper_dmfc0_lladdr(CPUMIPSState *env)
 {
     return env->lladdr >> env->CP0_LLAddr_shift;
 }
 
-target_ulong helper_dmfc0_watchlo (uint32_t sel)
+target_ulong helper_dmfc0_watchlo(CPUMIPSState *env, uint32_t sel)
 {
     return env->CP0_WatchLo[sel];
 }
 #endif /* TARGET_MIPS64 */
 
-void helper_mtc0_index (target_ulong arg1)
+void helper_mtc0_index(CPUMIPSState *env, target_ulong arg1)
 {
     int num = 1;
     unsigned int tmp = env->tlb->nb_tlb;
@@ -1140,7 +955,7 @@ void helper_mtc0_index (target_ulong arg1)
     env->CP0_Index = (env->CP0_Index & 0x80000000) | (arg1 & (num - 1));
 }
 
-void helper_mtc0_mvpcontrol (target_ulong arg1)
+void helper_mtc0_mvpcontrol(CPUMIPSState *env, target_ulong arg1)
 {
     uint32_t mask = 0;
     uint32_t newval;
@@ -1157,7 +972,7 @@ void helper_mtc0_mvpcontrol (target_ulong arg1)
     env->mvp->CP0_MVPControl = newval;
 }
 
-void helper_mtc0_vpecontrol (target_ulong arg1)
+void helper_mtc0_vpecontrol(CPUMIPSState *env, target_ulong arg1)
 {
     uint32_t mask;
     uint32_t newval;
@@ -1174,10 +989,10 @@ void helper_mtc0_vpecontrol (target_ulong arg1)
     env->CP0_VPEControl = newval;
 }
 
-void helper_mttc0_vpecontrol(target_ulong arg1)
+void helper_mttc0_vpecontrol(CPUMIPSState *env, target_ulong arg1)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
-    CPUMIPSState *other = mips_cpu_map_tc(&other_tc);
+    CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
     uint32_t mask;
     uint32_t newval;
 
@@ -1190,23 +1005,23 @@ void helper_mttc0_vpecontrol(target_ulong arg1)
     other->CP0_VPEControl = newval;
 }
 
-target_ulong helper_mftc0_vpecontrol(void)
+target_ulong helper_mftc0_vpecontrol(CPUMIPSState *env)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
-    CPUMIPSState *other = mips_cpu_map_tc(&other_tc);
+    CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
     /* FIXME: Mask away return zero on read bits.  */
     return other->CP0_VPEControl;
 }
 
-target_ulong helper_mftc0_vpeconf0(void)
+target_ulong helper_mftc0_vpeconf0(CPUMIPSState *env)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
-    CPUMIPSState *other = mips_cpu_map_tc(&other_tc);
+    CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
 
     return other->CP0_VPEConf0;
 }
 
-void helper_mtc0_vpeconf0 (target_ulong arg1)
+void helper_mtc0_vpeconf0(CPUMIPSState *env, target_ulong arg1)
 {
     uint32_t mask = 0;
     uint32_t newval;
@@ -1223,10 +1038,10 @@ void helper_mtc0_vpeconf0 (target_ulong arg1)
     env->CP0_VPEConf0 = newval;
 }
 
-void helper_mttc0_vpeconf0(target_ulong arg1)
+void helper_mttc0_vpeconf0(CPUMIPSState *env, target_ulong arg1)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
-    CPUMIPSState *other = mips_cpu_map_tc(&other_tc);
+    CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
     uint32_t mask = 0;
     uint32_t newval;
 
@@ -1237,7 +1052,7 @@ void helper_mttc0_vpeconf0(target_ulong arg1)
     other->CP0_VPEConf0 = newval;
 }
 
-void helper_mtc0_vpeconf1 (target_ulong arg1)
+void helper_mtc0_vpeconf1(CPUMIPSState *env, target_ulong arg1)
 {
     uint32_t mask = 0;
     uint32_t newval;
@@ -1255,25 +1070,25 @@ void helper_mtc0_vpeconf1 (target_ulong arg1)
     env->CP0_VPEConf1 = newval;
 }
 
-void helper_mtc0_yqmask (target_ulong arg1)
+void helper_mtc0_yqmask(CPUMIPSState *env, target_ulong arg1)
 {
     /* Yield qualifier inputs not implemented. */
     env->CP0_YQMask = 0x00000000;
 }
 
-void helper_mtc0_vpeopt (target_ulong arg1)
+void helper_mtc0_vpeopt(CPUMIPSState *env, target_ulong arg1)
 {
     env->CP0_VPEOpt = arg1 & 0x0000ffff;
 }
 
-void helper_mtc0_entrylo0 (target_ulong arg1)
+void helper_mtc0_entrylo0(CPUMIPSState *env, target_ulong arg1)
 {
     /* Large physaddr (PABITS) not implemented */
     /* 1k pages not implemented */
     env->CP0_EntryLo0 = arg1 & 0x3FFFFFFF;
 }
 
-void helper_mtc0_tcstatus (target_ulong arg1)
+void helper_mtc0_tcstatus(CPUMIPSState *env, target_ulong arg1)
 {
     uint32_t mask = env->CP0_TCStatus_rw_bitmask;
     uint32_t newval;
@@ -1284,10 +1099,10 @@ void helper_mtc0_tcstatus (target_ulong arg1)
     sync_c0_tcstatus(env, env->current_tc, newval);
 }
 
-void helper_mttc0_tcstatus (target_ulong arg1)
+void helper_mttc0_tcstatus(CPUMIPSState *env, target_ulong arg1)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
-    CPUMIPSState *other = mips_cpu_map_tc(&other_tc);
+    CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
 
     if (other_tc == other->current_tc)
         other->active_tc.CP0_TCStatus = arg1;
@@ -1296,7 +1111,7 @@ void helper_mttc0_tcstatus (target_ulong arg1)
     sync_c0_tcstatus(other, other_tc, arg1);
 }
 
-void helper_mtc0_tcbind (target_ulong arg1)
+void helper_mtc0_tcbind(CPUMIPSState *env, target_ulong arg1)
 {
     uint32_t mask = (1 << CP0TCBd_TBE);
     uint32_t newval;
@@ -1307,12 +1122,12 @@ void helper_mtc0_tcbind (target_ulong arg1)
     env->active_tc.CP0_TCBind = newval;
 }
 
-void helper_mttc0_tcbind (target_ulong arg1)
+void helper_mttc0_tcbind(CPUMIPSState *env, target_ulong arg1)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
     uint32_t mask = (1 << CP0TCBd_TBE);
     uint32_t newval;
-    CPUMIPSState *other = mips_cpu_map_tc(&other_tc);
+    CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
 
     if (other->mvp->CP0_MVPControl & (1 << CP0MVPCo_VPC))
         mask |= (1 << CP0TCBd_CurVPE);
@@ -1325,7 +1140,7 @@ void helper_mttc0_tcbind (target_ulong arg1)
     }
 }
 
-void helper_mtc0_tcrestart (target_ulong arg1)
+void helper_mtc0_tcrestart(CPUMIPSState *env, target_ulong arg1)
 {
     env->active_tc.PC = arg1;
     env->active_tc.CP0_TCStatus &= ~(1 << CP0TCSt_TDS);
@@ -1333,10 +1148,10 @@ void helper_mtc0_tcrestart (target_ulong arg1)
     /* MIPS16 not implemented. */
 }
 
-void helper_mttc0_tcrestart (target_ulong arg1)
+void helper_mttc0_tcrestart(CPUMIPSState *env, target_ulong arg1)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
-    CPUMIPSState *other = mips_cpu_map_tc(&other_tc);
+    CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
 
     if (other_tc == other->current_tc) {
         other->active_tc.PC = arg1;
@@ -1351,22 +1166,25 @@ void helper_mttc0_tcrestart (target_ulong arg1)
     }
 }
 
-void helper_mtc0_tchalt (target_ulong arg1)
+void helper_mtc0_tchalt(CPUMIPSState *env, target_ulong arg1)
 {
+    MIPSCPU *cpu = mips_env_get_cpu(env);
+
     env->active_tc.CP0_TCHalt = arg1 & 0x1;
 
     // TODO: Halt TC / Restart (if allocated+active) TC.
     if (env->active_tc.CP0_TCHalt & 1) {
-        mips_tc_sleep(env, env->current_tc);
+        mips_tc_sleep(cpu, env->current_tc);
     } else {
-        mips_tc_wake(env, env->current_tc);
+        mips_tc_wake(cpu, env->current_tc);
     }
 }
 
-void helper_mttc0_tchalt (target_ulong arg1)
+void helper_mttc0_tchalt(CPUMIPSState *env, target_ulong arg1)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
-    CPUMIPSState *other = mips_cpu_map_tc(&other_tc);
+    CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
+    MIPSCPU *other_cpu = mips_env_get_cpu(other);
 
     // TODO: Halt TC / Restart (if allocated+active) TC.
 
@@ -1376,21 +1194,21 @@ void helper_mttc0_tchalt (target_ulong arg1)
         other->tcs[other_tc].CP0_TCHalt = arg1;
 
     if (arg1 & 1) {
-        mips_tc_sleep(other, other_tc);
+        mips_tc_sleep(other_cpu, other_tc);
     } else {
-        mips_tc_wake(other, other_tc);
+        mips_tc_wake(other_cpu, other_tc);
     }
 }
 
-void helper_mtc0_tccontext (target_ulong arg1)
+void helper_mtc0_tccontext(CPUMIPSState *env, target_ulong arg1)
 {
     env->active_tc.CP0_TCContext = arg1;
 }
 
-void helper_mttc0_tccontext (target_ulong arg1)
+void helper_mttc0_tccontext(CPUMIPSState *env, target_ulong arg1)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
-    CPUMIPSState *other = mips_cpu_map_tc(&other_tc);
+    CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
 
     if (other_tc == other->current_tc)
         other->active_tc.CP0_TCContext = arg1;
@@ -1398,15 +1216,15 @@ void helper_mttc0_tccontext (target_ulong arg1)
         other->tcs[other_tc].CP0_TCContext = arg1;
 }
 
-void helper_mtc0_tcschedule (target_ulong arg1)
+void helper_mtc0_tcschedule(CPUMIPSState *env, target_ulong arg1)
 {
     env->active_tc.CP0_TCSchedule = arg1;
 }
 
-void helper_mttc0_tcschedule (target_ulong arg1)
+void helper_mttc0_tcschedule(CPUMIPSState *env, target_ulong arg1)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
-    CPUMIPSState *other = mips_cpu_map_tc(&other_tc);
+    CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
 
     if (other_tc == other->current_tc)
         other->active_tc.CP0_TCSchedule = arg1;
@@ -1414,15 +1232,15 @@ void helper_mttc0_tcschedule (target_ulong arg1)
         other->tcs[other_tc].CP0_TCSchedule = arg1;
 }
 
-void helper_mtc0_tcschefback (target_ulong arg1)
+void helper_mtc0_tcschefback(CPUMIPSState *env, target_ulong arg1)
 {
     env->active_tc.CP0_TCScheFBack = arg1;
 }
 
-void helper_mttc0_tcschefback (target_ulong arg1)
+void helper_mttc0_tcschefback(CPUMIPSState *env, target_ulong arg1)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
-    CPUMIPSState *other = mips_cpu_map_tc(&other_tc);
+    CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
 
     if (other_tc == other->current_tc)
         other->active_tc.CP0_TCScheFBack = arg1;
@@ -1430,25 +1248,25 @@ void helper_mttc0_tcschefback (target_ulong arg1)
         other->tcs[other_tc].CP0_TCScheFBack = arg1;
 }
 
-void helper_mtc0_entrylo1 (target_ulong arg1)
+void helper_mtc0_entrylo1(CPUMIPSState *env, target_ulong arg1)
 {
     /* Large physaddr (PABITS) not implemented */
     /* 1k pages not implemented */
     env->CP0_EntryLo1 = arg1 & 0x3FFFFFFF;
 }
 
-void helper_mtc0_context (target_ulong arg1)
+void helper_mtc0_context(CPUMIPSState *env, target_ulong arg1)
 {
     env->CP0_Context = (env->CP0_Context & 0x007FFFFF) | (arg1 & ~0x007FFFFF);
 }
 
-void helper_mtc0_pagemask (target_ulong arg1)
+void helper_mtc0_pagemask(CPUMIPSState *env, target_ulong arg1)
 {
     /* 1k pages not implemented */
     env->CP0_PageMask = arg1 & (0x1FFFFFFF & (TARGET_PAGE_MASK << 1));
 }
 
-void helper_mtc0_pagegrain (target_ulong arg1)
+void helper_mtc0_pagegrain(CPUMIPSState *env, target_ulong arg1)
 {
     /* SmartMIPS not implemented */
     /* Large physaddr (PABITS) not implemented */
@@ -1456,47 +1274,47 @@ void helper_mtc0_pagegrain (target_ulong arg1)
     env->CP0_PageGrain = 0;
 }
 
-void helper_mtc0_wired (target_ulong arg1)
+void helper_mtc0_wired(CPUMIPSState *env, target_ulong arg1)
 {
     env->CP0_Wired = arg1 % env->tlb->nb_tlb;
 }
 
-void helper_mtc0_srsconf0 (target_ulong arg1)
+void helper_mtc0_srsconf0(CPUMIPSState *env, target_ulong arg1)
 {
     env->CP0_SRSConf0 |= arg1 & env->CP0_SRSConf0_rw_bitmask;
 }
 
-void helper_mtc0_srsconf1 (target_ulong arg1)
+void helper_mtc0_srsconf1(CPUMIPSState *env, target_ulong arg1)
 {
     env->CP0_SRSConf1 |= arg1 & env->CP0_SRSConf1_rw_bitmask;
 }
 
-void helper_mtc0_srsconf2 (target_ulong arg1)
+void helper_mtc0_srsconf2(CPUMIPSState *env, target_ulong arg1)
 {
     env->CP0_SRSConf2 |= arg1 & env->CP0_SRSConf2_rw_bitmask;
 }
 
-void helper_mtc0_srsconf3 (target_ulong arg1)
+void helper_mtc0_srsconf3(CPUMIPSState *env, target_ulong arg1)
 {
     env->CP0_SRSConf3 |= arg1 & env->CP0_SRSConf3_rw_bitmask;
 }
 
-void helper_mtc0_srsconf4 (target_ulong arg1)
+void helper_mtc0_srsconf4(CPUMIPSState *env, target_ulong arg1)
 {
     env->CP0_SRSConf4 |= arg1 & env->CP0_SRSConf4_rw_bitmask;
 }
 
-void helper_mtc0_hwrena (target_ulong arg1)
+void helper_mtc0_hwrena(CPUMIPSState *env, target_ulong arg1)
 {
     env->CP0_HWREna = arg1 & 0x0000000F;
 }
 
-void helper_mtc0_count (target_ulong arg1)
+void helper_mtc0_count(CPUMIPSState *env, target_ulong arg1)
 {
     cpu_mips_store_count(env, arg1);
 }
 
-void helper_mtc0_entryhi (target_ulong arg1)
+void helper_mtc0_entryhi(CPUMIPSState *env, target_ulong arg1)
 {
     target_ulong old, val;
 
@@ -1515,21 +1333,21 @@ void helper_mtc0_entryhi (target_ulong arg1)
         cpu_mips_tlb_flush(env, 1);
 }
 
-void helper_mttc0_entryhi(target_ulong arg1)
+void helper_mttc0_entryhi(CPUMIPSState *env, target_ulong arg1)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
-    CPUMIPSState *other = mips_cpu_map_tc(&other_tc);
+    CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
 
     other->CP0_EntryHi = arg1;
     sync_c0_entryhi(other, other_tc);
 }
 
-void helper_mtc0_compare (target_ulong arg1)
+void helper_mtc0_compare(CPUMIPSState *env, target_ulong arg1)
 {
     cpu_mips_store_compare(env, arg1);
 }
 
-void helper_mtc0_status (target_ulong arg1)
+void helper_mtc0_status(CPUMIPSState *env, target_ulong arg1)
 {
     uint32_t val, old;
     uint32_t mask = env->CP0_Status_rw_bitmask;
@@ -1538,7 +1356,7 @@ void helper_mtc0_status (target_ulong arg1)
     old = env->CP0_Status;
     env->CP0_Status = (env->CP0_Status & ~mask) | val;
     if (env->CP0_Config3 & (1 << CP0C3_MT)) {
-        sync_c0_status(env, env->current_tc);
+        sync_c0_status(env, env, env->current_tc);
     } else {
         compute_hflags(env);
     }
@@ -1557,22 +1375,22 @@ void helper_mtc0_status (target_ulong arg1)
     }
 }
 
-void helper_mttc0_status(target_ulong arg1)
+void helper_mttc0_status(CPUMIPSState *env, target_ulong arg1)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
-    CPUMIPSState *other = mips_cpu_map_tc(&other_tc);
+    CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
 
     other->CP0_Status = arg1 & ~0xf1000018;
-    sync_c0_status(other, other_tc);
+    sync_c0_status(env, other, other_tc);
 }
 
-void helper_mtc0_intctl (target_ulong arg1)
+void helper_mtc0_intctl(CPUMIPSState *env, target_ulong arg1)
 {
     /* vectored interrupts not implemented, no performance counters. */
     env->CP0_IntCtl = (env->CP0_IntCtl & ~0x000003e0) | (arg1 & 0x000003e0);
 }
 
-void helper_mtc0_srsctl (target_ulong arg1)
+void helper_mtc0_srsctl(CPUMIPSState *env, target_ulong arg1)
 {
     uint32_t mask = (0xf << CP0SRSCtl_ESS) | (0xf << CP0SRSCtl_PSS);
     env->CP0_SRSCtl = (env->CP0_SRSCtl & ~mask) | (arg1 & mask);
@@ -1606,52 +1424,52 @@ static void mtc0_cause(CPUMIPSState *cpu, target_ulong arg1)
     }
 }
 
-void helper_mtc0_cause(target_ulong arg1)
+void helper_mtc0_cause(CPUMIPSState *env, target_ulong arg1)
 {
     mtc0_cause(env, arg1);
 }
 
-void helper_mttc0_cause(target_ulong arg1)
+void helper_mttc0_cause(CPUMIPSState *env, target_ulong arg1)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
-    CPUMIPSState *other = mips_cpu_map_tc(&other_tc);
+    CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
 
     mtc0_cause(other, arg1);
 }
 
-target_ulong helper_mftc0_epc(void)
+target_ulong helper_mftc0_epc(CPUMIPSState *env)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
-    CPUMIPSState *other = mips_cpu_map_tc(&other_tc);
+    CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
 
     return other->CP0_EPC;
 }
 
-target_ulong helper_mftc0_ebase(void)
+target_ulong helper_mftc0_ebase(CPUMIPSState *env)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
-    CPUMIPSState *other = mips_cpu_map_tc(&other_tc);
+    CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
 
     return other->CP0_EBase;
 }
 
-void helper_mtc0_ebase (target_ulong arg1)
+void helper_mtc0_ebase(CPUMIPSState *env, target_ulong arg1)
 {
     /* vectored interrupts not implemented */
     env->CP0_EBase = (env->CP0_EBase & ~0x3FFFF000) | (arg1 & 0x3FFFF000);
 }
 
-void helper_mttc0_ebase(target_ulong arg1)
+void helper_mttc0_ebase(CPUMIPSState *env, target_ulong arg1)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
-    CPUMIPSState *other = mips_cpu_map_tc(&other_tc);
+    CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
     other->CP0_EBase = (other->CP0_EBase & ~0x3FFFF000) | (arg1 & 0x3FFFF000);
 }
 
-target_ulong helper_mftc0_configx(target_ulong idx)
+target_ulong helper_mftc0_configx(CPUMIPSState *env, target_ulong idx)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
-    CPUMIPSState *other = mips_cpu_map_tc(&other_tc);
+    CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
 
     switch (idx) {
     case 0: return other->CP0_Config0;
@@ -1667,49 +1485,49 @@ target_ulong helper_mftc0_configx(target_ulong idx)
     return 0;
 }
 
-void helper_mtc0_config0 (target_ulong arg1)
+void helper_mtc0_config0(CPUMIPSState *env, target_ulong arg1)
 {
     env->CP0_Config0 = (env->CP0_Config0 & 0x81FFFFF8) | (arg1 & 0x00000007);
 }
 
-void helper_mtc0_config2 (target_ulong arg1)
+void helper_mtc0_config2(CPUMIPSState *env, target_ulong arg1)
 {
     /* tertiary/secondary caches not implemented */
     env->CP0_Config2 = (env->CP0_Config2 & 0x8FFF0FFF);
 }
 
-void helper_mtc0_lladdr (target_ulong arg1)
+void helper_mtc0_lladdr(CPUMIPSState *env, target_ulong arg1)
 {
     target_long mask = env->CP0_LLAddr_rw_bitmask;
     arg1 = arg1 << env->CP0_LLAddr_shift;
     env->lladdr = (env->lladdr & ~mask) | (arg1 & mask);
 }
 
-void helper_mtc0_watchlo (target_ulong arg1, uint32_t sel)
+void helper_mtc0_watchlo(CPUMIPSState *env, target_ulong arg1, uint32_t sel)
 {
     /* Watch exceptions for instructions, data loads, data stores
        not implemented. */
     env->CP0_WatchLo[sel] = (arg1 & ~0x7);
 }
 
-void helper_mtc0_watchhi (target_ulong arg1, uint32_t sel)
+void helper_mtc0_watchhi(CPUMIPSState *env, target_ulong arg1, uint32_t sel)
 {
     env->CP0_WatchHi[sel] = (arg1 & 0x40FF0FF8);
     env->CP0_WatchHi[sel] &= ~(env->CP0_WatchHi[sel] & arg1 & 0x7);
 }
 
-void helper_mtc0_xcontext (target_ulong arg1)
+void helper_mtc0_xcontext(CPUMIPSState *env, target_ulong arg1)
 {
     target_ulong mask = (1ULL << (env->SEGBITS - 7)) - 1;
     env->CP0_XContext = (env->CP0_XContext & mask) | (arg1 & ~mask);
 }
 
-void helper_mtc0_framemask (target_ulong arg1)
+void helper_mtc0_framemask(CPUMIPSState *env, target_ulong arg1)
 {
     env->CP0_Framemask = arg1; /* XXX */
 }
 
-void helper_mtc0_debug (target_ulong arg1)
+void helper_mtc0_debug(CPUMIPSState *env, target_ulong arg1)
 {
     env->CP0_Debug = (env->CP0_Debug & 0x8C03FC1F) | (arg1 & 0x13300120);
     if (arg1 & (1 << CP0DB_DM))
@@ -1718,11 +1536,11 @@ void helper_mtc0_debug (target_ulong arg1)
         env->hflags &= ~MIPS_HFLAG_DM;
 }
 
-void helper_mttc0_debug(target_ulong arg1)
+void helper_mttc0_debug(CPUMIPSState *env, target_ulong arg1)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
     uint32_t val = arg1 & ((1 << CP0DB_SSt) | (1 << CP0DB_Halt));
-    CPUMIPSState *other = mips_cpu_map_tc(&other_tc);
+    CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
 
     /* XXX: Might be wrong, check with EJTAG spec. */
     if (other_tc == other->current_tc)
@@ -1734,36 +1552,36 @@ void helper_mttc0_debug(target_ulong arg1)
                      (arg1 & ~((1 << CP0DB_SSt) | (1 << CP0DB_Halt)));
 }
 
-void helper_mtc0_performance0 (target_ulong arg1)
+void helper_mtc0_performance0(CPUMIPSState *env, target_ulong arg1)
 {
     env->CP0_Performance0 = arg1 & 0x000007ff;
 }
 
-void helper_mtc0_taglo (target_ulong arg1)
+void helper_mtc0_taglo(CPUMIPSState *env, target_ulong arg1)
 {
     env->CP0_TagLo = arg1 & 0xFFFFFCF6;
 }
 
-void helper_mtc0_datalo (target_ulong arg1)
+void helper_mtc0_datalo(CPUMIPSState *env, target_ulong arg1)
 {
     env->CP0_DataLo = arg1; /* XXX */
 }
 
-void helper_mtc0_taghi (target_ulong arg1)
+void helper_mtc0_taghi(CPUMIPSState *env, target_ulong arg1)
 {
     env->CP0_TagHi = arg1; /* XXX */
 }
 
-void helper_mtc0_datahi (target_ulong arg1)
+void helper_mtc0_datahi(CPUMIPSState *env, target_ulong arg1)
 {
     env->CP0_DataHi = arg1; /* XXX */
 }
 
 /* MIPS MT functions */
-target_ulong helper_mftgpr(uint32_t sel)
+target_ulong helper_mftgpr(CPUMIPSState *env, uint32_t sel)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
-    CPUMIPSState *other = mips_cpu_map_tc(&other_tc);
+    CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
 
     if (other_tc == other->current_tc)
         return other->active_tc.gpr[sel];
@@ -1771,10 +1589,10 @@ target_ulong helper_mftgpr(uint32_t sel)
         return other->tcs[other_tc].gpr[sel];
 }
 
-target_ulong helper_mftlo(uint32_t sel)
+target_ulong helper_mftlo(CPUMIPSState *env, uint32_t sel)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
-    CPUMIPSState *other = mips_cpu_map_tc(&other_tc);
+    CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
 
     if (other_tc == other->current_tc)
         return other->active_tc.LO[sel];
@@ -1782,10 +1600,10 @@ target_ulong helper_mftlo(uint32_t sel)
         return other->tcs[other_tc].LO[sel];
 }
 
-target_ulong helper_mfthi(uint32_t sel)
+target_ulong helper_mfthi(CPUMIPSState *env, uint32_t sel)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
-    CPUMIPSState *other = mips_cpu_map_tc(&other_tc);
+    CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
 
     if (other_tc == other->current_tc)
         return other->active_tc.HI[sel];
@@ -1793,10 +1611,10 @@ target_ulong helper_mfthi(uint32_t sel)
         return other->tcs[other_tc].HI[sel];
 }
 
-target_ulong helper_mftacx(uint32_t sel)
+target_ulong helper_mftacx(CPUMIPSState *env, uint32_t sel)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
-    CPUMIPSState *other = mips_cpu_map_tc(&other_tc);
+    CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
 
     if (other_tc == other->current_tc)
         return other->active_tc.ACX[sel];
@@ -1804,10 +1622,10 @@ target_ulong helper_mftacx(uint32_t sel)
         return other->tcs[other_tc].ACX[sel];
 }
 
-target_ulong helper_mftdsp(void)
+target_ulong helper_mftdsp(CPUMIPSState *env)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
-    CPUMIPSState *other = mips_cpu_map_tc(&other_tc);
+    CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
 
     if (other_tc == other->current_tc)
         return other->active_tc.DSPControl;
@@ -1815,10 +1633,10 @@ target_ulong helper_mftdsp(void)
         return other->tcs[other_tc].DSPControl;
 }
 
-void helper_mttgpr(target_ulong arg1, uint32_t sel)
+void helper_mttgpr(CPUMIPSState *env, target_ulong arg1, uint32_t sel)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
-    CPUMIPSState *other = mips_cpu_map_tc(&other_tc);
+    CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
 
     if (other_tc == other->current_tc)
         other->active_tc.gpr[sel] = arg1;
@@ -1826,10 +1644,10 @@ void helper_mttgpr(target_ulong arg1, uint32_t sel)
         other->tcs[other_tc].gpr[sel] = arg1;
 }
 
-void helper_mttlo(target_ulong arg1, uint32_t sel)
+void helper_mttlo(CPUMIPSState *env, target_ulong arg1, uint32_t sel)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
-    CPUMIPSState *other = mips_cpu_map_tc(&other_tc);
+    CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
 
     if (other_tc == other->current_tc)
         other->active_tc.LO[sel] = arg1;
@@ -1837,10 +1655,10 @@ void helper_mttlo(target_ulong arg1, uint32_t sel)
         other->tcs[other_tc].LO[sel] = arg1;
 }
 
-void helper_mtthi(target_ulong arg1, uint32_t sel)
+void helper_mtthi(CPUMIPSState *env, target_ulong arg1, uint32_t sel)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
-    CPUMIPSState *other = mips_cpu_map_tc(&other_tc);
+    CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
 
     if (other_tc == other->current_tc)
         other->active_tc.HI[sel] = arg1;
@@ -1848,10 +1666,10 @@ void helper_mtthi(target_ulong arg1, uint32_t sel)
         other->tcs[other_tc].HI[sel] = arg1;
 }
 
-void helper_mttacx(target_ulong arg1, uint32_t sel)
+void helper_mttacx(CPUMIPSState *env, target_ulong arg1, uint32_t sel)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
-    CPUMIPSState *other = mips_cpu_map_tc(&other_tc);
+    CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
 
     if (other_tc == other->current_tc)
         other->active_tc.ACX[sel] = arg1;
@@ -1859,10 +1677,10 @@ void helper_mttacx(target_ulong arg1, uint32_t sel)
         other->tcs[other_tc].ACX[sel] = arg1;
 }
 
-void helper_mttdsp(target_ulong arg1)
+void helper_mttdsp(CPUMIPSState *env, target_ulong arg1)
 {
     int other_tc = env->CP0_VPEControl & (0xff << CP0VPECo_TargTC);
-    CPUMIPSState *other = mips_cpu_map_tc(&other_tc);
+    CPUMIPSState *other = mips_cpu_map_tc(env, &other_tc);
 
     if (other_tc == other->current_tc)
         other->active_tc.DSPControl = arg1;
@@ -1883,37 +1701,41 @@ target_ulong helper_emt(void)
     return 0;
 }
 
-target_ulong helper_dvpe(void)
+target_ulong helper_dvpe(CPUMIPSState *env)
 {
-    CPUMIPSState *other_cpu = first_cpu;
+    CPUMIPSState *other_cpu_env = first_cpu;
     target_ulong prev = env->mvp->CP0_MVPControl;
 
     do {
         /* Turn off all VPEs except the one executing the dvpe.  */
-        if (other_cpu != env) {
-            other_cpu->mvp->CP0_MVPControl &= ~(1 << CP0MVPCo_EVP);
+        if (other_cpu_env != env) {
+            MIPSCPU *other_cpu = mips_env_get_cpu(other_cpu_env);
+
+            other_cpu_env->mvp->CP0_MVPControl &= ~(1 << CP0MVPCo_EVP);
             mips_vpe_sleep(other_cpu);
         }
-        other_cpu = other_cpu->next_cpu;
-    } while (other_cpu);
+        other_cpu_env = other_cpu_env->next_cpu;
+    } while (other_cpu_env);
     return prev;
 }
 
-target_ulong helper_evpe(void)
+target_ulong helper_evpe(CPUMIPSState *env)
 {
-    CPUMIPSState *other_cpu = first_cpu;
+    CPUMIPSState *other_cpu_env = first_cpu;
     target_ulong prev = env->mvp->CP0_MVPControl;
 
     do {
-        if (other_cpu != env
-           /* If the VPE is WFI, don't disturb its sleep.  */
-           && !mips_vpe_is_wfi(other_cpu)) {
+        MIPSCPU *other_cpu = mips_env_get_cpu(other_cpu_env);
+
+        if (other_cpu_env != env
+            /* If the VPE is WFI, don't disturb its sleep.  */
+            && !mips_vpe_is_wfi(other_cpu)) {
             /* Enable the VPE.  */
-            other_cpu->mvp->CP0_MVPControl |= (1 << CP0MVPCo_EVP);
-            mips_vpe_wake(other_cpu); /* And wake it up.  */
+            other_cpu_env->mvp->CP0_MVPControl |= (1 << CP0MVPCo_EVP);
+            mips_vpe_wake(other_cpu_env); /* And wake it up.  */
         }
-        other_cpu = other_cpu->next_cpu;
-    } while (other_cpu);
+        other_cpu_env = other_cpu_env->next_cpu;
+    } while (other_cpu_env);
     return prev;
 }
 #endif /* !CONFIG_USER_ONLY */
@@ -1925,7 +1747,7 @@ void helper_fork(target_ulong arg1, target_ulong arg2)
     // TODO: store to TC register
 }
 
-target_ulong helper_yield(target_ulong arg)
+target_ulong helper_yield(CPUMIPSState *env, target_ulong arg)
 {
     target_long arg1 = arg;
 
@@ -1936,13 +1758,13 @@ target_ulong helper_yield(target_ulong arg)
                 env->active_tc.CP0_TCStatus & (1 << CP0TCSt_DT)) {
                 env->CP0_VPEControl &= ~(0x7 << CP0VPECo_EXCPT);
                 env->CP0_VPEControl |= 4 << CP0VPECo_EXCPT;
-                helper_raise_exception(EXCP_THREAD);
+                helper_raise_exception(env, EXCP_THREAD);
             }
         }
     } else if (arg1 == 0) {
         if (0 /* TODO: TC underflow */) {
             env->CP0_VPEControl &= ~(0x7 << CP0VPECo_EXCPT);
-            helper_raise_exception(EXCP_THREAD);
+            helper_raise_exception(env, EXCP_THREAD);
         } else {
             // TODO: Deallocate TC
         }
@@ -1950,7 +1772,7 @@ target_ulong helper_yield(target_ulong arg)
         /* Yield qualifier inputs not implemented. */
         env->CP0_VPEControl &= ~(0x7 << CP0VPECo_EXCPT);
         env->CP0_VPEControl |= 2 << CP0VPECo_EXCPT;
-        helper_raise_exception(EXCP_THREAD);
+        helper_raise_exception(env, EXCP_THREAD);
     }
     return env->CP0_YQMask;
 }
@@ -1972,7 +1794,7 @@ static void r4k_mips_tlb_flush_extra (CPUMIPSState *env, int first)
     }
 }
 
-static void r4k_fill_tlb (int idx)
+static void r4k_fill_tlb(CPUMIPSState *env, int idx)
 {
     r4k_tlb_t *tlb;
 
@@ -1995,30 +1817,48 @@ static void r4k_fill_tlb (int idx)
     tlb->PFN[1] = (env->CP0_EntryLo1 >> 6) << 12;
 }
 
-void r4k_helper_tlbwi (void)
+void r4k_helper_tlbwi(CPUMIPSState *env)
 {
+    r4k_tlb_t *tlb;
     int idx;
+    target_ulong VPN;
+    uint8_t ASID;
+    bool G, V0, D0, V1, D1;
 
     idx = (env->CP0_Index & ~0x80000000) % env->tlb->nb_tlb;
+    tlb = &env->tlb->mmu.r4k.tlb[idx];
+    VPN = env->CP0_EntryHi & (TARGET_PAGE_MASK << 1);
+#if defined(TARGET_MIPS64)
+    VPN &= env->SEGMask;
+#endif
+    ASID = env->CP0_EntryHi & 0xff;
+    G = env->CP0_EntryLo0 & env->CP0_EntryLo1 & 1;
+    V0 = (env->CP0_EntryLo0 & 2) != 0;
+    D0 = (env->CP0_EntryLo0 & 4) != 0;
+    V1 = (env->CP0_EntryLo1 & 2) != 0;
+    D1 = (env->CP0_EntryLo1 & 4) != 0;
 
-    /* Discard cached TLB entries.  We could avoid doing this if the
-       tlbwi is just upgrading access permissions on the current entry;
-       that might be a further win.  */
-    r4k_mips_tlb_flush_extra (env, env->tlb->nb_tlb);
+    /* Discard cached TLB entries, unless tlbwi is just upgrading access
+       permissions on the current entry. */
+    if (tlb->VPN != VPN || tlb->ASID != ASID || tlb->G != G ||
+        (tlb->V0 && !V0) || (tlb->D0 && !D0) ||
+        (tlb->V1 && !V1) || (tlb->D1 && !D1)) {
+        r4k_mips_tlb_flush_extra(env, env->tlb->nb_tlb);
+    }
 
     r4k_invalidate_tlb(env, idx, 0);
-    r4k_fill_tlb(idx);
+    r4k_fill_tlb(env, idx);
 }
 
-void r4k_helper_tlbwr (void)
+void r4k_helper_tlbwr(CPUMIPSState *env)
 {
     int r = cpu_mips_get_random(env);
 
     r4k_invalidate_tlb(env, r, 1);
-    r4k_fill_tlb(r);
+    r4k_fill_tlb(env, r);
 }
 
-void r4k_helper_tlbp (void)
+void r4k_helper_tlbp(CPUMIPSState *env)
 {
     r4k_tlb_t *tlb;
     target_ulong mask;
@@ -2034,6 +1874,9 @@ void r4k_helper_tlbp (void)
         mask = tlb->PageMask | ~(TARGET_PAGE_MASK << 1);
         tag = env->CP0_EntryHi & ~mask;
         VPN = tlb->VPN & ~mask;
+#if defined(TARGET_MIPS64)
+        tag &= env->SEGMask;
+#endif
         /* Check ASID, virtual page number & size */
         if ((tlb->G == 1 || tlb->ASID == ASID) && VPN == tag) {
             /* TLB match */
@@ -2049,6 +1892,9 @@ void r4k_helper_tlbp (void)
             mask = tlb->PageMask | ~(TARGET_PAGE_MASK << 1);
             tag = env->CP0_EntryHi & ~mask;
             VPN = tlb->VPN & ~mask;
+#if defined(TARGET_MIPS64)
+            tag &= env->SEGMask;
+#endif
             /* Check ASID, virtual page number & size */
             if ((tlb->G == 1 || tlb->ASID == ASID) && VPN == tag) {
                 r4k_mips_tlb_flush_extra (env, i);
@@ -2060,7 +1906,7 @@ void r4k_helper_tlbp (void)
     }
 }
 
-void r4k_helper_tlbr (void)
+void r4k_helper_tlbr(CPUMIPSState *env)
 {
     r4k_tlb_t *tlb;
     uint8_t ASID;
@@ -2084,28 +1930,28 @@ void r4k_helper_tlbr (void)
                         (tlb->C1 << 3) | (tlb->PFN[1] >> 6);
 }
 
-void helper_tlbwi(void)
+void helper_tlbwi(CPUMIPSState *env)
 {
-    env->tlb->helper_tlbwi();
+    env->tlb->helper_tlbwi(env);
 }
 
-void helper_tlbwr(void)
+void helper_tlbwr(CPUMIPSState *env)
 {
-    env->tlb->helper_tlbwr();
+    env->tlb->helper_tlbwr(env);
 }
 
-void helper_tlbp(void)
+void helper_tlbp(CPUMIPSState *env)
 {
-    env->tlb->helper_tlbp();
+    env->tlb->helper_tlbp(env);
 }
 
-void helper_tlbr(void)
+void helper_tlbr(CPUMIPSState *env)
 {
-    env->tlb->helper_tlbr();
+    env->tlb->helper_tlbr(env);
 }
 
 /* Specials */
-target_ulong helper_di (void)
+target_ulong helper_di(CPUMIPSState *env)
 {
     target_ulong t0 = env->CP0_Status;
 
@@ -2113,7 +1959,7 @@ target_ulong helper_di (void)
     return t0;
 }
 
-target_ulong helper_ei (void)
+target_ulong helper_ei(CPUMIPSState *env)
 {
     target_ulong t0 = env->CP0_Status;
 
@@ -2121,7 +1967,7 @@ target_ulong helper_ei (void)
     return t0;
 }
 
-static void debug_pre_eret (void)
+static void debug_pre_eret(CPUMIPSState *env)
 {
     if (qemu_loglevel_mask(CPU_LOG_EXEC)) {
         qemu_log("ERET: PC " TARGET_FMT_lx " EPC " TARGET_FMT_lx,
@@ -2134,7 +1980,7 @@ static void debug_pre_eret (void)
     }
 }
 
-static void debug_post_eret (void)
+static void debug_post_eret(CPUMIPSState *env)
 {
     if (qemu_loglevel_mask(CPU_LOG_EXEC)) {
         qemu_log("  =>  PC " TARGET_FMT_lx " EPC " TARGET_FMT_lx,
@@ -2152,7 +1998,7 @@ static void debug_post_eret (void)
     }
 }
 
-static void set_pc (target_ulong error_pc)
+static void set_pc(CPUMIPSState *env, target_ulong error_pc)
 {
     env->active_tc.PC = error_pc & ~(target_ulong)1;
     if (error_pc & 1) {
@@ -2162,78 +2008,78 @@ static void set_pc (target_ulong error_pc)
     }
 }
 
-void helper_eret (void)
+void helper_eret(CPUMIPSState *env)
 {
-    debug_pre_eret();
+    debug_pre_eret(env);
     if (env->CP0_Status & (1 << CP0St_ERL)) {
-        set_pc(env->CP0_ErrorEPC);
+        set_pc(env, env->CP0_ErrorEPC);
         env->CP0_Status &= ~(1 << CP0St_ERL);
     } else {
-        set_pc(env->CP0_EPC);
+        set_pc(env, env->CP0_EPC);
         env->CP0_Status &= ~(1 << CP0St_EXL);
     }
     compute_hflags(env);
-    debug_post_eret();
+    debug_post_eret(env);
     env->lladdr = 1;
 }
 
-void helper_deret (void)
+void helper_deret(CPUMIPSState *env)
 {
-    debug_pre_eret();
-    set_pc(env->CP0_DEPC);
+    debug_pre_eret(env);
+    set_pc(env, env->CP0_DEPC);
 
     env->hflags &= MIPS_HFLAG_DM;
     compute_hflags(env);
-    debug_post_eret();
+    debug_post_eret(env);
     env->lladdr = 1;
 }
 #endif /* !CONFIG_USER_ONLY */
 
-target_ulong helper_rdhwr_cpunum(void)
+target_ulong helper_rdhwr_cpunum(CPUMIPSState *env)
 {
     if ((env->hflags & MIPS_HFLAG_CP0) ||
         (env->CP0_HWREna & (1 << 0)))
         return env->CP0_EBase & 0x3ff;
     else
-        helper_raise_exception(EXCP_RI);
+        helper_raise_exception(env, EXCP_RI);
 
     return 0;
 }
 
-target_ulong helper_rdhwr_synci_step(void)
+target_ulong helper_rdhwr_synci_step(CPUMIPSState *env)
 {
     if ((env->hflags & MIPS_HFLAG_CP0) ||
         (env->CP0_HWREna & (1 << 1)))
         return env->SYNCI_Step;
     else
-        helper_raise_exception(EXCP_RI);
+        helper_raise_exception(env, EXCP_RI);
 
     return 0;
 }
 
-target_ulong helper_rdhwr_cc(void)
+target_ulong helper_rdhwr_cc(CPUMIPSState *env)
 {
     if ((env->hflags & MIPS_HFLAG_CP0) ||
         (env->CP0_HWREna & (1 << 2)))
         return env->CP0_Count;
     else
-        helper_raise_exception(EXCP_RI);
+        helper_raise_exception(env, EXCP_RI);
 
     return 0;
 }
 
-target_ulong helper_rdhwr_ccres(void)
+target_ulong helper_rdhwr_ccres(CPUMIPSState *env)
 {
     if ((env->hflags & MIPS_HFLAG_CP0) ||
         (env->CP0_HWREna & (1 << 3)))
         return env->CCRes;
     else
-        helper_raise_exception(EXCP_RI);
+        helper_raise_exception(env, EXCP_RI);
 
     return 0;
 }
 
-void helper_pmon (int function)
+void helper_pmon(CPUMIPSState *env, int function)
 {
     function /= 2;
     switch (function) {
@@ -2259,16 +2105,17 @@ void helper_pmon (int function)
     }
 }
 
-void helper_wait (void)
+void helper_wait(CPUMIPSState *env)
 {
     env->halted = 1;
     cpu_reset_interrupt(env, CPU_INTERRUPT_WAKE);
-    helper_raise_exception(EXCP_HLT);
+    helper_raise_exception(env, EXCP_HLT);
 }
 
 #if !defined(CONFIG_USER_ONLY)
 
-static void QEMU_NORETURN do_unaligned_access(target_ulong addr, int is_write,
+static void QEMU_NORETURN do_unaligned_access(CPUMIPSState *env,
+                                              target_ulong addr, int is_write,
                                               int is_user, uintptr_t retaddr);
 
 #define MMUSUFFIX _mmu
@@ -2286,61 +2133,41 @@ static void QEMU_NORETURN do_unaligned_access(target_ulong addr, int is_write,
 #define SHIFT 3
 #include "softmmu_template.h"
 
-static void do_unaligned_access(target_ulong addr, int is_write,
-                                int is_user, uintptr_t retaddr)
+static void do_unaligned_access(CPUMIPSState *env, target_ulong addr,
+                                int is_write, int is_user, uintptr_t retaddr)
 {
     env->CP0_BadVAddr = addr;
-    do_restore_state (retaddr);
-    helper_raise_exception ((is_write == 1) ? EXCP_AdES : EXCP_AdEL);
+    do_raise_exception(env, (is_write == 1) ? EXCP_AdES : EXCP_AdEL, retaddr);
 }
 
-void tlb_fill(CPUMIPSState *env1, target_ulong addr, int is_write, int mmu_idx,
+void tlb_fill(CPUMIPSState *env, target_ulong addr, int is_write, int mmu_idx,
               uintptr_t retaddr)
 {
-    TranslationBlock *tb;
-    CPUMIPSState *saved_env;
     int ret;
 
-    saved_env = env;
-    env = env1;
     ret = cpu_mips_handle_mmu_fault(env, addr, is_write, mmu_idx);
     if (ret) {
-        if (retaddr) {
-            /* now we have a real cpu fault */
-            tb = tb_find_pc(retaddr);
-            if (tb) {
-                /* the PC is inside the translated code. It means that we have
-                   a virtual CPU fault */
-                cpu_restore_state(tb, env, retaddr);
-            }
-        }
-        helper_raise_exception_err(env->exception_index, env->error_code);
+        do_raise_exception_err(env, env->exception_index,
+                               env->error_code, retaddr);
     }
-    env = saved_env;
 }
 
-void cpu_unassigned_access(CPUMIPSState *env1, target_phys_addr_t addr,
+void cpu_unassigned_access(CPUMIPSState *env, hwaddr addr,
                            int is_write, int is_exec, int unused, int size)
 {
-    env = env1;
-
     if (is_exec)
-        helper_raise_exception(EXCP_IBE);
+        helper_raise_exception(env, EXCP_IBE);
     else
-        helper_raise_exception(EXCP_DBE);
+        helper_raise_exception(env, EXCP_DBE);
 }
 #endif /* !CONFIG_USER_ONLY */
 
 /* Complex FPU operations which may need stack space. */
 
-#define FLOAT_ONE32 make_float32(0x3f8 << 20)
-#define FLOAT_ONE64 make_float64(0x3ffULL << 52)
 #define FLOAT_TWO32 make_float32(1 << 30)
 #define FLOAT_TWO64 make_float64(1ULL << 62)
-#define FLOAT_QNAN32 0x7fbfffff
-#define FLOAT_QNAN64 0x7ff7ffffffffffffULL
-#define FLOAT_SNAN32 0x7fffffff
-#define FLOAT_SNAN64 0x7fffffffffffffffULL
+#define FP_TO_INT32_OVERFLOW 0x7fffffff
+#define FP_TO_INT64_OVERFLOW 0x7fffffffffffffffULL
 
 /* convert MIPS rounding mode in FCR31 to IEEE library */
 static unsigned int ieee_rm[] = {
@@ -2356,7 +2183,7 @@ static unsigned int ieee_rm[] = {
 #define RESTORE_FLUSH_MODE \
     set_flush_to_zero((env->active_fpu.fcr31 & (1 << 24)) != 0, &env->active_fpu.fp_status);
 
-target_ulong helper_cfc1 (uint32_t reg)
+target_ulong helper_cfc1(CPUMIPSState *env, uint32_t reg)
 {
     target_ulong arg1;
 
@@ -2381,7 +2208,7 @@ target_ulong helper_cfc1 (uint32_t reg)
     return arg1;
 }
 
-void helper_ctc1 (target_ulong arg1, uint32_t reg)
+void helper_ctc1(CPUMIPSState *env, target_ulong arg1, uint32_t reg)
 {
     switch(reg) {
     case 25:
@@ -2415,7 +2242,7 @@ void helper_ctc1 (target_ulong arg1, uint32_t reg)
     RESTORE_FLUSH_MODE;
     set_float_exception_flags(0, &env->active_fpu.fp_status);
     if ((GET_FP_ENABLE(env->active_fpu.fcr31) | 0x20) & GET_FP_CAUSE(env->active_fpu.fcr31))
-        helper_raise_exception(EXCP_FPE);
+        do_raise_exception(env, EXCP_FPE, GETPC());
 }
 
 static inline int ieee_ex_to_mips(int xcpt)
@@ -2441,15 +2268,21 @@ static inline int ieee_ex_to_mips(int xcpt)
     return ret;
 }
 
-static inline void update_fcr31(void)
+static inline void update_fcr31(CPUMIPSState *env, uintptr_t pc)
 {
     int tmp = ieee_ex_to_mips(get_float_exception_flags(&env->active_fpu.fp_status));
 
     SET_FP_CAUSE(env->active_fpu.fcr31, tmp);
-    if (GET_FP_ENABLE(env->active_fpu.fcr31) & tmp)
-        helper_raise_exception(EXCP_FPE);
-    else
-        UPDATE_FP_FLAGS(env->active_fpu.fcr31, tmp);
+
+    if (tmp) {
+        set_float_exception_flags(0, &env->active_fpu.fp_status);
+
+        if (GET_FP_ENABLE(env->active_fpu.fcr31) & tmp) {
+            do_raise_exception(env, EXCP_FPE, pc);
+        } else {
+            UPDATE_FP_FLAGS(env->active_fpu.fcr31, tmp);
+        }
+    }
 }
 
 /* Float support.
@@ -2458,385 +2291,409 @@ static inline void update_fcr31(void)
    paired single lower "pl", paired single upper "pu".  */
 
 /* unary operations, modifying fp status  */
-uint64_t helper_float_sqrt_d(uint64_t fdt0)
+uint64_t helper_float_sqrt_d(CPUMIPSState *env, uint64_t fdt0)
 {
-    return float64_sqrt(fdt0, &env->active_fpu.fp_status);
+    fdt0 = float64_sqrt(fdt0, &env->active_fpu.fp_status);
+    update_fcr31(env, GETPC());
+    return fdt0;
 }
 
-uint32_t helper_float_sqrt_s(uint32_t fst0)
+uint32_t helper_float_sqrt_s(CPUMIPSState *env, uint32_t fst0)
 {
-    return float32_sqrt(fst0, &env->active_fpu.fp_status);
+    fst0 = float32_sqrt(fst0, &env->active_fpu.fp_status);
+    update_fcr31(env, GETPC());
+    return fst0;
 }
 
-uint64_t helper_float_cvtd_s(uint32_t fst0)
+uint64_t helper_float_cvtd_s(CPUMIPSState *env, uint32_t fst0)
 {
     uint64_t fdt2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     fdt2 = float32_to_float64(fst0, &env->active_fpu.fp_status);
-    update_fcr31();
+    update_fcr31(env, GETPC());
     return fdt2;
 }
 
-uint64_t helper_float_cvtd_w(uint32_t wt0)
+uint64_t helper_float_cvtd_w(CPUMIPSState *env, uint32_t wt0)
 {
     uint64_t fdt2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     fdt2 = int32_to_float64(wt0, &env->active_fpu.fp_status);
-    update_fcr31();
+    update_fcr31(env, GETPC());
     return fdt2;
 }
 
-uint64_t helper_float_cvtd_l(uint64_t dt0)
+uint64_t helper_float_cvtd_l(CPUMIPSState *env, uint64_t dt0)
 {
     uint64_t fdt2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     fdt2 = int64_to_float64(dt0, &env->active_fpu.fp_status);
-    update_fcr31();
+    update_fcr31(env, GETPC());
     return fdt2;
 }
 
-uint64_t helper_float_cvtl_d(uint64_t fdt0)
+uint64_t helper_float_cvtl_d(CPUMIPSState *env, uint64_t fdt0)
 {
     uint64_t dt2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     dt2 = float64_to_int64(fdt0, &env->active_fpu.fp_status);
-    update_fcr31();
-    if (GET_FP_CAUSE(env->active_fpu.fcr31) & (FP_OVERFLOW | FP_INVALID))
-        dt2 = FLOAT_SNAN64;
+    if (get_float_exception_flags(&env->active_fpu.fp_status)
+        & (float_flag_invalid | float_flag_overflow)) {
+        dt2 = FP_TO_INT64_OVERFLOW;
+    }
+    update_fcr31(env, GETPC());
     return dt2;
 }
 
-uint64_t helper_float_cvtl_s(uint32_t fst0)
+uint64_t helper_float_cvtl_s(CPUMIPSState *env, uint32_t fst0)
 {
     uint64_t dt2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     dt2 = float32_to_int64(fst0, &env->active_fpu.fp_status);
-    update_fcr31();
-    if (GET_FP_CAUSE(env->active_fpu.fcr31) & (FP_OVERFLOW | FP_INVALID))
-        dt2 = FLOAT_SNAN64;
+    if (get_float_exception_flags(&env->active_fpu.fp_status)
+        & (float_flag_invalid | float_flag_overflow)) {
+        dt2 = FP_TO_INT64_OVERFLOW;
+    }
+    update_fcr31(env, GETPC());
     return dt2;
 }
 
-uint64_t helper_float_cvtps_pw(uint64_t dt0)
+uint64_t helper_float_cvtps_pw(CPUMIPSState *env, uint64_t dt0)
 {
     uint32_t fst2;
     uint32_t fsth2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     fst2 = int32_to_float32(dt0 & 0XFFFFFFFF, &env->active_fpu.fp_status);
     fsth2 = int32_to_float32(dt0 >> 32, &env->active_fpu.fp_status);
-    update_fcr31();
+    update_fcr31(env, GETPC());
     return ((uint64_t)fsth2 << 32) | fst2;
 }
 
-uint64_t helper_float_cvtpw_ps(uint64_t fdt0)
+uint64_t helper_float_cvtpw_ps(CPUMIPSState *env, uint64_t fdt0)
 {
     uint32_t wt2;
     uint32_t wth2;
+    int excp, excph;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     wt2 = float32_to_int32(fdt0 & 0XFFFFFFFF, &env->active_fpu.fp_status);
+    excp = get_float_exception_flags(&env->active_fpu.fp_status);
+    if (excp & (float_flag_overflow | float_flag_invalid)) {
+        wt2 = FP_TO_INT32_OVERFLOW;
+    }
+
+    set_float_exception_flags(0, &env->active_fpu.fp_status);
     wth2 = float32_to_int32(fdt0 >> 32, &env->active_fpu.fp_status);
-    update_fcr31();
-    if (GET_FP_CAUSE(env->active_fpu.fcr31) & (FP_OVERFLOW | FP_INVALID)) {
-        wt2 = FLOAT_SNAN32;
-        wth2 = FLOAT_SNAN32;
+    excph = get_float_exception_flags(&env->active_fpu.fp_status);
+    if (excph & (float_flag_overflow | float_flag_invalid)) {
+        wth2 = FP_TO_INT32_OVERFLOW;
     }
+
+    set_float_exception_flags(excp | excph, &env->active_fpu.fp_status);
+    update_fcr31(env, GETPC());
+
     return ((uint64_t)wth2 << 32) | wt2;
 }
 
-uint32_t helper_float_cvts_d(uint64_t fdt0)
+uint32_t helper_float_cvts_d(CPUMIPSState *env, uint64_t fdt0)
 {
     uint32_t fst2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     fst2 = float64_to_float32(fdt0, &env->active_fpu.fp_status);
-    update_fcr31();
+    update_fcr31(env, GETPC());
     return fst2;
 }
 
-uint32_t helper_float_cvts_w(uint32_t wt0)
+uint32_t helper_float_cvts_w(CPUMIPSState *env, uint32_t wt0)
 {
     uint32_t fst2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     fst2 = int32_to_float32(wt0, &env->active_fpu.fp_status);
-    update_fcr31();
+    update_fcr31(env, GETPC());
     return fst2;
 }
 
-uint32_t helper_float_cvts_l(uint64_t dt0)
+uint32_t helper_float_cvts_l(CPUMIPSState *env, uint64_t dt0)
 {
     uint32_t fst2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     fst2 = int64_to_float32(dt0, &env->active_fpu.fp_status);
-    update_fcr31();
+    update_fcr31(env, GETPC());
     return fst2;
 }
 
-uint32_t helper_float_cvts_pl(uint32_t wt0)
+uint32_t helper_float_cvts_pl(CPUMIPSState *env, uint32_t wt0)
 {
     uint32_t wt2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     wt2 = wt0;
-    update_fcr31();
+    update_fcr31(env, GETPC());
     return wt2;
 }
 
-uint32_t helper_float_cvts_pu(uint32_t wth0)
+uint32_t helper_float_cvts_pu(CPUMIPSState *env, uint32_t wth0)
 {
     uint32_t wt2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     wt2 = wth0;
-    update_fcr31();
+    update_fcr31(env, GETPC());
     return wt2;
 }
 
-uint32_t helper_float_cvtw_s(uint32_t fst0)
+uint32_t helper_float_cvtw_s(CPUMIPSState *env, uint32_t fst0)
 {
     uint32_t wt2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     wt2 = float32_to_int32(fst0, &env->active_fpu.fp_status);
-    update_fcr31();
-    if (GET_FP_CAUSE(env->active_fpu.fcr31) & (FP_OVERFLOW | FP_INVALID))
-        wt2 = FLOAT_SNAN32;
+    update_fcr31(env, GETPC());
+    if (get_float_exception_flags(&env->active_fpu.fp_status)
+        & (float_flag_invalid | float_flag_overflow)) {
+        wt2 = FP_TO_INT32_OVERFLOW;
+    }
     return wt2;
 }
 
-uint32_t helper_float_cvtw_d(uint64_t fdt0)
+uint32_t helper_float_cvtw_d(CPUMIPSState *env, uint64_t fdt0)
 {
     uint32_t wt2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     wt2 = float64_to_int32(fdt0, &env->active_fpu.fp_status);
-    update_fcr31();
-    if (GET_FP_CAUSE(env->active_fpu.fcr31) & (FP_OVERFLOW | FP_INVALID))
-        wt2 = FLOAT_SNAN32;
+    if (get_float_exception_flags(&env->active_fpu.fp_status)
+        & (float_flag_invalid | float_flag_overflow)) {
+        wt2 = FP_TO_INT32_OVERFLOW;
+    }
+    update_fcr31(env, GETPC());
     return wt2;
 }
 
-uint64_t helper_float_roundl_d(uint64_t fdt0)
+uint64_t helper_float_roundl_d(CPUMIPSState *env, uint64_t fdt0)
 {
     uint64_t dt2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     set_float_rounding_mode(float_round_nearest_even, &env->active_fpu.fp_status);
     dt2 = float64_to_int64(fdt0, &env->active_fpu.fp_status);
     RESTORE_ROUNDING_MODE;
-    update_fcr31();
-    if (GET_FP_CAUSE(env->active_fpu.fcr31) & (FP_OVERFLOW | FP_INVALID))
-        dt2 = FLOAT_SNAN64;
+    if (get_float_exception_flags(&env->active_fpu.fp_status)
+        & (float_flag_invalid | float_flag_overflow)) {
+        dt2 = FP_TO_INT64_OVERFLOW;
+    }
+    update_fcr31(env, GETPC());
     return dt2;
 }
 
-uint64_t helper_float_roundl_s(uint32_t fst0)
+uint64_t helper_float_roundl_s(CPUMIPSState *env, uint32_t fst0)
 {
     uint64_t dt2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     set_float_rounding_mode(float_round_nearest_even, &env->active_fpu.fp_status);
     dt2 = float32_to_int64(fst0, &env->active_fpu.fp_status);
     RESTORE_ROUNDING_MODE;
-    update_fcr31();
-    if (GET_FP_CAUSE(env->active_fpu.fcr31) & (FP_OVERFLOW | FP_INVALID))
-        dt2 = FLOAT_SNAN64;
+    if (get_float_exception_flags(&env->active_fpu.fp_status)
+        & (float_flag_invalid | float_flag_overflow)) {
+        dt2 = FP_TO_INT64_OVERFLOW;
+    }
+    update_fcr31(env, GETPC());
     return dt2;
 }
 
-uint32_t helper_float_roundw_d(uint64_t fdt0)
+uint32_t helper_float_roundw_d(CPUMIPSState *env, uint64_t fdt0)
 {
     uint32_t wt2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     set_float_rounding_mode(float_round_nearest_even, &env->active_fpu.fp_status);
     wt2 = float64_to_int32(fdt0, &env->active_fpu.fp_status);
     RESTORE_ROUNDING_MODE;
-    update_fcr31();
-    if (GET_FP_CAUSE(env->active_fpu.fcr31) & (FP_OVERFLOW | FP_INVALID))
-        wt2 = FLOAT_SNAN32;
+    if (get_float_exception_flags(&env->active_fpu.fp_status)
+        & (float_flag_invalid | float_flag_overflow)) {
+        wt2 = FP_TO_INT32_OVERFLOW;
+    }
+    update_fcr31(env, GETPC());
     return wt2;
 }
 
-uint32_t helper_float_roundw_s(uint32_t fst0)
+uint32_t helper_float_roundw_s(CPUMIPSState *env, uint32_t fst0)
 {
     uint32_t wt2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     set_float_rounding_mode(float_round_nearest_even, &env->active_fpu.fp_status);
     wt2 = float32_to_int32(fst0, &env->active_fpu.fp_status);
     RESTORE_ROUNDING_MODE;
-    update_fcr31();
-    if (GET_FP_CAUSE(env->active_fpu.fcr31) & (FP_OVERFLOW | FP_INVALID))
-        wt2 = FLOAT_SNAN32;
+    if (get_float_exception_flags(&env->active_fpu.fp_status)
+        & (float_flag_invalid | float_flag_overflow)) {
+        wt2 = FP_TO_INT32_OVERFLOW;
+    }
+    update_fcr31(env, GETPC());
     return wt2;
 }
 
-uint64_t helper_float_truncl_d(uint64_t fdt0)
+uint64_t helper_float_truncl_d(CPUMIPSState *env, uint64_t fdt0)
 {
     uint64_t dt2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     dt2 = float64_to_int64_round_to_zero(fdt0, &env->active_fpu.fp_status);
-    update_fcr31();
-    if (GET_FP_CAUSE(env->active_fpu.fcr31) & (FP_OVERFLOW | FP_INVALID))
-        dt2 = FLOAT_SNAN64;
+    if (get_float_exception_flags(&env->active_fpu.fp_status)
+        & (float_flag_invalid | float_flag_overflow)) {
+        dt2 = FP_TO_INT64_OVERFLOW;
+    }
+    update_fcr31(env, GETPC());
     return dt2;
 }
 
-uint64_t helper_float_truncl_s(uint32_t fst0)
+uint64_t helper_float_truncl_s(CPUMIPSState *env, uint32_t fst0)
 {
     uint64_t dt2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     dt2 = float32_to_int64_round_to_zero(fst0, &env->active_fpu.fp_status);
-    update_fcr31();
-    if (GET_FP_CAUSE(env->active_fpu.fcr31) & (FP_OVERFLOW | FP_INVALID))
-        dt2 = FLOAT_SNAN64;
+    if (get_float_exception_flags(&env->active_fpu.fp_status)
+        & (float_flag_invalid | float_flag_overflow)) {
+        dt2 = FP_TO_INT64_OVERFLOW;
+    }
+    update_fcr31(env, GETPC());
     return dt2;
 }
 
-uint32_t helper_float_truncw_d(uint64_t fdt0)
+uint32_t helper_float_truncw_d(CPUMIPSState *env, uint64_t fdt0)
 {
     uint32_t wt2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     wt2 = float64_to_int32_round_to_zero(fdt0, &env->active_fpu.fp_status);
-    update_fcr31();
-    if (GET_FP_CAUSE(env->active_fpu.fcr31) & (FP_OVERFLOW | FP_INVALID))
-        wt2 = FLOAT_SNAN32;
+    if (get_float_exception_flags(&env->active_fpu.fp_status)
+        & (float_flag_invalid | float_flag_overflow)) {
+        wt2 = FP_TO_INT32_OVERFLOW;
+    }
+    update_fcr31(env, GETPC());
     return wt2;
 }
 
-uint32_t helper_float_truncw_s(uint32_t fst0)
+uint32_t helper_float_truncw_s(CPUMIPSState *env, uint32_t fst0)
 {
     uint32_t wt2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     wt2 = float32_to_int32_round_to_zero(fst0, &env->active_fpu.fp_status);
-    update_fcr31();
-    if (GET_FP_CAUSE(env->active_fpu.fcr31) & (FP_OVERFLOW | FP_INVALID))
-        wt2 = FLOAT_SNAN32;
+    if (get_float_exception_flags(&env->active_fpu.fp_status)
+        & (float_flag_invalid | float_flag_overflow)) {
+        wt2 = FP_TO_INT32_OVERFLOW;
+    }
+    update_fcr31(env, GETPC());
     return wt2;
 }
 
-uint64_t helper_float_ceill_d(uint64_t fdt0)
+uint64_t helper_float_ceill_d(CPUMIPSState *env, uint64_t fdt0)
 {
     uint64_t dt2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     set_float_rounding_mode(float_round_up, &env->active_fpu.fp_status);
     dt2 = float64_to_int64(fdt0, &env->active_fpu.fp_status);
     RESTORE_ROUNDING_MODE;
-    update_fcr31();
-    if (GET_FP_CAUSE(env->active_fpu.fcr31) & (FP_OVERFLOW | FP_INVALID))
-        dt2 = FLOAT_SNAN64;
+    if (get_float_exception_flags(&env->active_fpu.fp_status)
+        & (float_flag_invalid | float_flag_overflow)) {
+        dt2 = FP_TO_INT64_OVERFLOW;
+    }
+    update_fcr31(env, GETPC());
     return dt2;
 }
 
-uint64_t helper_float_ceill_s(uint32_t fst0)
+uint64_t helper_float_ceill_s(CPUMIPSState *env, uint32_t fst0)
 {
     uint64_t dt2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     set_float_rounding_mode(float_round_up, &env->active_fpu.fp_status);
     dt2 = float32_to_int64(fst0, &env->active_fpu.fp_status);
     RESTORE_ROUNDING_MODE;
-    update_fcr31();
-    if (GET_FP_CAUSE(env->active_fpu.fcr31) & (FP_OVERFLOW | FP_INVALID))
-        dt2 = FLOAT_SNAN64;
+    if (get_float_exception_flags(&env->active_fpu.fp_status)
+        & (float_flag_invalid | float_flag_overflow)) {
+        dt2 = FP_TO_INT64_OVERFLOW;
+    }
+    update_fcr31(env, GETPC());
     return dt2;
 }
 
-uint32_t helper_float_ceilw_d(uint64_t fdt0)
+uint32_t helper_float_ceilw_d(CPUMIPSState *env, uint64_t fdt0)
 {
     uint32_t wt2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     set_float_rounding_mode(float_round_up, &env->active_fpu.fp_status);
     wt2 = float64_to_int32(fdt0, &env->active_fpu.fp_status);
     RESTORE_ROUNDING_MODE;
-    update_fcr31();
-    if (GET_FP_CAUSE(env->active_fpu.fcr31) & (FP_OVERFLOW | FP_INVALID))
-        wt2 = FLOAT_SNAN32;
+    if (get_float_exception_flags(&env->active_fpu.fp_status)
+        & (float_flag_invalid | float_flag_overflow)) {
+        wt2 = FP_TO_INT32_OVERFLOW;
+    }
+    update_fcr31(env, GETPC());
     return wt2;
 }
 
-uint32_t helper_float_ceilw_s(uint32_t fst0)
+uint32_t helper_float_ceilw_s(CPUMIPSState *env, uint32_t fst0)
 {
     uint32_t wt2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     set_float_rounding_mode(float_round_up, &env->active_fpu.fp_status);
     wt2 = float32_to_int32(fst0, &env->active_fpu.fp_status);
     RESTORE_ROUNDING_MODE;
-    update_fcr31();
-    if (GET_FP_CAUSE(env->active_fpu.fcr31) & (FP_OVERFLOW | FP_INVALID))
-        wt2 = FLOAT_SNAN32;
+    if (get_float_exception_flags(&env->active_fpu.fp_status)
+        & (float_flag_invalid | float_flag_overflow)) {
+        wt2 = FP_TO_INT32_OVERFLOW;
+    }
+    update_fcr31(env, GETPC());
     return wt2;
 }
 
-uint64_t helper_float_floorl_d(uint64_t fdt0)
+uint64_t helper_float_floorl_d(CPUMIPSState *env, uint64_t fdt0)
 {
     uint64_t dt2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     set_float_rounding_mode(float_round_down, &env->active_fpu.fp_status);
     dt2 = float64_to_int64(fdt0, &env->active_fpu.fp_status);
     RESTORE_ROUNDING_MODE;
-    update_fcr31();
-    if (GET_FP_CAUSE(env->active_fpu.fcr31) & (FP_OVERFLOW | FP_INVALID))
-        dt2 = FLOAT_SNAN64;
+    if (get_float_exception_flags(&env->active_fpu.fp_status)
+        & (float_flag_invalid | float_flag_overflow)) {
+        dt2 = FP_TO_INT64_OVERFLOW;
+    }
+    update_fcr31(env, GETPC());
     return dt2;
 }
 
-uint64_t helper_float_floorl_s(uint32_t fst0)
+uint64_t helper_float_floorl_s(CPUMIPSState *env, uint32_t fst0)
 {
     uint64_t dt2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     set_float_rounding_mode(float_round_down, &env->active_fpu.fp_status);
     dt2 = float32_to_int64(fst0, &env->active_fpu.fp_status);
     RESTORE_ROUNDING_MODE;
-    update_fcr31();
-    if (GET_FP_CAUSE(env->active_fpu.fcr31) & (FP_OVERFLOW | FP_INVALID))
-        dt2 = FLOAT_SNAN64;
+    if (get_float_exception_flags(&env->active_fpu.fp_status)
+        & (float_flag_invalid | float_flag_overflow)) {
+        dt2 = FP_TO_INT64_OVERFLOW;
+    }
+    update_fcr31(env, GETPC());
     return dt2;
 }
 
-uint32_t helper_float_floorw_d(uint64_t fdt0)
+uint32_t helper_float_floorw_d(CPUMIPSState *env, uint64_t fdt0)
 {
     uint32_t wt2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     set_float_rounding_mode(float_round_down, &env->active_fpu.fp_status);
     wt2 = float64_to_int32(fdt0, &env->active_fpu.fp_status);
     RESTORE_ROUNDING_MODE;
-    update_fcr31();
-    if (GET_FP_CAUSE(env->active_fpu.fcr31) & (FP_OVERFLOW | FP_INVALID))
-        wt2 = FLOAT_SNAN32;
+    if (get_float_exception_flags(&env->active_fpu.fp_status)
+        & (float_flag_invalid | float_flag_overflow)) {
+        wt2 = FP_TO_INT32_OVERFLOW;
+    }
+    update_fcr31(env, GETPC());
     return wt2;
 }
 
-uint32_t helper_float_floorw_s(uint32_t fst0)
+uint32_t helper_float_floorw_s(CPUMIPSState *env, uint32_t fst0)
 {
     uint32_t wt2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     set_float_rounding_mode(float_round_down, &env->active_fpu.fp_status);
     wt2 = float32_to_int32(fst0, &env->active_fpu.fp_status);
     RESTORE_ROUNDING_MODE;
-    update_fcr31();
-    if (GET_FP_CAUSE(env->active_fpu.fcr31) & (FP_OVERFLOW | FP_INVALID))
-        wt2 = FLOAT_SNAN32;
+    if (get_float_exception_flags(&env->active_fpu.fp_status)
+        & (float_flag_invalid | float_flag_overflow)) {
+        wt2 = FP_TO_INT32_OVERFLOW;
+    }
+    update_fcr31(env, GETPC());
     return wt2;
 }
 
@@ -2864,145 +2721,133 @@ FLOAT_UNOP(chs)
 #undef FLOAT_UNOP
 
 /* MIPS specific unary operations */
-uint64_t helper_float_recip_d(uint64_t fdt0)
+uint64_t helper_float_recip_d(CPUMIPSState *env, uint64_t fdt0)
 {
     uint64_t fdt2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
-    fdt2 = float64_div(FLOAT_ONE64, fdt0, &env->active_fpu.fp_status);
-    update_fcr31();
+    fdt2 = float64_div(float64_one, fdt0, &env->active_fpu.fp_status);
+    update_fcr31(env, GETPC());
     return fdt2;
 }
 
-uint32_t helper_float_recip_s(uint32_t fst0)
+uint32_t helper_float_recip_s(CPUMIPSState *env, uint32_t fst0)
 {
     uint32_t fst2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
-    fst2 = float32_div(FLOAT_ONE32, fst0, &env->active_fpu.fp_status);
-    update_fcr31();
+    fst2 = float32_div(float32_one, fst0, &env->active_fpu.fp_status);
+    update_fcr31(env, GETPC());
     return fst2;
 }
 
-uint64_t helper_float_rsqrt_d(uint64_t fdt0)
+uint64_t helper_float_rsqrt_d(CPUMIPSState *env, uint64_t fdt0)
 {
     uint64_t fdt2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     fdt2 = float64_sqrt(fdt0, &env->active_fpu.fp_status);
-    fdt2 = float64_div(FLOAT_ONE64, fdt2, &env->active_fpu.fp_status);
-    update_fcr31();
+    fdt2 = float64_div(float64_one, fdt2, &env->active_fpu.fp_status);
+    update_fcr31(env, GETPC());
     return fdt2;
 }
 
-uint32_t helper_float_rsqrt_s(uint32_t fst0)
+uint32_t helper_float_rsqrt_s(CPUMIPSState *env, uint32_t fst0)
 {
     uint32_t fst2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     fst2 = float32_sqrt(fst0, &env->active_fpu.fp_status);
-    fst2 = float32_div(FLOAT_ONE32, fst2, &env->active_fpu.fp_status);
-    update_fcr31();
+    fst2 = float32_div(float32_one, fst2, &env->active_fpu.fp_status);
+    update_fcr31(env, GETPC());
     return fst2;
 }
 
-uint64_t helper_float_recip1_d(uint64_t fdt0)
+uint64_t helper_float_recip1_d(CPUMIPSState *env, uint64_t fdt0)
 {
     uint64_t fdt2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
-    fdt2 = float64_div(FLOAT_ONE64, fdt0, &env->active_fpu.fp_status);
-    update_fcr31();
+    fdt2 = float64_div(float64_one, fdt0, &env->active_fpu.fp_status);
+    update_fcr31(env, GETPC());
     return fdt2;
 }
 
-uint32_t helper_float_recip1_s(uint32_t fst0)
+uint32_t helper_float_recip1_s(CPUMIPSState *env, uint32_t fst0)
 {
     uint32_t fst2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
-    fst2 = float32_div(FLOAT_ONE32, fst0, &env->active_fpu.fp_status);
-    update_fcr31();
+    fst2 = float32_div(float32_one, fst0, &env->active_fpu.fp_status);
+    update_fcr31(env, GETPC());
     return fst2;
 }
 
-uint64_t helper_float_recip1_ps(uint64_t fdt0)
+uint64_t helper_float_recip1_ps(CPUMIPSState *env, uint64_t fdt0)
 {
     uint32_t fst2;
     uint32_t fsth2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
-    fst2 = float32_div(FLOAT_ONE32, fdt0 & 0XFFFFFFFF, &env->active_fpu.fp_status);
-    fsth2 = float32_div(FLOAT_ONE32, fdt0 >> 32, &env->active_fpu.fp_status);
-    update_fcr31();
+    fst2 = float32_div(float32_one, fdt0 & 0XFFFFFFFF, &env->active_fpu.fp_status);
+    fsth2 = float32_div(float32_one, fdt0 >> 32, &env->active_fpu.fp_status);
+    update_fcr31(env, GETPC());
     return ((uint64_t)fsth2 << 32) | fst2;
 }
 
-uint64_t helper_float_rsqrt1_d(uint64_t fdt0)
+uint64_t helper_float_rsqrt1_d(CPUMIPSState *env, uint64_t fdt0)
 {
     uint64_t fdt2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     fdt2 = float64_sqrt(fdt0, &env->active_fpu.fp_status);
-    fdt2 = float64_div(FLOAT_ONE64, fdt2, &env->active_fpu.fp_status);
-    update_fcr31();
+    fdt2 = float64_div(float64_one, fdt2, &env->active_fpu.fp_status);
+    update_fcr31(env, GETPC());
     return fdt2;
 }
 
-uint32_t helper_float_rsqrt1_s(uint32_t fst0)
+uint32_t helper_float_rsqrt1_s(CPUMIPSState *env, uint32_t fst0)
 {
     uint32_t fst2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     fst2 = float32_sqrt(fst0, &env->active_fpu.fp_status);
-    fst2 = float32_div(FLOAT_ONE32, fst2, &env->active_fpu.fp_status);
-    update_fcr31();
+    fst2 = float32_div(float32_one, fst2, &env->active_fpu.fp_status);
+    update_fcr31(env, GETPC());
     return fst2;
 }
 
-uint64_t helper_float_rsqrt1_ps(uint64_t fdt0)
+uint64_t helper_float_rsqrt1_ps(CPUMIPSState *env, uint64_t fdt0)
 {
     uint32_t fst2;
     uint32_t fsth2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     fst2 = float32_sqrt(fdt0 & 0XFFFFFFFF, &env->active_fpu.fp_status);
     fsth2 = float32_sqrt(fdt0 >> 32, &env->active_fpu.fp_status);
-    fst2 = float32_div(FLOAT_ONE32, fst2, &env->active_fpu.fp_status);
-    fsth2 = float32_div(FLOAT_ONE32, fsth2, &env->active_fpu.fp_status);
-    update_fcr31();
+    fst2 = float32_div(float32_one, fst2, &env->active_fpu.fp_status);
+    fsth2 = float32_div(float32_one, fsth2, &env->active_fpu.fp_status);
+    update_fcr31(env, GETPC());
     return ((uint64_t)fsth2 << 32) | fst2;
 }
 
-#define FLOAT_OP(name, p) void helper_float_##name##_##p(void)
+#define FLOAT_OP(name, p) void helper_float_##name##_##p(CPUMIPSState *env)
 
 /* binary operations */
 #define FLOAT_BINOP(name)                                          \
-uint64_t helper_float_ ## name ## _d(uint64_t fdt0, uint64_t fdt1)     \
+uint64_t helper_float_ ## name ## _d(CPUMIPSState *env,            \
+                                     uint64_t fdt0, uint64_t fdt1) \
 {                                                                  \
     uint64_t dt2;                                                  \
                                                                    \
-    set_float_exception_flags(0, &env->active_fpu.fp_status);            \
     dt2 = float64_ ## name (fdt0, fdt1, &env->active_fpu.fp_status);     \
-    update_fcr31();                                                \
-    if (GET_FP_CAUSE(env->active_fpu.fcr31) & FP_INVALID)                \
-        dt2 = FLOAT_QNAN64;                                        \
+    update_fcr31(env, GETPC());                                    \
     return dt2;                                                    \
 }                                                                  \
                                                                    \
-uint32_t helper_float_ ## name ## _s(uint32_t fst0, uint32_t fst1)     \
+uint32_t helper_float_ ## name ## _s(CPUMIPSState *env,            \
+                                     uint32_t fst0, uint32_t fst1) \
 {                                                                  \
     uint32_t wt2;                                                  \
                                                                    \
-    set_float_exception_flags(0, &env->active_fpu.fp_status);            \
     wt2 = float32_ ## name (fst0, fst1, &env->active_fpu.fp_status);     \
-    update_fcr31();                                                \
-    if (GET_FP_CAUSE(env->active_fpu.fcr31) & FP_INVALID)                \
-        wt2 = FLOAT_QNAN32;                                        \
+    update_fcr31(env, GETPC());                                    \
     return wt2;                                                    \
 }                                                                  \
                                                                    \
-uint64_t helper_float_ ## name ## _ps(uint64_t fdt0, uint64_t fdt1)    \
+uint64_t helper_float_ ## name ## _ps(CPUMIPSState *env,           \
+                                      uint64_t fdt0,               \
+                                      uint64_t fdt1)               \
 {                                                                  \
     uint32_t fst0 = fdt0 & 0XFFFFFFFF;                             \
     uint32_t fsth0 = fdt0 >> 32;                                   \
@@ -3011,14 +2856,9 @@ uint64_t helper_float_ ## name ## _ps(uint64_t fdt0, uint64_t fdt1)    \
     uint32_t wt2;                                                  \
     uint32_t wth2;                                                 \
                                                                    \
-    set_float_exception_flags(0, &env->active_fpu.fp_status);            \
     wt2 = float32_ ## name (fst0, fst1, &env->active_fpu.fp_status);     \
     wth2 = float32_ ## name (fsth0, fsth1, &env->active_fpu.fp_status);  \
-    update_fcr31();                                                \
-    if (GET_FP_CAUSE(env->active_fpu.fcr31) & FP_INVALID) {              \
-        wt2 = FLOAT_QNAN32;                                        \
-        wth2 = FLOAT_QNAN32;                                       \
-    }                                                              \
+    update_fcr31(env, GETPC());                                    \
     return ((uint64_t)wth2 << 32) | wt2;                           \
 }
 
@@ -3028,158 +2868,120 @@ FLOAT_BINOP(mul)
 FLOAT_BINOP(div)
 #undef FLOAT_BINOP
 
-/* ternary operations */
-#define FLOAT_TERNOP(name1, name2)                                        \
-uint64_t helper_float_ ## name1 ## name2 ## _d(uint64_t fdt0, uint64_t fdt1,  \
-                                           uint64_t fdt2)                 \
-{                                                                         \
-    fdt0 = float64_ ## name1 (fdt0, fdt1, &env->active_fpu.fp_status);          \
-    return float64_ ## name2 (fdt0, fdt2, &env->active_fpu.fp_status);          \
-}                                                                         \
-                                                                          \
-uint32_t helper_float_ ## name1 ## name2 ## _s(uint32_t fst0, uint32_t fst1,  \
-                                           uint32_t fst2)                 \
-{                                                                         \
-    fst0 = float32_ ## name1 (fst0, fst1, &env->active_fpu.fp_status);          \
-    return float32_ ## name2 (fst0, fst2, &env->active_fpu.fp_status);          \
-}                                                                         \
-                                                                          \
-uint64_t helper_float_ ## name1 ## name2 ## _ps(uint64_t fdt0, uint64_t fdt1, \
-                                            uint64_t fdt2)                \
-{                                                                         \
-    uint32_t fst0 = fdt0 & 0XFFFFFFFF;                                    \
-    uint32_t fsth0 = fdt0 >> 32;                                          \
-    uint32_t fst1 = fdt1 & 0XFFFFFFFF;                                    \
-    uint32_t fsth1 = fdt1 >> 32;                                          \
-    uint32_t fst2 = fdt2 & 0XFFFFFFFF;                                    \
-    uint32_t fsth2 = fdt2 >> 32;                                          \
-                                                                          \
-    fst0 = float32_ ## name1 (fst0, fst1, &env->active_fpu.fp_status);          \
-    fsth0 = float32_ ## name1 (fsth0, fsth1, &env->active_fpu.fp_status);       \
-    fst2 = float32_ ## name2 (fst0, fst2, &env->active_fpu.fp_status);          \
-    fsth2 = float32_ ## name2 (fsth0, fsth2, &env->active_fpu.fp_status);       \
-    return ((uint64_t)fsth2 << 32) | fst2;                                \
-}
-
-FLOAT_TERNOP(mul, add)
-FLOAT_TERNOP(mul, sub)
-#undef FLOAT_TERNOP
-
-/* negated ternary operations */
-#define FLOAT_NTERNOP(name1, name2)                                       \
-uint64_t helper_float_n ## name1 ## name2 ## _d(uint64_t fdt0, uint64_t fdt1, \
-                                           uint64_t fdt2)                 \
-{                                                                         \
-    fdt0 = float64_ ## name1 (fdt0, fdt1, &env->active_fpu.fp_status);          \
-    fdt2 = float64_ ## name2 (fdt0, fdt2, &env->active_fpu.fp_status);          \
-    return float64_chs(fdt2);                                             \
-}                                                                         \
-                                                                          \
-uint32_t helper_float_n ## name1 ## name2 ## _s(uint32_t fst0, uint32_t fst1, \
-                                           uint32_t fst2)                 \
-{                                                                         \
-    fst0 = float32_ ## name1 (fst0, fst1, &env->active_fpu.fp_status);          \
-    fst2 = float32_ ## name2 (fst0, fst2, &env->active_fpu.fp_status);          \
-    return float32_chs(fst2);                                             \
-}                                                                         \
-                                                                          \
-uint64_t helper_float_n ## name1 ## name2 ## _ps(uint64_t fdt0, uint64_t fdt1,\
-                                           uint64_t fdt2)                 \
-{                                                                         \
-    uint32_t fst0 = fdt0 & 0XFFFFFFFF;                                    \
-    uint32_t fsth0 = fdt0 >> 32;                                          \
-    uint32_t fst1 = fdt1 & 0XFFFFFFFF;                                    \
-    uint32_t fsth1 = fdt1 >> 32;                                          \
-    uint32_t fst2 = fdt2 & 0XFFFFFFFF;                                    \
-    uint32_t fsth2 = fdt2 >> 32;                                          \
-                                                                          \
-    fst0 = float32_ ## name1 (fst0, fst1, &env->active_fpu.fp_status);          \
-    fsth0 = float32_ ## name1 (fsth0, fsth1, &env->active_fpu.fp_status);       \
-    fst2 = float32_ ## name2 (fst0, fst2, &env->active_fpu.fp_status);          \
-    fsth2 = float32_ ## name2 (fsth0, fsth2, &env->active_fpu.fp_status);       \
-    fst2 = float32_chs(fst2);                                             \
-    fsth2 = float32_chs(fsth2);                                           \
-    return ((uint64_t)fsth2 << 32) | fst2;                                \
-}
-
-FLOAT_NTERNOP(mul, add)
-FLOAT_NTERNOP(mul, sub)
-#undef FLOAT_NTERNOP
+/* FMA based operations */
+#define FLOAT_FMA(name, type)                                        \
+uint64_t helper_float_ ## name ## _d(CPUMIPSState *env,              \
+                                     uint64_t fdt0, uint64_t fdt1,   \
+                                     uint64_t fdt2)                  \
+{                                                                    \
+    fdt0 = float64_muladd(fdt0, fdt1, fdt2, type,                    \
+                         &env->active_fpu.fp_status);                \
+    update_fcr31(env, GETPC());                                      \
+    return fdt0;                                                     \
+}                                                                    \
+                                                                     \
+uint32_t helper_float_ ## name ## _s(CPUMIPSState *env,              \
+                                     uint32_t fst0, uint32_t fst1,   \
+                                     uint32_t fst2)                  \
+{                                                                    \
+    fst0 = float32_muladd(fst0, fst1, fst2, type,                    \
+                         &env->active_fpu.fp_status);                \
+    update_fcr31(env, GETPC());                                      \
+    return fst0;                                                     \
+}                                                                    \
+                                                                     \
+uint64_t helper_float_ ## name ## _ps(CPUMIPSState *env,             \
+                                      uint64_t fdt0, uint64_t fdt1,  \
+                                      uint64_t fdt2)                 \
+{                                                                    \
+    uint32_t fst0 = fdt0 & 0XFFFFFFFF;                               \
+    uint32_t fsth0 = fdt0 >> 32;                                     \
+    uint32_t fst1 = fdt1 & 0XFFFFFFFF;                               \
+    uint32_t fsth1 = fdt1 >> 32;                                     \
+    uint32_t fst2 = fdt2 & 0XFFFFFFFF;                               \
+    uint32_t fsth2 = fdt2 >> 32;                                     \
+                                                                     \
+    fst0 = float32_muladd(fst0, fst1, fst2, type,                    \
+                          &env->active_fpu.fp_status);               \
+    fsth0 = float32_muladd(fsth0, fsth1, fsth2, type,                \
+                           &env->active_fpu.fp_status);              \
+    update_fcr31(env, GETPC());                                      \
+    return ((uint64_t)fsth0 << 32) | fst0;                           \
+}
+FLOAT_FMA(madd, 0)
+FLOAT_FMA(msub, float_muladd_negate_c)
+FLOAT_FMA(nmadd, float_muladd_negate_result)
+FLOAT_FMA(nmsub, float_muladd_negate_result | float_muladd_negate_c)
+#undef FLOAT_FMA
 
 /* MIPS specific binary operations */
-uint64_t helper_float_recip2_d(uint64_t fdt0, uint64_t fdt2)
+uint64_t helper_float_recip2_d(CPUMIPSState *env, uint64_t fdt0, uint64_t fdt2)
 {
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     fdt2 = float64_mul(fdt0, fdt2, &env->active_fpu.fp_status);
-    fdt2 = float64_chs(float64_sub(fdt2, FLOAT_ONE64, &env->active_fpu.fp_status));
-    update_fcr31();
+    fdt2 = float64_chs(float64_sub(fdt2, float64_one, &env->active_fpu.fp_status));
+    update_fcr31(env, GETPC());
     return fdt2;
 }
 
-uint32_t helper_float_recip2_s(uint32_t fst0, uint32_t fst2)
+uint32_t helper_float_recip2_s(CPUMIPSState *env, uint32_t fst0, uint32_t fst2)
 {
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     fst2 = float32_mul(fst0, fst2, &env->active_fpu.fp_status);
-    fst2 = float32_chs(float32_sub(fst2, FLOAT_ONE32, &env->active_fpu.fp_status));
-    update_fcr31();
+    fst2 = float32_chs(float32_sub(fst2, float32_one, &env->active_fpu.fp_status));
+    update_fcr31(env, GETPC());
     return fst2;
 }
 
-uint64_t helper_float_recip2_ps(uint64_t fdt0, uint64_t fdt2)
+uint64_t helper_float_recip2_ps(CPUMIPSState *env, uint64_t fdt0, uint64_t fdt2)
 {
     uint32_t fst0 = fdt0 & 0XFFFFFFFF;
     uint32_t fsth0 = fdt0 >> 32;
     uint32_t fst2 = fdt2 & 0XFFFFFFFF;
     uint32_t fsth2 = fdt2 >> 32;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     fst2 = float32_mul(fst0, fst2, &env->active_fpu.fp_status);
     fsth2 = float32_mul(fsth0, fsth2, &env->active_fpu.fp_status);
-    fst2 = float32_chs(float32_sub(fst2, FLOAT_ONE32, &env->active_fpu.fp_status));
-    fsth2 = float32_chs(float32_sub(fsth2, FLOAT_ONE32, &env->active_fpu.fp_status));
-    update_fcr31();
+    fst2 = float32_chs(float32_sub(fst2, float32_one, &env->active_fpu.fp_status));
+    fsth2 = float32_chs(float32_sub(fsth2, float32_one, &env->active_fpu.fp_status));
+    update_fcr31(env, GETPC());
     return ((uint64_t)fsth2 << 32) | fst2;
 }
 
-uint64_t helper_float_rsqrt2_d(uint64_t fdt0, uint64_t fdt2)
+uint64_t helper_float_rsqrt2_d(CPUMIPSState *env, uint64_t fdt0, uint64_t fdt2)
 {
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     fdt2 = float64_mul(fdt0, fdt2, &env->active_fpu.fp_status);
-    fdt2 = float64_sub(fdt2, FLOAT_ONE64, &env->active_fpu.fp_status);
+    fdt2 = float64_sub(fdt2, float64_one, &env->active_fpu.fp_status);
     fdt2 = float64_chs(float64_div(fdt2, FLOAT_TWO64, &env->active_fpu.fp_status));
-    update_fcr31();
+    update_fcr31(env, GETPC());
     return fdt2;
 }
 
-uint32_t helper_float_rsqrt2_s(uint32_t fst0, uint32_t fst2)
+uint32_t helper_float_rsqrt2_s(CPUMIPSState *env, uint32_t fst0, uint32_t fst2)
 {
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     fst2 = float32_mul(fst0, fst2, &env->active_fpu.fp_status);
-    fst2 = float32_sub(fst2, FLOAT_ONE32, &env->active_fpu.fp_status);
+    fst2 = float32_sub(fst2, float32_one, &env->active_fpu.fp_status);
     fst2 = float32_chs(float32_div(fst2, FLOAT_TWO32, &env->active_fpu.fp_status));
-    update_fcr31();
+    update_fcr31(env, GETPC());
     return fst2;
 }
 
-uint64_t helper_float_rsqrt2_ps(uint64_t fdt0, uint64_t fdt2)
+uint64_t helper_float_rsqrt2_ps(CPUMIPSState *env, uint64_t fdt0, uint64_t fdt2)
 {
     uint32_t fst0 = fdt0 & 0XFFFFFFFF;
     uint32_t fsth0 = fdt0 >> 32;
     uint32_t fst2 = fdt2 & 0XFFFFFFFF;
     uint32_t fsth2 = fdt2 >> 32;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     fst2 = float32_mul(fst0, fst2, &env->active_fpu.fp_status);
     fsth2 = float32_mul(fsth0, fsth2, &env->active_fpu.fp_status);
-    fst2 = float32_sub(fst2, FLOAT_ONE32, &env->active_fpu.fp_status);
-    fsth2 = float32_sub(fsth2, FLOAT_ONE32, &env->active_fpu.fp_status);
+    fst2 = float32_sub(fst2, float32_one, &env->active_fpu.fp_status);
+    fsth2 = float32_sub(fsth2, float32_one, &env->active_fpu.fp_status);
     fst2 = float32_chs(float32_div(fst2, FLOAT_TWO32, &env->active_fpu.fp_status));
     fsth2 = float32_chs(float32_div(fsth2, FLOAT_TWO32, &env->active_fpu.fp_status));
-    update_fcr31();
+    update_fcr31(env, GETPC());
     return ((uint64_t)fsth2 << 32) | fst2;
 }
 
-uint64_t helper_float_addr_ps(uint64_t fdt0, uint64_t fdt1)
+uint64_t helper_float_addr_ps(CPUMIPSState *env, uint64_t fdt0, uint64_t fdt1)
 {
     uint32_t fst0 = fdt0 & 0XFFFFFFFF;
     uint32_t fsth0 = fdt0 >> 32;
@@ -3188,14 +2990,13 @@ uint64_t helper_float_addr_ps(uint64_t fdt0, uint64_t fdt1)
     uint32_t fst2;
     uint32_t fsth2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     fst2 = float32_add (fst0, fsth0, &env->active_fpu.fp_status);
     fsth2 = float32_add (fst1, fsth1, &env->active_fpu.fp_status);
-    update_fcr31();
+    update_fcr31(env, GETPC());
     return ((uint64_t)fsth2 << 32) | fst2;
 }
 
-uint64_t helper_float_mulr_ps(uint64_t fdt0, uint64_t fdt1)
+uint64_t helper_float_mulr_ps(CPUMIPSState *env, uint64_t fdt0, uint64_t fdt1)
 {
     uint32_t fst0 = fdt0 & 0XFFFFFFFF;
     uint32_t fsth0 = fdt0 >> 32;
@@ -3204,34 +3005,33 @@ uint64_t helper_float_mulr_ps(uint64_t fdt0, uint64_t fdt1)
     uint32_t fst2;
     uint32_t fsth2;
 
-    set_float_exception_flags(0, &env->active_fpu.fp_status);
     fst2 = float32_mul (fst0, fsth0, &env->active_fpu.fp_status);
     fsth2 = float32_mul (fst1, fsth1, &env->active_fpu.fp_status);
-    update_fcr31();
+    update_fcr31(env, GETPC());
     return ((uint64_t)fsth2 << 32) | fst2;
 }
 
 /* compare operations */
 #define FOP_COND_D(op, cond)                                   \
-void helper_cmp_d_ ## op (uint64_t fdt0, uint64_t fdt1, int cc)    \
+void helper_cmp_d_ ## op(CPUMIPSState *env, uint64_t fdt0,     \
+                         uint64_t fdt1, int cc)                \
 {                                                              \
     int c;                                                     \
-    set_float_exception_flags(0, &env->active_fpu.fp_status);  \
     c = cond;                                                  \
-    update_fcr31();                                            \
+    update_fcr31(env, GETPC());                                \
     if (c)                                                     \
         SET_FP_COND(cc, env->active_fpu);                      \
     else                                                       \
         CLEAR_FP_COND(cc, env->active_fpu);                    \
 }                                                              \
-void helper_cmpabs_d_ ## op (uint64_t fdt0, uint64_t fdt1, int cc) \
+void helper_cmpabs_d_ ## op(CPUMIPSState *env, uint64_t fdt0,  \
+                            uint64_t fdt1, int cc)             \
 {                                                              \
     int c;                                                     \
-    set_float_exception_flags(0, &env->active_fpu.fp_status);  \
     fdt0 = float64_abs(fdt0);                                  \
     fdt1 = float64_abs(fdt1);                                  \
     c = cond;                                                  \
-    update_fcr31();                                            \
+    update_fcr31(env, GETPC());                                \
     if (c)                                                     \
         SET_FP_COND(cc, env->active_fpu);                      \
     else                                                       \
@@ -3260,25 +3060,25 @@ FOP_COND_D(le,  float64_le(fdt0, fdt1, &env->active_fpu.fp_status))
 FOP_COND_D(ngt, float64_unordered(fdt1, fdt0, &env->active_fpu.fp_status)  || float64_le(fdt0, fdt1, &env->active_fpu.fp_status))
 
 #define FOP_COND_S(op, cond)                                   \
-void helper_cmp_s_ ## op (uint32_t fst0, uint32_t fst1, int cc)    \
+void helper_cmp_s_ ## op(CPUMIPSState *env, uint32_t fst0,     \
+                         uint32_t fst1, int cc)                \
 {                                                              \
     int c;                                                     \
-    set_float_exception_flags(0, &env->active_fpu.fp_status);  \
     c = cond;                                                  \
-    update_fcr31();                                            \
+    update_fcr31(env, GETPC());                                \
     if (c)                                                     \
         SET_FP_COND(cc, env->active_fpu);                      \
     else                                                       \
         CLEAR_FP_COND(cc, env->active_fpu);                    \
 }                                                              \
-void helper_cmpabs_s_ ## op (uint32_t fst0, uint32_t fst1, int cc) \
+void helper_cmpabs_s_ ## op(CPUMIPSState *env, uint32_t fst0,  \
+                            uint32_t fst1, int cc)             \
 {                                                              \
     int c;                                                     \
-    set_float_exception_flags(0, &env->active_fpu.fp_status);  \
     fst0 = float32_abs(fst0);                                  \
     fst1 = float32_abs(fst1);                                  \
     c = cond;                                                  \
-    update_fcr31();                                            \
+    update_fcr31(env, GETPC());                                \
     if (c)                                                     \
         SET_FP_COND(cc, env->active_fpu);                      \
     else                                                       \
@@ -3307,18 +3107,18 @@ FOP_COND_S(le,  float32_le(fst0, fst1, &env->active_fpu.fp_status))
 FOP_COND_S(ngt, float32_unordered(fst1, fst0, &env->active_fpu.fp_status)  || float32_le(fst0, fst1, &env->active_fpu.fp_status))
 
 #define FOP_COND_PS(op, condl, condh)                           \
-void helper_cmp_ps_ ## op (uint64_t fdt0, uint64_t fdt1, int cc)    \
+void helper_cmp_ps_ ## op(CPUMIPSState *env, uint64_t fdt0,     \
+                          uint64_t fdt1, int cc)                \
 {                                                               \
     uint32_t fst0, fsth0, fst1, fsth1;                          \
     int ch, cl;                                                 \
-    set_float_exception_flags(0, &env->active_fpu.fp_status);   \
     fst0 = fdt0 & 0XFFFFFFFF;                                   \
     fsth0 = fdt0 >> 32;                                         \
     fst1 = fdt1 & 0XFFFFFFFF;                                   \
     fsth1 = fdt1 >> 32;                                         \
     cl = condl;                                                 \
     ch = condh;                                                 \
-    update_fcr31();                                             \
+    update_fcr31(env, GETPC());                                 \
     if (cl)                                                     \
         SET_FP_COND(cc, env->active_fpu);                       \
     else                                                        \
@@ -3328,7 +3128,8 @@ void helper_cmp_ps_ ## op (uint64_t fdt0, uint64_t fdt1, int cc)    \
     else                                                        \
         CLEAR_FP_COND(cc + 1, env->active_fpu);                 \
 }                                                               \
-void helper_cmpabs_ps_ ## op (uint64_t fdt0, uint64_t fdt1, int cc) \
+void helper_cmpabs_ps_ ## op(CPUMIPSState *env, uint64_t fdt0,  \
+                             uint64_t fdt1, int cc)             \
 {                                                               \
     uint32_t fst0, fsth0, fst1, fsth1;                          \
     int ch, cl;                                                 \
@@ -3338,7 +3139,7 @@ void helper_cmpabs_ps_ ## op (uint64_t fdt0, uint64_t fdt1, int cc) \
     fsth1 = float32_abs(fdt1 >> 32);                            \
     cl = condl;                                                 \
     ch = condh;                                                 \
-    update_fcr31();                                             \
+    update_fcr31(env, GETPC());                                 \
     if (cl)                                                     \
         SET_FP_COND(cc, env->active_fpu);                       \
     else                                                        \
diff --git a/target-mips/translate.c b/target-mips/translate.c
index b29341953..71c55bcad 100644
--- a/target-mips/translate.c
+++ b/target-mips/translate.c
@@ -5,6 +5,7 @@
  *  Copyright (c) 2006 Marius Groeger (FPU operations)
  *  Copyright (c) 2006 Thiemo Seufer (MIPS32R2 support)
  *  Copyright (c) 2009 CodeSourcery (MIPS16 and microMIPS support)
+ *  Copyright (c) 2012 Jia Liu & Dongxue Zhang (MIPS ASE DSP support)
  *
  * This library is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Lesser General Public
@@ -28,7 +29,7 @@
 #define GEN_HELPER 1
 #include "helper.h"
 
-//#define MIPS_DEBUG_DISAS
+#define MIPS_DEBUG_DISAS 0
 //#define MIPS_DEBUG_SIGN_EXTENSIONS
 
 /* MIPS major opcodes */
@@ -312,6 +313,35 @@ enum {
     OPC_MODU_G_2E   = 0x23 | OPC_SPECIAL3,
     OPC_DMOD_G_2E   = 0x26 | OPC_SPECIAL3,
     OPC_DMODU_G_2E  = 0x27 | OPC_SPECIAL3,
+
+    /* MIPS DSP Load */
+    OPC_LX_DSP         = 0x0A | OPC_SPECIAL3,
+    /* MIPS DSP Arithmetic */
+    OPC_ADDU_QB_DSP    = 0x10 | OPC_SPECIAL3,
+    OPC_ADDU_OB_DSP    = 0x14 | OPC_SPECIAL3,
+    OPC_ABSQ_S_PH_DSP  = 0x12 | OPC_SPECIAL3,
+    OPC_ABSQ_S_QH_DSP  = 0x16 | OPC_SPECIAL3,
+    /* OPC_ADDUH_QB_DSP is same as OPC_MULT_G_2E.  */
+    /* OPC_ADDUH_QB_DSP   = 0x18 | OPC_SPECIAL3,  */
+    OPC_CMPU_EQ_QB_DSP = 0x11 | OPC_SPECIAL3,
+    OPC_CMPU_EQ_OB_DSP = 0x15 | OPC_SPECIAL3,
+    /* MIPS DSP GPR-Based Shift Sub-class */
+    OPC_SHLL_QB_DSP    = 0x13 | OPC_SPECIAL3,
+    OPC_SHLL_OB_DSP    = 0x17 | OPC_SPECIAL3,
+    /* MIPS DSP Multiply Sub-class insns */
+    /* OPC_MUL_PH_DSP is same as OPC_ADDUH_QB_DSP.  */
+    /* OPC_MUL_PH_DSP     = 0x18 | OPC_SPECIAL3,  */
+    OPC_DPA_W_PH_DSP   = 0x30 | OPC_SPECIAL3,
+    OPC_DPAQ_W_QH_DSP  = 0x34 | OPC_SPECIAL3,
+    /* DSP Bit/Manipulation Sub-class */
+    OPC_INSV_DSP       = 0x0C | OPC_SPECIAL3,
+    OPC_DINSV_DSP      = 0x0D | OPC_SPECIAL3,
+    /* MIPS DSP Compare-Pick Sub-class */
+    OPC_APPEND_DSP     = 0x31 | OPC_SPECIAL3,
+    OPC_DAPPEND_DSP    = 0x35 | OPC_SPECIAL3,
+    /* MIPS DSP Accumulator and DSPControl Access Sub-class */
+    OPC_EXTR_W_DSP     = 0x38 | OPC_SPECIAL3,
+    OPC_DEXTR_W_DSP    = 0x3C | OPC_SPECIAL3,
 };
 
 /* BSHFL opcodes */
@@ -331,6 +361,413 @@ enum {
     OPC_DSHD     = (0x05 << 6) | OPC_DBSHFL,
 };
 
+/* MIPS DSP REGIMM opcodes */
+enum {
+    OPC_BPOSGE32 = (0x1C << 16) | OPC_REGIMM,
+    OPC_BPOSGE64 = (0x1D << 16) | OPC_REGIMM,
+};
+
+#define MASK_LX(op) (MASK_SPECIAL3(op) | (op & (0x1F << 6)))
+/* MIPS DSP Load */
+enum {
+    OPC_LBUX = (0x06 << 6) | OPC_LX_DSP,
+    OPC_LHX  = (0x04 << 6) | OPC_LX_DSP,
+    OPC_LWX  = (0x00 << 6) | OPC_LX_DSP,
+    OPC_LDX = (0x08 << 6) | OPC_LX_DSP,
+};
+
+#define MASK_ADDU_QB(op) (MASK_SPECIAL3(op) | (op & (0x1F << 6)))
+enum {
+    /* MIPS DSP Arithmetic Sub-class */
+    OPC_ADDQ_PH        = (0x0A << 6) | OPC_ADDU_QB_DSP,
+    OPC_ADDQ_S_PH      = (0x0E << 6) | OPC_ADDU_QB_DSP,
+    OPC_ADDQ_S_W       = (0x16 << 6) | OPC_ADDU_QB_DSP,
+    OPC_ADDU_QB        = (0x00 << 6) | OPC_ADDU_QB_DSP,
+    OPC_ADDU_S_QB      = (0x04 << 6) | OPC_ADDU_QB_DSP,
+    OPC_ADDU_PH        = (0x08 << 6) | OPC_ADDU_QB_DSP,
+    OPC_ADDU_S_PH      = (0x0C << 6) | OPC_ADDU_QB_DSP,
+    OPC_SUBQ_PH        = (0x0B << 6) | OPC_ADDU_QB_DSP,
+    OPC_SUBQ_S_PH      = (0x0F << 6) | OPC_ADDU_QB_DSP,
+    OPC_SUBQ_S_W       = (0x17 << 6) | OPC_ADDU_QB_DSP,
+    OPC_SUBU_QB        = (0x01 << 6) | OPC_ADDU_QB_DSP,
+    OPC_SUBU_S_QB      = (0x05 << 6) | OPC_ADDU_QB_DSP,
+    OPC_SUBU_PH        = (0x09 << 6) | OPC_ADDU_QB_DSP,
+    OPC_SUBU_S_PH      = (0x0D << 6) | OPC_ADDU_QB_DSP,
+    OPC_ADDSC          = (0x10 << 6) | OPC_ADDU_QB_DSP,
+    OPC_ADDWC          = (0x11 << 6) | OPC_ADDU_QB_DSP,
+    OPC_MODSUB         = (0x12 << 6) | OPC_ADDU_QB_DSP,
+    OPC_RADDU_W_QB     = (0x14 << 6) | OPC_ADDU_QB_DSP,
+    /* MIPS DSP Multiply Sub-class insns */
+    OPC_MULEU_S_PH_QBL = (0x06 << 6) | OPC_ADDU_QB_DSP,
+    OPC_MULEU_S_PH_QBR = (0x07 << 6) | OPC_ADDU_QB_DSP,
+    OPC_MULQ_RS_PH     = (0x1F << 6) | OPC_ADDU_QB_DSP,
+    OPC_MULEQ_S_W_PHL  = (0x1C << 6) | OPC_ADDU_QB_DSP,
+    OPC_MULEQ_S_W_PHR  = (0x1D << 6) | OPC_ADDU_QB_DSP,
+    OPC_MULQ_S_PH      = (0x1E << 6) | OPC_ADDU_QB_DSP,
+};
+
+#define OPC_ADDUH_QB_DSP OPC_MULT_G_2E
+#define MASK_ADDUH_QB(op) (MASK_SPECIAL3(op) | (op & (0x1F << 6)))
+enum {
+    /* MIPS DSP Arithmetic Sub-class */
+    OPC_ADDUH_QB   = (0x00 << 6) | OPC_ADDUH_QB_DSP,
+    OPC_ADDUH_R_QB = (0x02 << 6) | OPC_ADDUH_QB_DSP,
+    OPC_ADDQH_PH   = (0x08 << 6) | OPC_ADDUH_QB_DSP,
+    OPC_ADDQH_R_PH = (0x0A << 6) | OPC_ADDUH_QB_DSP,
+    OPC_ADDQH_W    = (0x10 << 6) | OPC_ADDUH_QB_DSP,
+    OPC_ADDQH_R_W  = (0x12 << 6) | OPC_ADDUH_QB_DSP,
+    OPC_SUBUH_QB   = (0x01 << 6) | OPC_ADDUH_QB_DSP,
+    OPC_SUBUH_R_QB = (0x03 << 6) | OPC_ADDUH_QB_DSP,
+    OPC_SUBQH_PH   = (0x09 << 6) | OPC_ADDUH_QB_DSP,
+    OPC_SUBQH_R_PH = (0x0B << 6) | OPC_ADDUH_QB_DSP,
+    OPC_SUBQH_W    = (0x11 << 6) | OPC_ADDUH_QB_DSP,
+    OPC_SUBQH_R_W  = (0x13 << 6) | OPC_ADDUH_QB_DSP,
+    /* MIPS DSP Multiply Sub-class insns */
+    OPC_MUL_PH     = (0x0C << 6) | OPC_ADDUH_QB_DSP,
+    OPC_MUL_S_PH   = (0x0E << 6) | OPC_ADDUH_QB_DSP,
+    OPC_MULQ_S_W   = (0x16 << 6) | OPC_ADDUH_QB_DSP,
+    OPC_MULQ_RS_W  = (0x17 << 6) | OPC_ADDUH_QB_DSP,
+};
+
+#define MASK_ABSQ_S_PH(op) (MASK_SPECIAL3(op) | (op & (0x1F << 6)))
+enum {
+    /* MIPS DSP Arithmetic Sub-class */
+    OPC_ABSQ_S_QB       = (0x01 << 6) | OPC_ABSQ_S_PH_DSP,
+    OPC_ABSQ_S_PH       = (0x09 << 6) | OPC_ABSQ_S_PH_DSP,
+    OPC_ABSQ_S_W        = (0x11 << 6) | OPC_ABSQ_S_PH_DSP,
+    OPC_PRECEQ_W_PHL    = (0x0C << 6) | OPC_ABSQ_S_PH_DSP,
+    OPC_PRECEQ_W_PHR    = (0x0D << 6) | OPC_ABSQ_S_PH_DSP,
+    OPC_PRECEQU_PH_QBL  = (0x04 << 6) | OPC_ABSQ_S_PH_DSP,
+    OPC_PRECEQU_PH_QBR  = (0x05 << 6) | OPC_ABSQ_S_PH_DSP,
+    OPC_PRECEQU_PH_QBLA = (0x06 << 6) | OPC_ABSQ_S_PH_DSP,
+    OPC_PRECEQU_PH_QBRA = (0x07 << 6) | OPC_ABSQ_S_PH_DSP,
+    OPC_PRECEU_PH_QBL   = (0x1C << 6) | OPC_ABSQ_S_PH_DSP,
+    OPC_PRECEU_PH_QBR   = (0x1D << 6) | OPC_ABSQ_S_PH_DSP,
+    OPC_PRECEU_PH_QBLA  = (0x1E << 6) | OPC_ABSQ_S_PH_DSP,
+    OPC_PRECEU_PH_QBRA  = (0x1F << 6) | OPC_ABSQ_S_PH_DSP,
+    /* DSP Bit/Manipulation Sub-class */
+    OPC_BITREV          = (0x1B << 6) | OPC_ABSQ_S_PH_DSP,
+    OPC_REPL_QB         = (0x02 << 6) | OPC_ABSQ_S_PH_DSP,
+    OPC_REPLV_QB        = (0x03 << 6) | OPC_ABSQ_S_PH_DSP,
+    OPC_REPL_PH         = (0x0A << 6) | OPC_ABSQ_S_PH_DSP,
+    OPC_REPLV_PH        = (0x0B << 6) | OPC_ABSQ_S_PH_DSP,
+};
+
+#define MASK_CMPU_EQ_QB(op) (MASK_SPECIAL3(op) | (op & (0x1F << 6)))
+enum {
+    /* MIPS DSP Arithmetic Sub-class */
+    OPC_PRECR_QB_PH      = (0x0D << 6) | OPC_CMPU_EQ_QB_DSP,
+    OPC_PRECRQ_QB_PH     = (0x0C << 6) | OPC_CMPU_EQ_QB_DSP,
+    OPC_PRECR_SRA_PH_W   = (0x1E << 6) | OPC_CMPU_EQ_QB_DSP,
+    OPC_PRECR_SRA_R_PH_W = (0x1F << 6) | OPC_CMPU_EQ_QB_DSP,
+    OPC_PRECRQ_PH_W      = (0x14 << 6) | OPC_CMPU_EQ_QB_DSP,
+    OPC_PRECRQ_RS_PH_W   = (0x15 << 6) | OPC_CMPU_EQ_QB_DSP,
+    OPC_PRECRQU_S_QB_PH  = (0x0F << 6) | OPC_CMPU_EQ_QB_DSP,
+    /* DSP Compare-Pick Sub-class */
+    OPC_CMPU_EQ_QB       = (0x00 << 6) | OPC_CMPU_EQ_QB_DSP,
+    OPC_CMPU_LT_QB       = (0x01 << 6) | OPC_CMPU_EQ_QB_DSP,
+    OPC_CMPU_LE_QB       = (0x02 << 6) | OPC_CMPU_EQ_QB_DSP,
+    OPC_CMPGU_EQ_QB      = (0x04 << 6) | OPC_CMPU_EQ_QB_DSP,
+    OPC_CMPGU_LT_QB      = (0x05 << 6) | OPC_CMPU_EQ_QB_DSP,
+    OPC_CMPGU_LE_QB      = (0x06 << 6) | OPC_CMPU_EQ_QB_DSP,
+    OPC_CMPGDU_EQ_QB     = (0x18 << 6) | OPC_CMPU_EQ_QB_DSP,
+    OPC_CMPGDU_LT_QB     = (0x19 << 6) | OPC_CMPU_EQ_QB_DSP,
+    OPC_CMPGDU_LE_QB     = (0x1A << 6) | OPC_CMPU_EQ_QB_DSP,
+    OPC_CMP_EQ_PH        = (0x08 << 6) | OPC_CMPU_EQ_QB_DSP,
+    OPC_CMP_LT_PH        = (0x09 << 6) | OPC_CMPU_EQ_QB_DSP,
+    OPC_CMP_LE_PH        = (0x0A << 6) | OPC_CMPU_EQ_QB_DSP,
+    OPC_PICK_QB          = (0x03 << 6) | OPC_CMPU_EQ_QB_DSP,
+    OPC_PICK_PH          = (0x0B << 6) | OPC_CMPU_EQ_QB_DSP,
+    OPC_PACKRL_PH        = (0x0E << 6) | OPC_CMPU_EQ_QB_DSP,
+};
+
+#define MASK_SHLL_QB(op) (MASK_SPECIAL3(op) | (op & (0x1F << 6)))
+enum {
+    /* MIPS DSP GPR-Based Shift Sub-class */
+    OPC_SHLL_QB    = (0x00 << 6) | OPC_SHLL_QB_DSP,
+    OPC_SHLLV_QB   = (0x02 << 6) | OPC_SHLL_QB_DSP,
+    OPC_SHLL_PH    = (0x08 << 6) | OPC_SHLL_QB_DSP,
+    OPC_SHLLV_PH   = (0x0A << 6) | OPC_SHLL_QB_DSP,
+    OPC_SHLL_S_PH  = (0x0C << 6) | OPC_SHLL_QB_DSP,
+    OPC_SHLLV_S_PH = (0x0E << 6) | OPC_SHLL_QB_DSP,
+    OPC_SHLL_S_W   = (0x14 << 6) | OPC_SHLL_QB_DSP,
+    OPC_SHLLV_S_W  = (0x16 << 6) | OPC_SHLL_QB_DSP,
+    OPC_SHRL_QB    = (0x01 << 6) | OPC_SHLL_QB_DSP,
+    OPC_SHRLV_QB   = (0x03 << 6) | OPC_SHLL_QB_DSP,
+    OPC_SHRL_PH    = (0x19 << 6) | OPC_SHLL_QB_DSP,
+    OPC_SHRLV_PH   = (0x1B << 6) | OPC_SHLL_QB_DSP,
+    OPC_SHRA_QB    = (0x04 << 6) | OPC_SHLL_QB_DSP,
+    OPC_SHRA_R_QB  = (0x05 << 6) | OPC_SHLL_QB_DSP,
+    OPC_SHRAV_QB   = (0x06 << 6) | OPC_SHLL_QB_DSP,
+    OPC_SHRAV_R_QB = (0x07 << 6) | OPC_SHLL_QB_DSP,
+    OPC_SHRA_PH    = (0x09 << 6) | OPC_SHLL_QB_DSP,
+    OPC_SHRAV_PH   = (0x0B << 6) | OPC_SHLL_QB_DSP,
+    OPC_SHRA_R_PH  = (0x0D << 6) | OPC_SHLL_QB_DSP,
+    OPC_SHRAV_R_PH = (0x0F << 6) | OPC_SHLL_QB_DSP,
+    OPC_SHRA_R_W   = (0x15 << 6) | OPC_SHLL_QB_DSP,
+    OPC_SHRAV_R_W  = (0x17 << 6) | OPC_SHLL_QB_DSP,
+};
+
+#define MASK_DPA_W_PH(op) (MASK_SPECIAL3(op) | (op & (0x1F << 6)))
+enum {
+    /* MIPS DSP Multiply Sub-class insns */
+    OPC_DPAU_H_QBL    = (0x03 << 6) | OPC_DPA_W_PH_DSP,
+    OPC_DPAU_H_QBR    = (0x07 << 6) | OPC_DPA_W_PH_DSP,
+    OPC_DPSU_H_QBL    = (0x0B << 6) | OPC_DPA_W_PH_DSP,
+    OPC_DPSU_H_QBR    = (0x0F << 6) | OPC_DPA_W_PH_DSP,
+    OPC_DPA_W_PH      = (0x00 << 6) | OPC_DPA_W_PH_DSP,
+    OPC_DPAX_W_PH     = (0x08 << 6) | OPC_DPA_W_PH_DSP,
+    OPC_DPAQ_S_W_PH   = (0x04 << 6) | OPC_DPA_W_PH_DSP,
+    OPC_DPAQX_S_W_PH  = (0x18 << 6) | OPC_DPA_W_PH_DSP,
+    OPC_DPAQX_SA_W_PH = (0x1A << 6) | OPC_DPA_W_PH_DSP,
+    OPC_DPS_W_PH      = (0x01 << 6) | OPC_DPA_W_PH_DSP,
+    OPC_DPSX_W_PH     = (0x09 << 6) | OPC_DPA_W_PH_DSP,
+    OPC_DPSQ_S_W_PH   = (0x05 << 6) | OPC_DPA_W_PH_DSP,
+    OPC_DPSQX_S_W_PH  = (0x19 << 6) | OPC_DPA_W_PH_DSP,
+    OPC_DPSQX_SA_W_PH = (0x1B << 6) | OPC_DPA_W_PH_DSP,
+    OPC_MULSAQ_S_W_PH = (0x06 << 6) | OPC_DPA_W_PH_DSP,
+    OPC_DPAQ_SA_L_W   = (0x0C << 6) | OPC_DPA_W_PH_DSP,
+    OPC_DPSQ_SA_L_W   = (0x0D << 6) | OPC_DPA_W_PH_DSP,
+    OPC_MAQ_S_W_PHL   = (0x14 << 6) | OPC_DPA_W_PH_DSP,
+    OPC_MAQ_S_W_PHR   = (0x16 << 6) | OPC_DPA_W_PH_DSP,
+    OPC_MAQ_SA_W_PHL  = (0x10 << 6) | OPC_DPA_W_PH_DSP,
+    OPC_MAQ_SA_W_PHR  = (0x12 << 6) | OPC_DPA_W_PH_DSP,
+    OPC_MULSA_W_PH    = (0x02 << 6) | OPC_DPA_W_PH_DSP,
+};
+
+#define MASK_INSV(op) (MASK_SPECIAL3(op) | (op & (0x1F << 6)))
+enum {
+    /* DSP Bit/Manipulation Sub-class */
+    OPC_INSV = (0x00 << 6) | OPC_INSV_DSP,
+};
+
+#define MASK_APPEND(op) (MASK_SPECIAL3(op) | (op & (0x1F << 6)))
+enum {
+    /* MIPS DSP Compare-Pick Sub-class */
+    OPC_APPEND  = (0x00 << 6) | OPC_APPEND_DSP,
+    OPC_PREPEND = (0x01 << 6) | OPC_APPEND_DSP,
+    OPC_BALIGN  = (0x10 << 6) | OPC_APPEND_DSP,
+};
+
+#define MASK_EXTR_W(op) (MASK_SPECIAL3(op) | (op & (0x1F << 6)))
+enum {
+    /* MIPS DSP Accumulator and DSPControl Access Sub-class */
+    OPC_EXTR_W     = (0x00 << 6) | OPC_EXTR_W_DSP,
+    OPC_EXTR_R_W   = (0x04 << 6) | OPC_EXTR_W_DSP,
+    OPC_EXTR_RS_W  = (0x06 << 6) | OPC_EXTR_W_DSP,
+    OPC_EXTR_S_H   = (0x0E << 6) | OPC_EXTR_W_DSP,
+    OPC_EXTRV_S_H  = (0x0F << 6) | OPC_EXTR_W_DSP,
+    OPC_EXTRV_W    = (0x01 << 6) | OPC_EXTR_W_DSP,
+    OPC_EXTRV_R_W  = (0x05 << 6) | OPC_EXTR_W_DSP,
+    OPC_EXTRV_RS_W = (0x07 << 6) | OPC_EXTR_W_DSP,
+    OPC_EXTP       = (0x02 << 6) | OPC_EXTR_W_DSP,
+    OPC_EXTPV      = (0x03 << 6) | OPC_EXTR_W_DSP,
+    OPC_EXTPDP     = (0x0A << 6) | OPC_EXTR_W_DSP,
+    OPC_EXTPDPV    = (0x0B << 6) | OPC_EXTR_W_DSP,
+    OPC_SHILO      = (0x1A << 6) | OPC_EXTR_W_DSP,
+    OPC_SHILOV     = (0x1B << 6) | OPC_EXTR_W_DSP,
+    OPC_MTHLIP     = (0x1F << 6) | OPC_EXTR_W_DSP,
+    OPC_WRDSP      = (0x13 << 6) | OPC_EXTR_W_DSP,
+    OPC_RDDSP      = (0x12 << 6) | OPC_EXTR_W_DSP,
+};
+
+#define MASK_ABSQ_S_QH(op) (MASK_SPECIAL3(op) | (op & (0x1F << 6)))
+enum {
+    /* MIPS DSP Arithmetic Sub-class */
+    OPC_PRECEQ_L_PWL    = (0x14 << 6) | OPC_ABSQ_S_QH_DSP,
+    OPC_PRECEQ_L_PWR    = (0x15 << 6) | OPC_ABSQ_S_QH_DSP,
+    OPC_PRECEQ_PW_QHL   = (0x0C << 6) | OPC_ABSQ_S_QH_DSP,
+    OPC_PRECEQ_PW_QHR   = (0x0D << 6) | OPC_ABSQ_S_QH_DSP,
+    OPC_PRECEQ_PW_QHLA  = (0x0E << 6) | OPC_ABSQ_S_QH_DSP,
+    OPC_PRECEQ_PW_QHRA  = (0x0F << 6) | OPC_ABSQ_S_QH_DSP,
+    OPC_PRECEQU_QH_OBL  = (0x04 << 6) | OPC_ABSQ_S_QH_DSP,
+    OPC_PRECEQU_QH_OBR  = (0x05 << 6) | OPC_ABSQ_S_QH_DSP,
+    OPC_PRECEQU_QH_OBLA = (0x06 << 6) | OPC_ABSQ_S_QH_DSP,
+    OPC_PRECEQU_QH_OBRA = (0x07 << 6) | OPC_ABSQ_S_QH_DSP,
+    OPC_PRECEU_QH_OBL   = (0x1C << 6) | OPC_ABSQ_S_QH_DSP,
+    OPC_PRECEU_QH_OBR   = (0x1D << 6) | OPC_ABSQ_S_QH_DSP,
+    OPC_PRECEU_QH_OBLA  = (0x1E << 6) | OPC_ABSQ_S_QH_DSP,
+    OPC_PRECEU_QH_OBRA  = (0x1F << 6) | OPC_ABSQ_S_QH_DSP,
+    OPC_ABSQ_S_OB       = (0x01 << 6) | OPC_ABSQ_S_QH_DSP,
+    OPC_ABSQ_S_PW       = (0x11 << 6) | OPC_ABSQ_S_QH_DSP,
+    OPC_ABSQ_S_QH       = (0x09 << 6) | OPC_ABSQ_S_QH_DSP,
+    /* DSP Bit/Manipulation Sub-class */
+    OPC_REPL_OB         = (0x02 << 6) | OPC_ABSQ_S_QH_DSP,
+    OPC_REPL_PW         = (0x12 << 6) | OPC_ABSQ_S_QH_DSP,
+    OPC_REPL_QH         = (0x0A << 6) | OPC_ABSQ_S_QH_DSP,
+    OPC_REPLV_OB        = (0x03 << 6) | OPC_ABSQ_S_QH_DSP,
+    OPC_REPLV_PW        = (0x13 << 6) | OPC_ABSQ_S_QH_DSP,
+    OPC_REPLV_QH        = (0x0B << 6) | OPC_ABSQ_S_QH_DSP,
+};
+
+#define MASK_ADDU_OB(op) (MASK_SPECIAL3(op) | (op & (0x1F << 6)))
+enum {
+    /* MIPS DSP Multiply Sub-class insns */
+    OPC_MULEQ_S_PW_QHL = (0x1C << 6) | OPC_ADDU_OB_DSP,
+    OPC_MULEQ_S_PW_QHR = (0x1D << 6) | OPC_ADDU_OB_DSP,
+    OPC_MULEU_S_QH_OBL = (0x06 << 6) | OPC_ADDU_OB_DSP,
+    OPC_MULEU_S_QH_OBR = (0x07 << 6) | OPC_ADDU_OB_DSP,
+    OPC_MULQ_RS_QH     = (0x1F << 6) | OPC_ADDU_OB_DSP,
+    /* MIPS DSP Arithmetic Sub-class */
+    OPC_RADDU_L_OB     = (0x14 << 6) | OPC_ADDU_OB_DSP,
+    OPC_SUBQ_PW        = (0x13 << 6) | OPC_ADDU_OB_DSP,
+    OPC_SUBQ_S_PW      = (0x17 << 6) | OPC_ADDU_OB_DSP,
+    OPC_SUBQ_QH        = (0x0B << 6) | OPC_ADDU_OB_DSP,
+    OPC_SUBQ_S_QH      = (0x0F << 6) | OPC_ADDU_OB_DSP,
+    OPC_SUBU_OB        = (0x01 << 6) | OPC_ADDU_OB_DSP,
+    OPC_SUBU_S_OB      = (0x05 << 6) | OPC_ADDU_OB_DSP,
+    OPC_SUBU_QH        = (0x09 << 6) | OPC_ADDU_OB_DSP,
+    OPC_SUBU_S_QH      = (0x0D << 6) | OPC_ADDU_OB_DSP,
+    OPC_SUBUH_OB       = (0x19 << 6) | OPC_ADDU_OB_DSP,
+    OPC_SUBUH_R_OB     = (0x1B << 6) | OPC_ADDU_OB_DSP,
+    OPC_ADDQ_PW        = (0x12 << 6) | OPC_ADDU_OB_DSP,
+    OPC_ADDQ_S_PW      = (0x16 << 6) | OPC_ADDU_OB_DSP,
+    OPC_ADDQ_QH        = (0x0A << 6) | OPC_ADDU_OB_DSP,
+    OPC_ADDQ_S_QH      = (0x0E << 6) | OPC_ADDU_OB_DSP,
+    OPC_ADDU_OB        = (0x00 << 6) | OPC_ADDU_OB_DSP,
+    OPC_ADDU_S_OB      = (0x04 << 6) | OPC_ADDU_OB_DSP,
+    OPC_ADDU_QH        = (0x08 << 6) | OPC_ADDU_OB_DSP,
+    OPC_ADDU_S_QH      = (0x0C << 6) | OPC_ADDU_OB_DSP,
+    OPC_ADDUH_OB       = (0x18 << 6) | OPC_ADDU_OB_DSP,
+    OPC_ADDUH_R_OB     = (0x1A << 6) | OPC_ADDU_OB_DSP,
+};
+
+#define MASK_CMPU_EQ_OB(op) (MASK_SPECIAL3(op) | (op & (0x1F << 6)))
+enum {
+    /* DSP Compare-Pick Sub-class */
+    OPC_CMP_EQ_PW         = (0x10 << 6) | OPC_CMPU_EQ_OB_DSP,
+    OPC_CMP_LT_PW         = (0x11 << 6) | OPC_CMPU_EQ_OB_DSP,
+    OPC_CMP_LE_PW         = (0x12 << 6) | OPC_CMPU_EQ_OB_DSP,
+    OPC_CMP_EQ_QH         = (0x08 << 6) | OPC_CMPU_EQ_OB_DSP,
+    OPC_CMP_LT_QH         = (0x09 << 6) | OPC_CMPU_EQ_OB_DSP,
+    OPC_CMP_LE_QH         = (0x0A << 6) | OPC_CMPU_EQ_OB_DSP,
+    OPC_CMPGDU_EQ_OB      = (0x18 << 6) | OPC_CMPU_EQ_OB_DSP,
+    OPC_CMPGDU_LT_OB      = (0x19 << 6) | OPC_CMPU_EQ_OB_DSP,
+    OPC_CMPGDU_LE_OB      = (0x1A << 6) | OPC_CMPU_EQ_OB_DSP,
+    OPC_CMPGU_EQ_OB       = (0x04 << 6) | OPC_CMPU_EQ_OB_DSP,
+    OPC_CMPGU_LT_OB       = (0x05 << 6) | OPC_CMPU_EQ_OB_DSP,
+    OPC_CMPGU_LE_OB       = (0x06 << 6) | OPC_CMPU_EQ_OB_DSP,
+    OPC_CMPU_EQ_OB        = (0x00 << 6) | OPC_CMPU_EQ_OB_DSP,
+    OPC_CMPU_LT_OB        = (0x01 << 6) | OPC_CMPU_EQ_OB_DSP,
+    OPC_CMPU_LE_OB        = (0x02 << 6) | OPC_CMPU_EQ_OB_DSP,
+    OPC_PACKRL_PW         = (0x0E << 6) | OPC_CMPU_EQ_OB_DSP,
+    OPC_PICK_OB           = (0x03 << 6) | OPC_CMPU_EQ_OB_DSP,
+    OPC_PICK_PW           = (0x13 << 6) | OPC_CMPU_EQ_OB_DSP,
+    OPC_PICK_QH           = (0x0B << 6) | OPC_CMPU_EQ_OB_DSP,
+    /* MIPS DSP Arithmetic Sub-class */
+    OPC_PRECR_OB_QH       = (0x0D << 6) | OPC_CMPU_EQ_OB_DSP,
+    OPC_PRECR_SRA_QH_PW   = (0x1E << 6) | OPC_CMPU_EQ_OB_DSP,
+    OPC_PRECR_SRA_R_QH_PW = (0x1F << 6) | OPC_CMPU_EQ_OB_DSP,
+    OPC_PRECRQ_OB_QH      = (0x0C << 6) | OPC_CMPU_EQ_OB_DSP,
+    OPC_PRECRQ_PW_L       = (0x1C << 6) | OPC_CMPU_EQ_OB_DSP,
+    OPC_PRECRQ_QH_PW      = (0x14 << 6) | OPC_CMPU_EQ_OB_DSP,
+    OPC_PRECRQ_RS_QH_PW   = (0x15 << 6) | OPC_CMPU_EQ_OB_DSP,
+    OPC_PRECRQU_S_OB_QH   = (0x0F << 6) | OPC_CMPU_EQ_OB_DSP,
+};
+
+#define MASK_DAPPEND(op) (MASK_SPECIAL3(op) | (op & (0x1F << 6)))
+enum {
+    /* DSP Compare-Pick Sub-class */
+    OPC_DAPPEND  = (0x00 << 6) | OPC_DAPPEND_DSP,
+    OPC_PREPENDD = (0x03 << 6) | OPC_DAPPEND_DSP,
+    OPC_PREPENDW = (0x01 << 6) | OPC_DAPPEND_DSP,
+    OPC_DBALIGN  = (0x10 << 6) | OPC_DAPPEND_DSP,
+};
+
+#define MASK_DEXTR_W(op) (MASK_SPECIAL3(op) | (op & (0x1F << 6)))
+enum {
+    /* MIPS DSP Accumulator and DSPControl Access Sub-class */
+    OPC_DMTHLIP     = (0x1F << 6) | OPC_DEXTR_W_DSP,
+    OPC_DSHILO      = (0x1A << 6) | OPC_DEXTR_W_DSP,
+    OPC_DEXTP       = (0x02 << 6) | OPC_DEXTR_W_DSP,
+    OPC_DEXTPDP     = (0x0A << 6) | OPC_DEXTR_W_DSP,
+    OPC_DEXTPDPV    = (0x0B << 6) | OPC_DEXTR_W_DSP,
+    OPC_DEXTPV      = (0x03 << 6) | OPC_DEXTR_W_DSP,
+    OPC_DEXTR_L     = (0x10 << 6) | OPC_DEXTR_W_DSP,
+    OPC_DEXTR_R_L   = (0x14 << 6) | OPC_DEXTR_W_DSP,
+    OPC_DEXTR_RS_L  = (0x16 << 6) | OPC_DEXTR_W_DSP,
+    OPC_DEXTR_W     = (0x00 << 6) | OPC_DEXTR_W_DSP,
+    OPC_DEXTR_R_W   = (0x04 << 6) | OPC_DEXTR_W_DSP,
+    OPC_DEXTR_RS_W  = (0x06 << 6) | OPC_DEXTR_W_DSP,
+    OPC_DEXTR_S_H   = (0x0E << 6) | OPC_DEXTR_W_DSP,
+    OPC_DEXTRV_L    = (0x11 << 6) | OPC_DEXTR_W_DSP,
+    OPC_DEXTRV_R_L  = (0x15 << 6) | OPC_DEXTR_W_DSP,
+    OPC_DEXTRV_RS_L = (0x17 << 6) | OPC_DEXTR_W_DSP,
+    OPC_DEXTRV_S_H  = (0x0F << 6) | OPC_DEXTR_W_DSP,
+    OPC_DEXTRV_W    = (0x01 << 6) | OPC_DEXTR_W_DSP,
+    OPC_DEXTRV_R_W  = (0x05 << 6) | OPC_DEXTR_W_DSP,
+    OPC_DEXTRV_RS_W = (0x07 << 6) | OPC_DEXTR_W_DSP,
+    OPC_DSHILOV     = (0x1B << 6) | OPC_DEXTR_W_DSP,
+};
+
+#define MASK_DINSV(op) (MASK_SPECIAL3(op) | (op & (0x1F << 6)))
+enum {
+    /* DSP Bit/Manipulation Sub-class */
+    OPC_DINSV = (0x00 << 6) | OPC_DINSV_DSP,
+};
+
+#define MASK_DPAQ_W_QH(op) (MASK_SPECIAL3(op) | (op & (0x1F << 6)))
+enum {
+    /* MIPS DSP Multiply Sub-class insns */
+    OPC_DMADD         = (0x19 << 6) | OPC_DPAQ_W_QH_DSP,
+    OPC_DMADDU        = (0x1D << 6) | OPC_DPAQ_W_QH_DSP,
+    OPC_DMSUB         = (0x1B << 6) | OPC_DPAQ_W_QH_DSP,
+    OPC_DMSUBU        = (0x1F << 6) | OPC_DPAQ_W_QH_DSP,
+    OPC_DPA_W_QH      = (0x00 << 6) | OPC_DPAQ_W_QH_DSP,
+    OPC_DPAQ_S_W_QH   = (0x04 << 6) | OPC_DPAQ_W_QH_DSP,
+    OPC_DPAQ_SA_L_PW  = (0x0C << 6) | OPC_DPAQ_W_QH_DSP,
+    OPC_DPAU_H_OBL    = (0x03 << 6) | OPC_DPAQ_W_QH_DSP,
+    OPC_DPAU_H_OBR    = (0x07 << 6) | OPC_DPAQ_W_QH_DSP,
+    OPC_DPS_W_QH      = (0x01 << 6) | OPC_DPAQ_W_QH_DSP,
+    OPC_DPSQ_S_W_QH   = (0x05 << 6) | OPC_DPAQ_W_QH_DSP,
+    OPC_DPSQ_SA_L_PW  = (0x0D << 6) | OPC_DPAQ_W_QH_DSP,
+    OPC_DPSU_H_OBL    = (0x0B << 6) | OPC_DPAQ_W_QH_DSP,
+    OPC_DPSU_H_OBR    = (0x0F << 6) | OPC_DPAQ_W_QH_DSP,
+    OPC_MAQ_S_L_PWL   = (0x1C << 6) | OPC_DPAQ_W_QH_DSP,
+    OPC_MAQ_S_L_PWR   = (0x1E << 6) | OPC_DPAQ_W_QH_DSP,
+    OPC_MAQ_S_W_QHLL  = (0x14 << 6) | OPC_DPAQ_W_QH_DSP,
+    OPC_MAQ_SA_W_QHLL = (0x10 << 6) | OPC_DPAQ_W_QH_DSP,
+    OPC_MAQ_S_W_QHLR  = (0x15 << 6) | OPC_DPAQ_W_QH_DSP,
+    OPC_MAQ_SA_W_QHLR = (0x11 << 6) | OPC_DPAQ_W_QH_DSP,
+    OPC_MAQ_S_W_QHRL  = (0x16 << 6) | OPC_DPAQ_W_QH_DSP,
+    OPC_MAQ_SA_W_QHRL = (0x12 << 6) | OPC_DPAQ_W_QH_DSP,
+    OPC_MAQ_S_W_QHRR  = (0x17 << 6) | OPC_DPAQ_W_QH_DSP,
+    OPC_MAQ_SA_W_QHRR = (0x13 << 6) | OPC_DPAQ_W_QH_DSP,
+    OPC_MULSAQ_S_L_PW = (0x0E << 6) | OPC_DPAQ_W_QH_DSP,
+    OPC_MULSAQ_S_W_QH = (0x06 << 6) | OPC_DPAQ_W_QH_DSP,
+};
+
+#define MASK_SHLL_OB(op) (MASK_SPECIAL3(op) | (op & (0x1F << 6)))
+enum {
+    /* MIPS DSP GPR-Based Shift Sub-class */
+    OPC_SHLL_PW    = (0x10 << 6) | OPC_SHLL_OB_DSP,
+    OPC_SHLL_S_PW  = (0x14 << 6) | OPC_SHLL_OB_DSP,
+    OPC_SHLLV_OB   = (0x02 << 6) | OPC_SHLL_OB_DSP,
+    OPC_SHLLV_PW   = (0x12 << 6) | OPC_SHLL_OB_DSP,
+    OPC_SHLLV_S_PW = (0x16 << 6) | OPC_SHLL_OB_DSP,
+    OPC_SHLLV_QH   = (0x0A << 6) | OPC_SHLL_OB_DSP,
+    OPC_SHLLV_S_QH = (0x0E << 6) | OPC_SHLL_OB_DSP,
+    OPC_SHRA_PW    = (0x11 << 6) | OPC_SHLL_OB_DSP,
+    OPC_SHRA_R_PW  = (0x15 << 6) | OPC_SHLL_OB_DSP,
+    OPC_SHRAV_OB   = (0x06 << 6) | OPC_SHLL_OB_DSP,
+    OPC_SHRAV_R_OB = (0x07 << 6) | OPC_SHLL_OB_DSP,
+    OPC_SHRAV_PW   = (0x13 << 6) | OPC_SHLL_OB_DSP,
+    OPC_SHRAV_R_PW = (0x17 << 6) | OPC_SHLL_OB_DSP,
+    OPC_SHRAV_QH   = (0x0B << 6) | OPC_SHLL_OB_DSP,
+    OPC_SHRAV_R_QH = (0x0F << 6) | OPC_SHLL_OB_DSP,
+    OPC_SHRLV_OB   = (0x03 << 6) | OPC_SHLL_OB_DSP,
+    OPC_SHRLV_QH   = (0x1B << 6) | OPC_SHLL_OB_DSP,
+    OPC_SHLL_OB    = (0x00 << 6) | OPC_SHLL_OB_DSP,
+    OPC_SHLL_QH    = (0x08 << 6) | OPC_SHLL_OB_DSP,
+    OPC_SHLL_S_QH  = (0x0C << 6) | OPC_SHLL_OB_DSP,
+    OPC_SHRA_OB    = (0x04 << 6) | OPC_SHLL_OB_DSP,
+    OPC_SHRA_R_OB  = (0x05 << 6) | OPC_SHLL_OB_DSP,
+    OPC_SHRA_QH    = (0x09 << 6) | OPC_SHLL_OB_DSP,
+    OPC_SHRA_R_QH  = (0x0D << 6) | OPC_SHLL_OB_DSP,
+    OPC_SHRL_OB    = (0x01 << 6) | OPC_SHLL_OB_DSP,
+    OPC_SHRL_QH    = (0x19 << 6) | OPC_SHLL_OB_DSP,
+};
+
 /* Coprocessor 0 (rs field) */
 #define MASK_CP0(op)       MASK_OP_MAJOR(op) | (op & (0x1F << 21))
 
@@ -446,6 +883,103 @@ enum {
     OPC_BC2     = (0x08 << 21) | OPC_CP2,
 };
 
+#define MASK_LMI(op)  (MASK_OP_MAJOR(op) | (op & (0x1F << 21)) | (op & 0x1F))
+
+enum {
+    OPC_PADDSH  = (24 << 21) | (0x00) | OPC_CP2,
+    OPC_PADDUSH = (25 << 21) | (0x00) | OPC_CP2,
+    OPC_PADDH   = (26 << 21) | (0x00) | OPC_CP2,
+    OPC_PADDW   = (27 << 21) | (0x00) | OPC_CP2,
+    OPC_PADDSB  = (28 << 21) | (0x00) | OPC_CP2,
+    OPC_PADDUSB = (29 << 21) | (0x00) | OPC_CP2,
+    OPC_PADDB   = (30 << 21) | (0x00) | OPC_CP2,
+    OPC_PADDD   = (31 << 21) | (0x00) | OPC_CP2,
+
+    OPC_PSUBSH  = (24 << 21) | (0x01) | OPC_CP2,
+    OPC_PSUBUSH = (25 << 21) | (0x01) | OPC_CP2,
+    OPC_PSUBH   = (26 << 21) | (0x01) | OPC_CP2,
+    OPC_PSUBW   = (27 << 21) | (0x01) | OPC_CP2,
+    OPC_PSUBSB  = (28 << 21) | (0x01) | OPC_CP2,
+    OPC_PSUBUSB = (29 << 21) | (0x01) | OPC_CP2,
+    OPC_PSUBB   = (30 << 21) | (0x01) | OPC_CP2,
+    OPC_PSUBD   = (31 << 21) | (0x01) | OPC_CP2,
+
+    OPC_PSHUFH   = (24 << 21) | (0x02) | OPC_CP2,
+    OPC_PACKSSWH = (25 << 21) | (0x02) | OPC_CP2,
+    OPC_PACKSSHB = (26 << 21) | (0x02) | OPC_CP2,
+    OPC_PACKUSHB = (27 << 21) | (0x02) | OPC_CP2,
+    OPC_XOR_CP2  = (28 << 21) | (0x02) | OPC_CP2,
+    OPC_NOR_CP2  = (29 << 21) | (0x02) | OPC_CP2,
+    OPC_AND_CP2  = (30 << 21) | (0x02) | OPC_CP2,
+    OPC_PANDN    = (31 << 21) | (0x02) | OPC_CP2,
+
+    OPC_PUNPCKLHW = (24 << 21) | (0x03) | OPC_CP2,
+    OPC_PUNPCKHHW = (25 << 21) | (0x03) | OPC_CP2,
+    OPC_PUNPCKLBH = (26 << 21) | (0x03) | OPC_CP2,
+    OPC_PUNPCKHBH = (27 << 21) | (0x03) | OPC_CP2,
+    OPC_PINSRH_0  = (28 << 21) | (0x03) | OPC_CP2,
+    OPC_PINSRH_1  = (29 << 21) | (0x03) | OPC_CP2,
+    OPC_PINSRH_2  = (30 << 21) | (0x03) | OPC_CP2,
+    OPC_PINSRH_3  = (31 << 21) | (0x03) | OPC_CP2,
+
+    OPC_PAVGH   = (24 << 21) | (0x08) | OPC_CP2,
+    OPC_PAVGB   = (25 << 21) | (0x08) | OPC_CP2,
+    OPC_PMAXSH  = (26 << 21) | (0x08) | OPC_CP2,
+    OPC_PMINSH  = (27 << 21) | (0x08) | OPC_CP2,
+    OPC_PMAXUB  = (28 << 21) | (0x08) | OPC_CP2,
+    OPC_PMINUB  = (29 << 21) | (0x08) | OPC_CP2,
+
+    OPC_PCMPEQW = (24 << 21) | (0x09) | OPC_CP2,
+    OPC_PCMPGTW = (25 << 21) | (0x09) | OPC_CP2,
+    OPC_PCMPEQH = (26 << 21) | (0x09) | OPC_CP2,
+    OPC_PCMPGTH = (27 << 21) | (0x09) | OPC_CP2,
+    OPC_PCMPEQB = (28 << 21) | (0x09) | OPC_CP2,
+    OPC_PCMPGTB = (29 << 21) | (0x09) | OPC_CP2,
+
+    OPC_PSLLW   = (24 << 21) | (0x0A) | OPC_CP2,
+    OPC_PSLLH   = (25 << 21) | (0x0A) | OPC_CP2,
+    OPC_PMULLH  = (26 << 21) | (0x0A) | OPC_CP2,
+    OPC_PMULHH  = (27 << 21) | (0x0A) | OPC_CP2,
+    OPC_PMULUW  = (28 << 21) | (0x0A) | OPC_CP2,
+    OPC_PMULHUH = (29 << 21) | (0x0A) | OPC_CP2,
+
+    OPC_PSRLW     = (24 << 21) | (0x0B) | OPC_CP2,
+    OPC_PSRLH     = (25 << 21) | (0x0B) | OPC_CP2,
+    OPC_PSRAW     = (26 << 21) | (0x0B) | OPC_CP2,
+    OPC_PSRAH     = (27 << 21) | (0x0B) | OPC_CP2,
+    OPC_PUNPCKLWD = (28 << 21) | (0x0B) | OPC_CP2,
+    OPC_PUNPCKHWD = (29 << 21) | (0x0B) | OPC_CP2,
+
+    OPC_ADDU_CP2 = (24 << 21) | (0x0C) | OPC_CP2,
+    OPC_OR_CP2   = (25 << 21) | (0x0C) | OPC_CP2,
+    OPC_ADD_CP2  = (26 << 21) | (0x0C) | OPC_CP2,
+    OPC_DADD_CP2 = (27 << 21) | (0x0C) | OPC_CP2,
+    OPC_SEQU_CP2 = (28 << 21) | (0x0C) | OPC_CP2,
+    OPC_SEQ_CP2  = (29 << 21) | (0x0C) | OPC_CP2,
+
+    OPC_SUBU_CP2 = (24 << 21) | (0x0D) | OPC_CP2,
+    OPC_PASUBUB  = (25 << 21) | (0x0D) | OPC_CP2,
+    OPC_SUB_CP2  = (26 << 21) | (0x0D) | OPC_CP2,
+    OPC_DSUB_CP2 = (27 << 21) | (0x0D) | OPC_CP2,
+    OPC_SLTU_CP2 = (28 << 21) | (0x0D) | OPC_CP2,
+    OPC_SLT_CP2  = (29 << 21) | (0x0D) | OPC_CP2,
+
+    OPC_SLL_CP2  = (24 << 21) | (0x0E) | OPC_CP2,
+    OPC_DSLL_CP2 = (25 << 21) | (0x0E) | OPC_CP2,
+    OPC_PEXTRH   = (26 << 21) | (0x0E) | OPC_CP2,
+    OPC_PMADDHW  = (27 << 21) | (0x0E) | OPC_CP2,
+    OPC_SLEU_CP2 = (28 << 21) | (0x0E) | OPC_CP2,
+    OPC_SLE_CP2  = (29 << 21) | (0x0E) | OPC_CP2,
+
+    OPC_SRL_CP2  = (24 << 21) | (0x0F) | OPC_CP2,
+    OPC_DSRL_CP2 = (25 << 21) | (0x0F) | OPC_CP2,
+    OPC_SRA_CP2  = (26 << 21) | (0x0F) | OPC_CP2,
+    OPC_DSRA_CP2 = (27 << 21) | (0x0F) | OPC_CP2,
+    OPC_BIADD    = (28 << 21) | (0x0F) | OPC_CP2,
+    OPC_PMOVMSKB = (29 << 21) | (0x0F) | OPC_CP2,
+};
+
+
 #define MASK_CP3(op)       MASK_OP_MAJOR(op) | (op & 0x3F)
 
 enum {
@@ -478,32 +1012,52 @@ static TCGv cpu_HI[MIPS_DSP_ACC], cpu_LO[MIPS_DSP_ACC], cpu_ACX[MIPS_DSP_ACC];
 static TCGv cpu_dspctrl, btarget, bcond;
 static TCGv_i32 hflags;
 static TCGv_i32 fpu_fcr0, fpu_fcr31;
+static TCGv_i64 fpu_f64[32];
 
 static uint32_t gen_opc_hflags[OPC_BUF_SIZE];
+static target_ulong gen_opc_btarget[OPC_BUF_SIZE];
 
 #include "gen-icount.h"
 
-#define gen_helper_0i(name, arg) do {                             \
+#define gen_helper_0e0i(name, arg) do {                           \
     TCGv_i32 helper_tmp = tcg_const_i32(arg);                     \
-    gen_helper_##name(helper_tmp);                                \
+    gen_helper_##name(cpu_env, helper_tmp);                       \
     tcg_temp_free_i32(helper_tmp);                                \
     } while(0)
 
-#define gen_helper_1i(name, arg1, arg2) do {                      \
+#define gen_helper_0e1i(name, arg1, arg2) do {                    \
     TCGv_i32 helper_tmp = tcg_const_i32(arg2);                    \
-    gen_helper_##name(arg1, helper_tmp);                          \
+    gen_helper_##name(cpu_env, arg1, helper_tmp);                 \
+    tcg_temp_free_i32(helper_tmp);                                \
+    } while(0)
+
+#define gen_helper_1e0i(name, ret, arg1) do {                     \
+    TCGv_i32 helper_tmp = tcg_const_i32(arg1);                    \
+    gen_helper_##name(ret, cpu_env, helper_tmp);                  \
     tcg_temp_free_i32(helper_tmp);                                \
     } while(0)
 
-#define gen_helper_2i(name, arg1, arg2, arg3) do {                \
+#define gen_helper_1e1i(name, ret, arg1, arg2) do {               \
+    TCGv_i32 helper_tmp = tcg_const_i32(arg2);                    \
+    gen_helper_##name(ret, cpu_env, arg1, helper_tmp);            \
+    tcg_temp_free_i32(helper_tmp);                                \
+    } while(0)
+
+#define gen_helper_0e2i(name, arg1, arg2, arg3) do {              \
     TCGv_i32 helper_tmp = tcg_const_i32(arg3);                    \
-    gen_helper_##name(arg1, arg2, helper_tmp);                    \
+    gen_helper_##name(cpu_env, arg1, arg2, helper_tmp);           \
     tcg_temp_free_i32(helper_tmp);                                \
     } while(0)
 
-#define gen_helper_3i(name, arg1, arg2, arg3, arg4) do {          \
+#define gen_helper_1e2i(name, ret, arg1, arg2, arg3) do {         \
+    TCGv_i32 helper_tmp = tcg_const_i32(arg3);                    \
+    gen_helper_##name(ret, cpu_env, arg1, arg2, helper_tmp);      \
+    tcg_temp_free_i32(helper_tmp);                                \
+    } while(0)
+
+#define gen_helper_0e3i(name, arg1, arg2, arg3, arg4) do {        \
     TCGv_i32 helper_tmp = tcg_const_i32(arg4);                    \
-    gen_helper_##name(arg1, arg2, arg3, helper_tmp);              \
+    gen_helper_##name(cpu_env, arg1, arg2, arg3, helper_tmp);     \
     tcg_temp_free_i32(helper_tmp);                                \
     } while(0)
 
@@ -527,43 +1081,51 @@ enum {
     BS_EXCP     = 3, /* We reached an exception condition */
 };
 
-static const char *regnames[] =
-    { "r0", "at", "v0", "v1", "a0", "a1", "a2", "a3",
-      "t0", "t1", "t2", "t3", "t4", "t5", "t6", "t7",
-      "s0", "s1", "s2", "s3", "s4", "s5", "s6", "s7",
-      "t8", "t9", "k0", "k1", "gp", "sp", "s8", "ra", };
+static const char * const regnames[] = {
+    "r0", "at", "v0", "v1", "a0", "a1", "a2", "a3",
+    "t0", "t1", "t2", "t3", "t4", "t5", "t6", "t7",
+    "s0", "s1", "s2", "s3", "s4", "s5", "s6", "s7",
+    "t8", "t9", "k0", "k1", "gp", "sp", "s8", "ra",
+};
 
-static const char *regnames_HI[] =
-    { "HI0", "HI1", "HI2", "HI3", };
+static const char * const regnames_HI[] = {
+    "HI0", "HI1", "HI2", "HI3",
+};
 
-static const char *regnames_LO[] =
-    { "LO0", "LO1", "LO2", "LO3", };
+static const char * const regnames_LO[] = {
+    "LO0", "LO1", "LO2", "LO3",
+};
 
-static const char *regnames_ACX[] =
-    { "ACX0", "ACX1", "ACX2", "ACX3", };
+static const char * const regnames_ACX[] = {
+    "ACX0", "ACX1", "ACX2", "ACX3",
+};
 
-static const char *fregnames[] =
-    { "f0",  "f1",  "f2",  "f3",  "f4",  "f5",  "f6",  "f7",
-      "f8",  "f9",  "f10", "f11", "f12", "f13", "f14", "f15",
-      "f16", "f17", "f18", "f19", "f20", "f21", "f22", "f23",
-      "f24", "f25", "f26", "f27", "f28", "f29", "f30", "f31", };
+static const char * const fregnames[] = {
+    "f0",  "f1",  "f2",  "f3",  "f4",  "f5",  "f6",  "f7",
+    "f8",  "f9",  "f10", "f11", "f12", "f13", "f14", "f15",
+    "f16", "f17", "f18", "f19", "f20", "f21", "f22", "f23",
+    "f24", "f25", "f26", "f27", "f28", "f29", "f30", "f31",
+};
 
-#ifdef MIPS_DEBUG_DISAS
-#define MIPS_DEBUG(fmt, ...)                         \
-        qemu_log_mask(CPU_LOG_TB_IN_ASM,                \
-                       TARGET_FMT_lx ": %08x " fmt "\n", \
-                       ctx->pc, ctx->opcode , ## __VA_ARGS__)
-#define LOG_DISAS(...) qemu_log_mask(CPU_LOG_TB_IN_ASM, ## __VA_ARGS__)
-#else
-#define MIPS_DEBUG(fmt, ...) do { } while(0)
-#define LOG_DISAS(...) do { } while (0)
-#endif
+#define MIPS_DEBUG(fmt, ...)                                                  \
+    do {                                                                      \
+        if (MIPS_DEBUG_DISAS) {                                               \
+            qemu_log_mask(CPU_LOG_TB_IN_ASM,                                  \
+                          TARGET_FMT_lx ": %08x " fmt "\n",                   \
+                          ctx->pc, ctx->opcode , ## __VA_ARGS__);             \
+        }                                                                     \
+    } while (0)
+
+#define LOG_DISAS(...)                                                        \
+    do {                                                                      \
+        if (MIPS_DEBUG_DISAS) {                                               \
+            qemu_log_mask(CPU_LOG_TB_IN_ASM, ## __VA_ARGS__);                 \
+        }                                                                     \
+    } while (0)
 
 #define MIPS_INVAL(op)                                                        \
-do {                                                                          \
     MIPS_DEBUG("Invalid %s %03x %03x %03x", op, ctx->opcode >> 26,            \
-               ctx->opcode & 0x3F, ((ctx->opcode >> 16) & 0x1F));             \
-} while (0)
+               ctx->opcode & 0x3F, ((ctx->opcode >> 16) & 0x1F))
 
 /* General purpose registers moves. */
 static inline void gen_load_gpr (TCGv t, int reg)
@@ -640,54 +1202,54 @@ static inline void gen_store_srsgpr (int from, int to)
 }
 
 /* Floating point register moves. */
-static inline void gen_load_fpr32 (TCGv_i32 t, int reg)
+static void gen_load_fpr32(TCGv_i32 t, int reg)
 {
-    tcg_gen_ld_i32(t, cpu_env, offsetof(CPUMIPSState, active_fpu.fpr[reg].w[FP_ENDIAN_IDX]));
+    tcg_gen_trunc_i64_i32(t, fpu_f64[reg]);
 }
 
-static inline void gen_store_fpr32 (TCGv_i32 t, int reg)
+static void gen_store_fpr32(TCGv_i32 t, int reg)
 {
-    tcg_gen_st_i32(t, cpu_env, offsetof(CPUMIPSState, active_fpu.fpr[reg].w[FP_ENDIAN_IDX]));
+    TCGv_i64 t64 = tcg_temp_new_i64();
+    tcg_gen_extu_i32_i64(t64, t);
+    tcg_gen_deposit_i64(fpu_f64[reg], fpu_f64[reg], t64, 0, 32);
+    tcg_temp_free_i64(t64);
 }
 
-static inline void gen_load_fpr32h (TCGv_i32 t, int reg)
+static void gen_load_fpr32h(TCGv_i32 t, int reg)
 {
-    tcg_gen_ld_i32(t, cpu_env, offsetof(CPUMIPSState, active_fpu.fpr[reg].w[!FP_ENDIAN_IDX]));
+    TCGv_i64 t64 = tcg_temp_new_i64();
+    tcg_gen_shri_i64(t64, fpu_f64[reg], 32);
+    tcg_gen_trunc_i64_i32(t, t64);
+    tcg_temp_free_i64(t64);
 }
 
-static inline void gen_store_fpr32h (TCGv_i32 t, int reg)
+static void gen_store_fpr32h(TCGv_i32 t, int reg)
 {
-    tcg_gen_st_i32(t, cpu_env, offsetof(CPUMIPSState, active_fpu.fpr[reg].w[!FP_ENDIAN_IDX]));
+    TCGv_i64 t64 = tcg_temp_new_i64();
+    tcg_gen_extu_i32_i64(t64, t);
+    tcg_gen_deposit_i64(fpu_f64[reg], fpu_f64[reg], t64, 32, 32);
+    tcg_temp_free_i64(t64);
 }
 
-static inline void gen_load_fpr64 (DisasContext *ctx, TCGv_i64 t, int reg)
+static void gen_load_fpr64(DisasContext *ctx, TCGv_i64 t, int reg)
 {
     if (ctx->hflags & MIPS_HFLAG_F64) {
-        tcg_gen_ld_i64(t, cpu_env, offsetof(CPUMIPSState, active_fpu.fpr[reg].d));
+        tcg_gen_mov_i64(t, fpu_f64[reg]);
     } else {
-        TCGv_i32 t0 = tcg_temp_new_i32();
-        TCGv_i32 t1 = tcg_temp_new_i32();
-        gen_load_fpr32(t0, reg & ~1);
-        gen_load_fpr32(t1, reg | 1);
-        tcg_gen_concat_i32_i64(t, t0, t1);
-        tcg_temp_free_i32(t0);
-        tcg_temp_free_i32(t1);
+        tcg_gen_concat32_i64(t, fpu_f64[reg & ~1], fpu_f64[reg | 1]);
     }
 }
 
-static inline void gen_store_fpr64 (DisasContext *ctx, TCGv_i64 t, int reg)
+static void gen_store_fpr64(DisasContext *ctx, TCGv_i64 t, int reg)
 {
     if (ctx->hflags & MIPS_HFLAG_F64) {
-        tcg_gen_st_i64(t, cpu_env, offsetof(CPUMIPSState, active_fpu.fpr[reg].d));
+        tcg_gen_mov_i64(fpu_f64[reg], t);
     } else {
-        TCGv_i64 t0 = tcg_temp_new_i64();
-        TCGv_i32 t1 = tcg_temp_new_i32();
-        tcg_gen_trunc_i64_i32(t1, t);
-        gen_store_fpr32(t1, reg & ~1);
+        TCGv_i64 t0;
+        tcg_gen_deposit_i64(fpu_f64[reg & ~1], fpu_f64[reg & ~1], t, 0, 32);
+        t0 = tcg_temp_new_i64();
         tcg_gen_shri_i64(t0, t, 32);
-        tcg_gen_trunc_i64_i32(t1, t0);
-        gen_store_fpr32(t1, reg | 1);
-        tcg_temp_free_i32(t1);
+        tcg_gen_deposit_i64(fpu_f64[reg | 1], fpu_f64[reg | 1], t0, 0, 32);
         tcg_temp_free_i64(t0);
     }
 }
@@ -748,7 +1310,7 @@ generate_exception_err (DisasContext *ctx, int excp, int err)
     TCGv_i32 texcp = tcg_const_i32(excp);
     TCGv_i32 terr = tcg_const_i32(err);
     save_cpu_state(ctx, 1);
-    gen_helper_raise_exception_err(texcp, terr);
+    gen_helper_raise_exception_err(cpu_env, texcp, terr);
     tcg_temp_free_i32(terr);
     tcg_temp_free_i32(texcp);
 }
@@ -757,7 +1319,7 @@ static inline void
 generate_exception (DisasContext *ctx, int excp)
 {
     save_cpu_state(ctx, 1);
-    gen_helper_0i(raise_exception, excp);
+    gen_helper_0e0i(raise_exception, excp);
 }
 
 /* Addresses computation */
@@ -824,6 +1386,24 @@ static inline void check_cp1_registers(DisasContext *ctx, int regs)
         generate_exception(ctx, EXCP_RI);
 }
 
+/* Verify that the processor is running with DSP instructions enabled.
+   This is enabled by CP0 Status register MX(24) bit.
+ */
+
+static inline void check_dsp(DisasContext *ctx)
+{
+    if (unlikely(!(ctx->hflags & MIPS_HFLAG_DSP))) {
+        generate_exception(ctx, EXCP_DSPDIS);
+    }
+}
+
+static inline void check_dspr2(DisasContext *ctx)
+{
+    if (unlikely(!(ctx->hflags & MIPS_HFLAG_DSPR2))) {
+        generate_exception(ctx, EXCP_DSPDIS);
+    }
+}
+
 /* This code generates a "reserved instruction" exception if the
    CPU does not support the instruction set corresponding to flags. */
 static inline void check_insn(CPUMIPSState *env, DisasContext *ctx, int flags)
@@ -871,22 +1451,22 @@ static inline void gen_cmp ## type ## _ ## fmt(DisasContext *ctx, int n,      \
     gen_ldcmp_fpr##bits (ctx, fp0, fs);                                       \
     gen_ldcmp_fpr##bits (ctx, fp1, ft);                                       \
     switch (n) {                                                              \
-    case  0: gen_helper_2i(cmp ## type ## _ ## fmt ## _f, fp0, fp1, cc);    break;\
-    case  1: gen_helper_2i(cmp ## type ## _ ## fmt ## _un, fp0, fp1, cc);   break;\
-    case  2: gen_helper_2i(cmp ## type ## _ ## fmt ## _eq, fp0, fp1, cc);   break;\
-    case  3: gen_helper_2i(cmp ## type ## _ ## fmt ## _ueq, fp0, fp1, cc);  break;\
-    case  4: gen_helper_2i(cmp ## type ## _ ## fmt ## _olt, fp0, fp1, cc);  break;\
-    case  5: gen_helper_2i(cmp ## type ## _ ## fmt ## _ult, fp0, fp1, cc);  break;\
-    case  6: gen_helper_2i(cmp ## type ## _ ## fmt ## _ole, fp0, fp1, cc);  break;\
-    case  7: gen_helper_2i(cmp ## type ## _ ## fmt ## _ule, fp0, fp1, cc);  break;\
-    case  8: gen_helper_2i(cmp ## type ## _ ## fmt ## _sf, fp0, fp1, cc);   break;\
-    case  9: gen_helper_2i(cmp ## type ## _ ## fmt ## _ngle, fp0, fp1, cc); break;\
-    case 10: gen_helper_2i(cmp ## type ## _ ## fmt ## _seq, fp0, fp1, cc);  break;\
-    case 11: gen_helper_2i(cmp ## type ## _ ## fmt ## _ngl, fp0, fp1, cc);  break;\
-    case 12: gen_helper_2i(cmp ## type ## _ ## fmt ## _lt, fp0, fp1, cc);   break;\
-    case 13: gen_helper_2i(cmp ## type ## _ ## fmt ## _nge, fp0, fp1, cc);  break;\
-    case 14: gen_helper_2i(cmp ## type ## _ ## fmt ## _le, fp0, fp1, cc);   break;\
-    case 15: gen_helper_2i(cmp ## type ## _ ## fmt ## _ngt, fp0, fp1, cc);  break;\
+    case  0: gen_helper_0e2i(cmp ## type ## _ ## fmt ## _f, fp0, fp1, cc);    break;\
+    case  1: gen_helper_0e2i(cmp ## type ## _ ## fmt ## _un, fp0, fp1, cc);   break;\
+    case  2: gen_helper_0e2i(cmp ## type ## _ ## fmt ## _eq, fp0, fp1, cc);   break;\
+    case  3: gen_helper_0e2i(cmp ## type ## _ ## fmt ## _ueq, fp0, fp1, cc);  break;\
+    case  4: gen_helper_0e2i(cmp ## type ## _ ## fmt ## _olt, fp0, fp1, cc);  break;\
+    case  5: gen_helper_0e2i(cmp ## type ## _ ## fmt ## _ult, fp0, fp1, cc);  break;\
+    case  6: gen_helper_0e2i(cmp ## type ## _ ## fmt ## _ole, fp0, fp1, cc);  break;\
+    case  7: gen_helper_0e2i(cmp ## type ## _ ## fmt ## _ule, fp0, fp1, cc);  break;\
+    case  8: gen_helper_0e2i(cmp ## type ## _ ## fmt ## _sf, fp0, fp1, cc);   break;\
+    case  9: gen_helper_0e2i(cmp ## type ## _ ## fmt ## _ngle, fp0, fp1, cc); break;\
+    case 10: gen_helper_0e2i(cmp ## type ## _ ## fmt ## _seq, fp0, fp1, cc);  break;\
+    case 11: gen_helper_0e2i(cmp ## type ## _ ## fmt ## _ngl, fp0, fp1, cc);  break;\
+    case 12: gen_helper_0e2i(cmp ## type ## _ ## fmt ## _lt, fp0, fp1, cc);   break;\
+    case 13: gen_helper_0e2i(cmp ## type ## _ ## fmt ## _nge, fp0, fp1, cc);  break;\
+    case 14: gen_helper_0e2i(cmp ## type ## _ ## fmt ## _le, fp0, fp1, cc);   break;\
+    case 15: gen_helper_0e2i(cmp ## type ## _ ## fmt ## _ngt, fp0, fp1, cc);  break;\
     default: abort();                                                         \
     }                                                                         \
     tcg_temp_free_i##bits (fp0);                                              \
@@ -904,35 +1484,6 @@ FOP_CONDS(abs, 1, ps, FMT_PS, 64)
 #undef gen_ldcmp_fpr64
 
 /* load/store instructions. */
-#define OP_LD(insn,fname)                                                 \
-static inline void op_ld_##insn(TCGv ret, TCGv arg1, DisasContext *ctx)   \
-{                                                                         \
-    tcg_gen_qemu_##fname(ret, arg1, ctx->mem_idx);                        \
-}
-OP_LD(lb,ld8s);
-OP_LD(lbu,ld8u);
-OP_LD(lh,ld16s);
-OP_LD(lhu,ld16u);
-OP_LD(lw,ld32s);
-#if defined(TARGET_MIPS64)
-OP_LD(lwu,ld32u);
-OP_LD(ld,ld64);
-#endif
-#undef OP_LD
-
-#define OP_ST(insn,fname)                                                  \
-static inline void op_st_##insn(TCGv arg1, TCGv arg2, DisasContext *ctx)   \
-{                                                                          \
-    tcg_gen_qemu_##fname(arg1, arg2, ctx->mem_idx);                        \
-}
-OP_ST(sb,st8);
-OP_ST(sh,st16);
-OP_ST(sw,st32);
-#if defined(TARGET_MIPS64)
-OP_ST(sd,st64);
-#endif
-#undef OP_ST
-
 #ifdef CONFIG_USER_ONLY
 #define OP_LD_ATOMIC(insn,fname)                                           \
 static inline void op_ld_##insn(TCGv ret, TCGv arg1, DisasContext *ctx)    \
@@ -948,7 +1499,7 @@ static inline void op_ld_##insn(TCGv ret, TCGv arg1, DisasContext *ctx)    \
 #define OP_LD_ATOMIC(insn,fname)                                           \
 static inline void op_ld_##insn(TCGv ret, TCGv arg1, DisasContext *ctx)    \
 {                                                                          \
-    gen_helper_2i(insn, ret, arg1, ctx->mem_idx);                          \
+    gen_helper_1e1i(insn, ret, arg1, ctx->mem_idx);                        \
 }
 #endif
 OP_LD_ATOMIC(ll,ld32s);
@@ -975,7 +1526,7 @@ static inline void op_st_##insn(TCGv arg1, TCGv arg2, int rt, DisasContext *ctx)
     tcg_gen_movi_tl(t0, rt | ((almask << 3) & 0x20));                        \
     tcg_gen_st_tl(t0, cpu_env, offsetof(CPUMIPSState, llreg));                   \
     tcg_gen_st_tl(arg1, cpu_env, offsetof(CPUMIPSState, llnewval));              \
-    gen_helper_0i(raise_exception, EXCP_SC);                                 \
+    gen_helper_0e0i(raise_exception, EXCP_SC);                               \
     gen_set_label(l2);                                                       \
     tcg_gen_movi_tl(t0, 0);                                                  \
     gen_store_gpr(t0, rt);                                                   \
@@ -986,7 +1537,7 @@ static inline void op_st_##insn(TCGv arg1, TCGv arg2, int rt, DisasContext *ctx)
 static inline void op_st_##insn(TCGv arg1, TCGv arg2, int rt, DisasContext *ctx) \
 {                                                                            \
     TCGv t0 = tcg_temp_new();                                                \
-    gen_helper_3i(insn, t0, arg1, arg2, ctx->mem_idx);                       \
+    gen_helper_1e2i(insn, t0, arg1, arg2, ctx->mem_idx);                     \
     gen_store_gpr(t0, rt);                                                   \
     tcg_temp_free(t0);                                                       \
 }
@@ -1029,7 +1580,7 @@ static void gen_ld (CPUMIPSState *env, DisasContext *ctx, uint32_t opc,
                     int rt, int base, int16_t offset)
 {
     const char *opn = "ld";
-    TCGv t0, t1;
+    TCGv t0, t1, t2;
 
     if (rt == 0 && env->insn_flags & (INSN_LOONGSON2E | INSN_LOONGSON2F)) {
         /* Loongson CPU uses a load to zero register for prefetch.
@@ -1040,20 +1591,17 @@ static void gen_ld (CPUMIPSState *env, DisasContext *ctx, uint32_t opc,
     }
 
     t0 = tcg_temp_new();
-    t1 = tcg_temp_new();
     gen_base_offset_addr(ctx, t0, base, offset);
 
     switch (opc) {
 #if defined(TARGET_MIPS64)
     case OPC_LWU:
-        save_cpu_state(ctx, 0);
-        op_ld_lwu(t0, t0, ctx);
+        tcg_gen_qemu_ld32u(t0, t0, ctx->mem_idx);
         gen_store_gpr(t0, rt);
         opn = "lwu";
         break;
     case OPC_LD:
-        save_cpu_state(ctx, 0);
-        op_ld_ld(t0, t0, ctx);
+        tcg_gen_qemu_ld64(t0, t0, ctx->mem_idx);
         gen_store_gpr(t0, rt);
         opn = "ld";
         break;
@@ -1064,78 +1612,130 @@ static void gen_ld (CPUMIPSState *env, DisasContext *ctx, uint32_t opc,
         opn = "lld";
         break;
     case OPC_LDL:
-        save_cpu_state(ctx, 1);
+        t1 = tcg_temp_new();
+        tcg_gen_andi_tl(t1, t0, 7);
+#ifndef TARGET_WORDS_BIGENDIAN
+        tcg_gen_xori_tl(t1, t1, 7);
+#endif
+        tcg_gen_shli_tl(t1, t1, 3);
+        tcg_gen_andi_tl(t0, t0, ~7);
+        tcg_gen_qemu_ld64(t0, t0, ctx->mem_idx);
+        tcg_gen_shl_tl(t0, t0, t1);
+        tcg_gen_xori_tl(t1, t1, 63);
+        t2 = tcg_const_tl(0x7fffffffffffffffull);
+        tcg_gen_shr_tl(t2, t2, t1);
         gen_load_gpr(t1, rt);
-        gen_helper_3i(ldl, t1, t1, t0, ctx->mem_idx);
-        gen_store_gpr(t1, rt);
+        tcg_gen_and_tl(t1, t1, t2);
+        tcg_temp_free(t2);
+        tcg_gen_or_tl(t0, t0, t1);
+        tcg_temp_free(t1);
+        gen_store_gpr(t0, rt);
         opn = "ldl";
         break;
     case OPC_LDR:
-        save_cpu_state(ctx, 1);
+        t1 = tcg_temp_new();
+        tcg_gen_andi_tl(t1, t0, 7);
+#ifdef TARGET_WORDS_BIGENDIAN
+        tcg_gen_xori_tl(t1, t1, 7);
+#endif
+        tcg_gen_shli_tl(t1, t1, 3);
+        tcg_gen_andi_tl(t0, t0, ~7);
+        tcg_gen_qemu_ld64(t0, t0, ctx->mem_idx);
+        tcg_gen_shr_tl(t0, t0, t1);
+        tcg_gen_xori_tl(t1, t1, 63);
+        t2 = tcg_const_tl(0xfffffffffffffffeull);
+        tcg_gen_shl_tl(t2, t2, t1);
         gen_load_gpr(t1, rt);
-        gen_helper_3i(ldr, t1, t1, t0, ctx->mem_idx);
-        gen_store_gpr(t1, rt);
+        tcg_gen_and_tl(t1, t1, t2);
+        tcg_temp_free(t2);
+        tcg_gen_or_tl(t0, t0, t1);
+        tcg_temp_free(t1);
+        gen_store_gpr(t0, rt);
         opn = "ldr";
         break;
     case OPC_LDPC:
-        save_cpu_state(ctx, 0);
-        tcg_gen_movi_tl(t1, pc_relative_pc(ctx));
+        t1 = tcg_const_tl(pc_relative_pc(ctx));
         gen_op_addr_add(ctx, t0, t0, t1);
-        op_ld_ld(t0, t0, ctx);
+        tcg_temp_free(t1);
+        tcg_gen_qemu_ld64(t0, t0, ctx->mem_idx);
         gen_store_gpr(t0, rt);
         opn = "ldpc";
         break;
 #endif
     case OPC_LWPC:
-        save_cpu_state(ctx, 0);
-        tcg_gen_movi_tl(t1, pc_relative_pc(ctx));
+        t1 = tcg_const_tl(pc_relative_pc(ctx));
         gen_op_addr_add(ctx, t0, t0, t1);
-        op_ld_lw(t0, t0, ctx);
+        tcg_temp_free(t1);
+        tcg_gen_qemu_ld32s(t0, t0, ctx->mem_idx);
         gen_store_gpr(t0, rt);
         opn = "lwpc";
         break;
     case OPC_LW:
-        save_cpu_state(ctx, 0);
-        op_ld_lw(t0, t0, ctx);
+        tcg_gen_qemu_ld32s(t0, t0, ctx->mem_idx);
         gen_store_gpr(t0, rt);
         opn = "lw";
         break;
     case OPC_LH:
-        save_cpu_state(ctx, 0);
-        op_ld_lh(t0, t0, ctx);
+        tcg_gen_qemu_ld16s(t0, t0, ctx->mem_idx);
         gen_store_gpr(t0, rt);
         opn = "lh";
         break;
     case OPC_LHU:
-        save_cpu_state(ctx, 0);
-        op_ld_lhu(t0, t0, ctx);
+        tcg_gen_qemu_ld16u(t0, t0, ctx->mem_idx);
         gen_store_gpr(t0, rt);
         opn = "lhu";
         break;
     case OPC_LB:
-        save_cpu_state(ctx, 0);
-        op_ld_lb(t0, t0, ctx);
+        tcg_gen_qemu_ld8s(t0, t0, ctx->mem_idx);
         gen_store_gpr(t0, rt);
         opn = "lb";
         break;
     case OPC_LBU:
-        save_cpu_state(ctx, 0);
-        op_ld_lbu(t0, t0, ctx);
+        tcg_gen_qemu_ld8u(t0, t0, ctx->mem_idx);
         gen_store_gpr(t0, rt);
         opn = "lbu";
         break;
     case OPC_LWL:
-        save_cpu_state(ctx, 1);
+        t1 = tcg_temp_new();
+        tcg_gen_andi_tl(t1, t0, 3);
+#ifndef TARGET_WORDS_BIGENDIAN
+        tcg_gen_xori_tl(t1, t1, 3);
+#endif
+        tcg_gen_shli_tl(t1, t1, 3);
+        tcg_gen_andi_tl(t0, t0, ~3);
+        tcg_gen_qemu_ld32u(t0, t0, ctx->mem_idx);
+        tcg_gen_shl_tl(t0, t0, t1);
+        tcg_gen_xori_tl(t1, t1, 31);
+        t2 = tcg_const_tl(0x7fffffffull);
+        tcg_gen_shr_tl(t2, t2, t1);
         gen_load_gpr(t1, rt);
-        gen_helper_3i(lwl, t1, t1, t0, ctx->mem_idx);
-        gen_store_gpr(t1, rt);
+        tcg_gen_and_tl(t1, t1, t2);
+        tcg_temp_free(t2);
+        tcg_gen_or_tl(t0, t0, t1);
+        tcg_temp_free(t1);
+        tcg_gen_ext32s_tl(t0, t0);
+        gen_store_gpr(t0, rt);
         opn = "lwl";
         break;
     case OPC_LWR:
-        save_cpu_state(ctx, 1);
+        t1 = tcg_temp_new();
+        tcg_gen_andi_tl(t1, t0, 3);
+#ifdef TARGET_WORDS_BIGENDIAN
+        tcg_gen_xori_tl(t1, t1, 3);
+#endif
+        tcg_gen_shli_tl(t1, t1, 3);
+        tcg_gen_andi_tl(t0, t0, ~3);
+        tcg_gen_qemu_ld32u(t0, t0, ctx->mem_idx);
+        tcg_gen_shr_tl(t0, t0, t1);
+        tcg_gen_xori_tl(t1, t1, 31);
+        t2 = tcg_const_tl(0xfffffffeull);
+        tcg_gen_shl_tl(t2, t2, t1);
         gen_load_gpr(t1, rt);
-        gen_helper_3i(lwr, t1, t1, t0, ctx->mem_idx);
-        gen_store_gpr(t1, rt);
+        tcg_gen_and_tl(t1, t1, t2);
+        tcg_temp_free(t2);
+        tcg_gen_or_tl(t0, t0, t1);
+        tcg_temp_free(t1);
+        gen_store_gpr(t0, rt);
         opn = "lwr";
         break;
     case OPC_LL:
@@ -1148,7 +1748,6 @@ static void gen_ld (CPUMIPSState *env, DisasContext *ctx, uint32_t opc,
     (void)opn; /* avoid a compiler warning */
     MIPS_DEBUG("%s %s, %d(%s)", opn, regnames[rt], offset, regnames[base]);
     tcg_temp_free(t0);
-    tcg_temp_free(t1);
 }
 
 /* Store */
@@ -1164,44 +1763,40 @@ static void gen_st (DisasContext *ctx, uint32_t opc, int rt,
     switch (opc) {
 #if defined(TARGET_MIPS64)
     case OPC_SD:
-        save_cpu_state(ctx, 0);
-        op_st_sd(t1, t0, ctx);
+        tcg_gen_qemu_st64(t1, t0, ctx->mem_idx);
         opn = "sd";
         break;
     case OPC_SDL:
         save_cpu_state(ctx, 1);
-        gen_helper_2i(sdl, t1, t0, ctx->mem_idx);
+        gen_helper_0e2i(sdl, t1, t0, ctx->mem_idx);
         opn = "sdl";
         break;
     case OPC_SDR:
         save_cpu_state(ctx, 1);
-        gen_helper_2i(sdr, t1, t0, ctx->mem_idx);
+        gen_helper_0e2i(sdr, t1, t0, ctx->mem_idx);
         opn = "sdr";
         break;
 #endif
     case OPC_SW:
-        save_cpu_state(ctx, 0);
-        op_st_sw(t1, t0, ctx);
+        tcg_gen_qemu_st32(t1, t0, ctx->mem_idx);
         opn = "sw";
         break;
     case OPC_SH:
-        save_cpu_state(ctx, 0);
-        op_st_sh(t1, t0, ctx);
+        tcg_gen_qemu_st16(t1, t0, ctx->mem_idx);
         opn = "sh";
         break;
     case OPC_SB:
-        save_cpu_state(ctx, 0);
-        op_st_sb(t1, t0, ctx);
+        tcg_gen_qemu_st8(t1, t0, ctx->mem_idx);
         opn = "sb";
         break;
     case OPC_SWL:
         save_cpu_state(ctx, 1);
-        gen_helper_2i(swl, t1, t0, ctx->mem_idx);
+        gen_helper_0e2i(swl, t1, t0, ctx->mem_idx);
         opn = "swl";
         break;
     case OPC_SWR:
         save_cpu_state(ctx, 1);
-        gen_helper_2i(swr, t1, t0, ctx->mem_idx);
+        gen_helper_0e2i(swr, t1, t0, ctx->mem_idx);
         opn = "swr";
         break;
     }
@@ -1219,13 +1814,14 @@ static void gen_st_cond (DisasContext *ctx, uint32_t opc, int rt,
     const char *opn = "st_cond";
     TCGv t0, t1;
 
+#ifdef CONFIG_USER_ONLY
     t0 = tcg_temp_local_new();
-
-    gen_base_offset_addr(ctx, t0, base, offset);
-    /* Don't do NOP if destination is zero: we must perform the actual
-       memory access. */
-
     t1 = tcg_temp_local_new();
+#else
+    t0 = tcg_temp_new();
+    t1 = tcg_temp_new();
+#endif
+    gen_base_offset_addr(ctx, t0, base, offset);
     gen_load_gpr(t1, rt);
     switch (opc) {
 #if defined(TARGET_MIPS64)
@@ -1413,10 +2009,10 @@ static void gen_arith_imm (CPUMIPSState *env, DisasContext *ctx, uint32_t opc,
 }
 
 /* Logic with immediate operand */
-static void gen_logic_imm (CPUMIPSState *env, uint32_t opc, int rt, int rs, int16_t imm)
+static void gen_logic_imm(CPUMIPSState *env, DisasContext *ctx, uint32_t opc,
+                          int rt, int rs, int16_t imm)
 {
     target_ulong uimm;
-    const char *opn = "imm logic";
 
     if (rt == 0) {
         /* If no destination, treat it as a NOP. */
@@ -1430,33 +2026,39 @@ static void gen_logic_imm (CPUMIPSState *env, uint32_t opc, int rt, int rs, int1
             tcg_gen_andi_tl(cpu_gpr[rt], cpu_gpr[rs], uimm);
         else
             tcg_gen_movi_tl(cpu_gpr[rt], 0);
-        opn = "andi";
+        MIPS_DEBUG("andi %s, %s, " TARGET_FMT_lx, regnames[rt],
+                   regnames[rs], uimm);
         break;
     case OPC_ORI:
         if (rs != 0)
             tcg_gen_ori_tl(cpu_gpr[rt], cpu_gpr[rs], uimm);
         else
             tcg_gen_movi_tl(cpu_gpr[rt], uimm);
-        opn = "ori";
+        MIPS_DEBUG("ori %s, %s, " TARGET_FMT_lx, regnames[rt],
+                   regnames[rs], uimm);
         break;
     case OPC_XORI:
         if (likely(rs != 0))
             tcg_gen_xori_tl(cpu_gpr[rt], cpu_gpr[rs], uimm);
         else
             tcg_gen_movi_tl(cpu_gpr[rt], uimm);
-        opn = "xori";
+        MIPS_DEBUG("xori %s, %s, " TARGET_FMT_lx, regnames[rt],
+                   regnames[rs], uimm);
         break;
     case OPC_LUI:
         tcg_gen_movi_tl(cpu_gpr[rt], imm << 16);
-        opn = "lui";
+        MIPS_DEBUG("lui %s, " TARGET_FMT_lx, regnames[rt], uimm);
+        break;
+
+    default:
+        MIPS_DEBUG("Unknown logical immediate opcode %08x", opc);
         break;
     }
-    (void)opn; /* avoid a compiler warning */
-    MIPS_DEBUG("%s %s, %s, " TARGET_FMT_lx, opn, regnames[rt], regnames[rs], uimm);
 }
 
 /* Set on less than with immediate operand */
-static void gen_slt_imm (CPUMIPSState *env, uint32_t opc, int rt, int rs, int16_t imm)
+static void gen_slt_imm(CPUMIPSState *env, DisasContext *ctx, uint32_t opc,
+                        int rt, int rs, int16_t imm)
 {
     target_ulong uimm = (target_long)imm; /* Sign extend to 32/64 bits */
     const char *opn = "imm arith";
@@ -1757,45 +2359,44 @@ static void gen_arith (CPUMIPSState *env, DisasContext *ctx, uint32_t opc,
 }
 
 /* Conditional move */
-static void gen_cond_move (CPUMIPSState *env, uint32_t opc, int rd, int rs, int rt)
+static void gen_cond_move(CPUMIPSState *env, DisasContext *ctx, uint32_t opc,
+                          int rd, int rs, int rt)
 {
     const char *opn = "cond move";
-    int l1;
+    TCGv t0, t1, t2;
 
     if (rd == 0) {
-        /* If no destination, treat it as a NOP.
-           For add & sub, we must generate the overflow exception when needed. */
+        /* If no destination, treat it as a NOP. */
         MIPS_DEBUG("NOP");
         return;
     }
 
-    l1 = gen_new_label();
+    t0 = tcg_temp_new();
+    gen_load_gpr(t0, rt);
+    t1 = tcg_const_tl(0);
+    t2 = tcg_temp_new();
+    gen_load_gpr(t2, rs);
     switch (opc) {
     case OPC_MOVN:
-        if (likely(rt != 0))
-            tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_gpr[rt], 0, l1);
-        else
-            tcg_gen_br(l1);
+        tcg_gen_movcond_tl(TCG_COND_NE, cpu_gpr[rd], t0, t1, t2, cpu_gpr[rd]);
         opn = "movn";
         break;
     case OPC_MOVZ:
-        if (likely(rt != 0))
-            tcg_gen_brcondi_tl(TCG_COND_NE, cpu_gpr[rt], 0, l1);
+        tcg_gen_movcond_tl(TCG_COND_EQ, cpu_gpr[rd], t0, t1, t2, cpu_gpr[rd]);
         opn = "movz";
         break;
     }
-    if (rs != 0)
-        tcg_gen_mov_tl(cpu_gpr[rd], cpu_gpr[rs]);
-    else
-        tcg_gen_movi_tl(cpu_gpr[rd], 0);
-    gen_set_label(l1);
+    tcg_temp_free(t2);
+    tcg_temp_free(t1);
+    tcg_temp_free(t0);
 
     (void)opn; /* avoid a compiler warning */
     MIPS_DEBUG("%s %s, %s, %s", opn, regnames[rd], regnames[rs], regnames[rt]);
 }
 
 /* Logic */
-static void gen_logic (CPUMIPSState *env, uint32_t opc, int rd, int rs, int rt)
+static void gen_logic(CPUMIPSState *env, DisasContext *ctx, uint32_t opc,
+                      int rd, int rs, int rt)
 {
     const char *opn = "logic";
 
@@ -1856,7 +2457,8 @@ static void gen_logic (CPUMIPSState *env, uint32_t opc, int rd, int rs, int rt)
 }
 
 /* Set on lower than */
-static void gen_slt (CPUMIPSState *env, uint32_t opc, int rd, int rs, int rt)
+static void gen_slt(CPUMIPSState *env, DisasContext *ctx, uint32_t opc,
+                    int rd, int rs, int rt)
 {
     const char *opn = "slt";
     TCGv t0, t1;
@@ -1972,33 +2574,75 @@ static void gen_shift (CPUMIPSState *env, DisasContext *ctx, uint32_t opc,
 static void gen_HILO (DisasContext *ctx, uint32_t opc, int reg)
 {
     const char *opn = "hilo";
+    unsigned int acc;
 
     if (reg == 0 && (opc == OPC_MFHI || opc == OPC_MFLO)) {
         /* Treat as NOP. */
         MIPS_DEBUG("NOP");
         return;
     }
+
+    if (opc == OPC_MFHI || opc == OPC_MFLO) {
+        acc = ((ctx->opcode) >> 21) & 0x03;
+    } else {
+        acc = ((ctx->opcode) >> 11) & 0x03;
+    }
+
+    if (acc != 0) {
+        check_dsp(ctx);
+    }
+
     switch (opc) {
     case OPC_MFHI:
-        tcg_gen_mov_tl(cpu_gpr[reg], cpu_HI[0]);
+#if defined(TARGET_MIPS64)
+        if (acc != 0) {
+            tcg_gen_ext32s_tl(cpu_gpr[reg], cpu_HI[acc]);
+        } else
+#endif
+        {
+            tcg_gen_mov_tl(cpu_gpr[reg], cpu_HI[acc]);
+        }
         opn = "mfhi";
         break;
     case OPC_MFLO:
-        tcg_gen_mov_tl(cpu_gpr[reg], cpu_LO[0]);
+#if defined(TARGET_MIPS64)
+        if (acc != 0) {
+            tcg_gen_ext32s_tl(cpu_gpr[reg], cpu_LO[acc]);
+        } else
+#endif
+        {
+            tcg_gen_mov_tl(cpu_gpr[reg], cpu_LO[acc]);
+        }
         opn = "mflo";
         break;
     case OPC_MTHI:
-        if (reg != 0)
-            tcg_gen_mov_tl(cpu_HI[0], cpu_gpr[reg]);
-        else
-            tcg_gen_movi_tl(cpu_HI[0], 0);
+        if (reg != 0) {
+#if defined(TARGET_MIPS64)
+            if (acc != 0) {
+                tcg_gen_ext32s_tl(cpu_HI[acc], cpu_gpr[reg]);
+            } else
+#endif
+            {
+                tcg_gen_mov_tl(cpu_HI[acc], cpu_gpr[reg]);
+            }
+        } else {
+            tcg_gen_movi_tl(cpu_HI[acc], 0);
+        }
         opn = "mthi";
         break;
     case OPC_MTLO:
-        if (reg != 0)
-            tcg_gen_mov_tl(cpu_LO[0], cpu_gpr[reg]);
-        else
-            tcg_gen_movi_tl(cpu_LO[0], 0);
+        if (reg != 0) {
+#if defined(TARGET_MIPS64)
+            if (acc != 0) {
+                tcg_gen_ext32s_tl(cpu_LO[acc], cpu_gpr[reg]);
+            } else
+#endif
+            {
+                tcg_gen_mov_tl(cpu_LO[acc], cpu_gpr[reg]);
+            }
+        } else {
+            tcg_gen_movi_tl(cpu_LO[acc], 0);
+        }
         opn = "mtlo";
         break;
     }
@@ -2011,61 +2655,50 @@ static void gen_muldiv (DisasContext *ctx, uint32_t opc,
 {
     const char *opn = "mul/div";
     TCGv t0, t1;
+    unsigned int acc;
 
-    switch (opc) {
-    case OPC_DIV:
-    case OPC_DIVU:
-#if defined(TARGET_MIPS64)
-    case OPC_DDIV:
-    case OPC_DDIVU:
-#endif
-        t0 = tcg_temp_local_new();
-        t1 = tcg_temp_local_new();
-        break;
-    default:
-        t0 = tcg_temp_new();
-        t1 = tcg_temp_new();
-        break;
-    }
+    t0 = tcg_temp_new();
+    t1 = tcg_temp_new();
 
     gen_load_gpr(t0, rs);
     gen_load_gpr(t1, rt);
+
     switch (opc) {
     case OPC_DIV:
         {
-            int l1 = gen_new_label();
-            int l2 = gen_new_label();
-
+            TCGv t2 = tcg_temp_new();
+            TCGv t3 = tcg_temp_new();
             tcg_gen_ext32s_tl(t0, t0);
             tcg_gen_ext32s_tl(t1, t1);
-            tcg_gen_brcondi_tl(TCG_COND_EQ, t1, 0, l1);
-            tcg_gen_brcondi_tl(TCG_COND_NE, t0, INT_MIN, l2);
-            tcg_gen_brcondi_tl(TCG_COND_NE, t1, -1, l2);
-
-            tcg_gen_mov_tl(cpu_LO[0], t0);
-            tcg_gen_movi_tl(cpu_HI[0], 0);
-            tcg_gen_br(l1);
-            gen_set_label(l2);
+            tcg_gen_setcondi_tl(TCG_COND_EQ, t2, t0, INT_MIN);
+            tcg_gen_setcondi_tl(TCG_COND_EQ, t3, t1, -1);
+            tcg_gen_and_tl(t2, t2, t3);
+            tcg_gen_setcondi_tl(TCG_COND_EQ, t3, t1, 0);
+            tcg_gen_or_tl(t2, t2, t3);
+            tcg_gen_movi_tl(t3, 0);
+            tcg_gen_movcond_tl(TCG_COND_NE, t1, t2, t3, t2, t1);
             tcg_gen_div_tl(cpu_LO[0], t0, t1);
             tcg_gen_rem_tl(cpu_HI[0], t0, t1);
             tcg_gen_ext32s_tl(cpu_LO[0], cpu_LO[0]);
             tcg_gen_ext32s_tl(cpu_HI[0], cpu_HI[0]);
-            gen_set_label(l1);
+            tcg_temp_free(t3);
+            tcg_temp_free(t2);
         }
         opn = "div";
         break;
     case OPC_DIVU:
         {
-            int l1 = gen_new_label();
-
+            TCGv t2 = tcg_const_tl(0);
+            TCGv t3 = tcg_const_tl(1);
             tcg_gen_ext32u_tl(t0, t0);
             tcg_gen_ext32u_tl(t1, t1);
-            tcg_gen_brcondi_tl(TCG_COND_EQ, t1, 0, l1);
+            tcg_gen_movcond_tl(TCG_COND_EQ, t1, t1, t2, t3, t1);
             tcg_gen_divu_tl(cpu_LO[0], t0, t1);
             tcg_gen_remu_tl(cpu_HI[0], t0, t1);
             tcg_gen_ext32s_tl(cpu_LO[0], cpu_LO[0]);
             tcg_gen_ext32s_tl(cpu_HI[0], cpu_HI[0]);
-            gen_set_label(l1);
+            tcg_temp_free(t3);
+            tcg_temp_free(t2);
         }
         opn = "divu";
         break;
@@ -2073,6 +2706,10 @@ static void gen_muldiv (DisasContext *ctx, uint32_t opc,
         {
             TCGv_i64 t2 = tcg_temp_new_i64();
             TCGv_i64 t3 = tcg_temp_new_i64();
+            acc = ((ctx->opcode) >> 11) & 0x03;
+            if (acc != 0) {
+                check_dsp(ctx);
+            }
 
             tcg_gen_ext_tl_i64(t2, t0);
             tcg_gen_ext_tl_i64(t3, t1);
@@ -2082,8 +2719,8 @@ static void gen_muldiv (DisasContext *ctx, uint32_t opc,
             tcg_gen_shri_i64(t2, t2, 32);
             tcg_gen_trunc_i64_tl(t1, t2);
             tcg_temp_free_i64(t2);
-            tcg_gen_ext32s_tl(cpu_LO[0], t0);
-            tcg_gen_ext32s_tl(cpu_HI[0], t1);
+            tcg_gen_ext32s_tl(cpu_LO[acc], t0);
+            tcg_gen_ext32s_tl(cpu_HI[acc], t1);
         }
         opn = "mult";
         break;
@@ -2091,6 +2728,10 @@ static void gen_muldiv (DisasContext *ctx, uint32_t opc,
         {
             TCGv_i64 t2 = tcg_temp_new_i64();
             TCGv_i64 t3 = tcg_temp_new_i64();
+            acc = ((ctx->opcode) >> 11) & 0x03;
+            if (acc != 0) {
+                check_dsp(ctx);
+            }
 
             tcg_gen_ext32u_tl(t0, t0);
             tcg_gen_ext32u_tl(t1, t1);
@@ -2102,47 +2743,48 @@ static void gen_muldiv (DisasContext *ctx, uint32_t opc,
             tcg_gen_shri_i64(t2, t2, 32);
             tcg_gen_trunc_i64_tl(t1, t2);
             tcg_temp_free_i64(t2);
-            tcg_gen_ext32s_tl(cpu_LO[0], t0);
-            tcg_gen_ext32s_tl(cpu_HI[0], t1);
+            tcg_gen_ext32s_tl(cpu_LO[acc], t0);
+            tcg_gen_ext32s_tl(cpu_HI[acc], t1);
         }
         opn = "multu";
         break;
 #if defined(TARGET_MIPS64)
     case OPC_DDIV:
         {
-            int l1 = gen_new_label();
-            int l2 = gen_new_label();
-
-            tcg_gen_brcondi_tl(TCG_COND_EQ, t1, 0, l1);
-            tcg_gen_brcondi_tl(TCG_COND_NE, t0, -1LL << 63, l2);
-            tcg_gen_brcondi_tl(TCG_COND_NE, t1, -1LL, l2);
-            tcg_gen_mov_tl(cpu_LO[0], t0);
-            tcg_gen_movi_tl(cpu_HI[0], 0);
-            tcg_gen_br(l1);
-            gen_set_label(l2);
-            tcg_gen_div_i64(cpu_LO[0], t0, t1);
-            tcg_gen_rem_i64(cpu_HI[0], t0, t1);
-            gen_set_label(l1);
+            TCGv t2 = tcg_temp_new();
+            TCGv t3 = tcg_temp_new();
+            tcg_gen_setcondi_tl(TCG_COND_EQ, t2, t0, -1LL << 63);
+            tcg_gen_setcondi_tl(TCG_COND_EQ, t3, t1, -1LL);
+            tcg_gen_and_tl(t2, t2, t3);
+            tcg_gen_setcondi_tl(TCG_COND_EQ, t3, t1, 0);
+            tcg_gen_or_tl(t2, t2, t3);
+            tcg_gen_movi_tl(t3, 0);
+            tcg_gen_movcond_tl(TCG_COND_NE, t1, t2, t3, t2, t1);
+            tcg_gen_div_tl(cpu_LO[0], t0, t1);
+            tcg_gen_rem_tl(cpu_HI[0], t0, t1);
+            tcg_temp_free(t3);
+            tcg_temp_free(t2);
         }
         opn = "ddiv";
         break;
     case OPC_DDIVU:
         {
-            int l1 = gen_new_label();
-
-            tcg_gen_brcondi_tl(TCG_COND_EQ, t1, 0, l1);
+            TCGv t2 = tcg_const_tl(0);
+            TCGv t3 = tcg_const_tl(1);
+            tcg_gen_movcond_tl(TCG_COND_EQ, t1, t1, t2, t3, t1);
             tcg_gen_divu_i64(cpu_LO[0], t0, t1);
             tcg_gen_remu_i64(cpu_HI[0], t0, t1);
-            gen_set_label(l1);
+            tcg_temp_free(t3);
+            tcg_temp_free(t2);
         }
         opn = "ddivu";
         break;
     case OPC_DMULT:
-        gen_helper_dmult(t0, t1);
+        gen_helper_dmult(cpu_env, t0, t1);
         opn = "dmult";
         break;
     case OPC_DMULTU:
-        gen_helper_dmultu(t0, t1);
+        gen_helper_dmultu(cpu_env, t0, t1);
         opn = "dmultu";
         break;
 #endif
@@ -2150,41 +2792,49 @@ static void gen_muldiv (DisasContext *ctx, uint32_t opc,
         {
             TCGv_i64 t2 = tcg_temp_new_i64();
             TCGv_i64 t3 = tcg_temp_new_i64();
+            acc = ((ctx->opcode) >> 11) & 0x03;
+            if (acc != 0) {
+                check_dsp(ctx);
+            }
 
             tcg_gen_ext_tl_i64(t2, t0);
             tcg_gen_ext_tl_i64(t3, t1);
             tcg_gen_mul_i64(t2, t2, t3);
-            tcg_gen_concat_tl_i64(t3, cpu_LO[0], cpu_HI[0]);
+            tcg_gen_concat_tl_i64(t3, cpu_LO[acc], cpu_HI[acc]);
             tcg_gen_add_i64(t2, t2, t3);
             tcg_temp_free_i64(t3);
             tcg_gen_trunc_i64_tl(t0, t2);
             tcg_gen_shri_i64(t2, t2, 32);
             tcg_gen_trunc_i64_tl(t1, t2);
             tcg_temp_free_i64(t2);
-            tcg_gen_ext32s_tl(cpu_LO[0], t0);
-            tcg_gen_ext32s_tl(cpu_HI[0], t1);
+            tcg_gen_ext32s_tl(cpu_LO[acc], t0);
+            tcg_gen_ext32s_tl(cpu_HI[acc], t1);
         }
         opn = "madd";
         break;
     case OPC_MADDU:
-       {
+        {
             TCGv_i64 t2 = tcg_temp_new_i64();
             TCGv_i64 t3 = tcg_temp_new_i64();
+            acc = ((ctx->opcode) >> 11) & 0x03;
+            if (acc != 0) {
+                check_dsp(ctx);
+            }
 
             tcg_gen_ext32u_tl(t0, t0);
             tcg_gen_ext32u_tl(t1, t1);
             tcg_gen_extu_tl_i64(t2, t0);
             tcg_gen_extu_tl_i64(t3, t1);
             tcg_gen_mul_i64(t2, t2, t3);
-            tcg_gen_concat_tl_i64(t3, cpu_LO[0], cpu_HI[0]);
+            tcg_gen_concat_tl_i64(t3, cpu_LO[acc], cpu_HI[acc]);
             tcg_gen_add_i64(t2, t2, t3);
             tcg_temp_free_i64(t3);
             tcg_gen_trunc_i64_tl(t0, t2);
             tcg_gen_shri_i64(t2, t2, 32);
             tcg_gen_trunc_i64_tl(t1, t2);
             tcg_temp_free_i64(t2);
-            tcg_gen_ext32s_tl(cpu_LO[0], t0);
-            tcg_gen_ext32s_tl(cpu_HI[0], t1);
+            tcg_gen_ext32s_tl(cpu_LO[acc], t0);
+            tcg_gen_ext32s_tl(cpu_HI[acc], t1);
         }
         opn = "maddu";
         break;
@@ -2192,19 +2842,23 @@ static void gen_muldiv (DisasContext *ctx, uint32_t opc,
         {
             TCGv_i64 t2 = tcg_temp_new_i64();
             TCGv_i64 t3 = tcg_temp_new_i64();
+            acc = ((ctx->opcode) >> 11) & 0x03;
+            if (acc != 0) {
+                check_dsp(ctx);
+            }
 
             tcg_gen_ext_tl_i64(t2, t0);
             tcg_gen_ext_tl_i64(t3, t1);
             tcg_gen_mul_i64(t2, t2, t3);
-            tcg_gen_concat_tl_i64(t3, cpu_LO[0], cpu_HI[0]);
+            tcg_gen_concat_tl_i64(t3, cpu_LO[acc], cpu_HI[acc]);
             tcg_gen_sub_i64(t2, t3, t2);
             tcg_temp_free_i64(t3);
             tcg_gen_trunc_i64_tl(t0, t2);
             tcg_gen_shri_i64(t2, t2, 32);
             tcg_gen_trunc_i64_tl(t1, t2);
             tcg_temp_free_i64(t2);
-            tcg_gen_ext32s_tl(cpu_LO[0], t0);
-            tcg_gen_ext32s_tl(cpu_HI[0], t1);
+            tcg_gen_ext32s_tl(cpu_LO[acc], t0);
+            tcg_gen_ext32s_tl(cpu_HI[acc], t1);
         }
         opn = "msub";
         break;
@@ -2212,21 +2866,25 @@ static void gen_muldiv (DisasContext *ctx, uint32_t opc,
         {
             TCGv_i64 t2 = tcg_temp_new_i64();
             TCGv_i64 t3 = tcg_temp_new_i64();
+            acc = ((ctx->opcode) >> 11) & 0x03;
+            if (acc != 0) {
+                check_dsp(ctx);
+            }
 
             tcg_gen_ext32u_tl(t0, t0);
             tcg_gen_ext32u_tl(t1, t1);
             tcg_gen_extu_tl_i64(t2, t0);
             tcg_gen_extu_tl_i64(t3, t1);
             tcg_gen_mul_i64(t2, t2, t3);
-            tcg_gen_concat_tl_i64(t3, cpu_LO[0], cpu_HI[0]);
+            tcg_gen_concat_tl_i64(t3, cpu_LO[acc], cpu_HI[acc]);
             tcg_gen_sub_i64(t2, t3, t2);
             tcg_temp_free_i64(t3);
             tcg_gen_trunc_i64_tl(t0, t2);
             tcg_gen_shri_i64(t2, t2, 32);
             tcg_gen_trunc_i64_tl(t1, t2);
             tcg_temp_free_i64(t2);
-            tcg_gen_ext32s_tl(cpu_LO[0], t0);
-            tcg_gen_ext32s_tl(cpu_HI[0], t1);
+            tcg_gen_ext32s_tl(cpu_LO[acc], t0);
+            tcg_gen_ext32s_tl(cpu_HI[acc], t1);
         }
         opn = "msubu";
         break;
@@ -2254,59 +2912,59 @@ static void gen_mul_vr54xx (DisasContext *ctx, uint32_t opc,
 
     switch (opc) {
     case OPC_VR54XX_MULS:
-        gen_helper_muls(t0, t0, t1);
+        gen_helper_muls(t0, cpu_env, t0, t1);
         opn = "muls";
         break;
     case OPC_VR54XX_MULSU:
-        gen_helper_mulsu(t0, t0, t1);
+        gen_helper_mulsu(t0, cpu_env, t0, t1);
         opn = "mulsu";
         break;
     case OPC_VR54XX_MACC:
-        gen_helper_macc(t0, t0, t1);
+        gen_helper_macc(t0, cpu_env, t0, t1);
         opn = "macc";
         break;
     case OPC_VR54XX_MACCU:
-        gen_helper_maccu(t0, t0, t1);
+        gen_helper_maccu(t0, cpu_env, t0, t1);
         opn = "maccu";
         break;
     case OPC_VR54XX_MSAC:
-        gen_helper_msac(t0, t0, t1);
+        gen_helper_msac(t0, cpu_env, t0, t1);
         opn = "msac";
         break;
     case OPC_VR54XX_MSACU:
-        gen_helper_msacu(t0, t0, t1);
+        gen_helper_msacu(t0, cpu_env, t0, t1);
         opn = "msacu";
         break;
     case OPC_VR54XX_MULHI:
-        gen_helper_mulhi(t0, t0, t1);
+        gen_helper_mulhi(t0, cpu_env, t0, t1);
         opn = "mulhi";
         break;
     case OPC_VR54XX_MULHIU:
-        gen_helper_mulhiu(t0, t0, t1);
+        gen_helper_mulhiu(t0, cpu_env, t0, t1);
         opn = "mulhiu";
         break;
     case OPC_VR54XX_MULSHI:
-        gen_helper_mulshi(t0, t0, t1);
+        gen_helper_mulshi(t0, cpu_env, t0, t1);
         opn = "mulshi";
         break;
     case OPC_VR54XX_MULSHIU:
-        gen_helper_mulshiu(t0, t0, t1);
+        gen_helper_mulshiu(t0, cpu_env, t0, t1);
         opn = "mulshiu";
         break;
     case OPC_VR54XX_MACCHI:
-        gen_helper_macchi(t0, t0, t1);
+        gen_helper_macchi(t0, cpu_env, t0, t1);
         opn = "macchi";
         break;
     case OPC_VR54XX_MACCHIU:
-        gen_helper_macchiu(t0, t0, t1);
+        gen_helper_macchiu(t0, cpu_env, t0, t1);
         opn = "macchiu";
         break;
     case OPC_VR54XX_MSACHI:
-        gen_helper_msachi(t0, t0, t1);
+        gen_helper_msachi(t0, cpu_env, t0, t1);
         opn = "msachi";
         break;
     case OPC_VR54XX_MSACHIU:
-        gen_helper_msachiu(t0, t0, t1);
+        gen_helper_msachiu(t0, cpu_env, t0, t1);
         opn = "msachiu";
         break;
     default:
@@ -2362,8 +3020,8 @@ static void gen_cl (DisasContext *ctx, uint32_t opc,
 }
 
 /* Godson integer instructions */
-static void gen_loongson_integer (DisasContext *ctx, uint32_t opc,
-                                int rd, int rs, int rt)
+static void gen_loongson_integer(DisasContext *ctx, uint32_t opc,
+                                 int rd, int rs, int rt)
 {
     const char *opn = "loongson";
     TCGv t0, t1;
@@ -2576,6 +3234,278 @@ static void gen_loongson_integer (DisasContext *ctx, uint32_t opc,
     tcg_temp_free(t1);
 }
 
+/* Loongson multimedia instructions */
+static void gen_loongson_multimedia(DisasContext *ctx, int rd, int rs, int rt)
+{
+    const char *opn = "loongson_cp2";
+    uint32_t opc, shift_max;
+    TCGv_i64 t0, t1;
+
+    opc = MASK_LMI(ctx->opcode);
+    switch (opc) {
+    case OPC_ADD_CP2:
+    case OPC_SUB_CP2:
+    case OPC_DADD_CP2:
+    case OPC_DSUB_CP2:
+        t0 = tcg_temp_local_new_i64();
+        t1 = tcg_temp_local_new_i64();
+        break;
+    default:
+        t0 = tcg_temp_new_i64();
+        t1 = tcg_temp_new_i64();
+        break;
+    }
+
+    gen_load_fpr64(ctx, t0, rs);
+    gen_load_fpr64(ctx, t1, rt);
+
+#define LMI_HELPER(UP, LO) \
+    case OPC_##UP: gen_helper_##LO(t0, t0, t1); opn = #LO; break
+#define LMI_HELPER_1(UP, LO) \
+    case OPC_##UP: gen_helper_##LO(t0, t0); opn = #LO; break
+#define LMI_DIRECT(UP, LO, OP) \
+    case OPC_##UP: tcg_gen_##OP##_i64(t0, t0, t1); opn = #LO; break
+
+    switch (opc) {
+    LMI_HELPER(PADDSH, paddsh);
+    LMI_HELPER(PADDUSH, paddush);
+    LMI_HELPER(PADDH, paddh);
+    LMI_HELPER(PADDW, paddw);
+    LMI_HELPER(PADDSB, paddsb);
+    LMI_HELPER(PADDUSB, paddusb);
+    LMI_HELPER(PADDB, paddb);
+
+    LMI_HELPER(PSUBSH, psubsh);
+    LMI_HELPER(PSUBUSH, psubush);
+    LMI_HELPER(PSUBH, psubh);
+    LMI_HELPER(PSUBW, psubw);
+    LMI_HELPER(PSUBSB, psubsb);
+    LMI_HELPER(PSUBUSB, psubusb);
+    LMI_HELPER(PSUBB, psubb);
+
+    LMI_HELPER(PSHUFH, pshufh);
+    LMI_HELPER(PACKSSWH, packsswh);
+    LMI_HELPER(PACKSSHB, packsshb);
+    LMI_HELPER(PACKUSHB, packushb);
+
+    LMI_HELPER(PUNPCKLHW, punpcklhw);
+    LMI_HELPER(PUNPCKHHW, punpckhhw);
+    LMI_HELPER(PUNPCKLBH, punpcklbh);
+    LMI_HELPER(PUNPCKHBH, punpckhbh);
+    LMI_HELPER(PUNPCKLWD, punpcklwd);
+    LMI_HELPER(PUNPCKHWD, punpckhwd);
+
+    LMI_HELPER(PAVGH, pavgh);
+    LMI_HELPER(PAVGB, pavgb);
+    LMI_HELPER(PMAXSH, pmaxsh);
+    LMI_HELPER(PMINSH, pminsh);
+    LMI_HELPER(PMAXUB, pmaxub);
+    LMI_HELPER(PMINUB, pminub);
+
+    LMI_HELPER(PCMPEQW, pcmpeqw);
+    LMI_HELPER(PCMPGTW, pcmpgtw);
+    LMI_HELPER(PCMPEQH, pcmpeqh);
+    LMI_HELPER(PCMPGTH, pcmpgth);
+    LMI_HELPER(PCMPEQB, pcmpeqb);
+    LMI_HELPER(PCMPGTB, pcmpgtb);
+
+    LMI_HELPER(PSLLW, psllw);
+    LMI_HELPER(PSLLH, psllh);
+    LMI_HELPER(PSRLW, psrlw);
+    LMI_HELPER(PSRLH, psrlh);
+    LMI_HELPER(PSRAW, psraw);
+    LMI_HELPER(PSRAH, psrah);
+
+    LMI_HELPER(PMULLH, pmullh);
+    LMI_HELPER(PMULHH, pmulhh);
+    LMI_HELPER(PMULHUH, pmulhuh);
+    LMI_HELPER(PMADDHW, pmaddhw);
+
+    LMI_HELPER(PASUBUB, pasubub);
+    LMI_HELPER_1(BIADD, biadd);
+    LMI_HELPER_1(PMOVMSKB, pmovmskb);
+
+    LMI_DIRECT(PADDD, paddd, add);
+    LMI_DIRECT(PSUBD, psubd, sub);
+    LMI_DIRECT(XOR_CP2, xor, xor);
+    LMI_DIRECT(NOR_CP2, nor, nor);
+    LMI_DIRECT(AND_CP2, and, and);
+    LMI_DIRECT(PANDN, pandn, andc);
+    LMI_DIRECT(OR, or, or);
+
+    case OPC_PINSRH_0:
+        tcg_gen_deposit_i64(t0, t0, t1, 0, 16);
+        opn = "pinsrh_0";
+        break;
+    case OPC_PINSRH_1:
+        tcg_gen_deposit_i64(t0, t0, t1, 16, 16);
+        opn = "pinsrh_1";
+        break;
+    case OPC_PINSRH_2:
+        tcg_gen_deposit_i64(t0, t0, t1, 32, 16);
+        opn = "pinsrh_2";
+        break;
+    case OPC_PINSRH_3:
+        tcg_gen_deposit_i64(t0, t0, t1, 48, 16);
+        opn = "pinsrh_3";
+        break;
+
+    case OPC_PEXTRH:
+        tcg_gen_andi_i64(t1, t1, 3);
+        tcg_gen_shli_i64(t1, t1, 4);
+        tcg_gen_shr_i64(t0, t0, t1);
+        tcg_gen_ext16u_i64(t0, t0);
+        opn = "pextrh";
+        break;
+
+    case OPC_ADDU_CP2:
+        tcg_gen_add_i64(t0, t0, t1);
+        tcg_gen_ext32s_i64(t0, t0);
+        opn = "addu";
+        break;
+    case OPC_SUBU_CP2:
+        tcg_gen_sub_i64(t0, t0, t1);
+        tcg_gen_ext32s_i64(t0, t0);
+        opn = "addu";
+        break;
+
+    case OPC_SLL_CP2:
+        opn = "sll";
+        shift_max = 32;
+        goto do_shift;
+    case OPC_SRL_CP2:
+        opn = "srl";
+        shift_max = 32;
+        goto do_shift;
+    case OPC_SRA_CP2:
+        opn = "sra";
+        shift_max = 32;
+        goto do_shift;
+    case OPC_DSLL_CP2:
+        opn = "dsll";
+        shift_max = 64;
+        goto do_shift;
+    case OPC_DSRL_CP2:
+        opn = "dsrl";
+        shift_max = 64;
+        goto do_shift;
+    case OPC_DSRA_CP2:
+        opn = "dsra";
+        shift_max = 64;
+        goto do_shift;
+    do_shift:
+        /* Make sure shift count isn't TCG undefined behaviour.  */
+        tcg_gen_andi_i64(t1, t1, shift_max - 1);
+
+        switch (opc) {
+        case OPC_SLL_CP2:
+        case OPC_DSLL_CP2:
+            tcg_gen_shl_i64(t0, t0, t1);
+            break;
+        case OPC_SRA_CP2:
+        case OPC_DSRA_CP2:
+            /* Since SRA is UndefinedResult without sign-extended inputs,
+               we can treat SRA and DSRA the same.  */
+            tcg_gen_sar_i64(t0, t0, t1);
+            break;
+        case OPC_SRL_CP2:
+            /* We want to shift in zeros for SRL; zero-extend first.  */
+            tcg_gen_ext32u_i64(t0, t0);
+            /* FALLTHRU */
+        case OPC_DSRL_CP2:
+            tcg_gen_shr_i64(t0, t0, t1);
+            break;
+        }
+
+        if (shift_max == 32) {
+            tcg_gen_ext32s_i64(t0, t0);
+        }
+
+        /* Shifts larger than MAX produce zero.  */
+        tcg_gen_setcondi_i64(TCG_COND_LTU, t1, t1, shift_max);
+        tcg_gen_neg_i64(t1, t1);
+        tcg_gen_and_i64(t0, t0, t1);
+        break;
+
+    case OPC_ADD_CP2:
+    case OPC_DADD_CP2:
+        {
+            TCGv_i64 t2 = tcg_temp_new_i64();
+            int lab = gen_new_label();
+
+            tcg_gen_mov_i64(t2, t0);
+            tcg_gen_add_i64(t0, t1, t2);
+            if (opc == OPC_ADD_CP2) {
+                tcg_gen_ext32s_i64(t0, t0);
+            }
+            tcg_gen_xor_i64(t1, t1, t2);
+            tcg_gen_xor_i64(t2, t2, t0);
+            tcg_gen_andc_i64(t1, t2, t1);
+            tcg_temp_free_i64(t2);
+            tcg_gen_brcondi_i64(TCG_COND_GE, t1, 0, lab);
+            generate_exception(ctx, EXCP_OVERFLOW);
+            gen_set_label(lab);
+
+            opn = (opc == OPC_ADD_CP2 ? "add" : "dadd");
+            break;
+        }
+
+    case OPC_SUB_CP2:
+    case OPC_DSUB_CP2:
+        {
+            TCGv_i64 t2 = tcg_temp_new_i64();
+            int lab = gen_new_label();
+
+            tcg_gen_mov_i64(t2, t0);
+            tcg_gen_sub_i64(t0, t1, t2);
+            if (opc == OPC_SUB_CP2) {
+                tcg_gen_ext32s_i64(t0, t0);
+            }
+            tcg_gen_xor_i64(t1, t1, t2);
+            tcg_gen_xor_i64(t2, t2, t0);
+            tcg_gen_and_i64(t1, t1, t2);
+            tcg_temp_free_i64(t2);
+            tcg_gen_brcondi_i64(TCG_COND_GE, t1, 0, lab);
+            generate_exception(ctx, EXCP_OVERFLOW);
+            gen_set_label(lab);
+
+            opn = (opc == OPC_SUB_CP2 ? "sub" : "dsub");
+            break;
+        }
+
+    case OPC_PMULUW:
+        tcg_gen_ext32u_i64(t0, t0);
+        tcg_gen_ext32u_i64(t1, t1);
+        tcg_gen_mul_i64(t0, t0, t1);
+        opn = "pmuluw";
+        break;
+
+    case OPC_SEQU_CP2:
+    case OPC_SEQ_CP2:
+    case OPC_SLTU_CP2:
+    case OPC_SLT_CP2:
+    case OPC_SLEU_CP2:
+    case OPC_SLE_CP2:
+        /* ??? Document is unclear: Set FCC[CC].  Does that mean the
+           FD field is the CC field?  */
+    default:
+        MIPS_INVAL(opn);
+        generate_exception(ctx, EXCP_RI);
+        return;
+    }
+
+#undef LMI_HELPER
+#undef LMI_DIRECT
+
+    gen_store_fpr64(ctx, t0, rd);
+
+    (void)opn; /* avoid a compiler warning */
+    MIPS_DEBUG("%s %s, %s, %s", opn,
+               fregnames[rd], fregnames[rs], fregnames[rt]);
+    tcg_temp_free_i64(t0);
+    tcg_temp_free_i64(t1);
+}
+
 /* Traps */
 static void gen_trap (DisasContext *ctx, uint32_t opc,
                       int rs, int rt, int16_t imm)
@@ -2683,7 +3613,7 @@ static inline void gen_goto_tb(DisasContext *ctx, int n, target_ulong dest)
         gen_save_pc(dest);
         if (ctx->singlestep_enabled) {
             save_cpu_state(ctx, 0);
-            gen_helper_0i(raise_exception, EXCP_DEBUG);
+            gen_helper_0e0i(raise_exception, EXCP_DEBUG);
         }
         tcg_gen_exit_tb(0);
     }
@@ -2743,6 +3673,16 @@ static void gen_compute_branch (DisasContext *ctx, uint32_t opc,
         }
         btgt = ctx->pc + insn_bytes + offset;
         break;
+    case OPC_BPOSGE32:
+#if defined(TARGET_MIPS64)
+    case OPC_BPOSGE64:
+        tcg_gen_andi_tl(t0, cpu_dspctrl, 0x7F);
+#else
+        tcg_gen_andi_tl(t0, cpu_dspctrl, 0x3F);
+#endif
+        bcond_compute = 1;
+        btgt = ctx->pc + insn_bytes + offset;
+        break;
     case OPC_J:
     case OPC_JAL:
     case OPC_JALX:
@@ -2931,6 +3871,16 @@ static void gen_compute_branch (DisasContext *ctx, uint32_t opc,
             tcg_gen_setcondi_tl(TCG_COND_LT, bcond, t0, 0);
             MIPS_DEBUG("bltzl %s, " TARGET_FMT_lx, regnames[rs], btgt);
             goto likely;
+        case OPC_BPOSGE32:
+            tcg_gen_setcondi_tl(TCG_COND_GE, bcond, t0, 32);
+            MIPS_DEBUG("bposge32 " TARGET_FMT_lx, btgt);
+            goto not_likely;
+#if defined(TARGET_MIPS64)
+        case OPC_BPOSGE64:
+            tcg_gen_setcondi_tl(TCG_COND_GE, bcond, t0, 64);
+            MIPS_DEBUG("bposge64 " TARGET_FMT_lx, btgt);
+            goto not_likely;
+#endif
         case OPC_BLTZALS:
         case OPC_BLTZAL:
             ctx->hflags |= (opc == OPC_BLTZALS
@@ -2982,7 +3932,6 @@ static void gen_bitops (DisasContext *ctx, uint32_t opc, int rt,
 {
     TCGv t0 = tcg_temp_new();
     TCGv t1 = tcg_temp_new();
-    target_ulong mask;
 
     gen_load_gpr(t1, rs);
     switch (opc) {
@@ -3015,45 +3964,22 @@ static void gen_bitops (DisasContext *ctx, uint32_t opc, int rt,
     case OPC_INS:
         if (lsb > msb)
             goto fail;
-        mask = ((msb - lsb + 1 < 32) ? ((1 << (msb - lsb + 1)) - 1) : ~0) << lsb;
         gen_load_gpr(t0, rt);
-        tcg_gen_andi_tl(t0, t0, ~mask);
-        tcg_gen_shli_tl(t1, t1, lsb);
-        tcg_gen_andi_tl(t1, t1, mask);
-        tcg_gen_or_tl(t0, t0, t1);
+        tcg_gen_deposit_tl(t0, t0, t1, lsb, msb - lsb + 1);
         tcg_gen_ext32s_tl(t0, t0);
         break;
 #if defined(TARGET_MIPS64)
     case OPC_DINSM:
-        if (lsb > msb)
-            goto fail;
-        mask = ((msb - lsb + 1 + 32 < 64) ? ((1ULL << (msb - lsb + 1 + 32)) - 1) : ~0ULL) << lsb;
         gen_load_gpr(t0, rt);
-        tcg_gen_andi_tl(t0, t0, ~mask);
-        tcg_gen_shli_tl(t1, t1, lsb);
-        tcg_gen_andi_tl(t1, t1, mask);
-        tcg_gen_or_tl(t0, t0, t1);
+        tcg_gen_deposit_tl(t0, t0, t1, lsb, msb + 32 - lsb + 1);
         break;
     case OPC_DINSU:
-        if (lsb > msb)
-            goto fail;
-        mask = ((1ULL << (msb - lsb + 1)) - 1) << (lsb + 32);
         gen_load_gpr(t0, rt);
-        tcg_gen_andi_tl(t0, t0, ~mask);
-        tcg_gen_shli_tl(t1, t1, lsb + 32);
-        tcg_gen_andi_tl(t1, t1, mask);
-        tcg_gen_or_tl(t0, t0, t1);
+        tcg_gen_deposit_tl(t0, t0, t1, lsb + 32, msb - lsb + 1);
         break;
     case OPC_DINS:
-        if (lsb > msb)
-            goto fail;
-        gen_load_gpr(t0, rt);
-        mask = ((1ULL << (msb - lsb + 1)) - 1) << lsb;
         gen_load_gpr(t0, rt);
-        tcg_gen_andi_tl(t0, t0, ~mask);
-        tcg_gen_shli_tl(t1, t1, lsb);
-        tcg_gen_andi_tl(t1, t1, mask);
-        tcg_gen_or_tl(t0, t0, t1);
+        tcg_gen_deposit_tl(t0, t0, t1, lsb, msb - lsb + 1);
         break;
 #endif
     default:
@@ -3187,17 +4113,17 @@ static void gen_mfc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
             break;
         case 1:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mfc0_mvpcontrol(arg);
+            gen_helper_mfc0_mvpcontrol(arg, cpu_env);
             rn = "MVPControl";
             break;
         case 2:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mfc0_mvpconf0(arg);
+            gen_helper_mfc0_mvpconf0(arg, cpu_env);
             rn = "MVPConf0";
             break;
         case 3:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mfc0_mvpconf1(arg);
+            gen_helper_mfc0_mvpconf1(arg, cpu_env);
             rn = "MVPConf1";
             break;
         default:
@@ -3207,7 +4133,7 @@ static void gen_mfc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
     case 1:
         switch (sel) {
         case 0:
-            gen_helper_mfc0_random(arg);
+            gen_helper_mfc0_random(arg, cpu_env);
             rn = "Random";
             break;
         case 1:
@@ -3258,37 +4184,37 @@ static void gen_mfc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
             break;
         case 1:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mfc0_tcstatus(arg);
+            gen_helper_mfc0_tcstatus(arg, cpu_env);
             rn = "TCStatus";
             break;
         case 2:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mfc0_tcbind(arg);
+            gen_helper_mfc0_tcbind(arg, cpu_env);
             rn = "TCBind";
             break;
         case 3:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mfc0_tcrestart(arg);
+            gen_helper_mfc0_tcrestart(arg, cpu_env);
             rn = "TCRestart";
             break;
         case 4:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mfc0_tchalt(arg);
+            gen_helper_mfc0_tchalt(arg, cpu_env);
             rn = "TCHalt";
             break;
         case 5:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mfc0_tccontext(arg);
+            gen_helper_mfc0_tccontext(arg, cpu_env);
             rn = "TCContext";
             break;
         case 6:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mfc0_tcschedule(arg);
+            gen_helper_mfc0_tcschedule(arg, cpu_env);
             rn = "TCSchedule";
             break;
         case 7:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mfc0_tcschefback(arg);
+            gen_helper_mfc0_tcschefback(arg, cpu_env);
             rn = "TCScheFBack";
             break;
         default:
@@ -3399,7 +4325,7 @@ static void gen_mfc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
             /* Mark as an IO operation because we read the time.  */
             if (use_icount)
                 gen_io_start();
-            gen_helper_mfc0_count(arg);
+            gen_helper_mfc0_count(arg, cpu_env);
             if (use_icount) {
                 gen_io_end();
             }
@@ -3531,7 +4457,7 @@ static void gen_mfc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
     case 17:
         switch (sel) {
         case 0:
-            gen_helper_mfc0_lladdr(arg);
+            gen_helper_mfc0_lladdr(arg, cpu_env);
             rn = "LLAddr";
             break;
         default:
@@ -3541,7 +4467,7 @@ static void gen_mfc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
     case 18:
         switch (sel) {
         case 0 ... 7:
-            gen_helper_1i(mfc0_watchlo, arg, sel);
+            gen_helper_1e0i(mfc0_watchlo, arg, sel);
             rn = "WatchLo";
             break;
         default:
@@ -3551,7 +4477,7 @@ static void gen_mfc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
     case 19:
         switch (sel) {
         case 0 ...7:
-            gen_helper_1i(mfc0_watchhi, arg, sel);
+            gen_helper_1e0i(mfc0_watchhi, arg, sel);
             rn = "WatchHi";
             break;
         default:
@@ -3590,7 +4516,7 @@ static void gen_mfc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
     case 23:
         switch (sel) {
         case 0:
-            gen_helper_mfc0_debug(arg); /* EJTAG support */
+            gen_helper_mfc0_debug(arg, cpu_env); /* EJTAG support */
             rn = "Debug";
             break;
         case 1:
@@ -3765,12 +4691,12 @@ static void gen_mtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
     case 0:
         switch (sel) {
         case 0:
-            gen_helper_mtc0_index(arg);
+            gen_helper_mtc0_index(cpu_env, arg);
             rn = "Index";
             break;
         case 1:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mtc0_mvpcontrol(arg);
+            gen_helper_mtc0_mvpcontrol(cpu_env, arg);
             rn = "MVPControl";
             break;
         case 2:
@@ -3795,22 +4721,22 @@ static void gen_mtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
             break;
         case 1:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mtc0_vpecontrol(arg);
+            gen_helper_mtc0_vpecontrol(cpu_env, arg);
             rn = "VPEControl";
             break;
         case 2:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mtc0_vpeconf0(arg);
+            gen_helper_mtc0_vpeconf0(cpu_env, arg);
             rn = "VPEConf0";
             break;
         case 3:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mtc0_vpeconf1(arg);
+            gen_helper_mtc0_vpeconf1(cpu_env, arg);
             rn = "VPEConf1";
             break;
         case 4:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mtc0_yqmask(arg);
+            gen_helper_mtc0_yqmask(cpu_env, arg);
             rn = "YQMask";
             break;
         case 5:
@@ -3825,7 +4751,7 @@ static void gen_mtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
             break;
         case 7:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mtc0_vpeopt(arg);
+            gen_helper_mtc0_vpeopt(cpu_env, arg);
             rn = "VPEOpt";
             break;
         default:
@@ -3835,42 +4761,42 @@ static void gen_mtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
     case 2:
         switch (sel) {
         case 0:
-            gen_helper_mtc0_entrylo0(arg);
+            gen_helper_mtc0_entrylo0(cpu_env, arg);
             rn = "EntryLo0";
             break;
         case 1:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mtc0_tcstatus(arg);
+            gen_helper_mtc0_tcstatus(cpu_env, arg);
             rn = "TCStatus";
             break;
         case 2:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mtc0_tcbind(arg);
+            gen_helper_mtc0_tcbind(cpu_env, arg);
             rn = "TCBind";
             break;
         case 3:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mtc0_tcrestart(arg);
+            gen_helper_mtc0_tcrestart(cpu_env, arg);
             rn = "TCRestart";
             break;
         case 4:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mtc0_tchalt(arg);
+            gen_helper_mtc0_tchalt(cpu_env, arg);
             rn = "TCHalt";
             break;
         case 5:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mtc0_tccontext(arg);
+            gen_helper_mtc0_tccontext(cpu_env, arg);
             rn = "TCContext";
             break;
         case 6:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mtc0_tcschedule(arg);
+            gen_helper_mtc0_tcschedule(cpu_env, arg);
             rn = "TCSchedule";
             break;
         case 7:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mtc0_tcschefback(arg);
+            gen_helper_mtc0_tcschefback(cpu_env, arg);
             rn = "TCScheFBack";
             break;
         default:
@@ -3880,7 +4806,7 @@ static void gen_mtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
     case 3:
         switch (sel) {
         case 0:
-            gen_helper_mtc0_entrylo1(arg);
+            gen_helper_mtc0_entrylo1(cpu_env, arg);
             rn = "EntryLo1";
             break;
         default:
@@ -3890,11 +4816,11 @@ static void gen_mtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
     case 4:
         switch (sel) {
         case 0:
-            gen_helper_mtc0_context(arg);
+            gen_helper_mtc0_context(cpu_env, arg);
             rn = "Context";
             break;
         case 1:
-//            gen_helper_mtc0_contextconfig(arg); /* SmartMIPS ASE */
+//            gen_helper_mtc0_contextconfig(cpu_env, arg); /* SmartMIPS ASE */
             rn = "ContextConfig";
 //            break;
         default:
@@ -3904,12 +4830,12 @@ static void gen_mtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
     case 5:
         switch (sel) {
         case 0:
-            gen_helper_mtc0_pagemask(arg);
+            gen_helper_mtc0_pagemask(cpu_env, arg);
             rn = "PageMask";
             break;
         case 1:
             check_insn(env, ctx, ISA_MIPS32R2);
-            gen_helper_mtc0_pagegrain(arg);
+            gen_helper_mtc0_pagegrain(cpu_env, arg);
             rn = "PageGrain";
             break;
         default:
@@ -3919,32 +4845,32 @@ static void gen_mtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
     case 6:
         switch (sel) {
         case 0:
-            gen_helper_mtc0_wired(arg);
+            gen_helper_mtc0_wired(cpu_env, arg);
             rn = "Wired";
             break;
         case 1:
             check_insn(env, ctx, ISA_MIPS32R2);
-            gen_helper_mtc0_srsconf0(arg);
+            gen_helper_mtc0_srsconf0(cpu_env, arg);
             rn = "SRSConf0";
             break;
         case 2:
             check_insn(env, ctx, ISA_MIPS32R2);
-            gen_helper_mtc0_srsconf1(arg);
+            gen_helper_mtc0_srsconf1(cpu_env, arg);
             rn = "SRSConf1";
             break;
         case 3:
             check_insn(env, ctx, ISA_MIPS32R2);
-            gen_helper_mtc0_srsconf2(arg);
+            gen_helper_mtc0_srsconf2(cpu_env, arg);
             rn = "SRSConf2";
             break;
         case 4:
             check_insn(env, ctx, ISA_MIPS32R2);
-            gen_helper_mtc0_srsconf3(arg);
+            gen_helper_mtc0_srsconf3(cpu_env, arg);
             rn = "SRSConf3";
             break;
         case 5:
             check_insn(env, ctx, ISA_MIPS32R2);
-            gen_helper_mtc0_srsconf4(arg);
+            gen_helper_mtc0_srsconf4(cpu_env, arg);
             rn = "SRSConf4";
             break;
         default:
@@ -3955,7 +4881,7 @@ static void gen_mtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
         switch (sel) {
         case 0:
             check_insn(env, ctx, ISA_MIPS32R2);
-            gen_helper_mtc0_hwrena(arg);
+            gen_helper_mtc0_hwrena(cpu_env, arg);
             rn = "HWREna";
             break;
         default:
@@ -3969,7 +4895,7 @@ static void gen_mtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
     case 9:
         switch (sel) {
         case 0:
-            gen_helper_mtc0_count(arg);
+            gen_helper_mtc0_count(cpu_env, arg);
             rn = "Count";
             break;
         /* 6,7 are implementation dependent */
@@ -3980,7 +4906,7 @@ static void gen_mtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
     case 10:
         switch (sel) {
         case 0:
-            gen_helper_mtc0_entryhi(arg);
+            gen_helper_mtc0_entryhi(cpu_env, arg);
             rn = "EntryHi";
             break;
         default:
@@ -3990,7 +4916,7 @@ static void gen_mtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
     case 11:
         switch (sel) {
         case 0:
-            gen_helper_mtc0_compare(arg);
+            gen_helper_mtc0_compare(cpu_env, arg);
             rn = "Compare";
             break;
         /* 6,7 are implementation dependent */
@@ -4002,7 +4928,7 @@ static void gen_mtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
         switch (sel) {
         case 0:
             save_cpu_state(ctx, 1);
-            gen_helper_mtc0_status(arg);
+            gen_helper_mtc0_status(cpu_env, arg);
             /* BS_STOP isn't good enough here, hflags may have changed. */
             gen_save_pc(ctx->pc + 4);
             ctx->bstate = BS_EXCP;
@@ -4010,14 +4936,14 @@ static void gen_mtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
             break;
         case 1:
             check_insn(env, ctx, ISA_MIPS32R2);
-            gen_helper_mtc0_intctl(arg);
+            gen_helper_mtc0_intctl(cpu_env, arg);
             /* Stop translation as we may have switched the execution mode */
             ctx->bstate = BS_STOP;
             rn = "IntCtl";
             break;
         case 2:
             check_insn(env, ctx, ISA_MIPS32R2);
-            gen_helper_mtc0_srsctl(arg);
+            gen_helper_mtc0_srsctl(cpu_env, arg);
             /* Stop translation as we may have switched the execution mode */
             ctx->bstate = BS_STOP;
             rn = "SRSCtl";
@@ -4037,7 +4963,7 @@ static void gen_mtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
         switch (sel) {
         case 0:
             save_cpu_state(ctx, 1);
-            gen_helper_mtc0_cause(arg);
+            gen_helper_mtc0_cause(cpu_env, arg);
             rn = "Cause";
             break;
         default:
@@ -4062,7 +4988,7 @@ static void gen_mtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
             break;
         case 1:
             check_insn(env, ctx, ISA_MIPS32R2);
-            gen_helper_mtc0_ebase(arg);
+            gen_helper_mtc0_ebase(cpu_env, arg);
             rn = "EBase";
             break;
         default:
@@ -4072,7 +4998,7 @@ static void gen_mtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
     case 16:
         switch (sel) {
         case 0:
-            gen_helper_mtc0_config0(arg);
+            gen_helper_mtc0_config0(cpu_env, arg);
             rn = "Config";
             /* Stop translation as we may have switched the execution mode */
             ctx->bstate = BS_STOP;
@@ -4082,7 +5008,7 @@ static void gen_mtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
             rn = "Config1";
             break;
         case 2:
-            gen_helper_mtc0_config2(arg);
+            gen_helper_mtc0_config2(cpu_env, arg);
             rn = "Config2";
             /* Stop translation as we may have switched the execution mode */
             ctx->bstate = BS_STOP;
@@ -4109,7 +5035,7 @@ static void gen_mtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
     case 17:
         switch (sel) {
         case 0:
-            gen_helper_mtc0_lladdr(arg);
+            gen_helper_mtc0_lladdr(cpu_env, arg);
             rn = "LLAddr";
             break;
         default:
@@ -4119,7 +5045,7 @@ static void gen_mtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
     case 18:
         switch (sel) {
         case 0 ... 7:
-            gen_helper_1i(mtc0_watchlo, arg, sel);
+            gen_helper_0e1i(mtc0_watchlo, arg, sel);
             rn = "WatchLo";
             break;
         default:
@@ -4129,7 +5055,7 @@ static void gen_mtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
     case 19:
         switch (sel) {
         case 0 ... 7:
-            gen_helper_1i(mtc0_watchhi, arg, sel);
+            gen_helper_0e1i(mtc0_watchhi, arg, sel);
             rn = "WatchHi";
             break;
         default:
@@ -4141,7 +5067,7 @@ static void gen_mtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
         case 0:
 #if defined(TARGET_MIPS64)
             check_insn(env, ctx, ISA_MIPS3);
-            gen_helper_mtc0_xcontext(arg);
+            gen_helper_mtc0_xcontext(cpu_env, arg);
             rn = "XContext";
             break;
 #endif
@@ -4153,7 +5079,7 @@ static void gen_mtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
        /* Officially reserved, but sel 0 is used for R1x000 framemask */
         switch (sel) {
         case 0:
-            gen_helper_mtc0_framemask(arg);
+            gen_helper_mtc0_framemask(cpu_env, arg);
             rn = "Framemask";
             break;
         default:
@@ -4167,20 +5093,20 @@ static void gen_mtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
     case 23:
         switch (sel) {
         case 0:
-            gen_helper_mtc0_debug(arg); /* EJTAG support */
+            gen_helper_mtc0_debug(cpu_env, arg); /* EJTAG support */
             /* BS_STOP isn't good enough here, hflags may have changed. */
             gen_save_pc(ctx->pc + 4);
             ctx->bstate = BS_EXCP;
             rn = "Debug";
             break;
         case 1:
-//            gen_helper_mtc0_tracecontrol(arg); /* PDtrace support */
+//            gen_helper_mtc0_tracecontrol(cpu_env, arg); /* PDtrace support */
             rn = "TraceControl";
             /* Stop translation as we may have switched the execution mode */
             ctx->bstate = BS_STOP;
 //            break;
         case 2:
-//            gen_helper_mtc0_tracecontrol2(arg); /* PDtrace support */
+//            gen_helper_mtc0_tracecontrol2(cpu_env, arg); /* PDtrace support */
             rn = "TraceControl2";
             /* Stop translation as we may have switched the execution mode */
             ctx->bstate = BS_STOP;
@@ -4188,13 +5114,13 @@ static void gen_mtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
         case 3:
             /* Stop translation as we may have switched the execution mode */
             ctx->bstate = BS_STOP;
-//            gen_helper_mtc0_usertracedata(arg); /* PDtrace support */
+//            gen_helper_mtc0_usertracedata(cpu_env, arg); /* PDtrace support */
             rn = "UserTraceData";
             /* Stop translation as we may have switched the execution mode */
             ctx->bstate = BS_STOP;
 //            break;
         case 4:
-//            gen_helper_mtc0_tracebpc(arg); /* PDtrace support */
+//            gen_helper_mtc0_tracebpc(cpu_env, arg); /* PDtrace support */
             /* Stop translation as we may have switched the execution mode */
             ctx->bstate = BS_STOP;
             rn = "TraceBPC";
@@ -4217,7 +5143,7 @@ static void gen_mtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
     case 25:
         switch (sel) {
         case 0:
-            gen_helper_mtc0_performance0(arg);
+            gen_helper_mtc0_performance0(cpu_env, arg);
             rn = "Performance0";
             break;
         case 1:
@@ -4272,14 +5198,14 @@ static void gen_mtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
         case 2:
         case 4:
         case 6:
-            gen_helper_mtc0_taglo(arg);
+            gen_helper_mtc0_taglo(cpu_env, arg);
             rn = "TagLo";
             break;
         case 1:
         case 3:
         case 5:
         case 7:
-            gen_helper_mtc0_datalo(arg);
+            gen_helper_mtc0_datalo(cpu_env, arg);
             rn = "DataLo";
             break;
         default:
@@ -4292,14 +5218,14 @@ static void gen_mtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg, i
         case 2:
         case 4:
         case 6:
-            gen_helper_mtc0_taghi(arg);
+            gen_helper_mtc0_taghi(cpu_env, arg);
             rn = "TagHi";
             break;
         case 1:
         case 3:
         case 5:
         case 7:
-            gen_helper_mtc0_datahi(arg);
+            gen_helper_mtc0_datahi(cpu_env, arg);
             rn = "DataHi";
             break;
         default:
@@ -4364,17 +5290,17 @@ static void gen_dmfc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
             break;
         case 1:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mfc0_mvpcontrol(arg);
+            gen_helper_mfc0_mvpcontrol(arg, cpu_env);
             rn = "MVPControl";
             break;
         case 2:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mfc0_mvpconf0(arg);
+            gen_helper_mfc0_mvpconf0(arg, cpu_env);
             rn = "MVPConf0";
             break;
         case 3:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mfc0_mvpconf1(arg);
+            gen_helper_mfc0_mvpconf1(arg, cpu_env);
             rn = "MVPConf1";
             break;
         default:
@@ -4384,7 +5310,7 @@ static void gen_dmfc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
     case 1:
         switch (sel) {
         case 0:
-            gen_helper_mfc0_random(arg);
+            gen_helper_mfc0_random(arg, cpu_env);
             rn = "Random";
             break;
         case 1:
@@ -4434,37 +5360,37 @@ static void gen_dmfc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
             break;
         case 1:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mfc0_tcstatus(arg);
+            gen_helper_mfc0_tcstatus(arg, cpu_env);
             rn = "TCStatus";
             break;
         case 2:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mfc0_tcbind(arg);
+            gen_helper_mfc0_tcbind(arg, cpu_env);
             rn = "TCBind";
             break;
         case 3:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_dmfc0_tcrestart(arg);
+            gen_helper_dmfc0_tcrestart(arg, cpu_env);
             rn = "TCRestart";
             break;
         case 4:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_dmfc0_tchalt(arg);
+            gen_helper_dmfc0_tchalt(arg, cpu_env);
             rn = "TCHalt";
             break;
         case 5:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_dmfc0_tccontext(arg);
+            gen_helper_dmfc0_tccontext(arg, cpu_env);
             rn = "TCContext";
             break;
         case 6:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_dmfc0_tcschedule(arg);
+            gen_helper_dmfc0_tcschedule(arg, cpu_env);
             rn = "TCSchedule";
             break;
         case 7:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_dmfc0_tcschefback(arg);
+            gen_helper_dmfc0_tcschefback(arg, cpu_env);
             rn = "TCScheFBack";
             break;
         default:
@@ -4572,7 +5498,7 @@ static void gen_dmfc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
             /* Mark as an IO operation because we read the time.  */
             if (use_icount)
                 gen_io_start();
-            gen_helper_mfc0_count(arg);
+            gen_helper_mfc0_count(arg, cpu_env);
             if (use_icount) {
                 gen_io_end();
             }
@@ -4701,7 +5627,7 @@ static void gen_dmfc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
     case 17:
         switch (sel) {
         case 0:
-            gen_helper_dmfc0_lladdr(arg);
+            gen_helper_dmfc0_lladdr(arg, cpu_env);
             rn = "LLAddr";
             break;
         default:
@@ -4711,7 +5637,7 @@ static void gen_dmfc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
     case 18:
         switch (sel) {
         case 0 ... 7:
-            gen_helper_1i(dmfc0_watchlo, arg, sel);
+            gen_helper_1e0i(dmfc0_watchlo, arg, sel);
             rn = "WatchLo";
             break;
         default:
@@ -4721,7 +5647,7 @@ static void gen_dmfc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
     case 19:
         switch (sel) {
         case 0 ... 7:
-            gen_helper_1i(mfc0_watchhi, arg, sel);
+            gen_helper_1e0i(mfc0_watchhi, arg, sel);
             rn = "WatchHi";
             break;
         default:
@@ -4757,23 +5683,23 @@ static void gen_dmfc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
     case 23:
         switch (sel) {
         case 0:
-            gen_helper_mfc0_debug(arg); /* EJTAG support */
+            gen_helper_mfc0_debug(arg, cpu_env); /* EJTAG support */
             rn = "Debug";
             break;
         case 1:
-//            gen_helper_dmfc0_tracecontrol(arg); /* PDtrace support */
+//            gen_helper_dmfc0_tracecontrol(arg, cpu_env); /* PDtrace support */
             rn = "TraceControl";
 //            break;
         case 2:
-//            gen_helper_dmfc0_tracecontrol2(arg); /* PDtrace support */
+//            gen_helper_dmfc0_tracecontrol2(arg, cpu_env); /* PDtrace support */
             rn = "TraceControl2";
 //            break;
         case 3:
-//            gen_helper_dmfc0_usertracedata(arg); /* PDtrace support */
+//            gen_helper_dmfc0_usertracedata(arg, cpu_env); /* PDtrace support */
             rn = "UserTraceData";
 //            break;
         case 4:
-//            gen_helper_dmfc0_tracebpc(arg); /* PDtrace support */
+//            gen_helper_dmfc0_tracebpc(arg, cpu_env); /* PDtrace support */
             rn = "TraceBPC";
 //            break;
         default:
@@ -4931,12 +5857,12 @@ static void gen_dmtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
     case 0:
         switch (sel) {
         case 0:
-            gen_helper_mtc0_index(arg);
+            gen_helper_mtc0_index(cpu_env, arg);
             rn = "Index";
             break;
         case 1:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mtc0_mvpcontrol(arg);
+            gen_helper_mtc0_mvpcontrol(cpu_env, arg);
             rn = "MVPControl";
             break;
         case 2:
@@ -4961,22 +5887,22 @@ static void gen_dmtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
             break;
         case 1:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mtc0_vpecontrol(arg);
+            gen_helper_mtc0_vpecontrol(cpu_env, arg);
             rn = "VPEControl";
             break;
         case 2:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mtc0_vpeconf0(arg);
+            gen_helper_mtc0_vpeconf0(cpu_env, arg);
             rn = "VPEConf0";
             break;
         case 3:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mtc0_vpeconf1(arg);
+            gen_helper_mtc0_vpeconf1(cpu_env, arg);
             rn = "VPEConf1";
             break;
         case 4:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mtc0_yqmask(arg);
+            gen_helper_mtc0_yqmask(cpu_env, arg);
             rn = "YQMask";
             break;
         case 5:
@@ -4991,7 +5917,7 @@ static void gen_dmtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
             break;
         case 7:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mtc0_vpeopt(arg);
+            gen_helper_mtc0_vpeopt(cpu_env, arg);
             rn = "VPEOpt";
             break;
         default:
@@ -5001,42 +5927,42 @@ static void gen_dmtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
     case 2:
         switch (sel) {
         case 0:
-            gen_helper_mtc0_entrylo0(arg);
+            gen_helper_mtc0_entrylo0(cpu_env, arg);
             rn = "EntryLo0";
             break;
         case 1:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mtc0_tcstatus(arg);
+            gen_helper_mtc0_tcstatus(cpu_env, arg);
             rn = "TCStatus";
             break;
         case 2:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mtc0_tcbind(arg);
+            gen_helper_mtc0_tcbind(cpu_env, arg);
             rn = "TCBind";
             break;
         case 3:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mtc0_tcrestart(arg);
+            gen_helper_mtc0_tcrestart(cpu_env, arg);
             rn = "TCRestart";
             break;
         case 4:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mtc0_tchalt(arg);
+            gen_helper_mtc0_tchalt(cpu_env, arg);
             rn = "TCHalt";
             break;
         case 5:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mtc0_tccontext(arg);
+            gen_helper_mtc0_tccontext(cpu_env, arg);
             rn = "TCContext";
             break;
         case 6:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mtc0_tcschedule(arg);
+            gen_helper_mtc0_tcschedule(cpu_env, arg);
             rn = "TCSchedule";
             break;
         case 7:
             check_insn(env, ctx, ASE_MT);
-            gen_helper_mtc0_tcschefback(arg);
+            gen_helper_mtc0_tcschefback(cpu_env, arg);
             rn = "TCScheFBack";
             break;
         default:
@@ -5046,7 +5972,7 @@ static void gen_dmtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
     case 3:
         switch (sel) {
         case 0:
-            gen_helper_mtc0_entrylo1(arg);
+            gen_helper_mtc0_entrylo1(cpu_env, arg);
             rn = "EntryLo1";
             break;
         default:
@@ -5056,11 +5982,11 @@ static void gen_dmtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
     case 4:
         switch (sel) {
         case 0:
-            gen_helper_mtc0_context(arg);
+            gen_helper_mtc0_context(cpu_env, arg);
             rn = "Context";
             break;
         case 1:
-//           gen_helper_mtc0_contextconfig(arg); /* SmartMIPS ASE */
+//           gen_helper_mtc0_contextconfig(cpu_env, arg); /* SmartMIPS ASE */
             rn = "ContextConfig";
 //           break;
         default:
@@ -5070,12 +5996,12 @@ static void gen_dmtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
     case 5:
         switch (sel) {
         case 0:
-            gen_helper_mtc0_pagemask(arg);
+            gen_helper_mtc0_pagemask(cpu_env, arg);
             rn = "PageMask";
             break;
         case 1:
             check_insn(env, ctx, ISA_MIPS32R2);
-            gen_helper_mtc0_pagegrain(arg);
+            gen_helper_mtc0_pagegrain(cpu_env, arg);
             rn = "PageGrain";
             break;
         default:
@@ -5085,32 +6011,32 @@ static void gen_dmtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
     case 6:
         switch (sel) {
         case 0:
-            gen_helper_mtc0_wired(arg);
+            gen_helper_mtc0_wired(cpu_env, arg);
             rn = "Wired";
             break;
         case 1:
             check_insn(env, ctx, ISA_MIPS32R2);
-            gen_helper_mtc0_srsconf0(arg);
+            gen_helper_mtc0_srsconf0(cpu_env, arg);
             rn = "SRSConf0";
             break;
         case 2:
             check_insn(env, ctx, ISA_MIPS32R2);
-            gen_helper_mtc0_srsconf1(arg);
+            gen_helper_mtc0_srsconf1(cpu_env, arg);
             rn = "SRSConf1";
             break;
         case 3:
             check_insn(env, ctx, ISA_MIPS32R2);
-            gen_helper_mtc0_srsconf2(arg);
+            gen_helper_mtc0_srsconf2(cpu_env, arg);
             rn = "SRSConf2";
             break;
         case 4:
             check_insn(env, ctx, ISA_MIPS32R2);
-            gen_helper_mtc0_srsconf3(arg);
+            gen_helper_mtc0_srsconf3(cpu_env, arg);
             rn = "SRSConf3";
             break;
         case 5:
             check_insn(env, ctx, ISA_MIPS32R2);
-            gen_helper_mtc0_srsconf4(arg);
+            gen_helper_mtc0_srsconf4(cpu_env, arg);
             rn = "SRSConf4";
             break;
         default:
@@ -5121,7 +6047,7 @@ static void gen_dmtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
         switch (sel) {
         case 0:
             check_insn(env, ctx, ISA_MIPS32R2);
-            gen_helper_mtc0_hwrena(arg);
+            gen_helper_mtc0_hwrena(cpu_env, arg);
             rn = "HWREna";
             break;
         default:
@@ -5135,7 +6061,7 @@ static void gen_dmtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
     case 9:
         switch (sel) {
         case 0:
-            gen_helper_mtc0_count(arg);
+            gen_helper_mtc0_count(cpu_env, arg);
             rn = "Count";
             break;
         /* 6,7 are implementation dependent */
@@ -5148,7 +6074,7 @@ static void gen_dmtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
     case 10:
         switch (sel) {
         case 0:
-            gen_helper_mtc0_entryhi(arg);
+            gen_helper_mtc0_entryhi(cpu_env, arg);
             rn = "EntryHi";
             break;
         default:
@@ -5158,7 +6084,7 @@ static void gen_dmtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
     case 11:
         switch (sel) {
         case 0:
-            gen_helper_mtc0_compare(arg);
+            gen_helper_mtc0_compare(cpu_env, arg);
             rn = "Compare";
             break;
         /* 6,7 are implementation dependent */
@@ -5172,7 +6098,7 @@ static void gen_dmtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
         switch (sel) {
         case 0:
             save_cpu_state(ctx, 1);
-            gen_helper_mtc0_status(arg);
+            gen_helper_mtc0_status(cpu_env, arg);
             /* BS_STOP isn't good enough here, hflags may have changed. */
             gen_save_pc(ctx->pc + 4);
             ctx->bstate = BS_EXCP;
@@ -5180,14 +6106,14 @@ static void gen_dmtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
             break;
         case 1:
             check_insn(env, ctx, ISA_MIPS32R2);
-            gen_helper_mtc0_intctl(arg);
+            gen_helper_mtc0_intctl(cpu_env, arg);
             /* Stop translation as we may have switched the execution mode */
             ctx->bstate = BS_STOP;
             rn = "IntCtl";
             break;
         case 2:
             check_insn(env, ctx, ISA_MIPS32R2);
-            gen_helper_mtc0_srsctl(arg);
+            gen_helper_mtc0_srsctl(cpu_env, arg);
             /* Stop translation as we may have switched the execution mode */
             ctx->bstate = BS_STOP;
             rn = "SRSCtl";
@@ -5212,7 +6138,7 @@ static void gen_dmtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
             if (use_icount) {
                 gen_io_start();
             }
-            gen_helper_mtc0_cause(arg);
+            gen_helper_mtc0_cause(cpu_env, arg);
             if (use_icount) {
                 gen_io_end();
             }
@@ -5242,7 +6168,7 @@ static void gen_dmtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
             break;
         case 1:
             check_insn(env, ctx, ISA_MIPS32R2);
-            gen_helper_mtc0_ebase(arg);
+            gen_helper_mtc0_ebase(cpu_env, arg);
             rn = "EBase";
             break;
         default:
@@ -5252,7 +6178,7 @@ static void gen_dmtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
     case 16:
         switch (sel) {
         case 0:
-            gen_helper_mtc0_config0(arg);
+            gen_helper_mtc0_config0(cpu_env, arg);
             rn = "Config";
             /* Stop translation as we may have switched the execution mode */
             ctx->bstate = BS_STOP;
@@ -5262,7 +6188,7 @@ static void gen_dmtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
             rn = "Config1";
             break;
         case 2:
-            gen_helper_mtc0_config2(arg);
+            gen_helper_mtc0_config2(cpu_env, arg);
             rn = "Config2";
             /* Stop translation as we may have switched the execution mode */
             ctx->bstate = BS_STOP;
@@ -5280,7 +6206,7 @@ static void gen_dmtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
     case 17:
         switch (sel) {
         case 0:
-            gen_helper_mtc0_lladdr(arg);
+            gen_helper_mtc0_lladdr(cpu_env, arg);
             rn = "LLAddr";
             break;
         default:
@@ -5290,7 +6216,7 @@ static void gen_dmtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
     case 18:
         switch (sel) {
         case 0 ... 7:
-            gen_helper_1i(mtc0_watchlo, arg, sel);
+            gen_helper_0e1i(mtc0_watchlo, arg, sel);
             rn = "WatchLo";
             break;
         default:
@@ -5300,7 +6226,7 @@ static void gen_dmtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
     case 19:
         switch (sel) {
         case 0 ... 7:
-            gen_helper_1i(mtc0_watchhi, arg, sel);
+            gen_helper_0e1i(mtc0_watchhi, arg, sel);
             rn = "WatchHi";
             break;
         default:
@@ -5311,7 +6237,7 @@ static void gen_dmtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
         switch (sel) {
         case 0:
             check_insn(env, ctx, ISA_MIPS3);
-            gen_helper_mtc0_xcontext(arg);
+            gen_helper_mtc0_xcontext(cpu_env, arg);
             rn = "XContext";
             break;
         default:
@@ -5322,7 +6248,7 @@ static void gen_dmtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
        /* Officially reserved, but sel 0 is used for R1x000 framemask */
         switch (sel) {
         case 0:
-            gen_helper_mtc0_framemask(arg);
+            gen_helper_mtc0_framemask(cpu_env, arg);
             rn = "Framemask";
             break;
         default:
@@ -5336,32 +6262,32 @@ static void gen_dmtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
     case 23:
         switch (sel) {
         case 0:
-            gen_helper_mtc0_debug(arg); /* EJTAG support */
+            gen_helper_mtc0_debug(cpu_env, arg); /* EJTAG support */
             /* BS_STOP isn't good enough here, hflags may have changed. */
             gen_save_pc(ctx->pc + 4);
             ctx->bstate = BS_EXCP;
             rn = "Debug";
             break;
         case 1:
-//            gen_helper_mtc0_tracecontrol(arg); /* PDtrace support */
+//            gen_helper_mtc0_tracecontrol(cpu_env, arg); /* PDtrace support */
             /* Stop translation as we may have switched the execution mode */
             ctx->bstate = BS_STOP;
             rn = "TraceControl";
 //            break;
         case 2:
-//            gen_helper_mtc0_tracecontrol2(arg); /* PDtrace support */
+//            gen_helper_mtc0_tracecontrol2(cpu_env, arg); /* PDtrace support */
             /* Stop translation as we may have switched the execution mode */
             ctx->bstate = BS_STOP;
             rn = "TraceControl2";
 //            break;
         case 3:
-//            gen_helper_mtc0_usertracedata(arg); /* PDtrace support */
+//            gen_helper_mtc0_usertracedata(cpu_env, arg); /* PDtrace support */
             /* Stop translation as we may have switched the execution mode */
             ctx->bstate = BS_STOP;
             rn = "UserTraceData";
 //            break;
         case 4:
-//            gen_helper_mtc0_tracebpc(arg); /* PDtrace support */
+//            gen_helper_mtc0_tracebpc(cpu_env, arg); /* PDtrace support */
             /* Stop translation as we may have switched the execution mode */
             ctx->bstate = BS_STOP;
             rn = "TraceBPC";
@@ -5384,35 +6310,35 @@ static void gen_dmtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
     case 25:
         switch (sel) {
         case 0:
-            gen_helper_mtc0_performance0(arg);
+            gen_helper_mtc0_performance0(cpu_env, arg);
             rn = "Performance0";
             break;
         case 1:
-//            gen_helper_mtc0_performance1(arg);
+//            gen_helper_mtc0_performance1(cpu_env, arg);
             rn = "Performance1";
 //            break;
         case 2:
-//            gen_helper_mtc0_performance2(arg);
+//            gen_helper_mtc0_performance2(cpu_env, arg);
             rn = "Performance2";
 //            break;
         case 3:
-//            gen_helper_mtc0_performance3(arg);
+//            gen_helper_mtc0_performance3(cpu_env, arg);
             rn = "Performance3";
 //            break;
         case 4:
-//            gen_helper_mtc0_performance4(arg);
+//            gen_helper_mtc0_performance4(cpu_env, arg);
             rn = "Performance4";
 //            break;
         case 5:
-//            gen_helper_mtc0_performance5(arg);
+//            gen_helper_mtc0_performance5(cpu_env, arg);
             rn = "Performance5";
 //            break;
         case 6:
-//            gen_helper_mtc0_performance6(arg);
+//            gen_helper_mtc0_performance6(cpu_env, arg);
             rn = "Performance6";
 //            break;
         case 7:
-//            gen_helper_mtc0_performance7(arg);
+//            gen_helper_mtc0_performance7(cpu_env, arg);
             rn = "Performance7";
 //            break;
         default:
@@ -5439,14 +6365,14 @@ static void gen_dmtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
         case 2:
         case 4:
         case 6:
-            gen_helper_mtc0_taglo(arg);
+            gen_helper_mtc0_taglo(cpu_env, arg);
             rn = "TagLo";
             break;
         case 1:
         case 3:
         case 5:
         case 7:
-            gen_helper_mtc0_datalo(arg);
+            gen_helper_mtc0_datalo(cpu_env, arg);
             rn = "DataLo";
             break;
         default:
@@ -5459,14 +6385,14 @@ static void gen_dmtc0 (CPUMIPSState *env, DisasContext *ctx, TCGv arg, int reg,
         case 2:
         case 4:
         case 6:
-            gen_helper_mtc0_taghi(arg);
+            gen_helper_mtc0_taghi(cpu_env, arg);
             rn = "TagHi";
             break;
         case 1:
         case 3:
         case 5:
         case 7:
-            gen_helper_mtc0_datahi(arg);
+            gen_helper_mtc0_datahi(cpu_env, arg);
             rn = "DataHi";
             break;
         default:
@@ -5533,10 +6459,10 @@ static void gen_mftr(CPUMIPSState *env, DisasContext *ctx, int rt, int rd,
         case 1:
             switch (sel) {
             case 1:
-                gen_helper_mftc0_vpecontrol(t0);
+                gen_helper_mftc0_vpecontrol(t0, cpu_env);
                 break;
             case 2:
-                gen_helper_mftc0_vpeconf0(t0);
+                gen_helper_mftc0_vpeconf0(t0, cpu_env);
                 break;
             default:
                 goto die;
@@ -5546,25 +6472,25 @@ static void gen_mftr(CPUMIPSState *env, DisasContext *ctx, int rt, int rd,
         case 2:
             switch (sel) {
             case 1:
-                gen_helper_mftc0_tcstatus(t0);
+                gen_helper_mftc0_tcstatus(t0, cpu_env);
                 break;
             case 2:
-                gen_helper_mftc0_tcbind(t0);
+                gen_helper_mftc0_tcbind(t0, cpu_env);
                 break;
             case 3:
-                gen_helper_mftc0_tcrestart(t0);
+                gen_helper_mftc0_tcrestart(t0, cpu_env);
                 break;
             case 4:
-                gen_helper_mftc0_tchalt(t0);
+                gen_helper_mftc0_tchalt(t0, cpu_env);
                 break;
             case 5:
-                gen_helper_mftc0_tccontext(t0);
+                gen_helper_mftc0_tccontext(t0, cpu_env);
                 break;
             case 6:
-                gen_helper_mftc0_tcschedule(t0);
+                gen_helper_mftc0_tcschedule(t0, cpu_env);
                 break;
             case 7:
-                gen_helper_mftc0_tcschefback(t0);
+                gen_helper_mftc0_tcschefback(t0, cpu_env);
                 break;
             default:
                 gen_mfc0(env, ctx, t0, rt, sel);
@@ -5574,7 +6500,7 @@ static void gen_mftr(CPUMIPSState *env, DisasContext *ctx, int rt, int rd,
         case 10:
             switch (sel) {
             case 0:
-                gen_helper_mftc0_entryhi(t0);
+                gen_helper_mftc0_entryhi(t0, cpu_env);
                 break;
             default:
                 gen_mfc0(env, ctx, t0, rt, sel);
@@ -5583,7 +6509,7 @@ static void gen_mftr(CPUMIPSState *env, DisasContext *ctx, int rt, int rd,
         case 12:
             switch (sel) {
             case 0:
-                gen_helper_mftc0_status(t0);
+                gen_helper_mftc0_status(t0, cpu_env);
                 break;
             default:
                 gen_mfc0(env, ctx, t0, rt, sel);
@@ -5592,7 +6518,7 @@ static void gen_mftr(CPUMIPSState *env, DisasContext *ctx, int rt, int rd,
         case 13:
             switch (sel) {
             case 0:
-                gen_helper_mftc0_cause(t0);
+                gen_helper_mftc0_cause(t0, cpu_env);
                 break;
             default:
                 goto die;
@@ -5602,7 +6528,7 @@ static void gen_mftr(CPUMIPSState *env, DisasContext *ctx, int rt, int rd,
         case 14:
             switch (sel) {
             case 0:
-                gen_helper_mftc0_epc(t0);
+                gen_helper_mftc0_epc(t0, cpu_env);
                 break;
             default:
                 goto die;
@@ -5612,7 +6538,7 @@ static void gen_mftr(CPUMIPSState *env, DisasContext *ctx, int rt, int rd,
         case 15:
             switch (sel) {
             case 1:
-                gen_helper_mftc0_ebase(t0);
+                gen_helper_mftc0_ebase(t0, cpu_env);
                 break;
             default:
                 goto die;
@@ -5622,7 +6548,7 @@ static void gen_mftr(CPUMIPSState *env, DisasContext *ctx, int rt, int rd,
         case 16:
             switch (sel) {
             case 0 ... 7:
-                gen_helper_mftc0_configx(t0, tcg_const_tl(sel));
+                gen_helper_mftc0_configx(t0, cpu_env, tcg_const_tl(sel));
                 break;
             default:
                 goto die;
@@ -5632,7 +6558,7 @@ static void gen_mftr(CPUMIPSState *env, DisasContext *ctx, int rt, int rd,
         case 23:
             switch (sel) {
             case 0:
-                gen_helper_mftc0_debug(t0);
+                gen_helper_mftc0_debug(t0, cpu_env);
                 break;
             default:
                 gen_mfc0(env, ctx, t0, rt, sel);
@@ -5645,49 +6571,49 @@ static void gen_mftr(CPUMIPSState *env, DisasContext *ctx, int rt, int rd,
     } else switch (sel) {
     /* GPR registers. */
     case 0:
-        gen_helper_1i(mftgpr, t0, rt);
+        gen_helper_1e0i(mftgpr, t0, rt);
         break;
     /* Auxiliary CPU registers */
     case 1:
         switch (rt) {
         case 0:
-            gen_helper_1i(mftlo, t0, 0);
+            gen_helper_1e0i(mftlo, t0, 0);
             break;
         case 1:
-            gen_helper_1i(mfthi, t0, 0);
+            gen_helper_1e0i(mfthi, t0, 0);
             break;
         case 2:
-            gen_helper_1i(mftacx, t0, 0);
+            gen_helper_1e0i(mftacx, t0, 0);
             break;
         case 4:
-            gen_helper_1i(mftlo, t0, 1);
+            gen_helper_1e0i(mftlo, t0, 1);
             break;
         case 5:
-            gen_helper_1i(mfthi, t0, 1);
+            gen_helper_1e0i(mfthi, t0, 1);
             break;
         case 6:
-            gen_helper_1i(mftacx, t0, 1);
+            gen_helper_1e0i(mftacx, t0, 1);
             break;
         case 8:
-            gen_helper_1i(mftlo, t0, 2);
+            gen_helper_1e0i(mftlo, t0, 2);
             break;
         case 9:
-            gen_helper_1i(mfthi, t0, 2);
+            gen_helper_1e0i(mfthi, t0, 2);
             break;
         case 10:
-            gen_helper_1i(mftacx, t0, 2);
+            gen_helper_1e0i(mftacx, t0, 2);
             break;
         case 12:
-            gen_helper_1i(mftlo, t0, 3);
+            gen_helper_1e0i(mftlo, t0, 3);
             break;
         case 13:
-            gen_helper_1i(mfthi, t0, 3);
+            gen_helper_1e0i(mfthi, t0, 3);
             break;
         case 14:
-            gen_helper_1i(mftacx, t0, 3);
+            gen_helper_1e0i(mftacx, t0, 3);
             break;
         case 16:
-            gen_helper_mftdsp(t0);
+            gen_helper_mftdsp(t0, cpu_env);
             break;
         default:
             goto die;
@@ -5712,7 +6638,7 @@ static void gen_mftr(CPUMIPSState *env, DisasContext *ctx, int rt, int rd,
         break;
     case 3:
         /* XXX: For now we support only a single FPU context. */
-        gen_helper_1i(cfc1, t0, rt);
+        gen_helper_1e0i(cfc1, t0, rt);
         break;
     /* COP2: Not implemented. */
     case 4:
@@ -5751,10 +6677,10 @@ static void gen_mttr(CPUMIPSState *env, DisasContext *ctx, int rd, int rt,
         case 1:
             switch (sel) {
             case 1:
-                gen_helper_mttc0_vpecontrol(t0);
+                gen_helper_mttc0_vpecontrol(cpu_env, t0);
                 break;
             case 2:
-                gen_helper_mttc0_vpeconf0(t0);
+                gen_helper_mttc0_vpeconf0(cpu_env, t0);
                 break;
             default:
                 goto die;
@@ -5764,25 +6690,25 @@ static void gen_mttr(CPUMIPSState *env, DisasContext *ctx, int rd, int rt,
         case 2:
             switch (sel) {
             case 1:
-                gen_helper_mttc0_tcstatus(t0);
+                gen_helper_mttc0_tcstatus(cpu_env, t0);
                 break;
             case 2:
-                gen_helper_mttc0_tcbind(t0);
+                gen_helper_mttc0_tcbind(cpu_env, t0);
                 break;
             case 3:
-                gen_helper_mttc0_tcrestart(t0);
+                gen_helper_mttc0_tcrestart(cpu_env, t0);
                 break;
             case 4:
-                gen_helper_mttc0_tchalt(t0);
+                gen_helper_mttc0_tchalt(cpu_env, t0);
                 break;
             case 5:
-                gen_helper_mttc0_tccontext(t0);
+                gen_helper_mttc0_tccontext(cpu_env, t0);
                 break;
             case 6:
-                gen_helper_mttc0_tcschedule(t0);
+                gen_helper_mttc0_tcschedule(cpu_env, t0);
                 break;
             case 7:
-                gen_helper_mttc0_tcschefback(t0);
+                gen_helper_mttc0_tcschefback(cpu_env, t0);
                 break;
             default:
                 gen_mtc0(env, ctx, t0, rd, sel);
@@ -5792,7 +6718,7 @@ static void gen_mttr(CPUMIPSState *env, DisasContext *ctx, int rd, int rt,
         case 10:
             switch (sel) {
             case 0:
-                gen_helper_mttc0_entryhi(t0);
+                gen_helper_mttc0_entryhi(cpu_env, t0);
                 break;
             default:
                 gen_mtc0(env, ctx, t0, rd, sel);
@@ -5801,7 +6727,7 @@ static void gen_mttr(CPUMIPSState *env, DisasContext *ctx, int rd, int rt,
         case 12:
             switch (sel) {
             case 0:
-                gen_helper_mttc0_status(t0);
+                gen_helper_mttc0_status(cpu_env, t0);
                 break;
             default:
                 gen_mtc0(env, ctx, t0, rd, sel);
@@ -5810,7 +6736,7 @@ static void gen_mttr(CPUMIPSState *env, DisasContext *ctx, int rd, int rt,
         case 13:
             switch (sel) {
             case 0:
-                gen_helper_mttc0_cause(t0);
+                gen_helper_mttc0_cause(cpu_env, t0);
                 break;
             default:
                 goto die;
@@ -5820,7 +6746,7 @@ static void gen_mttr(CPUMIPSState *env, DisasContext *ctx, int rd, int rt,
         case 15:
             switch (sel) {
             case 1:
-                gen_helper_mttc0_ebase(t0);
+                gen_helper_mttc0_ebase(cpu_env, t0);
                 break;
             default:
                 goto die;
@@ -5830,7 +6756,7 @@ static void gen_mttr(CPUMIPSState *env, DisasContext *ctx, int rd, int rt,
         case 23:
             switch (sel) {
             case 0:
-                gen_helper_mttc0_debug(t0);
+                gen_helper_mttc0_debug(cpu_env, t0);
                 break;
             default:
                 gen_mtc0(env, ctx, t0, rd, sel);
@@ -5843,49 +6769,49 @@ static void gen_mttr(CPUMIPSState *env, DisasContext *ctx, int rd, int rt,
     } else switch (sel) {
     /* GPR registers. */
     case 0:
-        gen_helper_1i(mttgpr, t0, rd);
+        gen_helper_0e1i(mttgpr, t0, rd);
         break;
     /* Auxiliary CPU registers */
     case 1:
         switch (rd) {
         case 0:
-            gen_helper_1i(mttlo, t0, 0);
+            gen_helper_0e1i(mttlo, t0, 0);
             break;
         case 1:
-            gen_helper_1i(mtthi, t0, 0);
+            gen_helper_0e1i(mtthi, t0, 0);
             break;
         case 2:
-            gen_helper_1i(mttacx, t0, 0);
+            gen_helper_0e1i(mttacx, t0, 0);
             break;
         case 4:
-            gen_helper_1i(mttlo, t0, 1);
+            gen_helper_0e1i(mttlo, t0, 1);
             break;
         case 5:
-            gen_helper_1i(mtthi, t0, 1);
+            gen_helper_0e1i(mtthi, t0, 1);
             break;
         case 6:
-            gen_helper_1i(mttacx, t0, 1);
+            gen_helper_0e1i(mttacx, t0, 1);
             break;
         case 8:
-            gen_helper_1i(mttlo, t0, 2);
+            gen_helper_0e1i(mttlo, t0, 2);
             break;
         case 9:
-            gen_helper_1i(mtthi, t0, 2);
+            gen_helper_0e1i(mtthi, t0, 2);
             break;
         case 10:
-            gen_helper_1i(mttacx, t0, 2);
+            gen_helper_0e1i(mttacx, t0, 2);
             break;
         case 12:
-            gen_helper_1i(mttlo, t0, 3);
+            gen_helper_0e1i(mttlo, t0, 3);
             break;
         case 13:
-            gen_helper_1i(mtthi, t0, 3);
+            gen_helper_0e1i(mtthi, t0, 3);
             break;
         case 14:
-            gen_helper_1i(mttacx, t0, 3);
+            gen_helper_0e1i(mttacx, t0, 3);
             break;
         case 16:
-            gen_helper_mttdsp(t0);
+            gen_helper_mttdsp(cpu_env, t0);
             break;
         default:
             goto die;
@@ -5910,7 +6836,7 @@ static void gen_mttr(CPUMIPSState *env, DisasContext *ctx, int rd, int rt,
         break;
     case 3:
         /* XXX: For now we support only a single FPU context. */
-        gen_helper_1i(ctc1, t0, rd);
+        gen_helper_0e1i(ctc1, t0, rd);
         break;
     /* COP2: Not implemented. */
     case 4:
@@ -5995,30 +6921,30 @@ static void gen_cp0 (CPUMIPSState *env, DisasContext *ctx, uint32_t opc, int rt,
         opn = "tlbwi";
         if (!env->tlb->helper_tlbwi)
             goto die;
-        gen_helper_tlbwi();
+        gen_helper_tlbwi(cpu_env);
         break;
     case OPC_TLBWR:
         opn = "tlbwr";
         if (!env->tlb->helper_tlbwr)
             goto die;
-        gen_helper_tlbwr();
+        gen_helper_tlbwr(cpu_env);
         break;
     case OPC_TLBP:
         opn = "tlbp";
         if (!env->tlb->helper_tlbp)
             goto die;
-        gen_helper_tlbp();
+        gen_helper_tlbp(cpu_env);
         break;
     case OPC_TLBR:
         opn = "tlbr";
         if (!env->tlb->helper_tlbr)
             goto die;
-        gen_helper_tlbr();
+        gen_helper_tlbr(cpu_env);
         break;
     case OPC_ERET:
         opn = "eret";
         check_insn(env, ctx, ISA_MIPS2);
-        gen_helper_eret();
+        gen_helper_eret(cpu_env);
         ctx->bstate = BS_EXCP;
         break;
     case OPC_DERET:
@@ -6028,7 +6954,7 @@ static void gen_cp0 (CPUMIPSState *env, DisasContext *ctx, uint32_t opc, int rt,
             MIPS_INVAL(opn);
             generate_exception(ctx, EXCP_RI);
         } else {
-            gen_helper_deret();
+            gen_helper_deret(cpu_env);
             ctx->bstate = BS_EXCP;
         }
         break;
@@ -6039,7 +6965,7 @@ static void gen_cp0 (CPUMIPSState *env, DisasContext *ctx, uint32_t opc, int rt,
         ctx->pc += 4;
         save_cpu_state(ctx, 1);
         ctx->pc -= 4;
-        gen_helper_wait();
+        gen_helper_wait(cpu_env);
         ctx->bstate = BS_EXCP;
         break;
     default:
@@ -6340,13 +7266,13 @@ static void gen_cp1 (DisasContext *ctx, uint32_t opc, int rt, int fs)
         opn = "mtc1";
         break;
     case OPC_CFC1:
-        gen_helper_1i(cfc1, t0, fs);
+        gen_helper_1e0i(cfc1, t0, fs);
         gen_store_gpr(t0, rt);
         opn = "cfc1";
         break;
     case OPC_CTC1:
         gen_load_gpr(t0, rt);
-        gen_helper_1i(ctc1, t0, fs);
+        gen_helper_0e1i(ctc1, t0, fs);
         opn = "ctc1";
         break;
 #if defined(TARGET_MIPS64)
@@ -6543,7 +7469,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
 
             gen_load_fpr32(fp0, fs);
             gen_load_fpr32(fp1, ft);
-            gen_helper_float_add_s(fp0, fp0, fp1);
+            gen_helper_float_add_s(fp0, cpu_env, fp0, fp1);
             tcg_temp_free_i32(fp1);
             gen_store_fpr32(fp0, fd);
             tcg_temp_free_i32(fp0);
@@ -6558,7 +7484,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
 
             gen_load_fpr32(fp0, fs);
             gen_load_fpr32(fp1, ft);
-            gen_helper_float_sub_s(fp0, fp0, fp1);
+            gen_helper_float_sub_s(fp0, cpu_env, fp0, fp1);
             tcg_temp_free_i32(fp1);
             gen_store_fpr32(fp0, fd);
             tcg_temp_free_i32(fp0);
@@ -6573,7 +7499,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
 
             gen_load_fpr32(fp0, fs);
             gen_load_fpr32(fp1, ft);
-            gen_helper_float_mul_s(fp0, fp0, fp1);
+            gen_helper_float_mul_s(fp0, cpu_env, fp0, fp1);
             tcg_temp_free_i32(fp1);
             gen_store_fpr32(fp0, fd);
             tcg_temp_free_i32(fp0);
@@ -6588,7 +7514,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
 
             gen_load_fpr32(fp0, fs);
             gen_load_fpr32(fp1, ft);
-            gen_helper_float_div_s(fp0, fp0, fp1);
+            gen_helper_float_div_s(fp0, cpu_env, fp0, fp1);
             tcg_temp_free_i32(fp1);
             gen_store_fpr32(fp0, fd);
             tcg_temp_free_i32(fp0);
@@ -6601,7 +7527,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i32 fp0 = tcg_temp_new_i32();
 
             gen_load_fpr32(fp0, fs);
-            gen_helper_float_sqrt_s(fp0, fp0);
+            gen_helper_float_sqrt_s(fp0, cpu_env, fp0);
             gen_store_fpr32(fp0, fd);
             tcg_temp_free_i32(fp0);
         }
@@ -6646,7 +7572,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i64 fp64 = tcg_temp_new_i64();
 
             gen_load_fpr32(fp32, fs);
-            gen_helper_float_roundl_s(fp64, fp32);
+            gen_helper_float_roundl_s(fp64, cpu_env, fp32);
             tcg_temp_free_i32(fp32);
             gen_store_fpr64(ctx, fp64, fd);
             tcg_temp_free_i64(fp64);
@@ -6660,7 +7586,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i64 fp64 = tcg_temp_new_i64();
 
             gen_load_fpr32(fp32, fs);
-            gen_helper_float_truncl_s(fp64, fp32);
+            gen_helper_float_truncl_s(fp64, cpu_env, fp32);
             tcg_temp_free_i32(fp32);
             gen_store_fpr64(ctx, fp64, fd);
             tcg_temp_free_i64(fp64);
@@ -6674,7 +7600,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i64 fp64 = tcg_temp_new_i64();
 
             gen_load_fpr32(fp32, fs);
-            gen_helper_float_ceill_s(fp64, fp32);
+            gen_helper_float_ceill_s(fp64, cpu_env, fp32);
             tcg_temp_free_i32(fp32);
             gen_store_fpr64(ctx, fp64, fd);
             tcg_temp_free_i64(fp64);
@@ -6688,7 +7614,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i64 fp64 = tcg_temp_new_i64();
 
             gen_load_fpr32(fp32, fs);
-            gen_helper_float_floorl_s(fp64, fp32);
+            gen_helper_float_floorl_s(fp64, cpu_env, fp32);
             tcg_temp_free_i32(fp32);
             gen_store_fpr64(ctx, fp64, fd);
             tcg_temp_free_i64(fp64);
@@ -6700,7 +7626,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i32 fp0 = tcg_temp_new_i32();
 
             gen_load_fpr32(fp0, fs);
-            gen_helper_float_roundw_s(fp0, fp0);
+            gen_helper_float_roundw_s(fp0, cpu_env, fp0);
             gen_store_fpr32(fp0, fd);
             tcg_temp_free_i32(fp0);
         }
@@ -6711,7 +7637,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i32 fp0 = tcg_temp_new_i32();
 
             gen_load_fpr32(fp0, fs);
-            gen_helper_float_truncw_s(fp0, fp0);
+            gen_helper_float_truncw_s(fp0, cpu_env, fp0);
             gen_store_fpr32(fp0, fd);
             tcg_temp_free_i32(fp0);
         }
@@ -6722,7 +7648,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i32 fp0 = tcg_temp_new_i32();
 
             gen_load_fpr32(fp0, fs);
-            gen_helper_float_ceilw_s(fp0, fp0);
+            gen_helper_float_ceilw_s(fp0, cpu_env, fp0);
             gen_store_fpr32(fp0, fd);
             tcg_temp_free_i32(fp0);
         }
@@ -6733,7 +7659,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i32 fp0 = tcg_temp_new_i32();
 
             gen_load_fpr32(fp0, fs);
-            gen_helper_float_floorw_s(fp0, fp0);
+            gen_helper_float_floorw_s(fp0, cpu_env, fp0);
             gen_store_fpr32(fp0, fd);
             tcg_temp_free_i32(fp0);
         }
@@ -6781,7 +7707,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i32 fp0 = tcg_temp_new_i32();
 
             gen_load_fpr32(fp0, fs);
-            gen_helper_float_recip_s(fp0, fp0);
+            gen_helper_float_recip_s(fp0, cpu_env, fp0);
             gen_store_fpr32(fp0, fd);
             tcg_temp_free_i32(fp0);
         }
@@ -6793,7 +7719,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i32 fp0 = tcg_temp_new_i32();
 
             gen_load_fpr32(fp0, fs);
-            gen_helper_float_rsqrt_s(fp0, fp0);
+            gen_helper_float_rsqrt_s(fp0, cpu_env, fp0);
             gen_store_fpr32(fp0, fd);
             tcg_temp_free_i32(fp0);
         }
@@ -6807,7 +7733,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
 
             gen_load_fpr32(fp0, fs);
             gen_load_fpr32(fp1, ft);
-            gen_helper_float_recip2_s(fp0, fp0, fp1);
+            gen_helper_float_recip2_s(fp0, cpu_env, fp0, fp1);
             tcg_temp_free_i32(fp1);
             gen_store_fpr32(fp0, fd);
             tcg_temp_free_i32(fp0);
@@ -6820,7 +7746,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i32 fp0 = tcg_temp_new_i32();
 
             gen_load_fpr32(fp0, fs);
-            gen_helper_float_recip1_s(fp0, fp0);
+            gen_helper_float_recip1_s(fp0, cpu_env, fp0);
             gen_store_fpr32(fp0, fd);
             tcg_temp_free_i32(fp0);
         }
@@ -6832,7 +7758,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i32 fp0 = tcg_temp_new_i32();
 
             gen_load_fpr32(fp0, fs);
-            gen_helper_float_rsqrt1_s(fp0, fp0);
+            gen_helper_float_rsqrt1_s(fp0, cpu_env, fp0);
             gen_store_fpr32(fp0, fd);
             tcg_temp_free_i32(fp0);
         }
@@ -6846,7 +7772,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
 
             gen_load_fpr32(fp0, fs);
             gen_load_fpr32(fp1, ft);
-            gen_helper_float_rsqrt2_s(fp0, fp0, fp1);
+            gen_helper_float_rsqrt2_s(fp0, cpu_env, fp0, fp1);
             tcg_temp_free_i32(fp1);
             gen_store_fpr32(fp0, fd);
             tcg_temp_free_i32(fp0);
@@ -6860,7 +7786,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i64 fp64 = tcg_temp_new_i64();
 
             gen_load_fpr32(fp32, fs);
-            gen_helper_float_cvtd_s(fp64, fp32);
+            gen_helper_float_cvtd_s(fp64, cpu_env, fp32);
             tcg_temp_free_i32(fp32);
             gen_store_fpr64(ctx, fp64, fd);
             tcg_temp_free_i64(fp64);
@@ -6872,7 +7798,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i32 fp0 = tcg_temp_new_i32();
 
             gen_load_fpr32(fp0, fs);
-            gen_helper_float_cvtw_s(fp0, fp0);
+            gen_helper_float_cvtw_s(fp0, cpu_env, fp0);
             gen_store_fpr32(fp0, fd);
             tcg_temp_free_i32(fp0);
         }
@@ -6885,7 +7811,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i64 fp64 = tcg_temp_new_i64();
 
             gen_load_fpr32(fp32, fs);
-            gen_helper_float_cvtl_s(fp64, fp32);
+            gen_helper_float_cvtl_s(fp64, cpu_env, fp32);
             tcg_temp_free_i32(fp32);
             gen_store_fpr64(ctx, fp64, fd);
             tcg_temp_free_i64(fp64);
@@ -6941,7 +7867,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
 
             gen_load_fpr64(ctx, fp0, fs);
             gen_load_fpr64(ctx, fp1, ft);
-            gen_helper_float_add_d(fp0, fp0, fp1);
+            gen_helper_float_add_d(fp0, cpu_env, fp0, fp1);
             tcg_temp_free_i64(fp1);
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
@@ -6957,7 +7883,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
 
             gen_load_fpr64(ctx, fp0, fs);
             gen_load_fpr64(ctx, fp1, ft);
-            gen_helper_float_sub_d(fp0, fp0, fp1);
+            gen_helper_float_sub_d(fp0, cpu_env, fp0, fp1);
             tcg_temp_free_i64(fp1);
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
@@ -6973,7 +7899,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
 
             gen_load_fpr64(ctx, fp0, fs);
             gen_load_fpr64(ctx, fp1, ft);
-            gen_helper_float_mul_d(fp0, fp0, fp1);
+            gen_helper_float_mul_d(fp0, cpu_env, fp0, fp1);
             tcg_temp_free_i64(fp1);
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
@@ -6989,7 +7915,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
 
             gen_load_fpr64(ctx, fp0, fs);
             gen_load_fpr64(ctx, fp1, ft);
-            gen_helper_float_div_d(fp0, fp0, fp1);
+            gen_helper_float_div_d(fp0, cpu_env, fp0, fp1);
             tcg_temp_free_i64(fp1);
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
@@ -7003,7 +7929,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i64 fp0 = tcg_temp_new_i64();
 
             gen_load_fpr64(ctx, fp0, fs);
-            gen_helper_float_sqrt_d(fp0, fp0);
+            gen_helper_float_sqrt_d(fp0, cpu_env, fp0);
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
@@ -7050,7 +7976,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i64 fp0 = tcg_temp_new_i64();
 
             gen_load_fpr64(ctx, fp0, fs);
-            gen_helper_float_roundl_d(fp0, fp0);
+            gen_helper_float_roundl_d(fp0, cpu_env, fp0);
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
@@ -7062,7 +7988,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i64 fp0 = tcg_temp_new_i64();
 
             gen_load_fpr64(ctx, fp0, fs);
-            gen_helper_float_truncl_d(fp0, fp0);
+            gen_helper_float_truncl_d(fp0, cpu_env, fp0);
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
@@ -7074,7 +8000,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i64 fp0 = tcg_temp_new_i64();
 
             gen_load_fpr64(ctx, fp0, fs);
-            gen_helper_float_ceill_d(fp0, fp0);
+            gen_helper_float_ceill_d(fp0, cpu_env, fp0);
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
@@ -7086,7 +8012,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i64 fp0 = tcg_temp_new_i64();
 
             gen_load_fpr64(ctx, fp0, fs);
-            gen_helper_float_floorl_d(fp0, fp0);
+            gen_helper_float_floorl_d(fp0, cpu_env, fp0);
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
@@ -7099,7 +8025,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i64 fp64 = tcg_temp_new_i64();
 
             gen_load_fpr64(ctx, fp64, fs);
-            gen_helper_float_roundw_d(fp32, fp64);
+            gen_helper_float_roundw_d(fp32, cpu_env, fp64);
             tcg_temp_free_i64(fp64);
             gen_store_fpr32(fp32, fd);
             tcg_temp_free_i32(fp32);
@@ -7113,7 +8039,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i64 fp64 = tcg_temp_new_i64();
 
             gen_load_fpr64(ctx, fp64, fs);
-            gen_helper_float_truncw_d(fp32, fp64);
+            gen_helper_float_truncw_d(fp32, cpu_env, fp64);
             tcg_temp_free_i64(fp64);
             gen_store_fpr32(fp32, fd);
             tcg_temp_free_i32(fp32);
@@ -7127,7 +8053,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i64 fp64 = tcg_temp_new_i64();
 
             gen_load_fpr64(ctx, fp64, fs);
-            gen_helper_float_ceilw_d(fp32, fp64);
+            gen_helper_float_ceilw_d(fp32, cpu_env, fp64);
             tcg_temp_free_i64(fp64);
             gen_store_fpr32(fp32, fd);
             tcg_temp_free_i32(fp32);
@@ -7141,7 +8067,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i64 fp64 = tcg_temp_new_i64();
 
             gen_load_fpr64(ctx, fp64, fs);
-            gen_helper_float_floorw_d(fp32, fp64);
+            gen_helper_float_floorw_d(fp32, cpu_env, fp64);
             tcg_temp_free_i64(fp64);
             gen_store_fpr32(fp32, fd);
             tcg_temp_free_i32(fp32);
@@ -7190,7 +8116,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i64 fp0 = tcg_temp_new_i64();
 
             gen_load_fpr64(ctx, fp0, fs);
-            gen_helper_float_recip_d(fp0, fp0);
+            gen_helper_float_recip_d(fp0, cpu_env, fp0);
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
@@ -7202,7 +8128,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i64 fp0 = tcg_temp_new_i64();
 
             gen_load_fpr64(ctx, fp0, fs);
-            gen_helper_float_rsqrt_d(fp0, fp0);
+            gen_helper_float_rsqrt_d(fp0, cpu_env, fp0);
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
@@ -7216,7 +8142,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
 
             gen_load_fpr64(ctx, fp0, fs);
             gen_load_fpr64(ctx, fp1, ft);
-            gen_helper_float_recip2_d(fp0, fp0, fp1);
+            gen_helper_float_recip2_d(fp0, cpu_env, fp0, fp1);
             tcg_temp_free_i64(fp1);
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
@@ -7229,7 +8155,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i64 fp0 = tcg_temp_new_i64();
 
             gen_load_fpr64(ctx, fp0, fs);
-            gen_helper_float_recip1_d(fp0, fp0);
+            gen_helper_float_recip1_d(fp0, cpu_env, fp0);
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
@@ -7241,7 +8167,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i64 fp0 = tcg_temp_new_i64();
 
             gen_load_fpr64(ctx, fp0, fs);
-            gen_helper_float_rsqrt1_d(fp0, fp0);
+            gen_helper_float_rsqrt1_d(fp0, cpu_env, fp0);
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
@@ -7255,7 +8181,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
 
             gen_load_fpr64(ctx, fp0, fs);
             gen_load_fpr64(ctx, fp1, ft);
-            gen_helper_float_rsqrt2_d(fp0, fp0, fp1);
+            gen_helper_float_rsqrt2_d(fp0, cpu_env, fp0, fp1);
             tcg_temp_free_i64(fp1);
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
@@ -7293,7 +8219,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i64 fp64 = tcg_temp_new_i64();
 
             gen_load_fpr64(ctx, fp64, fs);
-            gen_helper_float_cvts_d(fp32, fp64);
+            gen_helper_float_cvts_d(fp32, cpu_env, fp64);
             tcg_temp_free_i64(fp64);
             gen_store_fpr32(fp32, fd);
             tcg_temp_free_i32(fp32);
@@ -7307,7 +8233,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i64 fp64 = tcg_temp_new_i64();
 
             gen_load_fpr64(ctx, fp64, fs);
-            gen_helper_float_cvtw_d(fp32, fp64);
+            gen_helper_float_cvtw_d(fp32, cpu_env, fp64);
             tcg_temp_free_i64(fp64);
             gen_store_fpr32(fp32, fd);
             tcg_temp_free_i32(fp32);
@@ -7320,7 +8246,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i64 fp0 = tcg_temp_new_i64();
 
             gen_load_fpr64(ctx, fp0, fs);
-            gen_helper_float_cvtl_d(fp0, fp0);
+            gen_helper_float_cvtl_d(fp0, cpu_env, fp0);
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
@@ -7331,7 +8257,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i32 fp0 = tcg_temp_new_i32();
 
             gen_load_fpr32(fp0, fs);
-            gen_helper_float_cvts_w(fp0, fp0);
+            gen_helper_float_cvts_w(fp0, cpu_env, fp0);
             gen_store_fpr32(fp0, fd);
             tcg_temp_free_i32(fp0);
         }
@@ -7344,7 +8270,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i64 fp64 = tcg_temp_new_i64();
 
             gen_load_fpr32(fp32, fs);
-            gen_helper_float_cvtd_w(fp64, fp32);
+            gen_helper_float_cvtd_w(fp64, cpu_env, fp32);
             tcg_temp_free_i32(fp32);
             gen_store_fpr64(ctx, fp64, fd);
             tcg_temp_free_i64(fp64);
@@ -7358,7 +8284,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i64 fp64 = tcg_temp_new_i64();
 
             gen_load_fpr64(ctx, fp64, fs);
-            gen_helper_float_cvts_l(fp32, fp64);
+            gen_helper_float_cvts_l(fp32, cpu_env, fp64);
             tcg_temp_free_i64(fp64);
             gen_store_fpr32(fp32, fd);
             tcg_temp_free_i32(fp32);
@@ -7371,7 +8297,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i64 fp0 = tcg_temp_new_i64();
 
             gen_load_fpr64(ctx, fp0, fs);
-            gen_helper_float_cvtd_l(fp0, fp0);
+            gen_helper_float_cvtd_l(fp0, cpu_env, fp0);
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
@@ -7383,7 +8309,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i64 fp0 = tcg_temp_new_i64();
 
             gen_load_fpr64(ctx, fp0, fs);
-            gen_helper_float_cvtps_pw(fp0, fp0);
+            gen_helper_float_cvtps_pw(fp0, cpu_env, fp0);
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
@@ -7397,7 +8323,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
 
             gen_load_fpr64(ctx, fp0, fs);
             gen_load_fpr64(ctx, fp1, ft);
-            gen_helper_float_add_ps(fp0, fp0, fp1);
+            gen_helper_float_add_ps(fp0, cpu_env, fp0, fp1);
             tcg_temp_free_i64(fp1);
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
@@ -7412,7 +8338,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
 
             gen_load_fpr64(ctx, fp0, fs);
             gen_load_fpr64(ctx, fp1, ft);
-            gen_helper_float_sub_ps(fp0, fp0, fp1);
+            gen_helper_float_sub_ps(fp0, cpu_env, fp0, fp1);
             tcg_temp_free_i64(fp1);
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
@@ -7427,7 +8353,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
 
             gen_load_fpr64(ctx, fp0, fs);
             gen_load_fpr64(ctx, fp1, ft);
-            gen_helper_float_mul_ps(fp0, fp0, fp1);
+            gen_helper_float_mul_ps(fp0, cpu_env, fp0, fp1);
             tcg_temp_free_i64(fp1);
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
@@ -7515,7 +8441,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
 
             gen_load_fpr64(ctx, fp0, ft);
             gen_load_fpr64(ctx, fp1, fs);
-            gen_helper_float_addr_ps(fp0, fp0, fp1);
+            gen_helper_float_addr_ps(fp0, cpu_env, fp0, fp1);
             tcg_temp_free_i64(fp1);
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
@@ -7530,7 +8456,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
 
             gen_load_fpr64(ctx, fp0, ft);
             gen_load_fpr64(ctx, fp1, fs);
-            gen_helper_float_mulr_ps(fp0, fp0, fp1);
+            gen_helper_float_mulr_ps(fp0, cpu_env, fp0, fp1);
             tcg_temp_free_i64(fp1);
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
@@ -7545,7 +8471,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
 
             gen_load_fpr64(ctx, fp0, fs);
             gen_load_fpr64(ctx, fp1, ft);
-            gen_helper_float_recip2_ps(fp0, fp0, fp1);
+            gen_helper_float_recip2_ps(fp0, cpu_env, fp0, fp1);
             tcg_temp_free_i64(fp1);
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
@@ -7558,7 +8484,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i64 fp0 = tcg_temp_new_i64();
 
             gen_load_fpr64(ctx, fp0, fs);
-            gen_helper_float_recip1_ps(fp0, fp0);
+            gen_helper_float_recip1_ps(fp0, cpu_env, fp0);
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
@@ -7570,7 +8496,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i64 fp0 = tcg_temp_new_i64();
 
             gen_load_fpr64(ctx, fp0, fs);
-            gen_helper_float_rsqrt1_ps(fp0, fp0);
+            gen_helper_float_rsqrt1_ps(fp0, cpu_env, fp0);
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
@@ -7584,7 +8510,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
 
             gen_load_fpr64(ctx, fp0, fs);
             gen_load_fpr64(ctx, fp1, ft);
-            gen_helper_float_rsqrt2_ps(fp0, fp0, fp1);
+            gen_helper_float_rsqrt2_ps(fp0, cpu_env, fp0, fp1);
             tcg_temp_free_i64(fp1);
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
@@ -7597,7 +8523,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i32 fp0 = tcg_temp_new_i32();
 
             gen_load_fpr32h(fp0, fs);
-            gen_helper_float_cvts_pu(fp0, fp0);
+            gen_helper_float_cvts_pu(fp0, cpu_env, fp0);
             gen_store_fpr32(fp0, fd);
             tcg_temp_free_i32(fp0);
         }
@@ -7609,7 +8535,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i64 fp0 = tcg_temp_new_i64();
 
             gen_load_fpr64(ctx, fp0, fs);
-            gen_helper_float_cvtpw_ps(fp0, fp0);
+            gen_helper_float_cvtpw_ps(fp0, cpu_env, fp0);
             gen_store_fpr64(ctx, fp0, fd);
             tcg_temp_free_i64(fp0);
         }
@@ -7621,7 +8547,7 @@ static void gen_farith (DisasContext *ctx, enum fopcode op1,
             TCGv_i32 fp0 = tcg_temp_new_i32();
 
             gen_load_fpr32(fp0, fs);
-            gen_helper_float_cvts_pl(fp0, fp0);
+            gen_helper_float_cvts_pl(fp0, cpu_env, fp0);
             gen_store_fpr32(fp0, fd);
             tcg_temp_free_i32(fp0);
         }
@@ -7747,7 +8673,6 @@ static void gen_flt3_ldst (DisasContext *ctx, uint32_t opc,
     }
     /* Don't do NOP if destination is zero: we must perform the actual
        memory access. */
-    save_cpu_state(ctx, 0);
     switch (opc) {
     case OPC_LWXC1:
         check_cop1x(ctx);
@@ -7887,7 +8812,7 @@ static void gen_flt3_arith (DisasContext *ctx, uint32_t opc,
             gen_load_fpr32(fp0, fs);
             gen_load_fpr32(fp1, ft);
             gen_load_fpr32(fp2, fr);
-            gen_helper_float_muladd_s(fp2, fp0, fp1, fp2);
+            gen_helper_float_madd_s(fp2, cpu_env, fp0, fp1, fp2);
             tcg_temp_free_i32(fp0);
             tcg_temp_free_i32(fp1);
             gen_store_fpr32(fp2, fd);
@@ -7906,7 +8831,7 @@ static void gen_flt3_arith (DisasContext *ctx, uint32_t opc,
             gen_load_fpr64(ctx, fp0, fs);
             gen_load_fpr64(ctx, fp1, ft);
             gen_load_fpr64(ctx, fp2, fr);
-            gen_helper_float_muladd_d(fp2, fp0, fp1, fp2);
+            gen_helper_float_madd_d(fp2, cpu_env, fp0, fp1, fp2);
             tcg_temp_free_i64(fp0);
             tcg_temp_free_i64(fp1);
             gen_store_fpr64(ctx, fp2, fd);
@@ -7924,7 +8849,7 @@ static void gen_flt3_arith (DisasContext *ctx, uint32_t opc,
             gen_load_fpr64(ctx, fp0, fs);
             gen_load_fpr64(ctx, fp1, ft);
             gen_load_fpr64(ctx, fp2, fr);
-            gen_helper_float_muladd_ps(fp2, fp0, fp1, fp2);
+            gen_helper_float_madd_ps(fp2, cpu_env, fp0, fp1, fp2);
             tcg_temp_free_i64(fp0);
             tcg_temp_free_i64(fp1);
             gen_store_fpr64(ctx, fp2, fd);
@@ -7942,7 +8867,7 @@ static void gen_flt3_arith (DisasContext *ctx, uint32_t opc,
             gen_load_fpr32(fp0, fs);
             gen_load_fpr32(fp1, ft);
             gen_load_fpr32(fp2, fr);
-            gen_helper_float_mulsub_s(fp2, fp0, fp1, fp2);
+            gen_helper_float_msub_s(fp2, cpu_env, fp0, fp1, fp2);
             tcg_temp_free_i32(fp0);
             tcg_temp_free_i32(fp1);
             gen_store_fpr32(fp2, fd);
@@ -7961,7 +8886,7 @@ static void gen_flt3_arith (DisasContext *ctx, uint32_t opc,
             gen_load_fpr64(ctx, fp0, fs);
             gen_load_fpr64(ctx, fp1, ft);
             gen_load_fpr64(ctx, fp2, fr);
-            gen_helper_float_mulsub_d(fp2, fp0, fp1, fp2);
+            gen_helper_float_msub_d(fp2, cpu_env, fp0, fp1, fp2);
             tcg_temp_free_i64(fp0);
             tcg_temp_free_i64(fp1);
             gen_store_fpr64(ctx, fp2, fd);
@@ -7979,7 +8904,7 @@ static void gen_flt3_arith (DisasContext *ctx, uint32_t opc,
             gen_load_fpr64(ctx, fp0, fs);
             gen_load_fpr64(ctx, fp1, ft);
             gen_load_fpr64(ctx, fp2, fr);
-            gen_helper_float_mulsub_ps(fp2, fp0, fp1, fp2);
+            gen_helper_float_msub_ps(fp2, cpu_env, fp0, fp1, fp2);
             tcg_temp_free_i64(fp0);
             tcg_temp_free_i64(fp1);
             gen_store_fpr64(ctx, fp2, fd);
@@ -7997,7 +8922,7 @@ static void gen_flt3_arith (DisasContext *ctx, uint32_t opc,
             gen_load_fpr32(fp0, fs);
             gen_load_fpr32(fp1, ft);
             gen_load_fpr32(fp2, fr);
-            gen_helper_float_nmuladd_s(fp2, fp0, fp1, fp2);
+            gen_helper_float_nmadd_s(fp2, cpu_env, fp0, fp1, fp2);
             tcg_temp_free_i32(fp0);
             tcg_temp_free_i32(fp1);
             gen_store_fpr32(fp2, fd);
@@ -8016,7 +8941,7 @@ static void gen_flt3_arith (DisasContext *ctx, uint32_t opc,
             gen_load_fpr64(ctx, fp0, fs);
             gen_load_fpr64(ctx, fp1, ft);
             gen_load_fpr64(ctx, fp2, fr);
-            gen_helper_float_nmuladd_d(fp2, fp0, fp1, fp2);
+            gen_helper_float_nmadd_d(fp2, cpu_env, fp0, fp1, fp2);
             tcg_temp_free_i64(fp0);
             tcg_temp_free_i64(fp1);
             gen_store_fpr64(ctx, fp2, fd);
@@ -8034,7 +8959,7 @@ static void gen_flt3_arith (DisasContext *ctx, uint32_t opc,
             gen_load_fpr64(ctx, fp0, fs);
             gen_load_fpr64(ctx, fp1, ft);
             gen_load_fpr64(ctx, fp2, fr);
-            gen_helper_float_nmuladd_ps(fp2, fp0, fp1, fp2);
+            gen_helper_float_nmadd_ps(fp2, cpu_env, fp0, fp1, fp2);
             tcg_temp_free_i64(fp0);
             tcg_temp_free_i64(fp1);
             gen_store_fpr64(ctx, fp2, fd);
@@ -8052,7 +8977,7 @@ static void gen_flt3_arith (DisasContext *ctx, uint32_t opc,
             gen_load_fpr32(fp0, fs);
             gen_load_fpr32(fp1, ft);
             gen_load_fpr32(fp2, fr);
-            gen_helper_float_nmulsub_s(fp2, fp0, fp1, fp2);
+            gen_helper_float_nmsub_s(fp2, cpu_env, fp0, fp1, fp2);
             tcg_temp_free_i32(fp0);
             tcg_temp_free_i32(fp1);
             gen_store_fpr32(fp2, fd);
@@ -8071,7 +8996,7 @@ static void gen_flt3_arith (DisasContext *ctx, uint32_t opc,
             gen_load_fpr64(ctx, fp0, fs);
             gen_load_fpr64(ctx, fp1, ft);
             gen_load_fpr64(ctx, fp2, fr);
-            gen_helper_float_nmulsub_d(fp2, fp0, fp1, fp2);
+            gen_helper_float_nmsub_d(fp2, cpu_env, fp0, fp1, fp2);
             tcg_temp_free_i64(fp0);
             tcg_temp_free_i64(fp1);
             gen_store_fpr64(ctx, fp2, fd);
@@ -8089,7 +9014,7 @@ static void gen_flt3_arith (DisasContext *ctx, uint32_t opc,
             gen_load_fpr64(ctx, fp0, fs);
             gen_load_fpr64(ctx, fp1, ft);
             gen_load_fpr64(ctx, fp2, fr);
-            gen_helper_float_nmulsub_ps(fp2, fp0, fp1, fp2);
+            gen_helper_float_nmsub_ps(fp2, cpu_env, fp0, fp1, fp2);
             tcg_temp_free_i64(fp0);
             tcg_temp_free_i64(fp1);
             gen_store_fpr64(ctx, fp2, fd);
@@ -8122,22 +9047,22 @@ gen_rdhwr (CPUMIPSState *env, DisasContext *ctx, int rt, int rd)
     switch (rd) {
     case 0:
         save_cpu_state(ctx, 1);
-        gen_helper_rdhwr_cpunum(t0);
+        gen_helper_rdhwr_cpunum(t0, cpu_env);
         gen_store_gpr(t0, rt);
         break;
     case 1:
         save_cpu_state(ctx, 1);
-        gen_helper_rdhwr_synci_step(t0);
+        gen_helper_rdhwr_synci_step(t0, cpu_env);
         gen_store_gpr(t0, rt);
         break;
     case 2:
         save_cpu_state(ctx, 1);
-        gen_helper_rdhwr_cc(t0);
+        gen_helper_rdhwr_cc(t0, cpu_env);
         gen_store_gpr(t0, rt);
         break;
     case 3:
         save_cpu_state(ctx, 1);
-        gen_helper_rdhwr_ccres(t0);
+        gen_helper_rdhwr_ccres(t0, cpu_env);
         gen_store_gpr(t0, rt);
         break;
     case 29:
@@ -8214,7 +9139,7 @@ static void handle_delay_slot (CPUMIPSState *env, DisasContext *ctx,
             }
             if (ctx->singlestep_enabled) {
                 save_cpu_state(ctx, 0);
-                gen_helper_0i(raise_exception, EXCP_DEBUG);
+                gen_helper_0e0i(raise_exception, EXCP_DEBUG);
             }
             tcg_gen_exit_tb(0);
             break;
@@ -8390,22 +9315,22 @@ static void gen_mips16_save (DisasContext *ctx,
     case 4:
         gen_base_offset_addr(ctx, t0, 29, 12);
         gen_load_gpr(t1, 7);
-        op_st_sw(t1, t0, ctx);
+        tcg_gen_qemu_st32(t1, t0, ctx->mem_idx);
         /* Fall through */
     case 3:
         gen_base_offset_addr(ctx, t0, 29, 8);
         gen_load_gpr(t1, 6);
-        op_st_sw(t1, t0, ctx);
+        tcg_gen_qemu_st32(t1, t0, ctx->mem_idx);
         /* Fall through */
     case 2:
         gen_base_offset_addr(ctx, t0, 29, 4);
         gen_load_gpr(t1, 5);
-        op_st_sw(t1, t0, ctx);
+        tcg_gen_qemu_st32(t1, t0, ctx->mem_idx);
         /* Fall through */
     case 1:
         gen_base_offset_addr(ctx, t0, 29, 0);
         gen_load_gpr(t1, 4);
-        op_st_sw(t1, t0, ctx);
+        tcg_gen_qemu_st32(t1, t0, ctx->mem_idx);
     }
 
     gen_load_gpr(t0, 29);
@@ -8413,7 +9338,7 @@ static void gen_mips16_save (DisasContext *ctx,
 #define DECR_AND_STORE(reg) do {                \
         tcg_gen_subi_tl(t0, t0, 4);             \
         gen_load_gpr(t1, reg);                  \
-        op_st_sw(t1, t0, ctx);                  \
+        tcg_gen_qemu_st32(t1, t0, ctx->mem_idx);                  \
     } while (0)
 
     if (do_ra) {
@@ -8511,10 +9436,10 @@ static void gen_mips16_restore (DisasContext *ctx,
 
     tcg_gen_addi_tl(t0, cpu_gpr[29], framesize);
 
-#define DECR_AND_LOAD(reg) do {                 \
-        tcg_gen_subi_tl(t0, t0, 4);             \
-        op_ld_lw(t1, t0, ctx);                  \
-        gen_store_gpr(t1, reg);                 \
+#define DECR_AND_LOAD(reg) do {                   \
+        tcg_gen_subi_tl(t0, t0, 4);               \
+        tcg_gen_qemu_ld32u(t1, t0, ctx->mem_idx); \
+        gen_store_gpr(t1, reg);                   \
     } while (0)
 
     if (do_ra) {
@@ -8678,7 +9603,7 @@ static void decode_i64_mips16 (CPUMIPSState *env, DisasContext *ctx,
 static int decode_extended_mips16_opc (CPUMIPSState *env, DisasContext *ctx,
                                        int *is_branch)
 {
-    int extend = lduw_code(ctx->pc + 2);
+    int extend = cpu_lduw_code(env, ctx->pc + 2);
     int op, rx, ry, funct, sa;
     int16_t imm, offset;
 
@@ -8760,10 +9685,10 @@ static int decode_extended_mips16_opc (CPUMIPSState *env, DisasContext *ctx,
         gen_arith_imm(env, ctx, OPC_ADDIU, rx, rx, imm);
         break;
     case M16_OPC_SLTI:
-        gen_slt_imm(env, OPC_SLTI, 24, rx, imm);
+        gen_slt_imm(env, ctx, OPC_SLTI, 24, rx, imm);
         break;
     case M16_OPC_SLTIU:
-        gen_slt_imm(env, OPC_SLTIU, 24, rx, imm);
+        gen_slt_imm(env, ctx, OPC_SLTIU, 24, rx, imm);
         break;
     case M16_OPC_I8:
         switch (funct) {
@@ -8904,7 +9829,7 @@ static int decode_mips16_opc (CPUMIPSState *env, DisasContext *ctx,
         /* No delay slot, so just process as a normal instruction */
         break;
     case M16_OPC_JAL:
-        offset = lduw_code(ctx->pc + 2);
+        offset = cpu_lduw_code(env, ctx->pc + 2);
         offset = (((ctx->opcode & 0x1f) << 21)
                   | ((ctx->opcode >> 5) & 0x1f) << 16
                   | offset) << 2;
@@ -8974,15 +9899,13 @@ static int decode_mips16_opc (CPUMIPSState *env, DisasContext *ctx,
     case M16_OPC_SLTI:
         {
             int16_t imm = (uint8_t) ctx->opcode;
-
-            gen_slt_imm(env, OPC_SLTI, 24, rx, imm);
+            gen_slt_imm(env, ctx, OPC_SLTI, 24, rx, imm);
         }
         break;
     case M16_OPC_SLTIU:
         {
             int16_t imm = (uint8_t) ctx->opcode;
-
-            gen_slt_imm(env, OPC_SLTIU, 24, rx, imm);
+            gen_slt_imm(env, ctx, OPC_SLTIU, 24, rx, imm);
         }
         break;
     case M16_OPC_I8:
@@ -9057,8 +9980,7 @@ static int decode_mips16_opc (CPUMIPSState *env, DisasContext *ctx,
     case M16_OPC_CMPI:
         {
             int16_t imm = (uint8_t) ctx->opcode;
-
-            gen_logic_imm(env, OPC_XORI, 24, rx, imm);
+            gen_logic_imm(env, ctx, OPC_XORI, 24, rx, imm);
         }
         break;
 #if defined(TARGET_MIPS64)
@@ -9170,10 +10092,10 @@ static int decode_mips16_opc (CPUMIPSState *env, DisasContext *ctx,
             }
             break;
         case RR_SLT:
-            gen_slt(env, OPC_SLT, 24, rx, ry);
+            gen_slt(env, ctx, OPC_SLT, 24, rx, ry);
             break;
         case RR_SLTU:
-            gen_slt(env, OPC_SLTU, 24, rx, ry);
+            gen_slt(env, ctx, OPC_SLTU, 24, rx, ry);
             break;
         case RR_BREAK:
             generate_exception(ctx, EXCP_BREAK);
@@ -9194,22 +10116,22 @@ static int decode_mips16_opc (CPUMIPSState *env, DisasContext *ctx,
             break;
 #endif
         case RR_CMP:
-            gen_logic(env, OPC_XOR, 24, rx, ry);
+            gen_logic(env, ctx, OPC_XOR, 24, rx, ry);
             break;
         case RR_NEG:
             gen_arith(env, ctx, OPC_SUBU, rx, 0, ry);
             break;
         case RR_AND:
-            gen_logic(env, OPC_AND, rx, rx, ry);
+            gen_logic(env, ctx, OPC_AND, rx, rx, ry);
             break;
         case RR_OR:
-            gen_logic(env, OPC_OR, rx, rx, ry);
+            gen_logic(env, ctx, OPC_OR, rx, rx, ry);
             break;
         case RR_XOR:
-            gen_logic(env, OPC_XOR, rx, rx, ry);
+            gen_logic(env, ctx, OPC_XOR, rx, rx, ry);
             break;
         case RR_NOT:
-            gen_logic(env, OPC_NOR, rx, ry, 0);
+            gen_logic(env, ctx, OPC_NOR, rx, ry, 0);
             break;
         case RR_MFHI:
             gen_HILO(ctx, OPC_MFHI, rx);
@@ -9317,9 +10239,19 @@ static int decode_mips16_opc (CPUMIPSState *env, DisasContext *ctx,
     return n_bytes;
 }
 
-/* microMIPS extension to MIPS32 */
+/* microMIPS extension to MIPS32/MIPS64 */
 
-/* microMIPS32 major opcodes */
+/*
+ * microMIPS32/microMIPS64 major opcodes
+ *
+ * 1. MIPS Architecture for Programmers Volume II-B:
+ *      The microMIPS32 Instruction Set (Revision 3.05)
+ *
+ *    Table 6.2 microMIPS32 Encoding of Major Opcode Field
+ *
+ * 2. MIPS Architecture For Programmers Volume II-A:
+ *      The MIPS64 Instruction Set (Revision 3.51)
+ */
 
 enum {
     POOL32A = 0x00,
@@ -9346,9 +10278,10 @@ enum {
     POOL16D = 0x13,
     ORI32 = 0x14,
     POOL32F = 0x15,
-    POOL32S = 0x16,
-    DADDIU32 = 0x17,
+    POOL32S = 0x16,  /* MIPS64 */
+    DADDIU32 = 0x17, /* MIPS64 */
 
+    /* 0x1f is reserved */
     POOL32C = 0x18,
     LWGP16 = 0x19,
     LW16 = 0x1a,
@@ -9356,7 +10289,6 @@ enum {
     XORI32 = 0x1c,
     JALS32 = 0x1d,
     ADDIUPC = 0x1e,
-    POOL48A = 0x1f,
 
     /* 0x20 is reserved */
     RES_20 = 0x20,
@@ -9385,8 +10317,8 @@ enum {
     B16 = 0x33,
     ANDI32 = 0x34,
     J32 = 0x35,
-    SD32 = 0x36,
-    LD32 = 0x37,
+    SD32 = 0x36, /* MIPS64 */
+    LD32 = 0x37, /* MIPS64 */
 
     /* 0x38 and 0x39 are reserved */
     RES_38 = 0x38,
@@ -9437,6 +10369,19 @@ enum {
 
 /* POOL32AXF encoding of minor opcode field extension */
 
+/*
+ * 1. MIPS Architecture for Programmers Volume II-B:
+ *      The microMIPS32 Instruction Set (Revision 3.05)
+ *
+ *    Table 6.5 POOL32Axf Encoding of Minor Opcode Extension Field
+ *
+ * 2. MIPS Architecture for Programmers VolumeIV-e:
+ *      The MIPS DSP Application-Specific Extension
+ *        to the microMIPS32 Architecture (Revision 2.34)
+ *
+ *    Table 5.5 POOL32Axf Encoding of Minor Opcode Extension Field
+ */
+
 enum {
     /* bits 11..6 */
     TEQ = 0x00,
@@ -9449,6 +10394,8 @@ enum {
     MFC0 = 0x03,
     MTC0 = 0x0b,
 
+    /* begin of microMIPS32 DSP */
+
     /* bits 13..12 for 0x01 */
     MFHI_ACC = 0x0,
     MFLO_ACC = 0x1,
@@ -9463,7 +10410,9 @@ enum {
 
     /* bits 13..12 for 0x32 */
     MULT_ACC = 0x0,
-    MULTU_ACC = 0x0,
+    MULTU_ACC = 0x1,
+
+    /* end of microMIPS32 DSP */
 
     /* bits 15..12 for 0x2c */
     SEB = 0x2,
@@ -9831,12 +10780,13 @@ static void gen_andi16 (CPUMIPSState *env, DisasContext *ctx)
     int rs = mmreg(uMIPS_RS(ctx->opcode));
     int encoded = ZIMM(ctx->opcode, 0, 4);
 
-    gen_logic_imm(env, OPC_ANDI, rd, rs, decoded_imm[encoded]);
+    gen_logic_imm(env, ctx, OPC_ANDI, rd, rs, decoded_imm[encoded]);
 }
 
 static void gen_ldst_multiple (DisasContext *ctx, uint32_t opc, int reglist,
                                int base, int16_t offset)
 {
+    const char *opn = "ldst_multiple";
     TCGv t0, t1;
     TCGv_i32 t2;
 
@@ -9855,20 +10805,25 @@ static void gen_ldst_multiple (DisasContext *ctx, uint32_t opc, int reglist,
     save_cpu_state(ctx, 1);
     switch (opc) {
     case LWM32:
-        gen_helper_lwm(t0, t1, t2);
+        gen_helper_lwm(cpu_env, t0, t1, t2);
+        opn = "lwm";
         break;
     case SWM32:
-        gen_helper_swm(t0, t1, t2);
+        gen_helper_swm(cpu_env, t0, t1, t2);
+        opn = "swm";
         break;
 #ifdef TARGET_MIPS64
     case LDM:
-        gen_helper_ldm(t0, t1, t2);
+        gen_helper_ldm(cpu_env, t0, t1, t2);
+        opn = "ldm";
         break;
     case SDM:
-        gen_helper_sdm(t0, t1, t2);
+        gen_helper_sdm(cpu_env, t0, t1, t2);
+        opn = "sdm";
         break;
 #endif
     }
+    (void)opn;
     MIPS_DEBUG("%s, %x, %d(%s)", opn, reglist, offset, regnames[base]);
     tcg_temp_free(t0);
     tcg_temp_free(t1);
@@ -9887,25 +10842,25 @@ static void gen_pool16c_insn (CPUMIPSState *env, DisasContext *ctx, int *is_bran
     case NOT16 + 1:
     case NOT16 + 2:
     case NOT16 + 3:
-        gen_logic(env, OPC_NOR, rd, rs, 0);
+        gen_logic(env, ctx, OPC_NOR, rd, rs, 0);
         break;
     case XOR16 + 0:
     case XOR16 + 1:
     case XOR16 + 2:
     case XOR16 + 3:
-        gen_logic(env, OPC_XOR, rd, rd, rs);
+        gen_logic(env, ctx, OPC_XOR, rd, rd, rs);
         break;
     case AND16 + 0:
     case AND16 + 1:
     case AND16 + 2:
     case AND16 + 3:
-        gen_logic(env, OPC_AND, rd, rd, rs);
+        gen_logic(env, ctx, OPC_AND, rd, rd, rs);
         break;
     case OR16 + 0:
     case OR16 + 1:
     case OR16 + 2:
     case OR16 + 3:
-        gen_logic(env, OPC_OR, rd, rd, rs);
+        gen_logic(env, ctx, OPC_OR, rd, rd, rs);
         break;
     case LWM16 + 0:
     case LWM16 + 1:
@@ -10017,8 +10972,7 @@ static void gen_ldxs (DisasContext *ctx, int base, int index, int rd)
         gen_op_addr_add(ctx, t0, t1, t0);
     }
 
-    save_cpu_state(ctx, 0);
-    op_ld_lw(t1, t0, ctx);
+    tcg_gen_qemu_ld32s(t1, t0, ctx->mem_idx);
     gen_store_gpr(t1, rd);
 
     tcg_temp_free(t0);
@@ -10047,23 +11001,21 @@ static void gen_ldst_pair (DisasContext *ctx, uint32_t opc, int rd,
             generate_exception(ctx, EXCP_RI);
             return;
         }
-        save_cpu_state(ctx, 0);
-        op_ld_lw(t1, t0, ctx);
+        tcg_gen_qemu_ld32s(t1, t0, ctx->mem_idx);
         gen_store_gpr(t1, rd);
         tcg_gen_movi_tl(t1, 4);
         gen_op_addr_add(ctx, t0, t0, t1);
-        op_ld_lw(t1, t0, ctx);
+        tcg_gen_qemu_ld32s(t1, t0, ctx->mem_idx);
         gen_store_gpr(t1, rd+1);
         opn = "lwp";
         break;
     case SWP:
-        save_cpu_state(ctx, 0);
         gen_load_gpr(t1, rd);
-        op_st_sw(t1, t0, ctx);
+        tcg_gen_qemu_st32(t1, t0, ctx->mem_idx);
         tcg_gen_movi_tl(t1, 4);
         gen_op_addr_add(ctx, t0, t0, t1);
         gen_load_gpr(t1, rd+1);
-        op_st_sw(t1, t0, ctx);
+        tcg_gen_qemu_st32(t1, t0, ctx->mem_idx);
         opn = "swp";
         break;
 #ifdef TARGET_MIPS64
@@ -10072,23 +11024,21 @@ static void gen_ldst_pair (DisasContext *ctx, uint32_t opc, int rd,
             generate_exception(ctx, EXCP_RI);
             return;
         }
-        save_cpu_state(ctx, 0);
-        op_ld_ld(t1, t0, ctx);
+        tcg_gen_qemu_ld64(t1, t0, ctx->mem_idx);
         gen_store_gpr(t1, rd);
         tcg_gen_movi_tl(t1, 8);
         gen_op_addr_add(ctx, t0, t0, t1);
-        op_ld_ld(t1, t0, ctx);
+        tcg_gen_qemu_ld64(t1, t0, ctx->mem_idx);
         gen_store_gpr(t1, rd+1);
         opn = "ldp";
         break;
     case SDP:
-        save_cpu_state(ctx, 0);
         gen_load_gpr(t1, rd);
-        op_st_sd(t1, t0, ctx);
+        tcg_gen_qemu_st64(t1, t0, ctx->mem_idx);
         tcg_gen_movi_tl(t1, 8);
         gen_op_addr_add(ctx, t0, t0, t1);
         gen_load_gpr(t1, rd+1);
-        op_st_sd(t1, t0, ctx);
+        tcg_gen_qemu_st64(t1, t0, ctx->mem_idx);
         opn = "sdp";
         break;
 #endif
@@ -10287,7 +11237,7 @@ static void gen_pool32axf (CPUMIPSState *env, DisasContext *ctx, int rt, int rs,
                 TCGv t0 = tcg_temp_new();
 
                 save_cpu_state(ctx, 1);
-                gen_helper_di(t0);
+                gen_helper_di(t0, cpu_env);
                 gen_store_gpr(t0, rs);
                 /* Stop translation as we may have switched the execution mode */
                 ctx->bstate = BS_STOP;
@@ -10300,7 +11250,7 @@ static void gen_pool32axf (CPUMIPSState *env, DisasContext *ctx, int rt, int rs,
                 TCGv t0 = tcg_temp_new();
 
                 save_cpu_state(ctx, 1);
-                gen_helper_ei(t0);
+                gen_helper_ei(t0, cpu_env);
                 gen_store_gpr(t0, rs);
                 /* Stop translation as we may have switched the execution mode */
                 ctx->bstate = BS_STOP;
@@ -10635,7 +11585,7 @@ static void decode_micromips32_opc (CPUMIPSState *env, DisasContext *ctx,
     uint32_t op, minor, mips32_op;
     uint32_t cond, fmt, cc;
 
-    insn = lduw_code(ctx->pc + 2);
+    insn = cpu_lduw_code(env, ctx->pc + 2);
     ctx->opcode = (ctx->opcode << 16) | insn;
 
     rt = (ctx->opcode >> 21) & 0x1f;
@@ -10719,7 +11669,7 @@ static void decode_micromips32_opc (CPUMIPSState *env, DisasContext *ctx,
             case XOR32:
                 mips32_op = OPC_XOR;
             do_logic:
-                gen_logic(env, mips32_op, rd, rs, rt);
+                gen_logic(env, ctx, mips32_op, rd, rs, rt);
                 break;
                 /* Set less than */
             case SLT:
@@ -10728,7 +11678,7 @@ static void decode_micromips32_opc (CPUMIPSState *env, DisasContext *ctx,
             case SLTU:
                 mips32_op = OPC_SLTU;
             do_slt:
-                gen_slt(env, mips32_op, rd, rs, rt);
+                gen_slt(env, ctx, mips32_op, rd, rs, rt);
                 break;
             default:
                 goto pool32a_invalid;
@@ -10744,7 +11694,7 @@ static void decode_micromips32_opc (CPUMIPSState *env, DisasContext *ctx,
             case MOVZ:
                 mips32_op = OPC_MOVZ;
             do_cmov:
-                gen_cond_move(env, mips32_op, rd, rs, rt);
+                gen_cond_move(env, ctx, mips32_op, rd, rs, rt);
                 break;
             case LWXS:
                 gen_ldxs(ctx, rs, rt, rd);
@@ -11157,7 +12107,7 @@ static void decode_micromips32_opc (CPUMIPSState *env, DisasContext *ctx,
                target. */
             break;
         case LUI:
-            gen_logic_imm(env, OPC_LUI, rs, -1, imm);
+            gen_logic_imm(env, ctx, OPC_LUI, rs, -1, imm);
             break;
         case SYNCI:
             break;
@@ -11276,7 +12226,7 @@ static void decode_micromips32_opc (CPUMIPSState *env, DisasContext *ctx,
     case ANDI32:
         mips32_op = OPC_ANDI;
     do_logici:
-        gen_logic_imm(env, mips32_op, rt, rs, imm);
+        gen_logic_imm(env, ctx, mips32_op, rt, rs, imm);
         break;
 
         /* Set less than immediate */
@@ -11286,7 +12236,7 @@ static void decode_micromips32_opc (CPUMIPSState *env, DisasContext *ctx,
     case SLTIU32:
         mips32_op = OPC_SLTIU;
     do_slti:
-        gen_slt_imm(env, mips32_op, rt, rs, imm);
+        gen_slt_imm(env, ctx, mips32_op, rt, rs, imm);
         break;
     case JALX32:
         offset = (int32_t)(ctx->opcode & 0x3FFFFFF) << 2;
@@ -11433,7 +12383,6 @@ static int decode_micromips_opc (CPUMIPSState *env, DisasContext *ctx, int *is_b
         case LB32:
         case LH32:
         case DADDIU32:
-        case POOL48A:           /* ??? */
         case LWC132:
         case LDC132:
         case LD32:
@@ -11701,6 +12650,1686 @@ static int decode_micromips_opc (CPUMIPSState *env, DisasContext *ctx, int *is_b
 
 #endif
 
+/* MIPSDSP functions. */
+static void gen_mipsdsp_ld(CPUMIPSState *env, DisasContext *ctx, uint32_t opc,
+                           int rd, int base, int offset)
+{
+    const char *opn = "ldx";
+    TCGv t0;
+
+    if (rd == 0) {
+        MIPS_DEBUG("NOP");
+        return;
+    }
+
+    check_dsp(ctx);
+    t0 = tcg_temp_new();
+
+    if (base == 0) {
+        gen_load_gpr(t0, offset);
+    } else if (offset == 0) {
+        gen_load_gpr(t0, base);
+    } else {
+        gen_op_addr_add(ctx, t0, cpu_gpr[base], cpu_gpr[offset]);
+    }
+
+    switch (opc) {
+    case OPC_LBUX:
+        tcg_gen_qemu_ld8u(t0, t0, ctx->mem_idx);
+        gen_store_gpr(t0, rd);
+        opn = "lbux";
+        break;
+    case OPC_LHX:
+        tcg_gen_qemu_ld16s(t0, t0, ctx->mem_idx);
+        gen_store_gpr(t0, rd);
+        opn = "lhx";
+        break;
+    case OPC_LWX:
+        tcg_gen_qemu_ld32s(t0, t0, ctx->mem_idx);
+        gen_store_gpr(t0, rd);
+        opn = "lwx";
+        break;
+#if defined(TARGET_MIPS64)
+    case OPC_LDX:
+        tcg_gen_qemu_ld64(t0, t0, ctx->mem_idx);
+        gen_store_gpr(t0, rd);
+        opn = "ldx";
+        break;
+#endif
+    }
+    (void)opn; /* avoid a compiler warning */
+    MIPS_DEBUG("%s %s, %s(%s)", opn,
+               regnames[rd], regnames[offset], regnames[base]);
+    tcg_temp_free(t0);
+}
+
+static void gen_mipsdsp_arith(DisasContext *ctx, uint32_t op1, uint32_t op2,
+                              int ret, int v1, int v2)
+{
+    const char *opn = "mipsdsp arith";
+    TCGv v1_t;
+    TCGv v2_t;
+
+    if (ret == 0) {
+        /* Treat as NOP. */
+        MIPS_DEBUG("NOP");
+        return;
+    }
+
+    v1_t = tcg_temp_new();
+    v2_t = tcg_temp_new();
+
+    gen_load_gpr(v1_t, v1);
+    gen_load_gpr(v2_t, v2);
+
+    switch (op1) {
+    /* OPC_MULT_G_2E is equal OPC_ADDUH_QB_DSP */
+    case OPC_MULT_G_2E:
+        check_dspr2(ctx);
+        switch (op2) {
+        case OPC_ADDUH_QB:
+            gen_helper_adduh_qb(cpu_gpr[ret], v1_t, v2_t);
+            break;
+        case OPC_ADDUH_R_QB:
+            gen_helper_adduh_r_qb(cpu_gpr[ret], v1_t, v2_t);
+            break;
+        case OPC_ADDQH_PH:
+            gen_helper_addqh_ph(cpu_gpr[ret], v1_t, v2_t);
+            break;
+        case OPC_ADDQH_R_PH:
+            gen_helper_addqh_r_ph(cpu_gpr[ret], v1_t, v2_t);
+            break;
+        case OPC_ADDQH_W:
+            gen_helper_addqh_w(cpu_gpr[ret], v1_t, v2_t);
+            break;
+        case OPC_ADDQH_R_W:
+            gen_helper_addqh_r_w(cpu_gpr[ret], v1_t, v2_t);
+            break;
+        case OPC_SUBUH_QB:
+            gen_helper_subuh_qb(cpu_gpr[ret], v1_t, v2_t);
+            break;
+        case OPC_SUBUH_R_QB:
+            gen_helper_subuh_r_qb(cpu_gpr[ret], v1_t, v2_t);
+            break;
+        case OPC_SUBQH_PH:
+            gen_helper_subqh_ph(cpu_gpr[ret], v1_t, v2_t);
+            break;
+        case OPC_SUBQH_R_PH:
+            gen_helper_subqh_r_ph(cpu_gpr[ret], v1_t, v2_t);
+            break;
+        case OPC_SUBQH_W:
+            gen_helper_subqh_w(cpu_gpr[ret], v1_t, v2_t);
+            break;
+        case OPC_SUBQH_R_W:
+            gen_helper_subqh_r_w(cpu_gpr[ret], v1_t, v2_t);
+            break;
+        }
+        break;
+    case OPC_ABSQ_S_PH_DSP:
+        switch (op2) {
+        case OPC_ABSQ_S_QB:
+            check_dspr2(ctx);
+            gen_helper_absq_s_qb(cpu_gpr[ret], v2_t, cpu_env);
+            break;
+        case OPC_ABSQ_S_PH:
+            check_dsp(ctx);
+            gen_helper_absq_s_ph(cpu_gpr[ret], v2_t, cpu_env);
+            break;
+        case OPC_ABSQ_S_W:
+            check_dsp(ctx);
+            gen_helper_absq_s_w(cpu_gpr[ret], v2_t, cpu_env);
+            break;
+        case OPC_PRECEQ_W_PHL:
+            check_dsp(ctx);
+            tcg_gen_andi_tl(cpu_gpr[ret], v2_t, 0xFFFF0000);
+            tcg_gen_ext32s_tl(cpu_gpr[ret], cpu_gpr[ret]);
+            break;
+        case OPC_PRECEQ_W_PHR:
+            check_dsp(ctx);
+            tcg_gen_andi_tl(cpu_gpr[ret], v2_t, 0x0000FFFF);
+            tcg_gen_shli_tl(cpu_gpr[ret], cpu_gpr[ret], 16);
+            tcg_gen_ext32s_tl(cpu_gpr[ret], cpu_gpr[ret]);
+            break;
+        case OPC_PRECEQU_PH_QBL:
+            check_dsp(ctx);
+            gen_helper_precequ_ph_qbl(cpu_gpr[ret], v2_t);
+            break;
+        case OPC_PRECEQU_PH_QBR:
+            check_dsp(ctx);
+            gen_helper_precequ_ph_qbr(cpu_gpr[ret], v2_t);
+            break;
+        case OPC_PRECEQU_PH_QBLA:
+            check_dsp(ctx);
+            gen_helper_precequ_ph_qbla(cpu_gpr[ret], v2_t);
+            break;
+        case OPC_PRECEQU_PH_QBRA:
+            check_dsp(ctx);
+            gen_helper_precequ_ph_qbra(cpu_gpr[ret], v2_t);
+            break;
+        case OPC_PRECEU_PH_QBL:
+            check_dsp(ctx);
+            gen_helper_preceu_ph_qbl(cpu_gpr[ret], v2_t);
+            break;
+        case OPC_PRECEU_PH_QBR:
+            check_dsp(ctx);
+            gen_helper_preceu_ph_qbr(cpu_gpr[ret], v2_t);
+            break;
+        case OPC_PRECEU_PH_QBLA:
+            check_dsp(ctx);
+            gen_helper_preceu_ph_qbla(cpu_gpr[ret], v2_t);
+            break;
+        case OPC_PRECEU_PH_QBRA:
+            check_dsp(ctx);
+            gen_helper_preceu_ph_qbra(cpu_gpr[ret], v2_t);
+            break;
+        }
+        break;
+    case OPC_ADDU_QB_DSP:
+        switch (op2) {
+        case OPC_ADDQ_PH:
+            check_dsp(ctx);
+            gen_helper_addq_ph(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_ADDQ_S_PH:
+            check_dsp(ctx);
+            gen_helper_addq_s_ph(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_ADDQ_S_W:
+            check_dsp(ctx);
+            gen_helper_addq_s_w(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_ADDU_QB:
+            check_dsp(ctx);
+            gen_helper_addu_qb(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_ADDU_S_QB:
+            check_dsp(ctx);
+            gen_helper_addu_s_qb(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_ADDU_PH:
+            check_dspr2(ctx);
+            gen_helper_addu_ph(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_ADDU_S_PH:
+            check_dspr2(ctx);
+            gen_helper_addu_s_ph(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_SUBQ_PH:
+            check_dsp(ctx);
+            gen_helper_subq_ph(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_SUBQ_S_PH:
+            check_dsp(ctx);
+            gen_helper_subq_s_ph(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_SUBQ_S_W:
+            check_dsp(ctx);
+            gen_helper_subq_s_w(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_SUBU_QB:
+            check_dsp(ctx);
+            gen_helper_subu_qb(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_SUBU_S_QB:
+            check_dsp(ctx);
+            gen_helper_subu_s_qb(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_SUBU_PH:
+            check_dspr2(ctx);
+            gen_helper_subu_ph(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_SUBU_S_PH:
+            check_dspr2(ctx);
+            gen_helper_subu_s_ph(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_ADDSC:
+            check_dsp(ctx);
+            gen_helper_addsc(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_ADDWC:
+            check_dsp(ctx);
+            gen_helper_addwc(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_MODSUB:
+            check_dsp(ctx);
+            gen_helper_modsub(cpu_gpr[ret], v1_t, v2_t);
+            break;
+        case OPC_RADDU_W_QB:
+            check_dsp(ctx);
+            gen_helper_raddu_w_qb(cpu_gpr[ret], v1_t);
+            break;
+        }
+        break;
+    case OPC_CMPU_EQ_QB_DSP:
+        switch (op2) {
+        case OPC_PRECR_QB_PH:
+            check_dspr2(ctx);
+            gen_helper_precr_qb_ph(cpu_gpr[ret], v1_t, v2_t);
+            break;
+        case OPC_PRECRQ_QB_PH:
+            check_dsp(ctx);
+            gen_helper_precrq_qb_ph(cpu_gpr[ret], v1_t, v2_t);
+            break;
+        case OPC_PRECR_SRA_PH_W:
+            check_dspr2(ctx);
+            {
+                TCGv_i32 sa_t = tcg_const_i32(v2);
+                gen_helper_precr_sra_ph_w(cpu_gpr[ret], sa_t, v1_t,
+                                          cpu_gpr[ret]);
+                tcg_temp_free_i32(sa_t);
+                break;
+            }
+        case OPC_PRECR_SRA_R_PH_W:
+            check_dspr2(ctx);
+            {
+                TCGv_i32 sa_t = tcg_const_i32(v2);
+                gen_helper_precr_sra_r_ph_w(cpu_gpr[ret], sa_t, v1_t,
+                                            cpu_gpr[ret]);
+                tcg_temp_free_i32(sa_t);
+                break;
+            }
+        case OPC_PRECRQ_PH_W:
+            check_dsp(ctx);
+            gen_helper_precrq_ph_w(cpu_gpr[ret], v1_t, v2_t);
+            break;
+        case OPC_PRECRQ_RS_PH_W:
+            check_dsp(ctx);
+            gen_helper_precrq_rs_ph_w(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_PRECRQU_S_QB_PH:
+            check_dsp(ctx);
+            gen_helper_precrqu_s_qb_ph(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        }
+        break;
+#ifdef TARGET_MIPS64
+    case OPC_ABSQ_S_QH_DSP:
+        switch (op2) {
+        case OPC_PRECEQ_L_PWL:
+            check_dsp(ctx);
+            tcg_gen_andi_tl(cpu_gpr[ret], v2_t, 0xFFFFFFFF00000000ull);
+            break;
+        case OPC_PRECEQ_L_PWR:
+            check_dsp(ctx);
+            tcg_gen_shli_tl(cpu_gpr[ret], v2_t, 32);
+            break;
+        case OPC_PRECEQ_PW_QHL:
+            check_dsp(ctx);
+            gen_helper_preceq_pw_qhl(cpu_gpr[ret], v2_t);
+            break;
+        case OPC_PRECEQ_PW_QHR:
+            check_dsp(ctx);
+            gen_helper_preceq_pw_qhr(cpu_gpr[ret], v2_t);
+            break;
+        case OPC_PRECEQ_PW_QHLA:
+            check_dsp(ctx);
+            gen_helper_preceq_pw_qhla(cpu_gpr[ret], v2_t);
+            break;
+        case OPC_PRECEQ_PW_QHRA:
+            check_dsp(ctx);
+            gen_helper_preceq_pw_qhra(cpu_gpr[ret], v2_t);
+            break;
+        case OPC_PRECEQU_QH_OBL:
+            check_dsp(ctx);
+            gen_helper_precequ_qh_obl(cpu_gpr[ret], v2_t);
+            break;
+        case OPC_PRECEQU_QH_OBR:
+            check_dsp(ctx);
+            gen_helper_precequ_qh_obr(cpu_gpr[ret], v2_t);
+            break;
+        case OPC_PRECEQU_QH_OBLA:
+            check_dsp(ctx);
+            gen_helper_precequ_qh_obla(cpu_gpr[ret], v2_t);
+            break;
+        case OPC_PRECEQU_QH_OBRA:
+            check_dsp(ctx);
+            gen_helper_precequ_qh_obra(cpu_gpr[ret], v2_t);
+            break;
+        case OPC_PRECEU_QH_OBL:
+            check_dsp(ctx);
+            gen_helper_preceu_qh_obl(cpu_gpr[ret], v2_t);
+            break;
+        case OPC_PRECEU_QH_OBR:
+            check_dsp(ctx);
+            gen_helper_preceu_qh_obr(cpu_gpr[ret], v2_t);
+            break;
+        case OPC_PRECEU_QH_OBLA:
+            check_dsp(ctx);
+            gen_helper_preceu_qh_obla(cpu_gpr[ret], v2_t);
+            break;
+        case OPC_PRECEU_QH_OBRA:
+            check_dsp(ctx);
+            gen_helper_preceu_qh_obra(cpu_gpr[ret], v2_t);
+            break;
+        case OPC_ABSQ_S_OB:
+            check_dspr2(ctx);
+            gen_helper_absq_s_ob(cpu_gpr[ret], v2_t, cpu_env);
+            break;
+        case OPC_ABSQ_S_PW:
+            check_dsp(ctx);
+            gen_helper_absq_s_pw(cpu_gpr[ret], v2_t, cpu_env);
+            break;
+        case OPC_ABSQ_S_QH:
+            check_dsp(ctx);
+            gen_helper_absq_s_qh(cpu_gpr[ret], v2_t, cpu_env);
+            break;
+        }
+        break;
+    case OPC_ADDU_OB_DSP:
+        switch (op2) {
+        case OPC_RADDU_L_OB:
+            check_dsp(ctx);
+            gen_helper_raddu_l_ob(cpu_gpr[ret], v1_t);
+            break;
+        case OPC_SUBQ_PW:
+            check_dsp(ctx);
+            gen_helper_subq_pw(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_SUBQ_S_PW:
+            check_dsp(ctx);
+            gen_helper_subq_s_pw(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_SUBQ_QH:
+            check_dsp(ctx);
+            gen_helper_subq_qh(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_SUBQ_S_QH:
+            check_dsp(ctx);
+            gen_helper_subq_s_qh(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_SUBU_OB:
+            check_dsp(ctx);
+            gen_helper_subu_ob(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_SUBU_S_OB:
+            check_dsp(ctx);
+            gen_helper_subu_s_ob(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_SUBU_QH:
+            check_dspr2(ctx);
+            gen_helper_subu_qh(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_SUBU_S_QH:
+            check_dspr2(ctx);
+            gen_helper_subu_s_qh(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_SUBUH_OB:
+            check_dspr2(ctx);
+            gen_helper_subuh_ob(cpu_gpr[ret], v1_t, v2_t);
+            break;
+        case OPC_SUBUH_R_OB:
+            check_dspr2(ctx);
+            gen_helper_subuh_r_ob(cpu_gpr[ret], v1_t, v2_t);
+            break;
+        case OPC_ADDQ_PW:
+            check_dsp(ctx);
+            gen_helper_addq_pw(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_ADDQ_S_PW:
+            check_dsp(ctx);
+            gen_helper_addq_s_pw(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_ADDQ_QH:
+            check_dsp(ctx);
+            gen_helper_addq_qh(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_ADDQ_S_QH:
+            check_dsp(ctx);
+            gen_helper_addq_s_qh(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_ADDU_OB:
+            check_dsp(ctx);
+            gen_helper_addu_ob(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_ADDU_S_OB:
+            check_dsp(ctx);
+            gen_helper_addu_s_ob(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_ADDU_QH:
+            check_dspr2(ctx);
+            gen_helper_addu_qh(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_ADDU_S_QH:
+            check_dspr2(ctx);
+            gen_helper_addu_s_qh(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_ADDUH_OB:
+            check_dspr2(ctx);
+            gen_helper_adduh_ob(cpu_gpr[ret], v1_t, v2_t);
+            break;
+        case OPC_ADDUH_R_OB:
+            check_dspr2(ctx);
+            gen_helper_adduh_r_ob(cpu_gpr[ret], v1_t, v2_t);
+            break;
+        }
+        break;
+    case OPC_CMPU_EQ_OB_DSP:
+        switch (op2) {
+        case OPC_PRECR_OB_QH:
+            check_dspr2(ctx);
+            gen_helper_precr_ob_qh(cpu_gpr[ret], v1_t, v2_t);
+            break;
+        case OPC_PRECR_SRA_QH_PW:
+            check_dspr2(ctx);
+            {
+                TCGv_i32 ret_t = tcg_const_i32(ret);
+                gen_helper_precr_sra_qh_pw(v2_t, v1_t, v2_t, ret_t);
+                tcg_temp_free_i32(ret_t);
+                break;
+            }
+        case OPC_PRECR_SRA_R_QH_PW:
+            check_dspr2(ctx);
+            {
+                TCGv_i32 sa_v = tcg_const_i32(ret);
+                gen_helper_precr_sra_r_qh_pw(v2_t, v1_t, v2_t, sa_v);
+                tcg_temp_free_i32(sa_v);
+                break;
+            }
+        case OPC_PRECRQ_OB_QH:
+            check_dsp(ctx);
+            gen_helper_precrq_ob_qh(cpu_gpr[ret], v1_t, v2_t);
+            break;
+        case OPC_PRECRQ_PW_L:
+            check_dsp(ctx);
+            gen_helper_precrq_pw_l(cpu_gpr[ret], v1_t, v2_t);
+            break;
+        case OPC_PRECRQ_QH_PW:
+            check_dsp(ctx);
+            gen_helper_precrq_qh_pw(cpu_gpr[ret], v1_t, v2_t);
+            break;
+        case OPC_PRECRQ_RS_QH_PW:
+            check_dsp(ctx);
+            gen_helper_precrq_rs_qh_pw(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_PRECRQU_S_OB_QH:
+            check_dsp(ctx);
+            gen_helper_precrqu_s_ob_qh(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        }
+        break;
+#endif
+    }
+
+    tcg_temp_free(v1_t);
+    tcg_temp_free(v2_t);
+
+    (void)opn; /* avoid a compiler warning */
+    MIPS_DEBUG("%s", opn);
+}
+
+static void gen_mipsdsp_shift(DisasContext *ctx, uint32_t opc,
+                              int ret, int v1, int v2)
+{
+    uint32_t op2;
+    const char *opn = "mipsdsp shift";
+    TCGv t0;
+    TCGv v1_t;
+    TCGv v2_t;
+
+    if (ret == 0) {
+        /* Treat as NOP. */
+        MIPS_DEBUG("NOP");
+        return;
+    }
+
+    t0 = tcg_temp_new();
+    v1_t = tcg_temp_new();
+    v2_t = tcg_temp_new();
+
+    tcg_gen_movi_tl(t0, v1);
+    gen_load_gpr(v1_t, v1);
+    gen_load_gpr(v2_t, v2);
+
+    switch (opc) {
+    case OPC_SHLL_QB_DSP:
+        {
+            op2 = MASK_SHLL_QB(ctx->opcode);
+            switch (op2) {
+            case OPC_SHLL_QB:
+                check_dsp(ctx);
+                gen_helper_shll_qb(cpu_gpr[ret], t0, v2_t, cpu_env);
+                break;
+            case OPC_SHLLV_QB:
+                check_dsp(ctx);
+                gen_helper_shll_qb(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+                break;
+            case OPC_SHLL_PH:
+                check_dsp(ctx);
+                gen_helper_shll_ph(cpu_gpr[ret], t0, v2_t, cpu_env);
+                break;
+            case OPC_SHLLV_PH:
+                check_dsp(ctx);
+                gen_helper_shll_ph(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+                break;
+            case OPC_SHLL_S_PH:
+                check_dsp(ctx);
+                gen_helper_shll_s_ph(cpu_gpr[ret], t0, v2_t, cpu_env);
+                break;
+            case OPC_SHLLV_S_PH:
+                check_dsp(ctx);
+                gen_helper_shll_s_ph(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+                break;
+            case OPC_SHLL_S_W:
+                check_dsp(ctx);
+                gen_helper_shll_s_w(cpu_gpr[ret], t0, v2_t, cpu_env);
+                break;
+            case OPC_SHLLV_S_W:
+                check_dsp(ctx);
+                gen_helper_shll_s_w(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+                break;
+            case OPC_SHRL_QB:
+                check_dsp(ctx);
+                gen_helper_shrl_qb(cpu_gpr[ret], t0, v2_t);
+                break;
+            case OPC_SHRLV_QB:
+                check_dsp(ctx);
+                gen_helper_shrl_qb(cpu_gpr[ret], v1_t, v2_t);
+                break;
+            case OPC_SHRL_PH:
+                check_dspr2(ctx);
+                gen_helper_shrl_ph(cpu_gpr[ret], t0, v2_t);
+                break;
+            case OPC_SHRLV_PH:
+                check_dspr2(ctx);
+                gen_helper_shrl_ph(cpu_gpr[ret], v1_t, v2_t);
+                break;
+            case OPC_SHRA_QB:
+                check_dspr2(ctx);
+                gen_helper_shra_qb(cpu_gpr[ret], t0, v2_t);
+                break;
+            case OPC_SHRA_R_QB:
+                check_dspr2(ctx);
+                gen_helper_shra_r_qb(cpu_gpr[ret], t0, v2_t);
+                break;
+            case OPC_SHRAV_QB:
+                check_dspr2(ctx);
+                gen_helper_shra_qb(cpu_gpr[ret], v1_t, v2_t);
+                break;
+            case OPC_SHRAV_R_QB:
+                check_dspr2(ctx);
+                gen_helper_shra_r_qb(cpu_gpr[ret], v1_t, v2_t);
+                break;
+            case OPC_SHRA_PH:
+                check_dsp(ctx);
+                gen_helper_shra_ph(cpu_gpr[ret], t0, v2_t);
+                break;
+            case OPC_SHRA_R_PH:
+                check_dsp(ctx);
+                gen_helper_shra_r_ph(cpu_gpr[ret], t0, v2_t);
+                break;
+            case OPC_SHRAV_PH:
+                check_dsp(ctx);
+                gen_helper_shra_ph(cpu_gpr[ret], v1_t, v2_t);
+                break;
+            case OPC_SHRAV_R_PH:
+                check_dsp(ctx);
+                gen_helper_shra_r_ph(cpu_gpr[ret], v1_t, v2_t);
+                break;
+            case OPC_SHRA_R_W:
+                check_dsp(ctx);
+                gen_helper_shra_r_w(cpu_gpr[ret], t0, v2_t);
+                break;
+            case OPC_SHRAV_R_W:
+                check_dsp(ctx);
+                gen_helper_shra_r_w(cpu_gpr[ret], v1_t, v2_t);
+                break;
+            default:            /* Invalid */
+                MIPS_INVAL("MASK SHLL.QB");
+                generate_exception(ctx, EXCP_RI);
+                break;
+            }
+            break;
+        }
+#ifdef TARGET_MIPS64
+    case OPC_SHLL_OB_DSP:
+        op2 = MASK_SHLL_OB(ctx->opcode);
+        switch (op2) {
+        case OPC_SHLL_PW:
+            check_dsp(ctx);
+            gen_helper_shll_pw(cpu_gpr[ret], v2_t, t0, cpu_env);
+            break;
+        case OPC_SHLLV_PW:
+            check_dsp(ctx);
+            gen_helper_shll_pw(cpu_gpr[ret], v2_t, v1_t, cpu_env);
+            break;
+        case OPC_SHLL_S_PW:
+            check_dsp(ctx);
+            gen_helper_shll_s_pw(cpu_gpr[ret], v2_t, t0, cpu_env);
+            break;
+        case OPC_SHLLV_S_PW:
+            check_dsp(ctx);
+            gen_helper_shll_s_pw(cpu_gpr[ret], v2_t, v1_t, cpu_env);
+            break;
+        case OPC_SHLL_OB:
+            check_dsp(ctx);
+            gen_helper_shll_ob(cpu_gpr[ret], v2_t, t0, cpu_env);
+            break;
+        case OPC_SHLLV_OB:
+            check_dsp(ctx);
+            gen_helper_shll_ob(cpu_gpr[ret], v2_t, v1_t, cpu_env);
+            break;
+        case OPC_SHLL_QH:
+            check_dsp(ctx);
+            gen_helper_shll_qh(cpu_gpr[ret], v2_t, t0, cpu_env);
+            break;
+        case OPC_SHLLV_QH:
+            check_dsp(ctx);
+            gen_helper_shll_qh(cpu_gpr[ret], v2_t, v1_t, cpu_env);
+            break;
+        case OPC_SHLL_S_QH:
+            check_dsp(ctx);
+            gen_helper_shll_s_qh(cpu_gpr[ret], v2_t, t0, cpu_env);
+            break;
+        case OPC_SHLLV_S_QH:
+            check_dsp(ctx);
+            gen_helper_shll_s_qh(cpu_gpr[ret], v2_t, v1_t, cpu_env);
+            break;
+        case OPC_SHRA_OB:
+            check_dspr2(ctx);
+            gen_helper_shra_ob(cpu_gpr[ret], v2_t, t0);
+            break;
+        case OPC_SHRAV_OB:
+            check_dspr2(ctx);
+            gen_helper_shra_ob(cpu_gpr[ret], v2_t, v1_t);
+            break;
+        case OPC_SHRA_R_OB:
+            check_dspr2(ctx);
+            gen_helper_shra_r_ob(cpu_gpr[ret], v2_t, t0);
+            break;
+        case OPC_SHRAV_R_OB:
+            check_dspr2(ctx);
+            gen_helper_shra_r_ob(cpu_gpr[ret], v2_t, v1_t);
+            break;
+        case OPC_SHRA_PW:
+            check_dsp(ctx);
+            gen_helper_shra_pw(cpu_gpr[ret], v2_t, t0);
+            break;
+        case OPC_SHRAV_PW:
+            check_dsp(ctx);
+            gen_helper_shra_pw(cpu_gpr[ret], v2_t, v1_t);
+            break;
+        case OPC_SHRA_R_PW:
+            check_dsp(ctx);
+            gen_helper_shra_r_pw(cpu_gpr[ret], v2_t, t0);
+            break;
+        case OPC_SHRAV_R_PW:
+            check_dsp(ctx);
+            gen_helper_shra_r_pw(cpu_gpr[ret], v2_t, v1_t);
+            break;
+        case OPC_SHRA_QH:
+            check_dsp(ctx);
+            gen_helper_shra_qh(cpu_gpr[ret], v2_t, t0);
+            break;
+        case OPC_SHRAV_QH:
+            check_dsp(ctx);
+            gen_helper_shra_qh(cpu_gpr[ret], v2_t, v1_t);
+            break;
+        case OPC_SHRA_R_QH:
+            check_dsp(ctx);
+            gen_helper_shra_r_qh(cpu_gpr[ret], v2_t, t0);
+            break;
+        case OPC_SHRAV_R_QH:
+            check_dsp(ctx);
+            gen_helper_shra_r_qh(cpu_gpr[ret], v2_t, v1_t);
+            break;
+        case OPC_SHRL_OB:
+            check_dsp(ctx);
+            gen_helper_shrl_ob(cpu_gpr[ret], v2_t, t0);
+            break;
+        case OPC_SHRLV_OB:
+            check_dsp(ctx);
+            gen_helper_shrl_ob(cpu_gpr[ret], v2_t, v1_t);
+            break;
+        case OPC_SHRL_QH:
+            check_dspr2(ctx);
+            gen_helper_shrl_qh(cpu_gpr[ret], v2_t, t0);
+            break;
+        case OPC_SHRLV_QH:
+            check_dspr2(ctx);
+            gen_helper_shrl_qh(cpu_gpr[ret], v2_t, v1_t);
+            break;
+        default:            /* Invalid */
+            MIPS_INVAL("MASK SHLL.OB");
+            generate_exception(ctx, EXCP_RI);
+            break;
+        }
+        break;
+#endif
+    }
+
+    tcg_temp_free(t0);
+    tcg_temp_free(v1_t);
+    tcg_temp_free(v2_t);
+    (void)opn; /* avoid a compiler warning */
+    MIPS_DEBUG("%s", opn);
+}
+
+static void gen_mipsdsp_multiply(DisasContext *ctx, uint32_t op1, uint32_t op2,
+                                 int ret, int v1, int v2, int check_ret)
+{
+    const char *opn = "mipsdsp multiply";
+    TCGv_i32 t0;
+    TCGv v1_t;
+    TCGv v2_t;
+
+    if ((ret == 0) && (check_ret == 1)) {
+        /* Treat as NOP. */
+        MIPS_DEBUG("NOP");
+        return;
+    }
+
+    t0 = tcg_temp_new_i32();
+    v1_t = tcg_temp_new();
+    v2_t = tcg_temp_new();
+
+    tcg_gen_movi_i32(t0, ret);
+    gen_load_gpr(v1_t, v1);
+    gen_load_gpr(v2_t, v2);
+
+    switch (op1) {
+    /* OPC_MULT_G_2E, OPC_ADDUH_QB_DSP, OPC_MUL_PH_DSP have
+     * the same mask and op1. */
+    case OPC_MULT_G_2E:
+        switch (op2) {
+        case  OPC_MUL_PH:
+            gen_helper_mul_ph(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case  OPC_MUL_S_PH:
+            gen_helper_mul_s_ph(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_MULQ_S_W:
+            gen_helper_mulq_s_w(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_MULQ_RS_W:
+            gen_helper_mulq_rs_w(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        }
+        break;
+    case OPC_DPA_W_PH_DSP:
+        switch (op2) {
+        case OPC_DPAU_H_QBL:
+            check_dsp(ctx);
+            gen_helper_dpau_h_qbl(t0, v1_t, v2_t, cpu_env);
+            break;
+        case OPC_DPAU_H_QBR:
+            check_dsp(ctx);
+            gen_helper_dpau_h_qbr(t0, v1_t, v2_t, cpu_env);
+            break;
+        case OPC_DPSU_H_QBL:
+            check_dsp(ctx);
+            gen_helper_dpsu_h_qbl(t0, v1_t, v2_t, cpu_env);
+            break;
+        case OPC_DPSU_H_QBR:
+            check_dsp(ctx);
+            gen_helper_dpsu_h_qbr(t0, v1_t, v2_t, cpu_env);
+            break;
+        case OPC_DPA_W_PH:
+            check_dspr2(ctx);
+            gen_helper_dpa_w_ph(t0, v1_t, v2_t, cpu_env);
+            break;
+        case OPC_DPAX_W_PH:
+            check_dspr2(ctx);
+            gen_helper_dpax_w_ph(t0, v1_t, v2_t, cpu_env);
+            break;
+        case OPC_DPAQ_S_W_PH:
+            check_dsp(ctx);
+            gen_helper_dpaq_s_w_ph(t0, v1_t, v2_t, cpu_env);
+            break;
+        case OPC_DPAQX_S_W_PH:
+            check_dspr2(ctx);
+            gen_helper_dpaqx_s_w_ph(t0, v1_t, v2_t, cpu_env);
+            break;
+        case OPC_DPAQX_SA_W_PH:
+            check_dspr2(ctx);
+            gen_helper_dpaqx_sa_w_ph(t0, v1_t, v2_t, cpu_env);
+            break;
+        case OPC_DPS_W_PH:
+            check_dspr2(ctx);
+            gen_helper_dps_w_ph(t0, v1_t, v2_t, cpu_env);
+            break;
+        case OPC_DPSX_W_PH:
+            check_dspr2(ctx);
+            gen_helper_dpsx_w_ph(t0, v1_t, v2_t, cpu_env);
+            break;
+        case OPC_DPSQ_S_W_PH:
+            check_dsp(ctx);
+            gen_helper_dpsq_s_w_ph(t0, v1_t, v2_t, cpu_env);
+            break;
+        case OPC_DPSQX_S_W_PH:
+            check_dspr2(ctx);
+            gen_helper_dpsqx_s_w_ph(t0, v1_t, v2_t, cpu_env);
+            break;
+        case OPC_DPSQX_SA_W_PH:
+            check_dspr2(ctx);
+            gen_helper_dpsqx_sa_w_ph(t0, v1_t, v2_t, cpu_env);
+            break;
+        case OPC_MULSAQ_S_W_PH:
+            check_dsp(ctx);
+            gen_helper_mulsaq_s_w_ph(t0, v1_t, v2_t, cpu_env);
+            break;
+        case OPC_DPAQ_SA_L_W:
+            check_dsp(ctx);
+            gen_helper_dpaq_sa_l_w(t0, v1_t, v2_t, cpu_env);
+            break;
+        case OPC_DPSQ_SA_L_W:
+            check_dsp(ctx);
+            gen_helper_dpsq_sa_l_w(t0, v1_t, v2_t, cpu_env);
+            break;
+        case OPC_MAQ_S_W_PHL:
+            check_dsp(ctx);
+            gen_helper_maq_s_w_phl(t0, v1_t, v2_t, cpu_env);
+            break;
+        case OPC_MAQ_S_W_PHR:
+            check_dsp(ctx);
+            gen_helper_maq_s_w_phr(t0, v1_t, v2_t, cpu_env);
+            break;
+        case OPC_MAQ_SA_W_PHL:
+            check_dsp(ctx);
+            gen_helper_maq_sa_w_phl(t0, v1_t, v2_t, cpu_env);
+            break;
+        case OPC_MAQ_SA_W_PHR:
+            check_dsp(ctx);
+            gen_helper_maq_sa_w_phr(t0, v1_t, v2_t, cpu_env);
+            break;
+        case OPC_MULSA_W_PH:
+            check_dspr2(ctx);
+            gen_helper_mulsa_w_ph(t0, v1_t, v2_t, cpu_env);
+            break;
+        }
+        break;
+#ifdef TARGET_MIPS64
+    case OPC_DPAQ_W_QH_DSP:
+        {
+            int ac = ret & 0x03;
+            tcg_gen_movi_i32(t0, ac);
+
+            switch (op2) {
+            case OPC_DMADD:
+                check_dsp(ctx);
+                gen_helper_dmadd(v1_t, v2_t, t0, cpu_env);
+                break;
+            case OPC_DMADDU:
+                check_dsp(ctx);
+                gen_helper_dmaddu(v1_t, v2_t, t0, cpu_env);
+                break;
+            case OPC_DMSUB:
+                check_dsp(ctx);
+                gen_helper_dmsub(v1_t, v2_t, t0, cpu_env);
+                break;
+            case OPC_DMSUBU:
+                check_dsp(ctx);
+                gen_helper_dmsubu(v1_t, v2_t, t0, cpu_env);
+                break;
+            case OPC_DPA_W_QH:
+                check_dspr2(ctx);
+                gen_helper_dpa_w_qh(v1_t, v2_t, t0, cpu_env);
+                break;
+            case OPC_DPAQ_S_W_QH:
+                check_dsp(ctx);
+                gen_helper_dpaq_s_w_qh(v1_t, v2_t, t0, cpu_env);
+                break;
+            case OPC_DPAQ_SA_L_PW:
+                check_dsp(ctx);
+                gen_helper_dpaq_sa_l_pw(v1_t, v2_t, t0, cpu_env);
+                break;
+            case OPC_DPAU_H_OBL:
+                check_dsp(ctx);
+                gen_helper_dpau_h_obl(v1_t, v2_t, t0, cpu_env);
+                break;
+            case OPC_DPAU_H_OBR:
+                check_dsp(ctx);
+                gen_helper_dpau_h_obr(v1_t, v2_t, t0, cpu_env);
+                break;
+            case OPC_DPS_W_QH:
+                check_dspr2(ctx);
+                gen_helper_dps_w_qh(v1_t, v2_t, t0, cpu_env);
+                break;
+            case OPC_DPSQ_S_W_QH:
+                check_dsp(ctx);
+                gen_helper_dpsq_s_w_qh(v1_t, v2_t, t0, cpu_env);
+                break;
+            case OPC_DPSQ_SA_L_PW:
+                check_dsp(ctx);
+                gen_helper_dpsq_sa_l_pw(v1_t, v2_t, t0, cpu_env);
+                break;
+            case OPC_DPSU_H_OBL:
+                check_dsp(ctx);
+                gen_helper_dpsu_h_obl(v1_t, v2_t, t0, cpu_env);
+                break;
+            case OPC_DPSU_H_OBR:
+                check_dsp(ctx);
+                gen_helper_dpsu_h_obr(v1_t, v2_t, t0, cpu_env);
+                break;
+            case OPC_MAQ_S_L_PWL:
+                check_dsp(ctx);
+                gen_helper_maq_s_l_pwl(v1_t, v2_t, t0, cpu_env);
+                break;
+            case OPC_MAQ_S_L_PWR:
+                check_dsp(ctx);
+                gen_helper_maq_s_l_pwr(v1_t, v2_t, t0, cpu_env);
+                break;
+            case OPC_MAQ_S_W_QHLL:
+                check_dsp(ctx);
+                gen_helper_maq_s_w_qhll(v1_t, v2_t, t0, cpu_env);
+                break;
+            case OPC_MAQ_SA_W_QHLL:
+                check_dsp(ctx);
+                gen_helper_maq_sa_w_qhll(v1_t, v2_t, t0, cpu_env);
+                break;
+            case OPC_MAQ_S_W_QHLR:
+                check_dsp(ctx);
+                gen_helper_maq_s_w_qhlr(v1_t, v2_t, t0, cpu_env);
+                break;
+            case OPC_MAQ_SA_W_QHLR:
+                check_dsp(ctx);
+                gen_helper_maq_sa_w_qhlr(v1_t, v2_t, t0, cpu_env);
+                break;
+            case OPC_MAQ_S_W_QHRL:
+                check_dsp(ctx);
+                gen_helper_maq_s_w_qhrl(v1_t, v2_t, t0, cpu_env);
+                break;
+            case OPC_MAQ_SA_W_QHRL:
+                check_dsp(ctx);
+                gen_helper_maq_sa_w_qhrl(v1_t, v2_t, t0, cpu_env);
+                break;
+            case OPC_MAQ_S_W_QHRR:
+                check_dsp(ctx);
+                gen_helper_maq_s_w_qhrr(v1_t, v2_t, t0, cpu_env);
+                break;
+            case OPC_MAQ_SA_W_QHRR:
+                check_dsp(ctx);
+                gen_helper_maq_sa_w_qhrr(v1_t, v2_t, t0, cpu_env);
+                break;
+            case OPC_MULSAQ_S_L_PW:
+                check_dsp(ctx);
+                gen_helper_mulsaq_s_l_pw(v1_t, v2_t, t0, cpu_env);
+                break;
+            case OPC_MULSAQ_S_W_QH:
+                check_dsp(ctx);
+                gen_helper_mulsaq_s_w_qh(v1_t, v2_t, t0, cpu_env);
+                break;
+            }
+        }
+        break;
+#endif
+    case OPC_ADDU_QB_DSP:
+        switch (op2) {
+        case OPC_MULEU_S_PH_QBL:
+            check_dsp(ctx);
+            gen_helper_muleu_s_ph_qbl(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_MULEU_S_PH_QBR:
+            check_dsp(ctx);
+            gen_helper_muleu_s_ph_qbr(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_MULQ_RS_PH:
+            check_dsp(ctx);
+            gen_helper_mulq_rs_ph(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_MULEQ_S_W_PHL:
+            check_dsp(ctx);
+            gen_helper_muleq_s_w_phl(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_MULEQ_S_W_PHR:
+            check_dsp(ctx);
+            gen_helper_muleq_s_w_phr(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_MULQ_S_PH:
+            check_dspr2(ctx);
+            gen_helper_mulq_s_ph(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        }
+        break;
+#ifdef TARGET_MIPS64
+    case OPC_ADDU_OB_DSP:
+        switch (op2) {
+        case OPC_MULEQ_S_PW_QHL:
+            check_dsp(ctx);
+            gen_helper_muleq_s_pw_qhl(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_MULEQ_S_PW_QHR:
+            check_dsp(ctx);
+            gen_helper_muleq_s_pw_qhr(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_MULEU_S_QH_OBL:
+            check_dsp(ctx);
+            gen_helper_muleu_s_qh_obl(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_MULEU_S_QH_OBR:
+            check_dsp(ctx);
+            gen_helper_muleu_s_qh_obr(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_MULQ_RS_QH:
+            check_dsp(ctx);
+            gen_helper_mulq_rs_qh(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        }
+        break;
+#endif
+    }
+
+    tcg_temp_free_i32(t0);
+    tcg_temp_free(v1_t);
+    tcg_temp_free(v2_t);
+
+    (void)opn; /* avoid a compiler warning */
+    MIPS_DEBUG("%s", opn);
+
+}
+
+static void gen_mipsdsp_bitinsn(CPUMIPSState *env, DisasContext *ctx,
+                                uint32_t op1, uint32_t op2,
+                                int ret, int val)
+{
+    const char *opn = "mipsdsp Bit/ Manipulation";
+    int16_t imm;
+    TCGv t0;
+    TCGv val_t;
+
+    if (ret == 0) {
+        /* Treat as NOP. */
+        MIPS_DEBUG("NOP");
+        return;
+    }
+
+    t0 = tcg_temp_new();
+    val_t = tcg_temp_new();
+    gen_load_gpr(val_t, val);
+
+    switch (op1) {
+    case OPC_ABSQ_S_PH_DSP:
+        switch (op2) {
+        case OPC_BITREV:
+            check_dsp(ctx);
+            gen_helper_bitrev(cpu_gpr[ret], val_t);
+            break;
+        case OPC_REPL_QB:
+            check_dsp(ctx);
+            {
+                target_long result;
+                imm = (ctx->opcode >> 16) & 0xFF;
+                result = (uint32_t)imm << 24 |
+                         (uint32_t)imm << 16 |
+                         (uint32_t)imm << 8  |
+                         (uint32_t)imm;
+                result = (int32_t)result;
+                tcg_gen_movi_tl(cpu_gpr[ret], result);
+            }
+            break;
+        case OPC_REPLV_QB:
+            check_dsp(ctx);
+            tcg_gen_ext8u_tl(cpu_gpr[ret], val_t);
+            tcg_gen_shli_tl(t0, cpu_gpr[ret], 8);
+            tcg_gen_or_tl(cpu_gpr[ret], cpu_gpr[ret], t0);
+            tcg_gen_shli_tl(t0, cpu_gpr[ret], 16);
+            tcg_gen_or_tl(cpu_gpr[ret], cpu_gpr[ret], t0);
+            tcg_gen_ext32s_tl(cpu_gpr[ret], cpu_gpr[ret]);
+            break;
+        case OPC_REPL_PH:
+            check_dsp(ctx);
+            {
+                imm = (ctx->opcode >> 16) & 0x03FF;
+                tcg_gen_movi_tl(cpu_gpr[ret], \
+                                (target_long)((int32_t)imm << 16 | \
+                                (uint32_t)(uint16_t)imm));
+            }
+            break;
+        case OPC_REPLV_PH:
+            check_dsp(ctx);
+            tcg_gen_ext16u_tl(cpu_gpr[ret], val_t);
+            tcg_gen_shli_tl(t0, cpu_gpr[ret], 16);
+            tcg_gen_or_tl(cpu_gpr[ret], cpu_gpr[ret], t0);
+            tcg_gen_ext32s_tl(cpu_gpr[ret], cpu_gpr[ret]);
+            break;
+        }
+        break;
+#ifdef TARGET_MIPS64
+    case OPC_ABSQ_S_QH_DSP:
+        switch (op2) {
+        case OPC_REPL_OB:
+            check_dsp(ctx);
+            {
+                target_long temp;
+
+                imm = (ctx->opcode >> 16) & 0xFF;
+                temp = ((uint64_t)imm << 8) | (uint64_t)imm;
+                temp = (temp << 16) | temp;
+                temp = (temp << 32) | temp;
+                tcg_gen_movi_tl(cpu_gpr[ret], temp);
+                break;
+            }
+        case OPC_REPL_PW:
+            check_dsp(ctx);
+            {
+                target_long temp;
+
+                imm = (ctx->opcode >> 16) & 0x03FF;
+                imm = (int16_t)(imm << 6) >> 6;
+                temp = ((target_long)imm << 32) \
+                       | ((target_long)imm & 0xFFFFFFFF);
+                tcg_gen_movi_tl(cpu_gpr[ret], temp);
+                break;
+            }
+        case OPC_REPL_QH:
+            check_dsp(ctx);
+            {
+                target_long temp;
+
+                imm = (ctx->opcode >> 16) & 0x03FF;
+                imm = (int16_t)(imm << 6) >> 6;
+
+                temp = ((uint64_t)(uint16_t)imm << 48) |
+                       ((uint64_t)(uint16_t)imm << 32) |
+                       ((uint64_t)(uint16_t)imm << 16) |
+                       (uint64_t)(uint16_t)imm;
+                tcg_gen_movi_tl(cpu_gpr[ret], temp);
+                break;
+            }
+        case OPC_REPLV_OB:
+            check_dsp(ctx);
+            tcg_gen_ext8u_tl(cpu_gpr[ret], val_t);
+            tcg_gen_shli_tl(t0, cpu_gpr[ret], 8);
+            tcg_gen_or_tl(cpu_gpr[ret], cpu_gpr[ret], t0);
+            tcg_gen_shli_tl(t0, cpu_gpr[ret], 16);
+            tcg_gen_or_tl(cpu_gpr[ret], cpu_gpr[ret], t0);
+            tcg_gen_shli_tl(t0, cpu_gpr[ret], 32);
+            tcg_gen_or_tl(cpu_gpr[ret], cpu_gpr[ret], t0);
+            break;
+        case OPC_REPLV_PW:
+            check_dsp(ctx);
+            tcg_gen_ext32u_i64(cpu_gpr[ret], val_t);
+            tcg_gen_shli_tl(t0, cpu_gpr[ret], 32);
+            tcg_gen_or_tl(cpu_gpr[ret], cpu_gpr[ret], t0);
+            break;
+        case OPC_REPLV_QH:
+            check_dsp(ctx);
+            tcg_gen_ext16u_tl(cpu_gpr[ret], val_t);
+            tcg_gen_shli_tl(t0, cpu_gpr[ret], 16);
+            tcg_gen_or_tl(cpu_gpr[ret], cpu_gpr[ret], t0);
+            tcg_gen_shli_tl(t0, cpu_gpr[ret], 32);
+            tcg_gen_or_tl(cpu_gpr[ret], cpu_gpr[ret], t0);
+            break;
+        }
+        break;
+#endif
+    }
+    tcg_temp_free(t0);
+    tcg_temp_free(val_t);
+
+    (void)opn; /* avoid a compiler warning */
+    MIPS_DEBUG("%s", opn);
+}
+
+static void gen_mipsdsp_add_cmp_pick(DisasContext *ctx,
+                                     uint32_t op1, uint32_t op2,
+                                     int ret, int v1, int v2, int check_ret)
+{
+    const char *opn = "mipsdsp add compare pick";
+    TCGv_i32 t0;
+    TCGv t1;
+    TCGv v1_t;
+    TCGv v2_t;
+
+    if ((ret == 0) && (check_ret == 1)) {
+        /* Treat as NOP. */
+        MIPS_DEBUG("NOP");
+        return;
+    }
+
+    t0 = tcg_temp_new_i32();
+    t1 = tcg_temp_new();
+    v1_t = tcg_temp_new();
+    v2_t = tcg_temp_new();
+
+    gen_load_gpr(v1_t, v1);
+    gen_load_gpr(v2_t, v2);
+
+    switch (op1) {
+    case OPC_APPEND_DSP:
+        switch (op2) {
+        case OPC_APPEND:
+            tcg_gen_movi_i32(t0, v2);
+            gen_helper_append(cpu_gpr[ret], cpu_gpr[ret], v1_t, t0);
+            break;
+        case OPC_PREPEND:
+            tcg_gen_movi_i32(t0, v2);
+            gen_helper_prepend(cpu_gpr[ret], v1_t, cpu_gpr[ret], t0);
+            break;
+        case OPC_BALIGN:
+            tcg_gen_movi_i32(t0, v2);
+            gen_helper_balign(cpu_gpr[ret], v1_t, cpu_gpr[ret], t0);
+            break;
+        default:            /* Invid */
+            MIPS_INVAL("MASK APPEND");
+            generate_exception(ctx, EXCP_RI);
+            break;
+        }
+        break;
+    case OPC_CMPU_EQ_QB_DSP:
+        switch (op2) {
+        case OPC_CMPU_EQ_QB:
+            check_dsp(ctx);
+            gen_helper_cmpu_eq_qb(v1_t, v2_t, cpu_env);
+            break;
+        case OPC_CMPU_LT_QB:
+            check_dsp(ctx);
+            gen_helper_cmpu_lt_qb(v1_t, v2_t, cpu_env);
+            break;
+        case OPC_CMPU_LE_QB:
+            check_dsp(ctx);
+            gen_helper_cmpu_le_qb(v1_t, v2_t, cpu_env);
+            break;
+        case OPC_CMPGU_EQ_QB:
+            check_dsp(ctx);
+            gen_helper_cmpgu_eq_qb(cpu_gpr[ret], v1_t, v2_t);
+            break;
+        case OPC_CMPGU_LT_QB:
+            check_dsp(ctx);
+            gen_helper_cmpgu_lt_qb(cpu_gpr[ret], v1_t, v2_t);
+            break;
+        case OPC_CMPGU_LE_QB:
+            check_dsp(ctx);
+            gen_helper_cmpgu_le_qb(cpu_gpr[ret], v1_t, v2_t);
+            break;
+        case OPC_CMPGDU_EQ_QB:
+            check_dspr2(ctx);
+            gen_helper_cmpgu_eq_qb(t1, v1_t, v2_t);
+            tcg_gen_mov_tl(cpu_gpr[ret], t1);
+            tcg_gen_andi_tl(cpu_dspctrl, cpu_dspctrl, 0xF0FFFFFF);
+            tcg_gen_shli_tl(t1, t1, 24);
+            tcg_gen_or_tl(cpu_dspctrl, cpu_dspctrl, t1);
+            break;
+        case OPC_CMPGDU_LT_QB:
+            check_dspr2(ctx);
+            gen_helper_cmpgu_lt_qb(t1, v1_t, v2_t);
+            tcg_gen_mov_tl(cpu_gpr[ret], t1);
+            tcg_gen_andi_tl(cpu_dspctrl, cpu_dspctrl, 0xF0FFFFFF);
+            tcg_gen_shli_tl(t1, t1, 24);
+            tcg_gen_or_tl(cpu_dspctrl, cpu_dspctrl, t1);
+            break;
+        case OPC_CMPGDU_LE_QB:
+            check_dspr2(ctx);
+            gen_helper_cmpgu_le_qb(t1, v1_t, v2_t);
+            tcg_gen_mov_tl(cpu_gpr[ret], t1);
+            tcg_gen_andi_tl(cpu_dspctrl, cpu_dspctrl, 0xF0FFFFFF);
+            tcg_gen_shli_tl(t1, t1, 24);
+            tcg_gen_or_tl(cpu_dspctrl, cpu_dspctrl, t1);
+            break;
+        case OPC_CMP_EQ_PH:
+            check_dsp(ctx);
+            gen_helper_cmp_eq_ph(v1_t, v2_t, cpu_env);
+            break;
+        case OPC_CMP_LT_PH:
+            check_dsp(ctx);
+            gen_helper_cmp_lt_ph(v1_t, v2_t, cpu_env);
+            break;
+        case OPC_CMP_LE_PH:
+            check_dsp(ctx);
+            gen_helper_cmp_le_ph(v1_t, v2_t, cpu_env);
+            break;
+        case OPC_PICK_QB:
+            check_dsp(ctx);
+            gen_helper_pick_qb(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_PICK_PH:
+            check_dsp(ctx);
+            gen_helper_pick_ph(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_PACKRL_PH:
+            check_dsp(ctx);
+            gen_helper_packrl_ph(cpu_gpr[ret], v1_t, v2_t);
+            break;
+        }
+        break;
+#ifdef TARGET_MIPS64
+    case OPC_CMPU_EQ_OB_DSP:
+        switch (op2) {
+        case OPC_CMP_EQ_PW:
+            check_dsp(ctx);
+            gen_helper_cmp_eq_pw(v1_t, v2_t, cpu_env);
+            break;
+        case OPC_CMP_LT_PW:
+            check_dsp(ctx);
+            gen_helper_cmp_lt_pw(v1_t, v2_t, cpu_env);
+            break;
+        case OPC_CMP_LE_PW:
+            check_dsp(ctx);
+            gen_helper_cmp_le_pw(v1_t, v2_t, cpu_env);
+            break;
+        case OPC_CMP_EQ_QH:
+            check_dsp(ctx);
+            gen_helper_cmp_eq_qh(v1_t, v2_t, cpu_env);
+            break;
+        case OPC_CMP_LT_QH:
+            check_dsp(ctx);
+            gen_helper_cmp_lt_qh(v1_t, v2_t, cpu_env);
+            break;
+        case OPC_CMP_LE_QH:
+            check_dsp(ctx);
+            gen_helper_cmp_le_qh(v1_t, v2_t, cpu_env);
+            break;
+        case OPC_CMPGDU_EQ_OB:
+            check_dspr2(ctx);
+            gen_helper_cmpgdu_eq_ob(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_CMPGDU_LT_OB:
+            check_dspr2(ctx);
+            gen_helper_cmpgdu_lt_ob(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_CMPGDU_LE_OB:
+            check_dspr2(ctx);
+            gen_helper_cmpgdu_le_ob(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_CMPGU_EQ_OB:
+            check_dsp(ctx);
+            gen_helper_cmpgu_eq_ob(cpu_gpr[ret], v1_t, v2_t);
+            break;
+        case OPC_CMPGU_LT_OB:
+            check_dsp(ctx);
+            gen_helper_cmpgu_lt_ob(cpu_gpr[ret], v1_t, v2_t);
+            break;
+        case OPC_CMPGU_LE_OB:
+            check_dsp(ctx);
+            gen_helper_cmpgu_le_ob(cpu_gpr[ret], v1_t, v2_t);
+            break;
+        case OPC_CMPU_EQ_OB:
+            check_dsp(ctx);
+            gen_helper_cmpu_eq_ob(v1_t, v2_t, cpu_env);
+            break;
+        case OPC_CMPU_LT_OB:
+            check_dsp(ctx);
+            gen_helper_cmpu_lt_ob(v1_t, v2_t, cpu_env);
+            break;
+        case OPC_CMPU_LE_OB:
+            check_dsp(ctx);
+            gen_helper_cmpu_le_ob(v1_t, v2_t, cpu_env);
+            break;
+        case OPC_PACKRL_PW:
+            check_dsp(ctx);
+            gen_helper_packrl_pw(cpu_gpr[ret], v1_t, v2_t);
+            break;
+        case OPC_PICK_OB:
+            check_dsp(ctx);
+            gen_helper_pick_ob(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_PICK_PW:
+            check_dsp(ctx);
+            gen_helper_pick_pw(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        case OPC_PICK_QH:
+            check_dsp(ctx);
+            gen_helper_pick_qh(cpu_gpr[ret], v1_t, v2_t, cpu_env);
+            break;
+        }
+        break;
+    case OPC_DAPPEND_DSP:
+        switch (op2) {
+        case OPC_DAPPEND:
+            tcg_gen_movi_i32(t0, v2);
+            gen_helper_dappend(cpu_gpr[ret], v1_t, cpu_gpr[ret], t0);
+            break;
+        case OPC_PREPENDD:
+            tcg_gen_movi_i32(t0, v2);
+            gen_helper_prependd(cpu_gpr[ret], v1_t, cpu_gpr[ret], t0);
+            break;
+        case OPC_PREPENDW:
+            tcg_gen_movi_i32(t0, v2);
+            gen_helper_prependw(cpu_gpr[ret], v1_t, cpu_gpr[ret], t0);
+            break;
+        case OPC_DBALIGN:
+            tcg_gen_movi_i32(t0, v2);
+            gen_helper_dbalign(cpu_gpr[ret], v1_t, cpu_gpr[ret], t0);
+            break;
+        default:            /* Invalid */
+            MIPS_INVAL("MASK DAPPEND");
+            generate_exception(ctx, EXCP_RI);
+            break;
+        }
+        break;
+#endif
+    }
+
+    tcg_temp_free_i32(t0);
+    tcg_temp_free(t1);
+    tcg_temp_free(v1_t);
+    tcg_temp_free(v2_t);
+
+    (void)opn; /* avoid a compiler warning */
+    MIPS_DEBUG("%s", opn);
+}
+
+static void gen_mipsdsp_accinsn(DisasContext *ctx, uint32_t op1, uint32_t op2,
+                                int ret, int v1, int v2, int check_ret)
+
+{
+    const char *opn = "mipsdsp accumulator";
+    TCGv t0;
+    TCGv t1;
+    TCGv v1_t;
+    TCGv v2_t;
+    int16_t imm;
+
+    if ((ret == 0) && (check_ret == 1)) {
+        /* Treat as NOP. */
+        MIPS_DEBUG("NOP");
+        return;
+    }
+
+    t0 = tcg_temp_new();
+    t1 = tcg_temp_new();
+    v1_t = tcg_temp_new();
+    v2_t = tcg_temp_new();
+
+    gen_load_gpr(v1_t, v1);
+    gen_load_gpr(v2_t, v2);
+
+    switch (op1) {
+    case OPC_EXTR_W_DSP:
+        check_dsp(ctx);
+        switch (op2) {
+        case OPC_EXTR_W:
+            tcg_gen_movi_tl(t0, v2);
+            tcg_gen_movi_tl(t1, v1);
+            gen_helper_extr_w(cpu_gpr[ret], t0, t1, cpu_env);
+            break;
+        case OPC_EXTR_R_W:
+            tcg_gen_movi_tl(t0, v2);
+            tcg_gen_movi_tl(t1, v1);
+            gen_helper_extr_r_w(cpu_gpr[ret], t0, t1, cpu_env);
+            break;
+        case OPC_EXTR_RS_W:
+            tcg_gen_movi_tl(t0, v2);
+            tcg_gen_movi_tl(t1, v1);
+            gen_helper_extr_rs_w(cpu_gpr[ret], t0, t1, cpu_env);
+            break;
+        case OPC_EXTR_S_H:
+            tcg_gen_movi_tl(t0, v2);
+            tcg_gen_movi_tl(t1, v1);
+            gen_helper_extr_s_h(cpu_gpr[ret], t0, t1, cpu_env);
+            break;
+        case OPC_EXTRV_S_H:
+            tcg_gen_movi_tl(t0, v2);
+            gen_helper_extr_s_h(cpu_gpr[ret], t0, v1_t, cpu_env);
+            break;
+        case OPC_EXTRV_W:
+            tcg_gen_movi_tl(t0, v2);
+            gen_helper_extr_w(cpu_gpr[ret], t0, v1_t, cpu_env);
+            break;
+        case OPC_EXTRV_R_W:
+            tcg_gen_movi_tl(t0, v2);
+            gen_helper_extr_r_w(cpu_gpr[ret], t0, v1_t, cpu_env);
+            break;
+        case OPC_EXTRV_RS_W:
+            tcg_gen_movi_tl(t0, v2);
+            gen_helper_extr_rs_w(cpu_gpr[ret], t0, v1_t, cpu_env);
+            break;
+        case OPC_EXTP:
+            tcg_gen_movi_tl(t0, v2);
+            tcg_gen_movi_tl(t1, v1);
+            gen_helper_extp(cpu_gpr[ret], t0, t1, cpu_env);
+            break;
+        case OPC_EXTPV:
+            tcg_gen_movi_tl(t0, v2);
+            gen_helper_extp(cpu_gpr[ret], t0, v1_t, cpu_env);
+            break;
+        case OPC_EXTPDP:
+            tcg_gen_movi_tl(t0, v2);
+            tcg_gen_movi_tl(t1, v1);
+            gen_helper_extpdp(cpu_gpr[ret], t0, t1, cpu_env);
+            break;
+        case OPC_EXTPDPV:
+            tcg_gen_movi_tl(t0, v2);
+            gen_helper_extpdp(cpu_gpr[ret], t0, v1_t, cpu_env);
+            break;
+        case OPC_SHILO:
+            imm = (ctx->opcode >> 20) & 0x3F;
+            tcg_gen_movi_tl(t0, ret);
+            tcg_gen_movi_tl(t1, imm);
+            gen_helper_shilo(t0, t1, cpu_env);
+            break;
+        case OPC_SHILOV:
+            tcg_gen_movi_tl(t0, ret);
+            gen_helper_shilo(t0, v1_t, cpu_env);
+            break;
+        case OPC_MTHLIP:
+            tcg_gen_movi_tl(t0, ret);
+            gen_helper_mthlip(t0, v1_t, cpu_env);
+            break;
+        case OPC_WRDSP:
+            imm = (ctx->opcode >> 11) & 0x3FF;
+            tcg_gen_movi_tl(t0, imm);
+            gen_helper_wrdsp(v1_t, t0, cpu_env);
+            break;
+        case OPC_RDDSP:
+            imm = (ctx->opcode >> 16) & 0x03FF;
+            tcg_gen_movi_tl(t0, imm);
+            gen_helper_rddsp(cpu_gpr[ret], t0, cpu_env);
+            break;
+        }
+        break;
+#ifdef TARGET_MIPS64
+    case OPC_DEXTR_W_DSP:
+        check_dsp(ctx);
+        switch (op2) {
+        case OPC_DMTHLIP:
+            tcg_gen_movi_tl(t0, ret);
+            gen_helper_dmthlip(v1_t, t0, cpu_env);
+            break;
+        case OPC_DSHILO:
+            {
+                int shift = (ctx->opcode >> 19) & 0x7F;
+                int ac = (ctx->opcode >> 11) & 0x03;
+                tcg_gen_movi_tl(t0, shift);
+                tcg_gen_movi_tl(t1, ac);
+                gen_helper_dshilo(t0, t1, cpu_env);
+                break;
+            }
+        case OPC_DSHILOV:
+            {
+                int ac = (ctx->opcode >> 11) & 0x03;
+                tcg_gen_movi_tl(t0, ac);
+                gen_helper_dshilo(v1_t, t0, cpu_env);
+                break;
+            }
+        case OPC_DEXTP:
+            tcg_gen_movi_tl(t0, v2);
+            tcg_gen_movi_tl(t1, v1);
+
+            gen_helper_dextp(cpu_gpr[ret], t0, t1, cpu_env);
+            break;
+        case OPC_DEXTPV:
+            tcg_gen_movi_tl(t0, v2);
+            gen_helper_dextp(cpu_gpr[ret], t0, v1_t, cpu_env);
+            break;
+        case OPC_DEXTPDP:
+            tcg_gen_movi_tl(t0, v2);
+            tcg_gen_movi_tl(t1, v1);
+            gen_helper_dextpdp(cpu_gpr[ret], t0, t1, cpu_env);
+            break;
+        case OPC_DEXTPDPV:
+            tcg_gen_movi_tl(t0, v2);
+            gen_helper_dextpdp(cpu_gpr[ret], t0, v1_t, cpu_env);
+            break;
+        case OPC_DEXTR_L:
+            tcg_gen_movi_tl(t0, v2);
+            tcg_gen_movi_tl(t1, v1);
+            gen_helper_dextr_l(cpu_gpr[ret], t0, t1, cpu_env);
+            break;
+        case OPC_DEXTR_R_L:
+            tcg_gen_movi_tl(t0, v2);
+            tcg_gen_movi_tl(t1, v1);
+            gen_helper_dextr_r_l(cpu_gpr[ret], t0, t1, cpu_env);
+            break;
+        case OPC_DEXTR_RS_L:
+            tcg_gen_movi_tl(t0, v2);
+            tcg_gen_movi_tl(t1, v1);
+            gen_helper_dextr_rs_l(cpu_gpr[ret], t0, t1, cpu_env);
+            break;
+        case OPC_DEXTR_W:
+            tcg_gen_movi_tl(t0, v2);
+            tcg_gen_movi_tl(t1, v1);
+            gen_helper_dextr_w(cpu_gpr[ret], t0, t1, cpu_env);
+            break;
+        case OPC_DEXTR_R_W:
+            tcg_gen_movi_tl(t0, v2);
+            tcg_gen_movi_tl(t1, v1);
+            gen_helper_dextr_r_w(cpu_gpr[ret], t0, t1, cpu_env);
+            break;
+        case OPC_DEXTR_RS_W:
+            tcg_gen_movi_tl(t0, v2);
+            tcg_gen_movi_tl(t1, v1);
+            gen_helper_dextr_rs_w(cpu_gpr[ret], t0, t1, cpu_env);
+            break;
+        case OPC_DEXTR_S_H:
+            tcg_gen_movi_tl(t0, v2);
+            tcg_gen_movi_tl(t1, v1);
+            gen_helper_dextr_s_h(cpu_gpr[ret], t0, t1, cpu_env);
+            break;
+        case OPC_DEXTRV_S_H:
+            tcg_gen_movi_tl(t0, v2);
+            tcg_gen_movi_tl(t1, v1);
+            gen_helper_dextr_s_h(cpu_gpr[ret], t0, t1, cpu_env);
+            break;
+        case OPC_DEXTRV_L:
+            tcg_gen_movi_tl(t0, v2);
+            gen_helper_dextr_l(cpu_gpr[ret], t0, v1_t, cpu_env);
+            break;
+        case OPC_DEXTRV_R_L:
+            tcg_gen_movi_tl(t0, v2);
+            gen_helper_dextr_r_l(cpu_gpr[ret], t0, v1_t, cpu_env);
+            break;
+        case OPC_DEXTRV_RS_L:
+            tcg_gen_movi_tl(t0, v2);
+            gen_helper_dextr_rs_l(cpu_gpr[ret], t0, v1_t, cpu_env);
+            break;
+        case OPC_DEXTRV_W:
+            tcg_gen_movi_tl(t0, v2);
+            gen_helper_dextr_w(cpu_gpr[ret], t0, v1_t, cpu_env);
+            break;
+        case OPC_DEXTRV_R_W:
+            tcg_gen_movi_tl(t0, v2);
+            gen_helper_dextr_r_w(cpu_gpr[ret], t0, v1_t, cpu_env);
+            break;
+        case OPC_DEXTRV_RS_W:
+            tcg_gen_movi_tl(t0, v2);
+            gen_helper_dextr_rs_w(cpu_gpr[ret], t0, v1_t, cpu_env);
+            break;
+        }
+        break;
+#endif
+    }
+
+    tcg_temp_free(t0);
+    tcg_temp_free(t1);
+    tcg_temp_free(v1_t);
+    tcg_temp_free(v2_t);
+
+    (void)opn; /* avoid a compiler warning */
+    MIPS_DEBUG("%s", opn);
+}
+
+/* End MIPSDSP functions. */
+
 static void decode_opc (CPUMIPSState *env, DisasContext *ctx, int *is_branch)
 {
     int32_t offset;
@@ -11726,8 +14355,9 @@ static void decode_opc (CPUMIPSState *env, DisasContext *ctx, int *is_branch)
         gen_set_label(l1);
     }
 
-    if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP)))
+    if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
         tcg_gen_debug_insn_start(ctx->pc);
+    }
 
     op = MASK_OP_MAJOR(ctx->opcode);
     rs = (ctx->opcode >> 21) & 0x1f;
@@ -11763,7 +14393,7 @@ static void decode_opc (CPUMIPSState *env, DisasContext *ctx, int *is_branch)
         case OPC_MOVZ:
             check_insn(env, ctx, ISA_MIPS4 | ISA_MIPS32 |
                                  INSN_LOONGSON2E | INSN_LOONGSON2F);
-            gen_cond_move(env, op1, rd, rs, rt);
+            gen_cond_move(env, ctx, op1, rd, rs, rt);
             break;
         case OPC_ADD ... OPC_SUBU:
             gen_arith(env, ctx, op1, rd, rs, rt);
@@ -11790,13 +14420,13 @@ static void decode_opc (CPUMIPSState *env, DisasContext *ctx, int *is_branch)
             break;
         case OPC_SLT:          /* Set on less than */
         case OPC_SLTU:
-            gen_slt(env, op1, rd, rs, rt);
+            gen_slt(env, ctx, op1, rd, rs, rt);
             break;
         case OPC_AND:          /* Logic*/
         case OPC_OR:
         case OPC_NOR:
         case OPC_XOR:
-            gen_logic(env, op1, rd, rs, rt);
+            gen_logic(env, ctx, op1, rd, rs, rt);
             break;
         case OPC_MULT ... OPC_DIVU:
             if (sa) {
@@ -11827,7 +14457,7 @@ static void decode_opc (CPUMIPSState *env, DisasContext *ctx, int *is_branch)
             MIPS_INVAL("PMON / selsl");
             generate_exception(ctx, EXCP_RI);
 #else
-            gen_helper_0i(pmon, sa);
+            gen_helper_0e0i(pmon, sa);
 #endif
             break;
         case OPC_SYSCALL:
@@ -12045,16 +14675,278 @@ static void decode_opc (CPUMIPSState *env, DisasContext *ctx, int *is_branch)
 
                 save_cpu_state(ctx, 1);
                 gen_load_gpr(t0, rs);
-                gen_helper_yield(t0, t0);
+                gen_helper_yield(t0, cpu_env, t0);
                 gen_store_gpr(t0, rd);
                 tcg_temp_free(t0);
             }
             break;
         case OPC_DIV_G_2E ... OPC_DIVU_G_2E:
-        case OPC_MULT_G_2E ... OPC_MULTU_G_2E:
         case OPC_MOD_G_2E ... OPC_MODU_G_2E:
-            check_insn(env, ctx, INSN_LOONGSON2E);
-            gen_loongson_integer(ctx, op1, rd, rs, rt);
+        case OPC_MULT_G_2E ... OPC_MULTU_G_2E:
+        /* OPC_MULT_G_2E, OPC_ADDUH_QB_DSP, OPC_MUL_PH_DSP have
+         * the same mask and op1. */
+            if ((env->insn_flags & ASE_DSPR2) && (op1 == OPC_MULT_G_2E)) {
+                op2 = MASK_ADDUH_QB(ctx->opcode);
+                switch (op2) {
+                case OPC_ADDUH_QB:
+                case OPC_ADDUH_R_QB:
+                case OPC_ADDQH_PH:
+                case OPC_ADDQH_R_PH:
+                case OPC_ADDQH_W:
+                case OPC_ADDQH_R_W:
+                case OPC_SUBUH_QB:
+                case OPC_SUBUH_R_QB:
+                case OPC_SUBQH_PH:
+                case OPC_SUBQH_R_PH:
+                case OPC_SUBQH_W:
+                case OPC_SUBQH_R_W:
+                    gen_mipsdsp_arith(ctx, op1, op2, rd, rs, rt);
+                    break;
+                case OPC_MUL_PH:
+                case OPC_MUL_S_PH:
+                case OPC_MULQ_S_W:
+                case OPC_MULQ_RS_W:
+                    gen_mipsdsp_multiply(ctx, op1, op2, rd, rs, rt, 1);
+                    break;
+                default:
+                    MIPS_INVAL("MASK ADDUH.QB");
+                    generate_exception(ctx, EXCP_RI);
+                    break;
+                }
+            } else if (env->insn_flags & INSN_LOONGSON2E) {
+                gen_loongson_integer(ctx, op1, rd, rs, rt);
+            } else {
+                generate_exception(ctx, EXCP_RI);
+            }
+            break;
+        case OPC_LX_DSP:
+            op2 = MASK_LX(ctx->opcode);
+            switch (op2) {
+#if defined(TARGET_MIPS64)
+            case OPC_LDX:
+#endif
+            case OPC_LBUX:
+            case OPC_LHX:
+            case OPC_LWX:
+                gen_mipsdsp_ld(env, ctx, op2, rd, rs, rt);
+                break;
+            default:            /* Invalid */
+                MIPS_INVAL("MASK LX");
+                generate_exception(ctx, EXCP_RI);
+                break;
+            }
+            break;
+        case OPC_ABSQ_S_PH_DSP:
+            op2 = MASK_ABSQ_S_PH(ctx->opcode);
+            switch (op2) {
+            case OPC_ABSQ_S_QB:
+            case OPC_ABSQ_S_PH:
+            case OPC_ABSQ_S_W:
+            case OPC_PRECEQ_W_PHL:
+            case OPC_PRECEQ_W_PHR:
+            case OPC_PRECEQU_PH_QBL:
+            case OPC_PRECEQU_PH_QBR:
+            case OPC_PRECEQU_PH_QBLA:
+            case OPC_PRECEQU_PH_QBRA:
+            case OPC_PRECEU_PH_QBL:
+            case OPC_PRECEU_PH_QBR:
+            case OPC_PRECEU_PH_QBLA:
+            case OPC_PRECEU_PH_QBRA:
+                gen_mipsdsp_arith(ctx, op1, op2, rd, rs, rt);
+                break;
+            case OPC_BITREV:
+            case OPC_REPL_QB:
+            case OPC_REPLV_QB:
+            case OPC_REPL_PH:
+            case OPC_REPLV_PH:
+                gen_mipsdsp_bitinsn(env, ctx, op1, op2, rd, rt);
+                break;
+            default:
+                MIPS_INVAL("MASK ABSQ_S.PH");
+                generate_exception(ctx, EXCP_RI);
+                break;
+            }
+            break;
+        case OPC_ADDU_QB_DSP:
+            op2 = MASK_ADDU_QB(ctx->opcode);
+            switch (op2) {
+            case OPC_ADDQ_PH:
+            case OPC_ADDQ_S_PH:
+            case OPC_ADDQ_S_W:
+            case OPC_ADDU_QB:
+            case OPC_ADDU_S_QB:
+            case OPC_ADDU_PH:
+            case OPC_ADDU_S_PH:
+            case OPC_SUBQ_PH:
+            case OPC_SUBQ_S_PH:
+            case OPC_SUBQ_S_W:
+            case OPC_SUBU_QB:
+            case OPC_SUBU_S_QB:
+            case OPC_SUBU_PH:
+            case OPC_SUBU_S_PH:
+            case OPC_ADDSC:
+            case OPC_ADDWC:
+            case OPC_MODSUB:
+            case OPC_RADDU_W_QB:
+                gen_mipsdsp_arith(ctx, op1, op2, rd, rs, rt);
+                break;
+            case OPC_MULEU_S_PH_QBL:
+            case OPC_MULEU_S_PH_QBR:
+            case OPC_MULQ_RS_PH:
+            case OPC_MULEQ_S_W_PHL:
+            case OPC_MULEQ_S_W_PHR:
+            case OPC_MULQ_S_PH:
+                gen_mipsdsp_multiply(ctx, op1, op2, rd, rs, rt, 1);
+                break;
+            default:            /* Invalid */
+                MIPS_INVAL("MASK ADDU.QB");
+                generate_exception(ctx, EXCP_RI);
+                break;
+
+            }
+            break;
+        case OPC_CMPU_EQ_QB_DSP:
+            op2 = MASK_CMPU_EQ_QB(ctx->opcode);
+            switch (op2) {
+            case OPC_PRECR_SRA_PH_W:
+            case OPC_PRECR_SRA_R_PH_W:
+                gen_mipsdsp_arith(ctx, op1, op2, rt, rs, rd);
+                break;
+            case OPC_PRECR_QB_PH:
+            case OPC_PRECRQ_QB_PH:
+            case OPC_PRECRQ_PH_W:
+            case OPC_PRECRQ_RS_PH_W:
+            case OPC_PRECRQU_S_QB_PH:
+                gen_mipsdsp_arith(ctx, op1, op2, rd, rs, rt);
+                break;
+            case OPC_CMPU_EQ_QB:
+            case OPC_CMPU_LT_QB:
+            case OPC_CMPU_LE_QB:
+            case OPC_CMP_EQ_PH:
+            case OPC_CMP_LT_PH:
+            case OPC_CMP_LE_PH:
+                gen_mipsdsp_add_cmp_pick(ctx, op1, op2, rd, rs, rt, 0);
+                break;
+            case OPC_CMPGU_EQ_QB:
+            case OPC_CMPGU_LT_QB:
+            case OPC_CMPGU_LE_QB:
+            case OPC_CMPGDU_EQ_QB:
+            case OPC_CMPGDU_LT_QB:
+            case OPC_CMPGDU_LE_QB:
+            case OPC_PICK_QB:
+            case OPC_PICK_PH:
+            case OPC_PACKRL_PH:
+                gen_mipsdsp_add_cmp_pick(ctx, op1, op2, rd, rs, rt, 1);
+                break;
+            default:            /* Invalid */
+                MIPS_INVAL("MASK CMPU.EQ.QB");
+                generate_exception(ctx, EXCP_RI);
+                break;
+            }
+            break;
+        case OPC_SHLL_QB_DSP:
+            gen_mipsdsp_shift(ctx, op1, rd, rs, rt);
+            break;
+        case OPC_DPA_W_PH_DSP:
+            op2 = MASK_DPA_W_PH(ctx->opcode);
+            switch (op2) {
+            case OPC_DPAU_H_QBL:
+            case OPC_DPAU_H_QBR:
+            case OPC_DPSU_H_QBL:
+            case OPC_DPSU_H_QBR:
+            case OPC_DPA_W_PH:
+            case OPC_DPAX_W_PH:
+            case OPC_DPAQ_S_W_PH:
+            case OPC_DPAQX_S_W_PH:
+            case OPC_DPAQX_SA_W_PH:
+            case OPC_DPS_W_PH:
+            case OPC_DPSX_W_PH:
+            case OPC_DPSQ_S_W_PH:
+            case OPC_DPSQX_S_W_PH:
+            case OPC_DPSQX_SA_W_PH:
+            case OPC_MULSAQ_S_W_PH:
+            case OPC_DPAQ_SA_L_W:
+            case OPC_DPSQ_SA_L_W:
+            case OPC_MAQ_S_W_PHL:
+            case OPC_MAQ_S_W_PHR:
+            case OPC_MAQ_SA_W_PHL:
+            case OPC_MAQ_SA_W_PHR:
+            case OPC_MULSA_W_PH:
+                gen_mipsdsp_multiply(ctx, op1, op2, rd, rs, rt, 0);
+                break;
+            default:            /* Invalid */
+                MIPS_INVAL("MASK DPAW.PH");
+                generate_exception(ctx, EXCP_RI);
+                break;
+            }
+            break;
+        case OPC_INSV_DSP:
+            op2 = MASK_INSV(ctx->opcode);
+            switch (op2) {
+            case OPC_INSV:
+                check_dsp(ctx);
+                {
+                    TCGv t0, t1;
+
+                    if (rt == 0) {
+                        MIPS_DEBUG("NOP");
+                        break;
+                    }
+
+                    t0 = tcg_temp_new();
+                    t1 = tcg_temp_new();
+
+                    gen_load_gpr(t0, rt);
+                    gen_load_gpr(t1, rs);
+
+                    gen_helper_insv(cpu_gpr[rt], cpu_env, t1, t0);
+
+                    tcg_temp_free(t0);
+                    tcg_temp_free(t1);
+                    break;
+                }
+            default:            /* Invalid */
+                MIPS_INVAL("MASK INSV");
+                generate_exception(ctx, EXCP_RI);
+                break;
+            }
+            break;
+        case OPC_APPEND_DSP:
+            check_dspr2(ctx);
+            op2 = MASK_APPEND(ctx->opcode);
+            gen_mipsdsp_add_cmp_pick(ctx, op1, op2, rt, rs, rd, 1);
+            break;
+        case OPC_EXTR_W_DSP:
+            op2 = MASK_EXTR_W(ctx->opcode);
+            switch (op2) {
+            case OPC_EXTR_W:
+            case OPC_EXTR_R_W:
+            case OPC_EXTR_RS_W:
+            case OPC_EXTR_S_H:
+            case OPC_EXTRV_S_H:
+            case OPC_EXTRV_W:
+            case OPC_EXTRV_R_W:
+            case OPC_EXTRV_RS_W:
+            case OPC_EXTP:
+            case OPC_EXTPV:
+            case OPC_EXTPDP:
+            case OPC_EXTPDPV:
+                gen_mipsdsp_accinsn(ctx, op1, op2, rt, rs, rd, 1);
+                break;
+            case OPC_RDDSP:
+                gen_mipsdsp_accinsn(ctx, op1, op2, rd, rs, rt, 1);
+                break;
+            case OPC_SHILO:
+            case OPC_SHILOV:
+            case OPC_MTHLIP:
+            case OPC_WRDSP:
+                gen_mipsdsp_accinsn(ctx, op1, op2, rd, rs, rt, 0);
+                break;
+            default:            /* Invalid */
+                MIPS_INVAL("MASK EXTR.W");
+                generate_exception(ctx, EXCP_RI);
+                break;
+            }
             break;
 #if defined(TARGET_MIPS64)
         case OPC_DEXTM ... OPC_DEXT:
@@ -12075,6 +14967,235 @@ static void decode_opc (CPUMIPSState *env, DisasContext *ctx, int *is_branch)
             check_insn(env, ctx, INSN_LOONGSON2E);
             gen_loongson_integer(ctx, op1, rd, rs, rt);
             break;
+        case OPC_ABSQ_S_QH_DSP:
+            op2 = MASK_ABSQ_S_QH(ctx->opcode);
+            switch (op2) {
+            case OPC_PRECEQ_L_PWL:
+            case OPC_PRECEQ_L_PWR:
+            case OPC_PRECEQ_PW_QHL:
+            case OPC_PRECEQ_PW_QHR:
+            case OPC_PRECEQ_PW_QHLA:
+            case OPC_PRECEQ_PW_QHRA:
+            case OPC_PRECEQU_QH_OBL:
+            case OPC_PRECEQU_QH_OBR:
+            case OPC_PRECEQU_QH_OBLA:
+            case OPC_PRECEQU_QH_OBRA:
+            case OPC_PRECEU_QH_OBL:
+            case OPC_PRECEU_QH_OBR:
+            case OPC_PRECEU_QH_OBLA:
+            case OPC_PRECEU_QH_OBRA:
+            case OPC_ABSQ_S_OB:
+            case OPC_ABSQ_S_PW:
+            case OPC_ABSQ_S_QH:
+                gen_mipsdsp_arith(ctx, op1, op2, rd, rs, rt);
+                break;
+            case OPC_REPL_OB:
+            case OPC_REPL_PW:
+            case OPC_REPL_QH:
+            case OPC_REPLV_OB:
+            case OPC_REPLV_PW:
+            case OPC_REPLV_QH:
+                gen_mipsdsp_bitinsn(env, ctx, op1, op2, rd, rt);
+                break;
+            default:            /* Invalid */
+                MIPS_INVAL("MASK ABSQ_S.QH");
+                generate_exception(ctx, EXCP_RI);
+                break;
+            }
+            break;
+        case OPC_ADDU_OB_DSP:
+            op2 = MASK_ADDU_OB(ctx->opcode);
+            switch (op2) {
+            case OPC_RADDU_L_OB:
+            case OPC_SUBQ_PW:
+            case OPC_SUBQ_S_PW:
+            case OPC_SUBQ_QH:
+            case OPC_SUBQ_S_QH:
+            case OPC_SUBU_OB:
+            case OPC_SUBU_S_OB:
+            case OPC_SUBU_QH:
+            case OPC_SUBU_S_QH:
+            case OPC_SUBUH_OB:
+            case OPC_SUBUH_R_OB:
+            case OPC_ADDQ_PW:
+            case OPC_ADDQ_S_PW:
+            case OPC_ADDQ_QH:
+            case OPC_ADDQ_S_QH:
+            case OPC_ADDU_OB:
+            case OPC_ADDU_S_OB:
+            case OPC_ADDU_QH:
+            case OPC_ADDU_S_QH:
+            case OPC_ADDUH_OB:
+            case OPC_ADDUH_R_OB:
+                gen_mipsdsp_arith(ctx, op1, op2, rd, rs, rt);
+                break;
+            case OPC_MULEQ_S_PW_QHL:
+            case OPC_MULEQ_S_PW_QHR:
+            case OPC_MULEU_S_QH_OBL:
+            case OPC_MULEU_S_QH_OBR:
+            case OPC_MULQ_RS_QH:
+                gen_mipsdsp_multiply(ctx, op1, op2, rd, rs, rt, 1);
+                break;
+            default:            /* Invalid */
+                MIPS_INVAL("MASK ADDU.OB");
+                generate_exception(ctx, EXCP_RI);
+                break;
+            }
+            break;
+        case OPC_CMPU_EQ_OB_DSP:
+            op2 = MASK_CMPU_EQ_OB(ctx->opcode);
+            switch (op2) {
+            case OPC_PRECR_SRA_QH_PW:
+            case OPC_PRECR_SRA_R_QH_PW:
+                /* Return value is rt. */
+                gen_mipsdsp_arith(ctx, op1, op2, rt, rs, rd);
+                break;
+            case OPC_PRECR_OB_QH:
+            case OPC_PRECRQ_OB_QH:
+            case OPC_PRECRQ_PW_L:
+            case OPC_PRECRQ_QH_PW:
+            case OPC_PRECRQ_RS_QH_PW:
+            case OPC_PRECRQU_S_OB_QH:
+                gen_mipsdsp_arith(ctx, op1, op2, rd, rs, rt);
+                break;
+            case OPC_CMPU_EQ_OB:
+            case OPC_CMPU_LT_OB:
+            case OPC_CMPU_LE_OB:
+            case OPC_CMP_EQ_QH:
+            case OPC_CMP_LT_QH:
+            case OPC_CMP_LE_QH:
+            case OPC_CMP_EQ_PW:
+            case OPC_CMP_LT_PW:
+            case OPC_CMP_LE_PW:
+                gen_mipsdsp_add_cmp_pick(ctx, op1, op2, rd, rs, rt, 0);
+                break;
+            case OPC_CMPGDU_EQ_OB:
+            case OPC_CMPGDU_LT_OB:
+            case OPC_CMPGDU_LE_OB:
+            case OPC_CMPGU_EQ_OB:
+            case OPC_CMPGU_LT_OB:
+            case OPC_CMPGU_LE_OB:
+            case OPC_PACKRL_PW:
+            case OPC_PICK_OB:
+            case OPC_PICK_PW:
+            case OPC_PICK_QH:
+                gen_mipsdsp_add_cmp_pick(ctx, op1, op2, rd, rs, rt, 1);
+                break;
+            default:            /* Invalid */
+                MIPS_INVAL("MASK CMPU_EQ.OB");
+                generate_exception(ctx, EXCP_RI);
+                break;
+            }
+            break;
+        case OPC_DAPPEND_DSP:
+            check_dspr2(ctx);
+            op2 = MASK_DAPPEND(ctx->opcode);
+            gen_mipsdsp_add_cmp_pick(ctx, op1, op2, rt, rs, rd, 1);
+            break;
+        case OPC_DEXTR_W_DSP:
+            op2 = MASK_DEXTR_W(ctx->opcode);
+            switch (op2) {
+            case OPC_DEXTP:
+            case OPC_DEXTPDP:
+            case OPC_DEXTPDPV:
+            case OPC_DEXTPV:
+            case OPC_DEXTR_L:
+            case OPC_DEXTR_R_L:
+            case OPC_DEXTR_RS_L:
+            case OPC_DEXTR_W:
+            case OPC_DEXTR_R_W:
+            case OPC_DEXTR_RS_W:
+            case OPC_DEXTR_S_H:
+            case OPC_DEXTRV_L:
+            case OPC_DEXTRV_R_L:
+            case OPC_DEXTRV_RS_L:
+            case OPC_DEXTRV_S_H:
+            case OPC_DEXTRV_W:
+            case OPC_DEXTRV_R_W:
+            case OPC_DEXTRV_RS_W:
+                gen_mipsdsp_accinsn(ctx, op1, op2, rt, rs, rd, 1);
+                break;
+            case OPC_DMTHLIP:
+            case OPC_DSHILO:
+            case OPC_DSHILOV:
+                gen_mipsdsp_accinsn(ctx, op1, op2, rd, rs, rt, 0);
+                break;
+            default:            /* Invalid */
+                MIPS_INVAL("MASK EXTR.W");
+                generate_exception(ctx, EXCP_RI);
+                break;
+            }
+            break;
+        case OPC_DPAQ_W_QH_DSP:
+            op2 = MASK_DPAQ_W_QH(ctx->opcode);
+            switch (op2) {
+            case OPC_DPAU_H_OBL:
+            case OPC_DPAU_H_OBR:
+            case OPC_DPSU_H_OBL:
+            case OPC_DPSU_H_OBR:
+            case OPC_DPA_W_QH:
+            case OPC_DPAQ_S_W_QH:
+            case OPC_DPS_W_QH:
+            case OPC_DPSQ_S_W_QH:
+            case OPC_MULSAQ_S_W_QH:
+            case OPC_DPAQ_SA_L_PW:
+            case OPC_DPSQ_SA_L_PW:
+            case OPC_MULSAQ_S_L_PW:
+                gen_mipsdsp_multiply(ctx, op1, op2, rd, rs, rt, 0);
+                break;
+            case OPC_MAQ_S_W_QHLL:
+            case OPC_MAQ_S_W_QHLR:
+            case OPC_MAQ_S_W_QHRL:
+            case OPC_MAQ_S_W_QHRR:
+            case OPC_MAQ_SA_W_QHLL:
+            case OPC_MAQ_SA_W_QHLR:
+            case OPC_MAQ_SA_W_QHRL:
+            case OPC_MAQ_SA_W_QHRR:
+            case OPC_MAQ_S_L_PWL:
+            case OPC_MAQ_S_L_PWR:
+            case OPC_DMADD:
+            case OPC_DMADDU:
+            case OPC_DMSUB:
+            case OPC_DMSUBU:
+                gen_mipsdsp_multiply(ctx, op1, op2, rd, rs, rt, 0);
+                break;
+            default:            /* Invalid */
+                MIPS_INVAL("MASK DPAQ.W.QH");
+                generate_exception(ctx, EXCP_RI);
+                break;
+            }
+            break;
+        case OPC_DINSV_DSP:
+            op2 = MASK_INSV(ctx->opcode);
+            switch (op2) {
+            case OPC_DINSV:
+                {
+                    TCGv t0, t1;
+
+                    if (rt == 0) {
+                        MIPS_DEBUG("NOP");
+                        break;
+                    }
+                    check_dsp(ctx);
+
+                    t0 = tcg_temp_new();
+                    t1 = tcg_temp_new();
+
+                    gen_load_gpr(t0, rt);
+                    gen_load_gpr(t1, rs);
+
+                    gen_helper_dinsv(cpu_gpr[rt], cpu_env, t1, t0);
+                    break;
+                }
+            default:            /* Invalid */
+                MIPS_INVAL("MASK DINSV");
+                generate_exception(ctx, EXCP_RI);
+                break;
+            }
+            break;
+        case OPC_SHLL_OB_DSP:
+            gen_mipsdsp_shift(ctx, op1, rd, rs, rt);
+            break;
 #endif
         default:            /* Invalid */
             MIPS_INVAL("special3");
@@ -12098,6 +15219,14 @@ static void decode_opc (CPUMIPSState *env, DisasContext *ctx, int *is_branch)
             check_insn(env, ctx, ISA_MIPS32R2);
             /* Treat as NOP. */
             break;
+        case OPC_BPOSGE32:    /* MIPS DSP branch */
+#if defined(TARGET_MIPS64)
+        case OPC_BPOSGE64:
+#endif
+            check_dsp(ctx);
+            gen_compute_branch(ctx, op1, 4, -1, -2, (int32_t)imm << 2);
+            *is_branch = 1;
+            break;
         default:            /* Invalid */
             MIPS_INVAL("regimm");
             generate_exception(ctx, EXCP_RI);
@@ -12144,18 +15273,18 @@ static void decode_opc (CPUMIPSState *env, DisasContext *ctx, int *is_branch)
                     break;
                 case OPC_DVPE:
                     check_insn(env, ctx, ASE_MT);
-                    gen_helper_dvpe(t0);
+                    gen_helper_dvpe(t0, cpu_env);
                     gen_store_gpr(t0, rt);
                     break;
                 case OPC_EVPE:
                     check_insn(env, ctx, ASE_MT);
-                    gen_helper_evpe(t0);
+                    gen_helper_evpe(t0, cpu_env);
                     gen_store_gpr(t0, rt);
                     break;
                 case OPC_DI:
                     check_insn(env, ctx, ISA_MIPS32R2);
                     save_cpu_state(ctx, 1);
-                    gen_helper_di(t0);
+                    gen_helper_di(t0, cpu_env);
                     gen_store_gpr(t0, rt);
                     /* Stop translation as we may have switched the execution mode */
                     ctx->bstate = BS_STOP;
@@ -12163,7 +15292,7 @@ static void decode_opc (CPUMIPSState *env, DisasContext *ctx, int *is_branch)
                 case OPC_EI:
                     check_insn(env, ctx, ISA_MIPS32R2);
                     save_cpu_state(ctx, 1);
-                    gen_helper_ei(t0);
+                    gen_helper_ei(t0, cpu_env);
                     gen_store_gpr(t0, rt);
                     /* Stop translation as we may have switched the execution mode */
                     ctx->bstate = BS_STOP;
@@ -12197,13 +15326,13 @@ static void decode_opc (CPUMIPSState *env, DisasContext *ctx, int *is_branch)
          break;
     case OPC_SLTI: /* Set on less than with immediate opcode */
     case OPC_SLTIU:
-         gen_slt_imm(env, op, rt, rs, imm);
+         gen_slt_imm(env, ctx, op, rt, rs, imm);
          break;
     case OPC_ANDI: /* Arithmetic with immediate opcode */
     case OPC_LUI:
     case OPC_ORI:
     case OPC_XORI:
-         gen_logic_imm(env, op, rt, rs, imm);
+         gen_logic_imm(env, ctx, op, rt, rs, imm);
          break;
     case OPC_J ... OPC_JAL: /* Jump */
          offset = (int32_t)(ctx->opcode & 0x3FFFFFF) << 2;
@@ -12298,10 +15427,14 @@ static void decode_opc (CPUMIPSState *env, DisasContext *ctx, int *is_branch)
     case OPC_LDC2:
     case OPC_SWC2:
     case OPC_SDC2:
-    case OPC_CP2:
         /* COP2: Not implemented. */
         generate_exception_err(ctx, EXCP_CpU, 2);
         break;
+    case OPC_CP2:
+        check_insn(env, ctx, INSN_LOONGSON2F);
+        /* Note that these instructions use different fields.  */
+        gen_loongson_multimedia(ctx, sa, rd, rt);
+        break;
 
     case OPC_CP3:
         if (env->CP0_Config1 & (1 << CP0C1_FP)) {
@@ -12406,7 +15539,7 @@ gen_intermediate_code_internal (CPUMIPSState *env, TranslationBlock *tb,
         qemu_log("search pc %d\n", search_pc);
 
     pc_start = tb->pc;
-    gen_opc_end = gen_opc_buf + OPC_MAX_SIZE;
+    gen_opc_end = tcg_ctx.gen_opc_buf + OPC_MAX_SIZE;
     ctx.pc = pc_start;
     ctx.saved_pc = -1;
     ctx.singlestep_enabled = env->singlestep_enabled;
@@ -12432,7 +15565,7 @@ gen_intermediate_code_internal (CPUMIPSState *env, TranslationBlock *tb,
                 if (bp->pc == ctx.pc) {
                     save_cpu_state(&ctx, 1);
                     ctx.bstate = BS_BRANCH;
-                    gen_helper_0i(raise_exception, EXCP_DEBUG);
+                    gen_helper_0e0i(raise_exception, EXCP_DEBUG);
                     /* Include the breakpoint location or the tb won't
                      * be flushed when it must be.  */
                     ctx.pc += 4;
@@ -12442,7 +15575,7 @@ gen_intermediate_code_internal (CPUMIPSState *env, TranslationBlock *tb,
         }
 
         if (search_pc) {
-            j = gen_opc_ptr - gen_opc_buf;
+            j = tcg_ctx.gen_opc_ptr - tcg_ctx.gen_opc_buf;
             if (lj < j) {
                 lj++;
                 while (lj < j)
@@ -12450,6 +15583,7 @@ gen_intermediate_code_internal (CPUMIPSState *env, TranslationBlock *tb,
             }
             gen_opc_pc[lj] = ctx.pc;
             gen_opc_hflags[lj] = ctx.hflags & MIPS_HFLAG_BMASK;
+            gen_opc_btarget[lj] = ctx.btarget;
             gen_opc_instr_start[lj] = 1;
             gen_opc_icount[lj] = num_insns;
         }
@@ -12458,14 +15592,14 @@ gen_intermediate_code_internal (CPUMIPSState *env, TranslationBlock *tb,
 
         is_branch = 0;
         if (!(ctx.hflags & MIPS_HFLAG_M16)) {
-            ctx.opcode = ldl_code(ctx.pc);
+            ctx.opcode = cpu_ldl_code(env, ctx.pc);
             insn_bytes = 4;
             decode_opc(env, &ctx, &is_branch);
         } else if (env->insn_flags & ASE_MICROMIPS) {
-            ctx.opcode = lduw_code(ctx.pc);
+            ctx.opcode = cpu_lduw_code(env, ctx.pc);
             insn_bytes = decode_micromips_opc(env, &ctx, &is_branch);
         } else if (env->insn_flags & ASE_MIPS16) {
-            ctx.opcode = lduw_code(ctx.pc);
+            ctx.opcode = cpu_lduw_code(env, ctx.pc);
             insn_bytes = decode_mips16_opc(env, &ctx, &is_branch);
         } else {
             generate_exception(&ctx, EXCP_RI);
@@ -12489,8 +15623,9 @@ gen_intermediate_code_internal (CPUMIPSState *env, TranslationBlock *tb,
         if ((ctx.pc & (TARGET_PAGE_SIZE - 1)) == 0)
             break;
 
-        if (gen_opc_ptr >= gen_opc_end)
+        if (tcg_ctx.gen_opc_ptr >= gen_opc_end) {
             break;
+        }
 
         if (num_insns >= max_insns)
             break;
@@ -12502,7 +15637,7 @@ gen_intermediate_code_internal (CPUMIPSState *env, TranslationBlock *tb,
         gen_io_end();
     if (env->singlestep_enabled && ctx.bstate != BS_BRANCH) {
         save_cpu_state(&ctx, ctx.bstate == BS_NONE);
-        gen_helper_0i(raise_exception, EXCP_DEBUG);
+        gen_helper_0e0i(raise_exception, EXCP_DEBUG);
     } else {
         switch (ctx.bstate) {
         case BS_STOP:
@@ -12522,9 +15657,9 @@ gen_intermediate_code_internal (CPUMIPSState *env, TranslationBlock *tb,
     }
 done_generating:
     gen_icount_end(tb, num_insns);
-    *gen_opc_ptr = INDEX_op_end;
+    *tcg_ctx.gen_opc_ptr = INDEX_op_end;
     if (search_pc) {
-        j = gen_opc_ptr - gen_opc_buf;
+        j = tcg_ctx.gen_opc_ptr - tcg_ctx.gen_opc_buf;
         lj++;
         while (lj <= j)
             gen_opc_instr_start[lj++] = 0;
@@ -12536,7 +15671,7 @@ done_generating:
     LOG_DISAS("\n");
     if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
         qemu_log("IN: %s\n", lookup_symbol(pc_start));
-        log_target_disas(pc_start, ctx.pc - pc_start, 0);
+        log_target_disas(env, pc_start, ctx.pc - pc_start, 0);
         qemu_log("\n");
     }
 #endif
@@ -12670,6 +15805,12 @@ static void mips_tcg_init(void)
         cpu_gpr[i] = tcg_global_mem_new(TCG_AREG0,
                                         offsetof(CPUMIPSState, active_tc.gpr[i]),
                                         regnames[i]);
+
+    for (i = 0; i < 32; i++) {
+        int off = offsetof(CPUMIPSState, active_fpu.fpr[i]);
+        fpu_f64[i] = tcg_global_mem_new_i64(TCG_AREG0, off, fregnames[i]);
+    }
+
     cpu_PC = tcg_global_mem_new(TCG_AREG0,
                                 offsetof(CPUMIPSState, active_tc.PC), "PC");
     for (i = 0; i < MIPS_DSP_ACC; i++) {
@@ -12787,18 +15928,18 @@ void cpu_state_reset(CPUMIPSState *env)
     env->insn_flags = env->cpu_model->insn_flags;
 
 #if defined(CONFIG_USER_ONLY)
-    env->hflags = MIPS_HFLAG_UM;
+    env->CP0_Status = (MIPS_HFLAG_UM << CP0St_KSU);
     /* Enable access to the CPUNum, SYNCI_Step, CC, and CCRes RDHWR
        hardware registers.  */
     env->CP0_HWREna |= 0x0000000F;
     if (env->CP0_Config1 & (1 << CP0C1_FP)) {
-        env->hflags |= MIPS_HFLAG_FPU;
+        env->CP0_Status |= (1 << CP0St_CU1);
     }
-#ifdef TARGET_MIPS64
-    if (env->active_fpu.fcr0 & (1 << FCR0_F64)) {
-        env->hflags |= MIPS_HFLAG_F64;
+    if (env->cpu_model->insn_flags & ASE_DSPR2) {
+        env->hflags |= MIPS_HFLAG_DSP | MIPS_HFLAG_DSPR2;
+    } else if (env->cpu_model->insn_flags & ASE_DSP) {
+        env->hflags |= MIPS_HFLAG_DSP;
     }
-#endif
 #else
     if (env->hflags & MIPS_HFLAG_BMASK) {
         /* If the exception was raised from a delay slot,
@@ -12828,7 +15969,6 @@ void cpu_state_reset(CPUMIPSState *env)
     }
     /* Count register increments in debug mode, EJTAG version 1 */
     env->CP0_Debug = (1 << CP0DB_CNT) | (0x1 << CP0DB_VER);
-    env->hflags = MIPS_HFLAG_CP0;
 
     if (env->CP0_Config3 & (1 << CP0C3_MT)) {
         int i;
@@ -12856,11 +15996,7 @@ void cpu_state_reset(CPUMIPSState *env)
         }
     }
 #endif
-#if defined(TARGET_MIPS64)
-    if (env->cpu_model->insn_flags & ISA_MIPS3) {
-        env->hflags |= MIPS_HFLAG_64;
-    }
-#endif
+    compute_hflags(env);
     env->exception_index = EXCP_NONE;
 }
 
@@ -12869,4 +16005,13 @@ void restore_state_to_opc(CPUMIPSState *env, TranslationBlock *tb, int pc_pos)
     env->active_tc.PC = gen_opc_pc[pc_pos];
     env->hflags &= ~MIPS_HFLAG_BMASK;
     env->hflags |= gen_opc_hflags[pc_pos];
+    switch (env->hflags & MIPS_HFLAG_BMASK_BASE) {
+    case MIPS_HFLAG_BR:
+        break;
+    case MIPS_HFLAG_BC:
+    case MIPS_HFLAG_BL:
+    case MIPS_HFLAG_B:
+        env->btarget = gen_opc_btarget[pc_pos];
+        break;
+    }
 }
diff --git a/target-mips/translate_init.c b/target-mips/translate_init.c
index c39138f3c..7cf238f34 100644
--- a/target-mips/translate_init.c
+++ b/target-mips/translate_init.c
@@ -311,6 +311,29 @@ static const mips_def_t mips_defs[] =
         .insn_flags = CPU_MIPS32R2 | ASE_MIPS16 | ASE_DSP | ASE_MT,
         .mmu_type = MMU_TYPE_R4000,
     },
+    {
+        .name = "74Kf",
+        .CP0_PRid = 0x00019700,
+        .CP0_Config0 = MIPS_CONFIG0 | (0x1 << CP0C0_AR) |
+                    (MMU_TYPE_R4000 << CP0C0_MT),
+        .CP0_Config1 = MIPS_CONFIG1 | (1 << CP0C1_FP) | (15 << CP0C1_MMU) |
+                       (0 << CP0C1_IS) | (3 << CP0C1_IL) | (1 << CP0C1_IA) |
+                       (0 << CP0C1_DS) | (3 << CP0C1_DL) | (1 << CP0C1_DA) |
+                       (1 << CP0C1_CA),
+        .CP0_Config2 = MIPS_CONFIG2,
+        .CP0_Config3 = MIPS_CONFIG3 | (0 << CP0C3_VInt) | (1 << CP0C3_DSPP),
+        .CP0_LLAddr_rw_bitmask = 0,
+        .CP0_LLAddr_shift = 4,
+        .SYNCI_Step = 32,
+        .CCRes = 2,
+        .CP0_Status_rw_bitmask = 0x3778FF1F,
+        .CP1_fcr0 = (1 << FCR0_F64) | (1 << FCR0_L) | (1 << FCR0_W) |
+                    (1 << FCR0_D) | (1 << FCR0_S) | (0x93 << FCR0_PRID),
+        .SEGBITS = 32,
+        .PABITS = 32,
+        .insn_flags = CPU_MIPS32R2 | ASE_MIPS16 | ASE_DSP | ASE_DSPR2,
+        .mmu_type = MMU_TYPE_R4000,
+    },
 #if defined(TARGET_MIPS64)
     {
         .name = "R4000",
@@ -484,6 +507,35 @@ static const mips_def_t mips_defs[] =
       .insn_flags = CPU_LOONGSON2F,
       .mmu_type = MMU_TYPE_R4000,
     },
+    {
+        /* A generic CPU providing MIPS64 ASE DSP 2 features.
+           FIXME: Eventually this should be replaced by a real CPU model. */
+        .name = "mips64dspr2",
+        .CP0_PRid = 0x00010000,
+        .CP0_Config0 = MIPS_CONFIG0 | (0x1 << CP0C0_AR) | (0x2 << CP0C0_AT) |
+                       (MMU_TYPE_R4000 << CP0C0_MT),
+        .CP0_Config1 = MIPS_CONFIG1 | (1 << CP0C1_FP) | (63 << CP0C1_MMU) |
+                       (2 << CP0C1_IS) | (4 << CP0C1_IL) | (3 << CP0C1_IA) |
+                       (2 << CP0C1_DS) | (4 << CP0C1_DL) | (3 << CP0C1_DA) |
+                       (1 << CP0C1_PC) | (1 << CP0C1_WR) | (1 << CP0C1_EP),
+        .CP0_Config2 = MIPS_CONFIG2,
+        .CP0_Config3 = MIPS_CONFIG3 | (1 << CP0C3_LPA),
+        .CP0_LLAddr_rw_bitmask = 0,
+        .CP0_LLAddr_shift = 0,
+        .SYNCI_Step = 32,
+        .CCRes = 2,
+        .CP0_Status_rw_bitmask = 0x37FBFFFF,
+        .CP1_fcr0 = (1 << FCR0_F64) | (1 << FCR0_3D) | (1 << FCR0_PS) |
+                    (1 << FCR0_L) | (1 << FCR0_W) | (1 << FCR0_D) |
+                    (1 << FCR0_S) | (0x00 << FCR0_PRID) | (0x0 << FCR0_REV),
+        .SEGBITS = 42,
+        /* The architectural limit is 59, but we have hardcoded 36 bit
+           in some places...
+        .PABITS = 59, */ /* the architectural limit */
+        .PABITS = 36,
+        .insn_flags = CPU_MIPS64R2 | ASE_DSP | ASE_DSPR2,
+        .mmu_type = MMU_TYPE_R4000,
+    },
 
 #endif
 };
diff --git a/target-openrisc/cpu.h b/target-openrisc/cpu.h
index de21a877d..ebb5ad312 100644
--- a/target-openrisc/cpu.h
+++ b/target-openrisc/cpu.h
@@ -89,24 +89,6 @@ enum {
 /* Interrupt */
 #define NR_IRQS  32
 
-/* Registers */
-enum {
-    R0 = 0, R1, R2, R3, R4, R5, R6, R7, R8, R9, R10,
-    R11, R12, R13, R14, R15, R16, R17, R18, R19, R20,
-    R21, R22, R23, R24, R25, R26, R27, R28, R29, R30,
-    R31
-};
-
-/* Register aliases */
-enum {
-    R_ZERO = R0,
-    R_SP = R1,
-    R_FP = R2,
-    R_LR = R9,
-    R_RV = R11,
-    R_RVH = R12
-};
-
 /* Unit presece register */
 enum {
     UPR_UP = (1 << 0),
@@ -279,11 +261,11 @@ typedef struct CPUOpenRISCTLBContext {
     OpenRISCTLBEntry dtlb[DTLB_WAYS][DTLB_SIZE];
 
     int (*cpu_openrisc_map_address_code)(struct OpenRISCCPU *cpu,
-                                         target_phys_addr_t *physical,
+                                         hwaddr *physical,
                                          int *prot,
                                          target_ulong address, int rw);
     int (*cpu_openrisc_map_address_data)(struct OpenRISCCPU *cpu,
-                                         target_phys_addr_t *physical,
+                                         hwaddr *physical,
                                          int *prot,
                                          target_ulong address, int rw);
 } CPUOpenRISCTLBContext;
@@ -387,13 +369,13 @@ void cpu_openrisc_count_stop(OpenRISCCPU *cpu);
 
 void cpu_openrisc_mmu_init(OpenRISCCPU *cpu);
 int cpu_openrisc_get_phys_nommu(OpenRISCCPU *cpu,
-                                target_phys_addr_t *physical,
+                                hwaddr *physical,
                                 int *prot, target_ulong address, int rw);
 int cpu_openrisc_get_phys_code(OpenRISCCPU *cpu,
-                               target_phys_addr_t *physical,
+                               hwaddr *physical,
                                int *prot, target_ulong address, int rw);
 int cpu_openrisc_get_phys_data(OpenRISCCPU *cpu,
-                               target_phys_addr_t *physical,
+                               hwaddr *physical,
                                int *prot, target_ulong address, int rw);
 #endif
 
@@ -437,8 +419,10 @@ static inline int cpu_mmu_index(CPUOpenRISCState *env)
 }
 
 #define CPU_INTERRUPT_TIMER   CPU_INTERRUPT_TGT_INT_0
-static inline bool cpu_has_work(CPUOpenRISCState *env)
+static inline bool cpu_has_work(CPUState *cpu)
 {
+    CPUOpenRISCState *env = &OPENRISC_CPU(cpu)->env;
+
     return env->interrupt_request & (CPU_INTERRUPT_HARD |
                                      CPU_INTERRUPT_TIMER);
 }
diff --git a/target-openrisc/mmu.c b/target-openrisc/mmu.c
index 0be1d413c..f2a6523cf 100644
--- a/target-openrisc/mmu.c
+++ b/target-openrisc/mmu.c
@@ -28,7 +28,7 @@
 
 #ifndef CONFIG_USER_ONLY
 int cpu_openrisc_get_phys_nommu(OpenRISCCPU *cpu,
-                                target_phys_addr_t *physical,
+                                hwaddr *physical,
                                 int *prot, target_ulong address, int rw)
 {
     *physical = address;
@@ -37,7 +37,7 @@ int cpu_openrisc_get_phys_nommu(OpenRISCCPU *cpu,
 }
 
 int cpu_openrisc_get_phys_code(OpenRISCCPU *cpu,
-                               target_phys_addr_t *physical,
+                               hwaddr *physical,
                                int *prot, target_ulong address, int rw)
 {
     int vpn = address >> TARGET_PAGE_BITS;
@@ -72,7 +72,7 @@ int cpu_openrisc_get_phys_code(OpenRISCCPU *cpu,
 }
 
 int cpu_openrisc_get_phys_data(OpenRISCCPU *cpu,
-                               target_phys_addr_t *physical,
+                               hwaddr *physical,
                                int *prot, target_ulong address, int rw)
 {
     int vpn = address >> TARGET_PAGE_BITS;
@@ -116,7 +116,7 @@ int cpu_openrisc_get_phys_data(OpenRISCCPU *cpu,
 }
 
 static int cpu_openrisc_get_phys_addr(OpenRISCCPU *cpu,
-                                      target_phys_addr_t *physical,
+                                      hwaddr *physical,
                                       int *prot, target_ulong address,
                                       int rw)
 {
@@ -185,7 +185,7 @@ int cpu_openrisc_handle_mmu_fault(CPUOpenRISCState *env,
                                   target_ulong address, int rw, int mmu_idx)
 {
     int ret = 0;
-    target_phys_addr_t physical = 0;
+    hwaddr physical = 0;
     int prot = 0;
     OpenRISCCPU *cpu = OPENRISC_CPU(ENV_GET_CPU(env));
 
@@ -219,10 +219,10 @@ int cpu_openrisc_handle_mmu_fault(CPUOpenRISCState *env,
 #endif
 
 #ifndef CONFIG_USER_ONLY
-target_phys_addr_t cpu_get_phys_page_debug(CPUOpenRISCState *env,
+hwaddr cpu_get_phys_page_debug(CPUOpenRISCState *env,
                                            target_ulong addr)
 {
-    target_phys_addr_t phys_addr;
+    hwaddr phys_addr;
     int prot;
     OpenRISCCPU *cpu = OPENRISC_CPU(ENV_GET_CPU(env));
 
diff --git a/target-openrisc/translate.c b/target-openrisc/translate.c
index 325ba09cb..f14da7bd1 100644
--- a/target-openrisc/translate.c
+++ b/target-openrisc/translate.c
@@ -1675,7 +1675,7 @@ static inline void gen_intermediate_code_internal(OpenRISCCPU *cpu,
     pc_start = tb->pc;
     dc->tb = tb;
 
-    gen_opc_end = gen_opc_buf + OPC_MAX_SIZE;
+    gen_opc_end = tcg_ctx.gen_opc_buf + OPC_MAX_SIZE;
     dc->is_jmp = DISAS_NEXT;
     dc->ppc = pc_start;
     dc->pc = pc_start;
@@ -1703,7 +1703,7 @@ static inline void gen_intermediate_code_internal(OpenRISCCPU *cpu,
     do {
         check_breakpoint(cpu, dc);
         if (search_pc) {
-            j = gen_opc_ptr - gen_opc_buf;
+            j = tcg_ctx.gen_opc_ptr - tcg_ctx.gen_opc_buf;
             if (k < j) {
                 k++;
                 while (k < j) {
@@ -1715,7 +1715,7 @@ static inline void gen_intermediate_code_internal(OpenRISCCPU *cpu,
             gen_opc_icount[k] = num_insns;
         }
 
-        if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP))) {
+        if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
             tcg_gen_debug_insn_start(dc->pc);
         }
 
@@ -1744,7 +1744,7 @@ static inline void gen_intermediate_code_internal(OpenRISCCPU *cpu,
             }
         }
     } while (!dc->is_jmp
-             && gen_opc_ptr < gen_opc_end
+             && tcg_ctx.gen_opc_ptr < gen_opc_end
              && !cpu->env.singlestep_enabled
              && !singlestep
              && (dc->pc < next_page_start)
@@ -1782,9 +1782,9 @@ static inline void gen_intermediate_code_internal(OpenRISCCPU *cpu,
     }
 
     gen_icount_end(tb, num_insns);
-    *gen_opc_ptr = INDEX_op_end;
+    *tcg_ctx.gen_opc_ptr = INDEX_op_end;
     if (search_pc) {
-        j = gen_opc_ptr - gen_opc_buf;
+        j = tcg_ctx.gen_opc_ptr - tcg_ctx.gen_opc_buf;
         k++;
         while (k <= j) {
             gen_opc_instr_start[k++] = 0;
@@ -1797,9 +1797,10 @@ static inline void gen_intermediate_code_internal(OpenRISCCPU *cpu,
 #ifdef DEBUG_DISAS
     if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
         qemu_log("\n");
-        log_target_disas(pc_start, dc->pc - pc_start, 0);
+        log_target_disas(&cpu->env, pc_start, dc->pc - pc_start, 0);
         qemu_log("\nisize=%d osize=%td\n",
-            dc->pc - pc_start, gen_opc_ptr - gen_opc_buf);
+            dc->pc - pc_start, tcg_ctx.gen_opc_ptr -
+            tcg_ctx.gen_opc_buf);
     }
 #endif
 }
diff --git a/target-ppc/cpu.h b/target-ppc/cpu.h
index ca2fc2198..5f1dc8b7d 100644
--- a/target-ppc/cpu.h
+++ b/target-ppc/cpu.h
@@ -355,7 +355,7 @@ struct ppc6xx_tlb_t {
 
 typedef struct ppcemb_tlb_t ppcemb_tlb_t;
 struct ppcemb_tlb_t {
-    target_phys_addr_t RPN;
+    hwaddr RPN;
     target_ulong EPN;
     target_ulong PID;
     target_ulong size;
@@ -963,7 +963,7 @@ struct CPUPPCState {
     /* floating point registers */
     float64 fpr[32];
     /* floating point status and control register */
-    uint32_t fpscr;
+    target_ulong fpscr;
 
     /* Next instruction pointer */
     target_ulong nip;
@@ -983,8 +983,8 @@ struct CPUPPCState {
     int slb_nr;
 #endif
     /* segment registers */
-    target_phys_addr_t htab_base;
-    target_phys_addr_t htab_mask;
+    hwaddr htab_base;
+    hwaddr htab_mask;
     target_ulong sr[32];
     /* externally stored hash table */
     uint8_t *external_htab;
@@ -1014,6 +1014,8 @@ struct CPUPPCState {
     /* Altivec registers */
     ppc_avr_t avr[32];
     uint32_t vscr;
+    /* VSX registers */
+    uint64_t vsr[32];
     /* SPE registers */
     uint64_t spe_acc;
     uint32_t spe_fscr;
@@ -1045,10 +1047,9 @@ struct CPUPPCState {
 #endif
 
 #if defined(TARGET_PPC64) && !defined(CONFIG_USER_ONLY)
-    target_phys_addr_t vpa;
-    target_phys_addr_t slb_shadow;
-    target_phys_addr_t dispatch_trace_log;
-    uint32_t dtl_size;
+    uint64_t vpa_addr;
+    uint64_t slb_shadow_addr, slb_shadow_size;
+    uint64_t dtl_addr, dtl_size;
 #endif /* TARGET_PPC64 */
 
     int error_code;
@@ -1066,7 +1067,7 @@ struct CPUPPCState {
     target_ulong ivor_mask;
     target_ulong ivpr_mask;
     target_ulong hreset_vector;
-    target_phys_addr_t mpic_cpu_base;
+    hwaddr mpic_cpu_base;
 #endif
 
     /* Those resources are used only during code translation */
@@ -1079,7 +1080,6 @@ struct CPUPPCState {
     int mmu_idx;         /* precomputed MMU index to speed up mem accesses */
 
     /* Power management */
-    int power_mode;
     int (*check_pow)(CPUPPCState *env);
 
 #if !defined(CONFIG_USER_ONLY)
@@ -1118,10 +1118,10 @@ do {                                            \
 /* Context used internally during MMU translations */
 typedef struct mmu_ctx_t mmu_ctx_t;
 struct mmu_ctx_t {
-    target_phys_addr_t raddr;      /* Real address              */
-    target_phys_addr_t eaddr;      /* Effective address         */
+    hwaddr raddr;      /* Real address              */
+    hwaddr eaddr;      /* Effective address         */
     int prot;                      /* Protection bits           */
-    target_phys_addr_t hash[2];    /* Pagetable hash values     */
+    hwaddr hash[2];    /* Pagetable hash values     */
     target_ulong ptem;             /* Virtual segment ID | API  */
     int key;                       /* Access key                */
     int nx;                        /* Non-execute area          */
@@ -1142,10 +1142,6 @@ int cpu_ppc_signal_handler (int host_signum, void *pinfo,
 int cpu_ppc_handle_mmu_fault (CPUPPCState *env, target_ulong address, int rw,
                               int mmu_idx);
 #define cpu_handle_mmu_fault cpu_ppc_handle_mmu_fault
-#if !defined(CONFIG_USER_ONLY)
-int get_physical_address (CPUPPCState *env, mmu_ctx_t *ctx, target_ulong vaddr,
-                          int rw, int access_type);
-#endif
 void do_interrupt (CPUPPCState *env);
 void ppc_hw_interrupt (CPUPPCState *env);
 
@@ -1179,7 +1175,6 @@ void cpu_ppc_store_decr (CPUPPCState *env, uint32_t value);
 uint32_t cpu_ppc_load_hdecr (CPUPPCState *env);
 void cpu_ppc_store_hdecr (CPUPPCState *env, uint32_t value);
 uint64_t cpu_ppc_load_purr (CPUPPCState *env);
-void cpu_ppc_store_purr (CPUPPCState *env, uint64_t value);
 uint32_t cpu_ppc601_load_rtcl (CPUPPCState *env);
 uint32_t cpu_ppc601_load_rtcu (CPUPPCState *env);
 #if !defined(CONFIG_USER_ONLY)
@@ -1191,10 +1186,8 @@ void store_40x_dbcr0 (CPUPPCState *env, uint32_t val);
 void store_40x_sler (CPUPPCState *env, uint32_t val);
 void store_booke_tcr (CPUPPCState *env, target_ulong val);
 void store_booke_tsr (CPUPPCState *env, target_ulong val);
-void booke206_flush_tlb(CPUPPCState *env, int flags, const int check_iprot);
-target_phys_addr_t booke206_tlb_to_page_size(CPUPPCState *env, ppcmas_tlb_t *tlb);
 int ppcmas_tlb_check(CPUPPCState *env, ppcmas_tlb_t *tlb,
-                     target_phys_addr_t *raddrp, target_ulong address,
+                     hwaddr *raddrp, target_ulong address,
                      uint32_t pid);
 void ppc_tlb_invalidate_all (CPUPPCState *env);
 void ppc_tlb_invalidate_one (CPUPPCState *env, target_ulong addr);
@@ -2222,10 +2215,12 @@ static inline bool msr_is_64bit(CPUPPCState *env, target_ulong msr)
     return msr & (1ULL << MSR_SF);
 }
 
-extern void (*cpu_ppc_hypercall)(CPUPPCState *);
+extern void (*cpu_ppc_hypercall)(PowerPCCPU *);
 
-static inline bool cpu_has_work(CPUPPCState *env)
+static inline bool cpu_has_work(CPUState *cpu)
 {
+    CPUPPCState *env = &POWERPC_CPU(cpu)->env;
+
     return msr_ee && (env->interrupt_request & CPU_INTERRUPT_HARD);
 }
 
diff --git a/target-ppc/excp_helper.c b/target-ppc/excp_helper.c
index 1a593f6f3..5e34ad08a 100644
--- a/target-ppc/excp_helper.c
+++ b/target-ppc/excp_helper.c
@@ -33,7 +33,7 @@
 /*****************************************************************************/
 /* PowerPC Hypercall emulation */
 
-void (*cpu_ppc_hypercall)(CPUPPCState *);
+void (*cpu_ppc_hypercall)(PowerPCCPU *);
 
 /*****************************************************************************/
 /* Exception processing */
@@ -63,8 +63,9 @@ static inline void dump_syscall(CPUPPCState *env)
 /* Note that this function should be greatly optimized
  * when called with a constant excp, from ppc_hw_interrupt
  */
-static inline void powerpc_excp(CPUPPCState *env, int excp_model, int excp)
+static inline void powerpc_excp(PowerPCCPU *cpu, int excp_model, int excp)
 {
+    CPUPPCState *env = &cpu->env;
     target_ulong msr, new_msr, vector;
     int srr0, srr1, asrr0, asrr1;
     int lpes0, lpes1, lev;
@@ -238,7 +239,7 @@ static inline void powerpc_excp(CPUPPCState *env, int excp_model, int excp)
         dump_syscall(env);
         lev = env->error_code;
         if ((lev == 1) && cpu_ppc_hypercall) {
-            cpu_ppc_hypercall(env);
+            cpu_ppc_hypercall(cpu);
             return;
         }
         if (lev == 1 || (lpes0 == 0 && lpes1 == 0)) {
@@ -643,11 +644,14 @@ static inline void powerpc_excp(CPUPPCState *env, int excp_model, int excp)
 
 void do_interrupt(CPUPPCState *env)
 {
-    powerpc_excp(env, env->excp_model, env->exception_index);
+    PowerPCCPU *cpu = ppc_env_get_cpu(env);
+
+    powerpc_excp(cpu, env->excp_model, env->exception_index);
 }
 
 void ppc_hw_interrupt(CPUPPCState *env)
 {
+    PowerPCCPU *cpu = ppc_env_get_cpu(env);
     int hdice;
 
 #if 0
@@ -658,20 +662,20 @@ void ppc_hw_interrupt(CPUPPCState *env)
     /* External reset */
     if (env->pending_interrupts & (1 << PPC_INTERRUPT_RESET)) {
         env->pending_interrupts &= ~(1 << PPC_INTERRUPT_RESET);
-        powerpc_excp(env, env->excp_model, POWERPC_EXCP_RESET);
+        powerpc_excp(cpu, env->excp_model, POWERPC_EXCP_RESET);
         return;
     }
     /* Machine check exception */
     if (env->pending_interrupts & (1 << PPC_INTERRUPT_MCK)) {
         env->pending_interrupts &= ~(1 << PPC_INTERRUPT_MCK);
-        powerpc_excp(env, env->excp_model, POWERPC_EXCP_MCHECK);
+        powerpc_excp(cpu, env->excp_model, POWERPC_EXCP_MCHECK);
         return;
     }
 #if 0 /* TODO */
     /* External debug exception */
     if (env->pending_interrupts & (1 << PPC_INTERRUPT_DEBUG)) {
         env->pending_interrupts &= ~(1 << PPC_INTERRUPT_DEBUG);
-        powerpc_excp(env, env->excp_model, POWERPC_EXCP_DEBUG);
+        powerpc_excp(cpu, env->excp_model, POWERPC_EXCP_DEBUG);
         return;
     }
 #endif
@@ -685,7 +689,7 @@ void ppc_hw_interrupt(CPUPPCState *env)
         /* Hypervisor decrementer exception */
         if (env->pending_interrupts & (1 << PPC_INTERRUPT_HDECR)) {
             env->pending_interrupts &= ~(1 << PPC_INTERRUPT_HDECR);
-            powerpc_excp(env, env->excp_model, POWERPC_EXCP_HDECR);
+            powerpc_excp(cpu, env->excp_model, POWERPC_EXCP_HDECR);
             return;
         }
     }
@@ -698,7 +702,7 @@ void ppc_hw_interrupt(CPUPPCState *env)
 #if 0
             env->pending_interrupts &= ~(1 << PPC_INTERRUPT_CEXT);
 #endif
-            powerpc_excp(env, env->excp_model, POWERPC_EXCP_CRITICAL);
+            powerpc_excp(cpu, env->excp_model, POWERPC_EXCP_CRITICAL);
             return;
         }
     }
@@ -706,30 +710,30 @@ void ppc_hw_interrupt(CPUPPCState *env)
         /* Watchdog timer on embedded PowerPC */
         if (env->pending_interrupts & (1 << PPC_INTERRUPT_WDT)) {
             env->pending_interrupts &= ~(1 << PPC_INTERRUPT_WDT);
-            powerpc_excp(env, env->excp_model, POWERPC_EXCP_WDT);
+            powerpc_excp(cpu, env->excp_model, POWERPC_EXCP_WDT);
             return;
         }
         if (env->pending_interrupts & (1 << PPC_INTERRUPT_CDOORBELL)) {
             env->pending_interrupts &= ~(1 << PPC_INTERRUPT_CDOORBELL);
-            powerpc_excp(env, env->excp_model, POWERPC_EXCP_DOORCI);
+            powerpc_excp(cpu, env->excp_model, POWERPC_EXCP_DOORCI);
             return;
         }
         /* Fixed interval timer on embedded PowerPC */
         if (env->pending_interrupts & (1 << PPC_INTERRUPT_FIT)) {
             env->pending_interrupts &= ~(1 << PPC_INTERRUPT_FIT);
-            powerpc_excp(env, env->excp_model, POWERPC_EXCP_FIT);
+            powerpc_excp(cpu, env->excp_model, POWERPC_EXCP_FIT);
             return;
         }
         /* Programmable interval timer on embedded PowerPC */
         if (env->pending_interrupts & (1 << PPC_INTERRUPT_PIT)) {
             env->pending_interrupts &= ~(1 << PPC_INTERRUPT_PIT);
-            powerpc_excp(env, env->excp_model, POWERPC_EXCP_PIT);
+            powerpc_excp(cpu, env->excp_model, POWERPC_EXCP_PIT);
             return;
         }
         /* Decrementer exception */
         if (env->pending_interrupts & (1 << PPC_INTERRUPT_DECR)) {
             env->pending_interrupts &= ~(1 << PPC_INTERRUPT_DECR);
-            powerpc_excp(env, env->excp_model, POWERPC_EXCP_DECR);
+            powerpc_excp(cpu, env->excp_model, POWERPC_EXCP_DECR);
             return;
         }
         /* External interrupt */
@@ -740,23 +744,23 @@ void ppc_hw_interrupt(CPUPPCState *env)
 #if 0
             env->pending_interrupts &= ~(1 << PPC_INTERRUPT_EXT);
 #endif
-            powerpc_excp(env, env->excp_model, POWERPC_EXCP_EXTERNAL);
+            powerpc_excp(cpu, env->excp_model, POWERPC_EXCP_EXTERNAL);
             return;
         }
         if (env->pending_interrupts & (1 << PPC_INTERRUPT_DOORBELL)) {
             env->pending_interrupts &= ~(1 << PPC_INTERRUPT_DOORBELL);
-            powerpc_excp(env, env->excp_model, POWERPC_EXCP_DOORI);
+            powerpc_excp(cpu, env->excp_model, POWERPC_EXCP_DOORI);
             return;
         }
         if (env->pending_interrupts & (1 << PPC_INTERRUPT_PERFM)) {
             env->pending_interrupts &= ~(1 << PPC_INTERRUPT_PERFM);
-            powerpc_excp(env, env->excp_model, POWERPC_EXCP_PERFM);
+            powerpc_excp(cpu, env->excp_model, POWERPC_EXCP_PERFM);
             return;
         }
         /* Thermal interrupt */
         if (env->pending_interrupts & (1 << PPC_INTERRUPT_THERM)) {
             env->pending_interrupts &= ~(1 << PPC_INTERRUPT_THERM);
-            powerpc_excp(env, env->excp_model, POWERPC_EXCP_THERM);
+            powerpc_excp(cpu, env->excp_model, POWERPC_EXCP_THERM);
             return;
         }
     }
diff --git a/target-ppc/helper.h b/target-ppc/helper.h
index fd04c063e..e588370e2 100644
--- a/target-ppc/helper.h
+++ b/target-ppc/helper.h
@@ -31,24 +31,24 @@ DEF_HELPER_2(icbi, void, env, tl)
 DEF_HELPER_5(lscbx, tl, env, tl, i32, i32, i32)
 
 #if defined(TARGET_PPC64)
-DEF_HELPER_FLAGS_2(mulhd, TCG_CALL_CONST | TCG_CALL_PURE, i64, i64, i64)
-DEF_HELPER_FLAGS_2(mulhdu, TCG_CALL_CONST | TCG_CALL_PURE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(mulhd, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(mulhdu, TCG_CALL_NO_RWG_SE, i64, i64, i64)
 DEF_HELPER_3(mulldo, i64, env, i64, i64)
 #endif
 
-DEF_HELPER_FLAGS_1(cntlzw, TCG_CALL_CONST | TCG_CALL_PURE, tl, tl)
-DEF_HELPER_FLAGS_1(popcntb, TCG_CALL_CONST | TCG_CALL_PURE, tl, tl)
-DEF_HELPER_FLAGS_1(popcntw, TCG_CALL_CONST | TCG_CALL_PURE, tl, tl)
+DEF_HELPER_FLAGS_1(cntlzw, TCG_CALL_NO_RWG_SE, tl, tl)
+DEF_HELPER_FLAGS_1(popcntb, TCG_CALL_NO_RWG_SE, tl, tl)
+DEF_HELPER_FLAGS_1(popcntw, TCG_CALL_NO_RWG_SE, tl, tl)
 DEF_HELPER_3(sraw, tl, env, tl, tl)
 #if defined(TARGET_PPC64)
-DEF_HELPER_FLAGS_1(cntlzd, TCG_CALL_CONST | TCG_CALL_PURE, tl, tl)
-DEF_HELPER_FLAGS_1(popcntd, TCG_CALL_CONST | TCG_CALL_PURE, tl, tl)
+DEF_HELPER_FLAGS_1(cntlzd, TCG_CALL_NO_RWG_SE, tl, tl)
+DEF_HELPER_FLAGS_1(popcntd, TCG_CALL_NO_RWG_SE, tl, tl)
 DEF_HELPER_3(srad, tl, env, tl, tl)
 #endif
 
-DEF_HELPER_FLAGS_1(cntlsw32, TCG_CALL_CONST | TCG_CALL_PURE, i32, i32)
-DEF_HELPER_FLAGS_1(cntlzw32, TCG_CALL_CONST | TCG_CALL_PURE, i32, i32)
-DEF_HELPER_FLAGS_2(brinc, TCG_CALL_CONST | TCG_CALL_PURE, tl, tl, tl)
+DEF_HELPER_FLAGS_1(cntlsw32, TCG_CALL_NO_RWG_SE, i32, i32)
+DEF_HELPER_FLAGS_1(cntlzw32, TCG_CALL_NO_RWG_SE, i32, i32)
+DEF_HELPER_FLAGS_2(brinc, TCG_CALL_NO_RWG_SE, tl, tl, tl)
 
 DEF_HELPER_1(float_check_status, void, env)
 DEF_HELPER_1(reset_fpstatus, void, env)
@@ -345,25 +345,25 @@ DEF_HELPER_2(6xx_tlbd, void, env, tl)
 DEF_HELPER_2(6xx_tlbi, void, env, tl)
 DEF_HELPER_2(74xx_tlbd, void, env, tl)
 DEF_HELPER_2(74xx_tlbi, void, env, tl)
-DEF_HELPER_FLAGS_1(tlbia, TCG_CALL_CONST, void, env)
-DEF_HELPER_FLAGS_2(tlbie, TCG_CALL_CONST, void, env, tl)
+DEF_HELPER_FLAGS_1(tlbia, TCG_CALL_NO_RWG, void, env)
+DEF_HELPER_FLAGS_2(tlbie, TCG_CALL_NO_RWG, void, env, tl)
 #if defined(TARGET_PPC64)
-DEF_HELPER_FLAGS_3(store_slb, TCG_CALL_CONST, void, env, tl, tl)
+DEF_HELPER_FLAGS_3(store_slb, TCG_CALL_NO_RWG, void, env, tl, tl)
 DEF_HELPER_2(load_slb_esid, tl, env, tl)
 DEF_HELPER_2(load_slb_vsid, tl, env, tl)
-DEF_HELPER_FLAGS_1(slbia, TCG_CALL_CONST, void, env)
-DEF_HELPER_FLAGS_2(slbie, TCG_CALL_CONST, void, env, tl)
+DEF_HELPER_FLAGS_1(slbia, TCG_CALL_NO_RWG, void, env)
+DEF_HELPER_FLAGS_2(slbie, TCG_CALL_NO_RWG, void, env, tl)
 #endif
-DEF_HELPER_FLAGS_2(load_sr, TCG_CALL_CONST, tl, env, tl);
-DEF_HELPER_FLAGS_3(store_sr, TCG_CALL_CONST, void, env, tl, tl)
+DEF_HELPER_FLAGS_2(load_sr, TCG_CALL_NO_RWG, tl, env, tl);
+DEF_HELPER_FLAGS_3(store_sr, TCG_CALL_NO_RWG, void, env, tl, tl)
 
-DEF_HELPER_FLAGS_1(602_mfrom, TCG_CALL_CONST | TCG_CALL_PURE, tl, tl)
+DEF_HELPER_FLAGS_1(602_mfrom, TCG_CALL_NO_RWG_SE, tl, tl)
 DEF_HELPER_1(msgsnd, void, tl)
 DEF_HELPER_2(msgclr, void, env, tl)
 #endif
 
 DEF_HELPER_4(dlmzb, tl, env, tl, tl, i32)
-DEF_HELPER_FLAGS_2(clcs, TCG_CALL_CONST | TCG_CALL_PURE, tl, env, i32)
+DEF_HELPER_FLAGS_2(clcs, TCG_CALL_NO_RWG_SE, tl, env, i32)
 #if !defined(CONFIG_USER_ONLY)
 DEF_HELPER_2(rac, tl, env, tl)
 #endif
diff --git a/target-ppc/int_helper.c b/target-ppc/int_helper.c
index f638b2a07..f39b4f682 100644
--- a/target-ppc/int_helper.c
+++ b/target-ppc/int_helper.c
@@ -287,23 +287,6 @@ target_ulong helper_602_mfrom(target_ulong arg)
     for (index = ARRAY_SIZE(r->element)-1; index >= 0; index--)
 #endif
 
-/* If X is a NaN, store the corresponding QNaN into RESULT.  Otherwise,
- * execute the following block.  */
-#define DO_HANDLE_NAN(result, x)                        \
-    if (float32_is_any_nan(x)) {                        \
-        CPU_FloatU __f;                                 \
-        __f.f = x;                                      \
-        __f.l = __f.l | (1 << 22);  /* Set QNaN bit. */ \
-        result = __f.f;                                 \
-    } else
-
-#define HANDLE_NAN1(result, x)                  \
-    DO_HANDLE_NAN(result, x)
-#define HANDLE_NAN2(result, x, y)                       \
-    DO_HANDLE_NAN(result, x) DO_HANDLE_NAN(result, y)
-#define HANDLE_NAN3(result, x, y, z)                                    \
-    DO_HANDLE_NAN(result, x) DO_HANDLE_NAN(result, y) DO_HANDLE_NAN(result, z)
-
 /* Saturating arithmetic helpers.  */
 #define SATCVT(from, to, from_type, to_type, min, max)          \
     static inline to_type cvt##from##to(from_type x, int *sat)  \
@@ -409,15 +392,29 @@ VARITH(uwm, u32)
         int i;                                                          \
                                                                         \
         for (i = 0; i < ARRAY_SIZE(r->f); i++) {                        \
-            HANDLE_NAN2(r->f[i], a->f[i], b->f[i]) {                    \
-                r->f[i] = func(a->f[i], b->f[i], &env->vec_status);     \
-            }                                                           \
+            r->f[i] = func(a->f[i], b->f[i], &env->vec_status);         \
         }                                                               \
     }
 VARITHFP(addfp, float32_add)
 VARITHFP(subfp, float32_sub)
+VARITHFP(minfp, float32_min)
+VARITHFP(maxfp, float32_max)
 #undef VARITHFP
 
+#define VARITHFPFMA(suffix, type)                                       \
+    void helper_v##suffix(CPUPPCState *env, ppc_avr_t *r, ppc_avr_t *a, \
+                           ppc_avr_t *b, ppc_avr_t *c)                  \
+    {                                                                   \
+        int i;                                                          \
+        for (i = 0; i < ARRAY_SIZE(r->f); i++) {                        \
+            r->f[i] = float32_muladd(a->f[i], c->f[i], b->f[i],         \
+                                     type, &env->vec_status);           \
+        }                                                               \
+    }
+VARITHFPFMA(maddfp, 0);
+VARITHFPFMA(nmsubfp, float_muladd_negate_result | float_muladd_negate_c);
+#undef VARITHFPFMA
+
 #define VARITHSAT_CASE(type, op, cvt, element)                          \
     {                                                                   \
         type result = (type)a->element[i] op (type)b->element[i];       \
@@ -649,27 +646,6 @@ VCT(uxs, cvtsduw, u32)
 VCT(sxs, cvtsdsw, s32)
 #undef VCT
 
-void helper_vmaddfp(CPUPPCState *env, ppc_avr_t *r, ppc_avr_t *a, ppc_avr_t *b,
-                    ppc_avr_t *c)
-{
-    int i;
-
-    for (i = 0; i < ARRAY_SIZE(r->f); i++) {
-        HANDLE_NAN3(r->f[i], a->f[i], b->f[i], c->f[i]) {
-            /* Need to do the computation in higher precision and round
-             * once at the end.  */
-            float64 af, bf, cf, t;
-
-            af = float32_to_float64(a->f[i], &env->vec_status);
-            bf = float32_to_float64(b->f[i], &env->vec_status);
-            cf = float32_to_float64(c->f[i], &env->vec_status);
-            t = float64_mul(af, cf, &env->vec_status);
-            t = float64_add(t, bf, &env->vec_status);
-            r->f[i] = float64_to_float32(t, &env->vec_status);
-        }
-    }
-}
-
 void helper_vmhaddshs(CPUPPCState *env, ppc_avr_t *r, ppc_avr_t *a,
                       ppc_avr_t *b, ppc_avr_t *c)
 {
@@ -730,27 +706,6 @@ VMINMAX(uw, u32)
 #undef VMINMAX_DO
 #undef VMINMAX
 
-#define VMINMAXFP(suffix, rT, rF)                                       \
-    void helper_v##suffix(CPUPPCState *env, ppc_avr_t *r, ppc_avr_t *a, \
-                          ppc_avr_t *b)                                 \
-    {                                                                   \
-        int i;                                                          \
-                                                                        \
-        for (i = 0; i < ARRAY_SIZE(r->f); i++) {                        \
-            HANDLE_NAN2(r->f[i], a->f[i], b->f[i]) {                    \
-                if (float32_lt_quiet(a->f[i], b->f[i],                  \
-                                     &env->vec_status)) {               \
-                    r->f[i] = rT->f[i];                                 \
-                } else {                                                \
-                    r->f[i] = rF->f[i];                                 \
-                }                                                       \
-            }                                                           \
-        }                                                               \
-    }
-VMINMAXFP(minfp, a, b)
-VMINMAXFP(maxfp, b, a)
-#undef VMINMAXFP
-
 void helper_vmladduhm(ppc_avr_t *r, ppc_avr_t *a, ppc_avr_t *b, ppc_avr_t *c)
 {
     int i;
@@ -930,28 +885,6 @@ VMUL(uh, u16, u32)
 #undef VMUL_DO
 #undef VMUL
 
-void helper_vnmsubfp(CPUPPCState *env, ppc_avr_t *r, ppc_avr_t *a,
-                     ppc_avr_t *b, ppc_avr_t *c)
-{
-    int i;
-
-    for (i = 0; i < ARRAY_SIZE(r->f); i++) {
-        HANDLE_NAN3(r->f[i], a->f[i], b->f[i], c->f[i]) {
-            /* Need to do the computation is higher precision and round
-             * once at the end.  */
-            float64 af, bf, cf, t;
-
-            af = float32_to_float64(a->f[i], &env->vec_status);
-            bf = float32_to_float64(b->f[i], &env->vec_status);
-            cf = float32_to_float64(c->f[i], &env->vec_status);
-            t = float64_mul(af, cf, &env->vec_status);
-            t = float64_sub(t, bf, &env->vec_status);
-            t = float64_chs(t);
-            r->f[i] = float64_to_float32(t, &env->vec_status);
-        }
-    }
-}
-
 void helper_vperm(CPUPPCState *env, ppc_avr_t *r, ppc_avr_t *a, ppc_avr_t *b,
                   ppc_avr_t *c)
 {
@@ -1039,9 +972,7 @@ void helper_vrefp(CPUPPCState *env, ppc_avr_t *r, ppc_avr_t *b)
     int i;
 
     for (i = 0; i < ARRAY_SIZE(r->f); i++) {
-        HANDLE_NAN1(r->f[i], b->f[i]) {
-            r->f[i] = float32_div(float32_one, b->f[i], &env->vec_status);
-        }
+        r->f[i] = float32_div(float32_one, b->f[i], &env->vec_status);
     }
 }
 
@@ -1054,9 +985,7 @@ void helper_vrefp(CPUPPCState *env, ppc_avr_t *r, ppc_avr_t *b)
                                                                 \
         set_float_rounding_mode(rounding, &s);                  \
         for (i = 0; i < ARRAY_SIZE(r->f); i++) {                \
-            HANDLE_NAN1(r->f[i], b->f[i]) {                     \
-                r->f[i] = float32_round_to_int (b->f[i], &s);   \
-            }                                                   \
+            r->f[i] = float32_round_to_int (b->f[i], &s);       \
         }                                                       \
     }
 VRFI(n, float_round_nearest_even)
@@ -1089,11 +1018,9 @@ void helper_vrsqrtefp(CPUPPCState *env, ppc_avr_t *r, ppc_avr_t *b)
     int i;
 
     for (i = 0; i < ARRAY_SIZE(r->f); i++) {
-        HANDLE_NAN1(r->f[i], b->f[i]) {
-            float32 t = float32_sqrt(b->f[i], &env->vec_status);
+        float32 t = float32_sqrt(b->f[i], &env->vec_status);
 
-            r->f[i] = float32_div(float32_one, t, &env->vec_status);
-        }
+        r->f[i] = float32_div(float32_one, t, &env->vec_status);
     }
 }
 
@@ -1109,9 +1036,7 @@ void helper_vexptefp(CPUPPCState *env, ppc_avr_t *r, ppc_avr_t *b)
     int i;
 
     for (i = 0; i < ARRAY_SIZE(r->f); i++) {
-        HANDLE_NAN1(r->f[i], b->f[i]) {
-            r->f[i] = float32_exp2(b->f[i], &env->vec_status);
-        }
+        r->f[i] = float32_exp2(b->f[i], &env->vec_status);
     }
 }
 
@@ -1120,9 +1045,7 @@ void helper_vlogefp(CPUPPCState *env, ppc_avr_t *r, ppc_avr_t *b)
     int i;
 
     for (i = 0; i < ARRAY_SIZE(r->f); i++) {
-        HANDLE_NAN1(r->f[i], b->f[i]) {
-            r->f[i] = float32_log2(b->f[i], &env->vec_status);
-        }
+        r->f[i] = float32_log2(b->f[i], &env->vec_status);
     }
 }
 
@@ -1473,10 +1396,6 @@ VUPK(lsh, s32, s16, UPKLO)
 #undef UPKHI
 #undef UPKLO
 
-#undef DO_HANDLE_NAN
-#undef HANDLE_NAN1
-#undef HANDLE_NAN2
-#undef HANDLE_NAN3
 #undef VECTOR_FOR_INORDER_I
 #undef HI_IDX
 #undef LO_IDX
diff --git a/target-ppc/kvm.c b/target-ppc/kvm.c
index a31d278a5..3f5df5772 100644
--- a/target-ppc/kvm.c
+++ b/target-ppc/kvm.c
@@ -60,6 +60,7 @@ static int cap_booke_sregs;
 static int cap_ppc_smt;
 static int cap_ppc_rma;
 static int cap_spapr_tce;
+static int cap_hior;
 
 /* XXX We have a race condition where we actually have a level triggered
  *     interrupt, but the infrastructure can't expose that yet, so the guest
@@ -72,9 +73,11 @@ static int cap_spapr_tce;
  */
 static QEMUTimer *idle_timer;
 
-static void kvm_kick_env(void *env)
+static void kvm_kick_cpu(void *opaque)
 {
-    qemu_cpu_kick(env);
+    PowerPCCPU *cpu = opaque;
+
+    qemu_cpu_kick(CPU(cpu));
 }
 
 int kvm_arch_init(KVMState *s)
@@ -86,6 +89,7 @@ int kvm_arch_init(KVMState *s)
     cap_ppc_smt = kvm_check_extension(s, KVM_CAP_PPC_SMT);
     cap_ppc_rma = kvm_check_extension(s, KVM_CAP_PPC_RMA);
     cap_spapr_tce = kvm_check_extension(s, KVM_CAP_SPAPR_TCE);
+    cap_hior = kvm_check_extension(s, KVM_CAP_PPC_HIOR);
 
     if (!cap_interrupt_level) {
         fprintf(stderr, "KVM: Couldn't find level irq capability. Expect the "
@@ -373,6 +377,7 @@ static inline void kvm_fixup_page_sizes(CPUPPCState *env)
 
 int kvm_arch_init_vcpu(CPUPPCState *cenv)
 {
+    PowerPCCPU *cpu = ppc_env_get_cpu(cenv);
     int ret;
 
     /* Gather server mmu info from KVM and update the CPU state */
@@ -384,7 +389,7 @@ int kvm_arch_init_vcpu(CPUPPCState *cenv)
         return ret;
     }
 
-    idle_timer = qemu_new_timer_ns(vm_clock, kvm_kick_env, cenv);
+    idle_timer = qemu_new_timer_ns(vm_clock, kvm_kick_cpu, cpu);
 
     /* Some targets support access to KVM's guest TLB. */
     switch (cenv->mmu_model) {
@@ -469,6 +474,54 @@ int kvm_arch_put_registers(CPUPPCState *env, int level)
         env->tlb_dirty = false;
     }
 
+    if (cap_segstate && (level >= KVM_PUT_RESET_STATE)) {
+        struct kvm_sregs sregs;
+
+        sregs.pvr = env->spr[SPR_PVR];
+
+        sregs.u.s.sdr1 = env->spr[SPR_SDR1];
+
+        /* Sync SLB */
+#ifdef TARGET_PPC64
+        for (i = 0; i < 64; i++) {
+            sregs.u.s.ppc64.slb[i].slbe = env->slb[i].esid;
+            sregs.u.s.ppc64.slb[i].slbv = env->slb[i].vsid;
+        }
+#endif
+
+        /* Sync SRs */
+        for (i = 0; i < 16; i++) {
+            sregs.u.s.ppc32.sr[i] = env->sr[i];
+        }
+
+        /* Sync BATs */
+        for (i = 0; i < 8; i++) {
+            /* Beware. We have to swap upper and lower bits here */
+            sregs.u.s.ppc32.dbat[i] = ((uint64_t)env->DBAT[0][i] << 32)
+                | env->DBAT[1][i];
+            sregs.u.s.ppc32.ibat[i] = ((uint64_t)env->IBAT[0][i] << 32)
+                | env->IBAT[1][i];
+        }
+
+        ret = kvm_vcpu_ioctl(env, KVM_SET_SREGS, &sregs);
+        if (ret) {
+            return ret;
+        }
+    }
+
+    if (cap_hior && (level >= KVM_PUT_RESET_STATE)) {
+        uint64_t hior = env->spr[SPR_HIOR];
+        struct kvm_one_reg reg = {
+            .id = KVM_REG_PPC_HIOR,
+            .addr = (uintptr_t) &hior,
+        };
+
+        ret = kvm_vcpu_ioctl(env, KVM_SET_ONE_REG, &reg);
+        if (ret) {
+            return ret;
+        }
+    }
+
     return ret;
 }
 
@@ -764,7 +817,8 @@ int kvm_arch_handle_exit(CPUPPCState *env, struct kvm_run *run)
 #ifdef CONFIG_PSERIES
     case KVM_EXIT_PAPR_HCALL:
         dprintf("handle PAPR hypercall\n");
-        run->papr_hcall.ret = spapr_hypercall(env, run->papr_hcall.nr,
+        run->papr_hcall.ret = spapr_hypercall(ppc_env_get_cpu(env),
+                                              run->papr_hcall.nr,
                                               run->papr_hcall.args);
         ret = 0;
         break;
@@ -795,7 +849,7 @@ static int read_cpuinfo(const char *field, char *value, int len)
             break;
         }
         if (!strncmp(line, field, field_len)) {
-            strncpy(value, line, len);
+            pstrcpy(value, len, line);
             ret = 0;
             break;
         }
@@ -946,52 +1000,14 @@ int kvmppc_get_hypercall(CPUPPCState *env, uint8_t *buf, int buf_len)
 void kvmppc_set_papr(CPUPPCState *env)
 {
     struct kvm_enable_cap cap = {};
-    struct kvm_one_reg reg = {};
-    struct kvm_sregs sregs = {};
     int ret;
-    uint64_t hior = env->spr[SPR_HIOR];
 
     cap.cap = KVM_CAP_PPC_PAPR;
     ret = kvm_vcpu_ioctl(env, KVM_ENABLE_CAP, &cap);
 
     if (ret) {
-        goto fail;
+        cpu_abort(env, "This KVM version does not support PAPR\n");
     }
-
-    /*
-     * XXX We set HIOR here. It really should be a qdev property of
-     *     the CPU node, but we don't have CPUs converted to qdev yet.
-     *
-     *     Once we have qdev CPUs, move HIOR to a qdev property and
-     *     remove this chunk.
-     */
-    reg.id = KVM_REG_PPC_HIOR;
-    reg.addr = (uintptr_t)&hior;
-    ret = kvm_vcpu_ioctl(env, KVM_SET_ONE_REG, &reg);
-    if (ret) {
-        fprintf(stderr, "Couldn't set HIOR. Maybe you're running an old \n"
-                        "kernel with support for HV KVM but no PAPR PR \n"
-                        "KVM in which case things will work. If they don't \n"
-                        "please update your host kernel!\n");
-    }
-
-    /* Set SDR1 so kernel space finds the HTAB */
-    ret = kvm_vcpu_ioctl(env, KVM_GET_SREGS, &sregs);
-    if (ret) {
-        goto fail;
-    }
-
-    sregs.u.s.sdr1 = env->spr[SPR_SDR1];
-
-    ret = kvm_vcpu_ioctl(env, KVM_SET_SREGS, &sregs);
-    if (ret) {
-        goto fail;
-    }
-
-    return;
-
-fail:
-    cpu_abort(env, "This KVM version does not support PAPR\n");
 }
 
 int kvmppc_smt_threads(void)
@@ -999,6 +1015,7 @@ int kvmppc_smt_threads(void)
     return cap_ppc_smt ? cap_ppc_smt : 1;
 }
 
+#ifdef TARGET_PPC64
 off_t kvmppc_alloc_rma(const char *name, MemoryRegion *sysmem)
 {
     void *rma;
@@ -1042,6 +1059,16 @@ off_t kvmppc_alloc_rma(const char *name, MemoryRegion *sysmem)
     return size;
 }
 
+uint64_t kvmppc_rma_size(uint64_t current_size, unsigned int hash_shift)
+{
+    if (cap_ppc_rma >= 2) {
+        return current_size;
+    }
+    return MIN(current_size,
+               getrampagesize() << (hash_shift - 7));
+}
+#endif
+
 void *kvmppc_create_spapr_tce(uint32_t liobn, uint32_t window_size, int *pfd)
 {
     struct kvm_create_spapr_tce args = {
@@ -1101,6 +1128,44 @@ int kvmppc_remove_spapr_tce(void *table, int fd, uint32_t window_size)
     return 0;
 }
 
+int kvmppc_reset_htab(int shift_hint)
+{
+    uint32_t shift = shift_hint;
+
+    if (!kvm_enabled()) {
+        /* Full emulation, tell caller to allocate htab itself */
+        return 0;
+    }
+    if (kvm_check_extension(kvm_state, KVM_CAP_PPC_ALLOC_HTAB)) {
+        int ret;
+        ret = kvm_vm_ioctl(kvm_state, KVM_PPC_ALLOCATE_HTAB, &shift);
+        if (ret == -ENOTTY) {
+            /* At least some versions of PR KVM advertise the
+             * capability, but don't implement the ioctl().  Oops.
+             * Return 0 so that we allocate the htab in qemu, as is
+             * correct for PR. */
+            return 0;
+        } else if (ret < 0) {
+            return ret;
+        }
+        return shift;
+    }
+
+    /* We have a kernel that predates the htab reset calls.  For PR
+     * KVM, we need to allocate the htab ourselves, for an HV KVM of
+     * this era, it has allocated a 16MB fixed size hash table
+     * already.  Kernels of this era have the GET_PVINFO capability
+     * only on PR, so we use this hack to determine the right
+     * answer */
+    if (kvm_check_extension(kvm_state, KVM_CAP_PPC_GET_PVINFO)) {
+        /* PR - tell caller to allocate htab */
+        return 0;
+    } else {
+        /* HV - assume 16MB kernel allocated htab */
+        return 24;
+    }
+}
+
 static inline uint32_t mfpvr(void)
 {
     uint32_t pvr;
diff --git a/target-ppc/kvm_ppc.h b/target-ppc/kvm_ppc.h
index e2f870385..baad6eb75 100644
--- a/target-ppc/kvm_ppc.h
+++ b/target-ppc/kvm_ppc.h
@@ -27,6 +27,8 @@ int kvmppc_smt_threads(void);
 off_t kvmppc_alloc_rma(const char *name, MemoryRegion *sysmem);
 void *kvmppc_create_spapr_tce(uint32_t liobn, uint32_t window_size, int *pfd);
 int kvmppc_remove_spapr_tce(void *table, int pfd, uint32_t window_size);
+int kvmppc_reset_htab(int shift_hint);
+uint64_t kvmppc_rma_size(uint64_t current_size, unsigned int hash_shift);
 #endif /* !CONFIG_USER_ONLY */
 const ppc_def_t *kvmppc_host_cpu_def(void);
 int kvmppc_fixup_cpu(CPUPPCState *env);
@@ -94,6 +96,23 @@ static inline int kvmppc_remove_spapr_tce(void *table, int pfd,
 {
     return -1;
 }
+
+static inline int kvmppc_reset_htab(int shift_hint)
+{
+    return -1;
+}
+
+static inline uint64_t kvmppc_rma_size(uint64_t current_size,
+                                       unsigned int hash_shift)
+{
+    return ram_size;
+}
+
+static inline int kvmppc_update_sdr1(CPUPPCState *env)
+{
+    return 0;
+}
+
 #endif /* !CONFIG_USER_ONLY */
 
 static inline const ppc_def_t *kvmppc_host_cpu_def(void)
diff --git a/target-ppc/machine.c b/target-ppc/machine.c
index d6c2ee41b..5e7bc00e2 100644
--- a/target-ppc/machine.c
+++ b/target-ppc/machine.c
@@ -6,6 +6,7 @@ void cpu_save(QEMUFile *f, void *opaque)
 {
     CPUPPCState *env = (CPUPPCState *)opaque;
     unsigned int i, j;
+    uint32_t fpscr;
 
     for (i = 0; i < 32; i++)
         qemu_put_betls(f, &env->gpr[i]);
@@ -30,7 +31,8 @@ void cpu_save(QEMUFile *f, void *opaque)
         u.d = env->fpr[i];
         qemu_put_be64(f, u.l);
     }
-    qemu_put_be32s(f, &env->fpscr);
+    fpscr = env->fpscr;
+    qemu_put_be32s(f, &fpscr);
     qemu_put_sbe32s(f, &env->access_type);
 #if defined(TARGET_PPC64)
     qemu_put_betls(f, &env->asr);
@@ -82,7 +84,7 @@ void cpu_save(QEMUFile *f, void *opaque)
     qemu_put_betls(f, &env->hflags);
     qemu_put_betls(f, &env->hflags_nmsr);
     qemu_put_sbe32s(f, &env->mmu_idx);
-    qemu_put_sbe32s(f, &env->power_mode);
+    qemu_put_sbe32(f, 0);
 }
 
 int cpu_load(QEMUFile *f, void *opaque, int version_id)
@@ -90,6 +92,7 @@ int cpu_load(QEMUFile *f, void *opaque, int version_id)
     CPUPPCState *env = (CPUPPCState *)opaque;
     unsigned int i, j;
     target_ulong sdr1;
+    uint32_t fpscr;
 
     for (i = 0; i < 32; i++)
         qemu_get_betls(f, &env->gpr[i]);
@@ -114,7 +117,8 @@ int cpu_load(QEMUFile *f, void *opaque, int version_id)
         u.l = qemu_get_be64(f);
         env->fpr[i] = u.d;
     }
-    qemu_get_be32s(f, &env->fpscr);
+    qemu_get_be32s(f, &fpscr);
+    env->fpscr = fpscr;
     qemu_get_sbe32s(f, &env->access_type);
 #if defined(TARGET_PPC64)
     qemu_get_betls(f, &env->asr);
@@ -167,7 +171,7 @@ int cpu_load(QEMUFile *f, void *opaque, int version_id)
     qemu_get_betls(f, &env->hflags);
     qemu_get_betls(f, &env->hflags_nmsr);
     qemu_get_sbe32s(f, &env->mmu_idx);
-    qemu_get_sbe32s(f, &env->power_mode);
+    qemu_get_sbe32(f); /* Discard unused power_mode */
 
     return 0;
 }
diff --git a/target-ppc/mmu_helper.c b/target-ppc/mmu_helper.c
index d2664acef..318ce92ad 100644
--- a/target-ppc/mmu_helper.c
+++ b/target-ppc/mmu_helper.c
@@ -215,7 +215,7 @@ static inline int pte_check(mmu_ctx_t *ctx, int is_64b, target_ulong pte0,
             pp = pte1 & 0x00000003;
         }
         if (ptem == ctx->ptem) {
-            if (ctx->raddr != (target_phys_addr_t)-1ULL) {
+            if (ctx->raddr != (hwaddr)-1ULL) {
                 /* all matches should have equal RPN, WIMG & PP */
                 if ((ctx->raddr & mmask) != (pte1 & mmask)) {
                     qemu_log("Bad RPN/WIMG/PP\n");
@@ -556,8 +556,8 @@ static inline int get_bat(CPUPPCState *env, mmu_ctx_t *ctx,
     return ret;
 }
 
-static inline target_phys_addr_t get_pteg_offset(CPUPPCState *env,
-                                                 target_phys_addr_t hash,
+static inline hwaddr get_pteg_offset(CPUPPCState *env,
+                                                 hwaddr hash,
                                                  int pte_size)
 {
     return (hash * pte_size * 8) & env->htab_mask;
@@ -567,7 +567,7 @@ static inline target_phys_addr_t get_pteg_offset(CPUPPCState *env,
 static inline int find_pte2(CPUPPCState *env, mmu_ctx_t *ctx, int is_64b, int h,
                             int rw, int type, int target_page_bits)
 {
-    target_phys_addr_t pteg_off;
+    hwaddr pteg_off;
     target_ulong pte0, pte1;
     int i, good = -1;
     int ret, r;
@@ -817,7 +817,7 @@ static int ppc_load_slb_vsid(CPUPPCState *env, target_ulong rb,
 static inline int get_segment(CPUPPCState *env, mmu_ctx_t *ctx,
                               target_ulong eaddr, int rw, int type)
 {
-    target_phys_addr_t hash;
+    hwaddr hash;
     target_ulong vsid;
     int ds, pr, target_page_bits;
     int ret, ret2;
@@ -896,7 +896,7 @@ static inline int get_segment(CPUPPCState *env, mmu_ctx_t *ctx,
             ctx->hash[1] = ~hash;
 
             /* Initialize real address with an invalid value */
-            ctx->raddr = (target_phys_addr_t)-1ULL;
+            ctx->raddr = (hwaddr)-1ULL;
             if (unlikely(env->mmu_model == POWERPC_MMU_SOFT_6xx ||
                          env->mmu_model == POWERPC_MMU_SOFT_74xx)) {
                 /* Software TLB search */
@@ -926,7 +926,7 @@ static inline int get_segment(CPUPPCState *env, mmu_ctx_t *ctx,
             }
 #if defined(DUMP_PAGE_TABLES)
             if (qemu_log_enabled()) {
-                target_phys_addr_t curaddr;
+                hwaddr curaddr;
                 uint32_t a0, a1, a2, a3;
 
                 qemu_log("Page table: " TARGET_FMT_plx " len " TARGET_FMT_plx
@@ -1009,7 +1009,7 @@ static inline int get_segment(CPUPPCState *env, mmu_ctx_t *ctx,
 
 /* Generic TLB check function for embedded PowerPC implementations */
 static int ppcemb_tlb_check(CPUPPCState *env, ppcemb_tlb_t *tlb,
-                            target_phys_addr_t *raddrp,
+                            hwaddr *raddrp,
                             target_ulong address, uint32_t pid, int ext,
                             int i)
 {
@@ -1032,12 +1032,10 @@ static int ppcemb_tlb_check(CPUPPCState *env, ppcemb_tlb_t *tlb,
         return -1;
     }
     *raddrp = (tlb->RPN & mask) | (address & ~mask);
-#if (TARGET_PHYS_ADDR_BITS >= 36)
     if (ext) {
         /* Extend the physical address to 36 bits */
-        *raddrp |= (target_phys_addr_t)(tlb->RPN & 0xF) << 32;
+        *raddrp |= (uint64_t)(tlb->RPN & 0xF) << 32;
     }
-#endif
 
     return 0;
 }
@@ -1047,7 +1045,7 @@ static int ppcemb_tlb_search(CPUPPCState *env, target_ulong address,
                              uint32_t pid)
 {
     ppcemb_tlb_t *tlb;
-    target_phys_addr_t raddr;
+    hwaddr raddr;
     int i, ret;
 
     /* Default return value is no match */
@@ -1081,7 +1079,7 @@ static inline void ppc4xx_tlb_invalidate_virt(CPUPPCState *env,
 {
 #if !defined(FLUSH_ALL_TLBS)
     ppcemb_tlb_t *tlb;
-    target_phys_addr_t raddr;
+    hwaddr raddr;
     target_ulong page, end;
     int i;
 
@@ -1106,11 +1104,11 @@ static int mmu40x_get_physical_address(CPUPPCState *env, mmu_ctx_t *ctx,
                                        int access_type)
 {
     ppcemb_tlb_t *tlb;
-    target_phys_addr_t raddr;
+    hwaddr raddr;
     int i, ret, zsel, zpr, pr;
 
     ret = -1;
-    raddr = (target_phys_addr_t)-1ULL;
+    raddr = (hwaddr)-1ULL;
     pr = msr_pr;
     for (i = 0; i < env->nb_tlb; i++) {
         tlb = &env->tlb.tlbe[i];
@@ -1177,7 +1175,7 @@ void store_40x_sler(CPUPPCState *env, uint32_t val)
 }
 
 static inline int mmubooke_check_tlb(CPUPPCState *env, ppcemb_tlb_t *tlb,
-                                     target_phys_addr_t *raddr, int *prot,
+                                     hwaddr *raddr, int *prot,
                                      target_ulong address, int rw,
                                      int access_type, int i)
 {
@@ -1251,11 +1249,11 @@ static int mmubooke_get_physical_address(CPUPPCState *env, mmu_ctx_t *ctx,
                                          int access_type)
 {
     ppcemb_tlb_t *tlb;
-    target_phys_addr_t raddr;
+    hwaddr raddr;
     int i, ret;
 
     ret = -1;
-    raddr = (target_phys_addr_t)-1ULL;
+    raddr = (hwaddr)-1ULL;
     for (i = 0; i < env->nb_tlb; i++) {
         tlb = &env->tlb.tlbe[i];
         ret = mmubooke_check_tlb(env, tlb, &raddr, &ctx->prot, address, rw,
@@ -1278,7 +1276,8 @@ static int mmubooke_get_physical_address(CPUPPCState *env, mmu_ctx_t *ctx,
     return ret;
 }
 
-void booke206_flush_tlb(CPUPPCState *env, int flags, const int check_iprot)
+static void booke206_flush_tlb(CPUPPCState *env, int flags,
+                               const int check_iprot)
 {
     int tlb_size;
     int i, j;
@@ -1299,8 +1298,8 @@ void booke206_flush_tlb(CPUPPCState *env, int flags, const int check_iprot)
     tlb_flush(env, 1);
 }
 
-target_phys_addr_t booke206_tlb_to_page_size(CPUPPCState *env,
-                                             ppcmas_tlb_t *tlb)
+static hwaddr booke206_tlb_to_page_size(CPUPPCState *env,
+                                        ppcmas_tlb_t *tlb)
 {
     int tlbm_size;
 
@@ -1311,7 +1310,7 @@ target_phys_addr_t booke206_tlb_to_page_size(CPUPPCState *env,
 
 /* TLB check function for MAS based SoftTLBs */
 int ppcmas_tlb_check(CPUPPCState *env, ppcmas_tlb_t *tlb,
-                     target_phys_addr_t *raddrp,
+                     hwaddr *raddrp,
                      target_ulong address, uint32_t pid)
 {
     target_ulong mask;
@@ -1347,7 +1346,7 @@ int ppcmas_tlb_check(CPUPPCState *env, ppcmas_tlb_t *tlb,
 }
 
 static int mmubooke206_check_tlb(CPUPPCState *env, ppcmas_tlb_t *tlb,
-                                 target_phys_addr_t *raddr, int *prot,
+                                 hwaddr *raddr, int *prot,
                                  target_ulong address, int rw,
                                  int access_type)
 {
@@ -1437,11 +1436,11 @@ static int mmubooke206_get_physical_address(CPUPPCState *env, mmu_ctx_t *ctx,
                                             int access_type)
 {
     ppcmas_tlb_t *tlb;
-    target_phys_addr_t raddr;
+    hwaddr raddr;
     int i, j, ret;
 
     ret = -1;
-    raddr = (target_phys_addr_t)-1ULL;
+    raddr = (hwaddr)-1ULL;
 
     for (i = 0; i < BOOKE206_MAX_TLBN; i++) {
         int ways = booke206_tlb_ways(env, i);
@@ -1498,7 +1497,7 @@ static void mmubooke_dump_mmu(FILE *f, fprintf_function cpu_fprintf,
 
     entry = &env->tlb.tlbe[0];
     for (i = 0; i < env->nb_tlb; i++, entry++) {
-        target_phys_addr_t ea, pa;
+        hwaddr ea, pa;
         target_ulong mask;
         uint64_t size = (uint64_t)entry->size;
         char size_buf[20];
@@ -1511,10 +1510,8 @@ static void mmubooke_dump_mmu(FILE *f, fprintf_function cpu_fprintf,
         mask = ~(entry->size - 1);
         ea = entry->EPN & mask;
         pa = entry->RPN & mask;
-#if (TARGET_PHYS_ADDR_BITS >= 36)
         /* Extend the physical address to 36 bits */
-        pa |= (target_phys_addr_t)(entry->RPN & 0xF) << 32;
-#endif
+        pa |= (hwaddr)(entry->RPN & 0xF) << 32;
         size /= 1024;
         if (size >= 1024) {
             snprintf(size_buf, sizeof(size_buf), "%3" PRId64 "M", size / 1024);
@@ -1541,7 +1538,7 @@ static void mmubooke206_dump_one_tlb(FILE *f, fprintf_function cpu_fprintf,
 
     entry = &env->tlb.tlbm[offset];
     for (i = 0; i < tlbsize; i++, entry++) {
-        target_phys_addr_t ea, pa, size;
+        hwaddr ea, pa, size;
         int tsize;
 
         if (!(entry->mas1 & MAS1_VALID)) {
@@ -1710,8 +1707,8 @@ static inline int check_physical(CPUPPCState *env, mmu_ctx_t *ctx,
     return ret;
 }
 
-int get_physical_address(CPUPPCState *env, mmu_ctx_t *ctx, target_ulong eaddr,
-                         int rw, int access_type)
+static int get_physical_address(CPUPPCState *env, mmu_ctx_t *ctx,
+                                target_ulong eaddr, int rw, int access_type)
 {
     int ret;
 
@@ -1787,7 +1784,7 @@ int get_physical_address(CPUPPCState *env, mmu_ctx_t *ctx, target_ulong eaddr,
     return ret;
 }
 
-target_phys_addr_t cpu_get_phys_page_debug(CPUPPCState *env, target_ulong addr)
+hwaddr cpu_get_phys_page_debug(CPUPPCState *env, target_ulong addr)
 {
     mmu_ctx_t ctx;
 
@@ -3147,7 +3144,7 @@ void helper_booke206_tlbsx(CPUPPCState *env, target_ulong address)
 {
     ppcmas_tlb_t *tlb = NULL;
     int i, j;
-    target_phys_addr_t raddr;
+    hwaddr raddr;
     uint32_t spid, sas;
 
     spid = (env->spr[SPR_BOOKE_MAS6] & MAS6_SPID_MASK) >> MAS6_SPID_SHIFT;
diff --git a/target-ppc/translate.c b/target-ppc/translate.c
index ac915ccad..987b04eda 100644
--- a/target-ppc/translate.c
+++ b/target-ppc/translate.c
@@ -68,7 +68,7 @@ static TCGv cpu_cfar;
 #endif
 static TCGv cpu_xer;
 static TCGv cpu_reserve;
-static TCGv_i32 cpu_fpscr;
+static TCGv cpu_fpscr;
 static TCGv_i32 cpu_access_type;
 
 #include "gen-icount.h"
@@ -163,8 +163,8 @@ void ppc_translate_init(void)
                                      offsetof(CPUPPCState, reserve_addr),
                                      "reserve_addr");
 
-    cpu_fpscr = tcg_global_mem_new_i32(TCG_AREG0,
-                                       offsetof(CPUPPCState, fpscr), "fpscr");
+    cpu_fpscr = tcg_global_mem_new(TCG_AREG0,
+                                   offsetof(CPUPPCState, fpscr), "fpscr");
 
     cpu_access_type = tcg_global_mem_new_i32(TCG_AREG0,
                                              offsetof(CPUPPCState, access_type), "access_type");
@@ -2302,6 +2302,7 @@ GEN_FLOAT_B(neg, 0x08, 0x01, 0, PPC_FLOAT);
 /* mcrfs */
 static void gen_mcrfs(DisasContext *ctx)
 {
+    TCGv tmp = tcg_temp_new();
     int bfa;
 
     if (unlikely(!ctx->fpu_enabled)) {
@@ -2309,9 +2310,11 @@ static void gen_mcrfs(DisasContext *ctx)
         return;
     }
     bfa = 4 * (7 - crfS(ctx->opcode));
-    tcg_gen_shri_i32(cpu_crf[crfD(ctx->opcode)], cpu_fpscr, bfa);
+    tcg_gen_shri_tl(tmp, cpu_fpscr, bfa);
+    tcg_gen_trunc_tl_i32(cpu_crf[crfD(ctx->opcode)], tmp);
+    tcg_temp_free(tmp);
     tcg_gen_andi_i32(cpu_crf[crfD(ctx->opcode)], cpu_crf[crfD(ctx->opcode)], 0xf);
-    tcg_gen_andi_i32(cpu_fpscr, cpu_fpscr, ~(0xF << bfa));
+    tcg_gen_andi_tl(cpu_fpscr, cpu_fpscr, ~(0xF << bfa));
 }
 
 /* mffs */
@@ -2322,7 +2325,7 @@ static void gen_mffs(DisasContext *ctx)
         return;
     }
     gen_reset_fpstatus();
-    tcg_gen_extu_i32_i64(cpu_fpr[rD(ctx->opcode)], cpu_fpscr);
+    tcg_gen_extu_tl_i64(cpu_fpr[rD(ctx->opcode)], cpu_fpscr);
     gen_compute_fprf(cpu_fpr[rD(ctx->opcode)], 0, Rc(ctx->opcode) != 0);
 }
 
@@ -2346,7 +2349,8 @@ static void gen_mtfsb0(DisasContext *ctx)
         tcg_temp_free_i32(t0);
     }
     if (unlikely(Rc(ctx->opcode) != 0)) {
-        tcg_gen_shri_i32(cpu_crf[1], cpu_fpscr, FPSCR_OX);
+        tcg_gen_trunc_tl_i32(cpu_crf[1], cpu_fpscr);
+        tcg_gen_shri_i32(cpu_crf[1], cpu_crf[1], FPSCR_OX);
     }
 }
 
@@ -2371,7 +2375,8 @@ static void gen_mtfsb1(DisasContext *ctx)
         tcg_temp_free_i32(t0);
     }
     if (unlikely(Rc(ctx->opcode) != 0)) {
-        tcg_gen_shri_i32(cpu_crf[1], cpu_fpscr, FPSCR_OX);
+        tcg_gen_trunc_tl_i32(cpu_crf[1], cpu_fpscr);
+        tcg_gen_shri_i32(cpu_crf[1], cpu_crf[1], FPSCR_OX);
     }
     /* We can raise a differed exception */
     gen_helper_float_check_status(cpu_env);
@@ -2397,7 +2402,8 @@ static void gen_mtfsf(DisasContext *ctx)
     gen_helper_store_fpscr(cpu_env, cpu_fpr[rB(ctx->opcode)], t0);
     tcg_temp_free_i32(t0);
     if (unlikely(Rc(ctx->opcode) != 0)) {
-        tcg_gen_shri_i32(cpu_crf[1], cpu_fpscr, FPSCR_OX);
+        tcg_gen_trunc_tl_i32(cpu_crf[1], cpu_fpscr);
+        tcg_gen_shri_i32(cpu_crf[1], cpu_crf[1], FPSCR_OX);
     }
     /* We can raise a differed exception */
     gen_helper_float_check_status(cpu_env);
@@ -2425,7 +2431,8 @@ static void gen_mtfsfi(DisasContext *ctx)
     tcg_temp_free_i64(t0);
     tcg_temp_free_i32(t1);
     if (unlikely(Rc(ctx->opcode) != 0)) {
-        tcg_gen_shri_i32(cpu_crf[1], cpu_fpscr, FPSCR_OX);
+        tcg_gen_trunc_tl_i32(cpu_crf[1], cpu_fpscr);
+        tcg_gen_shri_i32(cpu_crf[1], cpu_crf[1], FPSCR_OX);
     }
     /* We can raise a differed exception */
     gen_helper_float_check_status(cpu_env);
@@ -3466,7 +3473,8 @@ static inline void gen_goto_tb(DisasContext *ctx, int n, target_ulong dest)
         if (unlikely(ctx->singlestep_enabled)) {
             if ((ctx->singlestep_enabled &
                 (CPU_BRANCH_STEP | CPU_SINGLE_STEP)) &&
-                ctx->exception == POWERPC_EXCP_BRANCH) {
+                (ctx->exception == POWERPC_EXCP_BRANCH ||
+                 ctx->exception == POWERPC_EXCP_TRACE)) {
                 target_ulong tmp = ctx->nip;
                 ctx->nip = dest;
                 gen_exception(ctx, POWERPC_EXCP_TRACE);
@@ -9463,7 +9471,7 @@ void cpu_dump_state (CPUPPCState *env, FILE *f, fprintf_function cpu_fprintf,
         if ((i & (RFPL - 1)) == (RFPL - 1))
             cpu_fprintf(f, "\n");
     }
-    cpu_fprintf(f, "FPSCR %08x\n", env->fpscr);
+    cpu_fprintf(f, "FPSCR " TARGET_FMT_lx "\n", env->fpscr);
 #if !defined(CONFIG_USER_ONLY)
     cpu_fprintf(f, " SRR0 " TARGET_FMT_lx "  SRR1 " TARGET_FMT_lx
                    "    PVR " TARGET_FMT_lx " VRSAVE " TARGET_FMT_lx "\n",
@@ -9617,7 +9625,7 @@ static inline void gen_intermediate_code_internal(CPUPPCState *env,
     int max_insns;
 
     pc_start = tb->pc;
-    gen_opc_end = gen_opc_buf + OPC_MAX_SIZE;
+    gen_opc_end = tcg_ctx.gen_opc_buf + OPC_MAX_SIZE;
     ctx.nip = pc_start;
     ctx.tb = tb;
     ctx.exception = POWERPC_EXCP_NONE;
@@ -9657,7 +9665,8 @@ static inline void gen_intermediate_code_internal(CPUPPCState *env,
 
     gen_icount_start();
     /* Set env in case of segfault during code fetch */
-    while (ctx.exception == POWERPC_EXCP_NONE && gen_opc_ptr < gen_opc_end) {
+    while (ctx.exception == POWERPC_EXCP_NONE
+            && tcg_ctx.gen_opc_ptr < gen_opc_end) {
         if (unlikely(!QTAILQ_EMPTY(&env->breakpoints))) {
             QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
                 if (bp->pc == ctx.nip) {
@@ -9667,7 +9676,7 @@ static inline void gen_intermediate_code_internal(CPUPPCState *env,
             }
         }
         if (unlikely(search_pc)) {
-            j = gen_opc_ptr - gen_opc_buf;
+            j = tcg_ctx.gen_opc_ptr - tcg_ctx.gen_opc_buf;
             if (lj < j) {
                 lj++;
                 while (lj < j)
@@ -9690,8 +9699,9 @@ static inline void gen_intermediate_code_internal(CPUPPCState *env,
         LOG_DISAS("translate opcode %08x (%02x %02x %02x) (%s)\n",
                     ctx.opcode, opc1(ctx.opcode), opc2(ctx.opcode),
                     opc3(ctx.opcode), little_endian ? "little" : "big");
-        if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP)))
+        if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
             tcg_gen_debug_insn_start(ctx.nip);
+        }
         ctx.nip += 4;
         table = env->opcodes;
         num_insns++;
@@ -9766,9 +9776,9 @@ static inline void gen_intermediate_code_internal(CPUPPCState *env,
         tcg_gen_exit_tb(0);
     }
     gen_icount_end(tb, num_insns);
-    *gen_opc_ptr = INDEX_op_end;
+    *tcg_ctx.gen_opc_ptr = INDEX_op_end;
     if (unlikely(search_pc)) {
-        j = gen_opc_ptr - gen_opc_buf;
+        j = tcg_ctx.gen_opc_ptr - tcg_ctx.gen_opc_buf;
         lj++;
         while (lj <= j)
             gen_opc_instr_start[lj++] = 0;
@@ -9782,7 +9792,7 @@ static inline void gen_intermediate_code_internal(CPUPPCState *env,
         flags = env->bfd_mach;
         flags |= ctx.le_mode << 16;
         qemu_log("IN: %s\n", lookup_symbol(pc_start));
-        log_target_disas(pc_start, ctx.nip - pc_start, flags);
+        log_target_disas(env, pc_start, ctx.nip - pc_start, flags);
         qemu_log("\n");
     }
 #endif
diff --git a/target-ppc/translate_init.c b/target-ppc/translate_init.c
index fba2b4242..e63627cac 100644
--- a/target-ppc/translate_init.c
+++ b/target-ppc/translate_init.c
@@ -1498,7 +1498,7 @@ static void gen_spr_BookE (CPUPPCState *env, uint64_t ivor_mask)
     /* XXX : not implemented */
     spr_register(env, SPR_BOOKE_DBCR0, "DBCR0",
                  SPR_NOACCESS, SPR_NOACCESS,
-                 &spr_read_generic, &spr_write_generic,
+                 &spr_read_generic, &spr_write_40x_dbcr0,
                  0x00000000);
     /* XXX : not implemented */
     spr_register(env, SPR_BOOKE_DBCR1, "DBCR1",
@@ -10423,6 +10423,15 @@ static void ppc_cpu_reset(CPUState *s)
     env->pending_interrupts = 0;
     env->exception_index = POWERPC_EXCP_NONE;
     env->error_code = 0;
+
+#if defined(TARGET_PPC64) && !defined(CONFIG_USER_ONLY)
+    env->vpa_addr = 0;
+    env->slb_shadow_addr = 0;
+    env->slb_shadow_size = 0;
+    env->dtl_addr = 0;
+    env->dtl_size = 0;
+#endif /* TARGET_PPC64 */
+
     /* Flush all TLBs */
     tlb_flush(env, 1);
 }
diff --git a/target-s390x/Makefile.objs b/target-s390x/Makefile.objs
index 80be3bbdb..e728abf75 100644
--- a/target-s390x/Makefile.objs
+++ b/target-s390x/Makefile.objs
@@ -1,5 +1,4 @@
-obj-y += translate.o op_helper.o helper.o cpu.o interrupt.o
+obj-y += translate.o helper.o cpu.o interrupt.o
+obj-y += int_helper.o fpu_helper.o cc_helper.o mem_helper.o misc_helper.o
 obj-$(CONFIG_SOFTMMU) += machine.o
 obj-$(CONFIG_KVM) += kvm.o
-
-$(obj)/op_helper.o: QEMU_CFLAGS += $(HELPER_CFLAGS)
diff --git a/target-s390x/cc_helper.c b/target-s390x/cc_helper.c
new file mode 100644
index 000000000..19ef145da
--- /dev/null
+++ b/target-s390x/cc_helper.c
@@ -0,0 +1,550 @@
+/*
+ *  S/390 condition code helper routines
+ *
+ *  Copyright (c) 2009 Ulrich Hecht
+ *  Copyright (c) 2009 Alexander Graf
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "cpu.h"
+#include "helper.h"
+
+/* #define DEBUG_HELPER */
+#ifdef DEBUG_HELPER
+#define HELPER_LOG(x...) qemu_log(x)
+#else
+#define HELPER_LOG(x...)
+#endif
+
+static inline uint32_t cc_calc_ltgt_32(CPUS390XState *env, int32_t src,
+                                       int32_t dst)
+{
+    if (src == dst) {
+        return 0;
+    } else if (src < dst) {
+        return 1;
+    } else {
+        return 2;
+    }
+}
+
+static inline uint32_t cc_calc_ltgt0_32(CPUS390XState *env, int32_t dst)
+{
+    return cc_calc_ltgt_32(env, dst, 0);
+}
+
+static inline uint32_t cc_calc_ltgt_64(CPUS390XState *env, int64_t src,
+                                       int64_t dst)
+{
+    if (src == dst) {
+        return 0;
+    } else if (src < dst) {
+        return 1;
+    } else {
+        return 2;
+    }
+}
+
+static inline uint32_t cc_calc_ltgt0_64(CPUS390XState *env, int64_t dst)
+{
+    return cc_calc_ltgt_64(env, dst, 0);
+}
+
+static inline uint32_t cc_calc_ltugtu_32(CPUS390XState *env, uint32_t src,
+                                         uint32_t dst)
+{
+    if (src == dst) {
+        return 0;
+    } else if (src < dst) {
+        return 1;
+    } else {
+        return 2;
+    }
+}
+
+static inline uint32_t cc_calc_ltugtu_64(CPUS390XState *env, uint64_t src,
+                                         uint64_t dst)
+{
+    if (src == dst) {
+        return 0;
+    } else if (src < dst) {
+        return 1;
+    } else {
+        return 2;
+    }
+}
+
+static inline uint32_t cc_calc_tm_32(CPUS390XState *env, uint32_t val,
+                                     uint32_t mask)
+{
+    uint16_t r = val & mask;
+
+    HELPER_LOG("%s: val 0x%x mask 0x%x\n", __func__, val, mask);
+    if (r == 0 || mask == 0) {
+        return 0;
+    } else if (r == mask) {
+        return 3;
+    } else {
+        return 1;
+    }
+}
+
+/* set condition code for test under mask */
+static inline uint32_t cc_calc_tm_64(CPUS390XState *env, uint64_t val,
+                                     uint32_t mask)
+{
+    uint16_t r = val & mask;
+
+    HELPER_LOG("%s: val 0x%lx mask 0x%x r 0x%x\n", __func__, val, mask, r);
+    if (r == 0 || mask == 0) {
+        return 0;
+    } else if (r == mask) {
+        return 3;
+    } else {
+        while (!(mask & 0x8000)) {
+            mask <<= 1;
+            val <<= 1;
+        }
+        if (val & 0x8000) {
+            return 2;
+        } else {
+            return 1;
+        }
+    }
+}
+
+static inline uint32_t cc_calc_nz(CPUS390XState *env, uint64_t dst)
+{
+    return !!dst;
+}
+
+static inline uint32_t cc_calc_add_64(CPUS390XState *env, int64_t a1,
+                                      int64_t a2, int64_t ar)
+{
+    if ((a1 > 0 && a2 > 0 && ar < 0) || (a1 < 0 && a2 < 0 && ar > 0)) {
+        return 3; /* overflow */
+    } else {
+        if (ar < 0) {
+            return 1;
+        } else if (ar > 0) {
+            return 2;
+        } else {
+            return 0;
+        }
+    }
+}
+
+static inline uint32_t cc_calc_addu_64(CPUS390XState *env, uint64_t a1,
+                                       uint64_t a2, uint64_t ar)
+{
+    if (ar == 0) {
+        if (a1) {
+            return 2;
+        } else {
+            return 0;
+        }
+    } else {
+        if (ar < a1 || ar < a2) {
+            return 3;
+        } else {
+            return 1;
+        }
+    }
+}
+
+static inline uint32_t cc_calc_sub_64(CPUS390XState *env, int64_t a1,
+                                      int64_t a2, int64_t ar)
+{
+    if ((a1 > 0 && a2 < 0 && ar < 0) || (a1 < 0 && a2 > 0 && ar > 0)) {
+        return 3; /* overflow */
+    } else {
+        if (ar < 0) {
+            return 1;
+        } else if (ar > 0) {
+            return 2;
+        } else {
+            return 0;
+        }
+    }
+}
+
+static inline uint32_t cc_calc_subu_64(CPUS390XState *env, uint64_t a1,
+                                       uint64_t a2, uint64_t ar)
+{
+    if (ar == 0) {
+        return 2;
+    } else {
+        if (a2 > a1) {
+            return 1;
+        } else {
+            return 3;
+        }
+    }
+}
+
+static inline uint32_t cc_calc_abs_64(CPUS390XState *env, int64_t dst)
+{
+    if ((uint64_t)dst == 0x8000000000000000ULL) {
+        return 3;
+    } else if (dst) {
+        return 1;
+    } else {
+        return 0;
+    }
+}
+
+static inline uint32_t cc_calc_nabs_64(CPUS390XState *env, int64_t dst)
+{
+    return !!dst;
+}
+
+static inline uint32_t cc_calc_comp_64(CPUS390XState *env, int64_t dst)
+{
+    if ((uint64_t)dst == 0x8000000000000000ULL) {
+        return 3;
+    } else if (dst < 0) {
+        return 1;
+    } else if (dst > 0) {
+        return 2;
+    } else {
+        return 0;
+    }
+}
+
+
+static inline uint32_t cc_calc_add_32(CPUS390XState *env, int32_t a1,
+                                      int32_t a2, int32_t ar)
+{
+    if ((a1 > 0 && a2 > 0 && ar < 0) || (a1 < 0 && a2 < 0 && ar > 0)) {
+        return 3; /* overflow */
+    } else {
+        if (ar < 0) {
+            return 1;
+        } else if (ar > 0) {
+            return 2;
+        } else {
+            return 0;
+        }
+    }
+}
+
+static inline uint32_t cc_calc_addu_32(CPUS390XState *env, uint32_t a1,
+                                       uint32_t a2, uint32_t ar)
+{
+    if (ar == 0) {
+        if (a1) {
+            return 2;
+        } else {
+            return 0;
+        }
+    } else {
+        if (ar < a1 || ar < a2) {
+            return 3;
+        } else {
+            return 1;
+        }
+    }
+}
+
+static inline uint32_t cc_calc_sub_32(CPUS390XState *env, int32_t a1,
+                                      int32_t a2, int32_t ar)
+{
+    if ((a1 > 0 && a2 < 0 && ar < 0) || (a1 < 0 && a2 > 0 && ar > 0)) {
+        return 3; /* overflow */
+    } else {
+        if (ar < 0) {
+            return 1;
+        } else if (ar > 0) {
+            return 2;
+        } else {
+            return 0;
+        }
+    }
+}
+
+static inline uint32_t cc_calc_subu_32(CPUS390XState *env, uint32_t a1,
+                                       uint32_t a2, uint32_t ar)
+{
+    if (ar == 0) {
+        return 2;
+    } else {
+        if (a2 > a1) {
+            return 1;
+        } else {
+            return 3;
+        }
+    }
+}
+
+static inline uint32_t cc_calc_abs_32(CPUS390XState *env, int32_t dst)
+{
+    if ((uint32_t)dst == 0x80000000UL) {
+        return 3;
+    } else if (dst) {
+        return 1;
+    } else {
+        return 0;
+    }
+}
+
+static inline uint32_t cc_calc_nabs_32(CPUS390XState *env, int32_t dst)
+{
+    return !!dst;
+}
+
+static inline uint32_t cc_calc_comp_32(CPUS390XState *env, int32_t dst)
+{
+    if ((uint32_t)dst == 0x80000000UL) {
+        return 3;
+    } else if (dst < 0) {
+        return 1;
+    } else if (dst > 0) {
+        return 2;
+    } else {
+        return 0;
+    }
+}
+
+/* calculate condition code for insert character under mask insn */
+static inline uint32_t cc_calc_icm_32(CPUS390XState *env, uint32_t mask,
+                                      uint32_t val)
+{
+    uint32_t cc;
+
+    HELPER_LOG("%s: mask 0x%x val %d\n", __func__, mask, val);
+    if (mask == 0xf) {
+        if (!val) {
+            return 0;
+        } else if (val & 0x80000000) {
+            return 1;
+        } else {
+            return 2;
+        }
+    }
+
+    if (!val || !mask) {
+        cc = 0;
+    } else {
+        while (mask != 1) {
+            mask >>= 1;
+            val >>= 8;
+        }
+        if (val & 0x80) {
+            cc = 1;
+        } else {
+            cc = 2;
+        }
+    }
+    return cc;
+}
+
+static inline uint32_t cc_calc_slag(CPUS390XState *env, uint64_t src,
+                                    uint64_t shift)
+{
+    uint64_t mask = ((1ULL << shift) - 1ULL) << (64 - shift);
+    uint64_t match, r;
+
+    /* check if the sign bit stays the same */
+    if (src & (1ULL << 63)) {
+        match = mask;
+    } else {
+        match = 0;
+    }
+
+    if ((src & mask) != match) {
+        /* overflow */
+        return 3;
+    }
+
+    r = ((src << shift) & ((1ULL << 63) - 1)) | (src & (1ULL << 63));
+
+    if ((int64_t)r == 0) {
+        return 0;
+    } else if ((int64_t)r < 0) {
+        return 1;
+    }
+
+    return 2;
+}
+
+
+static inline uint32_t do_calc_cc(CPUS390XState *env, uint32_t cc_op,
+                                  uint64_t src, uint64_t dst, uint64_t vr)
+{
+    uint32_t r = 0;
+
+    switch (cc_op) {
+    case CC_OP_CONST0:
+    case CC_OP_CONST1:
+    case CC_OP_CONST2:
+    case CC_OP_CONST3:
+        /* cc_op value _is_ cc */
+        r = cc_op;
+        break;
+    case CC_OP_LTGT0_32:
+        r = cc_calc_ltgt0_32(env, dst);
+        break;
+    case CC_OP_LTGT0_64:
+        r =  cc_calc_ltgt0_64(env, dst);
+        break;
+    case CC_OP_LTGT_32:
+        r =  cc_calc_ltgt_32(env, src, dst);
+        break;
+    case CC_OP_LTGT_64:
+        r =  cc_calc_ltgt_64(env, src, dst);
+        break;
+    case CC_OP_LTUGTU_32:
+        r =  cc_calc_ltugtu_32(env, src, dst);
+        break;
+    case CC_OP_LTUGTU_64:
+        r =  cc_calc_ltugtu_64(env, src, dst);
+        break;
+    case CC_OP_TM_32:
+        r =  cc_calc_tm_32(env, src, dst);
+        break;
+    case CC_OP_TM_64:
+        r =  cc_calc_tm_64(env, src, dst);
+        break;
+    case CC_OP_NZ:
+        r =  cc_calc_nz(env, dst);
+        break;
+    case CC_OP_ADD_64:
+        r =  cc_calc_add_64(env, src, dst, vr);
+        break;
+    case CC_OP_ADDU_64:
+        r =  cc_calc_addu_64(env, src, dst, vr);
+        break;
+    case CC_OP_SUB_64:
+        r =  cc_calc_sub_64(env, src, dst, vr);
+        break;
+    case CC_OP_SUBU_64:
+        r =  cc_calc_subu_64(env, src, dst, vr);
+        break;
+    case CC_OP_ABS_64:
+        r =  cc_calc_abs_64(env, dst);
+        break;
+    case CC_OP_NABS_64:
+        r =  cc_calc_nabs_64(env, dst);
+        break;
+    case CC_OP_COMP_64:
+        r =  cc_calc_comp_64(env, dst);
+        break;
+
+    case CC_OP_ADD_32:
+        r =  cc_calc_add_32(env, src, dst, vr);
+        break;
+    case CC_OP_ADDU_32:
+        r =  cc_calc_addu_32(env, src, dst, vr);
+        break;
+    case CC_OP_SUB_32:
+        r =  cc_calc_sub_32(env, src, dst, vr);
+        break;
+    case CC_OP_SUBU_32:
+        r =  cc_calc_subu_32(env, src, dst, vr);
+        break;
+    case CC_OP_ABS_32:
+        r =  cc_calc_abs_64(env, dst);
+        break;
+    case CC_OP_NABS_32:
+        r =  cc_calc_nabs_64(env, dst);
+        break;
+    case CC_OP_COMP_32:
+        r =  cc_calc_comp_32(env, dst);
+        break;
+
+    case CC_OP_ICM:
+        r =  cc_calc_icm_32(env, src, dst);
+        break;
+    case CC_OP_SLAG:
+        r =  cc_calc_slag(env, src, dst);
+        break;
+
+    case CC_OP_LTGT_F32:
+        r = set_cc_f32(env, src, dst);
+        break;
+    case CC_OP_LTGT_F64:
+        r = set_cc_f64(env, src, dst);
+        break;
+    case CC_OP_NZ_F32:
+        r = set_cc_nz_f32(dst);
+        break;
+    case CC_OP_NZ_F64:
+        r = set_cc_nz_f64(dst);
+        break;
+
+    default:
+        cpu_abort(env, "Unknown CC operation: %s\n", cc_name(cc_op));
+    }
+
+    HELPER_LOG("%s: %15s 0x%016lx 0x%016lx 0x%016lx = %d\n", __func__,
+               cc_name(cc_op), src, dst, vr, r);
+    return r;
+}
+
+uint32_t calc_cc(CPUS390XState *env, uint32_t cc_op, uint64_t src, uint64_t dst,
+                 uint64_t vr)
+{
+    return do_calc_cc(env, cc_op, src, dst, vr);
+}
+
+uint32_t HELPER(calc_cc)(CPUS390XState *env, uint32_t cc_op, uint64_t src,
+                         uint64_t dst, uint64_t vr)
+{
+    return do_calc_cc(env, cc_op, src, dst, vr);
+}
+
+/* insert psw mask and condition code into r1 */
+void HELPER(ipm)(CPUS390XState *env, uint32_t cc, uint32_t r1)
+{
+    uint64_t r = env->regs[r1];
+
+    r &= 0xffffffff00ffffffULL;
+    r |= (cc << 28) | ((env->psw.mask >> 40) & 0xf);
+    env->regs[r1] = r;
+    HELPER_LOG("%s: cc %d psw.mask 0x%lx r1 0x%lx\n", __func__,
+               cc, env->psw.mask, r);
+}
+
+#ifndef CONFIG_USER_ONLY
+void HELPER(load_psw)(CPUS390XState *env, uint64_t mask, uint64_t addr)
+{
+    load_psw(env, mask, addr);
+    cpu_loop_exit(env);
+}
+
+void HELPER(sacf)(CPUS390XState *env, uint64_t a1)
+{
+    HELPER_LOG("%s: %16" PRIx64 "\n", __func__, a1);
+
+    switch (a1 & 0xf00) {
+    case 0x000:
+        env->psw.mask &= ~PSW_MASK_ASC;
+        env->psw.mask |= PSW_ASC_PRIMARY;
+        break;
+    case 0x100:
+        env->psw.mask &= ~PSW_MASK_ASC;
+        env->psw.mask |= PSW_ASC_SECONDARY;
+        break;
+    case 0x300:
+        env->psw.mask &= ~PSW_MASK_ASC;
+        env->psw.mask |= PSW_ASC_HOME;
+        break;
+    default:
+        qemu_log("unknown sacf mode: %" PRIx64 "\n", a1);
+        program_interrupt(env, PGM_SPECIFICATION, 2);
+        break;
+    }
+}
+#endif
diff --git a/target-s390x/cpu.h b/target-s390x/cpu.h
index 18ac6e393..0f9a1f734 100644
--- a/target-s390x/cpu.h
+++ b/target-s390x/cpu.h
@@ -324,7 +324,7 @@ unsigned s390_del_running_cpu(CPUS390XState *env);
 void s390_sclp_extint(uint32_t parm);
 
 /* from s390-virtio-bus */
-extern const target_phys_addr_t virtio_size;
+extern const hwaddr virtio_size;
 
 #else
 static inline void s390_add_running_cpu(CPUS390XState *env)
@@ -596,17 +596,6 @@ static inline const char *cc_name(int cc_op)
     return cc_names[cc_op];
 }
 
-/* SCLP PV interface defines */
-#define SCLP_CMDW_READ_SCP_INFO         0x00020001
-#define SCLP_CMDW_READ_SCP_INFO_FORCED  0x00120001
-
-#define SCP_LENGTH                      0x00
-#define SCP_FUNCTION_CODE               0x02
-#define SCP_CONTROL_MASK                0x03
-#define SCP_RESPONSE_CODE               0x06
-#define SCP_MEM_CODE                    0x08
-#define SCP_INCREMENT                   0x0a
-
 typedef struct LowCore
 {
     /* prefix area: defined by architecture */
@@ -955,7 +944,7 @@ static inline void ebcdic_put(uint8_t *p, const char *ascii, int len)
 void load_psw(CPUS390XState *env, uint64_t mask, uint64_t addr);
 int mmu_translate(CPUS390XState *env, target_ulong vaddr, int rw, uint64_t asc,
                   target_ulong *raddr, int *flags);
-int sclp_service_call(CPUS390XState *env, uint32_t sccb, uint64_t code);
+int sclp_service_call(uint32_t sccb, uint64_t code);
 uint32_t calc_cc(CPUS390XState *env, uint32_t cc_op, uint64_t src, uint64_t dst,
                  uint64_t vr);
 
@@ -988,8 +977,10 @@ static inline void cpu_inject_ext(CPUS390XState *env, uint32_t code, uint32_t pa
     cpu_interrupt(env, CPU_INTERRUPT_HARD);
 }
 
-static inline bool cpu_has_work(CPUS390XState *env)
+static inline bool cpu_has_work(CPUState *cpu)
 {
+    CPUS390XState *env = &S390_CPU(cpu)->env;
+
     return (env->interrupt_request & CPU_INTERRUPT_HARD) &&
         (env->psw.mask & PSW_MASK_EXT);
 }
@@ -999,4 +990,13 @@ static inline void cpu_pc_from_tb(CPUS390XState *env, TranslationBlock* tb)
     env->psw.addr = tb->pc;
 }
 
+/* fpu_helper.c */
+uint32_t set_cc_f32(CPUS390XState *env, float32 v1, float32 v2);
+uint32_t set_cc_f64(CPUS390XState *env, float64 v1, float64 v2);
+uint32_t set_cc_nz_f32(float32 v);
+uint32_t set_cc_nz_f64(float64 v);
+
+/* misc_helper.c */
+void program_interrupt(CPUS390XState *env, uint32_t code, int ilc);
+
 #endif
diff --git a/target-s390x/fpu_helper.c b/target-s390x/fpu_helper.c
new file mode 100644
index 000000000..ee9420d44
--- /dev/null
+++ b/target-s390x/fpu_helper.c
@@ -0,0 +1,843 @@
+/*
+ *  S/390 FPU helper routines
+ *
+ *  Copyright (c) 2009 Ulrich Hecht
+ *  Copyright (c) 2009 Alexander Graf
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "cpu.h"
+#include "helper.h"
+
+#if !defined(CONFIG_USER_ONLY)
+#include "softmmu_exec.h"
+#endif
+
+/* #define DEBUG_HELPER */
+#ifdef DEBUG_HELPER
+#define HELPER_LOG(x...) qemu_log(x)
+#else
+#define HELPER_LOG(x...)
+#endif
+
+static inline int float_comp_to_cc(CPUS390XState *env, int float_compare)
+{
+    switch (float_compare) {
+    case float_relation_equal:
+        return 0;
+    case float_relation_less:
+        return 1;
+    case float_relation_greater:
+        return 2;
+    case float_relation_unordered:
+        return 3;
+    default:
+        cpu_abort(env, "unknown return value for float compare\n");
+    }
+}
+
+/* condition codes for binary FP ops */
+uint32_t set_cc_f32(CPUS390XState *env, float32 v1, float32 v2)
+{
+    return float_comp_to_cc(env, float32_compare_quiet(v1, v2,
+                                                       &env->fpu_status));
+}
+
+uint32_t set_cc_f64(CPUS390XState *env, float64 v1, float64 v2)
+{
+    return float_comp_to_cc(env, float64_compare_quiet(v1, v2,
+                                                       &env->fpu_status));
+}
+
+/* condition codes for unary FP ops */
+uint32_t set_cc_nz_f32(float32 v)
+{
+    if (float32_is_any_nan(v)) {
+        return 3;
+    } else if (float32_is_zero(v)) {
+        return 0;
+    } else if (float32_is_neg(v)) {
+        return 1;
+    } else {
+        return 2;
+    }
+}
+
+uint32_t set_cc_nz_f64(float64 v)
+{
+    if (float64_is_any_nan(v)) {
+        return 3;
+    } else if (float64_is_zero(v)) {
+        return 0;
+    } else if (float64_is_neg(v)) {
+        return 1;
+    } else {
+        return 2;
+    }
+}
+
+static uint32_t set_cc_nz_f128(float128 v)
+{
+    if (float128_is_any_nan(v)) {
+        return 3;
+    } else if (float128_is_zero(v)) {
+        return 0;
+    } else if (float128_is_neg(v)) {
+        return 1;
+    } else {
+        return 2;
+    }
+}
+
+/* convert 32-bit int to 64-bit float */
+void HELPER(cdfbr)(CPUS390XState *env, uint32_t f1, int32_t v2)
+{
+    HELPER_LOG("%s: converting %d to f%d\n", __func__, v2, f1);
+    env->fregs[f1].d = int32_to_float64(v2, &env->fpu_status);
+}
+
+/* convert 32-bit int to 128-bit float */
+void HELPER(cxfbr)(CPUS390XState *env, uint32_t f1, int32_t v2)
+{
+    CPU_QuadU v1;
+
+    v1.q = int32_to_float128(v2, &env->fpu_status);
+    env->fregs[f1].ll = v1.ll.upper;
+    env->fregs[f1 + 2].ll = v1.ll.lower;
+}
+
+/* convert 64-bit int to 32-bit float */
+void HELPER(cegbr)(CPUS390XState *env, uint32_t f1, int64_t v2)
+{
+    HELPER_LOG("%s: converting %ld to f%d\n", __func__, v2, f1);
+    env->fregs[f1].l.upper = int64_to_float32(v2, &env->fpu_status);
+}
+
+/* convert 64-bit int to 64-bit float */
+void HELPER(cdgbr)(CPUS390XState *env, uint32_t f1, int64_t v2)
+{
+    HELPER_LOG("%s: converting %ld to f%d\n", __func__, v2, f1);
+    env->fregs[f1].d = int64_to_float64(v2, &env->fpu_status);
+}
+
+/* convert 64-bit int to 128-bit float */
+void HELPER(cxgbr)(CPUS390XState *env, uint32_t f1, int64_t v2)
+{
+    CPU_QuadU x1;
+
+    x1.q = int64_to_float128(v2, &env->fpu_status);
+    HELPER_LOG("%s: converted %ld to 0x%lx and 0x%lx\n", __func__, v2,
+               x1.ll.upper, x1.ll.lower);
+    env->fregs[f1].ll = x1.ll.upper;
+    env->fregs[f1 + 2].ll = x1.ll.lower;
+}
+
+/* convert 32-bit int to 32-bit float */
+void HELPER(cefbr)(CPUS390XState *env, uint32_t f1, int32_t v2)
+{
+    env->fregs[f1].l.upper = int32_to_float32(v2, &env->fpu_status);
+    HELPER_LOG("%s: converting %d to 0x%d in f%d\n", __func__, v2,
+               env->fregs[f1].l.upper, f1);
+}
+
+/* 32-bit FP addition RR */
+uint32_t HELPER(aebr)(CPUS390XState *env, uint32_t f1, uint32_t f2)
+{
+    env->fregs[f1].l.upper = float32_add(env->fregs[f1].l.upper,
+                                         env->fregs[f2].l.upper,
+                                         &env->fpu_status);
+    HELPER_LOG("%s: adding 0x%d resulting in 0x%d in f%d\n", __func__,
+               env->fregs[f2].l.upper, env->fregs[f1].l.upper, f1);
+
+    return set_cc_nz_f32(env->fregs[f1].l.upper);
+}
+
+/* 64-bit FP addition RR */
+uint32_t HELPER(adbr)(CPUS390XState *env, uint32_t f1, uint32_t f2)
+{
+    env->fregs[f1].d = float64_add(env->fregs[f1].d, env->fregs[f2].d,
+                                   &env->fpu_status);
+    HELPER_LOG("%s: adding 0x%ld resulting in 0x%ld in f%d\n", __func__,
+               env->fregs[f2].d, env->fregs[f1].d, f1);
+
+    return set_cc_nz_f64(env->fregs[f1].d);
+}
+
+/* 32-bit FP subtraction RR */
+uint32_t HELPER(sebr)(CPUS390XState *env, uint32_t f1, uint32_t f2)
+{
+    env->fregs[f1].l.upper = float32_sub(env->fregs[f1].l.upper,
+                                         env->fregs[f2].l.upper,
+                                         &env->fpu_status);
+    HELPER_LOG("%s: adding 0x%d resulting in 0x%d in f%d\n", __func__,
+               env->fregs[f2].l.upper, env->fregs[f1].l.upper, f1);
+
+    return set_cc_nz_f32(env->fregs[f1].l.upper);
+}
+
+/* 64-bit FP subtraction RR */
+uint32_t HELPER(sdbr)(CPUS390XState *env, uint32_t f1, uint32_t f2)
+{
+    env->fregs[f1].d = float64_sub(env->fregs[f1].d, env->fregs[f2].d,
+                                   &env->fpu_status);
+    HELPER_LOG("%s: subtracting 0x%ld resulting in 0x%ld in f%d\n",
+               __func__, env->fregs[f2].d, env->fregs[f1].d, f1);
+
+    return set_cc_nz_f64(env->fregs[f1].d);
+}
+
+/* 32-bit FP division RR */
+void HELPER(debr)(CPUS390XState *env, uint32_t f1, uint32_t f2)
+{
+    env->fregs[f1].l.upper = float32_div(env->fregs[f1].l.upper,
+                                         env->fregs[f2].l.upper,
+                                         &env->fpu_status);
+}
+
+/* 128-bit FP division RR */
+void HELPER(dxbr)(CPUS390XState *env, uint32_t f1, uint32_t f2)
+{
+    CPU_QuadU v1;
+    CPU_QuadU v2;
+    CPU_QuadU res;
+
+    v1.ll.upper = env->fregs[f1].ll;
+    v1.ll.lower = env->fregs[f1 + 2].ll;
+    v2.ll.upper = env->fregs[f2].ll;
+    v2.ll.lower = env->fregs[f2 + 2].ll;
+    res.q = float128_div(v1.q, v2.q, &env->fpu_status);
+    env->fregs[f1].ll = res.ll.upper;
+    env->fregs[f1 + 2].ll = res.ll.lower;
+}
+
+/* 64-bit FP multiplication RR */
+void HELPER(mdbr)(CPUS390XState *env, uint32_t f1, uint32_t f2)
+{
+    env->fregs[f1].d = float64_mul(env->fregs[f1].d, env->fregs[f2].d,
+                                   &env->fpu_status);
+}
+
+/* 128-bit FP multiplication RR */
+void HELPER(mxbr)(CPUS390XState *env, uint32_t f1, uint32_t f2)
+{
+    CPU_QuadU v1;
+    CPU_QuadU v2;
+    CPU_QuadU res;
+
+    v1.ll.upper = env->fregs[f1].ll;
+    v1.ll.lower = env->fregs[f1 + 2].ll;
+    v2.ll.upper = env->fregs[f2].ll;
+    v2.ll.lower = env->fregs[f2 + 2].ll;
+    res.q = float128_mul(v1.q, v2.q, &env->fpu_status);
+    env->fregs[f1].ll = res.ll.upper;
+    env->fregs[f1 + 2].ll = res.ll.lower;
+}
+
+/* convert 32-bit float to 64-bit float */
+void HELPER(ldebr)(CPUS390XState *env, uint32_t r1, uint32_t r2)
+{
+    env->fregs[r1].d = float32_to_float64(env->fregs[r2].l.upper,
+                                          &env->fpu_status);
+}
+
+/* convert 128-bit float to 64-bit float */
+void HELPER(ldxbr)(CPUS390XState *env, uint32_t f1, uint32_t f2)
+{
+    CPU_QuadU x2;
+
+    x2.ll.upper = env->fregs[f2].ll;
+    x2.ll.lower = env->fregs[f2 + 2].ll;
+    env->fregs[f1].d = float128_to_float64(x2.q, &env->fpu_status);
+    HELPER_LOG("%s: to 0x%ld\n", __func__, env->fregs[f1].d);
+}
+
+/* convert 64-bit float to 128-bit float */
+void HELPER(lxdbr)(CPUS390XState *env, uint32_t f1, uint32_t f2)
+{
+    CPU_QuadU res;
+
+    res.q = float64_to_float128(env->fregs[f2].d, &env->fpu_status);
+    env->fregs[f1].ll = res.ll.upper;
+    env->fregs[f1 + 2].ll = res.ll.lower;
+}
+
+/* convert 64-bit float to 32-bit float */
+void HELPER(ledbr)(CPUS390XState *env, uint32_t f1, uint32_t f2)
+{
+    float64 d2 = env->fregs[f2].d;
+
+    env->fregs[f1].l.upper = float64_to_float32(d2, &env->fpu_status);
+}
+
+/* convert 128-bit float to 32-bit float */
+void HELPER(lexbr)(CPUS390XState *env, uint32_t f1, uint32_t f2)
+{
+    CPU_QuadU x2;
+
+    x2.ll.upper = env->fregs[f2].ll;
+    x2.ll.lower = env->fregs[f2 + 2].ll;
+    env->fregs[f1].l.upper = float128_to_float32(x2.q, &env->fpu_status);
+    HELPER_LOG("%s: to 0x%d\n", __func__, env->fregs[f1].l.upper);
+}
+
+/* absolute value of 32-bit float */
+uint32_t HELPER(lpebr)(CPUS390XState *env, uint32_t f1, uint32_t f2)
+{
+    float32 v1;
+    float32 v2 = env->fregs[f2].d;
+
+    v1 = float32_abs(v2);
+    env->fregs[f1].d = v1;
+    return set_cc_nz_f32(v1);
+}
+
+/* absolute value of 64-bit float */
+uint32_t HELPER(lpdbr)(CPUS390XState *env, uint32_t f1, uint32_t f2)
+{
+    float64 v1;
+    float64 v2 = env->fregs[f2].d;
+
+    v1 = float64_abs(v2);
+    env->fregs[f1].d = v1;
+    return set_cc_nz_f64(v1);
+}
+
+/* absolute value of 128-bit float */
+uint32_t HELPER(lpxbr)(CPUS390XState *env, uint32_t f1, uint32_t f2)
+{
+    CPU_QuadU v1;
+    CPU_QuadU v2;
+
+    v2.ll.upper = env->fregs[f2].ll;
+    v2.ll.lower = env->fregs[f2 + 2].ll;
+    v1.q = float128_abs(v2.q);
+    env->fregs[f1].ll = v1.ll.upper;
+    env->fregs[f1 + 2].ll = v1.ll.lower;
+    return set_cc_nz_f128(v1.q);
+}
+
+/* load and test 64-bit float */
+uint32_t HELPER(ltdbr)(CPUS390XState *env, uint32_t f1, uint32_t f2)
+{
+    env->fregs[f1].d = env->fregs[f2].d;
+    return set_cc_nz_f64(env->fregs[f1].d);
+}
+
+/* load and test 32-bit float */
+uint32_t HELPER(ltebr)(CPUS390XState *env, uint32_t f1, uint32_t f2)
+{
+    env->fregs[f1].l.upper = env->fregs[f2].l.upper;
+    return set_cc_nz_f32(env->fregs[f1].l.upper);
+}
+
+/* load and test 128-bit float */
+uint32_t HELPER(ltxbr)(CPUS390XState *env, uint32_t f1, uint32_t f2)
+{
+    CPU_QuadU x;
+
+    x.ll.upper = env->fregs[f2].ll;
+    x.ll.lower = env->fregs[f2 + 2].ll;
+    env->fregs[f1].ll = x.ll.upper;
+    env->fregs[f1 + 2].ll = x.ll.lower;
+    return set_cc_nz_f128(x.q);
+}
+
+/* load complement of 32-bit float */
+uint32_t HELPER(lcebr)(CPUS390XState *env, uint32_t f1, uint32_t f2)
+{
+    env->fregs[f1].l.upper = float32_chs(env->fregs[f2].l.upper);
+
+    return set_cc_nz_f32(env->fregs[f1].l.upper);
+}
+
+/* load complement of 64-bit float */
+uint32_t HELPER(lcdbr)(CPUS390XState *env, uint32_t f1, uint32_t f2)
+{
+    env->fregs[f1].d = float64_chs(env->fregs[f2].d);
+
+    return set_cc_nz_f64(env->fregs[f1].d);
+}
+
+/* load complement of 128-bit float */
+uint32_t HELPER(lcxbr)(CPUS390XState *env, uint32_t f1, uint32_t f2)
+{
+    CPU_QuadU x1, x2;
+
+    x2.ll.upper = env->fregs[f2].ll;
+    x2.ll.lower = env->fregs[f2 + 2].ll;
+    x1.q = float128_chs(x2.q);
+    env->fregs[f1].ll = x1.ll.upper;
+    env->fregs[f1 + 2].ll = x1.ll.lower;
+    return set_cc_nz_f128(x1.q);
+}
+
+/* 32-bit FP addition RM */
+void HELPER(aeb)(CPUS390XState *env, uint32_t f1, uint32_t val)
+{
+    float32 v1 = env->fregs[f1].l.upper;
+    CPU_FloatU v2;
+
+    v2.l = val;
+    HELPER_LOG("%s: adding 0x%d from f%d and 0x%d\n", __func__,
+               v1, f1, v2.f);
+    env->fregs[f1].l.upper = float32_add(v1, v2.f, &env->fpu_status);
+}
+
+/* 32-bit FP division RM */
+void HELPER(deb)(CPUS390XState *env, uint32_t f1, uint32_t val)
+{
+    float32 v1 = env->fregs[f1].l.upper;
+    CPU_FloatU v2;
+
+    v2.l = val;
+    HELPER_LOG("%s: dividing 0x%d from f%d by 0x%d\n", __func__,
+               v1, f1, v2.f);
+    env->fregs[f1].l.upper = float32_div(v1, v2.f, &env->fpu_status);
+}
+
+/* 32-bit FP multiplication RM */
+void HELPER(meeb)(CPUS390XState *env, uint32_t f1, uint32_t val)
+{
+    float32 v1 = env->fregs[f1].l.upper;
+    CPU_FloatU v2;
+
+    v2.l = val;
+    HELPER_LOG("%s: multiplying 0x%d from f%d and 0x%d\n", __func__,
+               v1, f1, v2.f);
+    env->fregs[f1].l.upper = float32_mul(v1, v2.f, &env->fpu_status);
+}
+
+/* 32-bit FP compare RR */
+uint32_t HELPER(cebr)(CPUS390XState *env, uint32_t f1, uint32_t f2)
+{
+    float32 v1 = env->fregs[f1].l.upper;
+    float32 v2 = env->fregs[f2].l.upper;
+
+    HELPER_LOG("%s: comparing 0x%d from f%d and 0x%d\n", __func__,
+               v1, f1, v2);
+    return set_cc_f32(env, v1, v2);
+}
+
+/* 64-bit FP compare RR */
+uint32_t HELPER(cdbr)(CPUS390XState *env, uint32_t f1, uint32_t f2)
+{
+    float64 v1 = env->fregs[f1].d;
+    float64 v2 = env->fregs[f2].d;
+
+    HELPER_LOG("%s: comparing 0x%ld from f%d and 0x%ld\n", __func__,
+               v1, f1, v2);
+    return set_cc_f64(env, v1, v2);
+}
+
+/* 128-bit FP compare RR */
+uint32_t HELPER(cxbr)(CPUS390XState *env, uint32_t f1, uint32_t f2)
+{
+    CPU_QuadU v1;
+    CPU_QuadU v2;
+
+    v1.ll.upper = env->fregs[f1].ll;
+    v1.ll.lower = env->fregs[f1 + 2].ll;
+    v2.ll.upper = env->fregs[f2].ll;
+    v2.ll.lower = env->fregs[f2 + 2].ll;
+
+    return float_comp_to_cc(env, float128_compare_quiet(v1.q, v2.q,
+                                                   &env->fpu_status));
+}
+
+/* 64-bit FP compare RM */
+uint32_t HELPER(cdb)(CPUS390XState *env, uint32_t f1, uint64_t a2)
+{
+    float64 v1 = env->fregs[f1].d;
+    CPU_DoubleU v2;
+
+    v2.ll = cpu_ldq_data(env, a2);
+    HELPER_LOG("%s: comparing 0x%ld from f%d and 0x%lx\n", __func__, v1,
+               f1, v2.d);
+    return set_cc_f64(env, v1, v2.d);
+}
+
+/* 64-bit FP addition RM */
+uint32_t HELPER(adb)(CPUS390XState *env, uint32_t f1, uint64_t a2)
+{
+    float64 v1 = env->fregs[f1].d;
+    CPU_DoubleU v2;
+
+    v2.ll = cpu_ldq_data(env, a2);
+    HELPER_LOG("%s: adding 0x%lx from f%d and 0x%lx\n", __func__,
+               v1, f1, v2.d);
+    env->fregs[f1].d = v1 = float64_add(v1, v2.d, &env->fpu_status);
+    return set_cc_nz_f64(v1);
+}
+
+/* 32-bit FP subtraction RM */
+void HELPER(seb)(CPUS390XState *env, uint32_t f1, uint32_t val)
+{
+    float32 v1 = env->fregs[f1].l.upper;
+    CPU_FloatU v2;
+
+    v2.l = val;
+    env->fregs[f1].l.upper = float32_sub(v1, v2.f, &env->fpu_status);
+}
+
+/* 64-bit FP subtraction RM */
+uint32_t HELPER(sdb)(CPUS390XState *env, uint32_t f1, uint64_t a2)
+{
+    float64 v1 = env->fregs[f1].d;
+    CPU_DoubleU v2;
+
+    v2.ll = cpu_ldq_data(env, a2);
+    env->fregs[f1].d = v1 = float64_sub(v1, v2.d, &env->fpu_status);
+    return set_cc_nz_f64(v1);
+}
+
+/* 64-bit FP multiplication RM */
+void HELPER(mdb)(CPUS390XState *env, uint32_t f1, uint64_t a2)
+{
+    float64 v1 = env->fregs[f1].d;
+    CPU_DoubleU v2;
+
+    v2.ll = cpu_ldq_data(env, a2);
+    HELPER_LOG("%s: multiplying 0x%lx from f%d and 0x%ld\n", __func__,
+               v1, f1, v2.d);
+    env->fregs[f1].d = float64_mul(v1, v2.d, &env->fpu_status);
+}
+
+/* 64-bit FP division RM */
+void HELPER(ddb)(CPUS390XState *env, uint32_t f1, uint64_t a2)
+{
+    float64 v1 = env->fregs[f1].d;
+    CPU_DoubleU v2;
+
+    v2.ll = cpu_ldq_data(env, a2);
+    HELPER_LOG("%s: dividing 0x%lx from f%d by 0x%ld\n", __func__,
+               v1, f1, v2.d);
+    env->fregs[f1].d = float64_div(v1, v2.d, &env->fpu_status);
+}
+
+static void set_round_mode(CPUS390XState *env, int m3)
+{
+    switch (m3) {
+    case 0:
+        /* current mode */
+        break;
+    case 1:
+        /* biased round no nearest */
+    case 4:
+        /* round to nearest */
+        set_float_rounding_mode(float_round_nearest_even, &env->fpu_status);
+        break;
+    case 5:
+        /* round to zero */
+        set_float_rounding_mode(float_round_to_zero, &env->fpu_status);
+        break;
+    case 6:
+        /* round to +inf */
+        set_float_rounding_mode(float_round_up, &env->fpu_status);
+        break;
+    case 7:
+        /* round to -inf */
+        set_float_rounding_mode(float_round_down, &env->fpu_status);
+        break;
+    }
+}
+
+/* convert 32-bit float to 64-bit int */
+uint32_t HELPER(cgebr)(CPUS390XState *env, uint32_t r1, uint32_t f2,
+                       uint32_t m3)
+{
+    float32 v2 = env->fregs[f2].l.upper;
+
+    set_round_mode(env, m3);
+    env->regs[r1] = float32_to_int64(v2, &env->fpu_status);
+    return set_cc_nz_f32(v2);
+}
+
+/* convert 64-bit float to 64-bit int */
+uint32_t HELPER(cgdbr)(CPUS390XState *env, uint32_t r1, uint32_t f2,
+                       uint32_t m3)
+{
+    float64 v2 = env->fregs[f2].d;
+
+    set_round_mode(env, m3);
+    env->regs[r1] = float64_to_int64(v2, &env->fpu_status);
+    return set_cc_nz_f64(v2);
+}
+
+/* convert 128-bit float to 64-bit int */
+uint32_t HELPER(cgxbr)(CPUS390XState *env, uint32_t r1, uint32_t f2,
+                       uint32_t m3)
+{
+    CPU_QuadU v2;
+
+    v2.ll.upper = env->fregs[f2].ll;
+    v2.ll.lower = env->fregs[f2 + 2].ll;
+    set_round_mode(env, m3);
+    env->regs[r1] = float128_to_int64(v2.q, &env->fpu_status);
+    if (float128_is_any_nan(v2.q)) {
+        return 3;
+    } else if (float128_is_zero(v2.q)) {
+        return 0;
+    } else if (float128_is_neg(v2.q)) {
+        return 1;
+    } else {
+        return 2;
+    }
+}
+
+/* convert 32-bit float to 32-bit int */
+uint32_t HELPER(cfebr)(CPUS390XState *env, uint32_t r1, uint32_t f2,
+                       uint32_t m3)
+{
+    float32 v2 = env->fregs[f2].l.upper;
+
+    set_round_mode(env, m3);
+    env->regs[r1] = (env->regs[r1] & 0xffffffff00000000ULL) |
+        float32_to_int32(v2, &env->fpu_status);
+    return set_cc_nz_f32(v2);
+}
+
+/* convert 64-bit float to 32-bit int */
+uint32_t HELPER(cfdbr)(CPUS390XState *env, uint32_t r1, uint32_t f2,
+                       uint32_t m3)
+{
+    float64 v2 = env->fregs[f2].d;
+
+    set_round_mode(env, m3);
+    env->regs[r1] = (env->regs[r1] & 0xffffffff00000000ULL) |
+        float64_to_int32(v2, &env->fpu_status);
+    return set_cc_nz_f64(v2);
+}
+
+/* convert 128-bit float to 32-bit int */
+uint32_t HELPER(cfxbr)(CPUS390XState *env, uint32_t r1, uint32_t f2,
+                       uint32_t m3)
+{
+    CPU_QuadU v2;
+
+    v2.ll.upper = env->fregs[f2].ll;
+    v2.ll.lower = env->fregs[f2 + 2].ll;
+    env->regs[r1] = (env->regs[r1] & 0xffffffff00000000ULL) |
+        float128_to_int32(v2.q, &env->fpu_status);
+    return set_cc_nz_f128(v2.q);
+}
+
+/* load 32-bit FP zero */
+void HELPER(lzer)(CPUS390XState *env, uint32_t f1)
+{
+    env->fregs[f1].l.upper = float32_zero;
+}
+
+/* load 64-bit FP zero */
+void HELPER(lzdr)(CPUS390XState *env, uint32_t f1)
+{
+    env->fregs[f1].d = float64_zero;
+}
+
+/* load 128-bit FP zero */
+void HELPER(lzxr)(CPUS390XState *env, uint32_t f1)
+{
+    CPU_QuadU x;
+
+    x.q = float64_to_float128(float64_zero, &env->fpu_status);
+    env->fregs[f1].ll = x.ll.upper;
+    env->fregs[f1 + 1].ll = x.ll.lower;
+}
+
+/* 128-bit FP subtraction RR */
+uint32_t HELPER(sxbr)(CPUS390XState *env, uint32_t f1, uint32_t f2)
+{
+    CPU_QuadU v1;
+    CPU_QuadU v2;
+    CPU_QuadU res;
+
+    v1.ll.upper = env->fregs[f1].ll;
+    v1.ll.lower = env->fregs[f1 + 2].ll;
+    v2.ll.upper = env->fregs[f2].ll;
+    v2.ll.lower = env->fregs[f2 + 2].ll;
+    res.q = float128_sub(v1.q, v2.q, &env->fpu_status);
+    env->fregs[f1].ll = res.ll.upper;
+    env->fregs[f1 + 2].ll = res.ll.lower;
+    return set_cc_nz_f128(res.q);
+}
+
+/* 128-bit FP addition RR */
+uint32_t HELPER(axbr)(CPUS390XState *env, uint32_t f1, uint32_t f2)
+{
+    CPU_QuadU v1;
+    CPU_QuadU v2;
+    CPU_QuadU res;
+
+    v1.ll.upper = env->fregs[f1].ll;
+    v1.ll.lower = env->fregs[f1 + 2].ll;
+    v2.ll.upper = env->fregs[f2].ll;
+    v2.ll.lower = env->fregs[f2 + 2].ll;
+    res.q = float128_add(v1.q, v2.q, &env->fpu_status);
+    env->fregs[f1].ll = res.ll.upper;
+    env->fregs[f1 + 2].ll = res.ll.lower;
+    return set_cc_nz_f128(res.q);
+}
+
+/* 32-bit FP multiplication RR */
+void HELPER(meebr)(CPUS390XState *env, uint32_t f1, uint32_t f2)
+{
+    env->fregs[f1].l.upper = float32_mul(env->fregs[f1].l.upper,
+                                         env->fregs[f2].l.upper,
+                                         &env->fpu_status);
+}
+
+/* 64-bit FP division RR */
+void HELPER(ddbr)(CPUS390XState *env, uint32_t f1, uint32_t f2)
+{
+    env->fregs[f1].d = float64_div(env->fregs[f1].d, env->fregs[f2].d,
+                                   &env->fpu_status);
+}
+
+/* 64-bit FP multiply and add RM */
+void HELPER(madb)(CPUS390XState *env, uint32_t f1, uint64_t a2, uint32_t f3)
+{
+    CPU_DoubleU v2;
+
+    HELPER_LOG("%s: f1 %d a2 0x%lx f3 %d\n", __func__, f1, a2, f3);
+    v2.ll = cpu_ldq_data(env, a2);
+    env->fregs[f1].d = float64_add(env->fregs[f1].d,
+                                   float64_mul(v2.d, env->fregs[f3].d,
+                                               &env->fpu_status),
+                                   &env->fpu_status);
+}
+
+/* 64-bit FP multiply and add RR */
+void HELPER(madbr)(CPUS390XState *env, uint32_t f1, uint32_t f3, uint32_t f2)
+{
+    HELPER_LOG("%s: f1 %d f2 %d f3 %d\n", __func__, f1, f2, f3);
+    env->fregs[f1].d = float64_add(float64_mul(env->fregs[f2].d,
+                                               env->fregs[f3].d,
+                                               &env->fpu_status),
+                                   env->fregs[f1].d, &env->fpu_status);
+}
+
+/* 64-bit FP multiply and subtract RR */
+void HELPER(msdbr)(CPUS390XState *env, uint32_t f1, uint32_t f3, uint32_t f2)
+{
+    HELPER_LOG("%s: f1 %d f2 %d f3 %d\n", __func__, f1, f2, f3);
+    env->fregs[f1].d = float64_sub(float64_mul(env->fregs[f2].d,
+                                               env->fregs[f3].d,
+                                               &env->fpu_status),
+                                   env->fregs[f1].d, &env->fpu_status);
+}
+
+/* 32-bit FP multiply and add RR */
+void HELPER(maebr)(CPUS390XState *env, uint32_t f1, uint32_t f3, uint32_t f2)
+{
+    env->fregs[f1].l.upper = float32_add(env->fregs[f1].l.upper,
+                                         float32_mul(env->fregs[f2].l.upper,
+                                                     env->fregs[f3].l.upper,
+                                                     &env->fpu_status),
+                                         &env->fpu_status);
+}
+
+/* convert 32-bit float to 64-bit float */
+void HELPER(ldeb)(CPUS390XState *env, uint32_t f1, uint64_t a2)
+{
+    uint32_t v2;
+
+    v2 = cpu_ldl_data(env, a2);
+    env->fregs[f1].d = float32_to_float64(v2,
+                                          &env->fpu_status);
+}
+
+/* convert 64-bit float to 128-bit float */
+void HELPER(lxdb)(CPUS390XState *env, uint32_t f1, uint64_t a2)
+{
+    CPU_DoubleU v2;
+    CPU_QuadU v1;
+
+    v2.ll = cpu_ldq_data(env, a2);
+    v1.q = float64_to_float128(v2.d, &env->fpu_status);
+    env->fregs[f1].ll = v1.ll.upper;
+    env->fregs[f1 + 2].ll = v1.ll.lower;
+}
+
+/* test data class 32-bit */
+uint32_t HELPER(tceb)(CPUS390XState *env, uint32_t f1, uint64_t m2)
+{
+    float32 v1 = env->fregs[f1].l.upper;
+    int neg = float32_is_neg(v1);
+    uint32_t cc = 0;
+
+    HELPER_LOG("%s: v1 0x%lx m2 0x%lx neg %d\n", __func__, (long)v1, m2, neg);
+    if ((float32_is_zero(v1) && (m2 & (1 << (11-neg)))) ||
+        (float32_is_infinity(v1) && (m2 & (1 << (5-neg)))) ||
+        (float32_is_any_nan(v1) && (m2 & (1 << (3-neg)))) ||
+        (float32_is_signaling_nan(v1) && (m2 & (1 << (1-neg))))) {
+        cc = 1;
+    } else if (m2 & (1 << (9-neg))) {
+        /* assume normalized number */
+        cc = 1;
+    }
+
+    /* FIXME: denormalized? */
+    return cc;
+}
+
+/* test data class 64-bit */
+uint32_t HELPER(tcdb)(CPUS390XState *env, uint32_t f1, uint64_t m2)
+{
+    float64 v1 = env->fregs[f1].d;
+    int neg = float64_is_neg(v1);
+    uint32_t cc = 0;
+
+    HELPER_LOG("%s: v1 0x%lx m2 0x%lx neg %d\n", __func__, v1, m2, neg);
+    if ((float64_is_zero(v1) && (m2 & (1 << (11-neg)))) ||
+        (float64_is_infinity(v1) && (m2 & (1 << (5-neg)))) ||
+        (float64_is_any_nan(v1) && (m2 & (1 << (3-neg)))) ||
+        (float64_is_signaling_nan(v1) && (m2 & (1 << (1-neg))))) {
+        cc = 1;
+    } else if (m2 & (1 << (9-neg))) {
+        /* assume normalized number */
+        cc = 1;
+    }
+    /* FIXME: denormalized? */
+    return cc;
+}
+
+/* test data class 128-bit */
+uint32_t HELPER(tcxb)(CPUS390XState *env, uint32_t f1, uint64_t m2)
+{
+    CPU_QuadU v1;
+    uint32_t cc = 0;
+    int neg;
+
+    v1.ll.upper = env->fregs[f1].ll;
+    v1.ll.lower = env->fregs[f1 + 2].ll;
+
+    neg = float128_is_neg(v1.q);
+    if ((float128_is_zero(v1.q) && (m2 & (1 << (11-neg)))) ||
+        (float128_is_infinity(v1.q) && (m2 & (1 << (5-neg)))) ||
+        (float128_is_any_nan(v1.q) && (m2 & (1 << (3-neg)))) ||
+        (float128_is_signaling_nan(v1.q) && (m2 & (1 << (1-neg))))) {
+        cc = 1;
+    } else if (m2 & (1 << (9-neg))) {
+        /* assume normalized number */
+        cc = 1;
+    }
+    /* FIXME: denormalized? */
+    return cc;
+}
+
+/* square root 64-bit RR */
+void HELPER(sqdbr)(CPUS390XState *env, uint32_t f1, uint32_t f2)
+{
+    env->fregs[f1].d = float64_sqrt(env->fregs[f2].d, &env->fpu_status);
+}
diff --git a/target-s390x/helper.c b/target-s390x/helper.c
index d0a1180a8..b7b812a7e 100644
--- a/target-s390x/helper.c
+++ b/target-s390x/helper.c
@@ -74,7 +74,7 @@ S390CPU *cpu_s390x_init(const char *cpu_model)
 {
     S390CPU *cpu;
     CPUS390XState *env;
-    static int inited = 0;
+    static int inited;
 
     cpu = S390_CPU(object_new(TYPE_S390_CPU));
     env = &cpu->env;
@@ -91,25 +91,27 @@ S390CPU *cpu_s390x_init(const char *cpu_model)
 
 #if defined(CONFIG_USER_ONLY)
 
-void do_interrupt (CPUS390XState *env)
+void do_interrupt(CPUS390XState *env)
 {
     env->exception_index = -1;
 }
 
-int cpu_s390x_handle_mmu_fault (CPUS390XState *env, target_ulong address, int rw,
-                                int mmu_idx)
+int cpu_s390x_handle_mmu_fault(CPUS390XState *env, target_ulong address,
+                               int rw, int mmu_idx)
 {
-    /* fprintf(stderr,"%s: address 0x%lx rw %d mmu_idx %d\n",
-            __FUNCTION__, address, rw, mmu_idx); */
+    /* fprintf(stderr, "%s: address 0x%lx rw %d mmu_idx %d\n",
+       __func__, address, rw, mmu_idx); */
     env->exception_index = EXCP_ADDR;
-    env->__excp_addr = address; /* FIXME: find out how this works on a real machine */
+    /* FIXME: find out how this works on a real machine */
+    env->__excp_addr = address;
     return 1;
 }
 
 #else /* !CONFIG_USER_ONLY */
 
 /* Ensure to exit the TB after this call! */
-static void trigger_pgm_exception(CPUS390XState *env, uint32_t code, uint32_t ilc)
+static void trigger_pgm_exception(CPUS390XState *env, uint32_t code,
+                                  uint32_t ilc)
 {
     env->exception_index = EXCP_PGM;
     env->int_pgm_code = code;
@@ -138,19 +140,20 @@ static int trans_bits(CPUS390XState *env, uint64_t mode)
     return bits;
 }
 
-static void trigger_prot_fault(CPUS390XState *env, target_ulong vaddr, uint64_t mode)
+static void trigger_prot_fault(CPUS390XState *env, target_ulong vaddr,
+                               uint64_t mode)
 {
     int ilc = ILC_LATER_INC_2;
     int bits = trans_bits(env, mode) | 4;
 
-    DPRINTF("%s: vaddr=%016" PRIx64 " bits=%d\n", __FUNCTION__, vaddr, bits);
+    DPRINTF("%s: vaddr=%016" PRIx64 " bits=%d\n", __func__, vaddr, bits);
 
     stq_phys(env->psa + offsetof(LowCore, trans_exc_code), vaddr | bits);
     trigger_pgm_exception(env, PGM_PROTECTION, ilc);
 }
 
-static void trigger_page_fault(CPUS390XState *env, target_ulong vaddr, uint32_t type,
-                               uint64_t asc, int rw)
+static void trigger_page_fault(CPUS390XState *env, target_ulong vaddr,
+                               uint32_t type, uint64_t asc, int rw)
 {
     int ilc = ILC_LATER;
     int bits = trans_bits(env, asc);
@@ -160,26 +163,26 @@ static void trigger_page_fault(CPUS390XState *env, target_ulong vaddr, uint32_t
         ilc = 2;
     }
 
-    DPRINTF("%s: vaddr=%016" PRIx64 " bits=%d\n", __FUNCTION__, vaddr, bits);
+    DPRINTF("%s: vaddr=%016" PRIx64 " bits=%d\n", __func__, vaddr, bits);
 
     stq_phys(env->psa + offsetof(LowCore, trans_exc_code), vaddr | bits);
     trigger_pgm_exception(env, type, ilc);
 }
 
-static int mmu_translate_asce(CPUS390XState *env, target_ulong vaddr, uint64_t asc,
-                              uint64_t asce, int level, target_ulong *raddr,
-                              int *flags, int rw)
+static int mmu_translate_asce(CPUS390XState *env, target_ulong vaddr,
+                              uint64_t asc, uint64_t asce, int level,
+                              target_ulong *raddr, int *flags, int rw)
 {
     uint64_t offs = 0;
     uint64_t origin;
     uint64_t new_asce;
 
-    PTE_DPRINTF("%s: 0x%" PRIx64 "\n", __FUNCTION__, asce);
+    PTE_DPRINTF("%s: 0x%" PRIx64 "\n", __func__, asce);
 
     if (((level != _ASCE_TYPE_SEGMENT) && (asce & _REGION_ENTRY_INV)) ||
         ((level == _ASCE_TYPE_SEGMENT) && (asce & _SEGMENT_ENTRY_INV))) {
         /* XXX different regions have different faults */
-        DPRINTF("%s: invalid region\n", __FUNCTION__);
+        DPRINTF("%s: invalid region\n", __func__);
         trigger_page_fault(env, vaddr, PGM_SEGMENT_TRANS, asc, rw);
         return -1;
     }
@@ -222,7 +225,7 @@ static int mmu_translate_asce(CPUS390XState *env, target_ulong vaddr, uint64_t a
 
     new_asce = ldq_phys(origin + offs);
     PTE_DPRINTF("%s: 0x%" PRIx64 " + 0x%" PRIx64 " => 0x%016" PRIx64 "\n",
-                __FUNCTION__, origin, offs, new_asce);
+                __func__, origin, offs, new_asce);
 
     if (level != _ASCE_TYPE_SEGMENT) {
         /* yet another region */
@@ -232,7 +235,7 @@ static int mmu_translate_asce(CPUS390XState *env, target_ulong vaddr, uint64_t a
 
     /* PTE */
     if (new_asce & _PAGE_INVALID) {
-        DPRINTF("%s: PTE=0x%" PRIx64 " invalid\n", __FUNCTION__, new_asce);
+        DPRINTF("%s: PTE=0x%" PRIx64 " invalid\n", __func__, new_asce);
         trigger_page_fault(env, vaddr, PGM_PAGE_TRANS, asc, rw);
         return -1;
     }
@@ -243,13 +246,14 @@ static int mmu_translate_asce(CPUS390XState *env, target_ulong vaddr, uint64_t a
 
     *raddr = new_asce & _ASCE_ORIGIN;
 
-    PTE_DPRINTF("%s: PTE=0x%" PRIx64 "\n", __FUNCTION__, new_asce);
+    PTE_DPRINTF("%s: PTE=0x%" PRIx64 "\n", __func__, new_asce);
 
     return 0;
 }
 
-static int mmu_translate_asc(CPUS390XState *env, target_ulong vaddr, uint64_t asc,
-                             target_ulong *raddr, int *flags, int rw)
+static int mmu_translate_asc(CPUS390XState *env, target_ulong vaddr,
+                             uint64_t asc, target_ulong *raddr, int *flags,
+                             int rw)
 {
     uint64_t asce = 0;
     int level, new_level;
@@ -257,15 +261,15 @@ static int mmu_translate_asc(CPUS390XState *env, target_ulong vaddr, uint64_t as
 
     switch (asc) {
     case PSW_ASC_PRIMARY:
-        PTE_DPRINTF("%s: asc=primary\n", __FUNCTION__);
+        PTE_DPRINTF("%s: asc=primary\n", __func__);
         asce = env->cregs[1];
         break;
     case PSW_ASC_SECONDARY:
-        PTE_DPRINTF("%s: asc=secondary\n", __FUNCTION__);
+        PTE_DPRINTF("%s: asc=secondary\n", __func__);
         asce = env->cregs[7];
         break;
     case PSW_ASC_HOME:
-        PTE_DPRINTF("%s: asc=home\n", __FUNCTION__);
+        PTE_DPRINTF("%s: asc=home\n", __func__);
         asce = env->cregs[13];
         break;
     }
@@ -276,8 +280,7 @@ static int mmu_translate_asc(CPUS390XState *env, target_ulong vaddr, uint64_t as
     case _ASCE_TYPE_REGION2:
         if (vaddr & 0xffe0000000000000ULL) {
             DPRINTF("%s: vaddr doesn't fit 0x%16" PRIx64
-                        " 0xffe0000000000000ULL\n", __FUNCTION__,
-                        vaddr);
+                    " 0xffe0000000000000ULL\n", __func__, vaddr);
             trigger_page_fault(env, vaddr, PGM_TRANS_SPEC, asc, rw);
             return -1;
         }
@@ -285,8 +288,7 @@ static int mmu_translate_asc(CPUS390XState *env, target_ulong vaddr, uint64_t as
     case _ASCE_TYPE_REGION3:
         if (vaddr & 0xfffffc0000000000ULL) {
             DPRINTF("%s: vaddr doesn't fit 0x%16" PRIx64
-                        " 0xfffffc0000000000ULL\n", __FUNCTION__,
-                        vaddr);
+                    " 0xfffffc0000000000ULL\n", __func__, vaddr);
             trigger_page_fault(env, vaddr, PGM_TRANS_SPEC, asc, rw);
             return -1;
         }
@@ -294,8 +296,7 @@ static int mmu_translate_asc(CPUS390XState *env, target_ulong vaddr, uint64_t as
     case _ASCE_TYPE_SEGMENT:
         if (vaddr & 0xffffffff80000000ULL) {
             DPRINTF("%s: vaddr doesn't fit 0x%16" PRIx64
-                        " 0xffffffff80000000ULL\n", __FUNCTION__,
-                        vaddr);
+                    " 0xffffffff80000000ULL\n", __func__, vaddr);
             trigger_page_fault(env, vaddr, PGM_TRANS_SPEC, asc, rw);
             return -1;
         }
@@ -358,7 +359,7 @@ int mmu_translate(CPUS390XState *env, target_ulong vaddr, int rw, uint64_t asc,
         break;
     }
 
-out:
+ out:
     /* Convert real address -> absolute address */
     if (*raddr < 0x2000) {
         *raddr = *raddr + env->psa;
@@ -378,18 +379,18 @@ out:
     return r;
 }
 
-int cpu_s390x_handle_mmu_fault (CPUS390XState *env, target_ulong _vaddr, int rw,
-                                int mmu_idx)
+int cpu_s390x_handle_mmu_fault(CPUS390XState *env, target_ulong orig_vaddr,
+                               int rw, int mmu_idx)
 {
     uint64_t asc = env->psw.mask & PSW_MASK_ASC;
     target_ulong vaddr, raddr;
     int prot;
 
     DPRINTF("%s: address 0x%" PRIx64 " rw %d mmu_idx %d\n",
-            __FUNCTION__, _vaddr, rw, mmu_idx);
+            __func__, _vaddr, rw, mmu_idx);
 
-    _vaddr &= TARGET_PAGE_MASK;
-    vaddr = _vaddr;
+    orig_vaddr &= TARGET_PAGE_MASK;
+    vaddr = orig_vaddr;
 
     /* 31-Bit mode */
     if (!(env->psw.mask & PSW_MASK_64)) {
@@ -403,22 +404,23 @@ int cpu_s390x_handle_mmu_fault (CPUS390XState *env, target_ulong _vaddr, int rw,
 
     /* check out of RAM access */
     if (raddr > (ram_size + virtio_size)) {
-        DPRINTF("%s: aaddr %" PRIx64 " > ram_size %" PRIx64 "\n", __FUNCTION__,
+        DPRINTF("%s: aaddr %" PRIx64 " > ram_size %" PRIx64 "\n", __func__,
                 (uint64_t)aaddr, (uint64_t)ram_size);
         trigger_pgm_exception(env, PGM_ADDRESSING, ILC_LATER);
         return 1;
     }
 
-    DPRINTF("%s: set tlb %" PRIx64 " -> %" PRIx64 " (%x)\n", __FUNCTION__,
+    DPRINTF("%s: set tlb %" PRIx64 " -> %" PRIx64 " (%x)\n", __func__,
             (uint64_t)vaddr, (uint64_t)raddr, prot);
 
-    tlb_set_page(env, _vaddr, raddr, prot,
+    tlb_set_page(env, orig_vaddr, raddr, prot,
                  mmu_idx, TARGET_PAGE_SIZE);
 
     return 0;
 }
 
-target_phys_addr_t cpu_get_phys_page_debug(CPUS390XState *env, target_ulong vaddr)
+hwaddr cpu_get_phys_page_debug(CPUS390XState *env,
+                                           target_ulong vaddr)
 {
     target_ulong raddr;
     int prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC;
@@ -472,7 +474,7 @@ static void do_svc_interrupt(CPUS390XState *env)
 {
     uint64_t mask, addr;
     LowCore *lowcore;
-    target_phys_addr_t len = TARGET_PAGE_SIZE;
+    hwaddr len = TARGET_PAGE_SIZE;
 
     lowcore = cpu_physical_memory_map(env->psa, &len, 1);
 
@@ -492,24 +494,25 @@ static void do_program_interrupt(CPUS390XState *env)
 {
     uint64_t mask, addr;
     LowCore *lowcore;
-    target_phys_addr_t len = TARGET_PAGE_SIZE;
+    hwaddr len = TARGET_PAGE_SIZE;
     int ilc = env->int_pgm_ilc;
 
     switch (ilc) {
     case ILC_LATER:
-        ilc = get_ilc(ldub_code(env->psw.addr));
+        ilc = get_ilc(cpu_ldub_code(env, env->psw.addr));
         break;
     case ILC_LATER_INC:
-        ilc = get_ilc(ldub_code(env->psw.addr));
+        ilc = get_ilc(cpu_ldub_code(env, env->psw.addr));
         env->psw.addr += ilc * 2;
         break;
     case ILC_LATER_INC_2:
-        ilc = get_ilc(ldub_code(env->psw.addr)) * 2;
+        ilc = get_ilc(cpu_ldub_code(env, env->psw.addr)) * 2;
         env->psw.addr += ilc;
         break;
     }
 
-    qemu_log("%s: code=0x%x ilc=%d\n", __FUNCTION__, env->int_pgm_code, ilc);
+    qemu_log_mask(CPU_LOG_INT, "%s: code=0x%x ilc=%d\n",
+                  __func__, env->int_pgm_code, ilc);
 
     lowcore = cpu_physical_memory_map(env->psa, &len, 1);
 
@@ -522,7 +525,7 @@ static void do_program_interrupt(CPUS390XState *env)
 
     cpu_physical_memory_unmap(lowcore, len, 1, len);
 
-    DPRINTF("%s: %x %x %" PRIx64 " %" PRIx64 "\n", __FUNCTION__,
+    DPRINTF("%s: %x %x %" PRIx64 " %" PRIx64 "\n", __func__,
             env->int_pgm_code, ilc, env->psw.mask,
             env->psw.addr);
 
@@ -535,7 +538,7 @@ static void do_ext_interrupt(CPUS390XState *env)
 {
     uint64_t mask, addr;
     LowCore *lowcore;
-    target_phys_addr_t len = TARGET_PAGE_SIZE;
+    hwaddr len = TARGET_PAGE_SIZE;
     ExtQueue *q;
 
     if (!(env->psw.mask & PSW_MASK_EXT)) {
@@ -565,16 +568,16 @@ static void do_ext_interrupt(CPUS390XState *env)
         env->pending_int &= ~INTERRUPT_EXT;
     }
 
-    DPRINTF("%s: %" PRIx64 " %" PRIx64 "\n", __FUNCTION__,
+    DPRINTF("%s: %" PRIx64 " %" PRIx64 "\n", __func__,
             env->psw.mask, env->psw.addr);
 
     load_psw(env, mask, addr);
 }
 
-void do_interrupt (CPUS390XState *env)
+void do_interrupt(CPUS390XState *env)
 {
-    qemu_log("%s: %d at pc=%" PRIx64 "\n", __FUNCTION__, env->exception_index,
-             env->psw.addr);
+    qemu_log_mask(CPU_LOG_INT, "%s: %d at pc=%" PRIx64 "\n",
+                  __func__, env->exception_index, env->psw.addr);
 
     s390_add_running_cpu(env);
     /* handle external interrupts */
diff --git a/target-s390x/helper.h b/target-s390x/helper.h
index 01c8d0ea8..ac44eabd5 100644
--- a/target-s390x/helper.h
+++ b/target-s390x/helper.h
@@ -1,152 +1,152 @@
 #include "def-helper.h"
 
-DEF_HELPER_1(exception, void, i32)
-DEF_HELPER_3(nc, i32, i32, i64, i64)
-DEF_HELPER_3(oc, i32, i32, i64, i64)
-DEF_HELPER_3(xc, i32, i32, i64, i64)
-DEF_HELPER_3(mvc, void, i32, i64, i64)
-DEF_HELPER_3(clc, i32, i32, i64, i64)
-DEF_HELPER_2(mvcl, i32, i32, i32)
-DEF_HELPER_FLAGS_1(set_cc_comp_s32, TCG_CALL_PURE|TCG_CALL_CONST, i32, s32)
-DEF_HELPER_FLAGS_1(set_cc_comp_s64, TCG_CALL_PURE|TCG_CALL_CONST, i32, s64)
-DEF_HELPER_FLAGS_2(set_cc_icm, TCG_CALL_PURE|TCG_CALL_CONST, i32, i32, i32)
-DEF_HELPER_3(clm, i32, i32, i32, i64)
-DEF_HELPER_3(stcm, void, i32, i32, i64)
-DEF_HELPER_2(mlg, void, i32, i64)
-DEF_HELPER_2(dlg, void, i32, i64)
-DEF_HELPER_FLAGS_3(set_cc_add64, TCG_CALL_PURE|TCG_CALL_CONST, i32, s64, s64, s64)
-DEF_HELPER_FLAGS_3(set_cc_addu64, TCG_CALL_PURE|TCG_CALL_CONST, i32, i64, i64, i64)
-DEF_HELPER_FLAGS_3(set_cc_add32, TCG_CALL_PURE|TCG_CALL_CONST, i32, s32, s32, s32)
-DEF_HELPER_FLAGS_3(set_cc_addu32, TCG_CALL_PURE|TCG_CALL_CONST, i32, i32, i32, i32)
-DEF_HELPER_FLAGS_3(set_cc_sub64, TCG_CALL_PURE|TCG_CALL_CONST, i32, s64, s64, s64)
-DEF_HELPER_FLAGS_3(set_cc_subu64, TCG_CALL_PURE|TCG_CALL_CONST, i32, i64, i64, i64)
-DEF_HELPER_FLAGS_3(set_cc_sub32, TCG_CALL_PURE|TCG_CALL_CONST, i32, s32, s32, s32)
-DEF_HELPER_FLAGS_3(set_cc_subu32, TCG_CALL_PURE|TCG_CALL_CONST, i32, i32, i32, i32)
-DEF_HELPER_3(srst, i32, i32, i32, i32)
-DEF_HELPER_3(clst, i32, i32, i32, i32)
-DEF_HELPER_3(mvpg, void, i64, i64, i64)
-DEF_HELPER_3(mvst, void, i32, i32, i32)
-DEF_HELPER_3(csg, i32, i32, i64, i32)
-DEF_HELPER_3(cdsg, i32, i32, i64, i32)
-DEF_HELPER_3(cs, i32, i32, i64, i32)
-DEF_HELPER_4(ex, i32, i32, i64, i64, i64)
-DEF_HELPER_FLAGS_1(abs_i32, TCG_CALL_PURE|TCG_CALL_CONST, i32, s32)
-DEF_HELPER_FLAGS_1(nabs_i32, TCG_CALL_PURE|TCG_CALL_CONST, s32, s32)
-DEF_HELPER_FLAGS_1(abs_i64, TCG_CALL_PURE|TCG_CALL_CONST, i64, s64)
-DEF_HELPER_FLAGS_1(nabs_i64, TCG_CALL_PURE|TCG_CALL_CONST, s64, s64)
-DEF_HELPER_3(stcmh, void, i32, i64, i32)
-DEF_HELPER_3(icmh, i32, i32, i64, i32)
-DEF_HELPER_2(ipm, void, i32, i32)
-DEF_HELPER_FLAGS_3(addc_u32, TCG_CALL_PURE|TCG_CALL_CONST, i32, i32, i32, i32)
-DEF_HELPER_FLAGS_3(set_cc_addc_u64, TCG_CALL_PURE|TCG_CALL_CONST, i32, i64, i64, i64)
-DEF_HELPER_3(stam, void, i32, i64, i32)
-DEF_HELPER_3(lam, void, i32, i64, i32)
-DEF_HELPER_3(mvcle, i32, i32, i64, i32)
-DEF_HELPER_3(clcle, i32, i32, i64, i32)
-DEF_HELPER_3(slb, i32, i32, i32, i32)
-DEF_HELPER_4(slbg, i32, i32, i32, i64, i64)
-DEF_HELPER_2(cefbr, void, i32, s32)
-DEF_HELPER_2(cdfbr, void, i32, s32)
-DEF_HELPER_2(cxfbr, void, i32, s32)
-DEF_HELPER_2(cegbr, void, i32, s64)
-DEF_HELPER_2(cdgbr, void, i32, s64)
-DEF_HELPER_2(cxgbr, void, i32, s64)
-DEF_HELPER_2(adbr, i32, i32, i32)
-DEF_HELPER_2(aebr, i32, i32, i32)
-DEF_HELPER_2(sebr, i32, i32, i32)
-DEF_HELPER_2(sdbr, i32, i32, i32)
-DEF_HELPER_2(debr, void, i32, i32)
-DEF_HELPER_2(dxbr, void, i32, i32)
-DEF_HELPER_2(mdbr, void, i32, i32)
-DEF_HELPER_2(mxbr, void, i32, i32)
-DEF_HELPER_2(ldebr, void, i32, i32)
-DEF_HELPER_2(ldxbr, void, i32, i32)
-DEF_HELPER_2(lxdbr, void, i32, i32)
-DEF_HELPER_2(ledbr, void, i32, i32)
-DEF_HELPER_2(lexbr, void, i32, i32)
-DEF_HELPER_2(lpebr, i32, i32, i32)
-DEF_HELPER_2(lpdbr, i32, i32, i32)
-DEF_HELPER_2(lpxbr, i32, i32, i32)
-DEF_HELPER_2(ltebr, i32, i32, i32)
-DEF_HELPER_2(ltdbr, i32, i32, i32)
-DEF_HELPER_2(ltxbr, i32, i32, i32)
-DEF_HELPER_2(lcebr, i32, i32, i32)
-DEF_HELPER_2(lcdbr, i32, i32, i32)
-DEF_HELPER_2(lcxbr, i32, i32, i32)
-DEF_HELPER_2(aeb, void, i32, i32)
-DEF_HELPER_2(deb, void, i32, i32)
-DEF_HELPER_2(meeb, void, i32, i32)
-DEF_HELPER_2(cdb, i32, i32, i64)
-DEF_HELPER_2(adb, i32, i32, i64)
-DEF_HELPER_2(seb, void, i32, i32)
-DEF_HELPER_2(sdb, i32, i32, i64)
-DEF_HELPER_2(mdb, void, i32, i64)
-DEF_HELPER_2(ddb, void, i32, i64)
-DEF_HELPER_FLAGS_2(cebr, TCG_CALL_PURE, i32, i32, i32)
-DEF_HELPER_FLAGS_2(cdbr, TCG_CALL_PURE, i32, i32, i32)
-DEF_HELPER_FLAGS_2(cxbr, TCG_CALL_PURE, i32, i32, i32)
-DEF_HELPER_3(cgebr, i32, i32, i32, i32)
-DEF_HELPER_3(cgdbr, i32, i32, i32, i32)
-DEF_HELPER_3(cgxbr, i32, i32, i32, i32)
-DEF_HELPER_1(lzer, void, i32)
-DEF_HELPER_1(lzdr, void, i32)
-DEF_HELPER_1(lzxr, void, i32)
-DEF_HELPER_3(cfebr, i32, i32, i32, i32)
-DEF_HELPER_3(cfdbr, i32, i32, i32, i32)
-DEF_HELPER_3(cfxbr, i32, i32, i32, i32)
-DEF_HELPER_2(axbr, i32, i32, i32)
-DEF_HELPER_2(sxbr, i32, i32, i32)
-DEF_HELPER_2(meebr, void, i32, i32)
-DEF_HELPER_2(ddbr, void, i32, i32)
-DEF_HELPER_3(madb, void, i32, i64, i32)
-DEF_HELPER_3(maebr, void, i32, i32, i32)
-DEF_HELPER_3(madbr, void, i32, i32, i32)
-DEF_HELPER_3(msdbr, void, i32, i32, i32)
-DEF_HELPER_2(ldeb, void, i32, i64)
-DEF_HELPER_2(lxdb, void, i32, i64)
-DEF_HELPER_FLAGS_2(tceb, TCG_CALL_PURE, i32, i32, i64)
-DEF_HELPER_FLAGS_2(tcdb, TCG_CALL_PURE, i32, i32, i64)
-DEF_HELPER_FLAGS_2(tcxb, TCG_CALL_PURE, i32, i32, i64)
-DEF_HELPER_2(flogr, i32, i32, i64)
-DEF_HELPER_2(sqdbr, void, i32, i32)
-DEF_HELPER_FLAGS_1(cvd, TCG_CALL_PURE|TCG_CALL_CONST, i64, s32)
-DEF_HELPER_3(unpk, void, i32, i64, i64)
-DEF_HELPER_3(tr, void, i32, i64, i64)
+DEF_HELPER_2(exception, void, env, i32)
+DEF_HELPER_4(nc, i32, env, i32, i64, i64)
+DEF_HELPER_4(oc, i32, env, i32, i64, i64)
+DEF_HELPER_4(xc, i32, env, i32, i64, i64)
+DEF_HELPER_4(mvc, void, env, i32, i64, i64)
+DEF_HELPER_4(clc, i32, env, i32, i64, i64)
+DEF_HELPER_3(mvcl, i32, env, i32, i32)
+DEF_HELPER_FLAGS_1(set_cc_comp_s32, TCG_CALL_NO_RWG_SE, i32, s32)
+DEF_HELPER_FLAGS_1(set_cc_comp_s64, TCG_CALL_NO_RWG_SE, i32, s64)
+DEF_HELPER_FLAGS_2(set_cc_icm, TCG_CALL_NO_RWG_SE, i32, i32, i32)
+DEF_HELPER_4(clm, i32, env, i32, i32, i64)
+DEF_HELPER_4(stcm, void, env, i32, i32, i64)
+DEF_HELPER_3(mlg, void, env, i32, i64)
+DEF_HELPER_3(dlg, void, env, i32, i64)
+DEF_HELPER_FLAGS_3(set_cc_add64, TCG_CALL_NO_RWG_SE, i32, s64, s64, s64)
+DEF_HELPER_FLAGS_3(set_cc_addu64, TCG_CALL_NO_RWG_SE, i32, i64, i64, i64)
+DEF_HELPER_FLAGS_3(set_cc_add32, TCG_CALL_NO_RWG_SE, i32, s32, s32, s32)
+DEF_HELPER_FLAGS_3(set_cc_addu32, TCG_CALL_NO_RWG_SE, i32, i32, i32, i32)
+DEF_HELPER_FLAGS_3(set_cc_sub64, TCG_CALL_NO_RWG_SE, i32, s64, s64, s64)
+DEF_HELPER_FLAGS_3(set_cc_subu64, TCG_CALL_NO_RWG_SE, i32, i64, i64, i64)
+DEF_HELPER_FLAGS_3(set_cc_sub32, TCG_CALL_NO_RWG_SE, i32, s32, s32, s32)
+DEF_HELPER_FLAGS_3(set_cc_subu32, TCG_CALL_NO_RWG_SE, i32, i32, i32, i32)
+DEF_HELPER_4(srst, i32, env, i32, i32, i32)
+DEF_HELPER_4(clst, i32, env, i32, i32, i32)
+DEF_HELPER_4(mvpg, void, env, i64, i64, i64)
+DEF_HELPER_4(mvst, void, env, i32, i32, i32)
+DEF_HELPER_4(csg, i32, env, i32, i64, i32)
+DEF_HELPER_4(cdsg, i32, env, i32, i64, i32)
+DEF_HELPER_4(cs, i32, env, i32, i64, i32)
+DEF_HELPER_5(ex, i32, env, i32, i64, i64, i64)
+DEF_HELPER_FLAGS_1(abs_i32, TCG_CALL_NO_RWG_SE, i32, s32)
+DEF_HELPER_FLAGS_1(nabs_i32, TCG_CALL_NO_RWG_SE, s32, s32)
+DEF_HELPER_FLAGS_1(abs_i64, TCG_CALL_NO_RWG_SE, i64, s64)
+DEF_HELPER_FLAGS_1(nabs_i64, TCG_CALL_NO_RWG_SE, s64, s64)
+DEF_HELPER_4(stcmh, void, env, i32, i64, i32)
+DEF_HELPER_4(icmh, i32, env, i32, i64, i32)
+DEF_HELPER_3(ipm, void, env, i32, i32)
+DEF_HELPER_FLAGS_3(addc_u32, TCG_CALL_NO_RWG_SE, i32, i32, i32, i32)
+DEF_HELPER_FLAGS_3(set_cc_addc_u64, TCG_CALL_NO_RWG_SE, i32, i64, i64, i64)
+DEF_HELPER_4(stam, void, env, i32, i64, i32)
+DEF_HELPER_4(lam, void, env, i32, i64, i32)
+DEF_HELPER_4(mvcle, i32, env, i32, i64, i32)
+DEF_HELPER_4(clcle, i32, env, i32, i64, i32)
+DEF_HELPER_4(slb, i32, env, i32, i32, i32)
+DEF_HELPER_5(slbg, i32, env, i32, i32, i64, i64)
+DEF_HELPER_3(cefbr, void, env, i32, s32)
+DEF_HELPER_3(cdfbr, void, env, i32, s32)
+DEF_HELPER_3(cxfbr, void, env, i32, s32)
+DEF_HELPER_3(cegbr, void, env, i32, s64)
+DEF_HELPER_3(cdgbr, void, env, i32, s64)
+DEF_HELPER_3(cxgbr, void, env, i32, s64)
+DEF_HELPER_3(adbr, i32, env, i32, i32)
+DEF_HELPER_3(aebr, i32, env, i32, i32)
+DEF_HELPER_3(sebr, i32, env, i32, i32)
+DEF_HELPER_3(sdbr, i32, env, i32, i32)
+DEF_HELPER_3(debr, void, env, i32, i32)
+DEF_HELPER_3(dxbr, void, env, i32, i32)
+DEF_HELPER_3(mdbr, void, env, i32, i32)
+DEF_HELPER_3(mxbr, void, env, i32, i32)
+DEF_HELPER_3(ldebr, void, env, i32, i32)
+DEF_HELPER_3(ldxbr, void, env, i32, i32)
+DEF_HELPER_3(lxdbr, void, env, i32, i32)
+DEF_HELPER_3(ledbr, void, env, i32, i32)
+DEF_HELPER_3(lexbr, void, env, i32, i32)
+DEF_HELPER_3(lpebr, i32, env, i32, i32)
+DEF_HELPER_3(lpdbr, i32, env, i32, i32)
+DEF_HELPER_3(lpxbr, i32, env, i32, i32)
+DEF_HELPER_3(ltebr, i32, env, i32, i32)
+DEF_HELPER_3(ltdbr, i32, env, i32, i32)
+DEF_HELPER_3(ltxbr, i32, env, i32, i32)
+DEF_HELPER_3(lcebr, i32, env, i32, i32)
+DEF_HELPER_3(lcdbr, i32, env, i32, i32)
+DEF_HELPER_3(lcxbr, i32, env, i32, i32)
+DEF_HELPER_3(aeb, void, env, i32, i32)
+DEF_HELPER_3(deb, void, env, i32, i32)
+DEF_HELPER_3(meeb, void, env, i32, i32)
+DEF_HELPER_3(cdb, i32, env, i32, i64)
+DEF_HELPER_3(adb, i32, env, i32, i64)
+DEF_HELPER_3(seb, void, env, i32, i32)
+DEF_HELPER_3(sdb, i32, env, i32, i64)
+DEF_HELPER_3(mdb, void, env, i32, i64)
+DEF_HELPER_3(ddb, void, env, i32, i64)
+DEF_HELPER_FLAGS_3(cebr, TCG_CALL_NO_SE, i32, env, i32, i32)
+DEF_HELPER_FLAGS_3(cdbr, TCG_CALL_NO_SE, i32, env, i32, i32)
+DEF_HELPER_FLAGS_3(cxbr, TCG_CALL_NO_SE, i32, env, i32, i32)
+DEF_HELPER_4(cgebr, i32, env, i32, i32, i32)
+DEF_HELPER_4(cgdbr, i32, env, i32, i32, i32)
+DEF_HELPER_4(cgxbr, i32, env, i32, i32, i32)
+DEF_HELPER_2(lzer, void, env, i32)
+DEF_HELPER_2(lzdr, void, env, i32)
+DEF_HELPER_2(lzxr, void, env, i32)
+DEF_HELPER_4(cfebr, i32, env, i32, i32, i32)
+DEF_HELPER_4(cfdbr, i32, env, i32, i32, i32)
+DEF_HELPER_4(cfxbr, i32, env, i32, i32, i32)
+DEF_HELPER_3(axbr, i32, env, i32, i32)
+DEF_HELPER_3(sxbr, i32, env, i32, i32)
+DEF_HELPER_3(meebr, void, env, i32, i32)
+DEF_HELPER_3(ddbr, void, env, i32, i32)
+DEF_HELPER_4(madb, void, env, i32, i64, i32)
+DEF_HELPER_4(maebr, void, env, i32, i32, i32)
+DEF_HELPER_4(madbr, void, env, i32, i32, i32)
+DEF_HELPER_4(msdbr, void, env, i32, i32, i32)
+DEF_HELPER_3(ldeb, void, env, i32, i64)
+DEF_HELPER_3(lxdb, void, env, i32, i64)
+DEF_HELPER_FLAGS_3(tceb, TCG_CALL_NO_SE, i32, env, i32, i64)
+DEF_HELPER_FLAGS_3(tcdb, TCG_CALL_NO_SE, i32, env, i32, i64)
+DEF_HELPER_FLAGS_3(tcxb, TCG_CALL_NO_SE, i32, env, i32, i64)
+DEF_HELPER_3(flogr, i32, env, i32, i64)
+DEF_HELPER_3(sqdbr, void, env, i32, i32)
+DEF_HELPER_FLAGS_1(cvd, TCG_CALL_NO_RWG_SE, i64, s32)
+DEF_HELPER_4(unpk, void, env, i32, i64, i64)
+DEF_HELPER_4(tr, void, env, i32, i64, i64)
 
-DEF_HELPER_2(servc, i32, i32, i64)
-DEF_HELPER_3(diag, i64, i32, i64, i64)
-DEF_HELPER_2(load_psw, void, i64, i64)
+DEF_HELPER_3(servc, i32, env, i32, i64)
+DEF_HELPER_4(diag, i64, env, i32, i64, i64)
+DEF_HELPER_3(load_psw, void, env, i64, i64)
 DEF_HELPER_1(program_interrupt, void, i32)
-DEF_HELPER_FLAGS_1(stidp, TCG_CALL_CONST, void, i64)
-DEF_HELPER_FLAGS_1(spx, TCG_CALL_CONST, void, i64)
-DEF_HELPER_FLAGS_1(sck, TCG_CALL_CONST, i32, i64)
-DEF_HELPER_1(stck, i32, i64)
-DEF_HELPER_1(stcke, i32, i64)
-DEF_HELPER_FLAGS_1(sckc, TCG_CALL_CONST, void, i64)
-DEF_HELPER_FLAGS_1(stckc, TCG_CALL_CONST, void, i64)
-DEF_HELPER_FLAGS_1(spt, TCG_CALL_CONST, void, i64)
-DEF_HELPER_FLAGS_1(stpt, TCG_CALL_CONST, void, i64)
-DEF_HELPER_3(stsi, i32, i64, i32, i32)
-DEF_HELPER_3(lctl, void, i32, i64, i32)
-DEF_HELPER_3(lctlg, void, i32, i64, i32)
-DEF_HELPER_3(stctl, void, i32, i64, i32)
-DEF_HELPER_3(stctg, void, i32, i64, i32)
-DEF_HELPER_FLAGS_2(tprot, TCG_CALL_CONST, i32, i64, i64)
-DEF_HELPER_FLAGS_1(iske, TCG_CALL_PURE|TCG_CALL_CONST, i64, i64)
-DEF_HELPER_FLAGS_2(sske, TCG_CALL_CONST, void, i32, i64)
-DEF_HELPER_FLAGS_2(rrbe, TCG_CALL_CONST, i32, i32, i64)
-DEF_HELPER_2(csp, i32, i32, i32)
-DEF_HELPER_3(mvcs, i32, i64, i64, i64)
-DEF_HELPER_3(mvcp, i32, i64, i64, i64)
-DEF_HELPER_3(sigp, i32, i64, i32, i64)
-DEF_HELPER_1(sacf, void, i64)
-DEF_HELPER_FLAGS_2(ipte, TCG_CALL_CONST, void, i64, i64)
-DEF_HELPER_FLAGS_0(ptlb, TCG_CALL_CONST, void)
-DEF_HELPER_2(lra, i32, i64, i32)
-DEF_HELPER_2(stura, void, i64, i32)
-DEF_HELPER_2(cksm, void, i32, i32)
+DEF_HELPER_FLAGS_2(stidp, TCG_CALL_NO_RWG, void, env, i64)
+DEF_HELPER_FLAGS_2(spx, TCG_CALL_NO_RWG, void, env, i64)
+DEF_HELPER_FLAGS_1(sck, TCG_CALL_NO_RWG, i32, i64)
+DEF_HELPER_2(stck, i32, env, i64)
+DEF_HELPER_2(stcke, i32, env, i64)
+DEF_HELPER_FLAGS_2(sckc, TCG_CALL_NO_RWG, void, env, i64)
+DEF_HELPER_FLAGS_2(stckc, TCG_CALL_NO_RWG, void, env, i64)
+DEF_HELPER_FLAGS_2(spt, TCG_CALL_NO_RWG, void, env, i64)
+DEF_HELPER_FLAGS_2(stpt, TCG_CALL_NO_RWG, void, env, i64)
+DEF_HELPER_4(stsi, i32, env, i64, i32, i32)
+DEF_HELPER_4(lctl, void, env, i32, i64, i32)
+DEF_HELPER_4(lctlg, void, env, i32, i64, i32)
+DEF_HELPER_4(stctl, void, env, i32, i64, i32)
+DEF_HELPER_4(stctg, void, env, i32, i64, i32)
+DEF_HELPER_FLAGS_2(tprot, TCG_CALL_NO_RWG, i32, i64, i64)
+DEF_HELPER_FLAGS_2(iske, TCG_CALL_NO_RWG_SE, i64, env, i64)
+DEF_HELPER_FLAGS_3(sske, TCG_CALL_NO_RWG, void, env, i32, i64)
+DEF_HELPER_FLAGS_3(rrbe, TCG_CALL_NO_RWG, i32, env, i32, i64)
+DEF_HELPER_3(csp, i32, env, i32, i32)
+DEF_HELPER_4(mvcs, i32, env, i64, i64, i64)
+DEF_HELPER_4(mvcp, i32, env, i64, i64, i64)
+DEF_HELPER_4(sigp, i32, env, i64, i32, i64)
+DEF_HELPER_2(sacf, void, env, i64)
+DEF_HELPER_FLAGS_3(ipte, TCG_CALL_NO_RWG, void, env, i64, i64)
+DEF_HELPER_FLAGS_1(ptlb, TCG_CALL_NO_RWG, void, env)
+DEF_HELPER_3(lra, i32, env, i64, i32)
+DEF_HELPER_3(stura, void, env, i64, i32)
+DEF_HELPER_3(cksm, void, env, i32, i32)
 
-DEF_HELPER_FLAGS_4(calc_cc, TCG_CALL_PURE|TCG_CALL_CONST,
-                   i32, i32, i64, i64, i64)
+DEF_HELPER_FLAGS_5(calc_cc, TCG_CALL_NO_RWG_SE,
+                   i32, env, i32, i64, i64, i64)
 
 #include "def-helper.h"
diff --git a/target-s390x/int_helper.c b/target-s390x/int_helper.c
new file mode 100644
index 000000000..f202a7e1d
--- /dev/null
+++ b/target-s390x/int_helper.c
@@ -0,0 +1,201 @@
+/*
+ *  S/390 integer helper routines
+ *
+ *  Copyright (c) 2009 Ulrich Hecht
+ *  Copyright (c) 2009 Alexander Graf
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "cpu.h"
+#include "host-utils.h"
+#include "helper.h"
+
+/* #define DEBUG_HELPER */
+#ifdef DEBUG_HELPER
+#define HELPER_LOG(x...) qemu_log(x)
+#else
+#define HELPER_LOG(x...)
+#endif
+
+/* 64/64 -> 128 unsigned multiplication */
+void HELPER(mlg)(CPUS390XState *env, uint32_t r1, uint64_t v2)
+{
+#if HOST_LONG_BITS == 64 && defined(__GNUC__)
+    /* assuming 64-bit hosts have __uint128_t */
+    __uint128_t res = (__uint128_t)env->regs[r1 + 1];
+
+    res *= (__uint128_t)v2;
+    env->regs[r1] = (uint64_t)(res >> 64);
+    env->regs[r1 + 1] = (uint64_t)res;
+#else
+    mulu64(&env->regs[r1 + 1], &env->regs[r1], env->regs[r1 + 1], v2);
+#endif
+}
+
+/* 128 -> 64/64 unsigned division */
+void HELPER(dlg)(CPUS390XState *env, uint32_t r1, uint64_t v2)
+{
+    uint64_t divisor = v2;
+
+    if (!env->regs[r1]) {
+        /* 64 -> 64/64 case */
+        env->regs[r1] = env->regs[r1 + 1] % divisor;
+        env->regs[r1 + 1] = env->regs[r1 + 1] / divisor;
+        return;
+    } else {
+#if HOST_LONG_BITS == 64 && defined(__GNUC__)
+        /* assuming 64-bit hosts have __uint128_t */
+        __uint128_t dividend = (((__uint128_t)env->regs[r1]) << 64) |
+            (env->regs[r1 + 1]);
+        __uint128_t quotient = dividend / divisor;
+        __uint128_t remainder = dividend % divisor;
+
+        env->regs[r1 + 1] = quotient;
+        env->regs[r1] = remainder;
+#else
+        /* 32-bit hosts would need special wrapper functionality - just abort if
+           we encounter such a case; it's very unlikely anyways. */
+        cpu_abort(env, "128 -> 64/64 division not implemented\n");
+#endif
+    }
+}
+
+/* absolute value 32-bit */
+uint32_t HELPER(abs_i32)(int32_t val)
+{
+    if (val < 0) {
+        return -val;
+    } else {
+        return val;
+    }
+}
+
+/* negative absolute value 32-bit */
+int32_t HELPER(nabs_i32)(int32_t val)
+{
+    if (val < 0) {
+        return val;
+    } else {
+        return -val;
+    }
+}
+
+/* absolute value 64-bit */
+uint64_t HELPER(abs_i64)(int64_t val)
+{
+    HELPER_LOG("%s: val 0x%" PRIx64 "\n", __func__, val);
+
+    if (val < 0) {
+        return -val;
+    } else {
+        return val;
+    }
+}
+
+/* negative absolute value 64-bit */
+int64_t HELPER(nabs_i64)(int64_t val)
+{
+    if (val < 0) {
+        return val;
+    } else {
+        return -val;
+    }
+}
+
+/* add with carry 32-bit unsigned */
+uint32_t HELPER(addc_u32)(uint32_t cc, uint32_t v1, uint32_t v2)
+{
+    uint32_t res;
+
+    res = v1 + v2;
+    if (cc & 2) {
+        res++;
+    }
+
+    return res;
+}
+
+/* subtract unsigned v2 from v1 with borrow */
+uint32_t HELPER(slb)(CPUS390XState *env, uint32_t cc, uint32_t r1, uint32_t v2)
+{
+    uint32_t v1 = env->regs[r1];
+    uint32_t res = v1 + (~v2) + (cc >> 1);
+
+    env->regs[r1] = (env->regs[r1] & 0xffffffff00000000ULL) | res;
+    if (cc & 2) {
+        /* borrow */
+        return v1 ? 1 : 0;
+    } else {
+        return v1 ? 3 : 2;
+    }
+}
+
+/* subtract unsigned v2 from v1 with borrow */
+uint32_t HELPER(slbg)(CPUS390XState *env, uint32_t cc, uint32_t r1,
+                      uint64_t v1, uint64_t v2)
+{
+    uint64_t res = v1 + (~v2) + (cc >> 1);
+
+    env->regs[r1] = res;
+    if (cc & 2) {
+        /* borrow */
+        return v1 ? 1 : 0;
+    } else {
+        return v1 ? 3 : 2;
+    }
+}
+
+/* find leftmost one */
+uint32_t HELPER(flogr)(CPUS390XState *env, uint32_t r1, uint64_t v2)
+{
+    uint64_t res = 0;
+    uint64_t ov2 = v2;
+
+    while (!(v2 & 0x8000000000000000ULL) && v2) {
+        v2 <<= 1;
+        res++;
+    }
+
+    if (!v2) {
+        env->regs[r1] = 64;
+        env->regs[r1 + 1] = 0;
+        return 0;
+    } else {
+        env->regs[r1] = res;
+        env->regs[r1 + 1] = ov2 & ~(0x8000000000000000ULL >> res);
+        return 2;
+    }
+}
+
+uint64_t HELPER(cvd)(int32_t bin)
+{
+    /* positive 0 */
+    uint64_t dec = 0x0c;
+    int shift = 4;
+
+    if (bin < 0) {
+        bin = -bin;
+        dec = 0x0d;
+    }
+
+    for (shift = 4; (shift < 64) && bin; shift += 4) {
+        int current_number = bin % 10;
+
+        dec |= (current_number) << shift;
+        bin /= 10;
+    }
+
+    return dec;
+}
diff --git a/target-s390x/kvm.c b/target-s390x/kvm.c
index 07edf9369..94de76426 100644
--- a/target-s390x/kvm.c
+++ b/target-s390x/kvm.c
@@ -60,15 +60,15 @@
 #define SIGP_STORE_STATUS_ADDR          0x0e
 #define SIGP_SET_ARCH                   0x12
 
-#define SCLP_CMDW_READ_SCP_INFO         0x00020001
-#define SCLP_CMDW_READ_SCP_INFO_FORCED  0x00120001
-
 const KVMCapabilityInfo kvm_arch_required_capabilities[] = {
     KVM_CAP_LAST_INFO
 };
 
+static int cap_sync_regs;
+
 int kvm_arch_init(KVMState *s)
 {
+    cap_sync_regs = kvm_check_extension(s, KVM_CAP_SYNC_REGS);
     return 0;
 }
 
@@ -90,47 +90,116 @@ void kvm_arch_reset_vcpu(CPUS390XState *env)
 
 int kvm_arch_put_registers(CPUS390XState *env, int level)
 {
+    struct kvm_sregs sregs;
     struct kvm_regs regs;
     int ret;
     int i;
 
-    ret = kvm_vcpu_ioctl(env, KVM_GET_REGS, &regs);
-    if (ret < 0) {
-        return ret;
-    }
+    /* always save the PSW  and the GPRS*/
+    env->kvm_run->psw_addr = env->psw.addr;
+    env->kvm_run->psw_mask = env->psw.mask;
 
-    for (i = 0; i < 16; i++) {
-        regs.gprs[i] = env->regs[i];
+    if (cap_sync_regs && env->kvm_run->kvm_valid_regs & KVM_SYNC_GPRS) {
+        for (i = 0; i < 16; i++) {
+            env->kvm_run->s.regs.gprs[i] = env->regs[i];
+            env->kvm_run->kvm_dirty_regs |= KVM_SYNC_GPRS;
+        }
+    } else {
+        for (i = 0; i < 16; i++) {
+            regs.gprs[i] = env->regs[i];
+        }
+        ret = kvm_vcpu_ioctl(env, KVM_SET_REGS, &regs);
+        if (ret < 0) {
+            return ret;
+        }
     }
 
-    ret = kvm_vcpu_ioctl(env, KVM_SET_REGS, &regs);
-    if (ret < 0) {
-        return ret;
+    /* Do we need to save more than that? */
+    if (level == KVM_PUT_RUNTIME_STATE) {
+        return 0;
     }
 
-    env->kvm_run->psw_addr = env->psw.addr;
-    env->kvm_run->psw_mask = env->psw.mask;
+    if (cap_sync_regs &&
+        env->kvm_run->kvm_valid_regs & KVM_SYNC_ACRS &&
+        env->kvm_run->kvm_valid_regs & KVM_SYNC_CRS) {
+        for (i = 0; i < 16; i++) {
+            env->kvm_run->s.regs.acrs[i] = env->aregs[i];
+            env->kvm_run->s.regs.crs[i] = env->cregs[i];
+        }
+        env->kvm_run->kvm_dirty_regs |= KVM_SYNC_ACRS;
+        env->kvm_run->kvm_dirty_regs |= KVM_SYNC_CRS;
+    } else {
+        for (i = 0; i < 16; i++) {
+            sregs.acrs[i] = env->aregs[i];
+            sregs.crs[i] = env->cregs[i];
+        }
+        ret = kvm_vcpu_ioctl(env, KVM_SET_SREGS, &sregs);
+        if (ret < 0) {
+            return ret;
+        }
+    }
 
-    return ret;
+    /* Finally the prefix */
+    if (cap_sync_regs && env->kvm_run->kvm_valid_regs & KVM_SYNC_PREFIX) {
+        env->kvm_run->s.regs.prefix = env->psa;
+        env->kvm_run->kvm_dirty_regs |= KVM_SYNC_PREFIX;
+    } else {
+        /* prefix is only supported via sync regs */
+    }
+    return 0;
 }
 
 int kvm_arch_get_registers(CPUS390XState *env)
 {
-    int ret;
+    struct kvm_sregs sregs;
     struct kvm_regs regs;
+    int ret;
     int i;
 
-    ret = kvm_vcpu_ioctl(env, KVM_GET_REGS, &regs);
-    if (ret < 0) {
-        return ret;
+    /* get the PSW */
+    env->psw.addr = env->kvm_run->psw_addr;
+    env->psw.mask = env->kvm_run->psw_mask;
+
+    /* the GPRS */
+    if (cap_sync_regs && env->kvm_run->kvm_valid_regs & KVM_SYNC_GPRS) {
+        for (i = 0; i < 16; i++) {
+            env->regs[i] = env->kvm_run->s.regs.gprs[i];
+        }
+    } else {
+        ret = kvm_vcpu_ioctl(env, KVM_GET_REGS, &regs);
+        if (ret < 0) {
+            return ret;
+        }
+         for (i = 0; i < 16; i++) {
+            env->regs[i] = regs.gprs[i];
+        }
     }
 
-    for (i = 0; i < 16; i++) {
-        env->regs[i] = regs.gprs[i];
+    /* The ACRS and CRS */
+    if (cap_sync_regs &&
+        env->kvm_run->kvm_valid_regs & KVM_SYNC_ACRS &&
+        env->kvm_run->kvm_valid_regs & KVM_SYNC_CRS) {
+        for (i = 0; i < 16; i++) {
+            env->aregs[i] = env->kvm_run->s.regs.acrs[i];
+            env->cregs[i] = env->kvm_run->s.regs.crs[i];
+        }
+    } else {
+        ret = kvm_vcpu_ioctl(env, KVM_GET_SREGS, &sregs);
+        if (ret < 0) {
+            return ret;
+        }
+         for (i = 0; i < 16; i++) {
+            env->aregs[i] = sregs.acrs[i];
+            env->cregs[i] = sregs.crs[i];
+        }
     }
 
-    env->psw.addr = env->kvm_run->psw_addr;
-    env->psw.mask = env->kvm_run->psw_mask;
+    /* Finally the prefix */
+    if (cap_sync_regs && env->kvm_run->kvm_valid_regs & KVM_SYNC_PREFIX) {
+        env->psa = env->kvm_run->s.regs.prefix;
+    } else {
+        /* no prefix without sync regs */
+    }
 
     return 0;
 }
@@ -272,7 +341,7 @@ static int kvm_sclp_service_call(CPUS390XState *env, struct kvm_run *run,
     sccb = env->regs[ipbh0 & 0xf];
     code = env->regs[(ipbh0 & 0xf0) >> 4];
 
-    r = sclp_service_call(env, sccb, code);
+    r = sclp_service_call(sccb, code);
     if (r < 0) {
         enter_pgmcheck(env, -r);
     }
@@ -334,7 +403,7 @@ static int s390_cpu_restart(S390CPU *cpu)
 
     kvm_s390_interrupt(env, KVM_S390_RESTART, 0);
     s390_add_running_cpu(env);
-    qemu_cpu_kick(env);
+    qemu_cpu_kick(CPU(cpu));
     dprintf("DONE: SIGP cpu restart: %p\n", env);
     return 0;
 }
diff --git a/target-s390x/mem_helper.c b/target-s390x/mem_helper.c
new file mode 100644
index 000000000..6ebc22dd1
--- /dev/null
+++ b/target-s390x/mem_helper.c
@@ -0,0 +1,1203 @@
+/*
+ *  S/390 memory access helper routines
+ *
+ *  Copyright (c) 2009 Ulrich Hecht
+ *  Copyright (c) 2009 Alexander Graf
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "cpu.h"
+#include "helper.h"
+
+/*****************************************************************************/
+/* Softmmu support */
+#if !defined(CONFIG_USER_ONLY)
+#include "softmmu_exec.h"
+
+#define MMUSUFFIX _mmu
+
+#define SHIFT 0
+#include "softmmu_template.h"
+
+#define SHIFT 1
+#include "softmmu_template.h"
+
+#define SHIFT 2
+#include "softmmu_template.h"
+
+#define SHIFT 3
+#include "softmmu_template.h"
+
+/* try to fill the TLB and return an exception if error. If retaddr is
+   NULL, it means that the function was called in C code (i.e. not
+   from generated code or from helper.c) */
+/* XXX: fix it to restore all registers */
+void tlb_fill(CPUS390XState *env, target_ulong addr, int is_write, int mmu_idx,
+              uintptr_t retaddr)
+{
+    TranslationBlock *tb;
+    int ret;
+
+    ret = cpu_s390x_handle_mmu_fault(env, addr, is_write, mmu_idx);
+    if (unlikely(ret != 0)) {
+        if (likely(retaddr)) {
+            /* now we have a real cpu fault */
+            tb = tb_find_pc(retaddr);
+            if (likely(tb)) {
+                /* the PC is inside the translated code. It means that we have
+                   a virtual CPU fault */
+                cpu_restore_state(tb, env, retaddr);
+            }
+        }
+        cpu_loop_exit(env);
+    }
+}
+
+#endif
+
+/* #define DEBUG_HELPER */
+#ifdef DEBUG_HELPER
+#define HELPER_LOG(x...) qemu_log(x)
+#else
+#define HELPER_LOG(x...)
+#endif
+
+#ifndef CONFIG_USER_ONLY
+static void mvc_fast_memset(CPUS390XState *env, uint32_t l, uint64_t dest,
+                            uint8_t byte)
+{
+    hwaddr dest_phys;
+    hwaddr len = l;
+    void *dest_p;
+    uint64_t asc = env->psw.mask & PSW_MASK_ASC;
+    int flags;
+
+    if (mmu_translate(env, dest, 1, asc, &dest_phys, &flags)) {
+        cpu_stb_data(env, dest, byte);
+        cpu_abort(env, "should never reach here");
+    }
+    dest_phys |= dest & ~TARGET_PAGE_MASK;
+
+    dest_p = cpu_physical_memory_map(dest_phys, &len, 1);
+
+    memset(dest_p, byte, len);
+
+    cpu_physical_memory_unmap(dest_p, 1, len, len);
+}
+
+static void mvc_fast_memmove(CPUS390XState *env, uint32_t l, uint64_t dest,
+                             uint64_t src)
+{
+    hwaddr dest_phys;
+    hwaddr src_phys;
+    hwaddr len = l;
+    void *dest_p;
+    void *src_p;
+    uint64_t asc = env->psw.mask & PSW_MASK_ASC;
+    int flags;
+
+    if (mmu_translate(env, dest, 1, asc, &dest_phys, &flags)) {
+        cpu_stb_data(env, dest, 0);
+        cpu_abort(env, "should never reach here");
+    }
+    dest_phys |= dest & ~TARGET_PAGE_MASK;
+
+    if (mmu_translate(env, src, 0, asc, &src_phys, &flags)) {
+        cpu_ldub_data(env, src);
+        cpu_abort(env, "should never reach here");
+    }
+    src_phys |= src & ~TARGET_PAGE_MASK;
+
+    dest_p = cpu_physical_memory_map(dest_phys, &len, 1);
+    src_p = cpu_physical_memory_map(src_phys, &len, 0);
+
+    memmove(dest_p, src_p, len);
+
+    cpu_physical_memory_unmap(dest_p, 1, len, len);
+    cpu_physical_memory_unmap(src_p, 0, len, len);
+}
+#endif
+
+/* and on array */
+uint32_t HELPER(nc)(CPUS390XState *env, uint32_t l, uint64_t dest,
+                    uint64_t src)
+{
+    int i;
+    unsigned char x;
+    uint32_t cc = 0;
+
+    HELPER_LOG("%s l %d dest %" PRIx64 " src %" PRIx64 "\n",
+               __func__, l, dest, src);
+    for (i = 0; i <= l; i++) {
+        x = cpu_ldub_data(env, dest + i) & cpu_ldub_data(env, src + i);
+        if (x) {
+            cc = 1;
+        }
+        cpu_stb_data(env, dest + i, x);
+    }
+    return cc;
+}
+
+/* xor on array */
+uint32_t HELPER(xc)(CPUS390XState *env, uint32_t l, uint64_t dest,
+                    uint64_t src)
+{
+    int i;
+    unsigned char x;
+    uint32_t cc = 0;
+
+    HELPER_LOG("%s l %d dest %" PRIx64 " src %" PRIx64 "\n",
+               __func__, l, dest, src);
+
+#ifndef CONFIG_USER_ONLY
+    /* xor with itself is the same as memset(0) */
+    if ((l > 32) && (src == dest) &&
+        (src & TARGET_PAGE_MASK) == ((src + l) & TARGET_PAGE_MASK)) {
+        mvc_fast_memset(env, l + 1, dest, 0);
+        return 0;
+    }
+#else
+    if (src == dest) {
+        memset(g2h(dest), 0, l + 1);
+        return 0;
+    }
+#endif
+
+    for (i = 0; i <= l; i++) {
+        x = cpu_ldub_data(env, dest + i) ^ cpu_ldub_data(env, src + i);
+        if (x) {
+            cc = 1;
+        }
+        cpu_stb_data(env, dest + i, x);
+    }
+    return cc;
+}
+
+/* or on array */
+uint32_t HELPER(oc)(CPUS390XState *env, uint32_t l, uint64_t dest,
+                    uint64_t src)
+{
+    int i;
+    unsigned char x;
+    uint32_t cc = 0;
+
+    HELPER_LOG("%s l %d dest %" PRIx64 " src %" PRIx64 "\n",
+               __func__, l, dest, src);
+    for (i = 0; i <= l; i++) {
+        x = cpu_ldub_data(env, dest + i) | cpu_ldub_data(env, src + i);
+        if (x) {
+            cc = 1;
+        }
+        cpu_stb_data(env, dest + i, x);
+    }
+    return cc;
+}
+
+/* memmove */
+void HELPER(mvc)(CPUS390XState *env, uint32_t l, uint64_t dest, uint64_t src)
+{
+    int i = 0;
+    int x = 0;
+    uint32_t l_64 = (l + 1) / 8;
+
+    HELPER_LOG("%s l %d dest %" PRIx64 " src %" PRIx64 "\n",
+               __func__, l, dest, src);
+
+#ifndef CONFIG_USER_ONLY
+    if ((l > 32) &&
+        (src & TARGET_PAGE_MASK) == ((src + l) & TARGET_PAGE_MASK) &&
+        (dest & TARGET_PAGE_MASK) == ((dest + l) & TARGET_PAGE_MASK)) {
+        if (dest == (src + 1)) {
+            mvc_fast_memset(env, l + 1, dest, cpu_ldub_data(env, src));
+            return;
+        } else if ((src & TARGET_PAGE_MASK) != (dest & TARGET_PAGE_MASK)) {
+            mvc_fast_memmove(env, l + 1, dest, src);
+            return;
+        }
+    }
+#else
+    if (dest == (src + 1)) {
+        memset(g2h(dest), cpu_ldub_data(env, src), l + 1);
+        return;
+    } else {
+        memmove(g2h(dest), g2h(src), l + 1);
+        return;
+    }
+#endif
+
+    /* handle the parts that fit into 8-byte loads/stores */
+    if (dest != (src + 1)) {
+        for (i = 0; i < l_64; i++) {
+            cpu_stq_data(env, dest + x, cpu_ldq_data(env, src + x));
+            x += 8;
+        }
+    }
+
+    /* slow version crossing pages with byte accesses */
+    for (i = x; i <= l; i++) {
+        cpu_stb_data(env, dest + i, cpu_ldub_data(env, src + i));
+    }
+}
+
+/* compare unsigned byte arrays */
+uint32_t HELPER(clc)(CPUS390XState *env, uint32_t l, uint64_t s1, uint64_t s2)
+{
+    int i;
+    unsigned char x, y;
+    uint32_t cc;
+
+    HELPER_LOG("%s l %d s1 %" PRIx64 " s2 %" PRIx64 "\n",
+               __func__, l, s1, s2);
+    for (i = 0; i <= l; i++) {
+        x = cpu_ldub_data(env, s1 + i);
+        y = cpu_ldub_data(env, s2 + i);
+        HELPER_LOG("%02x (%c)/%02x (%c) ", x, x, y, y);
+        if (x < y) {
+            cc = 1;
+            goto done;
+        } else if (x > y) {
+            cc = 2;
+            goto done;
+        }
+    }
+    cc = 0;
+ done:
+    HELPER_LOG("\n");
+    return cc;
+}
+
+/* compare logical under mask */
+uint32_t HELPER(clm)(CPUS390XState *env, uint32_t r1, uint32_t mask,
+                     uint64_t addr)
+{
+    uint8_t r, d;
+    uint32_t cc;
+
+    HELPER_LOG("%s: r1 0x%x mask 0x%x addr 0x%" PRIx64 "\n", __func__, r1,
+               mask, addr);
+    cc = 0;
+    while (mask) {
+        if (mask & 8) {
+            d = cpu_ldub_data(env, addr);
+            r = (r1 & 0xff000000UL) >> 24;
+            HELPER_LOG("mask 0x%x %02x/%02x (0x%" PRIx64 ") ", mask, r, d,
+                       addr);
+            if (r < d) {
+                cc = 1;
+                break;
+            } else if (r > d) {
+                cc = 2;
+                break;
+            }
+            addr++;
+        }
+        mask = (mask << 1) & 0xf;
+        r1 <<= 8;
+    }
+    HELPER_LOG("\n");
+    return cc;
+}
+
+/* store character under mask */
+void HELPER(stcm)(CPUS390XState *env, uint32_t r1, uint32_t mask,
+                  uint64_t addr)
+{
+    uint8_t r;
+
+    HELPER_LOG("%s: r1 0x%x mask 0x%x addr 0x%lx\n", __func__, r1, mask,
+               addr);
+    while (mask) {
+        if (mask & 8) {
+            r = (r1 & 0xff000000UL) >> 24;
+            cpu_stb_data(env, addr, r);
+            HELPER_LOG("mask 0x%x %02x (0x%lx) ", mask, r, addr);
+            addr++;
+        }
+        mask = (mask << 1) & 0xf;
+        r1 <<= 8;
+    }
+    HELPER_LOG("\n");
+}
+
+static inline uint64_t get_address(CPUS390XState *env, int x2, int b2, int d2)
+{
+    uint64_t r = d2;
+
+    if (x2) {
+        r += env->regs[x2];
+    }
+
+    if (b2) {
+        r += env->regs[b2];
+    }
+
+    /* 31-Bit mode */
+    if (!(env->psw.mask & PSW_MASK_64)) {
+        r &= 0x7fffffff;
+    }
+
+    return r;
+}
+
+static inline uint64_t get_address_31fix(CPUS390XState *env, int reg)
+{
+    uint64_t r = env->regs[reg];
+
+    /* 31-Bit mode */
+    if (!(env->psw.mask & PSW_MASK_64)) {
+        r &= 0x7fffffff;
+    }
+
+    return r;
+}
+
+/* search string (c is byte to search, r2 is string, r1 end of string) */
+uint32_t HELPER(srst)(CPUS390XState *env, uint32_t c, uint32_t r1, uint32_t r2)
+{
+    uint64_t i;
+    uint32_t cc = 2;
+    uint64_t str = get_address_31fix(env, r2);
+    uint64_t end = get_address_31fix(env, r1);
+
+    HELPER_LOG("%s: c %d *r1 0x%" PRIx64 " *r2 0x%" PRIx64 "\n", __func__,
+               c, env->regs[r1], env->regs[r2]);
+
+    for (i = str; i != end; i++) {
+        if (cpu_ldub_data(env, i) == c) {
+            env->regs[r1] = i;
+            cc = 1;
+            break;
+        }
+    }
+
+    return cc;
+}
+
+/* unsigned string compare (c is string terminator) */
+uint32_t HELPER(clst)(CPUS390XState *env, uint32_t c, uint32_t r1, uint32_t r2)
+{
+    uint64_t s1 = get_address_31fix(env, r1);
+    uint64_t s2 = get_address_31fix(env, r2);
+    uint8_t v1, v2;
+    uint32_t cc;
+
+    c = c & 0xff;
+#ifdef CONFIG_USER_ONLY
+    if (!c) {
+        HELPER_LOG("%s: comparing '%s' and '%s'\n",
+                   __func__, (char *)g2h(s1), (char *)g2h(s2));
+    }
+#endif
+    for (;;) {
+        v1 = cpu_ldub_data(env, s1);
+        v2 = cpu_ldub_data(env, s2);
+        if ((v1 == c || v2 == c) || (v1 != v2)) {
+            break;
+        }
+        s1++;
+        s2++;
+    }
+
+    if (v1 == v2) {
+        cc = 0;
+    } else {
+        cc = (v1 < v2) ? 1 : 2;
+        /* FIXME: 31-bit mode! */
+        env->regs[r1] = s1;
+        env->regs[r2] = s2;
+    }
+    return cc;
+}
+
+/* move page */
+void HELPER(mvpg)(CPUS390XState *env, uint64_t r0, uint64_t r1, uint64_t r2)
+{
+    /* XXX missing r0 handling */
+#ifdef CONFIG_USER_ONLY
+    int i;
+
+    for (i = 0; i < TARGET_PAGE_SIZE; i++) {
+        cpu_stb_data(env, r1 + i, cpu_ldub_data(env, r2 + i));
+    }
+#else
+    mvc_fast_memmove(env, TARGET_PAGE_SIZE, r1, r2);
+#endif
+}
+
+/* string copy (c is string terminator) */
+void HELPER(mvst)(CPUS390XState *env, uint32_t c, uint32_t r1, uint32_t r2)
+{
+    uint64_t dest = get_address_31fix(env, r1);
+    uint64_t src = get_address_31fix(env, r2);
+    uint8_t v;
+
+    c = c & 0xff;
+#ifdef CONFIG_USER_ONLY
+    if (!c) {
+        HELPER_LOG("%s: copy '%s' to 0x%lx\n", __func__, (char *)g2h(src),
+                   dest);
+    }
+#endif
+    for (;;) {
+        v = cpu_ldub_data(env, src);
+        cpu_stb_data(env, dest, v);
+        if (v == c) {
+            break;
+        }
+        src++;
+        dest++;
+    }
+    env->regs[r1] = dest; /* FIXME: 31-bit mode! */
+}
+
+/* compare and swap 64-bit */
+uint32_t HELPER(csg)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3)
+{
+    /* FIXME: locking? */
+    uint32_t cc;
+    uint64_t v2 = cpu_ldq_data(env, a2);
+
+    if (env->regs[r1] == v2) {
+        cc = 0;
+        cpu_stq_data(env, a2, env->regs[r3]);
+    } else {
+        cc = 1;
+        env->regs[r1] = v2;
+    }
+    return cc;
+}
+
+/* compare double and swap 64-bit */
+uint32_t HELPER(cdsg)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3)
+{
+    /* FIXME: locking? */
+    uint32_t cc;
+    uint64_t v2_hi = cpu_ldq_data(env, a2);
+    uint64_t v2_lo = cpu_ldq_data(env, a2 + 8);
+    uint64_t v1_hi = env->regs[r1];
+    uint64_t v1_lo = env->regs[r1 + 1];
+
+    if ((v1_hi == v2_hi) && (v1_lo == v2_lo)) {
+        cc = 0;
+        cpu_stq_data(env, a2, env->regs[r3]);
+        cpu_stq_data(env, a2 + 8, env->regs[r3 + 1]);
+    } else {
+        cc = 1;
+        env->regs[r1] = v2_hi;
+        env->regs[r1 + 1] = v2_lo;
+    }
+
+    return cc;
+}
+
+/* compare and swap 32-bit */
+uint32_t HELPER(cs)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3)
+{
+    /* FIXME: locking? */
+    uint32_t cc;
+    uint32_t v2 = cpu_ldl_data(env, a2);
+
+    HELPER_LOG("%s: r1 %d a2 0x%lx r3 %d\n", __func__, r1, a2, r3);
+    if (((uint32_t)env->regs[r1]) == v2) {
+        cc = 0;
+        cpu_stl_data(env, a2, (uint32_t)env->regs[r3]);
+    } else {
+        cc = 1;
+        env->regs[r1] = (env->regs[r1] & 0xffffffff00000000ULL) | v2;
+    }
+    return cc;
+}
+
+static uint32_t helper_icm(CPUS390XState *env, uint32_t r1, uint64_t address,
+                           uint32_t mask)
+{
+    int pos = 24; /* top of the lower half of r1 */
+    uint64_t rmask = 0xff000000ULL;
+    uint8_t val = 0;
+    int ccd = 0;
+    uint32_t cc = 0;
+
+    while (mask) {
+        if (mask & 8) {
+            env->regs[r1] &= ~rmask;
+            val = cpu_ldub_data(env, address);
+            if ((val & 0x80) && !ccd) {
+                cc = 1;
+            }
+            ccd = 1;
+            if (val && cc == 0) {
+                cc = 2;
+            }
+            env->regs[r1] |= (uint64_t)val << pos;
+            address++;
+        }
+        mask = (mask << 1) & 0xf;
+        pos -= 8;
+        rmask >>= 8;
+    }
+
+    return cc;
+}
+
+/* execute instruction
+   this instruction executes an insn modified with the contents of r1
+   it does not change the executed instruction in memory
+   it does not change the program counter
+   in other words: tricky...
+   currently implemented by interpreting the cases it is most commonly used in
+*/
+uint32_t HELPER(ex)(CPUS390XState *env, uint32_t cc, uint64_t v1,
+                    uint64_t addr, uint64_t ret)
+{
+    uint16_t insn = cpu_lduw_code(env, addr);
+
+    HELPER_LOG("%s: v1 0x%lx addr 0x%lx insn 0x%x\n", __func__, v1, addr,
+               insn);
+    if ((insn & 0xf0ff) == 0xd000) {
+        uint32_t l, insn2, b1, b2, d1, d2;
+
+        l = v1 & 0xff;
+        insn2 = cpu_ldl_code(env, addr + 2);
+        b1 = (insn2 >> 28) & 0xf;
+        b2 = (insn2 >> 12) & 0xf;
+        d1 = (insn2 >> 16) & 0xfff;
+        d2 = insn2 & 0xfff;
+        switch (insn & 0xf00) {
+        case 0x200:
+            helper_mvc(env, l, get_address(env, 0, b1, d1),
+                       get_address(env, 0, b2, d2));
+            break;
+        case 0x500:
+            cc = helper_clc(env, l, get_address(env, 0, b1, d1),
+                            get_address(env, 0, b2, d2));
+            break;
+        case 0x700:
+            cc = helper_xc(env, l, get_address(env, 0, b1, d1),
+                           get_address(env, 0, b2, d2));
+            break;
+        case 0xc00:
+            helper_tr(env, l, get_address(env, 0, b1, d1),
+                      get_address(env, 0, b2, d2));
+            break;
+        default:
+            goto abort;
+            break;
+        }
+    } else if ((insn & 0xff00) == 0x0a00) {
+        /* supervisor call */
+        HELPER_LOG("%s: svc %ld via execute\n", __func__, (insn | v1) & 0xff);
+        env->psw.addr = ret - 4;
+        env->int_svc_code = (insn | v1) & 0xff;
+        env->int_svc_ilc = 4;
+        helper_exception(env, EXCP_SVC);
+    } else if ((insn & 0xff00) == 0xbf00) {
+        uint32_t insn2, r1, r3, b2, d2;
+
+        insn2 = cpu_ldl_code(env, addr + 2);
+        r1 = (insn2 >> 20) & 0xf;
+        r3 = (insn2 >> 16) & 0xf;
+        b2 = (insn2 >> 12) & 0xf;
+        d2 = insn2 & 0xfff;
+        cc = helper_icm(env, r1, get_address(env, 0, b2, d2), r3);
+    } else {
+    abort:
+        cpu_abort(env, "EXECUTE on instruction prefix 0x%x not implemented\n",
+                  insn);
+    }
+    return cc;
+}
+
+/* store character under mask high operates on the upper half of r1 */
+void HELPER(stcmh)(CPUS390XState *env, uint32_t r1, uint64_t address,
+                   uint32_t mask)
+{
+    int pos = 56; /* top of the upper half of r1 */
+
+    while (mask) {
+        if (mask & 8) {
+            cpu_stb_data(env, address, (env->regs[r1] >> pos) & 0xff);
+            address++;
+        }
+        mask = (mask << 1) & 0xf;
+        pos -= 8;
+    }
+}
+
+/* insert character under mask high; same as icm, but operates on the
+   upper half of r1 */
+uint32_t HELPER(icmh)(CPUS390XState *env, uint32_t r1, uint64_t address,
+                      uint32_t mask)
+{
+    int pos = 56; /* top of the upper half of r1 */
+    uint64_t rmask = 0xff00000000000000ULL;
+    uint8_t val = 0;
+    int ccd = 0;
+    uint32_t cc = 0;
+
+    while (mask) {
+        if (mask & 8) {
+            env->regs[r1] &= ~rmask;
+            val = cpu_ldub_data(env, address);
+            if ((val & 0x80) && !ccd) {
+                cc = 1;
+            }
+            ccd = 1;
+            if (val && cc == 0) {
+                cc = 2;
+            }
+            env->regs[r1] |= (uint64_t)val << pos;
+            address++;
+        }
+        mask = (mask << 1) & 0xf;
+        pos -= 8;
+        rmask >>= 8;
+    }
+
+    return cc;
+}
+
+/* load access registers r1 to r3 from memory at a2 */
+void HELPER(lam)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3)
+{
+    int i;
+
+    for (i = r1;; i = (i + 1) % 16) {
+        env->aregs[i] = cpu_ldl_data(env, a2);
+        a2 += 4;
+
+        if (i == r3) {
+            break;
+        }
+    }
+}
+
+/* store access registers r1 to r3 in memory at a2 */
+void HELPER(stam)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3)
+{
+    int i;
+
+    for (i = r1;; i = (i + 1) % 16) {
+        cpu_stl_data(env, a2, env->aregs[i]);
+        a2 += 4;
+
+        if (i == r3) {
+            break;
+        }
+    }
+}
+
+/* move long */
+uint32_t HELPER(mvcl)(CPUS390XState *env, uint32_t r1, uint32_t r2)
+{
+    uint64_t destlen = env->regs[r1 + 1] & 0xffffff;
+    uint64_t dest = get_address_31fix(env, r1);
+    uint64_t srclen = env->regs[r2 + 1] & 0xffffff;
+    uint64_t src = get_address_31fix(env, r2);
+    uint8_t pad = src >> 24;
+    uint8_t v;
+    uint32_t cc;
+
+    if (destlen == srclen) {
+        cc = 0;
+    } else if (destlen < srclen) {
+        cc = 1;
+    } else {
+        cc = 2;
+    }
+
+    if (srclen > destlen) {
+        srclen = destlen;
+    }
+
+    for (; destlen && srclen; src++, dest++, destlen--, srclen--) {
+        v = cpu_ldub_data(env, src);
+        cpu_stb_data(env, dest, v);
+    }
+
+    for (; destlen; dest++, destlen--) {
+        cpu_stb_data(env, dest, pad);
+    }
+
+    env->regs[r1 + 1] = destlen;
+    /* can't use srclen here, we trunc'ed it */
+    env->regs[r2 + 1] -= src - env->regs[r2];
+    env->regs[r1] = dest;
+    env->regs[r2] = src;
+
+    return cc;
+}
+
+/* move long extended another memcopy insn with more bells and whistles */
+uint32_t HELPER(mvcle)(CPUS390XState *env, uint32_t r1, uint64_t a2,
+                       uint32_t r3)
+{
+    uint64_t destlen = env->regs[r1 + 1];
+    uint64_t dest = env->regs[r1];
+    uint64_t srclen = env->regs[r3 + 1];
+    uint64_t src = env->regs[r3];
+    uint8_t pad = a2 & 0xff;
+    uint8_t v;
+    uint32_t cc;
+
+    if (!(env->psw.mask & PSW_MASK_64)) {
+        destlen = (uint32_t)destlen;
+        srclen = (uint32_t)srclen;
+        dest &= 0x7fffffff;
+        src &= 0x7fffffff;
+    }
+
+    if (destlen == srclen) {
+        cc = 0;
+    } else if (destlen < srclen) {
+        cc = 1;
+    } else {
+        cc = 2;
+    }
+
+    if (srclen > destlen) {
+        srclen = destlen;
+    }
+
+    for (; destlen && srclen; src++, dest++, destlen--, srclen--) {
+        v = cpu_ldub_data(env, src);
+        cpu_stb_data(env, dest, v);
+    }
+
+    for (; destlen; dest++, destlen--) {
+        cpu_stb_data(env, dest, pad);
+    }
+
+    env->regs[r1 + 1] = destlen;
+    /* can't use srclen here, we trunc'ed it */
+    /* FIXME: 31-bit mode! */
+    env->regs[r3 + 1] -= src - env->regs[r3];
+    env->regs[r1] = dest;
+    env->regs[r3] = src;
+
+    return cc;
+}
+
+/* compare logical long extended memcompare insn with padding */
+uint32_t HELPER(clcle)(CPUS390XState *env, uint32_t r1, uint64_t a2,
+                       uint32_t r3)
+{
+    uint64_t destlen = env->regs[r1 + 1];
+    uint64_t dest = get_address_31fix(env, r1);
+    uint64_t srclen = env->regs[r3 + 1];
+    uint64_t src = get_address_31fix(env, r3);
+    uint8_t pad = a2 & 0xff;
+    uint8_t v1 = 0, v2 = 0;
+    uint32_t cc = 0;
+
+    if (!(destlen || srclen)) {
+        return cc;
+    }
+
+    if (srclen > destlen) {
+        srclen = destlen;
+    }
+
+    for (; destlen || srclen; src++, dest++, destlen--, srclen--) {
+        v1 = srclen ? cpu_ldub_data(env, src) : pad;
+        v2 = destlen ? cpu_ldub_data(env, dest) : pad;
+        if (v1 != v2) {
+            cc = (v1 < v2) ? 1 : 2;
+            break;
+        }
+    }
+
+    env->regs[r1 + 1] = destlen;
+    /* can't use srclen here, we trunc'ed it */
+    env->regs[r3 + 1] -= src - env->regs[r3];
+    env->regs[r1] = dest;
+    env->regs[r3] = src;
+
+    return cc;
+}
+
+/* checksum */
+void HELPER(cksm)(CPUS390XState *env, uint32_t r1, uint32_t r2)
+{
+    uint64_t src = get_address_31fix(env, r2);
+    uint64_t src_len = env->regs[(r2 + 1) & 15];
+    uint64_t cksm = (uint32_t)env->regs[r1];
+
+    while (src_len >= 4) {
+        cksm += cpu_ldl_data(env, src);
+
+        /* move to next word */
+        src_len -= 4;
+        src += 4;
+    }
+
+    switch (src_len) {
+    case 0:
+        break;
+    case 1:
+        cksm += cpu_ldub_data(env, src) << 24;
+        break;
+    case 2:
+        cksm += cpu_lduw_data(env, src) << 16;
+        break;
+    case 3:
+        cksm += cpu_lduw_data(env, src) << 16;
+        cksm += cpu_ldub_data(env, src + 2) << 8;
+        break;
+    }
+
+    /* indicate we've processed everything */
+    env->regs[r2] = src + src_len;
+    env->regs[(r2 + 1) & 15] = 0;
+
+    /* store result */
+    env->regs[r1] = (env->regs[r1] & 0xffffffff00000000ULL) |
+        ((uint32_t)cksm + (cksm >> 32));
+}
+
+void HELPER(unpk)(CPUS390XState *env, uint32_t len, uint64_t dest,
+                  uint64_t src)
+{
+    int len_dest = len >> 4;
+    int len_src = len & 0xf;
+    uint8_t b;
+    int second_nibble = 0;
+
+    dest += len_dest;
+    src += len_src;
+
+    /* last byte is special, it only flips the nibbles */
+    b = cpu_ldub_data(env, src);
+    cpu_stb_data(env, dest, (b << 4) | (b >> 4));
+    src--;
+    len_src--;
+
+    /* now pad every nibble with 0xf0 */
+
+    while (len_dest > 0) {
+        uint8_t cur_byte = 0;
+
+        if (len_src > 0) {
+            cur_byte = cpu_ldub_data(env, src);
+        }
+
+        len_dest--;
+        dest--;
+
+        /* only advance one nibble at a time */
+        if (second_nibble) {
+            cur_byte >>= 4;
+            len_src--;
+            src--;
+        }
+        second_nibble = !second_nibble;
+
+        /* digit */
+        cur_byte = (cur_byte & 0xf);
+        /* zone bits */
+        cur_byte |= 0xf0;
+
+        cpu_stb_data(env, dest, cur_byte);
+    }
+}
+
+void HELPER(tr)(CPUS390XState *env, uint32_t len, uint64_t array,
+                uint64_t trans)
+{
+    int i;
+
+    for (i = 0; i <= len; i++) {
+        uint8_t byte = cpu_ldub_data(env, array + i);
+        uint8_t new_byte = cpu_ldub_data(env, trans + byte);
+
+        cpu_stb_data(env, array + i, new_byte);
+    }
+}
+
+#if !defined(CONFIG_USER_ONLY)
+void HELPER(lctlg)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3)
+{
+    int i;
+    uint64_t src = a2;
+
+    for (i = r1;; i = (i + 1) % 16) {
+        env->cregs[i] = cpu_ldq_data(env, src);
+        HELPER_LOG("load ctl %d from 0x%" PRIx64 " == 0x%" PRIx64 "\n",
+                   i, src, env->cregs[i]);
+        src += sizeof(uint64_t);
+
+        if (i == r3) {
+            break;
+        }
+    }
+
+    tlb_flush(env, 1);
+}
+
+void HELPER(lctl)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3)
+{
+    int i;
+    uint64_t src = a2;
+
+    for (i = r1;; i = (i + 1) % 16) {
+        env->cregs[i] = (env->cregs[i] & 0xFFFFFFFF00000000ULL) |
+            cpu_ldl_data(env, src);
+        src += sizeof(uint32_t);
+
+        if (i == r3) {
+            break;
+        }
+    }
+
+    tlb_flush(env, 1);
+}
+
+void HELPER(stctg)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3)
+{
+    int i;
+    uint64_t dest = a2;
+
+    for (i = r1;; i = (i + 1) % 16) {
+        cpu_stq_data(env, dest, env->cregs[i]);
+        dest += sizeof(uint64_t);
+
+        if (i == r3) {
+            break;
+        }
+    }
+}
+
+void HELPER(stctl)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3)
+{
+    int i;
+    uint64_t dest = a2;
+
+    for (i = r1;; i = (i + 1) % 16) {
+        cpu_stl_data(env, dest, env->cregs[i]);
+        dest += sizeof(uint32_t);
+
+        if (i == r3) {
+            break;
+        }
+    }
+}
+
+uint32_t HELPER(tprot)(uint64_t a1, uint64_t a2)
+{
+    /* XXX implement */
+
+    return 0;
+}
+
+/* insert storage key extended */
+uint64_t HELPER(iske)(CPUS390XState *env, uint64_t r2)
+{
+    uint64_t addr = get_address(env, 0, 0, r2);
+
+    if (addr > ram_size) {
+        return 0;
+    }
+
+    return env->storage_keys[addr / TARGET_PAGE_SIZE];
+}
+
+/* set storage key extended */
+void HELPER(sske)(CPUS390XState *env, uint32_t r1, uint64_t r2)
+{
+    uint64_t addr = get_address(env, 0, 0, r2);
+
+    if (addr > ram_size) {
+        return;
+    }
+
+    env->storage_keys[addr / TARGET_PAGE_SIZE] = r1;
+}
+
+/* reset reference bit extended */
+uint32_t HELPER(rrbe)(CPUS390XState *env, uint32_t r1, uint64_t r2)
+{
+    uint8_t re;
+    uint8_t key;
+
+    if (r2 > ram_size) {
+        return 0;
+    }
+
+    key = env->storage_keys[r2 / TARGET_PAGE_SIZE];
+    re = key & (SK_R | SK_C);
+    env->storage_keys[r2 / TARGET_PAGE_SIZE] = (key & ~SK_R);
+
+    /*
+     * cc
+     *
+     * 0  Reference bit zero; change bit zero
+     * 1  Reference bit zero; change bit one
+     * 2  Reference bit one; change bit zero
+     * 3  Reference bit one; change bit one
+     */
+
+    return re >> 1;
+}
+
+/* compare and swap and purge */
+uint32_t HELPER(csp)(CPUS390XState *env, uint32_t r1, uint32_t r2)
+{
+    uint32_t cc;
+    uint32_t o1 = env->regs[r1];
+    uint64_t a2 = get_address_31fix(env, r2) & ~3ULL;
+    uint32_t o2 = cpu_ldl_data(env, a2);
+
+    if (o1 == o2) {
+        cpu_stl_data(env, a2, env->regs[(r1 + 1) & 15]);
+        if (env->regs[r2] & 0x3) {
+            /* flush TLB / ALB */
+            tlb_flush(env, 1);
+        }
+        cc = 0;
+    } else {
+        env->regs[r1] = (env->regs[r1] & 0xffffffff00000000ULL) | o2;
+        cc = 1;
+    }
+
+    return cc;
+}
+
+static uint32_t mvc_asc(CPUS390XState *env, int64_t l, uint64_t a1,
+                        uint64_t mode1, uint64_t a2, uint64_t mode2)
+{
+    target_ulong src, dest;
+    int flags, cc = 0, i;
+
+    if (!l) {
+        return 0;
+    } else if (l > 256) {
+        /* max 256 */
+        l = 256;
+        cc = 3;
+    }
+
+    if (mmu_translate(env, a1 & TARGET_PAGE_MASK, 1, mode1, &dest, &flags)) {
+        cpu_loop_exit(env);
+    }
+    dest |= a1 & ~TARGET_PAGE_MASK;
+
+    if (mmu_translate(env, a2 & TARGET_PAGE_MASK, 0, mode2, &src, &flags)) {
+        cpu_loop_exit(env);
+    }
+    src |= a2 & ~TARGET_PAGE_MASK;
+
+    /* XXX replace w/ memcpy */
+    for (i = 0; i < l; i++) {
+        /* XXX be more clever */
+        if ((((dest + i) & TARGET_PAGE_MASK) != (dest & TARGET_PAGE_MASK)) ||
+            (((src + i) & TARGET_PAGE_MASK) != (src & TARGET_PAGE_MASK))) {
+            mvc_asc(env, l - i, a1 + i, mode1, a2 + i, mode2);
+            break;
+        }
+        stb_phys(dest + i, ldub_phys(src + i));
+    }
+
+    return cc;
+}
+
+uint32_t HELPER(mvcs)(CPUS390XState *env, uint64_t l, uint64_t a1, uint64_t a2)
+{
+    HELPER_LOG("%s: %16" PRIx64 " %16" PRIx64 " %16" PRIx64 "\n",
+               __func__, l, a1, a2);
+
+    return mvc_asc(env, l, a1, PSW_ASC_SECONDARY, a2, PSW_ASC_PRIMARY);
+}
+
+uint32_t HELPER(mvcp)(CPUS390XState *env, uint64_t l, uint64_t a1, uint64_t a2)
+{
+    HELPER_LOG("%s: %16" PRIx64 " %16" PRIx64 " %16" PRIx64 "\n",
+               __func__, l, a1, a2);
+
+    return mvc_asc(env, l, a1, PSW_ASC_PRIMARY, a2, PSW_ASC_SECONDARY);
+}
+
+/* invalidate pte */
+void HELPER(ipte)(CPUS390XState *env, uint64_t pte_addr, uint64_t vaddr)
+{
+    uint64_t page = vaddr & TARGET_PAGE_MASK;
+    uint64_t pte = 0;
+
+    /* XXX broadcast to other CPUs */
+
+    /* XXX Linux is nice enough to give us the exact pte address.
+       According to spec we'd have to find it out ourselves */
+    /* XXX Linux is fine with overwriting the pte, the spec requires
+       us to only set the invalid bit */
+    stq_phys(pte_addr, pte | _PAGE_INVALID);
+
+    /* XXX we exploit the fact that Linux passes the exact virtual
+       address here - it's not obliged to! */
+    tlb_flush_page(env, page);
+
+    /* XXX 31-bit hack */
+    if (page & 0x80000000) {
+        tlb_flush_page(env, page & ~0x80000000);
+    } else {
+        tlb_flush_page(env, page | 0x80000000);
+    }
+}
+
+/* flush local tlb */
+void HELPER(ptlb)(CPUS390XState *env)
+{
+    tlb_flush(env, 1);
+}
+
+/* store using real address */
+void HELPER(stura)(CPUS390XState *env, uint64_t addr, uint32_t v1)
+{
+    stw_phys(get_address(env, 0, 0, addr), v1);
+}
+
+/* load real address */
+uint32_t HELPER(lra)(CPUS390XState *env, uint64_t addr, uint32_t r1)
+{
+    uint32_t cc = 0;
+    int old_exc = env->exception_index;
+    uint64_t asc = env->psw.mask & PSW_MASK_ASC;
+    uint64_t ret;
+    int flags;
+
+    /* XXX incomplete - has more corner cases */
+    if (!(env->psw.mask & PSW_MASK_64) && (addr >> 32)) {
+        program_interrupt(env, PGM_SPECIAL_OP, 2);
+    }
+
+    env->exception_index = old_exc;
+    if (mmu_translate(env, addr, 0, asc, &ret, &flags)) {
+        cc = 3;
+    }
+    if (env->exception_index == EXCP_PGM) {
+        ret = env->int_pgm_code | 0x80000000;
+    } else {
+        ret |= addr & ~TARGET_PAGE_MASK;
+    }
+    env->exception_index = old_exc;
+
+    if (!(env->psw.mask & PSW_MASK_64)) {
+        env->regs[r1] = (env->regs[r1] & 0xffffffff00000000ULL) |
+            (ret & 0xffffffffULL);
+    } else {
+        env->regs[r1] = ret;
+    }
+
+    return cc;
+}
+
+#endif
diff --git a/target-s390x/misc_helper.c b/target-s390x/misc_helper.c
new file mode 100644
index 000000000..38d8f2a62
--- /dev/null
+++ b/target-s390x/misc_helper.c
@@ -0,0 +1,387 @@
+/*
+ *  S/390 misc helper routines
+ *
+ *  Copyright (c) 2009 Ulrich Hecht
+ *  Copyright (c) 2009 Alexander Graf
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "cpu.h"
+#include "memory.h"
+#include "host-utils.h"
+#include "helper.h"
+#include <string.h>
+#include "kvm.h"
+#include "qemu-timer.h"
+#ifdef CONFIG_KVM
+#include <linux/kvm.h>
+#endif
+
+#if !defined(CONFIG_USER_ONLY)
+#include "softmmu_exec.h"
+#include "sysemu.h"
+#endif
+
+/* #define DEBUG_HELPER */
+#ifdef DEBUG_HELPER
+#define HELPER_LOG(x...) qemu_log(x)
+#else
+#define HELPER_LOG(x...)
+#endif
+
+/* raise an exception */
+void HELPER(exception)(CPUS390XState *env, uint32_t excp)
+{
+    HELPER_LOG("%s: exception %d\n", __func__, excp);
+    env->exception_index = excp;
+    cpu_loop_exit(env);
+}
+
+#ifndef CONFIG_USER_ONLY
+void program_interrupt(CPUS390XState *env, uint32_t code, int ilc)
+{
+    qemu_log_mask(CPU_LOG_INT, "program interrupt at %#" PRIx64 "\n",
+                  env->psw.addr);
+
+    if (kvm_enabled()) {
+#ifdef CONFIG_KVM
+        kvm_s390_interrupt(env, KVM_S390_PROGRAM_INT, code);
+#endif
+    } else {
+        env->int_pgm_code = code;
+        env->int_pgm_ilc = ilc;
+        env->exception_index = EXCP_PGM;
+        cpu_loop_exit(env);
+    }
+}
+
+/* SCLP service call */
+uint32_t HELPER(servc)(CPUS390XState *env, uint32_t r1, uint64_t r2)
+{
+    int r;
+
+    r = sclp_service_call(r1, r2);
+    if (r < 0) {
+        program_interrupt(env, -r, 4);
+        return 0;
+    }
+    return r;
+}
+
+/* DIAG */
+uint64_t HELPER(diag)(CPUS390XState *env, uint32_t num, uint64_t mem,
+                      uint64_t code)
+{
+    uint64_t r;
+
+    switch (num) {
+    case 0x500:
+        /* KVM hypercall */
+        r = s390_virtio_hypercall(env, mem, code);
+        break;
+    case 0x44:
+        /* yield */
+        r = 0;
+        break;
+    case 0x308:
+        /* ipl */
+        r = 0;
+        break;
+    default:
+        r = -1;
+        break;
+    }
+
+    if (r) {
+        program_interrupt(env, PGM_OPERATION, ILC_LATER_INC);
+    }
+
+    return r;
+}
+
+/* Store CPU ID */
+void HELPER(stidp)(CPUS390XState *env, uint64_t a1)
+{
+    cpu_stq_data(env, a1, env->cpu_num);
+}
+
+/* Set Prefix */
+void HELPER(spx)(CPUS390XState *env, uint64_t a1)
+{
+    uint32_t prefix;
+
+    prefix = cpu_ldl_data(env, a1);
+    env->psa = prefix & 0xfffff000;
+    qemu_log("prefix: %#x\n", prefix);
+    tlb_flush_page(env, 0);
+    tlb_flush_page(env, TARGET_PAGE_SIZE);
+}
+
+/* Set Clock */
+uint32_t HELPER(sck)(uint64_t a1)
+{
+    /* XXX not implemented - is it necessary? */
+
+    return 0;
+}
+
+static inline uint64_t clock_value(CPUS390XState *env)
+{
+    uint64_t time;
+
+    time = env->tod_offset +
+        time2tod(qemu_get_clock_ns(vm_clock) - env->tod_basetime);
+
+    return time;
+}
+
+/* Store Clock */
+uint32_t HELPER(stck)(CPUS390XState *env, uint64_t a1)
+{
+    cpu_stq_data(env, a1, clock_value(env));
+
+    return 0;
+}
+
+/* Store Clock Extended */
+uint32_t HELPER(stcke)(CPUS390XState *env, uint64_t a1)
+{
+    cpu_stb_data(env, a1, 0);
+    /* basically the same value as stck */
+    cpu_stq_data(env, a1 + 1, clock_value(env) | env->cpu_num);
+    /* more fine grained than stck */
+    cpu_stq_data(env, a1 + 9, 0);
+    /* XXX programmable fields */
+    cpu_stw_data(env, a1 + 17, 0);
+
+    return 0;
+}
+
+/* Set Clock Comparator */
+void HELPER(sckc)(CPUS390XState *env, uint64_t a1)
+{
+    uint64_t time = cpu_ldq_data(env, a1);
+
+    if (time == -1ULL) {
+        return;
+    }
+
+    /* difference between now and then */
+    time -= clock_value(env);
+    /* nanoseconds */
+    time = (time * 125) >> 9;
+
+    qemu_mod_timer(env->tod_timer, qemu_get_clock_ns(vm_clock) + time);
+}
+
+/* Store Clock Comparator */
+void HELPER(stckc)(CPUS390XState *env, uint64_t a1)
+{
+    /* XXX implement */
+    cpu_stq_data(env, a1, 0);
+}
+
+/* Set CPU Timer */
+void HELPER(spt)(CPUS390XState *env, uint64_t a1)
+{
+    uint64_t time = cpu_ldq_data(env, a1);
+
+    if (time == -1ULL) {
+        return;
+    }
+
+    /* nanoseconds */
+    time = (time * 125) >> 9;
+
+    qemu_mod_timer(env->cpu_timer, qemu_get_clock_ns(vm_clock) + time);
+}
+
+/* Store CPU Timer */
+void HELPER(stpt)(CPUS390XState *env, uint64_t a1)
+{
+    /* XXX implement */
+    cpu_stq_data(env, a1, 0);
+}
+
+/* Store System Information */
+uint32_t HELPER(stsi)(CPUS390XState *env, uint64_t a0, uint32_t r0,
+                      uint32_t r1)
+{
+    int cc = 0;
+    int sel1, sel2;
+
+    if ((r0 & STSI_LEVEL_MASK) <= STSI_LEVEL_3 &&
+        ((r0 & STSI_R0_RESERVED_MASK) || (r1 & STSI_R1_RESERVED_MASK))) {
+        /* valid function code, invalid reserved bits */
+        program_interrupt(env, PGM_SPECIFICATION, 2);
+    }
+
+    sel1 = r0 & STSI_R0_SEL1_MASK;
+    sel2 = r1 & STSI_R1_SEL2_MASK;
+
+    /* XXX: spec exception if sysib is not 4k-aligned */
+
+    switch (r0 & STSI_LEVEL_MASK) {
+    case STSI_LEVEL_1:
+        if ((sel1 == 1) && (sel2 == 1)) {
+            /* Basic Machine Configuration */
+            struct sysib_111 sysib;
+
+            memset(&sysib, 0, sizeof(sysib));
+            ebcdic_put(sysib.manuf, "QEMU            ", 16);
+            /* same as machine type number in STORE CPU ID */
+            ebcdic_put(sysib.type, "QEMU", 4);
+            /* same as model number in STORE CPU ID */
+            ebcdic_put(sysib.model, "QEMU            ", 16);
+            ebcdic_put(sysib.sequence, "QEMU            ", 16);
+            ebcdic_put(sysib.plant, "QEMU", 4);
+            cpu_physical_memory_rw(a0, (uint8_t *)&sysib, sizeof(sysib), 1);
+        } else if ((sel1 == 2) && (sel2 == 1)) {
+            /* Basic Machine CPU */
+            struct sysib_121 sysib;
+
+            memset(&sysib, 0, sizeof(sysib));
+            /* XXX make different for different CPUs? */
+            ebcdic_put(sysib.sequence, "QEMUQEMUQEMUQEMU", 16);
+            ebcdic_put(sysib.plant, "QEMU", 4);
+            stw_p(&sysib.cpu_addr, env->cpu_num);
+            cpu_physical_memory_rw(a0, (uint8_t *)&sysib, sizeof(sysib), 1);
+        } else if ((sel1 == 2) && (sel2 == 2)) {
+            /* Basic Machine CPUs */
+            struct sysib_122 sysib;
+
+            memset(&sysib, 0, sizeof(sysib));
+            stl_p(&sysib.capability, 0x443afc29);
+            /* XXX change when SMP comes */
+            stw_p(&sysib.total_cpus, 1);
+            stw_p(&sysib.active_cpus, 1);
+            stw_p(&sysib.standby_cpus, 0);
+            stw_p(&sysib.reserved_cpus, 0);
+            cpu_physical_memory_rw(a0, (uint8_t *)&sysib, sizeof(sysib), 1);
+        } else {
+            cc = 3;
+        }
+        break;
+    case STSI_LEVEL_2:
+        {
+            if ((sel1 == 2) && (sel2 == 1)) {
+                /* LPAR CPU */
+                struct sysib_221 sysib;
+
+                memset(&sysib, 0, sizeof(sysib));
+                /* XXX make different for different CPUs? */
+                ebcdic_put(sysib.sequence, "QEMUQEMUQEMUQEMU", 16);
+                ebcdic_put(sysib.plant, "QEMU", 4);
+                stw_p(&sysib.cpu_addr, env->cpu_num);
+                stw_p(&sysib.cpu_id, 0);
+                cpu_physical_memory_rw(a0, (uint8_t *)&sysib, sizeof(sysib), 1);
+            } else if ((sel1 == 2) && (sel2 == 2)) {
+                /* LPAR CPUs */
+                struct sysib_222 sysib;
+
+                memset(&sysib, 0, sizeof(sysib));
+                stw_p(&sysib.lpar_num, 0);
+                sysib.lcpuc = 0;
+                /* XXX change when SMP comes */
+                stw_p(&sysib.total_cpus, 1);
+                stw_p(&sysib.conf_cpus, 1);
+                stw_p(&sysib.standby_cpus, 0);
+                stw_p(&sysib.reserved_cpus, 0);
+                ebcdic_put(sysib.name, "QEMU    ", 8);
+                stl_p(&sysib.caf, 1000);
+                stw_p(&sysib.dedicated_cpus, 0);
+                stw_p(&sysib.shared_cpus, 0);
+                cpu_physical_memory_rw(a0, (uint8_t *)&sysib, sizeof(sysib), 1);
+            } else {
+                cc = 3;
+            }
+            break;
+        }
+    case STSI_LEVEL_3:
+        {
+            if ((sel1 == 2) && (sel2 == 2)) {
+                /* VM CPUs */
+                struct sysib_322 sysib;
+
+                memset(&sysib, 0, sizeof(sysib));
+                sysib.count = 1;
+                /* XXX change when SMP comes */
+                stw_p(&sysib.vm[0].total_cpus, 1);
+                stw_p(&sysib.vm[0].conf_cpus, 1);
+                stw_p(&sysib.vm[0].standby_cpus, 0);
+                stw_p(&sysib.vm[0].reserved_cpus, 0);
+                ebcdic_put(sysib.vm[0].name, "KVMguest", 8);
+                stl_p(&sysib.vm[0].caf, 1000);
+                ebcdic_put(sysib.vm[0].cpi, "KVM/Linux       ", 16);
+                cpu_physical_memory_rw(a0, (uint8_t *)&sysib, sizeof(sysib), 1);
+            } else {
+                cc = 3;
+            }
+            break;
+        }
+    case STSI_LEVEL_CURRENT:
+        env->regs[0] = STSI_LEVEL_3;
+        break;
+    default:
+        cc = 3;
+        break;
+    }
+
+    return cc;
+}
+
+uint32_t HELPER(sigp)(CPUS390XState *env, uint64_t order_code, uint32_t r1,
+                      uint64_t cpu_addr)
+{
+    int cc = 0;
+
+    HELPER_LOG("%s: %016" PRIx64 " %08x %016" PRIx64 "\n",
+               __func__, order_code, r1, cpu_addr);
+
+    /* Remember: Use "R1 or R1 + 1, whichever is the odd-numbered register"
+       as parameter (input). Status (output) is always R1. */
+
+    switch (order_code) {
+    case SIGP_SET_ARCH:
+        /* switch arch */
+        break;
+    case SIGP_SENSE:
+        /* enumerate CPU status */
+        if (cpu_addr) {
+            /* XXX implement when SMP comes */
+            return 3;
+        }
+        env->regs[r1] &= 0xffffffff00000000ULL;
+        cc = 1;
+        break;
+#if !defined(CONFIG_USER_ONLY)
+    case SIGP_RESTART:
+        qemu_system_reset_request();
+        cpu_loop_exit(env);
+        break;
+    case SIGP_STOP:
+        qemu_system_shutdown_request();
+        cpu_loop_exit(env);
+        break;
+#endif
+    default:
+        /* unknown sigp */
+        fprintf(stderr, "XXX unknown sigp: 0x%" PRIx64 "\n", order_code);
+        cc = 3;
+    }
+
+    return cc;
+}
+#endif
diff --git a/target-s390x/op_helper.c b/target-s390x/op_helper.c
deleted file mode 100644
index abc35ddd7..000000000
--- a/target-s390x/op_helper.c
+++ /dev/null
@@ -1,3019 +0,0 @@
-/*
- *  S/390 helper routines
- *
- *  Copyright (c) 2009 Ulrich Hecht
- *  Copyright (c) 2009 Alexander Graf
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Lesser General Public
- * License as published by the Free Software Foundation; either
- * version 2 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
- * Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public
- * License along with this library; if not, see <http://www.gnu.org/licenses/>.
- */
-
-#include "cpu.h"
-#include "memory.h"
-#include "cputlb.h"
-#include "dyngen-exec.h"
-#include "host-utils.h"
-#include "helper.h"
-#include <string.h>
-#include "kvm.h"
-#include "qemu-timer.h"
-#ifdef CONFIG_KVM
-#include <linux/kvm.h>
-#endif
-
-#if !defined (CONFIG_USER_ONLY)
-#include "sysemu.h"
-#endif
-
-/*****************************************************************************/
-/* Softmmu support */
-#if !defined (CONFIG_USER_ONLY)
-#include "softmmu_exec.h"
-
-#define MMUSUFFIX _mmu
-
-#define SHIFT 0
-#include "softmmu_template.h"
-
-#define SHIFT 1
-#include "softmmu_template.h"
-
-#define SHIFT 2
-#include "softmmu_template.h"
-
-#define SHIFT 3
-#include "softmmu_template.h"
-
-/* try to fill the TLB and return an exception if error. If retaddr is
-   NULL, it means that the function was called in C code (i.e. not
-   from generated code or from helper.c) */
-/* XXX: fix it to restore all registers */
-void tlb_fill(CPUS390XState *env1, target_ulong addr, int is_write, int mmu_idx,
-              uintptr_t retaddr)
-{
-    TranslationBlock *tb;
-    CPUS390XState *saved_env;
-    int ret;
-
-    saved_env = env;
-    env = env1;
-    ret = cpu_s390x_handle_mmu_fault(env, addr, is_write, mmu_idx);
-    if (unlikely(ret != 0)) {
-        if (likely(retaddr)) {
-            /* now we have a real cpu fault */
-            tb = tb_find_pc(retaddr);
-            if (likely(tb)) {
-                /* the PC is inside the translated code. It means that we have
-                   a virtual CPU fault */
-                cpu_restore_state(tb, env, retaddr);
-            }
-        }
-        cpu_loop_exit(env);
-    }
-    env = saved_env;
-}
-
-#endif
-
-/* #define DEBUG_HELPER */
-#ifdef DEBUG_HELPER
-#define HELPER_LOG(x...) qemu_log(x)
-#else
-#define HELPER_LOG(x...)
-#endif
-
-/* raise an exception */
-void HELPER(exception)(uint32_t excp)
-{
-    HELPER_LOG("%s: exception %d\n", __FUNCTION__, excp);
-    env->exception_index = excp;
-    cpu_loop_exit(env);
-}
-
-#ifndef CONFIG_USER_ONLY
-static void mvc_fast_memset(CPUS390XState *env, uint32_t l, uint64_t dest,
-                            uint8_t byte)
-{
-    target_phys_addr_t dest_phys;
-    target_phys_addr_t len = l;
-    void *dest_p;
-    uint64_t asc = env->psw.mask & PSW_MASK_ASC;
-    int flags;
-
-    if (mmu_translate(env, dest, 1, asc, &dest_phys, &flags)) {
-        stb(dest, byte);
-        cpu_abort(env, "should never reach here");
-    }
-    dest_phys |= dest & ~TARGET_PAGE_MASK;
-
-    dest_p = cpu_physical_memory_map(dest_phys, &len, 1);
-
-    memset(dest_p, byte, len);
-
-    cpu_physical_memory_unmap(dest_p, 1, len, len);
-}
-
-static void mvc_fast_memmove(CPUS390XState *env, uint32_t l, uint64_t dest,
-                             uint64_t src)
-{
-    target_phys_addr_t dest_phys;
-    target_phys_addr_t src_phys;
-    target_phys_addr_t len = l;
-    void *dest_p;
-    void *src_p;
-    uint64_t asc = env->psw.mask & PSW_MASK_ASC;
-    int flags;
-
-    if (mmu_translate(env, dest, 1, asc, &dest_phys, &flags)) {
-        stb(dest, 0);
-        cpu_abort(env, "should never reach here");
-    }
-    dest_phys |= dest & ~TARGET_PAGE_MASK;
-
-    if (mmu_translate(env, src, 0, asc, &src_phys, &flags)) {
-        ldub(src);
-        cpu_abort(env, "should never reach here");
-    }
-    src_phys |= src & ~TARGET_PAGE_MASK;
-
-    dest_p = cpu_physical_memory_map(dest_phys, &len, 1);
-    src_p = cpu_physical_memory_map(src_phys, &len, 0);
-
-    memmove(dest_p, src_p, len);
-
-    cpu_physical_memory_unmap(dest_p, 1, len, len);
-    cpu_physical_memory_unmap(src_p, 0, len, len);
-}
-#endif
-
-/* and on array */
-uint32_t HELPER(nc)(uint32_t l, uint64_t dest, uint64_t src)
-{
-    int i;
-    unsigned char x;
-    uint32_t cc = 0;
-
-    HELPER_LOG("%s l %d dest %" PRIx64 " src %" PRIx64 "\n",
-               __FUNCTION__, l, dest, src);
-    for (i = 0; i <= l; i++) {
-        x = ldub(dest + i) & ldub(src + i);
-        if (x) {
-            cc = 1;
-        }
-        stb(dest + i, x);
-    }
-    return cc;
-}
-
-/* xor on array */
-uint32_t HELPER(xc)(uint32_t l, uint64_t dest, uint64_t src)
-{
-    int i;
-    unsigned char x;
-    uint32_t cc = 0;
-
-    HELPER_LOG("%s l %d dest %" PRIx64 " src %" PRIx64 "\n",
-               __FUNCTION__, l, dest, src);
-
-#ifndef CONFIG_USER_ONLY
-    /* xor with itself is the same as memset(0) */
-    if ((l > 32) && (src == dest) &&
-        (src & TARGET_PAGE_MASK) == ((src + l) & TARGET_PAGE_MASK)) {
-        mvc_fast_memset(env, l + 1, dest, 0);
-        return 0;
-    }
-#else
-    if (src == dest) {
-        memset(g2h(dest), 0, l + 1);
-        return 0;
-    }
-#endif
-
-    for (i = 0; i <= l; i++) {
-        x = ldub(dest + i) ^ ldub(src + i);
-        if (x) {
-            cc = 1;
-        }
-        stb(dest + i, x);
-    }
-    return cc;
-}
-
-/* or on array */
-uint32_t HELPER(oc)(uint32_t l, uint64_t dest, uint64_t src)
-{
-    int i;
-    unsigned char x;
-    uint32_t cc = 0;
-
-    HELPER_LOG("%s l %d dest %" PRIx64 " src %" PRIx64 "\n",
-               __FUNCTION__, l, dest, src);
-    for (i = 0; i <= l; i++) {
-        x = ldub(dest + i) | ldub(src + i);
-        if (x) {
-            cc = 1;
-        }
-        stb(dest + i, x);
-    }
-    return cc;
-}
-
-/* memmove */
-void HELPER(mvc)(uint32_t l, uint64_t dest, uint64_t src)
-{
-    int i = 0;
-    int x = 0;
-    uint32_t l_64 = (l + 1) / 8;
-
-    HELPER_LOG("%s l %d dest %" PRIx64 " src %" PRIx64 "\n",
-               __FUNCTION__, l, dest, src);
-
-#ifndef CONFIG_USER_ONLY
-    if ((l > 32) &&
-        (src & TARGET_PAGE_MASK) == ((src + l) & TARGET_PAGE_MASK) &&
-        (dest & TARGET_PAGE_MASK) == ((dest + l) & TARGET_PAGE_MASK)) {
-        if (dest == (src + 1)) {
-            mvc_fast_memset(env, l + 1, dest, ldub(src));
-            return;
-        } else if ((src & TARGET_PAGE_MASK) != (dest & TARGET_PAGE_MASK)) {
-            mvc_fast_memmove(env, l + 1, dest, src);
-            return;
-        }
-    }
-#else
-    if (dest == (src + 1)) {
-        memset(g2h(dest), ldub(src), l + 1);
-        return;
-    } else {
-        memmove(g2h(dest), g2h(src), l + 1);
-        return;
-    }
-#endif
-
-    /* handle the parts that fit into 8-byte loads/stores */
-    if (dest != (src + 1)) {
-        for (i = 0; i < l_64; i++) {
-            stq(dest + x, ldq(src + x));
-            x += 8;
-        }
-    }
-
-    /* slow version crossing pages with byte accesses */
-    for (i = x; i <= l; i++) {
-        stb(dest + i, ldub(src + i));
-    }
-}
-
-/* compare unsigned byte arrays */
-uint32_t HELPER(clc)(uint32_t l, uint64_t s1, uint64_t s2)
-{
-    int i;
-    unsigned char x,y;
-    uint32_t cc;
-    HELPER_LOG("%s l %d s1 %" PRIx64 " s2 %" PRIx64 "\n",
-               __FUNCTION__, l, s1, s2);
-    for (i = 0; i <= l; i++) {
-        x = ldub(s1 + i);
-        y = ldub(s2 + i);
-        HELPER_LOG("%02x (%c)/%02x (%c) ", x, x, y, y);
-        if (x < y) {
-            cc = 1;
-            goto done;
-        } else if (x > y) {
-            cc = 2;
-            goto done;
-        }
-    }
-    cc = 0;
-done:
-    HELPER_LOG("\n");
-    return cc;
-}
-
-/* compare logical under mask */
-uint32_t HELPER(clm)(uint32_t r1, uint32_t mask, uint64_t addr)
-{
-    uint8_t r,d;
-    uint32_t cc;
-    HELPER_LOG("%s: r1 0x%x mask 0x%x addr 0x%" PRIx64 "\n", __FUNCTION__, r1,
-               mask, addr);
-    cc = 0;
-    while (mask) {
-        if (mask & 8) {
-            d = ldub(addr);
-            r = (r1 & 0xff000000UL) >> 24;
-            HELPER_LOG("mask 0x%x %02x/%02x (0x%" PRIx64 ") ", mask, r, d,
-                        addr);
-            if (r < d) {
-                cc = 1;
-                break;
-            } else if (r > d) {
-                cc = 2;
-                break;
-            }
-            addr++;
-        }
-        mask = (mask << 1) & 0xf;
-        r1 <<= 8;
-    }
-    HELPER_LOG("\n");
-    return cc;
-}
-
-/* store character under mask */
-void HELPER(stcm)(uint32_t r1, uint32_t mask, uint64_t addr)
-{
-    uint8_t r;
-    HELPER_LOG("%s: r1 0x%x mask 0x%x addr 0x%lx\n", __FUNCTION__, r1, mask,
-               addr);
-    while (mask) {
-        if (mask & 8) {
-            r = (r1 & 0xff000000UL) >> 24;
-            stb(addr, r);
-            HELPER_LOG("mask 0x%x %02x (0x%lx) ", mask, r, addr);
-            addr++;
-        }
-        mask = (mask << 1) & 0xf;
-        r1 <<= 8;
-    }
-    HELPER_LOG("\n");
-}
-
-/* 64/64 -> 128 unsigned multiplication */
-void HELPER(mlg)(uint32_t r1, uint64_t v2)
-{
-#if HOST_LONG_BITS == 64 && defined(__GNUC__)
-    /* assuming 64-bit hosts have __uint128_t */
-    __uint128_t res = (__uint128_t)env->regs[r1 + 1];
-    res *= (__uint128_t)v2;
-    env->regs[r1] = (uint64_t)(res >> 64);
-    env->regs[r1 + 1] = (uint64_t)res;
-#else
-    mulu64(&env->regs[r1 + 1], &env->regs[r1], env->regs[r1 + 1], v2);
-#endif
-}
-
-/* 128 -> 64/64 unsigned division */
-void HELPER(dlg)(uint32_t r1, uint64_t v2)
-{
-    uint64_t divisor = v2;
-
-    if (!env->regs[r1]) {
-        /* 64 -> 64/64 case */
-        env->regs[r1] = env->regs[r1+1] % divisor;
-        env->regs[r1+1] = env->regs[r1+1] / divisor;
-        return;
-    } else {
-
-#if HOST_LONG_BITS == 64 && defined(__GNUC__)
-        /* assuming 64-bit hosts have __uint128_t */
-        __uint128_t dividend = (((__uint128_t)env->regs[r1]) << 64) |
-                               (env->regs[r1+1]);
-        __uint128_t quotient = dividend / divisor;
-        env->regs[r1+1] = quotient;
-        __uint128_t remainder = dividend % divisor;
-        env->regs[r1] = remainder;
-#else
-        /* 32-bit hosts would need special wrapper functionality - just abort if
-           we encounter such a case; it's very unlikely anyways. */
-        cpu_abort(env, "128 -> 64/64 division not implemented\n");
-#endif
-    }
-}
-
-static inline uint64_t get_address(int x2, int b2, int d2)
-{
-    uint64_t r = d2;
-
-    if (x2) {
-        r += env->regs[x2];
-    }
-
-    if (b2) {
-        r += env->regs[b2];
-    }
-
-    /* 31-Bit mode */
-    if (!(env->psw.mask & PSW_MASK_64)) {
-        r &= 0x7fffffff;
-    }
-
-    return r;
-}
-
-static inline uint64_t get_address_31fix(int reg)
-{
-    uint64_t r = env->regs[reg];
-
-    /* 31-Bit mode */
-    if (!(env->psw.mask & PSW_MASK_64)) {
-        r &= 0x7fffffff;
-    }
-
-    return r;
-}
-
-/* search string (c is byte to search, r2 is string, r1 end of string) */
-uint32_t HELPER(srst)(uint32_t c, uint32_t r1, uint32_t r2)
-{
-    uint64_t i;
-    uint32_t cc = 2;
-    uint64_t str = get_address_31fix(r2);
-    uint64_t end = get_address_31fix(r1);
-
-    HELPER_LOG("%s: c %d *r1 0x%" PRIx64 " *r2 0x%" PRIx64 "\n", __FUNCTION__,
-               c, env->regs[r1], env->regs[r2]);
-
-    for (i = str; i != end; i++) {
-        if (ldub(i) == c) {
-            env->regs[r1] = i;
-            cc = 1;
-            break;
-        }
-    }
-
-    return cc;
-}
-
-/* unsigned string compare (c is string terminator) */
-uint32_t HELPER(clst)(uint32_t c, uint32_t r1, uint32_t r2)
-{
-    uint64_t s1 = get_address_31fix(r1);
-    uint64_t s2 = get_address_31fix(r2);
-    uint8_t v1, v2;
-    uint32_t cc;
-    c = c & 0xff;
-#ifdef CONFIG_USER_ONLY
-    if (!c) {
-        HELPER_LOG("%s: comparing '%s' and '%s'\n",
-                   __FUNCTION__, (char*)g2h(s1), (char*)g2h(s2));
-    }
-#endif
-    for (;;) {
-        v1 = ldub(s1);
-        v2 = ldub(s2);
-        if ((v1 == c || v2 == c) || (v1 != v2)) {
-            break;
-        }
-        s1++;
-        s2++;
-    }
-
-    if (v1 == v2) {
-        cc = 0;
-    } else {
-        cc = (v1 < v2) ? 1 : 2;
-        /* FIXME: 31-bit mode! */
-        env->regs[r1] = s1;
-        env->regs[r2] = s2;
-    }
-    return cc;
-}
-
-/* move page */
-void HELPER(mvpg)(uint64_t r0, uint64_t r1, uint64_t r2)
-{
-    /* XXX missing r0 handling */
-#ifdef CONFIG_USER_ONLY
-    int i;
-
-    for (i = 0; i < TARGET_PAGE_SIZE; i++) {
-        stb(r1 + i, ldub(r2 + i));
-    }
-#else
-    mvc_fast_memmove(env, TARGET_PAGE_SIZE, r1, r2);
-#endif
-}
-
-/* string copy (c is string terminator) */
-void HELPER(mvst)(uint32_t c, uint32_t r1, uint32_t r2)
-{
-    uint64_t dest = get_address_31fix(r1);
-    uint64_t src = get_address_31fix(r2);
-    uint8_t v;
-    c = c & 0xff;
-#ifdef CONFIG_USER_ONLY
-    if (!c) {
-        HELPER_LOG("%s: copy '%s' to 0x%lx\n", __FUNCTION__, (char*)g2h(src),
-                   dest);
-    }
-#endif
-    for (;;) {
-        v = ldub(src);
-        stb(dest, v);
-        if (v == c) {
-            break;
-        }
-        src++;
-        dest++;
-    }
-    env->regs[r1] = dest; /* FIXME: 31-bit mode! */
-}
-
-/* compare and swap 64-bit */
-uint32_t HELPER(csg)(uint32_t r1, uint64_t a2, uint32_t r3)
-{
-    /* FIXME: locking? */
-    uint32_t cc;
-    uint64_t v2 = ldq(a2);
-    if (env->regs[r1] == v2) {
-        cc = 0;
-        stq(a2, env->regs[r3]);
-    } else {
-        cc = 1;
-        env->regs[r1] = v2;
-    }
-    return cc;
-}
-
-/* compare double and swap 64-bit */
-uint32_t HELPER(cdsg)(uint32_t r1, uint64_t a2, uint32_t r3)
-{
-    /* FIXME: locking? */
-    uint32_t cc;
-    uint64_t v2_hi = ldq(a2);
-    uint64_t v2_lo = ldq(a2 + 8);
-    uint64_t v1_hi = env->regs[r1];
-    uint64_t v1_lo = env->regs[r1 + 1];
-
-    if ((v1_hi == v2_hi) && (v1_lo == v2_lo)) {
-        cc = 0;
-        stq(a2, env->regs[r3]);
-        stq(a2 + 8, env->regs[r3 + 1]);
-    } else {
-        cc = 1;
-        env->regs[r1] = v2_hi;
-        env->regs[r1 + 1] = v2_lo;
-    }
-
-    return cc;
-}
-
-/* compare and swap 32-bit */
-uint32_t HELPER(cs)(uint32_t r1, uint64_t a2, uint32_t r3)
-{
-    /* FIXME: locking? */
-    uint32_t cc;
-    HELPER_LOG("%s: r1 %d a2 0x%lx r3 %d\n", __FUNCTION__, r1, a2, r3);
-    uint32_t v2 = ldl(a2);
-    if (((uint32_t)env->regs[r1]) == v2) {
-        cc = 0;
-        stl(a2, (uint32_t)env->regs[r3]);
-    } else {
-        cc = 1;
-        env->regs[r1] = (env->regs[r1] & 0xffffffff00000000ULL) | v2;
-    }
-    return cc;
-}
-
-static uint32_t helper_icm(uint32_t r1, uint64_t address, uint32_t mask)
-{
-    int pos = 24; /* top of the lower half of r1 */
-    uint64_t rmask = 0xff000000ULL;
-    uint8_t val = 0;
-    int ccd = 0;
-    uint32_t cc = 0;
-
-    while (mask) {
-        if (mask & 8) {
-            env->regs[r1] &= ~rmask;
-            val = ldub(address);
-            if ((val & 0x80) && !ccd) {
-                cc = 1;
-            }
-            ccd = 1;
-            if (val && cc == 0) {
-                cc = 2;
-            }
-            env->regs[r1] |= (uint64_t)val << pos;
-            address++;
-        }
-        mask = (mask << 1) & 0xf;
-        pos -= 8;
-        rmask >>= 8;
-    }
-
-    return cc;
-}
-
-/* execute instruction
-   this instruction executes an insn modified with the contents of r1
-   it does not change the executed instruction in memory
-   it does not change the program counter
-   in other words: tricky...
-   currently implemented by interpreting the cases it is most commonly used in
- */
-uint32_t HELPER(ex)(uint32_t cc, uint64_t v1, uint64_t addr, uint64_t ret)
-{
-    uint16_t insn = lduw_code(addr);
-    HELPER_LOG("%s: v1 0x%lx addr 0x%lx insn 0x%x\n", __FUNCTION__, v1, addr,
-             insn);
-    if ((insn & 0xf0ff) == 0xd000) {
-        uint32_t l, insn2, b1, b2, d1, d2;
-        l = v1 & 0xff;
-        insn2 = ldl_code(addr + 2);
-        b1 = (insn2 >> 28) & 0xf;
-        b2 = (insn2 >> 12) & 0xf;
-        d1 = (insn2 >> 16) & 0xfff;
-        d2 = insn2 & 0xfff;
-        switch (insn & 0xf00) {
-        case 0x200:
-            helper_mvc(l, get_address(0, b1, d1), get_address(0, b2, d2));
-            break;
-        case 0x500:
-            cc = helper_clc(l, get_address(0, b1, d1), get_address(0, b2, d2));
-            break;
-        case 0x700:
-            cc = helper_xc(l, get_address(0, b1, d1), get_address(0, b2, d2));
-            break;
-        case 0xc00:
-            helper_tr(l, get_address(0, b1, d1), get_address(0, b2, d2));
-            break;
-        default:
-            goto abort;
-            break;
-        }
-    } else if ((insn & 0xff00) == 0x0a00) {
-        /* supervisor call */
-        HELPER_LOG("%s: svc %ld via execute\n", __FUNCTION__, (insn|v1) & 0xff);
-        env->psw.addr = ret - 4;
-        env->int_svc_code = (insn|v1) & 0xff;
-        env->int_svc_ilc = 4;
-        helper_exception(EXCP_SVC);
-    } else if ((insn & 0xff00) == 0xbf00) {
-        uint32_t insn2, r1, r3, b2, d2;
-        insn2 = ldl_code(addr + 2);
-        r1 = (insn2 >> 20) & 0xf;
-        r3 = (insn2 >> 16) & 0xf;
-        b2 = (insn2 >> 12) & 0xf;
-        d2 = insn2 & 0xfff;
-        cc = helper_icm(r1, get_address(0, b2, d2), r3);
-    } else {
-abort:
-        cpu_abort(env, "EXECUTE on instruction prefix 0x%x not implemented\n",
-                  insn);
-    }
-    return cc;
-}
-
-/* absolute value 32-bit */
-uint32_t HELPER(abs_i32)(int32_t val)
-{
-    if (val < 0) {
-        return -val;
-    } else {
-        return val;
-    }
-}
-
-/* negative absolute value 32-bit */
-int32_t HELPER(nabs_i32)(int32_t val)
-{
-    if (val < 0) {
-        return val;
-    } else {
-        return -val;
-    }
-}
-
-/* absolute value 64-bit */
-uint64_t HELPER(abs_i64)(int64_t val)
-{
-    HELPER_LOG("%s: val 0x%" PRIx64 "\n", __FUNCTION__, val);
-
-    if (val < 0) {
-        return -val;
-    } else {
-        return val;
-    }
-}
-
-/* negative absolute value 64-bit */
-int64_t HELPER(nabs_i64)(int64_t val)
-{
-    if (val < 0) {
-        return val;
-    } else {
-        return -val;
-    }
-}
-
-/* add with carry 32-bit unsigned */
-uint32_t HELPER(addc_u32)(uint32_t cc, uint32_t v1, uint32_t v2)
-{
-    uint32_t res;
-
-    res = v1 + v2;
-    if (cc & 2) {
-        res++;
-    }
-
-    return res;
-}
-
-/* store character under mask high operates on the upper half of r1 */
-void HELPER(stcmh)(uint32_t r1, uint64_t address, uint32_t mask)
-{
-    int pos = 56; /* top of the upper half of r1 */
-
-    while (mask) {
-        if (mask & 8) {
-            stb(address, (env->regs[r1] >> pos) & 0xff);
-            address++;
-        }
-        mask = (mask << 1) & 0xf;
-        pos -= 8;
-    }
-}
-
-/* insert character under mask high; same as icm, but operates on the
-   upper half of r1 */
-uint32_t HELPER(icmh)(uint32_t r1, uint64_t address, uint32_t mask)
-{
-    int pos = 56; /* top of the upper half of r1 */
-    uint64_t rmask = 0xff00000000000000ULL;
-    uint8_t val = 0;
-    int ccd = 0;
-    uint32_t cc = 0;
-
-    while (mask) {
-        if (mask & 8) {
-            env->regs[r1] &= ~rmask;
-            val = ldub(address);
-            if ((val & 0x80) && !ccd) {
-                cc = 1;
-            }
-            ccd = 1;
-            if (val && cc == 0) {
-                cc = 2;
-            }
-            env->regs[r1] |= (uint64_t)val << pos;
-            address++;
-        }
-        mask = (mask << 1) & 0xf;
-        pos -= 8;
-        rmask >>= 8;
-    }
-
-    return cc;
-}
-
-/* insert psw mask and condition code into r1 */
-void HELPER(ipm)(uint32_t cc, uint32_t r1)
-{
-    uint64_t r = env->regs[r1];
-
-    r &= 0xffffffff00ffffffULL;
-    r |= (cc << 28) | ( (env->psw.mask >> 40) & 0xf );
-    env->regs[r1] = r;
-    HELPER_LOG("%s: cc %d psw.mask 0x%lx r1 0x%lx\n", __FUNCTION__,
-               cc, env->psw.mask, r);
-}
-
-/* load access registers r1 to r3 from memory at a2 */
-void HELPER(lam)(uint32_t r1, uint64_t a2, uint32_t r3)
-{
-    int i;
-
-    for (i = r1;; i = (i + 1) % 16) {
-        env->aregs[i] = ldl(a2);
-        a2 += 4;
-
-        if (i == r3) {
-            break;
-        }
-    }
-}
-
-/* store access registers r1 to r3 in memory at a2 */
-void HELPER(stam)(uint32_t r1, uint64_t a2, uint32_t r3)
-{
-    int i;
-
-    for (i = r1;; i = (i + 1) % 16) {
-        stl(a2, env->aregs[i]);
-        a2 += 4;
-
-        if (i == r3) {
-            break;
-        }
-    }
-}
-
-/* move long */
-uint32_t HELPER(mvcl)(uint32_t r1, uint32_t r2)
-{
-    uint64_t destlen = env->regs[r1 + 1] & 0xffffff;
-    uint64_t dest = get_address_31fix(r1);
-    uint64_t srclen = env->regs[r2 + 1] & 0xffffff;
-    uint64_t src = get_address_31fix(r2);
-    uint8_t pad = src >> 24;
-    uint8_t v;
-    uint32_t cc;
-
-    if (destlen == srclen) {
-        cc = 0;
-    } else if (destlen < srclen) {
-        cc = 1;
-    } else {
-        cc = 2;
-    }
-
-    if (srclen > destlen) {
-        srclen = destlen;
-    }
-
-    for (; destlen && srclen; src++, dest++, destlen--, srclen--) {
-        v = ldub(src);
-        stb(dest, v);
-    }
-
-    for (; destlen; dest++, destlen--) {
-        stb(dest, pad);
-    }
-
-    env->regs[r1 + 1] = destlen;
-    /* can't use srclen here, we trunc'ed it */
-    env->regs[r2 + 1] -= src - env->regs[r2];
-    env->regs[r1] = dest;
-    env->regs[r2] = src;
-
-    return cc;
-}
-
-/* move long extended another memcopy insn with more bells and whistles */
-uint32_t HELPER(mvcle)(uint32_t r1, uint64_t a2, uint32_t r3)
-{
-    uint64_t destlen = env->regs[r1 + 1];
-    uint64_t dest = env->regs[r1];
-    uint64_t srclen = env->regs[r3 + 1];
-    uint64_t src = env->regs[r3];
-    uint8_t pad = a2 & 0xff;
-    uint8_t v;
-    uint32_t cc;
-
-    if (!(env->psw.mask & PSW_MASK_64)) {
-        destlen = (uint32_t)destlen;
-        srclen = (uint32_t)srclen;
-        dest &= 0x7fffffff;
-        src &= 0x7fffffff;
-    }
-
-    if (destlen == srclen) {
-        cc = 0;
-    } else if (destlen < srclen) {
-        cc = 1;
-    } else {
-        cc = 2;
-    }
-
-    if (srclen > destlen) {
-        srclen = destlen;
-    }
-
-    for (; destlen && srclen; src++, dest++, destlen--, srclen--) {
-        v = ldub(src);
-        stb(dest, v);
-    }
-
-    for (; destlen; dest++, destlen--) {
-        stb(dest, pad);
-    }
-
-    env->regs[r1 + 1] = destlen;
-    /* can't use srclen here, we trunc'ed it */
-    /* FIXME: 31-bit mode! */
-    env->regs[r3 + 1] -= src - env->regs[r3];
-    env->regs[r1] = dest;
-    env->regs[r3] = src;
-
-    return cc;
-}
-
-/* compare logical long extended memcompare insn with padding */
-uint32_t HELPER(clcle)(uint32_t r1, uint64_t a2, uint32_t r3)
-{
-    uint64_t destlen = env->regs[r1 + 1];
-    uint64_t dest = get_address_31fix(r1);
-    uint64_t srclen = env->regs[r3 + 1];
-    uint64_t src = get_address_31fix(r3);
-    uint8_t pad = a2 & 0xff;
-    uint8_t v1 = 0,v2 = 0;
-    uint32_t cc = 0;
-
-    if (!(destlen || srclen)) {
-        return cc;
-    }
-
-    if (srclen > destlen) {
-        srclen = destlen;
-    }
-
-    for (; destlen || srclen; src++, dest++, destlen--, srclen--) {
-        v1 = srclen ? ldub(src) : pad;
-        v2 = destlen ? ldub(dest) : pad;
-        if (v1 != v2) {
-            cc = (v1 < v2) ? 1 : 2;
-            break;
-        }
-    }
-
-    env->regs[r1 + 1] = destlen;
-    /* can't use srclen here, we trunc'ed it */
-    env->regs[r3 + 1] -= src - env->regs[r3];
-    env->regs[r1] = dest;
-    env->regs[r3] = src;
-
-    return cc;
-}
-
-/* subtract unsigned v2 from v1 with borrow */
-uint32_t HELPER(slb)(uint32_t cc, uint32_t r1, uint32_t v2)
-{
-    uint32_t v1 = env->regs[r1];
-    uint32_t res = v1 + (~v2) + (cc >> 1);
-
-    env->regs[r1] = (env->regs[r1] & 0xffffffff00000000ULL) | res;
-    if (cc & 2) {
-        /* borrow */
-        return v1 ? 1 : 0;
-    } else {
-        return v1 ? 3 : 2;
-    }
-}
-
-/* subtract unsigned v2 from v1 with borrow */
-uint32_t HELPER(slbg)(uint32_t cc, uint32_t r1, uint64_t v1, uint64_t v2)
-{
-    uint64_t res = v1 + (~v2) + (cc >> 1);
-
-    env->regs[r1] = res;
-    if (cc & 2) {
-        /* borrow */
-        return v1 ? 1 : 0;
-    } else {
-        return v1 ? 3 : 2;
-    }
-}
-
-static inline int float_comp_to_cc(int float_compare)
-{
-    switch (float_compare) {
-    case float_relation_equal:
-        return 0;
-    case float_relation_less:
-        return 1;
-    case float_relation_greater:
-        return 2;
-    case float_relation_unordered:
-        return 3;
-    default:
-        cpu_abort(env, "unknown return value for float compare\n");
-    }
-}
-
-/* condition codes for binary FP ops */
-static uint32_t set_cc_f32(float32 v1, float32 v2)
-{
-    return float_comp_to_cc(float32_compare_quiet(v1, v2, &env->fpu_status));
-}
-
-static uint32_t set_cc_f64(float64 v1, float64 v2)
-{
-    return float_comp_to_cc(float64_compare_quiet(v1, v2, &env->fpu_status));
-}
-
-/* condition codes for unary FP ops */
-static uint32_t set_cc_nz_f32(float32 v)
-{
-    if (float32_is_any_nan(v)) {
-        return 3;
-    } else if (float32_is_zero(v)) {
-        return 0;
-    } else if (float32_is_neg(v)) {
-        return 1;
-    } else {
-        return 2;
-    }
-}
-
-static uint32_t set_cc_nz_f64(float64 v)
-{
-    if (float64_is_any_nan(v)) {
-        return 3;
-    } else if (float64_is_zero(v)) {
-        return 0;
-    } else if (float64_is_neg(v)) {
-        return 1;
-    } else {
-        return 2;
-    }
-}
-
-static uint32_t set_cc_nz_f128(float128 v)
-{
-    if (float128_is_any_nan(v)) {
-        return 3;
-    } else if (float128_is_zero(v)) {
-        return 0;
-    } else if (float128_is_neg(v)) {
-        return 1;
-    } else {
-        return 2;
-    }
-}
-
-/* convert 32-bit int to 64-bit float */
-void HELPER(cdfbr)(uint32_t f1, int32_t v2)
-{
-    HELPER_LOG("%s: converting %d to f%d\n", __FUNCTION__, v2, f1);
-    env->fregs[f1].d = int32_to_float64(v2, &env->fpu_status);
-}
-
-/* convert 32-bit int to 128-bit float */
-void HELPER(cxfbr)(uint32_t f1, int32_t v2)
-{
-    CPU_QuadU v1;
-    v1.q = int32_to_float128(v2, &env->fpu_status);
-    env->fregs[f1].ll = v1.ll.upper;
-    env->fregs[f1 + 2].ll = v1.ll.lower;
-}
-
-/* convert 64-bit int to 32-bit float */
-void HELPER(cegbr)(uint32_t f1, int64_t v2)
-{
-    HELPER_LOG("%s: converting %ld to f%d\n", __FUNCTION__, v2, f1);
-    env->fregs[f1].l.upper = int64_to_float32(v2, &env->fpu_status);
-}
-
-/* convert 64-bit int to 64-bit float */
-void HELPER(cdgbr)(uint32_t f1, int64_t v2)
-{
-    HELPER_LOG("%s: converting %ld to f%d\n", __FUNCTION__, v2, f1);
-    env->fregs[f1].d = int64_to_float64(v2, &env->fpu_status);
-}
-
-/* convert 64-bit int to 128-bit float */
-void HELPER(cxgbr)(uint32_t f1, int64_t v2)
-{
-    CPU_QuadU x1;
-    x1.q = int64_to_float128(v2, &env->fpu_status);
-    HELPER_LOG("%s: converted %ld to 0x%lx and 0x%lx\n", __FUNCTION__, v2,
-               x1.ll.upper, x1.ll.lower);
-    env->fregs[f1].ll = x1.ll.upper;
-    env->fregs[f1 + 2].ll = x1.ll.lower;
-}
-
-/* convert 32-bit int to 32-bit float */
-void HELPER(cefbr)(uint32_t f1, int32_t v2)
-{
-    env->fregs[f1].l.upper = int32_to_float32(v2, &env->fpu_status);
-    HELPER_LOG("%s: converting %d to 0x%d in f%d\n", __FUNCTION__, v2,
-               env->fregs[f1].l.upper, f1);
-}
-
-/* 32-bit FP addition RR */
-uint32_t HELPER(aebr)(uint32_t f1, uint32_t f2)
-{
-    env->fregs[f1].l.upper = float32_add(env->fregs[f1].l.upper,
-                                         env->fregs[f2].l.upper,
-                                         &env->fpu_status);
-    HELPER_LOG("%s: adding 0x%d resulting in 0x%d in f%d\n", __FUNCTION__,
-               env->fregs[f2].l.upper, env->fregs[f1].l.upper, f1);
-
-    return set_cc_nz_f32(env->fregs[f1].l.upper);
-}
-
-/* 64-bit FP addition RR */
-uint32_t HELPER(adbr)(uint32_t f1, uint32_t f2)
-{
-    env->fregs[f1].d = float64_add(env->fregs[f1].d, env->fregs[f2].d,
-                                   &env->fpu_status);
-    HELPER_LOG("%s: adding 0x%ld resulting in 0x%ld in f%d\n", __FUNCTION__,
-               env->fregs[f2].d, env->fregs[f1].d, f1);
-
-    return set_cc_nz_f64(env->fregs[f1].d);
-}
-
-/* 32-bit FP subtraction RR */
-uint32_t HELPER(sebr)(uint32_t f1, uint32_t f2)
-{
-    env->fregs[f1].l.upper = float32_sub(env->fregs[f1].l.upper,
-                                         env->fregs[f2].l.upper,
-                                         &env->fpu_status);
-    HELPER_LOG("%s: adding 0x%d resulting in 0x%d in f%d\n", __FUNCTION__,
-               env->fregs[f2].l.upper, env->fregs[f1].l.upper, f1);
-
-    return set_cc_nz_f32(env->fregs[f1].l.upper);
-}
-
-/* 64-bit FP subtraction RR */
-uint32_t HELPER(sdbr)(uint32_t f1, uint32_t f2)
-{
-    env->fregs[f1].d = float64_sub(env->fregs[f1].d, env->fregs[f2].d,
-                                   &env->fpu_status);
-    HELPER_LOG("%s: subtracting 0x%ld resulting in 0x%ld in f%d\n",
-               __FUNCTION__, env->fregs[f2].d, env->fregs[f1].d, f1);
-
-    return set_cc_nz_f64(env->fregs[f1].d);
-}
-
-/* 32-bit FP division RR */
-void HELPER(debr)(uint32_t f1, uint32_t f2)
-{
-    env->fregs[f1].l.upper = float32_div(env->fregs[f1].l.upper,
-                                         env->fregs[f2].l.upper,
-                                         &env->fpu_status);
-}
-
-/* 128-bit FP division RR */
-void HELPER(dxbr)(uint32_t f1, uint32_t f2)
-{
-    CPU_QuadU v1;
-    v1.ll.upper = env->fregs[f1].ll;
-    v1.ll.lower = env->fregs[f1 + 2].ll;
-    CPU_QuadU v2;
-    v2.ll.upper = env->fregs[f2].ll;
-    v2.ll.lower = env->fregs[f2 + 2].ll;
-    CPU_QuadU res;
-    res.q = float128_div(v1.q, v2.q, &env->fpu_status);
-    env->fregs[f1].ll = res.ll.upper;
-    env->fregs[f1 + 2].ll = res.ll.lower;
-}
-
-/* 64-bit FP multiplication RR */
-void HELPER(mdbr)(uint32_t f1, uint32_t f2)
-{
-    env->fregs[f1].d = float64_mul(env->fregs[f1].d, env->fregs[f2].d,
-                                   &env->fpu_status);
-}
-
-/* 128-bit FP multiplication RR */
-void HELPER(mxbr)(uint32_t f1, uint32_t f2)
-{
-    CPU_QuadU v1;
-    v1.ll.upper = env->fregs[f1].ll;
-    v1.ll.lower = env->fregs[f1 + 2].ll;
-    CPU_QuadU v2;
-    v2.ll.upper = env->fregs[f2].ll;
-    v2.ll.lower = env->fregs[f2 + 2].ll;
-    CPU_QuadU res;
-    res.q = float128_mul(v1.q, v2.q, &env->fpu_status);
-    env->fregs[f1].ll = res.ll.upper;
-    env->fregs[f1 + 2].ll = res.ll.lower;
-}
-
-/* convert 32-bit float to 64-bit float */
-void HELPER(ldebr)(uint32_t r1, uint32_t r2)
-{
-    env->fregs[r1].d = float32_to_float64(env->fregs[r2].l.upper,
-                                          &env->fpu_status);
-}
-
-/* convert 128-bit float to 64-bit float */
-void HELPER(ldxbr)(uint32_t f1, uint32_t f2)
-{
-    CPU_QuadU x2;
-    x2.ll.upper = env->fregs[f2].ll;
-    x2.ll.lower = env->fregs[f2 + 2].ll;
-    env->fregs[f1].d = float128_to_float64(x2.q, &env->fpu_status);
-    HELPER_LOG("%s: to 0x%ld\n", __FUNCTION__, env->fregs[f1].d);
-}
-
-/* convert 64-bit float to 128-bit float */
-void HELPER(lxdbr)(uint32_t f1, uint32_t f2)
-{
-    CPU_QuadU res;
-    res.q = float64_to_float128(env->fregs[f2].d, &env->fpu_status);
-    env->fregs[f1].ll = res.ll.upper;
-    env->fregs[f1 + 2].ll = res.ll.lower;
-}
-
-/* convert 64-bit float to 32-bit float */
-void HELPER(ledbr)(uint32_t f1, uint32_t f2)
-{
-    float64 d2 = env->fregs[f2].d;
-    env->fregs[f1].l.upper = float64_to_float32(d2, &env->fpu_status);
-}
-
-/* convert 128-bit float to 32-bit float */
-void HELPER(lexbr)(uint32_t f1, uint32_t f2)
-{
-    CPU_QuadU x2;
-    x2.ll.upper = env->fregs[f2].ll;
-    x2.ll.lower = env->fregs[f2 + 2].ll;
-    env->fregs[f1].l.upper = float128_to_float32(x2.q, &env->fpu_status);
-    HELPER_LOG("%s: to 0x%d\n", __FUNCTION__, env->fregs[f1].l.upper);
-}
-
-/* absolute value of 32-bit float */
-uint32_t HELPER(lpebr)(uint32_t f1, uint32_t f2)
-{
-    float32 v1;
-    float32 v2 = env->fregs[f2].d;
-    v1 = float32_abs(v2);
-    env->fregs[f1].d = v1;
-    return set_cc_nz_f32(v1);
-}
-
-/* absolute value of 64-bit float */
-uint32_t HELPER(lpdbr)(uint32_t f1, uint32_t f2)
-{
-    float64 v1;
-    float64 v2 = env->fregs[f2].d;
-    v1 = float64_abs(v2);
-    env->fregs[f1].d = v1;
-    return set_cc_nz_f64(v1);
-}
-
-/* absolute value of 128-bit float */
-uint32_t HELPER(lpxbr)(uint32_t f1, uint32_t f2)
-{
-    CPU_QuadU v1;
-    CPU_QuadU v2;
-    v2.ll.upper = env->fregs[f2].ll;
-    v2.ll.lower = env->fregs[f2 + 2].ll;
-    v1.q = float128_abs(v2.q);
-    env->fregs[f1].ll = v1.ll.upper;
-    env->fregs[f1 + 2].ll = v1.ll.lower;
-    return set_cc_nz_f128(v1.q);
-}
-
-/* load and test 64-bit float */
-uint32_t HELPER(ltdbr)(uint32_t f1, uint32_t f2)
-{
-    env->fregs[f1].d = env->fregs[f2].d;
-    return set_cc_nz_f64(env->fregs[f1].d);
-}
-
-/* load and test 32-bit float */
-uint32_t HELPER(ltebr)(uint32_t f1, uint32_t f2)
-{
-    env->fregs[f1].l.upper = env->fregs[f2].l.upper;
-    return set_cc_nz_f32(env->fregs[f1].l.upper);
-}
-
-/* load and test 128-bit float */
-uint32_t HELPER(ltxbr)(uint32_t f1, uint32_t f2)
-{
-    CPU_QuadU x;
-    x.ll.upper = env->fregs[f2].ll;
-    x.ll.lower = env->fregs[f2 + 2].ll;
-    env->fregs[f1].ll = x.ll.upper;
-    env->fregs[f1 + 2].ll = x.ll.lower;
-    return set_cc_nz_f128(x.q);
-}
-
-/* load complement of 32-bit float */
-uint32_t HELPER(lcebr)(uint32_t f1, uint32_t f2)
-{
-    env->fregs[f1].l.upper = float32_chs(env->fregs[f2].l.upper);
-
-    return set_cc_nz_f32(env->fregs[f1].l.upper);
-}
-
-/* load complement of 64-bit float */
-uint32_t HELPER(lcdbr)(uint32_t f1, uint32_t f2)
-{
-    env->fregs[f1].d = float64_chs(env->fregs[f2].d);
-
-    return set_cc_nz_f64(env->fregs[f1].d);
-}
-
-/* load complement of 128-bit float */
-uint32_t HELPER(lcxbr)(uint32_t f1, uint32_t f2)
-{
-    CPU_QuadU x1, x2;
-    x2.ll.upper = env->fregs[f2].ll;
-    x2.ll.lower = env->fregs[f2 + 2].ll;
-    x1.q = float128_chs(x2.q);
-    env->fregs[f1].ll = x1.ll.upper;
-    env->fregs[f1 + 2].ll = x1.ll.lower;
-    return set_cc_nz_f128(x1.q);
-}
-
-/* 32-bit FP addition RM */
-void HELPER(aeb)(uint32_t f1, uint32_t val)
-{
-    float32 v1 = env->fregs[f1].l.upper;
-    CPU_FloatU v2;
-    v2.l = val;
-    HELPER_LOG("%s: adding 0x%d from f%d and 0x%d\n", __FUNCTION__,
-               v1, f1, v2.f);
-    env->fregs[f1].l.upper = float32_add(v1, v2.f, &env->fpu_status);
-}
-
-/* 32-bit FP division RM */
-void HELPER(deb)(uint32_t f1, uint32_t val)
-{
-    float32 v1 = env->fregs[f1].l.upper;
-    CPU_FloatU v2;
-    v2.l = val;
-    HELPER_LOG("%s: dividing 0x%d from f%d by 0x%d\n", __FUNCTION__,
-               v1, f1, v2.f);
-    env->fregs[f1].l.upper = float32_div(v1, v2.f, &env->fpu_status);
-}
-
-/* 32-bit FP multiplication RM */
-void HELPER(meeb)(uint32_t f1, uint32_t val)
-{
-    float32 v1 = env->fregs[f1].l.upper;
-    CPU_FloatU v2;
-    v2.l = val;
-    HELPER_LOG("%s: multiplying 0x%d from f%d and 0x%d\n", __FUNCTION__,
-               v1, f1, v2.f);
-    env->fregs[f1].l.upper = float32_mul(v1, v2.f, &env->fpu_status);
-}
-
-/* 32-bit FP compare RR */
-uint32_t HELPER(cebr)(uint32_t f1, uint32_t f2)
-{
-    float32 v1 = env->fregs[f1].l.upper;
-    float32 v2 = env->fregs[f2].l.upper;
-    HELPER_LOG("%s: comparing 0x%d from f%d and 0x%d\n", __FUNCTION__,
-               v1, f1, v2);
-    return set_cc_f32(v1, v2);
-}
-
-/* 64-bit FP compare RR */
-uint32_t HELPER(cdbr)(uint32_t f1, uint32_t f2)
-{
-    float64 v1 = env->fregs[f1].d;
-    float64 v2 = env->fregs[f2].d;
-    HELPER_LOG("%s: comparing 0x%ld from f%d and 0x%ld\n", __FUNCTION__,
-               v1, f1, v2);
-    return set_cc_f64(v1, v2);
-}
-
-/* 128-bit FP compare RR */
-uint32_t HELPER(cxbr)(uint32_t f1, uint32_t f2)
-{
-    CPU_QuadU v1;
-    v1.ll.upper = env->fregs[f1].ll;
-    v1.ll.lower = env->fregs[f1 + 2].ll;
-    CPU_QuadU v2;
-    v2.ll.upper = env->fregs[f2].ll;
-    v2.ll.lower = env->fregs[f2 + 2].ll;
-
-    return float_comp_to_cc(float128_compare_quiet(v1.q, v2.q,
-                            &env->fpu_status));
-}
-
-/* 64-bit FP compare RM */
-uint32_t HELPER(cdb)(uint32_t f1, uint64_t a2)
-{
-    float64 v1 = env->fregs[f1].d;
-    CPU_DoubleU v2;
-    v2.ll = ldq(a2);
-    HELPER_LOG("%s: comparing 0x%ld from f%d and 0x%lx\n", __FUNCTION__, v1,
-               f1, v2.d);
-    return set_cc_f64(v1, v2.d);
-}
-
-/* 64-bit FP addition RM */
-uint32_t HELPER(adb)(uint32_t f1, uint64_t a2)
-{
-    float64 v1 = env->fregs[f1].d;
-    CPU_DoubleU v2;
-    v2.ll = ldq(a2);
-    HELPER_LOG("%s: adding 0x%lx from f%d and 0x%lx\n", __FUNCTION__,
-               v1, f1, v2.d);
-    env->fregs[f1].d = v1 = float64_add(v1, v2.d, &env->fpu_status);
-    return set_cc_nz_f64(v1);
-}
-
-/* 32-bit FP subtraction RM */
-void HELPER(seb)(uint32_t f1, uint32_t val)
-{
-    float32 v1 = env->fregs[f1].l.upper;
-    CPU_FloatU v2;
-    v2.l = val;
-    env->fregs[f1].l.upper = float32_sub(v1, v2.f, &env->fpu_status);
-}
-
-/* 64-bit FP subtraction RM */
-uint32_t HELPER(sdb)(uint32_t f1, uint64_t a2)
-{
-    float64 v1 = env->fregs[f1].d;
-    CPU_DoubleU v2;
-    v2.ll = ldq(a2);
-    env->fregs[f1].d = v1 = float64_sub(v1, v2.d, &env->fpu_status);
-    return set_cc_nz_f64(v1);
-}
-
-/* 64-bit FP multiplication RM */
-void HELPER(mdb)(uint32_t f1, uint64_t a2)
-{
-    float64 v1 = env->fregs[f1].d;
-    CPU_DoubleU v2;
-    v2.ll = ldq(a2);
-    HELPER_LOG("%s: multiplying 0x%lx from f%d and 0x%ld\n", __FUNCTION__,
-               v1, f1, v2.d);
-    env->fregs[f1].d = float64_mul(v1, v2.d, &env->fpu_status);
-}
-
-/* 64-bit FP division RM */
-void HELPER(ddb)(uint32_t f1, uint64_t a2)
-{
-    float64 v1 = env->fregs[f1].d;
-    CPU_DoubleU v2;
-    v2.ll = ldq(a2);
-    HELPER_LOG("%s: dividing 0x%lx from f%d by 0x%ld\n", __FUNCTION__,
-               v1, f1, v2.d);
-    env->fregs[f1].d = float64_div(v1, v2.d, &env->fpu_status);
-}
-
-static void set_round_mode(int m3)
-{
-    switch (m3) {
-    case 0:
-        /* current mode */
-        break;
-    case 1:
-        /* biased round no nearest */
-    case 4:
-        /* round to nearest */
-        set_float_rounding_mode(float_round_nearest_even, &env->fpu_status);
-        break;
-    case 5:
-        /* round to zero */
-        set_float_rounding_mode(float_round_to_zero, &env->fpu_status);
-        break;
-    case 6:
-        /* round to +inf */
-        set_float_rounding_mode(float_round_up, &env->fpu_status);
-        break;
-    case 7:
-        /* round to -inf */
-        set_float_rounding_mode(float_round_down, &env->fpu_status);
-        break;
-    }
-}
-
-/* convert 32-bit float to 64-bit int */
-uint32_t HELPER(cgebr)(uint32_t r1, uint32_t f2, uint32_t m3)
-{
-    float32 v2 = env->fregs[f2].l.upper;
-    set_round_mode(m3);
-    env->regs[r1] = float32_to_int64(v2, &env->fpu_status);
-    return set_cc_nz_f32(v2);
-}
-
-/* convert 64-bit float to 64-bit int */
-uint32_t HELPER(cgdbr)(uint32_t r1, uint32_t f2, uint32_t m3)
-{
-    float64 v2 = env->fregs[f2].d;
-    set_round_mode(m3);
-    env->regs[r1] = float64_to_int64(v2, &env->fpu_status);
-    return set_cc_nz_f64(v2);
-}
-
-/* convert 128-bit float to 64-bit int */
-uint32_t HELPER(cgxbr)(uint32_t r1, uint32_t f2, uint32_t m3)
-{
-    CPU_QuadU v2;
-    v2.ll.upper = env->fregs[f2].ll;
-    v2.ll.lower = env->fregs[f2 + 2].ll;
-    set_round_mode(m3);
-    env->regs[r1] = float128_to_int64(v2.q, &env->fpu_status);
-    if (float128_is_any_nan(v2.q)) {
-        return 3;
-    } else if (float128_is_zero(v2.q)) {
-        return 0;
-    } else if (float128_is_neg(v2.q)) {
-        return 1;
-    } else {
-        return 2;
-    }
-}
-
-/* convert 32-bit float to 32-bit int */
-uint32_t HELPER(cfebr)(uint32_t r1, uint32_t f2, uint32_t m3)
-{
-    float32 v2 = env->fregs[f2].l.upper;
-    set_round_mode(m3);
-    env->regs[r1] = (env->regs[r1] & 0xffffffff00000000ULL) |
-                     float32_to_int32(v2, &env->fpu_status);
-    return set_cc_nz_f32(v2);
-}
-
-/* convert 64-bit float to 32-bit int */
-uint32_t HELPER(cfdbr)(uint32_t r1, uint32_t f2, uint32_t m3)
-{
-    float64 v2 = env->fregs[f2].d;
-    set_round_mode(m3);
-    env->regs[r1] = (env->regs[r1] & 0xffffffff00000000ULL) |
-                     float64_to_int32(v2, &env->fpu_status);
-    return set_cc_nz_f64(v2);
-}
-
-/* convert 128-bit float to 32-bit int */
-uint32_t HELPER(cfxbr)(uint32_t r1, uint32_t f2, uint32_t m3)
-{
-    CPU_QuadU v2;
-    v2.ll.upper = env->fregs[f2].ll;
-    v2.ll.lower = env->fregs[f2 + 2].ll;
-    env->regs[r1] = (env->regs[r1] & 0xffffffff00000000ULL) |
-                     float128_to_int32(v2.q, &env->fpu_status);
-    return set_cc_nz_f128(v2.q);
-}
-
-/* load 32-bit FP zero */
-void HELPER(lzer)(uint32_t f1)
-{
-    env->fregs[f1].l.upper = float32_zero;
-}
-
-/* load 64-bit FP zero */
-void HELPER(lzdr)(uint32_t f1)
-{
-    env->fregs[f1].d = float64_zero;
-}
-
-/* load 128-bit FP zero */
-void HELPER(lzxr)(uint32_t f1)
-{
-    CPU_QuadU x;
-    x.q = float64_to_float128(float64_zero, &env->fpu_status);
-    env->fregs[f1].ll = x.ll.upper;
-    env->fregs[f1 + 1].ll = x.ll.lower;
-}
-
-/* 128-bit FP subtraction RR */
-uint32_t HELPER(sxbr)(uint32_t f1, uint32_t f2)
-{
-    CPU_QuadU v1;
-    v1.ll.upper = env->fregs[f1].ll;
-    v1.ll.lower = env->fregs[f1 + 2].ll;
-    CPU_QuadU v2;
-    v2.ll.upper = env->fregs[f2].ll;
-    v2.ll.lower = env->fregs[f2 + 2].ll;
-    CPU_QuadU res;
-    res.q = float128_sub(v1.q, v2.q, &env->fpu_status);
-    env->fregs[f1].ll = res.ll.upper;
-    env->fregs[f1 + 2].ll = res.ll.lower;
-    return set_cc_nz_f128(res.q);
-}
-
-/* 128-bit FP addition RR */
-uint32_t HELPER(axbr)(uint32_t f1, uint32_t f2)
-{
-    CPU_QuadU v1;
-    v1.ll.upper = env->fregs[f1].ll;
-    v1.ll.lower = env->fregs[f1 + 2].ll;
-    CPU_QuadU v2;
-    v2.ll.upper = env->fregs[f2].ll;
-    v2.ll.lower = env->fregs[f2 + 2].ll;
-    CPU_QuadU res;
-    res.q = float128_add(v1.q, v2.q, &env->fpu_status);
-    env->fregs[f1].ll = res.ll.upper;
-    env->fregs[f1 + 2].ll = res.ll.lower;
-    return set_cc_nz_f128(res.q);
-}
-
-/* 32-bit FP multiplication RR */
-void HELPER(meebr)(uint32_t f1, uint32_t f2)
-{
-    env->fregs[f1].l.upper = float32_mul(env->fregs[f1].l.upper,
-                                         env->fregs[f2].l.upper,
-                                         &env->fpu_status);
-}
-
-/* 64-bit FP division RR */
-void HELPER(ddbr)(uint32_t f1, uint32_t f2)
-{
-    env->fregs[f1].d = float64_div(env->fregs[f1].d, env->fregs[f2].d,
-                                   &env->fpu_status);
-}
-
-/* 64-bit FP multiply and add RM */
-void HELPER(madb)(uint32_t f1, uint64_t a2, uint32_t f3)
-{
-    HELPER_LOG("%s: f1 %d a2 0x%lx f3 %d\n", __FUNCTION__, f1, a2, f3);
-    CPU_DoubleU v2;
-    v2.ll = ldq(a2);
-    env->fregs[f1].d = float64_add(env->fregs[f1].d,
-                                   float64_mul(v2.d, env->fregs[f3].d,
-                                               &env->fpu_status),
-                                   &env->fpu_status);
-}
-
-/* 64-bit FP multiply and add RR */
-void HELPER(madbr)(uint32_t f1, uint32_t f3, uint32_t f2)
-{
-    HELPER_LOG("%s: f1 %d f2 %d f3 %d\n", __FUNCTION__, f1, f2, f3);
-    env->fregs[f1].d = float64_add(float64_mul(env->fregs[f2].d,
-                                               env->fregs[f3].d,
-                                               &env->fpu_status),
-                                   env->fregs[f1].d, &env->fpu_status);
-}
-
-/* 64-bit FP multiply and subtract RR */
-void HELPER(msdbr)(uint32_t f1, uint32_t f3, uint32_t f2)
-{
-    HELPER_LOG("%s: f1 %d f2 %d f3 %d\n", __FUNCTION__, f1, f2, f3);
-    env->fregs[f1].d = float64_sub(float64_mul(env->fregs[f2].d,
-                                               env->fregs[f3].d,
-                                               &env->fpu_status),
-                                   env->fregs[f1].d, &env->fpu_status);
-}
-
-/* 32-bit FP multiply and add RR */
-void HELPER(maebr)(uint32_t f1, uint32_t f3, uint32_t f2)
-{
-    env->fregs[f1].l.upper = float32_add(env->fregs[f1].l.upper,
-                                         float32_mul(env->fregs[f2].l.upper,
-                                                     env->fregs[f3].l.upper,
-                                                     &env->fpu_status),
-                                         &env->fpu_status);
-}
-
-/* convert 32-bit float to 64-bit float */
-void HELPER(ldeb)(uint32_t f1, uint64_t a2)
-{
-    uint32_t v2;
-    v2 = ldl(a2);
-    env->fregs[f1].d = float32_to_float64(v2,
-                                          &env->fpu_status);
-}
-
-/* convert 64-bit float to 128-bit float */
-void HELPER(lxdb)(uint32_t f1, uint64_t a2)
-{
-    CPU_DoubleU v2;
-    v2.ll = ldq(a2);
-    CPU_QuadU v1;
-    v1.q = float64_to_float128(v2.d, &env->fpu_status);
-    env->fregs[f1].ll = v1.ll.upper;
-    env->fregs[f1 + 2].ll = v1.ll.lower;
-}
-
-/* test data class 32-bit */
-uint32_t HELPER(tceb)(uint32_t f1, uint64_t m2)
-{
-    float32 v1 = env->fregs[f1].l.upper;
-    int neg = float32_is_neg(v1);
-    uint32_t cc = 0;
-
-    HELPER_LOG("%s: v1 0x%lx m2 0x%lx neg %d\n", __FUNCTION__, (long)v1, m2, neg);
-    if ((float32_is_zero(v1) && (m2 & (1 << (11-neg)))) ||
-        (float32_is_infinity(v1) && (m2 & (1 << (5-neg)))) ||
-        (float32_is_any_nan(v1) && (m2 & (1 << (3-neg)))) ||
-        (float32_is_signaling_nan(v1) && (m2 & (1 << (1-neg))))) {
-        cc = 1;
-    } else if (m2 & (1 << (9-neg))) {
-        /* assume normalized number */
-        cc = 1;
-    }
-
-    /* FIXME: denormalized? */
-    return cc;
-}
-
-/* test data class 64-bit */
-uint32_t HELPER(tcdb)(uint32_t f1, uint64_t m2)
-{
-    float64 v1 = env->fregs[f1].d;
-    int neg = float64_is_neg(v1);
-    uint32_t cc = 0;
-
-    HELPER_LOG("%s: v1 0x%lx m2 0x%lx neg %d\n", __FUNCTION__, v1, m2, neg);
-    if ((float64_is_zero(v1) && (m2 & (1 << (11-neg)))) ||
-        (float64_is_infinity(v1) && (m2 & (1 << (5-neg)))) ||
-        (float64_is_any_nan(v1) && (m2 & (1 << (3-neg)))) ||
-        (float64_is_signaling_nan(v1) && (m2 & (1 << (1-neg))))) {
-        cc = 1;
-    } else if (m2 & (1 << (9-neg))) {
-        /* assume normalized number */
-        cc = 1;
-    }
-    /* FIXME: denormalized? */
-    return cc;
-}
-
-/* test data class 128-bit */
-uint32_t HELPER(tcxb)(uint32_t f1, uint64_t m2)
-{
-    CPU_QuadU v1;
-    uint32_t cc = 0;
-    v1.ll.upper = env->fregs[f1].ll;
-    v1.ll.lower = env->fregs[f1 + 2].ll;
-
-    int neg = float128_is_neg(v1.q);
-    if ((float128_is_zero(v1.q) && (m2 & (1 << (11-neg)))) ||
-        (float128_is_infinity(v1.q) && (m2 & (1 << (5-neg)))) ||
-        (float128_is_any_nan(v1.q) && (m2 & (1 << (3-neg)))) ||
-        (float128_is_signaling_nan(v1.q) && (m2 & (1 << (1-neg))))) {
-        cc = 1;
-    } else if (m2 & (1 << (9-neg))) {
-        /* assume normalized number */
-        cc = 1;
-    }
-    /* FIXME: denormalized? */
-    return cc;
-}
-
-/* find leftmost one */
-uint32_t HELPER(flogr)(uint32_t r1, uint64_t v2)
-{
-    uint64_t res = 0;
-    uint64_t ov2 = v2;
-
-    while (!(v2 & 0x8000000000000000ULL) && v2) {
-        v2 <<= 1;
-        res++;
-    }
-
-    if (!v2) {
-        env->regs[r1] = 64;
-        env->regs[r1 + 1] = 0;
-        return 0;
-    } else {
-        env->regs[r1] = res;
-        env->regs[r1 + 1] = ov2 & ~(0x8000000000000000ULL >> res);
-        return 2;
-    }
-}
-
-/* square root 64-bit RR */
-void HELPER(sqdbr)(uint32_t f1, uint32_t f2)
-{
-    env->fregs[f1].d = float64_sqrt(env->fregs[f2].d, &env->fpu_status);
-}
-
-/* checksum */
-void HELPER(cksm)(uint32_t r1, uint32_t r2)
-{
-    uint64_t src = get_address_31fix(r2);
-    uint64_t src_len = env->regs[(r2 + 1) & 15];
-    uint64_t cksm = (uint32_t)env->regs[r1];
-
-    while (src_len >= 4) {
-        cksm += ldl(src);
-
-        /* move to next word */
-        src_len -= 4;
-        src += 4;
-    }
-
-    switch (src_len) {
-    case 0:
-        break;
-    case 1:
-        cksm += ldub(src) << 24;
-        break;
-    case 2:
-        cksm += lduw(src) << 16;
-        break;
-    case 3:
-        cksm += lduw(src) << 16;
-        cksm += ldub(src + 2) << 8;
-        break;
-    }
-
-    /* indicate we've processed everything */
-    env->regs[r2] = src + src_len;
-    env->regs[(r2 + 1) & 15] = 0;
-
-    /* store result */
-    env->regs[r1] = (env->regs[r1] & 0xffffffff00000000ULL) |
-                    ((uint32_t)cksm + (cksm >> 32));
-}
-
-static inline uint32_t cc_calc_ltgt_32(CPUS390XState *env, int32_t src,
-                                       int32_t dst)
-{
-    if (src == dst) {
-        return 0;
-    } else if (src < dst) {
-        return 1;
-    } else {
-        return 2;
-    }
-}
-
-static inline uint32_t cc_calc_ltgt0_32(CPUS390XState *env, int32_t dst)
-{
-    return cc_calc_ltgt_32(env, dst, 0);
-}
-
-static inline uint32_t cc_calc_ltgt_64(CPUS390XState *env, int64_t src,
-                                       int64_t dst)
-{
-    if (src == dst) {
-        return 0;
-    } else if (src < dst) {
-        return 1;
-    } else {
-        return 2;
-    }
-}
-
-static inline uint32_t cc_calc_ltgt0_64(CPUS390XState *env, int64_t dst)
-{
-    return cc_calc_ltgt_64(env, dst, 0);
-}
-
-static inline uint32_t cc_calc_ltugtu_32(CPUS390XState *env, uint32_t src,
-                                         uint32_t dst)
-{
-    if (src == dst) {
-        return 0;
-    } else if (src < dst) {
-        return 1;
-    } else {
-        return 2;
-    }
-}
-
-static inline uint32_t cc_calc_ltugtu_64(CPUS390XState *env, uint64_t src,
-                                         uint64_t dst)
-{
-    if (src == dst) {
-        return 0;
-    } else if (src < dst) {
-        return 1;
-    } else {
-        return 2;
-    }
-}
-
-static inline uint32_t cc_calc_tm_32(CPUS390XState *env, uint32_t val, uint32_t mask)
-{
-    HELPER_LOG("%s: val 0x%x mask 0x%x\n", __FUNCTION__, val, mask);
-    uint16_t r = val & mask;
-    if (r == 0 || mask == 0) {
-        return 0;
-    } else if (r == mask) {
-        return 3;
-    } else {
-        return 1;
-    }
-}
-
-/* set condition code for test under mask */
-static inline uint32_t cc_calc_tm_64(CPUS390XState *env, uint64_t val, uint32_t mask)
-{
-    uint16_t r = val & mask;
-    HELPER_LOG("%s: val 0x%lx mask 0x%x r 0x%x\n", __FUNCTION__, val, mask, r);
-    if (r == 0 || mask == 0) {
-        return 0;
-    } else if (r == mask) {
-        return 3;
-    } else {
-        while (!(mask & 0x8000)) {
-            mask <<= 1;
-            val <<= 1;
-        }
-        if (val & 0x8000) {
-            return 2;
-        } else {
-            return 1;
-        }
-    }
-}
-
-static inline uint32_t cc_calc_nz(CPUS390XState *env, uint64_t dst)
-{
-    return !!dst;
-}
-
-static inline uint32_t cc_calc_add_64(CPUS390XState *env, int64_t a1, int64_t a2,
-                                      int64_t ar)
-{
-    if ((a1 > 0 && a2 > 0 && ar < 0) || (a1 < 0 && a2 < 0 && ar > 0)) {
-        return 3; /* overflow */
-    } else {
-        if (ar < 0) {
-            return 1;
-        } else if (ar > 0) {
-            return 2;
-        } else {
-            return 0;
-        }
-    }
-}
-
-static inline uint32_t cc_calc_addu_64(CPUS390XState *env, uint64_t a1, uint64_t a2,
-                                       uint64_t ar)
-{
-    if (ar == 0) {
-        if (a1) {
-            return 2;
-        } else {
-            return 0;
-        }
-    } else {
-        if (ar < a1 || ar < a2) {
-          return 3;
-        } else {
-          return 1;
-        }
-    }
-}
-
-static inline uint32_t cc_calc_sub_64(CPUS390XState *env, int64_t a1, int64_t a2,
-                                      int64_t ar)
-{
-    if ((a1 > 0 && a2 < 0 && ar < 0) || (a1 < 0 && a2 > 0 && ar > 0)) {
-        return 3; /* overflow */
-    } else {
-        if (ar < 0) {
-            return 1;
-        } else if (ar > 0) {
-            return 2;
-        } else {
-            return 0;
-        }
-    }
-}
-
-static inline uint32_t cc_calc_subu_64(CPUS390XState *env, uint64_t a1, uint64_t a2,
-                                       uint64_t ar)
-{
-    if (ar == 0) {
-        return 2;
-    } else {
-        if (a2 > a1) {
-            return 1;
-        } else {
-            return 3;
-        }
-    }
-}
-
-static inline uint32_t cc_calc_abs_64(CPUS390XState *env, int64_t dst)
-{
-    if ((uint64_t)dst == 0x8000000000000000ULL) {
-        return 3;
-    } else if (dst) {
-        return 1;
-    } else {
-        return 0;
-    }
-}
-
-static inline uint32_t cc_calc_nabs_64(CPUS390XState *env, int64_t dst)
-{
-    return !!dst;
-}
-
-static inline uint32_t cc_calc_comp_64(CPUS390XState *env, int64_t dst)
-{
-    if ((uint64_t)dst == 0x8000000000000000ULL) {
-        return 3;
-    } else if (dst < 0) {
-        return 1;
-    } else if (dst > 0) {
-        return 2;
-    } else {
-        return 0;
-    }
-}
-
-
-static inline uint32_t cc_calc_add_32(CPUS390XState *env, int32_t a1, int32_t a2,
-                                      int32_t ar)
-{
-    if ((a1 > 0 && a2 > 0 && ar < 0) || (a1 < 0 && a2 < 0 && ar > 0)) {
-        return 3; /* overflow */
-    } else {
-        if (ar < 0) {
-            return 1;
-        } else if (ar > 0) {
-            return 2;
-        } else {
-            return 0;
-        }
-    }
-}
-
-static inline uint32_t cc_calc_addu_32(CPUS390XState *env, uint32_t a1, uint32_t a2,
-                                       uint32_t ar)
-{
-    if (ar == 0) {
-        if (a1) {
-          return 2;
-        } else {
-          return 0;
-        }
-    } else {
-        if (ar < a1 || ar < a2) {
-          return 3;
-        } else {
-          return 1;
-        }
-    }
-}
-
-static inline uint32_t cc_calc_sub_32(CPUS390XState *env, int32_t a1, int32_t a2,
-                                      int32_t ar)
-{
-    if ((a1 > 0 && a2 < 0 && ar < 0) || (a1 < 0 && a2 > 0 && ar > 0)) {
-        return 3; /* overflow */
-    } else {
-        if (ar < 0) {
-            return 1;
-        } else if (ar > 0) {
-            return 2;
-        } else {
-            return 0;
-        }
-    }
-}
-
-static inline uint32_t cc_calc_subu_32(CPUS390XState *env, uint32_t a1, uint32_t a2,
-                                       uint32_t ar)
-{
-    if (ar == 0) {
-        return 2;
-    } else {
-        if (a2 > a1) {
-            return 1;
-        } else {
-            return 3;
-        }
-    }
-}
-
-static inline uint32_t cc_calc_abs_32(CPUS390XState *env, int32_t dst)
-{
-    if ((uint32_t)dst == 0x80000000UL) {
-        return 3;
-    } else if (dst) {
-        return 1;
-    } else {
-        return 0;
-    }
-}
-
-static inline uint32_t cc_calc_nabs_32(CPUS390XState *env, int32_t dst)
-{
-    return !!dst;
-}
-
-static inline uint32_t cc_calc_comp_32(CPUS390XState *env, int32_t dst)
-{
-    if ((uint32_t)dst == 0x80000000UL) {
-        return 3;
-    } else if (dst < 0) {
-        return 1;
-    } else if (dst > 0) {
-        return 2;
-    } else {
-        return 0;
-    }
-}
-
-/* calculate condition code for insert character under mask insn */
-static inline uint32_t cc_calc_icm_32(CPUS390XState *env, uint32_t mask, uint32_t val)
-{
-    HELPER_LOG("%s: mask 0x%x val %d\n", __FUNCTION__, mask, val);
-    uint32_t cc;
-
-    if (mask == 0xf) {
-        if (!val) {
-            return 0;
-        } else if (val & 0x80000000) {
-            return 1;
-        } else {
-            return 2;
-        }
-    }
-
-    if (!val || !mask) {
-        cc = 0;
-    } else {
-        while (mask != 1) {
-            mask >>= 1;
-            val >>= 8;
-        }
-        if (val & 0x80) {
-            cc = 1;
-        } else {
-            cc = 2;
-        }
-    }
-    return cc;
-}
-
-static inline uint32_t cc_calc_slag(CPUS390XState *env, uint64_t src, uint64_t shift)
-{
-    uint64_t mask = ((1ULL << shift) - 1ULL) << (64 - shift);
-    uint64_t match, r;
-
-    /* check if the sign bit stays the same */
-    if (src & (1ULL << 63)) {
-        match = mask;
-    } else {
-        match = 0;
-    }
-
-    if ((src & mask) != match) {
-        /* overflow */
-        return 3;
-    }
-
-    r = ((src << shift) & ((1ULL << 63) - 1)) | (src & (1ULL << 63));
-
-    if ((int64_t)r == 0) {
-        return 0;
-    } else if ((int64_t)r < 0) {
-        return 1;
-    }
-
-    return 2;
-}
-
-
-static inline uint32_t do_calc_cc(CPUS390XState *env, uint32_t cc_op, uint64_t src,
-                                  uint64_t dst, uint64_t vr)
-{
-    uint32_t r = 0;
-
-    switch (cc_op) {
-    case CC_OP_CONST0:
-    case CC_OP_CONST1:
-    case CC_OP_CONST2:
-    case CC_OP_CONST3:
-        /* cc_op value _is_ cc */
-        r = cc_op;
-        break;
-    case CC_OP_LTGT0_32:
-        r = cc_calc_ltgt0_32(env, dst);
-        break;
-    case CC_OP_LTGT0_64:
-        r =  cc_calc_ltgt0_64(env, dst);
-        break;
-    case CC_OP_LTGT_32:
-        r =  cc_calc_ltgt_32(env, src, dst);
-        break;
-    case CC_OP_LTGT_64:
-        r =  cc_calc_ltgt_64(env, src, dst);
-        break;
-    case CC_OP_LTUGTU_32:
-        r =  cc_calc_ltugtu_32(env, src, dst);
-        break;
-    case CC_OP_LTUGTU_64:
-        r =  cc_calc_ltugtu_64(env, src, dst);
-        break;
-    case CC_OP_TM_32:
-        r =  cc_calc_tm_32(env, src, dst);
-        break;
-    case CC_OP_TM_64:
-        r =  cc_calc_tm_64(env, src, dst);
-        break;
-    case CC_OP_NZ:
-        r =  cc_calc_nz(env, dst);
-        break;
-    case CC_OP_ADD_64:
-        r =  cc_calc_add_64(env, src, dst, vr);
-        break;
-    case CC_OP_ADDU_64:
-        r =  cc_calc_addu_64(env, src, dst, vr);
-        break;
-    case CC_OP_SUB_64:
-        r =  cc_calc_sub_64(env, src, dst, vr);
-        break;
-    case CC_OP_SUBU_64:
-        r =  cc_calc_subu_64(env, src, dst, vr);
-        break;
-    case CC_OP_ABS_64:
-        r =  cc_calc_abs_64(env, dst);
-        break;
-    case CC_OP_NABS_64:
-        r =  cc_calc_nabs_64(env, dst);
-        break;
-    case CC_OP_COMP_64:
-        r =  cc_calc_comp_64(env, dst);
-        break;
-
-    case CC_OP_ADD_32:
-        r =  cc_calc_add_32(env, src, dst, vr);
-        break;
-    case CC_OP_ADDU_32:
-        r =  cc_calc_addu_32(env, src, dst, vr);
-        break;
-    case CC_OP_SUB_32:
-        r =  cc_calc_sub_32(env, src, dst, vr);
-        break;
-    case CC_OP_SUBU_32:
-        r =  cc_calc_subu_32(env, src, dst, vr);
-        break;
-    case CC_OP_ABS_32:
-        r =  cc_calc_abs_64(env, dst);
-        break;
-    case CC_OP_NABS_32:
-        r =  cc_calc_nabs_64(env, dst);
-        break;
-    case CC_OP_COMP_32:
-        r =  cc_calc_comp_32(env, dst);
-        break;
-
-    case CC_OP_ICM:
-        r =  cc_calc_icm_32(env, src, dst);
-        break;
-    case CC_OP_SLAG:
-        r =  cc_calc_slag(env, src, dst);
-        break;
-
-    case CC_OP_LTGT_F32:
-        r = set_cc_f32(src, dst);
-        break;
-    case CC_OP_LTGT_F64:
-        r = set_cc_f64(src, dst);
-        break;
-    case CC_OP_NZ_F32:
-        r = set_cc_nz_f32(dst);
-        break;
-    case CC_OP_NZ_F64:
-        r = set_cc_nz_f64(dst);
-        break;
-
-    default:
-        cpu_abort(env, "Unknown CC operation: %s\n", cc_name(cc_op));
-    }
-
-    HELPER_LOG("%s: %15s 0x%016lx 0x%016lx 0x%016lx = %d\n", __FUNCTION__,
-               cc_name(cc_op), src, dst, vr, r);
-    return r;
-}
-
-uint32_t calc_cc(CPUS390XState *env, uint32_t cc_op, uint64_t src, uint64_t dst,
-                 uint64_t vr)
-{
-    return do_calc_cc(env, cc_op, src, dst, vr);
-}
-
-uint32_t HELPER(calc_cc)(uint32_t cc_op, uint64_t src, uint64_t dst,
-                         uint64_t vr)
-{
-    return do_calc_cc(env, cc_op, src, dst, vr);
-}
-
-uint64_t HELPER(cvd)(int32_t bin)
-{
-    /* positive 0 */
-    uint64_t dec = 0x0c;
-    int shift = 4;
-
-    if (bin < 0) {
-        bin = -bin;
-        dec = 0x0d;
-    }
-
-    for (shift = 4; (shift < 64) && bin; shift += 4) {
-        int current_number = bin % 10;
-
-        dec |= (current_number) << shift;
-        bin /= 10;
-    }
-
-    return dec;
-}
-
-void HELPER(unpk)(uint32_t len, uint64_t dest, uint64_t src)
-{
-    int len_dest = len >> 4;
-    int len_src = len & 0xf;
-    uint8_t b;
-    int second_nibble = 0;
-
-    dest += len_dest;
-    src += len_src;
-
-    /* last byte is special, it only flips the nibbles */
-    b = ldub(src);
-    stb(dest, (b << 4) | (b >> 4));
-    src--;
-    len_src--;
-
-    /* now pad every nibble with 0xf0 */
-
-    while (len_dest > 0) {
-        uint8_t cur_byte = 0;
-
-        if (len_src > 0) {
-            cur_byte = ldub(src);
-        }
-
-        len_dest--;
-        dest--;
-
-        /* only advance one nibble at a time */
-        if (second_nibble) {
-            cur_byte >>= 4;
-            len_src--;
-            src--;
-        }
-        second_nibble = !second_nibble;
-
-        /* digit */
-        cur_byte = (cur_byte & 0xf);
-        /* zone bits */
-        cur_byte |= 0xf0;
-
-        stb(dest, cur_byte);
-    }
-}
-
-void HELPER(tr)(uint32_t len, uint64_t array, uint64_t trans)
-{
-    int i;
-
-    for (i = 0; i <= len; i++) {
-        uint8_t byte = ldub(array + i);
-        uint8_t new_byte = ldub(trans + byte);
-        stb(array + i, new_byte);
-    }
-}
-
-#ifndef CONFIG_USER_ONLY
-
-void HELPER(load_psw)(uint64_t mask, uint64_t addr)
-{
-    load_psw(env, mask, addr);
-    cpu_loop_exit(env);
-}
-
-static void program_interrupt(CPUS390XState *env, uint32_t code, int ilc)
-{
-    qemu_log("program interrupt at %#" PRIx64 "\n", env->psw.addr);
-
-    if (kvm_enabled()) {
-#ifdef CONFIG_KVM
-        kvm_s390_interrupt(env, KVM_S390_PROGRAM_INT, code);
-#endif
-    } else {
-        env->int_pgm_code = code;
-        env->int_pgm_ilc = ilc;
-        env->exception_index = EXCP_PGM;
-        cpu_loop_exit(env);
-    }
-}
-
-/*
- * ret < 0 indicates program check, ret = 0,1,2,3 -> cc
- */
-int sclp_service_call(CPUS390XState *env, uint32_t sccb, uint64_t code)
-{
-    int r = 0;
-    int shift = 0;
-
-#ifdef DEBUG_HELPER
-    printf("sclp(0x%x, 0x%" PRIx64 ")\n", sccb, code);
-#endif
-
-    /* basic checks */
-    if (!memory_region_is_ram(phys_page_find(sccb >> TARGET_PAGE_BITS)->mr)) {
-        return -PGM_ADDRESSING;
-    }
-    if (sccb & ~0x7ffffff8ul) {
-        return -PGM_SPECIFICATION;
-    }
-
-    switch(code) {
-        case SCLP_CMDW_READ_SCP_INFO:
-        case SCLP_CMDW_READ_SCP_INFO_FORCED:
-            while ((ram_size >> (20 + shift)) > 65535) {
-                shift++;
-            }
-            stw_phys(sccb + SCP_MEM_CODE, ram_size >> (20 + shift));
-            stb_phys(sccb + SCP_INCREMENT, 1 << shift);
-            stw_phys(sccb + SCP_RESPONSE_CODE, 0x10);
-
-            s390_sclp_extint(sccb & ~3);
-            break;
-        default:
-#ifdef DEBUG_HELPER
-            printf("KVM: invalid sclp call 0x%x / 0x%" PRIx64 "x\n", sccb, code);
-#endif
-            r = 3;
-            break;
-    }
-
-    return r;
-}
-
-/* SCLP service call */
-uint32_t HELPER(servc)(uint32_t r1, uint64_t r2)
-{
-    int r;
-
-    r = sclp_service_call(env, r1, r2);
-    if (r < 0) {
-        program_interrupt(env, -r, 4);
-        return 0;
-    }
-    return r;
-}
-
-/* DIAG */
-uint64_t HELPER(diag)(uint32_t num, uint64_t mem, uint64_t code)
-{
-    uint64_t r;
-
-    switch (num) {
-    case 0x500:
-        /* KVM hypercall */
-        r = s390_virtio_hypercall(env, mem, code);
-        break;
-    case 0x44:
-        /* yield */
-        r = 0;
-        break;
-    case 0x308:
-        /* ipl */
-        r = 0;
-        break;
-    default:
-        r = -1;
-        break;
-    }
-
-    if (r) {
-        program_interrupt(env, PGM_OPERATION, ILC_LATER_INC);
-    }
-
-    return r;
-}
-
-/* Store CPU ID */
-void HELPER(stidp)(uint64_t a1)
-{
-    stq(a1, env->cpu_num);
-}
-
-/* Set Prefix */
-void HELPER(spx)(uint64_t a1)
-{
-    uint32_t prefix;
-
-    prefix = ldl(a1);
-    env->psa = prefix & 0xfffff000;
-    qemu_log("prefix: %#x\n", prefix);
-    tlb_flush_page(env, 0);
-    tlb_flush_page(env, TARGET_PAGE_SIZE);
-}
-
-/* Set Clock */
-uint32_t HELPER(sck)(uint64_t a1)
-{
-    /* XXX not implemented - is it necessary? */
-
-    return 0;
-}
-
-static inline uint64_t clock_value(CPUS390XState *env)
-{
-    uint64_t time;
-
-    time = env->tod_offset +
-           time2tod(qemu_get_clock_ns(vm_clock) - env->tod_basetime);
-
-    return time;
-}
-
-/* Store Clock */
-uint32_t HELPER(stck)(uint64_t a1)
-{
-    stq(a1, clock_value(env));
-
-    return 0;
-}
-
-/* Store Clock Extended */
-uint32_t HELPER(stcke)(uint64_t a1)
-{
-    stb(a1, 0);
-    /* basically the same value as stck */
-    stq(a1 + 1, clock_value(env) | env->cpu_num);
-    /* more fine grained than stck */
-    stq(a1 + 9, 0);
-    /* XXX programmable fields */
-    stw(a1 + 17, 0);
-
-
-    return 0;
-}
-
-/* Set Clock Comparator */
-void HELPER(sckc)(uint64_t a1)
-{
-    uint64_t time = ldq(a1);
-
-    if (time == -1ULL) {
-        return;
-    }
-
-    /* difference between now and then */
-    time -= clock_value(env);
-    /* nanoseconds */
-    time = (time * 125) >> 9;
-
-    qemu_mod_timer(env->tod_timer, qemu_get_clock_ns(vm_clock) + time);
-}
-
-/* Store Clock Comparator */
-void HELPER(stckc)(uint64_t a1)
-{
-    /* XXX implement */
-    stq(a1, 0);
-}
-
-/* Set CPU Timer */
-void HELPER(spt)(uint64_t a1)
-{
-    uint64_t time = ldq(a1);
-
-    if (time == -1ULL) {
-        return;
-    }
-
-    /* nanoseconds */
-    time = (time * 125) >> 9;
-
-    qemu_mod_timer(env->cpu_timer, qemu_get_clock_ns(vm_clock) + time);
-}
-
-/* Store CPU Timer */
-void HELPER(stpt)(uint64_t a1)
-{
-    /* XXX implement */
-    stq(a1, 0);
-}
-
-/* Store System Information */
-uint32_t HELPER(stsi)(uint64_t a0, uint32_t r0, uint32_t r1)
-{
-    int cc = 0;
-    int sel1, sel2;
-
-    if ((r0 & STSI_LEVEL_MASK) <= STSI_LEVEL_3 &&
-        ((r0 & STSI_R0_RESERVED_MASK) || (r1 & STSI_R1_RESERVED_MASK))) {
-        /* valid function code, invalid reserved bits */
-        program_interrupt(env, PGM_SPECIFICATION, 2);
-    }
-
-    sel1 = r0 & STSI_R0_SEL1_MASK;
-    sel2 = r1 & STSI_R1_SEL2_MASK;
-
-    /* XXX: spec exception if sysib is not 4k-aligned */
-
-    switch (r0 & STSI_LEVEL_MASK) {
-    case STSI_LEVEL_1:
-        if ((sel1 == 1) && (sel2 == 1)) {
-            /* Basic Machine Configuration */
-            struct sysib_111 sysib;
-
-            memset(&sysib, 0, sizeof(sysib));
-            ebcdic_put(sysib.manuf, "QEMU            ", 16);
-            /* same as machine type number in STORE CPU ID */
-            ebcdic_put(sysib.type, "QEMU", 4);
-            /* same as model number in STORE CPU ID */
-            ebcdic_put(sysib.model, "QEMU            ", 16);
-            ebcdic_put(sysib.sequence, "QEMU            ", 16);
-            ebcdic_put(sysib.plant, "QEMU", 4);
-            cpu_physical_memory_rw(a0, (uint8_t*)&sysib, sizeof(sysib), 1);
-        } else if ((sel1 == 2) && (sel2 == 1)) {
-            /* Basic Machine CPU */
-            struct sysib_121 sysib;
-
-            memset(&sysib, 0, sizeof(sysib));
-            /* XXX make different for different CPUs? */
-            ebcdic_put(sysib.sequence, "QEMUQEMUQEMUQEMU", 16);
-            ebcdic_put(sysib.plant, "QEMU", 4);
-            stw_p(&sysib.cpu_addr, env->cpu_num);
-            cpu_physical_memory_rw(a0, (uint8_t*)&sysib, sizeof(sysib), 1);
-        } else if ((sel1 == 2) && (sel2 == 2)) {
-            /* Basic Machine CPUs */
-            struct sysib_122 sysib;
-
-            memset(&sysib, 0, sizeof(sysib));
-            stl_p(&sysib.capability, 0x443afc29);
-            /* XXX change when SMP comes */
-            stw_p(&sysib.total_cpus, 1);
-            stw_p(&sysib.active_cpus, 1);
-            stw_p(&sysib.standby_cpus, 0);
-            stw_p(&sysib.reserved_cpus, 0);
-            cpu_physical_memory_rw(a0, (uint8_t*)&sysib, sizeof(sysib), 1);
-        } else {
-            cc = 3;
-        }
-        break;
-    case STSI_LEVEL_2:
-    {
-        if ((sel1 == 2) && (sel2 == 1)) {
-            /* LPAR CPU */
-            struct sysib_221 sysib;
-
-            memset(&sysib, 0, sizeof(sysib));
-            /* XXX make different for different CPUs? */
-            ebcdic_put(sysib.sequence, "QEMUQEMUQEMUQEMU", 16);
-            ebcdic_put(sysib.plant, "QEMU", 4);
-            stw_p(&sysib.cpu_addr, env->cpu_num);
-            stw_p(&sysib.cpu_id, 0);
-            cpu_physical_memory_rw(a0, (uint8_t*)&sysib, sizeof(sysib), 1);
-        } else if ((sel1 == 2) && (sel2 == 2)) {
-            /* LPAR CPUs */
-            struct sysib_222 sysib;
-
-            memset(&sysib, 0, sizeof(sysib));
-            stw_p(&sysib.lpar_num, 0);
-            sysib.lcpuc = 0;
-            /* XXX change when SMP comes */
-            stw_p(&sysib.total_cpus, 1);
-            stw_p(&sysib.conf_cpus, 1);
-            stw_p(&sysib.standby_cpus, 0);
-            stw_p(&sysib.reserved_cpus, 0);
-            ebcdic_put(sysib.name, "QEMU    ", 8);
-            stl_p(&sysib.caf, 1000);
-            stw_p(&sysib.dedicated_cpus, 0);
-            stw_p(&sysib.shared_cpus, 0);
-            cpu_physical_memory_rw(a0, (uint8_t*)&sysib, sizeof(sysib), 1);
-        } else {
-            cc = 3;
-        }
-        break;
-    }
-    case STSI_LEVEL_3:
-    {
-        if ((sel1 == 2) && (sel2 == 2)) {
-            /* VM CPUs */
-            struct sysib_322 sysib;
-
-            memset(&sysib, 0, sizeof(sysib));
-            sysib.count = 1;
-            /* XXX change when SMP comes */
-            stw_p(&sysib.vm[0].total_cpus, 1);
-            stw_p(&sysib.vm[0].conf_cpus, 1);
-            stw_p(&sysib.vm[0].standby_cpus, 0);
-            stw_p(&sysib.vm[0].reserved_cpus, 0);
-            ebcdic_put(sysib.vm[0].name, "KVMguest", 8);
-            stl_p(&sysib.vm[0].caf, 1000);
-            ebcdic_put(sysib.vm[0].cpi, "KVM/Linux       ", 16);
-            cpu_physical_memory_rw(a0, (uint8_t*)&sysib, sizeof(sysib), 1);
-        } else {
-            cc = 3;
-        }
-        break;
-    }
-    case STSI_LEVEL_CURRENT:
-        env->regs[0] = STSI_LEVEL_3;
-        break;
-    default:
-        cc = 3;
-        break;
-    }
-
-    return cc;
-}
-
-void HELPER(lctlg)(uint32_t r1, uint64_t a2, uint32_t r3)
-{
-    int i;
-    uint64_t src = a2;
-
-    for (i = r1;; i = (i + 1) % 16) {
-        env->cregs[i] = ldq(src);
-        HELPER_LOG("load ctl %d from 0x%" PRIx64 " == 0x%" PRIx64 "\n",
-                   i, src, env->cregs[i]);
-        src += sizeof(uint64_t);
-
-        if (i == r3) {
-            break;
-        }
-    }
-
-    tlb_flush(env, 1);
-}
-
-void HELPER(lctl)(uint32_t r1, uint64_t a2, uint32_t r3)
-{
-    int i;
-    uint64_t src = a2;
-
-    for (i = r1;; i = (i + 1) % 16) {
-        env->cregs[i] = (env->cregs[i] & 0xFFFFFFFF00000000ULL) | ldl(src);
-        src += sizeof(uint32_t);
-
-        if (i == r3) {
-            break;
-        }
-    }
-
-    tlb_flush(env, 1);
-}
-
-void HELPER(stctg)(uint32_t r1, uint64_t a2, uint32_t r3)
-{
-    int i;
-    uint64_t dest = a2;
-
-    for (i = r1;; i = (i + 1) % 16) {
-        stq(dest, env->cregs[i]);
-        dest += sizeof(uint64_t);
-
-        if (i == r3) {
-            break;
-        }
-    }
-}
-
-void HELPER(stctl)(uint32_t r1, uint64_t a2, uint32_t r3)
-{
-    int i;
-    uint64_t dest = a2;
-
-    for (i = r1;; i = (i + 1) % 16) {
-        stl(dest, env->cregs[i]);
-        dest += sizeof(uint32_t);
-
-        if (i == r3) {
-            break;
-        }
-    }
-}
-
-uint32_t HELPER(tprot)(uint64_t a1, uint64_t a2)
-{
-    /* XXX implement */
-
-    return 0;
-}
-
-/* insert storage key extended */
-uint64_t HELPER(iske)(uint64_t r2)
-{
-    uint64_t addr = get_address(0, 0, r2);
-
-    if (addr > ram_size) {
-        return 0;
-    }
-
-    return env->storage_keys[addr / TARGET_PAGE_SIZE];
-}
-
-/* set storage key extended */
-void HELPER(sske)(uint32_t r1, uint64_t r2)
-{
-    uint64_t addr = get_address(0, 0, r2);
-
-    if (addr > ram_size) {
-        return;
-    }
-
-    env->storage_keys[addr / TARGET_PAGE_SIZE] = r1;
-}
-
-/* reset reference bit extended */
-uint32_t HELPER(rrbe)(uint32_t r1, uint64_t r2)
-{
-    uint8_t re;
-    uint8_t key;
-    if (r2 > ram_size) {
-        return 0;
-    }
-
-    key = env->storage_keys[r2 / TARGET_PAGE_SIZE];
-    re = key & (SK_R | SK_C);
-    env->storage_keys[r2 / TARGET_PAGE_SIZE] = (key & ~SK_R);
-
-    /*
-     * cc
-     *
-     * 0  Reference bit zero; change bit zero
-     * 1  Reference bit zero; change bit one
-     * 2  Reference bit one; change bit zero
-     * 3  Reference bit one; change bit one
-     */
-
-    return re >> 1;
-}
-
-/* compare and swap and purge */
-uint32_t HELPER(csp)(uint32_t r1, uint32_t r2)
-{
-    uint32_t cc;
-    uint32_t o1 = env->regs[r1];
-    uint64_t a2 = get_address_31fix(r2) & ~3ULL;
-    uint32_t o2 = ldl(a2);
-
-    if (o1 == o2) {
-        stl(a2, env->regs[(r1 + 1) & 15]);
-        if (env->regs[r2] & 0x3) {
-            /* flush TLB / ALB */
-            tlb_flush(env, 1);
-        }
-        cc = 0;
-    } else {
-        env->regs[r1] = (env->regs[r1] & 0xffffffff00000000ULL) | o2;
-        cc = 1;
-    }
-
-    return cc;
-}
-
-static uint32_t mvc_asc(int64_t l, uint64_t a1, uint64_t mode1, uint64_t a2,
-                        uint64_t mode2)
-{
-    target_ulong src, dest;
-    int flags, cc = 0, i;
-
-    if (!l) {
-        return 0;
-    } else if (l > 256) {
-        /* max 256 */
-        l = 256;
-        cc = 3;
-    }
-
-    if (mmu_translate(env, a1 & TARGET_PAGE_MASK, 1, mode1, &dest, &flags)) {
-        cpu_loop_exit(env);
-    }
-    dest |= a1 & ~TARGET_PAGE_MASK;
-
-    if (mmu_translate(env, a2 & TARGET_PAGE_MASK, 0, mode2, &src, &flags)) {
-        cpu_loop_exit(env);
-    }
-    src |= a2 & ~TARGET_PAGE_MASK;
-
-    /* XXX replace w/ memcpy */
-    for (i = 0; i < l; i++) {
-        /* XXX be more clever */
-        if ((((dest + i) & TARGET_PAGE_MASK) != (dest & TARGET_PAGE_MASK)) ||
-            (((src + i) & TARGET_PAGE_MASK) != (src & TARGET_PAGE_MASK))) {
-            mvc_asc(l - i, a1 + i, mode1, a2 + i, mode2);
-            break;
-        }
-        stb_phys(dest + i, ldub_phys(src + i));
-    }
-
-    return cc;
-}
-
-uint32_t HELPER(mvcs)(uint64_t l, uint64_t a1, uint64_t a2)
-{
-    HELPER_LOG("%s: %16" PRIx64 " %16" PRIx64 " %16" PRIx64 "\n",
-               __FUNCTION__, l, a1, a2);
-
-    return mvc_asc(l, a1, PSW_ASC_SECONDARY, a2, PSW_ASC_PRIMARY);
-}
-
-uint32_t HELPER(mvcp)(uint64_t l, uint64_t a1, uint64_t a2)
-{
-    HELPER_LOG("%s: %16" PRIx64 " %16" PRIx64 " %16" PRIx64 "\n",
-               __FUNCTION__, l, a1, a2);
-
-    return mvc_asc(l, a1, PSW_ASC_PRIMARY, a2, PSW_ASC_SECONDARY);
-}
-
-uint32_t HELPER(sigp)(uint64_t order_code, uint32_t r1, uint64_t cpu_addr)
-{
-    int cc = 0;
-
-    HELPER_LOG("%s: %016" PRIx64 " %08x %016" PRIx64 "\n",
-               __FUNCTION__, order_code, r1, cpu_addr);
-
-    /* Remember: Use "R1 or R1+1, whichever is the odd-numbered register"
-       as parameter (input). Status (output) is always R1. */
-
-    switch (order_code) {
-    case SIGP_SET_ARCH:
-        /* switch arch */
-        break;
-    case SIGP_SENSE:
-        /* enumerate CPU status */
-        if (cpu_addr) {
-            /* XXX implement when SMP comes */
-            return 3;
-        }
-        env->regs[r1] &= 0xffffffff00000000ULL;
-        cc = 1;
-        break;
-#if !defined (CONFIG_USER_ONLY)
-    case SIGP_RESTART:
-        qemu_system_reset_request();
-        cpu_loop_exit(env);
-        break;
-    case SIGP_STOP:
-        qemu_system_shutdown_request();
-        cpu_loop_exit(env);
-        break;
-#endif
-    default:
-        /* unknown sigp */
-        fprintf(stderr, "XXX unknown sigp: 0x%" PRIx64 "\n", order_code);
-        cc = 3;
-    }
-
-    return cc;
-}
-
-void HELPER(sacf)(uint64_t a1)
-{
-    HELPER_LOG("%s: %16" PRIx64 "\n", __FUNCTION__, a1);
-
-    switch (a1 & 0xf00) {
-    case 0x000:
-        env->psw.mask &= ~PSW_MASK_ASC;
-        env->psw.mask |= PSW_ASC_PRIMARY;
-        break;
-    case 0x100:
-        env->psw.mask &= ~PSW_MASK_ASC;
-        env->psw.mask |= PSW_ASC_SECONDARY;
-        break;
-    case 0x300:
-        env->psw.mask &= ~PSW_MASK_ASC;
-        env->psw.mask |= PSW_ASC_HOME;
-        break;
-    default:
-        qemu_log("unknown sacf mode: %" PRIx64 "\n", a1);
-        program_interrupt(env, PGM_SPECIFICATION, 2);
-        break;
-    }
-}
-
-/* invalidate pte */
-void HELPER(ipte)(uint64_t pte_addr, uint64_t vaddr)
-{
-    uint64_t page = vaddr & TARGET_PAGE_MASK;
-    uint64_t pte = 0;
-
-    /* XXX broadcast to other CPUs */
-
-    /* XXX Linux is nice enough to give us the exact pte address.
-           According to spec we'd have to find it out ourselves */
-    /* XXX Linux is fine with overwriting the pte, the spec requires
-           us to only set the invalid bit */
-    stq_phys(pte_addr, pte | _PAGE_INVALID);
-
-    /* XXX we exploit the fact that Linux passes the exact virtual
-           address here - it's not obliged to! */
-    tlb_flush_page(env, page);
-
-    /* XXX 31-bit hack */
-    if (page & 0x80000000) {
-        tlb_flush_page(env, page & ~0x80000000);
-    } else {
-        tlb_flush_page(env, page | 0x80000000);
-    }
-}
-
-/* flush local tlb */
-void HELPER(ptlb)(void)
-{
-    tlb_flush(env, 1);
-}
-
-/* store using real address */
-void HELPER(stura)(uint64_t addr, uint32_t v1)
-{
-    stw_phys(get_address(0, 0, addr), v1);
-}
-
-/* load real address */
-uint32_t HELPER(lra)(uint64_t addr, uint32_t r1)
-{
-    uint32_t cc = 0;
-    int old_exc = env->exception_index;
-    uint64_t asc = env->psw.mask & PSW_MASK_ASC;
-    uint64_t ret;
-    int flags;
-
-    /* XXX incomplete - has more corner cases */
-    if (!(env->psw.mask & PSW_MASK_64) && (addr >> 32)) {
-        program_interrupt(env, PGM_SPECIAL_OP, 2);
-    }
-
-    env->exception_index = old_exc;
-    if (mmu_translate(env, addr, 0, asc, &ret, &flags)) {
-        cc = 3;
-    }
-    if (env->exception_index == EXCP_PGM) {
-        ret = env->int_pgm_code | 0x80000000;
-    } else {
-        ret |= addr & ~TARGET_PAGE_MASK;
-    }
-    env->exception_index = old_exc;
-
-    if (!(env->psw.mask & PSW_MASK_64)) {
-        env->regs[r1] = (env->regs[r1] & 0xffffffff00000000ULL) | (ret & 0xffffffffULL);
-    } else {
-        env->regs[r1] = ret;
-    }
-
-    return cc;
-}
-
-#endif
diff --git a/target-s390x/translate.c b/target-s390x/translate.c
index 1c1baf534..993f20752 100644
--- a/target-s390x/translate.c
+++ b/target-s390x/translate.c
@@ -79,6 +79,14 @@ void cpu_dump_state(CPUS390XState *env, FILE *f, fprintf_function cpu_fprintf,
 {
     int i;
 
+    if (env->cc_op > 3) {
+        cpu_fprintf(f, "PSW=mask %016" PRIx64 " addr %016" PRIx64 " cc %15s\n",
+                    env->psw.mask, env->psw.addr, cc_name(env->cc_op));
+    } else {
+        cpu_fprintf(f, "PSW=mask %016" PRIx64 " addr %016" PRIx64 " cc %02x\n",
+                    env->psw.mask, env->psw.addr, env->cc_op);
+    }
+
     for (i = 0; i < 16; i++) {
         cpu_fprintf(f, "R%02d=%016" PRIx64, i, env->regs[i]);
         if ((i % 4) == 3) {
@@ -97,8 +105,6 @@ void cpu_dump_state(CPUS390XState *env, FILE *f, fprintf_function cpu_fprintf,
         }
     }
 
-    cpu_fprintf(f, "\n");
-
 #ifndef CONFIG_USER_ONLY
     for (i = 0; i < 16; i++) {
         cpu_fprintf(f, "C%02d=%016" PRIx64, i, env->cregs[i]);
@@ -110,22 +116,14 @@ void cpu_dump_state(CPUS390XState *env, FILE *f, fprintf_function cpu_fprintf,
     }
 #endif
 
-    cpu_fprintf(f, "\n");
-
-    if (env->cc_op > 3) {
-        cpu_fprintf(f, "PSW=mask %016" PRIx64 " addr %016" PRIx64 " cc %15s\n",
-                    env->psw.mask, env->psw.addr, cc_name(env->cc_op));
-    } else {
-        cpu_fprintf(f, "PSW=mask %016" PRIx64 " addr %016" PRIx64 " cc %02x\n",
-                    env->psw.mask, env->psw.addr, env->cc_op);
-    }
-
 #ifdef DEBUG_INLINE_BRANCHES
     for (i = 0; i < CC_OP_MAX; i++) {
         cpu_fprintf(f, "  %15s = %10ld\t%10ld\n", cc_name(i),
                     inline_branch_miss[i], inline_branch_hit[i]);
     }
 #endif
+
+    cpu_fprintf(f, "\n");
 }
 
 static TCGv_i64 psw_addr;
@@ -274,21 +272,21 @@ static inline void potential_page_fault(DisasContext *s)
 #endif
 }
 
-static inline uint64_t ld_code2(uint64_t pc)
+static inline uint64_t ld_code2(CPUS390XState *env, uint64_t pc)
 {
-    return (uint64_t)lduw_code(pc);
+    return (uint64_t)cpu_lduw_code(env, pc);
 }
 
-static inline uint64_t ld_code4(uint64_t pc)
+static inline uint64_t ld_code4(CPUS390XState *env, uint64_t pc)
 {
-    return (uint64_t)ldl_code(pc);
+    return (uint64_t)cpu_ldl_code(env, pc);
 }
 
-static inline uint64_t ld_code6(uint64_t pc)
+static inline uint64_t ld_code6(CPUS390XState *env, uint64_t pc)
 {
     uint64_t opc;
-    opc = (uint64_t)lduw_code(pc) << 32;
-    opc |= (uint64_t)(uint32_t)ldl_code(pc+2);
+    opc = (uint64_t)cpu_lduw_code(env, pc) << 32;
+    opc |= (uint64_t)(uint32_t)cpu_ldl_code(env, pc + 2);
     return opc;
 }
 
@@ -312,39 +310,39 @@ static inline void gen_debug(DisasContext *s)
     TCGv_i32 tmp = tcg_const_i32(EXCP_DEBUG);
     update_psw_addr(s);
     gen_op_calc_cc(s);
-    gen_helper_exception(tmp);
+    gen_helper_exception(cpu_env, tmp);
     tcg_temp_free_i32(tmp);
     s->is_jmp = DISAS_EXCP;
 }
 
 #ifdef CONFIG_USER_ONLY
 
-static void gen_illegal_opcode(DisasContext *s, int ilc)
+static void gen_illegal_opcode(CPUS390XState *env, DisasContext *s, int ilc)
 {
     TCGv_i32 tmp = tcg_const_i32(EXCP_SPEC);
     update_psw_addr(s);
     gen_op_calc_cc(s);
-    gen_helper_exception(tmp);
+    gen_helper_exception(cpu_env, tmp);
     tcg_temp_free_i32(tmp);
     s->is_jmp = DISAS_EXCP;
 }
 
 #else /* CONFIG_USER_ONLY */
 
-static void debug_print_inst(DisasContext *s, int ilc)
+static void debug_print_inst(CPUS390XState *env, DisasContext *s, int ilc)
 {
 #ifdef DEBUG_ILLEGAL_INSTRUCTIONS
     uint64_t inst = 0;
 
     switch (ilc & 3) {
     case 1:
-        inst = ld_code2(s->pc);
+        inst = ld_code2(env, s->pc);
         break;
     case 2:
-        inst = ld_code4(s->pc);
+        inst = ld_code4(env, s->pc);
         break;
     case 3:
-        inst = ld_code6(s->pc);
+        inst = ld_code6(env, s->pc);
         break;
     }
 
@@ -353,11 +351,12 @@ static void debug_print_inst(DisasContext *s, int ilc)
 #endif
 }
 
-static void gen_program_exception(DisasContext *s, int ilc, int code)
+static void gen_program_exception(CPUS390XState *env, DisasContext *s, int ilc,
+                                  int code)
 {
     TCGv_i32 tmp;
 
-    debug_print_inst(s, ilc);
+    debug_print_inst(env, s, ilc);
 
     /* remember what pgm exeption this was */
     tmp = tcg_const_i32(code);
@@ -377,7 +376,7 @@ static void gen_program_exception(DisasContext *s, int ilc, int code)
 
     /* trigger exception */
     tmp = tcg_const_i32(EXCP_PGM);
-    gen_helper_exception(tmp);
+    gen_helper_exception(cpu_env, tmp);
     tcg_temp_free_i32(tmp);
 
     /* end TB here */
@@ -385,20 +384,21 @@ static void gen_program_exception(DisasContext *s, int ilc, int code)
 }
 
 
-static void gen_illegal_opcode(DisasContext *s, int ilc)
+static void gen_illegal_opcode(CPUS390XState *env, DisasContext *s, int ilc)
 {
-    gen_program_exception(s, ilc, PGM_SPECIFICATION);
+    gen_program_exception(env, s, ilc, PGM_SPECIFICATION);
 }
 
-static void gen_privileged_exception(DisasContext *s, int ilc)
+static void gen_privileged_exception(CPUS390XState *env, DisasContext *s,
+                                     int ilc)
 {
-    gen_program_exception(s, ilc, PGM_PRIVILEGED);
+    gen_program_exception(env, s, ilc, PGM_PRIVILEGED);
 }
 
-static void check_privileged(DisasContext *s, int ilc)
+static void check_privileged(CPUS390XState *env, DisasContext *s, int ilc)
 {
     if (s->tb->flags & (PSW_MASK_PSTATE >> 32)) {
-        gen_privileged_exception(s, ilc);
+        gen_privileged_exception(env, s, ilc);
     }
 }
 
@@ -667,16 +667,11 @@ static void set_cc_cmp_f32_i64(DisasContext *s, TCGv_i32 v1, TCGv_i64 v2)
     s->cc_op = CC_OP_LTGT_F32;
 }
 
-static void set_cc_nz_f32(DisasContext *s, TCGv_i32 v1)
+static void gen_set_cc_nz_f32(DisasContext *s, TCGv_i32 v1)
 {
     gen_op_update1_cc_i32(s, CC_OP_NZ_F32, v1);
 }
 
-static inline void set_cc_nz_f64(DisasContext *s, TCGv_i64 v1)
-{
-    gen_op_update1_cc_i64(s, CC_OP_NZ_F64, v1);
-}
-
 /* CC value is in env->cc_op */
 static inline void set_cc_static(DisasContext *s)
 {
@@ -727,7 +722,7 @@ static void gen_op_calc_cc(DisasContext *s)
     case CC_OP_NZ_F32:
     case CC_OP_NZ_F64:
         /* 1 argument */
-        gen_helper_calc_cc(cc_op, local_cc_op, dummy, cc_dst, dummy);
+        gen_helper_calc_cc(cc_op, cpu_env, local_cc_op, dummy, cc_dst, dummy);
         break;
     case CC_OP_ICM:
     case CC_OP_LTGT_32:
@@ -740,7 +735,7 @@ static void gen_op_calc_cc(DisasContext *s)
     case CC_OP_LTGT_F64:
     case CC_OP_SLAG:
         /* 2 arguments */
-        gen_helper_calc_cc(cc_op, local_cc_op, cc_src, cc_dst, dummy);
+        gen_helper_calc_cc(cc_op, cpu_env, local_cc_op, cc_src, cc_dst, dummy);
         break;
     case CC_OP_ADD_64:
     case CC_OP_ADDU_64:
@@ -751,11 +746,11 @@ static void gen_op_calc_cc(DisasContext *s)
     case CC_OP_SUB_32:
     case CC_OP_SUBU_32:
         /* 3 arguments */
-        gen_helper_calc_cc(cc_op, local_cc_op, cc_src, cc_dst, cc_vr);
+        gen_helper_calc_cc(cc_op, cpu_env, local_cc_op, cc_src, cc_dst, cc_vr);
         break;
     case CC_OP_DYNAMIC:
         /* unknown operation - assume 3 arguments and cc_op in env */
-        gen_helper_calc_cc(cc_op, cc_op, cc_src, cc_dst, cc_vr);
+        gen_helper_calc_cc(cc_op, cpu_env, cc_op, cc_src, cc_dst, cc_vr);
         break;
     default:
         tcg_abort();
@@ -1268,7 +1263,7 @@ static void gen_op_mvc(DisasContext *s, int l, TCGv_i64 s1, TCGv_i64 s2)
         /* Fall back to helper */
         vl = tcg_const_i32(l);
         potential_page_fault(s);
-        gen_helper_mvc(vl, s1, s2);
+        gen_helper_mvc(cpu_env, vl, s1, s2);
         tcg_temp_free_i32(vl);
         return;
     }
@@ -1460,12 +1455,13 @@ static void gen_op_clc(DisasContext *s, int l, TCGv_i64 s1, TCGv_i64 s2)
 
     potential_page_fault(s);
     vl = tcg_const_i32(l);
-    gen_helper_clc(cc_op, vl, s1, s2);
+    gen_helper_clc(cc_op, cpu_env, vl, s1, s2);
     tcg_temp_free_i32(vl);
     set_cc_static(s);
 }
 
-static void disas_e3(DisasContext* s, int op, int r1, int x2, int b2, int d2)
+static void disas_e3(CPUS390XState *env, DisasContext* s, int op, int r1,
+                     int x2, int b2, int d2)
 {
     TCGv_i64 addr, tmp, tmp2, tmp3, tmp4;
     TCGv_i32 tmp32_1, tmp32_2, tmp32_3;
@@ -1808,7 +1804,7 @@ static void disas_e3(DisasContext* s, int op, int r1, int x2, int b2, int d2)
         tmp2 = tcg_temp_new_i64();
         tmp32_1 = tcg_const_i32(r1);
         tcg_gen_qemu_ld64(tmp2, addr, get_mem_index(s));
-        gen_helper_mlg(tmp32_1, tmp2);
+        gen_helper_mlg(cpu_env, tmp32_1, tmp2);
         tcg_temp_free_i64(tmp2);
         tcg_temp_free_i32(tmp32_1);
         break;
@@ -1816,7 +1812,7 @@ static void disas_e3(DisasContext* s, int op, int r1, int x2, int b2, int d2)
         tmp2 = tcg_temp_new_i64();
         tmp32_1 = tcg_const_i32(r1);
         tcg_gen_qemu_ld64(tmp2, addr, get_mem_index(s));
-        gen_helper_dlg(tmp32_1, tmp2);
+        gen_helper_dlg(cpu_env, tmp32_1, tmp2);
         tcg_temp_free_i64(tmp2);
         tcg_temp_free_i32(tmp32_1);
         break;
@@ -1842,7 +1838,7 @@ static void disas_e3(DisasContext* s, int op, int r1, int x2, int b2, int d2)
         tcg_gen_qemu_ld64(tmp2, addr, get_mem_index(s));
         /* XXX possible optimization point */
         gen_op_calc_cc(s);
-        gen_helper_slbg(cc_op, cc_op, tmp32_1, regs[r1], tmp2);
+        gen_helper_slbg(cc_op, cpu_env, cc_op, tmp32_1, regs[r1], tmp2);
         set_cc_static(s);
         tcg_temp_free_i64(tmp2);
         tcg_temp_free_i32(tmp32_1);
@@ -1922,7 +1918,7 @@ static void disas_e3(DisasContext* s, int op, int r1, int x2, int b2, int d2)
         tcg_gen_trunc_i64_i32(tmp32_2, tmp2);
         /* XXX possible optimization point */
         gen_op_calc_cc(s);
-        gen_helper_slb(cc_op, cc_op, tmp32_1, tmp32_2);
+        gen_helper_slb(cc_op, cpu_env, cc_op, tmp32_1, tmp32_2);
         set_cc_static(s);
         tcg_temp_free_i64(tmp2);
         tcg_temp_free_i32(tmp32_1);
@@ -1930,14 +1926,14 @@ static void disas_e3(DisasContext* s, int op, int r1, int x2, int b2, int d2)
         break;
     default:
         LOG_DISAS("illegal e3 operation 0x%x\n", op);
-        gen_illegal_opcode(s, 3);
+        gen_illegal_opcode(env, s, 3);
         break;
     }
     tcg_temp_free_i64(addr);
 }
 
 #ifndef CONFIG_USER_ONLY
-static void disas_e5(DisasContext* s, uint64_t insn)
+static void disas_e5(CPUS390XState *env, DisasContext* s, uint64_t insn)
 {
     TCGv_i64 tmp, tmp2;
     int op = (insn >> 32) & 0xff;
@@ -1955,7 +1951,7 @@ static void disas_e5(DisasContext* s, uint64_t insn)
         break;
     default:
         LOG_DISAS("illegal e5 operation 0x%x\n", op);
-        gen_illegal_opcode(s, 3);
+        gen_illegal_opcode(env, s, 3);
         break;
     }
 
@@ -1964,7 +1960,8 @@ static void disas_e5(DisasContext* s, uint64_t insn)
 }
 #endif
 
-static void disas_eb(DisasContext *s, int op, int r1, int r3, int b2, int d2)
+static void disas_eb(CPUS390XState *env, DisasContext *s, int op, int r1,
+                     int r3, int b2, int d2)
 {
     TCGv_i64 tmp, tmp2, tmp3, tmp4;
     TCGv_i32 tmp32_1, tmp32_2;
@@ -2099,7 +2096,7 @@ do_mh:
         tmp32_1 = tcg_const_i32(r1);
         tmp32_2 = tcg_const_i32(r3);
         potential_page_fault(s);
-        gen_helper_stcmh(tmp32_1, tmp, tmp32_2);
+        gen_helper_stcmh(cpu_env, tmp32_1, tmp, tmp32_2);
         tcg_temp_free_i64(tmp);
         tcg_temp_free_i32(tmp32_1);
         tcg_temp_free_i32(tmp32_2);
@@ -2107,24 +2104,24 @@ do_mh:
 #ifndef CONFIG_USER_ONLY
     case 0x2f: /* LCTLG     R1,R3,D2(B2)     [RSE] */
         /* Load Control */
-        check_privileged(s, ilc);
+        check_privileged(env, s, ilc);
         tmp = get_address(s, 0, b2, d2);
         tmp32_1 = tcg_const_i32(r1);
         tmp32_2 = tcg_const_i32(r3);
         potential_page_fault(s);
-        gen_helper_lctlg(tmp32_1, tmp, tmp32_2);
+        gen_helper_lctlg(cpu_env, tmp32_1, tmp, tmp32_2);
         tcg_temp_free_i64(tmp);
         tcg_temp_free_i32(tmp32_1);
         tcg_temp_free_i32(tmp32_2);
         break;
     case 0x25: /* STCTG     R1,R3,D2(B2)     [RSE] */
         /* Store Control */
-        check_privileged(s, ilc);
+        check_privileged(env, s, ilc);
         tmp = get_address(s, 0, b2, d2);
         tmp32_1 = tcg_const_i32(r1);
         tmp32_2 = tcg_const_i32(r3);
         potential_page_fault(s);
-        gen_helper_stctg(tmp32_1, tmp, tmp32_2);
+        gen_helper_stctg(cpu_env, tmp32_1, tmp, tmp32_2);
         tcg_temp_free_i64(tmp);
         tcg_temp_free_i32(tmp32_1);
         tcg_temp_free_i32(tmp32_2);
@@ -2136,7 +2133,7 @@ do_mh:
         tmp32_2 = tcg_const_i32(r3);
         potential_page_fault(s);
         /* XXX rewrite in tcg */
-        gen_helper_csg(cc_op, tmp32_1, tmp, tmp32_2);
+        gen_helper_csg(cc_op, cpu_env, tmp32_1, tmp, tmp32_2);
         set_cc_static(s);
         tcg_temp_free_i64(tmp);
         tcg_temp_free_i32(tmp32_1);
@@ -2148,7 +2145,7 @@ do_mh:
         tmp32_2 = tcg_const_i32(r3);
         potential_page_fault(s);
         /* XXX rewrite in tcg */
-        gen_helper_cdsg(cc_op, tmp32_1, tmp, tmp32_2);
+        gen_helper_cdsg(cc_op, cpu_env, tmp32_1, tmp, tmp32_2);
         set_cc_static(s);
         tcg_temp_free_i64(tmp);
         tcg_temp_free_i32(tmp32_1);
@@ -2188,7 +2185,7 @@ do_mh:
         tmp32_2 = tcg_const_i32(r3);
         potential_page_fault(s);
         /* XXX split CC calculation out */
-        gen_helper_icmh(cc_op, tmp32_1, tmp, tmp32_2);
+        gen_helper_icmh(cc_op, cpu_env, tmp32_1, tmp, tmp32_2);
         set_cc_static(s);
         tcg_temp_free_i64(tmp);
         tcg_temp_free_i32(tmp32_1);
@@ -2196,13 +2193,13 @@ do_mh:
         break;
     default:
         LOG_DISAS("illegal eb operation 0x%x\n", op);
-        gen_illegal_opcode(s, ilc);
+        gen_illegal_opcode(env, s, ilc);
         break;
     }
 }
 
-static void disas_ed(DisasContext *s, int op, int r1, int x2, int b2, int d2,
-                     int r1b)
+static void disas_ed(CPUS390XState *env, DisasContext *s, int op, int r1,
+                     int x2, int b2, int d2, int r1b)
 {
     TCGv_i32 tmp_r1, tmp32;
     TCGv_i64 addr, tmp;
@@ -2211,11 +2208,11 @@ static void disas_ed(DisasContext *s, int op, int r1, int x2, int b2, int d2,
     switch (op) {
     case 0x4: /* LDEB R1,D2(X2,B2) [RXE] */
         potential_page_fault(s);
-        gen_helper_ldeb(tmp_r1, addr);
+        gen_helper_ldeb(cpu_env, tmp_r1, addr);
         break;
     case 0x5: /* LXDB R1,D2(X2,B2) [RXE] */
         potential_page_fault(s);
-        gen_helper_lxdb(tmp_r1, addr);
+        gen_helper_lxdb(cpu_env, tmp_r1, addr);
         break;
     case 0x9: /* CEB    R1,D2(X2,B2)       [RXE] */
         tmp = tcg_temp_new_i64();
@@ -2230,12 +2227,12 @@ static void disas_ed(DisasContext *s, int op, int r1, int x2, int b2, int d2,
         tmp32 = tcg_temp_new_i32();
         tcg_gen_qemu_ld32u(tmp, addr, get_mem_index(s));
         tcg_gen_trunc_i64_i32(tmp32, tmp);
-        gen_helper_aeb(tmp_r1, tmp32);
+        gen_helper_aeb(cpu_env, tmp_r1, tmp32);
         tcg_temp_free_i64(tmp);
         tcg_temp_free_i32(tmp32);
 
         tmp32 = load_freg32(r1);
-        set_cc_nz_f32(s, tmp32);
+        gen_set_cc_nz_f32(s, tmp32);
         tcg_temp_free_i32(tmp32);
         break;
     case 0xb: /* SEB    R1,D2(X2,B2)       [RXE] */
@@ -2243,12 +2240,12 @@ static void disas_ed(DisasContext *s, int op, int r1, int x2, int b2, int d2,
         tmp32 = tcg_temp_new_i32();
         tcg_gen_qemu_ld32u(tmp, addr, get_mem_index(s));
         tcg_gen_trunc_i64_i32(tmp32, tmp);
-        gen_helper_seb(tmp_r1, tmp32);
+        gen_helper_seb(cpu_env, tmp_r1, tmp32);
         tcg_temp_free_i64(tmp);
         tcg_temp_free_i32(tmp32);
 
         tmp32 = load_freg32(r1);
-        set_cc_nz_f32(s, tmp32);
+        gen_set_cc_nz_f32(s, tmp32);
         tcg_temp_free_i32(tmp32);
         break;
     case 0xd: /* DEB    R1,D2(X2,B2)       [RXE] */
@@ -2256,23 +2253,23 @@ static void disas_ed(DisasContext *s, int op, int r1, int x2, int b2, int d2,
         tmp32 = tcg_temp_new_i32();
         tcg_gen_qemu_ld32u(tmp, addr, get_mem_index(s));
         tcg_gen_trunc_i64_i32(tmp32, tmp);
-        gen_helper_deb(tmp_r1, tmp32);
+        gen_helper_deb(cpu_env, tmp_r1, tmp32);
         tcg_temp_free_i64(tmp);
         tcg_temp_free_i32(tmp32);
         break;
     case 0x10: /* TCEB   R1,D2(X2,B2)       [RXE] */
         potential_page_fault(s);
-        gen_helper_tceb(cc_op, tmp_r1, addr);
+        gen_helper_tceb(cc_op, cpu_env, tmp_r1, addr);
         set_cc_static(s);
         break;
     case 0x11: /* TCDB   R1,D2(X2,B2)       [RXE] */
         potential_page_fault(s);
-        gen_helper_tcdb(cc_op, tmp_r1, addr);
+        gen_helper_tcdb(cc_op, cpu_env, tmp_r1, addr);
         set_cc_static(s);
         break;
     case 0x12: /* TCXB   R1,D2(X2,B2)       [RXE] */
         potential_page_fault(s);
-        gen_helper_tcxb(cc_op, tmp_r1, addr);
+        gen_helper_tcxb(cc_op, cpu_env, tmp_r1, addr);
         set_cc_static(s);
         break;
     case 0x17: /* MEEB   R1,D2(X2,B2)       [RXE] */
@@ -2280,50 +2277,51 @@ static void disas_ed(DisasContext *s, int op, int r1, int x2, int b2, int d2,
         tmp32 = tcg_temp_new_i32();
         tcg_gen_qemu_ld32u(tmp, addr, get_mem_index(s));
         tcg_gen_trunc_i64_i32(tmp32, tmp);
-        gen_helper_meeb(tmp_r1, tmp32);
+        gen_helper_meeb(cpu_env, tmp_r1, tmp32);
         tcg_temp_free_i64(tmp);
         tcg_temp_free_i32(tmp32);
         break;
     case 0x19: /* CDB    R1,D2(X2,B2)       [RXE] */
         potential_page_fault(s);
-        gen_helper_cdb(cc_op, tmp_r1, addr);
+        gen_helper_cdb(cc_op, cpu_env, tmp_r1, addr);
         set_cc_static(s);
         break;
     case 0x1a: /* ADB    R1,D2(X2,B2)       [RXE] */
         potential_page_fault(s);
-        gen_helper_adb(cc_op, tmp_r1, addr);
+        gen_helper_adb(cc_op, cpu_env, tmp_r1, addr);
         set_cc_static(s);
         break;
     case 0x1b: /* SDB    R1,D2(X2,B2)       [RXE] */
         potential_page_fault(s);
-        gen_helper_sdb(cc_op, tmp_r1, addr);
+        gen_helper_sdb(cc_op, cpu_env, tmp_r1, addr);
         set_cc_static(s);
         break;
     case 0x1c: /* MDB    R1,D2(X2,B2)       [RXE] */
         potential_page_fault(s);
-        gen_helper_mdb(tmp_r1, addr);
+        gen_helper_mdb(cpu_env, tmp_r1, addr);
         break;
     case 0x1d: /* DDB    R1,D2(X2,B2)       [RXE] */
         potential_page_fault(s);
-        gen_helper_ddb(tmp_r1, addr);
+        gen_helper_ddb(cpu_env, tmp_r1, addr);
         break;
     case 0x1e: /* MADB  R1,R3,D2(X2,B2) [RXF] */
         /* for RXF insns, r1 is R3 and r1b is R1 */
         tmp32 = tcg_const_i32(r1b);
         potential_page_fault(s);
-        gen_helper_madb(tmp32, addr, tmp_r1);
+        gen_helper_madb(cpu_env, tmp32, addr, tmp_r1);
         tcg_temp_free_i32(tmp32);
         break;
     default:
         LOG_DISAS("illegal ed operation 0x%x\n", op);
-        gen_illegal_opcode(s, 3);
+        gen_illegal_opcode(env, s, 3);
         return;
     }
     tcg_temp_free_i32(tmp_r1);
     tcg_temp_free_i64(addr);
 }
 
-static void disas_a5(DisasContext *s, int op, int r1, int i2)
+static void disas_a5(CPUS390XState *env, DisasContext *s, int op, int r1,
+                     int i2)
 {
     TCGv_i64 tmp, tmp2;
     TCGv_i32 tmp32;
@@ -2472,12 +2470,13 @@ static void disas_a5(DisasContext *s, int op, int r1, int i2)
         break;
     default:
         LOG_DISAS("illegal a5 operation 0x%x\n", op);
-        gen_illegal_opcode(s, 2);
+        gen_illegal_opcode(env, s, 2);
         return;
     }
 }
 
-static void disas_a7(DisasContext *s, int op, int r1, int i2)
+static void disas_a7(CPUS390XState *env, DisasContext *s, int op, int r1,
+                     int i2)
 {
     TCGv_i64 tmp, tmp2;
     TCGv_i32 tmp32_1, tmp32_2, tmp32_3;
@@ -2609,12 +2608,13 @@ static void disas_a7(DisasContext *s, int op, int r1, int i2)
         break;
     default:
         LOG_DISAS("illegal a7 operation 0x%x\n", op);
-        gen_illegal_opcode(s, 2);
+        gen_illegal_opcode(env, s, 2);
         return;
     }
 }
 
-static void disas_b2(DisasContext *s, int op, uint32_t insn)
+static void disas_b2(CPUS390XState *env, DisasContext *s, int op,
+                     uint32_t insn)
 {
     TCGv_i64 tmp, tmp2, tmp3;
     TCGv_i32 tmp32_1, tmp32_2, tmp32_3;
@@ -2633,14 +2633,14 @@ static void disas_b2(DisasContext *s, int op, uint32_t insn)
     case 0x22: /* IPM    R1               [RRE] */
         tmp32_1 = tcg_const_i32(r1);
         gen_op_calc_cc(s);
-        gen_helper_ipm(cc_op, tmp32_1);
+        gen_helper_ipm(cpu_env, cc_op, tmp32_1);
         tcg_temp_free_i32(tmp32_1);
         break;
     case 0x41: /* CKSM    R1,R2     [RRE] */
         tmp32_1 = tcg_const_i32(r1);
         tmp32_2 = tcg_const_i32(r2);
         potential_page_fault(s);
-        gen_helper_cksm(tmp32_1, tmp32_2);
+        gen_helper_cksm(cpu_env, tmp32_1, tmp32_2);
         tcg_temp_free_i32(tmp32_1);
         tcg_temp_free_i32(tmp32_2);
         gen_op_movi_cc(s, 0);
@@ -2669,7 +2669,7 @@ static void disas_b2(DisasContext *s, int op, uint32_t insn)
         tmp2 = load_reg(r1);
         tmp3 = load_reg(r2);
         potential_page_fault(s);
-        gen_helper_mvpg(tmp, tmp2, tmp3);
+        gen_helper_mvpg(cpu_env, tmp, tmp2, tmp3);
         tcg_temp_free_i64(tmp);
         tcg_temp_free_i64(tmp2);
         tcg_temp_free_i64(tmp3);
@@ -2681,7 +2681,7 @@ static void disas_b2(DisasContext *s, int op, uint32_t insn)
         tmp32_2 = tcg_const_i32(r1);
         tmp32_3 = tcg_const_i32(r2);
         potential_page_fault(s);
-        gen_helper_mvst(tmp32_1, tmp32_2, tmp32_3);
+        gen_helper_mvst(cpu_env, tmp32_1, tmp32_2, tmp32_3);
         tcg_temp_free_i32(tmp32_1);
         tcg_temp_free_i32(tmp32_2);
         tcg_temp_free_i32(tmp32_3);
@@ -2692,7 +2692,7 @@ static void disas_b2(DisasContext *s, int op, uint32_t insn)
         tmp32_2 = tcg_const_i32(r1);
         tmp32_3 = tcg_const_i32(r2);
         potential_page_fault(s);
-        gen_helper_clst(cc_op, tmp32_1, tmp32_2, tmp32_3);
+        gen_helper_clst(cc_op, cpu_env, tmp32_1, tmp32_2, tmp32_3);
         set_cc_static(s);
         tcg_temp_free_i32(tmp32_1);
         tcg_temp_free_i32(tmp32_2);
@@ -2703,7 +2703,7 @@ static void disas_b2(DisasContext *s, int op, uint32_t insn)
         tmp32_2 = tcg_const_i32(r1);
         tmp32_3 = tcg_const_i32(r2);
         potential_page_fault(s);
-        gen_helper_srst(cc_op, tmp32_1, tmp32_2, tmp32_3);
+        gen_helper_srst(cc_op, cpu_env, tmp32_1, tmp32_2, tmp32_3);
         set_cc_static(s);
         tcg_temp_free_i32(tmp32_1);
         tcg_temp_free_i32(tmp32_2);
@@ -2713,16 +2713,16 @@ static void disas_b2(DisasContext *s, int op, uint32_t insn)
 #ifndef CONFIG_USER_ONLY
     case 0x02: /* STIDP     D2(B2)     [S] */
         /* Store CPU ID */
-        check_privileged(s, ilc);
+        check_privileged(env, s, ilc);
         decode_rs(s, insn, &r1, &r3, &b2, &d2);
         tmp = get_address(s, 0, b2, d2);
         potential_page_fault(s);
-        gen_helper_stidp(tmp);
+        gen_helper_stidp(cpu_env, tmp);
         tcg_temp_free_i64(tmp);
         break;
     case 0x04: /* SCK       D2(B2)     [S] */
         /* Set Clock */
-        check_privileged(s, ilc);
+        check_privileged(env, s, ilc);
         decode_rs(s, insn, &r1, &r3, &b2, &d2);
         tmp = get_address(s, 0, b2, d2);
         potential_page_fault(s);
@@ -2735,49 +2735,49 @@ static void disas_b2(DisasContext *s, int op, uint32_t insn)
         decode_rs(s, insn, &r1, &r3, &b2, &d2);
         tmp = get_address(s, 0, b2, d2);
         potential_page_fault(s);
-        gen_helper_stck(cc_op, tmp);
+        gen_helper_stck(cc_op, cpu_env, tmp);
         set_cc_static(s);
         tcg_temp_free_i64(tmp);
         break;
     case 0x06: /* SCKC     D2(B2)     [S] */
         /* Set Clock Comparator */
-        check_privileged(s, ilc);
+        check_privileged(env, s, ilc);
         decode_rs(s, insn, &r1, &r3, &b2, &d2);
         tmp = get_address(s, 0, b2, d2);
         potential_page_fault(s);
-        gen_helper_sckc(tmp);
+        gen_helper_sckc(cpu_env, tmp);
         tcg_temp_free_i64(tmp);
         break;
     case 0x07: /* STCKC    D2(B2)     [S] */
         /* Store Clock Comparator */
-        check_privileged(s, ilc);
+        check_privileged(env, s, ilc);
         decode_rs(s, insn, &r1, &r3, &b2, &d2);
         tmp = get_address(s, 0, b2, d2);
         potential_page_fault(s);
-        gen_helper_stckc(tmp);
+        gen_helper_stckc(cpu_env, tmp);
         tcg_temp_free_i64(tmp);
         break;
     case 0x08: /* SPT      D2(B2)     [S] */
         /* Set CPU Timer */
-        check_privileged(s, ilc);
+        check_privileged(env, s, ilc);
         decode_rs(s, insn, &r1, &r3, &b2, &d2);
         tmp = get_address(s, 0, b2, d2);
         potential_page_fault(s);
-        gen_helper_spt(tmp);
+        gen_helper_spt(cpu_env, tmp);
         tcg_temp_free_i64(tmp);
         break;
     case 0x09: /* STPT     D2(B2)     [S] */
         /* Store CPU Timer */
-        check_privileged(s, ilc);
+        check_privileged(env, s, ilc);
         decode_rs(s, insn, &r1, &r3, &b2, &d2);
         tmp = get_address(s, 0, b2, d2);
         potential_page_fault(s);
-        gen_helper_stpt(tmp);
+        gen_helper_stpt(cpu_env, tmp);
         tcg_temp_free_i64(tmp);
         break;
     case 0x0a: /* SPKA     D2(B2)     [S] */
         /* Set PSW Key from Address */
-        check_privileged(s, ilc);
+        check_privileged(env, s, ilc);
         decode_rs(s, insn, &r1, &r3, &b2, &d2);
         tmp = get_address(s, 0, b2, d2);
         tmp2 = tcg_temp_new_i64();
@@ -2789,21 +2789,21 @@ static void disas_b2(DisasContext *s, int op, uint32_t insn)
         break;
     case 0x0d: /* PTLB                [S] */
         /* Purge TLB */
-        check_privileged(s, ilc);
-        gen_helper_ptlb();
+        check_privileged(env, s, ilc);
+        gen_helper_ptlb(cpu_env);
         break;
     case 0x10: /* SPX      D2(B2)     [S] */
         /* Set Prefix Register */
-        check_privileged(s, ilc);
+        check_privileged(env, s, ilc);
         decode_rs(s, insn, &r1, &r3, &b2, &d2);
         tmp = get_address(s, 0, b2, d2);
         potential_page_fault(s);
-        gen_helper_spx(tmp);
+        gen_helper_spx(cpu_env, tmp);
         tcg_temp_free_i64(tmp);
         break;
     case 0x11: /* STPX     D2(B2)     [S] */
         /* Store Prefix */
-        check_privileged(s, ilc);
+        check_privileged(env, s, ilc);
         decode_rs(s, insn, &r1, &r3, &b2, &d2);
         tmp = get_address(s, 0, b2, d2);
         tmp2 = tcg_temp_new_i64();
@@ -2814,7 +2814,7 @@ static void disas_b2(DisasContext *s, int op, uint32_t insn)
         break;
     case 0x12: /* STAP     D2(B2)     [S] */
         /* Store CPU Address */
-        check_privileged(s, ilc);
+        check_privileged(env, s, ilc);
         decode_rs(s, insn, &r1, &r3, &b2, &d2);
         tmp = get_address(s, 0, b2, d2);
         tmp2 = tcg_temp_new_i64();
@@ -2828,82 +2828,82 @@ static void disas_b2(DisasContext *s, int op, uint32_t insn)
         break;
     case 0x21: /* IPTE     R1,R2      [RRE] */
         /* Invalidate PTE */
-        check_privileged(s, ilc);
+        check_privileged(env, s, ilc);
         r1 = (insn >> 4) & 0xf;
         r2 = insn & 0xf;
         tmp = load_reg(r1);
         tmp2 = load_reg(r2);
-        gen_helper_ipte(tmp, tmp2);
+        gen_helper_ipte(cpu_env, tmp, tmp2);
         tcg_temp_free_i64(tmp);
         tcg_temp_free_i64(tmp2);
         break;
     case 0x29: /* ISKE     R1,R2      [RRE] */
         /* Insert Storage Key Extended */
-        check_privileged(s, ilc);
+        check_privileged(env, s, ilc);
         r1 = (insn >> 4) & 0xf;
         r2 = insn & 0xf;
         tmp = load_reg(r2);
         tmp2 = tcg_temp_new_i64();
-        gen_helper_iske(tmp2, tmp);
+        gen_helper_iske(tmp2, cpu_env, tmp);
         store_reg(r1, tmp2);
         tcg_temp_free_i64(tmp);
         tcg_temp_free_i64(tmp2);
         break;
     case 0x2a: /* RRBE     R1,R2      [RRE] */
         /* Set Storage Key Extended */
-        check_privileged(s, ilc);
+        check_privileged(env, s, ilc);
         r1 = (insn >> 4) & 0xf;
         r2 = insn & 0xf;
         tmp32_1 = load_reg32(r1);
         tmp = load_reg(r2);
-        gen_helper_rrbe(cc_op, tmp32_1, tmp);
+        gen_helper_rrbe(cc_op, cpu_env, tmp32_1, tmp);
         set_cc_static(s);
         tcg_temp_free_i32(tmp32_1);
         tcg_temp_free_i64(tmp);
         break;
     case 0x2b: /* SSKE     R1,R2      [RRE] */
         /* Set Storage Key Extended */
-        check_privileged(s, ilc);
+        check_privileged(env, s, ilc);
         r1 = (insn >> 4) & 0xf;
         r2 = insn & 0xf;
         tmp32_1 = load_reg32(r1);
         tmp = load_reg(r2);
-        gen_helper_sske(tmp32_1, tmp);
+        gen_helper_sske(cpu_env, tmp32_1, tmp);
         tcg_temp_free_i32(tmp32_1);
         tcg_temp_free_i64(tmp);
         break;
     case 0x34: /* STCH ? */
         /* Store Subchannel */
-        check_privileged(s, ilc);
+        check_privileged(env, s, ilc);
         gen_op_movi_cc(s, 3);
         break;
     case 0x46: /* STURA    R1,R2      [RRE] */
         /* Store Using Real Address */
-        check_privileged(s, ilc);
+        check_privileged(env, s, ilc);
         r1 = (insn >> 4) & 0xf;
         r2 = insn & 0xf;
         tmp32_1 = load_reg32(r1);
         tmp = load_reg(r2);
         potential_page_fault(s);
-        gen_helper_stura(tmp, tmp32_1);
+        gen_helper_stura(cpu_env, tmp, tmp32_1);
         tcg_temp_free_i32(tmp32_1);
         tcg_temp_free_i64(tmp);
         break;
     case 0x50: /* CSP      R1,R2      [RRE] */
         /* Compare And Swap And Purge */
-        check_privileged(s, ilc);
+        check_privileged(env, s, ilc);
         r1 = (insn >> 4) & 0xf;
         r2 = insn & 0xf;
         tmp32_1 = tcg_const_i32(r1);
         tmp32_2 = tcg_const_i32(r2);
-        gen_helper_csp(cc_op, tmp32_1, tmp32_2);
+        gen_helper_csp(cc_op, cpu_env, tmp32_1, tmp32_2);
         set_cc_static(s);
         tcg_temp_free_i32(tmp32_1);
         tcg_temp_free_i32(tmp32_2);
         break;
     case 0x5f: /* CHSC ? */
         /* Channel Subsystem Call */
-        check_privileged(s, ilc);
+        check_privileged(env, s, ilc);
         gen_op_movi_cc(s, 3);
         break;
     case 0x78: /* STCKE    D2(B2)     [S] */
@@ -2911,17 +2911,17 @@ static void disas_b2(DisasContext *s, int op, uint32_t insn)
         decode_rs(s, insn, &r1, &r3, &b2, &d2);
         tmp = get_address(s, 0, b2, d2);
         potential_page_fault(s);
-        gen_helper_stcke(cc_op, tmp);
+        gen_helper_stcke(cc_op, cpu_env, tmp);
         set_cc_static(s);
         tcg_temp_free_i64(tmp);
         break;
     case 0x79: /* SACF    D2(B2)     [S] */
         /* Store Clock Extended */
-        check_privileged(s, ilc);
+        check_privileged(env, s, ilc);
         decode_rs(s, insn, &r1, &r3, &b2, &d2);
         tmp = get_address(s, 0, b2, d2);
         potential_page_fault(s);
-        gen_helper_sacf(tmp);
+        gen_helper_sacf(cpu_env, tmp);
         tcg_temp_free_i64(tmp);
         /* addressing mode has changed, so end the block */
         s->pc += ilc * 2;
@@ -2929,13 +2929,13 @@ static void disas_b2(DisasContext *s, int op, uint32_t insn)
         s->is_jmp = DISAS_EXCP;
         break;
     case 0x7d: /* STSI     D2,(B2)     [S] */
-        check_privileged(s, ilc);
+        check_privileged(env, s, ilc);
         decode_rs(s, insn, &r1, &r3, &b2, &d2);
         tmp = get_address(s, 0, b2, d2);
         tmp32_1 = load_reg32(0);
         tmp32_2 = load_reg32(1);
         potential_page_fault(s);
-        gen_helper_stsi(cc_op, tmp, tmp32_1, tmp32_2);
+        gen_helper_stsi(cc_op, cpu_env, tmp, tmp32_1, tmp32_2);
         set_cc_static(s);
         tcg_temp_free_i64(tmp);
         tcg_temp_free_i32(tmp32_1);
@@ -2955,7 +2955,7 @@ static void disas_b2(DisasContext *s, int op, uint32_t insn)
         break;
     case 0xb1: /* STFL     D2(B2)     [S] */
         /* Store Facility List (CPU features) at 200 */
-        check_privileged(s, ilc);
+        check_privileged(env, s, ilc);
         tmp2 = tcg_const_i64(0xc0000000);
         tmp = tcg_const_i64(200);
         tcg_gen_qemu_st32(tmp2, tmp, get_mem_index(s));
@@ -2964,7 +2964,7 @@ static void disas_b2(DisasContext *s, int op, uint32_t insn)
         break;
     case 0xb2: /* LPSWE    D2(B2)     [S] */
         /* Load PSW Extended */
-        check_privileged(s, ilc);
+        check_privileged(env, s, ilc);
         decode_rs(s, insn, &r1, &r3, &b2, &d2);
         tmp = get_address(s, 0, b2, d2);
         tmp2 = tcg_temp_new_i64();
@@ -2972,7 +2972,7 @@ static void disas_b2(DisasContext *s, int op, uint32_t insn)
         tcg_gen_qemu_ld64(tmp2, tmp, get_mem_index(s));
         tcg_gen_addi_i64(tmp, tmp, 8);
         tcg_gen_qemu_ld64(tmp3, tmp, get_mem_index(s));
-        gen_helper_load_psw(tmp2, tmp3);
+        gen_helper_load_psw(cpu_env, tmp2, tmp3);
         /* we need to keep cc_op intact */
         s->is_jmp = DISAS_JUMP;
         tcg_temp_free_i64(tmp);
@@ -2981,11 +2981,11 @@ static void disas_b2(DisasContext *s, int op, uint32_t insn)
         break;
     case 0x20: /* SERVC     R1,R2     [RRE] */
         /* SCLP Service call (PV hypercall) */
-        check_privileged(s, ilc);
+        check_privileged(env, s, ilc);
         potential_page_fault(s);
         tmp32_1 = load_reg32(r2);
         tmp = load_reg(r1);
-        gen_helper_servc(cc_op, tmp32_1, tmp);
+        gen_helper_servc(cc_op, cpu_env, tmp32_1, tmp);
         set_cc_static(s);
         tcg_temp_free_i32(tmp32_1);
         tcg_temp_free_i64(tmp);
@@ -2993,12 +2993,13 @@ static void disas_b2(DisasContext *s, int op, uint32_t insn)
 #endif
     default:
         LOG_DISAS("illegal b2 operation 0x%x\n", op);
-        gen_illegal_opcode(s, ilc);
+        gen_illegal_opcode(env, s, ilc);
         break;
     }
 }
 
-static void disas_b3(DisasContext *s, int op, int m3, int r1, int r2)
+static void disas_b3(CPUS390XState *env, DisasContext *s, int op, int m3,
+                     int r1, int r2)
 {
     TCGv_i64 tmp;
     TCGv_i32 tmp32_1, tmp32_2, tmp32_3;
@@ -3006,14 +3007,14 @@ static void disas_b3(DisasContext *s, int op, int m3, int r1, int r2)
 #define FP_HELPER(i) \
     tmp32_1 = tcg_const_i32(r1); \
     tmp32_2 = tcg_const_i32(r2); \
-    gen_helper_ ## i (tmp32_1, tmp32_2); \
+    gen_helper_ ## i(cpu_env, tmp32_1, tmp32_2); \
     tcg_temp_free_i32(tmp32_1); \
     tcg_temp_free_i32(tmp32_2);
 
 #define FP_HELPER_CC(i) \
     tmp32_1 = tcg_const_i32(r1); \
     tmp32_2 = tcg_const_i32(r2); \
-    gen_helper_ ## i (cc_op, tmp32_1, tmp32_2); \
+    gen_helper_ ## i(cc_op, cpu_env, tmp32_1, tmp32_2); \
     set_cc_static(s); \
     tcg_temp_free_i32(tmp32_1); \
     tcg_temp_free_i32(tmp32_2);
@@ -3085,13 +3086,13 @@ static void disas_b3(DisasContext *s, int op, int m3, int r1, int r2)
         tmp32_3 = tcg_const_i32(r1);
         switch (op) {
         case 0xe:
-            gen_helper_maebr(tmp32_1, tmp32_3, tmp32_2);
+            gen_helper_maebr(cpu_env, tmp32_1, tmp32_3, tmp32_2);
             break;
         case 0x1e:
-            gen_helper_madbr(tmp32_1, tmp32_3, tmp32_2);
+            gen_helper_madbr(cpu_env, tmp32_1, tmp32_3, tmp32_2);
             break;
         case 0x1f:
-            gen_helper_msdbr(tmp32_1, tmp32_3, tmp32_2);
+            gen_helper_msdbr(cpu_env, tmp32_1, tmp32_3, tmp32_2);
             break;
         default:
             tcg_abort();
@@ -3143,17 +3144,17 @@ static void disas_b3(DisasContext *s, int op, int m3, int r1, int r2)
         break;
     case 0x74: /* LZER        R1                [RRE] */
         tmp32_1 = tcg_const_i32(r1);
-        gen_helper_lzer(tmp32_1);
+        gen_helper_lzer(cpu_env, tmp32_1);
         tcg_temp_free_i32(tmp32_1);
         break;
     case 0x75: /* LZDR        R1                [RRE] */
         tmp32_1 = tcg_const_i32(r1);
-        gen_helper_lzdr(tmp32_1);
+        gen_helper_lzdr(cpu_env, tmp32_1);
         tcg_temp_free_i32(tmp32_1);
         break;
     case 0x76: /* LZXR        R1                [RRE] */
         tmp32_1 = tcg_const_i32(r1);
-        gen_helper_lzxr(tmp32_1);
+        gen_helper_lzxr(cpu_env, tmp32_1);
         tcg_temp_free_i32(tmp32_1);
         break;
     case 0x84: /* SFPC        R1                [RRE] */
@@ -3174,13 +3175,13 @@ static void disas_b3(DisasContext *s, int op, int m3, int r1, int r2)
         tmp32_2 = load_reg32(r2);
         switch (op) {
         case 0x94:
-            gen_helper_cefbr(tmp32_1, tmp32_2);
+            gen_helper_cefbr(cpu_env, tmp32_1, tmp32_2);
             break;
         case 0x95:
-            gen_helper_cdfbr(tmp32_1, tmp32_2);
+            gen_helper_cdfbr(cpu_env, tmp32_1, tmp32_2);
             break;
         case 0x96:
-            gen_helper_cxfbr(tmp32_1, tmp32_2);
+            gen_helper_cxfbr(cpu_env, tmp32_1, tmp32_2);
             break;
         default:
             tcg_abort();
@@ -3196,13 +3197,13 @@ static void disas_b3(DisasContext *s, int op, int m3, int r1, int r2)
         tmp32_3 = tcg_const_i32(m3);
         switch (op) {
         case 0x98:
-            gen_helper_cfebr(cc_op, tmp32_1, tmp32_2, tmp32_3);
+            gen_helper_cfebr(cc_op, cpu_env, tmp32_1, tmp32_2, tmp32_3);
             break;
         case 0x99:
-            gen_helper_cfdbr(cc_op, tmp32_1, tmp32_2, tmp32_3);
+            gen_helper_cfdbr(cc_op, cpu_env, tmp32_1, tmp32_2, tmp32_3);
             break;
         case 0x9a:
-            gen_helper_cfxbr(cc_op, tmp32_1, tmp32_2, tmp32_3);
+            gen_helper_cfxbr(cc_op, cpu_env, tmp32_1, tmp32_2, tmp32_3);
             break;
         default:
             tcg_abort();
@@ -3218,10 +3219,10 @@ static void disas_b3(DisasContext *s, int op, int m3, int r1, int r2)
         tmp = load_reg(r2);
         switch (op) {
         case 0xa4:
-            gen_helper_cegbr(tmp32_1, tmp);
+            gen_helper_cegbr(cpu_env, tmp32_1, tmp);
             break;
         case 0xa5:
-            gen_helper_cdgbr(tmp32_1, tmp);
+            gen_helper_cdgbr(cpu_env, tmp32_1, tmp);
             break;
         default:
             tcg_abort();
@@ -3232,7 +3233,7 @@ static void disas_b3(DisasContext *s, int op, int m3, int r1, int r2)
     case 0xa6: /* CXGBR       R1,R2             [RRE] */
         tmp32_1 = tcg_const_i32(r1);
         tmp = load_reg(r2);
-        gen_helper_cxgbr(tmp32_1, tmp);
+        gen_helper_cxgbr(cpu_env, tmp32_1, tmp);
         tcg_temp_free_i32(tmp32_1);
         tcg_temp_free_i64(tmp);
         break;
@@ -3240,7 +3241,7 @@ static void disas_b3(DisasContext *s, int op, int m3, int r1, int r2)
         tmp32_1 = tcg_const_i32(r1);
         tmp32_2 = tcg_const_i32(r2);
         tmp32_3 = tcg_const_i32(m3);
-        gen_helper_cgebr(cc_op, tmp32_1, tmp32_2, tmp32_3);
+        gen_helper_cgebr(cc_op, cpu_env, tmp32_1, tmp32_2, tmp32_3);
         set_cc_static(s);
         tcg_temp_free_i32(tmp32_1);
         tcg_temp_free_i32(tmp32_2);
@@ -3250,7 +3251,7 @@ static void disas_b3(DisasContext *s, int op, int m3, int r1, int r2)
         tmp32_1 = tcg_const_i32(r1);
         tmp32_2 = tcg_const_i32(r2);
         tmp32_3 = tcg_const_i32(m3);
-        gen_helper_cgdbr(cc_op, tmp32_1, tmp32_2, tmp32_3);
+        gen_helper_cgdbr(cc_op, cpu_env, tmp32_1, tmp32_2, tmp32_3);
         set_cc_static(s);
         tcg_temp_free_i32(tmp32_1);
         tcg_temp_free_i32(tmp32_2);
@@ -3260,7 +3261,7 @@ static void disas_b3(DisasContext *s, int op, int m3, int r1, int r2)
         tmp32_1 = tcg_const_i32(r1);
         tmp32_2 = tcg_const_i32(r2);
         tmp32_3 = tcg_const_i32(m3);
-        gen_helper_cgxbr(cc_op, tmp32_1, tmp32_2, tmp32_3);
+        gen_helper_cgxbr(cc_op, cpu_env, tmp32_1, tmp32_2, tmp32_3);
         set_cc_static(s);
         tcg_temp_free_i32(tmp32_1);
         tcg_temp_free_i32(tmp32_2);
@@ -3268,7 +3269,7 @@ static void disas_b3(DisasContext *s, int op, int m3, int r1, int r2)
         break;
     default:
         LOG_DISAS("illegal b3 operation 0x%x\n", op);
-        gen_illegal_opcode(s, 2);
+        gen_illegal_opcode(env, s, 2);
         break;
     }
 
@@ -3276,7 +3277,8 @@ static void disas_b3(DisasContext *s, int op, int m3, int r1, int r2)
 #undef FP_HELPER
 }
 
-static void disas_b9(DisasContext *s, int op, int r1, int r2)
+static void disas_b9(CPUS390XState *env, DisasContext *s, int op, int r1,
+                     int r2)
 {
     TCGv_i64 tmp, tmp2, tmp3;
     TCGv_i32 tmp32_1, tmp32_2, tmp32_3;
@@ -3540,7 +3542,7 @@ static void disas_b9(DisasContext *s, int op, int r1, int r2)
     case 0x83: /* FLOGR R1,R2 [RRE] */
         tmp = load_reg(r2);
         tmp32_1 = tcg_const_i32(r1);
-        gen_helper_flogr(cc_op, tmp32_1, tmp);
+        gen_helper_flogr(cc_op, cpu_env, tmp32_1, tmp);
         set_cc_static(s);
         tcg_temp_free_i64(tmp);
         tcg_temp_free_i32(tmp32_1);
@@ -3560,7 +3562,7 @@ static void disas_b9(DisasContext *s, int op, int r1, int r2)
     case 0x87: /* DLGR      R1,R2     [RRE] */
         tmp32_1 = tcg_const_i32(r1);
         tmp = load_reg(r2);
-        gen_helper_dlg(tmp32_1, tmp);
+        gen_helper_dlg(cpu_env, tmp32_1, tmp);
         tcg_temp_free_i64(tmp);
         tcg_temp_free_i32(tmp32_1);
         break;
@@ -3585,7 +3587,7 @@ static void disas_b9(DisasContext *s, int op, int r1, int r2)
         tmp2 = load_reg(r2);
         tmp32_1 = tcg_const_i32(r1);
         gen_op_calc_cc(s);
-        gen_helper_slbg(cc_op, cc_op, tmp32_1, tmp, tmp2);
+        gen_helper_slbg(cc_op, cpu_env, cc_op, tmp32_1, tmp, tmp2);
         set_cc_static(s);
         tcg_temp_free_i64(tmp);
         tcg_temp_free_i64(tmp2);
@@ -3652,19 +3654,19 @@ static void disas_b9(DisasContext *s, int op, int r1, int r2)
         tmp32_1 = load_reg32(r2);
         tmp32_2 = tcg_const_i32(r1);
         gen_op_calc_cc(s);
-        gen_helper_slb(cc_op, cc_op, tmp32_2, tmp32_1);
+        gen_helper_slb(cc_op, cpu_env, cc_op, tmp32_2, tmp32_1);
         set_cc_static(s);
         tcg_temp_free_i32(tmp32_1);
         tcg_temp_free_i32(tmp32_2);
         break;
     default:
         LOG_DISAS("illegal b9 operation 0x%x\n", op);
-        gen_illegal_opcode(s, 2);
+        gen_illegal_opcode(env, s, 2);
         break;
     }
 }
 
-static void disas_c0(DisasContext *s, int op, int r1, int i2)
+static void disas_c0(CPUS390XState *env, DisasContext *s, int op, int r1, int i2)
 {
     TCGv_i64 tmp;
     TCGv_i32 tmp32_1, tmp32_2;
@@ -3760,12 +3762,13 @@ static void disas_c0(DisasContext *s, int op, int r1, int i2)
         break;
     default:
         LOG_DISAS("illegal c0 operation 0x%x\n", op);
-        gen_illegal_opcode(s, 3);
+        gen_illegal_opcode(env, s, 3);
         break;
     }
 }
 
-static void disas_c2(DisasContext *s, int op, int r1, int i2)
+static void disas_c2(CPUS390XState *env, DisasContext *s, int op, int r1,
+                     int i2)
 {
     TCGv_i64 tmp, tmp2, tmp3;
     TCGv_i32 tmp32_1, tmp32_2, tmp32_3;
@@ -3837,7 +3840,7 @@ static void disas_c2(DisasContext *s, int op, int r1, int i2)
         break;
     default:
         LOG_DISAS("illegal c2 operation 0x%x\n", op);
-        gen_illegal_opcode(s, 3);
+        gen_illegal_opcode(env, s, 3);
         break;
     }
 }
@@ -3859,7 +3862,7 @@ static void gen_and_or_xor_i32(int opc, TCGv_i32 tmp, TCGv_i32 tmp2)
     }
 }
 
-static void disas_s390_insn(DisasContext *s)
+static void disas_s390_insn(CPUS390XState *env, DisasContext *s)
 {
     TCGv_i64 tmp, tmp2, tmp3, tmp4;
     TCGv_i32 tmp32_1, tmp32_2, tmp32_3, tmp32_4;
@@ -3870,7 +3873,7 @@ static void disas_s390_insn(DisasContext *s)
     int ilc;
     int l1;
 
-    opc = ldub_code(s->pc);
+    opc = cpu_ldub_code(env, s->pc);
     LOG_DISAS("opc 0x%x\n", opc);
 
     ilc = get_ilc(opc);
@@ -3878,12 +3881,12 @@ static void disas_s390_insn(DisasContext *s)
     switch (opc) {
 #ifndef CONFIG_USER_ONLY
     case 0x01: /* SAM */
-        insn = ld_code2(s->pc);
+        insn = ld_code2(env, s->pc);
         /* set addressing mode, but we only do 64bit anyways */
         break;
 #endif
     case 0x6: /* BCTR     R1,R2     [RR] */
-        insn = ld_code2(s->pc);
+        insn = ld_code2(env, s->pc);
         decode_rr(s, insn, &r1, &r2);
         tmp32_1 = load_reg32(r1);
         tcg_gen_subi_i32(tmp32_1, tmp32_1, 1);
@@ -3909,7 +3912,7 @@ static void disas_s390_insn(DisasContext *s)
         }
         break;
     case 0x7: /* BCR    M1,R2     [RR] */
-        insn = ld_code2(s->pc);
+        insn = ld_code2(env, s->pc);
         decode_rr(s, insn, &r1, &r2);
         if (r2) {
             tmp = load_reg(r2);
@@ -3921,7 +3924,7 @@ static void disas_s390_insn(DisasContext *s)
         }
         break;
     case 0xa: /* SVC    I         [RR] */
-        insn = ld_code2(s->pc);
+        insn = ld_code2(env, s->pc);
         debug_insn(insn);
         i = insn & 0xff;
         update_psw_addr(s);
@@ -3931,14 +3934,14 @@ static void disas_s390_insn(DisasContext *s)
         tmp32_3 = tcg_const_i32(EXCP_SVC);
         tcg_gen_st_i32(tmp32_1, cpu_env, offsetof(CPUS390XState, int_svc_code));
         tcg_gen_st_i32(tmp32_2, cpu_env, offsetof(CPUS390XState, int_svc_ilc));
-        gen_helper_exception(tmp32_3);
+        gen_helper_exception(cpu_env, tmp32_3);
         s->is_jmp = DISAS_EXCP;
         tcg_temp_free_i32(tmp32_1);
         tcg_temp_free_i32(tmp32_2);
         tcg_temp_free_i32(tmp32_3);
         break;
     case 0xd: /* BASR   R1,R2     [RR] */
-        insn = ld_code2(s->pc);
+        insn = ld_code2(env, s->pc);
         decode_rr(s, insn, &r1, &r2);
         tmp = tcg_const_i64(pc_to_link_info(s, s->pc + 2));
         store_reg(r1, tmp);
@@ -3951,18 +3954,18 @@ static void disas_s390_insn(DisasContext *s)
         tcg_temp_free_i64(tmp);
         break;
     case 0xe: /* MVCL   R1,R2     [RR] */
-        insn = ld_code2(s->pc);
+        insn = ld_code2(env, s->pc);
         decode_rr(s, insn, &r1, &r2);
         tmp32_1 = tcg_const_i32(r1);
         tmp32_2 = tcg_const_i32(r2);
         potential_page_fault(s);
-        gen_helper_mvcl(cc_op, tmp32_1, tmp32_2);
+        gen_helper_mvcl(cc_op, cpu_env, tmp32_1, tmp32_2);
         set_cc_static(s);
         tcg_temp_free_i32(tmp32_1);
         tcg_temp_free_i32(tmp32_2);
         break;
     case 0x10: /* LPR    R1,R2     [RR] */
-        insn = ld_code2(s->pc);
+        insn = ld_code2(env, s->pc);
         decode_rr(s, insn, &r1, &r2);
         tmp32_1 = load_reg32(r2);
         set_cc_abs32(s, tmp32_1);
@@ -3971,7 +3974,7 @@ static void disas_s390_insn(DisasContext *s)
         tcg_temp_free_i32(tmp32_1);
         break;
     case 0x11: /* LNR    R1,R2     [RR] */
-        insn = ld_code2(s->pc);
+        insn = ld_code2(env, s->pc);
         decode_rr(s, insn, &r1, &r2);
         tmp32_1 = load_reg32(r2);
         set_cc_nabs32(s, tmp32_1);
@@ -3980,7 +3983,7 @@ static void disas_s390_insn(DisasContext *s)
         tcg_temp_free_i32(tmp32_1);
         break;
     case 0x12: /* LTR    R1,R2     [RR] */
-        insn = ld_code2(s->pc);
+        insn = ld_code2(env, s->pc);
         decode_rr(s, insn, &r1, &r2);
         tmp32_1 = load_reg32(r2);
         if (r1 != r2) {
@@ -3990,7 +3993,7 @@ static void disas_s390_insn(DisasContext *s)
         tcg_temp_free_i32(tmp32_1);
         break;
     case 0x13: /* LCR    R1,R2     [RR] */
-        insn = ld_code2(s->pc);
+        insn = ld_code2(env, s->pc);
         decode_rr(s, insn, &r1, &r2);
         tmp32_1 = load_reg32(r2);
         tcg_gen_neg_i32(tmp32_1, tmp32_1);
@@ -4001,7 +4004,7 @@ static void disas_s390_insn(DisasContext *s)
     case 0x14: /* NR     R1,R2     [RR] */
     case 0x16: /* OR     R1,R2     [RR] */
     case 0x17: /* XR     R1,R2     [RR] */
-        insn = ld_code2(s->pc);
+        insn = ld_code2(env, s->pc);
         decode_rr(s, insn, &r1, &r2);
         tmp32_2 = load_reg32(r2);
         tmp32_1 = load_reg32(r1);
@@ -4012,7 +4015,7 @@ static void disas_s390_insn(DisasContext *s)
         tcg_temp_free_i32(tmp32_2);
         break;
     case 0x18: /* LR     R1,R2     [RR] */
-        insn = ld_code2(s->pc);
+        insn = ld_code2(env, s->pc);
         decode_rr(s, insn, &r1, &r2);
         tmp32_1 = load_reg32(r2);
         store_reg32(r1, tmp32_1);
@@ -4020,7 +4023,7 @@ static void disas_s390_insn(DisasContext *s)
         break;
     case 0x15: /* CLR    R1,R2     [RR] */
     case 0x19: /* CR     R1,R2     [RR] */
-        insn = ld_code2(s->pc);
+        insn = ld_code2(env, s->pc);
         decode_rr(s, insn, &r1, &r2);
         tmp32_1 = load_reg32(r1);
         tmp32_2 = load_reg32(r2);
@@ -4034,7 +4037,7 @@ static void disas_s390_insn(DisasContext *s)
         break;
     case 0x1a: /* AR     R1,R2     [RR] */
     case 0x1e: /* ALR    R1,R2     [RR] */
-        insn = ld_code2(s->pc);
+        insn = ld_code2(env, s->pc);
         decode_rr(s, insn, &r1, &r2);
         tmp32_1 = load_reg32(r1);
         tmp32_2 = load_reg32(r2);
@@ -4052,7 +4055,7 @@ static void disas_s390_insn(DisasContext *s)
         break;
     case 0x1b: /* SR     R1,R2     [RR] */
     case 0x1f: /* SLR    R1,R2     [RR] */
-        insn = ld_code2(s->pc);
+        insn = ld_code2(env, s->pc);
         decode_rr(s, insn, &r1, &r2);
         tmp32_1 = load_reg32(r1);
         tmp32_2 = load_reg32(r2);
@@ -4070,7 +4073,7 @@ static void disas_s390_insn(DisasContext *s)
         break;
     case 0x1c: /* MR     R1,R2     [RR] */
         /* reg(r1, r1+1) = reg(r1+1) * reg(r2) */
-        insn = ld_code2(s->pc);
+        insn = ld_code2(env, s->pc);
         decode_rr(s, insn, &r1, &r2);
         tmp2 = load_reg(r2);
         tmp3 = load_reg((r1 + 1) & 15);
@@ -4084,7 +4087,7 @@ static void disas_s390_insn(DisasContext *s)
         tcg_temp_free_i64(tmp3);
         break;
     case 0x1d: /* DR     R1,R2               [RR] */
-        insn = ld_code2(s->pc);
+        insn = ld_code2(env, s->pc);
         decode_rr(s, insn, &r1, &r2);
         tmp32_1 = load_reg32(r1);
         tmp32_2 = load_reg32(r1 + 1);
@@ -4119,21 +4122,21 @@ static void disas_s390_insn(DisasContext *s)
         tcg_temp_free_i64(tmp3);
         break;
     case 0x28: /* LDR    R1,R2               [RR] */
-        insn = ld_code2(s->pc);
+        insn = ld_code2(env, s->pc);
         decode_rr(s, insn, &r1, &r2);
         tmp = load_freg(r2);
         store_freg(r1, tmp);
         tcg_temp_free_i64(tmp);
         break;
     case 0x38: /* LER    R1,R2               [RR] */
-        insn = ld_code2(s->pc);
+        insn = ld_code2(env, s->pc);
         decode_rr(s, insn, &r1, &r2);
         tmp32_1 = load_freg32(r2);
         store_freg32(r1, tmp32_1);
         tcg_temp_free_i32(tmp32_1);
         break;
     case 0x40: /* STH    R1,D2(X2,B2)     [RX] */
-        insn = ld_code4(s->pc);
+        insn = ld_code4(env, s->pc);
         tmp = decode_rx(s, insn, &r1, &x2, &b2, &d2);
         tmp2 = load_reg(r1);
         tcg_gen_qemu_st16(tmp2, tmp, get_mem_index(s));
@@ -4141,13 +4144,13 @@ static void disas_s390_insn(DisasContext *s)
         tcg_temp_free_i64(tmp2);
         break;
     case 0x41:        /* la */
-        insn = ld_code4(s->pc);
+        insn = ld_code4(env, s->pc);
         tmp = decode_rx(s, insn, &r1, &x2, &b2, &d2);
         store_reg(r1, tmp); /* FIXME: 31/24-bit addressing */
         tcg_temp_free_i64(tmp);
         break;
     case 0x42: /* STC    R1,D2(X2,B2)     [RX] */
-        insn = ld_code4(s->pc);
+        insn = ld_code4(env, s->pc);
         tmp = decode_rx(s, insn, &r1, &x2, &b2, &d2);
         tmp2 = load_reg(r1);
         tcg_gen_qemu_st8(tmp2, tmp, get_mem_index(s));
@@ -4155,7 +4158,7 @@ static void disas_s390_insn(DisasContext *s)
         tcg_temp_free_i64(tmp2);
         break;
     case 0x43: /* IC     R1,D2(X2,B2)     [RX] */
-        insn = ld_code4(s->pc);
+        insn = ld_code4(env, s->pc);
         tmp = decode_rx(s, insn, &r1, &x2, &b2, &d2);
         tmp2 = tcg_temp_new_i64();
         tcg_gen_qemu_ld8u(tmp2, tmp, get_mem_index(s));
@@ -4164,20 +4167,20 @@ static void disas_s390_insn(DisasContext *s)
         tcg_temp_free_i64(tmp2);
         break;
     case 0x44: /* EX     R1,D2(X2,B2)     [RX] */
-        insn = ld_code4(s->pc);
+        insn = ld_code4(env, s->pc);
         tmp = decode_rx(s, insn, &r1, &x2, &b2, &d2);
         tmp2 = load_reg(r1);
         tmp3 = tcg_const_i64(s->pc + 4);
         update_psw_addr(s);
         gen_op_calc_cc(s);
-        gen_helper_ex(cc_op, cc_op, tmp2, tmp, tmp3);
+        gen_helper_ex(cc_op, cpu_env, cc_op, tmp2, tmp, tmp3);
         set_cc_static(s);
         tcg_temp_free_i64(tmp);
         tcg_temp_free_i64(tmp2);
         tcg_temp_free_i64(tmp3);
         break;
     case 0x46: /* BCT    R1,D2(X2,B2)     [RX] */
-        insn = ld_code4(s->pc);
+        insn = ld_code4(env, s->pc);
         tmp = decode_rx(s, insn, &r1, &x2, &b2, &d2);
         tcg_temp_free_i64(tmp);
 
@@ -4201,14 +4204,14 @@ static void disas_s390_insn(DisasContext *s)
         tcg_temp_free_i64(tmp);
         break;
     case 0x47: /* BC     M1,D2(X2,B2)     [RX] */
-        insn = ld_code4(s->pc);
+        insn = ld_code4(env, s->pc);
         tmp = decode_rx(s, insn, &r1, &x2, &b2, &d2);
         gen_bcr(s, r1, tmp, s->pc + 4);
         tcg_temp_free_i64(tmp);
         s->is_jmp = DISAS_TB_JUMP;
         break;
     case 0x48: /* LH     R1,D2(X2,B2)     [RX] */
-        insn = ld_code4(s->pc);
+        insn = ld_code4(env, s->pc);
         tmp = decode_rx(s, insn, &r1, &x2, &b2, &d2);
         tmp2 = tcg_temp_new_i64();
         tcg_gen_qemu_ld16s(tmp2, tmp, get_mem_index(s));
@@ -4217,7 +4220,7 @@ static void disas_s390_insn(DisasContext *s)
         tcg_temp_free_i64(tmp2);
         break;
     case 0x49: /* CH     R1,D2(X2,B2)     [RX] */
-        insn = ld_code4(s->pc);
+        insn = ld_code4(env, s->pc);
         tmp = decode_rx(s, insn, &r1, &x2, &b2, &d2);
         tmp32_1 = load_reg32(r1);
         tmp32_2 = tcg_temp_new_i32();
@@ -4233,7 +4236,7 @@ static void disas_s390_insn(DisasContext *s)
     case 0x4a: /* AH     R1,D2(X2,B2)     [RX] */
     case 0x4b: /* SH     R1,D2(X2,B2)     [RX] */
     case 0x4c: /* MH     R1,D2(X2,B2)     [RX] */
-        insn = ld_code4(s->pc);
+        insn = ld_code4(env, s->pc);
         tmp = decode_rx(s, insn, &r1, &x2, &b2, &d2);
         tmp2 = tcg_temp_new_i64();
         tmp32_1 = load_reg32(r1);
@@ -4266,7 +4269,7 @@ static void disas_s390_insn(DisasContext *s)
         tcg_temp_free_i64(tmp2);
         break;
     case 0x4d: /* BAS    R1,D2(X2,B2)     [RX] */
-        insn = ld_code4(s->pc);
+        insn = ld_code4(env, s->pc);
         tmp = decode_rx(s, insn, &r1, &x2, &b2, &d2);
         tmp2 = tcg_const_i64(pc_to_link_info(s, s->pc + 4));
         store_reg(r1, tmp2);
@@ -4276,7 +4279,7 @@ static void disas_s390_insn(DisasContext *s)
         s->is_jmp = DISAS_JUMP;
         break;
     case 0x4e: /* CVD    R1,D2(X2,B2)     [RX] */
-        insn = ld_code4(s->pc);
+        insn = ld_code4(env, s->pc);
         tmp = decode_rx(s, insn, &r1, &x2, &b2, &d2);
         tmp2 = tcg_temp_new_i64();
         tmp32_1 = tcg_temp_new_i32();
@@ -4288,7 +4291,7 @@ static void disas_s390_insn(DisasContext *s)
         tcg_temp_free_i32(tmp32_1);
         break;
     case 0x50: /* st r1, d2(x2, b2) */
-        insn = ld_code4(s->pc);
+        insn = ld_code4(env, s->pc);
         tmp = decode_rx(s, insn, &r1, &x2, &b2, &d2);
         tmp2 = load_reg(r1);
         tcg_gen_qemu_st32(tmp2, tmp, get_mem_index(s));
@@ -4296,7 +4299,7 @@ static void disas_s390_insn(DisasContext *s)
         tcg_temp_free_i64(tmp2);
         break;
     case 0x55: /* CL     R1,D2(X2,B2)     [RX] */
-        insn = ld_code4(s->pc);
+        insn = ld_code4(env, s->pc);
         tmp = decode_rx(s, insn, &r1, &x2, &b2, &d2);
         tmp2 = tcg_temp_new_i64();
         tmp32_1 = tcg_temp_new_i32();
@@ -4312,7 +4315,7 @@ static void disas_s390_insn(DisasContext *s)
     case 0x54: /* N      R1,D2(X2,B2)     [RX] */
     case 0x56: /* O      R1,D2(X2,B2)     [RX] */
     case 0x57: /* X      R1,D2(X2,B2)     [RX] */
-        insn = ld_code4(s->pc);
+        insn = ld_code4(env, s->pc);
         tmp = decode_rx(s, insn, &r1, &x2, &b2, &d2);
         tmp2 = tcg_temp_new_i64();
         tmp32_1 = load_reg32(r1);
@@ -4328,7 +4331,7 @@ static void disas_s390_insn(DisasContext *s)
         tcg_temp_free_i32(tmp32_2);
         break;
     case 0x58: /* l r1, d2(x2, b2) */
-        insn = ld_code4(s->pc);
+        insn = ld_code4(env, s->pc);
         tmp = decode_rx(s, insn, &r1, &x2, &b2, &d2);
         tmp2 = tcg_temp_new_i64();
         tmp32_1 = tcg_temp_new_i32();
@@ -4340,7 +4343,7 @@ static void disas_s390_insn(DisasContext *s)
         tcg_temp_free_i32(tmp32_1);
         break;
     case 0x59: /* C      R1,D2(X2,B2)     [RX] */
-        insn = ld_code4(s->pc);
+        insn = ld_code4(env, s->pc);
         tmp = decode_rx(s, insn, &r1, &x2, &b2, &d2);
         tmp2 = tcg_temp_new_i64();
         tmp32_1 = tcg_temp_new_i32();
@@ -4357,7 +4360,7 @@ static void disas_s390_insn(DisasContext *s)
     case 0x5b: /* S      R1,D2(X2,B2)     [RX] */
     case 0x5e: /* AL     R1,D2(X2,B2)     [RX] */
     case 0x5f: /* SL     R1,D2(X2,B2)     [RX] */
-        insn = ld_code4(s->pc);
+        insn = ld_code4(env, s->pc);
         tmp = decode_rx(s, insn, &r1, &x2, &b2, &d2);
         tmp32_1 = load_reg32(r1);
         tmp32_2 = tcg_temp_new_i32();
@@ -4400,7 +4403,7 @@ static void disas_s390_insn(DisasContext *s)
         break;
     case 0x5c: /* M      R1,D2(X2,B2)        [RX] */
         /* reg(r1, r1+1) = reg(r1+1) * *(s32*)addr */
-        insn = ld_code4(s->pc);
+        insn = ld_code4(env, s->pc);
         tmp = decode_rx(s, insn, &r1, &x2, &b2, &d2);
         tmp2 = tcg_temp_new_i64();
         tcg_gen_qemu_ld32s(tmp2, tmp, get_mem_index(s));
@@ -4416,7 +4419,7 @@ static void disas_s390_insn(DisasContext *s)
         tcg_temp_free_i64(tmp3);
         break;
     case 0x5d: /* D      R1,D2(X2,B2)        [RX] */
-        insn = ld_code4(s->pc);
+        insn = ld_code4(env, s->pc);
         tmp3 = decode_rx(s, insn, &r1, &x2, &b2, &d2);
         tmp32_1 = load_reg32(r1);
         tmp32_2 = load_reg32(r1 + 1);
@@ -4450,7 +4453,7 @@ static void disas_s390_insn(DisasContext *s)
         tcg_temp_free_i64(tmp3);
         break;
     case 0x60: /* STD    R1,D2(X2,B2)        [RX] */
-        insn = ld_code4(s->pc);
+        insn = ld_code4(env, s->pc);
         tmp = decode_rx(s, insn, &r1, &x2, &b2, &d2);
         tmp2 = load_freg(r1);
         tcg_gen_qemu_st64(tmp2, tmp, get_mem_index(s));
@@ -4458,7 +4461,7 @@ static void disas_s390_insn(DisasContext *s)
         tcg_temp_free_i64(tmp2);
         break;
     case 0x68: /* LD    R1,D2(X2,B2)        [RX] */
-        insn = ld_code4(s->pc);
+        insn = ld_code4(env, s->pc);
         tmp = decode_rx(s, insn, &r1, &x2, &b2, &d2);
         tmp2 = tcg_temp_new_i64();
         tcg_gen_qemu_ld64(tmp2, tmp, get_mem_index(s));
@@ -4467,7 +4470,7 @@ static void disas_s390_insn(DisasContext *s)
         tcg_temp_free_i64(tmp2);
         break;
     case 0x70: /* STE R1,D2(X2,B2) [RX] */
-        insn = ld_code4(s->pc);
+        insn = ld_code4(env, s->pc);
         tmp = decode_rx(s, insn, &r1, &x2, &b2, &d2);
         tmp2 = tcg_temp_new_i64();
         tmp32_1 = load_freg32(r1);
@@ -4478,7 +4481,7 @@ static void disas_s390_insn(DisasContext *s)
         tcg_temp_free_i32(tmp32_1);
         break;
     case 0x71: /* MS      R1,D2(X2,B2)     [RX] */
-        insn = ld_code4(s->pc);
+        insn = ld_code4(env, s->pc);
         tmp = decode_rx(s, insn, &r1, &x2, &b2, &d2);
         tmp2 = tcg_temp_new_i64();
         tmp32_1 = load_reg32(r1);
@@ -4493,7 +4496,7 @@ static void disas_s390_insn(DisasContext *s)
         tcg_temp_free_i32(tmp32_2);
         break;
     case 0x78: /* LE     R1,D2(X2,B2)        [RX] */
-        insn = ld_code4(s->pc);
+        insn = ld_code4(env, s->pc);
         tmp = decode_rx(s, insn, &r1, &x2, &b2, &d2);
         tmp2 = tcg_temp_new_i64();
         tmp32_1 = tcg_temp_new_i32();
@@ -4507,8 +4510,8 @@ static void disas_s390_insn(DisasContext *s)
 #ifndef CONFIG_USER_ONLY
     case 0x80: /* SSM      D2(B2)       [S] */
         /* Set System Mask */
-        check_privileged(s, ilc);
-        insn = ld_code4(s->pc);
+        check_privileged(env, s, ilc);
+        insn = ld_code4(env, s->pc);
         decode_rs(s, insn, &r1, &r3, &b2, &d2);
         tmp = get_address(s, 0, b2, d2);
         tmp2 = tcg_temp_new_i64();
@@ -4523,8 +4526,8 @@ static void disas_s390_insn(DisasContext *s)
         break;
     case 0x82: /* LPSW     D2(B2)       [S] */
         /* Load PSW */
-        check_privileged(s, ilc);
-        insn = ld_code4(s->pc);
+        check_privileged(env, s, ilc);
+        insn = ld_code4(env, s->pc);
         decode_rs(s, insn, &r1, &r3, &b2, &d2);
         tmp = get_address(s, 0, b2, d2);
         tmp2 = tcg_temp_new_i64();
@@ -4532,7 +4535,7 @@ static void disas_s390_insn(DisasContext *s)
         tcg_gen_qemu_ld32u(tmp2, tmp, get_mem_index(s));
         tcg_gen_addi_i64(tmp, tmp, 4);
         tcg_gen_qemu_ld32u(tmp3, tmp, get_mem_index(s));
-        gen_helper_load_psw(tmp2, tmp3);
+        gen_helper_load_psw(cpu_env, tmp2, tmp3);
         tcg_temp_free_i64(tmp);
         tcg_temp_free_i64(tmp2);
         tcg_temp_free_i64(tmp3);
@@ -4541,14 +4544,14 @@ static void disas_s390_insn(DisasContext *s)
         break;
     case 0x83: /* DIAG     R1,R3,D2     [RS] */
         /* Diagnose call (KVM hypercall) */
-        check_privileged(s, ilc);
+        check_privileged(env, s, ilc);
         potential_page_fault(s);
-        insn = ld_code4(s->pc);
+        insn = ld_code4(env, s->pc);
         decode_rs(s, insn, &r1, &r3, &b2, &d2);
         tmp32_1 = tcg_const_i32(insn & 0xfff);
         tmp2 = load_reg(2);
         tmp3 = load_reg(1);
-        gen_helper_diag(tmp2, tmp32_1, tmp2, tmp3);
+        gen_helper_diag(tmp2, cpu_env, tmp32_1, tmp2, tmp3);
         store_reg(2, tmp2);
         tcg_temp_free_i32(tmp32_1);
         tcg_temp_free_i64(tmp2);
@@ -4558,7 +4561,7 @@ static void disas_s390_insn(DisasContext *s)
     case 0x88: /* SRL    R1,D2(B2)        [RS] */
     case 0x89: /* SLL    R1,D2(B2)        [RS] */
     case 0x8a: /* SRA    R1,D2(B2)        [RS] */
-        insn = ld_code4(s->pc);
+        insn = ld_code4(env, s->pc);
         decode_rs(s, insn, &r1, &r3, &b2, &d2);
         tmp = get_address(s, 0, b2, d2);
         tmp32_1 = load_reg32(r1);
@@ -4587,7 +4590,7 @@ static void disas_s390_insn(DisasContext *s)
     case 0x8c: /* SRDL   R1,D2(B2)        [RS] */
     case 0x8d: /* SLDL   R1,D2(B2)        [RS] */
     case 0x8e: /* SRDA   R1,D2(B2)        [RS] */
-        insn = ld_code4(s->pc);
+        insn = ld_code4(env, s->pc);
         decode_rs(s, insn, &r1, &r3, &b2, &d2);
         tmp = get_address(s, 0, b2, d2); /* shift */
         tmp2 = tcg_temp_new_i64();
@@ -4616,7 +4619,7 @@ static void disas_s390_insn(DisasContext *s)
         break;
     case 0x98: /* LM     R1,R3,D2(B2)     [RS] */
     case 0x90: /* STM    R1,R3,D2(B2)     [RS] */
-        insn = ld_code4(s->pc);
+        insn = ld_code4(env, s->pc);
         decode_rs(s, insn, &r1, &r3, &b2, &d2);
 
         tmp = get_address(s, 0, b2, d2);
@@ -4642,7 +4645,7 @@ static void disas_s390_insn(DisasContext *s)
         tcg_temp_free_i64(tmp4);
         break;
     case 0x91: /* TM     D1(B1),I2        [SI] */
-        insn = ld_code4(s->pc);
+        insn = ld_code4(env, s->pc);
         tmp = decode_si(s, insn, &i2, &b1, &d1);
         tmp2 = tcg_const_i64(i2);
         tcg_gen_qemu_ld8u(tmp, tmp, get_mem_index(s));
@@ -4651,7 +4654,7 @@ static void disas_s390_insn(DisasContext *s)
         tcg_temp_free_i64(tmp2);
         break;
     case 0x92: /* MVI    D1(B1),I2        [SI] */
-        insn = ld_code4(s->pc);
+        insn = ld_code4(env, s->pc);
         tmp = decode_si(s, insn, &i2, &b1, &d1);
         tmp2 = tcg_const_i64(i2);
         tcg_gen_qemu_st8(tmp2, tmp, get_mem_index(s));
@@ -4661,7 +4664,7 @@ static void disas_s390_insn(DisasContext *s)
     case 0x94: /* NI     D1(B1),I2        [SI] */
     case 0x96: /* OI     D1(B1),I2        [SI] */
     case 0x97: /* XI     D1(B1),I2        [SI] */
-        insn = ld_code4(s->pc);
+        insn = ld_code4(env, s->pc);
         tmp = decode_si(s, insn, &i2, &b1, &d1);
         tmp2 = tcg_temp_new_i64();
         tcg_gen_qemu_ld8u(tmp2, tmp, get_mem_index(s));
@@ -4684,7 +4687,7 @@ static void disas_s390_insn(DisasContext *s)
         tcg_temp_free_i64(tmp2);
         break;
     case 0x95: /* CLI    D1(B1),I2        [SI] */
-        insn = ld_code4(s->pc);
+        insn = ld_code4(env, s->pc);
         tmp = decode_si(s, insn, &i2, &b1, &d1);
         tmp2 = tcg_temp_new_i64();
         tcg_gen_qemu_ld8u(tmp2, tmp, get_mem_index(s));
@@ -4693,64 +4696,64 @@ static void disas_s390_insn(DisasContext *s)
         tcg_temp_free_i64(tmp2);
         break;
     case 0x9a: /* LAM      R1,R3,D2(B2)     [RS] */
-        insn = ld_code4(s->pc);
+        insn = ld_code4(env, s->pc);
         decode_rs(s, insn, &r1, &r3, &b2, &d2);
         tmp = get_address(s, 0, b2, d2);
         tmp32_1 = tcg_const_i32(r1);
         tmp32_2 = tcg_const_i32(r3);
         potential_page_fault(s);
-        gen_helper_lam(tmp32_1, tmp, tmp32_2);
+        gen_helper_lam(cpu_env, tmp32_1, tmp, tmp32_2);
         tcg_temp_free_i64(tmp);
         tcg_temp_free_i32(tmp32_1);
         tcg_temp_free_i32(tmp32_2);
         break;
     case 0x9b: /* STAM     R1,R3,D2(B2)     [RS] */
-        insn = ld_code4(s->pc);
+        insn = ld_code4(env, s->pc);
         decode_rs(s, insn, &r1, &r3, &b2, &d2);
         tmp = get_address(s, 0, b2, d2);
         tmp32_1 = tcg_const_i32(r1);
         tmp32_2 = tcg_const_i32(r3);
         potential_page_fault(s);
-        gen_helper_stam(tmp32_1, tmp, tmp32_2);
+        gen_helper_stam(cpu_env, tmp32_1, tmp, tmp32_2);
         tcg_temp_free_i64(tmp);
         tcg_temp_free_i32(tmp32_1);
         tcg_temp_free_i32(tmp32_2);
         break;
     case 0xa5:
-        insn = ld_code4(s->pc);
+        insn = ld_code4(env, s->pc);
         r1 = (insn >> 20) & 0xf;
         op = (insn >> 16) & 0xf;
         i2 = insn & 0xffff;
-        disas_a5(s, op, r1, i2);
+        disas_a5(env, s, op, r1, i2);
         break;
     case 0xa7:
-        insn = ld_code4(s->pc);
+        insn = ld_code4(env, s->pc);
         r1 = (insn >> 20) & 0xf;
         op = (insn >> 16) & 0xf;
         i2 = (short)insn;
-        disas_a7(s, op, r1, i2);
+        disas_a7(env, s, op, r1, i2);
         break;
     case 0xa8: /* MVCLE   R1,R3,D2(B2)     [RS] */
-        insn = ld_code4(s->pc);
+        insn = ld_code4(env, s->pc);
         decode_rs(s, insn, &r1, &r3, &b2, &d2);
         tmp = get_address(s, 0, b2, d2);
         tmp32_1 = tcg_const_i32(r1);
         tmp32_2 = tcg_const_i32(r3);
         potential_page_fault(s);
-        gen_helper_mvcle(cc_op, tmp32_1, tmp, tmp32_2);
+        gen_helper_mvcle(cc_op, cpu_env, tmp32_1, tmp, tmp32_2);
         set_cc_static(s);
         tcg_temp_free_i64(tmp);
         tcg_temp_free_i32(tmp32_1);
         tcg_temp_free_i32(tmp32_2);
         break;
     case 0xa9: /* CLCLE   R1,R3,D2(B2)     [RS] */
-        insn = ld_code4(s->pc);
+        insn = ld_code4(env, s->pc);
         decode_rs(s, insn, &r1, &r3, &b2, &d2);
         tmp = get_address(s, 0, b2, d2);
         tmp32_1 = tcg_const_i32(r1);
         tmp32_2 = tcg_const_i32(r3);
         potential_page_fault(s);
-        gen_helper_clcle(cc_op, tmp32_1, tmp, tmp32_2);
+        gen_helper_clcle(cc_op, cpu_env, tmp32_1, tmp, tmp32_2);
         set_cc_static(s);
         tcg_temp_free_i64(tmp);
         tcg_temp_free_i32(tmp32_1);
@@ -4759,8 +4762,8 @@ static void disas_s390_insn(DisasContext *s)
 #ifndef CONFIG_USER_ONLY
     case 0xac: /* STNSM   D1(B1),I2     [SI] */
     case 0xad: /* STOSM   D1(B1),I2     [SI] */
-        check_privileged(s, ilc);
-        insn = ld_code4(s->pc);
+        check_privileged(env, s, ilc);
+        insn = ld_code4(env, s->pc);
         tmp = decode_si(s, insn, &i2, &b1, &d1);
         tmp2 = tcg_temp_new_i64();
         tcg_gen_shri_i64(tmp2, psw_mask, 56);
@@ -4775,33 +4778,33 @@ static void disas_s390_insn(DisasContext *s)
         tcg_temp_free_i64(tmp2);
         break;
     case 0xae: /* SIGP   R1,R3,D2(B2)     [RS] */
-        check_privileged(s, ilc);
-        insn = ld_code4(s->pc);
+        check_privileged(env, s, ilc);
+        insn = ld_code4(env, s->pc);
         decode_rs(s, insn, &r1, &r3, &b2, &d2);
         tmp = get_address(s, 0, b2, d2);
         tmp2 = load_reg(r3);
         tmp32_1 = tcg_const_i32(r1);
         potential_page_fault(s);
-        gen_helper_sigp(cc_op, tmp, tmp32_1, tmp2);
+        gen_helper_sigp(cc_op, cpu_env, tmp, tmp32_1, tmp2);
         set_cc_static(s);
         tcg_temp_free_i64(tmp);
         tcg_temp_free_i64(tmp2);
         tcg_temp_free_i32(tmp32_1);
         break;
     case 0xb1: /* LRA    R1,D2(X2, B2)     [RX] */
-        check_privileged(s, ilc);
-        insn = ld_code4(s->pc);
+        check_privileged(env, s, ilc);
+        insn = ld_code4(env, s->pc);
         tmp = decode_rx(s, insn, &r1, &x2, &b2, &d2);
         tmp32_1 = tcg_const_i32(r1);
         potential_page_fault(s);
-        gen_helper_lra(cc_op, tmp, tmp32_1);
+        gen_helper_lra(cc_op, cpu_env, tmp, tmp32_1);
         set_cc_static(s);
         tcg_temp_free_i64(tmp);
         tcg_temp_free_i32(tmp32_1);
         break;
 #endif
     case 0xb2:
-        insn = ld_code4(s->pc);
+        insn = ld_code4(env, s->pc);
         op = (insn >> 16) & 0xff;
         switch (op) {
         case 0x9c: /* STFPC    D2(B2) [S] */
@@ -4818,95 +4821,95 @@ static void disas_s390_insn(DisasContext *s)
             tcg_temp_free_i64(tmp2);
             break;
         default:
-            disas_b2(s, op, insn);
+            disas_b2(env, s, op, insn);
             break;
         }
         break;
     case 0xb3:
-        insn = ld_code4(s->pc);
+        insn = ld_code4(env, s->pc);
         op = (insn >> 16) & 0xff;
         r3 = (insn >> 12) & 0xf; /* aka m3 */
         r1 = (insn >> 4) & 0xf;
         r2 = insn & 0xf;
-        disas_b3(s, op, r3, r1, r2);
+        disas_b3(env, s, op, r3, r1, r2);
         break;
 #ifndef CONFIG_USER_ONLY
     case 0xb6: /* STCTL     R1,R3,D2(B2)     [RS] */
         /* Store Control */
-        check_privileged(s, ilc);
-        insn = ld_code4(s->pc);
+        check_privileged(env, s, ilc);
+        insn = ld_code4(env, s->pc);
         decode_rs(s, insn, &r1, &r3, &b2, &d2);
         tmp = get_address(s, 0, b2, d2);
         tmp32_1 = tcg_const_i32(r1);
         tmp32_2 = tcg_const_i32(r3);
         potential_page_fault(s);
-        gen_helper_stctl(tmp32_1, tmp, tmp32_2);
+        gen_helper_stctl(cpu_env, tmp32_1, tmp, tmp32_2);
         tcg_temp_free_i64(tmp);
         tcg_temp_free_i32(tmp32_1);
         tcg_temp_free_i32(tmp32_2);
         break;
     case 0xb7: /* LCTL      R1,R3,D2(B2)     [RS] */
         /* Load Control */
-        check_privileged(s, ilc);
-        insn = ld_code4(s->pc);
+        check_privileged(env, s, ilc);
+        insn = ld_code4(env, s->pc);
         decode_rs(s, insn, &r1, &r3, &b2, &d2);
         tmp = get_address(s, 0, b2, d2);
         tmp32_1 = tcg_const_i32(r1);
         tmp32_2 = tcg_const_i32(r3);
         potential_page_fault(s);
-        gen_helper_lctl(tmp32_1, tmp, tmp32_2);
+        gen_helper_lctl(cpu_env, tmp32_1, tmp, tmp32_2);
         tcg_temp_free_i64(tmp);
         tcg_temp_free_i32(tmp32_1);
         tcg_temp_free_i32(tmp32_2);
         break;
 #endif
     case 0xb9:
-        insn = ld_code4(s->pc);
+        insn = ld_code4(env, s->pc);
         r1 = (insn >> 4) & 0xf;
         r2 = insn & 0xf;
         op = (insn >> 16) & 0xff;
-        disas_b9(s, op, r1, r2);
+        disas_b9(env, s, op, r1, r2);
         break;
     case 0xba: /* CS     R1,R3,D2(B2)     [RS] */
-        insn = ld_code4(s->pc);
+        insn = ld_code4(env, s->pc);
         decode_rs(s, insn, &r1, &r3, &b2, &d2);
         tmp = get_address(s, 0, b2, d2);
         tmp32_1 = tcg_const_i32(r1);
         tmp32_2 = tcg_const_i32(r3);
         potential_page_fault(s);
-        gen_helper_cs(cc_op, tmp32_1, tmp, tmp32_2);
+        gen_helper_cs(cc_op, cpu_env, tmp32_1, tmp, tmp32_2);
         set_cc_static(s);
         tcg_temp_free_i64(tmp);
         tcg_temp_free_i32(tmp32_1);
         tcg_temp_free_i32(tmp32_2);
         break;
     case 0xbd: /* CLM    R1,M3,D2(B2)     [RS] */
-        insn = ld_code4(s->pc);
+        insn = ld_code4(env, s->pc);
         decode_rs(s, insn, &r1, &r3, &b2, &d2);
         tmp = get_address(s, 0, b2, d2);
         tmp32_1 = load_reg32(r1);
         tmp32_2 = tcg_const_i32(r3);
         potential_page_fault(s);
-        gen_helper_clm(cc_op, tmp32_1, tmp32_2, tmp);
+        gen_helper_clm(cc_op, cpu_env, tmp32_1, tmp32_2, tmp);
         set_cc_static(s);
         tcg_temp_free_i64(tmp);
         tcg_temp_free_i32(tmp32_1);
         tcg_temp_free_i32(tmp32_2);
         break;
     case 0xbe: /* STCM R1,M3,D2(B2) [RS] */
-        insn = ld_code4(s->pc);
+        insn = ld_code4(env, s->pc);
         decode_rs(s, insn, &r1, &r3, &b2, &d2);
         tmp = get_address(s, 0, b2, d2);
         tmp32_1 = load_reg32(r1);
         tmp32_2 = tcg_const_i32(r3);
         potential_page_fault(s);
-        gen_helper_stcm(tmp32_1, tmp32_2, tmp);
+        gen_helper_stcm(cpu_env, tmp32_1, tmp32_2, tmp);
         tcg_temp_free_i64(tmp);
         tcg_temp_free_i32(tmp32_1);
         tcg_temp_free_i32(tmp32_2);
         break;
     case 0xbf: /* ICM    R1,M3,D2(B2)     [RS] */
-        insn = ld_code4(s->pc);
+        insn = ld_code4(env, s->pc);
         decode_rs(s, insn, &r1, &r3, &b2, &d2);
         if (r3 == 15) {
             /* effectively a 32-bit load */
@@ -4961,16 +4964,16 @@ static void disas_s390_insn(DisasContext *s)
         break;
     case 0xc0:
     case 0xc2:
-        insn = ld_code6(s->pc);
+        insn = ld_code6(env, s->pc);
         r1 = (insn >> 36) & 0xf;
         op = (insn >> 32) & 0xf;
         i2 = (int)insn;
         switch (opc) {
         case 0xc0:
-            disas_c0(s, op, r1, i2);
+            disas_c0(env, s, op, r1, i2);
             break;
         case 0xc2:
-            disas_c2(s, op, r1, i2);
+            disas_c2(env, s, op, r1, i2);
             break;
         default:
             tcg_abort();
@@ -4983,7 +4986,7 @@ static void disas_s390_insn(DisasContext *s)
     case 0xd7: /* XC     D1(L,B1),D2(B2)         [SS] */
     case 0xdc: /* TR     D1(L,B1),D2(B2)         [SS] */
     case 0xf3: /* UNPK   D1(L1,B1),D2(L2,B2)     [SS] */
-        insn = ld_code6(s->pc);
+        insn = ld_code6(env, s->pc);
         vl = tcg_const_i32((insn >> 32) & 0xff);
         b1 = (insn >> 28) & 0xf;
         b2 = (insn >> 12) & 0xf;
@@ -4997,7 +5000,7 @@ static void disas_s390_insn(DisasContext *s)
             break;
         case 0xd4:
             potential_page_fault(s);
-            gen_helper_nc(cc_op, vl, tmp, tmp2);
+            gen_helper_nc(cc_op, cpu_env, vl, tmp, tmp2);
             set_cc_static(s);
             break;
         case 0xd5:
@@ -5005,22 +5008,22 @@ static void disas_s390_insn(DisasContext *s)
             break;
         case 0xd6:
             potential_page_fault(s);
-            gen_helper_oc(cc_op, vl, tmp, tmp2);
+            gen_helper_oc(cc_op, cpu_env, vl, tmp, tmp2);
             set_cc_static(s);
             break;
         case 0xd7:
             potential_page_fault(s);
-            gen_helper_xc(cc_op, vl, tmp, tmp2);
+            gen_helper_xc(cc_op, cpu_env, vl, tmp, tmp2);
             set_cc_static(s);
             break;
         case 0xdc:
             potential_page_fault(s);
-            gen_helper_tr(vl, tmp, tmp2);
+            gen_helper_tr(cpu_env, vl, tmp, tmp2);
             set_cc_static(s);
             break;
         case 0xf3:
             potential_page_fault(s);
-            gen_helper_unpk(vl, tmp, tmp2);
+            gen_helper_unpk(cpu_env, vl, tmp, tmp2);
             break;
         default:
             tcg_abort();
@@ -5031,9 +5034,9 @@ static void disas_s390_insn(DisasContext *s)
 #ifndef CONFIG_USER_ONLY
     case 0xda: /* MVCP     D1(R1,B1),D2(B2),R3   [SS] */
     case 0xdb: /* MVCS     D1(R1,B1),D2(B2),R3   [SS] */
-        check_privileged(s, ilc);
+        check_privileged(env, s, ilc);
         potential_page_fault(s);
-        insn = ld_code6(s->pc);
+        insn = ld_code6(env, s->pc);
         r1 = (insn >> 36) & 0xf;
         r3 = (insn >> 32) & 0xf;
         b1 = (insn >> 28) & 0xf;
@@ -5045,9 +5048,9 @@ static void disas_s390_insn(DisasContext *s)
         tmp2 = get_address(s, 0, b1, d1);
         tmp3 = get_address(s, 0, b2, d2);
         if (opc == 0xda) {
-            gen_helper_mvcp(cc_op, tmp, tmp2, tmp3);
+            gen_helper_mvcp(cc_op, cpu_env, tmp, tmp2, tmp3);
         } else {
-            gen_helper_mvcs(cc_op, tmp, tmp2, tmp3);
+            gen_helper_mvcs(cc_op, cpu_env, tmp, tmp2, tmp3);
         }
         set_cc_static(s);
         tcg_temp_free_i64(tmp);
@@ -5056,7 +5059,7 @@ static void disas_s390_insn(DisasContext *s)
         break;
 #endif
     case 0xe3:
-        insn = ld_code6(s->pc);
+        insn = ld_code6(env, s->pc);
         debug_insn(insn);
         op = insn & 0xff;
         r1 = (insn >> 36) & 0xf;
@@ -5064,19 +5067,19 @@ static void disas_s390_insn(DisasContext *s)
         b2 = (insn >> 28) & 0xf;
         d2 = ((int)((((insn >> 16) & 0xfff)
            | ((insn << 4) & 0xff000)) << 12)) >> 12;
-        disas_e3(s, op,  r1, x2, b2, d2 );
+        disas_e3(env, s, op,  r1, x2, b2, d2 );
         break;
 #ifndef CONFIG_USER_ONLY
     case 0xe5:
         /* Test Protection */
-        check_privileged(s, ilc);
-        insn = ld_code6(s->pc);
+        check_privileged(env, s, ilc);
+        insn = ld_code6(env, s->pc);
         debug_insn(insn);
-        disas_e5(s, insn);
+        disas_e5(env, s, insn);
         break;
 #endif
     case 0xeb:
-        insn = ld_code6(s->pc);
+        insn = ld_code6(env, s->pc);
         debug_insn(insn);
         op = insn & 0xff;
         r1 = (insn >> 36) & 0xf;
@@ -5084,10 +5087,10 @@ static void disas_s390_insn(DisasContext *s)
         b2 = (insn >> 28) & 0xf;
         d2 = ((int)((((insn >> 16) & 0xfff)
            | ((insn << 4) & 0xff000)) << 12)) >> 12;
-        disas_eb(s, op, r1, r3, b2, d2);
+        disas_eb(env, s, op, r1, r3, b2, d2);
         break;
     case 0xed:
-        insn = ld_code6(s->pc);
+        insn = ld_code6(env, s->pc);
         debug_insn(insn);
         op = insn & 0xff;
         r1 = (insn >> 36) & 0xf;
@@ -5095,11 +5098,11 @@ static void disas_s390_insn(DisasContext *s)
         b2 = (insn >> 28) & 0xf;
         d2 = (short)((insn >> 16) & 0xfff);
         r1b = (insn >> 12) & 0xf;
-        disas_ed(s, op, r1, x2, b2, d2, r1b);
+        disas_ed(env, s, op, r1, x2, b2, d2, r1b);
         break;
     default:
         qemu_log_mask(LOG_UNIMP, "unimplemented opcode 0x%x\n", opc);
-        gen_illegal_opcode(s, ilc);
+        gen_illegal_opcode(env, s, ilc);
         break;
     }
 
@@ -5131,7 +5134,7 @@ static inline void gen_intermediate_code_internal(CPUS390XState *env,
     dc.tb = tb;
     dc.cc_op = CC_OP_DYNAMIC;
 
-    gen_opc_end = gen_opc_buf + OPC_MAX_SIZE;
+    gen_opc_end = tcg_ctx.gen_opc_buf + OPC_MAX_SIZE;
 
     next_page_start = (pc_start & TARGET_PAGE_MASK) + TARGET_PAGE_SIZE;
 
@@ -5153,7 +5156,7 @@ static inline void gen_intermediate_code_internal(CPUS390XState *env,
             }
         }
         if (search_pc) {
-            j = gen_opc_ptr - gen_opc_buf;
+            j = tcg_ctx.gen_opc_ptr - tcg_ctx.gen_opc_buf;
             if (lj < j) {
                 lj++;
                 while (lj < j) {
@@ -5168,17 +5171,19 @@ static inline void gen_intermediate_code_internal(CPUS390XState *env,
         if (num_insns + 1 == max_insns && (tb->cflags & CF_LAST_IO)) {
             gen_io_start();
         }
-#if defined(S390X_DEBUG_DISAS_VERBOSE)
-        LOG_DISAS("pc " TARGET_FMT_lx "\n",
-                  dc.pc);
-#endif
-        disas_s390_insn(&dc);
+
+        if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
+            tcg_gen_debug_insn_start(dc.pc);
+        }
+
+        disas_s390_insn(env, &dc);
 
         num_insns++;
         if (env->singlestep_enabled) {
             gen_debug(&dc);
         }
-    } while (!dc.is_jmp && gen_opc_ptr < gen_opc_end && dc.pc < next_page_start
+    } while (!dc.is_jmp && tcg_ctx.gen_opc_ptr < gen_opc_end
+             && dc.pc < next_page_start
              && num_insns < max_insns && !env->singlestep_enabled
              && !singlestep);
 
@@ -5202,9 +5207,9 @@ static inline void gen_intermediate_code_internal(CPUS390XState *env,
         tcg_gen_exit_tb(0);
     }
     gen_icount_end(tb, num_insns);
-    *gen_opc_ptr = INDEX_op_end;
+    *tcg_ctx.gen_opc_ptr = INDEX_op_end;
     if (search_pc) {
-        j = gen_opc_ptr - gen_opc_buf;
+        j = tcg_ctx.gen_opc_ptr - tcg_ctx.gen_opc_buf;
         lj++;
         while (lj <= j) {
             gen_opc_instr_start[lj++] = 0;
@@ -5214,10 +5219,9 @@ static inline void gen_intermediate_code_internal(CPUS390XState *env,
         tb->icount = num_insns;
     }
 #if defined(S390X_DEBUG_DISAS)
-    log_cpu_state_mask(CPU_LOG_TB_CPU, env, 0);
     if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
         qemu_log("IN: %s\n", lookup_symbol(pc_start));
-        log_target_disas(pc_start, dc.pc - pc_start, 1);
+        log_target_disas(env, pc_start, dc.pc - pc_start, 1);
         qemu_log("\n");
     }
 #endif
diff --git a/target-sh4/Makefile.objs b/target-sh4/Makefile.objs
index 2e0e093e1..ca20f2144 100644
--- a/target-sh4/Makefile.objs
+++ b/target-sh4/Makefile.objs
@@ -1,4 +1,2 @@
 obj-y += translate.o op_helper.o helper.o cpu.o
 obj-$(CONFIG_SOFTMMU) += machine.o
-
-$(obj)/op_helper.o: QEMU_CFLAGS += $(HELPER_CFLAGS)
diff --git a/target-sh4/cpu.h b/target-sh4/cpu.h
index bf592227e..9a0e72b1f 100644
--- a/target-sh4/cpu.h
+++ b/target-sh4/cpu.h
@@ -204,20 +204,20 @@ void sh4_cpu_list(FILE *f, fprintf_function cpu_fprintf);
 #if !defined(CONFIG_USER_ONLY)
 void cpu_sh4_invalidate_tlb(CPUSH4State *s);
 uint32_t cpu_sh4_read_mmaped_itlb_addr(CPUSH4State *s,
-                                       target_phys_addr_t addr);
-void cpu_sh4_write_mmaped_itlb_addr(CPUSH4State *s, target_phys_addr_t addr,
+                                       hwaddr addr);
+void cpu_sh4_write_mmaped_itlb_addr(CPUSH4State *s, hwaddr addr,
                                     uint32_t mem_value);
 uint32_t cpu_sh4_read_mmaped_itlb_data(CPUSH4State *s,
-                                       target_phys_addr_t addr);
-void cpu_sh4_write_mmaped_itlb_data(CPUSH4State *s, target_phys_addr_t addr,
+                                       hwaddr addr);
+void cpu_sh4_write_mmaped_itlb_data(CPUSH4State *s, hwaddr addr,
                                     uint32_t mem_value);
 uint32_t cpu_sh4_read_mmaped_utlb_addr(CPUSH4State *s,
-                                       target_phys_addr_t addr);
-void cpu_sh4_write_mmaped_utlb_addr(CPUSH4State *s, target_phys_addr_t addr,
+                                       hwaddr addr);
+void cpu_sh4_write_mmaped_utlb_addr(CPUSH4State *s, hwaddr addr,
                                     uint32_t mem_value);
 uint32_t cpu_sh4_read_mmaped_utlb_data(CPUSH4State *s,
-                                       target_phys_addr_t addr);
-void cpu_sh4_write_mmaped_utlb_data(CPUSH4State *s, target_phys_addr_t addr,
+                                       hwaddr addr);
+void cpu_sh4_write_mmaped_utlb_data(CPUSH4State *s, hwaddr addr,
                                     uint32_t mem_value);
 #endif
 
@@ -371,8 +371,10 @@ static inline void cpu_get_tb_cpu_state(CPUSH4State *env, target_ulong *pc,
             | (env->movcal_backup ? TB_FLAG_PENDING_MOVCA : 0); /* Bit 4 */
 }
 
-static inline bool cpu_has_work(CPUSH4State *env)
+static inline bool cpu_has_work(CPUState *cpu)
 {
+    CPUSH4State *env = &SUPERH_CPU(cpu)->env;
+
     return env->interrupt_request & CPU_INTERRUPT_HARD;
 }
 
diff --git a/target-sh4/helper.c b/target-sh4/helper.c
index 5c57380f1..ddebc7896 100644
--- a/target-sh4/helper.c
+++ b/target-sh4/helper.c
@@ -503,7 +503,7 @@ int cpu_sh4_handle_mmu_fault(CPUSH4State * env, target_ulong address, int rw,
     return 0;
 }
 
-target_phys_addr_t cpu_get_phys_page_debug(CPUSH4State * env, target_ulong addr)
+hwaddr cpu_get_phys_page_debug(CPUSH4State * env, target_ulong addr)
 {
     target_ulong physical;
     int prot;
@@ -574,7 +574,7 @@ void cpu_load_tlb(CPUSH4State * env)
 }
 
 uint32_t cpu_sh4_read_mmaped_itlb_addr(CPUSH4State *s,
-                                       target_phys_addr_t addr)
+                                       hwaddr addr)
 {
     int index = (addr & 0x00000300) >> 8;
     tlb_t * entry = &s->itlb[index];
@@ -584,7 +584,7 @@ uint32_t cpu_sh4_read_mmaped_itlb_addr(CPUSH4State *s,
            (entry->asid);
 }
 
-void cpu_sh4_write_mmaped_itlb_addr(CPUSH4State *s, target_phys_addr_t addr,
+void cpu_sh4_write_mmaped_itlb_addr(CPUSH4State *s, hwaddr addr,
 				    uint32_t mem_value)
 {
     uint32_t vpn = (mem_value & 0xfffffc00) >> 10;
@@ -604,7 +604,7 @@ void cpu_sh4_write_mmaped_itlb_addr(CPUSH4State *s, target_phys_addr_t addr,
 }
 
 uint32_t cpu_sh4_read_mmaped_itlb_data(CPUSH4State *s,
-                                       target_phys_addr_t addr)
+                                       hwaddr addr)
 {
     int array = (addr & 0x00800000) >> 23;
     int index = (addr & 0x00000300) >> 8;
@@ -626,7 +626,7 @@ uint32_t cpu_sh4_read_mmaped_itlb_data(CPUSH4State *s,
     }
 }
 
-void cpu_sh4_write_mmaped_itlb_data(CPUSH4State *s, target_phys_addr_t addr,
+void cpu_sh4_write_mmaped_itlb_data(CPUSH4State *s, hwaddr addr,
                                     uint32_t mem_value)
 {
     int array = (addr & 0x00800000) >> 23;
@@ -655,7 +655,7 @@ void cpu_sh4_write_mmaped_itlb_data(CPUSH4State *s, target_phys_addr_t addr,
 }
 
 uint32_t cpu_sh4_read_mmaped_utlb_addr(CPUSH4State *s,
-                                       target_phys_addr_t addr)
+                                       hwaddr addr)
 {
     int index = (addr & 0x00003f00) >> 8;
     tlb_t * entry = &s->utlb[index];
@@ -667,7 +667,7 @@ uint32_t cpu_sh4_read_mmaped_utlb_addr(CPUSH4State *s,
            (entry->asid);
 }
 
-void cpu_sh4_write_mmaped_utlb_addr(CPUSH4State *s, target_phys_addr_t addr,
+void cpu_sh4_write_mmaped_utlb_addr(CPUSH4State *s, hwaddr addr,
 				    uint32_t mem_value)
 {
     int associate = addr & 0x0000080;
@@ -740,7 +740,7 @@ void cpu_sh4_write_mmaped_utlb_addr(CPUSH4State *s, target_phys_addr_t addr,
 }
 
 uint32_t cpu_sh4_read_mmaped_utlb_data(CPUSH4State *s,
-                                       target_phys_addr_t addr)
+                                       hwaddr addr)
 {
     int array = (addr & 0x00800000) >> 23;
     int index = (addr & 0x00003f00) >> 8;
@@ -766,7 +766,7 @@ uint32_t cpu_sh4_read_mmaped_utlb_data(CPUSH4State *s,
     }
 }
 
-void cpu_sh4_write_mmaped_utlb_data(CPUSH4State *s, target_phys_addr_t addr,
+void cpu_sh4_write_mmaped_utlb_data(CPUSH4State *s, hwaddr addr,
                                     uint32_t mem_value)
 {
     int array = (addr & 0x00800000) >> 23;
diff --git a/target-sh4/helper.h b/target-sh4/helper.h
index 95e3c7c8f..304b77ba3 100644
--- a/target-sh4/helper.h
+++ b/target-sh4/helper.h
@@ -1,54 +1,50 @@
 #include "def-helper.h"
 
-DEF_HELPER_0(ldtlb, void)
-DEF_HELPER_0(raise_illegal_instruction, void)
-DEF_HELPER_0(raise_slot_illegal_instruction, void)
-DEF_HELPER_0(raise_fpu_disable, void)
-DEF_HELPER_0(raise_slot_fpu_disable, void)
-DEF_HELPER_0(debug, void)
-DEF_HELPER_1(sleep, void, i32)
-DEF_HELPER_1(trapa, void, i32)
+DEF_HELPER_1(ldtlb, void, env)
+DEF_HELPER_1(raise_illegal_instruction, noreturn, env)
+DEF_HELPER_1(raise_slot_illegal_instruction, noreturn, env)
+DEF_HELPER_1(raise_fpu_disable, noreturn, env)
+DEF_HELPER_1(raise_slot_fpu_disable, noreturn, env)
+DEF_HELPER_1(debug, noreturn, env)
+DEF_HELPER_1(sleep, noreturn, env)
+DEF_HELPER_2(trapa, noreturn, env, i32)
 
-DEF_HELPER_2(movcal, void, i32, i32)
-DEF_HELPER_0(discard_movcal_backup, void)
-DEF_HELPER_1(ocbi, void, i32)
+DEF_HELPER_3(movcal, void, env, i32, i32)
+DEF_HELPER_1(discard_movcal_backup, void, env)
+DEF_HELPER_2(ocbi, void, env, i32)
 
-DEF_HELPER_2(addv, i32, i32, i32)
-DEF_HELPER_2(addc, i32, i32, i32)
-DEF_HELPER_2(subv, i32, i32, i32)
-DEF_HELPER_2(subc, i32, i32, i32)
-DEF_HELPER_2(div1, i32, i32, i32)
-DEF_HELPER_2(macl, void, i32, i32)
-DEF_HELPER_2(macw, void, i32, i32)
+DEF_HELPER_3(div1, i32, env, i32, i32)
+DEF_HELPER_3(macl, void, env, i32, i32)
+DEF_HELPER_3(macw, void, env, i32, i32)
 
-DEF_HELPER_1(ld_fpscr, void, i32)
+DEF_HELPER_2(ld_fpscr, void, env, i32)
 
-DEF_HELPER_1(fabs_FT, f32, f32)
-DEF_HELPER_1(fabs_DT, f64, f64)
-DEF_HELPER_2(fadd_FT, f32, f32, f32)
-DEF_HELPER_2(fadd_DT, f64, f64, f64)
-DEF_HELPER_1(fcnvsd_FT_DT, f64, f32)
-DEF_HELPER_1(fcnvds_DT_FT, f32, f64)
+DEF_HELPER_FLAGS_1(fabs_FT, TCG_CALL_NO_RWG_SE, f32, f32)
+DEF_HELPER_FLAGS_1(fabs_DT, TCG_CALL_NO_RWG_SE, f64, f64)
+DEF_HELPER_3(fadd_FT, f32, env, f32, f32)
+DEF_HELPER_3(fadd_DT, f64, env, f64, f64)
+DEF_HELPER_2(fcnvsd_FT_DT, f64, env, f32)
+DEF_HELPER_2(fcnvds_DT_FT, f32, env, f64)
 
-DEF_HELPER_2(fcmp_eq_FT, void, f32, f32)
-DEF_HELPER_2(fcmp_eq_DT, void, f64, f64)
-DEF_HELPER_2(fcmp_gt_FT, void, f32, f32)
-DEF_HELPER_2(fcmp_gt_DT, void, f64, f64)
-DEF_HELPER_2(fdiv_FT, f32, f32, f32)
-DEF_HELPER_2(fdiv_DT, f64, f64, f64)
-DEF_HELPER_1(float_FT, f32, i32)
-DEF_HELPER_1(float_DT, f64, i32)
-DEF_HELPER_3(fmac_FT, f32, f32, f32, f32)
-DEF_HELPER_2(fmul_FT, f32, f32, f32)
-DEF_HELPER_2(fmul_DT, f64, f64, f64)
-DEF_HELPER_1(fneg_T, f32, f32)
-DEF_HELPER_2(fsub_FT, f32, f32, f32)
-DEF_HELPER_2(fsub_DT, f64, f64, f64)
-DEF_HELPER_1(fsqrt_FT, f32, f32)
-DEF_HELPER_1(fsqrt_DT, f64, f64)
-DEF_HELPER_1(ftrc_FT, i32, f32)
-DEF_HELPER_1(ftrc_DT, i32, f64)
-DEF_HELPER_2(fipr, void, i32, i32)
-DEF_HELPER_1(ftrv, void, i32)
+DEF_HELPER_3(fcmp_eq_FT, void, env, f32, f32)
+DEF_HELPER_3(fcmp_eq_DT, void, env, f64, f64)
+DEF_HELPER_3(fcmp_gt_FT, void, env, f32, f32)
+DEF_HELPER_3(fcmp_gt_DT, void, env, f64, f64)
+DEF_HELPER_3(fdiv_FT, f32, env, f32, f32)
+DEF_HELPER_3(fdiv_DT, f64, env, f64, f64)
+DEF_HELPER_2(float_FT, f32, env, i32)
+DEF_HELPER_2(float_DT, f64, env, i32)
+DEF_HELPER_4(fmac_FT, f32, env, f32, f32, f32)
+DEF_HELPER_3(fmul_FT, f32, env, f32, f32)
+DEF_HELPER_3(fmul_DT, f64, env, f64, f64)
+DEF_HELPER_FLAGS_1(fneg_T, TCG_CALL_NO_RWG_SE, f32, f32)
+DEF_HELPER_3(fsub_FT, f32, env, f32, f32)
+DEF_HELPER_3(fsub_DT, f64, env, f64, f64)
+DEF_HELPER_2(fsqrt_FT, f32, env, f32)
+DEF_HELPER_2(fsqrt_DT, f64, env, f64)
+DEF_HELPER_2(ftrc_FT, i32, env, f32)
+DEF_HELPER_2(ftrc_DT, i32, env, f64)
+DEF_HELPER_3(fipr, void, env, i32, i32)
+DEF_HELPER_2(ftrv, void, env, i32)
 
 #include "def-helper.h"
diff --git a/target-sh4/op_helper.c b/target-sh4/op_helper.c
index 40547911c..60ec4cbc4 100644
--- a/target-sh4/op_helper.c
+++ b/target-sh4/op_helper.c
@@ -19,10 +19,10 @@
 #include <assert.h>
 #include <stdlib.h>
 #include "cpu.h"
-#include "dyngen-exec.h"
 #include "helper.h"
 
-static void cpu_restore_state_from_retaddr(uintptr_t retaddr)
+static inline void cpu_restore_state_from_retaddr(CPUSH4State *env,
+                                                  uintptr_t retaddr)
 {
     TranslationBlock *tb;
 
@@ -53,26 +53,22 @@ static void cpu_restore_state_from_retaddr(uintptr_t retaddr)
 #define SHIFT 3
 #include "softmmu_template.h"
 
-void tlb_fill(CPUSH4State *env1, target_ulong addr, int is_write, int mmu_idx,
+void tlb_fill(CPUSH4State *env, target_ulong addr, int is_write, int mmu_idx,
               uintptr_t retaddr)
 {
-    CPUSH4State *saved_env;
     int ret;
 
-    saved_env = env;
-    env = env1;
     ret = cpu_sh4_handle_mmu_fault(env, addr, is_write, mmu_idx);
     if (ret) {
         /* now we have a real cpu fault */
-        cpu_restore_state_from_retaddr(retaddr);
+        cpu_restore_state_from_retaddr(env, retaddr);
         cpu_loop_exit(env);
     }
-    env = saved_env;
 }
 
 #endif
 
-void helper_ldtlb(void)
+void helper_ldtlb(CPUSH4State *env)
 {
 #ifdef CONFIG_USER_ONLY
     /* XXXXX */
@@ -82,55 +78,53 @@ void helper_ldtlb(void)
 #endif
 }
 
-static inline void raise_exception(int index, uintptr_t retaddr)
+static inline void QEMU_NORETURN raise_exception(CPUSH4State *env, int index,
+                                                 uintptr_t retaddr)
 {
     env->exception_index = index;
-    cpu_restore_state_from_retaddr(retaddr);
+    cpu_restore_state_from_retaddr(env, retaddr);
     cpu_loop_exit(env);
 }
 
-void helper_raise_illegal_instruction(void)
+void helper_raise_illegal_instruction(CPUSH4State *env)
 {
-    raise_exception(0x180, GETPC());
+    raise_exception(env, 0x180, 0);
 }
 
-void helper_raise_slot_illegal_instruction(void)
+void helper_raise_slot_illegal_instruction(CPUSH4State *env)
 {
-    raise_exception(0x1a0, GETPC());
+    raise_exception(env, 0x1a0, 0);
 }
 
-void helper_raise_fpu_disable(void)
+void helper_raise_fpu_disable(CPUSH4State *env)
 {
-    raise_exception(0x800, GETPC());
+    raise_exception(env, 0x800, 0);
 }
 
-void helper_raise_slot_fpu_disable(void)
+void helper_raise_slot_fpu_disable(CPUSH4State *env)
 {
-    raise_exception(0x820, GETPC());
+    raise_exception(env, 0x820, 0);
 }
 
-void helper_debug(void)
+void helper_debug(CPUSH4State *env)
 {
-    env->exception_index = EXCP_DEBUG;
-    cpu_loop_exit(env);
+    raise_exception(env, EXCP_DEBUG, 0);
 }
 
-void helper_sleep(uint32_t next_pc)
+void helper_sleep(CPUSH4State *env)
 {
     env->halted = 1;
     env->in_sleep = 1;
-    env->exception_index = EXCP_HLT;
-    env->pc = next_pc;
-    cpu_loop_exit(env);
+    raise_exception(env, EXCP_HLT, 0);
 }
 
-void helper_trapa(uint32_t tra)
+void helper_trapa(CPUSH4State *env, uint32_t tra)
 {
     env->tra = tra << 2;
-    raise_exception(0x160, GETPC());
+    raise_exception(env, 0x160, 0);
 }
 
-void helper_movcal(uint32_t address, uint32_t value)
+void helper_movcal(CPUSH4State *env, uint32_t address, uint32_t value)
 {
     if (cpu_sh4_is_cached (env, address))
     {
@@ -144,7 +138,7 @@ void helper_movcal(uint32_t address, uint32_t value)
     }
 }
 
-void helper_discard_movcal_backup(void)
+void helper_discard_movcal_backup(CPUSH4State *env)
 {
     memory_content *current = env->movcal_backup;
 
@@ -158,7 +152,7 @@ void helper_discard_movcal_backup(void)
     } 
 }
 
-void helper_ocbi(uint32_t address)
+void helper_ocbi(CPUSH4State *env, uint32_t address)
 {
     memory_content **current = &(env->movcal_backup);
     while (*current)
@@ -167,7 +161,7 @@ void helper_ocbi(uint32_t address)
 	if ((a & ~0x1F) == (address & ~0x1F))
 	{
 	    memory_content *next = (*current)->next;
-	    stl(a, (*current)->value);
+            cpu_stl_data(env, a, (*current)->value);
 	    
 	    if (next == NULL)
 	    {
@@ -181,51 +175,6 @@ void helper_ocbi(uint32_t address)
     }
 }
 
-uint32_t helper_addc(uint32_t arg0, uint32_t arg1)
-{
-    uint32_t tmp0, tmp1;
-
-    tmp1 = arg0 + arg1;
-    tmp0 = arg1;
-    arg1 = tmp1 + (env->sr & 1);
-    if (tmp0 > tmp1)
-	env->sr |= SR_T;
-    else
-	env->sr &= ~SR_T;
-    if (tmp1 > arg1)
-	env->sr |= SR_T;
-    return arg1;
-}
-
-uint32_t helper_addv(uint32_t arg0, uint32_t arg1)
-{
-    uint32_t dest, src, ans;
-
-    if ((int32_t) arg1 >= 0)
-	dest = 0;
-    else
-	dest = 1;
-    if ((int32_t) arg0 >= 0)
-	src = 0;
-    else
-	src = 1;
-    src += dest;
-    arg1 += arg0;
-    if ((int32_t) arg1 >= 0)
-	ans = 0;
-    else
-	ans = 1;
-    ans += dest;
-    if (src == 0 || src == 2) {
-	if (ans == 1)
-	    env->sr |= SR_T;
-	else
-	    env->sr &= ~SR_T;
-    } else
-	env->sr &= ~SR_T;
-    return arg1;
-}
-
 #define T (env->sr & SR_T)
 #define Q (env->sr & SR_Q ? 1 : 0)
 #define M (env->sr & SR_M ? 1 : 0)
@@ -236,7 +185,7 @@ uint32_t helper_addv(uint32_t arg0, uint32_t arg1)
 #define SETM env->sr |= SR_M
 #define CLRM env->sr &= ~SR_M
 
-uint32_t helper_div1(uint32_t arg0, uint32_t arg1)
+uint32_t helper_div1(CPUSH4State *env, uint32_t arg0, uint32_t arg1)
 {
     uint32_t tmp0, tmp2;
     uint8_t old_q, tmp1 = 0xff;
@@ -344,7 +293,7 @@ uint32_t helper_div1(uint32_t arg0, uint32_t arg1)
     return arg1;
 }
 
-void helper_macl(uint32_t arg0, uint32_t arg1)
+void helper_macl(CPUSH4State *env, uint32_t arg0, uint32_t arg1)
 {
     int64_t res;
 
@@ -360,7 +309,7 @@ void helper_macl(uint32_t arg0, uint32_t arg1)
     }
 }
 
-void helper_macw(uint32_t arg0, uint32_t arg1)
+void helper_macw(CPUSH4State *env, uint32_t arg0, uint32_t arg1)
 {
     int64_t res;
 
@@ -379,62 +328,17 @@ void helper_macw(uint32_t arg0, uint32_t arg1)
     }
 }
 
-uint32_t helper_subc(uint32_t arg0, uint32_t arg1)
-{
-    uint32_t tmp0, tmp1;
-
-    tmp1 = arg1 - arg0;
-    tmp0 = arg1;
-    arg1 = tmp1 - (env->sr & SR_T);
-    if (tmp0 < tmp1)
-	env->sr |= SR_T;
-    else
-	env->sr &= ~SR_T;
-    if (tmp1 < arg1)
-	env->sr |= SR_T;
-    return arg1;
-}
-
-uint32_t helper_subv(uint32_t arg0, uint32_t arg1)
-{
-    int32_t dest, src, ans;
-
-    if ((int32_t) arg1 >= 0)
-	dest = 0;
-    else
-	dest = 1;
-    if ((int32_t) arg0 >= 0)
-	src = 0;
-    else
-	src = 1;
-    src += dest;
-    arg1 -= arg0;
-    if ((int32_t) arg1 >= 0)
-	ans = 0;
-    else
-	ans = 1;
-    ans += dest;
-    if (src == 1) {
-	if (ans == 1)
-	    env->sr |= SR_T;
-	else
-	    env->sr &= ~SR_T;
-    } else
-	env->sr &= ~SR_T;
-    return arg1;
-}
-
-static inline void set_t(void)
+static inline void set_t(CPUSH4State *env)
 {
     env->sr |= SR_T;
 }
 
-static inline void clr_t(void)
+static inline void clr_t(CPUSH4State *env)
 {
     env->sr &= ~SR_T;
 }
 
-void helper_ld_fpscr(uint32_t val)
+void helper_ld_fpscr(CPUSH4State *env, uint32_t val)
 {
     env->fpscr = val & FPSCR_MASK;
     if ((val & FPSCR_RM_MASK) == FPSCR_RM_ZERO) {
@@ -445,7 +349,7 @@ void helper_ld_fpscr(uint32_t val)
     set_flush_to_zero((val & FPSCR_DN) != 0, &env->fp_status);
 }
 
-static void update_fpscr(uintptr_t retaddr)
+static void update_fpscr(CPUSH4State *env, uintptr_t retaddr)
 {
     int xcpt, cause, enable;
 
@@ -479,9 +383,7 @@ static void update_fpscr(uintptr_t retaddr)
         cause = (env->fpscr & FPSCR_CAUSE_MASK) >> FPSCR_CAUSE_SHIFT;
         enable = (env->fpscr & FPSCR_ENABLE_MASK) >> FPSCR_ENABLE_SHIFT;
         if (cause & enable) {
-            cpu_restore_state_from_retaddr(retaddr);
-            env->exception_index = 0x120;
-            cpu_loop_exit(env);
+            raise_exception(env, 0x120, retaddr);
         }
     }
 }
@@ -496,156 +398,155 @@ float64 helper_fabs_DT(float64 t0)
     return float64_abs(t0);
 }
 
-float32 helper_fadd_FT(float32 t0, float32 t1)
+float32 helper_fadd_FT(CPUSH4State *env, float32 t0, float32 t1)
 {
     set_float_exception_flags(0, &env->fp_status);
     t0 = float32_add(t0, t1, &env->fp_status);
-    update_fpscr(GETPC());
+    update_fpscr(env, GETPC());
     return t0;
 }
 
-float64 helper_fadd_DT(float64 t0, float64 t1)
+float64 helper_fadd_DT(CPUSH4State *env, float64 t0, float64 t1)
 {
     set_float_exception_flags(0, &env->fp_status);
     t0 = float64_add(t0, t1, &env->fp_status);
-    update_fpscr(GETPC());
+    update_fpscr(env, GETPC());
     return t0;
 }
 
-void helper_fcmp_eq_FT(float32 t0, float32 t1)
+void helper_fcmp_eq_FT(CPUSH4State *env, float32 t0, float32 t1)
 {
     int relation;
 
     set_float_exception_flags(0, &env->fp_status);
     relation = float32_compare(t0, t1, &env->fp_status);
     if (unlikely(relation == float_relation_unordered)) {
-        update_fpscr(GETPC());
+        update_fpscr(env, GETPC());
     } else if (relation == float_relation_equal) {
-	set_t();
+        set_t(env);
     } else {
-	clr_t();
+        clr_t(env);
     }
 }
 
-void helper_fcmp_eq_DT(float64 t0, float64 t1)
+void helper_fcmp_eq_DT(CPUSH4State *env, float64 t0, float64 t1)
 {
     int relation;
 
     set_float_exception_flags(0, &env->fp_status);
     relation = float64_compare(t0, t1, &env->fp_status);
     if (unlikely(relation == float_relation_unordered)) {
-        update_fpscr(GETPC());
+        update_fpscr(env, GETPC());
     } else if (relation == float_relation_equal) {
-	set_t();
+        set_t(env);
     } else {
-	clr_t();
+        clr_t(env);
     }
 }
 
-void helper_fcmp_gt_FT(float32 t0, float32 t1)
+void helper_fcmp_gt_FT(CPUSH4State *env, float32 t0, float32 t1)
 {
     int relation;
 
     set_float_exception_flags(0, &env->fp_status);
     relation = float32_compare(t0, t1, &env->fp_status);
     if (unlikely(relation == float_relation_unordered)) {
-        update_fpscr(GETPC());
+        update_fpscr(env, GETPC());
     } else if (relation == float_relation_greater) {
-	set_t();
+        set_t(env);
     } else {
-	clr_t();
+        clr_t(env);
     }
 }
 
-void helper_fcmp_gt_DT(float64 t0, float64 t1)
+void helper_fcmp_gt_DT(CPUSH4State *env, float64 t0, float64 t1)
 {
     int relation;
 
     set_float_exception_flags(0, &env->fp_status);
     relation = float64_compare(t0, t1, &env->fp_status);
     if (unlikely(relation == float_relation_unordered)) {
-        update_fpscr(GETPC());
+        update_fpscr(env, GETPC());
     } else if (relation == float_relation_greater) {
-	set_t();
+        set_t(env);
     } else {
-	clr_t();
+        clr_t(env);
     }
 }
 
-float64 helper_fcnvsd_FT_DT(float32 t0)
+float64 helper_fcnvsd_FT_DT(CPUSH4State *env, float32 t0)
 {
     float64 ret;
     set_float_exception_flags(0, &env->fp_status);
     ret = float32_to_float64(t0, &env->fp_status);
-    update_fpscr(GETPC());
+    update_fpscr(env, GETPC());
     return ret;
 }
 
-float32 helper_fcnvds_DT_FT(float64 t0)
+float32 helper_fcnvds_DT_FT(CPUSH4State *env, float64 t0)
 {
     float32 ret;
     set_float_exception_flags(0, &env->fp_status);
     ret = float64_to_float32(t0, &env->fp_status);
-    update_fpscr(GETPC());
+    update_fpscr(env, GETPC());
     return ret;
 }
 
-float32 helper_fdiv_FT(float32 t0, float32 t1)
+float32 helper_fdiv_FT(CPUSH4State *env, float32 t0, float32 t1)
 {
     set_float_exception_flags(0, &env->fp_status);
     t0 = float32_div(t0, t1, &env->fp_status);
-    update_fpscr(GETPC());
+    update_fpscr(env, GETPC());
     return t0;
 }
 
-float64 helper_fdiv_DT(float64 t0, float64 t1)
+float64 helper_fdiv_DT(CPUSH4State *env, float64 t0, float64 t1)
 {
     set_float_exception_flags(0, &env->fp_status);
     t0 = float64_div(t0, t1, &env->fp_status);
-    update_fpscr(GETPC());
+    update_fpscr(env, GETPC());
     return t0;
 }
 
-float32 helper_float_FT(uint32_t t0)
+float32 helper_float_FT(CPUSH4State *env, uint32_t t0)
 {
     float32 ret;
     set_float_exception_flags(0, &env->fp_status);
     ret = int32_to_float32(t0, &env->fp_status);
-    update_fpscr(GETPC());
+    update_fpscr(env, GETPC());
     return ret;
 }
 
-float64 helper_float_DT(uint32_t t0)
+float64 helper_float_DT(CPUSH4State *env, uint32_t t0)
 {
     float64 ret;
     set_float_exception_flags(0, &env->fp_status);
     ret = int32_to_float64(t0, &env->fp_status);
-    update_fpscr(GETPC());
+    update_fpscr(env, GETPC());
     return ret;
 }
 
-float32 helper_fmac_FT(float32 t0, float32 t1, float32 t2)
+float32 helper_fmac_FT(CPUSH4State *env, float32 t0, float32 t1, float32 t2)
 {
     set_float_exception_flags(0, &env->fp_status);
-    t0 = float32_mul(t0, t1, &env->fp_status);
-    t0 = float32_add(t0, t2, &env->fp_status);
-    update_fpscr(GETPC());
+    t0 = float32_muladd(t0, t1, t2, 0, &env->fp_status);
+    update_fpscr(env, GETPC());
     return t0;
 }
 
-float32 helper_fmul_FT(float32 t0, float32 t1)
+float32 helper_fmul_FT(CPUSH4State *env, float32 t0, float32 t1)
 {
     set_float_exception_flags(0, &env->fp_status);
     t0 = float32_mul(t0, t1, &env->fp_status);
-    update_fpscr(GETPC());
+    update_fpscr(env, GETPC());
     return t0;
 }
 
-float64 helper_fmul_DT(float64 t0, float64 t1)
+float64 helper_fmul_DT(CPUSH4State *env, float64 t0, float64 t1)
 {
     set_float_exception_flags(0, &env->fp_status);
     t0 = float64_mul(t0, t1, &env->fp_status);
-    update_fpscr(GETPC());
+    update_fpscr(env, GETPC());
     return t0;
 }
 
@@ -654,57 +555,57 @@ float32 helper_fneg_T(float32 t0)
     return float32_chs(t0);
 }
 
-float32 helper_fsqrt_FT(float32 t0)
+float32 helper_fsqrt_FT(CPUSH4State *env, float32 t0)
 {
     set_float_exception_flags(0, &env->fp_status);
     t0 = float32_sqrt(t0, &env->fp_status);
-    update_fpscr(GETPC());
+    update_fpscr(env, GETPC());
     return t0;
 }
 
-float64 helper_fsqrt_DT(float64 t0)
+float64 helper_fsqrt_DT(CPUSH4State *env, float64 t0)
 {
     set_float_exception_flags(0, &env->fp_status);
     t0 = float64_sqrt(t0, &env->fp_status);
-    update_fpscr(GETPC());
+    update_fpscr(env, GETPC());
     return t0;
 }
 
-float32 helper_fsub_FT(float32 t0, float32 t1)
+float32 helper_fsub_FT(CPUSH4State *env, float32 t0, float32 t1)
 {
     set_float_exception_flags(0, &env->fp_status);
     t0 = float32_sub(t0, t1, &env->fp_status);
-    update_fpscr(GETPC());
+    update_fpscr(env, GETPC());
     return t0;
 }
 
-float64 helper_fsub_DT(float64 t0, float64 t1)
+float64 helper_fsub_DT(CPUSH4State *env, float64 t0, float64 t1)
 {
     set_float_exception_flags(0, &env->fp_status);
     t0 = float64_sub(t0, t1, &env->fp_status);
-    update_fpscr(GETPC());
+    update_fpscr(env, GETPC());
     return t0;
 }
 
-uint32_t helper_ftrc_FT(float32 t0)
+uint32_t helper_ftrc_FT(CPUSH4State *env, float32 t0)
 {
     uint32_t ret;
     set_float_exception_flags(0, &env->fp_status);
     ret = float32_to_int32_round_to_zero(t0, &env->fp_status);
-    update_fpscr(GETPC());
+    update_fpscr(env, GETPC());
     return ret;
 }
 
-uint32_t helper_ftrc_DT(float64 t0)
+uint32_t helper_ftrc_DT(CPUSH4State *env, float64 t0)
 {
     uint32_t ret;
     set_float_exception_flags(0, &env->fp_status);
     ret = float64_to_int32_round_to_zero(t0, &env->fp_status);
-    update_fpscr(GETPC());
+    update_fpscr(env, GETPC());
     return ret;
 }
 
-void helper_fipr(uint32_t m, uint32_t n)
+void helper_fipr(CPUSH4State *env, uint32_t m, uint32_t n)
 {
     int bank, i;
     float32 r, p;
@@ -719,12 +620,12 @@ void helper_fipr(uint32_t m, uint32_t n)
                         &env->fp_status);
         r = float32_add(r, p, &env->fp_status);
     }
-    update_fpscr(GETPC());
+    update_fpscr(env, GETPC());
 
     env->fregs[bank + n + 3] = r;
 }
 
-void helper_ftrv(uint32_t n)
+void helper_ftrv(CPUSH4State *env, uint32_t n)
 {
     int bank_matrix, bank_vector;
     int i, j;
@@ -743,7 +644,7 @@ void helper_ftrv(uint32_t n)
             r[i] = float32_add(r[i], p, &env->fp_status);
         }
     }
-    update_fpscr(GETPC());
+    update_fpscr(env, GETPC());
 
     for (i = 0 ; i < 4 ; i++) {
         env->fregs[bank_vector + i] = r[i];
diff --git a/target-sh4/translate.c b/target-sh4/translate.c
index 6532ad2ad..5497dede0 100644
--- a/target-sh4/translate.c
+++ b/target-sh4/translate.c
@@ -18,7 +18,6 @@
  */
 
 #define DEBUG_DISAS
-#define SH4_DEBUG_DISAS
 //#define SH4_SINGLE_STEP
 
 #include "cpu.h"
@@ -32,8 +31,6 @@
 typedef struct DisasContext {
     struct TranslationBlock *tb;
     target_ulong pc;
-    uint32_t sr;
-    uint32_t fpscr;
     uint16_t opcode;
     uint32_t flags;
     int bstate;
@@ -47,7 +44,7 @@ typedef struct DisasContext {
 #if defined(CONFIG_USER_ONLY)
 #define IS_USER(ctx) 1
 #else
-#define IS_USER(ctx) (!(ctx->sr & SR_MD))
+#define IS_USER(ctx) (!(ctx->flags & SR_MD))
 #endif
 
 enum {
@@ -276,7 +273,7 @@ static void gen_goto_tb(DisasContext * ctx, int n, target_ulong dest)
     } else {
         tcg_gen_movi_i32(cpu_pc, dest);
         if (ctx->singlestep_enabled)
-            gen_helper_debug();
+            gen_helper_debug(cpu_env);
         tcg_gen_exit_tb(0);
     }
 }
@@ -288,7 +285,7 @@ static void gen_jump(DisasContext * ctx)
 	   delayed jump as immediate jump are conditinal jumps */
 	tcg_gen_mov_i32(cpu_pc, cpu_delayed_pc);
 	if (ctx->singlestep_enabled)
-	    gen_helper_debug();
+            gen_helper_debug(cpu_env);
 	tcg_gen_exit_tb(0);
     } else {
 	gen_goto_tb(ctx, 0, ctx->delayed_pc);
@@ -339,16 +336,6 @@ static void gen_delayed_conditional_jump(DisasContext * ctx)
     gen_jump(ctx);
 }
 
-static inline void gen_set_t(void)
-{
-    tcg_gen_ori_i32(cpu_sr, cpu_sr, SR_T);
-}
-
-static inline void gen_clr_t(void)
-{
-    tcg_gen_andi_i32(cpu_sr, cpu_sr, ~SR_T);
-}
-
 static inline void gen_cmp(int cond, TCGv t0, TCGv t1)
 {
     TCGv t;
@@ -423,44 +410,47 @@ static inline void gen_store_fpr64 (TCGv_i64 t, int reg)
 #define B11_8 ((ctx->opcode >> 8) & 0xf)
 #define B15_12 ((ctx->opcode >> 12) & 0xf)
 
-#define REG(x) ((x) < 8 && (ctx->sr & (SR_MD | SR_RB)) == (SR_MD | SR_RB) ? \
-		(cpu_gregs[x + 16]) : (cpu_gregs[x]))
+#define REG(x) ((x) < 8 && (ctx->flags & (SR_MD | SR_RB)) == (SR_MD | SR_RB) \
+                ? (cpu_gregs[x + 16]) : (cpu_gregs[x]))
 
-#define ALTREG(x) ((x) < 8 && (ctx->sr & (SR_MD | SR_RB)) != (SR_MD | SR_RB) \
+#define ALTREG(x) ((x) < 8 && (ctx->flags & (SR_MD | SR_RB)) != (SR_MD | SR_RB)\
 		? (cpu_gregs[x + 16]) : (cpu_gregs[x]))
 
-#define FREG(x) (ctx->fpscr & FPSCR_FR ? (x) ^ 0x10 : (x))
+#define FREG(x) (ctx->flags & FPSCR_FR ? (x) ^ 0x10 : (x))
 #define XHACK(x) ((((x) & 1 ) << 4) | ((x) & 0xe))
-#define XREG(x) (ctx->fpscr & FPSCR_FR ? XHACK(x) ^ 0x10 : XHACK(x))
+#define XREG(x) (ctx->flags & FPSCR_FR ? XHACK(x) ^ 0x10 : XHACK(x))
 #define DREG(x) FREG(x) /* Assumes lsb of (x) is always 0 */
 
 #define CHECK_NOT_DELAY_SLOT \
   if (ctx->flags & (DELAY_SLOT | DELAY_SLOT_CONDITIONAL))     \
   {                                                           \
-      gen_helper_raise_slot_illegal_instruction();            \
-      ctx->bstate = BS_EXCP;                                  \
+      tcg_gen_movi_i32(cpu_pc, ctx->pc);                      \
+      gen_helper_raise_slot_illegal_instruction(cpu_env);     \
+      ctx->bstate = BS_BRANCH;                                \
       return;                                                 \
   }
 
 #define CHECK_PRIVILEGED                                        \
   if (IS_USER(ctx)) {                                           \
+      tcg_gen_movi_i32(cpu_pc, ctx->pc);                        \
       if (ctx->flags & (DELAY_SLOT | DELAY_SLOT_CONDITIONAL)) { \
-         gen_helper_raise_slot_illegal_instruction();           \
+          gen_helper_raise_slot_illegal_instruction(cpu_env);   \
       } else {                                                  \
-         gen_helper_raise_illegal_instruction();                \
+          gen_helper_raise_illegal_instruction(cpu_env);        \
       }                                                         \
-      ctx->bstate = BS_EXCP;                                    \
+      ctx->bstate = BS_BRANCH;                                  \
       return;                                                   \
   }
 
 #define CHECK_FPU_ENABLED                                       \
   if (ctx->flags & SR_FD) {                                     \
+      tcg_gen_movi_i32(cpu_pc, ctx->pc);                        \
       if (ctx->flags & (DELAY_SLOT | DELAY_SLOT_CONDITIONAL)) { \
-          gen_helper_raise_slot_fpu_disable();                  \
+          gen_helper_raise_slot_fpu_disable(cpu_env);           \
       } else {                                                  \
-          gen_helper_raise_fpu_disable();                       \
+          gen_helper_raise_fpu_disable(cpu_env);                \
       }                                                         \
-      ctx->bstate = BS_EXCP;                                    \
+      ctx->bstate = BS_BRANCH;                                  \
       return;                                                   \
   }
 
@@ -492,7 +482,7 @@ static void _decode_opc(DisasContext * ctx)
 	  if (opcode != 0x0093 /* ocbi */
 	      && opcode != 0x00c3 /* movca.l */)
 	      {
-		  gen_helper_discard_movcal_backup ();
+                  gen_helper_discard_movcal_backup(cpu_env);
 		  ctx->has_movcal = 0;
 	      }
 	}
@@ -519,11 +509,11 @@ static void _decode_opc(DisasContext * ctx)
 	tcg_gen_andi_i32(cpu_sr, cpu_sr, ~SR_S);
 	return;
     case 0x0008:		/* clrt */
-	gen_clr_t();
+        tcg_gen_andi_i32(cpu_sr, cpu_sr, ~SR_T);
 	return;
     case 0x0038:		/* ldtlb */
 	CHECK_PRIVILEGED
-	gen_helper_ldtlb();
+        gen_helper_ldtlb(cpu_env);
 	return;
     case 0x002b:		/* rte */
 	CHECK_PRIVILEGED
@@ -537,21 +527,22 @@ static void _decode_opc(DisasContext * ctx)
 	tcg_gen_ori_i32(cpu_sr, cpu_sr, SR_S);
 	return;
     case 0x0018:		/* sett */
-	gen_set_t();
+        tcg_gen_ori_i32(cpu_sr, cpu_sr, SR_T);
 	return;
     case 0xfbfd:		/* frchg */
 	tcg_gen_xori_i32(cpu_fpscr, cpu_fpscr, FPSCR_FR);
 	ctx->bstate = BS_STOP;
 	return;
     case 0xf3fd:		/* fschg */
-	tcg_gen_xori_i32(cpu_fpscr, cpu_fpscr, FPSCR_SZ);
+        tcg_gen_xori_i32(cpu_fpscr, cpu_fpscr, FPSCR_SZ);
 	ctx->bstate = BS_STOP;
 	return;
     case 0x0009:		/* nop */
 	return;
     case 0x001b:		/* sleep */
 	CHECK_PRIVILEGED
-	gen_helper_sleep(tcg_const_i32(ctx->pc + 2));
+        tcg_gen_movi_i32(cpu_pc, ctx->pc + 2);
+        gen_helper_sleep(cpu_env);
 	return;
     }
 
@@ -732,17 +723,7 @@ static void _decode_opc(DisasContext * ctx)
 	}
 	return;
     case 0x6009:		/* swap.w Rm,Rn */
-	{
-	    TCGv high, low;
-	    high = tcg_temp_new();
-	    tcg_gen_shli_i32(high, REG(B7_4), 16);
-	    low = tcg_temp_new();
-	    tcg_gen_shri_i32(low, REG(B7_4), 16);
-	    tcg_gen_ext16u_i32(low, low);
-	    tcg_gen_or_i32(REG(B11_8), high, low);
-	    tcg_temp_free(low);
-	    tcg_temp_free(high);
-	}
+        tcg_gen_rotli_i32(REG(B11_8), REG(B7_4), 16);
 	return;
     case 0x200d:		/* xtrct Rm,Rn */
 	{
@@ -751,7 +732,6 @@ static void _decode_opc(DisasContext * ctx)
 	    tcg_gen_shli_i32(high, REG(B7_4), 16);
 	    low = tcg_temp_new();
 	    tcg_gen_shri_i32(low, REG(B11_8), 16);
-	    tcg_gen_ext16u_i32(low, low);
 	    tcg_gen_or_i32(REG(B11_8), high, low);
 	    tcg_temp_free(low);
 	    tcg_temp_free(high);
@@ -761,10 +741,43 @@ static void _decode_opc(DisasContext * ctx)
 	tcg_gen_add_i32(REG(B11_8), REG(B11_8), REG(B7_4));
 	return;
     case 0x300e:		/* addc Rm,Rn */
-	gen_helper_addc(REG(B11_8), REG(B7_4), REG(B11_8));
+        {
+            TCGv t0, t1, t2;
+            t0 = tcg_temp_new();
+            tcg_gen_andi_i32(t0, cpu_sr, SR_T);
+            t1 = tcg_temp_new();
+            tcg_gen_add_i32(t1, REG(B7_4), REG(B11_8));
+            tcg_gen_add_i32(t0, t0, t1);
+            t2 = tcg_temp_new();
+            tcg_gen_setcond_i32(TCG_COND_GTU, t2, REG(B11_8), t1);
+            tcg_gen_setcond_i32(TCG_COND_GTU, t1, t1, t0);
+            tcg_gen_or_i32(t1, t1, t2);
+            tcg_temp_free(t2);
+            tcg_gen_andi_i32(cpu_sr, cpu_sr, ~SR_T);
+            tcg_gen_or_i32(cpu_sr, cpu_sr, t1);
+            tcg_temp_free(t1);
+            tcg_gen_mov_i32(REG(B11_8), t0);
+            tcg_temp_free(t0);
+        }
 	return;
     case 0x300f:		/* addv Rm,Rn */
-	gen_helper_addv(REG(B11_8), REG(B7_4), REG(B11_8));
+        {
+            TCGv t0, t1, t2;
+            t0 = tcg_temp_new();
+            tcg_gen_add_i32(t0, REG(B7_4), REG(B11_8));
+            t1 = tcg_temp_new();
+            tcg_gen_xor_i32(t1, t0, REG(B11_8));
+            t2 = tcg_temp_new();
+            tcg_gen_xor_i32(t2, REG(B7_4), REG(B11_8));
+            tcg_gen_andc_i32(t1, t1, t2);
+            tcg_temp_free(t2);
+            tcg_gen_shri_i32(t1, t1, 31);
+            tcg_gen_andi_i32(cpu_sr, cpu_sr, ~SR_T);
+            tcg_gen_or_i32(cpu_sr, cpu_sr, t1);
+            tcg_temp_free(t1);
+            tcg_gen_mov_i32(REG(B7_4), t0);
+            tcg_temp_free(t0);
+        }
 	return;
     case 0x2009:		/* and Rm,Rn */
 	tcg_gen_and_i32(REG(B11_8), REG(B11_8), REG(B7_4));
@@ -817,7 +830,7 @@ static void _decode_opc(DisasContext * ctx)
 	}
 	return;
     case 0x3004:		/* div1 Rm,Rn */
-	gen_helper_div1(REG(B11_8), REG(B7_4), REG(B11_8));
+        gen_helper_div1(REG(B11_8), cpu_env, REG(B7_4), REG(B11_8));
 	return;
     case 0x300d:		/* dmuls.l Rm,Rn */
 	{
@@ -870,7 +883,7 @@ static void _decode_opc(DisasContext * ctx)
 	    tcg_gen_qemu_ld32s(arg0, REG(B7_4), ctx->memidx);
 	    arg1 = tcg_temp_new();
 	    tcg_gen_qemu_ld32s(arg1, REG(B11_8), ctx->memidx);
-	    gen_helper_macl(arg0, arg1);
+            gen_helper_macl(cpu_env, arg0, arg1);
 	    tcg_temp_free(arg1);
 	    tcg_temp_free(arg0);
 	    tcg_gen_addi_i32(REG(B7_4), REG(B7_4), 4);
@@ -884,7 +897,7 @@ static void _decode_opc(DisasContext * ctx)
 	    tcg_gen_qemu_ld32s(arg0, REG(B7_4), ctx->memidx);
 	    arg1 = tcg_temp_new();
 	    tcg_gen_qemu_ld32s(arg1, REG(B11_8), ctx->memidx);
-	    gen_helper_macw(arg0, arg1);
+            gen_helper_macw(cpu_env, arg0, arg1);
 	    tcg_temp_free(arg1);
 	    tcg_temp_free(arg0);
 	    tcg_gen_addi_i32(REG(B11_8), REG(B11_8), 2);
@@ -1013,10 +1026,43 @@ static void _decode_opc(DisasContext * ctx)
 	tcg_gen_sub_i32(REG(B11_8), REG(B11_8), REG(B7_4));
 	return;
     case 0x300a:		/* subc Rm,Rn */
-	gen_helper_subc(REG(B11_8), REG(B7_4), REG(B11_8));
+        {
+            TCGv t0, t1, t2;
+            t0 = tcg_temp_new();
+            tcg_gen_andi_i32(t0, cpu_sr, SR_T);
+            t1 = tcg_temp_new();
+            tcg_gen_sub_i32(t1, REG(B11_8), REG(B7_4));
+            tcg_gen_sub_i32(t0, t1, t0);
+            t2 = tcg_temp_new();
+            tcg_gen_setcond_i32(TCG_COND_LTU, t2, REG(B11_8), t1);
+            tcg_gen_setcond_i32(TCG_COND_LTU, t1, t1, t0);
+            tcg_gen_or_i32(t1, t1, t2);
+            tcg_temp_free(t2);
+            tcg_gen_andi_i32(cpu_sr, cpu_sr, ~SR_T);
+            tcg_gen_or_i32(cpu_sr, cpu_sr, t1);
+            tcg_temp_free(t1);
+            tcg_gen_mov_i32(REG(B11_8), t0);
+            tcg_temp_free(t0);
+        }
 	return;
     case 0x300b:		/* subv Rm,Rn */
-	gen_helper_subv(REG(B11_8), REG(B7_4), REG(B11_8));
+        {
+            TCGv t0, t1, t2;
+            t0 = tcg_temp_new();
+            tcg_gen_sub_i32(t0, REG(B11_8), REG(B7_4));
+            t1 = tcg_temp_new();
+            tcg_gen_xor_i32(t1, t0, REG(B7_4));
+            t2 = tcg_temp_new();
+            tcg_gen_xor_i32(t2, REG(B11_8), REG(B7_4));
+            tcg_gen_and_i32(t1, t1, t2);
+            tcg_temp_free(t2);
+            tcg_gen_shri_i32(t1, t1, 31);
+            tcg_gen_andi_i32(cpu_sr, cpu_sr, ~SR_T);
+            tcg_gen_or_i32(cpu_sr, cpu_sr, t1);
+            tcg_temp_free(t1);
+            tcg_gen_mov_i32(REG(B11_8), t0);
+            tcg_temp_free(t0);
+        }
 	return;
     case 0x2008:		/* tst Rm,Rn */
 	{
@@ -1031,7 +1077,7 @@ static void _decode_opc(DisasContext * ctx)
 	return;
     case 0xf00c: /* fmov {F,D,X}Rm,{F,D,X}Rn - FPSCR: Nothing */
 	CHECK_FPU_ENABLED
-	if (ctx->fpscr & FPSCR_SZ) {
+        if (ctx->flags & FPSCR_SZ) {
 	    TCGv_i64 fp = tcg_temp_new_i64();
 	    gen_load_fpr64(fp, XREG(B7_4));
 	    gen_store_fpr64(fp, XREG(B11_8));
@@ -1042,7 +1088,7 @@ static void _decode_opc(DisasContext * ctx)
 	return;
     case 0xf00a: /* fmov {F,D,X}Rm,@Rn - FPSCR: Nothing */
 	CHECK_FPU_ENABLED
-	if (ctx->fpscr & FPSCR_SZ) {
+        if (ctx->flags & FPSCR_SZ) {
 	    TCGv addr_hi = tcg_temp_new();
 	    int fr = XREG(B7_4);
 	    tcg_gen_addi_i32(addr_hi, REG(B11_8), 4);
@@ -1055,7 +1101,7 @@ static void _decode_opc(DisasContext * ctx)
 	return;
     case 0xf008: /* fmov @Rm,{F,D,X}Rn - FPSCR: Nothing */
 	CHECK_FPU_ENABLED
-	if (ctx->fpscr & FPSCR_SZ) {
+        if (ctx->flags & FPSCR_SZ) {
 	    TCGv addr_hi = tcg_temp_new();
 	    int fr = XREG(B11_8);
 	    tcg_gen_addi_i32(addr_hi, REG(B7_4), 4);
@@ -1068,7 +1114,7 @@ static void _decode_opc(DisasContext * ctx)
 	return;
     case 0xf009: /* fmov @Rm+,{F,D,X}Rn - FPSCR: Nothing */
 	CHECK_FPU_ENABLED
-	if (ctx->fpscr & FPSCR_SZ) {
+        if (ctx->flags & FPSCR_SZ) {
 	    TCGv addr_hi = tcg_temp_new();
 	    int fr = XREG(B11_8);
 	    tcg_gen_addi_i32(addr_hi, REG(B7_4), 4);
@@ -1083,7 +1129,7 @@ static void _decode_opc(DisasContext * ctx)
 	return;
     case 0xf00b: /* fmov {F,D,X}Rm,@-Rn - FPSCR: Nothing */
 	CHECK_FPU_ENABLED
-	if (ctx->fpscr & FPSCR_SZ) {
+        if (ctx->flags & FPSCR_SZ) {
 	    TCGv addr = tcg_temp_new_i32();
 	    int fr = XREG(B7_4);
 	    tcg_gen_subi_i32(addr, REG(B11_8), 4);
@@ -1106,7 +1152,7 @@ static void _decode_opc(DisasContext * ctx)
 	{
 	    TCGv addr = tcg_temp_new_i32();
 	    tcg_gen_add_i32(addr, REG(B7_4), REG(0));
-	    if (ctx->fpscr & FPSCR_SZ) {
+            if (ctx->flags & FPSCR_SZ) {
 		int fr = XREG(B11_8);
 		tcg_gen_qemu_ld32u(cpu_fregs[fr	 ], addr, ctx->memidx);
 		tcg_gen_addi_i32(addr, addr, 4);
@@ -1122,7 +1168,7 @@ static void _decode_opc(DisasContext * ctx)
 	{
 	    TCGv addr = tcg_temp_new();
 	    tcg_gen_add_i32(addr, REG(B11_8), REG(0));
-	    if (ctx->fpscr & FPSCR_SZ) {
+            if (ctx->flags & FPSCR_SZ) {
 		int fr = XREG(B7_4);
 		tcg_gen_qemu_ld32u(cpu_fregs[fr	 ], addr, ctx->memidx);
 		tcg_gen_addi_i32(addr, addr, 4);
@@ -1141,7 +1187,7 @@ static void _decode_opc(DisasContext * ctx)
     case 0xf005: /* fcmp/gt Rm,Rn - FPSCR: R[PR,Enable.V]/W[Cause,Flag] */
 	{
 	    CHECK_FPU_ENABLED
-	    if (ctx->fpscr & FPSCR_PR) {
+            if (ctx->flags & FPSCR_PR) {
                 TCGv_i64 fp0, fp1;
 
 		if (ctx->opcode & 0x0110)
@@ -1152,22 +1198,22 @@ static void _decode_opc(DisasContext * ctx)
 		gen_load_fpr64(fp1, DREG(B7_4));
                 switch (ctx->opcode & 0xf00f) {
                 case 0xf000:		/* fadd Rm,Rn */
-                    gen_helper_fadd_DT(fp0, fp0, fp1);
+                    gen_helper_fadd_DT(fp0, cpu_env, fp0, fp1);
                     break;
                 case 0xf001:		/* fsub Rm,Rn */
-                    gen_helper_fsub_DT(fp0, fp0, fp1);
+                    gen_helper_fsub_DT(fp0, cpu_env, fp0, fp1);
                     break;
                 case 0xf002:		/* fmul Rm,Rn */
-                    gen_helper_fmul_DT(fp0, fp0, fp1);
+                    gen_helper_fmul_DT(fp0, cpu_env, fp0, fp1);
                     break;
                 case 0xf003:		/* fdiv Rm,Rn */
-                    gen_helper_fdiv_DT(fp0, fp0, fp1);
+                    gen_helper_fdiv_DT(fp0, cpu_env, fp0, fp1);
                     break;
                 case 0xf004:		/* fcmp/eq Rm,Rn */
-                    gen_helper_fcmp_eq_DT(fp0, fp1);
+                    gen_helper_fcmp_eq_DT(cpu_env, fp0, fp1);
                     return;
                 case 0xf005:		/* fcmp/gt Rm,Rn */
-                    gen_helper_fcmp_gt_DT(fp0, fp1);
+                    gen_helper_fcmp_gt_DT(cpu_env, fp0, fp1);
                     return;
                 }
 		gen_store_fpr64(fp0, DREG(B11_8));
@@ -1176,22 +1222,32 @@ static void _decode_opc(DisasContext * ctx)
 	    } else {
                 switch (ctx->opcode & 0xf00f) {
                 case 0xf000:		/* fadd Rm,Rn */
-                    gen_helper_fadd_FT(cpu_fregs[FREG(B11_8)], cpu_fregs[FREG(B11_8)], cpu_fregs[FREG(B7_4)]);
+                    gen_helper_fadd_FT(cpu_fregs[FREG(B11_8)], cpu_env,
+                                       cpu_fregs[FREG(B11_8)],
+                                       cpu_fregs[FREG(B7_4)]);
                     break;
                 case 0xf001:		/* fsub Rm,Rn */
-                    gen_helper_fsub_FT(cpu_fregs[FREG(B11_8)], cpu_fregs[FREG(B11_8)], cpu_fregs[FREG(B7_4)]);
+                    gen_helper_fsub_FT(cpu_fregs[FREG(B11_8)], cpu_env,
+                                       cpu_fregs[FREG(B11_8)],
+                                       cpu_fregs[FREG(B7_4)]);
                     break;
                 case 0xf002:		/* fmul Rm,Rn */
-                    gen_helper_fmul_FT(cpu_fregs[FREG(B11_8)], cpu_fregs[FREG(B11_8)], cpu_fregs[FREG(B7_4)]);
+                    gen_helper_fmul_FT(cpu_fregs[FREG(B11_8)], cpu_env,
+                                       cpu_fregs[FREG(B11_8)],
+                                       cpu_fregs[FREG(B7_4)]);
                     break;
                 case 0xf003:		/* fdiv Rm,Rn */
-                    gen_helper_fdiv_FT(cpu_fregs[FREG(B11_8)], cpu_fregs[FREG(B11_8)], cpu_fregs[FREG(B7_4)]);
+                    gen_helper_fdiv_FT(cpu_fregs[FREG(B11_8)], cpu_env,
+                                       cpu_fregs[FREG(B11_8)],
+                                       cpu_fregs[FREG(B7_4)]);
                     break;
                 case 0xf004:		/* fcmp/eq Rm,Rn */
-                    gen_helper_fcmp_eq_FT(cpu_fregs[FREG(B11_8)], cpu_fregs[FREG(B7_4)]);
+                    gen_helper_fcmp_eq_FT(cpu_env, cpu_fregs[FREG(B11_8)],
+                                          cpu_fregs[FREG(B7_4)]);
                     return;
                 case 0xf005:		/* fcmp/gt Rm,Rn */
-                    gen_helper_fcmp_gt_FT(cpu_fregs[FREG(B11_8)], cpu_fregs[FREG(B7_4)]);
+                    gen_helper_fcmp_gt_FT(cpu_env, cpu_fregs[FREG(B11_8)],
+                                          cpu_fregs[FREG(B7_4)]);
                     return;
                 }
 	    }
@@ -1200,11 +1256,12 @@ static void _decode_opc(DisasContext * ctx)
     case 0xf00e: /* fmac FR0,RM,Rn */
         {
             CHECK_FPU_ENABLED
-            if (ctx->fpscr & FPSCR_PR) {
+            if (ctx->flags & FPSCR_PR) {
                 break; /* illegal instruction */
             } else {
-                gen_helper_fmac_FT(cpu_fregs[FREG(B11_8)],
-                                   cpu_fregs[FREG(0)], cpu_fregs[FREG(B7_4)], cpu_fregs[FREG(B11_8)]);
+                gen_helper_fmac_FT(cpu_fregs[FREG(B11_8)], cpu_env,
+                                   cpu_fregs[FREG(0)], cpu_fregs[FREG(B7_4)],
+                                   cpu_fregs[FREG(B11_8)]);
                 return;
             }
         }
@@ -1355,8 +1412,9 @@ static void _decode_opc(DisasContext * ctx)
 	{
 	    TCGv imm;
 	    CHECK_NOT_DELAY_SLOT
+            tcg_gen_movi_i32(cpu_pc, ctx->pc);
 	    imm = tcg_const_i32(B7_0);
-	    gen_helper_trapa(imm);
+            gen_helper_trapa(cpu_env, imm);
 	    tcg_temp_free(imm);
 	    ctx->bstate = BS_BRANCH;
 	}
@@ -1531,7 +1589,7 @@ static void _decode_opc(DisasContext * ctx)
 	LDST(fpul, 0x405a, 0x4056, 0x005a, 0x4052, {CHECK_FPU_ENABLED})
     case 0x406a:		/* lds Rm,FPSCR */
 	CHECK_FPU_ENABLED
-	gen_helper_ld_fpscr(REG(B11_8));
+        gen_helper_ld_fpscr(cpu_env, REG(B11_8));
 	ctx->bstate = BS_STOP;
 	return;
     case 0x4066:		/* lds.l @Rm+,FPSCR */
@@ -1540,7 +1598,7 @@ static void _decode_opc(DisasContext * ctx)
 	    TCGv addr = tcg_temp_new();
 	    tcg_gen_qemu_ld32s(addr, REG(B11_8), ctx->memidx);
 	    tcg_gen_addi_i32(REG(B11_8), REG(B11_8), 4);
-	    gen_helper_ld_fpscr(addr);
+            gen_helper_ld_fpscr(cpu_env, addr);
 	    tcg_temp_free(addr);
 	    ctx->bstate = BS_STOP;
 	}
@@ -1567,7 +1625,7 @@ static void _decode_opc(DisasContext * ctx)
         {
             TCGv val = tcg_temp_new();
             tcg_gen_qemu_ld32u(val, REG(B11_8), ctx->memidx);
-            gen_helper_movcal (REG(B11_8), val);            
+            gen_helper_movcal(cpu_env, REG(B11_8), val);
             tcg_gen_qemu_st32(REG(0), REG(B11_8), ctx->memidx);
         }
         ctx->has_movcal = 1;
@@ -1594,7 +1652,7 @@ static void _decode_opc(DisasContext * ctx)
         */
         if (ctx->features & SH_FEATURE_SH4A) {
 	    int label = gen_new_label();
-	    gen_clr_t();
+            tcg_gen_andi_i32(cpu_sr, cpu_sr, ~SR_T);
 	    tcg_gen_or_i32(cpu_sr, cpu_sr, cpu_ldst);
 	    tcg_gen_brcondi_i32(TCG_COND_EQ, cpu_ldst, 0, label);
 	    tcg_gen_qemu_st32(REG(0), REG(B11_8), ctx->memidx);
@@ -1619,7 +1677,7 @@ static void _decode_opc(DisasContext * ctx)
 	    break;
     case 0x0093:		/* ocbi @Rn */
 	{
-	    gen_helper_ocbi (REG(B11_8));
+            gen_helper_ocbi(cpu_env, REG(B11_8));
 	}
 	return;
     case 0x00a3:		/* ocbp @Rn */
@@ -1728,32 +1786,32 @@ static void _decode_opc(DisasContext * ctx)
 	return;
     case 0xf02d: /* float FPUL,FRn/DRn - FPSCR: R[PR,Enable.I]/W[Cause,Flag] */
 	CHECK_FPU_ENABLED
-	if (ctx->fpscr & FPSCR_PR) {
+        if (ctx->flags & FPSCR_PR) {
 	    TCGv_i64 fp;
 	    if (ctx->opcode & 0x0100)
 		break; /* illegal instruction */
 	    fp = tcg_temp_new_i64();
-	    gen_helper_float_DT(fp, cpu_fpul);
+            gen_helper_float_DT(fp, cpu_env, cpu_fpul);
 	    gen_store_fpr64(fp, DREG(B11_8));
 	    tcg_temp_free_i64(fp);
 	}
 	else {
-	    gen_helper_float_FT(cpu_fregs[FREG(B11_8)], cpu_fpul);
+            gen_helper_float_FT(cpu_fregs[FREG(B11_8)], cpu_env, cpu_fpul);
 	}
 	return;
     case 0xf03d: /* ftrc FRm/DRm,FPUL - FPSCR: R[PR,Enable.V]/W[Cause,Flag] */
 	CHECK_FPU_ENABLED
-	if (ctx->fpscr & FPSCR_PR) {
+        if (ctx->flags & FPSCR_PR) {
 	    TCGv_i64 fp;
 	    if (ctx->opcode & 0x0100)
 		break; /* illegal instruction */
 	    fp = tcg_temp_new_i64();
 	    gen_load_fpr64(fp, DREG(B11_8));
-	    gen_helper_ftrc_DT(cpu_fpul, fp);
+            gen_helper_ftrc_DT(cpu_fpul, cpu_env, fp);
 	    tcg_temp_free_i64(fp);
 	}
 	else {
-	    gen_helper_ftrc_FT(cpu_fpul, cpu_fregs[FREG(B11_8)]);
+            gen_helper_ftrc_FT(cpu_fpul, cpu_env, cpu_fregs[FREG(B11_8)]);
 	}
 	return;
     case 0xf04d: /* fneg FRn/DRn - FPSCR: Nothing */
@@ -1764,7 +1822,7 @@ static void _decode_opc(DisasContext * ctx)
 	return;
     case 0xf05d: /* fabs FRn/DRn */
 	CHECK_FPU_ENABLED
-	if (ctx->fpscr & FPSCR_PR) {
+        if (ctx->flags & FPSCR_PR) {
 	    if (ctx->opcode & 0x0100)
 		break; /* illegal instruction */
 	    TCGv_i64 fp = tcg_temp_new_i64();
@@ -1778,16 +1836,17 @@ static void _decode_opc(DisasContext * ctx)
 	return;
     case 0xf06d: /* fsqrt FRn */
 	CHECK_FPU_ENABLED
-	if (ctx->fpscr & FPSCR_PR) {
+        if (ctx->flags & FPSCR_PR) {
 	    if (ctx->opcode & 0x0100)
 		break; /* illegal instruction */
 	    TCGv_i64 fp = tcg_temp_new_i64();
 	    gen_load_fpr64(fp, DREG(B11_8));
-	    gen_helper_fsqrt_DT(fp, fp);
+            gen_helper_fsqrt_DT(fp, cpu_env, fp);
 	    gen_store_fpr64(fp, DREG(B11_8));
 	    tcg_temp_free_i64(fp);
 	} else {
-	    gen_helper_fsqrt_FT(cpu_fregs[FREG(B11_8)], cpu_fregs[FREG(B11_8)]);
+            gen_helper_fsqrt_FT(cpu_fregs[FREG(B11_8)], cpu_env,
+                                cpu_fregs[FREG(B11_8)]);
 	}
 	return;
     case 0xf07d: /* fsrra FRn */
@@ -1795,13 +1854,13 @@ static void _decode_opc(DisasContext * ctx)
 	break;
     case 0xf08d: /* fldi0 FRn - FPSCR: R[PR] */
 	CHECK_FPU_ENABLED
-	if (!(ctx->fpscr & FPSCR_PR)) {
+        if (!(ctx->flags & FPSCR_PR)) {
 	    tcg_gen_movi_i32(cpu_fregs[FREG(B11_8)], 0);
 	}
 	return;
     case 0xf09d: /* fldi1 FRn - FPSCR: R[PR] */
 	CHECK_FPU_ENABLED
-	if (!(ctx->fpscr & FPSCR_PR)) {
+        if (!(ctx->flags & FPSCR_PR)) {
 	    tcg_gen_movi_i32(cpu_fregs[FREG(B11_8)], 0x3f800000);
 	}
 	return;
@@ -1809,7 +1868,7 @@ static void _decode_opc(DisasContext * ctx)
 	CHECK_FPU_ENABLED
 	{
 	    TCGv_i64 fp = tcg_temp_new_i64();
-	    gen_helper_fcnvsd_FT_DT(fp, cpu_fpul);
+            gen_helper_fcnvsd_FT_DT(fp, cpu_env, cpu_fpul);
 	    gen_store_fpr64(fp, DREG(B11_8));
 	    tcg_temp_free_i64(fp);
 	}
@@ -1819,17 +1878,17 @@ static void _decode_opc(DisasContext * ctx)
 	{
 	    TCGv_i64 fp = tcg_temp_new_i64();
 	    gen_load_fpr64(fp, DREG(B11_8));
-	    gen_helper_fcnvds_DT_FT(cpu_fpul, fp);
+            gen_helper_fcnvds_DT_FT(cpu_fpul, cpu_env, fp);
 	    tcg_temp_free_i64(fp);
 	}
 	return;
     case 0xf0ed: /* fipr FVm,FVn */
         CHECK_FPU_ENABLED
-        if ((ctx->fpscr & FPSCR_PR) == 0) {
+        if ((ctx->flags & FPSCR_PR) == 0) {
             TCGv m, n;
             m = tcg_const_i32((ctx->opcode >> 8) & 3);
             n = tcg_const_i32((ctx->opcode >> 10) & 3);
-            gen_helper_fipr(m, n);
+            gen_helper_fipr(cpu_env, m, n);
             tcg_temp_free(m);
             tcg_temp_free(n);
             return;
@@ -1838,10 +1897,10 @@ static void _decode_opc(DisasContext * ctx)
     case 0xf0fd: /* ftrv XMTRX,FVn */
         CHECK_FPU_ENABLED
         if ((ctx->opcode & 0x0300) == 0x0100 &&
-            (ctx->fpscr & FPSCR_PR) == 0) {
+            (ctx->flags & FPSCR_PR) == 0) {
             TCGv n;
             n = tcg_const_i32((ctx->opcode >> 10) & 3);
-            gen_helper_ftrv(n);
+            gen_helper_ftrv(cpu_env, n);
             tcg_temp_free(n);
             return;
         }
@@ -1852,19 +1911,20 @@ static void _decode_opc(DisasContext * ctx)
 	    ctx->opcode, ctx->pc);
     fflush(stderr);
 #endif
+    tcg_gen_movi_i32(cpu_pc, ctx->pc);
     if (ctx->flags & (DELAY_SLOT | DELAY_SLOT_CONDITIONAL)) {
-       gen_helper_raise_slot_illegal_instruction();
+        gen_helper_raise_slot_illegal_instruction(cpu_env);
     } else {
-       gen_helper_raise_illegal_instruction();
+        gen_helper_raise_illegal_instruction(cpu_env);
     }
-    ctx->bstate = BS_EXCP;
+    ctx->bstate = BS_BRANCH;
 }
 
 static void decode_opc(DisasContext * ctx)
 {
     uint32_t old_flags = ctx->flags;
 
-    if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP))) {
+    if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
         tcg_gen_debug_insn_start(ctx->pc);
     }
 
@@ -1907,20 +1967,18 @@ gen_intermediate_code_internal(CPUSH4State * env, TranslationBlock * tb,
     int max_insns;
 
     pc_start = tb->pc;
-    gen_opc_end = gen_opc_buf + OPC_MAX_SIZE;
+    gen_opc_end = tcg_ctx.gen_opc_buf + OPC_MAX_SIZE;
     ctx.pc = pc_start;
     ctx.flags = (uint32_t)tb->flags;
     ctx.bstate = BS_NONE;
-    ctx.sr = env->sr;
-    ctx.fpscr = env->fpscr;
-    ctx.memidx = (env->sr & SR_MD) == 0 ? 1 : 0;
+    ctx.memidx = (ctx.flags & SR_MD) == 0 ? 1 : 0;
     /* We don't know if the delayed pc came from a dynamic or static branch,
        so assume it is a dynamic branch.  */
     ctx.delayed_pc = -1; /* use delayed pc from env pointer */
     ctx.tb = tb;
     ctx.singlestep_enabled = env->singlestep_enabled;
     ctx.features = env->features;
-    ctx.has_movcal = (tb->flags & TB_FLAG_PENDING_MOVCA);
+    ctx.has_movcal = (ctx.flags & TB_FLAG_PENDING_MOVCA);
 
     ii = -1;
     num_insns = 0;
@@ -1928,20 +1986,20 @@ gen_intermediate_code_internal(CPUSH4State * env, TranslationBlock * tb,
     if (max_insns == 0)
         max_insns = CF_COUNT_MASK;
     gen_icount_start();
-    while (ctx.bstate == BS_NONE && gen_opc_ptr < gen_opc_end) {
+    while (ctx.bstate == BS_NONE && tcg_ctx.gen_opc_ptr < gen_opc_end) {
         if (unlikely(!QTAILQ_EMPTY(&env->breakpoints))) {
             QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
                 if (ctx.pc == bp->pc) {
 		    /* We have hit a breakpoint - make sure PC is up-to-date */
 		    tcg_gen_movi_i32(cpu_pc, ctx.pc);
-		    gen_helper_debug();
-		    ctx.bstate = BS_EXCP;
+                    gen_helper_debug(cpu_env);
+                    ctx.bstate = BS_BRANCH;
 		    break;
 		}
 	    }
 	}
         if (search_pc) {
-            i = gen_opc_ptr - gen_opc_buf;
+            i = tcg_ctx.gen_opc_ptr - tcg_ctx.gen_opc_buf;
             if (ii < i) {
                 ii++;
                 while (ii < i)
@@ -1958,7 +2016,7 @@ gen_intermediate_code_internal(CPUSH4State * env, TranslationBlock * tb,
 	fprintf(stderr, "Loading opcode at address 0x%08x\n", ctx.pc);
 	fflush(stderr);
 #endif
-	ctx.opcode = lduw_code(ctx.pc);
+        ctx.opcode = cpu_lduw_code(env, ctx.pc);
 	decode_opc(&ctx);
         num_insns++;
 	ctx.pc += 2;
@@ -1975,7 +2033,7 @@ gen_intermediate_code_internal(CPUSH4State * env, TranslationBlock * tb,
         gen_io_end();
     if (env->singlestep_enabled) {
         tcg_gen_movi_i32(cpu_pc, ctx.pc);
-        gen_helper_debug();
+        gen_helper_debug(cpu_env);
     } else {
 	switch (ctx.bstate) {
         case BS_STOP:
@@ -1998,9 +2056,9 @@ gen_intermediate_code_internal(CPUSH4State * env, TranslationBlock * tb,
     }
 
     gen_icount_end(tb, num_insns);
-    *gen_opc_ptr = INDEX_op_end;
+    *tcg_ctx.gen_opc_ptr = INDEX_op_end;
     if (search_pc) {
-        i = gen_opc_ptr - gen_opc_buf;
+        i = tcg_ctx.gen_opc_ptr - tcg_ctx.gen_opc_buf;
         ii++;
         while (ii <= i)
             gen_opc_instr_start[ii++] = 0;
@@ -2010,12 +2068,9 @@ gen_intermediate_code_internal(CPUSH4State * env, TranslationBlock * tb,
     }
 
 #ifdef DEBUG_DISAS
-#ifdef SH4_DEBUG_DISAS
-    qemu_log_mask(CPU_LOG_TB_IN_ASM, "\n");
-#endif
     if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
 	qemu_log("IN:\n");	/* , lookup_symbol(pc_start)); */
-	log_target_disas(pc_start, ctx.pc - pc_start, 0);
+        log_target_disas(env, pc_start, ctx.pc - pc_start, 0);
 	qemu_log("\n");
     }
 #endif
diff --git a/target-sparc/Makefile.objs b/target-sparc/Makefile.objs
index a93e07deb..9fc42ea9b 100644
--- a/target-sparc/Makefile.objs
+++ b/target-sparc/Makefile.objs
@@ -4,5 +4,3 @@ obj-y += fop_helper.o cc_helper.o win_helper.o mmu_helper.o ldst_helper.o
 obj-$(TARGET_SPARC) += int32_helper.o
 obj-$(TARGET_SPARC64) += int64_helper.o
 obj-$(TARGET_SPARC64) += vis_helper.o
-
-$(obj)/op_helper.o: QEMU_CFLAGS += $(HELPER_CFLAGS)
diff --git a/target-sparc/cpu.c b/target-sparc/cpu.c
index f7c004c7d..882d30642 100644
--- a/target-sparc/cpu.c
+++ b/target-sparc/cpu.c
@@ -643,7 +643,7 @@ static int cpu_sparc_find_by_name(sparc_def_t *cpu_def, const char *cpu_model)
 {
     unsigned int i;
     const sparc_def_t *def = NULL;
-    char *s = strdup(cpu_model);
+    char *s = g_strdup(cpu_model);
     char *featurestr, *name = strtok(s, ",");
     uint32_t plus_features = 0;
     uint32_t minus_features = 0;
@@ -735,7 +735,7 @@ static int cpu_sparc_find_by_name(sparc_def_t *cpu_def, const char *cpu_model)
 #ifdef DEBUG_FEATURES
     print_features(stderr, fprintf, cpu_def->features, NULL);
 #endif
-    free(s);
+    g_free(s);
     return 0;
 
  error:
@@ -792,7 +792,6 @@ void cpu_dump_state(CPUSPARCState *env, FILE *f, fprintf_function cpu_fprintf,
 
     cpu_fprintf(f, "pc: " TARGET_FMT_lx "  npc: " TARGET_FMT_lx "\n", env->pc,
                 env->npc);
-    cpu_fprintf(f, "General Registers:\n");
 
     for (i = 0; i < 8; i++) {
         if (i % REGS_PER_LINE == 0) {
@@ -803,7 +802,6 @@ void cpu_dump_state(CPUSPARCState *env, FILE *f, fprintf_function cpu_fprintf,
             cpu_fprintf(f, "\n");
         }
     }
-    cpu_fprintf(f, "\nCurrent Register Window:\n");
     for (x = 0; x < 3; x++) {
         for (i = 0; i < 8; i++) {
             if (i % REGS_PER_LINE == 0) {
@@ -817,10 +815,10 @@ void cpu_dump_state(CPUSPARCState *env, FILE *f, fprintf_function cpu_fprintf,
             }
         }
     }
-    cpu_fprintf(f, "\nFloating Point Registers:\n");
+
     for (i = 0; i < TARGET_DPREGS; i++) {
         if ((i & 3) == 0) {
-            cpu_fprintf(f, "%%f%02d:", i * 2);
+            cpu_fprintf(f, "%%f%02d: ", i * 2);
         }
         cpu_fprintf(f, " %016" PRIx64, env->fpr[i].ll);
         if ((i & 3) == 3) {
@@ -850,6 +848,7 @@ void cpu_dump_state(CPUSPARCState *env, FILE *f, fprintf_function cpu_fprintf,
     cpu_fprintf(f, "fsr: " TARGET_FMT_lx " y: " TARGET_FMT_lx "\n",
                 env->fsr, env->y);
 #endif
+    cpu_fprintf(f, "\n");
 }
 
 static void sparc_cpu_initfn(Object *obj)
diff --git a/target-sparc/cpu.h b/target-sparc/cpu.h
index e16b7b351..042d52a31 100644
--- a/target-sparc/cpu.h
+++ b/target-sparc/cpu.h
@@ -583,10 +583,10 @@ static inline int tlb_compare_context(const SparcTLBEntry *tlb,
 
 /* cpu-exec.c */
 #if !defined(CONFIG_USER_ONLY)
-void cpu_unassigned_access(CPUSPARCState *env1, target_phys_addr_t addr,
+void cpu_unassigned_access(CPUSPARCState *env1, hwaddr addr,
                            int is_write, int is_exec, int is_asi, int size);
 #if defined(TARGET_SPARC64)
-target_phys_addr_t cpu_get_phys_page_nofault(CPUSPARCState *env, target_ulong addr,
+hwaddr cpu_get_phys_page_nofault(CPUSPARCState *env, target_ulong addr,
                                            int mmu_idx);
 #endif
 #endif
@@ -711,9 +711,7 @@ uint64_t cpu_tick_get_count(CPUTimer *timer);
 void cpu_tick_set_limit(CPUTimer *timer, uint64_t limit);
 trap_state* cpu_tsptr(CPUSPARCState* env);
 #endif
-void QEMU_NORETURN do_unaligned_access(CPUSPARCState *env, target_ulong addr,
-                                       int is_write, int is_user,
-                                       uintptr_t retaddr);
+void cpu_restore_state2(CPUSPARCState *env, uintptr_t retaddr);
 
 #define TB_FLAG_FPU_ENABLED (1 << 4)
 #define TB_FLAG_AM_ENABLED (1 << 5)
@@ -763,8 +761,10 @@ static inline bool tb_am_enabled(int tb_flags)
 #endif
 }
 
-static inline bool cpu_has_work(CPUSPARCState *env1)
+static inline bool cpu_has_work(CPUState *cpu)
 {
+    CPUSPARCState *env1 = &SPARC_CPU(cpu)->env;
+
     return (env1->interrupt_request & CPU_INTERRUPT_HARD) &&
            cpu_interrupts_enabled(env1);
 }
diff --git a/target-sparc/fop_helper.c b/target-sparc/fop_helper.c
index 9c64ef89f..f4b62a5ba 100644
--- a/target-sparc/fop_helper.c
+++ b/target-sparc/fop_helper.c
@@ -334,34 +334,28 @@ void helper_fsqrtq(CPUSPARCState *env)
 }
 
 #define GEN_FCMP(name, size, reg1, reg2, FS, E)                         \
-    void glue(helper_, name) (CPUSPARCState *env)                            \
+    void glue(helper_, name) (CPUSPARCState *env)                       \
     {                                                                   \
-        env->fsr &= FSR_FTT_NMASK;                                      \
-        if (E && (glue(size, _is_any_nan)(reg1) ||                      \
-                  glue(size, _is_any_nan)(reg2)) &&                     \
-            (env->fsr & FSR_NVM)) {                                     \
-            env->fsr |= FSR_NVC;                                        \
-            env->fsr |= FSR_FTT_IEEE_EXCP;                              \
-            helper_raise_exception(env, TT_FP_EXCP);                    \
+        int ret;                                                        \
+        clear_float_exceptions(env);                                    \
+        if (E) {                                                        \
+            ret = glue(size, _compare)(reg1, reg2, &env->fp_status);    \
+        } else {                                                        \
+            ret = glue(size, _compare_quiet)(reg1, reg2,                \
+                                             &env->fp_status);          \
         }                                                               \
-        switch (glue(size, _compare) (reg1, reg2, &env->fp_status)) {   \
+        check_ieee_exceptions(env);                                     \
+        switch (ret) {                                                  \
         case float_relation_unordered:                                  \
-            if ((env->fsr & FSR_NVM)) {                                 \
-                env->fsr |= FSR_NVC;                                    \
-                env->fsr |= FSR_FTT_IEEE_EXCP;                          \
-                helper_raise_exception(env, TT_FP_EXCP);                \
-            } else {                                                    \
-                env->fsr &= ~((FSR_FCC1 | FSR_FCC0) << FS);             \
-                env->fsr |= (FSR_FCC1 | FSR_FCC0) << FS;                \
-                env->fsr |= FSR_NVA;                                    \
-            }                                                           \
+            env->fsr |= (FSR_FCC1 | FSR_FCC0) << FS;                    \
+            env->fsr |= FSR_NVA;                                        \
             break;                                                      \
         case float_relation_less:                                       \
-            env->fsr &= ~((FSR_FCC1 | FSR_FCC0) << FS);                 \
+            env->fsr &= ~(FSR_FCC1) << FS;                              \
             env->fsr |= FSR_FCC0 << FS;                                 \
             break;                                                      \
         case float_relation_greater:                                    \
-            env->fsr &= ~((FSR_FCC1 | FSR_FCC0) << FS);                 \
+            env->fsr &= ~(FSR_FCC0) << FS;                              \
             env->fsr |= FSR_FCC1 << FS;                                 \
             break;                                                      \
         default:                                                        \
@@ -370,34 +364,27 @@ void helper_fsqrtq(CPUSPARCState *env)
         }                                                               \
     }
 #define GEN_FCMP_T(name, size, FS, E)                                   \
-    void glue(helper_, name)(CPUSPARCState *env, size src1, size src2)       \
+    void glue(helper_, name)(CPUSPARCState *env, size src1, size src2)  \
     {                                                                   \
-        env->fsr &= FSR_FTT_NMASK;                                      \
-        if (E && (glue(size, _is_any_nan)(src1) ||                      \
-                  glue(size, _is_any_nan)(src2)) &&                     \
-            (env->fsr & FSR_NVM)) {                                     \
-            env->fsr |= FSR_NVC;                                        \
-            env->fsr |= FSR_FTT_IEEE_EXCP;                              \
-            helper_raise_exception(env, TT_FP_EXCP);                    \
+        int ret;                                                        \
+        clear_float_exceptions(env);                                    \
+        if (E) {                                                        \
+            ret = glue(size, _compare)(src1, src2, &env->fp_status);    \
+        } else {                                                        \
+            ret = glue(size, _compare_quiet)(src1, src2,                \
+                                             &env->fp_status);          \
         }                                                               \
-        switch (glue(size, _compare) (src1, src2, &env->fp_status)) {   \
+        check_ieee_exceptions(env);                                     \
+        switch (ret) {                                                  \
         case float_relation_unordered:                                  \
-            if ((env->fsr & FSR_NVM)) {                                 \
-                env->fsr |= FSR_NVC;                                    \
-                env->fsr |= FSR_FTT_IEEE_EXCP;                          \
-                helper_raise_exception(env, TT_FP_EXCP);                \
-            } else {                                                    \
-                env->fsr &= ~((FSR_FCC1 | FSR_FCC0) << FS);             \
-                env->fsr |= (FSR_FCC1 | FSR_FCC0) << FS;                \
-                env->fsr |= FSR_NVA;                                    \
-            }                                                           \
+            env->fsr |= (FSR_FCC1 | FSR_FCC0) << FS;                    \
             break;                                                      \
         case float_relation_less:                                       \
-            env->fsr &= ~((FSR_FCC1 | FSR_FCC0) << FS);                 \
+            env->fsr &= ~(FSR_FCC1 << FS);                              \
             env->fsr |= FSR_FCC0 << FS;                                 \
             break;                                                      \
         case float_relation_greater:                                    \
-            env->fsr &= ~((FSR_FCC1 | FSR_FCC0) << FS);                 \
+            env->fsr &= ~(FSR_FCC0 << FS);                              \
             env->fsr |= FSR_FCC1 << FS;                                 \
             break;                                                      \
         default:                                                        \
diff --git a/target-sparc/helper.c b/target-sparc/helper.c
index 65e1740e1..556ac286e 100644
--- a/target-sparc/helper.c
+++ b/target-sparc/helper.c
@@ -75,6 +75,7 @@ static target_ulong helper_udiv_common(CPUSPARCState *env, target_ulong a,
     x1 = (b & 0xffffffff);
 
     if (x1 == 0) {
+        cpu_restore_state2(env, GETPC());
         helper_raise_exception(env, TT_DIV_ZERO);
     }
 
@@ -113,6 +114,7 @@ static target_ulong helper_sdiv_common(CPUSPARCState *env, target_ulong a,
     x1 = (b & 0xffffffff);
 
     if (x1 == 0) {
+        cpu_restore_state2(env, GETPC());
         helper_raise_exception(env, TT_DIV_ZERO);
     }
 
@@ -139,3 +141,87 @@ target_ulong helper_sdiv_cc(CPUSPARCState *env, target_ulong a, target_ulong b)
 {
     return helper_sdiv_common(env, a, b, 1);
 }
+
+#ifdef TARGET_SPARC64
+int64_t helper_sdivx(CPUSPARCState *env, int64_t a, int64_t b)
+{
+    if (b == 0) {
+        /* Raise divide by zero trap.  */
+        cpu_restore_state2(env, GETPC());
+        helper_raise_exception(env, TT_DIV_ZERO);
+    } else if (b == -1) {
+        /* Avoid overflow trap with i386 divide insn.  */
+        return -a;
+    } else {
+        return a / b;
+    }
+}
+
+uint64_t helper_udivx(CPUSPARCState *env, uint64_t a, uint64_t b)
+{
+    if (b == 0) {
+        /* Raise divide by zero trap.  */
+        cpu_restore_state2(env, GETPC());
+        helper_raise_exception(env, TT_DIV_ZERO);
+    }
+    return a / b;
+}
+#endif
+
+target_ulong helper_taddcctv(CPUSPARCState *env, target_ulong src1,
+                             target_ulong src2)
+{
+    target_ulong dst;
+
+    /* Tag overflow occurs if either input has bits 0 or 1 set.  */
+    if ((src1 | src2) & 3) {
+        goto tag_overflow;
+    }
+
+    dst = src1 + src2;
+
+    /* Tag overflow occurs if the addition overflows.  */
+    if (~(src1 ^ src2) & (src1 ^ dst) & (1u << 31)) {
+        goto tag_overflow;
+    }
+
+    /* Only modify the CC after any exceptions have been generated.  */
+    env->cc_op = CC_OP_TADDTV;
+    env->cc_src = src1;
+    env->cc_src2 = src2;
+    env->cc_dst = dst;
+    return dst;
+
+ tag_overflow:
+    cpu_restore_state2(env, GETPC());
+    helper_raise_exception(env, TT_TOVF);
+}
+
+target_ulong helper_tsubcctv(CPUSPARCState *env, target_ulong src1,
+                             target_ulong src2)
+{
+    target_ulong dst;
+
+    /* Tag overflow occurs if either input has bits 0 or 1 set.  */
+    if ((src1 | src2) & 3) {
+        goto tag_overflow;
+    }
+
+    dst = src1 - src2;
+
+    /* Tag overflow occurs if the subtraction overflows.  */
+    if ((src1 ^ src2) & (src1 ^ dst) & (1u << 31)) {
+        goto tag_overflow;
+    }
+
+    /* Only modify the CC after any exceptions have been generated.  */
+    env->cc_op = CC_OP_TSUBTV;
+    env->cc_src = src1;
+    env->cc_src2 = src2;
+    env->cc_dst = dst;
+    return dst;
+
+ tag_overflow:
+    cpu_restore_state2(env, GETPC());
+    helper_raise_exception(env, TT_TOVF);
+}
diff --git a/target-sparc/helper.h b/target-sparc/helper.h
index e3c7fddbe..098c48221 100644
--- a/target-sparc/helper.h
+++ b/target-sparc/helper.h
@@ -16,7 +16,7 @@ DEF_HELPER_1(rdccr, tl, env)
 DEF_HELPER_2(wrccr, void, env, tl)
 DEF_HELPER_1(rdcwp, tl, env)
 DEF_HELPER_2(wrcwp, void, env, tl)
-DEF_HELPER_FLAGS_2(array8, TCG_CALL_CONST | TCG_CALL_PURE, tl, tl, tl)
+DEF_HELPER_FLAGS_2(array8, TCG_CALL_NO_RWG_SE, tl, tl, tl)
 DEF_HELPER_1(popc, tl, tl)
 DEF_HELPER_4(ldda_asi, void, env, tl, int, int)
 DEF_HELPER_5(ldf_asi, void, env, tl, int, int, int)
@@ -38,6 +38,12 @@ DEF_HELPER_3(udiv, tl, env, tl, tl)
 DEF_HELPER_3(udiv_cc, tl, env, tl, tl)
 DEF_HELPER_3(sdiv, tl, env, tl, tl)
 DEF_HELPER_3(sdiv_cc, tl, env, tl, tl)
+DEF_HELPER_3(taddcctv, tl, env, tl, tl)
+DEF_HELPER_3(tsubcctv, tl, env, tl, tl)
+#ifdef TARGET_SPARC64
+DEF_HELPER_3(sdivx, s64, env, s64, s64)
+DEF_HELPER_3(udivx, i64, env, i64, i64)
+#endif
 DEF_HELPER_3(ldqf, void, env, tl, int)
 DEF_HELPER_3(stqf, void, env, tl, int)
 #if !defined(CONFIG_USER_ONLY) || defined(TARGET_SPARC64)
@@ -45,7 +51,7 @@ DEF_HELPER_5(ld_asi, i64, env, tl, int, int, int)
 DEF_HELPER_5(st_asi, void, env, tl, i64, int, int)
 #endif
 DEF_HELPER_2(ldfsr, void, env, i32)
-DEF_HELPER_FLAGS_1(fabss, TCG_CALL_CONST | TCG_CALL_PURE, f32, f32)
+DEF_HELPER_FLAGS_1(fabss, TCG_CALL_NO_RWG_SE, f32, f32)
 DEF_HELPER_2(fsqrts, f32, env, f32)
 DEF_HELPER_2(fsqrtd, f64, env, f64)
 DEF_HELPER_3(fcmps, void, env, f32, f32)
@@ -57,7 +63,7 @@ DEF_HELPER_1(fcmpq, void, env)
 DEF_HELPER_1(fcmpeq, void, env)
 #ifdef TARGET_SPARC64
 DEF_HELPER_2(ldxfsr, void, env, i64)
-DEF_HELPER_FLAGS_1(fabsd, TCG_CALL_CONST | TCG_CALL_PURE, f64, f64)
+DEF_HELPER_FLAGS_1(fabsd, TCG_CALL_NO_RWG_SE, f64, f64)
 DEF_HELPER_3(fcmps_fcc1, void, env, f32, f32)
 DEF_HELPER_3(fcmps_fcc2, void, env, f32, f32)
 DEF_HELPER_3(fcmps_fcc3, void, env, f32, f32)
@@ -98,14 +104,14 @@ DEF_HELPER_3(fdivs, f32, env, f32, f32)
 DEF_HELPER_3(fsmuld, f64, env, f32, f32)
 DEF_HELPER_3(fdmulq, void, env, f64, f64);
 
-DEF_HELPER_FLAGS_1(fnegs, TCG_CALL_CONST | TCG_CALL_PURE, f32, f32)
+DEF_HELPER_FLAGS_1(fnegs, TCG_CALL_NO_RWG_SE, f32, f32)
 DEF_HELPER_2(fitod, f64, env, s32)
 DEF_HELPER_2(fitoq, void, env, s32)
 
 DEF_HELPER_2(fitos, f32, env, s32)
 
 #ifdef TARGET_SPARC64
-DEF_HELPER_FLAGS_1(fnegd, TCG_CALL_CONST | TCG_CALL_PURE, f64, f64)
+DEF_HELPER_FLAGS_1(fnegd, TCG_CALL_NO_RWG_SE, f64, f64)
 DEF_HELPER_1(fnegq, void, env)
 DEF_HELPER_2(fxtos, f32, env, s64)
 DEF_HELPER_2(fxtod, f64, env, s64)
@@ -125,36 +131,36 @@ DEF_HELPER_2(fstox, s64, env, f32)
 DEF_HELPER_2(fdtox, s64, env, f64)
 DEF_HELPER_1(fqtox, s64, env)
 
-DEF_HELPER_FLAGS_2(fpmerge, TCG_CALL_CONST | TCG_CALL_PURE, i64, i64, i64)
-DEF_HELPER_FLAGS_2(fmul8x16, TCG_CALL_CONST | TCG_CALL_PURE, i64, i64, i64)
-DEF_HELPER_FLAGS_2(fmul8x16al, TCG_CALL_CONST | TCG_CALL_PURE, i64, i64, i64)
-DEF_HELPER_FLAGS_2(fmul8x16au, TCG_CALL_CONST | TCG_CALL_PURE, i64, i64, i64)
-DEF_HELPER_FLAGS_2(fmul8sux16, TCG_CALL_CONST | TCG_CALL_PURE, i64, i64, i64)
-DEF_HELPER_FLAGS_2(fmul8ulx16, TCG_CALL_CONST | TCG_CALL_PURE, i64, i64, i64)
-DEF_HELPER_FLAGS_2(fmuld8sux16, TCG_CALL_CONST | TCG_CALL_PURE, i64, i64, i64)
-DEF_HELPER_FLAGS_2(fmuld8ulx16, TCG_CALL_CONST | TCG_CALL_PURE, i64, i64, i64)
-DEF_HELPER_FLAGS_2(fexpand, TCG_CALL_CONST | TCG_CALL_PURE, i64, i64, i64)
-DEF_HELPER_FLAGS_3(pdist, TCG_CALL_CONST | TCG_CALL_PURE, i64, i64, i64, i64)
-DEF_HELPER_FLAGS_2(fpack16, TCG_CALL_CONST | TCG_CALL_PURE, i32, i64, i64)
-DEF_HELPER_FLAGS_3(fpack32, TCG_CALL_CONST | TCG_CALL_PURE, i64, i64, i64, i64)
-DEF_HELPER_FLAGS_2(fpackfix, TCG_CALL_CONST | TCG_CALL_PURE, i32, i64, i64)
-DEF_HELPER_FLAGS_3(bshuffle, TCG_CALL_CONST | TCG_CALL_PURE, i64, i64, i64, i64)
+DEF_HELPER_FLAGS_2(fpmerge, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(fmul8x16, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(fmul8x16al, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(fmul8x16au, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(fmul8sux16, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(fmul8ulx16, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(fmuld8sux16, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(fmuld8ulx16, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_2(fexpand, TCG_CALL_NO_RWG_SE, i64, i64, i64)
+DEF_HELPER_FLAGS_3(pdist, TCG_CALL_NO_RWG_SE, i64, i64, i64, i64)
+DEF_HELPER_FLAGS_2(fpack16, TCG_CALL_NO_RWG_SE, i32, i64, i64)
+DEF_HELPER_FLAGS_3(fpack32, TCG_CALL_NO_RWG_SE, i64, i64, i64, i64)
+DEF_HELPER_FLAGS_2(fpackfix, TCG_CALL_NO_RWG_SE, i32, i64, i64)
+DEF_HELPER_FLAGS_3(bshuffle, TCG_CALL_NO_RWG_SE, i64, i64, i64, i64)
 #define VIS_HELPER(name)                                                 \
-    DEF_HELPER_FLAGS_2(f ## name ## 16, TCG_CALL_CONST | TCG_CALL_PURE,  \
+    DEF_HELPER_FLAGS_2(f ## name ## 16, TCG_CALL_NO_RWG_SE,  \
                        i64, i64, i64)                                    \
-    DEF_HELPER_FLAGS_2(f ## name ## 16s, TCG_CALL_CONST | TCG_CALL_PURE, \
+    DEF_HELPER_FLAGS_2(f ## name ## 16s, TCG_CALL_NO_RWG_SE, \
                        i32, i32, i32)                                    \
-    DEF_HELPER_FLAGS_2(f ## name ## 32, TCG_CALL_CONST | TCG_CALL_PURE,  \
+    DEF_HELPER_FLAGS_2(f ## name ## 32, TCG_CALL_NO_RWG_SE,  \
                        i64, i64, i64)                                    \
-    DEF_HELPER_FLAGS_2(f ## name ## 32s, TCG_CALL_CONST | TCG_CALL_PURE, \
+    DEF_HELPER_FLAGS_2(f ## name ## 32s, TCG_CALL_NO_RWG_SE, \
                        i32, i32, i32)
 
 VIS_HELPER(padd);
 VIS_HELPER(psub);
 #define VIS_CMPHELPER(name)                                              \
-    DEF_HELPER_FLAGS_2(f##name##16, TCG_CALL_CONST | TCG_CALL_PURE,      \
+    DEF_HELPER_FLAGS_2(f##name##16, TCG_CALL_NO_RWG_SE,      \
                        i64, i64, i64)                                    \
-    DEF_HELPER_FLAGS_2(f##name##32, TCG_CALL_CONST | TCG_CALL_PURE,      \
+    DEF_HELPER_FLAGS_2(f##name##32, TCG_CALL_NO_RWG_SE,      \
                        i64, i64, i64)
 VIS_CMPHELPER(cmpgt);
 VIS_CMPHELPER(cmpeq);
diff --git a/target-sparc/int32_helper.c b/target-sparc/int32_helper.c
index 5e33d50e2..507c355ca 100644
--- a/target-sparc/int32_helper.c
+++ b/target-sparc/int32_helper.c
@@ -21,7 +21,7 @@
 #include "trace.h"
 #include "sysemu.h"
 
-//#define DEBUG_PCALL
+#define DEBUG_PCALL
 
 #ifdef DEBUG_PCALL
 static const char * const excp_names[0x80] = {
@@ -62,6 +62,11 @@ void do_interrupt(CPUSPARCState *env)
 {
     int cwp, intno = env->exception_index;
 
+    /* Compute PSR before exposing state.  */
+    if (env->cc_op != CC_OP_FLAGS) {
+        cpu_get_psr(env);
+    }
+
 #ifdef DEBUG_PCALL
     if (qemu_loglevel_mask(CPU_LOG_INT)) {
         static int count;
@@ -78,10 +83,7 @@ void do_interrupt(CPUSPARCState *env)
             }
         }
 
-        qemu_log("%6d: %s (v=%02x) pc=%08x npc=%08x SP=%08x\n",
-                count, name, intno,
-                env->pc,
-                env->npc, env->regwptr[6]);
+        qemu_log("%6d: %s (v=%02x)\n", count, name, intno);
         log_cpu_state(env, 0);
 #if 0
         {
diff --git a/target-sparc/int64_helper.c b/target-sparc/int64_helper.c
index 5e3eff76d..df37aa1d1 100644
--- a/target-sparc/int64_helper.c
+++ b/target-sparc/int64_helper.c
@@ -21,7 +21,7 @@
 #include "helper.h"
 #include "trace.h"
 
-//#define DEBUG_PCALL
+#define DEBUG_PCALL
 
 #ifdef DEBUG_PCALL
 static const char * const excp_names[0x80] = {
@@ -64,6 +64,11 @@ void do_interrupt(CPUSPARCState *env)
     int intno = env->exception_index;
     trap_state *tsptr;
 
+    /* Compute PSR before exposing state.  */
+    if (env->cc_op != CC_OP_FLAGS) {
+        cpu_get_psr(env);
+    }
+
 #ifdef DEBUG_PCALL
     if (qemu_loglevel_mask(CPU_LOG_INT)) {
         static int count;
@@ -84,11 +89,7 @@ void do_interrupt(CPUSPARCState *env)
             }
         }
 
-        qemu_log("%6d: %s (v=%04x) pc=%016" PRIx64 " npc=%016" PRIx64
-                " SP=%016" PRIx64 "\n",
-                count, name, intno,
-                env->pc,
-                env->npc, env->regwptr[6]);
+        qemu_log("%6d: %s (v=%04x)\n", count, name, intno);
         log_cpu_state(env, 0);
 #if 0
         {
diff --git a/target-sparc/ldst_helper.c b/target-sparc/ldst_helper.c
index 9bec7a92f..f3e08fd6e 100644
--- a/target-sparc/ldst_helper.c
+++ b/target-sparc/ldst_helper.c
@@ -65,6 +65,9 @@
 #define QT1 (env->qt1)
 
 #if !defined(CONFIG_USER_ONLY)
+static void QEMU_NORETURN do_unaligned_access(CPUSPARCState *env,
+                                              target_ulong addr, int is_write,
+                                              int is_user, uintptr_t retaddr);
 #include "softmmu_exec.h"
 #define MMUSUFFIX _mmu
 #define ALIGNED_ONLY
@@ -619,21 +622,21 @@ uint64_t helper_ld_asi(CPUSPARCState *env, target_ulong addr, int asi, int size,
     case 0x21 ... 0x2f: /* MMU passthrough, 0x100000000 to 0xfffffffff */
         switch (size) {
         case 1:
-            ret = ldub_phys((target_phys_addr_t)addr
-                            | ((target_phys_addr_t)(asi & 0xf) << 32));
+            ret = ldub_phys((hwaddr)addr
+                            | ((hwaddr)(asi & 0xf) << 32));
             break;
         case 2:
-            ret = lduw_phys((target_phys_addr_t)addr
-                            | ((target_phys_addr_t)(asi & 0xf) << 32));
+            ret = lduw_phys((hwaddr)addr
+                            | ((hwaddr)(asi & 0xf) << 32));
             break;
         default:
         case 4:
-            ret = ldl_phys((target_phys_addr_t)addr
-                           | ((target_phys_addr_t)(asi & 0xf) << 32));
+            ret = ldl_phys((hwaddr)addr
+                           | ((hwaddr)(asi & 0xf) << 32));
             break;
         case 8:
-            ret = ldq_phys((target_phys_addr_t)addr
-                           | ((target_phys_addr_t)(asi & 0xf) << 32));
+            ret = ldq_phys((hwaddr)addr
+                           | ((hwaddr)(asi & 0xf) << 32));
             break;
         }
         break;
@@ -1015,21 +1018,21 @@ void helper_st_asi(CPUSPARCState *env, target_ulong addr, uint64_t val, int asi,
         {
             switch (size) {
             case 1:
-                stb_phys((target_phys_addr_t)addr
-                         | ((target_phys_addr_t)(asi & 0xf) << 32), val);
+                stb_phys((hwaddr)addr
+                         | ((hwaddr)(asi & 0xf) << 32), val);
                 break;
             case 2:
-                stw_phys((target_phys_addr_t)addr
-                         | ((target_phys_addr_t)(asi & 0xf) << 32), val);
+                stw_phys((hwaddr)addr
+                         | ((hwaddr)(asi & 0xf) << 32), val);
                 break;
             case 4:
             default:
-                stl_phys((target_phys_addr_t)addr
-                         | ((target_phys_addr_t)(asi & 0xf) << 32), val);
+                stl_phys((hwaddr)addr
+                         | ((hwaddr)(asi & 0xf) << 32), val);
                 break;
             case 8:
-                stq_phys((target_phys_addr_t)addr
-                         | ((target_phys_addr_t)(asi & 0xf) << 32), val);
+                stq_phys((hwaddr)addr
+                         | ((hwaddr)(asi & 0xf) << 32), val);
                 break;
             }
         }
@@ -2313,7 +2316,7 @@ void helper_stqf(CPUSPARCState *env, target_ulong addr, int mem_idx)
 
 #if !defined(CONFIG_USER_ONLY)
 #ifndef TARGET_SPARC64
-void cpu_unassigned_access(CPUSPARCState *env, target_phys_addr_t addr,
+void cpu_unassigned_access(CPUSPARCState *env, hwaddr addr,
                            int is_write, int is_exec, int is_asi, int size)
 {
     int fault_type;
@@ -2373,7 +2376,7 @@ void cpu_unassigned_access(CPUSPARCState *env, target_phys_addr_t addr,
     }
 }
 #else
-void cpu_unassigned_access(CPUSPARCState *env, target_phys_addr_t addr,
+void cpu_unassigned_access(CPUSPARCState *env, hwaddr addr,
                            int is_write, int is_exec, int is_asi, int size)
 {
 #ifdef DEBUG_UNASSIGNED
@@ -2390,9 +2393,8 @@ void cpu_unassigned_access(CPUSPARCState *env, target_phys_addr_t addr,
 #endif
 #endif
 
-#if !defined(CONFIG_USER_ONLY)
 /* XXX: make it generic ? */
-static void cpu_restore_state2(CPUSPARCState *env, uintptr_t retaddr)
+void cpu_restore_state2(CPUSPARCState *env, uintptr_t retaddr)
 {
     TranslationBlock *tb;
 
@@ -2407,8 +2409,10 @@ static void cpu_restore_state2(CPUSPARCState *env, uintptr_t retaddr)
     }
 }
 
-void do_unaligned_access(CPUSPARCState *env, target_ulong addr, int is_write,
-                         int is_user, uintptr_t retaddr)
+#if !defined(CONFIG_USER_ONLY)
+static void QEMU_NORETURN do_unaligned_access(CPUSPARCState *env,
+                                              target_ulong addr, int is_write,
+                                              int is_user, uintptr_t retaddr)
 {
 #ifdef DEBUG_UNALIGNED
     printf("Unaligned access to 0x" TARGET_FMT_lx " from 0x" TARGET_FMT_lx
diff --git a/target-sparc/mmu_helper.c b/target-sparc/mmu_helper.c
index cb73c449f..2c89b20b6 100644
--- a/target-sparc/mmu_helper.c
+++ b/target-sparc/mmu_helper.c
@@ -76,13 +76,13 @@ static const int perm_table[2][8] = {
     }
 };
 
-static int get_physical_address(CPUSPARCState *env, target_phys_addr_t *physical,
+static int get_physical_address(CPUSPARCState *env, hwaddr *physical,
                                 int *prot, int *access_index,
                                 target_ulong address, int rw, int mmu_idx,
                                 target_ulong *page_size)
 {
     int access_perms = 0;
-    target_phys_addr_t pde_ptr;
+    hwaddr pde_ptr;
     uint32_t pde;
     int error_code = 0, is_dirty, is_user;
     unsigned long page_offset;
@@ -192,7 +192,7 @@ static int get_physical_address(CPUSPARCState *env, target_phys_addr_t *physical
 
     /* Even if large ptes, we map only one 4KB page in the cache to
        avoid filling it too fast */
-    *physical = ((target_phys_addr_t)(pde & PTE_ADDR_MASK) << 4) + page_offset;
+    *physical = ((hwaddr)(pde & PTE_ADDR_MASK) << 4) + page_offset;
     return error_code;
 }
 
@@ -200,7 +200,7 @@ static int get_physical_address(CPUSPARCState *env, target_phys_addr_t *physical
 int cpu_sparc_handle_mmu_fault(CPUSPARCState *env, target_ulong address, int rw,
                                int mmu_idx)
 {
-    target_phys_addr_t paddr;
+    hwaddr paddr;
     target_ulong vaddr;
     target_ulong page_size;
     int error_code = 0, prot, access_index;
@@ -244,11 +244,11 @@ int cpu_sparc_handle_mmu_fault(CPUSPARCState *env, target_ulong address, int rw,
 
 target_ulong mmu_probe(CPUSPARCState *env, target_ulong address, int mmulev)
 {
-    target_phys_addr_t pde_ptr;
+    hwaddr pde_ptr;
     uint32_t pde;
 
     /* Context base + context number */
-    pde_ptr = (target_phys_addr_t)(env->mmuregs[1] << 4) +
+    pde_ptr = (hwaddr)(env->mmuregs[1] << 4) +
         (env->mmuregs[2] << 2);
     pde = ldl_phys(pde_ptr);
 
@@ -312,13 +312,13 @@ void dump_mmu(FILE *f, fprintf_function cpu_fprintf, CPUSPARCState *env)
 {
     target_ulong va, va1, va2;
     unsigned int n, m, o;
-    target_phys_addr_t pde_ptr, pa;
+    hwaddr pde_ptr, pa;
     uint32_t pde;
 
     pde_ptr = (env->mmuregs[1] << 4) + (env->mmuregs[2] << 2);
     pde = ldl_phys(pde_ptr);
     (*cpu_fprintf)(f, "Root ptr: " TARGET_FMT_plx ", ctx: %d\n",
-                   (target_phys_addr_t)env->mmuregs[1] << 4, env->mmuregs[2]);
+                   (hwaddr)env->mmuregs[1] << 4, env->mmuregs[2]);
     for (n = 0, va = 0; n < 256; n++, va += 16 * 1024 * 1024) {
         pde = mmu_probe(env, va, 2);
         if (pde) {
@@ -431,7 +431,7 @@ int target_memory_rw_debug(CPUSPARCState *env, target_ulong addr,
 #else /* !TARGET_SPARC64 */
 
 /* 41 bit physical address space */
-static inline target_phys_addr_t ultrasparc_truncate_physical(uint64_t x)
+static inline hwaddr ultrasparc_truncate_physical(uint64_t x)
 {
     return x & 0x1ffffffffffULL;
 }
@@ -445,7 +445,7 @@ static inline target_phys_addr_t ultrasparc_truncate_physical(uint64_t x)
    entry size */
 static inline int ultrasparc_tag_match(SparcTLBEntry *tlb,
                                        uint64_t address, uint64_t context,
-                                       target_phys_addr_t *physical)
+                                       hwaddr *physical)
 {
     uint64_t mask;
 
@@ -478,7 +478,7 @@ static inline int ultrasparc_tag_match(SparcTLBEntry *tlb,
 }
 
 static int get_physical_address_data(CPUSPARCState *env,
-                                     target_phys_addr_t *physical, int *prot,
+                                     hwaddr *physical, int *prot,
                                      target_ulong address, int rw, int mmu_idx)
 {
     unsigned int i;
@@ -597,7 +597,7 @@ static int get_physical_address_data(CPUSPARCState *env,
 }
 
 static int get_physical_address_code(CPUSPARCState *env,
-                                     target_phys_addr_t *physical, int *prot,
+                                     hwaddr *physical, int *prot,
                                      target_ulong address, int mmu_idx)
 {
     unsigned int i;
@@ -665,7 +665,7 @@ static int get_physical_address_code(CPUSPARCState *env,
     return 1;
 }
 
-static int get_physical_address(CPUSPARCState *env, target_phys_addr_t *physical,
+static int get_physical_address(CPUSPARCState *env, hwaddr *physical,
                                 int *prot, int *access_index,
                                 target_ulong address, int rw, int mmu_idx,
                                 target_ulong *page_size)
@@ -703,7 +703,7 @@ int cpu_sparc_handle_mmu_fault(CPUSPARCState *env, target_ulong address, int rw,
                                int mmu_idx)
 {
     target_ulong vaddr;
-    target_phys_addr_t paddr;
+    hwaddr paddr;
     target_ulong page_size;
     int error_code = 0, prot, access_index;
 
@@ -810,7 +810,7 @@ void dump_mmu(FILE *f, fprintf_function cpu_fprintf, CPUSPARCState *env)
 
 #endif /* TARGET_SPARC64 */
 
-static int cpu_sparc_get_phys_page(CPUSPARCState *env, target_phys_addr_t *phys,
+static int cpu_sparc_get_phys_page(CPUSPARCState *env, hwaddr *phys,
                                    target_ulong addr, int rw, int mmu_idx)
 {
     target_ulong page_size;
@@ -821,10 +821,10 @@ static int cpu_sparc_get_phys_page(CPUSPARCState *env, target_phys_addr_t *phys,
 }
 
 #if defined(TARGET_SPARC64)
-target_phys_addr_t cpu_get_phys_page_nofault(CPUSPARCState *env, target_ulong addr,
+hwaddr cpu_get_phys_page_nofault(CPUSPARCState *env, target_ulong addr,
                                            int mmu_idx)
 {
-    target_phys_addr_t phys_addr;
+    hwaddr phys_addr;
 
     if (cpu_sparc_get_phys_page(env, &phys_addr, addr, 4, mmu_idx) != 0) {
         return -1;
@@ -833,9 +833,9 @@ target_phys_addr_t cpu_get_phys_page_nofault(CPUSPARCState *env, target_ulong ad
 }
 #endif
 
-target_phys_addr_t cpu_get_phys_page_debug(CPUSPARCState *env, target_ulong addr)
+hwaddr cpu_get_phys_page_debug(CPUSPARCState *env, target_ulong addr)
 {
-    target_phys_addr_t phys_addr;
+    hwaddr phys_addr;
     int mmu_idx = cpu_mmu_index(env);
     MemoryRegionSection section;
 
diff --git a/target-sparc/translate.c b/target-sparc/translate.c
index b95f91cd3..2ae803695 100644
--- a/target-sparc/translate.c
+++ b/target-sparc/translate.c
@@ -48,7 +48,7 @@ static TCGv cpu_y;
 #ifndef CONFIG_USER_ONLY
 static TCGv cpu_tbr;
 #endif
-static TCGv cpu_cond, cpu_dst, cpu_addr, cpu_val;
+static TCGv cpu_cond;
 #ifdef TARGET_SPARC64
 static TCGv_i32 cpu_xcc, cpu_asi, cpu_fprs;
 static TCGv cpu_gsr;
@@ -58,10 +58,6 @@ static TCGv_i32 cpu_softint;
 #else
 static TCGv cpu_wim;
 #endif
-/* local register indexes (only used inside old micro ops) */
-static TCGv cpu_tmp0;
-static TCGv_i32 cpu_tmp32;
-static TCGv_i64 cpu_tmp64;
 /* Floating point registers */
 static TCGv_i64 cpu_fpr[TARGET_DPREGS];
 
@@ -83,9 +79,18 @@ typedef struct DisasContext {
     struct TranslationBlock *tb;
     sparc_def_t *def;
     TCGv_i32 t32[3];
+    TCGv ttl[5];
     int n_t32;
+    int n_ttl;
 } DisasContext;
 
+typedef struct {
+    TCGCond cond;
+    bool is_bool;
+    bool g1, g2;
+    TCGv c1, c2;
+} DisasCompare;
+
 // This function uses non-native bit order
 #define GET_FIELD(X, FROM, TO)                                  \
     ((X) >> (31 - (TO)) & ((1 << ((TO) - (FROM) + 1)) - 1))
@@ -116,6 +121,22 @@ static int sign_extend(int x, int len)
 
 #define IS_IMM (insn & (1<<13))
 
+static inline TCGv_i32 get_temp_i32(DisasContext *dc)
+{
+    TCGv_i32 t;
+    assert(dc->n_t32 < ARRAY_SIZE(dc->t32));
+    dc->t32[dc->n_t32++] = t = tcg_temp_new_i32();
+    return t;
+}
+
+static inline TCGv get_temp_tl(DisasContext *dc)
+{
+    TCGv t;
+    assert(dc->n_ttl < ARRAY_SIZE(dc->ttl));
+    dc->ttl[dc->n_ttl++] = t = tcg_temp_new();
+    return t;
+}
+
 static inline void gen_update_fprs_dirty(int rd)
 {
 #if defined(TARGET_SPARC64)
@@ -136,16 +157,13 @@ static TCGv_i32 gen_load_fpr_F(DisasContext *dc, unsigned int src)
     if (src & 1) {
         return MAKE_TCGV_I32(GET_TCGV_I64(cpu_fpr[src / 2]));
     } else {
-        TCGv_i32 ret = tcg_temp_local_new_i32();
+        TCGv_i32 ret = get_temp_i32(dc);
         TCGv_i64 t = tcg_temp_new_i64();
 
         tcg_gen_shri_i64(t, cpu_fpr[src / 2], 32);
         tcg_gen_trunc_i64_i32(ret, t);
         tcg_temp_free_i64(t);
 
-        dc->t32[dc->n_t32++] = ret;
-        assert(dc->n_t32 <= ARRAY_SIZE(dc->t32));
-
         return ret;
     }
 #endif
@@ -167,9 +185,9 @@ static void gen_store_fpr_F(DisasContext *dc, unsigned int dst, TCGv_i32 v)
     gen_update_fprs_dirty(dst);
 }
 
-static TCGv_i32 gen_dest_fpr_F(void)
+static TCGv_i32 gen_dest_fpr_F(DisasContext *dc)
 {
-    return cpu_tmp32;
+    return get_temp_i32(dc);
 }
 
 static TCGv_i64 gen_load_fpr_D(DisasContext *dc, unsigned int src)
@@ -185,9 +203,9 @@ static void gen_store_fpr_D(DisasContext *dc, unsigned int dst, TCGv_i64 v)
     gen_update_fprs_dirty(dst);
 }
 
-static TCGv_i64 gen_dest_fpr_D(void)
+static TCGv_i64 gen_dest_fpr_D(DisasContext *dc, unsigned int dst)
 {
-    return cpu_tmp64;
+    return cpu_fpr[DFPREG(dst) / 2];
 }
 
 static void gen_op_load_fpr_QT0(unsigned int src)
@@ -256,25 +274,38 @@ static inline void gen_address_mask(DisasContext *dc, TCGv addr)
 #endif
 }
 
-static inline void gen_movl_reg_TN(int reg, TCGv tn)
+static inline TCGv gen_load_gpr(DisasContext *dc, int reg)
 {
-    if (reg == 0)
-        tcg_gen_movi_tl(tn, 0);
-    else if (reg < 8)
-        tcg_gen_mov_tl(tn, cpu_gregs[reg]);
-    else {
-        tcg_gen_ld_tl(tn, cpu_regwptr, (reg - 8) * sizeof(target_ulong));
+    if (reg == 0 || reg >= 8) {
+        TCGv t = get_temp_tl(dc);
+        if (reg == 0) {
+            tcg_gen_movi_tl(t, 0);
+        } else {
+            tcg_gen_ld_tl(t, cpu_regwptr, (reg - 8) * sizeof(target_ulong));
+        }
+        return t;
+    } else {
+        return cpu_gregs[reg];
     }
 }
 
-static inline void gen_movl_TN_reg(int reg, TCGv tn)
+static inline void gen_store_gpr(DisasContext *dc, int reg, TCGv v)
 {
-    if (reg == 0)
-        return;
-    else if (reg < 8)
-        tcg_gen_mov_tl(cpu_gregs[reg], tn);
-    else {
-        tcg_gen_st_tl(tn, cpu_regwptr, (reg - 8) * sizeof(target_ulong));
+    if (reg > 0) {
+        if (reg < 8) {
+            tcg_gen_mov_tl(cpu_gregs[reg], v);
+        } else {
+            tcg_gen_st_tl(v, cpu_regwptr, (reg - 8) * sizeof(target_ulong));
+        }
+    }
+}
+
+static inline TCGv gen_dest_gpr(DisasContext *dc, int reg)
+{
+    if (reg == 0 || reg >= 8) {
+        return get_temp_tl(dc);
+    } else {
+        return cpu_gregs[reg];
     }
 }
 
@@ -329,43 +360,6 @@ static inline void gen_mov_reg_C(TCGv reg, TCGv_i32 src)
     tcg_gen_andi_tl(reg, reg, 0x1);
 }
 
-static inline void gen_add_tv(TCGv dst, TCGv src1, TCGv src2)
-{
-    TCGv r_temp;
-    TCGv_i32 r_const;
-    int l1;
-
-    l1 = gen_new_label();
-
-    r_temp = tcg_temp_new();
-    tcg_gen_xor_tl(r_temp, src1, src2);
-    tcg_gen_not_tl(r_temp, r_temp);
-    tcg_gen_xor_tl(cpu_tmp0, src1, dst);
-    tcg_gen_and_tl(r_temp, r_temp, cpu_tmp0);
-    tcg_gen_andi_tl(r_temp, r_temp, (1ULL << 31));
-    tcg_gen_brcondi_tl(TCG_COND_EQ, r_temp, 0, l1);
-    r_const = tcg_const_i32(TT_TOVF);
-    gen_helper_raise_exception(cpu_env, r_const);
-    tcg_temp_free_i32(r_const);
-    gen_set_label(l1);
-    tcg_temp_free(r_temp);
-}
-
-static inline void gen_tag_tv(TCGv src1, TCGv src2)
-{
-    int l1;
-    TCGv_i32 r_const;
-
-    l1 = gen_new_label();
-    tcg_gen_or_tl(cpu_tmp0, src1, src2);
-    tcg_gen_andi_tl(cpu_tmp0, cpu_tmp0, 0x3);
-    tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_tmp0, 0, l1);
-    r_const = tcg_const_i32(TT_TOVF);
-    gen_helper_raise_exception(cpu_env, r_const);
-    tcg_temp_free_i32(r_const);
-    gen_set_label(l1);
-}
-
 static inline void gen_op_addi_cc(TCGv dst, TCGv src1, target_long src2)
 {
     tcg_gen_mov_tl(cpu_cc_src, src1);
@@ -510,45 +504,6 @@ static void gen_op_addx_int(DisasContext *dc, TCGv dst, TCGv src1,
     }
 }
 
-static inline void gen_op_tadd_cc(TCGv dst, TCGv src1, TCGv src2)
-{
-    tcg_gen_mov_tl(cpu_cc_src, src1);
-    tcg_gen_mov_tl(cpu_cc_src2, src2);
-    tcg_gen_add_tl(cpu_cc_dst, cpu_cc_src, cpu_cc_src2);
-    tcg_gen_mov_tl(dst, cpu_cc_dst);
-}
-
-static inline void gen_op_tadd_ccTV(TCGv dst, TCGv src1, TCGv src2)
-{
-    tcg_gen_mov_tl(cpu_cc_src, src1);
-    tcg_gen_mov_tl(cpu_cc_src2, src2);
-    gen_tag_tv(cpu_cc_src, cpu_cc_src2);
-    tcg_gen_add_tl(cpu_cc_dst, cpu_cc_src, cpu_cc_src2);
-    gen_add_tv(cpu_cc_dst, cpu_cc_src, cpu_cc_src2);
-    tcg_gen_mov_tl(dst, cpu_cc_dst);
-}
-
-static inline void gen_sub_tv(TCGv dst, TCGv src1, TCGv src2)
-{
-    TCGv r_temp;
-    TCGv_i32 r_const;
-    int l1;
-
-    l1 = gen_new_label();
-
-    r_temp = tcg_temp_new();
-    tcg_gen_xor_tl(r_temp, src1, src2);
-    tcg_gen_xor_tl(cpu_tmp0, src1, dst);
-    tcg_gen_and_tl(r_temp, r_temp, cpu_tmp0);
-    tcg_gen_andi_tl(r_temp, r_temp, (1ULL << 31));
-    tcg_gen_brcondi_tl(TCG_COND_EQ, r_temp, 0, l1);
-    r_const = tcg_const_i32(TT_TOVF);
-    gen_helper_raise_exception(cpu_env, r_const);
-    tcg_temp_free_i32(r_const);
-    gen_set_label(l1);
-    tcg_temp_free(r_temp);
-}
-
 static inline void gen_op_subi_cc(TCGv dst, TCGv src1, target_long src2, DisasContext *dc)
 {
     tcg_gen_mov_tl(cpu_cc_src, src1);
@@ -649,63 +604,46 @@ static void gen_op_subx_int(DisasContext *dc, TCGv dst, TCGv src1,
     }
 }
 
-static inline void gen_op_tsub_cc(TCGv dst, TCGv src1, TCGv src2)
-{
-    tcg_gen_mov_tl(cpu_cc_src, src1);
-    tcg_gen_mov_tl(cpu_cc_src2, src2);
-    tcg_gen_sub_tl(cpu_cc_dst, cpu_cc_src, cpu_cc_src2);
-    tcg_gen_mov_tl(dst, cpu_cc_dst);
-}
-
-static inline void gen_op_tsub_ccTV(TCGv dst, TCGv src1, TCGv src2)
-{
-    tcg_gen_mov_tl(cpu_cc_src, src1);
-    tcg_gen_mov_tl(cpu_cc_src2, src2);
-    gen_tag_tv(cpu_cc_src, cpu_cc_src2);
-    tcg_gen_sub_tl(cpu_cc_dst, cpu_cc_src, cpu_cc_src2);
-    gen_sub_tv(cpu_cc_dst, cpu_cc_src, cpu_cc_src2);
-    tcg_gen_mov_tl(dst, cpu_cc_dst);
-}
-
 static inline void gen_op_mulscc(TCGv dst, TCGv src1, TCGv src2)
 {
-    TCGv r_temp;
-    int l1;
+    TCGv r_temp, zero, t0;
 
-    l1 = gen_new_label();
     r_temp = tcg_temp_new();
+    t0 = tcg_temp_new();
 
     /* old op:
     if (!(env->y & 1))
         T1 = 0;
     */
+    zero = tcg_const_tl(0);
     tcg_gen_andi_tl(cpu_cc_src, src1, 0xffffffff);
     tcg_gen_andi_tl(r_temp, cpu_y, 0x1);
     tcg_gen_andi_tl(cpu_cc_src2, src2, 0xffffffff);
-    tcg_gen_brcondi_tl(TCG_COND_NE, r_temp, 0, l1);
-    tcg_gen_movi_tl(cpu_cc_src2, 0);
-    gen_set_label(l1);
+    tcg_gen_movcond_tl(TCG_COND_EQ, cpu_cc_src2, r_temp, zero,
+                       zero, cpu_cc_src2);
+    tcg_temp_free(zero);
 
     // b2 = T0 & 1;
     // env->y = (b2 << 31) | (env->y >> 1);
     tcg_gen_andi_tl(r_temp, cpu_cc_src, 0x1);
     tcg_gen_shli_tl(r_temp, r_temp, 31);
-    tcg_gen_shri_tl(cpu_tmp0, cpu_y, 1);
-    tcg_gen_andi_tl(cpu_tmp0, cpu_tmp0, 0x7fffffff);
-    tcg_gen_or_tl(cpu_tmp0, cpu_tmp0, r_temp);
-    tcg_gen_andi_tl(cpu_y, cpu_tmp0, 0xffffffff);
+    tcg_gen_shri_tl(t0, cpu_y, 1);
+    tcg_gen_andi_tl(t0, t0, 0x7fffffff);
+    tcg_gen_or_tl(t0, t0, r_temp);
+    tcg_gen_andi_tl(cpu_y, t0, 0xffffffff);
 
     // b1 = N ^ V;
-    gen_mov_reg_N(cpu_tmp0, cpu_psr);
+    gen_mov_reg_N(t0, cpu_psr);
     gen_mov_reg_V(r_temp, cpu_psr);
-    tcg_gen_xor_tl(cpu_tmp0, cpu_tmp0, r_temp);
+    tcg_gen_xor_tl(t0, t0, r_temp);
     tcg_temp_free(r_temp);
 
     // T0 = (b1 << 31) | (T0 >> 1);
     // src1 = T0;
-    tcg_gen_shli_tl(cpu_tmp0, cpu_tmp0, 31);
+    tcg_gen_shli_tl(t0, t0, 31);
     tcg_gen_shri_tl(cpu_cc_src, cpu_cc_src, 1);
-    tcg_gen_or_tl(cpu_cc_src, cpu_cc_src, cpu_tmp0);
+    tcg_gen_or_tl(cpu_cc_src, cpu_cc_src, t0);
+    tcg_temp_free(t0);
 
     tcg_gen_add_tl(cpu_cc_dst, cpu_cc_src, cpu_cc_src2);
 
@@ -737,9 +675,9 @@ static inline void gen_op_multiply(TCGv dst, TCGv src1, TCGv src2, int sign_ext)
     tcg_gen_mul_i64(r_temp2, r_temp, r_temp2);
 
     tcg_gen_shri_i64(r_temp, r_temp2, 32);
-    tcg_gen_trunc_i64_tl(cpu_tmp0, r_temp);
+    tcg_gen_trunc_i64_tl(cpu_y, r_temp);
     tcg_temp_free_i64(r_temp);
-    tcg_gen_andi_tl(cpu_y, cpu_tmp0, 0xffffffff);
+    tcg_gen_andi_tl(cpu_y, cpu_y, 0xffffffff);
 
     tcg_gen_trunc_i64_tl(dst, r_temp2);
 
@@ -761,44 +699,6 @@ static inline void gen_op_smul(TCGv dst, TCGv src1, TCGv src2)
     gen_op_multiply(dst, src1, src2, 1);
 }
 
-#ifdef TARGET_SPARC64
-static inline void gen_trap_ifdivzero_tl(TCGv divisor)
-{
-    TCGv_i32 r_const;
-    int l1;
-
-    l1 = gen_new_label();
-    tcg_gen_brcondi_tl(TCG_COND_NE, divisor, 0, l1);
-    r_const = tcg_const_i32(TT_DIV_ZERO);
-    gen_helper_raise_exception(cpu_env, r_const);
-    tcg_temp_free_i32(r_const);
-    gen_set_label(l1);
-}
-
-static inline void gen_op_sdivx(TCGv dst, TCGv src1, TCGv src2)
-{
-    int l1, l2;
-    TCGv r_temp1, r_temp2;
-
-    l1 = gen_new_label();
-    l2 = gen_new_label();
-    r_temp1 = tcg_temp_local_new();
-    r_temp2 = tcg_temp_local_new();
-    tcg_gen_mov_tl(r_temp1, src1);
-    tcg_gen_mov_tl(r_temp2, src2);
-    gen_trap_ifdivzero_tl(r_temp2);
-    tcg_gen_brcondi_tl(TCG_COND_NE, r_temp1, INT64_MIN, l1);
-    tcg_gen_brcondi_tl(TCG_COND_NE, r_temp2, -1, l1);
-    tcg_gen_movi_i64(dst, INT64_MIN);
-    tcg_gen_br(l2);
-    gen_set_label(l1);
-    tcg_gen_div_i64(dst, r_temp1, r_temp2);
-    gen_set_label(l2);
-    tcg_temp_free(r_temp1);
-    tcg_temp_free(r_temp2);
-}
-#endif
-
 // 1
 static inline void gen_op_eval_ba(TCGv dst)
 {
@@ -814,27 +714,33 @@ static inline void gen_op_eval_be(TCGv dst, TCGv_i32 src)
 // Z | (N ^ V)
 static inline void gen_op_eval_ble(TCGv dst, TCGv_i32 src)
 {
-    gen_mov_reg_N(cpu_tmp0, src);
+    TCGv t0 = tcg_temp_new();
+    gen_mov_reg_N(t0, src);
     gen_mov_reg_V(dst, src);
-    tcg_gen_xor_tl(dst, dst, cpu_tmp0);
-    gen_mov_reg_Z(cpu_tmp0, src);
-    tcg_gen_or_tl(dst, dst, cpu_tmp0);
+    tcg_gen_xor_tl(dst, dst, t0);
+    gen_mov_reg_Z(t0, src);
+    tcg_gen_or_tl(dst, dst, t0);
+    tcg_temp_free(t0);
 }
 
 // N ^ V
 static inline void gen_op_eval_bl(TCGv dst, TCGv_i32 src)
 {
-    gen_mov_reg_V(cpu_tmp0, src);
+    TCGv t0 = tcg_temp_new();
+    gen_mov_reg_V(t0, src);
     gen_mov_reg_N(dst, src);
-    tcg_gen_xor_tl(dst, dst, cpu_tmp0);
+    tcg_gen_xor_tl(dst, dst, t0);
+    tcg_temp_free(t0);
 }
 
 // C | Z
 static inline void gen_op_eval_bleu(TCGv dst, TCGv_i32 src)
 {
-    gen_mov_reg_Z(cpu_tmp0, src);
+    TCGv t0 = tcg_temp_new();
+    gen_mov_reg_Z(t0, src);
     gen_mov_reg_C(dst, src);
-    tcg_gen_or_tl(dst, dst, cpu_tmp0);
+    tcg_gen_or_tl(dst, dst, t0);
+    tcg_temp_free(t0);
 }
 
 // C
@@ -871,29 +777,21 @@ static inline void gen_op_eval_bne(TCGv dst, TCGv_i32 src)
 // !(Z | (N ^ V))
 static inline void gen_op_eval_bg(TCGv dst, TCGv_i32 src)
 {
-    gen_mov_reg_N(cpu_tmp0, src);
-    gen_mov_reg_V(dst, src);
-    tcg_gen_xor_tl(dst, dst, cpu_tmp0);
-    gen_mov_reg_Z(cpu_tmp0, src);
-    tcg_gen_or_tl(dst, dst, cpu_tmp0);
+    gen_op_eval_ble(dst, src);
     tcg_gen_xori_tl(dst, dst, 0x1);
 }
 
 // !(N ^ V)
 static inline void gen_op_eval_bge(TCGv dst, TCGv_i32 src)
 {
-    gen_mov_reg_V(cpu_tmp0, src);
-    gen_mov_reg_N(dst, src);
-    tcg_gen_xor_tl(dst, dst, cpu_tmp0);
+    gen_op_eval_bl(dst, src);
     tcg_gen_xori_tl(dst, dst, 0x1);
 }
 
 // !(C | Z)
 static inline void gen_op_eval_bgu(TCGv dst, TCGv_i32 src)
 {
-    gen_mov_reg_Z(cpu_tmp0, src);
-    gen_mov_reg_C(dst, src);
-    tcg_gen_or_tl(dst, dst, cpu_tmp0);
+    gen_op_eval_bleu(dst, src);
     tcg_gen_xori_tl(dst, dst, 0x1);
 }
 
@@ -943,18 +841,22 @@ static inline void gen_mov_reg_FCC1(TCGv reg, TCGv src,
 static inline void gen_op_eval_fbne(TCGv dst, TCGv src,
                                     unsigned int fcc_offset)
 {
+    TCGv t0 = tcg_temp_new();
     gen_mov_reg_FCC0(dst, src, fcc_offset);
-    gen_mov_reg_FCC1(cpu_tmp0, src, fcc_offset);
-    tcg_gen_or_tl(dst, dst, cpu_tmp0);
+    gen_mov_reg_FCC1(t0, src, fcc_offset);
+    tcg_gen_or_tl(dst, dst, t0);
+    tcg_temp_free(t0);
 }
 
 // 1 or 2: FCC0 ^ FCC1
 static inline void gen_op_eval_fblg(TCGv dst, TCGv src,
                                     unsigned int fcc_offset)
 {
+    TCGv t0 = tcg_temp_new();
     gen_mov_reg_FCC0(dst, src, fcc_offset);
-    gen_mov_reg_FCC1(cpu_tmp0, src, fcc_offset);
-    tcg_gen_xor_tl(dst, dst, cpu_tmp0);
+    gen_mov_reg_FCC1(t0, src, fcc_offset);
+    tcg_gen_xor_tl(dst, dst, t0);
+    tcg_temp_free(t0);
 }
 
 // 1 or 3: FCC0
@@ -968,10 +870,11 @@ static inline void gen_op_eval_fbul(TCGv dst, TCGv src,
 static inline void gen_op_eval_fbl(TCGv dst, TCGv src,
                                     unsigned int fcc_offset)
 {
+    TCGv t0 = tcg_temp_new();
     gen_mov_reg_FCC0(dst, src, fcc_offset);
-    gen_mov_reg_FCC1(cpu_tmp0, src, fcc_offset);
-    tcg_gen_xori_tl(cpu_tmp0, cpu_tmp0, 0x1);
-    tcg_gen_and_tl(dst, dst, cpu_tmp0);
+    gen_mov_reg_FCC1(t0, src, fcc_offset);
+    tcg_gen_andc_tl(dst, dst, t0);
+    tcg_temp_free(t0);
 }
 
 // 2 or 3: FCC1
@@ -985,39 +888,46 @@ static inline void gen_op_eval_fbug(TCGv dst, TCGv src,
 static inline void gen_op_eval_fbg(TCGv dst, TCGv src,
                                     unsigned int fcc_offset)
 {
+    TCGv t0 = tcg_temp_new();
     gen_mov_reg_FCC0(dst, src, fcc_offset);
-    tcg_gen_xori_tl(dst, dst, 0x1);
-    gen_mov_reg_FCC1(cpu_tmp0, src, fcc_offset);
-    tcg_gen_and_tl(dst, dst, cpu_tmp0);
+    gen_mov_reg_FCC1(t0, src, fcc_offset);
+    tcg_gen_andc_tl(dst, t0, dst);
+    tcg_temp_free(t0);
 }
 
 // 3: FCC0 & FCC1
 static inline void gen_op_eval_fbu(TCGv dst, TCGv src,
                                     unsigned int fcc_offset)
 {
+    TCGv t0 = tcg_temp_new();
     gen_mov_reg_FCC0(dst, src, fcc_offset);
-    gen_mov_reg_FCC1(cpu_tmp0, src, fcc_offset);
-    tcg_gen_and_tl(dst, dst, cpu_tmp0);
+    gen_mov_reg_FCC1(t0, src, fcc_offset);
+    tcg_gen_and_tl(dst, dst, t0);
+    tcg_temp_free(t0);
 }
 
 // 0: !(FCC0 | FCC1)
 static inline void gen_op_eval_fbe(TCGv dst, TCGv src,
                                     unsigned int fcc_offset)
 {
+    TCGv t0 = tcg_temp_new();
     gen_mov_reg_FCC0(dst, src, fcc_offset);
-    gen_mov_reg_FCC1(cpu_tmp0, src, fcc_offset);
-    tcg_gen_or_tl(dst, dst, cpu_tmp0);
+    gen_mov_reg_FCC1(t0, src, fcc_offset);
+    tcg_gen_or_tl(dst, dst, t0);
     tcg_gen_xori_tl(dst, dst, 0x1);
+    tcg_temp_free(t0);
 }
 
 // 0 or 3: !(FCC0 ^ FCC1)
 static inline void gen_op_eval_fbue(TCGv dst, TCGv src,
                                     unsigned int fcc_offset)
 {
+    TCGv t0 = tcg_temp_new();
     gen_mov_reg_FCC0(dst, src, fcc_offset);
-    gen_mov_reg_FCC1(cpu_tmp0, src, fcc_offset);
-    tcg_gen_xor_tl(dst, dst, cpu_tmp0);
+    gen_mov_reg_FCC1(t0, src, fcc_offset);
+    tcg_gen_xor_tl(dst, dst, t0);
     tcg_gen_xori_tl(dst, dst, 0x1);
+    tcg_temp_free(t0);
 }
 
 // 0 or 2: !FCC0
@@ -1032,11 +942,12 @@ static inline void gen_op_eval_fbge(TCGv dst, TCGv src,
 static inline void gen_op_eval_fbuge(TCGv dst, TCGv src,
                                     unsigned int fcc_offset)
 {
+    TCGv t0 = tcg_temp_new();
     gen_mov_reg_FCC0(dst, src, fcc_offset);
-    gen_mov_reg_FCC1(cpu_tmp0, src, fcc_offset);
-    tcg_gen_xori_tl(cpu_tmp0, cpu_tmp0, 0x1);
-    tcg_gen_and_tl(dst, dst, cpu_tmp0);
+    gen_mov_reg_FCC1(t0, src, fcc_offset);
+    tcg_gen_andc_tl(dst, dst, t0);
     tcg_gen_xori_tl(dst, dst, 0x1);
+    tcg_temp_free(t0);
 }
 
 // 0 or 1: !FCC1
@@ -1051,21 +962,24 @@ static inline void gen_op_eval_fble(TCGv dst, TCGv src,
 static inline void gen_op_eval_fbule(TCGv dst, TCGv src,
                                     unsigned int fcc_offset)
 {
+    TCGv t0 = tcg_temp_new();
     gen_mov_reg_FCC0(dst, src, fcc_offset);
+    gen_mov_reg_FCC1(t0, src, fcc_offset);
+    tcg_gen_andc_tl(dst, t0, dst);
     tcg_gen_xori_tl(dst, dst, 0x1);
-    gen_mov_reg_FCC1(cpu_tmp0, src, fcc_offset);
-    tcg_gen_and_tl(dst, dst, cpu_tmp0);
-    tcg_gen_xori_tl(dst, dst, 0x1);
+    tcg_temp_free(t0);
 }
 
 // !3: !(FCC0 & FCC1)
 static inline void gen_op_eval_fbo(TCGv dst, TCGv src,
                                     unsigned int fcc_offset)
 {
+    TCGv t0 = tcg_temp_new();
     gen_mov_reg_FCC0(dst, src, fcc_offset);
-    gen_mov_reg_FCC1(cpu_tmp0, src, fcc_offset);
-    tcg_gen_and_tl(dst, dst, cpu_tmp0);
+    gen_mov_reg_FCC1(t0, src, fcc_offset);
+    tcg_gen_and_tl(dst, dst, t0);
     tcg_gen_xori_tl(dst, dst, 0x1);
+    tcg_temp_free(t0);
 }
 
 static inline void gen_branch2(DisasContext *dc, target_ulong pc1,
@@ -1098,59 +1012,57 @@ static inline void gen_branch_a(DisasContext *dc, target_ulong pc1,
     gen_goto_tb(dc, 1, pc2 + 4, pc2 + 8);
 }
 
-static inline void gen_generic_branch(target_ulong npc1, target_ulong npc2,
-                                      TCGv r_cond)
+static inline void gen_generic_branch(DisasContext *dc)
 {
-    int l1, l2;
-
-    l1 = gen_new_label();
-    l2 = gen_new_label();
-
-    tcg_gen_brcondi_tl(TCG_COND_EQ, r_cond, 0, l1);
+    TCGv npc0 = tcg_const_tl(dc->jump_pc[0]);
+    TCGv npc1 = tcg_const_tl(dc->jump_pc[1]);
+    TCGv zero = tcg_const_tl(0);
 
-    tcg_gen_movi_tl(cpu_npc, npc1);
-    tcg_gen_br(l2);
+    tcg_gen_movcond_tl(TCG_COND_NE, cpu_npc, cpu_cond, zero, npc0, npc1);
 
-    gen_set_label(l1);
-    tcg_gen_movi_tl(cpu_npc, npc2);
-    gen_set_label(l2);
+    tcg_temp_free(npc0);
+    tcg_temp_free(npc1);
+    tcg_temp_free(zero);
 }
 
 /* call this function before using the condition register as it may
    have been set for a jump */
-static inline void flush_cond(DisasContext *dc, TCGv cond)
+static inline void flush_cond(DisasContext *dc)
 {
     if (dc->npc == JUMP_PC) {
-        gen_generic_branch(dc->jump_pc[0], dc->jump_pc[1], cond);
+        gen_generic_branch(dc);
         dc->npc = DYNAMIC_PC;
     }
 }
 
-static inline void save_npc(DisasContext *dc, TCGv cond)
+static inline void save_npc(DisasContext *dc)
 {
     if (dc->npc == JUMP_PC) {
-        gen_generic_branch(dc->jump_pc[0], dc->jump_pc[1], cond);
+        gen_generic_branch(dc);
         dc->npc = DYNAMIC_PC;
     } else if (dc->npc != DYNAMIC_PC) {
         tcg_gen_movi_tl(cpu_npc, dc->npc);
     }
 }
 
-static inline void save_state(DisasContext *dc, TCGv cond)
+static inline void update_psr(DisasContext *dc)
 {
-    tcg_gen_movi_tl(cpu_pc, dc->pc);
-    /* flush pending conditional evaluations before exposing cpu state */
     if (dc->cc_op != CC_OP_FLAGS) {
         dc->cc_op = CC_OP_FLAGS;
         gen_helper_compute_psr(cpu_env);
     }
-    save_npc(dc, cond);
 }
 
-static inline void gen_mov_pc_npc(DisasContext *dc, TCGv cond)
+static inline void save_state(DisasContext *dc)
+{
+    tcg_gen_movi_tl(cpu_pc, dc->pc);
+    save_npc(dc);
+}
+
+static inline void gen_mov_pc_npc(DisasContext *dc)
 {
     if (dc->npc == JUMP_PC) {
-        gen_generic_branch(dc->jump_pc[0], dc->jump_pc[1], cond);
+        gen_generic_branch(dc);
         tcg_gen_mov_tl(cpu_pc, cpu_npc);
         dc->pc = DYNAMIC_PC;
     } else if (dc->npc == DYNAMIC_PC) {
@@ -1167,82 +1079,201 @@ static inline void gen_op_next_insn(void)
     tcg_gen_addi_tl(cpu_npc, cpu_npc, 4);
 }
 
-static inline void gen_cond(TCGv r_dst, unsigned int cc, unsigned int cond,
-                            DisasContext *dc)
+static void free_compare(DisasCompare *cmp)
 {
+    if (!cmp->g1) {
+        tcg_temp_free(cmp->c1);
+    }
+    if (!cmp->g2) {
+        tcg_temp_free(cmp->c2);
+    }
+}
+
+static void gen_compare(DisasCompare *cmp, bool xcc, unsigned int cond,
+                        DisasContext *dc)
+{
+    static int subcc_cond[16] = {
+        TCG_COND_NEVER,
+        TCG_COND_EQ,
+        TCG_COND_LE,
+        TCG_COND_LT,
+        TCG_COND_LEU,
+        TCG_COND_LTU,
+        -1, /* neg */
+        -1, /* overflow */
+        TCG_COND_ALWAYS,
+        TCG_COND_NE,
+        TCG_COND_GT,
+        TCG_COND_GE,
+        TCG_COND_GTU,
+        TCG_COND_GEU,
+        -1, /* pos */
+        -1, /* no overflow */
+    };
+
+    static int logic_cond[16] = {
+        TCG_COND_NEVER,
+        TCG_COND_EQ,     /* eq:  Z */
+        TCG_COND_LE,     /* le:  Z | (N ^ V) -> Z | N */
+        TCG_COND_LT,     /* lt:  N ^ V -> N */
+        TCG_COND_EQ,     /* leu: C | Z -> Z */
+        TCG_COND_NEVER,  /* ltu: C -> 0 */
+        TCG_COND_LT,     /* neg: N */
+        TCG_COND_NEVER,  /* vs:  V -> 0 */
+        TCG_COND_ALWAYS,
+        TCG_COND_NE,     /* ne:  !Z */
+        TCG_COND_GT,     /* gt:  !(Z | (N ^ V)) -> !(Z | N) */
+        TCG_COND_GE,     /* ge:  !(N ^ V) -> !N */
+        TCG_COND_NE,     /* gtu: !(C | Z) -> !Z */
+        TCG_COND_ALWAYS, /* geu: !C -> 1 */
+        TCG_COND_GE,     /* pos: !N */
+        TCG_COND_ALWAYS, /* vc:  !V -> 1 */
+    };
+
     TCGv_i32 r_src;
+    TCGv r_dst;
 
 #ifdef TARGET_SPARC64
-    if (cc)
+    if (xcc) {
         r_src = cpu_xcc;
-    else
+    } else {
         r_src = cpu_psr;
+    }
 #else
     r_src = cpu_psr;
 #endif
+
     switch (dc->cc_op) {
-    case CC_OP_FLAGS:
+    case CC_OP_LOGIC:
+        cmp->cond = logic_cond[cond];
+    do_compare_dst_0:
+        cmp->is_bool = false;
+        cmp->g2 = false;
+        cmp->c2 = tcg_const_tl(0);
+#ifdef TARGET_SPARC64
+        if (!xcc) {
+            cmp->g1 = false;
+            cmp->c1 = tcg_temp_new();
+            tcg_gen_ext32s_tl(cmp->c1, cpu_cc_dst);
+            break;
+        }
+#endif
+        cmp->g1 = true;
+        cmp->c1 = cpu_cc_dst;
+        break;
+
+    case CC_OP_SUB:
+        switch (cond) {
+        case 6:  /* neg */
+        case 14: /* pos */
+            cmp->cond = (cond == 6 ? TCG_COND_LT : TCG_COND_GE);
+            goto do_compare_dst_0;
+
+        case 7: /* overflow */
+        case 15: /* !overflow */
+            goto do_dynamic;
+
+        default:
+            cmp->cond = subcc_cond[cond];
+            cmp->is_bool = false;
+#ifdef TARGET_SPARC64
+            if (!xcc) {
+                /* Note that sign-extension works for unsigned compares as
+                   long as both operands are sign-extended.  */
+                cmp->g1 = cmp->g2 = false;
+                cmp->c1 = tcg_temp_new();
+                cmp->c2 = tcg_temp_new();
+                tcg_gen_ext32s_tl(cmp->c1, cpu_cc_src);
+                tcg_gen_ext32s_tl(cmp->c2, cpu_cc_src2);
+                break;
+            }
+#endif
+            cmp->g1 = cmp->g2 = true;
+            cmp->c1 = cpu_cc_src;
+            cmp->c2 = cpu_cc_src2;
+            break;
+        }
         break;
+
     default:
+    do_dynamic:
         gen_helper_compute_psr(cpu_env);
         dc->cc_op = CC_OP_FLAGS;
-        break;
-    }
-    switch (cond) {
-    case 0x0:
-        gen_op_eval_bn(r_dst);
-        break;
-    case 0x1:
-        gen_op_eval_be(r_dst, r_src);
-        break;
-    case 0x2:
-        gen_op_eval_ble(r_dst, r_src);
-        break;
-    case 0x3:
-        gen_op_eval_bl(r_dst, r_src);
-        break;
-    case 0x4:
-        gen_op_eval_bleu(r_dst, r_src);
-        break;
-    case 0x5:
-        gen_op_eval_bcs(r_dst, r_src);
-        break;
-    case 0x6:
-        gen_op_eval_bneg(r_dst, r_src);
-        break;
-    case 0x7:
-        gen_op_eval_bvs(r_dst, r_src);
-        break;
-    case 0x8:
-        gen_op_eval_ba(r_dst);
-        break;
-    case 0x9:
-        gen_op_eval_bne(r_dst, r_src);
-        break;
-    case 0xa:
-        gen_op_eval_bg(r_dst, r_src);
-        break;
-    case 0xb:
-        gen_op_eval_bge(r_dst, r_src);
-        break;
-    case 0xc:
-        gen_op_eval_bgu(r_dst, r_src);
-        break;
-    case 0xd:
-        gen_op_eval_bcc(r_dst, r_src);
-        break;
-    case 0xe:
-        gen_op_eval_bpos(r_dst, r_src);
-        break;
-    case 0xf:
-        gen_op_eval_bvc(r_dst, r_src);
+        /* FALLTHRU */
+
+    case CC_OP_FLAGS:
+        /* We're going to generate a boolean result.  */
+        cmp->cond = TCG_COND_NE;
+        cmp->is_bool = true;
+        cmp->g1 = cmp->g2 = false;
+        cmp->c1 = r_dst = tcg_temp_new();
+        cmp->c2 = tcg_const_tl(0);
+
+        switch (cond) {
+        case 0x0:
+            gen_op_eval_bn(r_dst);
+            break;
+        case 0x1:
+            gen_op_eval_be(r_dst, r_src);
+            break;
+        case 0x2:
+            gen_op_eval_ble(r_dst, r_src);
+            break;
+        case 0x3:
+            gen_op_eval_bl(r_dst, r_src);
+            break;
+        case 0x4:
+            gen_op_eval_bleu(r_dst, r_src);
+            break;
+        case 0x5:
+            gen_op_eval_bcs(r_dst, r_src);
+            break;
+        case 0x6:
+            gen_op_eval_bneg(r_dst, r_src);
+            break;
+        case 0x7:
+            gen_op_eval_bvs(r_dst, r_src);
+            break;
+        case 0x8:
+            gen_op_eval_ba(r_dst);
+            break;
+        case 0x9:
+            gen_op_eval_bne(r_dst, r_src);
+            break;
+        case 0xa:
+            gen_op_eval_bg(r_dst, r_src);
+            break;
+        case 0xb:
+            gen_op_eval_bge(r_dst, r_src);
+            break;
+        case 0xc:
+            gen_op_eval_bgu(r_dst, r_src);
+            break;
+        case 0xd:
+            gen_op_eval_bcc(r_dst, r_src);
+            break;
+        case 0xe:
+            gen_op_eval_bpos(r_dst, r_src);
+            break;
+        case 0xf:
+            gen_op_eval_bvc(r_dst, r_src);
+            break;
+        }
         break;
     }
 }
 
-static inline void gen_fcond(TCGv r_dst, unsigned int cc, unsigned int cond)
+static void gen_fcompare(DisasCompare *cmp, unsigned int cc, unsigned int cond)
 {
     unsigned int offset;
+    TCGv r_dst;
+
+    /* For now we still generate a straight boolean result.  */
+    cmp->cond = TCG_COND_NE;
+    cmp->is_bool = true;
+    cmp->g1 = cmp->g2 = false;
+    cmp->c1 = r_dst = tcg_temp_new();
+    cmp->c2 = tcg_const_tl(0);
 
     switch (cc) {
     default:
@@ -1312,6 +1343,37 @@ static inline void gen_fcond(TCGv r_dst, unsigned int cc, unsigned int cond)
     }
 }
 
+static void gen_cond(TCGv r_dst, unsigned int cc, unsigned int cond,
+                     DisasContext *dc)
+{
+    DisasCompare cmp;
+    gen_compare(&cmp, cc, cond, dc);
+
+    /* The interface is to return a boolean in r_dst.  */
+    if (cmp.is_bool) {
+        tcg_gen_mov_tl(r_dst, cmp.c1);
+    } else {
+        tcg_gen_setcond_tl(cmp.cond, r_dst, cmp.c1, cmp.c2);
+    }
+
+    free_compare(&cmp);
+}
+
+static void gen_fcond(TCGv r_dst, unsigned int cc, unsigned int cond)
+{
+    DisasCompare cmp;
+    gen_fcompare(&cmp, cc, cond);
+
+    /* The interface is to return a boolean in r_dst.  */
+    if (cmp.is_bool) {
+        tcg_gen_mov_tl(r_dst, cmp.c1);
+    } else {
+        tcg_gen_setcond_tl(cmp.cond, r_dst, cmp.c1, cmp.c2);
+    }
+
+    free_compare(&cmp);
+}
+
 #ifdef TARGET_SPARC64
 // Inverted logic
 static const int gen_tcg_cond_reg[8] = {
@@ -1325,20 +1387,29 @@ static const int gen_tcg_cond_reg[8] = {
     TCG_COND_LT,
 };
 
+static void gen_compare_reg(DisasCompare *cmp, int cond, TCGv r_src)
+{
+    cmp->cond = tcg_invert_cond(gen_tcg_cond_reg[cond]);
+    cmp->is_bool = false;
+    cmp->g1 = true;
+    cmp->g2 = false;
+    cmp->c1 = r_src;
+    cmp->c2 = tcg_const_tl(0);
+}
+
 static inline void gen_cond_reg(TCGv r_dst, int cond, TCGv r_src)
 {
-    int l1;
+    DisasCompare cmp;
+    gen_compare_reg(&cmp, cond, r_src);
 
-    l1 = gen_new_label();
-    tcg_gen_movi_tl(r_dst, 0);
-    tcg_gen_brcondi_tl(gen_tcg_cond_reg[cond], r_src, 0, l1);
-    tcg_gen_movi_tl(r_dst, 1);
-    gen_set_label(l1);
+    /* The interface is to return a boolean in r_dst.  */
+    tcg_gen_setcond_tl(cmp.cond, r_dst, cmp.c1, cmp.c2);
+
+    free_compare(&cmp);
 }
 #endif
 
-static void do_branch(DisasContext *dc, int32_t offset, uint32_t insn, int cc,
-                      TCGv r_cond)
+static void do_branch(DisasContext *dc, int32_t offset, uint32_t insn, int cc)
 {
     unsigned int cond = GET_FIELD(insn, 3, 6), a = (insn & (1 << 29));
     target_ulong target = dc->pc + offset;
@@ -1368,10 +1439,10 @@ static void do_branch(DisasContext *dc, int32_t offset, uint32_t insn, int cc,
             tcg_gen_mov_tl(cpu_pc, cpu_npc);
         }
     } else {
-        flush_cond(dc, r_cond);
-        gen_cond(r_cond, cc, cond, dc);
+        flush_cond(dc);
+        gen_cond(cpu_cond, cc, cond, dc);
         if (a) {
-            gen_branch_a(dc, target, dc->npc, r_cond);
+            gen_branch_a(dc, target, dc->npc, cpu_cond);
             dc->is_br = 1;
         } else {
             dc->pc = dc->npc;
@@ -1387,8 +1458,7 @@ static void do_branch(DisasContext *dc, int32_t offset, uint32_t insn, int cc,
     }
 }
 
-static void do_fbranch(DisasContext *dc, int32_t offset, uint32_t insn, int cc,
-                      TCGv r_cond)
+static void do_fbranch(DisasContext *dc, int32_t offset, uint32_t insn, int cc)
 {
     unsigned int cond = GET_FIELD(insn, 3, 6), a = (insn & (1 << 29));
     target_ulong target = dc->pc + offset;
@@ -1418,10 +1488,10 @@ static void do_fbranch(DisasContext *dc, int32_t offset, uint32_t insn, int cc,
             tcg_gen_mov_tl(cpu_pc, cpu_npc);
         }
     } else {
-        flush_cond(dc, r_cond);
-        gen_fcond(r_cond, cc, cond);
+        flush_cond(dc);
+        gen_fcond(cpu_cond, cc, cond);
         if (a) {
-            gen_branch_a(dc, target, dc->npc, r_cond);
+            gen_branch_a(dc, target, dc->npc, cpu_cond);
             dc->is_br = 1;
         } else {
             dc->pc = dc->npc;
@@ -1439,7 +1509,7 @@ static void do_fbranch(DisasContext *dc, int32_t offset, uint32_t insn, int cc,
 
 #ifdef TARGET_SPARC64
 static void do_branch_reg(DisasContext *dc, int32_t offset, uint32_t insn,
-                          TCGv r_cond, TCGv r_reg)
+                          TCGv r_reg)
 {
     unsigned int cond = GET_FIELD_SP(insn, 25, 27), a = (insn & (1 << 29));
     target_ulong target = dc->pc + offset;
@@ -1447,10 +1517,10 @@ static void do_branch_reg(DisasContext *dc, int32_t offset, uint32_t insn,
     if (unlikely(AM_CHECK(dc))) {
         target &= 0xffffffffULL;
     }
-    flush_cond(dc, r_cond);
-    gen_cond_reg(r_cond, cond, r_reg);
+    flush_cond(dc);
+    gen_cond_reg(cpu_cond, cond, r_reg);
     if (a) {
-        gen_branch_a(dc, target, dc->npc, r_cond);
+        gen_branch_a(dc, target, dc->npc, cpu_cond);
         dc->is_br = 1;
     } else {
         dc->pc = dc->npc;
@@ -1617,13 +1687,13 @@ static inline void gen_op_fpexception_im(int fsr_flags)
     tcg_temp_free_i32(r_const);
 }
 
-static int gen_trap_ifnofpu(DisasContext *dc, TCGv r_cond)
+static int gen_trap_ifnofpu(DisasContext *dc)
 {
 #if !defined(CONFIG_USER_ONLY)
     if (!dc->fpu_enabled) {
         TCGv_i32 r_const;
 
-        save_state(dc, r_cond);
+        save_state(dc);
         r_const = tcg_const_i32(TT_NFPU_INSN);
         gen_helper_raise_exception(cpu_env, r_const);
         tcg_temp_free_i32(r_const);
@@ -1645,7 +1715,7 @@ static inline void gen_fop_FF(DisasContext *dc, int rd, int rs,
     TCGv_i32 dst, src;
 
     src = gen_load_fpr_F(dc, rs);
-    dst = gen_dest_fpr_F();
+    dst = gen_dest_fpr_F(dc);
 
     gen(dst, cpu_env, src);
 
@@ -1658,7 +1728,7 @@ static inline void gen_ne_fop_FF(DisasContext *dc, int rd, int rs,
     TCGv_i32 dst, src;
 
     src = gen_load_fpr_F(dc, rs);
-    dst = gen_dest_fpr_F();
+    dst = gen_dest_fpr_F(dc);
 
     gen(dst, src);
 
@@ -1672,7 +1742,7 @@ static inline void gen_fop_FFF(DisasContext *dc, int rd, int rs1, int rs2,
 
     src1 = gen_load_fpr_F(dc, rs1);
     src2 = gen_load_fpr_F(dc, rs2);
-    dst = gen_dest_fpr_F();
+    dst = gen_dest_fpr_F(dc);
 
     gen(dst, cpu_env, src1, src2);
 
@@ -1687,7 +1757,7 @@ static inline void gen_ne_fop_FFF(DisasContext *dc, int rd, int rs1, int rs2,
 
     src1 = gen_load_fpr_F(dc, rs1);
     src2 = gen_load_fpr_F(dc, rs2);
-    dst = gen_dest_fpr_F();
+    dst = gen_dest_fpr_F(dc);
 
     gen(dst, src1, src2);
 
@@ -1701,7 +1771,7 @@ static inline void gen_fop_DD(DisasContext *dc, int rd, int rs,
     TCGv_i64 dst, src;
 
     src = gen_load_fpr_D(dc, rs);
-    dst = gen_dest_fpr_D();
+    dst = gen_dest_fpr_D(dc, rd);
 
     gen(dst, cpu_env, src);
 
@@ -1715,7 +1785,7 @@ static inline void gen_ne_fop_DD(DisasContext *dc, int rd, int rs,
     TCGv_i64 dst, src;
 
     src = gen_load_fpr_D(dc, rs);
-    dst = gen_dest_fpr_D();
+    dst = gen_dest_fpr_D(dc, rd);
 
     gen(dst, src);
 
@@ -1730,7 +1800,7 @@ static inline void gen_fop_DDD(DisasContext *dc, int rd, int rs1, int rs2,
 
     src1 = gen_load_fpr_D(dc, rs1);
     src2 = gen_load_fpr_D(dc, rs2);
-    dst = gen_dest_fpr_D();
+    dst = gen_dest_fpr_D(dc, rd);
 
     gen(dst, cpu_env, src1, src2);
 
@@ -1745,7 +1815,7 @@ static inline void gen_ne_fop_DDD(DisasContext *dc, int rd, int rs1, int rs2,
 
     src1 = gen_load_fpr_D(dc, rs1);
     src2 = gen_load_fpr_D(dc, rs2);
-    dst = gen_dest_fpr_D();
+    dst = gen_dest_fpr_D(dc, rd);
 
     gen(dst, src1, src2);
 
@@ -1759,7 +1829,7 @@ static inline void gen_gsr_fop_DDD(DisasContext *dc, int rd, int rs1, int rs2,
 
     src1 = gen_load_fpr_D(dc, rs1);
     src2 = gen_load_fpr_D(dc, rs2);
-    dst = gen_dest_fpr_D();
+    dst = gen_dest_fpr_D(dc, rd);
 
     gen(dst, cpu_gsr, src1, src2);
 
@@ -1774,7 +1844,7 @@ static inline void gen_ne_fop_DDDD(DisasContext *dc, int rd, int rs1, int rs2,
     src1 = gen_load_fpr_D(dc, rs1);
     src2 = gen_load_fpr_D(dc, rs2);
     src0 = gen_load_fpr_D(dc, rd);
-    dst = gen_dest_fpr_D();
+    dst = gen_dest_fpr_D(dc, rd);
 
     gen(dst, src0, src1, src2);
 
@@ -1826,7 +1896,7 @@ static inline void gen_fop_DFF(DisasContext *dc, int rd, int rs1, int rs2,
 
     src1 = gen_load_fpr_F(dc, rs1);
     src2 = gen_load_fpr_F(dc, rs2);
-    dst = gen_dest_fpr_D();
+    dst = gen_dest_fpr_D(dc, rd);
 
     gen(dst, cpu_env, src1, src2);
 
@@ -1855,7 +1925,7 @@ static inline void gen_fop_DF(DisasContext *dc, int rd, int rs,
     TCGv_i32 src;
 
     src = gen_load_fpr_F(dc, rs);
-    dst = gen_dest_fpr_D();
+    dst = gen_dest_fpr_D(dc, rd);
 
     gen(dst, cpu_env, src);
 
@@ -1870,7 +1940,7 @@ static inline void gen_ne_fop_DF(DisasContext *dc, int rd, int rs,
     TCGv_i32 src;
 
     src = gen_load_fpr_F(dc, rs);
-    dst = gen_dest_fpr_D();
+    dst = gen_dest_fpr_D(dc, rd);
 
     gen(dst, cpu_env, src);
 
@@ -1884,7 +1954,7 @@ static inline void gen_fop_FD(DisasContext *dc, int rd, int rs,
     TCGv_i64 src;
 
     src = gen_load_fpr_D(dc, rs);
-    dst = gen_dest_fpr_F();
+    dst = gen_dest_fpr_F(dc);
 
     gen(dst, cpu_env, src);
 
@@ -1897,7 +1967,7 @@ static inline void gen_fop_FQ(DisasContext *dc, int rd, int rs,
     TCGv_i32 dst;
 
     gen_op_load_fpr_QT1(QFPREG(rs));
-    dst = gen_dest_fpr_F();
+    dst = gen_dest_fpr_F(dc);
 
     gen(dst, cpu_env);
 
@@ -1910,7 +1980,7 @@ static inline void gen_fop_DQ(DisasContext *dc, int rd, int rs,
     TCGv_i64 dst;
 
     gen_op_load_fpr_QT1(QFPREG(rs));
-    dst = gen_dest_fpr_D();
+    dst = gen_dest_fpr_D(dc, rd);
 
     gen(dst, cpu_env);
 
@@ -2011,22 +2081,25 @@ static inline void gen_stf_asi(TCGv addr, int insn, int size, int rd)
     tcg_temp_free_i32(r_asi);
 }
 
-static inline void gen_swap_asi(TCGv dst, TCGv addr, int insn)
+static inline void gen_swap_asi(TCGv dst, TCGv src, TCGv addr, int insn)
 {
     TCGv_i32 r_asi, r_size, r_sign;
+    TCGv_i64 t64 = tcg_temp_new_i64();
 
     r_asi = gen_get_asi(insn, addr);
     r_size = tcg_const_i32(4);
     r_sign = tcg_const_i32(0);
-    gen_helper_ld_asi(cpu_tmp64, cpu_env, addr, r_asi, r_size, r_sign);
+    gen_helper_ld_asi(t64, cpu_env, addr, r_asi, r_size, r_sign);
     tcg_temp_free_i32(r_sign);
-    gen_helper_st_asi(cpu_env, addr, dst, r_asi, r_size);
+    gen_helper_st_asi(cpu_env, addr, src, r_asi, r_size);
     tcg_temp_free_i32(r_size);
     tcg_temp_free_i32(r_asi);
-    tcg_gen_trunc_i64_tl(dst, cpu_tmp64);
+    tcg_gen_trunc_i64_tl(dst, t64);
+    tcg_temp_free_i64(t64);
 }
 
-static inline void gen_ldda_asi(TCGv hi, TCGv addr, int insn, int rd)
+static inline void gen_ldda_asi(DisasContext *dc, TCGv hi, TCGv addr,
+                                int insn, int rd)
 {
     TCGv_i32 r_asi, r_rd;
 
@@ -2037,42 +2110,44 @@ static inline void gen_ldda_asi(TCGv hi, TCGv addr, int insn, int rd)
     tcg_temp_free_i32(r_asi);
 }
 
-static inline void gen_stda_asi(TCGv hi, TCGv addr, int insn, int rd)
+static inline void gen_stda_asi(DisasContext *dc, TCGv hi, TCGv addr,
+                                int insn, int rd)
 {
     TCGv_i32 r_asi, r_size;
+    TCGv lo = gen_load_gpr(dc, rd + 1);
+    TCGv_i64 t64 = tcg_temp_new_i64();
 
-    gen_movl_reg_TN(rd + 1, cpu_tmp0);
-    tcg_gen_concat_tl_i64(cpu_tmp64, cpu_tmp0, hi);
+    tcg_gen_concat_tl_i64(t64, lo, hi);
     r_asi = gen_get_asi(insn, addr);
     r_size = tcg_const_i32(8);
-    gen_helper_st_asi(cpu_env, addr, cpu_tmp64, r_asi, r_size);
+    gen_helper_st_asi(cpu_env, addr, t64, r_asi, r_size);
     tcg_temp_free_i32(r_size);
     tcg_temp_free_i32(r_asi);
+    tcg_temp_free_i64(t64);
 }
 
-static inline void gen_cas_asi(TCGv dst, TCGv addr, TCGv val2, int insn,
-                               int rd)
+static inline void gen_cas_asi(DisasContext *dc, TCGv addr,
+                               TCGv val2, int insn, int rd)
 {
-    TCGv r_val1;
-    TCGv_i32 r_asi;
+    TCGv val1 = gen_load_gpr(dc, rd);
+    TCGv dst = gen_dest_gpr(dc, rd);
+    TCGv_i32 r_asi = gen_get_asi(insn, addr);
 
-    r_val1 = tcg_temp_new();
-    gen_movl_reg_TN(rd, r_val1);
-    r_asi = gen_get_asi(insn, addr);
-    gen_helper_cas_asi(dst, cpu_env, addr, r_val1, val2, r_asi);
+    gen_helper_cas_asi(dst, cpu_env, addr, val1, val2, r_asi);
     tcg_temp_free_i32(r_asi);
-    tcg_temp_free(r_val1);
+    gen_store_gpr(dc, rd, dst);
 }
 
-static inline void gen_casx_asi(TCGv dst, TCGv addr, TCGv val2, int insn,
-                                int rd)
+static inline void gen_casx_asi(DisasContext *dc, TCGv addr,
+                                TCGv val2, int insn, int rd)
 {
-    TCGv_i32 r_asi;
+    TCGv val1 = gen_load_gpr(dc, rd);
+    TCGv dst = gen_dest_gpr(dc, rd);
+    TCGv_i32 r_asi = gen_get_asi(insn, addr);
 
-    gen_movl_reg_TN(rd, cpu_tmp64);
-    r_asi = gen_get_asi(insn, addr);
-    gen_helper_casx_asi(dst, cpu_env, addr, cpu_tmp64, val2, r_asi);
+    gen_helper_casx_asi(dst, cpu_env, addr, val1, val2, r_asi);
     tcg_temp_free_i32(r_asi);
+    gen_store_gpr(dc, rd, dst);
 }
 
 #elif !defined(CONFIG_USER_ONLY)
@@ -2081,77 +2156,94 @@ static inline void gen_ld_asi(TCGv dst, TCGv addr, int insn, int size,
                               int sign)
 {
     TCGv_i32 r_asi, r_size, r_sign;
+    TCGv_i64 t64 = tcg_temp_new_i64();
 
     r_asi = tcg_const_i32(GET_FIELD(insn, 19, 26));
     r_size = tcg_const_i32(size);
     r_sign = tcg_const_i32(sign);
-    gen_helper_ld_asi(cpu_tmp64, cpu_env, addr, r_asi, r_size, r_sign);
-    tcg_temp_free(r_sign);
-    tcg_temp_free(r_size);
-    tcg_temp_free(r_asi);
-    tcg_gen_trunc_i64_tl(dst, cpu_tmp64);
+    gen_helper_ld_asi(t64, cpu_env, addr, r_asi, r_size, r_sign);
+    tcg_temp_free_i32(r_sign);
+    tcg_temp_free_i32(r_size);
+    tcg_temp_free_i32(r_asi);
+    tcg_gen_trunc_i64_tl(dst, t64);
+    tcg_temp_free_i64(t64);
 }
 
 static inline void gen_st_asi(TCGv src, TCGv addr, int insn, int size)
 {
     TCGv_i32 r_asi, r_size;
+    TCGv_i64 t64 = tcg_temp_new_i64();
 
-    tcg_gen_extu_tl_i64(cpu_tmp64, src);
+    tcg_gen_extu_tl_i64(t64, src);
     r_asi = tcg_const_i32(GET_FIELD(insn, 19, 26));
     r_size = tcg_const_i32(size);
-    gen_helper_st_asi(cpu_env, addr, cpu_tmp64, r_asi, r_size);
-    tcg_temp_free(r_size);
-    tcg_temp_free(r_asi);
+    gen_helper_st_asi(cpu_env, addr, t64, r_asi, r_size);
+    tcg_temp_free_i32(r_size);
+    tcg_temp_free_i32(r_asi);
+    tcg_temp_free_i64(t64);
 }
 
-static inline void gen_swap_asi(TCGv dst, TCGv addr, int insn)
+static inline void gen_swap_asi(TCGv dst, TCGv src, TCGv addr, int insn)
 {
     TCGv_i32 r_asi, r_size, r_sign;
-    TCGv_i64 r_val;
+    TCGv_i64 r_val, t64;
 
     r_asi = tcg_const_i32(GET_FIELD(insn, 19, 26));
     r_size = tcg_const_i32(4);
     r_sign = tcg_const_i32(0);
-    gen_helper_ld_asi(cpu_tmp64, cpu_env, addr, r_asi, r_size, r_sign);
+    t64 = tcg_temp_new_i64();
+    gen_helper_ld_asi(t64, cpu_env, addr, r_asi, r_size, r_sign);
     tcg_temp_free(r_sign);
     r_val = tcg_temp_new_i64();
-    tcg_gen_extu_tl_i64(r_val, dst);
+    tcg_gen_extu_tl_i64(r_val, src);
     gen_helper_st_asi(cpu_env, addr, r_val, r_asi, r_size);
     tcg_temp_free_i64(r_val);
-    tcg_temp_free(r_size);
-    tcg_temp_free(r_asi);
-    tcg_gen_trunc_i64_tl(dst, cpu_tmp64);
+    tcg_temp_free_i32(r_size);
+    tcg_temp_free_i32(r_asi);
+    tcg_gen_trunc_i64_tl(dst, t64);
+    tcg_temp_free_i64(t64);
 }
 
-static inline void gen_ldda_asi(TCGv hi, TCGv addr, int insn, int rd)
+static inline void gen_ldda_asi(DisasContext *dc, TCGv hi, TCGv addr,
+                                int insn, int rd)
 {
     TCGv_i32 r_asi, r_size, r_sign;
+    TCGv t;
+    TCGv_i64 t64;
 
     r_asi = tcg_const_i32(GET_FIELD(insn, 19, 26));
     r_size = tcg_const_i32(8);
     r_sign = tcg_const_i32(0);
-    gen_helper_ld_asi(cpu_tmp64, cpu_env, addr, r_asi, r_size, r_sign);
-    tcg_temp_free(r_sign);
-    tcg_temp_free(r_size);
-    tcg_temp_free(r_asi);
-    tcg_gen_trunc_i64_tl(cpu_tmp0, cpu_tmp64);
-    gen_movl_TN_reg(rd + 1, cpu_tmp0);
-    tcg_gen_shri_i64(cpu_tmp64, cpu_tmp64, 32);
-    tcg_gen_trunc_i64_tl(hi, cpu_tmp64);
-    gen_movl_TN_reg(rd, hi);
+    t64 = tcg_temp_new_i64();
+    gen_helper_ld_asi(t64, cpu_env, addr, r_asi, r_size, r_sign);
+    tcg_temp_free_i32(r_sign);
+    tcg_temp_free_i32(r_size);
+    tcg_temp_free_i32(r_asi);
+
+    t = gen_dest_gpr(dc, rd + 1);
+    tcg_gen_trunc_i64_tl(t, t64);
+    gen_store_gpr(dc, rd + 1, t);
+
+    tcg_gen_shri_i64(t64, t64, 32);
+    tcg_gen_trunc_i64_tl(hi, t64);
+    tcg_temp_free_i64(t64);
+    gen_store_gpr(dc, rd, hi);
 }
 
-static inline void gen_stda_asi(TCGv hi, TCGv addr, int insn, int rd)
+static inline void gen_stda_asi(DisasContext *dc, TCGv hi, TCGv addr,
+                                int insn, int rd)
 {
     TCGv_i32 r_asi, r_size;
+    TCGv lo = gen_load_gpr(dc, rd + 1);
+    TCGv_i64 t64 = tcg_temp_new_i64();
 
-    gen_movl_reg_TN(rd + 1, cpu_tmp0);
-    tcg_gen_concat_tl_i64(cpu_tmp64, cpu_tmp0, hi);
+    tcg_gen_concat_tl_i64(t64, lo, hi);
     r_asi = tcg_const_i32(GET_FIELD(insn, 19, 26));
     r_size = tcg_const_i32(8);
-    gen_helper_st_asi(cpu_env, addr, cpu_tmp64, r_asi, r_size);
-    tcg_temp_free(r_size);
-    tcg_temp_free(r_asi);
+    gen_helper_st_asi(cpu_env, addr, t64, r_asi, r_size);
+    tcg_temp_free_i32(r_size);
+    tcg_temp_free_i32(r_asi);
+    tcg_temp_free_i64(t64);
 }
 #endif
 
@@ -2173,43 +2265,77 @@ static inline void gen_ldstub_asi(TCGv dst, TCGv addr, int insn)
 }
 #endif
 
-static inline TCGv get_src1(unsigned int insn, TCGv def)
+static TCGv get_src1(DisasContext *dc, unsigned int insn)
 {
-    TCGv r_rs1 = def;
-    unsigned int rs1;
-
-    rs1 = GET_FIELD(insn, 13, 17);
-    if (rs1 == 0) {
-        tcg_gen_movi_tl(def, 0);
-    } else if (rs1 < 8) {
-        r_rs1 = cpu_gregs[rs1];
-    } else {
-        tcg_gen_ld_tl(def, cpu_regwptr, (rs1 - 8) * sizeof(target_ulong));
-    }
-    return r_rs1;
+    unsigned int rs1 = GET_FIELD(insn, 13, 17);
+    return gen_load_gpr(dc, rs1);
 }
 
-static inline TCGv get_src2(unsigned int insn, TCGv def)
+static TCGv get_src2(DisasContext *dc, unsigned int insn)
 {
-    TCGv r_rs2 = def;
-
     if (IS_IMM) { /* immediate */
         target_long simm = GET_FIELDs(insn, 19, 31);
-        tcg_gen_movi_tl(def, simm);
-    } else { /* register */
+        TCGv t = get_temp_tl(dc);
+        tcg_gen_movi_tl(t, simm);
+        return t;
+    } else {      /* register */
         unsigned int rs2 = GET_FIELD(insn, 27, 31);
-        if (rs2 == 0) {
-            tcg_gen_movi_tl(def, 0);
-        } else if (rs2 < 8) {
-            r_rs2 = cpu_gregs[rs2];
-        } else {
-            tcg_gen_ld_tl(def, cpu_regwptr, (rs2 - 8) * sizeof(target_ulong));
-        }
+        return gen_load_gpr(dc, rs2);
     }
-    return r_rs2;
 }
 
 #ifdef TARGET_SPARC64
+static void gen_fmovs(DisasContext *dc, DisasCompare *cmp, int rd, int rs)
+{
+    TCGv_i32 c32, zero, dst, s1, s2;
+
+    /* We have two choices here: extend the 32 bit data and use movcond_i64,
+       or fold the comparison down to 32 bits and use movcond_i32.  Choose
+       the later.  */
+    c32 = tcg_temp_new_i32();
+    if (cmp->is_bool) {
+        tcg_gen_trunc_i64_i32(c32, cmp->c1);
+    } else {
+        TCGv_i64 c64 = tcg_temp_new_i64();
+        tcg_gen_setcond_i64(cmp->cond, c64, cmp->c1, cmp->c2);
+        tcg_gen_trunc_i64_i32(c32, c64);
+        tcg_temp_free_i64(c64);
+    }
+
+    s1 = gen_load_fpr_F(dc, rs);
+    s2 = gen_load_fpr_F(dc, rd);
+    dst = gen_dest_fpr_F(dc);
+    zero = tcg_const_i32(0);
+
+    tcg_gen_movcond_i32(TCG_COND_NE, dst, c32, zero, s1, s2);
+
+    tcg_temp_free_i32(c32);
+    tcg_temp_free_i32(zero);
+    gen_store_fpr_F(dc, rd, dst);
+}
+
+static void gen_fmovd(DisasContext *dc, DisasCompare *cmp, int rd, int rs)
+{
+    TCGv_i64 dst = gen_dest_fpr_D(dc, rd);
+    tcg_gen_movcond_i64(cmp->cond, dst, cmp->c1, cmp->c2,
+                        gen_load_fpr_D(dc, rs),
+                        gen_load_fpr_D(dc, rd));
+    gen_store_fpr_D(dc, rd, dst);
+}
+
+static void gen_fmovq(DisasContext *dc, DisasCompare *cmp, int rd, int rs)
+{
+    int qd = QFPREG(rd);
+    int qs = QFPREG(rs);
+
+    tcg_gen_movcond_i64(cmp->cond, cpu_fpr[qd / 2], cmp->c1, cmp->c2,
+                        cpu_fpr[qs / 2], cpu_fpr[qd / 2]);
+    tcg_gen_movcond_i64(cmp->cond, cpu_fpr[qd / 2 + 1], cmp->c1, cmp->c2,
+                        cpu_fpr[qs / 2 + 1], cpu_fpr[qd / 2 + 1]);
+
+    gen_update_fprs_dirty(qd);
+}
+
 static inline void gen_load_trap_state_at_tl(TCGv_ptr r_tsptr, TCGv_ptr cpu_env)
 {
     TCGv_i32 r_tl = tcg_temp_new_i32();
@@ -2389,21 +2515,18 @@ static void gen_faligndata(TCGv dst, TCGv gsr, TCGv s1, TCGv s2)
 static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
 {
     unsigned int opc, rs1, rs2, rd;
-    TCGv cpu_src1, cpu_src2, cpu_tmp1, cpu_tmp2;
+    TCGv cpu_src1, cpu_src2;
     TCGv_i32 cpu_src1_32, cpu_src2_32, cpu_dst_32;
     TCGv_i64 cpu_src1_64, cpu_src2_64, cpu_dst_64;
     target_long simm;
 
-    if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP)))
+    if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
         tcg_gen_debug_insn_start(dc->pc);
+    }
 
     opc = GET_FIELD(insn, 0, 1);
-
     rd = GET_FIELD(insn, 2, 6);
 
-    cpu_tmp1 = cpu_src1 = tcg_temp_new();
-    cpu_tmp2 = cpu_src2 = tcg_temp_new();
-
     switch (opc) {
     case 0:                     /* branches/sethi */
         {
@@ -2420,9 +2543,9 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                     target <<= 2;
                     cc = GET_FIELD_SP(insn, 20, 21);
                     if (cc == 0)
-                        do_branch(dc, target, insn, 0, cpu_cond);
+                        do_branch(dc, target, insn, 0);
                     else if (cc == 2)
-                        do_branch(dc, target, insn, 1, cpu_cond);
+                        do_branch(dc, target, insn, 1);
                     else
                         goto illegal_insn;
                     goto jmp_insn;
@@ -2433,19 +2556,20 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                         (GET_FIELD_SP(insn, 20, 21) << 14);
                     target = sign_extend(target, 16);
                     target <<= 2;
-                    cpu_src1 = get_src1(insn, cpu_src1);
-                    do_branch_reg(dc, target, insn, cpu_cond, cpu_src1);
+                    cpu_src1 = get_src1(dc, insn);
+                    do_branch_reg(dc, target, insn, cpu_src1);
                     goto jmp_insn;
                 }
             case 0x5:           /* V9 FBPcc */
                 {
                     int cc = GET_FIELD_SP(insn, 20, 21);
-                    if (gen_trap_ifnofpu(dc, cpu_cond))
+                    if (gen_trap_ifnofpu(dc)) {
                         goto jmp_insn;
+                    }
                     target = GET_FIELD_SP(insn, 0, 18);
                     target = sign_extend(target, 19);
                     target <<= 2;
-                    do_fbranch(dc, target, insn, cc, cpu_cond);
+                    do_fbranch(dc, target, insn, cc);
                     goto jmp_insn;
                 }
 #else
@@ -2459,27 +2583,27 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                     target = GET_FIELD(insn, 10, 31);
                     target = sign_extend(target, 22);
                     target <<= 2;
-                    do_branch(dc, target, insn, 0, cpu_cond);
+                    do_branch(dc, target, insn, 0);
                     goto jmp_insn;
                 }
             case 0x6:           /* FBN+x */
                 {
-                    if (gen_trap_ifnofpu(dc, cpu_cond))
+                    if (gen_trap_ifnofpu(dc)) {
                         goto jmp_insn;
+                    }
                     target = GET_FIELD(insn, 10, 31);
                     target = sign_extend(target, 22);
                     target <<= 2;
-                    do_fbranch(dc, target, insn, 0, cpu_cond);
+                    do_fbranch(dc, target, insn, 0);
                     goto jmp_insn;
                 }
             case 0x4:           /* SETHI */
-                if (rd) { // nop
+                /* Special-case %g0 because that's the canonical nop.  */
+                if (rd) {
                     uint32_t value = GET_FIELD(insn, 10, 31);
-                    TCGv r_const;
-
-                    r_const = tcg_const_tl(value << 10);
-                    gen_movl_TN_reg(rd, r_const);
-                    tcg_temp_free(r_const);
+                    TCGv t = gen_dest_gpr(dc, rd);
+                    tcg_gen_movi_tl(t, value << 10);
+                    gen_store_gpr(dc, rd, t);
                 }
                 break;
             case 0x0:           /* UNIMPL */
@@ -2492,13 +2616,12 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
     case 1:                     /*CALL*/
         {
             target_long target = GET_FIELDs(insn, 2, 31) << 2;
-            TCGv r_const;
+            TCGv o7 = gen_dest_gpr(dc, 15);
 
-            r_const = tcg_const_tl(dc->pc);
-            gen_movl_TN_reg(15, r_const);
-            tcg_temp_free(r_const);
+            tcg_gen_movi_tl(o7, dc->pc);
+            gen_store_gpr(dc, 15, o7);
             target += dc->pc;
-            gen_mov_pc_npc(dc, cpu_cond);
+            gen_mov_pc_npc(dc);
 #ifdef TARGET_SPARC64
             if (unlikely(AM_CHECK(dc))) {
                 target &= 0xffffffffULL;
@@ -2510,71 +2633,88 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
     case 2:                     /* FPU & Logical Operations */
         {
             unsigned int xop = GET_FIELD(insn, 7, 12);
+            TCGv cpu_dst = get_temp_tl(dc);
+            TCGv cpu_tmp0;
+
             if (xop == 0x3a) {  /* generate trap */
-                int cond;
+                int cond = GET_FIELD(insn, 3, 6);
+                TCGv_i32 trap;
+                int l1 = -1, mask;
 
-                cpu_src1 = get_src1(insn, cpu_src1);
-                if (IS_IMM) {
-                    rs2 = GET_FIELD(insn, 25, 31);
-                    tcg_gen_addi_tl(cpu_dst, cpu_src1, rs2);
-                } else {
-                    rs2 = GET_FIELD(insn, 27, 31);
-                    if (rs2 != 0) {
-                        gen_movl_reg_TN(rs2, cpu_src2);
-                        tcg_gen_add_tl(cpu_dst, cpu_src1, cpu_src2);
-                    } else
-                        tcg_gen_mov_tl(cpu_dst, cpu_src1);
+                if (cond == 0) {
+                    /* Trap never.  */
+                    break;
                 }
 
-                cond = GET_FIELD(insn, 3, 6);
-                if (cond == 0x8) { /* Trap Always */
-                    save_state(dc, cpu_cond);
-                    if ((dc->def->features & CPU_FEATURE_HYPV) &&
-                        supervisor(dc))
-                        tcg_gen_andi_tl(cpu_dst, cpu_dst, UA2005_HTRAP_MASK);
-                    else
-                        tcg_gen_andi_tl(cpu_dst, cpu_dst, V8_TRAP_MASK);
-                    tcg_gen_addi_tl(cpu_dst, cpu_dst, TT_TRAP);
-                    tcg_gen_trunc_tl_i32(cpu_tmp32, cpu_dst);
-                    gen_helper_raise_exception(cpu_env, cpu_tmp32);
-
-                } else if (cond != 0) {
-                    TCGv r_cond = tcg_temp_new();
-                    int l1;
+                save_state(dc);
+
+                if (cond != 8) {
+                    /* Conditional trap.  */
+                    DisasCompare cmp;
 #ifdef TARGET_SPARC64
                     /* V9 icc/xcc */
                     int cc = GET_FIELD_SP(insn, 11, 12);
-
-                    save_state(dc, cpu_cond);
-                    if (cc == 0)
-                        gen_cond(r_cond, 0, cond, dc);
-                    else if (cc == 2)
-                        gen_cond(r_cond, 1, cond, dc);
-                    else
+                    if (cc == 0) {
+                        gen_compare(&cmp, 0, cond, dc);
+                    } else if (cc == 2) {
+                        gen_compare(&cmp, 1, cond, dc);
+                    } else {
                         goto illegal_insn;
+                    }
 #else
-                    save_state(dc, cpu_cond);
-                    gen_cond(r_cond, 0, cond, dc);
+                    gen_compare(&cmp, 0, cond, dc);
 #endif
                     l1 = gen_new_label();
-                    tcg_gen_brcondi_tl(TCG_COND_EQ, r_cond, 0, l1);
+                    tcg_gen_brcond_tl(tcg_invert_cond(cmp.cond),
+                                      cmp.c1, cmp.c2, l1);
+                    free_compare(&cmp);
+                }
 
-                    if ((dc->def->features & CPU_FEATURE_HYPV) &&
-                        supervisor(dc))
-                        tcg_gen_andi_tl(cpu_dst, cpu_dst, UA2005_HTRAP_MASK);
-                    else
-                        tcg_gen_andi_tl(cpu_dst, cpu_dst, V8_TRAP_MASK);
-                    tcg_gen_addi_tl(cpu_dst, cpu_dst, TT_TRAP);
-                    tcg_gen_trunc_tl_i32(cpu_tmp32, cpu_dst);
-                    gen_helper_raise_exception(cpu_env, cpu_tmp32);
+                mask = ((dc->def->features & CPU_FEATURE_HYPV) && supervisor(dc)
+                        ? UA2005_HTRAP_MASK : V8_TRAP_MASK);
 
+                /* Don't use the normal temporaries, as they may well have
+                   gone out of scope with the branch above.  While we're
+                   doing that we might as well pre-truncate to 32-bit.  */
+                trap = tcg_temp_new_i32();
+
+                rs1 = GET_FIELD_SP(insn, 14, 18);
+                if (IS_IMM) {
+                    rs2 = GET_FIELD_SP(insn, 0, 6);
+                    if (rs1 == 0) {
+                        tcg_gen_movi_i32(trap, (rs2 & mask) + TT_TRAP);
+                        /* Signal that the trap value is fully constant.  */
+                        mask = 0;
+                    } else {
+                        TCGv t1 = gen_load_gpr(dc, rs1);
+                        tcg_gen_trunc_tl_i32(trap, t1);
+                        tcg_gen_addi_i32(trap, trap, rs2);
+                    }
+                } else {
+                    TCGv t1, t2;
+                    rs2 = GET_FIELD_SP(insn, 0, 4);
+                    t1 = gen_load_gpr(dc, rs1);
+                    t2 = gen_load_gpr(dc, rs2);
+                    tcg_gen_add_tl(t1, t1, t2);
+                    tcg_gen_trunc_tl_i32(trap, t1);
+                }
+                if (mask != 0) {
+                    tcg_gen_andi_i32(trap, trap, mask);
+                    tcg_gen_addi_i32(trap, trap, TT_TRAP);
+                }
+
+                gen_helper_raise_exception(cpu_env, trap);
+                tcg_temp_free_i32(trap);
+
+                if (cond == 8) {
+                    /* An unconditional trap ends the TB.  */
+                    dc->is_br = 1;
+                    goto jmp_insn;
+                } else {
+                    /* A conditional trap falls through to the next insn.  */
                     gen_set_label(l1);
-                    tcg_temp_free(r_cond);
+                    break;
                 }
-                gen_op_next_insn();
-                tcg_gen_exit_tb(0);
-                dc->is_br = 1;
-                goto jmp_insn;
             } else if (xop == 0x28) {
                 rs1 = GET_FIELD(insn, 13, 17);
                 switch(rs1) {
@@ -2590,27 +2730,24 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                                        microSPARC II */
                     /* Read Asr17 */
                     if (rs1 == 0x11 && dc->def->features & CPU_FEATURE_ASR17) {
-                        TCGv r_const;
-
+                        TCGv t = gen_dest_gpr(dc, rd);
                         /* Read Asr17 for a Leon3 monoprocessor */
-                        r_const = tcg_const_tl((1 << 8)
-                                               | (dc->def->nwindows - 1));
-                        gen_movl_TN_reg(rd, r_const);
-                        tcg_temp_free(r_const);
+                        tcg_gen_movi_tl(t, (1 << 8) | (dc->def->nwindows - 1));
+                        gen_store_gpr(dc, rd, t);
                         break;
                     }
 #endif
-                    gen_movl_TN_reg(rd, cpu_y);
+                    gen_store_gpr(dc, rd, cpu_y);
                     break;
 #ifdef TARGET_SPARC64
                 case 0x2: /* V9 rdccr */
-                    gen_helper_compute_psr(cpu_env);
+                    update_psr(dc);
                     gen_helper_rdccr(cpu_dst, cpu_env);
-                    gen_movl_TN_reg(rd, cpu_dst);
+                    gen_store_gpr(dc, rd, cpu_dst);
                     break;
                 case 0x3: /* V9 rdasi */
                     tcg_gen_ext_i32_tl(cpu_dst, cpu_asi);
-                    gen_movl_TN_reg(rd, cpu_dst);
+                    gen_store_gpr(dc, rd, cpu_dst);
                     break;
                 case 0x4: /* V9 rdtick */
                     {
@@ -2621,39 +2758,38 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                                        offsetof(CPUSPARCState, tick));
                         gen_helper_tick_get_count(cpu_dst, r_tickptr);
                         tcg_temp_free_ptr(r_tickptr);
-                        gen_movl_TN_reg(rd, cpu_dst);
+                        gen_store_gpr(dc, rd, cpu_dst);
                     }
                     break;
                 case 0x5: /* V9 rdpc */
                     {
-                        TCGv r_const;
-
+                        TCGv t = gen_dest_gpr(dc, rd);
                         if (unlikely(AM_CHECK(dc))) {
-                            r_const = tcg_const_tl(dc->pc & 0xffffffffULL);
+                            tcg_gen_movi_tl(t, dc->pc & 0xffffffffULL);
                         } else {
-                           r_const = tcg_const_tl(dc->pc);
+                            tcg_gen_movi_tl(t, dc->pc);
                         }
-                        gen_movl_TN_reg(rd, r_const);
-                        tcg_temp_free(r_const);
+                        gen_store_gpr(dc, rd, t);
                     }
                     break;
                 case 0x6: /* V9 rdfprs */
                     tcg_gen_ext_i32_tl(cpu_dst, cpu_fprs);
-                    gen_movl_TN_reg(rd, cpu_dst);
+                    gen_store_gpr(dc, rd, cpu_dst);
                     break;
                 case 0xf: /* V9 membar */
                     break; /* no effect */
                 case 0x13: /* Graphics Status */
-                    if (gen_trap_ifnofpu(dc, cpu_cond))
+                    if (gen_trap_ifnofpu(dc)) {
                         goto jmp_insn;
-                    gen_movl_TN_reg(rd, cpu_gsr);
+                    }
+                    gen_store_gpr(dc, rd, cpu_gsr);
                     break;
                 case 0x16: /* Softint */
                     tcg_gen_ext_i32_tl(cpu_dst, cpu_softint);
-                    gen_movl_TN_reg(rd, cpu_dst);
+                    gen_store_gpr(dc, rd, cpu_dst);
                     break;
                 case 0x17: /* Tick compare */
-                    gen_movl_TN_reg(rd, cpu_tick_cmpr);
+                    gen_store_gpr(dc, rd, cpu_tick_cmpr);
                     break;
                 case 0x18: /* System tick */
                     {
@@ -2664,11 +2800,11 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                                        offsetof(CPUSPARCState, stick));
                         gen_helper_tick_get_count(cpu_dst, r_tickptr);
                         tcg_temp_free_ptr(r_tickptr);
-                        gen_movl_TN_reg(rd, cpu_dst);
+                        gen_store_gpr(dc, rd, cpu_dst);
                     }
                     break;
                 case 0x19: /* System tick compare */
-                    gen_movl_TN_reg(rd, cpu_stick_cmpr);
+                    gen_store_gpr(dc, rd, cpu_stick_cmpr);
                     break;
                 case 0x10: /* Performance Control */
                 case 0x11: /* Performance Instrumentation Counter */
@@ -2682,10 +2818,10 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
 #if !defined(CONFIG_USER_ONLY)
             } else if (xop == 0x29) { /* rdpsr / UA2005 rdhpr */
 #ifndef TARGET_SPARC64
-                if (!supervisor(dc))
+                if (!supervisor(dc)) {
                     goto priv_insn;
-                gen_helper_compute_psr(cpu_env);
-                dc->cc_op = CC_OP_FLAGS;
+                }
+                update_psr(dc);
                 gen_helper_rdpsr(cpu_dst, cpu_env);
 #else
                 CHECK_IU_FEATURE(dc, HYPV);
@@ -2715,11 +2851,13 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                     goto illegal_insn;
                 }
 #endif
-                gen_movl_TN_reg(rd, cpu_dst);
+                gen_store_gpr(dc, rd, cpu_dst);
                 break;
             } else if (xop == 0x2a) { /* rdwim / V9 rdpr */
-                if (!supervisor(dc))
+                if (!supervisor(dc)) {
                     goto priv_insn;
+                }
+                cpu_tmp0 = get_temp_tl(dc);
 #ifdef TARGET_SPARC64
                 rs1 = GET_FIELD(insn, 13, 17);
                 switch (rs1) {
@@ -2758,14 +2896,12 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                     break;
                 case 3: // tt
                     {
-                        TCGv_ptr r_tsptr;
+                        TCGv_ptr r_tsptr = tcg_temp_new_ptr();
 
-                        r_tsptr = tcg_temp_new_ptr();
                         gen_load_trap_state_at_tl(r_tsptr, cpu_env);
-                        tcg_gen_ld_i32(cpu_tmp32, r_tsptr,
-                                       offsetof(trap_state, tt));
+                        tcg_gen_ld32s_tl(cpu_tmp0, r_tsptr,
+                                         offsetof(trap_state, tt));
                         tcg_temp_free_ptr(r_tsptr);
-                        tcg_gen_ext_i32_tl(cpu_tmp0, cpu_tmp32);
                     }
                     break;
                 case 4: // tick
@@ -2776,7 +2912,6 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                         tcg_gen_ld_ptr(r_tickptr, cpu_env,
                                        offsetof(CPUSPARCState, tick));
                         gen_helper_tick_get_count(cpu_tmp0, r_tickptr);
-                        gen_movl_TN_reg(rd, cpu_tmp0);
                         tcg_temp_free_ptr(r_tickptr);
                     }
                     break;
@@ -2784,53 +2919,44 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                     tcg_gen_mov_tl(cpu_tmp0, cpu_tbr);
                     break;
                 case 6: // pstate
-                    tcg_gen_ld_i32(cpu_tmp32, cpu_env,
-                                   offsetof(CPUSPARCState, pstate));
-                    tcg_gen_ext_i32_tl(cpu_tmp0, cpu_tmp32);
+                    tcg_gen_ld32s_tl(cpu_tmp0, cpu_env,
+                                     offsetof(CPUSPARCState, pstate));
                     break;
                 case 7: // tl
-                    tcg_gen_ld_i32(cpu_tmp32, cpu_env,
-                                   offsetof(CPUSPARCState, tl));
-                    tcg_gen_ext_i32_tl(cpu_tmp0, cpu_tmp32);
+                    tcg_gen_ld32s_tl(cpu_tmp0, cpu_env,
+                                     offsetof(CPUSPARCState, tl));
                     break;
                 case 8: // pil
-                    tcg_gen_ld_i32(cpu_tmp32, cpu_env,
-                                   offsetof(CPUSPARCState, psrpil));
-                    tcg_gen_ext_i32_tl(cpu_tmp0, cpu_tmp32);
+                    tcg_gen_ld32s_tl(cpu_tmp0, cpu_env,
+                                     offsetof(CPUSPARCState, psrpil));
                     break;
                 case 9: // cwp
                     gen_helper_rdcwp(cpu_tmp0, cpu_env);
                     break;
                 case 10: // cansave
-                    tcg_gen_ld_i32(cpu_tmp32, cpu_env,
-                                   offsetof(CPUSPARCState, cansave));
-                    tcg_gen_ext_i32_tl(cpu_tmp0, cpu_tmp32);
+                    tcg_gen_ld32s_tl(cpu_tmp0, cpu_env,
+                                     offsetof(CPUSPARCState, cansave));
                     break;
                 case 11: // canrestore
-                    tcg_gen_ld_i32(cpu_tmp32, cpu_env,
-                                   offsetof(CPUSPARCState, canrestore));
-                    tcg_gen_ext_i32_tl(cpu_tmp0, cpu_tmp32);
+                    tcg_gen_ld32s_tl(cpu_tmp0, cpu_env,
+                                     offsetof(CPUSPARCState, canrestore));
                     break;
                 case 12: // cleanwin
-                    tcg_gen_ld_i32(cpu_tmp32, cpu_env,
-                                   offsetof(CPUSPARCState, cleanwin));
-                    tcg_gen_ext_i32_tl(cpu_tmp0, cpu_tmp32);
+                    tcg_gen_ld32s_tl(cpu_tmp0, cpu_env,
+                                     offsetof(CPUSPARCState, cleanwin));
                     break;
                 case 13: // otherwin
-                    tcg_gen_ld_i32(cpu_tmp32, cpu_env,
-                                   offsetof(CPUSPARCState, otherwin));
-                    tcg_gen_ext_i32_tl(cpu_tmp0, cpu_tmp32);
+                    tcg_gen_ld32s_tl(cpu_tmp0, cpu_env,
+                                     offsetof(CPUSPARCState, otherwin));
                     break;
                 case 14: // wstate
-                    tcg_gen_ld_i32(cpu_tmp32, cpu_env,
-                                   offsetof(CPUSPARCState, wstate));
-                    tcg_gen_ext_i32_tl(cpu_tmp0, cpu_tmp32);
+                    tcg_gen_ld32s_tl(cpu_tmp0, cpu_env,
+                                     offsetof(CPUSPARCState, wstate));
                     break;
                 case 16: // UA2005 gl
                     CHECK_IU_FEATURE(dc, GL);
-                    tcg_gen_ld_i32(cpu_tmp32, cpu_env,
-                                   offsetof(CPUSPARCState, gl));
-                    tcg_gen_ext_i32_tl(cpu_tmp0, cpu_tmp32);
+                    tcg_gen_ld32s_tl(cpu_tmp0, cpu_env,
+                                     offsetof(CPUSPARCState, gl));
                     break;
                 case 26: // UA2005 strand status
                     CHECK_IU_FEATURE(dc, HYPV);
@@ -2848,27 +2974,28 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
 #else
                 tcg_gen_ext_i32_tl(cpu_tmp0, cpu_wim);
 #endif
-                gen_movl_TN_reg(rd, cpu_tmp0);
+                gen_store_gpr(dc, rd, cpu_tmp0);
                 break;
             } else if (xop == 0x2b) { /* rdtbr / V9 flushw */
 #ifdef TARGET_SPARC64
-                save_state(dc, cpu_cond);
+                save_state(dc);
                 gen_helper_flushw(cpu_env);
 #else
                 if (!supervisor(dc))
                     goto priv_insn;
-                gen_movl_TN_reg(rd, cpu_tbr);
+                gen_store_gpr(dc, rd, cpu_tbr);
 #endif
                 break;
 #endif
             } else if (xop == 0x34) {   /* FPU Operations */
-                if (gen_trap_ifnofpu(dc, cpu_cond))
+                if (gen_trap_ifnofpu(dc)) {
                     goto jmp_insn;
+                }
                 gen_op_clear_ieee_excp_and_FTT();
                 rs1 = GET_FIELD(insn, 13, 17);
                 rs2 = GET_FIELD(insn, 27, 31);
                 xop = GET_FIELD(insn, 18, 26);
-                save_state(dc, cpu_cond);
+                save_state(dc);
                 switch (xop) {
                 case 0x1: /* fmovs */
                     cpu_src1_32 = gen_load_fpr_F(dc, rs2);
@@ -3036,216 +3163,121 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
 #ifdef TARGET_SPARC64
                 int cond;
 #endif
-                if (gen_trap_ifnofpu(dc, cpu_cond))
+                if (gen_trap_ifnofpu(dc)) {
                     goto jmp_insn;
+                }
                 gen_op_clear_ieee_excp_and_FTT();
                 rs1 = GET_FIELD(insn, 13, 17);
                 rs2 = GET_FIELD(insn, 27, 31);
                 xop = GET_FIELD(insn, 18, 26);
-                save_state(dc, cpu_cond);
-#ifdef TARGET_SPARC64
-                if ((xop & 0x11f) == 0x005) { // V9 fmovsr
-                    int l1;
+                save_state(dc);
 
-                    l1 = gen_new_label();
-                    cond = GET_FIELD_SP(insn, 14, 17);
-                    cpu_src1 = get_src1(insn, cpu_src1);
-                    tcg_gen_brcondi_tl(gen_tcg_cond_reg[cond], cpu_src1,
-                                       0, l1);
-                    cpu_src1_32 = gen_load_fpr_F(dc, rs2);
-                    gen_store_fpr_F(dc, rd, cpu_src1_32);
-                    gen_set_label(l1);
+#ifdef TARGET_SPARC64
+#define FMOVR(sz)                                                  \
+                do {                                               \
+                    DisasCompare cmp;                              \
+                    cond = GET_FIELD_SP(insn, 10, 12);             \
+                    cpu_src1 = get_src1(dc, insn);                 \
+                    gen_compare_reg(&cmp, cond, cpu_src1);         \
+                    gen_fmov##sz(dc, &cmp, rd, rs2);               \
+                    free_compare(&cmp);                            \
+                } while (0)
+
+                if ((xop & 0x11f) == 0x005) { /* V9 fmovsr */
+                    FMOVR(s);
                     break;
                 } else if ((xop & 0x11f) == 0x006) { // V9 fmovdr
-                    int l1;
-
-                    l1 = gen_new_label();
-                    cond = GET_FIELD_SP(insn, 14, 17);
-                    cpu_src1 = get_src1(insn, cpu_src1);
-                    tcg_gen_brcondi_tl(gen_tcg_cond_reg[cond], cpu_src1,
-                                       0, l1);
-                    cpu_src1_64 = gen_load_fpr_D(dc, rs2);
-                    gen_store_fpr_D(dc, rd, cpu_src1_64);
-                    gen_set_label(l1);
+                    FMOVR(d);
                     break;
                 } else if ((xop & 0x11f) == 0x007) { // V9 fmovqr
-                    int l1;
-
                     CHECK_FPU_FEATURE(dc, FLOAT128);
-                    l1 = gen_new_label();
-                    cond = GET_FIELD_SP(insn, 14, 17);
-                    cpu_src1 = get_src1(insn, cpu_src1);
-                    tcg_gen_brcondi_tl(gen_tcg_cond_reg[cond], cpu_src1,
-                                       0, l1);
-                    gen_move_Q(rd, rs2);
-                    gen_set_label(l1);
+                    FMOVR(q);
                     break;
                 }
+#undef FMOVR
 #endif
                 switch (xop) {
 #ifdef TARGET_SPARC64
-#define FMOVSCC(fcc)                                                    \
-                    {                                                   \
-                        TCGv r_cond;                                    \
-                        int l1;                                         \
-                                                                        \
-                        l1 = gen_new_label();                           \
-                        r_cond = tcg_temp_new();                        \
+#define FMOVCC(fcc, sz)                                                 \
+                    do {                                                \
+                        DisasCompare cmp;                               \
                         cond = GET_FIELD_SP(insn, 14, 17);              \
-                        gen_fcond(r_cond, fcc, cond);                   \
-                        tcg_gen_brcondi_tl(TCG_COND_EQ, r_cond,         \
-                                           0, l1);                      \
-                        cpu_src1_32 = gen_load_fpr_F(dc, rs2);          \
-                        gen_store_fpr_F(dc, rd, cpu_src1_32);           \
-                        gen_set_label(l1);                              \
-                        tcg_temp_free(r_cond);                          \
-                    }
-#define FMOVDCC(fcc)                                                    \
-                    {                                                   \
-                        TCGv r_cond;                                    \
-                        int l1;                                         \
-                                                                        \
-                        l1 = gen_new_label();                           \
-                        r_cond = tcg_temp_new();                        \
-                        cond = GET_FIELD_SP(insn, 14, 17);              \
-                        gen_fcond(r_cond, fcc, cond);                   \
-                        tcg_gen_brcondi_tl(TCG_COND_EQ, r_cond,         \
-                                           0, l1);                      \
-                        cpu_src1_64 = gen_load_fpr_D(dc, rs2);          \
-                        gen_store_fpr_D(dc, rd, cpu_src1_64);           \
-                        gen_set_label(l1);                              \
-                        tcg_temp_free(r_cond);                          \
-                    }
-#define FMOVQCC(fcc)                                                    \
-                    {                                                   \
-                        TCGv r_cond;                                    \
-                        int l1;                                         \
-                                                                        \
-                        l1 = gen_new_label();                           \
-                        r_cond = tcg_temp_new();                        \
-                        cond = GET_FIELD_SP(insn, 14, 17);              \
-                        gen_fcond(r_cond, fcc, cond);                   \
-                        tcg_gen_brcondi_tl(TCG_COND_EQ, r_cond,         \
-                                           0, l1);                      \
-                        gen_move_Q(rd, rs2);                            \
-                        gen_set_label(l1);                              \
-                        tcg_temp_free(r_cond);                          \
-                    }
+                        gen_fcompare(&cmp, fcc, cond);                  \
+                        gen_fmov##sz(dc, &cmp, rd, rs2);                \
+                        free_compare(&cmp);                             \
+                    } while (0)
+
                     case 0x001: /* V9 fmovscc %fcc0 */
-                        FMOVSCC(0);
+                        FMOVCC(0, s);
                         break;
                     case 0x002: /* V9 fmovdcc %fcc0 */
-                        FMOVDCC(0);
+                        FMOVCC(0, d);
                         break;
                     case 0x003: /* V9 fmovqcc %fcc0 */
                         CHECK_FPU_FEATURE(dc, FLOAT128);
-                        FMOVQCC(0);
+                        FMOVCC(0, q);
                         break;
                     case 0x041: /* V9 fmovscc %fcc1 */
-                        FMOVSCC(1);
+                        FMOVCC(1, s);
                         break;
                     case 0x042: /* V9 fmovdcc %fcc1 */
-                        FMOVDCC(1);
+                        FMOVCC(1, d);
                         break;
                     case 0x043: /* V9 fmovqcc %fcc1 */
                         CHECK_FPU_FEATURE(dc, FLOAT128);
-                        FMOVQCC(1);
+                        FMOVCC(1, q);
                         break;
                     case 0x081: /* V9 fmovscc %fcc2 */
-                        FMOVSCC(2);
+                        FMOVCC(2, s);
                         break;
                     case 0x082: /* V9 fmovdcc %fcc2 */
-                        FMOVDCC(2);
+                        FMOVCC(2, d);
                         break;
                     case 0x083: /* V9 fmovqcc %fcc2 */
                         CHECK_FPU_FEATURE(dc, FLOAT128);
-                        FMOVQCC(2);
+                        FMOVCC(2, q);
                         break;
                     case 0x0c1: /* V9 fmovscc %fcc3 */
-                        FMOVSCC(3);
+                        FMOVCC(3, s);
                         break;
                     case 0x0c2: /* V9 fmovdcc %fcc3 */
-                        FMOVDCC(3);
+                        FMOVCC(3, d);
                         break;
                     case 0x0c3: /* V9 fmovqcc %fcc3 */
                         CHECK_FPU_FEATURE(dc, FLOAT128);
-                        FMOVQCC(3);
+                        FMOVCC(3, q);
                         break;
-#undef FMOVSCC
-#undef FMOVDCC
-#undef FMOVQCC
-#define FMOVSCC(icc)                                                    \
-                    {                                                   \
-                        TCGv r_cond;                                    \
-                        int l1;                                         \
-                                                                        \
-                        l1 = gen_new_label();                           \
-                        r_cond = tcg_temp_new();                        \
-                        cond = GET_FIELD_SP(insn, 14, 17);              \
-                        gen_cond(r_cond, icc, cond, dc);                \
-                        tcg_gen_brcondi_tl(TCG_COND_EQ, r_cond,         \
-                                           0, l1);                      \
-                        cpu_src1_32 = gen_load_fpr_F(dc, rs2);          \
-                        gen_store_fpr_F(dc, rd, cpu_src1_32);           \
-                        gen_set_label(l1);                              \
-                        tcg_temp_free(r_cond);                          \
-                    }
-#define FMOVDCC(icc)                                                    \
-                    {                                                   \
-                        TCGv r_cond;                                    \
-                        int l1;                                         \
-                                                                        \
-                        l1 = gen_new_label();                           \
-                        r_cond = tcg_temp_new();                        \
-                        cond = GET_FIELD_SP(insn, 14, 17);              \
-                        gen_cond(r_cond, icc, cond, dc);                \
-                        tcg_gen_brcondi_tl(TCG_COND_EQ, r_cond,         \
-                                           0, l1);                      \
-                        cpu_src1_64 = gen_load_fpr_D(dc, rs2);          \
-                        gen_store_fpr_D(dc, rd, cpu_src1_64);           \
-                        gen_update_fprs_dirty(DFPREG(rd));              \
-                        gen_set_label(l1);                              \
-                        tcg_temp_free(r_cond);                          \
-                    }
-#define FMOVQCC(icc)                                                    \
-                    {                                                   \
-                        TCGv r_cond;                                    \
-                        int l1;                                         \
-                                                                        \
-                        l1 = gen_new_label();                           \
-                        r_cond = tcg_temp_new();                        \
+#undef FMOVCC
+#define FMOVCC(xcc, sz)                                                 \
+                    do {                                                \
+                        DisasCompare cmp;                               \
                         cond = GET_FIELD_SP(insn, 14, 17);              \
-                        gen_cond(r_cond, icc, cond, dc);                \
-                        tcg_gen_brcondi_tl(TCG_COND_EQ, r_cond,         \
-                                           0, l1);                      \
-                        gen_move_Q(rd, rs2);                            \
-                        gen_set_label(l1);                              \
-                        tcg_temp_free(r_cond);                          \
-                    }
+                        gen_compare(&cmp, xcc, cond, dc);               \
+                        gen_fmov##sz(dc, &cmp, rd, rs2);                \
+                        free_compare(&cmp);                             \
+                    } while (0)
 
                     case 0x101: /* V9 fmovscc %icc */
-                        FMOVSCC(0);
+                        FMOVCC(0, s);
                         break;
                     case 0x102: /* V9 fmovdcc %icc */
-                        FMOVDCC(0);
+                        FMOVCC(0, d);
                         break;
                     case 0x103: /* V9 fmovqcc %icc */
                         CHECK_FPU_FEATURE(dc, FLOAT128);
-                        FMOVQCC(0);
+                        FMOVCC(0, q);
                         break;
                     case 0x181: /* V9 fmovscc %xcc */
-                        FMOVSCC(1);
+                        FMOVCC(1, s);
                         break;
                     case 0x182: /* V9 fmovdcc %xcc */
-                        FMOVDCC(1);
+                        FMOVCC(1, d);
                         break;
                     case 0x183: /* V9 fmovqcc %xcc */
                         CHECK_FPU_FEATURE(dc, FLOAT128);
-                        FMOVQCC(1);
+                        FMOVCC(1, q);
                         break;
-#undef FMOVSCC
-#undef FMOVDCC
-#undef FMOVQCC
+#undef FMOVCC
 #endif
                     case 0x51: /* fcmps, V9 %fcc */
                         cpu_src1_32 = gen_load_fpr_F(dc, rs1);
@@ -3283,43 +3315,45 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                         goto illegal_insn;
                 }
             } else if (xop == 0x2) {
-                // clr/mov shortcut
-
+                TCGv dst = gen_dest_gpr(dc, rd);
                 rs1 = GET_FIELD(insn, 13, 17);
                 if (rs1 == 0) {
-                    // or %g0, x, y -> mov T0, x; mov y, T0
+                    /* clr/mov shortcut : or %g0, x, y -> mov x, y */
                     if (IS_IMM) {       /* immediate */
-                        TCGv r_const;
-
                         simm = GET_FIELDs(insn, 19, 31);
-                        r_const = tcg_const_tl(simm);
-                        gen_movl_TN_reg(rd, r_const);
-                        tcg_temp_free(r_const);
+                        tcg_gen_movi_tl(dst, simm);
+                        gen_store_gpr(dc, rd, dst);
                     } else {            /* register */
                         rs2 = GET_FIELD(insn, 27, 31);
-                        gen_movl_reg_TN(rs2, cpu_dst);
-                        gen_movl_TN_reg(rd, cpu_dst);
+                        if (rs2 == 0) {
+                            tcg_gen_movi_tl(dst, 0);
+                            gen_store_gpr(dc, rd, dst);
+                        } else {
+                            cpu_src2 = gen_load_gpr(dc, rs2);
+                            gen_store_gpr(dc, rd, cpu_src2);
+                        }
                     }
                 } else {
-                    cpu_src1 = get_src1(insn, cpu_src1);
+                    cpu_src1 = get_src1(dc, insn);
                     if (IS_IMM) {       /* immediate */
                         simm = GET_FIELDs(insn, 19, 31);
-                        tcg_gen_ori_tl(cpu_dst, cpu_src1, simm);
-                        gen_movl_TN_reg(rd, cpu_dst);
+                        tcg_gen_ori_tl(dst, cpu_src1, simm);
+                        gen_store_gpr(dc, rd, dst);
                     } else {            /* register */
-                        // or x, %g0, y -> mov T1, x; mov y, T1
                         rs2 = GET_FIELD(insn, 27, 31);
-                        if (rs2 != 0) {
-                            gen_movl_reg_TN(rs2, cpu_src2);
-                            tcg_gen_or_tl(cpu_dst, cpu_src1, cpu_src2);
-                            gen_movl_TN_reg(rd, cpu_dst);
-                        } else
-                            gen_movl_TN_reg(rd, cpu_src1);
+                        if (rs2 == 0) {
+                            /* mov shortcut:  or x, %g0, y -> mov x, y */
+                            gen_store_gpr(dc, rd, cpu_src1);
+                        } else {
+                            cpu_src2 = gen_load_gpr(dc, rs2);
+                            tcg_gen_or_tl(dst, cpu_src1, cpu_src2);
+                            gen_store_gpr(dc, rd, dst);
+                        }
                     }
                 }
 #ifdef TARGET_SPARC64
             } else if (xop == 0x25) { /* sll, V9 sllx */
-                cpu_src1 = get_src1(insn, cpu_src1);
+                cpu_src1 = get_src1(dc, insn);
                 if (IS_IMM) {   /* immediate */
                     simm = GET_FIELDs(insn, 20, 31);
                     if (insn & (1 << 12)) {
@@ -3329,7 +3363,8 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                     }
                 } else {                /* register */
                     rs2 = GET_FIELD(insn, 27, 31);
-                    gen_movl_reg_TN(rs2, cpu_src2);
+                    cpu_src2 = gen_load_gpr(dc, rs2);
+                    cpu_tmp0 = get_temp_tl(dc);
                     if (insn & (1 << 12)) {
                         tcg_gen_andi_i64(cpu_tmp0, cpu_src2, 0x3f);
                     } else {
@@ -3337,9 +3372,9 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                     }
                     tcg_gen_shl_i64(cpu_dst, cpu_src1, cpu_tmp0);
                 }
-                gen_movl_TN_reg(rd, cpu_dst);
+                gen_store_gpr(dc, rd, cpu_dst);
             } else if (xop == 0x26) { /* srl, V9 srlx */
-                cpu_src1 = get_src1(insn, cpu_src1);
+                cpu_src1 = get_src1(dc, insn);
                 if (IS_IMM) {   /* immediate */
                     simm = GET_FIELDs(insn, 20, 31);
                     if (insn & (1 << 12)) {
@@ -3350,7 +3385,8 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                     }
                 } else {                /* register */
                     rs2 = GET_FIELD(insn, 27, 31);
-                    gen_movl_reg_TN(rs2, cpu_src2);
+                    cpu_src2 = gen_load_gpr(dc, rs2);
+                    cpu_tmp0 = get_temp_tl(dc);
                     if (insn & (1 << 12)) {
                         tcg_gen_andi_i64(cpu_tmp0, cpu_src2, 0x3f);
                         tcg_gen_shr_i64(cpu_dst, cpu_src1, cpu_tmp0);
@@ -3360,65 +3396,48 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                         tcg_gen_shr_i64(cpu_dst, cpu_dst, cpu_tmp0);
                     }
                 }
-                gen_movl_TN_reg(rd, cpu_dst);
+                gen_store_gpr(dc, rd, cpu_dst);
             } else if (xop == 0x27) { /* sra, V9 srax */
-                cpu_src1 = get_src1(insn, cpu_src1);
+                cpu_src1 = get_src1(dc, insn);
                 if (IS_IMM) {   /* immediate */
                     simm = GET_FIELDs(insn, 20, 31);
                     if (insn & (1 << 12)) {
                         tcg_gen_sari_i64(cpu_dst, cpu_src1, simm & 0x3f);
                     } else {
-                        tcg_gen_andi_i64(cpu_dst, cpu_src1, 0xffffffffULL);
-                        tcg_gen_ext32s_i64(cpu_dst, cpu_dst);
+                        tcg_gen_ext32s_i64(cpu_dst, cpu_src1);
                         tcg_gen_sari_i64(cpu_dst, cpu_dst, simm & 0x1f);
                     }
                 } else {                /* register */
                     rs2 = GET_FIELD(insn, 27, 31);
-                    gen_movl_reg_TN(rs2, cpu_src2);
+                    cpu_src2 = gen_load_gpr(dc, rs2);
+                    cpu_tmp0 = get_temp_tl(dc);
                     if (insn & (1 << 12)) {
                         tcg_gen_andi_i64(cpu_tmp0, cpu_src2, 0x3f);
                         tcg_gen_sar_i64(cpu_dst, cpu_src1, cpu_tmp0);
                     } else {
                         tcg_gen_andi_i64(cpu_tmp0, cpu_src2, 0x1f);
-                        tcg_gen_andi_i64(cpu_dst, cpu_src1, 0xffffffffULL);
-                        tcg_gen_ext32s_i64(cpu_dst, cpu_dst);
+                        tcg_gen_ext32s_i64(cpu_dst, cpu_src1);
                         tcg_gen_sar_i64(cpu_dst, cpu_dst, cpu_tmp0);
                     }
                 }
-                gen_movl_TN_reg(rd, cpu_dst);
+                gen_store_gpr(dc, rd, cpu_dst);
 #endif
             } else if (xop < 0x36) {
                 if (xop < 0x20) {
-                    cpu_src1 = get_src1(insn, cpu_src1);
-                    cpu_src2 = get_src2(insn, cpu_src2);
+                    cpu_src1 = get_src1(dc, insn);
+                    cpu_src2 = get_src2(dc, insn);
                     switch (xop & ~0x10) {
                     case 0x0: /* add */
-                        if (IS_IMM) {
-                            simm = GET_FIELDs(insn, 19, 31);
-                            if (xop & 0x10) {
-                                gen_op_addi_cc(cpu_dst, cpu_src1, simm);
-                                tcg_gen_movi_i32(cpu_cc_op, CC_OP_ADD);
-                                dc->cc_op = CC_OP_ADD;
-                            } else {
-                                tcg_gen_addi_tl(cpu_dst, cpu_src1, simm);
-                            }
+                        if (xop & 0x10) {
+                            gen_op_add_cc(cpu_dst, cpu_src1, cpu_src2);
+                            tcg_gen_movi_i32(cpu_cc_op, CC_OP_ADD);
+                            dc->cc_op = CC_OP_ADD;
                         } else {
-                            if (xop & 0x10) {
-                                gen_op_add_cc(cpu_dst, cpu_src1, cpu_src2);
-                                tcg_gen_movi_i32(cpu_cc_op, CC_OP_ADD);
-                                dc->cc_op = CC_OP_ADD;
-                            } else {
-                                tcg_gen_add_tl(cpu_dst, cpu_src1, cpu_src2);
-                            }
+                            tcg_gen_add_tl(cpu_dst, cpu_src1, cpu_src2);
                         }
                         break;
                     case 0x1: /* and */
-                        if (IS_IMM) {
-                            simm = GET_FIELDs(insn, 19, 31);
-                            tcg_gen_andi_tl(cpu_dst, cpu_src1, simm);
-                        } else {
-                            tcg_gen_and_tl(cpu_dst, cpu_src1, cpu_src2);
-                        }
+                        tcg_gen_and_tl(cpu_dst, cpu_src1, cpu_src2);
                         if (xop & 0x10) {
                             tcg_gen_mov_tl(cpu_cc_dst, cpu_dst);
                             tcg_gen_movi_i32(cpu_cc_op, CC_OP_LOGIC);
@@ -3426,12 +3445,7 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                         }
                         break;
                     case 0x2: /* or */
-                        if (IS_IMM) {
-                            simm = GET_FIELDs(insn, 19, 31);
-                            tcg_gen_ori_tl(cpu_dst, cpu_src1, simm);
-                        } else {
-                            tcg_gen_or_tl(cpu_dst, cpu_src1, cpu_src2);
-                        }
+                        tcg_gen_or_tl(cpu_dst, cpu_src1, cpu_src2);
                         if (xop & 0x10) {
                             tcg_gen_mov_tl(cpu_cc_dst, cpu_dst);
                             tcg_gen_movi_i32(cpu_cc_op, CC_OP_LOGIC);
@@ -3439,12 +3453,7 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                         }
                         break;
                     case 0x3: /* xor */
-                        if (IS_IMM) {
-                            simm = GET_FIELDs(insn, 19, 31);
-                            tcg_gen_xori_tl(cpu_dst, cpu_src1, simm);
-                        } else {
-                            tcg_gen_xor_tl(cpu_dst, cpu_src1, cpu_src2);
-                        }
+                        tcg_gen_xor_tl(cpu_dst, cpu_src1, cpu_src2);
                         if (xop & 0x10) {
                             tcg_gen_mov_tl(cpu_cc_dst, cpu_dst);
                             tcg_gen_movi_i32(cpu_cc_op, CC_OP_LOGIC);
@@ -3452,30 +3461,16 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                         }
                         break;
                     case 0x4: /* sub */
-                        if (IS_IMM) {
-                            simm = GET_FIELDs(insn, 19, 31);
-                            if (xop & 0x10) {
-                                gen_op_subi_cc(cpu_dst, cpu_src1, simm, dc);
-                            } else {
-                                tcg_gen_subi_tl(cpu_dst, cpu_src1, simm);
-                            }
+                        if (xop & 0x10) {
+                            gen_op_sub_cc(cpu_dst, cpu_src1, cpu_src2);
+                            tcg_gen_movi_i32(cpu_cc_op, CC_OP_SUB);
+                            dc->cc_op = CC_OP_SUB;
                         } else {
-                            if (xop & 0x10) {
-                                gen_op_sub_cc(cpu_dst, cpu_src1, cpu_src2);
-                                tcg_gen_movi_i32(cpu_cc_op, CC_OP_SUB);
-                                dc->cc_op = CC_OP_SUB;
-                            } else {
-                                tcg_gen_sub_tl(cpu_dst, cpu_src1, cpu_src2);
-                            }
+                            tcg_gen_sub_tl(cpu_dst, cpu_src1, cpu_src2);
                         }
                         break;
                     case 0x5: /* andn */
-                        if (IS_IMM) {
-                            simm = GET_FIELDs(insn, 19, 31);
-                            tcg_gen_andi_tl(cpu_dst, cpu_src1, ~simm);
-                        } else {
-                            tcg_gen_andc_tl(cpu_dst, cpu_src1, cpu_src2);
-                        }
+                        tcg_gen_andc_tl(cpu_dst, cpu_src1, cpu_src2);
                         if (xop & 0x10) {
                             tcg_gen_mov_tl(cpu_cc_dst, cpu_dst);
                             tcg_gen_movi_i32(cpu_cc_op, CC_OP_LOGIC);
@@ -3483,12 +3478,7 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                         }
                         break;
                     case 0x6: /* orn */
-                        if (IS_IMM) {
-                            simm = GET_FIELDs(insn, 19, 31);
-                            tcg_gen_ori_tl(cpu_dst, cpu_src1, ~simm);
-                        } else {
-                            tcg_gen_orc_tl(cpu_dst, cpu_src1, cpu_src2);
-                        }
+                        tcg_gen_orc_tl(cpu_dst, cpu_src1, cpu_src2);
                         if (xop & 0x10) {
                             tcg_gen_mov_tl(cpu_cc_dst, cpu_dst);
                             tcg_gen_movi_i32(cpu_cc_op, CC_OP_LOGIC);
@@ -3496,13 +3486,7 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                         }
                         break;
                     case 0x7: /* xorn */
-                        if (IS_IMM) {
-                            simm = GET_FIELDs(insn, 19, 31);
-                            tcg_gen_xori_tl(cpu_dst, cpu_src1, ~simm);
-                        } else {
-                            tcg_gen_not_tl(cpu_tmp0, cpu_src2);
-                            tcg_gen_xor_tl(cpu_dst, cpu_src1, cpu_tmp0);
-                        }
+                        tcg_gen_eqv_tl(cpu_dst, cpu_src1, cpu_src2);
                         if (xop & 0x10) {
                             tcg_gen_mov_tl(cpu_cc_dst, cpu_dst);
                             tcg_gen_movi_i32(cpu_cc_op, CC_OP_LOGIC);
@@ -3515,12 +3499,7 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                         break;
 #ifdef TARGET_SPARC64
                     case 0x9: /* V9 mulx */
-                        if (IS_IMM) {
-                            simm = GET_FIELDs(insn, 19, 31);
-                            tcg_gen_muli_i64(cpu_dst, cpu_src1, simm);
-                        } else {
-                            tcg_gen_mul_i64(cpu_dst, cpu_src1, cpu_src2);
-                        }
+                        tcg_gen_mul_i64(cpu_dst, cpu_src1, cpu_src2);
                         break;
 #endif
                     case 0xa: /* umul */
@@ -3547,17 +3526,7 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                         break;
 #ifdef TARGET_SPARC64
                     case 0xd: /* V9 udivx */
-                        {
-                            TCGv r_temp1, r_temp2;
-                            r_temp1 = tcg_temp_local_new();
-                            r_temp2 = tcg_temp_local_new();
-                            tcg_gen_mov_tl(r_temp1, cpu_src1);
-                            tcg_gen_mov_tl(r_temp2, cpu_src2);
-                            gen_trap_ifdivzero_tl(r_temp2);
-                            tcg_gen_divu_i64(cpu_dst, r_temp1, r_temp2);
-                            tcg_temp_free(r_temp1);
-                            tcg_temp_free(r_temp2);
-                        }
+                        gen_helper_udivx(cpu_dst, cpu_env, cpu_src1, cpu_src2);
                         break;
 #endif
                     case 0xe: /* udiv */
@@ -3585,41 +3554,39 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                     default:
                         goto illegal_insn;
                     }
-                    gen_movl_TN_reg(rd, cpu_dst);
+                    gen_store_gpr(dc, rd, cpu_dst);
                 } else {
-                    cpu_src1 = get_src1(insn, cpu_src1);
-                    cpu_src2 = get_src2(insn, cpu_src2);
+                    cpu_src1 = get_src1(dc, insn);
+                    cpu_src2 = get_src2(dc, insn);
                     switch (xop) {
                     case 0x20: /* taddcc */
-                        gen_op_tadd_cc(cpu_dst, cpu_src1, cpu_src2);
-                        gen_movl_TN_reg(rd, cpu_dst);
+                        gen_op_add_cc(cpu_dst, cpu_src1, cpu_src2);
+                        gen_store_gpr(dc, rd, cpu_dst);
                         tcg_gen_movi_i32(cpu_cc_op, CC_OP_TADD);
                         dc->cc_op = CC_OP_TADD;
                         break;
                     case 0x21: /* tsubcc */
-                        gen_op_tsub_cc(cpu_dst, cpu_src1, cpu_src2);
-                        gen_movl_TN_reg(rd, cpu_dst);
+                        gen_op_sub_cc(cpu_dst, cpu_src1, cpu_src2);
+                        gen_store_gpr(dc, rd, cpu_dst);
                         tcg_gen_movi_i32(cpu_cc_op, CC_OP_TSUB);
                         dc->cc_op = CC_OP_TSUB;
                         break;
                     case 0x22: /* taddcctv */
-                        save_state(dc, cpu_cond);
-                        gen_op_tadd_ccTV(cpu_dst, cpu_src1, cpu_src2);
-                        gen_movl_TN_reg(rd, cpu_dst);
-                        tcg_gen_movi_i32(cpu_cc_op, CC_OP_TADDTV);
+                        gen_helper_taddcctv(cpu_dst, cpu_env,
+                                            cpu_src1, cpu_src2);
+                        gen_store_gpr(dc, rd, cpu_dst);
                         dc->cc_op = CC_OP_TADDTV;
                         break;
                     case 0x23: /* tsubcctv */
-                        save_state(dc, cpu_cond);
-                        gen_op_tsub_ccTV(cpu_dst, cpu_src1, cpu_src2);
-                        gen_movl_TN_reg(rd, cpu_dst);
-                        tcg_gen_movi_i32(cpu_cc_op, CC_OP_TSUBTV);
+                        gen_helper_tsubcctv(cpu_dst, cpu_env,
+                                            cpu_src1, cpu_src2);
+                        gen_store_gpr(dc, rd, cpu_dst);
                         dc->cc_op = CC_OP_TSUBTV;
                         break;
                     case 0x24: /* mulscc */
-                        gen_helper_compute_psr(cpu_env);
+                        update_psr(dc);
                         gen_op_mulscc(cpu_dst, cpu_src1, cpu_src2);
-                        gen_movl_TN_reg(rd, cpu_dst);
+                        gen_store_gpr(dc, rd, cpu_dst);
                         tcg_gen_movi_i32(cpu_cc_op, CC_OP_ADD);
                         dc->cc_op = CC_OP_ADD;
                         break;
@@ -3629,34 +3596,38 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                             simm = GET_FIELDs(insn, 20, 31);
                             tcg_gen_shli_tl(cpu_dst, cpu_src1, simm & 0x1f);
                         } else { /* register */
+                            cpu_tmp0 = get_temp_tl(dc);
                             tcg_gen_andi_tl(cpu_tmp0, cpu_src2, 0x1f);
                             tcg_gen_shl_tl(cpu_dst, cpu_src1, cpu_tmp0);
                         }
-                        gen_movl_TN_reg(rd, cpu_dst);
+                        gen_store_gpr(dc, rd, cpu_dst);
                         break;
                     case 0x26:  /* srl */
                         if (IS_IMM) { /* immediate */
                             simm = GET_FIELDs(insn, 20, 31);
                             tcg_gen_shri_tl(cpu_dst, cpu_src1, simm & 0x1f);
                         } else { /* register */
+                            cpu_tmp0 = get_temp_tl(dc);
                             tcg_gen_andi_tl(cpu_tmp0, cpu_src2, 0x1f);
                             tcg_gen_shr_tl(cpu_dst, cpu_src1, cpu_tmp0);
                         }
-                        gen_movl_TN_reg(rd, cpu_dst);
+                        gen_store_gpr(dc, rd, cpu_dst);
                         break;
                     case 0x27:  /* sra */
                         if (IS_IMM) { /* immediate */
                             simm = GET_FIELDs(insn, 20, 31);
                             tcg_gen_sari_tl(cpu_dst, cpu_src1, simm & 0x1f);
                         } else { /* register */
+                            cpu_tmp0 = get_temp_tl(dc);
                             tcg_gen_andi_tl(cpu_tmp0, cpu_src2, 0x1f);
                             tcg_gen_sar_tl(cpu_dst, cpu_src1, cpu_tmp0);
                         }
-                        gen_movl_TN_reg(rd, cpu_dst);
+                        gen_store_gpr(dc, rd, cpu_dst);
                         break;
 #endif
                     case 0x30:
                         {
+                            cpu_tmp0 = get_temp_tl(dc);
                             switch(rd) {
                             case 0: /* wry */
                                 tcg_gen_xor_tl(cpu_tmp0, cpu_src1, cpu_src2);
@@ -3674,20 +3645,20 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                                 break;
 #else
                             case 0x2: /* V9 wrccr */
-                                tcg_gen_xor_tl(cpu_dst, cpu_src1, cpu_src2);
-                                gen_helper_wrccr(cpu_env, cpu_dst);
+                                tcg_gen_xor_tl(cpu_tmp0, cpu_src1, cpu_src2);
+                                gen_helper_wrccr(cpu_env, cpu_tmp0);
                                 tcg_gen_movi_i32(cpu_cc_op, CC_OP_FLAGS);
                                 dc->cc_op = CC_OP_FLAGS;
                                 break;
                             case 0x3: /* V9 wrasi */
-                                tcg_gen_xor_tl(cpu_dst, cpu_src1, cpu_src2);
-                                tcg_gen_andi_tl(cpu_dst, cpu_dst, 0xff);
-                                tcg_gen_trunc_tl_i32(cpu_asi, cpu_dst);
+                                tcg_gen_xor_tl(cpu_tmp0, cpu_src1, cpu_src2);
+                                tcg_gen_andi_tl(cpu_tmp0, cpu_tmp0, 0xff);
+                                tcg_gen_trunc_tl_i32(cpu_asi, cpu_tmp0);
                                 break;
                             case 0x6: /* V9 wrfprs */
-                                tcg_gen_xor_tl(cpu_dst, cpu_src1, cpu_src2);
-                                tcg_gen_trunc_tl_i32(cpu_fprs, cpu_dst);
-                                save_state(dc, cpu_cond);
+                                tcg_gen_xor_tl(cpu_tmp0, cpu_src1, cpu_src2);
+                                tcg_gen_trunc_tl_i32(cpu_fprs, cpu_tmp0);
+                                save_state(dc);
                                 gen_op_next_insn();
                                 tcg_gen_exit_tb(0);
                                 dc->is_br = 1;
@@ -3700,27 +3671,28 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
 #endif
                                 break;
                             case 0x13: /* Graphics Status */
-                                if (gen_trap_ifnofpu(dc, cpu_cond))
+                                if (gen_trap_ifnofpu(dc)) {
                                     goto jmp_insn;
+                                }
                                 tcg_gen_xor_tl(cpu_gsr, cpu_src1, cpu_src2);
                                 break;
                             case 0x14: /* Softint set */
                                 if (!supervisor(dc))
                                     goto illegal_insn;
-                                tcg_gen_xor_tl(cpu_tmp64, cpu_src1, cpu_src2);
-                                gen_helper_set_softint(cpu_env, cpu_tmp64);
+                                tcg_gen_xor_tl(cpu_tmp0, cpu_src1, cpu_src2);
+                                gen_helper_set_softint(cpu_env, cpu_tmp0);
                                 break;
                             case 0x15: /* Softint clear */
                                 if (!supervisor(dc))
                                     goto illegal_insn;
-                                tcg_gen_xor_tl(cpu_tmp64, cpu_src1, cpu_src2);
-                                gen_helper_clear_softint(cpu_env, cpu_tmp64);
+                                tcg_gen_xor_tl(cpu_tmp0, cpu_src1, cpu_src2);
+                                gen_helper_clear_softint(cpu_env, cpu_tmp0);
                                 break;
                             case 0x16: /* Softint write */
                                 if (!supervisor(dc))
                                     goto illegal_insn;
-                                tcg_gen_xor_tl(cpu_tmp64, cpu_src1, cpu_src2);
-                                gen_helper_write_softint(cpu_env, cpu_tmp64);
+                                tcg_gen_xor_tl(cpu_tmp0, cpu_src1, cpu_src2);
+                                gen_helper_write_softint(cpu_env, cpu_tmp0);
                                 break;
                             case 0x17: /* Tick compare */
 #if !defined(CONFIG_USER_ONLY)
@@ -3748,13 +3720,13 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                                 {
                                     TCGv_ptr r_tickptr;
 
-                                    tcg_gen_xor_tl(cpu_dst, cpu_src1,
+                                    tcg_gen_xor_tl(cpu_tmp0, cpu_src1,
                                                    cpu_src2);
                                     r_tickptr = tcg_temp_new_ptr();
                                     tcg_gen_ld_ptr(r_tickptr, cpu_env,
                                                    offsetof(CPUSPARCState, stick));
                                     gen_helper_tick_set_count(r_tickptr,
-                                                              cpu_dst);
+                                                              cpu_tmp0);
                                     tcg_temp_free_ptr(r_tickptr);
                                 }
                                 break;
@@ -3809,11 +3781,12 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                                 goto illegal_insn;
                             }
 #else
-                            tcg_gen_xor_tl(cpu_dst, cpu_src1, cpu_src2);
-                            gen_helper_wrpsr(cpu_env, cpu_dst);
+                            cpu_tmp0 = get_temp_tl(dc);
+                            tcg_gen_xor_tl(cpu_tmp0, cpu_src1, cpu_src2);
+                            gen_helper_wrpsr(cpu_env, cpu_tmp0);
                             tcg_gen_movi_i32(cpu_cc_op, CC_OP_FLAGS);
                             dc->cc_op = CC_OP_FLAGS;
-                            save_state(dc, cpu_cond);
+                            save_state(dc);
                             gen_op_next_insn();
                             tcg_gen_exit_tb(0);
                             dc->is_br = 1;
@@ -3824,6 +3797,7 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                         {
                             if (!supervisor(dc))
                                 goto priv_insn;
+                            cpu_tmp0 = get_temp_tl(dc);
                             tcg_gen_xor_tl(cpu_tmp0, cpu_src1, cpu_src2);
 #ifdef TARGET_SPARC64
                             switch (rd) {
@@ -3867,9 +3841,8 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
 
                                     r_tsptr = tcg_temp_new_ptr();
                                     gen_load_trap_state_at_tl(r_tsptr, cpu_env);
-                                    tcg_gen_trunc_tl_i32(cpu_tmp32, cpu_tmp0);
-                                    tcg_gen_st_i32(cpu_tmp32, r_tsptr,
-                                                   offsetof(trap_state, tt));
+                                    tcg_gen_st32_tl(cpu_tmp0, r_tsptr,
+                                                    offsetof(trap_state, tt));
                                     tcg_temp_free_ptr(r_tsptr);
                                 }
                                 break;
@@ -3889,28 +3862,15 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                                 tcg_gen_mov_tl(cpu_tbr, cpu_tmp0);
                                 break;
                             case 6: // pstate
-                                {
-                                    TCGv r_tmp = tcg_temp_local_new();
-
-                                    tcg_gen_mov_tl(r_tmp, cpu_tmp0);
-                                    save_state(dc, cpu_cond);
-                                    gen_helper_wrpstate(cpu_env, r_tmp);
-                                    tcg_temp_free(r_tmp);
-                                    dc->npc = DYNAMIC_PC;
-                                }
+                                save_state(dc);
+                                gen_helper_wrpstate(cpu_env, cpu_tmp0);
+                                dc->npc = DYNAMIC_PC;
                                 break;
                             case 7: // tl
-                                {
-                                    TCGv r_tmp = tcg_temp_local_new();
-
-                                    tcg_gen_mov_tl(r_tmp, cpu_tmp0);
-                                    save_state(dc, cpu_cond);
-                                    tcg_gen_trunc_tl_i32(cpu_tmp32, r_tmp);
-                                    tcg_temp_free(r_tmp);
-                                    tcg_gen_st_i32(cpu_tmp32, cpu_env,
-                                                   offsetof(CPUSPARCState, tl));
-                                    dc->npc = DYNAMIC_PC;
-                                }
+                                save_state(dc);
+                                tcg_gen_st32_tl(cpu_tmp0, cpu_env,
+                                               offsetof(CPUSPARCState, tl));
+                                dc->npc = DYNAMIC_PC;
                                 break;
                             case 8: // pil
                                 gen_helper_wrpil(cpu_env, cpu_tmp0);
@@ -3919,40 +3879,34 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                                 gen_helper_wrcwp(cpu_env, cpu_tmp0);
                                 break;
                             case 10: // cansave
-                                tcg_gen_trunc_tl_i32(cpu_tmp32, cpu_tmp0);
-                                tcg_gen_st_i32(cpu_tmp32, cpu_env,
-                                               offsetof(CPUSPARCState,
-                                                        cansave));
+                                tcg_gen_st32_tl(cpu_tmp0, cpu_env,
+                                                offsetof(CPUSPARCState,
+                                                         cansave));
                                 break;
                             case 11: // canrestore
-                                tcg_gen_trunc_tl_i32(cpu_tmp32, cpu_tmp0);
-                                tcg_gen_st_i32(cpu_tmp32, cpu_env,
-                                               offsetof(CPUSPARCState,
-                                                        canrestore));
+                                tcg_gen_st32_tl(cpu_tmp0, cpu_env,
+                                                offsetof(CPUSPARCState,
+                                                         canrestore));
                                 break;
                             case 12: // cleanwin
-                                tcg_gen_trunc_tl_i32(cpu_tmp32, cpu_tmp0);
-                                tcg_gen_st_i32(cpu_tmp32, cpu_env,
-                                               offsetof(CPUSPARCState,
-                                                        cleanwin));
+                                tcg_gen_st32_tl(cpu_tmp0, cpu_env,
+                                                offsetof(CPUSPARCState,
+                                                         cleanwin));
                                 break;
                             case 13: // otherwin
-                                tcg_gen_trunc_tl_i32(cpu_tmp32, cpu_tmp0);
-                                tcg_gen_st_i32(cpu_tmp32, cpu_env,
-                                               offsetof(CPUSPARCState,
-                                                        otherwin));
+                                tcg_gen_st32_tl(cpu_tmp0, cpu_env,
+                                                offsetof(CPUSPARCState,
+                                                         otherwin));
                                 break;
                             case 14: // wstate
-                                tcg_gen_trunc_tl_i32(cpu_tmp32, cpu_tmp0);
-                                tcg_gen_st_i32(cpu_tmp32, cpu_env,
-                                               offsetof(CPUSPARCState,
-                                                        wstate));
+                                tcg_gen_st32_tl(cpu_tmp0, cpu_env,
+                                                offsetof(CPUSPARCState,
+                                                         wstate));
                                 break;
                             case 16: // UA2005 gl
                                 CHECK_IU_FEATURE(dc, GL);
-                                tcg_gen_trunc_tl_i32(cpu_tmp32, cpu_tmp0);
-                                tcg_gen_st_i32(cpu_tmp32, cpu_env,
-                                               offsetof(CPUSPARCState, gl));
+                                tcg_gen_st32_tl(cpu_tmp0, cpu_env,
+                                                offsetof(CPUSPARCState, gl));
                                 break;
                             case 26: // UA2005 strand status
                                 CHECK_IU_FEATURE(dc, HYPV);
@@ -3964,11 +3918,11 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                                 goto illegal_insn;
                             }
 #else
-                            tcg_gen_trunc_tl_i32(cpu_tmp32, cpu_tmp0);
-                            if (dc->def->nwindows != 32)
-                                tcg_gen_andi_tl(cpu_tmp32, cpu_tmp32,
+                            tcg_gen_trunc_tl_i32(cpu_wim, cpu_tmp0);
+                            if (dc->def->nwindows != 32) {
+                                tcg_gen_andi_tl(cpu_wim, cpu_wim,
                                                 (1 << dc->def->nwindows) - 1);
-                            tcg_gen_mov_i32(cpu_wim, cpu_tmp32);
+                            }
 #endif
                         }
                         break;
@@ -3982,11 +3936,12 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                             CHECK_IU_FEATURE(dc, HYPV);
                             if (!hypervisor(dc))
                                 goto priv_insn;
+                            cpu_tmp0 = get_temp_tl(dc);
                             tcg_gen_xor_tl(cpu_tmp0, cpu_src1, cpu_src2);
                             switch (rd) {
                             case 0: // hpstate
                                 // XXX gen_op_wrhpstate();
-                                save_state(dc, cpu_cond);
+                                save_state(dc);
                                 gen_op_next_insn();
                                 tcg_gen_exit_tb(0);
                                 dc->is_br = 1;
@@ -4026,74 +3981,67 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                         {
                             int cc = GET_FIELD_SP(insn, 11, 12);
                             int cond = GET_FIELD_SP(insn, 14, 17);
-                            TCGv r_cond;
-                            int l1;
+                            DisasCompare cmp;
+                            TCGv dst;
 
-                            r_cond = tcg_temp_new();
                             if (insn & (1 << 18)) {
-                                if (cc == 0)
-                                    gen_cond(r_cond, 0, cond, dc);
-                                else if (cc == 2)
-                                    gen_cond(r_cond, 1, cond, dc);
-                                else
+                                if (cc == 0) {
+                                    gen_compare(&cmp, 0, cond, dc);
+                                } else if (cc == 2) {
+                                    gen_compare(&cmp, 1, cond, dc);
+                                } else {
                                     goto illegal_insn;
+                                }
                             } else {
-                                gen_fcond(r_cond, cc, cond);
+                                gen_fcompare(&cmp, cc, cond);
                             }
 
-                            l1 = gen_new_label();
-
-                            tcg_gen_brcondi_tl(TCG_COND_EQ, r_cond, 0, l1);
-                            if (IS_IMM) {       /* immediate */
-                                TCGv r_const;
-
+                            /* The get_src2 above loaded the normal 13-bit
+                               immediate field, not the 11-bit field we have
+                               in movcc.  But it did handle the reg case.  */
+                            if (IS_IMM) {
                                 simm = GET_FIELD_SPs(insn, 0, 10);
-                                r_const = tcg_const_tl(simm);
-                                gen_movl_TN_reg(rd, r_const);
-                                tcg_temp_free(r_const);
-                            } else {
-                                rs2 = GET_FIELD_SP(insn, 0, 4);
-                                gen_movl_reg_TN(rs2, cpu_tmp0);
-                                gen_movl_TN_reg(rd, cpu_tmp0);
+                                tcg_gen_movi_tl(cpu_src2, simm);
                             }
-                            gen_set_label(l1);
-                            tcg_temp_free(r_cond);
+
+                            dst = gen_load_gpr(dc, rd);
+                            tcg_gen_movcond_tl(cmp.cond, dst,
+                                               cmp.c1, cmp.c2,
+                                               cpu_src2, dst);
+                            free_compare(&cmp);
+                            gen_store_gpr(dc, rd, dst);
                             break;
                         }
                     case 0x2d: /* V9 sdivx */
-                        gen_op_sdivx(cpu_dst, cpu_src1, cpu_src2);
-                        gen_movl_TN_reg(rd, cpu_dst);
+                        gen_helper_sdivx(cpu_dst, cpu_env, cpu_src1, cpu_src2);
+                        gen_store_gpr(dc, rd, cpu_dst);
                         break;
                     case 0x2e: /* V9 popc */
-                        {
-                            cpu_src2 = get_src2(insn, cpu_src2);
-                            gen_helper_popc(cpu_dst, cpu_src2);
-                            gen_movl_TN_reg(rd, cpu_dst);
-                        }
+                        gen_helper_popc(cpu_dst, cpu_src2);
+                        gen_store_gpr(dc, rd, cpu_dst);
+                        break;
                     case 0x2f: /* V9 movr */
                         {
                             int cond = GET_FIELD_SP(insn, 10, 12);
-                            int l1;
-
-                            cpu_src1 = get_src1(insn, cpu_src1);
-
-                            l1 = gen_new_label();
+                            DisasCompare cmp;
+                            TCGv dst;
 
-                            tcg_gen_brcondi_tl(gen_tcg_cond_reg[cond],
-                                              cpu_src1, 0, l1);
-                            if (IS_IMM) {       /* immediate */
-                                TCGv r_const;
+                            gen_compare_reg(&cmp, cond, cpu_src1);
 
+                            /* The get_src2 above loaded the normal 13-bit
+                               immediate field, not the 10-bit field we have
+                               in movr.  But it did handle the reg case.  */
+                            if (IS_IMM) {
                                 simm = GET_FIELD_SPs(insn, 0, 9);
-                                r_const = tcg_const_tl(simm);
-                                gen_movl_TN_reg(rd, r_const);
-                                tcg_temp_free(r_const);
-                            } else {
-                                rs2 = GET_FIELD_SP(insn, 0, 4);
-                                gen_movl_reg_TN(rs2, cpu_tmp0);
-                                gen_movl_TN_reg(rd, cpu_tmp0);
+                                tcg_gen_movi_tl(cpu_src2, simm);
                             }
-                            gen_set_label(l1);
+
+                            dst = gen_load_gpr(dc, rd);
+                            tcg_gen_movcond_tl(cmp.cond, dst,
+                                               cmp.c1, cmp.c2,
+                                               cpu_src2, dst);
+                            free_compare(&cmp);
+                            gen_store_gpr(dc, rd, dst);
                             break;
                         }
 #endif
@@ -4106,194 +4054,195 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                 int opf = GET_FIELD_SP(insn, 5, 13);
                 rs1 = GET_FIELD(insn, 13, 17);
                 rs2 = GET_FIELD(insn, 27, 31);
-                if (gen_trap_ifnofpu(dc, cpu_cond))
+                if (gen_trap_ifnofpu(dc)) {
                     goto jmp_insn;
+                }
 
                 switch (opf) {
                 case 0x000: /* VIS I edge8cc */
                     CHECK_FPU_FEATURE(dc, VIS1);
-                    gen_movl_reg_TN(rs1, cpu_src1);
-                    gen_movl_reg_TN(rs2, cpu_src2);
+                    cpu_src1 = gen_load_gpr(dc, rs1);
+                    cpu_src2 = gen_load_gpr(dc, rs2);
                     gen_edge(dc, cpu_dst, cpu_src1, cpu_src2, 8, 1, 0);
-                    gen_movl_TN_reg(rd, cpu_dst);
+                    gen_store_gpr(dc, rd, cpu_dst);
                     break;
                 case 0x001: /* VIS II edge8n */
                     CHECK_FPU_FEATURE(dc, VIS2);
-                    gen_movl_reg_TN(rs1, cpu_src1);
-                    gen_movl_reg_TN(rs2, cpu_src2);
+                    cpu_src1 = gen_load_gpr(dc, rs1);
+                    cpu_src2 = gen_load_gpr(dc, rs2);
                     gen_edge(dc, cpu_dst, cpu_src1, cpu_src2, 8, 0, 0);
-                    gen_movl_TN_reg(rd, cpu_dst);
+                    gen_store_gpr(dc, rd, cpu_dst);
                     break;
                 case 0x002: /* VIS I edge8lcc */
                     CHECK_FPU_FEATURE(dc, VIS1);
-                    gen_movl_reg_TN(rs1, cpu_src1);
-                    gen_movl_reg_TN(rs2, cpu_src2);
+                    cpu_src1 = gen_load_gpr(dc, rs1);
+                    cpu_src2 = gen_load_gpr(dc, rs2);
                     gen_edge(dc, cpu_dst, cpu_src1, cpu_src2, 8, 1, 1);
-                    gen_movl_TN_reg(rd, cpu_dst);
+                    gen_store_gpr(dc, rd, cpu_dst);
                     break;
                 case 0x003: /* VIS II edge8ln */
                     CHECK_FPU_FEATURE(dc, VIS2);
-                    gen_movl_reg_TN(rs1, cpu_src1);
-                    gen_movl_reg_TN(rs2, cpu_src2);
+                    cpu_src1 = gen_load_gpr(dc, rs1);
+                    cpu_src2 = gen_load_gpr(dc, rs2);
                     gen_edge(dc, cpu_dst, cpu_src1, cpu_src2, 8, 0, 1);
-                    gen_movl_TN_reg(rd, cpu_dst);
+                    gen_store_gpr(dc, rd, cpu_dst);
                     break;
                 case 0x004: /* VIS I edge16cc */
                     CHECK_FPU_FEATURE(dc, VIS1);
-                    gen_movl_reg_TN(rs1, cpu_src1);
-                    gen_movl_reg_TN(rs2, cpu_src2);
+                    cpu_src1 = gen_load_gpr(dc, rs1);
+                    cpu_src2 = gen_load_gpr(dc, rs2);
                     gen_edge(dc, cpu_dst, cpu_src1, cpu_src2, 16, 1, 0);
-                    gen_movl_TN_reg(rd, cpu_dst);
+                    gen_store_gpr(dc, rd, cpu_dst);
                     break;
                 case 0x005: /* VIS II edge16n */
                     CHECK_FPU_FEATURE(dc, VIS2);
-                    gen_movl_reg_TN(rs1, cpu_src1);
-                    gen_movl_reg_TN(rs2, cpu_src2);
+                    cpu_src1 = gen_load_gpr(dc, rs1);
+                    cpu_src2 = gen_load_gpr(dc, rs2);
                     gen_edge(dc, cpu_dst, cpu_src1, cpu_src2, 16, 0, 0);
-                    gen_movl_TN_reg(rd, cpu_dst);
+                    gen_store_gpr(dc, rd, cpu_dst);
                     break;
                 case 0x006: /* VIS I edge16lcc */
                     CHECK_FPU_FEATURE(dc, VIS1);
-                    gen_movl_reg_TN(rs1, cpu_src1);
-                    gen_movl_reg_TN(rs2, cpu_src2);
+                    cpu_src1 = gen_load_gpr(dc, rs1);
+                    cpu_src2 = gen_load_gpr(dc, rs2);
                     gen_edge(dc, cpu_dst, cpu_src1, cpu_src2, 16, 1, 1);
-                    gen_movl_TN_reg(rd, cpu_dst);
+                    gen_store_gpr(dc, rd, cpu_dst);
                     break;
                 case 0x007: /* VIS II edge16ln */
                     CHECK_FPU_FEATURE(dc, VIS2);
-                    gen_movl_reg_TN(rs1, cpu_src1);
-                    gen_movl_reg_TN(rs2, cpu_src2);
+                    cpu_src1 = gen_load_gpr(dc, rs1);
+                    cpu_src2 = gen_load_gpr(dc, rs2);
                     gen_edge(dc, cpu_dst, cpu_src1, cpu_src2, 16, 0, 1);
-                    gen_movl_TN_reg(rd, cpu_dst);
+                    gen_store_gpr(dc, rd, cpu_dst);
                     break;
                 case 0x008: /* VIS I edge32cc */
                     CHECK_FPU_FEATURE(dc, VIS1);
-                    gen_movl_reg_TN(rs1, cpu_src1);
-                    gen_movl_reg_TN(rs2, cpu_src2);
+                    cpu_src1 = gen_load_gpr(dc, rs1);
+                    cpu_src2 = gen_load_gpr(dc, rs2);
                     gen_edge(dc, cpu_dst, cpu_src1, cpu_src2, 32, 1, 0);
-                    gen_movl_TN_reg(rd, cpu_dst);
+                    gen_store_gpr(dc, rd, cpu_dst);
                     break;
                 case 0x009: /* VIS II edge32n */
                     CHECK_FPU_FEATURE(dc, VIS2);
-                    gen_movl_reg_TN(rs1, cpu_src1);
-                    gen_movl_reg_TN(rs2, cpu_src2);
+                    cpu_src1 = gen_load_gpr(dc, rs1);
+                    cpu_src2 = gen_load_gpr(dc, rs2);
                     gen_edge(dc, cpu_dst, cpu_src1, cpu_src2, 32, 0, 0);
-                    gen_movl_TN_reg(rd, cpu_dst);
+                    gen_store_gpr(dc, rd, cpu_dst);
                     break;
                 case 0x00a: /* VIS I edge32lcc */
                     CHECK_FPU_FEATURE(dc, VIS1);
-                    gen_movl_reg_TN(rs1, cpu_src1);
-                    gen_movl_reg_TN(rs2, cpu_src2);
+                    cpu_src1 = gen_load_gpr(dc, rs1);
+                    cpu_src2 = gen_load_gpr(dc, rs2);
                     gen_edge(dc, cpu_dst, cpu_src1, cpu_src2, 32, 1, 1);
-                    gen_movl_TN_reg(rd, cpu_dst);
+                    gen_store_gpr(dc, rd, cpu_dst);
                     break;
                 case 0x00b: /* VIS II edge32ln */
                     CHECK_FPU_FEATURE(dc, VIS2);
-                    gen_movl_reg_TN(rs1, cpu_src1);
-                    gen_movl_reg_TN(rs2, cpu_src2);
+                    cpu_src1 = gen_load_gpr(dc, rs1);
+                    cpu_src2 = gen_load_gpr(dc, rs2);
                     gen_edge(dc, cpu_dst, cpu_src1, cpu_src2, 32, 0, 1);
-                    gen_movl_TN_reg(rd, cpu_dst);
+                    gen_store_gpr(dc, rd, cpu_dst);
                     break;
                 case 0x010: /* VIS I array8 */
                     CHECK_FPU_FEATURE(dc, VIS1);
-                    cpu_src1 = get_src1(insn, cpu_src1);
-                    gen_movl_reg_TN(rs2, cpu_src2);
+                    cpu_src1 = gen_load_gpr(dc, rs1);
+                    cpu_src2 = gen_load_gpr(dc, rs2);
                     gen_helper_array8(cpu_dst, cpu_src1, cpu_src2);
-                    gen_movl_TN_reg(rd, cpu_dst);
+                    gen_store_gpr(dc, rd, cpu_dst);
                     break;
                 case 0x012: /* VIS I array16 */
                     CHECK_FPU_FEATURE(dc, VIS1);
-                    cpu_src1 = get_src1(insn, cpu_src1);
-                    gen_movl_reg_TN(rs2, cpu_src2);
+                    cpu_src1 = gen_load_gpr(dc, rs1);
+                    cpu_src2 = gen_load_gpr(dc, rs2);
                     gen_helper_array8(cpu_dst, cpu_src1, cpu_src2);
                     tcg_gen_shli_i64(cpu_dst, cpu_dst, 1);
-                    gen_movl_TN_reg(rd, cpu_dst);
+                    gen_store_gpr(dc, rd, cpu_dst);
                     break;
                 case 0x014: /* VIS I array32 */
                     CHECK_FPU_FEATURE(dc, VIS1);
-                    cpu_src1 = get_src1(insn, cpu_src1);
-                    gen_movl_reg_TN(rs2, cpu_src2);
+                    cpu_src1 = gen_load_gpr(dc, rs1);
+                    cpu_src2 = gen_load_gpr(dc, rs2);
                     gen_helper_array8(cpu_dst, cpu_src1, cpu_src2);
                     tcg_gen_shli_i64(cpu_dst, cpu_dst, 2);
-                    gen_movl_TN_reg(rd, cpu_dst);
+                    gen_store_gpr(dc, rd, cpu_dst);
                     break;
                 case 0x018: /* VIS I alignaddr */
                     CHECK_FPU_FEATURE(dc, VIS1);
-                    cpu_src1 = get_src1(insn, cpu_src1);
-                    gen_movl_reg_TN(rs2, cpu_src2);
+                    cpu_src1 = gen_load_gpr(dc, rs1);
+                    cpu_src2 = gen_load_gpr(dc, rs2);
                     gen_alignaddr(cpu_dst, cpu_src1, cpu_src2, 0);
-                    gen_movl_TN_reg(rd, cpu_dst);
+                    gen_store_gpr(dc, rd, cpu_dst);
                     break;
                 case 0x01a: /* VIS I alignaddrl */
                     CHECK_FPU_FEATURE(dc, VIS1);
-                    cpu_src1 = get_src1(insn, cpu_src1);
-                    gen_movl_reg_TN(rs2, cpu_src2);
+                    cpu_src1 = gen_load_gpr(dc, rs1);
+                    cpu_src2 = gen_load_gpr(dc, rs2);
                     gen_alignaddr(cpu_dst, cpu_src1, cpu_src2, 1);
-                    gen_movl_TN_reg(rd, cpu_dst);
+                    gen_store_gpr(dc, rd, cpu_dst);
                     break;
                 case 0x019: /* VIS II bmask */
                     CHECK_FPU_FEATURE(dc, VIS2);
-                    cpu_src1 = get_src1(insn, cpu_src1);
-                    cpu_src2 = get_src1(insn, cpu_src2);
+                    cpu_src1 = gen_load_gpr(dc, rs1);
+                    cpu_src2 = gen_load_gpr(dc, rs2);
                     tcg_gen_add_tl(cpu_dst, cpu_src1, cpu_src2);
                     tcg_gen_deposit_tl(cpu_gsr, cpu_gsr, cpu_dst, 32, 32);
-                    gen_movl_TN_reg(rd, cpu_dst);
+                    gen_store_gpr(dc, rd, cpu_dst);
                     break;
                 case 0x020: /* VIS I fcmple16 */
                     CHECK_FPU_FEATURE(dc, VIS1);
                     cpu_src1_64 = gen_load_fpr_D(dc, rs1);
                     cpu_src2_64 = gen_load_fpr_D(dc, rs2);
                     gen_helper_fcmple16(cpu_dst, cpu_src1_64, cpu_src2_64);
-                    gen_movl_TN_reg(rd, cpu_dst);
+                    gen_store_gpr(dc, rd, cpu_dst);
                     break;
                 case 0x022: /* VIS I fcmpne16 */
                     CHECK_FPU_FEATURE(dc, VIS1);
                     cpu_src1_64 = gen_load_fpr_D(dc, rs1);
                     cpu_src2_64 = gen_load_fpr_D(dc, rs2);
                     gen_helper_fcmpne16(cpu_dst, cpu_src1_64, cpu_src2_64);
-                    gen_movl_TN_reg(rd, cpu_dst);
+                    gen_store_gpr(dc, rd, cpu_dst);
                     break;
                 case 0x024: /* VIS I fcmple32 */
                     CHECK_FPU_FEATURE(dc, VIS1);
                     cpu_src1_64 = gen_load_fpr_D(dc, rs1);
                     cpu_src2_64 = gen_load_fpr_D(dc, rs2);
                     gen_helper_fcmple32(cpu_dst, cpu_src1_64, cpu_src2_64);
-                    gen_movl_TN_reg(rd, cpu_dst);
+                    gen_store_gpr(dc, rd, cpu_dst);
                     break;
                 case 0x026: /* VIS I fcmpne32 */
                     CHECK_FPU_FEATURE(dc, VIS1);
                     cpu_src1_64 = gen_load_fpr_D(dc, rs1);
                     cpu_src2_64 = gen_load_fpr_D(dc, rs2);
                     gen_helper_fcmpne32(cpu_dst, cpu_src1_64, cpu_src2_64);
-                    gen_movl_TN_reg(rd, cpu_dst);
+                    gen_store_gpr(dc, rd, cpu_dst);
                     break;
                 case 0x028: /* VIS I fcmpgt16 */
                     CHECK_FPU_FEATURE(dc, VIS1);
                     cpu_src1_64 = gen_load_fpr_D(dc, rs1);
                     cpu_src2_64 = gen_load_fpr_D(dc, rs2);
                     gen_helper_fcmpgt16(cpu_dst, cpu_src1_64, cpu_src2_64);
-                    gen_movl_TN_reg(rd, cpu_dst);
+                    gen_store_gpr(dc, rd, cpu_dst);
                     break;
                 case 0x02a: /* VIS I fcmpeq16 */
                     CHECK_FPU_FEATURE(dc, VIS1);
                     cpu_src1_64 = gen_load_fpr_D(dc, rs1);
                     cpu_src2_64 = gen_load_fpr_D(dc, rs2);
                     gen_helper_fcmpeq16(cpu_dst, cpu_src1_64, cpu_src2_64);
-                    gen_movl_TN_reg(rd, cpu_dst);
+                    gen_store_gpr(dc, rd, cpu_dst);
                     break;
                 case 0x02c: /* VIS I fcmpgt32 */
                     CHECK_FPU_FEATURE(dc, VIS1);
                     cpu_src1_64 = gen_load_fpr_D(dc, rs1);
                     cpu_src2_64 = gen_load_fpr_D(dc, rs2);
                     gen_helper_fcmpgt32(cpu_dst, cpu_src1_64, cpu_src2_64);
-                    gen_movl_TN_reg(rd, cpu_dst);
+                    gen_store_gpr(dc, rd, cpu_dst);
                     break;
                 case 0x02e: /* VIS I fcmpeq32 */
                     CHECK_FPU_FEATURE(dc, VIS1);
                     cpu_src1_64 = gen_load_fpr_D(dc, rs1);
                     cpu_src2_64 = gen_load_fpr_D(dc, rs2);
                     gen_helper_fcmpeq32(cpu_dst, cpu_src1_64, cpu_src2_64);
-                    gen_movl_TN_reg(rd, cpu_dst);
+                    gen_store_gpr(dc, rd, cpu_dst);
                     break;
                 case 0x031: /* VIS I fmul8x16 */
                     CHECK_FPU_FEATURE(dc, VIS1);
@@ -4330,14 +4279,14 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                 case 0x03b: /* VIS I fpack16 */
                     CHECK_FPU_FEATURE(dc, VIS1);
                     cpu_src1_64 = gen_load_fpr_D(dc, rs2);
-                    cpu_dst_32 = gen_dest_fpr_F();
+                    cpu_dst_32 = gen_dest_fpr_F(dc);
                     gen_helper_fpack16(cpu_dst_32, cpu_gsr, cpu_src1_64);
                     gen_store_fpr_F(dc, rd, cpu_dst_32);
                     break;
                 case 0x03d: /* VIS I fpackfix */
                     CHECK_FPU_FEATURE(dc, VIS1);
                     cpu_src1_64 = gen_load_fpr_D(dc, rs2);
-                    cpu_dst_32 = gen_dest_fpr_F();
+                    cpu_dst_32 = gen_dest_fpr_F(dc);
                     gen_helper_fpackfix(cpu_dst_32, cpu_gsr, cpu_src1_64);
                     gen_store_fpr_F(dc, rd, cpu_dst_32);
                     break;
@@ -4395,13 +4344,13 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                     break;
                 case 0x060: /* VIS I fzero */
                     CHECK_FPU_FEATURE(dc, VIS1);
-                    cpu_dst_64 = gen_dest_fpr_D();
+                    cpu_dst_64 = gen_dest_fpr_D(dc, rd);
                     tcg_gen_movi_i64(cpu_dst_64, 0);
                     gen_store_fpr_D(dc, rd, cpu_dst_64);
                     break;
                 case 0x061: /* VIS I fzeros */
                     CHECK_FPU_FEATURE(dc, VIS1);
-                    cpu_dst_32 = gen_dest_fpr_F();
+                    cpu_dst_32 = gen_dest_fpr_F(dc);
                     tcg_gen_movi_i32(cpu_dst_32, 0);
                     gen_store_fpr_F(dc, rd, cpu_dst_32);
                     break;
@@ -4523,13 +4472,13 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                     break;
                 case 0x07e: /* VIS I fone */
                     CHECK_FPU_FEATURE(dc, VIS1);
-                    cpu_dst_64 = gen_dest_fpr_D();
+                    cpu_dst_64 = gen_dest_fpr_D(dc, rd);
                     tcg_gen_movi_i64(cpu_dst_64, -1);
                     gen_store_fpr_D(dc, rd, cpu_dst_64);
                     break;
                 case 0x07f: /* VIS I fones */
                     CHECK_FPU_FEATURE(dc, VIS1);
-                    cpu_dst_32 = gen_dest_fpr_F();
+                    cpu_dst_32 = gen_dest_fpr_F(dc);
                     tcg_gen_movi_i32(cpu_dst_32, -1);
                     gen_store_fpr_F(dc, rd, cpu_dst_32);
                     break;
@@ -4553,56 +4502,60 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
             } else if (xop == 0x39) { /* V9 return */
                 TCGv_i32 r_const;
 
-                save_state(dc, cpu_cond);
-                cpu_src1 = get_src1(insn, cpu_src1);
+                save_state(dc);
+                cpu_src1 = get_src1(dc, insn);
+                cpu_tmp0 = get_temp_tl(dc);
                 if (IS_IMM) {   /* immediate */
                     simm = GET_FIELDs(insn, 19, 31);
-                    tcg_gen_addi_tl(cpu_dst, cpu_src1, simm);
+                    tcg_gen_addi_tl(cpu_tmp0, cpu_src1, simm);
                 } else {                /* register */
                     rs2 = GET_FIELD(insn, 27, 31);
                     if (rs2) {
-                        gen_movl_reg_TN(rs2, cpu_src2);
-                        tcg_gen_add_tl(cpu_dst, cpu_src1, cpu_src2);
-                    } else
-                        tcg_gen_mov_tl(cpu_dst, cpu_src1);
+                        cpu_src2 = gen_load_gpr(dc, rs2);
+                        tcg_gen_add_tl(cpu_tmp0, cpu_src1, cpu_src2);
+                    } else {
+                        tcg_gen_mov_tl(cpu_tmp0, cpu_src1);
+                    }
                 }
                 gen_helper_restore(cpu_env);
-                gen_mov_pc_npc(dc, cpu_cond);
+                gen_mov_pc_npc(dc);
                 r_const = tcg_const_i32(3);
-                gen_helper_check_align(cpu_env, cpu_dst, r_const);
+                gen_helper_check_align(cpu_env, cpu_tmp0, r_const);
                 tcg_temp_free_i32(r_const);
-                tcg_gen_mov_tl(cpu_npc, cpu_dst);
+                tcg_gen_mov_tl(cpu_npc, cpu_tmp0);
                 dc->npc = DYNAMIC_PC;
                 goto jmp_insn;
 #endif
             } else {
-                cpu_src1 = get_src1(insn, cpu_src1);
+                cpu_src1 = get_src1(dc, insn);
+                cpu_tmp0 = get_temp_tl(dc);
                 if (IS_IMM) {   /* immediate */
                     simm = GET_FIELDs(insn, 19, 31);
-                    tcg_gen_addi_tl(cpu_dst, cpu_src1, simm);
+                    tcg_gen_addi_tl(cpu_tmp0, cpu_src1, simm);
                 } else {                /* register */
                     rs2 = GET_FIELD(insn, 27, 31);
                     if (rs2) {
-                        gen_movl_reg_TN(rs2, cpu_src2);
-                        tcg_gen_add_tl(cpu_dst, cpu_src1, cpu_src2);
-                    } else
-                        tcg_gen_mov_tl(cpu_dst, cpu_src1);
+                        cpu_src2 = gen_load_gpr(dc, rs2);
+                        tcg_gen_add_tl(cpu_tmp0, cpu_src1, cpu_src2);
+                    } else {
+                        tcg_gen_mov_tl(cpu_tmp0, cpu_src1);
+                    }
                 }
                 switch (xop) {
                 case 0x38:      /* jmpl */
                     {
-                        TCGv r_pc;
+                        TCGv t;
                         TCGv_i32 r_const;
 
-                        r_pc = tcg_const_tl(dc->pc);
-                        gen_movl_TN_reg(rd, r_pc);
-                        tcg_temp_free(r_pc);
-                        gen_mov_pc_npc(dc, cpu_cond);
+                        t = gen_dest_gpr(dc, rd);
+                        tcg_gen_movi_tl(t, dc->pc);
+                        gen_store_gpr(dc, rd, t);
+                        gen_mov_pc_npc(dc);
                         r_const = tcg_const_i32(3);
-                        gen_helper_check_align(cpu_env, cpu_dst, r_const);
+                        gen_helper_check_align(cpu_env, cpu_tmp0, r_const);
                         tcg_temp_free_i32(r_const);
-                        gen_address_mask(dc, cpu_dst);
-                        tcg_gen_mov_tl(cpu_npc, cpu_dst);
+                        gen_address_mask(dc, cpu_tmp0);
+                        tcg_gen_mov_tl(cpu_npc, cpu_tmp0);
                         dc->npc = DYNAMIC_PC;
                     }
                     goto jmp_insn;
@@ -4613,11 +4566,11 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
 
                         if (!supervisor(dc))
                             goto priv_insn;
-                        gen_mov_pc_npc(dc, cpu_cond);
+                        gen_mov_pc_npc(dc);
                         r_const = tcg_const_i32(3);
-                        gen_helper_check_align(cpu_env, cpu_dst, r_const);
+                        gen_helper_check_align(cpu_env, cpu_tmp0, r_const);
                         tcg_temp_free_i32(r_const);
-                        tcg_gen_mov_tl(cpu_npc, cpu_dst);
+                        tcg_gen_mov_tl(cpu_npc, cpu_tmp0);
                         dc->npc = DYNAMIC_PC;
                         gen_helper_rett(cpu_env);
                     }
@@ -4629,14 +4582,14 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                     /* nop */
                     break;
                 case 0x3c:      /* save */
-                    save_state(dc, cpu_cond);
+                    save_state(dc);
                     gen_helper_save(cpu_env);
-                    gen_movl_TN_reg(rd, cpu_dst);
+                    gen_store_gpr(dc, rd, cpu_tmp0);
                     break;
                 case 0x3d:      /* restore */
-                    save_state(dc, cpu_cond);
+                    save_state(dc);
                     gen_helper_restore(cpu_env);
-                    gen_movl_TN_reg(rd, cpu_dst);
+                    gen_store_gpr(dc, rd, cpu_tmp0);
                     break;
 #if !defined(CONFIG_USER_ONLY) && defined(TARGET_SPARC64)
                 case 0x3e:      /* V9 done/retry */
@@ -4672,32 +4625,29 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
     case 3:                     /* load/store instructions */
         {
             unsigned int xop = GET_FIELD(insn, 7, 12);
+            /* ??? gen_address_mask prevents us from using a source
+               register directly.  Always generate a temporary.  */
+            TCGv cpu_addr = get_temp_tl(dc);
 
-            /* flush pending conditional evaluations before exposing
-               cpu state */
-            if (dc->cc_op != CC_OP_FLAGS) {
-                dc->cc_op = CC_OP_FLAGS;
-                gen_helper_compute_psr(cpu_env);
-            }
-            cpu_src1 = get_src1(insn, cpu_src1);
-            if (xop == 0x3c || xop == 0x3e) { // V9 casa/casxa
-                rs2 = GET_FIELD(insn, 27, 31);
-                gen_movl_reg_TN(rs2, cpu_src2);
-                tcg_gen_mov_tl(cpu_addr, cpu_src1);
+            tcg_gen_mov_tl(cpu_addr, get_src1(dc, insn));
+            if (xop == 0x3c || xop == 0x3e) {
+                /* V9 casa/casxa : no offset */
             } else if (IS_IMM) {     /* immediate */
                 simm = GET_FIELDs(insn, 19, 31);
-                tcg_gen_addi_tl(cpu_addr, cpu_src1, simm);
+                if (simm != 0) {
+                    tcg_gen_addi_tl(cpu_addr, cpu_addr, simm);
+                }
             } else {            /* register */
                 rs2 = GET_FIELD(insn, 27, 31);
                 if (rs2 != 0) {
-                    gen_movl_reg_TN(rs2, cpu_src2);
-                    tcg_gen_add_tl(cpu_addr, cpu_src1, cpu_src2);
-                } else
-                    tcg_gen_mov_tl(cpu_addr, cpu_src1);
+                    tcg_gen_add_tl(cpu_addr, cpu_addr, gen_load_gpr(dc, rs2));
+                }
             }
             if (xop < 4 || (xop > 7 && xop < 0x14 && xop != 0x0e) ||
                 (xop > 0x17 && xop <= 0x1d ) ||
                 (xop > 0x2c && xop <= 0x33) || xop == 0x1f || xop == 0x3d) {
+                TCGv cpu_val = gen_dest_gpr(dc, rd);
+
                 switch (xop) {
                 case 0x0:       /* ld, V9 lduw, load unsigned word */
                     gen_address_mask(dc, cpu_addr);
@@ -4716,20 +4666,23 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                         goto illegal_insn;
                     else {
                         TCGv_i32 r_const;
+                        TCGv_i64 t64;
 
-                        save_state(dc, cpu_cond);
+                        save_state(dc);
                         r_const = tcg_const_i32(7);
                         /* XXX remove alignment check */
                         gen_helper_check_align(cpu_env, cpu_addr, r_const);
                         tcg_temp_free_i32(r_const);
                         gen_address_mask(dc, cpu_addr);
-                        tcg_gen_qemu_ld64(cpu_tmp64, cpu_addr, dc->mem_idx);
-                        tcg_gen_trunc_i64_tl(cpu_tmp0, cpu_tmp64);
-                        tcg_gen_andi_tl(cpu_tmp0, cpu_tmp0, 0xffffffffULL);
-                        gen_movl_TN_reg(rd + 1, cpu_tmp0);
-                        tcg_gen_shri_i64(cpu_tmp64, cpu_tmp64, 32);
-                        tcg_gen_trunc_i64_tl(cpu_val, cpu_tmp64);
-                        tcg_gen_andi_tl(cpu_val, cpu_val, 0xffffffffULL);
+                        t64 = tcg_temp_new_i64();
+                        tcg_gen_qemu_ld64(t64, cpu_addr, dc->mem_idx);
+                        tcg_gen_trunc_i64_tl(cpu_val, t64);
+                        tcg_gen_ext32u_tl(cpu_val, cpu_val);
+                        gen_store_gpr(dc, rd + 1, cpu_val);
+                        tcg_gen_shri_i64(t64, t64, 32);
+                        tcg_gen_trunc_i64_tl(cpu_val, t64);
+                        tcg_temp_free_i64(t64);
+                        tcg_gen_ext32u_tl(cpu_val, cpu_val);
                     }
                     break;
                 case 0x9:       /* ldsb, load signed byte */
@@ -4751,14 +4704,17 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                         tcg_temp_free(r_const);
                     }
                     break;
-                case 0x0f:      /* swap, swap register with memory. Also
-                                   atomically */
-                    CHECK_IU_FEATURE(dc, SWAP);
-                    gen_movl_reg_TN(rd, cpu_val);
-                    gen_address_mask(dc, cpu_addr);
-                    tcg_gen_qemu_ld32u(cpu_tmp0, cpu_addr, dc->mem_idx);
-                    tcg_gen_qemu_st32(cpu_val, cpu_addr, dc->mem_idx);
-                    tcg_gen_mov_tl(cpu_val, cpu_tmp0);
+                case 0x0f:
+                    /* swap, swap register with memory. Also atomically */
+                    {
+                        TCGv t0 = get_temp_tl(dc);
+                        CHECK_IU_FEATURE(dc, SWAP);
+                        cpu_src1 = gen_load_gpr(dc, rd);
+                        gen_address_mask(dc, cpu_addr);
+                        tcg_gen_qemu_ld32u(t0, cpu_addr, dc->mem_idx);
+                        tcg_gen_qemu_st32(cpu_src1, cpu_addr, dc->mem_idx);
+                        tcg_gen_mov_tl(cpu_val, t0);
+                    }
                     break;
 #if !defined(CONFIG_USER_ONLY) || defined(TARGET_SPARC64)
                 case 0x10:      /* lda, V9 lduwa, load word alternate */
@@ -4768,7 +4724,7 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                     if (!supervisor(dc))
                         goto priv_insn;
 #endif
-                    save_state(dc, cpu_cond);
+                    save_state(dc);
                     gen_ld_asi(cpu_val, cpu_addr, insn, 4, 0);
                     break;
                 case 0x11:      /* lduba, load unsigned byte alternate */
@@ -4778,7 +4734,7 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                     if (!supervisor(dc))
                         goto priv_insn;
 #endif
-                    save_state(dc, cpu_cond);
+                    save_state(dc);
                     gen_ld_asi(cpu_val, cpu_addr, insn, 1, 0);
                     break;
                 case 0x12:      /* lduha, load unsigned halfword alternate */
@@ -4788,7 +4744,7 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                     if (!supervisor(dc))
                         goto priv_insn;
 #endif
-                    save_state(dc, cpu_cond);
+                    save_state(dc);
                     gen_ld_asi(cpu_val, cpu_addr, insn, 2, 0);
                     break;
                 case 0x13:      /* ldda, load double word alternate */
@@ -4800,8 +4756,8 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
 #endif
                     if (rd & 1)
                         goto illegal_insn;
-                    save_state(dc, cpu_cond);
-                    gen_ldda_asi(cpu_val, cpu_addr, insn, rd);
+                    save_state(dc);
+                    gen_ldda_asi(dc, cpu_val, cpu_addr, insn, rd);
                     goto skip_move;
                 case 0x19:      /* ldsba, load signed byte alternate */
 #ifndef TARGET_SPARC64
@@ -4810,7 +4766,7 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                     if (!supervisor(dc))
                         goto priv_insn;
 #endif
-                    save_state(dc, cpu_cond);
+                    save_state(dc);
                     gen_ld_asi(cpu_val, cpu_addr, insn, 1, 1);
                     break;
                 case 0x1a:      /* ldsha, load signed halfword alternate */
@@ -4820,7 +4776,7 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                     if (!supervisor(dc))
                         goto priv_insn;
 #endif
-                    save_state(dc, cpu_cond);
+                    save_state(dc);
                     gen_ld_asi(cpu_val, cpu_addr, insn, 2, 1);
                     break;
                 case 0x1d:      /* ldstuba -- XXX: should be atomically */
@@ -4830,7 +4786,7 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                     if (!supervisor(dc))
                         goto priv_insn;
 #endif
-                    save_state(dc, cpu_cond);
+                    save_state(dc);
                     gen_ldstub_asi(cpu_val, cpu_addr, insn);
                     break;
                 case 0x1f:      /* swapa, swap reg with alt. memory. Also
@@ -4842,9 +4798,9 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                     if (!supervisor(dc))
                         goto priv_insn;
 #endif
-                    save_state(dc, cpu_cond);
-                    gen_movl_reg_TN(rd, cpu_val);
-                    gen_swap_asi(cpu_val, cpu_addr, insn);
+                    save_state(dc);
+                    cpu_src1 = gen_load_gpr(dc, rd);
+                    gen_swap_asi(cpu_val, cpu_src1, cpu_addr, insn);
                     break;
 
 #ifndef TARGET_SPARC64
@@ -4864,28 +4820,28 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                     tcg_gen_qemu_ld64(cpu_val, cpu_addr, dc->mem_idx);
                     break;
                 case 0x18: /* V9 ldswa */
-                    save_state(dc, cpu_cond);
+                    save_state(dc);
                     gen_ld_asi(cpu_val, cpu_addr, insn, 4, 1);
                     break;
                 case 0x1b: /* V9 ldxa */
-                    save_state(dc, cpu_cond);
+                    save_state(dc);
                     gen_ld_asi(cpu_val, cpu_addr, insn, 8, 0);
                     break;
                 case 0x2d: /* V9 prefetch, no effect */
                     goto skip_move;
                 case 0x30: /* V9 ldfa */
-                    if (gen_trap_ifnofpu(dc, cpu_cond)) {
+                    if (gen_trap_ifnofpu(dc)) {
                         goto jmp_insn;
                     }
-                    save_state(dc, cpu_cond);
+                    save_state(dc);
                     gen_ldf_asi(cpu_addr, insn, 4, rd);
                     gen_update_fprs_dirty(rd);
                     goto skip_move;
                 case 0x33: /* V9 lddfa */
-                    if (gen_trap_ifnofpu(dc, cpu_cond)) {
+                    if (gen_trap_ifnofpu(dc)) {
                         goto jmp_insn;
                     }
-                    save_state(dc, cpu_cond);
+                    save_state(dc);
                     gen_ldf_asi(cpu_addr, insn, 8, DFPREG(rd));
                     gen_update_fprs_dirty(DFPREG(rd));
                     goto skip_move;
@@ -4893,10 +4849,10 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                     goto skip_move;
                 case 0x32: /* V9 ldqfa */
                     CHECK_FPU_FEATURE(dc, FLOAT128);
-                    if (gen_trap_ifnofpu(dc, cpu_cond)) {
+                    if (gen_trap_ifnofpu(dc)) {
                         goto jmp_insn;
                     }
-                    save_state(dc, cpu_cond);
+                    save_state(dc);
                     gen_ldf_asi(cpu_addr, insn, 16, QFPREG(rd));
                     gen_update_fprs_dirty(QFPREG(rd));
                     goto skip_move;
@@ -4904,39 +4860,42 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                 default:
                     goto illegal_insn;
                 }
-                gen_movl_TN_reg(rd, cpu_val);
+                gen_store_gpr(dc, rd, cpu_val);
 #if !defined(CONFIG_USER_ONLY) || defined(TARGET_SPARC64)
             skip_move: ;
 #endif
             } else if (xop >= 0x20 && xop < 0x24) {
-                if (gen_trap_ifnofpu(dc, cpu_cond))
+                TCGv t0;
+
+                if (gen_trap_ifnofpu(dc)) {
                     goto jmp_insn;
-                save_state(dc, cpu_cond);
+                }
+                save_state(dc);
                 switch (xop) {
                 case 0x20:      /* ldf, load fpreg */
                     gen_address_mask(dc, cpu_addr);
-                    tcg_gen_qemu_ld32u(cpu_tmp0, cpu_addr, dc->mem_idx);
-                    cpu_dst_32 = gen_dest_fpr_F();
-                    tcg_gen_trunc_tl_i32(cpu_dst_32, cpu_tmp0);
+                    t0 = get_temp_tl(dc);
+                    tcg_gen_qemu_ld32u(t0, cpu_addr, dc->mem_idx);
+                    cpu_dst_32 = gen_dest_fpr_F(dc);
+                    tcg_gen_trunc_tl_i32(cpu_dst_32, t0);
                     gen_store_fpr_F(dc, rd, cpu_dst_32);
                     break;
                 case 0x21:      /* ldfsr, V9 ldxfsr */
 #ifdef TARGET_SPARC64
                     gen_address_mask(dc, cpu_addr);
                     if (rd == 1) {
-                        tcg_gen_qemu_ld64(cpu_tmp64, cpu_addr, dc->mem_idx);
-                        gen_helper_ldxfsr(cpu_env, cpu_tmp64);
-                    } else {
-                        tcg_gen_qemu_ld32u(cpu_tmp0, cpu_addr, dc->mem_idx);
-                        tcg_gen_trunc_tl_i32(cpu_tmp32, cpu_tmp0);
-                        gen_helper_ldfsr(cpu_env, cpu_tmp32);
-                    }
-#else
-                    {
-                        tcg_gen_qemu_ld32u(cpu_tmp32, cpu_addr, dc->mem_idx);
-                        gen_helper_ldfsr(cpu_env, cpu_tmp32);
+                        TCGv_i64 t64 = tcg_temp_new_i64();
+                        tcg_gen_qemu_ld64(t64, cpu_addr, dc->mem_idx);
+                        gen_helper_ldxfsr(cpu_env, t64);
+                        tcg_temp_free_i64(t64);
+                        break;
                     }
 #endif
+                    cpu_dst_32 = get_temp_i32(dc);
+                    t0 = get_temp_tl(dc);
+                    tcg_gen_qemu_ld32u(t0, cpu_addr, dc->mem_idx);
+                    tcg_gen_trunc_tl_i32(cpu_dst_32, t0);
+                    gen_helper_ldfsr(cpu_env, cpu_dst_32);
                     break;
                 case 0x22:      /* ldqf, load quad fpreg */
                     {
@@ -4953,7 +4912,7 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                     break;
                 case 0x23:      /* lddf, load double fpreg */
                     gen_address_mask(dc, cpu_addr);
-                    cpu_dst_64 = gen_dest_fpr_D();
+                    cpu_dst_64 = gen_dest_fpr_D(dc, rd);
                     tcg_gen_qemu_ld64(cpu_dst_64, cpu_addr, dc->mem_idx);
                     gen_store_fpr_D(dc, rd, cpu_dst_64);
                     break;
@@ -4962,7 +4921,8 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                 }
             } else if (xop < 8 || (xop >= 0x14 && xop < 0x18) ||
                        xop == 0xe || xop == 0x1e) {
-                gen_movl_reg_TN(rd, cpu_val);
+                TCGv cpu_val = gen_load_gpr(dc, rd);
+
                 switch (xop) {
                 case 0x4: /* st, store word */
                     gen_address_mask(dc, cpu_addr);
@@ -4981,16 +4941,21 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                         goto illegal_insn;
                     else {
                         TCGv_i32 r_const;
+                        TCGv_i64 t64;
+                        TCGv lo;
 
-                        save_state(dc, cpu_cond);
+                        save_state(dc);
                         gen_address_mask(dc, cpu_addr);
                         r_const = tcg_const_i32(7);
                         /* XXX remove alignment check */
                         gen_helper_check_align(cpu_env, cpu_addr, r_const);
                         tcg_temp_free_i32(r_const);
-                        gen_movl_reg_TN(rd + 1, cpu_tmp0);
-                        tcg_gen_concat_tl_i64(cpu_tmp64, cpu_tmp0, cpu_val);
-                        tcg_gen_qemu_st64(cpu_tmp64, cpu_addr, dc->mem_idx);
+                        lo = gen_load_gpr(dc, rd + 1);
+
+                        t64 = tcg_temp_new_i64();
+                        tcg_gen_concat_tl_i64(t64, lo, cpu_val);
+                        tcg_gen_qemu_st64(t64, cpu_addr, dc->mem_idx);
+                        tcg_temp_free_i64(t64);
                     }
                     break;
 #if !defined(CONFIG_USER_ONLY) || defined(TARGET_SPARC64)
@@ -5001,7 +4966,7 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                     if (!supervisor(dc))
                         goto priv_insn;
 #endif
-                    save_state(dc, cpu_cond);
+                    save_state(dc);
                     gen_st_asi(cpu_val, cpu_addr, insn, 4);
                     dc->npc = DYNAMIC_PC;
                     break;
@@ -5012,7 +4977,7 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                     if (!supervisor(dc))
                         goto priv_insn;
 #endif
-                    save_state(dc, cpu_cond);
+                    save_state(dc);
                     gen_st_asi(cpu_val, cpu_addr, insn, 1);
                     dc->npc = DYNAMIC_PC;
                     break;
@@ -5023,7 +4988,7 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                     if (!supervisor(dc))
                         goto priv_insn;
 #endif
-                    save_state(dc, cpu_cond);
+                    save_state(dc);
                     gen_st_asi(cpu_val, cpu_addr, insn, 2);
                     dc->npc = DYNAMIC_PC;
                     break;
@@ -5037,8 +5002,8 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                     if (rd & 1)
                         goto illegal_insn;
                     else {
-                        save_state(dc, cpu_cond);
-                        gen_stda_asi(cpu_val, cpu_addr, insn, rd);
+                        save_state(dc);
+                        gen_stda_asi(dc, cpu_val, cpu_addr, insn, rd);
                     }
                     break;
 #endif
@@ -5048,7 +5013,7 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                     tcg_gen_qemu_st64(cpu_val, cpu_addr, dc->mem_idx);
                     break;
                 case 0x1e: /* V9 stxa */
-                    save_state(dc, cpu_cond);
+                    save_state(dc);
                     gen_st_asi(cpu_val, cpu_addr, insn, 8);
                     dc->npc = DYNAMIC_PC;
                     break;
@@ -5057,28 +5022,34 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                     goto illegal_insn;
                 }
             } else if (xop > 0x23 && xop < 0x28) {
-                if (gen_trap_ifnofpu(dc, cpu_cond))
+                if (gen_trap_ifnofpu(dc)) {
                     goto jmp_insn;
-                save_state(dc, cpu_cond);
+                }
+                save_state(dc);
                 switch (xop) {
                 case 0x24: /* stf, store fpreg */
-                    gen_address_mask(dc, cpu_addr);
-                    cpu_src1_32 = gen_load_fpr_F(dc, rd);
-                    tcg_gen_ext_i32_tl(cpu_tmp0, cpu_src1_32);
-                    tcg_gen_qemu_st32(cpu_tmp0, cpu_addr, dc->mem_idx);
+                    {
+                        TCGv t = get_temp_tl(dc);
+                        gen_address_mask(dc, cpu_addr);
+                        cpu_src1_32 = gen_load_fpr_F(dc, rd);
+                        tcg_gen_ext_i32_tl(t, cpu_src1_32);
+                        tcg_gen_qemu_st32(t, cpu_addr, dc->mem_idx);
+                    }
                     break;
                 case 0x25: /* stfsr, V9 stxfsr */
+                    {
+                        TCGv t = get_temp_tl(dc);
+
+                        tcg_gen_ld_tl(t, cpu_env, offsetof(CPUSPARCState, fsr));
 #ifdef TARGET_SPARC64
-                    gen_address_mask(dc, cpu_addr);
-                    tcg_gen_ld_i64(cpu_tmp64, cpu_env, offsetof(CPUSPARCState, fsr));
-                    if (rd == 1)
-                        tcg_gen_qemu_st64(cpu_tmp64, cpu_addr, dc->mem_idx);
-                    else
-                        tcg_gen_qemu_st32(cpu_tmp64, cpu_addr, dc->mem_idx);
-#else
-                    tcg_gen_ld_i32(cpu_tmp32, cpu_env, offsetof(CPUSPARCState, fsr));
-                    tcg_gen_qemu_st32(cpu_tmp32, cpu_addr, dc->mem_idx);
+                        gen_address_mask(dc, cpu_addr);
+                        if (rd == 1) {
+                            tcg_gen_qemu_st64(t, cpu_addr, dc->mem_idx);
+                            break;
+                        }
 #endif
+                        tcg_gen_qemu_st32(t, cpu_addr, dc->mem_idx);
+                    }
                     break;
                 case 0x26:
 #ifdef TARGET_SPARC64
@@ -5101,8 +5072,9 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
 #else
                     if (!supervisor(dc))
                         goto priv_insn;
-                    if (gen_trap_ifnofpu(dc, cpu_cond))
+                    if (gen_trap_ifnofpu(dc)) {
                         goto jmp_insn;
+                    }
                     goto nfq_insn;
 #endif
 #endif
@@ -5115,11 +5087,11 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                     goto illegal_insn;
                 }
             } else if (xop > 0x33 && xop < 0x3f) {
-                save_state(dc, cpu_cond);
+                save_state(dc);
                 switch (xop) {
 #ifdef TARGET_SPARC64
                 case 0x34: /* V9 stfa */
-                    if (gen_trap_ifnofpu(dc, cpu_cond)) {
+                    if (gen_trap_ifnofpu(dc)) {
                         goto jmp_insn;
                     }
                     gen_stf_asi(cpu_addr, insn, 4, rd);
@@ -5129,7 +5101,7 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                         TCGv_i32 r_const;
 
                         CHECK_FPU_FEATURE(dc, FLOAT128);
-                        if (gen_trap_ifnofpu(dc, cpu_cond)) {
+                        if (gen_trap_ifnofpu(dc)) {
                             goto jmp_insn;
                         }
                         r_const = tcg_const_i32(7);
@@ -5139,18 +5111,20 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                     }
                     break;
                 case 0x37: /* V9 stdfa */
-                    if (gen_trap_ifnofpu(dc, cpu_cond)) {
+                    if (gen_trap_ifnofpu(dc)) {
                         goto jmp_insn;
                     }
                     gen_stf_asi(cpu_addr, insn, 8, DFPREG(rd));
                     break;
                 case 0x3c: /* V9 casa */
-                    gen_cas_asi(cpu_val, cpu_addr, cpu_src2, insn, rd);
-                    gen_movl_TN_reg(rd, cpu_val);
+                    rs2 = GET_FIELD(insn, 27, 31);
+                    cpu_src2 = gen_load_gpr(dc, rs2);
+                    gen_cas_asi(dc, cpu_addr, cpu_src2, insn, rd);
                     break;
                 case 0x3e: /* V9 casxa */
-                    gen_casx_asi(cpu_val, cpu_addr, cpu_src2, insn, rd);
-                    gen_movl_TN_reg(rd, cpu_val);
+                    rs2 = GET_FIELD(insn, 27, 31);
+                    cpu_src2 = gen_load_gpr(dc, rs2);
+                    gen_casx_asi(dc, cpu_addr, cpu_src2, insn, rd);
                     break;
 #else
                 case 0x34: /* stc */
@@ -5162,8 +5136,9 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
                 default:
                     goto illegal_insn;
                 }
-            } else
+            } else {
                 goto illegal_insn;
+            }
         }
         break;
     }
@@ -5185,7 +5160,7 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
     {
         TCGv_i32 r_const;
 
-        save_state(dc, cpu_cond);
+        save_state(dc);
         r_const = tcg_const_i32(TT_ILL_INSN);
         gen_helper_raise_exception(cpu_env, r_const);
         tcg_temp_free_i32(r_const);
@@ -5196,7 +5171,7 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
     {
         TCGv_i32 r_const;
 
-        save_state(dc, cpu_cond);
+        save_state(dc);
         r_const = tcg_const_i32(TT_UNIMP_FLUSH);
         gen_helper_raise_exception(cpu_env, r_const);
         tcg_temp_free_i32(r_const);
@@ -5208,7 +5183,7 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
     {
         TCGv_i32 r_const;
 
-        save_state(dc, cpu_cond);
+        save_state(dc);
         r_const = tcg_const_i32(TT_PRIV_INSN);
         gen_helper_raise_exception(cpu_env, r_const);
         tcg_temp_free_i32(r_const);
@@ -5217,13 +5192,13 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
     goto egress;
 #endif
  nfpu_insn:
-    save_state(dc, cpu_cond);
+    save_state(dc);
     gen_op_fpexception_im(FSR_FTT_UNIMPFPOP);
     dc->is_br = 1;
     goto egress;
 #if !defined(CONFIG_USER_ONLY) && !defined(TARGET_SPARC64)
  nfq_insn:
-    save_state(dc, cpu_cond);
+    save_state(dc);
     gen_op_fpexception_im(FSR_FTT_SEQ_ERROR);
     dc->is_br = 1;
     goto egress;
@@ -5233,7 +5208,7 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
     {
         TCGv r_const;
 
-        save_state(dc, cpu_cond);
+        save_state(dc);
         r_const = tcg_const_i32(TT_NCP_INSN);
         gen_helper_raise_exception(cpu_env, r_const);
         tcg_temp_free(r_const);
@@ -5242,8 +5217,6 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
     goto egress;
 #endif
  egress:
-    tcg_temp_free(cpu_tmp1);
-    tcg_temp_free(cpu_tmp2);
     if (dc->n_t32 != 0) {
         int i;
         for (i = dc->n_t32 - 1; i >= 0; --i) {
@@ -5251,6 +5224,13 @@ static void disas_sparc_insn(DisasContext * dc, unsigned int insn)
         }
         dc->n_t32 = 0;
     }
+    if (dc->n_ttl != 0) {
+        int i;
+        for (i = dc->n_ttl - 1; i >= 0; --i) {
+            tcg_temp_free(dc->ttl[i]);
+        }
+        dc->n_ttl = 0;
+    }
 }
 
 static inline void gen_intermediate_code_internal(TranslationBlock * tb,
@@ -5277,17 +5257,7 @@ static inline void gen_intermediate_code_internal(TranslationBlock * tb,
     dc->fpu_enabled = tb_fpu_enabled(tb->flags);
     dc->address_mask_32bit = tb_am_enabled(tb->flags);
     dc->singlestep = (env->singlestep_enabled || singlestep);
-    gen_opc_end = gen_opc_buf + OPC_MAX_SIZE;
-
-    cpu_tmp0 = tcg_temp_new();
-    cpu_tmp32 = tcg_temp_new_i32();
-    cpu_tmp64 = tcg_temp_new_i64();
-
-    cpu_dst = tcg_temp_local_new();
-
-    // loads and stores
-    cpu_val = tcg_temp_local_new();
-    cpu_addr = tcg_temp_local_new();
+    gen_opc_end = tcg_ctx.gen_opc_buf + OPC_MAX_SIZE;
 
     num_insns = 0;
     max_insns = tb->cflags & CF_COUNT_MASK;
@@ -5299,7 +5269,7 @@ static inline void gen_intermediate_code_internal(TranslationBlock * tb,
             QTAILQ_FOREACH(bp, &env->breakpoints, entry) {
                 if (bp->pc == dc->pc) {
                     if (dc->pc != pc_start)
-                        save_state(dc, cpu_cond);
+                        save_state(dc);
                     gen_helper_debug(cpu_env);
                     tcg_gen_exit_tb(0);
                     dc->is_br = 1;
@@ -5309,7 +5279,7 @@ static inline void gen_intermediate_code_internal(TranslationBlock * tb,
         }
         if (spc) {
             qemu_log("Search PC...\n");
-            j = gen_opc_ptr - gen_opc_buf;
+            j = tcg_ctx.gen_opc_ptr - tcg_ctx.gen_opc_buf;
             if (lj < j) {
                 lj++;
                 while (lj < j)
@@ -5324,6 +5294,7 @@ static inline void gen_intermediate_code_internal(TranslationBlock * tb,
             gen_io_start();
         last_pc = dc->pc;
         insn = cpu_ldl_code(env, dc->pc);
+
         disas_sparc_insn(dc, insn);
         num_insns++;
 
@@ -5341,36 +5312,31 @@ static inline void gen_intermediate_code_internal(TranslationBlock * tb,
         if (dc->singlestep) {
             break;
         }
-    } while ((gen_opc_ptr < gen_opc_end) &&
+    } while ((tcg_ctx.gen_opc_ptr < gen_opc_end) &&
              (dc->pc - pc_start) < (TARGET_PAGE_SIZE - 32) &&
              num_insns < max_insns);
 
  exit_gen_loop:
-    tcg_temp_free(cpu_addr);
-    tcg_temp_free(cpu_val);
-    tcg_temp_free(cpu_dst);
-    tcg_temp_free_i64(cpu_tmp64);
-    tcg_temp_free_i32(cpu_tmp32);
-    tcg_temp_free(cpu_tmp0);
-
-    if (tb->cflags & CF_LAST_IO)
+    if (tb->cflags & CF_LAST_IO) {
         gen_io_end();
+    }
     if (!dc->is_br) {
         if (dc->pc != DYNAMIC_PC &&
             (dc->npc != DYNAMIC_PC && dc->npc != JUMP_PC)) {
             /* static PC and NPC: we can use direct chaining */
             gen_goto_tb(dc, 0, dc->pc, dc->npc);
         } else {
-            if (dc->pc != DYNAMIC_PC)
+            if (dc->pc != DYNAMIC_PC) {
                 tcg_gen_movi_tl(cpu_pc, dc->pc);
-            save_npc(dc, cpu_cond);
+            }
+            save_npc(dc);
             tcg_gen_exit_tb(0);
         }
     }
     gen_icount_end(tb, num_insns);
-    *gen_opc_ptr = INDEX_op_end;
+    *tcg_ctx.gen_opc_ptr = INDEX_op_end;
     if (spc) {
-        j = gen_opc_ptr - gen_opc_buf;
+        j = tcg_ctx.gen_opc_ptr - tcg_ctx.gen_opc_buf;
         lj++;
         while (lj <= j)
             gen_opc_instr_start[lj++] = 0;
@@ -5387,7 +5353,7 @@ static inline void gen_intermediate_code_internal(TranslationBlock * tb,
     if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
         qemu_log("--------------\n");
         qemu_log("IN: %s\n", lookup_symbol(pc_start));
-        log_target_disas(pc_start, last_pc + 4 - pc_start, 0);
+        log_target_disas(env, pc_start, last_pc + 4 - pc_start, 0);
         qemu_log("\n");
     }
 #endif
@@ -5526,9 +5492,4 @@ void restore_state_to_opc(CPUSPARCState *env, TranslationBlock *tb, int pc_pos)
     } else {
         env->npc = npc;
     }
-
-    /* flush pending conditional evaluations before exposing cpu state */
-    if (CC_OP != CC_OP_FLAGS) {
-        helper_compute_psr(env);
-    }
 }
diff --git a/target-unicore32/Makefile.objs b/target-unicore32/Makefile.objs
index 777f01fef..8e143da93 100644
--- a/target-unicore32/Makefile.objs
+++ b/target-unicore32/Makefile.objs
@@ -2,5 +2,3 @@ obj-y += translate.o op_helper.o helper.o cpu.o
 obj-y += ucf64_helper.o
 
 obj-$(CONFIG_SOFTMMU) += machine.o softmmu.o
-
-$(obj)/op_helper.o: QEMU_CFLAGS += $(HELPER_CFLAGS)
diff --git a/target-unicore32/cpu.c b/target-unicore32/cpu.c
index 3425bbeac..884c10101 100644
--- a/target-unicore32/cpu.c
+++ b/target-unicore32/cpu.c
@@ -12,7 +12,7 @@
  * or (at your option) any later version.
  */
 
-#include "cpu-qom.h"
+#include "cpu.h"
 #include "qemu-common.h"
 
 static inline void set_feature(CPUUniCore32State *env, int feature)
diff --git a/target-unicore32/cpu.h b/target-unicore32/cpu.h
index 06508a127..676c5d9d9 100644
--- a/target-unicore32/cpu.h
+++ b/target-unicore32/cpu.h
@@ -181,8 +181,10 @@ void uc32_translate_init(void);
 void do_interrupt(CPUUniCore32State *);
 void switch_mode(CPUUniCore32State *, int);
 
-static inline bool cpu_has_work(CPUUniCore32State *env)
+static inline bool cpu_has_work(CPUState *cpu)
 {
+    CPUUniCore32State *env = &UNICORE32_CPU(cpu)->env;
+
     return env->interrupt_request &
         (CPU_INTERRUPT_HARD | CPU_INTERRUPT_EXITTB);
 }
diff --git a/target-unicore32/helper.h b/target-unicore32/helper.h
index 305318ae5..a4b81494a 100644
--- a/target-unicore32/helper.h
+++ b/target-unicore32/helper.h
@@ -17,26 +17,26 @@ DEF_HELPER_1(cp1_putc, void, i32)
 DEF_HELPER_1(clz, i32, i32)
 DEF_HELPER_1(clo, i32, i32)
 
-DEF_HELPER_1(exception, void, i32)
+DEF_HELPER_2(exception, void, env, i32)
 
-DEF_HELPER_2(asr_write, void, i32, i32)
-DEF_HELPER_0(asr_read, i32)
+DEF_HELPER_3(asr_write, void, env, i32, i32)
+DEF_HELPER_1(asr_read, i32, env)
 
-DEF_HELPER_1(get_user_reg, i32, i32)
-DEF_HELPER_2(set_user_reg, void, i32, i32)
+DEF_HELPER_2(get_user_reg, i32, env, i32)
+DEF_HELPER_3(set_user_reg, void, env, i32, i32)
 
-DEF_HELPER_2(add_cc, i32, i32, i32)
-DEF_HELPER_2(adc_cc, i32, i32, i32)
-DEF_HELPER_2(sub_cc, i32, i32, i32)
-DEF_HELPER_2(sbc_cc, i32, i32, i32)
+DEF_HELPER_3(add_cc, i32, env, i32, i32)
+DEF_HELPER_3(adc_cc, i32, env, i32, i32)
+DEF_HELPER_3(sub_cc, i32, env, i32, i32)
+DEF_HELPER_3(sbc_cc, i32, env, i32, i32)
 
 DEF_HELPER_2(shl, i32, i32, i32)
 DEF_HELPER_2(shr, i32, i32, i32)
 DEF_HELPER_2(sar, i32, i32, i32)
-DEF_HELPER_2(shl_cc, i32, i32, i32)
-DEF_HELPER_2(shr_cc, i32, i32, i32)
-DEF_HELPER_2(sar_cc, i32, i32, i32)
-DEF_HELPER_2(ror_cc, i32, i32, i32)
+DEF_HELPER_3(shl_cc, i32, env, i32, i32)
+DEF_HELPER_3(shr_cc, i32, env, i32, i32)
+DEF_HELPER_3(sar_cc, i32, env, i32, i32)
+DEF_HELPER_3(ror_cc, i32, env, i32, i32)
 
 DEF_HELPER_1(ucf64_get_fpscr, i32, env)
 DEF_HELPER_2(ucf64_set_fpscr, void, env, i32)
diff --git a/target-unicore32/op_helper.c b/target-unicore32/op_helper.c
index c63789d6c..f474d1b59 100644
--- a/target-unicore32/op_helper.c
+++ b/target-unicore32/op_helper.c
@@ -9,19 +9,18 @@
  * later version. See the COPYING file in the top-level directory.
  */
 #include "cpu.h"
-#include "dyngen-exec.h"
 #include "helper.h"
 
 #define SIGNBIT (uint32_t)0x80000000
 #define SIGNBIT64 ((uint64_t)1 << 63)
 
-void HELPER(exception)(uint32_t excp)
+void HELPER(exception)(CPUUniCore32State *env, uint32_t excp)
 {
     env->exception_index = excp;
     cpu_loop_exit(env);
 }
 
-static target_ulong asr_read(void)
+static target_ulong asr_read(CPUUniCore32State *env)
 {
     int ZF;
     ZF = (env->ZF == 0);
@@ -29,24 +28,18 @@ static target_ulong asr_read(void)
         (env->CF << 29) | ((env->VF & 0x80000000) >> 3);
 }
 
-target_ulong cpu_asr_read(CPUUniCore32State *env1)
+target_ulong cpu_asr_read(CPUUniCore32State *env)
 {
-    CPUUniCore32State *saved_env;
-    target_ulong ret;
-
-    saved_env = env;
-    env = env1;
-    ret = asr_read();
-    env = saved_env;
-    return ret;
+    return asr_read(env);
 }
 
-target_ulong HELPER(asr_read)(void)
+target_ulong HELPER(asr_read)(CPUUniCore32State *env)
 {
-    return asr_read();
+    return asr_read(env);
 }
 
-static void asr_write(target_ulong val, target_ulong mask)
+static void asr_write(CPUUniCore32State *env, target_ulong val,
+                      target_ulong mask)
 {
     if (mask & ASR_NZCV) {
         env->ZF = (~val) & ASR_Z;
@@ -62,23 +55,19 @@ static void asr_write(target_ulong val, target_ulong mask)
     env->uncached_asr = (env->uncached_asr & ~mask) | (val & mask);
 }
 
-void cpu_asr_write(CPUUniCore32State *env1, target_ulong val, target_ulong mask)
+void cpu_asr_write(CPUUniCore32State *env, target_ulong val, target_ulong mask)
 {
-    CPUUniCore32State *saved_env;
-
-    saved_env = env;
-    env = env1;
-    asr_write(val, mask);
-    env = saved_env;
+    asr_write(env, val, mask);
 }
 
-void HELPER(asr_write)(target_ulong val, target_ulong mask)
+void HELPER(asr_write)(CPUUniCore32State *env, target_ulong val,
+                       target_ulong mask)
 {
-    asr_write(val, mask);
+    asr_write(env, val, mask);
 }
 
 /* Access to user mode registers from privileged modes.  */
-uint32_t HELPER(get_user_reg)(uint32_t regno)
+uint32_t HELPER(get_user_reg)(CPUUniCore32State *env, uint32_t regno)
 {
     uint32_t val;
 
@@ -92,7 +81,7 @@ uint32_t HELPER(get_user_reg)(uint32_t regno)
     return val;
 }
 
-void HELPER(set_user_reg)(uint32_t regno, uint32_t val)
+void HELPER(set_user_reg)(CPUUniCore32State *env, uint32_t regno, uint32_t val)
 {
     if (regno == 29) {
         env->banked_r29[0] = val;
@@ -107,7 +96,7 @@ void HELPER(set_user_reg)(uint32_t regno, uint32_t val)
    The only way to do that in TCG is a conditional branch, which clobbers
    all our temporaries.  For now implement these as helper functions.  */
 
-uint32_t HELPER(add_cc)(uint32_t a, uint32_t b)
+uint32_t HELPER(add_cc)(CPUUniCore32State *env, uint32_t a, uint32_t b)
 {
     uint32_t result;
     result = a + b;
@@ -117,7 +106,7 @@ uint32_t HELPER(add_cc)(uint32_t a, uint32_t b)
     return result;
 }
 
-uint32_t HELPER(adc_cc)(uint32_t a, uint32_t b)
+uint32_t HELPER(adc_cc)(CPUUniCore32State *env, uint32_t a, uint32_t b)
 {
     uint32_t result;
     if (!env->CF) {
@@ -132,7 +121,7 @@ uint32_t HELPER(adc_cc)(uint32_t a, uint32_t b)
     return result;
 }
 
-uint32_t HELPER(sub_cc)(uint32_t a, uint32_t b)
+uint32_t HELPER(sub_cc)(CPUUniCore32State *env, uint32_t a, uint32_t b)
 {
     uint32_t result;
     result = a - b;
@@ -142,7 +131,7 @@ uint32_t HELPER(sub_cc)(uint32_t a, uint32_t b)
     return result;
 }
 
-uint32_t HELPER(sbc_cc)(uint32_t a, uint32_t b)
+uint32_t HELPER(sbc_cc)(CPUUniCore32State *env, uint32_t a, uint32_t b)
 {
     uint32_t result;
     if (!env->CF) {
@@ -186,7 +175,7 @@ uint32_t HELPER(sar)(uint32_t x, uint32_t i)
     return (int32_t)x >> shift;
 }
 
-uint32_t HELPER(shl_cc)(uint32_t x, uint32_t i)
+uint32_t HELPER(shl_cc)(CPUUniCore32State *env, uint32_t x, uint32_t i)
 {
     int shift = i & 0xff;
     if (shift >= 32) {
@@ -203,7 +192,7 @@ uint32_t HELPER(shl_cc)(uint32_t x, uint32_t i)
     return x;
 }
 
-uint32_t HELPER(shr_cc)(uint32_t x, uint32_t i)
+uint32_t HELPER(shr_cc)(CPUUniCore32State *env, uint32_t x, uint32_t i)
 {
     int shift = i & 0xff;
     if (shift >= 32) {
@@ -220,7 +209,7 @@ uint32_t HELPER(shr_cc)(uint32_t x, uint32_t i)
     return x;
 }
 
-uint32_t HELPER(sar_cc)(uint32_t x, uint32_t i)
+uint32_t HELPER(sar_cc)(CPUUniCore32State *env, uint32_t x, uint32_t i)
 {
     int shift = i & 0xff;
     if (shift >= 32) {
@@ -233,7 +222,7 @@ uint32_t HELPER(sar_cc)(uint32_t x, uint32_t i)
     return x;
 }
 
-uint32_t HELPER(ror_cc)(uint32_t x, uint32_t i)
+uint32_t HELPER(ror_cc)(CPUUniCore32State *env, uint32_t x, uint32_t i)
 {
     int shift1, shift;
     shift1 = i & 0xff;
@@ -264,16 +253,13 @@ uint32_t HELPER(ror_cc)(uint32_t x, uint32_t i)
 #define SHIFT 3
 #include "softmmu_template.h"
 
-void tlb_fill(CPUUniCore32State *env1, target_ulong addr, int is_write,
-        int mmu_idx, uintptr_t retaddr)
+void tlb_fill(CPUUniCore32State *env, target_ulong addr, int is_write,
+              int mmu_idx, uintptr_t retaddr)
 {
     TranslationBlock *tb;
-    CPUUniCore32State *saved_env;
     unsigned long pc;
     int ret;
 
-    saved_env = env;
-    env = env1;
     ret = uc32_cpu_handle_mmu_fault(env, addr, is_write, mmu_idx);
     if (unlikely(ret)) {
         if (retaddr) {
@@ -287,6 +273,5 @@ void tlb_fill(CPUUniCore32State *env1, target_ulong addr, int is_write,
         }
         cpu_loop_exit(env);
     }
-    env = saved_env;
 }
 #endif
diff --git a/target-unicore32/softmmu.c b/target-unicore32/softmmu.c
index 373f94b27..fc27100f2 100644
--- a/target-unicore32/softmmu.c
+++ b/target-unicore32/softmmu.c
@@ -31,7 +31,7 @@
 
 
 /* Map CPU modes onto saved register banks.  */
-static inline int bank_number(int mode)
+static inline int bank_number(CPUUniCore32State *env, int mode)
 {
     switch (mode) {
     case ASR_MODE_USER:
@@ -46,7 +46,7 @@ static inline int bank_number(int mode)
     case ASR_MODE_INTR:
         return 4;
     }
-    cpu_abort(cpu_single_env, "Bad mode %x\n", mode);
+    cpu_abort(env, "Bad mode %x\n", mode);
     return -1;
 }
 
@@ -60,12 +60,12 @@ void switch_mode(CPUUniCore32State *env, int mode)
         return;
     }
 
-    i = bank_number(old_mode);
+    i = bank_number(env, old_mode);
     env->banked_r29[i] = env->regs[29];
     env->banked_r30[i] = env->regs[30];
     env->banked_bsr[i] = env->bsr;
 
-    i = bank_number(mode);
+    i = bank_number(env, mode);
     env->regs[29] = env->banked_r29[i];
     env->regs[30] = env->banked_r30[i];
     env->bsr = env->banked_bsr[i];
@@ -259,7 +259,7 @@ int uc32_cpu_handle_mmu_fault(CPUUniCore32State *env, target_ulong address,
     return ret;
 }
 
-target_phys_addr_t cpu_get_phys_page_debug(CPUUniCore32State *env,
+hwaddr cpu_get_phys_page_debug(CPUUniCore32State *env,
         target_ulong addr)
 {
     cpu_abort(env, "%s not supported yet\n", __func__);
diff --git a/target-unicore32/translate.c b/target-unicore32/translate.c
index 188bf8c52..052bb45d7 100644
--- a/target-unicore32/translate.c
+++ b/target-unicore32/translate.c
@@ -253,7 +253,7 @@ static void disas_ocd_insn(CPUUniCore32State *env, DisasContext *s,
 static inline void gen_set_asr(TCGv var, uint32_t mask)
 {
     TCGv tmp_mask = tcg_const_i32(mask);
-    gen_helper_asr_write(var, tmp_mask);
+    gen_helper_asr_write(cpu_env, var, tmp_mask);
     tcg_temp_free_i32(tmp_mask);
 }
 /* Set NZCV flags from the high 4 bits of var.  */
@@ -263,7 +263,7 @@ static void gen_exception(int excp)
 {
     TCGv tmp = new_tmp();
     tcg_gen_movi_i32(tmp, excp);
-    gen_helper_exception(tmp);
+    gen_helper_exception(cpu_env, tmp);
     dead_tmp(tmp);
 }
 
@@ -416,16 +416,16 @@ static inline void gen_uc32_shift_reg(TCGv var, int shiftop,
     if (flags) {
         switch (shiftop) {
         case 0:
-            gen_helper_shl_cc(var, var, shift);
+            gen_helper_shl_cc(var, cpu_env, var, shift);
             break;
         case 1:
-            gen_helper_shr_cc(var, var, shift);
+            gen_helper_shr_cc(var, cpu_env, var, shift);
             break;
         case 2:
-            gen_helper_sar_cc(var, var, shift);
+            gen_helper_sar_cc(var, cpu_env, var, shift);
             break;
         case 3:
-            gen_helper_ror_cc(var, var, shift);
+            gen_helper_ror_cc(var, cpu_env, var, shift);
             break;
         }
     } else {
@@ -1323,11 +1323,11 @@ static void do_datap(CPUUniCore32State *env, DisasContext *s, uint32_t insn)
             if (IS_USER(s)) {
                 ILLEGAL;
             }
-            gen_helper_sub_cc(tmp, tmp, tmp2);
+            gen_helper_sub_cc(tmp, cpu_env, tmp, tmp2);
             gen_exception_return(s, tmp);
         } else {
             if (UCOP_SET_S) {
-                gen_helper_sub_cc(tmp, tmp, tmp2);
+                gen_helper_sub_cc(tmp, cpu_env, tmp, tmp2);
             } else {
                 tcg_gen_sub_i32(tmp, tmp, tmp2);
             }
@@ -1336,7 +1336,7 @@ static void do_datap(CPUUniCore32State *env, DisasContext *s, uint32_t insn)
         break;
     case 0x03:
         if (UCOP_SET_S) {
-            gen_helper_sub_cc(tmp, tmp2, tmp);
+            gen_helper_sub_cc(tmp, cpu_env, tmp2, tmp);
         } else {
             tcg_gen_sub_i32(tmp, tmp2, tmp);
         }
@@ -1344,7 +1344,7 @@ static void do_datap(CPUUniCore32State *env, DisasContext *s, uint32_t insn)
         break;
     case 0x04:
         if (UCOP_SET_S) {
-            gen_helper_add_cc(tmp, tmp, tmp2);
+            gen_helper_add_cc(tmp, cpu_env, tmp, tmp2);
         } else {
             tcg_gen_add_i32(tmp, tmp, tmp2);
         }
@@ -1352,7 +1352,7 @@ static void do_datap(CPUUniCore32State *env, DisasContext *s, uint32_t insn)
         break;
     case 0x05:
         if (UCOP_SET_S) {
-            gen_helper_adc_cc(tmp, tmp, tmp2);
+            gen_helper_adc_cc(tmp, cpu_env, tmp, tmp2);
         } else {
             gen_add_carry(tmp, tmp, tmp2);
         }
@@ -1360,7 +1360,7 @@ static void do_datap(CPUUniCore32State *env, DisasContext *s, uint32_t insn)
         break;
     case 0x06:
         if (UCOP_SET_S) {
-            gen_helper_sbc_cc(tmp, tmp, tmp2);
+            gen_helper_sbc_cc(tmp, cpu_env, tmp, tmp2);
         } else {
             gen_sub_carry(tmp, tmp, tmp2);
         }
@@ -1368,7 +1368,7 @@ static void do_datap(CPUUniCore32State *env, DisasContext *s, uint32_t insn)
         break;
     case 0x07:
         if (UCOP_SET_S) {
-            gen_helper_sbc_cc(tmp, tmp2, tmp);
+            gen_helper_sbc_cc(tmp, cpu_env, tmp2, tmp);
         } else {
             gen_sub_carry(tmp, tmp2, tmp);
         }
@@ -1390,13 +1390,13 @@ static void do_datap(CPUUniCore32State *env, DisasContext *s, uint32_t insn)
         break;
     case 0x0a:
         if (UCOP_SET_S) {
-            gen_helper_sub_cc(tmp, tmp, tmp2);
+            gen_helper_sub_cc(tmp, cpu_env, tmp, tmp2);
         }
         dead_tmp(tmp);
         break;
     case 0x0b:
         if (UCOP_SET_S) {
-            gen_helper_add_cc(tmp, tmp, tmp2);
+            gen_helper_add_cc(tmp, cpu_env, tmp, tmp2);
         }
         dead_tmp(tmp);
         break;
@@ -1536,7 +1536,7 @@ static void do_misc(CPUUniCore32State *env, DisasContext *s, uint32_t insn)
             tmp = load_cpu_field(bsr);
         } else {
             tmp = new_tmp();
-            gen_helper_asr_read(tmp);
+            gen_helper_asr_read(tmp, cpu_env);
         }
         store_reg(s, UCOP_REG_D, tmp);
         return;
@@ -1760,7 +1760,7 @@ static void do_ldst_m(CPUUniCore32State *env, DisasContext *s, uint32_t insn)
                     gen_bx(s, tmp);
                 } else if (user) {
                     tmp2 = tcg_const_i32(reg);
-                    gen_helper_set_user_reg(tmp2, tmp);
+                    gen_helper_set_user_reg(cpu_env, tmp2, tmp);
                     tcg_temp_free_i32(tmp2);
                     dead_tmp(tmp);
                 } else if (reg == UCOP_REG_N) {
@@ -1778,7 +1778,7 @@ static void do_ldst_m(CPUUniCore32State *env, DisasContext *s, uint32_t insn)
                 } else if (user) {
                     tmp = new_tmp();
                     tmp2 = tcg_const_i32(reg);
-                    gen_helper_get_user_reg(tmp, tmp2);
+                    gen_helper_get_user_reg(tmp, cpu_env, tmp2);
                     tcg_temp_free_i32(tmp2);
                 } else {
                     tmp = load_reg(s, reg);
@@ -1861,7 +1861,11 @@ static void disas_uc32_insn(CPUUniCore32State *env, DisasContext *s)
 {
     unsigned int insn;
 
-    insn = ldl_code(s->pc);
+    if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
+        tcg_gen_debug_insn_start(s->pc);
+    }
+
+    insn = cpu_ldl_code(env, s->pc);
     s->pc += 4;
 
     /* UniCore instructions class:
@@ -1928,8 +1932,6 @@ static void disas_uc32_insn(CPUUniCore32State *env, DisasContext *s)
         }
         ILLEGAL;
     }
-
-    return;
 }
 
 /* generate intermediate code in gen_opc_buf and gen_opparam_buf for
@@ -1954,7 +1956,7 @@ static inline void gen_intermediate_code_internal(CPUUniCore32State *env,
 
     dc->tb = tb;
 
-    gen_opc_end = gen_opc_buf + OPC_MAX_SIZE;
+    gen_opc_end = tcg_ctx.gen_opc_buf + OPC_MAX_SIZE;
 
     dc->is_jmp = DISAS_NEXT;
     dc->pc = pc_start;
@@ -1997,7 +1999,7 @@ static inline void gen_intermediate_code_internal(CPUUniCore32State *env,
             }
         }
         if (search_pc) {
-            j = gen_opc_ptr - gen_opc_buf;
+            j = tcg_ctx.gen_opc_ptr - tcg_ctx.gen_opc_buf;
             if (lj < j) {
                 lj++;
                 while (lj < j) {
@@ -2029,7 +2031,7 @@ static inline void gen_intermediate_code_internal(CPUUniCore32State *env,
          * Also stop translation when a page boundary is reached.  This
          * ensures prefetch aborts occur at the right place.  */
         num_insns++;
-    } while (!dc->is_jmp && gen_opc_ptr < gen_opc_end &&
+    } while (!dc->is_jmp && tcg_ctx.gen_opc_ptr < gen_opc_end &&
              !env->singlestep_enabled &&
              !singlestep &&
              dc->pc < next_page_start &&
@@ -2101,18 +2103,18 @@ static inline void gen_intermediate_code_internal(CPUUniCore32State *env,
 
 done_generating:
     gen_icount_end(tb, num_insns);
-    *gen_opc_ptr = INDEX_op_end;
+    *tcg_ctx.gen_opc_ptr = INDEX_op_end;
 
 #ifdef DEBUG_DISAS
     if (qemu_loglevel_mask(CPU_LOG_TB_IN_ASM)) {
         qemu_log("----------------\n");
         qemu_log("IN: %s\n", lookup_symbol(pc_start));
-        log_target_disas(pc_start, dc->pc - pc_start, 0);
+        log_target_disas(env, pc_start, dc->pc - pc_start, 0);
         qemu_log("\n");
     }
 #endif
     if (search_pc) {
-        j = gen_opc_ptr - gen_opc_buf;
+        j = tcg_ctx.gen_opc_ptr - tcg_ctx.gen_opc_buf;
         lj++;
         while (lj <= j) {
             gen_opc_instr_start[lj++] = 0;
diff --git a/target-xtensa/cpu.h b/target-xtensa/cpu.h
index 177094ae9..74e98883b 100644
--- a/target-xtensa/cpu.h
+++ b/target-xtensa/cpu.h
@@ -36,6 +36,7 @@
 #include "config.h"
 #include "qemu-common.h"
 #include "cpu-defs.h"
+#include "fpu/softfloat.h"
 
 #define TARGET_HAS_ICE 1
 
@@ -325,6 +326,8 @@ typedef struct CPUXtensaState {
     uint32_t sregs[256];
     uint32_t uregs[256];
     uint32_t phys_regs[MAX_NAREG];
+    float32 fregs[16];
+    float_status fp_status;
 
     xtensa_tlb_entry itlb[7][MAX_TLB_WAY_SIZE];
     xtensa_tlb_entry dtlb[10][MAX_TLB_WAY_SIZE];
@@ -465,6 +468,8 @@ static inline int cpu_mmu_index(CPUXtensaState *env)
 #define XTENSA_TBFLAG_LITBASE 0x8
 #define XTENSA_TBFLAG_DEBUG 0x10
 #define XTENSA_TBFLAG_ICOUNT 0x20
+#define XTENSA_TBFLAG_CPENABLE_MASK 0x3fc0
+#define XTENSA_TBFLAG_CPENABLE_SHIFT 6
 
 static inline void cpu_get_tb_cpu_state(CPUXtensaState *env, target_ulong *pc,
         target_ulong *cs_base, int *flags)
@@ -488,13 +493,18 @@ static inline void cpu_get_tb_cpu_state(CPUXtensaState *env, target_ulong *pc,
             *flags |= XTENSA_TBFLAG_ICOUNT;
         }
     }
+    if (xtensa_option_enabled(env->config, XTENSA_OPTION_COPROCESSOR)) {
+        *flags |= env->sregs[CPENABLE] << XTENSA_TBFLAG_CPENABLE_SHIFT;
+    }
 }
 
 #include "cpu-all.h"
 #include "exec-all.h"
 
-static inline int cpu_has_work(CPUXtensaState *env)
+static inline int cpu_has_work(CPUState *cpu)
 {
+    CPUXtensaState *env = &XTENSA_CPU(cpu)->env;
+
     return env->pending_irq_level;
 }
 
diff --git a/target-xtensa/helper.c b/target-xtensa/helper.c
index d5bb171fc..d94bae210 100644
--- a/target-xtensa/helper.c
+++ b/target-xtensa/helper.c
@@ -118,7 +118,7 @@ void xtensa_cpu_list(FILE *f, fprintf_function cpu_fprintf)
     }
 }
 
-target_phys_addr_t cpu_get_phys_page_debug(CPUXtensaState *env, target_ulong addr)
+hwaddr cpu_get_phys_page_debug(CPUXtensaState *env, target_ulong addr)
 {
     uint32_t paddr;
     uint32_t page_size;
diff --git a/target-xtensa/helper.h b/target-xtensa/helper.h
index 152fec044..1163c0983 100644
--- a/target-xtensa/helper.h
+++ b/target-xtensa/helper.h
@@ -5,8 +5,8 @@ DEF_HELPER_3(exception_cause, noreturn, env, i32, i32)
 DEF_HELPER_4(exception_cause_vaddr, noreturn, env, i32, i32, i32)
 DEF_HELPER_3(debug_exception, noreturn, env, i32, i32)
 
-DEF_HELPER_FLAGS_1(nsa, TCG_CALL_CONST | TCG_CALL_PURE, i32, i32)
-DEF_HELPER_FLAGS_1(nsau, TCG_CALL_CONST | TCG_CALL_PURE, i32, i32)
+DEF_HELPER_FLAGS_1(nsa, TCG_CALL_NO_RWG_SE, i32, i32)
+DEF_HELPER_FLAGS_1(nsau, TCG_CALL_NO_RWG_SE, i32, i32)
 DEF_HELPER_2(wsr_windowbase, void, env, i32)
 DEF_HELPER_4(entry, void, env, i32, i32, i32)
 DEF_HELPER_2(retw, i32, env, i32)
@@ -25,8 +25,8 @@ DEF_HELPER_2(advance_ccount, void, env, i32)
 DEF_HELPER_1(check_interrupts, void, env)
 
 DEF_HELPER_2(wsr_rasid, void, env, i32)
-DEF_HELPER_FLAGS_3(rtlb0, TCG_CALL_CONST | TCG_CALL_PURE, i32, env, i32, i32)
-DEF_HELPER_FLAGS_3(rtlb1, TCG_CALL_CONST | TCG_CALL_PURE, i32, env, i32, i32)
+DEF_HELPER_FLAGS_3(rtlb0, TCG_CALL_NO_RWG_SE, i32, env, i32, i32)
+DEF_HELPER_FLAGS_3(rtlb1, TCG_CALL_NO_RWG_SE, i32, env, i32, i32)
 DEF_HELPER_3(itlb, void, env, i32, i32)
 DEF_HELPER_3(ptlb, i32, env, i32, i32)
 DEF_HELPER_4(wtlb, void, env, i32, i32, i32)
@@ -36,4 +36,25 @@ DEF_HELPER_3(wsr_ibreaka, void, env, i32, i32)
 DEF_HELPER_3(wsr_dbreaka, void, env, i32, i32)
 DEF_HELPER_3(wsr_dbreakc, void, env, i32, i32)
 
+DEF_HELPER_2(wur_fcr, void, env, i32)
+DEF_HELPER_FLAGS_1(abs_s, TCG_CALL_NO_RWG_SE, f32, f32)
+DEF_HELPER_FLAGS_1(neg_s, TCG_CALL_NO_RWG_SE, f32, f32)
+DEF_HELPER_3(add_s, f32, env, f32, f32)
+DEF_HELPER_3(sub_s, f32, env, f32, f32)
+DEF_HELPER_3(mul_s, f32, env, f32, f32)
+DEF_HELPER_4(madd_s, f32, env, f32, f32, f32)
+DEF_HELPER_4(msub_s, f32, env, f32, f32, f32)
+DEF_HELPER_FLAGS_3(ftoi, TCG_CALL_NO_RWG_SE, i32, f32, i32, i32)
+DEF_HELPER_FLAGS_3(ftoui, TCG_CALL_NO_RWG_SE, i32, f32, i32, i32)
+DEF_HELPER_3(itof, f32, env, i32, i32)
+DEF_HELPER_3(uitof, f32, env, i32, i32)
+
+DEF_HELPER_4(un_s, void, env, i32, f32, f32)
+DEF_HELPER_4(oeq_s, void, env, i32, f32, f32)
+DEF_HELPER_4(ueq_s, void, env, i32, f32, f32)
+DEF_HELPER_4(olt_s, void, env, i32, f32, f32)
+DEF_HELPER_4(ult_s, void, env, i32, f32, f32)
+DEF_HELPER_4(ole_s, void, env, i32, f32, f32)
+DEF_HELPER_4(ule_s, void, env, i32, f32, f32)
+
 #include "def-helper.h"
diff --git a/target-xtensa/op_helper.c b/target-xtensa/op_helper.c
index 2659c0e00..ae0c09977 100644
--- a/target-xtensa/op_helper.c
+++ b/target-xtensa/op_helper.c
@@ -771,3 +771,137 @@ void HELPER(wsr_dbreakc)(CPUXtensaState *env, uint32_t i, uint32_t v)
     }
     env->sregs[DBREAKC + i] = v;
 }
+
+void HELPER(wur_fcr)(CPUXtensaState *env, uint32_t v)
+{
+    static const int rounding_mode[] = {
+        float_round_nearest_even,
+        float_round_to_zero,
+        float_round_up,
+        float_round_down,
+    };
+
+    env->uregs[FCR] = v & 0xfffff07f;
+    set_float_rounding_mode(rounding_mode[v & 3], &env->fp_status);
+}
+
+float32 HELPER(abs_s)(float32 v)
+{
+    return float32_abs(v);
+}
+
+float32 HELPER(neg_s)(float32 v)
+{
+    return float32_chs(v);
+}
+
+float32 HELPER(add_s)(CPUXtensaState *env, float32 a, float32 b)
+{
+    return float32_add(a, b, &env->fp_status);
+}
+
+float32 HELPER(sub_s)(CPUXtensaState *env, float32 a, float32 b)
+{
+    return float32_sub(a, b, &env->fp_status);
+}
+
+float32 HELPER(mul_s)(CPUXtensaState *env, float32 a, float32 b)
+{
+    return float32_mul(a, b, &env->fp_status);
+}
+
+float32 HELPER(madd_s)(CPUXtensaState *env, float32 a, float32 b, float32 c)
+{
+    return float32_muladd(b, c, a, 0,
+            &env->fp_status);
+}
+
+float32 HELPER(msub_s)(CPUXtensaState *env, float32 a, float32 b, float32 c)
+{
+    return float32_muladd(b, c, a, float_muladd_negate_product,
+            &env->fp_status);
+}
+
+uint32_t HELPER(ftoi)(float32 v, uint32_t rounding_mode, uint32_t scale)
+{
+    float_status fp_status = {0};
+
+    set_float_rounding_mode(rounding_mode, &fp_status);
+    return float32_to_int32(
+            float32_scalbn(v, scale, &fp_status), &fp_status);
+}
+
+uint32_t HELPER(ftoui)(float32 v, uint32_t rounding_mode, uint32_t scale)
+{
+    float_status fp_status = {0};
+    float32 res;
+
+    set_float_rounding_mode(rounding_mode, &fp_status);
+
+    res = float32_scalbn(v, scale, &fp_status);
+
+    if (float32_is_neg(v) && !float32_is_any_nan(v)) {
+        return float32_to_int32(res, &fp_status);
+    } else {
+        return float32_to_uint32(res, &fp_status);
+    }
+}
+
+float32 HELPER(itof)(CPUXtensaState *env, uint32_t v, uint32_t scale)
+{
+    return float32_scalbn(int32_to_float32(v, &env->fp_status),
+            (int32_t)scale, &env->fp_status);
+}
+
+float32 HELPER(uitof)(CPUXtensaState *env, uint32_t v, uint32_t scale)
+{
+    return float32_scalbn(uint32_to_float32(v, &env->fp_status),
+            (int32_t)scale, &env->fp_status);
+}
+
+static inline void set_br(CPUXtensaState *env, bool v, uint32_t br)
+{
+    if (v) {
+        env->sregs[BR] |= br;
+    } else {
+        env->sregs[BR] &= ~br;
+    }
+}
+
+void HELPER(un_s)(CPUXtensaState *env, uint32_t br, float32 a, float32 b)
+{
+    set_br(env, float32_unordered_quiet(a, b, &env->fp_status), br);
+}
+
+void HELPER(oeq_s)(CPUXtensaState *env, uint32_t br, float32 a, float32 b)
+{
+    set_br(env, float32_eq_quiet(a, b, &env->fp_status), br);
+}
+
+void HELPER(ueq_s)(CPUXtensaState *env, uint32_t br, float32 a, float32 b)
+{
+    int v = float32_compare_quiet(a, b, &env->fp_status);
+    set_br(env, v == float_relation_equal || v == float_relation_unordered, br);
+}
+
+void HELPER(olt_s)(CPUXtensaState *env, uint32_t br, float32 a, float32 b)
+{
+    set_br(env, float32_lt_quiet(a, b, &env->fp_status), br);
+}
+
+void HELPER(ult_s)(CPUXtensaState *env, uint32_t br, float32 a, float32 b)
+{
+    int v = float32_compare_quiet(a, b, &env->fp_status);
+    set_br(env, v == float_relation_less || v == float_relation_unordered, br);
+}
+
+void HELPER(ole_s)(CPUXtensaState *env, uint32_t br, float32 a, float32 b)
+{
+    set_br(env, float32_le_quiet(a, b, &env->fp_status), br);
+}
+
+void HELPER(ule_s)(CPUXtensaState *env, uint32_t br, float32 a, float32 b)
+{
+    int v = float32_compare_quiet(a, b, &env->fp_status);
+    set_br(env, v != float_relation_greater, br);
+}
diff --git a/target-xtensa/overlay_tool.h b/target-xtensa/overlay_tool.h
index a3a5650fb..e39505316 100644
--- a/target-xtensa/overlay_tool.h
+++ b/target-xtensa/overlay_tool.h
@@ -58,6 +58,7 @@
     XCHAL_OPTION(XCHAL_HAVE_SEXT, XTENSA_OPTION_MISC_OP_SEXT) | \
     XCHAL_OPTION(XCHAL_HAVE_CLAMPS, XTENSA_OPTION_MISC_OP_CLAMPS) | \
     XCHAL_OPTION(XCHAL_HAVE_CP, XTENSA_OPTION_COPROCESSOR) | \
+    XCHAL_OPTION(XCHAL_HAVE_BOOLEANS, XTENSA_OPTION_BOOLEAN) | \
     XCHAL_OPTION(XCHAL_HAVE_FP, XTENSA_OPTION_FP_COPROCESSOR) | \
     XCHAL_OPTION(XCHAL_HAVE_RELEASE_SYNC, XTENSA_OPTION_MP_SYNCHRO) | \
     XCHAL_OPTION(XCHAL_HAVE_S32C1I, XTENSA_OPTION_CONDITIONAL_STORE) | \
diff --git a/target-xtensa/translate.c b/target-xtensa/translate.c
index 1900bd5d4..e5a3f49a7 100644
--- a/target-xtensa/translate.c
+++ b/target-xtensa/translate.c
@@ -65,11 +65,14 @@ typedef struct DisasContext {
     bool debug;
     bool icount;
     TCGv_i32 next_icount;
+
+    unsigned cpenable;
 } DisasContext;
 
 static TCGv_ptr cpu_env;
 static TCGv_i32 cpu_pc;
 static TCGv_i32 cpu_R[16];
+static TCGv_i32 cpu_FR[16];
 static TCGv_i32 cpu_SR[256];
 static TCGv_i32 cpu_UR[256];
 
@@ -155,6 +158,12 @@ void xtensa_translate_init(void)
         "ar8", "ar9", "ar10", "ar11",
         "ar12", "ar13", "ar14", "ar15",
     };
+    static const char * const fregnames[] = {
+        "f0", "f1", "f2", "f3",
+        "f4", "f5", "f6", "f7",
+        "f8", "f9", "f10", "f11",
+        "f12", "f13", "f14", "f15",
+    };
     int i;
 
     cpu_env = tcg_global_reg_new_ptr(TCG_AREG0, "env");
@@ -167,6 +176,12 @@ void xtensa_translate_init(void)
                 regnames[i]);
     }
 
+    for (i = 0; i < 16; i++) {
+        cpu_FR[i] = tcg_global_mem_new_i32(TCG_AREG0,
+                offsetof(CPUXtensaState, fregs[i]),
+                fregnames[i]);
+    }
+
     for (i = 0; i < 256; ++i) {
         if (sregnames[i]) {
             cpu_SR[i] = tcg_global_mem_new_i32(TCG_AREG0,
@@ -318,6 +333,15 @@ static void gen_check_privilege(DisasContext *dc)
     }
 }
 
+static void gen_check_cpenable(DisasContext *dc, unsigned cp)
+{
+    if (option_enabled(dc, XTENSA_OPTION_COPROCESSOR) &&
+            !(dc->cpenable & (1 << cp))) {
+        gen_exception_cause(dc, COPROCESSOR0_DISABLED + cp);
+        dc->is_jmp = DISAS_UPDATE;
+    }
+}
+
 static void gen_jump_slot(DisasContext *dc, TCGv dest, int slot)
 {
     tcg_gen_mov_i32(cpu_pc, dest);
@@ -566,6 +590,13 @@ static void gen_wsr_dbreakc(DisasContext *dc, uint32_t sr, TCGv_i32 v)
     }
 }
 
+static void gen_wsr_cpenable(DisasContext *dc, uint32_t sr, TCGv_i32 v)
+{
+    tcg_gen_andi_i32(cpu_SR[sr], v, 0xff);
+    /* This can change tb->flags, so exit tb */
+    gen_jumpi_check_loop_end(dc, -1);
+}
+
 static void gen_wsr_intset(DisasContext *dc, uint32_t sr, TCGv_i32 v)
 {
     tcg_gen_andi_i32(cpu_SR[sr], v,
@@ -668,6 +699,7 @@ static void gen_wsr(DisasContext *dc, uint32_t sr, TCGv_i32 s)
         [DBREAKA + 1] = gen_wsr_dbreaka,
         [DBREAKC] = gen_wsr_dbreakc,
         [DBREAKC + 1] = gen_wsr_dbreakc,
+        [CPENABLE] = gen_wsr_cpenable,
         [INTSET] = gen_wsr_intset,
         [INTCLEAR] = gen_wsr_intclear,
         [INTENABLE] = gen_wsr_intenable,
@@ -692,6 +724,23 @@ static void gen_wsr(DisasContext *dc, uint32_t sr, TCGv_i32 s)
     }
 }
 
+static void gen_wur(uint32_t ur, TCGv_i32 s)
+{
+    switch (ur) {
+    case FCR:
+        gen_helper_wur_fcr(cpu_env, s);
+        break;
+
+    case FSR:
+        tcg_gen_andi_i32(cpu_UR[ur], s, 0xffffff80);
+        break;
+
+    default:
+        tcg_gen_mov_i32(cpu_UR[ur], s);
+        break;
+    }
+}
+
 static void gen_load_store_alignment(DisasContext *dc, int shift,
         TCGv_i32 addr, bool no_hw_alignment)
 {
@@ -761,7 +810,7 @@ static TCGv_i32 gen_mac16_m(TCGv_i32 v, bool hi, bool is_unsigned)
     return m;
 }
 
-static void disas_xtensa_insn(DisasContext *dc)
+static void disas_xtensa_insn(CPUXtensaState *env, DisasContext *dc)
 {
 #define HAS_OPTION_BITS(opt) do { \
         if (!option_bits_enabled(dc, opt)) { \
@@ -851,8 +900,8 @@ static void disas_xtensa_insn(DisasContext *dc)
 
 #define RSR_SR (b1)
 
-    uint8_t b0 = cpu_ldub_code(cpu_single_env, dc->pc);
-    uint8_t b1 = cpu_ldub_code(cpu_single_env, dc->pc + 1);
+    uint8_t b0 = cpu_ldub_code(env, dc->pc);
+    uint8_t b1 = cpu_ldub_code(env, dc->pc + 1);
     uint8_t b2 = 0;
 
     static const uint32_t B4CONST[] = {
@@ -868,7 +917,7 @@ static void disas_xtensa_insn(DisasContext *dc)
         HAS_OPTION(XTENSA_OPTION_CODE_DENSITY);
     } else {
         dc->next_pc = dc->pc + 3;
-        b2 = cpu_ldub_code(cpu_single_env, dc->pc + 2);
+        b2 = cpu_ldub_code(env, dc->pc + 2);
     }
 
     switch (OP0) {
@@ -1761,13 +1810,11 @@ static void disas_xtensa_insn(DisasContext *dc)
 
             case 15: /*WUR*/
                 gen_window_check1(dc, RRR_T);
-                {
-                    if (uregnames[RSR_SR]) {
-                        tcg_gen_mov_i32(cpu_UR[RSR_SR], cpu_R[RRR_T]);
-                    } else {
-                        qemu_log("WUR %d not implemented, ", RSR_SR);
-                        TBD();
-                    }
+                if (uregnames[RSR_SR]) {
+                    gen_wur(RSR_SR, cpu_R[RRR_T]);
+                } else {
+                    qemu_log("WUR %d not implemented, ", RSR_SR);
+                    TBD();
                 }
                 break;
 
@@ -1778,7 +1825,7 @@ static void disas_xtensa_insn(DisasContext *dc)
         case 5:
             gen_window_check2(dc, RRR_R, RRR_T);
             {
-                int shiftimm = RRR_S | (OP1 << 4);
+                int shiftimm = RRR_S | ((OP1 & 1) << 4);
                 int maskimm = (1 << (OP2 + 1)) - 1;
 
                 TCGv_i32 tmp = tcg_temp_new_i32();
@@ -1797,8 +1844,34 @@ static void disas_xtensa_insn(DisasContext *dc)
             break;
 
         case 8: /*LSCXp*/
-            HAS_OPTION(XTENSA_OPTION_COPROCESSOR);
-            TBD();
+            switch (OP2) {
+            case 0: /*LSXf*/
+            case 1: /*LSXUf*/
+            case 4: /*SSXf*/
+            case 5: /*SSXUf*/
+                HAS_OPTION(XTENSA_OPTION_FP_COPROCESSOR);
+                gen_window_check2(dc, RRR_S, RRR_T);
+                gen_check_cpenable(dc, 0);
+                {
+                    TCGv_i32 addr = tcg_temp_new_i32();
+                    tcg_gen_add_i32(addr, cpu_R[RRR_S], cpu_R[RRR_T]);
+                    gen_load_store_alignment(dc, 2, addr, false);
+                    if (OP2 & 0x4) {
+                        tcg_gen_qemu_st32(cpu_FR[RRR_R], addr, dc->cring);
+                    } else {
+                        tcg_gen_qemu_ld32u(cpu_FR[RRR_R], addr, dc->cring);
+                    }
+                    if (OP2 & 0x1) {
+                        tcg_gen_mov_i32(cpu_R[RRR_S], addr);
+                    }
+                    tcg_temp_free(addr);
+                }
+                break;
+
+            default: /*reserved*/
+                RESERVED();
+                break;
+            }
             break;
 
         case 9: /*LSC4*/
@@ -1836,12 +1909,213 @@ static void disas_xtensa_insn(DisasContext *dc)
 
         case 10: /*FP0*/
             HAS_OPTION(XTENSA_OPTION_FP_COPROCESSOR);
-            TBD();
+            switch (OP2) {
+            case 0: /*ADD.Sf*/
+                gen_check_cpenable(dc, 0);
+                gen_helper_add_s(cpu_FR[RRR_R], cpu_env,
+                        cpu_FR[RRR_S], cpu_FR[RRR_T]);
+                break;
+
+            case 1: /*SUB.Sf*/
+                gen_check_cpenable(dc, 0);
+                gen_helper_sub_s(cpu_FR[RRR_R], cpu_env,
+                        cpu_FR[RRR_S], cpu_FR[RRR_T]);
+                break;
+
+            case 2: /*MUL.Sf*/
+                gen_check_cpenable(dc, 0);
+                gen_helper_mul_s(cpu_FR[RRR_R], cpu_env,
+                        cpu_FR[RRR_S], cpu_FR[RRR_T]);
+                break;
+
+            case 4: /*MADD.Sf*/
+                gen_check_cpenable(dc, 0);
+                gen_helper_madd_s(cpu_FR[RRR_R], cpu_env,
+                        cpu_FR[RRR_R], cpu_FR[RRR_S], cpu_FR[RRR_T]);
+                break;
+
+            case 5: /*MSUB.Sf*/
+                gen_check_cpenable(dc, 0);
+                gen_helper_msub_s(cpu_FR[RRR_R], cpu_env,
+                        cpu_FR[RRR_R], cpu_FR[RRR_S], cpu_FR[RRR_T]);
+                break;
+
+            case 8: /*ROUND.Sf*/
+            case 9: /*TRUNC.Sf*/
+            case 10: /*FLOOR.Sf*/
+            case 11: /*CEIL.Sf*/
+            case 14: /*UTRUNC.Sf*/
+                gen_window_check1(dc, RRR_R);
+                gen_check_cpenable(dc, 0);
+                {
+                    static const unsigned rounding_mode_const[] = {
+                        float_round_nearest_even,
+                        float_round_to_zero,
+                        float_round_down,
+                        float_round_up,
+                        [6] = float_round_to_zero,
+                    };
+                    TCGv_i32 rounding_mode = tcg_const_i32(
+                            rounding_mode_const[OP2 & 7]);
+                    TCGv_i32 scale = tcg_const_i32(RRR_T);
+
+                    if (OP2 == 14) {
+                        gen_helper_ftoui(cpu_R[RRR_R], cpu_FR[RRR_S],
+                                rounding_mode, scale);
+                    } else {
+                        gen_helper_ftoi(cpu_R[RRR_R], cpu_FR[RRR_S],
+                                rounding_mode, scale);
+                    }
+
+                    tcg_temp_free(rounding_mode);
+                    tcg_temp_free(scale);
+                }
+                break;
+
+            case 12: /*FLOAT.Sf*/
+            case 13: /*UFLOAT.Sf*/
+                gen_window_check1(dc, RRR_S);
+                gen_check_cpenable(dc, 0);
+                {
+                    TCGv_i32 scale = tcg_const_i32(-RRR_T);
+
+                    if (OP2 == 13) {
+                        gen_helper_uitof(cpu_FR[RRR_R], cpu_env,
+                                cpu_R[RRR_S], scale);
+                    } else {
+                        gen_helper_itof(cpu_FR[RRR_R], cpu_env,
+                                cpu_R[RRR_S], scale);
+                    }
+                    tcg_temp_free(scale);
+                }
+                break;
+
+            case 15: /*FP1OP*/
+                switch (RRR_T) {
+                case 0: /*MOV.Sf*/
+                    gen_check_cpenable(dc, 0);
+                    tcg_gen_mov_i32(cpu_FR[RRR_R], cpu_FR[RRR_S]);
+                    break;
+
+                case 1: /*ABS.Sf*/
+                    gen_check_cpenable(dc, 0);
+                    gen_helper_abs_s(cpu_FR[RRR_R], cpu_FR[RRR_S]);
+                    break;
+
+                case 4: /*RFRf*/
+                    gen_window_check1(dc, RRR_R);
+                    gen_check_cpenable(dc, 0);
+                    tcg_gen_mov_i32(cpu_R[RRR_R], cpu_FR[RRR_S]);
+                    break;
+
+                case 5: /*WFRf*/
+                    gen_window_check1(dc, RRR_S);
+                    gen_check_cpenable(dc, 0);
+                    tcg_gen_mov_i32(cpu_FR[RRR_R], cpu_R[RRR_S]);
+                    break;
+
+                case 6: /*NEG.Sf*/
+                    gen_check_cpenable(dc, 0);
+                    gen_helper_neg_s(cpu_FR[RRR_R], cpu_FR[RRR_S]);
+                    break;
+
+                default: /*reserved*/
+                    RESERVED();
+                    break;
+                }
+                break;
+
+            default: /*reserved*/
+                RESERVED();
+                break;
+            }
             break;
 
         case 11: /*FP1*/
             HAS_OPTION(XTENSA_OPTION_FP_COPROCESSOR);
-            TBD();
+
+#define gen_compare(rel, br, a, b) \
+    do { \
+        TCGv_i32 bit = tcg_const_i32(1 << br); \
+        \
+        gen_check_cpenable(dc, 0); \
+        gen_helper_##rel(cpu_env, bit, cpu_FR[a], cpu_FR[b]); \
+        tcg_temp_free(bit); \
+    } while (0)
+
+            switch (OP2) {
+            case 1: /*UN.Sf*/
+                gen_compare(un_s, RRR_R, RRR_S, RRR_T);
+                break;
+
+            case 2: /*OEQ.Sf*/
+                gen_compare(oeq_s, RRR_R, RRR_S, RRR_T);
+                break;
+
+            case 3: /*UEQ.Sf*/
+                gen_compare(ueq_s, RRR_R, RRR_S, RRR_T);
+                break;
+
+            case 4: /*OLT.Sf*/
+                gen_compare(olt_s, RRR_R, RRR_S, RRR_T);
+                break;
+
+            case 5: /*ULT.Sf*/
+                gen_compare(ult_s, RRR_R, RRR_S, RRR_T);
+                break;
+
+            case 6: /*OLE.Sf*/
+                gen_compare(ole_s, RRR_R, RRR_S, RRR_T);
+                break;
+
+            case 7: /*ULE.Sf*/
+                gen_compare(ule_s, RRR_R, RRR_S, RRR_T);
+                break;
+
+#undef gen_compare
+
+            case 8: /*MOVEQZ.Sf*/
+            case 9: /*MOVNEZ.Sf*/
+            case 10: /*MOVLTZ.Sf*/
+            case 11: /*MOVGEZ.Sf*/
+                gen_window_check1(dc, RRR_T);
+                gen_check_cpenable(dc, 0);
+                {
+                    static const TCGCond cond[] = {
+                        TCG_COND_NE,
+                        TCG_COND_EQ,
+                        TCG_COND_GE,
+                        TCG_COND_LT
+                    };
+                    int label = gen_new_label();
+                    tcg_gen_brcondi_i32(cond[OP2 - 8], cpu_R[RRR_T], 0, label);
+                    tcg_gen_mov_i32(cpu_FR[RRR_R], cpu_FR[RRR_S]);
+                    gen_set_label(label);
+                }
+                break;
+
+            case 12: /*MOVF.Sf*/
+            case 13: /*MOVT.Sf*/
+                HAS_OPTION(XTENSA_OPTION_BOOLEAN);
+                gen_check_cpenable(dc, 0);
+                {
+                    int label = gen_new_label();
+                    TCGv_i32 tmp = tcg_temp_new_i32();
+
+                    tcg_gen_andi_i32(tmp, cpu_SR[BR], 1 << RRR_T);
+                    tcg_gen_brcondi_i32(
+                            OP2 & 1 ? TCG_COND_EQ : TCG_COND_NE,
+                            tmp, 0, label);
+                    tcg_gen_mov_i32(cpu_FR[RRR_R], cpu_FR[RRR_S]);
+                    gen_set_label(label);
+                    tcg_temp_free(tmp);
+                }
+                break;
+
+            default: /*reserved*/
+                RESERVED();
+                break;
+            }
             break;
 
         default: /*reserved*/
@@ -2072,8 +2346,34 @@ static void disas_xtensa_insn(DisasContext *dc)
         break;
 
     case 3: /*LSCIp*/
-        HAS_OPTION(XTENSA_OPTION_COPROCESSOR);
-        TBD();
+        switch (RRI8_R) {
+        case 0: /*LSIf*/
+        case 4: /*SSIf*/
+        case 8: /*LSIUf*/
+        case 12: /*SSIUf*/
+            HAS_OPTION(XTENSA_OPTION_FP_COPROCESSOR);
+            gen_window_check1(dc, RRI8_S);
+            gen_check_cpenable(dc, 0);
+            {
+                TCGv_i32 addr = tcg_temp_new_i32();
+                tcg_gen_addi_i32(addr, cpu_R[RRI8_S], RRI8_IMM8 << 2);
+                gen_load_store_alignment(dc, 2, addr, false);
+                if (RRI8_R & 0x4) {
+                    tcg_gen_qemu_st32(cpu_FR[RRI8_T], addr, dc->cring);
+                } else {
+                    tcg_gen_qemu_ld32u(cpu_FR[RRI8_T], addr, dc->cring);
+                }
+                if (RRI8_R & 0x8) {
+                    tcg_gen_mov_i32(cpu_R[RRI8_S], addr);
+                }
+                tcg_temp_free(addr);
+            }
+            break;
+
+        default: /*reserved*/
+            RESERVED();
+            break;
+        }
         break;
 
     case 4: /*MAC16d*/
@@ -2502,7 +2802,9 @@ static void disas_xtensa_insn(DisasContext *dc)
         break;
     }
 
-    gen_check_loop_end(dc, 0);
+    if (dc->is_jmp == DISAS_NEXT) {
+        gen_check_loop_end(dc, 0);
+    }
     dc->pc = dc->next_pc;
 
     return;
@@ -2547,7 +2849,7 @@ static void gen_intermediate_code_internal(
     DisasContext dc;
     int insn_count = 0;
     int j, lj = -1;
-    uint16_t *gen_opc_end = gen_opc_buf + OPC_MAX_SIZE;
+    uint16_t *gen_opc_end = tcg_ctx.gen_opc_buf + OPC_MAX_SIZE;
     int max_insns = tb->cflags & CF_COUNT_MASK;
     uint32_t pc_start = tb->pc;
     uint32_t next_page_start =
@@ -2569,6 +2871,8 @@ static void gen_intermediate_code_internal(
     dc.ccount_delta = 0;
     dc.debug = tb->flags & XTENSA_TBFLAG_DEBUG;
     dc.icount = tb->flags & XTENSA_TBFLAG_ICOUNT;
+    dc.cpenable = (tb->flags & XTENSA_TBFLAG_CPENABLE_MASK) >>
+        XTENSA_TBFLAG_CPENABLE_SHIFT;
 
     init_litbase(&dc);
     init_sar_tracker(&dc);
@@ -2589,7 +2893,7 @@ static void gen_intermediate_code_internal(
         check_breakpoint(env, &dc);
 
         if (search_pc) {
-            j = gen_opc_ptr - gen_opc_buf;
+            j = tcg_ctx.gen_opc_ptr - tcg_ctx.gen_opc_buf;
             if (lj < j) {
                 lj++;
                 while (lj < j) {
@@ -2601,7 +2905,7 @@ static void gen_intermediate_code_internal(
             gen_opc_icount[lj] = insn_count;
         }
 
-        if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP))) {
+        if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP | CPU_LOG_TB_OP_OPT))) {
             tcg_gen_debug_insn_start(dc.pc);
         }
 
@@ -2627,7 +2931,7 @@ static void gen_intermediate_code_internal(
             gen_ibreak_check(env, &dc);
         }
 
-        disas_xtensa_insn(&dc);
+        disas_xtensa_insn(env, &dc);
         ++insn_count;
         if (dc.icount) {
             tcg_gen_mov_i32(cpu_SR[ICOUNT], dc.next_icount);
@@ -2640,7 +2944,7 @@ static void gen_intermediate_code_internal(
     } while (dc.is_jmp == DISAS_NEXT &&
             insn_count < max_insns &&
             dc.pc < next_page_start &&
-            gen_opc_ptr < gen_opc_end);
+            tcg_ctx.gen_opc_ptr < gen_opc_end);
 
     reset_litbase(&dc);
     reset_sar_tracker(&dc);
@@ -2656,7 +2960,7 @@ static void gen_intermediate_code_internal(
         gen_jumpi(&dc, dc.pc, 0);
     }
     gen_icount_end(tb, insn_count);
-    *gen_opc_ptr = INDEX_op_end;
+    *tcg_ctx.gen_opc_ptr = INDEX_op_end;
 
     if (!search_pc) {
         tb->size = dc.pc - pc_start;
@@ -2710,6 +3014,16 @@ void cpu_dump_state(CPUXtensaState *env, FILE *f, fprintf_function cpu_fprintf,
         cpu_fprintf(f, "AR%02d=%08x%c", i, env->phys_regs[i],
                 (i % 4) == 3 ? '\n' : ' ');
     }
+
+    if (xtensa_option_enabled(env->config, XTENSA_OPTION_FP_COPROCESSOR)) {
+        cpu_fprintf(f, "\n");
+
+        for (i = 0; i < 16; ++i) {
+            cpu_fprintf(f, "F%02d=%08x (%+10.8e)%c", i,
+                    float32_val(env->fregs[i]),
+                    *(float *)&env->fregs[i], (i % 2) == 1 ? '\n' : ' ');
+        }
+    }
 }
 
 void restore_state_to_opc(CPUXtensaState *env, TranslationBlock *tb, int pc_pos)
diff --git a/target-xtensa/xtensa-semi.c b/target-xtensa/xtensa-semi.c
index 6d001c2c1..851ff546f 100644
--- a/target-xtensa/xtensa-semi.c
+++ b/target-xtensa/xtensa-semi.c
@@ -54,6 +54,102 @@ enum {
     SELECT_ONE_EXCEPT = 3,
 };
 
+enum {
+    TARGET_EPERM        =  1,
+    TARGET_ENOENT       =  2,
+    TARGET_ESRCH        =  3,
+    TARGET_EINTR        =  4,
+    TARGET_EIO          =  5,
+    TARGET_ENXIO        =  6,
+    TARGET_E2BIG        =  7,
+    TARGET_ENOEXEC      =  8,
+    TARGET_EBADF        =  9,
+    TARGET_ECHILD       = 10,
+    TARGET_EAGAIN       = 11,
+    TARGET_ENOMEM       = 12,
+    TARGET_EACCES       = 13,
+    TARGET_EFAULT       = 14,
+    TARGET_ENOTBLK      = 15,
+    TARGET_EBUSY        = 16,
+    TARGET_EEXIST       = 17,
+    TARGET_EXDEV        = 18,
+    TARGET_ENODEV       = 19,
+    TARGET_ENOTDIR      = 20,
+    TARGET_EISDIR       = 21,
+    TARGET_EINVAL       = 22,
+    TARGET_ENFILE       = 23,
+    TARGET_EMFILE       = 24,
+    TARGET_ENOTTY       = 25,
+    TARGET_ETXTBSY      = 26,
+    TARGET_EFBIG        = 27,
+    TARGET_ENOSPC       = 28,
+    TARGET_ESPIPE       = 29,
+    TARGET_EROFS        = 30,
+    TARGET_EMLINK       = 31,
+    TARGET_EPIPE        = 32,
+    TARGET_EDOM         = 33,
+    TARGET_ERANGE       = 34,
+    TARGET_ENOSYS       = 88,
+    TARGET_ELOOP        = 92,
+};
+
+static uint32_t errno_h2g(int host_errno)
+{
+    static const uint32_t guest_errno[] = {
+        [EPERM]         = TARGET_EPERM,
+        [ENOENT]        = TARGET_ENOENT,
+        [ESRCH]         = TARGET_ESRCH,
+        [EINTR]         = TARGET_EINTR,
+        [EIO]           = TARGET_EIO,
+        [ENXIO]         = TARGET_ENXIO,
+        [E2BIG]         = TARGET_E2BIG,
+        [ENOEXEC]       = TARGET_ENOEXEC,
+        [EBADF]         = TARGET_EBADF,
+        [ECHILD]        = TARGET_ECHILD,
+        [EAGAIN]        = TARGET_EAGAIN,
+        [ENOMEM]        = TARGET_ENOMEM,
+        [EACCES]        = TARGET_EACCES,
+        [EFAULT]        = TARGET_EFAULT,
+#ifdef ENOTBLK
+        [ENOTBLK]       = TARGET_ENOTBLK,
+#endif
+        [EBUSY]         = TARGET_EBUSY,
+        [EEXIST]        = TARGET_EEXIST,
+        [EXDEV]         = TARGET_EXDEV,
+        [ENODEV]        = TARGET_ENODEV,
+        [ENOTDIR]       = TARGET_ENOTDIR,
+        [EISDIR]        = TARGET_EISDIR,
+        [EINVAL]        = TARGET_EINVAL,
+        [ENFILE]        = TARGET_ENFILE,
+        [EMFILE]        = TARGET_EMFILE,
+        [ENOTTY]        = TARGET_ENOTTY,
+#ifdef ETXTBSY
+        [ETXTBSY]       = TARGET_ETXTBSY,
+#endif
+        [EFBIG]         = TARGET_EFBIG,
+        [ENOSPC]        = TARGET_ENOSPC,
+        [ESPIPE]        = TARGET_ESPIPE,
+        [EROFS]         = TARGET_EROFS,
+        [EMLINK]        = TARGET_EMLINK,
+        [EPIPE]         = TARGET_EPIPE,
+        [EDOM]          = TARGET_EDOM,
+        [ERANGE]        = TARGET_ERANGE,
+        [ENOSYS]        = TARGET_ENOSYS,
+#ifdef ELOOP
+        [ELOOP]         = TARGET_ELOOP,
+#endif
+    };
+
+    if (host_errno == 0) {
+        return 0;
+    } else if (host_errno > 0 && host_errno < ARRAY_SIZE(guest_errno) &&
+            guest_errno[host_errno]) {
+        return guest_errno[host_errno];
+    } else {
+        return TARGET_EINVAL;
+    }
+}
+
 void HELPER(simcall)(CPUXtensaState *env)
 {
     uint32_t *regs = env->regs;
@@ -73,12 +169,12 @@ void HELPER(simcall)(CPUXtensaState *env)
             uint32_t len = regs[5];
 
             while (len > 0) {
-                target_phys_addr_t paddr =
+                hwaddr paddr =
                     cpu_get_phys_page_debug(env, vaddr);
                 uint32_t page_left =
                     TARGET_PAGE_SIZE - (vaddr & (TARGET_PAGE_SIZE - 1));
                 uint32_t io_sz = page_left < len ? page_left : len;
-                target_phys_addr_t sz = io_sz;
+                hwaddr sz = io_sz;
                 void *buf = cpu_physical_memory_map(paddr, &sz, is_write);
 
                 if (buf) {
@@ -87,14 +183,14 @@ void HELPER(simcall)(CPUXtensaState *env)
                     regs[2] = is_write ?
                         write(fd, buf, io_sz) :
                         read(fd, buf, io_sz);
-                    regs[3] = errno;
+                    regs[3] = errno_h2g(errno);
                     cpu_physical_memory_unmap(buf, sz, is_write, sz);
                     if (regs[2] == -1) {
                         break;
                     }
                 } else {
                     regs[2] = -1;
-                    regs[3] = EINVAL;
+                    regs[3] = TARGET_EINVAL;
                     break;
                 }
             }
@@ -117,10 +213,10 @@ void HELPER(simcall)(CPUXtensaState *env)
 
             if (rc == 0 && i < ARRAY_SIZE(name)) {
                 regs[2] = open(name, regs[4], regs[5]);
-                regs[3] = errno;
+                regs[3] = errno_h2g(errno);
             } else {
                 regs[2] = -1;
-                regs[3] = EINVAL;
+                regs[3] = TARGET_EINVAL;
             }
         }
         break;
@@ -130,13 +226,13 @@ void HELPER(simcall)(CPUXtensaState *env)
             regs[2] = regs[3] = 0;
         } else {
             regs[2] = close(regs[3]);
-            regs[3] = errno;
+            regs[3] = errno_h2g(errno);
         }
         break;
 
     case TARGET_SYS_lseek:
         regs[2] = lseek(regs[3], (off_t)(int32_t)regs[4], regs[5]);
-        regs[3] = errno;
+        regs[3] = errno_h2g(errno);
         break;
 
     case TARGET_SYS_select_one:
@@ -163,7 +259,7 @@ void HELPER(simcall)(CPUXtensaState *env)
                     rq == SELECT_ONE_WRITE  ? &fdset : NULL,
                     rq == SELECT_ONE_EXCEPT ? &fdset : NULL,
                     target_tv ? &tv : NULL);
-            regs[3] = errno;
+            regs[3] = errno_h2g(errno);
         }
         break;
 
@@ -199,7 +295,7 @@ void HELPER(simcall)(CPUXtensaState *env)
             uint32_t sz = regs[5];
 
             while (sz) {
-                target_phys_addr_t len = sz;
+                hwaddr len = sz;
                 void *buf = cpu_physical_memory_map(base, &len, 1);
 
                 if (buf && len) {
@@ -219,7 +315,7 @@ void HELPER(simcall)(CPUXtensaState *env)
     default:
         qemu_log("%s(%d): not implemented\n", __func__, regs[2]);
         regs[2] = -1;
-        regs[3] = ENOSYS;
+        regs[3] = TARGET_ENOSYS;
         break;
     }
 }
diff --git a/targphys.h b/targphys.h
deleted file mode 100644
index bd4938fc0..000000000
--- a/targphys.h
+++ /dev/null
@@ -1,37 +0,0 @@
-/* Define target_phys_addr_t if it exists.  */
-
-#ifndef TARGPHYS_H
-#define TARGPHYS_H
-
-#ifdef TARGET_PHYS_ADDR_BITS
-/* target_phys_addr_t is the type of a physical address (its size can
-   be different from 'target_ulong').  */
-
-#if TARGET_PHYS_ADDR_BITS == 32
-typedef uint32_t target_phys_addr_t;
-#define TARGET_PHYS_ADDR_MAX UINT32_MAX
-#define TARGET_FMT_plx "%08x"
-/* Format strings for printing target_phys_addr_t types.
- * These are recommended over the less flexible TARGET_FMT_plx,
- * which is retained for the benefit of existing code.
- */
-#define TARGET_PRIdPHYS PRId32
-#define TARGET_PRIiPHYS PRIi32
-#define TARGET_PRIoPHYS PRIo32
-#define TARGET_PRIuPHYS PRIu32
-#define TARGET_PRIxPHYS PRIx32
-#define TARGET_PRIXPHYS PRIX32
-#elif TARGET_PHYS_ADDR_BITS == 64
-typedef uint64_t target_phys_addr_t;
-#define TARGET_PHYS_ADDR_MAX UINT64_MAX
-#define TARGET_FMT_plx "%016" PRIx64
-#define TARGET_PRIdPHYS PRId64
-#define TARGET_PRIiPHYS PRIi64
-#define TARGET_PRIoPHYS PRIo64
-#define TARGET_PRIuPHYS PRIu64
-#define TARGET_PRIxPHYS PRIx64
-#define TARGET_PRIXPHYS PRIX64
-#endif
-#endif
-
-#endif
diff --git a/tcg/README b/tcg/README
index cfdfd96d0..ec1ac7937 100644
--- a/tcg/README
+++ b/tcg/README
@@ -77,19 +77,27 @@ destroyed, but local temporaries and globals are preserved.
 Using the tcg_gen_helper_x_y it is possible to call any function
 taking i32, i64 or pointer types. By default, before calling a helper,
 all globals are stored at their canonical location and it is assumed
-that the function can modify them. This can be overridden by the
-TCG_CALL_CONST function modifier. By default, the helper is allowed to
-modify the CPU state or raise an exception. This can be overridden by
-the TCG_CALL_PURE function modifier, in which case the call to the
-function is removed if the return value is not used.
+that the function can modify them. By default, the helper is allowed to
+modify the CPU state or raise an exception.
+
+This can be overridden using the following function modifiers:
+- TCG_CALL_NO_READ_GLOBALS means that the helper does not read globals,
+  either directly or via an exception. They will not be saved to their
+  canonical locations before calling the helper.
+- TCG_CALL_NO_WRITE_GLOBALS means that the helper does not modify any globals.
+  They will only be saved to their canonical location before calling helpers,
+  but they won't be reloaded afterwise.
+- TCG_CALL_NO_SIDE_EFFECTS means that the call to the function is removed if
+  the return value is not used.
+
+Note that TCG_CALL_NO_READ_GLOBALS implies TCG_CALL_NO_WRITE_GLOBALS.
 
 On some TCG targets (e.g. x86), several calling conventions are
 supported.
 
 * Branches:
 
-Use the instruction 'br' to jump to a label. Use 'jmp' to jump to an
-explicit address. Conditional branches can only jump to labels.
+Use the instruction 'br' to jump to a label.
 
 3.3) Code Optimizations
 
@@ -129,10 +137,6 @@ call function 'ptr' (pointer type)
 
 ********* Jumps/Labels
 
-* jmp t0
-
-Absolute jump to address t0 (pointer type).
-
 * set_label $label
 
 Define label 'label' at the current program point.
@@ -141,7 +145,7 @@ Define label 'label' at the current program point.
 
 Jump to label.
 
-* brcond_i32/i64 cond, t0, t1, label
+* brcond_i32/i64 t0, t1, cond, label
 
 Conditional jump if t0 cond t1 is true. cond can be:
     TCG_COND_EQ
@@ -301,12 +305,18 @@ This operation would be equivalent to
 
 ********* Conditional moves
 
-* setcond_i32/i64 cond, dest, t1, t2
+* setcond_i32/i64 dest, t1, t2, cond
 
 dest = (t1 cond t2)
 
 Set DEST to 1 if (T1 cond T2) is true, otherwise set to 0.
 
+* movcond_i32/i64 dest, c1, c2, v1, v2, cond
+
+dest = (c1 cond c2 ? v1 : v2)
+
+Set DEST to V1 if (C1 cond C2) is true, otherwise set to V2.
+
 ********* Type conversions
 
 * ext_i32_i64 t0, t1
@@ -348,13 +358,16 @@ st32_i64 t0, t1, offset
 write(t0, t1 + offset)
 Write 8, 16, 32 or 64 bits to host memory.
 
+All this opcodes assume that the pointed host memory doesn't correspond
+to a global. In the latter case the behaviour is unpredictable.
+
 ********* 64-bit target on 32-bit host support
 
 The following opcodes are internal to TCG.  Thus they are to be implemented by
 32-bit host code generators, but are not to be emitted by guest translators.
 They are emitted as needed by inline functions within "tcg-op.h".
 
-* brcond2_i32 cond, t0_low, t0_high, t1_low, t1_high, label
+* brcond2_i32 t0_low, t0_high, t1_low, t1_high, cond, label
 
 Similar to brcond, except that the 64-bit values T0 and T1
 are formed from two 32-bit arguments.
@@ -371,7 +384,7 @@ is returned in two 32-bit outputs.
 Similar to mul, except two 32-bit (unsigned) inputs T1 and T2 yielding
 the full 64-bit product T0.  The later is returned in two 32-bit outputs.
 
-* setcond2_i32 cond, dest, t1_low, t1_high, t2_low, t2_high
+* setcond2_i32 dest, t1_low, t1_high, t2_low, t2_high, cond
 
 Similar to setcond, except that the 64-bit values T1 and T2 are
 formed from two 32-bit arguments.  The result is a 32-bit value.
@@ -386,7 +399,8 @@ Exit the current TB and return the value t0 (word type).
 
 Exit the current TB and jump to the TB index 'index' (constant) if the
 current TB was linked to this TB. Otherwise execute the next
-instructions.
+instructions. Only indices 0 and 1 are valid and tcg_gen_goto_tb may be issued
+at most once with each slot index per TB.
 
 * qemu_ld8u t0, t1, flags
 qemu_ld8s t0, t1, flags
diff --git a/tcg/arm/tcg-target.c b/tcg/arm/tcg-target.c
index cf0ca3d0b..47612fe26 100644
--- a/tcg/arm/tcg-target.c
+++ b/tcg/arm/tcg-target.c
@@ -145,12 +145,6 @@ static void patch_reloc(uint8_t *code_ptr, int type,
     }
 }
 
-/* maximum number of register used for input function arguments */
-static inline int tcg_target_get_call_iarg_regs_count(int flags)
-{
-    return 4;
-}
-
 /* parse target specific constraints */
 static int target_parse_constraint(TCGArgConstraint *ct, const char **pct_str)
 {
@@ -176,7 +170,7 @@ static int target_parse_constraint(TCGArgConstraint *ct, const char **pct_str)
            so don't use these. */
         tcg_regset_reset_reg(ct->u.regs, TCG_REG_R0);
         tcg_regset_reset_reg(ct->u.regs, TCG_REG_R1);
-#if defined(CONFIG_TCG_PASS_AREG0) && (TARGET_LONG_BITS == 64)
+#if TARGET_LONG_BITS == 64
         /* If we're passing env to the helper as r0 and need a regpair
          * for the address then r2 will be overwritten as we're setting
          * up the args to the helper.
@@ -204,8 +198,7 @@ static int target_parse_constraint(TCGArgConstraint *ct, const char **pct_str)
            use these. */
         tcg_regset_reset_reg(ct->u.regs, TCG_REG_R0);
         tcg_regset_reset_reg(ct->u.regs, TCG_REG_R1);
-#if defined(CONFIG_SOFTMMU) && \
-    defined(CONFIG_TCG_PASS_AREG0) && (TARGET_LONG_BITS == 64)
+#if defined(CONFIG_SOFTMMU) && (TARGET_LONG_BITS == 64)
         /* Avoid clashes with registers being used for helper args */
         tcg_regset_reset_reg(ct->u.regs, TCG_REG_R2);
         tcg_regset_reset_reg(ct->u.regs, TCG_REG_R3);
@@ -223,7 +216,7 @@ static int target_parse_constraint(TCGArgConstraint *ct, const char **pct_str)
 #ifdef CONFIG_SOFTMMU
         /* r2 is still needed to load data_reg, so don't use it. */
         tcg_regset_reset_reg(ct->u.regs, TCG_REG_R2);
-#if defined(CONFIG_TCG_PASS_AREG0) && (TARGET_LONG_BITS == 64)
+#if TARGET_LONG_BITS == 64
         /* Avoid clashes with registers being used for helper args */
         tcg_regset_reset_reg(ct->u.regs, TCG_REG_R3);
 #endif
@@ -342,7 +335,7 @@ enum arm_cond_code_e {
     COND_AL = 0xe,
 };
 
-static const uint8_t tcg_cond_to_arm_cond[10] = {
+static const uint8_t tcg_cond_to_arm_cond[] = {
     [TCG_COND_EQ] = COND_EQ,
     [TCG_COND_NE] = COND_NE,
     [TCG_COND_LT] = COND_LT,
@@ -474,6 +467,21 @@ static inline void tcg_out_movi32(TCGContext *s,
     }
 }
 
+static inline void tcg_out_dat_rI(TCGContext *s, int cond, int opc, TCGArg dst,
+                                  TCGArg lhs, TCGArg rhs, int rhs_is_const)
+{
+    /* Emit either the reg,imm or reg,reg form of a data-processing insn.
+     * rhs must satisfy the "rI" constraint.
+     */
+    if (rhs_is_const) {
+        int rot = encode_imm(rhs);
+        assert(rot >= 0);
+        tcg_out_dat_imm(s, cond, opc, dst, lhs, rotl(rhs, rot) | (rot << 7));
+    } else {
+        tcg_out_dat_reg(s, cond, opc, dst, lhs, rhs, SHIFT_IMM_LSL(0));
+    }
+}
+
 static inline void tcg_out_mul32(TCGContext *s,
                 int cond, int rd, int rs, int rm)
 {
@@ -603,6 +611,22 @@ static inline void tcg_out_bswap16(TCGContext *s, int cond, int rd, int rn)
     }
 }
 
+/* swap the two low bytes assuming that the two high input bytes and the
+   two high output bit can hold any value. */
+static inline void tcg_out_bswap16st(TCGContext *s, int cond, int rd, int rn)
+{
+    if (use_armv6_instructions) {
+        /* rev16 */
+        tcg_out32(s, 0x06bf0fb0 | (cond << 28) | (rd << 12) | rn);
+    } else {
+        tcg_out_dat_reg(s, cond, ARITH_MOV,
+                        TCG_REG_R8, 0, rn, SHIFT_IMM_LSR(8));
+        tcg_out_dat_imm(s, cond, ARITH_AND, TCG_REG_R8, TCG_REG_R8, 0xff);
+        tcg_out_dat_reg(s, cond, ARITH_ORR,
+                        rd, TCG_REG_R8, rn, SHIFT_IMM_LSL(8));
+    }
+}
+
 static inline void tcg_out_bswap32(TCGContext *s, int cond, int rd, int rn)
 {
     if (use_armv6_instructions) {
@@ -631,6 +655,22 @@ static inline void tcg_out_ld32_12(TCGContext *s, int cond,
                         (rn << 16) | (rd << 12) | ((-im) & 0xfff));
 }
 
+/* Offset pre-increment with base writeback.  */
+static inline void tcg_out_ld32_12wb(TCGContext *s, int cond,
+                                     int rd, int rn, tcg_target_long im)
+{
+    /* ldr with writeback and both register equals is UNPREDICTABLE */
+    assert(rd != rn);
+
+    if (im >= 0) {
+        tcg_out32(s, (cond << 28) | 0x05b00000 |
+                        (rn << 16) | (rd << 12) | (im & 0xfff));
+    } else {
+        tcg_out32(s, (cond << 28) | 0x05300000 |
+                        (rn << 16) | (rd << 12) | ((-im) & 0xfff));
+    }
+}
+
 static inline void tcg_out_st32_12(TCGContext *s, int cond,
                 int rd, int rn, tcg_target_long im)
 {
@@ -954,7 +994,6 @@ static inline void tcg_out_goto_label(TCGContext *s, int cond, int label_index)
 
 #include "../../softmmu_defs.h"
 
-#ifdef CONFIG_TCG_PASS_AREG0
 /* helper signature: helper_ld_mmu(CPUState *env, target_ulong addr,
    int mmu_idx) */
 static const void * const qemu_ld_helpers[4] = {
@@ -972,25 +1011,6 @@ static const void * const qemu_st_helpers[4] = {
     helper_stl_mmu,
     helper_stq_mmu,
 };
-#else
-/* legacy helper signature: __ld_mmu(target_ulong addr, int
-   mmu_idx) */
-static void *qemu_ld_helpers[4] = {
-    __ldb_mmu,
-    __ldw_mmu,
-    __ldl_mmu,
-    __ldq_mmu,
-};
-
-/* legacy helper signature: __st_mmu(target_ulong addr, uintxx_t val,
-   int mmu_idx) */
-static void *qemu_st_helpers[4] = {
-    __stb_mmu,
-    __stw_mmu,
-    __stl_mmu,
-    __stq_mmu,
-};
-#endif
 
 /* Helper routines for marshalling helper function arguments into
  * the correct registers and stack.
@@ -1083,7 +1103,7 @@ static inline void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, int opc)
 {
     int addr_reg, data_reg, data_reg2, bswap;
 #ifdef CONFIG_SOFTMMU
-    int mem_index, s_bits;
+    int mem_index, s_bits, tlb_offset;
     TCGReg argreg;
 # if TARGET_LONG_BITS == 64
     int addr_reg2;
@@ -1123,19 +1143,15 @@ static inline void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, int opc)
                     TCG_REG_R0, TCG_REG_R8, CPU_TLB_SIZE - 1);
     tcg_out_dat_reg(s, COND_AL, ARITH_ADD, TCG_REG_R0, TCG_AREG0,
                     TCG_REG_R0, SHIFT_IMM_LSL(CPU_TLB_ENTRY_BITS));
-    /* In the
-     *  ldr r1 [r0, #(offsetof(CPUArchState, tlb_table[mem_index][0].addr_read))]
-     * below, the offset is likely to exceed 12 bits if mem_index != 0 and
-     * not exceed otherwise, so use an
-     *  add r0, r0, #(mem_index * sizeof *CPUArchState.tlb_table)
-     * before.
-     */
-    if (mem_index)
+    /* We assume that the offset is contained within 20 bits.  */
+    tlb_offset = offsetof(CPUArchState, tlb_table[mem_index][0].addr_read);
+    assert(tlb_offset & ~0xfffff == 0);
+    if (tlb_offset > 0xfff) {
         tcg_out_dat_imm(s, COND_AL, ARITH_ADD, TCG_REG_R0, TCG_REG_R0,
-                        (mem_index << (TLB_SHIFT & 1)) |
-                        ((16 - (TLB_SHIFT >> 1)) << 8));
-    tcg_out_ld32_12(s, COND_AL, TCG_REG_R1, TCG_REG_R0,
-                    offsetof(CPUArchState, tlb_table[0][0].addr_read));
+                        0xa00 | (tlb_offset >> 12));
+        tlb_offset &= 0xfff;
+    }
+    tcg_out_ld32_12wb(s, COND_AL, TCG_REG_R1, TCG_REG_R0, tlb_offset);
     tcg_out_dat_reg(s, COND_AL, ARITH_CMP, 0, TCG_REG_R1,
                     TCG_REG_R8, SHIFT_IMM_LSL(TARGET_PAGE_BITS));
     /* Check alignment.  */
@@ -1143,15 +1159,14 @@ static inline void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, int opc)
         tcg_out_dat_imm(s, COND_EQ, ARITH_TST,
                         0, addr_reg, (1 << s_bits) - 1);
 #  if TARGET_LONG_BITS == 64
-    /* XXX: possibly we could use a block data load or writeback in
-     * the first access.  */
-    tcg_out_ld32_12(s, COND_EQ, TCG_REG_R1, TCG_REG_R0,
-                    offsetof(CPUArchState, tlb_table[0][0].addr_read) + 4);
+    /* XXX: possibly we could use a block data load in the first access.  */
+    tcg_out_ld32_12(s, COND_EQ, TCG_REG_R1, TCG_REG_R0, 4);
     tcg_out_dat_reg(s, COND_EQ, ARITH_CMP, 0,
                     TCG_REG_R1, addr_reg2, SHIFT_IMM_LSL(0));
 #  endif
     tcg_out_ld32_12(s, COND_EQ, TCG_REG_R1, TCG_REG_R0,
-                    offsetof(CPUArchState, tlb_table[0][0].addend));
+                    offsetof(CPUTLBEntry, addend)
+                    - offsetof(CPUTLBEntry, addr_read));
 
     switch (opc) {
     case 0:
@@ -1203,9 +1218,7 @@ static inline void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, int opc)
      * trash by moving the earlier arguments into them.
      */
     argreg = TCG_REG_R0;
-#ifdef CONFIG_TCG_PASS_AREG0
     argreg = tcg_out_arg_reg32(s, argreg, TCG_AREG0);
-#endif
 #if TARGET_LONG_BITS == 64
     argreg = tcg_out_arg_reg64(s, argreg, addr_reg, addr_reg2);
 #else
@@ -1226,20 +1239,11 @@ static inline void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, int opc)
     case 1:
     case 2:
     default:
-        if (data_reg != TCG_REG_R0) {
-            tcg_out_dat_reg(s, COND_AL, ARITH_MOV,
-                            data_reg, 0, TCG_REG_R0, SHIFT_IMM_LSL(0));
-        }
+        tcg_out_mov_reg(s, COND_AL, data_reg, TCG_REG_R0);
         break;
     case 3:
-        if (data_reg != TCG_REG_R0) {
-            tcg_out_dat_reg(s, COND_AL, ARITH_MOV,
-                            data_reg, 0, TCG_REG_R0, SHIFT_IMM_LSL(0));
-        }
-        if (data_reg2 != TCG_REG_R1) {
-            tcg_out_dat_reg(s, COND_AL, ARITH_MOV,
-                            data_reg2, 0, TCG_REG_R1, SHIFT_IMM_LSL(0));
-        }
+        tcg_out_mov_reg(s, COND_AL, data_reg, TCG_REG_R0);
+        tcg_out_mov_reg(s, COND_AL, data_reg2, TCG_REG_R1);
         break;
     }
 
@@ -1311,7 +1315,7 @@ static inline void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, int opc)
 {
     int addr_reg, data_reg, data_reg2, bswap;
 #ifdef CONFIG_SOFTMMU
-    int mem_index, s_bits;
+    int mem_index, s_bits, tlb_offset;
     TCGReg argreg;
 # if TARGET_LONG_BITS == 64
     int addr_reg2;
@@ -1348,19 +1352,15 @@ static inline void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, int opc)
                     TCG_REG_R0, TCG_REG_R8, CPU_TLB_SIZE - 1);
     tcg_out_dat_reg(s, COND_AL, ARITH_ADD, TCG_REG_R0,
                     TCG_AREG0, TCG_REG_R0, SHIFT_IMM_LSL(CPU_TLB_ENTRY_BITS));
-    /* In the
-     *  ldr r1 [r0, #(offsetof(CPUArchState, tlb_table[mem_index][0].addr_write))]
-     * below, the offset is likely to exceed 12 bits if mem_index != 0 and
-     * not exceed otherwise, so use an
-     *  add r0, r0, #(mem_index * sizeof *CPUArchState.tlb_table)
-     * before.
-     */
-    if (mem_index)
+    /* We assume that the offset is contained within 20 bits.  */
+    tlb_offset = offsetof(CPUArchState, tlb_table[mem_index][0].addr_write);
+    assert(tlb_offset & ~0xfffff == 0);
+    if (tlb_offset > 0xfff) {
         tcg_out_dat_imm(s, COND_AL, ARITH_ADD, TCG_REG_R0, TCG_REG_R0,
-                        (mem_index << (TLB_SHIFT & 1)) |
-                        ((16 - (TLB_SHIFT >> 1)) << 8));
-    tcg_out_ld32_12(s, COND_AL, TCG_REG_R1, TCG_REG_R0,
-                    offsetof(CPUArchState, tlb_table[0][0].addr_write));
+                        0xa00 | (tlb_offset >> 12));
+        tlb_offset &= 0xfff;
+    }
+    tcg_out_ld32_12wb(s, COND_AL, TCG_REG_R1, TCG_REG_R0, tlb_offset);
     tcg_out_dat_reg(s, COND_AL, ARITH_CMP, 0, TCG_REG_R1,
                     TCG_REG_R8, SHIFT_IMM_LSL(TARGET_PAGE_BITS));
     /* Check alignment.  */
@@ -1368,15 +1368,14 @@ static inline void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, int opc)
         tcg_out_dat_imm(s, COND_EQ, ARITH_TST,
                         0, addr_reg, (1 << s_bits) - 1);
 #  if TARGET_LONG_BITS == 64
-    /* XXX: possibly we could use a block data load or writeback in
-     * the first access.  */
-    tcg_out_ld32_12(s, COND_EQ, TCG_REG_R1, TCG_REG_R0,
-                    offsetof(CPUArchState, tlb_table[0][0].addr_write) + 4);
+    /* XXX: possibly we could use a block data load in the first access.  */
+    tcg_out_ld32_12(s, COND_EQ, TCG_REG_R1, TCG_REG_R0, 4);
     tcg_out_dat_reg(s, COND_EQ, ARITH_CMP, 0,
                     TCG_REG_R1, addr_reg2, SHIFT_IMM_LSL(0));
 #  endif
     tcg_out_ld32_12(s, COND_EQ, TCG_REG_R1, TCG_REG_R0,
-                    offsetof(CPUArchState, tlb_table[0][0].addend));
+                    offsetof(CPUTLBEntry, addend)
+                    - offsetof(CPUTLBEntry, addr_write));
 
     switch (opc) {
     case 0:
@@ -1384,7 +1383,7 @@ static inline void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, int opc)
         break;
     case 1:
         if (bswap) {
-            tcg_out_bswap16(s, COND_EQ, TCG_REG_R0, data_reg);
+            tcg_out_bswap16st(s, COND_EQ, TCG_REG_R0, data_reg);
             tcg_out_st16_r(s, COND_EQ, TCG_REG_R0, addr_reg, TCG_REG_R1);
         } else {
             tcg_out_st16_r(s, COND_EQ, data_reg, addr_reg, TCG_REG_R1);
@@ -1421,9 +1420,7 @@ static inline void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, int opc)
      * trash by moving the earlier arguments into them.
      */
     argreg = TCG_REG_R0;
-#ifdef CONFIG_TCG_PASS_AREG0
     argreg = tcg_out_arg_reg32(s, argreg, TCG_AREG0);
-#endif
 #if TARGET_LONG_BITS == 64
     argreg = tcg_out_arg_reg64(s, argreg, addr_reg, addr_reg2);
 #else
@@ -1472,7 +1469,7 @@ static inline void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, int opc)
         break;
     case 1:
         if (bswap) {
-            tcg_out_bswap16(s, COND_AL, TCG_REG_R0, data_reg);
+            tcg_out_bswap16st(s, COND_AL, TCG_REG_R0, data_reg);
             tcg_out_st16_8(s, COND_AL, TCG_REG_R0, addr_reg, 0);
         } else {
             tcg_out_st16_8(s, COND_AL, data_reg, addr_reg, 0);
@@ -1561,12 +1558,6 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
         else
             tcg_out_callr(s, COND_AL, args[0]);
         break;
-    case INDEX_op_jmp:
-        if (const_args[0])
-            tcg_out_goto(s, COND_AL, args[0]);
-        else
-            tcg_out_bx(s, COND_AL, args[0]);
-        break;
     case INDEX_op_br:
         tcg_out_goto_label(s, COND_AL, args[0]);
         break;
@@ -1603,6 +1594,15 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_movi_i32:
         tcg_out_movi32(s, COND_AL, args[0], args[1]);
         break;
+    case INDEX_op_movcond_i32:
+        /* Constraints mean that v2 is always in the same register as dest,
+         * so we only need to do "if condition passed, move v1 to dest".
+         */
+        tcg_out_dat_rI(s, COND_AL, ARITH_CMP, 0,
+                       args[1], args[2], const_args[2]);
+        tcg_out_dat_rI(s, tcg_cond_to_arm_cond[args[5]],
+                       ARITH_MOV, args[0], 0, args[3], const_args[3]);
+        break;
     case INDEX_op_add_i32:
         c = ARITH_ADD;
         goto gen_arith;
@@ -1622,14 +1622,7 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
         c = ARITH_EOR;
         /* Fall through.  */
     gen_arith:
-        if (const_args[2]) {
-            int rot;
-            rot = encode_imm(args[2]);
-            tcg_out_dat_imm(s, COND_AL, c,
-                            args[0], args[1], rotl(args[2], rot) | (rot << 7));
-        } else
-            tcg_out_dat_reg(s, COND_AL, c,
-                            args[0], args[1], args[2], SHIFT_IMM_LSL(0));
+        tcg_out_dat_rI(s, COND_AL, c, args[0], args[1], args[2], const_args[2]);
         break;
     case INDEX_op_add2_i32:
         tcg_out_dat_reg2(s, COND_AL, ARITH_ADD, ARITH_ADC,
@@ -1689,15 +1682,8 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
         break;
 
     case INDEX_op_brcond_i32:
-        if (const_args[1]) {
-            int rot;
-            rot = encode_imm(args[1]);
-            tcg_out_dat_imm(s, COND_AL, ARITH_CMP, 0,
-                            args[0], rotl(args[1], rot) | (rot << 7));
-        } else {
-            tcg_out_dat_reg(s, COND_AL, ARITH_CMP, 0,
-                            args[0], args[1], SHIFT_IMM_LSL(0));
-        }
+        tcg_out_dat_rI(s, COND_AL, ARITH_CMP, 0,
+                       args[0], args[1], const_args[1]);
         tcg_out_goto_label(s, tcg_cond_to_arm_cond[args[2]], args[3]);
         break;
     case INDEX_op_brcond2_i32:
@@ -1716,15 +1702,8 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
         tcg_out_goto_label(s, tcg_cond_to_arm_cond[args[4]], args[5]);
         break;
     case INDEX_op_setcond_i32:
-        if (const_args[2]) {
-            int rot;
-            rot = encode_imm(args[2]);
-            tcg_out_dat_imm(s, COND_AL, ARITH_CMP, 0,
-                            args[1], rotl(args[2], rot) | (rot << 7));
-        } else {
-            tcg_out_dat_reg(s, COND_AL, ARITH_CMP, 0,
-                            args[1], args[2], SHIFT_IMM_LSL(0));
-        }
+        tcg_out_dat_rI(s, COND_AL, ARITH_CMP, 0,
+                       args[1], args[2], const_args[2]);
         tcg_out_dat_imm(s, tcg_cond_to_arm_cond[args[3]],
                         ARITH_MOV, args[0], 0, 1);
         tcg_out_dat_imm(s, tcg_cond_to_arm_cond[tcg_invert_cond(args[3])],
@@ -1800,7 +1779,6 @@ static const TCGTargetOpDef arm_op_defs[] = {
     { INDEX_op_exit_tb, { } },
     { INDEX_op_goto_tb, { } },
     { INDEX_op_call, { "ri" } },
-    { INDEX_op_jmp, { "ri" } },
     { INDEX_op_br, { } },
 
     { INDEX_op_mov_i32, { "r", "r" } },
@@ -1835,6 +1813,7 @@ static const TCGTargetOpDef arm_op_defs[] = {
 
     { INDEX_op_brcond_i32, { "r", "rI" } },
     { INDEX_op_setcond_i32, { "r", "r", "rI" } },
+    { INDEX_op_movcond_i32, { "r", "r", "rI", "rI", "0" } },
 
     /* TODO: "r", "r", "r", "r", "ri", "ri" */
     { INDEX_op_add2_i32, { "r", "r", "r", "r", "r", "r" } },
diff --git a/tcg/arm/tcg-target.h b/tcg/arm/tcg-target.h
index f90b83479..98fa11b28 100644
--- a/tcg/arm/tcg-target.h
+++ b/tcg/arm/tcg-target.h
@@ -73,11 +73,9 @@ typedef enum {
 #define TCG_TARGET_HAS_nand_i32         0
 #define TCG_TARGET_HAS_nor_i32          0
 #define TCG_TARGET_HAS_deposit_i32      0
-
-#define TCG_TARGET_HAS_GUEST_BASE
+#define TCG_TARGET_HAS_movcond_i32      1
 
 enum {
-    /* Note: must be synced with dyngen-exec.h */
     TCG_AREG0 = TCG_REG_R6,
 };
 
diff --git a/tcg/hppa/tcg-target.c b/tcg/hppa/tcg-target.c
index 2885212e3..de500ae18 100644
--- a/tcg/hppa/tcg-target.c
+++ b/tcg/hppa/tcg-target.c
@@ -175,12 +175,6 @@ static void patch_reloc(uint8_t *code_ptr, int type,
     *insn_ptr = insn;
 }
 
-/* maximum number of register used for input function arguments */
-static inline int tcg_target_get_call_iarg_regs_count(int flags)
-{
-    return 4;
-}
-
 /* parse target specific constraints */
 static int target_parse_constraint(TCGArgConstraint *ct, const char **pct_str)
 {
@@ -738,7 +732,7 @@ static void tcg_out_branch(TCGContext *s, int label_index, int nul)
     }
 }
 
-static const uint8_t tcg_cond_to_cmp_cond[10] =
+static const uint8_t tcg_cond_to_cmp_cond[] =
 {
     [TCG_COND_EQ] = COND_EQ,
     [TCG_COND_NE] = COND_EQ | COND_FALSE,
@@ -826,13 +820,15 @@ static void tcg_out_brcond2(TCGContext *s, int cond, TCGArg al, TCGArg ah,
 {
     switch (cond) {
     case TCG_COND_EQ:
+        tcg_out_comclr(s, TCG_COND_NE, TCG_REG_R0, al, bl, blconst);
+        tcg_out_brcond(s, TCG_COND_EQ, ah, bh, bhconst, label_index);
+        break;
     case TCG_COND_NE:
-        tcg_out_comclr(s, tcg_invert_cond(cond), TCG_REG_R0, al, bl, blconst);
-        tcg_out_brcond(s, cond, ah, bh, bhconst, label_index);
+        tcg_out_brcond(s, TCG_COND_NE, al, bl, bhconst, label_index);
+        tcg_out_brcond(s, TCG_COND_NE, ah, bh, bhconst, label_index);
         break;
-
     default:
-        tcg_out_brcond(s, cond, ah, bh, bhconst, label_index);
+        tcg_out_brcond(s, tcg_high_cond(cond), ah, bh, bhconst, label_index);
         tcg_out_comclr(s, TCG_COND_NE, TCG_REG_R0, ah, bh, bhconst);
         tcg_out_brcond(s, tcg_unsigned_cond(cond),
                        al, bl, blconst, label_index);
@@ -853,9 +849,8 @@ static void tcg_out_setcond2(TCGContext *s, int cond, TCGArg ret,
 {
     int scratch = TCG_REG_R20;
 
-    if (ret != al && ret != ah
-        && (blconst || ret != bl)
-        && (bhconst || ret != bh)) {
+    /* Note that the low parts are fully consumed before scratch is set.  */
+    if (ret != ah && (bhconst || ret != bh)) {
         scratch = ret;
     }
 
@@ -867,22 +862,52 @@ static void tcg_out_setcond2(TCGContext *s, int cond, TCGArg ret,
         tcg_out_movi(s, TCG_TYPE_I32, scratch, cond == TCG_COND_NE);
         break;
 
-    default:
+    case TCG_COND_GE:
+    case TCG_COND_GEU:
+    case TCG_COND_LT:
+    case TCG_COND_LTU:
+        /* Optimize compares with low part zero.  */
+        if (bl == 0) {
+            tcg_out_setcond(s, cond, ret, ah, bh, bhconst);
+            return;
+        }
+        /* FALLTHRU */
+
+    case TCG_COND_LE:
+    case TCG_COND_LEU:
+    case TCG_COND_GT:
+    case TCG_COND_GTU:
+        /* <= : ah < bh | (ah == bh && al <= bl) */
         tcg_out_setcond(s, tcg_unsigned_cond(cond), scratch, al, bl, blconst);
         tcg_out_comclr(s, TCG_COND_EQ, TCG_REG_R0, ah, bh, bhconst);
         tcg_out_movi(s, TCG_TYPE_I32, scratch, 0);
-        tcg_out_comclr(s, cond, TCG_REG_R0, ah, bh, bhconst);
+        tcg_out_comclr(s, tcg_invert_cond(tcg_high_cond(cond)),
+                       TCG_REG_R0, ah, bh, bhconst);
         tcg_out_movi(s, TCG_TYPE_I32, scratch, 1);
         break;
+
+    default:
+        tcg_abort();
     }
 
     tcg_out_mov(s, TCG_TYPE_I32, ret, scratch);
 }
 
+static void tcg_out_movcond(TCGContext *s, int cond, TCGArg ret,
+                            TCGArg c1, TCGArg c2, int c2const,
+                            TCGArg v1, int v1const)
+{
+    tcg_out_comclr(s, tcg_invert_cond(cond), TCG_REG_R0, c1, c2, c2const);
+    if (v1const) {
+        tcg_out_movi(s, TCG_TYPE_I32, ret, v1);
+    } else {
+        tcg_out_mov(s, TCG_TYPE_I32, ret, v1);
+    }
+}
+
 #if defined(CONFIG_SOFTMMU)
 #include "../../softmmu_defs.h"
 
-#ifdef CONFIG_TCG_PASS_AREG0
 /* helper signature: helper_ld_mmu(CPUState *env, target_ulong addr,
    int mmu_idx) */
 static const void * const qemu_ld_helpers[4] = {
@@ -900,25 +925,6 @@ static const void * const qemu_st_helpers[4] = {
     helper_stl_mmu,
     helper_stq_mmu,
 };
-#else
-/* legacy helper signature: __ld_mmu(target_ulong addr, int
-   mmu_idx) */
-static void *qemu_ld_helpers[4] = {
-    __ldb_mmu,
-    __ldw_mmu,
-    __ldl_mmu,
-    __ldq_mmu,
-};
-
-/* legacy helper signature: __st_mmu(target_ulong addr, uintxx_t val,
-   int mmu_idx) */
-static void *qemu_st_helpers[4] = {
-    __stb_mmu,
-    __stw_mmu,
-    __stl_mmu,
-    __stq_mmu,
-};
-#endif
 
 /* Load and compare a TLB entry, and branch if TLB miss.  OFFSET is set to
    the offset of the first ADDR_READ or ADDR_WRITE member of the appropriate
@@ -963,10 +969,11 @@ static int tcg_out_tlb_read(TCGContext *s, int r0, int r1, int addrlo,
         tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_R20, r1, offset);
     }
 
-    /* Compute the value that ought to appear in the TLB for a hit, namely, the page
-       of the address.  We include the low N bits of the address to catch unaligned
-       accesses and force them onto the slow path.  Do this computation after having
-       issued the load from the TLB slot to give the load time to complete.  */
+    /* Compute the value that ought to appear in the TLB for a hit, namely,
+       the page of the address.  We include the low N bits of the address
+       to catch unaligned accesses and force them onto the slow path.  Do
+       this computation after having issued the load from the TLB slot to
+       give the load time to complete.  */
     tcg_out_andi(s, r0, addrlo, TARGET_PAGE_MASK | ((1 << s_bits) - 1));
 
     /* If not equal, jump to lab_miss. */
@@ -979,6 +986,36 @@ static int tcg_out_tlb_read(TCGContext *s, int r0, int r1, int addrlo,
 
     return ret;
 }
+
+static int tcg_out_arg_reg32(TCGContext *s, int argno, TCGArg v, bool vconst)
+{
+    if (argno < 4) {
+        if (vconst) {
+            tcg_out_movi(s, TCG_TYPE_I32, tcg_target_call_iarg_regs[argno], v);
+        } else {
+            tcg_out_mov(s, TCG_TYPE_I32, tcg_target_call_iarg_regs[argno], v);
+        }
+    } else {
+        if (vconst && v != 0) {
+            tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_R20, v);
+            v = TCG_REG_R20;
+        }
+        tcg_out_st(s, TCG_TYPE_I32, v, TCG_REG_CALL_STACK,
+                   TCG_TARGET_CALL_STACK_OFFSET - ((argno - 3) * 4));
+    }
+    return argno + 1;
+}
+
+static int tcg_out_arg_reg64(TCGContext *s, int argno, TCGArg vl, TCGArg vh)
+{
+    /* 64-bit arguments must go in even reg pairs and stack slots.  */
+    if (argno & 1) {
+        argno++;
+    }
+    argno = tcg_out_arg_reg32(s, argno, vl, false);
+    argno = tcg_out_arg_reg32(s, argno, vh, false);
+    return argno;
+}
 #endif
 
 static void tcg_out_qemu_ld_direct(TCGContext *s, int datalo_reg, int datahi_reg,
@@ -1059,41 +1096,36 @@ static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, int opc)
     /* Note that addrhi_reg is only used for 64-bit guests.  */
     int addrhi_reg = (TARGET_LONG_BITS == 64 ? *args++ : TCG_REG_R0);
     int mem_index = *args;
-    int lab1, lab2, argreg, offset;
+    int lab1, lab2, argno, offset;
 
     lab1 = gen_new_label();
     lab2 = gen_new_label();
 
     offset = offsetof(CPUArchState, tlb_table[mem_index][0].addr_read);
-    offset = tcg_out_tlb_read(s, TCG_REG_R26, TCG_REG_R25, addrlo_reg, addrhi_reg,
-                              opc & 3, lab1, offset);
+    offset = tcg_out_tlb_read(s, TCG_REG_R26, TCG_REG_R25, addrlo_reg,
+                              addrhi_reg, opc & 3, lab1, offset);
 
     /* TLB Hit.  */
-    tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_R20, (offset ? TCG_REG_R1 : TCG_REG_R25),
+    tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_R20,
+               (offset ? TCG_REG_R1 : TCG_REG_R25),
                offsetof(CPUArchState, tlb_table[mem_index][0].addend) - offset);
-    tcg_out_qemu_ld_direct(s, datalo_reg, datahi_reg, addrlo_reg, TCG_REG_R20, opc);
+    tcg_out_qemu_ld_direct(s, datalo_reg, datahi_reg, addrlo_reg,
+                           TCG_REG_R20, opc);
     tcg_out_branch(s, lab2, 1);
 
     /* TLB Miss.  */
     /* label1: */
     tcg_out_label(s, lab1, s->code_ptr);
 
-    argreg = TCG_REG_R26;
-    tcg_out_mov(s, TCG_TYPE_I32, argreg--, addrlo_reg);
+    argno = 0;
+    argno = tcg_out_arg_reg32(s, argno, TCG_AREG0, false);
     if (TARGET_LONG_BITS == 64) {
-        tcg_out_mov(s, TCG_TYPE_I32, argreg--, addrhi_reg);
+        argno = tcg_out_arg_reg64(s, argno, addrlo_reg, addrhi_reg);
+    } else {
+        argno = tcg_out_arg_reg32(s, argno, addrlo_reg, false);
     }
-    tcg_out_movi(s, TCG_TYPE_I32, argreg, mem_index);
-
-#ifdef CONFIG_TCG_PASS_AREG0
-    /* XXX/FIXME: suboptimal */
-    tcg_out_mov(s, TCG_TYPE_I32, tcg_target_call_iarg_regs[2],
-                tcg_target_call_iarg_regs[1]);
-    tcg_out_mov(s, TCG_TYPE_TL, tcg_target_call_iarg_regs[1],
-                tcg_target_call_iarg_regs[0]);
-    tcg_out_mov(s, TCG_TYPE_PTR, tcg_target_call_iarg_regs[0],
-                TCG_AREG0);
-#endif
+    argno = tcg_out_arg_reg32(s, argno, mem_index, true);
+
     tcg_out_call(s, qemu_ld_helpers[opc & 3]);
 
     switch (opc) {
@@ -1129,8 +1161,8 @@ static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, int opc)
 #endif
 }
 
-static void tcg_out_qemu_st_direct(TCGContext *s, int datalo_reg, int datahi_reg,
-                                   int addr_reg, int opc)
+static void tcg_out_qemu_st_direct(TCGContext *s, int datalo_reg,
+                                   int datahi_reg, int addr_reg, int opc)
 {
 #ifdef TARGET_WORDS_BIGENDIAN
     const int bswap = 0;
@@ -1183,17 +1215,18 @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, int opc)
     /* Note that addrhi_reg is only used for 64-bit guests.  */
     int addrhi_reg = (TARGET_LONG_BITS == 64 ? *args++ : TCG_REG_R0);
     int mem_index = *args;
-    int lab1, lab2, argreg, offset;
+    int lab1, lab2, argno, next, offset;
 
     lab1 = gen_new_label();
     lab2 = gen_new_label();
 
     offset = offsetof(CPUArchState, tlb_table[mem_index][0].addr_write);
-    offset = tcg_out_tlb_read(s, TCG_REG_R26, TCG_REG_R25, addrlo_reg, addrhi_reg,
-                              opc, lab1, offset);
+    offset = tcg_out_tlb_read(s, TCG_REG_R26, TCG_REG_R25, addrlo_reg,
+                              addrhi_reg, opc, lab1, offset);
 
     /* TLB Hit.  */
-    tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_R20, (offset ? TCG_REG_R1 : TCG_REG_R25),
+    tcg_out_ld(s, TCG_TYPE_PTR, TCG_REG_R20,
+               (offset ? TCG_REG_R1 : TCG_REG_R25),
                offsetof(CPUArchState, tlb_table[mem_index][0].addend) - offset);
 
     /* There are no indexed stores, so we must do this addition explitly.
@@ -1206,65 +1239,46 @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, int opc)
     /* label1: */
     tcg_out_label(s, lab1, s->code_ptr);
 
-    argreg = TCG_REG_R26;
-    tcg_out_mov(s, TCG_TYPE_I32, argreg--, addrlo_reg);
+    argno = 0;
+    argno = tcg_out_arg_reg32(s, argno, TCG_AREG0, false);
     if (TARGET_LONG_BITS == 64) {
-        tcg_out_mov(s, TCG_TYPE_I32, argreg--, addrhi_reg);
+        argno = tcg_out_arg_reg64(s, argno, addrlo_reg, addrhi_reg);
+    } else {
+        argno = tcg_out_arg_reg32(s, argno, addrlo_reg, false);
     }
 
+    next = (argno < 4 ? tcg_target_call_iarg_regs[argno] : TCG_REG_R20);
     switch(opc) {
     case 0:
-        tcg_out_andi(s, argreg--, datalo_reg, 0xff);
-        tcg_out_movi(s, TCG_TYPE_I32, argreg, mem_index);
+        tcg_out_andi(s, next, datalo_reg, 0xff);
+        argno = tcg_out_arg_reg32(s, argno, next, false);
         break;
     case 1:
-        tcg_out_andi(s, argreg--, datalo_reg, 0xffff);
-        tcg_out_movi(s, TCG_TYPE_I32, argreg, mem_index);
+        tcg_out_andi(s, next, datalo_reg, 0xffff);
+        argno = tcg_out_arg_reg32(s, argno, next, false);
         break;
     case 2:
-        tcg_out_mov(s, TCG_TYPE_I32, argreg--, datalo_reg);
-        tcg_out_movi(s, TCG_TYPE_I32, argreg, mem_index);
+        argno = tcg_out_arg_reg32(s, argno, datalo_reg, false);
         break;
     case 3:
-        /* Because of the alignment required by the 64-bit data argument,
-           we will always use R23/R24.  Also, we will always run out of
-           argument registers for storing mem_index, so that will have
-           to go on the stack.  */
-        if (mem_index == 0) {
-            argreg = TCG_REG_R0;
-        } else {
-            argreg = TCG_REG_R20;
-            tcg_out_movi(s, TCG_TYPE_I32, argreg, mem_index);
-        }
-        tcg_out_mov(s, TCG_TYPE_I32, TCG_REG_R23, datahi_reg);
-        tcg_out_mov(s, TCG_TYPE_I32, TCG_REG_R24, datalo_reg);
-        tcg_out_st(s, TCG_TYPE_I32, argreg, TCG_REG_CALL_STACK,
-                   TCG_TARGET_CALL_STACK_OFFSET - 4);
+        argno = tcg_out_arg_reg64(s, argno, datalo_reg, datahi_reg);
         break;
     default:
         tcg_abort();
     }
+    argno = tcg_out_arg_reg32(s, argno, mem_index, true);
 
-#ifdef CONFIG_TCG_PASS_AREG0
-    /* XXX/FIXME: suboptimal */
-    tcg_out_mov(s, TCG_TYPE_I32, tcg_target_call_iarg_regs[3],
-                tcg_target_call_iarg_regs[2]);
-    tcg_out_mov(s, TCG_TYPE_I64, tcg_target_call_iarg_regs[2],
-                tcg_target_call_iarg_regs[1]);
-    tcg_out_mov(s, TCG_TYPE_TL, tcg_target_call_iarg_regs[1],
-                tcg_target_call_iarg_regs[0]);
-    tcg_out_mov(s, TCG_TYPE_PTR, tcg_target_call_iarg_regs[0],
-                TCG_AREG0);
-#endif
     tcg_out_call(s, qemu_st_helpers[opc]);
 
     /* label2: */
     tcg_out_label(s, lab2, s->code_ptr);
 #else
-    /* There are no indexed stores, so if GUEST_BASE is set we must do the add
-       explicitly.  Careful to avoid R20, which is used for the bswaps to follow.  */
+    /* There are no indexed stores, so if GUEST_BASE is set we must do
+       the add explicitly.  Careful to avoid R20, which is used for the
+       bswaps to follow.  */
     if (GUEST_BASE != 0) {
-        tcg_out_arith(s, TCG_REG_R31, addrlo_reg, TCG_GUEST_BASE_REG, INSN_ADDL);
+        tcg_out_arith(s, TCG_REG_R31, addrlo_reg,
+                      TCG_GUEST_BASE_REG, INSN_ADDL);
         addrlo_reg = TCG_REG_R31;
     }
     tcg_out_qemu_st_direct(s, datalo_reg, datahi_reg, addrlo_reg, opc);
@@ -1326,11 +1340,6 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
         }
         break;
 
-    case INDEX_op_jmp:
-        fprintf(stderr, "unimplemented jmp\n");
-        tcg_abort();
-        break;
-
     case INDEX_op_br:
         tcg_out_branch(s, args[0], 1);
         break;
@@ -1499,6 +1508,11 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
                          args[3], const_args[3], args[4], const_args[4]);
         break;
 
+    case INDEX_op_movcond_i32:
+        tcg_out_movcond(s, args[5], args[0], args[1], args[2], const_args[2],
+                        args[3], const_args[3]);
+        break;
+
     case INDEX_op_add2_i32:
         tcg_out_add2(s, args[0], args[1], args[2], args[3],
                      args[4], args[5], const_args[4]);
@@ -1560,7 +1574,6 @@ static const TCGTargetOpDef hppa_op_defs[] = {
     { INDEX_op_goto_tb, { } },
 
     { INDEX_op_call, { "ri" } },
-    { INDEX_op_jmp, { "r" } },
     { INDEX_op_br, { } },
 
     { INDEX_op_mov_i32, { "r", "r" } },
@@ -1607,6 +1620,10 @@ static const TCGTargetOpDef hppa_op_defs[] = {
     { INDEX_op_setcond_i32, { "r", "rZ", "rI" } },
     { INDEX_op_setcond2_i32, { "r", "rZ", "rZ", "rI", "rI" } },
 
+    /* ??? We can actually support a signed 14-bit arg3, but we
+       only have existing constraints for a signed 11-bit.  */
+    { INDEX_op_movcond_i32, { "r", "rZ", "rI", "rI", "0" } },
+
     { INDEX_op_add2_i32, { "r", "r", "rZ", "rZ", "rI", "rZ" } },
     { INDEX_op_sub2_i32, { "r", "r", "rI", "rZ", "rK", "rZ" } },
 
diff --git a/tcg/hppa/tcg-target.h b/tcg/hppa/tcg-target.h
index d4bf6fe75..f43fb419a 100644
--- a/tcg/hppa/tcg-target.h
+++ b/tcg/hppa/tcg-target.h
@@ -96,15 +96,13 @@ typedef enum {
 #define TCG_TARGET_HAS_nand_i32         0
 #define TCG_TARGET_HAS_nor_i32          0
 #define TCG_TARGET_HAS_deposit_i32      1
+#define TCG_TARGET_HAS_movcond_i32      1
 
 /* optional instructions automatically implemented */
 #define TCG_TARGET_HAS_neg_i32          0 /* sub rd, 0, rs */
 #define TCG_TARGET_HAS_ext8u_i32        0 /* and rd, rs, 0xff */
 #define TCG_TARGET_HAS_ext16u_i32       0 /* and rd, rs, 0xffff */
 
-#define TCG_TARGET_HAS_GUEST_BASE
-
-/* Note: must be synced with dyngen-exec.h */
 #define TCG_AREG0 TCG_REG_R17
 
 
diff --git a/tcg/i386/tcg-target.c b/tcg/i386/tcg-target.c
index da17bba39..6f3ad3cea 100644
--- a/tcg/i386/tcg-target.c
+++ b/tcg/i386/tcg-target.c
@@ -75,9 +75,7 @@ static const int tcg_target_call_iarg_regs[] = {
     TCG_REG_R8,
     TCG_REG_R9,
 #else
-    TCG_REG_EAX,
-    TCG_REG_EDX,
-    TCG_REG_ECX
+    /* 32 bit mode uses stack based calling convention (GCC default). */
 #endif
 };
 
@@ -88,6 +86,17 @@ static const int tcg_target_call_oarg_regs[] = {
 #endif
 };
 
+/* Registers used with L constraint, which are the first argument 
+   registers on x86_64, and two random call clobbered registers on
+   i386. */
+#if TCG_TARGET_REG_BITS == 64
+# define TCG_REG_L0 tcg_target_call_iarg_regs[0]
+# define TCG_REG_L1 tcg_target_call_iarg_regs[1]
+#else
+# define TCG_REG_L0 TCG_REG_EAX
+# define TCG_REG_L1 TCG_REG_EDX
+#endif
+
 static uint8_t *tb_ret_addr;
 
 static void patch_reloc(uint8_t *code_ptr, int type,
@@ -114,16 +123,6 @@ static void patch_reloc(uint8_t *code_ptr, int type,
     }
 }
 
-/* maximum number of register used for input function arguments */
-static inline int tcg_target_get_call_iarg_regs_count(int flags)
-{
-    if (TCG_TARGET_REG_BITS == 64) {
-        return 6;
-    }
-
-    return 0;
-}
-
 /* parse target specific constraints */
 static int target_parse_constraint(TCGArgConstraint *ct, const char **pct_str)
 {
@@ -179,18 +178,13 @@ static int target_parse_constraint(TCGArgConstraint *ct, const char **pct_str)
         /* qemu_ld/st address constraint */
     case 'L':
         ct->ct |= TCG_CT_REG;
-        if (TCG_TARGET_REG_BITS == 64) {
+#if TCG_TARGET_REG_BITS == 64
             tcg_regset_set32(ct->u.regs, 0, 0xffff);
-            tcg_regset_reset_reg(ct->u.regs, tcg_target_call_iarg_regs[0]);
-            tcg_regset_reset_reg(ct->u.regs, tcg_target_call_iarg_regs[1]);
-#ifdef CONFIG_TCG_PASS_AREG0
-            tcg_regset_reset_reg(ct->u.regs, tcg_target_call_iarg_regs[2]);
-#endif
-        } else {
+#else
             tcg_regset_set32(ct->u.regs, 0, 0xff);
-            tcg_regset_reset_reg(ct->u.regs, TCG_REG_EAX);
-            tcg_regset_reset_reg(ct->u.regs, TCG_REG_EDX);
-        }
+#endif
+        tcg_regset_reset_reg(ct->u.regs, TCG_REG_L0);
+        tcg_regset_reset_reg(ct->u.regs, TCG_REG_L1);
         break;
 
     case 'e':
@@ -238,11 +232,13 @@ static inline int tcg_target_const_match(tcg_target_long val,
 # define P_REXW		0x800		/* Set REX.W = 1 */
 # define P_REXB_R	0x1000		/* REG field as byte register */
 # define P_REXB_RM	0x2000		/* R/M field as byte register */
+# define P_GS           0x4000          /* gs segment override */
 #else
 # define P_ADDR32	0
 # define P_REXW		0
 # define P_REXB_R	0
 # define P_REXB_RM	0
+# define P_GS           0
 #endif
 
 #define OPC_ARITH_EvIz	(0x81)
@@ -251,6 +247,7 @@ static inline int tcg_target_const_match(tcg_target_long val,
 #define OPC_ADD_GvEv	(OPC_ARITH_GvEv | (ARITH_ADD << 3))
 #define OPC_BSWAP	(0xc8 | P_EXT)
 #define OPC_CALL_Jz	(0xe8)
+#define OPC_CMOVCC      (0x40 | P_EXT)  /* ... plus condition code */
 #define OPC_CMP_GvEv	(OPC_ARITH_GvEv | (ARITH_CMP << 3))
 #define OPC_DEC_r32	(0x48)
 #define OPC_IMUL_GvEv	(0xaf | P_EXT)
@@ -265,6 +262,7 @@ static inline int tcg_target_const_match(tcg_target_long val,
 #define OPC_MOVB_EvGv	(0x88)		/* stores, more or less */
 #define OPC_MOVL_EvGv	(0x89)		/* stores, more or less */
 #define OPC_MOVL_GvEv	(0x8b)		/* loads, more or less */
+#define OPC_MOVB_EvIz   (0xc6)
 #define OPC_MOVL_EvIz	(0xc7)
 #define OPC_MOVL_Iv     (0xb8)
 #define OPC_MOVSBL	(0xbe | P_EXT)
@@ -338,7 +336,7 @@ static inline int tcg_target_const_match(tcg_target_long val,
 #define JCC_JLE 0xe
 #define JCC_JG  0xf
 
-static const uint8_t tcg_cond_to_jcc[10] = {
+static const uint8_t tcg_cond_to_jcc[] = {
     [TCG_COND_EQ] = JCC_JE,
     [TCG_COND_NE] = JCC_JNE,
     [TCG_COND_LT] = JCC_JL,
@@ -356,6 +354,9 @@ static void tcg_out_opc(TCGContext *s, int opc, int r, int rm, int x)
 {
     int rex;
 
+    if (opc & P_GS) {
+        tcg_out8(s, 0x65);
+    }
     if (opc & P_DATA16) {
         /* We should never be asking for both 16 and 64-bit operation.  */
         assert((opc & P_REXW) == 0);
@@ -937,6 +938,24 @@ static void tcg_out_setcond2(TCGContext *s, const TCGArg *args,
 }
 #endif
 
+static void tcg_out_movcond32(TCGContext *s, TCGCond cond, TCGArg dest,
+                              TCGArg c1, TCGArg c2, int const_c2,
+                              TCGArg v1)
+{
+    tcg_out_cmp(s, c1, c2, const_c2, 0);
+    tcg_out_modrm(s, OPC_CMOVCC | tcg_cond_to_jcc[cond], dest, v1);
+}
+
+#if TCG_TARGET_REG_BITS == 64
+static void tcg_out_movcond64(TCGContext *s, TCGCond cond, TCGArg dest,
+                              TCGArg c1, TCGArg c2, int const_c2,
+                              TCGArg v1)
+{
+    tcg_out_cmp(s, c1, c2, const_c2, P_REXW);
+    tcg_out_modrm(s, OPC_CMOVCC | tcg_cond_to_jcc[cond] | P_REXW, dest, v1);
+}
+#endif
+
 static void tcg_out_branch(TCGContext *s, int call, tcg_target_long dest)
 {
     tcg_target_long disp = dest - (tcg_target_long)s->code_ptr - 5;
@@ -965,7 +984,6 @@ static void tcg_out_jmp(TCGContext *s, tcg_target_long dest)
 
 #include "../../softmmu_defs.h"
 
-#ifdef CONFIG_TCG_PASS_AREG0
 /* helper signature: helper_ld_mmu(CPUState *env, target_ulong addr,
    int mmu_idx) */
 static const void *qemu_ld_helpers[4] = {
@@ -983,25 +1001,17 @@ static const void *qemu_st_helpers[4] = {
     helper_stl_mmu,
     helper_stq_mmu,
 };
-#else
-/* legacy helper signature: __ld_mmu(target_ulong addr, int
-   mmu_idx) */
-static void *qemu_ld_helpers[4] = {
-    __ldb_mmu,
-    __ldw_mmu,
-    __ldl_mmu,
-    __ldq_mmu,
-};
 
-/* legacy helper signature: __st_mmu(target_ulong addr, uintxx_t val,
-   int mmu_idx) */
-static void *qemu_st_helpers[4] = {
-    __stb_mmu,
-    __stw_mmu,
-    __stl_mmu,
-    __stq_mmu,
-};
-#endif
+static void add_qemu_ldst_label(TCGContext *s,
+                                int is_ld,
+                                int opc,
+                                int data_reg,
+                                int data_reg2,
+                                int addrlo_reg,
+                                int addrhi_reg,
+                                int mem_index,
+                                uint8_t *raddr,
+                                uint8_t **label_ptr);
 
 /* Perform the TLB load and compare.
 
@@ -1018,12 +1028,12 @@ static void *qemu_st_helpers[4] = {
    LABEL_PTRS is filled with 1 (32-bit addresses) or 2 (64-bit addresses)
    positions of the displacements of forward jumps to the TLB miss case.
 
-   First argument register is loaded with the low part of the address.
+   Second argument register is loaded with the low part of the address.
    In the TLB hit case, it has been adjusted as indicated by the TLB
    and so is a host address.  In the TLB miss case, it continues to
    hold a guest address.
 
-   Second argument register is clobbered.  */
+   First argument register is clobbered.  */
 
 static inline void tcg_out_tlb_load(TCGContext *s, int addrlo_idx,
                                     int mem_index, int s_bits,
@@ -1031,8 +1041,8 @@ static inline void tcg_out_tlb_load(TCGContext *s, int addrlo_idx,
                                     uint8_t **label_ptr, int which)
 {
     const int addrlo = args[addrlo_idx];
-    const int r0 = tcg_target_call_iarg_regs[0];
-    const int r1 = tcg_target_call_iarg_regs[1];
+    const int r0 = TCG_REG_L0;
+    const int r1 = TCG_REG_L1;
     TCGType type = TCG_TYPE_I32;
     int rexw = 0;
 
@@ -1041,51 +1051,68 @@ static inline void tcg_out_tlb_load(TCGContext *s, int addrlo_idx,
         rexw = P_REXW;
     }
 
-    tcg_out_mov(s, type, r1, addrlo);
     tcg_out_mov(s, type, r0, addrlo);
+    tcg_out_mov(s, type, r1, addrlo);
 
-    tcg_out_shifti(s, SHIFT_SHR + rexw, r1,
+    tcg_out_shifti(s, SHIFT_SHR + rexw, r0,
                    TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS);
 
-    tgen_arithi(s, ARITH_AND + rexw, r0,
-                TARGET_PAGE_MASK | ((1 << s_bits) - 1), 0);
     tgen_arithi(s, ARITH_AND + rexw, r1,
+                TARGET_PAGE_MASK | ((1 << s_bits) - 1), 0);
+    tgen_arithi(s, ARITH_AND + rexw, r0,
                 (CPU_TLB_SIZE - 1) << CPU_TLB_ENTRY_BITS, 0);
 
-    tcg_out_modrm_sib_offset(s, OPC_LEA + P_REXW, r1, TCG_AREG0, r1, 0,
+    tcg_out_modrm_sib_offset(s, OPC_LEA + P_REXW, r0, TCG_AREG0, r0, 0,
                              offsetof(CPUArchState, tlb_table[mem_index][0])
                              + which);
 
-    /* cmp 0(r1), r0 */
-    tcg_out_modrm_offset(s, OPC_CMP_GvEv + rexw, r0, r1, 0);
+    /* cmp 0(r0), r1 */
+    tcg_out_modrm_offset(s, OPC_CMP_GvEv + rexw, r1, r0, 0);
 
-    tcg_out_mov(s, type, r0, addrlo);
+    tcg_out_mov(s, type, r1, addrlo);
 
-    /* jne label1 */
-    tcg_out8(s, OPC_JCC_short + JCC_JNE);
+    /* jne slow_path */
+    tcg_out_opc(s, OPC_JCC_long + JCC_JNE, 0, 0, 0);
     label_ptr[0] = s->code_ptr;
-    s->code_ptr++;
+    s->code_ptr += 4;
 
     if (TARGET_LONG_BITS > TCG_TARGET_REG_BITS) {
-        /* cmp 4(r1), addrhi */
-        tcg_out_modrm_offset(s, OPC_CMP_GvEv, args[addrlo_idx+1], r1, 4);
+        /* cmp 4(r0), addrhi */
+        tcg_out_modrm_offset(s, OPC_CMP_GvEv, args[addrlo_idx+1], r0, 4);
 
-        /* jne label1 */
-        tcg_out8(s, OPC_JCC_short + JCC_JNE);
+        /* jne slow_path */
+        tcg_out_opc(s, OPC_JCC_long + JCC_JNE, 0, 0, 0);
         label_ptr[1] = s->code_ptr;
-        s->code_ptr++;
+        s->code_ptr += 4;
     }
 
     /* TLB Hit.  */
 
-    /* add addend(r1), r0 */
-    tcg_out_modrm_offset(s, OPC_ADD_GvEv + P_REXW, r0, r1,
+    /* add addend(r0), r1 */
+    tcg_out_modrm_offset(s, OPC_ADD_GvEv + P_REXW, r1, r0,
                          offsetof(CPUTLBEntry, addend) - which);
 }
-#endif
+#elif defined(__x86_64__) && defined(__linux__)
+# include <asm/prctl.h>
+# include <sys/prctl.h>
+
+int arch_prctl(int code, unsigned long addr);
+
+static int guest_base_flags;
+static inline void setup_guest_base_seg(void)
+{
+    if (arch_prctl(ARCH_SET_GS, GUEST_BASE) == 0) {
+        guest_base_flags = P_GS;
+    }
+}
+#else
+# define guest_base_flags 0
+static inline void setup_guest_base_seg(void) { }
+#endif /* SOFTMMU */
 
 static void tcg_out_qemu_ld_direct(TCGContext *s, int datalo, int datahi,
-                                   int base, tcg_target_long ofs, int sizeop)
+                                   int base, tcg_target_long ofs, int seg,
+                                   int sizeop)
 {
 #ifdef TARGET_WORDS_BIGENDIAN
     const int bswap = 1;
@@ -1094,28 +1121,29 @@ static void tcg_out_qemu_ld_direct(TCGContext *s, int datalo, int datahi,
 #endif
     switch (sizeop) {
     case 0:
-        tcg_out_modrm_offset(s, OPC_MOVZBL, datalo, base, ofs);
+        tcg_out_modrm_offset(s, OPC_MOVZBL + seg, datalo, base, ofs);
         break;
     case 0 | 4:
-        tcg_out_modrm_offset(s, OPC_MOVSBL + P_REXW, datalo, base, ofs);
+        tcg_out_modrm_offset(s, OPC_MOVSBL + P_REXW + seg, datalo, base, ofs);
         break;
     case 1:
-        tcg_out_modrm_offset(s, OPC_MOVZWL, datalo, base, ofs);
+        tcg_out_modrm_offset(s, OPC_MOVZWL + seg, datalo, base, ofs);
         if (bswap) {
             tcg_out_rolw_8(s, datalo);
         }
         break;
     case 1 | 4:
         if (bswap) {
-            tcg_out_modrm_offset(s, OPC_MOVZWL, datalo, base, ofs);
+            tcg_out_modrm_offset(s, OPC_MOVZWL + seg, datalo, base, ofs);
             tcg_out_rolw_8(s, datalo);
             tcg_out_modrm(s, OPC_MOVSWL + P_REXW, datalo, datalo);
         } else {
-            tcg_out_modrm_offset(s, OPC_MOVSWL + P_REXW, datalo, base, ofs);
+            tcg_out_modrm_offset(s, OPC_MOVSWL + P_REXW + seg,
+                                 datalo, base, ofs);
         }
         break;
     case 2:
-        tcg_out_ld(s, TCG_TYPE_I32, datalo, base, ofs);
+        tcg_out_modrm_offset(s, OPC_MOVL_GvEv + seg, datalo, base, ofs);
         if (bswap) {
             tcg_out_bswap32(s, datalo);
         }
@@ -1123,17 +1151,18 @@ static void tcg_out_qemu_ld_direct(TCGContext *s, int datalo, int datahi,
 #if TCG_TARGET_REG_BITS == 64
     case 2 | 4:
         if (bswap) {
-            tcg_out_ld(s, TCG_TYPE_I32, datalo, base, ofs);
+            tcg_out_modrm_offset(s, OPC_MOVL_GvEv + seg, datalo, base, ofs);
             tcg_out_bswap32(s, datalo);
             tcg_out_ext32s(s, datalo, datalo);
         } else {
-            tcg_out_modrm_offset(s, OPC_MOVSLQ, datalo, base, ofs);
+            tcg_out_modrm_offset(s, OPC_MOVSLQ + seg, datalo, base, ofs);
         }
         break;
 #endif
     case 3:
         if (TCG_TARGET_REG_BITS == 64) {
-            tcg_out_ld(s, TCG_TYPE_I64, datalo, base, ofs);
+            tcg_out_modrm_offset(s, OPC_MOVL_GvEv + P_REXW + seg,
+                                 datalo, base, ofs);
             if (bswap) {
                 tcg_out_bswap64(s, datalo);
             }
@@ -1144,11 +1173,15 @@ static void tcg_out_qemu_ld_direct(TCGContext *s, int datalo, int datahi,
                 datahi = t;
             }
             if (base != datalo) {
-                tcg_out_ld(s, TCG_TYPE_I32, datalo, base, ofs);
-                tcg_out_ld(s, TCG_TYPE_I32, datahi, base, ofs + 4);
+                tcg_out_modrm_offset(s, OPC_MOVL_GvEv + seg,
+                                     datalo, base, ofs);
+                tcg_out_modrm_offset(s, OPC_MOVL_GvEv + seg,
+                                     datahi, base, ofs + 4);
             } else {
-                tcg_out_ld(s, TCG_TYPE_I32, datahi, base, ofs + 4);
-                tcg_out_ld(s, TCG_TYPE_I32, datalo, base, ofs);
+                tcg_out_modrm_offset(s, OPC_MOVL_GvEv + seg,
+                                     datahi, base, ofs + 4);
+                tcg_out_modrm_offset(s, OPC_MOVL_GvEv + seg,
+                                     datalo, base, ofs);
             }
             if (bswap) {
                 tcg_out_bswap32(s, datalo);
@@ -1171,12 +1204,7 @@ static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args,
     int addrlo_idx;
 #if defined(CONFIG_SOFTMMU)
     int mem_index, s_bits;
-#if TCG_TARGET_REG_BITS == 64
-    int arg_idx;
-#else
-    int stack_adjust;
-#endif
-    uint8_t *label_ptr[3];
+    uint8_t *label_ptr[2];
 #endif
 
     data_reg = args[0];
@@ -1194,133 +1222,48 @@ static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args,
                      label_ptr, offsetof(CPUTLBEntry, addr_read));
 
     /* TLB Hit.  */
-    tcg_out_qemu_ld_direct(s, data_reg, data_reg2,
-                           tcg_target_call_iarg_regs[0], 0, opc);
-
-    /* jmp label2 */
-    tcg_out8(s, OPC_JMP_short);
-    label_ptr[2] = s->code_ptr;
-    s->code_ptr++;
-
-    /* TLB Miss.  */
-
-    /* label1: */
-    *label_ptr[0] = s->code_ptr - label_ptr[0] - 1;
-    if (TARGET_LONG_BITS > TCG_TARGET_REG_BITS) {
-        *label_ptr[1] = s->code_ptr - label_ptr[1] - 1;
-    }
-
-    /* XXX: move that code at the end of the TB */
-#if TCG_TARGET_REG_BITS == 32
-    tcg_out_pushi(s, mem_index);
-    stack_adjust = 4;
-    if (TARGET_LONG_BITS == 64) {
-        tcg_out_push(s, args[addrlo_idx + 1]);
-        stack_adjust += 4;
-    }
-    tcg_out_push(s, args[addrlo_idx]);
-    stack_adjust += 4;
-#ifdef CONFIG_TCG_PASS_AREG0
-    tcg_out_push(s, TCG_AREG0);
-    stack_adjust += 4;
-#endif
-#else
-    /* The first argument is already loaded with addrlo.  */
-    arg_idx = 1;
-    tcg_out_movi(s, TCG_TYPE_I32, tcg_target_call_iarg_regs[arg_idx],
-                 mem_index);
-#ifdef CONFIG_TCG_PASS_AREG0
-    /* XXX/FIXME: suboptimal */
-    tcg_out_mov(s, TCG_TYPE_I64, tcg_target_call_iarg_regs[3],
-                tcg_target_call_iarg_regs[2]);
-    tcg_out_mov(s, TCG_TYPE_I64, tcg_target_call_iarg_regs[2],
-                tcg_target_call_iarg_regs[1]);
-    tcg_out_mov(s, TCG_TYPE_I64, tcg_target_call_iarg_regs[1],
-                tcg_target_call_iarg_regs[0]);
-    tcg_out_mov(s, TCG_TYPE_I64, tcg_target_call_iarg_regs[0],
-                TCG_AREG0);
-#endif
-#endif
-
-    tcg_out_calli(s, (tcg_target_long)qemu_ld_helpers[s_bits]);
-
-#if TCG_TARGET_REG_BITS == 32
-    if (stack_adjust == (TCG_TARGET_REG_BITS / 8)) {
-        /* Pop and discard.  This is 2 bytes smaller than the add.  */
-        tcg_out_pop(s, TCG_REG_ECX);
-    } else if (stack_adjust != 0) {
-        tcg_out_addi(s, TCG_REG_CALL_STACK, stack_adjust);
-    }
-#endif
-
-    switch(opc) {
-    case 0 | 4:
-        tcg_out_ext8s(s, data_reg, TCG_REG_EAX, P_REXW);
-        break;
-    case 1 | 4:
-        tcg_out_ext16s(s, data_reg, TCG_REG_EAX, P_REXW);
-        break;
-    case 0:
-        tcg_out_ext8u(s, data_reg, TCG_REG_EAX);
-        break;
-    case 1:
-        tcg_out_ext16u(s, data_reg, TCG_REG_EAX);
-        break;
-    case 2:
-        tcg_out_mov(s, TCG_TYPE_I32, data_reg, TCG_REG_EAX);
-        break;
-#if TCG_TARGET_REG_BITS == 64
-    case 2 | 4:
-        tcg_out_ext32s(s, data_reg, TCG_REG_EAX);
-        break;
-#endif
-    case 3:
-        if (TCG_TARGET_REG_BITS == 64) {
-            tcg_out_mov(s, TCG_TYPE_I64, data_reg, TCG_REG_RAX);
-        } else if (data_reg == TCG_REG_EDX) {
-            /* xchg %edx, %eax */
-            tcg_out_opc(s, OPC_XCHG_ax_r32 + TCG_REG_EDX, 0, 0, 0);
-            tcg_out_mov(s, TCG_TYPE_I32, data_reg2, TCG_REG_EAX);
-        } else {
-            tcg_out_mov(s, TCG_TYPE_I32, data_reg, TCG_REG_EAX);
-            tcg_out_mov(s, TCG_TYPE_I32, data_reg2, TCG_REG_EDX);
-        }
-        break;
-    default:
-        tcg_abort();
-    }
-
-    /* label2: */
-    *label_ptr[2] = s->code_ptr - label_ptr[2] - 1;
+    tcg_out_qemu_ld_direct(s, data_reg, data_reg2, TCG_REG_L1, 0, 0, opc);
+
+    /* Record the current context of a load into ldst label */
+    add_qemu_ldst_label(s,
+                        1,
+                        opc,
+                        data_reg,
+                        data_reg2,
+                        args[addrlo_idx],
+                        args[addrlo_idx + 1],
+                        mem_index,
+                        s->code_ptr,
+                        label_ptr);
 #else
     {
         int32_t offset = GUEST_BASE;
         int base = args[addrlo_idx];
-
-        if (TCG_TARGET_REG_BITS == 64) {
-            /* ??? We assume all operations have left us with register
-               contents that are zero extended.  So far this appears to
-               be true.  If we want to enforce this, we can either do
-               an explicit zero-extension here, or (if GUEST_BASE == 0)
-               use the ADDR32 prefix.  For now, do nothing.  */
-
-            if (offset != GUEST_BASE) {
-                tcg_out_movi(s, TCG_TYPE_I64,
-                             tcg_target_call_iarg_regs[0], GUEST_BASE);
-                tgen_arithr(s, ARITH_ADD + P_REXW,
-                            tcg_target_call_iarg_regs[0], base);
-                base = tcg_target_call_iarg_regs[0];
-                offset = 0;
-            }
+        int seg = 0;
+
+        /* ??? We assume all operations have left us with register contents
+           that are zero extended.  So far this appears to be true.  If we
+           want to enforce this, we can either do an explicit zero-extension
+           here, or (if GUEST_BASE == 0, or a segment register is in use)
+           use the ADDR32 prefix.  For now, do nothing.  */
+        if (GUEST_BASE && guest_base_flags) {
+            seg = guest_base_flags;
+            offset = 0;
+        } else if (TCG_TARGET_REG_BITS == 64 && offset != GUEST_BASE) {
+            tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_L1, GUEST_BASE);
+            tgen_arithr(s, ARITH_ADD + P_REXW, TCG_REG_L1, base);
+            base = TCG_REG_L1;
+            offset = 0;
         }
 
-        tcg_out_qemu_ld_direct(s, data_reg, data_reg2, base, offset, opc);
+        tcg_out_qemu_ld_direct(s, data_reg, data_reg2, base, offset, seg, opc);
     }
 #endif
 }
 
 static void tcg_out_qemu_st_direct(TCGContext *s, int datalo, int datahi,
-                                   int base, tcg_target_long ofs, int sizeop)
+                                   int base, tcg_target_long ofs, int seg,
+                                   int sizeop)
 {
 #ifdef TARGET_WORDS_BIGENDIAN
     const int bswap = 1;
@@ -1330,12 +1273,13 @@ static void tcg_out_qemu_st_direct(TCGContext *s, int datalo, int datahi,
     /* ??? Ideally we wouldn't need a scratch register.  For user-only,
        we could perform the bswap twice to restore the original value
        instead of moving to the scratch.  But as it is, the L constraint
-       means that the second argument reg is definitely free here.  */
-    int scratch = tcg_target_call_iarg_regs[1];
+       means that TCG_REG_L0 is definitely free here.  */
+    const int scratch = TCG_REG_L0;
 
     switch (sizeop) {
     case 0:
-        tcg_out_modrm_offset(s, OPC_MOVB_EvGv + P_REXB_R, datalo, base, ofs);
+        tcg_out_modrm_offset(s, OPC_MOVB_EvGv + P_REXB_R + seg,
+                             datalo, base, ofs);
         break;
     case 1:
         if (bswap) {
@@ -1343,7 +1287,8 @@ static void tcg_out_qemu_st_direct(TCGContext *s, int datalo, int datahi,
             tcg_out_rolw_8(s, scratch);
             datalo = scratch;
         }
-        tcg_out_modrm_offset(s, OPC_MOVL_EvGv + P_DATA16, datalo, base, ofs);
+        tcg_out_modrm_offset(s, OPC_MOVL_EvGv + P_DATA16 + seg,
+                             datalo, base, ofs);
         break;
     case 2:
         if (bswap) {
@@ -1351,7 +1296,7 @@ static void tcg_out_qemu_st_direct(TCGContext *s, int datalo, int datahi,
             tcg_out_bswap32(s, scratch);
             datalo = scratch;
         }
-        tcg_out_st(s, TCG_TYPE_I32, datalo, base, ofs);
+        tcg_out_modrm_offset(s, OPC_MOVL_EvGv + seg, datalo, base, ofs);
         break;
     case 3:
         if (TCG_TARGET_REG_BITS == 64) {
@@ -1360,17 +1305,18 @@ static void tcg_out_qemu_st_direct(TCGContext *s, int datalo, int datahi,
                 tcg_out_bswap64(s, scratch);
                 datalo = scratch;
             }
-            tcg_out_st(s, TCG_TYPE_I64, datalo, base, ofs);
+            tcg_out_modrm_offset(s, OPC_MOVL_EvGv + P_REXW + seg,
+                                 datalo, base, ofs);
         } else if (bswap) {
             tcg_out_mov(s, TCG_TYPE_I32, scratch, datahi);
             tcg_out_bswap32(s, scratch);
-            tcg_out_st(s, TCG_TYPE_I32, scratch, base, ofs);
+            tcg_out_modrm_offset(s, OPC_MOVL_EvGv + seg, scratch, base, ofs);
             tcg_out_mov(s, TCG_TYPE_I32, scratch, datalo);
             tcg_out_bswap32(s, scratch);
-            tcg_out_st(s, TCG_TYPE_I32, scratch, base, ofs + 4);
+            tcg_out_modrm_offset(s, OPC_MOVL_EvGv + seg, scratch, base, ofs+4);
         } else {
-            tcg_out_st(s, TCG_TYPE_I32, datalo, base, ofs);
-            tcg_out_st(s, TCG_TYPE_I32, datahi, base, ofs + 4);
+            tcg_out_modrm_offset(s, OPC_MOVL_EvGv + seg, datalo, base, ofs);
+            tcg_out_modrm_offset(s, OPC_MOVL_EvGv + seg, datahi, base, ofs+4);
         }
         break;
     default:
@@ -1385,8 +1331,7 @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args,
     int addrlo_idx;
 #if defined(CONFIG_SOFTMMU)
     int mem_index, s_bits;
-    int stack_adjust;
-    uint8_t *label_ptr[3];
+    uint8_t *label_ptr[2];
 #endif
 
     data_reg = args[0];
@@ -1404,23 +1349,223 @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args,
                      label_ptr, offsetof(CPUTLBEntry, addr_write));
 
     /* TLB Hit.  */
-    tcg_out_qemu_st_direct(s, data_reg, data_reg2,
-                           tcg_target_call_iarg_regs[0], 0, opc);
+    tcg_out_qemu_st_direct(s, data_reg, data_reg2, TCG_REG_L1, 0, 0, opc);
+
+    /* Record the current context of a store into ldst label */
+    add_qemu_ldst_label(s,
+                        0,
+                        opc,
+                        data_reg,
+                        data_reg2,
+                        args[addrlo_idx],
+                        args[addrlo_idx + 1],
+                        mem_index,
+                        s->code_ptr,
+                        label_ptr);
+#else
+    {
+        int32_t offset = GUEST_BASE;
+        int base = args[addrlo_idx];
+        int seg = 0;
+
+        /* ??? We assume all operations have left us with register contents
+           that are zero extended.  So far this appears to be true.  If we
+           want to enforce this, we can either do an explicit zero-extension
+           here, or (if GUEST_BASE == 0, or a segment register is in use)
+           use the ADDR32 prefix.  For now, do nothing.  */
+        if (GUEST_BASE && guest_base_flags) {
+            seg = guest_base_flags;
+            offset = 0;
+        } else if (TCG_TARGET_REG_BITS == 64 && offset != GUEST_BASE) {
+            tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_L1, GUEST_BASE);
+            tgen_arithr(s, ARITH_ADD + P_REXW, TCG_REG_L1, base);
+            base = TCG_REG_L1;
+            offset = 0;
+        }
+
+        tcg_out_qemu_st_direct(s, data_reg, data_reg2, base, offset, seg, opc);
+    }
+#endif
+}
+
+#if defined(CONFIG_SOFTMMU)
+/*
+ * Record the context of a call to the out of line helper code for the slow path
+ * for a load or store, so that we can later generate the correct helper code
+ */
+static void add_qemu_ldst_label(TCGContext *s,
+                                int is_ld,
+                                int opc,
+                                int data_reg,
+                                int data_reg2,
+                                int addrlo_reg,
+                                int addrhi_reg,
+                                int mem_index,
+                                uint8_t *raddr,
+                                uint8_t **label_ptr)
+{
+    int idx;
+    TCGLabelQemuLdst *label;
+
+    if (s->nb_qemu_ldst_labels >= TCG_MAX_QEMU_LDST) {
+        tcg_abort();
+    }
+
+    idx = s->nb_qemu_ldst_labels++;
+    label = (TCGLabelQemuLdst *)&s->qemu_ldst_labels[idx];
+    label->is_ld = is_ld;
+    label->opc = opc;
+    label->datalo_reg = data_reg;
+    label->datahi_reg = data_reg2;
+    label->addrlo_reg = addrlo_reg;
+    label->addrhi_reg = addrhi_reg;
+    label->mem_index = mem_index;
+    label->raddr = raddr;
+    label->label_ptr[0] = label_ptr[0];
+    if (TARGET_LONG_BITS > TCG_TARGET_REG_BITS) {
+        label->label_ptr[1] = label_ptr[1];
+    }
+}
+
+/*
+ * Generate code for the slow path for a load at the end of block
+ */
+static void tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *label)
+{
+    int s_bits;
+    int opc = label->opc;
+    int mem_index = label->mem_index;
+#if TCG_TARGET_REG_BITS == 32
+    int stack_adjust;
+    int addrlo_reg = label->addrlo_reg;
+    int addrhi_reg = label->addrhi_reg;
+#endif
+    int data_reg = label->datalo_reg;
+    int data_reg2 = label->datahi_reg;
+    uint8_t *raddr = label->raddr;
+    uint8_t **label_ptr = &label->label_ptr[0];
+
+    s_bits = opc & 3;
+
+    /* resolve label address */
+    *(uint32_t *)label_ptr[0] = (uint32_t)(s->code_ptr - label_ptr[0] - 4);
+    if (TARGET_LONG_BITS > TCG_TARGET_REG_BITS) {
+        *(uint32_t *)label_ptr[1] = (uint32_t)(s->code_ptr - label_ptr[1] - 4);
+    }
+
+#if TCG_TARGET_REG_BITS == 32
+    tcg_out_pushi(s, mem_index);
+    stack_adjust = 4;
+    if (TARGET_LONG_BITS == 64) {
+        tcg_out_push(s, addrhi_reg);
+        stack_adjust += 4;
+    }
+    tcg_out_push(s, addrlo_reg);
+    stack_adjust += 4;
+    tcg_out_push(s, TCG_AREG0);
+    stack_adjust += 4;
+#else
+    tcg_out_mov(s, TCG_TYPE_I64, tcg_target_call_iarg_regs[0], TCG_AREG0);
+    /* The second argument is already loaded with addrlo.  */
+    tcg_out_movi(s, TCG_TYPE_I32, tcg_target_call_iarg_regs[2], mem_index);
+#endif
+
+    /* Code generation of qemu_ld/st's slow path calling MMU helper
+
+       PRE_PROC ...
+       call MMU helper
+       jmp POST_PROC (2b) : short forward jump <- GETRA()
+       jmp next_code (5b) : dummy long backward jump which is never executed
+       POST_PROC ... : do post-processing <- GETRA() + 7
+       jmp next_code : jump to the code corresponding to next IR of qemu_ld/st
+    */
 
-    /* jmp label2 */
+    tcg_out_calli(s, (tcg_target_long)qemu_ld_helpers[s_bits]);
+
+    /* Jump to post-processing code */
     tcg_out8(s, OPC_JMP_short);
-    label_ptr[2] = s->code_ptr;
-    s->code_ptr++;
+    tcg_out8(s, 5);
+    /* Dummy backward jump having information of fast path'pc for MMU helpers */
+    tcg_out8(s, OPC_JMP_long);
+    *(int32_t *)s->code_ptr = (int32_t)(raddr - s->code_ptr - 4);
+    s->code_ptr += 4;
+
+#if TCG_TARGET_REG_BITS == 32
+    if (stack_adjust == (TCG_TARGET_REG_BITS / 8)) {
+        /* Pop and discard.  This is 2 bytes smaller than the add.  */
+        tcg_out_pop(s, TCG_REG_ECX);
+    } else if (stack_adjust != 0) {
+        tcg_out_addi(s, TCG_REG_CALL_STACK, stack_adjust);
+    }
+#endif
+
+    switch(opc) {
+    case 0 | 4:
+        tcg_out_ext8s(s, data_reg, TCG_REG_EAX, P_REXW);
+        break;
+    case 1 | 4:
+        tcg_out_ext16s(s, data_reg, TCG_REG_EAX, P_REXW);
+        break;
+    case 0:
+        tcg_out_ext8u(s, data_reg, TCG_REG_EAX);
+        break;
+    case 1:
+        tcg_out_ext16u(s, data_reg, TCG_REG_EAX);
+        break;
+    case 2:
+        tcg_out_mov(s, TCG_TYPE_I32, data_reg, TCG_REG_EAX);
+        break;
+#if TCG_TARGET_REG_BITS == 64
+    case 2 | 4:
+        tcg_out_ext32s(s, data_reg, TCG_REG_EAX);
+        break;
+#endif
+    case 3:
+        if (TCG_TARGET_REG_BITS == 64) {
+            tcg_out_mov(s, TCG_TYPE_I64, data_reg, TCG_REG_RAX);
+        } else if (data_reg == TCG_REG_EDX) {
+            /* xchg %edx, %eax */
+            tcg_out_opc(s, OPC_XCHG_ax_r32 + TCG_REG_EDX, 0, 0, 0);
+            tcg_out_mov(s, TCG_TYPE_I32, data_reg2, TCG_REG_EAX);
+        } else {
+            tcg_out_mov(s, TCG_TYPE_I32, data_reg, TCG_REG_EAX);
+            tcg_out_mov(s, TCG_TYPE_I32, data_reg2, TCG_REG_EDX);
+        }
+        break;
+    default:
+        tcg_abort();
+    }
 
-    /* TLB Miss.  */
+    /* Jump to the code corresponding to next IR of qemu_st */
+    tcg_out_jmp(s, (tcg_target_long)raddr);
+}
 
-    /* label1: */
-    *label_ptr[0] = s->code_ptr - label_ptr[0] - 1;
+/*
+ * Generate code for the slow path for a store at the end of block
+ */
+static void tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *label)
+{
+    int s_bits;
+    int stack_adjust;
+    int opc = label->opc;
+    int mem_index = label->mem_index;
+    int data_reg = label->datalo_reg;
+#if TCG_TARGET_REG_BITS == 32
+    int data_reg2 = label->datahi_reg;
+    int addrlo_reg = label->addrlo_reg;
+    int addrhi_reg = label->addrhi_reg;
+#endif
+    uint8_t *raddr = label->raddr;
+    uint8_t **label_ptr = &label->label_ptr[0];
+
+    s_bits = opc & 3;
+
+    /* resolve label address */
+    *(uint32_t *)label_ptr[0] = (uint32_t)(s->code_ptr - label_ptr[0] - 4);
     if (TARGET_LONG_BITS > TCG_TARGET_REG_BITS) {
-        *label_ptr[1] = s->code_ptr - label_ptr[1] - 1;
+        *(uint32_t *)label_ptr[1] = (uint32_t)(s->code_ptr - label_ptr[1] - 4);
     }
 
-    /* XXX: move that code at the end of the TB */
 #if TCG_TARGET_REG_BITS == 32
     tcg_out_pushi(s, mem_index);
     stack_adjust = 4;
@@ -1431,35 +1576,42 @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args,
     tcg_out_push(s, data_reg);
     stack_adjust += 4;
     if (TARGET_LONG_BITS == 64) {
-        tcg_out_push(s, args[addrlo_idx + 1]);
+        tcg_out_push(s, addrhi_reg);
         stack_adjust += 4;
     }
-    tcg_out_push(s, args[addrlo_idx]);
+    tcg_out_push(s, addrlo_reg);
     stack_adjust += 4;
-#ifdef CONFIG_TCG_PASS_AREG0
     tcg_out_push(s, TCG_AREG0);
     stack_adjust += 4;
-#endif
 #else
+    tcg_out_mov(s, TCG_TYPE_I64, tcg_target_call_iarg_regs[0], TCG_AREG0);
+    /* The second argument is already loaded with addrlo.  */
     tcg_out_mov(s, (opc == 3 ? TCG_TYPE_I64 : TCG_TYPE_I32),
-                tcg_target_call_iarg_regs[1], data_reg);
-    tcg_out_movi(s, TCG_TYPE_I32, tcg_target_call_iarg_regs[2], mem_index);
+                tcg_target_call_iarg_regs[2], data_reg);
+    tcg_out_movi(s, TCG_TYPE_I32, tcg_target_call_iarg_regs[3], mem_index);
     stack_adjust = 0;
-#ifdef CONFIG_TCG_PASS_AREG0
-    /* XXX/FIXME: suboptimal */
-    tcg_out_mov(s, TCG_TYPE_I64, tcg_target_call_iarg_regs[3],
-                tcg_target_call_iarg_regs[2]);
-    tcg_out_mov(s, TCG_TYPE_I64, tcg_target_call_iarg_regs[2],
-                tcg_target_call_iarg_regs[1]);
-    tcg_out_mov(s, TCG_TYPE_I64, tcg_target_call_iarg_regs[1],
-                tcg_target_call_iarg_regs[0]);
-    tcg_out_mov(s, TCG_TYPE_I64, tcg_target_call_iarg_regs[0],
-                TCG_AREG0);
-#endif
 #endif
 
+    /* Code generation of qemu_ld/st's slow path calling MMU helper
+
+       PRE_PROC ...
+       call MMU helper
+       jmp POST_PROC (2b) : short forward jump <- GETRA()
+       jmp next_code (5b) : dummy long backward jump which is never executed
+       POST_PROC ... : do post-processing <- GETRA() + 7
+       jmp next_code : jump to the code corresponding to next IR of qemu_ld/st
+    */
+
     tcg_out_calli(s, (tcg_target_long)qemu_st_helpers[s_bits]);
 
+    /* Jump to post-processing code */
+    tcg_out8(s, OPC_JMP_short);
+    tcg_out8(s, 5);
+    /* Dummy backward jump having information of fast path'pc for MMU helpers */
+    tcg_out8(s, OPC_JMP_long);
+    *(int32_t *)s->code_ptr = (int32_t)(raddr - s->code_ptr - 4);
+    s->code_ptr += 4;
+
     if (stack_adjust == (TCG_TARGET_REG_BITS / 8)) {
         /* Pop and discard.  This is 2 bytes smaller than the add.  */
         tcg_out_pop(s, TCG_REG_ECX);
@@ -1467,34 +1619,29 @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args,
         tcg_out_addi(s, TCG_REG_CALL_STACK, stack_adjust);
     }
 
-    /* label2: */
-    *label_ptr[2] = s->code_ptr - label_ptr[2] - 1;
-#else
-    {
-        int32_t offset = GUEST_BASE;
-        int base = args[addrlo_idx];
+    /* Jump to the code corresponding to next IR of qemu_st */
+    tcg_out_jmp(s, (tcg_target_long)raddr);
+}
 
-        if (TCG_TARGET_REG_BITS == 64) {
-            /* ??? We assume all operations have left us with register
-               contents that are zero extended.  So far this appears to
-               be true.  If we want to enforce this, we can either do
-               an explicit zero-extension here, or (if GUEST_BASE == 0)
-               use the ADDR32 prefix.  For now, do nothing.  */
-
-            if (offset != GUEST_BASE) {
-                tcg_out_movi(s, TCG_TYPE_I64,
-                             tcg_target_call_iarg_regs[0], GUEST_BASE);
-                tgen_arithr(s, ARITH_ADD + P_REXW,
-                            tcg_target_call_iarg_regs[0], base);
-                base = tcg_target_call_iarg_regs[0];
-                offset = 0;
-            }
+/*
+ * Generate TB finalization at the end of block
+ */
+void tcg_out_tb_finalize(TCGContext *s)
+{
+    int i;
+    TCGLabelQemuLdst *label;
+
+    /* qemu_ld/st slow paths */
+    for (i = 0; i < s->nb_qemu_ldst_labels; i++) {
+        label = (TCGLabelQemuLdst *)&s->qemu_ldst_labels[i];
+        if (label->is_ld) {
+            tcg_out_qemu_ld_slow_path(s, label);
+        } else {
+            tcg_out_qemu_st_slow_path(s, label);
         }
-
-        tcg_out_qemu_st_direct(s, data_reg, data_reg2, base, offset, opc);
     }
-#endif
 }
+#endif  /* CONFIG_SOFTMMU */
 
 static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
                               const TCGArg *args, const int *const_args)
@@ -1537,14 +1684,6 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
             tcg_out_modrm(s, OPC_GRP5, EXT5_CALLN_Ev, args[0]);
         }
         break;
-    case INDEX_op_jmp:
-        if (const_args[0]) {
-            tcg_out_jmp(s, args[0]);
-        } else {
-            /* jmp *reg */
-            tcg_out_modrm(s, OPC_GRP5, EXT5_JMPN_Ev, args[0]);
-        }
-        break;
     case INDEX_op_br:
         tcg_out_jxx(s, JCC_JMP, args[0], 0);
         break;
@@ -1573,18 +1712,35 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
         break;
 
     OP_32_64(st8):
-        tcg_out_modrm_offset(s, OPC_MOVB_EvGv | P_REXB_R,
-                             args[0], args[1], args[2]);
+        if (const_args[0]) {
+            tcg_out_modrm_offset(s, OPC_MOVB_EvIz,
+                                 0, args[1], args[2]);
+            tcg_out8(s, args[0]);
+        } else {
+            tcg_out_modrm_offset(s, OPC_MOVB_EvGv | P_REXB_R,
+                                 args[0], args[1], args[2]);
+        }
         break;
     OP_32_64(st16):
-        tcg_out_modrm_offset(s, OPC_MOVL_EvGv | P_DATA16,
-                             args[0], args[1], args[2]);
+        if (const_args[0]) {
+            tcg_out_modrm_offset(s, OPC_MOVL_EvIz | P_DATA16,
+                                 0, args[1], args[2]);
+            tcg_out16(s, args[0]);
+        } else {
+            tcg_out_modrm_offset(s, OPC_MOVL_EvGv | P_DATA16,
+                                 args[0], args[1], args[2]);
+        }
         break;
 #if TCG_TARGET_REG_BITS == 64
     case INDEX_op_st32_i64:
 #endif
     case INDEX_op_st_i32:
-        tcg_out_st(s, TCG_TYPE_I32, args[0], args[1], args[2]);
+        if (const_args[0]) {
+            tcg_out_modrm_offset(s, OPC_MOVL_EvIz, 0, args[1], args[2]);
+            tcg_out32(s, args[0]);
+        } else {
+            tcg_out_st(s, TCG_TYPE_I32, args[0], args[1], args[2]);
+        }
         break;
 
     OP_32_64(add):
@@ -1680,6 +1836,10 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
         tcg_out_setcond32(s, args[3], args[0], args[1],
                           args[2], const_args[2]);
         break;
+    case INDEX_op_movcond_i32:
+        tcg_out_movcond32(s, args[5], args[0], args[1],
+                          args[2], const_args[2], args[3]);
+        break;
 
     OP_32_64(bswap16):
         tcg_out_rolw_8(s, args[0]);
@@ -1788,7 +1948,13 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
         tcg_out_ld(s, TCG_TYPE_I64, args[0], args[1], args[2]);
         break;
     case INDEX_op_st_i64:
-        tcg_out_st(s, TCG_TYPE_I64, args[0], args[1], args[2]);
+        if (const_args[0]) {
+            tcg_out_modrm_offset(s, OPC_MOVL_EvIz | P_REXW,
+                                 0, args[1], args[2]);
+            tcg_out32(s, args[0]);
+        } else {
+            tcg_out_st(s, TCG_TYPE_I64, args[0], args[1], args[2]);
+        }
         break;
     case INDEX_op_qemu_ld32s:
         tcg_out_qemu_ld(s, args, 2 | 4);
@@ -1802,6 +1968,10 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
         tcg_out_setcond64(s, args[3], args[0], args[1],
                           args[2], const_args[2]);
         break;
+    case INDEX_op_movcond_i64:
+        tcg_out_movcond64(s, args[5], args[0], args[1],
+                          args[2], const_args[2], args[3]);
+        break;
 
     case INDEX_op_bswap64_i64:
         tcg_out_bswap64(s, args[0]);
@@ -1841,7 +2011,6 @@ static const TCGTargetOpDef x86_op_defs[] = {
     { INDEX_op_exit_tb, { } },
     { INDEX_op_goto_tb, { } },
     { INDEX_op_call, { "ri" } },
-    { INDEX_op_jmp, { "ri" } },
     { INDEX_op_br, { } },
     { INDEX_op_mov_i32, { "r", "r" } },
     { INDEX_op_movi_i32, { "r" } },
@@ -1850,9 +2019,9 @@ static const TCGTargetOpDef x86_op_defs[] = {
     { INDEX_op_ld16u_i32, { "r", "r" } },
     { INDEX_op_ld16s_i32, { "r", "r" } },
     { INDEX_op_ld_i32, { "r", "r" } },
-    { INDEX_op_st8_i32, { "q", "r" } },
-    { INDEX_op_st16_i32, { "r", "r" } },
-    { INDEX_op_st_i32, { "r", "r" } },
+    { INDEX_op_st8_i32, { "qi", "r" } },
+    { INDEX_op_st16_i32, { "ri", "r" } },
+    { INDEX_op_st_i32, { "ri", "r" } },
 
     { INDEX_op_add_i32, { "r", "r", "ri" } },
     { INDEX_op_sub_i32, { "r", "0", "ri" } },
@@ -1886,6 +2055,9 @@ static const TCGTargetOpDef x86_op_defs[] = {
     { INDEX_op_setcond_i32, { "q", "r", "ri" } },
 
     { INDEX_op_deposit_i32, { "Q", "0", "Q" } },
+#if TCG_TARGET_HAS_movcond_i32
+    { INDEX_op_movcond_i32, { "r", "r", "ri", "r", "0" } },
+#endif
 
 #if TCG_TARGET_REG_BITS == 32
     { INDEX_op_mulu2_i32, { "a", "d", "a", "r" } },
@@ -1903,10 +2075,10 @@ static const TCGTargetOpDef x86_op_defs[] = {
     { INDEX_op_ld32u_i64, { "r", "r" } },
     { INDEX_op_ld32s_i64, { "r", "r" } },
     { INDEX_op_ld_i64, { "r", "r" } },
-    { INDEX_op_st8_i64, { "r", "r" } },
-    { INDEX_op_st16_i64, { "r", "r" } },
-    { INDEX_op_st32_i64, { "r", "r" } },
-    { INDEX_op_st_i64, { "r", "r" } },
+    { INDEX_op_st8_i64, { "ri", "r" } },
+    { INDEX_op_st16_i64, { "ri", "r" } },
+    { INDEX_op_st32_i64, { "ri", "r" } },
+    { INDEX_op_st_i64, { "re", "r" } },
 
     { INDEX_op_add_i64, { "r", "0", "re" } },
     { INDEX_op_mul_i64, { "r", "0", "re" } },
@@ -1940,6 +2112,7 @@ static const TCGTargetOpDef x86_op_defs[] = {
     { INDEX_op_ext32u_i64, { "r", "r" } },
 
     { INDEX_op_deposit_i64, { "Q", "0", "Q" } },
+    { INDEX_op_movcond_i64, { "r", "r", "re", "r", "0" } },
 #endif
 
 #if TCG_TARGET_REG_BITS == 64
@@ -2038,15 +2211,17 @@ static void tcg_target_qemu_prologue(TCGContext *s)
 #if TCG_TARGET_REG_BITS == 32
     tcg_out_ld(s, TCG_TYPE_PTR, TCG_AREG0, TCG_REG_ESP,
                (ARRAY_SIZE(tcg_target_callee_save_regs) + 1) * 4);
-    tcg_out_ld(s, TCG_TYPE_PTR, tcg_target_call_iarg_regs[1], TCG_REG_ESP,
-               (ARRAY_SIZE(tcg_target_callee_save_regs) + 2) * 4);
+    tcg_out_addi(s, TCG_REG_ESP, -stack_addend);
+    /* jmp *tb.  */
+    tcg_out_modrm_offset(s, OPC_GRP5, EXT5_JMPN_Ev, TCG_REG_ESP,
+		         (ARRAY_SIZE(tcg_target_callee_save_regs) + 2) * 4
+			 + stack_addend);
 #else
     tcg_out_mov(s, TCG_TYPE_PTR, TCG_AREG0, tcg_target_call_iarg_regs[0]);
-#endif
     tcg_out_addi(s, TCG_REG_ESP, -stack_addend);
-
     /* jmp *tb.  */
     tcg_out_modrm(s, OPC_GRP5, EXT5_JMPN_Ev, tcg_target_call_iarg_regs[1]);
+#endif
 
     /* TB epilogue */
     tb_ret_addr = s->code_ptr;
@@ -2057,6 +2232,13 @@ static void tcg_target_qemu_prologue(TCGContext *s)
         tcg_out_pop(s, tcg_target_callee_save_regs[i]);
     }
     tcg_out_opc(s, OPC_RET, 0, 0, 0);
+
+#if !defined(CONFIG_SOFTMMU)
+    /* Try to set up a segment register to point to GUEST_BASE.  */
+    if (GUEST_BASE) {
+        setup_guest_base_seg();
+    }
+#endif
 }
 
 static void tcg_target_init(TCGContext *s)
diff --git a/tcg/i386/tcg-target.h b/tcg/i386/tcg-target.h
index c3cfe05f6..dbc675641 100644
--- a/tcg/i386/tcg-target.h
+++ b/tcg/i386/tcg-target.h
@@ -67,7 +67,11 @@ typedef enum {
 /* used for function call generation */
 #define TCG_REG_CALL_STACK TCG_REG_ESP 
 #define TCG_TARGET_STACK_ALIGN 16
+#if defined(_WIN64)
+#define TCG_TARGET_CALL_STACK_OFFSET 32
+#else
 #define TCG_TARGET_CALL_STACK_OFFSET 0
+#endif
 
 /* optional instructions */
 #define TCG_TARGET_HAS_div2_i32         1
@@ -86,6 +90,12 @@ typedef enum {
 #define TCG_TARGET_HAS_nand_i32         0
 #define TCG_TARGET_HAS_nor_i32          0
 #define TCG_TARGET_HAS_deposit_i32      1
+#if defined(__x86_64__) || defined(__i686__)
+/* Use cmov only if the compiler is already doing so.  */
+#define TCG_TARGET_HAS_movcond_i32      1
+#else
+#define TCG_TARGET_HAS_movcond_i32      0
+#endif
 
 #if TCG_TARGET_REG_BITS == 64
 #define TCG_TARGET_HAS_div2_i64         1
@@ -107,6 +117,7 @@ typedef enum {
 #define TCG_TARGET_HAS_nand_i64         0
 #define TCG_TARGET_HAS_nor_i64          0
 #define TCG_TARGET_HAS_deposit_i64      1
+#define TCG_TARGET_HAS_movcond_i64      1
 #endif
 
 #define TCG_TARGET_deposit_i32_valid(ofs, len) \
@@ -114,9 +125,6 @@ typedef enum {
      ((ofs) == 0 && (len) == 16))
 #define TCG_TARGET_deposit_i64_valid    TCG_TARGET_deposit_i32_valid
 
-#define TCG_TARGET_HAS_GUEST_BASE
-
-/* Note: must be synced with dyngen-exec.h */
 #if TCG_TARGET_REG_BITS == 64
 # define TCG_AREG0 TCG_REG_R14
 #else
diff --git a/tcg/ia64/tcg-target.c b/tcg/ia64/tcg-target.c
index dc588dbea..06570bea3 100644
--- a/tcg/ia64/tcg-target.c
+++ b/tcg/ia64/tcg-target.c
@@ -176,12 +176,6 @@ static const int tcg_target_call_oarg_regs[] = {
     TCG_REG_R8
 };
 
-/* maximum number of register used for input function arguments */
-static inline int tcg_target_get_call_iarg_regs_count(int flags)
-{
-    return 8;
-}
-
 /*
  * opcode formation
  */
@@ -236,6 +230,8 @@ enum {
     OPC_CMP4_LT_A6            = 0x18400000000ull,
     OPC_CMP4_LTU_A6           = 0x1a400000000ull,
     OPC_CMP4_EQ_A6            = 0x1c400000000ull,
+    OPC_DEP_I14               = 0x0ae00000000ull,
+    OPC_DEP_I15               = 0x08000000000ull,
     OPC_DEP_Z_I12             = 0x0a600000000ull,
     OPC_EXTR_I11              = 0x0a400002000ull,
     OPC_EXTR_U_I11            = 0x0a400000000ull,
@@ -507,6 +503,30 @@ static inline uint64_t tcg_opc_i12(int qp, uint64_t opc, int r1,
            | (qp & 0x3f);
 }
 
+static inline uint64_t tcg_opc_i14(int qp, uint64_t opc, int r1, uint64_t imm,
+                                   int r3, uint64_t pos, uint64_t len)
+{
+    return opc
+           | ((imm & 0x01) << 36)
+           | ((len & 0x3f) << 27)
+           | ((r3 & 0x7f) << 20)
+           | ((pos & 0x3f) << 14)
+           | ((r1 & 0x7f) << 6)
+           | (qp & 0x3f);
+}
+
+static inline uint64_t tcg_opc_i15(int qp, uint64_t opc, int r1, int r2,
+                                   int r3, uint64_t pos, uint64_t len)
+{
+    return opc
+           | ((pos & 0x3f) << 31)
+           | ((len & 0x0f) << 27)
+           | ((r3 & 0x7f) << 20)
+           | ((r2 & 0x7f) << 13)
+           | ((r1 & 0x7f) << 6)
+           | (qp & 0x3f);
+}
+
 static inline uint64_t tcg_opc_i18(int qp, uint64_t opc, uint64_t imm)
 {
     return opc
@@ -1318,6 +1338,37 @@ static inline void tcg_out_bswap64(TCGContext *s, TCGArg ret, TCGArg arg)
                    tcg_opc_i3 (TCG_REG_P0, OPC_MUX1_I3, ret, arg, 0xb));
 }
 
+static inline void tcg_out_deposit(TCGContext *s, TCGArg ret, TCGArg a1,
+                                   TCGArg a2, int const_a2, int pos, int len)
+{
+    uint64_t i1 = 0, i2 = 0;
+    int cpos = 63 - pos, lm1 = len - 1;
+
+    if (const_a2) {
+        /* Truncate the value of a constant a2 to the width of the field.  */
+        int mask = (1u << len) - 1;
+        a2 &= mask;
+
+        if (a2 == 0 || a2 == mask) {
+            /* 1-bit signed constant inserted into register.  */
+            i2 = tcg_opc_i14(TCG_REG_P0, OPC_DEP_I14, ret, a2, a1, cpos, lm1);
+        } else {
+            /* Otherwise, load any constant into a temporary.  Do this into
+               the first I slot to help out with cross-unit delays.  */
+            i1 = tcg_opc_a5(TCG_REG_P0, OPC_ADDL_A5,
+                            TCG_REG_R2, a2, TCG_REG_R0);
+            a2 = TCG_REG_R2;
+        }
+    }
+    if (i2 == 0) {
+        i2 = tcg_opc_i15(TCG_REG_P0, OPC_DEP_I15, ret, a2, a1, cpos, lm1);
+    }
+    tcg_out_bundle(s, (i1 ? mII : miI),
+                   tcg_opc_m48(TCG_REG_P0, OPC_NOP_M48, 0),
+                   i1 ? i1 : tcg_opc_i18(TCG_REG_P0, OPC_NOP_I18, 0),
+                   i2);
+}
+
 static inline uint64_t tcg_opc_cmp_a(int qp, TCGCond cond, TCGArg arg1,
                                      TCGArg arg2, int cmp4)
 {
@@ -1410,21 +1461,47 @@ static inline void tcg_out_setcond(TCGContext *s, TCGCond cond, TCGArg ret,
                    tcg_opc_a5(TCG_REG_P7, OPC_ADDL_A5, ret, 0, TCG_REG_R0));
 }
 
+static inline void tcg_out_movcond(TCGContext *s, TCGCond cond, TCGArg ret,
+                                   TCGArg c1, TCGArg c2,
+                                   TCGArg v1, int const_v1,
+                                   TCGArg v2, int const_v2, int cmp4)
+{
+    uint64_t opc1, opc2;
+
+    if (const_v1) {
+        opc1 = tcg_opc_a5(TCG_REG_P6, OPC_ADDL_A5, ret, v1, TCG_REG_R0);
+    } else if (ret == v1) {
+        opc1 = tcg_opc_m48(TCG_REG_P0, OPC_NOP_M48, 0);
+    } else {
+        opc1 = tcg_opc_a4(TCG_REG_P6, OPC_ADDS_A4, ret, 0, v1);
+    }
+    if (const_v2) {
+        opc2 = tcg_opc_a5(TCG_REG_P7, OPC_ADDL_A5, ret, v2, TCG_REG_R0);
+    } else if (ret == v2) {
+        opc2 = tcg_opc_m48(TCG_REG_P0, OPC_NOP_M48, 0);
+    } else {
+        opc2 = tcg_opc_a4(TCG_REG_P7, OPC_ADDS_A4, ret, 0, v2);
+    }
+
+    tcg_out_bundle(s, MmI,
+                   tcg_opc_cmp_a(TCG_REG_P0, cond, c1, c2, cmp4),
+                   opc1,
+                   opc2);
+}
+
 #if defined(CONFIG_SOFTMMU)
 
 #include "../../softmmu_defs.h"
 
 /* Load and compare a TLB entry, and return the result in (p6, p7).
    R2 is loaded with the address of the addend TLB entry.
-   R56 is loaded with the address, zero extented on 32-bit targets. */
+   R57 is loaded with the address, zero extented on 32-bit targets. */
 static inline void tcg_out_qemu_tlb(TCGContext *s, TCGArg addr_reg,
                                     int s_bits, uint64_t offset_rw,
                                     uint64_t offset_addend)
 {
     tcg_out_bundle(s, mII,
-                   tcg_opc_a5 (TCG_REG_P0, OPC_ADDL_A5, TCG_REG_R3,
-                               TARGET_PAGE_MASK | ((1 << s_bits) - 1),
-                               TCG_REG_R0),
+                   tcg_opc_m48(TCG_REG_P0, OPC_NOP_M48, 0),
                    tcg_opc_i11(TCG_REG_P0, OPC_EXTR_U_I11, TCG_REG_R2,
                                addr_reg, TARGET_PAGE_BITS, CPU_TLB_BITS - 1),
                    tcg_opc_i12(TCG_REG_P0, OPC_DEP_Z_I12, TCG_REG_R2,
@@ -1434,9 +1511,9 @@ static inline void tcg_out_qemu_tlb(TCGContext *s, TCGArg addr_reg,
                    tcg_opc_a5 (TCG_REG_P0, OPC_ADDL_A5, TCG_REG_R2,
                                offset_rw, TCG_REG_R2),
 #if TARGET_LONG_BITS == 32
-                   tcg_opc_i29(TCG_REG_P0, OPC_ZXT4_I29, TCG_REG_R56, addr_reg),
+                   tcg_opc_i29(TCG_REG_P0, OPC_ZXT4_I29, TCG_REG_R57, addr_reg),
 #else
-                   tcg_opc_a4(TCG_REG_P0, OPC_ADDS_A4, TCG_REG_R56,
+                   tcg_opc_a4(TCG_REG_P0, OPC_ADDS_A4, TCG_REG_R57,
                               0, addr_reg),
 #endif
                    tcg_opc_a1 (TCG_REG_P0, OPC_ADD_A1, TCG_REG_R2,
@@ -1444,15 +1521,15 @@ static inline void tcg_out_qemu_tlb(TCGContext *s, TCGArg addr_reg,
     tcg_out_bundle(s, mII,
                    tcg_opc_m3 (TCG_REG_P0,
                                (TARGET_LONG_BITS == 32
-                                ? OPC_LD4_M3 : OPC_LD8_M3), TCG_REG_R57,
+                                ? OPC_LD4_M3 : OPC_LD8_M3), TCG_REG_R56,
                                TCG_REG_R2, offset_addend - offset_rw),
-                   tcg_opc_a1 (TCG_REG_P0, OPC_AND_A1, TCG_REG_R3,
-                               TCG_REG_R3, TCG_REG_R56),
+                   tcg_opc_i14(TCG_REG_P0, OPC_DEP_I14, TCG_REG_R3, 0,
+                               TCG_REG_R57, 63 - s_bits,
+                               TARGET_PAGE_BITS - s_bits - 1),
                    tcg_opc_a6 (TCG_REG_P0, OPC_CMP_EQ_A6, TCG_REG_P6,
-                               TCG_REG_P7, TCG_REG_R3, TCG_REG_R57));
+                               TCG_REG_P7, TCG_REG_R3, TCG_REG_R56));
 }
 
-#ifdef CONFIG_TCG_PASS_AREG0
 /* helper signature: helper_ld_mmu(CPUState *env, target_ulong addr,
    int mmu_idx) */
 static const void * const qemu_ld_helpers[4] = {
@@ -1461,16 +1538,6 @@ static const void * const qemu_ld_helpers[4] = {
     helper_ldl_mmu,
     helper_ldq_mmu,
 };
-#else
-/* legacy helper signature: __ld_mmu(target_ulong addr, int
-   mmu_idx) */
-static void *qemu_ld_helpers[4] = {
-    __ldb_mmu,
-    __ldw_mmu,
-    __ldl_mmu,
-    __ldq_mmu,
-};
-#endif
 
 static inline void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, int opc)
 {
@@ -1497,8 +1564,8 @@ static inline void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, int opc)
 
     /* P6 is the fast path, and P7 the slow path */
     tcg_out_bundle(s, mLX,
-                   tcg_opc_a5 (TCG_REG_P7, OPC_ADDL_A5, TCG_REG_R57,
-                               mem_index, TCG_REG_R0),
+                   tcg_opc_a4 (TCG_REG_P7, OPC_ADDS_A4,
+                               TCG_REG_R56, 0, TCG_AREG0),
                    tcg_opc_l2 ((tcg_target_long) qemu_ld_helpers[s_bits]),
                    tcg_opc_x2 (TCG_REG_P7, OPC_MOVL_X2, TCG_REG_R2,
                                (tcg_target_long) qemu_ld_helpers[s_bits]));
@@ -1506,7 +1573,7 @@ static inline void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, int opc)
                    tcg_opc_m3 (TCG_REG_P0, OPC_LD8_M3, TCG_REG_R3,
                                TCG_REG_R2, 8),
                    tcg_opc_a1 (TCG_REG_P6, OPC_ADD_A1, TCG_REG_R3,
-                               TCG_REG_R3, TCG_REG_R56),
+                               TCG_REG_R3, TCG_REG_R57),
                    tcg_opc_i21(TCG_REG_P7, OPC_MOV_I21, TCG_REG_B6,
                                TCG_REG_R3, 0));
     if (bswap && s_bits == 1) {
@@ -1530,25 +1597,17 @@ static inline void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, int opc)
                        tcg_opc_m1 (TCG_REG_P7, OPC_LD8_M1, TCG_REG_R1, TCG_REG_R2),
                        tcg_opc_i18(TCG_REG_P0, OPC_NOP_I18, 0));
     }
-#ifdef CONFIG_TCG_PASS_AREG0
-    /* XXX/FIXME: suboptimal */
-    tcg_out_bundle(s, mII,
-                   tcg_opc_a5 (TCG_REG_P7, OPC_ADDL_A5, TCG_REG_R58,
-                               mem_index, TCG_REG_R0),
-                   tcg_opc_a4 (TCG_REG_P7, OPC_ADDS_A4,
-                               TCG_REG_R57, 0, TCG_REG_R56),
-                   tcg_opc_a4 (TCG_REG_P7, OPC_ADDS_A4,
-                               TCG_REG_R56, 0, TCG_AREG0));
-#endif
     if (!bswap || s_bits == 0) {
         tcg_out_bundle(s, miB,
-                       tcg_opc_m48(TCG_REG_P0, OPC_NOP_M48, 0),
+                       tcg_opc_a5 (TCG_REG_P7, OPC_ADDL_A5, TCG_REG_R58,
+                                   mem_index, TCG_REG_R0),
                        tcg_opc_i18(TCG_REG_P0, OPC_NOP_I18, 0),
                        tcg_opc_b5 (TCG_REG_P7, OPC_BR_CALL_SPTK_MANY_B5,
                                    TCG_REG_B0, TCG_REG_B6));
     } else {
         tcg_out_bundle(s, miB,
-                       tcg_opc_m48(TCG_REG_P0, OPC_NOP_M48, 0),
+                       tcg_opc_a5 (TCG_REG_P7, OPC_ADDL_A5, TCG_REG_R58,
+                                   mem_index, TCG_REG_R0),
                        tcg_opc_i3 (TCG_REG_P6, OPC_MUX1_I3,
                                    TCG_REG_R8, TCG_REG_R8, 0xb),
                        tcg_opc_b5 (TCG_REG_P7, OPC_BR_CALL_SPTK_MANY_B5,
@@ -1570,7 +1629,6 @@ static inline void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, int opc)
     }
 }
 
-#ifdef CONFIG_TCG_PASS_AREG0
 /* helper signature: helper_st_mmu(CPUState *env, target_ulong addr,
    uintxx_t val, int mmu_idx) */
 static const void * const qemu_st_helpers[4] = {
@@ -1579,16 +1637,6 @@ static const void * const qemu_st_helpers[4] = {
     helper_stl_mmu,
     helper_stq_mmu,
 };
-#else
-/* legacy helper signature: __st_mmu(target_ulong addr, uintxx_t val,
-   int mmu_idx) */
-static void *qemu_st_helpers[4] = {
-    __stb_mmu,
-    __stw_mmu,
-    __stl_mmu,
-    __stq_mmu,
-};
-#endif
 
 static inline void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, int opc)
 {
@@ -1611,8 +1659,8 @@ static inline void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, int opc)
 
     /* P6 is the fast path, and P7 the slow path */
     tcg_out_bundle(s, mLX,
-                   tcg_opc_a4(TCG_REG_P7, OPC_ADDS_A4, TCG_REG_R57,
-                              0, data_reg),
+                   tcg_opc_a4 (TCG_REG_P7, OPC_ADDS_A4,
+                               TCG_REG_R56, 0, TCG_AREG0),
                    tcg_opc_l2 ((tcg_target_long) qemu_st_helpers[opc]),
                    tcg_opc_x2 (TCG_REG_P7, OPC_MOVL_X2, TCG_REG_R2,
                                (tcg_target_long) qemu_st_helpers[opc]));
@@ -1620,31 +1668,42 @@ static inline void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, int opc)
                    tcg_opc_m3 (TCG_REG_P0, OPC_LD8_M3, TCG_REG_R3,
                                TCG_REG_R2, 8),
                    tcg_opc_a1 (TCG_REG_P6, OPC_ADD_A1, TCG_REG_R3,
-                               TCG_REG_R3, TCG_REG_R56),
+                               TCG_REG_R3, TCG_REG_R57),
                    tcg_opc_i21(TCG_REG_P7, OPC_MOV_I21, TCG_REG_B6,
                                TCG_REG_R3, 0));
 
     if (!bswap || opc == 0) {
-        tcg_out_bundle(s, mII,
+        tcg_out_bundle(s, mii,
                        tcg_opc_m1 (TCG_REG_P7, OPC_LD8_M1,
                                    TCG_REG_R1, TCG_REG_R2),
-                       tcg_opc_i18(TCG_REG_P0, OPC_NOP_I18, 0),
+                       tcg_opc_a4 (TCG_REG_P7, OPC_ADDS_A4, TCG_REG_R58,
+                                   0, data_reg),
                        tcg_opc_i18(TCG_REG_P0, OPC_NOP_I18, 0));
     } else if (opc == 1) {
-        tcg_out_bundle(s, mII,
+        tcg_out_bundle(s, miI,
                        tcg_opc_m1 (TCG_REG_P7, OPC_LD8_M1,
                                    TCG_REG_R1, TCG_REG_R2),
+                       tcg_opc_i18(TCG_REG_P0, OPC_NOP_I18, 0),
                        tcg_opc_i12(TCG_REG_P6, OPC_DEP_Z_I12,
-                                   TCG_REG_R2, data_reg, 15, 15),
+                                   TCG_REG_R2, data_reg, 15, 15));
+        tcg_out_bundle(s, miI,
+                       tcg_opc_a4 (TCG_REG_P7, OPC_ADDS_A4, TCG_REG_R58,
+                                   0, data_reg),
+                       tcg_opc_i18(TCG_REG_P0, OPC_NOP_I18, 0),
                        tcg_opc_i3 (TCG_REG_P6, OPC_MUX1_I3,
                                    TCG_REG_R2, TCG_REG_R2, 0xb));
         data_reg = TCG_REG_R2;
     } else if (opc == 2) {
-        tcg_out_bundle(s, mII,
+        tcg_out_bundle(s, miI,
                        tcg_opc_m1 (TCG_REG_P7, OPC_LD8_M1,
                                    TCG_REG_R1, TCG_REG_R2),
+                       tcg_opc_i18(TCG_REG_P0, OPC_NOP_I18, 0),
                        tcg_opc_i12(TCG_REG_P6, OPC_DEP_Z_I12,
-                                   TCG_REG_R2, data_reg, 31, 31),
+                                   TCG_REG_R2, data_reg, 31, 31));
+        tcg_out_bundle(s, miI,
+                       tcg_opc_a4 (TCG_REG_P7, OPC_ADDS_A4, TCG_REG_R58,
+                                   0, data_reg),
+                       tcg_opc_i18(TCG_REG_P0, OPC_NOP_I18, 0),
                        tcg_opc_i3 (TCG_REG_P6, OPC_MUX1_I3,
                                    TCG_REG_R2, TCG_REG_R2, 0xb));
         data_reg = TCG_REG_R2;
@@ -1652,37 +1711,20 @@ static inline void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, int opc)
         tcg_out_bundle(s, miI,
                        tcg_opc_m1 (TCG_REG_P7, OPC_LD8_M1,
                                    TCG_REG_R1, TCG_REG_R2),
-                       tcg_opc_i18(TCG_REG_P0, OPC_NOP_I18, 0),
+                       tcg_opc_a4 (TCG_REG_P7, OPC_ADDS_A4, TCG_REG_R58,
+                                   0, data_reg),
                        tcg_opc_i3 (TCG_REG_P6, OPC_MUX1_I3,
                                    TCG_REG_R2, data_reg, 0xb));
         data_reg = TCG_REG_R2;
     }
 
-#ifdef CONFIG_TCG_PASS_AREG0
-    /* XXX/FIXME: suboptimal */
-    tcg_out_bundle(s, mII,
-                   tcg_opc_a5 (TCG_REG_P7, OPC_ADDL_A5, TCG_REG_R59,
-                               mem_index, TCG_REG_R0),
-                   tcg_opc_a4 (TCG_REG_P7, OPC_ADDS_A4,
-                               TCG_REG_R58, 0, TCG_REG_R57),
-                   tcg_opc_a4 (TCG_REG_P7, OPC_ADDS_A4,
-                               TCG_REG_R57, 0, TCG_REG_R56));
     tcg_out_bundle(s, miB,
                    tcg_opc_m4 (TCG_REG_P6, opc_st_m4[opc],
                                data_reg, TCG_REG_R3),
-                   tcg_opc_a4 (TCG_REG_P7, OPC_ADDS_A4,
-                               TCG_REG_R56, 0, TCG_AREG0),
-                   tcg_opc_b5 (TCG_REG_P7, OPC_BR_CALL_SPTK_MANY_B5,
-                               TCG_REG_B0, TCG_REG_B6));
-#else
-    tcg_out_bundle(s, miB,
-                   tcg_opc_m4 (TCG_REG_P6, opc_st_m4[opc],
-                               data_reg, TCG_REG_R3),
-                   tcg_opc_a5 (TCG_REG_P7, OPC_ADDL_A5, TCG_REG_R58,
+                   tcg_opc_a5 (TCG_REG_P7, OPC_ADDL_A5, TCG_REG_R59,
                                mem_index, TCG_REG_R0),
                    tcg_opc_b5 (TCG_REG_P7, OPC_BR_CALL_SPTK_MANY_B5,
                                TCG_REG_B0, TCG_REG_B6));
-#endif
 }
 
 #else /* !CONFIG_SOFTMMU */
@@ -1956,9 +1998,6 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_goto_tb:
         tcg_out_goto_tb(s, args[0]);
         break;
-    case INDEX_op_jmp:
-        tcg_out_jmp(s, args[0]);
-        break;
 
     case INDEX_op_movi_i32:
         tcg_out_movi(s, TCG_TYPE_I32, args[0], args[1]);
@@ -2135,6 +2174,12 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
         tcg_out_bswap64(s, args[0], args[1]);
         break;
 
+    case INDEX_op_deposit_i32:
+    case INDEX_op_deposit_i64:
+        tcg_out_deposit(s, args[0], args[1], args[2], const_args[2],
+                        args[3], args[4]);
+        break;
+
     case INDEX_op_brcond_i32:
         tcg_out_brcond(s, args[2], args[0], const_args[0],
                        args[1], const_args[1], args[3], 1);
@@ -2149,6 +2194,14 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
     case INDEX_op_setcond_i64:
         tcg_out_setcond(s, args[3], args[0], args[1], args[2], 0);
         break;
+    case INDEX_op_movcond_i32:
+        tcg_out_movcond(s, args[5], args[0], args[1], args[2],
+                        args[3], const_args[3], args[4], const_args[4], 1);
+        break;
+    case INDEX_op_movcond_i64:
+        tcg_out_movcond(s, args[5], args[0], args[1], args[2],
+                        args[3], const_args[3], args[4], const_args[4], 0);
+        break;
 
     case INDEX_op_qemu_ld8u:
         tcg_out_qemu_ld(s, args, 0);
@@ -2196,7 +2249,6 @@ static const TCGTargetOpDef ia64_op_defs[] = {
     { INDEX_op_call, { "r" } },
     { INDEX_op_exit_tb, { } },
     { INDEX_op_goto_tb, { } },
-    { INDEX_op_jmp, { "r" } },
 
     { INDEX_op_mov_i32, { "r", "r" } },
     { INDEX_op_movi_i32, { "r" } },
@@ -2240,6 +2292,7 @@ static const TCGTargetOpDef ia64_op_defs[] = {
 
     { INDEX_op_brcond_i32, { "rI", "rI" } },
     { INDEX_op_setcond_i32, { "r", "rZ", "rZ" } },
+    { INDEX_op_movcond_i32, { "r", "rZ", "rZ", "rI", "rI" } },
 
     { INDEX_op_mov_i64, { "r", "r" } },
     { INDEX_op_movi_i64, { "r" } },
@@ -2289,6 +2342,10 @@ static const TCGTargetOpDef ia64_op_defs[] = {
 
     { INDEX_op_brcond_i64, { "rI", "rI" } },
     { INDEX_op_setcond_i64, { "r", "rZ", "rZ" } },
+    { INDEX_op_movcond_i64, { "r", "rZ", "rZ", "rI", "rI" } },
+
+    { INDEX_op_deposit_i32, { "r", "rZ", "ri" } },
+    { INDEX_op_deposit_i64, { "r", "rZ", "ri" } },
 
     { INDEX_op_qemu_ld8u, { "r", "r" } },
     { INDEX_op_qemu_ld8s, { "r", "r" } },
@@ -2313,9 +2370,12 @@ static void tcg_target_qemu_prologue(TCGContext *s)
     int frame_size;
 
     /* reserve some stack space */
-    frame_size = TCG_STATIC_CALL_ARGS_SIZE;
+    frame_size = TCG_STATIC_CALL_ARGS_SIZE +
+                 CPU_TEMP_BUF_NLONGS * sizeof(long);
     frame_size = (frame_size + TCG_TARGET_STACK_ALIGN - 1) &
                  ~(TCG_TARGET_STACK_ALIGN - 1);
+    tcg_set_frame(s, TCG_REG_CALL_STACK, TCG_STATIC_CALL_ARGS_SIZE,
+                  CPU_TEMP_BUF_NLONGS * sizeof(long));
 
     /* First emit adhoc function descriptor */
     *(uint64_t *)(s->code_ptr) = (uint64_t)s->code_ptr + 16; /* entry point */
@@ -2422,6 +2482,4 @@ static void tcg_target_init(TCGContext *s)
     tcg_regset_set_reg(s->reserved_regs, TCG_REG_R6);
 
     tcg_add_target_add_op_defs(ia64_op_defs);
-    tcg_set_frame(s, TCG_AREG0, offsetof(CPUArchState, temp_buf),
-                  CPU_TEMP_BUF_NLONGS * sizeof(long));
 }
diff --git a/tcg/ia64/tcg-target.h b/tcg/ia64/tcg-target.h
index 0631b9f18..91fe7a3b0 100644
--- a/tcg/ia64/tcg-target.h
+++ b/tcg/ia64/tcg-target.h
@@ -131,8 +131,13 @@ typedef enum {
 #define TCG_TARGET_HAS_orc_i64          1
 #define TCG_TARGET_HAS_rot_i32          1
 #define TCG_TARGET_HAS_rot_i64          1
-#define TCG_TARGET_HAS_deposit_i32      0
-#define TCG_TARGET_HAS_deposit_i64      0
+#define TCG_TARGET_HAS_movcond_i32      1
+#define TCG_TARGET_HAS_movcond_i64      1
+#define TCG_TARGET_HAS_deposit_i32      1
+#define TCG_TARGET_HAS_deposit_i64      1
+
+#define TCG_TARGET_deposit_i32_valid(ofs, len) ((len) <= 16)
+#define TCG_TARGET_deposit_i64_valid(ofs, len) ((len) <= 16)
 
 /* optional instructions automatically implemented */
 #define TCG_TARGET_HAS_neg_i32          0 /* sub r1, r0, r3 */
@@ -140,12 +145,8 @@ typedef enum {
 #define TCG_TARGET_HAS_not_i32          0 /* xor r1, -1, r3 */
 #define TCG_TARGET_HAS_not_i64          0 /* xor r1, -1, r3 */
 
-/* Note: must be synced with dyngen-exec.h */
 #define TCG_AREG0 TCG_REG_R7
 
-/* Guest base is supported */
-#define TCG_TARGET_HAS_GUEST_BASE
-
 static inline void flush_icache_range(tcg_target_ulong start,
                                       tcg_target_ulong stop)
 {
diff --git a/tcg/mips/tcg-target.c b/tcg/mips/tcg-target.c
index 1006e2800..ae2b274d8 100644
--- a/tcg/mips/tcg-target.c
+++ b/tcg/mips/tcg-target.c
@@ -68,7 +68,7 @@ static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = {
 #endif
 
 /* check if we really need so many registers :P */
-static const int tcg_target_reg_alloc_order[] = {
+static const TCGReg tcg_target_reg_alloc_order[] = {
     TCG_REG_S0,
     TCG_REG_S1,
     TCG_REG_S2,
@@ -94,14 +94,14 @@ static const int tcg_target_reg_alloc_order[] = {
     TCG_REG_V1
 };
 
-static const int tcg_target_call_iarg_regs[4] = {
+static const TCGReg tcg_target_call_iarg_regs[4] = {
     TCG_REG_A0,
     TCG_REG_A1,
     TCG_REG_A2,
     TCG_REG_A3
 };
 
-static const int tcg_target_call_oarg_regs[2] = {
+static const TCGReg tcg_target_call_oarg_regs[2] = {
     TCG_REG_V0,
     TCG_REG_V1
 };
@@ -185,12 +185,6 @@ static void patch_reloc(uint8_t *code_ptr, int type,
     }
 }
 
-/* maximum number of register used for input function arguments */
-static inline int tcg_target_get_call_iarg_regs_count(int flags)
-{
-    return 4;
-}
-
 /* parse target specific constraints */
 static int target_parse_constraint(TCGArgConstraint *ct, const char **pct_str)
 {
@@ -217,7 +211,7 @@ static int target_parse_constraint(TCGArgConstraint *ct, const char **pct_str)
         tcg_regset_set(ct->u.regs, 0xffffffff);
 #if defined(CONFIG_SOFTMMU)
         tcg_regset_reset_reg(ct->u.regs, TCG_REG_A0);
-# if defined(CONFIG_TCG_PASS_AREG0) && (TARGET_LONG_BITS == 64)
+# if (TARGET_LONG_BITS == 64)
         tcg_regset_reset_reg(ct->u.regs, TCG_REG_A2);
 # endif
 #endif
@@ -227,12 +221,11 @@ static int target_parse_constraint(TCGArgConstraint *ct, const char **pct_str)
         tcg_regset_set(ct->u.regs, 0xffffffff);
         tcg_regset_reset_reg(ct->u.regs, TCG_REG_A0);
 #if defined(CONFIG_SOFTMMU)
-# if (defined(CONFIG_TCG_PASS_AREG0) && TARGET_LONG_BITS == 32) || \
-     (!defined(CONFIG_TCG_PASS_AREG0) && TARGET_LONG_BITS == 64)
+# if (TARGET_LONG_BITS == 32)
         tcg_regset_reset_reg(ct->u.regs, TCG_REG_A1);
 # endif
         tcg_regset_reset_reg(ct->u.regs, TCG_REG_A2);
-# if defined(CONFIG_TCG_PASS_AREG0) && TARGET_LONG_BITS == 64
+# if TARGET_LONG_BITS == 64
         tcg_regset_reset_reg(ct->u.regs, TCG_REG_A3);
 # endif
 #endif
@@ -279,6 +272,8 @@ static inline int tcg_target_const_match(tcg_target_long val,
 enum {
     OPC_BEQ      = 0x04 << 26,
     OPC_BNE      = 0x05 << 26,
+    OPC_BLEZ     = 0x06 << 26,
+    OPC_BGTZ     = 0x07 << 26,
     OPC_ADDIU    = 0x09 << 26,
     OPC_SLTI     = 0x0A << 26,
     OPC_SLTIU    = 0x0B << 26,
@@ -299,12 +294,16 @@ enum {
     OPC_SPECIAL  = 0x00 << 26,
     OPC_SLL      = OPC_SPECIAL | 0x00,
     OPC_SRL      = OPC_SPECIAL | 0x02,
+    OPC_ROTR     = OPC_SPECIAL | (0x01 << 21) | 0x02,
     OPC_SRA      = OPC_SPECIAL | 0x03,
     OPC_SLLV     = OPC_SPECIAL | 0x04,
     OPC_SRLV     = OPC_SPECIAL | 0x06,
+    OPC_ROTRV    = OPC_SPECIAL | (0x01 <<  6) | 0x06,
     OPC_SRAV     = OPC_SPECIAL | 0x07,
     OPC_JR       = OPC_SPECIAL | 0x08,
     OPC_JALR     = OPC_SPECIAL | 0x09,
+    OPC_MOVZ     = OPC_SPECIAL | 0x0A,
+    OPC_MOVN     = OPC_SPECIAL | 0x0B,
     OPC_MFHI     = OPC_SPECIAL | 0x10,
     OPC_MFLO     = OPC_SPECIAL | 0x12,
     OPC_MULT     = OPC_SPECIAL | 0x18,
@@ -320,7 +319,16 @@ enum {
     OPC_SLT      = OPC_SPECIAL | 0x2A,
     OPC_SLTU     = OPC_SPECIAL | 0x2B,
 
+    OPC_REGIMM   = 0x01 << 26,
+    OPC_BLTZ     = OPC_REGIMM | (0x00 << 16),
+    OPC_BGEZ     = OPC_REGIMM | (0x01 << 16),
+
+    OPC_SPECIAL2 = 0x1c << 26,
+    OPC_MUL      = OPC_SPECIAL2 | 0x002,
+
     OPC_SPECIAL3 = 0x1f << 26,
+    OPC_INS      = OPC_SPECIAL3 | 0x004,
+    OPC_WSBH     = OPC_SPECIAL3 | 0x0a0,
     OPC_SEB      = OPC_SPECIAL3 | 0x420,
     OPC_SEH      = OPC_SPECIAL3 | 0x620,
 };
@@ -328,7 +336,8 @@ enum {
 /*
  * Type reg
  */
-static inline void tcg_out_opc_reg(TCGContext *s, int opc, int rd, int rs, int rt)
+static inline void tcg_out_opc_reg(TCGContext *s, int opc,
+                                   TCGReg rd, TCGReg rs, TCGReg rt)
 {
     int32_t inst;
 
@@ -342,7 +351,8 @@ static inline void tcg_out_opc_reg(TCGContext *s, int opc, int rd, int rs, int r
 /*
  * Type immediate
  */
-static inline void tcg_out_opc_imm(TCGContext *s, int opc, int rt, int rs, int imm)
+static inline void tcg_out_opc_imm(TCGContext *s, int opc,
+                                   TCGReg rt, TCGReg rs, TCGArg imm)
 {
     int32_t inst;
 
@@ -356,7 +366,8 @@ static inline void tcg_out_opc_imm(TCGContext *s, int opc, int rt, int rs, int i
 /*
  * Type branch
  */
-static inline void tcg_out_opc_br(TCGContext *s, int opc, int rt, int rs)
+static inline void tcg_out_opc_br(TCGContext *s, int opc,
+                                  TCGReg rt, TCGReg rs)
 {
     /* We pay attention here to not modify the branch target by reading
        the existing value and using it again. This ensure that caches and
@@ -369,7 +380,8 @@ static inline void tcg_out_opc_br(TCGContext *s, int opc, int rt, int rs)
 /*
  * Type sa
  */
-static inline void tcg_out_opc_sa(TCGContext *s, int opc, int rd, int rt, int sa)
+static inline void tcg_out_opc_sa(TCGContext *s, int opc,
+                                  TCGReg rd, TCGReg rt, TCGArg sa)
 {
     int32_t inst;
 
@@ -408,38 +420,47 @@ static inline void tcg_out_movi(TCGContext *s, TCGType type,
     }
 }
 
-static inline void tcg_out_bswap16(TCGContext *s, int ret, int arg)
+static inline void tcg_out_bswap16(TCGContext *s, TCGReg ret, TCGReg arg)
 {
+#if defined(__mips_isa_rev) && (__mips_isa_rev >= 2)
+    tcg_out_opc_reg(s, OPC_WSBH, ret, 0, arg);
+#else
     /* ret and arg can't be register at */
     if (ret == TCG_REG_AT || arg == TCG_REG_AT) {
         tcg_abort();
     }
 
     tcg_out_opc_sa(s, OPC_SRL, TCG_REG_AT, arg, 8);
-    tcg_out_opc_imm(s, OPC_ANDI, TCG_REG_AT, TCG_REG_AT, 0x00ff);
-
     tcg_out_opc_sa(s, OPC_SLL, ret, arg, 8);
     tcg_out_opc_imm(s, OPC_ANDI, ret, ret, 0xff00);
     tcg_out_opc_reg(s, OPC_OR, ret, ret, TCG_REG_AT);
+#endif
 }
 
-static inline void tcg_out_bswap16s(TCGContext *s, int ret, int arg)
+static inline void tcg_out_bswap16s(TCGContext *s, TCGReg ret, TCGReg arg)
 {
+#if defined(__mips_isa_rev) && (__mips_isa_rev >= 2)
+    tcg_out_opc_reg(s, OPC_WSBH, ret, 0, arg);
+    tcg_out_opc_reg(s, OPC_SEH, ret, 0, ret);
+#else
     /* ret and arg can't be register at */
     if (ret == TCG_REG_AT || arg == TCG_REG_AT) {
         tcg_abort();
     }
 
     tcg_out_opc_sa(s, OPC_SRL, TCG_REG_AT, arg, 8);
-    tcg_out_opc_imm(s, OPC_ANDI, TCG_REG_AT, TCG_REG_AT, 0xff);
-
     tcg_out_opc_sa(s, OPC_SLL, ret, arg, 24);
     tcg_out_opc_sa(s, OPC_SRA, ret, ret, 16);
     tcg_out_opc_reg(s, OPC_OR, ret, ret, TCG_REG_AT);
+#endif
 }
 
-static inline void tcg_out_bswap32(TCGContext *s, int ret, int arg)
+static inline void tcg_out_bswap32(TCGContext *s, TCGReg ret, TCGReg arg)
 {
+#if defined(__mips_isa_rev) && (__mips_isa_rev >= 2)
+    tcg_out_opc_reg(s, OPC_WSBH, ret, 0, arg);
+    tcg_out_opc_sa(s, OPC_ROTR, ret, ret, 16);
+#else
     /* ret and arg must be different and can't be register at */
     if (ret == arg || ret == TCG_REG_AT || arg == TCG_REG_AT) {
         tcg_abort();
@@ -457,11 +478,12 @@ static inline void tcg_out_bswap32(TCGContext *s, int ret, int arg)
     tcg_out_opc_sa(s, OPC_SRL, TCG_REG_AT, arg, 8);
     tcg_out_opc_imm(s, OPC_ANDI, TCG_REG_AT, TCG_REG_AT, 0xff00);
     tcg_out_opc_reg(s, OPC_OR, ret, ret, TCG_REG_AT);
+#endif
 }
 
-static inline void tcg_out_ext8s(TCGContext *s, int ret, int arg)
+static inline void tcg_out_ext8s(TCGContext *s, TCGReg ret, TCGReg arg)
 {
-#ifdef _MIPS_ARCH_MIPS32R2
+#if defined(__mips_isa_rev) && (__mips_isa_rev >= 2)
     tcg_out_opc_reg(s, OPC_SEB, ret, 0, arg);
 #else
     tcg_out_opc_sa(s, OPC_SLL, ret, arg, 24);
@@ -469,9 +491,9 @@ static inline void tcg_out_ext8s(TCGContext *s, int ret, int arg)
 #endif
 }
 
-static inline void tcg_out_ext16s(TCGContext *s, int ret, int arg)
+static inline void tcg_out_ext16s(TCGContext *s, TCGReg ret, TCGReg arg)
 {
-#ifdef _MIPS_ARCH_MIPS32R2
+#if defined(__mips_isa_rev) && (__mips_isa_rev >= 2)
     tcg_out_opc_reg(s, OPC_SEH, ret, 0, arg);
 #else
     tcg_out_opc_sa(s, OPC_SLL, ret, arg, 16);
@@ -479,8 +501,8 @@ static inline void tcg_out_ext16s(TCGContext *s, int ret, int arg)
 #endif
 }
 
-static inline void tcg_out_ldst(TCGContext *s, int opc, int arg,
-                              int arg1, tcg_target_long arg2)
+static inline void tcg_out_ldst(TCGContext *s, int opc, TCGArg arg,
+                                TCGReg arg1, TCGArg arg2)
 {
     if (arg2 == (int16_t) arg2) {
         tcg_out_opc_imm(s, opc, arg, arg1, arg2);
@@ -503,7 +525,7 @@ static inline void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg,
     tcg_out_ldst(s, OPC_SW, arg, arg1, arg2);
 }
 
-static inline void tcg_out_addi(TCGContext *s, int reg, tcg_target_long val)
+static inline void tcg_out_addi(TCGContext *s, TCGReg reg, TCGArg val)
 {
     if (val == (int16_t)val) {
         tcg_out_opc_imm(s, OPC_ADDIU, reg, reg, val);
@@ -544,7 +566,7 @@ DEFINE_TCG_OUT_CALL_IARG(tcg_out_call_iarg_reg16, TCGReg arg)
 #undef DEFINE_TCG_OUT_CALL_IARG_GET_ARG
 #define DEFINE_TCG_OUT_CALL_IARG_GET_ARG(A) \
     tcg_out_movi(s, TCG_TYPE_I32, A, arg);
-DEFINE_TCG_OUT_CALL_IARG(tcg_out_call_iarg_imm32, uint32_t arg)
+DEFINE_TCG_OUT_CALL_IARG(tcg_out_call_iarg_imm32, TCGArg arg)
 #undef DEFINE_TCG_OUT_CALL_IARG_GET_ARG
 
 /* We don't use the macro for this one to avoid an unnecessary reg-reg
@@ -574,8 +596,8 @@ static inline void tcg_out_call_iarg_reg64(TCGContext *s, int *arg_num,
 #endif
 }
 
-static void tcg_out_brcond(TCGContext *s, TCGCond cond, int arg1,
-                           int arg2, int label_index)
+static void tcg_out_brcond(TCGContext *s, TCGCond cond, TCGArg arg1,
+                           TCGArg arg2, int label_index)
 {
     TCGLabel *l = &s->labels[label_index];
 
@@ -587,32 +609,48 @@ static void tcg_out_brcond(TCGContext *s, TCGCond cond, int arg1,
         tcg_out_opc_br(s, OPC_BNE, arg1, arg2);
         break;
     case TCG_COND_LT:
-        tcg_out_opc_reg(s, OPC_SLT, TCG_REG_AT, arg1, arg2);
-        tcg_out_opc_br(s, OPC_BNE, TCG_REG_AT, TCG_REG_ZERO);
+        if (arg2 == 0) {
+            tcg_out_opc_br(s, OPC_BLTZ, 0, arg1);
+        } else {
+            tcg_out_opc_reg(s, OPC_SLT, TCG_REG_AT, arg1, arg2);
+            tcg_out_opc_br(s, OPC_BNE, TCG_REG_AT, TCG_REG_ZERO);
+        }
         break;
     case TCG_COND_LTU:
         tcg_out_opc_reg(s, OPC_SLTU, TCG_REG_AT, arg1, arg2);
         tcg_out_opc_br(s, OPC_BNE, TCG_REG_AT, TCG_REG_ZERO);
         break;
     case TCG_COND_GE:
-        tcg_out_opc_reg(s, OPC_SLT, TCG_REG_AT, arg1, arg2);
-        tcg_out_opc_br(s, OPC_BEQ, TCG_REG_AT, TCG_REG_ZERO);
+        if (arg2 == 0) {
+            tcg_out_opc_br(s, OPC_BGEZ, 0, arg1);
+        } else {
+            tcg_out_opc_reg(s, OPC_SLT, TCG_REG_AT, arg1, arg2);
+            tcg_out_opc_br(s, OPC_BEQ, TCG_REG_AT, TCG_REG_ZERO);
+        }
         break;
     case TCG_COND_GEU:
         tcg_out_opc_reg(s, OPC_SLTU, TCG_REG_AT, arg1, arg2);
         tcg_out_opc_br(s, OPC_BEQ, TCG_REG_AT, TCG_REG_ZERO);
         break;
     case TCG_COND_LE:
-        tcg_out_opc_reg(s, OPC_SLT, TCG_REG_AT, arg2, arg1);
-        tcg_out_opc_br(s, OPC_BEQ, TCG_REG_AT, TCG_REG_ZERO);
+        if (arg2 == 0) {
+            tcg_out_opc_br(s, OPC_BLEZ, 0, arg1);
+        } else {
+            tcg_out_opc_reg(s, OPC_SLT, TCG_REG_AT, arg2, arg1);
+            tcg_out_opc_br(s, OPC_BEQ, TCG_REG_AT, TCG_REG_ZERO);
+        }
         break;
     case TCG_COND_LEU:
         tcg_out_opc_reg(s, OPC_SLTU, TCG_REG_AT, arg2, arg1);
         tcg_out_opc_br(s, OPC_BEQ, TCG_REG_AT, TCG_REG_ZERO);
         break;
     case TCG_COND_GT:
-        tcg_out_opc_reg(s, OPC_SLT, TCG_REG_AT, arg2, arg1);
-        tcg_out_opc_br(s, OPC_BNE, TCG_REG_AT, TCG_REG_ZERO);
+        if (arg2 == 0) {
+            tcg_out_opc_br(s, OPC_BGTZ, 0, arg1);
+        } else {
+            tcg_out_opc_reg(s, OPC_SLT, TCG_REG_AT, arg2, arg1);
+            tcg_out_opc_br(s, OPC_BNE, TCG_REG_AT, TCG_REG_ZERO);
+        }
         break;
     case TCG_COND_GTU:
         tcg_out_opc_reg(s, OPC_SLTU, TCG_REG_AT, arg2, arg1);
@@ -632,8 +670,9 @@ static void tcg_out_brcond(TCGContext *s, TCGCond cond, int arg1,
 
 /* XXX: we implement it at the target level to avoid having to
    handle cross basic blocks temporaries */
-static void tcg_out_brcond2(TCGContext *s, TCGCond cond, int arg1,
-                            int arg2, int arg3, int arg4, int label_index)
+static void tcg_out_brcond2(TCGContext *s, TCGCond cond, TCGArg arg1,
+                            TCGArg arg2, TCGArg arg3, TCGArg arg4,
+                            int label_index)
 {
     void *label_ptr;
 
@@ -695,8 +734,70 @@ static void tcg_out_brcond2(TCGContext *s, TCGCond cond, int arg1,
     reloc_pc16(label_ptr, (tcg_target_long) s->code_ptr);
 }
 
-static void tcg_out_setcond(TCGContext *s, TCGCond cond, int ret,
-                            int arg1, int arg2)
+static void tcg_out_movcond(TCGContext *s, TCGCond cond, TCGReg ret,
+                            TCGArg c1, TCGArg c2, TCGArg v)
+{
+    switch (cond) {
+    case TCG_COND_EQ:
+        if (c1 == 0) {
+            tcg_out_opc_reg(s, OPC_MOVZ, ret, v, c2);
+        } else if (c2 == 0) {
+            tcg_out_opc_reg(s, OPC_MOVZ, ret, v, c1);
+        } else {
+            tcg_out_opc_reg(s, OPC_XOR, TCG_REG_AT, c1, c2);
+            tcg_out_opc_reg(s, OPC_MOVZ, ret, v, TCG_REG_AT);
+        }
+        break;
+    case TCG_COND_NE:
+        if (c1 == 0) {
+            tcg_out_opc_reg(s, OPC_MOVN, ret, v, c2);
+        } else if (c2 == 0) {
+            tcg_out_opc_reg(s, OPC_MOVN, ret, v, c1);
+        } else {
+            tcg_out_opc_reg(s, OPC_XOR, TCG_REG_AT, c1, c2);
+            tcg_out_opc_reg(s, OPC_MOVN, ret, v, TCG_REG_AT);
+        }
+        break;
+    case TCG_COND_LT:
+        tcg_out_opc_reg(s, OPC_SLT, TCG_REG_AT, c1, c2);
+        tcg_out_opc_reg(s, OPC_MOVN, ret, v, TCG_REG_AT);
+        break;
+    case TCG_COND_LTU:
+        tcg_out_opc_reg(s, OPC_SLTU, TCG_REG_AT, c1, c2);
+        tcg_out_opc_reg(s, OPC_MOVN, ret, v, TCG_REG_AT);
+        break;
+    case TCG_COND_GE:
+        tcg_out_opc_reg(s, OPC_SLT, TCG_REG_AT, c1, c2);
+        tcg_out_opc_reg(s, OPC_MOVZ, ret, v, TCG_REG_AT);
+        break;
+    case TCG_COND_GEU:
+        tcg_out_opc_reg(s, OPC_SLTU, TCG_REG_AT, c1, c2);
+        tcg_out_opc_reg(s, OPC_MOVZ, ret, v, TCG_REG_AT);
+        break;
+    case TCG_COND_LE:
+        tcg_out_opc_reg(s, OPC_SLT, TCG_REG_AT, c2, c1);
+        tcg_out_opc_reg(s, OPC_MOVZ, ret, v, TCG_REG_AT);
+        break;
+    case TCG_COND_LEU:
+        tcg_out_opc_reg(s, OPC_SLTU, TCG_REG_AT, c2, c1);
+        tcg_out_opc_reg(s, OPC_MOVZ, ret, v, TCG_REG_AT);
+        break;
+    case TCG_COND_GT:
+        tcg_out_opc_reg(s, OPC_SLT, TCG_REG_AT, c2, c1);
+        tcg_out_opc_reg(s, OPC_MOVN, ret, v, TCG_REG_AT);
+        break;
+    case TCG_COND_GTU:
+        tcg_out_opc_reg(s, OPC_SLTU, TCG_REG_AT, c2, c1);
+        tcg_out_opc_reg(s, OPC_MOVN, ret, v, TCG_REG_AT);
+        break;
+    default:
+        tcg_abort();
+        break;
+    }
+}
+
+static void tcg_out_setcond(TCGContext *s, TCGCond cond, TCGReg ret,
+                            TCGArg arg1, TCGArg arg2)
 {
     switch (cond) {
     case TCG_COND_EQ:
@@ -755,8 +856,8 @@ static void tcg_out_setcond(TCGContext *s, TCGCond cond, int ret,
 
 /* XXX: we implement it at the target level to avoid having to
    handle cross basic blocks temporaries */
-static void tcg_out_setcond2(TCGContext *s, TCGCond cond, int ret,
-                             int arg1, int arg2, int arg3, int arg4)
+static void tcg_out_setcond2(TCGContext *s, TCGCond cond, TCGReg ret,
+                             TCGArg arg1, TCGArg arg2, TCGArg arg3, TCGArg arg4)
 {
     switch (cond) {
     case TCG_COND_EQ:
@@ -821,7 +922,6 @@ static void tcg_out_setcond2(TCGContext *s, TCGCond cond, int ret,
 
 #include "../../softmmu_defs.h"
 
-#ifdef CONFIG_TCG_PASS_AREG0
 /* helper signature: helper_ld_mmu(CPUState *env, target_ulong addr,
    int mmu_idx) */
 static const void * const qemu_ld_helpers[4] = {
@@ -839,42 +939,22 @@ static const void * const qemu_st_helpers[4] = {
     helper_stl_mmu,
     helper_stq_mmu,
 };
-#else
-/* legacy helper signature: __ld_mmu(target_ulong addr, int
-   mmu_idx) */
-static void *qemu_ld_helpers[4] = {
-    __ldb_mmu,
-    __ldw_mmu,
-    __ldl_mmu,
-    __ldq_mmu,
-};
-
-/* legacy helper signature: __st_mmu(target_ulong addr, uintxx_t val,
-   int mmu_idx) */
-static void *qemu_st_helpers[4] = {
-    __stb_mmu,
-    __stw_mmu,
-    __stl_mmu,
-    __stq_mmu,
-};
-#endif
 #endif
 
 static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args,
                             int opc)
 {
-    int addr_regl, addr_meml;
-    int data_regl, data_regh, data_reg1, data_reg2;
-    int mem_index, s_bits;
+    TCGReg addr_regl, data_regl, data_regh, data_reg1, data_reg2;
 #if defined(CONFIG_SOFTMMU)
     void *label1_ptr, *label2_ptr;
     int arg_num;
-#endif
-#if TARGET_LONG_BITS == 64
-# if defined(CONFIG_SOFTMMU)
+    int mem_index, s_bits;
+    int addr_meml;
+# if TARGET_LONG_BITS == 64
     uint8_t *label3_ptr;
+    TCGReg addr_regh;
+    int addr_memh;
 # endif
-    int addr_regh, addr_memh;
 #endif
     data_regl = *args++;
     if (opc == 3)
@@ -882,11 +962,22 @@ static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args,
     else
         data_regh = 0;
     addr_regl = *args++;
-#if TARGET_LONG_BITS == 64
+#if defined(CONFIG_SOFTMMU)
+# if TARGET_LONG_BITS == 64
     addr_regh = *args++;
-#endif
+#  if defined(TCG_TARGET_WORDS_BIGENDIAN)
+    addr_memh = 0;
+    addr_meml = 4;
+#  else
+    addr_memh = 4;
+    addr_meml = 0;
+#  endif
+# else
+    addr_meml = 0;
+# endif
     mem_index = *args;
     s_bits = opc & 3;
+#endif
 
     if (opc == 3) {
 #if defined(TCG_TARGET_WORDS_BIGENDIAN)
@@ -900,18 +991,6 @@ static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args,
         data_reg1 = data_regl;
         data_reg2 = 0;
     }
-#if TARGET_LONG_BITS == 64
-# if defined(TCG_TARGET_WORDS_BIGENDIAN)
-    addr_memh = 0;
-    addr_meml = 4;
-# else
-    addr_memh = 4;
-    addr_meml = 0;
-# endif
-#else
-    addr_meml = 0;
-#endif
-
 #if defined(CONFIG_SOFTMMU)
     tcg_out_opc_sa(s, OPC_SRL, TCG_REG_A0, addr_regl, TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS);
     tcg_out_opc_imm(s, OPC_ANDI, TCG_REG_A0, TCG_REG_A0, (CPU_TLB_SIZE - 1) << CPU_TLB_ENTRY_BITS);
@@ -942,9 +1021,7 @@ static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args,
 
     /* slow path */
     arg_num = 0;
-# ifdef CONFIG_TCG_PASS_AREG0
     tcg_out_call_iarg_reg32(s, &arg_num, TCG_AREG0);
-# endif
 # if TARGET_LONG_BITS == 64
     tcg_out_call_iarg_reg64(s, &arg_num, addr_regl, addr_regh);
 # else
@@ -1052,50 +1129,56 @@ static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args,
 static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args,
                             int opc)
 {
-    int addr_regl, addr_meml;
-    int data_regl, data_regh, data_reg1, data_reg2;
-    int mem_index, s_bits;
+    TCGReg addr_regl, data_regl, data_regh, data_reg1, data_reg2;
 #if defined(CONFIG_SOFTMMU)
     uint8_t *label1_ptr, *label2_ptr;
     int arg_num;
+    int mem_index, s_bits;
+    int addr_meml;
 #endif
 #if TARGET_LONG_BITS == 64
 # if defined(CONFIG_SOFTMMU)
     uint8_t *label3_ptr;
+    TCGReg addr_regh;
+    int addr_memh;
 # endif
-    int addr_regh, addr_memh;
 #endif
-
     data_regl = *args++;
     if (opc == 3) {
         data_regh = *args++;
-#if defined(TCG_TARGET_WORDS_BIGENDIAN)
-        data_reg1 = data_regh;
-        data_reg2 = data_regl;
-#else
-        data_reg1 = data_regl;
-        data_reg2 = data_regh;
-#endif
     } else {
-        data_reg1 = data_regl;
-        data_reg2 = 0;
         data_regh = 0;
     }
     addr_regl = *args++;
-#if TARGET_LONG_BITS == 64
+#if defined(CONFIG_SOFTMMU)
+# if TARGET_LONG_BITS == 64
     addr_regh = *args++;
-# if defined(TCG_TARGET_WORDS_BIGENDIAN)
+#  if defined(TCG_TARGET_WORDS_BIGENDIAN)
     addr_memh = 0;
     addr_meml = 4;
-# else
+#  else
     addr_memh = 4;
     addr_meml = 0;
-# endif
-#else
+#  endif
+# else
     addr_meml = 0;
-#endif
+# endif
     mem_index = *args;
     s_bits = opc;
+#endif
+
+    if (opc == 3) {
+#if defined(TCG_TARGET_WORDS_BIGENDIAN)
+        data_reg1 = data_regh;
+        data_reg2 = data_regl;
+#else
+        data_reg1 = data_regl;
+        data_reg2 = data_regh;
+#endif
+    } else {
+        data_reg1 = data_regl;
+        data_reg2 = 0;
+    }
 
 #if defined(CONFIG_SOFTMMU)
     tcg_out_opc_sa(s, OPC_SRL, TCG_REG_A0, addr_regl, TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS);
@@ -1127,9 +1210,7 @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args,
 
     /* slow path */
     arg_num = 0;
-# ifdef CONFIG_TCG_PASS_AREG0
     tcg_out_call_iarg_reg32(s, &arg_num, TCG_AREG0);
-# endif
 # if TARGET_LONG_BITS == 64
     tcg_out_call_iarg_reg64(s, &arg_num, addr_regl, addr_regh);
 # else
@@ -1182,7 +1263,8 @@ static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args,
         break;
     case 1:
         if (TCG_NEED_BSWAP) {
-            tcg_out_bswap16(s, TCG_REG_T0, data_reg1);
+            tcg_out_opc_imm(s, OPC_ANDI, TCG_REG_T0, data_reg1, 0xffff);
+            tcg_out_bswap16(s, TCG_REG_T0, TCG_REG_T0);
             tcg_out_opc_imm(s, OPC_SH, TCG_REG_T0, TCG_REG_A0, 0);
         } else {
             tcg_out_opc_imm(s, OPC_SH, data_reg1, TCG_REG_A0, 0);
@@ -1243,10 +1325,6 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
         tcg_out_opc_reg(s, OPC_JALR, TCG_REG_RA, args[0], 0);
         tcg_out_nop(s);
         break;
-    case INDEX_op_jmp:
-        tcg_out_opc_reg(s, OPC_JR, 0, args[0], 0);
-        tcg_out_nop(s);
-        break;
     case INDEX_op_br:
         tcg_out_brcond(s, TCG_COND_EQ, TCG_REG_ZERO, TCG_REG_ZERO, args[0]);
         break;
@@ -1328,8 +1406,12 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
         tcg_out_mov(s, TCG_TYPE_I32, args[0], TCG_REG_AT);
         break;
     case INDEX_op_mul_i32:
+#if defined(__mips_isa_rev) && (__mips_isa_rev >= 1)
+        tcg_out_opc_reg(s, OPC_MUL, args[0], args[1], args[2]);
+#else
         tcg_out_opc_reg(s, OPC_MULT, 0, args[1], args[2]);
         tcg_out_opc_reg(s, OPC_MFLO, args[0], 0, 0);
+#endif
         break;
     case INDEX_op_mulu2_i32:
         tcg_out_opc_reg(s, OPC_MULTU, 0, args[2], args[3]);
@@ -1402,6 +1484,31 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
             tcg_out_opc_reg(s, OPC_SRLV, args[0], args[2], args[1]);
         }
         break;
+    case INDEX_op_rotl_i32:
+        if (const_args[2]) {
+            tcg_out_opc_sa(s, OPC_ROTR, args[0], args[1], 0x20 - args[2]);
+        } else {
+            tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_AT, 32);
+            tcg_out_opc_reg(s, OPC_SUBU, TCG_REG_AT, TCG_REG_AT, args[2]);
+            tcg_out_opc_reg(s, OPC_ROTRV, args[0], TCG_REG_AT, args[1]);
+        }
+        break;
+    case INDEX_op_rotr_i32:
+        if (const_args[2]) {
+            tcg_out_opc_sa(s, OPC_ROTR, args[0], args[1], args[2]);
+        } else {
+            tcg_out_opc_reg(s, OPC_ROTRV, args[0], args[2], args[1]);
+        }
+        break;
+
+    /* The bswap routines do not work on non-R2 CPU. In that case
+       we let TCG generating the corresponding code. */
+    case INDEX_op_bswap16_i32:
+        tcg_out_bswap16(s, args[0], args[1]);
+        break;
+    case INDEX_op_bswap32_i32:
+        tcg_out_bswap32(s, args[0], args[1]);
+        break;
 
     case INDEX_op_ext8s_i32:
         tcg_out_ext8s(s, args[0], args[1]);
@@ -1410,6 +1517,11 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
         tcg_out_ext16s(s, args[0], args[1]);
         break;
 
+    case INDEX_op_deposit_i32:
+        tcg_out_opc_imm(s, OPC_INS, args[0], args[2],
+                        ((args[3] + args[4] - 1) << 11) | (args[3] << 6));
+        break;
+
     case INDEX_op_brcond_i32:
         tcg_out_brcond(s, args[2], args[0], args[1], args[3]);
         break;
@@ -1417,6 +1529,10 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
         tcg_out_brcond2(s, args[4], args[0], args[1], args[2], args[3], args[5]);
         break;
 
+    case INDEX_op_movcond_i32:
+        tcg_out_movcond(s, args[5], args[0], args[1], args[2], args[3]);
+        break;
+
     case INDEX_op_setcond_i32:
         tcg_out_setcond(s, args[3], args[0], args[1], args[2]);
         break;
@@ -1464,7 +1580,6 @@ static const TCGTargetOpDef mips_op_defs[] = {
     { INDEX_op_exit_tb, { } },
     { INDEX_op_goto_tb, { } },
     { INDEX_op_call, { "C" } },
-    { INDEX_op_jmp, { "r" } },
     { INDEX_op_br, { } },
 
     { INDEX_op_mov_i32, { "r", "r" } },
@@ -1478,34 +1593,42 @@ static const TCGTargetOpDef mips_op_defs[] = {
     { INDEX_op_st16_i32, { "rZ", "r" } },
     { INDEX_op_st_i32, { "rZ", "r" } },
 
-    { INDEX_op_add_i32, { "r", "rZ", "rJZ" } },
+    { INDEX_op_add_i32, { "r", "rZ", "rJ" } },
     { INDEX_op_mul_i32, { "r", "rZ", "rZ" } },
     { INDEX_op_mulu2_i32, { "r", "r", "rZ", "rZ" } },
     { INDEX_op_div_i32, { "r", "rZ", "rZ" } },
     { INDEX_op_divu_i32, { "r", "rZ", "rZ" } },
     { INDEX_op_rem_i32, { "r", "rZ", "rZ" } },
     { INDEX_op_remu_i32, { "r", "rZ", "rZ" } },
-    { INDEX_op_sub_i32, { "r", "rZ", "rJZ" } },
+    { INDEX_op_sub_i32, { "r", "rZ", "rJ" } },
 
-    { INDEX_op_and_i32, { "r", "rZ", "rIZ" } },
+    { INDEX_op_and_i32, { "r", "rZ", "rI" } },
     { INDEX_op_nor_i32, { "r", "rZ", "rZ" } },
     { INDEX_op_not_i32, { "r", "rZ" } },
     { INDEX_op_or_i32, { "r", "rZ", "rIZ" } },
     { INDEX_op_xor_i32, { "r", "rZ", "rIZ" } },
 
-    { INDEX_op_shl_i32, { "r", "rZ", "riZ" } },
-    { INDEX_op_shr_i32, { "r", "rZ", "riZ" } },
-    { INDEX_op_sar_i32, { "r", "rZ", "riZ" } },
+    { INDEX_op_shl_i32, { "r", "rZ", "ri" } },
+    { INDEX_op_shr_i32, { "r", "rZ", "ri" } },
+    { INDEX_op_sar_i32, { "r", "rZ", "ri" } },
+    { INDEX_op_rotr_i32, { "r", "rZ", "ri" } },
+    { INDEX_op_rotl_i32, { "r", "rZ", "ri" } },
+
+    { INDEX_op_bswap16_i32, { "r", "r" } },
+    { INDEX_op_bswap32_i32, { "r", "r" } },
 
     { INDEX_op_ext8s_i32, { "r", "rZ" } },
     { INDEX_op_ext16s_i32, { "r", "rZ" } },
 
+    { INDEX_op_deposit_i32, { "r", "0", "rZ" } },
+
     { INDEX_op_brcond_i32, { "rZ", "rZ" } },
+    { INDEX_op_movcond_i32, { "r", "rZ", "rZ", "rZ", "0" } },
     { INDEX_op_setcond_i32, { "r", "rZ", "rZ" } },
     { INDEX_op_setcond2_i32, { "r", "rZ", "rZ", "rZ", "rZ" } },
 
-    { INDEX_op_add2_i32, { "r", "r", "rZ", "rZ", "rJZ", "rJZ" } },
-    { INDEX_op_sub2_i32, { "r", "r", "rZ", "rZ", "rJZ", "rJZ" } },
+    { INDEX_op_add2_i32, { "r", "r", "rZ", "rZ", "rJ", "rJ" } },
+    { INDEX_op_sub2_i32, { "r", "r", "rZ", "rZ", "rJ", "rJ" } },
     { INDEX_op_brcond2_i32, { "rZ", "rZ", "rZ", "rZ" } },
 
 #if TARGET_LONG_BITS == 32
@@ -1545,7 +1668,6 @@ static int tcg_target_callee_save_regs[] = {
     TCG_REG_S5,
     TCG_REG_S6,
     TCG_REG_S7,
-    TCG_REG_GP,
     TCG_REG_FP,
     TCG_REG_RA,       /* should be last for ABI compliance */
 };
@@ -1555,11 +1677,15 @@ static void tcg_target_qemu_prologue(TCGContext *s)
 {
     int i, frame_size;
 
-    /* reserve some stack space */
+    /* reserve some stack space, also for TCG temps. */
     frame_size = ARRAY_SIZE(tcg_target_callee_save_regs) * 4
-                 + TCG_STATIC_CALL_ARGS_SIZE;
+                 + TCG_STATIC_CALL_ARGS_SIZE
+                 + CPU_TEMP_BUF_NLONGS * sizeof(long);
     frame_size = (frame_size + TCG_TARGET_STACK_ALIGN - 1) &
                  ~(TCG_TARGET_STACK_ALIGN - 1);
+    tcg_set_frame(s, TCG_REG_SP, ARRAY_SIZE(tcg_target_callee_save_regs) * 4
+                  + TCG_STATIC_CALL_ARGS_SIZE,
+                  CPU_TEMP_BUF_NLONGS * sizeof(long));
 
     /* TB prologue */
     tcg_out_addi(s, TCG_REG_SP, -frame_size);
@@ -1611,8 +1737,7 @@ static void tcg_target_init(TCGContext *s)
     tcg_regset_set_reg(s->reserved_regs, TCG_REG_T0);   /* internal use */
     tcg_regset_set_reg(s->reserved_regs, TCG_REG_RA);   /* return address */
     tcg_regset_set_reg(s->reserved_regs, TCG_REG_SP);   /* stack pointer */
+    tcg_regset_set_reg(s->reserved_regs, TCG_REG_GP);   /* global pointer */
 
     tcg_add_target_add_op_defs(mips_op_defs);
-    tcg_set_frame(s, TCG_AREG0, offsetof(CPUArchState, temp_buf),
-                  CPU_TEMP_BUF_NLONGS * sizeof(long));
 }
diff --git a/tcg/mips/tcg-target.h b/tcg/mips/tcg-target.h
index d3c804d9a..65b5c59e8 100644
--- a/tcg/mips/tcg-target.h
+++ b/tcg/mips/tcg-target.h
@@ -80,28 +80,42 @@ typedef enum {
 #define TCG_TARGET_HAS_div_i32          1
 #define TCG_TARGET_HAS_not_i32          1
 #define TCG_TARGET_HAS_nor_i32          1
-#define TCG_TARGET_HAS_rot_i32          0
 #define TCG_TARGET_HAS_ext8s_i32        1
 #define TCG_TARGET_HAS_ext16s_i32       1
-#define TCG_TARGET_HAS_bswap32_i32      0
-#define TCG_TARGET_HAS_bswap16_i32      0
 #define TCG_TARGET_HAS_andc_i32         0
 #define TCG_TARGET_HAS_orc_i32          0
 #define TCG_TARGET_HAS_eqv_i32          0
 #define TCG_TARGET_HAS_nand_i32         0
+
+/* optional instructions only implemented on MIPS4, MIPS32 and Loongson 2 */
+#if (defined(__mips_isa_rev) && (__mips_isa_rev >= 1)) || \
+    defined(_MIPS_ARCH_LOONGSON2E) || defined(_MIPS_ARCH_LOONGSON2F) || \
+    defined(_MIPS_ARCH_MIPS4)
+#define TCG_TARGET_HAS_movcond_i32      1
+#else
+#define TCG_TARGET_HAS_movcond_i32      0
+#endif
+
+/* optional instructions only implemented on MIPS32R2 */
+#if defined(__mips_isa_rev) && (__mips_isa_rev >= 2)
+#define TCG_TARGET_HAS_bswap16_i32      1
+#define TCG_TARGET_HAS_bswap32_i32      1
+#define TCG_TARGET_HAS_rot_i32          1
+#define TCG_TARGET_HAS_deposit_i32      1
+#else
+#define TCG_TARGET_HAS_bswap16_i32      0
+#define TCG_TARGET_HAS_bswap32_i32      0
+#define TCG_TARGET_HAS_rot_i32          0
 #define TCG_TARGET_HAS_deposit_i32      0
+#endif
 
 /* optional instructions automatically implemented */
 #define TCG_TARGET_HAS_neg_i32          0 /* sub  rd, zero, rt   */
 #define TCG_TARGET_HAS_ext8u_i32        0 /* andi rt, rs, 0xff   */
 #define TCG_TARGET_HAS_ext16u_i32       0 /* andi rt, rs, 0xffff */
 
-/* Note: must be synced with dyngen-exec.h */
 #define TCG_AREG0 TCG_REG_S0
 
-/* guest base is supported */
-#define TCG_TARGET_HAS_GUEST_BASE
-
 #ifdef __OpenBSD__
 #include <machine/sysarch.h>
 #else
diff --git a/tcg/optimize.c b/tcg/optimize.c
index 9c65474a8..9109b813e 100644
--- a/tcg/optimize.c
+++ b/tcg/optimize.c
@@ -39,8 +39,6 @@ typedef enum {
     TCG_TEMP_UNDEF = 0,
     TCG_TEMP_CONST,
     TCG_TEMP_COPY,
-    TCG_TEMP_HAS_COPY,
-    TCG_TEMP_ANY
 } tcg_temp_state;
 
 struct tcg_temp_info {
@@ -52,39 +50,19 @@ struct tcg_temp_info {
 
 static struct tcg_temp_info temps[TCG_MAX_TEMPS];
 
-/* Reset TEMP's state to TCG_TEMP_ANY.  If TEMP was a representative of some
-   class of equivalent temp's, a new representative should be chosen in this
-   class. */
-static void reset_temp(TCGArg temp, int nb_temps, int nb_globals)
+/* Reset TEMP's state to TCG_TEMP_UNDEF.  If TEMP only had one copy, remove
+   the copy flag from the left temp.  */
+static void reset_temp(TCGArg temp)
 {
-    int i;
-    TCGArg new_base = (TCGArg)-1;
-    if (temps[temp].state == TCG_TEMP_HAS_COPY) {
-        for (i = temps[temp].next_copy; i != temp; i = temps[i].next_copy) {
-            if (i >= nb_globals) {
-                temps[i].state = TCG_TEMP_HAS_COPY;
-                new_base = i;
-                break;
-            }
+    if (temps[temp].state == TCG_TEMP_COPY) {
+        if (temps[temp].prev_copy == temps[temp].next_copy) {
+            temps[temps[temp].next_copy].state = TCG_TEMP_UNDEF;
+        } else {
+            temps[temps[temp].next_copy].prev_copy = temps[temp].prev_copy;
+            temps[temps[temp].prev_copy].next_copy = temps[temp].next_copy;
         }
-        for (i = temps[temp].next_copy; i != temp; i = temps[i].next_copy) {
-            if (new_base == (TCGArg)-1) {
-                temps[i].state = TCG_TEMP_ANY;
-            } else {
-                temps[i].val = new_base;
-            }
-        }
-        temps[temps[temp].next_copy].prev_copy = temps[temp].prev_copy;
-        temps[temps[temp].prev_copy].next_copy = temps[temp].next_copy;
-    } else if (temps[temp].state == TCG_TEMP_COPY) {
-        temps[temps[temp].next_copy].prev_copy = temps[temp].prev_copy;
-        temps[temps[temp].prev_copy].next_copy = temps[temp].next_copy;
-        new_base = temps[temp].val;
-    }
-    temps[temp].state = TCG_TEMP_ANY;
-    if (new_base != (TCGArg)-1 && temps[new_base].next_copy == new_base) {
-        temps[new_base].state = TCG_TEMP_ANY;
     }
+    temps[temp].state = TCG_TEMP_UNDEF;
 }
 
 static int op_bits(TCGOpcode op)
@@ -107,36 +85,83 @@ static TCGOpcode op_to_movi(TCGOpcode op)
     }
 }
 
-static void tcg_opt_gen_mov(TCGContext *s, TCGArg *gen_args, TCGArg dst,
-                            TCGArg src, int nb_temps, int nb_globals)
+static TCGArg find_better_copy(TCGContext *s, TCGArg temp)
+{
+    TCGArg i;
+
+    /* If this is already a global, we can't do better. */
+    if (temp < s->nb_globals) {
+        return temp;
+    }
+
+    /* Search for a global first. */
+    for (i = temps[temp].next_copy ; i != temp ; i = temps[i].next_copy) {
+        if (i < s->nb_globals) {
+            return i;
+        }
+    }
+
+    /* If it is a temp, search for a temp local. */
+    if (!s->temps[temp].temp_local) {
+        for (i = temps[temp].next_copy ; i != temp ; i = temps[i].next_copy) {
+            if (s->temps[i].temp_local) {
+                return i;
+            }
+        }
+    }
+
+    /* Failure to find a better representation, return the same temp. */
+    return temp;
+}
+
+static bool temps_are_copies(TCGArg arg1, TCGArg arg2)
+{
+    TCGArg i;
+
+    if (arg1 == arg2) {
+        return true;
+    }
+
+    if (temps[arg1].state != TCG_TEMP_COPY
+        || temps[arg2].state != TCG_TEMP_COPY) {
+        return false;
+    }
+
+    for (i = temps[arg1].next_copy ; i != arg1 ; i = temps[i].next_copy) {
+        if (i == arg2) {
+            return true;
+        }
+    }
+
+    return false;
+}
+
+static void tcg_opt_gen_mov(TCGContext *s, TCGArg *gen_args,
+                            TCGArg dst, TCGArg src)
 {
-        reset_temp(dst, nb_temps, nb_globals);
-        assert(temps[src].state != TCG_TEMP_COPY);
-        /* Don't try to copy if one of temps is a global or either one
-           is local and another is register */
-        if (src >= nb_globals && dst >= nb_globals &&
-            tcg_arg_is_local(s, src) == tcg_arg_is_local(s, dst)) {
-            assert(temps[src].state != TCG_TEMP_CONST);
-            if (temps[src].state != TCG_TEMP_HAS_COPY) {
-                temps[src].state = TCG_TEMP_HAS_COPY;
+        reset_temp(dst);
+        assert(temps[src].state != TCG_TEMP_CONST);
+
+        if (s->temps[src].type == s->temps[dst].type) {
+            if (temps[src].state != TCG_TEMP_COPY) {
+                temps[src].state = TCG_TEMP_COPY;
                 temps[src].next_copy = src;
                 temps[src].prev_copy = src;
             }
             temps[dst].state = TCG_TEMP_COPY;
-            temps[dst].val = src;
             temps[dst].next_copy = temps[src].next_copy;
             temps[dst].prev_copy = src;
             temps[temps[dst].next_copy].prev_copy = dst;
             temps[src].next_copy = dst;
         }
+
         gen_args[0] = dst;
         gen_args[1] = src;
 }
 
-static void tcg_opt_gen_movi(TCGArg *gen_args, TCGArg dst, TCGArg val,
-                             int nb_temps, int nb_globals)
+static void tcg_opt_gen_movi(TCGArg *gen_args, TCGArg dst, TCGArg val)
 {
-        reset_temp(dst, nb_temps, nb_globals);
+        reset_temp(dst);
         temps[dst].state = TCG_TEMP_CONST;
         temps[dst].val = val;
         gen_args[0] = dst;
@@ -267,6 +292,179 @@ static TCGArg do_constant_folding(TCGOpcode op, TCGArg x, TCGArg y)
     return res;
 }
 
+static bool do_constant_folding_cond_32(uint32_t x, uint32_t y, TCGCond c)
+{
+    switch (c) {
+    case TCG_COND_EQ:
+        return x == y;
+    case TCG_COND_NE:
+        return x != y;
+    case TCG_COND_LT:
+        return (int32_t)x < (int32_t)y;
+    case TCG_COND_GE:
+        return (int32_t)x >= (int32_t)y;
+    case TCG_COND_LE:
+        return (int32_t)x <= (int32_t)y;
+    case TCG_COND_GT:
+        return (int32_t)x > (int32_t)y;
+    case TCG_COND_LTU:
+        return x < y;
+    case TCG_COND_GEU:
+        return x >= y;
+    case TCG_COND_LEU:
+        return x <= y;
+    case TCG_COND_GTU:
+        return x > y;
+    default:
+        tcg_abort();
+    }
+}
+
+static bool do_constant_folding_cond_64(uint64_t x, uint64_t y, TCGCond c)
+{
+    switch (c) {
+    case TCG_COND_EQ:
+        return x == y;
+    case TCG_COND_NE:
+        return x != y;
+    case TCG_COND_LT:
+        return (int64_t)x < (int64_t)y;
+    case TCG_COND_GE:
+        return (int64_t)x >= (int64_t)y;
+    case TCG_COND_LE:
+        return (int64_t)x <= (int64_t)y;
+    case TCG_COND_GT:
+        return (int64_t)x > (int64_t)y;
+    case TCG_COND_LTU:
+        return x < y;
+    case TCG_COND_GEU:
+        return x >= y;
+    case TCG_COND_LEU:
+        return x <= y;
+    case TCG_COND_GTU:
+        return x > y;
+    default:
+        tcg_abort();
+    }
+}
+
+static bool do_constant_folding_cond_eq(TCGCond c)
+{
+    switch (c) {
+    case TCG_COND_GT:
+    case TCG_COND_LTU:
+    case TCG_COND_LT:
+    case TCG_COND_GTU:
+    case TCG_COND_NE:
+        return 0;
+    case TCG_COND_GE:
+    case TCG_COND_GEU:
+    case TCG_COND_LE:
+    case TCG_COND_LEU:
+    case TCG_COND_EQ:
+        return 1;
+    default:
+        tcg_abort();
+    }
+}
+
+/* Return 2 if the condition can't be simplified, and the result
+   of the condition (0 or 1) if it can */
+static TCGArg do_constant_folding_cond(TCGOpcode op, TCGArg x,
+                                       TCGArg y, TCGCond c)
+{
+    if (temps[x].state == TCG_TEMP_CONST && temps[y].state == TCG_TEMP_CONST) {
+        switch (op_bits(op)) {
+        case 32:
+            return do_constant_folding_cond_32(temps[x].val, temps[y].val, c);
+        case 64:
+            return do_constant_folding_cond_64(temps[x].val, temps[y].val, c);
+        default:
+            tcg_abort();
+        }
+    } else if (temps_are_copies(x, y)) {
+        return do_constant_folding_cond_eq(c);
+    } else if (temps[y].state == TCG_TEMP_CONST && temps[y].val == 0) {
+        switch (c) {
+        case TCG_COND_LTU:
+            return 0;
+        case TCG_COND_GEU:
+            return 1;
+        default:
+            return 2;
+        }
+    } else {
+        return 2;
+    }
+}
+
+/* Return 2 if the condition can't be simplified, and the result
+   of the condition (0 or 1) if it can */
+static TCGArg do_constant_folding_cond2(TCGArg *p1, TCGArg *p2, TCGCond c)
+{
+    TCGArg al = p1[0], ah = p1[1];
+    TCGArg bl = p2[0], bh = p2[1];
+
+    if (temps[bl].state == TCG_TEMP_CONST
+        && temps[bh].state == TCG_TEMP_CONST) {
+        uint64_t b = ((uint64_t)temps[bh].val << 32) | (uint32_t)temps[bl].val;
+
+        if (temps[al].state == TCG_TEMP_CONST
+            && temps[ah].state == TCG_TEMP_CONST) {
+            uint64_t a;
+            a = ((uint64_t)temps[ah].val << 32) | (uint32_t)temps[al].val;
+            return do_constant_folding_cond_64(a, b, c);
+        }
+        if (b == 0) {
+            switch (c) {
+            case TCG_COND_LTU:
+                return 0;
+            case TCG_COND_GEU:
+                return 1;
+            default:
+                break;
+            }
+        }
+    }
+    if (temps_are_copies(al, bl) && temps_are_copies(ah, bh)) {
+        return do_constant_folding_cond_eq(c);
+    }
+    return 2;
+}
+
+static bool swap_commutative(TCGArg dest, TCGArg *p1, TCGArg *p2)
+{
+    TCGArg a1 = *p1, a2 = *p2;
+    int sum = 0;
+    sum += temps[a1].state == TCG_TEMP_CONST;
+    sum -= temps[a2].state == TCG_TEMP_CONST;
+
+    /* Prefer the constant in second argument, and then the form
+       op a, a, b, which is better handled on non-RISC hosts. */
+    if (sum > 0 || (sum == 0 && dest == a2)) {
+        *p1 = a2;
+        *p2 = a1;
+        return true;
+    }
+    return false;
+}
+
+static bool swap_commutative2(TCGArg *p1, TCGArg *p2)
+{
+    int sum = 0;
+    sum += temps[p1[0]].state == TCG_TEMP_CONST;
+    sum += temps[p1[1]].state == TCG_TEMP_CONST;
+    sum -= temps[p2[0]].state == TCG_TEMP_CONST;
+    sum -= temps[p2[1]].state == TCG_TEMP_CONST;
+    if (sum > 0) {
+        TCGArg t;
+        t = p1[0], p1[0] = p2[0], p2[0] = t;
+        t = p1[1], p1[1] = p2[1], p2[1] = t;
+        return true;
+    }
+    return false;
+}
+
 /* Propagate constants and copies, fold constant expressions. */
 static TCGArg *tcg_constant_folding(TCGContext *s, uint16_t *tcg_opc_ptr,
                                     TCGArg *args, TCGOpDef *tcg_op_defs)
@@ -276,28 +474,34 @@ static TCGArg *tcg_constant_folding(TCGContext *s, uint16_t *tcg_opc_ptr,
     const TCGOpDef *def;
     TCGArg *gen_args;
     TCGArg tmp;
+
     /* Array VALS has an element for each temp.
        If this temp holds a constant then its value is kept in VALS' element.
-       If this temp is a copy of other ones then this equivalence class'
-       representative is kept in VALS' element.
-       If this temp is neither copy nor constant then corresponding VALS'
-       element is unused. */
+       If this temp is a copy of other ones then the other copies are
+       available through the doubly linked circular list. */
 
     nb_temps = s->nb_temps;
     nb_globals = s->nb_globals;
     memset(temps, 0, nb_temps * sizeof(struct tcg_temp_info));
 
-    nb_ops = tcg_opc_ptr - gen_opc_buf;
+    nb_ops = tcg_opc_ptr - s->gen_opc_buf;
     gen_args = args;
     for (op_index = 0; op_index < nb_ops; op_index++) {
-        op = gen_opc_buf[op_index];
+        op = s->gen_opc_buf[op_index];
         def = &tcg_op_defs[op];
         /* Do copy propagation */
-        if (!(def->flags & (TCG_OPF_CALL_CLOBBER | TCG_OPF_SIDE_EFFECTS))) {
-            assert(op != INDEX_op_call);
+        if (op == INDEX_op_call) {
+            int nb_oargs = args[0] >> 16;
+            int nb_iargs = args[0] & 0xffff;
+            for (i = nb_oargs + 1; i < nb_oargs + nb_iargs + 1; i++) {
+                if (temps[args[i]].state == TCG_TEMP_COPY) {
+                    args[i] = find_better_copy(s, args[i]);
+                }
+            }
+        } else {
             for (i = def->nb_oargs; i < def->nb_oargs + def->nb_iargs; i++) {
                 if (temps[args[i]].state == TCG_TEMP_COPY) {
-                    args[i] = temps[args[i]].val;
+                    args[i] = find_better_copy(s, args[i]);
                 }
             }
         }
@@ -312,17 +516,71 @@ static TCGArg *tcg_constant_folding(TCGContext *s, uint16_t *tcg_opc_ptr,
         CASE_OP_32_64(eqv):
         CASE_OP_32_64(nand):
         CASE_OP_32_64(nor):
-            if (temps[args[1]].state == TCG_TEMP_CONST) {
-                tmp = args[1];
-                args[1] = args[2];
-                args[2] = tmp;
+            swap_commutative(args[0], &args[1], &args[2]);
+            break;
+        CASE_OP_32_64(brcond):
+            if (swap_commutative(-1, &args[0], &args[1])) {
+                args[2] = tcg_swap_cond(args[2]);
+            }
+            break;
+        CASE_OP_32_64(setcond):
+            if (swap_commutative(args[0], &args[1], &args[2])) {
+                args[3] = tcg_swap_cond(args[3]);
+            }
+            break;
+        CASE_OP_32_64(movcond):
+            if (swap_commutative(-1, &args[1], &args[2])) {
+                args[5] = tcg_swap_cond(args[5]);
+            }
+            /* For movcond, we canonicalize the "false" input reg to match
+               the destination reg so that the tcg backend can implement
+               a "move if true" operation.  */
+            if (swap_commutative(args[0], &args[4], &args[3])) {
+                args[5] = tcg_invert_cond(args[5]);
+            }
+            break;
+        case INDEX_op_add2_i32:
+            swap_commutative(args[0], &args[2], &args[4]);
+            swap_commutative(args[1], &args[3], &args[5]);
+            break;
+        case INDEX_op_mulu2_i32:
+            swap_commutative(args[0], &args[2], &args[3]);
+            break;
+        case INDEX_op_brcond2_i32:
+            if (swap_commutative2(&args[0], &args[2])) {
+                args[4] = tcg_swap_cond(args[4]);
+            }
+            break;
+        case INDEX_op_setcond2_i32:
+            if (swap_commutative2(&args[1], &args[3])) {
+                args[5] = tcg_swap_cond(args[5]);
+            }
+            break;
+        default:
+            break;
+        }
+
+        /* Simplify expressions for "shift/rot r, 0, a => movi r, 0" */
+        switch (op) {
+        CASE_OP_32_64(shl):
+        CASE_OP_32_64(shr):
+        CASE_OP_32_64(sar):
+        CASE_OP_32_64(rotl):
+        CASE_OP_32_64(rotr):
+            if (temps[args[1]].state == TCG_TEMP_CONST
+                && temps[args[1]].val == 0) {
+                s->gen_opc_buf[op_index] = op_to_movi(op);
+                tcg_opt_gen_movi(gen_args, args[0], 0);
+                args += 3;
+                gen_args += 2;
+                continue;
             }
             break;
         default:
             break;
         }
 
-        /* Simplify expression if possible. */
+        /* Simplify expression for "op r, a, 0 => mov r, a" cases */
         switch (op) {
         CASE_OP_32_64(add):
         CASE_OP_32_64(sub):
@@ -331,50 +589,75 @@ static TCGArg *tcg_constant_folding(TCGContext *s, uint16_t *tcg_opc_ptr,
         CASE_OP_32_64(sar):
         CASE_OP_32_64(rotl):
         CASE_OP_32_64(rotr):
+        CASE_OP_32_64(or):
+        CASE_OP_32_64(xor):
             if (temps[args[1]].state == TCG_TEMP_CONST) {
                 /* Proceed with possible constant folding. */
                 break;
             }
             if (temps[args[2]].state == TCG_TEMP_CONST
                 && temps[args[2]].val == 0) {
-                if ((temps[args[0]].state == TCG_TEMP_COPY
-                    && temps[args[0]].val == args[1])
-                    || args[0] == args[1]) {
-                    args += 3;
-                    gen_opc_buf[op_index] = INDEX_op_nop;
+                if (temps_are_copies(args[0], args[1])) {
+                    s->gen_opc_buf[op_index] = INDEX_op_nop;
                 } else {
-                    gen_opc_buf[op_index] = op_to_mov(op);
-                    tcg_opt_gen_mov(s, gen_args, args[0], args[1],
-                                    nb_temps, nb_globals);
+                    s->gen_opc_buf[op_index] = op_to_mov(op);
+                    tcg_opt_gen_mov(s, gen_args, args[0], args[1]);
                     gen_args += 2;
-                    args += 3;
                 }
+                args += 3;
                 continue;
             }
             break;
+        default:
+            break;
+        }
+
+        /* Simplify expression for "op r, a, 0 => movi r, 0" cases */
+        switch (op) {
+        CASE_OP_32_64(and):
         CASE_OP_32_64(mul):
             if ((temps[args[2]].state == TCG_TEMP_CONST
                 && temps[args[2]].val == 0)) {
-                gen_opc_buf[op_index] = op_to_movi(op);
-                tcg_opt_gen_movi(gen_args, args[0], 0, nb_temps, nb_globals);
+                s->gen_opc_buf[op_index] = op_to_movi(op);
+                tcg_opt_gen_movi(gen_args, args[0], 0);
                 args += 3;
                 gen_args += 2;
                 continue;
             }
             break;
+        default:
+            break;
+        }
+
+        /* Simplify expression for "op r, a, a => mov r, a" cases */
+        switch (op) {
         CASE_OP_32_64(or):
         CASE_OP_32_64(and):
-            if (args[1] == args[2]) {
-                if (args[1] == args[0]) {
-                    args += 3;
-                    gen_opc_buf[op_index] = INDEX_op_nop;
+            if (temps_are_copies(args[1], args[2])) {
+                if (temps_are_copies(args[0], args[1])) {
+                    s->gen_opc_buf[op_index] = INDEX_op_nop;
                 } else {
-                    gen_opc_buf[op_index] = op_to_mov(op);
-                    tcg_opt_gen_mov(s, gen_args, args[0], args[1], nb_temps,
-                                    nb_globals);
+                    s->gen_opc_buf[op_index] = op_to_mov(op);
+                    tcg_opt_gen_mov(s, gen_args, args[0], args[1]);
                     gen_args += 2;
-                    args += 3;
                 }
+                args += 3;
+                continue;
+            }
+            break;
+        default:
+            break;
+        }
+
+        /* Simplify expression for "op r, a, a => movi r, 0" cases */
+        switch (op) {
+        CASE_OP_32_64(sub):
+        CASE_OP_32_64(xor):
+            if (temps_are_copies(args[1], args[2])) {
+                s->gen_opc_buf[op_index] = op_to_movi(op);
+                tcg_opt_gen_movi(gen_args, args[0], 0);
+                gen_args += 2;
+                args += 3;
                 continue;
             }
             break;
@@ -387,16 +670,13 @@ static TCGArg *tcg_constant_folding(TCGContext *s, uint16_t *tcg_opc_ptr,
            allocator where needed and possible.  Also detect copies. */
         switch (op) {
         CASE_OP_32_64(mov):
-            if ((temps[args[1]].state == TCG_TEMP_COPY
-                && temps[args[1]].val == args[0])
-                || args[0] == args[1]) {
+            if (temps_are_copies(args[0], args[1])) {
                 args += 2;
-                gen_opc_buf[op_index] = INDEX_op_nop;
+                s->gen_opc_buf[op_index] = INDEX_op_nop;
                 break;
             }
             if (temps[args[1]].state != TCG_TEMP_CONST) {
-                tcg_opt_gen_mov(s, gen_args, args[0], args[1],
-                                nb_temps, nb_globals);
+                tcg_opt_gen_mov(s, gen_args, args[0], args[1]);
                 gen_args += 2;
                 args += 2;
                 break;
@@ -404,14 +684,15 @@ static TCGArg *tcg_constant_folding(TCGContext *s, uint16_t *tcg_opc_ptr,
             /* Source argument is constant.  Rewrite the operation and
                let movi case handle it. */
             op = op_to_movi(op);
-            gen_opc_buf[op_index] = op;
+            s->gen_opc_buf[op_index] = op;
             args[1] = temps[args[1]].val;
             /* fallthrough */
         CASE_OP_32_64(movi):
-            tcg_opt_gen_movi(gen_args, args[0], args[1], nb_temps, nb_globals);
+            tcg_opt_gen_movi(gen_args, args[0], args[1]);
             gen_args += 2;
             args += 2;
             break;
+
         CASE_OP_32_64(not):
         CASE_OP_32_64(neg):
         CASE_OP_32_64(ext8s):
@@ -421,20 +702,15 @@ static TCGArg *tcg_constant_folding(TCGContext *s, uint16_t *tcg_opc_ptr,
         case INDEX_op_ext32s_i64:
         case INDEX_op_ext32u_i64:
             if (temps[args[1]].state == TCG_TEMP_CONST) {
-                gen_opc_buf[op_index] = op_to_movi(op);
+                s->gen_opc_buf[op_index] = op_to_movi(op);
                 tmp = do_constant_folding(op, temps[args[1]].val, 0);
-                tcg_opt_gen_movi(gen_args, args[0], tmp, nb_temps, nb_globals);
-                gen_args += 2;
-                args += 2;
-                break;
-            } else {
-                reset_temp(args[0], nb_temps, nb_globals);
-                gen_args[0] = args[0];
-                gen_args[1] = args[1];
+                tcg_opt_gen_movi(gen_args, args[0], tmp);
                 gen_args += 2;
                 args += 2;
                 break;
             }
+            goto do_default;
+
         CASE_OP_32_64(add):
         CASE_OP_32_64(sub):
         CASE_OP_32_64(mul):
@@ -453,31 +729,200 @@ static TCGArg *tcg_constant_folding(TCGContext *s, uint16_t *tcg_opc_ptr,
         CASE_OP_32_64(nor):
             if (temps[args[1]].state == TCG_TEMP_CONST
                 && temps[args[2]].state == TCG_TEMP_CONST) {
-                gen_opc_buf[op_index] = op_to_movi(op);
+                s->gen_opc_buf[op_index] = op_to_movi(op);
                 tmp = do_constant_folding(op, temps[args[1]].val,
                                           temps[args[2]].val);
-                tcg_opt_gen_movi(gen_args, args[0], tmp, nb_temps, nb_globals);
+                tcg_opt_gen_movi(gen_args, args[0], tmp);
                 gen_args += 2;
                 args += 3;
                 break;
+            }
+            goto do_default;
+
+        CASE_OP_32_64(deposit):
+            if (temps[args[1]].state == TCG_TEMP_CONST
+                && temps[args[2]].state == TCG_TEMP_CONST) {
+                s->gen_opc_buf[op_index] = op_to_movi(op);
+                tmp = ((1ull << args[4]) - 1);
+                tmp = (temps[args[1]].val & ~(tmp << args[3]))
+                      | ((temps[args[2]].val & tmp) << args[3]);
+                tcg_opt_gen_movi(gen_args, args[0], tmp);
+                gen_args += 2;
+                args += 5;
+                break;
+            }
+            goto do_default;
+
+        CASE_OP_32_64(setcond):
+            tmp = do_constant_folding_cond(op, args[1], args[2], args[3]);
+            if (tmp != 2) {
+                s->gen_opc_buf[op_index] = op_to_movi(op);
+                tcg_opt_gen_movi(gen_args, args[0], tmp);
+                gen_args += 2;
+                args += 4;
+                break;
+            }
+            goto do_default;
+
+        CASE_OP_32_64(brcond):
+            tmp = do_constant_folding_cond(op, args[0], args[1], args[2]);
+            if (tmp != 2) {
+                if (tmp) {
+                    memset(temps, 0, nb_temps * sizeof(struct tcg_temp_info));
+                    s->gen_opc_buf[op_index] = INDEX_op_br;
+                    gen_args[0] = args[3];
+                    gen_args += 1;
+                } else {
+                    s->gen_opc_buf[op_index] = INDEX_op_nop;
+                }
+                args += 4;
+                break;
+            }
+            goto do_default;
+
+        CASE_OP_32_64(movcond):
+            tmp = do_constant_folding_cond(op, args[1], args[2], args[5]);
+            if (tmp != 2) {
+                if (temps_are_copies(args[0], args[4-tmp])) {
+                    s->gen_opc_buf[op_index] = INDEX_op_nop;
+                } else if (temps[args[4-tmp]].state == TCG_TEMP_CONST) {
+                    s->gen_opc_buf[op_index] = op_to_movi(op);
+                    tcg_opt_gen_movi(gen_args, args[0], temps[args[4-tmp]].val);
+                    gen_args += 2;
+                } else {
+                    s->gen_opc_buf[op_index] = op_to_mov(op);
+                    tcg_opt_gen_mov(s, gen_args, args[0], args[4-tmp]);
+                    gen_args += 2;
+                }
+                args += 6;
+                break;
+            }
+            goto do_default;
+
+        case INDEX_op_add2_i32:
+        case INDEX_op_sub2_i32:
+            if (temps[args[2]].state == TCG_TEMP_CONST
+                && temps[args[3]].state == TCG_TEMP_CONST
+                && temps[args[4]].state == TCG_TEMP_CONST
+                && temps[args[5]].state == TCG_TEMP_CONST) {
+                uint32_t al = temps[args[2]].val;
+                uint32_t ah = temps[args[3]].val;
+                uint32_t bl = temps[args[4]].val;
+                uint32_t bh = temps[args[5]].val;
+                uint64_t a = ((uint64_t)ah << 32) | al;
+                uint64_t b = ((uint64_t)bh << 32) | bl;
+                TCGArg rl, rh;
+
+                if (op == INDEX_op_add2_i32) {
+                    a += b;
+                } else {
+                    a -= b;
+                }
+
+                /* We emit the extra nop when we emit the add2/sub2.  */
+                assert(s->gen_opc_buf[op_index + 1] == INDEX_op_nop);
+
+                rl = args[0];
+                rh = args[1];
+                s->gen_opc_buf[op_index] = INDEX_op_movi_i32;
+                s->gen_opc_buf[++op_index] = INDEX_op_movi_i32;
+                tcg_opt_gen_movi(&gen_args[0], rl, (uint32_t)a);
+                tcg_opt_gen_movi(&gen_args[2], rh, (uint32_t)(a >> 32));
+                gen_args += 4;
+                args += 6;
+                break;
+            }
+            goto do_default;
+
+        case INDEX_op_mulu2_i32:
+            if (temps[args[2]].state == TCG_TEMP_CONST
+                && temps[args[3]].state == TCG_TEMP_CONST) {
+                uint32_t a = temps[args[2]].val;
+                uint32_t b = temps[args[3]].val;
+                uint64_t r = (uint64_t)a * b;
+                TCGArg rl, rh;
+
+                /* We emit the extra nop when we emit the mulu2.  */
+                assert(s->gen_opc_buf[op_index + 1] == INDEX_op_nop);
+
+                rl = args[0];
+                rh = args[1];
+                s->gen_opc_buf[op_index] = INDEX_op_movi_i32;
+                s->gen_opc_buf[++op_index] = INDEX_op_movi_i32;
+                tcg_opt_gen_movi(&gen_args[0], rl, (uint32_t)r);
+                tcg_opt_gen_movi(&gen_args[2], rh, (uint32_t)(r >> 32));
+                gen_args += 4;
+                args += 4;
+                break;
+            }
+            goto do_default;
+
+        case INDEX_op_brcond2_i32:
+            tmp = do_constant_folding_cond2(&args[0], &args[2], args[4]);
+            if (tmp != 2) {
+                if (tmp) {
+                    memset(temps, 0, nb_temps * sizeof(struct tcg_temp_info));
+                    s->gen_opc_buf[op_index] = INDEX_op_br;
+                    gen_args[0] = args[5];
+                    gen_args += 1;
+                } else {
+                    s->gen_opc_buf[op_index] = INDEX_op_nop;
+                }
+            } else if ((args[4] == TCG_COND_LT || args[4] == TCG_COND_GE)
+                       && temps[args[2]].state == TCG_TEMP_CONST
+                       && temps[args[3]].state == TCG_TEMP_CONST
+                       && temps[args[2]].val == 0
+                       && temps[args[3]].val == 0) {
+                /* Simplify LT/GE comparisons vs zero to a single compare
+                   vs the high word of the input.  */
+                memset(temps, 0, nb_temps * sizeof(struct tcg_temp_info));
+                s->gen_opc_buf[op_index] = INDEX_op_brcond_i32;
+                gen_args[0] = args[1];
+                gen_args[1] = args[3];
+                gen_args[2] = args[4];
+                gen_args[3] = args[5];
+                gen_args += 4;
             } else {
-                reset_temp(args[0], nb_temps, nb_globals);
+                goto do_default;
+            }
+            args += 6;
+            break;
+
+        case INDEX_op_setcond2_i32:
+            tmp = do_constant_folding_cond2(&args[1], &args[3], args[5]);
+            if (tmp != 2) {
+                s->gen_opc_buf[op_index] = INDEX_op_movi_i32;
+                tcg_opt_gen_movi(gen_args, args[0], tmp);
+                gen_args += 2;
+            } else if ((args[5] == TCG_COND_LT || args[5] == TCG_COND_GE)
+                       && temps[args[3]].state == TCG_TEMP_CONST
+                       && temps[args[4]].state == TCG_TEMP_CONST
+                       && temps[args[3]].val == 0
+                       && temps[args[4]].val == 0) {
+                /* Simplify LT/GE comparisons vs zero to a single compare
+                   vs the high word of the input.  */
+                s->gen_opc_buf[op_index] = INDEX_op_setcond_i32;
                 gen_args[0] = args[0];
-                gen_args[1] = args[1];
-                gen_args[2] = args[2];
-                gen_args += 3;
-                args += 3;
-                break;
+                gen_args[1] = args[2];
+                gen_args[2] = args[4];
+                gen_args[3] = args[5];
+                gen_args += 4;
+            } else {
+                goto do_default;
             }
+            args += 6;
+            break;
+
         case INDEX_op_call:
             nb_call_args = (args[0] >> 16) + (args[0] & 0xffff);
-            if (!(args[nb_call_args + 1] & (TCG_CALL_CONST | TCG_CALL_PURE))) {
+            if (!(args[nb_call_args + 1] & (TCG_CALL_NO_READ_GLOBALS |
+                                            TCG_CALL_NO_WRITE_GLOBALS))) {
                 for (i = 0; i < nb_globals; i++) {
-                    reset_temp(i, nb_temps, nb_globals);
+                    reset_temp(i);
                 }
             }
             for (i = 0; i < (args[0] >> 16); i++) {
-                reset_temp(args[i + 1], nb_temps, nb_globals);
+                reset_temp(args[i + 1]);
             }
             i = nb_call_args + 3;
             while (i) {
@@ -487,22 +932,19 @@ static TCGArg *tcg_constant_folding(TCGContext *s, uint16_t *tcg_opc_ptr,
                 i--;
             }
             break;
-        case INDEX_op_set_label:
-        case INDEX_op_jmp:
-        case INDEX_op_br:
-        CASE_OP_32_64(brcond):
-            memset(temps, 0, nb_temps * sizeof(struct tcg_temp_info));
-            for (i = 0; i < def->nb_args; i++) {
-                *gen_args = *args;
-                args++;
-                gen_args++;
-            }
-            break;
+
         default:
-            /* Default case: we do know nothing about operation so no
-               propagation is done.  We only trash output args.  */
-            for (i = 0; i < def->nb_oargs; i++) {
-                reset_temp(args[i], nb_temps, nb_globals);
+        do_default:
+            /* Default case: we know nothing about operation (or were unable
+               to compute the operation result) so no propagation is done.
+               We trash everything if the operation is the end of a basic
+               block, otherwise we only trash the output args.  */
+            if (def->flags & TCG_OPF_BB_END) {
+                memset(temps, 0, nb_temps * sizeof(struct tcg_temp_info));
+            } else {
+                for (i = 0; i < def->nb_oargs; i++) {
+                    reset_temp(args[i]);
+                }
             }
             for (i = 0; i < def->nb_args; i++) {
                 gen_args[i] = args[i];
diff --git a/tcg/ppc/tcg-target.c b/tcg/ppc/tcg-target.c
index 0cff18125..d72d39627 100644
--- a/tcg/ppc/tcg-target.c
+++ b/tcg/ppc/tcg-target.c
@@ -39,8 +39,6 @@ static uint8_t *tb_ret_addr;
 #define LR_OFFSET 4
 #endif
 
-#define FAST_PATH
-
 #ifndef GUEST_BASE
 #define GUEST_BASE 0
 #endif
@@ -221,12 +219,6 @@ static void patch_reloc(uint8_t *code_ptr, int type,
     }
 }
 
-/* maximum number of register used for input function arguments */
-static int tcg_target_get_call_iarg_regs_count(int flags)
-{
-    return ARRAY_SIZE (tcg_target_call_iarg_regs);
-}
-
 /* parse target specific constraints */
 static int target_parse_constraint(TCGArgConstraint *ct, const char **pct_str)
 {
@@ -248,7 +240,6 @@ static int target_parse_constraint(TCGArgConstraint *ct, const char **pct_str)
         tcg_regset_set32(ct->u.regs, 0, 0xffffffff);
         tcg_regset_reset_reg(ct->u.regs, TCG_REG_R3);
         tcg_regset_reset_reg(ct->u.regs, TCG_REG_R4);
-#ifdef CONFIG_TCG_PASS_AREG0
         tcg_regset_reset_reg(ct->u.regs, TCG_REG_R5);
 #if TARGET_LONG_BITS == 64
         tcg_regset_reset_reg(ct->u.regs, TCG_REG_R6);
@@ -256,11 +247,6 @@ static int target_parse_constraint(TCGArgConstraint *ct, const char **pct_str)
         tcg_regset_reset_reg(ct->u.regs, TCG_REG_R7);
 #endif
 #endif
-#else /* !AREG0 */
-#if TARGET_LONG_BITS == 64
-        tcg_regset_reset_reg(ct->u.regs, TCG_REG_R5);
-#endif
-#endif
         break;
     case 'K':                   /* qemu_st[8..32] constraint */
         ct->ct |= TCG_CT_REG;
@@ -268,7 +254,6 @@ static int target_parse_constraint(TCGArgConstraint *ct, const char **pct_str)
         tcg_regset_reset_reg(ct->u.regs, TCG_REG_R3);
         tcg_regset_reset_reg(ct->u.regs, TCG_REG_R4);
         tcg_regset_reset_reg(ct->u.regs, TCG_REG_R5);
-#ifdef CONFIG_TCG_PASS_AREG0
         tcg_regset_reset_reg(ct->u.regs, TCG_REG_R6);
 #if TARGET_LONG_BITS == 64
         tcg_regset_reset_reg(ct->u.regs, TCG_REG_R7);
@@ -276,11 +261,6 @@ static int target_parse_constraint(TCGArgConstraint *ct, const char **pct_str)
         tcg_regset_reset_reg(ct->u.regs, TCG_REG_R8);
 #endif
 #endif
-#else /* !AREG0 */
-#if TARGET_LONG_BITS == 64
-        tcg_regset_reset_reg(ct->u.regs, TCG_REG_R6);
-#endif
-#endif
         break;
     case 'M':                   /* qemu_st64 constraint */
         ct->ct |= TCG_CT_REG;
@@ -290,12 +270,10 @@ static int target_parse_constraint(TCGArgConstraint *ct, const char **pct_str)
         tcg_regset_reset_reg(ct->u.regs, TCG_REG_R5);
         tcg_regset_reset_reg(ct->u.regs, TCG_REG_R6);
         tcg_regset_reset_reg(ct->u.regs, TCG_REG_R7);
-#if defined(CONFIG_TCG_PASS_AREG0)
         tcg_regset_reset_reg(ct->u.regs, TCG_REG_R8);
 #ifdef TCG_TARGET_CALL_ALIGN_ARGS
         tcg_regset_reset_reg(ct->u.regs, TCG_REG_R9);
 #endif
-#endif
         break;
 #else
     case 'L':
@@ -404,6 +382,7 @@ static int tcg_target_const_match(tcg_target_long val,
 #define ORC    XO31(412)
 #define EQV    XO31(284)
 #define NAND   XO31(476)
+#define ISEL   XO31( 15)
 
 #define LBZX   XO31( 87)
 #define LHZX   XO31(279)
@@ -456,7 +435,7 @@ enum {
     CR_SO
 };
 
-static const uint32_t tcg_to_bc[10] = {
+static const uint32_t tcg_to_bc[] = {
     [TCG_COND_EQ]  = BC | BI (7, CR_EQ) | BO_COND_TRUE,
     [TCG_COND_NE]  = BC | BI (7, CR_EQ) | BO_COND_FALSE,
     [TCG_COND_LT]  = BC | BI (7, CR_LT) | BO_COND_TRUE,
@@ -539,9 +518,39 @@ static void tcg_out_call (TCGContext *s, tcg_target_long arg, int const_arg)
 
 #if defined(CONFIG_SOFTMMU)
 
+static void add_qemu_ldst_label (TCGContext *s,
+                                 int is_ld,
+                                 int opc,
+                                 int data_reg,
+                                 int data_reg2,
+                                 int addrlo_reg,
+                                 int addrhi_reg,
+                                 int mem_index,
+                                 uint8_t *raddr,
+                                 uint8_t *label_ptr)
+{
+    int idx;
+    TCGLabelQemuLdst *label;
+
+    if (s->nb_qemu_ldst_labels >= TCG_MAX_QEMU_LDST) {
+        tcg_abort();
+    }
+
+    idx = s->nb_qemu_ldst_labels++;
+    label = (TCGLabelQemuLdst *)&s->qemu_ldst_labels[idx];
+    label->is_ld = is_ld;
+    label->opc = opc;
+    label->datalo_reg = data_reg;
+    label->datahi_reg = data_reg2;
+    label->addrlo_reg = addrlo_reg;
+    label->addrhi_reg = addrhi_reg;
+    label->mem_index = mem_index;
+    label->raddr = raddr;
+    label->label_ptr[0] = label_ptr;
+}
+
 #include "../../softmmu_defs.h"
 
-#ifdef CONFIG_TCG_PASS_AREG0
 /* helper signature: helper_ld_mmu(CPUState *env, target_ulong addr,
    int mmu_idx) */
 static const void * const qemu_ld_helpers[4] = {
@@ -559,55 +568,15 @@ static const void * const qemu_st_helpers[4] = {
     helper_stl_mmu,
     helper_stq_mmu,
 };
-#else
-/* legacy helper signature: __ld_mmu(target_ulong addr, int
-   mmu_idx) */
-static void *qemu_ld_helpers[4] = {
-    __ldb_mmu,
-    __ldw_mmu,
-    __ldl_mmu,
-    __ldq_mmu,
-};
 
-/* legacy helper signature: __ld_mmu(target_ulong addr, int
-   mmu_idx) */
-static void *qemu_st_helpers[4] = {
-    __stb_mmu,
-    __stw_mmu,
-    __stl_mmu,
-    __stq_mmu,
-};
-#endif
-#endif
+static void *ld_trampolines[4];
+static void *st_trampolines[4];
 
-static void tcg_out_qemu_ld (TCGContext *s, const TCGArg *args, int opc)
+static void tcg_out_tlb_check (TCGContext *s, int r0, int r1, int r2,
+                               int addr_reg, int addr_reg2, int s_bits,
+                               int offset1, int offset2, uint8_t **label_ptr)
 {
-    int addr_reg, data_reg, data_reg2, r0, r1, rbase, bswap;
-#ifdef CONFIG_SOFTMMU
-    int mem_index, s_bits, r2, ir;
-    void *label1_ptr, *label2_ptr;
-#if TARGET_LONG_BITS == 64
-    int addr_reg2;
-#endif
-#endif
-
-    data_reg = *args++;
-    if (opc == 3)
-        data_reg2 = *args++;
-    else
-        data_reg2 = 0;
-    addr_reg = *args++;
-
-#ifdef CONFIG_SOFTMMU
-#if TARGET_LONG_BITS == 64
-    addr_reg2 = *args++;
-#endif
-    mem_index = *args;
-    s_bits = opc & 3;
-    r0 = 3;
-    r1 = 4;
-    r2 = 0;
-    rbase = 0;
+    uint16_t retranst;
 
     tcg_out32 (s, (RLWINM
                    | RA (r0)
@@ -621,7 +590,7 @@ static void tcg_out_qemu_ld (TCGContext *s, const TCGArg *args, int opc)
     tcg_out32 (s, (LWZU
                    | RT (r1)
                    | RA (r0)
-                   | offsetof (CPUArchState, tlb_table[mem_index][0].addr_read)
+                   | offset1
                    )
         );
     tcg_out32 (s, (RLWINM
@@ -639,79 +608,58 @@ static void tcg_out_qemu_ld (TCGContext *s, const TCGArg *args, int opc)
     tcg_out32 (s, CMP | BF (6) | RA (addr_reg2) | RB (r1));
     tcg_out32 (s, CRAND | BT (7, CR_EQ) | BA (6, CR_EQ) | BB (7, CR_EQ));
 #endif
+    *label_ptr = s->code_ptr;
+    retranst = ((uint16_t *) s->code_ptr)[1] & ~3;
+    tcg_out32 (s, BC | BI (7, CR_EQ) | retranst | BO_COND_FALSE);
 
-    label1_ptr = s->code_ptr;
-#ifdef FAST_PATH
-    tcg_out32 (s, BC | BI (7, CR_EQ) | BO_COND_TRUE);
-#endif
-
-    /* slow path */
-    ir = 3;
-#ifdef CONFIG_TCG_PASS_AREG0
-    tcg_out_mov (s, TCG_TYPE_I32, ir++, TCG_AREG0);
-#endif
-#if TARGET_LONG_BITS == 32
-    tcg_out_mov (s, TCG_TYPE_I32, ir++, addr_reg);
-#else
-#ifdef TCG_TARGET_CALL_ALIGN_ARGS
-    ir |= 1;
-#endif
-    tcg_out_mov (s, TCG_TYPE_I32, ir++, addr_reg2);
-    tcg_out_mov (s, TCG_TYPE_I32, ir++, addr_reg);
-#endif
-    tcg_out_movi (s, TCG_TYPE_I32, ir, mem_index);
-
-    tcg_out_call (s, (tcg_target_long) qemu_ld_helpers[s_bits], 1);
-    switch (opc) {
-    case 0|4:
-        tcg_out32 (s, EXTSB | RA (data_reg) | RS (3));
-        break;
-    case 1|4:
-        tcg_out32 (s, EXTSH | RA (data_reg) | RS (3));
-        break;
-    case 0:
-    case 1:
-    case 2:
-        if (data_reg != 3)
-            tcg_out_mov (s, TCG_TYPE_I32, data_reg, 3);
-        break;
-    case 3:
-        if (data_reg == 3) {
-            if (data_reg2 == 4) {
-                tcg_out_mov (s, TCG_TYPE_I32, 0, 4);
-                tcg_out_mov (s, TCG_TYPE_I32, 4, 3);
-                tcg_out_mov (s, TCG_TYPE_I32, 3, 0);
-            }
-            else {
-                tcg_out_mov (s, TCG_TYPE_I32, data_reg2, 3);
-                tcg_out_mov (s, TCG_TYPE_I32, 3, 4);
-            }
-        }
-        else {
-            if (data_reg != 4) tcg_out_mov (s, TCG_TYPE_I32, data_reg, 4);
-            if (data_reg2 != 3) tcg_out_mov (s, TCG_TYPE_I32, data_reg2, 3);
-        }
-        break;
-    }
-    label2_ptr = s->code_ptr;
-    tcg_out32 (s, B);
-
-    /* label1: fast path */
-#ifdef FAST_PATH
-    reloc_pc14 (label1_ptr, (tcg_target_long) s->code_ptr);
-#endif
-
-    /* r0 now contains &env->tlb_table[mem_index][index].addr_read */
+    /* r0 now contains &env->tlb_table[mem_index][index].addr_x */
     tcg_out32 (s, (LWZ
                    | RT (r0)
                    | RA (r0)
-                   | (offsetof (CPUTLBEntry, addend)
-                      - offsetof (CPUTLBEntry, addr_read))
-                   ));
+                   | offset2
+                   )
+        );
     /* r0 = env->tlb_table[mem_index][index].addend */
     tcg_out32 (s, ADD | RT (r0) | RA (r0) | RB (addr_reg));
     /* r0 = env->tlb_table[mem_index][index].addend + addr */
 
+}
+#endif
+
+static void tcg_out_qemu_ld (TCGContext *s, const TCGArg *args, int opc)
+{
+    int addr_reg, data_reg, data_reg2, r0, r1, rbase, bswap;
+#ifdef CONFIG_SOFTMMU
+    int mem_index, s_bits, r2, addr_reg2;
+    uint8_t *label_ptr;
+#endif
+
+    data_reg = *args++;
+    if (opc == 3)
+        data_reg2 = *args++;
+    else
+        data_reg2 = 0;
+    addr_reg = *args++;
+
+#ifdef CONFIG_SOFTMMU
+#if TARGET_LONG_BITS == 64
+    addr_reg2 = *args++;
+#else
+    addr_reg2 = 0;
+#endif
+    mem_index = *args;
+    s_bits = opc & 3;
+    r0 = 3;
+    r1 = 4;
+    r2 = 0;
+    rbase = 0;
+
+    tcg_out_tlb_check (
+        s, r0, r1, r2, addr_reg, addr_reg2, s_bits,
+        offsetof (CPUArchState, tlb_table[mem_index][0].addr_read),
+        offsetof (CPUTLBEntry, addend) - offsetof (CPUTLBEntry, addr_read),
+        &label_ptr
+        );
 #else  /* !CONFIG_SOFTMMU */
     r0 = addr_reg;
     r1 = 3;
@@ -777,9 +725,17 @@ static void tcg_out_qemu_ld (TCGContext *s, const TCGArg *args, int opc)
         }
         break;
     }
-
 #ifdef CONFIG_SOFTMMU
-    reloc_pc24 (label2_ptr, (tcg_target_long) s->code_ptr);
+    add_qemu_ldst_label (s,
+                         1,
+                         opc,
+                         data_reg,
+                         data_reg2,
+                         addr_reg,
+                         addr_reg2,
+                         mem_index,
+                         s->code_ptr,
+                         label_ptr);
 #endif
 }
 
@@ -787,11 +743,8 @@ static void tcg_out_qemu_st (TCGContext *s, const TCGArg *args, int opc)
 {
     int addr_reg, r0, r1, data_reg, data_reg2, bswap, rbase;
 #ifdef CONFIG_SOFTMMU
-    int mem_index, r2, ir;
-    void *label1_ptr, *label2_ptr;
-#if TARGET_LONG_BITS == 64
-    int addr_reg2;
-#endif
+    int mem_index, r2, addr_reg2;
+    uint8_t *label_ptr;
 #endif
 
     data_reg = *args++;
@@ -804,6 +757,8 @@ static void tcg_out_qemu_st (TCGContext *s, const TCGArg *args, int opc)
 #ifdef CONFIG_SOFTMMU
 #if TARGET_LONG_BITS == 64
     addr_reg2 = *args++;
+#else
+    addr_reg2 = 0;
 #endif
     mem_index = *args;
     r0 = 3;
@@ -811,54 +766,162 @@ static void tcg_out_qemu_st (TCGContext *s, const TCGArg *args, int opc)
     r2 = 0;
     rbase = 0;
 
-    tcg_out32 (s, (RLWINM
-                   | RA (r0)
-                   | RS (addr_reg)
-                   | SH (32 - (TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS))
-                   | MB (32 - (CPU_TLB_ENTRY_BITS + CPU_TLB_BITS))
-                   | ME (31 - CPU_TLB_ENTRY_BITS)
-                   )
-        );
-    tcg_out32 (s, ADD | RT (r0) | RA (r0) | RB (TCG_AREG0));
-    tcg_out32 (s, (LWZU
-                   | RT (r1)
-                   | RA (r0)
-                   | offsetof (CPUArchState, tlb_table[mem_index][0].addr_write)
-                   )
-        );
-    tcg_out32 (s, (RLWINM
-                   | RA (r2)
-                   | RS (addr_reg)
-                   | SH (0)
-                   | MB ((32 - opc) & 31)
-                   | ME (31 - TARGET_PAGE_BITS)
-                   )
+    tcg_out_tlb_check (
+        s, r0, r1, r2, addr_reg, addr_reg2, opc & 3,
+        offsetof (CPUArchState, tlb_table[mem_index][0].addr_write),
+        offsetof (CPUTLBEntry, addend) - offsetof (CPUTLBEntry, addr_write),
+        &label_ptr
         );
+#else  /* !CONFIG_SOFTMMU */
+    r0 = addr_reg;
+    r1 = 3;
+    rbase = GUEST_BASE ? TCG_GUEST_BASE_REG : 0;
+#endif
 
-    tcg_out32 (s, CMP | (7 << 23) | RA (r2) | RB (r1));
-#if TARGET_LONG_BITS == 64
-    tcg_out32 (s, LWZ | RT (r1) | RA (r0) | 4);
-    tcg_out32 (s, CMP | BF (6) | RA (addr_reg2) | RB (r1));
-    tcg_out32 (s, CRAND | BT (7, CR_EQ) | BA (6, CR_EQ) | BB (7, CR_EQ));
+#ifdef TARGET_WORDS_BIGENDIAN
+    bswap = 0;
+#else
+    bswap = 1;
+#endif
+    switch (opc) {
+    case 0:
+        tcg_out32 (s, STBX | SAB (data_reg, rbase, r0));
+        break;
+    case 1:
+        if (bswap)
+            tcg_out32 (s, STHBRX | SAB (data_reg, rbase, r0));
+        else
+            tcg_out32 (s, STHX | SAB (data_reg, rbase, r0));
+        break;
+    case 2:
+        if (bswap)
+            tcg_out32 (s, STWBRX | SAB (data_reg, rbase, r0));
+        else
+            tcg_out32 (s, STWX | SAB (data_reg, rbase, r0));
+        break;
+    case 3:
+        if (bswap) {
+            tcg_out32 (s, ADDI | RT (r1) | RA (r0) | 4);
+            tcg_out32 (s, STWBRX | SAB (data_reg,  rbase, r0));
+            tcg_out32 (s, STWBRX | SAB (data_reg2, rbase, r1));
+        }
+        else {
+#ifdef CONFIG_USE_GUEST_BASE
+            tcg_out32 (s, STWX | SAB (data_reg2, rbase, r0));
+            tcg_out32 (s, ADDI | RT (r1) | RA (r0) | 4);
+            tcg_out32 (s, STWX | SAB (data_reg,  rbase, r1));
+#else
+            tcg_out32 (s, STW | RS (data_reg2) | RA (r0));
+            tcg_out32 (s, STW | RS (data_reg) | RA (r0) | 4);
 #endif
+        }
+        break;
+    }
 
-    label1_ptr = s->code_ptr;
-#ifdef FAST_PATH
-    tcg_out32 (s, BC | BI (7, CR_EQ) | BO_COND_TRUE);
+#ifdef CONFIG_SOFTMMU
+    add_qemu_ldst_label (s,
+                         0,
+                         opc,
+                         data_reg,
+                         data_reg2,
+                         addr_reg,
+                         addr_reg2,
+                         mem_index,
+                         s->code_ptr,
+                         label_ptr);
 #endif
+}
+
+#if defined(CONFIG_SOFTMMU)
+static void tcg_out_qemu_ld_slow_path (TCGContext *s, TCGLabelQemuLdst *label)
+{
+    int s_bits;
+    int ir;
+    int opc = label->opc;
+    int mem_index = label->mem_index;
+    int data_reg = label->datalo_reg;
+    int data_reg2 = label->datahi_reg;
+    int addr_reg = label->addrlo_reg;
+    uint8_t *raddr = label->raddr;
+    uint8_t **label_ptr = &label->label_ptr[0];
+
+    s_bits = opc & 3;
+
+    /* resolve label address */
+    reloc_pc14 (label_ptr[0], (tcg_target_long) s->code_ptr);
 
     /* slow path */
-    ir = 3;
-#ifdef CONFIG_TCG_PASS_AREG0
-    tcg_out_mov (s, TCG_TYPE_I32, ir++, TCG_AREG0);
+    ir = 4;
+#if TARGET_LONG_BITS == 32
+    tcg_out_mov (s, TCG_TYPE_I32, ir++, addr_reg);
+#else
+#ifdef TCG_TARGET_CALL_ALIGN_ARGS
+    ir |= 1;
+#endif
+    tcg_out_mov (s, TCG_TYPE_I32, ir++, label->addrhi_reg);
+    tcg_out_mov (s, TCG_TYPE_I32, ir++, addr_reg);
 #endif
+    tcg_out_movi (s, TCG_TYPE_I32, ir, mem_index);
+    tcg_out_call (s, (tcg_target_long) ld_trampolines[s_bits], 1);
+    tcg_out32 (s, (tcg_target_long) raddr);
+    switch (opc) {
+    case 0|4:
+        tcg_out32 (s, EXTSB | RA (data_reg) | RS (3));
+        break;
+    case 1|4:
+        tcg_out32 (s, EXTSH | RA (data_reg) | RS (3));
+        break;
+    case 0:
+    case 1:
+    case 2:
+        if (data_reg != 3)
+            tcg_out_mov (s, TCG_TYPE_I32, data_reg, 3);
+        break;
+    case 3:
+        if (data_reg == 3) {
+            if (data_reg2 == 4) {
+                tcg_out_mov (s, TCG_TYPE_I32, 0, 4);
+                tcg_out_mov (s, TCG_TYPE_I32, 4, 3);
+                tcg_out_mov (s, TCG_TYPE_I32, 3, 0);
+            }
+            else {
+                tcg_out_mov (s, TCG_TYPE_I32, data_reg2, 3);
+                tcg_out_mov (s, TCG_TYPE_I32, 3, 4);
+            }
+        }
+        else {
+            if (data_reg != 4) tcg_out_mov (s, TCG_TYPE_I32, data_reg, 4);
+            if (data_reg2 != 3) tcg_out_mov (s, TCG_TYPE_I32, data_reg2, 3);
+        }
+        break;
+    }
+    /* Jump to the code corresponding to next IR of qemu_st */
+    tcg_out_b (s, 0, (tcg_target_long) raddr);
+}
+
+static void tcg_out_qemu_st_slow_path (TCGContext *s, TCGLabelQemuLdst *label)
+{
+    int ir;
+    int opc = label->opc;
+    int mem_index = label->mem_index;
+    int data_reg = label->datalo_reg;
+    int data_reg2 = label->datahi_reg;
+    int addr_reg = label->addrlo_reg;
+    uint8_t *raddr = label->raddr;
+    uint8_t **label_ptr = &label->label_ptr[0];
+
+    /* resolve label address */
+    reloc_pc14 (label_ptr[0], (tcg_target_long) s->code_ptr);
+
+    /* slow path */
+    ir = 4;
 #if TARGET_LONG_BITS == 32
     tcg_out_mov (s, TCG_TYPE_I32, ir++, addr_reg);
 #else
 #ifdef TCG_TARGET_CALL_ALIGN_ARGS
     ir |= 1;
 #endif
-    tcg_out_mov (s, TCG_TYPE_I32, ir++, addr_reg2);
+    tcg_out_mov (s, TCG_TYPE_I32, ir++, label->addrhi_reg);
     tcg_out_mov (s, TCG_TYPE_I32, ir++, addr_reg);
 #endif
 
@@ -893,75 +956,39 @@ static void tcg_out_qemu_st (TCGContext *s, const TCGArg *args, int opc)
     ir++;
 
     tcg_out_movi (s, TCG_TYPE_I32, ir, mem_index);
-    tcg_out_call (s, (tcg_target_long) qemu_st_helpers[opc], 1);
-    label2_ptr = s->code_ptr;
-    tcg_out32 (s, B);
-
-    /* label1: fast path */
-#ifdef FAST_PATH
-    reloc_pc14 (label1_ptr, (tcg_target_long) s->code_ptr);
-#endif
-
-    tcg_out32 (s, (LWZ
-                   | RT (r0)
-                   | RA (r0)
-                   | (offsetof (CPUTLBEntry, addend)
-                      - offsetof (CPUTLBEntry, addr_write))
-                   ));
-    /* r0 = env->tlb_table[mem_index][index].addend */
-    tcg_out32 (s, ADD | RT (r0) | RA (r0) | RB (addr_reg));
-    /* r0 = env->tlb_table[mem_index][index].addend + addr */
-
-#else  /* !CONFIG_SOFTMMU */
-    r0 = addr_reg;
-    r1 = 3;
-    rbase = GUEST_BASE ? TCG_GUEST_BASE_REG : 0;
-#endif
+    tcg_out_call (s, (tcg_target_long) st_trampolines[opc], 1);
+    tcg_out32 (s, (tcg_target_long) raddr);
+    tcg_out_b (s, 0, (tcg_target_long) raddr);
+}
 
-#ifdef TARGET_WORDS_BIGENDIAN
-    bswap = 0;
-#else
-    bswap = 1;
-#endif
-    switch (opc) {
-    case 0:
-        tcg_out32 (s, STBX | SAB (data_reg, rbase, r0));
-        break;
-    case 1:
-        if (bswap)
-            tcg_out32 (s, STHBRX | SAB (data_reg, rbase, r0));
-        else
-            tcg_out32 (s, STHX | SAB (data_reg, rbase, r0));
-        break;
-    case 2:
-        if (bswap)
-            tcg_out32 (s, STWBRX | SAB (data_reg, rbase, r0));
-        else
-            tcg_out32 (s, STWX | SAB (data_reg, rbase, r0));
-        break;
-    case 3:
-        if (bswap) {
-            tcg_out32 (s, ADDI | RT (r1) | RA (r0) | 4);
-            tcg_out32 (s, STWBRX | SAB (data_reg,  rbase, r0));
-            tcg_out32 (s, STWBRX | SAB (data_reg2, rbase, r1));
+void tcg_out_tb_finalize(TCGContext *s)
+{
+    int i;
+    TCGLabelQemuLdst *label;
+
+    /* qemu_ld/st slow paths */
+    for (i = 0; i < s->nb_qemu_ldst_labels; i++) {
+        label = (TCGLabelQemuLdst *) &s->qemu_ldst_labels[i];
+        if (label->is_ld) {
+            tcg_out_qemu_ld_slow_path (s, label);
         }
         else {
-#ifdef CONFIG_USE_GUEST_BASE
-            tcg_out32 (s, STWX | SAB (data_reg2, rbase, r0));
-            tcg_out32 (s, ADDI | RT (r1) | RA (r0) | 4);
-            tcg_out32 (s, STWX | SAB (data_reg,  rbase, r1));
-#else
-            tcg_out32 (s, STW | RS (data_reg2) | RA (r0));
-            tcg_out32 (s, STW | RS (data_reg) | RA (r0) | 4);
-#endif
+            tcg_out_qemu_st_slow_path (s, label);
         }
-        break;
     }
+}
+#endif
 
 #ifdef CONFIG_SOFTMMU
-    reloc_pc24 (label2_ptr, (tcg_target_long) s->code_ptr);
-#endif
+static void emit_ldst_trampoline (TCGContext *s, const void *ptr)
+{
+    tcg_out32 (s, MFSPR | RT (3) | LR);
+    tcg_out32 (s, ADDI | RT (3) | RA (3) | 4);
+    tcg_out32 (s, MTSPR | RS (3) | LR);
+    tcg_out_mov (s, TCG_TYPE_I32, 3, TCG_AREG0);
+    tcg_out_b (s, 0, (tcg_target_long) ptr);
 }
+#endif
 
 static void tcg_target_qemu_prologue (TCGContext *s)
 {
@@ -1023,6 +1050,16 @@ static void tcg_target_qemu_prologue (TCGContext *s)
     tcg_out32 (s, MTSPR | RS (0) | LR);
     tcg_out32 (s, ADDI | RT (1) | RA (1) | frame_size);
     tcg_out32 (s, BCLR | BO_ALWAYS);
+
+#ifdef CONFIG_SOFTMMU
+    for (i = 0; i < 4; ++i) {
+        ld_trampolines[i] = s->code_ptr;
+        emit_ldst_trampoline (s, qemu_ld_helpers[i]);
+
+        st_trampolines[i] = s->code_ptr;
+        emit_ldst_trampoline (s, qemu_st_helpers[i]);
+    }
+#endif
 }
 
 static void tcg_out_ld (TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1,
@@ -1307,6 +1344,72 @@ static void tcg_out_setcond2 (TCGContext *s, const TCGArg *args,
         );
 }
 
+static void tcg_out_movcond (TCGContext *s, TCGCond cond,
+                             TCGArg dest,
+                             TCGArg c1, TCGArg c2,
+                             TCGArg v1, TCGArg v2,
+                             int const_c2)
+{
+    tcg_out_cmp (s, cond, c1, c2, const_c2, 7);
+
+    if (1) {
+        /* At least here on 7747A bit twiddling hacks are outperformed
+           by jumpy code (the testing was not scientific) */
+        if (dest == v2) {
+            cond = tcg_invert_cond (cond);
+            v2 = v1;
+        }
+        else {
+            if (dest != v1) {
+                tcg_out_mov (s, TCG_TYPE_I32, dest, v1);
+            }
+        }
+        /* Branch forward over one insn */
+        tcg_out32 (s, tcg_to_bc[cond] | 8);
+        tcg_out_mov (s, TCG_TYPE_I32, dest, v2);
+    }
+    else {
+        /* isel version, "if (1)" above should be replaced once a way
+           to figure out availability of isel on the underlying
+           hardware is found */
+        int tab, bc;
+
+        switch (cond) {
+        case TCG_COND_EQ:
+            tab = TAB (dest, v1, v2);
+            bc = CR_EQ;
+            break;
+        case TCG_COND_NE:
+            tab = TAB (dest, v2, v1);
+            bc = CR_EQ;
+            break;
+        case TCG_COND_LTU:
+        case TCG_COND_LT:
+            tab = TAB (dest, v1, v2);
+            bc = CR_LT;
+            break;
+        case TCG_COND_GEU:
+        case TCG_COND_GE:
+            tab = TAB (dest, v2, v1);
+            bc = CR_LT;
+            break;
+        case TCG_COND_LEU:
+        case TCG_COND_LE:
+            tab = TAB (dest, v2, v1);
+            bc = CR_GT;
+            break;
+        case TCG_COND_GTU:
+        case TCG_COND_GT:
+            tab = TAB (dest, v1, v2);
+            bc = CR_GT;
+            break;
+        default:
+            tcg_abort ();
+        }
+        tcg_out32 (s, ISEL | tab | ((bc + 28) << 6));
+    }
+}
+
 static void tcg_out_brcond (TCGContext *s, TCGCond cond,
                             TCGArg arg1, TCGArg arg2, int const_arg2,
                             int label_index)
@@ -1394,15 +1497,6 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
     case INDEX_op_call:
         tcg_out_call (s, args[0], const_args[0]);
         break;
-    case INDEX_op_jmp:
-        if (const_args[0]) {
-            tcg_out_b (s, 0, args[0]);
-        }
-        else {
-            tcg_out32 (s, MTSPR | RS (args[0]) | CTR);
-            tcg_out32 (s, BCCTR | BO_ALWAYS);
-        }
-        break;
     case INDEX_op_movi_i32:
         tcg_out_movi(s, TCG_TYPE_I32, args[0], args[1]);
         break;
@@ -1864,6 +1958,13 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
             );
         break;
 
+    case INDEX_op_movcond_i32:
+        tcg_out_movcond (s, args[5], args[0],
+                         args[1], args[2],
+                         args[3], args[4],
+                         const_args[2]);
+        break;
+
     default:
         tcg_dump_ops (s);
         tcg_abort ();
@@ -1874,7 +1975,6 @@ static const TCGTargetOpDef ppc_op_defs[] = {
     { INDEX_op_exit_tb, { } },
     { INDEX_op_goto_tb, { } },
     { INDEX_op_call, { "ri" } },
-    { INDEX_op_jmp, { "ri" } },
     { INDEX_op_br, { } },
 
     { INDEX_op_mov_i32, { "r", "r" } },
@@ -1960,6 +2060,7 @@ static const TCGTargetOpDef ppc_op_defs[] = {
     { INDEX_op_ext16u_i32, { "r", "r" } },
 
     { INDEX_op_deposit_i32, { "r", "0", "r" } },
+    { INDEX_op_movcond_i32, { "r", "r", "ri", "r", "r" } },
 
     { -1 },
 };
diff --git a/tcg/ppc/tcg-target.h b/tcg/ppc/tcg-target.h
index 2f37fd289..ad433ae5b 100644
--- a/tcg/ppc/tcg-target.h
+++ b/tcg/ppc/tcg-target.h
@@ -92,11 +92,10 @@ typedef enum {
 #define TCG_TARGET_HAS_nand_i32         1
 #define TCG_TARGET_HAS_nor_i32          1
 #define TCG_TARGET_HAS_deposit_i32      1
+#define TCG_TARGET_HAS_movcond_i32      1
 
 #define TCG_AREG0 TCG_REG_R27
 
-#define TCG_TARGET_HAS_GUEST_BASE
-
 #define tcg_qemu_tb_exec(env, tb_ptr) \
     ((long __attribute__ ((longcall)) \
       (*)(void *, void *))code_gen_prologue)(env, tb_ptr)
diff --git a/tcg/ppc64/tcg-target.c b/tcg/ppc64/tcg-target.c
index 27a0ae88e..5403fc1f9 100644
--- a/tcg/ppc64/tcg-target.c
+++ b/tcg/ppc64/tcg-target.c
@@ -208,12 +208,6 @@ static void patch_reloc (uint8_t *code_ptr, int type,
     }
 }
 
-/* maximum number of register used for input function arguments */
-static int tcg_target_get_call_iarg_regs_count (int flags)
-{
-    return ARRAY_SIZE (tcg_target_call_iarg_regs);
-}
-
 /* parse target specific constraints */
 static int target_parse_constraint (TCGArgConstraint *ct, const char **pct_str)
 {
@@ -235,10 +229,8 @@ static int target_parse_constraint (TCGArgConstraint *ct, const char **pct_str)
         tcg_regset_reset_reg (ct->u.regs, TCG_REG_R3);
 #ifdef CONFIG_SOFTMMU
         tcg_regset_reset_reg (ct->u.regs, TCG_REG_R4);
-#ifdef CONFIG_TCG_PASS_AREG0
         tcg_regset_reset_reg (ct->u.regs, TCG_REG_R5);
 #endif
-#endif
         break;
     case 'S':                   /* qemu_st constraint */
         ct->ct |= TCG_CT_REG;
@@ -247,10 +239,8 @@ static int target_parse_constraint (TCGArgConstraint *ct, const char **pct_str)
 #ifdef CONFIG_SOFTMMU
         tcg_regset_reset_reg (ct->u.regs, TCG_REG_R4);
         tcg_regset_reset_reg (ct->u.regs, TCG_REG_R5);
-#ifdef CONFIG_TCG_PASS_AREG0
         tcg_regset_reset_reg (ct->u.regs, TCG_REG_R6);
 #endif
-#endif
         break;
     case 'Z':
         ct->ct |= TCG_CT_CONST_U32;
@@ -428,7 +418,7 @@ enum {
     CR_SO
 };
 
-static const uint32_t tcg_to_bc[10] = {
+static const uint32_t tcg_to_bc[] = {
     [TCG_COND_EQ]  = BC | BI (7, CR_EQ) | BO_COND_TRUE,
     [TCG_COND_NE]  = BC | BI (7, CR_EQ) | BO_COND_FALSE,
     [TCG_COND_LT]  = BC | BI (7, CR_LT) | BO_COND_TRUE,
@@ -558,7 +548,6 @@ static void tcg_out_ldsta (TCGContext *s, int ret, int addr,
 
 #include "../../softmmu_defs.h"
 
-#ifdef CONFIG_TCG_PASS_AREG0
 /* helper signature: helper_ld_mmu(CPUState *env, target_ulong addr,
    int mmu_idx) */
 static const void * const qemu_ld_helpers[4] = {
@@ -576,25 +565,6 @@ static const void * const qemu_st_helpers[4] = {
     helper_stl_mmu,
     helper_stq_mmu,
 };
-#else
-/* legacy helper signature: __ld_mmu(target_ulong addr, int
-   mmu_idx) */
-static void *qemu_ld_helpers[4] = {
-    __ldb_mmu,
-    __ldw_mmu,
-    __ldl_mmu,
-    __ldq_mmu,
-};
-
-/* legacy helper signature: __st_mmu(target_ulong addr, uintxx_t val,
-   int mmu_idx) */
-static void *qemu_st_helpers[4] = {
-    __stb_mmu,
-    __stw_mmu,
-    __stl_mmu,
-    __stq_mmu,
-};
-#endif
 
 static void tcg_out_tlb_read (TCGContext *s, int r0, int r1, int r2,
                               int addr_reg, int s_bits, int offset)
@@ -676,9 +646,7 @@ static void tcg_out_qemu_ld (TCGContext *s, const TCGArg *args, int opc)
 
     /* slow path */
     ir = 3;
-#ifdef CONFIG_TCG_PASS_AREG0
     tcg_out_mov (s, TCG_TYPE_I64, ir++, TCG_AREG0);
-#endif
     tcg_out_mov (s, TCG_TYPE_I64, ir++, addr_reg);
     tcg_out_movi (s, TCG_TYPE_I64, ir++, mem_index);
 
@@ -827,9 +795,7 @@ static void tcg_out_qemu_st (TCGContext *s, const TCGArg *args, int opc)
 
     /* slow path */
     ir = 3;
-#ifdef CONFIG_TCG_PASS_AREG0
     tcg_out_mov (s, TCG_TYPE_I64, ir++, TCG_AREG0);
-#endif
     tcg_out_mov (s, TCG_TYPE_I64, ir++, addr_reg);
     tcg_out_rld (s, RLDICL, ir++, data_reg, 0, 64 - (1 << (3 + opc)));
     tcg_out_movi (s, TCG_TYPE_I64, ir++, mem_index);
@@ -1279,15 +1245,6 @@ static void tcg_out_op (TCGContext *s, TCGOpcode opc, const TCGArg *args,
     case INDEX_op_call:
         tcg_out_call (s, args[0], const_args[0]);
         break;
-    case INDEX_op_jmp:
-        if (const_args[0]) {
-            tcg_out_b (s, 0, args[0]);
-        }
-        else {
-            tcg_out32 (s, MTSPR | RS (args[0]) | CTR);
-            tcg_out32 (s, BCCTR | BO_ALWAYS);
-        }
-        break;
     case INDEX_op_movi_i32:
         tcg_out_movi (s, TCG_TYPE_I32, args[0], args[1]);
         break;
@@ -1622,7 +1579,6 @@ static const TCGTargetOpDef ppc_op_defs[] = {
     { INDEX_op_exit_tb, { } },
     { INDEX_op_goto_tb, { } },
     { INDEX_op_call, { "ri" } },
-    { INDEX_op_jmp, { "ri" } },
     { INDEX_op_br, { } },
 
     { INDEX_op_mov_i32, { "r", "r" } },
diff --git a/tcg/ppc64/tcg-target.h b/tcg/ppc64/tcg-target.h
index 97eec0843..97fc5c988 100644
--- a/tcg/ppc64/tcg-target.h
+++ b/tcg/ppc64/tcg-target.h
@@ -83,6 +83,7 @@ typedef enum {
 #define TCG_TARGET_HAS_nand_i32         0
 #define TCG_TARGET_HAS_nor_i32          0
 #define TCG_TARGET_HAS_deposit_i32      0
+#define TCG_TARGET_HAS_movcond_i32      0
 
 #define TCG_TARGET_HAS_div_i64          1
 #define TCG_TARGET_HAS_rot_i64          0
@@ -103,8 +104,8 @@ typedef enum {
 #define TCG_TARGET_HAS_nand_i64         0
 #define TCG_TARGET_HAS_nor_i64          0
 #define TCG_TARGET_HAS_deposit_i64      0
+#define TCG_TARGET_HAS_movcond_i64      0
 
 #define TCG_AREG0 TCG_REG_R27
 
-#define TCG_TARGET_HAS_GUEST_BASE
 #define TCG_TARGET_EXTEND_ARGS 1
diff --git a/tcg/s390/tcg-target.c b/tcg/s390/tcg-target.c
index 04662c15f..fd9286f52 100644
--- a/tcg/s390/tcg-target.c
+++ b/tcg/s390/tcg-target.c
@@ -268,7 +268,7 @@ static const int tcg_target_call_oarg_regs[] = {
 #define S390_CC_ALWAYS  15
 
 /* Condition codes that result from a COMPARE and COMPARE LOGICAL.  */
-static const uint8_t tcg_cond_to_s390_cond[10] = {
+static const uint8_t tcg_cond_to_s390_cond[] = {
     [TCG_COND_EQ]  = S390_CC_EQ,
     [TCG_COND_NE]  = S390_CC_NE,
     [TCG_COND_LT]  = S390_CC_LT,
@@ -284,7 +284,7 @@ static const uint8_t tcg_cond_to_s390_cond[10] = {
 /* Condition codes that result from a LOAD AND TEST.  Here, we have no
    unsigned instruction variation, however since the test is vs zero we
    can re-map the outcomes appropriately.  */
-static const uint8_t tcg_cond_to_ltr_cond[10] = {
+static const uint8_t tcg_cond_to_ltr_cond[] = {
     [TCG_COND_EQ]  = S390_CC_EQ,
     [TCG_COND_NE]  = S390_CC_NE,
     [TCG_COND_LT]  = S390_CC_LT,
@@ -301,7 +301,6 @@ static const uint8_t tcg_cond_to_ltr_cond[10] = {
 
 #include "../../softmmu_defs.h"
 
-#ifdef CONFIG_TCG_PASS_AREG0
 /* helper signature: helper_ld_mmu(CPUState *env, target_ulong addr,
    int mmu_idx) */
 static const void * const qemu_ld_helpers[4] = {
@@ -319,25 +318,6 @@ static const void * const qemu_st_helpers[4] = {
     helper_stl_mmu,
     helper_stq_mmu,
 };
-#else
-/* legacy helper signature: __ld_mmu(target_ulong addr, int
-   mmu_idx) */
-static void *qemu_ld_helpers[4] = {
-    __ldb_mmu,
-    __ldw_mmu,
-    __ldl_mmu,
-    __ldq_mmu,
-};
-
-/* legacy helper signature: __st_mmu(target_ulong addr, uintxx_t val,
-   int mmu_idx) */
-static void *qemu_st_helpers[4] = {
-    __stb_mmu,
-    __stw_mmu,
-    __stl_mmu,
-    __stq_mmu,
-};
-#endif
 #endif
 
 static uint8_t *tb_ret_addr;
@@ -376,11 +356,6 @@ static void patch_reloc(uint8_t *code_ptr, int type,
     }
 }
 
-static int tcg_target_get_call_iarg_regs_count(int flags)
-{
-    return sizeof(tcg_target_call_iarg_regs) / sizeof(int);
-}
-
 /* parse target specific constraints */
 static int target_parse_constraint(TCGArgConstraint *ct, const char **pct_str)
 {
@@ -1138,7 +1113,7 @@ static void tgen64_xori(TCGContext *s, TCGReg dest, tcg_target_ulong val)
 static int tgen_cmp(TCGContext *s, TCGType type, TCGCond c, TCGReg r1,
                     TCGArg c2, int c2const)
 {
-    bool is_unsigned = (c > TCG_COND_GT);
+    bool is_unsigned = is_unsigned_cond(c);
     if (c2const) {
         if (c2 == 0) {
             if (type == TCG_TYPE_I32) {
@@ -1507,29 +1482,25 @@ static void tcg_prepare_qemu_ldst(TCGContext* s, TCGReg data_reg,
             tcg_abort();
         }
         tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_R4, mem_index);
-#ifdef CONFIG_TCG_PASS_AREG0
         /* XXX/FIXME: suboptimal */
-        tcg_out_mov(s, TCG_TYPE_I32, tcg_target_call_iarg_regs[2],
+        tcg_out_mov(s, TCG_TYPE_I64, tcg_target_call_iarg_regs[3],
+                    tcg_target_call_iarg_regs[2]);
+        tcg_out_mov(s, TCG_TYPE_I64, tcg_target_call_iarg_regs[2],
                     tcg_target_call_iarg_regs[1]);
-        tcg_out_mov(s, TCG_TYPE_TL, tcg_target_call_iarg_regs[1],
+        tcg_out_mov(s, TCG_TYPE_I64, tcg_target_call_iarg_regs[1],
                     tcg_target_call_iarg_regs[0]);
-        tcg_out_mov(s, TCG_TYPE_PTR, tcg_target_call_iarg_regs[0],
+        tcg_out_mov(s, TCG_TYPE_I64, tcg_target_call_iarg_regs[0],
                     TCG_AREG0);
-#endif
         tgen_calli(s, (tcg_target_ulong)qemu_st_helpers[s_bits]);
     } else {
         tcg_out_movi(s, TCG_TYPE_I32, arg1, mem_index);
-#ifdef CONFIG_TCG_PASS_AREG0
         /* XXX/FIXME: suboptimal */
-        tcg_out_mov(s, TCG_TYPE_I32, tcg_target_call_iarg_regs[3],
-                    tcg_target_call_iarg_regs[2]);
         tcg_out_mov(s, TCG_TYPE_I64, tcg_target_call_iarg_regs[2],
                     tcg_target_call_iarg_regs[1]);
-        tcg_out_mov(s, TCG_TYPE_TL, tcg_target_call_iarg_regs[1],
+        tcg_out_mov(s, TCG_TYPE_I64, tcg_target_call_iarg_regs[1],
                     tcg_target_call_iarg_regs[0]);
-        tcg_out_mov(s, TCG_TYPE_PTR, tcg_target_call_iarg_regs[0],
+        tcg_out_mov(s, TCG_TYPE_I64, tcg_target_call_iarg_regs[0],
                     TCG_AREG0);
-#endif
         tgen_calli(s, (tcg_target_ulong)qemu_ld_helpers[s_bits]);
 
         /* sign extension */
@@ -2066,11 +2037,6 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc,
         break;
 #endif /* TCG_TARGET_REG_BITS == 64 */
 
-    case INDEX_op_jmp:
-        /* This one is obsolete and never emitted.  */
-        tcg_abort();
-        break;
-
     default:
         fprintf(stderr,"unimplemented opc 0x%x\n",opc);
         tcg_abort();
@@ -2081,7 +2047,6 @@ static const TCGTargetOpDef s390_op_defs[] = {
     { INDEX_op_exit_tb, { } },
     { INDEX_op_goto_tb, { } },
     { INDEX_op_call, { "ri" } },
-    { INDEX_op_jmp, { "ri" } },
     { INDEX_op_br, { } },
 
     { INDEX_op_mov_i32, { "r", "r" } },
diff --git a/tcg/s390/tcg-target.h b/tcg/s390/tcg-target.h
index d12f90b70..a0181aef7 100644
--- a/tcg/s390/tcg-target.h
+++ b/tcg/s390/tcg-target.h
@@ -63,6 +63,7 @@ typedef enum TCGReg {
 #define TCG_TARGET_HAS_nand_i32         0
 #define TCG_TARGET_HAS_nor_i32          0
 #define TCG_TARGET_HAS_deposit_i32      0
+#define TCG_TARGET_HAS_movcond_i32      0
 
 #if TCG_TARGET_REG_BITS == 64
 #define TCG_TARGET_HAS_div2_i64         1
@@ -84,10 +85,9 @@ typedef enum TCGReg {
 #define TCG_TARGET_HAS_nand_i64         0
 #define TCG_TARGET_HAS_nor_i64          0
 #define TCG_TARGET_HAS_deposit_i64      0
+#define TCG_TARGET_HAS_movcond_i64      0
 #endif
 
-#define TCG_TARGET_HAS_GUEST_BASE
-
 /* used for function call generation */
 #define TCG_REG_CALL_STACK		TCG_REG_R15
 #define TCG_TARGET_STACK_ALIGN		8
@@ -96,7 +96,6 @@ typedef enum TCGReg {
 #define TCG_TARGET_EXTEND_ARGS 1
 
 enum {
-    /* Note: must be synced with dyngen-exec.h */
     TCG_AREG0 = TCG_REG_R10,
 };
 
diff --git a/tcg/sparc/tcg-target.c b/tcg/sparc/tcg-target.c
index 247a27814..f14664787 100644
--- a/tcg/sparc/tcg-target.c
+++ b/tcg/sparc/tcg-target.c
@@ -59,10 +59,14 @@ static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = {
 };
 #endif
 
-#ifdef CONFIG_TCG_PASS_AREG0
-#define ARG_OFFSET 1
+/* Define some temporary registers.  T2 is used for constant generation.  */
+#define TCG_REG_T1  TCG_REG_G1
+#define TCG_REG_T2  TCG_REG_O7
+
+#ifdef CONFIG_USE_GUEST_BASE
+# define TCG_GUEST_BASE_REG TCG_REG_I5
 #else
-#define ARG_OFFSET 0
+# define TCG_GUEST_BASE_REG TCG_REG_G0
 #endif
 
 static const int tcg_target_reg_alloc_order[] = {
@@ -74,11 +78,25 @@ static const int tcg_target_reg_alloc_order[] = {
     TCG_REG_L5,
     TCG_REG_L6,
     TCG_REG_L7,
+
     TCG_REG_I0,
     TCG_REG_I1,
     TCG_REG_I2,
     TCG_REG_I3,
     TCG_REG_I4,
+    TCG_REG_I5,
+
+    TCG_REG_G2,
+    TCG_REG_G3,
+    TCG_REG_G4,
+    TCG_REG_G5,
+
+    TCG_REG_O0,
+    TCG_REG_O1,
+    TCG_REG_O2,
+    TCG_REG_O3,
+    TCG_REG_O4,
+    TCG_REG_O5,
 };
 
 static const int tcg_target_call_iarg_regs[6] = {
@@ -97,105 +115,6 @@ static const int tcg_target_call_oarg_regs[] = {
     TCG_REG_O3,
 };
 
-static inline int check_fit_tl(tcg_target_long val, unsigned int bits)
-{
-    return (val << ((sizeof(tcg_target_long) * 8 - bits))
-            >> (sizeof(tcg_target_long) * 8 - bits)) == val;
-}
-
-static inline int check_fit_i32(uint32_t val, unsigned int bits)
-{
-    return ((val << (32 - bits)) >> (32 - bits)) == val;
-}
-
-static void patch_reloc(uint8_t *code_ptr, int type,
-                        tcg_target_long value, tcg_target_long addend)
-{
-    value += addend;
-    switch (type) {
-    case R_SPARC_32:
-        if (value != (uint32_t)value)
-            tcg_abort();
-        *(uint32_t *)code_ptr = value;
-        break;
-    case R_SPARC_WDISP22:
-        value -= (long)code_ptr;
-        value >>= 2;
-        if (!check_fit_tl(value, 22))
-            tcg_abort();
-        *(uint32_t *)code_ptr = ((*(uint32_t *)code_ptr) & ~0x3fffff) | value;
-        break;
-    case R_SPARC_WDISP19:
-        value -= (long)code_ptr;
-        value >>= 2;
-        if (!check_fit_tl(value, 19))
-            tcg_abort();
-        *(uint32_t *)code_ptr = ((*(uint32_t *)code_ptr) & ~0x7ffff) | value;
-        break;
-    default:
-        tcg_abort();
-    }
-}
-
-/* maximum number of register used for input function arguments */
-static inline int tcg_target_get_call_iarg_regs_count(int flags)
-{
-    return 6;
-}
-
-/* parse target specific constraints */
-static int target_parse_constraint(TCGArgConstraint *ct, const char **pct_str)
-{
-    const char *ct_str;
-
-    ct_str = *pct_str;
-    switch (ct_str[0]) {
-    case 'r':
-        ct->ct |= TCG_CT_REG;
-        tcg_regset_set32(ct->u.regs, 0, 0xffffffff);
-        break;
-    case 'L': /* qemu_ld/st constraint */
-        ct->ct |= TCG_CT_REG;
-        tcg_regset_set32(ct->u.regs, 0, 0xffffffff);
-        // Helper args
-        tcg_regset_reset_reg(ct->u.regs, TCG_REG_O0);
-        tcg_regset_reset_reg(ct->u.regs, TCG_REG_O1);
-        tcg_regset_reset_reg(ct->u.regs, TCG_REG_O2);
-#ifdef CONFIG_TCG_PASS_AREG0
-        tcg_regset_reset_reg(ct->u.regs, TCG_REG_O3);
-#endif
-        break;
-    case 'I':
-        ct->ct |= TCG_CT_CONST_S11;
-        break;
-    case 'J':
-        ct->ct |= TCG_CT_CONST_S13;
-        break;
-    default:
-        return -1;
-    }
-    ct_str++;
-    *pct_str = ct_str;
-    return 0;
-}
-
-/* test if a constant matches the constraint */
-static inline int tcg_target_const_match(tcg_target_long val,
-                                         const TCGArgConstraint *arg_ct)
-{
-    int ct;
-
-    ct = arg_ct->ct;
-    if (ct & TCG_CT_CONST)
-        return 1;
-    else if ((ct & TCG_CT_CONST_S11) && check_fit_tl(val, 11))
-        return 1;
-    else if ((ct & TCG_CT_CONST_S13) && check_fit_tl(val, 13))
-        return 1;
-    else
-        return 0;
-}
-
 #define INSN_OP(x)  ((x) << 30)
 #define INSN_OP2(x) ((x) << 22)
 #define INSN_OP3(x) ((x) << 19)
@@ -205,12 +124,13 @@ static inline int tcg_target_const_match(tcg_target_long val,
 #define INSN_RS2(x) (x)
 #define INSN_ASI(x) ((x) << 5)
 
+#define INSN_IMM10(x) ((1 << 13) | ((x) & 0x3ff))
 #define INSN_IMM11(x) ((1 << 13) | ((x) & 0x7ff))
 #define INSN_IMM13(x) ((1 << 13) | ((x) & 0x1fff))
+#define INSN_OFF16(x) ((((x) >> 2) & 0x3fff) | ((((x) >> 16) & 3) << 20))
 #define INSN_OFF19(x) (((x) >> 2) & 0x07ffff)
-#define INSN_OFF22(x) (((x) >> 2) & 0x3fffff)
+#define INSN_COND(x) ((x) << 25)
 
-#define INSN_COND(x, a) (((x) << 25) | ((a) << 29))
 #define COND_N     0x0
 #define COND_E     0x1
 #define COND_LE    0x2
@@ -227,11 +147,26 @@ static inline int tcg_target_const_match(tcg_target_long val,
 #define COND_CC    0xd
 #define COND_POS   0xe
 #define COND_VC    0xf
-#define BA         (INSN_OP(0) | INSN_COND(COND_A, 0) | INSN_OP2(0x2))
+#define BA         (INSN_OP(0) | INSN_COND(COND_A) | INSN_OP2(0x2))
+
+#define RCOND_Z    1
+#define RCOND_LEZ  2
+#define RCOND_LZ   3
+#define RCOND_NZ   5
+#define RCOND_GZ   6
+#define RCOND_GEZ  7
 
 #define MOVCC_ICC  (1 << 18)
 #define MOVCC_XCC  (1 << 18 | 1 << 12)
 
+#define BPCC_ICC   0
+#define BPCC_XCC   (2 << 20)
+#define BPCC_PT    (1 << 19)
+#define BPCC_PN    0
+#define BPCC_A     (1 << 29)
+
+#define BPR_PT     BPCC_PT
+
 #define ARITH_ADD  (INSN_OP(2) | INSN_OP3(0x00))
 #define ARITH_ADDCC (INSN_OP(2) | INSN_OP3(0x10))
 #define ARITH_AND  (INSN_OP(2) | INSN_OP3(0x01))
@@ -242,7 +177,7 @@ static inline int tcg_target_const_match(tcg_target_long val,
 #define ARITH_XOR  (INSN_OP(2) | INSN_OP3(0x03))
 #define ARITH_SUB  (INSN_OP(2) | INSN_OP3(0x04))
 #define ARITH_SUBCC (INSN_OP(2) | INSN_OP3(0x14))
-#define ARITH_ADDX (INSN_OP(2) | INSN_OP3(0x10))
+#define ARITH_ADDX (INSN_OP(2) | INSN_OP3(0x08))
 #define ARITH_SUBX (INSN_OP(2) | INSN_OP3(0x0c))
 #define ARITH_UMUL (INSN_OP(2) | INSN_OP3(0x0a))
 #define ARITH_UDIV (INSN_OP(2) | INSN_OP3(0x0e))
@@ -251,6 +186,7 @@ static inline int tcg_target_const_match(tcg_target_long val,
 #define ARITH_UDIVX (INSN_OP(2) | INSN_OP3(0x0d))
 #define ARITH_SDIVX (INSN_OP(2) | INSN_OP3(0x2d))
 #define ARITH_MOVCC (INSN_OP(2) | INSN_OP3(0x2c))
+#define ARITH_MOVR (INSN_OP(2) | INSN_OP3(0x2f))
 
 #define SHIFT_SLL  (INSN_OP(2) | INSN_OP3(0x25))
 #define SHIFT_SRL  (INSN_OP(2) | INSN_OP3(0x26))
@@ -294,6 +230,119 @@ static inline int tcg_target_const_match(tcg_target_long val,
 #define ASI_PRIMARY_LITTLE 0x88
 #endif
 
+#define LDUH_LE    (LDUHA | INSN_ASI(ASI_PRIMARY_LITTLE))
+#define LDSH_LE    (LDSHA | INSN_ASI(ASI_PRIMARY_LITTLE))
+#define LDUW_LE    (LDUWA | INSN_ASI(ASI_PRIMARY_LITTLE))
+#define LDSW_LE    (LDSWA | INSN_ASI(ASI_PRIMARY_LITTLE))
+#define LDX_LE     (LDXA  | INSN_ASI(ASI_PRIMARY_LITTLE))
+
+#define STH_LE     (STHA  | INSN_ASI(ASI_PRIMARY_LITTLE))
+#define STW_LE     (STWA  | INSN_ASI(ASI_PRIMARY_LITTLE))
+#define STX_LE     (STXA  | INSN_ASI(ASI_PRIMARY_LITTLE))
+
+static inline int check_fit_tl(tcg_target_long val, unsigned int bits)
+{
+    return (val << ((sizeof(tcg_target_long) * 8 - bits))
+            >> (sizeof(tcg_target_long) * 8 - bits)) == val;
+}
+
+static inline int check_fit_i32(uint32_t val, unsigned int bits)
+{
+    return ((val << (32 - bits)) >> (32 - bits)) == val;
+}
+
+static void patch_reloc(uint8_t *code_ptr, int type,
+                        tcg_target_long value, tcg_target_long addend)
+{
+    uint32_t insn;
+    value += addend;
+    switch (type) {
+    case R_SPARC_32:
+        if (value != (uint32_t)value) {
+            tcg_abort();
+        }
+        *(uint32_t *)code_ptr = value;
+        break;
+    case R_SPARC_WDISP16:
+        value -= (long)code_ptr;
+        if (!check_fit_tl(value >> 2, 16)) {
+            tcg_abort();
+        }
+        insn = *(uint32_t *)code_ptr;
+        insn &= ~INSN_OFF16(-1);
+        insn |= INSN_OFF16(value);
+        *(uint32_t *)code_ptr = insn;
+        break;
+    case R_SPARC_WDISP19:
+        value -= (long)code_ptr;
+        if (!check_fit_tl(value >> 2, 19)) {
+            tcg_abort();
+        }
+        insn = *(uint32_t *)code_ptr;
+        insn &= ~INSN_OFF19(-1);
+        insn |= INSN_OFF19(value);
+        *(uint32_t *)code_ptr = insn;
+        break;
+    default:
+        tcg_abort();
+    }
+}
+
+/* parse target specific constraints */
+static int target_parse_constraint(TCGArgConstraint *ct, const char **pct_str)
+{
+    const char *ct_str;
+
+    ct_str = *pct_str;
+    switch (ct_str[0]) {
+    case 'r':
+        ct->ct |= TCG_CT_REG;
+        tcg_regset_set32(ct->u.regs, 0, 0xffffffff);
+        break;
+    case 'L': /* qemu_ld/st constraint */
+        ct->ct |= TCG_CT_REG;
+        tcg_regset_set32(ct->u.regs, 0, 0xffffffff);
+        // Helper args
+        tcg_regset_reset_reg(ct->u.regs, TCG_REG_O0);
+        tcg_regset_reset_reg(ct->u.regs, TCG_REG_O1);
+        tcg_regset_reset_reg(ct->u.regs, TCG_REG_O2);
+        break;
+    case 'I':
+        ct->ct |= TCG_CT_CONST_S11;
+        break;
+    case 'J':
+        ct->ct |= TCG_CT_CONST_S13;
+        break;
+    case 'Z':
+        ct->ct |= TCG_CT_CONST_ZERO;
+        break;
+    default:
+        return -1;
+    }
+    ct_str++;
+    *pct_str = ct_str;
+    return 0;
+}
+
+/* test if a constant matches the constraint */
+static inline int tcg_target_const_match(tcg_target_long val,
+                                         const TCGArgConstraint *arg_ct)
+{
+    int ct = arg_ct->ct;
+
+    if (ct & TCG_CT_CONST) {
+        return 1;
+    } else if ((ct & TCG_CT_CONST_ZERO) && val == 0) {
+        return 1;
+    } else if ((ct & TCG_CT_CONST_S11) && check_fit_tl(val, 11)) {
+        return 1;
+    } else if ((ct & TCG_CT_CONST_S13) && check_fit_tl(val, 13)) {
+        return 1;
+    } else {
+        return 0;
+    }
+}
+
 static inline void tcg_out_arith(TCGContext *s, int rd, int rs1, int rs2,
                                  int op)
 {
@@ -318,7 +367,9 @@ static void tcg_out_arithc(TCGContext *s, int rd, int rs1,
 static inline void tcg_out_mov(TCGContext *s, TCGType type,
                                TCGReg ret, TCGReg arg)
 {
-    tcg_out_arith(s, ret, arg, TCG_REG_G0, ARITH_OR);
+    if (ret != arg) {
+        tcg_out_arith(s, ret, arg, TCG_REG_G0, ARITH_OR);
+    }
 }
 
 static inline void tcg_out_sethi(TCGContext *s, int ret, uint32_t arg)
@@ -359,71 +410,50 @@ static inline void tcg_out_movi(TCGContext *s, TCGType type,
         tcg_out_sethi(s, ret, ~arg);
         tcg_out_arithi(s, ret, ret, (arg & 0x3ff) | -0x400, ARITH_XOR);
     } else {
-        tcg_out_movi_imm32(s, TCG_REG_I4, arg >> (TCG_TARGET_REG_BITS / 2));
-        tcg_out_arithi(s, TCG_REG_I4, TCG_REG_I4, 32, SHIFT_SLLX);
-        tcg_out_movi_imm32(s, ret, arg);
-        tcg_out_arith(s, ret, ret, TCG_REG_I4, ARITH_OR);
+        tcg_out_movi_imm32(s, ret, arg >> (TCG_TARGET_REG_BITS / 2));
+        tcg_out_arithi(s, ret, ret, 32, SHIFT_SLLX);
+        tcg_out_movi_imm32(s, TCG_REG_T2, arg);
+        tcg_out_arith(s, ret, ret, TCG_REG_T2, ARITH_OR);
     }
 }
 
-static inline void tcg_out_ld_raw(TCGContext *s, int ret,
-                                  tcg_target_long arg)
-{
-    tcg_out_sethi(s, ret, arg);
-    tcg_out32(s, LDUW | INSN_RD(ret) | INSN_RS1(ret) |
-              INSN_IMM13(arg & 0x3ff));
-}
-
-static inline void tcg_out_ld_ptr(TCGContext *s, int ret,
-                                  tcg_target_long arg)
+static inline void tcg_out_ldst_rr(TCGContext *s, int data, int a1,
+                                   int a2, int op)
 {
-    if (!check_fit_tl(arg, 10))
-        tcg_out_movi(s, TCG_TYPE_PTR, ret, arg & ~0x3ffULL);
-    if (TCG_TARGET_REG_BITS == 64) {
-        tcg_out32(s, LDX | INSN_RD(ret) | INSN_RS1(ret) |
-                  INSN_IMM13(arg & 0x3ff));
-    } else {
-        tcg_out32(s, LDUW | INSN_RD(ret) | INSN_RS1(ret) |
-                  INSN_IMM13(arg & 0x3ff));
-    }
+    tcg_out32(s, op | INSN_RD(data) | INSN_RS1(a1) | INSN_RS2(a2));
 }
 
-static inline void tcg_out_ldst(TCGContext *s, int ret, int addr, int offset, int op)
+static inline void tcg_out_ldst(TCGContext *s, int ret, int addr,
+                                int offset, int op)
 {
-    if (check_fit_tl(offset, 13))
+    if (check_fit_tl(offset, 13)) {
         tcg_out32(s, op | INSN_RD(ret) | INSN_RS1(addr) |
                   INSN_IMM13(offset));
-    else {
-        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_I5, offset);
-        tcg_out32(s, op | INSN_RD(ret) | INSN_RS1(TCG_REG_I5) |
-                  INSN_RS2(addr));
+    } else {
+        tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_T1, offset);
+        tcg_out_ldst_rr(s, ret, addr, TCG_REG_T1, op);
     }
 }
 
-static inline void tcg_out_ldst_asi(TCGContext *s, int ret, int addr,
-                                    int offset, int op, int asi)
-{
-    tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_I5, offset);
-    tcg_out32(s, op | INSN_RD(ret) | INSN_RS1(TCG_REG_I5) |
-              INSN_ASI(asi) | INSN_RS2(addr));
-}
-
 static inline void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret,
                               TCGReg arg1, tcg_target_long arg2)
 {
-    if (type == TCG_TYPE_I32)
-        tcg_out_ldst(s, ret, arg1, arg2, LDUW);
-    else
-        tcg_out_ldst(s, ret, arg1, arg2, LDX);
+    tcg_out_ldst(s, ret, arg1, arg2, (type == TCG_TYPE_I32 ? LDUW : LDX));
 }
 
 static inline void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg,
                               TCGReg arg1, tcg_target_long arg2)
 {
-    if (type == TCG_TYPE_I32)
-        tcg_out_ldst(s, arg, arg1, arg2, STW);
-    else
-        tcg_out_ldst(s, arg, arg1, arg2, STX);
+    tcg_out_ldst(s, arg, arg1, arg2, (type == TCG_TYPE_I32 ? STW : STX));
+}
+
+static inline void tcg_out_ld_ptr(TCGContext *s, int ret,
+                                  tcg_target_long arg)
+{
+    if (!check_fit_tl(arg, 10)) {
+        tcg_out_movi(s, TCG_TYPE_PTR, ret, arg & ~0x3ff);
+    }
+    tcg_out_ld(s, TCG_TYPE_PTR, ret, ret, arg & 0x3ff);
 }
 
 static inline void tcg_out_sety(TCGContext *s, int rs)
@@ -442,20 +472,21 @@ static inline void tcg_out_addi(TCGContext *s, int reg, tcg_target_long val)
         if (check_fit_tl(val, 13))
             tcg_out_arithi(s, reg, reg, val, ARITH_ADD);
         else {
-            tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_I5, val);
-            tcg_out_arith(s, reg, reg, TCG_REG_I5, ARITH_ADD);
+            tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_T1, val);
+            tcg_out_arith(s, reg, reg, TCG_REG_T1, ARITH_ADD);
         }
     }
 }
 
-static inline void tcg_out_andi(TCGContext *s, int reg, tcg_target_long val)
+static inline void tcg_out_andi(TCGContext *s, int rd, int rs,
+                                tcg_target_long val)
 {
     if (val != 0) {
         if (check_fit_tl(val, 13))
-            tcg_out_arithi(s, reg, reg, val, ARITH_AND);
+            tcg_out_arithi(s, rd, rs, val, ARITH_AND);
         else {
-            tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_I5, val);
-            tcg_out_arith(s, reg, reg, TCG_REG_I5, ARITH_AND);
+            tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_T1, val);
+            tcg_out_arith(s, rd, rs, TCG_REG_T1, ARITH_AND);
         }
     }
 }
@@ -467,8 +498,8 @@ static void tcg_out_div32(TCGContext *s, int rd, int rs1,
     if (uns) {
         tcg_out_sety(s, TCG_REG_G0);
     } else {
-        tcg_out_arithi(s, TCG_REG_I5, rs1, 31, SHIFT_SRA);
-        tcg_out_sety(s, TCG_REG_I5);
+        tcg_out_arithi(s, TCG_REG_T1, rs1, 31, SHIFT_SRA);
+        tcg_out_sety(s, TCG_REG_T1);
     }
 
     tcg_out_arithc(s, rd, rs1, val2, val2const,
@@ -480,37 +511,7 @@ static inline void tcg_out_nop(TCGContext *s)
     tcg_out_sethi(s, TCG_REG_G0, 0);
 }
 
-static void tcg_out_branch_i32(TCGContext *s, int opc, int label_index)
-{
-    TCGLabel *l = &s->labels[label_index];
-
-    if (l->has_value) {
-        tcg_out32(s, (INSN_OP(0) | INSN_COND(opc, 0) | INSN_OP2(0x2)
-                      | INSN_OFF22(l->u.value - (unsigned long)s->code_ptr)));
-    } else {
-        tcg_out_reloc(s, s->code_ptr, R_SPARC_WDISP22, label_index, 0);
-        tcg_out32(s, (INSN_OP(0) | INSN_COND(opc, 0) | INSN_OP2(0x2) | 0));
-    }
-}
-
-#if TCG_TARGET_REG_BITS == 64
-static void tcg_out_branch_i64(TCGContext *s, int opc, int label_index)
-{
-    TCGLabel *l = &s->labels[label_index];
-
-    if (l->has_value) {
-        tcg_out32(s, (INSN_OP(0) | INSN_COND(opc, 0) | INSN_OP2(0x1) |
-                      (0x5 << 19) |
-                      INSN_OFF19(l->u.value - (unsigned long)s->code_ptr)));
-    } else {
-        tcg_out_reloc(s, s->code_ptr, R_SPARC_WDISP19, label_index, 0);
-        tcg_out32(s, (INSN_OP(0) | INSN_COND(opc, 0) | INSN_OP2(0x1) |
-                      (0x5 << 19) | 0));
-    }
-}
-#endif
-
-static const uint8_t tcg_cond_to_bcond[10] = {
+static const uint8_t tcg_cond_to_bcond[] = {
     [TCG_COND_EQ] = COND_E,
     [TCG_COND_NE] = COND_NE,
     [TCG_COND_LT] = COND_L,
@@ -523,70 +524,144 @@ static const uint8_t tcg_cond_to_bcond[10] = {
     [TCG_COND_GTU] = COND_GU,
 };
 
+static const uint8_t tcg_cond_to_rcond[] = {
+    [TCG_COND_EQ] = RCOND_Z,
+    [TCG_COND_NE] = RCOND_NZ,
+    [TCG_COND_LT] = RCOND_LZ,
+    [TCG_COND_GT] = RCOND_GZ,
+    [TCG_COND_LE] = RCOND_LEZ,
+    [TCG_COND_GE] = RCOND_GEZ
+};
+
+static void tcg_out_bpcc0(TCGContext *s, int scond, int flags, int off19)
+{
+    tcg_out32(s, INSN_OP(0) | INSN_OP2(1) | INSN_COND(scond) | flags | off19);
+}
+
+static void tcg_out_bpcc(TCGContext *s, int scond, int flags, int label)
+{
+    TCGLabel *l = &s->labels[label];
+    int off19;
+
+    if (l->has_value) {
+        off19 = INSN_OFF19(l->u.value - (unsigned long)s->code_ptr);
+    } else {
+        /* Make sure to preserve destinations during retranslation.  */
+        off19 = *(uint32_t *)s->code_ptr & INSN_OFF19(-1);
+        tcg_out_reloc(s, s->code_ptr, R_SPARC_WDISP19, label, 0);
+    }
+    tcg_out_bpcc0(s, scond, flags, off19);
+}
+
 static void tcg_out_cmp(TCGContext *s, TCGArg c1, TCGArg c2, int c2const)
 {
     tcg_out_arithc(s, TCG_REG_G0, c1, c2, c2const, ARITH_SUBCC);
 }
 
-static void tcg_out_brcond_i32(TCGContext *s, TCGCond cond,
-                               TCGArg arg1, TCGArg arg2, int const_arg2,
-                               int label_index)
+static void tcg_out_brcond_i32(TCGContext *s, TCGCond cond, TCGArg arg1,
+                               TCGArg arg2, int const_arg2, int label)
 {
     tcg_out_cmp(s, arg1, arg2, const_arg2);
-    tcg_out_branch_i32(s, tcg_cond_to_bcond[cond], label_index);
+    tcg_out_bpcc(s, tcg_cond_to_bcond[cond], BPCC_ICC | BPCC_PT, label);
     tcg_out_nop(s);
 }
 
+static void tcg_out_movcc(TCGContext *s, TCGCond cond, int cc, TCGArg ret,
+                          TCGArg v1, int v1const)
+{
+    tcg_out32(s, ARITH_MOVCC | cc | INSN_RD(ret)
+              | INSN_RS1(tcg_cond_to_bcond[cond])
+              | (v1const ? INSN_IMM11(v1) : INSN_RS2(v1)));
+}
+
+static void tcg_out_movcond_i32(TCGContext *s, TCGCond cond, TCGArg ret,
+                                TCGArg c1, TCGArg c2, int c2const,
+                                TCGArg v1, int v1const)
+{
+    tcg_out_cmp(s, c1, c2, c2const);
+    tcg_out_movcc(s, cond, MOVCC_ICC, ret, v1, v1const);
+}
+
 #if TCG_TARGET_REG_BITS == 64
-static void tcg_out_brcond_i64(TCGContext *s, TCGCond cond,
-                               TCGArg arg1, TCGArg arg2, int const_arg2,
-                               int label_index)
+static void tcg_out_brcond_i64(TCGContext *s, TCGCond cond, TCGArg arg1,
+                               TCGArg arg2, int const_arg2, int label)
 {
-    tcg_out_cmp(s, arg1, arg2, const_arg2);
-    tcg_out_branch_i64(s, tcg_cond_to_bcond[cond], label_index);
+    /* For 64-bit signed comparisons vs zero, we can avoid the compare.  */
+    if (arg2 == 0 && !is_unsigned_cond(cond)) {
+        TCGLabel *l = &s->labels[label];
+        int off16;
+
+        if (l->has_value) {
+            off16 = INSN_OFF16(l->u.value - (unsigned long)s->code_ptr);
+        } else {
+            /* Make sure to preserve destinations during retranslation.  */
+            off16 = *(uint32_t *)s->code_ptr & INSN_OFF16(-1);
+            tcg_out_reloc(s, s->code_ptr, R_SPARC_WDISP16, label, 0);
+        }
+        tcg_out32(s, INSN_OP(0) | INSN_OP2(3) | BPR_PT | INSN_RS1(arg1)
+                  | INSN_COND(tcg_cond_to_rcond[cond]) | off16);
+    } else {
+        tcg_out_cmp(s, arg1, arg2, const_arg2);
+        tcg_out_bpcc(s, tcg_cond_to_bcond[cond], BPCC_XCC | BPCC_PT, label);
+    }
     tcg_out_nop(s);
 }
+
+static void tcg_out_movr(TCGContext *s, TCGCond cond, TCGArg ret, TCGArg c1,
+                         TCGArg v1, int v1const)
+{
+    tcg_out32(s, ARITH_MOVR | INSN_RD(ret) | INSN_RS1(c1)
+              | (tcg_cond_to_rcond[cond] << 10)
+              | (v1const ? INSN_IMM10(v1) : INSN_RS2(v1)));
+}
+
+static void tcg_out_movcond_i64(TCGContext *s, TCGCond cond, TCGArg ret,
+                                TCGArg c1, TCGArg c2, int c2const,
+                                TCGArg v1, int v1const)
+{
+    /* For 64-bit signed comparisons vs zero, we can avoid the compare.
+       Note that the immediate range is one bit smaller, so we must check
+       for that as well.  */
+    if (c2 == 0 && !is_unsigned_cond(cond)
+        && (!v1const || check_fit_tl(v1, 10))) {
+        tcg_out_movr(s, cond, ret, c1, v1, v1const);
+    } else {
+        tcg_out_cmp(s, c1, c2, c2const);
+        tcg_out_movcc(s, cond, MOVCC_XCC, ret, v1, v1const);
+    }
+}
 #else
 static void tcg_out_brcond2_i32(TCGContext *s, TCGCond cond,
                                 TCGArg al, TCGArg ah,
                                 TCGArg bl, int blconst,
                                 TCGArg bh, int bhconst, int label_dest)
 {
-    int cc, label_next = gen_new_label();
+    int scond, label_next = gen_new_label();
 
     tcg_out_cmp(s, ah, bh, bhconst);
 
     /* Note that we fill one of the delay slots with the second compare.  */
     switch (cond) {
     case TCG_COND_EQ:
-        cc = INSN_COND(tcg_cond_to_bcond[TCG_COND_NE], 0);
-        tcg_out_branch_i32(s, cc, label_next);
+        tcg_out_bpcc(s, COND_NE, BPCC_ICC | BPCC_PT, label_next);
         tcg_out_cmp(s, al, bl, blconst);
-        cc = INSN_COND(tcg_cond_to_bcond[TCG_COND_EQ], 0);
-        tcg_out_branch_i32(s, cc, label_dest);
+        tcg_out_bpcc(s, COND_E, BPCC_ICC | BPCC_PT, label_dest);
         break;
 
     case TCG_COND_NE:
-        cc = INSN_COND(tcg_cond_to_bcond[TCG_COND_NE], 0);
-        tcg_out_branch_i32(s, cc, label_dest);
+        tcg_out_bpcc(s, COND_NE, BPCC_ICC | BPCC_PT, label_dest);
         tcg_out_cmp(s, al, bl, blconst);
-        tcg_out_branch_i32(s, cc, label_dest);
+        tcg_out_bpcc(s, COND_NE, BPCC_ICC | BPCC_PT, label_dest);
         break;
 
     default:
-        /* ??? One could fairly easily special-case 64-bit unsigned
-           compares against 32-bit zero-extended constants.  For instance,
-           we know that (unsigned)AH < 0 is false and need not emit it.
-           Similarly, (unsigned)AH > 0 being true implies AH != 0, so the
-           second branch will never be taken.  */
-        cc = INSN_COND(tcg_cond_to_bcond[cond], 0);
-        tcg_out_branch_i32(s, cc, label_dest);
+        scond = tcg_cond_to_bcond[tcg_high_cond(cond)];
+        tcg_out_bpcc(s, scond, BPCC_ICC | BPCC_PT, label_dest);
         tcg_out_nop(s);
-        cc = INSN_COND(tcg_cond_to_bcond[TCG_COND_NE], 0);
-        tcg_out_branch_i32(s, cc, label_next);
+        tcg_out_bpcc(s, COND_NE, BPCC_ICC | BPCC_PT, label_next);
         tcg_out_cmp(s, al, bl, blconst);
-        cc = INSN_COND(tcg_cond_to_bcond[tcg_unsigned_cond(cond)], 0);
-        tcg_out_branch_i32(s, cc, label_dest);
+        scond = tcg_cond_to_bcond[tcg_unsigned_cond(cond)];
+        tcg_out_bpcc(s, scond, BPCC_ICC | BPCC_PT, label_dest);
         break;
     }
     tcg_out_nop(s);
@@ -598,47 +673,42 @@ static void tcg_out_brcond2_i32(TCGContext *s, TCGCond cond,
 static void tcg_out_setcond_i32(TCGContext *s, TCGCond cond, TCGArg ret,
                                 TCGArg c1, TCGArg c2, int c2const)
 {
-    TCGArg t;
-
     /* For 32-bit comparisons, we can play games with ADDX/SUBX.  */
     switch (cond) {
+    case TCG_COND_LTU:
+    case TCG_COND_GEU:
+        /* The result of the comparison is in the carry bit.  */
+        break;
+
     case TCG_COND_EQ:
     case TCG_COND_NE:
+        /* For equality, we can transform to inequality vs zero.  */
         if (c2 != 0) {
             tcg_out_arithc(s, ret, c1, c2, c2const, ARITH_XOR);
         }
         c1 = TCG_REG_G0, c2 = ret, c2const = 0;
-        cond = (cond == TCG_COND_EQ ? TCG_COND_LEU : TCG_COND_LTU);
+        cond = (cond == TCG_COND_EQ ? TCG_COND_GEU : TCG_COND_LTU);
 	break;
 
     case TCG_COND_GTU:
-    case TCG_COND_GEU:
-        if (c2const && c2 != 0) {
-            tcg_out_movi_imm13(s, TCG_REG_I5, c2);
-            c2 = TCG_REG_I5;
-        }
-        t = c1, c1 = c2, c2 = t, c2const = 0;
-        cond = tcg_swap_cond(cond);
-        break;
-
-    case TCG_COND_LTU:
     case TCG_COND_LEU:
-        break;
+        /* If we don't need to load a constant into a register, we can
+           swap the operands on GTU/LEU.  There's no benefit to loading
+           the constant into a temporary register.  */
+        if (!c2const || c2 == 0) {
+            TCGArg t = c1;
+            c1 = c2;
+            c2 = t;
+            c2const = 0;
+            cond = tcg_swap_cond(cond);
+            break;
+        }
+        /* FALLTHRU */
 
     default:
         tcg_out_cmp(s, c1, c2, c2const);
-#if defined(__sparc_v9__) || defined(__sparc_v8plus__)
         tcg_out_movi_imm13(s, ret, 0);
-        tcg_out32 (s, ARITH_MOVCC | INSN_RD(ret)
-                   | INSN_RS1(tcg_cond_to_bcond[cond])
-                   | MOVCC_ICC | INSN_IMM11(1));
-#else
-        t = gen_new_label();
-        tcg_out_branch_i32(s, INSN_COND(tcg_cond_to_bcond[cond], 1), t);
-        tcg_out_movi_imm13(s, ret, 1);
-        tcg_out_movi_imm13(s, ret, 0);
-        tcg_out_label(s, t, s->code_ptr);
-#endif
+        tcg_out_movcc(s, cond, MOVCC_ICC, ret, 1, 1);
         return;
     }
 
@@ -654,11 +724,16 @@ static void tcg_out_setcond_i32(TCGContext *s, TCGCond cond, TCGArg ret,
 static void tcg_out_setcond_i64(TCGContext *s, TCGCond cond, TCGArg ret,
                                 TCGArg c1, TCGArg c2, int c2const)
 {
-    tcg_out_cmp(s, c1, c2, c2const);
-    tcg_out_movi_imm13(s, ret, 0);
-    tcg_out32 (s, ARITH_MOVCC | INSN_RD(ret)
-               | INSN_RS1(tcg_cond_to_bcond[cond])
-               | MOVCC_XCC | INSN_IMM11(1));
+    /* For 64-bit signed comparisons vs zero, we can avoid the compare
+       if the input does not overlap the output.  */
+    if (c2 == 0 && !is_unsigned_cond(cond) && c1 != ret) {
+        tcg_out_movi_imm13(s, ret, 0);
+        tcg_out_movr(s, cond, ret, c1, 1, 1);
+    } else {
+        tcg_out_cmp(s, c1, c2, c2const);
+        tcg_out_movi_imm13(s, ret, 0);
+        tcg_out_movcc(s, cond, MOVCC_XCC, ret, 1, 1);
+    }
 }
 #else
 static void tcg_out_setcond2_i32(TCGContext *s, TCGCond cond, TCGArg ret,
@@ -666,56 +741,98 @@ static void tcg_out_setcond2_i32(TCGContext *s, TCGCond cond, TCGArg ret,
                                  TCGArg bl, int blconst,
                                  TCGArg bh, int bhconst)
 {
-    int lab;
+    int tmp = TCG_REG_T1;
+
+    /* Note that the low parts are fully consumed before tmp is set.  */
+    if (ret != ah && (bhconst || ret != bh)) {
+        tmp = ret;
+    }
 
     switch (cond) {
     case TCG_COND_EQ:
-        tcg_out_setcond_i32(s, TCG_COND_EQ, TCG_REG_I5, al, bl, blconst);
-        tcg_out_setcond_i32(s, TCG_COND_EQ, ret, ah, bh, bhconst);
-        tcg_out_arith(s, ret, ret, TCG_REG_I5, ARITH_AND);
-        break;
-
     case TCG_COND_NE:
-        tcg_out_setcond_i32(s, TCG_COND_NE, TCG_REG_I5, al, al, blconst);
-        tcg_out_setcond_i32(s, TCG_COND_NE, ret, ah, bh, bhconst);
-        tcg_out_arith(s, ret, ret, TCG_REG_I5, ARITH_OR);
+        if (bl == 0 && bh == 0) {
+            if (cond == TCG_COND_EQ) {
+                tcg_out_arith(s, TCG_REG_G0, al, ah, ARITH_ORCC);
+                tcg_out_movi(s, TCG_TYPE_I32, ret, 1);
+            } else {
+                tcg_out_arith(s, ret, al, ah, ARITH_ORCC);
+            }
+        } else {
+            tcg_out_setcond_i32(s, cond, tmp, al, bl, blconst);
+            tcg_out_cmp(s, ah, bh, bhconst);
+            tcg_out_mov(s, TCG_TYPE_I32, ret, tmp);
+        }
+        tcg_out_movcc(s, TCG_COND_NE, MOVCC_ICC, ret, cond == TCG_COND_NE, 1);
         break;
 
     default:
-        lab = gen_new_label();
-
+        /* <= : ah < bh | (ah == bh && al <= bl) */
+        tcg_out_setcond_i32(s, tcg_unsigned_cond(cond), tmp, al, bl, blconst);
         tcg_out_cmp(s, ah, bh, bhconst);
-        tcg_out_branch_i32(s, INSN_COND(tcg_cond_to_bcond[cond], 1), lab);
-        tcg_out_movi_imm13(s, ret, 1);
-        tcg_out_branch_i32(s, INSN_COND(COND_NE, 1), lab);
-        tcg_out_movi_imm13(s, ret, 0);
+        tcg_out_mov(s, TCG_TYPE_I32, ret, tmp);
+        tcg_out_movcc(s, TCG_COND_NE, MOVCC_ICC, ret, 0, 1);
+        tcg_out_movcc(s, tcg_high_cond(cond), MOVCC_ICC, ret, 1, 1);
+        break;
+    }
+}
 
-        tcg_out_setcond_i32(s, tcg_unsigned_cond(cond), ret, al, bl, blconst);
+static void tcg_out_addsub2(TCGContext *s, TCGArg rl, TCGArg rh,
+                            TCGArg al, TCGArg ah, TCGArg bl, int blconst,
+                            TCGArg bh, int bhconst, int opl, int oph)
+{
+    TCGArg tmp = TCG_REG_T1;
 
-        tcg_out_label(s, lab, s->code_ptr);
-        break;
+    /* Note that the low parts are fully consumed before tmp is set.  */
+    if (rl != ah && (bhconst || rl != bh)) {
+        tmp = rl;
     }
+
+    tcg_out_arithc(s, tmp, al, bl, blconst, opl);
+    tcg_out_arithc(s, rh, ah, bh, bhconst, oph);
+    tcg_out_mov(s, TCG_TYPE_I32, rl, tmp);
 }
 #endif
 
 /* Generate global QEMU prologue and epilogue code */
 static void tcg_target_qemu_prologue(TCGContext *s)
 {
-    tcg_set_frame(s, TCG_REG_I6, TCG_TARGET_CALL_STACK_OFFSET,
-                  CPU_TEMP_BUF_NLONGS * (int)sizeof(long));
+    int tmp_buf_size, frame_size;
+
+    /* The TCG temp buffer is at the top of the frame, immediately
+       below the frame pointer.  */
+    tmp_buf_size = CPU_TEMP_BUF_NLONGS * (int)sizeof(long);
+    tcg_set_frame(s, TCG_REG_I6, TCG_TARGET_STACK_BIAS - tmp_buf_size,
+                  tmp_buf_size);
+
+    /* TCG_TARGET_CALL_STACK_OFFSET includes the stack bias, but is
+       otherwise the minimal frame usable by callees.  */
+    frame_size = TCG_TARGET_CALL_STACK_OFFSET - TCG_TARGET_STACK_BIAS;
+    frame_size += TCG_STATIC_CALL_ARGS_SIZE + tmp_buf_size;
+    frame_size += TCG_TARGET_STACK_ALIGN - 1;
+    frame_size &= -TCG_TARGET_STACK_ALIGN;
     tcg_out32(s, SAVE | INSN_RD(TCG_REG_O6) | INSN_RS1(TCG_REG_O6) |
-              INSN_IMM13(-(TCG_TARGET_STACK_MINFRAME +
-                           CPU_TEMP_BUF_NLONGS * (int)sizeof(long))));
+              INSN_IMM13(-frame_size));
+
+#ifdef CONFIG_USE_GUEST_BASE
+    if (GUEST_BASE != 0) {
+        tcg_out_movi(s, TCG_TYPE_PTR, TCG_GUEST_BASE_REG, GUEST_BASE);
+        tcg_regset_set_reg(s->reserved_regs, TCG_GUEST_BASE_REG);
+    }
+#endif
+
     tcg_out32(s, JMPL | INSN_RD(TCG_REG_G0) | INSN_RS1(TCG_REG_I1) |
               INSN_RS2(TCG_REG_G0));
-    tcg_out_mov(s, TCG_TYPE_PTR, TCG_AREG0, TCG_REG_I0);
+    /* delay slot */
+    tcg_out_nop(s);
+
+    /* No epilogue required.  We issue ret + restore directly in the TB.  */
 }
 
 #if defined(CONFIG_SOFTMMU)
 
 #include "../../softmmu_defs.h"
 
-#ifdef CONFIG_TCG_PASS_AREG0
 /* helper signature: helper_ld_mmu(CPUState *env, target_ulong addr,
    int mmu_idx) */
 static const void * const qemu_ld_helpers[4] = {
@@ -733,441 +850,307 @@ static const void * const qemu_st_helpers[4] = {
     helper_stl_mmu,
     helper_stq_mmu,
 };
-#else
-/* legacy helper signature: __ld_mmu(target_ulong addr, int
-   mmu_idx) */
-static const void * const qemu_ld_helpers[4] = {
-    __ldb_mmu,
-    __ldw_mmu,
-    __ldl_mmu,
-    __ldq_mmu,
-};
 
-/* legacy helper signature: __st_mmu(target_ulong addr, uintxx_t val,
-   int mmu_idx) */
-static const void * const qemu_st_helpers[4] = {
-    __stb_mmu,
-    __stw_mmu,
-    __stl_mmu,
-    __stq_mmu,
-};
-#endif
-#endif
+/* Perform the TLB load and compare.
 
-#if TARGET_LONG_BITS == 32
-#define TARGET_LD_OP LDUW
-#else
-#define TARGET_LD_OP LDX
-#endif
+   Inputs:
+   ADDRLO_IDX contains the index into ARGS of the low part of the
+   address; the high part of the address is at ADDR_LOW_IDX+1.
 
-#if defined(CONFIG_SOFTMMU)
-#if HOST_LONG_BITS == 32
-#define TARGET_ADDEND_LD_OP LDUW
+   MEM_INDEX and S_BITS are the memory context and log2 size of the load.
+
+   WHICH is the offset into the CPUTLBEntry structure of the slot to read.
+   This should be offsetof addr_read or addr_write.
+
+   The result of the TLB comparison is in %[ix]cc.  The sanitized address
+   is in the returned register, maybe %o0.  The TLB addend is in %o1.  */
+
+static int tcg_out_tlb_load(TCGContext *s, int addrlo_idx, int mem_index,
+                            int s_bits, const TCGArg *args, int which)
+{
+    const int addrlo = args[addrlo_idx];
+    const int r0 = TCG_REG_O0;
+    const int r1 = TCG_REG_O1;
+    const int r2 = TCG_REG_O2;
+    int addr = addrlo;
+    int tlb_ofs;
+
+    if (TCG_TARGET_REG_BITS == 32 && TARGET_LONG_BITS == 64) {
+        /* Assemble the 64-bit address in R0.  */
+        tcg_out_arithi(s, r0, addrlo, 0, SHIFT_SRL);
+        tcg_out_arithi(s, r1, args[addrlo_idx + 1], 32, SHIFT_SLLX);
+        tcg_out_arith(s, r0, r0, r1, ARITH_OR);
+    }
+
+    /* Shift the page number down to tlb-entry.  */
+    tcg_out_arithi(s, r1, addrlo,
+                   TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS, SHIFT_SRL);
+
+    /* Mask out the page offset, except for the required alignment.  */
+    tcg_out_andi(s, r0, addr, TARGET_PAGE_MASK | ((1 << s_bits) - 1));
+
+    /* Compute tlb index, modulo tlb size.  */
+    tcg_out_andi(s, r1, r1, (CPU_TLB_SIZE - 1) << CPU_TLB_ENTRY_BITS);
+
+    /* Relative to the current ENV.  */
+    tcg_out_arith(s, r1, TCG_AREG0, r1, ARITH_ADD);
+
+    /* Find a base address that can load both tlb comparator and addend.  */
+    tlb_ofs = offsetof(CPUArchState, tlb_table[mem_index][0]);
+    if (!check_fit_tl(tlb_ofs + sizeof(CPUTLBEntry), 13)) {
+        tcg_out_addi(s, r1, tlb_ofs);
+        tlb_ofs = 0;
+    }
+
+    /* Load the tlb comparator and the addend.  */
+    tcg_out_ld(s, TCG_TYPE_TL, r2, r1, tlb_ofs + which);
+    tcg_out_ld(s, TCG_TYPE_PTR, r1, r1, tlb_ofs+offsetof(CPUTLBEntry, addend));
+
+    /* subcc arg0, arg2, %g0 */
+    tcg_out_cmp(s, r0, r2, 0);
+
+    /* If the guest address must be zero-extended, do so now.  */
+    if (TCG_TARGET_REG_BITS == 64 && TARGET_LONG_BITS == 32) {
+        tcg_out_arithi(s, r0, addrlo, 0, SHIFT_SRL);
+        return r0;
+    }
+    return addrlo;
+}
+#endif /* CONFIG_SOFTMMU */
+
+static const int qemu_ld_opc[8] = {
+#ifdef TARGET_WORDS_BIGENDIAN
+    LDUB, LDUH, LDUW, LDX, LDSB, LDSH, LDSW, LDX
 #else
-#define TARGET_ADDEND_LD_OP LDX
-#endif
+    LDUB, LDUH_LE, LDUW_LE, LDX_LE, LDSB, LDSH_LE, LDSW_LE, LDX_LE
 #endif
+};
 
-#ifdef __arch64__
-#define HOST_LD_OP LDX
-#define HOST_ST_OP STX
-#define HOST_SLL_OP SHIFT_SLLX
-#define HOST_SRA_OP SHIFT_SRAX
+static const int qemu_st_opc[4] = {
+#ifdef TARGET_WORDS_BIGENDIAN
+    STB, STH, STW, STX
 #else
-#define HOST_LD_OP LDUW
-#define HOST_ST_OP STW
-#define HOST_SLL_OP SHIFT_SLL
-#define HOST_SRA_OP SHIFT_SRA
+    STB, STH_LE, STW_LE, STX_LE
 #endif
+};
 
-static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args,
-                            int opc)
+static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, int sizeop)
 {
-    int addr_reg, data_reg, arg0, arg1, arg2, mem_index, s_bits;
+    int addrlo_idx = 1, datalo, datahi, addr_reg;
 #if defined(CONFIG_SOFTMMU)
-    uint32_t *label1_ptr, *label2_ptr;
+    int memi_idx, memi, s_bits, n;
+    uint32_t *label_ptr[2];
 #endif
 
-    data_reg = *args++;
-    addr_reg = *args++;
-    mem_index = *args;
-    s_bits = opc & 3;
-
-    arg0 = TCG_REG_O0;
-    arg1 = TCG_REG_O1;
-    arg2 = TCG_REG_O2;
+    datahi = datalo = args[0];
+    if (TCG_TARGET_REG_BITS == 32 && sizeop == 3) {
+        datahi = args[1];
+        addrlo_idx = 2;
+    }
 
 #if defined(CONFIG_SOFTMMU)
-    /* srl addr_reg, x, arg1 */
-    tcg_out_arithi(s, arg1, addr_reg, TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS,
-                   SHIFT_SRL);
-    /* and addr_reg, x, arg0 */
-    tcg_out_arithi(s, arg0, addr_reg, TARGET_PAGE_MASK | ((1 << s_bits) - 1),
-                   ARITH_AND);
-
-    /* and arg1, x, arg1 */
-    tcg_out_andi(s, arg1, (CPU_TLB_SIZE - 1) << CPU_TLB_ENTRY_BITS);
-
-    /* add arg1, x, arg1 */
-    tcg_out_addi(s, arg1, offsetof(CPUArchState,
-                                   tlb_table[mem_index][0].addr_read));
+    memi_idx = addrlo_idx + 1 + (TARGET_LONG_BITS > TCG_TARGET_REG_BITS);
+    memi = args[memi_idx];
+    s_bits = sizeop & 3;
+
+    addr_reg = tcg_out_tlb_load(s, addrlo_idx, memi, s_bits, args,
+                                offsetof(CPUTLBEntry, addr_read));
+
+    if (TCG_TARGET_REG_BITS == 32 && sizeop == 3) {
+        int reg64;
+
+        /* bne,pn %[xi]cc, label0 */
+        label_ptr[0] = (uint32_t *)s->code_ptr;
+        tcg_out_bpcc0(s, COND_NE, BPCC_PN
+                      | (TARGET_LONG_BITS == 64 ? BPCC_XCC : BPCC_ICC), 0);
+
+        /* TLB Hit.  */
+        /* Load all 64-bits into an O/G register.  */
+        reg64 = (datalo < 16 ? datalo : TCG_REG_O0);
+        tcg_out_ldst_rr(s, reg64, addr_reg, TCG_REG_O1, qemu_ld_opc[sizeop]);
+
+        /* Move the two 32-bit pieces into the destination registers.  */
+        tcg_out_arithi(s, datahi, reg64, 32, SHIFT_SRLX);
+        if (reg64 != datalo) {
+            tcg_out_mov(s, TCG_TYPE_I32, datalo, reg64);
+        }
 
-    /* add env, arg1, arg1 */
-    tcg_out_arith(s, arg1, TCG_AREG0, arg1, ARITH_ADD);
+        /* b,a,pt label1 */
+        label_ptr[1] = (uint32_t *)s->code_ptr;
+        tcg_out_bpcc0(s, COND_A, BPCC_A | BPCC_PT, 0);
+    } else {
+        /* The fast path is exactly one insn.  Thus we can perform the
+           entire TLB Hit in the (annulled) delay slot of the branch
+           over the TLB Miss case.  */
+
+        /* beq,a,pt %[xi]cc, label0 */
+        label_ptr[0] = NULL;
+        label_ptr[1] = (uint32_t *)s->code_ptr;
+        tcg_out_bpcc0(s, COND_E, BPCC_A | BPCC_PT
+                      | (TARGET_LONG_BITS == 64 ? BPCC_XCC : BPCC_ICC), 0);
+        /* delay slot */
+        tcg_out_ldst_rr(s, datalo, addr_reg, TCG_REG_O1, qemu_ld_opc[sizeop]);
+    }
 
-    /* ld [arg1], arg2 */
-    tcg_out32(s, TARGET_LD_OP | INSN_RD(arg2) | INSN_RS1(arg1) |
-              INSN_RS2(TCG_REG_G0));
+    /* TLB Miss.  */
 
-    /* subcc arg0, arg2, %g0 */
-    tcg_out_arith(s, TCG_REG_G0, arg0, arg2, ARITH_SUBCC);
-
-    /* will become:
-       be label1
-        or
-       be,pt %xcc label1 */
-    label1_ptr = (uint32_t *)s->code_ptr;
-    tcg_out32(s, 0);
-
-    /* mov (delay slot) */
-    tcg_out_mov(s, TCG_TYPE_PTR, arg0, addr_reg);
-
-    /* mov */
-    tcg_out_movi(s, TCG_TYPE_I32, arg1, mem_index);
-#ifdef CONFIG_TCG_PASS_AREG0
-    /* XXX/FIXME: suboptimal */
-    tcg_out_mov(s, TCG_TYPE_I32, tcg_target_call_iarg_regs[3],
-                tcg_target_call_iarg_regs[2]);
-    tcg_out_mov(s, TCG_TYPE_I64, tcg_target_call_iarg_regs[2],
-                tcg_target_call_iarg_regs[1]);
-    tcg_out_mov(s, TCG_TYPE_TL, tcg_target_call_iarg_regs[1],
-                tcg_target_call_iarg_regs[0]);
-    tcg_out_mov(s, TCG_TYPE_PTR, tcg_target_call_iarg_regs[0],
-                TCG_AREG0);
-#endif
+    if (label_ptr[0]) {
+        *label_ptr[0] |= INSN_OFF19((unsigned long)s->code_ptr -
+                                    (unsigned long)label_ptr[0]);
+    }
+    n = 0;
+    tcg_out_mov(s, TCG_TYPE_PTR, tcg_target_call_iarg_regs[n++], TCG_AREG0);
+    if (TARGET_LONG_BITS > TCG_TARGET_REG_BITS) {
+        tcg_out_mov(s, TCG_TYPE_REG, tcg_target_call_iarg_regs[n++],
+                    args[addrlo_idx + 1]);
+    }
+    tcg_out_mov(s, TCG_TYPE_REG, tcg_target_call_iarg_regs[n++],
+                args[addrlo_idx]);
 
-    /* XXX: move that code at the end of the TB */
     /* qemu_ld_helper[s_bits](arg0, arg1) */
     tcg_out32(s, CALL | ((((tcg_target_ulong)qemu_ld_helpers[s_bits]
                            - (tcg_target_ulong)s->code_ptr) >> 2)
                          & 0x3fffffff));
-    /* Store AREG0 in stack to avoid ugly glibc bugs that mangle
-       global registers */
-    // delay slot
-    tcg_out_ldst(s, TCG_AREG0, TCG_REG_CALL_STACK,
-                 TCG_TARGET_CALL_STACK_OFFSET - TCG_STATIC_CALL_ARGS_SIZE -
-                 sizeof(long), HOST_ST_OP);
-    tcg_out_ldst(s, TCG_AREG0, TCG_REG_CALL_STACK,
-                 TCG_TARGET_CALL_STACK_OFFSET - TCG_STATIC_CALL_ARGS_SIZE -
-                 sizeof(long), HOST_LD_OP);
-
-    /* data_reg = sign_extend(arg0) */
-    switch(opc) {
+    /* delay slot */
+    tcg_out_movi(s, TCG_TYPE_I32, tcg_target_call_iarg_regs[n], memi);
+
+    n = tcg_target_call_oarg_regs[0];
+    /* datalo = sign_extend(arg0) */
+    switch (sizeop) {
     case 0 | 4:
-        /* sll arg0, 24/56, data_reg */
-        tcg_out_arithi(s, data_reg, arg0, (int)sizeof(tcg_target_long) * 8 - 8,
-                       HOST_SLL_OP);
-        /* sra data_reg, 24/56, data_reg */
-        tcg_out_arithi(s, data_reg, data_reg,
-                       (int)sizeof(tcg_target_long) * 8 - 8, HOST_SRA_OP);
+        /* Recall that SRA sign extends from bit 31 through bit 63.  */
+        tcg_out_arithi(s, datalo, n, 24, SHIFT_SLL);
+        tcg_out_arithi(s, datalo, datalo, 24, SHIFT_SRA);
         break;
     case 1 | 4:
-        /* sll arg0, 16/48, data_reg */
-        tcg_out_arithi(s, data_reg, arg0,
-                       (int)sizeof(tcg_target_long) * 8 - 16, HOST_SLL_OP);
-        /* sra data_reg, 16/48, data_reg */
-        tcg_out_arithi(s, data_reg, data_reg,
-                       (int)sizeof(tcg_target_long) * 8 - 16, HOST_SRA_OP);
+        tcg_out_arithi(s, datalo, n, 16, SHIFT_SLL);
+        tcg_out_arithi(s, datalo, datalo, 16, SHIFT_SRA);
         break;
     case 2 | 4:
-        /* sll arg0, 32, data_reg */
-        tcg_out_arithi(s, data_reg, arg0, 32, HOST_SLL_OP);
-        /* sra data_reg, 32, data_reg */
-        tcg_out_arithi(s, data_reg, data_reg, 32, HOST_SRA_OP);
+        tcg_out_arithi(s, datalo, n, 0, SHIFT_SRA);
         break;
+    case 3:
+        if (TCG_TARGET_REG_BITS == 32) {
+            tcg_out_mov(s, TCG_TYPE_REG, datahi, n);
+            tcg_out_mov(s, TCG_TYPE_REG, datalo, n + 1);
+            break;
+        }
+        /* FALLTHRU */
     case 0:
     case 1:
     case 2:
-    case 3:
     default:
         /* mov */
-        tcg_out_mov(s, TCG_TYPE_REG, data_reg, arg0);
+        tcg_out_mov(s, TCG_TYPE_REG, datalo, n);
         break;
     }
 
-    /* will become:
-       ba label2 */
-    label2_ptr = (uint32_t *)s->code_ptr;
-    tcg_out32(s, 0);
-
-    /* nop (delay slot */
-    tcg_out_nop(s);
-
-    /* label1: */
-#if TARGET_LONG_BITS == 32
-    /* be label1 */
-    *label1_ptr = (INSN_OP(0) | INSN_COND(COND_E, 0) | INSN_OP2(0x2) |
-                   INSN_OFF22((unsigned long)s->code_ptr -
-                              (unsigned long)label1_ptr));
-#else
-    /* be,pt %xcc label1 */
-    *label1_ptr = (INSN_OP(0) | INSN_COND(COND_E, 0) | INSN_OP2(0x1) |
-                   (0x5 << 19) | INSN_OFF19((unsigned long)s->code_ptr -
-                              (unsigned long)label1_ptr));
-#endif
-
-    /* ld [arg1 + x], arg1 */
-    tcg_out_ldst(s, arg1, arg1, offsetof(CPUTLBEntry, addend) -
-                 offsetof(CPUTLBEntry, addr_read), TARGET_ADDEND_LD_OP);
-
-#if TARGET_LONG_BITS == 32
-    /* and addr_reg, x, arg0 */
-    tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_I5, 0xffffffff);
-    tcg_out_arith(s, arg0, addr_reg, TCG_REG_I5, ARITH_AND);
-    /* add arg0, arg1, arg0 */
-    tcg_out_arith(s, arg0, arg0, arg1, ARITH_ADD);
+    *label_ptr[1] |= INSN_OFF19((unsigned long)s->code_ptr -
+                                (unsigned long)label_ptr[1]);
 #else
-    /* add addr_reg, arg1, arg0 */
-    tcg_out_arith(s, arg0, addr_reg, arg1, ARITH_ADD);
-#endif
+    addr_reg = args[addrlo_idx];
+    if (TCG_TARGET_REG_BITS == 64 && TARGET_LONG_BITS == 32) {
+        tcg_out_arithi(s, TCG_REG_T1, addr_reg, 0, SHIFT_SRL);
+        addr_reg = TCG_REG_T1;
+    }
+    if (TCG_TARGET_REG_BITS == 32 && sizeop == 3) {
+        int reg64 = (datalo < 16 ? datalo : TCG_REG_O0);
 
-#else
-    arg0 = addr_reg;
-#endif
+        tcg_out_ldst_rr(s, reg64, addr_reg,
+                        (GUEST_BASE ? TCG_GUEST_BASE_REG : TCG_REG_G0),
+                        qemu_ld_opc[sizeop]);
 
-    switch(opc) {
-    case 0:
-        /* ldub [arg0], data_reg */
-        tcg_out_ldst(s, data_reg, arg0, 0, LDUB);
-        break;
-    case 0 | 4:
-        /* ldsb [arg0], data_reg */
-        tcg_out_ldst(s, data_reg, arg0, 0, LDSB);
-        break;
-    case 1:
-#ifdef TARGET_WORDS_BIGENDIAN
-        /* lduh [arg0], data_reg */
-        tcg_out_ldst(s, data_reg, arg0, 0, LDUH);
-#else
-        /* lduha [arg0] ASI_PRIMARY_LITTLE, data_reg */
-        tcg_out_ldst_asi(s, data_reg, arg0, 0, LDUHA, ASI_PRIMARY_LITTLE);
-#endif
-        break;
-    case 1 | 4:
-#ifdef TARGET_WORDS_BIGENDIAN
-        /* ldsh [arg0], data_reg */
-        tcg_out_ldst(s, data_reg, arg0, 0, LDSH);
-#else
-        /* ldsha [arg0] ASI_PRIMARY_LITTLE, data_reg */
-        tcg_out_ldst_asi(s, data_reg, arg0, 0, LDSHA, ASI_PRIMARY_LITTLE);
-#endif
-        break;
-    case 2:
-#ifdef TARGET_WORDS_BIGENDIAN
-        /* lduw [arg0], data_reg */
-        tcg_out_ldst(s, data_reg, arg0, 0, LDUW);
-#else
-        /* lduwa [arg0] ASI_PRIMARY_LITTLE, data_reg */
-        tcg_out_ldst_asi(s, data_reg, arg0, 0, LDUWA, ASI_PRIMARY_LITTLE);
-#endif
-        break;
-    case 2 | 4:
-#ifdef TARGET_WORDS_BIGENDIAN
-        /* ldsw [arg0], data_reg */
-        tcg_out_ldst(s, data_reg, arg0, 0, LDSW);
-#else
-        /* ldswa [arg0] ASI_PRIMARY_LITTLE, data_reg */
-        tcg_out_ldst_asi(s, data_reg, arg0, 0, LDSWA, ASI_PRIMARY_LITTLE);
-#endif
-        break;
-    case 3:
-#ifdef TARGET_WORDS_BIGENDIAN
-        /* ldx [arg0], data_reg */
-        tcg_out_ldst(s, data_reg, arg0, 0, LDX);
-#else
-        /* ldxa [arg0] ASI_PRIMARY_LITTLE, data_reg */
-        tcg_out_ldst_asi(s, data_reg, arg0, 0, LDXA, ASI_PRIMARY_LITTLE);
-#endif
-        break;
-    default:
-        tcg_abort();
+        tcg_out_arithi(s, datahi, reg64, 32, SHIFT_SRLX);
+        if (reg64 != datalo) {
+            tcg_out_mov(s, TCG_TYPE_I32, datalo, reg64);
+        }
+    } else {
+        tcg_out_ldst_rr(s, datalo, addr_reg,
+                        (GUEST_BASE ? TCG_GUEST_BASE_REG : TCG_REG_G0),
+                        qemu_ld_opc[sizeop]);
     }
-
-#if defined(CONFIG_SOFTMMU)
-    /* label2: */
-    *label2_ptr = (INSN_OP(0) | INSN_COND(COND_A, 0) | INSN_OP2(0x2) |
-                   INSN_OFF22((unsigned long)s->code_ptr -
-                              (unsigned long)label2_ptr));
-#endif
+#endif /* CONFIG_SOFTMMU */
 }
 
-static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args,
-                            int opc)
+static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, int sizeop)
 {
-    int addr_reg, data_reg, arg0, arg1, arg2, mem_index, s_bits;
+    int addrlo_idx = 1, datalo, datahi, addr_reg;
 #if defined(CONFIG_SOFTMMU)
-    uint32_t *label1_ptr, *label2_ptr;
+    int memi_idx, memi, n, datafull;
+    uint32_t *label_ptr;
 #endif
 
-    data_reg = *args++;
-    addr_reg = *args++;
-    mem_index = *args;
-
-    s_bits = opc;
-
-    arg0 = TCG_REG_O0;
-    arg1 = TCG_REG_O1;
-    arg2 = TCG_REG_O2;
+    datahi = datalo = args[0];
+    if (TCG_TARGET_REG_BITS == 32 && sizeop == 3) {
+        datahi = args[1];
+        addrlo_idx = 2;
+    }
 
 #if defined(CONFIG_SOFTMMU)
-    /* srl addr_reg, x, arg1 */
-    tcg_out_arithi(s, arg1, addr_reg, TARGET_PAGE_BITS - CPU_TLB_ENTRY_BITS,
-                   SHIFT_SRL);
-
-    /* and addr_reg, x, arg0 */
-    tcg_out_arithi(s, arg0, addr_reg, TARGET_PAGE_MASK | ((1 << s_bits) - 1),
-                   ARITH_AND);
-
-    /* and arg1, x, arg1 */
-    tcg_out_andi(s, arg1, (CPU_TLB_SIZE - 1) << CPU_TLB_ENTRY_BITS);
-
-    /* add arg1, x, arg1 */
-    tcg_out_addi(s, arg1, offsetof(CPUArchState,
-                                   tlb_table[mem_index][0].addr_write));
-
-    /* add env, arg1, arg1 */
-    tcg_out_arith(s, arg1, TCG_AREG0, arg1, ARITH_ADD);
+    memi_idx = addrlo_idx + 1 + (TARGET_LONG_BITS > TCG_TARGET_REG_BITS);
+    memi = args[memi_idx];
+
+    addr_reg = tcg_out_tlb_load(s, addrlo_idx, memi, sizeop, args,
+                                offsetof(CPUTLBEntry, addr_write));
+
+    datafull = datalo;
+    if (TCG_TARGET_REG_BITS == 32 && sizeop == 3) {
+        /* Reconstruct the full 64-bit value.  */
+        tcg_out_arithi(s, TCG_REG_T1, datalo, 0, SHIFT_SRL);
+        tcg_out_arithi(s, TCG_REG_O2, datahi, 32, SHIFT_SLLX);
+        tcg_out_arith(s, TCG_REG_O2, TCG_REG_T1, TCG_REG_O2, ARITH_OR);
+        datafull = TCG_REG_O2;
+    }
 
-    /* ld [arg1], arg2 */
-    tcg_out32(s, TARGET_LD_OP | INSN_RD(arg2) | INSN_RS1(arg1) |
-              INSN_RS2(TCG_REG_G0));
+    /* The fast path is exactly one insn.  Thus we can perform the entire
+       TLB Hit in the (annulled) delay slot of the branch over TLB Miss.  */
+    /* beq,a,pt %[xi]cc, label0 */
+    label_ptr = (uint32_t *)s->code_ptr;
+    tcg_out_bpcc0(s, COND_E, BPCC_A | BPCC_PT
+                  | (TARGET_LONG_BITS == 64 ? BPCC_XCC : BPCC_ICC), 0);
+    /* delay slot */
+    tcg_out_ldst_rr(s, datafull, addr_reg, TCG_REG_O1, qemu_st_opc[sizeop]);
+
+    /* TLB Miss.  */
+
+    n = 0;
+    tcg_out_mov(s, TCG_TYPE_PTR, tcg_target_call_iarg_regs[n++], TCG_AREG0);
+    if (TARGET_LONG_BITS > TCG_TARGET_REG_BITS) {
+        tcg_out_mov(s, TCG_TYPE_REG, tcg_target_call_iarg_regs[n++],
+                    args[addrlo_idx + 1]);
+    }
+    tcg_out_mov(s, TCG_TYPE_REG, tcg_target_call_iarg_regs[n++],
+                args[addrlo_idx]);
+    if (TCG_TARGET_REG_BITS == 32 && sizeop == 3) {
+        tcg_out_mov(s, TCG_TYPE_REG, tcg_target_call_iarg_regs[n++], datahi);
+    }
+    tcg_out_mov(s, TCG_TYPE_REG, tcg_target_call_iarg_regs[n++], datalo);
 
-    /* subcc arg0, arg2, %g0 */
-    tcg_out_arith(s, TCG_REG_G0, arg0, arg2, ARITH_SUBCC);
-
-    /* will become:
-       be label1
-        or
-       be,pt %xcc label1 */
-    label1_ptr = (uint32_t *)s->code_ptr;
-    tcg_out32(s, 0);
-
-    /* mov (delay slot) */
-    tcg_out_mov(s, TCG_TYPE_PTR, arg0, addr_reg);
-
-    /* mov */
-    tcg_out_mov(s, TCG_TYPE_REG, arg1, data_reg);
-
-    /* mov */
-    tcg_out_movi(s, TCG_TYPE_I32, arg2, mem_index);
-
-#ifdef CONFIG_TCG_PASS_AREG0
-    /* XXX/FIXME: suboptimal */
-    tcg_out_mov(s, TCG_TYPE_I32, tcg_target_call_iarg_regs[3],
-                tcg_target_call_iarg_regs[2]);
-    tcg_out_mov(s, TCG_TYPE_I64, tcg_target_call_iarg_regs[2],
-                tcg_target_call_iarg_regs[1]);
-    tcg_out_mov(s, TCG_TYPE_TL, tcg_target_call_iarg_regs[1],
-                tcg_target_call_iarg_regs[0]);
-    tcg_out_mov(s, TCG_TYPE_PTR, tcg_target_call_iarg_regs[0],
-                TCG_AREG0);
-#endif
-    /* XXX: move that code at the end of the TB */
     /* qemu_st_helper[s_bits](arg0, arg1, arg2) */
-    tcg_out32(s, CALL | ((((tcg_target_ulong)qemu_st_helpers[s_bits]
+    tcg_out32(s, CALL | ((((tcg_target_ulong)qemu_st_helpers[sizeop]
                            - (tcg_target_ulong)s->code_ptr) >> 2)
                          & 0x3fffffff));
-    /* Store AREG0 in stack to avoid ugly glibc bugs that mangle
-       global registers */
-    // delay slot
-    tcg_out_ldst(s, TCG_AREG0, TCG_REG_CALL_STACK,
-                 TCG_TARGET_CALL_STACK_OFFSET - TCG_STATIC_CALL_ARGS_SIZE -
-                 sizeof(long), HOST_ST_OP);
-    tcg_out_ldst(s, TCG_AREG0, TCG_REG_CALL_STACK,
-                 TCG_TARGET_CALL_STACK_OFFSET - TCG_STATIC_CALL_ARGS_SIZE -
-                 sizeof(long), HOST_LD_OP);
-
-    /* will become:
-       ba label2 */
-    label2_ptr = (uint32_t *)s->code_ptr;
-    tcg_out32(s, 0);
-
-    /* nop (delay slot) */
-    tcg_out_nop(s);
-
-#if TARGET_LONG_BITS == 32
-    /* be label1 */
-    *label1_ptr = (INSN_OP(0) | INSN_COND(COND_E, 0) | INSN_OP2(0x2) |
-                   INSN_OFF22((unsigned long)s->code_ptr -
-                              (unsigned long)label1_ptr));
-#else
-    /* be,pt %xcc label1 */
-    *label1_ptr = (INSN_OP(0) | INSN_COND(COND_E, 0) | INSN_OP2(0x1) |
-                   (0x5 << 19) | INSN_OFF19((unsigned long)s->code_ptr -
-                              (unsigned long)label1_ptr));
-#endif
-
-    /* ld [arg1 + x], arg1 */
-    tcg_out_ldst(s, arg1, arg1, offsetof(CPUTLBEntry, addend) -
-                 offsetof(CPUTLBEntry, addr_write), TARGET_ADDEND_LD_OP);
-
-#if TARGET_LONG_BITS == 32
-    /* and addr_reg, x, arg0 */
-    tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_I5, 0xffffffff);
-    tcg_out_arith(s, arg0, addr_reg, TCG_REG_I5, ARITH_AND);
-    /* add arg0, arg1, arg0 */
-    tcg_out_arith(s, arg0, arg0, arg1, ARITH_ADD);
-#else
-    /* add addr_reg, arg1, arg0 */
-    tcg_out_arith(s, arg0, addr_reg, arg1, ARITH_ADD);
-#endif
-
-#else
-    arg0 = addr_reg;
-#endif
+    /* delay slot */
+    tcg_out_movi(s, TCG_TYPE_REG, tcg_target_call_iarg_regs[n], memi);
 
-    switch(opc) {
-    case 0:
-        /* stb data_reg, [arg0] */
-        tcg_out_ldst(s, data_reg, arg0, 0, STB);
-        break;
-    case 1:
-#ifdef TARGET_WORDS_BIGENDIAN
-        /* sth data_reg, [arg0] */
-        tcg_out_ldst(s, data_reg, arg0, 0, STH);
-#else
-        /* stha data_reg, [arg0] ASI_PRIMARY_LITTLE */
-        tcg_out_ldst_asi(s, data_reg, arg0, 0, STHA, ASI_PRIMARY_LITTLE);
-#endif
-        break;
-    case 2:
-#ifdef TARGET_WORDS_BIGENDIAN
-        /* stw data_reg, [arg0] */
-        tcg_out_ldst(s, data_reg, arg0, 0, STW);
-#else
-        /* stwa data_reg, [arg0] ASI_PRIMARY_LITTLE */
-        tcg_out_ldst_asi(s, data_reg, arg0, 0, STWA, ASI_PRIMARY_LITTLE);
-#endif
-        break;
-    case 3:
-#ifdef TARGET_WORDS_BIGENDIAN
-        /* stx data_reg, [arg0] */
-        tcg_out_ldst(s, data_reg, arg0, 0, STX);
+    *label_ptr |= INSN_OFF19((unsigned long)s->code_ptr -
+                             (unsigned long)label_ptr);
 #else
-        /* stxa data_reg, [arg0] ASI_PRIMARY_LITTLE */
-        tcg_out_ldst_asi(s, data_reg, arg0, 0, STXA, ASI_PRIMARY_LITTLE);
-#endif
-        break;
-    default:
-        tcg_abort();
+    addr_reg = args[addrlo_idx];
+    if (TCG_TARGET_REG_BITS == 64 && TARGET_LONG_BITS == 32) {
+        tcg_out_arithi(s, TCG_REG_T1, addr_reg, 0, SHIFT_SRL);
+        addr_reg = TCG_REG_T1;
     }
-
-#if defined(CONFIG_SOFTMMU)
-    /* label2: */
-    *label2_ptr = (INSN_OP(0) | INSN_COND(COND_A, 0) | INSN_OP2(0x2) |
-                   INSN_OFF22((unsigned long)s->code_ptr -
-                              (unsigned long)label2_ptr));
-#endif
+    if (TCG_TARGET_REG_BITS == 32 && sizeop == 3) {
+        tcg_out_arithi(s, TCG_REG_T1, datalo, 0, SHIFT_SRL);
+        tcg_out_arithi(s, TCG_REG_O2, datahi, 32, SHIFT_SLLX);
+        tcg_out_arith(s, TCG_REG_O2, TCG_REG_T1, TCG_REG_O2, ARITH_OR);
+        datalo = TCG_REG_O2;
+    }
+    tcg_out_ldst_rr(s, datalo, addr_reg,
+                    (GUEST_BASE ? TCG_GUEST_BASE_REG : TCG_REG_G0),
+                    qemu_st_opc[sizeop]);
+#endif /* CONFIG_SOFTMMU */
 }
 
 static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
@@ -1186,43 +1169,36 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
     case INDEX_op_goto_tb:
         if (s->tb_jmp_offset) {
             /* direct jump method */
-            tcg_out_sethi(s, TCG_REG_I5, args[0] & 0xffffe000);
-            tcg_out32(s, JMPL | INSN_RD(TCG_REG_G0) | INSN_RS1(TCG_REG_I5) |
-                      INSN_IMM13((args[0] & 0x1fff)));
+            uint32_t old_insn = *(uint32_t *)s->code_ptr;
             s->tb_jmp_offset[args[0]] = s->code_ptr - s->code_buf;
+            /* Make sure to preserve links during retranslation.  */
+            tcg_out32(s, CALL | (old_insn & ~INSN_OP(-1)));
         } else {
             /* indirect jump method */
-            tcg_out_ld_ptr(s, TCG_REG_I5, (tcg_target_long)(s->tb_next + args[0]));
-            tcg_out32(s, JMPL | INSN_RD(TCG_REG_G0) | INSN_RS1(TCG_REG_I5) |
+            tcg_out_ld_ptr(s, TCG_REG_T1,
+                           (tcg_target_long)(s->tb_next + args[0]));
+            tcg_out32(s, JMPL | INSN_RD(TCG_REG_G0) | INSN_RS1(TCG_REG_T1) |
                       INSN_RS2(TCG_REG_G0));
         }
         tcg_out_nop(s);
         s->tb_next_offset[args[0]] = s->code_ptr - s->code_buf;
         break;
     case INDEX_op_call:
-        if (const_args[0])
+        if (const_args[0]) {
             tcg_out32(s, CALL | ((((tcg_target_ulong)args[0]
                                    - (tcg_target_ulong)s->code_ptr) >> 2)
                                  & 0x3fffffff));
-        else {
-            tcg_out_ld_ptr(s, TCG_REG_I5,
+        } else {
+            tcg_out_ld_ptr(s, TCG_REG_T1,
                            (tcg_target_long)(s->tb_next + args[0]));
-            tcg_out32(s, JMPL | INSN_RD(TCG_REG_O7) | INSN_RS1(TCG_REG_I5) |
+            tcg_out32(s, JMPL | INSN_RD(TCG_REG_O7) | INSN_RS1(TCG_REG_T1) |
                       INSN_RS2(TCG_REG_G0));
         }
-        /* Store AREG0 in stack to avoid ugly glibc bugs that mangle
-           global registers */
-        // delay slot
-        tcg_out_ldst(s, TCG_AREG0, TCG_REG_CALL_STACK,
-                     TCG_TARGET_CALL_STACK_OFFSET - TCG_STATIC_CALL_ARGS_SIZE -
-                     sizeof(long), HOST_ST_OP);
-        tcg_out_ldst(s, TCG_AREG0, TCG_REG_CALL_STACK,
-                     TCG_TARGET_CALL_STACK_OFFSET - TCG_STATIC_CALL_ARGS_SIZE -
-                     sizeof(long), HOST_LD_OP);
-        break;
-    case INDEX_op_jmp:
+        /* delay slot */
+        tcg_out_nop(s);
+        break;
     case INDEX_op_br:
-        tcg_out_branch_i32(s, COND_A, args[0]);
+        tcg_out_bpcc(s, COND_A, BPCC_PT, args[0]);
         tcg_out_nop(s);
         break;
     case INDEX_op_movi_i32:
@@ -1290,13 +1266,16 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
         goto gen_arith;
     case INDEX_op_shl_i32:
         c = SHIFT_SLL;
-        goto gen_arith;
+    do_shift32:
+        /* Limit immediate shift count lest we create an illegal insn.  */
+        tcg_out_arithc(s, args[0], args[1], args[2] & 31, const_args[2], c);
+        break;
     case INDEX_op_shr_i32:
         c = SHIFT_SRL;
-        goto gen_arith;
+        goto do_shift32;
     case INDEX_op_sar_i32:
         c = SHIFT_SRA;
-        goto gen_arith;
+        goto do_shift32;
     case INDEX_op_mul_i32:
         c = ARITH_UMUL;
         goto gen_arith;
@@ -1317,11 +1296,11 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
 
     case INDEX_op_rem_i32:
     case INDEX_op_remu_i32:
-        tcg_out_div32(s, TCG_REG_I5, args[1], args[2], const_args[2],
+        tcg_out_div32(s, TCG_REG_T1, args[1], args[2], const_args[2],
                       opc == INDEX_op_remu_i32);
-        tcg_out_arithc(s, TCG_REG_I5, TCG_REG_I5, args[2], const_args[2],
+        tcg_out_arithc(s, TCG_REG_T1, TCG_REG_T1, args[2], const_args[2],
                        ARITH_UMUL);
-        tcg_out_arith(s, args[0], args[1], TCG_REG_I5, ARITH_SUB);
+        tcg_out_arith(s, args[0], args[1], TCG_REG_T1, ARITH_SUB);
         break;
 
     case INDEX_op_brcond_i32:
@@ -1332,6 +1311,10 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
         tcg_out_setcond_i32(s, args[3], args[0], args[1],
                             args[2], const_args[2]);
         break;
+    case INDEX_op_movcond_i32:
+        tcg_out_movcond_i32(s, args[5], args[0], args[1],
+                            args[2], const_args[2], args[3], const_args[3]);
+        break;
 
 #if TCG_TARGET_REG_BITS == 32
     case INDEX_op_brcond2_i32:
@@ -1345,16 +1328,14 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
                              args[4], const_args[4]);
         break;
     case INDEX_op_add2_i32:
-        tcg_out_arithc(s, args[0], args[2], args[4], const_args[4],
-                       ARITH_ADDCC);
-        tcg_out_arithc(s, args[1], args[3], args[5], const_args[5],
-                       ARITH_ADDX);
+        tcg_out_addsub2(s, args[0], args[1], args[2], args[3],
+                        args[4], const_args[4], args[5], const_args[5],
+                        ARITH_ADDCC, ARITH_ADDX);
         break;
     case INDEX_op_sub2_i32:
-        tcg_out_arithc(s, args[0], args[2], args[4], const_args[4],
-                       ARITH_SUBCC);
-        tcg_out_arithc(s, args[1], args[3], args[5], const_args[5],
-                       ARITH_SUBX);
+        tcg_out_addsub2(s, args[0], args[1], args[2], args[3],
+                        args[4], const_args[4], args[5], const_args[5],
+                        ARITH_SUBCC, ARITH_SUBX);
         break;
     case INDEX_op_mulu2_i32:
         tcg_out_arithc(s, args[0], args[2], args[3], const_args[3],
@@ -1386,6 +1367,9 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
         tcg_out_qemu_ld(s, args, 2 | 4);
         break;
 #endif
+    case INDEX_op_qemu_ld64:
+        tcg_out_qemu_ld(s, args, 3);
+        break;
     case INDEX_op_qemu_st8:
         tcg_out_qemu_st(s, args, 0);
         break;
@@ -1395,6 +1379,9 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
     case INDEX_op_qemu_st32:
         tcg_out_qemu_st(s, args, 2);
         break;
+    case INDEX_op_qemu_st64:
+        tcg_out_qemu_st(s, args, 3);
+        break;
 
 #if TCG_TARGET_REG_BITS == 64
     case INDEX_op_movi_i64:
@@ -1411,13 +1398,16 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
         break;
     case INDEX_op_shl_i64:
         c = SHIFT_SLLX;
-        goto gen_arith;
+    do_shift64:
+        /* Limit immediate shift count lest we create an illegal insn.  */
+        tcg_out_arithc(s, args[0], args[1], args[2] & 63, const_args[2], c);
+        break;
     case INDEX_op_shr_i64:
         c = SHIFT_SRLX;
-        goto gen_arith;
+        goto do_shift64;
     case INDEX_op_sar_i64:
         c = SHIFT_SRAX;
-        goto gen_arith;
+        goto do_shift64;
     case INDEX_op_mul_i64:
         c = ARITH_MULX;
         goto gen_arith;
@@ -1429,11 +1419,11 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
         goto gen_arith;
     case INDEX_op_rem_i64:
     case INDEX_op_remu_i64:
-        tcg_out_arithc(s, TCG_REG_I5, args[1], args[2], const_args[2],
+        tcg_out_arithc(s, TCG_REG_T1, args[1], args[2], const_args[2],
                        opc == INDEX_op_rem_i64 ? ARITH_SDIVX : ARITH_UDIVX);
-        tcg_out_arithc(s, TCG_REG_I5, TCG_REG_I5, args[2], const_args[2],
+        tcg_out_arithc(s, TCG_REG_T1, TCG_REG_T1, args[2], const_args[2],
                        ARITH_MULX);
-        tcg_out_arith(s, args[0], args[1], TCG_REG_I5, ARITH_SUB);
+        tcg_out_arith(s, args[0], args[1], TCG_REG_T1, ARITH_SUB);
         break;
     case INDEX_op_ext32s_i64:
         if (const_args[1]) {
@@ -1458,14 +1448,10 @@ static inline void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
         tcg_out_setcond_i64(s, args[3], args[0], args[1],
                             args[2], const_args[2]);
         break;
-
-    case INDEX_op_qemu_ld64:
-        tcg_out_qemu_ld(s, args, 3);
-        break;
-    case INDEX_op_qemu_st64:
-        tcg_out_qemu_st(s, args, 3);
+    case INDEX_op_movcond_i64:
+        tcg_out_movcond_i64(s, args[5], args[0], args[1],
+                            args[2], const_args[2], args[3], const_args[3]);
         break;
-
 #endif
     gen_arith:
         tcg_out_arithc(s, args[0], args[1], args[2], const_args[2], c);
@@ -1485,7 +1471,6 @@ static const TCGTargetOpDef sparc_op_defs[] = {
     { INDEX_op_exit_tb, { } },
     { INDEX_op_goto_tb, { } },
     { INDEX_op_call, { "ri" } },
-    { INDEX_op_jmp, { "ri" } },
     { INDEX_op_br, { } },
 
     { INDEX_op_mov_i32, { "r", "r" } },
@@ -1495,55 +1480,42 @@ static const TCGTargetOpDef sparc_op_defs[] = {
     { INDEX_op_ld16u_i32, { "r", "r" } },
     { INDEX_op_ld16s_i32, { "r", "r" } },
     { INDEX_op_ld_i32, { "r", "r" } },
-    { INDEX_op_st8_i32, { "r", "r" } },
-    { INDEX_op_st16_i32, { "r", "r" } },
-    { INDEX_op_st_i32, { "r", "r" } },
-
-    { INDEX_op_add_i32, { "r", "r", "rJ" } },
-    { INDEX_op_mul_i32, { "r", "r", "rJ" } },
-    { INDEX_op_div_i32, { "r", "r", "rJ" } },
-    { INDEX_op_divu_i32, { "r", "r", "rJ" } },
-    { INDEX_op_rem_i32, { "r", "r", "rJ" } },
-    { INDEX_op_remu_i32, { "r", "r", "rJ" } },
-    { INDEX_op_sub_i32, { "r", "r", "rJ" } },
-    { INDEX_op_and_i32, { "r", "r", "rJ" } },
-    { INDEX_op_andc_i32, { "r", "r", "rJ" } },
-    { INDEX_op_or_i32, { "r", "r", "rJ" } },
-    { INDEX_op_orc_i32, { "r", "r", "rJ" } },
-    { INDEX_op_xor_i32, { "r", "r", "rJ" } },
-
-    { INDEX_op_shl_i32, { "r", "r", "rJ" } },
-    { INDEX_op_shr_i32, { "r", "r", "rJ" } },
-    { INDEX_op_sar_i32, { "r", "r", "rJ" } },
+    { INDEX_op_st8_i32, { "rZ", "r" } },
+    { INDEX_op_st16_i32, { "rZ", "r" } },
+    { INDEX_op_st_i32, { "rZ", "r" } },
+
+    { INDEX_op_add_i32, { "r", "rZ", "rJ" } },
+    { INDEX_op_mul_i32, { "r", "rZ", "rJ" } },
+    { INDEX_op_div_i32, { "r", "rZ", "rJ" } },
+    { INDEX_op_divu_i32, { "r", "rZ", "rJ" } },
+    { INDEX_op_rem_i32, { "r", "rZ", "rJ" } },
+    { INDEX_op_remu_i32, { "r", "rZ", "rJ" } },
+    { INDEX_op_sub_i32, { "r", "rZ", "rJ" } },
+    { INDEX_op_and_i32, { "r", "rZ", "rJ" } },
+    { INDEX_op_andc_i32, { "r", "rZ", "rJ" } },
+    { INDEX_op_or_i32, { "r", "rZ", "rJ" } },
+    { INDEX_op_orc_i32, { "r", "rZ", "rJ" } },
+    { INDEX_op_xor_i32, { "r", "rZ", "rJ" } },
+
+    { INDEX_op_shl_i32, { "r", "rZ", "rJ" } },
+    { INDEX_op_shr_i32, { "r", "rZ", "rJ" } },
+    { INDEX_op_sar_i32, { "r", "rZ", "rJ" } },
 
     { INDEX_op_neg_i32, { "r", "rJ" } },
     { INDEX_op_not_i32, { "r", "rJ" } },
 
-    { INDEX_op_brcond_i32, { "r", "rJ" } },
-    { INDEX_op_setcond_i32, { "r", "r", "rJ" } },
+    { INDEX_op_brcond_i32, { "rZ", "rJ" } },
+    { INDEX_op_setcond_i32, { "r", "rZ", "rJ" } },
+    { INDEX_op_movcond_i32, { "r", "rZ", "rJ", "rI", "0" } },
 
 #if TCG_TARGET_REG_BITS == 32
-    { INDEX_op_brcond2_i32, { "r", "r", "rJ", "rJ" } },
-    { INDEX_op_setcond2_i32, { "r", "r", "r", "rJ", "rJ" } },
-    { INDEX_op_add2_i32, { "r", "r", "r", "r", "rJ", "rJ" } },
-    { INDEX_op_sub2_i32, { "r", "r", "r", "r", "rJ", "rJ" } },
-    { INDEX_op_mulu2_i32, { "r", "r", "r", "rJ" } },
-#endif
-
-    { INDEX_op_qemu_ld8u, { "r", "L" } },
-    { INDEX_op_qemu_ld8s, { "r", "L" } },
-    { INDEX_op_qemu_ld16u, { "r", "L" } },
-    { INDEX_op_qemu_ld16s, { "r", "L" } },
-    { INDEX_op_qemu_ld32, { "r", "L" } },
-#if TCG_TARGET_REG_BITS == 64
-    { INDEX_op_qemu_ld32u, { "r", "L" } },
-    { INDEX_op_qemu_ld32s, { "r", "L" } },
+    { INDEX_op_brcond2_i32, { "rZ", "rZ", "rJ", "rJ" } },
+    { INDEX_op_setcond2_i32, { "r", "rZ", "rZ", "rJ", "rJ" } },
+    { INDEX_op_add2_i32, { "r", "r", "rZ", "rZ", "rJ", "rJ" } },
+    { INDEX_op_sub2_i32, { "r", "r", "rZ", "rZ", "rJ", "rJ" } },
+    { INDEX_op_mulu2_i32, { "r", "r", "rZ", "rJ" } },
 #endif
 
-    { INDEX_op_qemu_st8, { "L", "L" } },
-    { INDEX_op_qemu_st16, { "L", "L" } },
-    { INDEX_op_qemu_st32, { "L", "L" } },
-
 #if TCG_TARGET_REG_BITS == 64
     { INDEX_op_mov_i64, { "r", "r" } },
     { INDEX_op_movi_i64, { "r" } },
@@ -1554,29 +1526,27 @@ static const TCGTargetOpDef sparc_op_defs[] = {
     { INDEX_op_ld32u_i64, { "r", "r" } },
     { INDEX_op_ld32s_i64, { "r", "r" } },
     { INDEX_op_ld_i64, { "r", "r" } },
-    { INDEX_op_st8_i64, { "r", "r" } },
-    { INDEX_op_st16_i64, { "r", "r" } },
-    { INDEX_op_st32_i64, { "r", "r" } },
-    { INDEX_op_st_i64, { "r", "r" } },
-    { INDEX_op_qemu_ld64, { "L", "L" } },
-    { INDEX_op_qemu_st64, { "L", "L" } },
-
-    { INDEX_op_add_i64, { "r", "r", "rJ" } },
-    { INDEX_op_mul_i64, { "r", "r", "rJ" } },
-    { INDEX_op_div_i64, { "r", "r", "rJ" } },
-    { INDEX_op_divu_i64, { "r", "r", "rJ" } },
-    { INDEX_op_rem_i64, { "r", "r", "rJ" } },
-    { INDEX_op_remu_i64, { "r", "r", "rJ" } },
-    { INDEX_op_sub_i64, { "r", "r", "rJ" } },
-    { INDEX_op_and_i64, { "r", "r", "rJ" } },
-    { INDEX_op_andc_i64, { "r", "r", "rJ" } },
-    { INDEX_op_or_i64, { "r", "r", "rJ" } },
-    { INDEX_op_orc_i64, { "r", "r", "rJ" } },
-    { INDEX_op_xor_i64, { "r", "r", "rJ" } },
-
-    { INDEX_op_shl_i64, { "r", "r", "rJ" } },
-    { INDEX_op_shr_i64, { "r", "r", "rJ" } },
-    { INDEX_op_sar_i64, { "r", "r", "rJ" } },
+    { INDEX_op_st8_i64, { "rZ", "r" } },
+    { INDEX_op_st16_i64, { "rZ", "r" } },
+    { INDEX_op_st32_i64, { "rZ", "r" } },
+    { INDEX_op_st_i64, { "rZ", "r" } },
+
+    { INDEX_op_add_i64, { "r", "rZ", "rJ" } },
+    { INDEX_op_mul_i64, { "r", "rZ", "rJ" } },
+    { INDEX_op_div_i64, { "r", "rZ", "rJ" } },
+    { INDEX_op_divu_i64, { "r", "rZ", "rJ" } },
+    { INDEX_op_rem_i64, { "r", "rZ", "rJ" } },
+    { INDEX_op_remu_i64, { "r", "rZ", "rJ" } },
+    { INDEX_op_sub_i64, { "r", "rZ", "rJ" } },
+    { INDEX_op_and_i64, { "r", "rZ", "rJ" } },
+    { INDEX_op_andc_i64, { "r", "rZ", "rJ" } },
+    { INDEX_op_or_i64, { "r", "rZ", "rJ" } },
+    { INDEX_op_orc_i64, { "r", "rZ", "rJ" } },
+    { INDEX_op_xor_i64, { "r", "rZ", "rJ" } },
+
+    { INDEX_op_shl_i64, { "r", "rZ", "rJ" } },
+    { INDEX_op_shr_i64, { "r", "rZ", "rJ" } },
+    { INDEX_op_sar_i64, { "r", "rZ", "rJ" } },
 
     { INDEX_op_neg_i64, { "r", "rJ" } },
     { INDEX_op_not_i64, { "r", "rJ" } },
@@ -1584,9 +1554,51 @@ static const TCGTargetOpDef sparc_op_defs[] = {
     { INDEX_op_ext32s_i64, { "r", "ri" } },
     { INDEX_op_ext32u_i64, { "r", "ri" } },
 
-    { INDEX_op_brcond_i64, { "r", "rJ" } },
-    { INDEX_op_setcond_i64, { "r", "r", "rJ" } },
+    { INDEX_op_brcond_i64, { "rZ", "rJ" } },
+    { INDEX_op_setcond_i64, { "r", "rZ", "rJ" } },
+    { INDEX_op_movcond_i64, { "r", "rZ", "rJ", "rI", "0" } },
+#endif
+
+#if TCG_TARGET_REG_BITS == 64
+    { INDEX_op_qemu_ld8u, { "r", "L" } },
+    { INDEX_op_qemu_ld8s, { "r", "L" } },
+    { INDEX_op_qemu_ld16u, { "r", "L" } },
+    { INDEX_op_qemu_ld16s, { "r", "L" } },
+    { INDEX_op_qemu_ld32, { "r", "L" } },
+    { INDEX_op_qemu_ld32u, { "r", "L" } },
+    { INDEX_op_qemu_ld32s, { "r", "L" } },
+    { INDEX_op_qemu_ld64, { "r", "L" } },
+
+    { INDEX_op_qemu_st8, { "L", "L" } },
+    { INDEX_op_qemu_st16, { "L", "L" } },
+    { INDEX_op_qemu_st32, { "L", "L" } },
+    { INDEX_op_qemu_st64, { "L", "L" } },
+#elif TARGET_LONG_BITS <= TCG_TARGET_REG_BITS
+    { INDEX_op_qemu_ld8u, { "r", "L" } },
+    { INDEX_op_qemu_ld8s, { "r", "L" } },
+    { INDEX_op_qemu_ld16u, { "r", "L" } },
+    { INDEX_op_qemu_ld16s, { "r", "L" } },
+    { INDEX_op_qemu_ld32, { "r", "L" } },
+    { INDEX_op_qemu_ld64, { "r", "r", "L" } },
+
+    { INDEX_op_qemu_st8, { "L", "L" } },
+    { INDEX_op_qemu_st16, { "L", "L" } },
+    { INDEX_op_qemu_st32, { "L", "L" } },
+    { INDEX_op_qemu_st64, { "L", "L", "L" } },
+#else
+    { INDEX_op_qemu_ld8u, { "r", "L", "L" } },
+    { INDEX_op_qemu_ld8s, { "r", "L", "L" } },
+    { INDEX_op_qemu_ld16u, { "r", "L", "L" } },
+    { INDEX_op_qemu_ld16s, { "r", "L", "L" } },
+    { INDEX_op_qemu_ld32, { "r", "L", "L" } },
+    { INDEX_op_qemu_ld64, { "L", "L", "L", "L" } },
+
+    { INDEX_op_qemu_st8, { "L", "L", "L" } },
+    { INDEX_op_qemu_st16, { "L", "L", "L" } },
+    { INDEX_op_qemu_st32, { "L", "L", "L" } },
+    { INDEX_op_qemu_st64, { "L", "L", "L", "L" } },
 #endif
+
     { -1 },
 };
 
@@ -1613,25 +1625,23 @@ static void tcg_target_init(TCGContext *s)
                      (1 << TCG_REG_O7));
 
     tcg_regset_clear(s->reserved_regs);
-    tcg_regset_set_reg(s->reserved_regs, TCG_REG_G0);
-#if TCG_TARGET_REG_BITS == 64
-    tcg_regset_set_reg(s->reserved_regs, TCG_REG_I4); // for internal use
-#endif
-    tcg_regset_set_reg(s->reserved_regs, TCG_REG_I5); // for internal use
-    tcg_regset_set_reg(s->reserved_regs, TCG_REG_I6);
-    tcg_regset_set_reg(s->reserved_regs, TCG_REG_I7);
-    tcg_regset_set_reg(s->reserved_regs, TCG_REG_O6);
-    tcg_regset_set_reg(s->reserved_regs, TCG_REG_O7);
+    tcg_regset_set_reg(s->reserved_regs, TCG_REG_G0); /* zero */
+    tcg_regset_set_reg(s->reserved_regs, TCG_REG_G6); /* reserved for os */
+    tcg_regset_set_reg(s->reserved_regs, TCG_REG_G7); /* thread pointer */
+    tcg_regset_set_reg(s->reserved_regs, TCG_REG_I6); /* frame pointer */
+    tcg_regset_set_reg(s->reserved_regs, TCG_REG_I7); /* return address */
+    tcg_regset_set_reg(s->reserved_regs, TCG_REG_O6); /* stack pointer */
+    tcg_regset_set_reg(s->reserved_regs, TCG_REG_T1); /* for internal use */
+    tcg_regset_set_reg(s->reserved_regs, TCG_REG_T2); /* for internal use */
+
     tcg_add_target_add_op_defs(sparc_op_defs);
 }
 
 #if TCG_TARGET_REG_BITS == 64
 # define ELF_HOST_MACHINE  EM_SPARCV9
-#elif defined(__sparc_v8plus__)
+#else
 # define ELF_HOST_MACHINE  EM_SPARC32PLUS
 # define ELF_HOST_FLAGS    EF_SPARC_32PLUS
-#else
-# define ELF_HOST_MACHINE  EM_SPARC
 #endif
 
 typedef struct {
@@ -1687,3 +1697,18 @@ void tcg_register_jit(void *buf, size_t buf_size)
 
     tcg_register_jit_int(buf, buf_size, &debug_frame, sizeof(debug_frame));
 }
+
+void tb_set_jmp_target1(uintptr_t jmp_addr, uintptr_t addr)
+{
+    uint32_t *ptr = (uint32_t *)jmp_addr;
+    tcg_target_long disp = (tcg_target_long)(addr - jmp_addr) >> 2;
+
+    /* We can reach the entire address space for 32-bit.  For 64-bit
+       the code_gen_buffer can't be larger than 2GB.  */
+    if (TCG_TARGET_REG_BITS == 64 && !check_fit_tl(disp, 30)) {
+        tcg_abort();
+    }
+
+    *ptr = CALL | (disp & 0x3fffffff);
+    flush_icache_range(jmp_addr, jmp_addr + 4);
+}
diff --git a/tcg/sparc/tcg-target.h b/tcg/sparc/tcg-target.h
index ee2274d59..0e7d398a4 100644
--- a/tcg/sparc/tcg-target.h
+++ b/tcg/sparc/tcg-target.h
@@ -62,26 +62,24 @@ typedef enum {
     TCG_REG_I7,
 } TCGReg;
 
-#define TCG_CT_CONST_S11 0x100
-#define TCG_CT_CONST_S13 0x200
+#define TCG_CT_CONST_S11  0x100
+#define TCG_CT_CONST_S13  0x200
+#define TCG_CT_CONST_ZERO 0x400
 
 /* used for function call generation */
-#define TCG_REG_CALL_STACK TCG_REG_I6
-#ifdef __arch64__
-// Reserve space for AREG0
-#define TCG_TARGET_STACK_MINFRAME (176 + 4 * (int)sizeof(long) + \
-                                   TCG_STATIC_CALL_ARGS_SIZE)
-#define TCG_TARGET_CALL_STACK_OFFSET (2047 - 16)
-#define TCG_TARGET_STACK_ALIGN 16
+#define TCG_REG_CALL_STACK TCG_REG_O6
+
+#if TCG_TARGET_REG_BITS == 64
+#define TCG_TARGET_STACK_BIAS           2047
+#define TCG_TARGET_STACK_ALIGN          16
+#define TCG_TARGET_CALL_STACK_OFFSET    (128 + 6*8 + TCG_TARGET_STACK_BIAS)
 #else
-// AREG0 + one word for alignment
-#define TCG_TARGET_STACK_MINFRAME (92 + (2 + 1) * (int)sizeof(long) + \
-                                   TCG_STATIC_CALL_ARGS_SIZE)
-#define TCG_TARGET_CALL_STACK_OFFSET TCG_TARGET_STACK_MINFRAME
-#define TCG_TARGET_STACK_ALIGN 8
+#define TCG_TARGET_STACK_BIAS           0
+#define TCG_TARGET_STACK_ALIGN          8
+#define TCG_TARGET_CALL_STACK_OFFSET    (64 + 4 + 6*4)
 #endif
 
-#ifdef __arch64__
+#if TCG_TARGET_REG_BITS == 64
 #define TCG_TARGET_EXTEND_ARGS 1
 #endif
 
@@ -102,6 +100,7 @@ typedef enum {
 #define TCG_TARGET_HAS_nand_i32         0
 #define TCG_TARGET_HAS_nor_i32          0
 #define TCG_TARGET_HAS_deposit_i32      0
+#define TCG_TARGET_HAS_movcond_i32      1
 
 #if TCG_TARGET_REG_BITS == 64
 #define TCG_TARGET_HAS_div_i64          1
@@ -123,16 +122,10 @@ typedef enum {
 #define TCG_TARGET_HAS_nand_i64         0
 #define TCG_TARGET_HAS_nor_i64          0
 #define TCG_TARGET_HAS_deposit_i64      0
+#define TCG_TARGET_HAS_movcond_i64      1
 #endif
 
-/* Note: must be synced with dyngen-exec.h */
-#ifdef CONFIG_SOLARIS
-#define TCG_AREG0 TCG_REG_G2
-#elif defined(__sparc_v9__)
-#define TCG_AREG0 TCG_REG_G5
-#else
-#define TCG_AREG0 TCG_REG_G6
-#endif
+#define TCG_AREG0 TCG_REG_I0
 
 static inline void flush_icache_range(tcg_target_ulong start,
                                       tcg_target_ulong stop)
diff --git a/tcg/tcg-op.h b/tcg/tcg-op.h
index 169d3b2b0..0b3cb0be3 100644
--- a/tcg/tcg-op.h
+++ b/tcg/tcg-op.h
@@ -25,335 +25,340 @@
 
 int gen_new_label(void);
 
+static inline void tcg_gen_op0(TCGOpcode opc)
+{
+    *tcg_ctx.gen_opc_ptr++ = opc;
+}
+
 static inline void tcg_gen_op1_i32(TCGOpcode opc, TCGv_i32 arg1)
 {
-    *gen_opc_ptr++ = opc;
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg1);
+    *tcg_ctx.gen_opc_ptr++ = opc;
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg1);
 }
 
 static inline void tcg_gen_op1_i64(TCGOpcode opc, TCGv_i64 arg1)
 {
-    *gen_opc_ptr++ = opc;
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg1);
+    *tcg_ctx.gen_opc_ptr++ = opc;
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg1);
 }
 
 static inline void tcg_gen_op1i(TCGOpcode opc, TCGArg arg1)
 {
-    *gen_opc_ptr++ = opc;
-    *gen_opparam_ptr++ = arg1;
+    *tcg_ctx.gen_opc_ptr++ = opc;
+    *tcg_ctx.gen_opparam_ptr++ = arg1;
 }
 
 static inline void tcg_gen_op2_i32(TCGOpcode opc, TCGv_i32 arg1, TCGv_i32 arg2)
 {
-    *gen_opc_ptr++ = opc;
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg1);
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg2);
+    *tcg_ctx.gen_opc_ptr++ = opc;
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg1);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg2);
 }
 
 static inline void tcg_gen_op2_i64(TCGOpcode opc, TCGv_i64 arg1, TCGv_i64 arg2)
 {
-    *gen_opc_ptr++ = opc;
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg1);
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg2);
+    *tcg_ctx.gen_opc_ptr++ = opc;
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg1);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg2);
 }
 
 static inline void tcg_gen_op2i_i32(TCGOpcode opc, TCGv_i32 arg1, TCGArg arg2)
 {
-    *gen_opc_ptr++ = opc;
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg1);
-    *gen_opparam_ptr++ = arg2;
+    *tcg_ctx.gen_opc_ptr++ = opc;
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg1);
+    *tcg_ctx.gen_opparam_ptr++ = arg2;
 }
 
 static inline void tcg_gen_op2i_i64(TCGOpcode opc, TCGv_i64 arg1, TCGArg arg2)
 {
-    *gen_opc_ptr++ = opc;
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg1);
-    *gen_opparam_ptr++ = arg2;
+    *tcg_ctx.gen_opc_ptr++ = opc;
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg1);
+    *tcg_ctx.gen_opparam_ptr++ = arg2;
 }
 
 static inline void tcg_gen_op2ii(TCGOpcode opc, TCGArg arg1, TCGArg arg2)
 {
-    *gen_opc_ptr++ = opc;
-    *gen_opparam_ptr++ = arg1;
-    *gen_opparam_ptr++ = arg2;
+    *tcg_ctx.gen_opc_ptr++ = opc;
+    *tcg_ctx.gen_opparam_ptr++ = arg1;
+    *tcg_ctx.gen_opparam_ptr++ = arg2;
 }
 
 static inline void tcg_gen_op3_i32(TCGOpcode opc, TCGv_i32 arg1, TCGv_i32 arg2,
                                    TCGv_i32 arg3)
 {
-    *gen_opc_ptr++ = opc;
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg1);
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg2);
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg3);
+    *tcg_ctx.gen_opc_ptr++ = opc;
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg1);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg2);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg3);
 }
 
 static inline void tcg_gen_op3_i64(TCGOpcode opc, TCGv_i64 arg1, TCGv_i64 arg2,
                                    TCGv_i64 arg3)
 {
-    *gen_opc_ptr++ = opc;
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg1);
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg2);
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg3);
+    *tcg_ctx.gen_opc_ptr++ = opc;
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg1);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg2);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg3);
 }
 
 static inline void tcg_gen_op3i_i32(TCGOpcode opc, TCGv_i32 arg1,
                                     TCGv_i32 arg2, TCGArg arg3)
 {
-    *gen_opc_ptr++ = opc;
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg1);
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg2);
-    *gen_opparam_ptr++ = arg3;
+    *tcg_ctx.gen_opc_ptr++ = opc;
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg1);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg2);
+    *tcg_ctx.gen_opparam_ptr++ = arg3;
 }
 
 static inline void tcg_gen_op3i_i64(TCGOpcode opc, TCGv_i64 arg1,
                                     TCGv_i64 arg2, TCGArg arg3)
 {
-    *gen_opc_ptr++ = opc;
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg1);
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg2);
-    *gen_opparam_ptr++ = arg3;
+    *tcg_ctx.gen_opc_ptr++ = opc;
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg1);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg2);
+    *tcg_ctx.gen_opparam_ptr++ = arg3;
 }
 
 static inline void tcg_gen_ldst_op_i32(TCGOpcode opc, TCGv_i32 val,
                                        TCGv_ptr base, TCGArg offset)
 {
-    *gen_opc_ptr++ = opc;
-    *gen_opparam_ptr++ = GET_TCGV_I32(val);
-    *gen_opparam_ptr++ = GET_TCGV_PTR(base);
-    *gen_opparam_ptr++ = offset;
+    *tcg_ctx.gen_opc_ptr++ = opc;
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(val);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_PTR(base);
+    *tcg_ctx.gen_opparam_ptr++ = offset;
 }
 
 static inline void tcg_gen_ldst_op_i64(TCGOpcode opc, TCGv_i64 val,
                                        TCGv_ptr base, TCGArg offset)
 {
-    *gen_opc_ptr++ = opc;
-    *gen_opparam_ptr++ = GET_TCGV_I64(val);
-    *gen_opparam_ptr++ = GET_TCGV_PTR(base);
-    *gen_opparam_ptr++ = offset;
+    *tcg_ctx.gen_opc_ptr++ = opc;
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(val);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_PTR(base);
+    *tcg_ctx.gen_opparam_ptr++ = offset;
 }
 
 static inline void tcg_gen_qemu_ldst_op_i64_i32(TCGOpcode opc, TCGv_i64 val,
                                                 TCGv_i32 addr, TCGArg mem_index)
 {
-    *gen_opc_ptr++ = opc;
-    *gen_opparam_ptr++ = GET_TCGV_I64(val);
-    *gen_opparam_ptr++ = GET_TCGV_I32(addr);
-    *gen_opparam_ptr++ = mem_index;
+    *tcg_ctx.gen_opc_ptr++ = opc;
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(val);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(addr);
+    *tcg_ctx.gen_opparam_ptr++ = mem_index;
 }
 
 static inline void tcg_gen_qemu_ldst_op_i64_i64(TCGOpcode opc, TCGv_i64 val,
                                                 TCGv_i64 addr, TCGArg mem_index)
 {
-    *gen_opc_ptr++ = opc;
-    *gen_opparam_ptr++ = GET_TCGV_I64(val);
-    *gen_opparam_ptr++ = GET_TCGV_I64(addr);
-    *gen_opparam_ptr++ = mem_index;
+    *tcg_ctx.gen_opc_ptr++ = opc;
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(val);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(addr);
+    *tcg_ctx.gen_opparam_ptr++ = mem_index;
 }
 
 static inline void tcg_gen_op4_i32(TCGOpcode opc, TCGv_i32 arg1, TCGv_i32 arg2,
                                    TCGv_i32 arg3, TCGv_i32 arg4)
 {
-    *gen_opc_ptr++ = opc;
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg1);
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg2);
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg3);
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg4);
+    *tcg_ctx.gen_opc_ptr++ = opc;
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg1);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg2);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg3);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg4);
 }
 
 static inline void tcg_gen_op4_i64(TCGOpcode opc, TCGv_i64 arg1, TCGv_i64 arg2,
                                    TCGv_i64 arg3, TCGv_i64 arg4)
 {
-    *gen_opc_ptr++ = opc;
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg1);
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg2);
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg3);
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg4);
+    *tcg_ctx.gen_opc_ptr++ = opc;
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg1);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg2);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg3);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg4);
 }
 
 static inline void tcg_gen_op4i_i32(TCGOpcode opc, TCGv_i32 arg1, TCGv_i32 arg2,
                                     TCGv_i32 arg3, TCGArg arg4)
 {
-    *gen_opc_ptr++ = opc;
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg1);
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg2);
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg3);
-    *gen_opparam_ptr++ = arg4;
+    *tcg_ctx.gen_opc_ptr++ = opc;
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg1);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg2);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg3);
+    *tcg_ctx.gen_opparam_ptr++ = arg4;
 }
 
 static inline void tcg_gen_op4i_i64(TCGOpcode opc, TCGv_i64 arg1, TCGv_i64 arg2,
                                     TCGv_i64 arg3, TCGArg arg4)
 {
-    *gen_opc_ptr++ = opc;
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg1);
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg2);
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg3);
-    *gen_opparam_ptr++ = arg4;
+    *tcg_ctx.gen_opc_ptr++ = opc;
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg1);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg2);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg3);
+    *tcg_ctx.gen_opparam_ptr++ = arg4;
 }
 
 static inline void tcg_gen_op4ii_i32(TCGOpcode opc, TCGv_i32 arg1, TCGv_i32 arg2,
                                      TCGArg arg3, TCGArg arg4)
 {
-    *gen_opc_ptr++ = opc;
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg1);
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg2);
-    *gen_opparam_ptr++ = arg3;
-    *gen_opparam_ptr++ = arg4;
+    *tcg_ctx.gen_opc_ptr++ = opc;
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg1);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg2);
+    *tcg_ctx.gen_opparam_ptr++ = arg3;
+    *tcg_ctx.gen_opparam_ptr++ = arg4;
 }
 
 static inline void tcg_gen_op4ii_i64(TCGOpcode opc, TCGv_i64 arg1, TCGv_i64 arg2,
                                      TCGArg arg3, TCGArg arg4)
 {
-    *gen_opc_ptr++ = opc;
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg1);
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg2);
-    *gen_opparam_ptr++ = arg3;
-    *gen_opparam_ptr++ = arg4;
+    *tcg_ctx.gen_opc_ptr++ = opc;
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg1);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg2);
+    *tcg_ctx.gen_opparam_ptr++ = arg3;
+    *tcg_ctx.gen_opparam_ptr++ = arg4;
 }
 
 static inline void tcg_gen_op5_i32(TCGOpcode opc, TCGv_i32 arg1, TCGv_i32 arg2,
                                    TCGv_i32 arg3, TCGv_i32 arg4, TCGv_i32 arg5)
 {
-    *gen_opc_ptr++ = opc;
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg1);
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg2);
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg3);
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg4);
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg5);
+    *tcg_ctx.gen_opc_ptr++ = opc;
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg1);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg2);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg3);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg4);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg5);
 }
 
 static inline void tcg_gen_op5_i64(TCGOpcode opc, TCGv_i64 arg1, TCGv_i64 arg2,
                                    TCGv_i64 arg3, TCGv_i64 arg4, TCGv_i64 arg5)
 {
-    *gen_opc_ptr++ = opc;
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg1);
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg2);
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg3);
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg4);
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg5);
+    *tcg_ctx.gen_opc_ptr++ = opc;
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg1);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg2);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg3);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg4);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg5);
 }
 
 static inline void tcg_gen_op5i_i32(TCGOpcode opc, TCGv_i32 arg1, TCGv_i32 arg2,
                                     TCGv_i32 arg3, TCGv_i32 arg4, TCGArg arg5)
 {
-    *gen_opc_ptr++ = opc;
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg1);
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg2);
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg3);
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg4);
-    *gen_opparam_ptr++ = arg5;
+    *tcg_ctx.gen_opc_ptr++ = opc;
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg1);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg2);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg3);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg4);
+    *tcg_ctx.gen_opparam_ptr++ = arg5;
 }
 
 static inline void tcg_gen_op5i_i64(TCGOpcode opc, TCGv_i64 arg1, TCGv_i64 arg2,
                                     TCGv_i64 arg3, TCGv_i64 arg4, TCGArg arg5)
 {
-    *gen_opc_ptr++ = opc;
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg1);
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg2);
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg3);
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg4);
-    *gen_opparam_ptr++ = arg5;
+    *tcg_ctx.gen_opc_ptr++ = opc;
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg1);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg2);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg3);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg4);
+    *tcg_ctx.gen_opparam_ptr++ = arg5;
 }
 
 static inline void tcg_gen_op5ii_i32(TCGOpcode opc, TCGv_i32 arg1,
                                      TCGv_i32 arg2, TCGv_i32 arg3,
                                      TCGArg arg4, TCGArg arg5)
 {
-    *gen_opc_ptr++ = opc;
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg1);
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg2);
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg3);
-    *gen_opparam_ptr++ = arg4;
-    *gen_opparam_ptr++ = arg5;
+    *tcg_ctx.gen_opc_ptr++ = opc;
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg1);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg2);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg3);
+    *tcg_ctx.gen_opparam_ptr++ = arg4;
+    *tcg_ctx.gen_opparam_ptr++ = arg5;
 }
 
 static inline void tcg_gen_op5ii_i64(TCGOpcode opc, TCGv_i64 arg1,
                                      TCGv_i64 arg2, TCGv_i64 arg3,
                                      TCGArg arg4, TCGArg arg5)
 {
-    *gen_opc_ptr++ = opc;
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg1);
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg2);
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg3);
-    *gen_opparam_ptr++ = arg4;
-    *gen_opparam_ptr++ = arg5;
+    *tcg_ctx.gen_opc_ptr++ = opc;
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg1);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg2);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg3);
+    *tcg_ctx.gen_opparam_ptr++ = arg4;
+    *tcg_ctx.gen_opparam_ptr++ = arg5;
 }
 
 static inline void tcg_gen_op6_i32(TCGOpcode opc, TCGv_i32 arg1, TCGv_i32 arg2,
                                    TCGv_i32 arg3, TCGv_i32 arg4, TCGv_i32 arg5,
                                    TCGv_i32 arg6)
 {
-    *gen_opc_ptr++ = opc;
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg1);
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg2);
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg3);
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg4);
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg5);
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg6);
+    *tcg_ctx.gen_opc_ptr++ = opc;
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg1);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg2);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg3);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg4);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg5);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg6);
 }
 
 static inline void tcg_gen_op6_i64(TCGOpcode opc, TCGv_i64 arg1, TCGv_i64 arg2,
                                    TCGv_i64 arg3, TCGv_i64 arg4, TCGv_i64 arg5,
                                    TCGv_i64 arg6)
 {
-    *gen_opc_ptr++ = opc;
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg1);
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg2);
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg3);
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg4);
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg5);
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg6);
+    *tcg_ctx.gen_opc_ptr++ = opc;
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg1);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg2);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg3);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg4);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg5);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg6);
 }
 
 static inline void tcg_gen_op6i_i32(TCGOpcode opc, TCGv_i32 arg1, TCGv_i32 arg2,
                                     TCGv_i32 arg3, TCGv_i32 arg4,
                                     TCGv_i32 arg5, TCGArg arg6)
 {
-    *gen_opc_ptr++ = opc;
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg1);
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg2);
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg3);
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg4);
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg5);
-    *gen_opparam_ptr++ = arg6;
+    *tcg_ctx.gen_opc_ptr++ = opc;
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg1);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg2);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg3);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg4);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg5);
+    *tcg_ctx.gen_opparam_ptr++ = arg6;
 }
 
 static inline void tcg_gen_op6i_i64(TCGOpcode opc, TCGv_i64 arg1, TCGv_i64 arg2,
                                     TCGv_i64 arg3, TCGv_i64 arg4,
                                     TCGv_i64 arg5, TCGArg arg6)
 {
-    *gen_opc_ptr++ = opc;
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg1);
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg2);
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg3);
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg4);
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg5);
-    *gen_opparam_ptr++ = arg6;
+    *tcg_ctx.gen_opc_ptr++ = opc;
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg1);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg2);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg3);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg4);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg5);
+    *tcg_ctx.gen_opparam_ptr++ = arg6;
 }
 
 static inline void tcg_gen_op6ii_i32(TCGOpcode opc, TCGv_i32 arg1,
                                      TCGv_i32 arg2, TCGv_i32 arg3,
                                      TCGv_i32 arg4, TCGArg arg5, TCGArg arg6)
 {
-    *gen_opc_ptr++ = opc;
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg1);
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg2);
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg3);
-    *gen_opparam_ptr++ = GET_TCGV_I32(arg4);
-    *gen_opparam_ptr++ = arg5;
-    *gen_opparam_ptr++ = arg6;
+    *tcg_ctx.gen_opc_ptr++ = opc;
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg1);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg2);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg3);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I32(arg4);
+    *tcg_ctx.gen_opparam_ptr++ = arg5;
+    *tcg_ctx.gen_opparam_ptr++ = arg6;
 }
 
 static inline void tcg_gen_op6ii_i64(TCGOpcode opc, TCGv_i64 arg1,
                                      TCGv_i64 arg2, TCGv_i64 arg3,
                                      TCGv_i64 arg4, TCGArg arg5, TCGArg arg6)
 {
-    *gen_opc_ptr++ = opc;
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg1);
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg2);
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg3);
-    *gen_opparam_ptr++ = GET_TCGV_I64(arg4);
-    *gen_opparam_ptr++ = arg5;
-    *gen_opparam_ptr++ = arg6;
+    *tcg_ctx.gen_opc_ptr++ = opc;
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg1);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg2);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg3);
+    *tcg_ctx.gen_opparam_ptr++ = GET_TCGV_I64(arg4);
+    *tcg_ctx.gen_opparam_ptr++ = arg5;
+    *tcg_ctx.gen_opparam_ptr++ = arg6;
 }
 
 static inline void gen_set_label(int n)
@@ -396,10 +401,10 @@ static inline void tcg_gen_helperN(void *func, int flags, int sizemask,
 }
 
 /* Note: Both tcg_gen_helper32() and tcg_gen_helper64() are currently
-   reserved for helpers in tcg-runtime.c. These helpers are all const
-   and pure, hence the call to tcg_gen_callN() with TCG_CALL_CONST |
-   TCG_CALL_PURE. This may need to be adjusted if these functions
-   start to be used with other helpers. */
+   reserved for helpers in tcg-runtime.c. These helpers all do not read
+   globals and do not have side effects, hence the call to tcg_gen_callN()
+   with TCG_CALL_NO_READ_GLOBALS | TCG_CALL_NO_SIDE_EFFECTS. This may need
+   to be adjusted if these functions start to be used with other helpers. */
 static inline void tcg_gen_helper32(void *func, int sizemask, TCGv_i32 ret,
                                     TCGv_i32 a, TCGv_i32 b)
 {
@@ -408,8 +413,9 @@ static inline void tcg_gen_helper32(void *func, int sizemask, TCGv_i32 ret,
     fn = tcg_const_ptr(func);
     args[0] = GET_TCGV_I32(a);
     args[1] = GET_TCGV_I32(b);
-    tcg_gen_callN(&tcg_ctx, fn, TCG_CALL_CONST | TCG_CALL_PURE, sizemask,
-                  GET_TCGV_I32(ret), 2, args);
+    tcg_gen_callN(&tcg_ctx, fn,
+                  TCG_CALL_NO_READ_GLOBALS | TCG_CALL_NO_SIDE_EFFECTS,
+                  sizemask, GET_TCGV_I32(ret), 2, args);
     tcg_temp_free_ptr(fn);
 }
 
@@ -421,8 +427,9 @@ static inline void tcg_gen_helper64(void *func, int sizemask, TCGv_i64 ret,
     fn = tcg_const_ptr(func);
     args[0] = GET_TCGV_I64(a);
     args[1] = GET_TCGV_I64(b);
-    tcg_gen_callN(&tcg_ctx, fn, TCG_CALL_CONST | TCG_CALL_PURE, sizemask,
-                  GET_TCGV_I64(ret), 2, args);
+    tcg_gen_callN(&tcg_ctx, fn,
+                  TCG_CALL_NO_READ_GLOBALS | TCG_CALL_NO_SIDE_EFFECTS,
+                  sizemask, GET_TCGV_I64(ret), 2, args);
     tcg_temp_free_ptr(fn);
 }
 
@@ -518,18 +525,34 @@ static inline void tcg_gen_and_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
     }
 }
 
-static inline void tcg_gen_andi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
+static inline void tcg_gen_andi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2)
 {
-    /* some cases can be optimized here */
-    if (arg2 == 0) {
+    TCGv_i32 t0;
+    /* Some cases can be optimized here.  */
+    switch (arg2) {
+    case 0:
         tcg_gen_movi_i32(ret, 0);
-    } else if (arg2 == 0xffffffff) {
+        return;
+    case 0xffffffffu:
         tcg_gen_mov_i32(ret, arg1);
-    } else {
-        TCGv_i32 t0 = tcg_const_i32(arg2);
-        tcg_gen_and_i32(ret, arg1, t0);
-        tcg_temp_free_i32(t0);
-    }
+        return;
+    case 0xffu:
+        /* Don't recurse with tcg_gen_ext8u_i32.  */
+        if (TCG_TARGET_HAS_ext8u_i32) {
+            tcg_gen_op2_i32(INDEX_op_ext8u_i32, ret, arg1);
+            return;
+        }
+        break;
+    case 0xffffu:
+        if (TCG_TARGET_HAS_ext16u_i32) {
+            tcg_gen_op2_i32(INDEX_op_ext16u_i32, ret, arg1);
+            return;
+        }
+        break;
+    }
+    t0 = tcg_const_i32(arg2);
+    tcg_gen_and_i32(ret, arg1, t0);
+    tcg_temp_free_i32(t0);
 }
 
 static inline void tcg_gen_or_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
@@ -543,9 +566,9 @@ static inline void tcg_gen_or_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
 
 static inline void tcg_gen_ori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
 {
-    /* some cases can be optimized here */
-    if (arg2 == 0xffffffff) {
-        tcg_gen_movi_i32(ret, 0xffffffff);
+    /* Some cases can be optimized here.  */
+    if (arg2 == -1) {
+        tcg_gen_movi_i32(ret, -1);
     } else if (arg2 == 0) {
         tcg_gen_mov_i32(ret, arg1);
     } else {
@@ -566,9 +589,12 @@ static inline void tcg_gen_xor_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
 
 static inline void tcg_gen_xori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
 {
-    /* some cases can be optimized here */
+    /* Some cases can be optimized here.  */
     if (arg2 == 0) {
         tcg_gen_mov_i32(ret, arg1);
+    } else if (arg2 == -1 && TCG_TARGET_HAS_not_i32) {
+        /* Don't recurse with tcg_gen_not_i32.  */
+        tcg_gen_op2_i32(INDEX_op_not_i32, ret, arg1);
     } else {
         TCGv_i32 t0 = tcg_const_i32(arg2);
         tcg_gen_xor_i32(ret, arg1, t0);
@@ -627,29 +653,49 @@ static inline void tcg_gen_sari_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
 static inline void tcg_gen_brcond_i32(TCGCond cond, TCGv_i32 arg1,
                                       TCGv_i32 arg2, int label_index)
 {
-    tcg_gen_op4ii_i32(INDEX_op_brcond_i32, arg1, arg2, cond, label_index);
+    if (cond == TCG_COND_ALWAYS) {
+        tcg_gen_br(label_index);
+    } else if (cond != TCG_COND_NEVER) {
+        tcg_gen_op4ii_i32(INDEX_op_brcond_i32, arg1, arg2, cond, label_index);
+    }
 }
 
 static inline void tcg_gen_brcondi_i32(TCGCond cond, TCGv_i32 arg1,
                                        int32_t arg2, int label_index)
 {
-    TCGv_i32 t0 = tcg_const_i32(arg2);
-    tcg_gen_brcond_i32(cond, arg1, t0, label_index);
-    tcg_temp_free_i32(t0);
+    if (cond == TCG_COND_ALWAYS) {
+        tcg_gen_br(label_index);
+    } else if (cond != TCG_COND_NEVER) {
+        TCGv_i32 t0 = tcg_const_i32(arg2);
+        tcg_gen_brcond_i32(cond, arg1, t0, label_index);
+        tcg_temp_free_i32(t0);
+    }
 }
 
 static inline void tcg_gen_setcond_i32(TCGCond cond, TCGv_i32 ret,
                                        TCGv_i32 arg1, TCGv_i32 arg2)
 {
-    tcg_gen_op4i_i32(INDEX_op_setcond_i32, ret, arg1, arg2, cond);
+    if (cond == TCG_COND_ALWAYS) {
+        tcg_gen_movi_i32(ret, 1);
+    } else if (cond == TCG_COND_NEVER) {
+        tcg_gen_movi_i32(ret, 0);
+    } else {
+        tcg_gen_op4i_i32(INDEX_op_setcond_i32, ret, arg1, arg2, cond);
+    }
 }
 
 static inline void tcg_gen_setcondi_i32(TCGCond cond, TCGv_i32 ret,
                                         TCGv_i32 arg1, int32_t arg2)
 {
-    TCGv_i32 t0 = tcg_const_i32(arg2);
-    tcg_gen_setcond_i32(cond, ret, arg1, t0);
-    tcg_temp_free_i32(t0);
+    if (cond == TCG_COND_ALWAYS) {
+        tcg_gen_movi_i32(ret, 1);
+    } else if (cond == TCG_COND_NEVER) {
+        tcg_gen_movi_i32(ret, 0);
+    } else {
+        TCGv_i32 t0 = tcg_const_i32(arg2);
+        tcg_gen_setcond_i32(cond, ret, arg1, t0);
+        tcg_temp_free_i32(t0);
+    }
 }
 
 static inline void tcg_gen_mul_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
@@ -847,6 +893,8 @@ static inline void tcg_gen_add_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
     tcg_gen_op6_i32(INDEX_op_add2_i32, TCGV_LOW(ret), TCGV_HIGH(ret),
                     TCGV_LOW(arg1), TCGV_HIGH(arg1), TCGV_LOW(arg2),
                     TCGV_HIGH(arg2));
+    /* Allow the optimizer room to replace add2 with two moves.  */
+    tcg_gen_op0(INDEX_op_nop);
 }
 
 static inline void tcg_gen_sub_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
@@ -854,6 +902,8 @@ static inline void tcg_gen_sub_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
     tcg_gen_op6_i32(INDEX_op_sub2_i32, TCGV_LOW(ret), TCGV_HIGH(ret),
                     TCGV_LOW(arg1), TCGV_HIGH(arg1), TCGV_LOW(arg2),
                     TCGV_HIGH(arg2));
+    /* Allow the optimizer room to replace sub2 with two moves.  */
+    tcg_gen_op0(INDEX_op_nop);
 }
 
 static inline void tcg_gen_and_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
@@ -945,17 +995,27 @@ static inline void tcg_gen_sari_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
 static inline void tcg_gen_brcond_i64(TCGCond cond, TCGv_i64 arg1,
                                       TCGv_i64 arg2, int label_index)
 {
-    tcg_gen_op6ii_i32(INDEX_op_brcond2_i32,
-                      TCGV_LOW(arg1), TCGV_HIGH(arg1), TCGV_LOW(arg2),
-                      TCGV_HIGH(arg2), cond, label_index);
+    if (cond == TCG_COND_ALWAYS) {
+        tcg_gen_br(label_index);
+    } else if (cond != TCG_COND_NEVER) {
+        tcg_gen_op6ii_i32(INDEX_op_brcond2_i32,
+                          TCGV_LOW(arg1), TCGV_HIGH(arg1), TCGV_LOW(arg2),
+                          TCGV_HIGH(arg2), cond, label_index);
+    }
 }
 
 static inline void tcg_gen_setcond_i64(TCGCond cond, TCGv_i64 ret,
                                        TCGv_i64 arg1, TCGv_i64 arg2)
 {
-    tcg_gen_op6i_i32(INDEX_op_setcond2_i32, TCGV_LOW(ret),
-                     TCGV_LOW(arg1), TCGV_HIGH(arg1),
-                     TCGV_LOW(arg2), TCGV_HIGH(arg2), cond);
+    if (cond == TCG_COND_ALWAYS) {
+        tcg_gen_movi_i32(TCGV_LOW(ret), 1);
+    } else if (cond == TCG_COND_NEVER) {
+        tcg_gen_movi_i32(TCGV_LOW(ret), 0);
+    } else {
+        tcg_gen_op6i_i32(INDEX_op_setcond2_i32, TCGV_LOW(ret),
+                         TCGV_LOW(arg1), TCGV_HIGH(arg1),
+                         TCGV_LOW(arg2), TCGV_HIGH(arg2), cond);
+    }
     tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
 }
 
@@ -969,6 +1029,8 @@ static inline void tcg_gen_mul_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
 
     tcg_gen_op4_i32(INDEX_op_mulu2_i32, TCGV_LOW(t0), TCGV_HIGH(t0),
                     TCGV_LOW(arg1), TCGV_LOW(arg2));
+    /* Allow the optimizer room to replace mulu2 with two moves.  */
+    tcg_gen_op0(INDEX_op_nop);
 
     tcg_gen_mul_i32(t1, TCGV_LOW(arg1), TCGV_HIGH(arg2));
     tcg_gen_add_i32(TCGV_HIGH(t0), TCGV_HIGH(t0), t1);
@@ -1120,9 +1182,38 @@ static inline void tcg_gen_and_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
     }
 }
 
-static inline void tcg_gen_andi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
+static inline void tcg_gen_andi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2)
 {
-    TCGv_i64 t0 = tcg_const_i64(arg2);
+    TCGv_i64 t0;
+    /* Some cases can be optimized here.  */
+    switch (arg2) {
+    case 0:
+        tcg_gen_movi_i64(ret, 0);
+        return;
+    case 0xffffffffffffffffull:
+        tcg_gen_mov_i64(ret, arg1);
+        return;
+    case 0xffull:
+        /* Don't recurse with tcg_gen_ext8u_i32.  */
+        if (TCG_TARGET_HAS_ext8u_i64) {
+            tcg_gen_op2_i64(INDEX_op_ext8u_i64, ret, arg1);
+            return;
+        }
+        break;
+    case 0xffffu:
+        if (TCG_TARGET_HAS_ext16u_i64) {
+            tcg_gen_op2_i64(INDEX_op_ext16u_i64, ret, arg1);
+            return;
+        }
+        break;
+    case 0xffffffffull:
+        if (TCG_TARGET_HAS_ext32u_i64) {
+            tcg_gen_op2_i64(INDEX_op_ext32u_i64, ret, arg1);
+            return;
+        }
+        break;
+    }
+    t0 = tcg_const_i64(arg2);
     tcg_gen_and_i64(ret, arg1, t0);
     tcg_temp_free_i64(t0);
 }
@@ -1138,9 +1229,16 @@ static inline void tcg_gen_or_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
 
 static inline void tcg_gen_ori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
 {
-    TCGv_i64 t0 = tcg_const_i64(arg2);
-    tcg_gen_or_i64(ret, arg1, t0);
-    tcg_temp_free_i64(t0);
+    /* Some cases can be optimized here.  */
+    if (arg2 == -1) {
+        tcg_gen_movi_i64(ret, -1);
+    } else if (arg2 == 0) {
+        tcg_gen_mov_i64(ret, arg1);
+    } else {
+        TCGv_i64 t0 = tcg_const_i64(arg2);
+        tcg_gen_or_i64(ret, arg1, t0);
+        tcg_temp_free_i64(t0);
+    }
 }
 
 static inline void tcg_gen_xor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
@@ -1154,9 +1252,17 @@ static inline void tcg_gen_xor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
 
 static inline void tcg_gen_xori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
 {
-    TCGv_i64 t0 = tcg_const_i64(arg2);
-    tcg_gen_xor_i64(ret, arg1, t0);
-    tcg_temp_free_i64(t0);
+    /* Some cases can be optimized here.  */
+    if (arg2 == 0) {
+        tcg_gen_mov_i64(ret, arg1);
+    } else if (arg2 == -1 && TCG_TARGET_HAS_not_i64) {
+        /* Don't recurse with tcg_gen_not_i64.  */
+        tcg_gen_op2_i64(INDEX_op_not_i64, ret, arg1);
+    } else {
+        TCGv_i64 t0 = tcg_const_i64(arg2);
+        tcg_gen_xor_i64(ret, arg1, t0);
+        tcg_temp_free_i64(t0);
+    }
 }
 
 static inline void tcg_gen_shl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
@@ -1210,13 +1316,23 @@ static inline void tcg_gen_sari_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
 static inline void tcg_gen_brcond_i64(TCGCond cond, TCGv_i64 arg1,
                                       TCGv_i64 arg2, int label_index)
 {
-    tcg_gen_op4ii_i64(INDEX_op_brcond_i64, arg1, arg2, cond, label_index);
+    if (cond == TCG_COND_ALWAYS) {
+        tcg_gen_br(label_index);
+    } else if (cond != TCG_COND_NEVER) {
+        tcg_gen_op4ii_i64(INDEX_op_brcond_i64, arg1, arg2, cond, label_index);
+    }
 }
 
 static inline void tcg_gen_setcond_i64(TCGCond cond, TCGv_i64 ret,
                                        TCGv_i64 arg1, TCGv_i64 arg2)
 {
-    tcg_gen_op4i_i64(INDEX_op_setcond_i64, ret, arg1, arg2, cond);
+    if (cond == TCG_COND_ALWAYS) {
+        tcg_gen_movi_i64(ret, 1);
+    } else if (cond == TCG_COND_NEVER) {
+        tcg_gen_movi_i64(ret, 0);
+    } else {
+        tcg_gen_op4i_i64(INDEX_op_setcond_i64, ret, arg1, arg2, cond);
+    }
 }
 
 static inline void tcg_gen_mul_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
@@ -1334,9 +1450,13 @@ static inline void tcg_gen_subi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
 static inline void tcg_gen_brcondi_i64(TCGCond cond, TCGv_i64 arg1,
                                        int64_t arg2, int label_index)
 {
-    TCGv_i64 t0 = tcg_const_i64(arg2);
-    tcg_gen_brcond_i64(cond, arg1, t0, label_index);
-    tcg_temp_free_i64(t0);
+    if (cond == TCG_COND_ALWAYS) {
+        tcg_gen_br(label_index);
+    } else if (cond != TCG_COND_NEVER) {
+        TCGv_i64 t0 = tcg_const_i64(arg2);
+        tcg_gen_brcond_i64(cond, arg1, t0, label_index);
+        tcg_temp_free_i64(t0);
+    }
 }
 
 static inline void tcg_gen_setcondi_i64(TCGCond cond, TCGv_i64 ret,
@@ -1746,36 +1866,6 @@ static inline void tcg_gen_discard_i64(TCGv_i64 arg)
 #endif
 }
 
-static inline void tcg_gen_concat_i32_i64(TCGv_i64 dest, TCGv_i32 low, TCGv_i32 high)
-{
-#if TCG_TARGET_REG_BITS == 32
-    tcg_gen_mov_i32(TCGV_LOW(dest), low);
-    tcg_gen_mov_i32(TCGV_HIGH(dest), high);
-#else
-    TCGv_i64 tmp = tcg_temp_new_i64();
-    /* This extension is only needed for type correctness.
-       We may be able to do better given target specific information.  */
-    tcg_gen_extu_i32_i64(tmp, high);
-    tcg_gen_shli_i64(tmp, tmp, 32);
-    tcg_gen_extu_i32_i64(dest, low);
-    tcg_gen_or_i64(dest, dest, tmp);
-    tcg_temp_free_i64(tmp);
-#endif
-}
-
-static inline void tcg_gen_concat32_i64(TCGv_i64 dest, TCGv_i64 low, TCGv_i64 high)
-{
-#if TCG_TARGET_REG_BITS == 32
-    tcg_gen_concat_i32_i64(dest, TCGV_LOW(low), TCGV_LOW(high));
-#else
-    TCGv_i64 tmp = tcg_temp_new_i64();
-    tcg_gen_ext32u_i64(dest, low);
-    tcg_gen_shli_i64(tmp, high, 32);
-    tcg_gen_or_i64(dest, dest, tmp);
-    tcg_temp_free_i64(tmp);
-#endif
-}
-
 static inline void tcg_gen_andc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
 {
     if (TCG_TARGET_HAS_andc_i32) {
@@ -2048,6 +2138,10 @@ static inline void tcg_gen_deposit_i32(TCGv_i32 ret, TCGv_i32 arg1,
     uint32_t mask;
     TCGv_i32 t1;
 
+    tcg_debug_assert(ofs < 32);
+    tcg_debug_assert(len <= 32);
+    tcg_debug_assert(ofs + len <= 32);
+
     if (ofs == 0 && len == 32) {
         tcg_gen_mov_i32(ret, arg2);
         return;
@@ -2079,6 +2173,10 @@ static inline void tcg_gen_deposit_i64(TCGv_i64 ret, TCGv_i64 arg1,
     uint64_t mask;
     TCGv_i64 t1;
 
+    tcg_debug_assert(ofs < 64);
+    tcg_debug_assert(len <= 64);
+    tcg_debug_assert(ofs + len <= 64);
+
     if (ofs == 0 && len == 64) {
         tcg_gen_mov_i64(ret, arg2);
         return;
@@ -2118,6 +2216,102 @@ static inline void tcg_gen_deposit_i64(TCGv_i64 ret, TCGv_i64 arg1,
     tcg_temp_free_i64(t1);
 }
 
+static inline void tcg_gen_concat_i32_i64(TCGv_i64 dest, TCGv_i32 low,
+                                          TCGv_i32 high)
+{
+#if TCG_TARGET_REG_BITS == 32
+    tcg_gen_mov_i32(TCGV_LOW(dest), low);
+    tcg_gen_mov_i32(TCGV_HIGH(dest), high);
+#else
+    TCGv_i64 tmp = tcg_temp_new_i64();
+    /* These extensions are only needed for type correctness.
+       We may be able to do better given target specific information.  */
+    tcg_gen_extu_i32_i64(tmp, high);
+    tcg_gen_extu_i32_i64(dest, low);
+    /* If deposit is available, use it.  Otherwise use the extra
+       knowledge that we have of the zero-extensions above.  */
+    if (TCG_TARGET_HAS_deposit_i64 && TCG_TARGET_deposit_i64_valid(32, 32)) {
+        tcg_gen_deposit_i64(dest, dest, tmp, 32, 32);
+    } else {
+        tcg_gen_shli_i64(tmp, tmp, 32);
+        tcg_gen_or_i64(dest, dest, tmp);
+    }
+    tcg_temp_free_i64(tmp);
+#endif
+}
+
+static inline void tcg_gen_concat32_i64(TCGv_i64 dest, TCGv_i64 low,
+                                        TCGv_i64 high)
+{
+    tcg_gen_deposit_i64(dest, low, high, 32, 32);
+}
+
+static inline void tcg_gen_movcond_i32(TCGCond cond, TCGv_i32 ret,
+                                       TCGv_i32 c1, TCGv_i32 c2,
+                                       TCGv_i32 v1, TCGv_i32 v2)
+{
+    if (TCG_TARGET_HAS_movcond_i32) {
+        tcg_gen_op6i_i32(INDEX_op_movcond_i32, ret, c1, c2, v1, v2, cond);
+    } else {
+        TCGv_i32 t0 = tcg_temp_new_i32();
+        TCGv_i32 t1 = tcg_temp_new_i32();
+        tcg_gen_setcond_i32(cond, t0, c1, c2);
+        tcg_gen_neg_i32(t0, t0);
+        tcg_gen_and_i32(t1, v1, t0);
+        tcg_gen_andc_i32(ret, v2, t0);
+        tcg_gen_or_i32(ret, ret, t1);
+        tcg_temp_free_i32(t0);
+        tcg_temp_free_i32(t1);
+    }
+}
+
+static inline void tcg_gen_movcond_i64(TCGCond cond, TCGv_i64 ret,
+                                       TCGv_i64 c1, TCGv_i64 c2,
+                                       TCGv_i64 v1, TCGv_i64 v2)
+{
+#if TCG_TARGET_REG_BITS == 32
+    TCGv_i32 t0 = tcg_temp_new_i32();
+    TCGv_i32 t1 = tcg_temp_new_i32();
+    tcg_gen_op6i_i32(INDEX_op_setcond2_i32, t0,
+                     TCGV_LOW(c1), TCGV_HIGH(c1),
+                     TCGV_LOW(c2), TCGV_HIGH(c2), cond);
+
+    if (TCG_TARGET_HAS_movcond_i32) {
+        tcg_gen_movi_i32(t1, 0);
+        tcg_gen_movcond_i32(TCG_COND_NE, TCGV_LOW(ret), t0, t1,
+                            TCGV_LOW(v1), TCGV_LOW(v2));
+        tcg_gen_movcond_i32(TCG_COND_NE, TCGV_HIGH(ret), t0, t1,
+                            TCGV_HIGH(v1), TCGV_HIGH(v2));
+    } else {
+        tcg_gen_neg_i32(t0, t0);
+
+        tcg_gen_and_i32(t1, TCGV_LOW(v1), t0);
+        tcg_gen_andc_i32(TCGV_LOW(ret), TCGV_LOW(v2), t0);
+        tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(ret), t1);
+
+        tcg_gen_and_i32(t1, TCGV_HIGH(v1), t0);
+        tcg_gen_andc_i32(TCGV_HIGH(ret), TCGV_HIGH(v2), t0);
+        tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(ret), t1);
+    }
+    tcg_temp_free_i32(t0);
+    tcg_temp_free_i32(t1);
+#else
+    if (TCG_TARGET_HAS_movcond_i64) {
+        tcg_gen_op6i_i64(INDEX_op_movcond_i64, ret, c1, c2, v1, v2, cond);
+    } else {
+        TCGv_i64 t0 = tcg_temp_new_i64();
+        TCGv_i64 t1 = tcg_temp_new_i64();
+        tcg_gen_setcond_i64(cond, t0, c1, c2);
+        tcg_gen_neg_i64(t0, t0);
+        tcg_gen_and_i64(t1, v1, t0);
+        tcg_gen_andc_i64(ret, v2, t0);
+        tcg_gen_or_i64(ret, ret, t1);
+        tcg_temp_free_i64(t0);
+        tcg_temp_free_i64(t1);
+    }
+#endif
+}
+
 /***************************************/
 /* QEMU specific operations. Their type depend on the QEMU CPU
    type. */
@@ -2166,8 +2360,15 @@ static inline void tcg_gen_exit_tb(tcg_target_long val)
     tcg_gen_op1i(INDEX_op_exit_tb, val);
 }
 
-static inline void tcg_gen_goto_tb(int idx)
+static inline void tcg_gen_goto_tb(unsigned idx)
 {
+    /* We only support two chained exits.  */
+    tcg_debug_assert(idx <= 1);
+#ifdef CONFIG_DEBUG_TCG
+    /* Verify that we havn't seen this numbered exit before.  */
+    tcg_debug_assert((tcg_ctx.goto_tb_issue_mask & (1 << idx)) == 0);
+    tcg_ctx.goto_tb_issue_mask |= 1 << idx;
+#endif
     tcg_gen_op1i(INDEX_op_goto_tb, idx);
 }
 
@@ -2434,6 +2635,7 @@ static inline void tcg_gen_qemu_st64(TCGv_i64 arg, TCGv addr, int mem_index)
 #define tcg_gen_deposit_tl tcg_gen_deposit_i64
 #define tcg_const_tl tcg_const_i64
 #define tcg_const_local_tl tcg_const_local_i64
+#define tcg_gen_movcond_tl tcg_gen_movcond_i64
 #else
 #define tcg_gen_movi_tl tcg_gen_movi_i32
 #define tcg_gen_mov_tl tcg_gen_mov_i32
@@ -2505,6 +2707,7 @@ static inline void tcg_gen_qemu_st64(TCGv_i64 arg, TCGv addr, int mem_index)
 #define tcg_gen_deposit_tl tcg_gen_deposit_i32
 #define tcg_const_tl tcg_const_i32
 #define tcg_const_local_tl tcg_const_local_i32
+#define tcg_gen_movcond_tl tcg_gen_movcond_i32
 #endif
 
 #if TCG_TARGET_REG_BITS == 32
diff --git a/tcg/tcg-opc.h b/tcg/tcg-opc.h
index 8e06d03b1..965106341 100644
--- a/tcg/tcg-opc.h
+++ b/tcg/tcg-opc.h
@@ -36,10 +36,9 @@ DEF(nopn, 0, 0, 1, 0) /* variable number of parameters */
 
 DEF(discard, 1, 0, 0, 0)
 
-DEF(set_label, 0, 0, 1, 0)
-DEF(call, 0, 1, 2, TCG_OPF_SIDE_EFFECTS) /* variable number of parameters */
-DEF(jmp, 0, 1, 0, TCG_OPF_BB_END | TCG_OPF_SIDE_EFFECTS)
-DEF(br, 0, 0, 1, TCG_OPF_BB_END | TCG_OPF_SIDE_EFFECTS)
+DEF(set_label, 0, 0, 1, TCG_OPF_BB_END)
+DEF(call, 0, 1, 2, TCG_OPF_CALL_CLOBBER) /* variable number of parameters */
+DEF(br, 0, 0, 1, TCG_OPF_BB_END)
 
 #define IMPL(X) (X ? 0 : TCG_OPF_NOT_PRESENT)
 #if TCG_TARGET_REG_BITS == 32
@@ -51,15 +50,16 @@ DEF(br, 0, 0, 1, TCG_OPF_BB_END | TCG_OPF_SIDE_EFFECTS)
 DEF(mov_i32, 1, 1, 0, 0)
 DEF(movi_i32, 1, 0, 1, 0)
 DEF(setcond_i32, 1, 2, 1, 0)
+DEF(movcond_i32, 1, 4, 1, IMPL(TCG_TARGET_HAS_movcond_i32))
 /* load/store */
 DEF(ld8u_i32, 1, 1, 1, 0)
 DEF(ld8s_i32, 1, 1, 1, 0)
 DEF(ld16u_i32, 1, 1, 1, 0)
 DEF(ld16s_i32, 1, 1, 1, 0)
 DEF(ld_i32, 1, 1, 1, 0)
-DEF(st8_i32, 0, 2, 1, TCG_OPF_SIDE_EFFECTS)
-DEF(st16_i32, 0, 2, 1, TCG_OPF_SIDE_EFFECTS)
-DEF(st_i32, 0, 2, 1, TCG_OPF_SIDE_EFFECTS)
+DEF(st8_i32, 0, 2, 1, 0)
+DEF(st16_i32, 0, 2, 1, 0)
+DEF(st_i32, 0, 2, 1, 0)
 /* arith */
 DEF(add_i32, 1, 2, 0, 0)
 DEF(sub_i32, 1, 2, 0, 0)
@@ -81,12 +81,11 @@ DEF(rotl_i32, 1, 2, 0, IMPL(TCG_TARGET_HAS_rot_i32))
 DEF(rotr_i32, 1, 2, 0, IMPL(TCG_TARGET_HAS_rot_i32))
 DEF(deposit_i32, 1, 2, 2, IMPL(TCG_TARGET_HAS_deposit_i32))
 
-DEF(brcond_i32, 0, 2, 2, TCG_OPF_BB_END | TCG_OPF_SIDE_EFFECTS)
+DEF(brcond_i32, 0, 2, 2, TCG_OPF_BB_END)
 
 DEF(add2_i32, 2, 4, 0, IMPL(TCG_TARGET_REG_BITS == 32))
 DEF(sub2_i32, 2, 4, 0, IMPL(TCG_TARGET_REG_BITS == 32))
-DEF(brcond2_i32, 0, 4, 2,
-    TCG_OPF_BB_END | TCG_OPF_SIDE_EFFECTS | IMPL(TCG_TARGET_REG_BITS == 32))
+DEF(brcond2_i32, 0, 4, 2, TCG_OPF_BB_END | IMPL(TCG_TARGET_REG_BITS == 32))
 DEF(mulu2_i32, 2, 2, 0, IMPL(TCG_TARGET_REG_BITS == 32))
 DEF(setcond2_i32, 1, 4, 1, IMPL(TCG_TARGET_REG_BITS == 32))
 
@@ -107,6 +106,7 @@ DEF(nor_i32, 1, 2, 0, IMPL(TCG_TARGET_HAS_nor_i32))
 DEF(mov_i64, 1, 1, 0, IMPL64)
 DEF(movi_i64, 1, 0, 1, IMPL64)
 DEF(setcond_i64, 1, 2, 1, IMPL64)
+DEF(movcond_i64, 1, 4, 1, IMPL64 | IMPL(TCG_TARGET_HAS_movcond_i64))
 /* load/store */
 DEF(ld8u_i64, 1, 1, 1, IMPL64)
 DEF(ld8s_i64, 1, 1, 1, IMPL64)
@@ -115,10 +115,10 @@ DEF(ld16s_i64, 1, 1, 1, IMPL64)
 DEF(ld32u_i64, 1, 1, 1, IMPL64)
 DEF(ld32s_i64, 1, 1, 1, IMPL64)
 DEF(ld_i64, 1, 1, 1, IMPL64)
-DEF(st8_i64, 0, 2, 1, TCG_OPF_SIDE_EFFECTS | IMPL64)
-DEF(st16_i64, 0, 2, 1, TCG_OPF_SIDE_EFFECTS | IMPL64)
-DEF(st32_i64, 0, 2, 1, TCG_OPF_SIDE_EFFECTS | IMPL64)
-DEF(st_i64, 0, 2, 1, TCG_OPF_SIDE_EFFECTS | IMPL64)
+DEF(st8_i64, 0, 2, 1, IMPL64)
+DEF(st16_i64, 0, 2, 1, IMPL64)
+DEF(st32_i64, 0, 2, 1, IMPL64)
+DEF(st_i64, 0, 2, 1, IMPL64)
 /* arith */
 DEF(add_i64, 1, 2, 0, IMPL64)
 DEF(sub_i64, 1, 2, 0, IMPL64)
@@ -140,7 +140,7 @@ DEF(rotl_i64, 1, 2, 0, IMPL64 | IMPL(TCG_TARGET_HAS_rot_i64))
 DEF(rotr_i64, 1, 2, 0, IMPL64 | IMPL(TCG_TARGET_HAS_rot_i64))
 DEF(deposit_i64, 1, 2, 2, IMPL64 | IMPL(TCG_TARGET_HAS_deposit_i64))
 
-DEF(brcond_i64, 0, 2, 2, TCG_OPF_BB_END | TCG_OPF_SIDE_EFFECTS | IMPL64)
+DEF(brcond_i64, 0, 2, 2, TCG_OPF_BB_END | IMPL64)
 DEF(ext8s_i64, 1, 1, 0, IMPL64 | IMPL(TCG_TARGET_HAS_ext8s_i64))
 DEF(ext16s_i64, 1, 1, 0, IMPL64 | IMPL(TCG_TARGET_HAS_ext16s_i64))
 DEF(ext32s_i64, 1, 1, 0, IMPL64 | IMPL(TCG_TARGET_HAS_ext32s_i64))
@@ -164,8 +164,8 @@ DEF(debug_insn_start, 0, 0, 2, 0)
 #else
 DEF(debug_insn_start, 0, 0, 1, 0)
 #endif
-DEF(exit_tb, 0, 0, 1, TCG_OPF_BB_END | TCG_OPF_SIDE_EFFECTS)
-DEF(goto_tb, 0, 0, 1, TCG_OPF_BB_END | TCG_OPF_SIDE_EFFECTS)
+DEF(exit_tb, 0, 0, 1, TCG_OPF_BB_END)
+DEF(goto_tb, 0, 0, 1, TCG_OPF_BB_END)
 /* Note: even if TARGET_LONG_BITS is not defined, the INDEX_op
    constants must be defined */
 #if TCG_TARGET_REG_BITS == 32
diff --git a/tcg/tcg.c b/tcg/tcg.c
index 8386b70ab..cb193f268 100644
--- a/tcg/tcg.c
+++ b/tcg/tcg.c
@@ -62,10 +62,6 @@
 
 #include "elf.h"
 
-#if defined(CONFIG_USE_GUEST_BASE) && !defined(TCG_TARGET_HAS_GUEST_BASE)
-#error GUEST_BASE not supported on this host.
-#endif
-
 /* Forward declarations for functions declared in tcg-target.c and used here. */
 static void tcg_target_init(TCGContext *s);
 static void tcg_target_qemu_prologue(TCGContext *s);
@@ -89,7 +85,6 @@ static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1,
                        tcg_target_long arg2);
 static int tcg_target_const_match(tcg_target_long val,
                                   const TCGArgConstraint *arg_ct);
-static int tcg_target_get_call_iarg_regs_count(int flags);
 
 TCGOpDef tcg_op_defs[] = {
 #define DEF(s, oargs, iargs, cargs, flags) { #s, oargs, iargs, cargs, iargs + oargs + cargs, flags },
@@ -101,10 +96,6 @@ const size_t tcg_op_defs_max = ARRAY_SIZE(tcg_op_defs);
 static TCGRegSet tcg_target_available_regs[2];
 static TCGRegSet tcg_target_call_clobber_regs;
 
-/* XXX: move that inside the context */
-uint16_t *gen_opc_ptr;
-TCGArg *gen_opparam_ptr;
-
 static inline void tcg_out8(TCGContext *s, uint8_t v)
 {
     *s->code_ptr++ = v;
@@ -243,7 +234,6 @@ void tcg_context_init(TCGContext *s)
     int *sorted_args;
 
     memset(s, 0, sizeof(*s));
-    s->temps = s->static_temps;
     s->nb_globals = 0;
     
     /* Count total number of arguments and allocate the corresponding
@@ -299,8 +289,20 @@ void tcg_func_start(TCGContext *s)
     s->nb_labels = 0;
     s->current_frame_offset = s->frame_start;
 
-    gen_opc_ptr = gen_opc_buf;
-    gen_opparam_ptr = gen_opparam_buf;
+#ifdef CONFIG_DEBUG_TCG
+    s->goto_tb_issue_mask = 0;
+#endif
+
+    s->gen_opc_ptr = s->gen_opc_buf;
+    s->gen_opparam_ptr = s->gen_opparam_buf;
+
+#if defined(CONFIG_QEMU_LDST_OPTIMIZATION) && defined(CONFIG_SOFTMMU)
+    /* Initialize qemu_ld/st labels to assist code generation at the end of TB
+       for TLB miss cases at the end of TB */
+    s->qemu_ldst_labels = tcg_malloc(sizeof(TCGLabelQemuLdst) *
+                                     TCG_MAX_QEMU_LDST);
+    s->nb_qemu_ldst_labels = 0;
+#endif
 }
 
 static inline void tcg_temp_alloc(TCGContext *s, int n)
@@ -635,23 +637,23 @@ void tcg_gen_callN(TCGContext *s, TCGv_ptr func, unsigned int flags,
     }
 #endif /* TCG_TARGET_EXTEND_ARGS */
 
-    *gen_opc_ptr++ = INDEX_op_call;
-    nparam = gen_opparam_ptr++;
+    *s->gen_opc_ptr++ = INDEX_op_call;
+    nparam = s->gen_opparam_ptr++;
     if (ret != TCG_CALL_DUMMY_ARG) {
 #if TCG_TARGET_REG_BITS < 64
         if (sizemask & 1) {
 #ifdef TCG_TARGET_WORDS_BIGENDIAN
-            *gen_opparam_ptr++ = ret + 1;
-            *gen_opparam_ptr++ = ret;
+            *s->gen_opparam_ptr++ = ret + 1;
+            *s->gen_opparam_ptr++ = ret;
 #else
-            *gen_opparam_ptr++ = ret;
-            *gen_opparam_ptr++ = ret + 1;
+            *s->gen_opparam_ptr++ = ret;
+            *s->gen_opparam_ptr++ = ret + 1;
 #endif
             nb_rets = 2;
         } else
 #endif
         {
-            *gen_opparam_ptr++ = ret;
+            *s->gen_opparam_ptr++ = ret;
             nb_rets = 1;
         }
     } else {
@@ -665,7 +667,7 @@ void tcg_gen_callN(TCGContext *s, TCGv_ptr func, unsigned int flags,
 #ifdef TCG_TARGET_CALL_ALIGN_ARGS
             /* some targets want aligned 64 bit args */
             if (real_args & 1) {
-                *gen_opparam_ptr++ = TCG_CALL_DUMMY_ARG;
+                *s->gen_opparam_ptr++ = TCG_CALL_DUMMY_ARG;
                 real_args++;
             }
 #endif
@@ -680,28 +682,28 @@ void tcg_gen_callN(TCGContext *s, TCGv_ptr func, unsigned int flags,
 	       have to get more complicated to differentiate between
 	       stack arguments and register arguments.  */
 #if defined(TCG_TARGET_WORDS_BIGENDIAN) != defined(TCG_TARGET_STACK_GROWSUP)
-            *gen_opparam_ptr++ = args[i] + 1;
-            *gen_opparam_ptr++ = args[i];
+            *s->gen_opparam_ptr++ = args[i] + 1;
+            *s->gen_opparam_ptr++ = args[i];
 #else
-            *gen_opparam_ptr++ = args[i];
-            *gen_opparam_ptr++ = args[i] + 1;
+            *s->gen_opparam_ptr++ = args[i];
+            *s->gen_opparam_ptr++ = args[i] + 1;
 #endif
             real_args += 2;
             continue;
         }
 #endif /* TCG_TARGET_REG_BITS < 64 */
 
-        *gen_opparam_ptr++ = args[i];
+        *s->gen_opparam_ptr++ = args[i];
         real_args++;
     }
-    *gen_opparam_ptr++ = GET_TCGV_PTR(func);
+    *s->gen_opparam_ptr++ = GET_TCGV_PTR(func);
 
-    *gen_opparam_ptr++ = flags;
+    *s->gen_opparam_ptr++ = flags;
 
     *nparam = (nb_rets << 16) | (real_args + 1);
 
     /* total parameters, needed to go backward in the instruction stream */
-    *gen_opparam_ptr++ = 1 + nb_rets + real_args + 3;
+    *s->gen_opparam_ptr++ = 1 + nb_rets + real_args + 3;
 
 #if defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64
     for (i = 0; i < nargs; ++i) {
@@ -778,7 +780,11 @@ static void tcg_reg_alloc_start(TCGContext *s)
     }
     for(i = s->nb_globals; i < s->nb_temps; i++) {
         ts = &s->temps[i];
-        ts->val_type = TEMP_VAL_DEAD;
+        if (ts->temp_local) {
+            ts->val_type = TEMP_VAL_MEM;
+        } else {
+            ts->val_type = TEMP_VAL_DEAD;
+        }
         ts->mem_allocated = 0;
         ts->fixed_reg = 0;
     }
@@ -861,6 +867,8 @@ static TCGHelperInfo *tcg_find_helper(TCGContext *s, tcg_target_ulong val)
 
 static const char * const cond_name[] =
 {
+    [TCG_COND_NEVER] = "never",
+    [TCG_COND_ALWAYS] = "always",
     [TCG_COND_EQ] = "eq",
     [TCG_COND_NE] = "ne",
     [TCG_COND_LT] = "lt",
@@ -884,9 +892,9 @@ void tcg_dump_ops(TCGContext *s)
     char buf[128];
 
     first_insn = 1;
-    opc_ptr = gen_opc_buf;
-    args = gen_opparam_buf;
-    while (opc_ptr < gen_opc_ptr) {
+    opc_ptr = s->gen_opc_buf;
+    args = s->gen_opparam_buf;
+    while (opc_ptr < s->gen_opc_ptr) {
         c = *opc_ptr++;
         def = &tcg_op_defs[c];
         if (c == INDEX_op_debug_insn_start) {
@@ -937,11 +945,7 @@ void tcg_dump_ops(TCGContext *s)
                                                        args[nb_oargs + i]));
                 }
             }
-        } else if (c == INDEX_op_movi_i32 
-#if TCG_TARGET_REG_BITS == 64
-                   || c == INDEX_op_movi_i64
-#endif
-                   ) {
+        } else if (c == INDEX_op_movi_i32 || c == INDEX_op_movi_i64) {
             tcg_target_ulong val;
             TCGHelperInfo *th;
 
@@ -991,17 +995,13 @@ void tcg_dump_ops(TCGContext *s)
             }
             switch (c) {
             case INDEX_op_brcond_i32:
-#if TCG_TARGET_REG_BITS == 32
-            case INDEX_op_brcond2_i32:
-#elif TCG_TARGET_REG_BITS == 64
-            case INDEX_op_brcond_i64:
-#endif
             case INDEX_op_setcond_i32:
-#if TCG_TARGET_REG_BITS == 32
+            case INDEX_op_movcond_i32:
+            case INDEX_op_brcond2_i32:
             case INDEX_op_setcond2_i32:
-#elif TCG_TARGET_REG_BITS == 64
+            case INDEX_op_brcond_i64:
             case INDEX_op_setcond_i64:
-#endif
+            case INDEX_op_movcond_i64:
                 if (args[k] < ARRAY_SIZE(cond_name) && cond_name[args[k]]) {
                     qemu_log(",%s", cond_name[args[k++]]);
                 } else {
@@ -1188,31 +1188,27 @@ static inline void tcg_set_nop(TCGContext *s, uint16_t *opc_ptr,
     }
 }
 
-/* liveness analysis: end of function: globals are live, temps are
-   dead. */
-/* XXX: at this stage, not used as there would be little gains because
-   most TBs end with a conditional jump. */
-static inline void tcg_la_func_end(TCGContext *s, uint8_t *dead_temps)
+/* liveness analysis: end of function: all temps are dead, and globals
+   should be in memory. */
+static inline void tcg_la_func_end(TCGContext *s, uint8_t *dead_temps,
+                                   uint8_t *mem_temps)
 {
-    memset(dead_temps, 0, s->nb_globals);
-    memset(dead_temps + s->nb_globals, 1, s->nb_temps - s->nb_globals);
+    memset(dead_temps, 1, s->nb_temps);
+    memset(mem_temps, 1, s->nb_globals);
+    memset(mem_temps + s->nb_globals, 0, s->nb_temps - s->nb_globals);
 }
 
-/* liveness analysis: end of basic block: globals are live, temps are
-   dead, local temps are live. */
-static inline void tcg_la_bb_end(TCGContext *s, uint8_t *dead_temps)
+/* liveness analysis: end of basic block: all temps are dead, globals
+   and local temps should be in memory. */
+static inline void tcg_la_bb_end(TCGContext *s, uint8_t *dead_temps,
+                                 uint8_t *mem_temps)
 {
     int i;
-    TCGTemp *ts;
 
-    memset(dead_temps, 0, s->nb_globals);
-    ts = &s->temps[s->nb_globals];
+    memset(dead_temps, 1, s->nb_temps);
+    memset(mem_temps, 1, s->nb_globals);
     for(i = s->nb_globals; i < s->nb_temps; i++) {
-        if (ts->temp_local)
-            dead_temps[i] = 0;
-        else
-            dead_temps[i] = 1;
-        ts++;
+        mem_temps[i] = s->temps[i].temp_local;
     }
 }
 
@@ -1225,22 +1221,25 @@ static void tcg_liveness_analysis(TCGContext *s)
     TCGOpcode op;
     TCGArg *args;
     const TCGOpDef *def;
-    uint8_t *dead_temps;
-    unsigned int dead_args;
+    uint8_t *dead_temps, *mem_temps;
+    uint16_t dead_args;
+    uint8_t sync_args;
     
-    gen_opc_ptr++; /* skip end */
+    s->gen_opc_ptr++; /* skip end */
 
-    nb_ops = gen_opc_ptr - gen_opc_buf;
+    nb_ops = s->gen_opc_ptr - s->gen_opc_buf;
 
     s->op_dead_args = tcg_malloc(nb_ops * sizeof(uint16_t));
+    s->op_sync_args = tcg_malloc(nb_ops * sizeof(uint8_t));
     
     dead_temps = tcg_malloc(s->nb_temps);
-    memset(dead_temps, 1, s->nb_temps);
+    mem_temps = tcg_malloc(s->nb_temps);
+    tcg_la_func_end(s, dead_temps, mem_temps);
 
-    args = gen_opparam_ptr;
+    args = s->gen_opparam_ptr;
     op_index = nb_ops - 1;
     while (op_index >= 0) {
-        op = gen_opc_buf[op_index];
+        op = s->gen_opc_buf[op_index];
         def = &tcg_op_defs[op];
         switch(op) {
         case INDEX_op_call:
@@ -1256,30 +1255,41 @@ static void tcg_liveness_analysis(TCGContext *s)
 
                 /* pure functions can be removed if their result is not
                    used */
-                if (call_flags & TCG_CALL_PURE) {
+                if (call_flags & TCG_CALL_NO_SIDE_EFFECTS) {
                     for(i = 0; i < nb_oargs; i++) {
                         arg = args[i];
-                        if (!dead_temps[arg])
+                        if (!dead_temps[arg] || mem_temps[arg]) {
                             goto do_not_remove_call;
+                        }
                     }
-                    tcg_set_nop(s, gen_opc_buf + op_index, 
+                    tcg_set_nop(s, s->gen_opc_buf + op_index,
                                 args - 1, nb_args);
                 } else {
                 do_not_remove_call:
 
                     /* output args are dead */
                     dead_args = 0;
+                    sync_args = 0;
                     for(i = 0; i < nb_oargs; i++) {
                         arg = args[i];
                         if (dead_temps[arg]) {
                             dead_args |= (1 << i);
                         }
+                        if (mem_temps[arg]) {
+                            sync_args |= (1 << i);
+                        }
                         dead_temps[arg] = 1;
+                        mem_temps[arg] = 0;
                     }
-                    
-                    if (!(call_flags & TCG_CALL_CONST)) {
-                        /* globals are live (they may be used by the call) */
-                        memset(dead_temps, 0, s->nb_globals);
+
+                    if (!(call_flags & TCG_CALL_NO_READ_GLOBALS)) {
+                        /* globals should be synced to memory */
+                        memset(mem_temps, 1, s->nb_globals);
+                    }
+                    if (!(call_flags & (TCG_CALL_NO_WRITE_GLOBALS |
+                                        TCG_CALL_NO_READ_GLOBALS))) {
+                        /* globals should go back to memory */
+                        memset(dead_temps, 1, s->nb_globals);
                     }
 
                     /* input args are live */
@@ -1293,15 +1303,11 @@ static void tcg_liveness_analysis(TCGContext *s)
                         }
                     }
                     s->op_dead_args[op_index] = dead_args;
+                    s->op_sync_args[op_index] = sync_args;
                 }
                 args--;
             }
             break;
-        case INDEX_op_set_label:
-            args--;
-            /* mark end of basic block */
-            tcg_la_bb_end(s, dead_temps);
-            break;
         case INDEX_op_debug_insn_start:
             args -= def->nb_args;
             break;
@@ -1313,11 +1319,62 @@ static void tcg_liveness_analysis(TCGContext *s)
             args--;
             /* mark the temporary as dead */
             dead_temps[args[0]] = 1;
+            mem_temps[args[0]] = 0;
             break;
         case INDEX_op_end:
             break;
-            /* XXX: optimize by hardcoding common cases (e.g. triadic ops) */
+
+        case INDEX_op_add2_i32:
+        case INDEX_op_sub2_i32:
+            args -= 6;
+            nb_iargs = 4;
+            nb_oargs = 2;
+            /* Test if the high part of the operation is dead, but not
+               the low part.  The result can be optimized to a simple
+               add or sub.  This happens often for x86_64 guest when the
+               cpu mode is set to 32 bit.  */
+            if (dead_temps[args[1]] && !mem_temps[args[1]]) {
+                if (dead_temps[args[0]] && !mem_temps[args[0]]) {
+                    goto do_remove;
+                }
+                /* Create the single operation plus nop.  */
+                if (op == INDEX_op_add2_i32) {
+                    op = INDEX_op_add_i32;
+                } else {
+                    op = INDEX_op_sub_i32;
+                }
+                s->gen_opc_buf[op_index] = op;
+                args[1] = args[2];
+                args[2] = args[4];
+                assert(s->gen_opc_buf[op_index + 1] == INDEX_op_nop);
+                tcg_set_nop(s, s->gen_opc_buf + op_index + 1, args + 3, 3);
+                /* Fall through and mark the single-word operation live.  */
+                nb_iargs = 2;
+                nb_oargs = 1;
+            }
+            goto do_not_remove;
+
+        case INDEX_op_mulu2_i32:
+            args -= 4;
+            nb_iargs = 2;
+            nb_oargs = 2;
+            /* Likewise, test for the high part of the operation dead.  */
+            if (dead_temps[args[1]] && !mem_temps[args[1]]) {
+                if (dead_temps[args[0]] && !mem_temps[args[0]]) {
+                    goto do_remove;
+                }
+                s->gen_opc_buf[op_index] = op = INDEX_op_mul_i32;
+                args[1] = args[2];
+                args[2] = args[3];
+                assert(s->gen_opc_buf[op_index + 1] == INDEX_op_nop);
+                tcg_set_nop(s, s->gen_opc_buf + op_index + 1, args + 3, 1);
+                /* Fall through and mark the single-word operation live.  */
+                nb_oargs = 1;
+            }
+            goto do_not_remove;
+
         default:
+            /* XXX: optimize by hardcoding common cases (e.g. triadic ops) */
             args -= def->nb_args;
             nb_iargs = def->nb_iargs;
             nb_oargs = def->nb_oargs;
@@ -1328,10 +1385,12 @@ static void tcg_liveness_analysis(TCGContext *s)
             if (!(def->flags & TCG_OPF_SIDE_EFFECTS) && nb_oargs != 0) {
                 for(i = 0; i < nb_oargs; i++) {
                     arg = args[i];
-                    if (!dead_temps[arg])
+                    if (!dead_temps[arg] || mem_temps[arg]) {
                         goto do_not_remove;
+                    }
                 }
-                tcg_set_nop(s, gen_opc_buf + op_index, args, def->nb_args);
+            do_remove:
+                tcg_set_nop(s, s->gen_opc_buf + op_index, args, def->nb_args);
 #ifdef CONFIG_PROFILER
                 s->del_op_count++;
 #endif
@@ -1340,20 +1399,25 @@ static void tcg_liveness_analysis(TCGContext *s)
 
                 /* output args are dead */
                 dead_args = 0;
+                sync_args = 0;
                 for(i = 0; i < nb_oargs; i++) {
                     arg = args[i];
                     if (dead_temps[arg]) {
                         dead_args |= (1 << i);
                     }
+                    if (mem_temps[arg]) {
+                        sync_args |= (1 << i);
+                    }
                     dead_temps[arg] = 1;
+                    mem_temps[arg] = 0;
                 }
 
                 /* if end of basic block, update */
                 if (def->flags & TCG_OPF_BB_END) {
-                    tcg_la_bb_end(s, dead_temps);
-                } else if (def->flags & TCG_OPF_CALL_CLOBBER) {
-                    /* globals are live */
-                    memset(dead_temps, 0, s->nb_globals);
+                    tcg_la_bb_end(s, dead_temps, mem_temps);
+                } else if (def->flags & TCG_OPF_SIDE_EFFECTS) {
+                    /* globals should be synced to memory */
+                    memset(mem_temps, 1, s->nb_globals);
                 }
 
                 /* input args are live */
@@ -1365,24 +1429,28 @@ static void tcg_liveness_analysis(TCGContext *s)
                     dead_temps[arg] = 0;
                 }
                 s->op_dead_args[op_index] = dead_args;
+                s->op_sync_args[op_index] = sync_args;
             }
             break;
         }
         op_index--;
     }
 
-    if (args != gen_opparam_buf)
+    if (args != s->gen_opparam_buf) {
         tcg_abort();
+    }
 }
 #else
 /* dummy liveness analysis */
 static void tcg_liveness_analysis(TCGContext *s)
 {
     int nb_ops;
-    nb_ops = gen_opc_ptr - gen_opc_buf;
+    nb_ops = s->gen_opc_ptr - s->gen_opc_buf;
 
     s->op_dead_args = tcg_malloc(nb_ops * sizeof(uint16_t));
     memset(s->op_dead_args, 0, nb_ops * sizeof(uint16_t));
+    s->op_sync_args = tcg_malloc(nb_ops * sizeof(uint8_t));
+    memset(s->op_sync_args, 0, nb_ops * sizeof(uint8_t));
 }
 #endif
 
@@ -1463,7 +1531,8 @@ static void temp_allocate_frame(TCGContext *s, int temp)
 {
     TCGTemp *ts;
     ts = &s->temps[temp];
-#ifndef __sparc_v9__ /* Sparc64 stack is accessed with offset of 2047 */
+#if !(defined(__sparc__) && TCG_TARGET_REG_BITS == 64)
+    /* Sparc64 stack is accessed with offset of 2047 */
     s->current_frame_offset = (s->current_frame_offset +
                                (tcg_target_long)sizeof(tcg_target_long) - 1) &
         ~(sizeof(tcg_target_long) - 1);
@@ -1478,22 +1547,33 @@ static void temp_allocate_frame(TCGContext *s, int temp)
     s->current_frame_offset += (tcg_target_long)sizeof(tcg_target_long);
 }
 
+/* sync register 'reg' by saving it to the corresponding temporary */
+static inline void tcg_reg_sync(TCGContext *s, int reg)
+{
+    TCGTemp *ts;
+    int temp;
+
+    temp = s->reg_to_temp[reg];
+    ts = &s->temps[temp];
+    assert(ts->val_type == TEMP_VAL_REG);
+    if (!ts->mem_coherent && !ts->fixed_reg) {
+        if (!ts->mem_allocated) {
+            temp_allocate_frame(s, temp);
+        }
+        tcg_out_st(s, ts->type, reg, ts->mem_reg, ts->mem_offset);
+    }
+    ts->mem_coherent = 1;
+}
+
 /* free register 'reg' by spilling the corresponding temporary if necessary */
 static void tcg_reg_free(TCGContext *s, int reg)
 {
-    TCGTemp *ts;
     int temp;
 
     temp = s->reg_to_temp[reg];
     if (temp != -1) {
-        ts = &s->temps[temp];
-        assert(ts->val_type == TEMP_VAL_REG);
-        if (!ts->mem_coherent) {
-            if (!ts->mem_allocated) 
-                temp_allocate_frame(s, temp);
-            tcg_out_st(s, ts->type, reg, ts->mem_reg, ts->mem_offset);
-        }
-        ts->val_type = TEMP_VAL_MEM;
+        tcg_reg_sync(s, reg);
+        s->temps[temp].val_type = TEMP_VAL_MEM;
         s->reg_to_temp[reg] = -1;
     }
 }
@@ -1525,31 +1605,45 @@ static int tcg_reg_alloc(TCGContext *s, TCGRegSet reg1, TCGRegSet reg2)
     tcg_abort();
 }
 
-/* save a temporary to memory. 'allocated_regs' is used in case a
+/* mark a temporary as dead. */
+static inline void temp_dead(TCGContext *s, int temp)
+{
+    TCGTemp *ts;
+
+    ts = &s->temps[temp];
+    if (!ts->fixed_reg) {
+        if (ts->val_type == TEMP_VAL_REG) {
+            s->reg_to_temp[ts->reg] = -1;
+        }
+        if (temp < s->nb_globals || ts->temp_local) {
+            ts->val_type = TEMP_VAL_MEM;
+        } else {
+            ts->val_type = TEMP_VAL_DEAD;
+        }
+    }
+}
+
+/* sync a temporary to memory. 'allocated_regs' is used in case a
    temporary registers needs to be allocated to store a constant. */
-static void temp_save(TCGContext *s, int temp, TCGRegSet allocated_regs)
+static inline void temp_sync(TCGContext *s, int temp, TCGRegSet allocated_regs)
 {
     TCGTemp *ts;
-    int reg;
 
     ts = &s->temps[temp];
     if (!ts->fixed_reg) {
         switch(ts->val_type) {
+        case TEMP_VAL_CONST:
+            ts->reg = tcg_reg_alloc(s, tcg_target_available_regs[ts->type],
+                                    allocated_regs);
+            ts->val_type = TEMP_VAL_REG;
+            s->reg_to_temp[ts->reg] = temp;
+            ts->mem_coherent = 0;
+            tcg_out_movi(s, ts->type, ts->reg, ts->val);
+            /* fallthrough*/
         case TEMP_VAL_REG:
-            tcg_reg_free(s, ts->reg);
+            tcg_reg_sync(s, ts->reg);
             break;
         case TEMP_VAL_DEAD:
-            ts->val_type = TEMP_VAL_MEM;
-            break;
-        case TEMP_VAL_CONST:
-            reg = tcg_reg_alloc(s, tcg_target_available_regs[ts->type], 
-                                allocated_regs);
-            if (!ts->mem_allocated) 
-                temp_allocate_frame(s, temp);
-            tcg_out_movi(s, ts->type, reg, ts->val);
-            tcg_out_st(s, ts->type, reg, ts->mem_reg, ts->mem_offset);
-            ts->val_type = TEMP_VAL_MEM;
-            break;
         case TEMP_VAL_MEM:
             break;
         default:
@@ -1558,6 +1652,20 @@ static void temp_save(TCGContext *s, int temp, TCGRegSet allocated_regs)
     }
 }
 
+/* save a temporary to memory. 'allocated_regs' is used in case a
+   temporary registers needs to be allocated to store a constant. */
+static inline void temp_save(TCGContext *s, int temp, TCGRegSet allocated_regs)
+{
+#ifdef USE_LIVENESS_ANALYSIS
+    /* The liveness analysis already ensures that globals are back
+       in memory. Keep an assert for safety. */
+    assert(s->temps[temp].val_type == TEMP_VAL_MEM || s->temps[temp].fixed_reg);
+#else
+    temp_sync(s, temp, allocated_regs);
+    temp_dead(s, temp);
+#endif
+}
+
 /* save globals to their canonical location and assume they can be
    modified be the following code. 'allocated_regs' is used in case a
    temporary registers needs to be allocated to store a constant. */
@@ -1570,6 +1678,23 @@ static void save_globals(TCGContext *s, TCGRegSet allocated_regs)
     }
 }
 
+/* sync globals to their canonical location and assume they can be
+   read by the following code. 'allocated_regs' is used in case a
+   temporary registers needs to be allocated to store a constant. */
+static void sync_globals(TCGContext *s, TCGRegSet allocated_regs)
+{
+    int i;
+
+    for (i = 0; i < s->nb_globals; i++) {
+#ifdef USE_LIVENESS_ANALYSIS
+        assert(s->temps[i].val_type != TEMP_VAL_REG || s->temps[i].fixed_reg ||
+               s->temps[i].mem_coherent);
+#else
+        temp_sync(s, i, allocated_regs);
+#endif
+    }
+}
+
 /* at the end of a basic block, we assume all temporaries are dead and
    all globals are stored at their canonical location. */
 static void tcg_reg_alloc_bb_end(TCGContext *s, TCGRegSet allocated_regs)
@@ -1582,10 +1707,13 @@ static void tcg_reg_alloc_bb_end(TCGContext *s, TCGRegSet allocated_regs)
         if (ts->temp_local) {
             temp_save(s, i, allocated_regs);
         } else {
-            if (ts->val_type == TEMP_VAL_REG) {
-                s->reg_to_temp[ts->reg] = -1;
-            }
-            ts->val_type = TEMP_VAL_DEAD;
+#ifdef USE_LIVENESS_ANALYSIS
+            /* The liveness analysis already ensures that temps are dead.
+               Keep an assert for safety. */
+            assert(ts->val_type == TEMP_VAL_DEAD);
+#else
+            temp_dead(s, i);
+#endif
         }
     }
 
@@ -1593,8 +1721,10 @@ static void tcg_reg_alloc_bb_end(TCGContext *s, TCGRegSet allocated_regs)
 }
 
 #define IS_DEAD_ARG(n) ((dead_args >> (n)) & 1)
+#define NEED_SYNC_ARG(n) ((sync_args >> (n)) & 1)
 
-static void tcg_reg_alloc_movi(TCGContext *s, const TCGArg *args)
+static void tcg_reg_alloc_movi(TCGContext *s, const TCGArg *args,
+                               uint16_t dead_args, uint8_t sync_args)
 {
     TCGTemp *ots;
     tcg_target_ulong val;
@@ -1613,71 +1743,99 @@ static void tcg_reg_alloc_movi(TCGContext *s, const TCGArg *args)
         ots->val_type = TEMP_VAL_CONST;
         ots->val = val;
     }
+    if (NEED_SYNC_ARG(0)) {
+        temp_sync(s, args[0], s->reserved_regs);
+    }
+    if (IS_DEAD_ARG(0)) {
+        temp_dead(s, args[0]);
+    }
 }
 
 static void tcg_reg_alloc_mov(TCGContext *s, const TCGOpDef *def,
-                              const TCGArg *args,
-                              unsigned int dead_args)
+                              const TCGArg *args, uint16_t dead_args,
+                              uint8_t sync_args)
 {
+    TCGRegSet allocated_regs;
     TCGTemp *ts, *ots;
-    int reg;
-    const TCGArgConstraint *arg_ct;
+    const TCGArgConstraint *arg_ct, *oarg_ct;
 
+    tcg_regset_set(allocated_regs, s->reserved_regs);
     ots = &s->temps[args[0]];
     ts = &s->temps[args[1]];
-    arg_ct = &def->args_ct[0];
+    oarg_ct = &def->args_ct[0];
+    arg_ct = &def->args_ct[1];
+
+    /* If the source value is not in a register, and we're going to be
+       forced to have it in a register in order to perform the copy,
+       then copy the SOURCE value into its own register first.  That way
+       we don't have to reload SOURCE the next time it is used. */
+    if (((NEED_SYNC_ARG(0) || ots->fixed_reg) && ts->val_type != TEMP_VAL_REG)
+        || ts->val_type == TEMP_VAL_MEM) {
+        ts->reg = tcg_reg_alloc(s, arg_ct->u.regs, allocated_regs);
+        if (ts->val_type == TEMP_VAL_MEM) {
+            tcg_out_ld(s, ts->type, ts->reg, ts->mem_reg, ts->mem_offset);
+            ts->mem_coherent = 1;
+        } else if (ts->val_type == TEMP_VAL_CONST) {
+            tcg_out_movi(s, ts->type, ts->reg, ts->val);
+        }
+        s->reg_to_temp[ts->reg] = args[1];
+        ts->val_type = TEMP_VAL_REG;
+    }
 
-    /* XXX: always mark arg dead if IS_DEAD_ARG(1) */
-    if (ts->val_type == TEMP_VAL_REG) {
+    if (IS_DEAD_ARG(0) && !ots->fixed_reg) {
+        /* mov to a non-saved dead register makes no sense (even with
+           liveness analysis disabled). */
+        assert(NEED_SYNC_ARG(0));
+        /* The code above should have moved the temp to a register. */
+        assert(ts->val_type == TEMP_VAL_REG);
+        if (!ots->mem_allocated) {
+            temp_allocate_frame(s, args[0]);
+        }
+        tcg_out_st(s, ots->type, ts->reg, ots->mem_reg, ots->mem_offset);
+        if (IS_DEAD_ARG(1)) {
+            temp_dead(s, args[1]);
+        }
+        temp_dead(s, args[0]);
+    } else if (ts->val_type == TEMP_VAL_CONST) {
+        /* propagate constant */
+        if (ots->val_type == TEMP_VAL_REG) {
+            s->reg_to_temp[ots->reg] = -1;
+        }
+        ots->val_type = TEMP_VAL_CONST;
+        ots->val = ts->val;
+    } else {
+        /* The code in the first if block should have moved the
+           temp to a register. */
+        assert(ts->val_type == TEMP_VAL_REG);
         if (IS_DEAD_ARG(1) && !ts->fixed_reg && !ots->fixed_reg) {
             /* the mov can be suppressed */
-            if (ots->val_type == TEMP_VAL_REG)
-                s->reg_to_temp[ots->reg] = -1;
-            reg = ts->reg;
-            s->reg_to_temp[reg] = -1;
-            ts->val_type = TEMP_VAL_DEAD;
-        } else {
             if (ots->val_type == TEMP_VAL_REG) {
-                reg = ots->reg;
-            } else {
-                reg = tcg_reg_alloc(s, arg_ct->u.regs, s->reserved_regs);
-            }
-            if (ts->reg != reg) {
-                tcg_out_mov(s, ots->type, reg, ts->reg);
+                s->reg_to_temp[ots->reg] = -1;
             }
-        }
-    } else if (ts->val_type == TEMP_VAL_MEM) {
-        if (ots->val_type == TEMP_VAL_REG) {
-            reg = ots->reg;
+            ots->reg = ts->reg;
+            temp_dead(s, args[1]);
         } else {
-            reg = tcg_reg_alloc(s, arg_ct->u.regs, s->reserved_regs);
+            if (ots->val_type != TEMP_VAL_REG) {
+                /* When allocating a new register, make sure to not spill the
+                   input one. */
+                tcg_regset_set_reg(allocated_regs, ts->reg);
+                ots->reg = tcg_reg_alloc(s, oarg_ct->u.regs, allocated_regs);
+            }
+            tcg_out_mov(s, ots->type, ots->reg, ts->reg);
         }
-        tcg_out_ld(s, ts->type, reg, ts->mem_reg, ts->mem_offset);
-    } else if (ts->val_type == TEMP_VAL_CONST) {
-        if (ots->fixed_reg) {
-            reg = ots->reg;
-            tcg_out_movi(s, ots->type, reg, ts->val);
-        } else {
-            /* propagate constant */
-            if (ots->val_type == TEMP_VAL_REG)
-                s->reg_to_temp[ots->reg] = -1;
-            ots->val_type = TEMP_VAL_CONST;
-            ots->val = ts->val;
-            return;
+        ots->val_type = TEMP_VAL_REG;
+        ots->mem_coherent = 0;
+        s->reg_to_temp[ots->reg] = args[0];
+        if (NEED_SYNC_ARG(0)) {
+            tcg_reg_sync(s, ots->reg);
         }
-    } else {
-        tcg_abort();
     }
-    s->reg_to_temp[reg] = args[0];
-    ots->reg = reg;
-    ots->val_type = TEMP_VAL_REG;
-    ots->mem_coherent = 0;
 }
 
 static void tcg_reg_alloc_op(TCGContext *s, 
                              const TCGOpDef *def, TCGOpcode opc,
-                             const TCGArg *args,
-                             unsigned int dead_args)
+                             const TCGArg *args, uint16_t dead_args,
+                             uint8_t sync_args)
 {
     TCGRegSet allocated_regs;
     int i, k, nb_iargs, nb_oargs, reg;
@@ -1757,22 +1915,16 @@ static void tcg_reg_alloc_op(TCGContext *s,
     iarg_end: ;
     }
     
+    /* mark dead temporaries and free the associated registers */
+    for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
+        if (IS_DEAD_ARG(i)) {
+            temp_dead(s, args[i]);
+        }
+    }
+
     if (def->flags & TCG_OPF_BB_END) {
         tcg_reg_alloc_bb_end(s, allocated_regs);
     } else {
-        /* mark dead temporaries and free the associated registers */
-        for(i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
-            arg = args[i];
-            if (IS_DEAD_ARG(i)) {
-                ts = &s->temps[arg];
-                if (!ts->fixed_reg) {
-                    if (ts->val_type == TEMP_VAL_REG)
-                        s->reg_to_temp[ts->reg] = -1;
-                    ts->val_type = TEMP_VAL_DEAD;
-                }
-            }
-        }
-        
         if (def->flags & TCG_OPF_CALL_CLOBBER) {
             /* XXX: permit generic clobber register list ? */ 
             for(reg = 0; reg < TCG_TARGET_NB_REGS; reg++) {
@@ -1780,12 +1932,11 @@ static void tcg_reg_alloc_op(TCGContext *s,
                     tcg_reg_free(s, reg);
                 }
             }
-            /* XXX: for load/store we could do that only for the slow path
-               (i.e. when a memory callback is called) */
-            
-            /* store globals and free associated registers (we assume the insn
-               can modify any global. */
-            save_globals(s, allocated_regs);
+        }
+        if (def->flags & TCG_OPF_SIDE_EFFECTS) {
+            /* sync globals if the op has side effects and might trigger
+               an exception. */
+            sync_globals(s, allocated_regs);
         }
         
         /* satisfy the output constraints */
@@ -1809,18 +1960,15 @@ static void tcg_reg_alloc_op(TCGContext *s,
             tcg_regset_set_reg(allocated_regs, reg);
             /* if a fixed register is used, then a move will be done afterwards */
             if (!ts->fixed_reg) {
-                if (ts->val_type == TEMP_VAL_REG)
+                if (ts->val_type == TEMP_VAL_REG) {
                     s->reg_to_temp[ts->reg] = -1;
-                if (IS_DEAD_ARG(i)) {
-                    ts->val_type = TEMP_VAL_DEAD;
-                } else {
-                    ts->val_type = TEMP_VAL_REG;
-                    ts->reg = reg;
-                    /* temp value is modified, so the value kept in memory is
-                       potentially not the same */
-                    ts->mem_coherent = 0;
-                    s->reg_to_temp[reg] = arg;
-               }
+                }
+                ts->val_type = TEMP_VAL_REG;
+                ts->reg = reg;
+                /* temp value is modified, so the value kept in memory is
+                   potentially not the same */
+                ts->mem_coherent = 0;
+                s->reg_to_temp[reg] = arg;
             }
         oarg_end:
             new_args[i] = reg;
@@ -1837,6 +1985,12 @@ static void tcg_reg_alloc_op(TCGContext *s,
         if (ts->fixed_reg && ts->reg != reg) {
             tcg_out_mov(s, ts->type, ts->reg, reg);
         }
+        if (NEED_SYNC_ARG(i)) {
+            tcg_reg_sync(s, reg);
+        }
+        if (IS_DEAD_ARG(i)) {
+            temp_dead(s, args[i]);
+        }
     }
 }
 
@@ -1848,7 +2002,7 @@ static void tcg_reg_alloc_op(TCGContext *s,
 
 static int tcg_reg_alloc_call(TCGContext *s, const TCGOpDef *def,
                               TCGOpcode opc, const TCGArg *args,
-                              unsigned int dead_args)
+                              uint16_t dead_args, uint8_t sync_args)
 {
     int nb_iargs, nb_oargs, flags, nb_regs, i, reg, nb_params;
     TCGArg arg, func_arg;
@@ -1866,7 +2020,7 @@ static int tcg_reg_alloc_call(TCGContext *s, const TCGOpDef *def,
 
     flags = args[nb_oargs + nb_iargs];
 
-    nb_regs = tcg_target_get_call_iarg_regs_count(flags);
+    nb_regs = ARRAY_SIZE(tcg_target_call_iarg_regs);
     if (nb_regs > nb_params)
         nb_regs = nb_params;
 
@@ -1972,14 +2126,8 @@ static int tcg_reg_alloc_call(TCGContext *s, const TCGOpDef *def,
     
     /* mark dead temporaries and free the associated registers */
     for(i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
-        arg = args[i];
         if (IS_DEAD_ARG(i)) {
-            ts = &s->temps[arg];
-            if (!ts->fixed_reg) {
-                if (ts->val_type == TEMP_VAL_REG)
-                    s->reg_to_temp[ts->reg] = -1;
-                ts->val_type = TEMP_VAL_DEAD;
-            }
+            temp_dead(s, args[i]);
         }
     }
     
@@ -1989,10 +2137,14 @@ static int tcg_reg_alloc_call(TCGContext *s, const TCGOpDef *def,
             tcg_reg_free(s, reg);
         }
     }
-    
-    /* store globals and free associated registers (we assume the call
-       can modify any global. */
-    if (!(flags & TCG_CALL_CONST)) {
+
+    /* Save globals if they might be written by the helper, sync them if
+       they might be read. */
+    if (flags & TCG_CALL_NO_READ_GLOBALS) {
+        /* Nothing to do */
+    } else if (flags & TCG_CALL_NO_WRITE_GLOBALS) {
+        sync_globals(s, allocated_regs);
+    } else {
         save_globals(s, allocated_regs);
     }
 
@@ -2009,15 +2161,18 @@ static int tcg_reg_alloc_call(TCGContext *s, const TCGOpDef *def,
                 tcg_out_mov(s, ts->type, ts->reg, reg);
             }
         } else {
-            if (ts->val_type == TEMP_VAL_REG)
+            if (ts->val_type == TEMP_VAL_REG) {
                 s->reg_to_temp[ts->reg] = -1;
+            }
+            ts->val_type = TEMP_VAL_REG;
+            ts->reg = reg;
+            ts->mem_coherent = 0;
+            s->reg_to_temp[reg] = arg;
+            if (NEED_SYNC_ARG(i)) {
+                tcg_reg_sync(s, reg);
+            }
             if (IS_DEAD_ARG(i)) {
-                ts->val_type = TEMP_VAL_DEAD;
-            } else {
-                ts->val_type = TEMP_VAL_REG;
-                ts->reg = reg;
-                ts->mem_coherent = 0;
-                s->reg_to_temp[reg] = arg;
+                temp_dead(s, args[i]);
             }
         }
     }
@@ -2048,7 +2203,6 @@ static inline int tcg_gen_code_common(TCGContext *s, uint8_t *gen_code_buf,
     TCGOpcode opc;
     int op_index;
     const TCGOpDef *def;
-    unsigned int dead_args;
     const TCGArg *args;
 
 #ifdef DEBUG_DISAS
@@ -2059,22 +2213,29 @@ static inline int tcg_gen_code_common(TCGContext *s, uint8_t *gen_code_buf,
     }
 #endif
 
+#ifdef CONFIG_PROFILER
+    s->opt_time -= profile_getclock();
+#endif
+
 #ifdef USE_TCG_OPTIMIZATIONS
-    gen_opparam_ptr =
-        tcg_optimize(s, gen_opc_ptr, gen_opparam_buf, tcg_op_defs);
+    s->gen_opparam_ptr =
+        tcg_optimize(s, s->gen_opc_ptr, s->gen_opparam_buf, tcg_op_defs);
 #endif
 
 #ifdef CONFIG_PROFILER
+    s->opt_time += profile_getclock();
     s->la_time -= profile_getclock();
 #endif
+
     tcg_liveness_analysis(s);
+
 #ifdef CONFIG_PROFILER
     s->la_time += profile_getclock();
 #endif
 
 #ifdef DEBUG_DISAS
     if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_OPT))) {
-        qemu_log("OP after liveness analysis:\n");
+        qemu_log("OP after optimization and liveness analysis:\n");
         tcg_dump_ops(s);
         qemu_log("\n");
     }
@@ -2085,11 +2246,11 @@ static inline int tcg_gen_code_common(TCGContext *s, uint8_t *gen_code_buf,
     s->code_buf = gen_code_buf;
     s->code_ptr = gen_code_buf;
 
-    args = gen_opparam_buf;
+    args = s->gen_opparam_buf;
     op_index = 0;
 
     for(;;) {
-        opc = gen_opc_buf[op_index];
+        opc = s->gen_opc_buf[op_index];
 #ifdef CONFIG_PROFILER
         tcg_table_op_count[opc]++;
 #endif
@@ -2101,17 +2262,14 @@ static inline int tcg_gen_code_common(TCGContext *s, uint8_t *gen_code_buf,
 #endif
         switch(opc) {
         case INDEX_op_mov_i32:
-#if TCG_TARGET_REG_BITS == 64
         case INDEX_op_mov_i64:
-#endif
-            dead_args = s->op_dead_args[op_index];
-            tcg_reg_alloc_mov(s, def, args, dead_args);
+            tcg_reg_alloc_mov(s, def, args, s->op_dead_args[op_index],
+                              s->op_sync_args[op_index]);
             break;
         case INDEX_op_movi_i32:
-#if TCG_TARGET_REG_BITS == 64
         case INDEX_op_movi_i64:
-#endif
-            tcg_reg_alloc_movi(s, args);
+            tcg_reg_alloc_movi(s, args, s->op_dead_args[op_index],
+                               s->op_sync_args[op_index]);
             break;
         case INDEX_op_debug_insn_start:
             /* debug instruction */
@@ -2125,24 +2283,16 @@ static inline int tcg_gen_code_common(TCGContext *s, uint8_t *gen_code_buf,
             args += args[0];
             goto next;
         case INDEX_op_discard:
-            {
-                TCGTemp *ts;
-                ts = &s->temps[args[0]];
-                /* mark the temporary as dead */
-                if (!ts->fixed_reg) {
-                    if (ts->val_type == TEMP_VAL_REG)
-                        s->reg_to_temp[ts->reg] = -1;
-                    ts->val_type = TEMP_VAL_DEAD;
-                }
-            }
+            temp_dead(s, args[0]);
             break;
         case INDEX_op_set_label:
             tcg_reg_alloc_bb_end(s, s->reserved_regs);
             tcg_out_label(s, args[0], s->code_ptr);
             break;
         case INDEX_op_call:
-            dead_args = s->op_dead_args[op_index];
-            args += tcg_reg_alloc_call(s, def, opc, args, dead_args);
+            args += tcg_reg_alloc_call(s, def, opc, args,
+                                       s->op_dead_args[op_index],
+                                       s->op_sync_args[op_index]);
             goto next;
         case INDEX_op_end:
             goto the_end;
@@ -2154,8 +2304,8 @@ static inline int tcg_gen_code_common(TCGContext *s, uint8_t *gen_code_buf,
             /* Note: in order to speed up the code, it would be much
                faster to have specialized register allocator functions for
                some common argument patterns */
-            dead_args = s->op_dead_args[op_index];
-            tcg_reg_alloc_op(s, def, opc, args, dead_args);
+            tcg_reg_alloc_op(s, def, opc, args, s->op_dead_args[op_index],
+                             s->op_sync_args[op_index]);
             break;
         }
         args += def->nb_args;
@@ -2169,6 +2319,10 @@ static inline int tcg_gen_code_common(TCGContext *s, uint8_t *gen_code_buf,
 #endif
     }
  the_end:
+#if defined(CONFIG_QEMU_LDST_OPTIMIZATION) && defined(CONFIG_SOFTMMU)
+    /* Generate TB finalization at the end of block */
+    tcg_out_tb_finalize(s);
+#endif
     return -1;
 }
 
@@ -2177,7 +2331,7 @@ int tcg_gen_code(TCGContext *s, uint8_t *gen_code_buf)
 #ifdef CONFIG_PROFILER
     {
         int n;
-        n = (gen_opc_ptr - gen_opc_buf);
+        n = (s->gen_opc_ptr - s->gen_opc_buf);
         s->op_count += n;
         if (n > s->op_count_max)
             s->op_count_max = n;
@@ -2241,6 +2395,9 @@ void tcg_dump_info(FILE *f, fprintf_function cpu_fprintf)
                 (double)s->interm_time / tot * 100.0);
     cpu_fprintf(f, "  gen_code time     %0.1f%%\n", 
                 (double)s->code_time / tot * 100.0);
+    cpu_fprintf(f, "optim./code time    %0.1f%%\n",
+                (double)s->opt_time / (s->code_time ? s->code_time : 1)
+                * 100.0);
     cpu_fprintf(f, "liveness/code time  %0.1f%%\n", 
                 (double)s->la_time / (s->code_time ? s->code_time : 1) * 100.0);
     cpu_fprintf(f, "cpu_restore count   %" PRId64 "\n",
diff --git a/tcg/tcg.h b/tcg/tcg.h
index d710694e0..9481e35ab 100644
--- a/tcg/tcg.h
+++ b/tcg/tcg.h
@@ -79,6 +79,7 @@ typedef uint64_t TCGRegSet;
 #define TCG_TARGET_HAS_nand_i64         0
 #define TCG_TARGET_HAS_nor_i64          0
 #define TCG_TARGET_HAS_deposit_i64      0
+#define TCG_TARGET_HAS_movcond_i64      0
 #endif
 
 #ifndef TCG_TARGET_deposit_i32_valid
@@ -187,6 +188,24 @@ typedef tcg_target_ulong TCGArg;
    are aliases for target_ulong and host pointer sized values respectively.
  */
 
+#if defined(CONFIG_QEMU_LDST_OPTIMIZATION) && defined(CONFIG_SOFTMMU)
+/* Macros/structures for qemu_ld/st IR code optimization:
+   TCG_MAX_HELPER_LABELS is defined as same as OPC_BUF_SIZE in exec-all.h. */
+#define TCG_MAX_QEMU_LDST       640
+
+typedef struct TCGLabelQemuLdst {
+    int is_ld:1;            /* qemu_ld: 1, qemu_st: 0 */
+    int opc:4;
+    int addrlo_reg;         /* reg index for low word of guest virtual addr */
+    int addrhi_reg;         /* reg index for high word of guest virtual addr */
+    int datalo_reg;         /* reg index for low word to be loaded or stored */
+    int datahi_reg;         /* reg index for high word to be loaded or stored */
+    int mem_index;          /* soft MMU memory index */
+    uint8_t *raddr;         /* gen code addr of the next IR of qemu_ld/st IR */
+    uint8_t *label_ptr[2];  /* label pointers to be updated */
+} TCGLabelQemuLdst;
+#endif
+
 #ifdef CONFIG_DEBUG_TCG
 #define DEBUG_TCGV 1
 #endif
@@ -252,31 +271,47 @@ typedef int TCGv_i64;
 #define TCGV_UNUSED_I64(x) x = MAKE_TCGV_I64(-1)
 
 /* call flags */
-/* A pure function only reads its arguments and TCG global variables
-   and cannot raise exceptions. Hence a call to a pure function can be
-   safely suppressed if the return value is not used. */
-#define TCG_CALL_PURE           0x0010 
-/* A const function only reads its arguments and does not use TCG
-   global variables. Hence a call to such a function does not
-   save TCG global variables back to their canonical location. */
-#define TCG_CALL_CONST          0x0020
+/* Helper does not read globals (either directly or through an exception). It
+   implies TCG_CALL_NO_WRITE_GLOBALS. */
+#define TCG_CALL_NO_READ_GLOBALS    0x0010
+/* Helper does not write globals */
+#define TCG_CALL_NO_WRITE_GLOBALS   0x0020
+/* Helper can be safely suppressed if the return value is not used. */
+#define TCG_CALL_NO_SIDE_EFFECTS    0x0040
+
+/* convenience version of most used call flags */
+#define TCG_CALL_NO_RWG         TCG_CALL_NO_READ_GLOBALS
+#define TCG_CALL_NO_WG          TCG_CALL_NO_WRITE_GLOBALS
+#define TCG_CALL_NO_SE          TCG_CALL_NO_SIDE_EFFECTS
+#define TCG_CALL_NO_RWG_SE      (TCG_CALL_NO_RWG | TCG_CALL_NO_SE)
+#define TCG_CALL_NO_WG_SE       (TCG_CALL_NO_WG | TCG_CALL_NO_SE)
 
 /* used to align parameters */
 #define TCG_CALL_DUMMY_TCGV     MAKE_TCGV_I32(-1)
 #define TCG_CALL_DUMMY_ARG      ((TCGArg)(-1))
 
+/* Conditions.  Note that these are layed out for easy manipulation by
+   the the functions below:
+     bit 0 is used for inverting;
+     bit 1 is signed,
+     bit 2 is unsigned,
+     bit 3 is used with bit 0 for swapping signed/unsigned.  */
 typedef enum {
-    TCG_COND_EQ,
-    TCG_COND_NE,
-    TCG_COND_LT,
-    TCG_COND_GE,
-    TCG_COND_LE,
-    TCG_COND_GT,
+    /* non-signed */
+    TCG_COND_NEVER  = 0 | 0 | 0 | 0,
+    TCG_COND_ALWAYS = 0 | 0 | 0 | 1,
+    TCG_COND_EQ     = 8 | 0 | 0 | 0,
+    TCG_COND_NE     = 8 | 0 | 0 | 1,
+    /* signed */
+    TCG_COND_LT     = 0 | 0 | 2 | 0,
+    TCG_COND_GE     = 0 | 0 | 2 | 1,
+    TCG_COND_LE     = 8 | 0 | 2 | 0,
+    TCG_COND_GT     = 8 | 0 | 2 | 1,
     /* unsigned */
-    TCG_COND_LTU,
-    TCG_COND_GEU,
-    TCG_COND_LEU,
-    TCG_COND_GTU,
+    TCG_COND_LTU    = 0 | 4 | 0 | 0,
+    TCG_COND_GEU    = 0 | 4 | 0 | 1,
+    TCG_COND_LEU    = 8 | 4 | 0 | 0,
+    TCG_COND_GTU    = 8 | 4 | 0 | 1,
 } TCGCond;
 
 /* Invert the sense of the comparison.  */
@@ -288,13 +323,34 @@ static inline TCGCond tcg_invert_cond(TCGCond c)
 /* Swap the operands in a comparison.  */
 static inline TCGCond tcg_swap_cond(TCGCond c)
 {
-    int mask = (c < TCG_COND_LT ? 0 : c < TCG_COND_LTU ? 7 : 15);
-    return (TCGCond)(c ^ mask);
+    return c & 6 ? (TCGCond)(c ^ 9) : c;
 }
 
+/* Create an "unsigned" version of a "signed" comparison.  */
 static inline TCGCond tcg_unsigned_cond(TCGCond c)
 {
-    return (c >= TCG_COND_LT && c <= TCG_COND_GT ? c + 4 : c);
+    return c & 2 ? (TCGCond)(c ^ 6) : c;
+}
+
+/* Must a comparison be considered unsigned?  */
+static inline bool is_unsigned_cond(TCGCond c)
+{
+    return (c & 4) != 0;
+}
+
+/* Create a "high" version of a double-word comparison.
+   This removes equality from a LTE or GTE comparison.  */
+static inline TCGCond tcg_high_cond(TCGCond c)
+{
+    switch (c) {
+    case TCG_COND_GE:
+    case TCG_COND_LE:
+    case TCG_COND_GEU:
+    case TCG_COND_LEU:
+        return (TCGCond)(c ^ 8);
+    default:
+        return c;
+    }
 }
 
 #define TEMP_VAL_DEAD  0
@@ -335,7 +391,6 @@ struct TCGContext {
     TCGPool *pool_first, *pool_current, *pool_first_large;
     TCGLabel *labels;
     int nb_labels;
-    TCGTemp *temps; /* globals first, temps after */
     int nb_globals;
     int nb_temps;
     /* index of free temps, -1 if none */
@@ -343,13 +398,16 @@ struct TCGContext {
 
     /* goto_tb support */
     uint8_t *code_buf;
-    unsigned long *tb_next;
+    uintptr_t *tb_next;
     uint16_t *tb_next_offset;
     uint16_t *tb_jmp_offset; /* != NULL if USE_DIRECT_JUMP */
 
     /* liveness analysis */
     uint16_t *op_dead_args; /* for each operation, each bit tells if the
                                corresponding argument is dead */
+    uint8_t *op_sync_args;  /* for each operation, each bit tells if the
+                               corresponding output argument needs to be
+                               sync to memory. */
     
     /* tells in which temporary a given register is. It does not take
        into account fixed registers */
@@ -361,7 +419,7 @@ struct TCGContext {
     int frame_reg;
 
     uint8_t *code_ptr;
-    TCGTemp static_temps[TCG_MAX_TEMPS];
+    TCGTemp temps[TCG_MAX_TEMPS]; /* globals first, temps after */
 
     TCGHelperInfo *helpers;
     int nb_helpers;
@@ -382,20 +440,31 @@ struct TCGContext {
     int64_t interm_time;
     int64_t code_time;
     int64_t la_time;
+    int64_t opt_time;
     int64_t restore_count;
     int64_t restore_time;
 #endif
 
 #ifdef CONFIG_DEBUG_TCG
     int temps_in_use;
+    int goto_tb_issue_mask;
+#endif
+
+    uint16_t gen_opc_buf[OPC_BUF_SIZE];
+    TCGArg gen_opparam_buf[OPPARAM_BUF_SIZE];
+
+    uint16_t *gen_opc_ptr;
+    TCGArg *gen_opparam_ptr;
+
+#if defined(CONFIG_QEMU_LDST_OPTIMIZATION) && defined(CONFIG_SOFTMMU)
+    /* labels info for qemu_ld/st IRs
+       The labels help to generate TLB miss case codes at the end of TB */
+    TCGLabelQemuLdst *qemu_ldst_labels;
+    int nb_qemu_ldst_labels;
 #endif
 };
 
 extern TCGContext tcg_ctx;
-extern uint16_t *gen_opc_ptr;
-extern TCGArg *gen_opparam_ptr;
-extern uint16_t gen_opc_buf[];
-extern TCGArg gen_opparam_buf[];
 
 /* pool based memory allocation */
 
@@ -458,11 +527,6 @@ static inline TCGv_i64 tcg_temp_local_new_i64(void)
 void tcg_temp_free_i64(TCGv_i64 arg);
 char *tcg_get_arg_str_i64(TCGContext *s, char *buf, int buf_size, TCGv_i64 arg);
 
-static inline bool tcg_arg_is_local(TCGContext *s, TCGArg arg)
-{
-    return s->temps[arg].temp_local;
-}
-
 #if defined(CONFIG_DEBUG_TCG)
 /* If you call tcg_clear_temp_count() at the start of a section of
  * code which is not supposed to leak any TCG temporaries, then
@@ -499,8 +563,8 @@ enum {
     TCG_OPF_BB_END       = 0x01,
     /* Instruction clobbers call registers and potentially update globals.  */
     TCG_OPF_CALL_CLOBBER = 0x02,
-    /* Instruction has side effects: it cannot be removed
-       if its outputs are not used.  */
+    /* Instruction has side effects: it cannot be removed if its outputs
+       are not used, and might trigger exceptions.  */
     TCG_OPF_SIDE_EFFECTS = 0x04,
     /* Instruction operands are 64-bits (otherwise 32-bits).  */
     TCG_OPF_64BIT        = 0x08,
@@ -533,6 +597,15 @@ do {\
     abort();\
 } while (0)
 
+#ifdef CONFIG_DEBUG_TCG
+# define tcg_debug_assert(X) do { assert(X); } while (0)
+#elif QEMU_GNUC_PREREQ(4, 5)
+# define tcg_debug_assert(X) \
+    do { if (!(X)) { __builtin_unreachable(); } } while (0)
+#else
+# define tcg_debug_assert(X) do { (void)(X); } while (0)
+#endif
+
 void tcg_add_target_add_op_defs(const TCGTargetOpDef *tdefs);
 
 #if TCG_TARGET_REG_BITS == 32
@@ -579,7 +652,7 @@ TCGv_i64 tcg_const_i64(int64_t val);
 TCGv_i32 tcg_const_local_i32(int32_t val);
 TCGv_i64 tcg_const_local_i64(int64_t val);
 
-extern uint8_t code_gen_prologue[];
+extern uint8_t *code_gen_prologue;
 
 /* TCG targets may use a different definition of tcg_qemu_tb_exec. */
 #if !defined(tcg_qemu_tb_exec)
@@ -588,3 +661,8 @@ extern uint8_t code_gen_prologue[];
 #endif
 
 void tcg_register_jit(void *buf, size_t buf_size);
+
+#if defined(CONFIG_QEMU_LDST_OPTIMIZATION) && defined(CONFIG_SOFTMMU)
+/* Generate TB finalization at the end of block */
+void tcg_out_tb_finalize(TCGContext *s);
+#endif
diff --git a/tcg/tci/tcg-target.c b/tcg/tci/tcg-target.c
index ef8580fc8..1707169ea 100644
--- a/tcg/tci/tcg-target.c
+++ b/tcg/tci/tcg-target.c
@@ -69,7 +69,6 @@ static const TCGTargetOpDef tcg_target_op_defs[] = {
     { INDEX_op_exit_tb, { NULL } },
     { INDEX_op_goto_tb, { NULL } },
     { INDEX_op_call, { RI } },
-    { INDEX_op_jmp, { RI } },
     { INDEX_op_br, { NULL } },
 
     { INDEX_op_mov_i32, { R, R } },
@@ -123,6 +122,9 @@ static const TCGTargetOpDef tcg_target_op_defs[] = {
     { INDEX_op_rotl_i32, { R, RI, RI } },
     { INDEX_op_rotr_i32, { R, RI, RI } },
 #endif
+#if TCG_TARGET_HAS_deposit_i32
+    { INDEX_op_deposit_i32, { R, "0", R } },
+#endif
 
     { INDEX_op_brcond_i32, { R, RI } },
 
@@ -201,6 +203,9 @@ static const TCGTargetOpDef tcg_target_op_defs[] = {
     { INDEX_op_rotl_i64, { R, RI, RI } },
     { INDEX_op_rotr_i64, { R, RI, RI } },
 #endif
+#if TCG_TARGET_HAS_deposit_i64
+    { INDEX_op_deposit_i64, { R, "0", R } },
+#endif
     { INDEX_op_brcond_i64, { R, RI } },
 
 #if TCG_TARGET_HAS_ext8s_i64
@@ -300,7 +305,7 @@ static const int tcg_target_reg_alloc_order[] = {
 #endif
 };
 
-#if MAX_OPC_PARAM_IARGS != 4
+#if MAX_OPC_PARAM_IARGS != 5
 # error Fix needed, number of supported input arguments changed!
 #endif
 
@@ -309,16 +314,18 @@ static const int tcg_target_call_iarg_regs[] = {
     TCG_REG_R1,
     TCG_REG_R2,
     TCG_REG_R3,
-#if TCG_TARGET_REG_BITS == 32
-    /* 32 bit hosts need 2 * MAX_OPC_PARAM_IARGS registers. */
 #if 0 /* used for TCG_REG_CALL_STACK */
     TCG_REG_R4,
 #endif
     TCG_REG_R5,
+#if TCG_TARGET_REG_BITS == 32
+    /* 32 bit hosts need 2 * MAX_OPC_PARAM_IARGS registers. */
     TCG_REG_R6,
     TCG_REG_R7,
 #if TCG_TARGET_NB_REGS >= 16
     TCG_REG_R8,
+    TCG_REG_R9,
+    TCG_REG_R10,
 #else
 # error Too few input registers available
 #endif
@@ -581,9 +588,6 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
     case INDEX_op_call:
         tcg_out_ri(s, const_args[0], args[0]);
         break;
-    case INDEX_op_jmp:
-        TODO();
-        break;
     case INDEX_op_setcond_i32:
         tcg_out_r(s, args[0]);
         tcg_out_r(s, args[1]);
@@ -655,6 +659,15 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
         tcg_out_ri32(s, const_args[1], args[1]);
         tcg_out_ri32(s, const_args[2], args[2]);
         break;
+    case INDEX_op_deposit_i32:  /* Optional (TCG_TARGET_HAS_deposit_i32). */
+        tcg_out_r(s, args[0]);
+        tcg_out_r(s, args[1]);
+        tcg_out_r(s, args[2]);
+        assert(args[3] <= UINT8_MAX);
+        tcg_out8(s, args[3]);
+        assert(args[4] <= UINT8_MAX);
+        tcg_out8(s, args[4]);
+        break;
 
 #if TCG_TARGET_REG_BITS == 64
     case INDEX_op_mov_i64:
@@ -682,6 +695,15 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
         tcg_out_ri64(s, const_args[1], args[1]);
         tcg_out_ri64(s, const_args[2], args[2]);
         break;
+    case INDEX_op_deposit_i64:  /* Optional (TCG_TARGET_HAS_deposit_i64). */
+        tcg_out_r(s, args[0]);
+        tcg_out_r(s, args[1]);
+        tcg_out_r(s, args[2]);
+        assert(args[3] <= UINT8_MAX);
+        tcg_out8(s, args[3]);
+        assert(args[4] <= UINT8_MAX);
+        tcg_out8(s, args[4]);
+        break;
     case INDEX_op_div_i64:      /* Optional (TCG_TARGET_HAS_div_i64). */
     case INDEX_op_divu_i64:     /* Optional (TCG_TARGET_HAS_div_i64). */
     case INDEX_op_rem_i64:      /* Optional (TCG_TARGET_HAS_div_i64). */
@@ -798,9 +820,6 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
     case INDEX_op_qemu_st8:
     case INDEX_op_qemu_st16:
     case INDEX_op_qemu_st32:
-#ifdef CONFIG_TCG_PASS_AREG0
-        tcg_out_r(s, TCG_AREG0);
-#endif
         tcg_out_r(s, *args++);
         tcg_out_r(s, *args++);
 #if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
@@ -811,9 +830,6 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
 #endif
         break;
     case INDEX_op_qemu_st64:
-#ifdef CONFIG_TCG_PASS_AREG0
-        tcg_out_r(s, TCG_AREG0);
-#endif
         tcg_out_r(s, *args++);
 #if TCG_TARGET_REG_BITS == 32
         tcg_out_r(s, *args++);
@@ -867,12 +883,6 @@ static int tcg_target_const_match(tcg_target_long val,
     return arg_ct->ct & TCG_CT_CONST;
 }
 
-/* Maximum number of register used for input function arguments. */
-static int tcg_target_get_call_iarg_regs_count(int flags)
-{
-    return ARRAY_SIZE(tcg_target_call_iarg_regs);
-}
-
 static void tcg_target_init(TCGContext *s)
 {
 #if defined(CONFIG_DEBUG_TCG_INTERPRETER)
diff --git a/tcg/tci/tcg-target.h b/tcg/tci/tcg-target.h
index 30a0f2159..a832f5cf5 100644
--- a/tcg/tci/tcg-target.h
+++ b/tcg/tci/tcg-target.h
@@ -67,7 +67,7 @@
 #define TCG_TARGET_HAS_ext8u_i32        1
 #define TCG_TARGET_HAS_ext16u_i32       1
 #define TCG_TARGET_HAS_andc_i32         0
-#define TCG_TARGET_HAS_deposit_i32      0
+#define TCG_TARGET_HAS_deposit_i32      1
 #define TCG_TARGET_HAS_eqv_i32          0
 #define TCG_TARGET_HAS_nand_i32         0
 #define TCG_TARGET_HAS_nor_i32          0
@@ -75,12 +75,13 @@
 #define TCG_TARGET_HAS_not_i32          1
 #define TCG_TARGET_HAS_orc_i32          0
 #define TCG_TARGET_HAS_rot_i32          1
+#define TCG_TARGET_HAS_movcond_i32      0
 
 #if TCG_TARGET_REG_BITS == 64
 #define TCG_TARGET_HAS_bswap16_i64      1
 #define TCG_TARGET_HAS_bswap32_i64      1
 #define TCG_TARGET_HAS_bswap64_i64      1
-#define TCG_TARGET_HAS_deposit_i64      0
+#define TCG_TARGET_HAS_deposit_i64      1
 /* Not more than one of the next two defines must be 1. */
 #define TCG_TARGET_HAS_div_i64          0
 #define TCG_TARGET_HAS_div2_i64         0
@@ -98,11 +99,9 @@
 #define TCG_TARGET_HAS_not_i64          1
 #define TCG_TARGET_HAS_orc_i64          0
 #define TCG_TARGET_HAS_rot_i64          1
+#define TCG_TARGET_HAS_movcond_i64      0
 #endif /* TCG_TARGET_REG_BITS == 64 */
 
-/* Offset to user memory in user mode. */
-#define TCG_TARGET_HAS_GUEST_BASE
-
 /* Number of registers available.
    For 32 bit hosts, we need more than 8 registers (call arguments). */
 /* #define TCG_TARGET_NB_REGS 8 */
diff --git a/tci.c b/tci.c
index c79350d24..54cf1d952 100644
--- a/tci.c
+++ b/tci.c
@@ -25,7 +25,6 @@
 #endif
 
 #include "qemu-common.h"
-#include "dyngen-exec.h"        /* env */
 #include "exec-all.h"           /* MAX_OPC_PARAM_IARGS */
 #include "tcg-op.h"
 
@@ -37,17 +36,19 @@
         tcg_abort(); \
     } while (0)
 
-#if MAX_OPC_PARAM_IARGS != 4
+#if MAX_OPC_PARAM_IARGS != 5
 # error Fix needed, number of supported input arguments changed!
 #endif
 #if TCG_TARGET_REG_BITS == 32
 typedef uint64_t (*helper_function)(tcg_target_ulong, tcg_target_ulong,
                                     tcg_target_ulong, tcg_target_ulong,
                                     tcg_target_ulong, tcg_target_ulong,
+                                    tcg_target_ulong, tcg_target_ulong,
                                     tcg_target_ulong, tcg_target_ulong);
 #else
 typedef uint64_t (*helper_function)(tcg_target_ulong, tcg_target_ulong,
-                                    tcg_target_ulong, tcg_target_ulong);
+                                    tcg_target_ulong, tcg_target_ulong,
+                                    tcg_target_ulong);
 #endif
 
 /* TCI can optionally use a global register variable for env. */
@@ -63,17 +64,6 @@ uintptr_t tci_tb_ptr;
 
 static tcg_target_ulong tci_reg[TCG_TARGET_NB_REGS];
 
-#if !defined(CONFIG_TCG_PASS_AREG0)
-# define helper_ldb_mmu(env, addr, mmu_idx) __ldb_mmu(addr, mmu_idx)
-# define helper_ldw_mmu(env, addr, mmu_idx) __ldw_mmu(addr, mmu_idx)
-# define helper_ldl_mmu(env, addr, mmu_idx) __ldl_mmu(addr, mmu_idx)
-# define helper_ldq_mmu(env, addr, mmu_idx) __ldq_mmu(addr, mmu_idx)
-# define helper_stb_mmu(env, addr, val, mmu_idx) __stb_mmu(addr, val, mmu_idx)
-# define helper_stw_mmu(env, addr, val, mmu_idx) __stw_mmu(addr, val, mmu_idx)
-# define helper_stl_mmu(env, addr, val, mmu_idx) __stl_mmu(addr, val, mmu_idx)
-# define helper_stq_mmu(env, addr, val, mmu_idx) __stq_mmu(addr, val, mmu_idx)
-#endif /* !CONFIG_TCG_PASS_AREG0 */
-
 static tcg_target_ulong tci_read_reg(TCGReg index)
 {
     assert(index < ARRAY_SIZE(tci_reg));
@@ -348,9 +338,9 @@ static uint64_t tci_read_ri64(uint8_t **tb_ptr)
 }
 #endif
 
-static target_ulong tci_read_label(uint8_t **tb_ptr)
+static tcg_target_ulong tci_read_label(uint8_t **tb_ptr)
 {
-    target_ulong label = tci_read_i(tb_ptr);
+    tcg_target_ulong label = tci_read_i(tb_ptr);
     assert(label != 0);
     return label;
 }
@@ -501,18 +491,20 @@ tcg_target_ulong tcg_qemu_tb_exec(CPUArchState *cpustate, uint8_t *tb_ptr)
                                           tci_read_reg(TCG_REG_R5),
                                           tci_read_reg(TCG_REG_R6),
                                           tci_read_reg(TCG_REG_R7),
-                                          tci_read_reg(TCG_REG_R8));
+                                          tci_read_reg(TCG_REG_R8),
+                                          tci_read_reg(TCG_REG_R9),
+                                          tci_read_reg(TCG_REG_R10));
             tci_write_reg(TCG_REG_R0, tmp64);
             tci_write_reg(TCG_REG_R1, tmp64 >> 32);
 #else
             tmp64 = ((helper_function)t0)(tci_read_reg(TCG_REG_R0),
                                           tci_read_reg(TCG_REG_R1),
                                           tci_read_reg(TCG_REG_R2),
-                                          tci_read_reg(TCG_REG_R3));
+                                          tci_read_reg(TCG_REG_R3),
+                                          tci_read_reg(TCG_REG_R5));
             tci_write_reg(TCG_REG_R0, tmp64);
 #endif
             break;
-        case INDEX_op_jmp:
         case INDEX_op_br:
             label = tci_read_label(&tb_ptr);
             assert(tb_ptr == old_code_ptr + op_size);
@@ -697,6 +689,17 @@ tcg_target_ulong tcg_qemu_tb_exec(CPUArchState *cpustate, uint8_t *tb_ptr)
             tci_write_reg32(t0, (t1 >> t2) | (t1 << (32 - t2)));
             break;
 #endif
+#if TCG_TARGET_HAS_deposit_i32
+        case INDEX_op_deposit_i32:
+            t0 = *tb_ptr++;
+            t1 = tci_read_r32(&tb_ptr);
+            t2 = tci_read_r32(&tb_ptr);
+            tmp16 = *tb_ptr++;
+            tmp8 = *tb_ptr++;
+            tmp32 = (((1 << tmp8) - 1) << tmp16);
+            tci_write_reg32(t0, (t1 & ~tmp32) | ((t2 << tmp16) & tmp32));
+            break;
+#endif
         case INDEX_op_brcond_i32:
             t0 = tci_read_r32(&tb_ptr);
             t1 = tci_read_ri32(&tb_ptr);
@@ -944,6 +947,17 @@ tcg_target_ulong tcg_qemu_tb_exec(CPUArchState *cpustate, uint8_t *tb_ptr)
             TODO();
             break;
 #endif
+#if TCG_TARGET_HAS_deposit_i64
+        case INDEX_op_deposit_i64:
+            t0 = *tb_ptr++;
+            t1 = tci_read_r64(&tb_ptr);
+            t2 = tci_read_r64(&tb_ptr);
+            tmp16 = *tb_ptr++;
+            tmp8 = *tb_ptr++;
+            tmp64 = (((1ULL << tmp8) - 1) << tmp16);
+            tci_write_reg64(t0, (t1 & ~tmp64) | ((t2 << tmp16) & tmp64));
+            break;
+#endif
         case INDEX_op_brcond_i64:
             t0 = tci_read_r64(&tb_ptr);
             t1 = tci_read_ri64(&tb_ptr);
diff --git a/tests/Makefile b/tests/Makefile
index 26a67ce6f..b60f0fb8f 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -15,6 +15,8 @@ check-unit-y += tests/test-string-output-visitor$(EXESUF)
 check-unit-y += tests/test-coroutine$(EXESUF)
 check-unit-y += tests/test-visitor-serialization$(EXESUF)
 check-unit-y += tests/test-iov$(EXESUF)
+check-unit-y += tests/test-aio$(EXESUF)
+check-unit-y += tests/test-thread-pool$(EXESUF)
 
 check-block-$(CONFIG_POSIX) += tests/qemu-iotests-quick.sh
 
@@ -36,19 +38,21 @@ test-obj-y = tests/check-qint.o tests/check-qstring.o tests/check-qdict.o \
 	tests/test-qmp-input-visitor.o tests/test-qmp-input-strict.o \
 	tests/test-qmp-commands.o tests/test-visitor-serialization.o
 
-test-qapi-obj-y =  $(qobject-obj-y) $(qapi-obj-y) $(tools-obj-y)
+test-qapi-obj-y =  $(qobject-obj-y) $(qapi-obj-y) qemu-tool.o
 test-qapi-obj-y += tests/test-qapi-visit.o tests/test-qapi-types.o
 test-qapi-obj-y += module.o
 
 $(test-obj-y): QEMU_INCLUDES += -Itests
 
-tests/check-qint$(EXESUF): tests/check-qint.o qint.o $(tools-obj-y)
-tests/check-qstring$(EXESUF): tests/check-qstring.o qstring.o $(tools-obj-y)
-tests/check-qdict$(EXESUF): tests/check-qdict.o qdict.o qfloat.o qint.o qstring.o qbool.o qlist.o $(tools-obj-y)
-tests/check-qlist$(EXESUF): tests/check-qlist.o qlist.o qint.o $(tools-obj-y)
-tests/check-qfloat$(EXESUF): tests/check-qfloat.o qfloat.o $(tools-obj-y)
-tests/check-qjson$(EXESUF): tests/check-qjson.o $(qobject-obj-y) $(tools-obj-y)
-tests/test-coroutine$(EXESUF): tests/test-coroutine.o $(coroutine-obj-y) $(tools-obj-y)
+tests/check-qint$(EXESUF): tests/check-qint.o qint.o
+tests/check-qstring$(EXESUF): tests/check-qstring.o qstring.o
+tests/check-qdict$(EXESUF): tests/check-qdict.o qdict.o qfloat.o qint.o qstring.o qbool.o qlist.o
+tests/check-qlist$(EXESUF): tests/check-qlist.o qlist.o qint.o
+tests/check-qfloat$(EXESUF): tests/check-qfloat.o qfloat.o
+tests/check-qjson$(EXESUF): tests/check-qjson.o $(qobject-obj-y) qemu-tool.o
+tests/test-coroutine$(EXESUF): tests/test-coroutine.o $(coroutine-obj-y) $(tools-obj-y) $(block-obj-y) iov.o libqemustub.a
+tests/test-aio$(EXESUF): tests/test-aio.o $(coroutine-obj-y) $(tools-obj-y) $(block-obj-y) libqemustub.a
+tests/test-thread-pool$(EXESUF): tests/test-thread-pool.o $(coroutine-obj-y) $(tools-obj-y) $(block-obj-y) libqemustub.a
 tests/test-iov$(EXESUF): tests/test-iov.o iov.o
 
 tests/test-qapi-types.c tests/test-qapi-types.h :\
@@ -81,7 +85,7 @@ TARGETS=$(patsubst %-softmmu,%, $(filter %-softmmu,$(TARGET_DIRS)))
 QTEST_TARGETS=$(foreach TARGET,$(TARGETS), $(if $(check-qtest-$(TARGET)-y), $(TARGET),))
 check-qtest-$(CONFIG_POSIX)=$(foreach TARGET,$(TARGETS), $(check-qtest-$(TARGET)-y))
 
-qtest-obj-y = tests/libqtest.o $(oslib-obj-y) $(tools-obj-y)
+qtest-obj-y = tests/libqtest.o $(oslib-obj-y) libqemustub.a
 $(check-qtest-y): $(qtest-obj-y)
 
 .PHONY: check-help
diff --git a/tests/fdc-test.c b/tests/fdc-test.c
index fa7441110..4b0301da4 100644
--- a/tests/fdc-test.c
+++ b/tests/fdc-test.c
@@ -48,13 +48,17 @@ enum {
 
 enum {
     CMD_SENSE_INT           = 0x08,
+    CMD_READ_ID             = 0x0a,
     CMD_SEEK                = 0x0f,
+    CMD_VERIFY              = 0x16,
     CMD_READ                = 0xe6,
     CMD_RELATIVE_SEEK_OUT   = 0x8f,
     CMD_RELATIVE_SEEK_IN    = 0xcf,
 };
 
 enum {
+    BUSY    = 0x10,
+    NONDMA  = 0x20,
     RQM     = 0x80,
     DIO     = 0x40,
 
@@ -110,7 +114,7 @@ static void ack_irq(uint8_t *pcn)
     g_assert(!get_irq(FLOPPY_IRQ));
 }
 
-static uint8_t send_read_command(void)
+static uint8_t send_read_command(uint8_t cmd)
 {
     uint8_t drive = 0;
     uint8_t head = 0;
@@ -126,7 +130,7 @@ static uint8_t send_read_command(void)
 
     uint8_t ret = 0;
 
-    floppy_send(CMD_READ);
+    floppy_send(cmd);
     floppy_send(head << 2 | drive);
     g_assert(!get_irq(FLOPPY_IRQ));
     floppy_send(cyl);
@@ -152,7 +156,70 @@ static uint8_t send_read_command(void)
     }
 
     st0 = floppy_recv();
-    if (st0 != 0x60) {
+    if (st0 != 0x40) {
+        ret = 1;
+    }
+
+    floppy_recv();
+    floppy_recv();
+    floppy_recv();
+    floppy_recv();
+    floppy_recv();
+    floppy_recv();
+
+    return ret;
+}
+
+static uint8_t send_read_no_dma_command(int nb_sect, uint8_t expected_st0)
+{
+    uint8_t drive = 0;
+    uint8_t head = 0;
+    uint8_t cyl = 0;
+    uint8_t sect_addr = 1;
+    uint8_t sect_size = 2;
+    uint8_t eot = nb_sect;
+    uint8_t gap = 0x1b;
+    uint8_t gpl = 0xff;
+
+    uint8_t msr = 0;
+    uint8_t st0;
+
+    uint8_t ret = 0;
+
+    floppy_send(CMD_READ);
+    floppy_send(head << 2 | drive);
+    g_assert(!get_irq(FLOPPY_IRQ));
+    floppy_send(cyl);
+    floppy_send(head);
+    floppy_send(sect_addr);
+    floppy_send(sect_size);
+    floppy_send(eot);
+    floppy_send(gap);
+    floppy_send(gpl);
+
+    uint16_t i = 0;
+    uint8_t n = 2;
+    for (; i < n; i++) {
+        msr = inb(FLOPPY_BASE + reg_msr);
+        if (msr == (BUSY | NONDMA | DIO | RQM)) {
+            break;
+        }
+        sleep(1);
+    }
+
+    if (i >= n) {
+        return 1;
+    }
+
+    /* Non-DMA mode */
+    for (i = 0; i < 512 * 2 * nb_sect; i++) {
+        msr = inb(FLOPPY_BASE + reg_msr);
+        assert_bit_set(msr, BUSY | RQM | DIO);
+        inb(FLOPPY_BASE + reg_fifo);
+    }
+
+    st0 = floppy_recv();
+    if (st0 != expected_st0) {
         ret = 1;
     }
 
@@ -213,11 +280,11 @@ static void test_read_without_media(void)
 {
     uint8_t ret;
 
-    ret = send_read_command();
+    ret = send_read_command(CMD_READ);
     g_assert(ret == 0);
 }
 
-static void test_media_change(void)
+static void test_media_insert(void)
 {
     uint8_t dir;
 
@@ -245,6 +312,13 @@ static void test_media_change(void)
     assert_bit_clear(dir, DSKCHG);
     dir = inb(FLOPPY_BASE + reg_dir);
     assert_bit_clear(dir, DSKCHG);
+}
+
+static void test_media_change(void)
+{
+    uint8_t dir;
+
+    test_media_insert();
 
     /* Eject the floppy and check that DSKCHG is set. Reading it out doesn't
      * reset the bit. */
@@ -320,6 +394,108 @@ static void test_relative_seek(void)
     g_assert(pcn == 0);
 }
 
+static void test_read_id(void)
+{
+    uint8_t drive = 0;
+    uint8_t head = 0;
+    uint8_t cyl;
+    uint8_t st0;
+
+    /* Seek to track 0 and check with READ ID */
+    send_seek(0);
+
+    floppy_send(CMD_READ_ID);
+    g_assert(!get_irq(FLOPPY_IRQ));
+    floppy_send(head << 2 | drive);
+
+    while (!get_irq(FLOPPY_IRQ)) {
+        /* qemu involves a timer with READ ID... */
+        clock_step(1000000000LL / 50);
+    }
+
+    st0 = floppy_recv();
+    floppy_recv();
+    floppy_recv();
+    cyl = floppy_recv();
+    head = floppy_recv();
+    floppy_recv();
+    floppy_recv();
+
+    g_assert_cmpint(cyl, ==, 0);
+    g_assert_cmpint(head, ==, 0);
+    g_assert_cmpint(st0, ==, head << 2);
+
+    /* Seek to track 8 on head 1 and check with READ ID */
+    head = 1;
+    cyl = 8;
+
+    floppy_send(CMD_SEEK);
+    floppy_send(head << 2 | drive);
+    g_assert(!get_irq(FLOPPY_IRQ));
+    floppy_send(cyl);
+    g_assert(get_irq(FLOPPY_IRQ));
+    ack_irq(NULL);
+
+    floppy_send(CMD_READ_ID);
+    g_assert(!get_irq(FLOPPY_IRQ));
+    floppy_send(head << 2 | drive);
+
+    while (!get_irq(FLOPPY_IRQ)) {
+        /* qemu involves a timer with READ ID... */
+        clock_step(1000000000LL / 50);
+    }
+
+    st0 = floppy_recv();
+    floppy_recv();
+    floppy_recv();
+    cyl = floppy_recv();
+    head = floppy_recv();
+    floppy_recv();
+    floppy_recv();
+
+    g_assert_cmpint(cyl, ==, 8);
+    g_assert_cmpint(head, ==, 1);
+    g_assert_cmpint(st0, ==, head << 2);
+}
+
+static void test_read_no_dma_1(void)
+{
+    uint8_t ret;
+
+    outb(FLOPPY_BASE + reg_dor, inb(FLOPPY_BASE + reg_dor) & ~0x08);
+    send_seek(0);
+    ret = send_read_no_dma_command(1, 0x04);
+    g_assert(ret == 0);
+}
+
+static void test_read_no_dma_18(void)
+{
+    uint8_t ret;
+
+    outb(FLOPPY_BASE + reg_dor, inb(FLOPPY_BASE + reg_dor) & ~0x08);
+    send_seek(0);
+    ret = send_read_no_dma_command(18, 0x04);
+    g_assert(ret == 0);
+}
+
+static void test_read_no_dma_19(void)
+{
+    uint8_t ret;
+
+    outb(FLOPPY_BASE + reg_dor, inb(FLOPPY_BASE + reg_dor) & ~0x08);
+    send_seek(0);
+    ret = send_read_no_dma_command(19, 0x20);
+    g_assert(ret == 0);
+}
+
+static void test_verify(void)
+{
+    uint8_t ret;
+
+    ret = send_read_command(CMD_VERIFY);
+    g_assert(ret == 0);
+}
+
 /* success if no crash or abort */
 static void fuzz_registers(void)
 {
@@ -369,6 +545,12 @@ int main(int argc, char **argv)
     qtest_add_func("/fdc/media_change", test_media_change);
     qtest_add_func("/fdc/sense_interrupt", test_sense_interrupt);
     qtest_add_func("/fdc/relative_seek", test_relative_seek);
+    qtest_add_func("/fdc/read_id", test_read_id);
+    qtest_add_func("/fdc/verify", test_verify);
+    qtest_add_func("/fdc/media_insert", test_media_insert);
+    qtest_add_func("/fdc/read_no_dma_1", test_read_no_dma_1);
+    qtest_add_func("/fdc/read_no_dma_18", test_read_no_dma_18);
+    qtest_add_func("/fdc/read_no_dma_19", test_read_no_dma_19);
     qtest_add_func("/fdc/fuzz-registers", fuzz_registers);
 
     ret = g_test_run();
diff --git a/tests/libqtest.c b/tests/libqtest.c
index 02d039218..71b84c12d 100644
--- a/tests/libqtest.c
+++ b/tests/libqtest.c
@@ -85,6 +85,22 @@ static int socket_accept(int sock)
     return ret;
 }
 
+static pid_t qtest_qemu_pid(QTestState *s)
+{
+    FILE *f;
+    char buffer[1024];
+    pid_t pid = -1;
+
+    f = fopen(s->pid_file, "r");
+    if (f) {
+        if (fgets(buffer, sizeof(buffer), f)) {
+            pid = atoi(buffer);
+        }
+    }
+    fclose(f);
+    return pid;
+}
+
 QTestState *qtest_init(const char *extra_args)
 {
     QTestState *s;
@@ -136,25 +152,21 @@ QTestState *qtest_init(const char *extra_args)
     qtest_qmp(s, "");
     qtest_qmp(s, "{ 'execute': 'qmp_capabilities' }");
 
+    if (getenv("QTEST_STOP")) {
+        kill(qtest_qemu_pid(s), SIGSTOP);
+    }
+
     return s;
 }
 
 void qtest_quit(QTestState *s)
 {
-    FILE *f;
-    char buffer[1024];
-
-    f = fopen(s->pid_file, "r");
-    if (f) {
-        if (fgets(buffer, sizeof(buffer), f)) {
-            pid_t pid = atoi(buffer);
-            int status = 0;
-
-            kill(pid, SIGTERM);
-            waitpid(pid, &status, 0);
-        }
+    int status;
 
-        fclose(f);
+    pid_t pid = qtest_qemu_pid(s);
+    if (pid != -1) {
+        kill(pid, SIGTERM);
+        waitpid(pid, &status, 0);
     }
 
     unlink(s->pid_file);
diff --git a/tests/qemu-iotests/030 b/tests/qemu-iotests/030
index 55b16f81d..dd4ef1199 100755
--- a/tests/qemu-iotests/030
+++ b/tests/qemu-iotests/030
@@ -18,6 +18,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 
+import time
 import os
 import iotests
 from iotests import qemu_img, qemu_io
@@ -98,6 +99,43 @@ class TestSingleDrive(ImageStreamingTestCase):
                          qemu_io('-c', 'map', test_img),
                          'image file map does not match backing file after streaming')
 
+    def test_stream_pause(self):
+        self.assert_no_active_streams()
+
+        result = self.vm.qmp('block-stream', device='drive0')
+        self.assert_qmp(result, 'return', {})
+
+        result = self.vm.qmp('block-job-pause', device='drive0')
+        self.assert_qmp(result, 'return', {})
+
+        time.sleep(1)
+        result = self.vm.qmp('query-block-jobs')
+        offset = self.dictpath(result, 'return[0]/offset')
+
+        time.sleep(1)
+        result = self.vm.qmp('query-block-jobs')
+        self.assert_qmp(result, 'return[0]/offset', offset)
+
+        result = self.vm.qmp('block-job-resume', device='drive0')
+        self.assert_qmp(result, 'return', {})
+
+        completed = False
+        while not completed:
+            for event in self.vm.get_qmp_events(wait=True):
+                if event['event'] == 'BLOCK_JOB_COMPLETED':
+                    self.assert_qmp(event, 'data/type', 'stream')
+                    self.assert_qmp(event, 'data/device', 'drive0')
+                    self.assert_qmp(event, 'data/offset', self.image_len)
+                    self.assert_qmp(event, 'data/len', self.image_len)
+                    completed = True
+
+        self.assert_no_active_streams()
+        self.vm.shutdown()
+
+        self.assertEqual(qemu_io('-c', 'map', backing_img),
+                         qemu_io('-c', 'map', test_img),
+                         'image file map does not match backing file after streaming')
+
     def test_stream_partial(self):
         self.assert_no_active_streams()
 
@@ -157,6 +195,226 @@ class TestSmallerBackingFile(ImageStreamingTestCase):
         self.assert_no_active_streams()
         self.vm.shutdown()
 
+class TestErrors(ImageStreamingTestCase):
+    image_len = 2 * 1024 * 1024 # MB
+
+    # this should match STREAM_BUFFER_SIZE/512 in block/stream.c
+    STREAM_BUFFER_SIZE = 512 * 1024
+
+    def create_blkdebug_file(self, name, event, errno):
+        file = open(name, 'w')
+        file.write('''
+[inject-error]
+state = "1"
+event = "%s"
+errno = "%d"
+immediately = "off"
+once = "on"
+sector = "%d"
+
+[set-state]
+state = "1"
+event = "%s"
+new_state = "2"
+
+[set-state]
+state = "2"
+event = "%s"
+new_state = "1"
+''' % (event, errno, self.STREAM_BUFFER_SIZE / 512, event, event))
+        file.close()
+
+class TestEIO(TestErrors):
+    def setUp(self):
+        self.blkdebug_file = backing_img + ".blkdebug"
+        self.create_image(backing_img, TestErrors.image_len)
+        self.create_blkdebug_file(self.blkdebug_file, "read_aio", 5)
+        qemu_img('create', '-f', iotests.imgfmt,
+                 '-o', 'backing_file=blkdebug:%s:%s,backing_fmt=raw'
+                       % (self.blkdebug_file, backing_img),
+                 test_img)
+        self.vm = iotests.VM().add_drive(test_img)
+        self.vm.launch()
+
+    def tearDown(self):
+        self.vm.shutdown()
+        os.remove(test_img)
+        os.remove(backing_img)
+        os.remove(self.blkdebug_file)
+
+    def test_report(self):
+        self.assert_no_active_streams()
+
+        result = self.vm.qmp('block-stream', device='drive0')
+        self.assert_qmp(result, 'return', {})
+
+        completed = False
+        error = False
+        while not completed:
+            for event in self.vm.get_qmp_events(wait=True):
+                if event['event'] == 'BLOCK_JOB_ERROR':
+                    self.assert_qmp(event, 'data/device', 'drive0')
+                    self.assert_qmp(event, 'data/operation', 'read')
+                    error = True
+                elif event['event'] == 'BLOCK_JOB_COMPLETED':
+                    self.assertTrue(error, 'job completed unexpectedly')
+                    self.assert_qmp(event, 'data/type', 'stream')
+                    self.assert_qmp(event, 'data/device', 'drive0')
+                    self.assert_qmp(event, 'data/error', 'Input/output error')
+                    self.assert_qmp(event, 'data/offset', self.STREAM_BUFFER_SIZE)
+                    self.assert_qmp(event, 'data/len', self.image_len)
+                    completed = True
+
+        self.assert_no_active_streams()
+        self.vm.shutdown()
+
+    def test_ignore(self):
+        self.assert_no_active_streams()
+
+        result = self.vm.qmp('block-stream', device='drive0', on_error='ignore')
+        self.assert_qmp(result, 'return', {})
+
+        error = False
+        completed = False
+        while not completed:
+            for event in self.vm.get_qmp_events(wait=True):
+                if event['event'] == 'BLOCK_JOB_ERROR':
+                    self.assert_qmp(event, 'data/device', 'drive0')
+                    self.assert_qmp(event, 'data/operation', 'read')
+                    result = self.vm.qmp('query-block-jobs')
+                    self.assert_qmp(result, 'return[0]/paused', False)
+                    error = True
+                elif event['event'] == 'BLOCK_JOB_COMPLETED':
+                    self.assertTrue(error, 'job completed unexpectedly')
+                    self.assert_qmp(event, 'data/type', 'stream')
+                    self.assert_qmp(event, 'data/device', 'drive0')
+                    self.assert_qmp(event, 'data/error', 'Input/output error')
+                    self.assert_qmp(event, 'data/offset', self.image_len)
+                    self.assert_qmp(event, 'data/len', self.image_len)
+                    completed = True
+
+        self.assert_no_active_streams()
+        self.vm.shutdown()
+
+    def test_stop(self):
+        self.assert_no_active_streams()
+
+        result = self.vm.qmp('block-stream', device='drive0', on_error='stop')
+        self.assert_qmp(result, 'return', {})
+
+        error = False
+        completed = False
+        while not completed:
+            for event in self.vm.get_qmp_events(wait=True):
+                if event['event'] == 'BLOCK_JOB_ERROR':
+                    self.assert_qmp(event, 'data/device', 'drive0')
+                    self.assert_qmp(event, 'data/operation', 'read')
+
+                    result = self.vm.qmp('query-block-jobs')
+                    self.assert_qmp(result, 'return[0]/paused', True)
+                    self.assert_qmp(result, 'return[0]/offset', self.STREAM_BUFFER_SIZE)
+                    self.assert_qmp(result, 'return[0]/io-status', 'failed')
+
+                    result = self.vm.qmp('block-job-resume', device='drive0')
+                    self.assert_qmp(result, 'return', {})
+
+                    result = self.vm.qmp('query-block-jobs')
+                    self.assert_qmp(result, 'return[0]/paused', False)
+                    self.assert_qmp(result, 'return[0]/io-status', 'ok')
+                    error = True
+                elif event['event'] == 'BLOCK_JOB_COMPLETED':
+                    self.assertTrue(error, 'job completed unexpectedly')
+                    self.assert_qmp(event, 'data/type', 'stream')
+                    self.assert_qmp(event, 'data/device', 'drive0')
+                    self.assert_qmp_absent(event, 'data/error')
+                    self.assert_qmp(event, 'data/offset', self.image_len)
+                    self.assert_qmp(event, 'data/len', self.image_len)
+                    completed = True
+
+        self.assert_no_active_streams()
+        self.vm.shutdown()
+
+    def test_enospc(self):
+        self.assert_no_active_streams()
+
+        result = self.vm.qmp('block-stream', device='drive0', on_error='enospc')
+        self.assert_qmp(result, 'return', {})
+
+        completed = False
+        error = False
+        while not completed:
+            for event in self.vm.get_qmp_events(wait=True):
+                if event['event'] == 'BLOCK_JOB_ERROR':
+                    self.assert_qmp(event, 'data/device', 'drive0')
+                    self.assert_qmp(event, 'data/operation', 'read')
+                    error = True
+                elif event['event'] == 'BLOCK_JOB_COMPLETED':
+                    self.assertTrue(error, 'job completed unexpectedly')
+                    self.assert_qmp(event, 'data/type', 'stream')
+                    self.assert_qmp(event, 'data/device', 'drive0')
+                    self.assert_qmp(event, 'data/error', 'Input/output error')
+                    self.assert_qmp(event, 'data/offset', self.STREAM_BUFFER_SIZE)
+                    self.assert_qmp(event, 'data/len', self.image_len)
+                    completed = True
+
+        self.assert_no_active_streams()
+        self.vm.shutdown()
+
+class TestENOSPC(TestErrors):
+    def setUp(self):
+        self.blkdebug_file = backing_img + ".blkdebug"
+        self.create_image(backing_img, TestErrors.image_len)
+        self.create_blkdebug_file(self.blkdebug_file, "read_aio", 28)
+        qemu_img('create', '-f', iotests.imgfmt,
+                 '-o', 'backing_file=blkdebug:%s:%s,backing_fmt=raw'
+                       % (self.blkdebug_file, backing_img),
+                 test_img)
+        self.vm = iotests.VM().add_drive(test_img)
+        self.vm.launch()
+
+    def tearDown(self):
+        self.vm.shutdown()
+        os.remove(test_img)
+        os.remove(backing_img)
+        os.remove(self.blkdebug_file)
+
+    def test_enospc(self):
+        self.assert_no_active_streams()
+
+        result = self.vm.qmp('block-stream', device='drive0', on_error='enospc')
+        self.assert_qmp(result, 'return', {})
+
+        error = False
+        completed = False
+        while not completed:
+            for event in self.vm.get_qmp_events(wait=True):
+                if event['event'] == 'BLOCK_JOB_ERROR':
+                    self.assert_qmp(event, 'data/device', 'drive0')
+                    self.assert_qmp(event, 'data/operation', 'read')
+
+                    result = self.vm.qmp('query-block-jobs')
+                    self.assert_qmp(result, 'return[0]/paused', True)
+                    self.assert_qmp(result, 'return[0]/offset', self.STREAM_BUFFER_SIZE)
+                    self.assert_qmp(result, 'return[0]/io-status', 'nospace')
+
+                    result = self.vm.qmp('block-job-resume', device='drive0')
+                    self.assert_qmp(result, 'return', {})
+
+                    result = self.vm.qmp('query-block-jobs')
+                    self.assert_qmp(result, 'return[0]/paused', False)
+                    self.assert_qmp(result, 'return[0]/io-status', 'ok')
+                    error = True
+                elif event['event'] == 'BLOCK_JOB_COMPLETED':
+                    self.assertTrue(error, 'job completed unexpectedly')
+                    self.assert_qmp(event, 'data/type', 'stream')
+                    self.assert_qmp(event, 'data/device', 'drive0')
+                    self.assert_qmp_absent(event, 'data/error')
+                    self.assert_qmp(event, 'data/offset', self.image_len)
+                    self.assert_qmp(event, 'data/len', self.image_len)
+                    completed = True
+
+        self.assert_no_active_streams()
+        self.vm.shutdown()
 
 class TestStreamStop(ImageStreamingTestCase):
     image_len = 8 * 1024 * 1024 * 1024 # GB
@@ -173,8 +431,6 @@ class TestStreamStop(ImageStreamingTestCase):
         os.remove(backing_img)
 
     def test_stream_stop(self):
-        import time
-
         self.assert_no_active_streams()
 
         result = self.vm.qmp('block-stream', device='drive0')
diff --git a/tests/qemu-iotests/030.out b/tests/qemu-iotests/030.out
index 2f7d3902f..fa16b5cce 100644
--- a/tests/qemu-iotests/030.out
+++ b/tests/qemu-iotests/030.out
@@ -1,5 +1,5 @@
-.......
+.............
 ----------------------------------------------------------------------
-Ran 7 tests
+Ran 13 tests
 
 OK
diff --git a/tests/qemu-iotests/040 b/tests/qemu-iotests/040
new file mode 100755
index 000000000..aad535a74
--- /dev/null
+++ b/tests/qemu-iotests/040
@@ -0,0 +1,280 @@
+#!/usr/bin/env python
+#
+# Tests for image block commit.
+#
+# Copyright (C) 2012 IBM, Corp.
+# Copyright (C) 2012 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# Test for live block commit
+# Derived from Image Streaming Test 030
+
+import time
+import os
+import iotests
+from iotests import qemu_img, qemu_io
+import struct
+import errno
+
+backing_img = os.path.join(iotests.test_dir, 'backing.img')
+mid_img = os.path.join(iotests.test_dir, 'mid.img')
+test_img = os.path.join(iotests.test_dir, 'test.img')
+
+class ImageCommitTestCase(iotests.QMPTestCase):
+    '''Abstract base class for image commit test cases'''
+
+    def assert_no_active_commit(self):
+        result = self.vm.qmp('query-block-jobs')
+        self.assert_qmp(result, 'return', [])
+
+    def cancel_and_wait(self, drive='drive0'):
+        '''Cancel a block job and wait for it to finish'''
+        result = self.vm.qmp('block-job-cancel', device=drive)
+        self.assert_qmp(result, 'return', {})
+
+        cancelled = False
+        while not cancelled:
+            for event in self.vm.get_qmp_events(wait=True):
+                if event['event'] == 'BLOCK_JOB_CANCELLED':
+                    self.assert_qmp(event, 'data/type', 'commit')
+                    self.assert_qmp(event, 'data/device', drive)
+                    cancelled = True
+
+        self.assert_no_active_commit()
+
+    def create_image(self, name, size):
+        file = open(name, 'w')
+        i = 0
+        while i < size:
+            sector = struct.pack('>l504xl', i / 512, i / 512)
+            file.write(sector)
+            i = i + 512
+        file.close()
+
+
+class TestSingleDrive(ImageCommitTestCase):
+    image_len = 1 * 1024 * 1024
+    test_len = 1 * 1024 * 256
+
+    def setUp(self):
+        self.create_image(backing_img, TestSingleDrive.image_len)
+        qemu_img('create', '-f', iotests.imgfmt, '-o', 'backing_file=%s' % backing_img, mid_img)
+        qemu_img('create', '-f', iotests.imgfmt, '-o', 'backing_file=%s' % mid_img, test_img)
+        qemu_io('-c', 'write -P 0xab 0 524288', backing_img)
+        qemu_io('-c', 'write -P 0xef 524288 524288', mid_img)
+        self.vm = iotests.VM().add_drive(test_img)
+        self.vm.launch()
+
+    def tearDown(self):
+        self.vm.shutdown()
+        os.remove(test_img)
+        os.remove(mid_img)
+        os.remove(backing_img)
+
+    def test_commit(self):
+        self.assert_no_active_commit()
+        result = self.vm.qmp('block-commit', device='drive0', top='%s' % mid_img)
+        self.assert_qmp(result, 'return', {})
+
+        completed = False
+        while not completed:
+            for event in self.vm.get_qmp_events(wait=True):
+                if event['event'] == 'BLOCK_JOB_COMPLETED':
+                    self.assert_qmp(event, 'data/type', 'commit')
+                    self.assert_qmp(event, 'data/device', 'drive0')
+                    self.assert_qmp(event, 'data/offset', self.image_len)
+                    self.assert_qmp(event, 'data/len', self.image_len)
+                    completed = True
+
+        self.assert_no_active_commit()
+        self.vm.shutdown()
+
+        self.assertEqual(-1, qemu_io('-c', 'read -P 0xab 0 524288', backing_img).find("verification failed"))
+        self.assertEqual(-1, qemu_io('-c', 'read -P 0xef 524288 524288', backing_img).find("verification failed"))
+
+    def test_device_not_found(self):
+        result = self.vm.qmp('block-commit', device='nonexistent', top='%s' % mid_img)
+        self.assert_qmp(result, 'error/class', 'DeviceNotFound')
+
+    def test_top_same_base(self):
+        self.assert_no_active_commit()
+        result = self.vm.qmp('block-commit', device='drive0', top='%s' % backing_img, base='%s' % backing_img)
+        self.assert_qmp(result, 'error/class', 'GenericError')
+        self.assert_qmp(result, 'error/desc', 'Base \'%s\' not found' % backing_img)
+
+    def test_top_invalid(self):
+        self.assert_no_active_commit()
+        result = self.vm.qmp('block-commit', device='drive0', top='badfile', base='%s' % backing_img)
+        self.assert_qmp(result, 'error/class', 'GenericError')
+        self.assert_qmp(result, 'error/desc', 'Top image file badfile not found')
+
+    def test_base_invalid(self):
+        self.assert_no_active_commit()
+        result = self.vm.qmp('block-commit', device='drive0', top='%s' % mid_img, base='badfile')
+        self.assert_qmp(result, 'error/class', 'GenericError')
+        self.assert_qmp(result, 'error/desc', 'Base \'badfile\' not found')
+
+    def test_top_is_active(self):
+        self.assert_no_active_commit()
+        result = self.vm.qmp('block-commit', device='drive0', top='%s' % test_img, base='%s' % backing_img)
+        self.assert_qmp(result, 'error/class', 'GenericError')
+        self.assert_qmp(result, 'error/desc', 'Top image as the active layer is currently unsupported')
+
+    def test_top_and_base_reversed(self):
+        self.assert_no_active_commit()
+        result = self.vm.qmp('block-commit', device='drive0', top='%s' % backing_img, base='%s' % mid_img)
+        self.assert_qmp(result, 'error/class', 'GenericError')
+        self.assert_qmp(result, 'error/desc', 'Base \'%s\' not found' % mid_img)
+
+    def test_top_omitted(self):
+        self.assert_no_active_commit()
+        result = self.vm.qmp('block-commit', device='drive0')
+        self.assert_qmp(result, 'error/class', 'GenericError')
+        self.assert_qmp(result, 'error/desc', "Parameter 'top' is missing")
+
+class TestRelativePaths(ImageCommitTestCase):
+    image_len = 1 * 1024 * 1024
+    test_len = 1 * 1024 * 256
+
+    dir1 = "dir1"
+    dir2 = "dir2/"
+    dir3 = "dir2/dir3/"
+
+    test_img = os.path.join(iotests.test_dir, dir3, 'test.img')
+    mid_img = "../mid.img"
+    backing_img = "../dir1/backing.img"
+
+    backing_img_abs = os.path.join(iotests.test_dir, dir1, 'backing.img')
+    mid_img_abs = os.path.join(iotests.test_dir, dir2, 'mid.img')
+
+    def setUp(self):
+        try:
+            os.mkdir(os.path.join(iotests.test_dir, self.dir1))
+            os.mkdir(os.path.join(iotests.test_dir, self.dir2))
+            os.mkdir(os.path.join(iotests.test_dir, self.dir3))
+        except OSError as exception:
+            if exception.errno != errno.EEXIST:
+                raise
+        self.create_image(self.backing_img_abs, TestRelativePaths.image_len)
+        qemu_img('create', '-f', iotests.imgfmt, '-o', 'backing_file=%s' % self.backing_img_abs, self.mid_img_abs)
+        qemu_img('create', '-f', iotests.imgfmt, '-o', 'backing_file=%s' % self.mid_img_abs, self.test_img)
+        qemu_img('rebase', '-u', '-b', self.backing_img, self.mid_img_abs)
+        qemu_img('rebase', '-u', '-b', self.mid_img, self.test_img)
+        qemu_io('-c', 'write -P 0xab 0 524288', self.backing_img_abs)
+        qemu_io('-c', 'write -P 0xef 524288 524288', self.mid_img_abs)
+        self.vm = iotests.VM().add_drive(self.test_img)
+        self.vm.launch()
+
+    def tearDown(self):
+        self.vm.shutdown()
+        os.remove(self.test_img)
+        os.remove(self.mid_img_abs)
+        os.remove(self.backing_img_abs)
+        try:
+            os.rmdir(os.path.join(iotests.test_dir, self.dir1))
+            os.rmdir(os.path.join(iotests.test_dir, self.dir3))
+            os.rmdir(os.path.join(iotests.test_dir, self.dir2))
+        except OSError as exception:
+            if exception.errno != errno.EEXIST and exception.errno != errno.ENOTEMPTY:
+                raise
+
+    def test_commit(self):
+        self.assert_no_active_commit()
+        result = self.vm.qmp('block-commit', device='drive0', top='%s' % self.mid_img)
+        self.assert_qmp(result, 'return', {})
+
+        completed = False
+        while not completed:
+            for event in self.vm.get_qmp_events(wait=True):
+                if event['event'] == 'BLOCK_JOB_COMPLETED':
+                    self.assert_qmp(event, 'data/type', 'commit')
+                    self.assert_qmp(event, 'data/device', 'drive0')
+                    self.assert_qmp(event, 'data/offset', self.image_len)
+                    self.assert_qmp(event, 'data/len', self.image_len)
+                    completed = True
+
+        self.assert_no_active_commit()
+        self.vm.shutdown()
+
+        self.assertEqual(-1, qemu_io('-c', 'read -P 0xab 0 524288', self.backing_img_abs).find("verification failed"))
+        self.assertEqual(-1, qemu_io('-c', 'read -P 0xef 524288 524288', self.backing_img_abs).find("verification failed"))
+
+    def test_device_not_found(self):
+        result = self.vm.qmp('block-commit', device='nonexistent', top='%s' % self.mid_img)
+        self.assert_qmp(result, 'error/class', 'DeviceNotFound')
+
+    def test_top_same_base(self):
+        self.assert_no_active_commit()
+        result = self.vm.qmp('block-commit', device='drive0', top='%s' % self.mid_img, base='%s' % self.mid_img)
+        self.assert_qmp(result, 'error/class', 'GenericError')
+        self.assert_qmp(result, 'error/desc', 'Base \'%s\' not found' % self.mid_img)
+
+    def test_top_invalid(self):
+        self.assert_no_active_commit()
+        result = self.vm.qmp('block-commit', device='drive0', top='badfile', base='%s' % self.backing_img)
+        self.assert_qmp(result, 'error/class', 'GenericError')
+        self.assert_qmp(result, 'error/desc', 'Top image file badfile not found')
+
+    def test_base_invalid(self):
+        self.assert_no_active_commit()
+        result = self.vm.qmp('block-commit', device='drive0', top='%s' % self.mid_img, base='badfile')
+        self.assert_qmp(result, 'error/class', 'GenericError')
+        self.assert_qmp(result, 'error/desc', 'Base \'badfile\' not found')
+
+    def test_top_is_active(self):
+        self.assert_no_active_commit()
+        result = self.vm.qmp('block-commit', device='drive0', top='%s' % self.test_img, base='%s' % self.backing_img)
+        self.assert_qmp(result, 'error/class', 'GenericError')
+        self.assert_qmp(result, 'error/desc', 'Top image as the active layer is currently unsupported')
+
+    def test_top_and_base_reversed(self):
+        self.assert_no_active_commit()
+        result = self.vm.qmp('block-commit', device='drive0', top='%s' % self.backing_img, base='%s' % self.mid_img)
+        self.assert_qmp(result, 'error/class', 'GenericError')
+        self.assert_qmp(result, 'error/desc', 'Base \'%s\' not found' % self.mid_img)
+
+
+class TestSetSpeed(ImageCommitTestCase):
+    image_len = 80 * 1024 * 1024 # MB
+
+    def setUp(self):
+        qemu_img('create', backing_img, str(TestSetSpeed.image_len))
+        qemu_img('create', '-f', iotests.imgfmt, '-o', 'backing_file=%s' % backing_img, mid_img)
+        qemu_img('create', '-f', iotests.imgfmt, '-o', 'backing_file=%s' % mid_img, test_img)
+        self.vm = iotests.VM().add_drive(test_img)
+        self.vm.launch()
+
+    def tearDown(self):
+        self.vm.shutdown()
+        os.remove(test_img)
+        os.remove(mid_img)
+        os.remove(backing_img)
+
+    def test_set_speed(self):
+        self.assert_no_active_commit()
+
+        result = self.vm.qmp('block-commit', device='drive0', top=mid_img, speed=1024 * 1024)
+        self.assert_qmp(result, 'return', {})
+
+        # Ensure the speed we set was accepted
+        result = self.vm.qmp('query-block-jobs')
+        self.assert_qmp(result, 'return[0]/device', 'drive0')
+        self.assert_qmp(result, 'return[0]/speed', 1024 * 1024)
+
+        self.cancel_and_wait()
+
+
+if __name__ == '__main__':
+    iotests.main(supported_fmts=['qcow2', 'qed'])
diff --git a/tests/qemu-iotests/040.out b/tests/qemu-iotests/040.out
new file mode 100644
index 000000000..b6f257674
--- /dev/null
+++ b/tests/qemu-iotests/040.out
@@ -0,0 +1,5 @@
+................
+----------------------------------------------------------------------
+Ran 16 tests
+
+OK
diff --git a/tests/qemu-iotests/041 b/tests/qemu-iotests/041
new file mode 100755
index 000000000..c6eb85187
--- /dev/null
+++ b/tests/qemu-iotests/041
@@ -0,0 +1,615 @@
+#!/usr/bin/env python
+#
+# Tests for image mirroring.
+#
+# Copyright (C) 2012 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import time
+import os
+import iotests
+from iotests import qemu_img, qemu_io
+import struct
+
+backing_img = os.path.join(iotests.test_dir, 'backing.img')
+target_backing_img = os.path.join(iotests.test_dir, 'target-backing.img')
+test_img = os.path.join(iotests.test_dir, 'test.img')
+target_img = os.path.join(iotests.test_dir, 'target.img')
+
+class ImageMirroringTestCase(iotests.QMPTestCase):
+    '''Abstract base class for image mirroring test cases'''
+
+    def assert_no_active_mirrors(self):
+        result = self.vm.qmp('query-block-jobs')
+        self.assert_qmp(result, 'return', [])
+
+    def cancel_and_wait(self, drive='drive0', wait_ready=True):
+        '''Cancel a block job and wait for it to finish'''
+        if wait_ready:
+            ready = False
+            while not ready:
+                for event in self.vm.get_qmp_events(wait=True):
+                    if event['event'] == 'BLOCK_JOB_READY':
+                        self.assert_qmp(event, 'data/type', 'mirror')
+                        self.assert_qmp(event, 'data/device', drive)
+                        ready = True
+
+        result = self.vm.qmp('block-job-cancel', device=drive,
+                             force=not wait_ready)
+        self.assert_qmp(result, 'return', {})
+
+        cancelled = False
+        while not cancelled:
+            for event in self.vm.get_qmp_events(wait=True):
+                if event['event'] == 'BLOCK_JOB_COMPLETED' or \
+                   event['event'] == 'BLOCK_JOB_CANCELLED':
+                    self.assert_qmp(event, 'data/type', 'mirror')
+                    self.assert_qmp(event, 'data/device', drive)
+                    if wait_ready:
+                        self.assertEquals(event['event'], 'BLOCK_JOB_COMPLETED')
+                        self.assert_qmp(event, 'data/offset', self.image_len)
+                        self.assert_qmp(event, 'data/len', self.image_len)
+                    cancelled = True
+
+        self.assert_no_active_mirrors()
+
+    def complete_and_wait(self, drive='drive0', wait_ready=True):
+        '''Complete a block job and wait for it to finish'''
+        if wait_ready:
+            ready = False
+            while not ready:
+                for event in self.vm.get_qmp_events(wait=True):
+                    if event['event'] == 'BLOCK_JOB_READY':
+                        self.assert_qmp(event, 'data/type', 'mirror')
+                        self.assert_qmp(event, 'data/device', drive)
+                        ready = True
+
+        result = self.vm.qmp('block-job-complete', device=drive)
+        self.assert_qmp(result, 'return', {})
+
+        completed = False
+        while not completed:
+            for event in self.vm.get_qmp_events(wait=True):
+                if event['event'] == 'BLOCK_JOB_COMPLETED':
+                    self.assert_qmp(event, 'data/type', 'mirror')
+                    self.assert_qmp(event, 'data/device', drive)
+                    self.assert_qmp_absent(event, 'data/error')
+                    self.assert_qmp(event, 'data/offset', self.image_len)
+                    self.assert_qmp(event, 'data/len', self.image_len)
+                    completed = True
+
+        self.assert_no_active_mirrors()
+
+    def create_image(self, name, size):
+        file = open(name, 'w')
+        i = 0
+        while i < size:
+            sector = struct.pack('>l504xl', i / 512, i / 512)
+            file.write(sector)
+            i = i + 512
+        file.close()
+
+    def compare_images(self, img1, img2):
+        try:
+            qemu_img('convert', '-f', iotests.imgfmt, '-O', 'raw', img1, img1 + '.raw')
+            qemu_img('convert', '-f', iotests.imgfmt, '-O', 'raw', img2, img2 + '.raw')
+            file1 = open(img1 + '.raw', 'r')
+            file2 = open(img2 + '.raw', 'r')
+            return file1.read() == file2.read()
+        finally:
+            if file1 is not None:
+                file1.close()
+            if file2 is not None:
+                file2.close()
+            try:
+                os.remove(img1 + '.raw')
+            except OSError:
+                pass
+            try:
+                os.remove(img2 + '.raw')
+            except OSError:
+                pass
+
+class TestSingleDrive(ImageMirroringTestCase):
+    image_len = 1 * 1024 * 1024 # MB
+
+    def setUp(self):
+        self.create_image(backing_img, TestSingleDrive.image_len)
+        qemu_img('create', '-f', iotests.imgfmt, '-o', 'backing_file=%s' % backing_img, test_img)
+        self.vm = iotests.VM().add_drive(test_img)
+        self.vm.launch()
+
+    def tearDown(self):
+        self.vm.shutdown()
+        os.remove(test_img)
+        os.remove(backing_img)
+        try:
+            os.remove(target_img)
+        except OSError:
+            pass
+
+    def test_complete(self):
+        self.assert_no_active_mirrors()
+
+        result = self.vm.qmp('drive-mirror', device='drive0', sync='full',
+                             target=target_img)
+        self.assert_qmp(result, 'return', {})
+
+        self.complete_and_wait()
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/inserted/file', target_img)
+        self.vm.shutdown()
+        self.assertTrue(self.compare_images(test_img, target_img),
+                        'target image does not match source after mirroring')
+
+    def test_cancel(self):
+        self.assert_no_active_mirrors()
+
+        result = self.vm.qmp('drive-mirror', device='drive0', sync='full',
+                             target=target_img)
+        self.assert_qmp(result, 'return', {})
+
+        self.cancel_and_wait(wait_ready=False)
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/inserted/file', test_img)
+        self.vm.shutdown()
+
+    def test_cancel_after_ready(self):
+        self.assert_no_active_mirrors()
+
+        result = self.vm.qmp('drive-mirror', device='drive0', sync='full',
+                             target=target_img)
+        self.assert_qmp(result, 'return', {})
+
+        self.cancel_and_wait()
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/inserted/file', test_img)
+        self.vm.shutdown()
+        self.assertTrue(self.compare_images(test_img, target_img),
+                        'target image does not match source after mirroring')
+
+    def test_pause(self):
+        self.assert_no_active_mirrors()
+
+        result = self.vm.qmp('drive-mirror', device='drive0', sync='full',
+                             target=target_img)
+        self.assert_qmp(result, 'return', {})
+
+        result = self.vm.qmp('block-job-pause', device='drive0')
+        self.assert_qmp(result, 'return', {})
+
+        time.sleep(1)
+        result = self.vm.qmp('query-block-jobs')
+        offset = self.dictpath(result, 'return[0]/offset')
+
+        time.sleep(1)
+        result = self.vm.qmp('query-block-jobs')
+        self.assert_qmp(result, 'return[0]/offset', offset)
+
+        result = self.vm.qmp('block-job-resume', device='drive0')
+        self.assert_qmp(result, 'return', {})
+
+        self.complete_and_wait()
+        self.vm.shutdown()
+        self.assertTrue(self.compare_images(test_img, target_img),
+                        'target image does not match source after mirroring')
+
+    def test_large_cluster(self):
+        self.assert_no_active_mirrors()
+
+        qemu_img('create', '-f', iotests.imgfmt, '-o', 'cluster_size=%d,backing_file=%s'
+                        % (TestSingleDrive.image_len, backing_img), target_img)
+        result = self.vm.qmp('drive-mirror', device='drive0', sync='full',
+                             mode='existing', target=target_img)
+        self.assert_qmp(result, 'return', {})
+
+        self.complete_and_wait()
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/inserted/file', target_img)
+        self.vm.shutdown()
+        self.assertTrue(self.compare_images(test_img, target_img),
+                        'target image does not match source after mirroring')
+
+    def test_medium_not_found(self):
+        result = self.vm.qmp('drive-mirror', device='ide1-cd0', sync='full',
+                             target=target_img)
+        self.assert_qmp(result, 'error/class', 'GenericError')
+
+    def test_image_not_found(self):
+        result = self.vm.qmp('drive-mirror', device='drive0', sync='full',
+                             mode='existing', target=target_img)
+        self.assert_qmp(result, 'error/class', 'GenericError')
+
+    def test_device_not_found(self):
+        result = self.vm.qmp('drive-mirror', device='nonexistent', sync='full',
+                             target=target_img)
+        self.assert_qmp(result, 'error/class', 'DeviceNotFound')
+
+class TestMirrorNoBacking(ImageMirroringTestCase):
+    image_len = 2 * 1024 * 1024 # MB
+
+    def complete_and_wait(self, drive='drive0', wait_ready=True):
+        self.create_image(target_backing_img, TestMirrorNoBacking.image_len)
+        return ImageMirroringTestCase.complete_and_wait(self, drive, wait_ready)
+
+    def compare_images(self, img1, img2):
+        self.create_image(target_backing_img, TestMirrorNoBacking.image_len)
+        return ImageMirroringTestCase.compare_images(self, img1, img2)
+
+    def setUp(self):
+        self.create_image(backing_img, TestMirrorNoBacking.image_len)
+        qemu_img('create', '-f', iotests.imgfmt, '-o', 'backing_file=%s' % backing_img, test_img)
+        self.vm = iotests.VM().add_drive(test_img)
+        self.vm.launch()
+
+    def tearDown(self):
+        self.vm.shutdown()
+        os.remove(test_img)
+        os.remove(backing_img)
+        os.remove(target_backing_img)
+        os.remove(target_img)
+
+    def test_complete(self):
+        self.assert_no_active_mirrors()
+
+        qemu_img('create', '-f', iotests.imgfmt, '-o', 'backing_file=%s' % backing_img, target_img)
+        result = self.vm.qmp('drive-mirror', device='drive0', sync='full',
+                             mode='existing', target=target_img)
+        self.assert_qmp(result, 'return', {})
+
+        self.complete_and_wait()
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/inserted/file', target_img)
+        self.vm.shutdown()
+        self.assertTrue(self.compare_images(test_img, target_img),
+                        'target image does not match source after mirroring')
+
+    def test_cancel(self):
+        self.assert_no_active_mirrors()
+
+        qemu_img('create', '-f', iotests.imgfmt, '-o', 'backing_file=%s' % backing_img, target_img)
+        result = self.vm.qmp('drive-mirror', device='drive0', sync='full',
+                             mode='existing', target=target_img)
+        self.assert_qmp(result, 'return', {})
+
+        self.cancel_and_wait()
+        result = self.vm.qmp('query-block')
+        self.assert_qmp(result, 'return[0]/inserted/file', test_img)
+        self.vm.shutdown()
+        self.assertTrue(self.compare_images(test_img, target_img),
+                        'target image does not match source after mirroring')
+
+class TestReadErrors(ImageMirroringTestCase):
+    image_len = 2 * 1024 * 1024 # MB
+
+    # this should be a multiple of twice the default granularity
+    # so that we hit this offset first in state 1
+    MIRROR_GRANULARITY = 1024 * 1024
+
+    def create_blkdebug_file(self, name, event, errno):
+        file = open(name, 'w')
+        file.write('''
+[inject-error]
+state = "1"
+event = "%s"
+errno = "%d"
+immediately = "off"
+once = "on"
+sector = "%d"
+
+[set-state]
+state = "1"
+event = "%s"
+new_state = "2"
+
+[set-state]
+state = "2"
+event = "%s"
+new_state = "1"
+''' % (event, errno, self.MIRROR_GRANULARITY / 512, event, event))
+        file.close()
+
+    def setUp(self):
+        self.blkdebug_file = backing_img + ".blkdebug"
+        self.create_image(backing_img, TestReadErrors.image_len)
+        self.create_blkdebug_file(self.blkdebug_file, "read_aio", 5)
+        qemu_img('create', '-f', iotests.imgfmt,
+                 '-o', 'backing_file=blkdebug:%s:%s,backing_fmt=raw'
+                       % (self.blkdebug_file, backing_img),
+                 test_img)
+        self.vm = iotests.VM().add_drive(test_img)
+        self.vm.launch()
+
+    def tearDown(self):
+        self.vm.shutdown()
+        os.remove(test_img)
+        os.remove(backing_img)
+        os.remove(self.blkdebug_file)
+
+    def test_report_read(self):
+        self.assert_no_active_mirrors()
+
+        result = self.vm.qmp('drive-mirror', device='drive0', sync='full',
+                             target=target_img)
+        self.assert_qmp(result, 'return', {})
+
+        completed = False
+        error = False
+        while not completed:
+            for event in self.vm.get_qmp_events(wait=True):
+                if event['event'] == 'BLOCK_JOB_ERROR':
+                    self.assert_qmp(event, 'data/device', 'drive0')
+                    self.assert_qmp(event, 'data/operation', 'read')
+                    error = True
+                elif event['event'] == 'BLOCK_JOB_READY':
+                    self.assertTrue(False, 'job completed unexpectedly')
+                elif event['event'] == 'BLOCK_JOB_COMPLETED':
+                    self.assertTrue(error, 'job completed unexpectedly')
+                    self.assert_qmp(event, 'data/type', 'mirror')
+                    self.assert_qmp(event, 'data/device', 'drive0')
+                    self.assert_qmp(event, 'data/error', 'Input/output error')
+                    self.assert_qmp(event, 'data/len', self.image_len)
+                    completed = True
+
+        self.assert_no_active_mirrors()
+        self.vm.shutdown()
+
+    def test_ignore_read(self):
+        self.assert_no_active_mirrors()
+
+        result = self.vm.qmp('drive-mirror', device='drive0', sync='full',
+                             target=target_img, on_source_error='ignore')
+        self.assert_qmp(result, 'return', {})
+
+        event = self.vm.get_qmp_event(wait=True)
+        self.assertEquals(event['event'], 'BLOCK_JOB_ERROR')
+        self.assert_qmp(event, 'data/device', 'drive0')
+        self.assert_qmp(event, 'data/operation', 'read')
+        result = self.vm.qmp('query-block-jobs')
+        self.assert_qmp(result, 'return[0]/paused', False)
+        self.complete_and_wait()
+        self.vm.shutdown()
+
+    def test_stop_read(self):
+        self.assert_no_active_mirrors()
+
+        result = self.vm.qmp('drive-mirror', device='drive0', sync='full',
+                             target=target_img, on_source_error='stop')
+        self.assert_qmp(result, 'return', {})
+
+        error = False
+        ready = False
+        while not ready:
+            for event in self.vm.get_qmp_events(wait=True):
+                if event['event'] == 'BLOCK_JOB_ERROR':
+                    self.assert_qmp(event, 'data/device', 'drive0')
+                    self.assert_qmp(event, 'data/operation', 'read')
+
+                    result = self.vm.qmp('query-block-jobs')
+                    self.assert_qmp(result, 'return[0]/paused', True)
+                    self.assert_qmp(result, 'return[0]/io-status', 'failed')
+
+                    result = self.vm.qmp('block-job-resume', device='drive0')
+                    self.assert_qmp(result, 'return', {})
+                    error = True
+                elif event['event'] == 'BLOCK_JOB_READY':
+                    self.assertTrue(error, 'job completed unexpectedly')
+                    self.assert_qmp(event, 'data/device', 'drive0')
+                    ready = True
+
+        result = self.vm.qmp('query-block-jobs')
+        self.assert_qmp(result, 'return[0]/paused', False)
+        self.assert_qmp(result, 'return[0]/io-status', 'ok')
+
+        self.complete_and_wait(wait_ready=False)
+        self.assert_no_active_mirrors()
+        self.vm.shutdown()
+
+class TestWriteErrors(ImageMirroringTestCase):
+    image_len = 2 * 1024 * 1024 # MB
+
+    # this should be a multiple of twice the default granularity
+    # so that we hit this offset first in state 1
+    MIRROR_GRANULARITY = 1024 * 1024
+
+    def create_blkdebug_file(self, name, event, errno):
+        file = open(name, 'w')
+        file.write('''
+[inject-error]
+state = "1"
+event = "%s"
+errno = "%d"
+immediately = "off"
+once = "on"
+sector = "%d"
+
+[set-state]
+state = "1"
+event = "%s"
+new_state = "2"
+
+[set-state]
+state = "2"
+event = "%s"
+new_state = "1"
+''' % (event, errno, self.MIRROR_GRANULARITY / 512, event, event))
+        file.close()
+
+    def setUp(self):
+        self.blkdebug_file = target_img + ".blkdebug"
+        self.create_image(backing_img, TestWriteErrors.image_len)
+        self.create_blkdebug_file(self.blkdebug_file, "write_aio", 5)
+        qemu_img('create', '-f', iotests.imgfmt, '-obacking_file=%s' %(backing_img), test_img)
+        self.vm = iotests.VM().add_drive(test_img)
+        self.target_img = 'blkdebug:%s:%s' % (self.blkdebug_file, target_img)
+        qemu_img('create', '-f', iotests.imgfmt, '-osize=%d' %(TestWriteErrors.image_len), target_img)
+        self.vm.launch()
+
+    def tearDown(self):
+        self.vm.shutdown()
+        os.remove(test_img)
+        os.remove(backing_img)
+        os.remove(self.blkdebug_file)
+
+    def test_report_write(self):
+        self.assert_no_active_mirrors()
+
+        result = self.vm.qmp('drive-mirror', device='drive0', sync='full',
+                             mode='existing', target=self.target_img)
+        self.assert_qmp(result, 'return', {})
+
+        completed = False
+        error = False
+        while not completed:
+            for event in self.vm.get_qmp_events(wait=True):
+                if event['event'] == 'BLOCK_JOB_ERROR':
+                    self.assert_qmp(event, 'data/device', 'drive0')
+                    self.assert_qmp(event, 'data/operation', 'write')
+                    error = True
+                elif event['event'] == 'BLOCK_JOB_READY':
+                    self.assertTrue(False, 'job completed unexpectedly')
+                elif event['event'] == 'BLOCK_JOB_COMPLETED':
+                    self.assertTrue(error, 'job completed unexpectedly')
+                    self.assert_qmp(event, 'data/type', 'mirror')
+                    self.assert_qmp(event, 'data/device', 'drive0')
+                    self.assert_qmp(event, 'data/error', 'Input/output error')
+                    self.assert_qmp(event, 'data/len', self.image_len)
+                    completed = True
+
+        self.assert_no_active_mirrors()
+        self.vm.shutdown()
+
+    def test_ignore_write(self):
+        self.assert_no_active_mirrors()
+
+        result = self.vm.qmp('drive-mirror', device='drive0', sync='full',
+                             mode='existing', target=self.target_img,
+                             on_target_error='ignore')
+        self.assert_qmp(result, 'return', {})
+
+        event = self.vm.get_qmp_event(wait=True)
+        self.assertEquals(event['event'], 'BLOCK_JOB_ERROR')
+        self.assert_qmp(event, 'data/device', 'drive0')
+        self.assert_qmp(event, 'data/operation', 'write')
+        result = self.vm.qmp('query-block-jobs')
+        self.assert_qmp(result, 'return[0]/paused', False)
+        self.complete_and_wait()
+        self.vm.shutdown()
+
+    def test_stop_write(self):
+        self.assert_no_active_mirrors()
+
+        result = self.vm.qmp('drive-mirror', device='drive0', sync='full',
+                             mode='existing', target=self.target_img,
+                             on_target_error='stop')
+        self.assert_qmp(result, 'return', {})
+
+        error = False
+        ready = False
+        while not ready:
+            for event in self.vm.get_qmp_events(wait=True):
+                if event['event'] == 'BLOCK_JOB_ERROR':
+                    self.assert_qmp(event, 'data/device', 'drive0')
+                    self.assert_qmp(event, 'data/operation', 'write')
+
+                    result = self.vm.qmp('query-block-jobs')
+                    self.assert_qmp(result, 'return[0]/paused', True)
+                    self.assert_qmp(result, 'return[0]/io-status', 'failed')
+
+                    result = self.vm.qmp('block-job-resume', device='drive0')
+                    self.assert_qmp(result, 'return', {})
+
+                    result = self.vm.qmp('query-block-jobs')
+                    self.assert_qmp(result, 'return[0]/paused', False)
+                    self.assert_qmp(result, 'return[0]/io-status', 'ok')
+                    error = True
+                elif event['event'] == 'BLOCK_JOB_READY':
+                    self.assertTrue(error, 'job completed unexpectedly')
+                    self.assert_qmp(event, 'data/device', 'drive0')
+                    ready = True
+
+        self.complete_and_wait(wait_ready=False)
+        self.assert_no_active_mirrors()
+        self.vm.shutdown()
+
+class TestSetSpeed(ImageMirroringTestCase):
+    image_len = 80 * 1024 * 1024 # MB
+
+    def setUp(self):
+        qemu_img('create', backing_img, str(TestSetSpeed.image_len))
+        qemu_img('create', '-f', iotests.imgfmt, '-o', 'backing_file=%s' % backing_img, test_img)
+        self.vm = iotests.VM().add_drive(test_img)
+        self.vm.launch()
+
+    def tearDown(self):
+        self.vm.shutdown()
+        os.remove(test_img)
+        os.remove(backing_img)
+        os.remove(target_img)
+
+    def test_set_speed(self):
+        self.assert_no_active_mirrors()
+
+        result = self.vm.qmp('drive-mirror', device='drive0', sync='full',
+                             target=target_img)
+        self.assert_qmp(result, 'return', {})
+
+        # Default speed is 0
+        result = self.vm.qmp('query-block-jobs')
+        self.assert_qmp(result, 'return[0]/device', 'drive0')
+        self.assert_qmp(result, 'return[0]/speed', 0)
+
+        result = self.vm.qmp('block-job-set-speed', device='drive0', speed=8 * 1024 * 1024)
+        self.assert_qmp(result, 'return', {})
+
+        # Ensure the speed we set was accepted
+        result = self.vm.qmp('query-block-jobs')
+        self.assert_qmp(result, 'return[0]/device', 'drive0')
+        self.assert_qmp(result, 'return[0]/speed', 8 * 1024 * 1024)
+
+        self.cancel_and_wait()
+
+        # Check setting speed in drive-mirror works
+        result = self.vm.qmp('drive-mirror', device='drive0', sync='full',
+                             target=target_img, speed=4*1024*1024)
+        self.assert_qmp(result, 'return', {})
+
+        result = self.vm.qmp('query-block-jobs')
+        self.assert_qmp(result, 'return[0]/device', 'drive0')
+        self.assert_qmp(result, 'return[0]/speed', 4 * 1024 * 1024)
+
+        self.cancel_and_wait()
+
+    def test_set_speed_invalid(self):
+        self.assert_no_active_mirrors()
+
+        result = self.vm.qmp('drive-mirror', device='drive0', sync='full',
+                             target=target_img, speed=-1)
+        self.assert_qmp(result, 'error/class', 'GenericError')
+
+        self.assert_no_active_mirrors()
+
+        result = self.vm.qmp('drive-mirror', device='drive0', sync='full',
+                             target=target_img)
+        self.assert_qmp(result, 'return', {})
+
+        result = self.vm.qmp('block-job-set-speed', device='drive0', speed=-1)
+        self.assert_qmp(result, 'error/class', 'GenericError')
+
+        self.cancel_and_wait()
+
+if __name__ == '__main__':
+    iotests.main(supported_fmts=['qcow2', 'qed'])
diff --git a/tests/qemu-iotests/041.out b/tests/qemu-iotests/041.out
new file mode 100644
index 000000000..71009c239
--- /dev/null
+++ b/tests/qemu-iotests/041.out
@@ -0,0 +1,5 @@
+..................
+----------------------------------------------------------------------
+Ran 18 tests
+
+OK
diff --git a/tests/qemu-iotests/042 b/tests/qemu-iotests/042
new file mode 100755
index 000000000..c3c3ca8d0
--- /dev/null
+++ b/tests/qemu-iotests/042
@@ -0,0 +1,78 @@
+#!/bin/bash
+#
+# Test qemu-img operation on zero size images
+#
+# Copyright (C) 2012 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
+owner=kwolf@redhat.com
+
+seq=`basename $0`
+echo "QA output created by $seq"
+
+here=`pwd`
+tmp=/tmp/$$
+status=1	# failure is the default!
+
+_cleanup()
+{
+	_cleanup_test_img
+}
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+
+_supported_fmt qcow2 qcow qed vmdk
+_supported_proto file
+_supported_os Linux
+
+echo
+echo "== Creating zero size image =="
+
+_make_test_img 0
+_check_test_img
+
+mv $TEST_IMG $TEST_IMG.orig
+
+echo
+echo "== Converting the image =="
+
+$QEMU_IMG convert -O $IMGFMT $TEST_IMG.orig $TEST_IMG
+_check_test_img
+
+echo
+echo "== Converting the image, compressed =="
+
+if [ "$IMGFMT" == "qcow2" ]; then
+    $QEMU_IMG convert -c -O $IMGFMT $TEST_IMG.orig $TEST_IMG
+fi
+_check_test_img
+
+echo
+echo "== Rebasing the image =="
+
+$QEMU_IMG rebase -u -b $TEST_IMG.orig $TEST_IMG
+$QEMU_IMG rebase -b $TEST_IMG.orig $TEST_IMG
+_check_test_img
+
+# success, all done
+echo "*** done"
+rm -f $seq.full
+status=0
+
diff --git a/tests/qemu-iotests/042.out b/tests/qemu-iotests/042.out
new file mode 100644
index 000000000..dc80f4b5b
--- /dev/null
+++ b/tests/qemu-iotests/042.out
@@ -0,0 +1,15 @@
+QA output created by 042
+
+== Creating zero size image ==
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=0 
+No errors were found on the image.
+
+== Converting the image ==
+No errors were found on the image.
+
+== Converting the image, compressed ==
+No errors were found on the image.
+
+== Rebasing the image ==
+No errors were found on the image.
+*** done
diff --git a/tests/qemu-iotests/043 b/tests/qemu-iotests/043
new file mode 100755
index 000000000..3ba08dc94
--- /dev/null
+++ b/tests/qemu-iotests/043
@@ -0,0 +1,95 @@
+#!/bin/bash
+#
+# Test that qemu-img info --backing-chain detects infinite loops
+#
+# Copyright (C) 2012 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+# creator
+owner=stefanha@redhat.com
+
+seq=`basename $0`
+echo "QA output created by $seq"
+
+here=`pwd`
+tmp=/tmp/$$
+status=1	# failure is the default!
+
+_cleanup()
+{
+    _cleanup_test_img
+    rm -f $TEST_IMG.[123].base
+}
+trap "_cleanup; exit \$status" 0 1 2 3 15
+
+# get standard environment, filters and checks
+. ./common.rc
+. ./common.filter
+
+# Any format supporting backing files
+_supported_fmt qcow qcow2 vmdk qed
+_supported_proto generic
+_supported_os Linux
+
+
+size=128M
+_make_test_img $size
+$QEMU_IMG rebase -u -b $TEST_IMG $TEST_IMG
+
+echo
+echo "== backing file references self =="
+_img_info --backing-chain
+
+_make_test_img $size
+mv $TEST_IMG $TEST_IMG.base
+_make_test_img -b $TEST_IMG.base $size
+$QEMU_IMG rebase -u -b $TEST_IMG $TEST_IMG.base
+
+echo
+echo "== parent references self =="
+_img_info --backing-chain
+
+_make_test_img $size
+mv $TEST_IMG $TEST_IMG.1.base
+_make_test_img -b $TEST_IMG.1.base $size
+mv $TEST_IMG $TEST_IMG.2.base
+_make_test_img -b $TEST_IMG.2.base $size
+mv $TEST_IMG $TEST_IMG.3.base
+_make_test_img -b $TEST_IMG.3.base $size
+$QEMU_IMG rebase -u -b $TEST_IMG.2.base $TEST_IMG.1.base
+
+echo
+echo "== ancestor references another ancestor =="
+_img_info --backing-chain
+
+_make_test_img $size
+mv $TEST_IMG $TEST_IMG.1.base
+_make_test_img -b $TEST_IMG.1.base $size
+mv $TEST_IMG $TEST_IMG.2.base
+_make_test_img -b $TEST_IMG.2.base $size
+
+echo
+echo "== finite chain of length 3 (human) =="
+_img_info --backing-chain
+
+echo
+echo "== finite chain of length 3 (json) =="
+_img_info --backing-chain --output=json
+
+# success, all done
+echo "*** done"
+rm -f $seq.full
+status=0
diff --git a/tests/qemu-iotests/043.out b/tests/qemu-iotests/043.out
new file mode 100644
index 000000000..ad23337db
--- /dev/null
+++ b/tests/qemu-iotests/043.out
@@ -0,0 +1,66 @@
+QA output created by 043
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 
+
+== backing file references self ==
+qemu-img: Backing file 'TEST_DIR/t.IMGFMT' creates an infinite loop.
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file='TEST_DIR/t.IMGFMT.base' 
+
+== parent references self ==
+qemu-img: Backing file 'TEST_DIR/t.IMGFMT' creates an infinite loop.
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file='TEST_DIR/t.IMGFMT.1.base' 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file='TEST_DIR/t.IMGFMT.2.base' 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file='TEST_DIR/t.IMGFMT.3.base' 
+
+== ancestor references another ancestor ==
+qemu-img: Backing file 'TEST_DIR/t.IMGFMT.2.base' creates an infinite loop.
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file='TEST_DIR/t.IMGFMT.1.base' 
+Formatting 'TEST_DIR/t.IMGFMT', fmt=IMGFMT size=134217728 backing_file='TEST_DIR/t.IMGFMT.2.base' 
+
+== finite chain of length 3 (human) ==
+image: TEST_DIR/t.IMGFMT
+file format: IMGFMT
+virtual size: 128M (134217728 bytes)
+cluster_size: 65536
+backing file: TEST_DIR/t.IMGFMT.2.base
+
+image: TEST_DIR/t.IMGFMT.2.base
+file format: IMGFMT
+virtual size: 128M (134217728 bytes)
+cluster_size: 65536
+backing file: TEST_DIR/t.IMGFMT.1.base
+
+image: TEST_DIR/t.IMGFMT.1.base
+file format: IMGFMT
+virtual size: 128M (134217728 bytes)
+cluster_size: 65536
+
+== finite chain of length 3 (json) ==
+[
+    {
+        "virtual-size": 134217728, 
+        "filename": "TEST_DIR/t.IMGFMT", 
+        "cluster-size": 65536, 
+        "format": "IMGFMT", 
+        "backing-filename": "TEST_DIR/t.IMGFMT.2.base", 
+        "dirty-flag": false
+    }, 
+    {
+        "virtual-size": 134217728, 
+        "filename": "TEST_DIR/t.IMGFMT.2.base", 
+        "cluster-size": 65536, 
+        "format": "IMGFMT", 
+        "backing-filename": "TEST_DIR/t.IMGFMT.1.base", 
+        "dirty-flag": false
+    }, 
+    {
+        "virtual-size": 134217728, 
+        "filename": "TEST_DIR/t.IMGFMT.1.base", 
+        "cluster-size": 65536, 
+        "format": "IMGFMT", 
+        "dirty-flag": false
+    }
+]
+*** done
diff --git a/tests/qemu-iotests/044 b/tests/qemu-iotests/044
new file mode 100755
index 000000000..11ea0f4d3
--- /dev/null
+++ b/tests/qemu-iotests/044
@@ -0,0 +1,117 @@
+#!/usr/bin/env python
+#
+# Tests growing a large refcount table.
+#
+# Copyright (C) 2012 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+
+import time
+import os
+import qcow2
+from qcow2 import QcowHeader
+import iotests
+from iotests import qemu_img, qemu_img_verbose, qemu_io
+import struct
+import subprocess
+
+test_img = os.path.join(iotests.test_dir, 'test.img')
+
+class TestRefcountTableGrowth(iotests.QMPTestCase):
+    '''Abstract base class for image mirroring test cases'''
+
+    def preallocate(self, name):
+        fd = open(name, "r+b")
+        try:
+            off_reftable = 512
+            off_refblock = off_reftable + (512 * 512)
+            off_l1       = off_refblock + (512 * 512 * 64)
+            off_l2       = off_l1 + (512 * 512 * 4 * 8)
+            off_data     = off_l2 + (512 * 512 * 4 * 512)
+
+            # Write a new header
+            h = QcowHeader(fd)
+            h.refcount_table_offset = off_reftable
+            h.refcount_table_clusters = 512
+            h.l1_table_offset = off_l1
+            h.l1_size = 512 * 512 * 4
+            h.update(fd)
+
+            # Write a refcount table
+            fd.seek(off_reftable)
+
+            for i in xrange(0, h.refcount_table_clusters):
+                sector = ''.join(struct.pack('>Q',
+                    off_refblock + i * 64 * 512 + j * 512)
+                    for j in xrange(0, 64))
+                fd.write(sector)
+
+            # Write the refcount blocks
+            assert(fd.tell() == off_refblock)
+            sector = ''.join(struct.pack('>H', 1) for j in xrange(0, 64 * 256))
+            for block in xrange(0, h.refcount_table_clusters):
+                fd.write(sector)
+
+            # Write the L1 table
+            assert(fd.tell() == off_l1)
+            assert(off_l2 + 512 * h.l1_size == off_data)
+            table = ''.join(struct.pack('>Q', (1 << 63) | off_l2 + 512 * j)
+                for j in xrange(0, h.l1_size))
+            fd.write(table)
+
+            # Write the L2 tables
+            assert(fd.tell() == off_l2)
+            img_file_size = h.refcount_table_clusters * 64 * 256 * 512
+            remaining = img_file_size - off_data
+
+            off = off_data
+            while remaining > 1024 * 512:
+                pytable = list((1 << 63) | off + 512 * j
+                    for j in xrange(0, 1024))
+                table = struct.pack('>1024Q', *pytable)
+                fd.write(table)
+                remaining = remaining - 1024 * 512
+                off = off + 1024 * 512
+
+            table = ''.join(struct.pack('>Q', (1 << 63) | off + 512 * j)
+                for j in xrange(0, remaining / 512))
+            fd.write(table)
+
+
+            # Data
+            fd.truncate(img_file_size)
+
+
+        finally:
+            fd.close()
+
+
+    def setUp(self):
+        qemu_img('create', '-f', iotests.imgfmt, '-o', 'cluster_size=512', test_img, '16G')
+        self.preallocate(test_img)
+        pass
+
+
+    def tearDown(self):
+        os.remove(test_img)
+        pass
+
+    def test_grow_refcount_table(self):
+        qemu_io('-c', 'write 3800M 1M', test_img)
+        qemu_img_verbose('check' , test_img)
+        pass
+
+if __name__ == '__main__':
+    iotests.main(supported_fmts=['qcow2'])
diff --git a/tests/qemu-iotests/044.out b/tests/qemu-iotests/044.out
new file mode 100644
index 000000000..7a4007137
--- /dev/null
+++ b/tests/qemu-iotests/044.out
@@ -0,0 +1,6 @@
+No errors were found on the image.
+.
+----------------------------------------------------------------------
+Ran 1 tests
+
+OK
diff --git a/tests/qemu-iotests/common b/tests/qemu-iotests/common
index 1f6fdf5c5..b3aad89e2 100644
--- a/tests/qemu-iotests/common
+++ b/tests/qemu-iotests/common
@@ -136,6 +136,7 @@ check options
     -vmdk               test vmdk
     -rbd                test rbd
     -sheepdog           test sheepdog
+    -nbd                test nbd
     -xdiff		graphical mode diff
     -nocache		use O_DIRECT on backing file
     -misalign		misalign memory allocations
@@ -197,12 +198,14 @@ testlist options
 	    IMGPROTO=rbd
 	    xpand=false
 	    ;;
-
 	-sheepdog)
 	    IMGPROTO=sheepdog
 	    xpand=false
 	    ;;
-
+	-nbd)
+	    IMGPROTO=nbd
+	    xpand=false
+	    ;;
 	-nocache)
 	    QEMU_IO_OPTIONS="$QEMU_IO_OPTIONS --nocache"
 	    xpand=false
@@ -350,7 +353,11 @@ fi
 
 [ "$QEMU" = "" ] && _fatal "qemu not found"
 [ "$QEMU_IMG" = "" ] && _fatal "qemu-img not found"
-[ "$QEMU_IO" = "" ] && _fatal "qemu-img not found"
+[ "$QEMU_IO" = "" ] && _fatal "qemu-io not found"
+
+if [ "$IMGPROTO" = "nbd" ] ; then
+    [ "$QEMU_NBD" = "" ] && _fatal "qemu-nbd not found"
+fi
 
 if $valgrind; then
     export REAL_QEMU_IO="$QEMU_IO_PROG"
diff --git a/tests/qemu-iotests/common.config b/tests/qemu-iotests/common.config
index df082e750..08a3f100b 100644
--- a/tests/qemu-iotests/common.config
+++ b/tests/qemu-iotests/common.config
@@ -90,21 +90,23 @@ export PS_ALL_FLAGS="-ef"
 if [ -z "$QEMU_PROG" ]; then
     export QEMU_PROG="`set_prog_path qemu`"
 fi
-[ "$QEMU_PROG" = "" ] && _fatal "qemu not found"
 
 if [ -z "$QEMU_IMG_PROG" ]; then
     export QEMU_IMG_PROG="`set_prog_path qemu-img`"
 fi
-[ "$QEMU_IMG_PROG" = "" ] && _fatal "qemu-img not found"
 
 if [ -z "$QEMU_IO_PROG" ]; then
     export QEMU_IO_PROG="`set_prog_path qemu-io`"
 fi
-[ "$QEMU_IO_PROG" = "" ] && _fatal "qemu-io not found"
+
+if [ -z "$QEMU_NBD_PROG" ]; then
+    export QEMU_NBD_PROG="`set_prog_path qemu-nbd`"
+fi
 
 export QEMU=$QEMU_PROG
-export QEMU_IMG=$QEMU_IMG_PROG 
+export QEMU_IMG=$QEMU_IMG_PROG
 export QEMU_IO="$QEMU_IO_PROG $QEMU_IO_OPTIONS"
+export QEMU_NBD=$QEMU_NBD_PROG
 
 [ -f /etc/qemu-iotest.config ]       && . /etc/qemu-iotest.config
 
diff --git a/tests/qemu-iotests/common.rc b/tests/qemu-iotests/common.rc
index d534e9466..aef5f52b4 100644
--- a/tests/qemu-iotests/common.rc
+++ b/tests/qemu-iotests/common.rc
@@ -49,6 +49,9 @@ umask 022
 
 if [ "$IMGPROTO" = "file" ]; then
     TEST_IMG=$TEST_DIR/t.$IMGFMT
+elif [ "$IMGPROTO" = "nbd" ]; then
+    TEST_IMG_FILE=$TEST_DIR/t.$IMGFMT
+    TEST_IMG="nbd:127.0.0.1:10810"
 else
     TEST_IMG=$IMGPROTO:$TEST_DIR/t.$IMGFMT
 fi
@@ -86,6 +89,13 @@ _make_test_img()
     local extra_img_options=""
     local image_size=$*
     local optstr=""
+    local img_name=""
+
+    if [ -n "$TEST_IMG_FILE" ]; then
+        img_name=$TEST_IMG_FILE
+    else
+        img_name=$TEST_IMG
+    fi
 
     if [ -n "$IMGOPTS" ]; then
         optstr=$(_optstr_add "$optstr" "$IMGOPTS")
@@ -104,7 +114,7 @@ _make_test_img()
     fi
 
     # XXX(hch): have global image options?
-    $QEMU_IMG create -f $IMGFMT $extra_img_options $TEST_IMG $image_size | \
+    $QEMU_IMG create -f $IMGFMT $extra_img_options $img_name $image_size | \
         sed -e "s#$IMGPROTO:$TEST_DIR#TEST_DIR#g" \
             -e "s#$TEST_DIR#TEST_DIR#g" \
             -e "s#$IMGFMT#IMGFMT#g" \
@@ -115,12 +125,23 @@ _make_test_img()
             -e "s# compat6=\\(on\\|off\\)##g" \
             -e "s# static=\\(on\\|off\\)##g" \
             -e "s# lazy_refcounts=\\(on\\|off\\)##g"
+
+    # Start an NBD server on the image file, which is what we'll be talking to
+    if [ $IMGPROTO = "nbd" ]; then
+        eval "$QEMU_NBD -v -t -b 127.0.0.1 -p 10810  $TEST_IMG_FILE &"
+        QEMU_NBD_PID=$!
+        sleep 1 # FIXME: qemu-nbd needs to be listening before we continue
+    fi
 }
 
 _cleanup_test_img()
 {
     case "$IMGPROTO" in
 
+        nbd)
+            kill $QEMU_NBD_PID
+            rm -f $TEST_IMG_FILE
+            ;;
         file)
             rm -f $TEST_DIR/t.$IMGFMT
             rm -f $TEST_DIR/t.$IMGFMT.orig
@@ -145,6 +166,16 @@ _check_test_img()
     	sed -e 's/qemu-img\: This image format does not support checks/No errors were found on the image./'
 }
 
+_img_info()
+{
+    $QEMU_IMG info "$@" $TEST_IMG 2>&1 | \
+        sed -e "s#$IMGPROTO:$TEST_DIR#TEST_DIR#g" \
+            -e "s#$TEST_DIR#TEST_DIR#g" \
+            -e "s#$IMGFMT#IMGFMT#g" \
+            -e "/^disk size:/ D" \
+            -e "/actual-size/ D"
+}
+
 _get_pids_by_name()
 {
     if [ $# -ne 1 ]
diff --git a/tests/qemu-iotests/group b/tests/qemu-iotests/group
index ebb5ca4b4..a4a9044f2 100644
--- a/tests/qemu-iotests/group
+++ b/tests/qemu-iotests/group
@@ -36,7 +36,7 @@
 027 rw auto quick
 028 rw backing auto
 029 rw auto quick
-030 rw auto
+030 rw auto backing
 031 rw auto quick
 032 rw auto
 033 rw auto
@@ -46,3 +46,8 @@
 037 rw auto backing
 038 rw auto backing
 039 rw auto
+040 rw auto
+041 rw auto backing
+042 rw auto quick
+043 rw auto backing
+044 rw auto
diff --git a/tests/qemu-iotests/iotests.py b/tests/qemu-iotests/iotests.py
index e05b1d640..b2eaf20f0 100644
--- a/tests/qemu-iotests/iotests.py
+++ b/tests/qemu-iotests/iotests.py
@@ -19,6 +19,7 @@
 import os
 import re
 import subprocess
+import string
 import unittest
 import sys; sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'QMP'))
 import qmp
@@ -41,6 +42,10 @@ def qemu_img(*args):
     devnull = open('/dev/null', 'r+')
     return subprocess.call(qemu_img_args + list(args), stdin=devnull, stdout=devnull)
 
+def qemu_img_verbose(*args):
+    '''Run qemu-img without supressing its output and return the exit code'''
+    return subprocess.call(qemu_img_args + list(args))
+
 def qemu_io(*args):
     '''Run qemu-io and return the stdout data'''
     args = qemu_io_args + list(args)
@@ -96,9 +101,18 @@ class VM(object):
             os.remove(self._qemu_log_path)
             self._popen = None
 
+    underscore_to_dash = string.maketrans('_', '-')
     def qmp(self, cmd, **args):
         '''Invoke a QMP command and return the result dict'''
-        return self._qmp.cmd(cmd, args=args)
+        qmp_args = dict()
+        for k in args.keys():
+            qmp_args[k.translate(self.underscore_to_dash)] = args[k]
+
+        return self._qmp.cmd(cmd, args=qmp_args)
+
+    def get_qmp_event(self, wait=False):
+        '''Poll for one queued QMP events and return it'''
+        return self._qmp.pull_event(wait=wait)
 
     def get_qmp_events(self, wait=False):
         '''Poll for queued QMP events and return a list of dicts'''
@@ -132,6 +146,13 @@ class QMPTestCase(unittest.TestCase):
                     self.fail('invalid index "%s" in path "%s" in "%s"' % (idx, path, str(d)))
         return d
 
+    def assert_qmp_absent(self, d, path):
+        try:
+            result = self.dictpath(d, path)
+        except AssertionError:
+            return
+        self.fail('path "%s" has value "%s"' % (path, str(result)))
+
     def assert_qmp(self, d, path, value):
         '''Assert that the value for a specific path in a QMP dict matches'''
         result = self.dictpath(d, path)
@@ -165,4 +186,4 @@ def main(supported_fmts=[]):
     try:
         unittest.main(testRunner=MyTestRunner)
     finally:
-        sys.stderr.write(re.sub(r'Ran (\d+) test[s] in [\d.]+s', r'Ran \1 tests', output.getvalue()))
+        sys.stderr.write(re.sub(r'Ran (\d+) tests? in [\d.]+s', r'Ran \1 tests', output.getvalue()))
diff --git a/tests/qemu-iotests/qcow2.py b/tests/qemu-iotests/qcow2.py
index 97f37707b..fecf5b9a5 100755
--- a/tests/qemu-iotests/qcow2.py
+++ b/tests/qemu-iotests/qcow2.py
@@ -233,8 +233,9 @@ def usage():
     for name, handler, num_args, desc in cmds:
         print "    %-20s - %s" % (name, desc)
 
-if len(sys.argv) < 3:
-    usage()
-    sys.exit(1)
+if __name__ == '__main__':
+    if len(sys.argv) < 3:
+        usage()
+        sys.exit(1)
 
-main(sys.argv[1], sys.argv[2], sys.argv[3:])
+    main(sys.argv[1], sys.argv[2], sys.argv[3:])
diff --git a/tests/rtc-test.c b/tests/rtc-test.c
index f23ac3a6c..02edbf572 100644
--- a/tests/rtc-test.c
+++ b/tests/rtc-test.c
@@ -179,6 +179,77 @@ static void check_time(int wiggle)
 
 static int wiggle = 2;
 
+static void set_year_20xx(void)
+{
+    /* Set BCD mode */
+    cmos_write(RTC_REG_B, cmos_read(RTC_REG_B) & ~REG_B_DM);
+    cmos_write(RTC_REG_A, 0x76);
+    cmos_write(RTC_YEAR, 0x11);
+    cmos_write(RTC_CENTURY, 0x20);
+    cmos_write(RTC_MONTH, 0x02);
+    cmos_write(RTC_DAY_OF_MONTH, 0x02);
+    cmos_write(RTC_HOURS, 0x02);
+    cmos_write(RTC_MINUTES, 0x04);
+    cmos_write(RTC_SECONDS, 0x58);
+    cmos_write(RTC_REG_A, 0x26);
+
+    g_assert_cmpint(cmos_read(RTC_HOURS), ==, 0x02);
+    g_assert_cmpint(cmos_read(RTC_MINUTES), ==, 0x04);
+    g_assert_cmpint(cmos_read(RTC_SECONDS), >=, 0x58);
+    g_assert_cmpint(cmos_read(RTC_DAY_OF_MONTH), ==, 0x02);
+    g_assert_cmpint(cmos_read(RTC_MONTH), ==, 0x02);
+    g_assert_cmpint(cmos_read(RTC_YEAR), ==, 0x11);
+    g_assert_cmpint(cmos_read(RTC_CENTURY), ==, 0x20);
+
+    /* Set a date in 2080 to ensure there is no year-2038 overflow.  */
+    cmos_write(RTC_REG_A, 0x76);
+    cmos_write(RTC_YEAR, 0x80);
+    cmos_write(RTC_REG_A, 0x26);
+
+    g_assert_cmpint(cmos_read(RTC_HOURS), ==, 0x02);
+    g_assert_cmpint(cmos_read(RTC_MINUTES), ==, 0x04);
+    g_assert_cmpint(cmos_read(RTC_SECONDS), >=, 0x58);
+    g_assert_cmpint(cmos_read(RTC_DAY_OF_MONTH), ==, 0x02);
+    g_assert_cmpint(cmos_read(RTC_MONTH), ==, 0x02);
+    g_assert_cmpint(cmos_read(RTC_YEAR), ==, 0x80);
+    g_assert_cmpint(cmos_read(RTC_CENTURY), ==, 0x20);
+
+    cmos_write(RTC_REG_A, 0x76);
+    cmos_write(RTC_YEAR, 0x11);
+    cmos_write(RTC_REG_A, 0x26);
+
+    g_assert_cmpint(cmos_read(RTC_HOURS), ==, 0x02);
+    g_assert_cmpint(cmos_read(RTC_MINUTES), ==, 0x04);
+    g_assert_cmpint(cmos_read(RTC_SECONDS), >=, 0x58);
+    g_assert_cmpint(cmos_read(RTC_DAY_OF_MONTH), ==, 0x02);
+    g_assert_cmpint(cmos_read(RTC_MONTH), ==, 0x02);
+    g_assert_cmpint(cmos_read(RTC_YEAR), ==, 0x11);
+    g_assert_cmpint(cmos_read(RTC_CENTURY), ==, 0x20);
+}
+
+static void set_year_1980(void)
+{
+    /* Set BCD mode */
+    cmos_write(RTC_REG_B, cmos_read(RTC_REG_B) & ~REG_B_DM);
+    cmos_write(RTC_REG_A, 0x76);
+    cmos_write(RTC_YEAR, 0x80);
+    cmos_write(RTC_CENTURY, 0x19);
+    cmos_write(RTC_MONTH, 0x02);
+    cmos_write(RTC_DAY_OF_MONTH, 0x02);
+    cmos_write(RTC_HOURS, 0x02);
+    cmos_write(RTC_MINUTES, 0x04);
+    cmos_write(RTC_SECONDS, 0x58);
+    cmos_write(RTC_REG_A, 0x26);
+
+    g_assert_cmpint(cmos_read(RTC_HOURS), ==, 0x02);
+    g_assert_cmpint(cmos_read(RTC_MINUTES), ==, 0x04);
+    g_assert_cmpint(cmos_read(RTC_SECONDS), >=, 0x58);
+    g_assert_cmpint(cmos_read(RTC_DAY_OF_MONTH), ==, 0x02);
+    g_assert_cmpint(cmos_read(RTC_MONTH), ==, 0x02);
+    g_assert_cmpint(cmos_read(RTC_YEAR), ==, 0x80);
+    g_assert_cmpint(cmos_read(RTC_CENTURY), ==, 0x19);
+}
+
 static void bcd_check_time(void)
 {
     /* Set BCD mode */
@@ -256,6 +327,45 @@ static void fuzz_registers(void)
     }
 }
 
+static void register_b_set_flag(void)
+{
+    /* Enable binary-coded decimal (BCD) mode and SET flag in Register B*/
+    cmos_write(RTC_REG_B, (cmos_read(RTC_REG_B) & ~REG_B_DM) | REG_B_SET);
+
+    cmos_write(RTC_REG_A, 0x76);
+    cmos_write(RTC_YEAR, 0x11);
+    cmos_write(RTC_CENTURY, 0x20);
+    cmos_write(RTC_MONTH, 0x02);
+    cmos_write(RTC_DAY_OF_MONTH, 0x02);
+    cmos_write(RTC_HOURS, 0x02);
+    cmos_write(RTC_MINUTES, 0x04);
+    cmos_write(RTC_SECONDS, 0x58);
+    cmos_write(RTC_REG_A, 0x26);
+
+    /* Since SET flag is still enabled, these are equality checks. */
+    g_assert_cmpint(cmos_read(RTC_HOURS), ==, 0x02);
+    g_assert_cmpint(cmos_read(RTC_MINUTES), ==, 0x04);
+    g_assert_cmpint(cmos_read(RTC_SECONDS), ==, 0x58);
+    g_assert_cmpint(cmos_read(RTC_DAY_OF_MONTH), ==, 0x02);
+    g_assert_cmpint(cmos_read(RTC_MONTH), ==, 0x02);
+    g_assert_cmpint(cmos_read(RTC_YEAR), ==, 0x11);
+    g_assert_cmpint(cmos_read(RTC_CENTURY), ==, 0x20);
+
+    /* Disable SET flag in Register B */
+    cmos_write(RTC_REG_B, cmos_read(RTC_REG_B) & ~REG_B_SET);
+
+    g_assert_cmpint(cmos_read(RTC_HOURS), ==, 0x02);
+    g_assert_cmpint(cmos_read(RTC_MINUTES), ==, 0x04);
+
+    /* Since SET flag is disabled, this is an inequality check.
+     * We (reasonably) assume that no (sexagesimal) overflow occurs. */
+    g_assert_cmpint(cmos_read(RTC_SECONDS), >=, 0x58);
+    g_assert_cmpint(cmos_read(RTC_DAY_OF_MONTH), ==, 0x02);
+    g_assert_cmpint(cmos_read(RTC_MONTH), ==, 0x02);
+    g_assert_cmpint(cmos_read(RTC_YEAR), ==, 0x11);
+    g_assert_cmpint(cmos_read(RTC_CENTURY), ==, 0x20);
+}
+
 int main(int argc, char **argv)
 {
     QTestState *s = NULL;
@@ -269,6 +379,9 @@ int main(int argc, char **argv)
     qtest_add_func("/rtc/bcd/check-time", bcd_check_time);
     qtest_add_func("/rtc/dec/check-time", dec_check_time);
     qtest_add_func("/rtc/alarm-time", alarm_time);
+    qtest_add_func("/rtc/set-year/20xx", set_year_20xx);
+    qtest_add_func("/rtc/set-year/1980", set_year_1980);
+    qtest_add_func("/rtc/register_b_set_flag", register_b_set_flag);
     qtest_add_func("/rtc/fuzz-registers", fuzz_registers);
     ret = g_test_run();
 
diff --git a/tests/tcg/Makefile b/tests/tcg/Makefile
index 15e36a208..24e3154ca 100644
--- a/tests/tcg/Makefile
+++ b/tests/tcg/Makefile
@@ -1,13 +1,13 @@
--include ../config-host.mak
+-include ../../config-host.mak
 -include $(SRC_PATH)/rules.mak
 
-$(call set-vpath, $(SRC_PATH)/tests)
+$(call set-vpath, $(SRC_PATH)/tests/tcg)
 
-QEMU=../i386-linux-user/qemu-i386
-QEMU_X86_64=../x86_64-linux-user/qemu-x86_64
+QEMU=../../i386-linux-user/qemu-i386
+QEMU_X86_64=../../x86_64-linux-user/qemu-x86_64
 CC_X86_64=$(CC_I386) -m64
 
-QEMU_INCLUDES += -I..
+QEMU_INCLUDES += -I../..
 CFLAGS=-Wall -O2 -g -fno-strict-aliasing
 #CFLAGS+=-msse2
 LDFLAGS=
@@ -22,6 +22,7 @@ I386_TESTS=hello-i386 \
 	   testthread \
 	   sha1-i386 \
 	   test-i386 \
+	   test-i386-fprem \
 	   test-mmap \
 	   # runcom
 
@@ -36,6 +37,7 @@ TESTS += $(I386_TESTS)
 endif
 
 all: $(patsubst %,run-%,$(TESTS))
+test: all
 
 # rules to run tests
 
@@ -54,6 +56,11 @@ run-test-i386: test-i386
 	-$(QEMU) test-i386 > test-i386.out
 	@if diff -u test-i386.ref test-i386.out ; then echo "Auto Test OK"; fi
 
+run-test-i386-fprem: test-i386-fprem
+	./test-i386-fprem > test-i386-fprem.ref
+	-$(QEMU) test-i386-fprem > test-i386-fprem.out
+	@if diff -u test-i386-fprem.ref test-i386-fprem.out ; then echo "Auto Test OK"; fi
+
 run-test-x86_64: test-x86_64
 	./test-x86_64 > test-x86_64.ref
 	-$(QEMU_X86_64) test-x86_64 > test-x86_64.out
@@ -74,7 +81,10 @@ run-test_path: test_path
 # rules to compile tests
 
 test_path: test_path.o
+	$(CC_I386) $(LDFLAGS) -o $@ $^ $(LIBS)
+
 test_path.o: test_path.c
+	$(CC_I386) $(QEMU_INCLUDES) $(GLIB_CFLAGS) $(CFLAGS) -c -o $@ $^
 
 hello-i386: hello-i386.c
 	$(CC_I386) -nostdlib $(CFLAGS) -static $(LDFLAGS) -o $@ $<
@@ -86,12 +96,15 @@ testthread: testthread.c
 # i386/x86_64 emulation test (test various opcodes) */
 test-i386: test-i386.c test-i386-code16.S test-i386-vm86.S \
            test-i386.h test-i386-shift.h test-i386-muldiv.h
-	$(CC_I386) $(CFLAGS) $(LDFLAGS) -o $@ \
+	$(CC_I386) $(QEMU_INCLUDES) $(CFLAGS) $(LDFLAGS) -o $@ \
               $(<D)/test-i386.c $(<D)/test-i386-code16.S $(<D)/test-i386-vm86.S -lm
 
+test-i386-fprem: test-i386-fprem.c
+	$(CC_I386) $(QEMU_INCLUDES) $(CFLAGS) $(LDFLAGS) -o $@ $^
+
 test-x86_64: test-i386.c \
            test-i386.h test-i386-shift.h test-i386-muldiv.h
-	$(CC_X86_64) $(CFLAGS) $(LDFLAGS) -o $@ $(<D)/test-i386.c -lm
+	$(CC_X86_64) $(QEMU_INCLUDES) $(CFLAGS) $(LDFLAGS) -o $@ $(<D)/test-i386.c -lm
 
 # generic Linux and CPU test
 linux-test: linux-test.c
diff --git a/tests/tcg/hello-i386.c b/tests/tcg/hello-i386.c
index 86afc34fb..fa00380de 100644
--- a/tests/tcg/hello-i386.c
+++ b/tests/tcg/hello-i386.c
@@ -1,6 +1,6 @@
 #include <asm/unistd.h>
 
-static inline volatile void exit(int status)
+static inline void exit(int status)
 {
   int __res;
   __asm__ volatile ("movl %%ecx,%%ebx\n"\
@@ -17,6 +17,7 @@ static inline int write(int fd, const char * buf, int len)
 		    "popl %%ebx\n"\
 		    : "=a" (status) \
 		    : "0" (__NR_write),"S" ((long)(fd)),"c" ((long)(buf)),"d" ((long)(len)));
+  return status;
 }
 
 void _start(void)
diff --git a/tests/tcg/linux-test.c b/tests/tcg/linux-test.c
index 2e4a746ac..83cb32ddb 100644
--- a/tests/tcg/linux-test.c
+++ b/tests/tcg/linux-test.c
@@ -16,6 +16,7 @@
  *  You should have received a copy of the GNU General Public License
  *  along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
+#define _GNU_SOURCE
 #include <stdarg.h>
 #include <stdlib.h>
 #include <stdio.h>
@@ -38,6 +39,7 @@
 #include <dirent.h>
 #include <setjmp.h>
 #include <sys/shm.h>
+#include <sched.h>
 
 #define TESTPATH "/tmp/linux-test.tmp"
 #define TESTPORT 7654
diff --git a/tests/tcg/mips/mips32-dsp/Makefile b/tests/tcg/mips/mips32-dsp/Makefile
new file mode 100644
index 000000000..c3a0a0094
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/Makefile
@@ -0,0 +1,136 @@
+-include ../../config-host.mak
+
+CROSS=mips64el-unknown-linux-gnu-
+
+SIM=qemu-mipsel
+SIM_FLAGS=-cpu 74Kf
+
+CC      = $(CROSS)gcc
+CFLAGS  = -mabi=32 -march=mips32r2 -mgp32 -mdsp -static
+
+TESTCASES = absq_s_ph.tst
+TESTCASES += absq_s_w.tst
+TESTCASES += addq_ph.tst
+TESTCASES += addq_s_ph.tst
+TESTCASES += addq_s_w.tst
+TESTCASES += addsc.tst
+TESTCASES += addu_qb.tst
+TESTCASES += addu_s_qb.tst
+TESTCASES += addwc.tst
+TESTCASES += bitrev.tst
+TESTCASES += bposge32.tst
+TESTCASES += cmp_eq_ph.tst
+TESTCASES += cmpgu_eq_qb.tst
+TESTCASES += cmpgu_le_qb.tst
+TESTCASES += cmpgu_lt_qb.tst
+TESTCASES += cmp_le_ph.tst
+TESTCASES += cmp_lt_ph.tst
+TESTCASES += cmpu_eq_qb.tst
+TESTCASES += cmpu_le_qb.tst
+TESTCASES += cmpu_lt_qb.tst
+TESTCASES += dpaq_sa_l_w.tst
+TESTCASES += dpaq_s_w_ph.tst
+TESTCASES += dpau_h_qbl.tst
+TESTCASES += dpau_h_qbr.tst
+TESTCASES += dpsq_sa_l_w.tst
+TESTCASES += dpsq_s_w_ph.tst
+TESTCASES += dpsu_h_qbl.tst
+TESTCASES += dpsu_h_qbr.tst
+TESTCASES += extp.tst
+TESTCASES += extpdp.tst
+TESTCASES += extpdpv.tst
+TESTCASES += extpv.tst
+TESTCASES += extr_rs_w.tst
+TESTCASES += extr_r_w.tst
+TESTCASES += extr_s_h.tst
+TESTCASES += extrv_rs_w.tst
+TESTCASES += extrv_r_w.tst
+TESTCASES += extrv_s_h.tst
+TESTCASES += extrv_w.tst
+TESTCASES += extr_w.tst
+TESTCASES += insv.tst
+TESTCASES += lbux.tst
+TESTCASES += lhx.tst
+TESTCASES += lwx.tst
+TESTCASES += madd.tst
+TESTCASES += maddu.tst
+TESTCASES += maq_sa_w_phl.tst
+TESTCASES += maq_sa_w_phr.tst
+TESTCASES += maq_s_w_phl.tst
+TESTCASES += maq_s_w_phr.tst
+TESTCASES += mfhi.tst
+TESTCASES += mflo.tst
+TESTCASES += modsub.tst
+TESTCASES += msub.tst
+TESTCASES += msubu.tst
+TESTCASES += mthi.tst
+TESTCASES += mthlip.tst
+TESTCASES += mtlo.tst
+TESTCASES += muleq_s_w_phl.tst
+TESTCASES += muleq_s_w_phr.tst
+TESTCASES += muleu_s_ph_qbl.tst
+TESTCASES += muleu_s_ph_qbr.tst
+TESTCASES += mulq_rs_ph.tst
+TESTCASES += mult.tst
+TESTCASES += multu.tst
+TESTCASES += packrl_ph.tst
+TESTCASES += pick_ph.tst
+TESTCASES += pick_qb.tst
+TESTCASES += precequ_ph_qbla.tst
+TESTCASES += precequ_ph_qbl.tst
+TESTCASES += precequ_ph_qbra.tst
+TESTCASES += precequ_ph_qbr.tst
+TESTCASES += preceq_w_phl.tst
+TESTCASES += preceq_w_phr.tst
+TESTCASES += preceu_ph_qbla.tst
+TESTCASES += preceu_ph_qbl.tst
+TESTCASES += preceu_ph_qbra.tst
+TESTCASES += preceu_ph_qbr.tst
+TESTCASES += precrq_ph_w.tst
+TESTCASES += precrq_qb_ph.tst
+TESTCASES += precrq_rs_ph_w.tst
+TESTCASES += precrqu_s_qb_ph.tst
+TESTCASES += raddu_w_qb.tst
+TESTCASES += rddsp.tst
+TESTCASES += repl_ph.tst
+TESTCASES += repl_qb.tst
+TESTCASES += replv_ph.tst
+TESTCASES += replv_qb.tst
+TESTCASES += shilo.tst
+TESTCASES += shilov.tst
+TESTCASES += shll_ph.tst
+TESTCASES += shll_qb.tst
+TESTCASES += shll_s_ph.tst
+TESTCASES += shll_s_w.tst
+TESTCASES += shllv_ph.tst
+TESTCASES += shllv_qb.tst
+TESTCASES += shllv_s_ph.tst
+TESTCASES += shllv_s_w.tst
+TESTCASES += shra_ph.tst
+TESTCASES += shra_r_ph.tst
+TESTCASES += shra_r_w.tst
+TESTCASES += shrav_ph.tst
+TESTCASES += shrav_r_ph.tst
+TESTCASES += shrav_r_w.tst
+TESTCASES += shrl_qb.tst
+TESTCASES += shrlv_qb.tst
+TESTCASES += subq_ph.tst
+TESTCASES += subq_s_ph.tst
+TESTCASES += subq_s_w.tst
+TESTCASES += subu_qb.tst
+TESTCASES += subu_s_qb.tst
+TESTCASES += wrdsp.tst
+
+all: $(TESTCASES)
+
+%.tst: %.c
+	$(CC) $(CFLAGS) $< -o $@
+
+check: $(TESTCASES)
+	@for case in $(TESTCASES); do \
+        echo $(SIM) $(SIM_FLAGS) ./$$case;\
+        $(SIM) $(SIM_FLAGS) ./$$case; \
+	done
+
+clean:
+	$(RM) -rf $(TESTCASES)
diff --git a/tests/tcg/mips/mips32-dsp/absq_s_ph.c b/tests/tcg/mips/mips32-dsp/absq_s_ph.c
new file mode 100644
index 000000000..aa8411202
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/absq_s_ph.c
@@ -0,0 +1,31 @@
+#include<stdio.h>
+#include<assert.h>
+
+
+int main()
+{
+    int rd, rt;
+    int result;
+
+    rt     = 0x10017EFD;
+    result = 0x10017EFD;
+
+    __asm
+        ("absq_s.ph %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    assert(rd == result);
+
+    rt     = 0x8000A536;
+    result = 0x7FFF5ACA;
+
+    __asm
+        ("absq_s.ph %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/absq_s_w.c b/tests/tcg/mips/mips32-dsp/absq_s_w.c
new file mode 100644
index 000000000..3f52a4803
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/absq_s_w.c
@@ -0,0 +1,37 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rt;
+    int result;
+
+    rt     = 0x80000000;
+    result = 0x7FFFFFFF;
+    __asm
+        ("absq_s.w %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    assert(rd == result);
+
+    rt     = 0x80030000;
+    result = 0x7FFD0000;
+    __asm
+        ("absq_s.w %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    assert(rd == result);
+
+    rt     = 0x31036080;
+    result = 0x31036080;
+    __asm
+        ("absq_s.w %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/addq_ph.c b/tests/tcg/mips/mips32-dsp/addq_ph.c
new file mode 100644
index 000000000..96a549637
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/addq_ph.c
@@ -0,0 +1,46 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int dsp;
+    int result;
+
+    rs     = 0xFFFFFFFF;
+    rt     = 0x10101010;
+    result = 0x100F100F;
+    __asm
+        ("addq.ph   %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    assert(result == rd);
+
+    rs     = 0x3712847D;
+    rt     = 0x0031AF2D;
+    result = 0x374333AA;
+    __asm
+        ("addq.ph   %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    assert(result == rd);
+
+    rs     = 0x7fff847D;
+    rt     = 0x0031AF2D;
+    result = 0x803033AA;
+    __asm
+        ("addq.ph   %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    assert(result == rd);
+
+    __asm("rddsp %0\n\t"
+          : "=r"(dsp)
+         );
+    assert(((dsp >> 20) & 0x01) == 1);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/addq_s_ph.c b/tests/tcg/mips/mips32-dsp/addq_s_ph.c
new file mode 100644
index 000000000..5f865f6cf
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/addq_s_ph.c
@@ -0,0 +1,69 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int dsp;
+    int result;
+
+    rs     = 0xFFFFFFFF;
+    rt     = 0x10101010;
+    result = 0x100F100F;
+    __asm
+        ("addq_s.ph   %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    assert(result == rd);
+
+    rs     = 0x3712847D;
+    rt     = 0x0031AF2D;
+    result = 0x37438000;
+    __asm
+        ("addq_s.ph   %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    assert(result == rd);
+
+    __asm
+        ("rddsp %0\n\t"
+         : "=r"(dsp)
+        );
+    assert(((dsp >> 20) & 0x01) == 1);
+
+    rs     = 0x7fff847D;
+    rt     = 0x0031AF2D;
+    result = 0x7fff8000;
+    __asm
+        ("addq_s.ph   %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    assert(result == rd);
+
+    __asm
+        ("rddsp %0\n\t"
+         : "=r"(dsp)
+        );
+    assert(((dsp >> 20) & 0x01) == 1);
+
+    rs     = 0x8030847D;
+    rt     = 0x8a00AF2D;
+    result = 0x80008000;
+    __asm
+        ("addq_s.ph   %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    assert(result == rd);
+
+    __asm
+        ("rddsp %0\n\t"
+         : "=r"(dsp)
+        );
+    assert(((dsp >> 20) & 0x01) == 1);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/addq_s_w.c b/tests/tcg/mips/mips32-dsp/addq_s_w.c
new file mode 100644
index 000000000..1e13acf68
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/addq_s_w.c
@@ -0,0 +1,44 @@
+#include<stdio.h>
+#include<assert.h>
+
+
+int main()
+{
+    int rd, rs, rt;
+    int result;
+
+    rt     = 0x10017EFD;
+    rs     = 0x11111111;
+    result = 0x2112900e;
+
+    __asm
+        ("addq_s.w %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    assert(rd == result);
+
+    rt     = 0x80017EFD;
+    rs     = 0x81111111;
+    result = 0x80000000;
+
+    __asm
+        ("addq_s.w %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    assert(rd == result);
+
+    rt     = 0x7fffffff;
+    rs     = 0x01111111;
+    result = 0x7fffffff;
+
+    __asm
+        ("addq_s.w %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/addsc.c b/tests/tcg/mips/mips32-dsp/addsc.c
new file mode 100644
index 000000000..ace749f66
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/addsc.c
@@ -0,0 +1,33 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int dsp;
+    int result;
+
+    rs     = 0x0000000F;
+    rt     = 0x00000001;
+    result = 0x00000010;
+    __asm
+        ("addsc %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    assert(rd == result);
+
+    rs     = 0xFFFF0FFF;
+    rt     = 0x00010111;
+    result = 0x00001110;
+    __asm
+        ("addsc %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    assert(rd == result);
+    assert(((dsp >> 13) & 0x01) == 1);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/addu_qb.c b/tests/tcg/mips/mips32-dsp/addu_qb.c
new file mode 100644
index 000000000..23ba2e90d
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/addu_qb.c
@@ -0,0 +1,35 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int dsp;
+    int result;
+
+    rs     = 0x00FF00FF;
+    rt     = 0x00010001;
+    result = 0x00000000;
+    __asm
+        ("addu.qb %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    assert(rd == result);
+    assert(((dsp >> 20) & 0x01) == 1);
+
+    rs     = 0xFFFF1111;
+    rt     = 0x00020001;
+    result = 0xFF011112;
+    __asm
+        ("addu.qb %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    assert(rd == result);
+    assert(((dsp >> 20) & 0x01) == 1);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/addu_s_qb.c b/tests/tcg/mips/mips32-dsp/addu_s_qb.c
new file mode 100644
index 000000000..fe7fd3e6a
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/addu_s_qb.c
@@ -0,0 +1,35 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int dsp;
+    int result;
+
+    rs     = 0x10FF01FF;
+    rt     = 0x10010001;
+    result = 0x20FF01FF;
+    __asm
+        ("addu_s.qb %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    assert(rd == result);
+    assert(((dsp >> 20) & 0x1) == 1);
+
+    rs     = 0xFFFF1111;
+    rt     = 0x00020001;
+    result = 0xFFFF1112;
+    __asm
+        ("addu_s.qb %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    assert(rd == result);
+    assert(((dsp >> 20) & 0x1) == 1);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/addwc.c b/tests/tcg/mips/mips32-dsp/addwc.c
new file mode 100644
index 000000000..8a8d81fab
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/addwc.c
@@ -0,0 +1,49 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int dspi, dspo;
+    int result;
+
+    rs     = 0x10FF01FF;
+    rt     = 0x10010001;
+    dspi   = 0x00002000;
+    result = 0x21000201;
+    __asm
+        ("wrdsp %3\n"
+         "addwc %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt), "r"(dspi)
+        );
+    assert(rd == result);
+
+    rs     = 0xFFFF1111;
+    rt     = 0x00020001;
+    dspi   = 0x00;
+    result = 0x00011112;
+    __asm
+        ("wrdsp %3\n"
+         "addwc %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt), "r"(dspi)
+        );
+    assert(rd == result);
+
+    rs     = 0x8FFF1111;
+    rt     = 0x80020001;
+    dspi   = 0x00;
+    result = 0x10011112;
+    __asm
+        ("wrdsp %4\n"
+         "addwc %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dspo)
+         : "r"(rs), "r"(rt), "r"(dspi)
+        );
+    assert(rd == result);
+    assert(((dspo >> 20) & 0x01) == 1);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/bitrev.c b/tests/tcg/mips/mips32-dsp/bitrev.c
new file mode 100644
index 000000000..04d8a3844
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/bitrev.c
@@ -0,0 +1,20 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rt;
+    int result;
+
+    rt     = 0x12345678;
+    result = 0x00001E6A;
+
+    __asm
+        ("bitrev %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/bposge32.c b/tests/tcg/mips/mips32-dsp/bposge32.c
new file mode 100644
index 000000000..d25417ea7
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/bposge32.c
@@ -0,0 +1,44 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int dsp, sum;
+    int result;
+
+    dsp =  0x20;
+    sum = 0x01;
+    result = 0x02;
+
+    __asm
+        ("wrdsp %1\n\t"
+         "bposge32 test1\n\t"
+         "nop\n\t"
+         "addi %0, 0xA2\n\t"
+         "nop\n\t"
+         "test1:\n\t"
+         "addi %0, 0x01\n\t"
+         : "+r"(sum)
+         : "r"(dsp)
+        );
+    assert(sum == result);
+
+    dsp =  0x10;
+    sum = 0x01;
+    result = 0xA4;
+
+    __asm
+        ("wrdsp %1\n\t"
+         "bposge32 test2\n\t"
+         "nop\n\t"
+         "addi %0, 0xA2\n\t"
+         "nop\n\t"
+         "test2:\n\t"
+         "addi %0, 0x01\n\t"
+         : "+r"(sum)
+         : "r"(dsp)
+        );
+    assert(sum == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/cmp_eq_ph.c b/tests/tcg/mips/mips32-dsp/cmp_eq_ph.c
new file mode 100644
index 000000000..957bd88ce
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/cmp_eq_ph.c
@@ -0,0 +1,35 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int result;
+
+    rs     = 0x11777066;
+    rt     = 0x55AA33FF;
+    result = 0x00;
+    __asm
+        ("cmp.eq.ph %1, %2\n\t"
+         "rddsp %0\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+
+    rd = (rd >> 24) & 0x03;
+    assert(rd == result);
+
+    rs     = 0x11777066;
+    rt     = 0x11777066;
+    result = 0x03;
+    __asm
+        ("cmp.eq.ph %1, %2\n\t"
+         "rddsp %0\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    rd = (rd >> 24) & 0x03;
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/cmp_le_ph.c b/tests/tcg/mips/mips32-dsp/cmp_le_ph.c
new file mode 100644
index 000000000..356f156c5
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/cmp_le_ph.c
@@ -0,0 +1,35 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int result;
+
+    rs     = 0x11777066;
+    rt     = 0x55AA33FF;
+    result = 0x02;
+    __asm
+        ("cmp.le.ph %1, %2\n\t"
+         "rddsp %0\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+
+    rd = (rd >> 24) & 0x03;
+    assert(rd == result);
+
+    rs     = 0x11777066;
+    rt     = 0x11777066;
+    result = 0x03;
+    __asm
+        ("cmp.le.ph %1, %2\n\t"
+         "rddsp %0\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    rd = (rd >> 24) & 0x03;
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/cmp_lt_ph.c b/tests/tcg/mips/mips32-dsp/cmp_lt_ph.c
new file mode 100644
index 000000000..3fb4827ad
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/cmp_lt_ph.c
@@ -0,0 +1,35 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int result;
+
+    rs     = 0x11777066;
+    rt     = 0x55AA33FF;
+    result = 0x02;
+    __asm
+        ("cmp.lt.ph %1, %2\n\t"
+         "rddsp %0\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+
+    rd = (rd >> 24) & 0x03;
+    assert(rd == result);
+
+    rs     = 0x11777066;
+    rt     = 0x11777066;
+    result = 0x00;
+    __asm
+        ("cmp.lt.ph %1, %2\n\t"
+         "rddsp %0\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    rd = (rd >> 24) & 0x03;
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/cmpgu_eq_qb.c b/tests/tcg/mips/mips32-dsp/cmpgu_eq_qb.c
new file mode 100644
index 000000000..2615c84c7
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/cmpgu_eq_qb.c
@@ -0,0 +1,31 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int result;
+
+    rs     = 0x11777066;
+    rt     = 0x55AA70FF;
+    result = 0x02;
+    __asm
+        ("cmpgu.eq.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+
+    assert(rd == result);
+
+    rs     = 0x11777066;
+    rt     = 0x11777066;
+    result = 0x0F;
+    __asm
+        ("cmpgu.eq.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/cmpgu_le_qb.c b/tests/tcg/mips/mips32-dsp/cmpgu_le_qb.c
new file mode 100644
index 000000000..65d0813c3
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/cmpgu_le_qb.c
@@ -0,0 +1,31 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int result;
+
+    rs     = 0x11777066;
+    rt     = 0x55AA70FF;
+    result = 0x0F;
+    __asm
+        ("cmpgu.le.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+
+    assert(rd == result);
+
+    rs     = 0x11777066;
+    rt     = 0x11766066;
+    result = 0x09;
+    __asm
+        ("cmpgu.le.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/cmpgu_lt_qb.c b/tests/tcg/mips/mips32-dsp/cmpgu_lt_qb.c
new file mode 100644
index 000000000..7dddad985
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/cmpgu_lt_qb.c
@@ -0,0 +1,31 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int result;
+
+    rs     = 0x11777066;
+    rt     = 0x55AA70FF;
+    result = 0x0D;
+    __asm
+        ("cmpgu.lt.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+
+    assert(rd == result);
+
+    rs     = 0x11777066;
+    rt     = 0x11766066;
+    result = 0x00;
+    __asm
+        ("cmpgu.lt.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/cmpu_eq_qb.c b/tests/tcg/mips/mips32-dsp/cmpu_eq_qb.c
new file mode 100644
index 000000000..680f2a199
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/cmpu_eq_qb.c
@@ -0,0 +1,35 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rs, rt;
+    int dsp;
+    int result;
+
+    rs         = 0x11777066;
+    rt         = 0x55AA70FF;
+    result     = 0x02;
+    __asm
+        ("cmpu.eq.qb %1, %2\n\t"
+         "rddsp %0\n\t"
+         : "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 24) & 0x0F;
+    assert(dsp == result);
+
+    rs     = 0x11777066;
+    rt     = 0x11777066;
+    result = 0x0F;
+    __asm
+        ("cmpu.eq.qb %1, %2\n\t"
+         "rddsp %0\n\t"
+         : "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 24) & 0x0F;
+    assert(dsp == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/cmpu_le_qb.c b/tests/tcg/mips/mips32-dsp/cmpu_le_qb.c
new file mode 100644
index 000000000..43cfa509c
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/cmpu_le_qb.c
@@ -0,0 +1,35 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rs, rt;
+    int dsp;
+    int result;
+
+    rs         = 0x11777066;
+    rt         = 0x55AA70FF;
+    result     = 0x0F;
+    __asm
+        ("cmpu.le.qb %1, %2\n\t"
+         "rddsp %0\n\t"
+         : "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 24) & 0x0F;
+    assert(dsp == result);
+
+    rs     = 0x11777066;
+    rt     = 0x11777066;
+    result = 0x0F;
+    __asm
+        ("cmpu.le.qb %1, %2\n\t"
+         "rddsp %0\n\t"
+         : "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 24) & 0x0F;
+    assert(dsp == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/cmpu_lt_qb.c b/tests/tcg/mips/mips32-dsp/cmpu_lt_qb.c
new file mode 100644
index 000000000..074ca5b40
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/cmpu_lt_qb.c
@@ -0,0 +1,35 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rs, rt;
+    int dsp;
+    int result;
+
+    rs         = 0x11777066;
+    rt         = 0x55AA70FF;
+    result     = 0x0D;
+    __asm
+        ("cmpu.lt.qb %1, %2\n\t"
+         "rddsp %0\n\t"
+         : "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 24) & 0x0F;
+    assert(dsp == result);
+
+    rs     = 0x11777066;
+    rt     = 0x11777066;
+    result = 0x00;
+    __asm
+        ("cmpu.lt.qb %1, %2\n\t"
+         "rddsp %0\n\t"
+         : "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 24) & 0x0F;
+    assert(dsp == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/dpaq_s_w_ph.c b/tests/tcg/mips/mips32-dsp/dpaq_s_w_ph.c
new file mode 100644
index 000000000..a6425b6ed
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/dpaq_s_w_ph.c
@@ -0,0 +1,31 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rs, rt, dsp;
+    int ach = 0, acl = 0;
+    int resulth, resultl, resultdsp;
+
+    rs        = 0x800000FF;
+    rt        = 0x80000002;
+    resulth   = 0x00;
+    resultl   = 0x800003FB;
+    resultdsp = 0x01;
+    __asm
+        ("mthi        %0, $ac1\n\t"
+         "mtlo        %1, $ac1\n\t"
+         "dpaq_s.w.ph $ac1, %3, %4\n\t"
+         "mfhi        %0,   $ac1\n\t"
+         "mflo        %1,   $ac1\n\t"
+         "rddsp       %2\n\t"
+         : "+r"(ach), "+r"(acl), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = dsp >> 17 & 0x01;
+    assert(dsp == resultdsp);
+    assert(ach == resulth);
+    assert(acl == resultl);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/dpaq_sa_l_w.c b/tests/tcg/mips/mips32-dsp/dpaq_sa_l_w.c
new file mode 100644
index 000000000..ce864844d
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/dpaq_sa_l_w.c
@@ -0,0 +1,77 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rs, rt, dsp;
+    int ach = 0, acl = 0;
+    int resulth, resultl, resultdsp;
+
+    rs        = 0x80000000;
+    rt        = 0x80000000;
+    resulth   = 0x7FFFFFFF;
+    resultl   = 0xFFFFFFFF;
+    resultdsp = 0x01;
+    __asm
+        ("mthi        %0, $ac1\n\t"
+         "mtlo        %0, $ac1\n\t"
+         "dpaq_sa.l.w $ac1, %3, %4\n\t"
+         "mfhi        %0,   $ac1\n\t"
+         "mflo        %1,   $ac1\n\t"
+         "rddsp       %2\n\t"
+         : "+r"(ach), "+r"(acl), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 17) & 0x01;
+    assert(dsp == resultdsp);
+    assert(ach == resulth);
+    assert(acl == resultl);
+
+    ach = 0x12;
+    acl = 0x48;
+    rs  = 0x80000000;
+    rt  = 0x80000000;
+
+    resulth   = 0x7FFFFFFF;
+    resultl   = 0xFFFFFFFF;
+    resultdsp = 0x01;
+    __asm
+        ("mthi        %0, $ac1\n\t"
+         "mtlo        %0, $ac1\n\t"
+         "dpaq_sa.l.w $ac1, %3, %4\n\t"
+         "mfhi        %0,   $ac1\n\t"
+         "mflo        %1,   $ac1\n\t"
+         "rddsp       %2\n\t"
+         : "+r"(ach), "+r"(acl), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 17) & 0x01;
+    assert(dsp == resultdsp);
+    assert(ach == resulth);
+    assert(acl == resultl);
+
+    ach = 0x741532A0;
+    acl = 0xfceabb08;
+    rs  = 0x80000000;
+    rt  = 0x80000000;
+
+    resulth   = 0x7fffffff;
+    resultl   = 0xffffffff;
+    resultdsp = 0x01;
+    __asm
+        ("mthi        %0, $ac1\n\t"
+         "mtlo        %0, $ac1\n\t"
+         "dpaq_sa.l.w $ac1, %3, %4\n\t"
+         "mfhi        %0,   $ac1\n\t"
+         "mflo        %1,   $ac1\n\t"
+         "rddsp       %2\n\t"
+         : "+r"(ach), "+r"(acl), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 17) & 0x01;
+    assert(dsp == resultdsp);
+    assert(ach == resulth);
+    assert(acl == resultl);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/dpau_h_qbl.c b/tests/tcg/mips/mips32-dsp/dpau_h_qbl.c
new file mode 100644
index 000000000..6017b5e73
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/dpau_h_qbl.c
@@ -0,0 +1,27 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rs, rt;
+    int ach = 5, acl = 3;
+    int resulth, resultl;
+
+    rs        = 0x800000FF;
+    rt        = 0x80000002;
+    resulth   = 0x05;
+    resultl   = 0x4003;
+    __asm
+        ("mthi       %0, $ac1\n\t"
+         "mtlo       %1, $ac1\n\t"
+         "dpau.h.qbl $ac1, %2, %3\n\t"
+         "mfhi       %0,   $ac1\n\t"
+         "mflo       %1,   $ac1\n\t"
+         : "+r"(ach), "+r"(acl)
+         : "r"(rs), "r"(rt)
+        );
+    assert(ach == resulth);
+    assert(acl == resultl);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/dpau_h_qbr.c b/tests/tcg/mips/mips32-dsp/dpau_h_qbr.c
new file mode 100644
index 000000000..e4abb2e2a
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/dpau_h_qbr.c
@@ -0,0 +1,27 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rs, rt;
+    int ach = 5, acl = 3;
+    int resulth, resultl;
+
+    rs        = 0x800000FF;
+    rt        = 0x80000002;
+    resulth   = 0x05;
+    resultl   = 0x0201;
+    __asm
+        ("mthi       %0, $ac1\n\t"
+         "mtlo       %1, $ac1\n\t"
+         "dpau.h.qbr $ac1, %2, %3\n\t"
+         "mfhi       %0,   $ac1\n\t"
+         "mflo       %1,   $ac1\n\t"
+         : "+r"(ach), "+r"(acl)
+         : "r"(rs), "r"(rt)
+        );
+    assert(ach == resulth);
+    assert(acl == resultl);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/dpsq_s_w_ph.c b/tests/tcg/mips/mips32-dsp/dpsq_s_w_ph.c
new file mode 100644
index 000000000..22ab4d57b
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/dpsq_s_w_ph.c
@@ -0,0 +1,45 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rs, rt;
+    int ach = 5, acl = 5;
+    int resulth, resultl;
+
+    rs      = 0xBC0123AD;
+    rt      = 0x01643721;
+    resulth = 0x04;
+    resultl = 0xEE9794A3;
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpsq_s.w.ph $ac1, %2, %3\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         : "+r"(ach), "+r"(acl)
+         : "r"(rs), "r"(rt)
+        );
+    assert(ach == resulth);
+    assert(acl == resultl);
+
+    ach = 0x1424Ef1f;
+    acl = 0x1035219A;
+    rs      = 0x800083AD;
+    rt      = 0x80003721;
+    resulth = 0x1424ef1e;
+    resultl = 0x577ed901;
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpsq_s.w.ph $ac1, %2, %3\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         : "+r"(ach), "+r"(acl)
+         : "r"(rs), "r"(rt)
+        );
+    assert(ach == resulth);
+    assert(acl == resultl);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/dpsq_sa_l_w.c b/tests/tcg/mips/mips32-dsp/dpsq_sa_l_w.c
new file mode 100644
index 000000000..b7b73fdb6
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/dpsq_sa_l_w.c
@@ -0,0 +1,55 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rs, rt, dsp;
+    int ach = 5, acl = 5;
+    int resulth, resultl, resultdsp;
+
+    rs      = 0xBC0123AD;
+    rt      = 0x01643721;
+    resulth = 0xfdf4cbe0;
+    resultl = 0xd138776b;
+    resultdsp = 0x00;
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpsq_sa.l.w $ac1, %3, %4\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "+r"(ach), "+r"(acl), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 17) & 0x01;
+    assert(dsp == resultdsp);
+    assert(ach == resulth);
+    assert(acl == resultl);
+
+    ach = 0x54321123;
+    acl = 5;
+    rs      = 0x80000000;
+    rt      = 0x80000000;
+
+    resulth = 0xd4321123;
+    resultl = 0x06;
+    resultdsp = 0x01;
+
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpsq_sa.l.w $ac1, %3, %4\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "+r"(ach), "+r"(acl), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 17) & 0x01;
+    assert(dsp == resultdsp);
+    assert(ach == resulth);
+    assert(acl == resultl);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/dpsu_h_qbl.c b/tests/tcg/mips/mips32-dsp/dpsu_h_qbl.c
new file mode 100644
index 000000000..94e2bf625
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/dpsu_h_qbl.c
@@ -0,0 +1,27 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rs, rt;
+    int ach = 5, acl = 5;
+    int resulth, resultl;
+
+    rs      = 0xBC0123AD;
+    rt      = 0x01643721;
+    resulth = 0x04;
+    resultl = 0xFFFFFEE5;
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpsu.h.qbl $ac1, %2, %3\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         : "+r"(ach), "+r"(acl)
+         : "r"(rs), "r"(rt)
+        );
+    assert(ach == resulth);
+    assert(acl == resultl);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/dpsu_h_qbr.c b/tests/tcg/mips/mips32-dsp/dpsu_h_qbr.c
new file mode 100644
index 000000000..a1e663563
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/dpsu_h_qbr.c
@@ -0,0 +1,27 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rs, rt;
+    int ach = 5, acl = 5;
+    int resulth, resultl;
+
+    rs      = 0xBC0123AD;
+    rt      = 0x01643721;
+    resulth = 0x04;
+    resultl = 0xFFFFE233;
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpsu.h.qbr $ac1, %2, %3\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         : "+r"(ach), "+r"(acl)
+         : "r"(rs), "r"(rt)
+        );
+    assert(ach == resulth);
+    assert(acl == resultl);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/extp.c b/tests/tcg/mips/mips32-dsp/extp.c
new file mode 100644
index 000000000..21a67af21
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/extp.c
@@ -0,0 +1,44 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rt, ach, acl, dsp;
+    int result;
+
+    ach = 0x05;
+    acl = 0xB4CB;
+    dsp = 0x07;
+    result = 0x000C;
+
+    __asm
+        ("wrdsp %1, 0x01\n\t"
+         "mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "extp %0, $ac1, 0x03\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "+r"(dsp)
+         : "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 14) & 0x01;
+    assert(dsp == 0);
+    assert(result == rt);
+
+    ach = 0x05;
+    acl = 0xB4CB;
+    dsp = 0x01;
+
+    __asm
+        ("wrdsp %1, 0x01\n\t"
+         "mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "extp %0, $ac1, 0x03\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "+r"(dsp)
+         : "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 14) & 0x01;
+    assert(dsp == 1);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/extpdp.c b/tests/tcg/mips/mips32-dsp/extpdp.c
new file mode 100644
index 000000000..15ba0828f
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/extpdp.c
@@ -0,0 +1,46 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rt, ach, acl, dsp, pos, efi;
+    int result;
+
+    ach = 0x05;
+    acl = 0xB4CB;
+    dsp = 0x07;
+    result = 0x000C;
+
+    __asm
+        ("wrdsp %1, 0x01\n\t"
+         "mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "extpdp %0, $ac1, 0x03\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "+r"(dsp)
+         : "r"(ach), "r"(acl)
+        );
+    pos =  dsp & 0x3F;
+    efi = (dsp >> 14) & 0x01;
+    assert(pos == 3);
+    assert(efi == 0);
+    assert(result == rt);
+
+    ach = 0x05;
+    acl = 0xB4CB;
+    dsp = 0x01;
+
+    __asm
+        ("wrdsp %1, 0x01\n\t"
+         "mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "extpdp %0, $ac1, 0x03\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "+r"(dsp)
+         : "r"(ach), "r"(acl)
+        );
+    efi = (dsp >> 14) & 0x01;
+    assert(efi == 1);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/extpdpv.c b/tests/tcg/mips/mips32-dsp/extpdpv.c
new file mode 100644
index 000000000..f5774eed3
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/extpdpv.c
@@ -0,0 +1,47 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rt, rs, ach, acl, dsp, pos, efi;
+    int result;
+
+    ach = 0x05;
+    acl = 0xB4CB;
+    dsp = 0x07;
+    rs  = 0x03;
+    result = 0x000C;
+
+    __asm
+        ("wrdsp %1, 0x01\n\t"
+         "mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "extpdpv %0, $ac1, %4\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "+r"(dsp)
+         : "r"(ach), "r"(acl), "r"(rs)
+        );
+    pos =  dsp & 0x3F;
+    efi = (dsp >> 14) & 0x01;
+    assert(pos == 3);
+    assert(efi == 0);
+    assert(result == rt);
+
+    ach = 0x05;
+    acl = 0xB4CB;
+    dsp = 0x01;
+
+    __asm
+        ("wrdsp %1, 0x01\n\t"
+         "mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "extpdpv %0, $ac1, %4\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "+r"(dsp)
+         : "r"(ach), "r"(acl), "r"(rs)
+        );
+    efi = (dsp >> 14) & 0x01;
+    assert(efi == 1);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/extpv.c b/tests/tcg/mips/mips32-dsp/extpv.c
new file mode 100644
index 000000000..401b94afa
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/extpv.c
@@ -0,0 +1,45 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rt, ac, ach, acl, dsp;
+    int result;
+
+    ach = 0x05;
+    acl = 0xB4CB;
+    dsp = 0x07;
+    ac  = 0x03;
+    result = 0x000C;
+
+    __asm
+        ("wrdsp %1, 0x01\n\t"
+         "mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "extpv %0, $ac1, %4\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "+r"(dsp)
+         : "r"(ach), "r"(acl), "r"(ac)
+        );
+    dsp = (dsp >> 14) & 0x01;
+    assert(dsp == 0);
+    assert(result == rt);
+
+    ach = 0x05;
+    acl = 0xB4CB;
+    dsp = 0x01;
+
+    __asm
+        ("wrdsp %1, 0x01\n\t"
+         "mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "extpv %0, $ac1, %4\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "+r"(dsp)
+         : "r"(ach), "r"(acl), "r"(ac)
+        );
+    dsp = (dsp >> 14) & 0x01;
+    assert(dsp == 1);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/extr_r_w.c b/tests/tcg/mips/mips32-dsp/extr_r_w.c
new file mode 100644
index 000000000..0beeefd36
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/extr_r_w.c
@@ -0,0 +1,48 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rt, ach, acl, dsp;
+    int result;
+
+    ach = 0x05;
+    acl = 0xB4CB;
+    result = 0xA0001699;
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "extr_r.w %0, $ac1, 0x03\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    assert(dsp == 1);
+    assert(result == rt);
+
+    /* Clear dspcontrol */
+    dsp = 0;
+    __asm
+        ("wrdsp %0\n\t"
+         :
+         : "r"(dsp)
+        );
+
+    ach = 0x01;
+    acl = 0xB4CB;
+    result = 0x10000B4D;
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "extr_r.w %0, $ac1, 0x04\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    assert(dsp == 0);
+    assert(result == rt);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/extr_rs_w.c b/tests/tcg/mips/mips32-dsp/extr_rs_w.c
new file mode 100644
index 000000000..24c748db2
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/extr_rs_w.c
@@ -0,0 +1,48 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rt, ach, acl, dsp;
+    int result;
+
+    ach = 0x05;
+    acl = 0xB4CB;
+    result = 0x7FFFFFFF;
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "extr_rs.w %0, $ac1, 0x03\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    assert(dsp == 1);
+    assert(result == rt);
+
+    /* Clear dspcontrol */
+    dsp = 0;
+    __asm
+        ("wrdsp %0\n\t"
+         :
+         : "r"(dsp)
+        );
+
+    ach = 0x01;
+    acl = 0xB4CB;
+    result = 0x10000B4D;
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "extr_rs.w %0, $ac1, 0x04\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    assert(dsp == 0);
+    assert(result == rt);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/extr_s_h.c b/tests/tcg/mips/mips32-dsp/extr_s_h.c
new file mode 100644
index 000000000..b2129134c
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/extr_s_h.c
@@ -0,0 +1,63 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rt, ach, acl, dsp;
+    int result;
+
+    ach = 0x05;
+    acl = 0xB4CB;
+    result = 0x00007FFF;
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "extr_s.h %0, $ac1, 0x03\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    assert(dsp == 1);
+    assert(result == rt);
+
+    ach = 0xffffffff;
+    acl = 0x12344321;
+    result = 0xFFFF8000;
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "extr_s.h %0, $ac1, 0x08\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    assert(dsp == 1);
+    assert(result == rt);
+
+    /* Clear dsp */
+    dsp = 0;
+    __asm
+        ("wrdsp %0\n\t"
+         :
+         : "r"(dsp)
+        );
+
+    ach = 0x00;
+    acl = 0x4321;
+    result = 0x432;
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "extr_s.h %0, $ac1, 0x04\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    assert(dsp == 0);
+    assert(result == rt);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/extr_w.c b/tests/tcg/mips/mips32-dsp/extr_w.c
new file mode 100644
index 000000000..02ab9ecaa
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/extr_w.c
@@ -0,0 +1,48 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rt, ach, acl, dsp;
+    int result;
+
+    ach = 0x05;
+    acl = 0xB4CB;
+    result = 0xA0001699;
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "extr.w %0, $ac1, 0x03\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    assert(dsp == 1);
+    assert(result == rt);
+
+    /* Clear dspcontrol */
+    dsp = 0;
+    __asm
+        ("wrdsp %0\n\t"
+         :
+         : "r"(dsp)
+        );
+
+    ach = 0x01;
+    acl = 0xB4CB;
+    result = 0x10000B4C;
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "extr.w %0, $ac1, 0x04\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    assert(dsp == 0);
+    assert(result == rt);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/extrv_r_w.c b/tests/tcg/mips/mips32-dsp/extrv_r_w.c
new file mode 100644
index 000000000..005807b14
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/extrv_r_w.c
@@ -0,0 +1,54 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rt, rs, ach, acl, dsp;
+    int result;
+
+    ach = 0x05;
+    acl = 0xB4CB;
+    dsp = 0x07;
+    rs  = 0x03;
+    result = 0xA0001699;
+
+    __asm
+        ("wrdsp %1, 0x01\n\t"
+         "mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "extrv_r.w %0, $ac1, %2\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "+r"(dsp)
+         : "r"(rs), "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    assert(dsp == 1);
+    assert(result == rt);
+
+    /* Clear dspcontrol */
+    dsp = 0;
+    __asm
+        ("wrdsp %0\n\t"
+         :
+         : "r"(dsp)
+        );
+
+    rs = 4;
+    ach = 0x01;
+    acl = 0xB4CB;
+    result = 0x10000B4D;
+    __asm
+        ("wrdsp %1, 0x01\n\t"
+         "mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "extrv_r.w %0, $ac1, %2\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "+r"(dsp)
+         : "r"(rs), "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    assert(dsp == 0);
+    assert(result == rt);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/extrv_rs_w.c b/tests/tcg/mips/mips32-dsp/extrv_rs_w.c
new file mode 100644
index 000000000..c2d8513bb
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/extrv_rs_w.c
@@ -0,0 +1,52 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rt, rs, ach, acl, dsp;
+    int result;
+
+    rs = 0x03;
+    ach = 0x05;
+    acl = 0xB4CB;
+    result = 0x7FFFFFFF;
+    __asm
+        ("wrdsp %1, 0x01\n\t"
+         "mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "extrv_rs.w %0, $ac1, %2\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "+r"(dsp)
+         : "r"(rs), "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    assert(dsp == 1);
+    assert(result == rt);
+
+    /* Clear dspcontrol */
+    dsp = 0;
+    __asm
+        ("wrdsp %0\n\t"
+         :
+         : "r"(dsp)
+        );
+
+    rs = 0x04;
+    ach = 0x01;
+    acl = 0xB4CB;
+    result = 0x10000B4D;
+    __asm
+        ("wrdsp %1, 0x01\n\t"
+         "mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "extrv_rs.w %0, $ac1, %2\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "+r"(dsp)
+         : "r"(rs), "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    assert(dsp == 0);
+    assert(result == rt);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/extrv_s_h.c b/tests/tcg/mips/mips32-dsp/extrv_s_h.c
new file mode 100644
index 000000000..8c13b5eda
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/extrv_s_h.c
@@ -0,0 +1,71 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rt, rs, ach, acl, dsp;
+    int result;
+
+    ach = 0x05;
+    acl = 0xB4CB;
+    dsp = 0x07;
+    rs  = 0x03;
+    result = 0x00007FFF;
+
+    __asm
+        ("wrdsp %1, 0x01\n\t"
+         "mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "extrv_s.h %0, $ac1, %2\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "+r"(dsp)
+         : "r"(rs), "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    assert(dsp == 1);
+    assert(result == rt);
+
+    rs = 0x08;
+    ach = 0xffffffff;
+    acl = 0x12344321;
+    result = 0xFFFF8000;
+    __asm
+        ("wrdsp %1, 0x01\n\t"
+         "mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "extrv_s.h %0, $ac1, %2\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "+r"(dsp)
+         : "r"(rs), "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    assert(dsp == 1);
+    assert(result == rt);
+
+    /* Clear dsp */
+    dsp = 0;
+    __asm
+        ("wrdsp %0\n\t"
+         :
+         : "r"(dsp)
+        );
+
+    rs = 0x04;
+    ach = 0x00;
+    acl = 0x4321;
+    result = 0x432;
+    __asm
+        ("wrdsp %1, 0x01\n\t"
+         "mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "extrv_s.h %0, $ac1, %2\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "+r"(dsp)
+         : "r"(rs), "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    assert(dsp == 0);
+    assert(result == rt);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/extrv_w.c b/tests/tcg/mips/mips32-dsp/extrv_w.c
new file mode 100644
index 000000000..9cb493df3
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/extrv_w.c
@@ -0,0 +1,54 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rt, rs, ach, acl, dsp;
+    int result;
+
+    ach = 0x05;
+    acl = 0xB4CB;
+    dsp = 0x07;
+    rs  = 0x03;
+    result = 0xA0001699;
+
+    __asm
+        ("wrdsp %1, 0x01\n\t"
+         "mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "extrv.w %0, $ac1, %2\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "+r"(dsp)
+         : "r"(rs), "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    assert(dsp == 1);
+    assert(result == rt);
+
+    /* Clear dspcontrol */
+    dsp = 0;
+    __asm
+        ("wrdsp %0\n\t"
+         :
+         : "r"(dsp)
+        );
+
+    rs = 4;
+    ach = 0x01;
+    acl = 0xB4CB;
+    result = 0x10000B4C;
+    __asm
+        ("wrdsp %1, 0x01\n\t"
+         "mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "extrv.w %0, $ac1, %2\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "+r"(dsp)
+         : "r"(rs), "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    assert(dsp == 0);
+    assert(result == rt);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/insv.c b/tests/tcg/mips/mips32-dsp/insv.c
new file mode 100644
index 000000000..7e3b04760
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/insv.c
@@ -0,0 +1,23 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rt, rs, dsp;
+    int result;
+
+    /* msb = 10, lsb = 5 */
+    dsp    = 0x305;
+    rt     = 0x12345678;
+    rs     = 0x87654321;
+    result = 0x12345338;
+    __asm
+        ("wrdsp %2, 0x03\n\t"
+         "insv  %0, %1\n\t"
+         : "+r"(rt)
+         : "r"(rs), "r"(dsp)
+        );
+    assert(rt == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/lbux.c b/tests/tcg/mips/mips32-dsp/lbux.c
new file mode 100644
index 000000000..2337abea2
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/lbux.c
@@ -0,0 +1,25 @@
+#include <stdio.h>
+#include <assert.h>
+
+int main(void)
+{
+    int value, rd;
+    int *p;
+    unsigned long addr, index;
+    int result;
+
+    value  = 0xBCDEF389;
+    p = &value;
+    addr = (unsigned long)p;
+    index  = 0;
+    result = value & 0xFF;
+    __asm
+        ("lbux %0, %1(%2)\n\t"
+         : "=r"(rd)
+         : "r"(index), "r"(addr)
+        );
+
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/lhx.c b/tests/tcg/mips/mips32-dsp/lhx.c
new file mode 100644
index 000000000..10be3b385
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/lhx.c
@@ -0,0 +1,25 @@
+#include <stdio.h>
+#include <assert.h>
+
+int main(void)
+{
+    int value, rd;
+    int *p;
+    unsigned long addr, index;
+    int result;
+
+    value  = 0xBCDEF389;
+    p = &value;
+    addr = (unsigned long)p;
+    index  = 0;
+    result = 0xFFFFF389;
+    __asm
+        ("lhx %0, %1(%2)\n\t"
+         : "=r"(rd)
+         : "r"(index), "r"(addr)
+        );
+
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/lwx.c b/tests/tcg/mips/mips32-dsp/lwx.c
new file mode 100644
index 000000000..e6543c9e7
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/lwx.c
@@ -0,0 +1,25 @@
+#include <stdio.h>
+#include <assert.h>
+
+int main(void)
+{
+    int value, rd;
+    int *p;
+    unsigned long addr, index;
+    int result;
+
+    value  = 0xBCDEF389;
+    p = &value;
+    addr = (unsigned long)p;
+    index  = 0;
+    result = 0xBCDEF389;
+    __asm
+        ("lwx %0, %1(%2)\n\t"
+         : "=r"(rd)
+         : "r"(index), "r"(addr)
+        );
+
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/madd.c b/tests/tcg/mips/mips32-dsp/madd.c
new file mode 100644
index 000000000..af4bfcfe9
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/madd.c
@@ -0,0 +1,31 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rt, rs;
+    int achi, acli;
+    int acho, aclo;
+    int resulth, resultl;
+
+    achi = 0x05;
+    acli = 0xB4CB;
+    rs  = 0x01;
+    rt  = 0x01;
+    resulth = 0x05;
+    resultl = 0xB4CC;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "madd $ac1, %4, %5\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+    assert(resulth == acho);
+    assert(resultl == aclo);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/maddu.c b/tests/tcg/mips/mips32-dsp/maddu.c
new file mode 100644
index 000000000..af4bfcfe9
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/maddu.c
@@ -0,0 +1,31 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rt, rs;
+    int achi, acli;
+    int acho, aclo;
+    int resulth, resultl;
+
+    achi = 0x05;
+    acli = 0xB4CB;
+    rs  = 0x01;
+    rt  = 0x01;
+    resulth = 0x05;
+    resultl = 0xB4CC;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "madd $ac1, %4, %5\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+    assert(resulth == acho);
+    assert(resultl == aclo);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/main.c b/tests/tcg/mips/mips32-dsp/main.c
new file mode 100644
index 000000000..b296b20c9
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/main.c
@@ -0,0 +1,6 @@
+#include<stdio.h>
+
+int main()
+{
+    printf("hello world\n");
+}
diff --git a/tests/tcg/mips/mips32-dsp/maq_s_w_phl.c b/tests/tcg/mips/mips32-dsp/maq_s_w_phl.c
new file mode 100644
index 000000000..292d68566
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/maq_s_w_phl.c
@@ -0,0 +1,55 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rt, rs;
+    int achi, acli;
+    int dsp;
+    int acho, aclo;
+    int resulth, resultl;
+    int resdsp;
+
+    achi = 0x05;
+    acli = 0xB4CB;
+    rs  = 0xFF060000;
+    rt  = 0xCB000000;
+    resulth = 0x04;
+    resultl = 0x947438CB;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "maq_s.w.phl $ac1, %4, %5\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+    assert(resulth == acho);
+    assert(resultl == aclo);
+
+    achi = 0x06;
+    acli = 0xB4CB;
+    rs  = 0x80000000;
+    rt  = 0x80000000;
+    resulth = 0x6;
+    resultl = 0x8000b4ca;
+    resdsp = 1;
+
+    __asm
+        ("mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "maq_s.w.phl $ac1, %5, %6\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "=r"(acho), "=r"(aclo), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+    assert(resulth == acho);
+    assert(resultl == aclo);
+    assert(((dsp >> 17) & 0x01) == resdsp);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/maq_s_w_phr.c b/tests/tcg/mips/mips32-dsp/maq_s_w_phr.c
new file mode 100644
index 000000000..7b2ef2ab7
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/maq_s_w_phr.c
@@ -0,0 +1,55 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rt, rs;
+    int achi, acli;
+    int dsp;
+    int acho, aclo;
+    int resulth, resultl;
+    int resdsp;
+
+    achi = 0x05;
+    acli = 0xB4CB;
+    rs  = 0xFF06;
+    rt  = 0xCB00;
+    resulth = 0x04;
+    resultl = 0x947438CB;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "maq_s.w.phr $ac1, %4, %5\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+    assert(resulth == acho);
+    assert(resultl == aclo);
+
+    achi = 0x06;
+    acli = 0xB4CB;
+    rs  = 0x8000;
+    rt  = 0x8000;
+    resulth = 0x6;
+    resultl = 0x8000b4ca;
+    resdsp = 1;
+
+    __asm
+        ("mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "maq_s.w.phr $ac1, %5, %6\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "=r"(acho), "=r"(aclo), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+    assert(resulth == acho);
+    assert(resultl == aclo);
+    assert(((dsp >> 17) & 0x01) == resdsp);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/maq_sa_w_phl.c b/tests/tcg/mips/mips32-dsp/maq_sa_w_phl.c
new file mode 100644
index 000000000..a75699172
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/maq_sa_w_phl.c
@@ -0,0 +1,55 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rt, rs;
+    int achi, acli;
+    int dsp;
+    int acho, aclo;
+    int resulth, resultl;
+    int resdsp;
+
+    achi = 0x05;
+    acli = 0xB4CB;
+    rs = 0xFF060000;
+    rt = 0xCB000000;
+    resulth = 0x00;
+    resultl = 0x7FFFFFFF;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "maq_sa.w.phl $ac1, %4, %5\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+    assert(resulth == acho);
+    assert(resultl == aclo);
+
+    achi = 0x06;
+    acli = 0xB4CB;
+    rs  = 0x80000000;
+    rt  = 0x80000000;
+    resulth = 0x00;
+    resultl = 0x7fffffff;
+    resdsp = 0x01;
+
+    __asm
+        ("mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "maq_sa.w.phl $ac1, %5, %6\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "=r"(acho), "=r"(aclo), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+    assert(resulth == acho);
+    assert(resultl == aclo);
+    assert(((dsp >> 17) & 0x01) == 0x01);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/maq_sa_w_phr.c b/tests/tcg/mips/mips32-dsp/maq_sa_w_phr.c
new file mode 100644
index 000000000..d6498f8df
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/maq_sa_w_phr.c
@@ -0,0 +1,55 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rt, rs;
+    int achi, acli;
+    int dsp;
+    int acho, aclo;
+    int resulth, resultl;
+    int resdsp;
+
+    achi = 0x05;
+    acli = 0xB4CB;
+    rs  = 0xFF06;
+    rt  = 0xCB00;
+    resulth = 0x00;
+    resultl = 0x7FFFFFFF;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "maq_sa.w.phr $ac1, %4, %5\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+    assert(resulth == acho);
+    assert(resultl == aclo);
+
+    achi = 0x06;
+    acli = 0xB4CB;
+    rs  = 0x8000;
+    rt  = 0x8000;
+    resulth = 0x00;
+    resultl = 0x7fffffff;
+    resdsp = 0x01;
+
+    __asm
+        ("mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "maq_sa.w.phr $ac1, %5, %6\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "=r"(acho), "=r"(aclo), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+    assert(resulth == acho);
+    assert(resultl == aclo);
+    assert(((dsp >> 17) & 0x01) == 0x01);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/mfhi.c b/tests/tcg/mips/mips32-dsp/mfhi.c
new file mode 100644
index 000000000..43a80669d
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/mfhi.c
@@ -0,0 +1,21 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int achi, acho;
+    int result;
+
+    achi   = 0x004433;
+    result = 0x004433;
+
+    __asm
+        ("mthi %1, $ac1\n\t"
+         "mfhi %0, $ac1\n\t"
+         : "=r"(acho)
+         : "r"(achi)
+        );
+    assert(result == acho);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/mflo.c b/tests/tcg/mips/mips32-dsp/mflo.c
new file mode 100644
index 000000000..caeafdb05
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/mflo.c
@@ -0,0 +1,21 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int acli, aclo;
+    int result;
+
+    acli   = 0x004433;
+    result = 0x004433;
+
+    __asm
+        ("mthi %1, $ac1\n\t"
+         "mfhi %0, $ac1\n\t"
+         : "=r"(aclo)
+         : "r"(acli)
+        );
+    assert(result == aclo);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/modsub.c b/tests/tcg/mips/mips32-dsp/modsub.c
new file mode 100644
index 000000000..c294eebb5
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/modsub.c
@@ -0,0 +1,30 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int result;
+
+    rs     = 0xFFFFFFFF;
+    rt     = 0x000000FF;
+    result = 0xFFFFFF00;
+    __asm
+        ("modsub %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    assert(result == rd);
+
+    rs     = 0x00000000;
+    rt     = 0x00CD1FFF;
+    result = 0x0000CD1F;
+    __asm
+        ("modsub %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    assert(result == rd);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/msub.c b/tests/tcg/mips/mips32-dsp/msub.c
new file mode 100644
index 000000000..5779e6f47
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/msub.c
@@ -0,0 +1,30 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int achi, acli, rs, rt;
+    int acho, aclo;
+    int resulth, resultl;
+
+    rs      = 0x00BBAACC;
+    rt      = 0x0B1C3D2F;
+    achi    = 0x00004433;
+    acli    = 0xFFCC0011;
+    resulth = 0xFFF81F29;
+    resultl = 0xB355089D;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "msub $ac1, %4, %5\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+    assert(acho == resulth);
+    assert(aclo == resultl);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/msubu.c b/tests/tcg/mips/mips32-dsp/msubu.c
new file mode 100644
index 000000000..e0f9b5a77
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/msubu.c
@@ -0,0 +1,30 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int achi, acli, rs, rt;
+    int acho, aclo;
+    int resulth, resultl;
+
+    rs      = 0x00BBAACC;
+    rt      = 0x0B1C3D2F;
+    achi    = 0x00004433;
+    acli    = 0xFFCC0011;
+    resulth = 0xFFF81F29;
+    resultl = 0xB355089D;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "msubu $ac1, %4, %5\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+    assert(acho == resulth);
+    assert(aclo == resultl);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/mthi.c b/tests/tcg/mips/mips32-dsp/mthi.c
new file mode 100644
index 000000000..43a80669d
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/mthi.c
@@ -0,0 +1,21 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int achi, acho;
+    int result;
+
+    achi   = 0x004433;
+    result = 0x004433;
+
+    __asm
+        ("mthi %1, $ac1\n\t"
+         "mfhi %0, $ac1\n\t"
+         : "=r"(acho)
+         : "r"(achi)
+        );
+    assert(result == acho);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/mthlip.c b/tests/tcg/mips/mips32-dsp/mthlip.c
new file mode 100644
index 000000000..9549aae36
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/mthlip.c
@@ -0,0 +1,58 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rs, ach, acl, dsp;
+    int result, resulth, resultl;
+
+    dsp = 0x07;
+    ach = 0x05;
+    acl = 0xB4CB;
+    rs  = 0x00FFBBAA;
+    resulth = 0xB4CB;
+    resultl = 0x00FFBBAA;
+    result  = 0x27;
+
+    __asm
+        ("wrdsp %0, 0x01\n\t"
+         "mthi %1, $ac1\n\t"
+         "mtlo %2, $ac1\n\t"
+         "mthlip %3, $ac1\n\t"
+         "mfhi %1, $ac1\n\t"
+         "mflo %2, $ac1\n\t"
+         "rddsp %0\n\t"
+         : "+r"(dsp), "+r"(ach), "+r"(acl)
+         : "r"(rs)
+        );
+    dsp = dsp & 0x3F;
+    assert(dsp == result);
+    assert(ach == resulth);
+    assert(acl == resultl);
+
+    dsp = 0x3f;
+    ach = 0x05;
+    acl = 0xB4CB;
+    rs  = 0x00FFBBAA;
+    resulth = 0xB4CB;
+    resultl = 0x00FFBBAA;
+    result  = 0x3f;
+
+    __asm
+        ("wrdsp %0, 0x01\n\t"
+         "mthi %1, $ac1\n\t"
+         "mtlo %2, $ac1\n\t"
+         "mthlip %3, $ac1\n\t"
+         "mfhi %1, $ac1\n\t"
+         "mflo %2, $ac1\n\t"
+         "rddsp %0\n\t"
+         : "+r"(dsp), "+r"(ach), "+r"(acl)
+         : "r"(rs)
+        );
+    dsp = dsp & 0x3F;
+    assert(dsp == result);
+    assert(ach == resulth);
+    assert(acl == resultl);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/mtlo.c b/tests/tcg/mips/mips32-dsp/mtlo.c
new file mode 100644
index 000000000..caeafdb05
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/mtlo.c
@@ -0,0 +1,21 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int acli, aclo;
+    int result;
+
+    acli   = 0x004433;
+    result = 0x004433;
+
+    __asm
+        ("mthi %1, $ac1\n\t"
+         "mfhi %0, $ac1\n\t"
+         : "=r"(aclo)
+         : "r"(acli)
+        );
+    assert(result == aclo);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/muleq_s_w_phl.c b/tests/tcg/mips/mips32-dsp/muleq_s_w_phl.c
new file mode 100644
index 000000000..b3a5370fe
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/muleq_s_w_phl.c
@@ -0,0 +1,41 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt, dsp;
+    int result, resultdsp;
+
+    rs = 0x80001234;
+    rt = 0x80001234;
+    result = 0x7FFFFFFF;
+    resultdsp = 1;
+
+    __asm
+        ("muleq_s.w.phl %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 21) & 0x01;
+    assert(rd  == result);
+    assert(dsp == resultdsp);
+
+    rs = 0x12349988;
+    rt = 0x43219988;
+    result = 0x98be968;
+    resultdsp = 1;
+
+    __asm
+        ("muleq_s.w.phl %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 21) & 0x01;
+    assert(rd  == result);
+    assert(dsp == resultdsp);
+
+    return 0;
+}
+
diff --git a/tests/tcg/mips/mips32-dsp/muleq_s_w_phr.c b/tests/tcg/mips/mips32-dsp/muleq_s_w_phr.c
new file mode 100644
index 000000000..8066d7d02
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/muleq_s_w_phr.c
@@ -0,0 +1,40 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt, dsp;
+    int result, resultdsp;
+
+    rs = 0x8000;
+    rt = 0x8000;
+    result = 0x7FFFFFFF;
+    resultdsp = 1;
+
+    __asm
+        ("muleq_s.w.phr %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 21) & 0x01;
+    assert(rd  == result);
+    assert(dsp == resultdsp);
+
+    rs = 0x1234;
+    rt = 0x4321;
+    result = 0x98be968;
+    resultdsp = 1;
+
+    __asm
+        ("muleq_s.w.phr %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 21) & 0x01;
+    assert(rd  == result);
+    assert(dsp == resultdsp);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/muleu_s_ph_qbl.c b/tests/tcg/mips/mips32-dsp/muleu_s_ph_qbl.c
new file mode 100644
index 000000000..66a382806
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/muleu_s_ph_qbl.c
@@ -0,0 +1,25 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt, dsp;
+    int result, resultdsp;
+
+    rs = 0x80001234;
+    rt = 0x80004321;
+    result = 0xFFFF0000;
+    resultdsp = 1;
+
+    __asm
+        ("muleu_s.ph.qbl %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 21) & 0x01;
+    assert(rd  == result);
+    assert(dsp == resultdsp);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/muleu_s_ph_qbr.c b/tests/tcg/mips/mips32-dsp/muleu_s_ph_qbr.c
new file mode 100644
index 000000000..4cc6c8f7c
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/muleu_s_ph_qbr.c
@@ -0,0 +1,25 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt, dsp;
+    int result, resultdsp;
+
+    rs = 0x8000;
+    rt = 0x80004321;
+    result = 0xFFFF0000;
+    resultdsp = 1;
+
+    __asm
+        ("muleu_s.ph.qbr %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 21) & 0x01;
+    assert(rd  == result);
+    assert(dsp == resultdsp);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/mulq_rs_ph.c b/tests/tcg/mips/mips32-dsp/mulq_rs_ph.c
new file mode 100644
index 000000000..c7206039e
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/mulq_rs_ph.c
@@ -0,0 +1,25 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt, dsp;
+    int result, resultdsp;
+
+    rs = 0x80001234;
+    rt = 0x80004321;
+    result = 0x7FFF098C;
+    resultdsp = 1;
+
+    __asm
+        ("mulq_rs.ph %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 21) & 0x01;
+    assert(rd  == result);
+    assert(dsp == resultdsp);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/mult.c b/tests/tcg/mips/mips32-dsp/mult.c
new file mode 100644
index 000000000..15e6fde92
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/mult.c
@@ -0,0 +1,24 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rs, rt, ach, acl;
+    int result, resulth, resultl;
+
+    rs  = 0x00FFBBAA;
+    rt  = 0x4B231000;
+    resulth = 0x4b0f01;
+    resultl = 0x71f8a000;
+    __asm
+        ("mult $ac1, %2, %3\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "=r"(ach), "=r"(acl)
+         : "r"(rs), "r"(rt)
+        );
+    assert(ach == resulth);
+    assert(acl == resultl);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/multu.c b/tests/tcg/mips/mips32-dsp/multu.c
new file mode 100644
index 000000000..85d36c1b6
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/multu.c
@@ -0,0 +1,24 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rs, rt, ach, acl;
+    int result, resulth, resultl;
+
+    rs  = 0x00FFBBAA;
+    rt  = 0x4B231000;
+    resulth = 0x4b0f01;
+    resultl = 0x71f8a000;
+    __asm
+        ("multu $ac1, %2, %3\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "=r"(ach), "=r"(acl)
+         : "r"(rs), "r"(rt)
+        );
+    assert(ach == resulth);
+    assert(acl == resultl);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/packrl_ph.c b/tests/tcg/mips/mips32-dsp/packrl_ph.c
new file mode 100644
index 000000000..1f8e69992
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/packrl_ph.c
@@ -0,0 +1,21 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int result;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result = 0x56788765;
+
+    __asm
+        ("packrl.ph %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    assert(result == rd);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/pick_ph.c b/tests/tcg/mips/mips32-dsp/pick_ph.c
new file mode 100644
index 000000000..929a002e7
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/pick_ph.c
@@ -0,0 +1,49 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt, dsp;
+    int result;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    dsp = 0x0A000000;
+    result = 0x12344321;
+
+    __asm
+        ("wrdsp %3, 0x10\n\t"
+         "pick.ph %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt), "r"(dsp)
+        );
+    assert(rd == result);
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    dsp = 0x03000000;
+    result = 0x12345678;
+
+    __asm
+        ("wrdsp %3, 0x10\n\t"
+         "pick.ph %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt), "r"(dsp)
+        );
+    assert(rd == result);
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    dsp = 0x00000000;
+    result = 0x87654321;
+
+    __asm
+        ("wrdsp %3, 0x10\n\t"
+         "pick.ph %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt), "r"(dsp)
+        );
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/pick_qb.c b/tests/tcg/mips/mips32-dsp/pick_qb.c
new file mode 100644
index 000000000..a79047524
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/pick_qb.c
@@ -0,0 +1,36 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt, dsp;
+    int result;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    dsp = 0x0f000000;
+    result = 0x12345678;
+
+    __asm
+        ("wrdsp %3, 0x10\n\t"
+         "pick.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt), "r"(dsp)
+        );
+    assert(rd == result);
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    dsp = 0x00000000;
+    result = 0x87654321;
+
+    __asm
+        ("wrdsp %3, 0x10\n\t"
+         "pick.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt), "r"(dsp)
+        );
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/preceq_w_phl.c b/tests/tcg/mips/mips32-dsp/preceq_w_phl.c
new file mode 100644
index 000000000..bf70bf7d3
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/preceq_w_phl.c
@@ -0,0 +1,20 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rt;
+    int result;
+
+    rt = 0x87654321;
+    result = 0x87650000;
+
+    __asm
+        ("preceq.w.phl %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    assert(result == rd);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/preceq_w_phr.c b/tests/tcg/mips/mips32-dsp/preceq_w_phr.c
new file mode 100644
index 000000000..3f885ef58
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/preceq_w_phr.c
@@ -0,0 +1,20 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rt;
+    int result;
+
+    rt = 0x87654321;
+    result = 0x43210000;
+
+    __asm
+        ("preceq.w.phr %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    assert(result == rd);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/precequ_ph_qbl.c b/tests/tcg/mips/mips32-dsp/precequ_ph_qbl.c
new file mode 100644
index 000000000..63b7a9568
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/precequ_ph_qbl.c
@@ -0,0 +1,20 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rt;
+    int result;
+
+    rt = 0x87654321;
+    result = 0x43803280;
+
+    __asm
+        ("precequ.ph.qbl %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    assert(result == rd);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/precequ_ph_qbla.c b/tests/tcg/mips/mips32-dsp/precequ_ph_qbla.c
new file mode 100644
index 000000000..31627f0bd
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/precequ_ph_qbla.c
@@ -0,0 +1,20 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rt;
+    int result;
+
+    rt = 0x87654321;
+    result = 0x43802180;
+
+    __asm
+        ("precequ.ph.qbla %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    assert(result == rd);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/precequ_ph_qbr.c b/tests/tcg/mips/mips32-dsp/precequ_ph_qbr.c
new file mode 100644
index 000000000..b6f72d3cb
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/precequ_ph_qbr.c
@@ -0,0 +1,20 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rt;
+    int result;
+
+    rt = 0x87654321;
+    result = 0x21801080;
+
+    __asm
+        ("precequ.ph.qbr %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    assert(result == rd);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/precequ_ph_qbra.c b/tests/tcg/mips/mips32-dsp/precequ_ph_qbra.c
new file mode 100644
index 000000000..4764fd031
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/precequ_ph_qbra.c
@@ -0,0 +1,20 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rt;
+    int result;
+
+    rt = 0x87654321;
+    result = 0x32801080;
+
+    __asm
+        ("precequ.ph.qbra %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    assert(result == rd);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/preceu_ph_qbl.c b/tests/tcg/mips/mips32-dsp/preceu_ph_qbl.c
new file mode 100644
index 000000000..fa95c26cc
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/preceu_ph_qbl.c
@@ -0,0 +1,20 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rt;
+    int result;
+
+    rt = 0x87654321;
+    result = 0x00870065;
+
+    __asm
+        ("preceu.ph.qbl %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    assert(result == rd);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/preceu_ph_qbla.c b/tests/tcg/mips/mips32-dsp/preceu_ph_qbla.c
new file mode 100644
index 000000000..021f21a74
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/preceu_ph_qbla.c
@@ -0,0 +1,20 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rt;
+    int result;
+
+    rt = 0x87654321;
+    result = 0x00870043;
+
+    __asm
+        ("preceu.ph.qbla %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    assert(result == rd);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/preceu_ph_qbr.c b/tests/tcg/mips/mips32-dsp/preceu_ph_qbr.c
new file mode 100644
index 000000000..03df18c72
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/preceu_ph_qbr.c
@@ -0,0 +1,20 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rt;
+    int result;
+
+    rt = 0x87654321;
+    result = 0x00430021;
+
+    __asm
+        ("preceu.ph.qbr %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    assert(result == rd);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/preceu_ph_qbra.c b/tests/tcg/mips/mips32-dsp/preceu_ph_qbra.c
new file mode 100644
index 000000000..634327618
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/preceu_ph_qbra.c
@@ -0,0 +1,20 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rt;
+    int result;
+
+    rt = 0x87654321;
+    result = 0x00650021;
+
+    __asm
+        ("preceu.ph.qbra %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    assert(result == rd);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/precrq_ph_w.c b/tests/tcg/mips/mips32-dsp/precrq_ph_w.c
new file mode 100644
index 000000000..25d45f1a9
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/precrq_ph_w.c
@@ -0,0 +1,21 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int result;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result = 0x12348765;
+
+    __asm
+        ("precrq.ph.w %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    assert(result == rd);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/precrq_qb_ph.c b/tests/tcg/mips/mips32-dsp/precrq_qb_ph.c
new file mode 100644
index 000000000..fe23acce8
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/precrq_qb_ph.c
@@ -0,0 +1,21 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int result;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result = 0x12568743;
+
+    __asm
+        ("precrq.qb.ph %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    assert(result == rd);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/precrq_rs_ph_w.c b/tests/tcg/mips/mips32-dsp/precrq_rs_ph_w.c
new file mode 100644
index 000000000..3535b37a5
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/precrq_rs_ph_w.c
@@ -0,0 +1,35 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int dsp;
+    int result;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result = 0x12348765;
+
+    __asm
+        ("precrq_rs.ph.w %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    assert(result == rd);
+
+    rs = 0x7fffC678;
+    rt = 0x865432A0;
+    result = 0x7fff8654;
+
+    __asm
+        ("precrq_rs.ph.w %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    assert(((dsp >> 22) & 0x01) == 1);
+    assert(result == rd);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/precrqu_s_qb_ph.c b/tests/tcg/mips/mips32-dsp/precrqu_s_qb_ph.c
new file mode 100644
index 000000000..7481d5af3
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/precrqu_s_qb_ph.c
@@ -0,0 +1,24 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int dsp;
+    int result;
+
+    rs = 0x12345678;
+    rt = 0x87657FFF;
+    result = 0x24AC00FF;
+
+    __asm
+        ("precrqu_s.qb.ph %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    assert(result == rd);
+    assert(((dsp >> 22) & 0x01) == 0x01);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/raddu_w_qb.c b/tests/tcg/mips/mips32-dsp/raddu_w_qb.c
new file mode 100644
index 000000000..77a983c0d
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/raddu_w_qb.c
@@ -0,0 +1,20 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs;
+    int result;
+
+    rs = 0x12345678;
+    result = 0x114;
+
+    __asm
+        ("raddu.w.qb %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rs)
+        );
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/rddsp.c b/tests/tcg/mips/mips32-dsp/rddsp.c
new file mode 100644
index 000000000..e8948ec1d
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/rddsp.c
@@ -0,0 +1,54 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int dsp_i, dsp_o;
+    int ccond_i, outflag_i, efi_i, c_i, scount_i, pos_i;
+    int ccond_o, outflag_o, efi_o, c_o, scount_o, pos_o;
+    int ccond_r, outflag_r, efi_r, c_r, scount_r, pos_r;
+
+    ccond_i   = 0x000000BC;/* 4 */
+    outflag_i = 0x0000001B;/* 3 */
+    efi_i     = 0x00000001;/* 5 */
+    c_i       = 0x00000001;/* 2 */
+    scount_i  = 0x0000000F;/* 1 */
+    pos_i     = 0x0000000C;/* 0 */
+
+    dsp_i = (ccond_i   << 24) | \
+            (outflag_i << 16) | \
+            (efi_i     << 14) | \
+            (c_i       << 13) | \
+            (scount_i  <<  7) | \
+            pos_i;
+
+    ccond_r   = ccond_i;
+    outflag_r = outflag_i;
+    efi_r     = efi_i;
+    c_r       = c_i;
+    scount_r  = scount_i;
+    pos_r     = pos_i;
+
+    __asm
+        ("wrdsp %1, 0x3F\n\t"
+         "rddsp %0, 0x3F\n\t"
+         : "=r"(dsp_o)
+         : "r"(dsp_i)
+        );
+
+    ccond_o   = (dsp_o >> 24) & 0xFF;
+    outflag_o = (dsp_o >> 16) & 0xFF;
+    efi_o     = (dsp_o >> 14) & 0x01;
+    c_o       = (dsp_o >> 14) & 0x01;
+    scount_o  = (dsp_o >>  7) & 0x3F;
+    pos_o     =  dsp_o & 0x1F;
+
+    assert(ccond_o   == ccond_r);
+    assert(outflag_o == outflag_r);
+    assert(efi_o     == efi_r);
+    assert(c_o       == c_r);
+    assert(scount_o  == scount_r);
+    assert(pos_o     == pos_r);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/repl_ph.c b/tests/tcg/mips/mips32-dsp/repl_ph.c
new file mode 100644
index 000000000..21074953b
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/repl_ph.c
@@ -0,0 +1,23 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, result;
+
+    result = 0x01BF01BF;
+    __asm
+        ("repl.ph %0, 0x1BF\n\t"
+         : "=r"(rd)
+        );
+    assert(rd == result);
+
+    result = 0x01FF01FF;
+    __asm
+        ("repl.ph %0, 0x01FF\n\t"
+         : "=r"(rd)
+        );
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/repl_qb.c b/tests/tcg/mips/mips32-dsp/repl_qb.c
new file mode 100644
index 000000000..6631393ea
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/repl_qb.c
@@ -0,0 +1,16 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, result;
+
+    result = 0xBFBFBFBF;
+    __asm
+        ("repl.qb %0, 0xBF\n\t"
+         : "=r"(rd)
+        );
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/replv_ph.c b/tests/tcg/mips/mips32-dsp/replv_ph.c
new file mode 100644
index 000000000..07fb15f1f
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/replv_ph.c
@@ -0,0 +1,19 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rt;
+    int result;
+
+    rt = 0x12345678;
+    result = 0x56785678;
+    __asm
+        ("replv.ph %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/replv_qb.c b/tests/tcg/mips/mips32-dsp/replv_qb.c
new file mode 100644
index 000000000..dd1271fed
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/replv_qb.c
@@ -0,0 +1,19 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rt;
+    int result;
+
+    rt     = 0x12345678;
+    result = 0x78787878;
+    __asm
+        ("replv.qb %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/shilo.c b/tests/tcg/mips/mips32-dsp/shilo.c
new file mode 100644
index 000000000..b686616d8
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/shilo.c
@@ -0,0 +1,27 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int ach, acl;
+    int resulth, resultl;
+
+    ach = 0xBBAACCFF;
+    acl = 0x1C3B001D;
+
+    resulth = 0x17755;
+    resultl = 0x99fe3876;
+
+    __asm
+        ("mthi %0, $ac1\n\t"
+         "mtlo %1, $ac1\n\t"
+         "shilo $ac1, 0x0F\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "+r"(ach), "+r"(acl)
+        );
+    assert(ach == resulth);
+    assert(acl == resultl);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/shilov.c b/tests/tcg/mips/mips32-dsp/shilov.c
new file mode 100644
index 000000000..f186032a1
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/shilov.c
@@ -0,0 +1,29 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rs, ach, acl;
+    int resulth, resultl;
+
+    rs  = 0x0F;
+    ach = 0xBBAACCFF;
+    acl = 0x1C3B001D;
+
+    resulth = 0x17755;
+    resultl = 0x99fe3876;
+
+    __asm
+        ("mthi %0, $ac1\n\t"
+         "mtlo %1, $ac1\n\t"
+         "shilov $ac1, %2\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "+r"(ach), "+r"(acl)
+         : "r"(rs)
+        );
+    assert(ach == resulth);
+    assert(acl == resultl);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/shll_ph.c b/tests/tcg/mips/mips32-dsp/shll_ph.c
new file mode 100644
index 000000000..b8f1ff528
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/shll_ph.c
@@ -0,0 +1,24 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rt, dsp;
+    int result, resultdsp;
+
+    rt        = 0x12345678;
+    result    = 0xA000C000;
+    resultdsp = 1;
+
+    __asm
+        ("shll.ph %0, %2, 0x0B\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt)
+        );
+    dsp = (dsp >> 22) & 0x01;
+    assert(dsp == resultdsp);
+    assert(rd  == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/shll_qb.c b/tests/tcg/mips/mips32-dsp/shll_qb.c
new file mode 100644
index 000000000..8c1b91c63
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/shll_qb.c
@@ -0,0 +1,36 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rt, dsp;
+    int result, resultdsp;
+
+    rt     = 0x87654321;
+    result  = 0x87654321;
+    resultdsp = 0x00;
+
+    __asm
+        ("shll.qb %0, %2, 0x00\n\t"
+         "rddsp   %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt)
+        );
+    dsp = (dsp >> 22) & 0x01;
+    assert(rd == result);
+
+    rt     = 0x87654321;
+    result = 0x38281808;
+    resultdsp = 0x01;
+
+    __asm
+        ("shll.qb %0, %2, 0x03\n\t"
+         "rddsp   %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt)
+        );
+    dsp = (dsp >> 22) & 0x01;
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/shll_s_ph.c b/tests/tcg/mips/mips32-dsp/shll_s_ph.c
new file mode 100644
index 000000000..910fea3b3
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/shll_s_ph.c
@@ -0,0 +1,24 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rt, dsp;
+    int result, resultdsp;
+
+    rt        = 0x12345678;
+    result    = 0x7FFF7FFF;
+    resultdsp = 0x01;
+
+    __asm
+        ("shll_s.ph %0, %2, 0x0B\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt)
+        );
+    dsp = (dsp >> 22) & 0x01;
+    assert(dsp == resultdsp);
+    assert(rd  == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/shll_s_w.c b/tests/tcg/mips/mips32-dsp/shll_s_w.c
new file mode 100644
index 000000000..628c75210
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/shll_s_w.c
@@ -0,0 +1,52 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rt, dsp;
+    int result, resultdsp;
+
+    rt        = 0x82345678;
+    result    = 0x82345678;
+    resultdsp = 0x00;
+
+    __asm
+        ("shll_s.w %0, %2, 0x0\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt)
+        );
+    dsp = (dsp >> 22) & 0x01;
+    assert(dsp == resultdsp);
+    assert(rd  == result);
+
+    rt        = 0x82345678;
+    result    = 0x80000000;
+    resultdsp = 0x01;
+
+    __asm
+        ("shll_s.w %0, %2, 0x0B\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt)
+        );
+    dsp = (dsp >> 22) & 0x01;
+    assert(dsp == resultdsp);
+    assert(rd  == result);
+
+    rt        = 0x12345678;
+    result    = 0x7FFFFFFF;
+    resultdsp = 0x01;
+
+    __asm
+        ("shll_s.w %0, %2, 0x0B\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt)
+        );
+    dsp = (dsp >> 22) & 0x01;
+    assert(dsp == resultdsp);
+    assert(rd  == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/shllv_ph.c b/tests/tcg/mips/mips32-dsp/shllv_ph.c
new file mode 100644
index 000000000..f98a6322d
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/shllv_ph.c
@@ -0,0 +1,40 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt, dsp;
+    int result, resultdsp;
+
+    rs        = 0x0;
+    rt        = 0x12345678;
+    result    = 0x12345678;
+    resultdsp = 0;
+
+    __asm
+        ("shllv.ph %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt), "r"(rs)
+        );
+    dsp = (dsp >> 22) & 0x01;
+    assert(dsp == resultdsp);
+    assert(rd  == result);
+
+    rs        = 0x0B;
+    rt        = 0x12345678;
+    result    = 0xA000C000;
+    resultdsp = 1;
+
+    __asm
+        ("shllv.ph %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt), "r"(rs)
+        );
+    dsp = (dsp >> 22) & 0x01;
+    assert(dsp == resultdsp);
+    assert(rd  == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/shllv_qb.c b/tests/tcg/mips/mips32-dsp/shllv_qb.c
new file mode 100644
index 000000000..6d8ff4a25
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/shllv_qb.c
@@ -0,0 +1,38 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt, dsp;
+    int result, resultdsp;
+
+    rs     = 0x03;
+    rt     = 0x87654321;
+    result = 0x38281808;
+    resultdsp = 0x01;
+
+    __asm
+        ("shllv.qb %0, %2, %3\n\t"
+         "rddsp   %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt), "r"(rs)
+        );
+    dsp = (dsp >> 22) & 0x01;
+    assert(rd == result);
+
+    rs     = 0x00;
+    rt     = 0x87654321;
+    result = 0x87654321;
+    resultdsp = 0x01;
+
+    __asm
+        ("shllv.qb %0, %2, %3\n\t"
+         "rddsp   %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt), "r"(rs)
+        );
+    dsp = (dsp >> 22) & 0x01;
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/shllv_s_ph.c b/tests/tcg/mips/mips32-dsp/shllv_s_ph.c
new file mode 100644
index 000000000..fc9bd3276
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/shllv_s_ph.c
@@ -0,0 +1,40 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt, dsp;
+    int result, resultdsp;
+
+    rs        = 0x0;
+    rt        = 0x12345678;
+    result    = 0x12345678;
+    resultdsp = 0x0;
+
+    __asm
+        ("shllv_s.ph %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt), "r"(rs)
+        );
+    dsp = (dsp >> 22) & 0x01;
+    assert(dsp == resultdsp);
+    assert(rd  == result);
+
+    rs        = 0x0B;
+    rt        = 0x12345678;
+    result    = 0x7FFF7FFF;
+    resultdsp = 0x01;
+
+    __asm
+        ("shllv_s.ph %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt), "r"(rs)
+        );
+    dsp = (dsp >> 22) & 0x01;
+    assert(dsp == resultdsp);
+    assert(rd  == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/shllv_s_w.c b/tests/tcg/mips/mips32-dsp/shllv_s_w.c
new file mode 100644
index 000000000..350c25617
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/shllv_s_w.c
@@ -0,0 +1,40 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt, dsp;
+    int result, resultdsp;
+
+    rs        = 0x0B;
+    rt        = 0x12345678;
+    result    = 0x7FFFFFFF;
+    resultdsp = 0x01;
+
+    __asm
+        ("shllv_s.w %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt), "r"(rs)
+        );
+    dsp = (dsp >> 22) & 0x01;
+    assert(dsp == resultdsp);
+    assert(rd  == result);
+
+    rs        = 0x0;
+    rt        = 0x12345678;
+    result    = 0x12345678;
+    resultdsp = 0x01;
+
+    __asm
+        ("shllv_s.w %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt), "r"(rs)
+        );
+    dsp = (dsp >> 22) & 0x01;
+    assert(dsp == resultdsp);
+    assert(rd  == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/shra_ph.c b/tests/tcg/mips/mips32-dsp/shra_ph.c
new file mode 100644
index 000000000..5b2d840a6
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/shra_ph.c
@@ -0,0 +1,30 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rt;
+    int result;
+
+    rt     = 0x87654321;
+    result = 0xF0EC0864;
+
+    __asm
+        ("shra.ph %0, %1, 0x03\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    assert(rd == result);
+
+    rt     = 0x87654321;
+    result = 0x87654321;
+
+    __asm
+        ("shra.ph %0, %1, 0x00\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/shra_r_ph.c b/tests/tcg/mips/mips32-dsp/shra_r_ph.c
new file mode 100644
index 000000000..adc4ae68b
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/shra_r_ph.c
@@ -0,0 +1,30 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rt;
+    int result;
+
+    rt     = 0x87654321;
+    result = 0xF0ED0864;
+
+    __asm
+        ("shra_r.ph %0, %1, 0x03\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    assert(rd == result);
+
+    rt     = 0x87654321;
+    result = 0x87654321;
+
+    __asm
+        ("shra_r.ph %0, %1, 0x00\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/shra_r_w.c b/tests/tcg/mips/mips32-dsp/shra_r_w.c
new file mode 100644
index 000000000..ec0cf2c72
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/shra_r_w.c
@@ -0,0 +1,30 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rt;
+    int result;
+
+    rt     = 0x87654321;
+    result = 0xF0ECA864;
+
+    __asm
+        ("shra_r.w %0, %1, 0x03\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    assert(rd == result);
+
+    rt     = 0x87654321;
+    result = 0x87654321;
+
+    __asm
+        ("shra_r.w %0, %1, 0x0\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/shrav_ph.c b/tests/tcg/mips/mips32-dsp/shrav_ph.c
new file mode 100644
index 000000000..6e42aaf8e
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/shrav_ph.c
@@ -0,0 +1,32 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int result;
+
+    rs     = 0x03;
+    rt     = 0x87654321;
+    result = 0xF0EC0864;
+
+    __asm
+        ("shrav.ph %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rt), "r"(rs)
+        );
+    assert(rd == result);
+
+    rs     = 0x00;
+    rt     = 0x87654321;
+    result = 0x87654321;
+
+    __asm
+        ("shrav.ph %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rt), "r"(rs)
+        );
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/shrav_r_ph.c b/tests/tcg/mips/mips32-dsp/shrav_r_ph.c
new file mode 100644
index 000000000..f03b978d0
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/shrav_r_ph.c
@@ -0,0 +1,32 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int result;
+
+    rs     = 0x03;
+    rt     = 0x87654321;
+    result = 0xF0ED0864;
+
+    __asm
+        ("shrav_r.ph %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rt), "r"(rs)
+        );
+    assert(rd == result);
+
+    rs     = 0x00;
+    rt     = 0x87654321;
+    result = 0x87654321;
+
+    __asm
+        ("shrav_r.ph %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rt), "r"(rs)
+        );
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/shrav_r_w.c b/tests/tcg/mips/mips32-dsp/shrav_r_w.c
new file mode 100644
index 000000000..2ab03bb5d
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/shrav_r_w.c
@@ -0,0 +1,32 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int result;
+
+    rs     = 0x03;
+    rt     = 0x87654321;
+    result = 0xF0ECA864;
+
+    __asm
+        ("shrav_r.w %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rt), "r"(rs)
+        );
+    assert(rd == result);
+
+    rs     = 0x00;
+    rt     = 0x40000000;
+    result = 0x40000000;
+
+    __asm
+        ("shrav_r.w %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rt), "r"(rs)
+        );
+
+    assert(rd == result);
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/shrl_qb.c b/tests/tcg/mips/mips32-dsp/shrl_qb.c
new file mode 100644
index 000000000..a7e4e6a5e
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/shrl_qb.c
@@ -0,0 +1,31 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rt;
+    int result;
+
+    rt     = 0x12345678;
+    result = 0x00010203;
+
+    __asm
+        ("shrl.qb %0, %1, 0x05\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    assert(rd == result);
+
+    rt     = 0x12345678;
+    result = 0x12345678;
+
+    __asm
+        ("shrl.qb %0, %1, 0x0\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/shrlv_qb.c b/tests/tcg/mips/mips32-dsp/shrlv_qb.c
new file mode 100644
index 000000000..db77f6d0e
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/shrlv_qb.c
@@ -0,0 +1,32 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int result;
+
+    rs     = 0x05;
+    rt     = 0x12345678;
+    result = 0x00010203;
+
+    __asm
+        ("shrlv.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rt), "r"(rs)
+        );
+    assert(rd == result);
+
+    rs     = 0x00;
+    rt     = 0x12345678;
+    result = 0x12345678;
+
+    __asm
+        ("shrlv.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rt), "r"(rs)
+        );
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/subq_ph.c b/tests/tcg/mips/mips32-dsp/subq_ph.c
new file mode 100644
index 000000000..fdd7b38b6
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/subq_ph.c
@@ -0,0 +1,40 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt, dsp;
+    int result, resultdsp;
+
+    rs = 0x77777777;
+    rt = 0x67654321;
+    result    = 0x10123456;
+    resultdsp = 0x0;
+
+    __asm
+        ("subq.ph %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 20) & 0x01;
+    assert(dsp == resultdsp);
+    assert(rd  == result);
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result    = 0x8ACF1357;
+    resultdsp = 0x01;
+
+    __asm
+        ("subq.ph %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 20) & 0x01;
+    assert(dsp == resultdsp);
+    assert(rd  == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/subq_s_ph.c b/tests/tcg/mips/mips32-dsp/subq_s_ph.c
new file mode 100644
index 000000000..8e36dadef
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/subq_s_ph.c
@@ -0,0 +1,40 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt, dsp;
+    int result, resultdsp;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result    = 0x7FFF1357;
+    resultdsp = 0x01;
+
+    __asm
+        ("subq_s.ph %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 20) & 0x01;
+    assert(dsp == resultdsp);
+    assert(rd  == result);
+
+    rs = 0x12348000;
+    rt = 0x87657000;
+    result    = 0x7FFF8000;
+    resultdsp = 0x01;
+
+    __asm
+        ("subq_s.ph %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 20) & 0x01;
+    assert(dsp == resultdsp);
+    assert(rd  == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/subq_s_w.c b/tests/tcg/mips/mips32-dsp/subq_s_w.c
new file mode 100644
index 000000000..09022e9c8
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/subq_s_w.c
@@ -0,0 +1,58 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt, dsp;
+    int result, resultdsp;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result    = 0x7FFFFFFF;
+    resultdsp = 0x01;
+
+    __asm
+        ("subq_s.w %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 20) & 0x01;
+    assert(dsp == resultdsp);
+    assert(rd  == result);
+
+    rs = 0x66666;
+    rt = 0x55555;
+    result    = 0x11111;
+    resultdsp = 0x01;
+
+    __asm
+        ("subq_s.w %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 20) & 0x01;
+    assert(dsp == resultdsp);
+    assert(rd  == result);
+
+
+#if 0
+    rs = 0x35555555;
+    rt = 0xf5555555;
+    result    = 0x80000000;
+    resultdsp = 0x01;
+
+    __asm
+        ("subq_s.w %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+
+    dsp = (dsp >> 20) & 0x01;
+    assert(dsp == resultdsp);
+    assert(rd  == result);
+#endif
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/subu_qb.c b/tests/tcg/mips/mips32-dsp/subu_qb.c
new file mode 100644
index 000000000..420909615
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/subu_qb.c
@@ -0,0 +1,25 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt, dsp;
+    int result, resultdsp;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result    = 0x8BCF1357;
+    resultdsp = 0x01;
+
+    __asm
+        ("subu.qb %0, %2, %3\n\t"
+         "rddsp   %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 20) & 0x01;
+    assert(dsp == resultdsp);
+    assert(rd  == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/subu_s_qb.c b/tests/tcg/mips/mips32-dsp/subu_s_qb.c
new file mode 100644
index 000000000..3d650533d
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/subu_s_qb.c
@@ -0,0 +1,25 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt, dsp;
+    int result, resultdsp;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result    = 0x00001357;
+    resultdsp = 0x01;
+
+    __asm
+        ("subu_s.qb %0, %2, %3\n\t"
+         "rddsp   %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 20) & 0x01;
+    assert(dsp == resultdsp);
+    assert(rd  == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dsp/wrdsp.c b/tests/tcg/mips/mips32-dsp/wrdsp.c
new file mode 100644
index 000000000..e8948ec1d
--- /dev/null
+++ b/tests/tcg/mips/mips32-dsp/wrdsp.c
@@ -0,0 +1,54 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int dsp_i, dsp_o;
+    int ccond_i, outflag_i, efi_i, c_i, scount_i, pos_i;
+    int ccond_o, outflag_o, efi_o, c_o, scount_o, pos_o;
+    int ccond_r, outflag_r, efi_r, c_r, scount_r, pos_r;
+
+    ccond_i   = 0x000000BC;/* 4 */
+    outflag_i = 0x0000001B;/* 3 */
+    efi_i     = 0x00000001;/* 5 */
+    c_i       = 0x00000001;/* 2 */
+    scount_i  = 0x0000000F;/* 1 */
+    pos_i     = 0x0000000C;/* 0 */
+
+    dsp_i = (ccond_i   << 24) | \
+            (outflag_i << 16) | \
+            (efi_i     << 14) | \
+            (c_i       << 13) | \
+            (scount_i  <<  7) | \
+            pos_i;
+
+    ccond_r   = ccond_i;
+    outflag_r = outflag_i;
+    efi_r     = efi_i;
+    c_r       = c_i;
+    scount_r  = scount_i;
+    pos_r     = pos_i;
+
+    __asm
+        ("wrdsp %1, 0x3F\n\t"
+         "rddsp %0, 0x3F\n\t"
+         : "=r"(dsp_o)
+         : "r"(dsp_i)
+        );
+
+    ccond_o   = (dsp_o >> 24) & 0xFF;
+    outflag_o = (dsp_o >> 16) & 0xFF;
+    efi_o     = (dsp_o >> 14) & 0x01;
+    c_o       = (dsp_o >> 14) & 0x01;
+    scount_o  = (dsp_o >>  7) & 0x3F;
+    pos_o     =  dsp_o & 0x1F;
+
+    assert(ccond_o   == ccond_r);
+    assert(outflag_o == outflag_r);
+    assert(efi_o     == efi_r);
+    assert(c_o       == c_r);
+    assert(scount_o  == scount_r);
+    assert(pos_o     == pos_r);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/Makefile b/tests/tcg/mips/mips32-dspr2/Makefile
new file mode 100644
index 000000000..ed19581c7
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/Makefile
@@ -0,0 +1,71 @@
+-include ../../config-host.mak
+
+CROSS=mips64el-unknown-linux-gnu-
+
+SIM=qemu-mipsel
+SIM_FLAGS=-cpu 74Kf
+
+CC      = $(CROSS)gcc
+CFLAGS  = -mabi=32 -march=mips32r2 -mgp32 -mdspr2 -static
+
+TESTCASES = absq_s_qb.tst
+TESTCASES += addqh_ph.tst
+TESTCASES += addqh_r_ph.tst
+TESTCASES += addqh_r_w.tst
+TESTCASES += addqh_w.tst
+TESTCASES += adduh_qb.tst
+TESTCASES += adduh_r_qb.tst
+TESTCASES += addu_ph.tst
+TESTCASES += addu_s_ph.tst
+TESTCASES += append.tst
+TESTCASES += balign.tst
+TESTCASES += cmpgdu_eq_qb.tst
+TESTCASES += cmpgdu_le_qb.tst
+TESTCASES += cmpgdu_lt_qb.tst
+TESTCASES += dpaqx_sa_w_ph.tst
+TESTCASES += dpa_w_ph.tst
+TESTCASES += dpax_w_ph.tst
+TESTCASES += dpaqx_s_w_ph.tst
+TESTCASES += dpsqx_sa_w_ph.tst
+TESTCASES += dpsqx_s_w_ph.tst
+TESTCASES += dps_w_ph.tst
+TESTCASES += dpsx_w_ph.tst
+TESTCASES += mul_ph.tst
+TESTCASES += mulq_rs_w.tst
+TESTCASES += mulq_s_ph.tst
+TESTCASES += mulq_s_w.tst
+TESTCASES += mulsaq_s_w_ph.tst
+TESTCASES += mulsa_w_ph.tst
+TESTCASES += mul_s_ph.tst
+TESTCASES += precr_qb_ph.tst
+TESTCASES += precr_sra_ph_w.tst
+TESTCASES += precr_sra_r_ph_w.tst
+TESTCASES += prepend.tst
+TESTCASES += shra_qb.tst
+TESTCASES += shra_r_qb.tst
+TESTCASES += shrav_qb.tst
+TESTCASES += shrav_r_qb.tst
+TESTCASES += shrl_ph.tst
+TESTCASES += shrlv_ph.tst
+TESTCASES += subqh_ph.tst
+TESTCASES += subqh_r_ph.tst
+TESTCASES += subqh_r_w.tst
+TESTCASES += subqh_w.tst
+TESTCASES += subuh_qb.tst
+TESTCASES += subuh_r_qb.tst
+TESTCASES += subu_ph.tst
+TESTCASES += subu_s_ph.tst
+
+all: $(TESTCASES)
+
+%.tst: %.c
+	$(CC) $(CFLAGS) $< -o $@
+
+check: $(TESTCASES)
+	@for case in $(TESTCASES); do \
+        echo $(SIM) $(SIM_FLAGS) ./$$case;\
+		$(SIM) $(SIM_FLAGS) ./$$case; \
+	done
+
+clean:
+	$(RM) -rf $(TESTCASES)
diff --git a/tests/tcg/mips/mips32-dspr2/absq_s_qb.c b/tests/tcg/mips/mips32-dspr2/absq_s_qb.c
new file mode 100644
index 000000000..af4683f30
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/absq_s_qb.c
@@ -0,0 +1,35 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int input, result, dsp;
+    int hope;
+
+    input = 0x701BA35E;
+    hope  = 0x701B5D5E;
+
+    __asm
+        ("absq_s.qb %0, %1\n\t"
+         : "=r"(result)
+         : "r"(input)
+        );
+    assert(result == hope);
+
+
+    input = 0x801BA35E;
+    hope  = 0x7F1B5D5E;
+
+    __asm
+        ("absq_s.qb %0, %2\n\t"
+         "rddsp %1\n\t"
+         : "=r"(result), "=r"(dsp)
+         : "r"(input)
+        );
+    dsp = dsp >> 20;
+    dsp &= 0x01;
+    assert(dsp == 1);
+    assert(result == hope);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/addqh_ph.c b/tests/tcg/mips/mips32-dspr2/addqh_ph.c
new file mode 100644
index 000000000..921f0eaf3
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/addqh_ph.c
@@ -0,0 +1,30 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int result;
+
+    rs     = 0x706A13FE;
+    rt     = 0x13065174;
+    result = 0x41B832B9;
+    __asm
+        ("addqh.ph %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    assert(rd == result);
+
+    rs     = 0x81000100;
+    rt     = 0xc2000100;
+    result = 0xa1800100;
+    __asm
+        ("addqh.ph %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/addqh_r_ph.c b/tests/tcg/mips/mips32-dspr2/addqh_r_ph.c
new file mode 100644
index 000000000..213ba3725
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/addqh_r_ph.c
@@ -0,0 +1,30 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int result;
+
+    rs     = 0x706A13FE;
+    rt     = 0x13065174;
+    result = 0x41B832B9;
+    __asm
+        ("addqh_r.ph %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    assert(rd == result);
+
+    rs     = 0x81010100;
+    rt     = 0xc2000100;
+    result = 0xa1810100;
+    __asm
+        ("addqh_r.ph %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/addqh_r_w.c b/tests/tcg/mips/mips32-dspr2/addqh_r_w.c
new file mode 100644
index 000000000..75a75c50f
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/addqh_r_w.c
@@ -0,0 +1,34 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int result;
+
+    rs     = 0x00000010;
+    rt     = 0x00000001;
+    result = 0x00000009;
+
+    __asm
+        ("addqh_r.w  %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+
+    assert(rd == result);
+
+    rs     = 0xFFFFFFFE;
+    rt     = 0x00000001;
+    result = 0x00000000;
+
+    __asm
+        ("addqh_r.w  %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/addqh_w.c b/tests/tcg/mips/mips32-dspr2/addqh_w.c
new file mode 100644
index 000000000..de6926ebb
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/addqh_w.c
@@ -0,0 +1,34 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int result;
+
+    rs     = 0x00000010;
+    rt     = 0x00000001;
+    result = 0x00000008;
+
+    __asm
+        ("addqh.w  %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+
+    assert(rd == result);
+
+    rs     = 0xFFFFFFFE;
+    rt     = 0x00000001;
+    result = 0xFFFFFFFF;
+
+    __asm
+        ("addqh.w  %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/addu_ph.c b/tests/tcg/mips/mips32-dspr2/addu_ph.c
new file mode 100644
index 000000000..1d7a25a2a
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/addu_ph.c
@@ -0,0 +1,33 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int dsp;
+    int result;
+
+    rs     = 0x00FF00FF;
+    rt     = 0x00010001;
+    result = 0x01000100;
+    __asm
+        ("addu.ph %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    assert(rd == result);
+
+    rs     = 0xFFFF1111;
+    rt     = 0x00020001;
+    result = 0x00011112;
+    __asm
+        ("addu.ph %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    assert(rd == result);
+    assert(((dsp >> 20) & 0x01) == 1);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/addu_s_ph.c b/tests/tcg/mips/mips32-dspr2/addu_s_ph.c
new file mode 100644
index 000000000..979651bfc
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/addu_s_ph.c
@@ -0,0 +1,33 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int dsp;
+    int result;
+
+    rs     = 0x00FE00FE;
+    rt     = 0x00020001;
+    result = 0x010000FF;
+    __asm
+        ("addu_s.ph %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    assert(rd == result);
+
+    rs     = 0xFFFF1111;
+    rt     = 0x00020001;
+    result = 0xFFFF1112;
+    __asm
+        ("addu_s.ph %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    assert(rd == result);
+    assert(((dsp >> 20) & 0x01) == 1);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/adduh_qb.c b/tests/tcg/mips/mips32-dspr2/adduh_qb.c
new file mode 100644
index 000000000..a1f5d631b
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/adduh_qb.c
@@ -0,0 +1,30 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int result;
+
+    rs     = 0xFF0055AA;
+    rt     = 0x0113421B;
+    result = 0x80094B62;
+    __asm
+        ("adduh.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    assert(rd == result);
+
+    rs     = 0xFFFF0FFF;
+    rt     = 0x00010111;
+    result = 0x7F800888;
+    __asm
+        ("adduh.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/adduh_r_qb.c b/tests/tcg/mips/mips32-dspr2/adduh_r_qb.c
new file mode 100644
index 000000000..81e98c190
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/adduh_r_qb.c
@@ -0,0 +1,30 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int result;
+
+    rs     = 0xFF0055AA;
+    rt     = 0x01112211;
+    result = 0x80093C5E;
+    __asm
+        ("adduh_r.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    assert(rd == result);
+
+    rs     = 0xFFFF0FFF;
+    rt     = 0x00010111;
+    result = 0x80800888;
+    __asm
+        ("adduh_r.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/append.c b/tests/tcg/mips/mips32-dspr2/append.c
new file mode 100644
index 000000000..9a91e1650
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/append.c
@@ -0,0 +1,30 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rs, rt;
+    int result;
+
+    rs     = 0xFF0055AA;
+    rt     = 0x0113421B;
+    result = 0x02268436;
+    __asm
+        ("append %0, %1, 0x01\n\t"
+         : "+r"(rt)
+         : "r"(rs)
+        );
+    assert(rt == result);
+
+    rs     = 0xFFFF0FFF;
+    rt     = 0x00010111;
+    result = 0x0010111F;
+    __asm
+        ("append %0, %1, 0x04\n\t"
+         : "+r"(rt)
+         : "r"(rs)
+        );
+    assert(rt == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/balign.c b/tests/tcg/mips/mips32-dspr2/balign.c
new file mode 100644
index 000000000..537cf0451
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/balign.c
@@ -0,0 +1,30 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rs, rt;
+    int result;
+
+    rs     = 0xFF0055AA;
+    rt     = 0x0113421B;
+    result = 0x13421BFF;
+    __asm
+        ("balign %0, %1, 0x01\n\t"
+         : "+r"(rt)
+         : "r"(rs)
+        );
+    assert(rt == result);
+
+    rs     = 0xFFFF0FFF;
+    rt     = 0x00010111;
+    result = 0x11FFFF0F;
+    __asm
+        ("balign %0, %1, 0x03\n\t"
+         : "+r"(rt)
+         : "r"(rs)
+        );
+    assert(rt == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/cmpgdu_eq_qb.c b/tests/tcg/mips/mips32-dspr2/cmpgdu_eq_qb.c
new file mode 100644
index 000000000..2d6340d6f
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/cmpgdu_eq_qb.c
@@ -0,0 +1,37 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int dsp;
+    int result;
+
+    rs = 0x11777066;
+    rt = 0x55AA70FF;
+    result = 0x02;
+    __asm
+        ("cmpgdu.eq.qb %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 24) & 0x0F;
+    assert(rd  == result);
+    assert(dsp == result);
+
+    rs     = 0x11777066;
+    rt     = 0x11777066;
+    result = 0x0F;
+    __asm
+        ("cmpgdu.eq.qb %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 24) & 0x0F;
+    assert(rd  == result);
+    assert(dsp == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/cmpgdu_le_qb.c b/tests/tcg/mips/mips32-dspr2/cmpgdu_le_qb.c
new file mode 100644
index 000000000..a0ecdca2a
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/cmpgdu_le_qb.c
@@ -0,0 +1,37 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int dsp;
+    int result;
+
+    rs         = 0x11777066;
+    rt         = 0x55AA70FF;
+    result     = 0x0F;
+    __asm
+        ("cmpgdu.le.qb %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 24) & 0x0F;
+    assert(rd  == result);
+    assert(dsp == result);
+
+    rs     = 0x11777066;
+    rt     = 0x11707066;
+    result = 0x0B;
+    __asm
+        ("cmpgdu.le.qb %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 24) & 0x0F;
+    assert(rd  == result);
+    assert(dsp == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/cmpgdu_lt_qb.c b/tests/tcg/mips/mips32-dspr2/cmpgdu_lt_qb.c
new file mode 100644
index 000000000..dba99e392
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/cmpgdu_lt_qb.c
@@ -0,0 +1,37 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int dsp;
+    int result;
+
+    rs         = 0x11777066;
+    rt         = 0x55AA70FF;
+    result     = 0x0D;
+    __asm
+        ("cmpgdu.lt.qb %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 24) & 0x0F;
+    assert(rd  == result);
+    assert(dsp == result);
+
+    rs     = 0x11777066;
+    rt     = 0x11777066;
+    result = 0x00;
+    __asm
+        ("cmpgdu.lt.qb %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 24) & 0x0F;
+    assert(rd  == result);
+    assert(dsp == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/dpa_w_ph.c b/tests/tcg/mips/mips32-dspr2/dpa_w_ph.c
new file mode 100644
index 000000000..1cfbdb080
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/dpa_w_ph.c
@@ -0,0 +1,44 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rs, rt;
+    int ach = 5, acl = 5;
+    int resulth, resultl;
+
+    rs     = 0x00FF00FF;
+    rt     = 0x00010002;
+    resulth = 0x05;
+    resultl = 0x0302;
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpa.w.ph $ac1, %2, %3\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         : "+r"(ach), "+r"(acl)
+         : "r"(rs), "r"(rt)
+        );
+    assert(ach == resulth);
+    assert(acl == resultl);
+
+    ach = 6, acl = 7;
+    rs     = 0xFFFF00FF;
+    rt     = 0xFFFF0002;
+    resulth = 0x05;
+    resultl = 0xfffe0206;
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpa.w.ph $ac1, %2, %3\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         : "+r"(ach), "+r"(acl)
+         : "r"(rs), "r"(rt)
+        );
+    assert(ach == resulth);
+    assert(acl == resultl);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/dpaqx_s_w_ph.c b/tests/tcg/mips/mips32-dspr2/dpaqx_s_w_ph.c
new file mode 100644
index 000000000..ce8783024
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/dpaqx_s_w_ph.c
@@ -0,0 +1,79 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rs, rt, dsp;
+    int ach = 5, acl = 5;
+    int resulth, resultl, resultdsp;
+
+    rs     = 0x800000FF;
+    rt     = 0x00018000;
+    resulth = 0x05;
+    resultl = 0x80000202;
+    resultdsp = 0x01;
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpaqx_s.w.ph $ac1, %3, %4\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "+r"(ach), "+r"(acl), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 17) & 0x01;
+    assert(dsp == resultdsp);
+    assert(ach == resulth);
+    assert(acl == resultl);
+
+    ach    = 5;
+    acl    = 5;
+    rs     = 0x00FF00FF;
+    rt     = 0x00010002;
+    resulth = 0x05;
+    resultl = 0x05FF;
+    /***********************************************************
+     * Because of we set outflag at last time, although this
+     * time we set nothing, but it is stay the last time value.
+     **********************************************************/
+    resultdsp = 0x01;
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpaqx_s.w.ph $ac1, %3, %4\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "+r"(ach), "+r"(acl), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 17) & 0x01;
+    assert(dsp == resultdsp);
+    assert(ach == resulth);
+    assert(acl == resultl);
+
+    ach    = 5;
+    acl    = 5;
+    rs     = 0x800000FF;
+    rt     = 0x00028000;
+    resulth = 0x05;
+    resultl = 0x80000400;
+    resultdsp = 0x01;
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpaqx_s.w.ph $ac1, %3, %4\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "+r"(ach), "+r"(acl), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 17) & 0x01;
+    assert(dsp == resultdsp);
+    assert(ach == resulth);
+    assert(acl == resultl);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/dpaqx_sa_w_ph.c b/tests/tcg/mips/mips32-dspr2/dpaqx_sa_w_ph.c
new file mode 100644
index 000000000..798c4da5c
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/dpaqx_sa_w_ph.c
@@ -0,0 +1,53 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rs, rt, dsp;
+    int ach = 5, acl = 5;
+    int resulth, resultl, resultdsp;
+
+    rs     = 0x00FF00FF;
+    rt     = 0x00010002;
+    resulth = 0x00;
+    resultl = 0x7FFFFFFF;
+    resultdsp = 0x01;
+    __asm
+        ("wrdsp %2\n\t"
+         "mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpaqx_sa.w.ph $ac1, %3, %4\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "+r"(ach), "+r"(acl), "+r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    assert(dsp >> (16 + 1) == resultdsp);
+    assert(ach == resulth);
+    assert(acl == resultl);
+
+    ach = 9;
+    acl = 0xb;
+    rs     = 0x800000FF;
+    rt     = 0x00018000;
+    resulth = 0x00;
+    resultl = 0x7fffffff;
+    resultdsp = 0x01;
+    __asm
+        ("wrdsp %2\n\t"
+         "mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpaqx_sa.w.ph $ac1, %3, %4\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "+r"(ach), "+r"(acl), "+r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    assert(dsp >> (16 + 1) == resultdsp);
+    assert(ach == resulth);
+    assert(acl == resultl);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/dpax_w_ph.c b/tests/tcg/mips/mips32-dspr2/dpax_w_ph.c
new file mode 100644
index 000000000..f75699755
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/dpax_w_ph.c
@@ -0,0 +1,27 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rs, rt;
+    int ach = 5, acl = 5;
+    int resulth, resultl;
+
+    rs     = 0x00FF00FF;
+    rt     = 0x00010002;
+    resulth = 0x05;
+    resultl = 0x0302;
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpax.w.ph $ac1, %2, %3\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         : "+r"(ach), "+r"(acl)
+         : "r"(rs), "r"(rt)
+        );
+    assert(ach == resulth);
+    assert(acl == resultl);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/dps_w_ph.c b/tests/tcg/mips/mips32-dspr2/dps_w_ph.c
new file mode 100644
index 000000000..8303643d1
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/dps_w_ph.c
@@ -0,0 +1,27 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rs, rt;
+    int ach = 5, acl = 5;
+    int resulth, resultl;
+
+    rs     = 0x00FF00FF;
+    rt     = 0x00010002;
+    resulth = 0x04;
+    resultl = 0xFFFFFD08;
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dps.w.ph $ac1, %2, %3\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         : "+r"(ach), "+r"(acl)
+         : "r"(rs), "r"(rt)
+        );
+    assert(ach == resulth);
+    assert(acl == resultl);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/dpsqx_s_w_ph.c b/tests/tcg/mips/mips32-dspr2/dpsqx_s_w_ph.c
new file mode 100644
index 000000000..14cdd7c0f
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/dpsqx_s_w_ph.c
@@ -0,0 +1,54 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rs, rt, dsp;
+    int ach = 5, acl = 5;
+    int resulth, resultl, resultdsp;
+
+    rs      = 0xBC0123AD;
+    rt      = 0x01643721;
+    resulth = 0x04;
+    resultl = 0xAEA3E09B;
+    resultdsp = 0x00;
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpsqx_s.w.ph $ac1, %3, %4\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "+r"(ach), "+r"(acl), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 17) & 0x01;
+    assert(dsp == resultdsp);
+    assert(ach == resulth);
+    assert(acl == resultl);
+
+    ach = 0x99f13005;
+    acl = 0x51730062;
+    rs      = 0x80008000;
+    rt      = 0x80008000;
+
+    resulth = 0x99f13004;
+    resultl = 0x51730064;
+    resultdsp = 0x01;
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpsqx_s.w.ph $ac1, %3, %4\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "+r"(ach), "+r"(acl), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 17) & 0x01;
+    assert(dsp == resultdsp);
+    assert(ach == resulth);
+    assert(acl == resultl);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/dpsqx_sa_w_ph.c b/tests/tcg/mips/mips32-dspr2/dpsqx_sa_w_ph.c
new file mode 100644
index 000000000..7da278eac
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/dpsqx_sa_w_ph.c
@@ -0,0 +1,53 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rs, rt, dsp;
+    int ach = 5, acl = 5;
+    int resulth, resultl, resultdsp;
+
+    rs = 0xBC0123AD;
+    rt = 0x01643721;
+    resulth = 0x00;
+    resultl = 0x7FFFFFFF;
+    resultdsp = 0x01;
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpsqx_sa.w.ph $ac1, %3, %4\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "+r"(ach), "+r"(acl), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 17) & 0x01;
+    assert(dsp == resultdsp);
+    assert(ach == resulth);
+    assert(acl == resultl);
+
+    ach = 0x8c0b354A;
+    acl = 0xbbc02249;
+    rs = 0x800023AD;
+    rt = 0x01648000;
+    resulth = 0xffffffff;
+    resultl = 0x80000000;
+    resultdsp = 0x01;
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpsqx_sa.w.ph $ac1, %3, %4\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "+r"(ach), "+r"(acl), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 17) & 0x01;
+    assert(dsp == resultdsp);
+    assert(ach == resulth);
+    assert(acl == resultl);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/dpsx_w_ph.c b/tests/tcg/mips/mips32-dspr2/dpsx_w_ph.c
new file mode 100644
index 000000000..6db59a4cc
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/dpsx_w_ph.c
@@ -0,0 +1,27 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rs, rt;
+    int ach = 5, acl = 5;
+    int resulth, resultl;
+
+    rs      = 0xBC0123AD;
+    rt      = 0x01643721;
+    resulth = 0x04;
+    resultl = 0xD751F050;
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpsx.w.ph $ac1, %2, %3\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         : "+r"(ach), "+r"(acl)
+         : "r"(rs), "r"(rt)
+        );
+    assert(ach == resulth);
+    assert(acl == resultl);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/mul_ph.c b/tests/tcg/mips/mips32-dspr2/mul_ph.c
new file mode 100644
index 000000000..c7e9d60d1
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/mul_ph.c
@@ -0,0 +1,47 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt, dsp;
+    int result, resultdsp;
+
+    rs = 0x03FB1234;
+    rt = 0x0BCC4321;
+    result = 0xF504F4B4;
+    resultdsp = 1;
+
+    __asm
+        ("mul.ph %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 21) & 0x01;
+    assert(rd  == result);
+    assert(dsp == resultdsp);
+
+    dsp = 0;
+    __asm
+        ("wrdsp %0\n\t"
+         :
+         : "r"(dsp)
+        );
+
+    rs = 0x00210010;
+    rt = 0x00110005;
+    result = 0x2310050;
+    resultdsp = 0;
+
+    __asm
+        ("mul.ph %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 21) & 0x01;
+    assert(rd  == result);
+    assert(dsp == resultdsp);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/mul_s_ph.c b/tests/tcg/mips/mips32-dspr2/mul_s_ph.c
new file mode 100644
index 000000000..33da110de
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/mul_s_ph.c
@@ -0,0 +1,62 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt, dsp;
+    int result, resultdsp;
+
+    rs = 0x03FB1234;
+    rt = 0x0BCC4321;
+    result = 0x7fff7FFF;
+    resultdsp = 1;
+
+    __asm
+        ("mul_s.ph %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 21) & 0x01;
+    assert(rd  == result);
+    assert(dsp == resultdsp);
+
+    rs = 0x7fffff00;
+    rt = 0xff007fff;
+    result = 0x80008000;
+    resultdsp = 1;
+
+    __asm
+        ("mul_s.ph %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 21) & 0x01;
+    assert(rd  == result);
+    assert(dsp == resultdsp);
+
+    dsp = 0;
+    __asm
+        ("wrdsp %0\n\t"
+         :
+         : "r"(dsp)
+        );
+
+    rs = 0x00320001;
+    rt = 0x00210002;
+    result = 0x06720002;
+    resultdsp = 0;
+
+    __asm
+        ("mul_s.ph %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 21) & 0x01;
+    assert(rd  == result);
+    assert(dsp == resultdsp);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/mulq_rs_w.c b/tests/tcg/mips/mips32-dspr2/mulq_rs_w.c
new file mode 100644
index 000000000..669405faf
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/mulq_rs_w.c
@@ -0,0 +1,36 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt, dsp;
+    int result, resultdsp;
+
+    rs = 0x80001234;
+    rt = 0x80004321;
+    result = 0x80005555;
+
+    __asm
+        ("mulq_rs.w %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    assert(rd  == result);
+
+    rs = 0x80000000;
+    rt = 0x80000000;
+    result = 0x7FFFFFFF;
+    resultdsp = 1;
+
+    __asm
+        ("mulq_rs.w %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 21) & 0x01;
+    assert(rd  == result);
+    assert(dsp == resultdsp);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/mulq_s_ph.c b/tests/tcg/mips/mips32-dspr2/mulq_s_ph.c
new file mode 100644
index 000000000..d0f7674a3
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/mulq_s_ph.c
@@ -0,0 +1,25 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt, dsp;
+    int result, resultdsp;
+
+    rs = 0x80001234;
+    rt = 0x80004321;
+    result = 0x7FFF098B;
+    resultdsp = 1;
+
+    __asm
+        ("mulq_s.ph %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 21) & 0x01;
+    assert(rd  == result);
+    assert(dsp == resultdsp);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/mulq_s_w.c b/tests/tcg/mips/mips32-dspr2/mulq_s_w.c
new file mode 100644
index 000000000..df148b7ff
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/mulq_s_w.c
@@ -0,0 +1,36 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt, dsp;
+    int result, resultdsp;
+
+    rs = 0x80001234;
+    rt = 0x80004321;
+    result = 0x80005555;
+
+    __asm
+        ("mulq_s.w %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    assert(rd  == result);
+
+    rs = 0x80000000;
+    rt = 0x80000000;
+    result = 0x7FFFFFFF;
+    resultdsp = 1;
+
+    __asm
+        ("mulq_s.w %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 21) & 0x01;
+    assert(rd  == result);
+    assert(dsp == resultdsp);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/mulsa_w_ph.c b/tests/tcg/mips/mips32-dspr2/mulsa_w_ph.c
new file mode 100644
index 000000000..a6940939c
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/mulsa_w_ph.c
@@ -0,0 +1,29 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rs, rt, ach, acl;
+    int resulth, resultl;
+
+    ach = 0x05;
+    acl = 0x00BBDDCC;
+    rs = 0x80001234;
+    rt = 0x80004321;
+    resulth = 0x05;
+    resultl = 0x3BF5E918;
+
+    __asm
+        ("mthi %0, $ac1\n\t"
+         "mtlo %1, $ac1\n\t"
+         "mulsa.w.ph $ac1, %2, %3\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "+r"(ach), "+r"(acl)
+         : "r"(rs), "r"(rt)
+        );
+    assert(ach == resulth);
+    assert(acl == resultl);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/mulsaq_s_w_ph.c b/tests/tcg/mips/mips32-dspr2/mulsaq_s_w_ph.c
new file mode 100644
index 000000000..06c91a43e
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/mulsaq_s_w_ph.c
@@ -0,0 +1,29 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rs, rt, ach, acl;
+    int resulth, resultl;
+
+    ach = 0x05;
+    acl = 0x00BBDDCC;
+    rs = 0x80001234;
+    rt = 0x80004321;
+    resulth = 0x05;
+    resultl = 0x772ff463;
+
+    __asm
+        ("mthi %0, $ac1\n\t"
+         "mtlo %1, $ac1\n\t"
+         "mulsaq_s.w.ph $ac1, %2, %3\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "+r"(ach), "+r"(acl)
+         : "r"(rs), "r"(rt)
+        );
+    assert(ach == resulth);
+    assert(acl == resultl);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/precr_qb_ph.c b/tests/tcg/mips/mips32-dspr2/precr_qb_ph.c
new file mode 100644
index 000000000..3a2b3fde0
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/precr_qb_ph.c
@@ -0,0 +1,21 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int result;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result = 0x34786521;
+
+    __asm
+        ("precr.qb.ph %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    assert(result == rd);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/precr_sra_ph_w.c b/tests/tcg/mips/mips32-dspr2/precr_sra_ph_w.c
new file mode 100644
index 000000000..5c9baab03
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/precr_sra_ph_w.c
@@ -0,0 +1,32 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rs, rt;
+    int result;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result = 0x43215678;
+
+    __asm
+        ("precr_sra.ph.w %0, %1, 0x00\n\t"
+         : "+r"(rt)
+         : "r"(rs)
+        );
+    assert(result == rt);
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result = 0xFFFF0000;
+
+    __asm
+        ("precr_sra.ph.w %0, %1, 0x1F\n\t"
+         : "+r"(rt)
+         : "r"(rs)
+        );
+    assert(result == rt);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/precr_sra_r_ph_w.c b/tests/tcg/mips/mips32-dspr2/precr_sra_r_ph_w.c
new file mode 100644
index 000000000..6474a108c
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/precr_sra_r_ph_w.c
@@ -0,0 +1,32 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rs, rt;
+    int result;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result = 0x43215678;
+
+    __asm
+        ("precr_sra_r.ph.w %0, %1, 0x00\n\t"
+         : "+r"(rt)
+         : "r"(rs)
+        );
+    assert(result == rt);
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result = 0xFFFF0000;
+
+    __asm
+        ("precr_sra_r.ph.w %0, %1, 0x1F\n\t"
+         : "+r"(rt)
+         : "r"(rs)
+        );
+    assert(result == rt);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/prepend.c b/tests/tcg/mips/mips32-dspr2/prepend.c
new file mode 100644
index 000000000..f6bcd47b2
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/prepend.c
@@ -0,0 +1,30 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rs, rt;
+    int result;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result = 0x87654321;
+    __asm
+        ("prepend %0, %1, 0x00\n\t"
+         : "+r"(rt)
+         : "r"(rs)
+        );
+    assert(rt == result);
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result = 0xACF10ECA;
+    __asm
+        ("prepend %0, %1, 0x0F\n\t"
+         : "+r"(rt)
+         : "r"(rs)
+        );
+    assert(rt == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/shra_qb.c b/tests/tcg/mips/mips32-dspr2/shra_qb.c
new file mode 100644
index 000000000..48193de87
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/shra_qb.c
@@ -0,0 +1,30 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rt;
+    int result;
+
+    rt = 0x12345678;
+    result = 0x02060A0F;
+
+    __asm
+        ("shra.qb %0, %1, 0x03\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    assert(rd == result);
+
+    rt = 0x87654321;
+    result = 0xF00C0804;
+
+    __asm
+        ("shra.qb %0, %1, 0x03\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/shra_r_qb.c b/tests/tcg/mips/mips32-dspr2/shra_r_qb.c
new file mode 100644
index 000000000..29afa0e4b
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/shra_r_qb.c
@@ -0,0 +1,30 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rt;
+    int result;
+
+    rt = 0x12345678;
+    result = 0x02070B0F;
+
+    __asm
+        ("shra_r.qb %0, %1, 0x03\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    assert(rd == result);
+
+    rt = 0x87654321;
+    result = 0xF10D0804;
+
+    __asm
+        ("shra_r.qb %0, %1, 0x03\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/shrav_qb.c b/tests/tcg/mips/mips32-dspr2/shrav_qb.c
new file mode 100644
index 000000000..b21e1b7ca
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/shrav_qb.c
@@ -0,0 +1,32 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int result;
+
+    rs = 0x03;
+    rt = 0x12345678;
+    result = 0x02060A0F;
+
+    __asm
+        ("shrav.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rt), "r"(rs)
+        );
+    assert(rd == result);
+
+    rs = 0x03;
+    rt = 0x87654321;
+    result = 0xF00C0804;
+
+    __asm
+        ("shrav.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rt), "r"(rs)
+        );
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/shrav_r_qb.c b/tests/tcg/mips/mips32-dspr2/shrav_r_qb.c
new file mode 100644
index 000000000..9ea8aa0cb
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/shrav_r_qb.c
@@ -0,0 +1,32 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int result;
+
+    rs = 0x03;
+    rt = 0x12345678;
+    result = 0x02070B0F;
+
+    __asm
+        ("shrav_r.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rt), "r"(rs)
+        );
+    assert(rd == result);
+
+    rs = 0x03;
+    rt = 0x87654321;
+    result = 0xF10D0804;
+
+    __asm
+        ("shrav_r.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rt), "r"(rs)
+        );
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/shrl_ph.c b/tests/tcg/mips/mips32-dspr2/shrl_ph.c
new file mode 100644
index 000000000..724b9a7a4
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/shrl_ph.c
@@ -0,0 +1,20 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rt;
+    int result;
+
+    rt     = 0x12345678;
+    result = 0x009102B3;
+
+    __asm
+        ("shrl.ph %0, %1, 0x05\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/shrlv_ph.c b/tests/tcg/mips/mips32-dspr2/shrlv_ph.c
new file mode 100644
index 000000000..ac79aa69a
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/shrlv_ph.c
@@ -0,0 +1,21 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int result;
+
+    rs     = 0x05;
+    rt     = 0x12345678;
+    result = 0x009102B3;
+
+    __asm
+        ("shrlv.ph %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rt), "r"(rs)
+        );
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/subqh_ph.c b/tests/tcg/mips/mips32-dspr2/subqh_ph.c
new file mode 100644
index 000000000..dbc096734
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/subqh_ph.c
@@ -0,0 +1,21 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int result;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result = 0x456709AB;
+
+    __asm
+        ("subqh.ph %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/subqh_r_ph.c b/tests/tcg/mips/mips32-dspr2/subqh_r_ph.c
new file mode 100644
index 000000000..24ef0f1ae
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/subqh_r_ph.c
@@ -0,0 +1,21 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int result;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result = 0x456809AC;
+
+    __asm
+        ("subqh_r.ph %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/subqh_r_w.c b/tests/tcg/mips/mips32-dspr2/subqh_r_w.c
new file mode 100644
index 000000000..d460f8630
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/subqh_r_w.c
@@ -0,0 +1,21 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int result;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result = 0x456789AC;
+
+    __asm
+        ("subqh_r.w %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/subqh_w.c b/tests/tcg/mips/mips32-dspr2/subqh_w.c
new file mode 100644
index 000000000..42be3deb8
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/subqh_w.c
@@ -0,0 +1,21 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int result;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result = 0x456789AB;
+
+    __asm
+        ("subqh.w %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/subu_ph.c b/tests/tcg/mips/mips32-dspr2/subu_ph.c
new file mode 100644
index 000000000..0d39a017c
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/subu_ph.c
@@ -0,0 +1,40 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt, dsp;
+    int result, resultdsp;
+
+    rs = 0x87654321;
+    rt = 0x11111111;
+    result    = 0x76543210;
+    resultdsp = 0x00;
+
+    __asm
+        ("subu.ph %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 20) & 0x01;
+    assert(dsp == resultdsp);
+    assert(rd  == result);
+
+    rs = 0x87654321;
+    rt = 0x12345678;
+    result    = 0x7531ECA9;
+    resultdsp = 0x01;
+
+    __asm
+        ("subu.ph %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 20) & 0x01;
+    assert(dsp == resultdsp);
+    assert(rd  == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/subu_s_ph.c b/tests/tcg/mips/mips32-dspr2/subu_s_ph.c
new file mode 100644
index 000000000..8e4da4f3e
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/subu_s_ph.c
@@ -0,0 +1,25 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt, dsp;
+    int result, resultdsp;
+
+    rs = 0x87654321;
+    rt = 0x12345678;
+    result    = 0x75310000;
+    resultdsp = 0x01;
+
+    __asm
+        ("subu_s.ph %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 20) & 0x01;
+    assert(dsp == resultdsp);
+    assert(rd  == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/subuh_qb.c b/tests/tcg/mips/mips32-dspr2/subuh_qb.c
new file mode 100644
index 000000000..92cfc764b
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/subuh_qb.c
@@ -0,0 +1,21 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int result;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result = 0xC5E7092B;
+
+    __asm
+        ("subuh.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips32-dspr2/subuh_r_qb.c b/tests/tcg/mips/mips32-dspr2/subuh_r_qb.c
new file mode 100644
index 000000000..dac81d47d
--- /dev/null
+++ b/tests/tcg/mips/mips32-dspr2/subuh_r_qb.c
@@ -0,0 +1,32 @@
+#include<stdio.h>
+#include<assert.h>
+
+int main()
+{
+    int rd, rs, rt;
+    int result;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result = 0xC6E80A2C;
+
+    __asm
+        ("subuh_r.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    assert(rd == result);
+
+    rs = 0xBEFC292A;
+    rt = 0x9205C1B4;
+    result = 0x167cb4bb;
+
+    __asm
+        ("subuh_r.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    assert(rd == result);
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/Makefile b/tests/tcg/mips/mips64-dsp/Makefile
new file mode 100644
index 000000000..b2ac6b3ff
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/Makefile
@@ -0,0 +1,306 @@
+
+CROSS_COMPILE	?= mips64el-unknown-linux-gnu-
+
+SIM = qemu-system-mips64el
+SIMFLAGS = -nographic -cpu mips64dspr2 -kernel
+
+AS      = $(CROSS_COMPILE)as
+LD      = $(CROSS_COMPILE)ld
+CC      = $(CROSS_COMPILE)gcc
+AR      = $(CROSS_COMPILE)ar
+NM      = $(CROSS_COMPILE)nm
+STRIP       = $(CROSS_COMPILE)strip
+RANLIB      = $(CROSS_COMPILE)ranlib
+OBJCOPY     = $(CROSS_COMPILE)objcopy
+OBJDUMP     = $(CROSS_COMPILE)objdump
+
+VECTORS_OBJ ?= ./head.o ./printf.o
+
+HEAD_FLAGS ?= -nostdinc -mabi=64 -G 0 -mno-abicalls -fno-pic -pipe \
+              -msoft-float -march=mips64 -Wa,-mips64 -Wa,--trap \
+              -msym32 -DKBUILD_64BIT_SYM32 -I./
+
+CFLAGS ?= -nostdinc -mabi=64 -G 0 -mno-abicalls -fno-pic -fno-builtin  \
+          -pipe -march=mips64r2 -mgp64 -mdsp -static -Wa,--trap -msym32 \
+          -DKBUILD_64BIT_SYM32 -I./
+
+LDFLAGS = -T./mips_boot.lds -L./
+FLAGS = -nostdlib -mabi=64 -march=mips64r2 -mgp64 -mdsp
+
+
+#TESTCASES = absq_s_ob.tst
+TESTCASES = absq_s_ph.tst
+TESTCASES += absq_s_pw.tst
+TESTCASES += absq_s_qh.tst
+TESTCASES += absq_s_w.tst
+TESTCASES += addq_ph.tst
+TESTCASES += addq_pw.tst
+TESTCASES += addq_qh.tst
+TESTCASES += addq_s_ph.tst
+TESTCASES += addq_s_pw.tst
+TESTCASES += addq_s_qh.tst
+TESTCASES += addq_s_w.tst
+TESTCASES += addsc.tst
+TESTCASES += addu_ob.tst
+TESTCASES += addu_qb.tst
+TESTCASES += addu_s_ob.tst
+TESTCASES += addu_s_qb.tst
+TESTCASES += addwc.tst
+TESTCASES += bitrev.tst
+TESTCASES += bposge32.tst
+TESTCASES += bposge64.tst
+TESTCASES += cmp_eq_ph.tst
+TESTCASES += cmp_eq_pw.tst
+TESTCASES += cmp_eq_qh.tst
+TESTCASES += cmpgu_eq_ob.tst
+TESTCASES += cmpgu_eq_qb.tst
+TESTCASES += cmpgu_le_ob.tst
+TESTCASES += cmpgu_le_qb.tst
+TESTCASES += cmpgu_lt_ob.tst
+TESTCASES += cmpgu_lt_qb.tst
+TESTCASES += cmp_le_ph.tst
+TESTCASES += cmp_le_pw.tst
+TESTCASES += cmp_le_qh.tst
+TESTCASES += cmp_lt_ph.tst
+TESTCASES += cmp_lt_pw.tst
+TESTCASES += cmp_lt_qh.tst
+TESTCASES += cmpu_eq_ob.tst
+TESTCASES += cmpu_eq_qb.tst
+TESTCASES += cmpu_le_ob.tst
+TESTCASES += cmpu_le_qb.tst
+TESTCASES += cmpu_lt_ob.tst
+TESTCASES += cmpu_lt_qb.tst
+#TESTCASES += dappend.tst
+TESTCASES += dextp.tst
+TESTCASES += dextpdp.tst
+TESTCASES += dextpdpv.tst
+TESTCASES += dextpv.tst
+TESTCASES += dextr_l.tst
+TESTCASES += dextr_r_l.tst
+TESTCASES += dextr_rs_l.tst
+TESTCASES += dextr_rs_w.tst
+TESTCASES += dextr_r_w.tst
+TESTCASES += dextr_s_h.tst
+TESTCASES += dextrv_l.tst
+TESTCASES += dextrv_r_l.tst
+TESTCASES += dextrv_rs_l.tst
+TESTCASES += dextrv_rs_w.tst
+TESTCASES += dextrv_r_w.tst
+TESTCASES += dextrv_s_h.tst
+TESTCASES += dextrv_w.tst
+TESTCASES += dextr_w.tst
+TESTCASES += dinsv.tst
+TESTCASES += dmadd.tst
+TESTCASES += dmaddu.tst
+TESTCASES += dmsub.tst
+TESTCASES += dmsubu.tst
+TESTCASES += dmthlip.tst
+TESTCASES += dpaq_sa_l_pw.tst
+TESTCASES += dpaq_sa_l_w.tst
+TESTCASES += dpaq_s_w_ph.tst
+TESTCASES += dpaq_s_w_qh.tst
+TESTCASES += dpau_h_obl.tst
+TESTCASES += dpau_h_obr.tst
+TESTCASES += dpau_h_qbl.tst
+TESTCASES += dpau_h_qbr.tst
+TESTCASES += dpsq_sa_l_pw.tst
+TESTCASES += dpsq_sa_l_w.tst
+TESTCASES += dpsq_s_w_ph.tst
+TESTCASES += dpsq_s_w_qh.tst
+TESTCASES += dpsu_h_obl.tst
+TESTCASES += dpsu_h_obr.tst
+TESTCASES += dpsu_h_qbl.tst
+TESTCASES += dpsu_h_qbr.tst
+TESTCASES += dshilo.tst
+TESTCASES += dshilov.tst
+TESTCASES += extp.tst
+TESTCASES += extpdp.tst
+TESTCASES += extpdpv.tst
+TESTCASES += extpv.tst
+TESTCASES += extr_rs_w.tst
+TESTCASES += extr_r_w.tst
+TESTCASES += extr_s_h.tst
+TESTCASES += extrv_rs_w.tst
+TESTCASES += extrv_r_w.tst
+TESTCASES += extrv_s_h.tst
+TESTCASES += extrv_w.tst
+TESTCASES += extr_w.tst
+TESTCASES += insv.tst
+TESTCASES += lbux.tst
+TESTCASES += lhx.tst
+TESTCASES += lwx.tst
+TESTCASES += ldx.tst
+TESTCASES += madd.tst
+TESTCASES += maddu.tst
+TESTCASES += maq_sa_w_phl.tst
+TESTCASES += maq_sa_w_phr.tst
+TESTCASES += maq_sa_w_qhll.tst
+TESTCASES += maq_sa_w_qhlr.tst
+TESTCASES += maq_sa_w_qhrl.tst
+TESTCASES += maq_sa_w_qhrr.tst
+TESTCASES += maq_s_l_pwl.tst
+TESTCASES += maq_s_l_pwr.tst
+TESTCASES += maq_s_w_phl.tst
+TESTCASES += maq_s_w_phr.tst
+TESTCASES += maq_s_w_qhll.tst
+TESTCASES += maq_s_w_qhlr.tst
+TESTCASES += maq_s_w_qhrl.tst
+TESTCASES += maq_s_w_qhrr.tst
+TESTCASES += mfhi.tst
+TESTCASES += mflo.tst
+TESTCASES += modsub.tst
+TESTCASES += msub.tst
+TESTCASES += msubu.tst
+TESTCASES += mthi.tst
+TESTCASES += mthlip.tst
+TESTCASES += mtlo.tst
+TESTCASES += muleq_s_pw_qhl.tst
+TESTCASES += muleq_s_pw_qhr.tst
+TESTCASES += muleq_s_w_phl.tst
+TESTCASES += muleq_s_w_phr.tst
+TESTCASES += muleu_s_ph_qbl.tst
+TESTCASES += muleu_s_ph_qbr.tst
+TESTCASES += muleu_s_qh_obl.tst
+TESTCASES += muleu_s_qh_obr.tst
+TESTCASES += mulq_rs_ph.tst
+TESTCASES += mulq_rs_qh.tst
+TESTCASES += mulsaq_s_l_pw.tst
+TESTCASES += mulsaq_s_w_qh.tst
+TESTCASES += mult.tst
+TESTCASES += multu.tst
+TESTCASES += packrl_ph.tst
+TESTCASES += packrl_pw.tst
+TESTCASES += pick_ob.tst
+TESTCASES += pick_ph.tst
+TESTCASES += pick_pw.tst
+TESTCASES += pick_qb.tst
+TESTCASES += pick_qh.tst
+#TESTCASES += preceq_l_pwl.tst
+#TESTCASES += preceq_l_pwr.tst
+TESTCASES += preceq_pw_qhla.tst
+TESTCASES += preceq_pw_qhl.tst
+TESTCASES += preceq_pw_qhra.tst
+TESTCASES += preceq_pw_qhr.tst
+TESTCASES += precequ_ph_qbla.tst
+TESTCASES += precequ_ph_qbl.tst
+TESTCASES += precequ_ph_qbra.tst
+TESTCASES += precequ_ph_qbr.tst
+#TESTCASES += precequ_qh_obla.tst
+#TESTCASES += precequ_qh_obl.tst
+#TESTCASES += precequ_qh_obra.tst
+#TESTCASES += precequ_qh_obr.tst
+TESTCASES += preceq_w_phl.tst
+TESTCASES += preceq_w_phr.tst
+TESTCASES += preceu_ph_qbla.tst
+TESTCASES += preceu_ph_qbl.tst
+TESTCASES += preceu_ph_qbra.tst
+TESTCASES += preceu_ph_qbr.tst
+TESTCASES += preceu_qh_obla.tst
+TESTCASES += preceu_qh_obl.tst
+TESTCASES += preceu_qh_obra.tst
+TESTCASES += preceu_qh_obr.tst
+#TESTCASES += precr_ob_qh.tst
+TESTCASES += precrq_ob_qh.tst
+TESTCASES += precrq_ph_w.tst
+TESTCASES += precrq_pw_l.tst
+TESTCASES += precrq_qb_ph.tst
+TESTCASES += precrq_qh_pw.tst
+TESTCASES += precrq_rs_ph_w.tst
+TESTCASES += precrq_rs_qh_pw.tst
+TESTCASES += precrqu_s_ob_qh.tst
+TESTCASES += precrqu_s_qb_ph.tst
+#TESTCASES += precr_sra_qh_pw.tst
+#TESTCASES += precr_sra_r_qh_pw.tst
+#TESTCASES += prependd.tst
+#TESTCASES += prependw.tst
+#TESTCASES += raddu_l_ob.tst
+TESTCASES += raddu_w_qb.tst
+TESTCASES += rddsp.tst
+TESTCASES += repl_ob.tst
+TESTCASES += repl_ph.tst
+TESTCASES += repl_pw.tst
+TESTCASES += repl_qb.tst
+TESTCASES += repl_qh.tst
+TESTCASES += replv_ob.tst
+TESTCASES += replv_ph.tst
+TESTCASES += replv_pw.tst
+TESTCASES += replv_qb.tst
+TESTCASES += shilo.tst
+TESTCASES += shilov.tst
+TESTCASES += shll_ob.tst
+TESTCASES += shll_ph.tst
+TESTCASES += shll_pw.tst
+TESTCASES += shll_qb.tst
+TESTCASES += shll_qh.tst
+TESTCASES += shll_s_ph.tst
+TESTCASES += shll_s_pw.tst
+TESTCASES += shll_s_qh.tst
+TESTCASES += shll_s_w.tst
+TESTCASES += shllv_ob.tst
+TESTCASES += shllv_ph.tst
+TESTCASES += shllv_pw.tst
+TESTCASES += shllv_qb.tst
+TESTCASES += shllv_qh.tst
+TESTCASES += shllv_s_ph.tst
+TESTCASES += shllv_s_pw.tst
+TESTCASES += shllv_s_qh.tst
+TESTCASES += shllv_s_w.tst
+#TESTCASES += shra_ob.tst
+TESTCASES += shra_ph.tst
+TESTCASES += shra_pw.tst
+TESTCASES += shra_qh.tst
+#TESTCASES += shra_r_ob.tst
+TESTCASES += shra_r_ph.tst
+TESTCASES += shra_r_pw.tst
+TESTCASES += shra_r_qh.tst
+TESTCASES += shra_r_w.tst
+TESTCASES += shrav_ph.tst
+TESTCASES += shrav_pw.tst
+TESTCASES += shrav_qh.tst
+TESTCASES += shrav_r_ph.tst
+TESTCASES += shrav_r_pw.tst
+TESTCASES += shrav_r_qh.tst
+TESTCASES += shrav_r_w.tst
+TESTCASES += shrl_ob.tst
+TESTCASES += shrl_qb.tst
+#TESTCASES += shrl_qh.tst
+TESTCASES += shrlv_ob.tst
+TESTCASES += shrlv_qb.tst
+#TESTCASES += shrlv_qh.tst
+TESTCASES += subq_ph.tst
+TESTCASES += subq_pw.tst
+TESTCASES += subq_qh.tst
+TESTCASES += subq_s_ph.tst
+TESTCASES += subq_s_pw.tst
+TESTCASES += subq_s_qh.tst
+TESTCASES += subq_s_w.tst
+TESTCASES += subu_ob.tst
+TESTCASES += subu_qb.tst
+TESTCASES += subu_s_ob.tst
+TESTCASES += subu_s_qb.tst
+TESTCASES += wrdsp.tst
+
+all: build
+
+head.o : head.S
+	$(Q)$(CC) $(HEAD_FLAGS) -D"STACK_TOP=0xffffffff80200000" -c $< -o $@
+
+%.o  : %.S
+	$(CC) $(CFLAGS) -c $< -o $@
+
+%.o  : %.c
+	$(CC) $(CFLAGS) -c $< -o $@
+
+%.tst: %.o $(VECTORS_OBJ)
+	$(CC) $(VECTORS_OBJ) $(FLAGS) $(LDFLAGS) $< -o $@
+
+build: $(VECTORS_OBJ) $(MIPSSOC_LIB) $(TESTCASES)
+
+check:  $(VECTORS_OBJ) $(MIPSSOC_LIB) $(TESTCASES)
+	@for case in $(TESTCASES); do \
+		echo $(SIM) $(SIMFLAGS) ./$$case; \
+		$(SIM) $(SIMFLAGS) ./$$case & (sleep 1; killall $(SIM)); \
+	done
+
+clean:
+	$(Q)rm -f *.o *.tst *.a
diff --git a/tests/tcg/mips/mips64-dsp/absq_s_ob.c b/tests/tcg/mips/mips64-dsp/absq_s_ob.c
new file mode 100644
index 000000000..621403157
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/absq_s_ob.c
@@ -0,0 +1,63 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt, result, dspcontrol;
+    rt = 0x7F7F7F7F7F7F7F7F;
+    result = 0x7F7F7F7F7F7F7F7F;
+
+
+    __asm
+        (".set mips64\n\t"
+         "absq_s.ob %0 %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+
+    if (result != rd) {
+        printf("absq_s.ob test 1 error\n");
+
+        return -1;
+    }
+
+    __asm
+        ("rddsp %0\n\t"
+         : "=r"(rd)
+        );
+    rd >> 20;
+    rd = rd & 0x1;
+    if (rd != 0) {
+        printf("absq_s.ob test 1 dspcontrol overflow flag error\n");
+
+        return -1;
+    }
+
+    rt = 0x80FFFFFFFFFFFFFF;
+    result = 0x7F01010101010101;
+
+    __asm
+        ("absq_s.ob %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    if (result != rd) {
+        printf("absq_s.ob test 2 error\n");
+
+        return -1;
+    }
+
+    __asm
+        ("rddsp %0\n\t"
+         : "=r"(rd)
+        );
+    rd = rd >> 20;
+    rd = rd & 0x1;
+    if (rd != 1) {
+        printf("absq_s.ob test 2 dspcontrol overflow flag error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
+
diff --git a/tests/tcg/mips/mips64-dsp/absq_s_ph.c b/tests/tcg/mips/mips64-dsp/absq_s_ph.c
new file mode 100644
index 000000000..238416d43
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/absq_s_ph.c
@@ -0,0 +1,37 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt;
+    long long result;
+
+    rt     = 0x10017EFD;
+    result = 0x10017EFD;
+
+    __asm
+        ("absq_s.ph %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    if (rd != result) {
+        printf("absq_s.ph wrong\n");
+
+        return -1;
+    }
+
+    rt     = 0x8000A536;
+    result = 0x7FFF5ACA;
+
+    __asm
+        ("absq_s.ph %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    if (rd != result) {
+        printf("absq_s.ph wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/absq_s_pw.c b/tests/tcg/mips/mips64-dsp/absq_s_pw.c
new file mode 100644
index 000000000..48fc763b4
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/absq_s_pw.c
@@ -0,0 +1,66 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt, result, dspcontrol;
+    rd = 0;
+    rt = 0x7F7F7F7F7F7F7F7F;
+    result = 0x7F7F7F7F7F7F7F7F;
+
+
+    __asm
+        ("absq_s.pw %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+
+    if (result != rd) {
+        printf("absq_s.pw test 1 error\n");
+
+        return -1;
+    }
+
+    rd = 0;
+    __asm
+        ("rddsp %0\n\t"
+         : "=r"(rd)
+        );
+    rd >> 20;
+    rd = rd & 0x1;
+    if (rd != 0) {
+        printf("absq_s.pw test 1 dspcontrol overflow flag error\n");
+
+        return -1;
+    }
+
+    rd = 0;
+    rt = 0x80000000FFFFFFFF;
+    result = 0x7FFFFFFF00000001;
+
+    __asm
+        ("absq_s.pw %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    if (result != rd) {
+        printf("absq_s.pw test 2 error\n");
+
+        return -1;
+    }
+
+    rd = 0;
+    __asm
+        ("rddsp %0\n\t"
+         : "=r"(rd)
+        );
+    rd = rd >> 20;
+    rd = rd & 0x1;
+    if (rd != 1) {
+        printf("absq_s.pw test 2 dspcontrol overflow flag error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
+
diff --git a/tests/tcg/mips/mips64-dsp/absq_s_qh.c b/tests/tcg/mips/mips64-dsp/absq_s_qh.c
new file mode 100644
index 000000000..9001a9e16
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/absq_s_qh.c
@@ -0,0 +1,40 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt, result, dspcontrol;
+    rd = 0;
+    rt = 0x7F7F7F7F7F7F7F7F;
+    result = 0x7F7F7F7F7F7F7F7F;
+
+
+    __asm
+        ("absq_s.qh %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+
+    if (result != rd) {
+        printf("absq_s.qh test 1 error\n");
+
+        return -1;
+    }
+
+    rd = 0;
+    rt = 0x8000FFFFFFFFFFFF;
+    result = 0x7FFF000100000001;
+
+    __asm
+        ("absq_s.pw %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    if (result != rd) {
+        printf("absq_s.rw test 2 error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
+
diff --git a/tests/tcg/mips/mips64-dsp/absq_s_w.c b/tests/tcg/mips/mips64-dsp/absq_s_w.c
new file mode 100644
index 000000000..414c8bd3f
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/absq_s_w.c
@@ -0,0 +1,48 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt;
+    long long result;
+
+    rt     = 0x80000000;
+    result = 0x7FFFFFFF;
+    __asm
+        ("absq_s.w %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    if (rd != result) {
+        printf("absq_s_w.ph wrong\n");
+
+        return -1;
+    }
+
+    rt     = 0x80030000;
+    result = 0x7FFD0000;
+    __asm
+        ("absq_s.w %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    if (rd != result) {
+        printf("absq_s_w.ph wrong\n");
+
+        return -1;
+    }
+
+    rt     = 0x31036080;
+    result = 0x31036080;
+    __asm
+        ("absq_s.w %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    if (rd != result) {
+        printf("absq_s_w.ph wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/addq_ph.c b/tests/tcg/mips/mips64-dsp/addq_ph.c
new file mode 100644
index 000000000..22a36d980
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/addq_ph.c
@@ -0,0 +1,57 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long dsp;
+    long long result;
+
+    rs     = 0xFFFFFFFF;
+    rt     = 0x10101010;
+    result = 0x100F100F;
+    __asm
+        ("addq.ph   %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    if (rd != result) {
+        printf("1 addq.ph wrong\n");
+
+        return -1;
+    }
+
+    rs     = 0x3712847D;
+    rt     = 0x0031AF2D;
+    result = 0x374333AA;
+    __asm
+        ("addq.ph   %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    if (rd != result) {
+        printf("2 addq.ph wrong\n");
+
+        return -1;
+    }
+
+    rs     = 0x7fff847D;
+    rt     = 0x0031AF2D;
+    result = 0xffffffff803033AA;
+    __asm
+        ("addq.ph   %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+
+    __asm("rddsp %0\n\t"
+          : "=r"(dsp)
+         );
+
+    if (rd != result || (((dsp >> 20) & 0x01) != 1)) {
+        printf("3 addq.ph wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/addq_pw.c b/tests/tcg/mips/mips64-dsp/addq_pw.c
new file mode 100644
index 000000000..99a7668c0
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/addq_pw.c
@@ -0,0 +1,46 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, result, dspreg, dspresult;
+
+    rs = 0x123456787FFFFFFF;
+    rt = 0x1111111100000101;
+    result = 0x2345678980000100;
+    dspresult = 0x1;
+
+    __asm
+        ("addq.pw %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dspreg)
+         : "r"(rs), "r"(rt)
+        );
+
+    dspreg = ((dspreg >> 20) & 0x01);
+    if ((rd != result) || (dspreg != dspresult)) {
+        printf("addq.pw error\n");
+
+        return -1;
+    }
+
+    rs = 0x1234567880FFFFFF;
+    rt = 0x1111111180000001;
+    result = 0x2345678901000000;
+    dspresult = 0x1;
+
+    __asm
+        ("addq.pw %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dspreg)
+         : "r"(rs), "r"(rt)
+        );
+
+    dspreg = ((dspreg >> 20) & 0x01);
+    if ((rd != result) || (dspreg != dspresult)) {
+        printf("addq.pw error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/addq_qh.c b/tests/tcg/mips/mips64-dsp/addq_qh.c
new file mode 100644
index 000000000..4b874afb8
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/addq_qh.c
@@ -0,0 +1,28 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, result, dspreg, dspresult;
+
+    rs = 0x123456787FFF8010;
+    rt = 0x1111111100018000;
+    result = 0x2345678980000010;
+    dspresult = 0x1;
+
+    __asm
+        ("addq.qh %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dspreg)
+         : "r"(rs), "r"(rt)
+        );
+
+    dspreg = ((dspreg >> 20) & 0x01);
+
+    if ((rd != result) || (dspreg != dspresult)) {
+        printf("addq.qh error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/addq_s_ph.c b/tests/tcg/mips/mips64-dsp/addq_s_ph.c
new file mode 100644
index 000000000..ad84cdcfe
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/addq_s_ph.c
@@ -0,0 +1,84 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long dsp;
+    long long result;
+
+    rs     = 0xFFFFFFFF;
+    rt     = 0x10101010;
+    result = 0x100F100F;
+    __asm
+        ("addq_s.ph   %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    if (rd != result) {
+        printf("1 addq_s.ph wrong\n");
+
+        return -1;
+    }
+
+    rs     = 0x3712847D;
+    rt     = 0x0031AF2D;
+    result = 0x37438000;
+    __asm
+        ("addq_s.ph   %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+
+    __asm
+        ("rddsp %0\n\t"
+         : "=r"(dsp)
+        );
+
+    if ((rd != result) || (((dsp >> 20) & 0x01) != 1)) {
+        printf("2 addq_s.ph wrong\n");
+
+        return -1;
+    }
+
+    rs     = 0x7fff847D;
+    rt     = 0x0031AF2D;
+    result = 0x7fff8000;
+    __asm
+        ("addq_s.ph   %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+
+    __asm
+        ("rddsp %0\n\t"
+         : "=r"(dsp)
+        );
+
+    if ((rd != result) || (((dsp >> 20) & 0x01) != 1)) {
+        printf("3 addq_s.ph wrong\n");
+
+        return -1;
+    }
+
+    rs     = 0x8030847D;
+    rt     = 0x8a00AF2D;
+    result = 0xffffffff80008000;
+    __asm
+        ("addq_s.ph   %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+
+    __asm
+        ("rddsp %0\n\t"
+         : "=r"(dsp)
+        );
+
+    if ((rd != result) || (((dsp >> 20) & 0x01) != 1)) {
+        printf("4 addq_s.ph wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/addq_s_pw.c b/tests/tcg/mips/mips64-dsp/addq_s_pw.c
new file mode 100644
index 000000000..2e380bbfc
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/addq_s_pw.c
@@ -0,0 +1,45 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, result, dspreg, dspresult;
+    rs = 0x123456787FFFFFFF;
+    rt = 0x1111111100000001;
+    result = 0x234567897FFFFFFF;
+    dspresult = 0x1;
+
+    __asm
+        ("addq_s.pw %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dspreg)
+         : "r"(rs), "r"(rt)
+        );
+
+    dspreg = ((dspreg >> 20) & 0x01);
+    if ((rd != result) || (dspreg != dspresult)) {
+        printf("addq_s.pw error\n");
+
+        return -1;
+    }
+
+    rs = 0x80FFFFFFE00000FF;
+    rt = 0x80000001200000DD;
+    result = 0x80000000000001DC;
+    dspresult = 0x01;
+
+    __asm
+        ("addq_s.pw %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dspreg)
+         : "r"(rs), "r"(rt)
+        );
+
+    dspreg = ((dspreg >> 20) & 0x01);
+    if ((rd != result) || (dspreg != dspresult)) {
+        printf("addq_s.pw error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/addq_s_qh.c b/tests/tcg/mips/mips64-dsp/addq_s_qh.c
new file mode 100644
index 000000000..b638a2b93
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/addq_s_qh.c
@@ -0,0 +1,26 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, result, dspreg, dspresult;
+    rs = 0x123456787FFF8000;
+    rt = 0x1111111100028000;
+    result = 0x234567897FFF8000;
+    dspresult = 0x1;
+
+    __asm
+        ("addq_s.qh %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dspreg)
+         : "r"(rs), "r"(rt)
+        );
+
+    dspreg = ((dspreg >> 20) & 0x01);
+    if ((rd != result) || (dspreg != dspresult)) {
+        printf("addq_s.qh error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/addq_s_w.c b/tests/tcg/mips/mips64-dsp/addq_s_w.c
new file mode 100644
index 000000000..3e08f5d48
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/addq_s_w.c
@@ -0,0 +1,48 @@
+#include "io.h"
+
+int main()
+{
+    long long rd, rs, rt;
+    long long result;
+
+    rt     = 0x10017EFD;
+    rs     = 0x11111111;
+    result = 0x2112900e;
+
+    __asm
+        ("addq_s.w %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    if (rd != result) {
+        printf("addq_s.w error\n");
+    }
+
+    rt     = 0x80017EFD;
+    rs     = 0x81111111;
+    result = 0xffffffff80000000;
+
+    __asm
+        ("addq_s.w %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    if (rd != result) {
+        printf("addq_s.w error\n");
+    }
+
+    rt     = 0x7fffffff;
+    rs     = 0x01111111;
+    result = 0x7fffffff;
+
+    __asm
+        ("addq_s.w %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    if (rd != result) {
+        printf("addq_s.w error\n");
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/addsc.c b/tests/tcg/mips/mips64-dsp/addsc.c
new file mode 100644
index 000000000..4b684b9b9
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/addsc.c
@@ -0,0 +1,39 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long dsp;
+    long long result;
+
+    rs     = 0x0000000F;
+    rt     = 0x00000001;
+    result = 0x00000010;
+    __asm
+        ("addsc %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    if (rd != result) {
+        printf("1 addsc wrong\n");
+
+        return -1;
+    }
+
+    rs     = 0xFFFF0FFF;
+    rt     = 0x00010111;
+    result = 0x00001110;
+    __asm
+        ("addsc %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    if ((rd != result) || (((dsp >> 13) & 0x01) != 1)) {
+        printf("2 addsc wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/addu_ob.c b/tests/tcg/mips/mips64-dsp/addu_ob.c
new file mode 100644
index 000000000..17f9c668c
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/addu_ob.c
@@ -0,0 +1,28 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, result, dspreg, dspresult;
+
+    rs = 0x123456789ABCDEF0;
+    rt = 0x3456123498DEF390;
+    result = 0x468A68AC329AD180;
+    dspresult = 0x01;
+
+    __asm
+        ("addu.ob %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dspreg)
+         : "r"(rs), "r"(rt)
+        );
+
+    dspreg = ((dspreg >> 20) & 0x01);
+
+    if ((rd != result) || (dspreg != dspresult)) {
+        printf("addu.ob error\n\t");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/addu_qb.c b/tests/tcg/mips/mips64-dsp/addu_qb.c
new file mode 100644
index 000000000..3b9b5fc5b
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/addu_qb.c
@@ -0,0 +1,40 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long dsp;
+    long long result;
+
+    rs     = 0x00FF00FF;
+    rt     = 0x00010001;
+    result = 0x00000000;
+    __asm
+        ("addu.qb %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    if ((rd != result) || (((dsp >> 20) & 0x01) != 1)) {
+        printf("1 addu.qb wrong\n");
+
+        return -1;
+    }
+
+    rs     = 0xFFFF1111;
+    rt     = 0x00020001;
+    result = 0xFFFFFFFFFF011112;
+    __asm
+        ("addu.qb %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    if ((rd != result) || (((dsp >> 20) & 0x01) != 1)) {
+        printf("2 addu.qb wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/addu_s_ob.c b/tests/tcg/mips/mips64-dsp/addu_s_ob.c
new file mode 100644
index 000000000..e89a4638b
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/addu_s_ob.c
@@ -0,0 +1,27 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, result, dspreg, dspresult;
+    rs = 0x123456789ABCDEF0;
+    rt = 0x3456123498DEF390;
+    result = 0x468A68ACFFFFFFFF;
+    dspresult = 0x01;
+
+    __asm
+        ("addu_s.ob %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dspreg)
+         : "r"(rs), "r"(rt)
+        );
+
+    dspreg = ((dspreg >> 20) & 0x01);
+
+    if ((rd != result) || (dspreg != dspresult)) {
+        printf("addu_s.ob error\n\t");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/addu_s_qb.c b/tests/tcg/mips/mips64-dsp/addu_s_qb.c
new file mode 100644
index 000000000..cb84293ad
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/addu_s_qb.c
@@ -0,0 +1,40 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long dsp;
+    long long result;
+
+    rs     = 0x10FF01FF;
+    rt     = 0x10010001;
+    result = 0x20FF01FF;
+    __asm
+        ("addu_s.qb %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    if ((rd != result) || (((dsp >> 20) & 0x1) != 1)) {
+        printf("1 addu_s.qb error 1\n");
+
+        return -1;
+    }
+
+    rs     = 0xFFFFFFFFFFFF1111;
+    rt     = 0x00020001;
+    result = 0xFFFFFFFFFFFF1112;
+    __asm
+        ("addu_s.qb %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    if ((rd != result) || (((dsp >> 20) & 0x1) != 1)) {
+        printf("2 addu_s.qb error 2\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/addwc.c b/tests/tcg/mips/mips64-dsp/addwc.c
new file mode 100644
index 000000000..5929cd2f5
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/addwc.c
@@ -0,0 +1,59 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long dspi, dspo;
+    long long result;
+
+    rs     = 0x10FF01FF;
+    rt     = 0x10010001;
+    dspi   = 0x00002000;
+    result = 0x21000201;
+    __asm
+        ("wrdsp %3\n"
+         "addwc %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt), "r"(dspi)
+        );
+    if (rd != result) {
+        printf("1 addwc wrong\n");
+
+        return -1;
+    }
+
+    rs     = 0xFFFF1111;
+    rt     = 0x00020001;
+    dspi   = 0x00;
+    result = 0x00011112;
+    __asm
+        ("wrdsp %3\n"
+         "addwc %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt), "r"(dspi)
+        );
+    if (rd != result) {
+        printf("2 addwc wrong\n");
+
+        return -1;
+    }
+
+    rs     = 0x8FFF1111;
+    rt     = 0x80020001;
+    dspi   = 0x00;
+    result = 0x10011112;
+    __asm
+        ("wrdsp %4\n"
+         "addwc %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dspo)
+         : "r"(rs), "r"(rt), "r"(dspi)
+        );
+    if ((rd != result) || (((dspo >> 20) & 0x01) != 1)) {
+        printf("3 addwc wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/bitrev.c b/tests/tcg/mips/mips64-dsp/bitrev.c
new file mode 100644
index 000000000..ac24ef3f5
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/bitrev.c
@@ -0,0 +1,23 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt;
+    long long result;
+
+    rt     = 0x12345678;
+    result = 0x00001E6A;
+
+    __asm
+        ("bitrev %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    if (rd != result) {
+        printf("bitrev wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/bposge32.c b/tests/tcg/mips/mips64-dsp/bposge32.c
new file mode 100644
index 000000000..97bce4460
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/bposge32.c
@@ -0,0 +1,50 @@
+#include "io.h"
+
+int main(void)
+{
+    long long dsp, sum;
+    long long result;
+
+    dsp =  0x20;
+    sum = 0x01;
+    result = 0x02;
+
+    __asm
+        ("wrdsp %1\n\t"
+         "bposge32 test1\n\t"
+         "nop\n\t"
+         "addi %0, 0xA2\n\t"
+         "nop\n\t"
+         "test1:\n\t"
+         "addi %0, 0x01\n\t"
+         : "+r"(sum)
+         : "r"(dsp)
+        );
+    if (sum != result) {
+        printf("bposge32 wrong\n");
+
+        return -1;
+    }
+
+    dsp =  0x10;
+    sum = 0x01;
+    result = 0xA4;
+
+    __asm
+        ("wrdsp %1\n\t"
+         "bposge32 test2\n\t"
+         "nop\n\t"
+         "addi %0, 0xA2\n\t"
+         "nop\n\t"
+         "test2:\n\t"
+         "addi %0, 0x01\n\t"
+         : "+r"(sum)
+         : "r"(dsp)
+        );
+    if (sum != result) {
+        printf("bposge32 wrong\n");
+
+        return -1;
+    }
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/bposge64.c b/tests/tcg/mips/mips64-dsp/bposge64.c
new file mode 100644
index 000000000..36161ad85
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/bposge64.c
@@ -0,0 +1,50 @@
+#include "io.h"
+
+int main(void)
+{
+    long long dsp, sum;
+    long long result;
+
+    dsp =  0x40;
+    sum = 0x01;
+    result = 0x02;
+
+    __asm
+        ("wrdsp %1\n\t"
+         "bposge64 test1\n\t"
+         "nop\n\t"
+         "addi %0, 0xA2\n\t"
+         "nop\n\t"
+         "test1:\n\t"
+         "addi %0, 0x01\n\t"
+         : "+r"(sum)
+         : "r"(dsp)
+        );
+    if (sum != result) {
+        printf("bposge64 wrong\n");
+
+        return -1;
+    }
+
+    dsp =  0x10;
+    sum = 0x01;
+    result = 0xA4;
+
+    __asm
+        ("wrdsp %1\n\t"
+         "bposge64 test2\n\t"
+         "nop\n\t"
+         "addi %0, 0xA2\n\t"
+         "nop\n\t"
+         "test2:\n\t"
+         "addi %0, 0x01\n\t"
+         : "+r"(sum)
+         : "r"(dsp)
+        );
+    if (sum != result) {
+        printf("bposge64 wrong\n");
+
+        return -1;
+    }
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/cmp_eq_ph.c b/tests/tcg/mips/mips64-dsp/cmp_eq_ph.c
new file mode 100644
index 000000000..63069d0da
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/cmp_eq_ph.c
@@ -0,0 +1,42 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long result;
+
+    rs     = 0x11777066;
+    rt     = 0x55AA33FF;
+    result = 0x00;
+    __asm
+        ("cmp.eq.ph %1, %2\n\t"
+         "rddsp %0\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+
+    rd = (rd >> 24) & 0x03;
+    if (rd != result) {
+        printf("cmp.eq.ph wrong\n");
+
+        return -1;
+    }
+
+    rs     = 0x11777066;
+    rt     = 0x11777066;
+    result = 0x03;
+    __asm
+        ("cmp.eq.ph %1, %2\n\t"
+         "rddsp %0\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    rd = (rd >> 24) & 0x03;
+    if (rd != result) {
+        printf("cmp.eq.ph wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/cmp_eq_pw.c b/tests/tcg/mips/mips64-dsp/cmp_eq_pw.c
new file mode 100644
index 000000000..bae4c06cc
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/cmp_eq_pw.c
@@ -0,0 +1,46 @@
+#include "io.h"
+
+int main(void)
+{
+  long long rs, rt, dspreg, dspresult;
+
+  rs = 0x123456789ABCDEFF;
+  rt = 0x123456789ABCDEFF;
+  dspresult = 0x03;
+
+  __asm
+      ("cmp.eq.pw %1, %2\n\t"
+       "rddsp %0\n\t"
+       : "=r"(dspreg)
+       : "r"(rs), "r"(rt)
+      );
+
+  dspreg = ((dspreg >> 24) & 0x03);
+
+  if (dspreg != dspresult) {
+    printf("1 cmp.eq.pw error\n");
+
+    return -1;
+  }
+
+  rs = 0x123456799ABCDEFe;
+  rt = 0x123456789ABCDEFF;
+  dspresult = 0x00;
+
+  __asm
+      ("cmp.eq.pw %1, %2\n\t"
+       "rddsp %0\n\t"
+       : "=r"(dspreg)
+       : "r"(rs), "r"(rt)
+      );
+
+  dspreg = ((dspreg >> 24) & 0x03);
+
+  if (dspreg != dspresult) {
+    printf("2 cmp.eq.pw error\n");
+
+    return -1;
+  }
+
+  return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/cmp_eq_qh.c b/tests/tcg/mips/mips64-dsp/cmp_eq_qh.c
new file mode 100644
index 000000000..49ea27100
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/cmp_eq_qh.c
@@ -0,0 +1,46 @@
+#include "io.h"
+
+int main(void)
+{
+  long long rs, rt, dspreg, dspresult;
+
+  rs = 0x123456789ABCDEF0;
+  rt = 0x123456789ABCDEFF;
+  dspresult = 0x0E;
+
+  __asm
+      ("cmp.eq.qh %1, %2\n\t"
+        "rddsp %0\n\t"
+        : "=r"(dspreg)
+        : "r"(rs), "r"(rt)
+       );
+
+  dspreg = ((dspreg >> 24) & 0x0F);
+
+  if (dspreg != dspresult) {
+    printf("cmp.eq.qh error\n");
+
+    return -1;
+  }
+
+  rs = 0x12355a789A4CD3F0;
+  rt = 0x123456789ABCDEFF;
+  dspresult = 0x00;
+
+  __asm
+      ("cmp.eq.qh %1, %2\n\t"
+        "rddsp %0\n\t"
+        : "=r"(dspreg)
+        : "r"(rs), "r"(rt)
+       );
+
+  dspreg = ((dspreg >> 24) & 0x0F);
+
+  if (dspreg != dspresult) {
+    printf("cmp.eq.qh error\n");
+
+    return -1;
+  }
+
+  return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/cmp_le_ph.c b/tests/tcg/mips/mips64-dsp/cmp_le_ph.c
new file mode 100644
index 000000000..12d24f178
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/cmp_le_ph.c
@@ -0,0 +1,40 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long result;
+
+    rs     = 0x11777066;
+    rt     = 0x55AA33FF;
+    result = 0x02;
+    __asm
+        ("cmp.le.ph %1, %2\n\t"
+         "rddsp %0\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+
+    rd = (rd >> 24) & 0x03;
+    if (rd != result) {
+        printf("cmp.le.ph wrong\n");
+
+        return -1;
+    }
+    rs     = 0x11777066;
+    rt     = 0x11777066;
+    result = 0x03;
+    __asm
+        ("cmp.le.ph %1, %2\n\t"
+         "rddsp %0\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    rd = (rd >> 24) & 0x03;
+    if (rd != result) {
+        printf("cmp.le.ph wrong\n");
+
+        return -1;
+    }
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/cmp_le_pw.c b/tests/tcg/mips/mips64-dsp/cmp_le_pw.c
new file mode 100644
index 000000000..6acc43cd5
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/cmp_le_pw.c
@@ -0,0 +1,46 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rs, rt, dspreg, dspresult;
+
+    rs = 0x123456789ABCDEF0;
+    rt = 0x123456789ABCDEFF;
+    dspresult = 0x03;
+
+    __asm
+        ("cmp.le.pw %1, %2\n\t"
+         "rddsp %0\n\t"
+         : "=r"(dspreg)
+         : "r"(rs), "r"(rt)
+        );
+
+    dspreg = ((dspreg >> 24) & 0x03);
+
+    if (dspreg != dspresult) {
+        printf("1 cmp.le.pw error\n");
+
+        return -1;
+    }
+
+    rs = 0x123456799ABCEEFF;
+    rt = 0x123456789ABCDEFF;
+    dspresult = 0x00;
+
+    __asm
+        ("cmp.le.pw %1, %2\n\t"
+         "rddsp %0\n\t"
+         : "=r"(dspreg)
+         : "r"(rs), "r"(rt)
+        );
+
+    dspreg = ((dspreg >> 24) & 0x03);
+
+    if (dspreg != dspresult) {
+        printf("2 cmp.le.pw error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/cmp_le_qh.c b/tests/tcg/mips/mips64-dsp/cmp_le_qh.c
new file mode 100644
index 000000000..c9ce21667
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/cmp_le_qh.c
@@ -0,0 +1,46 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rs, rt, dspreg, dspresult;
+
+    rs = 0x123456789ABCDEF0;
+    rt = 0x123456789ABCDEFF;
+    dspresult = 0x0F;
+
+    __asm
+        ("cmp.le.qh %1, %2\n\t"
+         "rddsp %0\n\t"
+         : "=r"(dspreg)
+         : "r"(rs), "r"(rt)
+        );
+
+    dspreg = ((dspreg >> 24) & 0x0F);
+
+    if (dspreg != dspresult) {
+        printf("cmp.le.qh error\n");
+
+        return -1;
+    }
+
+    rs = 0x823456789ABCDEF0;
+    rt = 0x123456789ABCDEFF;
+    dspresult = 0x0f;
+
+    __asm
+        ("cmp.le.qh %1, %2\n\t"
+         "rddsp %0\n\t"
+         : "=r"(dspreg)
+         : "r"(rs), "r"(rt)
+        );
+
+    dspreg = ((dspreg >> 24) & 0x0F);
+
+    if (dspreg != dspresult) {
+        printf("cmp.le.qh error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/cmp_lt_ph.c b/tests/tcg/mips/mips64-dsp/cmp_lt_ph.c
new file mode 100644
index 000000000..1d91228c3
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/cmp_lt_ph.c
@@ -0,0 +1,41 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long result;
+
+    rs     = 0x11777066;
+    rt     = 0x55AA33FF;
+    result = 0x02;
+    __asm
+        ("cmp.lt.ph %1, %2\n\t"
+         "rddsp %0\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+
+    rd = (rd >> 24) & 0x03;
+    if (rd != result) {
+        printf("cmp.lt.ph wrong\n");
+
+        return -1;
+    }
+    rs     = 0x11777066;
+    rt     = 0x11777066;
+    result = 0x00;
+    __asm
+        ("cmp.lt.ph %1, %2\n\t"
+         "rddsp %0\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    rd = (rd >> 24) & 0x03;
+    if (rd != result) {
+        printf("cmp.lt.ph2 wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/cmp_lt_pw.c b/tests/tcg/mips/mips64-dsp/cmp_lt_pw.c
new file mode 100644
index 000000000..87e74caf3
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/cmp_lt_pw.c
@@ -0,0 +1,46 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rs, rt, dspreg, dspresult;
+
+    rs = 0x123456789ABCDEF0;
+    rt = 0x123456789ABCDEFF;
+    dspresult = 0x01;
+
+    __asm
+        ("cmp.lt.pw %1, %2\n\t"
+         "rddsp %0\n\t"
+         : "=r"(dspreg)
+         : "r"(rs), "r"(rt)
+        );
+
+    dspreg = ((dspreg >> 24) & 0x03);
+
+    if (dspreg != dspresult) {
+        printf("cmp.lt.pw error\n");
+
+        return -1;
+    }
+
+    rs = 0x123456779ABCDEFf;
+    rt = 0x123456789ABCDEFF;
+    dspresult = 0x02;
+
+    __asm
+        ("cmp.lt.pw %1, %2\n\t"
+         "rddsp %0\n\t"
+         : "=r"(dspreg)
+         : "r"(rs), "r"(rt)
+        );
+
+    dspreg = ((dspreg >> 24) & 0x03);
+
+    if (dspreg != dspresult) {
+        printf("cmp.lt.pw error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/cmp_lt_qh.c b/tests/tcg/mips/mips64-dsp/cmp_lt_qh.c
new file mode 100644
index 000000000..0a13a5eaa
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/cmp_lt_qh.c
@@ -0,0 +1,46 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rs, rt, dspreg, dspresult;
+
+    rs = 0x123558789ABCDEF0;
+    rt = 0x123456789ABCDEFF;
+    dspresult = 0x01;
+
+    __asm
+        ("cmp.lt.qh %1, %2\n\t"
+         "rddsp %0\n\t"
+         : "=r"(dspreg)
+         : "r"(rs), "r"(rt)
+        );
+
+    dspreg = ((dspreg >> 24) & 0x0F);
+
+    if (dspreg != dspresult) {
+        printf("cmp.lt.qh error\n");
+
+        return -1;
+    }
+
+    rs = 0x123356779ABbDEF0;
+    rt = 0x123456789ABCDEFF;
+    dspresult = 0x0f;
+
+    __asm
+        ("cmp.lt.qh %1, %2\n\t"
+         "rddsp %0\n\t"
+         : "=r"(dspreg)
+         : "r"(rs), "r"(rt)
+        );
+
+    dspreg = ((dspreg >> 24) & 0x0F);
+
+    if (dspreg != dspresult) {
+        printf("cmp.lt.qh error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/cmpgu_eq_ob.c b/tests/tcg/mips/mips64-dsp/cmpgu_eq_ob.c
new file mode 100644
index 000000000..697d73dd1
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/cmpgu_eq_ob.c
@@ -0,0 +1,40 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, result;
+
+    rs = 0x123456789ABCDEF0;
+    rt = 0x123456789ABCDEFF;
+    result = 0xFE;
+
+    __asm
+        ("cmpgu.eq.ob %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+
+    if (rd != result) {
+        printf("cmpgu.eq.ob error\n");
+
+        return -1;
+    }
+
+    rs = 0x133456789ABCDEF0;
+    rt = 0x123556789ABCDEFF;
+    result = 0x3E;
+
+    __asm
+        ("cmpgu.eq.ob %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+
+    if (rd != result) {
+        printf("cmpgu.eq.ob error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/cmpgu_eq_qb.c b/tests/tcg/mips/mips64-dsp/cmpgu_eq_qb.c
new file mode 100644
index 000000000..b41c4430f
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/cmpgu_eq_qb.c
@@ -0,0 +1,38 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long result;
+
+    rs     = 0x11777066;
+    rt     = 0x55AA70FF;
+    result = 0x02;
+    __asm
+        ("cmpgu.eq.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+
+    if (rd != result) {
+        printf("cmpgu.eq.ph wrong\n");
+
+        return -1;
+    }
+
+    rs     = 0x11777066;
+    rt     = 0x11777066;
+    result = 0x0F;
+    __asm
+        ("cmpgu.eq.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    if (rd != result) {
+        printf("cmpgu.eq.ph wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/cmpgu_le_ob.c b/tests/tcg/mips/mips64-dsp/cmpgu_le_ob.c
new file mode 100644
index 000000000..8b65f18c0
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/cmpgu_le_ob.c
@@ -0,0 +1,40 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, result;
+
+    rs = 0x123456789ABCDEF0;
+    rt = 0x123456789ABCDEFF;
+    result = 0xFF;
+
+    __asm
+        ("cmpgu.le.ob %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+
+    if (rd != result) {
+        printf("cmpgu.le.ob error\n");
+
+        return -1;
+    }
+
+    rs = 0x823556789ABCDEF0;
+    rt = 0x123456789ABCDEFF;
+    result = 0x3F;
+
+    __asm
+        ("cmpgu.le.ob %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+
+    if (rd != result) {
+        printf("cmpgu.le.ob error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/cmpgu_le_qb.c b/tests/tcg/mips/mips64-dsp/cmpgu_le_qb.c
new file mode 100644
index 000000000..dd2b091f6
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/cmpgu_le_qb.c
@@ -0,0 +1,37 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long result;
+
+    rs     = 0x11777066;
+    rt     = 0x55AA70FF;
+    result = 0x0F;
+    __asm
+        ("cmpgu.le.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    if (rd != result) {
+        printf("cmpgu.le.qb wrong\n");
+
+        return -1;
+    }
+
+    rs     = 0x11777066;
+    rt     = 0x11766066;
+    result = 0x09;
+    __asm
+        ("cmpgu.le.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    if (rd != result) {
+        printf("cmpgu.le.qb wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/cmpgu_lt_ob.c b/tests/tcg/mips/mips64-dsp/cmpgu_lt_ob.c
new file mode 100644
index 000000000..3e5c9dd6d
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/cmpgu_lt_ob.c
@@ -0,0 +1,40 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, result;
+
+    rs = 0x123456789ABCDEF0;
+    rt = 0x123456789ABCDEFF;
+    result = 0x01;
+
+    __asm
+        ("cmpgu.lt.ob %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+
+    if (rd != result) {
+        printf("cmpgu.lt.ob error\n");
+
+        return -1;
+    }
+
+    rs = 0x823455789ABCDEF0;
+    rt = 0x123356789ABCDEFF;
+    result = 0x21;
+
+    __asm
+        ("cmpgu.lt.ob %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+
+    if (rd != result) {
+        printf("cmpgu.lt.ob error\n");
+
+        return -1;
+    }
+
+   return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/cmpgu_lt_qb.c b/tests/tcg/mips/mips64-dsp/cmpgu_lt_qb.c
new file mode 100644
index 000000000..a467cb78d
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/cmpgu_lt_qb.c
@@ -0,0 +1,38 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long result;
+
+    rs     = 0x11777066;
+    rt     = 0x55AA70FF;
+    result = 0x0D;
+    __asm
+        ("cmpgu.lt.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+
+    if (rd != result) {
+        printf("cmpgu.lt.qb wrong\n");
+
+        return -1;
+    }
+
+    rs     = 0x11777066;
+    rt     = 0x11766066;
+    result = 0x00;
+    __asm
+        ("cmpgu.lt.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    if (rd != result) {
+        printf("cmpgu.lt.qb wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/cmpu_eq_ob.c b/tests/tcg/mips/mips64-dsp/cmpu_eq_ob.c
new file mode 100644
index 000000000..4d1983e5e
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/cmpu_eq_ob.c
@@ -0,0 +1,46 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, dspreg, dspresult;
+
+    rs = 0x123456789ABCDEF0;
+    rt = 0x123456789ABCDEFF;
+    dspresult = 0xFE;
+
+    __asm
+        ("cmpu.eq.ob %1, %2\n\t"
+         "rddsp %0"
+         : "=r"(dspreg)
+         : "r"(rs), "r"(rt)
+        );
+
+    dspreg = ((dspreg >> 24) & 0xFF);
+
+    if (dspreg != dspresult) {
+        printf("cmpu.eq.ob error\n");
+
+        return -1;
+    }
+
+    rs = 0x133516713A0CD1F0;
+    rt = 0x123456789ABCDEFF;
+    dspresult = 0x00;
+
+    __asm
+        ("cmpu.eq.ob %1, %2\n\t"
+         "rddsp %0"
+         : "=r"(dspreg)
+         : "r"(rs), "r"(rt)
+        );
+
+    dspreg = ((dspreg >> 24) & 0xFF);
+
+    if (dspreg != dspresult) {
+        printf("cmpu.eq.ob error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/cmpu_eq_qb.c b/tests/tcg/mips/mips64-dsp/cmpu_eq_qb.c
new file mode 100644
index 000000000..28f3bec25
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/cmpu_eq_qb.c
@@ -0,0 +1,42 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rs, rt;
+    long long dsp;
+    long long result;
+
+    rs         = 0x11777066;
+    rt         = 0x55AA70FF;
+    result     = 0x02;
+    __asm
+        ("cmpu.eq.qb %1, %2\n\t"
+         "rddsp %0\n\t"
+         : "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 24) & 0x0F;
+    if (dsp != result) {
+        printf("cmpu.eq.qb wrong\n");
+
+        return -1;
+    }
+
+    rs     = 0x11777066;
+    rt     = 0x11777066;
+    result = 0x0F;
+    __asm
+        ("cmpu.eq.qb %1, %2\n\t"
+         "rddsp %0\n\t"
+         : "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 24) & 0x0F;
+    if (dsp != result) {
+        printf("cmpu.eq.qb wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/cmpu_le_ob.c b/tests/tcg/mips/mips64-dsp/cmpu_le_ob.c
new file mode 100644
index 000000000..8acbd1c4b
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/cmpu_le_ob.c
@@ -0,0 +1,44 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rs, rt, dspreg, dspresult;
+
+    rs = 0x123456789ABCDEF0;
+    rt = 0x123456789ABCDEFF;
+    dspresult = 0xFF;
+
+    __asm
+        ("cmpu.le.ob %1, %2\n\t"
+         "rddsp %0"
+         : "=r"(dspreg)
+         : "r"(rs), "r"(rt)
+        );
+
+    dspreg = dspreg >> 24;
+    if (dspreg != dspresult) {
+        printf("cmpu.le.ob error\n");
+
+        return -1;
+    }
+
+    rs = 0x823656789ABCDEF0;
+    rt = 0x123456789ABCDEFF;
+    dspresult = 0x3F;
+
+    __asm
+        ("cmpu.le.ob %1, %2\n\t"
+         "rddsp %0"
+         : "=r"(dspreg)
+         : "r"(rs), "r"(rt)
+        );
+
+    dspreg = dspreg >> 24;
+    if (dspreg != dspresult) {
+        printf("cmpu.le.ob error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/cmpu_le_qb.c b/tests/tcg/mips/mips64-dsp/cmpu_le_qb.c
new file mode 100644
index 000000000..8a17a0851
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/cmpu_le_qb.c
@@ -0,0 +1,41 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rs, rt;
+    long long dsp;
+    long long result;
+
+    rs         = 0x11777066;
+    rt         = 0x55AA70FF;
+    result     = 0x0F;
+    __asm
+        ("cmpu.le.qb %1, %2\n\t"
+         "rddsp %0\n\t"
+         : "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 24) & 0x0F;
+    if (dsp != result) {
+        printf("cmpu.le.qb wrong\n");
+
+        return -1;
+    }
+
+    rs     = 0x11777066;
+    rt     = 0x11777066;
+    result = 0x0F;
+    __asm
+        ("cmpu.le.qb %1, %2\n\t"
+         "rddsp %0\n\t"
+         : "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 24) & 0x0F;
+    if (dsp != result) {
+        printf("cmpu.le.qb wrong\n");
+
+        return -1;
+    }
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/cmpu_lt_ob.c b/tests/tcg/mips/mips64-dsp/cmpu_lt_ob.c
new file mode 100644
index 000000000..34e312d81
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/cmpu_lt_ob.c
@@ -0,0 +1,44 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rs, rt, dspreg, dspresult;
+
+    rs = 0x123456789ABCDEF0;
+    rt = 0x123456789ABCDEFF;
+    dspresult = 0x01;
+
+    __asm
+        ("cmpu.lt.ob %1, %2\n\t"
+         "rddsp %0"
+         : "=r"(dspreg)
+         : "r"(rs), "r"(rt)
+        );
+
+    dspreg = dspreg >> 24;
+    if (dspreg != dspresult) {
+        printf("cmpu.lt.ob error\n");
+
+        return -1;
+    }
+
+    rs = 0x823156789ABCDEF0;
+    rt = 0x123456789ABCDEFF;
+    dspresult = 0x41;
+
+    __asm
+        ("cmpu.lt.ob %1, %2\n\t"
+         "rddsp %0"
+         : "=r"(dspreg)
+         : "r"(rs), "r"(rt)
+        );
+
+    dspreg = dspreg >> 24;
+    if (dspreg != dspresult) {
+        printf("cmpu.lt.ob error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/cmpu_lt_qb.c b/tests/tcg/mips/mips64-dsp/cmpu_lt_qb.c
new file mode 100644
index 000000000..adb75eed5
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/cmpu_lt_qb.c
@@ -0,0 +1,42 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rs, rt;
+    long long dsp;
+    long long result;
+
+    rs         = 0x11777066;
+    rt         = 0x55AA70FF;
+    result     = 0x0D;
+    __asm
+        ("cmpu.lt.qb %1, %2\n\t"
+         "rddsp %0\n\t"
+         : "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 24) & 0x0F;
+    if (dsp != result) {
+        printf("cmpu.lt.qb wrong\n");
+
+        return -1;
+    }
+
+    rs     = 0x11777066;
+    rt     = 0x11777066;
+    result = 0x00;
+    __asm
+        ("cmpu.lt.qb %1, %2\n\t"
+         "rddsp %0\n\t"
+         : "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 24) & 0x0F;
+    if (dsp != result) {
+        printf("cmpu.lt.qb wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dappend.c b/tests/tcg/mips/mips64-dsp/dappend.c
new file mode 100644
index 000000000..ba8e12182
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dappend.c
@@ -0,0 +1,37 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, rs;
+    long long res;
+    rt = 0x1234567887654321;
+    rs = 0xabcd1234abcd8765;
+
+    res = 0x1234567887654321;
+    __asm
+        ("dappend %0, %1, 0x0\n\t"
+         : "=r"(rt)
+         : "r"(rs)
+        );
+
+    if (rt != res) {
+        printf("dappend error\n");
+        return -1;
+    }
+
+    rt = 0x1234567887654321;
+    rs = 0xabcd1234abcd8765;
+
+    res = 0x2345678876543215;
+    __asm
+        ("dappend %0, %1, 0x4\n\t"
+         : "=r"(rt)
+         : "r"(rs)
+        );
+
+    if (rt != res) {
+        printf("dappend error\n");
+        return -1;
+    }
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dextp.c b/tests/tcg/mips/mips64-dsp/dextp.c
new file mode 100644
index 000000000..a469cc036
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dextp.c
@@ -0,0 +1,54 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, dsp;
+    long long achi, acli;
+    long long res, resdsp;
+    int rs;
+
+    rs = 0xabcd1234;
+
+    achi = 0x12345678;
+    acli = 0x87654321;
+    res = 0xff;
+    resdsp = 0x0;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "wrdsp %4\n\t"
+         "dextp %0, $ac1, 0x7\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rs)
+        );
+    dsp = (dsp >> 14) & 0x1;
+    if ((dsp != resdsp) || (rt != res)) {
+        printf("dextp error\n");
+        return -1;
+    }
+
+    rs = 0xabcd1200;
+
+    achi = 0x12345678;
+    acli = 0x87654321;
+    resdsp = 0x1;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "wrdsp %4\n\t"
+         "dextp %0, $ac1, 0x7\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rs)
+        );
+    dsp = (dsp >> 14) & 0x1;
+    if (dsp != resdsp) {
+        printf("dextp error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dextpdp.c b/tests/tcg/mips/mips64-dsp/dextpdp.c
new file mode 100644
index 000000000..a2361e2d4
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dextpdp.c
@@ -0,0 +1,59 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, dsp;
+    long long achi, acli;
+    long long res, resdsp, resdsppos;
+    int rs;
+    int tmp1, tmp2;
+
+    rs = 0xabcd1234;
+
+    achi = 0x12345678;
+    acli = 0x87654321;
+    res = 0xff;
+    resdsp = 0x0;
+    resdsppos = 0x2c;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "wrdsp %4\n\t"
+         "dextpdp %0, $ac1, 0x7\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rs)
+        );
+    tmp1 = (dsp >> 14) & 0x1;
+    tmp2 = dsp & 0x3f;
+
+    if ((tmp1 != resdsp) || (rt != res) || (tmp2 != resdsppos)) {
+        printf("dextpdp error\n");
+        return -1;
+    }
+
+    rs = 0xabcd1200;
+
+    achi = 0x12345678;
+    acli = 0x87654321;
+    resdsp = 0x1;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "wrdsp %4\n\t"
+         "dextpdp %0, $ac1, 0x7\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rs)
+        );
+    tmp1 = (dsp >> 14) & 0x1;
+
+    if (tmp1 != resdsp) {
+        printf("dextpdp error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dextpdpv.c b/tests/tcg/mips/mips64-dsp/dextpdpv.c
new file mode 100644
index 000000000..09c0b5b41
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dextpdpv.c
@@ -0,0 +1,63 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, rs, dsp;
+    long long achi, acli;
+    long long res, resdsp, resdsppos;
+    int rsdsp;
+    int tmp1, tmp2;
+
+    rsdsp = 0xabcd1234;
+    rs = 0x7;
+    achi = 0x12345678;
+    acli = 0x87654321;
+    res = 0xff;
+    resdsp = 0x0;
+    resdsppos = 0x2c;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "wrdsp %4, 0x1\n\t"
+         "wrdsp %4\n\t"
+         "dextpdpv %0, $ac1, %5\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rsdsp), "r"(rs)
+        );
+
+    tmp1 = (dsp >> 14) & 0x1;
+    tmp2 = dsp & 0x3f;
+
+    if ((tmp1 != resdsp) || (rt != res) || (tmp2 != resdsppos)) {
+        printf("dextpdpv error\n");
+        return -1;
+    }
+
+    rsdsp = 0xabcd1200;
+    rs = 0x7;
+    achi = 0x12345678;
+    acli = 0x87654321;
+    resdsp = 0x1;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "wrdsp %4, 0x1\n\t"
+         "wrdsp %4\n\t"
+         "dextpdpv %0, $ac1, %5\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rsdsp), "r"(rs)
+        );
+
+    tmp1 = (dsp >> 14) & 0x1;
+
+    if (tmp1 != resdsp) {
+        printf("dextpdpv error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dextpv.c b/tests/tcg/mips/mips64-dsp/dextpv.c
new file mode 100644
index 000000000..2626f3d98
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dextpv.c
@@ -0,0 +1,58 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, rs, dsp;
+    long long achi, acli;
+    long long res, resdsp;
+    int rsdsp;
+
+    rsdsp = 0xabcd1234;
+    rs = 0x7;
+
+    achi = 0x12345678;
+    acli = 0x87654321;
+    res = 0xff;
+    resdsp = 0x0;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "wrdsp %4, 0x1\n\t"
+         "wrdsp %4\n\t"
+         "dextpv %0, $ac1, %5\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rsdsp), "r"(rs)
+        );
+    dsp = (dsp >> 14) & 0x1;
+    if ((dsp != resdsp) || (rt != res)) {
+        printf("dextpv error\n");
+        return -1;
+    }
+
+    rsdsp = 0xabcd1200;
+    rs = 0x7;
+
+    achi = 0x12345678;
+    acli = 0x87654321;
+    resdsp = 0x1;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "wrdsp %4, 0x1\n\t"
+         "wrdsp %4\n\t"
+         "dextpv %0, $ac1, %5\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rsdsp), "r"(rs)
+        );
+    dsp = (dsp >> 14) & 0x1;
+    if (dsp != resdsp) {
+        printf("dextpv error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dextr_l.c b/tests/tcg/mips/mips64-dsp/dextr_l.c
new file mode 100644
index 000000000..538846df1
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dextr_l.c
@@ -0,0 +1,44 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt;
+    long long achi, acli;
+    long long res;
+
+    achi = 0x87654321;
+    acli = 0x12345678;
+
+    res = 0x2100000000123456;
+
+    __asm
+        ("mthi %1, $ac1\n\t"
+         "mtlo %2, $ac1\n\t"
+         "dextr.l %0, $ac1, 0x8\n\t"
+         : "=r"(rt)
+         : "r"(achi), "r"(acli)
+        );
+    if (rt != res) {
+        printf("dextr.l error\n");
+        return -1;
+    }
+
+    achi = 0x87654321;
+    acli = 0x12345678;
+
+    res = 0x12345678;
+
+    __asm
+        ("mthi %1, $ac1\n\t"
+         "mtlo %2, $ac1\n\t"
+         "dextr.l %0, $ac1, 0x0\n\t"
+         : "=r"(rt)
+         : "r"(achi), "r"(acli)
+        );
+    if (rt != res) {
+        printf("dextr.l error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dextr_r_l.c b/tests/tcg/mips/mips64-dsp/dextr_r_l.c
new file mode 100644
index 000000000..a10a9ab40
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dextr_r_l.c
@@ -0,0 +1,54 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, dsp;
+    long long achi, acli;
+    long long res, resdsp;
+
+    achi = 0x87654321;
+    acli = 0x12345678;
+
+    res = 0x2100000000123456;
+    resdsp = 0x01;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "dextr_r.l %0, $ac1, 0x8\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(achi), "r"(acli)
+        );
+
+    dsp = (dsp >> 23) & 0x1;
+
+    if ((dsp != resdsp) || (rt != res)) {
+        printf("dextr_r.l error\n");
+        return -1;
+    }
+
+    achi = 0x87654321;
+    acli = 0x12345678;
+
+    res = 0x12345678;
+    resdsp = 0x01;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "dextr_r.l %0, $ac1, 0x0\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(achi), "r"(acli)
+        );
+
+    dsp = (dsp >> 23) & 0x1;
+
+    if ((dsp != resdsp) || (rt != res)) {
+        printf("dextr_r.l error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dextr_r_w.c b/tests/tcg/mips/mips64-dsp/dextr_r_w.c
new file mode 100644
index 000000000..2774e9bfc
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dextr_r_w.c
@@ -0,0 +1,54 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, dsp;
+    long long achi, acli;
+    long long res, resdsp;
+
+    achi = 0x87654321;
+    acli = 0x12345678;
+
+    res = 0x123456;
+    resdsp = 0x01;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "dextr_r.w %0, $ac1, 0x8\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(achi), "r"(acli)
+        );
+
+    dsp = (dsp >> 23) & 0x1;
+
+    if ((dsp != resdsp) || (rt != res)) {
+        printf("dextr_r.w error\n");
+        return -1;
+    }
+
+    achi = 0x87654321;
+    acli = 0x12345678;
+
+    res = 0x12345678;
+    resdsp = 0x01;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "dextr_r.w %0, $ac1, 0x0\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(achi), "r"(acli)
+        );
+
+    dsp = (dsp >> 23) & 0x1;
+
+    if ((dsp != resdsp) || (rt != res)) {
+        printf("dextr_r.w error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dextr_rs_l.c b/tests/tcg/mips/mips64-dsp/dextr_rs_l.c
new file mode 100644
index 000000000..1a202fefa
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dextr_rs_l.c
@@ -0,0 +1,52 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, dsp;
+    long long achi, acli;
+    long long res, resdsp;
+
+    achi = 0x87654321;
+    acli = 0x12345678;
+
+    res = 0x8000000000000000;
+    resdsp = 0x1;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "dextr_rs.l %0, $ac1, 0x8\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(achi), "r"(acli)
+        );
+    dsp = (dsp >> 23) & 0x1;
+
+    if ((dsp != resdsp) || (rt != res)) {
+        printf("dextr_rs.l error\n");
+        return -1;
+    }
+
+    achi = 0x00;
+    acli = 0x12345678;
+
+    res = 0x12345678;
+    resdsp = 0x1;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "dextr_rs.l %0, $ac1, 0x0\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(achi), "r"(acli)
+        );
+    dsp = (dsp >> 23) & 0x1;
+
+    if ((dsp != resdsp) || (rt != res)) {
+        printf("dextr_rs.l error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dextr_rs_w.c b/tests/tcg/mips/mips64-dsp/dextr_rs_w.c
new file mode 100644
index 000000000..ebe5f99db
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dextr_rs_w.c
@@ -0,0 +1,52 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, dsp;
+    long long achi, acli;
+    long long res, resdsp;
+
+    achi = 0x87654321;
+    acli = 0x12345678;
+
+    res = 0xffffffff80000000;
+    resdsp = 0x1;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "dextr_rs.w %0, $ac1, 0x8\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(achi), "r"(acli)
+        );
+    dsp = (dsp >> 23) & 0x1;
+
+    if ((dsp != resdsp) || (rt != res)) {
+        printf("dextr_rs.w error\n");
+        return -1;
+    }
+
+    achi = 0x00;
+    acli = 0x12345678;
+
+    res = 0x123456;
+    resdsp = 0x1;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "dextr_rs.w %0, $ac1, 0x8\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(achi), "r"(acli)
+        );
+    dsp = (dsp >> 23) & 0x1;
+
+    if ((dsp != resdsp) || (rt != res)) {
+        printf("dextr_rs.w error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dextr_s_h.c b/tests/tcg/mips/mips64-dsp/dextr_s_h.c
new file mode 100644
index 000000000..1adb5549a
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dextr_s_h.c
@@ -0,0 +1,73 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, dsp;
+    long long achi, acli;
+    long long res, resdsp;
+
+    achi = 0x87654321;
+    acli = 0x12345678;
+
+    res = 0xffffffffffff8000;
+    resdsp = 0x1;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "dextr_s.h %0, $ac1, 0x8\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(achi), "r"(acli)
+        );
+    dsp = (dsp >> 23) & 0x1;
+
+    if ((dsp != resdsp) || (rt != res)) {
+        printf("1 dextr_s.h error\n");
+        return -1;
+    }
+
+    achi = 0x77654321;
+    acli = 0x12345678;
+
+    res = 0x7fff;
+    resdsp = 0x1;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "dextr_s.h %0, $ac1, 0x8\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(achi), "r"(acli)
+        );
+    dsp = (dsp >> 23) & 0x1;
+
+    if ((dsp != resdsp) || (rt != res)) {
+        printf("2 dextr_s.h error\n");
+        return -1;
+    }
+
+    achi = 0x00;
+    acli = 0x78;
+
+    res = 0x7;
+    resdsp = 0x1;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "dextr_s.h %0, $ac1, 0x4\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(achi), "r"(acli)
+        );
+    dsp = (dsp >> 23) & 0x1;
+
+    if ((dsp != resdsp) || (rt != res)) {
+        printf("3 dextr_s.h error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dextr_w.c b/tests/tcg/mips/mips64-dsp/dextr_w.c
new file mode 100644
index 000000000..79bed5da3
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dextr_w.c
@@ -0,0 +1,44 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt;
+    long long achi, acli;
+    long long res;
+
+    achi = 0x87654321;
+    acli = 0x12345678;
+
+    res = 0x123456;
+
+    __asm
+        ("mthi %1, $ac1\n\t"
+         "mtlo %2, $ac1\n\t"
+         "dextr.w %0, $ac1, 0x8\n\t"
+         : "=r"(rt)
+         : "r"(achi), "r"(acli)
+        );
+    if (rt != res) {
+        printf("dextr.w error\n");
+        return -1;
+    }
+
+    achi = 0x87654321;
+    acli = 0x12345678;
+
+    res = 0x12345678;
+
+    __asm
+        ("mthi %1, $ac1\n\t"
+         "mtlo %2, $ac1\n\t"
+         "dextr.w %0, $ac1, 0x0\n\t"
+         : "=r"(rt)
+         : "r"(achi), "r"(acli)
+        );
+    if (rt != res) {
+        printf("dextr.w error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dextrv_l.c b/tests/tcg/mips/mips64-dsp/dextrv_l.c
new file mode 100644
index 000000000..2e6187f72
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dextrv_l.c
@@ -0,0 +1,46 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, rs;
+    long long achi, acli;
+    long long res;
+
+    achi = 0x87654321;
+    acli = 0x12345678;
+    rs = 0x8;
+
+    res = 0x2100000000123456;
+
+    __asm
+        ("mthi %1, $ac1\n\t"
+         "mtlo %2, $ac1\n\t"
+         "dextrv.l %0, $ac1, %3\n\t"
+         : "=r"(rt)
+         : "r"(achi), "r"(acli), "r"(rs)
+        );
+    if (rt != res) {
+        printf("dextrv.l error\n");
+        return -1;
+    }
+
+    achi = 0x87654321;
+    acli = 0x12345678;
+    rs = 0x0;
+
+    res = 0x12345678;
+
+    __asm
+        ("mthi %1, $ac1\n\t"
+         "mtlo %2, $ac1\n\t"
+         "dextrv.l %0, $ac1, %3\n\t"
+         : "=r"(rt)
+         : "r"(achi), "r"(acli), "r"(rs)
+        );
+    if (rt != res) {
+        printf("dextrv.l error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dextrv_r_l.c b/tests/tcg/mips/mips64-dsp/dextrv_r_l.c
new file mode 100644
index 000000000..b47a0177d
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dextrv_r_l.c
@@ -0,0 +1,56 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, dsp, rs;
+    long long achi, acli;
+    long long res, resdsp;
+
+    achi = 0x87654321;
+    acli = 0x12345678;
+    rs = 0x8;
+
+    res = 0x2100000000123456;
+    resdsp = 0x01;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "dextrv_r.l %0, $ac1, %4\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rs)
+        );
+
+    dsp = (dsp >> 23) & 0x1;
+
+    if ((dsp != resdsp) || (rt != res)) {
+        printf("dextrv_r.l error\n");
+        return -1;
+    }
+
+    achi = 0x87654321;
+    acli = 0x12345678;
+    rs = 0x0;
+
+    res = 0x12345678;
+    resdsp = 0x01;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "dextrv_r.l %0, $ac1, %4\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rs)
+        );
+
+    dsp = (dsp >> 23) & 0x1;
+
+    if ((dsp != resdsp) || (rt != res)) {
+        printf("dextrv_r.l error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dextrv_r_w.c b/tests/tcg/mips/mips64-dsp/dextrv_r_w.c
new file mode 100644
index 000000000..cd201deb2
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dextrv_r_w.c
@@ -0,0 +1,56 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, rs, dsp;
+    long long achi, acli;
+    long long res, resdsp;
+
+    achi = 0x87654321;
+    acli = 0x12345678;
+    rs = 0x8;
+
+    res = 0x123456;
+    resdsp = 0x01;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "dextrv_r.w %0, $ac1, %4\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rs)
+        );
+
+    dsp = (dsp >> 23) & 0x1;
+
+    if ((dsp != resdsp) || (rt != res)) {
+        printf("dextrv_r.w error\n");
+        return -1;
+    }
+
+    achi = 0x87654321;
+    acli = 0x12345678;
+    rs = 0x0;
+
+    res = 0x12345678;
+    resdsp = 0x01;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "dextrv_r.w %0, $ac1, %4\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rs)
+        );
+
+    dsp = (dsp >> 23) & 0x1;
+
+    if ((dsp != resdsp) || (rt != res)) {
+        printf("dextrv_r.w error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dextrv_rs_l.c b/tests/tcg/mips/mips64-dsp/dextrv_rs_l.c
new file mode 100644
index 000000000..6ce418546
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dextrv_rs_l.c
@@ -0,0 +1,54 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, rs, dsp;
+    long long achi, acli;
+    long long res, resdsp;
+
+    achi = 0x87654321;
+    acli = 0x12345678;
+    rs = 0x8;
+
+    res = 0x8000000000000000;
+    resdsp = 0x1;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "dextrv_rs.l %0, $ac1, %4\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rs)
+        );
+    dsp = (dsp >> 23) & 0x1;
+
+    if ((dsp != resdsp) || (rt != res)) {
+        printf("dextrv_rs.l error\n");
+        return -1;
+    }
+
+    achi = 0x00;
+    acli = 0x12345678;
+    rs = 0x0;
+
+    res = 0x12345678;
+    resdsp = 0x1;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "dextrv_rs.l %0, $ac1, %4\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rs)
+        );
+    dsp = (dsp >> 23) & 0x1;
+
+    if ((dsp != resdsp) || (rt != res)) {
+        printf("dextrv_rs.l error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dextrv_rs_w.c b/tests/tcg/mips/mips64-dsp/dextrv_rs_w.c
new file mode 100644
index 000000000..a65183c03
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dextrv_rs_w.c
@@ -0,0 +1,54 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, rs, dsp;
+    long long achi, acli;
+    long long res, resdsp;
+
+    achi = 0x87654321;
+    acli = 0x12345678;
+    rs = 0x8;
+
+    res = 0xffffffff80000000;
+    resdsp = 0x1;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "dextrv_rs.w %0, $ac1, %4\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rs)
+        );
+    dsp = (dsp >> 23) & 0x1;
+
+    if ((dsp != resdsp) || (rt != res)) {
+        printf("dextrv_rs.w error\n");
+        return -1;
+    }
+
+    achi = 0x00;
+    acli = 0x12345678;
+    rs = 0x8;
+
+    res = 0x123456;
+    resdsp = 0x1;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "dextrv_rs.w %0, $ac1, %4\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rs)
+        );
+    dsp = (dsp >> 23) & 0x1;
+
+    if ((dsp != resdsp) || (rt != res)) {
+        printf("dextrv_rs.w error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dextrv_s_h.c b/tests/tcg/mips/mips64-dsp/dextrv_s_h.c
new file mode 100644
index 000000000..87d3aeedc
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dextrv_s_h.c
@@ -0,0 +1,32 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, rs, dsp;
+    long long achi, acli;
+    long long res, resdsp;
+
+    achi = 0x87654321;
+    acli = 0x12345678;
+    rs = 0x8;
+
+    res = 0xffffffffffff8000;
+    resdsp = 0x1;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "dextrv_s.h %0, $ac1, %4\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rs)
+        );
+    dsp = (dsp >> 23) & 0x1;
+
+    if ((dsp != resdsp) || (rt != res)) {
+        printf("dextrv_s.h error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dextrv_w.c b/tests/tcg/mips/mips64-dsp/dextrv_w.c
new file mode 100644
index 000000000..973765c1c
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dextrv_w.c
@@ -0,0 +1,46 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, rs;
+    long long achi, acli;
+    long long res;
+
+    achi = 0x87654321;
+    acli = 0x12345678;
+    rs = 0x8;
+
+    res = 0x123456;
+
+    __asm
+        ("mthi %1, $ac1\n\t"
+         "mtlo %2, $ac1\n\t"
+         "dextrv.w %0, $ac1, %3\n\t"
+         : "=r"(rt)
+         : "r"(achi), "r"(acli), "r"(rs)
+        );
+    if (rt != res) {
+        printf("dextrv.w error\n");
+        return -1;
+    }
+
+    achi = 0x87654321;
+    acli = 0x12345678;
+    rs = 0x0;
+
+    res = 0x12345678;
+
+    __asm
+        ("mthi %1, $ac1\n\t"
+         "mtlo %2, $ac1\n\t"
+         "dextrv.w %0, $ac1, %3\n\t"
+         : "=r"(rt)
+         : "r"(achi), "r"(acli), "r"(rs)
+        );
+    if (rt != res) {
+        printf("dextrv.w error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dinsv.c b/tests/tcg/mips/mips64-dsp/dinsv.c
new file mode 100644
index 000000000..f6192188c
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dinsv.c
@@ -0,0 +1,26 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rs, rt, dsp;
+    long long res;
+
+    rs = 0x1234567887654321;
+    rt = 0x1234567812345678;
+    dsp = 0x2222;
+    res = 0x1234567812345678;
+    __asm
+        ("wrdsp  %1, 0x3\n\t"
+         "wrdsp %1\n\t"
+         "dinsv %0, %2\n\t"
+         : "+r"(rt)
+         : "r"(dsp), "r"(rs)
+        );
+
+    if (rt != res) {
+        printf("dinsv error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dmadd.c b/tests/tcg/mips/mips64-dsp/dmadd.c
new file mode 100644
index 000000000..fb2261472
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dmadd.c
@@ -0,0 +1,57 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, rs;
+    long long achi, acli;
+    long long acho, aclo;
+    long long resh, resl;
+
+    achi = 0x1;
+    acli = 0x1;
+
+    rs = 0x0000000100000001;
+    rt = 0x0000000200000002;
+
+    resh = 0x1;
+    resl = 0x5;
+    __asm
+       ("mthi %2, $ac1 \t\n"
+        "mtlo %3, $ac1 \t\n"
+        "dmadd $ac1, %4, %5\t\n"
+        "mfhi %0, $ac1 \t\n"
+        "mflo %1, $ac1 \t\n"
+        : "=r"(acho), "=r"(aclo)
+        : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+      );
+    if ((acho != resh) || (aclo != resl)) {
+        printf("1 dmadd error\n");
+
+        return -1;
+    }
+
+    achi = 0x1;
+    acli = 0x1;
+
+    rs = 0xaaaabbbbccccdddd;
+    rt = 0xaaaabbbbccccdddd;
+
+    resh = 0x0000000000000000;
+    resl = 0xffffffffca860b63;
+
+    __asm
+       ("mthi %2, $ac1 \t\n"
+        "mtlo %3, $ac1 \t\n"
+        "dmadd $ac1, %4, %5\t\n"
+        "mfhi %0, $ac1 \t\n"
+        "mflo %1, $ac1 \t\n"
+        : "=r"(acho), "=r"(aclo)
+        : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+      );
+    if ((acho != resh) || (aclo != resl)) {
+        printf("2 dmadd error\n");
+
+        return -1;
+    }
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dmaddu.c b/tests/tcg/mips/mips64-dsp/dmaddu.c
new file mode 100644
index 000000000..39ab0c10d
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dmaddu.c
@@ -0,0 +1,56 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, rs;
+    long long achi, acli;
+    long long acho, aclo;
+    long long resh, resl;
+    achi = 0x1;
+    acli = 0x2;
+
+    rs = 0x0000000200000002;
+    rt = 0x0000000200000002;
+    resh = 0x1;
+    resl = 0xa;
+    __asm
+        ("mthi %2, $ac1 \t\n"
+         "mtlo %3, $ac1 \t\n"
+         "dmaddu $ac1, %4, %5\t\n"
+         "mfhi %0, $ac1 \t\n"
+         "mflo %1, $ac1 \t\n"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+    if ((acho != resh) || (aclo != resl)) {
+        printf("1 dmaddu error\n");
+
+        return -1;
+    }
+
+    achi = 0x1;
+    acli = 0x1;
+
+    rs = 0xaaaabbbbccccdddd;
+    rt = 0xaaaabbbbccccdddd;
+
+    resh = 0x0000000000000002;
+    resl = 0xffffffffca860b63;
+
+    __asm
+        ("mthi %2, $ac1 \t\n"
+         "mtlo %3, $ac1 \t\n"
+         "dmaddu $ac1, %4, %5\t\n"
+         "mfhi %0, $ac1 \t\n"
+         "mflo %1, $ac1 \t\n"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+    if ((acho != resh) || (aclo != resl)) {
+        printf("2 dmaddu error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dmsub.c b/tests/tcg/mips/mips64-dsp/dmsub.c
new file mode 100644
index 000000000..16be6170e
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dmsub.c
@@ -0,0 +1,59 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, rs;
+    long long achi, acli;
+    long long acho, aclo;
+    long long resh, resl;
+    achi = 0x1;
+    acli = 0x8;
+
+    rs = 0x0000000100000001;
+    rt = 0x0000000200000002;
+
+    resh = 0x1;
+    resl = 0x4;
+
+    __asm
+        ("mthi %2, $ac1 \t\n"
+         "mtlo %3, $ac1 \t\n"
+         "dmsub $ac1, %4, %5\t\n"
+         "mfhi %0, $ac1 \t\n"
+         "mflo %1, $ac1 \t\n"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+    if ((acho != resh) || (aclo != resl)) {
+        printf("1 dmsub error\n");
+
+        return -1;
+    }
+
+    achi = 0xfffffffF;
+    acli = 0xfffffffF;
+
+    rs = 0x8888999977776666;
+    rt = 0x9999888877776666;
+
+    resh = 0xffffffffffffffff;
+    resl = 0x789aae13;
+
+    __asm
+        ("mthi %2, $ac1 \t\n"
+         "mtlo %3, $ac1 \t\n"
+         "dmsub $ac1, %4, %5\t\n"
+         "mfhi %0, $ac1 \t\n"
+         "mflo %1, $ac1 \t\n"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+
+    if ((acho != resh) || (aclo != resl)) {
+        printf("2 dmsub error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dmsubu.c b/tests/tcg/mips/mips64-dsp/dmsubu.c
new file mode 100644
index 000000000..cc4838ad5
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dmsubu.c
@@ -0,0 +1,59 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, rs;
+    long long achi, acli;
+    long long acho, aclo;
+    long long resh, resl;
+    achi = 0x1;
+    acli = 0x8;
+
+    rs = 0x0000000100000001;
+    rt = 0x0000000200000002;
+
+    resh = 0x1;
+    resl = 0x4;
+
+    __asm
+        ("mthi %2, $ac1 \t\n"
+         "mtlo %3, $ac1 \t\n"
+         "dmsubu $ac1, %4, %5\t\n"
+         "mfhi %0, $ac1 \t\n"
+         "mflo %1, $ac1 \t\n"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+    if ((acho != resh) || (aclo != resl)) {
+        printf("1 dmsubu error\n");
+
+        return -1;
+    }
+
+    achi = 0xfffffffF;
+    acli = 0xfffffffF;
+
+    rs = 0x8888999977776666;
+    rt = 0x9999888877776666;
+
+    resh = 0xffffffffffffffff;
+    resl = 0x789aae13;
+
+    __asm
+        ("mthi %2, $ac1 \t\n"
+         "mtlo %3, $ac1 \t\n"
+         "dmsubu $ac1, %4, %5\t\n"
+         "mfhi %0, $ac1 \t\n"
+         "mflo %1, $ac1 \t\n"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+
+    if ((acho != resh) || (aclo != resl)) {
+        printf("2 dmsubu error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dmthlip.c b/tests/tcg/mips/mips64-dsp/dmthlip.c
new file mode 100644
index 000000000..027555fb5
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dmthlip.c
@@ -0,0 +1,41 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rs, dsp;
+    long long achi, acli;
+
+    long long rsdsp;
+    long long acho, aclo;
+
+    long long res;
+    long long reshi, reslo;
+
+
+    rs = 0xaaaabbbbccccdddd;
+    achi = 0x87654321;
+    acli = 0x12345678;
+    dsp = 0x22;
+
+    res = 0x62;
+    reshi = 0x12345678;
+    reslo = 0xffffffffccccdddd;
+
+    __asm
+        ("mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "wrdsp %5\n\t"
+         "dmthlip %6, $ac1\n\t"
+         "rddsp %0\n\t"
+         "mfhi %1, $ac1\n\t"
+         "mflo %2, $ac1\n\t"
+         : "=r"(rsdsp), "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(dsp), "r"(rs)
+        );
+    if ((rsdsp != res) || (acho != reshi) || (aclo != reslo)) {
+        printf("dmthlip error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dpaq_s_w_ph.c b/tests/tcg/mips/mips64-dsp/dpaq_s_w_ph.c
new file mode 100644
index 000000000..1bca93500
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dpaq_s_w_ph.c
@@ -0,0 +1,32 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rs, rt, dsp;
+    long long ach = 0, acl = 0;
+    long long resulth, resultl, resultdsp;
+
+    rs        = 0x800000FF;
+    rt        = 0x80000002;
+    resulth   = 0x00;
+    resultl   = 0xFFFFFFFF800003FB;
+    resultdsp = 0x01;
+    __asm
+        ("mthi        %0, $ac1\n\t"
+         "mtlo        %1, $ac1\n\t"
+         "dpaq_s.w.ph $ac1, %3, %4\n\t"
+         "mfhi        %0,   $ac1\n\t"
+         "mflo        %1,   $ac1\n\t"
+         "rddsp       %2\n\t"
+         : "+r"(ach), "+r"(acl), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = dsp >> 17 & 0x01;
+    if ((dsp != resultdsp) || (ach != resulth) || (acl != resultl)) {
+        printf("dpaq_w.w.ph wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dpaq_s_w_qh.c b/tests/tcg/mips/mips64-dsp/dpaq_s_w_qh.c
new file mode 100644
index 000000000..844a34742
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dpaq_s_w_qh.c
@@ -0,0 +1,57 @@
+#include"io.h"
+int main(void)
+{
+    long long rt, rs;
+    long long achi, acli;
+    long long acho, aclo;
+    long long resh, resl;
+
+    achi = 0x1;
+    acli = 0x1;
+    rs = 0x0001000100010001;
+    rt = 0x0002000200020002;
+    resh = 0x1;
+    resl = 0x11;
+
+    __asm
+        ("mthi %2, $ac1\t\n"
+         "mtlo %3, $ac1\t\n"
+         "dpaq_s.w.qh $ac1, %4, %5\t\n"
+         "mfhi %0, $ac1\t\n"
+         "mflo %1, $ac1\t\n"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+    if ((acho != resh) || (aclo != resl)) {
+        printf("1 dpaq_s.w.qh error\n");
+
+        return -1;
+    }
+
+    achi = 0xffffffff;
+    acli = 0xaaaaaaaa;
+
+    rs = 0x1111222233334444;
+    rt = 0xffffeeeeddddcccc;
+
+    resh = 0x00;
+    resl = 0xffffffffd27ad82e;
+
+    __asm
+        ("mthi %2, $ac1\t\n"
+         "mtlo %3, $ac1\t\n"
+         "dpaq_s.w.qh $ac1, %4, %5\t\n"
+         "mfhi %0, $ac1\t\n"
+         "mflo %1, $ac1\t\n"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+
+    if ((acho != resh) || (aclo != resl)) {
+        printf("2 dpaq_s.w.qh error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dpaq_sa_l_pw.c b/tests/tcg/mips/mips64-dsp/dpaq_sa_l_pw.c
new file mode 100644
index 000000000..1bb2ec2f2
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dpaq_sa_l_pw.c
@@ -0,0 +1,88 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rs, rt;
+    long long achi, acli;
+    long long acho, aclo;
+    long long dsp;
+    long long resh, resl;
+    long long resdsp;
+
+    rs = 0x0000000100000001;
+    rt = 0x0000000200000002;
+    achi = 0x1;
+    acli = 0x1;
+    resh = 0xffffffffffffffff;
+    resl = 0x0;
+    resdsp = 0x01;
+
+    __asm
+        ("mthi        %3, $ac1\n\t"
+         "mtlo        %4, $ac1\n\t"
+         "dpaq_sa.l.pw $ac1, %5, %6\n\t"
+         "mfhi        %0,   $ac1\n\t"
+         "mflo        %1,   $ac1\n\t"
+         "rddsp       %2\n\t"
+         : "=r"(acho), "=r"(aclo), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+
+    if ((acho != resh) || (aclo != resl) || ((dsp >> (16 + 1)) != resdsp)) {
+        printf("1 dpaq_sa_l_pw error\n");
+
+        return -1;
+    }
+
+    rs = 0xaaaabbbbccccdddd;
+    rt = 0x3333444455556666;
+    achi = 0x88888888;
+    acli = 0x66666666;
+
+    resh = 0xffffffff88888887;
+    resl = 0xffffffff9e2661da;
+
+    __asm
+        ("mthi        %2, $ac1\n\t"
+         "mtlo        %3, $ac1\n\t"
+         "dpaq_sa.l.pw $ac1, %4, %5\n\t"
+         "mfhi        %0,   $ac1\n\t"
+         "mflo        %1,   $ac1\n\t"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+
+    if ((acho != resh) || (aclo != resl)) {
+        printf("2 dpaq_sa_l_pw error\n");
+
+        return -1;
+    }
+
+    rs = 0x8000000080000000;
+    rt = 0x8000000080000000;
+    achi = 0x88888888;
+    acli = 0x66666666;
+
+    resh = 0xffffffffffffffff;
+    resl = 0x00;
+    resdsp = 0x01;
+
+    __asm
+        ("mthi        %3, $ac1\n\t"
+         "mtlo        %4, $ac1\n\t"
+         "dpaq_sa.l.pw $ac1, %5, %6\n\t"
+         "mfhi        %0,   $ac1\n\t"
+         "mflo        %1,   $ac1\n\t"
+         "rddsp       %2\n\t"
+         : "=r"(acho), "=r"(aclo), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+
+    if ((acho != resh) || (aclo != resl) || ((dsp >> (16 + 1)) != resdsp)) {
+        printf("2 dpaq_sa_l_pw error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dpaq_sa_l_w.c b/tests/tcg/mips/mips64-dsp/dpaq_sa_l_w.c
new file mode 100644
index 000000000..f840cdd76
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dpaq_sa_l_w.c
@@ -0,0 +1,82 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rs, rt, dsp;
+    long long ach = 0, acl = 0;
+    long long resulth, resultl, resultdsp;
+
+    rs        = 0x80000000;
+    rt        = 0x80000000;
+    resulth   = 0x7FFFFFFF;
+    resultl   = 0xffffffffFFFFFFFF;
+    resultdsp = 0x01;
+    __asm
+        ("mthi        %0, $ac1\n\t"
+         "mtlo        %0, $ac1\n\t"
+         "dpaq_sa.l.w $ac1, %3, %4\n\t"
+         "mfhi        %0,   $ac1\n\t"
+         "mflo        %1,   $ac1\n\t"
+         "rddsp       %2\n\t"
+         : "+r"(ach), "+r"(acl), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 17) & 0x01;
+    if ((dsp != resultdsp) || (ach != resulth) || (acl != resultl)) {
+        printf("dpaq_sa.l.w error\n");
+
+        return -1;
+    }
+
+    ach = 0x12;
+    acl = 0x48;
+    rs  = 0x80000000;
+    rt  = 0x80000000;
+
+    resulth   = 0x7FFFFFFF;
+    resultl   = 0xffffffffFFFFFFFF;
+    resultdsp = 0x01;
+    __asm
+        ("mthi        %0, $ac1\n\t"
+         "mtlo        %0, $ac1\n\t"
+         "dpaq_sa.l.w $ac1, %3, %4\n\t"
+         "mfhi        %0,   $ac1\n\t"
+         "mflo        %1,   $ac1\n\t"
+         "rddsp       %2\n\t"
+         : "+r"(ach), "+r"(acl), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 17) & 0x01;
+    if ((dsp != resultdsp) || (ach != resulth) || (acl != resultl)) {
+        printf("dpaq_sa.l.w error\n");
+
+        return -1;
+    }
+
+    ach = 0x741532A0;
+    acl = 0xfceabb08;
+    rs  = 0x80000000;
+    rt  = 0x80000000;
+
+    resulth   = 0x7fffffff;
+    resultl   = 0xffffffffffffffff;
+    resultdsp = 0x01;
+    __asm
+        ("mthi        %0, $ac1\n\t"
+         "mtlo        %0, $ac1\n\t"
+         "dpaq_sa.l.w $ac1, %3, %4\n\t"
+         "mfhi        %0,   $ac1\n\t"
+         "mflo        %1,   $ac1\n\t"
+         "rddsp       %2\n\t"
+         : "+r"(ach), "+r"(acl), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 17) & 0x01;
+    if ((dsp != resultdsp) || (ach != resulth) || (acl != resultl)) {
+        printf("dpaq_sa.l.w error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dpau_h_obl.c b/tests/tcg/mips/mips64-dsp/dpau_h_obl.c
new file mode 100644
index 000000000..54905e8f9
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dpau_h_obl.c
@@ -0,0 +1,59 @@
+
+#include "io.h"
+
+int main(void)
+{
+    long long rs, rt;
+    long long achi, acli;
+    long long acho, aclo;
+    long long resh, resl;
+
+    rs = 0x0000000100000001;
+    rt = 0x0000000200000002;
+    achi = 0x1;
+    acli = 0x1;
+    resh = 0x1;
+    resl = 0x3;
+
+    __asm
+        ("mthi        %2, $ac1\n\t"
+         "mtlo        %3, $ac1\n\t"
+         "dpau.h.obl $ac1, %4, %5\n\t"
+         "mfhi        %0,   $ac1\n\t"
+         "mflo        %1,   $ac1\n\t"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+
+    if ((acho != resh) || (aclo != resl)) {
+        printf("1 dpau.h.obl error\n");
+
+        return -1;
+    }
+
+    rs = 0xaaaabbbbccccdddd;
+    rt = 0x3333444455556666;
+    achi = 0x88888888;
+    acli = 0x66666666;
+
+    resh = 0xffffffff88888888;
+    resl = 0x66670d7a;
+
+    __asm
+        ("mthi        %2, $ac1\n\t"
+         "mtlo        %3, $ac1\n\t"
+         "dpau.h.obl $ac1, %4, %5\n\t"
+         "mfhi        %0,   $ac1\n\t"
+         "mflo        %1,   $ac1\n\t"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+
+    if ((acho != resh) || (aclo != resl)) {
+        printf("1 dpau.h.obl error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dpau_h_obr.c b/tests/tcg/mips/mips64-dsp/dpau_h_obr.c
new file mode 100644
index 000000000..d7aa60b4b
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dpau_h_obr.c
@@ -0,0 +1,59 @@
+
+#include "io.h"
+
+int main(void)
+{
+    long long rs, rt;
+    long long achi, acli;
+    long long acho, aclo;
+    long long resh, resl;
+
+    rs = 0x0000000100000001;
+    rt = 0x0000000200000002;
+    achi = 0x1;
+    acli = 0x1;
+    resh = 0x1;
+    resl = 0x3;
+
+    __asm
+        ("mthi        %2, $ac1\n\t"
+         "mtlo        %3, $ac1\n\t"
+         "dpau.h.obr $ac1, %4, %5\n\t"
+         "mfhi        %0,   $ac1\n\t"
+         "mflo        %1,   $ac1\n\t"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+
+    if ((acho != resh) || (aclo != resl)) {
+        printf("1 dpau.h.obr error\n");
+
+        return -1;
+    }
+
+    rs = 0xccccddddaaaabbbb;
+    rt = 0x5555666633334444;
+    achi = 0x88888888;
+    acli = 0x66666666;
+
+    resh = 0xffffffff88888888;
+    resl = 0x66670d7a;
+
+    __asm
+        ("mthi        %2, $ac1\n\t"
+         "mtlo        %3, $ac1\n\t"
+         "dpau.h.obr $ac1, %4, %5\n\t"
+         "mfhi        %0,   $ac1\n\t"
+         "mflo        %1,   $ac1\n\t"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+
+    if ((acho != resh) || (aclo != resl)) {
+        printf("1 dpau.h.obr error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dpau_h_qbl.c b/tests/tcg/mips/mips64-dsp/dpau_h_qbl.c
new file mode 100644
index 000000000..fcfd76431
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dpau_h_qbl.c
@@ -0,0 +1,29 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rs, rt;
+    long long ach = 5, acl = 3;
+    long long resulth, resultl;
+
+    rs        = 0x800000FF;
+    rt        = 0x80000002;
+    resulth   = 0x05;
+    resultl   = 0x4003;
+    __asm
+        ("mthi       %0, $ac1\n\t"
+         "mtlo       %1, $ac1\n\t"
+         "dpau.h.qbl $ac1, %2, %3\n\t"
+         "mfhi       %0,   $ac1\n\t"
+         "mflo       %1,   $ac1\n\t"
+         : "+r"(ach), "+r"(acl)
+         : "r"(rs), "r"(rt)
+        );
+    if ((ach != resulth) || (acl != resultl)) {
+        printf("dpau.h.qbl wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dpau_h_qbr.c b/tests/tcg/mips/mips64-dsp/dpau_h_qbr.c
new file mode 100644
index 000000000..3282461a7
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dpau_h_qbr.c
@@ -0,0 +1,29 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rs, rt;
+    long long ach = 5, acl = 3;
+    long long resulth, resultl;
+
+    rs        = 0x800000FF;
+    rt        = 0x80000002;
+    resulth   = 0x05;
+    resultl   = 0x0201;
+    __asm
+        ("mthi       %0, $ac1\n\t"
+         "mtlo       %1, $ac1\n\t"
+         "dpau.h.qbr $ac1, %2, %3\n\t"
+         "mfhi       %0,   $ac1\n\t"
+         "mflo       %1,   $ac1\n\t"
+         : "+r"(ach), "+r"(acl)
+         : "r"(rs), "r"(rt)
+        );
+    if ((ach != resulth) || (acl != resultl)) {
+        printf("dpau.h.qbr wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dpsq_s_w_ph.c b/tests/tcg/mips/mips64-dsp/dpsq_s_w_ph.c
new file mode 100644
index 000000000..7660f037d
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dpsq_s_w_ph.c
@@ -0,0 +1,51 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rs, rt;
+    long long ach = 5, acl = 5;
+    long long resulth, resultl;
+
+    rs      = 0xBC0123AD;
+    rt      = 0x01643721;
+    resulth = 0x04;
+    resultl = 0xFFFFFFFFEE9794A3;
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpsq_s.w.ph $ac1, %2, %3\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         : "+r"(ach), "+r"(acl)
+         : "r"(rs), "r"(rt)
+        );
+    if ((ach != resulth) || (acl != resultl)) {
+        printf("1 dpsq_s.w.ph wrong\n");
+
+        return -1;
+    }
+
+    ach = 0x1424Ef1f;
+    acl = 0x1035219A;
+    rs      = 0x800083AD;
+    rt      = 0x80003721;
+    resulth = 0x1424ef1e;
+    resultl = 0x577ed901;
+
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpsq_s.w.ph $ac1, %2, %3\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         : "+r"(ach), "+r"(acl)
+         : "r"(rs), "r"(rt)
+        );
+    if ((ach != resulth) || (acl != resultl)) {
+        printf("2 dpsq_s.w.ph wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dpsq_s_w_qh.c b/tests/tcg/mips/mips64-dsp/dpsq_s_w_qh.c
new file mode 100644
index 000000000..2cc50c577
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dpsq_s_w_qh.c
@@ -0,0 +1,56 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rs, rt;
+    long long achi, acli;
+    long long acho, aclo;
+    long long resh, resl;
+
+    rs = 0xffffeeeeddddcccc;
+    rt = 0x9999888877776666;
+    achi = 0x67576;
+    acli = 0x98878;
+
+    resh = 0x67576;
+    resl = 0x5b1682c4;
+    __asm
+        ("mthi  %2, $ac1\n\t"
+         "mtlo  %3, $ac1\n\t"
+         "dpsq_s.w.qh $ac1, %4, %5\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+    if ((acho != resh) || (aclo != resl)) {
+        printf("1 dpsq_s.w.qh wrong\n");
+
+        return -1;
+    }
+
+    rs = 0x8000800080008000;
+    rt = 0x8000800080008000;
+    achi = 0x67576;
+    acli = 0x98878;
+
+    resh = 0x67575;
+    resl = 0x0009887c;
+
+    __asm
+        ("mthi  %2, $ac1\n\t"
+         "mtlo  %3, $ac1\n\t"
+         "dpsq_s.w.qh $ac1, %4, %5\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+    if ((acho != resh) || (aclo != resl)) {
+        printf("2 dpsq_s.w.qh wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dpsq_sa_l_pw.c b/tests/tcg/mips/mips64-dsp/dpsq_sa_l_pw.c
new file mode 100644
index 000000000..7fc2503fc
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dpsq_sa_l_pw.c
@@ -0,0 +1,76 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rs, rt, dsp;
+    long long achi, acli;
+    long long resh, resl, resdsp;
+
+    rs = 0x89789BC0123AD;
+    rt = 0x5467591643721;
+
+    achi = 0x98765437;
+    acli = 0x65489709;
+
+    resh = 0xffffffffffffffff;
+    resl = 0x00;
+
+    resdsp = 0x01;
+
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpsq_sa.l.pw $ac1, %3, %4\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "+r"(achi), "+r"(acli), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+
+    dsp = (dsp >> 17) & 0x01;
+    if ((dsp != resdsp) || (achi != resh) || (acli != resl)) {
+        printf("1 dpsq_sa.l.pw wrong\n");
+
+        return -1;
+    }
+
+    /* clear dspcontrol reg for next test use. */
+    dsp = 0;
+    __asm
+        ("wrdsp %0"
+         :
+         : "r"(dsp)
+        );
+
+    rs = 0x8B78980000000;
+    rt = 0x5867580000000;
+
+    achi = 0x98765437;
+    acli = 0x65489709;
+
+    resh = 0xffffffff98765436;
+    resl = 0x11d367d0;
+
+    resdsp = 0x01;
+
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpsq_sa.l.pw $ac1, %3, %4\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "+r"(achi), "+r"(acli), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+
+    dsp = (dsp >> 17) & 0x01;
+    if ((dsp != resdsp) || (achi != resh) || (acli != resl)) {
+        printf("2 dpsq_sa.l.pw wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dpsq_sa_l_w.c b/tests/tcg/mips/mips64-dsp/dpsq_sa_l_w.c
new file mode 100644
index 000000000..f55afc909
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dpsq_sa_l_w.c
@@ -0,0 +1,59 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rs, rt, dsp;
+    long long ach = 5, acl = 5;
+    long long resulth, resultl, resultdsp;
+
+    rs = 0xBC0123AD;
+    rt = 0x01643721;
+
+    resulth = 0xfffffffffdf4cbe0;
+    resultl = 0xFFFFFFFFd138776b;
+    resultdsp = 0x00;
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpsq_sa.l.w $ac1, %3, %4\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "+r"(ach), "+r"(acl), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 17) & 0x01;
+    if ((dsp != resultdsp) || (ach != resulth) || (acl != resultl)) {
+        printf("1 dpsq_sa.l.w wrong\n");
+
+        return -1;
+    }
+
+    ach = 0x54321123;
+    acl = 5;
+    rs = 0x80000000;
+    rt = 0x80000000;
+
+    resulth = 0xffffffffd4321123;
+    resultl = 0x06;
+    resultdsp = 0x01;
+
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpsq_sa.l.w $ac1, %3, %4\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "+r"(ach), "+r"(acl), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 17) & 0x01;
+    if ((dsp != resultdsp) || (ach != resulth) || (acl != resultl)) {
+        printf("2 dpsq_sa.l.w wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dpsu_h_obl.c b/tests/tcg/mips/mips64-dsp/dpsu_h_obl.c
new file mode 100644
index 000000000..c0a8f4d7a
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dpsu_h_obl.c
@@ -0,0 +1,32 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rs, rt;
+    long long ach = 5, acl = 5;
+    long long resulth, resultl;
+
+    rs      = 0x88886666BC0123AD;
+    rt      = 0x9999888801643721;
+
+    resulth = 0x04;
+    resultl = 0xFFFFFFFFFFFEF115;
+
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpsu.h.obl $ac1, %2, %3\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         : "+r"(ach), "+r"(acl)
+         : "r"(rs), "r"(rt)
+        );
+
+    if ((ach != resulth) || (acl != resultl)) {
+        printf("dpsu.h.obl wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dpsu_h_obr.c b/tests/tcg/mips/mips64-dsp/dpsu_h_obr.c
new file mode 100644
index 000000000..aa0d47a06
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dpsu_h_obr.c
@@ -0,0 +1,32 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rs, rt;
+    long long ach = 5, acl = 5;
+    long long resulth, resultl;
+
+    rs      = 0x7878878888886666;
+    rt      = 0x9865454399998888;
+
+    resulth = 0x04;
+    resultl = 0xFFFFFFFFFFFeF115;
+
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpsu.h.obr $ac1, %2, %3\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         : "+r"(ach), "+r"(acl)
+         : "r"(rs), "r"(rt)
+        );
+
+    if ((ach != resulth) || (acl != resultl)) {
+        printf("dpsu.h.qbr wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dpsu_h_qbl.c b/tests/tcg/mips/mips64-dsp/dpsu_h_qbl.c
new file mode 100644
index 000000000..da6dbb615
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dpsu_h_qbl.c
@@ -0,0 +1,29 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rs, rt;
+    long long ach = 5, acl = 5;
+    long long resulth, resultl;
+
+    rs      = 0xBC0123AD;
+    rt      = 0x01643721;
+    resulth = 0x04;
+    resultl = 0xFFFFFFFFFFFFFEE5;
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpsu.h.qbl $ac1, %2, %3\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         : "+r"(ach), "+r"(acl)
+         : "r"(rs), "r"(rt)
+        );
+    if ((ach != resulth) || (acl != resultl)) {
+        printf("dpsu.h.qbl wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dpsu_h_qbr.c b/tests/tcg/mips/mips64-dsp/dpsu_h_qbr.c
new file mode 100644
index 000000000..bf00b70aa
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dpsu_h_qbr.c
@@ -0,0 +1,29 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rs, rt;
+    long long ach = 5, acl = 5;
+    long long resulth, resultl;
+
+    rs      = 0xBC0123AD;
+    rt      = 0x01643721;
+    resulth = 0x04;
+    resultl = 0xFFFFFFFFFFFFE233;
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpsu.h.qbr $ac1, %2, %3\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         : "+r"(ach), "+r"(acl)
+         : "r"(rs), "r"(rt)
+        );
+    if ((ach != resulth) || (acl != resultl)) {
+        printf("dpsu.h.qbr wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dshilo.c b/tests/tcg/mips/mips64-dsp/dshilo.c
new file mode 100644
index 000000000..f50584b9c
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dshilo.c
@@ -0,0 +1,52 @@
+#include "io.h"
+
+int main(void)
+{
+    long long achi, acli;
+    long long acho, aclo;
+    long long reshi, reslo;
+
+    achi = 0x87654321;
+    acli = 0x12345678;
+
+    reshi = 0xfffffffff8765432;
+    reslo = 0x1234567;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "dshilo $ac1, 0x4\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli)
+        );
+
+    if ((acho != reshi) || (aclo != reslo)) {
+        printf("1 dshilo error\n");
+        return -1;
+    }
+
+    achi = 0x87654321;
+    acli = 0x12345678;
+
+    reshi = 0x1234567;
+    reslo = 0x00;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "dshilo $ac1, -60\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli)
+        );
+
+    if ((acho != reshi) || (aclo != reslo)) {
+        printf("2 dshilo error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/dshilov.c b/tests/tcg/mips/mips64-dsp/dshilov.c
new file mode 100644
index 000000000..792bd2373
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/dshilov.c
@@ -0,0 +1,54 @@
+#include "io.h"
+
+int main(void)
+{
+    long long achi, acli, rs;
+    long long acho, aclo;
+    long long reshi, reslo;
+
+    achi = 0x87654321;
+    acli = 0x12345678;
+    rs = 0x4;
+
+    reshi = 0xfffffffff8765432;
+    reslo = 0x1234567;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "dshilov $ac1, %4\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs)
+        );
+
+    if ((acho != reshi) || (aclo != reslo)) {
+        printf("dshilov error\n");
+        return -1;
+    }
+
+    rs = 0x44;
+    achi = 0x87654321;
+    acli = 0x12345678;
+
+    reshi = 0x1234567;
+    reslo = 0x00;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "dshilov $ac1, %4\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs)
+        );
+
+    if ((acho != reshi) || (aclo != reslo)) {
+        printf("dshilov error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/extp.c b/tests/tcg/mips/mips64-dsp/extp.c
new file mode 100644
index 000000000..c72f54bac
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/extp.c
@@ -0,0 +1,50 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, ach, acl, dsp;
+    long long result;
+
+    ach = 0x05;
+    acl = 0xB4CB;
+    dsp = 0x07;
+    result = 0x000C;
+
+    __asm
+        ("wrdsp %1, 0x01\n\t"
+         "mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "extp %0, $ac1, 0x03\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "+r"(dsp)
+         : "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 14) & 0x01;
+    if ((dsp != 0) || (result != rt)) {
+        printf("extp wrong\n");
+
+        return -1;
+    }
+
+    ach = 0x05;
+    acl = 0xB4CB;
+    dsp = 0x01;
+
+    __asm
+        ("wrdsp %1, 0x01\n\t"
+         "mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "extp %0, $ac1, 0x03\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "+r"(dsp)
+         : "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 14) & 0x01;
+    if (dsp != 1) {
+        printf("extp wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/extpdp.c b/tests/tcg/mips/mips64-dsp/extpdp.c
new file mode 100644
index 000000000..f43019384
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/extpdp.c
@@ -0,0 +1,51 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, ach, acl, dsp, pos, efi;
+    long long result;
+
+    ach = 0x05;
+    acl = 0xB4CB;
+    dsp = 0x07;
+    result = 0x000C;
+
+    __asm
+        ("wrdsp %1, 0x01\n\t"
+         "mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "extpdp %0, $ac1, 0x03\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "+r"(dsp)
+         : "r"(ach), "r"(acl)
+        );
+    pos =  dsp & 0x3F;
+    efi = (dsp >> 14) & 0x01;
+    if ((pos != 3) || (efi != 0) || (result != rt)) {
+        printf("extpdp wrong\n");
+
+        return -1;
+    }
+
+    ach = 0x05;
+    acl = 0xB4CB;
+    dsp = 0x01;
+
+    __asm
+        ("wrdsp %1, 0x01\n\t"
+         "mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "extpdp %0, $ac1, 0x03\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "+r"(dsp)
+         : "r"(ach), "r"(acl)
+        );
+    efi = (dsp >> 14) & 0x01;
+    if (efi != 1) {
+        printf("extpdp wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/extpdpv.c b/tests/tcg/mips/mips64-dsp/extpdpv.c
new file mode 100644
index 000000000..ba57426d2
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/extpdpv.c
@@ -0,0 +1,52 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, rs, ach, acl, dsp, pos, efi;
+    long long result;
+
+    ach = 0x05;
+    acl = 0xB4CB;
+    dsp = 0x07;
+    rs  = 0x03;
+    result = 0x000C;
+
+    __asm
+        ("wrdsp %1, 0x01\n\t"
+         "mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "extpdpv %0, $ac1, %4\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "+r"(dsp)
+         : "r"(ach), "r"(acl), "r"(rs)
+        );
+    pos =  dsp & 0x3F;
+    efi = (dsp >> 14) & 0x01;
+    if ((pos != 3) || (efi != 0) || (result != rt)) {
+        printf("extpdpv wrong\n");
+
+        return -1;
+    }
+
+    ach = 0x05;
+    acl = 0xB4CB;
+    dsp = 0x01;
+
+    __asm
+        ("wrdsp %1, 0x01\n\t"
+         "mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "extpdpv %0, $ac1, %4\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "+r"(dsp)
+         : "r"(ach), "r"(acl), "r"(rs)
+        );
+    efi = (dsp >> 14) & 0x01;
+    if (efi != 1) {
+        printf("extpdpv wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/extpv.c b/tests/tcg/mips/mips64-dsp/extpv.c
new file mode 100644
index 000000000..158472bf9
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/extpv.c
@@ -0,0 +1,51 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, ac, ach, acl, dsp;
+    long long result;
+
+    ach = 0x05;
+    acl = 0xB4CB;
+    dsp = 0x07;
+    ac  = 0x03;
+    result = 0x000C;
+
+    __asm
+        ("wrdsp %1, 0x01\n\t"
+         "mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "extpv %0, $ac1, %4\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "+r"(dsp)
+         : "r"(ach), "r"(acl), "r"(ac)
+        );
+    dsp = (dsp >> 14) & 0x01;
+    if ((dsp != 0) || (result != rt)) {
+        printf("extpv wrong\n");
+
+        return -1;
+    }
+
+    ach = 0x05;
+    acl = 0xB4CB;
+    dsp = 0x01;
+
+    __asm
+        ("wrdsp %1, 0x01\n\t"
+         "mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "extpv %0, $ac1, %4\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "+r"(dsp)
+         : "r"(ach), "r"(acl), "r"(ac)
+        );
+    dsp = (dsp >> 14) & 0x01;
+    if (dsp != 1) {
+        printf("extpv wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/extr_r_w.c b/tests/tcg/mips/mips64-dsp/extr_r_w.c
new file mode 100644
index 000000000..94572ad15
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/extr_r_w.c
@@ -0,0 +1,53 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, ach, acl, dsp;
+    long long result;
+
+    ach = 0x05;
+    acl = 0xB4CB;
+    result = 0xFFFFFFFFA0001699;
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "extr_r.w %0, $ac1, 0x03\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    if ((dsp != 1) || (result != rt)) {
+        printf("1 extr_r.w wrong\n");
+
+        return -1;
+    }
+
+    /* Clear dspcontrol */
+    dsp = 0;
+    __asm
+        ("wrdsp %0\n\t"
+         :
+         : "r"(dsp)
+        );
+
+    ach = 0x01;
+    acl = 0xB4CB;
+    result = 0x10000B4D;
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "extr_r.w %0, $ac1, 0x04\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    if ((dsp != 0) || (result != rt)) {
+        printf("2 extr_r.w wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/extr_rs_w.c b/tests/tcg/mips/mips64-dsp/extr_rs_w.c
new file mode 100644
index 000000000..73551f96b
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/extr_rs_w.c
@@ -0,0 +1,53 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, ach, acl, dsp;
+    long long result;
+
+    ach = 0x05;
+    acl = 0xB4CB;
+    result = 0x7FFFFFFF;
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "extr_rs.w %0, $ac1, 0x03\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    if ((dsp != 1) || (result != rt)) {
+        printf("1 extr_rs.w wrong\n");
+
+        return -1;
+    }
+
+    /* Clear dspcontrol */
+    dsp = 0;
+    __asm
+        ("wrdsp %0\n\t"
+         :
+         : "r"(dsp)
+        );
+
+    ach = 0x01;
+    acl = 0xB4CB;
+    result = 0x10000B4D;
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "extr_rs.w %0, $ac1, 0x04\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    if ((dsp != 0) || (result != rt)) {
+        printf("2 extr_rs.w wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/extr_s_h.c b/tests/tcg/mips/mips64-dsp/extr_s_h.c
new file mode 100644
index 000000000..de10cb57a
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/extr_s_h.c
@@ -0,0 +1,71 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, ach, acl, dsp;
+    long long result;
+
+    ach = 0x05;
+    acl = 0xB4CB;
+    result = 0x00007FFF;
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "extr_s.h %0, $ac1, 0x03\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    if ((dsp != 1) || (result != rt)) {
+        printf("extr_s.h wrong\n");
+
+        return -1;
+    }
+
+    ach = 0xffffffff;
+    acl = 0x12344321;
+    result = 0xffffffffFFFF8000;
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "extr_s.h %0, $ac1, 0x08\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    if ((dsp != 1) || (result != rt)) {
+        printf("extr_s.h wrong\n");
+
+        return -1;
+    }
+
+    /* Clear dsp */
+    dsp = 0;
+    __asm
+        ("wrdsp %0\n\t"
+         :
+         : "r"(dsp)
+        );
+
+    ach = 0x00;
+    acl = 0x4321;
+    result = 0x432;
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "extr_s.h %0, $ac1, 0x04\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    if ((dsp != 0) || (result != rt)) {
+        printf("extr_s.h wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/extr_w.c b/tests/tcg/mips/mips64-dsp/extr_w.c
new file mode 100644
index 000000000..bd6957668
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/extr_w.c
@@ -0,0 +1,53 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, ach, acl, dsp;
+    long long result;
+
+    ach = 0x05;
+    acl = 0xB4CB;
+    result = 0xFFFFFFFFA0001699;
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "extr.w %0, $ac1, 0x03\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    if ((dsp != 1) || (result != rt)) {
+        printf("extr.w wrong\n");
+
+        return -1;
+    }
+
+    /* Clear dspcontrol */
+    dsp = 0;
+    __asm
+        ("wrdsp %0\n\t"
+         :
+         : "r"(dsp)
+        );
+
+    ach = 0x01;
+    acl = 0xB4CB;
+    result = 0x10000B4C;
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "extr.w %0, $ac1, 0x04\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "=r"(dsp)
+         : "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    if ((dsp != 0) || (result != rt)) {
+        printf("extr.w wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/extrv_r_w.c b/tests/tcg/mips/mips64-dsp/extrv_r_w.c
new file mode 100644
index 000000000..837972978
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/extrv_r_w.c
@@ -0,0 +1,59 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, rs, ach, acl, dsp;
+    long long result;
+
+    ach = 0x05;
+    acl = 0xB4CB;
+    dsp = 0x07;
+    rs  = 0x03;
+    result = 0xFFFFFFFFA0001699;
+
+    __asm
+        ("wrdsp %1, 0x01\n\t"
+         "mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "extrv_r.w %0, $ac1, %2\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "+r"(dsp)
+         : "r"(rs), "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    if ((dsp != 1) || (result != rt)) {
+        printf("extrv_r.w wrong\n");
+
+        return -1;
+    }
+
+    /* Clear dspcontrol */
+    dsp = 0;
+    __asm
+        ("wrdsp %0\n\t"
+         :
+         : "r"(dsp)
+        );
+
+    rs = 4;
+    ach = 0x01;
+    acl = 0xB4CB;
+    result = 0x10000B4D;
+    __asm
+        ("wrdsp %1, 0x01\n\t"
+         "mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "extrv_r.w %0, $ac1, %2\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "+r"(dsp)
+         : "r"(rs), "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    if ((dsp != 0) || (result != rt)) {
+        printf("extrv_r.w wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/extrv_rs_w.c b/tests/tcg/mips/mips64-dsp/extrv_rs_w.c
new file mode 100644
index 000000000..8707cd117
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/extrv_rs_w.c
@@ -0,0 +1,59 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, rs, ach, acl, dsp;
+    long long result;
+
+    ach = 0x05;
+    acl = 0xB4CB;
+    dsp = 0x07;
+    rs  = 0x03;
+    result = 0x7FFFFFFF;
+
+    __asm
+        ("wrdsp %1, 0x01\n\t"
+         "mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "extrv_rs.w %0, $ac1, %2\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "+r"(dsp)
+         : "r"(rs), "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    if ((dsp != 1) || (result != rt)) {
+        printf("1 extrv_rs.w wrong\n");
+
+        return -1;
+    }
+
+    /* Clear dspcontrol */
+    dsp = 0;
+    __asm
+        ("wrdsp %0\n\t"
+         :
+         : "r"(dsp)
+        );
+
+    rs = 4;
+    ach = 0x01;
+    acl = 0xB4CB;
+    result = 0x10000B4D;
+    __asm
+        ("wrdsp %1, 0x01\n\t"
+         "mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "extrv_rs.w %0, $ac1, %2\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "+r"(dsp)
+         : "r"(rs), "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    if ((dsp != 0) || (result != rt)) {
+        printf("2 extrv_rs.w wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/extrv_s_h.c b/tests/tcg/mips/mips64-dsp/extrv_s_h.c
new file mode 100644
index 000000000..b6dcaebcb
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/extrv_s_h.c
@@ -0,0 +1,79 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, rs, ach, acl, dsp;
+    long long result;
+
+    ach = 0x05;
+    acl = 0xB4CB;
+    dsp = 0x07;
+    rs  = 0x03;
+    result = 0x00007FFF;
+
+    __asm
+        ("wrdsp %1, 0x01\n\t"
+         "mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "extrv_s.h %0, $ac1, %2\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "+r"(dsp)
+         : "r"(rs), "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    if ((dsp != 1) || (result != rt)) {
+        printf("extrv_s.h wrong\n");
+
+        return -1;
+    }
+
+    rs = 0x08;
+    ach = 0xffffffff;
+    acl = 0x12344321;
+    result = 0xffffffffFFFF8000;
+    __asm
+        ("wrdsp %1, 0x01\n\t"
+         "mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "extrv_s.h %0, $ac1, %2\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "+r"(dsp)
+         : "r"(rs), "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    if ((dsp != 1) || (result != rt)) {
+        printf("extrv_s.h wrong\n");
+
+        return -1;
+    }
+
+    /* Clear dsp */
+    dsp = 0;
+    __asm
+        ("wrdsp %0\n\t"
+         :
+         : "r"(dsp)
+        );
+
+    rs = 0x04;
+    ach = 0x00;
+    acl = 0x4321;
+    result = 0x432;
+    __asm
+        ("wrdsp %1, 0x01\n\t"
+         "mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "extrv_s.h %0, $ac1, %2\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "+r"(dsp)
+         : "r"(rs), "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    if ((dsp != 0) || (result != rt)) {
+        printf("extrv_s.h wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/extrv_w.c b/tests/tcg/mips/mips64-dsp/extrv_w.c
new file mode 100644
index 000000000..8adffb395
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/extrv_w.c
@@ -0,0 +1,59 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, rs, ach, acl, dsp;
+    long long result;
+
+    ach = 0x05;
+    acl = 0xB4CB;
+    dsp = 0x07;
+    rs  = 0x03;
+    result = 0xFFFFFFFFA0001699;
+
+    __asm
+        ("wrdsp %1, 0x01\n\t"
+         "mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "extrv.w %0, $ac1, %2\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "+r"(dsp)
+         : "r"(rs), "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    if ((dsp != 1) || (result != rt)) {
+        printf("extrv.w wrong\n");
+
+        return -1;
+    }
+
+    /* Clear dspcontrol */
+    dsp = 0;
+    __asm
+        ("wrdsp %0\n\t"
+         :
+         : "r"(dsp)
+        );
+
+    rs = 4;
+    ach = 0x01;
+    acl = 0xB4CB;
+    result = 0x10000B4C;
+    __asm
+        ("wrdsp %1, 0x01\n\t"
+         "mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "extrv.w %0, $ac1, %2\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rt), "+r"(dsp)
+         : "r"(rs), "r"(ach), "r"(acl)
+        );
+    dsp = (dsp >> 23) & 0x01;
+    if ((dsp != 0) || (result != rt)) {
+        printf("extrv.w wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/head.S b/tests/tcg/mips/mips64-dsp/head.S
new file mode 100644
index 000000000..9a099ae42
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/head.S
@@ -0,0 +1,16 @@
+/*
+ *  Startup Code for MIPS64 CPU-core
+ *
+ */
+.text
+.globl _start
+.align 4
+_start:
+    ori    $2, $2, 0xffff
+    sll    $2, $2, 16
+    ori    $2, $2, 0xffff
+    mtc0   $2, $12, 0
+    jal    main
+
+end:
+    b end
diff --git a/tests/tcg/mips/mips64-dsp/insv.c b/tests/tcg/mips/mips64-dsp/insv.c
new file mode 100644
index 000000000..fc5696f4c
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/insv.c
@@ -0,0 +1,26 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, rs, dsp;
+    long long result;
+
+    /* msb = 10, lsb = 5 */
+    dsp    = 0x305;
+    rt     = 0x12345678;
+    rs     = 0xffffffff87654321;
+    result = 0x12345338;
+    __asm
+        ("wrdsp %2, 0x03\n\t"
+         "insv  %0, %1\n\t"
+         : "+r"(rt)
+         : "r"(rs), "r"(dsp)
+        );
+    if (rt != result) {
+        printf("insv wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/io.h b/tests/tcg/mips/mips64-dsp/io.h
new file mode 100644
index 000000000..b7db61d7c
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/io.h
@@ -0,0 +1,22 @@
+#ifndef _ASM_IO_H
+#define _ASM_IO_H
+extern int printf(const char *fmt, ...);
+extern unsigned long get_ticks(void);
+
+#define _read(source)                \
+({ unsigned long __res;                \
+    __asm__ __volatile__(            \
+        "mfc0\t%0, " #source "\n\t"    \
+        : "=r" (__res));        \
+    __res;                    \
+})
+
+#define __read(source)                \
+({ unsigned long __res;                \
+    __asm__ __volatile__(            \
+        "move\t%0, " #source "\n\t"    \
+        : "=r" (__res));        \
+    __res;                    \
+})
+
+#endif
diff --git a/tests/tcg/mips/mips64-dsp/lbux.c b/tests/tcg/mips/mips64-dsp/lbux.c
new file mode 100644
index 000000000..dbdc87bff
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/lbux.c
@@ -0,0 +1,27 @@
+#include "io.h"
+
+int main(void)
+{
+    long long value, rd;
+    long long *p;
+    unsigned long long addr, index;
+    long long result;
+
+    value  = 0xBCDEF389;
+    p = &value;
+    addr = (unsigned long long)p;
+    index  = 0;
+    result = value & 0xFF;
+    __asm
+        ("lbux %0, %1(%2)\n\t"
+         : "=r"(rd)
+         : "r"(index), "r"(addr)
+        );
+    if (rd != result) {
+        printf("lbux wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/ldx.c b/tests/tcg/mips/mips64-dsp/ldx.c
new file mode 100644
index 000000000..787d9f00b
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/ldx.c
@@ -0,0 +1,27 @@
+#include "io.h"
+
+int main(void)
+{
+    long long value, rd;
+    long long *p;
+    unsigned long long addr, index;
+    long long result;
+
+    value  = 0xBCDEF389;
+    p = &value;
+    addr = (unsigned long long)p;
+    index  = 0;
+    result = 0xBCDEF389;
+    __asm
+        ("ldx %0, %1(%2)\n\t"
+         : "=r"(rd)
+         : "r"(index), "r"(addr)
+        );
+    if (rd != result) {
+        printf("lwx wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/lhx.c b/tests/tcg/mips/mips64-dsp/lhx.c
new file mode 100644
index 000000000..2020e5686
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/lhx.c
@@ -0,0 +1,27 @@
+#include "io.h"
+
+int main(void)
+{
+    long long value, rd;
+    long long *p;
+    unsigned long long addr, index;
+    long long result;
+
+    value  = 0xBCDEF389;
+    p = &value;
+    addr = (unsigned long long)p;
+    index  = 0;
+    result = 0xFFFFFFFFFFFFF389;
+    __asm
+        ("lhx %0, %1(%2)\n\t"
+         : "=r"(rd)
+         : "r"(index), "r"(addr)
+        );
+    if (rd != result) {
+        printf("lhx wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/lwx.c b/tests/tcg/mips/mips64-dsp/lwx.c
new file mode 100644
index 000000000..6a81414d6
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/lwx.c
@@ -0,0 +1,27 @@
+#include "io.h"
+
+int main(void)
+{
+    long long value, rd;
+    long long *p;
+    unsigned long long addr, index;
+    long long result;
+
+    value  = 0xBCDEF389;
+    p = &value;
+    addr = (unsigned long long)p;
+    index  = 0;
+    result = 0xFFFFFFFFBCDEF389;
+    __asm
+        ("lwx %0, %1(%2)\n\t"
+         : "=r"(rd)
+         : "r"(index), "r"(addr)
+        );
+    if (rd != result) {
+        printf("lwx wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/madd.c b/tests/tcg/mips/mips64-dsp/madd.c
new file mode 100644
index 000000000..de6e44fbc
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/madd.c
@@ -0,0 +1,33 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, rs;
+    long long achi, acli;
+    long long acho, aclo;
+    long long resulth, resultl;
+
+    achi = 0x05;
+    acli = 0xB4CB;
+    rs  = 0x01;
+    rt  = 0x01;
+    resulth = 0x05;
+    resultl = 0xB4CC;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "madd $ac1, %4, %5\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+    if ((resulth != acho) || (resultl != aclo)) {
+        printf("madd wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/maddu.c b/tests/tcg/mips/mips64-dsp/maddu.c
new file mode 100644
index 000000000..e9f426a37
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/maddu.c
@@ -0,0 +1,33 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, rs;
+    long long achi, acli;
+    long long acho, aclo;
+    long long resulth, resultl;
+
+    achi = 0x05;
+    acli = 0xB4CB;
+    rs  = 0x01;
+    rt  = 0x01;
+    resulth = 0x05;
+    resultl = 0xB4CC;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "madd $ac1, %4, %5\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+    if ((resulth != acho) || (resultl != aclo)) {
+        printf("maddu wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/maq_s_l_pwl.c b/tests/tcg/mips/mips64-dsp/maq_s_l_pwl.c
new file mode 100644
index 000000000..c196b4353
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/maq_s_l_pwl.c
@@ -0,0 +1,56 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, rs, dsp;
+    long long achi, acli;
+    long long acho, aclo;
+    long long resulth, resultl;
+
+    achi = 0x05;
+    acli = 0xB4CB;
+    rs  = 0x98765432FF060000;
+    rt  = 0xfdeca987CB000000;
+    resulth = 0x05;
+    resultl = 0x18278587;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "maq_s.l.pwl $ac1, %4, %5\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+    if ((resulth != acho) || (resultl != aclo)) {
+        printf("maq_s_l.w.pwl wrong 1\n");
+
+        return -1;
+    }
+
+    achi = 0x05;
+    acli = 0xB4CB;
+    rs  = 0x80000000FF060000;
+    rt  = 0x80000000CB000000;
+    resulth = 0x05;
+    resultl = 0xb4ca;
+
+    __asm
+        ("mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "maq_s.l.pwl $ac1, %5, %6\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "=r"(acho), "=r"(aclo), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 17) & 0x1;
+    if ((dsp != 0x1) || (resulth != acho) || (resultl != aclo)) {
+        printf("maq_s_l.w.pwl wrong 2\n");
+
+        return -1;
+    }
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/maq_s_l_pwr.c b/tests/tcg/mips/mips64-dsp/maq_s_l_pwr.c
new file mode 100644
index 000000000..e2af69fe2
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/maq_s_l_pwr.c
@@ -0,0 +1,56 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, rs, dsp;
+    long long achi, acli;
+    long long acho, aclo;
+    long long resulth, resultl;
+
+    achi = 0x05;
+    acli = 0xB4CB;
+    rs  = 0x87898765432;
+    rt  = 0x7878fdeca987;
+    resulth = 0x05;
+    resultl = 0x18278587;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "maq_s.l.pwr $ac1, %4, %5\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+    if ((resulth != acho) || (resultl != aclo)) {
+        printf("maq_s.w.pwr wrong\n");
+
+        return -1;
+    }
+
+    achi = 0x05;
+    acli = 0xB4CB;
+    rs  = 0x89899980000000;
+    rt  = 0x88780000000;
+    resulth = 0x05;
+    resultl = 0xb4ca;
+
+    __asm
+        ("mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "maq_s.l.pwr $ac1, %5, %6\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "=r"(acho), "=r"(aclo), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 17) & 0x1;
+    if ((dsp != 0x1) || (resulth != acho) || (resultl != aclo)) {
+        printf("maq_s.w.pwr wrong\n");
+
+        return -1;
+    }
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/maq_s_w_phl.c b/tests/tcg/mips/mips64-dsp/maq_s_w_phl.c
new file mode 100644
index 000000000..7dba8746e
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/maq_s_w_phl.c
@@ -0,0 +1,60 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, rs;
+    long long achi, acli;
+    long long dsp;
+    long long acho, aclo;
+    long long resulth, resultl;
+    long long resdsp;
+
+    achi = 0x05;
+    acli = 0xB4CB;
+    rs  = 0xFF060000;
+    rt  = 0xCB000000;
+    resulth = 0x04;
+    resultl = 0xffffffff947438CB;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "maq_s.w.phl $ac1, %4, %5\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+    if ((resulth != acho) || (resultl != aclo)) {
+        printf("1 maq_s.w.phl error\n");
+
+        return -1;
+    }
+
+    achi = 0x06;
+    acli = 0xB4CB;
+    rs  = 0x80000000;
+    rt  = 0x80000000;
+    resulth = 0x6;
+    resultl = 0xffffffff8000b4ca;
+    resdsp = 1;
+
+    __asm
+        ("mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "maq_s.w.phl $ac1, %5, %6\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "=r"(acho), "=r"(aclo), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+    if ((resulth != acho) || (resultl != aclo) ||
+        (((dsp >> 17) & 0x01) != resdsp)) {
+        printf("2 maq_s.w.phl error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/maq_s_w_phr.c b/tests/tcg/mips/mips64-dsp/maq_s_w_phr.c
new file mode 100644
index 000000000..138ee2a69
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/maq_s_w_phr.c
@@ -0,0 +1,60 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, rs;
+    long long achi, acli;
+    long long dsp;
+    long long acho, aclo;
+    long long resulth, resultl;
+    long long resdsp;
+
+    achi = 0x05;
+    acli = 0xB4CB;
+    rs  = 0xFF06;
+    rt  = 0xCB00;
+    resulth = 0x04;
+    resultl = 0xffffffff947438CB;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "maq_s.w.phr $ac1, %4, %5\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+    if ((resulth != acho) || (resultl != aclo)) {
+        printf("1 maq_s.w.phr error\n");
+
+        return -1;
+    }
+
+    achi = 0x06;
+    acli = 0xB4CB;
+    rs  = 0x8000;
+    rt  = 0x8000;
+    resulth = 0x6;
+    resultl = 0xffffffff8000b4ca;
+    resdsp = 1;
+
+    __asm
+        ("mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "maq_s.w.phr $ac1, %5, %6\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "=r"(acho), "=r"(aclo), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+    if ((resulth != acho) || (resultl != aclo) ||
+        (((dsp >> 17) & 0x01) != resdsp)) {
+        printf("2 maq_s.w.phr error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/maq_s_w_qhll.c b/tests/tcg/mips/mips64-dsp/maq_s_w_qhll.c
new file mode 100644
index 000000000..234a0af29
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/maq_s_w_qhll.c
@@ -0,0 +1,62 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, rs, dsp;
+    long long achi, acli;
+    long long acho, aclo;
+    long long resulth, resultl;
+
+    achi = 0x05;
+    acli = 0x05;
+
+    rs  = 0x1234888899990000;
+    rt  = 0x9876888899990000;
+
+    resulth = 0x05;
+    resultl = 0x15ae87f5;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "maq_s.w.qhll $ac1, %4, %5\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+
+    if ((resulth != acho) || (resultl != aclo)) {
+        printf("maq_s.w.qhll wrong\n");
+
+        return -1;
+    }
+
+
+    achi = 0x04;
+    acli = 0x06;
+    rs  = 0x8000888899990000;
+    rt  = 0x8000888899990000;
+
+    resulth = 0x04;
+    resultl = 0xffffffff80000005;
+
+    __asm
+        ("mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "maq_s.w.qhll $ac1, %5, %6\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "=r"(acho), "=r"(aclo), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+
+    dsp = (dsp >> 17) & 0x1;
+    if ((dsp != 0x1) || (resulth != acho) || (resultl != aclo)) {
+        printf("maq_s.w.qhll wrong\n");
+
+        return -1;
+    }
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/maq_s_w_qhlr.c b/tests/tcg/mips/mips64-dsp/maq_s_w_qhlr.c
new file mode 100644
index 000000000..8768cbaa3
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/maq_s_w_qhlr.c
@@ -0,0 +1,62 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, rs, dsp;
+    long long achi, acli;
+    long long acho, aclo;
+    long long resulth, resultl;
+
+    achi = 0x05;
+    acli = 0x05;
+
+    rs  = 0x1234123412340000;
+    rt  = 0x9876987698760000;
+
+    resulth = 0x05;
+    resultl = 0x15ae87f5;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "maq_s.w.qhlr $ac1, %4, %5\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+
+    if ((resulth != acho) || (resultl != aclo)) {
+        printf("1 maq_s.w.qhlr wrong\n");
+
+        return -1;
+    }
+
+
+    achi = 0x04;
+    acli = 0x06;
+    rs  = 0x8000800080000000;
+    rt  = 0x8000800080000000;
+
+    resulth = 0x04;
+    resultl = 0xffffffff80000005;
+
+    __asm
+        ("mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "maq_s.w.qhlr $ac1, %5, %6\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "=r"(acho), "=r"(aclo), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+
+    dsp = (dsp >> 17) & 0x1;
+    if ((dsp != 0x1) || (resulth != acho) || (resultl != aclo)) {
+        printf("2 maq_s.w.qhlr wrong\n");
+
+        return -1;
+    }
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/maq_s_w_qhrl.c b/tests/tcg/mips/mips64-dsp/maq_s_w_qhrl.c
new file mode 100644
index 000000000..5006e2be3
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/maq_s_w_qhrl.c
@@ -0,0 +1,63 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, rs, dsp;
+    long long achi, acli;
+    long long acho, aclo;
+    long long resulth, resultl;
+
+    achi = 0x05;
+    acli = 0x05;
+
+    rs  = 0x1234888812340000;
+    rt  = 0x9876888898760000;
+
+    resulth = 0x05;
+    resultl = 0x15ae87f5;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "maq_s.w.qhrl $ac1, %4, %5\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+
+    if ((resulth != acho) || (resultl != aclo)) {
+        printf("1 maq_s.w.qhrl wrong\n");
+
+        return -1;
+    }
+
+
+    achi = 0x04;
+    acli = 0x06;
+    rs  = 0x8888999980000000;
+    rt  = 0x8888999980000000;
+
+    resulth = 0x04;
+    resultl = 0xffffffff80000005;
+
+    __asm
+        ("mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "maq_s.w.qhrl $ac1, %5, %6\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "=r"(acho), "=r"(aclo), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+
+    dsp = (dsp >> 17) & 0x1;
+    if ((dsp != 0x1) || (resulth != acho) || (resultl != aclo)) {
+        printf("2 maq_s.w.qhrl wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/maq_s_w_qhrr.c b/tests/tcg/mips/mips64-dsp/maq_s_w_qhrr.c
new file mode 100644
index 000000000..1d213a51b
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/maq_s_w_qhrr.c
@@ -0,0 +1,63 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, rs, dsp;
+    long long achi, acli;
+    long long acho, aclo;
+    long long resulth, resultl;
+
+    achi = 0x05;
+    acli = 0x05;
+
+    rs  = 0x1234888812341234;
+    rt  = 0x9876888898769876;
+
+    resulth = 0x05;
+    resultl = 0x15ae87f5;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "maq_s.w.qhrr $ac1, %4, %5\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+
+    if ((resulth != acho) || (resultl != aclo)) {
+        printf("1 maq_s.w.qhrr wrong\n");
+
+        return -1;
+    }
+
+
+    achi = 0x04;
+    acli = 0x06;
+    rs  = 0x8000888899998000;
+    rt  = 0x8000888899998000;
+
+    resulth = 0x04;
+    resultl = 0xffffffff80000005;
+
+    __asm
+        ("mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "maq_s.w.qhrr $ac1, %5, %6\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "=r"(acho), "=r"(aclo), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+
+    dsp = (dsp >> 17) & 0x1;
+    if ((dsp != 0x1) || (resulth != acho) || (resultl != aclo)) {
+        printf("2 maq_s.w.qhrr wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/maq_sa_w_phl.c b/tests/tcg/mips/mips64-dsp/maq_sa_w_phl.c
new file mode 100644
index 000000000..5530ffbe6
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/maq_sa_w_phl.c
@@ -0,0 +1,60 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, rs;
+    long long achi, acli;
+    long long dsp;
+    long long acho, aclo;
+    long long resulth, resultl;
+    long long resdsp;
+
+    achi = 0x05;
+    acli = 0xB4CB;
+    rs = 0xFF060000;
+    rt = 0xCB000000;
+    resulth = 0xffffffffffffffff;
+    resultl = 0xffffffff947438cb;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "maq_sa.w.phl $ac1, %4, %5\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+    if ((resulth != acho) || (resultl != aclo)) {
+        printf("1 maq_sa.w.phl error\n");
+
+        return -1;
+    }
+
+    achi = 0x06;
+    acli = 0xB4CB;
+    rs  = 0x80000000;
+    rt  = 0x80000000;
+    resulth = 0x00;
+    resultl = 0x7fffffff;
+    resdsp = 0x01;
+
+    __asm
+        ("mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "maq_sa.w.phl $ac1, %5, %6\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "=r"(acho), "=r"(aclo), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+    if ((resulth != acho) || (resultl != aclo) ||
+        (((dsp >> 17) & 0x01) != 0x01)) {
+        printf("2 maq_sa.w.phl error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/maq_sa_w_phr.c b/tests/tcg/mips/mips64-dsp/maq_sa_w_phr.c
new file mode 100644
index 000000000..b611cfa91
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/maq_sa_w_phr.c
@@ -0,0 +1,60 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, rs;
+    long long achi, acli;
+    long long dsp;
+    long long acho, aclo;
+    long long resulth, resultl;
+    long long resdsp;
+
+    achi = 0x05;
+    acli = 0xB4CB;
+    rs = 0xFF06;
+    rt = 0xCB00;
+    resulth = 0xffffffffffffffff;
+    resultl = 0xffffffff947438cb;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "maq_sa.w.phr $ac1, %4, %5\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+    if ((resulth != acho) || (resultl != aclo)) {
+        printf("1 maq_sa.w.phr error\n");
+
+        return -1;
+    }
+
+    achi = 0x06;
+    acli = 0xB4CB;
+    rs  = 0x8000;
+    rt  = 0x8000;
+    resulth = 0x00;
+    resultl = 0x7fffffff;
+    resdsp = 0x01;
+
+    __asm
+        ("mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "maq_sa.w.phr $ac1, %5, %6\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "=r"(acho), "=r"(aclo), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+    if ((resulth != acho) || (resultl != aclo) ||
+        (((dsp >> 17) & 0x01) != 0x01)) {
+        printf("2 maq_sa.w.phr error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/maq_sa_w_qhll.c b/tests/tcg/mips/mips64-dsp/maq_sa_w_qhll.c
new file mode 100644
index 000000000..136ff2d77
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/maq_sa_w_qhll.c
@@ -0,0 +1,62 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, rs, dsp;
+    long long achi, acli;
+    long long acho, aclo;
+    long long resulth, resultl;
+
+    achi = 0x05;
+    acli = 0x05;
+
+    rs  = 0x1234888899990000;
+    rt  = 0x9876888899990000;
+
+    resulth = 0x00;
+    resultl = 0x15ae87f5;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "maq_sa.w.qhll $ac1, %4, %5\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+
+    if ((resulth != acho) || (resultl != aclo)) {
+        printf("1 maq_sa.w.qhll wrong\n");
+
+        return -1;
+    }
+
+
+    achi = 0x04;
+    acli = 0x06;
+    rs  = 0x8000888899990000;
+    rt  = 0x8000888899990000;
+
+    resulth = 0x00;
+    resultl = 0x7fffffff;
+
+    __asm
+        ("mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "maq_sa.w.qhll $ac1, %5, %6\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "=r"(acho), "=r"(aclo), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+
+    dsp = (dsp >> 17) & 0x1;
+    if ((dsp != 0x1) || (resulth != acho) || (resultl != aclo)) {
+        printf("2 maq_sa.w.qhll wrong\n");
+
+        return -1;
+    }
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/maq_sa_w_qhlr.c b/tests/tcg/mips/mips64-dsp/maq_sa_w_qhlr.c
new file mode 100644
index 000000000..dd0ae1cca
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/maq_sa_w_qhlr.c
@@ -0,0 +1,64 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, rs, dsp;
+    long long achi, acli;
+    long long acho, aclo;
+    long long resulth, resultl;
+
+    achi = 0x05;
+    acli = 0x05;
+
+    rs  = 0x1234123412340000;
+    rt  = 0x9876987699990000;
+
+    resulth = 0x0;
+    resultl = 0x15ae87f5;
+
+    __asm
+        ("mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "maq_sa.w.qhlr $ac1, %5, %6\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "=r"(acho), "=r"(aclo), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+
+    dsp = (dsp >> 17) & 0x1;
+    if ((dsp != 0x0) || (resulth != acho) || (resultl != aclo)) {
+        printf("maq_sa.w.qhlr wrong\n");
+
+        return -1;
+    }
+
+
+    achi = 0x04;
+    acli = 0x06;
+    rs  = 0x8000800099990000;
+    rt  = 0x8000800099990000;
+
+    resulth = 0x00;
+    resultl = 0x7fffffff;
+
+    __asm
+        ("mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "maq_sa.w.qhlr $ac1, %5, %6\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "=r"(acho), "=r"(aclo), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+
+    dsp = (dsp >> 17) & 0x1;
+    if ((dsp != 0x1) || (resulth != acho) || (resultl != aclo)) {
+        printf("maq_sa.w.qhlr wrong\n");
+
+        return -1;
+    }
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/maq_sa_w_qhrl.c b/tests/tcg/mips/mips64-dsp/maq_sa_w_qhrl.c
new file mode 100644
index 000000000..a3de6f8e2
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/maq_sa_w_qhrl.c
@@ -0,0 +1,64 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, rs, dsp;
+    long long achi, acli;
+    long long acho, aclo;
+    long long resulth, resultl;
+
+    achi = 0x05;
+    acli = 0x05;
+
+    rs  = 0x1234123412340000;
+    rt  = 0x9876987698760000;
+
+    resulth = 0x0;
+    resultl = 0x15ae87f5;
+
+    __asm
+        ("mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "maq_sa.w.qhrl $ac1, %5, %6\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "=r"(acho), "=r"(aclo), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+
+    dsp = (dsp >> 17) & 0x1;
+    if ((dsp != 0x0) || (resulth != acho) || (resultl != aclo)) {
+        printf("1 maq_sa.w.qhrl wrong\n");
+
+        return -1;
+    }
+
+
+    achi = 0x04;
+    acli = 0x06;
+    rs  = 0x8000800080000000;
+    rt  = 0x8000800080000000;
+
+    resulth = 0x00;
+    resultl = 0x7fffffff;
+
+    __asm
+        ("mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "maq_sa.w.qhrl $ac1, %5, %6\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "=r"(acho), "=r"(aclo), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+
+    dsp = (dsp >> 17) & 0x1;
+    if ((dsp != 0x1) || (resulth != acho) || (resultl != aclo)) {
+        printf("2 maq_sa.w.qhrl wrong\n");
+
+        return -1;
+    }
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/maq_sa_w_qhrr.c b/tests/tcg/mips/mips64-dsp/maq_sa_w_qhrr.c
new file mode 100644
index 000000000..f02173736
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/maq_sa_w_qhrr.c
@@ -0,0 +1,64 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, rs, dsp;
+    long long achi, acli;
+    long long acho, aclo;
+    long long resulth, resultl;
+
+    achi = 0x05;
+    acli = 0x05;
+
+    rs  = 0x1234123412341234;
+    rt  = 0x9876987698769876;
+
+    resulth = 0x0;
+    resultl = 0x15ae87f5;
+
+    __asm
+        ("mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "maq_sa.w.qhrr $ac1, %5, %6\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "=r"(acho), "=r"(aclo), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+
+    dsp = (dsp >> 17) & 0x1;
+    if ((dsp != 0x0) || (resulth != acho) || (resultl != aclo)) {
+        printf("1 maq_sa.w.qhrr wrong\n");
+
+        return -1;
+    }
+
+
+    achi = 0x04;
+    acli = 0x06;
+    rs  = 0x8000800080008000;
+    rt  = 0x8000800080008000;
+
+    resulth = 0x00;
+    resultl = 0x7fffffff;
+
+    __asm
+        ("mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "maq_sa.w.qhrr $ac1, %5, %6\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "=r"(acho), "=r"(aclo), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+
+    dsp = (dsp >> 17) & 0x1;
+    if ((dsp != 0x1) || (resulth != acho) || (resultl != aclo)) {
+        printf("2 maq_sa.w.qhrr wrong\n");
+
+        return -1;
+    }
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/mfhi.c b/tests/tcg/mips/mips64-dsp/mfhi.c
new file mode 100644
index 000000000..ee915f796
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/mfhi.c
@@ -0,0 +1,24 @@
+#include "io.h"
+
+int main(void)
+{
+    long long achi, acho;
+    long long result;
+
+    achi   = 0x004433;
+    result = 0x004433;
+
+    __asm
+        ("mthi %1, $ac1\n\t"
+         "mfhi %0, $ac1\n\t"
+         : "=r"(acho)
+         : "r"(achi)
+        );
+    if (result != acho) {
+        printf("mfhi wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/mflo.c b/tests/tcg/mips/mips64-dsp/mflo.c
new file mode 100644
index 000000000..cdc646b5f
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/mflo.c
@@ -0,0 +1,24 @@
+#include "io.h"
+
+int main(void)
+{
+    long long acli, aclo;
+    long long result;
+
+    acli   = 0x004433;
+    result = 0x004433;
+
+    __asm
+        ("mtlo %1, $ac1\n\t"
+         "mflo %0, $ac1\n\t"
+         : "=r"(aclo)
+         : "r"(acli)
+        );
+    if (result != aclo) {
+        printf("mflo wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/mips_boot.lds b/tests/tcg/mips/mips64-dsp/mips_boot.lds
new file mode 100644
index 000000000..bd7c0c0f3
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/mips_boot.lds
@@ -0,0 +1,31 @@
+OUTPUT_ARCH(mips)
+SECTIONS
+{
+    . = 0xffffffff80100000;
+    . = ALIGN((1 << 13));
+    .text :
+    {
+        *(.text)
+        *(.rodata)
+        *(.rodata.*)
+    }
+
+    __init_begin = .;
+    . = ALIGN((1 << 12));
+    .init.text : AT(ADDR(.init.text) - 0)
+    {
+        *(.init.text)
+    }
+    .init.data : AT(ADDR(.init.data) - 0)
+    {
+        *(.init.data)
+    }
+    . = ALIGN((1 << 12));
+    __init_end = .;
+
+    . = ALIGN((1 << 13));
+    .data :
+    {
+        *(.data)
+    }
+}
diff --git a/tests/tcg/mips/mips64-dsp/modsub.c b/tests/tcg/mips/mips64-dsp/modsub.c
new file mode 100644
index 000000000..2c91cb4c5
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/modsub.c
@@ -0,0 +1,37 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long result;
+
+    rs     = 0xFFFFFFFF;
+    rt     = 0x000000FF;
+    result = 0xFFFFFF00;
+    __asm
+        ("modsub %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    if (result != rd) {
+        printf("modsub wrong\n");
+
+        return -1;
+    }
+
+    rs     = 0x00000000;
+    rt     = 0x00CD1FFF;
+    result = 0x0000CD1F;
+    __asm
+        ("modsub %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    if (result != rd) {
+        printf("modsub wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/msub.c b/tests/tcg/mips/mips64-dsp/msub.c
new file mode 100644
index 000000000..75066b591
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/msub.c
@@ -0,0 +1,32 @@
+#include "io.h"
+
+int main(void)
+{
+    long long achi, acli, rs, rt;
+    long long acho, aclo;
+    long long resulth, resultl;
+
+    rs      = 0x00BBAACC;
+    rt      = 0x0B1C3D2F;
+    achi    = 0x00004433;
+    acli    = 0xFFCC0011;
+    resulth = 0xFFFFFFFFFFF81F29;
+    resultl = 0xFFFFFFFFB355089D;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "msub $ac1, %4, %5\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+    if ((acho != resulth) || (aclo != resultl)) {
+        printf("msub wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/msubu.c b/tests/tcg/mips/mips64-dsp/msubu.c
new file mode 100644
index 000000000..55f8ae046
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/msubu.c
@@ -0,0 +1,32 @@
+#include "io.h"
+
+int main(void)
+{
+    long long achi, acli, rs, rt;
+    long long acho, aclo;
+    long long resulth, resultl;
+
+    rs      = 0x00BBAACC;
+    rt      = 0x0B1C3D2F;
+    achi    = 0x00004433;
+    acli    = 0xFFCC0011;
+    resulth = 0xFFFFFFFFFFF81F29;
+    resultl = 0xFFFFFFFFB355089D;
+
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "msubu $ac1, %4, %5\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+    if ((acho != resulth) || (aclo != resultl)) {
+        printf("msubu wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/mthi.c b/tests/tcg/mips/mips64-dsp/mthi.c
new file mode 100644
index 000000000..857005139
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/mthi.c
@@ -0,0 +1,24 @@
+#include "io.h"
+
+int main(void)
+{
+    long long achi, acho;
+    long long result;
+
+    achi   = 0x004433;
+    result = 0x004433;
+
+    __asm
+        ("mthi %1, $ac1\n\t"
+         "mfhi %0, $ac1\n\t"
+         : "=r"(acho)
+         : "r"(achi)
+        );
+    if (result != acho) {
+        printf("mthi wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/mthlip.c b/tests/tcg/mips/mips64-dsp/mthlip.c
new file mode 100644
index 000000000..957cd426f
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/mthlip.c
@@ -0,0 +1,61 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rs, ach, acl, dsp;
+    long long result, resulth, resultl;
+
+    dsp = 0x07;
+    ach = 0x05;
+    acl = 0xB4CB;
+    rs  = 0x00FFBBAA;
+    resulth = 0xB4CB;
+    resultl = 0x00FFBBAA;
+    result  = 0x27;
+
+    __asm
+        ("wrdsp %0, 0x01\n\t"
+         "mthi %1, $ac1\n\t"
+         "mtlo %2, $ac1\n\t"
+         "mthlip %3, $ac1\n\t"
+         "mfhi %1, $ac1\n\t"
+         "mflo %2, $ac1\n\t"
+         "rddsp %0\n\t"
+         : "+r"(dsp), "+r"(ach), "+r"(acl)
+         : "r"(rs)
+        );
+    dsp = dsp & 0x3F;
+    if ((dsp != result) || (ach != resulth) || (acl != resultl)) {
+        printf("mthlip wrong\n");
+
+        return -1;
+    }
+
+    dsp = 0x3f;
+    ach = 0x05;
+    acl = 0xB4CB;
+    rs  = 0x00FFBBAA;
+    resulth = 0xB4CB;
+    resultl = 0x00FFBBAA;
+    result  = 0x3f;
+
+    __asm
+        ("wrdsp %0, 0x01\n\t"
+         "mthi %1, $ac1\n\t"
+         "mtlo %2, $ac1\n\t"
+         "mthlip %3, $ac1\n\t"
+         "mfhi %1, $ac1\n\t"
+         "mflo %2, $ac1\n\t"
+         "rddsp %0\n\t"
+         : "+r"(dsp), "+r"(ach), "+r"(acl)
+         : "r"(rs)
+        );
+    dsp = dsp & 0x3F;
+    if ((dsp != result) || (ach != resulth) || (acl != resultl)) {
+        printf("mthlip wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/mtlo.c b/tests/tcg/mips/mips64-dsp/mtlo.c
new file mode 100644
index 000000000..304fffbe7
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/mtlo.c
@@ -0,0 +1,22 @@
+#include "io.h"
+
+int main(void)
+{
+    long long acli, aclo;
+    long long result;
+
+    acli   = 0x004433;
+    result = 0x004433;
+
+    __asm
+        ("mthi %1, $ac1\n\t"
+         "mfhi %0, $ac1\n\t"
+         : "=r"(aclo)
+         : "r"(acli)
+        );
+    if (result != aclo) {
+        printf("mtlo wrong\n");
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/muleq_s_pw_qhl.c b/tests/tcg/mips/mips64-dsp/muleq_s_pw_qhl.c
new file mode 100644
index 000000000..6c68d45af
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/muleq_s_pw_qhl.c
@@ -0,0 +1,56 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, result;
+
+    rd = 0;
+    rs = 0x45BCFFFF12345678;
+    rt = 0x98529AD287654321;
+    result = 0x52fbec7035a2ca5c;
+
+    __asm
+        ("muleq_s.pw.qhl %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+
+    if (result != rd) {
+        printf("1 muleq_s.pw.qhl error\n");
+
+        return -1;
+    }
+
+    rd = 0;
+    rs = 0x45BC800012345678;
+    rt = 0x9852800087654321;
+    result = 0x52fbec707FFFFFFF;
+
+    __asm
+        ("muleq_s.pw.qhl %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+
+    if (result != rd) {
+        printf("2 muleq_s.pw.qhl error\n");
+
+        return -1;
+    }
+
+    rd = 0;
+    __asm
+        ("rddsp %0\n\t"
+         : "=r"(rd)
+        );
+    rd = rd >> 21;
+    rd = rd & 0x1;
+
+    if (rd != 1) {
+        printf("3 muleq_s.pw.qhl error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/muleq_s_pw_qhr.c b/tests/tcg/mips/mips64-dsp/muleq_s_pw_qhr.c
new file mode 100644
index 000000000..fa8b41fd3
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/muleq_s_pw_qhr.c
@@ -0,0 +1,57 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long result;
+
+    rd = 0;
+    rs = 0x1234567845BCFFFF;
+    rt = 0x8765432198529AD2;
+    result = 0x52fbec7035a2ca5c;
+
+    __asm
+        ("muleq_s.pw.qhr %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+
+    if (result != rd) {
+        printf("1 muleq_s.pw.qhr error\n");
+
+        return -1;
+    }
+
+    rd = 0;
+    rs = 0x1234567845BC8000;
+    rt = 0x8765432198528000;
+    result = 0x52fbec707FFFFFFF;
+
+    __asm
+        ("muleq_s.pw.qhr %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+
+    if (result != rd) {
+        printf("2 muleq_s.pw.qhr error\n");
+
+        return -1;
+    }
+
+    rd = 0;
+    __asm
+        ("rddsp %0\n\t"
+         : "=r"(rd)
+        );
+    rd = rd >> 21;
+    rd = rd & 0x1;
+
+    if (rd != 1) {
+        printf("3 muleq_s.pw.qhr error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/muleq_s_w_phl.c b/tests/tcg/mips/mips64-dsp/muleq_s_w_phl.c
new file mode 100644
index 000000000..997a9f64d
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/muleq_s_w_phl.c
@@ -0,0 +1,46 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, dsp;
+    long long result, resultdsp;
+
+    rs = 0x80009988;
+    rt = 0x80009988;
+    result = 0x7FFFFFFF;
+    resultdsp = 1;
+
+    __asm
+        ("muleq_s.w.phl %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 21) & 0x01;
+    if ((rd  != result) || (dsp != resultdsp)) {
+        printf("muleq_s.w.phl wrong\n");
+
+        return -1;
+    }
+
+    rs = 0x12343322;
+    rt = 0x43213322;
+    result = 0x98be968;
+    resultdsp = 1;
+
+    __asm
+        ("muleq_s.w.phl %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 21) & 0x01;
+    if ((rd  != result) || (dsp != resultdsp)) {
+        printf("muleq_s.w.phl wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
+
diff --git a/tests/tcg/mips/mips64-dsp/muleq_s_w_phr.c b/tests/tcg/mips/mips64-dsp/muleq_s_w_phr.c
new file mode 100644
index 000000000..0e594794d
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/muleq_s_w_phr.c
@@ -0,0 +1,45 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, dsp;
+    long long result, resultdsp;
+
+    rs = 0x8000;
+    rt = 0x8000;
+    result = 0x7FFFFFFF;
+    resultdsp = 1;
+
+    __asm
+        ("muleq_s.w.phr %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 21) & 0x01;
+    if ((rd  != result) || (dsp != resultdsp)) {
+        printf("muleq_s.w.phr wrong\n");
+
+        return -1;
+    }
+
+    rs = 0x1234;
+    rt = 0x4321;
+    result = 0x98be968;
+    resultdsp = 1;
+
+    __asm
+        ("muleq_s.w.phr %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 21) & 0x01;
+    if ((rd  != result) || (dsp != resultdsp)) {
+        printf("muleq_s.w.phr wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/muleu_s_ph_qbl.c b/tests/tcg/mips/mips64-dsp/muleu_s_ph_qbl.c
new file mode 100644
index 000000000..2f444c9f8
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/muleu_s_ph_qbl.c
@@ -0,0 +1,27 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, dsp;
+    long long result, resultdsp;
+
+    rs = 0x80001234;
+    rt = 0x80004321;
+    result = 0xFFFFFFFFFFFF0000;
+    resultdsp = 1;
+
+    __asm
+        ("muleu_s.ph.qbl %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 21) & 0x01;
+    if ((rd  != result) || (dsp != resultdsp)) {
+        printf("muleu_s.ph.qbl wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/muleu_s_ph_qbr.c b/tests/tcg/mips/mips64-dsp/muleu_s_ph_qbr.c
new file mode 100644
index 000000000..8bd0e9942
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/muleu_s_ph_qbr.c
@@ -0,0 +1,27 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, dsp;
+    long long result, resultdsp;
+
+    rs = 0x8000;
+    rt = 0x80004321;
+    result = 0xFFFFFFFFFFFF0000;
+    resultdsp = 1;
+
+    __asm
+        ("muleu_s.ph.qbr %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 21) & 0x01;
+    if ((rd  != result) || (dsp != resultdsp)) {
+        printf("muleu_s.ph.qbr wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/muleu_s_qh_obl.c b/tests/tcg/mips/mips64-dsp/muleu_s_qh_obl.c
new file mode 100644
index 000000000..db0d386e8
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/muleu_s_qh_obl.c
@@ -0,0 +1,30 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long dsp;
+    long long resdsp, result;
+
+    rd = 0;
+    rs = 0x1234567802020202;
+    rt = 0x0034432112344321;
+    result = 0x03A8FFFFFFFFFFFF;
+    resdsp = 0x01;
+
+    __asm
+        ("muleu_s.qh.obl %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+
+    dsp = (dsp >> 21) & 0x01;
+    if ((rd != result) || (resdsp != dsp)) {
+        printf("muleu_s.qh.obl error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/muleu_s_qh_obr.c b/tests/tcg/mips/mips64-dsp/muleu_s_qh_obr.c
new file mode 100644
index 000000000..52ed9c095
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/muleu_s_qh_obr.c
@@ -0,0 +1,31 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long dsp;
+    long long resdsp, result;
+
+    rd = 0;
+    rs = 0x0202020212345678;
+
+    rt = 0x0034432112344321;
+    result = 0x03A8FFFFFFFFFFFF;
+    resdsp = 0x01;
+
+    __asm
+        ("muleu_s.qh.obr %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+
+    dsp = (dsp >> 21) & 0x01;
+    if ((rd != result) || (resdsp != dsp)) {
+        printf("muleu_s.qh.obr error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/mulq_rs_ph.c b/tests/tcg/mips/mips64-dsp/mulq_rs_ph.c
new file mode 100644
index 000000000..fd6233d4d
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/mulq_rs_ph.c
@@ -0,0 +1,27 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, dsp;
+    long long result, resultdsp;
+
+    rs = 0x80001234;
+    rt = 0x80004321;
+    result = 0x7FFF098C;
+    resultdsp = 1;
+
+    __asm
+        ("mulq_rs.ph %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 21) & 0x01;
+    if ((rd  != result) || (dsp != resultdsp)) {
+        printf("mulq_rs.ph wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/mulq_rs_qh.c b/tests/tcg/mips/mips64-dsp/mulq_rs_qh.c
new file mode 100644
index 000000000..7863c0591
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/mulq_rs_qh.c
@@ -0,0 +1,33 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, result, dsp, dspresult;
+    rt = 0x80003698CE8F9201;
+    rs = 0x800034634BCDE321;
+    result = 0x7fff16587a530313;
+
+    dspresult = 0x01;
+
+    __asm
+        ("mulq_rs.qh %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt), "r"(rs)
+        );
+
+    if (rd != result) {
+        printf("mulq_rs.qh error\n");
+
+        return -1;
+    }
+
+    dsp = (dsp >> 21) & 0x01;
+    if (dsp != dspresult) {
+        printf("mulq_rs.qh DSPControl Reg ouflag error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/mulsaq_s_l_pw.c b/tests/tcg/mips/mips64-dsp/mulsaq_s_l_pw.c
new file mode 100644
index 000000000..02548f85c
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/mulsaq_s_l_pw.c
@@ -0,0 +1,59 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rs, rt, dsp;
+    long long achi, acli;
+    long long acho, aclo;
+    long long resl, resh;
+
+    achi = 0x4;
+    acli = 0x4;
+
+    rs = 0x1234567887654321;
+    rt = 0x8765432112345678;
+
+    resh = 0x4;
+    resl = 0x4;
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "mulsaq_s.l.pw $ac1, %4, %5\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+
+    if ((acho != resh) || (aclo != resl)) {
+        printf("1 mulsaq_s.l.pw wrong\n");
+
+        return -1;
+    }
+
+    achi = 0x4;
+    acli = 0x4;
+
+    rs = 0x8000000087654321;
+    rt = 0x8000000012345678;
+
+    resh = 0x4;
+    resl = 0x1e8ee513;
+    __asm
+        ("mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "mulsaq_s.l.pw $ac1, %5, %6\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "=r"(acho), "=r"(aclo), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 17) & 0x1;
+    if ((dsp != 0x1) || (acho != resh) || (aclo != resl)) {
+        printf("2 mulsaq_s.l.pw wrong\n");
+
+        return -1;
+    }
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/mulsaq_s_w_qh.c b/tests/tcg/mips/mips64-dsp/mulsaq_s_w_qh.c
new file mode 100644
index 000000000..92d7a0b4f
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/mulsaq_s_w_qh.c
@@ -0,0 +1,57 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rs, rt, dsp;
+    long long achi, acli;
+    long long acho, aclo;
+    long long resl, resh;
+
+    achi = 0x4;
+    acli = 0x4;
+
+    rs = 0x5678123443218765;
+    rt = 0x4321876556781234;
+
+    resh = 0x4;
+    resl = 0x342fcbd4;
+    __asm
+        ("mthi %2, $ac1\n\t"
+         "mtlo %3, $ac1\n\t"
+         "mulsaq_s.w.qh $ac1, %4, %5\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+
+    if ((acho != resh) || (aclo != resl)) {
+        printf("1 mulsaq_s.w.qh wrong\n");
+        return -1;
+    }
+
+    achi = 0x4;
+    acli = 0x4;
+
+    rs = 0x8000800087654321;
+    rt = 0x8000800012345678;
+
+    resh = 0x3;
+    resl = 0xffffffffe5e81a1c;
+    __asm
+        ("mthi %3, $ac1\n\t"
+         "mtlo %4, $ac1\n\t"
+         "mulsaq_s.w.qh $ac1, %5, %6\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "=r"(acho), "=r"(aclo), "=r"(dsp)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 17) & 0x1;
+    if ((dsp != 0x1) || (acho != resh) || (aclo != resl)) {
+        printf("2 mulsaq_s.w.qh wrong\n");
+        return -1;
+    }
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/mult.c b/tests/tcg/mips/mips64-dsp/mult.c
new file mode 100644
index 000000000..4a294d1a0
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/mult.c
@@ -0,0 +1,26 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rs, rt, ach, acl;
+    long long result, resulth, resultl;
+
+    rs  = 0x00FFBBAA;
+    rt  = 0x4B231000;
+    resulth = 0x4b0f01;
+    resultl = 0x71f8a000;
+    __asm
+        ("mult $ac1, %2, %3\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "=r"(ach), "=r"(acl)
+         : "r"(rs), "r"(rt)
+        );
+    if ((ach != resulth) || (acl != resultl)) {
+        printf("mult wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/multu.c b/tests/tcg/mips/mips64-dsp/multu.c
new file mode 100644
index 000000000..21a8a7c77
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/multu.c
@@ -0,0 +1,26 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rs, rt, ach, acl;
+    long long result, resulth, resultl;
+
+    rs  = 0x00FFBBAA;
+    rt  = 0x4B231000;
+    resulth = 0x4b0f01;
+    resultl = 0x71f8a000;
+    __asm
+        ("multu $ac1, %2, %3\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "=r"(ach), "=r"(acl)
+         : "r"(rs), "r"(rt)
+        );
+    if ((ach != resulth) || (acl != resultl)) {
+        printf("multu wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/packrl_ph.c b/tests/tcg/mips/mips64-dsp/packrl_ph.c
new file mode 100644
index 000000000..3722b0ae6
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/packrl_ph.c
@@ -0,0 +1,24 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long result;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result = 0x56788765;
+
+    __asm
+        ("packrl.ph %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    if (result != rd) {
+        printf("packrl.ph wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/packrl_pw.c b/tests/tcg/mips/mips64-dsp/packrl_pw.c
new file mode 100644
index 000000000..780741883
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/packrl_pw.c
@@ -0,0 +1,24 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long res;
+
+    rs = 0x1234567887654321;
+    rt = 0xabcdef9812345678;
+
+    res = 0x87654321abcdef98;
+
+    __asm
+        ("packrl.pw %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    if (rd != res) {
+        printf("packrl.pw error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/pick_ob.c b/tests/tcg/mips/mips64-dsp/pick_ob.c
new file mode 100644
index 000000000..160049ffd
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/pick_ob.c
@@ -0,0 +1,66 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, dsp;
+    long long res;
+
+    dsp = 0xff000000;
+
+    rs = 0x1234567812345678;
+    rt = 0x8765432187654321;
+
+    res = 0x1234567812345678;
+
+    __asm
+        ("wrdsp %1, 0x10\n\t"
+         "pick.ob %0, %2, %3\n\t"
+         : "=r"(rd)
+         : "r"(dsp), "r"(rs), "r"(rt)
+        );
+
+    if (rd != res) {
+        printf("1 pick.ob error\n");
+        return -1;
+    }
+
+    dsp = 0x00000000;
+
+    rs = 0x1234567812345678;
+    rt = 0x8765432187654321;
+
+    res = 0x8765432187654321;
+
+    __asm
+        ("wrdsp %1, 0x10\n\t"
+         "pick.ob %0, %2, %3\n\t"
+         : "=r"(rd)
+         : "r"(dsp), "r"(rs), "r"(rt)
+        );
+
+    if (rd != res) {
+        printf("2 pick.ob error\n");
+        return -1;
+    }
+
+    dsp = 0x34000000;
+
+    rs = 0x1234567812345678;
+    rt = 0x8765432187654321;
+
+    res = 0x8765567887344321;
+
+    __asm
+        ("wrdsp %1, 0x10\n\t"
+         "pick.ob %0, %2, %3\n\t"
+         : "=r"(rd)
+         : "r"(dsp), "r"(rs), "r"(rt)
+        );
+
+    if (rd != res) {
+        printf("3 pick.ob error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/pick_ph.c b/tests/tcg/mips/mips64-dsp/pick_ph.c
new file mode 100644
index 000000000..8800c14d1
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/pick_ph.c
@@ -0,0 +1,60 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, dsp;
+    long long result;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    dsp = 0x0A000000;
+    result = 0x12344321;
+
+    __asm
+        ("wrdsp %3, 0x10\n\t"
+         "pick.ph %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt), "r"(dsp)
+        );
+    if (rd != result) {
+        printf("1 pick.ph wrong\n");
+
+        return -1;
+    }
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    dsp = 0x03000000;
+    result = 0x12345678;
+
+    __asm
+        ("wrdsp %3, 0x10\n\t"
+         "pick.ph %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt), "r"(dsp)
+        );
+    if (rd != result) {
+        printf("2 pick.ph wrong\n");
+
+        return -1;
+    }
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    dsp = 0x00000000;
+    result = 0xffffffff87654321;
+
+    __asm
+        ("wrdsp %3, 0x10\n\t"
+         "pick.ph %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt), "r"(dsp)
+        );
+    if (rd != result) {
+        printf("3 pick.ph wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/pick_pw.c b/tests/tcg/mips/mips64-dsp/pick_pw.c
new file mode 100644
index 000000000..24d80f551
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/pick_pw.c
@@ -0,0 +1,48 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, dsp;
+    long long res;
+    dsp = 0xff000000;
+
+    rs = 0x1234567812345678;
+    rt = 0x8765432187654321;
+
+    res = 0x1234567812345678;
+
+    __asm
+        ("wrdsp %1, 0x10\n\t"
+         "wrdsp %1\n\t"
+         "pick.pw %0, %2, %3\n\t"
+         : "=r"(rd), "+r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+
+    if (rd != res) {
+        printf("pick.pw error\n");
+        return -1;
+    }
+
+    dsp = 0x00000000;
+
+    rs = 0x1234567812345678;
+    rt = 0x8765432187654321;
+
+    res = 0x8765432187654321;
+
+    __asm
+        ("wrdsp %1, 0x10\n\t"
+         "wrdsp %1\n\t"
+         "pick.pw %0, %2, %3\n\t"
+         : "=r"(rd), "+r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+
+    if (rd != res) {
+        printf("pick.pw error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/pick_qb.c b/tests/tcg/mips/mips64-dsp/pick_qb.c
new file mode 100644
index 000000000..0d5de9db9
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/pick_qb.c
@@ -0,0 +1,43 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, dsp;
+    long long result;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    dsp = 0x0f000000;
+    result = 0x12345678;
+
+    __asm
+        ("wrdsp %3, 0x10\n\t"
+         "pick.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt), "r"(dsp)
+        );
+    if (rd != result) {
+        printf("pick.qb wrong\n");
+
+        return -1;
+    }
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    dsp = 0x00000000;
+    result = 0xffffffff87654321;
+
+    __asm
+        ("wrdsp %3, 0x10\n\t"
+         "pick.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt), "r"(dsp)
+        );
+    if (rd != result) {
+        printf("pick.qb wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/pick_qh.c b/tests/tcg/mips/mips64-dsp/pick_qh.c
new file mode 100644
index 000000000..aa2e2938a
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/pick_qh.c
@@ -0,0 +1,48 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, dsp;
+    long long res;
+    dsp = 0xff000000;
+
+    rs = 0x1234567812345678;
+    rt = 0x8765432187654321;
+
+    res = 0x1234567812345678;
+
+    __asm
+        ("wrdsp %1, 0x10\n\t"
+         "wrdsp %1\n\t"
+         "pick.qh %0, %2, %3\n\t"
+         : "=r"(rd), "+r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+
+    if (rd != res) {
+        printf("pick.qh error\n");
+        return -1;
+    }
+
+    dsp = 0x00000000;
+
+    rs = 0x1234567812345678;
+    rt = 0x8765432187654321;
+
+    res = 0x8765432187654321;
+
+    __asm
+        ("wrdsp %1, 0x10\n\t"
+         "wrdsp %1\n\t"
+         "pick.qh %0, %2, %3\n\t"
+         : "=r"(rd), "+r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+
+    if (rd != res) {
+        printf("pick.qh error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/preceq_l_pwl.c b/tests/tcg/mips/mips64-dsp/preceq_l_pwl.c
new file mode 100644
index 000000000..64551007c
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/preceq_l_pwl.c
@@ -0,0 +1,24 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt;
+    long long result;
+    rt = 0xFFFFFFFF11111111;
+    result = 0xFFFFFFFF00000000;
+
+    __asm
+        ("preceq.l.pwl %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+
+    if (result != rd) {
+        printf("preceq.l.pwl wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
+
diff --git a/tests/tcg/mips/mips64-dsp/preceq_l_pwr.c b/tests/tcg/mips/mips64-dsp/preceq_l_pwr.c
new file mode 100644
index 000000000..1e05339d2
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/preceq_l_pwr.c
@@ -0,0 +1,24 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt;
+    long long result;
+    rt = 0xFFFFFFFF11111111;
+    result = 0x1111111100000000;
+
+    __asm
+        ("preceq.l.pwl %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+
+    if (result != rd) {
+        printf("preceq.l.pwr wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
+
diff --git a/tests/tcg/mips/mips64-dsp/preceq_pw_qhl.c b/tests/tcg/mips/mips64-dsp/preceq_pw_qhl.c
new file mode 100644
index 000000000..f44b94049
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/preceq_pw_qhl.c
@@ -0,0 +1,21 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt, result;
+    rt = 0x0123456789ABCDEF;
+    result = 0x0123000045670000;
+
+    __asm
+        ("preceq.pw.qhl %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    if (result != rd) {
+        printf("preceq.pw.qhl error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/preceq_pw_qhla.c b/tests/tcg/mips/mips64-dsp/preceq_pw_qhla.c
new file mode 100644
index 000000000..f0f78f43c
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/preceq_pw_qhla.c
@@ -0,0 +1,23 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt, result;
+
+    rt = 0x123456789ABCDEF0;
+    result = 0x123400009ABC0000;
+
+    __asm
+        ("preceq.pw.qhla %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+
+    if (result != rd) {
+        printf("preceq.pw.qhla error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/preceq_pw_qhr.c b/tests/tcg/mips/mips64-dsp/preceq_pw_qhr.c
new file mode 100644
index 000000000..709d4f900
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/preceq_pw_qhr.c
@@ -0,0 +1,21 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt, result;
+    rt = 0x0123456789ABCDEF;
+    result = 0x89AB0000CDEF0000;
+
+    __asm
+        ("preceq.pw.qhr %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    if (result != rd) {
+        printf("preceq.pw.qhr error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/preceq_pw_qhra.c b/tests/tcg/mips/mips64-dsp/preceq_pw_qhra.c
new file mode 100644
index 000000000..4d071ec86
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/preceq_pw_qhra.c
@@ -0,0 +1,23 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt, result;
+
+    rt = 0x123456789ABCDEF0;
+    result = 0x56780000DEF00000;
+
+    __asm
+        ("preceq.pw.qhra %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+
+    if (result != rd) {
+        printf("preceq.pw.qhra error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/preceq_w_phl.c b/tests/tcg/mips/mips64-dsp/preceq_w_phl.c
new file mode 100644
index 000000000..4ed3fc030
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/preceq_w_phl.c
@@ -0,0 +1,23 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt;
+    long long result;
+
+    rt = 0x87654321;
+    result = 0xFFFFFFFF87650000;
+
+    __asm
+        ("preceq.w.phl %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    if (result != rd) {
+        printf("preceq.w.phl wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/preceq_w_phr.c b/tests/tcg/mips/mips64-dsp/preceq_w_phr.c
new file mode 100644
index 000000000..e2ea0933b
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/preceq_w_phr.c
@@ -0,0 +1,23 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt;
+    long long result;
+
+    rt = 0x87654321;
+    result = 0x43210000;
+
+    __asm
+        ("preceq.w.phr %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    if (result != rd) {
+        printf("preceq.w.phr wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/precequ_ph_qbl.c b/tests/tcg/mips/mips64-dsp/precequ_ph_qbl.c
new file mode 100644
index 000000000..17b73311d
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/precequ_ph_qbl.c
@@ -0,0 +1,23 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt;
+    long long result;
+
+    rt = 0x87654321;
+    result = 0x43803280;
+
+    __asm
+        ("precequ.ph.qbl %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    if (result != rd) {
+        printf("precequ.ph.qbl wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/precequ_ph_qbla.c b/tests/tcg/mips/mips64-dsp/precequ_ph_qbla.c
new file mode 100644
index 000000000..15e94946b
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/precequ_ph_qbla.c
@@ -0,0 +1,23 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt;
+    long long result;
+
+    rt = 0x87654321;
+    result = 0x43802180;
+
+    __asm
+        ("precequ.ph.qbla %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    if (result != rd) {
+        printf("precequ.ph.qbla wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/precequ_ph_qbr.c b/tests/tcg/mips/mips64-dsp/precequ_ph_qbr.c
new file mode 100644
index 000000000..495368ce0
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/precequ_ph_qbr.c
@@ -0,0 +1,23 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt;
+    long long result;
+
+    rt = 0x87654321;
+    result = 0x21801080;
+
+    __asm
+        ("precequ.ph.qbr %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    if (result != rd) {
+        printf("precequ.ph.qbr wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/precequ_ph_qbra.c b/tests/tcg/mips/mips64-dsp/precequ_ph_qbra.c
new file mode 100644
index 000000000..7c6636975
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/precequ_ph_qbra.c
@@ -0,0 +1,23 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt;
+    long long result;
+
+    rt = 0x87654321;
+    result = 0x32801080;
+
+    __asm
+        ("precequ.ph.qbra %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    if (result != rd) {
+        printf("precequ.ph.qbra wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/precequ_qh_obl.c b/tests/tcg/mips/mips64-dsp/precequ_qh_obl.c
new file mode 100644
index 000000000..176d2365a
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/precequ_qh_obl.c
@@ -0,0 +1,22 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt, result;
+    rt = 0x123456789ABCDEF0;
+    result = 0x09001A002B003C00;
+
+    __asm
+        ("precequ.qh.obla %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+
+    if (result != rd) {
+        printf("precequ.qh.obla error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/precequ_qh_obla.c b/tests/tcg/mips/mips64-dsp/precequ_qh_obla.c
new file mode 100644
index 000000000..93a36a485
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/precequ_qh_obla.c
@@ -0,0 +1,22 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt, result;
+    rt = 0x123456789ABCDEF0;
+    result = 0x09002B004D006F00;
+
+    __asm
+        ("precequ.qh.obla %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+
+    if (result != rd) {
+        printf("precequ.qh.obla error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/precequ_qh_obr.c b/tests/tcg/mips/mips64-dsp/precequ_qh_obr.c
new file mode 100644
index 000000000..121473083
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/precequ_qh_obr.c
@@ -0,0 +1,24 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt, result;
+
+    rt = 0x123456789ABCDEF0;
+    result = 0x4D005E006F007000;
+
+    __asm
+        ("precequ.qh.obr %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+
+    if (result != rd) {
+        printf("precequ.qh.obr error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
+
diff --git a/tests/tcg/mips/mips64-dsp/precequ_qh_obra.c b/tests/tcg/mips/mips64-dsp/precequ_qh_obra.c
new file mode 100644
index 000000000..3aa0e096c
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/precequ_qh_obra.c
@@ -0,0 +1,24 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt, result;
+
+    rt = 0x123456789ABCDEF0;
+    result = 0x1A003C005D007000;
+
+    __asm
+        ("precequ.qh.obra %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+
+    if (result != rd) {
+        printf("precequ.qh.obra error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
+
diff --git a/tests/tcg/mips/mips64-dsp/preceu_ph_qbl.c b/tests/tcg/mips/mips64-dsp/preceu_ph_qbl.c
new file mode 100644
index 000000000..81f7917c1
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/preceu_ph_qbl.c
@@ -0,0 +1,23 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt;
+    long long result;
+
+    rt = 0x87654321;
+    result = 0x00870065;
+
+    __asm
+        ("preceu.ph.qbl %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    if (result != rd) {
+        printf("preceu.ph.qbl wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/preceu_ph_qbla.c b/tests/tcg/mips/mips64-dsp/preceu_ph_qbla.c
new file mode 100644
index 000000000..38cf6a62b
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/preceu_ph_qbla.c
@@ -0,0 +1,23 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt;
+    long long result;
+
+    rt = 0x87654321;
+    result = 0x00870043;
+
+    __asm
+        ("preceu.ph.qbla %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    if (result != rd) {
+        printf("preceu.ph.qbla wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/preceu_ph_qbr.c b/tests/tcg/mips/mips64-dsp/preceu_ph_qbr.c
new file mode 100644
index 000000000..70c32b671
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/preceu_ph_qbr.c
@@ -0,0 +1,23 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt;
+    long long result;
+
+    rt = 0x87654321;
+    result = 0x00430021;
+
+    __asm
+        ("preceu.ph.qbr %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    if (result != rd) {
+        printf("preceu.ph.qbr wrong");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/preceu_ph_qbra.c b/tests/tcg/mips/mips64-dsp/preceu_ph_qbra.c
new file mode 100644
index 000000000..c6638aaaf
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/preceu_ph_qbra.c
@@ -0,0 +1,23 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt;
+    long long result;
+
+    rt = 0x87654321;
+    result = 0x00650021;
+
+    __asm
+        ("preceu.ph.qbra %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    if (result != rd) {
+        printf("preceu.ph.qbra wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/preceu_qh_obl.c b/tests/tcg/mips/mips64-dsp/preceu_qh_obl.c
new file mode 100644
index 000000000..63f9373b0
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/preceu_qh_obl.c
@@ -0,0 +1,22 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt, result;
+    rt = 0x123456789ABCDEF0;
+    result = 0x0012003400560078;
+
+    __asm
+        ("preceu.qh.obl %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+
+    if (result != rd) {
+        printf("preceu.qh.obl error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/preceu_qh_obla.c b/tests/tcg/mips/mips64-dsp/preceu_qh_obla.c
new file mode 100644
index 000000000..5fb65e404
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/preceu_qh_obla.c
@@ -0,0 +1,22 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt, result;
+    rt = 0x123456789ABCDEF0;
+    result = 0x00120056009A00DE;
+
+    __asm
+        ("preceu.qh.obla %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+
+    if (result != rd) {
+        printf("preceu.qh.obla error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/preceu_qh_obr.c b/tests/tcg/mips/mips64-dsp/preceu_qh_obr.c
new file mode 100644
index 000000000..9af3b6372
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/preceu_qh_obr.c
@@ -0,0 +1,23 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt, result;
+
+    rt = 0x123456789ABCDEF0;
+    result = 0x009A00BC00DE00F0;
+
+    __asm
+        ("preceu.qh.obr %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+
+    if (result != rd) {
+        printf("preceu.qh.obr error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/preceu_qh_obra.c b/tests/tcg/mips/mips64-dsp/preceu_qh_obra.c
new file mode 100644
index 000000000..fd0408337
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/preceu_qh_obra.c
@@ -0,0 +1,23 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt, result;
+
+    rt = 0x123456789ABCDEF0;
+    result = 0x0034007800BC00F0;
+
+    __asm
+        ("preceu.qh.obra %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+
+    if (result != rd) {
+        printf("preceu.qh.obra error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/precr_ob_qh.c b/tests/tcg/mips/mips64-dsp/precr_ob_qh.c
new file mode 100644
index 000000000..ce2da79af
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/precr_ob_qh.c
@@ -0,0 +1,25 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long res;
+
+    rs = 0x1234567812345678;
+    rt = 0x8765432187654321;
+
+    res = 0x3478347865216521;
+
+    __asm
+        ("precr.ob.qh %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+
+    if (rd != res) {
+        printf("precr.ob.qh error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/precr_sra_qh_pw.c b/tests/tcg/mips/mips64-dsp/precr_sra_qh_pw.c
new file mode 100644
index 000000000..8bb16de9a
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/precr_sra_qh_pw.c
@@ -0,0 +1,40 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rs, rt;
+    long long res;
+
+    rt = 0x8765432187654321;
+    rs = 0x1234567812345678;
+
+    res = 0x4321432156785678;
+
+    __asm
+        ("precr_sra.qh.pw %0, %1, 0x0\n\t"
+         : "=r"(rt)
+         : "r"(rs)
+        );
+
+    if (rt != res) {
+        printf("precr_sra.qh.pw error\n");
+        return -1;
+    }
+
+    rt = 0x8765432187654321;
+    rs = 0x1234567812345678;
+
+    res = 0x5432543245674567;
+
+    __asm
+        ("precr_sra.qh.pw %0, %1, 0x4\n\t"
+         : "=r"(rt)
+         : "r"(rs)
+        );
+
+    if (rt != res) {
+        printf("precr_sra.qh.pw error\n");
+        return -1;
+    }
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/precr_sra_r_qh_pw.c b/tests/tcg/mips/mips64-dsp/precr_sra_r_qh_pw.c
new file mode 100644
index 000000000..734ac322e
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/precr_sra_r_qh_pw.c
@@ -0,0 +1,40 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rs, rt;
+    long long res;
+
+    rt = 0x8765432187654321;
+    rs = 0x1234567812345678;
+
+    res = 0x4321432156785678;
+
+    __asm
+        ("precr_sra_r.qh.pw %0, %1, 0x0\n\t"
+         : "=r"(rt)
+         : "r"(rs)
+        );
+
+    if (rt != res) {
+        printf("precr_sra_r.qh.pw error\n");
+        return -1;
+    }
+
+    rt = 0x8765432187654321;
+    rs = 0x1234567812345678;
+
+    res = 0x5432543245684568;
+
+    __asm
+        ("precr_sra_r.qh.pw %0, %1, 0x4\n\t"
+         : "=r"(rt)
+         : "r"(rs)
+        );
+
+    if (rt != res) {
+        printf("precr_sra_r.qh.pw error\n");
+        return -1;
+    }
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/precrq_ob_qh.c b/tests/tcg/mips/mips64-dsp/precrq_ob_qh.c
new file mode 100644
index 000000000..4f61b1709
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/precrq_ob_qh.c
@@ -0,0 +1,25 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long res;
+
+    rs = 0x1234567812345678;
+    rt = 0x8765432187654321;
+
+    res = 0x1256125687438743;
+
+    __asm
+        ("precrq.ob.qh %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+
+    if (rd != res) {
+        printf("precrq.ob.qh error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/precrq_ph_w.c b/tests/tcg/mips/mips64-dsp/precrq_ph_w.c
new file mode 100644
index 000000000..f0946abdc
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/precrq_ph_w.c
@@ -0,0 +1,24 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long result;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result = 0x12348765;
+
+    __asm
+        ("precrq.ph.w %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    if (result != rd) {
+        printf("precrq.ph.w wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/precrq_pw_l.c b/tests/tcg/mips/mips64-dsp/precrq_pw_l.c
new file mode 100644
index 000000000..da957c074
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/precrq_pw_l.c
@@ -0,0 +1,25 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long res;
+
+    rs = 0x1234567812345678;
+    rt = 0x8765432187654321;
+
+    res = 0x1234567887654321;
+
+    __asm
+        ("precrq.pw.l %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+
+    if (rd != res) {
+        printf("precrq.pw.l error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/precrq_qb_ph.c b/tests/tcg/mips/mips64-dsp/precrq_qb_ph.c
new file mode 100644
index 000000000..f417c9f34
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/precrq_qb_ph.c
@@ -0,0 +1,24 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long result;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result = 0x12568743;
+
+    __asm
+        ("precrq.qb.ph %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    if (result != rd) {
+        printf("precrq.qb.ph wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/precrq_qh_pw.c b/tests/tcg/mips/mips64-dsp/precrq_qh_pw.c
new file mode 100644
index 000000000..4a4ffef8e
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/precrq_qh_pw.c
@@ -0,0 +1,25 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long res;
+
+    rs = 0x1234567812345678;
+    rt = 0x8765432187654321;
+
+    res = 0x1234123487658765;
+
+    __asm
+        ("precrq.qh.pw %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+
+    if (rd != res) {
+        printf("precrq.qh.pw error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/precrq_rs_ph_w.c b/tests/tcg/mips/mips64-dsp/precrq_rs_ph_w.c
new file mode 100644
index 000000000..61da3331f
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/precrq_rs_ph_w.c
@@ -0,0 +1,41 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long dsp;
+    long long result;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result = 0x12348765;
+
+    __asm
+        ("precrq_rs.ph.w %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    if (result != rd) {
+        printf("1 precrq_rs.ph.w wrong\n");
+
+        return -1;
+    }
+
+    rs = 0x7fffC678;
+    rt = 0x865432A0;
+    result = 0x7fff8654;
+
+    __asm
+        ("precrq_rs.ph.w %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    if ((result != rd) || (((dsp >> 22) & 0x01) != 1)) {
+        printf("2 precrq_rs.ph.w wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/precrq_rs_qh_pw.c b/tests/tcg/mips/mips64-dsp/precrq_rs_qh_pw.c
new file mode 100644
index 000000000..ac78728ab
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/precrq_rs_qh_pw.c
@@ -0,0 +1,43 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long dsp;
+    long long res;
+
+    rs = 0x1234567812345678;
+    rt = 0x8765432187654321;
+
+    res = 0x1234123487658765;
+
+    __asm
+        ("precrq_rs.qh.pw %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+
+    if (rd != res) {
+        printf("precrq_rs.qh.pw error\n");
+        return -1;
+    }
+
+    rs = 0x7fffC67812345678;
+    rt = 0x8765432187654321;
+
+    res = 0x7fff123487658765;
+
+    __asm
+        ("precrq_rs.qh.pw %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+
+    if (rd != res) {
+        printf("precrq_rs.qh.pw error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/precrqu_s_ob_qh.c b/tests/tcg/mips/mips64-dsp/precrqu_s_ob_qh.c
new file mode 100644
index 000000000..e27c36b7f
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/precrqu_s_ob_qh.c
@@ -0,0 +1,27 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, dsp;
+    long long res, resdsp;
+
+    rs = 0x7fff567812345678;
+    rt = 0x8765432187654321;
+
+    res = 0xffac24ac00860086;
+    resdsp = 0x1;
+
+    __asm
+        ("precrqu_s.ob.qh %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 22) & 0x1;
+    if ((rd != res) || (dsp != resdsp)) {
+        printf("precrq_s.ob.qh error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/precrqu_s_qb_ph.c b/tests/tcg/mips/mips64-dsp/precrqu_s_qb_ph.c
new file mode 100644
index 000000000..cb1fee450
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/precrqu_s_qb_ph.c
@@ -0,0 +1,26 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long dsp;
+    long long result;
+
+    rs = 0x12345678;
+    rt = 0x87657fff;
+    result = 0x24AC00FF;
+
+    __asm
+        ("precrqu_s.qb.ph %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    if ((result != rd) || (((dsp >> 22) & 0x01) != 0x01)) {
+        printf("precrqu_s.qb.ph wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/prependd.c b/tests/tcg/mips/mips64-dsp/prependd.c
new file mode 100644
index 000000000..b4208c2da
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/prependd.c
@@ -0,0 +1,37 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, rs;
+    long long res;
+    rt = 0x1234567887654321;
+    rs = 0xabcd1234abcd8765;
+
+    res = 0x1234567887654321;
+    __asm
+        ("prependd %0, %1, 0x0\n\t"
+         : "=r"(rt)
+         : "r"(rs)
+        );
+
+    if (rt != res) {
+        printf("prependd error\n");
+        return -1;
+    }
+
+    rt = 0x1234567887654321;
+    rs = 0xabcd1234abcd8765;
+
+    res = 0xd876512345678876;
+    __asm
+        ("prependd %0, %1, 0x4\n\t"
+         : "=r"(rt)
+         : "r"(rs)
+        );
+
+    if (rt != res) {
+        printf("prependd error\n");
+        return -1;
+    }
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/prependw.c b/tests/tcg/mips/mips64-dsp/prependw.c
new file mode 100644
index 000000000..d91bd2023
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/prependw.c
@@ -0,0 +1,37 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, rs;
+    long long res;
+    rt = 0x1234567887654321;
+    rs = 0xabcd1234abcd8765;
+
+    res = 0x1234567887654321;
+    __asm
+        ("prependw %0, %1, 0x0\n\t"
+         : "=r"(rt)
+         : "r"(rs)
+        );
+
+    if (rt != res) {
+        printf("prependw error\n");
+        return -1;
+    }
+
+    rt = 0x1234567887654321;
+    rs = 0xabcd1234abcd8765;
+
+    res = 0x5123456788765432;
+    __asm
+        ("prependw %0, %1, 0x4\n\t"
+         : "=r"(rt)
+         : "r"(rs)
+        );
+
+    if (rt != res) {
+        printf("prependw error\n");
+        return -1;
+    }
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/printf.c b/tests/tcg/mips/mips64-dsp/printf.c
new file mode 100644
index 000000000..cf8676d39
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/printf.c
@@ -0,0 +1,266 @@
+
+typedef unsigned long va_list;
+
+#define ACC    4
+#define __read(source)                    \
+({ va_list __res;                    \
+    __asm__ __volatile__(                \
+        "move\t%0, " #source "\n\t"        \
+        : "=r" (__res));            \
+    __res;                        \
+})
+
+enum format_type {
+    FORMAT_TYPE_NONE,
+    FORMAT_TYPE_HEX,
+    FORMAT_TYPE_ULONG,
+    FORMAT_TYPE_FLOAT
+};
+
+struct printf_spec {
+    char    type;
+};
+
+static int format_decode(char *fmt, struct printf_spec *spec)
+{
+    char *start = fmt;
+
+    for (; *fmt ; ++fmt) {
+        if (*fmt == '%') {
+            break;
+        }
+    }
+
+    switch (*++fmt) {
+    case 'x':
+        spec->type = FORMAT_TYPE_HEX;
+        break;
+
+    case 'd':
+        spec->type = FORMAT_TYPE_ULONG;
+        break;
+
+    case 'f':
+        spec->type = FORMAT_TYPE_FLOAT;
+        break;
+
+    default:
+        spec->type = FORMAT_TYPE_NONE;
+    }
+
+    return ++fmt - start;
+}
+
+void *memcpy(void *dest, void *src, int n)
+{
+    int i;
+    char *s = src;
+    char *d = dest;
+
+    for (i = 0; i < n; i++) {
+        d[i] = s[i];
+    }
+    return dest;
+}
+
+char *number(char *buf, va_list num)
+{
+    int i;
+    char *str = buf;
+    static char digits[16] = "0123456789abcdef";
+    str = str + sizeof(num) * 2;
+
+    for (i = 0; i < sizeof(num) * 2; i++) {
+        *--str = digits[num & 15];
+        num >>= 4;
+    }
+
+    return buf + sizeof(num) * 2;
+}
+
+char *__number(char *buf, va_list num)
+{
+    int i;
+    va_list mm = num;
+    char *str = buf;
+
+    if (!num) {
+        *str++ = '0';
+        return str;
+    }
+
+    for (i = 0; mm; mm = mm/10, i++) {
+        /* Do nothing. */
+    }
+
+    str = str + i;
+
+    while (num) {
+        *--str = num % 10 + 48;
+        num = num / 10;
+    }
+
+    return str + i;
+}
+
+va_list modf(va_list args, va_list *integer, va_list *num)
+{
+    int i;
+    double dot_v = 0;
+    va_list E, DOT, DOT_V;
+
+    if (!args) {
+        return 0;
+    }
+
+    for (i = 0, args = args << 1 >> 1; i < 52; i++) {
+        if ((args >> i) & 0x1) {
+            break;
+        }
+    }
+
+    *integer = 0;
+
+    if ((args >> 56 != 0x3f) || (args >> 52 == 0x3ff)) {
+        E = (args >> 52) - 1023;
+        DOT = 52 - E - i;
+        DOT_V = args << (12 + E) >> (12 + E) >> i;
+        *integer = ((args << 12 >> 12) >> (i + DOT)) | (1 << E);
+    } else {
+        E = ~((args >> 52) - 1023) + 1;
+        DOT_V = args << 12 >> 12;
+
+        dot_v += 1.0 / (1 << E);
+
+        for (i = 1; i <= 16; i++) {
+            if ((DOT_V >> (52 - i)) & 0x1) {
+                dot_v += 1.0 / (1 << E + i);
+            }
+        }
+
+        for (i = 1, E = 0; i <= ACC; i++) {
+            dot_v *= 10;
+            if (!(va_list)dot_v) {
+                E++;
+            }
+    }
+
+    *num = E;
+
+    return dot_v;
+    }
+
+    if (args & 0xf) {
+        for (i = 1; i <= 16; i++) {
+            if ((DOT_V >> (DOT - i)) & 0x1) {
+                dot_v += 1.0 / (1 << i);
+            }
+        }
+
+        for (i = 1, E = 0; i <= ACC; i++) {
+            dot_v *= 10;
+            if (!(va_list)dot_v) {
+                E++;
+            }
+        }
+
+        *num = E;
+
+        return dot_v;
+    } else if (DOT) {
+        for (i = 1; i <= DOT; i++) {
+            if ((DOT_V >> (DOT - i)) & 0x1) {
+                dot_v += 1.0 / (1 << i);
+            }
+        }
+
+        for (i = 1; i <= ACC; i++) {
+            dot_v = dot_v * 10;
+        }
+
+    return dot_v;
+    }
+
+    return 0;
+}
+
+int vsnprintf(char *buf, int size, char *fmt, va_list args)
+{
+    char *str, *mm;
+    struct printf_spec spec = {0};
+
+    str = mm = buf;
+
+    while (*fmt) {
+        char *old_fmt = fmt;
+        int read = format_decode(fmt, &spec);
+
+        fmt += read;
+
+        switch (spec.type) {
+        case FORMAT_TYPE_NONE: {
+            memcpy(str, old_fmt, read);
+            str += read;
+            break;
+        }
+        case FORMAT_TYPE_HEX: {
+            memcpy(str, old_fmt, read);
+            str = number(str + read, args);
+            for (; *mm ; ++mm) {
+                if (*mm == '%') {
+                    *mm = '0';
+                break;
+                }
+            }
+        break;
+        }
+        case FORMAT_TYPE_ULONG: {
+            memcpy(str, old_fmt, read - 2);
+            str = __number(str + read - 2, args);
+            break;
+        }
+        case FORMAT_TYPE_FLOAT: {
+            va_list integer, dot_v, num;
+            dot_v = modf(args, &integer, &num);
+            memcpy(str, old_fmt, read - 2);
+            str += read - 2;
+            if ((args >> 63 & 0x1)) {
+                *str++ = '-';
+            }
+            str = __number(str, integer);
+            if (dot_v) {
+                *str++ = '.';
+                while (num--) {
+                    *str++ = '0';
+                }
+                str = __number(str, dot_v);
+            }
+            break;
+        }
+        }
+    }
+    *str = '\0';
+
+    return str - buf;
+}
+
+static void serial_out(char *str)
+{
+    while (*str) {
+        *(char *)0xffffffffb80003f8 = *str++;
+    }
+}
+
+int vprintf(char *fmt, va_list args)
+{
+    int printed_len = 0;
+    static char printf_buf[512];
+    printed_len = vsnprintf(printf_buf, sizeof(printf_buf), fmt, args);
+    serial_out(printf_buf);
+    return printed_len;
+}
+
+int printf(char *fmt, ...)
+{
+    return vprintf(fmt, __read($5));
+}
diff --git a/tests/tcg/mips/mips64-dsp/raddu_l_ob.c b/tests/tcg/mips/mips64-dsp/raddu_l_ob.c
new file mode 100644
index 000000000..76ddf25fb
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/raddu_l_ob.c
@@ -0,0 +1,22 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, result;
+    rs = 0x12345678ABCDEF0;
+    result = 0x000000000001E258;
+
+    __asm
+        ("raddu.l.ob %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs)
+        );
+
+    if (rd != result) {
+        printf("raddu.l.ob error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/raddu_w_qb.c b/tests/tcg/mips/mips64-dsp/raddu_w_qb.c
new file mode 100644
index 000000000..c9d6535bb
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/raddu_w_qb.c
@@ -0,0 +1,23 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs;
+    long long result;
+
+    rs = 0x12345678;
+    result = 0x114;
+
+    __asm
+        ("raddu.w.qb %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rs)
+        );
+    if (rd != result) {
+        printf("raddu.w.qb wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/rddsp.c b/tests/tcg/mips/mips64-dsp/rddsp.c
new file mode 100644
index 000000000..716557243
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/rddsp.c
@@ -0,0 +1,53 @@
+#include "io.h"
+
+int main(void)
+{
+    long long dsp_i, dsp_o;
+    long long ccond_i, outflag_i, efi_i, c_i, scount_i, pos_i;
+    long long ccond_o, outflag_o, efi_o, c_o, scount_o, pos_o;
+    long long ccond_r, outflag_r, efi_r, c_r, scount_r, pos_r;
+
+    ccond_i   = 0x000000BC;/* 4 */
+    outflag_i = 0x0000001B;/* 3 */
+    efi_i     = 0x00000001;/* 5 */
+    c_i       = 0x00000001;/* 2 */
+    scount_i  = 0x0000000F;/* 1 */
+    pos_i     = 0x0000000C;/* 0 */
+
+    dsp_i = (ccond_i   << 24) | \
+            (outflag_i << 16) | \
+            (efi_i     << 14) | \
+            (c_i       << 13) | \
+            (scount_i  <<  7) | \
+            pos_i;
+
+    ccond_r   = ccond_i;
+    outflag_r = outflag_i;
+    efi_r     = efi_i;
+    c_r       = c_i;
+    scount_r  = scount_i;
+    pos_r     = pos_i;
+
+    __asm
+        ("wrdsp %1, 0x3F\n\t"
+         "rddsp %0, 0x3F\n\t"
+         : "=r"(dsp_o)
+         : "r"(dsp_i)
+        );
+
+    ccond_o   = (dsp_o >> 24) & 0xFF;
+    outflag_o = (dsp_o >> 16) & 0xFF;
+    efi_o     = (dsp_o >> 14) & 0x01;
+    c_o       = (dsp_o >> 14) & 0x01;
+    scount_o  = (dsp_o >>  7) & 0x3F;
+    pos_o     =  dsp_o & 0x1F;
+
+    if ((ccond_o != ccond_r) || (outflag_o != outflag_r) || (efi_o != efi_r) \
+            || (c_o != c_r) || (scount_o != scount_r) || (pos_o != pos_r)) {
+        printf("rddsp wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/repl_ob.c b/tests/tcg/mips/mips64-dsp/repl_ob.c
new file mode 100644
index 000000000..20cb78013
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/repl_ob.c
@@ -0,0 +1,21 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, result;
+    rd = 0;
+    result = 0xFFFFFFFFFFFFFFFF;
+
+    __asm
+        ("repl.ob %0, 0xFF\n\t"
+         : "=r"(rd)
+        );
+
+    if (result != rd) {
+        printf("repl.ob error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/repl_ph.c b/tests/tcg/mips/mips64-dsp/repl_ph.c
new file mode 100644
index 000000000..11d29bdbc
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/repl_ph.c
@@ -0,0 +1,30 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, result;
+
+    result = 0x01BF01BF;
+    __asm
+        ("repl.ph %0, 0x1BF\n\t"
+         : "=r"(rd)
+        );
+    if (rd != result) {
+        printf("repl.ph wrong\n");
+
+        return -1;
+    }
+
+    result = 0x01FF01FF;
+    __asm
+        ("repl.ph %0, 0x01FF\n\t"
+         : "=r"(rd)
+        );
+    if (rd != result) {
+        printf("repl.ph wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/repl_pw.c b/tests/tcg/mips/mips64-dsp/repl_pw.c
new file mode 100644
index 000000000..d35376a2a
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/repl_pw.c
@@ -0,0 +1,34 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, result;
+    rd = 0;
+    result = 0x000001FF000001FF;
+
+    __asm
+        ("repl.pw %0, 0x1FF\n\t"
+         : "=r"(rd)
+        );
+
+    if (result != rd) {
+        printf("repl.pw error1\n");
+
+        return -1;
+    }
+
+    rd = 0;
+    result = 0xFFFFFE00FFFFFE00;
+    __asm
+        ("repl.pw %0, 0xFFFFFFFFFFFFFE00\n\t"
+         : "=r"(rd)
+        );
+
+    if (result != rd) {
+        printf("repl.pw error2\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/repl_qb.c b/tests/tcg/mips/mips64-dsp/repl_qb.c
new file mode 100644
index 000000000..592feaecb
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/repl_qb.c
@@ -0,0 +1,19 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, result;
+
+    result = 0xFFFFFFFFBFBFBFBF;
+    __asm
+        ("repl.qb %0, 0xBF\n\t"
+         : "=r"(rd)
+        );
+    if (rd != result) {
+        printf("repl.qb wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/repl_qh.c b/tests/tcg/mips/mips64-dsp/repl_qh.c
new file mode 100644
index 000000000..82afc3716
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/repl_qh.c
@@ -0,0 +1,34 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, result;
+    rd = 0;
+    result = 0x01FF01FF01FF01FF;
+
+    __asm
+        ("repl.qh %0, 0x1FF\n\t"
+         : "=r"(rd)
+        );
+
+    if (result != rd) {
+        printf("repl.qh error 1\n");
+
+        return -1;
+    }
+
+    rd = 0;
+    result = 0xFE00FE00FE00FE00;
+    __asm
+        ("repl.qh %0, 0xFFFFFFFFFFFFFE00\n\t"
+         : "=r"(rd)
+        );
+
+    if (result != rd) {
+        printf("repl.qh error 2\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/replv_ob.c b/tests/tcg/mips/mips64-dsp/replv_ob.c
new file mode 100644
index 000000000..31ff3186d
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/replv_ob.c
@@ -0,0 +1,23 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt, result;
+
+    rt = 0xFF;
+    result = 0xFFFFFFFFFFFFFFFF;
+
+    __asm
+        ("replv.ob %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+
+    if (result != rd) {
+        printf("replv.ob error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/replv_ph.c b/tests/tcg/mips/mips64-dsp/replv_ph.c
new file mode 100644
index 000000000..0af7a36b4
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/replv_ph.c
@@ -0,0 +1,22 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt;
+    long long result;
+
+    rt = 0x12345678;
+    result = 0x56785678;
+    __asm
+        ("replv.ph %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    if (rd != result) {
+        printf("replv.ph wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/replv_pw.c b/tests/tcg/mips/mips64-dsp/replv_pw.c
new file mode 100644
index 000000000..e1789af4c
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/replv_pw.c
@@ -0,0 +1,23 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt, result;
+    rd = 0;
+    rt = 0xFFFFFFFF;
+    result = 0xFFFFFFFFFFFFFFFF;
+
+    __asm
+        ("replv.pw %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+
+    if (result != rd) {
+        printf("replv.pw error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/replv_qb.c b/tests/tcg/mips/mips64-dsp/replv_qb.c
new file mode 100644
index 000000000..d99298c31
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/replv_qb.c
@@ -0,0 +1,22 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt;
+    long long result;
+
+    rt     = 0x12345678;
+    result = 0x78787878;
+    __asm
+        ("replv.qb %0, %1\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    if (rd != result) {
+        printf("replv.qb wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shilo.c b/tests/tcg/mips/mips64-dsp/shilo.c
new file mode 100644
index 000000000..5f454f69e
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shilo.c
@@ -0,0 +1,29 @@
+#include "io.h"
+
+int main(void)
+{
+    long long ach, acl;
+    long long resulth, resultl;
+
+    ach = 0xBBAACCFF;
+    acl = 0x1C3B001D;
+
+    resulth = 0x17755;
+    resultl = 0xFFFFFFFF99fe3876;
+
+    __asm
+        ("mthi %0, $ac1\n\t"
+         "mtlo %1, $ac1\n\t"
+         "shilo $ac1, 0x0F\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "+r"(ach), "+r"(acl)
+        );
+    if ((ach != resulth) || (acl != resultl)) {
+        printf("shilo wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shilov.c b/tests/tcg/mips/mips64-dsp/shilov.c
new file mode 100644
index 000000000..e82615a8c
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shilov.c
@@ -0,0 +1,31 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rs, ach, acl;
+    long long resulth, resultl;
+
+    rs  = 0x0F;
+    ach = 0xBBAACCFF;
+    acl = 0x1C3B001D;
+
+    resulth = 0x17755;
+    resultl = 0xFFFFFFFF99fe3876;
+
+    __asm
+        ("mthi %0, $ac1\n\t"
+         "mtlo %1, $ac1\n\t"
+         "shilov $ac1, %2\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "+r"(ach), "+r"(acl)
+         : "r"(rs)
+        );
+    if ((ach != resulth) || (acl != resultl)) {
+        printf("shilov wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shll_ob.c b/tests/tcg/mips/mips64-dsp/shll_ob.c
new file mode 100644
index 000000000..7dcb58ff4
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shll_ob.c
@@ -0,0 +1,43 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt, dsp;
+    long long res, resdsp;
+
+    rt = 0x9ba8765433456789;
+    res = 0x9ba8765433456789;
+    resdsp = 0x0;
+    __asm
+        ("shll.ob %0, %2, 0x0\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt)
+        );
+
+    dsp = (dsp >> 22) & 0x1;
+
+    if ((dsp != resdsp) || (rd != res)) {
+        printf("shll.ob error\n");
+        return -1;
+    }
+
+    rt = 0x9ba8765433456789;
+    res = 0xd840b0a098283848;
+    resdsp = 0x1;
+    __asm
+        ("shll.ob %0, %2, 0x3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt)
+        );
+
+    dsp = (dsp >> 22) & 0x1;
+
+    if ((dsp != resdsp) || (rd != res)) {
+        printf("shll.ob error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shll_ph.c b/tests/tcg/mips/mips64-dsp/shll_ph.c
new file mode 100644
index 000000000..42b462d20
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shll_ph.c
@@ -0,0 +1,43 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt, dsp;
+    long long result, resultdsp;
+
+    rt        = 0x12345678;
+    result    = 0x12345678;
+    resultdsp = 0;
+
+    __asm
+        ("shll.ph %0, %2, 0x0\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt)
+        );
+    dsp = (dsp >> 22) & 0x01;
+    if ((dsp != resultdsp) || (rd  != result)) {
+        printf("shll.ph wrong\n");
+
+        return -1;
+    }
+
+    rt        = 0x12345678;
+    result    = 0xFFFFFFFFA000C000;
+    resultdsp = 1;
+
+    __asm
+        ("shll.ph %0, %2, 0x0B\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt)
+        );
+    dsp = (dsp >> 22) & 0x01;
+    if ((dsp != resultdsp) || (rd  != result)) {
+        printf("shll.ph wrong1\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shll_pw.c b/tests/tcg/mips/mips64-dsp/shll_pw.c
new file mode 100644
index 000000000..d7878b279
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shll_pw.c
@@ -0,0 +1,43 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt, dsp;
+    long long result, resultdsp;
+
+    rt        = 0x8765432112345678;
+    result    = 0x8765432112345678;
+    resultdsp = 0;
+
+    __asm
+        ("shll.pw %0, %2, 0x0\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt)
+        );
+
+    dsp = (dsp >> 22) & 0x01;
+    if ((dsp != resultdsp) || (rd  != result)) {
+        printf("shll.pw wrong\n");
+        return -1;
+    }
+
+    rt        = 0x8765432112345678;
+    result    = 0x6543210034567800;
+    resultdsp = 1;
+
+    __asm
+        ("shll.pw %0, %2, 0x8\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt)
+        );
+
+    dsp = (dsp >> 22) & 0x01;
+    if ((dsp != resultdsp) || (rd  != result)) {
+        printf("shll.pw wrong\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shll_qb.c b/tests/tcg/mips/mips64-dsp/shll_qb.c
new file mode 100644
index 000000000..c21ab6698
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shll_qb.c
@@ -0,0 +1,26 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt, dsp;
+    long long result, resultdsp;
+
+    rt     = 0x87654321;
+    result = 0x38281808;
+    resultdsp = 0x01;
+
+    __asm
+        ("shll.qb %0, %2, 0x03\n\t"
+         "rddsp   %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt)
+        );
+    dsp = (dsp >> 22) & 0x01;
+    if (rd != result) {
+        printf("shll.qb wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shll_qh.c b/tests/tcg/mips/mips64-dsp/shll_qh.c
new file mode 100644
index 000000000..1380825a3
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shll_qh.c
@@ -0,0 +1,42 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt, dsp;
+    long long res, resdsp;
+
+    rt = 0x9ba8765433456789;
+    res = 0x9ba8765433456789;
+    resdsp = 0x0;
+    __asm
+        ("shll.qh %0, %2, 0x0\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt)
+        );
+    dsp = (dsp >> 22) & 0x1;
+
+    if ((dsp != resdsp) || (rd != res)) {
+        printf("shll.qh error\n");
+        return -1;
+    }
+
+    rt = 0x9ba8765433456789;
+    res = 0xdd40b2a09a283c48;
+    resdsp = 0x1;
+    __asm
+        ("shll.qh %0, %2, 0x3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt)
+        );
+
+    dsp = (dsp >> 22) & 0x1;
+
+    if ((dsp != resdsp) || (rd != res)) {
+        printf("shll.qh error1\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shll_s_ph.c b/tests/tcg/mips/mips64-dsp/shll_s_ph.c
new file mode 100644
index 000000000..1cf5d6da6
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shll_s_ph.c
@@ -0,0 +1,43 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt, dsp;
+    long long result, resultdsp;
+
+    rt        = 0x12345678;
+    result    = 0x12345678;
+    resultdsp = 0x0;
+
+    __asm
+        ("shll_s.ph %0, %2, 0x0\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt)
+        );
+    dsp = (dsp >> 22) & 0x01;
+    if ((dsp != resultdsp) || (rd  != result)) {
+        printf("shll_s.ph wrong\n");
+
+        return -1;
+    }
+
+    rt        = 0x12345678;
+    result    = 0x7FFF7FFF;
+    resultdsp = 0x01;
+
+    __asm
+        ("shll_s.ph %0, %2, 0x0B\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt)
+        );
+    dsp = (dsp >> 22) & 0x01;
+    if ((dsp != resultdsp) || (rd  != result)) {
+        printf("shll_s.ph wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shll_s_pw.c b/tests/tcg/mips/mips64-dsp/shll_s_pw.c
new file mode 100644
index 000000000..e38f6860c
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shll_s_pw.c
@@ -0,0 +1,43 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt, dsp;
+    long long result, resultdsp;
+
+    rt        = 0x8765432112345678;
+    result    = 0x8765432112345678;
+    resultdsp = 0;
+
+    __asm
+        ("shll_s.pw %0, %2, 0x0\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt)
+        );
+
+    dsp = (dsp >> 22) & 0x01;
+    if ((dsp != resultdsp) || (rd  != result)) {
+        printf("shll_s.pw wrong\n");
+        return -1;
+    }
+
+    rt        = 0x8765432112345678;
+    result    = 0x800000007fffffff;
+    resultdsp = 1;
+
+    __asm
+        ("shll_s.pw %0, %2, 0x8\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt)
+        );
+
+    dsp = (dsp >> 22) & 0x01;
+    if ((dsp != resultdsp) || (rd  != result)) {
+        printf("shll_s.pw wrong\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shll_s_qh.c b/tests/tcg/mips/mips64-dsp/shll_s_qh.c
new file mode 100644
index 000000000..f2f57fa27
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shll_s_qh.c
@@ -0,0 +1,43 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt, dsp;
+    long long res, resdsp;
+
+    rt = 0x9ba8765433456789;
+    res = 0x9ba8765433456789;
+    resdsp = 0x0;
+    __asm
+        ("shll_s.qh %0, %2, 0x0\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt)
+        );
+
+    dsp = (dsp >> 22) & 0x1;
+
+    if ((dsp != resdsp) || (rd != res)) {
+        printf("shll_s.qh error\n");
+        return -1;
+    }
+
+    rt = 0x9ba8765433456789;
+    res = 0x80007fff7fff7fff;
+    resdsp = 0x1;
+    __asm
+        ("shll_s.qh %0, %2, 0x3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt)
+        );
+
+    dsp = (dsp >> 22) & 0x1;
+
+    if ((dsp != resdsp) || (rd != res)) {
+        printf("shll_s.qh error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shll_s_w.c b/tests/tcg/mips/mips64-dsp/shll_s_w.c
new file mode 100644
index 000000000..57800615d
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shll_s_w.c
@@ -0,0 +1,26 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt, dsp;
+    long long result, resultdsp;
+
+    rt        = 0x12345678;
+    result    = 0x7FFFFFFF;
+    resultdsp = 0x01;
+
+    __asm
+        ("shll_s.w %0, %2, 0x0B\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt)
+        );
+    dsp = (dsp >> 22) & 0x01;
+    if ((dsp != resultdsp) || (rd  != result)) {
+        printf("shll_s.w wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shllv_ob.c b/tests/tcg/mips/mips64-dsp/shllv_ob.c
new file mode 100644
index 000000000..96a2e6f55
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shllv_ob.c
@@ -0,0 +1,45 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt, rs, dsp;
+    long long result, resultdsp;
+
+    rt        = 0x8765432112345678;
+    rs = 0x0;
+    result    = 0x8765432112345678;
+    resultdsp = 0;
+
+    __asm
+        ("shllv.ob %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt), "r"(rs)
+        );
+
+    dsp = (dsp >> 22) & 0x01;
+    if ((dsp != resultdsp) || (rd  != result)) {
+        printf("shllv.ob wrong\n");
+        return -1;
+    }
+
+    rt        = 0x8765432112345678;
+    rs = 0x4;
+    result    = 0x7050301020406080;
+    resultdsp = 1;
+
+    __asm
+        ("shllv.ob %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt), "r"(rs)
+        );
+
+    dsp = (dsp >> 22) & 0x01;
+    if ((dsp != resultdsp) || (rd  != result)) {
+        printf("shllv.ob wrong\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shllv_ph.c b/tests/tcg/mips/mips64-dsp/shllv_ph.c
new file mode 100644
index 000000000..532291f3f
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shllv_ph.c
@@ -0,0 +1,27 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, dsp;
+    long long result, resultdsp;
+
+    rs        = 0x0B;
+    rt        = 0x12345678;
+    result    = 0xFFFFFFFFA000C000;
+    resultdsp = 1;
+
+    __asm
+        ("shllv.ph %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt), "r"(rs)
+        );
+    dsp = (dsp >> 22) & 0x01;
+    if ((dsp != resultdsp) || (rd  != result)) {
+        printf("shllv.ph wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shllv_pw.c b/tests/tcg/mips/mips64-dsp/shllv_pw.c
new file mode 100644
index 000000000..8d4ec295b
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shllv_pw.c
@@ -0,0 +1,45 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt, rs, dsp;
+    long long result, resultdsp;
+    rt        = 0x8765432112345678;
+    rs = 0x0;
+    result    = 0x8765432112345678;
+    resultdsp = 0;
+
+    __asm
+        ("shllv.pw %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt), "r"(rs)
+        );
+
+    dsp = (dsp >> 22) & 0x01;
+    if ((dsp != resultdsp) || (rd  != result)) {
+        printf("shllv.pw wrong\n");
+        return -1;
+    }
+
+
+    rt        = 0x8765432112345678;
+    rs = 0x8;
+    result    = 0x6543210034567800;
+    resultdsp = 1;
+
+    __asm
+        ("shllv.pw %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt), "r"(rs)
+        );
+
+    dsp = (dsp >> 22) & 0x01;
+    if ((dsp != resultdsp) || (rd  != result)) {
+        printf("shllv.pw wrong\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shllv_qb.c b/tests/tcg/mips/mips64-dsp/shllv_qb.c
new file mode 100644
index 000000000..e49356b8e
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shllv_qb.c
@@ -0,0 +1,27 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, dsp;
+    long long result, resultdsp;
+
+    rs     = 0x03;
+    rt     = 0x87654321;
+    result = 0x38281808;
+    resultdsp = 0x01;
+
+    __asm
+        ("shllv.qb %0, %2, %3\n\t"
+         "rddsp   %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt), "r"(rs)
+        );
+    dsp = (dsp >> 22) & 0x01;
+    if (rd != result) {
+        printf("shllv.qb wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shllv_qh.c b/tests/tcg/mips/mips64-dsp/shllv_qh.c
new file mode 100644
index 000000000..0de4077e7
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shllv_qh.c
@@ -0,0 +1,45 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt, rs, dsp;
+    long long result, resultdsp;
+
+    rt        = 0x8765432112345678;
+    rs = 0x0;
+    result    = 0x8765432112345678;
+    resultdsp = 0;
+
+    __asm
+        ("shllv.qh %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt), "r"(rs)
+        );
+
+    dsp = (dsp >> 22) & 0x01;
+    if ((dsp != resultdsp) || (rd  != result)) {
+        printf("shllv.qh wrong\n");
+        return -1;
+    }
+
+    rt        = 0x8765432112345678;
+    rs = 0x4;
+    result    = 0x7650321023406780;
+    resultdsp = 1;
+
+    __asm
+        ("shllv.qh %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt), "r"(rs)
+        );
+
+    dsp = (dsp >> 22) & 0x01;
+    if ((dsp != resultdsp) || (rd  != result)) {
+        printf("shllv.qh wrong\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shllv_s_ph.c b/tests/tcg/mips/mips64-dsp/shllv_s_ph.c
new file mode 100644
index 000000000..7e69f941f
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shllv_s_ph.c
@@ -0,0 +1,27 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, dsp;
+    long long result, resultdsp;
+
+    rs        = 0x0B;
+    rt        = 0x12345678;
+    result    = 0x7FFF7FFF;
+    resultdsp = 0x01;
+
+    __asm
+        ("shllv_s.ph %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt), "r"(rs)
+        );
+    dsp = (dsp >> 22) & 0x01;
+    if ((dsp != resultdsp) || (rd  != result)) {
+        printf("shllv_s.ph wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shllv_s_pw.c b/tests/tcg/mips/mips64-dsp/shllv_s_pw.c
new file mode 100644
index 000000000..f8dc8d296
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shllv_s_pw.c
@@ -0,0 +1,45 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt, rs, dsp;
+    long long result, resultdsp;
+
+    rt        = 0x8765432112345678;
+    rs = 0x0;
+    result    = 0x8765432112345678;
+    resultdsp = 0;
+
+    __asm
+        ("shllv_s.pw %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt), "r"(rs)
+        );
+
+    dsp = (dsp >> 22) & 0x01;
+    if ((dsp != resultdsp) || (rd  != result)) {
+        printf("shllv_s.pw wrong\n");
+        return -1;
+    }
+
+    rt        = 0x8765432112345678;
+    rs = 0x8;
+    result    = 0x800000007fffffff;
+    resultdsp = 1;
+
+    __asm
+        ("shllv_s.pw %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt), "r"(rs)
+        );
+
+    dsp = (dsp >> 22) & 0x01;
+    if ((dsp != resultdsp) || (rd  != result)) {
+        printf("shllv_s.pw wrong\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shllv_s_qh.c b/tests/tcg/mips/mips64-dsp/shllv_s_qh.c
new file mode 100644
index 000000000..db3832d09
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shllv_s_qh.c
@@ -0,0 +1,45 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt, rs, dsp;
+    long long result, resultdsp;
+
+    rt        = 0x8765432112345678;
+    rs = 0x0;
+    result    = 0x8765432112345678;
+    resultdsp = 0;
+
+    __asm
+        ("shllv_s.qh %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt), "r"(rs)
+        );
+
+    dsp = (dsp >> 22) & 0x01;
+    if ((dsp != resultdsp) || (rd  != result)) {
+        printf("shllv_s.qh wrong\n");
+        return -1;
+    }
+
+    rt        = 0x8765432112345678;
+    rs = 0x4;
+    result    = 0x80007fff7fff7fff;
+    resultdsp = 1;
+
+    __asm
+        ("shllv_s.qh %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt), "r"(rs)
+        );
+
+    dsp = (dsp >> 22) & 0x01;
+    if ((dsp != resultdsp) || (rd  != result)) {
+        printf("shllv_s.qh wrong\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shllv_s_w.c b/tests/tcg/mips/mips64-dsp/shllv_s_w.c
new file mode 100644
index 000000000..5f6af8b8c
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shllv_s_w.c
@@ -0,0 +1,27 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, dsp;
+    long long result, resultdsp;
+
+    rs        = 0x0B;
+    rt        = 0x12345678;
+    result    = 0x7FFFFFFF;
+    resultdsp = 0x01;
+
+    __asm
+        ("shllv_s.w %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rt), "r"(rs)
+        );
+    dsp = (dsp >> 22) & 0x01;
+    if ((dsp != resultdsp) || (rd  != result)) {
+        printf("shllv_s.w wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shra_ob.c b/tests/tcg/mips/mips64-dsp/shra_ob.c
new file mode 100644
index 000000000..d7fcfa816
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shra_ob.c
@@ -0,0 +1,23 @@
+#include "io.h"
+
+int main()
+{
+    long long rd, rt;
+    long long res;
+
+    rt = 0xbc98756abc654389;
+    res = 0xfbf9f7f6fb0604f8;
+
+    __asm
+        ("shra.ob %0, %1, 0x4\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+
+    if (rd != res) {
+        printf("shra.ob error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shra_ph.c b/tests/tcg/mips/mips64-dsp/shra_ph.c
new file mode 100644
index 000000000..a2dc01474
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shra_ph.c
@@ -0,0 +1,23 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt;
+    long long result;
+
+    rt     = 0x87654321;
+    result = 0xFFFFFFFFF0EC0864;
+
+    __asm
+        ("shra.ph %0, %1, 0x03\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    if (rd != result) {
+        printf("shra.ph wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shra_pw.c b/tests/tcg/mips/mips64-dsp/shra_pw.c
new file mode 100644
index 000000000..33b1b8fe7
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shra_pw.c
@@ -0,0 +1,36 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt;
+    long long res;
+
+    rt = 0x1234567887654321;
+    res = 0x01234567f8765432;
+
+    __asm
+        ("shra.pw %0, %1, 0x4"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+
+    if (rd != res) {
+        printf("shra.pw error\n");
+        return -1;
+    }
+
+    rt = 0x1234567887654321;
+    res = 0x1234567887654321;
+
+    __asm
+        ("shra.pw %0, %1, 0x0"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+
+    if (rd != res) {
+        printf("shra.pw error\n");
+        return -1;
+    }
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shra_qh.c b/tests/tcg/mips/mips64-dsp/shra_qh.c
new file mode 100644
index 000000000..85dbfef3e
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shra_qh.c
@@ -0,0 +1,37 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt;
+    long long res;
+
+    rt = 0x8512345654323454;
+    res = 0xf851034505430345;
+
+    __asm
+        ("shra.qh %0, %1, 0x4\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+
+    if (rd != res) {
+        printf("shra.qh error\n");
+        return -1;
+    }
+
+    rt = 0x8512345654323454;
+    res = 0x8512345654323454;
+
+    __asm
+        ("shra.qh %0, %1, 0x0\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+
+    if (rd != res) {
+        printf("shra.qh error1\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shra_r_ob.c b/tests/tcg/mips/mips64-dsp/shra_r_ob.c
new file mode 100644
index 000000000..184709443
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shra_r_ob.c
@@ -0,0 +1,22 @@
+#include "io.h"
+
+int main()
+{
+    long long rd, rt;
+    long long res;
+
+    rt = 0xbc98756abc654389;
+    res = 0xfcfaf8f7fc0705f9;
+
+    __asm
+        ("shra_r.ob %0, %1, 0x4\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+
+    if (rd != res) {
+        printf("shra_r.ob error\n");
+        return -1;
+    }
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shra_r_ph.c b/tests/tcg/mips/mips64-dsp/shra_r_ph.c
new file mode 100644
index 000000000..e0943ad47
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shra_r_ph.c
@@ -0,0 +1,23 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt;
+    long long result;
+
+    rt     = 0x87654321;
+    result = 0xFFFFFFFFF0ED0864;
+
+    __asm
+        ("shra_r.ph %0, %1, 0x03\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    if (rd != result) {
+        printf("shra_r.ph wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shra_r_pw.c b/tests/tcg/mips/mips64-dsp/shra_r_pw.c
new file mode 100644
index 000000000..6a86e684b
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shra_r_pw.c
@@ -0,0 +1,36 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt;
+    long long res;
+
+    rt = 0x1234567887654321;
+    res = 0x01234568f8765432;
+
+    __asm
+        ("shra_r.pw %0, %1, 0x4"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+
+    if (rd != res) {
+        printf("shra_r.pw error\n");
+        return -1;
+    }
+
+    rt = 0x1234567887654321;
+    res = 0x1234567887654321;
+
+    __asm
+        ("shra_r.pw %0, %1, 0x0"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+
+    if (rd != res) {
+        printf("shra_r.pw error\n");
+        return -1;
+    }
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shra_r_qh.c b/tests/tcg/mips/mips64-dsp/shra_r_qh.c
new file mode 100644
index 000000000..d5c2110ef
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shra_r_qh.c
@@ -0,0 +1,37 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt;
+    long long res;
+
+    rt = 0x8512345654323454;
+    res = 0xf0a2068b0a86068b;
+
+    __asm
+        ("shra_r.qh %0, %1, 0x3\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+
+    if (rd != res) {
+        printf("shra_r.qh error\n");
+        return -1;
+    }
+
+    rt = 0x8512345654323454;
+    res = 0x8512345654323454;
+
+    __asm
+        ("shra_r.qh %0, %1, 0x0\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+
+    if (rd != res) {
+        printf("shra_r.qh error1\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shra_r_w.c b/tests/tcg/mips/mips64-dsp/shra_r_w.c
new file mode 100644
index 000000000..36d2c9c88
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shra_r_w.c
@@ -0,0 +1,23 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt;
+    long long result;
+
+    rt     = 0x87654321;
+    result = 0xFFFFFFFFF0ECA864;
+
+    __asm
+        ("shra_r.w %0, %1, 0x03\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    if (rd != result) {
+        printf("shra_r.w wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shrav_ph.c b/tests/tcg/mips/mips64-dsp/shrav_ph.c
new file mode 100644
index 000000000..1b4e98374
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shrav_ph.c
@@ -0,0 +1,24 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long result;
+
+    rs     = 0x03;
+    rt     = 0x87654321;
+    result = 0xFFFFFFFFF0EC0864;
+
+    __asm
+        ("shrav.ph %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rt), "r"(rs)
+        );
+    if (rd != result) {
+        printf("shrav.ph wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shrav_pw.c b/tests/tcg/mips/mips64-dsp/shrav_pw.c
new file mode 100644
index 000000000..e19d51579
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shrav_pw.c
@@ -0,0 +1,38 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt, rs;
+    long long res;
+
+    rt = 0x1234567887654321;
+    rs = 0x4;
+    res = 0x01234567f8765432;
+
+    __asm
+        ("shrav.pw %0, %1, %2"
+         : "=r"(rd)
+         : "r"(rt), "r"(rs)
+        );
+
+    if (rd != res) {
+        printf("shrav.pw error\n");
+        return -1;
+    }
+
+    rt = 0x1234567887654321;
+    rs = 0x0;
+    res = 0x1234567887654321;
+
+    __asm
+        ("shrav.pw %0, %1, %2"
+         : "=r"(rd)
+         : "r"(rt), "r"(rs)
+        );
+
+    if (rd != res) {
+        printf("shrav.pw error1\n");
+        return -1;
+    }
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shrav_qh.c b/tests/tcg/mips/mips64-dsp/shrav_qh.c
new file mode 100644
index 000000000..dc92e09d4
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shrav_qh.c
@@ -0,0 +1,39 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt, rs;
+    long long res;
+
+    rt = 0x8512345654323454;
+    rs = 0x4;
+    res = 0xf851034505430345;
+
+    __asm
+        ("shrav.qh %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rt), "r"(rs)
+        );
+
+    if (rd != res) {
+        printf("shrav.qh error\n");
+        return -1;
+    }
+
+    rt = 0x8512345654323454;
+    rs = 0x0;
+    res = 0x8512345654323454;
+
+    __asm
+        ("shrav.qh %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rt), "r"(rs)
+        );
+
+    if (rd != res) {
+        printf("shrav.qh error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shrav_r_ph.c b/tests/tcg/mips/mips64-dsp/shrav_r_ph.c
new file mode 100644
index 000000000..350d5294f
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shrav_r_ph.c
@@ -0,0 +1,24 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long result;
+
+    rs     = 0x03;
+    rt     = 0x87654321;
+    result = 0xFFFFFFFFF0ED0864;
+
+    __asm
+        ("shrav_r.ph %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rt), "r"(rs)
+        );
+    if (rd != result) {
+        printf("shrav_r.ph wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shrav_r_pw.c b/tests/tcg/mips/mips64-dsp/shrav_r_pw.c
new file mode 100644
index 000000000..25b054593
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shrav_r_pw.c
@@ -0,0 +1,37 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt, rs;
+    long long res;
+
+    rt = 0x1234567887654321;
+    rs = 0x4;
+    res = 0x01234568f8765432;
+
+    __asm
+        ("shrav_r.pw %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rt), "r"(rs)
+        );
+
+    if (rd != res) {
+        printf("shrav_r.pw error\n");
+        return -1;
+    }
+
+    rt = 0x1234567887654321;
+    rs = 0x0;
+    res = 0x1234567887654321;
+
+    __asm
+        ("shrav_r.pw %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rt), "r"(rs)
+        );
+    if (rd != res) {
+        printf("shrav_r.pw error\n");
+        return -1;
+    }
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shrav_r_qh.c b/tests/tcg/mips/mips64-dsp/shrav_r_qh.c
new file mode 100644
index 000000000..fd187a1e0
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shrav_r_qh.c
@@ -0,0 +1,39 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt, rs;
+    long long res;
+
+    rt = 0x8512345654323454;
+    rs = 0x3;
+    res = 0xf0a2068b0a86068b;
+
+    __asm
+        ("shrav_r.qh %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rt), "r"(rs)
+        );
+
+    if (rd != res) {
+        printf("shrav_r.qh error\n");
+        return -1;
+    }
+
+    rt = 0x400000000000000;
+    rs = 0x0;
+    res = 0x400000000000000;
+
+    __asm
+        ("shrav_r.qh %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rt), "r"(rs)
+        );
+
+    if (rd != res) {
+        printf("shrav_r.qh error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shrav_r_w.c b/tests/tcg/mips/mips64-dsp/shrav_r_w.c
new file mode 100644
index 000000000..3766c7255
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shrav_r_w.c
@@ -0,0 +1,24 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long result;
+
+    rs     = 0x03;
+    rt     = 0x87654321;
+    result = 0xFFFFFFFFF0ECA864;
+
+    __asm
+        ("shrav_r.w %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rt), "r"(rs)
+        );
+    if (rd != result) {
+        printf("shrav_r.w wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shrl_ob.c b/tests/tcg/mips/mips64-dsp/shrl_ob.c
new file mode 100644
index 000000000..a1145713a
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shrl_ob.c
@@ -0,0 +1,38 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt;
+    long long res;
+
+    rt = 0xab76543212345678;
+    res = 0x150e0a0602060a0f;
+
+    __asm
+        ("shrl.ob %0, %1, 0x3\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+
+    if (rd != res) {
+        printf("shrl.ob error\n");
+        return -1;
+    }
+
+    rt = 0xab76543212345678;
+    res = 0xab76543212345678;
+
+    __asm
+        ("shrl.ob %0, %1, 0x0\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+
+    if (rd != res) {
+        printf("shrl.ob error\n");
+        return -1;
+    }
+
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shrl_qb.c b/tests/tcg/mips/mips64-dsp/shrl_qb.c
new file mode 100644
index 000000000..c0e36dba1
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shrl_qb.c
@@ -0,0 +1,23 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt;
+    long long result;
+
+    rt     = 0x12345678;
+    result = 0x00010203;
+
+    __asm
+        ("shrl.qb %0, %1, 0x05\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    if (rd != result) {
+        printf("shrl.qb wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shrl_qh.c b/tests/tcg/mips/mips64-dsp/shrl_qh.c
new file mode 100644
index 000000000..c1562463e
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shrl_qh.c
@@ -0,0 +1,22 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt;
+    long long res;
+
+    rt = 0x8765679abc543786;
+    res = 0x087606790bc50378;
+
+    __asm
+        ("shrl.qh %0, %1, 0x4\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+
+    if (rd != res) {
+        printf("shrl.qh error\n");
+        return -1;
+    }
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shrlv_ob.c b/tests/tcg/mips/mips64-dsp/shrlv_ob.c
new file mode 100644
index 000000000..cb39c4671
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shrlv_ob.c
@@ -0,0 +1,39 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt, rs;
+    long long res;
+
+    rt = 0xab76543212345678;
+    rs = 0x3;
+    res = 0x150e0a0602060a0f;
+
+    __asm
+        ("shrlv.ob %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rt), "r"(rs)
+        );
+
+    if (rd != res) {
+        printf("shrlv.ob error\n");
+        return -1;
+    }
+
+    rt = 0xab76543212345678;
+    rs = 0x0;
+    res = 0xab76543212345678;
+
+    __asm
+        ("shrlv.ob %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rt), "r"(rs)
+        );
+
+    if (rd != res) {
+        printf("shrlv.ob error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shrlv_qb.c b/tests/tcg/mips/mips64-dsp/shrlv_qb.c
new file mode 100644
index 000000000..5616aa9c5
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shrlv_qb.c
@@ -0,0 +1,24 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long result;
+
+    rs     = 0x05;
+    rt     = 0x12345678;
+    result = 0x00010203;
+
+    __asm
+        ("shrlv.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rt), "r"(rs)
+        );
+    if (rd != result) {
+        printf("shrlv.qb wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/shrlv_qh.c b/tests/tcg/mips/mips64-dsp/shrlv_qh.c
new file mode 100644
index 000000000..05de2fd6c
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/shrlv_qh.c
@@ -0,0 +1,23 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt, rs;
+    long long res;
+
+    rt = 0x8765679abc543786;
+    rs = 0x4;
+    res = 0x087606790bc50378;
+
+    __asm
+        ("shrlv.qh %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rt), "r"(rs)
+        );
+
+    if (rd != res) {
+        printf("shrlv.qh error\n");
+        return -1;
+    }
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/subq_ph.c b/tests/tcg/mips/mips64-dsp/subq_ph.c
new file mode 100644
index 000000000..6a1b18610
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/subq_ph.c
@@ -0,0 +1,27 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, dsp;
+    long long result, resultdsp;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result    = 0xFFFFFFFF8ACF1357;
+    resultdsp = 0x01;
+
+    __asm
+        ("subq.ph %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 20) & 0x01;
+    if ((dsp != resultdsp) || (rd  != result)) {
+        printf("subq.ph wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/subq_pw.c b/tests/tcg/mips/mips64-dsp/subq_pw.c
new file mode 100644
index 000000000..32f96ba4b
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/subq_pw.c
@@ -0,0 +1,44 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, result, dspreg, dspresult;
+    rt = 0x123456789ABCDEF0;
+    rs = 0x123456789ABCDEF0;
+    result = 0x0;
+    dspresult = 0x0;
+
+    __asm
+        ("subq.pw %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dspreg)
+         : "r"(rs), "r"(rt)
+         );
+    dspreg = (dspreg >> 20) & 0x1;
+    if ((rd != result) || (dspreg != dspresult)) {
+        printf("subq.pw error1\n\t");
+
+        return -1;
+    }
+
+    rt = 0x123456789ABCDEF1;
+    rs = 0x123456789ABCDEF2;
+    result =  0x0000000000000001;
+    dspresult = 0x0;
+
+    __asm
+        ("subq.pw %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dspreg)
+         : "r"(rs), "r"(rt)
+        );
+    dspreg = (dspreg >> 20) & 0x1;
+    if ((rd != result) || (dspreg != dspresult)) {
+        printf("subq.pw error2\n");
+
+        return -1;
+    }
+
+    return 0;
+}
+
diff --git a/tests/tcg/mips/mips64-dsp/subq_qh.c b/tests/tcg/mips/mips64-dsp/subq_qh.c
new file mode 100644
index 000000000..76d5f0a10
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/subq_qh.c
@@ -0,0 +1,26 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, result, dspreg, dspresult;
+    rt = 0x123456789ABCDEF0;
+    rs = 0x123456789ABCDEF0;
+    result = 0x0;
+    dspresult = 0x0;
+
+    __asm
+        ("subq.qh %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dspreg)
+         : "r"(rs), "r"(rt)
+        );
+    dspreg = (dspreg >> 20) & 0x1;
+    if ((rd != result) || (dspreg != dspresult)) {
+        printf("subq.qh error\n\t");
+
+        return -1;
+    }
+
+    return 0;
+}
+
diff --git a/tests/tcg/mips/mips64-dsp/subq_s_ph.c b/tests/tcg/mips/mips64-dsp/subq_s_ph.c
new file mode 100644
index 000000000..0b162f07e
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/subq_s_ph.c
@@ -0,0 +1,27 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, dsp;
+    long long result, resultdsp;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result    = 0x7FFF1357;
+    resultdsp = 0x01;
+
+    __asm
+        ("subq_s.ph %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 20) & 0x01;
+    if ((dsp != resultdsp) || (rd  != result)) {
+        printf("subq_s.ph wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/subq_s_pw.c b/tests/tcg/mips/mips64-dsp/subq_s_pw.c
new file mode 100644
index 000000000..e8e0b0567
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/subq_s_pw.c
@@ -0,0 +1,63 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, result, dspreg, dspresult;
+    rt = 0x9FFFFFFD9FFFFFFD;
+    rs = 0x4000000080000000;
+    result = 0x7fffffffe0000003;
+    dspresult = 0x1;
+
+    __asm
+        ("subq_s.pw %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dspreg)
+         : "r"(rs), "r"(rt)
+        );
+    dspreg = (dspreg >> 20) & 0x1;
+    if ((rd != result) || (dspreg != dspresult)) {
+        printf("subq_s.pw error1\n");
+
+        return -1;
+    }
+
+    rt = 0x123456789ABCDEF1;
+    rs = 0x123456789ABCDEF2;
+    result =  0x0000000000000001;
+    /* This time we do not set dspctrl, but it setted in pre-action. */
+    dspresult = 0x1;
+
+    __asm
+        ("subq_s.pw %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dspreg)
+         : "r"(rs), "r"(rt)
+        );
+    dspreg = (dspreg >> 20) & 0x1;
+    if ((rd != result) || (dspreg != dspresult)) {
+        printf("subq_s.pw error2\n");
+
+        return -1;
+    }
+
+    rt = 0x8000000080000000;
+    rs = 0x7000000070000000;
+    dspresult = 0x1;
+
+    __asm
+        ("subq_s.pw %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dspreg)
+         : "r"(rs), "r"(rt)
+        );
+
+    dspreg = (dspreg >> 20) & 0x1;
+    if ((dspreg != dspresult)) {
+        printf("subq_s.pw error3\n");
+
+        return -1;
+    }
+
+    return 0;
+}
+
diff --git a/tests/tcg/mips/mips64-dsp/subq_s_qh.c b/tests/tcg/mips/mips64-dsp/subq_s_qh.c
new file mode 100644
index 000000000..4053b6b88
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/subq_s_qh.c
@@ -0,0 +1,61 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, result, dspreg, dspresult;
+    rs = 0x123456789ABCDEF0;
+    rt = 0x123456789ABCDEF0;
+    result = 0x0;
+    dspresult = 0x0;
+
+    __asm
+        ("subq_s.qh %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dspreg)
+         : "r"(rs), "r"(rt)
+        );
+    dspreg = (dspreg >> 20) & 0x1;
+    if ((rd != result) || (dspreg != dspresult)) {
+        printf("subq_s.qh error1\n");
+
+        return -1;
+    }
+
+    rs = 0x4000000080000000;
+    rt = 0x9FFD00009FFC0000;
+    result =  0x7FFF0000E0040000;
+    dspresult = 0x1;
+
+    __asm
+        ("subq_s.qh %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dspreg)
+         : "r"(rs), "r"(rt)
+        );
+    dspreg = (dspreg >> 20) & 0x1;
+    if ((rd != result) || (dspreg != dspresult)) {
+        printf("subq_s.qh error2\n");
+
+        return -1;
+    }
+
+    rs = 0x8000000000000000;
+    rt = 0x7000000000000000;
+    result =  0x8000000000000000;
+    dspresult = 0x1;
+    __asm
+        ("subq_s.qh %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dspreg)
+         : "r"(rs), "r"(rt)
+        );
+
+    dspreg = (dspreg >> 20) & 0x1;
+    if ((rd != result) || (dspreg != dspresult)) {
+        printf("subq_s.qh error3\n");
+        return -1;
+    }
+
+    return 0;
+}
+
diff --git a/tests/tcg/mips/mips64-dsp/subq_s_w.c b/tests/tcg/mips/mips64-dsp/subq_s_w.c
new file mode 100644
index 000000000..91d32da17
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/subq_s_w.c
@@ -0,0 +1,27 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, dsp;
+    long long result, resultdsp;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result    = 0x7FFFFFFF;
+    resultdsp = 0x01;
+
+    __asm
+        ("subq_s.w %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 20) & 0x01;
+    if ((dsp != resultdsp) || (rd  != result)) {
+        printf("subq_s.w wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/subu_ob.c b/tests/tcg/mips/mips64-dsp/subu_ob.c
new file mode 100644
index 000000000..f67096711
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/subu_ob.c
@@ -0,0 +1,26 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, result, dspreg, dspresult;
+    rs = 0x6F6F6F6F6F6F6F6F;
+    rt = 0x5E5E5E5E5E5E5E5E;
+    result = 0x1111111111111111;
+    dspresult = 0x0;
+
+    __asm
+        ("subu.ob %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dspreg)
+         : "r"(rs), "r"(rt)
+         );
+
+    if ((rd != result) || (dspreg != dspresult)) {
+        printf("subu.ob error\n");
+
+        return -1;
+    }
+
+    return 0;
+}
+
diff --git a/tests/tcg/mips/mips64-dsp/subu_qb.c b/tests/tcg/mips/mips64-dsp/subu_qb.c
new file mode 100644
index 000000000..9eb80df37
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/subu_qb.c
@@ -0,0 +1,27 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, dsp;
+    long long result, resultdsp;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result    = 0xFFFFFFFF8BCF1357;
+    resultdsp = 0x01;
+
+    __asm
+        ("subu.qb %0, %2, %3\n\t"
+         "rddsp   %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 20) & 0x01;
+    if ((dsp != resultdsp) || (rd  != result)) {
+        printf("subu.qb wrong\n");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/subu_s_ob.c b/tests/tcg/mips/mips64-dsp/subu_s_ob.c
new file mode 100644
index 000000000..5df64e5ff
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/subu_s_ob.c
@@ -0,0 +1,26 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, dspreg, result, dspresult;
+    rs = 0x12345678ABCDEF0;
+    rt = 0x12345678ABCDEF1;
+    result = 0x00000000000;
+    dspresult = 0x01;
+
+    __asm
+        ("subu_s.ob %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dspreg)
+         : "r"(rs), "r"(rt)
+         );
+
+    dspreg = ((dspreg >> 20) & 0x01);
+    if ((rd != result) || (dspreg != dspresult)) {
+        printf("subu_s.ob error\n\t");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/subu_s_qb.c b/tests/tcg/mips/mips64-dsp/subu_s_qb.c
new file mode 100644
index 000000000..9de76f4a1
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/subu_s_qb.c
@@ -0,0 +1,27 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, dsp;
+    long long result, resultdsp;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result    = 0x00001357;
+    resultdsp = 0x01;
+
+    __asm
+        ("subu_s.qb %0, %2, %3\n\t"
+         "rddsp   %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 20) & 0x01;
+    if ((dsp != resultdsp) || (rd  != result)) {
+        printf("subu_s_qb wrong");
+
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dsp/wrdsp.c b/tests/tcg/mips/mips64-dsp/wrdsp.c
new file mode 100644
index 000000000..3033fd88d
--- /dev/null
+++ b/tests/tcg/mips/mips64-dsp/wrdsp.c
@@ -0,0 +1,48 @@
+#include "io.h"
+
+int main(void)
+{
+    long long dsp_i, dsp_o;
+    long long ccond_i, outflag_i, efi_i, c_i, scount_i, pos_i;
+    long long ccond_o, outflag_o, efi_o, c_o, scount_o, pos_o;
+    long long ccond_r, outflag_r, efi_r, c_r, scount_r, pos_r;
+
+    ccond_i = 0x000000BC;/* 4 */
+    outflag_i = 0x0000001B;/* 3 */
+    efi_i = 0x00000001;/* 5 */
+    c_i = 0x00000001;/* 2 */
+    scount_i = 0x0000000F;/* 1 */
+    pos_i = 0x0000000C;/* 0 */
+
+    dsp_i = (ccond_i << 24) | (outflag_i << 16) | (efi_i << 14) | (c_i << 13)
+            | (scount_i << 7) | pos_i;
+
+    ccond_r = ccond_i;
+    outflag_r = outflag_i;
+    efi_r = efi_i;
+    c_r = c_i;
+    scount_r = scount_i;
+    pos_r = pos_i;
+
+    __asm
+        ("wrdsp %1, 0x3F\n\t"
+         "rddsp %0, 0x3F\n\t"
+         : "=r"(dsp_o)
+         : "r"(dsp_i)
+        );
+
+    ccond_o = (dsp_o >> 24) & 0xFF;
+    outflag_o = (dsp_o >> 16) & 0xFF;
+    efi_o = (dsp_o >> 14) & 0x01;
+    c_o = (dsp_o >> 14) & 0x01;
+    scount_o = (dsp_o >> 7) & 0x3F;
+    pos_o = dsp_o & 0x1F;
+
+    if ((ccond_o != ccond_r) || (outflag_o != outflag_r) || (efi_o != efi_r) \
+            || (c_o != c_r) || (scount_o != scount_r) || (pos_o != pos_r)) {
+        printf("wrddsp wrong\n");
+
+        return -1;
+    }
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/.directory b/tests/tcg/mips/mips64-dspr2/.directory
new file mode 100644
index 000000000..c75a91451
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/.directory
@@ -0,0 +1,2 @@
+[Dolphin]
+Timestamp=2012,8,3,16,41,52
diff --git a/tests/tcg/mips/mips64-dspr2/Makefile b/tests/tcg/mips/mips64-dspr2/Makefile
new file mode 100644
index 000000000..ba44bb9c0
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/Makefile
@@ -0,0 +1,116 @@
+CROSS_COMPILE	?= mips64el-unknown-linux-gnu-
+
+SIM = qemu-system-mips64el
+SIMFLAGS = -nographic -cpu mips64dspr2 -kernel
+
+AS      = $(CROSS_COMPILE)as
+LD      = $(CROSS_COMPILE)ld
+CC      = $(CROSS_COMPILE)gcc
+AR      = $(CROSS_COMPILE)ar
+NM      = $(CROSS_COMPILE)nm
+STRIP       = $(CROSS_COMPILE)strip
+RANLIB      = $(CROSS_COMPILE)ranlib
+OBJCOPY     = $(CROSS_COMPILE)objcopy
+OBJDUMP     = $(CROSS_COMPILE)objdump
+
+VECTORS_OBJ ?= ./head.o ./printf.o
+
+HEAD_FLAGS ?= -nostdinc -mabi=64 -G 0 -mno-abicalls -fno-pic -pipe \
+              -msoft-float -march=mips64 -Wa,-mips64 -Wa,--trap \
+              -msym32 -DKBUILD_64BIT_SYM32 -I./
+
+CFLAGS ?= -nostdinc -mabi=64 -G 0 -mno-abicalls -fno-pic -fno-builtin  \
+          -pipe -march=mips64r2 -mgp64 -mdspr2 -static -Wa,--trap -msym32 \
+          -DKBUILD_64BIT_SYM32 -I./
+
+LDFLAGS = -T./mips_boot.lds -L./
+FLAGS = -nostdlib -mabi=64 -march=mips64r2 -mgp64 -mdspr2
+
+TESTCASES = absq_s_qb.tst
+TESTCASES += addqh_ph.tst
+TESTCASES += addqh_r_ph.tst
+TESTCASES += addqh_r_w.tst
+TESTCASES += addqh_w.tst
+#TESTCASES += adduh_ob.tst
+TESTCASES += adduh_qb.tst
+#TESTCASES += adduh_r_ob.tst
+TESTCASES += adduh_r_qb.tst
+TESTCASES += addu_ph.tst
+#TESTCASES += addu_qh.tst
+TESTCASES += addu_s_ph.tst
+#TESTCASES += addu_s_qh.tst
+TESTCASES += append.tst
+TESTCASES += balign.tst
+#TESTCASES += cmpgdu_eq_ob.tst
+TESTCASES += cmpgdu_eq_qb.tst
+#TESTCASES += cmpgdu_le_ob.tst
+TESTCASES += cmpgdu_le_qb.tst
+#TESTCASES += cmpgdu_lt_ob.tst
+TESTCASES += cmpgdu_lt_qb.tst
+#TESTCASES += dbalign.tst
+TESTCASES += dpaqx_sa_w_ph.tst
+TESTCASES += dpaqx_s_w_ph.tst
+TESTCASES += dpa_w_ph.tst
+#TESTCASES += dpa_w_qh.tst
+TESTCASES += dpax_w_ph.tst
+TESTCASES += dpsqx_sa_w_ph.tst
+TESTCASES += dpsqx_s_w_ph.tst
+TESTCASES += dps_w_ph.tst
+#TESTCASES += dps_w_qh.tst
+TESTCASES += dpsx_w_ph.tst
+TESTCASES += mul_ph.tst
+TESTCASES += mulq_rs_w.tst
+TESTCASES += mulq_s_ph.tst
+TESTCASES += mulq_s_w.tst
+TESTCASES += mulsaq_s_w_ph.tst
+TESTCASES += mulsa_w_ph.tst
+TESTCASES += mul_s_ph.tst
+TESTCASES += precr_qb_ph.tst
+TESTCASES += precr_sra_ph_w.tst
+TESTCASES += precr_sra_r_ph_w.tst
+TESTCASES += prepend.tst
+TESTCASES += shra_qb.tst
+TESTCASES += shra_r_qb.tst
+#TESTCASES += shrav_ob.tst
+TESTCASES += shrav_qb.tst
+#TESTCASES += shrav_r_ob.tst
+TESTCASES += shrav_r_qb.tst
+TESTCASES += shrl_ph.tst
+TESTCASES += shrlv_ph.tst
+TESTCASES += subqh_ph.tst
+TESTCASES += subqh_r_ph.tst
+TESTCASES += subqh_r_w.tst
+TESTCASES += subqh_w.tst
+#TESTCASES += subuh_ob.tst
+TESTCASES += subuh_qb.tst
+#TESTCASES += subuh_r_ob.tst
+TESTCASES += subuh_r_qb.tst
+TESTCASES += subu_ph.tst
+#TESTCASES += subu_qh.tst
+TESTCASES += subu_s_ph.tst
+#TESTCASES += subu_s_qh.tst
+
+all: build
+
+head.o : head.S
+	$(Q)$(CC) $(HEAD_FLAGS) -D"STACK_TOP=0xffffffff80200000" -c $< -o $@
+
+%.o  : %.S
+	$(CC) $(CFLAGS) -c $< -o $@
+
+%.o  : %.c
+	$(CC) $(CFLAGS) -c $< -o $@
+
+%.tst: %.o $(VECTORS_OBJ)
+	$(CC) $(VECTORS_OBJ) $(FLAGS) $(LDFLAGS) $< -o $@
+
+build: $(VECTORS_OBJ) $(MIPSSOC_LIB) $(TESTCASES)
+
+check:  $(VECTORS_OBJ) $(MIPSSOC_LIB) $(TESTCASES)
+	@for case in $(TESTCASES); do \
+		echo $(SIM) $(SIMFLAGS) ./$$case; \
+		$(SIM) $(SIMFLAGS) ./$$case & (sleep 1; killall $(SIM)); \
+	done
+
+clean:
+	$(Q)rm -f *.o *.tst *.a
diff --git a/tests/tcg/mips/mips64-dspr2/absq_s_qb.c b/tests/tcg/mips/mips64-dspr2/absq_s_qb.c
new file mode 100644
index 000000000..f7aec3e56
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/absq_s_qb.c
@@ -0,0 +1,42 @@
+#include "io.h"
+int main()
+{
+    long long input, result, dsp;
+    long long hope;
+
+    input = 0x701BA35E;
+    hope  = 0x701B5D5E;
+
+    __asm
+        ("absq_s.qb %0, %1\n\t"
+         : "=r"(result)
+         : "r"(input)
+        );
+    if (result != hope) {
+        printf("absq_s.qb error\n");
+        return -1;
+    }
+
+    input = 0x801BA35E;
+    hope  = 0x7F1B5D5E;
+
+    __asm
+        ("absq_s.qb %0, %2\n\t"
+         "rddsp %1\n\t"
+         : "=r"(result), "=r"(dsp)
+         : "r"(input)
+        );
+    dsp = dsp >> 20;
+    dsp &= 0x01;
+    if (result != hope) {
+        printf("absq_s.qb error\n");
+        return -1;
+    }
+
+    if (dsp != 1) {
+        printf("absq_s.qb error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/addqh_ph.c b/tests/tcg/mips/mips64-dspr2/addqh_ph.c
new file mode 100644
index 000000000..6b43cb8d0
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/addqh_ph.c
@@ -0,0 +1,35 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long result;
+
+    rs     = 0x706A13FE;
+    rt     = 0x13065174;
+    result = 0x41B832B9;
+    __asm
+        ("addqh.ph %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    if (result != rd) {
+        printf("addqh.ph error!\n");
+        return -1;
+    }
+
+    rs     = 0x81000100;
+    rt     = 0xc2000100;
+    result = 0xffffffffa1800100;
+    __asm
+        ("addqh.ph %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    if (result != rd) {
+        printf("addqh.ph error!\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/addqh_r_ph.c b/tests/tcg/mips/mips64-dspr2/addqh_r_ph.c
new file mode 100644
index 000000000..890ec98d9
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/addqh_r_ph.c
@@ -0,0 +1,35 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long result;
+
+    rs     = 0x706A13FE;
+    rt     = 0x13065174;
+    result = 0x41B832B9;
+    __asm
+        ("addqh_r.ph %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    if (rd != result) {
+        printf("addqh_r.ph error\n");
+        return -1;
+    }
+
+    rs     = 0x81010100;
+    rt     = 0xc2000100;
+    result = 0xffffffffa1810100;
+    __asm
+        ("addqh_r.ph %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    if (rd != result) {
+        printf("addqh_r.ph error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/addqh_r_w.c b/tests/tcg/mips/mips64-dspr2/addqh_r_w.c
new file mode 100644
index 000000000..d324decbd
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/addqh_r_w.c
@@ -0,0 +1,38 @@
+#include"io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long result;
+
+    rs     = 0x00000010;
+    rt     = 0x00000001;
+    result = 0x00000009;
+
+    __asm
+        ("addqh_r.w  %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+
+    if (rd != result) {
+        printf("addqh_r.w error!\n");
+        return -1;
+    }
+    rs     = 0xFFFFFFFE;
+    rt     = 0x00000001;
+    result = 0x00000000;
+
+    __asm
+        ("addqh_r.w  %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+
+    if (rd != result) {
+        printf("addqh_r.w error!\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/addqh_w.c b/tests/tcg/mips/mips64-dspr2/addqh_w.c
new file mode 100644
index 000000000..78559e678
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/addqh_w.c
@@ -0,0 +1,39 @@
+#include"io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long result;
+
+    rs     = 0x00000010;
+    rt     = 0x00000001;
+    result = 0x00000008;
+
+    __asm
+        ("addqh.w  %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+
+    if (rd != result) {
+        printf("addqh.w wrong\n");
+        return -1;
+    }
+
+    rs     = 0xFFFFFFFE;
+    rt     = 0x00000001;
+    result = 0xFFFFFFFFFFFFFFFF;
+
+    __asm
+        ("addqh.w  %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+
+    if (rd != result) {
+        printf("addqh.w wrong\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/addu_ph.c b/tests/tcg/mips/mips64-dspr2/addu_ph.c
new file mode 100644
index 000000000..d64c8cde8
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/addu_ph.c
@@ -0,0 +1,37 @@
+#include"io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long dsp;
+    long long result;
+
+    rs     = 0x00FF00FF;
+    rt     = 0x00010001;
+    result = 0x01000100;
+    __asm
+        ("addu.ph %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    if (rd != result) {
+        printf("1 addu.ph error\n");
+        return -1;
+    }
+
+    rs     = 0xFFFF1111;
+    rt     = 0x00020001;
+    result = 0x00011112;
+    __asm
+        ("addu.ph %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    if ((rd != result) || (((dsp >> 20) & 0x01) != 1)) {
+        printf("2 addu.ph error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/addu_qh.c b/tests/tcg/mips/mips64-dspr2/addu_qh.c
new file mode 100644
index 000000000..edcbf342c
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/addu_qh.c
@@ -0,0 +1,43 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, dspreg;
+    long long result, dspresult;
+
+    rs = 0x123456787FFF0000;
+    rt = 0x1111111180000000;
+    result = 0x23456789FFFF0000;
+    dspresult = 0x0;
+
+    __asm("addu.qh %0, %2, %3\n\t"
+          "rddsp %1\n\t"
+          : "=r"(rd), "=r"(dspreg)
+          : "r"(rs), "r"(rt)
+         );
+
+    dspreg = ((dspreg >> 20) & 0x01);
+    if ((rd != result) || (dspreg != dspresult)) {
+        printf("addu.qh error\n");
+        return -1;
+    }
+
+    rs = 0x123456787FFF0000;
+    rt = 0x1111111180020000;
+    result = 0x23456789FFFF0000;
+    dspresult = 0x01;
+
+    __asm("addu.qh %0, %2, %3\n\t"
+          "rddsp %1\n\t"
+          : "=r"(rd), "=r"(dspreg)
+          : "r"(rs), "r"(rt)
+         );
+
+    dspreg = ((dspreg >> 20) & 0x01);
+    if ((rd != result) || (dspreg != dspresult)) {
+        printf("addu.qh overflow error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/addu_s_ph.c b/tests/tcg/mips/mips64-dspr2/addu_s_ph.c
new file mode 100644
index 000000000..9250edb45
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/addu_s_ph.c
@@ -0,0 +1,37 @@
+#include"io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long dsp;
+    long long result;
+
+    rs     = 0x00FE00FE;
+    rt     = 0x00020001;
+    result = 0x010000FF;
+    __asm
+        ("addu_s.ph %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    if (rd != result) {
+        printf("addu_s.ph error\n");
+        return -1;
+    }
+
+    rs     = 0xFFFF1111;
+    rt     = 0x00020001;
+    result = 0xFFFFFFFFFFFF1112;
+    __asm
+        ("addu_s.ph %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    if ((rd != result) || (((dsp >> 20) & 0x01) != 1)) {
+        printf("addu_s.ph error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/addu_s_qh.c b/tests/tcg/mips/mips64-dspr2/addu_s_qh.c
new file mode 100644
index 000000000..b0c162625
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/addu_s_qh.c
@@ -0,0 +1,43 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, dspreg;
+    long long result, dspresult;
+
+    rs = 0x123456787FFF0000;
+    rt = 0x1111111180000000;
+    result = 0x23456789FFFF0000;
+    dspresult = 0x0;
+
+    __asm("addu_s.qh %0, %2, %3\n\t"
+          "rddsp %1\n\t"
+          : "=r"(rd), "=r"(dspreg)
+          : "r"(rs), "r"(rt)
+         );
+
+    dspreg = ((dspreg >> 20) & 0x01);
+    if ((rd != result) || (dspreg != dspresult)) {
+        printf("1 addu_s.qh error\n");
+        return -1;
+    }
+
+    rs = 0x12345678FFFF0000;
+    rt = 0x11111111000F0000;
+    result = 0x23456789FFFF0000;
+    dspresult = 0x01;
+
+    __asm("addu_s.qh %0, %2, %3\n\t"
+          "rddsp %1\n\t"
+          : "=r"(rd), "=r"(dspreg)
+          : "r"(rs), "r"(rt)
+         );
+
+    dspreg = ((dspreg >> 20) & 0x01);
+    if ((rd != result) || (dspreg != dspresult)) {
+        printf("2 addu_s.qh error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/adduh_ob.c b/tests/tcg/mips/mips64-dspr2/adduh_ob.c
new file mode 100644
index 000000000..9b309f6f1
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/adduh_ob.c
@@ -0,0 +1,35 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, result;
+    rs = 0xFF987CDEBCEF2356;
+    rt = 0xFF987CDEBCEF2354;
+    result = 0xFF987CDEBCEF2355;
+
+    __asm("adduh.ob %0, %1, %2\n\t"
+          : "=r"(rd)
+          : "r"(rs), "r"(rt)
+         );
+
+    if (rd != result) {
+        printf("adduh.ob error\n\t");
+        return -1;
+    }
+
+    rs = 0xac50691729945316;
+    rt = 0xb9234ca3f5573162;
+    result = 0xb2395a5d8f75423c;
+
+    __asm("adduh.ob %0, %1, %2\n\t"
+          : "=r"(rd)
+          : "r"(rs), "r"(rt)
+         );
+
+    if (rd != result) {
+        printf("adduh.ob error\n\t");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/adduh_qb.c b/tests/tcg/mips/mips64-dspr2/adduh_qb.c
new file mode 100644
index 000000000..796b409a8
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/adduh_qb.c
@@ -0,0 +1,35 @@
+#include"io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long result;
+
+    rs     = 0xFF0055AA;
+    rt     = 0x0113421B;
+    result = 0xffffffff80094B62;
+    __asm
+        ("adduh.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    if (rd != result) {
+        printf("adduh.qb error\n");
+        return -1;
+    }
+    rs     = 0xFFFF0FFF;
+    rt     = 0x00010111;
+    result = 0x7F800888;
+
+    __asm
+        ("adduh.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    if (rd != result) {
+        printf("adduh.qb error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/adduh_r_ob.c b/tests/tcg/mips/mips64-dspr2/adduh_r_ob.c
new file mode 100644
index 000000000..832de833e
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/adduh_r_ob.c
@@ -0,0 +1,35 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, result;
+    rs = 0xFF987CDEBCEF2356;
+    rt = 0xFF987CDEBCEF2355;
+    result = 0xFF987CDEBCEF2356;
+
+    __asm("adduh_r.ob %0, %1, %2\n\t"
+          : "=r"(rd)
+          : "r"(rs), "r"(rt)
+         );
+
+    if (rd != result) {
+        printf("1 adduh_r.ob error\n\t");
+        return -1;
+    }
+
+    rs = 0xac50691729945316;
+    rt = 0xb9234ca3f5573162;
+    result = 0xb33a5b5d8f76423c;
+
+    __asm("adduh_r.ob %0, %1, %2\n\t"
+          : "=r"(rd)
+          : "r"(rs), "r"(rt)
+         );
+
+    if (rd != result) {
+        printf("2 adduh_r.ob error\n\t");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/adduh_r_qb.c b/tests/tcg/mips/mips64-dspr2/adduh_r_qb.c
new file mode 100644
index 000000000..ae65fa5e1
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/adduh_r_qb.c
@@ -0,0 +1,35 @@
+#include"io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long result;
+
+    rs     = 0xFF0055AA;
+    rt     = 0x01112211;
+    result = 0xffffffff80093C5E;
+    __asm
+        ("adduh_r.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    if (rd != result) {
+        printf("adduh_r.qb error\n");
+        return -1;
+    }
+
+    rs     = 0xFFFF0FFF;
+    rt     = 0x00010111;
+    result = 0xffffffff80800888;
+    __asm
+        ("adduh_r.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    if (rd != result) {
+        printf("adduh_r.qb error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/append.c b/tests/tcg/mips/mips64-dspr2/append.c
new file mode 100644
index 000000000..68a7cecc4
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/append.c
@@ -0,0 +1,35 @@
+#include"io.h"
+
+int main(void)
+{
+    long long rs, rt;
+    long long result;
+
+    rs     = 0xFF0055AA;
+    rt     = 0x0113421B;
+    result = 0x02268436;
+    __asm
+        ("append %0, %1, 0x01\n\t"
+         : "+r"(rt)
+         : "r"(rs)
+        );
+    if (rt != result) {
+        printf("append error\n");
+        return -1;
+    }
+
+    rs     = 0xFFFF0FFF;
+    rt     = 0x00010111;
+    result = 0x0010111F;
+    __asm
+        ("append %0, %1, 0x04\n\t"
+         : "+r"(rt)
+         : "r"(rs)
+        );
+    if (rt != result) {
+        printf("append error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/balign.c b/tests/tcg/mips/mips64-dspr2/balign.c
new file mode 100644
index 000000000..7fbe81578
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/balign.c
@@ -0,0 +1,35 @@
+#include"io.h"
+
+int main(void)
+{
+    long long rs, rt;
+    long long result;
+
+    rs     = 0xFF0055AA;
+    rt     = 0x0113421B;
+    result = 0x13421BFF;
+    __asm
+        ("balign %0, %1, 0x01\n\t"
+         : "+r"(rt)
+         : "r"(rs)
+        );
+    if (rt != result) {
+        printf("balign error\n");
+        return -1;
+    }
+
+    rs     = 0xFFFF0FFF;
+    rt     = 0x00010111;
+    result = 0x11FFFF0F;
+    __asm
+        ("balign %0, %1, 0x03\n\t"
+         : "+r"(rt)
+         : "r"(rs)
+        );
+    if (rt != result) {
+        printf("balign error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/cmpgdu_eq_ob.c b/tests/tcg/mips/mips64-dspr2/cmpgdu_eq_ob.c
new file mode 100644
index 000000000..61217f38c
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/cmpgdu_eq_ob.c
@@ -0,0 +1,44 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, result, dspreg, dspresult;
+
+    rs = 0x123456789ABCDEF0;
+    rt = 0x123456789ABCDEFF;
+    result = 0xFE;
+    dspresult = 0xFE;
+
+    __asm("cmpgdu.eq.ob %0, %2, %3\n\t"
+          "rddsp %1"
+          : "=r"(rd), "=r"(dspreg)
+          : "r"(rs), "r"(rt)
+         );
+
+    dspreg = ((dspreg >> 24) & 0xFF);
+
+    if ((rd != result) || (dspreg != dspresult)) {
+        printf("1 cmpgdu.eq.ob error\n");
+        return -1;
+    }
+
+    rs = 0x133256789ABCDEF0;
+    rt = 0x123456789ABCDEFF;
+    result = 0x3E;
+    dspresult = 0x3E;
+
+    __asm("cmpgdu.eq.ob %0, %2, %3\n\t"
+          "rddsp %1"
+          : "=r"(rd), "=r"(dspreg)
+          : "r"(rs), "r"(rt)
+         );
+
+    dspreg = ((dspreg >> 24) & 0xFF);
+
+    if ((rd != result) || (dspreg != dspresult)) {
+        printf("2 cmpgdu.eq.ob error\n");
+        return -1;
+    }
+
+   return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/cmpgdu_eq_qb.c b/tests/tcg/mips/mips64-dspr2/cmpgdu_eq_qb.c
new file mode 100644
index 000000000..c63f6480e
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/cmpgdu_eq_qb.c
@@ -0,0 +1,41 @@
+#include"io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long dsp;
+    long long result;
+
+    rs         = 0x11777066;
+    rt         = 0x55AA70FF;
+    result     = 0x02;
+    __asm
+        ("cmpgdu.eq.qb %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 24) & 0x0F;
+    if ((rd != result) || (dsp != result)) {
+        printf("cmpgdu.eq.qb error\n");
+        return -1;
+    }
+
+    rs     = 0x11777066;
+    rt     = 0x11777066;
+    result = 0x0F;
+    __asm
+        ("cmpgdu.eq.qb %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 24) & 0x0F;
+
+    if ((rd != result) || (dsp != result)) {
+        printf("cmpgdu.eq.qb error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/cmpgdu_le_ob.c b/tests/tcg/mips/mips64-dspr2/cmpgdu_le_ob.c
new file mode 100644
index 000000000..b3da09818
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/cmpgdu_le_ob.c
@@ -0,0 +1,44 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, result, dspreg, dspresult;
+
+    rs = 0x123456789abcdef0;
+    rt = 0x123456789abcdeff;
+    dspresult = 0xff;
+    result = 0xff;
+
+    __asm("cmpgdu.le.ob %0, %2, %3\n\t"
+          "rddsp %1"
+          : "=r"(rd), "=r"(dspreg)
+          : "r"(rs), "r"(rt)
+         );
+
+    dspreg = ((dspreg >> 24) & 0xff);
+
+    if ((rd != result) || (dspreg != dspresult)) {
+        printf("cmpgdu.le.ob error\n");
+        return -1;
+    }
+
+    rs = 0x113556789ABCDEF0;
+    rt = 0x123456789ABCDEFF;
+    result = 0xBE;
+    dspresult = 0xFE;
+
+    __asm("cmpgdu.eq.ob %0, %2, %3\n\t"
+          "rddsp %1"
+          : "=r"(rd), "=r"(dspreg)
+          : "r"(rs), "r"(rt)
+         );
+
+    dspreg = ((dspreg >> 24) & 0xFF);
+
+    if ((rd != result) || (dspreg != dspresult)) {
+        printf("cmpgdu.eq.ob error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/cmpgdu_le_qb.c b/tests/tcg/mips/mips64-dspr2/cmpgdu_le_qb.c
new file mode 100644
index 000000000..f0a60ea4e
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/cmpgdu_le_qb.c
@@ -0,0 +1,48 @@
+#include"io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long dsp;
+    long long result;
+
+    rs         = 0x11777066;
+    rt         = 0x55AA70FF;
+    result     = 0x0F;
+    __asm
+        ("cmpgdu.le.qb %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 24) & 0x0F;
+    if (rd  != result) {
+        printf("cmpgdu.le.qb error\n");
+        return -1;
+    }
+    if (dsp != result) {
+        printf("cmpgdu.le.qb error\n");
+        return -1;
+    }
+
+    rs     = 0x11777066;
+    rt     = 0x11707066;
+    result = 0x0B;
+    __asm
+        ("cmpgdu.le.qb %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 24) & 0x0F;
+    if (rd  != result) {
+        printf("cmpgdu.le.qb error\n");
+        return -1;
+    }
+    if (dsp != result) {
+        printf("cmpgdu.le.qb error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/cmpgdu_lt_ob.c b/tests/tcg/mips/mips64-dspr2/cmpgdu_lt_ob.c
new file mode 100644
index 000000000..d80b4e6ab
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/cmpgdu_lt_ob.c
@@ -0,0 +1,44 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, result, dspreg, dspresult;
+
+    rs = 0x123456789ABCDEF0;
+    rt = 0x123456789ABCDEFF;
+    dspresult = 0x01;
+    result = 0x01;
+
+    __asm("cmpgdu.lt.ob %0, %2, %3\n\t"
+          "rddsp %1"
+          : "=r"(rd), "=r"(dspreg)
+          : "r"(rs), "r"(rt)
+         );
+
+    dspreg = ((dspreg >> 24) & 0xFF);
+
+    if ((rd != result) || (dspreg != dspresult)) {
+        printf("cmpgdu.lt.ob error\n");
+        return -1;
+    }
+
+    rs = 0x143356789ABCDEF0;
+    rt = 0x123456789ABCDEFF;
+    dspresult = 0x41;
+    result = 0x41;
+
+    __asm("cmpgdu.lt.ob %0, %2, %3\n\t"
+          "rddsp %1"
+          : "=r"(rd), "=r"(dspreg)
+          : "r"(rs), "r"(rt)
+         );
+
+    dspreg = ((dspreg >> 24) & 0xFF);
+
+    if ((rd != result) || (dspreg != dspresult)) {
+        printf("cmpgdu.lt.ob error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/cmpgdu_lt_qb.c b/tests/tcg/mips/mips64-dspr2/cmpgdu_lt_qb.c
new file mode 100644
index 000000000..a71e4e307
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/cmpgdu_lt_qb.c
@@ -0,0 +1,48 @@
+#include"io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long dsp;
+    long long result;
+
+    rs         = 0x11777066;
+    rt         = 0x55AA70FF;
+    result     = 0x0D;
+    __asm
+        ("cmpgdu.lt.qb %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 24) & 0x0F;
+    if (rd  != result) {
+        printf("cmpgdu.lt.qb error\n");
+        return -1;
+    }
+    if (dsp != result) {
+        printf("cmpgdu.lt.qb error\n");
+        return -1;
+    }
+
+    rs     = 0x11777066;
+    rt     = 0x11777066;
+    result = 0x00;
+    __asm
+        ("cmpgdu.lt.qb %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 24) & 0x0F;
+    if (rd  != result) {
+        printf("cmpgdu.lt.qb error\n");
+        return -1;
+    }
+    if (dsp != result) {
+        printf("cmpgdu.lt.qb error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/dbalign.c b/tests/tcg/mips/mips64-dspr2/dbalign.c
new file mode 100644
index 000000000..c7431b185
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/dbalign.c
@@ -0,0 +1,39 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rt, rs;
+    long long res;
+
+    rt = 0x1234567887654321;
+    rs = 0xabcd1234abcd1234;
+
+    res = 0x34567887654321ab;
+
+    asm ("dbalign %0, %1, 0x1\n"
+         : "=r"(rt)
+         : "r"(rs)
+        );
+
+    if (rt != res) {
+        printf("dbalign error\n");
+        return -1;
+    }
+
+    rt = 0x1234567887654321;
+    rs = 0xabcd1234abcd1234;
+
+    res = 0x7887654321abcd12;
+
+    asm ("dbalign %0, %1, 0x3\n"
+         : "=r"(rt)
+         : "r"(rs)
+        );
+
+    if (rt != res) {
+        printf("dbalign error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/dpa_w_ph.c b/tests/tcg/mips/mips64-dspr2/dpa_w_ph.c
new file mode 100644
index 000000000..39dc99aa5
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/dpa_w_ph.c
@@ -0,0 +1,47 @@
+#include"io.h"
+
+int main(void)
+{
+    long long rs, rt;
+    long long ach = 5, acl = 5;
+    long long resulth, resultl;
+
+    rs = 0x00FF00FF;
+    rt = 0x00010002;
+    resulth = 0x05;
+    resultl = 0x0302;
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpa.w.ph $ac1, %2, %3\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         : "+r"(ach), "+r"(acl)
+         : "r"(rs), "r"(rt)
+        );
+    if ((ach != resulth) || (acl != resultl)) {
+        printf("1 dpa.w.ph error\n");
+        return -1;
+    }
+
+    ach = 6, acl = 7;
+    rs = 0xFFFF00FF;
+    rt = 0xFFFF0002;
+    resulth = 0x05;
+    resultl = 0xfffffffffffe0206;
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpa.w.ph $ac1, %2, %3\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         : "+r"(ach), "+r"(acl)
+         : "r"(rs), "r"(rt)
+        );
+    if ((ach != resulth) || (acl != resultl)) {
+        printf("2 dpa.w.ph error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/dpa_w_qh.c b/tests/tcg/mips/mips64-dspr2/dpa_w_qh.c
new file mode 100644
index 000000000..1411e44be
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/dpa_w_qh.c
@@ -0,0 +1,56 @@
+#include"io.h"
+int main(void)
+{
+    long long rt, rs;
+    long long achi, acli;
+    long long acho, aclo;
+    long long resh, resl;
+
+    achi = 0x1;
+    acli = 0x1;
+
+    rs = 0x0001000100010001;
+    rt = 0x0002000200020002;
+
+    resh = 0x1;
+    resl = 0x9;
+
+    asm("mthi %2, $ac1\t\n"
+        "mtlo %3, $ac1\t\n"
+        "dpa.w.qh $ac1, %4, %5\t\n"
+        "mfhi %0, $ac1\t\n"
+        "mflo %1, $ac1\t\n"
+        : "=r"(acho), "=r"(aclo)
+        : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+       );
+
+    if ((acho != resh) || (aclo != resl)) {
+        printf("1 dpa.w.qh error\n");
+        return -1;
+    }
+
+
+    achi = 0xffffffff;
+    acli = 0xaaaaaaaa;
+
+    rs = 0xaaaabbbbccccdddd;
+    rt = 0x7777888899996666;
+
+    resh = 0xffffffffffffffff;
+    resl = 0x320cdf02;
+
+    asm("mthi %2, $ac1\t\n"
+        "mtlo %3, $ac1\t\n"
+        "dpa.w.qh $ac1, %4, %5\t\n"
+        "mfhi %0, $ac1\t\n"
+        "mflo %1, $ac1\t\n"
+        : "=r"(acho), "=r"(aclo)
+        : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+       );
+    if ((acho != resh) || (aclo != resl)) {
+        printf("2 dpa.w.qh error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/dpaqx_s_w_ph.c b/tests/tcg/mips/mips64-dspr2/dpaqx_s_w_ph.c
new file mode 100644
index 000000000..51252fb98
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/dpaqx_s_w_ph.c
@@ -0,0 +1,97 @@
+#include"io.h"
+
+int main(void)
+{
+    long long rs, rt, dsp;
+    long long ach = 5, acl = 5;
+    long long resulth, resultl, resultdsp;
+
+    rs     = 0x800000FF;
+    rt     = 0x00018000;
+    resulth = 0x05;
+    resultl = 0xFFFFFFFF80000202;
+    resultdsp = 0x01;
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpaqx_s.w.ph $ac1, %3, %4\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "+r"(ach), "+r"(acl), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 17) & 0x01;
+    if (dsp != resultdsp) {
+        printf("dpaqx_s.w.ph error\n");
+        return -1;
+    }
+    if (ach != resulth) {
+        printf("dpaqx_s.w.ph error\n");
+        return -1;
+    }
+    if (acl != resultl) {
+        printf("dpaqx_s.w.ph error\n");
+        return -1;
+    }
+
+    ach    = 5;
+    acl    = 5;
+    rs     = 0x00FF00FF;
+    rt     = 0x00010002;
+    resulth = 0x05;
+    resultl = 0x05FF;
+    /***********************************************************
+     * Because of we set outflag at last time, although this
+     * time we set nothing, but it is stay the last time value.
+     **********************************************************/
+    resultdsp = 0x01;
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpaqx_s.w.ph $ac1, %3, %4\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "+r"(ach), "+r"(acl), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 17) & 0x01;
+    if (dsp != resultdsp) {
+        printf("dpaqx_s.w.ph error\n");
+        return -1;
+    }
+    if (ach != resulth) {
+        printf("dpaqx_s.w.ph error\n");
+        return -1;
+    }
+    if (acl != resultl) {
+        printf("dpaqx_s.w.ph error\n");
+        return -1;
+    }
+
+    ach    = 5;
+    acl    = 5;
+    rs     = 0x800000FF;
+    rt     = 0x00028000;
+    resulth = 0x05;
+    resultl = 0xffffffff80000400;
+    resultdsp = 0x01;
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpaqx_s.w.ph $ac1, %3, %4\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "+r"(ach), "+r"(acl), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 17) & 0x01;
+    if ((dsp != resultdsp) || (ach != resulth) || (acl != resultl)) {
+        printf("dpaqx_s.w.ph error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/dpaqx_sa_w_ph.c b/tests/tcg/mips/mips64-dspr2/dpaqx_sa_w_ph.c
new file mode 100644
index 000000000..18d6b3a98
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/dpaqx_sa_w_ph.c
@@ -0,0 +1,54 @@
+#include "io.h"
+
+int main()
+{
+    long long rs, rt, dsp;
+    long long ach = 5, acl = 5;
+    long long resulth, resultl, resultdsp;
+
+    rs     = 0x00FF00FF;
+    rt     = 0x00010002;
+    resulth = 0x00;
+    resultl = 0x7FFFFFFF;
+    resultdsp = 0x01;
+    __asm
+        ("wrdsp %2\n\t"
+         "mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpaqx_sa.w.ph $ac1, %3, %4\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "+r"(ach), "+r"(acl), "+r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    if ((dsp >> (16 + 1) != resultdsp) || (ach != resulth) ||
+        (acl != resultl)) {
+        printf("dpaqx_sa.w.ph errror\n");
+    }
+
+    ach = 9;
+    acl = 0xb;
+    rs     = 0x800000FF;
+    rt     = 0x00018000;
+    resulth = 0x00;
+    resultl = 0x7fffffff;
+    resultdsp = 0x01;
+    __asm
+        ("wrdsp %2\n\t"
+         "mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpaqx_sa.w.ph $ac1, %3, %4\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "+r"(ach), "+r"(acl), "+r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    if ((dsp >> (16 + 1) != resultdsp) || (ach != resulth) ||
+        (acl != resultl)) {
+        printf("dpaqx_sa.w.ph errror\n");
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/dpax_w_ph.c b/tests/tcg/mips/mips64-dspr2/dpax_w_ph.c
new file mode 100644
index 000000000..9d595fc14
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/dpax_w_ph.c
@@ -0,0 +1,32 @@
+#include"io.h"
+
+int main(void)
+{
+    long rs, rt;
+    long ach = 5, acl = 5;
+    long resulth, resultl;
+
+    rs     = 0x00FF00FF;
+    rt     = 0x00010002;
+    resulth = 0x05;
+    resultl = 0x0302;
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpax.w.ph $ac1, %2, %3\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         : "+r"(ach), "+r"(acl)
+         : "r"(rs), "r"(rt)
+        );
+    if (ach != resulth) {
+        printf("dpax.w.ph error\n");
+        return -1;
+    }
+    if (acl != resultl) {
+        printf("dpax.w.ph error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/dps_w_ph.c b/tests/tcg/mips/mips64-dspr2/dps_w_ph.c
new file mode 100644
index 000000000..99f292ecb
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/dps_w_ph.c
@@ -0,0 +1,28 @@
+#include"io.h"
+
+int main(void)
+{
+    long long rs, rt;
+    long long ach = 5, acl = 5;
+    long long resulth, resultl;
+
+    rs     = 0x00FF00FF;
+    rt     = 0x00010002;
+    resulth = 0x04;
+    resultl = 0xFFFFFFFFFFFFFFD08;
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dps.w.ph $ac1, %2, %3\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         : "+r"(ach), "+r"(acl)
+         : "r"(rs), "r"(rt)
+        );
+    if (ach != resulth || acl != resultl) {
+        printf("dps.w.ph error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/dps_w_qh.c b/tests/tcg/mips/mips64-dspr2/dps_w_qh.c
new file mode 100644
index 000000000..61277eb30
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/dps_w_qh.c
@@ -0,0 +1,55 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rs, rt;
+    long long achi, acli;
+    long long acho, aclo;
+    long long resh, resl;
+
+    rs = 0x0000000100000001;
+    rt = 0x0000000200000002;
+    achi = 0x1;
+    acli = 0x8;
+
+    resh = 0x1;
+    resl = 0x4;
+
+    asm ("mthi %2, $ac1\t\n"
+         "mtlo %3, $ac1\t\n"
+         "dps.w.qh $ac1, %4, %5\t\n"
+         "mfhi %0, $ac1\t\n"
+         "mflo %1, $ac1\t\n"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+
+    if ((acho != resh) || (aclo != resl)) {
+        printf("1 dps.w.qh error\n");
+        return -1;
+    }
+
+    rs = 0xaaaabbbbccccdddd;
+    rt = 0xaaaabbbbccccdddd;
+
+    achi = 0x88888888;
+    achi = 0x55555555;
+
+    resh = 0xfffffffff7777777;
+    resl = 0x0a38b181;
+
+    asm ("mthi %2, $ac1\t\n"
+         "mtlo %3, $ac1\t\n"
+         "dps.w.qh $ac1, %4, %5\t\n"
+         "mfhi %0, $ac1\t\n"
+         "mflo %1, $ac1\t\n"
+         : "=r"(acho), "=r"(aclo)
+         : "r"(achi), "r"(acli), "r"(rs), "r"(rt)
+        );
+
+    if ((acho != resh) || (aclo != resl)) {
+        printf("1 dps.w.qh error\n");
+        return -1;
+    }
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/dpsqx_s_w_ph.c b/tests/tcg/mips/mips64-dspr2/dpsqx_s_w_ph.c
new file mode 100644
index 000000000..ba46a9269
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/dpsqx_s_w_ph.c
@@ -0,0 +1,55 @@
+#include"io.h"
+
+int main(void)
+{
+    long long rs, rt, dsp;
+    long long ach = 5, acl = 5;
+    long long resulth, resultl, resultdsp;
+
+    rs = 0xBC0123AD;
+    rt = 0x01643721;
+    resulth = 0x04;
+    resultl = 0xFFFFFFFFAEA3E09B;
+    resultdsp = 0x00;
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpsqx_s.w.ph $ac1, %3, %4\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "+r"(ach), "+r"(acl), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 17) & 0x01;
+    if (dsp != resultdsp || ach != resulth || acl != resultl) {
+        printf("dpsqx_s.w.ph error\n");
+        return -1;
+    }
+
+    ach = 0x99f13005;
+    acl = 0x51730062;
+    rs = 0x80008000;
+    rt = 0x80008000;
+
+    resulth = 0xffffffff99f13004;
+    resultl = 0x51730064;
+    resultdsp = 0x01;
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpsqx_s.w.ph $ac1, %3, %4\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "+r"(ach), "+r"(acl), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 17) & 0x01;
+    if (dsp != resultdsp || ach != resulth || acl != resultl) {
+        printf("dpsqx_s.w.ph error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/dpsqx_sa_w_ph.c b/tests/tcg/mips/mips64-dspr2/dpsqx_sa_w_ph.c
new file mode 100644
index 000000000..24c888134
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/dpsqx_sa_w_ph.c
@@ -0,0 +1,53 @@
+#include"io.h"
+int main()
+{
+    long long rs, rt, dsp;
+    long long ach = 5, acl = 5;
+    long long resulth, resultl, resultdsp;
+
+    rs = 0xBC0123AD;
+    rt = 0x01643721;
+    resulth = 0x00;
+    resultl = 0x7FFFFFFF;
+    resultdsp = 0x01;
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpsqx_sa.w.ph $ac1, %3, %4\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "+r"(ach), "+r"(acl), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 17) & 0x01;
+    if (dsp != resultdsp || ach != resulth || acl != resultl) {
+        printf("dpsqx_sa.w.ph error\n");
+        return -1;
+    }
+
+    ach = 0x8c0b354A;
+    acl = 0xbbc02249;
+    rs      = 0x800023AD;
+    rt      = 0x01648000;
+    resulth = 0xffffffffffffffff;
+    resultl = 0xffffffff80000000;
+    resultdsp = 0x01;
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpsqx_sa.w.ph $ac1, %3, %4\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         "rddsp %2\n\t"
+         : "+r"(ach), "+r"(acl), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 17) & 0x01;
+    if (dsp != resultdsp || ach != resulth || acl != resultl) {
+        printf("dpsqx_sa.w.ph error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/dpsx_w_ph.c b/tests/tcg/mips/mips64-dspr2/dpsx_w_ph.c
new file mode 100644
index 000000000..b6291b5eb
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/dpsx_w_ph.c
@@ -0,0 +1,28 @@
+#include"io.h"
+
+int main(void)
+{
+    long long rs, rt;
+    long long ach = 5, acl = 5;
+    long long resulth, resultl;
+
+    rs      = 0xBC0123AD;
+    rt      = 0x01643721;
+    resulth = 0x04;
+    resultl = 0xFFFFFFFFD751F050;
+    __asm
+        ("mthi  %0, $ac1\n\t"
+         "mtlo  %1, $ac1\n\t"
+         "dpsx.w.ph $ac1, %2, %3\n\t"
+         "mfhi  %0, $ac1\n\t"
+         "mflo  %1, $ac1\n\t"
+         : "+r"(ach), "+r"(acl)
+         : "r"(rs), "r"(rt)
+        );
+    if (ach != resulth || acl != resultl) {
+        printf("dpsx.w.ph error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/head.S b/tests/tcg/mips/mips64-dspr2/head.S
new file mode 100644
index 000000000..9a099ae42
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/head.S
@@ -0,0 +1,16 @@
+/*
+ *  Startup Code for MIPS64 CPU-core
+ *
+ */
+.text
+.globl _start
+.align 4
+_start:
+    ori    $2, $2, 0xffff
+    sll    $2, $2, 16
+    ori    $2, $2, 0xffff
+    mtc0   $2, $12, 0
+    jal    main
+
+end:
+    b end
diff --git a/tests/tcg/mips/mips64-dspr2/io.h b/tests/tcg/mips/mips64-dspr2/io.h
new file mode 100644
index 000000000..b7db61d7c
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/io.h
@@ -0,0 +1,22 @@
+#ifndef _ASM_IO_H
+#define _ASM_IO_H
+extern int printf(const char *fmt, ...);
+extern unsigned long get_ticks(void);
+
+#define _read(source)                \
+({ unsigned long __res;                \
+    __asm__ __volatile__(            \
+        "mfc0\t%0, " #source "\n\t"    \
+        : "=r" (__res));        \
+    __res;                    \
+})
+
+#define __read(source)                \
+({ unsigned long __res;                \
+    __asm__ __volatile__(            \
+        "move\t%0, " #source "\n\t"    \
+        : "=r" (__res));        \
+    __res;                    \
+})
+
+#endif
diff --git a/tests/tcg/mips/mips64-dspr2/mips_boot.lds b/tests/tcg/mips/mips64-dspr2/mips_boot.lds
new file mode 100644
index 000000000..bd7c0c0f3
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/mips_boot.lds
@@ -0,0 +1,31 @@
+OUTPUT_ARCH(mips)
+SECTIONS
+{
+    . = 0xffffffff80100000;
+    . = ALIGN((1 << 13));
+    .text :
+    {
+        *(.text)
+        *(.rodata)
+        *(.rodata.*)
+    }
+
+    __init_begin = .;
+    . = ALIGN((1 << 12));
+    .init.text : AT(ADDR(.init.text) - 0)
+    {
+        *(.init.text)
+    }
+    .init.data : AT(ADDR(.init.data) - 0)
+    {
+        *(.init.data)
+    }
+    . = ALIGN((1 << 12));
+    __init_end = .;
+
+    . = ALIGN((1 << 13));
+    .data :
+    {
+        *(.data)
+    }
+}
diff --git a/tests/tcg/mips/mips64-dspr2/mul_ph.c b/tests/tcg/mips/mips64-dspr2/mul_ph.c
new file mode 100644
index 000000000..5a3d05cb2
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/mul_ph.c
@@ -0,0 +1,50 @@
+#include"io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, dsp;
+    long long result, resultdsp;
+
+    rs = 0x03FB1234;
+    rt = 0x0BCC4321;
+    result = 0xFFFFFFFFF504F4B4;
+    resultdsp = 1;
+
+    __asm
+        ("mul.ph %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 21) & 0x01;
+    if (rd  != result || dsp != resultdsp) {
+        printf("mul.ph wrong\n");
+        return -1;
+    }
+
+    dsp = 0;
+    __asm
+        ("wrdsp %0\n\t"
+         :
+         : "r"(dsp)
+        );
+
+    rs = 0x00210010;
+    rt = 0x00110005;
+    result = 0x2310050;
+    resultdsp = 0;
+
+    __asm
+        ("mul.ph %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 21) & 0x01;
+    if (rd  != result || dsp != resultdsp) {
+        printf("mul.ph wrong\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/mul_s_ph.c b/tests/tcg/mips/mips64-dspr2/mul_s_ph.c
new file mode 100644
index 000000000..7c8b2c718
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/mul_s_ph.c
@@ -0,0 +1,67 @@
+#include"io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, dsp;
+    long long result, resultdsp;
+
+    rs = 0x03FB1234;
+    rt = 0x0BCC4321;
+    result = 0x7fff7FFF;
+    resultdsp = 1;
+
+    __asm
+        ("mul_s.ph %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 21) & 0x01;
+    if (rd != result || dsp != resultdsp) {
+        printf("1 mul_s.ph error\n");
+        return -1;
+    }
+
+    rs = 0x7fffff00;
+    rt = 0xff007fff;
+    result = 0xffffffff80008000;
+    resultdsp = 1;
+
+    __asm
+        ("mul_s.ph %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 21) & 0x01;
+    if (rd != result || dsp != resultdsp) {
+        printf("2 mul_s.ph error\n");
+        return -1;
+    }
+
+    dsp = 0;
+    __asm
+        ("wrdsp %0\n\t"
+         :
+         : "r"(dsp)
+        );
+
+    rs = 0x00320001;
+    rt = 0x00210002;
+    result = 0x06720002;
+    resultdsp = 0;
+
+    __asm
+        ("mul_s.ph %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 21) & 0x01;
+    if (rd != result || dsp != resultdsp) {
+        printf("3 mul_s.ph error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/mulq_rs_w.c b/tests/tcg/mips/mips64-dspr2/mulq_rs_w.c
new file mode 100644
index 000000000..ffdc66d54
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/mulq_rs_w.c
@@ -0,0 +1,40 @@
+#include"io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, dsp;
+    long long result, resultdsp;
+
+    rs = 0x80001234;
+    rt = 0x80004321;
+    result = 0xFFFFFFFF80005555;
+
+    __asm
+        ("mulq_rs.w %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    if (rd != result) {
+        printf("mulq_rs.w error!\n");
+        return -1;
+    }
+
+    rs = 0x80000000;
+    rt = 0x80000000;
+    result = 0x7FFFFFFF;
+    resultdsp = 1;
+
+    __asm
+        ("mulq_rs.w %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 21) & 0x01;
+    if (rd != result || dsp != resultdsp) {
+        printf("mulq_rs.w error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/mulq_s_ph.c b/tests/tcg/mips/mips64-dspr2/mulq_s_ph.c
new file mode 100644
index 000000000..b8c20c68c
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/mulq_s_ph.c
@@ -0,0 +1,26 @@
+#include"io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, dsp;
+    long long result, resultdsp;
+
+    rs = 0x80001234;
+    rt = 0x80004321;
+    result = 0x7FFF098B;
+    resultdsp = 1;
+
+    __asm
+        ("mulq_s.ph %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 21) & 0x01;
+    if (rd  != result || dsp != resultdsp) {
+        printf("mulq_s.ph error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/mulq_s_w.c b/tests/tcg/mips/mips64-dspr2/mulq_s_w.c
new file mode 100644
index 000000000..db74b713f
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/mulq_s_w.c
@@ -0,0 +1,40 @@
+#include"io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, dsp;
+    long long result, resultdsp;
+
+    rs = 0x80001234;
+    rt = 0x80004321;
+    result = 0xFFFFFFFF80005555;
+
+    __asm
+        ("mulq_s.w %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    if (rd != result) {
+        printf("mulq_s.w error\n");
+        return -1;
+    }
+
+    rs = 0x80000000;
+    rt = 0x80000000;
+    result = 0x7FFFFFFF;
+    resultdsp = 1;
+
+    __asm
+        ("mulq_s.w %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 21) & 0x01;
+    if (rd != result || dsp != resultdsp) {
+        printf("mulq_s.w error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/mulsa_w_ph.c b/tests/tcg/mips/mips64-dspr2/mulsa_w_ph.c
new file mode 100644
index 000000000..5b22a60a8
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/mulsa_w_ph.c
@@ -0,0 +1,30 @@
+#include"io.h"
+
+int main(void)
+{
+    long long rs, rt, ach, acl;
+    long long resulth, resultl;
+
+    ach = 0x05;
+    acl = 0x00BBDDCC;
+    rs = 0x80001234;
+    rt = 0x80004321;
+    resulth = 0x05;
+    resultl = 0x3BF5E918;
+
+    __asm
+        ("mthi %0, $ac1\n\t"
+         "mtlo %1, $ac1\n\t"
+         "mulsa.w.ph $ac1, %2, %3\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "+r"(ach), "+r"(acl)
+         : "r"(rs), "r"(rt)
+        );
+    if (ach != resulth || acl != resultl) {
+        printf("mulsa.w.ph error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/mulsaq_s_w_ph.c b/tests/tcg/mips/mips64-dspr2/mulsaq_s_w_ph.c
new file mode 100644
index 000000000..835a73d47
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/mulsaq_s_w_ph.c
@@ -0,0 +1,30 @@
+#include"io.h"
+
+int main(void)
+{
+    long long rs, rt, ach, acl;
+    long long resulth, resultl;
+
+    ach = 0x05;
+    acl = 0x00BBDDCC;
+    rs = 0x80001234;
+    rt = 0x80004321;
+    resulth = 0x05;
+    resultl = 0x772ff463;
+
+    __asm
+        ("mthi %0, $ac1\n\t"
+         "mtlo %1, $ac1\n\t"
+         "mulsaq_s.w.ph $ac1, %2, %3\n\t"
+         "mfhi %0, $ac1\n\t"
+         "mflo %1, $ac1\n\t"
+         : "+r"(ach), "+r"(acl)
+         : "r"(rs), "r"(rt)
+        );
+    if (ach != resulth || acl != resultl) {
+        printf("mulsaq_s.w.ph error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/precr_qb_ph.c b/tests/tcg/mips/mips64-dspr2/precr_qb_ph.c
new file mode 100644
index 000000000..80d5e8dce
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/precr_qb_ph.c
@@ -0,0 +1,23 @@
+#include"io.h"
+
+int main()
+{
+    long long rd, rs, rt;
+    long long result;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result = 0x34786521;
+
+    __asm
+        ("precr.qb.ph %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    if (result != rd) {
+        printf("precr.qb.ph error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/precr_sra_ph_w.c b/tests/tcg/mips/mips64-dspr2/precr_sra_ph_w.c
new file mode 100644
index 000000000..b1d7bcdf8
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/precr_sra_ph_w.c
@@ -0,0 +1,37 @@
+#include"io.h"
+
+int main(void)
+{
+    long long rs, rt;
+    long long result;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result = 0x43215678;
+
+    __asm
+        ("precr_sra.ph.w %0, %1, 0x00\n\t"
+         : "+r"(rt)
+         : "r"(rs)
+        );
+    if (result != rt) {
+        printf("precr_sra.ph.w error\n");
+        return -1;
+    }
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result = 0xFFFFFFFFFFFF0000;
+
+    __asm
+        ("precr_sra.ph.w %0, %1, 0x1F\n\t"
+         : "+r"(rt)
+         : "r"(rs)
+        );
+    if (result != rt) {
+        printf("precr_sra.ph.w error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/precr_sra_r_ph_w.c b/tests/tcg/mips/mips64-dspr2/precr_sra_r_ph_w.c
new file mode 100644
index 000000000..62d220dca
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/precr_sra_r_ph_w.c
@@ -0,0 +1,37 @@
+#include"io.h"
+
+int main(void)
+{
+    long long rs, rt;
+    long long result;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result = 0x43215678;
+
+    __asm
+        ("precr_sra_r.ph.w %0, %1, 0x00\n\t"
+         : "+r"(rt)
+         : "r"(rs)
+        );
+    if (result != rt) {
+        printf("precr_sra_r.ph.w error\n");
+        return -1;
+    }
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result = 0xFFFFFFFFFFFF0000;
+
+    __asm
+        ("precr_sra_r.ph.w %0, %1, 0x1F\n\t"
+         : "+r"(rt)
+         : "r"(rs)
+        );
+    if (result != rt) {
+        printf("precr_sra_r.ph.w error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/prepend.c b/tests/tcg/mips/mips64-dspr2/prepend.c
new file mode 100644
index 000000000..4ab083e96
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/prepend.c
@@ -0,0 +1,35 @@
+#include"io.h"
+
+int main(void)
+{
+    long long rs, rt;
+    long long result;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result = 0xFFFFFFFF87654321;
+    __asm
+        ("prepend %0, %1, 0x00\n\t"
+         : "+r"(rt)
+         : "r"(rs)
+        );
+    if (rt != result) {
+        printf("prepend error\n");
+        return -1;
+    }
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result = 0xFFFFFFFFACF10ECA;
+    __asm
+        ("prepend %0, %1, 0x0F\n\t"
+         : "+r"(rt)
+         : "r"(rs)
+        );
+    if (rt != result) {
+        printf("prepend error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/printf.c b/tests/tcg/mips/mips64-dspr2/printf.c
new file mode 100644
index 000000000..cf8676d39
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/printf.c
@@ -0,0 +1,266 @@
+
+typedef unsigned long va_list;
+
+#define ACC    4
+#define __read(source)                    \
+({ va_list __res;                    \
+    __asm__ __volatile__(                \
+        "move\t%0, " #source "\n\t"        \
+        : "=r" (__res));            \
+    __res;                        \
+})
+
+enum format_type {
+    FORMAT_TYPE_NONE,
+    FORMAT_TYPE_HEX,
+    FORMAT_TYPE_ULONG,
+    FORMAT_TYPE_FLOAT
+};
+
+struct printf_spec {
+    char    type;
+};
+
+static int format_decode(char *fmt, struct printf_spec *spec)
+{
+    char *start = fmt;
+
+    for (; *fmt ; ++fmt) {
+        if (*fmt == '%') {
+            break;
+        }
+    }
+
+    switch (*++fmt) {
+    case 'x':
+        spec->type = FORMAT_TYPE_HEX;
+        break;
+
+    case 'd':
+        spec->type = FORMAT_TYPE_ULONG;
+        break;
+
+    case 'f':
+        spec->type = FORMAT_TYPE_FLOAT;
+        break;
+
+    default:
+        spec->type = FORMAT_TYPE_NONE;
+    }
+
+    return ++fmt - start;
+}
+
+void *memcpy(void *dest, void *src, int n)
+{
+    int i;
+    char *s = src;
+    char *d = dest;
+
+    for (i = 0; i < n; i++) {
+        d[i] = s[i];
+    }
+    return dest;
+}
+
+char *number(char *buf, va_list num)
+{
+    int i;
+    char *str = buf;
+    static char digits[16] = "0123456789abcdef";
+    str = str + sizeof(num) * 2;
+
+    for (i = 0; i < sizeof(num) * 2; i++) {
+        *--str = digits[num & 15];
+        num >>= 4;
+    }
+
+    return buf + sizeof(num) * 2;
+}
+
+char *__number(char *buf, va_list num)
+{
+    int i;
+    va_list mm = num;
+    char *str = buf;
+
+    if (!num) {
+        *str++ = '0';
+        return str;
+    }
+
+    for (i = 0; mm; mm = mm/10, i++) {
+        /* Do nothing. */
+    }
+
+    str = str + i;
+
+    while (num) {
+        *--str = num % 10 + 48;
+        num = num / 10;
+    }
+
+    return str + i;
+}
+
+va_list modf(va_list args, va_list *integer, va_list *num)
+{
+    int i;
+    double dot_v = 0;
+    va_list E, DOT, DOT_V;
+
+    if (!args) {
+        return 0;
+    }
+
+    for (i = 0, args = args << 1 >> 1; i < 52; i++) {
+        if ((args >> i) & 0x1) {
+            break;
+        }
+    }
+
+    *integer = 0;
+
+    if ((args >> 56 != 0x3f) || (args >> 52 == 0x3ff)) {
+        E = (args >> 52) - 1023;
+        DOT = 52 - E - i;
+        DOT_V = args << (12 + E) >> (12 + E) >> i;
+        *integer = ((args << 12 >> 12) >> (i + DOT)) | (1 << E);
+    } else {
+        E = ~((args >> 52) - 1023) + 1;
+        DOT_V = args << 12 >> 12;
+
+        dot_v += 1.0 / (1 << E);
+
+        for (i = 1; i <= 16; i++) {
+            if ((DOT_V >> (52 - i)) & 0x1) {
+                dot_v += 1.0 / (1 << E + i);
+            }
+        }
+
+        for (i = 1, E = 0; i <= ACC; i++) {
+            dot_v *= 10;
+            if (!(va_list)dot_v) {
+                E++;
+            }
+    }
+
+    *num = E;
+
+    return dot_v;
+    }
+
+    if (args & 0xf) {
+        for (i = 1; i <= 16; i++) {
+            if ((DOT_V >> (DOT - i)) & 0x1) {
+                dot_v += 1.0 / (1 << i);
+            }
+        }
+
+        for (i = 1, E = 0; i <= ACC; i++) {
+            dot_v *= 10;
+            if (!(va_list)dot_v) {
+                E++;
+            }
+        }
+
+        *num = E;
+
+        return dot_v;
+    } else if (DOT) {
+        for (i = 1; i <= DOT; i++) {
+            if ((DOT_V >> (DOT - i)) & 0x1) {
+                dot_v += 1.0 / (1 << i);
+            }
+        }
+
+        for (i = 1; i <= ACC; i++) {
+            dot_v = dot_v * 10;
+        }
+
+    return dot_v;
+    }
+
+    return 0;
+}
+
+int vsnprintf(char *buf, int size, char *fmt, va_list args)
+{
+    char *str, *mm;
+    struct printf_spec spec = {0};
+
+    str = mm = buf;
+
+    while (*fmt) {
+        char *old_fmt = fmt;
+        int read = format_decode(fmt, &spec);
+
+        fmt += read;
+
+        switch (spec.type) {
+        case FORMAT_TYPE_NONE: {
+            memcpy(str, old_fmt, read);
+            str += read;
+            break;
+        }
+        case FORMAT_TYPE_HEX: {
+            memcpy(str, old_fmt, read);
+            str = number(str + read, args);
+            for (; *mm ; ++mm) {
+                if (*mm == '%') {
+                    *mm = '0';
+                break;
+                }
+            }
+        break;
+        }
+        case FORMAT_TYPE_ULONG: {
+            memcpy(str, old_fmt, read - 2);
+            str = __number(str + read - 2, args);
+            break;
+        }
+        case FORMAT_TYPE_FLOAT: {
+            va_list integer, dot_v, num;
+            dot_v = modf(args, &integer, &num);
+            memcpy(str, old_fmt, read - 2);
+            str += read - 2;
+            if ((args >> 63 & 0x1)) {
+                *str++ = '-';
+            }
+            str = __number(str, integer);
+            if (dot_v) {
+                *str++ = '.';
+                while (num--) {
+                    *str++ = '0';
+                }
+                str = __number(str, dot_v);
+            }
+            break;
+        }
+        }
+    }
+    *str = '\0';
+
+    return str - buf;
+}
+
+static void serial_out(char *str)
+{
+    while (*str) {
+        *(char *)0xffffffffb80003f8 = *str++;
+    }
+}
+
+int vprintf(char *fmt, va_list args)
+{
+    int printed_len = 0;
+    static char printf_buf[512];
+    printed_len = vsnprintf(printf_buf, sizeof(printf_buf), fmt, args);
+    serial_out(printf_buf);
+    return printed_len;
+}
+
+int printf(char *fmt, ...)
+{
+    return vprintf(fmt, __read($5));
+}
diff --git a/tests/tcg/mips/mips64-dspr2/shra_qb.c b/tests/tcg/mips/mips64-dspr2/shra_qb.c
new file mode 100644
index 000000000..cac310235
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/shra_qb.c
@@ -0,0 +1,35 @@
+#include"io.h"
+
+int main(void)
+{
+    long long rd, rt;
+    long long result;
+
+    rt = 0x12345678;
+    result = 0x02060A0F;
+
+    __asm
+        ("shra.qb %0, %1, 0x03\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    if (rd != result) {
+        printf("shra.qb error\n");
+        return -1;
+    }
+
+    rt = 0x87654321;
+    result = 0xFFFFFFFFF00C0804;
+
+    __asm
+        ("shra.qb %0, %1, 0x03\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    if (rd != result) {
+        printf("shra.qb error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/shra_r_qb.c b/tests/tcg/mips/mips64-dspr2/shra_r_qb.c
new file mode 100644
index 000000000..9c64f75bd
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/shra_r_qb.c
@@ -0,0 +1,35 @@
+#include "io.h"
+
+int main()
+{
+    int rd, rt;
+    int result;
+
+    rt = 0x12345678;
+    result = 0x02070B0F;
+
+    __asm
+        ("shra_r.qb %0, %1, 0x03\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    if (rd != result) {
+        printf("shra_r.qb wrong\n");
+        return -1;
+    }
+
+    rt = 0x87654321;
+    result = 0xF10D0804;
+
+    __asm
+        ("shra_r.qb %0, %1, 0x03\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    if (rd != result) {
+        printf("shra_r.qb wrong\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/shrav_ob.c b/tests/tcg/mips/mips64-dspr2/shrav_ob.c
new file mode 100644
index 000000000..fbdfbab35
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/shrav_ob.c
@@ -0,0 +1,22 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt, rs;
+    long long res;
+
+    rt = 0x1234567887654321;
+    rs = 0x4;
+    res = 0xf1f3f5f7f8060402;
+
+    asm ("shrav.ob %0, %1, %2"
+        : "=r"(rd)
+        : "r"(rt), "r"(rs)
+        );
+
+    if (rd != res) {
+        printf("shra.ob error\n");
+        return -1;
+    }
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/shrav_qb.c b/tests/tcg/mips/mips64-dspr2/shrav_qb.c
new file mode 100644
index 000000000..a716203d8
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/shrav_qb.c
@@ -0,0 +1,37 @@
+#include"io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long result;
+
+    rs = 0x03;
+    rt = 0x12345678;
+    result = 0x02060A0F;
+
+    __asm
+        ("shrav.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rt), "r"(rs)
+        );
+    if (rd != result) {
+        printf("shrav.qb error\n");
+        return -1;
+    }
+
+    rs = 0x03;
+    rt = 0x87654321;
+    result = 0xFFFFFFFFF00C0804;
+
+    __asm
+        ("shrav.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rt), "r"(rs)
+        );
+    if (rd != result) {
+        printf("shrav.qb error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/shrav_r_ob.c b/tests/tcg/mips/mips64-dspr2/shrav_r_ob.c
new file mode 100644
index 000000000..b80100a7c
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/shrav_r_ob.c
@@ -0,0 +1,22 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rt, rs;
+    long long res;
+
+    rt = 0x1234567887654321;
+    rs = 0x4;
+    res = 0xe3e7ebf0f1ede9e5;
+
+    asm ("shrav_r.ob %0, %1, %2"
+        : "=r"(rd)
+        : "r"(rt), "r"(rs)
+        );
+
+    if (rd != res) {
+        printf("shra_r.ob error\n");
+        return -1;
+    }
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/shrav_r_qb.c b/tests/tcg/mips/mips64-dspr2/shrav_r_qb.c
new file mode 100644
index 000000000..009080b2a
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/shrav_r_qb.c
@@ -0,0 +1,37 @@
+#include"io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long result;
+
+    rs = 0x03;
+    rt = 0x12345678;
+    result = 0x02070B0F;
+
+    __asm
+        ("shrav_r.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rt), "r"(rs)
+        );
+    if (rd != result) {
+        printf("shrav_r.qb error\n");
+        return -1;
+    }
+
+    rs = 0x03;
+    rt = 0x87654321;
+    result = 0xFFFFFFFFF10D0804;
+
+    __asm
+        ("shrav_r.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rt), "r"(rs)
+        );
+    if (rd != result) {
+        printf("shrav_r.qb error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/shrl_ph.c b/tests/tcg/mips/mips64-dspr2/shrl_ph.c
new file mode 100644
index 000000000..e32d97662
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/shrl_ph.c
@@ -0,0 +1,22 @@
+#include"io.h"
+
+int main(void)
+{
+    long long rd, rt;
+    long long result;
+
+    rt     = 0x12345678;
+    result = 0x009102B3;
+
+    __asm
+        ("shrl.ph %0, %1, 0x05\n\t"
+         : "=r"(rd)
+         : "r"(rt)
+        );
+    if (rd != result) {
+        printf("shrl.ph error!\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/shrlv_ph.c b/tests/tcg/mips/mips64-dspr2/shrlv_ph.c
new file mode 100644
index 000000000..58c5488b5
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/shrlv_ph.c
@@ -0,0 +1,23 @@
+#include"io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long result;
+
+    rs     = 0x05;
+    rt     = 0x12345678;
+    result = 0x009102B3;
+
+    __asm
+        ("shrlv.ph %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rt), "r"(rs)
+        );
+    if (rd != result) {
+        printf("shrlv.ph error!\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/subqh_ph.c b/tests/tcg/mips/mips64-dspr2/subqh_ph.c
new file mode 100644
index 000000000..90374019a
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/subqh_ph.c
@@ -0,0 +1,23 @@
+#include"io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long result;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result = 0x456709AB;
+
+    __asm
+        ("subqh.ph %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    if (rd != result) {
+        printf("subqh.ph error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/subqh_r_ph.c b/tests/tcg/mips/mips64-dspr2/subqh_r_ph.c
new file mode 100644
index 000000000..b8f9d2fee
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/subqh_r_ph.c
@@ -0,0 +1,23 @@
+#include"io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long result;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result = 0x456809AC;
+
+    __asm
+        ("subqh_r.ph %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    if (rd != result) {
+        printf("subqh_r.ph error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/subqh_r_w.c b/tests/tcg/mips/mips64-dspr2/subqh_r_w.c
new file mode 100644
index 000000000..b025e40a3
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/subqh_r_w.c
@@ -0,0 +1,23 @@
+#include"io.h"
+
+int main()
+{
+    long long rd, rs, rt;
+    long long result;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result = 0x456789AC;
+
+    __asm
+        ("subqh_r.w %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    if (rd != result) {
+        printf("subqh_r.w error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/subqh_w.c b/tests/tcg/mips/mips64-dspr2/subqh_w.c
new file mode 100644
index 000000000..65f17603d
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/subqh_w.c
@@ -0,0 +1,23 @@
+#include"io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long result;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result = 0x456789AB;
+
+    __asm
+        ("subqh.w %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    if (rd != result) {
+        printf("subqh.w error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/subu_ph.c b/tests/tcg/mips/mips64-dspr2/subu_ph.c
new file mode 100644
index 000000000..60a6b1b7d
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/subu_ph.c
@@ -0,0 +1,26 @@
+#include"io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, dsp;
+    long long result, resultdsp;
+
+    rs = 0x87654321;
+    rt = 0x12345678;
+    result    = 0x7531ECA9;
+    resultdsp = 0x01;
+
+    __asm
+        ("subu.ph %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 20) & 0x01;
+    if (dsp != resultdsp || rd  != result) {
+        printf("subu.ph error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/subu_qh.c b/tests/tcg/mips/mips64-dspr2/subu_qh.c
new file mode 100644
index 000000000..911cb349d
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/subu_qh.c
@@ -0,0 +1,24 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, dspreg, result, dspresult;
+    rs = 0x123456789ABCDEF0;
+    rt = 0x123456789ABCDEF1;
+    result = 0x000000000000000F;
+    dspresult = 0x01;
+
+    __asm("subu.qh %0, %2, %3\n\t"
+          "rddsp %1\n\t"
+          : "=r"(rd), "=r"(dspreg)
+          : "r"(rs), "r"(rt)
+         );
+
+    dspreg = ((dspreg >> 20) & 0x01);
+    if ((rd != result) || (dspreg != dspresult)) {
+        printf("subu.qh error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/subu_s_ph.c b/tests/tcg/mips/mips64-dspr2/subu_s_ph.c
new file mode 100644
index 000000000..ae32cc06f
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/subu_s_ph.c
@@ -0,0 +1,25 @@
+#include"io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, dsp;
+    long long result, resultdsp;
+
+    rs = 0x87654321;
+    rt = 0x12345678;
+    result    = 0x75310000;
+    resultdsp = 0x01;
+
+    __asm
+        ("subu_s.ph %0, %2, %3\n\t"
+         "rddsp %1\n\t"
+         : "=r"(rd), "=r"(dsp)
+         : "r"(rs), "r"(rt)
+        );
+    dsp = (dsp >> 20) & 0x01;
+    if (dsp != resultdsp || rd  != result) {
+        printf("subu_s.ph error\n");
+        return -1;
+    }
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/subu_s_qh.c b/tests/tcg/mips/mips64-dspr2/subu_s_qh.c
new file mode 100644
index 000000000..de7a29e77
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/subu_s_qh.c
@@ -0,0 +1,42 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, dspreg, result, dspresult;
+    rs = 0x1111111111111111;
+    rt = 0x2222222222222222;
+    result = 0x1111111111111111;
+    dspresult = 0x00;
+
+    __asm("subu_s.qh %0, %2, %3\n\t"
+          "rddsp %1\n\t"
+          : "=r"(rd), "=r"(dspreg)
+          : "r"(rs), "r"(rt)
+         );
+
+    dspreg = ((dspreg >> 20) & 0x01);
+    if ((rd != result) || (dspreg != dspresult)) {
+        printf("subu_s.qh error\n\t");
+        return -1;
+    }
+
+
+    rs = 0x8888888888888888;
+    rt = 0xa888a888a888a888;
+    result = 0x0000000000000000;
+    dspresult = 0x01;
+
+    __asm("subu_s.qh %0, %2, %3\n\t"
+          "rddsp %1\n\t"
+          : "=r"(rd), "=r"(dspreg)
+          : "r"(rs), "r"(rt)
+         );
+
+    dspreg = ((dspreg >> 20) & 0x01);
+    if ((rd != result) || (dspreg != dspresult)) {
+        printf("subu_s.qh error\n\t");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/subuh_ob.c b/tests/tcg/mips/mips64-dspr2/subuh_ob.c
new file mode 100644
index 000000000..3fc452bf8
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/subuh_ob.c
@@ -0,0 +1,36 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, result;
+
+    rd = 0x0;
+    rs = 0x246856789ABCDEF0;
+    rt = 0x123456789ABCDEF0;
+    result = 0x091A000000000000;
+
+    __asm("subuh.ob %0, %1, %2\n\t"
+          : "=r"(rd)
+          : "r"(rs), "r"(rt)
+         );
+
+    if (rd != result) {
+        printf("subuh.ob error\n");
+        return -1;
+    }
+
+    rs = 0x246856789ABCDEF0;
+    rt = 0x1131517191B1D1F1;
+    result = 0x1b4f2d2d51637577;
+
+    __asm("subuh.ob %0, %1, %2\n\t"
+          : "=r"(rd)
+          : "r"(rs), "r"(rt)
+         );
+
+    if (rd != result) {
+        printf("subuh.ob error\n");
+        return -1;
+    }
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/subuh_qb.c b/tests/tcg/mips/mips64-dspr2/subuh_qb.c
new file mode 100644
index 000000000..aac7a834e
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/subuh_qb.c
@@ -0,0 +1,23 @@
+#include"io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long result;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result = 0xC5E7092B;
+
+    __asm
+        ("subuh.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    if (rd != result) {
+        printf("subuh.qb wrong\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/subuh_r_ob.c b/tests/tcg/mips/mips64-dspr2/subuh_r_ob.c
new file mode 100644
index 000000000..fc20ffd09
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/subuh_r_ob.c
@@ -0,0 +1,23 @@
+#include "io.h"
+
+int main(void)
+{
+    long long rd, rs, rt, result;
+
+    rd = 0x0;
+    rs = 0x246956789ABCDEF0;
+    rt = 0x123456789ABCDEF0;
+    result = 0x091B000000000000;
+
+    __asm("subuh.ob %0, %1, %2\n\t"
+          : "=r"(rd)
+          : "r"(rs), "r"(rt)
+         );
+
+    if (rd != result) {
+        printf("subuh.ob error\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/mips/mips64-dspr2/subuh_r_qb.c b/tests/tcg/mips/mips64-dspr2/subuh_r_qb.c
new file mode 100644
index 000000000..66d468044
--- /dev/null
+++ b/tests/tcg/mips/mips64-dspr2/subuh_r_qb.c
@@ -0,0 +1,37 @@
+#include"io.h"
+
+int main(void)
+{
+    long long rd, rs, rt;
+    long long result;
+
+    rs = 0x12345678;
+    rt = 0x87654321;
+    result = 0xC6E80A2C;
+
+    __asm
+        ("subuh_r.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    if (rd != result) {
+        printf("1 subuh_r.qb wrong\n");
+        return -1;
+    }
+
+    rs = 0xBEFC292A;
+    rt = 0x9205C1B4;
+    result = 0x167cb4bb;
+
+    __asm
+        ("subuh_r.qb %0, %1, %2\n\t"
+         : "=r"(rd)
+         : "r"(rs), "r"(rt)
+        );
+    if (rd != result) {
+        printf("2 subuh_r.qb wrong\n");
+        return -1;
+    }
+
+    return 0;
+}
diff --git a/tests/tcg/test-i386-fprem.c b/tests/tcg/test-i386-fprem.c
new file mode 100644
index 000000000..8c7a4d1ff
--- /dev/null
+++ b/tests/tcg/test-i386-fprem.c
@@ -0,0 +1,353 @@
+/*
+ *  x86 FPREM test - executes the FPREM and FPREM1 instructions with corner case
+ *  operands and prints the operands, result and FPU status word.
+ *
+ *  Run this on real hardware, then under QEMU, and diff the outputs, to compare
+ *  QEMU's implementation to your hardware. The 'run-test-i386-fprem' make
+ *  target does this.
+ *
+ *  Copyright (c) 2003 Fabrice Bellard
+ *  Copyright (c) 2012 Catalin Patulea
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, see <http://www.gnu.org/licenses/>.
+ */
+#include "compiler.h"
+#include "osdep.h"
+#include <stdio.h>
+#include <inttypes.h>
+
+/*
+ * Inspired by <ieee754.h>'s union ieee854_long_double, but with single
+ * long long mantissa fields and assuming little-endianness for simplicity.
+ */
+union float80u {
+    long double d;
+
+    /* This is the IEEE 854 double-extended-precision format.  */
+    struct {
+        unsigned long long mantissa:63;
+        unsigned int one:1;
+        unsigned int exponent:15;
+        unsigned int negative:1;
+        unsigned int empty:16;
+    } QEMU_PACKED ieee;
+
+    /* This is for NaNs in the IEEE 854 double-extended-precision format.  */
+    struct {
+        unsigned long long mantissa:62;
+        unsigned int quiet_nan:1;
+        unsigned int one:1;
+        unsigned int exponent:15;
+        unsigned int negative:1;
+        unsigned int empty:16;
+    } QEMU_PACKED ieee_nan;
+};
+
+#define IEEE854_LONG_DOUBLE_BIAS 0x3fff
+
+static const union float80u q_nan = {
+    .ieee_nan.negative = 0,  /* X */
+    .ieee_nan.exponent = 0x7fff,
+    .ieee_nan.one = 1,
+    .ieee_nan.quiet_nan = 1,
+    .ieee_nan.mantissa = 0,
+};
+
+static const union float80u s_nan = {
+    .ieee_nan.negative = 0,  /* X */
+    .ieee_nan.exponent = 0x7fff,
+    .ieee_nan.one = 1,
+    .ieee_nan.quiet_nan = 0,
+    .ieee_nan.mantissa = 1,  /* nonzero */
+};
+
+static const union float80u pos_inf = {
+    .ieee.negative = 0,
+    .ieee.exponent = 0x7fff,
+    .ieee.one = 1,
+    .ieee.mantissa = 0,
+};
+
+static const union float80u pseudo_pos_inf = {  /* "unsupported" */
+    .ieee.negative = 0,
+    .ieee.exponent = 0x7fff,
+    .ieee.one = 0,
+    .ieee.mantissa = 0,
+};
+
+static const union float80u pos_denorm = {
+    .ieee.negative = 0,
+    .ieee.exponent = 0,
+    .ieee.one = 0,
+    .ieee.mantissa = 1,
+};
+
+static const union float80u smallest_positive_norm = {
+    .ieee.negative = 0,
+    .ieee.exponent = 1,
+    .ieee.one = 1,
+    .ieee.mantissa = 0,
+};
+
+static void fninit()
+{
+    asm volatile ("fninit\n");
+}
+
+static long double fprem(long double a, long double b, uint16_t *sw)
+{
+    long double result;
+    asm volatile ("fprem\n"
+                  "fnstsw %1\n"
+                  : "=t" (result), "=m" (*sw)
+                  : "0" (a), "u" (b)
+                  : "st(1)");
+    return result;
+}
+
+static long double fprem1(long double a, long double b, uint16_t *sw)
+{
+    long double result;
+    asm volatile ("fprem1\n"
+                  "fnstsw %1\n"
+                  : "=t" (result), "=m" (*sw)
+                  : "0" (a), "u" (b)
+                  : "st(1)");
+    return result;
+}
+
+#define FPUS_IE (1 << 0)
+#define FPUS_DE (1 << 1)
+#define FPUS_ZE (1 << 2)
+#define FPUS_OE (1 << 3)
+#define FPUS_UE (1 << 4)
+#define FPUS_PE (1 << 5)
+#define FPUS_SF (1 << 6)
+#define FPUS_SE (1 << 7)
+#define FPUS_C0 (1 << 8)
+#define FPUS_C1 (1 << 9)
+#define FPUS_C2 (1 << 10)
+#define FPUS_TOP 0x3800
+#define FPUS_C3 (1 << 14)
+#define FPUS_B  (1 << 15)
+
+#define FPUS_EMASK 0x007f
+
+#define FPUC_EM 0x3f
+
+static void psw(uint16_t sw)
+{
+    printf("SW:  C3 TopC2C1C0\n");
+    printf("SW: %c %d %3d %d %d %d %c %c %c %c %c %c %c %c\n",
+           sw & FPUS_B ? 'B' : 'b',
+           !!(sw & FPUS_C3),
+           (sw & FPUS_TOP) >> 11,
+           !!(sw & FPUS_C2),
+           !!(sw & FPUS_C1),
+           !!(sw & FPUS_C0),
+           (sw & FPUS_SE) ? 'S' : 's',
+           (sw & FPUS_SF) ? 'F' : 'f',
+           (sw & FPUS_PE) ? 'P' : 'p',
+           (sw & FPUS_UE) ? 'U' : 'u',
+           (sw & FPUS_OE) ? 'O' : 'o',
+           (sw & FPUS_ZE) ? 'Z' : 'z',
+           (sw & FPUS_DE) ? 'D' : 'd',
+           (sw & FPUS_IE) ? 'I' : 'i');
+}
+
+static void do_fprem(long double a, long double b)
+{
+    const union float80u au = {.d = a};
+    const union float80u bu = {.d = b};
+    union float80u ru;
+    uint16_t sw;
+
+    printf("A: S=%d Exp=%04x Int=%d (QNaN=%d) Sig=%016llx (%.06Le)\n",
+           au.ieee.negative, au.ieee.exponent, au.ieee.one,
+           au.ieee_nan.quiet_nan, (unsigned long long)au.ieee.mantissa,
+           a);
+    printf("B: S=%d Exp=%04x Int=%d (QNaN=%d) Sig=%016llx (%.06Le)\n",
+           bu.ieee.negative, bu.ieee.exponent, bu.ieee.one,
+           bu.ieee_nan.quiet_nan, (unsigned long long)bu.ieee.mantissa,
+           b);
+    fflush(stdout);
+
+    fninit();
+    ru.d = fprem(a, b, &sw);
+    psw(sw);
+
+    printf("R : S=%d Exp=%04x Int=%d (QNaN=%d) Sig=%016llx (%.06Le)\n",
+           ru.ieee.negative, ru.ieee.exponent, ru.ieee.one,
+           ru.ieee_nan.quiet_nan, (unsigned long long)ru.ieee.mantissa,
+           ru.d);
+
+    fninit();
+    ru.d = fprem1(a, b, &sw);
+    psw(sw);
+
+    printf("R1: S=%d Exp=%04x Int=%d (QNaN=%d) Sig=%016llx (%.06Le)\n",
+           ru.ieee.negative, ru.ieee.exponent, ru.ieee.one,
+           ru.ieee_nan.quiet_nan, (unsigned long long)ru.ieee.mantissa,
+           ru.d);
+
+    printf("\n");
+}
+
+static void do_fprem_stack_underflow(void)
+{
+    const long double a = 1.0;
+    union float80u ru;
+    uint16_t sw;
+
+    fninit();
+    asm volatile ("fprem\n"
+                  "fnstsw %1\n"
+                  : "=t" (ru.d), "=m" (sw)
+                  : "0" (a)
+                  : "st(1)");
+    psw(sw);
+
+    printf("R: S=%d Exp=%04x Int=%d (QNaN=%d) Sig=%016llx (%.06Le)\n",
+           ru.ieee.negative, ru.ieee.exponent, ru.ieee.one,
+           ru.ieee_nan.quiet_nan, (unsigned long long)ru.ieee.mantissa,
+           ru.d);
+    printf("\n");
+}
+
+static void test_fprem_cases(void)
+{
+    printf("= stack underflow =\n");
+    do_fprem_stack_underflow();
+
+    printf("= invalid operation =\n");
+    do_fprem(s_nan.d, 1.0);
+    do_fprem(1.0, 0.0);
+    do_fprem(pos_inf.d, 1.0);
+    do_fprem(pseudo_pos_inf.d, 1.0);
+
+    printf("= denormal =\n");
+    do_fprem(pos_denorm.d, 1.0);
+    do_fprem(1.0, pos_denorm.d);
+
+    /* printf("= underflow =\n"); */
+    /* TODO: Is there a case where FPREM raises underflow? */
+}
+
+static void test_fprem_pairs(void)
+{
+    unsigned long long count;
+
+    unsigned int negative_index_a = 0;
+    unsigned int negative_index_b = 0;
+    static const unsigned int negative_values[] = {
+        0,
+        1,
+    };
+
+    unsigned int exponent_index_a = 0;
+    unsigned int exponent_index_b = 0;
+    static const unsigned int exponent_values[] = {
+        0,
+        1,
+        2,
+        IEEE854_LONG_DOUBLE_BIAS - 1,
+        IEEE854_LONG_DOUBLE_BIAS,
+        IEEE854_LONG_DOUBLE_BIAS + 1,
+        0x7ffd,
+        0x7ffe,
+        0x7fff,
+    };
+
+    unsigned int one_index_a = 0;
+    unsigned int one_index_b = 0;
+    static const unsigned int one_values[] = {
+        0,
+        1,
+    };
+
+    unsigned int quiet_nan_index_a = 0;
+    unsigned int quiet_nan_index_b = 0;
+    static const unsigned int quiet_nan_values[] = {
+        0,
+        1,
+    };
+
+    unsigned int mantissa_index_a = 0;
+    unsigned int mantissa_index_b = 0;
+    static const unsigned long long mantissa_values[] = {
+        0,
+        1,
+        2,
+        0x3ffffffffffffffdULL,
+        0x3ffffffffffffffeULL,
+        0x3fffffffffffffffULL,
+    };
+
+    for (count = 0; ; ++count) {
+#define INIT_FIELD(var, field) \
+            .ieee_nan.field = field##_values[field##_index_##var]
+        const union float80u a = {
+            INIT_FIELD(a, negative),
+            INIT_FIELD(a, exponent),
+            INIT_FIELD(a, one),
+            INIT_FIELD(a, quiet_nan),
+            INIT_FIELD(a, mantissa),
+        };
+        const union float80u b = {
+            INIT_FIELD(b, negative),
+            INIT_FIELD(b, exponent),
+            INIT_FIELD(b, one),
+            INIT_FIELD(b, quiet_nan),
+            INIT_FIELD(b, mantissa),
+        };
+#undef INIT_FIELD
+
+        do_fprem(a.d, b.d);
+
+        int carry = 1;
+#define CARRY_INTO(var, field) do { \
+            if (carry) { \
+                if (++field##_index_##var == ARRAY_SIZE(field##_values)) { \
+                    field##_index_##var = 0; \
+                } else { \
+                    carry = 0; \
+                } \
+            } \
+        } while (0)
+        CARRY_INTO(b, mantissa);
+        CARRY_INTO(b, quiet_nan);
+        CARRY_INTO(b, one);
+        CARRY_INTO(b, exponent);
+        CARRY_INTO(b, negative);
+        CARRY_INTO(a, mantissa);
+        CARRY_INTO(a, quiet_nan);
+        CARRY_INTO(a, one);
+        CARRY_INTO(a, exponent);
+        CARRY_INTO(a, negative);
+#undef CARRY_INTO
+
+        if (carry) {
+            break;
+        }
+    }
+
+    fprintf(stderr, "test-i386-fprem: tested %llu cases\n", count);
+}
+
+int main(int argc, char **argv)
+{
+    test_fprem_cases();
+    test_fprem_pairs();
+    return 0;
+}
diff --git a/tests/tcg/test-i386.c b/tests/tcg/test-i386.c
index 8e64bbaf3..40392ac51 100644
--- a/tests/tcg/test-i386.c
+++ b/tests/tcg/test-i386.c
@@ -17,6 +17,7 @@
  *  along with this program; if not, see <http://www.gnu.org/licenses/>.
  */
 #define _GNU_SOURCE
+#include "compiler.h"
 #include <stdlib.h>
 #include <stdio.h>
 #include <string.h>
@@ -784,7 +785,7 @@ void fpu_clear_exceptions(void)
         long double fpregs[8];
     } float_env32;
 
-    asm volatile ("fnstenv %0\n" : : "m" (float_env32));
+    asm volatile ("fnstenv %0\n" : "=m" (float_env32));
     float_env32.fpus &= ~0x7f;
     asm volatile ("fldenv %0\n" : : "m" (float_env32));
 }
@@ -1827,7 +1828,7 @@ void test_exceptions(void)
     printf("lock nop exception:\n");
     if (setjmp(jmp_env) == 0) {
         /* now execute an invalid instruction */
-        asm volatile("lock nop");
+        asm volatile(".byte 0xf0, 0x90"); /* lock nop */
     }
 
     printf("INT exception:\n");
diff --git a/tests/tcg/test-mmap.c b/tests/tcg/test-mmap.c
index c418b6766..3982fa2c7 100644
--- a/tests/tcg/test-mmap.c
+++ b/tests/tcg/test-mmap.c
@@ -429,6 +429,12 @@ void check_file_fixed_mmaps(void)
 	fprintf (stderr, " passed\n");
 }
 
+void checked_write(int fd, const void *buf, size_t count)
+{
+    ssize_t rc = write(fd, buf, count);
+    fail_unless(rc == count);
+}
+
 int main(int argc, char **argv)
 {
 	char tempname[] = "/tmp/.cmmapXXXXXX";
@@ -450,13 +456,15 @@ int main(int argc, char **argv)
 	unlink(tempname);
 
 	/* Fill the file with int's counting from zero and up.  */
-	for (i = 0; i < (pagesize * 4) / sizeof i; i++)
-		write (test_fd, &i, sizeof i);
+    for (i = 0; i < (pagesize * 4) / sizeof i; i++) {
+        checked_write(test_fd, &i, sizeof i);
+    }
+
 	/* Append a few extra writes to make the file end at non 
 	   page boundary.  */
-	write (test_fd, &i, sizeof i); i++;
-	write (test_fd, &i, sizeof i); i++;
-	write (test_fd, &i, sizeof i); i++;
+    checked_write(test_fd, &i, sizeof i); i++;
+    checked_write(test_fd, &i, sizeof i); i++;
+    checked_write(test_fd, &i, sizeof i); i++;
 
 	test_fsize = lseek(test_fd, 0, SEEK_CUR);
 
diff --git a/tests/tcg/test_path.c b/tests/tcg/test_path.c
index 7265a9445..a064eea8f 100644
--- a/tests/tcg/test_path.c
+++ b/tests/tcg/test_path.c
@@ -1,11 +1,12 @@
 /* Test path override code */
-#include "../config-host.h"
-#include "../qemu-malloc.c"
-#include "../cutils.c"
-#include "../path.c"
-#include "../trace.c"
+#define _GNU_SOURCE
+#include "config-host.h"
+#include "iov.c"
+#include "cutils.c"
+#include "path.c"
+#include "trace.c"
 #ifdef CONFIG_TRACE_SIMPLE
-#include "../simpletrace.c"
+#include "../trace/simple.c"
 #endif
 
 #include <stdarg.h>
diff --git a/tests/tcg/testthread.c b/tests/tcg/testthread.c
index 27e4825bc..2679af119 100644
--- a/tests/tcg/testthread.c
+++ b/tests/tcg/testthread.c
@@ -1,3 +1,4 @@
+#include <assert.h>
 #include <stdlib.h>
 #include <stdio.h>
 #include <string.h>
@@ -8,6 +9,12 @@
 #include <sys/wait.h>
 #include <sched.h>
 
+void checked_write(int fd, const void *buf, size_t count)
+{
+    ssize_t rc = write(fd, buf, count);
+    assert(rc == count);
+}
+
 void *thread1_func(void *arg)
 {
     int i;
@@ -15,7 +22,7 @@ void *thread1_func(void *arg)
 
     for(i=0;i<10;i++) {
         snprintf(buf, sizeof(buf), "thread1: %d %s\n", i, (char *)arg);
-        write(1, buf, strlen(buf));
+        checked_write(1, buf, strlen(buf));
         usleep(100 * 1000);
     }
     return NULL;
@@ -27,7 +34,7 @@ void *thread2_func(void *arg)
     char buf[512];
     for(i=0;i<20;i++) {
         snprintf(buf, sizeof(buf), "thread2: %d %s\n", i, (char *)arg);
-        write(1, buf, strlen(buf));
+        checked_write(1, buf, strlen(buf));
         usleep(150 * 1000);
     }
     return NULL;
diff --git a/tests/test-aio.c b/tests/test-aio.c
new file mode 100644
index 000000000..f53c90870
--- /dev/null
+++ b/tests/test-aio.c
@@ -0,0 +1,667 @@
+/*
+ * AioContext tests
+ *
+ * Copyright Red Hat, Inc. 2012
+ *
+ * Authors:
+ *  Paolo Bonzini    <pbonzini@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU LGPL, version 2 or later.
+ * See the COPYING.LIB file in the top-level directory.
+ */
+
+#include <glib.h>
+#include "qemu-aio.h"
+
+AioContext *ctx;
+
+/* Simple callbacks for testing.  */
+
+typedef struct {
+    QEMUBH *bh;
+    int n;
+    int max;
+} BHTestData;
+
+static void bh_test_cb(void *opaque)
+{
+    BHTestData *data = opaque;
+    if (++data->n < data->max) {
+        qemu_bh_schedule(data->bh);
+    }
+}
+
+static void bh_delete_cb(void *opaque)
+{
+    BHTestData *data = opaque;
+    if (++data->n < data->max) {
+        qemu_bh_schedule(data->bh);
+    } else {
+        qemu_bh_delete(data->bh);
+        data->bh = NULL;
+    }
+}
+
+typedef struct {
+    EventNotifier e;
+    int n;
+    int active;
+    bool auto_set;
+} EventNotifierTestData;
+
+static int event_active_cb(EventNotifier *e)
+{
+    EventNotifierTestData *data = container_of(e, EventNotifierTestData, e);
+    return data->active > 0;
+}
+
+static void event_ready_cb(EventNotifier *e)
+{
+    EventNotifierTestData *data = container_of(e, EventNotifierTestData, e);
+    g_assert(event_notifier_test_and_clear(e));
+    data->n++;
+    if (data->active > 0) {
+        data->active--;
+    }
+    if (data->auto_set && data->active) {
+        event_notifier_set(e);
+    }
+}
+
+/* Tests using aio_*.  */
+
+static void test_notify(void)
+{
+    g_assert(!aio_poll(ctx, false));
+    aio_notify(ctx);
+    g_assert(!aio_poll(ctx, true));
+    g_assert(!aio_poll(ctx, false));
+}
+
+static void test_flush(void)
+{
+    g_assert(!aio_poll(ctx, false));
+    aio_notify(ctx);
+    aio_flush(ctx);
+    g_assert(!aio_poll(ctx, false));
+}
+
+static void test_bh_schedule(void)
+{
+    BHTestData data = { .n = 0 };
+    data.bh = aio_bh_new(ctx, bh_test_cb, &data);
+
+    qemu_bh_schedule(data.bh);
+    g_assert_cmpint(data.n, ==, 0);
+
+    g_assert(aio_poll(ctx, true));
+    g_assert_cmpint(data.n, ==, 1);
+
+    g_assert(!aio_poll(ctx, false));
+    g_assert_cmpint(data.n, ==, 1);
+    qemu_bh_delete(data.bh);
+}
+
+static void test_bh_schedule10(void)
+{
+    BHTestData data = { .n = 0, .max = 10 };
+    data.bh = aio_bh_new(ctx, bh_test_cb, &data);
+
+    qemu_bh_schedule(data.bh);
+    g_assert_cmpint(data.n, ==, 0);
+
+    g_assert(aio_poll(ctx, false));
+    g_assert_cmpint(data.n, ==, 1);
+
+    g_assert(aio_poll(ctx, true));
+    g_assert_cmpint(data.n, ==, 2);
+
+    aio_flush(ctx);
+    g_assert_cmpint(data.n, ==, 10);
+
+    g_assert(!aio_poll(ctx, false));
+    g_assert_cmpint(data.n, ==, 10);
+    qemu_bh_delete(data.bh);
+}
+
+static void test_bh_cancel(void)
+{
+    BHTestData data = { .n = 0 };
+    data.bh = aio_bh_new(ctx, bh_test_cb, &data);
+
+    qemu_bh_schedule(data.bh);
+    g_assert_cmpint(data.n, ==, 0);
+
+    qemu_bh_cancel(data.bh);
+    g_assert_cmpint(data.n, ==, 0);
+
+    g_assert(!aio_poll(ctx, false));
+    g_assert_cmpint(data.n, ==, 0);
+    qemu_bh_delete(data.bh);
+}
+
+static void test_bh_delete(void)
+{
+    BHTestData data = { .n = 0 };
+    data.bh = aio_bh_new(ctx, bh_test_cb, &data);
+
+    qemu_bh_schedule(data.bh);
+    g_assert_cmpint(data.n, ==, 0);
+
+    qemu_bh_delete(data.bh);
+    g_assert_cmpint(data.n, ==, 0);
+
+    g_assert(!aio_poll(ctx, false));
+    g_assert_cmpint(data.n, ==, 0);
+}
+
+static void test_bh_delete_from_cb(void)
+{
+    BHTestData data1 = { .n = 0, .max = 1 };
+
+    data1.bh = aio_bh_new(ctx, bh_delete_cb, &data1);
+
+    qemu_bh_schedule(data1.bh);
+    g_assert_cmpint(data1.n, ==, 0);
+
+    aio_flush(ctx);
+    g_assert_cmpint(data1.n, ==, data1.max);
+    g_assert(data1.bh == NULL);
+
+    g_assert(!aio_poll(ctx, false));
+    g_assert(!aio_poll(ctx, true));
+}
+
+static void test_bh_delete_from_cb_many(void)
+{
+    BHTestData data1 = { .n = 0, .max = 1 };
+    BHTestData data2 = { .n = 0, .max = 3 };
+    BHTestData data3 = { .n = 0, .max = 2 };
+    BHTestData data4 = { .n = 0, .max = 4 };
+
+    data1.bh = aio_bh_new(ctx, bh_delete_cb, &data1);
+    data2.bh = aio_bh_new(ctx, bh_delete_cb, &data2);
+    data3.bh = aio_bh_new(ctx, bh_delete_cb, &data3);
+    data4.bh = aio_bh_new(ctx, bh_delete_cb, &data4);
+
+    qemu_bh_schedule(data1.bh);
+    qemu_bh_schedule(data2.bh);
+    qemu_bh_schedule(data3.bh);
+    qemu_bh_schedule(data4.bh);
+    g_assert_cmpint(data1.n, ==, 0);
+    g_assert_cmpint(data2.n, ==, 0);
+    g_assert_cmpint(data3.n, ==, 0);
+    g_assert_cmpint(data4.n, ==, 0);
+
+    g_assert(aio_poll(ctx, false));
+    g_assert_cmpint(data1.n, ==, 1);
+    g_assert_cmpint(data2.n, ==, 1);
+    g_assert_cmpint(data3.n, ==, 1);
+    g_assert_cmpint(data4.n, ==, 1);
+    g_assert(data1.bh == NULL);
+
+    aio_flush(ctx);
+    g_assert_cmpint(data1.n, ==, data1.max);
+    g_assert_cmpint(data2.n, ==, data2.max);
+    g_assert_cmpint(data3.n, ==, data3.max);
+    g_assert_cmpint(data4.n, ==, data4.max);
+    g_assert(data1.bh == NULL);
+    g_assert(data2.bh == NULL);
+    g_assert(data3.bh == NULL);
+    g_assert(data4.bh == NULL);
+}
+
+static void test_bh_flush(void)
+{
+    BHTestData data = { .n = 0 };
+    data.bh = aio_bh_new(ctx, bh_test_cb, &data);
+
+    qemu_bh_schedule(data.bh);
+    g_assert_cmpint(data.n, ==, 0);
+
+    aio_flush(ctx);
+    g_assert_cmpint(data.n, ==, 1);
+
+    g_assert(!aio_poll(ctx, false));
+    g_assert_cmpint(data.n, ==, 1);
+    qemu_bh_delete(data.bh);
+}
+
+static void test_set_event_notifier(void)
+{
+    EventNotifierTestData data = { .n = 0, .active = 0 };
+    event_notifier_init(&data.e, false);
+    aio_set_event_notifier(ctx, &data.e, event_ready_cb, event_active_cb);
+    g_assert(!aio_poll(ctx, false));
+    g_assert_cmpint(data.n, ==, 0);
+
+    aio_set_event_notifier(ctx, &data.e, NULL, NULL);
+    g_assert(!aio_poll(ctx, false));
+    g_assert_cmpint(data.n, ==, 0);
+    event_notifier_cleanup(&data.e);
+}
+
+static void test_wait_event_notifier(void)
+{
+    EventNotifierTestData data = { .n = 0, .active = 1 };
+    event_notifier_init(&data.e, false);
+    aio_set_event_notifier(ctx, &data.e, event_ready_cb, event_active_cb);
+    g_assert(aio_poll(ctx, false));
+    g_assert_cmpint(data.n, ==, 0);
+    g_assert_cmpint(data.active, ==, 1);
+
+    event_notifier_set(&data.e);
+    g_assert(aio_poll(ctx, false));
+    g_assert_cmpint(data.n, ==, 1);
+    g_assert_cmpint(data.active, ==, 0);
+
+    g_assert(!aio_poll(ctx, false));
+    g_assert_cmpint(data.n, ==, 1);
+    g_assert_cmpint(data.active, ==, 0);
+
+    aio_set_event_notifier(ctx, &data.e, NULL, NULL);
+    g_assert(!aio_poll(ctx, false));
+    g_assert_cmpint(data.n, ==, 1);
+
+    event_notifier_cleanup(&data.e);
+}
+
+static void test_flush_event_notifier(void)
+{
+    EventNotifierTestData data = { .n = 0, .active = 10, .auto_set = true };
+    event_notifier_init(&data.e, false);
+    aio_set_event_notifier(ctx, &data.e, event_ready_cb, event_active_cb);
+    g_assert(aio_poll(ctx, false));
+    g_assert_cmpint(data.n, ==, 0);
+    g_assert_cmpint(data.active, ==, 10);
+
+    event_notifier_set(&data.e);
+    g_assert(aio_poll(ctx, false));
+    g_assert_cmpint(data.n, ==, 1);
+    g_assert_cmpint(data.active, ==, 9);
+    g_assert(aio_poll(ctx, false));
+
+    aio_flush(ctx);
+    g_assert_cmpint(data.n, ==, 10);
+    g_assert_cmpint(data.active, ==, 0);
+    g_assert(!aio_poll(ctx, false));
+
+    aio_set_event_notifier(ctx, &data.e, NULL, NULL);
+    g_assert(!aio_poll(ctx, false));
+    event_notifier_cleanup(&data.e);
+}
+
+static void test_wait_event_notifier_noflush(void)
+{
+    EventNotifierTestData data = { .n = 0 };
+    EventNotifierTestData dummy = { .n = 0, .active = 1 };
+
+    event_notifier_init(&data.e, false);
+    aio_set_event_notifier(ctx, &data.e, event_ready_cb, NULL);
+
+    g_assert(!aio_poll(ctx, false));
+    g_assert_cmpint(data.n, ==, 0);
+
+    /* Until there is an active descriptor, aio_poll may or may not call
+     * event_ready_cb.  Still, it must not block.  */
+    event_notifier_set(&data.e);
+    g_assert(!aio_poll(ctx, true));
+    data.n = 0;
+
+    /* An active event notifier forces aio_poll to look at EventNotifiers.  */
+    event_notifier_init(&dummy.e, false);
+    aio_set_event_notifier(ctx, &dummy.e, event_ready_cb, event_active_cb);
+
+    event_notifier_set(&data.e);
+    g_assert(aio_poll(ctx, false));
+    g_assert_cmpint(data.n, ==, 1);
+    g_assert(!aio_poll(ctx, false));
+    g_assert_cmpint(data.n, ==, 1);
+
+    event_notifier_set(&data.e);
+    g_assert(aio_poll(ctx, false));
+    g_assert_cmpint(data.n, ==, 2);
+    g_assert(!aio_poll(ctx, false));
+    g_assert_cmpint(data.n, ==, 2);
+
+    event_notifier_set(&dummy.e);
+    aio_flush(ctx);
+    g_assert_cmpint(data.n, ==, 2);
+    g_assert_cmpint(dummy.n, ==, 1);
+    g_assert_cmpint(dummy.active, ==, 0);
+
+    aio_set_event_notifier(ctx, &dummy.e, NULL, NULL);
+    event_notifier_cleanup(&dummy.e);
+
+    aio_set_event_notifier(ctx, &data.e, NULL, NULL);
+    g_assert(!aio_poll(ctx, false));
+    g_assert_cmpint(data.n, ==, 2);
+
+    event_notifier_cleanup(&data.e);
+}
+
+/* Now the same tests, using the context as a GSource.  They are
+ * very similar to the ones above, with g_main_context_iteration
+ * replacing aio_poll.  However:
+ * - sometimes both the AioContext and the glib main loop wake
+ *   themselves up.  Hence, some "g_assert(!aio_poll(ctx, false));"
+ *   are replaced by "while (g_main_context_iteration(NULL, false));".
+ * - there is no exact replacement for aio_flush's blocking wait.
+ *   "while (g_main_context_iteration(NULL, true)" seems to work,
+ *   but it is not documented _why_ it works.  For these tests a
+ *   non-blocking loop like "while (g_main_context_iteration(NULL, false)"
+ *   works well, and that's what I am using.
+ */
+
+static void test_source_notify(void)
+{
+    while (g_main_context_iteration(NULL, false));
+    aio_notify(ctx);
+    g_assert(g_main_context_iteration(NULL, true));
+    g_assert(!g_main_context_iteration(NULL, false));
+}
+
+static void test_source_flush(void)
+{
+    g_assert(!g_main_context_iteration(NULL, false));
+    aio_notify(ctx);
+    while (g_main_context_iteration(NULL, false));
+    g_assert(!g_main_context_iteration(NULL, false));
+}
+
+static void test_source_bh_schedule(void)
+{
+    BHTestData data = { .n = 0 };
+    data.bh = aio_bh_new(ctx, bh_test_cb, &data);
+
+    qemu_bh_schedule(data.bh);
+    g_assert_cmpint(data.n, ==, 0);
+
+    g_assert(g_main_context_iteration(NULL, true));
+    g_assert_cmpint(data.n, ==, 1);
+
+    g_assert(!g_main_context_iteration(NULL, false));
+    g_assert_cmpint(data.n, ==, 1);
+    qemu_bh_delete(data.bh);
+}
+
+static void test_source_bh_schedule10(void)
+{
+    BHTestData data = { .n = 0, .max = 10 };
+    data.bh = aio_bh_new(ctx, bh_test_cb, &data);
+
+    qemu_bh_schedule(data.bh);
+    g_assert_cmpint(data.n, ==, 0);
+
+    g_assert(g_main_context_iteration(NULL, false));
+    g_assert_cmpint(data.n, ==, 1);
+
+    g_assert(g_main_context_iteration(NULL, true));
+    g_assert_cmpint(data.n, ==, 2);
+
+    while (g_main_context_iteration(NULL, false));
+    g_assert_cmpint(data.n, ==, 10);
+
+    g_assert(!g_main_context_iteration(NULL, false));
+    g_assert_cmpint(data.n, ==, 10);
+    qemu_bh_delete(data.bh);
+}
+
+static void test_source_bh_cancel(void)
+{
+    BHTestData data = { .n = 0 };
+    data.bh = aio_bh_new(ctx, bh_test_cb, &data);
+
+    qemu_bh_schedule(data.bh);
+    g_assert_cmpint(data.n, ==, 0);
+
+    qemu_bh_cancel(data.bh);
+    g_assert_cmpint(data.n, ==, 0);
+
+    while (g_main_context_iteration(NULL, false));
+    g_assert_cmpint(data.n, ==, 0);
+    qemu_bh_delete(data.bh);
+}
+
+static void test_source_bh_delete(void)
+{
+    BHTestData data = { .n = 0 };
+    data.bh = aio_bh_new(ctx, bh_test_cb, &data);
+
+    qemu_bh_schedule(data.bh);
+    g_assert_cmpint(data.n, ==, 0);
+
+    qemu_bh_delete(data.bh);
+    g_assert_cmpint(data.n, ==, 0);
+
+    while (g_main_context_iteration(NULL, false));
+    g_assert_cmpint(data.n, ==, 0);
+}
+
+static void test_source_bh_delete_from_cb(void)
+{
+    BHTestData data1 = { .n = 0, .max = 1 };
+
+    data1.bh = aio_bh_new(ctx, bh_delete_cb, &data1);
+
+    qemu_bh_schedule(data1.bh);
+    g_assert_cmpint(data1.n, ==, 0);
+
+    g_main_context_iteration(NULL, true);
+    g_assert_cmpint(data1.n, ==, data1.max);
+    g_assert(data1.bh == NULL);
+
+    g_assert(!g_main_context_iteration(NULL, false));
+}
+
+static void test_source_bh_delete_from_cb_many(void)
+{
+    BHTestData data1 = { .n = 0, .max = 1 };
+    BHTestData data2 = { .n = 0, .max = 3 };
+    BHTestData data3 = { .n = 0, .max = 2 };
+    BHTestData data4 = { .n = 0, .max = 4 };
+
+    data1.bh = aio_bh_new(ctx, bh_delete_cb, &data1);
+    data2.bh = aio_bh_new(ctx, bh_delete_cb, &data2);
+    data3.bh = aio_bh_new(ctx, bh_delete_cb, &data3);
+    data4.bh = aio_bh_new(ctx, bh_delete_cb, &data4);
+
+    qemu_bh_schedule(data1.bh);
+    qemu_bh_schedule(data2.bh);
+    qemu_bh_schedule(data3.bh);
+    qemu_bh_schedule(data4.bh);
+    g_assert_cmpint(data1.n, ==, 0);
+    g_assert_cmpint(data2.n, ==, 0);
+    g_assert_cmpint(data3.n, ==, 0);
+    g_assert_cmpint(data4.n, ==, 0);
+
+    g_assert(g_main_context_iteration(NULL, false));
+    g_assert_cmpint(data1.n, ==, 1);
+    g_assert_cmpint(data2.n, ==, 1);
+    g_assert_cmpint(data3.n, ==, 1);
+    g_assert_cmpint(data4.n, ==, 1);
+    g_assert(data1.bh == NULL);
+
+    while (g_main_context_iteration(NULL, false));
+    g_assert_cmpint(data1.n, ==, data1.max);
+    g_assert_cmpint(data2.n, ==, data2.max);
+    g_assert_cmpint(data3.n, ==, data3.max);
+    g_assert_cmpint(data4.n, ==, data4.max);
+    g_assert(data1.bh == NULL);
+    g_assert(data2.bh == NULL);
+    g_assert(data3.bh == NULL);
+    g_assert(data4.bh == NULL);
+}
+
+static void test_source_bh_flush(void)
+{
+    BHTestData data = { .n = 0 };
+    data.bh = aio_bh_new(ctx, bh_test_cb, &data);
+
+    qemu_bh_schedule(data.bh);
+    g_assert_cmpint(data.n, ==, 0);
+
+    g_assert(g_main_context_iteration(NULL, true));
+    g_assert_cmpint(data.n, ==, 1);
+
+    g_assert(!g_main_context_iteration(NULL, false));
+    g_assert_cmpint(data.n, ==, 1);
+    qemu_bh_delete(data.bh);
+}
+
+static void test_source_set_event_notifier(void)
+{
+    EventNotifierTestData data = { .n = 0, .active = 0 };
+    event_notifier_init(&data.e, false);
+    aio_set_event_notifier(ctx, &data.e, event_ready_cb, event_active_cb);
+    while (g_main_context_iteration(NULL, false));
+    g_assert_cmpint(data.n, ==, 0);
+
+    aio_set_event_notifier(ctx, &data.e, NULL, NULL);
+    while (g_main_context_iteration(NULL, false));
+    g_assert_cmpint(data.n, ==, 0);
+    event_notifier_cleanup(&data.e);
+}
+
+static void test_source_wait_event_notifier(void)
+{
+    EventNotifierTestData data = { .n = 0, .active = 1 };
+    event_notifier_init(&data.e, false);
+    aio_set_event_notifier(ctx, &data.e, event_ready_cb, event_active_cb);
+    g_assert(g_main_context_iteration(NULL, false));
+    g_assert_cmpint(data.n, ==, 0);
+    g_assert_cmpint(data.active, ==, 1);
+
+    event_notifier_set(&data.e);
+    g_assert(g_main_context_iteration(NULL, false));
+    g_assert_cmpint(data.n, ==, 1);
+    g_assert_cmpint(data.active, ==, 0);
+
+    while (g_main_context_iteration(NULL, false));
+    g_assert_cmpint(data.n, ==, 1);
+    g_assert_cmpint(data.active, ==, 0);
+
+    aio_set_event_notifier(ctx, &data.e, NULL, NULL);
+    while (g_main_context_iteration(NULL, false));
+    g_assert_cmpint(data.n, ==, 1);
+
+    event_notifier_cleanup(&data.e);
+}
+
+static void test_source_flush_event_notifier(void)
+{
+    EventNotifierTestData data = { .n = 0, .active = 10, .auto_set = true };
+    event_notifier_init(&data.e, false);
+    aio_set_event_notifier(ctx, &data.e, event_ready_cb, event_active_cb);
+    g_assert(g_main_context_iteration(NULL, false));
+    g_assert_cmpint(data.n, ==, 0);
+    g_assert_cmpint(data.active, ==, 10);
+
+    event_notifier_set(&data.e);
+    g_assert(g_main_context_iteration(NULL, false));
+    g_assert_cmpint(data.n, ==, 1);
+    g_assert_cmpint(data.active, ==, 9);
+    g_assert(g_main_context_iteration(NULL, false));
+
+    while (g_main_context_iteration(NULL, false));
+    g_assert_cmpint(data.n, ==, 10);
+    g_assert_cmpint(data.active, ==, 0);
+    g_assert(!g_main_context_iteration(NULL, false));
+
+    aio_set_event_notifier(ctx, &data.e, NULL, NULL);
+    while (g_main_context_iteration(NULL, false));
+    event_notifier_cleanup(&data.e);
+}
+
+static void test_source_wait_event_notifier_noflush(void)
+{
+    EventNotifierTestData data = { .n = 0 };
+    EventNotifierTestData dummy = { .n = 0, .active = 1 };
+
+    event_notifier_init(&data.e, false);
+    aio_set_event_notifier(ctx, &data.e, event_ready_cb, NULL);
+
+    while (g_main_context_iteration(NULL, false));
+    g_assert_cmpint(data.n, ==, 0);
+
+    /* Until there is an active descriptor, glib may or may not call
+     * event_ready_cb.  Still, it must not block.  */
+    event_notifier_set(&data.e);
+    g_main_context_iteration(NULL, true);
+    data.n = 0;
+
+    /* An active event notifier forces aio_poll to look at EventNotifiers.  */
+    event_notifier_init(&dummy.e, false);
+    aio_set_event_notifier(ctx, &dummy.e, event_ready_cb, event_active_cb);
+
+    event_notifier_set(&data.e);
+    g_assert(g_main_context_iteration(NULL, false));
+    g_assert_cmpint(data.n, ==, 1);
+    g_assert(!g_main_context_iteration(NULL, false));
+    g_assert_cmpint(data.n, ==, 1);
+
+    event_notifier_set(&data.e);
+    g_assert(g_main_context_iteration(NULL, false));
+    g_assert_cmpint(data.n, ==, 2);
+    g_assert(!g_main_context_iteration(NULL, false));
+    g_assert_cmpint(data.n, ==, 2);
+
+    event_notifier_set(&dummy.e);
+    while (g_main_context_iteration(NULL, false));
+    g_assert_cmpint(data.n, ==, 2);
+    g_assert_cmpint(dummy.n, ==, 1);
+    g_assert_cmpint(dummy.active, ==, 0);
+
+    aio_set_event_notifier(ctx, &dummy.e, NULL, NULL);
+    event_notifier_cleanup(&dummy.e);
+
+    aio_set_event_notifier(ctx, &data.e, NULL, NULL);
+    while (g_main_context_iteration(NULL, false));
+    g_assert_cmpint(data.n, ==, 2);
+
+    event_notifier_cleanup(&data.e);
+}
+
+/* End of tests.  */
+
+int main(int argc, char **argv)
+{
+    GSource *src;
+
+    ctx = aio_context_new();
+    src = aio_get_g_source(ctx);
+    g_source_attach(src, NULL);
+    g_source_unref(src);
+
+    while (g_main_context_iteration(NULL, false));
+
+    g_test_init(&argc, &argv, NULL);
+    g_test_add_func("/aio/notify",                  test_notify);
+    g_test_add_func("/aio/flush",                   test_flush);
+    g_test_add_func("/aio/bh/schedule",             test_bh_schedule);
+    g_test_add_func("/aio/bh/schedule10",           test_bh_schedule10);
+    g_test_add_func("/aio/bh/cancel",               test_bh_cancel);
+    g_test_add_func("/aio/bh/delete",               test_bh_delete);
+    g_test_add_func("/aio/bh/callback-delete/one",  test_bh_delete_from_cb);
+    g_test_add_func("/aio/bh/callback-delete/many", test_bh_delete_from_cb_many);
+    g_test_add_func("/aio/bh/flush",                test_bh_flush);
+    g_test_add_func("/aio/event/add-remove",        test_set_event_notifier);
+    g_test_add_func("/aio/event/wait",              test_wait_event_notifier);
+    g_test_add_func("/aio/event/wait/no-flush-cb",  test_wait_event_notifier_noflush);
+    g_test_add_func("/aio/event/flush",             test_flush_event_notifier);
+
+    g_test_add_func("/aio-gsource/notify",                  test_source_notify);
+    g_test_add_func("/aio-gsource/flush",                   test_source_flush);
+    g_test_add_func("/aio-gsource/bh/schedule",             test_source_bh_schedule);
+    g_test_add_func("/aio-gsource/bh/schedule10",           test_source_bh_schedule10);
+    g_test_add_func("/aio-gsource/bh/cancel",               test_source_bh_cancel);
+    g_test_add_func("/aio-gsource/bh/delete",               test_source_bh_delete);
+    g_test_add_func("/aio-gsource/bh/callback-delete/one",  test_source_bh_delete_from_cb);
+    g_test_add_func("/aio-gsource/bh/callback-delete/many", test_source_bh_delete_from_cb_many);
+    g_test_add_func("/aio-gsource/bh/flush",                test_source_bh_flush);
+    g_test_add_func("/aio-gsource/event/add-remove",        test_source_set_event_notifier);
+    g_test_add_func("/aio-gsource/event/wait",              test_source_wait_event_notifier);
+    g_test_add_func("/aio-gsource/event/wait/no-flush-cb",  test_source_wait_event_notifier_noflush);
+    g_test_add_func("/aio-gsource/event/flush",             test_source_flush_event_notifier);
+    return g_test_run();
+}
diff --git a/tests/test-thread-pool.c b/tests/test-thread-pool.c
new file mode 100644
index 000000000..fea0445fb
--- /dev/null
+++ b/tests/test-thread-pool.c
@@ -0,0 +1,216 @@
+#include <glib.h>
+#include "qemu-common.h"
+#include "qemu-aio.h"
+#include "thread-pool.h"
+#include "block.h"
+
+static int active;
+
+typedef struct {
+    BlockDriverAIOCB *aiocb;
+    int n;
+    int ret;
+} WorkerTestData;
+
+static int worker_cb(void *opaque)
+{
+    WorkerTestData *data = opaque;
+    return __sync_fetch_and_add(&data->n, 1);
+}
+
+static int long_cb(void *opaque)
+{
+    WorkerTestData *data = opaque;
+    __sync_fetch_and_add(&data->n, 1);
+    g_usleep(2000000);
+    __sync_fetch_and_add(&data->n, 1);
+    return 0;
+}
+
+static void done_cb(void *opaque, int ret)
+{
+    WorkerTestData *data = opaque;
+    g_assert_cmpint(data->ret, ==, -EINPROGRESS);
+    data->ret = ret;
+    data->aiocb = NULL;
+
+    /* Callbacks are serialized, so no need to use atomic ops.  */
+    active--;
+}
+
+/* A non-blocking poll of the main AIO context (we cannot use aio_poll
+ * because we do not know the AioContext).
+ */
+static void qemu_aio_wait_nonblocking(void)
+{
+    qemu_notify_event();
+    qemu_aio_wait();
+}
+
+static void test_submit(void)
+{
+    WorkerTestData data = { .n = 0 };
+    thread_pool_submit(worker_cb, &data);
+    qemu_aio_flush();
+    g_assert_cmpint(data.n, ==, 1);
+}
+
+static void test_submit_aio(void)
+{
+    WorkerTestData data = { .n = 0, .ret = -EINPROGRESS };
+    data.aiocb = thread_pool_submit_aio(worker_cb, &data, done_cb, &data);
+
+    /* The callbacks are not called until after the first wait.  */
+    active = 1;
+    g_assert_cmpint(data.ret, ==, -EINPROGRESS);
+    qemu_aio_flush();
+    g_assert_cmpint(active, ==, 0);
+    g_assert_cmpint(data.n, ==, 1);
+    g_assert_cmpint(data.ret, ==, 0);
+}
+
+static void co_test_cb(void *opaque)
+{
+    WorkerTestData *data = opaque;
+
+    active = 1;
+    data->n = 0;
+    data->ret = -EINPROGRESS;
+    thread_pool_submit_co(worker_cb, data);
+
+    /* The test continues in test_submit_co, after qemu_coroutine_enter... */
+
+    g_assert_cmpint(data->n, ==, 1);
+    data->ret = 0;
+    active--;
+
+    /* The test continues in test_submit_co, after qemu_aio_flush... */
+}
+
+static void test_submit_co(void)
+{
+    WorkerTestData data;
+    Coroutine *co = qemu_coroutine_create(co_test_cb);
+
+    qemu_coroutine_enter(co, &data);
+
+    /* Back here once the worker has started.  */
+
+    g_assert_cmpint(active, ==, 1);
+    g_assert_cmpint(data.ret, ==, -EINPROGRESS);
+
+    /* qemu_aio_flush will execute the rest of the coroutine.  */
+
+    qemu_aio_flush();
+
+    /* Back here after the coroutine has finished.  */
+
+    g_assert_cmpint(active, ==, 0);
+    g_assert_cmpint(data.ret, ==, 0);
+}
+
+static void test_submit_many(void)
+{
+    WorkerTestData data[100];
+    int i;
+
+    /* Start more work items than there will be threads.  */
+    for (i = 0; i < 100; i++) {
+        data[i].n = 0;
+        data[i].ret = -EINPROGRESS;
+        thread_pool_submit_aio(worker_cb, &data[i], done_cb, &data[i]);
+    }
+
+    active = 100;
+    while (active > 0) {
+        qemu_aio_wait();
+    }
+    for (i = 0; i < 100; i++) {
+        g_assert_cmpint(data[i].n, ==, 1);
+        g_assert_cmpint(data[i].ret, ==, 0);
+    }
+}
+
+static void test_cancel(void)
+{
+    WorkerTestData data[100];
+    int num_canceled;
+    int i;
+
+    /* Start more work items than there will be threads, to ensure
+     * the pool is full.
+     */
+    test_submit_many();
+
+    /* Start long running jobs, to ensure we can cancel some.  */
+    for (i = 0; i < 100; i++) {
+        data[i].n = 0;
+        data[i].ret = -EINPROGRESS;
+        data[i].aiocb = thread_pool_submit_aio(long_cb, &data[i],
+                                               done_cb, &data[i]);
+    }
+
+    /* Starting the threads may be left to a bottom half.  Let it
+     * run, but do not waste too much time...
+     */
+    active = 100;
+    qemu_aio_wait_nonblocking();
+
+    /* Wait some time for the threads to start, with some sanity
+     * testing on the behavior of the scheduler...
+     */
+    g_assert_cmpint(active, ==, 100);
+    g_usleep(1000000);
+    g_assert_cmpint(active, >, 50);
+
+    /* Cancel the jobs that haven't been started yet.  */
+    num_canceled = 0;
+    for (i = 0; i < 100; i++) {
+        if (__sync_val_compare_and_swap(&data[i].n, 0, 3) == 0) {
+            data[i].ret = -ECANCELED;
+            bdrv_aio_cancel(data[i].aiocb);
+            active--;
+            num_canceled++;
+        }
+    }
+    g_assert_cmpint(active, >, 0);
+    g_assert_cmpint(num_canceled, <, 100);
+
+    /* Canceling the others will be a blocking operation.  */
+    for (i = 0; i < 100; i++) {
+        if (data[i].n != 3) {
+            bdrv_aio_cancel(data[i].aiocb);
+        }
+    }
+
+    /* Finish execution and execute any remaining callbacks.  */
+    qemu_aio_flush();
+    g_assert_cmpint(active, ==, 0);
+    for (i = 0; i < 100; i++) {
+        if (data[i].n == 3) {
+            g_assert_cmpint(data[i].ret, ==, -ECANCELED);
+            g_assert(data[i].aiocb != NULL);
+        } else {
+            g_assert_cmpint(data[i].n, ==, 2);
+            g_assert_cmpint(data[i].ret, ==, 0);
+            g_assert(data[i].aiocb == NULL);
+        }
+    }
+}
+
+int main(int argc, char **argv)
+{
+    /* These should be removed once each AioContext has its thread pool.
+     * The test should create its own AioContext.
+     */
+    qemu_init_main_loop();
+    bdrv_init();
+
+    g_test_init(&argc, &argv, NULL);
+    g_test_add_func("/thread-pool/submit", test_submit);
+    g_test_add_func("/thread-pool/submit-aio", test_submit_aio);
+    g_test_add_func("/thread-pool/submit-co", test_submit_co);
+    g_test_add_func("/thread-pool/submit-many", test_submit_many);
+    g_test_add_func("/thread-pool/cancel", test_cancel);
+    return g_test_run();
+}
diff --git a/thread-pool.c b/thread-pool.c
new file mode 100644
index 000000000..204f70b7b
--- /dev/null
+++ b/thread-pool.c
@@ -0,0 +1,289 @@
+/*
+ * QEMU block layer thread pool
+ *
+ * Copyright IBM, Corp. 2008
+ * Copyright Red Hat, Inc. 2012
+ *
+ * Authors:
+ *  Anthony Liguori   <aliguori@us.ibm.com>
+ *  Paolo Bonzini     <pbonzini@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ *
+ * Contributions after 2012-01-13 are licensed under the terms of the
+ * GNU GPL, version 2 or (at your option) any later version.
+ */
+#include "qemu-common.h"
+#include "qemu-queue.h"
+#include "qemu-thread.h"
+#include "osdep.h"
+#include "qemu-coroutine.h"
+#include "trace.h"
+#include "block_int.h"
+#include "event_notifier.h"
+#include "thread-pool.h"
+
+static void do_spawn_thread(void);
+
+typedef struct ThreadPoolElement ThreadPoolElement;
+
+enum ThreadState {
+    THREAD_QUEUED,
+    THREAD_ACTIVE,
+    THREAD_DONE,
+    THREAD_CANCELED,
+};
+
+struct ThreadPoolElement {
+    BlockDriverAIOCB common;
+    ThreadPoolFunc *func;
+    void *arg;
+
+    /* Moving state out of THREAD_QUEUED is protected by lock.  After
+     * that, only the worker thread can write to it.  Reads and writes
+     * of state and ret are ordered with memory barriers.
+     */
+    enum ThreadState state;
+    int ret;
+
+    /* Access to this list is protected by lock.  */
+    QTAILQ_ENTRY(ThreadPoolElement) reqs;
+
+    /* Access to this list is protected by the global mutex.  */
+    QLIST_ENTRY(ThreadPoolElement) all;
+};
+
+static EventNotifier notifier;
+static QemuMutex lock;
+static QemuCond check_cancel;
+static QemuSemaphore sem;
+static int max_threads = 64;
+static QEMUBH *new_thread_bh;
+
+/* The following variables are protected by the global mutex.  */
+static QLIST_HEAD(, ThreadPoolElement) head;
+
+/* The following variables are protected by lock.  */
+static QTAILQ_HEAD(, ThreadPoolElement) request_list;
+static int cur_threads;
+static int idle_threads;
+static int new_threads;     /* backlog of threads we need to create */
+static int pending_threads; /* threads created but not running yet */
+static int pending_cancellations; /* whether we need a cond_broadcast */
+
+static void *worker_thread(void *unused)
+{
+    qemu_mutex_lock(&lock);
+    pending_threads--;
+    do_spawn_thread();
+
+    while (1) {
+        ThreadPoolElement *req;
+        int ret;
+
+        do {
+            idle_threads++;
+            qemu_mutex_unlock(&lock);
+            ret = qemu_sem_timedwait(&sem, 10000);
+            qemu_mutex_lock(&lock);
+            idle_threads--;
+        } while (ret == -1 && !QTAILQ_EMPTY(&request_list));
+        if (ret == -1) {
+            break;
+        }
+
+        req = QTAILQ_FIRST(&request_list);
+        QTAILQ_REMOVE(&request_list, req, reqs);
+        req->state = THREAD_ACTIVE;
+        qemu_mutex_unlock(&lock);
+
+        ret = req->func(req->arg);
+
+        req->ret = ret;
+        /* Write ret before state.  */
+        smp_wmb();
+        req->state = THREAD_DONE;
+
+        qemu_mutex_lock(&lock);
+        if (pending_cancellations) {
+            qemu_cond_broadcast(&check_cancel);
+        }
+
+        event_notifier_set(&notifier);
+    }
+
+    cur_threads--;
+    qemu_mutex_unlock(&lock);
+    return NULL;
+}
+
+static void do_spawn_thread(void)
+{
+    QemuThread t;
+
+    /* Runs with lock taken.  */
+    if (!new_threads) {
+        return;
+    }
+
+    new_threads--;
+    pending_threads++;
+
+    qemu_thread_create(&t, worker_thread, NULL, QEMU_THREAD_DETACHED);
+}
+
+static void spawn_thread_bh_fn(void *opaque)
+{
+    qemu_mutex_lock(&lock);
+    do_spawn_thread();
+    qemu_mutex_unlock(&lock);
+}
+
+static void spawn_thread(void)
+{
+    cur_threads++;
+    new_threads++;
+    /* If there are threads being created, they will spawn new workers, so
+     * we don't spend time creating many threads in a loop holding a mutex or
+     * starving the current vcpu.
+     *
+     * If there are no idle threads, ask the main thread to create one, so we
+     * inherit the correct affinity instead of the vcpu affinity.
+     */
+    if (!pending_threads) {
+        qemu_bh_schedule(new_thread_bh);
+    }
+}
+
+static void event_notifier_ready(EventNotifier *notifier)
+{
+    ThreadPoolElement *elem, *next;
+
+    event_notifier_test_and_clear(notifier);
+restart:
+    QLIST_FOREACH_SAFE(elem, &head, all, next) {
+        if (elem->state != THREAD_CANCELED && elem->state != THREAD_DONE) {
+            continue;
+        }
+        if (elem->state == THREAD_DONE) {
+            trace_thread_pool_complete(elem, elem->common.opaque, elem->ret);
+        }
+        if (elem->state == THREAD_DONE && elem->common.cb) {
+            QLIST_REMOVE(elem, all);
+            /* Read state before ret.  */
+            smp_rmb();
+            elem->common.cb(elem->common.opaque, elem->ret);
+            qemu_aio_release(elem);
+            goto restart;
+        } else {
+            /* remove the request */
+            QLIST_REMOVE(elem, all);
+            qemu_aio_release(elem);
+        }
+    }
+}
+
+static int thread_pool_active(EventNotifier *notifier)
+{
+    return !QLIST_EMPTY(&head);
+}
+
+static void thread_pool_cancel(BlockDriverAIOCB *acb)
+{
+    ThreadPoolElement *elem = (ThreadPoolElement *)acb;
+
+    trace_thread_pool_cancel(elem, elem->common.opaque);
+
+    qemu_mutex_lock(&lock);
+    if (elem->state == THREAD_QUEUED &&
+        /* No thread has yet started working on elem. we can try to "steal"
+         * the item from the worker if we can get a signal from the
+         * semaphore.  Because this is non-blocking, we can do it with
+         * the lock taken and ensure that elem will remain THREAD_QUEUED.
+         */
+        qemu_sem_timedwait(&sem, 0) == 0) {
+        QTAILQ_REMOVE(&request_list, elem, reqs);
+        elem->state = THREAD_CANCELED;
+        event_notifier_set(&notifier);
+    } else {
+        pending_cancellations++;
+        while (elem->state != THREAD_CANCELED && elem->state != THREAD_DONE) {
+            qemu_cond_wait(&check_cancel, &lock);
+        }
+        pending_cancellations--;
+    }
+    qemu_mutex_unlock(&lock);
+}
+
+static const AIOCBInfo thread_pool_aiocb_info = {
+    .aiocb_size         = sizeof(ThreadPoolElement),
+    .cancel             = thread_pool_cancel,
+};
+
+BlockDriverAIOCB *thread_pool_submit_aio(ThreadPoolFunc *func, void *arg,
+        BlockDriverCompletionFunc *cb, void *opaque)
+{
+    ThreadPoolElement *req;
+
+    req = qemu_aio_get(&thread_pool_aiocb_info, NULL, cb, opaque);
+    req->func = func;
+    req->arg = arg;
+    req->state = THREAD_QUEUED;
+
+    QLIST_INSERT_HEAD(&head, req, all);
+
+    trace_thread_pool_submit(req, arg);
+
+    qemu_mutex_lock(&lock);
+    if (idle_threads == 0 && cur_threads < max_threads) {
+        spawn_thread();
+    }
+    QTAILQ_INSERT_TAIL(&request_list, req, reqs);
+    qemu_mutex_unlock(&lock);
+    qemu_sem_post(&sem);
+    return &req->common;
+}
+
+typedef struct ThreadPoolCo {
+    Coroutine *co;
+    int ret;
+} ThreadPoolCo;
+
+static void thread_pool_co_cb(void *opaque, int ret)
+{
+    ThreadPoolCo *co = opaque;
+
+    co->ret = ret;
+    qemu_coroutine_enter(co->co, NULL);
+}
+
+int coroutine_fn thread_pool_submit_co(ThreadPoolFunc *func, void *arg)
+{
+    ThreadPoolCo tpc = { .co = qemu_coroutine_self(), .ret = -EINPROGRESS };
+    assert(qemu_in_coroutine());
+    thread_pool_submit_aio(func, arg, thread_pool_co_cb, &tpc);
+    qemu_coroutine_yield();
+    return tpc.ret;
+}
+
+void thread_pool_submit(ThreadPoolFunc *func, void *arg)
+{
+    thread_pool_submit_aio(func, arg, NULL, NULL);
+}
+
+static void thread_pool_init(void)
+{
+    QLIST_INIT(&head);
+    event_notifier_init(&notifier, false);
+    qemu_mutex_init(&lock);
+    qemu_cond_init(&check_cancel);
+    qemu_sem_init(&sem, 0);
+    qemu_aio_set_event_notifier(&notifier, event_notifier_ready,
+                                thread_pool_active);
+
+    QTAILQ_INIT(&request_list);
+    new_thread_bh = qemu_bh_new(spawn_thread_bh_fn, NULL);
+}
+
+block_init(thread_pool_init)
diff --git a/thread-pool.h b/thread-pool.h
new file mode 100644
index 000000000..378a4ac9f
--- /dev/null
+++ b/thread-pool.h
@@ -0,0 +1,34 @@
+/*
+ * QEMU block layer thread pool
+ *
+ * Copyright IBM, Corp. 2008
+ * Copyright Red Hat, Inc. 2012
+ *
+ * Authors:
+ *  Anthony Liguori   <aliguori@us.ibm.com>
+ *  Paolo Bonzini     <pbonzini@redhat.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2.  See
+ * the COPYING file in the top-level directory.
+ *
+ * Contributions after 2012-01-13 are licensed under the terms of the
+ * GNU GPL, version 2 or (at your option) any later version.
+ */
+
+#ifndef QEMU_THREAD_POOL_H
+#define QEMU_THREAD_POOL_H 1
+
+#include "qemu-common.h"
+#include "qemu-queue.h"
+#include "qemu-thread.h"
+#include "qemu-coroutine.h"
+#include "block_int.h"
+
+typedef int ThreadPoolFunc(void *opaque);
+
+BlockDriverAIOCB *thread_pool_submit_aio(ThreadPoolFunc *func, void *arg,
+     BlockDriverCompletionFunc *cb, void *opaque);
+int coroutine_fn thread_pool_submit_co(ThreadPoolFunc *func, void *arg);
+void thread_pool_submit(ThreadPoolFunc *func, void *arg);
+
+#endif
diff --git a/trace-events b/trace-events
index 04b072392..6c6cbf10f 100644
--- a/trace-events
+++ b/trace-events
@@ -74,10 +74,22 @@ bdrv_co_do_copy_on_readv(void *bs, int64_t sector_num, int nb_sectors, int64_t c
 # block/stream.c
 stream_one_iteration(void *s, int64_t sector_num, int nb_sectors, int is_allocated) "s %p sector_num %"PRId64" nb_sectors %d is_allocated %d"
 stream_start(void *bs, void *base, void *s, void *co, void *opaque) "bs %p base %p s %p co %p opaque %p"
+commit_one_iteration(void *s, int64_t sector_num, int nb_sectors, int is_allocated) "s %p sector_num %"PRId64" nb_sectors %d is_allocated %d"
+commit_start(void *bs, void *base, void *top, void *s, void *co, void *opaque) "bs %p base %p top %p s %p co %p opaque %p"
+
+# block/mirror.c
+mirror_start(void *bs, void *s, void *co, void *opaque) "bs %p s %p co %p opaque %p"
+mirror_before_flush(void *s) "s %p"
+mirror_before_drain(void *s, int64_t cnt) "s %p dirty count %"PRId64
+mirror_before_sleep(void *s, int64_t cnt, int synced) "s %p dirty count %"PRId64" synced %d"
+mirror_one_iteration(void *s, int64_t sector_num, int nb_sectors) "s %p sector_num %"PRId64" nb_sectors %d"
 
 # blockdev.c
 qmp_block_job_cancel(void *job) "job %p"
-block_stream_cb(void *bs, void *job, int ret) "bs %p job %p ret %d"
+qmp_block_job_pause(void *job) "job %p"
+qmp_block_job_resume(void *job) "job %p"
+qmp_block_job_complete(void *job) "job %p"
+block_job_cb(void *bs, void *job, int ret) "bs %p job %p ret %d"
 qmp_block_stream(void *bs, void *job) "bs %p job %p"
 
 # hw/virtio-blk.c
@@ -86,6 +98,11 @@ virtio_blk_rw_complete(void *req, int ret) "req %p ret %d"
 virtio_blk_handle_write(void *req, uint64_t sector, size_t nsectors) "req %p sector %"PRIu64" nsectors %zu"
 virtio_blk_handle_read(void *req, uint64_t sector, size_t nsectors) "req %p sector %"PRIu64" nsectors %zu"
 
+# thread-pool.c
+thread_pool_submit(void *req, void *opaque) "req %p opaque %p"
+thread_pool_complete(void *req, void *opaque, int ret) "req %p opaque %p ret %d"
+thread_pool_cancel(void *req, void *opaque) "req %p opaque %p"
+
 # posix-aio-compat.c
 paio_submit(void *acb, void *opaque, int64_t sector_num, int nb_sectors, int type) "acb %p opaque %p sector_num %"PRId64" nb_sectors %d type %d"
 paio_complete(void *acb, void *opaque, int ret) "acb %p opaque %p ret %d"
@@ -243,9 +260,12 @@ usb_port_release(int bus, const char *port) "bus %d, port %s"
 
 # hw/usb/hcd-ehci.c
 usb_ehci_reset(void) "=== RESET ==="
-usb_ehci_mmio_readl(uint32_t addr, const char *str, uint32_t val) "rd mmio %04x [%s] = %x"
-usb_ehci_mmio_writel(uint32_t addr, const char *str, uint32_t val) "wr mmio %04x [%s] = %x"
-usb_ehci_mmio_change(uint32_t addr, const char *str, uint32_t new, uint32_t old) "ch mmio %04x [%s] = %x (old: %x)"
+usb_ehci_opreg_read(uint32_t addr, const char *str, uint32_t val) "rd mmio %04x [%s] = %x"
+usb_ehci_opreg_write(uint32_t addr, const char *str, uint32_t val) "wr mmio %04x [%s] = %x"
+usb_ehci_opreg_change(uint32_t addr, const char *str, uint32_t new, uint32_t old) "ch mmio %04x [%s] = %x (old: %x)"
+usb_ehci_portsc_read(uint32_t addr, uint32_t port, uint32_t val) "rd mmio %04x [port %d] = %x"
+usb_ehci_portsc_write(uint32_t addr, uint32_t port, uint32_t val) "wr mmio %04x [port %d] = %x"
+usb_ehci_portsc_change(uint32_t addr, uint32_t port, uint32_t new, uint32_t old) "ch mmio %04x [port %d] = %x (old: %x)"
 usb_ehci_usbsts(const char *sts, int state) "usbsts %s %d"
 usb_ehci_state(const char *schedule, const char *state) "%s schedule %s"
 usb_ehci_qh_ptrs(void *q, uint32_t addr, uint32_t nxt, uint32_t c_qtd, uint32_t n_qtd, uint32_t a_qtd) "q %p - QH @ %08x: next %08x qtds %08x,%08x,%08x"
@@ -263,6 +283,10 @@ usb_ehci_data(int rw, uint32_t cpage, uint32_t offset, uint32_t addr, uint32_t l
 usb_ehci_queue_action(void *q, const char *action) "q %p: %s"
 usb_ehci_packet_action(void *q, void *p, const char *action) "q %p p %p: %s"
 usb_ehci_irq(uint32_t level, uint32_t frindex, uint32_t sts, uint32_t mask) "level %d, frindex 0x%04x, sts 0x%x, mask 0x%x"
+usb_ehci_guest_bug(const char *reason) "%s"
+usb_ehci_doorbell_ring(void) ""
+usb_ehci_doorbell_ack(void) ""
+usb_ehci_dma_error(void) ""
 
 # hw/usb/hcd-uhci.c
 usb_uhci_reset(void) "=== RESET ==="
@@ -277,7 +301,7 @@ usb_uhci_mmio_writew(uint32_t addr, uint32_t val) "addr 0x%04x, val 0x%04x"
 usb_uhci_mmio_readl(uint32_t addr, uint32_t val) "addr 0x%04x, ret 0x%08x"
 usb_uhci_mmio_writel(uint32_t addr, uint32_t val) "addr 0x%04x, val 0x%08x"
 usb_uhci_queue_add(uint32_t token) "token 0x%x"
-usb_uhci_queue_del(uint32_t token) "token 0x%x"
+usb_uhci_queue_del(uint32_t token, const char *reason) "token 0x%x: %s"
 usb_uhci_packet_add(uint32_t token, uint32_t addr) "token 0x%x, td 0x%x"
 usb_uhci_packet_link_async(uint32_t token, uint32_t addr) "token 0x%x, td 0x%x"
 usb_uhci_packet_unlink_async(uint32_t token, uint32_t addr) "token 0x%x, td 0x%x"
@@ -310,8 +334,13 @@ usb_xhci_runtime_write(uint32_t off, uint32_t val) "off 0x%04x, val 0x%08x"
 usb_xhci_doorbell_write(uint32_t off, uint32_t val) "off 0x%04x, val 0x%08x"
 usb_xhci_irq_intx(uint32_t level) "level %d"
 usb_xhci_irq_msi(uint32_t nr) "nr %d"
-usb_xhci_queue_event(uint32_t idx, const char *name, uint64_t param, uint32_t status, uint32_t control) "idx %d, %s, p %016" PRIx64 ", s %08x, c 0x%08x"
+usb_xhci_irq_msix(uint32_t nr) "nr %d"
+usb_xhci_irq_msix_use(uint32_t nr) "nr %d"
+usb_xhci_irq_msix_unuse(uint32_t nr) "nr %d"
+usb_xhci_queue_event(uint32_t vector, uint32_t idx, const char *trb, const char *evt, uint64_t param, uint32_t status, uint32_t control) "v %d, idx %d, %s, %s, p %016" PRIx64 ", s %08x, c 0x%08x"
 usb_xhci_fetch_trb(uint64_t addr, const char *name, uint64_t param, uint32_t status, uint32_t control) "addr %016" PRIx64 ", %s, p %016" PRIx64 ", s %08x, c 0x%08x"
+usb_xhci_port_reset(uint32_t port) "port %d"
+usb_xhci_port_link(uint32_t port, uint32_t pls) "port %d, pls %d"
 usb_xhci_slot_enable(uint32_t slotid) "slotid %d"
 usb_xhci_slot_disable(uint32_t slotid) "slotid %d"
 usb_xhci_slot_address(uint32_t slotid) "slotid %d"
@@ -320,10 +349,11 @@ usb_xhci_slot_evaluate(uint32_t slotid) "slotid %d"
 usb_xhci_slot_reset(uint32_t slotid) "slotid %d"
 usb_xhci_ep_enable(uint32_t slotid, uint32_t epid) "slotid %d, epid %d"
 usb_xhci_ep_disable(uint32_t slotid, uint32_t epid) "slotid %d, epid %d"
+usb_xhci_ep_set_dequeue(uint32_t slotid, uint32_t epid, uint64_t param) "slotid %d, epid %d, ptr %016" PRIx64
 usb_xhci_ep_kick(uint32_t slotid, uint32_t epid) "slotid %d, epid %d"
 usb_xhci_ep_stop(uint32_t slotid, uint32_t epid) "slotid %d, epid %d"
 usb_xhci_ep_reset(uint32_t slotid, uint32_t epid) "slotid %d, epid %d"
-usb_xhci_xfer_start(void *xfer, uint32_t slotid, uint32_t epid, uint32_t length) "%p: slotid %d, epid %d, length %d"
+usb_xhci_xfer_start(void *xfer, uint32_t slotid, uint32_t epid) "%p: slotid %d, epid %d"
 usb_xhci_xfer_async(void *xfer) "%p"
 usb_xhci_xfer_nak(void *xfer) "%p"
 usb_xhci_xfer_retry(void *xfer) "%p"
@@ -336,6 +366,7 @@ usb_desc_device_qualifier(int addr, int len, int ret) "dev %d query device quali
 usb_desc_config(int addr, int index, int len, int ret) "dev %d query config %d, len %d, ret %d"
 usb_desc_other_speed_config(int addr, int index, int len, int ret) "dev %d query config %d, len %d, ret %d"
 usb_desc_string(int addr, int index, int len, int ret) "dev %d query string %d, len %d, ret %d"
+usb_desc_bos(int addr, int len, int ret) "dev %d bos, len %d, ret %d"
 usb_set_addr(int addr) "dev %d"
 usb_set_config(int addr, int config, int ret) "dev %d, config %d, ret %d"
 usb_set_interface(int addr, int iface, int alt, int ret) "dev %d, interface %d, altsetting %d, ret %d"
@@ -378,7 +409,7 @@ usb_host_claim_interfaces(int bus, int addr, int config, int nif) "dev %d:%d, co
 usb_host_release_interfaces(int bus, int addr) "dev %d:%d"
 usb_host_req_control(int bus, int addr, void *p, int req, int value, int index) "dev %d:%d, packet %p, req 0x%x, value %d, index %d"
 usb_host_req_data(int bus, int addr, void *p, int in, int ep, int size) "dev %d:%d, packet %p, in %d, ep %d, size %d"
-usb_host_req_complete(int bus, int addr, void *p, int status) "dev %d:%d, packet %p, status %d"
+usb_host_req_complete(int bus, int addr, void *p, int status, int length) "dev %d:%d, packet %p, status %d, length %d"
 usb_host_req_emulated(int bus, int addr, void *p, int status) "dev %d:%d, packet %p, status %d"
 usb_host_req_canceled(int bus, int addr, void *p) "dev %d:%d, packet %p"
 usb_host_urb_submit(int bus, int addr, void *aurb, int length, int more) "dev %d:%d, aurb %p, length %d, more %d"
@@ -906,6 +937,10 @@ ppm_save(const char *filename, void *display_surface) "%s surface=%p"
 savevm_section_start(void) ""
 savevm_section_end(unsigned int section_id) "section_id %u"
 
+# arch_init.c
+migration_bitmap_sync_start(void) ""
+migration_bitmap_sync_end(uint64_t dirty_pages) "dirty_pages %" PRIu64""
+
 # hw/qxl.c
 disable qxl_interface_set_mm_time(int qid, uint32_t mm_time) "%d %d"
 disable qxl_io_write_vga(int qid, const char *mode, uint32_t addr, uint32_t val) "%d %s addr=%u val=%u"
@@ -924,8 +959,9 @@ qxl_interface_update_area_complete_rest(int qid, uint32_t num_updated_rects) "%d
 qxl_interface_update_area_complete_overflow(int qid, int max) "%d max=%d"
 qxl_interface_update_area_complete_schedule_bh(int qid, uint32_t num_dirty) "%d #dirty=%d"
 qxl_io_destroy_primary_ignored(int qid, const char *mode) "%d %s"
+qxl_io_log(int qid, const uint8_t *log_buf) "%d %s"
 qxl_io_read_unexpected(int qid) "%d"
-qxl_io_unexpected_vga_mode(int qid, uint32_t io_port, const char *desc) "%d 0x%x (%s)"
+qxl_io_unexpected_vga_mode(int qid, uint64_t addr, uint64_t val, const char *desc) "%d 0x%"PRIx64"=%"PRIu64" (%s)"
 qxl_io_write(int qid, const char *mode, uint64_t addr, uint64_t val, unsigned size, int async) "%d %s addr=%"PRIu64 " val=%"PRIu64" size=%u async=%d"
 qxl_memslot_add_guest(int qid, uint32_t slot_id, uint64_t guest_start, uint64_t guest_end) "%d %u: guest phys 0x%"PRIx64 " - 0x%" PRIx64
 qxl_post_load(int qid, const char *mode) "%d %s"
@@ -956,6 +992,7 @@ qxl_spice_destroy_surfaces(int qid, int async) "%d async=%d"
 qxl_spice_destroy_surface_wait_complete(int qid, uint32_t id) "%d sid=%d"
 qxl_spice_destroy_surface_wait(int qid, uint32_t id, int async) "%d sid=%d async=%d"
 qxl_spice_flush_surfaces_async(int qid, uint32_t surface_count, uint32_t num_free_res) "%d s#=%d, res#=%d"
+qxl_spice_monitors_config(int qid) "%d"
 qxl_spice_loadvm_commands(int qid, void *ext, uint32_t count) "%d ext=%p count=%d"
 qxl_spice_oom(int qid) "%d"
 qxl_spice_reset_cursor(int qid) "%d"
@@ -964,6 +1001,13 @@ qxl_spice_reset_memslots(int qid) "%d"
 qxl_spice_update_area(int qid, uint32_t surface_id, uint32_t left, uint32_t right, uint32_t top, uint32_t bottom) "%d sid=%d [%d,%d,%d,%d]"
 qxl_spice_update_area_rest(int qid, uint32_t num_dirty_rects, uint32_t clear_dirty_region) "%d #d=%d clear=%d"
 qxl_surfaces_dirty(int qid, int surface, int offset, int size) "%d surface=%d offset=%d size=%d"
+qxl_send_events(int qid, uint32_t events) "%d %d"
+qxl_send_events_vm_stopped(int qid, uint32_t events) "%d %d"
+qxl_set_guest_bug(int qid) "%d"
+qxl_interrupt_client_monitors_config(int qid, int num_heads, void *heads) "%d %d %p"
+qxl_client_monitors_config_unsupported_by_guest(int qid, uint32_t int_mask, void *client_monitors_config) "%d %X %p"
+qxl_client_monitors_config_capped(int qid, int requested, int limit) "%d %d %d"
+qxl_client_monitors_config_crc(int qid, unsigned size, uint32_t crc32) "%d %u %u"
 
 # hw/qxl-render.c
 qxl_render_blit_guest_primary_initialized(void) ""
diff --git a/trace/control.c b/trace/control.c
index 22d5863ee..be05efb99 100644
--- a/trace/control.c
+++ b/trace/control.c
@@ -12,6 +12,8 @@
 
 void trace_backend_init_events(const char *fname)
 {
+    int ret;
+
     if (fname == NULL) {
         return;
     }
@@ -30,7 +32,12 @@ void trace_backend_init_events(const char *fname)
             if ('#' == line_buf[0]) { /* skip commented lines */
                 continue;
             }
-            if (!trace_event_set_state(line_buf, true)) {
+            if ('-' == line_buf[0]) {
+                ret = trace_event_set_state(line_buf+1, false);
+            } else {
+                ret = trace_event_set_state(line_buf, true);
+            }
+            if (!ret) {
                 fprintf(stderr,
                         "error: trace event '%s' does not exist\n", line_buf);
                 exit(1);
diff --git a/translate-all.c b/translate-all.c
index 5bd2d3711..d9c2e5786 100644
--- a/translate-all.c
+++ b/translate-all.c
@@ -33,9 +33,6 @@
 /* code generation context */
 TCGContext tcg_ctx;
 
-uint16_t gen_opc_buf[OPC_BUF_SIZE];
-TCGArg gen_opparam_buf[OPPARAM_BUF_SIZE];
-
 target_ulong gen_opc_pc[OPC_BUF_SIZE];
 uint16_t gen_opc_icount[OPC_BUF_SIZE];
 uint8_t gen_opc_instr_start[OPC_BUF_SIZE];
diff --git a/ui/cocoa.m b/ui/cocoa.m
index 2383646dc..87d2e44c6 100644
--- a/ui/cocoa.m
+++ b/ui/cocoa.m
@@ -1017,8 +1017,8 @@ void cocoa_display_init(DisplayState *ds, int full_screen)
     dcl = g_malloc0(sizeof(DisplayChangeListener));
 
     // register vga output callbacks
-    dcl->dpy_update = cocoa_update;
-    dcl->dpy_resize = cocoa_resize;
+    dcl->dpy_gfx_update = cocoa_update;
+    dcl->dpy_gfx_resize = cocoa_resize;
     dcl->dpy_refresh = cocoa_refresh;
 
 	register_displaychangelistener(ds, dcl);
diff --git a/ui/curses.c b/ui/curses.c
index c2be2c641..b40b22307 100644
--- a/ui/curses.c
+++ b/ui/curses.c
@@ -95,17 +95,16 @@ static void curses_calc_pad(void)
     }
 }
 
-static void curses_resize(DisplayState *ds)
+static void curses_resize(DisplayState *ds, int width, int height)
 {
-    if (ds_get_width(ds) == gwidth && ds_get_height(ds) == gheight)
+    if (width == gwidth && height == gheight) {
         return;
+    }
 
-    gwidth = ds_get_width(ds);
-    gheight = ds_get_height(ds);
+    gwidth = width;
+    gheight = height;
 
     curses_calc_pad();
-    ds->surface->width = width * FONT_WIDTH;
-    ds->surface->height = height * FONT_HEIGHT;
 }
 
 #ifndef _WIN32
@@ -167,8 +166,6 @@ static void curses_refresh(DisplayState *ds)
         clear();
         refresh();
         curses_calc_pad();
-        ds->surface->width = FONT_WIDTH * width;
-        ds->surface->height = FONT_HEIGHT * height;
         vga_hw_invalidate();
         invalidate = 0;
     }
@@ -195,8 +192,6 @@ static void curses_refresh(DisplayState *ds)
             refresh();
             curses_calc_pad();
             curses_update(ds, 0, 0, width, height);
-            ds->surface->width = FONT_WIDTH * width;
-            ds->surface->height = FONT_HEIGHT * height;
             continue;
         }
 #endif
@@ -355,13 +350,11 @@ void curses_display_init(DisplayState *ds, int full_screen)
 #endif
 
     dcl = (DisplayChangeListener *) g_malloc0(sizeof(DisplayChangeListener));
-    dcl->dpy_update = curses_update;
-    dcl->dpy_resize = curses_resize;
+    dcl->dpy_text_update = curses_update;
+    dcl->dpy_text_resize = curses_resize;
     dcl->dpy_refresh = curses_refresh;
     dcl->dpy_text_cursor = curses_cursor_position;
     register_displaychangelistener(ds, dcl);
-    qemu_free_displaysurface(ds);
-    ds->surface = qemu_create_displaysurface_from(640, 400, 0, 0, (uint8_t*) screen);
 
     invalidate = 1;
 }
diff --git a/ui/sdl.c b/ui/sdl.c
index f6f711c1b..37f01b217 100644
--- a/ui/sdl.c
+++ b/ui/sdl.c
@@ -55,7 +55,6 @@ static int absolute_enabled = 0;
 static int guest_cursor = 0;
 static int guest_x, guest_y;
 static SDL_Cursor *guest_sprite = NULL;
-static uint8_t allocator;
 static SDL_PixelFormat host_format;
 static int scaling_active = 0;
 static Notifier mouse_mode_notifier;
@@ -117,108 +116,13 @@ static void do_sdl_resize(int width, int height, int bpp)
 
 static void sdl_resize(DisplayState *ds)
 {
-    if  (!allocator) {
-        if (!scaling_active)
-            do_sdl_resize(ds_get_width(ds), ds_get_height(ds), 0);
-        else if (real_screen->format->BitsPerPixel != ds_get_bits_per_pixel(ds))
-            do_sdl_resize(real_screen->w, real_screen->h, ds_get_bits_per_pixel(ds));
-        sdl_setdata(ds);
-    } else {
-        if (guest_screen != NULL) {
-            SDL_FreeSurface(guest_screen);
-            guest_screen = NULL;
-        }
-    }
-}
-
-static PixelFormat sdl_to_qemu_pixelformat(SDL_PixelFormat *sdl_pf)
-{
-    PixelFormat qemu_pf;
-
-    memset(&qemu_pf, 0x00, sizeof(PixelFormat));
-
-    qemu_pf.bits_per_pixel = sdl_pf->BitsPerPixel;
-    qemu_pf.bytes_per_pixel = sdl_pf->BytesPerPixel;
-    qemu_pf.depth = (qemu_pf.bits_per_pixel) == 32 ? 24 : (qemu_pf.bits_per_pixel);
-
-    qemu_pf.rmask = sdl_pf->Rmask;
-    qemu_pf.gmask = sdl_pf->Gmask;
-    qemu_pf.bmask = sdl_pf->Bmask;
-    qemu_pf.amask = sdl_pf->Amask;
-
-    qemu_pf.rshift = sdl_pf->Rshift;
-    qemu_pf.gshift = sdl_pf->Gshift;
-    qemu_pf.bshift = sdl_pf->Bshift;
-    qemu_pf.ashift = sdl_pf->Ashift;
-
-    qemu_pf.rbits = 8 - sdl_pf->Rloss;
-    qemu_pf.gbits = 8 - sdl_pf->Gloss;
-    qemu_pf.bbits = 8 - sdl_pf->Bloss;
-    qemu_pf.abits = 8 - sdl_pf->Aloss;
-
-    qemu_pf.rmax = ((1 << qemu_pf.rbits) - 1);
-    qemu_pf.gmax = ((1 << qemu_pf.gbits) - 1);
-    qemu_pf.bmax = ((1 << qemu_pf.bbits) - 1);
-    qemu_pf.amax = ((1 << qemu_pf.abits) - 1);
-
-    return qemu_pf;
-}
-
-static DisplaySurface* sdl_create_displaysurface(int width, int height)
-{
-    DisplaySurface *surface = (DisplaySurface*) g_malloc0(sizeof(DisplaySurface));
-
-    surface->width = width;
-    surface->height = height;
-
-    if (scaling_active) {
-        int linesize;
-        PixelFormat pf;
-        if (host_format.BytesPerPixel != 2 && host_format.BytesPerPixel != 4) {
-            linesize = width * 4;
-            pf = qemu_default_pixelformat(32);
-        } else {
-            linesize = width * host_format.BytesPerPixel;
-            pf = sdl_to_qemu_pixelformat(&host_format);
-        }
-        qemu_alloc_display(surface, width, height, linesize, pf, 0);
-        return surface;
+    if (!scaling_active) {
+        do_sdl_resize(ds_get_width(ds), ds_get_height(ds), 0);
+    } else if (real_screen->format->BitsPerPixel != ds_get_bits_per_pixel(ds)) {
+        do_sdl_resize(real_screen->w, real_screen->h,
+                      ds_get_bits_per_pixel(ds));
     }
-
-    if (host_format.BitsPerPixel == 16)
-        do_sdl_resize(width, height, 16);
-    else
-        do_sdl_resize(width, height, 32);
-
-    surface->pf = sdl_to_qemu_pixelformat(real_screen->format);
-    surface->linesize = real_screen->pitch;
-    surface->data = real_screen->pixels;
-
-#ifdef HOST_WORDS_BIGENDIAN
-    surface->flags = QEMU_REALPIXELS_FLAG | QEMU_BIG_ENDIAN_FLAG;
-#else
-    surface->flags = QEMU_REALPIXELS_FLAG;
-#endif
-    allocator = 1;
-
-    return surface;
-}
-
-static void sdl_free_displaysurface(DisplaySurface *surface)
-{
-    allocator = 0;
-    if (surface == NULL)
-        return;
-
-    if (surface->flags & QEMU_ALLOCATED_FLAG)
-        g_free(surface->data);
-    g_free(surface);
-}
-
-static DisplaySurface* sdl_resize_displaysurface(DisplaySurface *surface, int width, int height)
-{
-    sdl_free_displaysurface(surface);
-    return sdl_create_displaysurface(width, height);
+    sdl_setdata(ds);
 }
 
 /* generic keyboard conversion */
@@ -553,7 +457,7 @@ static void sdl_scale(DisplayState *ds, int width, int height)
     if (!is_buffer_shared(ds->surface)) {
         ds->surface = qemu_resize_displaysurface(ds, ds_get_width(ds),
                                                  ds_get_height(ds));
-        dpy_resize(ds);
+        dpy_gfx_resize(ds);
     }
 }
 
@@ -899,13 +803,7 @@ static void sdl_refresh(DisplayState *ds)
     }
 }
 
-static void sdl_fill(DisplayState *ds, int x, int y, int w, int h, uint32_t c)
-{
-    SDL_Rect dst = { x, y, w, h };
-    SDL_FillRect(real_screen, &dst, c);
-}
-
-static void sdl_mouse_warp(int x, int y, int on)
+static void sdl_mouse_warp(DisplayState *ds, int x, int y, int on)
 {
     if (on) {
         if (!guest_cursor)
@@ -921,7 +819,7 @@ static void sdl_mouse_warp(int x, int y, int on)
     guest_x = x, guest_y = y;
 }
 
-static void sdl_mouse_define(QEMUCursor *c)
+static void sdl_mouse_define(DisplayState *ds, QEMUCursor *c)
 {
     uint8_t *image, *mask;
     int bpl;
@@ -955,7 +853,6 @@ void sdl_display_init(DisplayState *ds, int full_screen, int no_frame)
 {
     int flags;
     uint8_t data = 0;
-    DisplayAllocator *da;
     const SDL_VideoInfo *vi;
     char *filename;
 
@@ -1020,23 +917,14 @@ void sdl_display_init(DisplayState *ds, int full_screen, int no_frame)
     }
 
     dcl = g_malloc0(sizeof(DisplayChangeListener));
-    dcl->dpy_update = sdl_update;
-    dcl->dpy_resize = sdl_resize;
+    dcl->dpy_gfx_update = sdl_update;
+    dcl->dpy_gfx_resize = sdl_resize;
     dcl->dpy_refresh = sdl_refresh;
-    dcl->dpy_setdata = sdl_setdata;
-    dcl->dpy_fill = sdl_fill;
-    ds->mouse_set = sdl_mouse_warp;
-    ds->cursor_define = sdl_mouse_define;
+    dcl->dpy_gfx_setdata = sdl_setdata;
+    dcl->dpy_mouse_set = sdl_mouse_warp;
+    dcl->dpy_cursor_define = sdl_mouse_define;
     register_displaychangelistener(ds, dcl);
 
-    da = g_malloc0(sizeof(DisplayAllocator));
-    da->create_displaysurface = sdl_create_displaysurface;
-    da->resize_displaysurface = sdl_resize_displaysurface;
-    da->free_displaysurface = sdl_free_displaysurface;
-    if (register_displayallocator(ds, da) == da) {
-        dpy_resize(ds);
-    }
-
     mouse_mode_notifier.notify = sdl_mouse_mode_change;
     qemu_add_mouse_mode_change_notifier(&mouse_mode_notifier);
 
diff --git a/ui/spice-core.c b/ui/spice-core.c
index 4fc48f890..261c6f2c1 100644
--- a/ui/spice-core.c
+++ b/ui/spice-core.c
@@ -37,6 +37,7 @@
 #include "migration.h"
 #include "monitor.h"
 #include "hw/hw.h"
+#include "spice-display.h"
 
 /* core bits */
 
@@ -45,6 +46,7 @@ static Notifier migration_state;
 static const char *auth = "spice";
 static char *auth_passwd;
 static time_t auth_expires = TIME_MAX;
+static int spice_migration_completed;
 int using_spice = 0;
 
 static QemuThread me;
@@ -221,7 +223,6 @@ static void channel_event(int event, SpiceChannelEventInfo *info)
     client = qdict_new();
     server = qdict_new();
 
-#ifdef SPICE_CHANNEL_EVENT_FLAG_ADDR_EXT
     if (info->flags & SPICE_CHANNEL_EVENT_FLAG_ADDR_EXT) {
         add_addr_info(client, (struct sockaddr *)&info->paddr_ext,
                       info->plen_ext);
@@ -230,12 +231,7 @@ static void channel_event(int event, SpiceChannelEventInfo *info)
     } else {
         error_report("spice: %s, extended address is expected",
                      __func__);
-#endif
-        add_addr_info(client, &info->paddr, info->plen);
-        add_addr_info(server, &info->laddr, info->llen);
-#ifdef SPICE_CHANNEL_EVENT_FLAG_ADDR_EXT
     }
-#endif
 
     if (event == SPICE_CHANNEL_EVENT_INITIALIZED) {
         qdict_put(server, "auth", qstring_from_str(auth));
@@ -274,7 +270,6 @@ static SpiceCoreInterface core_interface = {
     .channel_event      = channel_event,
 };
 
-#ifdef SPICE_INTERFACE_MIGRATION
 typedef struct SpiceMigration {
     SpiceMigrateInstance sin;
     struct {
@@ -284,6 +279,7 @@ typedef struct SpiceMigration {
 } SpiceMigration;
 
 static void migrate_connect_complete_cb(SpiceMigrateInstance *sin);
+static void migrate_end_complete_cb(SpiceMigrateInstance *sin);
 
 static const SpiceMigrateInterface migrate_interface = {
     .base.type = SPICE_INTERFACE_MIGRATION,
@@ -291,7 +287,7 @@ static const SpiceMigrateInterface migrate_interface = {
     .base.major_version = SPICE_INTERFACE_MIGRATION_MAJOR,
     .base.minor_version = SPICE_INTERFACE_MIGRATION_MINOR,
     .migrate_connect_complete = migrate_connect_complete_cb,
-    .migrate_end_complete = NULL,
+    .migrate_end_complete = migrate_end_complete_cb,
 };
 
 static SpiceMigration spice_migrate;
@@ -304,7 +300,12 @@ static void migrate_connect_complete_cb(SpiceMigrateInstance *sin)
     }
     sm->connect_complete.cb = NULL;
 }
-#endif
+
+static void migrate_end_complete_cb(SpiceMigrateInstance *sin)
+{
+    monitor_protocol_event(QEVENT_SPICE_MIGRATE_COMPLETED, NULL);
+    spice_migration_completed = true;
+}
 
 /* config string parsing */
 
@@ -344,7 +345,8 @@ static const char *stream_video_names[] = {
     [ SPICE_STREAM_VIDEO_FILTER ] = "filter",
 };
 #define parse_stream_video(_name) \
-    name2enum(_name, stream_video_names, ARRAY_SIZE(stream_video_names))
+    parse_name(_name, "stream video control", \
+               stream_video_names, ARRAY_SIZE(stream_video_names))
 
 static const char *compression_names[] = {
     [ SPICE_IMAGE_COMPRESS_OFF ]      = "off",
@@ -383,17 +385,13 @@ static SpiceChannelList *qmp_query_spice_channels(void)
         chan = g_malloc0(sizeof(*chan));
         chan->value = g_malloc0(sizeof(*chan->value));
 
-#ifdef SPICE_CHANNEL_EVENT_FLAG_ADDR_EXT
         if (item->info->flags & SPICE_CHANNEL_EVENT_FLAG_ADDR_EXT) {
             paddr = (struct sockaddr *)&item->info->paddr_ext;
             plen = item->info->plen_ext;
         } else {
-#endif
             paddr = &item->info->paddr;
             plen = item->info->plen;
-#ifdef SPICE_CHANNEL_EVENT_FLAG_ADDR_EXT
         }
-#endif
 
         getnameinfo(paddr, plen,
                     host, sizeof(host), port, sizeof(port),
@@ -435,6 +433,7 @@ SpiceInfo *qmp_query_spice(Error **errp)
     }
 
     info->enabled = true;
+    info->migrated = spice_migration_completed;
 
     addr = qemu_opt_get(opts, "addr");
     port = qemu_opt_get_number(opts, "port", 0);
@@ -462,13 +461,10 @@ SpiceInfo *qmp_query_spice(Error **errp)
         info->tls_port = tls_port;
     }
 
-#if SPICE_SERVER_VERSION >= 0x000a03 /* 0.10.3 */
     info->mouse_mode = spice_server_is_server_mouse(spice_server) ?
                        SPICE_QUERY_MOUSE_MODE_SERVER :
                        SPICE_QUERY_MOUSE_MODE_CLIENT;
-#else
-    info->mouse_mode = SPICE_QUERY_MOUSE_MODE_UNKNOWN;
-#endif
+
     /* for compatibility with the original command */
     info->has_channels = true;
     info->channels = qmp_query_spice_channels();
@@ -481,17 +477,11 @@ static void migration_state_notifier(Notifier *notifier, void *data)
     MigrationState *s = data;
 
     if (migration_is_active(s)) {
-#ifdef SPICE_INTERFACE_MIGRATION
         spice_server_migrate_start(spice_server);
-#endif
     } else if (migration_has_finished(s)) {
-#ifndef SPICE_INTERFACE_MIGRATION
-        spice_server_migrate_switch(spice_server);
-#else
         spice_server_migrate_end(spice_server, true);
     } else if (migration_has_failed(s)) {
         spice_server_migrate_end(spice_server, false);
-#endif
     }
 }
 
@@ -500,16 +490,11 @@ int qemu_spice_migrate_info(const char *hostname, int port, int tls_port,
                             MonitorCompletion *cb, void *opaque)
 {
     int ret;
-#ifdef SPICE_INTERFACE_MIGRATION
+
     spice_migrate.connect_complete.cb = cb;
     spice_migrate.connect_complete.opaque = opaque;
     ret = spice_server_migrate_connect(spice_server, hostname,
                                        port, tls_port, subject);
-#else
-    ret = spice_server_migrate_info(spice_server, hostname,
-                                    port, tls_port, subject);
-    cb(opaque, NULL);
-#endif
     return ret;
 }
 
@@ -545,6 +530,18 @@ static int add_channel(const char *name, const char *value, void *opaque)
     return 0;
 }
 
+static void vm_change_state_handler(void *opaque, int running,
+                                    RunState state)
+{
+    if (running) {
+        qemu_spice_display_start();
+        spice_server_vm_start(spice_server);
+    } else {
+        spice_server_vm_stop(spice_server);
+        qemu_spice_display_stop();
+    }
+}
+
 void qemu_spice_init(void)
 {
     QemuOpts *opts = QTAILQ_FIRST(&qemu_spice_opts.head);
@@ -558,6 +555,7 @@ void qemu_spice_init(void)
     int port, tls_port, len, addr_flags;
     spice_image_compression_t compression;
     spice_wan_compression_t wan_compr;
+    bool seamless_migration;
 
     qemu_thread_get_self(&me);
 
@@ -612,7 +610,7 @@ void qemu_spice_init(void)
         }
 
         x509_key_password = qemu_opt_get(opts, "x509-key-password");
-        x509_dh_file = qemu_opt_get(opts, "x509-dh-file");
+        x509_dh_file = qemu_opt_get(opts, "x509-dh-key-file");
         tls_ciphers = qemu_opt_get(opts, "tls-ciphers");
     }
 
@@ -642,16 +640,11 @@ void qemu_spice_init(void)
         spice_server_set_ticket(spice_server, password, 0, 0, 0);
     }
     if (qemu_opt_get_bool(opts, "sasl", 0)) {
-#if SPICE_SERVER_VERSION >= 0x000900 /* 0.9.0 */
         if (spice_server_set_sasl_appname(spice_server, "qemu") == -1 ||
             spice_server_set_sasl(spice_server, 1) == -1) {
             error_report("spice: failed to enable sasl");
             exit(1);
         }
-#else
-        error_report("spice: sasl is not available (spice >= 0.9 required)");
-        exit(1);
-#endif
     }
     if (qemu_opt_get_bool(opts, "disable-ticketing", 0)) {
         auth = "none";
@@ -696,11 +689,11 @@ void qemu_spice_init(void)
 
     qemu_opt_foreach(opts, add_channel, &tls_port, 0);
 
-#if SPICE_SERVER_VERSION >= 0x000a02 /* 0.10.2 */
     spice_server_set_name(spice_server, qemu_name);
     spice_server_set_uuid(spice_server, qemu_uuid);
-#endif
 
+    seamless_migration = qemu_opt_get_bool(opts, "seamless-migration", 0);
+    spice_server_set_seamless_migration(spice_server, seamless_migration);
     if (0 != spice_server_init(spice_server, &core_interface)) {
         error_report("failed to initialize spice server");
         exit(1);
@@ -709,15 +702,15 @@ void qemu_spice_init(void)
 
     migration_state.notify = migration_state_notifier;
     add_migration_state_change_notifier(&migration_state);
-#ifdef SPICE_INTERFACE_MIGRATION
     spice_migrate.sin.base.sif = &migrate_interface.base;
     spice_migrate.connect_complete.cb = NULL;
     qemu_spice_add_interface(&spice_migrate.sin.base);
-#endif
 
     qemu_spice_input_init();
     qemu_spice_audio_init();
 
+    qemu_add_vm_change_state_handler(vm_change_state_handler, &spice_server);
+
     g_free(x509_key_file);
     g_free(x509_cert_file);
     g_free(x509_cacert_file);
@@ -740,6 +733,7 @@ int qemu_spice_add_interface(SpiceBaseInstance *sin)
         spice_server = spice_server_new();
         spice_server_init(spice_server, &core_interface);
     }
+
     return spice_server_add_interface(spice_server, sin);
 }
 
@@ -778,15 +772,11 @@ int qemu_spice_set_pw_expire(time_t expires)
 
 int qemu_spice_display_add_client(int csock, int skipauth, int tls)
 {
-#if SPICE_SERVER_VERSION >= 0x000a01
     if (tls) {
         return spice_server_add_ssl_client(spice_server, csock, skipauth);
     } else {
         return spice_server_add_client(spice_server, csock, skipauth);
     }
-#else
-    return -1;
-#endif
 }
 
 static void spice_register_config(void)
diff --git a/ui/spice-display.c b/ui/spice-display.c
index 3e8f0b3ad..6aff3368c 100644
--- a/ui/spice-display.c
+++ b/ui/spice-display.c
@@ -126,46 +126,48 @@ void qemu_spice_wakeup(SimpleSpiceDisplay *ssd)
     ssd->worker->wakeup(ssd->worker);
 }
 
-void qemu_spice_start(SimpleSpiceDisplay *ssd)
+static int spice_display_is_running;
+
+void qemu_spice_display_start(void)
+{
+    spice_display_is_running = true;
+}
+
+void qemu_spice_display_stop(void)
 {
-    trace_qemu_spice_start(ssd->qxl.id);
-    ssd->worker->start(ssd->worker);
+    spice_display_is_running = false;
 }
 
-void qemu_spice_stop(SimpleSpiceDisplay *ssd)
+int qemu_spice_display_is_running(SimpleSpiceDisplay *ssd)
 {
-    trace_qemu_spice_stop(ssd->qxl.id);
-    ssd->worker->stop(ssd->worker);
+    return spice_display_is_running;
 }
 
-static SimpleSpiceUpdate *qemu_spice_create_update(SimpleSpiceDisplay *ssd)
+static void qemu_spice_create_one_update(SimpleSpiceDisplay *ssd,
+                                         QXLRect *rect)
 {
     SimpleSpiceUpdate *update;
     QXLDrawable *drawable;
     QXLImage *image;
     QXLCommand *cmd;
-    uint8_t *src, *dst;
-    int by, bw, bh;
+    int bw, bh;
     struct timespec time_space;
-
-    if (qemu_spice_rect_is_empty(&ssd->dirty)) {
-        return NULL;
-    };
+    pixman_image_t *dest;
 
     trace_qemu_spice_create_update(
-           ssd->dirty.left, ssd->dirty.right,
-           ssd->dirty.top, ssd->dirty.bottom);
+           rect->left, rect->right,
+           rect->top, rect->bottom);
 
     update   = g_malloc0(sizeof(*update));
     drawable = &update->drawable;
     image    = &update->image;
     cmd      = &update->ext.cmd;
 
-    bw       = ssd->dirty.right - ssd->dirty.left;
-    bh       = ssd->dirty.bottom - ssd->dirty.top;
+    bw       = rect->right - rect->left;
+    bh       = rect->bottom - rect->top;
     update->bitmap = g_malloc(bw * bh * 4);
 
-    drawable->bbox            = ssd->dirty;
+    drawable->bbox            = *rect;
     drawable->clip.type       = SPICE_CLIP_TYPE_NONE;
     drawable->effect          = QXL_EFFECT_OPAQUE;
     drawable->release_info.id = (uintptr_t)update;
@@ -193,31 +195,94 @@ static SimpleSpiceUpdate *qemu_spice_create_update(SimpleSpiceDisplay *ssd)
     image->bitmap.palette = 0;
     image->bitmap.format = SPICE_BITMAP_FMT_32BIT;
 
-    if (ssd->conv == NULL) {
-        PixelFormat dst = qemu_default_pixelformat(32);
-        ssd->conv = qemu_pf_conv_get(&dst, &ssd->ds->surface->pf);
-        assert(ssd->conv);
+    dest = pixman_image_create_bits(PIXMAN_x8r8g8b8, bw, bh,
+                                    (void *)update->bitmap, bw * 4);
+    pixman_image_composite(PIXMAN_OP_SRC, ssd->surface, NULL, ssd->mirror,
+                           rect->left, rect->top, 0, 0,
+                           rect->left, rect->top, bw, bh);
+    pixman_image_composite(PIXMAN_OP_SRC, ssd->mirror, NULL, dest,
+                           rect->left, rect->top, 0, 0,
+                           0, 0, bw, bh);
+    pixman_image_unref(dest);
+
+    cmd->type = QXL_CMD_DRAW;
+    cmd->data = (uintptr_t)drawable;
+
+    QTAILQ_INSERT_TAIL(&ssd->updates, update, next);
+}
+
+static void qemu_spice_create_update(SimpleSpiceDisplay *ssd)
+{
+    static const int blksize = 32;
+    int blocks = (ds_get_width(ssd->ds) + blksize - 1) / blksize;
+    int dirty_top[blocks];
+    int y, yoff, x, xoff, blk, bw;
+    int bpp = ds_get_bytes_per_pixel(ssd->ds);
+    uint8_t *guest, *mirror;
+
+    if (qemu_spice_rect_is_empty(&ssd->dirty)) {
+        return;
+    };
+
+    if (ssd->surface == NULL) {
+        ssd->surface = pixman_image_ref(ds_get_image(ssd->ds));
+        ssd->mirror  = qemu_pixman_mirror_create(ds_get_format(ssd->ds),
+                                                 ds_get_image(ssd->ds));
     }
 
-    src = ds_get_data(ssd->ds) +
-        ssd->dirty.top * ds_get_linesize(ssd->ds) +
-        ssd->dirty.left * ds_get_bytes_per_pixel(ssd->ds);
-    dst = update->bitmap;
-    for (by = 0; by < bh; by++) {
-        qemu_pf_conv_run(ssd->conv, dst, src, bw);
-        src += ds_get_linesize(ssd->ds);
-        dst += image->bitmap.stride;
+    for (blk = 0; blk < blocks; blk++) {
+        dirty_top[blk] = -1;
     }
 
-    cmd->type = QXL_CMD_DRAW;
-    cmd->data = (uintptr_t)drawable;
+    guest = ds_get_data(ssd->ds);
+    mirror = (void *)pixman_image_get_data(ssd->mirror);
+    for (y = ssd->dirty.top; y < ssd->dirty.bottom; y++) {
+        yoff = y * ds_get_linesize(ssd->ds);
+        for (x = ssd->dirty.left; x < ssd->dirty.right; x += blksize) {
+            xoff = x * bpp;
+            blk = x / blksize;
+            bw = MIN(blksize, ssd->dirty.right - x);
+            if (memcmp(guest + yoff + xoff,
+                       mirror + yoff + xoff,
+                       bw * bpp) == 0) {
+                if (dirty_top[blk] != -1) {
+                    QXLRect update = {
+                        .top    = dirty_top[blk],
+                        .bottom = y,
+                        .left   = x,
+                        .right  = x + bw,
+                    };
+                    qemu_spice_create_one_update(ssd, &update);
+                    dirty_top[blk] = -1;
+                }
+            } else {
+                if (dirty_top[blk] == -1) {
+                    dirty_top[blk] = y;
+                }
+            }
+        }
+    }
+
+    for (x = ssd->dirty.left; x < ssd->dirty.right; x += blksize) {
+        blk = x / blksize;
+        bw = MIN(blksize, ssd->dirty.right - x);
+        if (dirty_top[blk] != -1) {
+            QXLRect update = {
+                .top    = dirty_top[blk],
+                .bottom = ssd->dirty.bottom,
+                .left   = x,
+                .right  = x + bw,
+            };
+            qemu_spice_create_one_update(ssd, &update);
+            dirty_top[blk] = -1;
+        }
+    }
 
     memset(&ssd->dirty, 0, sizeof(ssd->dirty));
-    return update;
 }
 
 /*
- * Called from spice server thread context (via interface_release_ressource)
+ * Called from spice server thread context (via interface_release_resource)
  * We do *not* hold the global qemu mutex here, so extra care is needed
  * when calling qemu functions.  QEMU interfaces used:
  *    - g_free (underlying glibc free is re-entrant).
@@ -269,26 +334,16 @@ void qemu_spice_destroy_host_primary(SimpleSpiceDisplay *ssd)
     qemu_spice_destroy_primary_surface(ssd, 0, QXL_SYNC);
 }
 
-void qemu_spice_vm_change_state_handler(void *opaque, int running,
-                                        RunState state)
-{
-    SimpleSpiceDisplay *ssd = opaque;
-
-    if (running) {
-        ssd->running = true;
-        qemu_spice_start(ssd);
-    } else {
-        qemu_spice_stop(ssd);
-        ssd->running = false;
-    }
-}
-
 void qemu_spice_display_init_common(SimpleSpiceDisplay *ssd, DisplayState *ds)
 {
     ssd->ds = ds;
     qemu_mutex_init(&ssd->lock);
+    QTAILQ_INIT(&ssd->updates);
     ssd->mouse_x = -1;
     ssd->mouse_y = -1;
+    if (ssd->num_surfaces == 0) {
+        ssd->num_surfaces = 1024;
+    }
     ssd->bufsize = (16 * 1024 * 1024);
     ssd->buf = g_malloc(ssd->bufsize);
 }
@@ -314,16 +369,22 @@ void qemu_spice_display_update(SimpleSpiceDisplay *ssd,
 
 void qemu_spice_display_resize(SimpleSpiceDisplay *ssd)
 {
+    SimpleSpiceUpdate *update;
+
     dprint(1, "%s:\n", __FUNCTION__);
 
     memset(&ssd->dirty, 0, sizeof(ssd->dirty));
-    qemu_pf_conv_put(ssd->conv);
-    ssd->conv = NULL;
+    if (ssd->surface) {
+        pixman_image_unref(ssd->surface);
+        ssd->surface = NULL;
+        pixman_image_unref(ssd->mirror);
+        ssd->mirror = NULL;
+    }
 
     qemu_mutex_lock(&ssd->lock);
-    if (ssd->update != NULL) {
-        qemu_spice_destroy_update(ssd, ssd->update);
-        ssd->update = NULL;
+    while ((update = QTAILQ_FIRST(&ssd->updates)) != NULL) {
+        QTAILQ_REMOVE(&ssd->updates, update, next);
+        qemu_spice_destroy_update(ssd, update);
     }
     qemu_mutex_unlock(&ssd->lock);
     qemu_spice_destroy_host_primary(ssd);
@@ -336,12 +397,12 @@ void qemu_spice_display_resize(SimpleSpiceDisplay *ssd)
 void qemu_spice_cursor_refresh_unlocked(SimpleSpiceDisplay *ssd)
 {
     if (ssd->cursor) {
-        ssd->ds->cursor_define(ssd->cursor);
+        dpy_cursor_define(ssd->ds, ssd->cursor);
         cursor_put(ssd->cursor);
         ssd->cursor = NULL;
     }
     if (ssd->mouse_x != -1 && ssd->mouse_y != -1) {
-        ssd->ds->mouse_set(ssd->mouse_x, ssd->mouse_y, 1);
+        dpy_mouse_set(ssd->ds, ssd->mouse_x, ssd->mouse_y, 1);
         ssd->mouse_x = -1;
         ssd->mouse_y = -1;
     }
@@ -353,8 +414,8 @@ void qemu_spice_display_refresh(SimpleSpiceDisplay *ssd)
     vga_hw_update();
 
     qemu_mutex_lock(&ssd->lock);
-    if (ssd->update == NULL) {
-        ssd->update = qemu_spice_create_update(ssd);
+    if (QTAILQ_EMPTY(&ssd->updates)) {
+        qemu_spice_create_update(ssd);
         ssd->notify++;
     }
     qemu_spice_cursor_refresh_unlocked(ssd);
@@ -399,7 +460,7 @@ static void interface_get_init_info(QXLInstance *sin, QXLDevInitInfo *info)
     info->num_memslots_groups = NUM_MEMSLOTS_GROUPS;
     info->internal_groupslot_id = 0;
     info->qxl_ram_size = ssd->bufsize;
-    info->n_surfaces = NUM_SURFACES;
+    info->n_surfaces = ssd->num_surfaces;
 }
 
 static int interface_get_command(QXLInstance *sin, struct QXLCommandExt *ext)
@@ -411,9 +472,9 @@ static int interface_get_command(QXLInstance *sin, struct QXLCommandExt *ext)
     dprint(3, "%s:\n", __FUNCTION__);
 
     qemu_mutex_lock(&ssd->lock);
-    if (ssd->update != NULL) {
-        update = ssd->update;
-        ssd->update = NULL;
+    update = QTAILQ_FIRST(&ssd->updates);
+    if (update != NULL) {
+        QTAILQ_REMOVE(&ssd->updates, update, next);
         *ext = update->ext;
         ret = true;
     }
@@ -464,6 +525,37 @@ static int interface_flush_resources(QXLInstance *sin)
     return 0;
 }
 
+static void interface_update_area_complete(QXLInstance *sin,
+        uint32_t surface_id,
+        QXLRect *dirty, uint32_t num_updated_rects)
+{
+    /* should never be called, used in qxl native mode only */
+    fprintf(stderr, "%s: abort()\n", __func__);
+    abort();
+}
+
+/* called from spice server thread context only */
+static void interface_async_complete(QXLInstance *sin, uint64_t cookie_token)
+{
+    /* should never be called, used in qxl native mode only */
+    fprintf(stderr, "%s: abort()\n", __func__);
+    abort();
+}
+
+static void interface_set_client_capabilities(QXLInstance *sin,
+                                              uint8_t client_present,
+                                              uint8_t caps[58])
+{
+    dprint(3, "%s:\n", __func__);
+}
+
+static int interface_client_monitors_config(QXLInstance *sin,
+                                        VDAgentMonitorsConfig *monitors_config)
+{
+    dprint(3, "%s:\n", __func__);
+    return 0; /* == not supported by guest */
+}
+
 static const QXLInterface dpy_interface = {
     .base.type               = SPICE_INTERFACE_QXL,
     .base.description        = "qemu simple display",
@@ -483,6 +575,10 @@ static const QXLInterface dpy_interface = {
     .req_cursor_notification = interface_req_cursor_notification,
     .notify_update           = interface_notify_update,
     .flush_resources         = interface_flush_resources,
+    .async_complete          = interface_async_complete,
+    .update_area_complete    = interface_update_area_complete,
+    .set_client_capabilities = interface_set_client_capabilities,
+    .client_monitors_config  = interface_client_monitors_config,
 };
 
 static SimpleSpiceDisplay sdpy;
@@ -503,8 +599,8 @@ static void display_refresh(struct DisplayState *ds)
 }
 
 static DisplayChangeListener display_listener = {
-    .dpy_update  = display_update,
-    .dpy_resize  = display_resize,
+    .dpy_gfx_update  = display_update,
+    .dpy_gfx_resize  = display_resize,
     .dpy_refresh = display_refresh,
 };
 
@@ -512,13 +608,12 @@ void qemu_spice_display_init(DisplayState *ds)
 {
     assert(sdpy.ds == NULL);
     qemu_spice_display_init_common(&sdpy, ds);
-    register_displaychangelistener(ds, &display_listener);
 
     sdpy.qxl.base.sif = &dpy_interface.base;
     qemu_spice_add_interface(&sdpy.qxl.base);
     assert(sdpy.worker);
 
-    qemu_add_vm_change_state_handler(qemu_spice_vm_change_state_handler, &sdpy);
     qemu_spice_create_host_memslot(&sdpy);
     qemu_spice_create_host_primary(&sdpy);
+    register_displaychangelistener(ds, &display_listener);
 }
diff --git a/ui/spice-display.h b/ui/spice-display.h
index 12e50b6ef..38b6ea98b 100644
--- a/ui/spice-display.h
+++ b/ui/spice-display.h
@@ -20,8 +20,7 @@
 #include <spice/qxl_dev.h>
 
 #include "qemu-thread.h"
-#include "console.h"
-#include "pflib.h"
+#include "qemu-pixman.h"
 #include "sysemu.h"
 
 #define NUM_MEMSLOTS 8
@@ -32,8 +31,6 @@
 #define MEMSLOT_GROUP_GUEST 1
 #define NUM_MEMSLOTS_GROUPS 2
 
-#define NUM_SURFACES 1024
-
 /*
  * Internal enum to differenciate between options for
  * io calls that have a sync (old) version and an _async (new)
@@ -51,6 +48,7 @@ typedef enum qxl_async_io {
 enum {
     QXL_COOKIE_TYPE_IO,
     QXL_COOKIE_TYPE_RENDER_UPDATE_AREA,
+    QXL_COOKIE_TYPE_POST_LOAD_MONITORS_CONFIG,
 };
 
 typedef struct QXLCookie {
@@ -78,11 +76,12 @@ struct SimpleSpiceDisplay {
     QXLWorker *worker;
     QXLInstance qxl;
     uint32_t unique;
-    QemuPfConv *conv;
+    pixman_image_t *surface;
+    pixman_image_t *mirror;
+    int32_t num_surfaces;
 
     QXLRect dirty;
     int notify;
-    int running;
 
     /*
      * All struct members below this comment can be accessed from
@@ -90,7 +89,7 @@ struct SimpleSpiceDisplay {
      * to them must be protected by the lock.
      */
     QemuMutex lock;
-    SimpleSpiceUpdate *update;
+    QTAILQ_HEAD(, SimpleSpiceUpdate) updates;
     QEMUCursor *cursor;
     int mouse_x, mouse_y;
 };
@@ -100,6 +99,7 @@ struct SimpleSpiceUpdate {
     QXLImage image;
     QXLCommandExt ext;
     uint8_t *bitmap;
+    QTAILQ_ENTRY(SimpleSpiceUpdate) next;
 };
 
 int qemu_spice_rect_is_empty(const QXLRect* r);
@@ -129,5 +129,6 @@ void qemu_spice_create_primary_surface(SimpleSpiceDisplay *ssd, uint32_t id,
 void qemu_spice_destroy_primary_surface(SimpleSpiceDisplay *ssd,
                                         uint32_t id, qxl_async_io async);
 void qemu_spice_wakeup(SimpleSpiceDisplay *ssd);
-void qemu_spice_start(SimpleSpiceDisplay *ssd);
-void qemu_spice_stop(SimpleSpiceDisplay *ssd);
+void qemu_spice_display_start(void);
+void qemu_spice_display_stop(void);
+int qemu_spice_display_is_running(SimpleSpiceDisplay *ssd);
diff --git a/ui/vnc-auth-sasl.c b/ui/vnc-auth-sasl.c
index 8fba7702c..f3ad75d52 100644
--- a/ui/vnc-auth-sasl.c
+++ b/ui/vnc-auth-sasl.c
@@ -432,9 +432,7 @@ static int protocol_client_auth_sasl_start_len(VncState *vs, uint8_t *data, size
 
 static int protocol_client_auth_sasl_mechname(VncState *vs, uint8_t *data, size_t len)
 {
-    char *mechname = g_malloc(len + 1);
-    strncpy(mechname, (char*)data, len);
-    mechname[len] = '\0';
+    char *mechname = g_strndup((const char *) data, len);
     VNC_DEBUG("Got client mechname '%s' check against '%s'\n",
               mechname, vs->sasl.mechlist);
 
@@ -619,7 +617,6 @@ void start_auth_sasl(VncState *vs)
 
  authabort:
     vnc_client_error(vs);
-    return;
 }
 
 
diff --git a/ui/vnc-enc-hextile-template.h b/ui/vnc-enc-hextile-template.h
index a7310e194..d868d7572 100644
--- a/ui/vnc-enc-hextile-template.h
+++ b/ui/vnc-enc-hextile-template.h
@@ -14,7 +14,7 @@ static void CONCAT(send_hextile_tile_, NAME)(VncState *vs,
                                              int *has_bg, int *has_fg)
 {
     VncDisplay *vd = vs->vd;
-    uint8_t *row = vd->server->data + y * ds_get_linesize(vs->ds) + x * ds_get_bytes_per_pixel(vs->ds);
+    uint8_t *row = vnc_server_fb_ptr(vd, x, y);
     pixel_t *irow = (pixel_t *)row;
     int j, i;
     pixel_t *last_bg = (pixel_t *)last_bg_;
@@ -25,7 +25,7 @@ static void CONCAT(send_hextile_tile_, NAME)(VncState *vs,
     int bg_count = 0;
     int fg_count = 0;
     int flags = 0;
-    uint8_t data[(vs->clientds.pf.bytes_per_pixel + 2) * 16 * 16];
+    uint8_t data[(vs->client_pf.bytes_per_pixel + 2) * 16 * 16];
     int n_data = 0;
     int n_subtiles = 0;
 
@@ -58,7 +58,7 @@ static void CONCAT(send_hextile_tile_, NAME)(VncState *vs,
 	}
 	if (n_colors > 2)
 	    break;
-	irow += ds_get_linesize(vs->ds) / sizeof(pixel_t);
+	irow += vnc_server_fb_stride(vd) / sizeof(pixel_t);
     }
 
     if (n_colors > 1 && fg_count > bg_count) {
@@ -106,7 +106,7 @@ static void CONCAT(send_hextile_tile_, NAME)(VncState *vs,
 		n_data += 2;
 		n_subtiles++;
 	    }
-	    irow += ds_get_linesize(vs->ds) / sizeof(pixel_t);
+	    irow += vnc_server_fb_stride(vd) / sizeof(pixel_t);
 	}
 	break;
     case 3:
@@ -133,7 +133,7 @@ static void CONCAT(send_hextile_tile_, NAME)(VncState *vs,
 		    has_color = 0;
 #ifdef GENERIC
                     vnc_convert_pixel(vs, data + n_data, color);
-                    n_data += vs->clientds.pf.bytes_per_pixel;
+                    n_data += vs->client_pf.bytes_per_pixel;
 #else
 		    memcpy(data + n_data, &color, sizeof(color));
                     n_data += sizeof(pixel_t);
@@ -153,7 +153,7 @@ static void CONCAT(send_hextile_tile_, NAME)(VncState *vs,
 	    if (has_color) {
 #ifdef GENERIC
                 vnc_convert_pixel(vs, data + n_data, color);
-                n_data += vs->clientds.pf.bytes_per_pixel;
+                n_data += vs->client_pf.bytes_per_pixel;
 #else
                 memcpy(data + n_data, &color, sizeof(color));
                 n_data += sizeof(pixel_t);
@@ -162,7 +162,7 @@ static void CONCAT(send_hextile_tile_, NAME)(VncState *vs,
 		n_data += 2;
 		n_subtiles++;
 	    }
-	    irow += ds_get_linesize(vs->ds) / sizeof(pixel_t);
+	    irow += vnc_server_fb_stride(vd) / sizeof(pixel_t);
 	}
 
 	/* A SubrectsColoured subtile invalidates the foreground color */
@@ -190,18 +190,17 @@ static void CONCAT(send_hextile_tile_, NAME)(VncState *vs,
     vnc_write_u8(vs, flags);
     if (n_colors < 4) {
 	if (flags & 0x02)
-	    vs->write_pixels(vs, &vd->server->pf, last_bg, sizeof(pixel_t));
+	    vs->write_pixels(vs, last_bg, sizeof(pixel_t));
 	if (flags & 0x04)
-	    vs->write_pixels(vs, &vd->server->pf, last_fg, sizeof(pixel_t));
+	    vs->write_pixels(vs, last_fg, sizeof(pixel_t));
 	if (n_subtiles) {
 	    vnc_write_u8(vs, n_subtiles);
 	    vnc_write(vs, data, n_data);
 	}
     } else {
 	for (j = 0; j < h; j++) {
-	    vs->write_pixels(vs, &vd->server->pf, row,
-                             w * ds_get_bytes_per_pixel(vs->ds));
-	    row += ds_get_linesize(vs->ds);
+	    vs->write_pixels(vs, row, w * 4);
+	    row += vnc_server_fb_stride(vd);
 	}
     }
 }
diff --git a/ui/vnc-enc-hextile.c b/ui/vnc-enc-hextile.c
index c860dbb2e..2e768fd89 100644
--- a/ui/vnc-enc-hextile.c
+++ b/ui/vnc-enc-hextile.c
@@ -32,31 +32,11 @@ static void hextile_enc_cord(uint8_t *ptr, int x, int y, int w, int h)
     ptr[1] = (((w - 1) & 0x0F) << 4) | ((h - 1) & 0x0F);
 }
 
-#define BPP 8
-#include "vnc-enc-hextile-template.h"
-#undef BPP
-
-#define BPP 16
-#include "vnc-enc-hextile-template.h"
-#undef BPP
-
 #define BPP 32
 #include "vnc-enc-hextile-template.h"
 #undef BPP
 
 #define GENERIC
-#define BPP 8
-#include "vnc-enc-hextile-template.h"
-#undef BPP
-#undef GENERIC
-
-#define GENERIC
-#define BPP 16
-#include "vnc-enc-hextile-template.h"
-#undef BPP
-#undef GENERIC
-
-#define GENERIC
 #define BPP 32
 #include "vnc-enc-hextile-template.h"
 #undef BPP
@@ -68,10 +48,9 @@ int vnc_hextile_send_framebuffer_update(VncState *vs, int x,
     int i, j;
     int has_fg, has_bg;
     uint8_t *last_fg, *last_bg;
-    VncDisplay *vd = vs->vd;
 
-    last_fg = (uint8_t *) g_malloc(vd->server->pf.bytes_per_pixel);
-    last_bg = (uint8_t *) g_malloc(vd->server->pf.bytes_per_pixel);
+    last_fg = (uint8_t *) g_malloc(VNC_SERVER_FB_BYTES);
+    last_bg = (uint8_t *) g_malloc(VNC_SERVER_FB_BYTES);
     has_fg = has_bg = 0;
     for (j = y; j < (y + h); j += 16) {
         for (i = x; i < (x + w); i += 16) {
@@ -89,28 +68,16 @@ int vnc_hextile_send_framebuffer_update(VncState *vs, int x,
 void vnc_hextile_set_pixel_conversion(VncState *vs, int generic)
 {
     if (!generic) {
-        switch (vs->ds->surface->pf.bits_per_pixel) {
-            case 8:
-                vs->hextile.send_tile = send_hextile_tile_8;
-                break;
-            case 16:
-                vs->hextile.send_tile = send_hextile_tile_16;
-                break;
-            case 32:
-                vs->hextile.send_tile = send_hextile_tile_32;
-                break;
+        switch (VNC_SERVER_FB_BITS) {
+        case 32:
+            vs->hextile.send_tile = send_hextile_tile_32;
+            break;
         }
     } else {
-        switch (vs->ds->surface->pf.bits_per_pixel) {
-            case 8:
-                vs->hextile.send_tile = send_hextile_tile_generic_8;
-                break;
-            case 16:
-                vs->hextile.send_tile = send_hextile_tile_generic_16;
-                break;
-            case 32:
-                vs->hextile.send_tile = send_hextile_tile_generic_32;
-                break;
+        switch (VNC_SERVER_FB_BITS) {
+        case 32:
+            vs->hextile.send_tile = send_hextile_tile_generic_32;
+            break;
         }
     }
 }
diff --git a/ui/vnc-enc-tight.c b/ui/vnc-enc-tight.c
index 5d492abe9..9ae4cabff 100644
--- a/ui/vnc-enc-tight.c
+++ b/ui/vnc-enc-tight.c
@@ -124,7 +124,7 @@ static bool tight_can_send_png_rect(VncState *vs, int w, int h)
     }
 
     if (ds_get_bytes_per_pixel(vs->ds) == 1 ||
-        vs->clientds.pf.bytes_per_pixel == 1) {
+        vs->client_pf.bytes_per_pixel == 1) {
         return false;
     }
 
@@ -153,7 +153,7 @@ tight_detect_smooth_image24(VncState *vs, int w, int h)
      * If client is big-endian, color samples begin from the second
      * byte (offset 1) of a 32-bit pixel value.
      */
-    off = !!(vs->clientds.flags & QEMU_BIG_ENDIAN_FLAG);
+    off = vs->client_be;
 
     memset(stats, 0, sizeof (stats));
 
@@ -216,16 +216,16 @@ tight_detect_smooth_image24(VncState *vs, int w, int h)
         unsigned int errors;                                            \
         unsigned char *buf = vs->tight.tight.buffer;                    \
                                                                         \
-        endian = ((vs->clientds.flags & QEMU_BIG_ENDIAN_FLAG) !=        \
-                  (vs->ds->surface->flags & QEMU_BIG_ENDIAN_FLAG));     \
+        endian = 0; /* FIXME: ((vs->clientds.flags & QEMU_BIG_ENDIAN_FLAG) != \
+                      (vs->ds->surface->flags & QEMU_BIG_ENDIAN_FLAG)); */ \
                                                                         \
                                                                         \
-        max[0] = vs->clientds.pf.rmax;                                  \
-        max[1] = vs->clientds.pf.gmax;                                  \
-        max[2] = vs->clientds.pf.bmax;                                  \
-        shift[0] = vs->clientds.pf.rshift;                              \
-        shift[1] = vs->clientds.pf.gshift;                              \
-        shift[2] = vs->clientds.pf.bshift;                              \
+        max[0] = vs->client_pf.rmax;                                  \
+        max[1] = vs->client_pf.gmax;                                  \
+        max[2] = vs->client_pf.bmax;                                  \
+        shift[0] = vs->client_pf.rshift;                              \
+        shift[1] = vs->client_pf.gshift;                              \
+        shift[2] = vs->client_pf.bshift;                              \
                                                                         \
         memset(stats, 0, sizeof(stats));                                \
                                                                         \
@@ -302,7 +302,7 @@ tight_detect_smooth_image(VncState *vs, int w, int h)
     }
 
     if (ds_get_bytes_per_pixel(vs->ds) == 1 ||
-        vs->clientds.pf.bytes_per_pixel == 1 ||
+        vs->client_pf.bytes_per_pixel == 1 ||
         w < VNC_TIGHT_DETECT_MIN_WIDTH || h < VNC_TIGHT_DETECT_MIN_HEIGHT) {
         return 0;
     }
@@ -317,7 +317,7 @@ tight_detect_smooth_image(VncState *vs, int w, int h)
         }
     }
 
-    if (vs->clientds.pf.bytes_per_pixel == 4) {
+    if (vs->client_pf.bytes_per_pixel == 4) {
         if (vs->tight.pixel24) {
             errors = tight_detect_smooth_image24(vs, w, h);
             if (vs->tight.quality != (uint8_t)-1) {
@@ -430,7 +430,7 @@ static int tight_fill_palette(VncState *vs, int x, int y,
         max = 256;
     }
 
-    switch(vs->clientds.pf.bytes_per_pixel) {
+    switch (vs->client_pf.bytes_per_pixel) {
     case 4:
         return tight_fill_palette32(vs, x, y, max, count, bg, fg, palette);
     case 2:
@@ -557,15 +557,15 @@ tight_filter_gradient24(VncState *vs, uint8_t *buf, int w, int h)
     buf32 = (uint32_t *)buf;
     memset(vs->tight.gradient.buffer, 0, w * 3 * sizeof(int));
 
-    if ((vs->clientds.flags & QEMU_BIG_ENDIAN_FLAG) ==
-        (vs->ds->surface->flags & QEMU_BIG_ENDIAN_FLAG)) {
-        shift[0] = vs->clientds.pf.rshift;
-        shift[1] = vs->clientds.pf.gshift;
-        shift[2] = vs->clientds.pf.bshift;
+    if (1 /* FIXME: (vs->clientds.flags & QEMU_BIG_ENDIAN_FLAG) ==
+             (vs->ds->surface->flags & QEMU_BIG_ENDIAN_FLAG) */) {
+        shift[0] = vs->client_pf.rshift;
+        shift[1] = vs->client_pf.gshift;
+        shift[2] = vs->client_pf.bshift;
     } else {
-        shift[0] = 24 - vs->clientds.pf.rshift;
-        shift[1] = 24 - vs->clientds.pf.gshift;
-        shift[2] = 24 - vs->clientds.pf.bshift;
+        shift[0] = 24 - vs->client_pf.rshift;
+        shift[1] = 24 - vs->client_pf.gshift;
+        shift[2] = 24 - vs->client_pf.bshift;
     }
 
     for (y = 0; y < h; y++) {
@@ -615,15 +615,15 @@ tight_filter_gradient24(VncState *vs, uint8_t *buf, int w, int h)
                                                                         \
         memset (vs->tight.gradient.buffer, 0, w * 3 * sizeof(int));     \
                                                                         \
-        endian = ((vs->clientds.flags & QEMU_BIG_ENDIAN_FLAG) !=        \
-                  (vs->ds->surface->flags & QEMU_BIG_ENDIAN_FLAG));     \
+        endian = 0; /* FIXME: ((vs->clientds.flags & QEMU_BIG_ENDIAN_FLAG) != \
+                       (vs->ds->surface->flags & QEMU_BIG_ENDIAN_FLAG)); */ \
                                                                         \
-        max[0] = vs->clientds.pf.rmax;                                  \
-        max[1] = vs->clientds.pf.gmax;                                  \
-        max[2] = vs->clientds.pf.bmax;                                  \
-        shift[0] = vs->clientds.pf.rshift;                              \
-        shift[1] = vs->clientds.pf.gshift;                              \
-        shift[2] = vs->clientds.pf.bshift;                              \
+        max[0] = vs->client_pf.rmax;                                    \
+        max[1] = vs->client_pf.gmax;                                    \
+        max[2] = vs->client_pf.bmax;                                    \
+        shift[0] = vs->client_pf.rshift;                                \
+        shift[1] = vs->client_pf.gshift;                                \
+        shift[2] = vs->client_pf.bshift;                                \
                                                                         \
         for (y = 0; y < h; y++) {                                       \
             for (c = 0; c < 3; c++) {                                   \
@@ -671,56 +671,42 @@ DEFINE_GRADIENT_FILTER_FUNCTION(32)
  * that case new color will be stored in *colorPtr.
  */
 
-#define DEFINE_CHECK_SOLID_FUNCTION(bpp)                                \
-                                                                        \
-    static bool                                                         \
-    check_solid_tile##bpp(VncState *vs, int x, int y, int w, int h,     \
-                          uint32_t* color, bool samecolor)              \
-    {                                                                   \
-        VncDisplay *vd = vs->vd;                                        \
-        uint##bpp##_t *fbptr;                                           \
-        uint##bpp##_t c;                                                \
-        int dx, dy;                                                     \
-                                                                        \
-        fbptr = (uint##bpp##_t *)                                       \
-            (vd->server->data + y * ds_get_linesize(vs->ds) +           \
-             x * ds_get_bytes_per_pixel(vs->ds));                       \
-                                                                        \
-        c = *fbptr;                                                     \
-        if (samecolor && (uint32_t)c != *color) {                       \
-            return false;                                               \
-        }                                                               \
-                                                                        \
-        for (dy = 0; dy < h; dy++) {                                    \
-            for (dx = 0; dx < w; dx++) {                                \
-                if (c != fbptr[dx]) {                                   \
-                    return false;                                       \
-                }                                                       \
-            }                                                           \
-            fbptr = (uint##bpp##_t *)                                   \
-                ((uint8_t *)fbptr + ds_get_linesize(vs->ds));           \
-        }                                                               \
-                                                                        \
-        *color = (uint32_t)c;                                           \
-        return true;                                                    \
+static bool
+check_solid_tile32(VncState *vs, int x, int y, int w, int h,
+                   uint32_t *color, bool samecolor)
+{
+    VncDisplay *vd = vs->vd;
+    uint32_t *fbptr;
+    uint32_t c;
+    int dx, dy;
+
+    fbptr = vnc_server_fb_ptr(vd, x, y);
+
+    c = *fbptr;
+    if (samecolor && (uint32_t)c != *color) {
+        return false;
+    }
+
+    for (dy = 0; dy < h; dy++) {
+        for (dx = 0; dx < w; dx++) {
+            if (c != fbptr[dx]) {
+                return false;
+            }
+        }
+        fbptr = (uint32_t *)
+            ((uint8_t *)fbptr + vnc_server_fb_stride(vd));
     }
 
-DEFINE_CHECK_SOLID_FUNCTION(32)
-DEFINE_CHECK_SOLID_FUNCTION(16)
-DEFINE_CHECK_SOLID_FUNCTION(8)
+    *color = (uint32_t)c;
+    return true;
+}
 
 static bool check_solid_tile(VncState *vs, int x, int y, int w, int h,
                              uint32_t* color, bool samecolor)
 {
-    VncDisplay *vd = vs->vd;
-
-    switch(vd->server->pf.bytes_per_pixel) {
+    switch (VNC_SERVER_FB_BYTES) {
     case 4:
         return check_solid_tile32(vs, x, y, w, h, color, samecolor);
-    case 2:
-        return check_solid_tile16(vs, x, y, w, h, color, samecolor);
-    default:
-        return check_solid_tile8(vs, x, y, w, h, color, samecolor);
     }
 }
 
@@ -906,15 +892,15 @@ static void tight_pack24(VncState *vs, uint8_t *buf, size_t count, size_t *ret)
 
     buf32 = (uint32_t *)buf;
 
-    if ((vs->clientds.flags & QEMU_BIG_ENDIAN_FLAG) ==
-        (vs->ds->surface->flags & QEMU_BIG_ENDIAN_FLAG)) {
-        rshift = vs->clientds.pf.rshift;
-        gshift = vs->clientds.pf.gshift;
-        bshift = vs->clientds.pf.bshift;
+    if (1 /* FIXME: (vs->clientds.flags & QEMU_BIG_ENDIAN_FLAG) ==
+             (vs->ds->surface->flags & QEMU_BIG_ENDIAN_FLAG) */) {
+        rshift = vs->client_pf.rshift;
+        gshift = vs->client_pf.gshift;
+        bshift = vs->client_pf.bshift;
     } else {
-        rshift = 24 - vs->clientds.pf.rshift;
-        gshift = 24 - vs->clientds.pf.gshift;
-        bshift = 24 - vs->clientds.pf.bshift;
+        rshift = 24 - vs->client_pf.rshift;
+        gshift = 24 - vs->client_pf.gshift;
+        bshift = 24 - vs->client_pf.bshift;
     }
 
     if (ret) {
@@ -946,7 +932,7 @@ static int send_full_color_rect(VncState *vs, int x, int y, int w, int h)
         tight_pack24(vs, vs->tight.tight.buffer, w * h, &vs->tight.tight.offset);
         bytes = 3;
     } else {
-        bytes = vs->clientds.pf.bytes_per_pixel;
+        bytes = vs->client_pf.bytes_per_pixel;
     }
 
     bytes = tight_compress_data(vs, stream, w * h * bytes,
@@ -966,7 +952,7 @@ static int send_solid_rect(VncState *vs)
         tight_pack24(vs, vs->tight.tight.buffer, 1, &vs->tight.tight.offset);
         bytes = 3;
     } else {
-        bytes = vs->clientds.pf.bytes_per_pixel;
+        bytes = vs->client_pf.bytes_per_pixel;
     }
 
     vnc_write(vs, vs->tight.tight.buffer, bytes);
@@ -983,7 +969,7 @@ static int send_mono_rect(VncState *vs, int x, int y,
 #ifdef CONFIG_VNC_PNG
     if (tight_can_send_png_rect(vs, w, h)) {
         int ret;
-        int bpp = vs->clientds.pf.bytes_per_pixel * 8;
+        int bpp = vs->client_pf.bytes_per_pixel * 8;
         VncPalette *palette = palette_new(2, bpp);
 
         palette_put(palette, bg);
@@ -1000,7 +986,7 @@ static int send_mono_rect(VncState *vs, int x, int y,
     vnc_write_u8(vs, VNC_TIGHT_FILTER_PALETTE);
     vnc_write_u8(vs, 1);
 
-    switch(vs->clientds.pf.bytes_per_pixel) {
+    switch (vs->client_pf.bytes_per_pixel) {
     case 4:
     {
         uint32_t buf[2] = {bg, fg};
@@ -1043,7 +1029,7 @@ static void write_palette(int idx, uint32_t color, void *opaque)
 {
     struct palette_cb_priv *priv = opaque;
     VncState *vs = priv->vs;
-    uint32_t bytes = vs->clientds.pf.bytes_per_pixel;
+    uint32_t bytes = vs->client_pf.bytes_per_pixel;
 
     if (bytes == 4) {
         ((uint32_t*)priv->header)[idx] = color;
@@ -1058,8 +1044,9 @@ static bool send_gradient_rect(VncState *vs, int x, int y, int w, int h)
     int level = tight_conf[vs->tight.compression].gradient_zlib_level;
     ssize_t bytes;
 
-    if (vs->clientds.pf.bytes_per_pixel == 1)
+    if (vs->client_pf.bytes_per_pixel == 1) {
         return send_full_color_rect(vs, x, y, w, h);
+    }
 
     vnc_write_u8(vs, (stream | VNC_TIGHT_EXPLICIT_FILTER) << 4);
     vnc_write_u8(vs, VNC_TIGHT_FILTER_GRADIENT);
@@ -1069,7 +1056,7 @@ static bool send_gradient_rect(VncState *vs, int x, int y, int w, int h)
     if (vs->tight.pixel24) {
         tight_filter_gradient24(vs, vs->tight.tight.buffer, w, h);
         bytes = 3;
-    } else if (vs->clientds.pf.bytes_per_pixel == 4) {
+    } else if (vs->client_pf.bytes_per_pixel == 4) {
         tight_filter_gradient32(vs, (uint32_t *)vs->tight.tight.buffer, w, h);
         bytes = 4;
     } else {
@@ -1107,7 +1094,7 @@ static int send_palette_rect(VncState *vs, int x, int y,
     vnc_write_u8(vs, VNC_TIGHT_FILTER_PALETTE);
     vnc_write_u8(vs, colors - 1);
 
-    switch(vs->clientds.pf.bytes_per_pixel) {
+    switch (vs->client_pf.bytes_per_pixel) {
     case 4:
     {
         size_t old_offset, offset;
@@ -1148,79 +1135,6 @@ static int send_palette_rect(VncState *vs, int x, int y,
     return (bytes >= 0);
 }
 
-#if defined(CONFIG_VNC_JPEG) || defined(CONFIG_VNC_PNG)
-static void rgb_prepare_row24(VncState *vs, uint8_t *dst, int x, int y,
-                              int count)
-{
-    VncDisplay *vd = vs->vd;
-    uint32_t *fbptr;
-    uint32_t pix;
-
-    fbptr = (uint32_t *)(vd->server->data + y * ds_get_linesize(vs->ds) +
-                         x * ds_get_bytes_per_pixel(vs->ds));
-
-    while (count--) {
-        pix = *fbptr++;
-        *dst++ = (uint8_t)(pix >> vs->ds->surface->pf.rshift);
-        *dst++ = (uint8_t)(pix >> vs->ds->surface->pf.gshift);
-        *dst++ = (uint8_t)(pix >> vs->ds->surface->pf.bshift);
-    }
-}
-
-#define DEFINE_RGB_GET_ROW_FUNCTION(bpp)                                \
-                                                                        \
-    static void                                                         \
-    rgb_prepare_row##bpp(VncState *vs, uint8_t *dst,                    \
-                         int x, int y, int count)                       \
-    {                                                                   \
-        VncDisplay *vd = vs->vd;                                        \
-        uint##bpp##_t *fbptr;                                           \
-        uint##bpp##_t pix;                                              \
-        int r, g, b;                                                    \
-                                                                        \
-        fbptr = (uint##bpp##_t *)                                       \
-            (vd->server->data + y * ds_get_linesize(vs->ds) +           \
-             x * ds_get_bytes_per_pixel(vs->ds));                       \
-                                                                        \
-        while (count--) {                                               \
-            pix = *fbptr++;                                             \
-                                                                        \
-            r = (int)((pix >> vs->ds->surface->pf.rshift)               \
-                      & vs->ds->surface->pf.rmax);                      \
-            g = (int)((pix >> vs->ds->surface->pf.gshift)               \
-                      & vs->ds->surface->pf.gmax);                      \
-            b = (int)((pix >> vs->ds->surface->pf.bshift)               \
-                      & vs->ds->surface->pf.bmax);                      \
-                                                                        \
-            *dst++ = (uint8_t)((r * 255 + vs->ds->surface->pf.rmax / 2) \
-                               / vs->ds->surface->pf.rmax);             \
-            *dst++ = (uint8_t)((g * 255 + vs->ds->surface->pf.gmax / 2) \
-                               / vs->ds->surface->pf.gmax);             \
-            *dst++ = (uint8_t)((b * 255 + vs->ds->surface->pf.bmax / 2) \
-                               / vs->ds->surface->pf.bmax);             \
-        }                                                               \
-    }
-
-DEFINE_RGB_GET_ROW_FUNCTION(16)
-DEFINE_RGB_GET_ROW_FUNCTION(32)
-
-static void rgb_prepare_row(VncState *vs, uint8_t *dst, int x, int y,
-                            int count)
-{
-    if (ds_get_bytes_per_pixel(vs->ds) == 4) {
-        if (vs->ds->surface->pf.rmax == 0xFF &&
-            vs->ds->surface->pf.gmax == 0xFF &&
-            vs->ds->surface->pf.bmax == 0xFF) {
-            rgb_prepare_row24(vs, dst, x, y, count);
-        } else {
-            rgb_prepare_row32(vs, dst, x, y, count);
-        }
-    } else {
-        rgb_prepare_row16(vs, dst, x, y, count);
-    }
-}
-#endif /* CONFIG_VNC_JPEG or CONFIG_VNC_PNG */
-
 /*
  * JPEG compression stuff.
  */
@@ -1265,6 +1179,7 @@ static int send_jpeg_rect(VncState *vs, int x, int y, int w, int h, int quality)
     struct jpeg_compress_struct cinfo;
     struct jpeg_error_mgr jerr;
     struct jpeg_destination_mgr manager;
+    pixman_image_t *linebuf;
     JSAMPROW row[1];
     uint8_t *buf;
     int dy;
@@ -1293,13 +1208,14 @@ static int send_jpeg_rect(VncState *vs, int x, int y, int w, int h, int quality)
 
     jpeg_start_compress(&cinfo, true);
 
-    buf = g_malloc(w * 3);
+    linebuf = qemu_pixman_linebuf_create(PIXMAN_BE_r8g8b8, w);
+    buf = (uint8_t *)pixman_image_get_data(linebuf);
     row[0] = buf;
     for (dy = 0; dy < h; dy++) {
-        rgb_prepare_row(vs, buf, x, y + dy, w);
+        qemu_pixman_linebuf_fill(linebuf, vs->vd->server, w, dy);
         jpeg_write_scanlines(&cinfo, row, 1);
     }
-    g_free(buf);
+    qemu_pixman_image_unref(linebuf);
 
     jpeg_finish_compress(&cinfo);
     jpeg_destroy_compress(&cinfo);
@@ -1326,23 +1242,23 @@ static void write_png_palette(int idx, uint32_t pix, void *opaque)
 
     if (vs->tight.pixel24)
     {
-        color->red = (pix >> vs->clientds.pf.rshift) & vs->clientds.pf.rmax;
-        color->green = (pix >> vs->clientds.pf.gshift) & vs->clientds.pf.gmax;
-        color->blue = (pix >> vs->clientds.pf.bshift) & vs->clientds.pf.bmax;
+        color->red = (pix >> vs->client_pf.rshift) & vs->client_pf.rmax;
+        color->green = (pix >> vs->client_pf.gshift) & vs->client_pf.gmax;
+        color->blue = (pix >> vs->client_pf.bshift) & vs->client_pf.bmax;
     }
     else
     {
         int red, green, blue;
 
-        red = (pix >> vs->clientds.pf.rshift) & vs->clientds.pf.rmax;
-        green = (pix >> vs->clientds.pf.gshift) & vs->clientds.pf.gmax;
-        blue = (pix >> vs->clientds.pf.bshift) & vs->clientds.pf.bmax;
-        color->red = ((red * 255 + vs->clientds.pf.rmax / 2) /
-                      vs->clientds.pf.rmax);
-        color->green = ((green * 255 + vs->clientds.pf.gmax / 2) /
-                        vs->clientds.pf.gmax);
-        color->blue = ((blue * 255 + vs->clientds.pf.bmax / 2) /
-                       vs->clientds.pf.bmax);
+        red = (pix >> vs->client_pf.rshift) & vs->client_pf.rmax;
+        green = (pix >> vs->client_pf.gshift) & vs->client_pf.gmax;
+        blue = (pix >> vs->client_pf.bshift) & vs->client_pf.bmax;
+        color->red = ((red * 255 + vs->client_pf.rmax / 2) /
+                      vs->client_pf.rmax);
+        color->green = ((green * 255 + vs->client_pf.gmax / 2) /
+                        vs->client_pf.gmax);
+        color->blue = ((blue * 255 + vs->client_pf.bmax / 2) /
+                       vs->client_pf.bmax);
     }
 }
 
@@ -1378,6 +1294,7 @@ static int send_png_rect(VncState *vs, int x, int y, int w, int h,
     png_structp png_ptr;
     png_infop info_ptr;
     png_colorp png_palette = NULL;
+    pixman_image_t *linebuf;
     int level = tight_png_conf[vs->tight.compression].png_zlib_level;
     int filters = tight_png_conf[vs->tight.compression].png_filters;
     uint8_t *buf;
@@ -1422,7 +1339,7 @@ static int send_png_rect(VncState *vs, int x, int y, int w, int h,
 
         png_set_PLTE(png_ptr, info_ptr, png_palette, palette_size(palette));
 
-        if (vs->clientds.pf.bytes_per_pixel == 4) {
+        if (vs->client_pf.bytes_per_pixel == 4) {
             tight_encode_indexed_rect32(vs->tight.tight.buffer, w * h, palette);
         } else {
             tight_encode_indexed_rect16(vs->tight.tight.buffer, w * h, palette);
@@ -1432,17 +1349,18 @@ static int send_png_rect(VncState *vs, int x, int y, int w, int h,
     png_write_info(png_ptr, info_ptr);
 
     buffer_reserve(&vs->tight.png, 2048);
-    buf = g_malloc(w * 3);
+    linebuf = qemu_pixman_linebuf_create(PIXMAN_BE_r8g8b8, w);
+    buf = (uint8_t *)pixman_image_get_data(linebuf);
     for (dy = 0; dy < h; dy++)
     {
         if (color_type == PNG_COLOR_TYPE_PALETTE) {
             memcpy(buf, vs->tight.tight.buffer + (dy * w), w);
         } else {
-            rgb_prepare_row(vs, buf, x, y + dy, w);
+            qemu_pixman_linebuf_fill(linebuf, vs->vd->server, w, dy);
         }
         png_write_row(png_ptr, buf);
     }
-    g_free(buf);
+    qemu_pixman_image_unref(linebuf);
 
     png_write_end(png_ptr, NULL);
 
@@ -1713,8 +1631,8 @@ static int tight_send_framebuffer_update(VncState *vs, int x, int y,
 {
     int max_rows;
 
-    if (vs->clientds.pf.bytes_per_pixel == 4 && vs->clientds.pf.rmax == 0xFF &&
-        vs->clientds.pf.bmax == 0xFF && vs->clientds.pf.gmax == 0xFF) {
+    if (vs->client_pf.bytes_per_pixel == 4 && vs->client_pf.rmax == 0xFF &&
+        vs->client_pf.bmax == 0xFF && vs->client_pf.gmax == 0xFF) {
         vs->tight.pixel24 = true;
     } else {
         vs->tight.pixel24 = false;
diff --git a/ui/vnc-enc-zrle.c b/ui/vnc-enc-zrle.c
index 917d384f5..ed3b48465 100644
--- a/ui/vnc-enc-zrle.c
+++ b/ui/vnc-enc-zrle.c
@@ -255,7 +255,7 @@ static void zrle_write_u8(VncState *vs, uint8_t value)
 static int zrle_send_framebuffer_update(VncState *vs, int x, int y,
                                         int w, int h)
 {
-    bool be = !!(vs->clientds.flags & QEMU_BIG_ENDIAN_FLAG);
+    bool be = vs->client_be;
     size_t bytes;
     int zywrle_level;
 
@@ -277,13 +277,13 @@ static int zrle_send_framebuffer_update(VncState *vs, int x, int y,
 
     vnc_zrle_start(vs);
 
-    switch(vs->clientds.pf.bytes_per_pixel) {
+    switch (vs->client_pf.bytes_per_pixel) {
     case 1:
         zrle_encode_8ne(vs, x, y, w, h, zywrle_level);
         break;
 
     case 2:
-        if (vs->clientds.pf.gmax > 0x1F) {
+        if (vs->client_pf.gmax > 0x1F) {
             if (be) {
                 zrle_encode_16be(vs, x, y, w, h, zywrle_level);
             } else {
@@ -304,13 +304,13 @@ static int zrle_send_framebuffer_update(VncState *vs, int x, int y,
         bool fits_in_ms3bytes;
 
         fits_in_ls3bytes =
-            ((vs->clientds.pf.rmax << vs->clientds.pf.rshift) < (1 << 24) &&
-             (vs->clientds.pf.gmax << vs->clientds.pf.gshift) < (1 << 24) &&
-             (vs->clientds.pf.bmax << vs->clientds.pf.bshift) < (1 << 24));
+            ((vs->client_pf.rmax << vs->client_pf.rshift) < (1 << 24) &&
+             (vs->client_pf.gmax << vs->client_pf.gshift) < (1 << 24) &&
+             (vs->client_pf.bmax << vs->client_pf.bshift) < (1 << 24));
 
-        fits_in_ms3bytes = (vs->clientds.pf.rshift > 7 &&
-                            vs->clientds.pf.gshift > 7 &&
-                            vs->clientds.pf.bshift > 7);
+        fits_in_ms3bytes = (vs->client_pf.rshift > 7 &&
+                            vs->client_pf.gshift > 7 &&
+                            vs->client_pf.bshift > 7);
 
         if ((fits_in_ls3bytes && !be) || (fits_in_ms3bytes && be)) {
             if (be) {
diff --git a/ui/vnc-jobs.c b/ui/vnc-jobs.c
index 087b84d31..57c091683 100644
--- a/ui/vnc-jobs.c
+++ b/ui/vnc-jobs.c
@@ -33,21 +33,21 @@
 /*
  * Locking:
  *
- * There is three levels of locking:
+ * There are three levels of locking:
  * - jobs queue lock: for each operation on the queue (push, pop, isEmpty?)
  * - VncDisplay global lock: mainly used for framebuffer updates to avoid
  *                      screen corruption if the framebuffer is updated
- *			while the worker is doing something.
+ *                      while the worker is doing something.
  * - VncState::output lock: used to make sure the output buffer is not corrupted
- * 		   	 if two threads try to write on it at the same time
+ *                          if two threads try to write on it at the same time
  *
- * While the VNC worker thread is working, the VncDisplay global lock is hold
- * to avoid screen corruptions (this does not block vnc_refresh() because it
- * uses trylock()) but the output lock is not hold because the thread work on
+ * While the VNC worker thread is working, the VncDisplay global lock is held
+ * to avoid screen corruption (this does not block vnc_refresh() because it
+ * uses trylock()) but the output lock is not held because the thread works on
  * its own output buffer.
  * When the encoding job is done, the worker thread will hold the output lock
  * and copy its output buffer in vs->output.
-*/
+ */
 
 struct VncJobQueue {
     QemuCond cond;
@@ -62,7 +62,7 @@ typedef struct VncJobQueue VncJobQueue;
 
 /*
  * We use a single global queue, but most of the functions are
- * already reetrant, so we can easilly add more than one encoding thread
+ * already reentrant, so we can easily add more than one encoding thread
  */
 static VncJobQueue *queue;
 
@@ -187,7 +187,8 @@ static void vnc_async_encoding_start(VncState *orig, VncState *local)
     local->vd = orig->vd;
     local->lossy_rect = orig->lossy_rect;
     local->write_pixels = orig->write_pixels;
-    local->clientds = orig->clientds;
+    local->client_pf = orig->client_pf;
+    local->client_be = orig->client_be;
     local->tight = orig->tight;
     local->zlib = orig->zlib;
     local->hextile = orig->hextile;
@@ -320,6 +321,11 @@ static void *vnc_worker_thread(void *arg)
     return NULL;
 }
 
+static bool vnc_worker_thread_running(void)
+{
+    return queue; /* Check global queue */
+}
+
 void vnc_start_worker_thread(void)
 {
     VncJobQueue *q;
@@ -332,11 +338,6 @@ void vnc_start_worker_thread(void)
     queue = q; /* Set global queue */
 }
 
-bool vnc_worker_thread_running(void)
-{
-    return queue; /* Check global queue */
-}
-
 void vnc_stop_worker_thread(void)
 {
     if (!vnc_worker_thread_running())
diff --git a/ui/vnc-jobs.h b/ui/vnc-jobs.h
index 86e6d888c..31da103fa 100644
--- a/ui/vnc-jobs.h
+++ b/ui/vnc-jobs.h
@@ -40,7 +40,6 @@ void vnc_jobs_join(VncState *vs);
 
 void vnc_jobs_consume_buffer(VncState *vs);
 void vnc_start_worker_thread(void);
-bool vnc_worker_thread_running(void);
 void vnc_stop_worker_thread(void);
 
 /* Locks */
diff --git a/ui/vnc-palette.h b/ui/vnc-palette.h
index 3260885ff..b82dc5db9 100644
--- a/ui/vnc-palette.h
+++ b/ui/vnc-palette.h
@@ -32,6 +32,7 @@
 #include "qlist.h"
 #include "qemu-queue.h"
 #include <stdint.h>
+#include <stdbool.h>
 
 #define VNC_PALETTE_HASH_SIZE 256
 #define VNC_PALETTE_MAX_SIZE  256
diff --git a/ui/vnc-tls.c b/ui/vnc-tls.c
index 3aaa93928..a7f7d07ac 100644
--- a/ui/vnc-tls.c
+++ b/ui/vnc-tls.c
@@ -49,7 +49,7 @@ static int vnc_tls_initialize(void)
     if (gnutls_global_init () < 0)
         return 0;
 
-    /* XXX ought to re-generate diffie-hellmen params periodically */
+    /* XXX ought to re-generate diffie-hellman params periodically */
     if (gnutls_dh_params_init (&dh_params) < 0)
         return 0;
     if (gnutls_dh_params_generate2 (dh_params, DH_BITS) < 0)
diff --git a/ui/vnc.c b/ui/vnc.c
index 385e345c3..ba303626a 100644
--- a/ui/vnc.c
+++ b/ui/vnc.c
@@ -372,6 +372,10 @@ VncInfo *qmp_query_vnc(Error **errp)
             }
         }
 
+        if (vnc_display->lsock == -1) {
+            return info;
+        }
+
         if (getsockname(vnc_display->lsock, (struct sockaddr *)&sa,
                         &salen) == -1) {
             error_set(errp, QERR_UNDEFINED_ERROR);
@@ -432,6 +436,8 @@ static void vnc_dpy_update(DisplayState *ds, int x, int y, int w, int h)
     int i;
     VncDisplay *vd = ds->opaque;
     struct VncSurface *s = &vd->guest;
+    int width = ds_get_width(ds);
+    int height = ds_get_height(ds);
 
     h += y;
 
@@ -442,10 +448,10 @@ static void vnc_dpy_update(DisplayState *ds, int x, int y, int w, int h)
     w += (x % 16);
     x -= (x % 16);
 
-    x = MIN(x, s->ds->width);
-    y = MIN(y, s->ds->height);
-    w = MIN(x + w, s->ds->width) - x;
-    h = MIN(h, s->ds->height);
+    x = MIN(x, width);
+    y = MIN(y, height);
+    w = MIN(x + w, width) - x;
+    h = MIN(h, height);
 
     for (; y < h; y++)
         for (i = 0; i < w; i += 16)
@@ -475,12 +481,12 @@ void buffer_reserve(Buffer *buffer, size_t len)
     }
 }
 
-int buffer_empty(Buffer *buffer)
+static int buffer_empty(Buffer *buffer)
 {
     return buffer->offset == 0;
 }
 
-uint8_t *buffer_end(Buffer *buffer)
+static uint8_t *buffer_end(Buffer *buffer)
 {
     return buffer->buffer + buffer->offset;
 }
@@ -546,6 +552,21 @@ static void vnc_abort_display_jobs(VncDisplay *vd)
     }
 }
 
+int vnc_server_fb_stride(VncDisplay *vd)
+{
+    return pixman_image_get_stride(vd->server);
+}
+
+void *vnc_server_fb_ptr(VncDisplay *vd, int x, int y)
+{
+    uint8_t *ptr;
+
+    ptr  = (uint8_t *)pixman_image_get_data(vd->server);
+    ptr += y * vnc_server_fb_stride(vd);
+    ptr += x * VNC_SERVER_FB_BYTES;
+    return ptr;
+}
+
 static void vnc_dpy_resize(DisplayState *ds)
 {
     VncDisplay *vd = ds->opaque;
@@ -554,20 +575,20 @@ static void vnc_dpy_resize(DisplayState *ds)
     vnc_abort_display_jobs(vd);
 
     /* server surface */
-    if (!vd->server)
-        vd->server = g_malloc0(sizeof(*vd->server));
-    if (vd->server->data)
-        g_free(vd->server->data);
-    *(vd->server) = *(ds->surface);
-    vd->server->data = g_malloc0(vd->server->linesize *
-                                    vd->server->height);
+    qemu_pixman_image_unref(vd->server);
+    vd->server = pixman_image_create_bits(VNC_SERVER_FB_FORMAT,
+                                          ds_get_width(ds),
+                                          ds_get_height(ds),
+                                          NULL, 0);
 
     /* guest surface */
-    if (!vd->guest.ds)
-        vd->guest.ds = g_malloc0(sizeof(*vd->guest.ds));
+#if 0 /* FIXME */
     if (ds_get_bytes_per_pixel(ds) != vd->guest.ds->pf.bytes_per_pixel)
         console_color_init(ds);
-    *(vd->guest.ds) = *(ds->surface);
+#endif
+    qemu_pixman_image_unref(vd->guest.fb);
+    vd->guest.fb = pixman_image_ref(ds->surface->image);
+    vd->guest.format = ds->surface->format;
     memset(vd->guest.dirty, 0xFF, sizeof(vd->guest.dirty));
 
     QTAILQ_FOREACH(vs, &vd->clients, next) {
@@ -581,7 +602,7 @@ static void vnc_dpy_resize(DisplayState *ds)
 }
 
 /* fastest code */
-static void vnc_write_pixels_copy(VncState *vs, struct PixelFormat *pf,
+static void vnc_write_pixels_copy(VncState *vs,
                                   void *pixels, int size)
 {
     vnc_write(vs, pixels, size);
@@ -591,23 +612,23 @@ static void vnc_write_pixels_copy(VncState *vs, struct PixelFormat *pf,
 void vnc_convert_pixel(VncState *vs, uint8_t *buf, uint32_t v)
 {
     uint8_t r, g, b;
-    VncDisplay *vd = vs->vd;
 
-    r = ((((v & vd->server->pf.rmask) >> vd->server->pf.rshift) << vs->clientds.pf.rbits) >>
-        vd->server->pf.rbits);
-    g = ((((v & vd->server->pf.gmask) >> vd->server->pf.gshift) << vs->clientds.pf.gbits) >>
-        vd->server->pf.gbits);
-    b = ((((v & vd->server->pf.bmask) >> vd->server->pf.bshift) << vs->clientds.pf.bbits) >>
-        vd->server->pf.bbits);
-    v = (r << vs->clientds.pf.rshift) |
-        (g << vs->clientds.pf.gshift) |
-        (b << vs->clientds.pf.bshift);
-    switch(vs->clientds.pf.bytes_per_pixel) {
+#if VNC_SERVER_FB_FORMAT == PIXMAN_FORMAT(32, PIXMAN_TYPE_ARGB, 0, 8, 8, 8)
+    r = (((v & 0x00ff0000) >> 16) << vs->client_pf.rbits) >> 8;
+    g = (((v & 0x0000ff00) >>  8) << vs->client_pf.gbits) >> 8;
+    b = (((v & 0x000000ff) >>  0) << vs->client_pf.bbits) >> 8;
+#else
+# error need some bits here if you change VNC_SERVER_FB_FORMAT
+#endif
+    v = (r << vs->client_pf.rshift) |
+        (g << vs->client_pf.gshift) |
+        (b << vs->client_pf.bshift);
+    switch (vs->client_pf.bytes_per_pixel) {
     case 1:
         buf[0] = v;
         break;
     case 2:
-        if (vs->clientds.flags & QEMU_BIG_ENDIAN_FLAG) {
+        if (vs->client_be) {
             buf[0] = v >> 8;
             buf[1] = v;
         } else {
@@ -617,7 +638,7 @@ void vnc_convert_pixel(VncState *vs, uint8_t *buf, uint32_t v)
         break;
     default:
     case 4:
-        if (vs->clientds.flags & QEMU_BIG_ENDIAN_FLAG) {
+        if (vs->client_be) {
             buf[0] = v >> 24;
             buf[1] = v >> 16;
             buf[2] = v >> 8;
@@ -632,37 +653,19 @@ void vnc_convert_pixel(VncState *vs, uint8_t *buf, uint32_t v)
     }
 }
 
-static void vnc_write_pixels_generic(VncState *vs, struct PixelFormat *pf,
+static void vnc_write_pixels_generic(VncState *vs,
                                      void *pixels1, int size)
 {
     uint8_t buf[4];
 
-    if (pf->bytes_per_pixel == 4) {
+    if (VNC_SERVER_FB_BYTES == 4) {
         uint32_t *pixels = pixels1;
         int n, i;
         n = size >> 2;
-        for(i = 0; i < n; i++) {
+        for (i = 0; i < n; i++) {
             vnc_convert_pixel(vs, buf, pixels[i]);
-            vnc_write(vs, buf, vs->clientds.pf.bytes_per_pixel);
+            vnc_write(vs, buf, vs->client_pf.bytes_per_pixel);
         }
-    } else if (pf->bytes_per_pixel == 2) {
-        uint16_t *pixels = pixels1;
-        int n, i;
-        n = size >> 1;
-        for(i = 0; i < n; i++) {
-            vnc_convert_pixel(vs, buf, pixels[i]);
-            vnc_write(vs, buf, vs->clientds.pf.bytes_per_pixel);
-        }
-    } else if (pf->bytes_per_pixel == 1) {
-        uint8_t *pixels = pixels1;
-        int n, i;
-        n = size;
-        for(i = 0; i < n; i++) {
-            vnc_convert_pixel(vs, buf, pixels[i]);
-            vnc_write(vs, buf, vs->clientds.pf.bytes_per_pixel);
-        }
-    } else {
-        fprintf(stderr, "vnc_write_pixels_generic: VncState color depth not supported\n");
     }
 }
 
@@ -672,10 +675,10 @@ int vnc_raw_send_framebuffer_update(VncState *vs, int x, int y, int w, int h)
     uint8_t *row;
     VncDisplay *vd = vs->vd;
 
-    row = vd->server->data + y * ds_get_linesize(vs->ds) + x * ds_get_bytes_per_pixel(vs->ds);
+    row = vnc_server_fb_ptr(vd, x, y);
     for (i = 0; i < h; i++) {
-        vs->write_pixels(vs, &vd->server->pf, row, w * ds_get_bytes_per_pixel(vs->ds));
-        row += ds_get_linesize(vs->ds);
+        vs->write_pixels(vs, row, w * VNC_SERVER_FB_BYTES);
+        row += vnc_server_fb_stride(vd);
     }
     return 1;
 }
@@ -732,7 +735,7 @@ static void vnc_dpy_copy(DisplayState *ds, int src_x, int src_y, int dst_x, int
     VncState *vs, *vn;
     uint8_t *src_row;
     uint8_t *dst_row;
-    int i,x,y,pitch,depth,inc,w_lim,s;
+    int i, x, y, pitch, inc, w_lim, s;
     int cmp_bytes;
 
     vnc_refresh_server_surface(vd);
@@ -745,10 +748,9 @@ static void vnc_dpy_copy(DisplayState *ds, int src_x, int src_y, int dst_x, int
     }
 
     /* do bitblit op on the local surface too */
-    pitch = ds_get_linesize(vd->ds);
-    depth = ds_get_bytes_per_pixel(vd->ds);
-    src_row = vd->server->data + pitch * src_y + depth * src_x;
-    dst_row = vd->server->data + pitch * dst_y + depth * dst_x;
+    pitch = vnc_server_fb_stride(vd);
+    src_row = vnc_server_fb_ptr(vd, src_x, src_y);
+    dst_row = vnc_server_fb_ptr(vd, dst_x, dst_y);
     y = dst_y;
     inc = 1;
     if (dst_y > src_y) {
@@ -776,7 +778,7 @@ static void vnc_dpy_copy(DisplayState *ds, int src_x, int src_y, int dst_x, int
             } else {
                 s = 16;
             }
-            cmp_bytes = s * depth;
+            cmp_bytes = s * VNC_SERVER_FB_BYTES;
             if (memcmp(src_row, dst_row, cmp_bytes) == 0)
                 continue;
             memmove(dst_row, src_row, cmp_bytes);
@@ -786,8 +788,8 @@ static void vnc_dpy_copy(DisplayState *ds, int src_x, int src_y, int dst_x, int
                 }
             }
         }
-        src_row += pitch - w * depth;
-        dst_row += pitch - w * depth;
+        src_row += pitch - w * VNC_SERVER_FB_BYTES;
+        dst_row += pitch - w * VNC_SERVER_FB_BYTES;
         y += inc;
     }
 
@@ -798,7 +800,7 @@ static void vnc_dpy_copy(DisplayState *ds, int src_x, int src_y, int dst_x, int
     }
 }
 
-static void vnc_mouse_set(int x, int y, int visible)
+static void vnc_mouse_set(DisplayState *ds, int x, int y, int visible)
 {
     /* can we ask the client(s) to move the pointer ??? */
 }
@@ -806,7 +808,6 @@ static void vnc_mouse_set(int x, int y, int visible)
 static int vnc_cursor_define(VncState *vs)
 {
     QEMUCursor *c = vs->vd->cursor;
-    PixelFormat pf = qemu_default_pixelformat(32);
     int isize;
 
     if (vnc_has_feature(vs, VNC_FEATURE_RICH_CURSOR)) {
@@ -816,8 +817,8 @@ static int vnc_cursor_define(VncState *vs)
         vnc_write_u16(vs, 1);  /*  # of rects  */
         vnc_framebuffer_update(vs, c->hot_x, c->hot_y, c->width, c->height,
                                VNC_ENCODING_RICH_CURSOR);
-        isize = c->width * c->height * vs->clientds.pf.bytes_per_pixel;
-        vnc_write_pixels_generic(vs, &pf, c->data, isize);
+        isize = c->width * c->height * vs->client_pf.bytes_per_pixel;
+        vnc_write_pixels_generic(vs, c->data, isize);
         vnc_write(vs, vs->vd->cursor_mask, vs->vd->cursor_msize);
         vnc_unlock_output(vs);
         return 0;
@@ -825,7 +826,7 @@ static int vnc_cursor_define(VncState *vs)
     return -1;
 }
 
-static void vnc_dpy_cursor_define(QEMUCursor *c)
+static void vnc_dpy_cursor_define(DisplayState *ds, QEMUCursor *c)
 {
     VncDisplay *vd = vnc_display;
     VncState *vs;
@@ -894,8 +895,8 @@ static int vnc_update_client(VncState *vs, int has_dirty)
          */
         job = vnc_job_new(vs);
 
-        width = MIN(vd->server->width, vs->client_width);
-        height = MIN(vd->server->height, vs->client_height);
+        width = MIN(pixman_image_get_width(vd->server), vs->client_width);
+        height = MIN(pixman_image_get_height(vd->server), vs->client_height);
 
         for (y = 0; y < height; y++) {
             int x;
@@ -1372,17 +1373,17 @@ void vnc_flush(VncState *vs)
     vnc_unlock_output(vs);
 }
 
-uint8_t read_u8(uint8_t *data, size_t offset)
+static uint8_t read_u8(uint8_t *data, size_t offset)
 {
     return data[offset];
 }
 
-uint16_t read_u16(uint8_t *data, size_t offset)
+static uint16_t read_u16(uint8_t *data, size_t offset)
 {
     return ((data[offset] & 0xFF) << 8) | (data[offset + 1] & 0xFF);
 }
 
-int32_t read_s32(uint8_t *data, size_t offset)
+static int32_t read_s32(uint8_t *data, size_t offset)
 {
     return (int32_t)((data[offset] << 24) | (data[offset + 1] << 16) |
                      (data[offset + 2] << 8) | data[offset + 3]);
@@ -1802,10 +1803,12 @@ static void set_encodings(VncState *vs, int32_t *encodings, size_t n_encodings)
             vs->features |= VNC_FEATURE_TIGHT_MASK;
             vs->vnc_encoding = enc;
             break;
+#ifdef CONFIG_VNC_PNG
         case VNC_ENCODING_TIGHT_PNG:
             vs->features |= VNC_FEATURE_TIGHT_PNG_MASK;
             vs->vnc_encoding = enc;
             break;
+#endif
         case VNC_ENCODING_ZLIB:
             vs->features |= VNC_FEATURE_ZLIB_MASK;
             vs->vnc_encoding = enc;
@@ -1855,9 +1858,9 @@ static void set_encodings(VncState *vs, int32_t *encodings, size_t n_encodings)
 
 static void set_pixel_conversion(VncState *vs)
 {
-    if ((vs->clientds.flags & QEMU_BIG_ENDIAN_FLAG) ==
-        (vs->ds->surface->flags & QEMU_BIG_ENDIAN_FLAG) && 
-        !memcmp(&(vs->clientds.pf), &(vs->ds->surface->pf), sizeof(PixelFormat))) {
+    pixman_format_code_t fmt = qemu_pixman_get_format(&vs->client_pf);
+
+    if (fmt == VNC_SERVER_FB_FORMAT) {
         vs->write_pixels = vnc_write_pixels_copy;
         vnc_hextile_set_pixel_conversion(vs, 0);
     } else {
@@ -1877,23 +1880,22 @@ static void set_pixel_format(VncState *vs,
         return;
     }
 
-    vs->clientds = *(vs->vd->guest.ds);
-    vs->clientds.pf.rmax = red_max;
-    vs->clientds.pf.rbits = hweight_long(red_max);
-    vs->clientds.pf.rshift = red_shift;
-    vs->clientds.pf.rmask = red_max << red_shift;
-    vs->clientds.pf.gmax = green_max;
-    vs->clientds.pf.gbits = hweight_long(green_max);
-    vs->clientds.pf.gshift = green_shift;
-    vs->clientds.pf.gmask = green_max << green_shift;
-    vs->clientds.pf.bmax = blue_max;
-    vs->clientds.pf.bbits = hweight_long(blue_max);
-    vs->clientds.pf.bshift = blue_shift;
-    vs->clientds.pf.bmask = blue_max << blue_shift;
-    vs->clientds.pf.bits_per_pixel = bits_per_pixel;
-    vs->clientds.pf.bytes_per_pixel = bits_per_pixel / 8;
-    vs->clientds.pf.depth = bits_per_pixel == 32 ? 24 : bits_per_pixel;
-    vs->clientds.flags = big_endian_flag ? QEMU_BIG_ENDIAN_FLAG : 0x00;
+    vs->client_pf.rmax = red_max;
+    vs->client_pf.rbits = hweight_long(red_max);
+    vs->client_pf.rshift = red_shift;
+    vs->client_pf.rmask = red_max << red_shift;
+    vs->client_pf.gmax = green_max;
+    vs->client_pf.gbits = hweight_long(green_max);
+    vs->client_pf.gshift = green_shift;
+    vs->client_pf.gmask = green_max << green_shift;
+    vs->client_pf.bmax = blue_max;
+    vs->client_pf.bbits = hweight_long(blue_max);
+    vs->client_pf.bshift = blue_shift;
+    vs->client_pf.bmask = blue_max << blue_shift;
+    vs->client_pf.bits_per_pixel = bits_per_pixel;
+    vs->client_pf.bytes_per_pixel = bits_per_pixel / 8;
+    vs->client_pf.depth = bits_per_pixel == 32 ? 24 : bits_per_pixel;
+    vs->client_be = big_endian_flag;
 
     set_pixel_conversion(vs);
 
@@ -1904,8 +1906,10 @@ static void set_pixel_format(VncState *vs,
 static void pixel_format_message (VncState *vs) {
     char pad[3] = { 0, 0, 0 };
 
-    vnc_write_u8(vs, vs->ds->surface->pf.bits_per_pixel); /* bits-per-pixel */
-    vnc_write_u8(vs, vs->ds->surface->pf.depth); /* depth */
+    vs->client_pf = qemu_default_pixelformat(32);
+
+    vnc_write_u8(vs, vs->client_pf.bits_per_pixel); /* bits-per-pixel */
+    vnc_write_u8(vs, vs->client_pf.depth); /* depth */
 
 #ifdef HOST_WORDS_BIGENDIAN
     vnc_write_u8(vs, 1);             /* big-endian-flag */
@@ -1913,27 +1917,25 @@ static void pixel_format_message (VncState *vs) {
     vnc_write_u8(vs, 0);             /* big-endian-flag */
 #endif
     vnc_write_u8(vs, 1);             /* true-color-flag */
-    vnc_write_u16(vs, vs->ds->surface->pf.rmax);     /* red-max */
-    vnc_write_u16(vs, vs->ds->surface->pf.gmax);     /* green-max */
-    vnc_write_u16(vs, vs->ds->surface->pf.bmax);     /* blue-max */
-    vnc_write_u8(vs, vs->ds->surface->pf.rshift);    /* red-shift */
-    vnc_write_u8(vs, vs->ds->surface->pf.gshift);    /* green-shift */
-    vnc_write_u8(vs, vs->ds->surface->pf.bshift);    /* blue-shift */
+    vnc_write_u16(vs, vs->client_pf.rmax);     /* red-max */
+    vnc_write_u16(vs, vs->client_pf.gmax);     /* green-max */
+    vnc_write_u16(vs, vs->client_pf.bmax);     /* blue-max */
+    vnc_write_u8(vs, vs->client_pf.rshift);    /* red-shift */
+    vnc_write_u8(vs, vs->client_pf.gshift);    /* green-shift */
+    vnc_write_u8(vs, vs->client_pf.bshift);    /* blue-shift */
+    vnc_write(vs, pad, 3);           /* padding */
 
     vnc_hextile_set_pixel_conversion(vs, 0);
-
-    vs->clientds = *(vs->ds->surface);
-    vs->clientds.flags &= ~QEMU_ALLOCATED_FLAG;
     vs->write_pixels = vnc_write_pixels_copy;
-
-    vnc_write(vs, pad, 3);           /* padding */
 }
 
 static void vnc_dpy_setdata(DisplayState *ds)
 {
     VncDisplay *vd = ds->opaque;
 
-    *(vd->guest.ds) = *(ds->surface);
+    qemu_pixman_image_unref(vd->guest.fb);
+    vd->guest.fb = pixman_image_ref(ds->surface->image);
+    vd->guest.format = ds->surface->format;
     vnc_dpy_update(ds, 0, 0, ds_get_width(ds), ds_get_height(ds));
 }
 
@@ -2437,12 +2439,14 @@ static int vnc_refresh_lossy_rect(VncDisplay *vd, int x, int y)
 
 static int vnc_update_stats(VncDisplay *vd,  struct timeval * tv)
 {
+    int width = pixman_image_get_width(vd->guest.fb);
+    int height = pixman_image_get_height(vd->guest.fb);
     int x, y;
     struct timeval res;
     int has_dirty = 0;
 
-    for (y = 0; y < vd->guest.ds->height; y += VNC_STAT_RECT) {
-        for (x = 0; x < vd->guest.ds->width; x += VNC_STAT_RECT) {
+    for (y = 0; y < height; y += VNC_STAT_RECT) {
+        for (x = 0; x < width; x += VNC_STAT_RECT) {
             VncRectStat *rect = vnc_stat_rect(vd, x, y);
 
             rect->updated = false;
@@ -2456,8 +2460,8 @@ static int vnc_update_stats(VncDisplay *vd,  struct timeval * tv)
     }
     vd->guest.last_freq_check = *tv;
 
-    for (y = 0; y < vd->guest.ds->height; y += VNC_STAT_RECT) {
-        for (x = 0; x < vd->guest.ds->width; x += VNC_STAT_RECT) {
+    for (y = 0; y < height; y += VNC_STAT_RECT) {
+        for (x = 0; x < width; x += VNC_STAT_RECT) {
             VncRectStat *rect= vnc_stat_rect(vd, x, y);
             int count = ARRAY_SIZE(rect->times);
             struct timeval min, max;
@@ -2526,12 +2530,15 @@ static void vnc_rect_updated(VncDisplay *vd, int x, int y, struct timeval * tv)
 
 static int vnc_refresh_server_surface(VncDisplay *vd)
 {
+    int width = pixman_image_get_width(vd->guest.fb);
+    int height = pixman_image_get_height(vd->guest.fb);
     int y;
     uint8_t *guest_row;
     uint8_t *server_row;
     int cmp_bytes;
     VncState *vs;
     int has_dirty = 0;
+    pixman_image_t *tmpbuf = NULL;
 
     struct timeval tv = { 0, 0 };
 
@@ -2545,22 +2552,31 @@ static int vnc_refresh_server_surface(VncDisplay *vd)
      * Check and copy modified bits from guest to server surface.
      * Update server dirty map.
      */
-    cmp_bytes = 16 * ds_get_bytes_per_pixel(vd->ds);
-    if (cmp_bytes > vd->ds->surface->linesize) {
-        cmp_bytes = vd->ds->surface->linesize;
+    cmp_bytes = 64;
+    if (cmp_bytes > vnc_server_fb_stride(vd)) {
+        cmp_bytes = vnc_server_fb_stride(vd);
+    }
+    if (vd->guest.format != VNC_SERVER_FB_FORMAT) {
+        int width = pixman_image_get_width(vd->server);
+        tmpbuf = qemu_pixman_linebuf_create(VNC_SERVER_FB_FORMAT, width);
     }
-    guest_row  = vd->guest.ds->data;
-    server_row = vd->server->data;
-    for (y = 0; y < vd->guest.ds->height; y++) {
+    guest_row = (uint8_t *)pixman_image_get_data(vd->guest.fb);
+    server_row = (uint8_t *)pixman_image_get_data(vd->server);
+    for (y = 0; y < height; y++) {
         if (!bitmap_empty(vd->guest.dirty[y], VNC_DIRTY_BITS)) {
             int x;
             uint8_t *guest_ptr;
             uint8_t *server_ptr;
 
-            guest_ptr  = guest_row;
+            if (vd->guest.format != VNC_SERVER_FB_FORMAT) {
+                qemu_pixman_linebuf_fill(tmpbuf, vd->guest.fb, width, y);
+                guest_ptr = (uint8_t *)pixman_image_get_data(tmpbuf);
+            } else {
+                guest_ptr = guest_row;
+            }
             server_ptr = server_row;
 
-            for (x = 0; x + 15 < vd->guest.ds->width;
+            for (x = 0; x + 15 < width;
                     x += 16, guest_ptr += cmp_bytes, server_ptr += cmp_bytes) {
                 if (!test_and_clear_bit((x / 16), vd->guest.dirty[y]))
                     continue;
@@ -2575,9 +2591,10 @@ static int vnc_refresh_server_surface(VncDisplay *vd)
                 has_dirty++;
             }
         }
-        guest_row  += ds_get_linesize(vd->ds);
-        server_row += ds_get_linesize(vd->ds);
+        guest_row  += pixman_image_get_stride(vd->guest.fb);
+        server_row += pixman_image_get_stride(vd->server);
     }
+    qemu_pixman_image_unref(tmpbuf);
     return has_dirty;
 }
 
@@ -2747,17 +2764,17 @@ void vnc_display_init(DisplayState *ds)
     qemu_mutex_init(&vs->mutex);
     vnc_start_worker_thread();
 
-    dcl->dpy_copy = vnc_dpy_copy;
-    dcl->dpy_update = vnc_dpy_update;
-    dcl->dpy_resize = vnc_dpy_resize;
-    dcl->dpy_setdata = vnc_dpy_setdata;
+    dcl->dpy_gfx_copy = vnc_dpy_copy;
+    dcl->dpy_gfx_update = vnc_dpy_update;
+    dcl->dpy_gfx_resize = vnc_dpy_resize;
+    dcl->dpy_gfx_setdata = vnc_dpy_setdata;
+    dcl->dpy_mouse_set = vnc_mouse_set;
+    dcl->dpy_cursor_define = vnc_dpy_cursor_define;
     register_displaychangelistener(ds, dcl);
-    ds->mouse_set = vnc_mouse_set;
-    ds->cursor_define = vnc_dpy_cursor_define;
 }
 
 
-void vnc_display_close(DisplayState *ds)
+static void vnc_display_close(DisplayState *ds)
 {
     VncDisplay *vs = ds ? (VncDisplay *)ds->opaque : vnc_display;
 
@@ -2779,7 +2796,7 @@ void vnc_display_close(DisplayState *ds)
 #endif
 }
 
-int vnc_display_disable_login(DisplayState *ds)
+static int vnc_display_disable_login(DisplayState *ds)
 {
     VncDisplay *vs = ds ? (VncDisplay *)ds->opaque : vnc_display;
 
@@ -2844,7 +2861,7 @@ char *vnc_display_local_addr(DisplayState *ds)
     return vnc_socket_local_addr("%s:%s", vs->lsock);
 }
 
-int vnc_display_open(DisplayState *ds, const char *display)
+void vnc_display_open(DisplayState *ds, const char *display, Error **errp)
 {
     VncDisplay *vs = ds ? (VncDisplay *)ds->opaque : vnc_display;
     const char *options;
@@ -2862,14 +2879,15 @@ int vnc_display_open(DisplayState *ds, const char *display)
 #endif
     int lock_key_sync = 1;
 
-    if (!vnc_display)
-        return -1;
+    if (!vnc_display) {
+        error_setg(errp, "VNC display not active");
+        return;
+    }
     vnc_display_close(ds);
     if (strcmp(display, "none") == 0)
-        return 0;
+        return;
 
-    if (!(vs->display = strdup(display)))
-        return -1;
+    vs->display = g_strdup(display);
     vs->share_policy = VNC_SHARE_POLICY_ALLOW_EXCLUSIVE;
 
     options = display;
@@ -2877,13 +2895,11 @@ int vnc_display_open(DisplayState *ds, const char *display)
         options++;
         if (strncmp(options, "password", 8) == 0) {
             if (fips_get_state()) {
-                fprintf(stderr,
-                        "VNC password auth disabled due to FIPS mode, "
-                        "consider using the VeNCrypt or SASL authentication "
-                        "methods as an alternative\n");
-                g_free(vs->display);
-                vs->display = NULL;
-                return -1;
+                error_setg(errp,
+                           "VNC password auth disabled due to FIPS mode, "
+                           "consider using the VeNCrypt or SASL authentication "
+                           "methods as an alternative");
+                goto fail;
             }
             password = 1; /* Require password auth */
         } else if (strncmp(options, "reverse", 7) == 0) {
@@ -2913,18 +2929,14 @@ int vnc_display_open(DisplayState *ds, const char *display)
 
                 VNC_DEBUG("Trying certificate path '%s'\n", path);
                 if (vnc_tls_set_x509_creds_dir(vs, path) < 0) {
-                    fprintf(stderr, "Failed to find x509 certificates/keys in %s\n", path);
+                    error_setg(errp, "Failed to find x509 certificates/keys in %s", path);
                     g_free(path);
-                    g_free(vs->display);
-                    vs->display = NULL;
-                    return -1;
+                    goto fail;
                 }
                 g_free(path);
             } else {
-                fprintf(stderr, "No certificate path provided\n");
-                g_free(vs->display);
-                vs->display = NULL;
-                return -1;
+                error_setg(errp, "No certificate path provided");
+                goto fail;
             }
 #endif
 #if defined(CONFIG_VNC_TLS) || defined(CONFIG_VNC_SASL)
@@ -2933,7 +2945,7 @@ int vnc_display_open(DisplayState *ds, const char *display)
 #endif
         } else if (strncmp(options, "lossy", 5) == 0) {
             vs->lossy = true;
-        } else if (strncmp(options, "non-adapative", 13) == 0) {
+        } else if (strncmp(options, "non-adaptive", 12) == 0) {
             vs->non_adaptive = true;
         } else if (strncmp(options, "share=", 6) == 0) {
             if (strncmp(options+6, "ignore", 6) == 0) {
@@ -2943,10 +2955,8 @@ int vnc_display_open(DisplayState *ds, const char *display)
             } else if (strncmp(options+6, "force-shared", 12) == 0) {
                 vs->share_policy = VNC_SHARE_POLICY_FORCE_SHARED;
             } else {
-                fprintf(stderr, "unknown vnc share= option\n");
-                g_free(vs->display);
-                vs->display = NULL;
-                return -1;
+                error_setg(errp, "unknown vnc share= option");
+                goto fail;
             }
         }
     }
@@ -3047,52 +3057,50 @@ int vnc_display_open(DisplayState *ds, const char *display)
 
 #ifdef CONFIG_VNC_SASL
     if ((saslErr = sasl_server_init(NULL, "qemu")) != SASL_OK) {
-        fprintf(stderr, "Failed to initialize SASL auth %s",
-                sasl_errstring(saslErr, NULL, NULL));
-        g_free(vs->display);
-        vs->display = NULL;
-        return -1;
+        error_setg(errp, "Failed to initialize SASL auth: %s",
+                   sasl_errstring(saslErr, NULL, NULL));
+        goto fail;
     }
 #endif
     vs->lock_key_sync = lock_key_sync;
 
     if (reverse) {
         /* connect to viewer */
-        if (strncmp(display, "unix:", 5) == 0)
-            vs->lsock = unix_connect(display+5);
-        else
-            vs->lsock = inet_connect(display, true, NULL, NULL);
-        if (-1 == vs->lsock) {
-            g_free(vs->display);
-            vs->display = NULL;
-            return -1;
+        int csock;
+        vs->lsock = -1;
+        if (strncmp(display, "unix:", 5) == 0) {
+            csock = unix_connect(display+5, errp);
         } else {
-            int csock = vs->lsock;
-            vs->lsock = -1;
-            vnc_connect(vs, csock, 0);
+            csock = inet_connect(display, errp);
         }
-        return 0;
-
+        if (csock < 0) {
+            goto fail;
+        }
+        vnc_connect(vs, csock, 0);
     } else {
         /* listen for connects */
         char *dpy;
         dpy = g_malloc(256);
         if (strncmp(display, "unix:", 5) == 0) {
             pstrcpy(dpy, 256, "unix:");
-            vs->lsock = unix_listen(display+5, dpy+5, 256-5);
+            vs->lsock = unix_listen(display+5, dpy+5, 256-5, errp);
         } else {
             vs->lsock = inet_listen(display, dpy, 256,
-                                    SOCK_STREAM, 5900, NULL);
+                                    SOCK_STREAM, 5900, errp);
         }
-        if (-1 == vs->lsock) {
+        if (vs->lsock < 0) {
             g_free(dpy);
-            return -1;
-        } else {
-            g_free(vs->display);
-            vs->display = dpy;
+            goto fail;
         }
+        g_free(vs->display);
+        vs->display = dpy;
+        qemu_set_fd_handler2(vs->lsock, NULL, vnc_listen_read, NULL, vs);
     }
-    return qemu_set_fd_handler2(vs->lsock, NULL, vnc_listen_read, NULL, vs);
+    return;
+
+fail:
+    g_free(vs->display);
+    vs->display = NULL;
 }
 
 void vnc_display_add_client(DisplayState *ds, int csock, int skipauth)
diff --git a/ui/vnc.h b/ui/vnc.h
index 068c2fcda..6141e88b4 100644
--- a/ui/vnc.h
+++ b/ui/vnc.h
@@ -69,7 +69,7 @@ typedef struct VncRectEntry VncRectEntry;
 
 typedef int VncReadEvent(VncState *vs, uint8_t *data, size_t len);
 
-typedef void VncWritePixels(VncState *vs, struct PixelFormat *pf, void *data, int size);
+typedef void VncWritePixels(VncState *vs, void *data, int size);
 
 typedef void VncSendHextileTile(VncState *vs,
                                 int x, int y, int w, int h,
@@ -117,7 +117,8 @@ struct VncSurface
     struct timeval last_freq_check;
     DECLARE_BITMAP(dirty[VNC_MAX_HEIGHT], VNC_MAX_WIDTH / 16);
     VncRectStat stats[VNC_STAT_ROWS][VNC_STAT_COLS];
-    DisplaySurface *ds;
+    pixman_image_t *fb;
+    pixman_format_code_t format;
 };
 
 typedef enum VncShareMode {
@@ -151,7 +152,7 @@ struct VncDisplay
     uint8_t *cursor_mask;
 
     struct VncSurface guest;   /* guest visible surface (aka ds->surface) */
-    DisplaySurface *server;  /* vnc server surface */
+    pixman_image_t *server;    /* vnc server surface */
 
     char *display;
     char *password;
@@ -275,7 +276,9 @@ struct VncState
     Buffer input;
     /* current output mode information */
     VncWritePixels *write_pixels;
-    DisplaySurface clientds;
+    PixelFormat client_pf;
+    pixman_format_code_t client_format;
+    bool client_be;
 
     CaptureVoiceOut *audio_cap;
     struct audsettings as;
@@ -493,9 +496,6 @@ void vnc_read_when(VncState *vs, VncReadEvent *func, size_t expecting);
 
 
 /* Buffer I/O functions */
-uint8_t read_u8(uint8_t *data, size_t offset);
-uint16_t read_u16(uint8_t *data, size_t offset);
-int32_t read_s32(uint8_t *data, size_t offset);
 uint32_t read_u32(uint8_t *data, size_t offset);
 
 /* Protocol stage functions */
@@ -507,8 +507,6 @@ void start_auth_vnc(VncState *vs);
 
 /* Buffer management */
 void buffer_reserve(Buffer *buffer, size_t len);
-int buffer_empty(Buffer *buffer);
-uint8_t *buffer_end(Buffer *buffer);
 void buffer_reset(Buffer *buffer);
 void buffer_free(Buffer *buffer);
 void buffer_append(Buffer *buffer, const void *data, size_t len);
@@ -527,6 +525,14 @@ static inline uint32_t vnc_has_feature(VncState *vs, int feature) {
 void vnc_framebuffer_update(VncState *vs, int x, int y, int w, int h,
                             int32_t encoding);
 
+/* server fb is in PIXMAN_x8r8g8b8 */
+#define VNC_SERVER_FB_FORMAT PIXMAN_FORMAT(32, PIXMAN_TYPE_ARGB, 0, 8, 8, 8)
+#define VNC_SERVER_FB_BITS   (PIXMAN_FORMAT_BPP(VNC_SERVER_FB_FORMAT))
+#define VNC_SERVER_FB_BYTES  ((VNC_SERVER_FB_BITS+7)/8)
+
+void *vnc_server_fb_ptr(VncDisplay *vd, int x, int y);
+int vnc_server_fb_stride(VncDisplay *vd);
+
 void vnc_convert_pixel(VncState *vs, uint8_t *buf, uint32_t v);
 double vnc_update_freq(VncState *vs, int x, int y, int w, int h);
 void vnc_sent_lossy_rect(VncState *vs, int x, int y, int w, int h);
diff --git a/uri.c b/uri.c
new file mode 100644
index 000000000..dd922de33
--- /dev/null
+++ b/uri.c
@@ -0,0 +1,2249 @@
+/**
+ * uri.c: set of generic URI related routines
+ *
+ * Reference: RFCs 3986, 2732 and 2373
+ *
+ * Copyright (C) 1998-2003 Daniel Veillard.  All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+ * DANIEL VEILLARD BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Except as contained in this notice, the name of Daniel Veillard shall not
+ * be used in advertising or otherwise to promote the sale, use or other
+ * dealings in this Software without prior written authorization from him.
+ *
+ * daniel@veillard.com
+ *
+ **
+ *
+ * Copyright (C) 2007, 2009-2010 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307  USA
+ *
+ * Authors:
+ *    Richard W.M. Jones <rjones@redhat.com>
+ *
+ */
+
+#include <glib.h>
+#include <string.h>
+#include <stdio.h>
+
+#include "uri.h"
+
+static void uri_clean(URI *uri);
+
+/*
+ * Old rule from 2396 used in legacy handling code
+ * alpha    = lowalpha | upalpha
+ */
+#define IS_ALPHA(x) (IS_LOWALPHA(x) || IS_UPALPHA(x))
+
+
+/*
+ * lowalpha = "a" | "b" | "c" | "d" | "e" | "f" | "g" | "h" | "i" | "j" |
+ *            "k" | "l" | "m" | "n" | "o" | "p" | "q" | "r" | "s" | "t" |
+ *            "u" | "v" | "w" | "x" | "y" | "z"
+ */
+
+#define IS_LOWALPHA(x) (((x) >= 'a') && ((x) <= 'z'))
+
+/*
+ * upalpha = "A" | "B" | "C" | "D" | "E" | "F" | "G" | "H" | "I" | "J" |
+ *           "K" | "L" | "M" | "N" | "O" | "P" | "Q" | "R" | "S" | "T" |
+ *           "U" | "V" | "W" | "X" | "Y" | "Z"
+ */
+#define IS_UPALPHA(x) (((x) >= 'A') && ((x) <= 'Z'))
+
+#ifdef IS_DIGIT
+#undef IS_DIGIT
+#endif
+/*
+ * digit = "0" | "1" | "2" | "3" | "4" | "5" | "6" | "7" | "8" | "9"
+ */
+#define IS_DIGIT(x) (((x) >= '0') && ((x) <= '9'))
+
+/*
+ * alphanum = alpha | digit
+ */
+
+#define IS_ALPHANUM(x) (IS_ALPHA(x) || IS_DIGIT(x))
+
+/*
+ * mark = "-" | "_" | "." | "!" | "~" | "*" | "'" | "(" | ")"
+ */
+
+#define IS_MARK(x) (((x) == '-') || ((x) == '_') || ((x) == '.') ||     \
+    ((x) == '!') || ((x) == '~') || ((x) == '*') || ((x) == '\'') ||    \
+    ((x) == '(') || ((x) == ')'))
+
+/*
+ * unwise = "{" | "}" | "|" | "\" | "^" | "`"
+ */
+
+#define IS_UNWISE(p)                                                    \
+      (((*(p) == '{')) || ((*(p) == '}')) || ((*(p) == '|')) ||         \
+       ((*(p) == '\\')) || ((*(p) == '^')) || ((*(p) == '[')) ||        \
+       ((*(p) == ']')) || ((*(p) == '`')))
+/*
+ * reserved = ";" | "/" | "?" | ":" | "@" | "&" | "=" | "+" | "$" | "," |
+ *            "[" | "]"
+ */
+
+#define IS_RESERVED(x) (((x) == ';') || ((x) == '/') || ((x) == '?') || \
+        ((x) == ':') || ((x) == '@') || ((x) == '&') || ((x) == '=') || \
+        ((x) == '+') || ((x) == '$') || ((x) == ',') || ((x) == '[') || \
+        ((x) == ']'))
+
+/*
+ * unreserved = alphanum | mark
+ */
+
+#define IS_UNRESERVED(x) (IS_ALPHANUM(x) || IS_MARK(x))
+
+/*
+ * Skip to next pointer char, handle escaped sequences
+ */
+
+#define NEXT(p) ((*p == '%')? p += 3 : p++)
+
+/*
+ * Productions from the spec.
+ *
+ *    authority     = server | reg_name
+ *    reg_name      = 1*( unreserved | escaped | "$" | "," |
+ *                        ";" | ":" | "@" | "&" | "=" | "+" )
+ *
+ * path          = [ abs_path | opaque_part ]
+ */
+
+
+/************************************************************************
+ *									*
+ *                         RFC 3986 parser				*
+ *									*
+ ************************************************************************/
+
+#define ISA_DIGIT(p) ((*(p) >= '0') && (*(p) <= '9'))
+#define ISA_ALPHA(p) (((*(p) >= 'a') && (*(p) <= 'z')) ||		\
+                      ((*(p) >= 'A') && (*(p) <= 'Z')))
+#define ISA_HEXDIG(p)							\
+       (ISA_DIGIT(p) || ((*(p) >= 'a') && (*(p) <= 'f')) ||		\
+        ((*(p) >= 'A') && (*(p) <= 'F')))
+
+/*
+ *    sub-delims    = "!" / "$" / "&" / "'" / "(" / ")"
+ *                     / "*" / "+" / "," / ";" / "="
+ */
+#define ISA_SUB_DELIM(p)						\
+      (((*(p) == '!')) || ((*(p) == '$')) || ((*(p) == '&')) ||		\
+       ((*(p) == '(')) || ((*(p) == ')')) || ((*(p) == '*')) ||		\
+       ((*(p) == '+')) || ((*(p) == ',')) || ((*(p) == ';')) ||		\
+       ((*(p) == '=')) || ((*(p) == '\'')))
+
+/*
+ *    gen-delims    = ":" / "/" / "?" / "#" / "[" / "]" / "@"
+ */
+#define ISA_GEN_DELIM(p)						\
+      (((*(p) == ':')) || ((*(p) == '/')) || ((*(p) == '?')) ||         \
+       ((*(p) == '#')) || ((*(p) == '[')) || ((*(p) == ']')) ||         \
+       ((*(p) == '@')))
+
+/*
+ *    reserved      = gen-delims / sub-delims
+ */
+#define ISA_RESERVED(p) (ISA_GEN_DELIM(p) || (ISA_SUB_DELIM(p)))
+
+/*
+ *    unreserved    = ALPHA / DIGIT / "-" / "." / "_" / "~"
+ */
+#define ISA_UNRESERVED(p)						\
+      ((ISA_ALPHA(p)) || (ISA_DIGIT(p)) || ((*(p) == '-')) ||		\
+       ((*(p) == '.')) || ((*(p) == '_')) || ((*(p) == '~')))
+
+/*
+ *    pct-encoded   = "%" HEXDIG HEXDIG
+ */
+#define ISA_PCT_ENCODED(p)						\
+     ((*(p) == '%') && (ISA_HEXDIG(p + 1)) && (ISA_HEXDIG(p + 2)))
+
+/*
+ *    pchar         = unreserved / pct-encoded / sub-delims / ":" / "@"
+ */
+#define ISA_PCHAR(p)							\
+     (ISA_UNRESERVED(p) || ISA_PCT_ENCODED(p) || ISA_SUB_DELIM(p) ||	\
+      ((*(p) == ':')) || ((*(p) == '@')))
+
+/**
+ * rfc3986_parse_scheme:
+ * @uri:  pointer to an URI structure
+ * @str:  pointer to the string to analyze
+ *
+ * Parse an URI scheme
+ *
+ * ALPHA *( ALPHA / DIGIT / "+" / "-" / "." )
+ *
+ * Returns 0 or the error code
+ */
+static int
+rfc3986_parse_scheme(URI *uri, const char **str) {
+    const char *cur;
+
+    if (str == NULL)
+	return(-1);
+
+    cur = *str;
+    if (!ISA_ALPHA(cur))
+	return(2);
+    cur++;
+    while (ISA_ALPHA(cur) || ISA_DIGIT(cur) ||
+           (*cur == '+') || (*cur == '-') || (*cur == '.')) cur++;
+    if (uri != NULL) {
+	if (uri->scheme != NULL) g_free(uri->scheme);
+	uri->scheme = g_strndup(*str, cur - *str);
+    }
+    *str = cur;
+    return(0);
+}
+
+/**
+ * rfc3986_parse_fragment:
+ * @uri:  pointer to an URI structure
+ * @str:  pointer to the string to analyze
+ *
+ * Parse the query part of an URI
+ *
+ * fragment      = *( pchar / "/" / "?" )
+ * NOTE: the strict syntax as defined by 3986 does not allow '[' and ']'
+ *       in the fragment identifier but this is used very broadly for
+ *       xpointer scheme selection, so we are allowing it here to not break
+ *       for example all the DocBook processing chains.
+ *
+ * Returns 0 or the error code
+ */
+static int
+rfc3986_parse_fragment(URI *uri, const char **str)
+{
+    const char *cur;
+
+    if (str == NULL)
+        return (-1);
+
+    cur = *str;
+
+    while ((ISA_PCHAR(cur)) || (*cur == '/') || (*cur == '?') ||
+           (*cur == '[') || (*cur == ']') ||
+           ((uri != NULL) && (uri->cleanup & 1) && (IS_UNWISE(cur))))
+        NEXT(cur);
+    if (uri != NULL) {
+        if (uri->fragment != NULL)
+            g_free(uri->fragment);
+	if (uri->cleanup & 2)
+	    uri->fragment = g_strndup(*str, cur - *str);
+	else
+	    uri->fragment = uri_string_unescape(*str, cur - *str, NULL);
+    }
+    *str = cur;
+    return (0);
+}
+
+/**
+ * rfc3986_parse_query:
+ * @uri:  pointer to an URI structure
+ * @str:  pointer to the string to analyze
+ *
+ * Parse the query part of an URI
+ *
+ * query = *uric
+ *
+ * Returns 0 or the error code
+ */
+static int
+rfc3986_parse_query(URI *uri, const char **str)
+{
+    const char *cur;
+
+    if (str == NULL)
+        return (-1);
+
+    cur = *str;
+
+    while ((ISA_PCHAR(cur)) || (*cur == '/') || (*cur == '?') ||
+           ((uri != NULL) && (uri->cleanup & 1) && (IS_UNWISE(cur))))
+        NEXT(cur);
+    if (uri != NULL) {
+	if (uri->query != NULL)
+	    g_free (uri->query);
+	uri->query = g_strndup (*str, cur - *str);
+    }
+    *str = cur;
+    return (0);
+}
+
+/**
+ * rfc3986_parse_port:
+ * @uri:  pointer to an URI structure
+ * @str:  the string to analyze
+ *
+ * Parse a port  part and fills in the appropriate fields
+ * of the @uri structure
+ *
+ * port          = *DIGIT
+ *
+ * Returns 0 or the error code
+ */
+static int
+rfc3986_parse_port(URI *uri, const char **str)
+{
+    const char *cur = *str;
+
+    if (ISA_DIGIT(cur)) {
+	if (uri != NULL)
+	    uri->port = 0;
+	while (ISA_DIGIT(cur)) {
+	    if (uri != NULL)
+		uri->port = uri->port * 10 + (*cur - '0');
+	    cur++;
+	}
+	*str = cur;
+	return(0);
+    }
+    return(1);
+}
+
+/**
+ * rfc3986_parse_user_info:
+ * @uri:  pointer to an URI structure
+ * @str:  the string to analyze
+ *
+ * Parse an user informations part and fills in the appropriate fields
+ * of the @uri structure
+ *
+ * userinfo      = *( unreserved / pct-encoded / sub-delims / ":" )
+ *
+ * Returns 0 or the error code
+ */
+static int
+rfc3986_parse_user_info(URI *uri, const char **str)
+{
+    const char *cur;
+
+    cur = *str;
+    while (ISA_UNRESERVED(cur) || ISA_PCT_ENCODED(cur) ||
+           ISA_SUB_DELIM(cur) || (*cur == ':'))
+	NEXT(cur);
+    if (*cur == '@') {
+	if (uri != NULL) {
+	    if (uri->user != NULL) g_free(uri->user);
+	    if (uri->cleanup & 2)
+		uri->user = g_strndup(*str, cur - *str);
+	    else
+		uri->user = uri_string_unescape(*str, cur - *str, NULL);
+	}
+	*str = cur;
+	return(0);
+    }
+    return(1);
+}
+
+/**
+ * rfc3986_parse_dec_octet:
+ * @str:  the string to analyze
+ *
+ *    dec-octet     = DIGIT                 ; 0-9
+ *                  / %x31-39 DIGIT         ; 10-99
+ *                  / "1" 2DIGIT            ; 100-199
+ *                  / "2" %x30-34 DIGIT     ; 200-249
+ *                  / "25" %x30-35          ; 250-255
+ *
+ * Skip a dec-octet.
+ *
+ * Returns 0 if found and skipped, 1 otherwise
+ */
+static int
+rfc3986_parse_dec_octet(const char **str) {
+    const char *cur = *str;
+
+    if (!(ISA_DIGIT(cur)))
+        return(1);
+    if (!ISA_DIGIT(cur+1))
+	cur++;
+    else if ((*cur != '0') && (ISA_DIGIT(cur + 1)) && (!ISA_DIGIT(cur+2)))
+	cur += 2;
+    else if ((*cur == '1') && (ISA_DIGIT(cur + 1)) && (ISA_DIGIT(cur + 2)))
+	cur += 3;
+    else if ((*cur == '2') && (*(cur + 1) >= '0') &&
+	     (*(cur + 1) <= '4') && (ISA_DIGIT(cur + 2)))
+	cur += 3;
+    else if ((*cur == '2') && (*(cur + 1) == '5') &&
+	     (*(cur + 2) >= '0') && (*(cur + 1) <= '5'))
+	cur += 3;
+    else
+        return(1);
+    *str = cur;
+    return(0);
+}
+/**
+ * rfc3986_parse_host:
+ * @uri:  pointer to an URI structure
+ * @str:  the string to analyze
+ *
+ * Parse an host part and fills in the appropriate fields
+ * of the @uri structure
+ *
+ * host          = IP-literal / IPv4address / reg-name
+ * IP-literal    = "[" ( IPv6address / IPvFuture  ) "]"
+ * IPv4address   = dec-octet "." dec-octet "." dec-octet "." dec-octet
+ * reg-name      = *( unreserved / pct-encoded / sub-delims )
+ *
+ * Returns 0 or the error code
+ */
+static int
+rfc3986_parse_host(URI *uri, const char **str)
+{
+    const char *cur = *str;
+    const char *host;
+
+    host = cur;
+    /*
+     * IPv6 and future adressing scheme are enclosed between brackets
+     */
+    if (*cur == '[') {
+        cur++;
+	while ((*cur != ']') && (*cur != 0))
+	    cur++;
+	if (*cur != ']')
+	    return(1);
+	cur++;
+	goto found;
+    }
+    /*
+     * try to parse an IPv4
+     */
+    if (ISA_DIGIT(cur)) {
+        if (rfc3986_parse_dec_octet(&cur) != 0)
+	    goto not_ipv4;
+	if (*cur != '.')
+	    goto not_ipv4;
+	cur++;
+        if (rfc3986_parse_dec_octet(&cur) != 0)
+	    goto not_ipv4;
+	if (*cur != '.')
+	    goto not_ipv4;
+        if (rfc3986_parse_dec_octet(&cur) != 0)
+	    goto not_ipv4;
+	if (*cur != '.')
+	    goto not_ipv4;
+        if (rfc3986_parse_dec_octet(&cur) != 0)
+	    goto not_ipv4;
+	goto found;
+not_ipv4:
+        cur = *str;
+    }
+    /*
+     * then this should be a hostname which can be empty
+     */
+    while (ISA_UNRESERVED(cur) || ISA_PCT_ENCODED(cur) || ISA_SUB_DELIM(cur))
+        NEXT(cur);
+found:
+    if (uri != NULL) {
+	if (uri->authority != NULL) g_free(uri->authority);
+	uri->authority = NULL;
+	if (uri->server != NULL) g_free(uri->server);
+	if (cur != host) {
+	    if (uri->cleanup & 2)
+		uri->server = g_strndup(host, cur - host);
+	    else
+		uri->server = uri_string_unescape(host, cur - host, NULL);
+	} else
+	    uri->server = NULL;
+    }
+    *str = cur;
+    return(0);
+}
+
+/**
+ * rfc3986_parse_authority:
+ * @uri:  pointer to an URI structure
+ * @str:  the string to analyze
+ *
+ * Parse an authority part and fills in the appropriate fields
+ * of the @uri structure
+ *
+ * authority     = [ userinfo "@" ] host [ ":" port ]
+ *
+ * Returns 0 or the error code
+ */
+static int
+rfc3986_parse_authority(URI *uri, const char **str)
+{
+    const char *cur;
+    int ret;
+
+    cur = *str;
+    /*
+     * try to parse an userinfo and check for the trailing @
+     */
+    ret = rfc3986_parse_user_info(uri, &cur);
+    if ((ret != 0) || (*cur != '@'))
+        cur = *str;
+    else
+        cur++;
+    ret = rfc3986_parse_host(uri, &cur);
+    if (ret != 0) return(ret);
+    if (*cur == ':') {
+        cur++;
+        ret = rfc3986_parse_port(uri, &cur);
+	if (ret != 0) return(ret);
+    }
+    *str = cur;
+    return(0);
+}
+
+/**
+ * rfc3986_parse_segment:
+ * @str:  the string to analyze
+ * @forbid: an optional forbidden character
+ * @empty: allow an empty segment
+ *
+ * Parse a segment and fills in the appropriate fields
+ * of the @uri structure
+ *
+ * segment       = *pchar
+ * segment-nz    = 1*pchar
+ * segment-nz-nc = 1*( unreserved / pct-encoded / sub-delims / "@" )
+ *               ; non-zero-length segment without any colon ":"
+ *
+ * Returns 0 or the error code
+ */
+static int
+rfc3986_parse_segment(const char **str, char forbid, int empty)
+{
+    const char *cur;
+
+    cur = *str;
+    if (!ISA_PCHAR(cur)) {
+        if (empty)
+	    return(0);
+	return(1);
+    }
+    while (ISA_PCHAR(cur) && (*cur != forbid))
+        NEXT(cur);
+    *str = cur;
+    return (0);
+}
+
+/**
+ * rfc3986_parse_path_ab_empty:
+ * @uri:  pointer to an URI structure
+ * @str:  the string to analyze
+ *
+ * Parse an path absolute or empty and fills in the appropriate fields
+ * of the @uri structure
+ *
+ * path-abempty  = *( "/" segment )
+ *
+ * Returns 0 or the error code
+ */
+static int
+rfc3986_parse_path_ab_empty(URI *uri, const char **str)
+{
+    const char *cur;
+    int ret;
+
+    cur = *str;
+
+    while (*cur == '/') {
+        cur++;
+	ret = rfc3986_parse_segment(&cur, 0, 1);
+	if (ret != 0) return(ret);
+    }
+    if (uri != NULL) {
+	if (uri->path != NULL) g_free(uri->path);
+        if (*str != cur) {
+            if (uri->cleanup & 2)
+                uri->path = g_strndup(*str, cur - *str);
+            else
+                uri->path = uri_string_unescape(*str, cur - *str, NULL);
+        } else {
+            uri->path = NULL;
+        }
+    }
+    *str = cur;
+    return (0);
+}
+
+/**
+ * rfc3986_parse_path_absolute:
+ * @uri:  pointer to an URI structure
+ * @str:  the string to analyze
+ *
+ * Parse an path absolute and fills in the appropriate fields
+ * of the @uri structure
+ *
+ * path-absolute = "/" [ segment-nz *( "/" segment ) ]
+ *
+ * Returns 0 or the error code
+ */
+static int
+rfc3986_parse_path_absolute(URI *uri, const char **str)
+{
+    const char *cur;
+    int ret;
+
+    cur = *str;
+
+    if (*cur != '/')
+        return(1);
+    cur++;
+    ret = rfc3986_parse_segment(&cur, 0, 0);
+    if (ret == 0) {
+	while (*cur == '/') {
+	    cur++;
+	    ret = rfc3986_parse_segment(&cur, 0, 1);
+	    if (ret != 0) return(ret);
+	}
+    }
+    if (uri != NULL) {
+	if (uri->path != NULL) g_free(uri->path);
+        if (cur != *str) {
+            if (uri->cleanup & 2)
+                uri->path = g_strndup(*str, cur - *str);
+            else
+                uri->path = uri_string_unescape(*str, cur - *str, NULL);
+        } else {
+            uri->path = NULL;
+        }
+    }
+    *str = cur;
+    return (0);
+}
+
+/**
+ * rfc3986_parse_path_rootless:
+ * @uri:  pointer to an URI structure
+ * @str:  the string to analyze
+ *
+ * Parse an path without root and fills in the appropriate fields
+ * of the @uri structure
+ *
+ * path-rootless = segment-nz *( "/" segment )
+ *
+ * Returns 0 or the error code
+ */
+static int
+rfc3986_parse_path_rootless(URI *uri, const char **str)
+{
+    const char *cur;
+    int ret;
+
+    cur = *str;
+
+    ret = rfc3986_parse_segment(&cur, 0, 0);
+    if (ret != 0) return(ret);
+    while (*cur == '/') {
+        cur++;
+	ret = rfc3986_parse_segment(&cur, 0, 1);
+	if (ret != 0) return(ret);
+    }
+    if (uri != NULL) {
+	if (uri->path != NULL) g_free(uri->path);
+        if (cur != *str) {
+            if (uri->cleanup & 2)
+                uri->path = g_strndup(*str, cur - *str);
+            else
+                uri->path = uri_string_unescape(*str, cur - *str, NULL);
+        } else {
+            uri->path = NULL;
+        }
+    }
+    *str = cur;
+    return (0);
+}
+
+/**
+ * rfc3986_parse_path_no_scheme:
+ * @uri:  pointer to an URI structure
+ * @str:  the string to analyze
+ *
+ * Parse an path which is not a scheme and fills in the appropriate fields
+ * of the @uri structure
+ *
+ * path-noscheme = segment-nz-nc *( "/" segment )
+ *
+ * Returns 0 or the error code
+ */
+static int
+rfc3986_parse_path_no_scheme(URI *uri, const char **str)
+{
+    const char *cur;
+    int ret;
+
+    cur = *str;
+
+    ret = rfc3986_parse_segment(&cur, ':', 0);
+    if (ret != 0) return(ret);
+    while (*cur == '/') {
+        cur++;
+	ret = rfc3986_parse_segment(&cur, 0, 1);
+	if (ret != 0) return(ret);
+    }
+    if (uri != NULL) {
+	if (uri->path != NULL) g_free(uri->path);
+        if (cur != *str) {
+            if (uri->cleanup & 2)
+                uri->path = g_strndup(*str, cur - *str);
+            else
+                uri->path = uri_string_unescape(*str, cur - *str, NULL);
+        } else {
+            uri->path = NULL;
+        }
+    }
+    *str = cur;
+    return (0);
+}
+
+/**
+ * rfc3986_parse_hier_part:
+ * @uri:  pointer to an URI structure
+ * @str:  the string to analyze
+ *
+ * Parse an hierarchical part and fills in the appropriate fields
+ * of the @uri structure
+ *
+ * hier-part     = "//" authority path-abempty
+ *                / path-absolute
+ *                / path-rootless
+ *                / path-empty
+ *
+ * Returns 0 or the error code
+ */
+static int
+rfc3986_parse_hier_part(URI *uri, const char **str)
+{
+    const char *cur;
+    int ret;
+
+    cur = *str;
+
+    if ((*cur == '/') && (*(cur + 1) == '/')) {
+        cur += 2;
+	ret = rfc3986_parse_authority(uri, &cur);
+	if (ret != 0) return(ret);
+	ret = rfc3986_parse_path_ab_empty(uri, &cur);
+	if (ret != 0) return(ret);
+	*str = cur;
+	return(0);
+    } else if (*cur == '/') {
+        ret = rfc3986_parse_path_absolute(uri, &cur);
+	if (ret != 0) return(ret);
+    } else if (ISA_PCHAR(cur)) {
+        ret = rfc3986_parse_path_rootless(uri, &cur);
+	if (ret != 0) return(ret);
+    } else {
+	/* path-empty is effectively empty */
+	if (uri != NULL) {
+	    if (uri->path != NULL) g_free(uri->path);
+	    uri->path = NULL;
+	}
+    }
+    *str = cur;
+    return (0);
+}
+
+/**
+ * rfc3986_parse_relative_ref:
+ * @uri:  pointer to an URI structure
+ * @str:  the string to analyze
+ *
+ * Parse an URI string and fills in the appropriate fields
+ * of the @uri structure
+ *
+ * relative-ref  = relative-part [ "?" query ] [ "#" fragment ]
+ * relative-part = "//" authority path-abempty
+ *               / path-absolute
+ *               / path-noscheme
+ *               / path-empty
+ *
+ * Returns 0 or the error code
+ */
+static int
+rfc3986_parse_relative_ref(URI *uri, const char *str) {
+    int ret;
+
+    if ((*str == '/') && (*(str + 1) == '/')) {
+        str += 2;
+	ret = rfc3986_parse_authority(uri, &str);
+	if (ret != 0) return(ret);
+	ret = rfc3986_parse_path_ab_empty(uri, &str);
+	if (ret != 0) return(ret);
+    } else if (*str == '/') {
+	ret = rfc3986_parse_path_absolute(uri, &str);
+	if (ret != 0) return(ret);
+    } else if (ISA_PCHAR(str)) {
+        ret = rfc3986_parse_path_no_scheme(uri, &str);
+	if (ret != 0) return(ret);
+    } else {
+	/* path-empty is effectively empty */
+	if (uri != NULL) {
+	    if (uri->path != NULL) g_free(uri->path);
+	    uri->path = NULL;
+	}
+    }
+
+    if (*str == '?') {
+	str++;
+	ret = rfc3986_parse_query(uri, &str);
+	if (ret != 0) return(ret);
+    }
+    if (*str == '#') {
+	str++;
+	ret = rfc3986_parse_fragment(uri, &str);
+	if (ret != 0) return(ret);
+    }
+    if (*str != 0) {
+	uri_clean(uri);
+	return(1);
+    }
+    return(0);
+}
+
+
+/**
+ * rfc3986_parse:
+ * @uri:  pointer to an URI structure
+ * @str:  the string to analyze
+ *
+ * Parse an URI string and fills in the appropriate fields
+ * of the @uri structure
+ *
+ * scheme ":" hier-part [ "?" query ] [ "#" fragment ]
+ *
+ * Returns 0 or the error code
+ */
+static int
+rfc3986_parse(URI *uri, const char *str) {
+    int ret;
+
+    ret = rfc3986_parse_scheme(uri, &str);
+    if (ret != 0) return(ret);
+    if (*str != ':') {
+	return(1);
+    }
+    str++;
+    ret = rfc3986_parse_hier_part(uri, &str);
+    if (ret != 0) return(ret);
+    if (*str == '?') {
+	str++;
+	ret = rfc3986_parse_query(uri, &str);
+	if (ret != 0) return(ret);
+    }
+    if (*str == '#') {
+	str++;
+	ret = rfc3986_parse_fragment(uri, &str);
+	if (ret != 0) return(ret);
+    }
+    if (*str != 0) {
+	uri_clean(uri);
+	return(1);
+    }
+    return(0);
+}
+
+/**
+ * rfc3986_parse_uri_reference:
+ * @uri:  pointer to an URI structure
+ * @str:  the string to analyze
+ *
+ * Parse an URI reference string and fills in the appropriate fields
+ * of the @uri structure
+ *
+ * URI-reference = URI / relative-ref
+ *
+ * Returns 0 or the error code
+ */
+static int
+rfc3986_parse_uri_reference(URI *uri, const char *str) {
+    int ret;
+
+    if (str == NULL)
+	return(-1);
+    uri_clean(uri);
+
+    /*
+     * Try first to parse absolute refs, then fallback to relative if
+     * it fails.
+     */
+    ret = rfc3986_parse(uri, str);
+    if (ret != 0) {
+	uri_clean(uri);
+        ret = rfc3986_parse_relative_ref(uri, str);
+	if (ret != 0) {
+	    uri_clean(uri);
+	    return(ret);
+	}
+    }
+    return(0);
+}
+
+/**
+ * uri_parse:
+ * @str:  the URI string to analyze
+ *
+ * Parse an URI based on RFC 3986
+ *
+ * URI-reference = [ absoluteURI | relativeURI ] [ "#" fragment ]
+ *
+ * Returns a newly built URI or NULL in case of error
+ */
+URI *
+uri_parse(const char *str) {
+    URI *uri;
+    int ret;
+
+    if (str == NULL)
+	return(NULL);
+    uri = uri_new();
+    if (uri != NULL) {
+	ret = rfc3986_parse_uri_reference(uri, str);
+        if (ret) {
+	    uri_free(uri);
+	    return(NULL);
+	}
+    }
+    return(uri);
+}
+
+/**
+ * uri_parse_into:
+ * @uri:  pointer to an URI structure
+ * @str:  the string to analyze
+ *
+ * Parse an URI reference string based on RFC 3986 and fills in the
+ * appropriate fields of the @uri structure
+ *
+ * URI-reference = URI / relative-ref
+ *
+ * Returns 0 or the error code
+ */
+int
+uri_parse_into(URI *uri, const char *str) {
+    return(rfc3986_parse_uri_reference(uri, str));
+}
+
+/**
+ * uri_parse_raw:
+ * @str:  the URI string to analyze
+ * @raw:  if 1 unescaping of URI pieces are disabled
+ *
+ * Parse an URI but allows to keep intact the original fragments.
+ *
+ * URI-reference = URI / relative-ref
+ *
+ * Returns a newly built URI or NULL in case of error
+ */
+URI *
+uri_parse_raw(const char *str, int raw) {
+    URI *uri;
+    int ret;
+
+    if (str == NULL)
+	return(NULL);
+    uri = uri_new();
+    if (uri != NULL) {
+        if (raw) {
+	    uri->cleanup |= 2;
+	}
+	ret = uri_parse_into(uri, str);
+        if (ret) {
+	    uri_free(uri);
+	    return(NULL);
+	}
+    }
+    return(uri);
+}
+
+/************************************************************************
+ *									*
+ *			Generic URI structure functions			*
+ *									*
+ ************************************************************************/
+
+/**
+ * uri_new:
+ *
+ * Simply creates an empty URI
+ *
+ * Returns the new structure or NULL in case of error
+ */
+URI *
+uri_new(void) {
+    URI *ret;
+
+    ret = (URI *) g_malloc(sizeof(URI));
+    memset(ret, 0, sizeof(URI));
+    return(ret);
+}
+
+/**
+ * realloc2n:
+ *
+ * Function to handle properly a reallocation when saving an URI
+ * Also imposes some limit on the length of an URI string output
+ */
+static char *
+realloc2n(char *ret, int *max) {
+    char *temp;
+    int tmp;
+
+    tmp = *max * 2;
+    temp = g_realloc(ret, (tmp + 1));
+    *max = tmp;
+    return(temp);
+}
+
+/**
+ * uri_to_string:
+ * @uri:  pointer to an URI
+ *
+ * Save the URI as an escaped string
+ *
+ * Returns a new string (to be deallocated by caller)
+ */
+char *
+uri_to_string(URI *uri) {
+    char *ret = NULL;
+    char *temp;
+    const char *p;
+    int len;
+    int max;
+
+    if (uri == NULL) return(NULL);
+
+
+    max = 80;
+    ret = g_malloc(max + 1);
+    len = 0;
+
+    if (uri->scheme != NULL) {
+	p = uri->scheme;
+	while (*p != 0) {
+	    if (len >= max) {
+                temp = realloc2n(ret, &max);
+                if (temp == NULL) goto mem_error;
+		ret = temp;
+	    }
+	    ret[len++] = *p++;
+	}
+	if (len >= max) {
+            temp = realloc2n(ret, &max);
+            if (temp == NULL) goto mem_error;
+            ret = temp;
+	}
+	ret[len++] = ':';
+    }
+    if (uri->opaque != NULL) {
+	p = uri->opaque;
+	while (*p != 0) {
+	    if (len + 3 >= max) {
+                temp = realloc2n(ret, &max);
+                if (temp == NULL) goto mem_error;
+                ret = temp;
+	    }
+	    if (IS_RESERVED(*(p)) || IS_UNRESERVED(*(p)))
+		ret[len++] = *p++;
+	    else {
+		int val = *(unsigned char *)p++;
+		int hi = val / 0x10, lo = val % 0x10;
+		ret[len++] = '%';
+		ret[len++] = hi + (hi > 9? 'A'-10 : '0');
+		ret[len++] = lo + (lo > 9? 'A'-10 : '0');
+	    }
+	}
+    } else {
+	if (uri->server != NULL) {
+	    if (len + 3 >= max) {
+                temp = realloc2n(ret, &max);
+                if (temp == NULL) goto mem_error;
+                ret = temp;
+	    }
+	    ret[len++] = '/';
+	    ret[len++] = '/';
+	    if (uri->user != NULL) {
+		p = uri->user;
+		while (*p != 0) {
+		    if (len + 3 >= max) {
+                        temp = realloc2n(ret, &max);
+                        if (temp == NULL) goto mem_error;
+                        ret = temp;
+		    }
+		    if ((IS_UNRESERVED(*(p))) ||
+			((*(p) == ';')) || ((*(p) == ':')) ||
+			((*(p) == '&')) || ((*(p) == '=')) ||
+			((*(p) == '+')) || ((*(p) == '$')) ||
+			((*(p) == ',')))
+			ret[len++] = *p++;
+		    else {
+			int val = *(unsigned char *)p++;
+			int hi = val / 0x10, lo = val % 0x10;
+			ret[len++] = '%';
+			ret[len++] = hi + (hi > 9? 'A'-10 : '0');
+			ret[len++] = lo + (lo > 9? 'A'-10 : '0');
+		    }
+		}
+		if (len + 3 >= max) {
+                    temp = realloc2n(ret, &max);
+                    if (temp == NULL) goto mem_error;
+                    ret = temp;
+		}
+		ret[len++] = '@';
+	    }
+	    p = uri->server;
+	    while (*p != 0) {
+		if (len >= max) {
+                    temp = realloc2n(ret, &max);
+                    if (temp == NULL) goto mem_error;
+                    ret = temp;
+		}
+		ret[len++] = *p++;
+	    }
+	    if (uri->port > 0) {
+		if (len + 10 >= max) {
+                    temp = realloc2n(ret, &max);
+                    if (temp == NULL) goto mem_error;
+                    ret = temp;
+		}
+		len += snprintf(&ret[len], max - len, ":%d", uri->port);
+	    }
+	} else if (uri->authority != NULL) {
+	    if (len + 3 >= max) {
+                temp = realloc2n(ret, &max);
+                if (temp == NULL) goto mem_error;
+                ret = temp;
+	    }
+	    ret[len++] = '/';
+	    ret[len++] = '/';
+	    p = uri->authority;
+	    while (*p != 0) {
+		if (len + 3 >= max) {
+                    temp = realloc2n(ret, &max);
+                    if (temp == NULL) goto mem_error;
+                    ret = temp;
+		}
+		if ((IS_UNRESERVED(*(p))) ||
+                    ((*(p) == '$')) || ((*(p) == ',')) || ((*(p) == ';')) ||
+                    ((*(p) == ':')) || ((*(p) == '@')) || ((*(p) == '&')) ||
+                    ((*(p) == '=')) || ((*(p) == '+')))
+		    ret[len++] = *p++;
+		else {
+		    int val = *(unsigned char *)p++;
+		    int hi = val / 0x10, lo = val % 0x10;
+		    ret[len++] = '%';
+		    ret[len++] = hi + (hi > 9? 'A'-10 : '0');
+		    ret[len++] = lo + (lo > 9? 'A'-10 : '0');
+		}
+	    }
+	} else if (uri->scheme != NULL) {
+	    if (len + 3 >= max) {
+                temp = realloc2n(ret, &max);
+                if (temp == NULL) goto mem_error;
+                ret = temp;
+	    }
+	    ret[len++] = '/';
+	    ret[len++] = '/';
+	}
+	if (uri->path != NULL) {
+	    p = uri->path;
+	    /*
+	     * the colon in file:///d: should not be escaped or
+	     * Windows accesses fail later.
+	     */
+	    if ((uri->scheme != NULL) &&
+		(p[0] == '/') &&
+		(((p[1] >= 'a') && (p[1] <= 'z')) ||
+		 ((p[1] >= 'A') && (p[1] <= 'Z'))) &&
+		(p[2] == ':') &&
+	        (!strcmp(uri->scheme, "file"))) {
+		if (len + 3 >= max) {
+                    temp = realloc2n(ret, &max);
+                    if (temp == NULL) goto mem_error;
+                    ret = temp;
+		}
+		ret[len++] = *p++;
+		ret[len++] = *p++;
+		ret[len++] = *p++;
+	    }
+	    while (*p != 0) {
+		if (len + 3 >= max) {
+                    temp = realloc2n(ret, &max);
+                    if (temp == NULL) goto mem_error;
+                    ret = temp;
+		}
+		if ((IS_UNRESERVED(*(p))) || ((*(p) == '/')) ||
+                    ((*(p) == ';')) || ((*(p) == '@')) || ((*(p) == '&')) ||
+	            ((*(p) == '=')) || ((*(p) == '+')) || ((*(p) == '$')) ||
+	            ((*(p) == ',')))
+		    ret[len++] = *p++;
+		else {
+		    int val = *(unsigned char *)p++;
+		    int hi = val / 0x10, lo = val % 0x10;
+		    ret[len++] = '%';
+		    ret[len++] = hi + (hi > 9? 'A'-10 : '0');
+		    ret[len++] = lo + (lo > 9? 'A'-10 : '0');
+		}
+	    }
+	}
+	if (uri->query != NULL) {
+	    if (len + 1 >= max) {
+                temp = realloc2n(ret, &max);
+                if (temp == NULL) goto mem_error;
+                ret = temp;
+	    }
+	    ret[len++] = '?';
+	    p = uri->query;
+	    while (*p != 0) {
+		if (len + 1 >= max) {
+                    temp = realloc2n(ret, &max);
+                    if (temp == NULL) goto mem_error;
+                    ret = temp;
+		}
+		ret[len++] = *p++;
+	    }
+	}
+    }
+    if (uri->fragment != NULL) {
+	if (len + 3 >= max) {
+            temp = realloc2n(ret, &max);
+            if (temp == NULL) goto mem_error;
+            ret = temp;
+	}
+	ret[len++] = '#';
+	p = uri->fragment;
+	while (*p != 0) {
+	    if (len + 3 >= max) {
+                temp = realloc2n(ret, &max);
+                if (temp == NULL) goto mem_error;
+                ret = temp;
+	    }
+	    if ((IS_UNRESERVED(*(p))) || (IS_RESERVED(*(p))))
+		ret[len++] = *p++;
+	    else {
+		int val = *(unsigned char *)p++;
+		int hi = val / 0x10, lo = val % 0x10;
+		ret[len++] = '%';
+		ret[len++] = hi + (hi > 9? 'A'-10 : '0');
+		ret[len++] = lo + (lo > 9? 'A'-10 : '0');
+	    }
+	}
+    }
+    if (len >= max) {
+        temp = realloc2n(ret, &max);
+        if (temp == NULL) goto mem_error;
+        ret = temp;
+    }
+    ret[len] = 0;
+    return(ret);
+
+mem_error:
+    g_free(ret);
+    return(NULL);
+}
+
+/**
+ * uri_clean:
+ * @uri:  pointer to an URI
+ *
+ * Make sure the URI struct is free of content
+ */
+static void
+uri_clean(URI *uri) {
+    if (uri == NULL) return;
+
+    if (uri->scheme != NULL) g_free(uri->scheme);
+    uri->scheme = NULL;
+    if (uri->server != NULL) g_free(uri->server);
+    uri->server = NULL;
+    if (uri->user != NULL) g_free(uri->user);
+    uri->user = NULL;
+    if (uri->path != NULL) g_free(uri->path);
+    uri->path = NULL;
+    if (uri->fragment != NULL) g_free(uri->fragment);
+    uri->fragment = NULL;
+    if (uri->opaque != NULL) g_free(uri->opaque);
+    uri->opaque = NULL;
+    if (uri->authority != NULL) g_free(uri->authority);
+    uri->authority = NULL;
+    if (uri->query != NULL) g_free(uri->query);
+    uri->query = NULL;
+}
+
+/**
+ * uri_free:
+ * @uri:  pointer to an URI
+ *
+ * Free up the URI struct
+ */
+void
+uri_free(URI *uri) {
+    uri_clean(uri);
+    g_free(uri);
+}
+
+/************************************************************************
+ *									*
+ *			Helper functions				*
+ *									*
+ ************************************************************************/
+
+/**
+ * normalize_uri_path:
+ * @path:  pointer to the path string
+ *
+ * Applies the 5 normalization steps to a path string--that is, RFC 2396
+ * Section 5.2, steps 6.c through 6.g.
+ *
+ * Normalization occurs directly on the string, no new allocation is done
+ *
+ * Returns 0 or an error code
+ */
+static int
+normalize_uri_path(char *path) {
+    char *cur, *out;
+
+    if (path == NULL)
+	return(-1);
+
+    /* Skip all initial "/" chars.  We want to get to the beginning of the
+     * first non-empty segment.
+     */
+    cur = path;
+    while (cur[0] == '/')
+      ++cur;
+    if (cur[0] == '\0')
+      return(0);
+
+    /* Keep everything we've seen so far.  */
+    out = cur;
+
+    /*
+     * Analyze each segment in sequence for cases (c) and (d).
+     */
+    while (cur[0] != '\0') {
+	/*
+	 * c) All occurrences of "./", where "." is a complete path segment,
+	 *    are removed from the buffer string.
+	 */
+	if ((cur[0] == '.') && (cur[1] == '/')) {
+	    cur += 2;
+	    /* '//' normalization should be done at this point too */
+	    while (cur[0] == '/')
+		cur++;
+	    continue;
+	}
+
+	/*
+	 * d) If the buffer string ends with "." as a complete path segment,
+	 *    that "." is removed.
+	 */
+	if ((cur[0] == '.') && (cur[1] == '\0'))
+	    break;
+
+	/* Otherwise keep the segment.  */
+	while (cur[0] != '/') {
+            if (cur[0] == '\0')
+              goto done_cd;
+	    (out++)[0] = (cur++)[0];
+	}
+	/* nomalize // */
+	while ((cur[0] == '/') && (cur[1] == '/'))
+	    cur++;
+
+        (out++)[0] = (cur++)[0];
+    }
+ done_cd:
+    out[0] = '\0';
+
+    /* Reset to the beginning of the first segment for the next sequence.  */
+    cur = path;
+    while (cur[0] == '/')
+      ++cur;
+    if (cur[0] == '\0')
+	return(0);
+
+    /*
+     * Analyze each segment in sequence for cases (e) and (f).
+     *
+     * e) All occurrences of "<segment>/../", where <segment> is a
+     *    complete path segment not equal to "..", are removed from the
+     *    buffer string.  Removal of these path segments is performed
+     *    iteratively, removing the leftmost matching pattern on each
+     *    iteration, until no matching pattern remains.
+     *
+     * f) If the buffer string ends with "<segment>/..", where <segment>
+     *    is a complete path segment not equal to "..", that
+     *    "<segment>/.." is removed.
+     *
+     * To satisfy the "iterative" clause in (e), we need to collapse the
+     * string every time we find something that needs to be removed.  Thus,
+     * we don't need to keep two pointers into the string: we only need a
+     * "current position" pointer.
+     */
+    while (1) {
+        char *segp, *tmp;
+
+        /* At the beginning of each iteration of this loop, "cur" points to
+         * the first character of the segment we want to examine.
+         */
+
+        /* Find the end of the current segment.  */
+        segp = cur;
+        while ((segp[0] != '/') && (segp[0] != '\0'))
+          ++segp;
+
+        /* If this is the last segment, we're done (we need at least two
+         * segments to meet the criteria for the (e) and (f) cases).
+         */
+        if (segp[0] == '\0')
+          break;
+
+        /* If the first segment is "..", or if the next segment _isn't_ "..",
+         * keep this segment and try the next one.
+         */
+        ++segp;
+        if (((cur[0] == '.') && (cur[1] == '.') && (segp == cur+3))
+            || ((segp[0] != '.') || (segp[1] != '.')
+                || ((segp[2] != '/') && (segp[2] != '\0')))) {
+          cur = segp;
+          continue;
+        }
+
+        /* If we get here, remove this segment and the next one and back up
+         * to the previous segment (if there is one), to implement the
+         * "iteratively" clause.  It's pretty much impossible to back up
+         * while maintaining two pointers into the buffer, so just compact
+         * the whole buffer now.
+         */
+
+        /* If this is the end of the buffer, we're done.  */
+        if (segp[2] == '\0') {
+          cur[0] = '\0';
+          break;
+        }
+        /* Valgrind complained, strcpy(cur, segp + 3); */
+        /* string will overlap, do not use strcpy */
+        tmp = cur;
+        segp += 3;
+        while ((*tmp++ = *segp++) != 0)
+          ;
+
+        /* If there are no previous segments, then keep going from here.  */
+        segp = cur;
+        while ((segp > path) && ((--segp)[0] == '/'))
+          ;
+        if (segp == path)
+          continue;
+
+        /* "segp" is pointing to the end of a previous segment; find it's
+         * start.  We need to back up to the previous segment and start
+         * over with that to handle things like "foo/bar/../..".  If we
+         * don't do this, then on the first pass we'll remove the "bar/..",
+         * but be pointing at the second ".." so we won't realize we can also
+         * remove the "foo/..".
+         */
+        cur = segp;
+        while ((cur > path) && (cur[-1] != '/'))
+          --cur;
+    }
+    out[0] = '\0';
+
+    /*
+     * g) If the resulting buffer string still begins with one or more
+     *    complete path segments of "..", then the reference is
+     *    considered to be in error. Implementations may handle this
+     *    error by retaining these components in the resolved path (i.e.,
+     *    treating them as part of the final URI), by removing them from
+     *    the resolved path (i.e., discarding relative levels above the
+     *    root), or by avoiding traversal of the reference.
+     *
+     * We discard them from the final path.
+     */
+    if (path[0] == '/') {
+      cur = path;
+      while ((cur[0] == '/') && (cur[1] == '.') && (cur[2] == '.')
+             && ((cur[3] == '/') || (cur[3] == '\0')))
+	cur += 3;
+
+      if (cur != path) {
+	out = path;
+	while (cur[0] != '\0')
+          (out++)[0] = (cur++)[0];
+	out[0] = 0;
+      }
+    }
+
+    return(0);
+}
+
+static int is_hex(char c) {
+    if (((c >= '0') && (c <= '9')) ||
+        ((c >= 'a') && (c <= 'f')) ||
+        ((c >= 'A') && (c <= 'F')))
+	return(1);
+    return(0);
+}
+
+
+/**
+ * uri_string_unescape:
+ * @str:  the string to unescape
+ * @len:   the length in bytes to unescape (or <= 0 to indicate full string)
+ * @target:  optional destination buffer
+ *
+ * Unescaping routine, but does not check that the string is an URI. The
+ * output is a direct unsigned char translation of %XX values (no encoding)
+ * Note that the length of the result can only be smaller or same size as
+ * the input string.
+ *
+ * Returns a copy of the string, but unescaped, will return NULL only in case
+ * of error
+ */
+char *
+uri_string_unescape(const char *str, int len, char *target) {
+    char *ret, *out;
+    const char *in;
+
+    if (str == NULL)
+	return(NULL);
+    if (len <= 0) len = strlen(str);
+    if (len < 0) return(NULL);
+
+    if (target == NULL) {
+	ret = g_malloc(len + 1);
+    } else
+	ret = target;
+    in = str;
+    out = ret;
+    while(len > 0) {
+	if ((len > 2) && (*in == '%') && (is_hex(in[1])) && (is_hex(in[2]))) {
+	    in++;
+	    if ((*in >= '0') && (*in <= '9'))
+	        *out = (*in - '0');
+	    else if ((*in >= 'a') && (*in <= 'f'))
+	        *out = (*in - 'a') + 10;
+	    else if ((*in >= 'A') && (*in <= 'F'))
+	        *out = (*in - 'A') + 10;
+	    in++;
+	    if ((*in >= '0') && (*in <= '9'))
+	        *out = *out * 16 + (*in - '0');
+	    else if ((*in >= 'a') && (*in <= 'f'))
+	        *out = *out * 16 + (*in - 'a') + 10;
+	    else if ((*in >= 'A') && (*in <= 'F'))
+	        *out = *out * 16 + (*in - 'A') + 10;
+	    in++;
+	    len -= 3;
+	    out++;
+	} else {
+	    *out++ = *in++;
+	    len--;
+	}
+    }
+    *out = 0;
+    return(ret);
+}
+
+/**
+ * uri_string_escape:
+ * @str:  string to escape
+ * @list: exception list string of chars not to escape
+ *
+ * This routine escapes a string to hex, ignoring reserved characters (a-z)
+ * and the characters in the exception list.
+ *
+ * Returns a new escaped string or NULL in case of error.
+ */
+char *
+uri_string_escape(const char *str, const char *list) {
+    char *ret, ch;
+    char *temp;
+    const char *in;
+    int len, out;
+
+    if (str == NULL)
+	return(NULL);
+    if (str[0] == 0)
+	return(g_strdup(str));
+    len = strlen(str);
+    if (!(len > 0)) return(NULL);
+
+    len += 20;
+    ret = g_malloc(len);
+    in = str;
+    out = 0;
+    while(*in != 0) {
+	if (len - out <= 3) {
+            temp = realloc2n(ret, &len);
+	    ret = temp;
+	}
+
+	ch = *in;
+
+	if ((ch != '@') && (!IS_UNRESERVED(ch)) && (!strchr(list, ch))) {
+	    unsigned char val;
+	    ret[out++] = '%';
+	    val = ch >> 4;
+	    if (val <= 9)
+		ret[out++] = '0' + val;
+	    else
+		ret[out++] = 'A' + val - 0xA;
+	    val = ch & 0xF;
+	    if (val <= 9)
+		ret[out++] = '0' + val;
+	    else
+		ret[out++] = 'A' + val - 0xA;
+	    in++;
+	} else {
+	    ret[out++] = *in++;
+	}
+
+    }
+    ret[out] = 0;
+    return(ret);
+}
+
+/************************************************************************
+ *									*
+ *			Public functions				*
+ *									*
+ ************************************************************************/
+
+/**
+ * uri_resolve:
+ * @URI:  the URI instance found in the document
+ * @base:  the base value
+ *
+ * Computes he final URI of the reference done by checking that
+ * the given URI is valid, and building the final URI using the
+ * base URI. This is processed according to section 5.2 of the
+ * RFC 2396
+ *
+ * 5.2. Resolving Relative References to Absolute Form
+ *
+ * Returns a new URI string (to be freed by the caller) or NULL in case
+ *         of error.
+ */
+char *
+uri_resolve(const char *uri, const char *base) {
+    char *val = NULL;
+    int ret, len, indx, cur, out;
+    URI *ref = NULL;
+    URI *bas = NULL;
+    URI *res = NULL;
+
+    /*
+     * 1) The URI reference is parsed into the potential four components and
+     *    fragment identifier, as described in Section 4.3.
+     *
+     *    NOTE that a completely empty URI is treated by modern browsers
+     *    as a reference to "." rather than as a synonym for the current
+     *    URI.  Should we do that here?
+     */
+    if (uri == NULL)
+	ret = -1;
+    else {
+	if (*uri) {
+	    ref = uri_new();
+	    if (ref == NULL)
+		goto done;
+	    ret = uri_parse_into(ref, uri);
+	}
+	else
+	    ret = 0;
+    }
+    if (ret != 0)
+	goto done;
+    if ((ref != NULL) && (ref->scheme != NULL)) {
+	/*
+	 * The URI is absolute don't modify.
+	 */
+	val = g_strdup(uri);
+	goto done;
+    }
+    if (base == NULL)
+	ret = -1;
+    else {
+	bas = uri_new();
+	if (bas == NULL)
+	    goto done;
+	ret = uri_parse_into(bas, base);
+    }
+    if (ret != 0) {
+	if (ref)
+	    val = uri_to_string(ref);
+	goto done;
+    }
+    if (ref == NULL) {
+	/*
+	 * the base fragment must be ignored
+	 */
+	if (bas->fragment != NULL) {
+	    g_free(bas->fragment);
+	    bas->fragment = NULL;
+	}
+	val = uri_to_string(bas);
+	goto done;
+    }
+
+    /*
+     * 2) If the path component is empty and the scheme, authority, and
+     *    query components are undefined, then it is a reference to the
+     *    current document and we are done.  Otherwise, the reference URI's
+     *    query and fragment components are defined as found (or not found)
+     *    within the URI reference and not inherited from the base URI.
+     *
+     *    NOTE that in modern browsers, the parsing differs from the above
+     *    in the following aspect:  the query component is allowed to be
+     *    defined while still treating this as a reference to the current
+     *    document.
+     */
+    res = uri_new();
+    if (res == NULL)
+	goto done;
+    if ((ref->scheme == NULL) && (ref->path == NULL) &&
+	((ref->authority == NULL) && (ref->server == NULL))) {
+	if (bas->scheme != NULL)
+	    res->scheme = g_strdup(bas->scheme);
+	if (bas->authority != NULL)
+	    res->authority = g_strdup(bas->authority);
+	else if (bas->server != NULL) {
+	    res->server = g_strdup(bas->server);
+	    if (bas->user != NULL)
+		res->user = g_strdup(bas->user);
+	    res->port = bas->port;
+	}
+	if (bas->path != NULL)
+	    res->path = g_strdup(bas->path);
+	if (ref->query != NULL)
+	    res->query = g_strdup (ref->query);
+	else if (bas->query != NULL)
+	    res->query = g_strdup(bas->query);
+	if (ref->fragment != NULL)
+	    res->fragment = g_strdup(ref->fragment);
+	goto step_7;
+    }
+
+    /*
+     * 3) If the scheme component is defined, indicating that the reference
+     *    starts with a scheme name, then the reference is interpreted as an
+     *    absolute URI and we are done.  Otherwise, the reference URI's
+     *    scheme is inherited from the base URI's scheme component.
+     */
+    if (ref->scheme != NULL) {
+	val = uri_to_string(ref);
+	goto done;
+    }
+    if (bas->scheme != NULL)
+	res->scheme = g_strdup(bas->scheme);
+
+    if (ref->query != NULL)
+	res->query = g_strdup(ref->query);
+    if (ref->fragment != NULL)
+	res->fragment = g_strdup(ref->fragment);
+
+    /*
+     * 4) If the authority component is defined, then the reference is a
+     *    network-path and we skip to step 7.  Otherwise, the reference
+     *    URI's authority is inherited from the base URI's authority
+     *    component, which will also be undefined if the URI scheme does not
+     *    use an authority component.
+     */
+    if ((ref->authority != NULL) || (ref->server != NULL)) {
+	if (ref->authority != NULL)
+	    res->authority = g_strdup(ref->authority);
+	else {
+	    res->server = g_strdup(ref->server);
+	    if (ref->user != NULL)
+		res->user = g_strdup(ref->user);
+            res->port = ref->port;
+	}
+	if (ref->path != NULL)
+	    res->path = g_strdup(ref->path);
+	goto step_7;
+    }
+    if (bas->authority != NULL)
+	res->authority = g_strdup(bas->authority);
+    else if (bas->server != NULL) {
+	res->server = g_strdup(bas->server);
+	if (bas->user != NULL)
+	    res->user = g_strdup(bas->user);
+	res->port = bas->port;
+    }
+
+    /*
+     * 5) If the path component begins with a slash character ("/"), then
+     *    the reference is an absolute-path and we skip to step 7.
+     */
+    if ((ref->path != NULL) && (ref->path[0] == '/')) {
+	res->path = g_strdup(ref->path);
+	goto step_7;
+    }
+
+
+    /*
+     * 6) If this step is reached, then we are resolving a relative-path
+     *    reference.  The relative path needs to be merged with the base
+     *    URI's path.  Although there are many ways to do this, we will
+     *    describe a simple method using a separate string buffer.
+     *
+     * Allocate a buffer large enough for the result string.
+     */
+    len = 2; /* extra / and 0 */
+    if (ref->path != NULL)
+	len += strlen(ref->path);
+    if (bas->path != NULL)
+	len += strlen(bas->path);
+    res->path = g_malloc(len);
+    res->path[0] = 0;
+
+    /*
+     * a) All but the last segment of the base URI's path component is
+     *    copied to the buffer.  In other words, any characters after the
+     *    last (right-most) slash character, if any, are excluded.
+     */
+    cur = 0;
+    out = 0;
+    if (bas->path != NULL) {
+	while (bas->path[cur] != 0) {
+	    while ((bas->path[cur] != 0) && (bas->path[cur] != '/'))
+		cur++;
+	    if (bas->path[cur] == 0)
+		break;
+
+	    cur++;
+	    while (out < cur) {
+		res->path[out] = bas->path[out];
+		out++;
+	    }
+	}
+    }
+    res->path[out] = 0;
+
+    /*
+     * b) The reference's path component is appended to the buffer
+     *    string.
+     */
+    if (ref->path != NULL && ref->path[0] != 0) {
+	indx = 0;
+	/*
+	 * Ensure the path includes a '/'
+	 */
+	if ((out == 0) && (bas->server != NULL))
+	    res->path[out++] = '/';
+	while (ref->path[indx] != 0) {
+	    res->path[out++] = ref->path[indx++];
+	}
+    }
+    res->path[out] = 0;
+
+    /*
+     * Steps c) to h) are really path normalization steps
+     */
+    normalize_uri_path(res->path);
+
+step_7:
+
+    /*
+     * 7) The resulting URI components, including any inherited from the
+     *    base URI, are recombined to give the absolute form of the URI
+     *    reference.
+     */
+    val = uri_to_string(res);
+
+done:
+    if (ref != NULL)
+	uri_free(ref);
+    if (bas != NULL)
+	uri_free(bas);
+    if (res != NULL)
+	uri_free(res);
+    return(val);
+}
+
+/**
+ * uri_resolve_relative:
+ * @URI:  the URI reference under consideration
+ * @base:  the base value
+ *
+ * Expresses the URI of the reference in terms relative to the
+ * base.  Some examples of this operation include:
+ *     base = "http://site1.com/docs/book1.html"
+ *        URI input                        URI returned
+ *     docs/pic1.gif                    pic1.gif
+ *     docs/img/pic1.gif                img/pic1.gif
+ *     img/pic1.gif                     ../img/pic1.gif
+ *     http://site1.com/docs/pic1.gif   pic1.gif
+ *     http://site2.com/docs/pic1.gif   http://site2.com/docs/pic1.gif
+ *
+ *     base = "docs/book1.html"
+ *        URI input                        URI returned
+ *     docs/pic1.gif                    pic1.gif
+ *     docs/img/pic1.gif                img/pic1.gif
+ *     img/pic1.gif                     ../img/pic1.gif
+ *     http://site1.com/docs/pic1.gif   http://site1.com/docs/pic1.gif
+ *
+ *
+ * Note: if the URI reference is really wierd or complicated, it may be
+ *       worthwhile to first convert it into a "nice" one by calling
+ *       uri_resolve (using 'base') before calling this routine,
+ *       since this routine (for reasonable efficiency) assumes URI has
+ *       already been through some validation.
+ *
+ * Returns a new URI string (to be freed by the caller) or NULL in case
+ * error.
+ */
+char *
+uri_resolve_relative (const char *uri, const char * base)
+{
+    char *val = NULL;
+    int ret;
+    int ix;
+    int pos = 0;
+    int nbslash = 0;
+    int len;
+    URI *ref = NULL;
+    URI *bas = NULL;
+    char *bptr, *uptr, *vptr;
+    int remove_path = 0;
+
+    if ((uri == NULL) || (*uri == 0))
+	return NULL;
+
+    /*
+     * First parse URI into a standard form
+     */
+    ref = uri_new ();
+    if (ref == NULL)
+	return NULL;
+    /* If URI not already in "relative" form */
+    if (uri[0] != '.') {
+	ret = uri_parse_into (ref, uri);
+	if (ret != 0)
+	    goto done;		/* Error in URI, return NULL */
+    } else
+	ref->path = g_strdup(uri);
+
+    /*
+     * Next parse base into the same standard form
+     */
+    if ((base == NULL) || (*base == 0)) {
+	val = g_strdup (uri);
+	goto done;
+    }
+    bas = uri_new ();
+    if (bas == NULL)
+	goto done;
+    if (base[0] != '.') {
+	ret = uri_parse_into (bas, base);
+	if (ret != 0)
+	    goto done;		/* Error in base, return NULL */
+    } else
+	bas->path = g_strdup(base);
+
+    /*
+     * If the scheme / server on the URI differs from the base,
+     * just return the URI
+     */
+    if ((ref->scheme != NULL) &&
+	((bas->scheme == NULL) ||
+	 (strcmp (bas->scheme, ref->scheme)) ||
+	 (strcmp (bas->server, ref->server)))) {
+	val = g_strdup (uri);
+	goto done;
+    }
+    if (!strcmp(bas->path, ref->path)) {
+	val = g_strdup("");
+	goto done;
+    }
+    if (bas->path == NULL) {
+	val = g_strdup(ref->path);
+	goto done;
+    }
+    if (ref->path == NULL) {
+        ref->path = (char *) "/";
+	remove_path = 1;
+    }
+
+    /*
+     * At this point (at last!) we can compare the two paths
+     *
+     * First we take care of the special case where either of the
+     * two path components may be missing (bug 316224)
+     */
+    if (bas->path == NULL) {
+	if (ref->path != NULL) {
+	    uptr = ref->path;
+	    if (*uptr == '/')
+		uptr++;
+	    /* exception characters from uri_to_string */
+	    val = uri_string_escape(uptr, "/;&=+$,");
+	}
+	goto done;
+    }
+    bptr = bas->path;
+    if (ref->path == NULL) {
+	for (ix = 0; bptr[ix] != 0; ix++) {
+	    if (bptr[ix] == '/')
+		nbslash++;
+	}
+	uptr = NULL;
+	len = 1;	/* this is for a string terminator only */
+    } else {
+    /*
+     * Next we compare the two strings and find where they first differ
+     */
+	if ((ref->path[pos] == '.') && (ref->path[pos+1] == '/'))
+            pos += 2;
+	if ((*bptr == '.') && (bptr[1] == '/'))
+            bptr += 2;
+	else if ((*bptr == '/') && (ref->path[pos] != '/'))
+	    bptr++;
+	while ((bptr[pos] == ref->path[pos]) && (bptr[pos] != 0))
+	    pos++;
+
+	if (bptr[pos] == ref->path[pos]) {
+	    val = g_strdup("");
+	    goto done;		/* (I can't imagine why anyone would do this) */
+	}
+
+	/*
+	 * In URI, "back up" to the last '/' encountered.  This will be the
+	 * beginning of the "unique" suffix of URI
+	 */
+	ix = pos;
+	if ((ref->path[ix] == '/') && (ix > 0))
+	    ix--;
+	else if ((ref->path[ix] == 0) && (ix > 1) && (ref->path[ix - 1] == '/'))
+	    ix -= 2;
+	for (; ix > 0; ix--) {
+	    if (ref->path[ix] == '/')
+		break;
+	}
+	if (ix == 0) {
+	    uptr = ref->path;
+	} else {
+	    ix++;
+	    uptr = &ref->path[ix];
+	}
+
+	/*
+	 * In base, count the number of '/' from the differing point
+	 */
+	if (bptr[pos] != ref->path[pos]) {/* check for trivial URI == base */
+	    for (; bptr[ix] != 0; ix++) {
+		if (bptr[ix] == '/')
+		    nbslash++;
+	    }
+	}
+	len = strlen (uptr) + 1;
+    }
+
+    if (nbslash == 0) {
+	if (uptr != NULL)
+	    /* exception characters from uri_to_string */
+	    val = uri_string_escape(uptr, "/;&=+$,");
+	goto done;
+    }
+
+    /*
+     * Allocate just enough space for the returned string -
+     * length of the remainder of the URI, plus enough space
+     * for the "../" groups, plus one for the terminator
+     */
+    val = g_malloc (len + 3 * nbslash);
+    vptr = val;
+    /*
+     * Put in as many "../" as needed
+     */
+    for (; nbslash>0; nbslash--) {
+	*vptr++ = '.';
+	*vptr++ = '.';
+	*vptr++ = '/';
+    }
+    /*
+     * Finish up with the end of the URI
+     */
+    if (uptr != NULL) {
+        if ((vptr > val) && (len > 0) &&
+	    (uptr[0] == '/') && (vptr[-1] == '/')) {
+	    memcpy (vptr, uptr + 1, len - 1);
+	    vptr[len - 2] = 0;
+	} else {
+	    memcpy (vptr, uptr, len);
+	    vptr[len - 1] = 0;
+	}
+    } else {
+	vptr[len - 1] = 0;
+    }
+
+    /* escape the freshly-built path */
+    vptr = val;
+	/* exception characters from uri_to_string */
+    val = uri_string_escape(vptr, "/;&=+$,");
+    g_free(vptr);
+
+done:
+    /*
+     * Free the working variables
+     */
+    if (remove_path != 0)
+        ref->path = NULL;
+    if (ref != NULL)
+	uri_free (ref);
+    if (bas != NULL)
+	uri_free (bas);
+
+    return val;
+}
+
+/*
+ * Utility functions to help parse and assemble query strings.
+ */
+
+struct QueryParams *
+query_params_new (int init_alloc)
+{
+    struct QueryParams *ps;
+
+    if (init_alloc <= 0) init_alloc = 1;
+
+    ps = g_new(QueryParams, 1);
+    ps->n = 0;
+    ps->alloc = init_alloc;
+    ps->p = g_new(QueryParam, ps->alloc);
+
+    return ps;
+}
+
+/* Ensure there is space to store at least one more parameter
+ * at the end of the set.
+ */
+static int
+query_params_append (struct QueryParams *ps,
+               const char *name, const char *value)
+{
+    if (ps->n >= ps->alloc) {
+        ps->p = g_renew(QueryParam, ps->p, ps->alloc * 2);
+        ps->alloc *= 2;
+    }
+
+    ps->p[ps->n].name = g_strdup(name);
+    ps->p[ps->n].value = value ? g_strdup(value) : NULL;
+    ps->p[ps->n].ignore = 0;
+    ps->n++;
+
+    return 0;
+}
+
+void
+query_params_free (struct QueryParams *ps)
+{
+    int i;
+
+    for (i = 0; i < ps->n; ++i) {
+        g_free (ps->p[i].name);
+        g_free (ps->p[i].value);
+    }
+    g_free (ps->p);
+    g_free (ps);
+}
+
+struct QueryParams *
+query_params_parse (const char *query)
+{
+    struct QueryParams *ps;
+    const char *end, *eq;
+
+    ps = query_params_new (0);
+    if (!query || query[0] == '\0') return ps;
+
+    while (*query) {
+        char *name = NULL, *value = NULL;
+
+        /* Find the next separator, or end of the string. */
+        end = strchr (query, '&');
+        if (!end)
+            end = strchr (query, ';');
+        if (!end)
+            end = query + strlen (query);
+
+        /* Find the first '=' character between here and end. */
+        eq = strchr (query, '=');
+        if (eq && eq >= end) eq = NULL;
+
+        /* Empty section (eg. "&&"). */
+        if (end == query)
+            goto next;
+
+        /* If there is no '=' character, then we have just "name"
+         * and consistent with CGI.pm we assume value is "".
+         */
+        else if (!eq) {
+            name = uri_string_unescape (query, end - query, NULL);
+            value = NULL;
+        }
+        /* Or if we have "name=" here (works around annoying
+         * problem when calling uri_string_unescape with len = 0).
+         */
+        else if (eq+1 == end) {
+            name = uri_string_unescape (query, eq - query, NULL);
+            value = g_new0(char, 1);
+        }
+        /* If the '=' character is at the beginning then we have
+         * "=value" and consistent with CGI.pm we _ignore_ this.
+         */
+        else if (query == eq)
+            goto next;
+
+        /* Otherwise it's "name=value". */
+        else {
+            name = uri_string_unescape (query, eq - query, NULL);
+            value = uri_string_unescape (eq+1, end - (eq+1), NULL);
+        }
+
+        /* Append to the parameter set. */
+        query_params_append (ps, name, value);
+        g_free(name);
+        g_free(value);
+
+    next:
+        query = end;
+        if (*query) query ++; /* skip '&' separator */
+    }
+
+    return ps;
+}
diff --git a/uri.h b/uri.h
new file mode 100644
index 000000000..de99b3bd4
--- /dev/null
+++ b/uri.h
@@ -0,0 +1,113 @@
+/**
+ * Summary: library of generic URI related routines
+ * Description: library of generic URI related routines
+ *              Implements RFC 2396
+ *
+ * Copyright (C) 1998-2003 Daniel Veillard.  All Rights Reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+ * DANIEL VEILLARD BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Except as contained in this notice, the name of Daniel Veillard shall not
+ * be used in advertising or otherwise to promote the sale, use or other
+ * dealings in this Software without prior written authorization from him.
+ *
+ * Author: Daniel Veillard
+ **
+ * Copyright (C) 2007 Red Hat, Inc.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307  USA
+ *
+ * Authors:
+ *    Richard W.M. Jones <rjones@redhat.com>
+ *
+ * Utility functions to help parse and assemble query strings.
+ */
+
+#ifndef QEMU_URI_H
+#define QEMU_URI_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * URI:
+ *
+ * A parsed URI reference. This is a struct containing the various fields
+ * as described in RFC 2396 but separated for further processing.
+ */
+typedef struct URI {
+    char *scheme;	/* the URI scheme */
+    char *opaque;	/* opaque part */
+    char *authority;	/* the authority part */
+    char *server;	/* the server part */
+    char *user;		/* the user part */
+    int port;		/* the port number */
+    char *path;		/* the path string */
+    char *fragment;	/* the fragment identifier */
+    int  cleanup;	/* parsing potentially unclean URI */
+    char *query;	/* the query string (as it appears in the URI) */
+} URI;
+
+URI *uri_new(void);
+char *uri_resolve(const char *URI, const char *base);
+char *uri_resolve_relative(const char *URI, const char *base);
+URI *uri_parse(const char *str);
+URI *uri_parse_raw(const char *str, int raw);
+int uri_parse_into(URI *uri, const char *str);
+char *uri_to_string(URI *uri);
+char *uri_string_escape(const char *str, const char *list);
+char *uri_string_unescape(const char *str, int len, char *target);
+void uri_free(URI *uri);
+
+/* Single web service query parameter 'name=value'. */
+typedef struct QueryParam {
+  char *name;			/* Name (unescaped). */
+  char *value;			/* Value (unescaped). */
+  int ignore;			/* Ignore this field in qparam_get_query */
+} QueryParam;
+
+/* Set of parameters. */
+typedef struct QueryParams {
+  int n;			/* number of parameters used */
+  int alloc;			/* allocated space */
+  QueryParam *p;		/* array of parameters */
+} QueryParams;
+
+struct QueryParams *query_params_new (int init_alloc);
+int query_param_append (QueryParams *ps, const char *name, const char *value);
+extern char *query_param_to_string (const QueryParams *ps);
+extern QueryParams *query_params_parse (const char *query);
+extern void query_params_free (QueryParams *ps);
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* QEMU_URI_H */
diff --git a/user-exec.c b/user-exec.c
index b9ea9dd32..ef9b1727b 100644
--- a/user-exec.c
+++ b/user-exec.c
@@ -18,9 +18,6 @@
  */
 #include "config.h"
 #include "cpu.h"
-#ifndef CONFIG_TCG_PASS_AREG0
-#include "dyngen-exec.h"
-#endif
 #include "disas.h"
 #include "tcg.h"
 
@@ -60,12 +57,6 @@ void cpu_resume_from_signal(CPUArchState *env1, void *puc)
     struct sigcontext *uc = puc;
 #endif
 
-#ifndef CONFIG_TCG_PASS_AREG0
-    env = env1;
-
-    /* XXX: restore cpu registers saved in host registers */
-#endif
-
     if (puc) {
         /* XXX: use siglongjmp ? */
 #ifdef __linux__
@@ -93,11 +84,6 @@ static inline int handle_cpu_signal(uintptr_t pc, unsigned long address,
     TranslationBlock *tb;
     int ret;
 
-#ifndef CONFIG_TCG_PASS_AREG0
-    if (cpu_single_env) {
-        env = cpu_single_env; /* XXX: find a correct solution for multithread */
-    }
-#endif
 #if defined(DEBUG_SIGNAL)
     qemu_printf("qemu: SIGSEGV pc=0x%08lx address=%08lx w=%d oldset=0x%08lx\n",
                 pc, address, is_write, *(unsigned long *)old_set);
diff --git a/vl.c b/vl.c
index 7c577fa54..a3ab3841a 100644
--- a/vl.c
+++ b/vl.c
@@ -168,6 +168,7 @@ int main(int argc, char **argv)
 #include "osdep.h"
 
 #include "ui/qemu-spice.h"
+#include "qapi/string-input-visitor.h"
 
 //#define DEBUG_NET
 //#define DEBUG_SLIRP
@@ -180,7 +181,7 @@ static const char *data_dir;
 const char *bios_name = NULL;
 enum vga_retrace_method vga_retrace_method = VGA_RETRACE_DUMB;
 DisplayType display_type = DT_DEFAULT;
-int display_remote = 0;
+static int display_remote;
 const char* keyboard_layout = NULL;
 ram_addr_t ram_size;
 const char *mem_path = NULL;
@@ -203,7 +204,6 @@ CharDriverState *serial_hds[MAX_SERIAL_PORTS];
 CharDriverState *parallel_hds[MAX_PARALLEL_PORTS];
 CharDriverState *virtcon_hds[MAX_VIRTIO_CONSOLES];
 int win2k_install_hack = 0;
-int usb_enabled = 0;
 int singlestep = 0;
 int smp_cpus = 1;
 int max_cpus = 0;
@@ -215,7 +215,7 @@ const char *vnc_display;
 int acpi_enabled = 1;
 int no_hpet = 0;
 int fd_bootchk = 1;
-int no_reboot = 0;
+static int no_reboot;
 int no_shutdown = 0;
 int cursor_hide = 1;
 int graphic_rotate = 0;
@@ -243,7 +243,8 @@ struct FWBootEntry {
     char *suffix;
 };
 
-QTAILQ_HEAD(, FWBootEntry) fw_boot_order = QTAILQ_HEAD_INITIALIZER(fw_boot_order);
+static QTAILQ_HEAD(, FWBootEntry) fw_boot_order =
+    QTAILQ_HEAD_INITIALIZER(fw_boot_order);
 
 int nb_numa_nodes;
 uint64_t node_mem[MAX_NODES];
@@ -341,7 +342,7 @@ static const RunStateTransition runstate_transitions_def[] = {
     { RUN_STATE_DEBUG, RUN_STATE_RUNNING },
 
     { RUN_STATE_INMIGRATE, RUN_STATE_RUNNING },
-    { RUN_STATE_INMIGRATE, RUN_STATE_PRELAUNCH },
+    { RUN_STATE_INMIGRATE, RUN_STATE_PAUSED },
 
     { RUN_STATE_INTERNAL_ERROR, RUN_STATE_PAUSED },
     { RUN_STATE_INTERNAL_ERROR, RUN_STATE_FINISH_MIGRATE },
@@ -397,7 +398,7 @@ bool runstate_check(RunState state)
     return current_run_state == state;
 }
 
-void runstate_init(void)
+static void runstate_init(void)
 {
     const RunStateTransition *p;
 
@@ -790,6 +791,89 @@ static int parse_sandbox(QemuOpts *opts, void *opaque)
     return 0;
 }
 
+/*********QEMU USB setting******/
+bool usb_enabled(bool default_usb)
+{
+    QemuOpts *mach_opts;
+    mach_opts = qemu_opts_find(qemu_find_opts("machine"), 0);
+    if (mach_opts) {
+        return qemu_opt_get_bool(mach_opts, "usb", default_usb);
+    }
+    return default_usb;
+}
+
+#ifndef _WIN32
+static int parse_add_fd(QemuOpts *opts, void *opaque)
+{
+    int fd, dupfd, flags;
+    int64_t fdset_id;
+    const char *fd_opaque = NULL;
+
+    fd = qemu_opt_get_number(opts, "fd", -1);
+    fdset_id = qemu_opt_get_number(opts, "set", -1);
+    fd_opaque = qemu_opt_get(opts, "opaque");
+
+    if (fd < 0) {
+        qerror_report(ERROR_CLASS_GENERIC_ERROR,
+                      "fd option is required and must be non-negative");
+        return -1;
+    }
+
+    if (fd <= STDERR_FILENO) {
+        qerror_report(ERROR_CLASS_GENERIC_ERROR,
+                      "fd cannot be a standard I/O stream");
+        return -1;
+    }
+
+    /*
+     * All fds inherited across exec() necessarily have FD_CLOEXEC
+     * clear, while qemu sets FD_CLOEXEC on all other fds used internally.
+     */
+    flags = fcntl(fd, F_GETFD);
+    if (flags == -1 || (flags & FD_CLOEXEC)) {
+        qerror_report(ERROR_CLASS_GENERIC_ERROR,
+                      "fd is not valid or already in use");
+        return -1;
+    }
+
+    if (fdset_id < 0) {
+        qerror_report(ERROR_CLASS_GENERIC_ERROR,
+                      "set option is required and must be non-negative");
+        return -1;
+    }
+
+#ifdef F_DUPFD_CLOEXEC
+    dupfd = fcntl(fd, F_DUPFD_CLOEXEC, 0);
+#else
+    dupfd = dup(fd);
+    if (dupfd != -1) {
+        qemu_set_cloexec(dupfd);
+    }
+#endif
+    if (dupfd == -1) {
+        qerror_report(ERROR_CLASS_GENERIC_ERROR,
+                      "Error duplicating fd: %s", strerror(errno));
+        return -1;
+    }
+
+    /* add the duplicate fd, and optionally the opaque string, to the fd set */
+    monitor_fdset_add_fd(dupfd, true, fdset_id, fd_opaque ? true : false,
+                         fd_opaque, NULL);
+
+    return 0;
+}
+
+static int cleanup_add_fd(QemuOpts *opts, void *opaque)
+{
+    int fd;
+
+    fd = qemu_opt_get_number(opts, "fd", -1);
+    close(fd);
+
+    return 0;
+}
+#endif
+
 /***********************************************************/
 /* QEMU Block devices */
 
@@ -1023,7 +1107,6 @@ static void numa_add(const char *optarg)
         }
         nb_numa_nodes++;
     }
-    return;
 }
 
 static void smp_parse(const char *optarg)
@@ -1078,8 +1161,9 @@ static int usb_device_add(const char *devname)
     const char *p;
     USBDevice *dev = NULL;
 
-    if (!usb_enabled)
+    if (!usb_enabled(false)) {
         return -1;
+    }
 
     /* drivers with .usbdevice_name entry in USBDeviceInfo */
     dev = usbdevice_create(devname);
@@ -1115,8 +1199,9 @@ static int usb_device_del(const char *devname)
     if (strstart(devname, "host:", &p))
         return usb_host_device_close(p);
 
-    if (!usb_enabled)
+    if (!usb_enabled(false)) {
         return -1;
+    }
 
     p = strchr(devname, '.');
     if (!p)
@@ -1276,19 +1361,51 @@ static void gui_update(void *opaque)
 {
     uint64_t interval = GUI_REFRESH_INTERVAL;
     DisplayState *ds = opaque;
-    DisplayChangeListener *dcl = ds->listeners;
+    DisplayChangeListener *dcl;
 
     dpy_refresh(ds);
 
-    while (dcl != NULL) {
+    QLIST_FOREACH(dcl, &ds->listeners, next) {
         if (dcl->gui_timer_interval &&
             dcl->gui_timer_interval < interval)
             interval = dcl->gui_timer_interval;
-        dcl = dcl->next;
     }
     qemu_mod_timer(ds->gui_timer, interval + qemu_get_clock_ms(rt_clock));
 }
 
+void gui_setup_refresh(DisplayState *ds)
+{
+    DisplayChangeListener *dcl;
+    bool need_timer = false;
+    bool have_gfx = false;
+    bool have_text = false;
+
+    QLIST_FOREACH(dcl, &ds->listeners, next) {
+        if (dcl->dpy_refresh != NULL) {
+            need_timer = true;
+        }
+        if (dcl->dpy_gfx_update != NULL) {
+            have_gfx = true;
+        }
+        if (dcl->dpy_text_update != NULL) {
+            have_text = true;
+        }
+    }
+
+    if (need_timer && ds->gui_timer == NULL) {
+        ds->gui_timer = qemu_new_timer_ms(rt_clock, gui_update, ds);
+        qemu_mod_timer(ds->gui_timer, qemu_get_clock_ms(rt_clock));
+    }
+    if (!need_timer && ds->gui_timer != NULL) {
+        qemu_del_timer(ds->gui_timer);
+        qemu_free_timer(ds->gui_timer);
+        ds->gui_timer = NULL;
+    }
+
+    ds->have_gfx = have_gfx;
+    ds->have_text = have_text;
+}
+
 struct vm_change_state_entry {
     VMChangeStateHandler *cb;
     void *opaque;
@@ -1355,6 +1472,8 @@ static int powerdown_requested;
 static int debug_requested;
 static int suspend_requested;
 static int wakeup_requested;
+static NotifierList powerdown_notifiers =
+    NOTIFIER_LIST_INITIALIZER(powerdown_notifiers);
 static NotifierList suspend_notifiers =
     NOTIFIER_LIST_INITIALIZER(suspend_notifiers);
 static NotifierList wakeup_notifiers =
@@ -1372,14 +1491,14 @@ int qemu_reset_requested_get(void)
     return reset_requested;
 }
 
-int qemu_shutdown_requested(void)
+static int qemu_shutdown_requested(void)
 {
     int r = shutdown_requested;
     shutdown_requested = 0;
     return r;
 }
 
-void qemu_kill_report(void)
+static void qemu_kill_report(void)
 {
     if (!qtest_enabled() && shutdown_signal != -1) {
         fprintf(stderr, "qemu: terminating on signal %d", shutdown_signal);
@@ -1395,7 +1514,7 @@ void qemu_kill_report(void)
     }
 }
 
-int qemu_reset_requested(void)
+static int qemu_reset_requested(void)
 {
     int r = reset_requested;
     reset_requested = 0;
@@ -1416,7 +1535,7 @@ static int qemu_wakeup_requested(void)
     return r;
 }
 
-int qemu_powerdown_requested(void)
+static int qemu_powerdown_requested(void)
 {
     int r = powerdown_requested;
     powerdown_requested = 0;
@@ -1563,12 +1682,23 @@ void qemu_system_shutdown_request(void)
     qemu_notify_event();
 }
 
+static void qemu_system_powerdown(void)
+{
+    monitor_protocol_event(QEVENT_POWERDOWN, NULL);
+    notifier_list_notify(&powerdown_notifiers, NULL);
+}
+
 void qemu_system_powerdown_request(void)
 {
     powerdown_requested = 1;
     qemu_notify_event();
 }
 
+void qemu_register_powerdown_notifier(Notifier *notifier)
+{
+    notifier_list_add(&powerdown_notifiers, notifier);
+}
+
 void qemu_system_debug_request(void)
 {
     debug_requested = 1;
@@ -1581,8 +1711,6 @@ void qemu_system_vmstop_request(RunState state)
     qemu_notify_event();
 }
 
-qemu_irq qemu_system_powerdown;
-
 static bool main_loop_should_exit(void)
 {
     RunState r;
@@ -1619,8 +1747,7 @@ static bool main_loop_should_exit(void)
         monitor_protocol_event(QEVENT_WAKEUP, NULL);
     }
     if (qemu_powerdown_requested()) {
-        monitor_protocol_event(QEVENT_POWERDOWN, NULL);
-        qemu_irq_raise(qemu_system_powerdown);
+        qemu_system_powerdown();
     }
     if (qemu_vmstop_requested(&r)) {
         vm_stop(r);
@@ -1690,17 +1817,23 @@ static const QEMUOption qemu_options[] = {
 
 static bool vga_available(void)
 {
-    return qdev_exists("VGA") || qdev_exists("isa-vga");
+    return object_class_by_name("VGA") || object_class_by_name("isa-vga");
 }
 
 static bool cirrus_vga_available(void)
 {
-    return qdev_exists("cirrus-vga") || qdev_exists("isa-cirrus-vga");
+    return object_class_by_name("cirrus-vga")
+           || object_class_by_name("isa-cirrus-vga");
 }
 
 static bool vmware_vga_available(void)
 {
-    return qdev_exists("vmware-svga");
+    return object_class_by_name("vmware-svga");
+}
+
+static bool qxl_vga_available(void)
+{
+    return object_class_by_name("qxl-vga");
 }
 
 static void select_vgahw (const char *p)
@@ -1732,7 +1865,12 @@ static void select_vgahw (const char *p)
     } else if (strstart(p, "xenfb", &opts)) {
         vga_interface_type = VGA_XENFB;
     } else if (strstart(p, "qxl", &opts)) {
-        vga_interface_type = VGA_QXL;
+        if (qxl_vga_available()) {
+            vga_interface_type = VGA_QXL;
+        } else {
+            fprintf(stderr, "Error: QXL VGA not available\n");
+            exit(0);
+        }
     } else if (!strstart(p, "none", &opts)) {
     invalid_vga:
         fprintf(stderr, "Unknown vga type: %s\n", p);
@@ -2023,7 +2161,9 @@ struct device_config {
     Location loc;
     QTAILQ_ENTRY(device_config) next;
 };
-QTAILQ_HEAD(, device_config) device_configs = QTAILQ_HEAD_INITIALIZER(device_configs);
+
+static QTAILQ_HEAD(, device_config) device_configs =
+    QTAILQ_HEAD_INITIALIZER(device_configs);
 
 static void add_device_config(int type, const char *cmdline)
 {
@@ -2337,9 +2477,51 @@ static void free_and_trace(gpointer mem)
     free(mem);
 }
 
-int qemu_init_main_loop(void)
+static int object_set_property(const char *name, const char *value, void *opaque)
+{
+    Object *obj = OBJECT(opaque);
+    StringInputVisitor *siv;
+    Error *local_err = NULL;
+
+    if (strcmp(name, "qom-type") == 0 || strcmp(name, "id") == 0) {
+        return 0;
+    }
+
+    siv = string_input_visitor_new(value);
+    object_property_set(obj, string_input_get_visitor(siv), name, &local_err);
+    string_input_visitor_cleanup(siv);
+
+    if (local_err) {
+        qerror_report_err(local_err);
+        error_free(local_err);
+        return -1;
+    }
+
+    return 0;
+}
+
+static int object_create(QemuOpts *opts, void *opaque)
 {
-    return main_loop_init();
+    const char *type = qemu_opt_get(opts, "qom-type");
+    const char *id = qemu_opts_id(opts);
+    Object *obj;
+
+    g_assert(type != NULL);
+
+    if (id == NULL) {
+        qerror_report(QERR_MISSING_PARAMETER, "id");
+        return -1;
+    }
+
+    obj = object_new(type);
+    if (qemu_opt_foreach(opts, object_set_property, obj, 1) < 0) {
+        return -1;
+    }
+
+    object_property_add_child(container_get(object_get_root(), "/objects"),
+                              id, obj, NULL);
+
+    return 0;
 }
 
 int main(int argc, char **argv, char **envp)
@@ -2351,7 +2533,6 @@ int main(int argc, char **argv, char **envp)
     const char *kernel_filename, *kernel_cmdline;
     char boot_devices[33] = "cad"; /* default to HD->floppy->CD-ROM */
     DisplayState *ds;
-    DisplayChangeListener *dcl;
     int cyls, heads, secs, translation;
     QemuOpts *hda_opts = NULL, *opts, *machine_opts;
     QemuOptsList *olist;
@@ -2470,6 +2651,11 @@ int main(int argc, char **argv, char **envp)
             case QEMU_OPTION_M:
                 machine = machine_parse(optarg);
                 break;
+            case QEMU_OPTION_no_kvm_irqchip: {
+                olist = qemu_find_opts("machine");
+                qemu_opts_parse(olist, "kernel_irqchip=off", 0);
+                break;
+            }
             case QEMU_OPTION_cpu:
                 /* hw initialization will check this */
                 cpu_model = optarg;
@@ -2622,7 +2808,8 @@ int main(int argc, char **argv, char **envp)
                 {
                     static const char * const params[] = {
                         "order", "once", "menu",
-                        "splash", "splash-time", NULL
+                        "splash", "splash-time",
+                        "reboot-timeout", NULL
                     };
                     char buf[sizeof(boot_devices)];
                     char *standard_boot_devices;
@@ -3061,11 +3248,37 @@ int main(int argc, char **argv, char **envp)
                     machine = machine_parse(optarg);
                 }
                 break;
+             case QEMU_OPTION_no_kvm:
+                olist = qemu_find_opts("machine");
+                qemu_opts_parse(olist, "accel=tcg", 0);
+                break;
+            case QEMU_OPTION_no_kvm_pit: {
+                fprintf(stderr, "Warning: KVM PIT can no longer be disabled "
+                                "separately.\n");
+                break;
+            }
+            case QEMU_OPTION_no_kvm_pit_reinjection: {
+                static GlobalProperty kvm_pit_lost_tick_policy[] = {
+                    {
+                        .driver   = "kvm-pit",
+                        .property = "lost_tick_policy",
+                        .value    = "discard",
+                    },
+                    { /* end of list */ }
+                };
+
+                fprintf(stderr, "Warning: option deprecated, use "
+                        "lost_tick_policy property of kvm-pit instead.\n");
+                qdev_prop_register_global_list(kvm_pit_lost_tick_policy);
+                break;
+            }
             case QEMU_OPTION_usb:
-                usb_enabled = 1;
+                olist = qemu_find_opts("machine");
+                qemu_opts_parse(olist, "usb=on", 0);
                 break;
             case QEMU_OPTION_usbdevice:
-                usb_enabled = 1;
+                olist = qemu_find_opts("machine");
+                qemu_opts_parse(olist, "usb=on", 0);
                 add_device_config(DEV_USB, optarg);
                 break;
             case QEMU_OPTION_device:
@@ -3144,6 +3357,10 @@ int main(int argc, char **argv, char **envp)
             case QEMU_OPTION_semihosting:
                 semihosting_enabled = 1;
                 break;
+            case QEMU_OPTION_tdf:
+                fprintf(stderr, "Warning: user space PIT time drift fix "
+                                "is no longer supported.\n");
+                break;
             case QEMU_OPTION_name:
                 qemu_name = g_strdup(optarg);
 		 {
@@ -3288,6 +3505,21 @@ int main(int argc, char **argv, char **envp)
                     exit(0);
                 }
                 break;
+            case QEMU_OPTION_add_fd:
+#ifndef _WIN32
+                opts = qemu_opts_parse(qemu_find_opts("add-fd"), optarg, 0);
+                if (!opts) {
+                    exit(0);
+                }
+#else
+                error_report("File descriptor passing is disabled on this "
+                             "platform");
+                exit(1);
+#endif
+                break;
+            case QEMU_OPTION_object:
+                opts = qemu_opts_parse(qemu_find_opts("object"), optarg, 1);
+                break;
             default:
                 os_parse_cmd_args(popt->index, optarg);
             }
@@ -3295,10 +3527,25 @@ int main(int argc, char **argv, char **envp)
     }
     loc_set_none();
 
+    if (qemu_init_main_loop()) {
+        fprintf(stderr, "qemu_init_main_loop failed\n");
+        exit(1);
+    }
+
     if (qemu_opts_foreach(qemu_find_opts("sandbox"), parse_sandbox, NULL, 0)) {
         exit(1);
     }
 
+#ifndef _WIN32
+    if (qemu_opts_foreach(qemu_find_opts("add-fd"), parse_add_fd, NULL, 1)) {
+        exit(1);
+    }
+
+    if (qemu_opts_foreach(qemu_find_opts("add-fd"), cleanup_add_fd, NULL, 1)) {
+        exit(1);
+    }
+#endif
+
     if (machine == NULL) {
         fprintf(stderr, "No machine found.\n");
         exit(1);
@@ -3308,6 +3555,11 @@ int main(int argc, char **argv, char **envp)
         qemu_set_version(machine->hw_version);
     }
 
+    if (qemu_opts_foreach(qemu_find_opts("object"),
+                          object_create, NULL, 0) != 0) {
+        exit(1);
+    }
+
     /* Init CPU def lists, based on config
      * - Must be called after all the qemu_read_config_file() calls
      * - Must be called before list_cpus()
@@ -3447,12 +3699,6 @@ int main(int argc, char **argv, char **envp)
 
     configure_accelerator();
 
-    qemu_init_cpu_loop();
-    if (qemu_init_main_loop()) {
-        fprintf(stderr, "qemu_init_main_loop failed\n");
-        exit(1);
-    }
-
     machine_opts = qemu_opts_find(qemu_find_opts("machine"), 0);
     if (machine_opts) {
         kernel_filename = qemu_opt_get(machine_opts, "kernel");
@@ -3485,10 +3731,8 @@ int main(int argc, char **argv, char **envp)
 
     os_set_line_buffering();
 
-    if (init_timer_alarm() < 0) {
-        fprintf(stderr, "could not initialize alarm timer\n");
-        exit(1);
-    }
+    qemu_init_cpu_loop();
+    qemu_mutex_lock_iothread();
 
 #ifdef CONFIG_SPICE
     /* spice needs the timers to be initialized by this point */
@@ -3595,8 +3839,12 @@ int main(int argc, char **argv, char **envp)
         exit(1);
 
     /* If no default VGA is requested, the default is "none".  */
-    if (default_vga && cirrus_vga_available()) {
-        vga_model = "cirrus";
+    if (default_vga) {
+        if (cirrus_vga_available()) {
+            vga_model = "cirrus";
+        } else if (vga_available()) {
+            vga_model = "std";
+        }
     }
     select_vgahw(vga_model);
 
@@ -3613,8 +3861,13 @@ int main(int argc, char **argv, char **envp)
 
     qdev_machine_init();
 
-    machine->init(ram_size, boot_devices,
-                  kernel_filename, kernel_cmdline, initrd_filename, cpu_model);
+    QEMUMachineInitArgs args = { .ram_size = ram_size,
+                                 .boot_device = boot_devices,
+                                 .kernel_filename = kernel_filename,
+                                 .kernel_cmdline = kernel_cmdline,
+                                 .initrd_filename = initrd_filename,
+                                 .cpu_model = cpu_model };
+    machine->init(&args);
 
     cpu_synchronize_all_post_init();
 
@@ -3623,7 +3876,7 @@ int main(int argc, char **argv, char **envp)
     current_machine = machine;
 
     /* init USB devices */
-    if (usb_enabled) {
+    if (usb_enabled(false)) {
         if (foreach_device_config(DEV_USB, usb_parse) < 0)
             exit(1);
     }
@@ -3657,7 +3910,9 @@ int main(int argc, char **argv, char **envp)
         break;
 #if defined(CONFIG_CURSES)
     case DT_CURSES:
-        curses_display_init(ds, full_screen);
+        if (!is_daemonized()) {
+            curses_display_init(ds, full_screen);
+        }
         break;
 #endif
 #if defined(CONFIG_SDL)
@@ -3679,10 +3934,13 @@ int main(int argc, char **argv, char **envp)
 #ifdef CONFIG_VNC
     /* init remote displays */
     if (vnc_display) {
+        Error *local_err = NULL;
         vnc_display_init(ds);
-        if (vnc_display_open(ds, vnc_display) < 0) {
-            fprintf(stderr, "Failed to start VNC server on `%s'\n",
-                    vnc_display);
+        vnc_display_open(ds, vnc_display, &local_err);
+        if (local_err != NULL) {
+            fprintf(stderr, "Failed to start VNC server on `%s': %s\n",
+                    vnc_display, error_get_pretty(local_err));
+            error_free(local_err);
             exit(1);
         }
 
@@ -3698,16 +3956,6 @@ int main(int argc, char **argv, char **envp)
 #endif
 
     /* display setup */
-    dpy_resize(ds);
-    dcl = ds->listeners;
-    while (dcl != NULL) {
-        if (dcl->dpy_refresh != NULL) {
-            ds->gui_timer = qemu_new_timer_ms(rt_clock, gui_update, ds);
-            qemu_mod_timer(ds->gui_timer, qemu_get_clock_ms(rt_clock));
-            break;
-        }
-        dcl = dcl->next;
-    }
     text_consoles_set_display(ds);
 
     if (foreach_device_config(DEV_GDB, gdbserver_start) < 0) {
@@ -3734,16 +3982,12 @@ int main(int argc, char **argv, char **envp)
     }
 
     if (incoming) {
-        Error *errp = NULL;
-        int ret = qemu_start_incoming_migration(incoming, &errp);
-        if (ret < 0) {
-            if (error_is_set(&errp)) {
-                fprintf(stderr, "Migrate: %s\n", error_get_pretty(errp));
-                error_free(errp);
-            }
-            fprintf(stderr, "Migration failed. Exit code %s(%d), exiting.\n",
-                    incoming, ret);
-            exit(ret);
+        Error *local_err = NULL;
+        qemu_start_incoming_migration(incoming, &local_err);
+        if (local_err) {
+            fprintf(stderr, "-incoming %s: %s\n", incoming, error_get_pretty(local_err));
+            error_free(local_err);
+            exit(1);
         }
     } else if (autostart) {
         vm_start();
diff --git a/vmstate.h b/vmstate.h
index 5bd2b762a..623af0a29 100644
--- a/vmstate.h
+++ b/vmstate.h
@@ -139,6 +139,7 @@ extern const VMStateInfo vmstate_info_uint64;
 extern const VMStateInfo vmstate_info_timer;
 extern const VMStateInfo vmstate_info_buffer;
 extern const VMStateInfo vmstate_info_unused_buffer;
+extern const VMStateInfo vmstate_info_bitmap;
 
 #define type_check_array(t1,t2,n) ((t1(*)[n])0 - (t2*)0)
 #define type_check_pointer(t1,t2) ((t1**)0 - (t2*)0)
@@ -411,6 +412,18 @@ extern const VMStateInfo vmstate_info_unused_buffer;
     .flags        = VMS_BUFFER,                                      \
 }
 
+/* _field_size should be a int32_t field in the _state struct giving the
+ * size of the bitmap _field in bits.
+ */
+#define VMSTATE_BITMAP(_field, _state, _version, _field_size) {      \
+    .name         = (stringify(_field)),                             \
+    .version_id   = (_version),                                      \
+    .size_offset  = vmstate_offset_value(_state, _field_size, int32_t),\
+    .info         = &vmstate_info_bitmap,                            \
+    .flags        = VMS_VBUFFER|VMS_POINTER,                         \
+    .offset       = offsetof(_state, _field),                        \
+}
+
 /* _f : field name
    _f_n : num of elements field_name
    _n : num of elements
@@ -503,8 +516,11 @@ extern const VMStateInfo vmstate_info_unused_buffer;
 #define VMSTATE_TIMER_TEST(_f, _s, _test)                             \
     VMSTATE_POINTER_TEST(_f, _s, _test, vmstate_info_timer, QEMUTimer *)
 
+#define VMSTATE_TIMER_V(_f, _s, _v)                                   \
+    VMSTATE_POINTER(_f, _s, _v, vmstate_info_timer, QEMUTimer *)
+
 #define VMSTATE_TIMER(_f, _s)                                         \
-    VMSTATE_TIMER_TEST(_f, _s, NULL)
+    VMSTATE_TIMER_V(_f, _s, 0)
 
 #define VMSTATE_TIMER_ARRAY(_f, _s, _n)                              \
     VMSTATE_ARRAY_OF_POINTER(_f, _s, _n, 0, vmstate_info_timer, QEMUTimer *)
diff --git a/xen-all.c b/xen-all.c
index f76b051ee..046cc2ac3 100644
--- a/xen-all.c
+++ b/xen-all.c
@@ -14,6 +14,7 @@
 #include "hw/pc.h"
 #include "hw/xen_common.h"
 #include "hw/xen_backend.h"
+#include "qmp-commands.h"
 
 #include "range.h"
 #include "xen-mapcache.h"
@@ -36,6 +37,7 @@
 
 static MemoryRegion ram_memory, ram_640k, ram_lo, ram_hi;
 static MemoryRegion *framebuffer;
+static bool xen_in_migration;
 
 /* Compatibility with older version */
 #if __XEN_LATEST_INTERFACE_VERSION__ < 0x0003020a
@@ -66,10 +68,10 @@ static inline ioreq_t *xen_vcpu_ioreq(shared_iopage_t *shared_page, int vcpu)
 #define BUFFER_IO_MAX_DELAY  100
 
 typedef struct XenPhysmap {
-    target_phys_addr_t start_addr;
+    hwaddr start_addr;
     ram_addr_t size;
     char *name;
-    target_phys_addr_t phys_offset;
+    hwaddr phys_offset;
 
     QLIST_ENTRY(XenPhysmap) list;
 } XenPhysmap;
@@ -90,7 +92,7 @@ typedef struct XenIOState {
     struct xs_handle *xenstore;
     MemoryListener memory_listener;
     QLIST_HEAD(, XenPhysmap) physmap;
-    target_phys_addr_t free_phys_offset;
+    hwaddr free_phys_offset;
     const XenPhysmap *log_for_dirtybit;
 
     Notifier exit;
@@ -229,7 +231,7 @@ void xen_ram_alloc(ram_addr_t ram_addr, ram_addr_t size, MemoryRegion *mr)
 }
 
 static XenPhysmap *get_physmapping(XenIOState *state,
-                                   target_phys_addr_t start_addr, ram_addr_t size)
+                                   hwaddr start_addr, ram_addr_t size)
 {
     XenPhysmap *physmap = NULL;
 
@@ -243,10 +245,10 @@ static XenPhysmap *get_physmapping(XenIOState *state,
     return NULL;
 }
 
-static target_phys_addr_t xen_phys_offset_to_gaddr(target_phys_addr_t start_addr,
+static hwaddr xen_phys_offset_to_gaddr(hwaddr start_addr,
                                                    ram_addr_t size, void *opaque)
 {
-    target_phys_addr_t addr = start_addr & TARGET_PAGE_MASK;
+    hwaddr addr = start_addr & TARGET_PAGE_MASK;
     XenIOState *xen_io_state = opaque;
     XenPhysmap *physmap = NULL;
 
@@ -261,16 +263,16 @@ static target_phys_addr_t xen_phys_offset_to_gaddr(target_phys_addr_t start_addr
 
 #if CONFIG_XEN_CTRL_INTERFACE_VERSION >= 340
 static int xen_add_to_physmap(XenIOState *state,
-                              target_phys_addr_t start_addr,
+                              hwaddr start_addr,
                               ram_addr_t size,
                               MemoryRegion *mr,
-                              target_phys_addr_t offset_within_region)
+                              hwaddr offset_within_region)
 {
     unsigned long i = 0;
     int rc = 0;
     XenPhysmap *physmap = NULL;
-    target_phys_addr_t pfn, start_gpfn;
-    target_phys_addr_t phys_offset = memory_region_get_ram_addr(mr);
+    hwaddr pfn, start_gpfn;
+    hwaddr phys_offset = memory_region_get_ram_addr(mr);
     char path[80], value[17];
 
     if (get_physmapping(state, start_addr, size)) {
@@ -347,13 +349,13 @@ go_physmap:
 }
 
 static int xen_remove_from_physmap(XenIOState *state,
-                                   target_phys_addr_t start_addr,
+                                   hwaddr start_addr,
                                    ram_addr_t size)
 {
     unsigned long i = 0;
     int rc = 0;
     XenPhysmap *physmap = NULL;
-    target_phys_addr_t phys_offset = 0;
+    hwaddr phys_offset = 0;
 
     physmap = get_physmapping(state, start_addr, size);
     if (physmap == NULL) {
@@ -392,16 +394,16 @@ static int xen_remove_from_physmap(XenIOState *state,
 
 #else
 static int xen_add_to_physmap(XenIOState *state,
-                              target_phys_addr_t start_addr,
+                              hwaddr start_addr,
                               ram_addr_t size,
                               MemoryRegion *mr,
-                              target_phys_addr_t offset_within_region)
+                              hwaddr offset_within_region)
 {
     return -ENOSYS;
 }
 
 static int xen_remove_from_physmap(XenIOState *state,
-                                   target_phys_addr_t start_addr,
+                                   hwaddr start_addr,
                                    ram_addr_t size)
 {
     return -ENOSYS;
@@ -413,7 +415,7 @@ static void xen_set_memory(struct MemoryListener *listener,
                            bool add)
 {
     XenIOState *state = container_of(listener, XenIOState, memory_listener);
-    target_phys_addr_t start_addr = section->offset_within_address_space;
+    hwaddr start_addr = section->offset_within_address_space;
     ram_addr_t size = section->size;
     bool log_dirty = memory_region_is_logging(section->mr);
     hvmmem_type_t mem_type;
@@ -452,14 +454,6 @@ static void xen_set_memory(struct MemoryListener *listener,
     }
 }
 
-static void xen_begin(MemoryListener *listener)
-{
-}
-
-static void xen_commit(MemoryListener *listener)
-{
-}
-
 static void xen_region_add(MemoryListener *listener,
                            MemoryRegionSection *section)
 {
@@ -472,16 +466,11 @@ static void xen_region_del(MemoryListener *listener,
     xen_set_memory(listener, section, false);
 }
 
-static void xen_region_nop(MemoryListener *listener,
-                           MemoryRegionSection *section)
-{
-}
-
 static void xen_sync_dirty_bitmap(XenIOState *state,
-                                  target_phys_addr_t start_addr,
+                                  hwaddr start_addr,
                                   ram_addr_t size)
 {
-    target_phys_addr_t npages = size >> TARGET_PAGE_BITS;
+    hwaddr npages = size >> TARGET_PAGE_BITS;
     const int width = sizeof(unsigned long) * 8;
     unsigned long bitmap[(npages + width - 1) / width];
     int rc, i, j;
@@ -505,7 +494,8 @@ static void xen_sync_dirty_bitmap(XenIOState *state,
                                  bitmap);
     if (rc < 0) {
         if (rc != -ENODATA) {
-            fprintf(stderr, "xen: track_dirty_vram failed (0x" TARGET_FMT_plx
+            memory_region_set_dirty(framebuffer, 0, size);
+            DPRINTF("xen: track_dirty_vram failed (0x" TARGET_FMT_plx
                     ", 0x" TARGET_FMT_plx "): %s\n",
                     start_addr, start_addr + size, strerror(-rc));
         }
@@ -552,42 +542,36 @@ static void xen_log_sync(MemoryListener *listener, MemoryRegionSection *section)
 
 static void xen_log_global_start(MemoryListener *listener)
 {
+    if (xen_enabled()) {
+        xen_in_migration = true;
+    }
 }
 
 static void xen_log_global_stop(MemoryListener *listener)
 {
-}
-
-static void xen_eventfd_add(MemoryListener *listener,
-                            MemoryRegionSection *section,
-                            bool match_data, uint64_t data,
-                            EventNotifier *e)
-{
-}
-
-static void xen_eventfd_del(MemoryListener *listener,
-                            MemoryRegionSection *section,
-                            bool match_data, uint64_t data,
-                            EventNotifier *e)
-{
+    xen_in_migration = false;
 }
 
 static MemoryListener xen_memory_listener = {
-    .begin = xen_begin,
-    .commit = xen_commit,
     .region_add = xen_region_add,
     .region_del = xen_region_del,
-    .region_nop = xen_region_nop,
     .log_start = xen_log_start,
     .log_stop = xen_log_stop,
     .log_sync = xen_log_sync,
     .log_global_start = xen_log_global_start,
     .log_global_stop = xen_log_global_stop,
-    .eventfd_add = xen_eventfd_add,
-    .eventfd_del = xen_eventfd_del,
     .priority = 10,
 };
 
+void qmp_xen_set_global_dirty_log(bool enable, Error **errp)
+{
+    if (enable) {
+        memory_global_dirty_log_start();
+    } else {
+        memory_global_dirty_log_stop();
+    }
+}
+
 /* VCPU Operations, MMIO, IO ring ... */
 
 static void xen_reset_vcpu(void *opaque)
@@ -1076,7 +1060,6 @@ static void xen_read_physmap(XenIOState *state)
         QLIST_INSERT_HEAD(&state->physmap, physmap, list);
     }
     free(entries);
-    return;
 }
 
 int xen_hvm_init(void)
@@ -1158,7 +1141,7 @@ int xen_hvm_init(void)
 
     state->memory_listener = xen_memory_listener;
     QLIST_INIT(&state->physmap);
-    memory_listener_register(&state->memory_listener, get_system_memory());
+    memory_listener_register(&state->memory_listener, &address_space_memory);
     state->log_for_dirtybit = NULL;
 
     /* Initialize backend core & drivers */
@@ -1213,3 +1196,24 @@ void xen_shutdown_fatal_error(const char *fmt, ...)
     /* destroy the domain */
     qemu_system_shutdown_request();
 }
+
+void xen_modified_memory(ram_addr_t start, ram_addr_t length)
+{
+    if (unlikely(xen_in_migration)) {
+        int rc;
+        ram_addr_t start_pfn, nb_pages;
+
+        if (length == 0) {
+            length = TARGET_PAGE_SIZE;
+        }
+        start_pfn = start >> TARGET_PAGE_BITS;
+        nb_pages = ((start + length + TARGET_PAGE_SIZE - 1) >> TARGET_PAGE_BITS)
+            - start_pfn;
+        rc = xc_hvm_modified_memory(xen_xc, xen_domid, start_pfn, nb_pages);
+        if (rc) {
+            fprintf(stderr,
+                    "%s failed for "RAM_ADDR_FMT" ("RAM_ADDR_FMT"): %i, %s\n",
+                    __func__, start, nb_pages, rc, strerror(-rc));
+        }
+    }
+}
diff --git a/xen-mapcache.c b/xen-mapcache.c
index 9cd6db3d7..31c06dc95 100644
--- a/xen-mapcache.c
+++ b/xen-mapcache.c
@@ -53,18 +53,18 @@
 #define mapcache_unlock() ((void)0)
 
 typedef struct MapCacheEntry {
-    target_phys_addr_t paddr_index;
+    hwaddr paddr_index;
     uint8_t *vaddr_base;
     unsigned long *valid_mapping;
     uint8_t lock;
-    target_phys_addr_t size;
+    hwaddr size;
     struct MapCacheEntry *next;
 } MapCacheEntry;
 
 typedef struct MapCacheRev {
     uint8_t *vaddr_req;
-    target_phys_addr_t paddr_index;
-    target_phys_addr_t size;
+    hwaddr paddr_index;
+    hwaddr size;
     QTAILQ_ENTRY(MapCacheRev) next;
 } MapCacheRev;
 
@@ -74,7 +74,7 @@ typedef struct MapCache {
     QTAILQ_HEAD(map_cache_head, MapCacheRev) locked_entries;
 
     /* For most cases (>99.9%), the page address is the same. */
-    target_phys_addr_t last_address_index;
+    hwaddr last_address_index;
     uint8_t *last_address_vaddr;
     unsigned long max_mcache_size;
     unsigned int mcache_bucket_shift;
@@ -142,14 +142,14 @@ void xen_map_cache_init(phys_offset_to_gaddr_t f, void *opaque)
 }
 
 static void xen_remap_bucket(MapCacheEntry *entry,
-                             target_phys_addr_t size,
-                             target_phys_addr_t address_index)
+                             hwaddr size,
+                             hwaddr address_index)
 {
     uint8_t *vaddr_base;
     xen_pfn_t *pfns;
     int *err;
     unsigned int i;
-    target_phys_addr_t nb_pfn = size >> XC_PAGE_SHIFT;
+    hwaddr nb_pfn = size >> XC_PAGE_SHIFT;
 
     trace_xen_remap_bucket(address_index);
 
@@ -195,13 +195,13 @@ static void xen_remap_bucket(MapCacheEntry *entry,
     g_free(err);
 }
 
-uint8_t *xen_map_cache(target_phys_addr_t phys_addr, target_phys_addr_t size,
+uint8_t *xen_map_cache(hwaddr phys_addr, hwaddr size,
                        uint8_t lock)
 {
     MapCacheEntry *entry, *pentry = NULL;
-    target_phys_addr_t address_index;
-    target_phys_addr_t address_offset;
-    target_phys_addr_t __size = size;
+    hwaddr address_index;
+    hwaddr address_offset;
+    hwaddr __size = size;
     bool translated = false;
 
 tryagain:
@@ -278,8 +278,8 @@ ram_addr_t xen_ram_addr_from_mapcache(void *ptr)
 {
     MapCacheEntry *entry = NULL;
     MapCacheRev *reventry;
-    target_phys_addr_t paddr_index;
-    target_phys_addr_t size;
+    hwaddr paddr_index;
+    hwaddr size;
     int found = 0;
 
     QTAILQ_FOREACH(reventry, &mapcache->locked_entries, next) {
@@ -316,8 +316,8 @@ void xen_invalidate_map_cache_entry(uint8_t *buffer)
 {
     MapCacheEntry *entry = NULL, *pentry = NULL;
     MapCacheRev *reventry;
-    target_phys_addr_t paddr_index;
-    target_phys_addr_t size;
+    hwaddr paddr_index;
+    hwaddr size;
     int found = 0;
 
     QTAILQ_FOREACH(reventry, &mapcache->locked_entries, next) {
diff --git a/xen-mapcache.h b/xen-mapcache.h
index 70301a57e..c59804060 100644
--- a/xen-mapcache.h
+++ b/xen-mapcache.h
@@ -11,14 +11,14 @@
 
 #include <stdlib.h>
 
-typedef target_phys_addr_t (*phys_offset_to_gaddr_t)(target_phys_addr_t start_addr,
+typedef hwaddr (*phys_offset_to_gaddr_t)(hwaddr start_addr,
                                                      ram_addr_t size,
                                                      void *opaque);
 #ifdef CONFIG_XEN
 
 void xen_map_cache_init(phys_offset_to_gaddr_t f,
                         void *opaque);
-uint8_t *xen_map_cache(target_phys_addr_t phys_addr, target_phys_addr_t size,
+uint8_t *xen_map_cache(hwaddr phys_addr, hwaddr size,
                        uint8_t lock);
 ram_addr_t xen_ram_addr_from_mapcache(void *ptr);
 void xen_invalidate_map_cache_entry(uint8_t *buffer);
@@ -31,8 +31,8 @@ static inline void xen_map_cache_init(phys_offset_to_gaddr_t f,
 {
 }
 
-static inline uint8_t *xen_map_cache(target_phys_addr_t phys_addr,
-                                     target_phys_addr_t size,
+static inline uint8_t *xen_map_cache(hwaddr phys_addr,
+                                     hwaddr size,
                                      uint8_t lock)
 {
     abort();
diff --git a/xen-stub.c b/xen-stub.c
index 8ff2b79ac..921439263 100644
--- a/xen-stub.c
+++ b/xen-stub.c
@@ -11,6 +11,7 @@
 #include "qemu-common.h"
 #include "hw/xen.h"
 #include "memory.h"
+#include "qmp-commands.h"
 
 void xenstore_store_pv_console_info(int i, CharDriverState *chr)
 {
@@ -54,3 +55,11 @@ int xen_init(void)
 void xen_register_framebuffer(MemoryRegion *mr)
 {
 }
+
+void qmp_xen_set_global_dirty_log(bool enable, Error **errp)
+{
+}
+
+void xen_modified_memory(ram_addr_t start, ram_addr_t length)
+{
+}