|
@@ -1676,57 +1676,91 @@ MemTxResult address_space_write(AddressSpace *as, hwaddr addr,
|
|
|
* @result: location to write the success/failure of the transaction;
|
|
|
* if NULL, this information is discarded
|
|
|
*/
|
|
|
-uint32_t address_space_ldub(AddressSpace *as, hwaddr addr,
|
|
|
- MemTxAttrs attrs, MemTxResult *result);
|
|
|
-uint32_t address_space_lduw_le(AddressSpace *as, hwaddr addr,
|
|
|
- MemTxAttrs attrs, MemTxResult *result);
|
|
|
-uint32_t address_space_lduw_be(AddressSpace *as, hwaddr addr,
|
|
|
- MemTxAttrs attrs, MemTxResult *result);
|
|
|
-uint32_t address_space_ldl_le(AddressSpace *as, hwaddr addr,
|
|
|
- MemTxAttrs attrs, MemTxResult *result);
|
|
|
-uint32_t address_space_ldl_be(AddressSpace *as, hwaddr addr,
|
|
|
- MemTxAttrs attrs, MemTxResult *result);
|
|
|
-uint64_t address_space_ldq_le(AddressSpace *as, hwaddr addr,
|
|
|
- MemTxAttrs attrs, MemTxResult *result);
|
|
|
-uint64_t address_space_ldq_be(AddressSpace *as, hwaddr addr,
|
|
|
- MemTxAttrs attrs, MemTxResult *result);
|
|
|
-void address_space_stb(AddressSpace *as, hwaddr addr, uint32_t val,
|
|
|
- MemTxAttrs attrs, MemTxResult *result);
|
|
|
-void address_space_stw_le(AddressSpace *as, hwaddr addr, uint32_t val,
|
|
|
- MemTxAttrs attrs, MemTxResult *result);
|
|
|
-void address_space_stw_be(AddressSpace *as, hwaddr addr, uint32_t val,
|
|
|
- MemTxAttrs attrs, MemTxResult *result);
|
|
|
-void address_space_stl_le(AddressSpace *as, hwaddr addr, uint32_t val,
|
|
|
- MemTxAttrs attrs, MemTxResult *result);
|
|
|
-void address_space_stl_be(AddressSpace *as, hwaddr addr, uint32_t val,
|
|
|
- MemTxAttrs attrs, MemTxResult *result);
|
|
|
-void address_space_stq_le(AddressSpace *as, hwaddr addr, uint64_t val,
|
|
|
- MemTxAttrs attrs, MemTxResult *result);
|
|
|
-void address_space_stq_be(AddressSpace *as, hwaddr addr, uint64_t val,
|
|
|
- MemTxAttrs attrs, MemTxResult *result);
|
|
|
-
|
|
|
-uint32_t ldub_phys(AddressSpace *as, hwaddr addr);
|
|
|
-uint32_t lduw_le_phys(AddressSpace *as, hwaddr addr);
|
|
|
-uint32_t lduw_be_phys(AddressSpace *as, hwaddr addr);
|
|
|
-uint32_t ldl_le_phys(AddressSpace *as, hwaddr addr);
|
|
|
-uint32_t ldl_be_phys(AddressSpace *as, hwaddr addr);
|
|
|
-uint64_t ldq_le_phys(AddressSpace *as, hwaddr addr);
|
|
|
-uint64_t ldq_be_phys(AddressSpace *as, hwaddr addr);
|
|
|
-void stb_phys(AddressSpace *as, hwaddr addr, uint32_t val);
|
|
|
-void stw_le_phys(AddressSpace *as, hwaddr addr, uint32_t val);
|
|
|
-void stw_be_phys(AddressSpace *as, hwaddr addr, uint32_t val);
|
|
|
-void stl_le_phys(AddressSpace *as, hwaddr addr, uint32_t val);
|
|
|
-void stl_be_phys(AddressSpace *as, hwaddr addr, uint32_t val);
|
|
|
-void stq_le_phys(AddressSpace *as, hwaddr addr, uint64_t val);
|
|
|
-void stq_be_phys(AddressSpace *as, hwaddr addr, uint64_t val);
|
|
|
+
|
|
|
+#define SUFFIX
|
|
|
+#define ARG1 as
|
|
|
+#define ARG1_DECL AddressSpace *as
|
|
|
+#include "exec/memory_ldst.inc.h"
|
|
|
+
|
|
|
+#define SUFFIX
|
|
|
+#define ARG1 as
|
|
|
+#define ARG1_DECL AddressSpace *as
|
|
|
+#include "exec/memory_ldst_phys.inc.h"
|
|
|
|
|
|
struct MemoryRegionCache {
|
|
|
+ void *ptr;
|
|
|
hwaddr xlat;
|
|
|
hwaddr len;
|
|
|
- AddressSpace *as;
|
|
|
+ FlatView *fv;
|
|
|
+ MemoryRegionSection mrs;
|
|
|
+ bool is_write;
|
|
|
};
|
|
|
|
|
|
-#define MEMORY_REGION_CACHE_INVALID ((MemoryRegionCache) { .as = NULL })
|
|
|
+#define MEMORY_REGION_CACHE_INVALID ((MemoryRegionCache) { .mrs.mr = NULL })
|
|
|
+
|
|
|
+
|
|
|
+/* address_space_ld*_cached: load from a cached #MemoryRegion
|
|
|
+ * address_space_st*_cached: store into a cached #MemoryRegion
|
|
|
+ *
|
|
|
+ * These functions perform a load or store of the byte, word,
|
|
|
+ * longword or quad to the specified address. The address is
|
|
|
+ * a physical address in the AddressSpace, but it must lie within
|
|
|
+ * a #MemoryRegion that was mapped with address_space_cache_init.
|
|
|
+ *
|
|
|
+ * The _le suffixed functions treat the data as little endian;
|
|
|
+ * _be indicates big endian; no suffix indicates "same endianness
|
|
|
+ * as guest CPU".
|
|
|
+ *
|
|
|
+ * The "guest CPU endianness" accessors are deprecated for use outside
|
|
|
+ * target-* code; devices should be CPU-agnostic and use either the LE
|
|
|
+ * or the BE accessors.
|
|
|
+ *
|
|
|
+ * @cache: previously initialized #MemoryRegionCache to be accessed
|
|
|
+ * @addr: address within the address space
|
|
|
+ * @val: data value, for stores
|
|
|
+ * @attrs: memory transaction attributes
|
|
|
+ * @result: location to write the success/failure of the transaction;
|
|
|
+ * if NULL, this information is discarded
|
|
|
+ */
|
|
|
+
|
|
|
+#define SUFFIX _cached_slow
|
|
|
+#define ARG1 cache
|
|
|
+#define ARG1_DECL MemoryRegionCache *cache
|
|
|
+#include "exec/memory_ldst.inc.h"
|
|
|
+
|
|
|
+/* Inline fast path for direct RAM access. */
|
|
|
+static inline uint8_t address_space_ldub_cached(MemoryRegionCache *cache,
|
|
|
+ hwaddr addr, MemTxAttrs attrs, MemTxResult *result)
|
|
|
+{
|
|
|
+ assert(addr < cache->len);
|
|
|
+ if (likely(cache->ptr)) {
|
|
|
+ return ldub_p(cache->ptr + addr);
|
|
|
+ } else {
|
|
|
+ return address_space_ldub_cached_slow(cache, addr, attrs, result);
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+static inline void address_space_stb_cached(MemoryRegionCache *cache,
|
|
|
+ hwaddr addr, uint32_t val, MemTxAttrs attrs, MemTxResult *result)
|
|
|
+{
|
|
|
+ assert(addr < cache->len);
|
|
|
+ if (likely(cache->ptr)) {
|
|
|
+ stb_p(cache->ptr + addr, val);
|
|
|
+ } else {
|
|
|
+ address_space_stb_cached_slow(cache, addr, val, attrs, result);
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+#define ENDIANNESS _le
|
|
|
+#include "exec/memory_ldst_cached.inc.h"
|
|
|
+
|
|
|
+#define ENDIANNESS _be
|
|
|
+#include "exec/memory_ldst_cached.inc.h"
|
|
|
+
|
|
|
+#define SUFFIX _cached
|
|
|
+#define ARG1 cache
|
|
|
+#define ARG1_DECL MemoryRegionCache *cache
|
|
|
+#include "exec/memory_ldst_phys.inc.h"
|
|
|
|
|
|
/* address_space_cache_init: prepare for repeated access to a physical
|
|
|
* memory region
|
|
@@ -1772,72 +1806,6 @@ void address_space_cache_invalidate(MemoryRegionCache *cache,
|
|
|
*/
|
|
|
void address_space_cache_destroy(MemoryRegionCache *cache);
|
|
|
|
|
|
-/* address_space_ld*_cached: load from a cached #MemoryRegion
|
|
|
- * address_space_st*_cached: store into a cached #MemoryRegion
|
|
|
- *
|
|
|
- * These functions perform a load or store of the byte, word,
|
|
|
- * longword or quad to the specified address. The address is
|
|
|
- * a physical address in the AddressSpace, but it must lie within
|
|
|
- * a #MemoryRegion that was mapped with address_space_cache_init.
|
|
|
- *
|
|
|
- * The _le suffixed functions treat the data as little endian;
|
|
|
- * _be indicates big endian; no suffix indicates "same endianness
|
|
|
- * as guest CPU".
|
|
|
- *
|
|
|
- * The "guest CPU endianness" accessors are deprecated for use outside
|
|
|
- * target-* code; devices should be CPU-agnostic and use either the LE
|
|
|
- * or the BE accessors.
|
|
|
- *
|
|
|
- * @cache: previously initialized #MemoryRegionCache to be accessed
|
|
|
- * @addr: address within the address space
|
|
|
- * @val: data value, for stores
|
|
|
- * @attrs: memory transaction attributes
|
|
|
- * @result: location to write the success/failure of the transaction;
|
|
|
- * if NULL, this information is discarded
|
|
|
- */
|
|
|
-uint32_t address_space_ldub_cached(MemoryRegionCache *cache, hwaddr addr,
|
|
|
- MemTxAttrs attrs, MemTxResult *result);
|
|
|
-uint32_t address_space_lduw_le_cached(MemoryRegionCache *cache, hwaddr addr,
|
|
|
- MemTxAttrs attrs, MemTxResult *result);
|
|
|
-uint32_t address_space_lduw_be_cached(MemoryRegionCache *cache, hwaddr addr,
|
|
|
- MemTxAttrs attrs, MemTxResult *result);
|
|
|
-uint32_t address_space_ldl_le_cached(MemoryRegionCache *cache, hwaddr addr,
|
|
|
- MemTxAttrs attrs, MemTxResult *result);
|
|
|
-uint32_t address_space_ldl_be_cached(MemoryRegionCache *cache, hwaddr addr,
|
|
|
- MemTxAttrs attrs, MemTxResult *result);
|
|
|
-uint64_t address_space_ldq_le_cached(MemoryRegionCache *cache, hwaddr addr,
|
|
|
- MemTxAttrs attrs, MemTxResult *result);
|
|
|
-uint64_t address_space_ldq_be_cached(MemoryRegionCache *cache, hwaddr addr,
|
|
|
- MemTxAttrs attrs, MemTxResult *result);
|
|
|
-void address_space_stb_cached(MemoryRegionCache *cache, hwaddr addr, uint32_t val,
|
|
|
- MemTxAttrs attrs, MemTxResult *result);
|
|
|
-void address_space_stw_le_cached(MemoryRegionCache *cache, hwaddr addr, uint32_t val,
|
|
|
- MemTxAttrs attrs, MemTxResult *result);
|
|
|
-void address_space_stw_be_cached(MemoryRegionCache *cache, hwaddr addr, uint32_t val,
|
|
|
- MemTxAttrs attrs, MemTxResult *result);
|
|
|
-void address_space_stl_le_cached(MemoryRegionCache *cache, hwaddr addr, uint32_t val,
|
|
|
- MemTxAttrs attrs, MemTxResult *result);
|
|
|
-void address_space_stl_be_cached(MemoryRegionCache *cache, hwaddr addr, uint32_t val,
|
|
|
- MemTxAttrs attrs, MemTxResult *result);
|
|
|
-void address_space_stq_le_cached(MemoryRegionCache *cache, hwaddr addr, uint64_t val,
|
|
|
- MemTxAttrs attrs, MemTxResult *result);
|
|
|
-void address_space_stq_be_cached(MemoryRegionCache *cache, hwaddr addr, uint64_t val,
|
|
|
- MemTxAttrs attrs, MemTxResult *result);
|
|
|
-
|
|
|
-uint32_t ldub_phys_cached(MemoryRegionCache *cache, hwaddr addr);
|
|
|
-uint32_t lduw_le_phys_cached(MemoryRegionCache *cache, hwaddr addr);
|
|
|
-uint32_t lduw_be_phys_cached(MemoryRegionCache *cache, hwaddr addr);
|
|
|
-uint32_t ldl_le_phys_cached(MemoryRegionCache *cache, hwaddr addr);
|
|
|
-uint32_t ldl_be_phys_cached(MemoryRegionCache *cache, hwaddr addr);
|
|
|
-uint64_t ldq_le_phys_cached(MemoryRegionCache *cache, hwaddr addr);
|
|
|
-uint64_t ldq_be_phys_cached(MemoryRegionCache *cache, hwaddr addr);
|
|
|
-void stb_phys_cached(MemoryRegionCache *cache, hwaddr addr, uint32_t val);
|
|
|
-void stw_le_phys_cached(MemoryRegionCache *cache, hwaddr addr, uint32_t val);
|
|
|
-void stw_be_phys_cached(MemoryRegionCache *cache, hwaddr addr, uint32_t val);
|
|
|
-void stl_le_phys_cached(MemoryRegionCache *cache, hwaddr addr, uint32_t val);
|
|
|
-void stl_be_phys_cached(MemoryRegionCache *cache, hwaddr addr, uint32_t val);
|
|
|
-void stq_le_phys_cached(MemoryRegionCache *cache, hwaddr addr, uint64_t val);
|
|
|
-void stq_be_phys_cached(MemoryRegionCache *cache, hwaddr addr, uint64_t val);
|
|
|
/* address_space_get_iotlb_entry: translate an address into an IOTLB
|
|
|
* entry. Should be called from an RCU critical section.
|
|
|
*/
|
|
@@ -1925,6 +1893,13 @@ MemTxResult flatview_read_continue(FlatView *fv, hwaddr addr,
|
|
|
MemoryRegion *mr);
|
|
|
void *qemu_map_ram_ptr(RAMBlock *ram_block, ram_addr_t addr);
|
|
|
|
|
|
+/* Internal functions, part of the implementation of address_space_read_cached
|
|
|
+ * and address_space_write_cached. */
|
|
|
+void address_space_read_cached_slow(MemoryRegionCache *cache,
|
|
|
+ hwaddr addr, void *buf, int len);
|
|
|
+void address_space_write_cached_slow(MemoryRegionCache *cache,
|
|
|
+ hwaddr addr, const void *buf, int len);
|
|
|
+
|
|
|
static inline bool memory_access_is_direct(MemoryRegion *mr, bool is_write)
|
|
|
{
|
|
|
if (is_write) {
|
|
@@ -1993,7 +1968,11 @@ address_space_read_cached(MemoryRegionCache *cache, hwaddr addr,
|
|
|
void *buf, int len)
|
|
|
{
|
|
|
assert(addr < cache->len && len <= cache->len - addr);
|
|
|
- address_space_read(cache->as, cache->xlat + addr, MEMTXATTRS_UNSPECIFIED, buf, len);
|
|
|
+ if (likely(cache->ptr)) {
|
|
|
+ memcpy(buf, cache->ptr + addr, len);
|
|
|
+ } else {
|
|
|
+ address_space_read_cached_slow(cache, addr, buf, len);
|
|
|
+ }
|
|
|
}
|
|
|
|
|
|
/**
|
|
@@ -2009,7 +1988,11 @@ address_space_write_cached(MemoryRegionCache *cache, hwaddr addr,
|
|
|
void *buf, int len)
|
|
|
{
|
|
|
assert(addr < cache->len && len <= cache->len - addr);
|
|
|
- address_space_write(cache->as, cache->xlat + addr, MEMTXATTRS_UNSPECIFIED, buf, len);
|
|
|
+ if (likely(cache->ptr)) {
|
|
|
+ memcpy(cache->ptr + addr, buf, len);
|
|
|
+ } else {
|
|
|
+ address_space_write_cached_slow(cache, addr, buf, len);
|
|
|
+ }
|
|
|
}
|
|
|
|
|
|
#endif
|