xen-mapcache.c 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772
  1. /*
  2. * Copyright (C) 2011 Citrix Ltd.
  3. *
  4. * This work is licensed under the terms of the GNU GPL, version 2. See
  5. * the COPYING file in the top-level directory.
  6. *
  7. * Contributions after 2012-01-13 are licensed under the terms of the
  8. * GNU GPL, version 2 or (at your option) any later version.
  9. */
  10. #include "qemu/osdep.h"
  11. #include "qemu/units.h"
  12. #include "qemu/error-report.h"
  13. #include <sys/resource.h>
  14. #include "hw/xen/xen-hvm-common.h"
  15. #include "hw/xen/xen_native.h"
  16. #include "qemu/bitmap.h"
  17. #include "system/runstate.h"
  18. #include "system/xen-mapcache.h"
  19. #include "trace.h"
  20. #include <xenevtchn.h>
  21. #include <xengnttab.h>
  22. #if HOST_LONG_BITS == 32
  23. # define MCACHE_MAX_SIZE (1UL<<31) /* 2GB Cap */
  24. #else
  25. # define MCACHE_MAX_SIZE (1UL<<35) /* 32GB Cap */
  26. #endif
  27. /* This is the size of the virtual address space reserve to QEMU that will not
  28. * be use by MapCache.
  29. * From empirical tests I observed that qemu use 75MB more than the
  30. * max_mcache_size.
  31. */
  32. #define NON_MCACHE_MEMORY_SIZE (80 * MiB)
  33. typedef struct MapCacheEntry {
  34. hwaddr paddr_index;
  35. uint8_t *vaddr_base;
  36. unsigned long *valid_mapping;
  37. uint32_t lock;
  38. #define XEN_MAPCACHE_ENTRY_DUMMY (1 << 0)
  39. #define XEN_MAPCACHE_ENTRY_GRANT (1 << 1)
  40. uint8_t flags;
  41. hwaddr size;
  42. struct MapCacheEntry *next;
  43. } MapCacheEntry;
  44. typedef struct MapCacheRev {
  45. uint8_t *vaddr_req;
  46. hwaddr paddr_index;
  47. hwaddr size;
  48. QTAILQ_ENTRY(MapCacheRev) next;
  49. bool dma;
  50. } MapCacheRev;
  51. typedef struct MapCache {
  52. MapCacheEntry *entry;
  53. unsigned long nr_buckets;
  54. QTAILQ_HEAD(, MapCacheRev) locked_entries;
  55. /* For most cases (>99.9%), the page address is the same. */
  56. MapCacheEntry *last_entry;
  57. unsigned long max_mcache_size;
  58. unsigned int bucket_shift;
  59. unsigned long bucket_size;
  60. phys_offset_to_gaddr_t phys_offset_to_gaddr;
  61. QemuMutex lock;
  62. void *opaque;
  63. } MapCache;
  64. static MapCache *mapcache;
  65. static MapCache *mapcache_grants;
  66. static xengnttab_handle *xen_region_gnttabdev;
  67. static inline void mapcache_lock(MapCache *mc)
  68. {
  69. qemu_mutex_lock(&mc->lock);
  70. }
  71. static inline void mapcache_unlock(MapCache *mc)
  72. {
  73. qemu_mutex_unlock(&mc->lock);
  74. }
  75. static inline int test_bits(int nr, int size, const unsigned long *addr)
  76. {
  77. unsigned long res = find_next_zero_bit(addr, size + nr, nr);
  78. if (res >= nr + size)
  79. return 1;
  80. else
  81. return 0;
  82. }
  83. static MapCache *xen_map_cache_init_single(phys_offset_to_gaddr_t f,
  84. void *opaque,
  85. unsigned int bucket_shift,
  86. unsigned long max_size)
  87. {
  88. unsigned long size;
  89. MapCache *mc;
  90. assert(bucket_shift >= XC_PAGE_SHIFT);
  91. mc = g_new0(MapCache, 1);
  92. mc->phys_offset_to_gaddr = f;
  93. mc->opaque = opaque;
  94. qemu_mutex_init(&mc->lock);
  95. QTAILQ_INIT(&mc->locked_entries);
  96. mc->bucket_shift = bucket_shift;
  97. mc->bucket_size = 1UL << bucket_shift;
  98. mc->max_mcache_size = max_size;
  99. mc->nr_buckets =
  100. (((mc->max_mcache_size >> XC_PAGE_SHIFT) +
  101. (1UL << (bucket_shift - XC_PAGE_SHIFT)) - 1) >>
  102. (bucket_shift - XC_PAGE_SHIFT));
  103. size = mc->nr_buckets * sizeof(MapCacheEntry);
  104. size = (size + XC_PAGE_SIZE - 1) & ~(XC_PAGE_SIZE - 1);
  105. trace_xen_map_cache_init(mc->nr_buckets, size);
  106. mc->entry = g_malloc0(size);
  107. return mc;
  108. }
  109. void xen_map_cache_init(phys_offset_to_gaddr_t f, void *opaque)
  110. {
  111. struct rlimit rlimit_as;
  112. unsigned long max_mcache_size;
  113. unsigned int bucket_shift;
  114. xen_region_gnttabdev = xengnttab_open(NULL, 0);
  115. if (xen_region_gnttabdev == NULL) {
  116. error_report("mapcache: Failed to open gnttab device");
  117. exit(EXIT_FAILURE);
  118. }
  119. if (HOST_LONG_BITS == 32) {
  120. bucket_shift = 16;
  121. } else {
  122. bucket_shift = 20;
  123. }
  124. if (geteuid() == 0) {
  125. rlimit_as.rlim_cur = RLIM_INFINITY;
  126. rlimit_as.rlim_max = RLIM_INFINITY;
  127. max_mcache_size = MCACHE_MAX_SIZE;
  128. } else {
  129. getrlimit(RLIMIT_AS, &rlimit_as);
  130. rlimit_as.rlim_cur = rlimit_as.rlim_max;
  131. if (rlimit_as.rlim_max != RLIM_INFINITY) {
  132. warn_report("QEMU's maximum size of virtual"
  133. " memory is not infinity");
  134. }
  135. if (rlimit_as.rlim_max < MCACHE_MAX_SIZE + NON_MCACHE_MEMORY_SIZE) {
  136. max_mcache_size = rlimit_as.rlim_max - NON_MCACHE_MEMORY_SIZE;
  137. } else {
  138. max_mcache_size = MCACHE_MAX_SIZE;
  139. }
  140. }
  141. mapcache = xen_map_cache_init_single(f, opaque,
  142. bucket_shift,
  143. max_mcache_size);
  144. /*
  145. * Grant mappings must use XC_PAGE_SIZE granularity since we can't
  146. * map anything beyond the number of pages granted to us.
  147. */
  148. mapcache_grants = xen_map_cache_init_single(f, opaque,
  149. XC_PAGE_SHIFT,
  150. max_mcache_size);
  151. setrlimit(RLIMIT_AS, &rlimit_as);
  152. }
  153. static void xen_remap_bucket(MapCache *mc,
  154. MapCacheEntry *entry,
  155. void *vaddr,
  156. hwaddr size,
  157. hwaddr address_index,
  158. bool dummy,
  159. bool grant,
  160. bool is_write,
  161. ram_addr_t ram_offset)
  162. {
  163. uint8_t *vaddr_base;
  164. g_autofree uint32_t *refs = NULL;
  165. g_autofree xen_pfn_t *pfns = NULL;
  166. g_autofree int *err;
  167. unsigned int i;
  168. hwaddr nb_pfn = size >> XC_PAGE_SHIFT;
  169. trace_xen_remap_bucket(address_index);
  170. if (grant) {
  171. refs = g_new0(uint32_t, nb_pfn);
  172. } else {
  173. pfns = g_new0(xen_pfn_t, nb_pfn);
  174. }
  175. err = g_new0(int, nb_pfn);
  176. if (entry->vaddr_base != NULL) {
  177. if (!(entry->flags & XEN_MAPCACHE_ENTRY_DUMMY)) {
  178. ram_block_notify_remove(entry->vaddr_base, entry->size,
  179. entry->size);
  180. }
  181. /*
  182. * If an entry is being replaced by another mapping and we're using
  183. * MAP_FIXED flag for it - there is possibility of a race for vaddr
  184. * address with another thread doing an mmap call itself
  185. * (see man 2 mmap). To avoid that we skip explicit unmapping here
  186. * and allow the kernel to destroy the previous mappings by replacing
  187. * them in mmap call later.
  188. *
  189. * Non-identical replacements are not allowed therefore.
  190. */
  191. assert(!vaddr || (entry->vaddr_base == vaddr && entry->size == size));
  192. if (!vaddr && munmap(entry->vaddr_base, entry->size) != 0) {
  193. perror("unmap fails");
  194. exit(-1);
  195. }
  196. }
  197. g_free(entry->valid_mapping);
  198. entry->valid_mapping = NULL;
  199. if (grant) {
  200. hwaddr grant_base = address_index - (ram_offset >> XC_PAGE_SHIFT);
  201. for (i = 0; i < nb_pfn; i++) {
  202. refs[i] = grant_base + i;
  203. }
  204. } else {
  205. for (i = 0; i < nb_pfn; i++) {
  206. pfns[i] = (address_index << (mc->bucket_shift - XC_PAGE_SHIFT)) + i;
  207. }
  208. }
  209. entry->flags &= ~XEN_MAPCACHE_ENTRY_GRANT;
  210. if (!dummy) {
  211. if (grant) {
  212. int prot = PROT_READ;
  213. if (is_write) {
  214. prot |= PROT_WRITE;
  215. }
  216. entry->flags |= XEN_MAPCACHE_ENTRY_GRANT;
  217. assert(vaddr == NULL);
  218. vaddr_base = xengnttab_map_domain_grant_refs(xen_region_gnttabdev,
  219. nb_pfn,
  220. xen_domid, refs,
  221. prot);
  222. } else {
  223. /*
  224. * If the caller has requested the mapping at a specific address use
  225. * MAP_FIXED to make sure it's honored.
  226. *
  227. * We don't yet support upgrading mappings from RO to RW, to handle
  228. * models using ordinary address_space_rw(), foreign mappings ignore
  229. * is_write and are always mapped RW.
  230. */
  231. vaddr_base = xenforeignmemory_map2(xen_fmem, xen_domid, vaddr,
  232. PROT_READ | PROT_WRITE,
  233. vaddr ? MAP_FIXED : 0,
  234. nb_pfn, pfns, err);
  235. }
  236. if (vaddr_base == NULL) {
  237. perror(grant ? "xengnttab_map_domain_grant_refs"
  238. : "xenforeignmemory_map2");
  239. exit(-1);
  240. }
  241. } else {
  242. /*
  243. * We create dummy mappings where we are unable to create a foreign
  244. * mapping immediately due to certain circumstances (i.e. on resume now)
  245. */
  246. vaddr_base = mmap(vaddr, size, PROT_READ | PROT_WRITE,
  247. MAP_ANON | MAP_SHARED | (vaddr ? MAP_FIXED : 0),
  248. -1, 0);
  249. if (vaddr_base == MAP_FAILED) {
  250. perror("mmap");
  251. exit(-1);
  252. }
  253. }
  254. if (!(entry->flags & XEN_MAPCACHE_ENTRY_DUMMY)) {
  255. ram_block_notify_add(vaddr_base, size, size);
  256. }
  257. entry->vaddr_base = vaddr_base;
  258. entry->paddr_index = address_index;
  259. entry->size = size;
  260. entry->valid_mapping = g_new0(unsigned long,
  261. BITS_TO_LONGS(size >> XC_PAGE_SHIFT));
  262. if (dummy) {
  263. entry->flags |= XEN_MAPCACHE_ENTRY_DUMMY;
  264. } else {
  265. entry->flags &= ~(XEN_MAPCACHE_ENTRY_DUMMY);
  266. }
  267. bitmap_zero(entry->valid_mapping, nb_pfn);
  268. for (i = 0; i < nb_pfn; i++) {
  269. if (!err[i]) {
  270. bitmap_set(entry->valid_mapping, i, 1);
  271. }
  272. }
  273. }
  274. static uint8_t *xen_map_cache_unlocked(MapCache *mc,
  275. hwaddr phys_addr, hwaddr size,
  276. ram_addr_t ram_offset,
  277. uint8_t lock, bool dma,
  278. bool grant, bool is_write)
  279. {
  280. MapCacheEntry *entry, *pentry = NULL,
  281. *free_entry = NULL, *free_pentry = NULL;
  282. hwaddr address_index;
  283. hwaddr address_offset;
  284. hwaddr cache_size = size;
  285. hwaddr test_bit_size;
  286. bool translated G_GNUC_UNUSED = false;
  287. bool dummy = false;
  288. tryagain:
  289. address_index = phys_addr >> mc->bucket_shift;
  290. address_offset = phys_addr & (mc->bucket_size - 1);
  291. trace_xen_map_cache(phys_addr);
  292. /* test_bit_size is always a multiple of XC_PAGE_SIZE */
  293. if (size) {
  294. test_bit_size = size + (phys_addr & (XC_PAGE_SIZE - 1));
  295. if (test_bit_size % XC_PAGE_SIZE) {
  296. test_bit_size += XC_PAGE_SIZE - (test_bit_size % XC_PAGE_SIZE);
  297. }
  298. } else {
  299. test_bit_size = XC_PAGE_SIZE;
  300. }
  301. if (mc->last_entry != NULL &&
  302. mc->last_entry->paddr_index == address_index &&
  303. !lock && !size &&
  304. test_bits(address_offset >> XC_PAGE_SHIFT,
  305. test_bit_size >> XC_PAGE_SHIFT,
  306. mc->last_entry->valid_mapping)) {
  307. trace_xen_map_cache_return(
  308. mc->last_entry->vaddr_base + address_offset
  309. );
  310. return mc->last_entry->vaddr_base + address_offset;
  311. }
  312. /* size is always a multiple of mc->bucket_size */
  313. if (size) {
  314. cache_size = size + address_offset;
  315. if (cache_size % mc->bucket_size) {
  316. cache_size += mc->bucket_size - (cache_size % mc->bucket_size);
  317. }
  318. } else {
  319. cache_size = mc->bucket_size;
  320. }
  321. entry = &mc->entry[address_index % mc->nr_buckets];
  322. while (entry && (lock || entry->lock) && entry->vaddr_base &&
  323. (entry->paddr_index != address_index || entry->size != cache_size ||
  324. !test_bits(address_offset >> XC_PAGE_SHIFT,
  325. test_bit_size >> XC_PAGE_SHIFT,
  326. entry->valid_mapping))) {
  327. if (!free_entry && !entry->lock) {
  328. free_entry = entry;
  329. free_pentry = pentry;
  330. }
  331. pentry = entry;
  332. entry = entry->next;
  333. }
  334. if (!entry && free_entry) {
  335. entry = free_entry;
  336. pentry = free_pentry;
  337. }
  338. if (!entry) {
  339. entry = g_new0(MapCacheEntry, 1);
  340. pentry->next = entry;
  341. xen_remap_bucket(mc, entry, NULL, cache_size, address_index, dummy,
  342. grant, is_write, ram_offset);
  343. } else if (!entry->lock) {
  344. if (!entry->vaddr_base || entry->paddr_index != address_index ||
  345. entry->size != cache_size ||
  346. !test_bits(address_offset >> XC_PAGE_SHIFT,
  347. test_bit_size >> XC_PAGE_SHIFT,
  348. entry->valid_mapping)) {
  349. xen_remap_bucket(mc, entry, NULL, cache_size, address_index, dummy,
  350. grant, is_write, ram_offset);
  351. }
  352. }
  353. if(!test_bits(address_offset >> XC_PAGE_SHIFT,
  354. test_bit_size >> XC_PAGE_SHIFT,
  355. entry->valid_mapping)) {
  356. mc->last_entry = NULL;
  357. #ifdef XEN_COMPAT_PHYSMAP
  358. if (!translated && mc->phys_offset_to_gaddr) {
  359. phys_addr = mc->phys_offset_to_gaddr(phys_addr, size);
  360. translated = true;
  361. goto tryagain;
  362. }
  363. #endif
  364. if (!dummy && runstate_check(RUN_STATE_INMIGRATE)) {
  365. dummy = true;
  366. goto tryagain;
  367. }
  368. trace_xen_map_cache_return(NULL);
  369. return NULL;
  370. }
  371. mc->last_entry = entry;
  372. if (lock) {
  373. MapCacheRev *reventry = g_new0(MapCacheRev, 1);
  374. entry->lock++;
  375. if (entry->lock == 0) {
  376. error_report("mapcache entry lock overflow: "HWADDR_FMT_plx" -> %p",
  377. entry->paddr_index, entry->vaddr_base);
  378. abort();
  379. }
  380. reventry->dma = dma;
  381. reventry->vaddr_req = mc->last_entry->vaddr_base + address_offset;
  382. reventry->paddr_index = mc->last_entry->paddr_index;
  383. reventry->size = entry->size;
  384. QTAILQ_INSERT_HEAD(&mc->locked_entries, reventry, next);
  385. }
  386. trace_xen_map_cache_return(
  387. mc->last_entry->vaddr_base + address_offset
  388. );
  389. return mc->last_entry->vaddr_base + address_offset;
  390. }
  391. uint8_t *xen_map_cache(MemoryRegion *mr,
  392. hwaddr phys_addr, hwaddr size,
  393. ram_addr_t ram_addr_offset,
  394. uint8_t lock, bool dma,
  395. bool is_write)
  396. {
  397. bool grant = xen_mr_is_grants(mr);
  398. MapCache *mc = grant ? mapcache_grants : mapcache;
  399. uint8_t *p;
  400. if (grant && !lock) {
  401. /*
  402. * Grants are only supported via address_space_map(). Anything
  403. * else is considered a user/guest error.
  404. *
  405. * QEMU generally doesn't expect these mappings to ever fail, so
  406. * if this happens we report an error message and abort().
  407. */
  408. error_report("Tried to access a grant reference without mapping it.");
  409. abort();
  410. }
  411. mapcache_lock(mc);
  412. p = xen_map_cache_unlocked(mc, phys_addr, size, ram_addr_offset,
  413. lock, dma, grant, is_write);
  414. mapcache_unlock(mc);
  415. return p;
  416. }
  417. static ram_addr_t xen_ram_addr_from_mapcache_single(MapCache *mc, void *ptr)
  418. {
  419. MapCacheEntry *entry = NULL;
  420. MapCacheRev *reventry;
  421. hwaddr paddr_index;
  422. hwaddr size;
  423. ram_addr_t raddr;
  424. int found = 0;
  425. mapcache_lock(mc);
  426. QTAILQ_FOREACH(reventry, &mc->locked_entries, next) {
  427. if (reventry->vaddr_req == ptr) {
  428. paddr_index = reventry->paddr_index;
  429. size = reventry->size;
  430. found = 1;
  431. break;
  432. }
  433. }
  434. if (!found) {
  435. trace_xen_ram_addr_from_mapcache_not_found(ptr);
  436. mapcache_unlock(mc);
  437. return RAM_ADDR_INVALID;
  438. }
  439. entry = &mc->entry[paddr_index % mc->nr_buckets];
  440. while (entry && (entry->paddr_index != paddr_index || entry->size != size)) {
  441. entry = entry->next;
  442. }
  443. if (!entry) {
  444. trace_xen_ram_addr_from_mapcache_not_in_cache(ptr);
  445. raddr = RAM_ADDR_INVALID;
  446. } else {
  447. raddr = (reventry->paddr_index << mc->bucket_shift) +
  448. ((unsigned long) ptr - (unsigned long) entry->vaddr_base);
  449. }
  450. mapcache_unlock(mc);
  451. return raddr;
  452. }
  453. ram_addr_t xen_ram_addr_from_mapcache(void *ptr)
  454. {
  455. ram_addr_t addr;
  456. addr = xen_ram_addr_from_mapcache_single(mapcache, ptr);
  457. if (addr == RAM_ADDR_INVALID) {
  458. addr = xen_ram_addr_from_mapcache_single(mapcache_grants, ptr);
  459. }
  460. return addr;
  461. }
  462. static void xen_invalidate_map_cache_entry_unlocked(MapCache *mc,
  463. uint8_t *buffer)
  464. {
  465. MapCacheEntry *entry = NULL, *pentry = NULL;
  466. MapCacheRev *reventry;
  467. hwaddr paddr_index;
  468. hwaddr size;
  469. int found = 0;
  470. int rc;
  471. QTAILQ_FOREACH(reventry, &mc->locked_entries, next) {
  472. if (reventry->vaddr_req == buffer) {
  473. paddr_index = reventry->paddr_index;
  474. size = reventry->size;
  475. found = 1;
  476. break;
  477. }
  478. }
  479. if (!found) {
  480. trace_xen_invalidate_map_cache_entry_unlocked_not_found(buffer);
  481. QTAILQ_FOREACH(reventry, &mc->locked_entries, next) {
  482. trace_xen_invalidate_map_cache_entry_unlocked_found(
  483. reventry->paddr_index,
  484. reventry->vaddr_req
  485. );
  486. }
  487. return;
  488. }
  489. QTAILQ_REMOVE(&mc->locked_entries, reventry, next);
  490. g_free(reventry);
  491. if (mc->last_entry != NULL &&
  492. mc->last_entry->paddr_index == paddr_index) {
  493. mc->last_entry = NULL;
  494. }
  495. entry = &mc->entry[paddr_index % mc->nr_buckets];
  496. while (entry && (entry->paddr_index != paddr_index || entry->size != size)) {
  497. pentry = entry;
  498. entry = entry->next;
  499. }
  500. if (!entry) {
  501. trace_xen_invalidate_map_cache_entry_unlocked_miss(buffer);
  502. return;
  503. }
  504. entry->lock--;
  505. if (entry->lock > 0) {
  506. return;
  507. }
  508. ram_block_notify_remove(entry->vaddr_base, entry->size, entry->size);
  509. if (entry->flags & XEN_MAPCACHE_ENTRY_GRANT) {
  510. rc = xengnttab_unmap(xen_region_gnttabdev, entry->vaddr_base,
  511. entry->size >> mc->bucket_shift);
  512. } else {
  513. rc = munmap(entry->vaddr_base, entry->size);
  514. }
  515. if (rc) {
  516. perror("unmap fails");
  517. exit(-1);
  518. }
  519. g_free(entry->valid_mapping);
  520. if (pentry) {
  521. pentry->next = entry->next;
  522. g_free(entry);
  523. } else {
  524. /*
  525. * Invalidate mapping but keep entry->next pointing to the rest
  526. * of the list.
  527. *
  528. * Note that lock is already zero here, otherwise we don't unmap.
  529. */
  530. entry->paddr_index = 0;
  531. entry->vaddr_base = NULL;
  532. entry->valid_mapping = NULL;
  533. entry->flags = 0;
  534. entry->size = 0;
  535. }
  536. }
  537. typedef struct XenMapCacheData {
  538. Coroutine *co;
  539. uint8_t *buffer;
  540. } XenMapCacheData;
  541. static void xen_invalidate_map_cache_entry_single(MapCache *mc, uint8_t *buffer)
  542. {
  543. mapcache_lock(mc);
  544. xen_invalidate_map_cache_entry_unlocked(mc, buffer);
  545. mapcache_unlock(mc);
  546. }
  547. static void xen_invalidate_map_cache_entry_all(uint8_t *buffer)
  548. {
  549. xen_invalidate_map_cache_entry_single(mapcache, buffer);
  550. xen_invalidate_map_cache_entry_single(mapcache_grants, buffer);
  551. }
  552. static void xen_invalidate_map_cache_entry_bh(void *opaque)
  553. {
  554. XenMapCacheData *data = opaque;
  555. xen_invalidate_map_cache_entry_all(data->buffer);
  556. aio_co_wake(data->co);
  557. }
  558. void coroutine_mixed_fn xen_invalidate_map_cache_entry(uint8_t *buffer)
  559. {
  560. if (qemu_in_coroutine()) {
  561. XenMapCacheData data = {
  562. .co = qemu_coroutine_self(),
  563. .buffer = buffer,
  564. };
  565. aio_bh_schedule_oneshot(qemu_get_current_aio_context(),
  566. xen_invalidate_map_cache_entry_bh, &data);
  567. qemu_coroutine_yield();
  568. } else {
  569. xen_invalidate_map_cache_entry_all(buffer);
  570. }
  571. }
  572. static void xen_invalidate_map_cache_single(MapCache *mc)
  573. {
  574. unsigned long i;
  575. MapCacheRev *reventry;
  576. mapcache_lock(mc);
  577. QTAILQ_FOREACH(reventry, &mc->locked_entries, next) {
  578. if (!reventry->dma) {
  579. continue;
  580. }
  581. trace_xen_invalidate_map_cache(reventry->paddr_index,
  582. reventry->vaddr_req);
  583. }
  584. for (i = 0; i < mc->nr_buckets; i++) {
  585. MapCacheEntry *entry = &mc->entry[i];
  586. if (entry->vaddr_base == NULL) {
  587. continue;
  588. }
  589. if (entry->lock > 0) {
  590. continue;
  591. }
  592. if (munmap(entry->vaddr_base, entry->size) != 0) {
  593. perror("unmap fails");
  594. exit(-1);
  595. }
  596. entry->paddr_index = 0;
  597. entry->vaddr_base = NULL;
  598. entry->size = 0;
  599. g_free(entry->valid_mapping);
  600. entry->valid_mapping = NULL;
  601. }
  602. mc->last_entry = NULL;
  603. mapcache_unlock(mc);
  604. }
  605. void xen_invalidate_map_cache(void)
  606. {
  607. /* Flush pending AIO before destroying the mapcache */
  608. bdrv_drain_all();
  609. xen_invalidate_map_cache_single(mapcache);
  610. xen_invalidate_map_cache_single(mapcache_grants);
  611. }
  612. static uint8_t *xen_replace_cache_entry_unlocked(MapCache *mc,
  613. hwaddr old_phys_addr,
  614. hwaddr new_phys_addr,
  615. hwaddr size)
  616. {
  617. MapCacheEntry *entry;
  618. hwaddr address_index, address_offset;
  619. hwaddr test_bit_size, cache_size = size;
  620. address_index = old_phys_addr >> mc->bucket_shift;
  621. address_offset = old_phys_addr & (mc->bucket_size - 1);
  622. assert(size);
  623. /* test_bit_size is always a multiple of XC_PAGE_SIZE */
  624. test_bit_size = size + (old_phys_addr & (XC_PAGE_SIZE - 1));
  625. if (test_bit_size % XC_PAGE_SIZE) {
  626. test_bit_size += XC_PAGE_SIZE - (test_bit_size % XC_PAGE_SIZE);
  627. }
  628. cache_size = size + address_offset;
  629. if (cache_size % mc->bucket_size) {
  630. cache_size += mc->bucket_size - (cache_size % mc->bucket_size);
  631. }
  632. entry = &mc->entry[address_index % mc->nr_buckets];
  633. while (entry && !(entry->paddr_index == address_index &&
  634. entry->size == cache_size)) {
  635. entry = entry->next;
  636. }
  637. if (!entry) {
  638. trace_xen_replace_cache_entry_unlocked(old_phys_addr);
  639. return NULL;
  640. }
  641. assert((entry->flags & XEN_MAPCACHE_ENTRY_GRANT) == 0);
  642. address_index = new_phys_addr >> mc->bucket_shift;
  643. address_offset = new_phys_addr & (mc->bucket_size - 1);
  644. trace_xen_replace_cache_entry_dummy(old_phys_addr, new_phys_addr);
  645. xen_remap_bucket(mc, entry, entry->vaddr_base,
  646. cache_size, address_index, false,
  647. false, false, old_phys_addr);
  648. if (!test_bits(address_offset >> XC_PAGE_SHIFT,
  649. test_bit_size >> XC_PAGE_SHIFT,
  650. entry->valid_mapping)) {
  651. trace_xen_replace_cache_entry_unlocked_could_not_update_entry(
  652. old_phys_addr
  653. );
  654. return NULL;
  655. }
  656. return entry->vaddr_base + address_offset;
  657. }
  658. uint8_t *xen_replace_cache_entry(hwaddr old_phys_addr,
  659. hwaddr new_phys_addr,
  660. hwaddr size)
  661. {
  662. uint8_t *p;
  663. mapcache_lock(mapcache);
  664. p = xen_replace_cache_entry_unlocked(mapcache, old_phys_addr,
  665. new_phys_addr, size);
  666. mapcache_unlock(mapcache);
  667. return p;
  668. }