memory.h 117 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211
  1. /*
  2. * Physical memory management API
  3. *
  4. * Copyright 2011 Red Hat, Inc. and/or its affiliates
  5. *
  6. * Authors:
  7. * Avi Kivity <avi@redhat.com>
  8. *
  9. * This work is licensed under the terms of the GNU GPL, version 2. See
  10. * the COPYING file in the top-level directory.
  11. *
  12. */
  13. #ifndef MEMORY_H
  14. #define MEMORY_H
  15. #ifndef CONFIG_USER_ONLY
  16. #include "exec/cpu-common.h"
  17. #include "exec/hwaddr.h"
  18. #include "exec/memattrs.h"
  19. #include "exec/memop.h"
  20. #include "exec/ramlist.h"
  21. #include "qemu/bswap.h"
  22. #include "qemu/queue.h"
  23. #include "qemu/int128.h"
  24. #include "qemu/range.h"
  25. #include "qemu/notify.h"
  26. #include "qom/object.h"
  27. #include "qemu/rcu.h"
  28. #define RAM_ADDR_INVALID (~(ram_addr_t)0)
  29. #define MAX_PHYS_ADDR_SPACE_BITS 62
  30. #define MAX_PHYS_ADDR (((hwaddr)1 << MAX_PHYS_ADDR_SPACE_BITS) - 1)
  31. #define TYPE_MEMORY_REGION "memory-region"
  32. DECLARE_INSTANCE_CHECKER(MemoryRegion, MEMORY_REGION,
  33. TYPE_MEMORY_REGION)
  34. #define TYPE_IOMMU_MEMORY_REGION "iommu-memory-region"
  35. typedef struct IOMMUMemoryRegionClass IOMMUMemoryRegionClass;
  36. DECLARE_OBJ_CHECKERS(IOMMUMemoryRegion, IOMMUMemoryRegionClass,
  37. IOMMU_MEMORY_REGION, TYPE_IOMMU_MEMORY_REGION)
  38. #define TYPE_RAM_DISCARD_MANAGER "ram-discard-manager"
  39. typedef struct RamDiscardManagerClass RamDiscardManagerClass;
  40. typedef struct RamDiscardManager RamDiscardManager;
  41. DECLARE_OBJ_CHECKERS(RamDiscardManager, RamDiscardManagerClass,
  42. RAM_DISCARD_MANAGER, TYPE_RAM_DISCARD_MANAGER);
  43. #ifdef CONFIG_FUZZ
  44. void fuzz_dma_read_cb(size_t addr,
  45. size_t len,
  46. MemoryRegion *mr);
  47. #else
  48. static inline void fuzz_dma_read_cb(size_t addr,
  49. size_t len,
  50. MemoryRegion *mr)
  51. {
  52. /* Do Nothing */
  53. }
  54. #endif
  55. /* Possible bits for global_dirty_log_{start|stop} */
  56. /* Dirty tracking enabled because migration is running */
  57. #define GLOBAL_DIRTY_MIGRATION (1U << 0)
  58. /* Dirty tracking enabled because measuring dirty rate */
  59. #define GLOBAL_DIRTY_DIRTY_RATE (1U << 1)
  60. /* Dirty tracking enabled because dirty limit */
  61. #define GLOBAL_DIRTY_LIMIT (1U << 2)
  62. #define GLOBAL_DIRTY_MASK (0x7)
  63. extern unsigned int global_dirty_tracking;
  64. typedef struct MemoryRegionOps MemoryRegionOps;
  65. struct ReservedRegion {
  66. Range range;
  67. unsigned type;
  68. };
  69. /**
  70. * struct MemoryRegionSection: describes a fragment of a #MemoryRegion
  71. *
  72. * @mr: the region, or %NULL if empty
  73. * @fv: the flat view of the address space the region is mapped in
  74. * @offset_within_region: the beginning of the section, relative to @mr's start
  75. * @size: the size of the section; will not exceed @mr's boundaries
  76. * @offset_within_address_space: the address of the first byte of the section
  77. * relative to the region's address space
  78. * @readonly: writes to this section are ignored
  79. * @nonvolatile: this section is non-volatile
  80. * @unmergeable: this section should not get merged with adjacent sections
  81. */
  82. struct MemoryRegionSection {
  83. Int128 size;
  84. MemoryRegion *mr;
  85. FlatView *fv;
  86. hwaddr offset_within_region;
  87. hwaddr offset_within_address_space;
  88. bool readonly;
  89. bool nonvolatile;
  90. bool unmergeable;
  91. };
  92. typedef struct IOMMUTLBEntry IOMMUTLBEntry;
  93. /* See address_space_translate: bit 0 is read, bit 1 is write. */
  94. typedef enum {
  95. IOMMU_NONE = 0,
  96. IOMMU_RO = 1,
  97. IOMMU_WO = 2,
  98. IOMMU_RW = 3,
  99. } IOMMUAccessFlags;
  100. #define IOMMU_ACCESS_FLAG(r, w) (((r) ? IOMMU_RO : 0) | ((w) ? IOMMU_WO : 0))
  101. struct IOMMUTLBEntry {
  102. AddressSpace *target_as;
  103. hwaddr iova;
  104. hwaddr translated_addr;
  105. hwaddr addr_mask; /* 0xfff = 4k translation */
  106. IOMMUAccessFlags perm;
  107. };
  108. /*
  109. * Bitmap for different IOMMUNotifier capabilities. Each notifier can
  110. * register with one or multiple IOMMU Notifier capability bit(s).
  111. *
  112. * Normally there're two use cases for the notifiers:
  113. *
  114. * (1) When the device needs accurate synchronizations of the vIOMMU page
  115. * tables, it needs to register with both MAP|UNMAP notifies (which
  116. * is defined as IOMMU_NOTIFIER_IOTLB_EVENTS below).
  117. *
  118. * Regarding to accurate synchronization, it's when the notified
  119. * device maintains a shadow page table and must be notified on each
  120. * guest MAP (page table entry creation) and UNMAP (invalidation)
  121. * events (e.g. VFIO). Both notifications must be accurate so that
  122. * the shadow page table is fully in sync with the guest view.
  123. *
  124. * (2) When the device doesn't need accurate synchronizations of the
  125. * vIOMMU page tables, it needs to register only with UNMAP or
  126. * DEVIOTLB_UNMAP notifies.
  127. *
  128. * It's when the device maintains a cache of IOMMU translations
  129. * (IOTLB) and is able to fill that cache by requesting translations
  130. * from the vIOMMU through a protocol similar to ATS (Address
  131. * Translation Service).
  132. *
  133. * Note that in this mode the vIOMMU will not maintain a shadowed
  134. * page table for the address space, and the UNMAP messages can cover
  135. * more than the pages that used to get mapped. The IOMMU notifiee
  136. * should be able to take care of over-sized invalidations.
  137. */
  138. typedef enum {
  139. IOMMU_NOTIFIER_NONE = 0,
  140. /* Notify cache invalidations */
  141. IOMMU_NOTIFIER_UNMAP = 0x1,
  142. /* Notify entry changes (newly created entries) */
  143. IOMMU_NOTIFIER_MAP = 0x2,
  144. /* Notify changes on device IOTLB entries */
  145. IOMMU_NOTIFIER_DEVIOTLB_UNMAP = 0x04,
  146. } IOMMUNotifierFlag;
  147. #define IOMMU_NOTIFIER_IOTLB_EVENTS (IOMMU_NOTIFIER_MAP | IOMMU_NOTIFIER_UNMAP)
  148. #define IOMMU_NOTIFIER_DEVIOTLB_EVENTS IOMMU_NOTIFIER_DEVIOTLB_UNMAP
  149. #define IOMMU_NOTIFIER_ALL (IOMMU_NOTIFIER_IOTLB_EVENTS | \
  150. IOMMU_NOTIFIER_DEVIOTLB_EVENTS)
  151. struct IOMMUNotifier;
  152. typedef void (*IOMMUNotify)(struct IOMMUNotifier *notifier,
  153. IOMMUTLBEntry *data);
  154. struct IOMMUNotifier {
  155. IOMMUNotify notify;
  156. IOMMUNotifierFlag notifier_flags;
  157. /* Notify for address space range start <= addr <= end */
  158. hwaddr start;
  159. hwaddr end;
  160. int iommu_idx;
  161. QLIST_ENTRY(IOMMUNotifier) node;
  162. };
  163. typedef struct IOMMUNotifier IOMMUNotifier;
  164. typedef struct IOMMUTLBEvent {
  165. IOMMUNotifierFlag type;
  166. IOMMUTLBEntry entry;
  167. } IOMMUTLBEvent;
  168. /* RAM is pre-allocated and passed into qemu_ram_alloc_from_ptr */
  169. #define RAM_PREALLOC (1 << 0)
  170. /* RAM is mmap-ed with MAP_SHARED */
  171. #define RAM_SHARED (1 << 1)
  172. /* Only a portion of RAM (used_length) is actually used, and migrated.
  173. * Resizing RAM while migrating can result in the migration being canceled.
  174. */
  175. #define RAM_RESIZEABLE (1 << 2)
  176. /* UFFDIO_ZEROPAGE is available on this RAMBlock to atomically
  177. * zero the page and wake waiting processes.
  178. * (Set during postcopy)
  179. */
  180. #define RAM_UF_ZEROPAGE (1 << 3)
  181. /* RAM can be migrated */
  182. #define RAM_MIGRATABLE (1 << 4)
  183. /* RAM is a persistent kind memory */
  184. #define RAM_PMEM (1 << 5)
  185. /*
  186. * UFFDIO_WRITEPROTECT is used on this RAMBlock to
  187. * support 'write-tracking' migration type.
  188. * Implies ram_state->ram_wt_enabled.
  189. */
  190. #define RAM_UF_WRITEPROTECT (1 << 6)
  191. /*
  192. * RAM is mmap-ed with MAP_NORESERVE. When set, reserving swap space (or huge
  193. * pages if applicable) is skipped: will bail out if not supported. When not
  194. * set, the OS will do the reservation, if supported for the memory type.
  195. */
  196. #define RAM_NORESERVE (1 << 7)
  197. /* RAM that isn't accessible through normal means. */
  198. #define RAM_PROTECTED (1 << 8)
  199. /* RAM is an mmap-ed named file */
  200. #define RAM_NAMED_FILE (1 << 9)
  201. /* RAM is mmap-ed read-only */
  202. #define RAM_READONLY (1 << 10)
  203. /* RAM FD is opened read-only */
  204. #define RAM_READONLY_FD (1 << 11)
  205. /* RAM can be private that has kvm guest memfd backend */
  206. #define RAM_GUEST_MEMFD (1 << 12)
  207. /*
  208. * In RAMBlock creation functions, if MAP_SHARED is 0 in the flags parameter,
  209. * the implementation may still create a shared mapping if other conditions
  210. * require it. Callers who specifically want a private mapping, eg objects
  211. * specified by the user, must pass RAM_PRIVATE.
  212. * After RAMBlock creation, MAP_SHARED in the block's flags indicates whether
  213. * the block is shared or private, and MAP_PRIVATE is omitted.
  214. */
  215. #define RAM_PRIVATE (1 << 13)
  216. static inline void iommu_notifier_init(IOMMUNotifier *n, IOMMUNotify fn,
  217. IOMMUNotifierFlag flags,
  218. hwaddr start, hwaddr end,
  219. int iommu_idx)
  220. {
  221. n->notify = fn;
  222. n->notifier_flags = flags;
  223. n->start = start;
  224. n->end = end;
  225. n->iommu_idx = iommu_idx;
  226. }
  227. /*
  228. * Memory region callbacks
  229. */
  230. struct MemoryRegionOps {
  231. /* Read from the memory region. @addr is relative to @mr; @size is
  232. * in bytes. */
  233. uint64_t (*read)(void *opaque,
  234. hwaddr addr,
  235. unsigned size);
  236. /* Write to the memory region. @addr is relative to @mr; @size is
  237. * in bytes. */
  238. void (*write)(void *opaque,
  239. hwaddr addr,
  240. uint64_t data,
  241. unsigned size);
  242. MemTxResult (*read_with_attrs)(void *opaque,
  243. hwaddr addr,
  244. uint64_t *data,
  245. unsigned size,
  246. MemTxAttrs attrs);
  247. MemTxResult (*write_with_attrs)(void *opaque,
  248. hwaddr addr,
  249. uint64_t data,
  250. unsigned size,
  251. MemTxAttrs attrs);
  252. enum device_endian endianness;
  253. /* Guest-visible constraints: */
  254. struct {
  255. /* If nonzero, specify bounds on access sizes beyond which a machine
  256. * check is thrown.
  257. */
  258. unsigned min_access_size;
  259. unsigned max_access_size;
  260. /* If true, unaligned accesses are supported. Otherwise unaligned
  261. * accesses throw machine checks.
  262. */
  263. bool unaligned;
  264. /*
  265. * If present, and returns #false, the transaction is not accepted
  266. * by the device (and results in machine dependent behaviour such
  267. * as a machine check exception).
  268. */
  269. bool (*accepts)(void *opaque, hwaddr addr,
  270. unsigned size, bool is_write,
  271. MemTxAttrs attrs);
  272. } valid;
  273. /* Internal implementation constraints: */
  274. struct {
  275. /* If nonzero, specifies the minimum size implemented. Smaller sizes
  276. * will be rounded upwards and a partial result will be returned.
  277. */
  278. unsigned min_access_size;
  279. /* If nonzero, specifies the maximum size implemented. Larger sizes
  280. * will be done as a series of accesses with smaller sizes.
  281. */
  282. unsigned max_access_size;
  283. /* If true, unaligned accesses are supported. Otherwise all accesses
  284. * are converted to (possibly multiple) naturally aligned accesses.
  285. */
  286. bool unaligned;
  287. } impl;
  288. };
  289. typedef struct MemoryRegionClass {
  290. /* private */
  291. ObjectClass parent_class;
  292. } MemoryRegionClass;
  293. enum IOMMUMemoryRegionAttr {
  294. IOMMU_ATTR_SPAPR_TCE_FD
  295. };
  296. /*
  297. * IOMMUMemoryRegionClass:
  298. *
  299. * All IOMMU implementations need to subclass TYPE_IOMMU_MEMORY_REGION
  300. * and provide an implementation of at least the @translate method here
  301. * to handle requests to the memory region. Other methods are optional.
  302. *
  303. * The IOMMU implementation must use the IOMMU notifier infrastructure
  304. * to report whenever mappings are changed, by calling
  305. * memory_region_notify_iommu() (or, if necessary, by calling
  306. * memory_region_notify_iommu_one() for each registered notifier).
  307. *
  308. * Conceptually an IOMMU provides a mapping from input address
  309. * to an output TLB entry. If the IOMMU is aware of memory transaction
  310. * attributes and the output TLB entry depends on the transaction
  311. * attributes, we represent this using IOMMU indexes. Each index
  312. * selects a particular translation table that the IOMMU has:
  313. *
  314. * @attrs_to_index returns the IOMMU index for a set of transaction attributes
  315. *
  316. * @translate takes an input address and an IOMMU index
  317. *
  318. * and the mapping returned can only depend on the input address and the
  319. * IOMMU index.
  320. *
  321. * Most IOMMUs don't care about the transaction attributes and support
  322. * only a single IOMMU index. A more complex IOMMU might have one index
  323. * for secure transactions and one for non-secure transactions.
  324. */
  325. struct IOMMUMemoryRegionClass {
  326. /* private: */
  327. MemoryRegionClass parent_class;
  328. /* public: */
  329. /**
  330. * @translate:
  331. *
  332. * Return a TLB entry that contains a given address.
  333. *
  334. * The IOMMUAccessFlags indicated via @flag are optional and may
  335. * be specified as IOMMU_NONE to indicate that the caller needs
  336. * the full translation information for both reads and writes. If
  337. * the access flags are specified then the IOMMU implementation
  338. * may use this as an optimization, to stop doing a page table
  339. * walk as soon as it knows that the requested permissions are not
  340. * allowed. If IOMMU_NONE is passed then the IOMMU must do the
  341. * full page table walk and report the permissions in the returned
  342. * IOMMUTLBEntry. (Note that this implies that an IOMMU may not
  343. * return different mappings for reads and writes.)
  344. *
  345. * The returned information remains valid while the caller is
  346. * holding the big QEMU lock or is inside an RCU critical section;
  347. * if the caller wishes to cache the mapping beyond that it must
  348. * register an IOMMU notifier so it can invalidate its cached
  349. * information when the IOMMU mapping changes.
  350. *
  351. * @iommu: the IOMMUMemoryRegion
  352. *
  353. * @hwaddr: address to be translated within the memory region
  354. *
  355. * @flag: requested access permission
  356. *
  357. * @iommu_idx: IOMMU index for the translation
  358. */
  359. IOMMUTLBEntry (*translate)(IOMMUMemoryRegion *iommu, hwaddr addr,
  360. IOMMUAccessFlags flag, int iommu_idx);
  361. /**
  362. * @get_min_page_size:
  363. *
  364. * Returns minimum supported page size in bytes.
  365. *
  366. * If this method is not provided then the minimum is assumed to
  367. * be TARGET_PAGE_SIZE.
  368. *
  369. * @iommu: the IOMMUMemoryRegion
  370. */
  371. uint64_t (*get_min_page_size)(IOMMUMemoryRegion *iommu);
  372. /**
  373. * @notify_flag_changed:
  374. *
  375. * Called when IOMMU Notifier flag changes (ie when the set of
  376. * events which IOMMU users are requesting notification for changes).
  377. * Optional method -- need not be provided if the IOMMU does not
  378. * need to know exactly which events must be notified.
  379. *
  380. * @iommu: the IOMMUMemoryRegion
  381. *
  382. * @old_flags: events which previously needed to be notified
  383. *
  384. * @new_flags: events which now need to be notified
  385. *
  386. * Returns 0 on success, or a negative errno; in particular
  387. * returns -EINVAL if the new flag bitmap is not supported by the
  388. * IOMMU memory region. In case of failure, the error object
  389. * must be created
  390. */
  391. int (*notify_flag_changed)(IOMMUMemoryRegion *iommu,
  392. IOMMUNotifierFlag old_flags,
  393. IOMMUNotifierFlag new_flags,
  394. Error **errp);
  395. /**
  396. * @replay:
  397. *
  398. * Called to handle memory_region_iommu_replay().
  399. *
  400. * The default implementation of memory_region_iommu_replay() is to
  401. * call the IOMMU translate method for every page in the address space
  402. * with flag == IOMMU_NONE and then call the notifier if translate
  403. * returns a valid mapping. If this method is implemented then it
  404. * overrides the default behaviour, and must provide the full semantics
  405. * of memory_region_iommu_replay(), by calling @notifier for every
  406. * translation present in the IOMMU.
  407. *
  408. * Optional method -- an IOMMU only needs to provide this method
  409. * if the default is inefficient or produces undesirable side effects.
  410. *
  411. * Note: this is not related to record-and-replay functionality.
  412. */
  413. void (*replay)(IOMMUMemoryRegion *iommu, IOMMUNotifier *notifier);
  414. /**
  415. * @get_attr:
  416. *
  417. * Get IOMMU misc attributes. This is an optional method that
  418. * can be used to allow users of the IOMMU to get implementation-specific
  419. * information. The IOMMU implements this method to handle calls
  420. * by IOMMU users to memory_region_iommu_get_attr() by filling in
  421. * the arbitrary data pointer for any IOMMUMemoryRegionAttr values that
  422. * the IOMMU supports. If the method is unimplemented then
  423. * memory_region_iommu_get_attr() will always return -EINVAL.
  424. *
  425. * @iommu: the IOMMUMemoryRegion
  426. *
  427. * @attr: attribute being queried
  428. *
  429. * @data: memory to fill in with the attribute data
  430. *
  431. * Returns 0 on success, or a negative errno; in particular
  432. * returns -EINVAL for unrecognized or unimplemented attribute types.
  433. */
  434. int (*get_attr)(IOMMUMemoryRegion *iommu, enum IOMMUMemoryRegionAttr attr,
  435. void *data);
  436. /**
  437. * @attrs_to_index:
  438. *
  439. * Return the IOMMU index to use for a given set of transaction attributes.
  440. *
  441. * Optional method: if an IOMMU only supports a single IOMMU index then
  442. * the default implementation of memory_region_iommu_attrs_to_index()
  443. * will return 0.
  444. *
  445. * The indexes supported by an IOMMU must be contiguous, starting at 0.
  446. *
  447. * @iommu: the IOMMUMemoryRegion
  448. * @attrs: memory transaction attributes
  449. */
  450. int (*attrs_to_index)(IOMMUMemoryRegion *iommu, MemTxAttrs attrs);
  451. /**
  452. * @num_indexes:
  453. *
  454. * Return the number of IOMMU indexes this IOMMU supports.
  455. *
  456. * Optional method: if this method is not provided, then
  457. * memory_region_iommu_num_indexes() will return 1, indicating that
  458. * only a single IOMMU index is supported.
  459. *
  460. * @iommu: the IOMMUMemoryRegion
  461. */
  462. int (*num_indexes)(IOMMUMemoryRegion *iommu);
  463. };
  464. typedef struct RamDiscardListener RamDiscardListener;
  465. typedef int (*NotifyRamPopulate)(RamDiscardListener *rdl,
  466. MemoryRegionSection *section);
  467. typedef void (*NotifyRamDiscard)(RamDiscardListener *rdl,
  468. MemoryRegionSection *section);
  469. struct RamDiscardListener {
  470. /*
  471. * @notify_populate:
  472. *
  473. * Notification that previously discarded memory is about to get populated.
  474. * Listeners are able to object. If any listener objects, already
  475. * successfully notified listeners are notified about a discard again.
  476. *
  477. * @rdl: the #RamDiscardListener getting notified
  478. * @section: the #MemoryRegionSection to get populated. The section
  479. * is aligned within the memory region to the minimum granularity
  480. * unless it would exceed the registered section.
  481. *
  482. * Returns 0 on success. If the notification is rejected by the listener,
  483. * an error is returned.
  484. */
  485. NotifyRamPopulate notify_populate;
  486. /*
  487. * @notify_discard:
  488. *
  489. * Notification that previously populated memory was discarded successfully
  490. * and listeners should drop all references to such memory and prevent
  491. * new population (e.g., unmap).
  492. *
  493. * @rdl: the #RamDiscardListener getting notified
  494. * @section: the #MemoryRegionSection to get populated. The section
  495. * is aligned within the memory region to the minimum granularity
  496. * unless it would exceed the registered section.
  497. */
  498. NotifyRamDiscard notify_discard;
  499. /*
  500. * @double_discard_supported:
  501. *
  502. * The listener suppors getting @notify_discard notifications that span
  503. * already discarded parts.
  504. */
  505. bool double_discard_supported;
  506. MemoryRegionSection *section;
  507. QLIST_ENTRY(RamDiscardListener) next;
  508. };
  509. static inline void ram_discard_listener_init(RamDiscardListener *rdl,
  510. NotifyRamPopulate populate_fn,
  511. NotifyRamDiscard discard_fn,
  512. bool double_discard_supported)
  513. {
  514. rdl->notify_populate = populate_fn;
  515. rdl->notify_discard = discard_fn;
  516. rdl->double_discard_supported = double_discard_supported;
  517. }
  518. typedef int (*ReplayRamPopulate)(MemoryRegionSection *section, void *opaque);
  519. typedef void (*ReplayRamDiscard)(MemoryRegionSection *section, void *opaque);
  520. /*
  521. * RamDiscardManagerClass:
  522. *
  523. * A #RamDiscardManager coordinates which parts of specific RAM #MemoryRegion
  524. * regions are currently populated to be used/accessed by the VM, notifying
  525. * after parts were discarded (freeing up memory) and before parts will be
  526. * populated (consuming memory), to be used/accessed by the VM.
  527. *
  528. * A #RamDiscardManager can only be set for a RAM #MemoryRegion while the
  529. * #MemoryRegion isn't mapped into an address space yet (either directly
  530. * or via an alias); it cannot change while the #MemoryRegion is
  531. * mapped into an address space.
  532. *
  533. * The #RamDiscardManager is intended to be used by technologies that are
  534. * incompatible with discarding of RAM (e.g., VFIO, which may pin all
  535. * memory inside a #MemoryRegion), and require proper coordination to only
  536. * map the currently populated parts, to hinder parts that are expected to
  537. * remain discarded from silently getting populated and consuming memory.
  538. * Technologies that support discarding of RAM don't have to bother and can
  539. * simply map the whole #MemoryRegion.
  540. *
  541. * An example #RamDiscardManager is virtio-mem, which logically (un)plugs
  542. * memory within an assigned RAM #MemoryRegion, coordinated with the VM.
  543. * Logically unplugging memory consists of discarding RAM. The VM agreed to not
  544. * access unplugged (discarded) memory - especially via DMA. virtio-mem will
  545. * properly coordinate with listeners before memory is plugged (populated),
  546. * and after memory is unplugged (discarded).
  547. *
  548. * Listeners are called in multiples of the minimum granularity (unless it
  549. * would exceed the registered range) and changes are aligned to the minimum
  550. * granularity within the #MemoryRegion. Listeners have to prepare for memory
  551. * becoming discarded in a different granularity than it was populated and the
  552. * other way around.
  553. */
  554. struct RamDiscardManagerClass {
  555. /* private */
  556. InterfaceClass parent_class;
  557. /* public */
  558. /**
  559. * @get_min_granularity:
  560. *
  561. * Get the minimum granularity in which listeners will get notified
  562. * about changes within the #MemoryRegion via the #RamDiscardManager.
  563. *
  564. * @rdm: the #RamDiscardManager
  565. * @mr: the #MemoryRegion
  566. *
  567. * Returns the minimum granularity.
  568. */
  569. uint64_t (*get_min_granularity)(const RamDiscardManager *rdm,
  570. const MemoryRegion *mr);
  571. /**
  572. * @is_populated:
  573. *
  574. * Check whether the given #MemoryRegionSection is completely populated
  575. * (i.e., no parts are currently discarded) via the #RamDiscardManager.
  576. * There are no alignment requirements.
  577. *
  578. * @rdm: the #RamDiscardManager
  579. * @section: the #MemoryRegionSection
  580. *
  581. * Returns whether the given range is completely populated.
  582. */
  583. bool (*is_populated)(const RamDiscardManager *rdm,
  584. const MemoryRegionSection *section);
  585. /**
  586. * @replay_populated:
  587. *
  588. * Call the #ReplayRamPopulate callback for all populated parts within the
  589. * #MemoryRegionSection via the #RamDiscardManager.
  590. *
  591. * In case any call fails, no further calls are made.
  592. *
  593. * @rdm: the #RamDiscardManager
  594. * @section: the #MemoryRegionSection
  595. * @replay_fn: the #ReplayRamPopulate callback
  596. * @opaque: pointer to forward to the callback
  597. *
  598. * Returns 0 on success, or a negative error if any notification failed.
  599. */
  600. int (*replay_populated)(const RamDiscardManager *rdm,
  601. MemoryRegionSection *section,
  602. ReplayRamPopulate replay_fn, void *opaque);
  603. /**
  604. * @replay_discarded:
  605. *
  606. * Call the #ReplayRamDiscard callback for all discarded parts within the
  607. * #MemoryRegionSection via the #RamDiscardManager.
  608. *
  609. * @rdm: the #RamDiscardManager
  610. * @section: the #MemoryRegionSection
  611. * @replay_fn: the #ReplayRamDiscard callback
  612. * @opaque: pointer to forward to the callback
  613. */
  614. void (*replay_discarded)(const RamDiscardManager *rdm,
  615. MemoryRegionSection *section,
  616. ReplayRamDiscard replay_fn, void *opaque);
  617. /**
  618. * @register_listener:
  619. *
  620. * Register a #RamDiscardListener for the given #MemoryRegionSection and
  621. * immediately notify the #RamDiscardListener about all populated parts
  622. * within the #MemoryRegionSection via the #RamDiscardManager.
  623. *
  624. * In case any notification fails, no further notifications are triggered
  625. * and an error is logged.
  626. *
  627. * @rdm: the #RamDiscardManager
  628. * @rdl: the #RamDiscardListener
  629. * @section: the #MemoryRegionSection
  630. */
  631. void (*register_listener)(RamDiscardManager *rdm,
  632. RamDiscardListener *rdl,
  633. MemoryRegionSection *section);
  634. /**
  635. * @unregister_listener:
  636. *
  637. * Unregister a previously registered #RamDiscardListener via the
  638. * #RamDiscardManager after notifying the #RamDiscardListener about all
  639. * populated parts becoming unpopulated within the registered
  640. * #MemoryRegionSection.
  641. *
  642. * @rdm: the #RamDiscardManager
  643. * @rdl: the #RamDiscardListener
  644. */
  645. void (*unregister_listener)(RamDiscardManager *rdm,
  646. RamDiscardListener *rdl);
  647. };
  648. uint64_t ram_discard_manager_get_min_granularity(const RamDiscardManager *rdm,
  649. const MemoryRegion *mr);
  650. bool ram_discard_manager_is_populated(const RamDiscardManager *rdm,
  651. const MemoryRegionSection *section);
  652. int ram_discard_manager_replay_populated(const RamDiscardManager *rdm,
  653. MemoryRegionSection *section,
  654. ReplayRamPopulate replay_fn,
  655. void *opaque);
  656. void ram_discard_manager_replay_discarded(const RamDiscardManager *rdm,
  657. MemoryRegionSection *section,
  658. ReplayRamDiscard replay_fn,
  659. void *opaque);
  660. void ram_discard_manager_register_listener(RamDiscardManager *rdm,
  661. RamDiscardListener *rdl,
  662. MemoryRegionSection *section);
  663. void ram_discard_manager_unregister_listener(RamDiscardManager *rdm,
  664. RamDiscardListener *rdl);
  665. /**
  666. * memory_get_xlat_addr: Extract addresses from a TLB entry
  667. *
  668. * @iotlb: pointer to an #IOMMUTLBEntry
  669. * @vaddr: virtual address
  670. * @ram_addr: RAM address
  671. * @read_only: indicates if writes are allowed
  672. * @mr_has_discard_manager: indicates memory is controlled by a
  673. * RamDiscardManager
  674. * @errp: pointer to Error*, to store an error if it happens.
  675. *
  676. * Return: true on success, else false setting @errp with error.
  677. */
  678. bool memory_get_xlat_addr(IOMMUTLBEntry *iotlb, void **vaddr,
  679. ram_addr_t *ram_addr, bool *read_only,
  680. bool *mr_has_discard_manager, Error **errp);
  681. typedef struct CoalescedMemoryRange CoalescedMemoryRange;
  682. typedef struct MemoryRegionIoeventfd MemoryRegionIoeventfd;
  683. /** MemoryRegion:
  684. *
  685. * A struct representing a memory region.
  686. */
  687. struct MemoryRegion {
  688. Object parent_obj;
  689. /* private: */
  690. /* The following fields should fit in a cache line */
  691. bool romd_mode;
  692. bool ram;
  693. bool subpage;
  694. bool readonly; /* For RAM regions */
  695. bool nonvolatile;
  696. bool rom_device;
  697. bool flush_coalesced_mmio;
  698. bool unmergeable;
  699. uint8_t dirty_log_mask;
  700. bool is_iommu;
  701. RAMBlock *ram_block;
  702. Object *owner;
  703. /* owner as TYPE_DEVICE. Used for re-entrancy checks in MR access hotpath */
  704. DeviceState *dev;
  705. const MemoryRegionOps *ops;
  706. void *opaque;
  707. MemoryRegion *container;
  708. int mapped_via_alias; /* Mapped via an alias, container might be NULL */
  709. Int128 size;
  710. hwaddr addr;
  711. void (*destructor)(MemoryRegion *mr);
  712. uint64_t align;
  713. bool terminates;
  714. bool ram_device;
  715. bool enabled;
  716. uint8_t vga_logging_count;
  717. MemoryRegion *alias;
  718. hwaddr alias_offset;
  719. int32_t priority;
  720. QTAILQ_HEAD(, MemoryRegion) subregions;
  721. QTAILQ_ENTRY(MemoryRegion) subregions_link;
  722. QTAILQ_HEAD(, CoalescedMemoryRange) coalesced;
  723. const char *name;
  724. unsigned ioeventfd_nb;
  725. MemoryRegionIoeventfd *ioeventfds;
  726. RamDiscardManager *rdm; /* Only for RAM */
  727. /* For devices designed to perform re-entrant IO into their own IO MRs */
  728. bool disable_reentrancy_guard;
  729. };
  730. struct IOMMUMemoryRegion {
  731. MemoryRegion parent_obj;
  732. QLIST_HEAD(, IOMMUNotifier) iommu_notify;
  733. IOMMUNotifierFlag iommu_notify_flags;
  734. };
  735. #define IOMMU_NOTIFIER_FOREACH(n, mr) \
  736. QLIST_FOREACH((n), &(mr)->iommu_notify, node)
  737. #define MEMORY_LISTENER_PRIORITY_MIN 0
  738. #define MEMORY_LISTENER_PRIORITY_ACCEL 10
  739. #define MEMORY_LISTENER_PRIORITY_DEV_BACKEND 10
  740. /**
  741. * struct MemoryListener: callbacks structure for updates to the physical memory map
  742. *
  743. * Allows a component to adjust to changes in the guest-visible memory map.
  744. * Use with memory_listener_register() and memory_listener_unregister().
  745. */
  746. struct MemoryListener {
  747. /**
  748. * @begin:
  749. *
  750. * Called at the beginning of an address space update transaction.
  751. * Followed by calls to #MemoryListener.region_add(),
  752. * #MemoryListener.region_del(), #MemoryListener.region_nop(),
  753. * #MemoryListener.log_start() and #MemoryListener.log_stop() in
  754. * increasing address order.
  755. *
  756. * @listener: The #MemoryListener.
  757. */
  758. void (*begin)(MemoryListener *listener);
  759. /**
  760. * @commit:
  761. *
  762. * Called at the end of an address space update transaction,
  763. * after the last call to #MemoryListener.region_add(),
  764. * #MemoryListener.region_del() or #MemoryListener.region_nop(),
  765. * #MemoryListener.log_start() and #MemoryListener.log_stop().
  766. *
  767. * @listener: The #MemoryListener.
  768. */
  769. void (*commit)(MemoryListener *listener);
  770. /**
  771. * @region_add:
  772. *
  773. * Called during an address space update transaction,
  774. * for a section of the address space that is new in this address space
  775. * space since the last transaction.
  776. *
  777. * @listener: The #MemoryListener.
  778. * @section: The new #MemoryRegionSection.
  779. */
  780. void (*region_add)(MemoryListener *listener, MemoryRegionSection *section);
  781. /**
  782. * @region_del:
  783. *
  784. * Called during an address space update transaction,
  785. * for a section of the address space that has disappeared in the address
  786. * space since the last transaction.
  787. *
  788. * @listener: The #MemoryListener.
  789. * @section: The old #MemoryRegionSection.
  790. */
  791. void (*region_del)(MemoryListener *listener, MemoryRegionSection *section);
  792. /**
  793. * @region_nop:
  794. *
  795. * Called during an address space update transaction,
  796. * for a section of the address space that is in the same place in the address
  797. * space as in the last transaction.
  798. *
  799. * @listener: The #MemoryListener.
  800. * @section: The #MemoryRegionSection.
  801. */
  802. void (*region_nop)(MemoryListener *listener, MemoryRegionSection *section);
  803. /**
  804. * @log_start:
  805. *
  806. * Called during an address space update transaction, after
  807. * one of #MemoryListener.region_add(), #MemoryListener.region_del() or
  808. * #MemoryListener.region_nop(), if dirty memory logging clients have
  809. * become active since the last transaction.
  810. *
  811. * @listener: The #MemoryListener.
  812. * @section: The #MemoryRegionSection.
  813. * @old: A bitmap of dirty memory logging clients that were active in
  814. * the previous transaction.
  815. * @new: A bitmap of dirty memory logging clients that are active in
  816. * the current transaction.
  817. */
  818. void (*log_start)(MemoryListener *listener, MemoryRegionSection *section,
  819. int old_val, int new_val);
  820. /**
  821. * @log_stop:
  822. *
  823. * Called during an address space update transaction, after
  824. * one of #MemoryListener.region_add(), #MemoryListener.region_del() or
  825. * #MemoryListener.region_nop() and possibly after
  826. * #MemoryListener.log_start(), if dirty memory logging clients have
  827. * become inactive since the last transaction.
  828. *
  829. * @listener: The #MemoryListener.
  830. * @section: The #MemoryRegionSection.
  831. * @old: A bitmap of dirty memory logging clients that were active in
  832. * the previous transaction.
  833. * @new: A bitmap of dirty memory logging clients that are active in
  834. * the current transaction.
  835. */
  836. void (*log_stop)(MemoryListener *listener, MemoryRegionSection *section,
  837. int old_val, int new_val);
  838. /**
  839. * @log_sync:
  840. *
  841. * Called by memory_region_snapshot_and_clear_dirty() and
  842. * memory_global_dirty_log_sync(), before accessing QEMU's "official"
  843. * copy of the dirty memory bitmap for a #MemoryRegionSection.
  844. *
  845. * @listener: The #MemoryListener.
  846. * @section: The #MemoryRegionSection.
  847. */
  848. void (*log_sync)(MemoryListener *listener, MemoryRegionSection *section);
  849. /**
  850. * @log_sync_global:
  851. *
  852. * This is the global version of @log_sync when the listener does
  853. * not have a way to synchronize the log with finer granularity.
  854. * When the listener registers with @log_sync_global defined, then
  855. * its @log_sync must be NULL. Vice versa.
  856. *
  857. * @listener: The #MemoryListener.
  858. * @last_stage: The last stage to synchronize the log during migration.
  859. * The caller should guarantee that the synchronization with true for
  860. * @last_stage is triggered for once after all VCPUs have been stopped.
  861. */
  862. void (*log_sync_global)(MemoryListener *listener, bool last_stage);
  863. /**
  864. * @log_clear:
  865. *
  866. * Called before reading the dirty memory bitmap for a
  867. * #MemoryRegionSection.
  868. *
  869. * @listener: The #MemoryListener.
  870. * @section: The #MemoryRegionSection.
  871. */
  872. void (*log_clear)(MemoryListener *listener, MemoryRegionSection *section);
  873. /**
  874. * @log_global_start:
  875. *
  876. * Called by memory_global_dirty_log_start(), which
  877. * enables the %DIRTY_LOG_MIGRATION client on all memory regions in
  878. * the address space. #MemoryListener.log_global_start() is also
  879. * called when a #MemoryListener is added, if global dirty logging is
  880. * active at that time.
  881. *
  882. * @listener: The #MemoryListener.
  883. * @errp: pointer to Error*, to store an error if it happens.
  884. *
  885. * Return: true on success, else false setting @errp with error.
  886. */
  887. bool (*log_global_start)(MemoryListener *listener, Error **errp);
  888. /**
  889. * @log_global_stop:
  890. *
  891. * Called by memory_global_dirty_log_stop(), which
  892. * disables the %DIRTY_LOG_MIGRATION client on all memory regions in
  893. * the address space.
  894. *
  895. * @listener: The #MemoryListener.
  896. */
  897. void (*log_global_stop)(MemoryListener *listener);
  898. /**
  899. * @log_global_after_sync:
  900. *
  901. * Called after reading the dirty memory bitmap
  902. * for any #MemoryRegionSection.
  903. *
  904. * @listener: The #MemoryListener.
  905. */
  906. void (*log_global_after_sync)(MemoryListener *listener);
  907. /**
  908. * @eventfd_add:
  909. *
  910. * Called during an address space update transaction,
  911. * for a section of the address space that has had a new ioeventfd
  912. * registration since the last transaction.
  913. *
  914. * @listener: The #MemoryListener.
  915. * @section: The new #MemoryRegionSection.
  916. * @match_data: The @match_data parameter for the new ioeventfd.
  917. * @data: The @data parameter for the new ioeventfd.
  918. * @e: The #EventNotifier parameter for the new ioeventfd.
  919. */
  920. void (*eventfd_add)(MemoryListener *listener, MemoryRegionSection *section,
  921. bool match_data, uint64_t data, EventNotifier *e);
  922. /**
  923. * @eventfd_del:
  924. *
  925. * Called during an address space update transaction,
  926. * for a section of the address space that has dropped an ioeventfd
  927. * registration since the last transaction.
  928. *
  929. * @listener: The #MemoryListener.
  930. * @section: The new #MemoryRegionSection.
  931. * @match_data: The @match_data parameter for the dropped ioeventfd.
  932. * @data: The @data parameter for the dropped ioeventfd.
  933. * @e: The #EventNotifier parameter for the dropped ioeventfd.
  934. */
  935. void (*eventfd_del)(MemoryListener *listener, MemoryRegionSection *section,
  936. bool match_data, uint64_t data, EventNotifier *e);
  937. /**
  938. * @coalesced_io_add:
  939. *
  940. * Called during an address space update transaction,
  941. * for a section of the address space that has had a new coalesced
  942. * MMIO range registration since the last transaction.
  943. *
  944. * @listener: The #MemoryListener.
  945. * @section: The new #MemoryRegionSection.
  946. * @addr: The starting address for the coalesced MMIO range.
  947. * @len: The length of the coalesced MMIO range.
  948. */
  949. void (*coalesced_io_add)(MemoryListener *listener, MemoryRegionSection *section,
  950. hwaddr addr, hwaddr len);
  951. /**
  952. * @coalesced_io_del:
  953. *
  954. * Called during an address space update transaction,
  955. * for a section of the address space that has dropped a coalesced
  956. * MMIO range since the last transaction.
  957. *
  958. * @listener: The #MemoryListener.
  959. * @section: The new #MemoryRegionSection.
  960. * @addr: The starting address for the coalesced MMIO range.
  961. * @len: The length of the coalesced MMIO range.
  962. */
  963. void (*coalesced_io_del)(MemoryListener *listener, MemoryRegionSection *section,
  964. hwaddr addr, hwaddr len);
  965. /**
  966. * @priority:
  967. *
  968. * Govern the order in which memory listeners are invoked. Lower priorities
  969. * are invoked earlier for "add" or "start" callbacks, and later for "delete"
  970. * or "stop" callbacks.
  971. */
  972. unsigned priority;
  973. /**
  974. * @name:
  975. *
  976. * Name of the listener. It can be used in contexts where we'd like to
  977. * identify one memory listener with the rest.
  978. */
  979. const char *name;
  980. /* private: */
  981. AddressSpace *address_space;
  982. QTAILQ_ENTRY(MemoryListener) link;
  983. QTAILQ_ENTRY(MemoryListener) link_as;
  984. };
  985. typedef struct AddressSpaceMapClient {
  986. QEMUBH *bh;
  987. QLIST_ENTRY(AddressSpaceMapClient) link;
  988. } AddressSpaceMapClient;
  989. #define DEFAULT_MAX_BOUNCE_BUFFER_SIZE (4096)
  990. /**
  991. * struct AddressSpace: describes a mapping of addresses to #MemoryRegion objects
  992. */
  993. struct AddressSpace {
  994. /* private: */
  995. struct rcu_head rcu;
  996. char *name;
  997. MemoryRegion *root;
  998. /* Accessed via RCU. */
  999. struct FlatView *current_map;
  1000. int ioeventfd_nb;
  1001. int ioeventfd_notifiers;
  1002. struct MemoryRegionIoeventfd *ioeventfds;
  1003. QTAILQ_HEAD(, MemoryListener) listeners;
  1004. QTAILQ_ENTRY(AddressSpace) address_spaces_link;
  1005. /*
  1006. * Maximum DMA bounce buffer size used for indirect memory map requests.
  1007. * This limits the total size of bounce buffer allocations made for
  1008. * DMA requests to indirect memory regions within this AddressSpace. DMA
  1009. * requests that exceed the limit (e.g. due to overly large requested size
  1010. * or concurrent DMA requests having claimed too much buffer space) will be
  1011. * rejected and left to the caller to handle.
  1012. */
  1013. size_t max_bounce_buffer_size;
  1014. /* Total size of bounce buffers currently allocated, atomically accessed */
  1015. size_t bounce_buffer_size;
  1016. /* List of callbacks to invoke when buffers free up */
  1017. QemuMutex map_client_list_lock;
  1018. QLIST_HEAD(, AddressSpaceMapClient) map_client_list;
  1019. };
  1020. typedef struct AddressSpaceDispatch AddressSpaceDispatch;
  1021. typedef struct FlatRange FlatRange;
  1022. /* Flattened global view of current active memory hierarchy. Kept in sorted
  1023. * order.
  1024. */
  1025. struct FlatView {
  1026. struct rcu_head rcu;
  1027. unsigned ref;
  1028. FlatRange *ranges;
  1029. unsigned nr;
  1030. unsigned nr_allocated;
  1031. struct AddressSpaceDispatch *dispatch;
  1032. MemoryRegion *root;
  1033. };
  1034. static inline FlatView *address_space_to_flatview(AddressSpace *as)
  1035. {
  1036. return qatomic_rcu_read(&as->current_map);
  1037. }
  1038. /**
  1039. * typedef flatview_cb: callback for flatview_for_each_range()
  1040. *
  1041. * @start: start address of the range within the FlatView
  1042. * @len: length of the range in bytes
  1043. * @mr: MemoryRegion covering this range
  1044. * @offset_in_region: offset of the first byte of the range within @mr
  1045. * @opaque: data pointer passed to flatview_for_each_range()
  1046. *
  1047. * Returns: true to stop the iteration, false to keep going.
  1048. */
  1049. typedef bool (*flatview_cb)(Int128 start,
  1050. Int128 len,
  1051. const MemoryRegion *mr,
  1052. hwaddr offset_in_region,
  1053. void *opaque);
  1054. /**
  1055. * flatview_for_each_range: Iterate through a FlatView
  1056. * @fv: the FlatView to iterate through
  1057. * @cb: function to call for each range
  1058. * @opaque: opaque data pointer to pass to @cb
  1059. *
  1060. * A FlatView is made up of a list of non-overlapping ranges, each of
  1061. * which is a slice of a MemoryRegion. This function iterates through
  1062. * each range in @fv, calling @cb. The callback function can terminate
  1063. * iteration early by returning 'true'.
  1064. */
  1065. void flatview_for_each_range(FlatView *fv, flatview_cb cb, void *opaque);
  1066. static inline bool MemoryRegionSection_eq(MemoryRegionSection *a,
  1067. MemoryRegionSection *b)
  1068. {
  1069. return a->mr == b->mr &&
  1070. a->fv == b->fv &&
  1071. a->offset_within_region == b->offset_within_region &&
  1072. a->offset_within_address_space == b->offset_within_address_space &&
  1073. int128_eq(a->size, b->size) &&
  1074. a->readonly == b->readonly &&
  1075. a->nonvolatile == b->nonvolatile;
  1076. }
  1077. /**
  1078. * memory_region_section_new_copy: Copy a memory region section
  1079. *
  1080. * Allocate memory for a new copy, copy the memory region section, and
  1081. * properly take a reference on all relevant members.
  1082. *
  1083. * @s: the #MemoryRegionSection to copy
  1084. */
  1085. MemoryRegionSection *memory_region_section_new_copy(MemoryRegionSection *s);
  1086. /**
  1087. * memory_region_section_free_copy: Free a copied memory region section
  1088. *
  1089. * Free a copy of a memory section created via memory_region_section_new_copy().
  1090. * properly dropping references on all relevant members.
  1091. *
  1092. * @s: the #MemoryRegionSection to copy
  1093. */
  1094. void memory_region_section_free_copy(MemoryRegionSection *s);
  1095. /**
  1096. * memory_region_init: Initialize a memory region
  1097. *
  1098. * The region typically acts as a container for other memory regions. Use
  1099. * memory_region_add_subregion() to add subregions.
  1100. *
  1101. * @mr: the #MemoryRegion to be initialized
  1102. * @owner: the object that tracks the region's reference count
  1103. * @name: used for debugging; not visible to the user or ABI
  1104. * @size: size of the region; any subregions beyond this size will be clipped
  1105. */
  1106. void memory_region_init(MemoryRegion *mr,
  1107. Object *owner,
  1108. const char *name,
  1109. uint64_t size);
  1110. /**
  1111. * memory_region_ref: Add 1 to a memory region's reference count
  1112. *
  1113. * Whenever memory regions are accessed outside the BQL, they need to be
  1114. * preserved against hot-unplug. MemoryRegions actually do not have their
  1115. * own reference count; they piggyback on a QOM object, their "owner".
  1116. * This function adds a reference to the owner.
  1117. *
  1118. * All MemoryRegions must have an owner if they can disappear, even if the
  1119. * device they belong to operates exclusively under the BQL. This is because
  1120. * the region could be returned at any time by memory_region_find, and this
  1121. * is usually under guest control.
  1122. *
  1123. * @mr: the #MemoryRegion
  1124. */
  1125. void memory_region_ref(MemoryRegion *mr);
  1126. /**
  1127. * memory_region_unref: Remove 1 to a memory region's reference count
  1128. *
  1129. * Whenever memory regions are accessed outside the BQL, they need to be
  1130. * preserved against hot-unplug. MemoryRegions actually do not have their
  1131. * own reference count; they piggyback on a QOM object, their "owner".
  1132. * This function removes a reference to the owner and possibly destroys it.
  1133. *
  1134. * @mr: the #MemoryRegion
  1135. */
  1136. void memory_region_unref(MemoryRegion *mr);
  1137. /**
  1138. * memory_region_init_io: Initialize an I/O memory region.
  1139. *
  1140. * Accesses into the region will cause the callbacks in @ops to be called.
  1141. * if @size is nonzero, subregions will be clipped to @size.
  1142. *
  1143. * @mr: the #MemoryRegion to be initialized.
  1144. * @owner: the object that tracks the region's reference count
  1145. * @ops: a structure containing read and write callbacks to be used when
  1146. * I/O is performed on the region.
  1147. * @opaque: passed to the read and write callbacks of the @ops structure.
  1148. * @name: used for debugging; not visible to the user or ABI
  1149. * @size: size of the region.
  1150. */
  1151. void memory_region_init_io(MemoryRegion *mr,
  1152. Object *owner,
  1153. const MemoryRegionOps *ops,
  1154. void *opaque,
  1155. const char *name,
  1156. uint64_t size);
  1157. /**
  1158. * memory_region_init_ram_nomigrate: Initialize RAM memory region. Accesses
  1159. * into the region will modify memory
  1160. * directly.
  1161. *
  1162. * @mr: the #MemoryRegion to be initialized.
  1163. * @owner: the object that tracks the region's reference count
  1164. * @name: Region name, becomes part of RAMBlock name used in migration stream
  1165. * must be unique within any device
  1166. * @size: size of the region.
  1167. * @errp: pointer to Error*, to store an error if it happens.
  1168. *
  1169. * Note that this function does not do anything to cause the data in the
  1170. * RAM memory region to be migrated; that is the responsibility of the caller.
  1171. *
  1172. * Return: true on success, else false setting @errp with error.
  1173. */
  1174. bool memory_region_init_ram_nomigrate(MemoryRegion *mr,
  1175. Object *owner,
  1176. const char *name,
  1177. uint64_t size,
  1178. Error **errp);
  1179. /**
  1180. * memory_region_init_ram_flags_nomigrate: Initialize RAM memory region.
  1181. * Accesses into the region will
  1182. * modify memory directly.
  1183. *
  1184. * @mr: the #MemoryRegion to be initialized.
  1185. * @owner: the object that tracks the region's reference count
  1186. * @name: Region name, becomes part of RAMBlock name used in migration stream
  1187. * must be unique within any device
  1188. * @size: size of the region.
  1189. * @ram_flags: RamBlock flags. Supported flags: RAM_SHARED, RAM_NORESERVE,
  1190. * RAM_GUEST_MEMFD.
  1191. * @errp: pointer to Error*, to store an error if it happens.
  1192. *
  1193. * Note that this function does not do anything to cause the data in the
  1194. * RAM memory region to be migrated; that is the responsibility of the caller.
  1195. *
  1196. * Return: true on success, else false setting @errp with error.
  1197. */
  1198. bool memory_region_init_ram_flags_nomigrate(MemoryRegion *mr,
  1199. Object *owner,
  1200. const char *name,
  1201. uint64_t size,
  1202. uint32_t ram_flags,
  1203. Error **errp);
  1204. /**
  1205. * memory_region_init_resizeable_ram: Initialize memory region with resizable
  1206. * RAM. Accesses into the region will
  1207. * modify memory directly. Only an initial
  1208. * portion of this RAM is actually used.
  1209. * Changing the size while migrating
  1210. * can result in the migration being
  1211. * canceled.
  1212. *
  1213. * @mr: the #MemoryRegion to be initialized.
  1214. * @owner: the object that tracks the region's reference count
  1215. * @name: Region name, becomes part of RAMBlock name used in migration stream
  1216. * must be unique within any device
  1217. * @size: used size of the region.
  1218. * @max_size: max size of the region.
  1219. * @resized: callback to notify owner about used size change.
  1220. * @errp: pointer to Error*, to store an error if it happens.
  1221. *
  1222. * Note that this function does not do anything to cause the data in the
  1223. * RAM memory region to be migrated; that is the responsibility of the caller.
  1224. *
  1225. * Return: true on success, else false setting @errp with error.
  1226. */
  1227. bool memory_region_init_resizeable_ram(MemoryRegion *mr,
  1228. Object *owner,
  1229. const char *name,
  1230. uint64_t size,
  1231. uint64_t max_size,
  1232. void (*resized)(const char*,
  1233. uint64_t length,
  1234. void *host),
  1235. Error **errp);
  1236. #ifdef CONFIG_POSIX
  1237. /**
  1238. * memory_region_init_ram_from_file: Initialize RAM memory region with a
  1239. * mmap-ed backend.
  1240. *
  1241. * @mr: the #MemoryRegion to be initialized.
  1242. * @owner: the object that tracks the region's reference count
  1243. * @name: Region name, becomes part of RAMBlock name used in migration stream
  1244. * must be unique within any device
  1245. * @size: size of the region.
  1246. * @align: alignment of the region base address; if 0, the default alignment
  1247. * (getpagesize()) will be used.
  1248. * @ram_flags: RamBlock flags. Supported flags: RAM_SHARED, RAM_PMEM,
  1249. * RAM_NORESERVE, RAM_PROTECTED, RAM_NAMED_FILE, RAM_READONLY,
  1250. * RAM_READONLY_FD, RAM_GUEST_MEMFD
  1251. * @path: the path in which to allocate the RAM.
  1252. * @offset: offset within the file referenced by path
  1253. * @errp: pointer to Error*, to store an error if it happens.
  1254. *
  1255. * Note that this function does not do anything to cause the data in the
  1256. * RAM memory region to be migrated; that is the responsibility of the caller.
  1257. *
  1258. * Return: true on success, else false setting @errp with error.
  1259. */
  1260. bool memory_region_init_ram_from_file(MemoryRegion *mr,
  1261. Object *owner,
  1262. const char *name,
  1263. uint64_t size,
  1264. uint64_t align,
  1265. uint32_t ram_flags,
  1266. const char *path,
  1267. ram_addr_t offset,
  1268. Error **errp);
  1269. /**
  1270. * memory_region_init_ram_from_fd: Initialize RAM memory region with a
  1271. * mmap-ed backend.
  1272. *
  1273. * @mr: the #MemoryRegion to be initialized.
  1274. * @owner: the object that tracks the region's reference count
  1275. * @name: the name of the region.
  1276. * @size: size of the region.
  1277. * @ram_flags: RamBlock flags. Supported flags: RAM_SHARED, RAM_PMEM,
  1278. * RAM_NORESERVE, RAM_PROTECTED, RAM_NAMED_FILE, RAM_READONLY,
  1279. * RAM_READONLY_FD, RAM_GUEST_MEMFD
  1280. * @fd: the fd to mmap.
  1281. * @offset: offset within the file referenced by fd
  1282. * @errp: pointer to Error*, to store an error if it happens.
  1283. *
  1284. * Note that this function does not do anything to cause the data in the
  1285. * RAM memory region to be migrated; that is the responsibility of the caller.
  1286. *
  1287. * Return: true on success, else false setting @errp with error.
  1288. */
  1289. bool memory_region_init_ram_from_fd(MemoryRegion *mr,
  1290. Object *owner,
  1291. const char *name,
  1292. uint64_t size,
  1293. uint32_t ram_flags,
  1294. int fd,
  1295. ram_addr_t offset,
  1296. Error **errp);
  1297. #endif
  1298. /**
  1299. * memory_region_init_ram_ptr: Initialize RAM memory region from a
  1300. * user-provided pointer. Accesses into the
  1301. * region will modify memory directly.
  1302. *
  1303. * @mr: the #MemoryRegion to be initialized.
  1304. * @owner: the object that tracks the region's reference count
  1305. * @name: Region name, becomes part of RAMBlock name used in migration stream
  1306. * must be unique within any device
  1307. * @size: size of the region.
  1308. * @ptr: memory to be mapped; must contain at least @size bytes.
  1309. *
  1310. * Note that this function does not do anything to cause the data in the
  1311. * RAM memory region to be migrated; that is the responsibility of the caller.
  1312. */
  1313. void memory_region_init_ram_ptr(MemoryRegion *mr,
  1314. Object *owner,
  1315. const char *name,
  1316. uint64_t size,
  1317. void *ptr);
  1318. /**
  1319. * memory_region_init_ram_device_ptr: Initialize RAM device memory region from
  1320. * a user-provided pointer.
  1321. *
  1322. * A RAM device represents a mapping to a physical device, such as to a PCI
  1323. * MMIO BAR of an vfio-pci assigned device. The memory region may be mapped
  1324. * into the VM address space and access to the region will modify memory
  1325. * directly. However, the memory region should not be included in a memory
  1326. * dump (device may not be enabled/mapped at the time of the dump), and
  1327. * operations incompatible with manipulating MMIO should be avoided. Replaces
  1328. * skip_dump flag.
  1329. *
  1330. * @mr: the #MemoryRegion to be initialized.
  1331. * @owner: the object that tracks the region's reference count
  1332. * @name: the name of the region.
  1333. * @size: size of the region.
  1334. * @ptr: memory to be mapped; must contain at least @size bytes.
  1335. *
  1336. * Note that this function does not do anything to cause the data in the
  1337. * RAM memory region to be migrated; that is the responsibility of the caller.
  1338. * (For RAM device memory regions, migrating the contents rarely makes sense.)
  1339. */
  1340. void memory_region_init_ram_device_ptr(MemoryRegion *mr,
  1341. Object *owner,
  1342. const char *name,
  1343. uint64_t size,
  1344. void *ptr);
  1345. /**
  1346. * memory_region_init_alias: Initialize a memory region that aliases all or a
  1347. * part of another memory region.
  1348. *
  1349. * @mr: the #MemoryRegion to be initialized.
  1350. * @owner: the object that tracks the region's reference count
  1351. * @name: used for debugging; not visible to the user or ABI
  1352. * @orig: the region to be referenced; @mr will be equivalent to
  1353. * @orig between @offset and @offset + @size - 1.
  1354. * @offset: start of the section in @orig to be referenced.
  1355. * @size: size of the region.
  1356. */
  1357. void memory_region_init_alias(MemoryRegion *mr,
  1358. Object *owner,
  1359. const char *name,
  1360. MemoryRegion *orig,
  1361. hwaddr offset,
  1362. uint64_t size);
  1363. /**
  1364. * memory_region_init_rom_nomigrate: Initialize a ROM memory region.
  1365. *
  1366. * This has the same effect as calling memory_region_init_ram_nomigrate()
  1367. * and then marking the resulting region read-only with
  1368. * memory_region_set_readonly().
  1369. *
  1370. * Note that this function does not do anything to cause the data in the
  1371. * RAM side of the memory region to be migrated; that is the responsibility
  1372. * of the caller.
  1373. *
  1374. * @mr: the #MemoryRegion to be initialized.
  1375. * @owner: the object that tracks the region's reference count
  1376. * @name: Region name, becomes part of RAMBlock name used in migration stream
  1377. * must be unique within any device
  1378. * @size: size of the region.
  1379. * @errp: pointer to Error*, to store an error if it happens.
  1380. *
  1381. * Return: true on success, else false setting @errp with error.
  1382. */
  1383. bool memory_region_init_rom_nomigrate(MemoryRegion *mr,
  1384. Object *owner,
  1385. const char *name,
  1386. uint64_t size,
  1387. Error **errp);
  1388. /**
  1389. * memory_region_init_rom_device_nomigrate: Initialize a ROM memory region.
  1390. * Writes are handled via callbacks.
  1391. *
  1392. * Note that this function does not do anything to cause the data in the
  1393. * RAM side of the memory region to be migrated; that is the responsibility
  1394. * of the caller.
  1395. *
  1396. * @mr: the #MemoryRegion to be initialized.
  1397. * @owner: the object that tracks the region's reference count
  1398. * @ops: callbacks for write access handling (must not be NULL).
  1399. * @opaque: passed to the read and write callbacks of the @ops structure.
  1400. * @name: Region name, becomes part of RAMBlock name used in migration stream
  1401. * must be unique within any device
  1402. * @size: size of the region.
  1403. * @errp: pointer to Error*, to store an error if it happens.
  1404. *
  1405. * Return: true on success, else false setting @errp with error.
  1406. */
  1407. bool memory_region_init_rom_device_nomigrate(MemoryRegion *mr,
  1408. Object *owner,
  1409. const MemoryRegionOps *ops,
  1410. void *opaque,
  1411. const char *name,
  1412. uint64_t size,
  1413. Error **errp);
  1414. /**
  1415. * memory_region_init_iommu: Initialize a memory region of a custom type
  1416. * that translates addresses
  1417. *
  1418. * An IOMMU region translates addresses and forwards accesses to a target
  1419. * memory region.
  1420. *
  1421. * The IOMMU implementation must define a subclass of TYPE_IOMMU_MEMORY_REGION.
  1422. * @_iommu_mr should be a pointer to enough memory for an instance of
  1423. * that subclass, @instance_size is the size of that subclass, and
  1424. * @mrtypename is its name. This function will initialize @_iommu_mr as an
  1425. * instance of the subclass, and its methods will then be called to handle
  1426. * accesses to the memory region. See the documentation of
  1427. * #IOMMUMemoryRegionClass for further details.
  1428. *
  1429. * @_iommu_mr: the #IOMMUMemoryRegion to be initialized
  1430. * @instance_size: the IOMMUMemoryRegion subclass instance size
  1431. * @mrtypename: the type name of the #IOMMUMemoryRegion
  1432. * @owner: the object that tracks the region's reference count
  1433. * @name: used for debugging; not visible to the user or ABI
  1434. * @size: size of the region.
  1435. */
  1436. void memory_region_init_iommu(void *_iommu_mr,
  1437. size_t instance_size,
  1438. const char *mrtypename,
  1439. Object *owner,
  1440. const char *name,
  1441. uint64_t size);
  1442. /**
  1443. * memory_region_init_ram - Initialize RAM memory region. Accesses into the
  1444. * region will modify memory directly.
  1445. *
  1446. * @mr: the #MemoryRegion to be initialized
  1447. * @owner: the object that tracks the region's reference count (must be
  1448. * TYPE_DEVICE or a subclass of TYPE_DEVICE, or NULL)
  1449. * @name: name of the memory region
  1450. * @size: size of the region in bytes
  1451. * @errp: pointer to Error*, to store an error if it happens.
  1452. *
  1453. * This function allocates RAM for a board model or device, and
  1454. * arranges for it to be migrated (by calling vmstate_register_ram()
  1455. * if @owner is a DeviceState, or vmstate_register_ram_global() if
  1456. * @owner is NULL).
  1457. *
  1458. * TODO: Currently we restrict @owner to being either NULL (for
  1459. * global RAM regions with no owner) or devices, so that we can
  1460. * give the RAM block a unique name for migration purposes.
  1461. * We should lift this restriction and allow arbitrary Objects.
  1462. * If you pass a non-NULL non-device @owner then we will assert.
  1463. *
  1464. * Return: true on success, else false setting @errp with error.
  1465. */
  1466. bool memory_region_init_ram(MemoryRegion *mr,
  1467. Object *owner,
  1468. const char *name,
  1469. uint64_t size,
  1470. Error **errp);
  1471. bool memory_region_init_ram_guest_memfd(MemoryRegion *mr,
  1472. Object *owner,
  1473. const char *name,
  1474. uint64_t size,
  1475. Error **errp);
  1476. /**
  1477. * memory_region_init_rom: Initialize a ROM memory region.
  1478. *
  1479. * This has the same effect as calling memory_region_init_ram()
  1480. * and then marking the resulting region read-only with
  1481. * memory_region_set_readonly(). This includes arranging for the
  1482. * contents to be migrated.
  1483. *
  1484. * TODO: Currently we restrict @owner to being either NULL (for
  1485. * global RAM regions with no owner) or devices, so that we can
  1486. * give the RAM block a unique name for migration purposes.
  1487. * We should lift this restriction and allow arbitrary Objects.
  1488. * If you pass a non-NULL non-device @owner then we will assert.
  1489. *
  1490. * @mr: the #MemoryRegion to be initialized.
  1491. * @owner: the object that tracks the region's reference count
  1492. * @name: Region name, becomes part of RAMBlock name used in migration stream
  1493. * must be unique within any device
  1494. * @size: size of the region.
  1495. * @errp: pointer to Error*, to store an error if it happens.
  1496. *
  1497. * Return: true on success, else false setting @errp with error.
  1498. */
  1499. bool memory_region_init_rom(MemoryRegion *mr,
  1500. Object *owner,
  1501. const char *name,
  1502. uint64_t size,
  1503. Error **errp);
  1504. /**
  1505. * memory_region_init_rom_device: Initialize a ROM memory region.
  1506. * Writes are handled via callbacks.
  1507. *
  1508. * This function initializes a memory region backed by RAM for reads
  1509. * and callbacks for writes, and arranges for the RAM backing to
  1510. * be migrated (by calling vmstate_register_ram()
  1511. * if @owner is a DeviceState, or vmstate_register_ram_global() if
  1512. * @owner is NULL).
  1513. *
  1514. * TODO: Currently we restrict @owner to being either NULL (for
  1515. * global RAM regions with no owner) or devices, so that we can
  1516. * give the RAM block a unique name for migration purposes.
  1517. * We should lift this restriction and allow arbitrary Objects.
  1518. * If you pass a non-NULL non-device @owner then we will assert.
  1519. *
  1520. * @mr: the #MemoryRegion to be initialized.
  1521. * @owner: the object that tracks the region's reference count
  1522. * @ops: callbacks for write access handling (must not be NULL).
  1523. * @opaque: passed to the read and write callbacks of the @ops structure.
  1524. * @name: Region name, becomes part of RAMBlock name used in migration stream
  1525. * must be unique within any device
  1526. * @size: size of the region.
  1527. * @errp: pointer to Error*, to store an error if it happens.
  1528. *
  1529. * Return: true on success, else false setting @errp with error.
  1530. */
  1531. bool memory_region_init_rom_device(MemoryRegion *mr,
  1532. Object *owner,
  1533. const MemoryRegionOps *ops,
  1534. void *opaque,
  1535. const char *name,
  1536. uint64_t size,
  1537. Error **errp);
  1538. /**
  1539. * memory_region_owner: get a memory region's owner.
  1540. *
  1541. * @mr: the memory region being queried.
  1542. */
  1543. Object *memory_region_owner(MemoryRegion *mr);
  1544. /**
  1545. * memory_region_size: get a memory region's size.
  1546. *
  1547. * @mr: the memory region being queried.
  1548. */
  1549. uint64_t memory_region_size(MemoryRegion *mr);
  1550. /**
  1551. * memory_region_is_ram: check whether a memory region is random access
  1552. *
  1553. * Returns %true if a memory region is random access.
  1554. *
  1555. * @mr: the memory region being queried
  1556. */
  1557. static inline bool memory_region_is_ram(MemoryRegion *mr)
  1558. {
  1559. return mr->ram;
  1560. }
  1561. /**
  1562. * memory_region_is_ram_device: check whether a memory region is a ram device
  1563. *
  1564. * Returns %true if a memory region is a device backed ram region
  1565. *
  1566. * @mr: the memory region being queried
  1567. */
  1568. bool memory_region_is_ram_device(MemoryRegion *mr);
  1569. /**
  1570. * memory_region_is_romd: check whether a memory region is in ROMD mode
  1571. *
  1572. * Returns %true if a memory region is a ROM device and currently set to allow
  1573. * direct reads.
  1574. *
  1575. * @mr: the memory region being queried
  1576. */
  1577. static inline bool memory_region_is_romd(MemoryRegion *mr)
  1578. {
  1579. return mr->rom_device && mr->romd_mode;
  1580. }
  1581. /**
  1582. * memory_region_is_protected: check whether a memory region is protected
  1583. *
  1584. * Returns %true if a memory region is protected RAM and cannot be accessed
  1585. * via standard mechanisms, e.g. DMA.
  1586. *
  1587. * @mr: the memory region being queried
  1588. */
  1589. bool memory_region_is_protected(MemoryRegion *mr);
  1590. /**
  1591. * memory_region_has_guest_memfd: check whether a memory region has guest_memfd
  1592. * associated
  1593. *
  1594. * Returns %true if a memory region's ram_block has valid guest_memfd assigned.
  1595. *
  1596. * @mr: the memory region being queried
  1597. */
  1598. bool memory_region_has_guest_memfd(MemoryRegion *mr);
  1599. /**
  1600. * memory_region_get_iommu: check whether a memory region is an iommu
  1601. *
  1602. * Returns pointer to IOMMUMemoryRegion if a memory region is an iommu,
  1603. * otherwise NULL.
  1604. *
  1605. * @mr: the memory region being queried
  1606. */
  1607. static inline IOMMUMemoryRegion *memory_region_get_iommu(MemoryRegion *mr)
  1608. {
  1609. if (mr->alias) {
  1610. return memory_region_get_iommu(mr->alias);
  1611. }
  1612. if (mr->is_iommu) {
  1613. return (IOMMUMemoryRegion *) mr;
  1614. }
  1615. return NULL;
  1616. }
  1617. /**
  1618. * memory_region_get_iommu_class_nocheck: returns iommu memory region class
  1619. * if an iommu or NULL if not
  1620. *
  1621. * Returns pointer to IOMMUMemoryRegionClass if a memory region is an iommu,
  1622. * otherwise NULL. This is fast path avoiding QOM checking, use with caution.
  1623. *
  1624. * @iommu_mr: the memory region being queried
  1625. */
  1626. static inline IOMMUMemoryRegionClass *memory_region_get_iommu_class_nocheck(
  1627. IOMMUMemoryRegion *iommu_mr)
  1628. {
  1629. return (IOMMUMemoryRegionClass *) (((Object *)iommu_mr)->class);
  1630. }
  1631. #define memory_region_is_iommu(mr) (memory_region_get_iommu(mr) != NULL)
  1632. /**
  1633. * memory_region_iommu_get_min_page_size: get minimum supported page size
  1634. * for an iommu
  1635. *
  1636. * Returns minimum supported page size for an iommu.
  1637. *
  1638. * @iommu_mr: the memory region being queried
  1639. */
  1640. uint64_t memory_region_iommu_get_min_page_size(IOMMUMemoryRegion *iommu_mr);
  1641. /**
  1642. * memory_region_notify_iommu: notify a change in an IOMMU translation entry.
  1643. *
  1644. * Note: for any IOMMU implementation, an in-place mapping change
  1645. * should be notified with an UNMAP followed by a MAP.
  1646. *
  1647. * @iommu_mr: the memory region that was changed
  1648. * @iommu_idx: the IOMMU index for the translation table which has changed
  1649. * @event: TLB event with the new entry in the IOMMU translation table.
  1650. * The entry replaces all old entries for the same virtual I/O address
  1651. * range.
  1652. */
  1653. void memory_region_notify_iommu(IOMMUMemoryRegion *iommu_mr,
  1654. int iommu_idx,
  1655. const IOMMUTLBEvent event);
  1656. /**
  1657. * memory_region_notify_iommu_one: notify a change in an IOMMU translation
  1658. * entry to a single notifier
  1659. *
  1660. * This works just like memory_region_notify_iommu(), but it only
  1661. * notifies a specific notifier, not all of them.
  1662. *
  1663. * @notifier: the notifier to be notified
  1664. * @event: TLB event with the new entry in the IOMMU translation table.
  1665. * The entry replaces all old entries for the same virtual I/O address
  1666. * range.
  1667. */
  1668. void memory_region_notify_iommu_one(IOMMUNotifier *notifier,
  1669. const IOMMUTLBEvent *event);
  1670. /**
  1671. * memory_region_unmap_iommu_notifier_range: notify a unmap for an IOMMU
  1672. * translation that covers the
  1673. * range of a notifier
  1674. *
  1675. * @notifier: the notifier to be notified
  1676. */
  1677. void memory_region_unmap_iommu_notifier_range(IOMMUNotifier *notifier);
  1678. /**
  1679. * memory_region_register_iommu_notifier: register a notifier for changes to
  1680. * IOMMU translation entries.
  1681. *
  1682. * Returns 0 on success, or a negative errno otherwise. In particular,
  1683. * -EINVAL indicates that at least one of the attributes of the notifier
  1684. * is not supported (flag/range) by the IOMMU memory region. In case of error
  1685. * the error object must be created.
  1686. *
  1687. * @mr: the memory region to observe
  1688. * @n: the IOMMUNotifier to be added; the notify callback receives a
  1689. * pointer to an #IOMMUTLBEntry as the opaque value; the pointer
  1690. * ceases to be valid on exit from the notifier.
  1691. * @errp: pointer to Error*, to store an error if it happens.
  1692. */
  1693. int memory_region_register_iommu_notifier(MemoryRegion *mr,
  1694. IOMMUNotifier *n, Error **errp);
  1695. /**
  1696. * memory_region_iommu_replay: replay existing IOMMU translations to
  1697. * a notifier with the minimum page granularity returned by
  1698. * mr->iommu_ops->get_page_size().
  1699. *
  1700. * Note: this is not related to record-and-replay functionality.
  1701. *
  1702. * @iommu_mr: the memory region to observe
  1703. * @n: the notifier to which to replay iommu mappings
  1704. */
  1705. void memory_region_iommu_replay(IOMMUMemoryRegion *iommu_mr, IOMMUNotifier *n);
  1706. /**
  1707. * memory_region_unregister_iommu_notifier: unregister a notifier for
  1708. * changes to IOMMU translation entries.
  1709. *
  1710. * @mr: the memory region which was observed and for which notify_stopped()
  1711. * needs to be called
  1712. * @n: the notifier to be removed.
  1713. */
  1714. void memory_region_unregister_iommu_notifier(MemoryRegion *mr,
  1715. IOMMUNotifier *n);
  1716. /**
  1717. * memory_region_iommu_get_attr: return an IOMMU attr if get_attr() is
  1718. * defined on the IOMMU.
  1719. *
  1720. * Returns 0 on success, or a negative errno otherwise. In particular,
  1721. * -EINVAL indicates that the IOMMU does not support the requested
  1722. * attribute.
  1723. *
  1724. * @iommu_mr: the memory region
  1725. * @attr: the requested attribute
  1726. * @data: a pointer to the requested attribute data
  1727. */
  1728. int memory_region_iommu_get_attr(IOMMUMemoryRegion *iommu_mr,
  1729. enum IOMMUMemoryRegionAttr attr,
  1730. void *data);
  1731. /**
  1732. * memory_region_iommu_attrs_to_index: return the IOMMU index to
  1733. * use for translations with the given memory transaction attributes.
  1734. *
  1735. * @iommu_mr: the memory region
  1736. * @attrs: the memory transaction attributes
  1737. */
  1738. int memory_region_iommu_attrs_to_index(IOMMUMemoryRegion *iommu_mr,
  1739. MemTxAttrs attrs);
  1740. /**
  1741. * memory_region_iommu_num_indexes: return the total number of IOMMU
  1742. * indexes that this IOMMU supports.
  1743. *
  1744. * @iommu_mr: the memory region
  1745. */
  1746. int memory_region_iommu_num_indexes(IOMMUMemoryRegion *iommu_mr);
  1747. /**
  1748. * memory_region_name: get a memory region's name
  1749. *
  1750. * Returns the string that was used to initialize the memory region.
  1751. *
  1752. * @mr: the memory region being queried
  1753. */
  1754. const char *memory_region_name(const MemoryRegion *mr);
  1755. /**
  1756. * memory_region_is_logging: return whether a memory region is logging writes
  1757. *
  1758. * Returns %true if the memory region is logging writes for the given client
  1759. *
  1760. * @mr: the memory region being queried
  1761. * @client: the client being queried
  1762. */
  1763. bool memory_region_is_logging(MemoryRegion *mr, uint8_t client);
  1764. /**
  1765. * memory_region_get_dirty_log_mask: return the clients for which a
  1766. * memory region is logging writes.
  1767. *
  1768. * Returns a bitmap of clients, in which the DIRTY_MEMORY_* constants
  1769. * are the bit indices.
  1770. *
  1771. * @mr: the memory region being queried
  1772. */
  1773. uint8_t memory_region_get_dirty_log_mask(MemoryRegion *mr);
  1774. /**
  1775. * memory_region_is_rom: check whether a memory region is ROM
  1776. *
  1777. * Returns %true if a memory region is read-only memory.
  1778. *
  1779. * @mr: the memory region being queried
  1780. */
  1781. static inline bool memory_region_is_rom(MemoryRegion *mr)
  1782. {
  1783. return mr->ram && mr->readonly;
  1784. }
  1785. /**
  1786. * memory_region_is_nonvolatile: check whether a memory region is non-volatile
  1787. *
  1788. * Returns %true is a memory region is non-volatile memory.
  1789. *
  1790. * @mr: the memory region being queried
  1791. */
  1792. static inline bool memory_region_is_nonvolatile(MemoryRegion *mr)
  1793. {
  1794. return mr->nonvolatile;
  1795. }
  1796. /**
  1797. * memory_region_get_fd: Get a file descriptor backing a RAM memory region.
  1798. *
  1799. * Returns a file descriptor backing a file-based RAM memory region,
  1800. * or -1 if the region is not a file-based RAM memory region.
  1801. *
  1802. * @mr: the RAM or alias memory region being queried.
  1803. */
  1804. int memory_region_get_fd(MemoryRegion *mr);
  1805. /**
  1806. * memory_region_from_host: Convert a pointer into a RAM memory region
  1807. * and an offset within it.
  1808. *
  1809. * Given a host pointer inside a RAM memory region (created with
  1810. * memory_region_init_ram() or memory_region_init_ram_ptr()), return
  1811. * the MemoryRegion and the offset within it.
  1812. *
  1813. * Use with care; by the time this function returns, the returned pointer is
  1814. * not protected by RCU anymore. If the caller is not within an RCU critical
  1815. * section and does not hold the BQL, it must have other means of
  1816. * protecting the pointer, such as a reference to the region that includes
  1817. * the incoming ram_addr_t.
  1818. *
  1819. * @ptr: the host pointer to be converted
  1820. * @offset: the offset within memory region
  1821. */
  1822. MemoryRegion *memory_region_from_host(void *ptr, ram_addr_t *offset);
  1823. /**
  1824. * memory_region_get_ram_ptr: Get a pointer into a RAM memory region.
  1825. *
  1826. * Returns a host pointer to a RAM memory region (created with
  1827. * memory_region_init_ram() or memory_region_init_ram_ptr()).
  1828. *
  1829. * Use with care; by the time this function returns, the returned pointer is
  1830. * not protected by RCU anymore. If the caller is not within an RCU critical
  1831. * section and does not hold the BQL, it must have other means of
  1832. * protecting the pointer, such as a reference to the region that includes
  1833. * the incoming ram_addr_t.
  1834. *
  1835. * @mr: the memory region being queried.
  1836. */
  1837. void *memory_region_get_ram_ptr(MemoryRegion *mr);
  1838. /* memory_region_ram_resize: Resize a RAM region.
  1839. *
  1840. * Resizing RAM while migrating can result in the migration being canceled.
  1841. * Care has to be taken if the guest might have already detected the memory.
  1842. *
  1843. * @mr: a memory region created with @memory_region_init_resizeable_ram.
  1844. * @newsize: the new size the region
  1845. * @errp: pointer to Error*, to store an error if it happens.
  1846. */
  1847. void memory_region_ram_resize(MemoryRegion *mr, ram_addr_t newsize,
  1848. Error **errp);
  1849. /**
  1850. * memory_region_msync: Synchronize selected address range of
  1851. * a memory mapped region
  1852. *
  1853. * @mr: the memory region to be msync
  1854. * @addr: the initial address of the range to be sync
  1855. * @size: the size of the range to be sync
  1856. */
  1857. void memory_region_msync(MemoryRegion *mr, hwaddr addr, hwaddr size);
  1858. /**
  1859. * memory_region_writeback: Trigger cache writeback for
  1860. * selected address range
  1861. *
  1862. * @mr: the memory region to be updated
  1863. * @addr: the initial address of the range to be written back
  1864. * @size: the size of the range to be written back
  1865. */
  1866. void memory_region_writeback(MemoryRegion *mr, hwaddr addr, hwaddr size);
  1867. /**
  1868. * memory_region_set_log: Turn dirty logging on or off for a region.
  1869. *
  1870. * Turns dirty logging on or off for a specified client (display, migration).
  1871. * Only meaningful for RAM regions.
  1872. *
  1873. * @mr: the memory region being updated.
  1874. * @log: whether dirty logging is to be enabled or disabled.
  1875. * @client: the user of the logging information; %DIRTY_MEMORY_VGA only.
  1876. */
  1877. void memory_region_set_log(MemoryRegion *mr, bool log, unsigned client);
  1878. /**
  1879. * memory_region_set_dirty: Mark a range of bytes as dirty in a memory region.
  1880. *
  1881. * Marks a range of bytes as dirty, after it has been dirtied outside
  1882. * guest code.
  1883. *
  1884. * @mr: the memory region being dirtied.
  1885. * @addr: the address (relative to the start of the region) being dirtied.
  1886. * @size: size of the range being dirtied.
  1887. */
  1888. void memory_region_set_dirty(MemoryRegion *mr, hwaddr addr,
  1889. hwaddr size);
  1890. /**
  1891. * memory_region_clear_dirty_bitmap - clear dirty bitmap for memory range
  1892. *
  1893. * This function is called when the caller wants to clear the remote
  1894. * dirty bitmap of a memory range within the memory region. This can
  1895. * be used by e.g. KVM to manually clear dirty log when
  1896. * KVM_CAP_MANUAL_DIRTY_LOG_PROTECT is declared support by the host
  1897. * kernel.
  1898. *
  1899. * @mr: the memory region to clear the dirty log upon
  1900. * @start: start address offset within the memory region
  1901. * @len: length of the memory region to clear dirty bitmap
  1902. */
  1903. void memory_region_clear_dirty_bitmap(MemoryRegion *mr, hwaddr start,
  1904. hwaddr len);
  1905. /**
  1906. * memory_region_snapshot_and_clear_dirty: Get a snapshot of the dirty
  1907. * bitmap and clear it.
  1908. *
  1909. * Creates a snapshot of the dirty bitmap, clears the dirty bitmap and
  1910. * returns the snapshot. The snapshot can then be used to query dirty
  1911. * status, using memory_region_snapshot_get_dirty. Snapshotting allows
  1912. * querying the same page multiple times, which is especially useful for
  1913. * display updates where the scanlines often are not page aligned.
  1914. *
  1915. * The dirty bitmap region which gets copied into the snapshot (and
  1916. * cleared afterwards) can be larger than requested. The boundaries
  1917. * are rounded up/down so complete bitmap longs (covering 64 pages on
  1918. * 64bit hosts) can be copied over into the bitmap snapshot. Which
  1919. * isn't a problem for display updates as the extra pages are outside
  1920. * the visible area, and in case the visible area changes a full
  1921. * display redraw is due anyway. Should other use cases for this
  1922. * function emerge we might have to revisit this implementation
  1923. * detail.
  1924. *
  1925. * Use g_free to release DirtyBitmapSnapshot.
  1926. *
  1927. * @mr: the memory region being queried.
  1928. * @addr: the address (relative to the start of the region) being queried.
  1929. * @size: the size of the range being queried.
  1930. * @client: the user of the logging information; typically %DIRTY_MEMORY_VGA.
  1931. */
  1932. DirtyBitmapSnapshot *memory_region_snapshot_and_clear_dirty(MemoryRegion *mr,
  1933. hwaddr addr,
  1934. hwaddr size,
  1935. unsigned client);
  1936. /**
  1937. * memory_region_snapshot_get_dirty: Check whether a range of bytes is dirty
  1938. * in the specified dirty bitmap snapshot.
  1939. *
  1940. * @mr: the memory region being queried.
  1941. * @snap: the dirty bitmap snapshot
  1942. * @addr: the address (relative to the start of the region) being queried.
  1943. * @size: the size of the range being queried.
  1944. */
  1945. bool memory_region_snapshot_get_dirty(MemoryRegion *mr,
  1946. DirtyBitmapSnapshot *snap,
  1947. hwaddr addr, hwaddr size);
  1948. /**
  1949. * memory_region_reset_dirty: Mark a range of pages as clean, for a specified
  1950. * client.
  1951. *
  1952. * Marks a range of pages as no longer dirty.
  1953. *
  1954. * @mr: the region being updated.
  1955. * @addr: the start of the subrange being cleaned.
  1956. * @size: the size of the subrange being cleaned.
  1957. * @client: the user of the logging information; %DIRTY_MEMORY_MIGRATION or
  1958. * %DIRTY_MEMORY_VGA.
  1959. */
  1960. void memory_region_reset_dirty(MemoryRegion *mr, hwaddr addr,
  1961. hwaddr size, unsigned client);
  1962. /**
  1963. * memory_region_flush_rom_device: Mark a range of pages dirty and invalidate
  1964. * TBs (for self-modifying code).
  1965. *
  1966. * The MemoryRegionOps->write() callback of a ROM device must use this function
  1967. * to mark byte ranges that have been modified internally, such as by directly
  1968. * accessing the memory returned by memory_region_get_ram_ptr().
  1969. *
  1970. * This function marks the range dirty and invalidates TBs so that TCG can
  1971. * detect self-modifying code.
  1972. *
  1973. * @mr: the region being flushed.
  1974. * @addr: the start, relative to the start of the region, of the range being
  1975. * flushed.
  1976. * @size: the size, in bytes, of the range being flushed.
  1977. */
  1978. void memory_region_flush_rom_device(MemoryRegion *mr, hwaddr addr, hwaddr size);
  1979. /**
  1980. * memory_region_set_readonly: Turn a memory region read-only (or read-write)
  1981. *
  1982. * Allows a memory region to be marked as read-only (turning it into a ROM).
  1983. * only useful on RAM regions.
  1984. *
  1985. * @mr: the region being updated.
  1986. * @readonly: whether the region is to be ROM or RAM.
  1987. */
  1988. void memory_region_set_readonly(MemoryRegion *mr, bool readonly);
  1989. /**
  1990. * memory_region_set_nonvolatile: Turn a memory region non-volatile
  1991. *
  1992. * Allows a memory region to be marked as non-volatile.
  1993. * only useful on RAM regions.
  1994. *
  1995. * @mr: the region being updated.
  1996. * @nonvolatile: whether the region is to be non-volatile.
  1997. */
  1998. void memory_region_set_nonvolatile(MemoryRegion *mr, bool nonvolatile);
  1999. /**
  2000. * memory_region_rom_device_set_romd: enable/disable ROMD mode
  2001. *
  2002. * Allows a ROM device (initialized with memory_region_init_rom_device() to
  2003. * set to ROMD mode (default) or MMIO mode. When it is in ROMD mode, the
  2004. * device is mapped to guest memory and satisfies read access directly.
  2005. * When in MMIO mode, reads are forwarded to the #MemoryRegion.read function.
  2006. * Writes are always handled by the #MemoryRegion.write function.
  2007. *
  2008. * @mr: the memory region to be updated
  2009. * @romd_mode: %true to put the region into ROMD mode
  2010. */
  2011. void memory_region_rom_device_set_romd(MemoryRegion *mr, bool romd_mode);
  2012. /**
  2013. * memory_region_set_coalescing: Enable memory coalescing for the region.
  2014. *
  2015. * Enabled writes to a region to be queued for later processing. MMIO ->write
  2016. * callbacks may be delayed until a non-coalesced MMIO is issued.
  2017. * Only useful for IO regions. Roughly similar to write-combining hardware.
  2018. *
  2019. * @mr: the memory region to be write coalesced
  2020. */
  2021. void memory_region_set_coalescing(MemoryRegion *mr);
  2022. /**
  2023. * memory_region_add_coalescing: Enable memory coalescing for a sub-range of
  2024. * a region.
  2025. *
  2026. * Like memory_region_set_coalescing(), but works on a sub-range of a region.
  2027. * Multiple calls can be issued coalesced disjoint ranges.
  2028. *
  2029. * @mr: the memory region to be updated.
  2030. * @offset: the start of the range within the region to be coalesced.
  2031. * @size: the size of the subrange to be coalesced.
  2032. */
  2033. void memory_region_add_coalescing(MemoryRegion *mr,
  2034. hwaddr offset,
  2035. uint64_t size);
  2036. /**
  2037. * memory_region_clear_coalescing: Disable MMIO coalescing for the region.
  2038. *
  2039. * Disables any coalescing caused by memory_region_set_coalescing() or
  2040. * memory_region_add_coalescing(). Roughly equivalent to uncacheble memory
  2041. * hardware.
  2042. *
  2043. * @mr: the memory region to be updated.
  2044. */
  2045. void memory_region_clear_coalescing(MemoryRegion *mr);
  2046. /**
  2047. * memory_region_set_flush_coalesced: Enforce memory coalescing flush before
  2048. * accesses.
  2049. *
  2050. * Ensure that pending coalesced MMIO request are flushed before the memory
  2051. * region is accessed. This property is automatically enabled for all regions
  2052. * passed to memory_region_set_coalescing() and memory_region_add_coalescing().
  2053. *
  2054. * @mr: the memory region to be updated.
  2055. */
  2056. void memory_region_set_flush_coalesced(MemoryRegion *mr);
  2057. /**
  2058. * memory_region_clear_flush_coalesced: Disable memory coalescing flush before
  2059. * accesses.
  2060. *
  2061. * Clear the automatic coalesced MMIO flushing enabled via
  2062. * memory_region_set_flush_coalesced. Note that this service has no effect on
  2063. * memory regions that have MMIO coalescing enabled for themselves. For them,
  2064. * automatic flushing will stop once coalescing is disabled.
  2065. *
  2066. * @mr: the memory region to be updated.
  2067. */
  2068. void memory_region_clear_flush_coalesced(MemoryRegion *mr);
  2069. /**
  2070. * memory_region_add_eventfd: Request an eventfd to be triggered when a word
  2071. * is written to a location.
  2072. *
  2073. * Marks a word in an IO region (initialized with memory_region_init_io())
  2074. * as a trigger for an eventfd event. The I/O callback will not be called.
  2075. * The caller must be prepared to handle failure (that is, take the required
  2076. * action if the callback _is_ called).
  2077. *
  2078. * @mr: the memory region being updated.
  2079. * @addr: the address within @mr that is to be monitored
  2080. * @size: the size of the access to trigger the eventfd
  2081. * @match_data: whether to match against @data, instead of just @addr
  2082. * @data: the data to match against the guest write
  2083. * @e: event notifier to be triggered when @addr, @size, and @data all match.
  2084. **/
  2085. void memory_region_add_eventfd(MemoryRegion *mr,
  2086. hwaddr addr,
  2087. unsigned size,
  2088. bool match_data,
  2089. uint64_t data,
  2090. EventNotifier *e);
  2091. /**
  2092. * memory_region_del_eventfd: Cancel an eventfd.
  2093. *
  2094. * Cancels an eventfd trigger requested by a previous
  2095. * memory_region_add_eventfd() call.
  2096. *
  2097. * @mr: the memory region being updated.
  2098. * @addr: the address within @mr that is to be monitored
  2099. * @size: the size of the access to trigger the eventfd
  2100. * @match_data: whether to match against @data, instead of just @addr
  2101. * @data: the data to match against the guest write
  2102. * @e: event notifier to be triggered when @addr, @size, and @data all match.
  2103. */
  2104. void memory_region_del_eventfd(MemoryRegion *mr,
  2105. hwaddr addr,
  2106. unsigned size,
  2107. bool match_data,
  2108. uint64_t data,
  2109. EventNotifier *e);
  2110. /**
  2111. * memory_region_add_subregion: Add a subregion to a container.
  2112. *
  2113. * Adds a subregion at @offset. The subregion may not overlap with other
  2114. * subregions (except for those explicitly marked as overlapping). A region
  2115. * may only be added once as a subregion (unless removed with
  2116. * memory_region_del_subregion()); use memory_region_init_alias() if you
  2117. * want a region to be a subregion in multiple locations.
  2118. *
  2119. * @mr: the region to contain the new subregion; must be a container
  2120. * initialized with memory_region_init().
  2121. * @offset: the offset relative to @mr where @subregion is added.
  2122. * @subregion: the subregion to be added.
  2123. */
  2124. void memory_region_add_subregion(MemoryRegion *mr,
  2125. hwaddr offset,
  2126. MemoryRegion *subregion);
  2127. /**
  2128. * memory_region_add_subregion_overlap: Add a subregion to a container
  2129. * with overlap.
  2130. *
  2131. * Adds a subregion at @offset. The subregion may overlap with other
  2132. * subregions. Conflicts are resolved by having a higher @priority hide a
  2133. * lower @priority. Subregions without priority are taken as @priority 0.
  2134. * A region may only be added once as a subregion (unless removed with
  2135. * memory_region_del_subregion()); use memory_region_init_alias() if you
  2136. * want a region to be a subregion in multiple locations.
  2137. *
  2138. * @mr: the region to contain the new subregion; must be a container
  2139. * initialized with memory_region_init().
  2140. * @offset: the offset relative to @mr where @subregion is added.
  2141. * @subregion: the subregion to be added.
  2142. * @priority: used for resolving overlaps; highest priority wins.
  2143. */
  2144. void memory_region_add_subregion_overlap(MemoryRegion *mr,
  2145. hwaddr offset,
  2146. MemoryRegion *subregion,
  2147. int priority);
  2148. /**
  2149. * memory_region_get_ram_addr: Get the ram address associated with a memory
  2150. * region
  2151. *
  2152. * @mr: the region to be queried
  2153. */
  2154. ram_addr_t memory_region_get_ram_addr(MemoryRegion *mr);
  2155. uint64_t memory_region_get_alignment(const MemoryRegion *mr);
  2156. /**
  2157. * memory_region_del_subregion: Remove a subregion.
  2158. *
  2159. * Removes a subregion from its container.
  2160. *
  2161. * @mr: the container to be updated.
  2162. * @subregion: the region being removed; must be a current subregion of @mr.
  2163. */
  2164. void memory_region_del_subregion(MemoryRegion *mr,
  2165. MemoryRegion *subregion);
  2166. /*
  2167. * memory_region_set_enabled: dynamically enable or disable a region
  2168. *
  2169. * Enables or disables a memory region. A disabled memory region
  2170. * ignores all accesses to itself and its subregions. It does not
  2171. * obscure sibling subregions with lower priority - it simply behaves as
  2172. * if it was removed from the hierarchy.
  2173. *
  2174. * Regions default to being enabled.
  2175. *
  2176. * @mr: the region to be updated
  2177. * @enabled: whether to enable or disable the region
  2178. */
  2179. void memory_region_set_enabled(MemoryRegion *mr, bool enabled);
  2180. /*
  2181. * memory_region_set_address: dynamically update the address of a region
  2182. *
  2183. * Dynamically updates the address of a region, relative to its container.
  2184. * May be used on regions are currently part of a memory hierarchy.
  2185. *
  2186. * @mr: the region to be updated
  2187. * @addr: new address, relative to container region
  2188. */
  2189. void memory_region_set_address(MemoryRegion *mr, hwaddr addr);
  2190. /*
  2191. * memory_region_set_size: dynamically update the size of a region.
  2192. *
  2193. * Dynamically updates the size of a region.
  2194. *
  2195. * @mr: the region to be updated
  2196. * @size: used size of the region.
  2197. */
  2198. void memory_region_set_size(MemoryRegion *mr, uint64_t size);
  2199. /*
  2200. * memory_region_set_alias_offset: dynamically update a memory alias's offset
  2201. *
  2202. * Dynamically updates the offset into the target region that an alias points
  2203. * to, as if the fourth argument to memory_region_init_alias() has changed.
  2204. *
  2205. * @mr: the #MemoryRegion to be updated; should be an alias.
  2206. * @offset: the new offset into the target memory region
  2207. */
  2208. void memory_region_set_alias_offset(MemoryRegion *mr,
  2209. hwaddr offset);
  2210. /*
  2211. * memory_region_set_unmergeable: Set a memory region unmergeable
  2212. *
  2213. * Mark a memory region unmergeable, resulting in the memory region (or
  2214. * everything contained in a memory region container) not getting merged when
  2215. * simplifying the address space and notifying memory listeners. Consequently,
  2216. * memory listeners will never get notified about ranges that are larger than
  2217. * the original memory regions.
  2218. *
  2219. * This is primarily useful when multiple aliases to a RAM memory region are
  2220. * mapped into a memory region container, and updates (e.g., enable/disable or
  2221. * map/unmap) of individual memory region aliases are not supposed to affect
  2222. * other memory regions in the same container.
  2223. *
  2224. * @mr: the #MemoryRegion to be updated
  2225. * @unmergeable: whether to mark the #MemoryRegion unmergeable
  2226. */
  2227. void memory_region_set_unmergeable(MemoryRegion *mr, bool unmergeable);
  2228. /**
  2229. * memory_region_present: checks if an address relative to a @container
  2230. * translates into #MemoryRegion within @container
  2231. *
  2232. * Answer whether a #MemoryRegion within @container covers the address
  2233. * @addr.
  2234. *
  2235. * @container: a #MemoryRegion within which @addr is a relative address
  2236. * @addr: the area within @container to be searched
  2237. */
  2238. bool memory_region_present(MemoryRegion *container, hwaddr addr);
  2239. /**
  2240. * memory_region_is_mapped: returns true if #MemoryRegion is mapped
  2241. * into another memory region, which does not necessarily imply that it is
  2242. * mapped into an address space.
  2243. *
  2244. * @mr: a #MemoryRegion which should be checked if it's mapped
  2245. */
  2246. bool memory_region_is_mapped(MemoryRegion *mr);
  2247. /**
  2248. * memory_region_get_ram_discard_manager: get the #RamDiscardManager for a
  2249. * #MemoryRegion
  2250. *
  2251. * The #RamDiscardManager cannot change while a memory region is mapped.
  2252. *
  2253. * @mr: the #MemoryRegion
  2254. */
  2255. RamDiscardManager *memory_region_get_ram_discard_manager(MemoryRegion *mr);
  2256. /**
  2257. * memory_region_has_ram_discard_manager: check whether a #MemoryRegion has a
  2258. * #RamDiscardManager assigned
  2259. *
  2260. * @mr: the #MemoryRegion
  2261. */
  2262. static inline bool memory_region_has_ram_discard_manager(MemoryRegion *mr)
  2263. {
  2264. return !!memory_region_get_ram_discard_manager(mr);
  2265. }
  2266. /**
  2267. * memory_region_set_ram_discard_manager: set the #RamDiscardManager for a
  2268. * #MemoryRegion
  2269. *
  2270. * This function must not be called for a mapped #MemoryRegion, a #MemoryRegion
  2271. * that does not cover RAM, or a #MemoryRegion that already has a
  2272. * #RamDiscardManager assigned.
  2273. *
  2274. * @mr: the #MemoryRegion
  2275. * @rdm: #RamDiscardManager to set
  2276. */
  2277. void memory_region_set_ram_discard_manager(MemoryRegion *mr,
  2278. RamDiscardManager *rdm);
  2279. /**
  2280. * memory_region_find: translate an address/size relative to a
  2281. * MemoryRegion into a #MemoryRegionSection.
  2282. *
  2283. * Locates the first #MemoryRegion within @mr that overlaps the range
  2284. * given by @addr and @size.
  2285. *
  2286. * Returns a #MemoryRegionSection that describes a contiguous overlap.
  2287. * It will have the following characteristics:
  2288. * - @size = 0 iff no overlap was found
  2289. * - @mr is non-%NULL iff an overlap was found
  2290. *
  2291. * Remember that in the return value the @offset_within_region is
  2292. * relative to the returned region (in the .@mr field), not to the
  2293. * @mr argument.
  2294. *
  2295. * Similarly, the .@offset_within_address_space is relative to the
  2296. * address space that contains both regions, the passed and the
  2297. * returned one. However, in the special case where the @mr argument
  2298. * has no container (and thus is the root of the address space), the
  2299. * following will hold:
  2300. * - @offset_within_address_space >= @addr
  2301. * - @offset_within_address_space + .@size <= @addr + @size
  2302. *
  2303. * @mr: a MemoryRegion within which @addr is a relative address
  2304. * @addr: start of the area within @as to be searched
  2305. * @size: size of the area to be searched
  2306. */
  2307. MemoryRegionSection memory_region_find(MemoryRegion *mr,
  2308. hwaddr addr, uint64_t size);
  2309. /**
  2310. * memory_global_dirty_log_sync: synchronize the dirty log for all memory
  2311. *
  2312. * Synchronizes the dirty page log for all address spaces.
  2313. *
  2314. * @last_stage: whether this is the last stage of live migration
  2315. */
  2316. void memory_global_dirty_log_sync(bool last_stage);
  2317. /**
  2318. * memory_global_after_dirty_log_sync: synchronize the dirty log for all memory
  2319. *
  2320. * Synchronizes the vCPUs with a thread that is reading the dirty bitmap.
  2321. * This function must be called after the dirty log bitmap is cleared, and
  2322. * before dirty guest memory pages are read. If you are using
  2323. * #DirtyBitmapSnapshot, memory_region_snapshot_and_clear_dirty() takes
  2324. * care of doing this.
  2325. */
  2326. void memory_global_after_dirty_log_sync(void);
  2327. /**
  2328. * memory_region_transaction_begin: Start a transaction.
  2329. *
  2330. * During a transaction, changes will be accumulated and made visible
  2331. * only when the transaction ends (is committed).
  2332. */
  2333. void memory_region_transaction_begin(void);
  2334. /**
  2335. * memory_region_transaction_commit: Commit a transaction and make changes
  2336. * visible to the guest.
  2337. */
  2338. void memory_region_transaction_commit(void);
  2339. /**
  2340. * memory_listener_register: register callbacks to be called when memory
  2341. * sections are mapped or unmapped into an address
  2342. * space
  2343. *
  2344. * @listener: an object containing the callbacks to be called
  2345. * @filter: if non-%NULL, only regions in this address space will be observed
  2346. */
  2347. void memory_listener_register(MemoryListener *listener, AddressSpace *filter);
  2348. /**
  2349. * memory_listener_unregister: undo the effect of memory_listener_register()
  2350. *
  2351. * @listener: an object containing the callbacks to be removed
  2352. */
  2353. void memory_listener_unregister(MemoryListener *listener);
  2354. /**
  2355. * memory_global_dirty_log_start: begin dirty logging for all regions
  2356. *
  2357. * @flags: purpose of starting dirty log, migration or dirty rate
  2358. * @errp: pointer to Error*, to store an error if it happens.
  2359. *
  2360. * Return: true on success, else false setting @errp with error.
  2361. */
  2362. bool memory_global_dirty_log_start(unsigned int flags, Error **errp);
  2363. /**
  2364. * memory_global_dirty_log_stop: end dirty logging for all regions
  2365. *
  2366. * @flags: purpose of stopping dirty log, migration or dirty rate
  2367. */
  2368. void memory_global_dirty_log_stop(unsigned int flags);
  2369. void mtree_info(bool flatview, bool dispatch_tree, bool owner, bool disabled);
  2370. bool memory_region_access_valid(MemoryRegion *mr, hwaddr addr,
  2371. unsigned size, bool is_write,
  2372. MemTxAttrs attrs);
  2373. /**
  2374. * memory_region_dispatch_read: perform a read directly to the specified
  2375. * MemoryRegion.
  2376. *
  2377. * @mr: #MemoryRegion to access
  2378. * @addr: address within that region
  2379. * @pval: pointer to uint64_t which the data is written to
  2380. * @op: size, sign, and endianness of the memory operation
  2381. * @attrs: memory transaction attributes to use for the access
  2382. */
  2383. MemTxResult memory_region_dispatch_read(MemoryRegion *mr,
  2384. hwaddr addr,
  2385. uint64_t *pval,
  2386. MemOp op,
  2387. MemTxAttrs attrs);
  2388. /**
  2389. * memory_region_dispatch_write: perform a write directly to the specified
  2390. * MemoryRegion.
  2391. *
  2392. * @mr: #MemoryRegion to access
  2393. * @addr: address within that region
  2394. * @data: data to write
  2395. * @op: size, sign, and endianness of the memory operation
  2396. * @attrs: memory transaction attributes to use for the access
  2397. */
  2398. MemTxResult memory_region_dispatch_write(MemoryRegion *mr,
  2399. hwaddr addr,
  2400. uint64_t data,
  2401. MemOp op,
  2402. MemTxAttrs attrs);
  2403. /**
  2404. * address_space_init: initializes an address space
  2405. *
  2406. * @as: an uninitialized #AddressSpace
  2407. * @root: a #MemoryRegion that routes addresses for the address space
  2408. * @name: an address space name. The name is only used for debugging
  2409. * output.
  2410. */
  2411. void address_space_init(AddressSpace *as, MemoryRegion *root, const char *name);
  2412. /**
  2413. * address_space_destroy: destroy an address space
  2414. *
  2415. * Releases all resources associated with an address space. After an address space
  2416. * is destroyed, its root memory region (given by address_space_init()) may be destroyed
  2417. * as well.
  2418. *
  2419. * @as: address space to be destroyed
  2420. */
  2421. void address_space_destroy(AddressSpace *as);
  2422. /**
  2423. * address_space_remove_listeners: unregister all listeners of an address space
  2424. *
  2425. * Removes all callbacks previously registered with memory_listener_register()
  2426. * for @as.
  2427. *
  2428. * @as: an initialized #AddressSpace
  2429. */
  2430. void address_space_remove_listeners(AddressSpace *as);
  2431. /**
  2432. * address_space_rw: read from or write to an address space.
  2433. *
  2434. * Return a MemTxResult indicating whether the operation succeeded
  2435. * or failed (eg unassigned memory, device rejected the transaction,
  2436. * IOMMU fault).
  2437. *
  2438. * @as: #AddressSpace to be accessed
  2439. * @addr: address within that address space
  2440. * @attrs: memory transaction attributes
  2441. * @buf: buffer with the data transferred
  2442. * @len: the number of bytes to read or write
  2443. * @is_write: indicates the transfer direction
  2444. */
  2445. MemTxResult address_space_rw(AddressSpace *as, hwaddr addr,
  2446. MemTxAttrs attrs, void *buf,
  2447. hwaddr len, bool is_write);
  2448. /**
  2449. * address_space_write: write to address space.
  2450. *
  2451. * Return a MemTxResult indicating whether the operation succeeded
  2452. * or failed (eg unassigned memory, device rejected the transaction,
  2453. * IOMMU fault).
  2454. *
  2455. * @as: #AddressSpace to be accessed
  2456. * @addr: address within that address space
  2457. * @attrs: memory transaction attributes
  2458. * @buf: buffer with the data transferred
  2459. * @len: the number of bytes to write
  2460. */
  2461. MemTxResult address_space_write(AddressSpace *as, hwaddr addr,
  2462. MemTxAttrs attrs,
  2463. const void *buf, hwaddr len);
  2464. /**
  2465. * address_space_write_rom: write to address space, including ROM.
  2466. *
  2467. * This function writes to the specified address space, but will
  2468. * write data to both ROM and RAM. This is used for non-guest
  2469. * writes like writes from the gdb debug stub or initial loading
  2470. * of ROM contents.
  2471. *
  2472. * Note that portions of the write which attempt to write data to
  2473. * a device will be silently ignored -- only real RAM and ROM will
  2474. * be written to.
  2475. *
  2476. * Return a MemTxResult indicating whether the operation succeeded
  2477. * or failed (eg unassigned memory, device rejected the transaction,
  2478. * IOMMU fault).
  2479. *
  2480. * @as: #AddressSpace to be accessed
  2481. * @addr: address within that address space
  2482. * @attrs: memory transaction attributes
  2483. * @buf: buffer with the data transferred
  2484. * @len: the number of bytes to write
  2485. */
  2486. MemTxResult address_space_write_rom(AddressSpace *as, hwaddr addr,
  2487. MemTxAttrs attrs,
  2488. const void *buf, hwaddr len);
  2489. /* address_space_ld*: load from an address space
  2490. * address_space_st*: store to an address space
  2491. *
  2492. * These functions perform a load or store of the byte, word,
  2493. * longword or quad to the specified address within the AddressSpace.
  2494. * The _le suffixed functions treat the data as little endian;
  2495. * _be indicates big endian; no suffix indicates "same endianness
  2496. * as guest CPU".
  2497. *
  2498. * The "guest CPU endianness" accessors are deprecated for use outside
  2499. * target-* code; devices should be CPU-agnostic and use either the LE
  2500. * or the BE accessors.
  2501. *
  2502. * @as #AddressSpace to be accessed
  2503. * @addr: address within that address space
  2504. * @val: data value, for stores
  2505. * @attrs: memory transaction attributes
  2506. * @result: location to write the success/failure of the transaction;
  2507. * if NULL, this information is discarded
  2508. */
  2509. #define SUFFIX
  2510. #define ARG1 as
  2511. #define ARG1_DECL AddressSpace *as
  2512. #include "exec/memory_ldst.h.inc"
  2513. #define SUFFIX
  2514. #define ARG1 as
  2515. #define ARG1_DECL AddressSpace *as
  2516. #include "exec/memory_ldst_phys.h.inc"
  2517. struct MemoryRegionCache {
  2518. uint8_t *ptr;
  2519. hwaddr xlat;
  2520. hwaddr len;
  2521. FlatView *fv;
  2522. MemoryRegionSection mrs;
  2523. bool is_write;
  2524. };
  2525. /* address_space_ld*_cached: load from a cached #MemoryRegion
  2526. * address_space_st*_cached: store into a cached #MemoryRegion
  2527. *
  2528. * These functions perform a load or store of the byte, word,
  2529. * longword or quad to the specified address. The address is
  2530. * a physical address in the AddressSpace, but it must lie within
  2531. * a #MemoryRegion that was mapped with address_space_cache_init.
  2532. *
  2533. * The _le suffixed functions treat the data as little endian;
  2534. * _be indicates big endian; no suffix indicates "same endianness
  2535. * as guest CPU".
  2536. *
  2537. * The "guest CPU endianness" accessors are deprecated for use outside
  2538. * target-* code; devices should be CPU-agnostic and use either the LE
  2539. * or the BE accessors.
  2540. *
  2541. * @cache: previously initialized #MemoryRegionCache to be accessed
  2542. * @addr: address within the address space
  2543. * @val: data value, for stores
  2544. * @attrs: memory transaction attributes
  2545. * @result: location to write the success/failure of the transaction;
  2546. * if NULL, this information is discarded
  2547. */
  2548. #define SUFFIX _cached_slow
  2549. #define ARG1 cache
  2550. #define ARG1_DECL MemoryRegionCache *cache
  2551. #include "exec/memory_ldst.h.inc"
  2552. /* Inline fast path for direct RAM access. */
  2553. static inline uint8_t address_space_ldub_cached(MemoryRegionCache *cache,
  2554. hwaddr addr, MemTxAttrs attrs, MemTxResult *result)
  2555. {
  2556. assert(addr < cache->len);
  2557. if (likely(cache->ptr)) {
  2558. return ldub_p(cache->ptr + addr);
  2559. } else {
  2560. return address_space_ldub_cached_slow(cache, addr, attrs, result);
  2561. }
  2562. }
  2563. static inline void address_space_stb_cached(MemoryRegionCache *cache,
  2564. hwaddr addr, uint8_t val, MemTxAttrs attrs, MemTxResult *result)
  2565. {
  2566. assert(addr < cache->len);
  2567. if (likely(cache->ptr)) {
  2568. stb_p(cache->ptr + addr, val);
  2569. } else {
  2570. address_space_stb_cached_slow(cache, addr, val, attrs, result);
  2571. }
  2572. }
  2573. #define ENDIANNESS _le
  2574. #include "exec/memory_ldst_cached.h.inc"
  2575. #define ENDIANNESS _be
  2576. #include "exec/memory_ldst_cached.h.inc"
  2577. #define SUFFIX _cached
  2578. #define ARG1 cache
  2579. #define ARG1_DECL MemoryRegionCache *cache
  2580. #include "exec/memory_ldst_phys.h.inc"
  2581. /* address_space_cache_init: prepare for repeated access to a physical
  2582. * memory region
  2583. *
  2584. * @cache: #MemoryRegionCache to be filled
  2585. * @as: #AddressSpace to be accessed
  2586. * @addr: address within that address space
  2587. * @len: length of buffer
  2588. * @is_write: indicates the transfer direction
  2589. *
  2590. * Will only work with RAM, and may map a subset of the requested range by
  2591. * returning a value that is less than @len. On failure, return a negative
  2592. * errno value.
  2593. *
  2594. * Because it only works with RAM, this function can be used for
  2595. * read-modify-write operations. In this case, is_write should be %true.
  2596. *
  2597. * Note that addresses passed to the address_space_*_cached functions
  2598. * are relative to @addr.
  2599. */
  2600. int64_t address_space_cache_init(MemoryRegionCache *cache,
  2601. AddressSpace *as,
  2602. hwaddr addr,
  2603. hwaddr len,
  2604. bool is_write);
  2605. /**
  2606. * address_space_cache_init_empty: Initialize empty #MemoryRegionCache
  2607. *
  2608. * @cache: The #MemoryRegionCache to operate on.
  2609. *
  2610. * Initializes #MemoryRegionCache structure without memory region attached.
  2611. * Cache initialized this way can only be safely destroyed, but not used.
  2612. */
  2613. static inline void address_space_cache_init_empty(MemoryRegionCache *cache)
  2614. {
  2615. cache->mrs.mr = NULL;
  2616. /* There is no real need to initialize fv, but it makes Coverity happy. */
  2617. cache->fv = NULL;
  2618. }
  2619. /**
  2620. * address_space_cache_invalidate: complete a write to a #MemoryRegionCache
  2621. *
  2622. * @cache: The #MemoryRegionCache to operate on.
  2623. * @addr: The first physical address that was written, relative to the
  2624. * address that was passed to @address_space_cache_init.
  2625. * @access_len: The number of bytes that were written starting at @addr.
  2626. */
  2627. void address_space_cache_invalidate(MemoryRegionCache *cache,
  2628. hwaddr addr,
  2629. hwaddr access_len);
  2630. /**
  2631. * address_space_cache_destroy: free a #MemoryRegionCache
  2632. *
  2633. * @cache: The #MemoryRegionCache whose memory should be released.
  2634. */
  2635. void address_space_cache_destroy(MemoryRegionCache *cache);
  2636. /* address_space_get_iotlb_entry: translate an address into an IOTLB
  2637. * entry. Should be called from an RCU critical section.
  2638. */
  2639. IOMMUTLBEntry address_space_get_iotlb_entry(AddressSpace *as, hwaddr addr,
  2640. bool is_write, MemTxAttrs attrs);
  2641. /* address_space_translate: translate an address range into an address space
  2642. * into a MemoryRegion and an address range into that section. Should be
  2643. * called from an RCU critical section, to avoid that the last reference
  2644. * to the returned region disappears after address_space_translate returns.
  2645. *
  2646. * @fv: #FlatView to be accessed
  2647. * @addr: address within that address space
  2648. * @xlat: pointer to address within the returned memory region section's
  2649. * #MemoryRegion.
  2650. * @len: pointer to length
  2651. * @is_write: indicates the transfer direction
  2652. * @attrs: memory attributes
  2653. */
  2654. MemoryRegion *flatview_translate(FlatView *fv,
  2655. hwaddr addr, hwaddr *xlat,
  2656. hwaddr *len, bool is_write,
  2657. MemTxAttrs attrs);
  2658. static inline MemoryRegion *address_space_translate(AddressSpace *as,
  2659. hwaddr addr, hwaddr *xlat,
  2660. hwaddr *len, bool is_write,
  2661. MemTxAttrs attrs)
  2662. {
  2663. return flatview_translate(address_space_to_flatview(as),
  2664. addr, xlat, len, is_write, attrs);
  2665. }
  2666. /* address_space_access_valid: check for validity of accessing an address
  2667. * space range
  2668. *
  2669. * Check whether memory is assigned to the given address space range, and
  2670. * access is permitted by any IOMMU regions that are active for the address
  2671. * space.
  2672. *
  2673. * For now, addr and len should be aligned to a page size. This limitation
  2674. * will be lifted in the future.
  2675. *
  2676. * @as: #AddressSpace to be accessed
  2677. * @addr: address within that address space
  2678. * @len: length of the area to be checked
  2679. * @is_write: indicates the transfer direction
  2680. * @attrs: memory attributes
  2681. */
  2682. bool address_space_access_valid(AddressSpace *as, hwaddr addr, hwaddr len,
  2683. bool is_write, MemTxAttrs attrs);
  2684. /* address_space_map: map a physical memory region into a host virtual address
  2685. *
  2686. * May map a subset of the requested range, given by and returned in @plen.
  2687. * May return %NULL and set *@plen to zero(0), if resources needed to perform
  2688. * the mapping are exhausted.
  2689. * Use only for reads OR writes - not for read-modify-write operations.
  2690. * Use address_space_register_map_client() to know when retrying the map
  2691. * operation is likely to succeed.
  2692. *
  2693. * @as: #AddressSpace to be accessed
  2694. * @addr: address within that address space
  2695. * @plen: pointer to length of buffer; updated on return
  2696. * @is_write: indicates the transfer direction
  2697. * @attrs: memory attributes
  2698. */
  2699. void *address_space_map(AddressSpace *as, hwaddr addr,
  2700. hwaddr *plen, bool is_write, MemTxAttrs attrs);
  2701. /* address_space_unmap: Unmaps a memory region previously mapped by address_space_map()
  2702. *
  2703. * Will also mark the memory as dirty if @is_write == %true. @access_len gives
  2704. * the amount of memory that was actually read or written by the caller.
  2705. *
  2706. * @as: #AddressSpace used
  2707. * @buffer: host pointer as returned by address_space_map()
  2708. * @len: buffer length as returned by address_space_map()
  2709. * @access_len: amount of data actually transferred
  2710. * @is_write: indicates the transfer direction
  2711. */
  2712. void address_space_unmap(AddressSpace *as, void *buffer, hwaddr len,
  2713. bool is_write, hwaddr access_len);
  2714. /*
  2715. * address_space_register_map_client: Register a callback to invoke when
  2716. * resources for address_space_map() are available again.
  2717. *
  2718. * address_space_map may fail when there are not enough resources available,
  2719. * such as when bounce buffer memory would exceed the limit. The callback can
  2720. * be used to retry the address_space_map operation. Note that the callback
  2721. * gets automatically removed after firing.
  2722. *
  2723. * @as: #AddressSpace to be accessed
  2724. * @bh: callback to invoke when address_space_map() retry is appropriate
  2725. */
  2726. void address_space_register_map_client(AddressSpace *as, QEMUBH *bh);
  2727. /*
  2728. * address_space_unregister_map_client: Unregister a callback that has
  2729. * previously been registered and not fired yet.
  2730. *
  2731. * @as: #AddressSpace to be accessed
  2732. * @bh: callback to unregister
  2733. */
  2734. void address_space_unregister_map_client(AddressSpace *as, QEMUBH *bh);
  2735. /* Internal functions, part of the implementation of address_space_read. */
  2736. MemTxResult address_space_read_full(AddressSpace *as, hwaddr addr,
  2737. MemTxAttrs attrs, void *buf, hwaddr len);
  2738. MemTxResult flatview_read_continue(FlatView *fv, hwaddr addr,
  2739. MemTxAttrs attrs, void *buf,
  2740. hwaddr len, hwaddr addr1, hwaddr l,
  2741. MemoryRegion *mr);
  2742. void *qemu_map_ram_ptr(RAMBlock *ram_block, ram_addr_t addr);
  2743. /* Internal functions, part of the implementation of address_space_read_cached
  2744. * and address_space_write_cached. */
  2745. MemTxResult address_space_read_cached_slow(MemoryRegionCache *cache,
  2746. hwaddr addr, void *buf, hwaddr len);
  2747. MemTxResult address_space_write_cached_slow(MemoryRegionCache *cache,
  2748. hwaddr addr, const void *buf,
  2749. hwaddr len);
  2750. int memory_access_size(MemoryRegion *mr, unsigned l, hwaddr addr);
  2751. bool prepare_mmio_access(MemoryRegion *mr);
  2752. static inline bool memory_region_supports_direct_access(MemoryRegion *mr)
  2753. {
  2754. /* ROM DEVICE regions only allow direct access if in ROMD mode. */
  2755. if (memory_region_is_romd(mr)) {
  2756. return true;
  2757. }
  2758. if (!memory_region_is_ram(mr)) {
  2759. return false;
  2760. }
  2761. /*
  2762. * RAM DEVICE regions can be accessed directly using memcpy, but it might
  2763. * be MMIO and access using mempy can be wrong (e.g., using instructions not
  2764. * intended for MMIO access). So we treat this as IO.
  2765. */
  2766. return !memory_region_is_ram_device(mr);
  2767. }
  2768. static inline bool memory_access_is_direct(MemoryRegion *mr, bool is_write,
  2769. MemTxAttrs attrs)
  2770. {
  2771. if (!memory_region_supports_direct_access(mr)) {
  2772. return false;
  2773. }
  2774. /* Debug access can write to ROM. */
  2775. if (is_write && !attrs.debug) {
  2776. return !mr->readonly && !mr->rom_device;
  2777. }
  2778. return true;
  2779. }
  2780. /**
  2781. * address_space_read: read from an address space.
  2782. *
  2783. * Return a MemTxResult indicating whether the operation succeeded
  2784. * or failed (eg unassigned memory, device rejected the transaction,
  2785. * IOMMU fault). Called within RCU critical section.
  2786. *
  2787. * @as: #AddressSpace to be accessed
  2788. * @addr: address within that address space
  2789. * @attrs: memory transaction attributes
  2790. * @buf: buffer with the data transferred
  2791. * @len: length of the data transferred
  2792. */
  2793. static inline __attribute__((__always_inline__))
  2794. MemTxResult address_space_read(AddressSpace *as, hwaddr addr,
  2795. MemTxAttrs attrs, void *buf,
  2796. hwaddr len)
  2797. {
  2798. MemTxResult result = MEMTX_OK;
  2799. hwaddr l, addr1;
  2800. void *ptr;
  2801. MemoryRegion *mr;
  2802. FlatView *fv;
  2803. if (__builtin_constant_p(len)) {
  2804. if (len) {
  2805. RCU_READ_LOCK_GUARD();
  2806. fv = address_space_to_flatview(as);
  2807. l = len;
  2808. mr = flatview_translate(fv, addr, &addr1, &l, false, attrs);
  2809. if (len == l && memory_access_is_direct(mr, false, attrs)) {
  2810. ptr = qemu_map_ram_ptr(mr->ram_block, addr1);
  2811. memcpy(buf, ptr, len);
  2812. } else {
  2813. result = flatview_read_continue(fv, addr, attrs, buf, len,
  2814. addr1, l, mr);
  2815. }
  2816. }
  2817. } else {
  2818. result = address_space_read_full(as, addr, attrs, buf, len);
  2819. }
  2820. return result;
  2821. }
  2822. /**
  2823. * address_space_read_cached: read from a cached RAM region
  2824. *
  2825. * @cache: Cached region to be addressed
  2826. * @addr: address relative to the base of the RAM region
  2827. * @buf: buffer with the data transferred
  2828. * @len: length of the data transferred
  2829. */
  2830. static inline MemTxResult
  2831. address_space_read_cached(MemoryRegionCache *cache, hwaddr addr,
  2832. void *buf, hwaddr len)
  2833. {
  2834. assert(addr < cache->len && len <= cache->len - addr);
  2835. fuzz_dma_read_cb(cache->xlat + addr, len, cache->mrs.mr);
  2836. if (likely(cache->ptr)) {
  2837. memcpy(buf, cache->ptr + addr, len);
  2838. return MEMTX_OK;
  2839. } else {
  2840. return address_space_read_cached_slow(cache, addr, buf, len);
  2841. }
  2842. }
  2843. /**
  2844. * address_space_write_cached: write to a cached RAM region
  2845. *
  2846. * @cache: Cached region to be addressed
  2847. * @addr: address relative to the base of the RAM region
  2848. * @buf: buffer with the data transferred
  2849. * @len: length of the data transferred
  2850. */
  2851. static inline MemTxResult
  2852. address_space_write_cached(MemoryRegionCache *cache, hwaddr addr,
  2853. const void *buf, hwaddr len)
  2854. {
  2855. assert(addr < cache->len && len <= cache->len - addr);
  2856. if (likely(cache->ptr)) {
  2857. memcpy(cache->ptr + addr, buf, len);
  2858. return MEMTX_OK;
  2859. } else {
  2860. return address_space_write_cached_slow(cache, addr, buf, len);
  2861. }
  2862. }
  2863. /**
  2864. * address_space_set: Fill address space with a constant byte.
  2865. *
  2866. * Return a MemTxResult indicating whether the operation succeeded
  2867. * or failed (eg unassigned memory, device rejected the transaction,
  2868. * IOMMU fault).
  2869. *
  2870. * @as: #AddressSpace to be accessed
  2871. * @addr: address within that address space
  2872. * @c: constant byte to fill the memory
  2873. * @len: the number of bytes to fill with the constant byte
  2874. * @attrs: memory transaction attributes
  2875. */
  2876. MemTxResult address_space_set(AddressSpace *as, hwaddr addr,
  2877. uint8_t c, hwaddr len, MemTxAttrs attrs);
  2878. #ifdef COMPILING_PER_TARGET
  2879. /* enum device_endian to MemOp. */
  2880. static inline MemOp devend_memop(enum device_endian end)
  2881. {
  2882. QEMU_BUILD_BUG_ON(DEVICE_HOST_ENDIAN != DEVICE_LITTLE_ENDIAN &&
  2883. DEVICE_HOST_ENDIAN != DEVICE_BIG_ENDIAN);
  2884. #if HOST_BIG_ENDIAN != TARGET_BIG_ENDIAN
  2885. /* Swap if non-host endianness or native (target) endianness */
  2886. return (end == DEVICE_HOST_ENDIAN) ? 0 : MO_BSWAP;
  2887. #else
  2888. const int non_host_endianness =
  2889. DEVICE_LITTLE_ENDIAN ^ DEVICE_BIG_ENDIAN ^ DEVICE_HOST_ENDIAN;
  2890. /* In this case, native (target) endianness needs no swap. */
  2891. return (end == non_host_endianness) ? MO_BSWAP : 0;
  2892. #endif
  2893. }
  2894. #endif /* COMPILING_PER_TARGET */
  2895. /*
  2896. * Inhibit technologies that require discarding of pages in RAM blocks, e.g.,
  2897. * to manage the actual amount of memory consumed by the VM (then, the memory
  2898. * provided by RAM blocks might be bigger than the desired memory consumption).
  2899. * This *must* be set if:
  2900. * - Discarding parts of a RAM blocks does not result in the change being
  2901. * reflected in the VM and the pages getting freed.
  2902. * - All memory in RAM blocks is pinned or duplicated, invaldiating any previous
  2903. * discards blindly.
  2904. * - Discarding parts of a RAM blocks will result in integrity issues (e.g.,
  2905. * encrypted VMs).
  2906. * Technologies that only temporarily pin the current working set of a
  2907. * driver are fine, because we don't expect such pages to be discarded
  2908. * (esp. based on guest action like balloon inflation).
  2909. *
  2910. * This is *not* to be used to protect from concurrent discards (esp.,
  2911. * postcopy).
  2912. *
  2913. * Returns 0 if successful. Returns -EBUSY if a technology that relies on
  2914. * discards to work reliably is active.
  2915. */
  2916. int ram_block_discard_disable(bool state);
  2917. /*
  2918. * See ram_block_discard_disable(): only disable uncoordinated discards,
  2919. * keeping coordinated discards (via the RamDiscardManager) enabled.
  2920. */
  2921. int ram_block_uncoordinated_discard_disable(bool state);
  2922. /*
  2923. * Inhibit technologies that disable discarding of pages in RAM blocks.
  2924. *
  2925. * Returns 0 if successful. Returns -EBUSY if discards are already set to
  2926. * broken.
  2927. */
  2928. int ram_block_discard_require(bool state);
  2929. /*
  2930. * See ram_block_discard_require(): only inhibit technologies that disable
  2931. * uncoordinated discarding of pages in RAM blocks, allowing co-existence with
  2932. * technologies that only inhibit uncoordinated discards (via the
  2933. * RamDiscardManager).
  2934. */
  2935. int ram_block_coordinated_discard_require(bool state);
  2936. /*
  2937. * Test if any discarding of memory in ram blocks is disabled.
  2938. */
  2939. bool ram_block_discard_is_disabled(void);
  2940. /*
  2941. * Test if any discarding of memory in ram blocks is required to work reliably.
  2942. */
  2943. bool ram_block_discard_is_required(void);
  2944. void ram_block_add_cpr_blocker(RAMBlock *rb, Error **errp);
  2945. void ram_block_del_cpr_blocker(RAMBlock *rb);
  2946. #endif
  2947. #endif