vhost-user.c 85 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796
  1. /*
  2. * vhost-user
  3. *
  4. * Copyright (c) 2013 Virtual Open Systems Sarl.
  5. *
  6. * This work is licensed under the terms of the GNU GPL, version 2 or later.
  7. * See the COPYING file in the top-level directory.
  8. *
  9. */
  10. #include "qemu/osdep.h"
  11. #include "qapi/error.h"
  12. #include "hw/virtio/vhost.h"
  13. #include "hw/virtio/vhost-user.h"
  14. #include "hw/virtio/vhost-backend.h"
  15. #include "hw/virtio/virtio.h"
  16. #include "hw/virtio/virtio-net.h"
  17. #include "chardev/char-fe.h"
  18. #include "io/channel-socket.h"
  19. #include "sysemu/kvm.h"
  20. #include "qemu/error-report.h"
  21. #include "qemu/main-loop.h"
  22. #include "qemu/sockets.h"
  23. #include "sysemu/runstate.h"
  24. #include "sysemu/cryptodev.h"
  25. #include "migration/migration.h"
  26. #include "migration/postcopy-ram.h"
  27. #include "trace.h"
  28. #include "exec/ramblock.h"
  29. #include <sys/ioctl.h>
  30. #include <sys/socket.h>
  31. #include <sys/un.h>
  32. #include "standard-headers/linux/vhost_types.h"
  33. #ifdef CONFIG_LINUX
  34. #include <linux/userfaultfd.h>
  35. #endif
  36. #define VHOST_MEMORY_BASELINE_NREGIONS 8
  37. #define VHOST_USER_F_PROTOCOL_FEATURES 30
  38. #define VHOST_USER_SLAVE_MAX_FDS 8
  39. /*
  40. * Set maximum number of RAM slots supported to
  41. * the maximum number supported by the target
  42. * hardware plaform.
  43. */
  44. #if defined(TARGET_X86) || defined(TARGET_X86_64) || \
  45. defined(TARGET_ARM) || defined(TARGET_ARM_64)
  46. #include "hw/acpi/acpi.h"
  47. #define VHOST_USER_MAX_RAM_SLOTS ACPI_MAX_RAM_SLOTS
  48. #elif defined(TARGET_PPC) || defined(TARGET_PPC64)
  49. #include "hw/ppc/spapr.h"
  50. #define VHOST_USER_MAX_RAM_SLOTS SPAPR_MAX_RAM_SLOTS
  51. #else
  52. #define VHOST_USER_MAX_RAM_SLOTS 512
  53. #endif
  54. /*
  55. * Maximum size of virtio device config space
  56. */
  57. #define VHOST_USER_MAX_CONFIG_SIZE 256
  58. enum VhostUserProtocolFeature {
  59. VHOST_USER_PROTOCOL_F_MQ = 0,
  60. VHOST_USER_PROTOCOL_F_LOG_SHMFD = 1,
  61. VHOST_USER_PROTOCOL_F_RARP = 2,
  62. VHOST_USER_PROTOCOL_F_REPLY_ACK = 3,
  63. VHOST_USER_PROTOCOL_F_NET_MTU = 4,
  64. VHOST_USER_PROTOCOL_F_SLAVE_REQ = 5,
  65. VHOST_USER_PROTOCOL_F_CROSS_ENDIAN = 6,
  66. VHOST_USER_PROTOCOL_F_CRYPTO_SESSION = 7,
  67. VHOST_USER_PROTOCOL_F_PAGEFAULT = 8,
  68. VHOST_USER_PROTOCOL_F_CONFIG = 9,
  69. VHOST_USER_PROTOCOL_F_SLAVE_SEND_FD = 10,
  70. VHOST_USER_PROTOCOL_F_HOST_NOTIFIER = 11,
  71. VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD = 12,
  72. VHOST_USER_PROTOCOL_F_RESET_DEVICE = 13,
  73. /* Feature 14 reserved for VHOST_USER_PROTOCOL_F_INBAND_NOTIFICATIONS. */
  74. VHOST_USER_PROTOCOL_F_CONFIGURE_MEM_SLOTS = 15,
  75. VHOST_USER_PROTOCOL_F_STATUS = 16,
  76. VHOST_USER_PROTOCOL_F_MAX
  77. };
  78. #define VHOST_USER_PROTOCOL_FEATURE_MASK ((1 << VHOST_USER_PROTOCOL_F_MAX) - 1)
  79. typedef enum VhostUserRequest {
  80. VHOST_USER_NONE = 0,
  81. VHOST_USER_GET_FEATURES = 1,
  82. VHOST_USER_SET_FEATURES = 2,
  83. VHOST_USER_SET_OWNER = 3,
  84. VHOST_USER_RESET_OWNER = 4,
  85. VHOST_USER_SET_MEM_TABLE = 5,
  86. VHOST_USER_SET_LOG_BASE = 6,
  87. VHOST_USER_SET_LOG_FD = 7,
  88. VHOST_USER_SET_VRING_NUM = 8,
  89. VHOST_USER_SET_VRING_ADDR = 9,
  90. VHOST_USER_SET_VRING_BASE = 10,
  91. VHOST_USER_GET_VRING_BASE = 11,
  92. VHOST_USER_SET_VRING_KICK = 12,
  93. VHOST_USER_SET_VRING_CALL = 13,
  94. VHOST_USER_SET_VRING_ERR = 14,
  95. VHOST_USER_GET_PROTOCOL_FEATURES = 15,
  96. VHOST_USER_SET_PROTOCOL_FEATURES = 16,
  97. VHOST_USER_GET_QUEUE_NUM = 17,
  98. VHOST_USER_SET_VRING_ENABLE = 18,
  99. VHOST_USER_SEND_RARP = 19,
  100. VHOST_USER_NET_SET_MTU = 20,
  101. VHOST_USER_SET_SLAVE_REQ_FD = 21,
  102. VHOST_USER_IOTLB_MSG = 22,
  103. VHOST_USER_SET_VRING_ENDIAN = 23,
  104. VHOST_USER_GET_CONFIG = 24,
  105. VHOST_USER_SET_CONFIG = 25,
  106. VHOST_USER_CREATE_CRYPTO_SESSION = 26,
  107. VHOST_USER_CLOSE_CRYPTO_SESSION = 27,
  108. VHOST_USER_POSTCOPY_ADVISE = 28,
  109. VHOST_USER_POSTCOPY_LISTEN = 29,
  110. VHOST_USER_POSTCOPY_END = 30,
  111. VHOST_USER_GET_INFLIGHT_FD = 31,
  112. VHOST_USER_SET_INFLIGHT_FD = 32,
  113. VHOST_USER_GPU_SET_SOCKET = 33,
  114. VHOST_USER_RESET_DEVICE = 34,
  115. /* Message number 35 reserved for VHOST_USER_VRING_KICK. */
  116. VHOST_USER_GET_MAX_MEM_SLOTS = 36,
  117. VHOST_USER_ADD_MEM_REG = 37,
  118. VHOST_USER_REM_MEM_REG = 38,
  119. VHOST_USER_SET_STATUS = 39,
  120. VHOST_USER_GET_STATUS = 40,
  121. VHOST_USER_MAX
  122. } VhostUserRequest;
  123. typedef enum VhostUserSlaveRequest {
  124. VHOST_USER_SLAVE_NONE = 0,
  125. VHOST_USER_SLAVE_IOTLB_MSG = 1,
  126. VHOST_USER_SLAVE_CONFIG_CHANGE_MSG = 2,
  127. VHOST_USER_SLAVE_VRING_HOST_NOTIFIER_MSG = 3,
  128. VHOST_USER_SLAVE_MAX
  129. } VhostUserSlaveRequest;
  130. typedef struct VhostUserMemoryRegion {
  131. uint64_t guest_phys_addr;
  132. uint64_t memory_size;
  133. uint64_t userspace_addr;
  134. uint64_t mmap_offset;
  135. } VhostUserMemoryRegion;
  136. typedef struct VhostUserMemory {
  137. uint32_t nregions;
  138. uint32_t padding;
  139. VhostUserMemoryRegion regions[VHOST_MEMORY_BASELINE_NREGIONS];
  140. } VhostUserMemory;
  141. typedef struct VhostUserMemRegMsg {
  142. uint64_t padding;
  143. VhostUserMemoryRegion region;
  144. } VhostUserMemRegMsg;
  145. typedef struct VhostUserLog {
  146. uint64_t mmap_size;
  147. uint64_t mmap_offset;
  148. } VhostUserLog;
  149. typedef struct VhostUserConfig {
  150. uint32_t offset;
  151. uint32_t size;
  152. uint32_t flags;
  153. uint8_t region[VHOST_USER_MAX_CONFIG_SIZE];
  154. } VhostUserConfig;
  155. #define VHOST_CRYPTO_SYM_HMAC_MAX_KEY_LEN 512
  156. #define VHOST_CRYPTO_SYM_CIPHER_MAX_KEY_LEN 64
  157. typedef struct VhostUserCryptoSession {
  158. /* session id for success, -1 on errors */
  159. int64_t session_id;
  160. CryptoDevBackendSymSessionInfo session_setup_data;
  161. uint8_t key[VHOST_CRYPTO_SYM_CIPHER_MAX_KEY_LEN];
  162. uint8_t auth_key[VHOST_CRYPTO_SYM_HMAC_MAX_KEY_LEN];
  163. } VhostUserCryptoSession;
  164. static VhostUserConfig c __attribute__ ((unused));
  165. #define VHOST_USER_CONFIG_HDR_SIZE (sizeof(c.offset) \
  166. + sizeof(c.size) \
  167. + sizeof(c.flags))
  168. typedef struct VhostUserVringArea {
  169. uint64_t u64;
  170. uint64_t size;
  171. uint64_t offset;
  172. } VhostUserVringArea;
  173. typedef struct VhostUserInflight {
  174. uint64_t mmap_size;
  175. uint64_t mmap_offset;
  176. uint16_t num_queues;
  177. uint16_t queue_size;
  178. } VhostUserInflight;
  179. typedef struct {
  180. VhostUserRequest request;
  181. #define VHOST_USER_VERSION_MASK (0x3)
  182. #define VHOST_USER_REPLY_MASK (0x1 << 2)
  183. #define VHOST_USER_NEED_REPLY_MASK (0x1 << 3)
  184. uint32_t flags;
  185. uint32_t size; /* the following payload size */
  186. } QEMU_PACKED VhostUserHeader;
  187. typedef union {
  188. #define VHOST_USER_VRING_IDX_MASK (0xff)
  189. #define VHOST_USER_VRING_NOFD_MASK (0x1 << 8)
  190. uint64_t u64;
  191. struct vhost_vring_state state;
  192. struct vhost_vring_addr addr;
  193. VhostUserMemory memory;
  194. VhostUserMemRegMsg mem_reg;
  195. VhostUserLog log;
  196. struct vhost_iotlb_msg iotlb;
  197. VhostUserConfig config;
  198. VhostUserCryptoSession session;
  199. VhostUserVringArea area;
  200. VhostUserInflight inflight;
  201. } VhostUserPayload;
  202. typedef struct VhostUserMsg {
  203. VhostUserHeader hdr;
  204. VhostUserPayload payload;
  205. } QEMU_PACKED VhostUserMsg;
  206. static VhostUserMsg m __attribute__ ((unused));
  207. #define VHOST_USER_HDR_SIZE (sizeof(VhostUserHeader))
  208. #define VHOST_USER_PAYLOAD_SIZE (sizeof(VhostUserPayload))
  209. /* The version of the protocol we support */
  210. #define VHOST_USER_VERSION (0x1)
  211. struct vhost_user {
  212. struct vhost_dev *dev;
  213. /* Shared between vhost devs of the same virtio device */
  214. VhostUserState *user;
  215. QIOChannel *slave_ioc;
  216. GSource *slave_src;
  217. NotifierWithReturn postcopy_notifier;
  218. struct PostCopyFD postcopy_fd;
  219. uint64_t postcopy_client_bases[VHOST_USER_MAX_RAM_SLOTS];
  220. /* Length of the region_rb and region_rb_offset arrays */
  221. size_t region_rb_len;
  222. /* RAMBlock associated with a given region */
  223. RAMBlock **region_rb;
  224. /*
  225. * The offset from the start of the RAMBlock to the start of the
  226. * vhost region.
  227. */
  228. ram_addr_t *region_rb_offset;
  229. /* True once we've entered postcopy_listen */
  230. bool postcopy_listen;
  231. /* Our current regions */
  232. int num_shadow_regions;
  233. struct vhost_memory_region shadow_regions[VHOST_USER_MAX_RAM_SLOTS];
  234. };
  235. struct scrub_regions {
  236. struct vhost_memory_region *region;
  237. int reg_idx;
  238. int fd_idx;
  239. };
  240. static bool ioeventfd_enabled(void)
  241. {
  242. return !kvm_enabled() || kvm_eventfds_enabled();
  243. }
  244. static int vhost_user_read_header(struct vhost_dev *dev, VhostUserMsg *msg)
  245. {
  246. struct vhost_user *u = dev->opaque;
  247. CharBackend *chr = u->user->chr;
  248. uint8_t *p = (uint8_t *) msg;
  249. int r, size = VHOST_USER_HDR_SIZE;
  250. r = qemu_chr_fe_read_all(chr, p, size);
  251. if (r != size) {
  252. int saved_errno = errno;
  253. error_report("Failed to read msg header. Read %d instead of %d."
  254. " Original request %d.", r, size, msg->hdr.request);
  255. return r < 0 ? -saved_errno : -EIO;
  256. }
  257. /* validate received flags */
  258. if (msg->hdr.flags != (VHOST_USER_REPLY_MASK | VHOST_USER_VERSION)) {
  259. error_report("Failed to read msg header."
  260. " Flags 0x%x instead of 0x%x.", msg->hdr.flags,
  261. VHOST_USER_REPLY_MASK | VHOST_USER_VERSION);
  262. return -EPROTO;
  263. }
  264. trace_vhost_user_read(msg->hdr.request, msg->hdr.flags);
  265. return 0;
  266. }
  267. struct vhost_user_read_cb_data {
  268. struct vhost_dev *dev;
  269. VhostUserMsg *msg;
  270. GMainLoop *loop;
  271. int ret;
  272. };
  273. static gboolean vhost_user_read_cb(void *do_not_use, GIOCondition condition,
  274. gpointer opaque)
  275. {
  276. struct vhost_user_read_cb_data *data = opaque;
  277. struct vhost_dev *dev = data->dev;
  278. VhostUserMsg *msg = data->msg;
  279. struct vhost_user *u = dev->opaque;
  280. CharBackend *chr = u->user->chr;
  281. uint8_t *p = (uint8_t *) msg;
  282. int r, size;
  283. r = vhost_user_read_header(dev, msg);
  284. if (r < 0) {
  285. data->ret = r;
  286. goto end;
  287. }
  288. /* validate message size is sane */
  289. if (msg->hdr.size > VHOST_USER_PAYLOAD_SIZE) {
  290. error_report("Failed to read msg header."
  291. " Size %d exceeds the maximum %zu.", msg->hdr.size,
  292. VHOST_USER_PAYLOAD_SIZE);
  293. data->ret = -EPROTO;
  294. goto end;
  295. }
  296. if (msg->hdr.size) {
  297. p += VHOST_USER_HDR_SIZE;
  298. size = msg->hdr.size;
  299. r = qemu_chr_fe_read_all(chr, p, size);
  300. if (r != size) {
  301. int saved_errno = errno;
  302. error_report("Failed to read msg payload."
  303. " Read %d instead of %d.", r, msg->hdr.size);
  304. data->ret = r < 0 ? -saved_errno : -EIO;
  305. goto end;
  306. }
  307. }
  308. end:
  309. g_main_loop_quit(data->loop);
  310. return G_SOURCE_REMOVE;
  311. }
  312. static gboolean slave_read(QIOChannel *ioc, GIOCondition condition,
  313. gpointer opaque);
  314. /*
  315. * This updates the read handler to use a new event loop context.
  316. * Event sources are removed from the previous context : this ensures
  317. * that events detected in the previous context are purged. They will
  318. * be re-detected and processed in the new context.
  319. */
  320. static void slave_update_read_handler(struct vhost_dev *dev,
  321. GMainContext *ctxt)
  322. {
  323. struct vhost_user *u = dev->opaque;
  324. if (!u->slave_ioc) {
  325. return;
  326. }
  327. if (u->slave_src) {
  328. g_source_destroy(u->slave_src);
  329. g_source_unref(u->slave_src);
  330. }
  331. u->slave_src = qio_channel_add_watch_source(u->slave_ioc,
  332. G_IO_IN | G_IO_HUP,
  333. slave_read, dev, NULL,
  334. ctxt);
  335. }
  336. static int vhost_user_read(struct vhost_dev *dev, VhostUserMsg *msg)
  337. {
  338. struct vhost_user *u = dev->opaque;
  339. CharBackend *chr = u->user->chr;
  340. GMainContext *prev_ctxt = chr->chr->gcontext;
  341. GMainContext *ctxt = g_main_context_new();
  342. GMainLoop *loop = g_main_loop_new(ctxt, FALSE);
  343. struct vhost_user_read_cb_data data = {
  344. .dev = dev,
  345. .loop = loop,
  346. .msg = msg,
  347. .ret = 0
  348. };
  349. /*
  350. * We want to be able to monitor the slave channel fd while waiting
  351. * for chr I/O. This requires an event loop, but we can't nest the
  352. * one to which chr is currently attached : its fd handlers might not
  353. * be prepared for re-entrancy. So we create a new one and switch chr
  354. * to use it.
  355. */
  356. slave_update_read_handler(dev, ctxt);
  357. qemu_chr_be_update_read_handlers(chr->chr, ctxt);
  358. qemu_chr_fe_add_watch(chr, G_IO_IN | G_IO_HUP, vhost_user_read_cb, &data);
  359. g_main_loop_run(loop);
  360. /*
  361. * Restore the previous event loop context. This also destroys/recreates
  362. * event sources : this guarantees that all pending events in the original
  363. * context that have been processed by the nested loop are purged.
  364. */
  365. qemu_chr_be_update_read_handlers(chr->chr, prev_ctxt);
  366. slave_update_read_handler(dev, NULL);
  367. g_main_loop_unref(loop);
  368. g_main_context_unref(ctxt);
  369. return data.ret;
  370. }
  371. static int process_message_reply(struct vhost_dev *dev,
  372. const VhostUserMsg *msg)
  373. {
  374. int ret;
  375. VhostUserMsg msg_reply;
  376. if ((msg->hdr.flags & VHOST_USER_NEED_REPLY_MASK) == 0) {
  377. return 0;
  378. }
  379. ret = vhost_user_read(dev, &msg_reply);
  380. if (ret < 0) {
  381. return ret;
  382. }
  383. if (msg_reply.hdr.request != msg->hdr.request) {
  384. error_report("Received unexpected msg type. "
  385. "Expected %d received %d",
  386. msg->hdr.request, msg_reply.hdr.request);
  387. return -EPROTO;
  388. }
  389. return msg_reply.payload.u64 ? -EIO : 0;
  390. }
  391. static bool vhost_user_one_time_request(VhostUserRequest request)
  392. {
  393. switch (request) {
  394. case VHOST_USER_SET_OWNER:
  395. case VHOST_USER_RESET_OWNER:
  396. case VHOST_USER_SET_MEM_TABLE:
  397. case VHOST_USER_GET_QUEUE_NUM:
  398. case VHOST_USER_NET_SET_MTU:
  399. return true;
  400. default:
  401. return false;
  402. }
  403. }
  404. /* most non-init callers ignore the error */
  405. static int vhost_user_write(struct vhost_dev *dev, VhostUserMsg *msg,
  406. int *fds, int fd_num)
  407. {
  408. struct vhost_user *u = dev->opaque;
  409. CharBackend *chr = u->user->chr;
  410. int ret, size = VHOST_USER_HDR_SIZE + msg->hdr.size;
  411. /*
  412. * For non-vring specific requests, like VHOST_USER_SET_MEM_TABLE,
  413. * we just need send it once in the first time. For later such
  414. * request, we just ignore it.
  415. */
  416. if (vhost_user_one_time_request(msg->hdr.request) && dev->vq_index != 0) {
  417. msg->hdr.flags &= ~VHOST_USER_NEED_REPLY_MASK;
  418. return 0;
  419. }
  420. if (qemu_chr_fe_set_msgfds(chr, fds, fd_num) < 0) {
  421. error_report("Failed to set msg fds.");
  422. return -EINVAL;
  423. }
  424. ret = qemu_chr_fe_write_all(chr, (const uint8_t *) msg, size);
  425. if (ret != size) {
  426. int saved_errno = errno;
  427. error_report("Failed to write msg."
  428. " Wrote %d instead of %d.", ret, size);
  429. return ret < 0 ? -saved_errno : -EIO;
  430. }
  431. trace_vhost_user_write(msg->hdr.request, msg->hdr.flags);
  432. return 0;
  433. }
  434. int vhost_user_gpu_set_socket(struct vhost_dev *dev, int fd)
  435. {
  436. VhostUserMsg msg = {
  437. .hdr.request = VHOST_USER_GPU_SET_SOCKET,
  438. .hdr.flags = VHOST_USER_VERSION,
  439. };
  440. return vhost_user_write(dev, &msg, &fd, 1);
  441. }
  442. static int vhost_user_set_log_base(struct vhost_dev *dev, uint64_t base,
  443. struct vhost_log *log)
  444. {
  445. int fds[VHOST_USER_MAX_RAM_SLOTS];
  446. size_t fd_num = 0;
  447. bool shmfd = virtio_has_feature(dev->protocol_features,
  448. VHOST_USER_PROTOCOL_F_LOG_SHMFD);
  449. int ret;
  450. VhostUserMsg msg = {
  451. .hdr.request = VHOST_USER_SET_LOG_BASE,
  452. .hdr.flags = VHOST_USER_VERSION,
  453. .payload.log.mmap_size = log->size * sizeof(*(log->log)),
  454. .payload.log.mmap_offset = 0,
  455. .hdr.size = sizeof(msg.payload.log),
  456. };
  457. if (shmfd && log->fd != -1) {
  458. fds[fd_num++] = log->fd;
  459. }
  460. ret = vhost_user_write(dev, &msg, fds, fd_num);
  461. if (ret < 0) {
  462. return ret;
  463. }
  464. if (shmfd) {
  465. msg.hdr.size = 0;
  466. ret = vhost_user_read(dev, &msg);
  467. if (ret < 0) {
  468. return ret;
  469. }
  470. if (msg.hdr.request != VHOST_USER_SET_LOG_BASE) {
  471. error_report("Received unexpected msg type. "
  472. "Expected %d received %d",
  473. VHOST_USER_SET_LOG_BASE, msg.hdr.request);
  474. return -EPROTO;
  475. }
  476. }
  477. return 0;
  478. }
  479. static MemoryRegion *vhost_user_get_mr_data(uint64_t addr, ram_addr_t *offset,
  480. int *fd)
  481. {
  482. MemoryRegion *mr;
  483. assert((uintptr_t)addr == addr);
  484. mr = memory_region_from_host((void *)(uintptr_t)addr, offset);
  485. *fd = memory_region_get_fd(mr);
  486. return mr;
  487. }
  488. static void vhost_user_fill_msg_region(VhostUserMemoryRegion *dst,
  489. struct vhost_memory_region *src,
  490. uint64_t mmap_offset)
  491. {
  492. assert(src != NULL && dst != NULL);
  493. dst->userspace_addr = src->userspace_addr;
  494. dst->memory_size = src->memory_size;
  495. dst->guest_phys_addr = src->guest_phys_addr;
  496. dst->mmap_offset = mmap_offset;
  497. }
  498. static int vhost_user_fill_set_mem_table_msg(struct vhost_user *u,
  499. struct vhost_dev *dev,
  500. VhostUserMsg *msg,
  501. int *fds, size_t *fd_num,
  502. bool track_ramblocks)
  503. {
  504. int i, fd;
  505. ram_addr_t offset;
  506. MemoryRegion *mr;
  507. struct vhost_memory_region *reg;
  508. VhostUserMemoryRegion region_buffer;
  509. msg->hdr.request = VHOST_USER_SET_MEM_TABLE;
  510. for (i = 0; i < dev->mem->nregions; ++i) {
  511. reg = dev->mem->regions + i;
  512. mr = vhost_user_get_mr_data(reg->userspace_addr, &offset, &fd);
  513. if (fd > 0) {
  514. if (track_ramblocks) {
  515. assert(*fd_num < VHOST_MEMORY_BASELINE_NREGIONS);
  516. trace_vhost_user_set_mem_table_withfd(*fd_num, mr->name,
  517. reg->memory_size,
  518. reg->guest_phys_addr,
  519. reg->userspace_addr,
  520. offset);
  521. u->region_rb_offset[i] = offset;
  522. u->region_rb[i] = mr->ram_block;
  523. } else if (*fd_num == VHOST_MEMORY_BASELINE_NREGIONS) {
  524. error_report("Failed preparing vhost-user memory table msg");
  525. return -ENOBUFS;
  526. }
  527. vhost_user_fill_msg_region(&region_buffer, reg, offset);
  528. msg->payload.memory.regions[*fd_num] = region_buffer;
  529. fds[(*fd_num)++] = fd;
  530. } else if (track_ramblocks) {
  531. u->region_rb_offset[i] = 0;
  532. u->region_rb[i] = NULL;
  533. }
  534. }
  535. msg->payload.memory.nregions = *fd_num;
  536. if (!*fd_num) {
  537. error_report("Failed initializing vhost-user memory map, "
  538. "consider using -object memory-backend-file share=on");
  539. return -EINVAL;
  540. }
  541. msg->hdr.size = sizeof(msg->payload.memory.nregions);
  542. msg->hdr.size += sizeof(msg->payload.memory.padding);
  543. msg->hdr.size += *fd_num * sizeof(VhostUserMemoryRegion);
  544. return 0;
  545. }
  546. static inline bool reg_equal(struct vhost_memory_region *shadow_reg,
  547. struct vhost_memory_region *vdev_reg)
  548. {
  549. return shadow_reg->guest_phys_addr == vdev_reg->guest_phys_addr &&
  550. shadow_reg->userspace_addr == vdev_reg->userspace_addr &&
  551. shadow_reg->memory_size == vdev_reg->memory_size;
  552. }
  553. static void scrub_shadow_regions(struct vhost_dev *dev,
  554. struct scrub_regions *add_reg,
  555. int *nr_add_reg,
  556. struct scrub_regions *rem_reg,
  557. int *nr_rem_reg, uint64_t *shadow_pcb,
  558. bool track_ramblocks)
  559. {
  560. struct vhost_user *u = dev->opaque;
  561. bool found[VHOST_USER_MAX_RAM_SLOTS] = {};
  562. struct vhost_memory_region *reg, *shadow_reg;
  563. int i, j, fd, add_idx = 0, rm_idx = 0, fd_num = 0;
  564. ram_addr_t offset;
  565. MemoryRegion *mr;
  566. bool matching;
  567. /*
  568. * Find memory regions present in our shadow state which are not in
  569. * the device's current memory state.
  570. *
  571. * Mark regions in both the shadow and device state as "found".
  572. */
  573. for (i = 0; i < u->num_shadow_regions; i++) {
  574. shadow_reg = &u->shadow_regions[i];
  575. matching = false;
  576. for (j = 0; j < dev->mem->nregions; j++) {
  577. reg = &dev->mem->regions[j];
  578. mr = vhost_user_get_mr_data(reg->userspace_addr, &offset, &fd);
  579. if (reg_equal(shadow_reg, reg)) {
  580. matching = true;
  581. found[j] = true;
  582. if (track_ramblocks) {
  583. /*
  584. * Reset postcopy client bases, region_rb, and
  585. * region_rb_offset in case regions are removed.
  586. */
  587. if (fd > 0) {
  588. u->region_rb_offset[j] = offset;
  589. u->region_rb[j] = mr->ram_block;
  590. shadow_pcb[j] = u->postcopy_client_bases[i];
  591. } else {
  592. u->region_rb_offset[j] = 0;
  593. u->region_rb[j] = NULL;
  594. }
  595. }
  596. break;
  597. }
  598. }
  599. /*
  600. * If the region was not found in the current device memory state
  601. * create an entry for it in the removed list.
  602. */
  603. if (!matching) {
  604. rem_reg[rm_idx].region = shadow_reg;
  605. rem_reg[rm_idx++].reg_idx = i;
  606. }
  607. }
  608. /*
  609. * For regions not marked "found", create entries in the added list.
  610. *
  611. * Note their indexes in the device memory state and the indexes of their
  612. * file descriptors.
  613. */
  614. for (i = 0; i < dev->mem->nregions; i++) {
  615. reg = &dev->mem->regions[i];
  616. vhost_user_get_mr_data(reg->userspace_addr, &offset, &fd);
  617. if (fd > 0) {
  618. ++fd_num;
  619. }
  620. /*
  621. * If the region was in both the shadow and device state we don't
  622. * need to send a VHOST_USER_ADD_MEM_REG message for it.
  623. */
  624. if (found[i]) {
  625. continue;
  626. }
  627. add_reg[add_idx].region = reg;
  628. add_reg[add_idx].reg_idx = i;
  629. add_reg[add_idx++].fd_idx = fd_num;
  630. }
  631. *nr_rem_reg = rm_idx;
  632. *nr_add_reg = add_idx;
  633. return;
  634. }
  635. static int send_remove_regions(struct vhost_dev *dev,
  636. struct scrub_regions *remove_reg,
  637. int nr_rem_reg, VhostUserMsg *msg,
  638. bool reply_supported)
  639. {
  640. struct vhost_user *u = dev->opaque;
  641. struct vhost_memory_region *shadow_reg;
  642. int i, fd, shadow_reg_idx, ret;
  643. ram_addr_t offset;
  644. VhostUserMemoryRegion region_buffer;
  645. /*
  646. * The regions in remove_reg appear in the same order they do in the
  647. * shadow table. Therefore we can minimize memory copies by iterating
  648. * through remove_reg backwards.
  649. */
  650. for (i = nr_rem_reg - 1; i >= 0; i--) {
  651. shadow_reg = remove_reg[i].region;
  652. shadow_reg_idx = remove_reg[i].reg_idx;
  653. vhost_user_get_mr_data(shadow_reg->userspace_addr, &offset, &fd);
  654. if (fd > 0) {
  655. msg->hdr.request = VHOST_USER_REM_MEM_REG;
  656. vhost_user_fill_msg_region(&region_buffer, shadow_reg, 0);
  657. msg->payload.mem_reg.region = region_buffer;
  658. ret = vhost_user_write(dev, msg, NULL, 0);
  659. if (ret < 0) {
  660. return ret;
  661. }
  662. if (reply_supported) {
  663. ret = process_message_reply(dev, msg);
  664. if (ret) {
  665. return ret;
  666. }
  667. }
  668. }
  669. /*
  670. * At this point we know the backend has unmapped the region. It is now
  671. * safe to remove it from the shadow table.
  672. */
  673. memmove(&u->shadow_regions[shadow_reg_idx],
  674. &u->shadow_regions[shadow_reg_idx + 1],
  675. sizeof(struct vhost_memory_region) *
  676. (u->num_shadow_regions - shadow_reg_idx - 1));
  677. u->num_shadow_regions--;
  678. }
  679. return 0;
  680. }
  681. static int send_add_regions(struct vhost_dev *dev,
  682. struct scrub_regions *add_reg, int nr_add_reg,
  683. VhostUserMsg *msg, uint64_t *shadow_pcb,
  684. bool reply_supported, bool track_ramblocks)
  685. {
  686. struct vhost_user *u = dev->opaque;
  687. int i, fd, ret, reg_idx, reg_fd_idx;
  688. struct vhost_memory_region *reg;
  689. MemoryRegion *mr;
  690. ram_addr_t offset;
  691. VhostUserMsg msg_reply;
  692. VhostUserMemoryRegion region_buffer;
  693. for (i = 0; i < nr_add_reg; i++) {
  694. reg = add_reg[i].region;
  695. reg_idx = add_reg[i].reg_idx;
  696. reg_fd_idx = add_reg[i].fd_idx;
  697. mr = vhost_user_get_mr_data(reg->userspace_addr, &offset, &fd);
  698. if (fd > 0) {
  699. if (track_ramblocks) {
  700. trace_vhost_user_set_mem_table_withfd(reg_fd_idx, mr->name,
  701. reg->memory_size,
  702. reg->guest_phys_addr,
  703. reg->userspace_addr,
  704. offset);
  705. u->region_rb_offset[reg_idx] = offset;
  706. u->region_rb[reg_idx] = mr->ram_block;
  707. }
  708. msg->hdr.request = VHOST_USER_ADD_MEM_REG;
  709. vhost_user_fill_msg_region(&region_buffer, reg, offset);
  710. msg->payload.mem_reg.region = region_buffer;
  711. ret = vhost_user_write(dev, msg, &fd, 1);
  712. if (ret < 0) {
  713. return ret;
  714. }
  715. if (track_ramblocks) {
  716. uint64_t reply_gpa;
  717. ret = vhost_user_read(dev, &msg_reply);
  718. if (ret < 0) {
  719. return ret;
  720. }
  721. reply_gpa = msg_reply.payload.mem_reg.region.guest_phys_addr;
  722. if (msg_reply.hdr.request != VHOST_USER_ADD_MEM_REG) {
  723. error_report("%s: Received unexpected msg type."
  724. "Expected %d received %d", __func__,
  725. VHOST_USER_ADD_MEM_REG,
  726. msg_reply.hdr.request);
  727. return -EPROTO;
  728. }
  729. /*
  730. * We're using the same structure, just reusing one of the
  731. * fields, so it should be the same size.
  732. */
  733. if (msg_reply.hdr.size != msg->hdr.size) {
  734. error_report("%s: Unexpected size for postcopy reply "
  735. "%d vs %d", __func__, msg_reply.hdr.size,
  736. msg->hdr.size);
  737. return -EPROTO;
  738. }
  739. /* Get the postcopy client base from the backend's reply. */
  740. if (reply_gpa == dev->mem->regions[reg_idx].guest_phys_addr) {
  741. shadow_pcb[reg_idx] =
  742. msg_reply.payload.mem_reg.region.userspace_addr;
  743. trace_vhost_user_set_mem_table_postcopy(
  744. msg_reply.payload.mem_reg.region.userspace_addr,
  745. msg->payload.mem_reg.region.userspace_addr,
  746. reg_fd_idx, reg_idx);
  747. } else {
  748. error_report("%s: invalid postcopy reply for region. "
  749. "Got guest physical address %" PRIX64 ", expected "
  750. "%" PRIX64, __func__, reply_gpa,
  751. dev->mem->regions[reg_idx].guest_phys_addr);
  752. return -EPROTO;
  753. }
  754. } else if (reply_supported) {
  755. ret = process_message_reply(dev, msg);
  756. if (ret) {
  757. return ret;
  758. }
  759. }
  760. } else if (track_ramblocks) {
  761. u->region_rb_offset[reg_idx] = 0;
  762. u->region_rb[reg_idx] = NULL;
  763. }
  764. /*
  765. * At this point, we know the backend has mapped in the new
  766. * region, if the region has a valid file descriptor.
  767. *
  768. * The region should now be added to the shadow table.
  769. */
  770. u->shadow_regions[u->num_shadow_regions].guest_phys_addr =
  771. reg->guest_phys_addr;
  772. u->shadow_regions[u->num_shadow_regions].userspace_addr =
  773. reg->userspace_addr;
  774. u->shadow_regions[u->num_shadow_regions].memory_size =
  775. reg->memory_size;
  776. u->num_shadow_regions++;
  777. }
  778. return 0;
  779. }
  780. static int vhost_user_add_remove_regions(struct vhost_dev *dev,
  781. VhostUserMsg *msg,
  782. bool reply_supported,
  783. bool track_ramblocks)
  784. {
  785. struct vhost_user *u = dev->opaque;
  786. struct scrub_regions add_reg[VHOST_USER_MAX_RAM_SLOTS];
  787. struct scrub_regions rem_reg[VHOST_USER_MAX_RAM_SLOTS];
  788. uint64_t shadow_pcb[VHOST_USER_MAX_RAM_SLOTS] = {};
  789. int nr_add_reg, nr_rem_reg;
  790. int ret;
  791. msg->hdr.size = sizeof(msg->payload.mem_reg);
  792. /* Find the regions which need to be removed or added. */
  793. scrub_shadow_regions(dev, add_reg, &nr_add_reg, rem_reg, &nr_rem_reg,
  794. shadow_pcb, track_ramblocks);
  795. if (nr_rem_reg) {
  796. ret = send_remove_regions(dev, rem_reg, nr_rem_reg, msg,
  797. reply_supported);
  798. if (ret < 0) {
  799. goto err;
  800. }
  801. }
  802. if (nr_add_reg) {
  803. ret = send_add_regions(dev, add_reg, nr_add_reg, msg, shadow_pcb,
  804. reply_supported, track_ramblocks);
  805. if (ret < 0) {
  806. goto err;
  807. }
  808. }
  809. if (track_ramblocks) {
  810. memcpy(u->postcopy_client_bases, shadow_pcb,
  811. sizeof(uint64_t) * VHOST_USER_MAX_RAM_SLOTS);
  812. /*
  813. * Now we've registered this with the postcopy code, we ack to the
  814. * client, because now we're in the position to be able to deal with
  815. * any faults it generates.
  816. */
  817. /* TODO: Use this for failure cases as well with a bad value. */
  818. msg->hdr.size = sizeof(msg->payload.u64);
  819. msg->payload.u64 = 0; /* OK */
  820. ret = vhost_user_write(dev, msg, NULL, 0);
  821. if (ret < 0) {
  822. return ret;
  823. }
  824. }
  825. return 0;
  826. err:
  827. if (track_ramblocks) {
  828. memcpy(u->postcopy_client_bases, shadow_pcb,
  829. sizeof(uint64_t) * VHOST_USER_MAX_RAM_SLOTS);
  830. }
  831. return ret;
  832. }
  833. static int vhost_user_set_mem_table_postcopy(struct vhost_dev *dev,
  834. struct vhost_memory *mem,
  835. bool reply_supported,
  836. bool config_mem_slots)
  837. {
  838. struct vhost_user *u = dev->opaque;
  839. int fds[VHOST_MEMORY_BASELINE_NREGIONS];
  840. size_t fd_num = 0;
  841. VhostUserMsg msg_reply;
  842. int region_i, msg_i;
  843. int ret;
  844. VhostUserMsg msg = {
  845. .hdr.flags = VHOST_USER_VERSION,
  846. };
  847. if (u->region_rb_len < dev->mem->nregions) {
  848. u->region_rb = g_renew(RAMBlock*, u->region_rb, dev->mem->nregions);
  849. u->region_rb_offset = g_renew(ram_addr_t, u->region_rb_offset,
  850. dev->mem->nregions);
  851. memset(&(u->region_rb[u->region_rb_len]), '\0',
  852. sizeof(RAMBlock *) * (dev->mem->nregions - u->region_rb_len));
  853. memset(&(u->region_rb_offset[u->region_rb_len]), '\0',
  854. sizeof(ram_addr_t) * (dev->mem->nregions - u->region_rb_len));
  855. u->region_rb_len = dev->mem->nregions;
  856. }
  857. if (config_mem_slots) {
  858. ret = vhost_user_add_remove_regions(dev, &msg, reply_supported, true);
  859. if (ret < 0) {
  860. return ret;
  861. }
  862. } else {
  863. ret = vhost_user_fill_set_mem_table_msg(u, dev, &msg, fds, &fd_num,
  864. true);
  865. if (ret < 0) {
  866. return ret;
  867. }
  868. ret = vhost_user_write(dev, &msg, fds, fd_num);
  869. if (ret < 0) {
  870. return ret;
  871. }
  872. ret = vhost_user_read(dev, &msg_reply);
  873. if (ret < 0) {
  874. return ret;
  875. }
  876. if (msg_reply.hdr.request != VHOST_USER_SET_MEM_TABLE) {
  877. error_report("%s: Received unexpected msg type."
  878. "Expected %d received %d", __func__,
  879. VHOST_USER_SET_MEM_TABLE, msg_reply.hdr.request);
  880. return -EPROTO;
  881. }
  882. /*
  883. * We're using the same structure, just reusing one of the
  884. * fields, so it should be the same size.
  885. */
  886. if (msg_reply.hdr.size != msg.hdr.size) {
  887. error_report("%s: Unexpected size for postcopy reply "
  888. "%d vs %d", __func__, msg_reply.hdr.size,
  889. msg.hdr.size);
  890. return -EPROTO;
  891. }
  892. memset(u->postcopy_client_bases, 0,
  893. sizeof(uint64_t) * VHOST_USER_MAX_RAM_SLOTS);
  894. /*
  895. * They're in the same order as the regions that were sent
  896. * but some of the regions were skipped (above) if they
  897. * didn't have fd's
  898. */
  899. for (msg_i = 0, region_i = 0;
  900. region_i < dev->mem->nregions;
  901. region_i++) {
  902. if (msg_i < fd_num &&
  903. msg_reply.payload.memory.regions[msg_i].guest_phys_addr ==
  904. dev->mem->regions[region_i].guest_phys_addr) {
  905. u->postcopy_client_bases[region_i] =
  906. msg_reply.payload.memory.regions[msg_i].userspace_addr;
  907. trace_vhost_user_set_mem_table_postcopy(
  908. msg_reply.payload.memory.regions[msg_i].userspace_addr,
  909. msg.payload.memory.regions[msg_i].userspace_addr,
  910. msg_i, region_i);
  911. msg_i++;
  912. }
  913. }
  914. if (msg_i != fd_num) {
  915. error_report("%s: postcopy reply not fully consumed "
  916. "%d vs %zd",
  917. __func__, msg_i, fd_num);
  918. return -EIO;
  919. }
  920. /*
  921. * Now we've registered this with the postcopy code, we ack to the
  922. * client, because now we're in the position to be able to deal
  923. * with any faults it generates.
  924. */
  925. /* TODO: Use this for failure cases as well with a bad value. */
  926. msg.hdr.size = sizeof(msg.payload.u64);
  927. msg.payload.u64 = 0; /* OK */
  928. ret = vhost_user_write(dev, &msg, NULL, 0);
  929. if (ret < 0) {
  930. return ret;
  931. }
  932. }
  933. return 0;
  934. }
  935. static int vhost_user_set_mem_table(struct vhost_dev *dev,
  936. struct vhost_memory *mem)
  937. {
  938. struct vhost_user *u = dev->opaque;
  939. int fds[VHOST_MEMORY_BASELINE_NREGIONS];
  940. size_t fd_num = 0;
  941. bool do_postcopy = u->postcopy_listen && u->postcopy_fd.handler;
  942. bool reply_supported = virtio_has_feature(dev->protocol_features,
  943. VHOST_USER_PROTOCOL_F_REPLY_ACK);
  944. bool config_mem_slots =
  945. virtio_has_feature(dev->protocol_features,
  946. VHOST_USER_PROTOCOL_F_CONFIGURE_MEM_SLOTS);
  947. int ret;
  948. if (do_postcopy) {
  949. /*
  950. * Postcopy has enough differences that it's best done in it's own
  951. * version
  952. */
  953. return vhost_user_set_mem_table_postcopy(dev, mem, reply_supported,
  954. config_mem_slots);
  955. }
  956. VhostUserMsg msg = {
  957. .hdr.flags = VHOST_USER_VERSION,
  958. };
  959. if (reply_supported) {
  960. msg.hdr.flags |= VHOST_USER_NEED_REPLY_MASK;
  961. }
  962. if (config_mem_slots) {
  963. ret = vhost_user_add_remove_regions(dev, &msg, reply_supported, false);
  964. if (ret < 0) {
  965. return ret;
  966. }
  967. } else {
  968. ret = vhost_user_fill_set_mem_table_msg(u, dev, &msg, fds, &fd_num,
  969. false);
  970. if (ret < 0) {
  971. return ret;
  972. }
  973. ret = vhost_user_write(dev, &msg, fds, fd_num);
  974. if (ret < 0) {
  975. return ret;
  976. }
  977. if (reply_supported) {
  978. return process_message_reply(dev, &msg);
  979. }
  980. }
  981. return 0;
  982. }
  983. static int vhost_user_set_vring_endian(struct vhost_dev *dev,
  984. struct vhost_vring_state *ring)
  985. {
  986. bool cross_endian = virtio_has_feature(dev->protocol_features,
  987. VHOST_USER_PROTOCOL_F_CROSS_ENDIAN);
  988. VhostUserMsg msg = {
  989. .hdr.request = VHOST_USER_SET_VRING_ENDIAN,
  990. .hdr.flags = VHOST_USER_VERSION,
  991. .payload.state = *ring,
  992. .hdr.size = sizeof(msg.payload.state),
  993. };
  994. if (!cross_endian) {
  995. error_report("vhost-user trying to send unhandled ioctl");
  996. return -ENOTSUP;
  997. }
  998. return vhost_user_write(dev, &msg, NULL, 0);
  999. }
  1000. static int vhost_set_vring(struct vhost_dev *dev,
  1001. unsigned long int request,
  1002. struct vhost_vring_state *ring)
  1003. {
  1004. VhostUserMsg msg = {
  1005. .hdr.request = request,
  1006. .hdr.flags = VHOST_USER_VERSION,
  1007. .payload.state = *ring,
  1008. .hdr.size = sizeof(msg.payload.state),
  1009. };
  1010. return vhost_user_write(dev, &msg, NULL, 0);
  1011. }
  1012. static int vhost_user_set_vring_num(struct vhost_dev *dev,
  1013. struct vhost_vring_state *ring)
  1014. {
  1015. return vhost_set_vring(dev, VHOST_USER_SET_VRING_NUM, ring);
  1016. }
  1017. static void vhost_user_host_notifier_free(VhostUserHostNotifier *n)
  1018. {
  1019. assert(n && n->unmap_addr);
  1020. munmap(n->unmap_addr, qemu_real_host_page_size());
  1021. n->unmap_addr = NULL;
  1022. }
  1023. /*
  1024. * clean-up function for notifier, will finally free the structure
  1025. * under rcu.
  1026. */
  1027. static void vhost_user_host_notifier_remove(VhostUserHostNotifier *n,
  1028. VirtIODevice *vdev)
  1029. {
  1030. if (n->addr) {
  1031. if (vdev) {
  1032. virtio_queue_set_host_notifier_mr(vdev, n->idx, &n->mr, false);
  1033. }
  1034. assert(!n->unmap_addr);
  1035. n->unmap_addr = n->addr;
  1036. n->addr = NULL;
  1037. call_rcu(n, vhost_user_host_notifier_free, rcu);
  1038. }
  1039. }
  1040. static int vhost_user_set_vring_base(struct vhost_dev *dev,
  1041. struct vhost_vring_state *ring)
  1042. {
  1043. return vhost_set_vring(dev, VHOST_USER_SET_VRING_BASE, ring);
  1044. }
  1045. static int vhost_user_set_vring_enable(struct vhost_dev *dev, int enable)
  1046. {
  1047. int i;
  1048. if (!virtio_has_feature(dev->features, VHOST_USER_F_PROTOCOL_FEATURES)) {
  1049. return -EINVAL;
  1050. }
  1051. for (i = 0; i < dev->nvqs; ++i) {
  1052. int ret;
  1053. struct vhost_vring_state state = {
  1054. .index = dev->vq_index + i,
  1055. .num = enable,
  1056. };
  1057. ret = vhost_set_vring(dev, VHOST_USER_SET_VRING_ENABLE, &state);
  1058. if (ret < 0) {
  1059. /*
  1060. * Restoring the previous state is likely infeasible, as well as
  1061. * proceeding regardless the error, so just bail out and hope for
  1062. * the device-level recovery.
  1063. */
  1064. return ret;
  1065. }
  1066. }
  1067. return 0;
  1068. }
  1069. static VhostUserHostNotifier *fetch_notifier(VhostUserState *u,
  1070. int idx)
  1071. {
  1072. if (idx >= u->notifiers->len) {
  1073. return NULL;
  1074. }
  1075. return g_ptr_array_index(u->notifiers, idx);
  1076. }
  1077. static int vhost_user_get_vring_base(struct vhost_dev *dev,
  1078. struct vhost_vring_state *ring)
  1079. {
  1080. int ret;
  1081. VhostUserMsg msg = {
  1082. .hdr.request = VHOST_USER_GET_VRING_BASE,
  1083. .hdr.flags = VHOST_USER_VERSION,
  1084. .payload.state = *ring,
  1085. .hdr.size = sizeof(msg.payload.state),
  1086. };
  1087. struct vhost_user *u = dev->opaque;
  1088. VhostUserHostNotifier *n = fetch_notifier(u->user, ring->index);
  1089. if (n) {
  1090. vhost_user_host_notifier_remove(n, dev->vdev);
  1091. }
  1092. ret = vhost_user_write(dev, &msg, NULL, 0);
  1093. if (ret < 0) {
  1094. return ret;
  1095. }
  1096. ret = vhost_user_read(dev, &msg);
  1097. if (ret < 0) {
  1098. return ret;
  1099. }
  1100. if (msg.hdr.request != VHOST_USER_GET_VRING_BASE) {
  1101. error_report("Received unexpected msg type. Expected %d received %d",
  1102. VHOST_USER_GET_VRING_BASE, msg.hdr.request);
  1103. return -EPROTO;
  1104. }
  1105. if (msg.hdr.size != sizeof(msg.payload.state)) {
  1106. error_report("Received bad msg size.");
  1107. return -EPROTO;
  1108. }
  1109. *ring = msg.payload.state;
  1110. return 0;
  1111. }
  1112. static int vhost_set_vring_file(struct vhost_dev *dev,
  1113. VhostUserRequest request,
  1114. struct vhost_vring_file *file)
  1115. {
  1116. int fds[VHOST_USER_MAX_RAM_SLOTS];
  1117. size_t fd_num = 0;
  1118. VhostUserMsg msg = {
  1119. .hdr.request = request,
  1120. .hdr.flags = VHOST_USER_VERSION,
  1121. .payload.u64 = file->index & VHOST_USER_VRING_IDX_MASK,
  1122. .hdr.size = sizeof(msg.payload.u64),
  1123. };
  1124. if (ioeventfd_enabled() && file->fd > 0) {
  1125. fds[fd_num++] = file->fd;
  1126. } else {
  1127. msg.payload.u64 |= VHOST_USER_VRING_NOFD_MASK;
  1128. }
  1129. return vhost_user_write(dev, &msg, fds, fd_num);
  1130. }
  1131. static int vhost_user_set_vring_kick(struct vhost_dev *dev,
  1132. struct vhost_vring_file *file)
  1133. {
  1134. return vhost_set_vring_file(dev, VHOST_USER_SET_VRING_KICK, file);
  1135. }
  1136. static int vhost_user_set_vring_call(struct vhost_dev *dev,
  1137. struct vhost_vring_file *file)
  1138. {
  1139. return vhost_set_vring_file(dev, VHOST_USER_SET_VRING_CALL, file);
  1140. }
  1141. static int vhost_user_set_vring_err(struct vhost_dev *dev,
  1142. struct vhost_vring_file *file)
  1143. {
  1144. return vhost_set_vring_file(dev, VHOST_USER_SET_VRING_ERR, file);
  1145. }
  1146. static int vhost_user_get_u64(struct vhost_dev *dev, int request, uint64_t *u64)
  1147. {
  1148. int ret;
  1149. VhostUserMsg msg = {
  1150. .hdr.request = request,
  1151. .hdr.flags = VHOST_USER_VERSION,
  1152. };
  1153. if (vhost_user_one_time_request(request) && dev->vq_index != 0) {
  1154. return 0;
  1155. }
  1156. ret = vhost_user_write(dev, &msg, NULL, 0);
  1157. if (ret < 0) {
  1158. return ret;
  1159. }
  1160. ret = vhost_user_read(dev, &msg);
  1161. if (ret < 0) {
  1162. return ret;
  1163. }
  1164. if (msg.hdr.request != request) {
  1165. error_report("Received unexpected msg type. Expected %d received %d",
  1166. request, msg.hdr.request);
  1167. return -EPROTO;
  1168. }
  1169. if (msg.hdr.size != sizeof(msg.payload.u64)) {
  1170. error_report("Received bad msg size.");
  1171. return -EPROTO;
  1172. }
  1173. *u64 = msg.payload.u64;
  1174. return 0;
  1175. }
  1176. static int vhost_user_get_features(struct vhost_dev *dev, uint64_t *features)
  1177. {
  1178. if (vhost_user_get_u64(dev, VHOST_USER_GET_FEATURES, features) < 0) {
  1179. return -EPROTO;
  1180. }
  1181. return 0;
  1182. }
  1183. static int enforce_reply(struct vhost_dev *dev,
  1184. const VhostUserMsg *msg)
  1185. {
  1186. uint64_t dummy;
  1187. if (msg->hdr.flags & VHOST_USER_NEED_REPLY_MASK) {
  1188. return process_message_reply(dev, msg);
  1189. }
  1190. /*
  1191. * We need to wait for a reply but the backend does not
  1192. * support replies for the command we just sent.
  1193. * Send VHOST_USER_GET_FEATURES which makes all backends
  1194. * send a reply.
  1195. */
  1196. return vhost_user_get_features(dev, &dummy);
  1197. }
  1198. static int vhost_user_set_vring_addr(struct vhost_dev *dev,
  1199. struct vhost_vring_addr *addr)
  1200. {
  1201. int ret;
  1202. VhostUserMsg msg = {
  1203. .hdr.request = VHOST_USER_SET_VRING_ADDR,
  1204. .hdr.flags = VHOST_USER_VERSION,
  1205. .payload.addr = *addr,
  1206. .hdr.size = sizeof(msg.payload.addr),
  1207. };
  1208. bool reply_supported = virtio_has_feature(dev->protocol_features,
  1209. VHOST_USER_PROTOCOL_F_REPLY_ACK);
  1210. /*
  1211. * wait for a reply if logging is enabled to make sure
  1212. * backend is actually logging changes
  1213. */
  1214. bool wait_for_reply = addr->flags & (1 << VHOST_VRING_F_LOG);
  1215. if (reply_supported && wait_for_reply) {
  1216. msg.hdr.flags |= VHOST_USER_NEED_REPLY_MASK;
  1217. }
  1218. ret = vhost_user_write(dev, &msg, NULL, 0);
  1219. if (ret < 0) {
  1220. return ret;
  1221. }
  1222. if (wait_for_reply) {
  1223. return enforce_reply(dev, &msg);
  1224. }
  1225. return 0;
  1226. }
  1227. static int vhost_user_set_u64(struct vhost_dev *dev, int request, uint64_t u64,
  1228. bool wait_for_reply)
  1229. {
  1230. VhostUserMsg msg = {
  1231. .hdr.request = request,
  1232. .hdr.flags = VHOST_USER_VERSION,
  1233. .payload.u64 = u64,
  1234. .hdr.size = sizeof(msg.payload.u64),
  1235. };
  1236. int ret;
  1237. if (wait_for_reply) {
  1238. bool reply_supported = virtio_has_feature(dev->protocol_features,
  1239. VHOST_USER_PROTOCOL_F_REPLY_ACK);
  1240. if (reply_supported) {
  1241. msg.hdr.flags |= VHOST_USER_NEED_REPLY_MASK;
  1242. }
  1243. }
  1244. ret = vhost_user_write(dev, &msg, NULL, 0);
  1245. if (ret < 0) {
  1246. return ret;
  1247. }
  1248. if (wait_for_reply) {
  1249. return enforce_reply(dev, &msg);
  1250. }
  1251. return 0;
  1252. }
  1253. static int vhost_user_set_status(struct vhost_dev *dev, uint8_t status)
  1254. {
  1255. return vhost_user_set_u64(dev, VHOST_USER_SET_STATUS, status, false);
  1256. }
  1257. static int vhost_user_get_status(struct vhost_dev *dev, uint8_t *status)
  1258. {
  1259. uint64_t value;
  1260. int ret;
  1261. ret = vhost_user_get_u64(dev, VHOST_USER_GET_STATUS, &value);
  1262. if (ret < 0) {
  1263. return ret;
  1264. }
  1265. *status = value;
  1266. return 0;
  1267. }
  1268. static int vhost_user_add_status(struct vhost_dev *dev, uint8_t status)
  1269. {
  1270. uint8_t s;
  1271. int ret;
  1272. ret = vhost_user_get_status(dev, &s);
  1273. if (ret < 0) {
  1274. return ret;
  1275. }
  1276. if ((s & status) == status) {
  1277. return 0;
  1278. }
  1279. s |= status;
  1280. return vhost_user_set_status(dev, s);
  1281. }
  1282. static int vhost_user_set_features(struct vhost_dev *dev,
  1283. uint64_t features)
  1284. {
  1285. /*
  1286. * wait for a reply if logging is enabled to make sure
  1287. * backend is actually logging changes
  1288. */
  1289. bool log_enabled = features & (0x1ULL << VHOST_F_LOG_ALL);
  1290. int ret;
  1291. /*
  1292. * We need to include any extra backend only feature bits that
  1293. * might be needed by our device. Currently this includes the
  1294. * VHOST_USER_F_PROTOCOL_FEATURES bit for enabling protocol
  1295. * features.
  1296. */
  1297. ret = vhost_user_set_u64(dev, VHOST_USER_SET_FEATURES,
  1298. features | dev->backend_features,
  1299. log_enabled);
  1300. if (virtio_has_feature(dev->protocol_features,
  1301. VHOST_USER_PROTOCOL_F_STATUS)) {
  1302. if (!ret) {
  1303. return vhost_user_add_status(dev, VIRTIO_CONFIG_S_FEATURES_OK);
  1304. }
  1305. }
  1306. return ret;
  1307. }
  1308. static int vhost_user_set_protocol_features(struct vhost_dev *dev,
  1309. uint64_t features)
  1310. {
  1311. return vhost_user_set_u64(dev, VHOST_USER_SET_PROTOCOL_FEATURES, features,
  1312. false);
  1313. }
  1314. static int vhost_user_set_owner(struct vhost_dev *dev)
  1315. {
  1316. VhostUserMsg msg = {
  1317. .hdr.request = VHOST_USER_SET_OWNER,
  1318. .hdr.flags = VHOST_USER_VERSION,
  1319. };
  1320. return vhost_user_write(dev, &msg, NULL, 0);
  1321. }
  1322. static int vhost_user_get_max_memslots(struct vhost_dev *dev,
  1323. uint64_t *max_memslots)
  1324. {
  1325. uint64_t backend_max_memslots;
  1326. int err;
  1327. err = vhost_user_get_u64(dev, VHOST_USER_GET_MAX_MEM_SLOTS,
  1328. &backend_max_memslots);
  1329. if (err < 0) {
  1330. return err;
  1331. }
  1332. *max_memslots = backend_max_memslots;
  1333. return 0;
  1334. }
  1335. static int vhost_user_reset_device(struct vhost_dev *dev)
  1336. {
  1337. VhostUserMsg msg = {
  1338. .hdr.flags = VHOST_USER_VERSION,
  1339. };
  1340. msg.hdr.request = virtio_has_feature(dev->protocol_features,
  1341. VHOST_USER_PROTOCOL_F_RESET_DEVICE)
  1342. ? VHOST_USER_RESET_DEVICE
  1343. : VHOST_USER_RESET_OWNER;
  1344. return vhost_user_write(dev, &msg, NULL, 0);
  1345. }
  1346. static int vhost_user_slave_handle_config_change(struct vhost_dev *dev)
  1347. {
  1348. if (!dev->config_ops || !dev->config_ops->vhost_dev_config_notifier) {
  1349. return -ENOSYS;
  1350. }
  1351. return dev->config_ops->vhost_dev_config_notifier(dev);
  1352. }
  1353. /*
  1354. * Fetch or create the notifier for a given idx. Newly created
  1355. * notifiers are added to the pointer array that tracks them.
  1356. */
  1357. static VhostUserHostNotifier *fetch_or_create_notifier(VhostUserState *u,
  1358. int idx)
  1359. {
  1360. VhostUserHostNotifier *n = NULL;
  1361. if (idx >= u->notifiers->len) {
  1362. g_ptr_array_set_size(u->notifiers, idx + 1);
  1363. }
  1364. n = g_ptr_array_index(u->notifiers, idx);
  1365. if (!n) {
  1366. /*
  1367. * In case notification arrive out-of-order,
  1368. * make room for current index.
  1369. */
  1370. g_ptr_array_remove_index(u->notifiers, idx);
  1371. n = g_new0(VhostUserHostNotifier, 1);
  1372. n->idx = idx;
  1373. g_ptr_array_insert(u->notifiers, idx, n);
  1374. trace_vhost_user_create_notifier(idx, n);
  1375. }
  1376. return n;
  1377. }
  1378. static int vhost_user_slave_handle_vring_host_notifier(struct vhost_dev *dev,
  1379. VhostUserVringArea *area,
  1380. int fd)
  1381. {
  1382. int queue_idx = area->u64 & VHOST_USER_VRING_IDX_MASK;
  1383. size_t page_size = qemu_real_host_page_size();
  1384. struct vhost_user *u = dev->opaque;
  1385. VhostUserState *user = u->user;
  1386. VirtIODevice *vdev = dev->vdev;
  1387. VhostUserHostNotifier *n;
  1388. void *addr;
  1389. char *name;
  1390. if (!virtio_has_feature(dev->protocol_features,
  1391. VHOST_USER_PROTOCOL_F_HOST_NOTIFIER) ||
  1392. vdev == NULL || queue_idx >= virtio_get_num_queues(vdev)) {
  1393. return -EINVAL;
  1394. }
  1395. /*
  1396. * Fetch notifier and invalidate any old data before setting up
  1397. * new mapped address.
  1398. */
  1399. n = fetch_or_create_notifier(user, queue_idx);
  1400. vhost_user_host_notifier_remove(n, vdev);
  1401. if (area->u64 & VHOST_USER_VRING_NOFD_MASK) {
  1402. return 0;
  1403. }
  1404. /* Sanity check. */
  1405. if (area->size != page_size) {
  1406. return -EINVAL;
  1407. }
  1408. addr = mmap(NULL, page_size, PROT_READ | PROT_WRITE, MAP_SHARED,
  1409. fd, area->offset);
  1410. if (addr == MAP_FAILED) {
  1411. return -EFAULT;
  1412. }
  1413. name = g_strdup_printf("vhost-user/host-notifier@%p mmaps[%d]",
  1414. user, queue_idx);
  1415. if (!n->mr.ram) { /* Don't init again after suspend. */
  1416. memory_region_init_ram_device_ptr(&n->mr, OBJECT(vdev), name,
  1417. page_size, addr);
  1418. } else {
  1419. n->mr.ram_block->host = addr;
  1420. }
  1421. g_free(name);
  1422. if (virtio_queue_set_host_notifier_mr(vdev, queue_idx, &n->mr, true)) {
  1423. object_unparent(OBJECT(&n->mr));
  1424. munmap(addr, page_size);
  1425. return -ENXIO;
  1426. }
  1427. n->addr = addr;
  1428. return 0;
  1429. }
  1430. static void close_slave_channel(struct vhost_user *u)
  1431. {
  1432. g_source_destroy(u->slave_src);
  1433. g_source_unref(u->slave_src);
  1434. u->slave_src = NULL;
  1435. object_unref(OBJECT(u->slave_ioc));
  1436. u->slave_ioc = NULL;
  1437. }
  1438. static gboolean slave_read(QIOChannel *ioc, GIOCondition condition,
  1439. gpointer opaque)
  1440. {
  1441. struct vhost_dev *dev = opaque;
  1442. struct vhost_user *u = dev->opaque;
  1443. VhostUserHeader hdr = { 0, };
  1444. VhostUserPayload payload = { 0, };
  1445. Error *local_err = NULL;
  1446. gboolean rc = G_SOURCE_CONTINUE;
  1447. int ret = 0;
  1448. struct iovec iov;
  1449. g_autofree int *fd = NULL;
  1450. size_t fdsize = 0;
  1451. int i;
  1452. /* Read header */
  1453. iov.iov_base = &hdr;
  1454. iov.iov_len = VHOST_USER_HDR_SIZE;
  1455. if (qio_channel_readv_full_all(ioc, &iov, 1, &fd, &fdsize, &local_err)) {
  1456. error_report_err(local_err);
  1457. goto err;
  1458. }
  1459. if (hdr.size > VHOST_USER_PAYLOAD_SIZE) {
  1460. error_report("Failed to read msg header."
  1461. " Size %d exceeds the maximum %zu.", hdr.size,
  1462. VHOST_USER_PAYLOAD_SIZE);
  1463. goto err;
  1464. }
  1465. /* Read payload */
  1466. if (qio_channel_read_all(ioc, (char *) &payload, hdr.size, &local_err)) {
  1467. error_report_err(local_err);
  1468. goto err;
  1469. }
  1470. switch (hdr.request) {
  1471. case VHOST_USER_SLAVE_IOTLB_MSG:
  1472. ret = vhost_backend_handle_iotlb_msg(dev, &payload.iotlb);
  1473. break;
  1474. case VHOST_USER_SLAVE_CONFIG_CHANGE_MSG :
  1475. ret = vhost_user_slave_handle_config_change(dev);
  1476. break;
  1477. case VHOST_USER_SLAVE_VRING_HOST_NOTIFIER_MSG:
  1478. ret = vhost_user_slave_handle_vring_host_notifier(dev, &payload.area,
  1479. fd ? fd[0] : -1);
  1480. break;
  1481. default:
  1482. error_report("Received unexpected msg type: %d.", hdr.request);
  1483. ret = -EINVAL;
  1484. }
  1485. /*
  1486. * REPLY_ACK feature handling. Other reply types has to be managed
  1487. * directly in their request handlers.
  1488. */
  1489. if (hdr.flags & VHOST_USER_NEED_REPLY_MASK) {
  1490. struct iovec iovec[2];
  1491. hdr.flags &= ~VHOST_USER_NEED_REPLY_MASK;
  1492. hdr.flags |= VHOST_USER_REPLY_MASK;
  1493. payload.u64 = !!ret;
  1494. hdr.size = sizeof(payload.u64);
  1495. iovec[0].iov_base = &hdr;
  1496. iovec[0].iov_len = VHOST_USER_HDR_SIZE;
  1497. iovec[1].iov_base = &payload;
  1498. iovec[1].iov_len = hdr.size;
  1499. if (qio_channel_writev_all(ioc, iovec, ARRAY_SIZE(iovec), &local_err)) {
  1500. error_report_err(local_err);
  1501. goto err;
  1502. }
  1503. }
  1504. goto fdcleanup;
  1505. err:
  1506. close_slave_channel(u);
  1507. rc = G_SOURCE_REMOVE;
  1508. fdcleanup:
  1509. if (fd) {
  1510. for (i = 0; i < fdsize; i++) {
  1511. close(fd[i]);
  1512. }
  1513. }
  1514. return rc;
  1515. }
  1516. static int vhost_setup_slave_channel(struct vhost_dev *dev)
  1517. {
  1518. VhostUserMsg msg = {
  1519. .hdr.request = VHOST_USER_SET_SLAVE_REQ_FD,
  1520. .hdr.flags = VHOST_USER_VERSION,
  1521. };
  1522. struct vhost_user *u = dev->opaque;
  1523. int sv[2], ret = 0;
  1524. bool reply_supported = virtio_has_feature(dev->protocol_features,
  1525. VHOST_USER_PROTOCOL_F_REPLY_ACK);
  1526. Error *local_err = NULL;
  1527. QIOChannel *ioc;
  1528. if (!virtio_has_feature(dev->protocol_features,
  1529. VHOST_USER_PROTOCOL_F_SLAVE_REQ)) {
  1530. return 0;
  1531. }
  1532. if (qemu_socketpair(PF_UNIX, SOCK_STREAM, 0, sv) == -1) {
  1533. int saved_errno = errno;
  1534. error_report("socketpair() failed");
  1535. return -saved_errno;
  1536. }
  1537. ioc = QIO_CHANNEL(qio_channel_socket_new_fd(sv[0], &local_err));
  1538. if (!ioc) {
  1539. error_report_err(local_err);
  1540. return -ECONNREFUSED;
  1541. }
  1542. u->slave_ioc = ioc;
  1543. slave_update_read_handler(dev, NULL);
  1544. if (reply_supported) {
  1545. msg.hdr.flags |= VHOST_USER_NEED_REPLY_MASK;
  1546. }
  1547. ret = vhost_user_write(dev, &msg, &sv[1], 1);
  1548. if (ret) {
  1549. goto out;
  1550. }
  1551. if (reply_supported) {
  1552. ret = process_message_reply(dev, &msg);
  1553. }
  1554. out:
  1555. close(sv[1]);
  1556. if (ret) {
  1557. close_slave_channel(u);
  1558. }
  1559. return ret;
  1560. }
  1561. #ifdef CONFIG_LINUX
  1562. /*
  1563. * Called back from the postcopy fault thread when a fault is received on our
  1564. * ufd.
  1565. * TODO: This is Linux specific
  1566. */
  1567. static int vhost_user_postcopy_fault_handler(struct PostCopyFD *pcfd,
  1568. void *ufd)
  1569. {
  1570. struct vhost_dev *dev = pcfd->data;
  1571. struct vhost_user *u = dev->opaque;
  1572. struct uffd_msg *msg = ufd;
  1573. uint64_t faultaddr = msg->arg.pagefault.address;
  1574. RAMBlock *rb = NULL;
  1575. uint64_t rb_offset;
  1576. int i;
  1577. trace_vhost_user_postcopy_fault_handler(pcfd->idstr, faultaddr,
  1578. dev->mem->nregions);
  1579. for (i = 0; i < MIN(dev->mem->nregions, u->region_rb_len); i++) {
  1580. trace_vhost_user_postcopy_fault_handler_loop(i,
  1581. u->postcopy_client_bases[i], dev->mem->regions[i].memory_size);
  1582. if (faultaddr >= u->postcopy_client_bases[i]) {
  1583. /* Ofset of the fault address in the vhost region */
  1584. uint64_t region_offset = faultaddr - u->postcopy_client_bases[i];
  1585. if (region_offset < dev->mem->regions[i].memory_size) {
  1586. rb_offset = region_offset + u->region_rb_offset[i];
  1587. trace_vhost_user_postcopy_fault_handler_found(i,
  1588. region_offset, rb_offset);
  1589. rb = u->region_rb[i];
  1590. return postcopy_request_shared_page(pcfd, rb, faultaddr,
  1591. rb_offset);
  1592. }
  1593. }
  1594. }
  1595. error_report("%s: Failed to find region for fault %" PRIx64,
  1596. __func__, faultaddr);
  1597. return -1;
  1598. }
  1599. static int vhost_user_postcopy_waker(struct PostCopyFD *pcfd, RAMBlock *rb,
  1600. uint64_t offset)
  1601. {
  1602. struct vhost_dev *dev = pcfd->data;
  1603. struct vhost_user *u = dev->opaque;
  1604. int i;
  1605. trace_vhost_user_postcopy_waker(qemu_ram_get_idstr(rb), offset);
  1606. if (!u) {
  1607. return 0;
  1608. }
  1609. /* Translate the offset into an address in the clients address space */
  1610. for (i = 0; i < MIN(dev->mem->nregions, u->region_rb_len); i++) {
  1611. if (u->region_rb[i] == rb &&
  1612. offset >= u->region_rb_offset[i] &&
  1613. offset < (u->region_rb_offset[i] +
  1614. dev->mem->regions[i].memory_size)) {
  1615. uint64_t client_addr = (offset - u->region_rb_offset[i]) +
  1616. u->postcopy_client_bases[i];
  1617. trace_vhost_user_postcopy_waker_found(client_addr);
  1618. return postcopy_wake_shared(pcfd, client_addr, rb);
  1619. }
  1620. }
  1621. trace_vhost_user_postcopy_waker_nomatch(qemu_ram_get_idstr(rb), offset);
  1622. return 0;
  1623. }
  1624. #endif
  1625. /*
  1626. * Called at the start of an inbound postcopy on reception of the
  1627. * 'advise' command.
  1628. */
  1629. static int vhost_user_postcopy_advise(struct vhost_dev *dev, Error **errp)
  1630. {
  1631. #ifdef CONFIG_LINUX
  1632. struct vhost_user *u = dev->opaque;
  1633. CharBackend *chr = u->user->chr;
  1634. int ufd;
  1635. int ret;
  1636. VhostUserMsg msg = {
  1637. .hdr.request = VHOST_USER_POSTCOPY_ADVISE,
  1638. .hdr.flags = VHOST_USER_VERSION,
  1639. };
  1640. ret = vhost_user_write(dev, &msg, NULL, 0);
  1641. if (ret < 0) {
  1642. error_setg(errp, "Failed to send postcopy_advise to vhost");
  1643. return ret;
  1644. }
  1645. ret = vhost_user_read(dev, &msg);
  1646. if (ret < 0) {
  1647. error_setg(errp, "Failed to get postcopy_advise reply from vhost");
  1648. return ret;
  1649. }
  1650. if (msg.hdr.request != VHOST_USER_POSTCOPY_ADVISE) {
  1651. error_setg(errp, "Unexpected msg type. Expected %d received %d",
  1652. VHOST_USER_POSTCOPY_ADVISE, msg.hdr.request);
  1653. return -EPROTO;
  1654. }
  1655. if (msg.hdr.size) {
  1656. error_setg(errp, "Received bad msg size.");
  1657. return -EPROTO;
  1658. }
  1659. ufd = qemu_chr_fe_get_msgfd(chr);
  1660. if (ufd < 0) {
  1661. error_setg(errp, "%s: Failed to get ufd", __func__);
  1662. return -EIO;
  1663. }
  1664. qemu_socket_set_nonblock(ufd);
  1665. /* register ufd with userfault thread */
  1666. u->postcopy_fd.fd = ufd;
  1667. u->postcopy_fd.data = dev;
  1668. u->postcopy_fd.handler = vhost_user_postcopy_fault_handler;
  1669. u->postcopy_fd.waker = vhost_user_postcopy_waker;
  1670. u->postcopy_fd.idstr = "vhost-user"; /* Need to find unique name */
  1671. postcopy_register_shared_ufd(&u->postcopy_fd);
  1672. return 0;
  1673. #else
  1674. error_setg(errp, "Postcopy not supported on non-Linux systems");
  1675. return -ENOSYS;
  1676. #endif
  1677. }
  1678. /*
  1679. * Called at the switch to postcopy on reception of the 'listen' command.
  1680. */
  1681. static int vhost_user_postcopy_listen(struct vhost_dev *dev, Error **errp)
  1682. {
  1683. struct vhost_user *u = dev->opaque;
  1684. int ret;
  1685. VhostUserMsg msg = {
  1686. .hdr.request = VHOST_USER_POSTCOPY_LISTEN,
  1687. .hdr.flags = VHOST_USER_VERSION | VHOST_USER_NEED_REPLY_MASK,
  1688. };
  1689. u->postcopy_listen = true;
  1690. trace_vhost_user_postcopy_listen();
  1691. ret = vhost_user_write(dev, &msg, NULL, 0);
  1692. if (ret < 0) {
  1693. error_setg(errp, "Failed to send postcopy_listen to vhost");
  1694. return ret;
  1695. }
  1696. ret = process_message_reply(dev, &msg);
  1697. if (ret) {
  1698. error_setg(errp, "Failed to receive reply to postcopy_listen");
  1699. return ret;
  1700. }
  1701. return 0;
  1702. }
  1703. /*
  1704. * Called at the end of postcopy
  1705. */
  1706. static int vhost_user_postcopy_end(struct vhost_dev *dev, Error **errp)
  1707. {
  1708. VhostUserMsg msg = {
  1709. .hdr.request = VHOST_USER_POSTCOPY_END,
  1710. .hdr.flags = VHOST_USER_VERSION | VHOST_USER_NEED_REPLY_MASK,
  1711. };
  1712. int ret;
  1713. struct vhost_user *u = dev->opaque;
  1714. trace_vhost_user_postcopy_end_entry();
  1715. ret = vhost_user_write(dev, &msg, NULL, 0);
  1716. if (ret < 0) {
  1717. error_setg(errp, "Failed to send postcopy_end to vhost");
  1718. return ret;
  1719. }
  1720. ret = process_message_reply(dev, &msg);
  1721. if (ret) {
  1722. error_setg(errp, "Failed to receive reply to postcopy_end");
  1723. return ret;
  1724. }
  1725. postcopy_unregister_shared_ufd(&u->postcopy_fd);
  1726. close(u->postcopy_fd.fd);
  1727. u->postcopy_fd.handler = NULL;
  1728. trace_vhost_user_postcopy_end_exit();
  1729. return 0;
  1730. }
  1731. static int vhost_user_postcopy_notifier(NotifierWithReturn *notifier,
  1732. void *opaque)
  1733. {
  1734. struct PostcopyNotifyData *pnd = opaque;
  1735. struct vhost_user *u = container_of(notifier, struct vhost_user,
  1736. postcopy_notifier);
  1737. struct vhost_dev *dev = u->dev;
  1738. switch (pnd->reason) {
  1739. case POSTCOPY_NOTIFY_PROBE:
  1740. if (!virtio_has_feature(dev->protocol_features,
  1741. VHOST_USER_PROTOCOL_F_PAGEFAULT)) {
  1742. /* TODO: Get the device name into this error somehow */
  1743. error_setg(pnd->errp,
  1744. "vhost-user backend not capable of postcopy");
  1745. return -ENOENT;
  1746. }
  1747. break;
  1748. case POSTCOPY_NOTIFY_INBOUND_ADVISE:
  1749. return vhost_user_postcopy_advise(dev, pnd->errp);
  1750. case POSTCOPY_NOTIFY_INBOUND_LISTEN:
  1751. return vhost_user_postcopy_listen(dev, pnd->errp);
  1752. case POSTCOPY_NOTIFY_INBOUND_END:
  1753. return vhost_user_postcopy_end(dev, pnd->errp);
  1754. default:
  1755. /* We ignore notifications we don't know */
  1756. break;
  1757. }
  1758. return 0;
  1759. }
  1760. static int vhost_user_backend_init(struct vhost_dev *dev, void *opaque,
  1761. Error **errp)
  1762. {
  1763. uint64_t features, ram_slots;
  1764. struct vhost_user *u;
  1765. VhostUserState *vus = (VhostUserState *) opaque;
  1766. int err;
  1767. assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER);
  1768. u = g_new0(struct vhost_user, 1);
  1769. u->user = vus;
  1770. u->dev = dev;
  1771. dev->opaque = u;
  1772. err = vhost_user_get_features(dev, &features);
  1773. if (err < 0) {
  1774. error_setg_errno(errp, -err, "vhost_backend_init failed");
  1775. return err;
  1776. }
  1777. if (virtio_has_feature(features, VHOST_USER_F_PROTOCOL_FEATURES)) {
  1778. bool supports_f_config = vus->supports_config ||
  1779. (dev->config_ops && dev->config_ops->vhost_dev_config_notifier);
  1780. uint64_t protocol_features;
  1781. dev->backend_features |= 1ULL << VHOST_USER_F_PROTOCOL_FEATURES;
  1782. err = vhost_user_get_u64(dev, VHOST_USER_GET_PROTOCOL_FEATURES,
  1783. &protocol_features);
  1784. if (err < 0) {
  1785. error_setg_errno(errp, EPROTO, "vhost_backend_init failed");
  1786. return -EPROTO;
  1787. }
  1788. /*
  1789. * We will use all the protocol features we support - although
  1790. * we suppress F_CONFIG if we know QEMUs internal code can not support
  1791. * it.
  1792. */
  1793. protocol_features &= VHOST_USER_PROTOCOL_FEATURE_MASK;
  1794. if (supports_f_config) {
  1795. if (!virtio_has_feature(protocol_features,
  1796. VHOST_USER_PROTOCOL_F_CONFIG)) {
  1797. error_setg(errp, "vhost-user device expecting "
  1798. "VHOST_USER_PROTOCOL_F_CONFIG but the vhost-user backend does "
  1799. "not support it.");
  1800. return -EPROTO;
  1801. }
  1802. } else {
  1803. if (virtio_has_feature(protocol_features,
  1804. VHOST_USER_PROTOCOL_F_CONFIG)) {
  1805. warn_reportf_err(*errp, "vhost-user backend supports "
  1806. "VHOST_USER_PROTOCOL_F_CONFIG but QEMU does not.");
  1807. protocol_features &= ~(1ULL << VHOST_USER_PROTOCOL_F_CONFIG);
  1808. }
  1809. }
  1810. /* final set of protocol features */
  1811. dev->protocol_features = protocol_features;
  1812. err = vhost_user_set_protocol_features(dev, dev->protocol_features);
  1813. if (err < 0) {
  1814. error_setg_errno(errp, EPROTO, "vhost_backend_init failed");
  1815. return -EPROTO;
  1816. }
  1817. /* query the max queues we support if backend supports Multiple Queue */
  1818. if (dev->protocol_features & (1ULL << VHOST_USER_PROTOCOL_F_MQ)) {
  1819. err = vhost_user_get_u64(dev, VHOST_USER_GET_QUEUE_NUM,
  1820. &dev->max_queues);
  1821. if (err < 0) {
  1822. error_setg_errno(errp, EPROTO, "vhost_backend_init failed");
  1823. return -EPROTO;
  1824. }
  1825. } else {
  1826. dev->max_queues = 1;
  1827. }
  1828. if (dev->num_queues && dev->max_queues < dev->num_queues) {
  1829. error_setg(errp, "The maximum number of queues supported by the "
  1830. "backend is %" PRIu64, dev->max_queues);
  1831. return -EINVAL;
  1832. }
  1833. if (virtio_has_feature(features, VIRTIO_F_IOMMU_PLATFORM) &&
  1834. !(virtio_has_feature(dev->protocol_features,
  1835. VHOST_USER_PROTOCOL_F_SLAVE_REQ) &&
  1836. virtio_has_feature(dev->protocol_features,
  1837. VHOST_USER_PROTOCOL_F_REPLY_ACK))) {
  1838. error_setg(errp, "IOMMU support requires reply-ack and "
  1839. "slave-req protocol features.");
  1840. return -EINVAL;
  1841. }
  1842. /* get max memory regions if backend supports configurable RAM slots */
  1843. if (!virtio_has_feature(dev->protocol_features,
  1844. VHOST_USER_PROTOCOL_F_CONFIGURE_MEM_SLOTS)) {
  1845. u->user->memory_slots = VHOST_MEMORY_BASELINE_NREGIONS;
  1846. } else {
  1847. err = vhost_user_get_max_memslots(dev, &ram_slots);
  1848. if (err < 0) {
  1849. error_setg_errno(errp, EPROTO, "vhost_backend_init failed");
  1850. return -EPROTO;
  1851. }
  1852. if (ram_slots < u->user->memory_slots) {
  1853. error_setg(errp, "The backend specified a max ram slots limit "
  1854. "of %" PRIu64", when the prior validated limit was "
  1855. "%d. This limit should never decrease.", ram_slots,
  1856. u->user->memory_slots);
  1857. return -EINVAL;
  1858. }
  1859. u->user->memory_slots = MIN(ram_slots, VHOST_USER_MAX_RAM_SLOTS);
  1860. }
  1861. }
  1862. if (dev->migration_blocker == NULL &&
  1863. !virtio_has_feature(dev->protocol_features,
  1864. VHOST_USER_PROTOCOL_F_LOG_SHMFD)) {
  1865. error_setg(&dev->migration_blocker,
  1866. "Migration disabled: vhost-user backend lacks "
  1867. "VHOST_USER_PROTOCOL_F_LOG_SHMFD feature.");
  1868. }
  1869. if (dev->vq_index == 0) {
  1870. err = vhost_setup_slave_channel(dev);
  1871. if (err < 0) {
  1872. error_setg_errno(errp, EPROTO, "vhost_backend_init failed");
  1873. return -EPROTO;
  1874. }
  1875. }
  1876. u->postcopy_notifier.notify = vhost_user_postcopy_notifier;
  1877. postcopy_add_notifier(&u->postcopy_notifier);
  1878. return 0;
  1879. }
  1880. static int vhost_user_backend_cleanup(struct vhost_dev *dev)
  1881. {
  1882. struct vhost_user *u;
  1883. assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER);
  1884. u = dev->opaque;
  1885. if (u->postcopy_notifier.notify) {
  1886. postcopy_remove_notifier(&u->postcopy_notifier);
  1887. u->postcopy_notifier.notify = NULL;
  1888. }
  1889. u->postcopy_listen = false;
  1890. if (u->postcopy_fd.handler) {
  1891. postcopy_unregister_shared_ufd(&u->postcopy_fd);
  1892. close(u->postcopy_fd.fd);
  1893. u->postcopy_fd.handler = NULL;
  1894. }
  1895. if (u->slave_ioc) {
  1896. close_slave_channel(u);
  1897. }
  1898. g_free(u->region_rb);
  1899. u->region_rb = NULL;
  1900. g_free(u->region_rb_offset);
  1901. u->region_rb_offset = NULL;
  1902. u->region_rb_len = 0;
  1903. g_free(u);
  1904. dev->opaque = 0;
  1905. return 0;
  1906. }
  1907. static int vhost_user_get_vq_index(struct vhost_dev *dev, int idx)
  1908. {
  1909. assert(idx >= dev->vq_index && idx < dev->vq_index + dev->nvqs);
  1910. return idx;
  1911. }
  1912. static int vhost_user_memslots_limit(struct vhost_dev *dev)
  1913. {
  1914. struct vhost_user *u = dev->opaque;
  1915. return u->user->memory_slots;
  1916. }
  1917. static bool vhost_user_requires_shm_log(struct vhost_dev *dev)
  1918. {
  1919. assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER);
  1920. return virtio_has_feature(dev->protocol_features,
  1921. VHOST_USER_PROTOCOL_F_LOG_SHMFD);
  1922. }
  1923. static int vhost_user_migration_done(struct vhost_dev *dev, char* mac_addr)
  1924. {
  1925. VhostUserMsg msg = { };
  1926. assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER);
  1927. /* If guest supports GUEST_ANNOUNCE do nothing */
  1928. if (virtio_has_feature(dev->acked_features, VIRTIO_NET_F_GUEST_ANNOUNCE)) {
  1929. return 0;
  1930. }
  1931. /* if backend supports VHOST_USER_PROTOCOL_F_RARP ask it to send the RARP */
  1932. if (virtio_has_feature(dev->protocol_features,
  1933. VHOST_USER_PROTOCOL_F_RARP)) {
  1934. msg.hdr.request = VHOST_USER_SEND_RARP;
  1935. msg.hdr.flags = VHOST_USER_VERSION;
  1936. memcpy((char *)&msg.payload.u64, mac_addr, 6);
  1937. msg.hdr.size = sizeof(msg.payload.u64);
  1938. return vhost_user_write(dev, &msg, NULL, 0);
  1939. }
  1940. return -ENOTSUP;
  1941. }
  1942. static bool vhost_user_can_merge(struct vhost_dev *dev,
  1943. uint64_t start1, uint64_t size1,
  1944. uint64_t start2, uint64_t size2)
  1945. {
  1946. ram_addr_t offset;
  1947. int mfd, rfd;
  1948. (void)vhost_user_get_mr_data(start1, &offset, &mfd);
  1949. (void)vhost_user_get_mr_data(start2, &offset, &rfd);
  1950. return mfd == rfd;
  1951. }
  1952. static int vhost_user_net_set_mtu(struct vhost_dev *dev, uint16_t mtu)
  1953. {
  1954. VhostUserMsg msg;
  1955. bool reply_supported = virtio_has_feature(dev->protocol_features,
  1956. VHOST_USER_PROTOCOL_F_REPLY_ACK);
  1957. int ret;
  1958. if (!(dev->protocol_features & (1ULL << VHOST_USER_PROTOCOL_F_NET_MTU))) {
  1959. return 0;
  1960. }
  1961. msg.hdr.request = VHOST_USER_NET_SET_MTU;
  1962. msg.payload.u64 = mtu;
  1963. msg.hdr.size = sizeof(msg.payload.u64);
  1964. msg.hdr.flags = VHOST_USER_VERSION;
  1965. if (reply_supported) {
  1966. msg.hdr.flags |= VHOST_USER_NEED_REPLY_MASK;
  1967. }
  1968. ret = vhost_user_write(dev, &msg, NULL, 0);
  1969. if (ret < 0) {
  1970. return ret;
  1971. }
  1972. /* If reply_ack supported, slave has to ack specified MTU is valid */
  1973. if (reply_supported) {
  1974. return process_message_reply(dev, &msg);
  1975. }
  1976. return 0;
  1977. }
  1978. static int vhost_user_send_device_iotlb_msg(struct vhost_dev *dev,
  1979. struct vhost_iotlb_msg *imsg)
  1980. {
  1981. int ret;
  1982. VhostUserMsg msg = {
  1983. .hdr.request = VHOST_USER_IOTLB_MSG,
  1984. .hdr.size = sizeof(msg.payload.iotlb),
  1985. .hdr.flags = VHOST_USER_VERSION | VHOST_USER_NEED_REPLY_MASK,
  1986. .payload.iotlb = *imsg,
  1987. };
  1988. ret = vhost_user_write(dev, &msg, NULL, 0);
  1989. if (ret < 0) {
  1990. return ret;
  1991. }
  1992. return process_message_reply(dev, &msg);
  1993. }
  1994. static void vhost_user_set_iotlb_callback(struct vhost_dev *dev, int enabled)
  1995. {
  1996. /* No-op as the receive channel is not dedicated to IOTLB messages. */
  1997. }
  1998. static int vhost_user_get_config(struct vhost_dev *dev, uint8_t *config,
  1999. uint32_t config_len, Error **errp)
  2000. {
  2001. int ret;
  2002. VhostUserMsg msg = {
  2003. .hdr.request = VHOST_USER_GET_CONFIG,
  2004. .hdr.flags = VHOST_USER_VERSION,
  2005. .hdr.size = VHOST_USER_CONFIG_HDR_SIZE + config_len,
  2006. };
  2007. if (!virtio_has_feature(dev->protocol_features,
  2008. VHOST_USER_PROTOCOL_F_CONFIG)) {
  2009. error_setg(errp, "VHOST_USER_PROTOCOL_F_CONFIG not supported");
  2010. return -EINVAL;
  2011. }
  2012. assert(config_len <= VHOST_USER_MAX_CONFIG_SIZE);
  2013. msg.payload.config.offset = 0;
  2014. msg.payload.config.size = config_len;
  2015. ret = vhost_user_write(dev, &msg, NULL, 0);
  2016. if (ret < 0) {
  2017. error_setg_errno(errp, -ret, "vhost_get_config failed");
  2018. return ret;
  2019. }
  2020. ret = vhost_user_read(dev, &msg);
  2021. if (ret < 0) {
  2022. error_setg_errno(errp, -ret, "vhost_get_config failed");
  2023. return ret;
  2024. }
  2025. if (msg.hdr.request != VHOST_USER_GET_CONFIG) {
  2026. error_setg(errp,
  2027. "Received unexpected msg type. Expected %d received %d",
  2028. VHOST_USER_GET_CONFIG, msg.hdr.request);
  2029. return -EPROTO;
  2030. }
  2031. if (msg.hdr.size != VHOST_USER_CONFIG_HDR_SIZE + config_len) {
  2032. error_setg(errp, "Received bad msg size.");
  2033. return -EPROTO;
  2034. }
  2035. memcpy(config, msg.payload.config.region, config_len);
  2036. return 0;
  2037. }
  2038. static int vhost_user_set_config(struct vhost_dev *dev, const uint8_t *data,
  2039. uint32_t offset, uint32_t size, uint32_t flags)
  2040. {
  2041. int ret;
  2042. uint8_t *p;
  2043. bool reply_supported = virtio_has_feature(dev->protocol_features,
  2044. VHOST_USER_PROTOCOL_F_REPLY_ACK);
  2045. VhostUserMsg msg = {
  2046. .hdr.request = VHOST_USER_SET_CONFIG,
  2047. .hdr.flags = VHOST_USER_VERSION,
  2048. .hdr.size = VHOST_USER_CONFIG_HDR_SIZE + size,
  2049. };
  2050. if (!virtio_has_feature(dev->protocol_features,
  2051. VHOST_USER_PROTOCOL_F_CONFIG)) {
  2052. return -ENOTSUP;
  2053. }
  2054. if (reply_supported) {
  2055. msg.hdr.flags |= VHOST_USER_NEED_REPLY_MASK;
  2056. }
  2057. if (size > VHOST_USER_MAX_CONFIG_SIZE) {
  2058. return -EINVAL;
  2059. }
  2060. msg.payload.config.offset = offset,
  2061. msg.payload.config.size = size,
  2062. msg.payload.config.flags = flags,
  2063. p = msg.payload.config.region;
  2064. memcpy(p, data, size);
  2065. ret = vhost_user_write(dev, &msg, NULL, 0);
  2066. if (ret < 0) {
  2067. return ret;
  2068. }
  2069. if (reply_supported) {
  2070. return process_message_reply(dev, &msg);
  2071. }
  2072. return 0;
  2073. }
  2074. static int vhost_user_crypto_create_session(struct vhost_dev *dev,
  2075. void *session_info,
  2076. uint64_t *session_id)
  2077. {
  2078. int ret;
  2079. bool crypto_session = virtio_has_feature(dev->protocol_features,
  2080. VHOST_USER_PROTOCOL_F_CRYPTO_SESSION);
  2081. CryptoDevBackendSymSessionInfo *sess_info = session_info;
  2082. VhostUserMsg msg = {
  2083. .hdr.request = VHOST_USER_CREATE_CRYPTO_SESSION,
  2084. .hdr.flags = VHOST_USER_VERSION,
  2085. .hdr.size = sizeof(msg.payload.session),
  2086. };
  2087. assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_USER);
  2088. if (!crypto_session) {
  2089. error_report("vhost-user trying to send unhandled ioctl");
  2090. return -ENOTSUP;
  2091. }
  2092. memcpy(&msg.payload.session.session_setup_data, sess_info,
  2093. sizeof(CryptoDevBackendSymSessionInfo));
  2094. if (sess_info->key_len) {
  2095. memcpy(&msg.payload.session.key, sess_info->cipher_key,
  2096. sess_info->key_len);
  2097. }
  2098. if (sess_info->auth_key_len > 0) {
  2099. memcpy(&msg.payload.session.auth_key, sess_info->auth_key,
  2100. sess_info->auth_key_len);
  2101. }
  2102. ret = vhost_user_write(dev, &msg, NULL, 0);
  2103. if (ret < 0) {
  2104. error_report("vhost_user_write() return %d, create session failed",
  2105. ret);
  2106. return ret;
  2107. }
  2108. ret = vhost_user_read(dev, &msg);
  2109. if (ret < 0) {
  2110. error_report("vhost_user_read() return %d, create session failed",
  2111. ret);
  2112. return ret;
  2113. }
  2114. if (msg.hdr.request != VHOST_USER_CREATE_CRYPTO_SESSION) {
  2115. error_report("Received unexpected msg type. Expected %d received %d",
  2116. VHOST_USER_CREATE_CRYPTO_SESSION, msg.hdr.request);
  2117. return -EPROTO;
  2118. }
  2119. if (msg.hdr.size != sizeof(msg.payload.session)) {
  2120. error_report("Received bad msg size.");
  2121. return -EPROTO;
  2122. }
  2123. if (msg.payload.session.session_id < 0) {
  2124. error_report("Bad session id: %" PRId64 "",
  2125. msg.payload.session.session_id);
  2126. return -EINVAL;
  2127. }
  2128. *session_id = msg.payload.session.session_id;
  2129. return 0;
  2130. }
  2131. static int
  2132. vhost_user_crypto_close_session(struct vhost_dev *dev, uint64_t session_id)
  2133. {
  2134. int ret;
  2135. bool crypto_session = virtio_has_feature(dev->protocol_features,
  2136. VHOST_USER_PROTOCOL_F_CRYPTO_SESSION);
  2137. VhostUserMsg msg = {
  2138. .hdr.request = VHOST_USER_CLOSE_CRYPTO_SESSION,
  2139. .hdr.flags = VHOST_USER_VERSION,
  2140. .hdr.size = sizeof(msg.payload.u64),
  2141. };
  2142. msg.payload.u64 = session_id;
  2143. if (!crypto_session) {
  2144. error_report("vhost-user trying to send unhandled ioctl");
  2145. return -ENOTSUP;
  2146. }
  2147. ret = vhost_user_write(dev, &msg, NULL, 0);
  2148. if (ret < 0) {
  2149. error_report("vhost_user_write() return %d, close session failed",
  2150. ret);
  2151. return ret;
  2152. }
  2153. return 0;
  2154. }
  2155. static bool vhost_user_mem_section_filter(struct vhost_dev *dev,
  2156. MemoryRegionSection *section)
  2157. {
  2158. return memory_region_get_fd(section->mr) >= 0;
  2159. }
  2160. static int vhost_user_get_inflight_fd(struct vhost_dev *dev,
  2161. uint16_t queue_size,
  2162. struct vhost_inflight *inflight)
  2163. {
  2164. void *addr;
  2165. int fd;
  2166. int ret;
  2167. struct vhost_user *u = dev->opaque;
  2168. CharBackend *chr = u->user->chr;
  2169. VhostUserMsg msg = {
  2170. .hdr.request = VHOST_USER_GET_INFLIGHT_FD,
  2171. .hdr.flags = VHOST_USER_VERSION,
  2172. .payload.inflight.num_queues = dev->nvqs,
  2173. .payload.inflight.queue_size = queue_size,
  2174. .hdr.size = sizeof(msg.payload.inflight),
  2175. };
  2176. if (!virtio_has_feature(dev->protocol_features,
  2177. VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)) {
  2178. return 0;
  2179. }
  2180. ret = vhost_user_write(dev, &msg, NULL, 0);
  2181. if (ret < 0) {
  2182. return ret;
  2183. }
  2184. ret = vhost_user_read(dev, &msg);
  2185. if (ret < 0) {
  2186. return ret;
  2187. }
  2188. if (msg.hdr.request != VHOST_USER_GET_INFLIGHT_FD) {
  2189. error_report("Received unexpected msg type. "
  2190. "Expected %d received %d",
  2191. VHOST_USER_GET_INFLIGHT_FD, msg.hdr.request);
  2192. return -EPROTO;
  2193. }
  2194. if (msg.hdr.size != sizeof(msg.payload.inflight)) {
  2195. error_report("Received bad msg size.");
  2196. return -EPROTO;
  2197. }
  2198. if (!msg.payload.inflight.mmap_size) {
  2199. return 0;
  2200. }
  2201. fd = qemu_chr_fe_get_msgfd(chr);
  2202. if (fd < 0) {
  2203. error_report("Failed to get mem fd");
  2204. return -EIO;
  2205. }
  2206. addr = mmap(0, msg.payload.inflight.mmap_size, PROT_READ | PROT_WRITE,
  2207. MAP_SHARED, fd, msg.payload.inflight.mmap_offset);
  2208. if (addr == MAP_FAILED) {
  2209. error_report("Failed to mmap mem fd");
  2210. close(fd);
  2211. return -EFAULT;
  2212. }
  2213. inflight->addr = addr;
  2214. inflight->fd = fd;
  2215. inflight->size = msg.payload.inflight.mmap_size;
  2216. inflight->offset = msg.payload.inflight.mmap_offset;
  2217. inflight->queue_size = queue_size;
  2218. return 0;
  2219. }
  2220. static int vhost_user_set_inflight_fd(struct vhost_dev *dev,
  2221. struct vhost_inflight *inflight)
  2222. {
  2223. VhostUserMsg msg = {
  2224. .hdr.request = VHOST_USER_SET_INFLIGHT_FD,
  2225. .hdr.flags = VHOST_USER_VERSION,
  2226. .payload.inflight.mmap_size = inflight->size,
  2227. .payload.inflight.mmap_offset = inflight->offset,
  2228. .payload.inflight.num_queues = dev->nvqs,
  2229. .payload.inflight.queue_size = inflight->queue_size,
  2230. .hdr.size = sizeof(msg.payload.inflight),
  2231. };
  2232. if (!virtio_has_feature(dev->protocol_features,
  2233. VHOST_USER_PROTOCOL_F_INFLIGHT_SHMFD)) {
  2234. return 0;
  2235. }
  2236. return vhost_user_write(dev, &msg, &inflight->fd, 1);
  2237. }
  2238. static void vhost_user_state_destroy(gpointer data)
  2239. {
  2240. VhostUserHostNotifier *n = (VhostUserHostNotifier *) data;
  2241. if (n) {
  2242. vhost_user_host_notifier_remove(n, NULL);
  2243. object_unparent(OBJECT(&n->mr));
  2244. /*
  2245. * We can't free until vhost_user_host_notifier_remove has
  2246. * done it's thing so schedule the free with RCU.
  2247. */
  2248. g_free_rcu(n, rcu);
  2249. }
  2250. }
  2251. bool vhost_user_init(VhostUserState *user, CharBackend *chr, Error **errp)
  2252. {
  2253. if (user->chr) {
  2254. error_setg(errp, "Cannot initialize vhost-user state");
  2255. return false;
  2256. }
  2257. user->chr = chr;
  2258. user->memory_slots = 0;
  2259. user->notifiers = g_ptr_array_new_full(VIRTIO_QUEUE_MAX / 4,
  2260. &vhost_user_state_destroy);
  2261. return true;
  2262. }
  2263. void vhost_user_cleanup(VhostUserState *user)
  2264. {
  2265. if (!user->chr) {
  2266. return;
  2267. }
  2268. memory_region_transaction_begin();
  2269. user->notifiers = (GPtrArray *) g_ptr_array_free(user->notifiers, true);
  2270. memory_region_transaction_commit();
  2271. user->chr = NULL;
  2272. }
  2273. typedef struct {
  2274. vu_async_close_fn cb;
  2275. DeviceState *dev;
  2276. CharBackend *cd;
  2277. struct vhost_dev *vhost;
  2278. } VhostAsyncCallback;
  2279. static void vhost_user_async_close_bh(void *opaque)
  2280. {
  2281. VhostAsyncCallback *data = opaque;
  2282. struct vhost_dev *vhost = data->vhost;
  2283. /*
  2284. * If the vhost_dev has been cleared in the meantime there is
  2285. * nothing left to do as some other path has completed the
  2286. * cleanup.
  2287. */
  2288. if (vhost->vdev) {
  2289. data->cb(data->dev);
  2290. }
  2291. g_free(data);
  2292. }
  2293. /*
  2294. * We only schedule the work if the machine is running. If suspended
  2295. * we want to keep all the in-flight data as is for migration
  2296. * purposes.
  2297. */
  2298. void vhost_user_async_close(DeviceState *d,
  2299. CharBackend *chardev, struct vhost_dev *vhost,
  2300. vu_async_close_fn cb)
  2301. {
  2302. if (!runstate_check(RUN_STATE_SHUTDOWN)) {
  2303. /*
  2304. * A close event may happen during a read/write, but vhost
  2305. * code assumes the vhost_dev remains setup, so delay the
  2306. * stop & clear.
  2307. */
  2308. AioContext *ctx = qemu_get_current_aio_context();
  2309. VhostAsyncCallback *data = g_new0(VhostAsyncCallback, 1);
  2310. /* Save data for the callback */
  2311. data->cb = cb;
  2312. data->dev = d;
  2313. data->cd = chardev;
  2314. data->vhost = vhost;
  2315. /* Disable any further notifications on the chardev */
  2316. qemu_chr_fe_set_handlers(chardev,
  2317. NULL, NULL, NULL, NULL, NULL, NULL,
  2318. false);
  2319. aio_bh_schedule_oneshot(ctx, vhost_user_async_close_bh, data);
  2320. /*
  2321. * Move vhost device to the stopped state. The vhost-user device
  2322. * will be clean up and disconnected in BH. This can be useful in
  2323. * the vhost migration code. If disconnect was caught there is an
  2324. * option for the general vhost code to get the dev state without
  2325. * knowing its type (in this case vhost-user).
  2326. *
  2327. * Note if the vhost device is fully cleared by the time we
  2328. * execute the bottom half we won't continue with the cleanup.
  2329. */
  2330. vhost->started = false;
  2331. }
  2332. }
  2333. static int vhost_user_dev_start(struct vhost_dev *dev, bool started)
  2334. {
  2335. if (!virtio_has_feature(dev->protocol_features,
  2336. VHOST_USER_PROTOCOL_F_STATUS)) {
  2337. return 0;
  2338. }
  2339. /* Set device status only for last queue pair */
  2340. if (dev->vq_index + dev->nvqs != dev->vq_index_end) {
  2341. return 0;
  2342. }
  2343. if (started) {
  2344. return vhost_user_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE |
  2345. VIRTIO_CONFIG_S_DRIVER |
  2346. VIRTIO_CONFIG_S_DRIVER_OK);
  2347. } else {
  2348. return vhost_user_set_status(dev, 0);
  2349. }
  2350. }
  2351. const VhostOps user_ops = {
  2352. .backend_type = VHOST_BACKEND_TYPE_USER,
  2353. .vhost_backend_init = vhost_user_backend_init,
  2354. .vhost_backend_cleanup = vhost_user_backend_cleanup,
  2355. .vhost_backend_memslots_limit = vhost_user_memslots_limit,
  2356. .vhost_set_log_base = vhost_user_set_log_base,
  2357. .vhost_set_mem_table = vhost_user_set_mem_table,
  2358. .vhost_set_vring_addr = vhost_user_set_vring_addr,
  2359. .vhost_set_vring_endian = vhost_user_set_vring_endian,
  2360. .vhost_set_vring_num = vhost_user_set_vring_num,
  2361. .vhost_set_vring_base = vhost_user_set_vring_base,
  2362. .vhost_get_vring_base = vhost_user_get_vring_base,
  2363. .vhost_set_vring_kick = vhost_user_set_vring_kick,
  2364. .vhost_set_vring_call = vhost_user_set_vring_call,
  2365. .vhost_set_vring_err = vhost_user_set_vring_err,
  2366. .vhost_set_features = vhost_user_set_features,
  2367. .vhost_get_features = vhost_user_get_features,
  2368. .vhost_set_owner = vhost_user_set_owner,
  2369. .vhost_reset_device = vhost_user_reset_device,
  2370. .vhost_get_vq_index = vhost_user_get_vq_index,
  2371. .vhost_set_vring_enable = vhost_user_set_vring_enable,
  2372. .vhost_requires_shm_log = vhost_user_requires_shm_log,
  2373. .vhost_migration_done = vhost_user_migration_done,
  2374. .vhost_backend_can_merge = vhost_user_can_merge,
  2375. .vhost_net_set_mtu = vhost_user_net_set_mtu,
  2376. .vhost_set_iotlb_callback = vhost_user_set_iotlb_callback,
  2377. .vhost_send_device_iotlb_msg = vhost_user_send_device_iotlb_msg,
  2378. .vhost_get_config = vhost_user_get_config,
  2379. .vhost_set_config = vhost_user_set_config,
  2380. .vhost_crypto_create_session = vhost_user_crypto_create_session,
  2381. .vhost_crypto_close_session = vhost_user_crypto_close_session,
  2382. .vhost_backend_mem_section_filter = vhost_user_mem_section_filter,
  2383. .vhost_get_inflight_fd = vhost_user_get_inflight_fd,
  2384. .vhost_set_inflight_fd = vhost_user_set_inflight_fd,
  2385. .vhost_dev_start = vhost_user_dev_start,
  2386. };