2
0

optimize.c 80 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092
  1. /*
  2. * Optimizations for Tiny Code Generator for QEMU
  3. *
  4. * Copyright (c) 2010 Samsung Electronics.
  5. * Contributed by Kirill Batuzov <batuzovk@ispras.ru>
  6. *
  7. * Permission is hereby granted, free of charge, to any person obtaining a copy
  8. * of this software and associated documentation files (the "Software"), to deal
  9. * in the Software without restriction, including without limitation the rights
  10. * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  11. * copies of the Software, and to permit persons to whom the Software is
  12. * furnished to do so, subject to the following conditions:
  13. *
  14. * The above copyright notice and this permission notice shall be included in
  15. * all copies or substantial portions of the Software.
  16. *
  17. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  18. * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  19. * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
  20. * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  21. * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  22. * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  23. * THE SOFTWARE.
  24. */
  25. #include "qemu/osdep.h"
  26. #include "qemu/int128.h"
  27. #include "qemu/interval-tree.h"
  28. #include "tcg/tcg-op-common.h"
  29. #include "tcg-internal.h"
  30. #include "tcg-has.h"
  31. #define CASE_OP_32_64(x) \
  32. glue(glue(case INDEX_op_, x), _i32): \
  33. glue(glue(case INDEX_op_, x), _i64)
  34. #define CASE_OP_32_64_VEC(x) \
  35. glue(glue(case INDEX_op_, x), _i32): \
  36. glue(glue(case INDEX_op_, x), _i64): \
  37. glue(glue(case INDEX_op_, x), _vec)
  38. typedef struct MemCopyInfo {
  39. IntervalTreeNode itree;
  40. QSIMPLEQ_ENTRY (MemCopyInfo) next;
  41. TCGTemp *ts;
  42. TCGType type;
  43. } MemCopyInfo;
  44. typedef struct TempOptInfo {
  45. bool is_const;
  46. TCGTemp *prev_copy;
  47. TCGTemp *next_copy;
  48. QSIMPLEQ_HEAD(, MemCopyInfo) mem_copy;
  49. uint64_t val;
  50. uint64_t z_mask; /* mask bit is 0 if and only if value bit is 0 */
  51. uint64_t s_mask; /* mask bit is 1 if value bit matches msb */
  52. } TempOptInfo;
  53. typedef struct OptContext {
  54. TCGContext *tcg;
  55. TCGOp *prev_mb;
  56. TCGTempSet temps_used;
  57. IntervalTreeRoot mem_copy;
  58. QSIMPLEQ_HEAD(, MemCopyInfo) mem_free;
  59. /* In flight values from optimization. */
  60. TCGType type;
  61. } OptContext;
  62. static inline TempOptInfo *ts_info(TCGTemp *ts)
  63. {
  64. return ts->state_ptr;
  65. }
  66. static inline TempOptInfo *arg_info(TCGArg arg)
  67. {
  68. return ts_info(arg_temp(arg));
  69. }
  70. static inline bool ti_is_const(TempOptInfo *ti)
  71. {
  72. return ti->is_const;
  73. }
  74. static inline uint64_t ti_const_val(TempOptInfo *ti)
  75. {
  76. return ti->val;
  77. }
  78. static inline bool ti_is_const_val(TempOptInfo *ti, uint64_t val)
  79. {
  80. return ti_is_const(ti) && ti_const_val(ti) == val;
  81. }
  82. static inline bool ts_is_const(TCGTemp *ts)
  83. {
  84. return ti_is_const(ts_info(ts));
  85. }
  86. static inline bool ts_is_const_val(TCGTemp *ts, uint64_t val)
  87. {
  88. return ti_is_const_val(ts_info(ts), val);
  89. }
  90. static inline bool arg_is_const(TCGArg arg)
  91. {
  92. return ts_is_const(arg_temp(arg));
  93. }
  94. static inline bool arg_is_const_val(TCGArg arg, uint64_t val)
  95. {
  96. return ts_is_const_val(arg_temp(arg), val);
  97. }
  98. static inline bool ts_is_copy(TCGTemp *ts)
  99. {
  100. return ts_info(ts)->next_copy != ts;
  101. }
  102. static TCGTemp *cmp_better_copy(TCGTemp *a, TCGTemp *b)
  103. {
  104. return a->kind < b->kind ? b : a;
  105. }
  106. /* Initialize and activate a temporary. */
  107. static void init_ts_info(OptContext *ctx, TCGTemp *ts)
  108. {
  109. size_t idx = temp_idx(ts);
  110. TempOptInfo *ti;
  111. if (test_bit(idx, ctx->temps_used.l)) {
  112. return;
  113. }
  114. set_bit(idx, ctx->temps_used.l);
  115. ti = ts->state_ptr;
  116. if (ti == NULL) {
  117. ti = tcg_malloc(sizeof(TempOptInfo));
  118. ts->state_ptr = ti;
  119. }
  120. ti->next_copy = ts;
  121. ti->prev_copy = ts;
  122. QSIMPLEQ_INIT(&ti->mem_copy);
  123. if (ts->kind == TEMP_CONST) {
  124. ti->is_const = true;
  125. ti->val = ts->val;
  126. ti->z_mask = ts->val;
  127. ti->s_mask = INT64_MIN >> clrsb64(ts->val);
  128. } else {
  129. ti->is_const = false;
  130. ti->z_mask = -1;
  131. ti->s_mask = 0;
  132. }
  133. }
  134. static MemCopyInfo *mem_copy_first(OptContext *ctx, intptr_t s, intptr_t l)
  135. {
  136. IntervalTreeNode *r = interval_tree_iter_first(&ctx->mem_copy, s, l);
  137. return r ? container_of(r, MemCopyInfo, itree) : NULL;
  138. }
  139. static MemCopyInfo *mem_copy_next(MemCopyInfo *mem, intptr_t s, intptr_t l)
  140. {
  141. IntervalTreeNode *r = interval_tree_iter_next(&mem->itree, s, l);
  142. return r ? container_of(r, MemCopyInfo, itree) : NULL;
  143. }
  144. static void remove_mem_copy(OptContext *ctx, MemCopyInfo *mc)
  145. {
  146. TCGTemp *ts = mc->ts;
  147. TempOptInfo *ti = ts_info(ts);
  148. interval_tree_remove(&mc->itree, &ctx->mem_copy);
  149. QSIMPLEQ_REMOVE(&ti->mem_copy, mc, MemCopyInfo, next);
  150. QSIMPLEQ_INSERT_TAIL(&ctx->mem_free, mc, next);
  151. }
  152. static void remove_mem_copy_in(OptContext *ctx, intptr_t s, intptr_t l)
  153. {
  154. while (true) {
  155. MemCopyInfo *mc = mem_copy_first(ctx, s, l);
  156. if (!mc) {
  157. break;
  158. }
  159. remove_mem_copy(ctx, mc);
  160. }
  161. }
  162. static void remove_mem_copy_all(OptContext *ctx)
  163. {
  164. remove_mem_copy_in(ctx, 0, -1);
  165. tcg_debug_assert(interval_tree_is_empty(&ctx->mem_copy));
  166. }
  167. static TCGTemp *find_better_copy(TCGTemp *ts)
  168. {
  169. TCGTemp *i, *ret;
  170. /* If this is already readonly, we can't do better. */
  171. if (temp_readonly(ts)) {
  172. return ts;
  173. }
  174. ret = ts;
  175. for (i = ts_info(ts)->next_copy; i != ts; i = ts_info(i)->next_copy) {
  176. ret = cmp_better_copy(ret, i);
  177. }
  178. return ret;
  179. }
  180. static void move_mem_copies(TCGTemp *dst_ts, TCGTemp *src_ts)
  181. {
  182. TempOptInfo *si = ts_info(src_ts);
  183. TempOptInfo *di = ts_info(dst_ts);
  184. MemCopyInfo *mc;
  185. QSIMPLEQ_FOREACH(mc, &si->mem_copy, next) {
  186. tcg_debug_assert(mc->ts == src_ts);
  187. mc->ts = dst_ts;
  188. }
  189. QSIMPLEQ_CONCAT(&di->mem_copy, &si->mem_copy);
  190. }
  191. /* Reset TEMP's state, possibly removing the temp for the list of copies. */
  192. static void reset_ts(OptContext *ctx, TCGTemp *ts)
  193. {
  194. TempOptInfo *ti = ts_info(ts);
  195. TCGTemp *pts = ti->prev_copy;
  196. TCGTemp *nts = ti->next_copy;
  197. TempOptInfo *pi = ts_info(pts);
  198. TempOptInfo *ni = ts_info(nts);
  199. ni->prev_copy = ti->prev_copy;
  200. pi->next_copy = ti->next_copy;
  201. ti->next_copy = ts;
  202. ti->prev_copy = ts;
  203. ti->is_const = false;
  204. ti->z_mask = -1;
  205. ti->s_mask = 0;
  206. if (!QSIMPLEQ_EMPTY(&ti->mem_copy)) {
  207. if (ts == nts) {
  208. /* Last temp copy being removed, the mem copies die. */
  209. MemCopyInfo *mc;
  210. QSIMPLEQ_FOREACH(mc, &ti->mem_copy, next) {
  211. interval_tree_remove(&mc->itree, &ctx->mem_copy);
  212. }
  213. QSIMPLEQ_CONCAT(&ctx->mem_free, &ti->mem_copy);
  214. } else {
  215. move_mem_copies(find_better_copy(nts), ts);
  216. }
  217. }
  218. }
  219. static void reset_temp(OptContext *ctx, TCGArg arg)
  220. {
  221. reset_ts(ctx, arg_temp(arg));
  222. }
  223. static void record_mem_copy(OptContext *ctx, TCGType type,
  224. TCGTemp *ts, intptr_t start, intptr_t last)
  225. {
  226. MemCopyInfo *mc;
  227. TempOptInfo *ti;
  228. mc = QSIMPLEQ_FIRST(&ctx->mem_free);
  229. if (mc) {
  230. QSIMPLEQ_REMOVE_HEAD(&ctx->mem_free, next);
  231. } else {
  232. mc = tcg_malloc(sizeof(*mc));
  233. }
  234. memset(mc, 0, sizeof(*mc));
  235. mc->itree.start = start;
  236. mc->itree.last = last;
  237. mc->type = type;
  238. interval_tree_insert(&mc->itree, &ctx->mem_copy);
  239. ts = find_better_copy(ts);
  240. ti = ts_info(ts);
  241. mc->ts = ts;
  242. QSIMPLEQ_INSERT_TAIL(&ti->mem_copy, mc, next);
  243. }
  244. static bool ts_are_copies(TCGTemp *ts1, TCGTemp *ts2)
  245. {
  246. TCGTemp *i;
  247. if (ts1 == ts2) {
  248. return true;
  249. }
  250. if (!ts_is_copy(ts1) || !ts_is_copy(ts2)) {
  251. return false;
  252. }
  253. for (i = ts_info(ts1)->next_copy; i != ts1; i = ts_info(i)->next_copy) {
  254. if (i == ts2) {
  255. return true;
  256. }
  257. }
  258. return false;
  259. }
  260. static bool args_are_copies(TCGArg arg1, TCGArg arg2)
  261. {
  262. return ts_are_copies(arg_temp(arg1), arg_temp(arg2));
  263. }
  264. static TCGTemp *find_mem_copy_for(OptContext *ctx, TCGType type, intptr_t s)
  265. {
  266. MemCopyInfo *mc;
  267. for (mc = mem_copy_first(ctx, s, s); mc; mc = mem_copy_next(mc, s, s)) {
  268. if (mc->itree.start == s && mc->type == type) {
  269. return find_better_copy(mc->ts);
  270. }
  271. }
  272. return NULL;
  273. }
  274. static TCGArg arg_new_constant(OptContext *ctx, uint64_t val)
  275. {
  276. TCGType type = ctx->type;
  277. TCGTemp *ts;
  278. if (type == TCG_TYPE_I32) {
  279. val = (int32_t)val;
  280. }
  281. ts = tcg_constant_internal(type, val);
  282. init_ts_info(ctx, ts);
  283. return temp_arg(ts);
  284. }
  285. static TCGArg arg_new_temp(OptContext *ctx)
  286. {
  287. TCGTemp *ts = tcg_temp_new_internal(ctx->type, TEMP_EBB);
  288. init_ts_info(ctx, ts);
  289. return temp_arg(ts);
  290. }
  291. static bool tcg_opt_gen_mov(OptContext *ctx, TCGOp *op, TCGArg dst, TCGArg src)
  292. {
  293. TCGTemp *dst_ts = arg_temp(dst);
  294. TCGTemp *src_ts = arg_temp(src);
  295. TempOptInfo *di;
  296. TempOptInfo *si;
  297. TCGOpcode new_op;
  298. if (ts_are_copies(dst_ts, src_ts)) {
  299. tcg_op_remove(ctx->tcg, op);
  300. return true;
  301. }
  302. reset_ts(ctx, dst_ts);
  303. di = ts_info(dst_ts);
  304. si = ts_info(src_ts);
  305. switch (ctx->type) {
  306. case TCG_TYPE_I32:
  307. new_op = INDEX_op_mov_i32;
  308. break;
  309. case TCG_TYPE_I64:
  310. new_op = INDEX_op_mov_i64;
  311. break;
  312. case TCG_TYPE_V64:
  313. case TCG_TYPE_V128:
  314. case TCG_TYPE_V256:
  315. /* TCGOP_TYPE and TCGOP_VECE remain unchanged. */
  316. new_op = INDEX_op_mov_vec;
  317. break;
  318. default:
  319. g_assert_not_reached();
  320. }
  321. op->opc = new_op;
  322. op->args[0] = dst;
  323. op->args[1] = src;
  324. di->z_mask = si->z_mask;
  325. di->s_mask = si->s_mask;
  326. if (src_ts->type == dst_ts->type) {
  327. TempOptInfo *ni = ts_info(si->next_copy);
  328. di->next_copy = si->next_copy;
  329. di->prev_copy = src_ts;
  330. ni->prev_copy = dst_ts;
  331. si->next_copy = dst_ts;
  332. di->is_const = si->is_const;
  333. di->val = si->val;
  334. if (!QSIMPLEQ_EMPTY(&si->mem_copy)
  335. && cmp_better_copy(src_ts, dst_ts) == dst_ts) {
  336. move_mem_copies(dst_ts, src_ts);
  337. }
  338. }
  339. return true;
  340. }
  341. static bool tcg_opt_gen_movi(OptContext *ctx, TCGOp *op,
  342. TCGArg dst, uint64_t val)
  343. {
  344. /* Convert movi to mov with constant temp. */
  345. return tcg_opt_gen_mov(ctx, op, dst, arg_new_constant(ctx, val));
  346. }
  347. static uint64_t do_constant_folding_2(TCGOpcode op, uint64_t x, uint64_t y)
  348. {
  349. uint64_t l64, h64;
  350. switch (op) {
  351. CASE_OP_32_64(add):
  352. return x + y;
  353. CASE_OP_32_64(sub):
  354. return x - y;
  355. CASE_OP_32_64(mul):
  356. return x * y;
  357. CASE_OP_32_64_VEC(and):
  358. return x & y;
  359. CASE_OP_32_64_VEC(or):
  360. return x | y;
  361. CASE_OP_32_64_VEC(xor):
  362. return x ^ y;
  363. case INDEX_op_shl_i32:
  364. return (uint32_t)x << (y & 31);
  365. case INDEX_op_shl_i64:
  366. return (uint64_t)x << (y & 63);
  367. case INDEX_op_shr_i32:
  368. return (uint32_t)x >> (y & 31);
  369. case INDEX_op_shr_i64:
  370. return (uint64_t)x >> (y & 63);
  371. case INDEX_op_sar_i32:
  372. return (int32_t)x >> (y & 31);
  373. case INDEX_op_sar_i64:
  374. return (int64_t)x >> (y & 63);
  375. case INDEX_op_rotr_i32:
  376. return ror32(x, y & 31);
  377. case INDEX_op_rotr_i64:
  378. return ror64(x, y & 63);
  379. case INDEX_op_rotl_i32:
  380. return rol32(x, y & 31);
  381. case INDEX_op_rotl_i64:
  382. return rol64(x, y & 63);
  383. CASE_OP_32_64_VEC(not):
  384. return ~x;
  385. CASE_OP_32_64(neg):
  386. return -x;
  387. CASE_OP_32_64_VEC(andc):
  388. return x & ~y;
  389. CASE_OP_32_64_VEC(orc):
  390. return x | ~y;
  391. CASE_OP_32_64_VEC(eqv):
  392. return ~(x ^ y);
  393. CASE_OP_32_64_VEC(nand):
  394. return ~(x & y);
  395. CASE_OP_32_64_VEC(nor):
  396. return ~(x | y);
  397. case INDEX_op_clz_i32:
  398. return (uint32_t)x ? clz32(x) : y;
  399. case INDEX_op_clz_i64:
  400. return x ? clz64(x) : y;
  401. case INDEX_op_ctz_i32:
  402. return (uint32_t)x ? ctz32(x) : y;
  403. case INDEX_op_ctz_i64:
  404. return x ? ctz64(x) : y;
  405. case INDEX_op_ctpop_i32:
  406. return ctpop32(x);
  407. case INDEX_op_ctpop_i64:
  408. return ctpop64(x);
  409. CASE_OP_32_64(ext8s):
  410. return (int8_t)x;
  411. CASE_OP_32_64(ext16s):
  412. return (int16_t)x;
  413. CASE_OP_32_64(ext8u):
  414. return (uint8_t)x;
  415. CASE_OP_32_64(ext16u):
  416. return (uint16_t)x;
  417. CASE_OP_32_64(bswap16):
  418. x = bswap16(x);
  419. return y & TCG_BSWAP_OS ? (int16_t)x : x;
  420. CASE_OP_32_64(bswap32):
  421. x = bswap32(x);
  422. return y & TCG_BSWAP_OS ? (int32_t)x : x;
  423. case INDEX_op_bswap64_i64:
  424. return bswap64(x);
  425. case INDEX_op_ext_i32_i64:
  426. case INDEX_op_ext32s_i64:
  427. return (int32_t)x;
  428. case INDEX_op_extu_i32_i64:
  429. case INDEX_op_extrl_i64_i32:
  430. case INDEX_op_ext32u_i64:
  431. return (uint32_t)x;
  432. case INDEX_op_extrh_i64_i32:
  433. return (uint64_t)x >> 32;
  434. case INDEX_op_muluh_i32:
  435. return ((uint64_t)(uint32_t)x * (uint32_t)y) >> 32;
  436. case INDEX_op_mulsh_i32:
  437. return ((int64_t)(int32_t)x * (int32_t)y) >> 32;
  438. case INDEX_op_muluh_i64:
  439. mulu64(&l64, &h64, x, y);
  440. return h64;
  441. case INDEX_op_mulsh_i64:
  442. muls64(&l64, &h64, x, y);
  443. return h64;
  444. case INDEX_op_div_i32:
  445. /* Avoid crashing on divide by zero, otherwise undefined. */
  446. return (int32_t)x / ((int32_t)y ? : 1);
  447. case INDEX_op_divu_i32:
  448. return (uint32_t)x / ((uint32_t)y ? : 1);
  449. case INDEX_op_div_i64:
  450. return (int64_t)x / ((int64_t)y ? : 1);
  451. case INDEX_op_divu_i64:
  452. return (uint64_t)x / ((uint64_t)y ? : 1);
  453. case INDEX_op_rem_i32:
  454. return (int32_t)x % ((int32_t)y ? : 1);
  455. case INDEX_op_remu_i32:
  456. return (uint32_t)x % ((uint32_t)y ? : 1);
  457. case INDEX_op_rem_i64:
  458. return (int64_t)x % ((int64_t)y ? : 1);
  459. case INDEX_op_remu_i64:
  460. return (uint64_t)x % ((uint64_t)y ? : 1);
  461. default:
  462. g_assert_not_reached();
  463. }
  464. }
  465. static uint64_t do_constant_folding(TCGOpcode op, TCGType type,
  466. uint64_t x, uint64_t y)
  467. {
  468. uint64_t res = do_constant_folding_2(op, x, y);
  469. if (type == TCG_TYPE_I32) {
  470. res = (int32_t)res;
  471. }
  472. return res;
  473. }
  474. static bool do_constant_folding_cond_32(uint32_t x, uint32_t y, TCGCond c)
  475. {
  476. switch (c) {
  477. case TCG_COND_EQ:
  478. return x == y;
  479. case TCG_COND_NE:
  480. return x != y;
  481. case TCG_COND_LT:
  482. return (int32_t)x < (int32_t)y;
  483. case TCG_COND_GE:
  484. return (int32_t)x >= (int32_t)y;
  485. case TCG_COND_LE:
  486. return (int32_t)x <= (int32_t)y;
  487. case TCG_COND_GT:
  488. return (int32_t)x > (int32_t)y;
  489. case TCG_COND_LTU:
  490. return x < y;
  491. case TCG_COND_GEU:
  492. return x >= y;
  493. case TCG_COND_LEU:
  494. return x <= y;
  495. case TCG_COND_GTU:
  496. return x > y;
  497. case TCG_COND_TSTEQ:
  498. return (x & y) == 0;
  499. case TCG_COND_TSTNE:
  500. return (x & y) != 0;
  501. case TCG_COND_ALWAYS:
  502. case TCG_COND_NEVER:
  503. break;
  504. }
  505. g_assert_not_reached();
  506. }
  507. static bool do_constant_folding_cond_64(uint64_t x, uint64_t y, TCGCond c)
  508. {
  509. switch (c) {
  510. case TCG_COND_EQ:
  511. return x == y;
  512. case TCG_COND_NE:
  513. return x != y;
  514. case TCG_COND_LT:
  515. return (int64_t)x < (int64_t)y;
  516. case TCG_COND_GE:
  517. return (int64_t)x >= (int64_t)y;
  518. case TCG_COND_LE:
  519. return (int64_t)x <= (int64_t)y;
  520. case TCG_COND_GT:
  521. return (int64_t)x > (int64_t)y;
  522. case TCG_COND_LTU:
  523. return x < y;
  524. case TCG_COND_GEU:
  525. return x >= y;
  526. case TCG_COND_LEU:
  527. return x <= y;
  528. case TCG_COND_GTU:
  529. return x > y;
  530. case TCG_COND_TSTEQ:
  531. return (x & y) == 0;
  532. case TCG_COND_TSTNE:
  533. return (x & y) != 0;
  534. case TCG_COND_ALWAYS:
  535. case TCG_COND_NEVER:
  536. break;
  537. }
  538. g_assert_not_reached();
  539. }
  540. static int do_constant_folding_cond_eq(TCGCond c)
  541. {
  542. switch (c) {
  543. case TCG_COND_GT:
  544. case TCG_COND_LTU:
  545. case TCG_COND_LT:
  546. case TCG_COND_GTU:
  547. case TCG_COND_NE:
  548. return 0;
  549. case TCG_COND_GE:
  550. case TCG_COND_GEU:
  551. case TCG_COND_LE:
  552. case TCG_COND_LEU:
  553. case TCG_COND_EQ:
  554. return 1;
  555. case TCG_COND_TSTEQ:
  556. case TCG_COND_TSTNE:
  557. return -1;
  558. case TCG_COND_ALWAYS:
  559. case TCG_COND_NEVER:
  560. break;
  561. }
  562. g_assert_not_reached();
  563. }
  564. /*
  565. * Return -1 if the condition can't be simplified,
  566. * and the result of the condition (0 or 1) if it can.
  567. */
  568. static int do_constant_folding_cond(TCGType type, TCGArg x,
  569. TCGArg y, TCGCond c)
  570. {
  571. if (arg_is_const(x) && arg_is_const(y)) {
  572. uint64_t xv = arg_info(x)->val;
  573. uint64_t yv = arg_info(y)->val;
  574. switch (type) {
  575. case TCG_TYPE_I32:
  576. return do_constant_folding_cond_32(xv, yv, c);
  577. case TCG_TYPE_I64:
  578. return do_constant_folding_cond_64(xv, yv, c);
  579. default:
  580. /* Only scalar comparisons are optimizable */
  581. return -1;
  582. }
  583. } else if (args_are_copies(x, y)) {
  584. return do_constant_folding_cond_eq(c);
  585. } else if (arg_is_const_val(y, 0)) {
  586. switch (c) {
  587. case TCG_COND_LTU:
  588. case TCG_COND_TSTNE:
  589. return 0;
  590. case TCG_COND_GEU:
  591. case TCG_COND_TSTEQ:
  592. return 1;
  593. default:
  594. return -1;
  595. }
  596. }
  597. return -1;
  598. }
  599. /**
  600. * swap_commutative:
  601. * @dest: TCGArg of the destination argument, or NO_DEST.
  602. * @p1: first paired argument
  603. * @p2: second paired argument
  604. *
  605. * If *@p1 is a constant and *@p2 is not, swap.
  606. * If *@p2 matches @dest, swap.
  607. * Return true if a swap was performed.
  608. */
  609. #define NO_DEST temp_arg(NULL)
  610. static bool swap_commutative(TCGArg dest, TCGArg *p1, TCGArg *p2)
  611. {
  612. TCGArg a1 = *p1, a2 = *p2;
  613. int sum = 0;
  614. sum += arg_is_const(a1);
  615. sum -= arg_is_const(a2);
  616. /* Prefer the constant in second argument, and then the form
  617. op a, a, b, which is better handled on non-RISC hosts. */
  618. if (sum > 0 || (sum == 0 && dest == a2)) {
  619. *p1 = a2;
  620. *p2 = a1;
  621. return true;
  622. }
  623. return false;
  624. }
  625. static bool swap_commutative2(TCGArg *p1, TCGArg *p2)
  626. {
  627. int sum = 0;
  628. sum += arg_is_const(p1[0]);
  629. sum += arg_is_const(p1[1]);
  630. sum -= arg_is_const(p2[0]);
  631. sum -= arg_is_const(p2[1]);
  632. if (sum > 0) {
  633. TCGArg t;
  634. t = p1[0], p1[0] = p2[0], p2[0] = t;
  635. t = p1[1], p1[1] = p2[1], p2[1] = t;
  636. return true;
  637. }
  638. return false;
  639. }
  640. /*
  641. * Return -1 if the condition can't be simplified,
  642. * and the result of the condition (0 or 1) if it can.
  643. */
  644. static int do_constant_folding_cond1(OptContext *ctx, TCGOp *op, TCGArg dest,
  645. TCGArg *p1, TCGArg *p2, TCGArg *pcond)
  646. {
  647. TCGCond cond;
  648. TempOptInfo *i1;
  649. bool swap;
  650. int r;
  651. swap = swap_commutative(dest, p1, p2);
  652. cond = *pcond;
  653. if (swap) {
  654. *pcond = cond = tcg_swap_cond(cond);
  655. }
  656. r = do_constant_folding_cond(ctx->type, *p1, *p2, cond);
  657. if (r >= 0) {
  658. return r;
  659. }
  660. if (!is_tst_cond(cond)) {
  661. return -1;
  662. }
  663. i1 = arg_info(*p1);
  664. /*
  665. * TSTNE x,x -> NE x,0
  666. * TSTNE x,i -> NE x,0 if i includes all nonzero bits of x
  667. */
  668. if (args_are_copies(*p1, *p2) ||
  669. (arg_is_const(*p2) && (i1->z_mask & ~arg_info(*p2)->val) == 0)) {
  670. *p2 = arg_new_constant(ctx, 0);
  671. *pcond = tcg_tst_eqne_cond(cond);
  672. return -1;
  673. }
  674. /* TSTNE x,i -> LT x,0 if i only includes sign bit copies */
  675. if (arg_is_const(*p2) && (arg_info(*p2)->val & ~i1->s_mask) == 0) {
  676. *p2 = arg_new_constant(ctx, 0);
  677. *pcond = tcg_tst_ltge_cond(cond);
  678. return -1;
  679. }
  680. /* Expand to AND with a temporary if no backend support. */
  681. if (!TCG_TARGET_HAS_tst) {
  682. TCGOpcode and_opc = (ctx->type == TCG_TYPE_I32
  683. ? INDEX_op_and_i32 : INDEX_op_and_i64);
  684. TCGOp *op2 = tcg_op_insert_before(ctx->tcg, op, and_opc, 3);
  685. TCGArg tmp = arg_new_temp(ctx);
  686. op2->args[0] = tmp;
  687. op2->args[1] = *p1;
  688. op2->args[2] = *p2;
  689. *p1 = tmp;
  690. *p2 = arg_new_constant(ctx, 0);
  691. *pcond = tcg_tst_eqne_cond(cond);
  692. }
  693. return -1;
  694. }
  695. static int do_constant_folding_cond2(OptContext *ctx, TCGOp *op, TCGArg *args)
  696. {
  697. TCGArg al, ah, bl, bh;
  698. TCGCond c;
  699. bool swap;
  700. int r;
  701. swap = swap_commutative2(args, args + 2);
  702. c = args[4];
  703. if (swap) {
  704. args[4] = c = tcg_swap_cond(c);
  705. }
  706. al = args[0];
  707. ah = args[1];
  708. bl = args[2];
  709. bh = args[3];
  710. if (arg_is_const(bl) && arg_is_const(bh)) {
  711. tcg_target_ulong blv = arg_info(bl)->val;
  712. tcg_target_ulong bhv = arg_info(bh)->val;
  713. uint64_t b = deposit64(blv, 32, 32, bhv);
  714. if (arg_is_const(al) && arg_is_const(ah)) {
  715. tcg_target_ulong alv = arg_info(al)->val;
  716. tcg_target_ulong ahv = arg_info(ah)->val;
  717. uint64_t a = deposit64(alv, 32, 32, ahv);
  718. r = do_constant_folding_cond_64(a, b, c);
  719. if (r >= 0) {
  720. return r;
  721. }
  722. }
  723. if (b == 0) {
  724. switch (c) {
  725. case TCG_COND_LTU:
  726. case TCG_COND_TSTNE:
  727. return 0;
  728. case TCG_COND_GEU:
  729. case TCG_COND_TSTEQ:
  730. return 1;
  731. default:
  732. break;
  733. }
  734. }
  735. /* TSTNE x,-1 -> NE x,0 */
  736. if (b == -1 && is_tst_cond(c)) {
  737. args[3] = args[2] = arg_new_constant(ctx, 0);
  738. args[4] = tcg_tst_eqne_cond(c);
  739. return -1;
  740. }
  741. /* TSTNE x,sign -> LT x,0 */
  742. if (b == INT64_MIN && is_tst_cond(c)) {
  743. /* bl must be 0, so copy that to bh */
  744. args[3] = bl;
  745. args[4] = tcg_tst_ltge_cond(c);
  746. return -1;
  747. }
  748. }
  749. if (args_are_copies(al, bl) && args_are_copies(ah, bh)) {
  750. r = do_constant_folding_cond_eq(c);
  751. if (r >= 0) {
  752. return r;
  753. }
  754. /* TSTNE x,x -> NE x,0 */
  755. if (is_tst_cond(c)) {
  756. args[3] = args[2] = arg_new_constant(ctx, 0);
  757. args[4] = tcg_tst_eqne_cond(c);
  758. return -1;
  759. }
  760. }
  761. /* Expand to AND with a temporary if no backend support. */
  762. if (!TCG_TARGET_HAS_tst && is_tst_cond(c)) {
  763. TCGOp *op1 = tcg_op_insert_before(ctx->tcg, op, INDEX_op_and_i32, 3);
  764. TCGOp *op2 = tcg_op_insert_before(ctx->tcg, op, INDEX_op_and_i32, 3);
  765. TCGArg t1 = arg_new_temp(ctx);
  766. TCGArg t2 = arg_new_temp(ctx);
  767. op1->args[0] = t1;
  768. op1->args[1] = al;
  769. op1->args[2] = bl;
  770. op2->args[0] = t2;
  771. op2->args[1] = ah;
  772. op2->args[2] = bh;
  773. args[0] = t1;
  774. args[1] = t2;
  775. args[3] = args[2] = arg_new_constant(ctx, 0);
  776. args[4] = tcg_tst_eqne_cond(c);
  777. }
  778. return -1;
  779. }
  780. static void init_arguments(OptContext *ctx, TCGOp *op, int nb_args)
  781. {
  782. for (int i = 0; i < nb_args; i++) {
  783. TCGTemp *ts = arg_temp(op->args[i]);
  784. init_ts_info(ctx, ts);
  785. }
  786. }
  787. static void copy_propagate(OptContext *ctx, TCGOp *op,
  788. int nb_oargs, int nb_iargs)
  789. {
  790. for (int i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
  791. TCGTemp *ts = arg_temp(op->args[i]);
  792. if (ts_is_copy(ts)) {
  793. op->args[i] = temp_arg(find_better_copy(ts));
  794. }
  795. }
  796. }
  797. static void finish_bb(OptContext *ctx)
  798. {
  799. /* We only optimize memory barriers across basic blocks. */
  800. ctx->prev_mb = NULL;
  801. }
  802. static void finish_ebb(OptContext *ctx)
  803. {
  804. finish_bb(ctx);
  805. /* We only optimize across extended basic blocks. */
  806. memset(&ctx->temps_used, 0, sizeof(ctx->temps_used));
  807. remove_mem_copy_all(ctx);
  808. }
  809. static bool finish_folding(OptContext *ctx, TCGOp *op)
  810. {
  811. const TCGOpDef *def = &tcg_op_defs[op->opc];
  812. int i, nb_oargs;
  813. nb_oargs = def->nb_oargs;
  814. for (i = 0; i < nb_oargs; i++) {
  815. TCGTemp *ts = arg_temp(op->args[i]);
  816. reset_ts(ctx, ts);
  817. }
  818. return true;
  819. }
  820. /*
  821. * The fold_* functions return true when processing is complete,
  822. * usually by folding the operation to a constant or to a copy,
  823. * and calling tcg_opt_gen_{mov,movi}. They may do other things,
  824. * like collect information about the value produced, for use in
  825. * optimizing a subsequent operation.
  826. *
  827. * These first fold_* functions are all helpers, used by other
  828. * folders for more specific operations.
  829. */
  830. static bool fold_const1(OptContext *ctx, TCGOp *op)
  831. {
  832. if (arg_is_const(op->args[1])) {
  833. uint64_t t;
  834. t = arg_info(op->args[1])->val;
  835. t = do_constant_folding(op->opc, ctx->type, t, 0);
  836. return tcg_opt_gen_movi(ctx, op, op->args[0], t);
  837. }
  838. return false;
  839. }
  840. static bool fold_const2(OptContext *ctx, TCGOp *op)
  841. {
  842. if (arg_is_const(op->args[1]) && arg_is_const(op->args[2])) {
  843. uint64_t t1 = arg_info(op->args[1])->val;
  844. uint64_t t2 = arg_info(op->args[2])->val;
  845. t1 = do_constant_folding(op->opc, ctx->type, t1, t2);
  846. return tcg_opt_gen_movi(ctx, op, op->args[0], t1);
  847. }
  848. return false;
  849. }
  850. static bool fold_commutative(OptContext *ctx, TCGOp *op)
  851. {
  852. swap_commutative(op->args[0], &op->args[1], &op->args[2]);
  853. return false;
  854. }
  855. static bool fold_const2_commutative(OptContext *ctx, TCGOp *op)
  856. {
  857. swap_commutative(op->args[0], &op->args[1], &op->args[2]);
  858. return fold_const2(ctx, op);
  859. }
  860. /*
  861. * Record "zero" and "sign" masks for the single output of @op.
  862. * See TempOptInfo definition of z_mask and s_mask.
  863. * If z_mask allows, fold the output to constant zero.
  864. * The passed s_mask may be augmented by z_mask.
  865. */
  866. static bool fold_masks_zs(OptContext *ctx, TCGOp *op,
  867. uint64_t z_mask, int64_t s_mask)
  868. {
  869. const TCGOpDef *def = &tcg_op_defs[op->opc];
  870. TCGTemp *ts;
  871. TempOptInfo *ti;
  872. int rep;
  873. /* Only single-output opcodes are supported here. */
  874. tcg_debug_assert(def->nb_oargs == 1);
  875. /*
  876. * 32-bit ops generate 32-bit results, which for the purpose of
  877. * simplifying tcg are sign-extended. Certainly that's how we
  878. * represent our constants elsewhere. Note that the bits will
  879. * be reset properly for a 64-bit value when encountering the
  880. * type changing opcodes.
  881. */
  882. if (ctx->type == TCG_TYPE_I32) {
  883. z_mask = (int32_t)z_mask;
  884. s_mask |= INT32_MIN;
  885. }
  886. if (z_mask == 0) {
  887. return tcg_opt_gen_movi(ctx, op, op->args[0], 0);
  888. }
  889. ts = arg_temp(op->args[0]);
  890. reset_ts(ctx, ts);
  891. ti = ts_info(ts);
  892. ti->z_mask = z_mask;
  893. /* Canonicalize s_mask and incorporate data from z_mask. */
  894. rep = clz64(~s_mask);
  895. rep = MAX(rep, clz64(z_mask));
  896. rep = MAX(rep - 1, 0);
  897. ti->s_mask = INT64_MIN >> rep;
  898. return true;
  899. }
  900. static bool fold_masks_z(OptContext *ctx, TCGOp *op, uint64_t z_mask)
  901. {
  902. return fold_masks_zs(ctx, op, z_mask, 0);
  903. }
  904. static bool fold_masks_s(OptContext *ctx, TCGOp *op, uint64_t s_mask)
  905. {
  906. return fold_masks_zs(ctx, op, -1, s_mask);
  907. }
  908. /*
  909. * An "affected" mask bit is 0 if and only if the result is identical
  910. * to the first input. Thus if the entire mask is 0, the operation
  911. * is equivalent to a copy.
  912. */
  913. static bool fold_affected_mask(OptContext *ctx, TCGOp *op, uint64_t a_mask)
  914. {
  915. if (ctx->type == TCG_TYPE_I32) {
  916. a_mask = (uint32_t)a_mask;
  917. }
  918. if (a_mask == 0) {
  919. return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[1]);
  920. }
  921. return false;
  922. }
  923. /*
  924. * Convert @op to NOT, if NOT is supported by the host.
  925. * Return true f the conversion is successful, which will still
  926. * indicate that the processing is complete.
  927. */
  928. static bool fold_not(OptContext *ctx, TCGOp *op);
  929. static bool fold_to_not(OptContext *ctx, TCGOp *op, int idx)
  930. {
  931. TCGOpcode not_op;
  932. bool have_not;
  933. switch (ctx->type) {
  934. case TCG_TYPE_I32:
  935. not_op = INDEX_op_not_i32;
  936. have_not = TCG_TARGET_HAS_not_i32;
  937. break;
  938. case TCG_TYPE_I64:
  939. not_op = INDEX_op_not_i64;
  940. have_not = TCG_TARGET_HAS_not_i64;
  941. break;
  942. case TCG_TYPE_V64:
  943. case TCG_TYPE_V128:
  944. case TCG_TYPE_V256:
  945. not_op = INDEX_op_not_vec;
  946. have_not = TCG_TARGET_HAS_not_vec;
  947. break;
  948. default:
  949. g_assert_not_reached();
  950. }
  951. if (have_not) {
  952. op->opc = not_op;
  953. op->args[1] = op->args[idx];
  954. return fold_not(ctx, op);
  955. }
  956. return false;
  957. }
  958. /* If the binary operation has first argument @i, fold to @i. */
  959. static bool fold_ix_to_i(OptContext *ctx, TCGOp *op, uint64_t i)
  960. {
  961. if (arg_is_const_val(op->args[1], i)) {
  962. return tcg_opt_gen_movi(ctx, op, op->args[0], i);
  963. }
  964. return false;
  965. }
  966. /* If the binary operation has first argument @i, fold to NOT. */
  967. static bool fold_ix_to_not(OptContext *ctx, TCGOp *op, uint64_t i)
  968. {
  969. if (arg_is_const_val(op->args[1], i)) {
  970. return fold_to_not(ctx, op, 2);
  971. }
  972. return false;
  973. }
  974. /* If the binary operation has second argument @i, fold to @i. */
  975. static bool fold_xi_to_i(OptContext *ctx, TCGOp *op, uint64_t i)
  976. {
  977. if (arg_is_const_val(op->args[2], i)) {
  978. return tcg_opt_gen_movi(ctx, op, op->args[0], i);
  979. }
  980. return false;
  981. }
  982. /* If the binary operation has second argument @i, fold to identity. */
  983. static bool fold_xi_to_x(OptContext *ctx, TCGOp *op, uint64_t i)
  984. {
  985. if (arg_is_const_val(op->args[2], i)) {
  986. return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[1]);
  987. }
  988. return false;
  989. }
  990. /* If the binary operation has second argument @i, fold to NOT. */
  991. static bool fold_xi_to_not(OptContext *ctx, TCGOp *op, uint64_t i)
  992. {
  993. if (arg_is_const_val(op->args[2], i)) {
  994. return fold_to_not(ctx, op, 1);
  995. }
  996. return false;
  997. }
  998. /* If the binary operation has both arguments equal, fold to @i. */
  999. static bool fold_xx_to_i(OptContext *ctx, TCGOp *op, uint64_t i)
  1000. {
  1001. if (args_are_copies(op->args[1], op->args[2])) {
  1002. return tcg_opt_gen_movi(ctx, op, op->args[0], i);
  1003. }
  1004. return false;
  1005. }
  1006. /* If the binary operation has both arguments equal, fold to identity. */
  1007. static bool fold_xx_to_x(OptContext *ctx, TCGOp *op)
  1008. {
  1009. if (args_are_copies(op->args[1], op->args[2])) {
  1010. return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[1]);
  1011. }
  1012. return false;
  1013. }
  1014. /*
  1015. * These outermost fold_<op> functions are sorted alphabetically.
  1016. *
  1017. * The ordering of the transformations should be:
  1018. * 1) those that produce a constant
  1019. * 2) those that produce a copy
  1020. * 3) those that produce information about the result value.
  1021. */
  1022. static bool fold_or(OptContext *ctx, TCGOp *op);
  1023. static bool fold_orc(OptContext *ctx, TCGOp *op);
  1024. static bool fold_xor(OptContext *ctx, TCGOp *op);
  1025. static bool fold_add(OptContext *ctx, TCGOp *op)
  1026. {
  1027. if (fold_const2_commutative(ctx, op) ||
  1028. fold_xi_to_x(ctx, op, 0)) {
  1029. return true;
  1030. }
  1031. return finish_folding(ctx, op);
  1032. }
  1033. /* We cannot as yet do_constant_folding with vectors. */
  1034. static bool fold_add_vec(OptContext *ctx, TCGOp *op)
  1035. {
  1036. if (fold_commutative(ctx, op) ||
  1037. fold_xi_to_x(ctx, op, 0)) {
  1038. return true;
  1039. }
  1040. return finish_folding(ctx, op);
  1041. }
  1042. static bool fold_addsub2(OptContext *ctx, TCGOp *op, bool add)
  1043. {
  1044. bool a_const = arg_is_const(op->args[2]) && arg_is_const(op->args[3]);
  1045. bool b_const = arg_is_const(op->args[4]) && arg_is_const(op->args[5]);
  1046. if (a_const && b_const) {
  1047. uint64_t al = arg_info(op->args[2])->val;
  1048. uint64_t ah = arg_info(op->args[3])->val;
  1049. uint64_t bl = arg_info(op->args[4])->val;
  1050. uint64_t bh = arg_info(op->args[5])->val;
  1051. TCGArg rl, rh;
  1052. TCGOp *op2;
  1053. if (ctx->type == TCG_TYPE_I32) {
  1054. uint64_t a = deposit64(al, 32, 32, ah);
  1055. uint64_t b = deposit64(bl, 32, 32, bh);
  1056. if (add) {
  1057. a += b;
  1058. } else {
  1059. a -= b;
  1060. }
  1061. al = sextract64(a, 0, 32);
  1062. ah = sextract64(a, 32, 32);
  1063. } else {
  1064. Int128 a = int128_make128(al, ah);
  1065. Int128 b = int128_make128(bl, bh);
  1066. if (add) {
  1067. a = int128_add(a, b);
  1068. } else {
  1069. a = int128_sub(a, b);
  1070. }
  1071. al = int128_getlo(a);
  1072. ah = int128_gethi(a);
  1073. }
  1074. rl = op->args[0];
  1075. rh = op->args[1];
  1076. /* The proper opcode is supplied by tcg_opt_gen_mov. */
  1077. op2 = tcg_op_insert_before(ctx->tcg, op, 0, 2);
  1078. tcg_opt_gen_movi(ctx, op, rl, al);
  1079. tcg_opt_gen_movi(ctx, op2, rh, ah);
  1080. return true;
  1081. }
  1082. /* Fold sub2 r,x,i to add2 r,x,-i */
  1083. if (!add && b_const) {
  1084. uint64_t bl = arg_info(op->args[4])->val;
  1085. uint64_t bh = arg_info(op->args[5])->val;
  1086. /* Negate the two parts without assembling and disassembling. */
  1087. bl = -bl;
  1088. bh = ~bh + !bl;
  1089. op->opc = (ctx->type == TCG_TYPE_I32
  1090. ? INDEX_op_add2_i32 : INDEX_op_add2_i64);
  1091. op->args[4] = arg_new_constant(ctx, bl);
  1092. op->args[5] = arg_new_constant(ctx, bh);
  1093. }
  1094. return finish_folding(ctx, op);
  1095. }
  1096. static bool fold_add2(OptContext *ctx, TCGOp *op)
  1097. {
  1098. /* Note that the high and low parts may be independently swapped. */
  1099. swap_commutative(op->args[0], &op->args[2], &op->args[4]);
  1100. swap_commutative(op->args[1], &op->args[3], &op->args[5]);
  1101. return fold_addsub2(ctx, op, true);
  1102. }
  1103. static bool fold_and(OptContext *ctx, TCGOp *op)
  1104. {
  1105. uint64_t z1, z2, z_mask, s_mask;
  1106. TempOptInfo *t1, *t2;
  1107. if (fold_const2_commutative(ctx, op) ||
  1108. fold_xi_to_i(ctx, op, 0) ||
  1109. fold_xi_to_x(ctx, op, -1) ||
  1110. fold_xx_to_x(ctx, op)) {
  1111. return true;
  1112. }
  1113. t1 = arg_info(op->args[1]);
  1114. t2 = arg_info(op->args[2]);
  1115. z1 = t1->z_mask;
  1116. z2 = t2->z_mask;
  1117. /*
  1118. * Known-zeros does not imply known-ones. Therefore unless
  1119. * arg2 is constant, we can't infer affected bits from it.
  1120. */
  1121. if (ti_is_const(t2) && fold_affected_mask(ctx, op, z1 & ~z2)) {
  1122. return true;
  1123. }
  1124. z_mask = z1 & z2;
  1125. /*
  1126. * Sign repetitions are perforce all identical, whether they are 1 or 0.
  1127. * Bitwise operations preserve the relative quantity of the repetitions.
  1128. */
  1129. s_mask = t1->s_mask & t2->s_mask;
  1130. return fold_masks_zs(ctx, op, z_mask, s_mask);
  1131. }
  1132. static bool fold_andc(OptContext *ctx, TCGOp *op)
  1133. {
  1134. uint64_t z_mask, s_mask;
  1135. TempOptInfo *t1, *t2;
  1136. if (fold_const2(ctx, op) ||
  1137. fold_xx_to_i(ctx, op, 0) ||
  1138. fold_xi_to_x(ctx, op, 0) ||
  1139. fold_ix_to_not(ctx, op, -1)) {
  1140. return true;
  1141. }
  1142. t1 = arg_info(op->args[1]);
  1143. t2 = arg_info(op->args[2]);
  1144. z_mask = t1->z_mask;
  1145. /*
  1146. * Known-zeros does not imply known-ones. Therefore unless
  1147. * arg2 is constant, we can't infer anything from it.
  1148. */
  1149. if (ti_is_const(t2)) {
  1150. uint64_t v2 = ti_const_val(t2);
  1151. if (fold_affected_mask(ctx, op, z_mask & v2)) {
  1152. return true;
  1153. }
  1154. z_mask &= ~v2;
  1155. }
  1156. s_mask = t1->s_mask & t2->s_mask;
  1157. return fold_masks_zs(ctx, op, z_mask, s_mask);
  1158. }
  1159. static bool fold_bitsel_vec(OptContext *ctx, TCGOp *op)
  1160. {
  1161. /* If true and false values are the same, eliminate the cmp. */
  1162. if (args_are_copies(op->args[2], op->args[3])) {
  1163. return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[2]);
  1164. }
  1165. if (arg_is_const(op->args[2]) && arg_is_const(op->args[3])) {
  1166. uint64_t tv = arg_info(op->args[2])->val;
  1167. uint64_t fv = arg_info(op->args[3])->val;
  1168. if (tv == -1 && fv == 0) {
  1169. return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[1]);
  1170. }
  1171. if (tv == 0 && fv == -1) {
  1172. if (TCG_TARGET_HAS_not_vec) {
  1173. op->opc = INDEX_op_not_vec;
  1174. return fold_not(ctx, op);
  1175. } else {
  1176. op->opc = INDEX_op_xor_vec;
  1177. op->args[2] = arg_new_constant(ctx, -1);
  1178. return fold_xor(ctx, op);
  1179. }
  1180. }
  1181. }
  1182. if (arg_is_const(op->args[2])) {
  1183. uint64_t tv = arg_info(op->args[2])->val;
  1184. if (tv == -1) {
  1185. op->opc = INDEX_op_or_vec;
  1186. op->args[2] = op->args[3];
  1187. return fold_or(ctx, op);
  1188. }
  1189. if (tv == 0 && TCG_TARGET_HAS_andc_vec) {
  1190. op->opc = INDEX_op_andc_vec;
  1191. op->args[2] = op->args[1];
  1192. op->args[1] = op->args[3];
  1193. return fold_andc(ctx, op);
  1194. }
  1195. }
  1196. if (arg_is_const(op->args[3])) {
  1197. uint64_t fv = arg_info(op->args[3])->val;
  1198. if (fv == 0) {
  1199. op->opc = INDEX_op_and_vec;
  1200. return fold_and(ctx, op);
  1201. }
  1202. if (fv == -1 && TCG_TARGET_HAS_orc_vec) {
  1203. op->opc = INDEX_op_orc_vec;
  1204. op->args[2] = op->args[1];
  1205. op->args[1] = op->args[3];
  1206. return fold_orc(ctx, op);
  1207. }
  1208. }
  1209. return finish_folding(ctx, op);
  1210. }
  1211. static bool fold_brcond(OptContext *ctx, TCGOp *op)
  1212. {
  1213. int i = do_constant_folding_cond1(ctx, op, NO_DEST, &op->args[0],
  1214. &op->args[1], &op->args[2]);
  1215. if (i == 0) {
  1216. tcg_op_remove(ctx->tcg, op);
  1217. return true;
  1218. }
  1219. if (i > 0) {
  1220. op->opc = INDEX_op_br;
  1221. op->args[0] = op->args[3];
  1222. finish_ebb(ctx);
  1223. } else {
  1224. finish_bb(ctx);
  1225. }
  1226. return true;
  1227. }
  1228. static bool fold_brcond2(OptContext *ctx, TCGOp *op)
  1229. {
  1230. TCGCond cond;
  1231. TCGArg label;
  1232. int i, inv = 0;
  1233. i = do_constant_folding_cond2(ctx, op, &op->args[0]);
  1234. cond = op->args[4];
  1235. label = op->args[5];
  1236. if (i >= 0) {
  1237. goto do_brcond_const;
  1238. }
  1239. switch (cond) {
  1240. case TCG_COND_LT:
  1241. case TCG_COND_GE:
  1242. /*
  1243. * Simplify LT/GE comparisons vs zero to a single compare
  1244. * vs the high word of the input.
  1245. */
  1246. if (arg_is_const_val(op->args[2], 0) &&
  1247. arg_is_const_val(op->args[3], 0)) {
  1248. goto do_brcond_high;
  1249. }
  1250. break;
  1251. case TCG_COND_NE:
  1252. inv = 1;
  1253. QEMU_FALLTHROUGH;
  1254. case TCG_COND_EQ:
  1255. /*
  1256. * Simplify EQ/NE comparisons where one of the pairs
  1257. * can be simplified.
  1258. */
  1259. i = do_constant_folding_cond(TCG_TYPE_I32, op->args[0],
  1260. op->args[2], cond);
  1261. switch (i ^ inv) {
  1262. case 0:
  1263. goto do_brcond_const;
  1264. case 1:
  1265. goto do_brcond_high;
  1266. }
  1267. i = do_constant_folding_cond(TCG_TYPE_I32, op->args[1],
  1268. op->args[3], cond);
  1269. switch (i ^ inv) {
  1270. case 0:
  1271. goto do_brcond_const;
  1272. case 1:
  1273. goto do_brcond_low;
  1274. }
  1275. break;
  1276. case TCG_COND_TSTEQ:
  1277. case TCG_COND_TSTNE:
  1278. if (arg_is_const_val(op->args[2], 0)) {
  1279. goto do_brcond_high;
  1280. }
  1281. if (arg_is_const_val(op->args[3], 0)) {
  1282. goto do_brcond_low;
  1283. }
  1284. break;
  1285. default:
  1286. break;
  1287. do_brcond_low:
  1288. op->opc = INDEX_op_brcond_i32;
  1289. op->args[1] = op->args[2];
  1290. op->args[2] = cond;
  1291. op->args[3] = label;
  1292. return fold_brcond(ctx, op);
  1293. do_brcond_high:
  1294. op->opc = INDEX_op_brcond_i32;
  1295. op->args[0] = op->args[1];
  1296. op->args[1] = op->args[3];
  1297. op->args[2] = cond;
  1298. op->args[3] = label;
  1299. return fold_brcond(ctx, op);
  1300. do_brcond_const:
  1301. if (i == 0) {
  1302. tcg_op_remove(ctx->tcg, op);
  1303. return true;
  1304. }
  1305. op->opc = INDEX_op_br;
  1306. op->args[0] = label;
  1307. finish_ebb(ctx);
  1308. return true;
  1309. }
  1310. finish_bb(ctx);
  1311. return true;
  1312. }
  1313. static bool fold_bswap(OptContext *ctx, TCGOp *op)
  1314. {
  1315. uint64_t z_mask, s_mask, sign;
  1316. TempOptInfo *t1 = arg_info(op->args[1]);
  1317. if (ti_is_const(t1)) {
  1318. return tcg_opt_gen_movi(ctx, op, op->args[0],
  1319. do_constant_folding(op->opc, ctx->type,
  1320. ti_const_val(t1),
  1321. op->args[2]));
  1322. }
  1323. z_mask = t1->z_mask;
  1324. switch (op->opc) {
  1325. case INDEX_op_bswap16_i32:
  1326. case INDEX_op_bswap16_i64:
  1327. z_mask = bswap16(z_mask);
  1328. sign = INT16_MIN;
  1329. break;
  1330. case INDEX_op_bswap32_i32:
  1331. case INDEX_op_bswap32_i64:
  1332. z_mask = bswap32(z_mask);
  1333. sign = INT32_MIN;
  1334. break;
  1335. case INDEX_op_bswap64_i64:
  1336. z_mask = bswap64(z_mask);
  1337. sign = INT64_MIN;
  1338. break;
  1339. default:
  1340. g_assert_not_reached();
  1341. }
  1342. s_mask = 0;
  1343. switch (op->args[2] & (TCG_BSWAP_OZ | TCG_BSWAP_OS)) {
  1344. case TCG_BSWAP_OZ:
  1345. break;
  1346. case TCG_BSWAP_OS:
  1347. /* If the sign bit may be 1, force all the bits above to 1. */
  1348. if (z_mask & sign) {
  1349. z_mask |= sign;
  1350. }
  1351. /* The value and therefore s_mask is explicitly sign-extended. */
  1352. s_mask = sign;
  1353. break;
  1354. default:
  1355. /* The high bits are undefined: force all bits above the sign to 1. */
  1356. z_mask |= sign << 1;
  1357. break;
  1358. }
  1359. return fold_masks_zs(ctx, op, z_mask, s_mask);
  1360. }
  1361. static bool fold_call(OptContext *ctx, TCGOp *op)
  1362. {
  1363. TCGContext *s = ctx->tcg;
  1364. int nb_oargs = TCGOP_CALLO(op);
  1365. int nb_iargs = TCGOP_CALLI(op);
  1366. int flags, i;
  1367. init_arguments(ctx, op, nb_oargs + nb_iargs);
  1368. copy_propagate(ctx, op, nb_oargs, nb_iargs);
  1369. /* If the function reads or writes globals, reset temp data. */
  1370. flags = tcg_call_flags(op);
  1371. if (!(flags & (TCG_CALL_NO_READ_GLOBALS | TCG_CALL_NO_WRITE_GLOBALS))) {
  1372. int nb_globals = s->nb_globals;
  1373. for (i = 0; i < nb_globals; i++) {
  1374. if (test_bit(i, ctx->temps_used.l)) {
  1375. reset_ts(ctx, &ctx->tcg->temps[i]);
  1376. }
  1377. }
  1378. }
  1379. /* If the function has side effects, reset mem data. */
  1380. if (!(flags & TCG_CALL_NO_SIDE_EFFECTS)) {
  1381. remove_mem_copy_all(ctx);
  1382. }
  1383. /* Reset temp data for outputs. */
  1384. for (i = 0; i < nb_oargs; i++) {
  1385. reset_temp(ctx, op->args[i]);
  1386. }
  1387. /* Stop optimizing MB across calls. */
  1388. ctx->prev_mb = NULL;
  1389. return true;
  1390. }
  1391. static bool fold_cmp_vec(OptContext *ctx, TCGOp *op)
  1392. {
  1393. /* Canonicalize the comparison to put immediate second. */
  1394. if (swap_commutative(NO_DEST, &op->args[1], &op->args[2])) {
  1395. op->args[3] = tcg_swap_cond(op->args[3]);
  1396. }
  1397. return finish_folding(ctx, op);
  1398. }
  1399. static bool fold_cmpsel_vec(OptContext *ctx, TCGOp *op)
  1400. {
  1401. /* If true and false values are the same, eliminate the cmp. */
  1402. if (args_are_copies(op->args[3], op->args[4])) {
  1403. return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[3]);
  1404. }
  1405. /* Canonicalize the comparison to put immediate second. */
  1406. if (swap_commutative(NO_DEST, &op->args[1], &op->args[2])) {
  1407. op->args[5] = tcg_swap_cond(op->args[5]);
  1408. }
  1409. /*
  1410. * Canonicalize the "false" input reg to match the destination,
  1411. * so that the tcg backend can implement "move if true".
  1412. */
  1413. if (swap_commutative(op->args[0], &op->args[4], &op->args[3])) {
  1414. op->args[5] = tcg_invert_cond(op->args[5]);
  1415. }
  1416. return finish_folding(ctx, op);
  1417. }
  1418. static bool fold_count_zeros(OptContext *ctx, TCGOp *op)
  1419. {
  1420. uint64_t z_mask, s_mask;
  1421. TempOptInfo *t1 = arg_info(op->args[1]);
  1422. TempOptInfo *t2 = arg_info(op->args[2]);
  1423. if (ti_is_const(t1)) {
  1424. uint64_t t = ti_const_val(t1);
  1425. if (t != 0) {
  1426. t = do_constant_folding(op->opc, ctx->type, t, 0);
  1427. return tcg_opt_gen_movi(ctx, op, op->args[0], t);
  1428. }
  1429. return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[2]);
  1430. }
  1431. switch (ctx->type) {
  1432. case TCG_TYPE_I32:
  1433. z_mask = 31;
  1434. break;
  1435. case TCG_TYPE_I64:
  1436. z_mask = 63;
  1437. break;
  1438. default:
  1439. g_assert_not_reached();
  1440. }
  1441. s_mask = ~z_mask;
  1442. z_mask |= t2->z_mask;
  1443. s_mask &= t2->s_mask;
  1444. return fold_masks_zs(ctx, op, z_mask, s_mask);
  1445. }
  1446. static bool fold_ctpop(OptContext *ctx, TCGOp *op)
  1447. {
  1448. uint64_t z_mask;
  1449. if (fold_const1(ctx, op)) {
  1450. return true;
  1451. }
  1452. switch (ctx->type) {
  1453. case TCG_TYPE_I32:
  1454. z_mask = 32 | 31;
  1455. break;
  1456. case TCG_TYPE_I64:
  1457. z_mask = 64 | 63;
  1458. break;
  1459. default:
  1460. g_assert_not_reached();
  1461. }
  1462. return fold_masks_z(ctx, op, z_mask);
  1463. }
  1464. static bool fold_deposit(OptContext *ctx, TCGOp *op)
  1465. {
  1466. TempOptInfo *t1 = arg_info(op->args[1]);
  1467. TempOptInfo *t2 = arg_info(op->args[2]);
  1468. int ofs = op->args[3];
  1469. int len = op->args[4];
  1470. int width;
  1471. TCGOpcode and_opc;
  1472. uint64_t z_mask, s_mask;
  1473. if (ti_is_const(t1) && ti_is_const(t2)) {
  1474. return tcg_opt_gen_movi(ctx, op, op->args[0],
  1475. deposit64(ti_const_val(t1), ofs, len,
  1476. ti_const_val(t2)));
  1477. }
  1478. switch (ctx->type) {
  1479. case TCG_TYPE_I32:
  1480. and_opc = INDEX_op_and_i32;
  1481. width = 32;
  1482. break;
  1483. case TCG_TYPE_I64:
  1484. and_opc = INDEX_op_and_i64;
  1485. width = 64;
  1486. break;
  1487. default:
  1488. g_assert_not_reached();
  1489. }
  1490. /* Inserting a value into zero at offset 0. */
  1491. if (ti_is_const_val(t1, 0) && ofs == 0) {
  1492. uint64_t mask = MAKE_64BIT_MASK(0, len);
  1493. op->opc = and_opc;
  1494. op->args[1] = op->args[2];
  1495. op->args[2] = arg_new_constant(ctx, mask);
  1496. return fold_and(ctx, op);
  1497. }
  1498. /* Inserting zero into a value. */
  1499. if (ti_is_const_val(t2, 0)) {
  1500. uint64_t mask = deposit64(-1, ofs, len, 0);
  1501. op->opc = and_opc;
  1502. op->args[2] = arg_new_constant(ctx, mask);
  1503. return fold_and(ctx, op);
  1504. }
  1505. /* The s_mask from the top portion of the deposit is still valid. */
  1506. if (ofs + len == width) {
  1507. s_mask = t2->s_mask << ofs;
  1508. } else {
  1509. s_mask = t1->s_mask & ~MAKE_64BIT_MASK(0, ofs + len);
  1510. }
  1511. z_mask = deposit64(t1->z_mask, ofs, len, t2->z_mask);
  1512. return fold_masks_zs(ctx, op, z_mask, s_mask);
  1513. }
  1514. static bool fold_divide(OptContext *ctx, TCGOp *op)
  1515. {
  1516. if (fold_const2(ctx, op) ||
  1517. fold_xi_to_x(ctx, op, 1)) {
  1518. return true;
  1519. }
  1520. return finish_folding(ctx, op);
  1521. }
  1522. static bool fold_dup(OptContext *ctx, TCGOp *op)
  1523. {
  1524. if (arg_is_const(op->args[1])) {
  1525. uint64_t t = arg_info(op->args[1])->val;
  1526. t = dup_const(TCGOP_VECE(op), t);
  1527. return tcg_opt_gen_movi(ctx, op, op->args[0], t);
  1528. }
  1529. return finish_folding(ctx, op);
  1530. }
  1531. static bool fold_dup2(OptContext *ctx, TCGOp *op)
  1532. {
  1533. if (arg_is_const(op->args[1]) && arg_is_const(op->args[2])) {
  1534. uint64_t t = deposit64(arg_info(op->args[1])->val, 32, 32,
  1535. arg_info(op->args[2])->val);
  1536. return tcg_opt_gen_movi(ctx, op, op->args[0], t);
  1537. }
  1538. if (args_are_copies(op->args[1], op->args[2])) {
  1539. op->opc = INDEX_op_dup_vec;
  1540. TCGOP_VECE(op) = MO_32;
  1541. }
  1542. return finish_folding(ctx, op);
  1543. }
  1544. static bool fold_eqv(OptContext *ctx, TCGOp *op)
  1545. {
  1546. uint64_t s_mask;
  1547. if (fold_const2_commutative(ctx, op) ||
  1548. fold_xi_to_x(ctx, op, -1) ||
  1549. fold_xi_to_not(ctx, op, 0)) {
  1550. return true;
  1551. }
  1552. s_mask = arg_info(op->args[1])->s_mask
  1553. & arg_info(op->args[2])->s_mask;
  1554. return fold_masks_s(ctx, op, s_mask);
  1555. }
  1556. static bool fold_extract(OptContext *ctx, TCGOp *op)
  1557. {
  1558. uint64_t z_mask_old, z_mask;
  1559. TempOptInfo *t1 = arg_info(op->args[1]);
  1560. int pos = op->args[2];
  1561. int len = op->args[3];
  1562. if (ti_is_const(t1)) {
  1563. return tcg_opt_gen_movi(ctx, op, op->args[0],
  1564. extract64(ti_const_val(t1), pos, len));
  1565. }
  1566. z_mask_old = t1->z_mask;
  1567. z_mask = extract64(z_mask_old, pos, len);
  1568. if (pos == 0 && fold_affected_mask(ctx, op, z_mask_old ^ z_mask)) {
  1569. return true;
  1570. }
  1571. return fold_masks_z(ctx, op, z_mask);
  1572. }
  1573. static bool fold_extract2(OptContext *ctx, TCGOp *op)
  1574. {
  1575. if (arg_is_const(op->args[1]) && arg_is_const(op->args[2])) {
  1576. uint64_t v1 = arg_info(op->args[1])->val;
  1577. uint64_t v2 = arg_info(op->args[2])->val;
  1578. int shr = op->args[3];
  1579. if (op->opc == INDEX_op_extract2_i64) {
  1580. v1 >>= shr;
  1581. v2 <<= 64 - shr;
  1582. } else {
  1583. v1 = (uint32_t)v1 >> shr;
  1584. v2 = (uint64_t)((int32_t)v2 << (32 - shr));
  1585. }
  1586. return tcg_opt_gen_movi(ctx, op, op->args[0], v1 | v2);
  1587. }
  1588. return finish_folding(ctx, op);
  1589. }
  1590. static bool fold_exts(OptContext *ctx, TCGOp *op)
  1591. {
  1592. uint64_t s_mask_old, s_mask, z_mask;
  1593. bool type_change = false;
  1594. TempOptInfo *t1;
  1595. if (fold_const1(ctx, op)) {
  1596. return true;
  1597. }
  1598. t1 = arg_info(op->args[1]);
  1599. z_mask = t1->z_mask;
  1600. s_mask = t1->s_mask;
  1601. s_mask_old = s_mask;
  1602. switch (op->opc) {
  1603. CASE_OP_32_64(ext8s):
  1604. s_mask |= INT8_MIN;
  1605. z_mask = (int8_t)z_mask;
  1606. break;
  1607. CASE_OP_32_64(ext16s):
  1608. s_mask |= INT16_MIN;
  1609. z_mask = (int16_t)z_mask;
  1610. break;
  1611. case INDEX_op_ext_i32_i64:
  1612. type_change = true;
  1613. QEMU_FALLTHROUGH;
  1614. case INDEX_op_ext32s_i64:
  1615. s_mask |= INT32_MIN;
  1616. z_mask = (int32_t)z_mask;
  1617. break;
  1618. default:
  1619. g_assert_not_reached();
  1620. }
  1621. if (!type_change && fold_affected_mask(ctx, op, s_mask & ~s_mask_old)) {
  1622. return true;
  1623. }
  1624. return fold_masks_zs(ctx, op, z_mask, s_mask);
  1625. }
  1626. static bool fold_extu(OptContext *ctx, TCGOp *op)
  1627. {
  1628. uint64_t z_mask_old, z_mask;
  1629. bool type_change = false;
  1630. if (fold_const1(ctx, op)) {
  1631. return true;
  1632. }
  1633. z_mask_old = z_mask = arg_info(op->args[1])->z_mask;
  1634. switch (op->opc) {
  1635. CASE_OP_32_64(ext8u):
  1636. z_mask = (uint8_t)z_mask;
  1637. break;
  1638. CASE_OP_32_64(ext16u):
  1639. z_mask = (uint16_t)z_mask;
  1640. break;
  1641. case INDEX_op_extrl_i64_i32:
  1642. case INDEX_op_extu_i32_i64:
  1643. type_change = true;
  1644. QEMU_FALLTHROUGH;
  1645. case INDEX_op_ext32u_i64:
  1646. z_mask = (uint32_t)z_mask;
  1647. break;
  1648. case INDEX_op_extrh_i64_i32:
  1649. type_change = true;
  1650. z_mask >>= 32;
  1651. break;
  1652. default:
  1653. g_assert_not_reached();
  1654. }
  1655. if (!type_change && fold_affected_mask(ctx, op, z_mask_old ^ z_mask)) {
  1656. return true;
  1657. }
  1658. return fold_masks_z(ctx, op, z_mask);
  1659. }
  1660. static bool fold_mb(OptContext *ctx, TCGOp *op)
  1661. {
  1662. /* Eliminate duplicate and redundant fence instructions. */
  1663. if (ctx->prev_mb) {
  1664. /*
  1665. * Merge two barriers of the same type into one,
  1666. * or a weaker barrier into a stronger one,
  1667. * or two weaker barriers into a stronger one.
  1668. * mb X; mb Y => mb X|Y
  1669. * mb; strl => mb; st
  1670. * ldaq; mb => ld; mb
  1671. * ldaq; strl => ld; mb; st
  1672. * Other combinations are also merged into a strong
  1673. * barrier. This is stricter than specified but for
  1674. * the purposes of TCG is better than not optimizing.
  1675. */
  1676. ctx->prev_mb->args[0] |= op->args[0];
  1677. tcg_op_remove(ctx->tcg, op);
  1678. } else {
  1679. ctx->prev_mb = op;
  1680. }
  1681. return true;
  1682. }
  1683. static bool fold_mov(OptContext *ctx, TCGOp *op)
  1684. {
  1685. return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[1]);
  1686. }
  1687. static bool fold_movcond(OptContext *ctx, TCGOp *op)
  1688. {
  1689. uint64_t z_mask, s_mask;
  1690. TempOptInfo *tt, *ft;
  1691. int i;
  1692. /* If true and false values are the same, eliminate the cmp. */
  1693. if (args_are_copies(op->args[3], op->args[4])) {
  1694. return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[3]);
  1695. }
  1696. /*
  1697. * Canonicalize the "false" input reg to match the destination reg so
  1698. * that the tcg backend can implement a "move if true" operation.
  1699. */
  1700. if (swap_commutative(op->args[0], &op->args[4], &op->args[3])) {
  1701. op->args[5] = tcg_invert_cond(op->args[5]);
  1702. }
  1703. i = do_constant_folding_cond1(ctx, op, NO_DEST, &op->args[1],
  1704. &op->args[2], &op->args[5]);
  1705. if (i >= 0) {
  1706. return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[4 - i]);
  1707. }
  1708. tt = arg_info(op->args[3]);
  1709. ft = arg_info(op->args[4]);
  1710. z_mask = tt->z_mask | ft->z_mask;
  1711. s_mask = tt->s_mask & ft->s_mask;
  1712. if (ti_is_const(tt) && ti_is_const(ft)) {
  1713. uint64_t tv = ti_const_val(tt);
  1714. uint64_t fv = ti_const_val(ft);
  1715. TCGOpcode opc, negopc = 0;
  1716. TCGCond cond = op->args[5];
  1717. switch (ctx->type) {
  1718. case TCG_TYPE_I32:
  1719. opc = INDEX_op_setcond_i32;
  1720. if (TCG_TARGET_HAS_negsetcond_i32) {
  1721. negopc = INDEX_op_negsetcond_i32;
  1722. }
  1723. tv = (int32_t)tv;
  1724. fv = (int32_t)fv;
  1725. break;
  1726. case TCG_TYPE_I64:
  1727. opc = INDEX_op_setcond_i64;
  1728. if (TCG_TARGET_HAS_negsetcond_i64) {
  1729. negopc = INDEX_op_negsetcond_i64;
  1730. }
  1731. break;
  1732. default:
  1733. g_assert_not_reached();
  1734. }
  1735. if (tv == 1 && fv == 0) {
  1736. op->opc = opc;
  1737. op->args[3] = cond;
  1738. } else if (fv == 1 && tv == 0) {
  1739. op->opc = opc;
  1740. op->args[3] = tcg_invert_cond(cond);
  1741. } else if (negopc) {
  1742. if (tv == -1 && fv == 0) {
  1743. op->opc = negopc;
  1744. op->args[3] = cond;
  1745. } else if (fv == -1 && tv == 0) {
  1746. op->opc = negopc;
  1747. op->args[3] = tcg_invert_cond(cond);
  1748. }
  1749. }
  1750. }
  1751. return fold_masks_zs(ctx, op, z_mask, s_mask);
  1752. }
  1753. static bool fold_mul(OptContext *ctx, TCGOp *op)
  1754. {
  1755. if (fold_const2(ctx, op) ||
  1756. fold_xi_to_i(ctx, op, 0) ||
  1757. fold_xi_to_x(ctx, op, 1)) {
  1758. return true;
  1759. }
  1760. return finish_folding(ctx, op);
  1761. }
  1762. static bool fold_mul_highpart(OptContext *ctx, TCGOp *op)
  1763. {
  1764. if (fold_const2_commutative(ctx, op) ||
  1765. fold_xi_to_i(ctx, op, 0)) {
  1766. return true;
  1767. }
  1768. return finish_folding(ctx, op);
  1769. }
  1770. static bool fold_multiply2(OptContext *ctx, TCGOp *op)
  1771. {
  1772. swap_commutative(op->args[0], &op->args[2], &op->args[3]);
  1773. if (arg_is_const(op->args[2]) && arg_is_const(op->args[3])) {
  1774. uint64_t a = arg_info(op->args[2])->val;
  1775. uint64_t b = arg_info(op->args[3])->val;
  1776. uint64_t h, l;
  1777. TCGArg rl, rh;
  1778. TCGOp *op2;
  1779. switch (op->opc) {
  1780. case INDEX_op_mulu2_i32:
  1781. l = (uint64_t)(uint32_t)a * (uint32_t)b;
  1782. h = (int32_t)(l >> 32);
  1783. l = (int32_t)l;
  1784. break;
  1785. case INDEX_op_muls2_i32:
  1786. l = (int64_t)(int32_t)a * (int32_t)b;
  1787. h = l >> 32;
  1788. l = (int32_t)l;
  1789. break;
  1790. case INDEX_op_mulu2_i64:
  1791. mulu64(&l, &h, a, b);
  1792. break;
  1793. case INDEX_op_muls2_i64:
  1794. muls64(&l, &h, a, b);
  1795. break;
  1796. default:
  1797. g_assert_not_reached();
  1798. }
  1799. rl = op->args[0];
  1800. rh = op->args[1];
  1801. /* The proper opcode is supplied by tcg_opt_gen_mov. */
  1802. op2 = tcg_op_insert_before(ctx->tcg, op, 0, 2);
  1803. tcg_opt_gen_movi(ctx, op, rl, l);
  1804. tcg_opt_gen_movi(ctx, op2, rh, h);
  1805. return true;
  1806. }
  1807. return finish_folding(ctx, op);
  1808. }
  1809. static bool fold_nand(OptContext *ctx, TCGOp *op)
  1810. {
  1811. uint64_t s_mask;
  1812. if (fold_const2_commutative(ctx, op) ||
  1813. fold_xi_to_not(ctx, op, -1)) {
  1814. return true;
  1815. }
  1816. s_mask = arg_info(op->args[1])->s_mask
  1817. & arg_info(op->args[2])->s_mask;
  1818. return fold_masks_s(ctx, op, s_mask);
  1819. }
  1820. static bool fold_neg_no_const(OptContext *ctx, TCGOp *op)
  1821. {
  1822. /* Set to 1 all bits to the left of the rightmost. */
  1823. uint64_t z_mask = arg_info(op->args[1])->z_mask;
  1824. z_mask = -(z_mask & -z_mask);
  1825. return fold_masks_z(ctx, op, z_mask);
  1826. }
  1827. static bool fold_neg(OptContext *ctx, TCGOp *op)
  1828. {
  1829. return fold_const1(ctx, op) || fold_neg_no_const(ctx, op);
  1830. }
  1831. static bool fold_nor(OptContext *ctx, TCGOp *op)
  1832. {
  1833. uint64_t s_mask;
  1834. if (fold_const2_commutative(ctx, op) ||
  1835. fold_xi_to_not(ctx, op, 0)) {
  1836. return true;
  1837. }
  1838. s_mask = arg_info(op->args[1])->s_mask
  1839. & arg_info(op->args[2])->s_mask;
  1840. return fold_masks_s(ctx, op, s_mask);
  1841. }
  1842. static bool fold_not(OptContext *ctx, TCGOp *op)
  1843. {
  1844. if (fold_const1(ctx, op)) {
  1845. return true;
  1846. }
  1847. return fold_masks_s(ctx, op, arg_info(op->args[1])->s_mask);
  1848. }
  1849. static bool fold_or(OptContext *ctx, TCGOp *op)
  1850. {
  1851. uint64_t z_mask, s_mask;
  1852. TempOptInfo *t1, *t2;
  1853. if (fold_const2_commutative(ctx, op) ||
  1854. fold_xi_to_x(ctx, op, 0) ||
  1855. fold_xx_to_x(ctx, op)) {
  1856. return true;
  1857. }
  1858. t1 = arg_info(op->args[1]);
  1859. t2 = arg_info(op->args[2]);
  1860. z_mask = t1->z_mask | t2->z_mask;
  1861. s_mask = t1->s_mask & t2->s_mask;
  1862. return fold_masks_zs(ctx, op, z_mask, s_mask);
  1863. }
  1864. static bool fold_orc(OptContext *ctx, TCGOp *op)
  1865. {
  1866. uint64_t s_mask;
  1867. if (fold_const2(ctx, op) ||
  1868. fold_xx_to_i(ctx, op, -1) ||
  1869. fold_xi_to_x(ctx, op, -1) ||
  1870. fold_ix_to_not(ctx, op, 0)) {
  1871. return true;
  1872. }
  1873. s_mask = arg_info(op->args[1])->s_mask
  1874. & arg_info(op->args[2])->s_mask;
  1875. return fold_masks_s(ctx, op, s_mask);
  1876. }
  1877. static bool fold_qemu_ld_1reg(OptContext *ctx, TCGOp *op)
  1878. {
  1879. const TCGOpDef *def = &tcg_op_defs[op->opc];
  1880. MemOpIdx oi = op->args[def->nb_oargs + def->nb_iargs];
  1881. MemOp mop = get_memop(oi);
  1882. int width = 8 * memop_size(mop);
  1883. uint64_t z_mask = -1, s_mask = 0;
  1884. if (width < 64) {
  1885. if (mop & MO_SIGN) {
  1886. s_mask = MAKE_64BIT_MASK(width - 1, 64 - (width - 1));
  1887. } else {
  1888. z_mask = MAKE_64BIT_MASK(0, width);
  1889. }
  1890. }
  1891. /* Opcodes that touch guest memory stop the mb optimization. */
  1892. ctx->prev_mb = NULL;
  1893. return fold_masks_zs(ctx, op, z_mask, s_mask);
  1894. }
  1895. static bool fold_qemu_ld_2reg(OptContext *ctx, TCGOp *op)
  1896. {
  1897. /* Opcodes that touch guest memory stop the mb optimization. */
  1898. ctx->prev_mb = NULL;
  1899. return finish_folding(ctx, op);
  1900. }
  1901. static bool fold_qemu_st(OptContext *ctx, TCGOp *op)
  1902. {
  1903. /* Opcodes that touch guest memory stop the mb optimization. */
  1904. ctx->prev_mb = NULL;
  1905. return true;
  1906. }
  1907. static bool fold_remainder(OptContext *ctx, TCGOp *op)
  1908. {
  1909. if (fold_const2(ctx, op) ||
  1910. fold_xx_to_i(ctx, op, 0)) {
  1911. return true;
  1912. }
  1913. return finish_folding(ctx, op);
  1914. }
  1915. /* Return 1 if finished, -1 if simplified, 0 if unchanged. */
  1916. static int fold_setcond_zmask(OptContext *ctx, TCGOp *op, bool neg)
  1917. {
  1918. uint64_t a_zmask, b_val;
  1919. TCGCond cond;
  1920. if (!arg_is_const(op->args[2])) {
  1921. return false;
  1922. }
  1923. a_zmask = arg_info(op->args[1])->z_mask;
  1924. b_val = arg_info(op->args[2])->val;
  1925. cond = op->args[3];
  1926. if (ctx->type == TCG_TYPE_I32) {
  1927. a_zmask = (uint32_t)a_zmask;
  1928. b_val = (uint32_t)b_val;
  1929. }
  1930. /*
  1931. * A with only low bits set vs B with high bits set means that A < B.
  1932. */
  1933. if (a_zmask < b_val) {
  1934. bool inv = false;
  1935. switch (cond) {
  1936. case TCG_COND_NE:
  1937. case TCG_COND_LEU:
  1938. case TCG_COND_LTU:
  1939. inv = true;
  1940. /* fall through */
  1941. case TCG_COND_GTU:
  1942. case TCG_COND_GEU:
  1943. case TCG_COND_EQ:
  1944. return tcg_opt_gen_movi(ctx, op, op->args[0], neg ? -inv : inv);
  1945. default:
  1946. break;
  1947. }
  1948. }
  1949. /*
  1950. * A with only lsb set is already boolean.
  1951. */
  1952. if (a_zmask <= 1) {
  1953. bool convert = false;
  1954. bool inv = false;
  1955. switch (cond) {
  1956. case TCG_COND_EQ:
  1957. inv = true;
  1958. /* fall through */
  1959. case TCG_COND_NE:
  1960. convert = (b_val == 0);
  1961. break;
  1962. case TCG_COND_LTU:
  1963. case TCG_COND_TSTEQ:
  1964. inv = true;
  1965. /* fall through */
  1966. case TCG_COND_GEU:
  1967. case TCG_COND_TSTNE:
  1968. convert = (b_val == 1);
  1969. break;
  1970. default:
  1971. break;
  1972. }
  1973. if (convert) {
  1974. TCGOpcode add_opc, xor_opc, neg_opc;
  1975. if (!inv && !neg) {
  1976. return tcg_opt_gen_mov(ctx, op, op->args[0], op->args[1]);
  1977. }
  1978. switch (ctx->type) {
  1979. case TCG_TYPE_I32:
  1980. add_opc = INDEX_op_add_i32;
  1981. neg_opc = INDEX_op_neg_i32;
  1982. xor_opc = INDEX_op_xor_i32;
  1983. break;
  1984. case TCG_TYPE_I64:
  1985. add_opc = INDEX_op_add_i64;
  1986. neg_opc = INDEX_op_neg_i64;
  1987. xor_opc = INDEX_op_xor_i64;
  1988. break;
  1989. default:
  1990. g_assert_not_reached();
  1991. }
  1992. if (!inv) {
  1993. op->opc = neg_opc;
  1994. } else if (neg) {
  1995. op->opc = add_opc;
  1996. op->args[2] = arg_new_constant(ctx, -1);
  1997. } else {
  1998. op->opc = xor_opc;
  1999. op->args[2] = arg_new_constant(ctx, 1);
  2000. }
  2001. return -1;
  2002. }
  2003. }
  2004. return 0;
  2005. }
  2006. static void fold_setcond_tst_pow2(OptContext *ctx, TCGOp *op, bool neg)
  2007. {
  2008. TCGOpcode and_opc, sub_opc, xor_opc, neg_opc, shr_opc;
  2009. TCGOpcode uext_opc = 0, sext_opc = 0;
  2010. TCGCond cond = op->args[3];
  2011. TCGArg ret, src1, src2;
  2012. TCGOp *op2;
  2013. uint64_t val;
  2014. int sh;
  2015. bool inv;
  2016. if (!is_tst_cond(cond) || !arg_is_const(op->args[2])) {
  2017. return;
  2018. }
  2019. src2 = op->args[2];
  2020. val = arg_info(src2)->val;
  2021. if (!is_power_of_2(val)) {
  2022. return;
  2023. }
  2024. sh = ctz64(val);
  2025. switch (ctx->type) {
  2026. case TCG_TYPE_I32:
  2027. and_opc = INDEX_op_and_i32;
  2028. sub_opc = INDEX_op_sub_i32;
  2029. xor_opc = INDEX_op_xor_i32;
  2030. shr_opc = INDEX_op_shr_i32;
  2031. neg_opc = INDEX_op_neg_i32;
  2032. if (TCG_TARGET_extract_valid(TCG_TYPE_I32, sh, 1)) {
  2033. uext_opc = INDEX_op_extract_i32;
  2034. }
  2035. if (TCG_TARGET_sextract_valid(TCG_TYPE_I32, sh, 1)) {
  2036. sext_opc = INDEX_op_sextract_i32;
  2037. }
  2038. break;
  2039. case TCG_TYPE_I64:
  2040. and_opc = INDEX_op_and_i64;
  2041. sub_opc = INDEX_op_sub_i64;
  2042. xor_opc = INDEX_op_xor_i64;
  2043. shr_opc = INDEX_op_shr_i64;
  2044. neg_opc = INDEX_op_neg_i64;
  2045. if (TCG_TARGET_extract_valid(TCG_TYPE_I64, sh, 1)) {
  2046. uext_opc = INDEX_op_extract_i64;
  2047. }
  2048. if (TCG_TARGET_sextract_valid(TCG_TYPE_I64, sh, 1)) {
  2049. sext_opc = INDEX_op_sextract_i64;
  2050. }
  2051. break;
  2052. default:
  2053. g_assert_not_reached();
  2054. }
  2055. ret = op->args[0];
  2056. src1 = op->args[1];
  2057. inv = cond == TCG_COND_TSTEQ;
  2058. if (sh && sext_opc && neg && !inv) {
  2059. op->opc = sext_opc;
  2060. op->args[1] = src1;
  2061. op->args[2] = sh;
  2062. op->args[3] = 1;
  2063. return;
  2064. } else if (sh && uext_opc) {
  2065. op->opc = uext_opc;
  2066. op->args[1] = src1;
  2067. op->args[2] = sh;
  2068. op->args[3] = 1;
  2069. } else {
  2070. if (sh) {
  2071. op2 = tcg_op_insert_before(ctx->tcg, op, shr_opc, 3);
  2072. op2->args[0] = ret;
  2073. op2->args[1] = src1;
  2074. op2->args[2] = arg_new_constant(ctx, sh);
  2075. src1 = ret;
  2076. }
  2077. op->opc = and_opc;
  2078. op->args[1] = src1;
  2079. op->args[2] = arg_new_constant(ctx, 1);
  2080. }
  2081. if (neg && inv) {
  2082. op2 = tcg_op_insert_after(ctx->tcg, op, sub_opc, 3);
  2083. op2->args[0] = ret;
  2084. op2->args[1] = ret;
  2085. op2->args[2] = arg_new_constant(ctx, 1);
  2086. } else if (inv) {
  2087. op2 = tcg_op_insert_after(ctx->tcg, op, xor_opc, 3);
  2088. op2->args[0] = ret;
  2089. op2->args[1] = ret;
  2090. op2->args[2] = arg_new_constant(ctx, 1);
  2091. } else if (neg) {
  2092. op2 = tcg_op_insert_after(ctx->tcg, op, neg_opc, 2);
  2093. op2->args[0] = ret;
  2094. op2->args[1] = ret;
  2095. }
  2096. }
  2097. static bool fold_setcond(OptContext *ctx, TCGOp *op)
  2098. {
  2099. int i = do_constant_folding_cond1(ctx, op, op->args[0], &op->args[1],
  2100. &op->args[2], &op->args[3]);
  2101. if (i >= 0) {
  2102. return tcg_opt_gen_movi(ctx, op, op->args[0], i);
  2103. }
  2104. i = fold_setcond_zmask(ctx, op, false);
  2105. if (i > 0) {
  2106. return true;
  2107. }
  2108. if (i == 0) {
  2109. fold_setcond_tst_pow2(ctx, op, false);
  2110. }
  2111. return fold_masks_z(ctx, op, 1);
  2112. }
  2113. static bool fold_negsetcond(OptContext *ctx, TCGOp *op)
  2114. {
  2115. int i = do_constant_folding_cond1(ctx, op, op->args[0], &op->args[1],
  2116. &op->args[2], &op->args[3]);
  2117. if (i >= 0) {
  2118. return tcg_opt_gen_movi(ctx, op, op->args[0], -i);
  2119. }
  2120. i = fold_setcond_zmask(ctx, op, true);
  2121. if (i > 0) {
  2122. return true;
  2123. }
  2124. if (i == 0) {
  2125. fold_setcond_tst_pow2(ctx, op, true);
  2126. }
  2127. /* Value is {0,-1} so all bits are repetitions of the sign. */
  2128. return fold_masks_s(ctx, op, -1);
  2129. }
  2130. static bool fold_setcond2(OptContext *ctx, TCGOp *op)
  2131. {
  2132. TCGCond cond;
  2133. int i, inv = 0;
  2134. i = do_constant_folding_cond2(ctx, op, &op->args[1]);
  2135. cond = op->args[5];
  2136. if (i >= 0) {
  2137. goto do_setcond_const;
  2138. }
  2139. switch (cond) {
  2140. case TCG_COND_LT:
  2141. case TCG_COND_GE:
  2142. /*
  2143. * Simplify LT/GE comparisons vs zero to a single compare
  2144. * vs the high word of the input.
  2145. */
  2146. if (arg_is_const_val(op->args[3], 0) &&
  2147. arg_is_const_val(op->args[4], 0)) {
  2148. goto do_setcond_high;
  2149. }
  2150. break;
  2151. case TCG_COND_NE:
  2152. inv = 1;
  2153. QEMU_FALLTHROUGH;
  2154. case TCG_COND_EQ:
  2155. /*
  2156. * Simplify EQ/NE comparisons where one of the pairs
  2157. * can be simplified.
  2158. */
  2159. i = do_constant_folding_cond(TCG_TYPE_I32, op->args[1],
  2160. op->args[3], cond);
  2161. switch (i ^ inv) {
  2162. case 0:
  2163. goto do_setcond_const;
  2164. case 1:
  2165. goto do_setcond_high;
  2166. }
  2167. i = do_constant_folding_cond(TCG_TYPE_I32, op->args[2],
  2168. op->args[4], cond);
  2169. switch (i ^ inv) {
  2170. case 0:
  2171. goto do_setcond_const;
  2172. case 1:
  2173. goto do_setcond_low;
  2174. }
  2175. break;
  2176. case TCG_COND_TSTEQ:
  2177. case TCG_COND_TSTNE:
  2178. if (arg_is_const_val(op->args[3], 0)) {
  2179. goto do_setcond_high;
  2180. }
  2181. if (arg_is_const_val(op->args[4], 0)) {
  2182. goto do_setcond_low;
  2183. }
  2184. break;
  2185. default:
  2186. break;
  2187. do_setcond_low:
  2188. op->args[2] = op->args[3];
  2189. op->args[3] = cond;
  2190. op->opc = INDEX_op_setcond_i32;
  2191. return fold_setcond(ctx, op);
  2192. do_setcond_high:
  2193. op->args[1] = op->args[2];
  2194. op->args[2] = op->args[4];
  2195. op->args[3] = cond;
  2196. op->opc = INDEX_op_setcond_i32;
  2197. return fold_setcond(ctx, op);
  2198. }
  2199. return fold_masks_z(ctx, op, 1);
  2200. do_setcond_const:
  2201. return tcg_opt_gen_movi(ctx, op, op->args[0], i);
  2202. }
  2203. static bool fold_sextract(OptContext *ctx, TCGOp *op)
  2204. {
  2205. uint64_t z_mask, s_mask, s_mask_old;
  2206. TempOptInfo *t1 = arg_info(op->args[1]);
  2207. int pos = op->args[2];
  2208. int len = op->args[3];
  2209. if (ti_is_const(t1)) {
  2210. return tcg_opt_gen_movi(ctx, op, op->args[0],
  2211. sextract64(ti_const_val(t1), pos, len));
  2212. }
  2213. s_mask_old = t1->s_mask;
  2214. s_mask = s_mask_old >> pos;
  2215. s_mask |= -1ull << (len - 1);
  2216. if (pos == 0 && fold_affected_mask(ctx, op, s_mask & ~s_mask_old)) {
  2217. return true;
  2218. }
  2219. z_mask = sextract64(t1->z_mask, pos, len);
  2220. return fold_masks_zs(ctx, op, z_mask, s_mask);
  2221. }
  2222. static bool fold_shift(OptContext *ctx, TCGOp *op)
  2223. {
  2224. uint64_t s_mask, z_mask;
  2225. TempOptInfo *t1, *t2;
  2226. if (fold_const2(ctx, op) ||
  2227. fold_ix_to_i(ctx, op, 0) ||
  2228. fold_xi_to_x(ctx, op, 0)) {
  2229. return true;
  2230. }
  2231. t1 = arg_info(op->args[1]);
  2232. t2 = arg_info(op->args[2]);
  2233. s_mask = t1->s_mask;
  2234. z_mask = t1->z_mask;
  2235. if (ti_is_const(t2)) {
  2236. int sh = ti_const_val(t2);
  2237. z_mask = do_constant_folding(op->opc, ctx->type, z_mask, sh);
  2238. s_mask = do_constant_folding(op->opc, ctx->type, s_mask, sh);
  2239. return fold_masks_zs(ctx, op, z_mask, s_mask);
  2240. }
  2241. switch (op->opc) {
  2242. CASE_OP_32_64(sar):
  2243. /*
  2244. * Arithmetic right shift will not reduce the number of
  2245. * input sign repetitions.
  2246. */
  2247. return fold_masks_s(ctx, op, s_mask);
  2248. CASE_OP_32_64(shr):
  2249. /*
  2250. * If the sign bit is known zero, then logical right shift
  2251. * will not reduce the number of input sign repetitions.
  2252. */
  2253. if (~z_mask & -s_mask) {
  2254. return fold_masks_s(ctx, op, s_mask);
  2255. }
  2256. break;
  2257. default:
  2258. break;
  2259. }
  2260. return finish_folding(ctx, op);
  2261. }
  2262. static bool fold_sub_to_neg(OptContext *ctx, TCGOp *op)
  2263. {
  2264. TCGOpcode neg_op;
  2265. bool have_neg;
  2266. if (!arg_is_const(op->args[1]) || arg_info(op->args[1])->val != 0) {
  2267. return false;
  2268. }
  2269. switch (ctx->type) {
  2270. case TCG_TYPE_I32:
  2271. neg_op = INDEX_op_neg_i32;
  2272. have_neg = true;
  2273. break;
  2274. case TCG_TYPE_I64:
  2275. neg_op = INDEX_op_neg_i64;
  2276. have_neg = true;
  2277. break;
  2278. case TCG_TYPE_V64:
  2279. case TCG_TYPE_V128:
  2280. case TCG_TYPE_V256:
  2281. neg_op = INDEX_op_neg_vec;
  2282. have_neg = (TCG_TARGET_HAS_neg_vec &&
  2283. tcg_can_emit_vec_op(neg_op, ctx->type, TCGOP_VECE(op)) > 0);
  2284. break;
  2285. default:
  2286. g_assert_not_reached();
  2287. }
  2288. if (have_neg) {
  2289. op->opc = neg_op;
  2290. op->args[1] = op->args[2];
  2291. return fold_neg_no_const(ctx, op);
  2292. }
  2293. return false;
  2294. }
  2295. /* We cannot as yet do_constant_folding with vectors. */
  2296. static bool fold_sub_vec(OptContext *ctx, TCGOp *op)
  2297. {
  2298. if (fold_xx_to_i(ctx, op, 0) ||
  2299. fold_xi_to_x(ctx, op, 0) ||
  2300. fold_sub_to_neg(ctx, op)) {
  2301. return true;
  2302. }
  2303. return finish_folding(ctx, op);
  2304. }
  2305. static bool fold_sub(OptContext *ctx, TCGOp *op)
  2306. {
  2307. if (fold_const2(ctx, op) ||
  2308. fold_xx_to_i(ctx, op, 0) ||
  2309. fold_xi_to_x(ctx, op, 0) ||
  2310. fold_sub_to_neg(ctx, op)) {
  2311. return true;
  2312. }
  2313. /* Fold sub r,x,i to add r,x,-i */
  2314. if (arg_is_const(op->args[2])) {
  2315. uint64_t val = arg_info(op->args[2])->val;
  2316. op->opc = (ctx->type == TCG_TYPE_I32
  2317. ? INDEX_op_add_i32 : INDEX_op_add_i64);
  2318. op->args[2] = arg_new_constant(ctx, -val);
  2319. }
  2320. return finish_folding(ctx, op);
  2321. }
  2322. static bool fold_sub2(OptContext *ctx, TCGOp *op)
  2323. {
  2324. return fold_addsub2(ctx, op, false);
  2325. }
  2326. static bool fold_tcg_ld(OptContext *ctx, TCGOp *op)
  2327. {
  2328. uint64_t z_mask = -1, s_mask = 0;
  2329. /* We can't do any folding with a load, but we can record bits. */
  2330. switch (op->opc) {
  2331. CASE_OP_32_64(ld8s):
  2332. s_mask = INT8_MIN;
  2333. break;
  2334. CASE_OP_32_64(ld8u):
  2335. z_mask = MAKE_64BIT_MASK(0, 8);
  2336. break;
  2337. CASE_OP_32_64(ld16s):
  2338. s_mask = INT16_MIN;
  2339. break;
  2340. CASE_OP_32_64(ld16u):
  2341. z_mask = MAKE_64BIT_MASK(0, 16);
  2342. break;
  2343. case INDEX_op_ld32s_i64:
  2344. s_mask = INT32_MIN;
  2345. break;
  2346. case INDEX_op_ld32u_i64:
  2347. z_mask = MAKE_64BIT_MASK(0, 32);
  2348. break;
  2349. default:
  2350. g_assert_not_reached();
  2351. }
  2352. return fold_masks_zs(ctx, op, z_mask, s_mask);
  2353. }
  2354. static bool fold_tcg_ld_memcopy(OptContext *ctx, TCGOp *op)
  2355. {
  2356. TCGTemp *dst, *src;
  2357. intptr_t ofs;
  2358. TCGType type;
  2359. if (op->args[1] != tcgv_ptr_arg(tcg_env)) {
  2360. return finish_folding(ctx, op);
  2361. }
  2362. type = ctx->type;
  2363. ofs = op->args[2];
  2364. dst = arg_temp(op->args[0]);
  2365. src = find_mem_copy_for(ctx, type, ofs);
  2366. if (src && src->base_type == type) {
  2367. return tcg_opt_gen_mov(ctx, op, temp_arg(dst), temp_arg(src));
  2368. }
  2369. reset_ts(ctx, dst);
  2370. record_mem_copy(ctx, type, dst, ofs, ofs + tcg_type_size(type) - 1);
  2371. return true;
  2372. }
  2373. static bool fold_tcg_st(OptContext *ctx, TCGOp *op)
  2374. {
  2375. intptr_t ofs = op->args[2];
  2376. intptr_t lm1;
  2377. if (op->args[1] != tcgv_ptr_arg(tcg_env)) {
  2378. remove_mem_copy_all(ctx);
  2379. return true;
  2380. }
  2381. switch (op->opc) {
  2382. CASE_OP_32_64(st8):
  2383. lm1 = 0;
  2384. break;
  2385. CASE_OP_32_64(st16):
  2386. lm1 = 1;
  2387. break;
  2388. case INDEX_op_st32_i64:
  2389. case INDEX_op_st_i32:
  2390. lm1 = 3;
  2391. break;
  2392. case INDEX_op_st_i64:
  2393. lm1 = 7;
  2394. break;
  2395. case INDEX_op_st_vec:
  2396. lm1 = tcg_type_size(ctx->type) - 1;
  2397. break;
  2398. default:
  2399. g_assert_not_reached();
  2400. }
  2401. remove_mem_copy_in(ctx, ofs, ofs + lm1);
  2402. return true;
  2403. }
  2404. static bool fold_tcg_st_memcopy(OptContext *ctx, TCGOp *op)
  2405. {
  2406. TCGTemp *src;
  2407. intptr_t ofs, last;
  2408. TCGType type;
  2409. if (op->args[1] != tcgv_ptr_arg(tcg_env)) {
  2410. return fold_tcg_st(ctx, op);
  2411. }
  2412. src = arg_temp(op->args[0]);
  2413. ofs = op->args[2];
  2414. type = ctx->type;
  2415. /*
  2416. * Eliminate duplicate stores of a constant.
  2417. * This happens frequently when the target ISA zero-extends.
  2418. */
  2419. if (ts_is_const(src)) {
  2420. TCGTemp *prev = find_mem_copy_for(ctx, type, ofs);
  2421. if (src == prev) {
  2422. tcg_op_remove(ctx->tcg, op);
  2423. return true;
  2424. }
  2425. }
  2426. last = ofs + tcg_type_size(type) - 1;
  2427. remove_mem_copy_in(ctx, ofs, last);
  2428. record_mem_copy(ctx, type, src, ofs, last);
  2429. return true;
  2430. }
  2431. static bool fold_xor(OptContext *ctx, TCGOp *op)
  2432. {
  2433. uint64_t z_mask, s_mask;
  2434. TempOptInfo *t1, *t2;
  2435. if (fold_const2_commutative(ctx, op) ||
  2436. fold_xx_to_i(ctx, op, 0) ||
  2437. fold_xi_to_x(ctx, op, 0) ||
  2438. fold_xi_to_not(ctx, op, -1)) {
  2439. return true;
  2440. }
  2441. t1 = arg_info(op->args[1]);
  2442. t2 = arg_info(op->args[2]);
  2443. z_mask = t1->z_mask | t2->z_mask;
  2444. s_mask = t1->s_mask & t2->s_mask;
  2445. return fold_masks_zs(ctx, op, z_mask, s_mask);
  2446. }
  2447. /* Propagate constants and copies, fold constant expressions. */
  2448. void tcg_optimize(TCGContext *s)
  2449. {
  2450. int nb_temps, i;
  2451. TCGOp *op, *op_next;
  2452. OptContext ctx = { .tcg = s };
  2453. QSIMPLEQ_INIT(&ctx.mem_free);
  2454. /* Array VALS has an element for each temp.
  2455. If this temp holds a constant then its value is kept in VALS' element.
  2456. If this temp is a copy of other ones then the other copies are
  2457. available through the doubly linked circular list. */
  2458. nb_temps = s->nb_temps;
  2459. for (i = 0; i < nb_temps; ++i) {
  2460. s->temps[i].state_ptr = NULL;
  2461. }
  2462. QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) {
  2463. TCGOpcode opc = op->opc;
  2464. const TCGOpDef *def;
  2465. bool done = false;
  2466. /* Calls are special. */
  2467. if (opc == INDEX_op_call) {
  2468. fold_call(&ctx, op);
  2469. continue;
  2470. }
  2471. def = &tcg_op_defs[opc];
  2472. init_arguments(&ctx, op, def->nb_oargs + def->nb_iargs);
  2473. copy_propagate(&ctx, op, def->nb_oargs, def->nb_iargs);
  2474. /* Pre-compute the type of the operation. */
  2475. ctx.type = TCGOP_TYPE(op);
  2476. /*
  2477. * Process each opcode.
  2478. * Sorted alphabetically by opcode as much as possible.
  2479. */
  2480. switch (opc) {
  2481. CASE_OP_32_64(add):
  2482. done = fold_add(&ctx, op);
  2483. break;
  2484. case INDEX_op_add_vec:
  2485. done = fold_add_vec(&ctx, op);
  2486. break;
  2487. CASE_OP_32_64(add2):
  2488. done = fold_add2(&ctx, op);
  2489. break;
  2490. CASE_OP_32_64_VEC(and):
  2491. done = fold_and(&ctx, op);
  2492. break;
  2493. CASE_OP_32_64_VEC(andc):
  2494. done = fold_andc(&ctx, op);
  2495. break;
  2496. CASE_OP_32_64(brcond):
  2497. done = fold_brcond(&ctx, op);
  2498. break;
  2499. case INDEX_op_brcond2_i32:
  2500. done = fold_brcond2(&ctx, op);
  2501. break;
  2502. CASE_OP_32_64(bswap16):
  2503. CASE_OP_32_64(bswap32):
  2504. case INDEX_op_bswap64_i64:
  2505. done = fold_bswap(&ctx, op);
  2506. break;
  2507. CASE_OP_32_64(clz):
  2508. CASE_OP_32_64(ctz):
  2509. done = fold_count_zeros(&ctx, op);
  2510. break;
  2511. CASE_OP_32_64(ctpop):
  2512. done = fold_ctpop(&ctx, op);
  2513. break;
  2514. CASE_OP_32_64(deposit):
  2515. done = fold_deposit(&ctx, op);
  2516. break;
  2517. CASE_OP_32_64(div):
  2518. CASE_OP_32_64(divu):
  2519. done = fold_divide(&ctx, op);
  2520. break;
  2521. case INDEX_op_dup_vec:
  2522. done = fold_dup(&ctx, op);
  2523. break;
  2524. case INDEX_op_dup2_vec:
  2525. done = fold_dup2(&ctx, op);
  2526. break;
  2527. CASE_OP_32_64_VEC(eqv):
  2528. done = fold_eqv(&ctx, op);
  2529. break;
  2530. CASE_OP_32_64(extract):
  2531. done = fold_extract(&ctx, op);
  2532. break;
  2533. CASE_OP_32_64(extract2):
  2534. done = fold_extract2(&ctx, op);
  2535. break;
  2536. CASE_OP_32_64(ext8s):
  2537. CASE_OP_32_64(ext16s):
  2538. case INDEX_op_ext32s_i64:
  2539. case INDEX_op_ext_i32_i64:
  2540. done = fold_exts(&ctx, op);
  2541. break;
  2542. CASE_OP_32_64(ext8u):
  2543. CASE_OP_32_64(ext16u):
  2544. case INDEX_op_ext32u_i64:
  2545. case INDEX_op_extu_i32_i64:
  2546. case INDEX_op_extrl_i64_i32:
  2547. case INDEX_op_extrh_i64_i32:
  2548. done = fold_extu(&ctx, op);
  2549. break;
  2550. CASE_OP_32_64(ld8s):
  2551. CASE_OP_32_64(ld8u):
  2552. CASE_OP_32_64(ld16s):
  2553. CASE_OP_32_64(ld16u):
  2554. case INDEX_op_ld32s_i64:
  2555. case INDEX_op_ld32u_i64:
  2556. done = fold_tcg_ld(&ctx, op);
  2557. break;
  2558. #if !defined(CONFIG_TCG_THREADED_INTERPRETER) /* FIXME: this breaks TCTI */
  2559. case INDEX_op_ld_i32:
  2560. case INDEX_op_ld_i64:
  2561. case INDEX_op_ld_vec:
  2562. done = fold_tcg_ld_memcopy(&ctx, op);
  2563. break;
  2564. CASE_OP_32_64(st8):
  2565. CASE_OP_32_64(st16):
  2566. case INDEX_op_st32_i64:
  2567. done = fold_tcg_st(&ctx, op);
  2568. break;
  2569. case INDEX_op_st_i32:
  2570. case INDEX_op_st_i64:
  2571. case INDEX_op_st_vec:
  2572. done = fold_tcg_st_memcopy(&ctx, op);
  2573. break;
  2574. #endif
  2575. case INDEX_op_mb:
  2576. done = fold_mb(&ctx, op);
  2577. break;
  2578. CASE_OP_32_64_VEC(mov):
  2579. done = fold_mov(&ctx, op);
  2580. break;
  2581. CASE_OP_32_64(movcond):
  2582. done = fold_movcond(&ctx, op);
  2583. break;
  2584. CASE_OP_32_64(mul):
  2585. done = fold_mul(&ctx, op);
  2586. break;
  2587. CASE_OP_32_64(mulsh):
  2588. CASE_OP_32_64(muluh):
  2589. done = fold_mul_highpart(&ctx, op);
  2590. break;
  2591. CASE_OP_32_64(muls2):
  2592. CASE_OP_32_64(mulu2):
  2593. done = fold_multiply2(&ctx, op);
  2594. break;
  2595. CASE_OP_32_64_VEC(nand):
  2596. done = fold_nand(&ctx, op);
  2597. break;
  2598. CASE_OP_32_64(neg):
  2599. done = fold_neg(&ctx, op);
  2600. break;
  2601. CASE_OP_32_64_VEC(nor):
  2602. done = fold_nor(&ctx, op);
  2603. break;
  2604. CASE_OP_32_64_VEC(not):
  2605. done = fold_not(&ctx, op);
  2606. break;
  2607. CASE_OP_32_64_VEC(or):
  2608. done = fold_or(&ctx, op);
  2609. break;
  2610. CASE_OP_32_64_VEC(orc):
  2611. done = fold_orc(&ctx, op);
  2612. break;
  2613. case INDEX_op_qemu_ld_i32:
  2614. done = fold_qemu_ld_1reg(&ctx, op);
  2615. break;
  2616. case INDEX_op_qemu_ld_i64:
  2617. if (TCG_TARGET_REG_BITS == 64) {
  2618. done = fold_qemu_ld_1reg(&ctx, op);
  2619. break;
  2620. }
  2621. QEMU_FALLTHROUGH;
  2622. case INDEX_op_qemu_ld_i128:
  2623. done = fold_qemu_ld_2reg(&ctx, op);
  2624. break;
  2625. case INDEX_op_qemu_st8_i32:
  2626. case INDEX_op_qemu_st_i32:
  2627. case INDEX_op_qemu_st_i64:
  2628. case INDEX_op_qemu_st_i128:
  2629. done = fold_qemu_st(&ctx, op);
  2630. break;
  2631. CASE_OP_32_64(rem):
  2632. CASE_OP_32_64(remu):
  2633. done = fold_remainder(&ctx, op);
  2634. break;
  2635. CASE_OP_32_64(rotl):
  2636. CASE_OP_32_64(rotr):
  2637. CASE_OP_32_64(sar):
  2638. CASE_OP_32_64(shl):
  2639. CASE_OP_32_64(shr):
  2640. done = fold_shift(&ctx, op);
  2641. break;
  2642. CASE_OP_32_64(setcond):
  2643. done = fold_setcond(&ctx, op);
  2644. break;
  2645. CASE_OP_32_64(negsetcond):
  2646. done = fold_negsetcond(&ctx, op);
  2647. break;
  2648. case INDEX_op_setcond2_i32:
  2649. done = fold_setcond2(&ctx, op);
  2650. break;
  2651. case INDEX_op_cmp_vec:
  2652. done = fold_cmp_vec(&ctx, op);
  2653. break;
  2654. case INDEX_op_cmpsel_vec:
  2655. done = fold_cmpsel_vec(&ctx, op);
  2656. break;
  2657. case INDEX_op_bitsel_vec:
  2658. done = fold_bitsel_vec(&ctx, op);
  2659. break;
  2660. CASE_OP_32_64(sextract):
  2661. done = fold_sextract(&ctx, op);
  2662. break;
  2663. CASE_OP_32_64(sub):
  2664. done = fold_sub(&ctx, op);
  2665. break;
  2666. case INDEX_op_sub_vec:
  2667. done = fold_sub_vec(&ctx, op);
  2668. break;
  2669. CASE_OP_32_64(sub2):
  2670. done = fold_sub2(&ctx, op);
  2671. break;
  2672. CASE_OP_32_64_VEC(xor):
  2673. done = fold_xor(&ctx, op);
  2674. break;
  2675. case INDEX_op_set_label:
  2676. case INDEX_op_br:
  2677. case INDEX_op_exit_tb:
  2678. case INDEX_op_goto_tb:
  2679. case INDEX_op_goto_ptr:
  2680. finish_ebb(&ctx);
  2681. done = true;
  2682. break;
  2683. default:
  2684. done = finish_folding(&ctx, op);
  2685. break;
  2686. }
  2687. tcg_debug_assert(done);
  2688. }
  2689. }