ARMLoadStoreOptimizer.cpp 69 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999
  1. //===-- ARMLoadStoreOptimizer.cpp - ARM load / store opt. pass ------------===//
  2. //
  3. // The LLVM Compiler Infrastructure
  4. //
  5. // This file is distributed under the University of Illinois Open Source
  6. // License. See LICENSE.TXT for details.
  7. //
  8. //===----------------------------------------------------------------------===//
  9. //
  10. // This file contains a pass that performs load / store related peephole
  11. // optimizations. This pass should be run after register allocation.
  12. //
  13. //===----------------------------------------------------------------------===//
  14. #define DEBUG_TYPE "arm-ldst-opt"
  15. #include "ARM.h"
  16. #include "ARMBaseInstrInfo.h"
  17. #include "ARMBaseRegisterInfo.h"
  18. #include "ARMMachineFunctionInfo.h"
  19. #include "MCTargetDesc/ARMAddressingModes.h"
  20. #include "llvm/ADT/DenseMap.h"
  21. #include "llvm/ADT/STLExtras.h"
  22. #include "llvm/ADT/SmallPtrSet.h"
  23. #include "llvm/ADT/SmallSet.h"
  24. #include "llvm/ADT/SmallVector.h"
  25. #include "llvm/ADT/Statistic.h"
  26. #include "llvm/CodeGen/MachineBasicBlock.h"
  27. #include "llvm/CodeGen/MachineFunctionPass.h"
  28. #include "llvm/CodeGen/MachineInstr.h"
  29. #include "llvm/CodeGen/MachineInstrBuilder.h"
  30. #include "llvm/CodeGen/MachineRegisterInfo.h"
  31. #include "llvm/CodeGen/RegisterScavenging.h"
  32. #include "llvm/CodeGen/SelectionDAGNodes.h"
  33. #include "llvm/IR/DataLayout.h"
  34. #include "llvm/IR/DerivedTypes.h"
  35. #include "llvm/IR/Function.h"
  36. #include "llvm/Support/Debug.h"
  37. #include "llvm/Support/ErrorHandling.h"
  38. #include "llvm/Support/raw_ostream.h"
  39. #include "llvm/Target/TargetInstrInfo.h"
  40. #include "llvm/Target/TargetMachine.h"
  41. #include "llvm/Target/TargetRegisterInfo.h"
  42. using namespace llvm;
  43. STATISTIC(NumLDMGened , "Number of ldm instructions generated");
  44. STATISTIC(NumSTMGened , "Number of stm instructions generated");
  45. STATISTIC(NumVLDMGened, "Number of vldm instructions generated");
  46. STATISTIC(NumVSTMGened, "Number of vstm instructions generated");
  47. STATISTIC(NumLdStMoved, "Number of load / store instructions moved");
  48. STATISTIC(NumLDRDFormed,"Number of ldrd created before allocation");
  49. STATISTIC(NumSTRDFormed,"Number of strd created before allocation");
  50. STATISTIC(NumLDRD2LDM, "Number of ldrd instructions turned back into ldm");
  51. STATISTIC(NumSTRD2STM, "Number of strd instructions turned back into stm");
  52. STATISTIC(NumLDRD2LDR, "Number of ldrd instructions turned back into ldr's");
  53. STATISTIC(NumSTRD2STR, "Number of strd instructions turned back into str's");
  54. /// ARMAllocLoadStoreOpt - Post- register allocation pass the combine
  55. /// load / store instructions to form ldm / stm instructions.
  56. namespace {
  57. struct ARMLoadStoreOpt : public MachineFunctionPass {
  58. static char ID;
  59. ARMLoadStoreOpt() : MachineFunctionPass(ID) {}
  60. const TargetInstrInfo *TII;
  61. const TargetRegisterInfo *TRI;
  62. const ARMSubtarget *STI;
  63. ARMFunctionInfo *AFI;
  64. RegScavenger *RS;
  65. bool isThumb2;
  66. virtual bool runOnMachineFunction(MachineFunction &Fn);
  67. virtual const char *getPassName() const {
  68. return "ARM load / store optimization pass";
  69. }
  70. private:
  71. struct MemOpQueueEntry {
  72. int Offset;
  73. unsigned Reg;
  74. bool isKill;
  75. unsigned Position;
  76. MachineBasicBlock::iterator MBBI;
  77. bool Merged;
  78. MemOpQueueEntry(int o, unsigned r, bool k, unsigned p,
  79. MachineBasicBlock::iterator i)
  80. : Offset(o), Reg(r), isKill(k), Position(p), MBBI(i), Merged(false) {}
  81. };
  82. typedef SmallVector<MemOpQueueEntry,8> MemOpQueue;
  83. typedef MemOpQueue::iterator MemOpQueueIter;
  84. void findUsesOfImpDef(SmallVectorImpl<MachineOperand *> &UsesOfImpDefs,
  85. const MemOpQueue &MemOps, unsigned DefReg,
  86. unsigned RangeBegin, unsigned RangeEnd);
  87. bool MergeOps(MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI,
  88. int Offset, unsigned Base, bool BaseKill, int Opcode,
  89. ARMCC::CondCodes Pred, unsigned PredReg, unsigned Scratch,
  90. DebugLoc dl,
  91. ArrayRef<std::pair<unsigned, bool> > Regs,
  92. ArrayRef<unsigned> ImpDefs);
  93. void MergeOpsUpdate(MachineBasicBlock &MBB,
  94. MemOpQueue &MemOps,
  95. unsigned memOpsBegin,
  96. unsigned memOpsEnd,
  97. unsigned insertAfter,
  98. int Offset,
  99. unsigned Base,
  100. bool BaseKill,
  101. int Opcode,
  102. ARMCC::CondCodes Pred,
  103. unsigned PredReg,
  104. unsigned Scratch,
  105. DebugLoc dl,
  106. SmallVectorImpl<MachineBasicBlock::iterator> &Merges);
  107. void MergeLDR_STR(MachineBasicBlock &MBB, unsigned SIndex, unsigned Base,
  108. int Opcode, unsigned Size,
  109. ARMCC::CondCodes Pred, unsigned PredReg,
  110. unsigned Scratch, MemOpQueue &MemOps,
  111. SmallVectorImpl<MachineBasicBlock::iterator> &Merges);
  112. void AdvanceRS(MachineBasicBlock &MBB, MemOpQueue &MemOps);
  113. bool FixInvalidRegPairOp(MachineBasicBlock &MBB,
  114. MachineBasicBlock::iterator &MBBI);
  115. bool MergeBaseUpdateLoadStore(MachineBasicBlock &MBB,
  116. MachineBasicBlock::iterator MBBI,
  117. const TargetInstrInfo *TII,
  118. bool &Advance,
  119. MachineBasicBlock::iterator &I);
  120. bool MergeBaseUpdateLSMultiple(MachineBasicBlock &MBB,
  121. MachineBasicBlock::iterator MBBI,
  122. bool &Advance,
  123. MachineBasicBlock::iterator &I);
  124. bool LoadStoreMultipleOpti(MachineBasicBlock &MBB);
  125. bool MergeReturnIntoLDM(MachineBasicBlock &MBB);
  126. };
  127. char ARMLoadStoreOpt::ID = 0;
  128. }
  129. static int getLoadStoreMultipleOpcode(int Opcode, ARM_AM::AMSubMode Mode) {
  130. switch (Opcode) {
  131. default: llvm_unreachable("Unhandled opcode!");
  132. case ARM::LDRi12:
  133. ++NumLDMGened;
  134. switch (Mode) {
  135. default: llvm_unreachable("Unhandled submode!");
  136. case ARM_AM::ia: return ARM::LDMIA;
  137. case ARM_AM::da: return ARM::LDMDA;
  138. case ARM_AM::db: return ARM::LDMDB;
  139. case ARM_AM::ib: return ARM::LDMIB;
  140. }
  141. case ARM::STRi12:
  142. ++NumSTMGened;
  143. switch (Mode) {
  144. default: llvm_unreachable("Unhandled submode!");
  145. case ARM_AM::ia: return ARM::STMIA;
  146. case ARM_AM::da: return ARM::STMDA;
  147. case ARM_AM::db: return ARM::STMDB;
  148. case ARM_AM::ib: return ARM::STMIB;
  149. }
  150. case ARM::t2LDRi8:
  151. case ARM::t2LDRi12:
  152. ++NumLDMGened;
  153. switch (Mode) {
  154. default: llvm_unreachable("Unhandled submode!");
  155. case ARM_AM::ia: return ARM::t2LDMIA;
  156. case ARM_AM::db: return ARM::t2LDMDB;
  157. }
  158. case ARM::t2STRi8:
  159. case ARM::t2STRi12:
  160. ++NumSTMGened;
  161. switch (Mode) {
  162. default: llvm_unreachable("Unhandled submode!");
  163. case ARM_AM::ia: return ARM::t2STMIA;
  164. case ARM_AM::db: return ARM::t2STMDB;
  165. }
  166. case ARM::VLDRS:
  167. ++NumVLDMGened;
  168. switch (Mode) {
  169. default: llvm_unreachable("Unhandled submode!");
  170. case ARM_AM::ia: return ARM::VLDMSIA;
  171. case ARM_AM::db: return 0; // Only VLDMSDB_UPD exists.
  172. }
  173. case ARM::VSTRS:
  174. ++NumVSTMGened;
  175. switch (Mode) {
  176. default: llvm_unreachable("Unhandled submode!");
  177. case ARM_AM::ia: return ARM::VSTMSIA;
  178. case ARM_AM::db: return 0; // Only VSTMSDB_UPD exists.
  179. }
  180. case ARM::VLDRD:
  181. ++NumVLDMGened;
  182. switch (Mode) {
  183. default: llvm_unreachable("Unhandled submode!");
  184. case ARM_AM::ia: return ARM::VLDMDIA;
  185. case ARM_AM::db: return 0; // Only VLDMDDB_UPD exists.
  186. }
  187. case ARM::VSTRD:
  188. ++NumVSTMGened;
  189. switch (Mode) {
  190. default: llvm_unreachable("Unhandled submode!");
  191. case ARM_AM::ia: return ARM::VSTMDIA;
  192. case ARM_AM::db: return 0; // Only VSTMDDB_UPD exists.
  193. }
  194. }
  195. }
  196. namespace llvm {
  197. namespace ARM_AM {
  198. AMSubMode getLoadStoreMultipleSubMode(int Opcode) {
  199. switch (Opcode) {
  200. default: llvm_unreachable("Unhandled opcode!");
  201. case ARM::LDMIA_RET:
  202. case ARM::LDMIA:
  203. case ARM::LDMIA_UPD:
  204. case ARM::STMIA:
  205. case ARM::STMIA_UPD:
  206. case ARM::t2LDMIA_RET:
  207. case ARM::t2LDMIA:
  208. case ARM::t2LDMIA_UPD:
  209. case ARM::t2STMIA:
  210. case ARM::t2STMIA_UPD:
  211. case ARM::VLDMSIA:
  212. case ARM::VLDMSIA_UPD:
  213. case ARM::VSTMSIA:
  214. case ARM::VSTMSIA_UPD:
  215. case ARM::VLDMDIA:
  216. case ARM::VLDMDIA_UPD:
  217. case ARM::VSTMDIA:
  218. case ARM::VSTMDIA_UPD:
  219. return ARM_AM::ia;
  220. case ARM::LDMDA:
  221. case ARM::LDMDA_UPD:
  222. case ARM::STMDA:
  223. case ARM::STMDA_UPD:
  224. return ARM_AM::da;
  225. case ARM::LDMDB:
  226. case ARM::LDMDB_UPD:
  227. case ARM::STMDB:
  228. case ARM::STMDB_UPD:
  229. case ARM::t2LDMDB:
  230. case ARM::t2LDMDB_UPD:
  231. case ARM::t2STMDB:
  232. case ARM::t2STMDB_UPD:
  233. case ARM::VLDMSDB_UPD:
  234. case ARM::VSTMSDB_UPD:
  235. case ARM::VLDMDDB_UPD:
  236. case ARM::VSTMDDB_UPD:
  237. return ARM_AM::db;
  238. case ARM::LDMIB:
  239. case ARM::LDMIB_UPD:
  240. case ARM::STMIB:
  241. case ARM::STMIB_UPD:
  242. return ARM_AM::ib;
  243. }
  244. }
  245. } // end namespace ARM_AM
  246. } // end namespace llvm
  247. static bool isT2i32Load(unsigned Opc) {
  248. return Opc == ARM::t2LDRi12 || Opc == ARM::t2LDRi8;
  249. }
  250. static bool isi32Load(unsigned Opc) {
  251. return Opc == ARM::LDRi12 || isT2i32Load(Opc);
  252. }
  253. static bool isT2i32Store(unsigned Opc) {
  254. return Opc == ARM::t2STRi12 || Opc == ARM::t2STRi8;
  255. }
  256. static bool isi32Store(unsigned Opc) {
  257. return Opc == ARM::STRi12 || isT2i32Store(Opc);
  258. }
  259. /// MergeOps - Create and insert a LDM or STM with Base as base register and
  260. /// registers in Regs as the register operands that would be loaded / stored.
  261. /// It returns true if the transformation is done.
  262. bool
  263. ARMLoadStoreOpt::MergeOps(MachineBasicBlock &MBB,
  264. MachineBasicBlock::iterator MBBI,
  265. int Offset, unsigned Base, bool BaseKill,
  266. int Opcode, ARMCC::CondCodes Pred,
  267. unsigned PredReg, unsigned Scratch, DebugLoc dl,
  268. ArrayRef<std::pair<unsigned, bool> > Regs,
  269. ArrayRef<unsigned> ImpDefs) {
  270. // Only a single register to load / store. Don't bother.
  271. unsigned NumRegs = Regs.size();
  272. if (NumRegs <= 1)
  273. return false;
  274. ARM_AM::AMSubMode Mode = ARM_AM::ia;
  275. // VFP and Thumb2 do not support IB or DA modes.
  276. bool isNotVFP = isi32Load(Opcode) || isi32Store(Opcode);
  277. bool haveIBAndDA = isNotVFP && !isThumb2;
  278. if (Offset == 4 && haveIBAndDA)
  279. Mode = ARM_AM::ib;
  280. else if (Offset == -4 * (int)NumRegs + 4 && haveIBAndDA)
  281. Mode = ARM_AM::da;
  282. else if (Offset == -4 * (int)NumRegs && isNotVFP)
  283. // VLDM/VSTM do not support DB mode without also updating the base reg.
  284. Mode = ARM_AM::db;
  285. else if (Offset != 0) {
  286. // Check if this is a supported opcode before we insert instructions to
  287. // calculate a new base register.
  288. if (!getLoadStoreMultipleOpcode(Opcode, Mode)) return false;
  289. // If starting offset isn't zero, insert a MI to materialize a new base.
  290. // But only do so if it is cost effective, i.e. merging more than two
  291. // loads / stores.
  292. if (NumRegs <= 2)
  293. return false;
  294. unsigned NewBase;
  295. if (isi32Load(Opcode))
  296. // If it is a load, then just use one of the destination register to
  297. // use as the new base.
  298. NewBase = Regs[NumRegs-1].first;
  299. else {
  300. // Use the scratch register to use as a new base.
  301. NewBase = Scratch;
  302. if (NewBase == 0)
  303. return false;
  304. }
  305. int BaseOpc = !isThumb2 ? ARM::ADDri : ARM::t2ADDri;
  306. if (Offset < 0) {
  307. BaseOpc = !isThumb2 ? ARM::SUBri : ARM::t2SUBri;
  308. Offset = - Offset;
  309. }
  310. int ImmedOffset = isThumb2
  311. ? ARM_AM::getT2SOImmVal(Offset) : ARM_AM::getSOImmVal(Offset);
  312. if (ImmedOffset == -1)
  313. // FIXME: Try t2ADDri12 or t2SUBri12?
  314. return false; // Probably not worth it then.
  315. BuildMI(MBB, MBBI, dl, TII->get(BaseOpc), NewBase)
  316. .addReg(Base, getKillRegState(BaseKill)).addImm(Offset)
  317. .addImm(Pred).addReg(PredReg).addReg(0);
  318. Base = NewBase;
  319. BaseKill = true; // New base is always killed right its use.
  320. }
  321. bool isDef = (isi32Load(Opcode) || Opcode == ARM::VLDRS ||
  322. Opcode == ARM::VLDRD);
  323. Opcode = getLoadStoreMultipleOpcode(Opcode, Mode);
  324. if (!Opcode) return false;
  325. MachineInstrBuilder MIB = BuildMI(MBB, MBBI, dl, TII->get(Opcode))
  326. .addReg(Base, getKillRegState(BaseKill))
  327. .addImm(Pred).addReg(PredReg);
  328. for (unsigned i = 0; i != NumRegs; ++i)
  329. MIB = MIB.addReg(Regs[i].first, getDefRegState(isDef)
  330. | getKillRegState(Regs[i].second));
  331. // Add implicit defs for super-registers.
  332. for (unsigned i = 0, e = ImpDefs.size(); i != e; ++i)
  333. MIB.addReg(ImpDefs[i], RegState::ImplicitDefine);
  334. return true;
  335. }
  336. /// \brief Find all instructions using a given imp-def within a range.
  337. ///
  338. /// We are trying to combine a range of instructions, one of which (located at
  339. /// position RangeBegin) implicitly defines a register. The final LDM/STM will
  340. /// be placed at RangeEnd, and so any uses of this definition between RangeStart
  341. /// and RangeEnd must be modified to use an undefined value.
  342. ///
  343. /// The live range continues until we find a second definition or one of the
  344. /// uses we find is a kill. Unfortunately MemOps is not sorted by Position, so
  345. /// we must consider all uses and decide which are relevant in a second pass.
  346. void ARMLoadStoreOpt::findUsesOfImpDef(
  347. SmallVectorImpl<MachineOperand *> &UsesOfImpDefs, const MemOpQueue &MemOps,
  348. unsigned DefReg, unsigned RangeBegin, unsigned RangeEnd) {
  349. std::map<unsigned, MachineOperand *> Uses;
  350. unsigned LastLivePos = RangeEnd;
  351. // First we find all uses of this register with Position between RangeBegin
  352. // and RangeEnd, any or all of these could be uses of a definition at
  353. // RangeBegin. We also record the latest position a definition at RangeBegin
  354. // would be considered live.
  355. for (unsigned i = 0; i < MemOps.size(); ++i) {
  356. MachineInstr &MI = *MemOps[i].MBBI;
  357. unsigned MIPosition = MemOps[i].Position;
  358. if (MIPosition <= RangeBegin || MIPosition > RangeEnd)
  359. continue;
  360. // If this instruction defines the register, then any later use will be of
  361. // that definition rather than ours.
  362. if (MI.definesRegister(DefReg))
  363. LastLivePos = std::min(LastLivePos, MIPosition);
  364. MachineOperand *UseOp = MI.findRegisterUseOperand(DefReg);
  365. if (!UseOp)
  366. continue;
  367. // If this instruction kills the register then (assuming liveness is
  368. // correct when we start) we don't need to think about anything after here.
  369. if (UseOp->isKill())
  370. LastLivePos = std::min(LastLivePos, MIPosition);
  371. Uses[MIPosition] = UseOp;
  372. }
  373. // Now we traverse the list of all uses, and append the ones that actually use
  374. // our definition to the requested list.
  375. for (std::map<unsigned, MachineOperand *>::iterator I = Uses.begin(),
  376. E = Uses.end();
  377. I != E; ++I) {
  378. // List is sorted by position so once we've found one out of range there
  379. // will be no more to consider.
  380. if (I->first > LastLivePos)
  381. break;
  382. UsesOfImpDefs.push_back(I->second);
  383. }
  384. }
  385. // MergeOpsUpdate - call MergeOps and update MemOps and merges accordingly on
  386. // success.
  387. void ARMLoadStoreOpt::MergeOpsUpdate(MachineBasicBlock &MBB,
  388. MemOpQueue &memOps,
  389. unsigned memOpsBegin, unsigned memOpsEnd,
  390. unsigned insertAfter, int Offset,
  391. unsigned Base, bool BaseKill,
  392. int Opcode,
  393. ARMCC::CondCodes Pred, unsigned PredReg,
  394. unsigned Scratch,
  395. DebugLoc dl,
  396. SmallVectorImpl<MachineBasicBlock::iterator> &Merges) {
  397. // First calculate which of the registers should be killed by the merged
  398. // instruction.
  399. const unsigned insertPos = memOps[insertAfter].Position;
  400. SmallSet<unsigned, 4> KilledRegs;
  401. DenseMap<unsigned, unsigned> Killer;
  402. for (unsigned i = 0, e = memOps.size(); i != e; ++i) {
  403. if (i == memOpsBegin) {
  404. i = memOpsEnd;
  405. if (i == e)
  406. break;
  407. }
  408. if (memOps[i].Position < insertPos && memOps[i].isKill) {
  409. unsigned Reg = memOps[i].Reg;
  410. KilledRegs.insert(Reg);
  411. Killer[Reg] = i;
  412. }
  413. }
  414. SmallVector<std::pair<unsigned, bool>, 8> Regs;
  415. SmallVector<unsigned, 8> ImpDefs;
  416. SmallVector<MachineOperand *, 8> UsesOfImpDefs;
  417. for (unsigned i = memOpsBegin; i < memOpsEnd; ++i) {
  418. unsigned Reg = memOps[i].Reg;
  419. // If we are inserting the merged operation after an operation that
  420. // uses the same register, make sure to transfer any kill flag.
  421. bool isKill = memOps[i].isKill || KilledRegs.count(Reg);
  422. Regs.push_back(std::make_pair(Reg, isKill));
  423. // Collect any implicit defs of super-registers. They must be preserved.
  424. for (MIOperands MO(memOps[i].MBBI); MO.isValid(); ++MO) {
  425. if (!MO->isReg() || !MO->isDef() || !MO->isImplicit() || MO->isDead())
  426. continue;
  427. unsigned DefReg = MO->getReg();
  428. if (std::find(ImpDefs.begin(), ImpDefs.end(), DefReg) == ImpDefs.end())
  429. ImpDefs.push_back(DefReg);
  430. // There may be other uses of the definition between this instruction and
  431. // the eventual LDM/STM position. These should be marked undef if the
  432. // merge takes place.
  433. findUsesOfImpDef(UsesOfImpDefs, memOps, DefReg, memOps[i].Position,
  434. insertPos);
  435. }
  436. }
  437. // Try to do the merge.
  438. MachineBasicBlock::iterator Loc = memOps[insertAfter].MBBI;
  439. ++Loc;
  440. if (!MergeOps(MBB, Loc, Offset, Base, BaseKill, Opcode,
  441. Pred, PredReg, Scratch, dl, Regs, ImpDefs))
  442. return;
  443. // Merge succeeded, update records.
  444. Merges.push_back(std::prev(Loc));
  445. // In gathering loads together, we may have moved the imp-def of a register
  446. // past one of its uses. This is OK, since we know better than the rest of
  447. // LLVM what's OK with ARM loads and stores; but we still have to adjust the
  448. // affected uses.
  449. for (SmallVectorImpl<MachineOperand *>::iterator I = UsesOfImpDefs.begin(),
  450. E = UsesOfImpDefs.end();
  451. I != E; ++I)
  452. (*I)->setIsUndef();
  453. for (unsigned i = memOpsBegin; i < memOpsEnd; ++i) {
  454. // Remove kill flags from any memops that come before insertPos.
  455. if (Regs[i-memOpsBegin].second) {
  456. unsigned Reg = Regs[i-memOpsBegin].first;
  457. if (KilledRegs.count(Reg)) {
  458. unsigned j = Killer[Reg];
  459. int Idx = memOps[j].MBBI->findRegisterUseOperandIdx(Reg, true);
  460. assert(Idx >= 0 && "Cannot find killing operand");
  461. memOps[j].MBBI->getOperand(Idx).setIsKill(false);
  462. memOps[j].isKill = false;
  463. }
  464. memOps[i].isKill = true;
  465. }
  466. MBB.erase(memOps[i].MBBI);
  467. // Update this memop to refer to the merged instruction.
  468. // We may need to move kill flags again.
  469. memOps[i].Merged = true;
  470. memOps[i].MBBI = Merges.back();
  471. memOps[i].Position = insertPos;
  472. }
  473. }
  474. /// MergeLDR_STR - Merge a number of load / store instructions into one or more
  475. /// load / store multiple instructions.
  476. void
  477. ARMLoadStoreOpt::MergeLDR_STR(MachineBasicBlock &MBB, unsigned SIndex,
  478. unsigned Base, int Opcode, unsigned Size,
  479. ARMCC::CondCodes Pred, unsigned PredReg,
  480. unsigned Scratch, MemOpQueue &MemOps,
  481. SmallVectorImpl<MachineBasicBlock::iterator> &Merges) {
  482. bool isNotVFP = isi32Load(Opcode) || isi32Store(Opcode);
  483. int Offset = MemOps[SIndex].Offset;
  484. int SOffset = Offset;
  485. unsigned insertAfter = SIndex;
  486. MachineBasicBlock::iterator Loc = MemOps[SIndex].MBBI;
  487. DebugLoc dl = Loc->getDebugLoc();
  488. const MachineOperand &PMO = Loc->getOperand(0);
  489. unsigned PReg = PMO.getReg();
  490. unsigned PRegNum = PMO.isUndef() ? UINT_MAX : TRI->getEncodingValue(PReg);
  491. unsigned Count = 1;
  492. unsigned Limit = ~0U;
  493. // vldm / vstm limit are 32 for S variants, 16 for D variants.
  494. switch (Opcode) {
  495. default: break;
  496. case ARM::VSTRS:
  497. Limit = 32;
  498. break;
  499. case ARM::VSTRD:
  500. Limit = 16;
  501. break;
  502. case ARM::VLDRD:
  503. Limit = 16;
  504. break;
  505. case ARM::VLDRS:
  506. Limit = 32;
  507. break;
  508. }
  509. for (unsigned i = SIndex+1, e = MemOps.size(); i != e; ++i) {
  510. int NewOffset = MemOps[i].Offset;
  511. const MachineOperand &MO = MemOps[i].MBBI->getOperand(0);
  512. unsigned Reg = MO.getReg();
  513. unsigned RegNum = MO.isUndef() ? UINT_MAX : TRI->getEncodingValue(Reg);
  514. // Register numbers must be in ascending order. For VFP / NEON load and
  515. // store multiples, the registers must also be consecutive and within the
  516. // limit on the number of registers per instruction.
  517. if (Reg != ARM::SP &&
  518. NewOffset == Offset + (int)Size &&
  519. ((isNotVFP && RegNum > PRegNum) ||
  520. ((Count < Limit) && RegNum == PRegNum+1)) &&
  521. // On Swift we don't want vldm/vstm to start with a odd register num
  522. // because Q register unaligned vldm/vstm need more uops.
  523. (!STI->isSwift() || isNotVFP || Count != 1 || !(PRegNum & 0x1))) {
  524. Offset += Size;
  525. PRegNum = RegNum;
  526. ++Count;
  527. } else {
  528. // Can't merge this in. Try merge the earlier ones first.
  529. MergeOpsUpdate(MBB, MemOps, SIndex, i, insertAfter, SOffset,
  530. Base, false, Opcode, Pred, PredReg, Scratch, dl, Merges);
  531. MergeLDR_STR(MBB, i, Base, Opcode, Size, Pred, PredReg, Scratch,
  532. MemOps, Merges);
  533. return;
  534. }
  535. if (MemOps[i].Position > MemOps[insertAfter].Position)
  536. insertAfter = i;
  537. }
  538. bool BaseKill = Loc->findRegisterUseOperandIdx(Base, true) != -1;
  539. MergeOpsUpdate(MBB, MemOps, SIndex, MemOps.size(), insertAfter, SOffset,
  540. Base, BaseKill, Opcode, Pred, PredReg, Scratch, dl, Merges);
  541. return;
  542. }
  543. static bool definesCPSR(MachineInstr *MI) {
  544. for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
  545. const MachineOperand &MO = MI->getOperand(i);
  546. if (!MO.isReg())
  547. continue;
  548. if (MO.isDef() && MO.getReg() == ARM::CPSR && !MO.isDead())
  549. // If the instruction has live CPSR def, then it's not safe to fold it
  550. // into load / store.
  551. return true;
  552. }
  553. return false;
  554. }
  555. static bool isMatchingDecrement(MachineInstr *MI, unsigned Base,
  556. unsigned Bytes, unsigned Limit,
  557. ARMCC::CondCodes Pred, unsigned PredReg) {
  558. unsigned MyPredReg = 0;
  559. if (!MI)
  560. return false;
  561. bool CheckCPSRDef = false;
  562. switch (MI->getOpcode()) {
  563. default: return false;
  564. case ARM::t2SUBri:
  565. case ARM::SUBri:
  566. CheckCPSRDef = true;
  567. // fallthrough
  568. case ARM::tSUBspi:
  569. break;
  570. }
  571. // Make sure the offset fits in 8 bits.
  572. if (Bytes == 0 || (Limit && Bytes >= Limit))
  573. return false;
  574. unsigned Scale = (MI->getOpcode() == ARM::tSUBspi) ? 4 : 1; // FIXME
  575. if (!(MI->getOperand(0).getReg() == Base &&
  576. MI->getOperand(1).getReg() == Base &&
  577. (MI->getOperand(2).getImm()*Scale) == Bytes &&
  578. getInstrPredicate(MI, MyPredReg) == Pred &&
  579. MyPredReg == PredReg))
  580. return false;
  581. return CheckCPSRDef ? !definesCPSR(MI) : true;
  582. }
  583. static bool isMatchingIncrement(MachineInstr *MI, unsigned Base,
  584. unsigned Bytes, unsigned Limit,
  585. ARMCC::CondCodes Pred, unsigned PredReg) {
  586. unsigned MyPredReg = 0;
  587. if (!MI)
  588. return false;
  589. bool CheckCPSRDef = false;
  590. switch (MI->getOpcode()) {
  591. default: return false;
  592. case ARM::t2ADDri:
  593. case ARM::ADDri:
  594. CheckCPSRDef = true;
  595. // fallthrough
  596. case ARM::tADDspi:
  597. break;
  598. }
  599. if (Bytes == 0 || (Limit && Bytes >= Limit))
  600. // Make sure the offset fits in 8 bits.
  601. return false;
  602. unsigned Scale = (MI->getOpcode() == ARM::tADDspi) ? 4 : 1; // FIXME
  603. if (!(MI->getOperand(0).getReg() == Base &&
  604. MI->getOperand(1).getReg() == Base &&
  605. (MI->getOperand(2).getImm()*Scale) == Bytes &&
  606. getInstrPredicate(MI, MyPredReg) == Pred &&
  607. MyPredReg == PredReg))
  608. return false;
  609. return CheckCPSRDef ? !definesCPSR(MI) : true;
  610. }
  611. static inline unsigned getLSMultipleTransferSize(MachineInstr *MI) {
  612. switch (MI->getOpcode()) {
  613. default: return 0;
  614. case ARM::LDRi12:
  615. case ARM::STRi12:
  616. case ARM::t2LDRi8:
  617. case ARM::t2LDRi12:
  618. case ARM::t2STRi8:
  619. case ARM::t2STRi12:
  620. case ARM::VLDRS:
  621. case ARM::VSTRS:
  622. return 4;
  623. case ARM::VLDRD:
  624. case ARM::VSTRD:
  625. return 8;
  626. case ARM::LDMIA:
  627. case ARM::LDMDA:
  628. case ARM::LDMDB:
  629. case ARM::LDMIB:
  630. case ARM::STMIA:
  631. case ARM::STMDA:
  632. case ARM::STMDB:
  633. case ARM::STMIB:
  634. case ARM::t2LDMIA:
  635. case ARM::t2LDMDB:
  636. case ARM::t2STMIA:
  637. case ARM::t2STMDB:
  638. case ARM::VLDMSIA:
  639. case ARM::VSTMSIA:
  640. return (MI->getNumOperands() - MI->getDesc().getNumOperands() + 1) * 4;
  641. case ARM::VLDMDIA:
  642. case ARM::VSTMDIA:
  643. return (MI->getNumOperands() - MI->getDesc().getNumOperands() + 1) * 8;
  644. }
  645. }
  646. static unsigned getUpdatingLSMultipleOpcode(unsigned Opc,
  647. ARM_AM::AMSubMode Mode) {
  648. switch (Opc) {
  649. default: llvm_unreachable("Unhandled opcode!");
  650. case ARM::LDMIA:
  651. case ARM::LDMDA:
  652. case ARM::LDMDB:
  653. case ARM::LDMIB:
  654. switch (Mode) {
  655. default: llvm_unreachable("Unhandled submode!");
  656. case ARM_AM::ia: return ARM::LDMIA_UPD;
  657. case ARM_AM::ib: return ARM::LDMIB_UPD;
  658. case ARM_AM::da: return ARM::LDMDA_UPD;
  659. case ARM_AM::db: return ARM::LDMDB_UPD;
  660. }
  661. case ARM::STMIA:
  662. case ARM::STMDA:
  663. case ARM::STMDB:
  664. case ARM::STMIB:
  665. switch (Mode) {
  666. default: llvm_unreachable("Unhandled submode!");
  667. case ARM_AM::ia: return ARM::STMIA_UPD;
  668. case ARM_AM::ib: return ARM::STMIB_UPD;
  669. case ARM_AM::da: return ARM::STMDA_UPD;
  670. case ARM_AM::db: return ARM::STMDB_UPD;
  671. }
  672. case ARM::t2LDMIA:
  673. case ARM::t2LDMDB:
  674. switch (Mode) {
  675. default: llvm_unreachable("Unhandled submode!");
  676. case ARM_AM::ia: return ARM::t2LDMIA_UPD;
  677. case ARM_AM::db: return ARM::t2LDMDB_UPD;
  678. }
  679. case ARM::t2STMIA:
  680. case ARM::t2STMDB:
  681. switch (Mode) {
  682. default: llvm_unreachable("Unhandled submode!");
  683. case ARM_AM::ia: return ARM::t2STMIA_UPD;
  684. case ARM_AM::db: return ARM::t2STMDB_UPD;
  685. }
  686. case ARM::VLDMSIA:
  687. switch (Mode) {
  688. default: llvm_unreachable("Unhandled submode!");
  689. case ARM_AM::ia: return ARM::VLDMSIA_UPD;
  690. case ARM_AM::db: return ARM::VLDMSDB_UPD;
  691. }
  692. case ARM::VLDMDIA:
  693. switch (Mode) {
  694. default: llvm_unreachable("Unhandled submode!");
  695. case ARM_AM::ia: return ARM::VLDMDIA_UPD;
  696. case ARM_AM::db: return ARM::VLDMDDB_UPD;
  697. }
  698. case ARM::VSTMSIA:
  699. switch (Mode) {
  700. default: llvm_unreachable("Unhandled submode!");
  701. case ARM_AM::ia: return ARM::VSTMSIA_UPD;
  702. case ARM_AM::db: return ARM::VSTMSDB_UPD;
  703. }
  704. case ARM::VSTMDIA:
  705. switch (Mode) {
  706. default: llvm_unreachable("Unhandled submode!");
  707. case ARM_AM::ia: return ARM::VSTMDIA_UPD;
  708. case ARM_AM::db: return ARM::VSTMDDB_UPD;
  709. }
  710. }
  711. }
  712. /// MergeBaseUpdateLSMultiple - Fold proceeding/trailing inc/dec of base
  713. /// register into the LDM/STM/VLDM{D|S}/VSTM{D|S} op when possible:
  714. ///
  715. /// stmia rn, <ra, rb, rc>
  716. /// rn := rn + 4 * 3;
  717. /// =>
  718. /// stmia rn!, <ra, rb, rc>
  719. ///
  720. /// rn := rn - 4 * 3;
  721. /// ldmia rn, <ra, rb, rc>
  722. /// =>
  723. /// ldmdb rn!, <ra, rb, rc>
  724. bool ARMLoadStoreOpt::MergeBaseUpdateLSMultiple(MachineBasicBlock &MBB,
  725. MachineBasicBlock::iterator MBBI,
  726. bool &Advance,
  727. MachineBasicBlock::iterator &I) {
  728. MachineInstr *MI = MBBI;
  729. unsigned Base = MI->getOperand(0).getReg();
  730. bool BaseKill = MI->getOperand(0).isKill();
  731. unsigned Bytes = getLSMultipleTransferSize(MI);
  732. unsigned PredReg = 0;
  733. ARMCC::CondCodes Pred = getInstrPredicate(MI, PredReg);
  734. int Opcode = MI->getOpcode();
  735. DebugLoc dl = MI->getDebugLoc();
  736. // Can't use an updating ld/st if the base register is also a dest
  737. // register. e.g. ldmdb r0!, {r0, r1, r2}. The behavior is undefined.
  738. for (unsigned i = 2, e = MI->getNumOperands(); i != e; ++i)
  739. if (MI->getOperand(i).getReg() == Base)
  740. return false;
  741. bool DoMerge = false;
  742. ARM_AM::AMSubMode Mode = ARM_AM::getLoadStoreMultipleSubMode(Opcode);
  743. // Try merging with the previous instruction.
  744. MachineBasicBlock::iterator BeginMBBI = MBB.begin();
  745. if (MBBI != BeginMBBI) {
  746. MachineBasicBlock::iterator PrevMBBI = std::prev(MBBI);
  747. while (PrevMBBI != BeginMBBI && PrevMBBI->isDebugValue())
  748. --PrevMBBI;
  749. if (Mode == ARM_AM::ia &&
  750. isMatchingDecrement(PrevMBBI, Base, Bytes, 0, Pred, PredReg)) {
  751. Mode = ARM_AM::db;
  752. DoMerge = true;
  753. } else if (Mode == ARM_AM::ib &&
  754. isMatchingDecrement(PrevMBBI, Base, Bytes, 0, Pred, PredReg)) {
  755. Mode = ARM_AM::da;
  756. DoMerge = true;
  757. }
  758. if (DoMerge)
  759. MBB.erase(PrevMBBI);
  760. }
  761. // Try merging with the next instruction.
  762. MachineBasicBlock::iterator EndMBBI = MBB.end();
  763. if (!DoMerge && MBBI != EndMBBI) {
  764. MachineBasicBlock::iterator NextMBBI = std::next(MBBI);
  765. while (NextMBBI != EndMBBI && NextMBBI->isDebugValue())
  766. ++NextMBBI;
  767. if ((Mode == ARM_AM::ia || Mode == ARM_AM::ib) &&
  768. isMatchingIncrement(NextMBBI, Base, Bytes, 0, Pred, PredReg)) {
  769. DoMerge = true;
  770. } else if ((Mode == ARM_AM::da || Mode == ARM_AM::db) &&
  771. isMatchingDecrement(NextMBBI, Base, Bytes, 0, Pred, PredReg)) {
  772. DoMerge = true;
  773. }
  774. if (DoMerge) {
  775. if (NextMBBI == I) {
  776. Advance = true;
  777. ++I;
  778. }
  779. MBB.erase(NextMBBI);
  780. }
  781. }
  782. if (!DoMerge)
  783. return false;
  784. unsigned NewOpc = getUpdatingLSMultipleOpcode(Opcode, Mode);
  785. MachineInstrBuilder MIB = BuildMI(MBB, MBBI, dl, TII->get(NewOpc))
  786. .addReg(Base, getDefRegState(true)) // WB base register
  787. .addReg(Base, getKillRegState(BaseKill))
  788. .addImm(Pred).addReg(PredReg);
  789. // Transfer the rest of operands.
  790. for (unsigned OpNum = 3, e = MI->getNumOperands(); OpNum != e; ++OpNum)
  791. MIB.addOperand(MI->getOperand(OpNum));
  792. // Transfer memoperands.
  793. MIB->setMemRefs(MI->memoperands_begin(), MI->memoperands_end());
  794. MBB.erase(MBBI);
  795. return true;
  796. }
  797. static unsigned getPreIndexedLoadStoreOpcode(unsigned Opc,
  798. ARM_AM::AddrOpc Mode) {
  799. switch (Opc) {
  800. case ARM::LDRi12:
  801. return ARM::LDR_PRE_IMM;
  802. case ARM::STRi12:
  803. return ARM::STR_PRE_IMM;
  804. case ARM::VLDRS:
  805. return Mode == ARM_AM::add ? ARM::VLDMSIA_UPD : ARM::VLDMSDB_UPD;
  806. case ARM::VLDRD:
  807. return Mode == ARM_AM::add ? ARM::VLDMDIA_UPD : ARM::VLDMDDB_UPD;
  808. case ARM::VSTRS:
  809. return Mode == ARM_AM::add ? ARM::VSTMSIA_UPD : ARM::VSTMSDB_UPD;
  810. case ARM::VSTRD:
  811. return Mode == ARM_AM::add ? ARM::VSTMDIA_UPD : ARM::VSTMDDB_UPD;
  812. case ARM::t2LDRi8:
  813. case ARM::t2LDRi12:
  814. return ARM::t2LDR_PRE;
  815. case ARM::t2STRi8:
  816. case ARM::t2STRi12:
  817. return ARM::t2STR_PRE;
  818. default: llvm_unreachable("Unhandled opcode!");
  819. }
  820. }
  821. static unsigned getPostIndexedLoadStoreOpcode(unsigned Opc,
  822. ARM_AM::AddrOpc Mode) {
  823. switch (Opc) {
  824. case ARM::LDRi12:
  825. return ARM::LDR_POST_IMM;
  826. case ARM::STRi12:
  827. return ARM::STR_POST_IMM;
  828. case ARM::VLDRS:
  829. return Mode == ARM_AM::add ? ARM::VLDMSIA_UPD : ARM::VLDMSDB_UPD;
  830. case ARM::VLDRD:
  831. return Mode == ARM_AM::add ? ARM::VLDMDIA_UPD : ARM::VLDMDDB_UPD;
  832. case ARM::VSTRS:
  833. return Mode == ARM_AM::add ? ARM::VSTMSIA_UPD : ARM::VSTMSDB_UPD;
  834. case ARM::VSTRD:
  835. return Mode == ARM_AM::add ? ARM::VSTMDIA_UPD : ARM::VSTMDDB_UPD;
  836. case ARM::t2LDRi8:
  837. case ARM::t2LDRi12:
  838. return ARM::t2LDR_POST;
  839. case ARM::t2STRi8:
  840. case ARM::t2STRi12:
  841. return ARM::t2STR_POST;
  842. default: llvm_unreachable("Unhandled opcode!");
  843. }
  844. }
  845. /// MergeBaseUpdateLoadStore - Fold proceeding/trailing inc/dec of base
  846. /// register into the LDR/STR/FLD{D|S}/FST{D|S} op when possible:
  847. bool ARMLoadStoreOpt::MergeBaseUpdateLoadStore(MachineBasicBlock &MBB,
  848. MachineBasicBlock::iterator MBBI,
  849. const TargetInstrInfo *TII,
  850. bool &Advance,
  851. MachineBasicBlock::iterator &I) {
  852. MachineInstr *MI = MBBI;
  853. unsigned Base = MI->getOperand(1).getReg();
  854. bool BaseKill = MI->getOperand(1).isKill();
  855. unsigned Bytes = getLSMultipleTransferSize(MI);
  856. int Opcode = MI->getOpcode();
  857. DebugLoc dl = MI->getDebugLoc();
  858. bool isAM5 = (Opcode == ARM::VLDRD || Opcode == ARM::VLDRS ||
  859. Opcode == ARM::VSTRD || Opcode == ARM::VSTRS);
  860. bool isAM2 = (Opcode == ARM::LDRi12 || Opcode == ARM::STRi12);
  861. if (isi32Load(Opcode) || isi32Store(Opcode))
  862. if (MI->getOperand(2).getImm() != 0)
  863. return false;
  864. if (isAM5 && ARM_AM::getAM5Offset(MI->getOperand(2).getImm()) != 0)
  865. return false;
  866. bool isLd = isi32Load(Opcode) || Opcode == ARM::VLDRS || Opcode == ARM::VLDRD;
  867. // Can't do the merge if the destination register is the same as the would-be
  868. // writeback register.
  869. if (MI->getOperand(0).getReg() == Base)
  870. return false;
  871. unsigned PredReg = 0;
  872. ARMCC::CondCodes Pred = getInstrPredicate(MI, PredReg);
  873. bool DoMerge = false;
  874. ARM_AM::AddrOpc AddSub = ARM_AM::add;
  875. unsigned NewOpc = 0;
  876. // AM2 - 12 bits, thumb2 - 8 bits.
  877. unsigned Limit = isAM5 ? 0 : (isAM2 ? 0x1000 : 0x100);
  878. // Try merging with the previous instruction.
  879. MachineBasicBlock::iterator BeginMBBI = MBB.begin();
  880. if (MBBI != BeginMBBI) {
  881. MachineBasicBlock::iterator PrevMBBI = std::prev(MBBI);
  882. while (PrevMBBI != BeginMBBI && PrevMBBI->isDebugValue())
  883. --PrevMBBI;
  884. if (isMatchingDecrement(PrevMBBI, Base, Bytes, Limit, Pred, PredReg)) {
  885. DoMerge = true;
  886. AddSub = ARM_AM::sub;
  887. } else if (!isAM5 &&
  888. isMatchingIncrement(PrevMBBI, Base, Bytes, Limit,Pred,PredReg)) {
  889. DoMerge = true;
  890. }
  891. if (DoMerge) {
  892. NewOpc = getPreIndexedLoadStoreOpcode(Opcode, AddSub);
  893. MBB.erase(PrevMBBI);
  894. }
  895. }
  896. // Try merging with the next instruction.
  897. MachineBasicBlock::iterator EndMBBI = MBB.end();
  898. if (!DoMerge && MBBI != EndMBBI) {
  899. MachineBasicBlock::iterator NextMBBI = std::next(MBBI);
  900. while (NextMBBI != EndMBBI && NextMBBI->isDebugValue())
  901. ++NextMBBI;
  902. if (!isAM5 &&
  903. isMatchingDecrement(NextMBBI, Base, Bytes, Limit, Pred, PredReg)) {
  904. DoMerge = true;
  905. AddSub = ARM_AM::sub;
  906. } else if (isMatchingIncrement(NextMBBI, Base, Bytes, Limit,Pred,PredReg)) {
  907. DoMerge = true;
  908. }
  909. if (DoMerge) {
  910. NewOpc = getPostIndexedLoadStoreOpcode(Opcode, AddSub);
  911. if (NextMBBI == I) {
  912. Advance = true;
  913. ++I;
  914. }
  915. MBB.erase(NextMBBI);
  916. }
  917. }
  918. if (!DoMerge)
  919. return false;
  920. if (isAM5) {
  921. // VLDM[SD}_UPD, VSTM[SD]_UPD
  922. // (There are no base-updating versions of VLDR/VSTR instructions, but the
  923. // updating load/store-multiple instructions can be used with only one
  924. // register.)
  925. MachineOperand &MO = MI->getOperand(0);
  926. BuildMI(MBB, MBBI, dl, TII->get(NewOpc))
  927. .addReg(Base, getDefRegState(true)) // WB base register
  928. .addReg(Base, getKillRegState(isLd ? BaseKill : false))
  929. .addImm(Pred).addReg(PredReg)
  930. .addReg(MO.getReg(), (isLd ? getDefRegState(true) :
  931. getKillRegState(MO.isKill())));
  932. } else if (isLd) {
  933. if (isAM2) {
  934. // LDR_PRE, LDR_POST
  935. if (NewOpc == ARM::LDR_PRE_IMM || NewOpc == ARM::LDRB_PRE_IMM) {
  936. int Offset = AddSub == ARM_AM::sub ? -Bytes : Bytes;
  937. BuildMI(MBB, MBBI, dl, TII->get(NewOpc), MI->getOperand(0).getReg())
  938. .addReg(Base, RegState::Define)
  939. .addReg(Base).addImm(Offset).addImm(Pred).addReg(PredReg);
  940. } else {
  941. int Offset = ARM_AM::getAM2Opc(AddSub, Bytes, ARM_AM::no_shift);
  942. BuildMI(MBB, MBBI, dl, TII->get(NewOpc), MI->getOperand(0).getReg())
  943. .addReg(Base, RegState::Define)
  944. .addReg(Base).addReg(0).addImm(Offset).addImm(Pred).addReg(PredReg);
  945. }
  946. } else {
  947. int Offset = AddSub == ARM_AM::sub ? -Bytes : Bytes;
  948. // t2LDR_PRE, t2LDR_POST
  949. BuildMI(MBB, MBBI, dl, TII->get(NewOpc), MI->getOperand(0).getReg())
  950. .addReg(Base, RegState::Define)
  951. .addReg(Base).addImm(Offset).addImm(Pred).addReg(PredReg);
  952. }
  953. } else {
  954. MachineOperand &MO = MI->getOperand(0);
  955. // FIXME: post-indexed stores use am2offset_imm, which still encodes
  956. // the vestigal zero-reg offset register. When that's fixed, this clause
  957. // can be removed entirely.
  958. if (isAM2 && NewOpc == ARM::STR_POST_IMM) {
  959. int Offset = ARM_AM::getAM2Opc(AddSub, Bytes, ARM_AM::no_shift);
  960. // STR_PRE, STR_POST
  961. BuildMI(MBB, MBBI, dl, TII->get(NewOpc), Base)
  962. .addReg(MO.getReg(), getKillRegState(MO.isKill()))
  963. .addReg(Base).addReg(0).addImm(Offset).addImm(Pred).addReg(PredReg);
  964. } else {
  965. int Offset = AddSub == ARM_AM::sub ? -Bytes : Bytes;
  966. // t2STR_PRE, t2STR_POST
  967. BuildMI(MBB, MBBI, dl, TII->get(NewOpc), Base)
  968. .addReg(MO.getReg(), getKillRegState(MO.isKill()))
  969. .addReg(Base).addImm(Offset).addImm(Pred).addReg(PredReg);
  970. }
  971. }
  972. MBB.erase(MBBI);
  973. return true;
  974. }
  975. /// isMemoryOp - Returns true if instruction is a memory operation that this
  976. /// pass is capable of operating on.
  977. static bool isMemoryOp(const MachineInstr *MI) {
  978. // When no memory operands are present, conservatively assume unaligned,
  979. // volatile, unfoldable.
  980. if (!MI->hasOneMemOperand())
  981. return false;
  982. const MachineMemOperand *MMO = *MI->memoperands_begin();
  983. // Don't touch volatile memory accesses - we may be changing their order.
  984. if (MMO->isVolatile())
  985. return false;
  986. // Unaligned ldr/str is emulated by some kernels, but unaligned ldm/stm is
  987. // not.
  988. if (MMO->getAlignment() < 4)
  989. return false;
  990. // str <undef> could probably be eliminated entirely, but for now we just want
  991. // to avoid making a mess of it.
  992. // FIXME: Use str <undef> as a wildcard to enable better stm folding.
  993. if (MI->getNumOperands() > 0 && MI->getOperand(0).isReg() &&
  994. MI->getOperand(0).isUndef())
  995. return false;
  996. // Likewise don't mess with references to undefined addresses.
  997. if (MI->getNumOperands() > 1 && MI->getOperand(1).isReg() &&
  998. MI->getOperand(1).isUndef())
  999. return false;
  1000. int Opcode = MI->getOpcode();
  1001. switch (Opcode) {
  1002. default: break;
  1003. case ARM::VLDRS:
  1004. case ARM::VSTRS:
  1005. return MI->getOperand(1).isReg();
  1006. case ARM::VLDRD:
  1007. case ARM::VSTRD:
  1008. return MI->getOperand(1).isReg();
  1009. case ARM::LDRi12:
  1010. case ARM::STRi12:
  1011. case ARM::t2LDRi8:
  1012. case ARM::t2LDRi12:
  1013. case ARM::t2STRi8:
  1014. case ARM::t2STRi12:
  1015. return MI->getOperand(1).isReg();
  1016. }
  1017. return false;
  1018. }
  1019. /// AdvanceRS - Advance register scavenger to just before the earliest memory
  1020. /// op that is being merged.
  1021. void ARMLoadStoreOpt::AdvanceRS(MachineBasicBlock &MBB, MemOpQueue &MemOps) {
  1022. MachineBasicBlock::iterator Loc = MemOps[0].MBBI;
  1023. unsigned Position = MemOps[0].Position;
  1024. for (unsigned i = 1, e = MemOps.size(); i != e; ++i) {
  1025. if (MemOps[i].Position < Position) {
  1026. Position = MemOps[i].Position;
  1027. Loc = MemOps[i].MBBI;
  1028. }
  1029. }
  1030. if (Loc != MBB.begin())
  1031. RS->forward(std::prev(Loc));
  1032. }
  1033. static int getMemoryOpOffset(const MachineInstr *MI) {
  1034. int Opcode = MI->getOpcode();
  1035. bool isAM3 = Opcode == ARM::LDRD || Opcode == ARM::STRD;
  1036. unsigned NumOperands = MI->getDesc().getNumOperands();
  1037. unsigned OffField = MI->getOperand(NumOperands-3).getImm();
  1038. if (Opcode == ARM::t2LDRi12 || Opcode == ARM::t2LDRi8 ||
  1039. Opcode == ARM::t2STRi12 || Opcode == ARM::t2STRi8 ||
  1040. Opcode == ARM::t2LDRDi8 || Opcode == ARM::t2STRDi8 ||
  1041. Opcode == ARM::LDRi12 || Opcode == ARM::STRi12)
  1042. return OffField;
  1043. int Offset = isAM3 ? ARM_AM::getAM3Offset(OffField)
  1044. : ARM_AM::getAM5Offset(OffField) * 4;
  1045. if (isAM3) {
  1046. if (ARM_AM::getAM3Op(OffField) == ARM_AM::sub)
  1047. Offset = -Offset;
  1048. } else {
  1049. if (ARM_AM::getAM5Op(OffField) == ARM_AM::sub)
  1050. Offset = -Offset;
  1051. }
  1052. return Offset;
  1053. }
  1054. static void InsertLDR_STR(MachineBasicBlock &MBB,
  1055. MachineBasicBlock::iterator &MBBI,
  1056. int Offset, bool isDef,
  1057. DebugLoc dl, unsigned NewOpc,
  1058. unsigned Reg, bool RegDeadKill, bool RegUndef,
  1059. unsigned BaseReg, bool BaseKill, bool BaseUndef,
  1060. bool OffKill, bool OffUndef,
  1061. ARMCC::CondCodes Pred, unsigned PredReg,
  1062. const TargetInstrInfo *TII, bool isT2) {
  1063. if (isDef) {
  1064. MachineInstrBuilder MIB = BuildMI(MBB, MBBI, MBBI->getDebugLoc(),
  1065. TII->get(NewOpc))
  1066. .addReg(Reg, getDefRegState(true) | getDeadRegState(RegDeadKill))
  1067. .addReg(BaseReg, getKillRegState(BaseKill)|getUndefRegState(BaseUndef));
  1068. MIB.addImm(Offset).addImm(Pred).addReg(PredReg);
  1069. } else {
  1070. MachineInstrBuilder MIB = BuildMI(MBB, MBBI, MBBI->getDebugLoc(),
  1071. TII->get(NewOpc))
  1072. .addReg(Reg, getKillRegState(RegDeadKill) | getUndefRegState(RegUndef))
  1073. .addReg(BaseReg, getKillRegState(BaseKill)|getUndefRegState(BaseUndef));
  1074. MIB.addImm(Offset).addImm(Pred).addReg(PredReg);
  1075. }
  1076. }
  1077. bool ARMLoadStoreOpt::FixInvalidRegPairOp(MachineBasicBlock &MBB,
  1078. MachineBasicBlock::iterator &MBBI) {
  1079. MachineInstr *MI = &*MBBI;
  1080. unsigned Opcode = MI->getOpcode();
  1081. if (Opcode == ARM::LDRD || Opcode == ARM::STRD ||
  1082. Opcode == ARM::t2LDRDi8 || Opcode == ARM::t2STRDi8) {
  1083. const MachineOperand &BaseOp = MI->getOperand(2);
  1084. unsigned BaseReg = BaseOp.getReg();
  1085. unsigned EvenReg = MI->getOperand(0).getReg();
  1086. unsigned OddReg = MI->getOperand(1).getReg();
  1087. unsigned EvenRegNum = TRI->getDwarfRegNum(EvenReg, false);
  1088. unsigned OddRegNum = TRI->getDwarfRegNum(OddReg, false);
  1089. // ARM errata 602117: LDRD with base in list may result in incorrect base
  1090. // register when interrupted or faulted.
  1091. bool Errata602117 = EvenReg == BaseReg && STI->isCortexM3();
  1092. if (!Errata602117 &&
  1093. ((EvenRegNum & 1) == 0 && (EvenRegNum + 1) == OddRegNum))
  1094. return false;
  1095. MachineBasicBlock::iterator NewBBI = MBBI;
  1096. bool isT2 = Opcode == ARM::t2LDRDi8 || Opcode == ARM::t2STRDi8;
  1097. bool isLd = Opcode == ARM::LDRD || Opcode == ARM::t2LDRDi8;
  1098. bool EvenDeadKill = isLd ?
  1099. MI->getOperand(0).isDead() : MI->getOperand(0).isKill();
  1100. bool EvenUndef = MI->getOperand(0).isUndef();
  1101. bool OddDeadKill = isLd ?
  1102. MI->getOperand(1).isDead() : MI->getOperand(1).isKill();
  1103. bool OddUndef = MI->getOperand(1).isUndef();
  1104. bool BaseKill = BaseOp.isKill();
  1105. bool BaseUndef = BaseOp.isUndef();
  1106. bool OffKill = isT2 ? false : MI->getOperand(3).isKill();
  1107. bool OffUndef = isT2 ? false : MI->getOperand(3).isUndef();
  1108. int OffImm = getMemoryOpOffset(MI);
  1109. unsigned PredReg = 0;
  1110. ARMCC::CondCodes Pred = getInstrPredicate(MI, PredReg);
  1111. if (OddRegNum > EvenRegNum && OffImm == 0) {
  1112. // Ascending register numbers and no offset. It's safe to change it to a
  1113. // ldm or stm.
  1114. unsigned NewOpc = (isLd)
  1115. ? (isT2 ? ARM::t2LDMIA : ARM::LDMIA)
  1116. : (isT2 ? ARM::t2STMIA : ARM::STMIA);
  1117. if (isLd) {
  1118. BuildMI(MBB, MBBI, MBBI->getDebugLoc(), TII->get(NewOpc))
  1119. .addReg(BaseReg, getKillRegState(BaseKill))
  1120. .addImm(Pred).addReg(PredReg)
  1121. .addReg(EvenReg, getDefRegState(isLd) | getDeadRegState(EvenDeadKill))
  1122. .addReg(OddReg, getDefRegState(isLd) | getDeadRegState(OddDeadKill));
  1123. ++NumLDRD2LDM;
  1124. } else {
  1125. BuildMI(MBB, MBBI, MBBI->getDebugLoc(), TII->get(NewOpc))
  1126. .addReg(BaseReg, getKillRegState(BaseKill))
  1127. .addImm(Pred).addReg(PredReg)
  1128. .addReg(EvenReg,
  1129. getKillRegState(EvenDeadKill) | getUndefRegState(EvenUndef))
  1130. .addReg(OddReg,
  1131. getKillRegState(OddDeadKill) | getUndefRegState(OddUndef));
  1132. ++NumSTRD2STM;
  1133. }
  1134. NewBBI = std::prev(MBBI);
  1135. } else {
  1136. // Split into two instructions.
  1137. unsigned NewOpc = (isLd)
  1138. ? (isT2 ? (OffImm < 0 ? ARM::t2LDRi8 : ARM::t2LDRi12) : ARM::LDRi12)
  1139. : (isT2 ? (OffImm < 0 ? ARM::t2STRi8 : ARM::t2STRi12) : ARM::STRi12);
  1140. // Be extra careful for thumb2. t2LDRi8 can't reference a zero offset,
  1141. // so adjust and use t2LDRi12 here for that.
  1142. unsigned NewOpc2 = (isLd)
  1143. ? (isT2 ? (OffImm+4 < 0 ? ARM::t2LDRi8 : ARM::t2LDRi12) : ARM::LDRi12)
  1144. : (isT2 ? (OffImm+4 < 0 ? ARM::t2STRi8 : ARM::t2STRi12) : ARM::STRi12);
  1145. DebugLoc dl = MBBI->getDebugLoc();
  1146. // If this is a load and base register is killed, it may have been
  1147. // re-defed by the load, make sure the first load does not clobber it.
  1148. if (isLd &&
  1149. (BaseKill || OffKill) &&
  1150. (TRI->regsOverlap(EvenReg, BaseReg))) {
  1151. assert(!TRI->regsOverlap(OddReg, BaseReg));
  1152. InsertLDR_STR(MBB, MBBI, OffImm+4, isLd, dl, NewOpc2,
  1153. OddReg, OddDeadKill, false,
  1154. BaseReg, false, BaseUndef, false, OffUndef,
  1155. Pred, PredReg, TII, isT2);
  1156. NewBBI = std::prev(MBBI);
  1157. InsertLDR_STR(MBB, MBBI, OffImm, isLd, dl, NewOpc,
  1158. EvenReg, EvenDeadKill, false,
  1159. BaseReg, BaseKill, BaseUndef, OffKill, OffUndef,
  1160. Pred, PredReg, TII, isT2);
  1161. } else {
  1162. if (OddReg == EvenReg && EvenDeadKill) {
  1163. // If the two source operands are the same, the kill marker is
  1164. // probably on the first one. e.g.
  1165. // t2STRDi8 %R5<kill>, %R5, %R9<kill>, 0, 14, %reg0
  1166. EvenDeadKill = false;
  1167. OddDeadKill = true;
  1168. }
  1169. // Never kill the base register in the first instruction.
  1170. if (EvenReg == BaseReg)
  1171. EvenDeadKill = false;
  1172. InsertLDR_STR(MBB, MBBI, OffImm, isLd, dl, NewOpc,
  1173. EvenReg, EvenDeadKill, EvenUndef,
  1174. BaseReg, false, BaseUndef, false, OffUndef,
  1175. Pred, PredReg, TII, isT2);
  1176. NewBBI = std::prev(MBBI);
  1177. InsertLDR_STR(MBB, MBBI, OffImm+4, isLd, dl, NewOpc2,
  1178. OddReg, OddDeadKill, OddUndef,
  1179. BaseReg, BaseKill, BaseUndef, OffKill, OffUndef,
  1180. Pred, PredReg, TII, isT2);
  1181. }
  1182. if (isLd)
  1183. ++NumLDRD2LDR;
  1184. else
  1185. ++NumSTRD2STR;
  1186. }
  1187. MBB.erase(MI);
  1188. MBBI = NewBBI;
  1189. return true;
  1190. }
  1191. return false;
  1192. }
  1193. /// LoadStoreMultipleOpti - An optimization pass to turn multiple LDR / STR
  1194. /// ops of the same base and incrementing offset into LDM / STM ops.
  1195. bool ARMLoadStoreOpt::LoadStoreMultipleOpti(MachineBasicBlock &MBB) {
  1196. unsigned NumMerges = 0;
  1197. unsigned NumMemOps = 0;
  1198. MemOpQueue MemOps;
  1199. unsigned CurrBase = 0;
  1200. int CurrOpc = -1;
  1201. unsigned CurrSize = 0;
  1202. ARMCC::CondCodes CurrPred = ARMCC::AL;
  1203. unsigned CurrPredReg = 0;
  1204. unsigned Position = 0;
  1205. SmallVector<MachineBasicBlock::iterator,4> Merges;
  1206. RS->enterBasicBlock(&MBB);
  1207. MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end();
  1208. while (MBBI != E) {
  1209. if (FixInvalidRegPairOp(MBB, MBBI))
  1210. continue;
  1211. bool Advance = false;
  1212. bool TryMerge = false;
  1213. bool Clobber = false;
  1214. bool isMemOp = isMemoryOp(MBBI);
  1215. if (isMemOp) {
  1216. int Opcode = MBBI->getOpcode();
  1217. unsigned Size = getLSMultipleTransferSize(MBBI);
  1218. const MachineOperand &MO = MBBI->getOperand(0);
  1219. unsigned Reg = MO.getReg();
  1220. bool isKill = MO.isDef() ? false : MO.isKill();
  1221. unsigned Base = MBBI->getOperand(1).getReg();
  1222. unsigned PredReg = 0;
  1223. ARMCC::CondCodes Pred = getInstrPredicate(MBBI, PredReg);
  1224. int Offset = getMemoryOpOffset(MBBI);
  1225. // Watch out for:
  1226. // r4 := ldr [r5]
  1227. // r5 := ldr [r5, #4]
  1228. // r6 := ldr [r5, #8]
  1229. //
  1230. // The second ldr has effectively broken the chain even though it
  1231. // looks like the later ldr(s) use the same base register. Try to
  1232. // merge the ldr's so far, including this one. But don't try to
  1233. // combine the following ldr(s).
  1234. Clobber = (isi32Load(Opcode) && Base == MBBI->getOperand(0).getReg());
  1235. // Watch out for:
  1236. // r4 := ldr [r0, #8]
  1237. // r4 := ldr [r0, #4]
  1238. //
  1239. // The optimization may reorder the second ldr in front of the first
  1240. // ldr, which violates write after write(WAW) dependence. The same as
  1241. // str. Try to merge inst(s) already in MemOps.
  1242. bool Overlap = false;
  1243. for (MemOpQueueIter I = MemOps.begin(), E = MemOps.end(); I != E; ++I) {
  1244. if (TRI->regsOverlap(Reg, I->MBBI->getOperand(0).getReg())) {
  1245. Overlap = true;
  1246. break;
  1247. }
  1248. }
  1249. if (CurrBase == 0 && !Clobber) {
  1250. // Start of a new chain.
  1251. CurrBase = Base;
  1252. CurrOpc = Opcode;
  1253. CurrSize = Size;
  1254. CurrPred = Pred;
  1255. CurrPredReg = PredReg;
  1256. MemOps.push_back(MemOpQueueEntry(Offset, Reg, isKill, Position, MBBI));
  1257. ++NumMemOps;
  1258. Advance = true;
  1259. } else if (!Overlap) {
  1260. if (Clobber) {
  1261. TryMerge = true;
  1262. Advance = true;
  1263. }
  1264. if (CurrOpc == Opcode && CurrBase == Base && CurrPred == Pred) {
  1265. // No need to match PredReg.
  1266. // Continue adding to the queue.
  1267. if (Offset > MemOps.back().Offset) {
  1268. MemOps.push_back(MemOpQueueEntry(Offset, Reg, isKill,
  1269. Position, MBBI));
  1270. ++NumMemOps;
  1271. Advance = true;
  1272. } else {
  1273. for (MemOpQueueIter I = MemOps.begin(), E = MemOps.end();
  1274. I != E; ++I) {
  1275. if (Offset < I->Offset) {
  1276. MemOps.insert(I, MemOpQueueEntry(Offset, Reg, isKill,
  1277. Position, MBBI));
  1278. ++NumMemOps;
  1279. Advance = true;
  1280. break;
  1281. } else if (Offset == I->Offset) {
  1282. // Collision! This can't be merged!
  1283. break;
  1284. }
  1285. }
  1286. }
  1287. }
  1288. }
  1289. }
  1290. if (MBBI->isDebugValue()) {
  1291. ++MBBI;
  1292. if (MBBI == E)
  1293. // Reach the end of the block, try merging the memory instructions.
  1294. TryMerge = true;
  1295. } else if (Advance) {
  1296. ++Position;
  1297. ++MBBI;
  1298. if (MBBI == E)
  1299. // Reach the end of the block, try merging the memory instructions.
  1300. TryMerge = true;
  1301. } else
  1302. TryMerge = true;
  1303. if (TryMerge) {
  1304. if (NumMemOps > 1) {
  1305. // Try to find a free register to use as a new base in case it's needed.
  1306. // First advance to the instruction just before the start of the chain.
  1307. AdvanceRS(MBB, MemOps);
  1308. // Find a scratch register.
  1309. unsigned Scratch = RS->FindUnusedReg(&ARM::GPRRegClass);
  1310. // Process the load / store instructions.
  1311. RS->forward(std::prev(MBBI));
  1312. // Merge ops.
  1313. Merges.clear();
  1314. MergeLDR_STR(MBB, 0, CurrBase, CurrOpc, CurrSize,
  1315. CurrPred, CurrPredReg, Scratch, MemOps, Merges);
  1316. // Try folding preceding/trailing base inc/dec into the generated
  1317. // LDM/STM ops.
  1318. for (unsigned i = 0, e = Merges.size(); i < e; ++i)
  1319. if (MergeBaseUpdateLSMultiple(MBB, Merges[i], Advance, MBBI))
  1320. ++NumMerges;
  1321. NumMerges += Merges.size();
  1322. // Try folding preceding/trailing base inc/dec into those load/store
  1323. // that were not merged to form LDM/STM ops.
  1324. for (unsigned i = 0; i != NumMemOps; ++i)
  1325. if (!MemOps[i].Merged)
  1326. if (MergeBaseUpdateLoadStore(MBB, MemOps[i].MBBI, TII,Advance,MBBI))
  1327. ++NumMerges;
  1328. // RS may be pointing to an instruction that's deleted.
  1329. RS->skipTo(std::prev(MBBI));
  1330. } else if (NumMemOps == 1) {
  1331. // Try folding preceding/trailing base inc/dec into the single
  1332. // load/store.
  1333. if (MergeBaseUpdateLoadStore(MBB, MemOps[0].MBBI, TII, Advance, MBBI)) {
  1334. ++NumMerges;
  1335. RS->forward(std::prev(MBBI));
  1336. }
  1337. }
  1338. CurrBase = 0;
  1339. CurrOpc = -1;
  1340. CurrSize = 0;
  1341. CurrPred = ARMCC::AL;
  1342. CurrPredReg = 0;
  1343. if (NumMemOps) {
  1344. MemOps.clear();
  1345. NumMemOps = 0;
  1346. }
  1347. // If iterator hasn't been advanced and this is not a memory op, skip it.
  1348. // It can't start a new chain anyway.
  1349. if (!Advance && !isMemOp && MBBI != E) {
  1350. ++Position;
  1351. ++MBBI;
  1352. }
  1353. }
  1354. }
  1355. return NumMerges > 0;
  1356. }
  1357. /// MergeReturnIntoLDM - If this is a exit BB, try merging the return ops
  1358. /// ("bx lr" and "mov pc, lr") into the preceding stack restore so it
  1359. /// directly restore the value of LR into pc.
  1360. /// ldmfd sp!, {..., lr}
  1361. /// bx lr
  1362. /// or
  1363. /// ldmfd sp!, {..., lr}
  1364. /// mov pc, lr
  1365. /// =>
  1366. /// ldmfd sp!, {..., pc}
  1367. bool ARMLoadStoreOpt::MergeReturnIntoLDM(MachineBasicBlock &MBB) {
  1368. if (MBB.empty()) return false;
  1369. MachineBasicBlock::iterator MBBI = MBB.getLastNonDebugInstr();
  1370. if (MBBI != MBB.begin() &&
  1371. (MBBI->getOpcode() == ARM::BX_RET ||
  1372. MBBI->getOpcode() == ARM::tBX_RET ||
  1373. MBBI->getOpcode() == ARM::MOVPCLR)) {
  1374. MachineInstr *PrevMI = std::prev(MBBI);
  1375. unsigned Opcode = PrevMI->getOpcode();
  1376. if (Opcode == ARM::LDMIA_UPD || Opcode == ARM::LDMDA_UPD ||
  1377. Opcode == ARM::LDMDB_UPD || Opcode == ARM::LDMIB_UPD ||
  1378. Opcode == ARM::t2LDMIA_UPD || Opcode == ARM::t2LDMDB_UPD) {
  1379. MachineOperand &MO = PrevMI->getOperand(PrevMI->getNumOperands()-1);
  1380. if (MO.getReg() != ARM::LR)
  1381. return false;
  1382. unsigned NewOpc = (isThumb2 ? ARM::t2LDMIA_RET : ARM::LDMIA_RET);
  1383. assert(((isThumb2 && Opcode == ARM::t2LDMIA_UPD) ||
  1384. Opcode == ARM::LDMIA_UPD) && "Unsupported multiple load-return!");
  1385. PrevMI->setDesc(TII->get(NewOpc));
  1386. MO.setReg(ARM::PC);
  1387. PrevMI->copyImplicitOps(*MBB.getParent(), &*MBBI);
  1388. MBB.erase(MBBI);
  1389. return true;
  1390. }
  1391. }
  1392. return false;
  1393. }
  1394. bool ARMLoadStoreOpt::runOnMachineFunction(MachineFunction &Fn) {
  1395. const TargetMachine &TM = Fn.getTarget();
  1396. AFI = Fn.getInfo<ARMFunctionInfo>();
  1397. TII = TM.getInstrInfo();
  1398. TRI = TM.getRegisterInfo();
  1399. STI = &TM.getSubtarget<ARMSubtarget>();
  1400. RS = new RegScavenger();
  1401. isThumb2 = AFI->isThumb2Function();
  1402. bool Modified = false;
  1403. for (MachineFunction::iterator MFI = Fn.begin(), E = Fn.end(); MFI != E;
  1404. ++MFI) {
  1405. MachineBasicBlock &MBB = *MFI;
  1406. Modified |= LoadStoreMultipleOpti(MBB);
  1407. if (TM.getSubtarget<ARMSubtarget>().hasV5TOps())
  1408. Modified |= MergeReturnIntoLDM(MBB);
  1409. }
  1410. delete RS;
  1411. return Modified;
  1412. }
  1413. /// ARMPreAllocLoadStoreOpt - Pre- register allocation pass that move
  1414. /// load / stores from consecutive locations close to make it more
  1415. /// likely they will be combined later.
  1416. namespace {
  1417. struct ARMPreAllocLoadStoreOpt : public MachineFunctionPass{
  1418. static char ID;
  1419. ARMPreAllocLoadStoreOpt() : MachineFunctionPass(ID) {}
  1420. const DataLayout *TD;
  1421. const TargetInstrInfo *TII;
  1422. const TargetRegisterInfo *TRI;
  1423. const ARMSubtarget *STI;
  1424. MachineRegisterInfo *MRI;
  1425. MachineFunction *MF;
  1426. virtual bool runOnMachineFunction(MachineFunction &Fn);
  1427. virtual const char *getPassName() const {
  1428. return "ARM pre- register allocation load / store optimization pass";
  1429. }
  1430. private:
  1431. bool CanFormLdStDWord(MachineInstr *Op0, MachineInstr *Op1, DebugLoc &dl,
  1432. unsigned &NewOpc, unsigned &EvenReg,
  1433. unsigned &OddReg, unsigned &BaseReg,
  1434. int &Offset,
  1435. unsigned &PredReg, ARMCC::CondCodes &Pred,
  1436. bool &isT2);
  1437. bool RescheduleOps(MachineBasicBlock *MBB,
  1438. SmallVectorImpl<MachineInstr *> &Ops,
  1439. unsigned Base, bool isLd,
  1440. DenseMap<MachineInstr*, unsigned> &MI2LocMap);
  1441. bool RescheduleLoadStoreInstrs(MachineBasicBlock *MBB);
  1442. };
  1443. char ARMPreAllocLoadStoreOpt::ID = 0;
  1444. }
  1445. bool ARMPreAllocLoadStoreOpt::runOnMachineFunction(MachineFunction &Fn) {
  1446. TD = Fn.getTarget().getDataLayout();
  1447. TII = Fn.getTarget().getInstrInfo();
  1448. TRI = Fn.getTarget().getRegisterInfo();
  1449. STI = &Fn.getTarget().getSubtarget<ARMSubtarget>();
  1450. MRI = &Fn.getRegInfo();
  1451. MF = &Fn;
  1452. bool Modified = false;
  1453. for (MachineFunction::iterator MFI = Fn.begin(), E = Fn.end(); MFI != E;
  1454. ++MFI)
  1455. Modified |= RescheduleLoadStoreInstrs(MFI);
  1456. return Modified;
  1457. }
  1458. static bool IsSafeAndProfitableToMove(bool isLd, unsigned Base,
  1459. MachineBasicBlock::iterator I,
  1460. MachineBasicBlock::iterator E,
  1461. SmallPtrSet<MachineInstr*, 4> &MemOps,
  1462. SmallSet<unsigned, 4> &MemRegs,
  1463. const TargetRegisterInfo *TRI) {
  1464. // Are there stores / loads / calls between them?
  1465. // FIXME: This is overly conservative. We should make use of alias information
  1466. // some day.
  1467. SmallSet<unsigned, 4> AddedRegPressure;
  1468. while (++I != E) {
  1469. if (I->isDebugValue() || MemOps.count(&*I))
  1470. continue;
  1471. if (I->isCall() || I->isTerminator() || I->hasUnmodeledSideEffects())
  1472. return false;
  1473. if (isLd && I->mayStore())
  1474. return false;
  1475. if (!isLd) {
  1476. if (I->mayLoad())
  1477. return false;
  1478. // It's not safe to move the first 'str' down.
  1479. // str r1, [r0]
  1480. // strh r5, [r0]
  1481. // str r4, [r0, #+4]
  1482. if (I->mayStore())
  1483. return false;
  1484. }
  1485. for (unsigned j = 0, NumOps = I->getNumOperands(); j != NumOps; ++j) {
  1486. MachineOperand &MO = I->getOperand(j);
  1487. if (!MO.isReg())
  1488. continue;
  1489. unsigned Reg = MO.getReg();
  1490. if (MO.isDef() && TRI->regsOverlap(Reg, Base))
  1491. return false;
  1492. if (Reg != Base && !MemRegs.count(Reg))
  1493. AddedRegPressure.insert(Reg);
  1494. }
  1495. }
  1496. // Estimate register pressure increase due to the transformation.
  1497. if (MemRegs.size() <= 4)
  1498. // Ok if we are moving small number of instructions.
  1499. return true;
  1500. return AddedRegPressure.size() <= MemRegs.size() * 2;
  1501. }
  1502. /// Copy Op0 and Op1 operands into a new array assigned to MI.
  1503. static void concatenateMemOperands(MachineInstr *MI, MachineInstr *Op0,
  1504. MachineInstr *Op1) {
  1505. assert(MI->memoperands_empty() && "expected a new machineinstr");
  1506. size_t numMemRefs = (Op0->memoperands_end() - Op0->memoperands_begin())
  1507. + (Op1->memoperands_end() - Op1->memoperands_begin());
  1508. MachineFunction *MF = MI->getParent()->getParent();
  1509. MachineSDNode::mmo_iterator MemBegin = MF->allocateMemRefsArray(numMemRefs);
  1510. MachineSDNode::mmo_iterator MemEnd =
  1511. std::copy(Op0->memoperands_begin(), Op0->memoperands_end(), MemBegin);
  1512. MemEnd =
  1513. std::copy(Op1->memoperands_begin(), Op1->memoperands_end(), MemEnd);
  1514. MI->setMemRefs(MemBegin, MemEnd);
  1515. }
  1516. bool
  1517. ARMPreAllocLoadStoreOpt::CanFormLdStDWord(MachineInstr *Op0, MachineInstr *Op1,
  1518. DebugLoc &dl,
  1519. unsigned &NewOpc, unsigned &EvenReg,
  1520. unsigned &OddReg, unsigned &BaseReg,
  1521. int &Offset, unsigned &PredReg,
  1522. ARMCC::CondCodes &Pred,
  1523. bool &isT2) {
  1524. // Make sure we're allowed to generate LDRD/STRD.
  1525. if (!STI->hasV5TEOps())
  1526. return false;
  1527. // FIXME: VLDRS / VSTRS -> VLDRD / VSTRD
  1528. unsigned Scale = 1;
  1529. unsigned Opcode = Op0->getOpcode();
  1530. if (Opcode == ARM::LDRi12)
  1531. NewOpc = ARM::LDRD;
  1532. else if (Opcode == ARM::STRi12)
  1533. NewOpc = ARM::STRD;
  1534. else if (Opcode == ARM::t2LDRi8 || Opcode == ARM::t2LDRi12) {
  1535. NewOpc = ARM::t2LDRDi8;
  1536. Scale = 4;
  1537. isT2 = true;
  1538. } else if (Opcode == ARM::t2STRi8 || Opcode == ARM::t2STRi12) {
  1539. NewOpc = ARM::t2STRDi8;
  1540. Scale = 4;
  1541. isT2 = true;
  1542. } else
  1543. return false;
  1544. // Make sure the base address satisfies i64 ld / st alignment requirement.
  1545. // At the moment, we ignore the memoryoperand's value.
  1546. // If we want to use AliasAnalysis, we should check it accordingly.
  1547. if (!Op0->hasOneMemOperand() ||
  1548. (*Op0->memoperands_begin())->isVolatile())
  1549. return false;
  1550. unsigned Align = (*Op0->memoperands_begin())->getAlignment();
  1551. const Function *Func = MF->getFunction();
  1552. unsigned ReqAlign = STI->hasV6Ops()
  1553. ? TD->getABITypeAlignment(Type::getInt64Ty(Func->getContext()))
  1554. : 8; // Pre-v6 need 8-byte align
  1555. if (Align < ReqAlign)
  1556. return false;
  1557. // Then make sure the immediate offset fits.
  1558. int OffImm = getMemoryOpOffset(Op0);
  1559. if (isT2) {
  1560. int Limit = (1 << 8) * Scale;
  1561. if (OffImm >= Limit || (OffImm <= -Limit) || (OffImm & (Scale-1)))
  1562. return false;
  1563. Offset = OffImm;
  1564. } else {
  1565. ARM_AM::AddrOpc AddSub = ARM_AM::add;
  1566. if (OffImm < 0) {
  1567. AddSub = ARM_AM::sub;
  1568. OffImm = - OffImm;
  1569. }
  1570. int Limit = (1 << 8) * Scale;
  1571. if (OffImm >= Limit || (OffImm & (Scale-1)))
  1572. return false;
  1573. Offset = ARM_AM::getAM3Opc(AddSub, OffImm);
  1574. }
  1575. EvenReg = Op0->getOperand(0).getReg();
  1576. OddReg = Op1->getOperand(0).getReg();
  1577. if (EvenReg == OddReg)
  1578. return false;
  1579. BaseReg = Op0->getOperand(1).getReg();
  1580. Pred = getInstrPredicate(Op0, PredReg);
  1581. dl = Op0->getDebugLoc();
  1582. return true;
  1583. }
  1584. bool ARMPreAllocLoadStoreOpt::RescheduleOps(MachineBasicBlock *MBB,
  1585. SmallVectorImpl<MachineInstr *> &Ops,
  1586. unsigned Base, bool isLd,
  1587. DenseMap<MachineInstr*, unsigned> &MI2LocMap) {
  1588. bool RetVal = false;
  1589. // Sort by offset (in reverse order).
  1590. std::sort(Ops.begin(), Ops.end(),
  1591. [](const MachineInstr *LHS, const MachineInstr *RHS) {
  1592. int LOffset = getMemoryOpOffset(LHS);
  1593. int ROffset = getMemoryOpOffset(RHS);
  1594. assert(LHS == RHS || LOffset != ROffset);
  1595. return LOffset > ROffset;
  1596. });
  1597. // The loads / stores of the same base are in order. Scan them from first to
  1598. // last and check for the following:
  1599. // 1. Any def of base.
  1600. // 2. Any gaps.
  1601. while (Ops.size() > 1) {
  1602. unsigned FirstLoc = ~0U;
  1603. unsigned LastLoc = 0;
  1604. MachineInstr *FirstOp = 0;
  1605. MachineInstr *LastOp = 0;
  1606. int LastOffset = 0;
  1607. unsigned LastOpcode = 0;
  1608. unsigned LastBytes = 0;
  1609. unsigned NumMove = 0;
  1610. for (int i = Ops.size() - 1; i >= 0; --i) {
  1611. MachineInstr *Op = Ops[i];
  1612. unsigned Loc = MI2LocMap[Op];
  1613. if (Loc <= FirstLoc) {
  1614. FirstLoc = Loc;
  1615. FirstOp = Op;
  1616. }
  1617. if (Loc >= LastLoc) {
  1618. LastLoc = Loc;
  1619. LastOp = Op;
  1620. }
  1621. unsigned LSMOpcode
  1622. = getLoadStoreMultipleOpcode(Op->getOpcode(), ARM_AM::ia);
  1623. if (LastOpcode && LSMOpcode != LastOpcode)
  1624. break;
  1625. int Offset = getMemoryOpOffset(Op);
  1626. unsigned Bytes = getLSMultipleTransferSize(Op);
  1627. if (LastBytes) {
  1628. if (Bytes != LastBytes || Offset != (LastOffset + (int)Bytes))
  1629. break;
  1630. }
  1631. LastOffset = Offset;
  1632. LastBytes = Bytes;
  1633. LastOpcode = LSMOpcode;
  1634. if (++NumMove == 8) // FIXME: Tune this limit.
  1635. break;
  1636. }
  1637. if (NumMove <= 1)
  1638. Ops.pop_back();
  1639. else {
  1640. SmallPtrSet<MachineInstr*, 4> MemOps;
  1641. SmallSet<unsigned, 4> MemRegs;
  1642. for (int i = NumMove-1; i >= 0; --i) {
  1643. MemOps.insert(Ops[i]);
  1644. MemRegs.insert(Ops[i]->getOperand(0).getReg());
  1645. }
  1646. // Be conservative, if the instructions are too far apart, don't
  1647. // move them. We want to limit the increase of register pressure.
  1648. bool DoMove = (LastLoc - FirstLoc) <= NumMove*4; // FIXME: Tune this.
  1649. if (DoMove)
  1650. DoMove = IsSafeAndProfitableToMove(isLd, Base, FirstOp, LastOp,
  1651. MemOps, MemRegs, TRI);
  1652. if (!DoMove) {
  1653. for (unsigned i = 0; i != NumMove; ++i)
  1654. Ops.pop_back();
  1655. } else {
  1656. // This is the new location for the loads / stores.
  1657. MachineBasicBlock::iterator InsertPos = isLd ? FirstOp : LastOp;
  1658. while (InsertPos != MBB->end()
  1659. && (MemOps.count(InsertPos) || InsertPos->isDebugValue()))
  1660. ++InsertPos;
  1661. // If we are moving a pair of loads / stores, see if it makes sense
  1662. // to try to allocate a pair of registers that can form register pairs.
  1663. MachineInstr *Op0 = Ops.back();
  1664. MachineInstr *Op1 = Ops[Ops.size()-2];
  1665. unsigned EvenReg = 0, OddReg = 0;
  1666. unsigned BaseReg = 0, PredReg = 0;
  1667. ARMCC::CondCodes Pred = ARMCC::AL;
  1668. bool isT2 = false;
  1669. unsigned NewOpc = 0;
  1670. int Offset = 0;
  1671. DebugLoc dl;
  1672. if (NumMove == 2 && CanFormLdStDWord(Op0, Op1, dl, NewOpc,
  1673. EvenReg, OddReg, BaseReg,
  1674. Offset, PredReg, Pred, isT2)) {
  1675. Ops.pop_back();
  1676. Ops.pop_back();
  1677. const MCInstrDesc &MCID = TII->get(NewOpc);
  1678. const TargetRegisterClass *TRC = TII->getRegClass(MCID, 0, TRI, *MF);
  1679. MRI->constrainRegClass(EvenReg, TRC);
  1680. MRI->constrainRegClass(OddReg, TRC);
  1681. // Form the pair instruction.
  1682. if (isLd) {
  1683. MachineInstrBuilder MIB = BuildMI(*MBB, InsertPos, dl, MCID)
  1684. .addReg(EvenReg, RegState::Define)
  1685. .addReg(OddReg, RegState::Define)
  1686. .addReg(BaseReg);
  1687. // FIXME: We're converting from LDRi12 to an insn that still
  1688. // uses addrmode2, so we need an explicit offset reg. It should
  1689. // always by reg0 since we're transforming LDRi12s.
  1690. if (!isT2)
  1691. MIB.addReg(0);
  1692. MIB.addImm(Offset).addImm(Pred).addReg(PredReg);
  1693. concatenateMemOperands(MIB, Op0, Op1);
  1694. DEBUG(dbgs() << "Formed " << *MIB << "\n");
  1695. ++NumLDRDFormed;
  1696. } else {
  1697. MachineInstrBuilder MIB = BuildMI(*MBB, InsertPos, dl, MCID)
  1698. .addReg(EvenReg)
  1699. .addReg(OddReg)
  1700. .addReg(BaseReg);
  1701. // FIXME: We're converting from LDRi12 to an insn that still
  1702. // uses addrmode2, so we need an explicit offset reg. It should
  1703. // always by reg0 since we're transforming STRi12s.
  1704. if (!isT2)
  1705. MIB.addReg(0);
  1706. MIB.addImm(Offset).addImm(Pred).addReg(PredReg);
  1707. concatenateMemOperands(MIB, Op0, Op1);
  1708. DEBUG(dbgs() << "Formed " << *MIB << "\n");
  1709. ++NumSTRDFormed;
  1710. }
  1711. MBB->erase(Op0);
  1712. MBB->erase(Op1);
  1713. // Add register allocation hints to form register pairs.
  1714. MRI->setRegAllocationHint(EvenReg, ARMRI::RegPairEven, OddReg);
  1715. MRI->setRegAllocationHint(OddReg, ARMRI::RegPairOdd, EvenReg);
  1716. } else {
  1717. for (unsigned i = 0; i != NumMove; ++i) {
  1718. MachineInstr *Op = Ops.back();
  1719. Ops.pop_back();
  1720. MBB->splice(InsertPos, MBB, Op);
  1721. }
  1722. }
  1723. NumLdStMoved += NumMove;
  1724. RetVal = true;
  1725. }
  1726. }
  1727. }
  1728. return RetVal;
  1729. }
  1730. bool
  1731. ARMPreAllocLoadStoreOpt::RescheduleLoadStoreInstrs(MachineBasicBlock *MBB) {
  1732. bool RetVal = false;
  1733. DenseMap<MachineInstr*, unsigned> MI2LocMap;
  1734. DenseMap<unsigned, SmallVector<MachineInstr*, 4> > Base2LdsMap;
  1735. DenseMap<unsigned, SmallVector<MachineInstr*, 4> > Base2StsMap;
  1736. SmallVector<unsigned, 4> LdBases;
  1737. SmallVector<unsigned, 4> StBases;
  1738. unsigned Loc = 0;
  1739. MachineBasicBlock::iterator MBBI = MBB->begin();
  1740. MachineBasicBlock::iterator E = MBB->end();
  1741. while (MBBI != E) {
  1742. for (; MBBI != E; ++MBBI) {
  1743. MachineInstr *MI = MBBI;
  1744. if (MI->isCall() || MI->isTerminator()) {
  1745. // Stop at barriers.
  1746. ++MBBI;
  1747. break;
  1748. }
  1749. if (!MI->isDebugValue())
  1750. MI2LocMap[MI] = ++Loc;
  1751. if (!isMemoryOp(MI))
  1752. continue;
  1753. unsigned PredReg = 0;
  1754. if (getInstrPredicate(MI, PredReg) != ARMCC::AL)
  1755. continue;
  1756. int Opc = MI->getOpcode();
  1757. bool isLd = isi32Load(Opc) || Opc == ARM::VLDRS || Opc == ARM::VLDRD;
  1758. unsigned Base = MI->getOperand(1).getReg();
  1759. int Offset = getMemoryOpOffset(MI);
  1760. bool StopHere = false;
  1761. if (isLd) {
  1762. DenseMap<unsigned, SmallVector<MachineInstr*, 4> >::iterator BI =
  1763. Base2LdsMap.find(Base);
  1764. if (BI != Base2LdsMap.end()) {
  1765. for (unsigned i = 0, e = BI->second.size(); i != e; ++i) {
  1766. if (Offset == getMemoryOpOffset(BI->second[i])) {
  1767. StopHere = true;
  1768. break;
  1769. }
  1770. }
  1771. if (!StopHere)
  1772. BI->second.push_back(MI);
  1773. } else {
  1774. Base2LdsMap[Base].push_back(MI);
  1775. LdBases.push_back(Base);
  1776. }
  1777. } else {
  1778. DenseMap<unsigned, SmallVector<MachineInstr*, 4> >::iterator BI =
  1779. Base2StsMap.find(Base);
  1780. if (BI != Base2StsMap.end()) {
  1781. for (unsigned i = 0, e = BI->second.size(); i != e; ++i) {
  1782. if (Offset == getMemoryOpOffset(BI->second[i])) {
  1783. StopHere = true;
  1784. break;
  1785. }
  1786. }
  1787. if (!StopHere)
  1788. BI->second.push_back(MI);
  1789. } else {
  1790. Base2StsMap[Base].push_back(MI);
  1791. StBases.push_back(Base);
  1792. }
  1793. }
  1794. if (StopHere) {
  1795. // Found a duplicate (a base+offset combination that's seen earlier).
  1796. // Backtrack.
  1797. --Loc;
  1798. break;
  1799. }
  1800. }
  1801. // Re-schedule loads.
  1802. for (unsigned i = 0, e = LdBases.size(); i != e; ++i) {
  1803. unsigned Base = LdBases[i];
  1804. SmallVectorImpl<MachineInstr *> &Lds = Base2LdsMap[Base];
  1805. if (Lds.size() > 1)
  1806. RetVal |= RescheduleOps(MBB, Lds, Base, true, MI2LocMap);
  1807. }
  1808. // Re-schedule stores.
  1809. for (unsigned i = 0, e = StBases.size(); i != e; ++i) {
  1810. unsigned Base = StBases[i];
  1811. SmallVectorImpl<MachineInstr *> &Sts = Base2StsMap[Base];
  1812. if (Sts.size() > 1)
  1813. RetVal |= RescheduleOps(MBB, Sts, Base, false, MI2LocMap);
  1814. }
  1815. if (MBBI != E) {
  1816. Base2LdsMap.clear();
  1817. Base2StsMap.clear();
  1818. LdBases.clear();
  1819. StBases.clear();
  1820. }
  1821. }
  1822. return RetVal;
  1823. }
  1824. /// createARMLoadStoreOptimizationPass - returns an instance of the load / store
  1825. /// optimization pass.
  1826. FunctionPass *llvm::createARMLoadStoreOptimizationPass(bool PreAlloc) {
  1827. if (PreAlloc)
  1828. return new ARMPreAllocLoadStoreOpt();
  1829. return new ARMLoadStoreOpt();
  1830. }