CGStmtOpenMP.cpp 209 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149315031513152315331543155315631573158315931603161316231633164316531663167316831693170317131723173317431753176317731783179318031813182318331843185318631873188318931903191319231933194319531963197319831993200320132023203320432053206320732083209321032113212321332143215321632173218321932203221322232233224322532263227322832293230323132323233323432353236323732383239324032413242324332443245324632473248324932503251325232533254325532563257325832593260326132623263326432653266326732683269327032713272327332743275327632773278327932803281328232833284328532863287328832893290329132923293329432953296329732983299330033013302330333043305330633073308330933103311331233133314331533163317331833193320332133223323332433253326332733283329333033313332333333343335333633373338333933403341334233433344334533463347334833493350335133523353335433553356335733583359336033613362336333643365336633673368336933703371337233733374337533763377337833793380338133823383338433853386338733883389339033913392339333943395339633973398339934003401340234033404340534063407340834093410341134123413341434153416341734183419342034213422342334243425342634273428342934303431343234333434343534363437343834393440344134423443344434453446344734483449345034513452345334543455345634573458345934603461346234633464346534663467346834693470347134723473347434753476347734783479348034813482348334843485348634873488348934903491349234933494349534963497349834993500350135023503350435053506350735083509351035113512351335143515351635173518351935203521352235233524352535263527352835293530353135323533353435353536353735383539354035413542354335443545354635473548354935503551355235533554355535563557355835593560356135623563356435653566356735683569357035713572357335743575357635773578357935803581358235833584358535863587358835893590359135923593359435953596359735983599360036013602360336043605360636073608360936103611361236133614361536163617361836193620362136223623362436253626362736283629363036313632363336343635363636373638363936403641364236433644364536463647364836493650365136523653365436553656365736583659366036613662366336643665366636673668366936703671367236733674367536763677367836793680368136823683368436853686368736883689369036913692369336943695369636973698369937003701370237033704370537063707370837093710371137123713371437153716371737183719372037213722372337243725372637273728372937303731373237333734373537363737373837393740374137423743374437453746374737483749375037513752375337543755375637573758375937603761376237633764376537663767376837693770377137723773377437753776377737783779378037813782378337843785378637873788378937903791379237933794379537963797379837993800380138023803380438053806380738083809381038113812381338143815381638173818381938203821382238233824382538263827382838293830383138323833383438353836383738383839384038413842384338443845384638473848384938503851385238533854385538563857385838593860386138623863386438653866386738683869387038713872387338743875387638773878387938803881388238833884388538863887388838893890389138923893389438953896389738983899390039013902390339043905390639073908390939103911391239133914391539163917391839193920392139223923392439253926392739283929393039313932393339343935393639373938393939403941394239433944394539463947394839493950395139523953395439553956395739583959396039613962396339643965396639673968396939703971397239733974397539763977397839793980398139823983398439853986398739883989399039913992399339943995399639973998399940004001400240034004400540064007400840094010401140124013401440154016401740184019402040214022402340244025402640274028402940304031403240334034403540364037403840394040404140424043404440454046404740484049405040514052405340544055405640574058405940604061406240634064406540664067406840694070407140724073407440754076407740784079408040814082408340844085408640874088408940904091409240934094409540964097409840994100410141024103410441054106410741084109411041114112411341144115411641174118411941204121412241234124412541264127412841294130413141324133413441354136413741384139414041414142414341444145414641474148414941504151415241534154415541564157415841594160416141624163416441654166416741684169417041714172417341744175417641774178417941804181418241834184418541864187418841894190419141924193419441954196419741984199420042014202420342044205420642074208420942104211421242134214421542164217421842194220422142224223422442254226422742284229423042314232423342344235423642374238423942404241424242434244424542464247424842494250425142524253425442554256425742584259426042614262426342644265426642674268426942704271427242734274427542764277427842794280428142824283428442854286428742884289429042914292429342944295429642974298429943004301430243034304430543064307430843094310431143124313431443154316431743184319432043214322432343244325432643274328432943304331433243334334433543364337433843394340434143424343434443454346434743484349435043514352435343544355435643574358435943604361436243634364436543664367436843694370437143724373437443754376437743784379438043814382438343844385438643874388438943904391439243934394439543964397439843994400440144024403440444054406440744084409441044114412441344144415441644174418441944204421442244234424442544264427442844294430443144324433443444354436443744384439444044414442444344444445444644474448444944504451445244534454445544564457445844594460446144624463446444654466446744684469447044714472447344744475447644774478447944804481448244834484448544864487448844894490449144924493449444954496449744984499450045014502450345044505450645074508450945104511451245134514451545164517451845194520452145224523452445254526452745284529453045314532453345344535453645374538453945404541454245434544454545464547454845494550455145524553455445554556455745584559456045614562456345644565456645674568456945704571457245734574457545764577457845794580458145824583458445854586458745884589459045914592459345944595459645974598459946004601460246034604460546064607460846094610461146124613461446154616461746184619462046214622462346244625462646274628462946304631463246334634463546364637463846394640464146424643464446454646464746484649465046514652465346544655465646574658465946604661466246634664466546664667466846694670467146724673467446754676467746784679468046814682468346844685468646874688468946904691469246934694469546964697469846994700470147024703470447054706470747084709471047114712471347144715471647174718471947204721472247234724472547264727472847294730473147324733473447354736473747384739474047414742474347444745474647474748474947504751475247534754475547564757475847594760476147624763476447654766476747684769477047714772477347744775477647774778477947804781478247834784478547864787478847894790479147924793479447954796479747984799480048014802480348044805480648074808480948104811481248134814481548164817481848194820482148224823482448254826482748284829483048314832483348344835483648374838483948404841484248434844484548464847484848494850485148524853485448554856485748584859486048614862486348644865486648674868486948704871487248734874487548764877487848794880488148824883488448854886488748884889489048914892489348944895489648974898489949004901490249034904490549064907490849094910491149124913491449154916491749184919492049214922492349244925492649274928492949304931493249334934493549364937493849394940494149424943494449454946494749484949495049514952495349544955495649574958495949604961496249634964496549664967496849694970497149724973497449754976497749784979498049814982498349844985498649874988
  1. //===--- CGStmtOpenMP.cpp - Emit LLVM Code from Statements ----------------===//
  2. //
  3. // The LLVM Compiler Infrastructure
  4. //
  5. // This file is distributed under the University of Illinois Open Source
  6. // License. See LICENSE.TXT for details.
  7. //
  8. //===----------------------------------------------------------------------===//
  9. //
  10. // This contains code to emit OpenMP nodes as LLVM code.
  11. //
  12. //===----------------------------------------------------------------------===//
  13. #include "CGCleanup.h"
  14. #include "CGOpenMPRuntime.h"
  15. #include "CodeGenFunction.h"
  16. #include "CodeGenModule.h"
  17. #include "TargetInfo.h"
  18. #include "clang/AST/Stmt.h"
  19. #include "clang/AST/StmtOpenMP.h"
  20. #include "clang/AST/DeclOpenMP.h"
  21. #include "llvm/IR/CallSite.h"
  22. using namespace clang;
  23. using namespace CodeGen;
  24. namespace {
  25. /// Lexical scope for OpenMP executable constructs, that handles correct codegen
  26. /// for captured expressions.
  27. class OMPLexicalScope : public CodeGenFunction::LexicalScope {
  28. void emitPreInitStmt(CodeGenFunction &CGF, const OMPExecutableDirective &S) {
  29. for (const auto *C : S.clauses()) {
  30. if (const auto *CPI = OMPClauseWithPreInit::get(C)) {
  31. if (const auto *PreInit =
  32. cast_or_null<DeclStmt>(CPI->getPreInitStmt())) {
  33. for (const auto *I : PreInit->decls()) {
  34. if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
  35. CGF.EmitVarDecl(cast<VarDecl>(*I));
  36. } else {
  37. CodeGenFunction::AutoVarEmission Emission =
  38. CGF.EmitAutoVarAlloca(cast<VarDecl>(*I));
  39. CGF.EmitAutoVarCleanups(Emission);
  40. }
  41. }
  42. }
  43. }
  44. }
  45. }
  46. CodeGenFunction::OMPPrivateScope InlinedShareds;
  47. static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) {
  48. return CGF.LambdaCaptureFields.lookup(VD) ||
  49. (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) ||
  50. (CGF.CurCodeDecl && isa<BlockDecl>(CGF.CurCodeDecl));
  51. }
  52. public:
  53. OMPLexicalScope(
  54. CodeGenFunction &CGF, const OMPExecutableDirective &S,
  55. const llvm::Optional<OpenMPDirectiveKind> CapturedRegion = llvm::None,
  56. const bool EmitPreInitStmt = true)
  57. : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()),
  58. InlinedShareds(CGF) {
  59. if (EmitPreInitStmt)
  60. emitPreInitStmt(CGF, S);
  61. if (!CapturedRegion.hasValue())
  62. return;
  63. assert(S.hasAssociatedStmt() &&
  64. "Expected associated statement for inlined directive.");
  65. const CapturedStmt *CS = S.getCapturedStmt(*CapturedRegion);
  66. for (const auto &C : CS->captures()) {
  67. if (C.capturesVariable() || C.capturesVariableByCopy()) {
  68. auto *VD = C.getCapturedVar();
  69. assert(VD == VD->getCanonicalDecl() &&
  70. "Canonical decl must be captured.");
  71. DeclRefExpr DRE(
  72. const_cast<VarDecl *>(VD),
  73. isCapturedVar(CGF, VD) || (CGF.CapturedStmtInfo &&
  74. InlinedShareds.isGlobalVarCaptured(VD)),
  75. VD->getType().getNonReferenceType(), VK_LValue, C.getLocation());
  76. InlinedShareds.addPrivate(VD, [&CGF, &DRE]() -> Address {
  77. return CGF.EmitLValue(&DRE).getAddress();
  78. });
  79. }
  80. }
  81. (void)InlinedShareds.Privatize();
  82. }
  83. };
  84. /// Lexical scope for OpenMP parallel construct, that handles correct codegen
  85. /// for captured expressions.
  86. class OMPParallelScope final : public OMPLexicalScope {
  87. bool EmitPreInitStmt(const OMPExecutableDirective &S) {
  88. OpenMPDirectiveKind Kind = S.getDirectiveKind();
  89. return !(isOpenMPTargetExecutionDirective(Kind) ||
  90. isOpenMPLoopBoundSharingDirective(Kind)) &&
  91. isOpenMPParallelDirective(Kind);
  92. }
  93. public:
  94. OMPParallelScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
  95. : OMPLexicalScope(CGF, S, /*CapturedRegion=*/llvm::None,
  96. EmitPreInitStmt(S)) {}
  97. };
  98. /// Lexical scope for OpenMP teams construct, that handles correct codegen
  99. /// for captured expressions.
  100. class OMPTeamsScope final : public OMPLexicalScope {
  101. bool EmitPreInitStmt(const OMPExecutableDirective &S) {
  102. OpenMPDirectiveKind Kind = S.getDirectiveKind();
  103. return !isOpenMPTargetExecutionDirective(Kind) &&
  104. isOpenMPTeamsDirective(Kind);
  105. }
  106. public:
  107. OMPTeamsScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
  108. : OMPLexicalScope(CGF, S, /*CapturedRegion=*/llvm::None,
  109. EmitPreInitStmt(S)) {}
  110. };
  111. /// Private scope for OpenMP loop-based directives, that supports capturing
  112. /// of used expression from loop statement.
  113. class OMPLoopScope : public CodeGenFunction::RunCleanupsScope {
  114. void emitPreInitStmt(CodeGenFunction &CGF, const OMPLoopDirective &S) {
  115. CodeGenFunction::OMPMapVars PreCondVars;
  116. for (const auto *E : S.counters()) {
  117. const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
  118. (void)PreCondVars.setVarAddr(
  119. CGF, VD, CGF.CreateMemTemp(VD->getType().getNonReferenceType()));
  120. }
  121. (void)PreCondVars.apply(CGF);
  122. if (const auto *PreInits = cast_or_null<DeclStmt>(S.getPreInits())) {
  123. for (const auto *I : PreInits->decls())
  124. CGF.EmitVarDecl(cast<VarDecl>(*I));
  125. }
  126. PreCondVars.restore(CGF);
  127. }
  128. public:
  129. OMPLoopScope(CodeGenFunction &CGF, const OMPLoopDirective &S)
  130. : CodeGenFunction::RunCleanupsScope(CGF) {
  131. emitPreInitStmt(CGF, S);
  132. }
  133. };
  134. class OMPSimdLexicalScope : public CodeGenFunction::LexicalScope {
  135. CodeGenFunction::OMPPrivateScope InlinedShareds;
  136. static bool isCapturedVar(CodeGenFunction &CGF, const VarDecl *VD) {
  137. return CGF.LambdaCaptureFields.lookup(VD) ||
  138. (CGF.CapturedStmtInfo && CGF.CapturedStmtInfo->lookup(VD)) ||
  139. (CGF.CurCodeDecl && isa<BlockDecl>(CGF.CurCodeDecl) &&
  140. cast<BlockDecl>(CGF.CurCodeDecl)->capturesVariable(VD));
  141. }
  142. public:
  143. OMPSimdLexicalScope(CodeGenFunction &CGF, const OMPExecutableDirective &S)
  144. : CodeGenFunction::LexicalScope(CGF, S.getSourceRange()),
  145. InlinedShareds(CGF) {
  146. for (const auto *C : S.clauses()) {
  147. if (const auto *CPI = OMPClauseWithPreInit::get(C)) {
  148. if (const auto *PreInit =
  149. cast_or_null<DeclStmt>(CPI->getPreInitStmt())) {
  150. for (const auto *I : PreInit->decls()) {
  151. if (!I->hasAttr<OMPCaptureNoInitAttr>()) {
  152. CGF.EmitVarDecl(cast<VarDecl>(*I));
  153. } else {
  154. CodeGenFunction::AutoVarEmission Emission =
  155. CGF.EmitAutoVarAlloca(cast<VarDecl>(*I));
  156. CGF.EmitAutoVarCleanups(Emission);
  157. }
  158. }
  159. }
  160. } else if (const auto *UDP = dyn_cast<OMPUseDevicePtrClause>(C)) {
  161. for (const Expr *E : UDP->varlists()) {
  162. const Decl *D = cast<DeclRefExpr>(E)->getDecl();
  163. if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(D))
  164. CGF.EmitVarDecl(*OED);
  165. }
  166. }
  167. }
  168. if (!isOpenMPSimdDirective(S.getDirectiveKind()))
  169. CGF.EmitOMPPrivateClause(S, InlinedShareds);
  170. if (const auto *TG = dyn_cast<OMPTaskgroupDirective>(&S)) {
  171. if (const Expr *E = TG->getReductionRef())
  172. CGF.EmitVarDecl(*cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl()));
  173. }
  174. const auto *CS = cast_or_null<CapturedStmt>(S.getAssociatedStmt());
  175. while (CS) {
  176. for (auto &C : CS->captures()) {
  177. if (C.capturesVariable() || C.capturesVariableByCopy()) {
  178. auto *VD = C.getCapturedVar();
  179. assert(VD == VD->getCanonicalDecl() &&
  180. "Canonical decl must be captured.");
  181. DeclRefExpr DRE(const_cast<VarDecl *>(VD),
  182. isCapturedVar(CGF, VD) ||
  183. (CGF.CapturedStmtInfo &&
  184. InlinedShareds.isGlobalVarCaptured(VD)),
  185. VD->getType().getNonReferenceType(), VK_LValue,
  186. C.getLocation());
  187. InlinedShareds.addPrivate(VD, [&CGF, &DRE]() -> Address {
  188. return CGF.EmitLValue(&DRE).getAddress();
  189. });
  190. }
  191. }
  192. CS = dyn_cast<CapturedStmt>(CS->getCapturedStmt());
  193. }
  194. (void)InlinedShareds.Privatize();
  195. }
  196. };
  197. } // namespace
  198. static void emitCommonOMPTargetDirective(CodeGenFunction &CGF,
  199. const OMPExecutableDirective &S,
  200. const RegionCodeGenTy &CodeGen);
  201. LValue CodeGenFunction::EmitOMPSharedLValue(const Expr *E) {
  202. if (const auto *OrigDRE = dyn_cast<DeclRefExpr>(E)) {
  203. if (const auto *OrigVD = dyn_cast<VarDecl>(OrigDRE->getDecl())) {
  204. OrigVD = OrigVD->getCanonicalDecl();
  205. bool IsCaptured =
  206. LambdaCaptureFields.lookup(OrigVD) ||
  207. (CapturedStmtInfo && CapturedStmtInfo->lookup(OrigVD)) ||
  208. (CurCodeDecl && isa<BlockDecl>(CurCodeDecl));
  209. DeclRefExpr DRE(const_cast<VarDecl *>(OrigVD), IsCaptured,
  210. OrigDRE->getType(), VK_LValue, OrigDRE->getExprLoc());
  211. return EmitLValue(&DRE);
  212. }
  213. }
  214. return EmitLValue(E);
  215. }
  216. llvm::Value *CodeGenFunction::getTypeSize(QualType Ty) {
  217. ASTContext &C = getContext();
  218. llvm::Value *Size = nullptr;
  219. auto SizeInChars = C.getTypeSizeInChars(Ty);
  220. if (SizeInChars.isZero()) {
  221. // getTypeSizeInChars() returns 0 for a VLA.
  222. while (const VariableArrayType *VAT = C.getAsVariableArrayType(Ty)) {
  223. VlaSizePair VlaSize = getVLASize(VAT);
  224. Ty = VlaSize.Type;
  225. Size = Size ? Builder.CreateNUWMul(Size, VlaSize.NumElts)
  226. : VlaSize.NumElts;
  227. }
  228. SizeInChars = C.getTypeSizeInChars(Ty);
  229. if (SizeInChars.isZero())
  230. return llvm::ConstantInt::get(SizeTy, /*V=*/0);
  231. return Builder.CreateNUWMul(Size, CGM.getSize(SizeInChars));
  232. }
  233. return CGM.getSize(SizeInChars);
  234. }
  235. void CodeGenFunction::GenerateOpenMPCapturedVars(
  236. const CapturedStmt &S, SmallVectorImpl<llvm::Value *> &CapturedVars) {
  237. const RecordDecl *RD = S.getCapturedRecordDecl();
  238. auto CurField = RD->field_begin();
  239. auto CurCap = S.captures().begin();
  240. for (CapturedStmt::const_capture_init_iterator I = S.capture_init_begin(),
  241. E = S.capture_init_end();
  242. I != E; ++I, ++CurField, ++CurCap) {
  243. if (CurField->hasCapturedVLAType()) {
  244. const VariableArrayType *VAT = CurField->getCapturedVLAType();
  245. llvm::Value *Val = VLASizeMap[VAT->getSizeExpr()];
  246. CapturedVars.push_back(Val);
  247. } else if (CurCap->capturesThis()) {
  248. CapturedVars.push_back(CXXThisValue);
  249. } else if (CurCap->capturesVariableByCopy()) {
  250. llvm::Value *CV = EmitLoadOfScalar(EmitLValue(*I), CurCap->getLocation());
  251. // If the field is not a pointer, we need to save the actual value
  252. // and load it as a void pointer.
  253. if (!CurField->getType()->isAnyPointerType()) {
  254. ASTContext &Ctx = getContext();
  255. Address DstAddr = CreateMemTemp(
  256. Ctx.getUIntPtrType(),
  257. Twine(CurCap->getCapturedVar()->getName(), ".casted"));
  258. LValue DstLV = MakeAddrLValue(DstAddr, Ctx.getUIntPtrType());
  259. llvm::Value *SrcAddrVal = EmitScalarConversion(
  260. DstAddr.getPointer(), Ctx.getPointerType(Ctx.getUIntPtrType()),
  261. Ctx.getPointerType(CurField->getType()), CurCap->getLocation());
  262. LValue SrcLV =
  263. MakeNaturalAlignAddrLValue(SrcAddrVal, CurField->getType());
  264. // Store the value using the source type pointer.
  265. EmitStoreThroughLValue(RValue::get(CV), SrcLV);
  266. // Load the value using the destination type pointer.
  267. CV = EmitLoadOfScalar(DstLV, CurCap->getLocation());
  268. }
  269. CapturedVars.push_back(CV);
  270. } else {
  271. assert(CurCap->capturesVariable() && "Expected capture by reference.");
  272. CapturedVars.push_back(EmitLValue(*I).getAddress().getPointer());
  273. }
  274. }
  275. }
  276. static Address castValueFromUintptr(CodeGenFunction &CGF, SourceLocation Loc,
  277. QualType DstType, StringRef Name,
  278. LValue AddrLV,
  279. bool isReferenceType = false) {
  280. ASTContext &Ctx = CGF.getContext();
  281. llvm::Value *CastedPtr = CGF.EmitScalarConversion(
  282. AddrLV.getAddress().getPointer(), Ctx.getUIntPtrType(),
  283. Ctx.getPointerType(DstType), Loc);
  284. Address TmpAddr =
  285. CGF.MakeNaturalAlignAddrLValue(CastedPtr, Ctx.getPointerType(DstType))
  286. .getAddress();
  287. // If we are dealing with references we need to return the address of the
  288. // reference instead of the reference of the value.
  289. if (isReferenceType) {
  290. QualType RefType = Ctx.getLValueReferenceType(DstType);
  291. llvm::Value *RefVal = TmpAddr.getPointer();
  292. TmpAddr = CGF.CreateMemTemp(RefType, Twine(Name, ".ref"));
  293. LValue TmpLVal = CGF.MakeAddrLValue(TmpAddr, RefType);
  294. CGF.EmitStoreThroughLValue(RValue::get(RefVal), TmpLVal, /*isInit=*/true);
  295. }
  296. return TmpAddr;
  297. }
  298. static QualType getCanonicalParamType(ASTContext &C, QualType T) {
  299. if (T->isLValueReferenceType())
  300. return C.getLValueReferenceType(
  301. getCanonicalParamType(C, T.getNonReferenceType()),
  302. /*SpelledAsLValue=*/false);
  303. if (T->isPointerType())
  304. return C.getPointerType(getCanonicalParamType(C, T->getPointeeType()));
  305. if (const ArrayType *A = T->getAsArrayTypeUnsafe()) {
  306. if (const auto *VLA = dyn_cast<VariableArrayType>(A))
  307. return getCanonicalParamType(C, VLA->getElementType());
  308. if (!A->isVariablyModifiedType())
  309. return C.getCanonicalType(T);
  310. }
  311. return C.getCanonicalParamType(T);
  312. }
  313. namespace {
  314. /// Contains required data for proper outlined function codegen.
  315. struct FunctionOptions {
  316. /// Captured statement for which the function is generated.
  317. const CapturedStmt *S = nullptr;
  318. /// true if cast to/from UIntPtr is required for variables captured by
  319. /// value.
  320. const bool UIntPtrCastRequired = true;
  321. /// true if only casted arguments must be registered as local args or VLA
  322. /// sizes.
  323. const bool RegisterCastedArgsOnly = false;
  324. /// Name of the generated function.
  325. const StringRef FunctionName;
  326. explicit FunctionOptions(const CapturedStmt *S, bool UIntPtrCastRequired,
  327. bool RegisterCastedArgsOnly,
  328. StringRef FunctionName)
  329. : S(S), UIntPtrCastRequired(UIntPtrCastRequired),
  330. RegisterCastedArgsOnly(UIntPtrCastRequired && RegisterCastedArgsOnly),
  331. FunctionName(FunctionName) {}
  332. };
  333. }
  334. static llvm::Function *emitOutlinedFunctionPrologue(
  335. CodeGenFunction &CGF, FunctionArgList &Args,
  336. llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>>
  337. &LocalAddrs,
  338. llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>>
  339. &VLASizes,
  340. llvm::Value *&CXXThisValue, const FunctionOptions &FO) {
  341. const CapturedDecl *CD = FO.S->getCapturedDecl();
  342. const RecordDecl *RD = FO.S->getCapturedRecordDecl();
  343. assert(CD->hasBody() && "missing CapturedDecl body");
  344. CXXThisValue = nullptr;
  345. // Build the argument list.
  346. CodeGenModule &CGM = CGF.CGM;
  347. ASTContext &Ctx = CGM.getContext();
  348. FunctionArgList TargetArgs;
  349. Args.append(CD->param_begin(),
  350. std::next(CD->param_begin(), CD->getContextParamPosition()));
  351. TargetArgs.append(
  352. CD->param_begin(),
  353. std::next(CD->param_begin(), CD->getContextParamPosition()));
  354. auto I = FO.S->captures().begin();
  355. FunctionDecl *DebugFunctionDecl = nullptr;
  356. if (!FO.UIntPtrCastRequired) {
  357. FunctionProtoType::ExtProtoInfo EPI;
  358. DebugFunctionDecl = FunctionDecl::Create(
  359. Ctx, Ctx.getTranslationUnitDecl(), FO.S->getBeginLoc(),
  360. SourceLocation(), DeclarationName(), Ctx.VoidTy,
  361. Ctx.getTrivialTypeSourceInfo(
  362. Ctx.getFunctionType(Ctx.VoidTy, llvm::None, EPI)),
  363. SC_Static, /*isInlineSpecified=*/false, /*hasWrittenPrototype=*/false);
  364. }
  365. for (const FieldDecl *FD : RD->fields()) {
  366. QualType ArgType = FD->getType();
  367. IdentifierInfo *II = nullptr;
  368. VarDecl *CapVar = nullptr;
  369. // If this is a capture by copy and the type is not a pointer, the outlined
  370. // function argument type should be uintptr and the value properly casted to
  371. // uintptr. This is necessary given that the runtime library is only able to
  372. // deal with pointers. We can pass in the same way the VLA type sizes to the
  373. // outlined function.
  374. if (FO.UIntPtrCastRequired &&
  375. ((I->capturesVariableByCopy() && !ArgType->isAnyPointerType()) ||
  376. I->capturesVariableArrayType()))
  377. ArgType = Ctx.getUIntPtrType();
  378. if (I->capturesVariable() || I->capturesVariableByCopy()) {
  379. CapVar = I->getCapturedVar();
  380. II = CapVar->getIdentifier();
  381. } else if (I->capturesThis()) {
  382. II = &Ctx.Idents.get("this");
  383. } else {
  384. assert(I->capturesVariableArrayType());
  385. II = &Ctx.Idents.get("vla");
  386. }
  387. if (ArgType->isVariablyModifiedType())
  388. ArgType = getCanonicalParamType(Ctx, ArgType);
  389. VarDecl *Arg;
  390. if (DebugFunctionDecl && (CapVar || I->capturesThis())) {
  391. Arg = ParmVarDecl::Create(
  392. Ctx, DebugFunctionDecl,
  393. CapVar ? CapVar->getBeginLoc() : FD->getBeginLoc(),
  394. CapVar ? CapVar->getLocation() : FD->getLocation(), II, ArgType,
  395. /*TInfo=*/nullptr, SC_None, /*DefArg=*/nullptr);
  396. } else {
  397. Arg = ImplicitParamDecl::Create(Ctx, /*DC=*/nullptr, FD->getLocation(),
  398. II, ArgType, ImplicitParamDecl::Other);
  399. }
  400. Args.emplace_back(Arg);
  401. // Do not cast arguments if we emit function with non-original types.
  402. TargetArgs.emplace_back(
  403. FO.UIntPtrCastRequired
  404. ? Arg
  405. : CGM.getOpenMPRuntime().translateParameter(FD, Arg));
  406. ++I;
  407. }
  408. Args.append(
  409. std::next(CD->param_begin(), CD->getContextParamPosition() + 1),
  410. CD->param_end());
  411. TargetArgs.append(
  412. std::next(CD->param_begin(), CD->getContextParamPosition() + 1),
  413. CD->param_end());
  414. // Create the function declaration.
  415. const CGFunctionInfo &FuncInfo =
  416. CGM.getTypes().arrangeBuiltinFunctionDeclaration(Ctx.VoidTy, TargetArgs);
  417. llvm::FunctionType *FuncLLVMTy = CGM.getTypes().GetFunctionType(FuncInfo);
  418. auto *F =
  419. llvm::Function::Create(FuncLLVMTy, llvm::GlobalValue::InternalLinkage,
  420. FO.FunctionName, &CGM.getModule());
  421. CGM.SetInternalFunctionAttributes(CD, F, FuncInfo);
  422. if (CD->isNothrow())
  423. F->setDoesNotThrow();
  424. F->setDoesNotRecurse();
  425. // Generate the function.
  426. CGF.StartFunction(CD, Ctx.VoidTy, F, FuncInfo, TargetArgs,
  427. FO.S->getBeginLoc(), CD->getBody()->getBeginLoc());
  428. unsigned Cnt = CD->getContextParamPosition();
  429. I = FO.S->captures().begin();
  430. for (const FieldDecl *FD : RD->fields()) {
  431. // Do not map arguments if we emit function with non-original types.
  432. Address LocalAddr(Address::invalid());
  433. if (!FO.UIntPtrCastRequired && Args[Cnt] != TargetArgs[Cnt]) {
  434. LocalAddr = CGM.getOpenMPRuntime().getParameterAddress(CGF, Args[Cnt],
  435. TargetArgs[Cnt]);
  436. } else {
  437. LocalAddr = CGF.GetAddrOfLocalVar(Args[Cnt]);
  438. }
  439. // If we are capturing a pointer by copy we don't need to do anything, just
  440. // use the value that we get from the arguments.
  441. if (I->capturesVariableByCopy() && FD->getType()->isAnyPointerType()) {
  442. const VarDecl *CurVD = I->getCapturedVar();
  443. // If the variable is a reference we need to materialize it here.
  444. if (CurVD->getType()->isReferenceType()) {
  445. Address RefAddr = CGF.CreateMemTemp(
  446. CurVD->getType(), CGM.getPointerAlign(), ".materialized_ref");
  447. CGF.EmitStoreOfScalar(LocalAddr.getPointer(), RefAddr,
  448. /*Volatile=*/false, CurVD->getType());
  449. LocalAddr = RefAddr;
  450. }
  451. if (!FO.RegisterCastedArgsOnly)
  452. LocalAddrs.insert({Args[Cnt], {CurVD, LocalAddr}});
  453. ++Cnt;
  454. ++I;
  455. continue;
  456. }
  457. LValue ArgLVal = CGF.MakeAddrLValue(LocalAddr, Args[Cnt]->getType(),
  458. AlignmentSource::Decl);
  459. if (FD->hasCapturedVLAType()) {
  460. if (FO.UIntPtrCastRequired) {
  461. ArgLVal = CGF.MakeAddrLValue(
  462. castValueFromUintptr(CGF, I->getLocation(), FD->getType(),
  463. Args[Cnt]->getName(), ArgLVal),
  464. FD->getType(), AlignmentSource::Decl);
  465. }
  466. llvm::Value *ExprArg = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation());
  467. const VariableArrayType *VAT = FD->getCapturedVLAType();
  468. VLASizes.try_emplace(Args[Cnt], VAT->getSizeExpr(), ExprArg);
  469. } else if (I->capturesVariable()) {
  470. const VarDecl *Var = I->getCapturedVar();
  471. QualType VarTy = Var->getType();
  472. Address ArgAddr = ArgLVal.getAddress();
  473. if (!VarTy->isReferenceType()) {
  474. if (ArgLVal.getType()->isLValueReferenceType()) {
  475. ArgAddr = CGF.EmitLoadOfReference(ArgLVal);
  476. } else if (!VarTy->isVariablyModifiedType() ||
  477. !VarTy->isPointerType()) {
  478. assert(ArgLVal.getType()->isPointerType());
  479. ArgAddr = CGF.EmitLoadOfPointer(
  480. ArgAddr, ArgLVal.getType()->castAs<PointerType>());
  481. }
  482. }
  483. if (!FO.RegisterCastedArgsOnly) {
  484. LocalAddrs.insert(
  485. {Args[Cnt],
  486. {Var, Address(ArgAddr.getPointer(), Ctx.getDeclAlign(Var))}});
  487. }
  488. } else if (I->capturesVariableByCopy()) {
  489. assert(!FD->getType()->isAnyPointerType() &&
  490. "Not expecting a captured pointer.");
  491. const VarDecl *Var = I->getCapturedVar();
  492. QualType VarTy = Var->getType();
  493. LocalAddrs.insert(
  494. {Args[Cnt],
  495. {Var, FO.UIntPtrCastRequired
  496. ? castValueFromUintptr(CGF, I->getLocation(),
  497. FD->getType(), Args[Cnt]->getName(),
  498. ArgLVal, VarTy->isReferenceType())
  499. : ArgLVal.getAddress()}});
  500. } else {
  501. // If 'this' is captured, load it into CXXThisValue.
  502. assert(I->capturesThis());
  503. CXXThisValue = CGF.EmitLoadOfScalar(ArgLVal, I->getLocation());
  504. LocalAddrs.insert({Args[Cnt], {nullptr, ArgLVal.getAddress()}});
  505. }
  506. ++Cnt;
  507. ++I;
  508. }
  509. return F;
  510. }
  511. llvm::Function *
  512. CodeGenFunction::GenerateOpenMPCapturedStmtFunction(const CapturedStmt &S) {
  513. assert(
  514. CapturedStmtInfo &&
  515. "CapturedStmtInfo should be set when generating the captured function");
  516. const CapturedDecl *CD = S.getCapturedDecl();
  517. // Build the argument list.
  518. bool NeedWrapperFunction =
  519. getDebugInfo() &&
  520. CGM.getCodeGenOpts().getDebugInfo() >= codegenoptions::LimitedDebugInfo;
  521. FunctionArgList Args;
  522. llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>> LocalAddrs;
  523. llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>> VLASizes;
  524. SmallString<256> Buffer;
  525. llvm::raw_svector_ostream Out(Buffer);
  526. Out << CapturedStmtInfo->getHelperName();
  527. if (NeedWrapperFunction)
  528. Out << "_debug__";
  529. FunctionOptions FO(&S, !NeedWrapperFunction, /*RegisterCastedArgsOnly=*/false,
  530. Out.str());
  531. llvm::Function *F = emitOutlinedFunctionPrologue(*this, Args, LocalAddrs,
  532. VLASizes, CXXThisValue, FO);
  533. for (const auto &LocalAddrPair : LocalAddrs) {
  534. if (LocalAddrPair.second.first) {
  535. setAddrOfLocalVar(LocalAddrPair.second.first,
  536. LocalAddrPair.second.second);
  537. }
  538. }
  539. for (const auto &VLASizePair : VLASizes)
  540. VLASizeMap[VLASizePair.second.first] = VLASizePair.second.second;
  541. PGO.assignRegionCounters(GlobalDecl(CD), F);
  542. CapturedStmtInfo->EmitBody(*this, CD->getBody());
  543. FinishFunction(CD->getBodyRBrace());
  544. if (!NeedWrapperFunction)
  545. return F;
  546. FunctionOptions WrapperFO(&S, /*UIntPtrCastRequired=*/true,
  547. /*RegisterCastedArgsOnly=*/true,
  548. CapturedStmtInfo->getHelperName());
  549. CodeGenFunction WrapperCGF(CGM, /*suppressNewContext=*/true);
  550. WrapperCGF.CapturedStmtInfo = CapturedStmtInfo;
  551. Args.clear();
  552. LocalAddrs.clear();
  553. VLASizes.clear();
  554. llvm::Function *WrapperF =
  555. emitOutlinedFunctionPrologue(WrapperCGF, Args, LocalAddrs, VLASizes,
  556. WrapperCGF.CXXThisValue, WrapperFO);
  557. llvm::SmallVector<llvm::Value *, 4> CallArgs;
  558. for (const auto *Arg : Args) {
  559. llvm::Value *CallArg;
  560. auto I = LocalAddrs.find(Arg);
  561. if (I != LocalAddrs.end()) {
  562. LValue LV = WrapperCGF.MakeAddrLValue(
  563. I->second.second,
  564. I->second.first ? I->second.first->getType() : Arg->getType(),
  565. AlignmentSource::Decl);
  566. CallArg = WrapperCGF.EmitLoadOfScalar(LV, S.getBeginLoc());
  567. } else {
  568. auto EI = VLASizes.find(Arg);
  569. if (EI != VLASizes.end()) {
  570. CallArg = EI->second.second;
  571. } else {
  572. LValue LV = WrapperCGF.MakeAddrLValue(WrapperCGF.GetAddrOfLocalVar(Arg),
  573. Arg->getType(),
  574. AlignmentSource::Decl);
  575. CallArg = WrapperCGF.EmitLoadOfScalar(LV, S.getBeginLoc());
  576. }
  577. }
  578. CallArgs.emplace_back(WrapperCGF.EmitFromMemory(CallArg, Arg->getType()));
  579. }
  580. CGM.getOpenMPRuntime().emitOutlinedFunctionCall(WrapperCGF, S.getBeginLoc(),
  581. F, CallArgs);
  582. WrapperCGF.FinishFunction();
  583. return WrapperF;
  584. }
  585. //===----------------------------------------------------------------------===//
  586. // OpenMP Directive Emission
  587. //===----------------------------------------------------------------------===//
  588. void CodeGenFunction::EmitOMPAggregateAssign(
  589. Address DestAddr, Address SrcAddr, QualType OriginalType,
  590. const llvm::function_ref<void(Address, Address)> CopyGen) {
  591. // Perform element-by-element initialization.
  592. QualType ElementTy;
  593. // Drill down to the base element type on both arrays.
  594. const ArrayType *ArrayTy = OriginalType->getAsArrayTypeUnsafe();
  595. llvm::Value *NumElements = emitArrayLength(ArrayTy, ElementTy, DestAddr);
  596. SrcAddr = Builder.CreateElementBitCast(SrcAddr, DestAddr.getElementType());
  597. llvm::Value *SrcBegin = SrcAddr.getPointer();
  598. llvm::Value *DestBegin = DestAddr.getPointer();
  599. // Cast from pointer to array type to pointer to single element.
  600. llvm::Value *DestEnd = Builder.CreateGEP(DestBegin, NumElements);
  601. // The basic structure here is a while-do loop.
  602. llvm::BasicBlock *BodyBB = createBasicBlock("omp.arraycpy.body");
  603. llvm::BasicBlock *DoneBB = createBasicBlock("omp.arraycpy.done");
  604. llvm::Value *IsEmpty =
  605. Builder.CreateICmpEQ(DestBegin, DestEnd, "omp.arraycpy.isempty");
  606. Builder.CreateCondBr(IsEmpty, DoneBB, BodyBB);
  607. // Enter the loop body, making that address the current address.
  608. llvm::BasicBlock *EntryBB = Builder.GetInsertBlock();
  609. EmitBlock(BodyBB);
  610. CharUnits ElementSize = getContext().getTypeSizeInChars(ElementTy);
  611. llvm::PHINode *SrcElementPHI =
  612. Builder.CreatePHI(SrcBegin->getType(), 2, "omp.arraycpy.srcElementPast");
  613. SrcElementPHI->addIncoming(SrcBegin, EntryBB);
  614. Address SrcElementCurrent =
  615. Address(SrcElementPHI,
  616. SrcAddr.getAlignment().alignmentOfArrayElement(ElementSize));
  617. llvm::PHINode *DestElementPHI =
  618. Builder.CreatePHI(DestBegin->getType(), 2, "omp.arraycpy.destElementPast");
  619. DestElementPHI->addIncoming(DestBegin, EntryBB);
  620. Address DestElementCurrent =
  621. Address(DestElementPHI,
  622. DestAddr.getAlignment().alignmentOfArrayElement(ElementSize));
  623. // Emit copy.
  624. CopyGen(DestElementCurrent, SrcElementCurrent);
  625. // Shift the address forward by one element.
  626. llvm::Value *DestElementNext = Builder.CreateConstGEP1_32(
  627. DestElementPHI, /*Idx0=*/1, "omp.arraycpy.dest.element");
  628. llvm::Value *SrcElementNext = Builder.CreateConstGEP1_32(
  629. SrcElementPHI, /*Idx0=*/1, "omp.arraycpy.src.element");
  630. // Check whether we've reached the end.
  631. llvm::Value *Done =
  632. Builder.CreateICmpEQ(DestElementNext, DestEnd, "omp.arraycpy.done");
  633. Builder.CreateCondBr(Done, DoneBB, BodyBB);
  634. DestElementPHI->addIncoming(DestElementNext, Builder.GetInsertBlock());
  635. SrcElementPHI->addIncoming(SrcElementNext, Builder.GetInsertBlock());
  636. // Done.
  637. EmitBlock(DoneBB, /*IsFinished=*/true);
  638. }
  639. void CodeGenFunction::EmitOMPCopy(QualType OriginalType, Address DestAddr,
  640. Address SrcAddr, const VarDecl *DestVD,
  641. const VarDecl *SrcVD, const Expr *Copy) {
  642. if (OriginalType->isArrayType()) {
  643. const auto *BO = dyn_cast<BinaryOperator>(Copy);
  644. if (BO && BO->getOpcode() == BO_Assign) {
  645. // Perform simple memcpy for simple copying.
  646. LValue Dest = MakeAddrLValue(DestAddr, OriginalType);
  647. LValue Src = MakeAddrLValue(SrcAddr, OriginalType);
  648. EmitAggregateAssign(Dest, Src, OriginalType);
  649. } else {
  650. // For arrays with complex element types perform element by element
  651. // copying.
  652. EmitOMPAggregateAssign(
  653. DestAddr, SrcAddr, OriginalType,
  654. [this, Copy, SrcVD, DestVD](Address DestElement, Address SrcElement) {
  655. // Working with the single array element, so have to remap
  656. // destination and source variables to corresponding array
  657. // elements.
  658. CodeGenFunction::OMPPrivateScope Remap(*this);
  659. Remap.addPrivate(DestVD, [DestElement]() { return DestElement; });
  660. Remap.addPrivate(SrcVD, [SrcElement]() { return SrcElement; });
  661. (void)Remap.Privatize();
  662. EmitIgnoredExpr(Copy);
  663. });
  664. }
  665. } else {
  666. // Remap pseudo source variable to private copy.
  667. CodeGenFunction::OMPPrivateScope Remap(*this);
  668. Remap.addPrivate(SrcVD, [SrcAddr]() { return SrcAddr; });
  669. Remap.addPrivate(DestVD, [DestAddr]() { return DestAddr; });
  670. (void)Remap.Privatize();
  671. // Emit copying of the whole variable.
  672. EmitIgnoredExpr(Copy);
  673. }
  674. }
  675. bool CodeGenFunction::EmitOMPFirstprivateClause(const OMPExecutableDirective &D,
  676. OMPPrivateScope &PrivateScope) {
  677. if (!HaveInsertPoint())
  678. return false;
  679. bool FirstprivateIsLastprivate = false;
  680. llvm::DenseSet<const VarDecl *> Lastprivates;
  681. for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
  682. for (const auto *D : C->varlists())
  683. Lastprivates.insert(
  684. cast<VarDecl>(cast<DeclRefExpr>(D)->getDecl())->getCanonicalDecl());
  685. }
  686. llvm::DenseSet<const VarDecl *> EmittedAsFirstprivate;
  687. llvm::SmallVector<OpenMPDirectiveKind, 4> CaptureRegions;
  688. getOpenMPCaptureRegions(CaptureRegions, D.getDirectiveKind());
  689. // Force emission of the firstprivate copy if the directive does not emit
  690. // outlined function, like omp for, omp simd, omp distribute etc.
  691. bool MustEmitFirstprivateCopy =
  692. CaptureRegions.size() == 1 && CaptureRegions.back() == OMPD_unknown;
  693. for (const auto *C : D.getClausesOfKind<OMPFirstprivateClause>()) {
  694. auto IRef = C->varlist_begin();
  695. auto InitsRef = C->inits().begin();
  696. for (const Expr *IInit : C->private_copies()) {
  697. const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
  698. bool ThisFirstprivateIsLastprivate =
  699. Lastprivates.count(OrigVD->getCanonicalDecl()) > 0;
  700. const FieldDecl *FD = CapturedStmtInfo->lookup(OrigVD);
  701. if (!MustEmitFirstprivateCopy && !ThisFirstprivateIsLastprivate && FD &&
  702. !FD->getType()->isReferenceType()) {
  703. EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl());
  704. ++IRef;
  705. ++InitsRef;
  706. continue;
  707. }
  708. FirstprivateIsLastprivate =
  709. FirstprivateIsLastprivate || ThisFirstprivateIsLastprivate;
  710. if (EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl()).second) {
  711. const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
  712. const auto *VDInit =
  713. cast<VarDecl>(cast<DeclRefExpr>(*InitsRef)->getDecl());
  714. bool IsRegistered;
  715. DeclRefExpr DRE(const_cast<VarDecl *>(OrigVD),
  716. /*RefersToEnclosingVariableOrCapture=*/FD != nullptr,
  717. (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
  718. LValue OriginalLVal = EmitLValue(&DRE);
  719. QualType Type = VD->getType();
  720. if (Type->isArrayType()) {
  721. // Emit VarDecl with copy init for arrays.
  722. // Get the address of the original variable captured in current
  723. // captured region.
  724. IsRegistered = PrivateScope.addPrivate(
  725. OrigVD, [this, VD, Type, OriginalLVal, VDInit]() {
  726. AutoVarEmission Emission = EmitAutoVarAlloca(*VD);
  727. const Expr *Init = VD->getInit();
  728. if (!isa<CXXConstructExpr>(Init) ||
  729. isTrivialInitializer(Init)) {
  730. // Perform simple memcpy.
  731. LValue Dest =
  732. MakeAddrLValue(Emission.getAllocatedAddress(), Type);
  733. EmitAggregateAssign(Dest, OriginalLVal, Type);
  734. } else {
  735. EmitOMPAggregateAssign(
  736. Emission.getAllocatedAddress(), OriginalLVal.getAddress(),
  737. Type,
  738. [this, VDInit, Init](Address DestElement,
  739. Address SrcElement) {
  740. // Clean up any temporaries needed by the
  741. // initialization.
  742. RunCleanupsScope InitScope(*this);
  743. // Emit initialization for single element.
  744. setAddrOfLocalVar(VDInit, SrcElement);
  745. EmitAnyExprToMem(Init, DestElement,
  746. Init->getType().getQualifiers(),
  747. /*IsInitializer*/ false);
  748. LocalDeclMap.erase(VDInit);
  749. });
  750. }
  751. EmitAutoVarCleanups(Emission);
  752. return Emission.getAllocatedAddress();
  753. });
  754. } else {
  755. Address OriginalAddr = OriginalLVal.getAddress();
  756. IsRegistered = PrivateScope.addPrivate(
  757. OrigVD, [this, VDInit, OriginalAddr, VD]() {
  758. // Emit private VarDecl with copy init.
  759. // Remap temp VDInit variable to the address of the original
  760. // variable (for proper handling of captured global variables).
  761. setAddrOfLocalVar(VDInit, OriginalAddr);
  762. EmitDecl(*VD);
  763. LocalDeclMap.erase(VDInit);
  764. return GetAddrOfLocalVar(VD);
  765. });
  766. }
  767. assert(IsRegistered &&
  768. "firstprivate var already registered as private");
  769. // Silence the warning about unused variable.
  770. (void)IsRegistered;
  771. }
  772. ++IRef;
  773. ++InitsRef;
  774. }
  775. }
  776. return FirstprivateIsLastprivate && !EmittedAsFirstprivate.empty();
  777. }
  778. void CodeGenFunction::EmitOMPPrivateClause(
  779. const OMPExecutableDirective &D,
  780. CodeGenFunction::OMPPrivateScope &PrivateScope) {
  781. if (!HaveInsertPoint())
  782. return;
  783. llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
  784. for (const auto *C : D.getClausesOfKind<OMPPrivateClause>()) {
  785. auto IRef = C->varlist_begin();
  786. for (const Expr *IInit : C->private_copies()) {
  787. const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
  788. if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
  789. const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
  790. bool IsRegistered = PrivateScope.addPrivate(OrigVD, [this, VD]() {
  791. // Emit private VarDecl with copy init.
  792. EmitDecl(*VD);
  793. return GetAddrOfLocalVar(VD);
  794. });
  795. assert(IsRegistered && "private var already registered as private");
  796. // Silence the warning about unused variable.
  797. (void)IsRegistered;
  798. }
  799. ++IRef;
  800. }
  801. }
  802. }
  803. bool CodeGenFunction::EmitOMPCopyinClause(const OMPExecutableDirective &D) {
  804. if (!HaveInsertPoint())
  805. return false;
  806. // threadprivate_var1 = master_threadprivate_var1;
  807. // operator=(threadprivate_var2, master_threadprivate_var2);
  808. // ...
  809. // __kmpc_barrier(&loc, global_tid);
  810. llvm::DenseSet<const VarDecl *> CopiedVars;
  811. llvm::BasicBlock *CopyBegin = nullptr, *CopyEnd = nullptr;
  812. for (const auto *C : D.getClausesOfKind<OMPCopyinClause>()) {
  813. auto IRef = C->varlist_begin();
  814. auto ISrcRef = C->source_exprs().begin();
  815. auto IDestRef = C->destination_exprs().begin();
  816. for (const Expr *AssignOp : C->assignment_ops()) {
  817. const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
  818. QualType Type = VD->getType();
  819. if (CopiedVars.insert(VD->getCanonicalDecl()).second) {
  820. // Get the address of the master variable. If we are emitting code with
  821. // TLS support, the address is passed from the master as field in the
  822. // captured declaration.
  823. Address MasterAddr = Address::invalid();
  824. if (getLangOpts().OpenMPUseTLS &&
  825. getContext().getTargetInfo().isTLSSupported()) {
  826. assert(CapturedStmtInfo->lookup(VD) &&
  827. "Copyin threadprivates should have been captured!");
  828. DeclRefExpr DRE(const_cast<VarDecl *>(VD), true, (*IRef)->getType(),
  829. VK_LValue, (*IRef)->getExprLoc());
  830. MasterAddr = EmitLValue(&DRE).getAddress();
  831. LocalDeclMap.erase(VD);
  832. } else {
  833. MasterAddr =
  834. Address(VD->isStaticLocal() ? CGM.getStaticLocalDeclAddress(VD)
  835. : CGM.GetAddrOfGlobal(VD),
  836. getContext().getDeclAlign(VD));
  837. }
  838. // Get the address of the threadprivate variable.
  839. Address PrivateAddr = EmitLValue(*IRef).getAddress();
  840. if (CopiedVars.size() == 1) {
  841. // At first check if current thread is a master thread. If it is, no
  842. // need to copy data.
  843. CopyBegin = createBasicBlock("copyin.not.master");
  844. CopyEnd = createBasicBlock("copyin.not.master.end");
  845. Builder.CreateCondBr(
  846. Builder.CreateICmpNE(
  847. Builder.CreatePtrToInt(MasterAddr.getPointer(), CGM.IntPtrTy),
  848. Builder.CreatePtrToInt(PrivateAddr.getPointer(),
  849. CGM.IntPtrTy)),
  850. CopyBegin, CopyEnd);
  851. EmitBlock(CopyBegin);
  852. }
  853. const auto *SrcVD =
  854. cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
  855. const auto *DestVD =
  856. cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
  857. EmitOMPCopy(Type, PrivateAddr, MasterAddr, DestVD, SrcVD, AssignOp);
  858. }
  859. ++IRef;
  860. ++ISrcRef;
  861. ++IDestRef;
  862. }
  863. }
  864. if (CopyEnd) {
  865. // Exit out of copying procedure for non-master thread.
  866. EmitBlock(CopyEnd, /*IsFinished=*/true);
  867. return true;
  868. }
  869. return false;
  870. }
  871. bool CodeGenFunction::EmitOMPLastprivateClauseInit(
  872. const OMPExecutableDirective &D, OMPPrivateScope &PrivateScope) {
  873. if (!HaveInsertPoint())
  874. return false;
  875. bool HasAtLeastOneLastprivate = false;
  876. llvm::DenseSet<const VarDecl *> SIMDLCVs;
  877. if (isOpenMPSimdDirective(D.getDirectiveKind())) {
  878. const auto *LoopDirective = cast<OMPLoopDirective>(&D);
  879. for (const Expr *C : LoopDirective->counters()) {
  880. SIMDLCVs.insert(
  881. cast<VarDecl>(cast<DeclRefExpr>(C)->getDecl())->getCanonicalDecl());
  882. }
  883. }
  884. llvm::DenseSet<const VarDecl *> AlreadyEmittedVars;
  885. for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
  886. HasAtLeastOneLastprivate = true;
  887. if (isOpenMPTaskLoopDirective(D.getDirectiveKind()) &&
  888. !getLangOpts().OpenMPSimd)
  889. break;
  890. auto IRef = C->varlist_begin();
  891. auto IDestRef = C->destination_exprs().begin();
  892. for (const Expr *IInit : C->private_copies()) {
  893. // Keep the address of the original variable for future update at the end
  894. // of the loop.
  895. const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
  896. // Taskloops do not require additional initialization, it is done in
  897. // runtime support library.
  898. if (AlreadyEmittedVars.insert(OrigVD->getCanonicalDecl()).second) {
  899. const auto *DestVD =
  900. cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
  901. PrivateScope.addPrivate(DestVD, [this, OrigVD, IRef]() {
  902. DeclRefExpr DRE(
  903. const_cast<VarDecl *>(OrigVD),
  904. /*RefersToEnclosingVariableOrCapture=*/CapturedStmtInfo->lookup(
  905. OrigVD) != nullptr,
  906. (*IRef)->getType(), VK_LValue, (*IRef)->getExprLoc());
  907. return EmitLValue(&DRE).getAddress();
  908. });
  909. // Check if the variable is also a firstprivate: in this case IInit is
  910. // not generated. Initialization of this variable will happen in codegen
  911. // for 'firstprivate' clause.
  912. if (IInit && !SIMDLCVs.count(OrigVD->getCanonicalDecl())) {
  913. const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());
  914. bool IsRegistered = PrivateScope.addPrivate(OrigVD, [this, VD]() {
  915. // Emit private VarDecl with copy init.
  916. EmitDecl(*VD);
  917. return GetAddrOfLocalVar(VD);
  918. });
  919. assert(IsRegistered &&
  920. "lastprivate var already registered as private");
  921. (void)IsRegistered;
  922. }
  923. }
  924. ++IRef;
  925. ++IDestRef;
  926. }
  927. }
  928. return HasAtLeastOneLastprivate;
  929. }
  930. void CodeGenFunction::EmitOMPLastprivateClauseFinal(
  931. const OMPExecutableDirective &D, bool NoFinals,
  932. llvm::Value *IsLastIterCond) {
  933. if (!HaveInsertPoint())
  934. return;
  935. // Emit following code:
  936. // if (<IsLastIterCond>) {
  937. // orig_var1 = private_orig_var1;
  938. // ...
  939. // orig_varn = private_orig_varn;
  940. // }
  941. llvm::BasicBlock *ThenBB = nullptr;
  942. llvm::BasicBlock *DoneBB = nullptr;
  943. if (IsLastIterCond) {
  944. ThenBB = createBasicBlock(".omp.lastprivate.then");
  945. DoneBB = createBasicBlock(".omp.lastprivate.done");
  946. Builder.CreateCondBr(IsLastIterCond, ThenBB, DoneBB);
  947. EmitBlock(ThenBB);
  948. }
  949. llvm::DenseSet<const VarDecl *> AlreadyEmittedVars;
  950. llvm::DenseMap<const VarDecl *, const Expr *> LoopCountersAndUpdates;
  951. if (const auto *LoopDirective = dyn_cast<OMPLoopDirective>(&D)) {
  952. auto IC = LoopDirective->counters().begin();
  953. for (const Expr *F : LoopDirective->finals()) {
  954. const auto *D =
  955. cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl())->getCanonicalDecl();
  956. if (NoFinals)
  957. AlreadyEmittedVars.insert(D);
  958. else
  959. LoopCountersAndUpdates[D] = F;
  960. ++IC;
  961. }
  962. }
  963. for (const auto *C : D.getClausesOfKind<OMPLastprivateClause>()) {
  964. auto IRef = C->varlist_begin();
  965. auto ISrcRef = C->source_exprs().begin();
  966. auto IDestRef = C->destination_exprs().begin();
  967. for (const Expr *AssignOp : C->assignment_ops()) {
  968. const auto *PrivateVD =
  969. cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
  970. QualType Type = PrivateVD->getType();
  971. const auto *CanonicalVD = PrivateVD->getCanonicalDecl();
  972. if (AlreadyEmittedVars.insert(CanonicalVD).second) {
  973. // If lastprivate variable is a loop control variable for loop-based
  974. // directive, update its value before copyin back to original
  975. // variable.
  976. if (const Expr *FinalExpr = LoopCountersAndUpdates.lookup(CanonicalVD))
  977. EmitIgnoredExpr(FinalExpr);
  978. const auto *SrcVD =
  979. cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());
  980. const auto *DestVD =
  981. cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());
  982. // Get the address of the original variable.
  983. Address OriginalAddr = GetAddrOfLocalVar(DestVD);
  984. // Get the address of the private variable.
  985. Address PrivateAddr = GetAddrOfLocalVar(PrivateVD);
  986. if (const auto *RefTy = PrivateVD->getType()->getAs<ReferenceType>())
  987. PrivateAddr =
  988. Address(Builder.CreateLoad(PrivateAddr),
  989. getNaturalTypeAlignment(RefTy->getPointeeType()));
  990. EmitOMPCopy(Type, OriginalAddr, PrivateAddr, DestVD, SrcVD, AssignOp);
  991. }
  992. ++IRef;
  993. ++ISrcRef;
  994. ++IDestRef;
  995. }
  996. if (const Expr *PostUpdate = C->getPostUpdateExpr())
  997. EmitIgnoredExpr(PostUpdate);
  998. }
  999. if (IsLastIterCond)
  1000. EmitBlock(DoneBB, /*IsFinished=*/true);
  1001. }
  1002. void CodeGenFunction::EmitOMPReductionClauseInit(
  1003. const OMPExecutableDirective &D,
  1004. CodeGenFunction::OMPPrivateScope &PrivateScope) {
  1005. if (!HaveInsertPoint())
  1006. return;
  1007. SmallVector<const Expr *, 4> Shareds;
  1008. SmallVector<const Expr *, 4> Privates;
  1009. SmallVector<const Expr *, 4> ReductionOps;
  1010. SmallVector<const Expr *, 4> LHSs;
  1011. SmallVector<const Expr *, 4> RHSs;
  1012. for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
  1013. auto IPriv = C->privates().begin();
  1014. auto IRed = C->reduction_ops().begin();
  1015. auto ILHS = C->lhs_exprs().begin();
  1016. auto IRHS = C->rhs_exprs().begin();
  1017. for (const Expr *Ref : C->varlists()) {
  1018. Shareds.emplace_back(Ref);
  1019. Privates.emplace_back(*IPriv);
  1020. ReductionOps.emplace_back(*IRed);
  1021. LHSs.emplace_back(*ILHS);
  1022. RHSs.emplace_back(*IRHS);
  1023. std::advance(IPriv, 1);
  1024. std::advance(IRed, 1);
  1025. std::advance(ILHS, 1);
  1026. std::advance(IRHS, 1);
  1027. }
  1028. }
  1029. ReductionCodeGen RedCG(Shareds, Privates, ReductionOps);
  1030. unsigned Count = 0;
  1031. auto ILHS = LHSs.begin();
  1032. auto IRHS = RHSs.begin();
  1033. auto IPriv = Privates.begin();
  1034. for (const Expr *IRef : Shareds) {
  1035. const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*IPriv)->getDecl());
  1036. // Emit private VarDecl with reduction init.
  1037. RedCG.emitSharedLValue(*this, Count);
  1038. RedCG.emitAggregateType(*this, Count);
  1039. AutoVarEmission Emission = EmitAutoVarAlloca(*PrivateVD);
  1040. RedCG.emitInitialization(*this, Count, Emission.getAllocatedAddress(),
  1041. RedCG.getSharedLValue(Count),
  1042. [&Emission](CodeGenFunction &CGF) {
  1043. CGF.EmitAutoVarInit(Emission);
  1044. return true;
  1045. });
  1046. EmitAutoVarCleanups(Emission);
  1047. Address BaseAddr = RedCG.adjustPrivateAddress(
  1048. *this, Count, Emission.getAllocatedAddress());
  1049. bool IsRegistered = PrivateScope.addPrivate(
  1050. RedCG.getBaseDecl(Count), [BaseAddr]() { return BaseAddr; });
  1051. assert(IsRegistered && "private var already registered as private");
  1052. // Silence the warning about unused variable.
  1053. (void)IsRegistered;
  1054. const auto *LHSVD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());
  1055. const auto *RHSVD = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());
  1056. QualType Type = PrivateVD->getType();
  1057. bool isaOMPArraySectionExpr = isa<OMPArraySectionExpr>(IRef);
  1058. if (isaOMPArraySectionExpr && Type->isVariablyModifiedType()) {
  1059. // Store the address of the original variable associated with the LHS
  1060. // implicit variable.
  1061. PrivateScope.addPrivate(LHSVD, [&RedCG, Count]() {
  1062. return RedCG.getSharedLValue(Count).getAddress();
  1063. });
  1064. PrivateScope.addPrivate(
  1065. RHSVD, [this, PrivateVD]() { return GetAddrOfLocalVar(PrivateVD); });
  1066. } else if ((isaOMPArraySectionExpr && Type->isScalarType()) ||
  1067. isa<ArraySubscriptExpr>(IRef)) {
  1068. // Store the address of the original variable associated with the LHS
  1069. // implicit variable.
  1070. PrivateScope.addPrivate(LHSVD, [&RedCG, Count]() {
  1071. return RedCG.getSharedLValue(Count).getAddress();
  1072. });
  1073. PrivateScope.addPrivate(RHSVD, [this, PrivateVD, RHSVD]() {
  1074. return Builder.CreateElementBitCast(GetAddrOfLocalVar(PrivateVD),
  1075. ConvertTypeForMem(RHSVD->getType()),
  1076. "rhs.begin");
  1077. });
  1078. } else {
  1079. QualType Type = PrivateVD->getType();
  1080. bool IsArray = getContext().getAsArrayType(Type) != nullptr;
  1081. Address OriginalAddr = RedCG.getSharedLValue(Count).getAddress();
  1082. // Store the address of the original variable associated with the LHS
  1083. // implicit variable.
  1084. if (IsArray) {
  1085. OriginalAddr = Builder.CreateElementBitCast(
  1086. OriginalAddr, ConvertTypeForMem(LHSVD->getType()), "lhs.begin");
  1087. }
  1088. PrivateScope.addPrivate(LHSVD, [OriginalAddr]() { return OriginalAddr; });
  1089. PrivateScope.addPrivate(
  1090. RHSVD, [this, PrivateVD, RHSVD, IsArray]() {
  1091. return IsArray
  1092. ? Builder.CreateElementBitCast(
  1093. GetAddrOfLocalVar(PrivateVD),
  1094. ConvertTypeForMem(RHSVD->getType()), "rhs.begin")
  1095. : GetAddrOfLocalVar(PrivateVD);
  1096. });
  1097. }
  1098. ++ILHS;
  1099. ++IRHS;
  1100. ++IPriv;
  1101. ++Count;
  1102. }
  1103. }
  1104. void CodeGenFunction::EmitOMPReductionClauseFinal(
  1105. const OMPExecutableDirective &D, const OpenMPDirectiveKind ReductionKind) {
  1106. if (!HaveInsertPoint())
  1107. return;
  1108. llvm::SmallVector<const Expr *, 8> Privates;
  1109. llvm::SmallVector<const Expr *, 8> LHSExprs;
  1110. llvm::SmallVector<const Expr *, 8> RHSExprs;
  1111. llvm::SmallVector<const Expr *, 8> ReductionOps;
  1112. bool HasAtLeastOneReduction = false;
  1113. for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
  1114. HasAtLeastOneReduction = true;
  1115. Privates.append(C->privates().begin(), C->privates().end());
  1116. LHSExprs.append(C->lhs_exprs().begin(), C->lhs_exprs().end());
  1117. RHSExprs.append(C->rhs_exprs().begin(), C->rhs_exprs().end());
  1118. ReductionOps.append(C->reduction_ops().begin(), C->reduction_ops().end());
  1119. }
  1120. if (HasAtLeastOneReduction) {
  1121. bool WithNowait = D.getSingleClause<OMPNowaitClause>() ||
  1122. isOpenMPParallelDirective(D.getDirectiveKind()) ||
  1123. ReductionKind == OMPD_simd;
  1124. bool SimpleReduction = ReductionKind == OMPD_simd;
  1125. // Emit nowait reduction if nowait clause is present or directive is a
  1126. // parallel directive (it always has implicit barrier).
  1127. CGM.getOpenMPRuntime().emitReduction(
  1128. *this, D.getEndLoc(), Privates, LHSExprs, RHSExprs, ReductionOps,
  1129. {WithNowait, SimpleReduction, ReductionKind});
  1130. }
  1131. }
  1132. static void emitPostUpdateForReductionClause(
  1133. CodeGenFunction &CGF, const OMPExecutableDirective &D,
  1134. const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
  1135. if (!CGF.HaveInsertPoint())
  1136. return;
  1137. llvm::BasicBlock *DoneBB = nullptr;
  1138. for (const auto *C : D.getClausesOfKind<OMPReductionClause>()) {
  1139. if (const Expr *PostUpdate = C->getPostUpdateExpr()) {
  1140. if (!DoneBB) {
  1141. if (llvm::Value *Cond = CondGen(CGF)) {
  1142. // If the first post-update expression is found, emit conditional
  1143. // block if it was requested.
  1144. llvm::BasicBlock *ThenBB = CGF.createBasicBlock(".omp.reduction.pu");
  1145. DoneBB = CGF.createBasicBlock(".omp.reduction.pu.done");
  1146. CGF.Builder.CreateCondBr(Cond, ThenBB, DoneBB);
  1147. CGF.EmitBlock(ThenBB);
  1148. }
  1149. }
  1150. CGF.EmitIgnoredExpr(PostUpdate);
  1151. }
  1152. }
  1153. if (DoneBB)
  1154. CGF.EmitBlock(DoneBB, /*IsFinished=*/true);
  1155. }
  1156. namespace {
  1157. /// Codegen lambda for appending distribute lower and upper bounds to outlined
  1158. /// parallel function. This is necessary for combined constructs such as
  1159. /// 'distribute parallel for'
  1160. typedef llvm::function_ref<void(CodeGenFunction &,
  1161. const OMPExecutableDirective &,
  1162. llvm::SmallVectorImpl<llvm::Value *> &)>
  1163. CodeGenBoundParametersTy;
  1164. } // anonymous namespace
  1165. static void emitCommonOMPParallelDirective(
  1166. CodeGenFunction &CGF, const OMPExecutableDirective &S,
  1167. OpenMPDirectiveKind InnermostKind, const RegionCodeGenTy &CodeGen,
  1168. const CodeGenBoundParametersTy &CodeGenBoundParameters) {
  1169. const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);
  1170. llvm::Value *OutlinedFn =
  1171. CGF.CGM.getOpenMPRuntime().emitParallelOutlinedFunction(
  1172. S, *CS->getCapturedDecl()->param_begin(), InnermostKind, CodeGen);
  1173. if (const auto *NumThreadsClause = S.getSingleClause<OMPNumThreadsClause>()) {
  1174. CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF);
  1175. llvm::Value *NumThreads =
  1176. CGF.EmitScalarExpr(NumThreadsClause->getNumThreads(),
  1177. /*IgnoreResultAssign=*/true);
  1178. CGF.CGM.getOpenMPRuntime().emitNumThreadsClause(
  1179. CGF, NumThreads, NumThreadsClause->getBeginLoc());
  1180. }
  1181. if (const auto *ProcBindClause = S.getSingleClause<OMPProcBindClause>()) {
  1182. CodeGenFunction::RunCleanupsScope ProcBindScope(CGF);
  1183. CGF.CGM.getOpenMPRuntime().emitProcBindClause(
  1184. CGF, ProcBindClause->getProcBindKind(), ProcBindClause->getBeginLoc());
  1185. }
  1186. const Expr *IfCond = nullptr;
  1187. for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
  1188. if (C->getNameModifier() == OMPD_unknown ||
  1189. C->getNameModifier() == OMPD_parallel) {
  1190. IfCond = C->getCondition();
  1191. break;
  1192. }
  1193. }
  1194. OMPParallelScope Scope(CGF, S);
  1195. llvm::SmallVector<llvm::Value *, 16> CapturedVars;
  1196. // Combining 'distribute' with 'for' requires sharing each 'distribute' chunk
  1197. // lower and upper bounds with the pragma 'for' chunking mechanism.
  1198. // The following lambda takes care of appending the lower and upper bound
  1199. // parameters when necessary
  1200. CodeGenBoundParameters(CGF, S, CapturedVars);
  1201. CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
  1202. CGF.CGM.getOpenMPRuntime().emitParallelCall(CGF, S.getBeginLoc(), OutlinedFn,
  1203. CapturedVars, IfCond);
  1204. }
  1205. static void emitEmptyBoundParameters(CodeGenFunction &,
  1206. const OMPExecutableDirective &,
  1207. llvm::SmallVectorImpl<llvm::Value *> &) {}
  1208. void CodeGenFunction::EmitOMPParallelDirective(const OMPParallelDirective &S) {
  1209. // Emit parallel region as a standalone region.
  1210. auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
  1211. Action.Enter(CGF);
  1212. OMPPrivateScope PrivateScope(CGF);
  1213. bool Copyins = CGF.EmitOMPCopyinClause(S);
  1214. (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
  1215. if (Copyins) {
  1216. // Emit implicit barrier to synchronize threads and avoid data races on
  1217. // propagation master's thread values of threadprivate variables to local
  1218. // instances of that variables of all other implicit threads.
  1219. CGF.CGM.getOpenMPRuntime().emitBarrierCall(
  1220. CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
  1221. /*ForceSimpleCall=*/true);
  1222. }
  1223. CGF.EmitOMPPrivateClause(S, PrivateScope);
  1224. CGF.EmitOMPReductionClauseInit(S, PrivateScope);
  1225. (void)PrivateScope.Privatize();
  1226. CGF.EmitStmt(S.getCapturedStmt(OMPD_parallel)->getCapturedStmt());
  1227. CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
  1228. };
  1229. emitCommonOMPParallelDirective(*this, S, OMPD_parallel, CodeGen,
  1230. emitEmptyBoundParameters);
  1231. emitPostUpdateForReductionClause(*this, S,
  1232. [](CodeGenFunction &) { return nullptr; });
  1233. }
  1234. void CodeGenFunction::EmitOMPLoopBody(const OMPLoopDirective &D,
  1235. JumpDest LoopExit) {
  1236. RunCleanupsScope BodyScope(*this);
  1237. // Update counters values on current iteration.
  1238. for (const Expr *UE : D.updates())
  1239. EmitIgnoredExpr(UE);
  1240. // Update the linear variables.
  1241. // In distribute directives only loop counters may be marked as linear, no
  1242. // need to generate the code for them.
  1243. if (!isOpenMPDistributeDirective(D.getDirectiveKind())) {
  1244. for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
  1245. for (const Expr *UE : C->updates())
  1246. EmitIgnoredExpr(UE);
  1247. }
  1248. }
  1249. // On a continue in the body, jump to the end.
  1250. JumpDest Continue = getJumpDestInCurrentScope("omp.body.continue");
  1251. BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
  1252. // Emit loop body.
  1253. EmitStmt(D.getBody());
  1254. // The end (updates/cleanups).
  1255. EmitBlock(Continue.getBlock());
  1256. BreakContinueStack.pop_back();
  1257. }
  1258. void CodeGenFunction::EmitOMPInnerLoop(
  1259. const Stmt &S, bool RequiresCleanup, const Expr *LoopCond,
  1260. const Expr *IncExpr,
  1261. const llvm::function_ref<void(CodeGenFunction &)> BodyGen,
  1262. const llvm::function_ref<void(CodeGenFunction &)> PostIncGen) {
  1263. auto LoopExit = getJumpDestInCurrentScope("omp.inner.for.end");
  1264. // Start the loop with a block that tests the condition.
  1265. auto CondBlock = createBasicBlock("omp.inner.for.cond");
  1266. EmitBlock(CondBlock);
  1267. const SourceRange R = S.getSourceRange();
  1268. LoopStack.push(CondBlock, SourceLocToDebugLoc(R.getBegin()),
  1269. SourceLocToDebugLoc(R.getEnd()));
  1270. // If there are any cleanups between here and the loop-exit scope,
  1271. // create a block to stage a loop exit along.
  1272. llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
  1273. if (RequiresCleanup)
  1274. ExitBlock = createBasicBlock("omp.inner.for.cond.cleanup");
  1275. llvm::BasicBlock *LoopBody = createBasicBlock("omp.inner.for.body");
  1276. // Emit condition.
  1277. EmitBranchOnBoolExpr(LoopCond, LoopBody, ExitBlock, getProfileCount(&S));
  1278. if (ExitBlock != LoopExit.getBlock()) {
  1279. EmitBlock(ExitBlock);
  1280. EmitBranchThroughCleanup(LoopExit);
  1281. }
  1282. EmitBlock(LoopBody);
  1283. incrementProfileCounter(&S);
  1284. // Create a block for the increment.
  1285. JumpDest Continue = getJumpDestInCurrentScope("omp.inner.for.inc");
  1286. BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
  1287. BodyGen(*this);
  1288. // Emit "IV = IV + 1" and a back-edge to the condition block.
  1289. EmitBlock(Continue.getBlock());
  1290. EmitIgnoredExpr(IncExpr);
  1291. PostIncGen(*this);
  1292. BreakContinueStack.pop_back();
  1293. EmitBranch(CondBlock);
  1294. LoopStack.pop();
  1295. // Emit the fall-through block.
  1296. EmitBlock(LoopExit.getBlock());
  1297. }
  1298. bool CodeGenFunction::EmitOMPLinearClauseInit(const OMPLoopDirective &D) {
  1299. if (!HaveInsertPoint())
  1300. return false;
  1301. // Emit inits for the linear variables.
  1302. bool HasLinears = false;
  1303. for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
  1304. for (const Expr *Init : C->inits()) {
  1305. HasLinears = true;
  1306. const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(Init)->getDecl());
  1307. if (const auto *Ref =
  1308. dyn_cast<DeclRefExpr>(VD->getInit()->IgnoreImpCasts())) {
  1309. AutoVarEmission Emission = EmitAutoVarAlloca(*VD);
  1310. const auto *OrigVD = cast<VarDecl>(Ref->getDecl());
  1311. DeclRefExpr DRE(const_cast<VarDecl *>(OrigVD),
  1312. CapturedStmtInfo->lookup(OrigVD) != nullptr,
  1313. VD->getInit()->getType(), VK_LValue,
  1314. VD->getInit()->getExprLoc());
  1315. EmitExprAsInit(&DRE, VD, MakeAddrLValue(Emission.getAllocatedAddress(),
  1316. VD->getType()),
  1317. /*capturedByInit=*/false);
  1318. EmitAutoVarCleanups(Emission);
  1319. } else {
  1320. EmitVarDecl(*VD);
  1321. }
  1322. }
  1323. // Emit the linear steps for the linear clauses.
  1324. // If a step is not constant, it is pre-calculated before the loop.
  1325. if (const auto *CS = cast_or_null<BinaryOperator>(C->getCalcStep()))
  1326. if (const auto *SaveRef = cast<DeclRefExpr>(CS->getLHS())) {
  1327. EmitVarDecl(*cast<VarDecl>(SaveRef->getDecl()));
  1328. // Emit calculation of the linear step.
  1329. EmitIgnoredExpr(CS);
  1330. }
  1331. }
  1332. return HasLinears;
  1333. }
  1334. void CodeGenFunction::EmitOMPLinearClauseFinal(
  1335. const OMPLoopDirective &D,
  1336. const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
  1337. if (!HaveInsertPoint())
  1338. return;
  1339. llvm::BasicBlock *DoneBB = nullptr;
  1340. // Emit the final values of the linear variables.
  1341. for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
  1342. auto IC = C->varlist_begin();
  1343. for (const Expr *F : C->finals()) {
  1344. if (!DoneBB) {
  1345. if (llvm::Value *Cond = CondGen(*this)) {
  1346. // If the first post-update expression is found, emit conditional
  1347. // block if it was requested.
  1348. llvm::BasicBlock *ThenBB = createBasicBlock(".omp.linear.pu");
  1349. DoneBB = createBasicBlock(".omp.linear.pu.done");
  1350. Builder.CreateCondBr(Cond, ThenBB, DoneBB);
  1351. EmitBlock(ThenBB);
  1352. }
  1353. }
  1354. const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl());
  1355. DeclRefExpr DRE(const_cast<VarDecl *>(OrigVD),
  1356. CapturedStmtInfo->lookup(OrigVD) != nullptr,
  1357. (*IC)->getType(), VK_LValue, (*IC)->getExprLoc());
  1358. Address OrigAddr = EmitLValue(&DRE).getAddress();
  1359. CodeGenFunction::OMPPrivateScope VarScope(*this);
  1360. VarScope.addPrivate(OrigVD, [OrigAddr]() { return OrigAddr; });
  1361. (void)VarScope.Privatize();
  1362. EmitIgnoredExpr(F);
  1363. ++IC;
  1364. }
  1365. if (const Expr *PostUpdate = C->getPostUpdateExpr())
  1366. EmitIgnoredExpr(PostUpdate);
  1367. }
  1368. if (DoneBB)
  1369. EmitBlock(DoneBB, /*IsFinished=*/true);
  1370. }
  1371. static void emitAlignedClause(CodeGenFunction &CGF,
  1372. const OMPExecutableDirective &D) {
  1373. if (!CGF.HaveInsertPoint())
  1374. return;
  1375. for (const auto *Clause : D.getClausesOfKind<OMPAlignedClause>()) {
  1376. unsigned ClauseAlignment = 0;
  1377. if (const Expr *AlignmentExpr = Clause->getAlignment()) {
  1378. auto *AlignmentCI =
  1379. cast<llvm::ConstantInt>(CGF.EmitScalarExpr(AlignmentExpr));
  1380. ClauseAlignment = static_cast<unsigned>(AlignmentCI->getZExtValue());
  1381. }
  1382. for (const Expr *E : Clause->varlists()) {
  1383. unsigned Alignment = ClauseAlignment;
  1384. if (Alignment == 0) {
  1385. // OpenMP [2.8.1, Description]
  1386. // If no optional parameter is specified, implementation-defined default
  1387. // alignments for SIMD instructions on the target platforms are assumed.
  1388. Alignment =
  1389. CGF.getContext()
  1390. .toCharUnitsFromBits(CGF.getContext().getOpenMPDefaultSimdAlign(
  1391. E->getType()->getPointeeType()))
  1392. .getQuantity();
  1393. }
  1394. assert((Alignment == 0 || llvm::isPowerOf2_32(Alignment)) &&
  1395. "alignment is not power of 2");
  1396. if (Alignment != 0) {
  1397. llvm::Value *PtrValue = CGF.EmitScalarExpr(E);
  1398. CGF.EmitAlignmentAssumption(PtrValue, Alignment);
  1399. }
  1400. }
  1401. }
  1402. }
  1403. void CodeGenFunction::EmitOMPPrivateLoopCounters(
  1404. const OMPLoopDirective &S, CodeGenFunction::OMPPrivateScope &LoopScope) {
  1405. if (!HaveInsertPoint())
  1406. return;
  1407. auto I = S.private_counters().begin();
  1408. for (const Expr *E : S.counters()) {
  1409. const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
  1410. const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl());
  1411. // Emit var without initialization.
  1412. AutoVarEmission VarEmission = EmitAutoVarAlloca(*PrivateVD);
  1413. EmitAutoVarCleanups(VarEmission);
  1414. LocalDeclMap.erase(PrivateVD);
  1415. (void)LoopScope.addPrivate(VD, [&VarEmission]() {
  1416. return VarEmission.getAllocatedAddress();
  1417. });
  1418. if (LocalDeclMap.count(VD) || CapturedStmtInfo->lookup(VD) ||
  1419. VD->hasGlobalStorage()) {
  1420. (void)LoopScope.addPrivate(PrivateVD, [this, VD, E]() {
  1421. DeclRefExpr DRE(const_cast<VarDecl *>(VD),
  1422. LocalDeclMap.count(VD) || CapturedStmtInfo->lookup(VD),
  1423. E->getType(), VK_LValue, E->getExprLoc());
  1424. return EmitLValue(&DRE).getAddress();
  1425. });
  1426. } else {
  1427. (void)LoopScope.addPrivate(PrivateVD, [&VarEmission]() {
  1428. return VarEmission.getAllocatedAddress();
  1429. });
  1430. }
  1431. ++I;
  1432. }
  1433. // Privatize extra loop counters used in loops for ordered(n) clauses.
  1434. for (const auto *C : S.getClausesOfKind<OMPOrderedClause>()) {
  1435. if (!C->getNumForLoops())
  1436. continue;
  1437. for (unsigned I = S.getCollapsedNumber(),
  1438. E = C->getLoopNumIterations().size();
  1439. I < E; ++I) {
  1440. const auto *DRE = cast<DeclRefExpr>(C->getLoopCounter(I));
  1441. const auto *VD = cast<VarDecl>(DRE->getDecl());
  1442. // Override only those variables that are really emitted already.
  1443. if (LocalDeclMap.count(VD)) {
  1444. (void)LoopScope.addPrivate(VD, [this, DRE, VD]() {
  1445. return CreateMemTemp(DRE->getType(), VD->getName());
  1446. });
  1447. }
  1448. }
  1449. }
  1450. }
  1451. static void emitPreCond(CodeGenFunction &CGF, const OMPLoopDirective &S,
  1452. const Expr *Cond, llvm::BasicBlock *TrueBlock,
  1453. llvm::BasicBlock *FalseBlock, uint64_t TrueCount) {
  1454. if (!CGF.HaveInsertPoint())
  1455. return;
  1456. {
  1457. CodeGenFunction::OMPPrivateScope PreCondScope(CGF);
  1458. CGF.EmitOMPPrivateLoopCounters(S, PreCondScope);
  1459. (void)PreCondScope.Privatize();
  1460. // Get initial values of real counters.
  1461. for (const Expr *I : S.inits()) {
  1462. CGF.EmitIgnoredExpr(I);
  1463. }
  1464. }
  1465. // Check that loop is executed at least one time.
  1466. CGF.EmitBranchOnBoolExpr(Cond, TrueBlock, FalseBlock, TrueCount);
  1467. }
  1468. void CodeGenFunction::EmitOMPLinearClause(
  1469. const OMPLoopDirective &D, CodeGenFunction::OMPPrivateScope &PrivateScope) {
  1470. if (!HaveInsertPoint())
  1471. return;
  1472. llvm::DenseSet<const VarDecl *> SIMDLCVs;
  1473. if (isOpenMPSimdDirective(D.getDirectiveKind())) {
  1474. const auto *LoopDirective = cast<OMPLoopDirective>(&D);
  1475. for (const Expr *C : LoopDirective->counters()) {
  1476. SIMDLCVs.insert(
  1477. cast<VarDecl>(cast<DeclRefExpr>(C)->getDecl())->getCanonicalDecl());
  1478. }
  1479. }
  1480. for (const auto *C : D.getClausesOfKind<OMPLinearClause>()) {
  1481. auto CurPrivate = C->privates().begin();
  1482. for (const Expr *E : C->varlists()) {
  1483. const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
  1484. const auto *PrivateVD =
  1485. cast<VarDecl>(cast<DeclRefExpr>(*CurPrivate)->getDecl());
  1486. if (!SIMDLCVs.count(VD->getCanonicalDecl())) {
  1487. bool IsRegistered = PrivateScope.addPrivate(VD, [this, PrivateVD]() {
  1488. // Emit private VarDecl with copy init.
  1489. EmitVarDecl(*PrivateVD);
  1490. return GetAddrOfLocalVar(PrivateVD);
  1491. });
  1492. assert(IsRegistered && "linear var already registered as private");
  1493. // Silence the warning about unused variable.
  1494. (void)IsRegistered;
  1495. } else {
  1496. EmitVarDecl(*PrivateVD);
  1497. }
  1498. ++CurPrivate;
  1499. }
  1500. }
  1501. }
  1502. static void emitSimdlenSafelenClause(CodeGenFunction &CGF,
  1503. const OMPExecutableDirective &D,
  1504. bool IsMonotonic) {
  1505. if (!CGF.HaveInsertPoint())
  1506. return;
  1507. if (const auto *C = D.getSingleClause<OMPSimdlenClause>()) {
  1508. RValue Len = CGF.EmitAnyExpr(C->getSimdlen(), AggValueSlot::ignored(),
  1509. /*ignoreResult=*/true);
  1510. auto *Val = cast<llvm::ConstantInt>(Len.getScalarVal());
  1511. CGF.LoopStack.setVectorizeWidth(Val->getZExtValue());
  1512. // In presence of finite 'safelen', it may be unsafe to mark all
  1513. // the memory instructions parallel, because loop-carried
  1514. // dependences of 'safelen' iterations are possible.
  1515. if (!IsMonotonic)
  1516. CGF.LoopStack.setParallel(!D.getSingleClause<OMPSafelenClause>());
  1517. } else if (const auto *C = D.getSingleClause<OMPSafelenClause>()) {
  1518. RValue Len = CGF.EmitAnyExpr(C->getSafelen(), AggValueSlot::ignored(),
  1519. /*ignoreResult=*/true);
  1520. auto *Val = cast<llvm::ConstantInt>(Len.getScalarVal());
  1521. CGF.LoopStack.setVectorizeWidth(Val->getZExtValue());
  1522. // In presence of finite 'safelen', it may be unsafe to mark all
  1523. // the memory instructions parallel, because loop-carried
  1524. // dependences of 'safelen' iterations are possible.
  1525. CGF.LoopStack.setParallel(/*Enable=*/false);
  1526. }
  1527. }
  1528. void CodeGenFunction::EmitOMPSimdInit(const OMPLoopDirective &D,
  1529. bool IsMonotonic) {
  1530. // Walk clauses and process safelen/lastprivate.
  1531. LoopStack.setParallel(!IsMonotonic);
  1532. LoopStack.setVectorizeEnable();
  1533. emitSimdlenSafelenClause(*this, D, IsMonotonic);
  1534. }
  1535. void CodeGenFunction::EmitOMPSimdFinal(
  1536. const OMPLoopDirective &D,
  1537. const llvm::function_ref<llvm::Value *(CodeGenFunction &)> CondGen) {
  1538. if (!HaveInsertPoint())
  1539. return;
  1540. llvm::BasicBlock *DoneBB = nullptr;
  1541. auto IC = D.counters().begin();
  1542. auto IPC = D.private_counters().begin();
  1543. for (const Expr *F : D.finals()) {
  1544. const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>((*IC))->getDecl());
  1545. const auto *PrivateVD = cast<VarDecl>(cast<DeclRefExpr>((*IPC))->getDecl());
  1546. const auto *CED = dyn_cast<OMPCapturedExprDecl>(OrigVD);
  1547. if (LocalDeclMap.count(OrigVD) || CapturedStmtInfo->lookup(OrigVD) ||
  1548. OrigVD->hasGlobalStorage() || CED) {
  1549. if (!DoneBB) {
  1550. if (llvm::Value *Cond = CondGen(*this)) {
  1551. // If the first post-update expression is found, emit conditional
  1552. // block if it was requested.
  1553. llvm::BasicBlock *ThenBB = createBasicBlock(".omp.final.then");
  1554. DoneBB = createBasicBlock(".omp.final.done");
  1555. Builder.CreateCondBr(Cond, ThenBB, DoneBB);
  1556. EmitBlock(ThenBB);
  1557. }
  1558. }
  1559. Address OrigAddr = Address::invalid();
  1560. if (CED) {
  1561. OrigAddr = EmitLValue(CED->getInit()->IgnoreImpCasts()).getAddress();
  1562. } else {
  1563. DeclRefExpr DRE(const_cast<VarDecl *>(PrivateVD),
  1564. /*RefersToEnclosingVariableOrCapture=*/false,
  1565. (*IPC)->getType(), VK_LValue, (*IPC)->getExprLoc());
  1566. OrigAddr = EmitLValue(&DRE).getAddress();
  1567. }
  1568. OMPPrivateScope VarScope(*this);
  1569. VarScope.addPrivate(OrigVD, [OrigAddr]() { return OrigAddr; });
  1570. (void)VarScope.Privatize();
  1571. EmitIgnoredExpr(F);
  1572. }
  1573. ++IC;
  1574. ++IPC;
  1575. }
  1576. if (DoneBB)
  1577. EmitBlock(DoneBB, /*IsFinished=*/true);
  1578. }
  1579. static void emitOMPLoopBodyWithStopPoint(CodeGenFunction &CGF,
  1580. const OMPLoopDirective &S,
  1581. CodeGenFunction::JumpDest LoopExit) {
  1582. CGF.EmitOMPLoopBody(S, LoopExit);
  1583. CGF.EmitStopPoint(&S);
  1584. }
  1585. /// Emit a helper variable and return corresponding lvalue.
  1586. static LValue EmitOMPHelperVar(CodeGenFunction &CGF,
  1587. const DeclRefExpr *Helper) {
  1588. auto VDecl = cast<VarDecl>(Helper->getDecl());
  1589. CGF.EmitVarDecl(*VDecl);
  1590. return CGF.EmitLValue(Helper);
  1591. }
  1592. static void emitOMPSimdRegion(CodeGenFunction &CGF, const OMPLoopDirective &S,
  1593. PrePostActionTy &Action) {
  1594. Action.Enter(CGF);
  1595. assert(isOpenMPSimdDirective(S.getDirectiveKind()) &&
  1596. "Expected simd directive");
  1597. OMPLoopScope PreInitScope(CGF, S);
  1598. // if (PreCond) {
  1599. // for (IV in 0..LastIteration) BODY;
  1600. // <Final counter/linear vars updates>;
  1601. // }
  1602. //
  1603. if (isOpenMPDistributeDirective(S.getDirectiveKind()) ||
  1604. isOpenMPWorksharingDirective(S.getDirectiveKind()) ||
  1605. isOpenMPTaskLoopDirective(S.getDirectiveKind())) {
  1606. (void)EmitOMPHelperVar(CGF, cast<DeclRefExpr>(S.getLowerBoundVariable()));
  1607. (void)EmitOMPHelperVar(CGF, cast<DeclRefExpr>(S.getUpperBoundVariable()));
  1608. }
  1609. // Emit: if (PreCond) - begin.
  1610. // If the condition constant folds and can be elided, avoid emitting the
  1611. // whole loop.
  1612. bool CondConstant;
  1613. llvm::BasicBlock *ContBlock = nullptr;
  1614. if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
  1615. if (!CondConstant)
  1616. return;
  1617. } else {
  1618. llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("simd.if.then");
  1619. ContBlock = CGF.createBasicBlock("simd.if.end");
  1620. emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock,
  1621. CGF.getProfileCount(&S));
  1622. CGF.EmitBlock(ThenBlock);
  1623. CGF.incrementProfileCounter(&S);
  1624. }
  1625. // Emit the loop iteration variable.
  1626. const Expr *IVExpr = S.getIterationVariable();
  1627. const auto *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl());
  1628. CGF.EmitVarDecl(*IVDecl);
  1629. CGF.EmitIgnoredExpr(S.getInit());
  1630. // Emit the iterations count variable.
  1631. // If it is not a variable, Sema decided to calculate iterations count on
  1632. // each iteration (e.g., it is foldable into a constant).
  1633. if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
  1634. CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
  1635. // Emit calculation of the iterations count.
  1636. CGF.EmitIgnoredExpr(S.getCalcLastIteration());
  1637. }
  1638. CGF.EmitOMPSimdInit(S);
  1639. emitAlignedClause(CGF, S);
  1640. (void)CGF.EmitOMPLinearClauseInit(S);
  1641. {
  1642. CodeGenFunction::OMPPrivateScope LoopScope(CGF);
  1643. CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
  1644. CGF.EmitOMPLinearClause(S, LoopScope);
  1645. CGF.EmitOMPPrivateClause(S, LoopScope);
  1646. CGF.EmitOMPReductionClauseInit(S, LoopScope);
  1647. bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
  1648. (void)LoopScope.Privatize();
  1649. CGF.EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), S.getCond(),
  1650. S.getInc(),
  1651. [&S](CodeGenFunction &CGF) {
  1652. CGF.EmitOMPLoopBody(S, CodeGenFunction::JumpDest());
  1653. CGF.EmitStopPoint(&S);
  1654. },
  1655. [](CodeGenFunction &) {});
  1656. CGF.EmitOMPSimdFinal(S, [](CodeGenFunction &) { return nullptr; });
  1657. // Emit final copy of the lastprivate variables at the end of loops.
  1658. if (HasLastprivateClause)
  1659. CGF.EmitOMPLastprivateClauseFinal(S, /*NoFinals=*/true);
  1660. CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_simd);
  1661. emitPostUpdateForReductionClause(CGF, S,
  1662. [](CodeGenFunction &) { return nullptr; });
  1663. }
  1664. CGF.EmitOMPLinearClauseFinal(S, [](CodeGenFunction &) { return nullptr; });
  1665. // Emit: if (PreCond) - end.
  1666. if (ContBlock) {
  1667. CGF.EmitBranch(ContBlock);
  1668. CGF.EmitBlock(ContBlock, true);
  1669. }
  1670. }
  1671. void CodeGenFunction::EmitOMPSimdDirective(const OMPSimdDirective &S) {
  1672. auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
  1673. emitOMPSimdRegion(CGF, S, Action);
  1674. };
  1675. OMPLexicalScope Scope(*this, S, OMPD_unknown);
  1676. CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
  1677. }
  1678. void CodeGenFunction::EmitOMPOuterLoop(
  1679. bool DynamicOrOrdered, bool IsMonotonic, const OMPLoopDirective &S,
  1680. CodeGenFunction::OMPPrivateScope &LoopScope,
  1681. const CodeGenFunction::OMPLoopArguments &LoopArgs,
  1682. const CodeGenFunction::CodeGenLoopTy &CodeGenLoop,
  1683. const CodeGenFunction::CodeGenOrderedTy &CodeGenOrdered) {
  1684. CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
  1685. const Expr *IVExpr = S.getIterationVariable();
  1686. const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
  1687. const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
  1688. JumpDest LoopExit = getJumpDestInCurrentScope("omp.dispatch.end");
  1689. // Start the loop with a block that tests the condition.
  1690. llvm::BasicBlock *CondBlock = createBasicBlock("omp.dispatch.cond");
  1691. EmitBlock(CondBlock);
  1692. const SourceRange R = S.getSourceRange();
  1693. LoopStack.push(CondBlock, SourceLocToDebugLoc(R.getBegin()),
  1694. SourceLocToDebugLoc(R.getEnd()));
  1695. llvm::Value *BoolCondVal = nullptr;
  1696. if (!DynamicOrOrdered) {
  1697. // UB = min(UB, GlobalUB) or
  1698. // UB = min(UB, PrevUB) for combined loop sharing constructs (e.g.
  1699. // 'distribute parallel for')
  1700. EmitIgnoredExpr(LoopArgs.EUB);
  1701. // IV = LB
  1702. EmitIgnoredExpr(LoopArgs.Init);
  1703. // IV < UB
  1704. BoolCondVal = EvaluateExprAsBool(LoopArgs.Cond);
  1705. } else {
  1706. BoolCondVal =
  1707. RT.emitForNext(*this, S.getBeginLoc(), IVSize, IVSigned, LoopArgs.IL,
  1708. LoopArgs.LB, LoopArgs.UB, LoopArgs.ST);
  1709. }
  1710. // If there are any cleanups between here and the loop-exit scope,
  1711. // create a block to stage a loop exit along.
  1712. llvm::BasicBlock *ExitBlock = LoopExit.getBlock();
  1713. if (LoopScope.requiresCleanups())
  1714. ExitBlock = createBasicBlock("omp.dispatch.cleanup");
  1715. llvm::BasicBlock *LoopBody = createBasicBlock("omp.dispatch.body");
  1716. Builder.CreateCondBr(BoolCondVal, LoopBody, ExitBlock);
  1717. if (ExitBlock != LoopExit.getBlock()) {
  1718. EmitBlock(ExitBlock);
  1719. EmitBranchThroughCleanup(LoopExit);
  1720. }
  1721. EmitBlock(LoopBody);
  1722. // Emit "IV = LB" (in case of static schedule, we have already calculated new
  1723. // LB for loop condition and emitted it above).
  1724. if (DynamicOrOrdered)
  1725. EmitIgnoredExpr(LoopArgs.Init);
  1726. // Create a block for the increment.
  1727. JumpDest Continue = getJumpDestInCurrentScope("omp.dispatch.inc");
  1728. BreakContinueStack.push_back(BreakContinue(LoopExit, Continue));
  1729. // Generate !llvm.loop.parallel metadata for loads and stores for loops
  1730. // with dynamic/guided scheduling and without ordered clause.
  1731. if (!isOpenMPSimdDirective(S.getDirectiveKind()))
  1732. LoopStack.setParallel(!IsMonotonic);
  1733. else
  1734. EmitOMPSimdInit(S, IsMonotonic);
  1735. SourceLocation Loc = S.getBeginLoc();
  1736. // when 'distribute' is not combined with a 'for':
  1737. // while (idx <= UB) { BODY; ++idx; }
  1738. // when 'distribute' is combined with a 'for'
  1739. // (e.g. 'distribute parallel for')
  1740. // while (idx <= UB) { <CodeGen rest of pragma>; idx += ST; }
  1741. EmitOMPInnerLoop(
  1742. S, LoopScope.requiresCleanups(), LoopArgs.Cond, LoopArgs.IncExpr,
  1743. [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) {
  1744. CodeGenLoop(CGF, S, LoopExit);
  1745. },
  1746. [IVSize, IVSigned, Loc, &CodeGenOrdered](CodeGenFunction &CGF) {
  1747. CodeGenOrdered(CGF, Loc, IVSize, IVSigned);
  1748. });
  1749. EmitBlock(Continue.getBlock());
  1750. BreakContinueStack.pop_back();
  1751. if (!DynamicOrOrdered) {
  1752. // Emit "LB = LB + Stride", "UB = UB + Stride".
  1753. EmitIgnoredExpr(LoopArgs.NextLB);
  1754. EmitIgnoredExpr(LoopArgs.NextUB);
  1755. }
  1756. EmitBranch(CondBlock);
  1757. LoopStack.pop();
  1758. // Emit the fall-through block.
  1759. EmitBlock(LoopExit.getBlock());
  1760. // Tell the runtime we are done.
  1761. auto &&CodeGen = [DynamicOrOrdered, &S](CodeGenFunction &CGF) {
  1762. if (!DynamicOrOrdered)
  1763. CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
  1764. S.getDirectiveKind());
  1765. };
  1766. OMPCancelStack.emitExit(*this, S.getDirectiveKind(), CodeGen);
  1767. }
  1768. void CodeGenFunction::EmitOMPForOuterLoop(
  1769. const OpenMPScheduleTy &ScheduleKind, bool IsMonotonic,
  1770. const OMPLoopDirective &S, OMPPrivateScope &LoopScope, bool Ordered,
  1771. const OMPLoopArguments &LoopArgs,
  1772. const CodeGenDispatchBoundsTy &CGDispatchBounds) {
  1773. CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
  1774. // Dynamic scheduling of the outer loop (dynamic, guided, auto, runtime).
  1775. const bool DynamicOrOrdered =
  1776. Ordered || RT.isDynamic(ScheduleKind.Schedule);
  1777. assert((Ordered ||
  1778. !RT.isStaticNonchunked(ScheduleKind.Schedule,
  1779. LoopArgs.Chunk != nullptr)) &&
  1780. "static non-chunked schedule does not need outer loop");
  1781. // Emit outer loop.
  1782. //
  1783. // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
  1784. // When schedule(dynamic,chunk_size) is specified, the iterations are
  1785. // distributed to threads in the team in chunks as the threads request them.
  1786. // Each thread executes a chunk of iterations, then requests another chunk,
  1787. // until no chunks remain to be distributed. Each chunk contains chunk_size
  1788. // iterations, except for the last chunk to be distributed, which may have
  1789. // fewer iterations. When no chunk_size is specified, it defaults to 1.
  1790. //
  1791. // When schedule(guided,chunk_size) is specified, the iterations are assigned
  1792. // to threads in the team in chunks as the executing threads request them.
  1793. // Each thread executes a chunk of iterations, then requests another chunk,
  1794. // until no chunks remain to be assigned. For a chunk_size of 1, the size of
  1795. // each chunk is proportional to the number of unassigned iterations divided
  1796. // by the number of threads in the team, decreasing to 1. For a chunk_size
  1797. // with value k (greater than 1), the size of each chunk is determined in the
  1798. // same way, with the restriction that the chunks do not contain fewer than k
  1799. // iterations (except for the last chunk to be assigned, which may have fewer
  1800. // than k iterations).
  1801. //
  1802. // When schedule(auto) is specified, the decision regarding scheduling is
  1803. // delegated to the compiler and/or runtime system. The programmer gives the
  1804. // implementation the freedom to choose any possible mapping of iterations to
  1805. // threads in the team.
  1806. //
  1807. // When schedule(runtime) is specified, the decision regarding scheduling is
  1808. // deferred until run time, and the schedule and chunk size are taken from the
  1809. // run-sched-var ICV. If the ICV is set to auto, the schedule is
  1810. // implementation defined
  1811. //
  1812. // while(__kmpc_dispatch_next(&LB, &UB)) {
  1813. // idx = LB;
  1814. // while (idx <= UB) { BODY; ++idx;
  1815. // __kmpc_dispatch_fini_(4|8)[u](); // For ordered loops only.
  1816. // } // inner loop
  1817. // }
  1818. //
  1819. // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
  1820. // When schedule(static, chunk_size) is specified, iterations are divided into
  1821. // chunks of size chunk_size, and the chunks are assigned to the threads in
  1822. // the team in a round-robin fashion in the order of the thread number.
  1823. //
  1824. // while(UB = min(UB, GlobalUB), idx = LB, idx < UB) {
  1825. // while (idx <= UB) { BODY; ++idx; } // inner loop
  1826. // LB = LB + ST;
  1827. // UB = UB + ST;
  1828. // }
  1829. //
  1830. const Expr *IVExpr = S.getIterationVariable();
  1831. const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
  1832. const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
  1833. if (DynamicOrOrdered) {
  1834. const std::pair<llvm::Value *, llvm::Value *> DispatchBounds =
  1835. CGDispatchBounds(*this, S, LoopArgs.LB, LoopArgs.UB);
  1836. llvm::Value *LBVal = DispatchBounds.first;
  1837. llvm::Value *UBVal = DispatchBounds.second;
  1838. CGOpenMPRuntime::DispatchRTInput DipatchRTInputValues = {LBVal, UBVal,
  1839. LoopArgs.Chunk};
  1840. RT.emitForDispatchInit(*this, S.getBeginLoc(), ScheduleKind, IVSize,
  1841. IVSigned, Ordered, DipatchRTInputValues);
  1842. } else {
  1843. CGOpenMPRuntime::StaticRTInput StaticInit(
  1844. IVSize, IVSigned, Ordered, LoopArgs.IL, LoopArgs.LB, LoopArgs.UB,
  1845. LoopArgs.ST, LoopArgs.Chunk);
  1846. RT.emitForStaticInit(*this, S.getBeginLoc(), S.getDirectiveKind(),
  1847. ScheduleKind, StaticInit);
  1848. }
  1849. auto &&CodeGenOrdered = [Ordered](CodeGenFunction &CGF, SourceLocation Loc,
  1850. const unsigned IVSize,
  1851. const bool IVSigned) {
  1852. if (Ordered) {
  1853. CGF.CGM.getOpenMPRuntime().emitForOrderedIterationEnd(CGF, Loc, IVSize,
  1854. IVSigned);
  1855. }
  1856. };
  1857. OMPLoopArguments OuterLoopArgs(LoopArgs.LB, LoopArgs.UB, LoopArgs.ST,
  1858. LoopArgs.IL, LoopArgs.Chunk, LoopArgs.EUB);
  1859. OuterLoopArgs.IncExpr = S.getInc();
  1860. OuterLoopArgs.Init = S.getInit();
  1861. OuterLoopArgs.Cond = S.getCond();
  1862. OuterLoopArgs.NextLB = S.getNextLowerBound();
  1863. OuterLoopArgs.NextUB = S.getNextUpperBound();
  1864. EmitOMPOuterLoop(DynamicOrOrdered, IsMonotonic, S, LoopScope, OuterLoopArgs,
  1865. emitOMPLoopBodyWithStopPoint, CodeGenOrdered);
  1866. }
  1867. static void emitEmptyOrdered(CodeGenFunction &, SourceLocation Loc,
  1868. const unsigned IVSize, const bool IVSigned) {}
  1869. void CodeGenFunction::EmitOMPDistributeOuterLoop(
  1870. OpenMPDistScheduleClauseKind ScheduleKind, const OMPLoopDirective &S,
  1871. OMPPrivateScope &LoopScope, const OMPLoopArguments &LoopArgs,
  1872. const CodeGenLoopTy &CodeGenLoopContent) {
  1873. CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
  1874. // Emit outer loop.
  1875. // Same behavior as a OMPForOuterLoop, except that schedule cannot be
  1876. // dynamic
  1877. //
  1878. const Expr *IVExpr = S.getIterationVariable();
  1879. const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
  1880. const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
  1881. CGOpenMPRuntime::StaticRTInput StaticInit(
  1882. IVSize, IVSigned, /* Ordered = */ false, LoopArgs.IL, LoopArgs.LB,
  1883. LoopArgs.UB, LoopArgs.ST, LoopArgs.Chunk);
  1884. RT.emitDistributeStaticInit(*this, S.getBeginLoc(), ScheduleKind, StaticInit);
  1885. // for combined 'distribute' and 'for' the increment expression of distribute
  1886. // is store in DistInc. For 'distribute' alone, it is in Inc.
  1887. Expr *IncExpr;
  1888. if (isOpenMPLoopBoundSharingDirective(S.getDirectiveKind()))
  1889. IncExpr = S.getDistInc();
  1890. else
  1891. IncExpr = S.getInc();
  1892. // this routine is shared by 'omp distribute parallel for' and
  1893. // 'omp distribute': select the right EUB expression depending on the
  1894. // directive
  1895. OMPLoopArguments OuterLoopArgs;
  1896. OuterLoopArgs.LB = LoopArgs.LB;
  1897. OuterLoopArgs.UB = LoopArgs.UB;
  1898. OuterLoopArgs.ST = LoopArgs.ST;
  1899. OuterLoopArgs.IL = LoopArgs.IL;
  1900. OuterLoopArgs.Chunk = LoopArgs.Chunk;
  1901. OuterLoopArgs.EUB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
  1902. ? S.getCombinedEnsureUpperBound()
  1903. : S.getEnsureUpperBound();
  1904. OuterLoopArgs.IncExpr = IncExpr;
  1905. OuterLoopArgs.Init = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
  1906. ? S.getCombinedInit()
  1907. : S.getInit();
  1908. OuterLoopArgs.Cond = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
  1909. ? S.getCombinedCond()
  1910. : S.getCond();
  1911. OuterLoopArgs.NextLB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
  1912. ? S.getCombinedNextLowerBound()
  1913. : S.getNextLowerBound();
  1914. OuterLoopArgs.NextUB = isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
  1915. ? S.getCombinedNextUpperBound()
  1916. : S.getNextUpperBound();
  1917. EmitOMPOuterLoop(/* DynamicOrOrdered = */ false, /* IsMonotonic = */ false, S,
  1918. LoopScope, OuterLoopArgs, CodeGenLoopContent,
  1919. emitEmptyOrdered);
  1920. }
  1921. static std::pair<LValue, LValue>
  1922. emitDistributeParallelForInnerBounds(CodeGenFunction &CGF,
  1923. const OMPExecutableDirective &S) {
  1924. const OMPLoopDirective &LS = cast<OMPLoopDirective>(S);
  1925. LValue LB =
  1926. EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getLowerBoundVariable()));
  1927. LValue UB =
  1928. EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getUpperBoundVariable()));
  1929. // When composing 'distribute' with 'for' (e.g. as in 'distribute
  1930. // parallel for') we need to use the 'distribute'
  1931. // chunk lower and upper bounds rather than the whole loop iteration
  1932. // space. These are parameters to the outlined function for 'parallel'
  1933. // and we copy the bounds of the previous schedule into the
  1934. // the current ones.
  1935. LValue PrevLB = CGF.EmitLValue(LS.getPrevLowerBoundVariable());
  1936. LValue PrevUB = CGF.EmitLValue(LS.getPrevUpperBoundVariable());
  1937. llvm::Value *PrevLBVal = CGF.EmitLoadOfScalar(
  1938. PrevLB, LS.getPrevLowerBoundVariable()->getExprLoc());
  1939. PrevLBVal = CGF.EmitScalarConversion(
  1940. PrevLBVal, LS.getPrevLowerBoundVariable()->getType(),
  1941. LS.getIterationVariable()->getType(),
  1942. LS.getPrevLowerBoundVariable()->getExprLoc());
  1943. llvm::Value *PrevUBVal = CGF.EmitLoadOfScalar(
  1944. PrevUB, LS.getPrevUpperBoundVariable()->getExprLoc());
  1945. PrevUBVal = CGF.EmitScalarConversion(
  1946. PrevUBVal, LS.getPrevUpperBoundVariable()->getType(),
  1947. LS.getIterationVariable()->getType(),
  1948. LS.getPrevUpperBoundVariable()->getExprLoc());
  1949. CGF.EmitStoreOfScalar(PrevLBVal, LB);
  1950. CGF.EmitStoreOfScalar(PrevUBVal, UB);
  1951. return {LB, UB};
  1952. }
  1953. /// if the 'for' loop has a dispatch schedule (e.g. dynamic, guided) then
  1954. /// we need to use the LB and UB expressions generated by the worksharing
  1955. /// code generation support, whereas in non combined situations we would
  1956. /// just emit 0 and the LastIteration expression
  1957. /// This function is necessary due to the difference of the LB and UB
  1958. /// types for the RT emission routines for 'for_static_init' and
  1959. /// 'for_dispatch_init'
  1960. static std::pair<llvm::Value *, llvm::Value *>
  1961. emitDistributeParallelForDispatchBounds(CodeGenFunction &CGF,
  1962. const OMPExecutableDirective &S,
  1963. Address LB, Address UB) {
  1964. const OMPLoopDirective &LS = cast<OMPLoopDirective>(S);
  1965. const Expr *IVExpr = LS.getIterationVariable();
  1966. // when implementing a dynamic schedule for a 'for' combined with a
  1967. // 'distribute' (e.g. 'distribute parallel for'), the 'for' loop
  1968. // is not normalized as each team only executes its own assigned
  1969. // distribute chunk
  1970. QualType IteratorTy = IVExpr->getType();
  1971. llvm::Value *LBVal =
  1972. CGF.EmitLoadOfScalar(LB, /*Volatile=*/false, IteratorTy, S.getBeginLoc());
  1973. llvm::Value *UBVal =
  1974. CGF.EmitLoadOfScalar(UB, /*Volatile=*/false, IteratorTy, S.getBeginLoc());
  1975. return {LBVal, UBVal};
  1976. }
  1977. static void emitDistributeParallelForDistributeInnerBoundParams(
  1978. CodeGenFunction &CGF, const OMPExecutableDirective &S,
  1979. llvm::SmallVectorImpl<llvm::Value *> &CapturedVars) {
  1980. const auto &Dir = cast<OMPLoopDirective>(S);
  1981. LValue LB =
  1982. CGF.EmitLValue(cast<DeclRefExpr>(Dir.getCombinedLowerBoundVariable()));
  1983. llvm::Value *LBCast = CGF.Builder.CreateIntCast(
  1984. CGF.Builder.CreateLoad(LB.getAddress()), CGF.SizeTy, /*isSigned=*/false);
  1985. CapturedVars.push_back(LBCast);
  1986. LValue UB =
  1987. CGF.EmitLValue(cast<DeclRefExpr>(Dir.getCombinedUpperBoundVariable()));
  1988. llvm::Value *UBCast = CGF.Builder.CreateIntCast(
  1989. CGF.Builder.CreateLoad(UB.getAddress()), CGF.SizeTy, /*isSigned=*/false);
  1990. CapturedVars.push_back(UBCast);
  1991. }
  1992. static void
  1993. emitInnerParallelForWhenCombined(CodeGenFunction &CGF,
  1994. const OMPLoopDirective &S,
  1995. CodeGenFunction::JumpDest LoopExit) {
  1996. auto &&CGInlinedWorksharingLoop = [&S](CodeGenFunction &CGF,
  1997. PrePostActionTy &Action) {
  1998. Action.Enter(CGF);
  1999. bool HasCancel = false;
  2000. if (!isOpenMPSimdDirective(S.getDirectiveKind())) {
  2001. if (const auto *D = dyn_cast<OMPTeamsDistributeParallelForDirective>(&S))
  2002. HasCancel = D->hasCancel();
  2003. else if (const auto *D = dyn_cast<OMPDistributeParallelForDirective>(&S))
  2004. HasCancel = D->hasCancel();
  2005. else if (const auto *D =
  2006. dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(&S))
  2007. HasCancel = D->hasCancel();
  2008. }
  2009. CodeGenFunction::OMPCancelStackRAII CancelRegion(CGF, S.getDirectiveKind(),
  2010. HasCancel);
  2011. CGF.EmitOMPWorksharingLoop(S, S.getPrevEnsureUpperBound(),
  2012. emitDistributeParallelForInnerBounds,
  2013. emitDistributeParallelForDispatchBounds);
  2014. };
  2015. emitCommonOMPParallelDirective(
  2016. CGF, S,
  2017. isOpenMPSimdDirective(S.getDirectiveKind()) ? OMPD_for_simd : OMPD_for,
  2018. CGInlinedWorksharingLoop,
  2019. emitDistributeParallelForDistributeInnerBoundParams);
  2020. }
  2021. void CodeGenFunction::EmitOMPDistributeParallelForDirective(
  2022. const OMPDistributeParallelForDirective &S) {
  2023. auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
  2024. CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined,
  2025. S.getDistInc());
  2026. };
  2027. OMPLexicalScope Scope(*this, S, OMPD_parallel);
  2028. CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen);
  2029. }
  2030. void CodeGenFunction::EmitOMPDistributeParallelForSimdDirective(
  2031. const OMPDistributeParallelForSimdDirective &S) {
  2032. auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
  2033. CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined,
  2034. S.getDistInc());
  2035. };
  2036. OMPLexicalScope Scope(*this, S, OMPD_parallel);
  2037. CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen);
  2038. }
  2039. void CodeGenFunction::EmitOMPDistributeSimdDirective(
  2040. const OMPDistributeSimdDirective &S) {
  2041. auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
  2042. CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc());
  2043. };
  2044. OMPLexicalScope Scope(*this, S, OMPD_unknown);
  2045. CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
  2046. }
  2047. void CodeGenFunction::EmitOMPTargetSimdDeviceFunction(
  2048. CodeGenModule &CGM, StringRef ParentName, const OMPTargetSimdDirective &S) {
  2049. // Emit SPMD target parallel for region as a standalone region.
  2050. auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
  2051. emitOMPSimdRegion(CGF, S, Action);
  2052. };
  2053. llvm::Function *Fn;
  2054. llvm::Constant *Addr;
  2055. // Emit target region as a standalone region.
  2056. CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
  2057. S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
  2058. assert(Fn && Addr && "Target device function emission failed.");
  2059. }
  2060. void CodeGenFunction::EmitOMPTargetSimdDirective(
  2061. const OMPTargetSimdDirective &S) {
  2062. auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
  2063. emitOMPSimdRegion(CGF, S, Action);
  2064. };
  2065. emitCommonOMPTargetDirective(*this, S, CodeGen);
  2066. }
  2067. namespace {
  2068. struct ScheduleKindModifiersTy {
  2069. OpenMPScheduleClauseKind Kind;
  2070. OpenMPScheduleClauseModifier M1;
  2071. OpenMPScheduleClauseModifier M2;
  2072. ScheduleKindModifiersTy(OpenMPScheduleClauseKind Kind,
  2073. OpenMPScheduleClauseModifier M1,
  2074. OpenMPScheduleClauseModifier M2)
  2075. : Kind(Kind), M1(M1), M2(M2) {}
  2076. };
  2077. } // namespace
  2078. bool CodeGenFunction::EmitOMPWorksharingLoop(
  2079. const OMPLoopDirective &S, Expr *EUB,
  2080. const CodeGenLoopBoundsTy &CodeGenLoopBounds,
  2081. const CodeGenDispatchBoundsTy &CGDispatchBounds) {
  2082. // Emit the loop iteration variable.
  2083. const auto *IVExpr = cast<DeclRefExpr>(S.getIterationVariable());
  2084. const auto *IVDecl = cast<VarDecl>(IVExpr->getDecl());
  2085. EmitVarDecl(*IVDecl);
  2086. // Emit the iterations count variable.
  2087. // If it is not a variable, Sema decided to calculate iterations count on each
  2088. // iteration (e.g., it is foldable into a constant).
  2089. if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
  2090. EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
  2091. // Emit calculation of the iterations count.
  2092. EmitIgnoredExpr(S.getCalcLastIteration());
  2093. }
  2094. CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
  2095. bool HasLastprivateClause;
  2096. // Check pre-condition.
  2097. {
  2098. OMPLoopScope PreInitScope(*this, S);
  2099. // Skip the entire loop if we don't meet the precondition.
  2100. // If the condition constant folds and can be elided, avoid emitting the
  2101. // whole loop.
  2102. bool CondConstant;
  2103. llvm::BasicBlock *ContBlock = nullptr;
  2104. if (ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
  2105. if (!CondConstant)
  2106. return false;
  2107. } else {
  2108. llvm::BasicBlock *ThenBlock = createBasicBlock("omp.precond.then");
  2109. ContBlock = createBasicBlock("omp.precond.end");
  2110. emitPreCond(*this, S, S.getPreCond(), ThenBlock, ContBlock,
  2111. getProfileCount(&S));
  2112. EmitBlock(ThenBlock);
  2113. incrementProfileCounter(&S);
  2114. }
  2115. RunCleanupsScope DoacrossCleanupScope(*this);
  2116. bool Ordered = false;
  2117. if (const auto *OrderedClause = S.getSingleClause<OMPOrderedClause>()) {
  2118. if (OrderedClause->getNumForLoops())
  2119. RT.emitDoacrossInit(*this, S, OrderedClause->getLoopNumIterations());
  2120. else
  2121. Ordered = true;
  2122. }
  2123. llvm::DenseSet<const Expr *> EmittedFinals;
  2124. emitAlignedClause(*this, S);
  2125. bool HasLinears = EmitOMPLinearClauseInit(S);
  2126. // Emit helper vars inits.
  2127. std::pair<LValue, LValue> Bounds = CodeGenLoopBounds(*this, S);
  2128. LValue LB = Bounds.first;
  2129. LValue UB = Bounds.second;
  2130. LValue ST =
  2131. EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getStrideVariable()));
  2132. LValue IL =
  2133. EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getIsLastIterVariable()));
  2134. // Emit 'then' code.
  2135. {
  2136. OMPPrivateScope LoopScope(*this);
  2137. if (EmitOMPFirstprivateClause(S, LoopScope) || HasLinears) {
  2138. // Emit implicit barrier to synchronize threads and avoid data races on
  2139. // initialization of firstprivate variables and post-update of
  2140. // lastprivate variables.
  2141. CGM.getOpenMPRuntime().emitBarrierCall(
  2142. *this, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
  2143. /*ForceSimpleCall=*/true);
  2144. }
  2145. EmitOMPPrivateClause(S, LoopScope);
  2146. HasLastprivateClause = EmitOMPLastprivateClauseInit(S, LoopScope);
  2147. EmitOMPReductionClauseInit(S, LoopScope);
  2148. EmitOMPPrivateLoopCounters(S, LoopScope);
  2149. EmitOMPLinearClause(S, LoopScope);
  2150. (void)LoopScope.Privatize();
  2151. // Detect the loop schedule kind and chunk.
  2152. llvm::Value *Chunk = nullptr;
  2153. OpenMPScheduleTy ScheduleKind;
  2154. if (const auto *C = S.getSingleClause<OMPScheduleClause>()) {
  2155. ScheduleKind.Schedule = C->getScheduleKind();
  2156. ScheduleKind.M1 = C->getFirstScheduleModifier();
  2157. ScheduleKind.M2 = C->getSecondScheduleModifier();
  2158. if (const Expr *Ch = C->getChunkSize()) {
  2159. Chunk = EmitScalarExpr(Ch);
  2160. Chunk = EmitScalarConversion(Chunk, Ch->getType(),
  2161. S.getIterationVariable()->getType(),
  2162. S.getBeginLoc());
  2163. }
  2164. }
  2165. const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
  2166. const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
  2167. // OpenMP 4.5, 2.7.1 Loop Construct, Description.
  2168. // If the static schedule kind is specified or if the ordered clause is
  2169. // specified, and if no monotonic modifier is specified, the effect will
  2170. // be as if the monotonic modifier was specified.
  2171. if (RT.isStaticNonchunked(ScheduleKind.Schedule,
  2172. /* Chunked */ Chunk != nullptr) &&
  2173. !Ordered) {
  2174. if (isOpenMPSimdDirective(S.getDirectiveKind()))
  2175. EmitOMPSimdInit(S, /*IsMonotonic=*/true);
  2176. // OpenMP [2.7.1, Loop Construct, Description, table 2-1]
  2177. // When no chunk_size is specified, the iteration space is divided into
  2178. // chunks that are approximately equal in size, and at most one chunk is
  2179. // distributed to each thread. Note that the size of the chunks is
  2180. // unspecified in this case.
  2181. CGOpenMPRuntime::StaticRTInput StaticInit(
  2182. IVSize, IVSigned, Ordered, IL.getAddress(), LB.getAddress(),
  2183. UB.getAddress(), ST.getAddress());
  2184. RT.emitForStaticInit(*this, S.getBeginLoc(), S.getDirectiveKind(),
  2185. ScheduleKind, StaticInit);
  2186. JumpDest LoopExit =
  2187. getJumpDestInCurrentScope(createBasicBlock("omp.loop.exit"));
  2188. // UB = min(UB, GlobalUB);
  2189. EmitIgnoredExpr(S.getEnsureUpperBound());
  2190. // IV = LB;
  2191. EmitIgnoredExpr(S.getInit());
  2192. // while (idx <= UB) { BODY; ++idx; }
  2193. EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), S.getCond(),
  2194. S.getInc(),
  2195. [&S, LoopExit](CodeGenFunction &CGF) {
  2196. CGF.EmitOMPLoopBody(S, LoopExit);
  2197. CGF.EmitStopPoint(&S);
  2198. },
  2199. [](CodeGenFunction &) {});
  2200. EmitBlock(LoopExit.getBlock());
  2201. // Tell the runtime we are done.
  2202. auto &&CodeGen = [&S](CodeGenFunction &CGF) {
  2203. CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
  2204. S.getDirectiveKind());
  2205. };
  2206. OMPCancelStack.emitExit(*this, S.getDirectiveKind(), CodeGen);
  2207. } else {
  2208. const bool IsMonotonic =
  2209. Ordered || ScheduleKind.Schedule == OMPC_SCHEDULE_static ||
  2210. ScheduleKind.Schedule == OMPC_SCHEDULE_unknown ||
  2211. ScheduleKind.M1 == OMPC_SCHEDULE_MODIFIER_monotonic ||
  2212. ScheduleKind.M2 == OMPC_SCHEDULE_MODIFIER_monotonic;
  2213. // Emit the outer loop, which requests its work chunk [LB..UB] from
  2214. // runtime and runs the inner loop to process it.
  2215. const OMPLoopArguments LoopArguments(LB.getAddress(), UB.getAddress(),
  2216. ST.getAddress(), IL.getAddress(),
  2217. Chunk, EUB);
  2218. EmitOMPForOuterLoop(ScheduleKind, IsMonotonic, S, LoopScope, Ordered,
  2219. LoopArguments, CGDispatchBounds);
  2220. }
  2221. if (isOpenMPSimdDirective(S.getDirectiveKind())) {
  2222. EmitOMPSimdFinal(S, [IL, &S](CodeGenFunction &CGF) {
  2223. return CGF.Builder.CreateIsNotNull(
  2224. CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
  2225. });
  2226. }
  2227. EmitOMPReductionClauseFinal(
  2228. S, /*ReductionKind=*/isOpenMPSimdDirective(S.getDirectiveKind())
  2229. ? /*Parallel and Simd*/ OMPD_parallel_for_simd
  2230. : /*Parallel only*/ OMPD_parallel);
  2231. // Emit post-update of the reduction variables if IsLastIter != 0.
  2232. emitPostUpdateForReductionClause(
  2233. *this, S, [IL, &S](CodeGenFunction &CGF) {
  2234. return CGF.Builder.CreateIsNotNull(
  2235. CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
  2236. });
  2237. // Emit final copy of the lastprivate variables if IsLastIter != 0.
  2238. if (HasLastprivateClause)
  2239. EmitOMPLastprivateClauseFinal(
  2240. S, isOpenMPSimdDirective(S.getDirectiveKind()),
  2241. Builder.CreateIsNotNull(EmitLoadOfScalar(IL, S.getBeginLoc())));
  2242. }
  2243. EmitOMPLinearClauseFinal(S, [IL, &S](CodeGenFunction &CGF) {
  2244. return CGF.Builder.CreateIsNotNull(
  2245. CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
  2246. });
  2247. DoacrossCleanupScope.ForceCleanup();
  2248. // We're now done with the loop, so jump to the continuation block.
  2249. if (ContBlock) {
  2250. EmitBranch(ContBlock);
  2251. EmitBlock(ContBlock, /*IsFinished=*/true);
  2252. }
  2253. }
  2254. return HasLastprivateClause;
  2255. }
  2256. /// The following two functions generate expressions for the loop lower
  2257. /// and upper bounds in case of static and dynamic (dispatch) schedule
  2258. /// of the associated 'for' or 'distribute' loop.
  2259. static std::pair<LValue, LValue>
  2260. emitForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S) {
  2261. const auto &LS = cast<OMPLoopDirective>(S);
  2262. LValue LB =
  2263. EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getLowerBoundVariable()));
  2264. LValue UB =
  2265. EmitOMPHelperVar(CGF, cast<DeclRefExpr>(LS.getUpperBoundVariable()));
  2266. return {LB, UB};
  2267. }
  2268. /// When dealing with dispatch schedules (e.g. dynamic, guided) we do not
  2269. /// consider the lower and upper bound expressions generated by the
  2270. /// worksharing loop support, but we use 0 and the iteration space size as
  2271. /// constants
  2272. static std::pair<llvm::Value *, llvm::Value *>
  2273. emitDispatchForLoopBounds(CodeGenFunction &CGF, const OMPExecutableDirective &S,
  2274. Address LB, Address UB) {
  2275. const auto &LS = cast<OMPLoopDirective>(S);
  2276. const Expr *IVExpr = LS.getIterationVariable();
  2277. const unsigned IVSize = CGF.getContext().getTypeSize(IVExpr->getType());
  2278. llvm::Value *LBVal = CGF.Builder.getIntN(IVSize, 0);
  2279. llvm::Value *UBVal = CGF.EmitScalarExpr(LS.getLastIteration());
  2280. return {LBVal, UBVal};
  2281. }
  2282. void CodeGenFunction::EmitOMPForDirective(const OMPForDirective &S) {
  2283. bool HasLastprivates = false;
  2284. auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF,
  2285. PrePostActionTy &) {
  2286. OMPCancelStackRAII CancelRegion(CGF, OMPD_for, S.hasCancel());
  2287. HasLastprivates = CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(),
  2288. emitForLoopBounds,
  2289. emitDispatchForLoopBounds);
  2290. };
  2291. {
  2292. OMPLexicalScope Scope(*this, S, OMPD_unknown);
  2293. CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_for, CodeGen,
  2294. S.hasCancel());
  2295. }
  2296. // Emit an implicit barrier at the end.
  2297. if (!S.getSingleClause<OMPNowaitClause>() || HasLastprivates)
  2298. CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(), OMPD_for);
  2299. }
  2300. void CodeGenFunction::EmitOMPForSimdDirective(const OMPForSimdDirective &S) {
  2301. bool HasLastprivates = false;
  2302. auto &&CodeGen = [&S, &HasLastprivates](CodeGenFunction &CGF,
  2303. PrePostActionTy &) {
  2304. HasLastprivates = CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(),
  2305. emitForLoopBounds,
  2306. emitDispatchForLoopBounds);
  2307. };
  2308. {
  2309. OMPLexicalScope Scope(*this, S, OMPD_unknown);
  2310. CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_simd, CodeGen);
  2311. }
  2312. // Emit an implicit barrier at the end.
  2313. if (!S.getSingleClause<OMPNowaitClause>() || HasLastprivates)
  2314. CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(), OMPD_for);
  2315. }
  2316. static LValue createSectionLVal(CodeGenFunction &CGF, QualType Ty,
  2317. const Twine &Name,
  2318. llvm::Value *Init = nullptr) {
  2319. LValue LVal = CGF.MakeAddrLValue(CGF.CreateMemTemp(Ty, Name), Ty);
  2320. if (Init)
  2321. CGF.EmitStoreThroughLValue(RValue::get(Init), LVal, /*isInit*/ true);
  2322. return LVal;
  2323. }
  2324. void CodeGenFunction::EmitSections(const OMPExecutableDirective &S) {
  2325. const Stmt *CapturedStmt = S.getInnermostCapturedStmt()->getCapturedStmt();
  2326. const auto *CS = dyn_cast<CompoundStmt>(CapturedStmt);
  2327. bool HasLastprivates = false;
  2328. auto &&CodeGen = [&S, CapturedStmt, CS,
  2329. &HasLastprivates](CodeGenFunction &CGF, PrePostActionTy &) {
  2330. ASTContext &C = CGF.getContext();
  2331. QualType KmpInt32Ty =
  2332. C.getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1);
  2333. // Emit helper vars inits.
  2334. LValue LB = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.lb.",
  2335. CGF.Builder.getInt32(0));
  2336. llvm::ConstantInt *GlobalUBVal = CS != nullptr
  2337. ? CGF.Builder.getInt32(CS->size() - 1)
  2338. : CGF.Builder.getInt32(0);
  2339. LValue UB =
  2340. createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.ub.", GlobalUBVal);
  2341. LValue ST = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.st.",
  2342. CGF.Builder.getInt32(1));
  2343. LValue IL = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.il.",
  2344. CGF.Builder.getInt32(0));
  2345. // Loop counter.
  2346. LValue IV = createSectionLVal(CGF, KmpInt32Ty, ".omp.sections.iv.");
  2347. OpaqueValueExpr IVRefExpr(S.getBeginLoc(), KmpInt32Ty, VK_LValue);
  2348. CodeGenFunction::OpaqueValueMapping OpaqueIV(CGF, &IVRefExpr, IV);
  2349. OpaqueValueExpr UBRefExpr(S.getBeginLoc(), KmpInt32Ty, VK_LValue);
  2350. CodeGenFunction::OpaqueValueMapping OpaqueUB(CGF, &UBRefExpr, UB);
  2351. // Generate condition for loop.
  2352. BinaryOperator Cond(&IVRefExpr, &UBRefExpr, BO_LE, C.BoolTy, VK_RValue,
  2353. OK_Ordinary, S.getBeginLoc(), FPOptions());
  2354. // Increment for loop counter.
  2355. UnaryOperator Inc(&IVRefExpr, UO_PreInc, KmpInt32Ty, VK_RValue, OK_Ordinary,
  2356. S.getBeginLoc(), true);
  2357. auto &&BodyGen = [CapturedStmt, CS, &S, &IV](CodeGenFunction &CGF) {
  2358. // Iterate through all sections and emit a switch construct:
  2359. // switch (IV) {
  2360. // case 0:
  2361. // <SectionStmt[0]>;
  2362. // break;
  2363. // ...
  2364. // case <NumSection> - 1:
  2365. // <SectionStmt[<NumSection> - 1]>;
  2366. // break;
  2367. // }
  2368. // .omp.sections.exit:
  2369. llvm::BasicBlock *ExitBB = CGF.createBasicBlock(".omp.sections.exit");
  2370. llvm::SwitchInst *SwitchStmt =
  2371. CGF.Builder.CreateSwitch(CGF.EmitLoadOfScalar(IV, S.getBeginLoc()),
  2372. ExitBB, CS == nullptr ? 1 : CS->size());
  2373. if (CS) {
  2374. unsigned CaseNumber = 0;
  2375. for (const Stmt *SubStmt : CS->children()) {
  2376. auto CaseBB = CGF.createBasicBlock(".omp.sections.case");
  2377. CGF.EmitBlock(CaseBB);
  2378. SwitchStmt->addCase(CGF.Builder.getInt32(CaseNumber), CaseBB);
  2379. CGF.EmitStmt(SubStmt);
  2380. CGF.EmitBranch(ExitBB);
  2381. ++CaseNumber;
  2382. }
  2383. } else {
  2384. llvm::BasicBlock *CaseBB = CGF.createBasicBlock(".omp.sections.case");
  2385. CGF.EmitBlock(CaseBB);
  2386. SwitchStmt->addCase(CGF.Builder.getInt32(0), CaseBB);
  2387. CGF.EmitStmt(CapturedStmt);
  2388. CGF.EmitBranch(ExitBB);
  2389. }
  2390. CGF.EmitBlock(ExitBB, /*IsFinished=*/true);
  2391. };
  2392. CodeGenFunction::OMPPrivateScope LoopScope(CGF);
  2393. if (CGF.EmitOMPFirstprivateClause(S, LoopScope)) {
  2394. // Emit implicit barrier to synchronize threads and avoid data races on
  2395. // initialization of firstprivate variables and post-update of lastprivate
  2396. // variables.
  2397. CGF.CGM.getOpenMPRuntime().emitBarrierCall(
  2398. CGF, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
  2399. /*ForceSimpleCall=*/true);
  2400. }
  2401. CGF.EmitOMPPrivateClause(S, LoopScope);
  2402. HasLastprivates = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
  2403. CGF.EmitOMPReductionClauseInit(S, LoopScope);
  2404. (void)LoopScope.Privatize();
  2405. // Emit static non-chunked loop.
  2406. OpenMPScheduleTy ScheduleKind;
  2407. ScheduleKind.Schedule = OMPC_SCHEDULE_static;
  2408. CGOpenMPRuntime::StaticRTInput StaticInit(
  2409. /*IVSize=*/32, /*IVSigned=*/true, /*Ordered=*/false, IL.getAddress(),
  2410. LB.getAddress(), UB.getAddress(), ST.getAddress());
  2411. CGF.CGM.getOpenMPRuntime().emitForStaticInit(
  2412. CGF, S.getBeginLoc(), S.getDirectiveKind(), ScheduleKind, StaticInit);
  2413. // UB = min(UB, GlobalUB);
  2414. llvm::Value *UBVal = CGF.EmitLoadOfScalar(UB, S.getBeginLoc());
  2415. llvm::Value *MinUBGlobalUB = CGF.Builder.CreateSelect(
  2416. CGF.Builder.CreateICmpSLT(UBVal, GlobalUBVal), UBVal, GlobalUBVal);
  2417. CGF.EmitStoreOfScalar(MinUBGlobalUB, UB);
  2418. // IV = LB;
  2419. CGF.EmitStoreOfScalar(CGF.EmitLoadOfScalar(LB, S.getBeginLoc()), IV);
  2420. // while (idx <= UB) { BODY; ++idx; }
  2421. CGF.EmitOMPInnerLoop(S, /*RequiresCleanup=*/false, &Cond, &Inc, BodyGen,
  2422. [](CodeGenFunction &) {});
  2423. // Tell the runtime we are done.
  2424. auto &&CodeGen = [&S](CodeGenFunction &CGF) {
  2425. CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),
  2426. S.getDirectiveKind());
  2427. };
  2428. CGF.OMPCancelStack.emitExit(CGF, S.getDirectiveKind(), CodeGen);
  2429. CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
  2430. // Emit post-update of the reduction variables if IsLastIter != 0.
  2431. emitPostUpdateForReductionClause(CGF, S, [IL, &S](CodeGenFunction &CGF) {
  2432. return CGF.Builder.CreateIsNotNull(
  2433. CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
  2434. });
  2435. // Emit final copy of the lastprivate variables if IsLastIter != 0.
  2436. if (HasLastprivates)
  2437. CGF.EmitOMPLastprivateClauseFinal(
  2438. S, /*NoFinals=*/false,
  2439. CGF.Builder.CreateIsNotNull(
  2440. CGF.EmitLoadOfScalar(IL, S.getBeginLoc())));
  2441. };
  2442. bool HasCancel = false;
  2443. if (auto *OSD = dyn_cast<OMPSectionsDirective>(&S))
  2444. HasCancel = OSD->hasCancel();
  2445. else if (auto *OPSD = dyn_cast<OMPParallelSectionsDirective>(&S))
  2446. HasCancel = OPSD->hasCancel();
  2447. OMPCancelStackRAII CancelRegion(*this, S.getDirectiveKind(), HasCancel);
  2448. CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_sections, CodeGen,
  2449. HasCancel);
  2450. // Emit barrier for lastprivates only if 'sections' directive has 'nowait'
  2451. // clause. Otherwise the barrier will be generated by the codegen for the
  2452. // directive.
  2453. if (HasLastprivates && S.getSingleClause<OMPNowaitClause>()) {
  2454. // Emit implicit barrier to synchronize threads and avoid data races on
  2455. // initialization of firstprivate variables.
  2456. CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(),
  2457. OMPD_unknown);
  2458. }
  2459. }
  2460. void CodeGenFunction::EmitOMPSectionsDirective(const OMPSectionsDirective &S) {
  2461. {
  2462. OMPLexicalScope Scope(*this, S, OMPD_unknown);
  2463. EmitSections(S);
  2464. }
  2465. // Emit an implicit barrier at the end.
  2466. if (!S.getSingleClause<OMPNowaitClause>()) {
  2467. CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(),
  2468. OMPD_sections);
  2469. }
  2470. }
  2471. void CodeGenFunction::EmitOMPSectionDirective(const OMPSectionDirective &S) {
  2472. auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
  2473. CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
  2474. };
  2475. OMPLexicalScope Scope(*this, S, OMPD_unknown);
  2476. CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_section, CodeGen,
  2477. S.hasCancel());
  2478. }
  2479. void CodeGenFunction::EmitOMPSingleDirective(const OMPSingleDirective &S) {
  2480. llvm::SmallVector<const Expr *, 8> CopyprivateVars;
  2481. llvm::SmallVector<const Expr *, 8> DestExprs;
  2482. llvm::SmallVector<const Expr *, 8> SrcExprs;
  2483. llvm::SmallVector<const Expr *, 8> AssignmentOps;
  2484. // Check if there are any 'copyprivate' clauses associated with this
  2485. // 'single' construct.
  2486. // Build a list of copyprivate variables along with helper expressions
  2487. // (<source>, <destination>, <destination>=<source> expressions)
  2488. for (const auto *C : S.getClausesOfKind<OMPCopyprivateClause>()) {
  2489. CopyprivateVars.append(C->varlists().begin(), C->varlists().end());
  2490. DestExprs.append(C->destination_exprs().begin(),
  2491. C->destination_exprs().end());
  2492. SrcExprs.append(C->source_exprs().begin(), C->source_exprs().end());
  2493. AssignmentOps.append(C->assignment_ops().begin(),
  2494. C->assignment_ops().end());
  2495. }
  2496. // Emit code for 'single' region along with 'copyprivate' clauses
  2497. auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
  2498. Action.Enter(CGF);
  2499. OMPPrivateScope SingleScope(CGF);
  2500. (void)CGF.EmitOMPFirstprivateClause(S, SingleScope);
  2501. CGF.EmitOMPPrivateClause(S, SingleScope);
  2502. (void)SingleScope.Privatize();
  2503. CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
  2504. };
  2505. {
  2506. OMPLexicalScope Scope(*this, S, OMPD_unknown);
  2507. CGM.getOpenMPRuntime().emitSingleRegion(*this, CodeGen, S.getBeginLoc(),
  2508. CopyprivateVars, DestExprs,
  2509. SrcExprs, AssignmentOps);
  2510. }
  2511. // Emit an implicit barrier at the end (to avoid data race on firstprivate
  2512. // init or if no 'nowait' clause was specified and no 'copyprivate' clause).
  2513. if (!S.getSingleClause<OMPNowaitClause>() && CopyprivateVars.empty()) {
  2514. CGM.getOpenMPRuntime().emitBarrierCall(
  2515. *this, S.getBeginLoc(),
  2516. S.getSingleClause<OMPNowaitClause>() ? OMPD_unknown : OMPD_single);
  2517. }
  2518. }
  2519. void CodeGenFunction::EmitOMPMasterDirective(const OMPMasterDirective &S) {
  2520. auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
  2521. Action.Enter(CGF);
  2522. CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
  2523. };
  2524. OMPLexicalScope Scope(*this, S, OMPD_unknown);
  2525. CGM.getOpenMPRuntime().emitMasterRegion(*this, CodeGen, S.getBeginLoc());
  2526. }
  2527. void CodeGenFunction::EmitOMPCriticalDirective(const OMPCriticalDirective &S) {
  2528. auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
  2529. Action.Enter(CGF);
  2530. CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
  2531. };
  2532. const Expr *Hint = nullptr;
  2533. if (const auto *HintClause = S.getSingleClause<OMPHintClause>())
  2534. Hint = HintClause->getHint();
  2535. OMPLexicalScope Scope(*this, S, OMPD_unknown);
  2536. CGM.getOpenMPRuntime().emitCriticalRegion(*this,
  2537. S.getDirectiveName().getAsString(),
  2538. CodeGen, S.getBeginLoc(), Hint);
  2539. }
  2540. void CodeGenFunction::EmitOMPParallelForDirective(
  2541. const OMPParallelForDirective &S) {
  2542. // Emit directive as a combined directive that consists of two implicit
  2543. // directives: 'parallel' with 'for' directive.
  2544. auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
  2545. Action.Enter(CGF);
  2546. OMPCancelStackRAII CancelRegion(CGF, OMPD_parallel_for, S.hasCancel());
  2547. CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds,
  2548. emitDispatchForLoopBounds);
  2549. };
  2550. emitCommonOMPParallelDirective(*this, S, OMPD_for, CodeGen,
  2551. emitEmptyBoundParameters);
  2552. }
  2553. void CodeGenFunction::EmitOMPParallelForSimdDirective(
  2554. const OMPParallelForSimdDirective &S) {
  2555. // Emit directive as a combined directive that consists of two implicit
  2556. // directives: 'parallel' with 'for' directive.
  2557. auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
  2558. Action.Enter(CGF);
  2559. CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds,
  2560. emitDispatchForLoopBounds);
  2561. };
  2562. emitCommonOMPParallelDirective(*this, S, OMPD_simd, CodeGen,
  2563. emitEmptyBoundParameters);
  2564. }
  2565. void CodeGenFunction::EmitOMPParallelSectionsDirective(
  2566. const OMPParallelSectionsDirective &S) {
  2567. // Emit directive as a combined directive that consists of two implicit
  2568. // directives: 'parallel' with 'sections' directive.
  2569. auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
  2570. Action.Enter(CGF);
  2571. CGF.EmitSections(S);
  2572. };
  2573. emitCommonOMPParallelDirective(*this, S, OMPD_sections, CodeGen,
  2574. emitEmptyBoundParameters);
  2575. }
  2576. void CodeGenFunction::EmitOMPTaskBasedDirective(
  2577. const OMPExecutableDirective &S, const OpenMPDirectiveKind CapturedRegion,
  2578. const RegionCodeGenTy &BodyGen, const TaskGenTy &TaskGen,
  2579. OMPTaskDataTy &Data) {
  2580. // Emit outlined function for task construct.
  2581. const CapturedStmt *CS = S.getCapturedStmt(CapturedRegion);
  2582. auto I = CS->getCapturedDecl()->param_begin();
  2583. auto PartId = std::next(I);
  2584. auto TaskT = std::next(I, 4);
  2585. // Check if the task is final
  2586. if (const auto *Clause = S.getSingleClause<OMPFinalClause>()) {
  2587. // If the condition constant folds and can be elided, try to avoid emitting
  2588. // the condition and the dead arm of the if/else.
  2589. const Expr *Cond = Clause->getCondition();
  2590. bool CondConstant;
  2591. if (ConstantFoldsToSimpleInteger(Cond, CondConstant))
  2592. Data.Final.setInt(CondConstant);
  2593. else
  2594. Data.Final.setPointer(EvaluateExprAsBool(Cond));
  2595. } else {
  2596. // By default the task is not final.
  2597. Data.Final.setInt(/*IntVal=*/false);
  2598. }
  2599. // Check if the task has 'priority' clause.
  2600. if (const auto *Clause = S.getSingleClause<OMPPriorityClause>()) {
  2601. const Expr *Prio = Clause->getPriority();
  2602. Data.Priority.setInt(/*IntVal=*/true);
  2603. Data.Priority.setPointer(EmitScalarConversion(
  2604. EmitScalarExpr(Prio), Prio->getType(),
  2605. getContext().getIntTypeForBitwidth(/*DestWidth=*/32, /*Signed=*/1),
  2606. Prio->getExprLoc()));
  2607. }
  2608. // The first function argument for tasks is a thread id, the second one is a
  2609. // part id (0 for tied tasks, >=0 for untied task).
  2610. llvm::DenseSet<const VarDecl *> EmittedAsPrivate;
  2611. // Get list of private variables.
  2612. for (const auto *C : S.getClausesOfKind<OMPPrivateClause>()) {
  2613. auto IRef = C->varlist_begin();
  2614. for (const Expr *IInit : C->private_copies()) {
  2615. const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
  2616. if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
  2617. Data.PrivateVars.push_back(*IRef);
  2618. Data.PrivateCopies.push_back(IInit);
  2619. }
  2620. ++IRef;
  2621. }
  2622. }
  2623. EmittedAsPrivate.clear();
  2624. // Get list of firstprivate variables.
  2625. for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) {
  2626. auto IRef = C->varlist_begin();
  2627. auto IElemInitRef = C->inits().begin();
  2628. for (const Expr *IInit : C->private_copies()) {
  2629. const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
  2630. if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
  2631. Data.FirstprivateVars.push_back(*IRef);
  2632. Data.FirstprivateCopies.push_back(IInit);
  2633. Data.FirstprivateInits.push_back(*IElemInitRef);
  2634. }
  2635. ++IRef;
  2636. ++IElemInitRef;
  2637. }
  2638. }
  2639. // Get list of lastprivate variables (for taskloops).
  2640. llvm::DenseMap<const VarDecl *, const DeclRefExpr *> LastprivateDstsOrigs;
  2641. for (const auto *C : S.getClausesOfKind<OMPLastprivateClause>()) {
  2642. auto IRef = C->varlist_begin();
  2643. auto ID = C->destination_exprs().begin();
  2644. for (const Expr *IInit : C->private_copies()) {
  2645. const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());
  2646. if (EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {
  2647. Data.LastprivateVars.push_back(*IRef);
  2648. Data.LastprivateCopies.push_back(IInit);
  2649. }
  2650. LastprivateDstsOrigs.insert(
  2651. {cast<VarDecl>(cast<DeclRefExpr>(*ID)->getDecl()),
  2652. cast<DeclRefExpr>(*IRef)});
  2653. ++IRef;
  2654. ++ID;
  2655. }
  2656. }
  2657. SmallVector<const Expr *, 4> LHSs;
  2658. SmallVector<const Expr *, 4> RHSs;
  2659. for (const auto *C : S.getClausesOfKind<OMPReductionClause>()) {
  2660. auto IPriv = C->privates().begin();
  2661. auto IRed = C->reduction_ops().begin();
  2662. auto ILHS = C->lhs_exprs().begin();
  2663. auto IRHS = C->rhs_exprs().begin();
  2664. for (const Expr *Ref : C->varlists()) {
  2665. Data.ReductionVars.emplace_back(Ref);
  2666. Data.ReductionCopies.emplace_back(*IPriv);
  2667. Data.ReductionOps.emplace_back(*IRed);
  2668. LHSs.emplace_back(*ILHS);
  2669. RHSs.emplace_back(*IRHS);
  2670. std::advance(IPriv, 1);
  2671. std::advance(IRed, 1);
  2672. std::advance(ILHS, 1);
  2673. std::advance(IRHS, 1);
  2674. }
  2675. }
  2676. Data.Reductions = CGM.getOpenMPRuntime().emitTaskReductionInit(
  2677. *this, S.getBeginLoc(), LHSs, RHSs, Data);
  2678. // Build list of dependences.
  2679. for (const auto *C : S.getClausesOfKind<OMPDependClause>())
  2680. for (const Expr *IRef : C->varlists())
  2681. Data.Dependences.emplace_back(C->getDependencyKind(), IRef);
  2682. auto &&CodeGen = [&Data, &S, CS, &BodyGen, &LastprivateDstsOrigs,
  2683. CapturedRegion](CodeGenFunction &CGF,
  2684. PrePostActionTy &Action) {
  2685. // Set proper addresses for generated private copies.
  2686. OMPPrivateScope Scope(CGF);
  2687. if (!Data.PrivateVars.empty() || !Data.FirstprivateVars.empty() ||
  2688. !Data.LastprivateVars.empty()) {
  2689. enum { PrivatesParam = 2, CopyFnParam = 3 };
  2690. llvm::Value *CopyFn = CGF.Builder.CreateLoad(
  2691. CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam)));
  2692. llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(
  2693. CS->getCapturedDecl()->getParam(PrivatesParam)));
  2694. // Map privates.
  2695. llvm::SmallVector<std::pair<const VarDecl *, Address>, 16> PrivatePtrs;
  2696. llvm::SmallVector<llvm::Value *, 16> CallArgs;
  2697. CallArgs.push_back(PrivatesPtr);
  2698. for (const Expr *E : Data.PrivateVars) {
  2699. const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
  2700. Address PrivatePtr = CGF.CreateMemTemp(
  2701. CGF.getContext().getPointerType(E->getType()), ".priv.ptr.addr");
  2702. PrivatePtrs.emplace_back(VD, PrivatePtr);
  2703. CallArgs.push_back(PrivatePtr.getPointer());
  2704. }
  2705. for (const Expr *E : Data.FirstprivateVars) {
  2706. const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
  2707. Address PrivatePtr =
  2708. CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
  2709. ".firstpriv.ptr.addr");
  2710. PrivatePtrs.emplace_back(VD, PrivatePtr);
  2711. CallArgs.push_back(PrivatePtr.getPointer());
  2712. }
  2713. for (const Expr *E : Data.LastprivateVars) {
  2714. const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
  2715. Address PrivatePtr =
  2716. CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
  2717. ".lastpriv.ptr.addr");
  2718. PrivatePtrs.emplace_back(VD, PrivatePtr);
  2719. CallArgs.push_back(PrivatePtr.getPointer());
  2720. }
  2721. CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, S.getBeginLoc(),
  2722. CopyFn, CallArgs);
  2723. for (const auto &Pair : LastprivateDstsOrigs) {
  2724. const auto *OrigVD = cast<VarDecl>(Pair.second->getDecl());
  2725. DeclRefExpr DRE(
  2726. const_cast<VarDecl *>(OrigVD),
  2727. /*RefersToEnclosingVariableOrCapture=*/CGF.CapturedStmtInfo->lookup(
  2728. OrigVD) != nullptr,
  2729. Pair.second->getType(), VK_LValue, Pair.second->getExprLoc());
  2730. Scope.addPrivate(Pair.first, [&CGF, &DRE]() {
  2731. return CGF.EmitLValue(&DRE).getAddress();
  2732. });
  2733. }
  2734. for (const auto &Pair : PrivatePtrs) {
  2735. Address Replacement(CGF.Builder.CreateLoad(Pair.second),
  2736. CGF.getContext().getDeclAlign(Pair.first));
  2737. Scope.addPrivate(Pair.first, [Replacement]() { return Replacement; });
  2738. }
  2739. }
  2740. if (Data.Reductions) {
  2741. OMPLexicalScope LexScope(CGF, S, CapturedRegion);
  2742. ReductionCodeGen RedCG(Data.ReductionVars, Data.ReductionCopies,
  2743. Data.ReductionOps);
  2744. llvm::Value *ReductionsPtr = CGF.Builder.CreateLoad(
  2745. CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(9)));
  2746. for (unsigned Cnt = 0, E = Data.ReductionVars.size(); Cnt < E; ++Cnt) {
  2747. RedCG.emitSharedLValue(CGF, Cnt);
  2748. RedCG.emitAggregateType(CGF, Cnt);
  2749. // FIXME: This must removed once the runtime library is fixed.
  2750. // Emit required threadprivate variables for
  2751. // initilizer/combiner/finalizer.
  2752. CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(),
  2753. RedCG, Cnt);
  2754. Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
  2755. CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
  2756. Replacement =
  2757. Address(CGF.EmitScalarConversion(
  2758. Replacement.getPointer(), CGF.getContext().VoidPtrTy,
  2759. CGF.getContext().getPointerType(
  2760. Data.ReductionCopies[Cnt]->getType()),
  2761. Data.ReductionCopies[Cnt]->getExprLoc()),
  2762. Replacement.getAlignment());
  2763. Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);
  2764. Scope.addPrivate(RedCG.getBaseDecl(Cnt),
  2765. [Replacement]() { return Replacement; });
  2766. }
  2767. }
  2768. // Privatize all private variables except for in_reduction items.
  2769. (void)Scope.Privatize();
  2770. SmallVector<const Expr *, 4> InRedVars;
  2771. SmallVector<const Expr *, 4> InRedPrivs;
  2772. SmallVector<const Expr *, 4> InRedOps;
  2773. SmallVector<const Expr *, 4> TaskgroupDescriptors;
  2774. for (const auto *C : S.getClausesOfKind<OMPInReductionClause>()) {
  2775. auto IPriv = C->privates().begin();
  2776. auto IRed = C->reduction_ops().begin();
  2777. auto ITD = C->taskgroup_descriptors().begin();
  2778. for (const Expr *Ref : C->varlists()) {
  2779. InRedVars.emplace_back(Ref);
  2780. InRedPrivs.emplace_back(*IPriv);
  2781. InRedOps.emplace_back(*IRed);
  2782. TaskgroupDescriptors.emplace_back(*ITD);
  2783. std::advance(IPriv, 1);
  2784. std::advance(IRed, 1);
  2785. std::advance(ITD, 1);
  2786. }
  2787. }
  2788. // Privatize in_reduction items here, because taskgroup descriptors must be
  2789. // privatized earlier.
  2790. OMPPrivateScope InRedScope(CGF);
  2791. if (!InRedVars.empty()) {
  2792. ReductionCodeGen RedCG(InRedVars, InRedPrivs, InRedOps);
  2793. for (unsigned Cnt = 0, E = InRedVars.size(); Cnt < E; ++Cnt) {
  2794. RedCG.emitSharedLValue(CGF, Cnt);
  2795. RedCG.emitAggregateType(CGF, Cnt);
  2796. // The taskgroup descriptor variable is always implicit firstprivate and
  2797. // privatized already during procoessing of the firstprivates.
  2798. // FIXME: This must removed once the runtime library is fixed.
  2799. // Emit required threadprivate variables for
  2800. // initilizer/combiner/finalizer.
  2801. CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(),
  2802. RedCG, Cnt);
  2803. llvm::Value *ReductionsPtr =
  2804. CGF.EmitLoadOfScalar(CGF.EmitLValue(TaskgroupDescriptors[Cnt]),
  2805. TaskgroupDescriptors[Cnt]->getExprLoc());
  2806. Address Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(
  2807. CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt));
  2808. Replacement = Address(
  2809. CGF.EmitScalarConversion(
  2810. Replacement.getPointer(), CGF.getContext().VoidPtrTy,
  2811. CGF.getContext().getPointerType(InRedPrivs[Cnt]->getType()),
  2812. InRedPrivs[Cnt]->getExprLoc()),
  2813. Replacement.getAlignment());
  2814. Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);
  2815. InRedScope.addPrivate(RedCG.getBaseDecl(Cnt),
  2816. [Replacement]() { return Replacement; });
  2817. }
  2818. }
  2819. (void)InRedScope.Privatize();
  2820. Action.Enter(CGF);
  2821. BodyGen(CGF);
  2822. };
  2823. llvm::Value *OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction(
  2824. S, *I, *PartId, *TaskT, S.getDirectiveKind(), CodeGen, Data.Tied,
  2825. Data.NumberOfParts);
  2826. OMPLexicalScope Scope(*this, S);
  2827. TaskGen(*this, OutlinedFn, Data);
  2828. }
  2829. static ImplicitParamDecl *
  2830. createImplicitFirstprivateForType(ASTContext &C, OMPTaskDataTy &Data,
  2831. QualType Ty, CapturedDecl *CD,
  2832. SourceLocation Loc) {
  2833. auto *OrigVD = ImplicitParamDecl::Create(C, CD, Loc, /*Id=*/nullptr, Ty,
  2834. ImplicitParamDecl::Other);
  2835. auto *OrigRef = DeclRefExpr::Create(
  2836. C, NestedNameSpecifierLoc(), SourceLocation(), OrigVD,
  2837. /*RefersToEnclosingVariableOrCapture=*/false, Loc, Ty, VK_LValue);
  2838. auto *PrivateVD = ImplicitParamDecl::Create(C, CD, Loc, /*Id=*/nullptr, Ty,
  2839. ImplicitParamDecl::Other);
  2840. auto *PrivateRef = DeclRefExpr::Create(
  2841. C, NestedNameSpecifierLoc(), SourceLocation(), PrivateVD,
  2842. /*RefersToEnclosingVariableOrCapture=*/false, Loc, Ty, VK_LValue);
  2843. QualType ElemType = C.getBaseElementType(Ty);
  2844. auto *InitVD = ImplicitParamDecl::Create(C, CD, Loc, /*Id=*/nullptr, ElemType,
  2845. ImplicitParamDecl::Other);
  2846. auto *InitRef = DeclRefExpr::Create(
  2847. C, NestedNameSpecifierLoc(), SourceLocation(), InitVD,
  2848. /*RefersToEnclosingVariableOrCapture=*/false, Loc, ElemType, VK_LValue);
  2849. PrivateVD->setInitStyle(VarDecl::CInit);
  2850. PrivateVD->setInit(ImplicitCastExpr::Create(C, ElemType, CK_LValueToRValue,
  2851. InitRef, /*BasePath=*/nullptr,
  2852. VK_RValue));
  2853. Data.FirstprivateVars.emplace_back(OrigRef);
  2854. Data.FirstprivateCopies.emplace_back(PrivateRef);
  2855. Data.FirstprivateInits.emplace_back(InitRef);
  2856. return OrigVD;
  2857. }
  2858. void CodeGenFunction::EmitOMPTargetTaskBasedDirective(
  2859. const OMPExecutableDirective &S, const RegionCodeGenTy &BodyGen,
  2860. OMPTargetDataInfo &InputInfo) {
  2861. // Emit outlined function for task construct.
  2862. const CapturedStmt *CS = S.getCapturedStmt(OMPD_task);
  2863. Address CapturedStruct = GenerateCapturedStmtArgument(*CS);
  2864. QualType SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl());
  2865. auto I = CS->getCapturedDecl()->param_begin();
  2866. auto PartId = std::next(I);
  2867. auto TaskT = std::next(I, 4);
  2868. OMPTaskDataTy Data;
  2869. // The task is not final.
  2870. Data.Final.setInt(/*IntVal=*/false);
  2871. // Get list of firstprivate variables.
  2872. for (const auto *C : S.getClausesOfKind<OMPFirstprivateClause>()) {
  2873. auto IRef = C->varlist_begin();
  2874. auto IElemInitRef = C->inits().begin();
  2875. for (auto *IInit : C->private_copies()) {
  2876. Data.FirstprivateVars.push_back(*IRef);
  2877. Data.FirstprivateCopies.push_back(IInit);
  2878. Data.FirstprivateInits.push_back(*IElemInitRef);
  2879. ++IRef;
  2880. ++IElemInitRef;
  2881. }
  2882. }
  2883. OMPPrivateScope TargetScope(*this);
  2884. VarDecl *BPVD = nullptr;
  2885. VarDecl *PVD = nullptr;
  2886. VarDecl *SVD = nullptr;
  2887. if (InputInfo.NumberOfTargetItems > 0) {
  2888. auto *CD = CapturedDecl::Create(
  2889. getContext(), getContext().getTranslationUnitDecl(), /*NumParams=*/0);
  2890. llvm::APInt ArrSize(/*numBits=*/32, InputInfo.NumberOfTargetItems);
  2891. QualType BaseAndPointersType = getContext().getConstantArrayType(
  2892. getContext().VoidPtrTy, ArrSize, ArrayType::Normal,
  2893. /*IndexTypeQuals=*/0);
  2894. BPVD = createImplicitFirstprivateForType(
  2895. getContext(), Data, BaseAndPointersType, CD, S.getBeginLoc());
  2896. PVD = createImplicitFirstprivateForType(
  2897. getContext(), Data, BaseAndPointersType, CD, S.getBeginLoc());
  2898. QualType SizesType = getContext().getConstantArrayType(
  2899. getContext().getSizeType(), ArrSize, ArrayType::Normal,
  2900. /*IndexTypeQuals=*/0);
  2901. SVD = createImplicitFirstprivateForType(getContext(), Data, SizesType, CD,
  2902. S.getBeginLoc());
  2903. TargetScope.addPrivate(
  2904. BPVD, [&InputInfo]() { return InputInfo.BasePointersArray; });
  2905. TargetScope.addPrivate(PVD,
  2906. [&InputInfo]() { return InputInfo.PointersArray; });
  2907. TargetScope.addPrivate(SVD,
  2908. [&InputInfo]() { return InputInfo.SizesArray; });
  2909. }
  2910. (void)TargetScope.Privatize();
  2911. // Build list of dependences.
  2912. for (const auto *C : S.getClausesOfKind<OMPDependClause>())
  2913. for (const Expr *IRef : C->varlists())
  2914. Data.Dependences.emplace_back(C->getDependencyKind(), IRef);
  2915. auto &&CodeGen = [&Data, &S, CS, &BodyGen, BPVD, PVD, SVD,
  2916. &InputInfo](CodeGenFunction &CGF, PrePostActionTy &Action) {
  2917. // Set proper addresses for generated private copies.
  2918. OMPPrivateScope Scope(CGF);
  2919. if (!Data.FirstprivateVars.empty()) {
  2920. enum { PrivatesParam = 2, CopyFnParam = 3 };
  2921. llvm::Value *CopyFn = CGF.Builder.CreateLoad(
  2922. CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam)));
  2923. llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(
  2924. CS->getCapturedDecl()->getParam(PrivatesParam)));
  2925. // Map privates.
  2926. llvm::SmallVector<std::pair<const VarDecl *, Address>, 16> PrivatePtrs;
  2927. llvm::SmallVector<llvm::Value *, 16> CallArgs;
  2928. CallArgs.push_back(PrivatesPtr);
  2929. for (const Expr *E : Data.FirstprivateVars) {
  2930. const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
  2931. Address PrivatePtr =
  2932. CGF.CreateMemTemp(CGF.getContext().getPointerType(E->getType()),
  2933. ".firstpriv.ptr.addr");
  2934. PrivatePtrs.emplace_back(VD, PrivatePtr);
  2935. CallArgs.push_back(PrivatePtr.getPointer());
  2936. }
  2937. CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, S.getBeginLoc(),
  2938. CopyFn, CallArgs);
  2939. for (const auto &Pair : PrivatePtrs) {
  2940. Address Replacement(CGF.Builder.CreateLoad(Pair.second),
  2941. CGF.getContext().getDeclAlign(Pair.first));
  2942. Scope.addPrivate(Pair.first, [Replacement]() { return Replacement; });
  2943. }
  2944. }
  2945. // Privatize all private variables except for in_reduction items.
  2946. (void)Scope.Privatize();
  2947. if (InputInfo.NumberOfTargetItems > 0) {
  2948. InputInfo.BasePointersArray = CGF.Builder.CreateConstArrayGEP(
  2949. CGF.GetAddrOfLocalVar(BPVD), /*Index=*/0, CGF.getPointerSize());
  2950. InputInfo.PointersArray = CGF.Builder.CreateConstArrayGEP(
  2951. CGF.GetAddrOfLocalVar(PVD), /*Index=*/0, CGF.getPointerSize());
  2952. InputInfo.SizesArray = CGF.Builder.CreateConstArrayGEP(
  2953. CGF.GetAddrOfLocalVar(SVD), /*Index=*/0, CGF.getSizeSize());
  2954. }
  2955. Action.Enter(CGF);
  2956. OMPLexicalScope LexScope(CGF, S, OMPD_task, /*EmitPreInitStmt=*/false);
  2957. BodyGen(CGF);
  2958. };
  2959. llvm::Value *OutlinedFn = CGM.getOpenMPRuntime().emitTaskOutlinedFunction(
  2960. S, *I, *PartId, *TaskT, S.getDirectiveKind(), CodeGen, /*Tied=*/true,
  2961. Data.NumberOfParts);
  2962. llvm::APInt TrueOrFalse(32, S.hasClausesOfKind<OMPNowaitClause>() ? 1 : 0);
  2963. IntegerLiteral IfCond(getContext(), TrueOrFalse,
  2964. getContext().getIntTypeForBitwidth(32, /*Signed=*/0),
  2965. SourceLocation());
  2966. CGM.getOpenMPRuntime().emitTaskCall(*this, S.getBeginLoc(), S, OutlinedFn,
  2967. SharedsTy, CapturedStruct, &IfCond, Data);
  2968. }
  2969. void CodeGenFunction::EmitOMPTaskDirective(const OMPTaskDirective &S) {
  2970. // Emit outlined function for task construct.
  2971. const CapturedStmt *CS = S.getCapturedStmt(OMPD_task);
  2972. Address CapturedStruct = GenerateCapturedStmtArgument(*CS);
  2973. QualType SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl());
  2974. const Expr *IfCond = nullptr;
  2975. for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
  2976. if (C->getNameModifier() == OMPD_unknown ||
  2977. C->getNameModifier() == OMPD_task) {
  2978. IfCond = C->getCondition();
  2979. break;
  2980. }
  2981. }
  2982. OMPTaskDataTy Data;
  2983. // Check if we should emit tied or untied task.
  2984. Data.Tied = !S.getSingleClause<OMPUntiedClause>();
  2985. auto &&BodyGen = [CS](CodeGenFunction &CGF, PrePostActionTy &) {
  2986. CGF.EmitStmt(CS->getCapturedStmt());
  2987. };
  2988. auto &&TaskGen = [&S, SharedsTy, CapturedStruct,
  2989. IfCond](CodeGenFunction &CGF, llvm::Value *OutlinedFn,
  2990. const OMPTaskDataTy &Data) {
  2991. CGF.CGM.getOpenMPRuntime().emitTaskCall(CGF, S.getBeginLoc(), S, OutlinedFn,
  2992. SharedsTy, CapturedStruct, IfCond,
  2993. Data);
  2994. };
  2995. EmitOMPTaskBasedDirective(S, OMPD_task, BodyGen, TaskGen, Data);
  2996. }
  2997. void CodeGenFunction::EmitOMPTaskyieldDirective(
  2998. const OMPTaskyieldDirective &S) {
  2999. CGM.getOpenMPRuntime().emitTaskyieldCall(*this, S.getBeginLoc());
  3000. }
  3001. void CodeGenFunction::EmitOMPBarrierDirective(const OMPBarrierDirective &S) {
  3002. CGM.getOpenMPRuntime().emitBarrierCall(*this, S.getBeginLoc(), OMPD_barrier);
  3003. }
  3004. void CodeGenFunction::EmitOMPTaskwaitDirective(const OMPTaskwaitDirective &S) {
  3005. CGM.getOpenMPRuntime().emitTaskwaitCall(*this, S.getBeginLoc());
  3006. }
  3007. void CodeGenFunction::EmitOMPTaskgroupDirective(
  3008. const OMPTaskgroupDirective &S) {
  3009. auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
  3010. Action.Enter(CGF);
  3011. if (const Expr *E = S.getReductionRef()) {
  3012. SmallVector<const Expr *, 4> LHSs;
  3013. SmallVector<const Expr *, 4> RHSs;
  3014. OMPTaskDataTy Data;
  3015. for (const auto *C : S.getClausesOfKind<OMPTaskReductionClause>()) {
  3016. auto IPriv = C->privates().begin();
  3017. auto IRed = C->reduction_ops().begin();
  3018. auto ILHS = C->lhs_exprs().begin();
  3019. auto IRHS = C->rhs_exprs().begin();
  3020. for (const Expr *Ref : C->varlists()) {
  3021. Data.ReductionVars.emplace_back(Ref);
  3022. Data.ReductionCopies.emplace_back(*IPriv);
  3023. Data.ReductionOps.emplace_back(*IRed);
  3024. LHSs.emplace_back(*ILHS);
  3025. RHSs.emplace_back(*IRHS);
  3026. std::advance(IPriv, 1);
  3027. std::advance(IRed, 1);
  3028. std::advance(ILHS, 1);
  3029. std::advance(IRHS, 1);
  3030. }
  3031. }
  3032. llvm::Value *ReductionDesc =
  3033. CGF.CGM.getOpenMPRuntime().emitTaskReductionInit(CGF, S.getBeginLoc(),
  3034. LHSs, RHSs, Data);
  3035. const auto *VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
  3036. CGF.EmitVarDecl(*VD);
  3037. CGF.EmitStoreOfScalar(ReductionDesc, CGF.GetAddrOfLocalVar(VD),
  3038. /*Volatile=*/false, E->getType());
  3039. }
  3040. CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
  3041. };
  3042. OMPLexicalScope Scope(*this, S, OMPD_unknown);
  3043. CGM.getOpenMPRuntime().emitTaskgroupRegion(*this, CodeGen, S.getBeginLoc());
  3044. }
  3045. void CodeGenFunction::EmitOMPFlushDirective(const OMPFlushDirective &S) {
  3046. CGM.getOpenMPRuntime().emitFlush(
  3047. *this,
  3048. [&S]() -> ArrayRef<const Expr *> {
  3049. if (const auto *FlushClause = S.getSingleClause<OMPFlushClause>())
  3050. return llvm::makeArrayRef(FlushClause->varlist_begin(),
  3051. FlushClause->varlist_end());
  3052. return llvm::None;
  3053. }(),
  3054. S.getBeginLoc());
  3055. }
  3056. void CodeGenFunction::EmitOMPDistributeLoop(const OMPLoopDirective &S,
  3057. const CodeGenLoopTy &CodeGenLoop,
  3058. Expr *IncExpr) {
  3059. // Emit the loop iteration variable.
  3060. const auto *IVExpr = cast<DeclRefExpr>(S.getIterationVariable());
  3061. const auto *IVDecl = cast<VarDecl>(IVExpr->getDecl());
  3062. EmitVarDecl(*IVDecl);
  3063. // Emit the iterations count variable.
  3064. // If it is not a variable, Sema decided to calculate iterations count on each
  3065. // iteration (e.g., it is foldable into a constant).
  3066. if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
  3067. EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
  3068. // Emit calculation of the iterations count.
  3069. EmitIgnoredExpr(S.getCalcLastIteration());
  3070. }
  3071. CGOpenMPRuntime &RT = CGM.getOpenMPRuntime();
  3072. bool HasLastprivateClause = false;
  3073. // Check pre-condition.
  3074. {
  3075. OMPLoopScope PreInitScope(*this, S);
  3076. // Skip the entire loop if we don't meet the precondition.
  3077. // If the condition constant folds and can be elided, avoid emitting the
  3078. // whole loop.
  3079. bool CondConstant;
  3080. llvm::BasicBlock *ContBlock = nullptr;
  3081. if (ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
  3082. if (!CondConstant)
  3083. return;
  3084. } else {
  3085. llvm::BasicBlock *ThenBlock = createBasicBlock("omp.precond.then");
  3086. ContBlock = createBasicBlock("omp.precond.end");
  3087. emitPreCond(*this, S, S.getPreCond(), ThenBlock, ContBlock,
  3088. getProfileCount(&S));
  3089. EmitBlock(ThenBlock);
  3090. incrementProfileCounter(&S);
  3091. }
  3092. emitAlignedClause(*this, S);
  3093. // Emit 'then' code.
  3094. {
  3095. // Emit helper vars inits.
  3096. LValue LB = EmitOMPHelperVar(
  3097. *this, cast<DeclRefExpr>(
  3098. (isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
  3099. ? S.getCombinedLowerBoundVariable()
  3100. : S.getLowerBoundVariable())));
  3101. LValue UB = EmitOMPHelperVar(
  3102. *this, cast<DeclRefExpr>(
  3103. (isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
  3104. ? S.getCombinedUpperBoundVariable()
  3105. : S.getUpperBoundVariable())));
  3106. LValue ST =
  3107. EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getStrideVariable()));
  3108. LValue IL =
  3109. EmitOMPHelperVar(*this, cast<DeclRefExpr>(S.getIsLastIterVariable()));
  3110. OMPPrivateScope LoopScope(*this);
  3111. if (EmitOMPFirstprivateClause(S, LoopScope)) {
  3112. // Emit implicit barrier to synchronize threads and avoid data races
  3113. // on initialization of firstprivate variables and post-update of
  3114. // lastprivate variables.
  3115. CGM.getOpenMPRuntime().emitBarrierCall(
  3116. *this, S.getBeginLoc(), OMPD_unknown, /*EmitChecks=*/false,
  3117. /*ForceSimpleCall=*/true);
  3118. }
  3119. EmitOMPPrivateClause(S, LoopScope);
  3120. if (isOpenMPSimdDirective(S.getDirectiveKind()) &&
  3121. !isOpenMPParallelDirective(S.getDirectiveKind()) &&
  3122. !isOpenMPTeamsDirective(S.getDirectiveKind()))
  3123. EmitOMPReductionClauseInit(S, LoopScope);
  3124. HasLastprivateClause = EmitOMPLastprivateClauseInit(S, LoopScope);
  3125. EmitOMPPrivateLoopCounters(S, LoopScope);
  3126. (void)LoopScope.Privatize();
  3127. // Detect the distribute schedule kind and chunk.
  3128. llvm::Value *Chunk = nullptr;
  3129. OpenMPDistScheduleClauseKind ScheduleKind = OMPC_DIST_SCHEDULE_unknown;
  3130. if (const auto *C = S.getSingleClause<OMPDistScheduleClause>()) {
  3131. ScheduleKind = C->getDistScheduleKind();
  3132. if (const Expr *Ch = C->getChunkSize()) {
  3133. Chunk = EmitScalarExpr(Ch);
  3134. Chunk = EmitScalarConversion(Chunk, Ch->getType(),
  3135. S.getIterationVariable()->getType(),
  3136. S.getBeginLoc());
  3137. }
  3138. }
  3139. const unsigned IVSize = getContext().getTypeSize(IVExpr->getType());
  3140. const bool IVSigned = IVExpr->getType()->hasSignedIntegerRepresentation();
  3141. // OpenMP [2.10.8, distribute Construct, Description]
  3142. // If dist_schedule is specified, kind must be static. If specified,
  3143. // iterations are divided into chunks of size chunk_size, chunks are
  3144. // assigned to the teams of the league in a round-robin fashion in the
  3145. // order of the team number. When no chunk_size is specified, the
  3146. // iteration space is divided into chunks that are approximately equal
  3147. // in size, and at most one chunk is distributed to each team of the
  3148. // league. The size of the chunks is unspecified in this case.
  3149. if (RT.isStaticNonchunked(ScheduleKind,
  3150. /* Chunked */ Chunk != nullptr)) {
  3151. if (isOpenMPSimdDirective(S.getDirectiveKind()))
  3152. EmitOMPSimdInit(S, /*IsMonotonic=*/true);
  3153. CGOpenMPRuntime::StaticRTInput StaticInit(
  3154. IVSize, IVSigned, /* Ordered = */ false, IL.getAddress(),
  3155. LB.getAddress(), UB.getAddress(), ST.getAddress());
  3156. RT.emitDistributeStaticInit(*this, S.getBeginLoc(), ScheduleKind,
  3157. StaticInit);
  3158. JumpDest LoopExit =
  3159. getJumpDestInCurrentScope(createBasicBlock("omp.loop.exit"));
  3160. // UB = min(UB, GlobalUB);
  3161. EmitIgnoredExpr(isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
  3162. ? S.getCombinedEnsureUpperBound()
  3163. : S.getEnsureUpperBound());
  3164. // IV = LB;
  3165. EmitIgnoredExpr(isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
  3166. ? S.getCombinedInit()
  3167. : S.getInit());
  3168. const Expr *Cond =
  3169. isOpenMPLoopBoundSharingDirective(S.getDirectiveKind())
  3170. ? S.getCombinedCond()
  3171. : S.getCond();
  3172. // for distribute alone, codegen
  3173. // while (idx <= UB) { BODY; ++idx; }
  3174. // when combined with 'for' (e.g. as in 'distribute parallel for')
  3175. // while (idx <= UB) { <CodeGen rest of pragma>; idx += ST; }
  3176. EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), Cond, IncExpr,
  3177. [&S, LoopExit, &CodeGenLoop](CodeGenFunction &CGF) {
  3178. CodeGenLoop(CGF, S, LoopExit);
  3179. },
  3180. [](CodeGenFunction &) {});
  3181. EmitBlock(LoopExit.getBlock());
  3182. // Tell the runtime we are done.
  3183. RT.emitForStaticFinish(*this, S.getBeginLoc(), S.getDirectiveKind());
  3184. } else {
  3185. // Emit the outer loop, which requests its work chunk [LB..UB] from
  3186. // runtime and runs the inner loop to process it.
  3187. const OMPLoopArguments LoopArguments = {
  3188. LB.getAddress(), UB.getAddress(), ST.getAddress(), IL.getAddress(),
  3189. Chunk};
  3190. EmitOMPDistributeOuterLoop(ScheduleKind, S, LoopScope, LoopArguments,
  3191. CodeGenLoop);
  3192. }
  3193. if (isOpenMPSimdDirective(S.getDirectiveKind())) {
  3194. EmitOMPSimdFinal(S, [IL, &S](CodeGenFunction &CGF) {
  3195. return CGF.Builder.CreateIsNotNull(
  3196. CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
  3197. });
  3198. }
  3199. if (isOpenMPSimdDirective(S.getDirectiveKind()) &&
  3200. !isOpenMPParallelDirective(S.getDirectiveKind()) &&
  3201. !isOpenMPTeamsDirective(S.getDirectiveKind())) {
  3202. OpenMPDirectiveKind ReductionKind = OMPD_unknown;
  3203. if (isOpenMPParallelDirective(S.getDirectiveKind()) &&
  3204. isOpenMPSimdDirective(S.getDirectiveKind())) {
  3205. ReductionKind = OMPD_parallel_for_simd;
  3206. } else if (isOpenMPParallelDirective(S.getDirectiveKind())) {
  3207. ReductionKind = OMPD_parallel_for;
  3208. } else if (isOpenMPSimdDirective(S.getDirectiveKind())) {
  3209. ReductionKind = OMPD_simd;
  3210. } else if (!isOpenMPTeamsDirective(S.getDirectiveKind()) &&
  3211. S.hasClausesOfKind<OMPReductionClause>()) {
  3212. llvm_unreachable(
  3213. "No reduction clauses is allowed in distribute directive.");
  3214. }
  3215. EmitOMPReductionClauseFinal(S, ReductionKind);
  3216. // Emit post-update of the reduction variables if IsLastIter != 0.
  3217. emitPostUpdateForReductionClause(
  3218. *this, S, [IL, &S](CodeGenFunction &CGF) {
  3219. return CGF.Builder.CreateIsNotNull(
  3220. CGF.EmitLoadOfScalar(IL, S.getBeginLoc()));
  3221. });
  3222. }
  3223. // Emit final copy of the lastprivate variables if IsLastIter != 0.
  3224. if (HasLastprivateClause) {
  3225. EmitOMPLastprivateClauseFinal(
  3226. S, /*NoFinals=*/false,
  3227. Builder.CreateIsNotNull(EmitLoadOfScalar(IL, S.getBeginLoc())));
  3228. }
  3229. }
  3230. // We're now done with the loop, so jump to the continuation block.
  3231. if (ContBlock) {
  3232. EmitBranch(ContBlock);
  3233. EmitBlock(ContBlock, true);
  3234. }
  3235. }
  3236. }
  3237. void CodeGenFunction::EmitOMPDistributeDirective(
  3238. const OMPDistributeDirective &S) {
  3239. auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
  3240. CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc());
  3241. };
  3242. OMPLexicalScope Scope(*this, S, OMPD_unknown);
  3243. CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_distribute, CodeGen);
  3244. }
  3245. static llvm::Function *emitOutlinedOrderedFunction(CodeGenModule &CGM,
  3246. const CapturedStmt *S) {
  3247. CodeGenFunction CGF(CGM, /*suppressNewContext=*/true);
  3248. CodeGenFunction::CGCapturedStmtInfo CapStmtInfo;
  3249. CGF.CapturedStmtInfo = &CapStmtInfo;
  3250. llvm::Function *Fn = CGF.GenerateOpenMPCapturedStmtFunction(*S);
  3251. Fn->setDoesNotRecurse();
  3252. return Fn;
  3253. }
  3254. void CodeGenFunction::EmitOMPOrderedDirective(const OMPOrderedDirective &S) {
  3255. if (S.hasClausesOfKind<OMPDependClause>()) {
  3256. assert(!S.getAssociatedStmt() &&
  3257. "No associated statement must be in ordered depend construct.");
  3258. for (const auto *DC : S.getClausesOfKind<OMPDependClause>())
  3259. CGM.getOpenMPRuntime().emitDoacrossOrdered(*this, DC);
  3260. return;
  3261. }
  3262. const auto *C = S.getSingleClause<OMPSIMDClause>();
  3263. auto &&CodeGen = [&S, C, this](CodeGenFunction &CGF,
  3264. PrePostActionTy &Action) {
  3265. const CapturedStmt *CS = S.getInnermostCapturedStmt();
  3266. if (C) {
  3267. llvm::SmallVector<llvm::Value *, 16> CapturedVars;
  3268. CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
  3269. llvm::Function *OutlinedFn = emitOutlinedOrderedFunction(CGM, CS);
  3270. CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF, S.getBeginLoc(),
  3271. OutlinedFn, CapturedVars);
  3272. } else {
  3273. Action.Enter(CGF);
  3274. CGF.EmitStmt(CS->getCapturedStmt());
  3275. }
  3276. };
  3277. OMPLexicalScope Scope(*this, S, OMPD_unknown);
  3278. CGM.getOpenMPRuntime().emitOrderedRegion(*this, CodeGen, S.getBeginLoc(), !C);
  3279. }
  3280. static llvm::Value *convertToScalarValue(CodeGenFunction &CGF, RValue Val,
  3281. QualType SrcType, QualType DestType,
  3282. SourceLocation Loc) {
  3283. assert(CGF.hasScalarEvaluationKind(DestType) &&
  3284. "DestType must have scalar evaluation kind.");
  3285. assert(!Val.isAggregate() && "Must be a scalar or complex.");
  3286. return Val.isScalar() ? CGF.EmitScalarConversion(Val.getScalarVal(), SrcType,
  3287. DestType, Loc)
  3288. : CGF.EmitComplexToScalarConversion(
  3289. Val.getComplexVal(), SrcType, DestType, Loc);
  3290. }
  3291. static CodeGenFunction::ComplexPairTy
  3292. convertToComplexValue(CodeGenFunction &CGF, RValue Val, QualType SrcType,
  3293. QualType DestType, SourceLocation Loc) {
  3294. assert(CGF.getEvaluationKind(DestType) == TEK_Complex &&
  3295. "DestType must have complex evaluation kind.");
  3296. CodeGenFunction::ComplexPairTy ComplexVal;
  3297. if (Val.isScalar()) {
  3298. // Convert the input element to the element type of the complex.
  3299. QualType DestElementType =
  3300. DestType->castAs<ComplexType>()->getElementType();
  3301. llvm::Value *ScalarVal = CGF.EmitScalarConversion(
  3302. Val.getScalarVal(), SrcType, DestElementType, Loc);
  3303. ComplexVal = CodeGenFunction::ComplexPairTy(
  3304. ScalarVal, llvm::Constant::getNullValue(ScalarVal->getType()));
  3305. } else {
  3306. assert(Val.isComplex() && "Must be a scalar or complex.");
  3307. QualType SrcElementType = SrcType->castAs<ComplexType>()->getElementType();
  3308. QualType DestElementType =
  3309. DestType->castAs<ComplexType>()->getElementType();
  3310. ComplexVal.first = CGF.EmitScalarConversion(
  3311. Val.getComplexVal().first, SrcElementType, DestElementType, Loc);
  3312. ComplexVal.second = CGF.EmitScalarConversion(
  3313. Val.getComplexVal().second, SrcElementType, DestElementType, Loc);
  3314. }
  3315. return ComplexVal;
  3316. }
  3317. static void emitSimpleAtomicStore(CodeGenFunction &CGF, bool IsSeqCst,
  3318. LValue LVal, RValue RVal) {
  3319. if (LVal.isGlobalReg()) {
  3320. CGF.EmitStoreThroughGlobalRegLValue(RVal, LVal);
  3321. } else {
  3322. CGF.EmitAtomicStore(RVal, LVal,
  3323. IsSeqCst ? llvm::AtomicOrdering::SequentiallyConsistent
  3324. : llvm::AtomicOrdering::Monotonic,
  3325. LVal.isVolatile(), /*IsInit=*/false);
  3326. }
  3327. }
  3328. void CodeGenFunction::emitOMPSimpleStore(LValue LVal, RValue RVal,
  3329. QualType RValTy, SourceLocation Loc) {
  3330. switch (getEvaluationKind(LVal.getType())) {
  3331. case TEK_Scalar:
  3332. EmitStoreThroughLValue(RValue::get(convertToScalarValue(
  3333. *this, RVal, RValTy, LVal.getType(), Loc)),
  3334. LVal);
  3335. break;
  3336. case TEK_Complex:
  3337. EmitStoreOfComplex(
  3338. convertToComplexValue(*this, RVal, RValTy, LVal.getType(), Loc), LVal,
  3339. /*isInit=*/false);
  3340. break;
  3341. case TEK_Aggregate:
  3342. llvm_unreachable("Must be a scalar or complex.");
  3343. }
  3344. }
  3345. static void emitOMPAtomicReadExpr(CodeGenFunction &CGF, bool IsSeqCst,
  3346. const Expr *X, const Expr *V,
  3347. SourceLocation Loc) {
  3348. // v = x;
  3349. assert(V->isLValue() && "V of 'omp atomic read' is not lvalue");
  3350. assert(X->isLValue() && "X of 'omp atomic read' is not lvalue");
  3351. LValue XLValue = CGF.EmitLValue(X);
  3352. LValue VLValue = CGF.EmitLValue(V);
  3353. RValue Res = XLValue.isGlobalReg()
  3354. ? CGF.EmitLoadOfLValue(XLValue, Loc)
  3355. : CGF.EmitAtomicLoad(
  3356. XLValue, Loc,
  3357. IsSeqCst ? llvm::AtomicOrdering::SequentiallyConsistent
  3358. : llvm::AtomicOrdering::Monotonic,
  3359. XLValue.isVolatile());
  3360. // OpenMP, 2.12.6, atomic Construct
  3361. // Any atomic construct with a seq_cst clause forces the atomically
  3362. // performed operation to include an implicit flush operation without a
  3363. // list.
  3364. if (IsSeqCst)
  3365. CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
  3366. CGF.emitOMPSimpleStore(VLValue, Res, X->getType().getNonReferenceType(), Loc);
  3367. }
  3368. static void emitOMPAtomicWriteExpr(CodeGenFunction &CGF, bool IsSeqCst,
  3369. const Expr *X, const Expr *E,
  3370. SourceLocation Loc) {
  3371. // x = expr;
  3372. assert(X->isLValue() && "X of 'omp atomic write' is not lvalue");
  3373. emitSimpleAtomicStore(CGF, IsSeqCst, CGF.EmitLValue(X), CGF.EmitAnyExpr(E));
  3374. // OpenMP, 2.12.6, atomic Construct
  3375. // Any atomic construct with a seq_cst clause forces the atomically
  3376. // performed operation to include an implicit flush operation without a
  3377. // list.
  3378. if (IsSeqCst)
  3379. CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
  3380. }
  3381. static std::pair<bool, RValue> emitOMPAtomicRMW(CodeGenFunction &CGF, LValue X,
  3382. RValue Update,
  3383. BinaryOperatorKind BO,
  3384. llvm::AtomicOrdering AO,
  3385. bool IsXLHSInRHSPart) {
  3386. ASTContext &Context = CGF.getContext();
  3387. // Allow atomicrmw only if 'x' and 'update' are integer values, lvalue for 'x'
  3388. // expression is simple and atomic is allowed for the given type for the
  3389. // target platform.
  3390. if (BO == BO_Comma || !Update.isScalar() ||
  3391. !Update.getScalarVal()->getType()->isIntegerTy() ||
  3392. !X.isSimple() || (!isa<llvm::ConstantInt>(Update.getScalarVal()) &&
  3393. (Update.getScalarVal()->getType() !=
  3394. X.getAddress().getElementType())) ||
  3395. !X.getAddress().getElementType()->isIntegerTy() ||
  3396. !Context.getTargetInfo().hasBuiltinAtomic(
  3397. Context.getTypeSize(X.getType()), Context.toBits(X.getAlignment())))
  3398. return std::make_pair(false, RValue::get(nullptr));
  3399. llvm::AtomicRMWInst::BinOp RMWOp;
  3400. switch (BO) {
  3401. case BO_Add:
  3402. RMWOp = llvm::AtomicRMWInst::Add;
  3403. break;
  3404. case BO_Sub:
  3405. if (!IsXLHSInRHSPart)
  3406. return std::make_pair(false, RValue::get(nullptr));
  3407. RMWOp = llvm::AtomicRMWInst::Sub;
  3408. break;
  3409. case BO_And:
  3410. RMWOp = llvm::AtomicRMWInst::And;
  3411. break;
  3412. case BO_Or:
  3413. RMWOp = llvm::AtomicRMWInst::Or;
  3414. break;
  3415. case BO_Xor:
  3416. RMWOp = llvm::AtomicRMWInst::Xor;
  3417. break;
  3418. case BO_LT:
  3419. RMWOp = X.getType()->hasSignedIntegerRepresentation()
  3420. ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Min
  3421. : llvm::AtomicRMWInst::Max)
  3422. : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMin
  3423. : llvm::AtomicRMWInst::UMax);
  3424. break;
  3425. case BO_GT:
  3426. RMWOp = X.getType()->hasSignedIntegerRepresentation()
  3427. ? (IsXLHSInRHSPart ? llvm::AtomicRMWInst::Max
  3428. : llvm::AtomicRMWInst::Min)
  3429. : (IsXLHSInRHSPart ? llvm::AtomicRMWInst::UMax
  3430. : llvm::AtomicRMWInst::UMin);
  3431. break;
  3432. case BO_Assign:
  3433. RMWOp = llvm::AtomicRMWInst::Xchg;
  3434. break;
  3435. case BO_Mul:
  3436. case BO_Div:
  3437. case BO_Rem:
  3438. case BO_Shl:
  3439. case BO_Shr:
  3440. case BO_LAnd:
  3441. case BO_LOr:
  3442. return std::make_pair(false, RValue::get(nullptr));
  3443. case BO_PtrMemD:
  3444. case BO_PtrMemI:
  3445. case BO_LE:
  3446. case BO_GE:
  3447. case BO_EQ:
  3448. case BO_NE:
  3449. case BO_Cmp:
  3450. case BO_AddAssign:
  3451. case BO_SubAssign:
  3452. case BO_AndAssign:
  3453. case BO_OrAssign:
  3454. case BO_XorAssign:
  3455. case BO_MulAssign:
  3456. case BO_DivAssign:
  3457. case BO_RemAssign:
  3458. case BO_ShlAssign:
  3459. case BO_ShrAssign:
  3460. case BO_Comma:
  3461. llvm_unreachable("Unsupported atomic update operation");
  3462. }
  3463. llvm::Value *UpdateVal = Update.getScalarVal();
  3464. if (auto *IC = dyn_cast<llvm::ConstantInt>(UpdateVal)) {
  3465. UpdateVal = CGF.Builder.CreateIntCast(
  3466. IC, X.getAddress().getElementType(),
  3467. X.getType()->hasSignedIntegerRepresentation());
  3468. }
  3469. llvm::Value *Res =
  3470. CGF.Builder.CreateAtomicRMW(RMWOp, X.getPointer(), UpdateVal, AO);
  3471. return std::make_pair(true, RValue::get(Res));
  3472. }
  3473. std::pair<bool, RValue> CodeGenFunction::EmitOMPAtomicSimpleUpdateExpr(
  3474. LValue X, RValue E, BinaryOperatorKind BO, bool IsXLHSInRHSPart,
  3475. llvm::AtomicOrdering AO, SourceLocation Loc,
  3476. const llvm::function_ref<RValue(RValue)> CommonGen) {
  3477. // Update expressions are allowed to have the following forms:
  3478. // x binop= expr; -> xrval + expr;
  3479. // x++, ++x -> xrval + 1;
  3480. // x--, --x -> xrval - 1;
  3481. // x = x binop expr; -> xrval binop expr
  3482. // x = expr Op x; - > expr binop xrval;
  3483. auto Res = emitOMPAtomicRMW(*this, X, E, BO, AO, IsXLHSInRHSPart);
  3484. if (!Res.first) {
  3485. if (X.isGlobalReg()) {
  3486. // Emit an update expression: 'xrval' binop 'expr' or 'expr' binop
  3487. // 'xrval'.
  3488. EmitStoreThroughLValue(CommonGen(EmitLoadOfLValue(X, Loc)), X);
  3489. } else {
  3490. // Perform compare-and-swap procedure.
  3491. EmitAtomicUpdate(X, AO, CommonGen, X.getType().isVolatileQualified());
  3492. }
  3493. }
  3494. return Res;
  3495. }
  3496. static void emitOMPAtomicUpdateExpr(CodeGenFunction &CGF, bool IsSeqCst,
  3497. const Expr *X, const Expr *E,
  3498. const Expr *UE, bool IsXLHSInRHSPart,
  3499. SourceLocation Loc) {
  3500. assert(isa<BinaryOperator>(UE->IgnoreImpCasts()) &&
  3501. "Update expr in 'atomic update' must be a binary operator.");
  3502. const auto *BOUE = cast<BinaryOperator>(UE->IgnoreImpCasts());
  3503. // Update expressions are allowed to have the following forms:
  3504. // x binop= expr; -> xrval + expr;
  3505. // x++, ++x -> xrval + 1;
  3506. // x--, --x -> xrval - 1;
  3507. // x = x binop expr; -> xrval binop expr
  3508. // x = expr Op x; - > expr binop xrval;
  3509. assert(X->isLValue() && "X of 'omp atomic update' is not lvalue");
  3510. LValue XLValue = CGF.EmitLValue(X);
  3511. RValue ExprRValue = CGF.EmitAnyExpr(E);
  3512. llvm::AtomicOrdering AO = IsSeqCst
  3513. ? llvm::AtomicOrdering::SequentiallyConsistent
  3514. : llvm::AtomicOrdering::Monotonic;
  3515. const auto *LHS = cast<OpaqueValueExpr>(BOUE->getLHS()->IgnoreImpCasts());
  3516. const auto *RHS = cast<OpaqueValueExpr>(BOUE->getRHS()->IgnoreImpCasts());
  3517. const OpaqueValueExpr *XRValExpr = IsXLHSInRHSPart ? LHS : RHS;
  3518. const OpaqueValueExpr *ERValExpr = IsXLHSInRHSPart ? RHS : LHS;
  3519. auto &&Gen = [&CGF, UE, ExprRValue, XRValExpr, ERValExpr](RValue XRValue) {
  3520. CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
  3521. CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue);
  3522. return CGF.EmitAnyExpr(UE);
  3523. };
  3524. (void)CGF.EmitOMPAtomicSimpleUpdateExpr(
  3525. XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, Gen);
  3526. // OpenMP, 2.12.6, atomic Construct
  3527. // Any atomic construct with a seq_cst clause forces the atomically
  3528. // performed operation to include an implicit flush operation without a
  3529. // list.
  3530. if (IsSeqCst)
  3531. CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
  3532. }
  3533. static RValue convertToType(CodeGenFunction &CGF, RValue Value,
  3534. QualType SourceType, QualType ResType,
  3535. SourceLocation Loc) {
  3536. switch (CGF.getEvaluationKind(ResType)) {
  3537. case TEK_Scalar:
  3538. return RValue::get(
  3539. convertToScalarValue(CGF, Value, SourceType, ResType, Loc));
  3540. case TEK_Complex: {
  3541. auto Res = convertToComplexValue(CGF, Value, SourceType, ResType, Loc);
  3542. return RValue::getComplex(Res.first, Res.second);
  3543. }
  3544. case TEK_Aggregate:
  3545. break;
  3546. }
  3547. llvm_unreachable("Must be a scalar or complex.");
  3548. }
  3549. static void emitOMPAtomicCaptureExpr(CodeGenFunction &CGF, bool IsSeqCst,
  3550. bool IsPostfixUpdate, const Expr *V,
  3551. const Expr *X, const Expr *E,
  3552. const Expr *UE, bool IsXLHSInRHSPart,
  3553. SourceLocation Loc) {
  3554. assert(X->isLValue() && "X of 'omp atomic capture' is not lvalue");
  3555. assert(V->isLValue() && "V of 'omp atomic capture' is not lvalue");
  3556. RValue NewVVal;
  3557. LValue VLValue = CGF.EmitLValue(V);
  3558. LValue XLValue = CGF.EmitLValue(X);
  3559. RValue ExprRValue = CGF.EmitAnyExpr(E);
  3560. llvm::AtomicOrdering AO = IsSeqCst
  3561. ? llvm::AtomicOrdering::SequentiallyConsistent
  3562. : llvm::AtomicOrdering::Monotonic;
  3563. QualType NewVValType;
  3564. if (UE) {
  3565. // 'x' is updated with some additional value.
  3566. assert(isa<BinaryOperator>(UE->IgnoreImpCasts()) &&
  3567. "Update expr in 'atomic capture' must be a binary operator.");
  3568. const auto *BOUE = cast<BinaryOperator>(UE->IgnoreImpCasts());
  3569. // Update expressions are allowed to have the following forms:
  3570. // x binop= expr; -> xrval + expr;
  3571. // x++, ++x -> xrval + 1;
  3572. // x--, --x -> xrval - 1;
  3573. // x = x binop expr; -> xrval binop expr
  3574. // x = expr Op x; - > expr binop xrval;
  3575. const auto *LHS = cast<OpaqueValueExpr>(BOUE->getLHS()->IgnoreImpCasts());
  3576. const auto *RHS = cast<OpaqueValueExpr>(BOUE->getRHS()->IgnoreImpCasts());
  3577. const OpaqueValueExpr *XRValExpr = IsXLHSInRHSPart ? LHS : RHS;
  3578. NewVValType = XRValExpr->getType();
  3579. const OpaqueValueExpr *ERValExpr = IsXLHSInRHSPart ? RHS : LHS;
  3580. auto &&Gen = [&CGF, &NewVVal, UE, ExprRValue, XRValExpr, ERValExpr,
  3581. IsPostfixUpdate](RValue XRValue) {
  3582. CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
  3583. CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, XRValue);
  3584. RValue Res = CGF.EmitAnyExpr(UE);
  3585. NewVVal = IsPostfixUpdate ? XRValue : Res;
  3586. return Res;
  3587. };
  3588. auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr(
  3589. XLValue, ExprRValue, BOUE->getOpcode(), IsXLHSInRHSPart, AO, Loc, Gen);
  3590. if (Res.first) {
  3591. // 'atomicrmw' instruction was generated.
  3592. if (IsPostfixUpdate) {
  3593. // Use old value from 'atomicrmw'.
  3594. NewVVal = Res.second;
  3595. } else {
  3596. // 'atomicrmw' does not provide new value, so evaluate it using old
  3597. // value of 'x'.
  3598. CodeGenFunction::OpaqueValueMapping MapExpr(CGF, ERValExpr, ExprRValue);
  3599. CodeGenFunction::OpaqueValueMapping MapX(CGF, XRValExpr, Res.second);
  3600. NewVVal = CGF.EmitAnyExpr(UE);
  3601. }
  3602. }
  3603. } else {
  3604. // 'x' is simply rewritten with some 'expr'.
  3605. NewVValType = X->getType().getNonReferenceType();
  3606. ExprRValue = convertToType(CGF, ExprRValue, E->getType(),
  3607. X->getType().getNonReferenceType(), Loc);
  3608. auto &&Gen = [&NewVVal, ExprRValue](RValue XRValue) {
  3609. NewVVal = XRValue;
  3610. return ExprRValue;
  3611. };
  3612. // Try to perform atomicrmw xchg, otherwise simple exchange.
  3613. auto Res = CGF.EmitOMPAtomicSimpleUpdateExpr(
  3614. XLValue, ExprRValue, /*BO=*/BO_Assign, /*IsXLHSInRHSPart=*/false, AO,
  3615. Loc, Gen);
  3616. if (Res.first) {
  3617. // 'atomicrmw' instruction was generated.
  3618. NewVVal = IsPostfixUpdate ? Res.second : ExprRValue;
  3619. }
  3620. }
  3621. // Emit post-update store to 'v' of old/new 'x' value.
  3622. CGF.emitOMPSimpleStore(VLValue, NewVVal, NewVValType, Loc);
  3623. // OpenMP, 2.12.6, atomic Construct
  3624. // Any atomic construct with a seq_cst clause forces the atomically
  3625. // performed operation to include an implicit flush operation without a
  3626. // list.
  3627. if (IsSeqCst)
  3628. CGF.CGM.getOpenMPRuntime().emitFlush(CGF, llvm::None, Loc);
  3629. }
  3630. static void emitOMPAtomicExpr(CodeGenFunction &CGF, OpenMPClauseKind Kind,
  3631. bool IsSeqCst, bool IsPostfixUpdate,
  3632. const Expr *X, const Expr *V, const Expr *E,
  3633. const Expr *UE, bool IsXLHSInRHSPart,
  3634. SourceLocation Loc) {
  3635. switch (Kind) {
  3636. case OMPC_read:
  3637. emitOMPAtomicReadExpr(CGF, IsSeqCst, X, V, Loc);
  3638. break;
  3639. case OMPC_write:
  3640. emitOMPAtomicWriteExpr(CGF, IsSeqCst, X, E, Loc);
  3641. break;
  3642. case OMPC_unknown:
  3643. case OMPC_update:
  3644. emitOMPAtomicUpdateExpr(CGF, IsSeqCst, X, E, UE, IsXLHSInRHSPart, Loc);
  3645. break;
  3646. case OMPC_capture:
  3647. emitOMPAtomicCaptureExpr(CGF, IsSeqCst, IsPostfixUpdate, V, X, E, UE,
  3648. IsXLHSInRHSPart, Loc);
  3649. break;
  3650. case OMPC_if:
  3651. case OMPC_final:
  3652. case OMPC_num_threads:
  3653. case OMPC_private:
  3654. case OMPC_firstprivate:
  3655. case OMPC_lastprivate:
  3656. case OMPC_reduction:
  3657. case OMPC_task_reduction:
  3658. case OMPC_in_reduction:
  3659. case OMPC_safelen:
  3660. case OMPC_simdlen:
  3661. case OMPC_collapse:
  3662. case OMPC_default:
  3663. case OMPC_seq_cst:
  3664. case OMPC_shared:
  3665. case OMPC_linear:
  3666. case OMPC_aligned:
  3667. case OMPC_copyin:
  3668. case OMPC_copyprivate:
  3669. case OMPC_flush:
  3670. case OMPC_proc_bind:
  3671. case OMPC_schedule:
  3672. case OMPC_ordered:
  3673. case OMPC_nowait:
  3674. case OMPC_untied:
  3675. case OMPC_threadprivate:
  3676. case OMPC_depend:
  3677. case OMPC_mergeable:
  3678. case OMPC_device:
  3679. case OMPC_threads:
  3680. case OMPC_simd:
  3681. case OMPC_map:
  3682. case OMPC_num_teams:
  3683. case OMPC_thread_limit:
  3684. case OMPC_priority:
  3685. case OMPC_grainsize:
  3686. case OMPC_nogroup:
  3687. case OMPC_num_tasks:
  3688. case OMPC_hint:
  3689. case OMPC_dist_schedule:
  3690. case OMPC_defaultmap:
  3691. case OMPC_uniform:
  3692. case OMPC_to:
  3693. case OMPC_from:
  3694. case OMPC_use_device_ptr:
  3695. case OMPC_is_device_ptr:
  3696. case OMPC_unified_address:
  3697. llvm_unreachable("Clause is not allowed in 'omp atomic'.");
  3698. }
  3699. }
  3700. void CodeGenFunction::EmitOMPAtomicDirective(const OMPAtomicDirective &S) {
  3701. bool IsSeqCst = S.getSingleClause<OMPSeqCstClause>();
  3702. OpenMPClauseKind Kind = OMPC_unknown;
  3703. for (const OMPClause *C : S.clauses()) {
  3704. // Find first clause (skip seq_cst clause, if it is first).
  3705. if (C->getClauseKind() != OMPC_seq_cst) {
  3706. Kind = C->getClauseKind();
  3707. break;
  3708. }
  3709. }
  3710. const Stmt *CS = S.getInnermostCapturedStmt()->IgnoreContainers();
  3711. if (const auto *EWC = dyn_cast<ExprWithCleanups>(CS))
  3712. enterFullExpression(EWC);
  3713. // Processing for statements under 'atomic capture'.
  3714. if (const auto *Compound = dyn_cast<CompoundStmt>(CS)) {
  3715. for (const Stmt *C : Compound->body()) {
  3716. if (const auto *EWC = dyn_cast<ExprWithCleanups>(C))
  3717. enterFullExpression(EWC);
  3718. }
  3719. }
  3720. auto &&CodeGen = [&S, Kind, IsSeqCst, CS](CodeGenFunction &CGF,
  3721. PrePostActionTy &) {
  3722. CGF.EmitStopPoint(CS);
  3723. emitOMPAtomicExpr(CGF, Kind, IsSeqCst, S.isPostfixUpdate(), S.getX(),
  3724. S.getV(), S.getExpr(), S.getUpdateExpr(),
  3725. S.isXLHSInRHSPart(), S.getBeginLoc());
  3726. };
  3727. OMPLexicalScope Scope(*this, S, OMPD_unknown);
  3728. CGM.getOpenMPRuntime().emitInlinedDirective(*this, OMPD_atomic, CodeGen);
  3729. }
  3730. static void emitCommonOMPTargetDirective(CodeGenFunction &CGF,
  3731. const OMPExecutableDirective &S,
  3732. const RegionCodeGenTy &CodeGen) {
  3733. assert(isOpenMPTargetExecutionDirective(S.getDirectiveKind()));
  3734. CodeGenModule &CGM = CGF.CGM;
  3735. // On device emit this construct as inlined code.
  3736. if (CGM.getLangOpts().OpenMPIsDevice) {
  3737. OMPLexicalScope Scope(CGF, S, OMPD_target);
  3738. CGM.getOpenMPRuntime().emitInlinedDirective(
  3739. CGF, OMPD_target, [&S](CodeGenFunction &CGF, PrePostActionTy &) {
  3740. CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
  3741. });
  3742. return;
  3743. }
  3744. llvm::Function *Fn = nullptr;
  3745. llvm::Constant *FnID = nullptr;
  3746. const Expr *IfCond = nullptr;
  3747. // Check for the at most one if clause associated with the target region.
  3748. for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
  3749. if (C->getNameModifier() == OMPD_unknown ||
  3750. C->getNameModifier() == OMPD_target) {
  3751. IfCond = C->getCondition();
  3752. break;
  3753. }
  3754. }
  3755. // Check if we have any device clause associated with the directive.
  3756. const Expr *Device = nullptr;
  3757. if (auto *C = S.getSingleClause<OMPDeviceClause>())
  3758. Device = C->getDevice();
  3759. // Check if we have an if clause whose conditional always evaluates to false
  3760. // or if we do not have any targets specified. If so the target region is not
  3761. // an offload entry point.
  3762. bool IsOffloadEntry = true;
  3763. if (IfCond) {
  3764. bool Val;
  3765. if (CGF.ConstantFoldsToSimpleInteger(IfCond, Val) && !Val)
  3766. IsOffloadEntry = false;
  3767. }
  3768. if (CGM.getLangOpts().OMPTargetTriples.empty())
  3769. IsOffloadEntry = false;
  3770. assert(CGF.CurFuncDecl && "No parent declaration for target region!");
  3771. StringRef ParentName;
  3772. // In case we have Ctors/Dtors we use the complete type variant to produce
  3773. // the mangling of the device outlined kernel.
  3774. if (const auto *D = dyn_cast<CXXConstructorDecl>(CGF.CurFuncDecl))
  3775. ParentName = CGM.getMangledName(GlobalDecl(D, Ctor_Complete));
  3776. else if (const auto *D = dyn_cast<CXXDestructorDecl>(CGF.CurFuncDecl))
  3777. ParentName = CGM.getMangledName(GlobalDecl(D, Dtor_Complete));
  3778. else
  3779. ParentName =
  3780. CGM.getMangledName(GlobalDecl(cast<FunctionDecl>(CGF.CurFuncDecl)));
  3781. // Emit target region as a standalone region.
  3782. CGM.getOpenMPRuntime().emitTargetOutlinedFunction(S, ParentName, Fn, FnID,
  3783. IsOffloadEntry, CodeGen);
  3784. OMPLexicalScope Scope(CGF, S, OMPD_task);
  3785. CGM.getOpenMPRuntime().emitTargetCall(CGF, S, Fn, FnID, IfCond, Device);
  3786. }
  3787. static void emitTargetRegion(CodeGenFunction &CGF, const OMPTargetDirective &S,
  3788. PrePostActionTy &Action) {
  3789. Action.Enter(CGF);
  3790. CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
  3791. (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
  3792. CGF.EmitOMPPrivateClause(S, PrivateScope);
  3793. (void)PrivateScope.Privatize();
  3794. CGF.EmitStmt(S.getCapturedStmt(OMPD_target)->getCapturedStmt());
  3795. }
  3796. void CodeGenFunction::EmitOMPTargetDeviceFunction(CodeGenModule &CGM,
  3797. StringRef ParentName,
  3798. const OMPTargetDirective &S) {
  3799. auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
  3800. emitTargetRegion(CGF, S, Action);
  3801. };
  3802. llvm::Function *Fn;
  3803. llvm::Constant *Addr;
  3804. // Emit target region as a standalone region.
  3805. CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
  3806. S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
  3807. assert(Fn && Addr && "Target device function emission failed.");
  3808. }
  3809. void CodeGenFunction::EmitOMPTargetDirective(const OMPTargetDirective &S) {
  3810. auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
  3811. emitTargetRegion(CGF, S, Action);
  3812. };
  3813. emitCommonOMPTargetDirective(*this, S, CodeGen);
  3814. }
  3815. static void emitCommonOMPTeamsDirective(CodeGenFunction &CGF,
  3816. const OMPExecutableDirective &S,
  3817. OpenMPDirectiveKind InnermostKind,
  3818. const RegionCodeGenTy &CodeGen) {
  3819. const CapturedStmt *CS = S.getCapturedStmt(OMPD_teams);
  3820. llvm::Value *OutlinedFn =
  3821. CGF.CGM.getOpenMPRuntime().emitTeamsOutlinedFunction(
  3822. S, *CS->getCapturedDecl()->param_begin(), InnermostKind, CodeGen);
  3823. const auto *NT = S.getSingleClause<OMPNumTeamsClause>();
  3824. const auto *TL = S.getSingleClause<OMPThreadLimitClause>();
  3825. if (NT || TL) {
  3826. const Expr *NumTeams = NT ? NT->getNumTeams() : nullptr;
  3827. const Expr *ThreadLimit = TL ? TL->getThreadLimit() : nullptr;
  3828. CGF.CGM.getOpenMPRuntime().emitNumTeamsClause(CGF, NumTeams, ThreadLimit,
  3829. S.getBeginLoc());
  3830. }
  3831. OMPTeamsScope Scope(CGF, S);
  3832. llvm::SmallVector<llvm::Value *, 16> CapturedVars;
  3833. CGF.GenerateOpenMPCapturedVars(*CS, CapturedVars);
  3834. CGF.CGM.getOpenMPRuntime().emitTeamsCall(CGF, S, S.getBeginLoc(), OutlinedFn,
  3835. CapturedVars);
  3836. }
  3837. void CodeGenFunction::EmitOMPTeamsDirective(const OMPTeamsDirective &S) {
  3838. // Emit teams region as a standalone region.
  3839. auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
  3840. Action.Enter(CGF);
  3841. OMPPrivateScope PrivateScope(CGF);
  3842. (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
  3843. CGF.EmitOMPPrivateClause(S, PrivateScope);
  3844. CGF.EmitOMPReductionClauseInit(S, PrivateScope);
  3845. (void)PrivateScope.Privatize();
  3846. CGF.EmitStmt(S.getCapturedStmt(OMPD_teams)->getCapturedStmt());
  3847. CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
  3848. };
  3849. emitCommonOMPTeamsDirective(*this, S, OMPD_distribute, CodeGen);
  3850. emitPostUpdateForReductionClause(*this, S,
  3851. [](CodeGenFunction &) { return nullptr; });
  3852. }
  3853. static void emitTargetTeamsRegion(CodeGenFunction &CGF, PrePostActionTy &Action,
  3854. const OMPTargetTeamsDirective &S) {
  3855. auto *CS = S.getCapturedStmt(OMPD_teams);
  3856. Action.Enter(CGF);
  3857. // Emit teams region as a standalone region.
  3858. auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &Action) {
  3859. Action.Enter(CGF);
  3860. CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
  3861. (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
  3862. CGF.EmitOMPPrivateClause(S, PrivateScope);
  3863. CGF.EmitOMPReductionClauseInit(S, PrivateScope);
  3864. (void)PrivateScope.Privatize();
  3865. CGF.EmitStmt(CS->getCapturedStmt());
  3866. CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
  3867. };
  3868. emitCommonOMPTeamsDirective(CGF, S, OMPD_teams, CodeGen);
  3869. emitPostUpdateForReductionClause(CGF, S,
  3870. [](CodeGenFunction &) { return nullptr; });
  3871. }
  3872. void CodeGenFunction::EmitOMPTargetTeamsDeviceFunction(
  3873. CodeGenModule &CGM, StringRef ParentName,
  3874. const OMPTargetTeamsDirective &S) {
  3875. auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
  3876. emitTargetTeamsRegion(CGF, Action, S);
  3877. };
  3878. llvm::Function *Fn;
  3879. llvm::Constant *Addr;
  3880. // Emit target region as a standalone region.
  3881. CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
  3882. S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
  3883. assert(Fn && Addr && "Target device function emission failed.");
  3884. }
  3885. void CodeGenFunction::EmitOMPTargetTeamsDirective(
  3886. const OMPTargetTeamsDirective &S) {
  3887. auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
  3888. emitTargetTeamsRegion(CGF, Action, S);
  3889. };
  3890. emitCommonOMPTargetDirective(*this, S, CodeGen);
  3891. }
  3892. static void
  3893. emitTargetTeamsDistributeRegion(CodeGenFunction &CGF, PrePostActionTy &Action,
  3894. const OMPTargetTeamsDistributeDirective &S) {
  3895. Action.Enter(CGF);
  3896. auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
  3897. CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc());
  3898. };
  3899. // Emit teams region as a standalone region.
  3900. auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
  3901. PrePostActionTy &Action) {
  3902. Action.Enter(CGF);
  3903. CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
  3904. CGF.EmitOMPReductionClauseInit(S, PrivateScope);
  3905. (void)PrivateScope.Privatize();
  3906. CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
  3907. CodeGenDistribute);
  3908. CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
  3909. };
  3910. emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute, CodeGen);
  3911. emitPostUpdateForReductionClause(CGF, S,
  3912. [](CodeGenFunction &) { return nullptr; });
  3913. }
  3914. void CodeGenFunction::EmitOMPTargetTeamsDistributeDeviceFunction(
  3915. CodeGenModule &CGM, StringRef ParentName,
  3916. const OMPTargetTeamsDistributeDirective &S) {
  3917. auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
  3918. emitTargetTeamsDistributeRegion(CGF, Action, S);
  3919. };
  3920. llvm::Function *Fn;
  3921. llvm::Constant *Addr;
  3922. // Emit target region as a standalone region.
  3923. CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
  3924. S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
  3925. assert(Fn && Addr && "Target device function emission failed.");
  3926. }
  3927. void CodeGenFunction::EmitOMPTargetTeamsDistributeDirective(
  3928. const OMPTargetTeamsDistributeDirective &S) {
  3929. auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
  3930. emitTargetTeamsDistributeRegion(CGF, Action, S);
  3931. };
  3932. emitCommonOMPTargetDirective(*this, S, CodeGen);
  3933. }
  3934. static void emitTargetTeamsDistributeSimdRegion(
  3935. CodeGenFunction &CGF, PrePostActionTy &Action,
  3936. const OMPTargetTeamsDistributeSimdDirective &S) {
  3937. Action.Enter(CGF);
  3938. auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
  3939. CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc());
  3940. };
  3941. // Emit teams region as a standalone region.
  3942. auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
  3943. PrePostActionTy &Action) {
  3944. Action.Enter(CGF);
  3945. CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
  3946. CGF.EmitOMPReductionClauseInit(S, PrivateScope);
  3947. (void)PrivateScope.Privatize();
  3948. CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
  3949. CodeGenDistribute);
  3950. CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
  3951. };
  3952. emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_simd, CodeGen);
  3953. emitPostUpdateForReductionClause(CGF, S,
  3954. [](CodeGenFunction &) { return nullptr; });
  3955. }
  3956. void CodeGenFunction::EmitOMPTargetTeamsDistributeSimdDeviceFunction(
  3957. CodeGenModule &CGM, StringRef ParentName,
  3958. const OMPTargetTeamsDistributeSimdDirective &S) {
  3959. auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
  3960. emitTargetTeamsDistributeSimdRegion(CGF, Action, S);
  3961. };
  3962. llvm::Function *Fn;
  3963. llvm::Constant *Addr;
  3964. // Emit target region as a standalone region.
  3965. CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
  3966. S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
  3967. assert(Fn && Addr && "Target device function emission failed.");
  3968. }
  3969. void CodeGenFunction::EmitOMPTargetTeamsDistributeSimdDirective(
  3970. const OMPTargetTeamsDistributeSimdDirective &S) {
  3971. auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
  3972. emitTargetTeamsDistributeSimdRegion(CGF, Action, S);
  3973. };
  3974. emitCommonOMPTargetDirective(*this, S, CodeGen);
  3975. }
  3976. void CodeGenFunction::EmitOMPTeamsDistributeDirective(
  3977. const OMPTeamsDistributeDirective &S) {
  3978. auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
  3979. CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc());
  3980. };
  3981. // Emit teams region as a standalone region.
  3982. auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
  3983. PrePostActionTy &Action) {
  3984. Action.Enter(CGF);
  3985. OMPPrivateScope PrivateScope(CGF);
  3986. CGF.EmitOMPReductionClauseInit(S, PrivateScope);
  3987. (void)PrivateScope.Privatize();
  3988. CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
  3989. CodeGenDistribute);
  3990. CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
  3991. };
  3992. emitCommonOMPTeamsDirective(*this, S, OMPD_distribute, CodeGen);
  3993. emitPostUpdateForReductionClause(*this, S,
  3994. [](CodeGenFunction &) { return nullptr; });
  3995. }
  3996. void CodeGenFunction::EmitOMPTeamsDistributeSimdDirective(
  3997. const OMPTeamsDistributeSimdDirective &S) {
  3998. auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
  3999. CGF.EmitOMPDistributeLoop(S, emitOMPLoopBodyWithStopPoint, S.getInc());
  4000. };
  4001. // Emit teams region as a standalone region.
  4002. auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
  4003. PrePostActionTy &Action) {
  4004. Action.Enter(CGF);
  4005. OMPPrivateScope PrivateScope(CGF);
  4006. CGF.EmitOMPReductionClauseInit(S, PrivateScope);
  4007. (void)PrivateScope.Privatize();
  4008. CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_simd,
  4009. CodeGenDistribute);
  4010. CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
  4011. };
  4012. emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_simd, CodeGen);
  4013. emitPostUpdateForReductionClause(*this, S,
  4014. [](CodeGenFunction &) { return nullptr; });
  4015. }
  4016. void CodeGenFunction::EmitOMPTeamsDistributeParallelForDirective(
  4017. const OMPTeamsDistributeParallelForDirective &S) {
  4018. auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
  4019. CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined,
  4020. S.getDistInc());
  4021. };
  4022. // Emit teams region as a standalone region.
  4023. auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
  4024. PrePostActionTy &Action) {
  4025. Action.Enter(CGF);
  4026. OMPPrivateScope PrivateScope(CGF);
  4027. CGF.EmitOMPReductionClauseInit(S, PrivateScope);
  4028. (void)PrivateScope.Privatize();
  4029. CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_distribute,
  4030. CodeGenDistribute);
  4031. CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
  4032. };
  4033. emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_parallel_for, CodeGen);
  4034. emitPostUpdateForReductionClause(*this, S,
  4035. [](CodeGenFunction &) { return nullptr; });
  4036. }
  4037. void CodeGenFunction::EmitOMPTeamsDistributeParallelForSimdDirective(
  4038. const OMPTeamsDistributeParallelForSimdDirective &S) {
  4039. auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
  4040. CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined,
  4041. S.getDistInc());
  4042. };
  4043. // Emit teams region as a standalone region.
  4044. auto &&CodeGen = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
  4045. PrePostActionTy &Action) {
  4046. Action.Enter(CGF);
  4047. OMPPrivateScope PrivateScope(CGF);
  4048. CGF.EmitOMPReductionClauseInit(S, PrivateScope);
  4049. (void)PrivateScope.Privatize();
  4050. CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
  4051. CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);
  4052. CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
  4053. };
  4054. emitCommonOMPTeamsDirective(*this, S, OMPD_distribute_parallel_for, CodeGen);
  4055. emitPostUpdateForReductionClause(*this, S,
  4056. [](CodeGenFunction &) { return nullptr; });
  4057. }
  4058. static void emitTargetTeamsDistributeParallelForRegion(
  4059. CodeGenFunction &CGF, const OMPTargetTeamsDistributeParallelForDirective &S,
  4060. PrePostActionTy &Action) {
  4061. Action.Enter(CGF);
  4062. auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
  4063. CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined,
  4064. S.getDistInc());
  4065. };
  4066. // Emit teams region as a standalone region.
  4067. auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
  4068. PrePostActionTy &Action) {
  4069. Action.Enter(CGF);
  4070. CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
  4071. CGF.EmitOMPReductionClauseInit(S, PrivateScope);
  4072. (void)PrivateScope.Privatize();
  4073. CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
  4074. CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);
  4075. CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
  4076. };
  4077. emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_parallel_for,
  4078. CodeGenTeams);
  4079. emitPostUpdateForReductionClause(CGF, S,
  4080. [](CodeGenFunction &) { return nullptr; });
  4081. }
  4082. void CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForDeviceFunction(
  4083. CodeGenModule &CGM, StringRef ParentName,
  4084. const OMPTargetTeamsDistributeParallelForDirective &S) {
  4085. // Emit SPMD target teams distribute parallel for region as a standalone
  4086. // region.
  4087. auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
  4088. emitTargetTeamsDistributeParallelForRegion(CGF, S, Action);
  4089. };
  4090. llvm::Function *Fn;
  4091. llvm::Constant *Addr;
  4092. // Emit target region as a standalone region.
  4093. CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
  4094. S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
  4095. assert(Fn && Addr && "Target device function emission failed.");
  4096. }
  4097. void CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForDirective(
  4098. const OMPTargetTeamsDistributeParallelForDirective &S) {
  4099. auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
  4100. emitTargetTeamsDistributeParallelForRegion(CGF, S, Action);
  4101. };
  4102. emitCommonOMPTargetDirective(*this, S, CodeGen);
  4103. }
  4104. static void emitTargetTeamsDistributeParallelForSimdRegion(
  4105. CodeGenFunction &CGF,
  4106. const OMPTargetTeamsDistributeParallelForSimdDirective &S,
  4107. PrePostActionTy &Action) {
  4108. Action.Enter(CGF);
  4109. auto &&CodeGenDistribute = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
  4110. CGF.EmitOMPDistributeLoop(S, emitInnerParallelForWhenCombined,
  4111. S.getDistInc());
  4112. };
  4113. // Emit teams region as a standalone region.
  4114. auto &&CodeGenTeams = [&S, &CodeGenDistribute](CodeGenFunction &CGF,
  4115. PrePostActionTy &Action) {
  4116. Action.Enter(CGF);
  4117. CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
  4118. CGF.EmitOMPReductionClauseInit(S, PrivateScope);
  4119. (void)PrivateScope.Privatize();
  4120. CGF.CGM.getOpenMPRuntime().emitInlinedDirective(
  4121. CGF, OMPD_distribute, CodeGenDistribute, /*HasCancel=*/false);
  4122. CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_teams);
  4123. };
  4124. emitCommonOMPTeamsDirective(CGF, S, OMPD_distribute_parallel_for_simd,
  4125. CodeGenTeams);
  4126. emitPostUpdateForReductionClause(CGF, S,
  4127. [](CodeGenFunction &) { return nullptr; });
  4128. }
  4129. void CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForSimdDeviceFunction(
  4130. CodeGenModule &CGM, StringRef ParentName,
  4131. const OMPTargetTeamsDistributeParallelForSimdDirective &S) {
  4132. // Emit SPMD target teams distribute parallel for simd region as a standalone
  4133. // region.
  4134. auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
  4135. emitTargetTeamsDistributeParallelForSimdRegion(CGF, S, Action);
  4136. };
  4137. llvm::Function *Fn;
  4138. llvm::Constant *Addr;
  4139. // Emit target region as a standalone region.
  4140. CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
  4141. S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
  4142. assert(Fn && Addr && "Target device function emission failed.");
  4143. }
  4144. void CodeGenFunction::EmitOMPTargetTeamsDistributeParallelForSimdDirective(
  4145. const OMPTargetTeamsDistributeParallelForSimdDirective &S) {
  4146. auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
  4147. emitTargetTeamsDistributeParallelForSimdRegion(CGF, S, Action);
  4148. };
  4149. emitCommonOMPTargetDirective(*this, S, CodeGen);
  4150. }
  4151. void CodeGenFunction::EmitOMPCancellationPointDirective(
  4152. const OMPCancellationPointDirective &S) {
  4153. CGM.getOpenMPRuntime().emitCancellationPointCall(*this, S.getBeginLoc(),
  4154. S.getCancelRegion());
  4155. }
  4156. void CodeGenFunction::EmitOMPCancelDirective(const OMPCancelDirective &S) {
  4157. const Expr *IfCond = nullptr;
  4158. for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
  4159. if (C->getNameModifier() == OMPD_unknown ||
  4160. C->getNameModifier() == OMPD_cancel) {
  4161. IfCond = C->getCondition();
  4162. break;
  4163. }
  4164. }
  4165. CGM.getOpenMPRuntime().emitCancelCall(*this, S.getBeginLoc(), IfCond,
  4166. S.getCancelRegion());
  4167. }
  4168. CodeGenFunction::JumpDest
  4169. CodeGenFunction::getOMPCancelDestination(OpenMPDirectiveKind Kind) {
  4170. if (Kind == OMPD_parallel || Kind == OMPD_task ||
  4171. Kind == OMPD_target_parallel)
  4172. return ReturnBlock;
  4173. assert(Kind == OMPD_for || Kind == OMPD_section || Kind == OMPD_sections ||
  4174. Kind == OMPD_parallel_sections || Kind == OMPD_parallel_for ||
  4175. Kind == OMPD_distribute_parallel_for ||
  4176. Kind == OMPD_target_parallel_for ||
  4177. Kind == OMPD_teams_distribute_parallel_for ||
  4178. Kind == OMPD_target_teams_distribute_parallel_for);
  4179. return OMPCancelStack.getExitBlock();
  4180. }
  4181. void CodeGenFunction::EmitOMPUseDevicePtrClause(
  4182. const OMPClause &NC, OMPPrivateScope &PrivateScope,
  4183. const llvm::DenseMap<const ValueDecl *, Address> &CaptureDeviceAddrMap) {
  4184. const auto &C = cast<OMPUseDevicePtrClause>(NC);
  4185. auto OrigVarIt = C.varlist_begin();
  4186. auto InitIt = C.inits().begin();
  4187. for (const Expr *PvtVarIt : C.private_copies()) {
  4188. const auto *OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*OrigVarIt)->getDecl());
  4189. const auto *InitVD = cast<VarDecl>(cast<DeclRefExpr>(*InitIt)->getDecl());
  4190. const auto *PvtVD = cast<VarDecl>(cast<DeclRefExpr>(PvtVarIt)->getDecl());
  4191. // In order to identify the right initializer we need to match the
  4192. // declaration used by the mapping logic. In some cases we may get
  4193. // OMPCapturedExprDecl that refers to the original declaration.
  4194. const ValueDecl *MatchingVD = OrigVD;
  4195. if (const auto *OED = dyn_cast<OMPCapturedExprDecl>(MatchingVD)) {
  4196. // OMPCapturedExprDecl are used to privative fields of the current
  4197. // structure.
  4198. const auto *ME = cast<MemberExpr>(OED->getInit());
  4199. assert(isa<CXXThisExpr>(ME->getBase()) &&
  4200. "Base should be the current struct!");
  4201. MatchingVD = ME->getMemberDecl();
  4202. }
  4203. // If we don't have information about the current list item, move on to
  4204. // the next one.
  4205. auto InitAddrIt = CaptureDeviceAddrMap.find(MatchingVD);
  4206. if (InitAddrIt == CaptureDeviceAddrMap.end())
  4207. continue;
  4208. bool IsRegistered = PrivateScope.addPrivate(OrigVD, [this, OrigVD,
  4209. InitAddrIt, InitVD,
  4210. PvtVD]() {
  4211. // Initialize the temporary initialization variable with the address we
  4212. // get from the runtime library. We have to cast the source address
  4213. // because it is always a void *. References are materialized in the
  4214. // privatization scope, so the initialization here disregards the fact
  4215. // the original variable is a reference.
  4216. QualType AddrQTy =
  4217. getContext().getPointerType(OrigVD->getType().getNonReferenceType());
  4218. llvm::Type *AddrTy = ConvertTypeForMem(AddrQTy);
  4219. Address InitAddr = Builder.CreateBitCast(InitAddrIt->second, AddrTy);
  4220. setAddrOfLocalVar(InitVD, InitAddr);
  4221. // Emit private declaration, it will be initialized by the value we
  4222. // declaration we just added to the local declarations map.
  4223. EmitDecl(*PvtVD);
  4224. // The initialization variables reached its purpose in the emission
  4225. // of the previous declaration, so we don't need it anymore.
  4226. LocalDeclMap.erase(InitVD);
  4227. // Return the address of the private variable.
  4228. return GetAddrOfLocalVar(PvtVD);
  4229. });
  4230. assert(IsRegistered && "firstprivate var already registered as private");
  4231. // Silence the warning about unused variable.
  4232. (void)IsRegistered;
  4233. ++OrigVarIt;
  4234. ++InitIt;
  4235. }
  4236. }
  4237. // Generate the instructions for '#pragma omp target data' directive.
  4238. void CodeGenFunction::EmitOMPTargetDataDirective(
  4239. const OMPTargetDataDirective &S) {
  4240. CGOpenMPRuntime::TargetDataInfo Info(/*RequiresDevicePointerInfo=*/true);
  4241. // Create a pre/post action to signal the privatization of the device pointer.
  4242. // This action can be replaced by the OpenMP runtime code generation to
  4243. // deactivate privatization.
  4244. bool PrivatizeDevicePointers = false;
  4245. class DevicePointerPrivActionTy : public PrePostActionTy {
  4246. bool &PrivatizeDevicePointers;
  4247. public:
  4248. explicit DevicePointerPrivActionTy(bool &PrivatizeDevicePointers)
  4249. : PrePostActionTy(), PrivatizeDevicePointers(PrivatizeDevicePointers) {}
  4250. void Enter(CodeGenFunction &CGF) override {
  4251. PrivatizeDevicePointers = true;
  4252. }
  4253. };
  4254. DevicePointerPrivActionTy PrivAction(PrivatizeDevicePointers);
  4255. auto &&CodeGen = [&S, &Info, &PrivatizeDevicePointers](
  4256. CodeGenFunction &CGF, PrePostActionTy &Action) {
  4257. auto &&InnermostCodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &) {
  4258. CGF.EmitStmt(S.getInnermostCapturedStmt()->getCapturedStmt());
  4259. };
  4260. // Codegen that selects whether to generate the privatization code or not.
  4261. auto &&PrivCodeGen = [&S, &Info, &PrivatizeDevicePointers,
  4262. &InnermostCodeGen](CodeGenFunction &CGF,
  4263. PrePostActionTy &Action) {
  4264. RegionCodeGenTy RCG(InnermostCodeGen);
  4265. PrivatizeDevicePointers = false;
  4266. // Call the pre-action to change the status of PrivatizeDevicePointers if
  4267. // needed.
  4268. Action.Enter(CGF);
  4269. if (PrivatizeDevicePointers) {
  4270. OMPPrivateScope PrivateScope(CGF);
  4271. // Emit all instances of the use_device_ptr clause.
  4272. for (const auto *C : S.getClausesOfKind<OMPUseDevicePtrClause>())
  4273. CGF.EmitOMPUseDevicePtrClause(*C, PrivateScope,
  4274. Info.CaptureDeviceAddrMap);
  4275. (void)PrivateScope.Privatize();
  4276. RCG(CGF);
  4277. } else {
  4278. RCG(CGF);
  4279. }
  4280. };
  4281. // Forward the provided action to the privatization codegen.
  4282. RegionCodeGenTy PrivRCG(PrivCodeGen);
  4283. PrivRCG.setAction(Action);
  4284. // Notwithstanding the body of the region is emitted as inlined directive,
  4285. // we don't use an inline scope as changes in the references inside the
  4286. // region are expected to be visible outside, so we do not privative them.
  4287. OMPLexicalScope Scope(CGF, S);
  4288. CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_target_data,
  4289. PrivRCG);
  4290. };
  4291. RegionCodeGenTy RCG(CodeGen);
  4292. // If we don't have target devices, don't bother emitting the data mapping
  4293. // code.
  4294. if (CGM.getLangOpts().OMPTargetTriples.empty()) {
  4295. RCG(*this);
  4296. return;
  4297. }
  4298. // Check if we have any if clause associated with the directive.
  4299. const Expr *IfCond = nullptr;
  4300. if (const auto *C = S.getSingleClause<OMPIfClause>())
  4301. IfCond = C->getCondition();
  4302. // Check if we have any device clause associated with the directive.
  4303. const Expr *Device = nullptr;
  4304. if (const auto *C = S.getSingleClause<OMPDeviceClause>())
  4305. Device = C->getDevice();
  4306. // Set the action to signal privatization of device pointers.
  4307. RCG.setAction(PrivAction);
  4308. // Emit region code.
  4309. CGM.getOpenMPRuntime().emitTargetDataCalls(*this, S, IfCond, Device, RCG,
  4310. Info);
  4311. }
  4312. void CodeGenFunction::EmitOMPTargetEnterDataDirective(
  4313. const OMPTargetEnterDataDirective &S) {
  4314. // If we don't have target devices, don't bother emitting the data mapping
  4315. // code.
  4316. if (CGM.getLangOpts().OMPTargetTriples.empty())
  4317. return;
  4318. // Check if we have any if clause associated with the directive.
  4319. const Expr *IfCond = nullptr;
  4320. if (const auto *C = S.getSingleClause<OMPIfClause>())
  4321. IfCond = C->getCondition();
  4322. // Check if we have any device clause associated with the directive.
  4323. const Expr *Device = nullptr;
  4324. if (const auto *C = S.getSingleClause<OMPDeviceClause>())
  4325. Device = C->getDevice();
  4326. OMPLexicalScope Scope(*this, S, OMPD_task);
  4327. CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device);
  4328. }
  4329. void CodeGenFunction::EmitOMPTargetExitDataDirective(
  4330. const OMPTargetExitDataDirective &S) {
  4331. // If we don't have target devices, don't bother emitting the data mapping
  4332. // code.
  4333. if (CGM.getLangOpts().OMPTargetTriples.empty())
  4334. return;
  4335. // Check if we have any if clause associated with the directive.
  4336. const Expr *IfCond = nullptr;
  4337. if (const auto *C = S.getSingleClause<OMPIfClause>())
  4338. IfCond = C->getCondition();
  4339. // Check if we have any device clause associated with the directive.
  4340. const Expr *Device = nullptr;
  4341. if (const auto *C = S.getSingleClause<OMPDeviceClause>())
  4342. Device = C->getDevice();
  4343. OMPLexicalScope Scope(*this, S, OMPD_task);
  4344. CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device);
  4345. }
  4346. static void emitTargetParallelRegion(CodeGenFunction &CGF,
  4347. const OMPTargetParallelDirective &S,
  4348. PrePostActionTy &Action) {
  4349. // Get the captured statement associated with the 'parallel' region.
  4350. const CapturedStmt *CS = S.getCapturedStmt(OMPD_parallel);
  4351. Action.Enter(CGF);
  4352. auto &&CodeGen = [&S, CS](CodeGenFunction &CGF, PrePostActionTy &Action) {
  4353. Action.Enter(CGF);
  4354. CodeGenFunction::OMPPrivateScope PrivateScope(CGF);
  4355. (void)CGF.EmitOMPFirstprivateClause(S, PrivateScope);
  4356. CGF.EmitOMPPrivateClause(S, PrivateScope);
  4357. CGF.EmitOMPReductionClauseInit(S, PrivateScope);
  4358. (void)PrivateScope.Privatize();
  4359. // TODO: Add support for clauses.
  4360. CGF.EmitStmt(CS->getCapturedStmt());
  4361. CGF.EmitOMPReductionClauseFinal(S, /*ReductionKind=*/OMPD_parallel);
  4362. };
  4363. emitCommonOMPParallelDirective(CGF, S, OMPD_parallel, CodeGen,
  4364. emitEmptyBoundParameters);
  4365. emitPostUpdateForReductionClause(CGF, S,
  4366. [](CodeGenFunction &) { return nullptr; });
  4367. }
  4368. void CodeGenFunction::EmitOMPTargetParallelDeviceFunction(
  4369. CodeGenModule &CGM, StringRef ParentName,
  4370. const OMPTargetParallelDirective &S) {
  4371. auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
  4372. emitTargetParallelRegion(CGF, S, Action);
  4373. };
  4374. llvm::Function *Fn;
  4375. llvm::Constant *Addr;
  4376. // Emit target region as a standalone region.
  4377. CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
  4378. S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
  4379. assert(Fn && Addr && "Target device function emission failed.");
  4380. }
  4381. void CodeGenFunction::EmitOMPTargetParallelDirective(
  4382. const OMPTargetParallelDirective &S) {
  4383. auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
  4384. emitTargetParallelRegion(CGF, S, Action);
  4385. };
  4386. emitCommonOMPTargetDirective(*this, S, CodeGen);
  4387. }
  4388. static void emitTargetParallelForRegion(CodeGenFunction &CGF,
  4389. const OMPTargetParallelForDirective &S,
  4390. PrePostActionTy &Action) {
  4391. Action.Enter(CGF);
  4392. // Emit directive as a combined directive that consists of two implicit
  4393. // directives: 'parallel' with 'for' directive.
  4394. auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
  4395. Action.Enter(CGF);
  4396. CodeGenFunction::OMPCancelStackRAII CancelRegion(
  4397. CGF, OMPD_target_parallel_for, S.hasCancel());
  4398. CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds,
  4399. emitDispatchForLoopBounds);
  4400. };
  4401. emitCommonOMPParallelDirective(CGF, S, OMPD_for, CodeGen,
  4402. emitEmptyBoundParameters);
  4403. }
  4404. void CodeGenFunction::EmitOMPTargetParallelForDeviceFunction(
  4405. CodeGenModule &CGM, StringRef ParentName,
  4406. const OMPTargetParallelForDirective &S) {
  4407. // Emit SPMD target parallel for region as a standalone region.
  4408. auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
  4409. emitTargetParallelForRegion(CGF, S, Action);
  4410. };
  4411. llvm::Function *Fn;
  4412. llvm::Constant *Addr;
  4413. // Emit target region as a standalone region.
  4414. CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
  4415. S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
  4416. assert(Fn && Addr && "Target device function emission failed.");
  4417. }
  4418. void CodeGenFunction::EmitOMPTargetParallelForDirective(
  4419. const OMPTargetParallelForDirective &S) {
  4420. auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
  4421. emitTargetParallelForRegion(CGF, S, Action);
  4422. };
  4423. emitCommonOMPTargetDirective(*this, S, CodeGen);
  4424. }
  4425. static void
  4426. emitTargetParallelForSimdRegion(CodeGenFunction &CGF,
  4427. const OMPTargetParallelForSimdDirective &S,
  4428. PrePostActionTy &Action) {
  4429. Action.Enter(CGF);
  4430. // Emit directive as a combined directive that consists of two implicit
  4431. // directives: 'parallel' with 'for' directive.
  4432. auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
  4433. Action.Enter(CGF);
  4434. CGF.EmitOMPWorksharingLoop(S, S.getEnsureUpperBound(), emitForLoopBounds,
  4435. emitDispatchForLoopBounds);
  4436. };
  4437. emitCommonOMPParallelDirective(CGF, S, OMPD_simd, CodeGen,
  4438. emitEmptyBoundParameters);
  4439. }
  4440. void CodeGenFunction::EmitOMPTargetParallelForSimdDeviceFunction(
  4441. CodeGenModule &CGM, StringRef ParentName,
  4442. const OMPTargetParallelForSimdDirective &S) {
  4443. // Emit SPMD target parallel for region as a standalone region.
  4444. auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
  4445. emitTargetParallelForSimdRegion(CGF, S, Action);
  4446. };
  4447. llvm::Function *Fn;
  4448. llvm::Constant *Addr;
  4449. // Emit target region as a standalone region.
  4450. CGM.getOpenMPRuntime().emitTargetOutlinedFunction(
  4451. S, ParentName, Fn, Addr, /*IsOffloadEntry=*/true, CodeGen);
  4452. assert(Fn && Addr && "Target device function emission failed.");
  4453. }
  4454. void CodeGenFunction::EmitOMPTargetParallelForSimdDirective(
  4455. const OMPTargetParallelForSimdDirective &S) {
  4456. auto &&CodeGen = [&S](CodeGenFunction &CGF, PrePostActionTy &Action) {
  4457. emitTargetParallelForSimdRegion(CGF, S, Action);
  4458. };
  4459. emitCommonOMPTargetDirective(*this, S, CodeGen);
  4460. }
  4461. /// Emit a helper variable and return corresponding lvalue.
  4462. static void mapParam(CodeGenFunction &CGF, const DeclRefExpr *Helper,
  4463. const ImplicitParamDecl *PVD,
  4464. CodeGenFunction::OMPPrivateScope &Privates) {
  4465. const auto *VDecl = cast<VarDecl>(Helper->getDecl());
  4466. Privates.addPrivate(VDecl,
  4467. [&CGF, PVD]() { return CGF.GetAddrOfLocalVar(PVD); });
  4468. }
  4469. void CodeGenFunction::EmitOMPTaskLoopBasedDirective(const OMPLoopDirective &S) {
  4470. assert(isOpenMPTaskLoopDirective(S.getDirectiveKind()));
  4471. // Emit outlined function for task construct.
  4472. const CapturedStmt *CS = S.getCapturedStmt(OMPD_taskloop);
  4473. Address CapturedStruct = GenerateCapturedStmtArgument(*CS);
  4474. QualType SharedsTy = getContext().getRecordType(CS->getCapturedRecordDecl());
  4475. const Expr *IfCond = nullptr;
  4476. for (const auto *C : S.getClausesOfKind<OMPIfClause>()) {
  4477. if (C->getNameModifier() == OMPD_unknown ||
  4478. C->getNameModifier() == OMPD_taskloop) {
  4479. IfCond = C->getCondition();
  4480. break;
  4481. }
  4482. }
  4483. OMPTaskDataTy Data;
  4484. // Check if taskloop must be emitted without taskgroup.
  4485. Data.Nogroup = S.getSingleClause<OMPNogroupClause>();
  4486. // TODO: Check if we should emit tied or untied task.
  4487. Data.Tied = true;
  4488. // Set scheduling for taskloop
  4489. if (const auto* Clause = S.getSingleClause<OMPGrainsizeClause>()) {
  4490. // grainsize clause
  4491. Data.Schedule.setInt(/*IntVal=*/false);
  4492. Data.Schedule.setPointer(EmitScalarExpr(Clause->getGrainsize()));
  4493. } else if (const auto* Clause = S.getSingleClause<OMPNumTasksClause>()) {
  4494. // num_tasks clause
  4495. Data.Schedule.setInt(/*IntVal=*/true);
  4496. Data.Schedule.setPointer(EmitScalarExpr(Clause->getNumTasks()));
  4497. }
  4498. auto &&BodyGen = [CS, &S](CodeGenFunction &CGF, PrePostActionTy &) {
  4499. // if (PreCond) {
  4500. // for (IV in 0..LastIteration) BODY;
  4501. // <Final counter/linear vars updates>;
  4502. // }
  4503. //
  4504. // Emit: if (PreCond) - begin.
  4505. // If the condition constant folds and can be elided, avoid emitting the
  4506. // whole loop.
  4507. bool CondConstant;
  4508. llvm::BasicBlock *ContBlock = nullptr;
  4509. OMPLoopScope PreInitScope(CGF, S);
  4510. if (CGF.ConstantFoldsToSimpleInteger(S.getPreCond(), CondConstant)) {
  4511. if (!CondConstant)
  4512. return;
  4513. } else {
  4514. llvm::BasicBlock *ThenBlock = CGF.createBasicBlock("taskloop.if.then");
  4515. ContBlock = CGF.createBasicBlock("taskloop.if.end");
  4516. emitPreCond(CGF, S, S.getPreCond(), ThenBlock, ContBlock,
  4517. CGF.getProfileCount(&S));
  4518. CGF.EmitBlock(ThenBlock);
  4519. CGF.incrementProfileCounter(&S);
  4520. }
  4521. if (isOpenMPSimdDirective(S.getDirectiveKind()))
  4522. CGF.EmitOMPSimdInit(S);
  4523. OMPPrivateScope LoopScope(CGF);
  4524. // Emit helper vars inits.
  4525. enum { LowerBound = 5, UpperBound, Stride, LastIter };
  4526. auto *I = CS->getCapturedDecl()->param_begin();
  4527. auto *LBP = std::next(I, LowerBound);
  4528. auto *UBP = std::next(I, UpperBound);
  4529. auto *STP = std::next(I, Stride);
  4530. auto *LIP = std::next(I, LastIter);
  4531. mapParam(CGF, cast<DeclRefExpr>(S.getLowerBoundVariable()), *LBP,
  4532. LoopScope);
  4533. mapParam(CGF, cast<DeclRefExpr>(S.getUpperBoundVariable()), *UBP,
  4534. LoopScope);
  4535. mapParam(CGF, cast<DeclRefExpr>(S.getStrideVariable()), *STP, LoopScope);
  4536. mapParam(CGF, cast<DeclRefExpr>(S.getIsLastIterVariable()), *LIP,
  4537. LoopScope);
  4538. CGF.EmitOMPPrivateLoopCounters(S, LoopScope);
  4539. bool HasLastprivateClause = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);
  4540. (void)LoopScope.Privatize();
  4541. // Emit the loop iteration variable.
  4542. const Expr *IVExpr = S.getIterationVariable();
  4543. const auto *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl());
  4544. CGF.EmitVarDecl(*IVDecl);
  4545. CGF.EmitIgnoredExpr(S.getInit());
  4546. // Emit the iterations count variable.
  4547. // If it is not a variable, Sema decided to calculate iterations count on
  4548. // each iteration (e.g., it is foldable into a constant).
  4549. if (const auto *LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {
  4550. CGF.EmitVarDecl(*cast<VarDecl>(LIExpr->getDecl()));
  4551. // Emit calculation of the iterations count.
  4552. CGF.EmitIgnoredExpr(S.getCalcLastIteration());
  4553. }
  4554. CGF.EmitOMPInnerLoop(S, LoopScope.requiresCleanups(), S.getCond(),
  4555. S.getInc(),
  4556. [&S](CodeGenFunction &CGF) {
  4557. CGF.EmitOMPLoopBody(S, JumpDest());
  4558. CGF.EmitStopPoint(&S);
  4559. },
  4560. [](CodeGenFunction &) {});
  4561. // Emit: if (PreCond) - end.
  4562. if (ContBlock) {
  4563. CGF.EmitBranch(ContBlock);
  4564. CGF.EmitBlock(ContBlock, true);
  4565. }
  4566. // Emit final copy of the lastprivate variables if IsLastIter != 0.
  4567. if (HasLastprivateClause) {
  4568. CGF.EmitOMPLastprivateClauseFinal(
  4569. S, isOpenMPSimdDirective(S.getDirectiveKind()),
  4570. CGF.Builder.CreateIsNotNull(CGF.EmitLoadOfScalar(
  4571. CGF.GetAddrOfLocalVar(*LIP), /*Volatile=*/false,
  4572. (*LIP)->getType(), S.getBeginLoc())));
  4573. }
  4574. };
  4575. auto &&TaskGen = [&S, SharedsTy, CapturedStruct,
  4576. IfCond](CodeGenFunction &CGF, llvm::Value *OutlinedFn,
  4577. const OMPTaskDataTy &Data) {
  4578. auto &&CodeGen = [&S, OutlinedFn, SharedsTy, CapturedStruct, IfCond,
  4579. &Data](CodeGenFunction &CGF, PrePostActionTy &) {
  4580. OMPLoopScope PreInitScope(CGF, S);
  4581. CGF.CGM.getOpenMPRuntime().emitTaskLoopCall(CGF, S.getBeginLoc(), S,
  4582. OutlinedFn, SharedsTy,
  4583. CapturedStruct, IfCond, Data);
  4584. };
  4585. CGF.CGM.getOpenMPRuntime().emitInlinedDirective(CGF, OMPD_taskloop,
  4586. CodeGen);
  4587. };
  4588. if (Data.Nogroup) {
  4589. EmitOMPTaskBasedDirective(S, OMPD_taskloop, BodyGen, TaskGen, Data);
  4590. } else {
  4591. CGM.getOpenMPRuntime().emitTaskgroupRegion(
  4592. *this,
  4593. [&S, &BodyGen, &TaskGen, &Data](CodeGenFunction &CGF,
  4594. PrePostActionTy &Action) {
  4595. Action.Enter(CGF);
  4596. CGF.EmitOMPTaskBasedDirective(S, OMPD_taskloop, BodyGen, TaskGen,
  4597. Data);
  4598. },
  4599. S.getBeginLoc());
  4600. }
  4601. }
  4602. void CodeGenFunction::EmitOMPTaskLoopDirective(const OMPTaskLoopDirective &S) {
  4603. EmitOMPTaskLoopBasedDirective(S);
  4604. }
  4605. void CodeGenFunction::EmitOMPTaskLoopSimdDirective(
  4606. const OMPTaskLoopSimdDirective &S) {
  4607. EmitOMPTaskLoopBasedDirective(S);
  4608. }
  4609. // Generate the instructions for '#pragma omp target update' directive.
  4610. void CodeGenFunction::EmitOMPTargetUpdateDirective(
  4611. const OMPTargetUpdateDirective &S) {
  4612. // If we don't have target devices, don't bother emitting the data mapping
  4613. // code.
  4614. if (CGM.getLangOpts().OMPTargetTriples.empty())
  4615. return;
  4616. // Check if we have any if clause associated with the directive.
  4617. const Expr *IfCond = nullptr;
  4618. if (const auto *C = S.getSingleClause<OMPIfClause>())
  4619. IfCond = C->getCondition();
  4620. // Check if we have any device clause associated with the directive.
  4621. const Expr *Device = nullptr;
  4622. if (const auto *C = S.getSingleClause<OMPDeviceClause>())
  4623. Device = C->getDevice();
  4624. OMPLexicalScope Scope(*this, S, OMPD_task);
  4625. CGM.getOpenMPRuntime().emitTargetDataStandAloneCall(*this, S, IfCond, Device);
  4626. }
  4627. void CodeGenFunction::EmitSimpleOMPExecutableDirective(
  4628. const OMPExecutableDirective &D) {
  4629. if (!D.hasAssociatedStmt() || !D.getAssociatedStmt())
  4630. return;
  4631. auto &&CodeGen = [&D](CodeGenFunction &CGF, PrePostActionTy &Action) {
  4632. if (isOpenMPSimdDirective(D.getDirectiveKind())) {
  4633. emitOMPSimdRegion(CGF, cast<OMPLoopDirective>(D), Action);
  4634. } else {
  4635. if (const auto *LD = dyn_cast<OMPLoopDirective>(&D)) {
  4636. for (const Expr *E : LD->counters()) {
  4637. if (const auto *VD = dyn_cast<OMPCapturedExprDecl>(
  4638. cast<DeclRefExpr>(E)->getDecl())) {
  4639. // Emit only those that were not explicitly referenced in clauses.
  4640. if (!CGF.LocalDeclMap.count(VD))
  4641. CGF.EmitVarDecl(*VD);
  4642. }
  4643. }
  4644. for (const auto *C : D.getClausesOfKind<OMPOrderedClause>()) {
  4645. if (!C->getNumForLoops())
  4646. continue;
  4647. for (unsigned I = LD->getCollapsedNumber(),
  4648. E = C->getLoopNumIterations().size();
  4649. I < E; ++I) {
  4650. if (const auto *VD = dyn_cast<OMPCapturedExprDecl>(
  4651. cast<DeclRefExpr>(C->getLoopCounter(I))->getDecl())) {
  4652. // Emit only those that were not explicitly referenced in clauses.
  4653. if (!CGF.LocalDeclMap.count(VD))
  4654. CGF.EmitVarDecl(*VD);
  4655. }
  4656. }
  4657. }
  4658. }
  4659. CGF.EmitStmt(D.getInnermostCapturedStmt()->getCapturedStmt());
  4660. }
  4661. };
  4662. OMPSimdLexicalScope Scope(*this, D);
  4663. CGM.getOpenMPRuntime().emitInlinedDirective(
  4664. *this,
  4665. isOpenMPSimdDirective(D.getDirectiveKind()) ? OMPD_simd
  4666. : D.getDirectiveKind(),
  4667. CodeGen);
  4668. }