starpu.texi 193 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149315031513152315331543155315631573158315931603161316231633164316531663167316831693170317131723173317431753176317731783179318031813182318331843185318631873188318931903191319231933194319531963197319831993200320132023203320432053206320732083209321032113212321332143215321632173218321932203221322232233224322532263227322832293230323132323233323432353236323732383239324032413242324332443245324632473248324932503251325232533254325532563257325832593260326132623263326432653266326732683269327032713272327332743275327632773278327932803281328232833284328532863287328832893290329132923293329432953296329732983299330033013302330333043305330633073308330933103311331233133314331533163317331833193320332133223323332433253326332733283329333033313332333333343335333633373338333933403341334233433344334533463347334833493350335133523353335433553356335733583359336033613362336333643365336633673368336933703371337233733374337533763377337833793380338133823383338433853386338733883389339033913392339333943395339633973398339934003401340234033404340534063407340834093410341134123413341434153416341734183419342034213422342334243425342634273428342934303431343234333434343534363437343834393440344134423443344434453446344734483449345034513452345334543455345634573458345934603461346234633464346534663467346834693470347134723473347434753476347734783479348034813482348334843485348634873488348934903491349234933494349534963497349834993500350135023503350435053506350735083509351035113512351335143515351635173518351935203521352235233524352535263527352835293530353135323533353435353536353735383539354035413542354335443545354635473548354935503551355235533554355535563557355835593560356135623563356435653566356735683569357035713572357335743575357635773578357935803581358235833584358535863587358835893590359135923593359435953596359735983599360036013602360336043605360636073608360936103611361236133614361536163617361836193620362136223623362436253626362736283629363036313632363336343635363636373638363936403641364236433644364536463647364836493650365136523653365436553656365736583659366036613662366336643665366636673668366936703671367236733674367536763677367836793680368136823683368436853686368736883689369036913692369336943695369636973698369937003701370237033704370537063707370837093710371137123713371437153716371737183719372037213722372337243725372637273728372937303731373237333734373537363737373837393740374137423743374437453746374737483749375037513752375337543755375637573758375937603761376237633764376537663767376837693770377137723773377437753776377737783779378037813782378337843785378637873788378937903791379237933794379537963797379837993800380138023803380438053806380738083809381038113812381338143815381638173818381938203821382238233824382538263827382838293830383138323833383438353836383738383839384038413842384338443845384638473848384938503851385238533854385538563857385838593860386138623863386438653866386738683869387038713872387338743875387638773878387938803881388238833884388538863887388838893890389138923893389438953896389738983899390039013902390339043905390639073908390939103911391239133914391539163917391839193920392139223923392439253926392739283929393039313932393339343935393639373938393939403941394239433944394539463947394839493950395139523953395439553956395739583959396039613962396339643965396639673968396939703971397239733974397539763977397839793980398139823983398439853986398739883989399039913992399339943995399639973998399940004001400240034004400540064007400840094010401140124013401440154016401740184019402040214022402340244025402640274028402940304031403240334034403540364037403840394040404140424043404440454046404740484049405040514052405340544055405640574058405940604061406240634064406540664067406840694070407140724073407440754076407740784079408040814082408340844085408640874088408940904091409240934094409540964097409840994100410141024103410441054106410741084109411041114112411341144115411641174118411941204121412241234124412541264127412841294130413141324133413441354136413741384139414041414142414341444145414641474148414941504151415241534154415541564157415841594160416141624163416441654166416741684169417041714172417341744175417641774178417941804181418241834184418541864187418841894190419141924193419441954196419741984199420042014202420342044205420642074208420942104211421242134214421542164217421842194220422142224223422442254226422742284229423042314232423342344235423642374238423942404241424242434244424542464247424842494250425142524253425442554256425742584259426042614262426342644265426642674268426942704271427242734274427542764277427842794280428142824283428442854286428742884289429042914292429342944295429642974298429943004301430243034304430543064307430843094310431143124313431443154316431743184319432043214322432343244325432643274328432943304331433243334334433543364337433843394340434143424343434443454346434743484349435043514352435343544355435643574358435943604361436243634364436543664367436843694370437143724373437443754376437743784379438043814382438343844385438643874388438943904391439243934394439543964397439843994400440144024403440444054406440744084409441044114412441344144415441644174418441944204421442244234424442544264427442844294430443144324433443444354436443744384439444044414442444344444445444644474448444944504451445244534454445544564457445844594460446144624463446444654466446744684469447044714472447344744475447644774478447944804481448244834484448544864487448844894490449144924493449444954496449744984499450045014502450345044505450645074508450945104511451245134514451545164517451845194520452145224523452445254526452745284529453045314532453345344535453645374538453945404541454245434544454545464547454845494550455145524553455445554556455745584559456045614562456345644565456645674568456945704571457245734574457545764577457845794580458145824583458445854586458745884589459045914592459345944595459645974598459946004601460246034604460546064607460846094610461146124613461446154616461746184619462046214622462346244625462646274628462946304631463246334634463546364637463846394640464146424643464446454646464746484649465046514652465346544655465646574658465946604661466246634664466546664667466846694670467146724673467446754676467746784679468046814682468346844685468646874688468946904691469246934694469546964697469846994700470147024703470447054706470747084709471047114712471347144715471647174718471947204721472247234724472547264727472847294730473147324733473447354736473747384739474047414742474347444745474647474748474947504751475247534754475547564757475847594760476147624763476447654766476747684769477047714772477347744775477647774778477947804781478247834784478547864787478847894790479147924793479447954796479747984799480048014802480348044805480648074808480948104811481248134814481548164817481848194820482148224823482448254826482748284829483048314832483348344835483648374838483948404841484248434844484548464847484848494850485148524853485448554856485748584859486048614862486348644865486648674868486948704871487248734874487548764877487848794880488148824883488448854886488748884889489048914892489348944895489648974898489949004901490249034904490549064907490849094910491149124913491449154916491749184919492049214922492349244925492649274928492949304931493249334934493549364937493849394940494149424943494449454946494749484949495049514952495349544955495649574958495949604961496249634964496549664967496849694970497149724973497449754976497749784979498049814982498349844985498649874988498949904991499249934994499549964997499849995000500150025003500450055006500750085009501050115012501350145015501650175018501950205021502250235024502550265027502850295030503150325033503450355036503750385039504050415042504350445045504650475048504950505051505250535054505550565057505850595060506150625063506450655066506750685069507050715072507350745075
  1. @c StarPU --- Runtime system for heterogeneous multicore architectures.
  2. @c
  3. @c Copyright (C) 2009-2011 Université de Bordeaux 1
  4. @c Copyright (C) 2010, 2011 Centre National de la Recherche Scientifique
  5. @c
  6. @c StarPU is free software; you can redistribute it and/or modify
  7. @c it under the terms of the GNU Lesser General Public License as published by
  8. @c the Free Software Foundation; either version 2.1 of the License, or (at
  9. @c your option) any later version.
  10. @c
  11. @c StarPU is distributed in the hope that it will be useful, but
  12. @c WITHOUT ANY WARRANTY; without even the implied warranty of
  13. @c MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
  14. @c
  15. @c See the GNU Lesser General Public License in COPYING.LGPL for more details.
  16. \input texinfo @c -*-texinfo-*-
  17. @c %**start of header
  18. @setfilename starpu.info
  19. @settitle StarPU Handbook
  20. @c %**end of header
  21. @include version.texi
  22. @setchapternewpage odd
  23. @dircategory Development
  24. @direntry
  25. * StarPU: (starpu). StarPU Handbook
  26. @end direntry
  27. @titlepage
  28. @title StarPU Handbook
  29. @subtitle for StarPU @value{VERSION}
  30. @page
  31. @vskip 0pt plus 1fill
  32. @comment For the @value{version-GCC} Version*
  33. @end titlepage
  34. @c @summarycontents
  35. @contents
  36. @page
  37. @node Top
  38. @top Preface
  39. @cindex Preface
  40. This manual documents the usage of StarPU version @value{VERSION}. It
  41. was last updated on @value{UPDATED}.
  42. @comment
  43. @comment When you add a new menu item, please keep the right hand
  44. @comment aligned to the same column. Do not use tabs. This provides
  45. @comment better formatting.
  46. @comment
  47. @menu
  48. * Introduction:: A basic introduction to using StarPU
  49. * Installing StarPU:: How to configure, build and install StarPU
  50. * Using StarPU:: How to run StarPU application
  51. * Basic Examples:: Basic examples of the use of StarPU
  52. * Performance optimization:: How to optimize performance with StarPU
  53. * Performance feedback:: Performance debugging tools
  54. * StarPU MPI support:: How to combine StarPU with MPI
  55. * Configuring StarPU:: How to configure StarPU
  56. * StarPU API:: The API to use StarPU
  57. * Advanced Topics:: Advanced use of StarPU
  58. * C Extensions:: Easier StarPU programming with GCC
  59. * Full source code for the 'Scaling a Vector' example::
  60. * Function Index:: Index of C functions.
  61. @end menu
  62. @c ---------------------------------------------------------------------
  63. @c Introduction to StarPU
  64. @c ---------------------------------------------------------------------
  65. @node Introduction
  66. @chapter Introduction to StarPU
  67. @menu
  68. * Motivation:: Why StarPU ?
  69. * StarPU in a Nutshell:: The Fundamentals of StarPU
  70. @end menu
  71. @node Motivation
  72. @section Motivation
  73. @c complex machines with heterogeneous cores/devices
  74. The use of specialized hardware such as accelerators or coprocessors offers an
  75. interesting approach to overcome the physical limits encountered by processor
  76. architects. As a result, many machines are now equipped with one or several
  77. accelerators (e.g. a GPU), in addition to the usual processor(s). While a lot of
  78. efforts have been devoted to offload computation onto such accelerators, very
  79. little attention as been paid to portability concerns on the one hand, and to the
  80. possibility of having heterogeneous accelerators and processors to interact on the other hand.
  81. StarPU is a runtime system that offers support for heterogeneous multicore
  82. architectures, it not only offers a unified view of the computational resources
  83. (i.e. CPUs and accelerators at the same time), but it also takes care of
  84. efficiently mapping and executing tasks onto an heterogeneous machine while
  85. transparently handling low-level issues such as data transfers in a portable
  86. fashion.
  87. @c this leads to a complicated distributed memory design
  88. @c which is not (easily) manageable by hand
  89. @c added value/benefits of StarPU
  90. @c - portability
  91. @c - scheduling, perf. portability
  92. @node StarPU in a Nutshell
  93. @section StarPU in a Nutshell
  94. @menu
  95. * Codelet and Tasks::
  96. * StarPU Data Management Library::
  97. * Glossary::
  98. * Research Papers::
  99. @end menu
  100. From a programming point of view, StarPU is not a new language but a library
  101. that executes tasks explicitly submitted by the application. The data that a
  102. task manipulates are automatically transferred onto the accelerator so that the
  103. programmer does not have to take care of complex data movements. StarPU also
  104. takes particular care of scheduling those tasks efficiently and allows
  105. scheduling experts to implement custom scheduling policies in a portable
  106. fashion.
  107. @c explain the notion of codelet and task (i.e. g(A, B)
  108. @node Codelet and Tasks
  109. @subsection Codelet and Tasks
  110. One of the StarPU primary data structures is the @b{codelet}. A codelet describes a
  111. computational kernel that can possibly be implemented on multiple architectures
  112. such as a CPU, a CUDA device or a Cell's SPU.
  113. @c TODO insert illustration f : f_spu, f_cpu, ...
  114. Another important data structure is the @b{task}. Executing a StarPU task
  115. consists in applying a codelet on a data set, on one of the architectures on
  116. which the codelet is implemented. A task thus describes the codelet that it
  117. uses, but also which data are accessed, and how they are
  118. accessed during the computation (read and/or write).
  119. StarPU tasks are asynchronous: submitting a task to StarPU is a non-blocking
  120. operation. The task structure can also specify a @b{callback} function that is
  121. called once StarPU has properly executed the task. It also contains optional
  122. fields that the application may use to give hints to the scheduler (such as
  123. priority levels).
  124. By default, task dependencies are inferred from data dependency (sequential
  125. coherence) by StarPU. The application can however disable sequential coherency
  126. for some data, and dependencies be expressed by hand.
  127. A task may be identified by a unique 64-bit number chosen by the application
  128. which we refer as a @b{tag}.
  129. Task dependencies can be enforced by hand either by the means of callback functions, by
  130. submitting other tasks, or by expressing dependencies
  131. between tags (which can thus correspond to tasks that have not been submitted
  132. yet).
  133. @c TODO insert illustration f(Ar, Brw, Cr) + ..
  134. @c DSM
  135. @node StarPU Data Management Library
  136. @subsection StarPU Data Management Library
  137. Because StarPU schedules tasks at runtime, data transfers have to be
  138. done automatically and ``just-in-time'' between processing units,
  139. relieving the application programmer from explicit data transfers.
  140. Moreover, to avoid unnecessary transfers, StarPU keeps data
  141. where it was last needed, even if was modified there, and it
  142. allows multiple copies of the same data to reside at the same time on
  143. several processing units as long as it is not modified.
  144. @node Glossary
  145. @subsection Glossary
  146. A @b{codelet} records pointers to various implementations of the same
  147. theoretical function.
  148. A @b{memory node} can be either the main RAM or GPU-embedded memory.
  149. A @b{bus} is a link between memory nodes.
  150. A @b{data handle} keeps track of replicates of the same data (@b{registered} by the
  151. application) over various memory nodes. The data management library manages
  152. keeping them coherent.
  153. The @b{home} memory node of a data handle is the memory node from which the data
  154. was registered (usually the main memory node).
  155. A @b{task} represents a scheduled execution of a codelet on some data handles.
  156. A @b{tag} is a rendez-vous point. Tasks typically have their own tag, and can
  157. depend on other tags. The value is chosen by the application.
  158. A @b{worker} execute tasks. There is typically one per CPU computation core and
  159. one per accelerator (for which a whole CPU core is dedicated).
  160. A @b{driver} drives a given kind of workers. There are currently CPU, CUDA,
  161. OpenCL and Gordon drivers. They usually start several workers to actually drive
  162. them.
  163. A @b{performance model} is a (dynamic or static) model of the performance of a
  164. given codelet. Codelets can have execution time performance model as well as
  165. power consumption performance models.
  166. A data @b{interface} describes the layout of the data: for a vector, a pointer
  167. for the start, the number of elements and the size of elements ; for a matrix, a
  168. pointer for the start, the number of elements per row, the offset between rows,
  169. and the size of each element ; etc. To access their data, codelet functions are
  170. given interfaces for the local memory node replicates of the data handles of the
  171. scheduled task.
  172. @b{Partitioning} data means dividing the data of a given data handle (called
  173. @b{father}) into a series of @b{children} data handles which designate various
  174. portions of the former.
  175. A @b{filter} is the function which computes children data handles from a father
  176. data handle, and thus describes how the partitioning should be done (horizontal,
  177. vertical, etc.)
  178. @b{Acquiring} a data handle can be done from the main application, to safely
  179. access the data of a data handle from its home node, without having to
  180. unregister it.
  181. @node Research Papers
  182. @subsection Research Papers
  183. Research papers about StarPU can be found at
  184. @indicateurl{http://runtime.bordeaux.inria.fr/Publis/Keyword/STARPU.html}
  185. Notably a good overview in the research report
  186. @indicateurl{http://hal.archives-ouvertes.fr/inria-00467677}
  187. @c ---------------------------------------------------------------------
  188. @c Installing StarPU
  189. @c ---------------------------------------------------------------------
  190. @node Installing StarPU
  191. @chapter Installing StarPU
  192. @menu
  193. * Downloading StarPU::
  194. * Configuration of StarPU::
  195. * Building and Installing StarPU::
  196. @end menu
  197. StarPU can be built and installed by the standard means of the GNU
  198. autotools. The following chapter is intended to briefly remind how these tools
  199. can be used to install StarPU.
  200. @node Downloading StarPU
  201. @section Downloading StarPU
  202. @menu
  203. * Getting Sources::
  204. * Optional dependencies::
  205. @end menu
  206. @node Getting Sources
  207. @subsection Getting Sources
  208. The simplest way to get StarPU sources is to download the latest official
  209. release tarball from @indicateurl{https://gforge.inria.fr/frs/?group_id=1570} ,
  210. or the latest nightly snapshot from
  211. @indicateurl{http://starpu.gforge.inria.fr/testing/} . The following documents
  212. how to get the very latest version from the subversion repository itself, it
  213. should be needed only if you need the very latest changes (i.e. less than a
  214. day!)
  215. The source code is managed by a Subversion server hosted by the
  216. InriaGforge. To get the source code, you need:
  217. @itemize
  218. @item
  219. To install the client side of the software Subversion if it is
  220. not already available on your system. The software can be obtained from
  221. @indicateurl{http://subversion.tigris.org} . If you are running
  222. on Windows, you will probably prefer to use TortoiseSVN from
  223. @indicateurl{http://tortoisesvn.tigris.org/} .
  224. @item
  225. You can check out the project's SVN repository through anonymous
  226. access. This will provide you with a read access to the
  227. repository.
  228. If you need to have write access on the StarPU project, you can also choose to
  229. become a member of the project @code{starpu}. For this, you first need to get
  230. an account to the gForge server. You can then send a request to join the project
  231. (@indicateurl{https://gforge.inria.fr/project/request.php?group_id=1570}).
  232. @item
  233. More information on how to get a gForge account, to become a member of
  234. a project, or on any other related task can be obtained from the
  235. InriaGforge at @indicateurl{https://gforge.inria.fr/}. The most important
  236. thing is to upload your public SSH key on the gForge server (see the
  237. FAQ at @indicateurl{http://siteadmin.gforge.inria.fr/FAQ.html#Q6} for
  238. instructions).
  239. @end itemize
  240. You can now check out the latest version from the Subversion server:
  241. @itemize
  242. @item
  243. using the anonymous access via svn:
  244. @example
  245. % svn checkout svn://scm.gforge.inria.fr/svn/starpu/trunk
  246. @end example
  247. @item
  248. using the anonymous access via https:
  249. @example
  250. % svn checkout --username anonsvn https://scm.gforge.inria.fr/svn/starpu/trunk
  251. @end example
  252. The password is @code{anonsvn}.
  253. @item
  254. using your gForge account
  255. @example
  256. % svn checkout svn+ssh://<login>@@scm.gforge.inria.fr/svn/starpu/trunk
  257. @end example
  258. @end itemize
  259. The following step requires the availability of @code{autoconf} and
  260. @code{automake} to generate the @code{./configure} script. This is
  261. done by calling @code{./autogen.sh}. The required version for
  262. @code{autoconf} is 2.60 or higher. You will also need @code{makeinfo}.
  263. @example
  264. % ./autogen.sh
  265. @end example
  266. If the autotools are not available on your machine or not recent
  267. enough, you can choose to download the latest nightly tarball, which
  268. is provided with a @code{configure} script.
  269. @example
  270. % wget http://starpu.gforge.inria.fr/testing/starpu-nightly-latest.tar.gz
  271. @end example
  272. @node Optional dependencies
  273. @subsection Optional dependencies
  274. The topology discovery library, @code{hwloc}, is not mandatory to use StarPU
  275. but strongly recommended. It allows to increase performance, and to
  276. perform some topology aware scheduling.
  277. @code{hwloc} is available in major distributions and for most OSes and can be
  278. downloaded from @indicateurl{http://www.open-mpi.org/software/hwloc}.
  279. @node Configuration of StarPU
  280. @section Configuration of StarPU
  281. @menu
  282. * Generating Makefiles and configuration scripts::
  283. * Running the configuration::
  284. @end menu
  285. @node Generating Makefiles and configuration scripts
  286. @subsection Generating Makefiles and configuration scripts
  287. This step is not necessary when using the tarball releases of StarPU. If you
  288. are using the source code from the svn repository, you first need to generate
  289. the configure scripts and the Makefiles.
  290. @example
  291. % ./autogen.sh
  292. @end example
  293. @node Running the configuration
  294. @subsection Running the configuration
  295. @example
  296. % ./configure
  297. @end example
  298. Details about options that are useful to give to @code{./configure} are given in
  299. @ref{Compilation configuration}.
  300. @node Building and Installing StarPU
  301. @section Building and Installing StarPU
  302. @menu
  303. * Building::
  304. * Sanity Checks::
  305. * Installing::
  306. @end menu
  307. @node Building
  308. @subsection Building
  309. @example
  310. % make
  311. @end example
  312. @node Sanity Checks
  313. @subsection Sanity Checks
  314. In order to make sure that StarPU is working properly on the system, it is also
  315. possible to run a test suite.
  316. @example
  317. % make check
  318. @end example
  319. @node Installing
  320. @subsection Installing
  321. In order to install StarPU at the location that was specified during
  322. configuration:
  323. @example
  324. % make install
  325. @end example
  326. @c ---------------------------------------------------------------------
  327. @c Using StarPU
  328. @c ---------------------------------------------------------------------
  329. @node Using StarPU
  330. @chapter Using StarPU
  331. @menu
  332. * Setting flags for compiling and linking applications::
  333. * Running a basic StarPU application::
  334. * Kernel threads started by StarPU::
  335. * Enabling OpenCL::
  336. @end menu
  337. @node Setting flags for compiling and linking applications
  338. @section Setting flags for compiling and linking applications
  339. Compiling and linking an application against StarPU may require to use
  340. specific flags or libraries (for instance @code{CUDA} or @code{libspe2}).
  341. To this end, it is possible to use the @code{pkg-config} tool.
  342. If StarPU was not installed at some standard location, the path of StarPU's
  343. library must be specified in the @code{PKG_CONFIG_PATH} environment variable so
  344. that @code{pkg-config} can find it. For example if StarPU was installed in
  345. @code{$prefix_dir}:
  346. @example
  347. % PKG_CONFIG_PATH=$PKG_CONFIG_PATH:$prefix_dir/lib/pkgconfig
  348. @end example
  349. The flags required to compile or link against StarPU are then
  350. accessible with the following commands:
  351. @example
  352. % pkg-config --cflags libstarpu # options for the compiler
  353. % pkg-config --libs libstarpu # options for the linker
  354. @end example
  355. @node Running a basic StarPU application
  356. @section Running a basic StarPU application
  357. Basic examples using StarPU are built in the directory
  358. @code{examples/basic_examples/} (and installed in
  359. @code{$prefix_dir/lib/starpu/examples/}). You can for example run the example
  360. @code{vector_scal}.
  361. @example
  362. % ./examples/basic_examples/vector_scal
  363. BEFORE : First element was 1.000000
  364. AFTER First element is 3.140000
  365. %
  366. @end example
  367. When StarPU is used for the first time, the directory
  368. @code{$HOME/.starpu/} is created, performance models will be stored in
  369. that directory.
  370. Please note that buses are benchmarked when StarPU is launched for the
  371. first time. This may take a few minutes, or less if @code{hwloc} is
  372. installed. This step is done only once per user and per machine.
  373. @node Kernel threads started by StarPU
  374. @section Kernel threads started by StarPU
  375. StarPU automatically binds one thread per CPU core. It does not use
  376. SMT/hyperthreading because kernels are usually already optimized for using a
  377. full core, and using hyperthreading would make kernel calibration rather random.
  378. Since driving GPUs is a CPU-consuming task, StarPU dedicates one core per GPU
  379. While StarPU tasks are executing, the application is not supposed to do
  380. computations in the threads it starts itself, tasks should be used instead.
  381. TODO: add a StarPU function to bind an application thread (e.g. the main thread)
  382. to a dedicated core (and thus disable the corresponding StarPU CPU worker).
  383. @node Enabling OpenCL
  384. @section Enabling OpenCL
  385. When both CUDA and OpenCL drivers are enabled, StarPU will launch an
  386. OpenCL worker for NVIDIA GPUs only if CUDA is not already running on them.
  387. This design choice was necessary as OpenCL and CUDA can not run at the
  388. same time on the same NVIDIA GPU, as there is currently no interoperability
  389. between them.
  390. To enable OpenCL, you need either to disable CUDA when configuring StarPU:
  391. @example
  392. % ./configure --disable-cuda
  393. @end example
  394. or when running applications:
  395. @example
  396. % STARPU_NCUDA=0 ./application
  397. @end example
  398. OpenCL will automatically be started on any device not yet used by
  399. CUDA. So on a machine running 4 GPUS, it is therefore possible to
  400. enable CUDA on 2 devices, and OpenCL on the 2 other devices by doing
  401. so:
  402. @example
  403. % STARPU_NCUDA=2 ./application
  404. @end example
  405. @c ---------------------------------------------------------------------
  406. @c Basic Examples
  407. @c ---------------------------------------------------------------------
  408. @node Basic Examples
  409. @chapter Basic Examples
  410. @menu
  411. * Compiling and linking options::
  412. * Hello World:: Submitting Tasks
  413. * Scaling a Vector:: Manipulating Data
  414. * Vector Scaling on an Hybrid CPU/GPU Machine:: Handling Heterogeneous Architectures
  415. * Using multiple implentations of a codelet::
  416. * Task and Worker Profiling::
  417. * Partitioning Data:: Partitioning Data
  418. * Performance model example::
  419. * Theoretical lower bound on execution time::
  420. * Insert Task Utility::
  421. * More examples:: More examples shipped with StarPU
  422. * Debugging:: When things go wrong.
  423. @end menu
  424. @node Compiling and linking options
  425. @section Compiling and linking options
  426. Let's suppose StarPU has been installed in the directory
  427. @code{$STARPU_DIR}. As explained in @ref{Setting flags for compiling and linking applications},
  428. the variable @code{PKG_CONFIG_PATH} needs to be set. It is also
  429. necessary to set the variable @code{LD_LIBRARY_PATH} to locate dynamic
  430. libraries at runtime.
  431. @example
  432. % PKG_CONFIG_PATH=$STARPU_DIR/lib/pkgconfig:$PKG_CONFIG_PATH
  433. % LD_LIBRARY_PATH=$STARPU_DIR/lib:$LD_LIBRARY_PATH
  434. @end example
  435. The Makefile could for instance contain the following lines to define which
  436. options must be given to the compiler and to the linker:
  437. @cartouche
  438. @example
  439. CFLAGS += $$(pkg-config --cflags libstarpu)
  440. LDFLAGS += $$(pkg-config --libs libstarpu)
  441. @end example
  442. @end cartouche
  443. @node Hello World
  444. @section Hello World
  445. @menu
  446. * Required Headers::
  447. * Defining a Codelet::
  448. * Submitting a Task::
  449. * Execution of Hello World::
  450. @end menu
  451. In this section, we show how to implement a simple program that submits a task to StarPU.
  452. @node Required Headers
  453. @subsection Required Headers
  454. The @code{starpu.h} header should be included in any code using StarPU.
  455. @cartouche
  456. @smallexample
  457. #include <starpu.h>
  458. @end smallexample
  459. @end cartouche
  460. @node Defining a Codelet
  461. @subsection Defining a Codelet
  462. @cartouche
  463. @smallexample
  464. struct params @{
  465. int i;
  466. float f;
  467. @};
  468. void cpu_func(void *buffers[], void *cl_arg)
  469. @{
  470. struct params *params = cl_arg;
  471. printf("Hello world (params = @{%i, %f@} )\n", params->i, params->f);
  472. @}
  473. starpu_codelet cl =
  474. @{
  475. .where = STARPU_CPU,
  476. .cpu_func = cpu_func,
  477. .nbuffers = 0
  478. @};
  479. @end smallexample
  480. @end cartouche
  481. A codelet is a structure that represents a computational kernel. Such a codelet
  482. may contain an implementation of the same kernel on different architectures
  483. (e.g. CUDA, Cell's SPU, x86, ...).
  484. The @code{nbuffers} field specifies the number of data buffers that are
  485. manipulated by the codelet: here the codelet does not access or modify any data
  486. that is controlled by our data management library. Note that the argument
  487. passed to the codelet (the @code{cl_arg} field of the @code{starpu_task}
  488. structure) does not count as a buffer since it is not managed by our data
  489. management library, but just contain trivial parameters.
  490. @c TODO need a crossref to the proper description of "where" see bla for more ...
  491. We create a codelet which may only be executed on the CPUs. The @code{where}
  492. field is a bitmask that defines where the codelet may be executed. Here, the
  493. @code{STARPU_CPU} value means that only CPUs can execute this codelet
  494. (@pxref{Codelets and Tasks} for more details on this field).
  495. When a CPU core executes a codelet, it calls the @code{cpu_func} function,
  496. which @emph{must} have the following prototype:
  497. @code{void (*cpu_func)(void *buffers[], void *cl_arg);}
  498. In this example, we can ignore the first argument of this function which gives a
  499. description of the input and output buffers (e.g. the size and the location of
  500. the matrices) since there is none.
  501. The second argument is a pointer to a buffer passed as an
  502. argument to the codelet by the means of the @code{cl_arg} field of the
  503. @code{starpu_task} structure.
  504. @c TODO rewrite so that it is a little clearer ?
  505. Be aware that this may be a pointer to a
  506. @emph{copy} of the actual buffer, and not the pointer given by the programmer:
  507. if the codelet modifies this buffer, there is no guarantee that the initial
  508. buffer will be modified as well: this for instance implies that the buffer
  509. cannot be used as a synchronization medium. If synchronization is needed, data
  510. has to be registered to StarPU, see @ref{Scaling a Vector}.
  511. @node Submitting a Task
  512. @subsection Submitting a Task
  513. @cartouche
  514. @smallexample
  515. void callback_func(void *callback_arg)
  516. @{
  517. printf("Callback function (arg %x)\n", callback_arg);
  518. @}
  519. int main(int argc, char **argv)
  520. @{
  521. /* @b{initialize StarPU} */
  522. starpu_init(NULL);
  523. struct starpu_task *task = starpu_task_create();
  524. task->cl = &cl; /* @b{Pointer to the codelet defined above} */
  525. struct params params = @{ 1, 2.0f @};
  526. task->cl_arg = &params;
  527. task->cl_arg_size = sizeof(params);
  528. task->callback_func = callback_func;
  529. task->callback_arg = 0x42;
  530. /* @b{starpu_task_submit will be a blocking call} */
  531. task->synchronous = 1;
  532. /* @b{submit the task to StarPU} */
  533. starpu_task_submit(task);
  534. /* @b{terminate StarPU} */
  535. starpu_shutdown();
  536. return 0;
  537. @}
  538. @end smallexample
  539. @end cartouche
  540. Before submitting any tasks to StarPU, @code{starpu_init} must be called. The
  541. @code{NULL} argument specifies that we use default configuration. Tasks cannot
  542. be submitted after the termination of StarPU by a call to
  543. @code{starpu_shutdown}.
  544. In the example above, a task structure is allocated by a call to
  545. @code{starpu_task_create}. This function only allocates and fills the
  546. corresponding structure with the default settings (@pxref{Codelets and
  547. Tasks, starpu_task_create}), but it does not submit the task to StarPU.
  548. @c not really clear ;)
  549. The @code{cl} field is a pointer to the codelet which the task will
  550. execute: in other words, the codelet structure describes which computational
  551. kernel should be offloaded on the different architectures, and the task
  552. structure is a wrapper containing a codelet and the piece of data on which the
  553. codelet should operate.
  554. The optional @code{cl_arg} field is a pointer to a buffer (of size
  555. @code{cl_arg_size}) with some parameters for the kernel
  556. described by the codelet. For instance, if a codelet implements a computational
  557. kernel that multiplies its input vector by a constant, the constant could be
  558. specified by the means of this buffer, instead of registering it as a StarPU
  559. data. It must however be noted that StarPU avoids making copy whenever possible
  560. and rather passes the pointer as such, so the buffer which is pointed at must
  561. kept allocated until the task terminates, and if several tasks are submitted
  562. with various parameters, each of them must be given a pointer to their own
  563. buffer.
  564. Once a task has been executed, an optional callback function is be called.
  565. While the computational kernel could be offloaded on various architectures, the
  566. callback function is always executed on a CPU. The @code{callback_arg}
  567. pointer is passed as an argument of the callback. The prototype of a callback
  568. function must be:
  569. @code{void (*callback_function)(void *);}
  570. If the @code{synchronous} field is non-zero, task submission will be
  571. synchronous: the @code{starpu_task_submit} function will not return until the
  572. task was executed. Note that the @code{starpu_shutdown} method does not
  573. guarantee that asynchronous tasks have been executed before it returns,
  574. @code{starpu_task_wait_for_all} can be used to that effect, or data can be
  575. unregistered (@code{starpu_data_unregister(vector_handle);}), which will
  576. implicitly wait for all the tasks scheduled to work on it, unless explicitly
  577. disabled thanks to @code{starpu_data_set_default_sequential_consistency_flag} or
  578. @code{starpu_data_set_sequential_consistency_flag}.
  579. @node Execution of Hello World
  580. @subsection Execution of Hello World
  581. @smallexample
  582. % make hello_world
  583. cc $(pkg-config --cflags libstarpu) $(pkg-config --libs libstarpu) hello_world.c -o hello_world
  584. % ./hello_world
  585. Hello world (params = @{1, 2.000000@} )
  586. Callback function (arg 42)
  587. @end smallexample
  588. @node Scaling a Vector
  589. @section Manipulating Data: Scaling a Vector
  590. The previous example has shown how to submit tasks. In this section,
  591. we show how StarPU tasks can manipulate data. The full source code for
  592. this example is given in @ref{Full source code for the 'Scaling a Vector' example}.
  593. @menu
  594. * Source code of Vector Scaling::
  595. * Execution of Vector Scaling::
  596. @end menu
  597. @node Source code of Vector Scaling
  598. @subsection Source code of Vector Scaling
  599. Programmers can describe the data layout of their application so that StarPU is
  600. responsible for enforcing data coherency and availability across the machine.
  601. Instead of handling complex (and non-portable) mechanisms to perform data
  602. movements, programmers only declare which piece of data is accessed and/or
  603. modified by a task, and StarPU makes sure that when a computational kernel
  604. starts somewhere (e.g. on a GPU), its data are available locally.
  605. Before submitting those tasks, the programmer first needs to declare the
  606. different pieces of data to StarPU using the @code{starpu_*_data_register}
  607. functions. To ease the development of applications for StarPU, it is possible
  608. to describe multiple types of data layout. A type of data layout is called an
  609. @b{interface}. There are different predefined interfaces available in StarPU:
  610. here we will consider the @b{vector interface}.
  611. The following lines show how to declare an array of @code{NX} elements of type
  612. @code{float} using the vector interface:
  613. @cartouche
  614. @smallexample
  615. float vector[NX];
  616. starpu_data_handle vector_handle;
  617. starpu_vector_data_register(&vector_handle, 0, (uintptr_t)vector, NX,
  618. sizeof(vector[0]));
  619. @end smallexample
  620. @end cartouche
  621. The first argument, called the @b{data handle}, is an opaque pointer which
  622. designates the array in StarPU. This is also the structure which is used to
  623. describe which data is used by a task. The second argument is the node number
  624. where the data originally resides. Here it is 0 since the @code{vector} array is in
  625. the main memory. Then comes the pointer @code{vector} where the data can be found in main memory,
  626. the number of elements in the vector and the size of each element.
  627. The following shows how to construct a StarPU task that will manipulate the
  628. vector and a constant factor.
  629. @cartouche
  630. @smallexample
  631. float factor = 3.14;
  632. struct starpu_task *task = starpu_task_create();
  633. task->cl = &cl; /* @b{Pointer to the codelet defined below} */
  634. task->buffers[0].handle = vector_handle; /* @b{First parameter of the codelet} */
  635. task->buffers[0].mode = STARPU_RW;
  636. task->cl_arg = &factor;
  637. task->cl_arg_size = sizeof(factor);
  638. task->synchronous = 1;
  639. starpu_task_submit(task);
  640. @end smallexample
  641. @end cartouche
  642. Since the factor is a mere constant float value parameter,
  643. it does not need a preliminary registration, and
  644. can just be passed through the @code{cl_arg} pointer like in the previous
  645. example. The vector parameter is described by its handle.
  646. There are two fields in each element of the @code{buffers} array.
  647. @code{handle} is the handle of the data, and @code{mode} specifies how the
  648. kernel will access the data (@code{STARPU_R} for read-only, @code{STARPU_W} for
  649. write-only and @code{STARPU_RW} for read and write access).
  650. The definition of the codelet can be written as follows:
  651. @cartouche
  652. @smallexample
  653. void scal_cpu_func(void *buffers[], void *cl_arg)
  654. @{
  655. unsigned i;
  656. float *factor = cl_arg;
  657. /* length of the vector */
  658. unsigned n = STARPU_VECTOR_GET_NX(buffers[0]);
  659. /* CPU copy of the vector pointer */
  660. float *val = (float *)STARPU_VECTOR_GET_PTR(buffers[0]);
  661. for (i = 0; i < n; i++)
  662. val[i] *= *factor;
  663. @}
  664. starpu_codelet cl = @{
  665. .where = STARPU_CPU,
  666. .cpu_func = scal_cpu_func,
  667. .nbuffers = 1
  668. @};
  669. @end smallexample
  670. @end cartouche
  671. The first argument is an array that gives
  672. a description of all the buffers passed in the @code{task->buffers}@ array. The
  673. size of this array is given by the @code{nbuffers} field of the codelet
  674. structure. For the sake of genericity, this array contains pointers to the
  675. different interfaces describing each buffer. In the case of the @b{vector
  676. interface}, the location of the vector (resp. its length) is accessible in the
  677. @code{ptr} (resp. @code{nx}) of this array. Since the vector is accessed in a
  678. read-write fashion, any modification will automatically affect future accesses
  679. to this vector made by other tasks.
  680. The second argument of the @code{scal_cpu_func} function contains a pointer to the
  681. parameters of the codelet (given in @code{task->cl_arg}), so that we read the
  682. constant factor from this pointer.
  683. @node Execution of Vector Scaling
  684. @subsection Execution of Vector Scaling
  685. @smallexample
  686. % make vector_scal
  687. cc $(pkg-config --cflags libstarpu) $(pkg-config --libs libstarpu) vector_scal.c -o vector_scal
  688. % ./vector_scal
  689. 0.000000 3.000000 6.000000 9.000000 12.000000
  690. @end smallexample
  691. @node Vector Scaling on an Hybrid CPU/GPU Machine
  692. @section Vector Scaling on an Hybrid CPU/GPU Machine
  693. Contrary to the previous examples, the task submitted in this example may not
  694. only be executed by the CPUs, but also by a CUDA device.
  695. @menu
  696. * Definition of the CUDA Kernel::
  697. * Definition of the OpenCL Kernel::
  698. * Definition of the Main Code::
  699. * Execution of Hybrid Vector Scaling::
  700. @end menu
  701. @node Definition of the CUDA Kernel
  702. @subsection Definition of the CUDA Kernel
  703. The CUDA implementation can be written as follows. It needs to be compiled with
  704. a CUDA compiler such as nvcc, the NVIDIA CUDA compiler driver. It must be noted
  705. that the vector pointer returned by STARPU_VECTOR_GET_PTR is here a pointer in GPU
  706. memory, so that it can be passed as such to the @code{vector_mult_cuda} kernel
  707. call.
  708. @cartouche
  709. @smallexample
  710. #include <starpu.h>
  711. #include <starpu_cuda.h>
  712. static __global__ void vector_mult_cuda(float *val, unsigned n,
  713. float factor)
  714. @{
  715. unsigned i = blockIdx.x*blockDim.x + threadIdx.x;
  716. if (i < n)
  717. val[i] *= factor;
  718. @}
  719. extern "C" void scal_cuda_func(void *buffers[], void *_args)
  720. @{
  721. float *factor = (float *)_args;
  722. /* length of the vector */
  723. unsigned n = STARPU_VECTOR_GET_NX(buffers[0]);
  724. /* CUDA copy of the vector pointer */
  725. float *val = (float *)STARPU_VECTOR_GET_PTR(buffers[0]);
  726. unsigned threads_per_block = 64;
  727. unsigned nblocks = (n + threads_per_block-1) / threads_per_block;
  728. @i{ vector_mult_cuda<<<nblocks,threads_per_block, 0, starpu_cuda_get_local_stream()>>>(val, n, *factor);}
  729. @i{ cudaStreamSynchronize(starpu_cuda_get_local_stream());}
  730. @}
  731. @end smallexample
  732. @end cartouche
  733. @node Definition of the OpenCL Kernel
  734. @subsection Definition of the OpenCL Kernel
  735. The OpenCL implementation can be written as follows. StarPU provides
  736. tools to compile a OpenCL kernel stored in a file.
  737. @cartouche
  738. @smallexample
  739. __kernel void vector_mult_opencl(__global float* val, int nx, float factor)
  740. @{
  741. const int i = get_global_id(0);
  742. if (i < nx) @{
  743. val[i] *= factor;
  744. @}
  745. @}
  746. @end smallexample
  747. @end cartouche
  748. Similarly to CUDA, the pointer returned by @code{STARPU_VECTOR_GET_PTR} is here
  749. a device pointer, so that it is passed as such to the OpenCL kernel.
  750. @cartouche
  751. @smallexample
  752. #include <starpu.h>
  753. @i{#include <starpu_opencl.h>}
  754. @i{extern struct starpu_opencl_program programs;}
  755. void scal_opencl_func(void *buffers[], void *_args)
  756. @{
  757. float *factor = _args;
  758. @i{ int id, devid, err;}
  759. @i{ cl_kernel kernel;}
  760. @i{ cl_command_queue queue;}
  761. @i{ cl_event event;}
  762. /* length of the vector */
  763. unsigned n = STARPU_VECTOR_GET_NX(buffers[0]);
  764. /* OpenCL copy of the vector pointer */
  765. cl_mem val = (cl_mem) STARPU_VECTOR_GET_PTR(buffers[0]);
  766. @i{ id = starpu_worker_get_id();}
  767. @i{ devid = starpu_worker_get_devid(id);}
  768. @i{ err = starpu_opencl_load_kernel(&kernel, &queue, &programs,}
  769. @i{ "vector_mult_opencl", devid); /* @b{Name of the codelet defined above} */}
  770. @i{ if (err != CL_SUCCESS) STARPU_OPENCL_REPORT_ERROR(err);}
  771. @i{ err = clSetKernelArg(kernel, 0, sizeof(val), &val);}
  772. @i{ err |= clSetKernelArg(kernel, 1, sizeof(n), &n);}
  773. @i{ err |= clSetKernelArg(kernel, 2, sizeof(*factor), factor);}
  774. @i{ if (err) STARPU_OPENCL_REPORT_ERROR(err);}
  775. @i{ @{}
  776. @i{ size_t global=1;}
  777. @i{ size_t local=1;}
  778. @i{ err = clEnqueueNDRangeKernel(queue, kernel, 1, NULL, &global, &local, 0, NULL, &event);}
  779. @i{ if (err != CL_SUCCESS) STARPU_OPENCL_REPORT_ERROR(err);}
  780. @i{ @}}
  781. @i{ clFinish(queue);}
  782. @i{ starpu_opencl_collect_stats(event);}
  783. @i{ clReleaseEvent(event);}
  784. @i{ starpu_opencl_release_kernel(kernel);}
  785. @}
  786. @end smallexample
  787. @end cartouche
  788. @node Definition of the Main Code
  789. @subsection Definition of the Main Code
  790. The CPU implementation is the same as in the previous section.
  791. Here is the source of the main application. You can notice the value of the
  792. field @code{where} for the codelet. We specify
  793. @code{STARPU_CPU|STARPU_CUDA|STARPU_OPENCL} to indicate to StarPU that the codelet
  794. can be executed either on a CPU or on a CUDA or an OpenCL device.
  795. @cartouche
  796. @smallexample
  797. #include <starpu.h>
  798. #define NX 2048
  799. extern void scal_cuda_func(void *buffers[], void *_args);
  800. extern void scal_cpu_func(void *buffers[], void *_args);
  801. extern void scal_opencl_func(void *buffers[], void *_args);
  802. /* @b{Definition of the codelet} */
  803. static starpu_codelet cl = @{
  804. .where = STARPU_CPU|STARPU_CUDA|STARPU_OPENCL; /* @b{It can be executed on a CPU,} */
  805. /* @b{on a CUDA device, or on an OpenCL device} */
  806. .cuda_func = scal_cuda_func,
  807. .cpu_func = scal_cpu_func,
  808. .opencl_func = scal_opencl_func,
  809. .nbuffers = 1
  810. @}
  811. #ifdef STARPU_USE_OPENCL
  812. /* @b{The compiled version of the OpenCL program} */
  813. struct starpu_opencl_program programs;
  814. #endif
  815. int main(int argc, char **argv)
  816. @{
  817. float *vector;
  818. int i, ret;
  819. float factor=3.0;
  820. struct starpu_task *task;
  821. starpu_data_handle vector_handle;
  822. starpu_init(NULL); /* @b{Initialising StarPU} */
  823. #ifdef STARPU_USE_OPENCL
  824. starpu_opencl_load_opencl_from_file(
  825. "examples/basic_examples/vector_scal_opencl_codelet.cl",
  826. &programs, NULL);
  827. #endif
  828. vector = malloc(NX*sizeof(vector[0]));
  829. assert(vector);
  830. for(i=0 ; i<NX ; i++) vector[i] = i;
  831. @end smallexample
  832. @end cartouche
  833. @cartouche
  834. @smallexample
  835. /* @b{Registering data within StarPU} */
  836. starpu_vector_data_register(&vector_handle, 0, (uintptr_t)vector,
  837. NX, sizeof(vector[0]));
  838. /* @b{Definition of the task} */
  839. task = starpu_task_create();
  840. task->cl = &cl;
  841. task->buffers[0].handle = vector_handle;
  842. task->buffers[0].mode = STARPU_RW;
  843. task->cl_arg = &factor;
  844. task->cl_arg_size = sizeof(factor);
  845. @end smallexample
  846. @end cartouche
  847. @cartouche
  848. @smallexample
  849. /* @b{Submitting the task} */
  850. ret = starpu_task_submit(task);
  851. if (ret == -ENODEV) @{
  852. fprintf(stderr, "No worker may execute this task\n");
  853. return 1;
  854. @}
  855. @c TODO: Mmm, should rather be an unregistration with an implicit dependency, no?
  856. /* @b{Waiting for its termination} */
  857. starpu_task_wait_for_all();
  858. /* @b{Update the vector in RAM} */
  859. starpu_data_acquire(vector_handle, STARPU_R);
  860. @end smallexample
  861. @end cartouche
  862. @cartouche
  863. @smallexample
  864. /* @b{Access the data} */
  865. for(i=0 ; i<NX; i++) @{
  866. fprintf(stderr, "%f ", vector[i]);
  867. @}
  868. fprintf(stderr, "\n");
  869. /* @b{Release the RAM view of the data before unregistering it and shutting down StarPU} */
  870. starpu_data_release(vector_handle);
  871. starpu_data_unregister(vector_handle);
  872. starpu_shutdown();
  873. return 0;
  874. @}
  875. @end smallexample
  876. @end cartouche
  877. @node Execution of Hybrid Vector Scaling
  878. @subsection Execution of Hybrid Vector Scaling
  879. The Makefile given at the beginning of the section must be extended to
  880. give the rules to compile the CUDA source code. Note that the source
  881. file of the OpenCL kernel does not need to be compiled now, it will
  882. be compiled at run-time when calling the function
  883. @code{starpu_opencl_load_opencl_from_file()} (@pxref{starpu_opencl_load_opencl_from_file}).
  884. @cartouche
  885. @smallexample
  886. CFLAGS += $(shell pkg-config --cflags libstarpu)
  887. LDFLAGS += $(shell pkg-config --libs libstarpu)
  888. CC = gcc
  889. vector_scal: vector_scal.o vector_scal_cpu.o vector_scal_cuda.o vector_scal_opencl.o
  890. %.o: %.cu
  891. nvcc $(CFLAGS) $< -c $@
  892. clean:
  893. rm -f vector_scal *.o
  894. @end smallexample
  895. @end cartouche
  896. @smallexample
  897. % make
  898. @end smallexample
  899. and to execute it, with the default configuration:
  900. @smallexample
  901. % ./vector_scal
  902. 0.000000 3.000000 6.000000 9.000000 12.000000
  903. @end smallexample
  904. or for example, by disabling CPU devices:
  905. @smallexample
  906. % STARPU_NCPUS=0 ./vector_scal
  907. 0.000000 3.000000 6.000000 9.000000 12.000000
  908. @end smallexample
  909. or by disabling CUDA devices (which may permit to enable the use of OpenCL,
  910. see @ref{Enabling OpenCL}):
  911. @smallexample
  912. % STARPU_NCUDA=0 ./vector_scal
  913. 0.000000 3.000000 6.000000 9.000000 12.000000
  914. @end smallexample
  915. @node Using multiple implentations of a codelet
  916. @section Using multiple implentations of a codelet
  917. One may want to write multiple implementations of a codelet for a single type of
  918. device and let StarPU choose which one to run. As an example, we will show how
  919. to use SSE to scale a vector. The codelet can be written as follows :
  920. @cartouche
  921. @smallexample
  922. #include <xmmintrin.h>
  923. void scal_sse_func(void *buffers[], void *cl_arg)
  924. @{
  925. float *vector = (float *) STARPU_VECTOR_GET_PTR(buffers[0]);
  926. unsigned int n = STARPU_VECTOR_GET_NX(buffers[0]);
  927. unsigned int n_iterations = n/4;
  928. if (n % 4 != 0)
  929. n_iterations++;
  930. __m128 *VECTOR = (__m128*) vector;
  931. __m128 factor __attribute__((aligned(16)));
  932. factor = _mm_set1_ps(*(float *) cl_arg);
  933. unsigned int i;
  934. for (i = 0; i < n_iterations; i++)
  935. VECTOR[i] = _mm_mul_ps(factor, VECTOR[i]);
  936. @}
  937. @end smallexample
  938. @end cartouche
  939. The @code{cpu_func} field of the @code{starpu_codelet} structure has to be set
  940. to the special value @code{STARPU_MULTIPLE_CPU_IMPLEMENTATIONS}. Note that
  941. @code{STARPU_MULTIPLE_CUDA_IMPLEMENTATIONS} and
  942. @code{STARPU_MULTIPLE_OPENCL_IMPLEMENTATIONS} are also available.
  943. @cartouche
  944. @smallexample
  945. starpu_codelet cl = @{
  946. .where = STARPU_CPU,
  947. .cpu_func = STARPU_MULTIPLE_CPU_IMPLEMENTATIONS,
  948. .cpu_funcs = @{ scal_cpu_func, scal_sse_func @},
  949. .nbuffers = 1
  950. @};
  951. @end smallexample
  952. @end cartouche
  953. @node Task and Worker Profiling
  954. @section Task and Worker Profiling
  955. A full example showing how to use the profiling API is available in
  956. the StarPU sources in the directory @code{examples/profiling/}.
  957. @cartouche
  958. @smallexample
  959. struct starpu_task *task = starpu_task_create();
  960. task->cl = &cl;
  961. task->synchronous = 1;
  962. /* We will destroy the task structure by hand so that we can
  963. * query the profiling info before the task is destroyed. */
  964. task->destroy = 0;
  965. /* Submit and wait for completion (since synchronous was set to 1) */
  966. starpu_task_submit(task);
  967. /* The task is finished, get profiling information */
  968. struct starpu_task_profiling_info *info = task->profiling_info;
  969. /* How much time did it take before the task started ? */
  970. double delay += starpu_timing_timespec_delay_us(&info->submit_time, &info->start_time);
  971. /* How long was the task execution ? */
  972. double length += starpu_timing_timespec_delay_us(&info->start_time, &info->end_time);
  973. /* We don't need the task structure anymore */
  974. starpu_task_destroy(task);
  975. @end smallexample
  976. @end cartouche
  977. @cartouche
  978. @smallexample
  979. /* Display the occupancy of all workers during the test */
  980. int worker;
  981. for (worker = 0; worker < starpu_worker_get_count(); worker++)
  982. @{
  983. struct starpu_worker_profiling_info worker_info;
  984. int ret = starpu_worker_get_profiling_info(worker, &worker_info);
  985. STARPU_ASSERT(!ret);
  986. double total_time = starpu_timing_timespec_to_us(&worker_info.total_time);
  987. double executing_time = starpu_timing_timespec_to_us(&worker_info.executing_time);
  988. double sleeping_time = starpu_timing_timespec_to_us(&worker_info.sleeping_time);
  989. float executing_ratio = 100.0*executing_time/total_time;
  990. float sleeping_ratio = 100.0*sleeping_time/total_time;
  991. char workername[128];
  992. starpu_worker_get_name(worker, workername, 128);
  993. fprintf(stderr, "Worker %s:\n", workername);
  994. fprintf(stderr, "\ttotal time : %.2lf ms\n", total_time*1e-3);
  995. fprintf(stderr, "\texec time : %.2lf ms (%.2f %%)\n", executing_time*1e-3,
  996. executing_ratio);
  997. fprintf(stderr, "\tblocked time : %.2lf ms (%.2f %%)\n", sleeping_time*1e-3,
  998. sleeping_ratio);
  999. @}
  1000. @end smallexample
  1001. @end cartouche
  1002. @node Partitioning Data
  1003. @section Partitioning Data
  1004. An existing piece of data can be partitioned in sub parts to be used by different tasks, for instance:
  1005. @cartouche
  1006. @smallexample
  1007. int vector[NX];
  1008. starpu_data_handle handle;
  1009. /* Declare data to StarPU */
  1010. starpu_vector_data_register(&handle, 0, (uintptr_t)vector, NX, sizeof(vector[0]));
  1011. /* Partition the vector in PARTS sub-vectors */
  1012. starpu_filter f =
  1013. @{
  1014. .filter_func = starpu_block_filter_func_vector,
  1015. .nchildren = PARTS
  1016. @};
  1017. starpu_data_partition(handle, &f);
  1018. @end smallexample
  1019. @end cartouche
  1020. @cartouche
  1021. @smallexample
  1022. /* Submit a task on each sub-vector */
  1023. for (i=0; i<starpu_data_get_nb_children(handle); i++) @{
  1024. /* Get subdata number i (there is only 1 dimension) */
  1025. starpu_data_handle sub_handle = starpu_data_get_sub_data(handle, 1, i);
  1026. struct starpu_task *task = starpu_task_create();
  1027. task->buffers[0].handle = sub_handle;
  1028. task->buffers[0].mode = STARPU_RW;
  1029. task->cl = &cl;
  1030. task->synchronous = 1;
  1031. task->cl_arg = &factor;
  1032. task->cl_arg_size = sizeof(factor);
  1033. starpu_task_submit(task);
  1034. @}
  1035. @end smallexample
  1036. @end cartouche
  1037. Partitioning can be applied several times, see
  1038. @code{examples/basic_examples/mult.c} and @code{examples/filters/}.
  1039. @node Performance model example
  1040. @section Performance model example
  1041. To achieve good scheduling, StarPU scheduling policies need to be able to
  1042. estimate in advance the duration of a task. This is done by giving to codelets
  1043. a performance model, by defining a @code{starpu_perfmodel_t} structure and
  1044. providing its address in the @code{model} field of the @code{starpu_codelet}
  1045. structure. The @code{symbol} and @code{type} fields of @code{starpu_perfmodel_t}
  1046. are mandatory, to give a name to the model, and the type of the model, since
  1047. there are several kinds of performance models.
  1048. @itemize
  1049. @item
  1050. Measured at runtime (@code{STARPU_HISTORY_BASED} model type). This assumes that for a
  1051. given set of data input/output sizes, the performance will always be about the
  1052. same. This is very true for regular kernels on GPUs for instance (<0.1% error),
  1053. and just a bit less true on CPUs (~=1% error). This also assumes that there are
  1054. few different sets of data input/output sizes. StarPU will then keep record of
  1055. the average time of previous executions on the various processing units, and use
  1056. it as an estimation. History is done per task size, by using a hash of the input
  1057. and ouput sizes as an index.
  1058. It will also save it in @code{~/.starpu/sampling/codelets}
  1059. for further executions, and can be observed by using the
  1060. @code{starpu_perfmodel_display} command, or drawn by using
  1061. the @code{starpu_perfmodel_plot}. The models are indexed by machine name. To
  1062. share the models between machines (e.g. for a homogeneous cluster), use
  1063. @code{export STARPU_HOSTNAME=some_global_name}. The following is a small code
  1064. example.
  1065. If e.g. the code is recompiled with other compilation options, or several
  1066. variants of the code are used, the symbol string should be changed to reflect
  1067. that, in order to recalibrate a new model from zero. The symbol string can even
  1068. be constructed dynamically at execution time, as long as this is done before
  1069. submitting any task using it.
  1070. @cartouche
  1071. @smallexample
  1072. static struct starpu_perfmodel_t mult_perf_model = @{
  1073. .type = STARPU_HISTORY_BASED,
  1074. .symbol = "mult_perf_model"
  1075. @};
  1076. starpu_codelet cl = @{
  1077. .where = STARPU_CPU,
  1078. .cpu_func = cpu_mult,
  1079. .nbuffers = 3,
  1080. /* for the scheduling policy to be able to use performance models */
  1081. .model = &mult_perf_model
  1082. @};
  1083. @end smallexample
  1084. @end cartouche
  1085. @item
  1086. Measured at runtime and refined by regression (@code{STARPU_REGRESSION_*_BASED}
  1087. model type). This still assumes performance regularity, but can work
  1088. with various data input sizes, by applying regression over observed
  1089. execution times. STARPU_REGRESSION_BASED uses an a*n^b regression
  1090. form, STARPU_NL_REGRESSION_BASED uses an a*n^b+c (more precise than
  1091. STARPU_REGRESSION_BASED, but costs a lot more to compute). For instance,
  1092. @code{tests/perfmodels/regression_based.c} uses a regression-based performance
  1093. model for the @code{memset} operation.
  1094. @item
  1095. Provided as an estimation from the application itself (@code{STARPU_COMMON} model type and @code{cost_model} field),
  1096. see for instance
  1097. @code{examples/common/blas_model.h} and @code{examples/common/blas_model.c}.
  1098. @item
  1099. Provided explicitly by the application (@code{STARPU_PER_ARCH} model type): the
  1100. @code{.per_arch[i].cost_model} fields have to be filled with pointers to
  1101. functions which return the expected duration of the task in micro-seconds, one
  1102. per architecture.
  1103. @end itemize
  1104. How to use schedulers which can benefit from such performance model is explained
  1105. in @ref{Task scheduling policy}.
  1106. The same can be done for task power consumption estimation, by setting the
  1107. @code{power_model} field the same way as the @code{model} field. Note: for
  1108. now, the application has to give to the power consumption performance model
  1109. a name which is different from the execution time performance model.
  1110. The application can request time estimations from the StarPU performance
  1111. models by filling a task structure as usual without actually submitting
  1112. it. The data handles can be created by calling @code{starpu_data_register}
  1113. functions with a @code{NULL} pointer (and need to be unregistered as usual)
  1114. and the desired data sizes. The @code{starpu_task_expected_length} and
  1115. @code{starpu_task_expected_power} functions can then be called to get an
  1116. estimation of the task duration on a given arch. @code{starpu_task_destroy}
  1117. needs to be called to destroy the dummy task afterwards. See
  1118. @code{tests/perfmodels/regression_based.c} for an example.
  1119. @node Theoretical lower bound on execution time
  1120. @section Theoretical lower bound on execution time
  1121. For kernels with history-based performance models, StarPU can very easily provide a theoretical lower
  1122. bound for the execution time of a whole set of tasks. See for
  1123. instance @code{examples/lu/lu_example.c}: before submitting tasks,
  1124. call @code{starpu_bound_start}, and after complete execution, call
  1125. @code{starpu_bound_stop}. @code{starpu_bound_print_lp} or
  1126. @code{starpu_bound_print_mps} can then be used to output a Linear Programming
  1127. problem corresponding to the schedule of your tasks. Run it through
  1128. @code{lp_solve} or any other linear programming solver, and that will give you a
  1129. lower bound for the total execution time of your tasks. If StarPU was compiled
  1130. with the glpk library installed, @code{starpu_bound_compute} can be used to
  1131. solve it immediately and get the optimized minimum. Its @code{integer}
  1132. parameter allows to decide whether integer resolution should be computed
  1133. and returned.
  1134. The @code{deps} parameter tells StarPU whether to take tasks and implicit data
  1135. dependencies into account. It must be understood that the linear programming
  1136. problem size is quadratic with the number of tasks and thus the time to solve it
  1137. will be very long, it could be minutes for just a few dozen tasks. You should
  1138. probably use @code{lp_solve -timeout 1 test.pl -wmps test.mps} to convert the
  1139. problem to MPS format and then use a better solver, @code{glpsol} might be
  1140. better than @code{lp_solve} for instance (the @code{--pcost} option may be
  1141. useful), but sometimes doesn't manage to converge. @code{cbc} might look
  1142. slower, but it is parallel. Be sure to try at least all the @code{-B} options
  1143. of @code{lp_solve}. For instance, we often just use
  1144. @code{lp_solve -cc -B1 -Bb -Bg -Bp -Bf -Br -BG -Bd -Bs -BB -Bo -Bc -Bi} , and
  1145. the @code{-gr} option can also be quite useful.
  1146. Setting @code{deps} to 0 will only take into account the actual computations
  1147. on processing units. It however still properly takes into account the varying
  1148. performances of kernels and processing units, which is quite more accurate than
  1149. just comparing StarPU performances with the fastest of the kernels being used.
  1150. The @code{prio} parameter tells StarPU whether to simulate taking into account
  1151. the priorities as the StarPU scheduler would, i.e. schedule prioritized
  1152. tasks before less prioritized tasks, to check to which extend this results
  1153. to a less optimal solution. This increases even more computation time.
  1154. Note that for simplicity, all this however doesn't take into account data
  1155. transfers, which are assumed to be completely overlapped.
  1156. @node Insert Task Utility
  1157. @section Insert Task Utility
  1158. StarPU provides the wrapper function @code{starpu_insert_task} to ease
  1159. the creation and submission of tasks.
  1160. @deftypefun int starpu_insert_task (starpu_codelet *@var{cl}, ...)
  1161. Create and submit a task corresponding to @var{cl} with the following
  1162. arguments. The argument list must be zero-terminated.
  1163. The arguments following the codelets can be of the following types:
  1164. @itemize
  1165. @item
  1166. @code{STARPU_R}, @code{STARPU_W}, @code{STARPU_RW}, @code{STARPU_SCRATCH}, @code{STARPU_REDUX} an access mode followed by a data handle;
  1167. @item
  1168. @code{STARPU_VALUE} followed by a pointer to a constant value and
  1169. the size of the constant;
  1170. @item
  1171. @code{STARPU_CALLBACK} followed by a pointer to a callback function;
  1172. @item
  1173. @code{STARPU_CALLBACK_ARG} followed by a pointer to be given as an
  1174. argument to the callback function;
  1175. @item
  1176. @code{STARPU_PRIORITY} followed by a integer defining a priority level.
  1177. @end itemize
  1178. Parameters to be passed to the codelet implementation are defined
  1179. through the type @code{STARPU_VALUE}. The function
  1180. @code{starpu_unpack_cl_args} must be called within the codelet
  1181. implementation to retrieve them.
  1182. @end deftypefun
  1183. Here the implementation of the codelet:
  1184. @smallexample
  1185. void func_cpu(void *descr[], void *_args)
  1186. @{
  1187. int *x0 = (int *)STARPU_VARIABLE_GET_PTR(descr[0]);
  1188. float *x1 = (float *)STARPU_VARIABLE_GET_PTR(descr[1]);
  1189. int ifactor;
  1190. float ffactor;
  1191. starpu_unpack_cl_args(_args, &ifactor, &ffactor);
  1192. *x0 = *x0 * ifactor;
  1193. *x1 = *x1 * ffactor;
  1194. @}
  1195. starpu_codelet mycodelet = @{
  1196. .where = STARPU_CPU,
  1197. .cpu_func = func_cpu,
  1198. .nbuffers = 2
  1199. @};
  1200. @end smallexample
  1201. And the call to the @code{starpu_insert_task} wrapper:
  1202. @smallexample
  1203. starpu_insert_task(&mycodelet,
  1204. STARPU_VALUE, &ifactor, sizeof(ifactor),
  1205. STARPU_VALUE, &ffactor, sizeof(ffactor),
  1206. STARPU_RW, data_handles[0], STARPU_RW, data_handles[1],
  1207. 0);
  1208. @end smallexample
  1209. The call to @code{starpu_insert_task} is equivalent to the following
  1210. code:
  1211. @smallexample
  1212. struct starpu_task *task = starpu_task_create();
  1213. task->cl = &mycodelet;
  1214. task->buffers[0].handle = data_handles[0];
  1215. task->buffers[0].mode = STARPU_RW;
  1216. task->buffers[1].handle = data_handles[1];
  1217. task->buffers[1].mode = STARPU_RW;
  1218. char *arg_buffer;
  1219. size_t arg_buffer_size;
  1220. starpu_pack_cl_args(&arg_buffer, &arg_buffer_size,
  1221. STARPU_VALUE, &ifactor, sizeof(ifactor),
  1222. STARPU_VALUE, &ffactor, sizeof(ffactor),
  1223. 0);
  1224. task->cl_arg = arg_buffer;
  1225. task->cl_arg_size = arg_buffer_size;
  1226. int ret = starpu_task_submit(task);
  1227. @end smallexample
  1228. If some part of the task insertion depends on the value of some computation,
  1229. the @code{STARPU_DATA_ACQUIRE_CB} macro can be very convenient. For
  1230. instance, assuming that the index variable @code{i} was registered as handle
  1231. @code{i_handle}:
  1232. @smallexample
  1233. /* Compute which portion we will work on, e.g. pivot */
  1234. starpu_insert_task(&which_index, STARPU_W, i_handle, 0);
  1235. /* And submit the corresponding task */
  1236. STARPU_DATA_ACQUIRE_CB(i_handle, STARPU_R, starpu_insert_task(&work, STARPU_RW, A_handle[i], 0));
  1237. @end smallexample
  1238. The @code{STARPU_DATA_ACQUIRE_CB} macro submits an asynchronous request for
  1239. acquiring data @code{i} for the main application, and will execute the code
  1240. given as third parameter when it is acquired. In other words, as soon as the
  1241. value of @code{i} computed by the @code{which_index} codelet can be read, the
  1242. portion of code passed as third parameter of @code{STARPU_DATA_ACQUIRE_CB} will
  1243. be executed, and is allowed to read from @code{i} to use it e.g. as an
  1244. index. Note that this macro is only avaible when compiling StarPU with
  1245. the compiler @code{gcc}.
  1246. @node Debugging
  1247. @section Debugging
  1248. StarPU provides several tools to help debugging aplications. Execution traces
  1249. can be generated and displayed graphically, see @ref{Generating traces}. Some
  1250. gdb helpers are also provided to show the whole StarPU state:
  1251. @smallexample
  1252. (gdb) source tools/gdbinit
  1253. (gdb) help starpu
  1254. @end smallexample
  1255. @node More examples
  1256. @section More examples
  1257. More examples are available in the StarPU sources in the @code{examples/}
  1258. directory. Simple examples include:
  1259. @table @asis
  1260. @item @code{incrementer/}:
  1261. Trivial incrementation test.
  1262. @item @code{basic_examples/}:
  1263. Simple documented Hello world (as shown in @ref{Hello World}), vector/scalar product (as shown
  1264. in @ref{Vector Scaling on an Hybrid CPU/GPU Machine}), matrix
  1265. product examples (as shown in @ref{Performance model example}), an example using the blocked matrix data
  1266. interface, and an example using the variable data interface.
  1267. @item @code{matvecmult/}:
  1268. OpenCL example from NVidia, adapted to StarPU.
  1269. @item @code{axpy/}:
  1270. AXPY CUBLAS operation adapted to StarPU.
  1271. @item @code{fortran/}:
  1272. Example of Fortran bindings.
  1273. @end table
  1274. More advanced examples include:
  1275. @table @asis
  1276. @item @code{filters/}:
  1277. Examples using filters, as shown in @ref{Partitioning Data}.
  1278. @item @code{lu/}:
  1279. LU matrix factorization, see for instance @code{xlu_implicit.c}
  1280. @item @code{cholesky/}:
  1281. Cholesky matrix factorization, see for instance @code{cholesky_implicit.c}.
  1282. @end table
  1283. @c ---------------------------------------------------------------------
  1284. @c Performance options
  1285. @c ---------------------------------------------------------------------
  1286. @node Performance optimization
  1287. @chapter How to optimize performance with StarPU
  1288. TODO: improve!
  1289. @menu
  1290. * Data management::
  1291. * Task submission::
  1292. * Task priorities::
  1293. * Task scheduling policy::
  1294. * Performance model calibration::
  1295. * Task distribution vs Data transfer::
  1296. * Data prefetch::
  1297. * Power-based scheduling::
  1298. * Profiling::
  1299. * CUDA-specific optimizations::
  1300. @end menu
  1301. Simply encapsulating application kernels into tasks already permits to
  1302. seamlessly support CPU and GPUs at the same time. To achieve good performance, a
  1303. few additional changes are needed.
  1304. @node Data management
  1305. @section Data management
  1306. When the application allocates data, whenever possible it should use the
  1307. @code{starpu_malloc} function, which will ask CUDA or
  1308. OpenCL to make the allocation itself and pin the corresponding allocated
  1309. memory. This is needed to permit asynchronous data transfer, i.e. permit data
  1310. transfer to overlap with computations. Otherwise, the trace will show that the
  1311. @code{DriverCopyAsync} state takes a lot of time, this is because CUDA or OpenCL
  1312. then reverts to synchronous transfers.
  1313. By default, StarPU leaves replicates of data wherever they were used, in case they
  1314. will be re-used by other tasks, thus saving the data transfer time. When some
  1315. task modifies some data, all the other replicates are invalidated, and only the
  1316. processing unit which ran that task will have a valid replicate of the data. If the application knows
  1317. that this data will not be re-used by further tasks, it should advise StarPU to
  1318. immediately replicate it to a desired list of memory nodes (given through a
  1319. bitmask). This can be understood like the write-through mode of CPU caches.
  1320. @example
  1321. starpu_data_set_wt_mask(img_handle, 1<<0);
  1322. @end example
  1323. will for instance request to always transfer a replicate into the main memory (node
  1324. 0), as bit 0 of the write-through bitmask is being set.
  1325. @node Task submission
  1326. @section Task submission
  1327. To let StarPU make online optimizations, tasks should be submitted
  1328. asynchronously as much as possible. Ideally, all the tasks should be
  1329. submitted, and mere calls to @code{starpu_task_wait_for_all} or
  1330. @code{starpu_data_unregister} be done to wait for
  1331. termination. StarPU will then be able to rework the whole schedule, overlap
  1332. computation with communication, manage accelerator local memory usage, etc.
  1333. @node Task priorities
  1334. @section Task priorities
  1335. By default, StarPU will consider the tasks in the order they are submitted by
  1336. the application. If the application programmer knows that some tasks should
  1337. be performed in priority (for instance because their output is needed by many
  1338. other tasks and may thus be a bottleneck if not executed early enough), the
  1339. @code{priority} field of the task structure should be set to transmit the
  1340. priority information to StarPU.
  1341. @node Task scheduling policy
  1342. @section Task scheduling policy
  1343. By default, StarPU uses the @code{eager} simple greedy scheduler. This is
  1344. because it provides correct load balance even if the application codelets do not
  1345. have performance models. If your application codelets have performance models
  1346. (@pxref{Performance model example} for examples showing how to do it),
  1347. you should change the scheduler thanks to the @code{STARPU_SCHED} environment
  1348. variable. For instance @code{export STARPU_SCHED=dmda} . Use @code{help} to get
  1349. the list of available schedulers.
  1350. The @b{eager} scheduler uses a central task queue, from which workers draw tasks
  1351. to work on. This however does not permit to prefetch data since the scheduling
  1352. decision is taken late. If a task has a non-0 priority, it is put at the front of the queue.
  1353. The @b{prio} scheduler also uses a central task queue, but sorts tasks by
  1354. priority (between -5 and 5).
  1355. The @b{random} scheduler distributes tasks randomly according to assumed worker
  1356. overall performance.
  1357. The @b{ws} (work stealing) scheduler schedules tasks on the local worker by
  1358. default. When a worker becomes idle, it steals a task from the most loaded
  1359. worker.
  1360. The @b{dm} (deque model) scheduler uses task execution performance models into account to
  1361. perform an HEFT-similar scheduling strategy: it schedules tasks where their
  1362. termination time will be minimal.
  1363. The @b{dmda} (deque model data aware) scheduler is similar to dm, it also takes
  1364. into account data transfer time.
  1365. The @b{dmdar} (deque model data aware ready) scheduler is similar to dmda,
  1366. it also sorts tasks on per-worker queues by number of already-available data
  1367. buffers.
  1368. The @b{dmdas} (deque model data aware sorted) scheduler is similar to dmda, it
  1369. also supports arbitrary priority values.
  1370. The @b{heft} (HEFT) scheduler is similar to dmda, it also supports task bundles.
  1371. The @b{pheft} (parallel HEFT) scheduler is similar to heft, it also supports
  1372. parallel tasks (still experimental).
  1373. The @b{pgreedy} (parallel greedy) scheduler is similar to greedy, it also
  1374. supports parallel tasks (still experimental).
  1375. @node Performance model calibration
  1376. @section Performance model calibration
  1377. Most schedulers are based on an estimation of codelet duration on each kind
  1378. of processing unit. For this to be possible, the application programmer needs
  1379. to configure a performance model for the codelets of the application (see
  1380. @ref{Performance model example} for instance). History-based performance models
  1381. use on-line calibration. StarPU will automatically calibrate codelets
  1382. which have never been calibrated yet, and save the result in
  1383. @code{~/.starpu/sampling/codelets}.
  1384. The models are indexed by machine name. To share the models between machines (e.g. for a homogeneous cluster), use @code{export STARPU_HOSTNAME=some_global_name}. To force continuing calibration, use
  1385. @code{export STARPU_CALIBRATE=1} . This may be necessary if your application
  1386. has not-so-stable performance. StarPU will force calibration (and thus ignore
  1387. the current result) until 10 (STARPU_CALIBRATION_MINIMUM) measurements have been
  1388. made on each architecture, to avoid badly scheduling tasks just because the
  1389. first measurements were not so good. Details on the current performance model status
  1390. can be obtained from the @code{starpu_perfmodel_display} command: the @code{-l}
  1391. option lists the available performance models, and the @code{-s} option permits
  1392. to choose the performance model to be displayed. The result looks like:
  1393. @example
  1394. $ starpu_perfmodel_display -s starpu_dlu_lu_model_22
  1395. performance model for cpu
  1396. # hash size mean dev n
  1397. 880805ba 98304 2.731309e+02 6.010210e+01 1240
  1398. b50b6605 393216 1.469926e+03 1.088828e+02 1240
  1399. 5c6c3401 1572864 1.125983e+04 3.265296e+03 1240
  1400. @end example
  1401. Which shows that for the LU 22 kernel with a 1.5MiB matrix, the average
  1402. execution time on CPUs was about 12ms, with a 2ms standard deviation, over
  1403. 1240 samples. It is a good idea to check this before doing actual performance
  1404. measurements.
  1405. A graph can be drawn by using the @code{starpu_perfmodel_plot}:
  1406. @example
  1407. $ starpu_perfmodel_plot -s starpu_dlu_lu_model_22
  1408. 98304 393216 1572864
  1409. $ gnuplot starpu_starpu_dlu_lu_model_22.gp
  1410. $ gv starpu_starpu_dlu_lu_model_22.eps
  1411. @end example
  1412. If a kernel source code was modified (e.g. performance improvement), the
  1413. calibration information is stale and should be dropped, to re-calibrate from
  1414. start. This can be done by using @code{export STARPU_CALIBRATE=2}.
  1415. Note: due to CUDA limitations, to be able to measure kernel duration,
  1416. calibration mode needs to disable asynchronous data transfers. Calibration thus
  1417. disables data transfer / computation overlapping, and should thus not be used
  1418. for eventual benchmarks. Note 2: history-based performance models get calibrated
  1419. only if a performance-model-based scheduler is chosen.
  1420. @node Task distribution vs Data transfer
  1421. @section Task distribution vs Data transfer
  1422. Distributing tasks to balance the load induces data transfer penalty. StarPU
  1423. thus needs to find a balance between both. The target function that the
  1424. @code{dmda} scheduler of StarPU
  1425. tries to minimize is @code{alpha * T_execution + beta * T_data_transfer}, where
  1426. @code{T_execution} is the estimated execution time of the codelet (usually
  1427. accurate), and @code{T_data_transfer} is the estimated data transfer time. The
  1428. latter is estimated based on bus calibration before execution start,
  1429. i.e. with an idle machine, thus without contention. You can force bus re-calibration by running
  1430. @code{starpu_calibrate_bus}. The beta parameter defaults to 1, but it can be
  1431. worth trying to tweak it by using @code{export STARPU_BETA=2} for instance,
  1432. since during real application execution, contention makes transfer times bigger.
  1433. This is of course imprecise, but in practice, a rough estimation already gives
  1434. the good results that a precise estimation would give.
  1435. @node Data prefetch
  1436. @section Data prefetch
  1437. The @code{heft}, @code{dmda} and @code{pheft} scheduling policies perform data prefetch (see @ref{STARPU_PREFETCH}):
  1438. as soon as a scheduling decision is taken for a task, requests are issued to
  1439. transfer its required data to the target processing unit, if needeed, so that
  1440. when the processing unit actually starts the task, its data will hopefully be
  1441. already available and it will not have to wait for the transfer to finish.
  1442. The application may want to perform some manual prefetching, for several reasons
  1443. such as excluding initial data transfers from performance measurements, or
  1444. setting up an initial statically-computed data distribution on the machine
  1445. before submitting tasks, which will thus guide StarPU toward an initial task
  1446. distribution (since StarPU will try to avoid further transfers).
  1447. This can be achieved by giving the @code{starpu_data_prefetch_on_node} function
  1448. the handle and the desired target memory node.
  1449. @node Power-based scheduling
  1450. @section Power-based scheduling
  1451. If the application can provide some power performance model (through
  1452. the @code{power_model} field of the codelet structure), StarPU will
  1453. take it into account when distributing tasks. The target function that
  1454. the @code{dmda} scheduler minimizes becomes @code{alpha * T_execution +
  1455. beta * T_data_transfer + gamma * Consumption} , where @code{Consumption}
  1456. is the estimated task consumption in Joules. To tune this parameter, use
  1457. @code{export STARPU_GAMMA=3000} for instance, to express that each Joule
  1458. (i.e kW during 1000us) is worth 3000us execution time penalty. Setting
  1459. @code{alpha} and @code{beta} to zero permits to only take into account power consumption.
  1460. This is however not sufficient to correctly optimize power: the scheduler would
  1461. simply tend to run all computations on the most energy-conservative processing
  1462. unit. To account for the consumption of the whole machine (including idle
  1463. processing units), the idle power of the machine should be given by setting
  1464. @code{export STARPU_IDLE_POWER=200} for 200W, for instance. This value can often
  1465. be obtained from the machine power supplier.
  1466. The power actually consumed by the total execution can be displayed by setting
  1467. @code{export STARPU_PROFILING=1 STARPU_WORKER_STATS=1} .
  1468. @node Profiling
  1469. @section Profiling
  1470. A quick view of how many tasks each worker has executed can be obtained by setting
  1471. @code{export STARPU_WORKER_STATS=1} This is a convenient way to check that
  1472. execution did happen on accelerators without penalizing performance with
  1473. the profiling overhead.
  1474. A quick view of how much data transfers have been issued can be obtained by setting
  1475. @code{export STARPU_BUS_STATS=1} .
  1476. More detailed profiling information can be enabled by using @code{export STARPU_PROFILING=1} or by
  1477. calling @code{starpu_profiling_status_set} from the source code.
  1478. Statistics on the execution can then be obtained by using @code{export
  1479. STARPU_BUS_STATS=1} and @code{export STARPU_WORKER_STATS=1} .
  1480. More details on performance feedback are provided by the next chapter.
  1481. @node CUDA-specific optimizations
  1482. @section CUDA-specific optimizations
  1483. Due to CUDA limitations, StarPU will have a hard time overlapping its own
  1484. communications and the codelet computations if the application does not use a
  1485. dedicated CUDA stream for its computations. StarPU provides one by the use of
  1486. @code{starpu_cuda_get_local_stream()} which should be used by all CUDA codelet
  1487. operations. For instance:
  1488. @example
  1489. func <<<grid,block,0,starpu_cuda_get_local_stream()>>> (foo, bar);
  1490. cudaStreamSynchronize(starpu_cuda_get_local_stream());
  1491. @end example
  1492. StarPU already does appropriate calls for the CUBLAS library.
  1493. Unfortunately, some CUDA libraries do not have stream variants of
  1494. kernels. That will lower the potential for overlapping.
  1495. @c ---------------------------------------------------------------------
  1496. @c Performance feedback
  1497. @c ---------------------------------------------------------------------
  1498. @node Performance feedback
  1499. @chapter Performance feedback
  1500. @menu
  1501. * On-line:: On-line performance feedback
  1502. * Off-line:: Off-line performance feedback
  1503. * Codelet performance:: Performance of codelets
  1504. @end menu
  1505. @node On-line
  1506. @section On-line performance feedback
  1507. @menu
  1508. * Enabling monitoring:: Enabling on-line performance monitoring
  1509. * Task feedback:: Per-task feedback
  1510. * Codelet feedback:: Per-codelet feedback
  1511. * Worker feedback:: Per-worker feedback
  1512. * Bus feedback:: Bus-related feedback
  1513. * StarPU-Top:: StarPU-Top interface
  1514. @end menu
  1515. @node Enabling monitoring
  1516. @subsection Enabling on-line performance monitoring
  1517. In order to enable online performance monitoring, the application can call
  1518. @code{starpu_profiling_status_set(STARPU_PROFILING_ENABLE)}. It is possible to
  1519. detect whether monitoring is already enabled or not by calling
  1520. @code{starpu_profiling_status_get()}. Enabling monitoring also reinitialize all
  1521. previously collected feedback. The @code{STARPU_PROFILING} environment variable
  1522. can also be set to 1 to achieve the same effect.
  1523. Likewise, performance monitoring is stopped by calling
  1524. @code{starpu_profiling_status_set(STARPU_PROFILING_DISABLE)}. Note that this
  1525. does not reset the performance counters so that the application may consult
  1526. them later on.
  1527. More details about the performance monitoring API are available in section
  1528. @ref{Profiling API}.
  1529. @node Task feedback
  1530. @subsection Per-task feedback
  1531. If profiling is enabled, a pointer to a @code{starpu_task_profiling_info}
  1532. structure is put in the @code{.profiling_info} field of the @code{starpu_task}
  1533. structure when a task terminates.
  1534. This structure is automatically destroyed when the task structure is destroyed,
  1535. either automatically or by calling @code{starpu_task_destroy}.
  1536. The @code{starpu_task_profiling_info} structure indicates the date when the
  1537. task was submitted (@code{submit_time}), started (@code{start_time}), and
  1538. terminated (@code{end_time}), relative to the initialization of
  1539. StarPU with @code{starpu_init}. It also specifies the identifier of the worker
  1540. that has executed the task (@code{workerid}).
  1541. These date are stored as @code{timespec} structures which the user may convert
  1542. into micro-seconds using the @code{starpu_timing_timespec_to_us} helper
  1543. function.
  1544. It it worth noting that the application may directly access this structure from
  1545. the callback executed at the end of the task. The @code{starpu_task} structure
  1546. associated to the callback currently being executed is indeed accessible with
  1547. the @code{starpu_get_current_task()} function.
  1548. @node Codelet feedback
  1549. @subsection Per-codelet feedback
  1550. The @code{per_worker_stats} field of the @code{starpu_codelet_t} structure is
  1551. an array of counters. The i-th entry of the array is incremented every time a
  1552. task implementing the codelet is executed on the i-th worker.
  1553. This array is not reinitialized when profiling is enabled or disabled.
  1554. @node Worker feedback
  1555. @subsection Per-worker feedback
  1556. The second argument returned by the @code{starpu_worker_get_profiling_info}
  1557. function is a @code{starpu_worker_profiling_info} structure that gives
  1558. statistics about the specified worker. This structure specifies when StarPU
  1559. started collecting profiling information for that worker (@code{start_time}),
  1560. the duration of the profiling measurement interval (@code{total_time}), the
  1561. time spent executing kernels (@code{executing_time}), the time spent sleeping
  1562. because there is no task to execute at all (@code{sleeping_time}), and the
  1563. number of tasks that were executed while profiling was enabled.
  1564. These values give an estimation of the proportion of time spent do real work,
  1565. and the time spent either sleeping because there are not enough executable
  1566. tasks or simply wasted in pure StarPU overhead.
  1567. Calling @code{starpu_worker_get_profiling_info} resets the profiling
  1568. information associated to a worker.
  1569. When an FxT trace is generated (see @ref{Generating traces}), it is also
  1570. possible to use the @code{starpu_top} script (described in @ref{starpu-top}) to
  1571. generate a graphic showing the evolution of these values during the time, for
  1572. the different workers.
  1573. @node Bus feedback
  1574. @subsection Bus-related feedback
  1575. TODO
  1576. @c how to enable/disable performance monitoring
  1577. @c what kind of information do we get ?
  1578. The bus speed measured by StarPU can be displayed by using the
  1579. @code{starpu_machine_display} tool, for instance:
  1580. @example
  1581. StarPU has found :
  1582. 3 CUDA devices
  1583. CUDA 0 (Tesla C2050 02:00.0)
  1584. CUDA 1 (Tesla C2050 03:00.0)
  1585. CUDA 2 (Tesla C2050 84:00.0)
  1586. from to RAM to CUDA 0 to CUDA 1 to CUDA 2
  1587. RAM 0.000000 5176.530428 5176.492994 5191.710722
  1588. CUDA 0 4523.732446 0.000000 2414.074751 2417.379201
  1589. CUDA 1 4523.718152 2414.078822 0.000000 2417.375119
  1590. CUDA 2 4534.229519 2417.069025 2417.060863 0.000000
  1591. @end example
  1592. @node StarPU-Top
  1593. @subsection StarPU-Top interface
  1594. StarPU-Top is an interface which remotely displays the on-line state of a StarPU
  1595. application and permits the user to change parameters on the fly.
  1596. Variables to be monitored can be registered by calling the
  1597. @code{starputop_add_data_boolean}, @code{starputop_add_data_integer},
  1598. @code{starputop_add_data_float} functions, e.g.:
  1599. @example
  1600. starputop_data *data = starputop_add_data_integer("mynum", 0, 100, 1);
  1601. @end example
  1602. The application should then call @code{starputop_init_and_wait} to give its name
  1603. and wait for StarPU-Top to get a start request from the user. The name is used
  1604. by StarPU-Top to quickly reload a previously-saved layout of parameter display.
  1605. @example
  1606. starputop_init_and_wait("the application");
  1607. @end example
  1608. The new values can then be provided thanks to
  1609. @code{starputop_update_data_boolean}, @code{starputop_update_data_integer},
  1610. @code{starputop_update_data_float}, e.g.:
  1611. @example
  1612. starputop_update_data_integer(data, mynum);
  1613. @end example
  1614. Updateable parameters can be registered thanks to @code{starputop_register_parameter_boolean}, @code{starputop_register_parameter_integer}, @code{starputop_register_parameter_float}, e.g.:
  1615. @example
  1616. float apha;
  1617. starputop_register_parameter_float("alpha", &alpha, 0, 10, modif_hook);
  1618. @end example
  1619. @code{modif_hook} is a function which will be called when the parameter is being modified, it can for instance print the new value:
  1620. @example
  1621. void modif_hook(struct starputop_param_t *d) @{
  1622. fprintf(stderr,"%s has been modified: %f\n", d->name, alpha);
  1623. @}
  1624. @end example
  1625. Task schedulers should notify StarPU-Top when it has decided when a task will be
  1626. scheduled, so that it can show it in its Gantt chart, for instance:
  1627. @example
  1628. starputop_task_prevision(task, workerid, begin, end);
  1629. @end example
  1630. Starting StarPU-Top and the application can be done two ways:
  1631. @itemize
  1632. @item The application is started by hand on some machine (and thus already
  1633. waiting for the start event). In the Preference dialog of StarPU-Top, the SSH
  1634. checkbox should be unchecked, and the hostname and port (default is 2011) on
  1635. which the application is already running should be specified. Clicking on the
  1636. connection button will thus connect to the already-running application.
  1637. @item StarPU-Top is started first, and clicking on the connection button will
  1638. start the application itself (possibly on a remote machine). The SSH checkbox
  1639. should be checked, and a command line provided, e.g.:
  1640. @example
  1641. ssh myserver STARPU_SCHED=heft ./application
  1642. @end example
  1643. If port 2011 of the remote machine can not be accessed directly, an ssh port bridge should be added:
  1644. @example
  1645. ssh -L 2011:localhost:2011 myserver STARPU_SCHED=heft ./application
  1646. @end example
  1647. and "localhost" should be used as IP Address to connect to.
  1648. @end itemize
  1649. @node Off-line
  1650. @section Off-line performance feedback
  1651. @menu
  1652. * Generating traces:: Generating traces with FxT
  1653. * Gantt diagram:: Creating a Gantt Diagram
  1654. * DAG:: Creating a DAG with graphviz
  1655. * starpu-top:: Monitoring activity
  1656. @end menu
  1657. @node Generating traces
  1658. @subsection Generating traces with FxT
  1659. StarPU can use the FxT library (see
  1660. @indicateurl{https://savannah.nongnu.org/projects/fkt/}) to generate traces
  1661. with a limited runtime overhead.
  1662. You can either get a tarball:
  1663. @example
  1664. % wget http://download.savannah.gnu.org/releases/fkt/fxt-0.2.2.tar.gz
  1665. @end example
  1666. or use the FxT library from CVS (autotools are required):
  1667. @example
  1668. % cvs -d :pserver:anonymous@@cvs.sv.gnu.org:/sources/fkt co FxT
  1669. % ./bootstrap
  1670. @end example
  1671. Compiling and installing the FxT library in the @code{$FXTDIR} path is
  1672. done following the standard procedure:
  1673. @example
  1674. % ./configure --prefix=$FXTDIR
  1675. % make
  1676. % make install
  1677. @end example
  1678. In order to have StarPU to generate traces, StarPU should be configured with
  1679. the @code{--with-fxt} option:
  1680. @example
  1681. $ ./configure --with-fxt=$FXTDIR
  1682. @end example
  1683. Or you can simply point the @code{PKG_CONFIG_PATH} to
  1684. @code{$FXTDIR/lib/pkgconfig} and pass @code{--with-fxt} to @code{./configure}
  1685. When FxT is enabled, a trace is generated when StarPU is terminated by calling
  1686. @code{starpu_shutdown()}). The trace is a binary file whose name has the form
  1687. @code{prof_file_XXX_YYY} where @code{XXX} is the user name, and
  1688. @code{YYY} is the pid of the process that used StarPU. This file is saved in the
  1689. @code{/tmp/} directory by default, or by the directory specified by
  1690. the @code{STARPU_FXT_PREFIX} environment variable.
  1691. @node Gantt diagram
  1692. @subsection Creating a Gantt Diagram
  1693. When the FxT trace file @code{filename} has been generated, it is possible to
  1694. generate a trace in the Paje format by calling:
  1695. @example
  1696. % starpu_fxt_tool -i filename
  1697. @end example
  1698. Or alternatively, setting the @code{STARPU_GENERATE_TRACE} environment variable
  1699. to 1 before application execution will make StarPU do it automatically at
  1700. application shutdown.
  1701. This will create a @code{paje.trace} file in the current directory that can be
  1702. inspected with the ViTE trace visualizing open-source tool. More information
  1703. about ViTE is available at @indicateurl{http://vite.gforge.inria.fr/}. It is
  1704. possible to open the @code{paje.trace} file with ViTE by using the following
  1705. command:
  1706. @example
  1707. % vite paje.trace
  1708. @end example
  1709. @node DAG
  1710. @subsection Creating a DAG with graphviz
  1711. When the FxT trace file @code{filename} has been generated, it is possible to
  1712. generate a task graph in the DOT format by calling:
  1713. @example
  1714. $ starpu_fxt_tool -i filename
  1715. @end example
  1716. This will create a @code{dag.dot} file in the current directory. This file is a
  1717. task graph described using the DOT language. It is possible to get a
  1718. graphical output of the graph by using the graphviz library:
  1719. @example
  1720. $ dot -Tpdf dag.dot -o output.pdf
  1721. @end example
  1722. @node starpu-top
  1723. @subsection Monitoring activity
  1724. When the FxT trace file @code{filename} has been generated, it is possible to
  1725. generate a activity trace by calling:
  1726. @example
  1727. $ starpu_fxt_tool -i filename
  1728. @end example
  1729. This will create an @code{activity.data} file in the current
  1730. directory. A profile of the application showing the activity of StarPU
  1731. during the execution of the program can be generated:
  1732. @example
  1733. $ starpu_top activity.data
  1734. @end example
  1735. This will create a file named @code{activity.eps} in the current directory.
  1736. This picture is composed of two parts.
  1737. The first part shows the activity of the different workers. The green sections
  1738. indicate which proportion of the time was spent executed kernels on the
  1739. processing unit. The red sections indicate the proportion of time spent in
  1740. StartPU: an important overhead may indicate that the granularity may be too
  1741. low, and that bigger tasks may be appropriate to use the processing unit more
  1742. efficiently. The black sections indicate that the processing unit was blocked
  1743. because there was no task to process: this may indicate a lack of parallelism
  1744. which may be alleviated by creating more tasks when it is possible.
  1745. The second part of the @code{activity.eps} picture is a graph showing the
  1746. evolution of the number of tasks available in the system during the execution.
  1747. Ready tasks are shown in black, and tasks that are submitted but not
  1748. schedulable yet are shown in grey.
  1749. @node Codelet performance
  1750. @section Performance of codelets
  1751. The performance model of codelets can be examined by using the
  1752. @code{starpu_perfmodel_display} tool:
  1753. @example
  1754. $ starpu_perfmodel_display -l
  1755. file: <malloc_pinned.hannibal>
  1756. file: <starpu_slu_lu_model_21.hannibal>
  1757. file: <starpu_slu_lu_model_11.hannibal>
  1758. file: <starpu_slu_lu_model_22.hannibal>
  1759. file: <starpu_slu_lu_model_12.hannibal>
  1760. @end example
  1761. Here, the codelets of the lu example are available. We can examine the
  1762. performance of the 22 kernel:
  1763. @example
  1764. $ starpu_perfmodel_display -s starpu_slu_lu_model_22
  1765. performance model for cpu
  1766. # hash size mean dev n
  1767. 57618ab0 19660800 2.851069e+05 1.829369e+04 109
  1768. performance model for cuda_0
  1769. # hash size mean dev n
  1770. 57618ab0 19660800 1.164144e+04 1.556094e+01 315
  1771. performance model for cuda_1
  1772. # hash size mean dev n
  1773. 57618ab0 19660800 1.164271e+04 1.330628e+01 360
  1774. performance model for cuda_2
  1775. # hash size mean dev n
  1776. 57618ab0 19660800 1.166730e+04 3.390395e+02 456
  1777. @end example
  1778. We can see that for the given size, over a sample of a few hundreds of
  1779. execution, the GPUs are about 20 times faster than the CPUs (numbers are in
  1780. us). The standard deviation is extremely low for the GPUs, and less than 10% for
  1781. CPUs.
  1782. The @code{starpu_regression_display} tool does the same for regression-based
  1783. performance models. It also writes a @code{.gp} file in the current directory,
  1784. to be run in the @code{gnuplot} tool, which shows the corresponding curve.
  1785. @c ---------------------------------------------------------------------
  1786. @c MPI support
  1787. @c ---------------------------------------------------------------------
  1788. @node StarPU MPI support
  1789. @chapter StarPU MPI support
  1790. The integration of MPI transfers within task parallelism is done in a
  1791. very natural way by the means of asynchronous interactions between the
  1792. application and StarPU. This is implemented in a separate libstarpumpi library
  1793. which basically provides "StarPU" equivalents of @code{MPI_*} functions, where
  1794. @code{void *} buffers are replaced with @code{starpu_data_handle}s, and all
  1795. GPU-RAM-NIC transfers are handled efficiently by StarPU-MPI. The user has to
  1796. use the usual @code{mpirun} command of the MPI implementation to start StarPU on
  1797. the different MPI nodes.
  1798. @menu
  1799. * The API::
  1800. * Simple Example::
  1801. * MPI Insert Task Utility::
  1802. * MPI Collective Operations::
  1803. @end menu
  1804. @node The API
  1805. @section The API
  1806. @subsection Initialisation
  1807. @deftypefun int starpu_mpi_initialize (void)
  1808. Initializes the starpumpi library. This must be called between calling
  1809. @code{starpu_init} and other @code{starpu_mpi} functions. This
  1810. function does not call @code{MPI_Init}, it should be called beforehand.
  1811. @end deftypefun
  1812. @deftypefun int starpu_mpi_initialize_extended (int *@var{rank}, int *@var{world_size})
  1813. Initializes the starpumpi library. This must be called between calling
  1814. @code{starpu_init} and other @code{starpu_mpi} functions.
  1815. This function calls @code{MPI_Init}, and therefore should be prefered
  1816. to the previous one for MPI implementations which are not thread-safe.
  1817. Returns the current MPI node rank and world size.
  1818. @end deftypefun
  1819. @deftypefun int starpu_mpi_shutdown (void)
  1820. Cleans the starpumpi library. This must be called between calling
  1821. @code{starpu_mpi} functions and @code{starpu_shutdown}.
  1822. @code{MPI_Finalize} will be called if StarPU-MPI has been initialized
  1823. by calling @code{starpu_mpi_initialize_extended}.
  1824. @end deftypefun
  1825. @subsection Communication
  1826. @deftypefun int starpu_mpi_send (starpu_data_handle @var{data_handle}, int @var{dest}, int @var{mpi_tag}, MPI_Comm @var{comm})
  1827. @end deftypefun
  1828. @deftypefun int starpu_mpi_recv (starpu_data_handle @var{data_handle}, int @var{source}, int @var{mpi_tag}, MPI_Comm @var{comm}, MPI_Status *@var{status})
  1829. @end deftypefun
  1830. @deftypefun int starpu_mpi_isend (starpu_data_handle @var{data_handle}, starpu_mpi_req *@var{req}, int @var{dest}, int @var{mpi_tag}, MPI_Comm @var{comm})
  1831. @end deftypefun
  1832. @deftypefun int starpu_mpi_irecv (starpu_data_handle @var{data_handle}, starpu_mpi_req *@var{req}, int @var{source}, int @var{mpi_tag}, MPI_Comm @var{comm})
  1833. @end deftypefun
  1834. @deftypefun int starpu_mpi_isend_detached (starpu_data_handle @var{data_handle}, int @var{dest}, int @var{mpi_tag}, MPI_Comm @var{comm}, void (*@var{callback})(void *), void *@var{arg})
  1835. @end deftypefun
  1836. @deftypefun int starpu_mpi_irecv_detached (starpu_data_handle @var{data_handle}, int @var{source}, int @var{mpi_tag}, MPI_Comm @var{comm}, void (*@var{callback})(void *), void *@var{arg})
  1837. @end deftypefun
  1838. @deftypefun int starpu_mpi_wait (starpu_mpi_req *@var{req}, MPI_Status *@var{status})
  1839. @end deftypefun
  1840. @deftypefun int starpu_mpi_test (starpu_mpi_req *@var{req}, int *@var{flag}, MPI_Status *@var{status})
  1841. @end deftypefun
  1842. @deftypefun int starpu_mpi_barrier (MPI_Comm @var{comm})
  1843. @end deftypefun
  1844. @deftypefun int starpu_mpi_isend_detached_unlock_tag (starpu_data_handle @var{data_handle}, int @var{dest}, int @var{mpi_tag}, MPI_Comm @var{comm}, starpu_tag_t @var{tag})
  1845. When the transfer is completed, the tag is unlocked
  1846. @end deftypefun
  1847. @deftypefun int starpu_mpi_irecv_detached_unlock_tag (starpu_data_handle @var{data_handle}, int @var{source}, int @var{mpi_tag}, MPI_Comm @var{comm}, starpu_tag_t @var{tag})
  1848. @end deftypefun
  1849. @deftypefun int starpu_mpi_isend_array_detached_unlock_tag (unsigned @var{array_size}, starpu_data_handle *@var{data_handle}, int *@var{dest}, int *@var{mpi_tag}, MPI_Comm *@var{comm}, starpu_tag_t @var{tag})
  1850. Asynchronously send an array of buffers, and unlocks the tag once all
  1851. of them are transmitted.
  1852. @end deftypefun
  1853. @deftypefun int starpu_mpi_irecv_array_detached_unlock_tag (unsigned @var{array_size}, starpu_data_handle *@var{data_handle}, int *@var{source}, int *@var{mpi_tag}, MPI_Comm *@var{comm}, starpu_tag_t @var{tag})
  1854. @end deftypefun
  1855. @page
  1856. @node Simple Example
  1857. @section Simple Example
  1858. @cartouche
  1859. @smallexample
  1860. void increment_token(void)
  1861. @{
  1862. struct starpu_task *task = starpu_task_create();
  1863. task->cl = &increment_cl;
  1864. task->buffers[0].handle = token_handle;
  1865. task->buffers[0].mode = STARPU_RW;
  1866. starpu_task_submit(task);
  1867. @}
  1868. @end smallexample
  1869. @end cartouche
  1870. @cartouche
  1871. @smallexample
  1872. int main(int argc, char **argv)
  1873. @{
  1874. int rank, size;
  1875. starpu_init(NULL);
  1876. starpu_mpi_initialize_extended(&rank, &size);
  1877. starpu_vector_data_register(&token_handle, 0, (uintptr_t)&token, 1, sizeof(unsigned));
  1878. unsigned nloops = NITER;
  1879. unsigned loop;
  1880. unsigned last_loop = nloops - 1;
  1881. unsigned last_rank = size - 1;
  1882. @end smallexample
  1883. @end cartouche
  1884. @cartouche
  1885. @smallexample
  1886. for (loop = 0; loop < nloops; loop++) @{
  1887. int tag = loop*size + rank;
  1888. if (loop == 0 && rank == 0)
  1889. @{
  1890. token = 0;
  1891. fprintf(stdout, "Start with token value %d\n", token);
  1892. @}
  1893. else
  1894. @{
  1895. starpu_mpi_irecv_detached(token_handle, (rank+size-1)%size, tag,
  1896. MPI_COMM_WORLD, NULL, NULL);
  1897. @}
  1898. increment_token();
  1899. if (loop == last_loop && rank == last_rank)
  1900. @{
  1901. starpu_data_acquire(token_handle, STARPU_R);
  1902. fprintf(stdout, "Finished : token value %d\n", token);
  1903. starpu_data_release(token_handle);
  1904. @}
  1905. else
  1906. @{
  1907. starpu_mpi_isend_detached(token_handle, (rank+1)%size, tag+1,
  1908. MPI_COMM_WORLD, NULL, NULL);
  1909. @}
  1910. @}
  1911. starpu_task_wait_for_all();
  1912. @end smallexample
  1913. @end cartouche
  1914. @cartouche
  1915. @smallexample
  1916. starpu_mpi_shutdown();
  1917. starpu_shutdown();
  1918. if (rank == last_rank)
  1919. @{
  1920. fprintf(stderr, "[%d] token = %d == %d * %d ?\n", rank, token, nloops, size);
  1921. STARPU_ASSERT(token == nloops*size);
  1922. @}
  1923. @end smallexample
  1924. @end cartouche
  1925. @page
  1926. @node MPI Insert Task Utility
  1927. @section MPI Insert Task Utility
  1928. @deftypefun void starpu_mpi_insert_task (MPI_Comm @var{comm}, starpu_codelet *@var{cl}, ...)
  1929. Create and submit a task corresponding to @var{cl} with the following
  1930. arguments. The argument list must be zero-terminated.
  1931. The arguments following the codelets are the same types as for the
  1932. function @code{starpu_insert_task} defined in @ref{Insert Task
  1933. Utility}. The extra argument @code{STARPU_EXECUTE_ON_NODE} followed by an
  1934. integer allows to specify the node to execute the codelet. It is also
  1935. possible to specify that the node owning a specific data will execute
  1936. the codelet, by using @code{STARPU_EXECUTE_ON_DATA} followed by a data
  1937. handle.
  1938. The algorithm is as follows:
  1939. @enumerate
  1940. @item Find out whether we are to execute the codelet because we own the
  1941. data to be written to. If different tasks own data to be written to,
  1942. the argument @code{STARPU_EXECUTE_ON_NODE} or
  1943. @code{STARPU_EXECUTE_ON_DATA} should be used to specify the executing
  1944. task @code{ET}.
  1945. @item Send and receive data as requested. Tasks owning data which need
  1946. to be read by the executing task @code{ET} are sending them to @code{ET}.
  1947. @item Execute the codelet. This is done by the task selected in the
  1948. 1st step of the algorithm.
  1949. @item In the case when different tasks own data to be written to, send
  1950. W data back to their owners.
  1951. @end enumerate
  1952. The algorithm also includes a cache mechanism that allows not to send
  1953. data twice to the same task, unless the data has been modified.
  1954. @end deftypefun
  1955. @deftypefun void starpu_mpi_get_data_on_node (MPI_Comm @var{comm}, starpu_data_handle @var{data_handle}, int @var{node})
  1956. @end deftypefun
  1957. @page
  1958. Here an example showing how to use @code{starpu_mpi_insert_task}. One
  1959. first needs to define a distribution function which specifies the
  1960. locality of the data. Note that that distribution information needs to
  1961. be given to StarPU by calling @code{starpu_data_set_rank}.
  1962. @cartouche
  1963. @smallexample
  1964. /* Returns the MPI node number where data is */
  1965. int my_distrib(int x, int y, int nb_nodes) @{
  1966. /* Cyclic distrib */
  1967. return ((int)(x / sqrt(nb_nodes) + (y / sqrt(nb_nodes)) * sqrt(nb_nodes))) % nb_nodes;
  1968. // /* Linear distrib */
  1969. // return x / sqrt(nb_nodes) + (y / sqrt(nb_nodes)) * X;
  1970. @}
  1971. @end smallexample
  1972. @end cartouche
  1973. Now the data can be registered within StarPU. Data which are not
  1974. owned but will be needed for computations can be registered through
  1975. the lazy allocation mechanism, i.e. with a @code{home_node} set to -1.
  1976. StarPU will automatically allocate the memory when it is used for the
  1977. first time.
  1978. @cartouche
  1979. @smallexample
  1980. unsigned matrix[X][Y];
  1981. starpu_data_handle data_handles[X][Y];
  1982. for(x = 0; x < X; x++) @{
  1983. for (y = 0; y < Y; y++) @{
  1984. int mpi_rank = my_distrib(x, y, size);
  1985. if (mpi_rank == rank)
  1986. /* Owning data */
  1987. starpu_variable_data_register(&data_handles[x][y], 0,
  1988. (uintptr_t)&(matrix[x][y]), sizeof(unsigned));
  1989. else if (rank == mpi_rank+1 || rank == mpi_rank-1)
  1990. /* I don't own that index, but will need it for my computations */
  1991. starpu_variable_data_register(&data_handles[x][y], -1,
  1992. (uintptr_t)NULL, sizeof(unsigned));
  1993. else
  1994. /* I know it's useless to allocate anything for this */
  1995. data_handles[x][y] = NULL;
  1996. if (data_handles[x][y])
  1997. starpu_data_set_rank(data_handles[x][y], mpi_rank);
  1998. @}
  1999. @}
  2000. @end smallexample
  2001. @end cartouche
  2002. Now @code{starpu_mpi_insert_task()} can be called for the different
  2003. steps of the application.
  2004. @cartouche
  2005. @smallexample
  2006. for(loop=0 ; loop<niter; loop++)
  2007. for (x = 1; x < X-1; x++)
  2008. for (y = 1; y < Y-1; y++)
  2009. starpu_mpi_insert_task(MPI_COMM_WORLD, &stencil5_cl,
  2010. STARPU_RW, data_handles[x][y],
  2011. STARPU_R, data_handles[x-1][y],
  2012. STARPU_R, data_handles[x+1][y],
  2013. STARPU_R, data_handles[x][y-1],
  2014. STARPU_R, data_handles[x][y+1],
  2015. 0);
  2016. starpu_task_wait_for_all();
  2017. @end smallexample
  2018. @end cartouche
  2019. @node MPI Collective Operations
  2020. @section MPI Collective Operations
  2021. @deftypefun int starpu_mpi_scatter_detached (starpu_data_handle *@var{data_handles}, int @var{count}, int @var{root}, MPI_Comm @var{comm})
  2022. Scatter data among processes of the communicator based on the ownership of
  2023. the data. For each data of the array @var{data_handles}, the
  2024. process @var{root} sends the data to the process owning this data.
  2025. Processes receiving data must have valid data handles to receive them.
  2026. @end deftypefun
  2027. @deftypefun int starpu_mpi_gather_detached (starpu_data_handle *@var{data_handles}, int @var{count}, int @var{root}, MPI_Comm @var{comm})
  2028. Gather data from the different processes of the communicator onto the
  2029. process @var{root}. Each process owning data handle in the array
  2030. @var{data_handles} will send them to the process @var{root}. The
  2031. process @var{root} must have valid data handles to receive the data.
  2032. @end deftypefun
  2033. @page
  2034. @cartouche
  2035. @smallexample
  2036. if (rank == root)
  2037. @{
  2038. /* Allocate the vector */
  2039. vector = malloc(nblocks * sizeof(float *));
  2040. for(x=0 ; x<nblocks ; x++)
  2041. @{
  2042. starpu_malloc((void **)&vector[x], block_size*sizeof(float));
  2043. @}
  2044. @}
  2045. /* Allocate data handles and register data to StarPU */
  2046. data_handles = malloc(nblocks*sizeof(starpu_data_handle *));
  2047. for(x = 0; x < nblocks ; x++)
  2048. @{
  2049. int mpi_rank = my_distrib(x, nodes);
  2050. if (rank == root) @{
  2051. starpu_vector_data_register(&data_handles[x], 0, (uintptr_t)vector[x],
  2052. blocks_size, sizeof(float));
  2053. @}
  2054. else if ((mpi_rank == rank) || ((rank == mpi_rank+1 || rank == mpi_rank-1))) @{
  2055. /* I own that index, or i will need it for my computations */
  2056. starpu_vector_data_register(&data_handles[x], -1, (uintptr_t)NULL,
  2057. block_size, sizeof(float));
  2058. @}
  2059. else @{
  2060. /* I know it's useless to allocate anything for this */
  2061. data_handles[x] = NULL;
  2062. @}
  2063. if (data_handles[x]) @{
  2064. starpu_data_set_rank(data_handles[x], mpi_rank);
  2065. @}
  2066. @}
  2067. /* Scatter the matrix among the nodes */
  2068. starpu_mpi_scatter_detached(data_handles, nblocks, root, MPI_COMM_WORLD);
  2069. /* Calculation */
  2070. for(x = 0; x < nblocks ; x++) @{
  2071. if (data_handles[x]) @{
  2072. int owner = starpu_data_get_rank(data_handles[x]);
  2073. if (owner == rank) @{
  2074. starpu_insert_task(&cl, STARPU_RW, data_handles[x], 0);
  2075. @}
  2076. @}
  2077. @}
  2078. /* Gather the matrix on main node */
  2079. starpu_mpi_gather_detached(data_handles, nblocks, 0, MPI_COMM_WORLD);
  2080. @end smallexample
  2081. @end cartouche
  2082. @c ---------------------------------------------------------------------
  2083. @c Configuration options
  2084. @c ---------------------------------------------------------------------
  2085. @node Configuring StarPU
  2086. @chapter Configuring StarPU
  2087. @menu
  2088. * Compilation configuration::
  2089. * Execution configuration through environment variables::
  2090. @end menu
  2091. @node Compilation configuration
  2092. @section Compilation configuration
  2093. The following arguments can be given to the @code{configure} script.
  2094. @menu
  2095. * Common configuration::
  2096. * Configuring workers::
  2097. * Advanced configuration::
  2098. @end menu
  2099. @node Common configuration
  2100. @subsection Common configuration
  2101. @menu
  2102. * --enable-debug::
  2103. * --enable-fast::
  2104. * --enable-verbose::
  2105. * --enable-coverage::
  2106. @end menu
  2107. @node --enable-debug
  2108. @subsubsection @code{--enable-debug}
  2109. @table @asis
  2110. @item @emph{Description}:
  2111. Enable debugging messages.
  2112. @end table
  2113. @node --enable-fast
  2114. @subsubsection @code{--enable-fast}
  2115. @table @asis
  2116. @item @emph{Description}:
  2117. Do not enforce assertions, saves a lot of time spent to compute them otherwise.
  2118. @end table
  2119. @node --enable-verbose
  2120. @subsubsection @code{--enable-verbose}
  2121. @table @asis
  2122. @item @emph{Description}:
  2123. Augment the verbosity of the debugging messages. This can be disabled
  2124. at runtime by setting the environment variable @code{STARPU_SILENT} to
  2125. any value.
  2126. @smallexample
  2127. % STARPU_SILENT=1 ./vector_scal
  2128. @end smallexample
  2129. @end table
  2130. @node --enable-coverage
  2131. @subsubsection @code{--enable-coverage}
  2132. @table @asis
  2133. @item @emph{Description}:
  2134. Enable flags for the @code{gcov} coverage tool.
  2135. @end table
  2136. @node Configuring workers
  2137. @subsection Configuring workers
  2138. @menu
  2139. * --enable-maxcpus::
  2140. * --disable-cpu::
  2141. * --enable-maxcudadev::
  2142. * --disable-cuda::
  2143. * --with-cuda-dir::
  2144. * --with-cuda-include-dir::
  2145. * --with-cuda-lib-dir::
  2146. * --disable-cuda-memcpy-peer::
  2147. * --enable-maxopencldev::
  2148. * --disable-opencl::
  2149. * --with-opencl-dir::
  2150. * --with-opencl-include-dir::
  2151. * --with-opencl-lib-dir::
  2152. * --enable-gordon::
  2153. * --with-gordon-dir::
  2154. * --enable-maximplementations::
  2155. @end menu
  2156. @node --enable-maxcpus
  2157. @subsubsection @code{--enable-maxcpus=<number>}
  2158. @table @asis
  2159. @item @emph{Description}:
  2160. Defines the maximum number of CPU cores that StarPU will support, then
  2161. available as the @code{STARPU_MAXCPUS} macro.
  2162. @end table
  2163. @node --disable-cpu
  2164. @subsubsection @code{--disable-cpu}
  2165. @table @asis
  2166. @item @emph{Description}:
  2167. Disable the use of CPUs of the machine. Only GPUs etc. will be used.
  2168. @end table
  2169. @node --enable-maxcudadev
  2170. @subsubsection @code{--enable-maxcudadev=<number>}
  2171. @table @asis
  2172. @item @emph{Description}:
  2173. Defines the maximum number of CUDA devices that StarPU will support, then
  2174. available as the @code{STARPU_MAXCUDADEVS} macro.
  2175. @end table
  2176. @node --disable-cuda
  2177. @subsubsection @code{--disable-cuda}
  2178. @table @asis
  2179. @item @emph{Description}:
  2180. Disable the use of CUDA, even if a valid CUDA installation was detected.
  2181. @end table
  2182. @node --with-cuda-dir
  2183. @subsubsection @code{--with-cuda-dir=<path>}
  2184. @table @asis
  2185. @item @emph{Description}:
  2186. Specify the directory where CUDA is installed. This directory should notably contain
  2187. @code{include/cuda.h}.
  2188. @end table
  2189. @node --with-cuda-include-dir
  2190. @subsubsection @code{--with-cuda-include-dir=<path>}
  2191. @table @asis
  2192. @item @emph{Description}:
  2193. Specify the directory where CUDA headers are installed. This directory should
  2194. notably contain @code{cuda.h}. This defaults to @code{/include} appended to the
  2195. value given to @code{--with-cuda-dir}.
  2196. @end table
  2197. @node --with-cuda-lib-dir
  2198. @subsubsection @code{--with-cuda-lib-dir=<path>}
  2199. @table @asis
  2200. @item @emph{Description}:
  2201. Specify the directory where the CUDA library is installed. This directory should
  2202. notably contain the CUDA shared libraries (e.g. libcuda.so). This defaults to
  2203. @code{/lib} appended to the value given to @code{--with-cuda-dir}.
  2204. @end table
  2205. @node --disable-cuda-memcpy-peer
  2206. @subsubsection @code{--disable-cuda-memcpy-peer}
  2207. @table @asis
  2208. @item @emph{Description}
  2209. Explicitely disables peer transfers when using CUDA 4.0
  2210. @end table
  2211. @node --enable-maxopencldev
  2212. @subsubsection @code{--enable-maxopencldev=<number>}
  2213. @table @asis
  2214. @item @emph{Description}:
  2215. Defines the maximum number of OpenCL devices that StarPU will support, then
  2216. available as the @code{STARPU_MAXOPENCLDEVS} macro.
  2217. @end table
  2218. @node --disable-opencl
  2219. @subsubsection @code{--disable-opencl}
  2220. @table @asis
  2221. @item @emph{Description}:
  2222. Disable the use of OpenCL, even if the SDK is detected.
  2223. @end table
  2224. @node --with-opencl-dir
  2225. @subsubsection @code{--with-opencl-dir=<path>}
  2226. @table @asis
  2227. @item @emph{Description}:
  2228. Specify the location of the OpenCL SDK. This directory should notably contain
  2229. @code{include/CL/cl.h} (or @code{include/OpenCL/cl.h} on Mac OS).
  2230. @end table
  2231. @node --with-opencl-include-dir
  2232. @subsubsection @code{--with-opencl-include-dir=<path>}
  2233. @table @asis
  2234. @item @emph{Description}:
  2235. Specify the location of OpenCL headers. This directory should notably contain
  2236. @code{CL/cl.h} (or @code{OpenCL/cl.h} on Mac OS). This defaults to
  2237. @code{/include} appended to the value given to @code{--with-opencl-dir}.
  2238. @end table
  2239. @node --with-opencl-lib-dir
  2240. @subsubsection @code{--with-opencl-lib-dir=<path>}
  2241. @table @asis
  2242. @item @emph{Description}:
  2243. Specify the location of the OpenCL library. This directory should notably
  2244. contain the OpenCL shared libraries (e.g. libOpenCL.so). This defaults to
  2245. @code{/lib} appended to the value given to @code{--with-opencl-dir}.
  2246. @end table
  2247. @node --enable-gordon
  2248. @subsubsection @code{--enable-gordon}
  2249. @table @asis
  2250. @item @emph{Description}:
  2251. Enable the use of the Gordon runtime for Cell SPUs.
  2252. @c TODO: rather default to enabled when detected
  2253. @end table
  2254. @node --with-gordon-dir
  2255. @subsubsection @code{--with-gordon-dir=<path>}
  2256. @table @asis
  2257. @item @emph{Description}:
  2258. Specify the location of the Gordon SDK.
  2259. @end table
  2260. @node --enable-maximplementations
  2261. @subsubsection @code{--enable-maximplementations=<number>}
  2262. @table @asis
  2263. @item @emph{Description}:
  2264. Defines the number of implementations that can be defined for a single kind of
  2265. device. It is then available as the @code{STARPU_MAXIMPLEMENTATIONS} macro.
  2266. @end table
  2267. @node Advanced configuration
  2268. @subsection Advanced configuration
  2269. @menu
  2270. * --enable-perf-debug::
  2271. * --enable-model-debug::
  2272. * --enable-stats::
  2273. * --enable-maxbuffers::
  2274. * --enable-allocation-cache::
  2275. * --enable-opengl-render::
  2276. * --enable-blas-lib::
  2277. * --with-magma::
  2278. * --with-fxt::
  2279. * --with-perf-model-dir::
  2280. * --with-mpicc::
  2281. * --with-goto-dir::
  2282. * --with-atlas-dir::
  2283. * --with-mkl-cflags::
  2284. * --with-mkl-ldflags::
  2285. @end menu
  2286. @node --enable-perf-debug
  2287. @subsubsection @code{--enable-perf-debug}
  2288. @table @asis
  2289. @item @emph{Description}:
  2290. Enable performance debugging through gprof.
  2291. @end table
  2292. @node --enable-model-debug
  2293. @subsubsection @code{--enable-model-debug}
  2294. @table @asis
  2295. @item @emph{Description}:
  2296. Enable performance model debugging.
  2297. @end table
  2298. @node --enable-stats
  2299. @subsubsection @code{--enable-stats}
  2300. @table @asis
  2301. @item @emph{Description}:
  2302. Enable statistics.
  2303. @end table
  2304. @node --enable-maxbuffers
  2305. @subsubsection @code{--enable-maxbuffers=<nbuffers>}
  2306. @table @asis
  2307. @item @emph{Description}:
  2308. Define the maximum number of buffers that tasks will be able to take
  2309. as parameters, then available as the @code{STARPU_NMAXBUFS} macro.
  2310. @end table
  2311. @node --enable-allocation-cache
  2312. @subsubsection @code{--enable-allocation-cache}
  2313. @table @asis
  2314. @item @emph{Description}:
  2315. Enable the use of a data allocation cache to avoid the cost of it with
  2316. CUDA. Still experimental.
  2317. @end table
  2318. @node --enable-opengl-render
  2319. @subsubsection @code{--enable-opengl-render}
  2320. @table @asis
  2321. @item @emph{Description}:
  2322. Enable the use of OpenGL for the rendering of some examples.
  2323. @c TODO: rather default to enabled when detected
  2324. @end table
  2325. @node --enable-blas-lib
  2326. @subsubsection @code{--enable-blas-lib=<name>}
  2327. @table @asis
  2328. @item @emph{Description}:
  2329. Specify the blas library to be used by some of the examples. The
  2330. library has to be 'atlas' or 'goto'.
  2331. @end table
  2332. @node --with-magma
  2333. @subsubsection @code{--with-magma=<path>}
  2334. @table @asis
  2335. @item @emph{Description}:
  2336. Specify where magma is installed. This directory should notably contain
  2337. @code{include/magmablas.h}.
  2338. @end table
  2339. @node --with-fxt
  2340. @subsubsection @code{--with-fxt=<path>}
  2341. @table @asis
  2342. @item @emph{Description}:
  2343. Specify the location of FxT (for generating traces and rendering them
  2344. using ViTE). This directory should notably contain
  2345. @code{include/fxt/fxt.h}.
  2346. @c TODO add ref to other section
  2347. @end table
  2348. @node --with-perf-model-dir
  2349. @subsubsection @code{--with-perf-model-dir=<dir>}
  2350. @table @asis
  2351. @item @emph{Description}:
  2352. Specify where performance models should be stored (instead of defaulting to the
  2353. current user's home).
  2354. @end table
  2355. @node --with-mpicc
  2356. @subsubsection @code{--with-mpicc=<path to mpicc>}
  2357. @table @asis
  2358. @item @emph{Description}:
  2359. Specify the location of the @code{mpicc} compiler to be used for starpumpi.
  2360. @end table
  2361. @node --with-goto-dir
  2362. @subsubsection @code{--with-goto-dir=<dir>}
  2363. @table @asis
  2364. @item @emph{Description}:
  2365. Specify the location of GotoBLAS.
  2366. @end table
  2367. @node --with-atlas-dir
  2368. @subsubsection @code{--with-atlas-dir=<dir>}
  2369. @table @asis
  2370. @item @emph{Description}:
  2371. Specify the location of ATLAS. This directory should notably contain
  2372. @code{include/cblas.h}.
  2373. @end table
  2374. @node --with-mkl-cflags
  2375. @subsubsection @code{--with-mkl-cflags=<cflags>}
  2376. @table @asis
  2377. @item @emph{Description}:
  2378. Specify the compilation flags for the MKL Library.
  2379. @end table
  2380. @node --with-mkl-ldflags
  2381. @subsubsection @code{--with-mkl-ldflags=<ldflags>}
  2382. @table @asis
  2383. @item @emph{Description}:
  2384. Specify the linking flags for the MKL Library. Note that the
  2385. @url{http://software.intel.com/en-us/articles/intel-mkl-link-line-advisor/}
  2386. website provides a script to determine the linking flags.
  2387. @end table
  2388. @c ---------------------------------------------------------------------
  2389. @c Environment variables
  2390. @c ---------------------------------------------------------------------
  2391. @node Execution configuration through environment variables
  2392. @section Execution configuration through environment variables
  2393. @menu
  2394. * Workers:: Configuring workers
  2395. * Scheduling:: Configuring the Scheduling engine
  2396. * Misc:: Miscellaneous and debug
  2397. @end menu
  2398. Note: the values given in @code{starpu_conf} structure passed when
  2399. calling @code{starpu_init} will override the values of the environment
  2400. variables.
  2401. @node Workers
  2402. @subsection Configuring workers
  2403. @menu
  2404. * STARPU_NCPUS:: Number of CPU workers
  2405. * STARPU_NCUDA:: Number of CUDA workers
  2406. * STARPU_NOPENCL:: Number of OpenCL workers
  2407. * STARPU_NGORDON:: Number of SPU workers (Cell)
  2408. * STARPU_WORKERS_CPUID:: Bind workers to specific CPUs
  2409. * STARPU_WORKERS_CUDAID:: Select specific CUDA devices
  2410. * STARPU_WORKERS_OPENCLID:: Select specific OpenCL devices
  2411. @end menu
  2412. @node STARPU_NCPUS
  2413. @subsubsection @code{STARPU_NCPUS} -- Number of CPU workers
  2414. @table @asis
  2415. @item @emph{Description}:
  2416. Specify the number of CPU workers (thus not including workers dedicated to control acceleratores). Note that by default, StarPU will not allocate
  2417. more CPU workers than there are physical CPUs, and that some CPUs are used to control
  2418. the accelerators.
  2419. @end table
  2420. @node STARPU_NCUDA
  2421. @subsubsection @code{STARPU_NCUDA} -- Number of CUDA workers
  2422. @table @asis
  2423. @item @emph{Description}:
  2424. Specify the number of CUDA devices that StarPU can use. If
  2425. @code{STARPU_NCUDA} is lower than the number of physical devices, it is
  2426. possible to select which CUDA devices should be used by the means of the
  2427. @code{STARPU_WORKERS_CUDAID} environment variable. By default, StarPU will
  2428. create as many CUDA workers as there are CUDA devices.
  2429. @end table
  2430. @node STARPU_NOPENCL
  2431. @subsubsection @code{STARPU_NOPENCL} -- Number of OpenCL workers
  2432. @table @asis
  2433. @item @emph{Description}:
  2434. OpenCL equivalent of the @code{STARPU_NCUDA} environment variable.
  2435. @end table
  2436. @node STARPU_NGORDON
  2437. @subsubsection @code{STARPU_NGORDON} -- Number of SPU workers (Cell)
  2438. @table @asis
  2439. @item @emph{Description}:
  2440. Specify the number of SPUs that StarPU can use.
  2441. @end table
  2442. @node STARPU_WORKERS_CPUID
  2443. @subsubsection @code{STARPU_WORKERS_CPUID} -- Bind workers to specific CPUs
  2444. @table @asis
  2445. @item @emph{Description}:
  2446. Passing an array of integers (starting from 0) in @code{STARPU_WORKERS_CPUID}
  2447. specifies on which logical CPU the different workers should be
  2448. bound. For instance, if @code{STARPU_WORKERS_CPUID = "0 1 4 5"}, the first
  2449. worker will be bound to logical CPU #0, the second CPU worker will be bound to
  2450. logical CPU #1 and so on. Note that the logical ordering of the CPUs is either
  2451. determined by the OS, or provided by the @code{hwloc} library in case it is
  2452. available.
  2453. Note that the first workers correspond to the CUDA workers, then come the
  2454. OpenCL and the SPU, and finally the CPU workers. For example if
  2455. we have @code{STARPU_NCUDA=1}, @code{STARPU_NOPENCL=1}, @code{STARPU_NCPUS=2}
  2456. and @code{STARPU_WORKERS_CPUID = "0 2 1 3"}, the CUDA device will be controlled
  2457. by logical CPU #0, the OpenCL device will be controlled by logical CPU #2, and
  2458. the logical CPUs #1 and #3 will be used by the CPU workers.
  2459. If the number of workers is larger than the array given in
  2460. @code{STARPU_WORKERS_CPUID}, the workers are bound to the logical CPUs in a
  2461. round-robin fashion: if @code{STARPU_WORKERS_CPUID = "0 1"}, the first and the
  2462. third (resp. second and fourth) workers will be put on CPU #0 (resp. CPU #1).
  2463. This variable is ignored if the @code{use_explicit_workers_bindid} flag of the
  2464. @code{starpu_conf} structure passed to @code{starpu_init} is set.
  2465. @end table
  2466. @node STARPU_WORKERS_CUDAID
  2467. @subsubsection @code{STARPU_WORKERS_CUDAID} -- Select specific CUDA devices
  2468. @table @asis
  2469. @item @emph{Description}:
  2470. Similarly to the @code{STARPU_WORKERS_CPUID} environment variable, it is
  2471. possible to select which CUDA devices should be used by StarPU. On a machine
  2472. equipped with 4 GPUs, setting @code{STARPU_WORKERS_CUDAID = "1 3"} and
  2473. @code{STARPU_NCUDA=2} specifies that 2 CUDA workers should be created, and that
  2474. they should use CUDA devices #1 and #3 (the logical ordering of the devices is
  2475. the one reported by CUDA).
  2476. This variable is ignored if the @code{use_explicit_workers_cuda_gpuid} flag of
  2477. the @code{starpu_conf} structure passed to @code{starpu_init} is set.
  2478. @end table
  2479. @node STARPU_WORKERS_OPENCLID
  2480. @subsubsection @code{STARPU_WORKERS_OPENCLID} -- Select specific OpenCL devices
  2481. @table @asis
  2482. @item @emph{Description}:
  2483. OpenCL equivalent of the @code{STARPU_WORKERS_CUDAID} environment variable.
  2484. This variable is ignored if the @code{use_explicit_workers_opencl_gpuid} flag of
  2485. the @code{starpu_conf} structure passed to @code{starpu_init} is set.
  2486. @end table
  2487. @node Scheduling
  2488. @subsection Configuring the Scheduling engine
  2489. @menu
  2490. * STARPU_SCHED:: Scheduling policy
  2491. * STARPU_CALIBRATE:: Calibrate performance models
  2492. * STARPU_PREFETCH:: Use data prefetch
  2493. * STARPU_SCHED_ALPHA:: Computation factor
  2494. * STARPU_SCHED_BETA:: Communication factor
  2495. @end menu
  2496. @node STARPU_SCHED
  2497. @subsubsection @code{STARPU_SCHED} -- Scheduling policy
  2498. @table @asis
  2499. @item @emph{Description}:
  2500. This chooses between the different scheduling policies proposed by StarPU: work
  2501. random, stealing, greedy, with performance models, etc.
  2502. Use @code{STARPU_SCHED=help} to get the list of available schedulers.
  2503. @end table
  2504. @node STARPU_CALIBRATE
  2505. @subsubsection @code{STARPU_CALIBRATE} -- Calibrate performance models
  2506. @table @asis
  2507. @item @emph{Description}:
  2508. If this variable is set to 1, the performance models are calibrated during
  2509. the execution. If it is set to 2, the previous values are dropped to restart
  2510. calibration from scratch. Setting this variable to 0 disable calibration, this
  2511. is the default behaviour.
  2512. Note: this currently only applies to @code{dm}, @code{dmda} and @code{heft} scheduling policies.
  2513. @end table
  2514. @node STARPU_PREFETCH
  2515. @subsubsection @code{STARPU_PREFETCH} -- Use data prefetch
  2516. @table @asis
  2517. @item @emph{Description}:
  2518. This variable indicates whether data prefetching should be enabled (0 means
  2519. that it is disabled). If prefetching is enabled, when a task is scheduled to be
  2520. executed e.g. on a GPU, StarPU will request an asynchronous transfer in
  2521. advance, so that data is already present on the GPU when the task starts. As a
  2522. result, computation and data transfers are overlapped.
  2523. Note that prefetching is enabled by default in StarPU.
  2524. @end table
  2525. @node STARPU_SCHED_ALPHA
  2526. @subsubsection @code{STARPU_SCHED_ALPHA} -- Computation factor
  2527. @table @asis
  2528. @item @emph{Description}:
  2529. To estimate the cost of a task StarPU takes into account the estimated
  2530. computation time (obtained thanks to performance models). The alpha factor is
  2531. the coefficient to be applied to it before adding it to the communication part.
  2532. @end table
  2533. @node STARPU_SCHED_BETA
  2534. @subsubsection @code{STARPU_SCHED_BETA} -- Communication factor
  2535. @table @asis
  2536. @item @emph{Description}:
  2537. To estimate the cost of a task StarPU takes into account the estimated
  2538. data transfer time (obtained thanks to performance models). The beta factor is
  2539. the coefficient to be applied to it before adding it to the computation part.
  2540. @end table
  2541. @node Misc
  2542. @subsection Miscellaneous and debug
  2543. @menu
  2544. * STARPU_SILENT:: Disable verbose mode
  2545. * STARPU_LOGFILENAME:: Select debug file name
  2546. * STARPU_FXT_PREFIX:: FxT trace location
  2547. * STARPU_LIMIT_GPU_MEM:: Restrict memory size on the GPUs
  2548. * STARPU_GENERATE_TRACE:: Generate a Paje trace when StarPU is shut down
  2549. @end menu
  2550. @node STARPU_SILENT
  2551. @subsubsection @code{STARPU_SILENT} -- Disable verbose mode
  2552. @table @asis
  2553. @item @emph{Description}:
  2554. This variable allows to disable verbose mode at runtime when StarPU
  2555. has been configured with the option @code{--enable-verbose}.
  2556. @end table
  2557. @node STARPU_LOGFILENAME
  2558. @subsubsection @code{STARPU_LOGFILENAME} -- Select debug file name
  2559. @table @asis
  2560. @item @emph{Description}:
  2561. This variable specifies in which file the debugging output should be saved to.
  2562. @end table
  2563. @node STARPU_FXT_PREFIX
  2564. @subsubsection @code{STARPU_FXT_PREFIX} -- FxT trace location
  2565. @table @asis
  2566. @item @emph{Description}
  2567. This variable specifies in which directory to save the trace generated if FxT is enabled. It needs to have a trailing '/' character.
  2568. @end table
  2569. @node STARPU_LIMIT_GPU_MEM
  2570. @subsubsection @code{STARPU_LIMIT_GPU_MEM} -- Restrict memory size on the GPUs
  2571. @table @asis
  2572. @item @emph{Description}
  2573. This variable specifies the maximum number of megabytes that should be
  2574. available to the application on each GPUs. In case this value is smaller than
  2575. the size of the memory of a GPU, StarPU pre-allocates a buffer to waste memory
  2576. on the device. This variable is intended to be used for experimental purposes
  2577. as it emulates devices that have a limited amount of memory.
  2578. @end table
  2579. @node STARPU_GENERATE_TRACE
  2580. @subsubsection @code{STARPU_GENERATE_TRACE} -- Generate a Paje trace when StarPU is shut down
  2581. @table @asis
  2582. @item @emph{Description}
  2583. When set to 1, this variable indicates that StarPU should automatically
  2584. generate a Paje trace when starpu_shutdown is called.
  2585. @end table
  2586. @c ---------------------------------------------------------------------
  2587. @c StarPU API
  2588. @c ---------------------------------------------------------------------
  2589. @node StarPU API
  2590. @chapter StarPU API
  2591. @menu
  2592. * Initialization and Termination:: Initialization and Termination methods
  2593. * Workers' Properties:: Methods to enumerate workers' properties
  2594. * Data Library:: Methods to manipulate data
  2595. * Data Interfaces::
  2596. * Data Partition::
  2597. * Codelets and Tasks:: Methods to construct tasks
  2598. * Explicit Dependencies:: Explicit Dependencies
  2599. * Implicit Data Dependencies:: Implicit Data Dependencies
  2600. * Performance Model API::
  2601. * Profiling API:: Profiling API
  2602. * CUDA extensions:: CUDA extensions
  2603. * OpenCL extensions:: OpenCL extensions
  2604. * Cell extensions:: Cell extensions
  2605. * Miscellaneous helpers::
  2606. @end menu
  2607. @node Initialization and Termination
  2608. @section Initialization and Termination
  2609. @menu
  2610. * starpu_init:: Initialize StarPU
  2611. * struct starpu_conf:: StarPU runtime configuration
  2612. * starpu_conf_init:: Initialize starpu_conf structure
  2613. * starpu_shutdown:: Terminate StarPU
  2614. @end menu
  2615. @node starpu_init
  2616. @subsection @code{starpu_init} -- Initialize StarPU
  2617. @table @asis
  2618. @item @emph{Description}:
  2619. This is StarPU initialization method, which must be called prior to any other
  2620. StarPU call. It is possible to specify StarPU's configuration (e.g. scheduling
  2621. policy, number of cores, ...) by passing a non-null argument. Default
  2622. configuration is used if the passed argument is @code{NULL}.
  2623. @item @emph{Return value}:
  2624. Upon successful completion, this function returns 0. Otherwise, @code{-ENODEV}
  2625. indicates that no worker was available (so that StarPU was not initialized).
  2626. @item @emph{Prototype}:
  2627. @code{int starpu_init(struct starpu_conf *conf);}
  2628. @end table
  2629. @node struct starpu_conf
  2630. @subsection @code{struct starpu_conf} -- StarPU runtime configuration
  2631. @table @asis
  2632. @item @emph{Description}:
  2633. This structure is passed to the @code{starpu_init} function in order
  2634. to configure StarPU.
  2635. When the default value is used, StarPU automatically selects the number
  2636. of processing units and takes the default scheduling policy. This parameter
  2637. overwrites the equivalent environment variables.
  2638. @item @emph{Fields}:
  2639. @table @asis
  2640. @item @code{sched_policy_name} (default = NULL):
  2641. This is the name of the scheduling policy. This can also be specified with the
  2642. @code{STARPU_SCHED} environment variable.
  2643. @item @code{sched_policy} (default = NULL):
  2644. This is the definition of the scheduling policy. This field is ignored
  2645. if @code{sched_policy_name} is set.
  2646. @item @code{ncpus} (default = -1):
  2647. This is the number of CPU cores that StarPU can use. This can also be
  2648. specified with the @code{STARPU_NCPUS} environment variable.
  2649. @item @code{ncuda} (default = -1):
  2650. This is the number of CUDA devices that StarPU can use. This can also be
  2651. specified with the @code{STARPU_NCUDA} environment variable.
  2652. @item @code{nopencl} (default = -1):
  2653. This is the number of OpenCL devices that StarPU can use. This can also be
  2654. specified with the @code{STARPU_NOPENCL} environment variable.
  2655. @item @code{nspus} (default = -1):
  2656. This is the number of Cell SPUs that StarPU can use. This can also be
  2657. specified with the @code{STARPU_NGORDON} environment variable.
  2658. @item @code{use_explicit_workers_bindid} (default = 0)
  2659. If this flag is set, the @code{workers_bindid} array indicates where the
  2660. different workers are bound, otherwise StarPU automatically selects where to
  2661. bind the different workers unless the @code{STARPU_WORKERS_CPUID} environment
  2662. variable is set. The @code{STARPU_WORKERS_CPUID} environment variable is
  2663. ignored if the @code{use_explicit_workers_bindid} flag is set.
  2664. @item @code{workers_bindid[STARPU_NMAXWORKERS]}
  2665. If the @code{use_explicit_workers_bindid} flag is set, this array indicates
  2666. where to bind the different workers. The i-th entry of the
  2667. @code{workers_bindid} indicates the logical identifier of the processor which
  2668. should execute the i-th worker. Note that the logical ordering of the CPUs is
  2669. either determined by the OS, or provided by the @code{hwloc} library in case it
  2670. is available.
  2671. When this flag is set, the @ref{STARPU_WORKERS_CPUID} environment variable is
  2672. ignored.
  2673. @item @code{use_explicit_workers_cuda_gpuid} (default = 0)
  2674. If this flag is set, the CUDA workers will be attached to the CUDA devices
  2675. specified in the @code{workers_cuda_gpuid} array. Otherwise, StarPU affects the
  2676. CUDA devices in a round-robin fashion.
  2677. When this flag is set, the @ref{STARPU_WORKERS_CUDAID} environment variable is
  2678. ignored.
  2679. @item @code{workers_cuda_gpuid[STARPU_NMAXWORKERS]}
  2680. If the @code{use_explicit_workers_cuda_gpuid} flag is set, this array contains
  2681. the logical identifiers of the CUDA devices (as used by @code{cudaGetDevice}).
  2682. @item @code{use_explicit_workers_opencl_gpuid} (default = 0)
  2683. If this flag is set, the OpenCL workers will be attached to the OpenCL devices
  2684. specified in the @code{workers_opencl_gpuid} array. Otherwise, StarPU affects the
  2685. OpenCL devices in a round-robin fashion.
  2686. @item @code{workers_opencl_gpuid[STARPU_NMAXWORKERS]}:
  2687. @item @code{calibrate} (default = 0):
  2688. If this flag is set, StarPU will calibrate the performance models when
  2689. executing tasks. If this value is equal to -1, the default value is used. The
  2690. default value is overwritten by the @code{STARPU_CALIBRATE} environment
  2691. variable when it is set.
  2692. @end table
  2693. @item @code{single_combined_worker} (default = 0):
  2694. By default, StarPU creates various combined workers according to the machine
  2695. structure. Some parallel libraries (e.g. most OpenMP implementations) however do
  2696. not support concurrent calls to parallel code. In such case, setting this flag
  2697. makes StarPU only create one combined worker, containing all
  2698. the CPU workers. The default value is overwritten by the
  2699. @code{STARPU_SINGLE_COMBINED_WORKER} environment variable when it is set.
  2700. @end table
  2701. @node starpu_conf_init
  2702. @subsection @code{starpu_conf_init} -- Initialize starpu_conf structure
  2703. @table @asis
  2704. This function initializes the @code{starpu_conf} structure passed as argument
  2705. with the default values. In case some configuration parameters are already
  2706. specified through environment variables, @code{starpu_conf_init} initializes
  2707. the fields of the structure according to the environment variables. For
  2708. instance if @code{STARPU_CALIBRATE} is set, its value is put in the
  2709. @code{.ncuda} field of the structure passed as argument.
  2710. @item @emph{Return value}:
  2711. Upon successful completion, this function returns 0. Otherwise, @code{-EINVAL}
  2712. indicates that the argument was NULL.
  2713. @item @emph{Prototype}:
  2714. @code{int starpu_conf_init(struct starpu_conf *conf);}
  2715. @end table
  2716. @node starpu_shutdown
  2717. @subsection @code{starpu_shutdown} -- Terminate StarPU
  2718. @deftypefun void starpu_shutdown (void)
  2719. This is StarPU termination method. It must be called at the end of the
  2720. application: statistics and other post-mortem debugging information are not
  2721. guaranteed to be available until this method has been called.
  2722. @end deftypefun
  2723. @node Workers' Properties
  2724. @section Workers' Properties
  2725. @menu
  2726. * starpu_worker_get_count:: Get the number of processing units
  2727. * starpu_worker_get_count_by_type:: Get the number of processing units of a given type
  2728. * starpu_cpu_worker_get_count:: Get the number of CPU controlled by StarPU
  2729. * starpu_cuda_worker_get_count:: Get the number of CUDA devices controlled by StarPU
  2730. * starpu_opencl_worker_get_count:: Get the number of OpenCL devices controlled by StarPU
  2731. * starpu_spu_worker_get_count:: Get the number of Cell SPUs controlled by StarPU
  2732. * starpu_worker_get_id:: Get the identifier of the current worker
  2733. * starpu_worker_get_ids_by_type:: Get the list of identifiers of workers with a given type
  2734. * starpu_worker_get_devid:: Get the device identifier of a worker
  2735. * starpu_worker_get_type:: Get the type of processing unit associated to a worker
  2736. * starpu_worker_get_name:: Get the name of a worker
  2737. * starpu_worker_get_memory_node:: Get the memory node of a worker
  2738. @end menu
  2739. @node starpu_worker_get_count
  2740. @subsection @code{starpu_worker_get_count} -- Get the number of processing units
  2741. @deftypefun unsigned starpu_worker_get_count (void)
  2742. This function returns the number of workers (i.e. processing units executing
  2743. StarPU tasks). The returned value should be at most @code{STARPU_NMAXWORKERS}.
  2744. @end deftypefun
  2745. @node starpu_worker_get_count_by_type
  2746. @subsection @code{starpu_worker_get_count_by_type} -- Get the number of processing units of a given type
  2747. @deftypefun int starpu_worker_get_count_by_type ({enum starpu_archtype} @var{type})
  2748. Returns the number of workers of the type indicated by the argument. A positive
  2749. (or null) value is returned in case of success, @code{-EINVAL} indicates that
  2750. the type is not valid otherwise.
  2751. @end deftypefun
  2752. @node starpu_cpu_worker_get_count
  2753. @subsection @code{starpu_cpu_worker_get_count} -- Get the number of CPU controlled by StarPU
  2754. @deftypefun unsigned starpu_cpu_worker_get_count (void)
  2755. This function returns the number of CPUs controlled by StarPU. The returned
  2756. value should be at most @code{STARPU_MAXCPUS}.
  2757. @end deftypefun
  2758. @node starpu_cuda_worker_get_count
  2759. @subsection @code{starpu_cuda_worker_get_count} -- Get the number of CUDA devices controlled by StarPU
  2760. @deftypefun unsigned starpu_cuda_worker_get_count (void)
  2761. This function returns the number of CUDA devices controlled by StarPU. The returned
  2762. value should be at most @code{STARPU_MAXCUDADEVS}.
  2763. @end deftypefun
  2764. @node starpu_opencl_worker_get_count
  2765. @subsection @code{starpu_opencl_worker_get_count} -- Get the number of OpenCL devices controlled by StarPU
  2766. @deftypefun unsigned starpu_opencl_worker_get_count (void)
  2767. This function returns the number of OpenCL devices controlled by StarPU. The returned
  2768. value should be at most @code{STARPU_MAXOPENCLDEVS}.
  2769. @end deftypefun
  2770. @node starpu_spu_worker_get_count
  2771. @subsection @code{starpu_spu_worker_get_count} -- Get the number of Cell SPUs controlled by StarPU
  2772. @deftypefun unsigned starpu_spu_worker_get_count (void)
  2773. This function returns the number of Cell SPUs controlled by StarPU.
  2774. @end deftypefun
  2775. @node starpu_worker_get_id
  2776. @subsection @code{starpu_worker_get_id} -- Get the identifier of the current worker
  2777. @deftypefun int starpu_worker_get_id (void)
  2778. This function returns the identifier of the worker associated to the calling
  2779. thread. The returned value is either -1 if the current context is not a StarPU
  2780. worker (i.e. when called from the application outside a task or a callback), or
  2781. an integer between 0 and @code{starpu_worker_get_count() - 1}.
  2782. @end deftypefun
  2783. @node starpu_worker_get_ids_by_type
  2784. @subsection @code{starpu_worker_get_ids_by_type} -- Get the list of identifiers of workers with a given type
  2785. @deftypefun int starpu_worker_get_ids_by_type ({enum starpu_archtype} @var{type}, int *@var{workerids}, int @var{maxsize})
  2786. Fill the workerids array with the identifiers of the workers that have the type
  2787. indicated in the first argument. The maxsize argument indicates the size of the
  2788. workids array. The returned value gives the number of identifiers that were put
  2789. in the array. @code{-ERANGE} is returned is maxsize is lower than the number of
  2790. workers with the appropriate type: in that case, the array is filled with the
  2791. maxsize first elements. To avoid such overflows, the value of maxsize can be
  2792. chosen by the means of the @code{starpu_worker_get_count_by_type} function, or
  2793. by passing a value greater or equal to @code{STARPU_NMAXWORKERS}.
  2794. @end deftypefun
  2795. @node starpu_worker_get_devid
  2796. @subsection @code{starpu_worker_get_devid} -- Get the device identifier of a worker
  2797. @deftypefun int starpu_worker_get_devid (int @var{id})
  2798. This functions returns the device id of the worker associated to an identifier
  2799. (as returned by the @code{starpu_worker_get_id} function). In the case of a
  2800. CUDA worker, this device identifier is the logical device identifier exposed by
  2801. CUDA (used by the @code{cudaGetDevice} function for instance). The device
  2802. identifier of a CPU worker is the logical identifier of the core on which the
  2803. worker was bound; this identifier is either provided by the OS or by the
  2804. @code{hwloc} library in case it is available.
  2805. @end deftypefun
  2806. @node starpu_worker_get_type
  2807. @subsection @code{starpu_worker_get_type} -- Get the type of processing unit associated to a worker
  2808. @deftypefun {enum starpu_archtype} starpu_worker_get_type (int @var{id})
  2809. This function returns the type of worker associated to an identifier (as
  2810. returned by the @code{starpu_worker_get_id} function). The returned value
  2811. indicates the architecture of the worker: @code{STARPU_CPU_WORKER} for a CPU
  2812. core, @code{STARPU_CUDA_WORKER} for a CUDA device,
  2813. @code{STARPU_OPENCL_WORKER} for a OpenCL device, and
  2814. @code{STARPU_GORDON_WORKER} for a Cell SPU. The value returned for an invalid
  2815. identifier is unspecified.
  2816. @end deftypefun
  2817. @node starpu_worker_get_name
  2818. @subsection @code{starpu_worker_get_name} -- Get the name of a worker
  2819. @deftypefun void starpu_worker_get_name (int @var{id}, char *@var{dst}, size_t @var{maxlen})
  2820. StarPU associates a unique human readable string to each processing unit. This
  2821. function copies at most the @var{maxlen} first bytes of the unique string
  2822. associated to a worker identified by its identifier @var{id} into the
  2823. @var{dst} buffer. The caller is responsible for ensuring that the @var{dst}
  2824. is a valid pointer to a buffer of @var{maxlen} bytes at least. Calling this
  2825. function on an invalid identifier results in an unspecified behaviour.
  2826. @end deftypefun
  2827. @node starpu_worker_get_memory_node
  2828. @subsection @code{starpu_worker_get_memory_node} -- Get the memory node of a worker
  2829. @deftypefun unsigned starpu_worker_get_memory_node (unsigned @var{workerid})
  2830. This function returns the identifier of the memory node associated to the
  2831. worker identified by @var{workerid}.
  2832. @end deftypefun
  2833. @node Data Library
  2834. @section Data Library
  2835. This section describes the data management facilities provided by StarPU.
  2836. We show how to use existing data interfaces in @ref{Data Interfaces}, but developers can
  2837. design their own data interfaces if required.
  2838. @menu
  2839. * starpu_malloc:: Allocate data and pin it
  2840. * starpu_access_mode:: Data access mode
  2841. * unsigned memory_node:: Memory node
  2842. * starpu_data_handle:: StarPU opaque data handle
  2843. * void *interface:: StarPU data interface
  2844. * starpu_data_register:: Register a piece of data to StarPU
  2845. * starpu_data_unregister:: Unregister a piece of data from StarPU
  2846. * starpu_data_unregister_no_coherency:: Unregister a piece of data from StarPU without coherency
  2847. * starpu_data_invalidate:: Invalidate all data replicates
  2848. * starpu_data_acquire:: Access registered data from the application
  2849. * starpu_data_acquire_cb:: Access registered data from the application asynchronously
  2850. * STARPU_DATA_ACQUIRE_CB:: Access registered data from the application asynchronously, macro
  2851. * starpu_data_release:: Release registered data from the application
  2852. * starpu_data_set_wt_mask:: Set the Write-Through mask
  2853. * starpu_data_prefetch_on_node:: Prefetch data to a given node
  2854. @end menu
  2855. @node starpu_malloc
  2856. @subsection @code{starpu_malloc} -- Allocate data and pin it
  2857. @deftypefun int starpu_malloc (void **@var{A}, size_t @var{dim})
  2858. This function allocates data of the given size in main memory. It will also try to pin it in
  2859. CUDA or OpenCL, so that data transfers from this buffer can be asynchronous, and
  2860. thus permit data transfer and computation overlapping. The allocated buffer must
  2861. be freed thanks to the @code{starpu_free} function.
  2862. @end deftypefun
  2863. @node starpu_access_mode
  2864. @subsection @code{starpu_access_mode} -- Data access mode
  2865. This datatype describes a data access mode. The different available modes are:
  2866. @table @asis
  2867. @table @asis
  2868. @item @code{STARPU_R} read-only mode.
  2869. @item @code{STARPU_W} write-only mode.
  2870. @item @code{STARPU_RW} read-write mode. This is equivalent to @code{STARPU_R|STARPU_W}.
  2871. @item @code{STARPU_SCRATCH} scratch memory. A temporary buffer is allocated for the task, but StarPU does not enforce data consistency, i.e. each device has its own buffer, independently from each other (even for CPUs). This is useful for temporary variables. For now, no behaviour is defined concerning the relation with STARPU_R/W modes and the value provided at registration, i.e. the value of the scratch buffer is undefined at entry of the codelet function, but this is being considered for future extensions.
  2872. @item @code{STARPU_REDUX} reduction mode. TODO: document, as well as @code{starpu_data_set_reduction_methods}
  2873. @end table
  2874. @end table
  2875. @node unsigned memory_node
  2876. @subsection @code{unsigned memory_node} -- Memory node
  2877. @table @asis
  2878. @item @emph{Description}:
  2879. Every worker is associated to a memory node which is a logical abstraction of
  2880. the address space from which the processing unit gets its data. For instance,
  2881. the memory node associated to the different CPU workers represents main memory
  2882. (RAM), the memory node associated to a GPU is DRAM embedded on the device.
  2883. Every memory node is identified by a logical index which is accessible from the
  2884. @code{starpu_worker_get_memory_node} function. When registering a piece of data
  2885. to StarPU, the specified memory node indicates where the piece of data
  2886. initially resides (we also call this memory node the home node of a piece of
  2887. data).
  2888. @end table
  2889. @node starpu_data_handle
  2890. @subsection @code{starpu_data_handle} -- StarPU opaque data handle
  2891. @table @asis
  2892. @item @emph{Description}:
  2893. StarPU uses @code{starpu_data_handle} as an opaque handle to manage a piece of
  2894. data. Once a piece of data has been registered to StarPU, it is associated to a
  2895. @code{starpu_data_handle} which keeps track of the state of the piece of data
  2896. over the entire machine, so that we can maintain data consistency and locate
  2897. data replicates for instance.
  2898. @end table
  2899. @node void *interface
  2900. @subsection @code{void *interface} -- StarPU data interface
  2901. @table @asis
  2902. @item @emph{Description}:
  2903. Data management is done at a high-level in StarPU: rather than accessing a mere
  2904. list of contiguous buffers, the tasks may manipulate data that are described by
  2905. a high-level construct which we call data interface.
  2906. An example of data interface is the "vector" interface which describes a
  2907. contiguous data array on a spefic memory node. This interface is a simple
  2908. structure containing the number of elements in the array, the size of the
  2909. elements, and the address of the array in the appropriate address space (this
  2910. address may be invalid if there is no valid copy of the array in the memory
  2911. node). More informations on the data interfaces provided by StarPU are
  2912. given in @ref{Data Interfaces}.
  2913. When a piece of data managed by StarPU is used by a task, the task
  2914. implementation is given a pointer to an interface describing a valid copy of
  2915. the data that is accessible from the current processing unit.
  2916. @end table
  2917. @node starpu_data_register
  2918. @subsection @code{starpu_data_register} -- Register a piece of data to StarPU
  2919. @deftypefun void starpu_data_register (starpu_data_handle *@var{handleptr}, uint32_t @var{home_node}, void *@var{interface}, {struct starpu_data_interface_ops_t} *@var{ops})
  2920. Register a piece of data into the handle located at the @var{handleptr}
  2921. address. The @var{interface} buffer contains the initial description of the
  2922. data in the home node. The @var{ops} argument is a pointer to a structure
  2923. describing the different methods used to manipulate this type of interface. See
  2924. @ref{struct starpu_data_interface_ops_t} for more details on this structure.
  2925. If @code{home_node} is -1, StarPU will automatically
  2926. allocate the memory when it is used for the
  2927. first time in write-only mode. Once such data handle has been automatically
  2928. allocated, it is possible to access it using any access mode.
  2929. Note that StarPU supplies a set of predefined types of interface (e.g. vector or
  2930. matrix) which can be registered by the means of helper functions (e.g.
  2931. @code{starpu_vector_data_register} or @code{starpu_matrix_data_register}).
  2932. @end deftypefun
  2933. @node starpu_data_unregister
  2934. @subsection @code{starpu_data_unregister} -- Unregister a piece of data from StarPU
  2935. @deftypefun void starpu_data_unregister (starpu_data_handle @var{handle})
  2936. This function unregisters a data handle from StarPU. If the data was
  2937. automatically allocated by StarPU because the home node was -1, all
  2938. automatically allocated buffers are freed. Otherwise, a valid copy of the data
  2939. is put back into the home node in the buffer that was initially registered.
  2940. Using a data handle that has been unregistered from StarPU results in an
  2941. undefined behaviour.
  2942. @end deftypefun
  2943. @node starpu_data_unregister_no_coherency
  2944. @subsection @code{starpu_data_unregister_no_coherency} -- Unregister a piece of data from StarPU
  2945. @deftypefun void starpu_data_unregister_no_coherency (starpu_data_handle @var{handle})
  2946. This is the same as starpu_data_unregister, except that StarPU does not put back
  2947. a valid copy into the home node, in the buffer that was initially registered.
  2948. @end deftypefun
  2949. @node starpu_data_invalidate
  2950. @subsection @code{starpu_data_invalidate} -- Invalidate all data replicates
  2951. @deftypefun void starpu_data_invalidate (starpu_data_handle @var{handle})
  2952. Destroy all replicates of the data handle. After data invalidation, the first
  2953. access to the handle must be performed in write-only mode. Accessing an
  2954. invalidated data in read-mode results in undefined behaviour.
  2955. @end deftypefun
  2956. @c TODO create a specific sections about user interaction with the DSM ?
  2957. @node starpu_data_acquire
  2958. @subsection @code{starpu_data_acquire} -- Access registered data from the application
  2959. @deftypefun int starpu_data_acquire (starpu_data_handle @var{handle}, starpu_access_mode @var{mode})
  2960. The application must call this function prior to accessing registered data from
  2961. main memory outside tasks. StarPU ensures that the application will get an
  2962. up-to-date copy of the data in main memory located where the data was
  2963. originally registered, and that all concurrent accesses (e.g. from tasks) will
  2964. be consistent with the access mode specified in the @var{mode} argument.
  2965. @code{starpu_data_release} must be called once the application does not need to
  2966. access the piece of data anymore. Note that implicit data
  2967. dependencies are also enforced by @code{starpu_data_acquire}, i.e.
  2968. @code{starpu_data_acquire} will wait for all tasks scheduled to work on
  2969. the data, unless that they have not been disabled explictly by calling
  2970. @code{starpu_data_set_default_sequential_consistency_flag} or
  2971. @code{starpu_data_set_sequential_consistency_flag}.
  2972. @code{starpu_data_acquire} is a blocking call, so that it cannot be called from
  2973. tasks or from their callbacks (in that case, @code{starpu_data_acquire} returns
  2974. @code{-EDEADLK}). Upon successful completion, this function returns 0.
  2975. @end deftypefun
  2976. @node starpu_data_acquire_cb
  2977. @subsection @code{starpu_data_acquire_cb} -- Access registered data from the application asynchronously
  2978. @deftypefun int starpu_data_acquire_cb (starpu_data_handle @var{handle}, starpu_access_mode @var{mode}, void (*@var{callback})(void *), void *@var{arg})
  2979. @code{starpu_data_acquire_cb} is the asynchronous equivalent of
  2980. @code{starpu_data_release}. When the data specified in the first argument is
  2981. available in the appropriate access mode, the callback function is executed.
  2982. The application may access the requested data during the execution of this
  2983. callback. The callback function must call @code{starpu_data_release} once the
  2984. application does not need to access the piece of data anymore.
  2985. Note that implicit data dependencies are also enforced by
  2986. @code{starpu_data_acquire_cb} in case they are enabled.
  2987. Contrary to @code{starpu_data_acquire}, this function is non-blocking and may
  2988. be called from task callbacks. Upon successful completion, this function
  2989. returns 0.
  2990. @end deftypefun
  2991. @node STARPU_DATA_ACQUIRE_CB
  2992. @subsection @code{STARPU_DATA_ACQUIRE_CB} -- Access registered data from the application asynchronously, macro
  2993. @deftypefun STARPU_DATA_ACQUIRE_CB (starpu_data_handle @var{handle}, starpu_access_mode @var{mode}, code)
  2994. @code{STARPU_DATA_ACQUIRE_CB} is the same as @code{starpu_data_acquire_cb},
  2995. except that the code to be executed in a callback is directly provided as a
  2996. macro parameter, and the data handle is automatically released after it. This
  2997. permit to easily execute code which depends on the value of some registered
  2998. data. This is non-blocking too and may be called from task callbacks.
  2999. @end deftypefun
  3000. @node starpu_data_release
  3001. @subsection @code{starpu_data_release} -- Release registered data from the application
  3002. @deftypefun void starpu_data_release (starpu_data_handle @var{handle})
  3003. This function releases the piece of data acquired by the application either by
  3004. @code{starpu_data_acquire} or by @code{starpu_data_acquire_cb}.
  3005. @end deftypefun
  3006. @node starpu_data_set_wt_mask
  3007. @subsection @code{starpu_data_set_wt_mask} -- Set the Write-Through mask
  3008. @deftypefun void starpu_data_set_wt_mask (starpu_data_handle @var{handle}, uint32_t @var{wt_mask})
  3009. This function sets the write-through mask of a given data, i.e. a bitmask of
  3010. nodes where the data should be always replicated after modification.
  3011. @end deftypefun
  3012. @node starpu_data_prefetch_on_node
  3013. @subsection @code{starpu_data_prefetch_on_node} -- Prefetch data to a given node
  3014. @deftypefun int starpu_data_prefetch_on_node (starpu_data_handle @var{handle}, unsigned @var{node}, unsigned @var{async})
  3015. Issue a prefetch request for a given data to a given node, i.e.
  3016. requests that the data be replicated to the given node, so that it is available
  3017. there for tasks. If the @var{async} parameter is 0, the call will block until
  3018. the transfer is achieved, else the call will return as soon as the request is
  3019. scheduled (which may however have to wait for a task completion).
  3020. @end deftypefun
  3021. @node Data Interfaces
  3022. @section Data Interfaces
  3023. @menu
  3024. * Variable Interface::
  3025. * Vector Interface::
  3026. * Matrix Interface::
  3027. * 3D Matrix Interface::
  3028. * BCSR Interface for Sparse Matrices (Blocked Compressed Sparse Row Representation)::
  3029. * CSR Interface for Sparse Matrices (Compressed Sparse Row Representation)::
  3030. @end menu
  3031. @node Variable Interface
  3032. @subsection Variable Interface
  3033. @table @asis
  3034. @item @emph{Description}:
  3035. This variant of @code{starpu_data_register} uses the variable interface,
  3036. i.e. for a mere single variable. @code{ptr} is the address of the variable,
  3037. and @code{elemsize} is the size of the variable.
  3038. @item @emph{Prototype}:
  3039. @code{void starpu_variable_data_register(starpu_data_handle *handle,
  3040. uint32_t home_node,
  3041. uintptr_t ptr, size_t elemsize);}
  3042. @item @emph{Example}:
  3043. @cartouche
  3044. @smallexample
  3045. float var;
  3046. starpu_data_handle var_handle;
  3047. starpu_variable_data_register(&var_handle, 0, (uintptr_t)&var, sizeof(var));
  3048. @end smallexample
  3049. @end cartouche
  3050. @end table
  3051. @node Vector Interface
  3052. @subsection Vector Interface
  3053. @table @asis
  3054. @item @emph{Description}:
  3055. This variant of @code{starpu_data_register} uses the vector interface,
  3056. i.e. for mere arrays of elements. @code{ptr} is the address of the first
  3057. element in the home node. @code{nx} is the number of elements in the vector.
  3058. @code{elemsize} is the size of each element.
  3059. @item @emph{Prototype}:
  3060. @code{void starpu_vector_data_register(starpu_data_handle *handle, uint32_t home_node,
  3061. uintptr_t ptr, uint32_t nx, size_t elemsize);}
  3062. @item @emph{Example}:
  3063. @cartouche
  3064. @smallexample
  3065. float vector[NX];
  3066. starpu_data_handle vector_handle;
  3067. starpu_vector_data_register(&vector_handle, 0, (uintptr_t)vector, NX,
  3068. sizeof(vector[0]));
  3069. @end smallexample
  3070. @end cartouche
  3071. @end table
  3072. @node Matrix Interface
  3073. @subsection Matrix Interface
  3074. @table @asis
  3075. @item @emph{Description}:
  3076. This variant of @code{starpu_data_register} uses the matrix interface, i.e. for
  3077. matrices of elements. @code{ptr} is the address of the first element in the home
  3078. node. @code{ld} is the number of elements between rows. @code{nx} is the number
  3079. of elements in a row (this can be different from @code{ld} if there are extra
  3080. elements for alignment for instance). @code{ny} is the number of rows.
  3081. @code{elemsize} is the size of each element.
  3082. @item @emph{Prototype}:
  3083. @code{void starpu_matrix_data_register(starpu_data_handle *handle, uint32_t home_node,
  3084. uintptr_t ptr, uint32_t ld, uint32_t nx,
  3085. uint32_t ny, size_t elemsize);}
  3086. @item @emph{Example}:
  3087. @cartouche
  3088. @smallexample
  3089. float *matrix;
  3090. starpu_data_handle matrix_handle;
  3091. matrix = (float*)malloc(width * height * sizeof(float));
  3092. starpu_matrix_data_register(&matrix_handle, 0, (uintptr_t)matrix,
  3093. width, width, height, sizeof(float));
  3094. @end smallexample
  3095. @end cartouche
  3096. @end table
  3097. @node 3D Matrix Interface
  3098. @subsection 3D Matrix Interface
  3099. @table @asis
  3100. @item @emph{Description}:
  3101. This variant of @code{starpu_data_register} uses the 3D matrix interface.
  3102. @code{ptr} is the address of the array of first element in the home node.
  3103. @code{ldy} is the number of elements between rows. @code{ldz} is the number
  3104. of rows between z planes. @code{nx} is the number of elements in a row (this
  3105. can be different from @code{ldy} if there are extra elements for alignment
  3106. for instance). @code{ny} is the number of rows in a z plane (likewise with
  3107. @code{ldz}). @code{nz} is the number of z planes. @code{elemsize} is the size of
  3108. each element.
  3109. @item @emph{Prototype}:
  3110. @code{void starpu_block_data_register(starpu_data_handle *handle, uint32_t home_node,
  3111. uintptr_t ptr, uint32_t ldy, uint32_t ldz, uint32_t nx,
  3112. uint32_t ny, uint32_t nz, size_t elemsize);}
  3113. @item @emph{Example}:
  3114. @cartouche
  3115. @smallexample
  3116. float *block;
  3117. starpu_data_handle block_handle;
  3118. block = (float*)malloc(nx*ny*nz*sizeof(float));
  3119. starpu_block_data_register(&block_handle, 0, (uintptr_t)block,
  3120. nx, nx*ny, nx, ny, nz, sizeof(float));
  3121. @end smallexample
  3122. @end cartouche
  3123. @end table
  3124. @node BCSR Interface for Sparse Matrices (Blocked Compressed Sparse Row Representation)
  3125. @subsection BCSR Interface for Sparse Matrices (Blocked Compressed Sparse Row Representation)
  3126. @deftypefun void starpu_bcsr_data_register (starpu_data_handle *@var{handle}, uint32_t @var{home_node}, uint32_t @var{nnz}, uint32_t @var{nrow}, uintptr_t @var{nzval}, uint32_t *@var{colind}, uint32_t *@var{rowptr}, uint32_t @var{firstentry}, uint32_t @var{r}, uint32_t @var{c}, size_t @var{elemsize})
  3127. This variant of @code{starpu_data_register} uses the BCSR sparse matrix interface.
  3128. TODO
  3129. @end deftypefun
  3130. @node CSR Interface for Sparse Matrices (Compressed Sparse Row Representation)
  3131. @subsection CSR Interface for Sparse Matrices (Compressed Sparse Row Representation)
  3132. @deftypefun void starpu_csr_data_register (starpu_data_handle *@var{handle}, uint32_t @var{home_node}, uint32_t @var{nnz}, uint32_t @var{nrow}, uintptr_t @var{nzval}, uint32_t *@var{colind}, uint32_t *@var{rowptr}, uint32_t @var{firstentry}, size_t @var{elemsize})
  3133. This variant of @code{starpu_data_register} uses the CSR sparse matrix interface.
  3134. TODO
  3135. @end deftypefun
  3136. @node Data Partition
  3137. @section Data Partition
  3138. @menu
  3139. * struct starpu_data_filter:: StarPU filter structure
  3140. * starpu_data_partition:: Partition Data
  3141. * starpu_data_unpartition:: Unpartition Data
  3142. * starpu_data_get_nb_children::
  3143. * starpu_data_get_sub_data::
  3144. * Predefined filter functions::
  3145. @end menu
  3146. @node struct starpu_data_filter
  3147. @subsection @code{struct starpu_data_filter} -- StarPU filter structure
  3148. @table @asis
  3149. @item @emph{Description}:
  3150. The filter structure describes a data partitioning operation, to be given to the
  3151. @code{starpu_data_partition} function, see @ref{starpu_data_partition} for an example.
  3152. @item @emph{Fields}:
  3153. @table @asis
  3154. @item @code{filter_func}:
  3155. This function fills the @code{child_interface} structure with interface
  3156. information for the @code{id}-th child of the parent @code{father_interface} (among @code{nparts}).
  3157. @code{void (*filter_func)(void *father_interface, void* child_interface, struct starpu_data_filter *, unsigned id, unsigned nparts);}
  3158. @item @code{nchildren}:
  3159. This is the number of parts to partition the data into.
  3160. @item @code{get_nchildren}:
  3161. This returns the number of children. This can be used instead of @code{nchildren} when the number of
  3162. children depends on the actual data (e.g. the number of blocks in a sparse
  3163. matrix).
  3164. @code{unsigned (*get_nchildren)(struct starpu_data_filter *, starpu_data_handle initial_handle);}
  3165. @item @code{get_child_ops}:
  3166. In case the resulting children use a different data interface, this function
  3167. returns which interface is used by child number @code{id}.
  3168. @code{struct starpu_data_interface_ops_t *(*get_child_ops)(struct starpu_data_filter *, unsigned id);}
  3169. @item @code{filter_arg}:
  3170. Some filters take an addition parameter, but this is usually unused.
  3171. @item @code{filter_arg_ptr}:
  3172. Some filters take an additional array parameter like the sizes of the parts, but
  3173. this is usually unused.
  3174. @end table
  3175. @end table
  3176. @node starpu_data_partition
  3177. @subsection starpu_data_partition -- Partition Data
  3178. @table @asis
  3179. @item @emph{Description}:
  3180. This requests partitioning one StarPU data @code{initial_handle} into several
  3181. subdata according to the filter @code{f}
  3182. @item @emph{Prototype}:
  3183. @code{void starpu_data_partition(starpu_data_handle initial_handle, struct starpu_data_filter *f);}
  3184. @item @emph{Example}:
  3185. @cartouche
  3186. @smallexample
  3187. struct starpu_data_filter f = @{
  3188. .filter_func = starpu_vertical_block_filter_func,
  3189. .nchildren = nslicesx,
  3190. .get_nchildren = NULL,
  3191. .get_child_ops = NULL
  3192. @};
  3193. starpu_data_partition(A_handle, &f);
  3194. @end smallexample
  3195. @end cartouche
  3196. @end table
  3197. @node starpu_data_unpartition
  3198. @subsection starpu_data_unpartition -- Unpartition data
  3199. @table @asis
  3200. @item @emph{Description}:
  3201. This unapplies one filter, thus unpartitioning the data. The pieces of data are
  3202. collected back into one big piece in the @code{gathering_node} (usually 0).
  3203. @item @emph{Prototype}:
  3204. @code{void starpu_data_unpartition(starpu_data_handle root_data, uint32_t gathering_node);}
  3205. @item @emph{Example}:
  3206. @cartouche
  3207. @smallexample
  3208. starpu_data_unpartition(A_handle, 0);
  3209. @end smallexample
  3210. @end cartouche
  3211. @end table
  3212. @node starpu_data_get_nb_children
  3213. @subsection starpu_data_get_nb_children
  3214. @table @asis
  3215. @item @emph{Description}:
  3216. This function returns the number of children.
  3217. @item @emph{Return value}:
  3218. The number of children.
  3219. @item @emph{Prototype}:
  3220. @code{int starpu_data_get_nb_children(starpu_data_handle handle);}
  3221. @end table
  3222. @c starpu_data_handle starpu_data_get_child(starpu_data_handle handle, unsigned i);
  3223. @node starpu_data_get_sub_data
  3224. @subsection starpu_data_get_sub_data
  3225. @table @asis
  3226. @item @emph{Description}:
  3227. After partitioning a StarPU data by applying a filter,
  3228. @code{starpu_data_get_sub_data} can be used to get handles for each of the data
  3229. portions. @code{root_data} is the parent data that was partitioned. @code{depth}
  3230. is the number of filters to traverse (in case several filters have been applied,
  3231. to e.g. partition in row blocks, and then in column blocks), and the subsequent
  3232. parameters are the indexes.
  3233. @item @emph{Return value}:
  3234. A handle to the subdata.
  3235. @item @emph{Prototype}:
  3236. @code{starpu_data_handle starpu_data_get_sub_data(starpu_data_handle root_data, unsigned depth, ... );}
  3237. @item @emph{Example}:
  3238. @cartouche
  3239. @smallexample
  3240. h = starpu_data_get_sub_data(A_handle, 1, taskx);
  3241. @end smallexample
  3242. @end cartouche
  3243. @end table
  3244. @node Predefined filter functions
  3245. @subsection Predefined filter functions
  3246. @menu
  3247. * Partitioning BCSR Data::
  3248. * Partitioning BLAS interface::
  3249. * Partitioning Vector Data::
  3250. * Partitioning Block Data::
  3251. @end menu
  3252. This section gives a partial list of the predefined partitioning functions.
  3253. Examples on how to use them are shown in @ref{Partitioning Data}. The complete
  3254. list can be found in @code{starpu_data_filters.h} .
  3255. @node Partitioning BCSR Data
  3256. @subsubsection Partitioning BCSR Data
  3257. @deftypefun void starpu_canonical_block_filter_bcsr (void *@var{father_interface}, void *@var{child_interface}, {struct starpu_data_filter} *@var{f}, unsigned @var{id}, unsigned @var{nparts})
  3258. TODO
  3259. @end deftypefun
  3260. @deftypefun void starpu_vertical_block_filter_func_csr (void *@var{father_interface}, void *@var{child_interface}, {struct starpu_data_filter} *@var{f}, unsigned @var{id}, unsigned @var{nparts})
  3261. TODO
  3262. @end deftypefun
  3263. @node Partitioning BLAS interface
  3264. @subsubsection Partitioning BLAS interface
  3265. @deftypefun void starpu_block_filter_func (void *@var{father_interface}, void *@var{child_interface}, {struct starpu_data_filter} *@var{f}, unsigned @var{id}, unsigned @var{nparts})
  3266. This partitions a dense Matrix into horizontal blocks.
  3267. @end deftypefun
  3268. @deftypefun void starpu_vertical_block_filter_func (void *@var{father_interface}, void *@var{child_interface}, {struct starpu_data_filter} *@var{f}, unsigned @var{id}, unsigned @var{nparts})
  3269. This partitions a dense Matrix into vertical blocks.
  3270. @end deftypefun
  3271. @node Partitioning Vector Data
  3272. @subsubsection Partitioning Vector Data
  3273. @deftypefun void starpu_block_filter_func_vector (void *@var{father_interface}, void *@var{child_interface}, {struct starpu_data_filter} *@var{f}, unsigned @var{id}, unsigned @var{nparts})
  3274. This partitions a vector into blocks of the same size.
  3275. @end deftypefun
  3276. @deftypefun void starpu_vector_list_filter_func (void *@var{father_interface}, void *@var{child_interface}, {struct starpu_data_filter} *@var{f}, unsigned @var{id}, unsigned @var{nparts})
  3277. This partitions a vector into blocks of sizes given in the @var{filter_arg_ptr}
  3278. field of @var{f}, supposed to point on a @code{uint32_t*} array.
  3279. @end deftypefun
  3280. @deftypefun void starpu_vector_divide_in_2_filter_func (void *@var{father_interface}, void *@var{child_interface}, {struct starpu_data_filter} *@var{f}, unsigned @var{id}, unsigned @var{nparts})
  3281. This partitions a vector into two blocks, the first block size being given in
  3282. the @var{filter_arg} field of @var{f}.
  3283. @end deftypefun
  3284. @node Partitioning Block Data
  3285. @subsubsection Partitioning Block Data
  3286. @deftypefun void starpu_block_filter_func_block (void *@var{father_interface}, void *@var{child_interface}, {struct starpu_data_filter} *@var{f}, unsigned @var{id}, unsigned @var{nparts})
  3287. This partitions a 3D matrix along the X axis.
  3288. @end deftypefun
  3289. @node Codelets and Tasks
  3290. @section Codelets and Tasks
  3291. This section describes the interface to manipulate codelets and tasks.
  3292. @deftp {Data Type} {struct starpu_codelet}
  3293. The codelet structure describes a kernel that is possibly implemented on various
  3294. targets. For compatibility, make sure to initialize the whole structure to zero.
  3295. @table @asis
  3296. @item @code{where}
  3297. Indicates which types of processing units are able to execute the codelet.
  3298. @code{STARPU_CPU|STARPU_CUDA} for instance indicates that the codelet is
  3299. implemented for both CPU cores and CUDA devices while @code{STARPU_GORDON}
  3300. indicates that it is only available on Cell SPUs.
  3301. @item @code{cpu_func} (optional)
  3302. Is a function pointer to the CPU implementation of the codelet. Its prototype
  3303. must be: @code{void cpu_func(void *buffers[], void *cl_arg)}. The first
  3304. argument being the array of data managed by the data management library, and
  3305. the second argument is a pointer to the argument passed from the @code{cl_arg}
  3306. field of the @code{starpu_task} structure.
  3307. The @code{cpu_func} field is ignored if @code{STARPU_CPU} does not appear in
  3308. the @code{where} field, it must be non-null otherwise.
  3309. @item @code{cuda_func} (optional)
  3310. Is a function pointer to the CUDA implementation of the codelet. @emph{This
  3311. must be a host-function written in the CUDA runtime API}. Its prototype must
  3312. be: @code{void cuda_func(void *buffers[], void *cl_arg);}. The @code{cuda_func}
  3313. field is ignored if @code{STARPU_CUDA} does not appear in the @code{where}
  3314. field, it must be non-null otherwise.
  3315. @item @code{opencl_func} (optional)
  3316. Is a function pointer to the OpenCL implementation of the codelet. Its
  3317. prototype must be:
  3318. @code{void opencl_func(starpu_data_interface_t *descr, void *arg);}.
  3319. This pointer is ignored if @code{STARPU_OPENCL} does not appear in the
  3320. @code{where} field, it must be non-null otherwise.
  3321. @item @code{gordon_func} (optional)
  3322. This is the index of the Cell SPU implementation within the Gordon library.
  3323. See Gordon documentation for more details on how to register a kernel and
  3324. retrieve its index.
  3325. @item @code{nbuffers}
  3326. Specifies the number of arguments taken by the codelet. These arguments are
  3327. managed by the DSM and are accessed from the @code{void *buffers[]}
  3328. array. The constant argument passed with the @code{cl_arg} field of the
  3329. @code{starpu_task} structure is not counted in this number. This value should
  3330. not be above @code{STARPU_NMAXBUFS}.
  3331. @item @code{model} (optional)
  3332. This is a pointer to the task duration performance model associated to this
  3333. codelet. This optional field is ignored when set to @code{NULL}.
  3334. TODO
  3335. @item @code{power_model} (optional)
  3336. This is a pointer to the task power consumption performance model associated
  3337. to this codelet. This optional field is ignored when set to @code{NULL}.
  3338. In the case of parallel codelets, this has to account for all processing units
  3339. involved in the parallel execution.
  3340. TODO
  3341. @end table
  3342. @end deftp
  3343. @deftp {Data Type} {struct starpu_task}
  3344. The @code{starpu_task} structure describes a task that can be offloaded on the various
  3345. processing units managed by StarPU. It instantiates a codelet. It can either be
  3346. allocated dynamically with the @code{starpu_task_create} method, or declared
  3347. statically. In the latter case, the programmer has to zero the
  3348. @code{starpu_task} structure and to fill the different fields properly. The
  3349. indicated default values correspond to the configuration of a task allocated
  3350. with @code{starpu_task_create}.
  3351. @table @asis
  3352. @item @code{cl}
  3353. Is a pointer to the corresponding @code{starpu_codelet} data structure. This
  3354. describes where the kernel should be executed, and supplies the appropriate
  3355. implementations. When set to @code{NULL}, no code is executed during the tasks,
  3356. such empty tasks can be useful for synchronization purposes.
  3357. @item @code{buffers}
  3358. Is an array of @code{starpu_buffer_descr_t} structures. It describes the
  3359. different pieces of data accessed by the task, and how they should be accessed.
  3360. The @code{starpu_buffer_descr_t} structure is composed of two fields, the
  3361. @code{handle} field specifies the handle of the piece of data, and the
  3362. @code{mode} field is the required access mode (eg @code{STARPU_RW}). The number
  3363. of entries in this array must be specified in the @code{nbuffers} field of the
  3364. @code{starpu_codelet} structure, and should not excede @code{STARPU_NMAXBUFS}.
  3365. If unsufficient, this value can be set with the @code{--enable-maxbuffers}
  3366. option when configuring StarPU.
  3367. @item @code{cl_arg} (optional; default: @code{NULL})
  3368. This pointer is passed to the codelet through the second argument
  3369. of the codelet implementation (e.g. @code{cpu_func} or @code{cuda_func}).
  3370. In the specific case of the Cell processor, see the @code{cl_arg_size}
  3371. argument.
  3372. @item @code{cl_arg_size} (optional, Cell-specific)
  3373. In the case of the Cell processor, the @code{cl_arg} pointer is not directly
  3374. given to the SPU function. A buffer of size @code{cl_arg_size} is allocated on
  3375. the SPU. This buffer is then filled with the @code{cl_arg_size} bytes starting
  3376. at address @code{cl_arg}. In this case, the argument given to the SPU codelet
  3377. is therefore not the @code{cl_arg} pointer, but the address of the buffer in
  3378. local store (LS) instead. This field is ignored for CPU, CUDA and OpenCL
  3379. codelets, where the @code{cl_arg} pointer is given as such.
  3380. @item @code{callback_func} (optional) (default: @code{NULL})
  3381. This is a function pointer of prototype @code{void (*f)(void *)} which
  3382. specifies a possible callback. If this pointer is non-null, the callback
  3383. function is executed @emph{on the host} after the execution of the task. The
  3384. callback is passed the value contained in the @code{callback_arg} field. No
  3385. callback is executed if the field is set to @code{NULL}.
  3386. @item @code{callback_arg} (optional) (default: @code{NULL})
  3387. This is the pointer passed to the callback function. This field is ignored if
  3388. the @code{callback_func} is set to @code{NULL}.
  3389. @item @code{use_tag} (optional) (default: @code{0})
  3390. If set, this flag indicates that the task should be associated with the tag
  3391. contained in the @code{tag_id} field. Tag allow the application to synchronize
  3392. with the task and to express task dependencies easily.
  3393. @item @code{tag_id}
  3394. This fields contains the tag associated to the task if the @code{use_tag} field
  3395. was set, it is ignored otherwise.
  3396. @item @code{synchronous}
  3397. If this flag is set, the @code{starpu_task_submit} function is blocking and
  3398. returns only when the task has been executed (or if no worker is able to
  3399. process the task). Otherwise, @code{starpu_task_submit} returns immediately.
  3400. @item @code{priority} (optional) (default: @code{STARPU_DEFAULT_PRIO})
  3401. This field indicates a level of priority for the task. This is an integer value
  3402. that must be set between the return values of the
  3403. @code{starpu_sched_get_min_priority} function for the least important tasks,
  3404. and that of the @code{starpu_sched_get_max_priority} for the most important
  3405. tasks (included). The @code{STARPU_MIN_PRIO} and @code{STARPU_MAX_PRIO} macros
  3406. are provided for convenience and respectively returns value of
  3407. @code{starpu_sched_get_min_priority} and @code{starpu_sched_get_max_priority}.
  3408. Default priority is @code{STARPU_DEFAULT_PRIO}, which is always defined as 0 in
  3409. order to allow static task initialization. Scheduling strategies that take
  3410. priorities into account can use this parameter to take better scheduling
  3411. decisions, but the scheduling policy may also ignore it.
  3412. @item @code{execute_on_a_specific_worker} (default: @code{0})
  3413. If this flag is set, StarPU will bypass the scheduler and directly affect this
  3414. task to the worker specified by the @code{workerid} field.
  3415. @item @code{workerid} (optional)
  3416. If the @code{execute_on_a_specific_worker} field is set, this field indicates
  3417. which is the identifier of the worker that should process this task (as
  3418. returned by @code{starpu_worker_get_id}). This field is ignored if
  3419. @code{execute_on_a_specific_worker} field is set to 0.
  3420. @item @code{detach} (optional) (default: @code{1})
  3421. If this flag is set, it is not possible to synchronize with the task
  3422. by the means of @code{starpu_task_wait} later on. Internal data structures
  3423. are only guaranteed to be freed once @code{starpu_task_wait} is called if the
  3424. flag is not set.
  3425. @item @code{destroy} (optional) (default: @code{1})
  3426. If this flag is set, the task structure will automatically be freed, either
  3427. after the execution of the callback if the task is detached, or during
  3428. @code{starpu_task_wait} otherwise. If this flag is not set, dynamically
  3429. allocated data structures will not be freed until @code{starpu_task_destroy} is
  3430. called explicitly. Setting this flag for a statically allocated task structure
  3431. will result in undefined behaviour.
  3432. @item @code{predicted} (output field)
  3433. Predicted duration of the task. This field is only set if the scheduling
  3434. strategy used performance models.
  3435. @end table
  3436. @end deftp
  3437. @deftypefun void starpu_task_init ({struct starpu_task} *@var{task})
  3438. Initialize @var{task} with default values. This function is implicitly
  3439. called by @code{starpu_task_create}. By default, tasks initialized with
  3440. @code{starpu_task_init} must be deinitialized explicitly with
  3441. @code{starpu_task_deinit}. Tasks can also be initialized statically, using the
  3442. constant @code{STARPU_TASK_INITIALIZER}.
  3443. @end deftypefun
  3444. @deftypefun {struct starpu_task *} starpu_task_create (void)
  3445. Allocate a task structure and initialize it with default values. Tasks
  3446. allocated dynamically with @code{starpu_task_create} are automatically freed when the
  3447. task is terminated. If the destroy flag is explicitly unset, the resources used
  3448. by the task are freed by calling
  3449. @code{starpu_task_destroy}.
  3450. @end deftypefun
  3451. @deftypefun void starpu_task_deinit ({struct starpu_task} *@var{task})
  3452. Release all the structures automatically allocated to execute @var{task}. This is
  3453. called automatically by @code{starpu_task_destroy}, but the task structure itself is not
  3454. freed. This should be used for statically allocated tasks for instance.
  3455. @end deftypefun
  3456. @deftypefun void starpu_task_destroy ({struct starpu_task} *@var{task})
  3457. Free the resource allocated during @code{starpu_task_create} and
  3458. associated with @var{task}. This function can be called automatically
  3459. after the execution of a task by setting the @code{destroy} flag of the
  3460. @code{starpu_task} structure (default behaviour). Calling this function
  3461. on a statically allocated task results in an undefined behaviour.
  3462. @end deftypefun
  3463. @deftypefun int starpu_task_wait ({struct starpu_task} *@var{task})
  3464. This function blocks until @var{task} has been executed. It is not possible to
  3465. synchronize with a task more than once. It is not possible to wait for
  3466. synchronous or detached tasks.
  3467. Upon successful completion, this function returns 0. Otherwise, @code{-EINVAL}
  3468. indicates that the specified task was either synchronous or detached.
  3469. @end deftypefun
  3470. @deftypefun int starpu_task_submit ({struct starpu_task} *@var{task})
  3471. This function submits @var{task} to StarPU. Calling this function does
  3472. not mean that the task will be executed immediately as there can be data or task
  3473. (tag) dependencies that are not fulfilled yet: StarPU will take care of
  3474. scheduling this task with respect to such dependencies.
  3475. This function returns immediately if the @code{synchronous} field of the
  3476. @code{starpu_task} structure was set to 0, and block until the termination of
  3477. the task otherwise. It is also possible to synchronize the application with
  3478. asynchronous tasks by the means of tags, using the @code{starpu_tag_wait}
  3479. function for instance.
  3480. In case of success, this function returns 0, a return value of @code{-ENODEV}
  3481. means that there is no worker able to process this task (e.g. there is no GPU
  3482. available and this task is only implemented for CUDA devices).
  3483. @end deftypefun
  3484. @deftypefun int starpu_task_wait_for_all (void)
  3485. This function blocks until all the tasks that were submitted are terminated.
  3486. @end deftypefun
  3487. @deftypefun {struct starpu_task *} starpu_get_current_task (void)
  3488. This function returns the task currently executed by the worker, or
  3489. NULL if it is called either from a thread that is not a task or simply
  3490. because there is no task being executed at the moment.
  3491. @end deftypefun
  3492. @deftypefun void starpu_display_codelet_stats ({struct starpu_codelet_t} *@var{cl})
  3493. Output on @code{stderr} some statistics on the codelet @var{cl}.
  3494. @end deftypefun
  3495. @c Callbacks : what can we put in callbacks ?
  3496. @node Explicit Dependencies
  3497. @section Explicit Dependencies
  3498. @menu
  3499. * starpu_task_declare_deps_array:: starpu_task_declare_deps_array
  3500. * starpu_tag_t:: Task logical identifier
  3501. * starpu_tag_declare_deps:: Declare the Dependencies of a Tag
  3502. * starpu_tag_declare_deps_array:: Declare the Dependencies of a Tag
  3503. * starpu_tag_wait:: Block until a Tag is terminated
  3504. * starpu_tag_wait_array:: Block until a set of Tags is terminated
  3505. * starpu_tag_remove:: Destroy a Tag
  3506. * starpu_tag_notify_from_apps:: Feed a tag explicitly
  3507. @end menu
  3508. @node starpu_task_declare_deps_array
  3509. @subsection @code{starpu_task_declare_deps_array} -- Declare task dependencies
  3510. @deftypefun void starpu_task_declare_deps_array ({struct starpu_task} *@var{task}, unsigned @var{ndeps}, {struct starpu_task} *@var{task_array}[])
  3511. Declare task dependencies between a @var{task} and an array of tasks of length
  3512. @var{ndeps}. This function must be called prior to the submission of the task,
  3513. but it may called after the submission or the execution of the tasks in the
  3514. array provided the tasks are still valid (ie. they were not automatically
  3515. destroyed). Calling this function on a task that was already submitted or with
  3516. an entry of @var{task_array} that is not a valid task anymore results in an
  3517. undefined behaviour. If @var{ndeps} is null, no dependency is added. It is
  3518. possible to call @code{starpu_task_declare_deps_array} multiple times on the
  3519. same task, in this case, the dependencies are added. It is possible to have
  3520. redundancy in the task dependencies.
  3521. @end deftypefun
  3522. @node starpu_tag_t
  3523. @subsection @code{starpu_tag_t} -- Task logical identifier
  3524. @table @asis
  3525. @item @emph{Description}:
  3526. It is possible to associate a task with a unique ``tag'' chosen by the application, and to express
  3527. dependencies between tasks by the means of those tags. To do so, fill the
  3528. @code{tag_id} field of the @code{starpu_task} structure with a tag number (can
  3529. be arbitrary) and set the @code{use_tag} field to 1.
  3530. If @code{starpu_tag_declare_deps} is called with this tag number, the task will
  3531. not be started until the tasks which holds the declared dependency tags are
  3532. completed.
  3533. @end table
  3534. @node starpu_tag_declare_deps
  3535. @subsection @code{starpu_tag_declare_deps} -- Declare the Dependencies of a Tag
  3536. @table @asis
  3537. @item @emph{Description}:
  3538. Specify the dependencies of the task identified by tag @code{id}. The first
  3539. argument specifies the tag which is configured, the second argument gives the
  3540. number of tag(s) on which @code{id} depends. The following arguments are the
  3541. tags which have to be terminated to unlock the task.
  3542. This function must be called before the associated task is submitted to StarPU
  3543. with @code{starpu_task_submit}.
  3544. @item @emph{Remark}
  3545. Because of the variable arity of @code{starpu_tag_declare_deps}, note that the
  3546. last arguments @emph{must} be of type @code{starpu_tag_t}: constant values
  3547. typically need to be explicitly casted. Using the
  3548. @code{starpu_tag_declare_deps_array} function avoids this hazard.
  3549. @item @emph{Prototype}:
  3550. @code{void starpu_tag_declare_deps(starpu_tag_t id, unsigned ndeps, ...);}
  3551. @item @emph{Example}:
  3552. @cartouche
  3553. @example
  3554. /* Tag 0x1 depends on tags 0x32 and 0x52 */
  3555. starpu_tag_declare_deps((starpu_tag_t)0x1,
  3556. 2, (starpu_tag_t)0x32, (starpu_tag_t)0x52);
  3557. @end example
  3558. @end cartouche
  3559. @end table
  3560. @node starpu_tag_declare_deps_array
  3561. @subsection @code{starpu_tag_declare_deps_array} -- Declare the Dependencies of a Tag
  3562. @table @asis
  3563. @item @emph{Description}:
  3564. This function is similar to @code{starpu_tag_declare_deps}, except that its
  3565. does not take a variable number of arguments but an array of tags of size
  3566. @code{ndeps}.
  3567. @item @emph{Prototype}:
  3568. @code{void starpu_tag_declare_deps_array(starpu_tag_t id, unsigned ndeps, starpu_tag_t *array);}
  3569. @item @emph{Example}:
  3570. @cartouche
  3571. @example
  3572. /* Tag 0x1 depends on tags 0x32 and 0x52 */
  3573. starpu_tag_t tag_array[2] = @{0x32, 0x52@};
  3574. starpu_tag_declare_deps_array((starpu_tag_t)0x1, 2, tag_array);
  3575. @end example
  3576. @end cartouche
  3577. @end table
  3578. @node starpu_tag_wait
  3579. @subsection @code{starpu_tag_wait} -- Block until a Tag is terminated
  3580. @deftypefun void starpu_tag_wait (starpu_tag_t @var{id})
  3581. This function blocks until the task associated to tag @var{id} has been
  3582. executed. This is a blocking call which must therefore not be called within
  3583. tasks or callbacks, but only from the application directly. It is possible to
  3584. synchronize with the same tag multiple times, as long as the
  3585. @code{starpu_tag_remove} function is not called. Note that it is still
  3586. possible to synchronize with a tag associated to a task which @code{starpu_task}
  3587. data structure was freed (e.g. if the @code{destroy} flag of the
  3588. @code{starpu_task} was enabled).
  3589. @end deftypefun
  3590. @node starpu_tag_wait_array
  3591. @subsection @code{starpu_tag_wait_array} -- Block until a set of Tags is terminated
  3592. @deftypefun void starpu_tag_wait_array (unsigned @var{ntags}, starpu_tag_t *@var{id})
  3593. This function is similar to @code{starpu_tag_wait} except that it blocks until
  3594. @emph{all} the @var{ntags} tags contained in the @var{id} array are
  3595. terminated.
  3596. @end deftypefun
  3597. @node starpu_tag_remove
  3598. @subsection @code{starpu_tag_remove} -- Destroy a Tag
  3599. @deftypefun void starpu_tag_remove (starpu_tag_t @var{id})
  3600. This function releases the resources associated to tag @var{id}. It can be
  3601. called once the corresponding task has been executed and when there is
  3602. no other tag that depend on this tag anymore.
  3603. @end deftypefun
  3604. @node starpu_tag_notify_from_apps
  3605. @subsection @code{starpu_tag_notify_from_apps} -- Feed a Tag explicitly
  3606. @deftypefun void starpu_tag_notify_from_apps (starpu_tag_t @var{id})
  3607. This function explicitly unlocks tag @var{id}. It may be useful in the
  3608. case of applications which execute part of their computation outside StarPU
  3609. tasks (e.g. third-party libraries). It is also provided as a
  3610. convenient tool for the programmer, for instance to entirely construct the task
  3611. DAG before actually giving StarPU the opportunity to execute the tasks.
  3612. @end deftypefun
  3613. @node Implicit Data Dependencies
  3614. @section Implicit Data Dependencies
  3615. @menu
  3616. * starpu_data_set_default_sequential_consistency_flag:: starpu_data_set_default_sequential_consistency_flag
  3617. * starpu_data_get_default_sequential_consistency_flag:: starpu_data_get_default_sequential_consistency_flag
  3618. * starpu_data_set_sequential_consistency_flag:: starpu_data_set_sequential_consistency_flag
  3619. @end menu
  3620. In this section, we describe how StarPU makes it possible to insert implicit
  3621. task dependencies in order to enforce sequential data consistency. When this
  3622. data consistency is enabled on a specific data handle, any data access will
  3623. appear as sequentially consistent from the application. For instance, if the
  3624. application submits two tasks that access the same piece of data in read-only
  3625. mode, and then a third task that access it in write mode, dependencies will be
  3626. added between the two first tasks and the third one. Implicit data dependencies
  3627. are also inserted in the case of data accesses from the application.
  3628. @node starpu_data_set_default_sequential_consistency_flag
  3629. @subsection @code{starpu_data_set_default_sequential_consistency_flag} -- Set default sequential consistency flag
  3630. @deftypefun void starpu_data_set_default_sequential_consistency_flag (unsigned @var{flag})
  3631. Set the default sequential consistency flag. If a non-zero value is passed, a
  3632. sequential data consistency will be enforced for all handles registered after
  3633. this function call, otherwise it is disabled. By default, StarPU enables
  3634. sequential data consistency. It is also possible to select the data consistency
  3635. mode of a specific data handle with the
  3636. @code{starpu_data_set_sequential_consistency_flag} function.
  3637. @end deftypefun
  3638. @node starpu_data_get_default_sequential_consistency_flag
  3639. @subsection @code{starpu_data_get_default_sequential_consistency_flag} -- Get current default sequential consistency flag
  3640. @deftypefun unsigned starpu_data_set_default_sequential_consistency_flag (void)
  3641. This function returns the current default sequential consistency flag.
  3642. @end deftypefun
  3643. @node starpu_data_set_sequential_consistency_flag
  3644. @subsection @code{starpu_data_set_sequential_consistency_flag} -- Set data sequential consistency mode
  3645. @deftypefun void starpu_data_set_sequential_consistency_flag (starpu_data_handle @var{handle}, unsigned @var{flag})
  3646. Select the data consistency mode associated to a data handle. The consistency
  3647. mode set using this function has the priority over the default mode which can
  3648. be set with @code{starpu_data_set_sequential_consistency_flag}.
  3649. @end deftypefun
  3650. @node Performance Model API
  3651. @section Performance Model API
  3652. @menu
  3653. * starpu_load_history_debug::
  3654. * starpu_perfmodel_debugfilepath::
  3655. * starpu_perfmodel_get_arch_name::
  3656. * starpu_force_bus_sampling::
  3657. @end menu
  3658. @node starpu_load_history_debug
  3659. @subsection @code{starpu_load_history_debug}
  3660. @deftypefun int starpu_load_history_debug ({const char} *@var{symbol}, {struct starpu_perfmodel_t} *@var{model})
  3661. TODO
  3662. @end deftypefun
  3663. @node starpu_perfmodel_debugfilepath
  3664. @subsection @code{starpu_perfmodel_debugfilepath}
  3665. @deftypefun void starpu_perfmodel_debugfilepath ({struct starpu_perfmodel_t} *@var{model}, {enum starpu_perf_archtype} @var{arch}, char *@var{path}, size_t @var{maxlen})
  3666. TODO
  3667. @end deftypefun
  3668. @node starpu_perfmodel_get_arch_name
  3669. @subsection @code{starpu_perfmodel_get_arch_name}
  3670. @deftypefun void starpu_perfmodel_get_arch_name ({enum starpu_perf_archtype} @var{arch}, char *@var{archname}, size_t @var{maxlen})
  3671. TODO
  3672. @end deftypefun
  3673. @node starpu_force_bus_sampling
  3674. @subsection @code{starpu_force_bus_sampling}
  3675. @deftypefun void starpu_force_bus_sampling (void)
  3676. This forces sampling the bus performance model again.
  3677. @end deftypefun
  3678. @node Profiling API
  3679. @section Profiling API
  3680. @menu
  3681. * starpu_profiling_status_set:: starpu_profiling_status_set
  3682. * starpu_profiling_status_get:: starpu_profiling_status_get
  3683. * struct starpu_task_profiling_info:: task profiling information
  3684. * struct starpu_worker_profiling_info:: worker profiling information
  3685. * starpu_worker_get_profiling_info:: starpu_worker_get_profiling_info
  3686. * struct starpu_bus_profiling_info:: bus profiling information
  3687. * starpu_bus_get_count::
  3688. * starpu_bus_get_id::
  3689. * starpu_bus_get_src::
  3690. * starpu_bus_get_dst::
  3691. * starpu_timing_timespec_delay_us::
  3692. * starpu_timing_timespec_to_us::
  3693. * starpu_bus_profiling_helper_display_summary::
  3694. * starpu_worker_profiling_helper_display_summary::
  3695. @end menu
  3696. @node starpu_profiling_status_set
  3697. @subsection @code{starpu_profiling_status_set} -- Set current profiling status
  3698. @table @asis
  3699. @item @emph{Description}:
  3700. Thie function sets the profiling status. Profiling is activated by passing
  3701. @code{STARPU_PROFILING_ENABLE} in @code{status}. Passing
  3702. @code{STARPU_PROFILING_DISABLE} disables profiling. Calling this function
  3703. resets all profiling measurements. When profiling is enabled, the
  3704. @code{profiling_info} field of the @code{struct starpu_task} structure points
  3705. to a valid @code{struct starpu_task_profiling_info} structure containing
  3706. information about the execution of the task.
  3707. @item @emph{Return value}:
  3708. Negative return values indicate an error, otherwise the previous status is
  3709. returned.
  3710. @item @emph{Prototype}:
  3711. @code{int starpu_profiling_status_set(int status);}
  3712. @end table
  3713. @node starpu_profiling_status_get
  3714. @subsection @code{starpu_profiling_status_get} -- Get current profiling status
  3715. @deftypefun int starpu_profiling_status_get (void)
  3716. Return the current profiling status or a negative value in case there was an error.
  3717. @end deftypefun
  3718. @node struct starpu_task_profiling_info
  3719. @subsection @code{struct starpu_task_profiling_info} -- Task profiling information
  3720. @table @asis
  3721. @item @emph{Description}:
  3722. This structure contains information about the execution of a task. It is
  3723. accessible from the @code{.profiling_info} field of the @code{starpu_task}
  3724. structure if profiling was enabled.
  3725. @item @emph{Fields}:
  3726. @table @asis
  3727. @item @code{submit_time}:
  3728. Date of task submission (relative to the initialization of StarPU).
  3729. @item @code{start_time}:
  3730. Date of task execution beginning (relative to the initialization of StarPU).
  3731. @item @code{end_time}:
  3732. Date of task execution termination (relative to the initialization of StarPU).
  3733. @item @code{workerid}:
  3734. Identifier of the worker which has executed the task.
  3735. @end table
  3736. @end table
  3737. @node struct starpu_worker_profiling_info
  3738. @subsection @code{struct starpu_worker_profiling_info} -- Worker profiling information
  3739. @table @asis
  3740. @item @emph{Description}:
  3741. This structure contains the profiling information associated to a worker.
  3742. @item @emph{Fields}:
  3743. @table @asis
  3744. @item @code{start_time}:
  3745. Starting date for the reported profiling measurements.
  3746. @item @code{total_time}:
  3747. Duration of the profiling measurement interval.
  3748. @item @code{executing_time}:
  3749. Time spent by the worker to execute tasks during the profiling measurement interval.
  3750. @item @code{sleeping_time}:
  3751. Time spent idling by the worker during the profiling measurement interval.
  3752. @item @code{executed_tasks}:
  3753. Number of tasks executed by the worker during the profiling measurement interval.
  3754. @end table
  3755. @end table
  3756. @node starpu_worker_get_profiling_info
  3757. @subsection @code{starpu_worker_get_profiling_info} -- Get worker profiling info
  3758. @table @asis
  3759. @item @emph{Description}:
  3760. Get the profiling info associated to the worker identified by @code{workerid},
  3761. and reset the profiling measurements. If the @code{worker_info} argument is
  3762. NULL, only reset the counters associated to worker @code{workerid}.
  3763. @item @emph{Return value}:
  3764. Upon successful completion, this function returns 0. Otherwise, a negative
  3765. value is returned.
  3766. @item @emph{Prototype}:
  3767. @code{int starpu_worker_get_profiling_info(int workerid, struct starpu_worker_profiling_info *worker_info);}
  3768. @end table
  3769. @node struct starpu_bus_profiling_info
  3770. @subsection @code{struct starpu_bus_profiling_info} -- Bus profiling information
  3771. @table @asis
  3772. @item @emph{Description}:
  3773. TODO
  3774. @item @emph{Fields}:
  3775. @table @asis
  3776. @item @code{start_time}:
  3777. TODO
  3778. @item @code{total_time}:
  3779. TODO
  3780. @item @code{transferred_bytes}:
  3781. TODO
  3782. @item @code{transfer_count}:
  3783. TODO
  3784. @end table
  3785. @end table
  3786. @node starpu_bus_get_count
  3787. @subsection @code{starpu_bus_get_count}
  3788. @deftypefun int starpu_bus_get_count (void)
  3789. TODO
  3790. @end deftypefun
  3791. @node starpu_bus_get_id
  3792. @subsection @code{starpu_bus_get_id}
  3793. @deftypefun int starpu_bus_get_id (int @var{src}, int @var{dst})
  3794. TODO
  3795. @end deftypefun
  3796. @node starpu_bus_get_src
  3797. @subsection @code{starpu_bus_get_src}
  3798. @deftypefun int starpu_bus_get_src (int @var{busid})
  3799. TODO
  3800. @end deftypefun
  3801. @node starpu_bus_get_dst
  3802. @subsection @code{starpu_bus_get_dst}
  3803. @deftypefun int starpu_bus_get_dst (int @var{busid})
  3804. TODO
  3805. @end deftypefun
  3806. @node starpu_timing_timespec_delay_us
  3807. @subsection @code{starpu_timing_timespec_delay_us}
  3808. @deftypefun double starpu_timing_timespec_delay_us ({struct timespec} *@var{start}, {struct timespec} *@var{end})
  3809. TODO
  3810. @end deftypefun
  3811. @node starpu_timing_timespec_to_us
  3812. @subsection @code{starpu_timing_timespec_to_us}
  3813. @deftypefun double starpu_timing_timespec_to_us ({struct timespec} *@var{ts})
  3814. TODO
  3815. @end deftypefun
  3816. @node starpu_bus_profiling_helper_display_summary
  3817. @subsection @code{starpu_bus_profiling_helper_display_summary}
  3818. @deftypefun void starpu_bus_profiling_helper_display_summary (void)
  3819. TODO
  3820. @end deftypefun
  3821. @node starpu_worker_profiling_helper_display_summary
  3822. @subsection @code{starpu_worker_profiling_helper_display_summary}
  3823. @deftypefun void starpu_worker_profiling_helper_display_summary (void)
  3824. TODO
  3825. @end deftypefun
  3826. @node CUDA extensions
  3827. @section CUDA extensions
  3828. @c void starpu_malloc(float **A, size_t dim);
  3829. @menu
  3830. * starpu_cuda_get_local_stream:: Get current worker's CUDA stream
  3831. * starpu_helper_cublas_init:: Initialize CUBLAS on every CUDA device
  3832. * starpu_helper_cublas_shutdown:: Deinitialize CUBLAS on every CUDA device
  3833. @end menu
  3834. @node starpu_cuda_get_local_stream
  3835. @subsection @code{starpu_cuda_get_local_stream} -- Get current worker's CUDA stream
  3836. @deftypefun {cudaStream_t *} starpu_cuda_get_local_stream (void)
  3837. StarPU provides a stream for every CUDA device controlled by StarPU. This
  3838. function is only provided for convenience so that programmers can easily use
  3839. asynchronous operations within codelets without having to create a stream by
  3840. hand. Note that the application is not forced to use the stream provided by
  3841. @code{starpu_cuda_get_local_stream} and may also create its own streams.
  3842. Synchronizing with @code{cudaThreadSynchronize()} is allowed, but will reduce
  3843. the likelihood of having all transfers overlapped.
  3844. @end deftypefun
  3845. @node starpu_helper_cublas_init
  3846. @subsection @code{starpu_helper_cublas_init} -- Initialize CUBLAS on every CUDA device
  3847. @deftypefun void starpu_helper_cublas_init (void)
  3848. The CUBLAS library must be initialized prior to any CUBLAS call. Calling
  3849. @code{starpu_helper_cublas_init} will initialize CUBLAS on every CUDA device
  3850. controlled by StarPU. This call blocks until CUBLAS has been properly
  3851. initialized on every device.
  3852. @end deftypefun
  3853. @node starpu_helper_cublas_shutdown
  3854. @subsection @code{starpu_helper_cublas_shutdown} -- Deinitialize CUBLAS on every CUDA device
  3855. @deftypefun void starpu_helper_cublas_shutdown (void)
  3856. This function synchronously deinitializes the CUBLAS library on every CUDA device.
  3857. @end deftypefun
  3858. @node OpenCL extensions
  3859. @section OpenCL extensions
  3860. @menu
  3861. * Compiling OpenCL kernels:: Compiling OpenCL kernels
  3862. * Loading OpenCL kernels:: Loading OpenCL kernels
  3863. * OpenCL statistics:: Collecting statistics from OpenCL
  3864. @end menu
  3865. @node Compiling OpenCL kernels
  3866. @subsection Compiling OpenCL kernels
  3867. Source codes for OpenCL kernels can be stored in a file or in a
  3868. string. StarPU provides functions to build the program executable for
  3869. each available OpenCL device as a @code{cl_program} object. This
  3870. program executable can then be loaded within a specific queue as
  3871. explained in the next section. These are only helpers, Applications
  3872. can also fill a @code{starpu_opencl_program} array by hand for more advanced
  3873. use (e.g. different programs on the different OpenCL devices, for
  3874. relocation purpose for instance).
  3875. @menu
  3876. * starpu_opencl_load_opencl_from_file:: Compiling OpenCL source code
  3877. * starpu_opencl_load_opencl_from_string:: Compiling OpenCL source code
  3878. * starpu_opencl_unload_opencl:: Releasing OpenCL code
  3879. @end menu
  3880. @node starpu_opencl_load_opencl_from_file
  3881. @subsubsection @code{starpu_opencl_load_opencl_from_file} -- Compiling OpenCL source code
  3882. @deftypefun int starpu_opencl_load_opencl_from_file (char *@var{source_file_name}, {struct starpu_opencl_program} *@var{opencl_programs}, {const char}* @var{build_options})
  3883. TODO
  3884. @end deftypefun
  3885. @node starpu_opencl_load_opencl_from_string
  3886. @subsubsection @code{starpu_opencl_load_opencl_from_string} -- Compiling OpenCL source code
  3887. @deftypefun int starpu_opencl_load_opencl_from_string (char *@var{opencl_program_source}, {struct starpu_opencl_program} *@var{opencl_programs}, {const char}* @var{build_options})
  3888. TODO
  3889. @end deftypefun
  3890. @node starpu_opencl_unload_opencl
  3891. @subsubsection @code{starpu_opencl_unload_opencl} -- Releasing OpenCL code
  3892. @deftypefun int starpu_opencl_unload_opencl ({struct starpu_opencl_program} *@var{opencl_programs})
  3893. TODO
  3894. @end deftypefun
  3895. @node Loading OpenCL kernels
  3896. @subsection Loading OpenCL kernels
  3897. @menu
  3898. * starpu_opencl_load_kernel:: Loading a kernel
  3899. * starpu_opencl_relase_kernel:: Releasing a kernel
  3900. @end menu
  3901. @node starpu_opencl_load_kernel
  3902. @subsubsection @code{starpu_opencl_load_kernel} -- Loading a kernel
  3903. @deftypefun int starpu_opencl_load_kernel (cl_kernel *@var{kernel}, cl_command_queue *@var{queue}, {struct starpu_opencl_program} *@var{opencl_programs}, char *@var{kernel_name}, int @var{devid})
  3904. TODO
  3905. @end deftypefun
  3906. @node starpu_opencl_relase_kernel
  3907. @subsubsection @code{starpu_opencl_release_kernel} -- Releasing a kernel
  3908. @deftypefun int starpu_opencl_release_kernel (cl_kernel @var{kernel})
  3909. TODO
  3910. @end deftypefun
  3911. @node OpenCL statistics
  3912. @subsection OpenCL statistics
  3913. @menu
  3914. * starpu_opencl_collect_stats:: Collect statistics on a kernel execution
  3915. @end menu
  3916. @node starpu_opencl_collect_stats
  3917. @subsubsection @code{starpu_opencl_collect_stats} -- Collect statistics on a kernel execution
  3918. @deftypefun int starpu_opencl_collect_stats (cl_event @var{event})
  3919. After termination of the kernels, the OpenCL codelet should call this function
  3920. to pass it the even returned by @code{clEnqueueNDRangeKernel}, to let StarPU
  3921. collect statistics about the kernel execution (used cycles, consumed power).
  3922. @end deftypefun
  3923. @node Cell extensions
  3924. @section Cell extensions
  3925. nothing yet.
  3926. @node Miscellaneous helpers
  3927. @section Miscellaneous helpers
  3928. @menu
  3929. * starpu_data_cpy:: Copy a data handle into another data handle
  3930. * starpu_execute_on_each_worker:: Execute a function on a subset of workers
  3931. @end menu
  3932. @node starpu_data_cpy
  3933. @subsection @code{starpu_data_cpy} -- Copy a data handle into another data handle
  3934. @deftypefun int starpu_data_cpy (starpu_data_handle @var{dst_handle}, starpu_data_handle @var{src_handle}, int @var{asynchronous}, void (*@var{callback_func})(void*), void *@var{callback_arg})
  3935. Copy the content of the @var{src_handle} into the @var{dst_handle} handle.
  3936. The @var{asynchronous} parameter indicates whether the function should
  3937. block or not. In the case of an asynchronous call, it is possible to
  3938. synchronize with the termination of this operation either by the means of
  3939. implicit dependencies (if enabled) or by calling
  3940. @code{starpu_task_wait_for_all()}. If @var{callback_func} is not @code{NULL},
  3941. this callback function is executed after the handle has been copied, and it is
  3942. given the @var{callback_arg} pointer as argument.
  3943. @end deftypefun
  3944. @node starpu_execute_on_each_worker
  3945. @subsection @code{starpu_execute_on_each_worker} -- Execute a function on a subset of workers
  3946. @deftypefun void starpu_execute_on_each_worker (void (*@var{func})(void *), void *@var{arg}, uint32_t @var{where})
  3947. When calling this method, the offloaded function specified by the first argument is
  3948. executed by every StarPU worker that may execute the function.
  3949. The second argument is passed to the offloaded function.
  3950. The last argument specifies on which types of processing units the function
  3951. should be executed. Similarly to the @var{where} field of the
  3952. @code{starpu_codelet} structure, it is possible to specify that the function
  3953. should be executed on every CUDA device and every CPU by passing
  3954. @code{STARPU_CPU|STARPU_CUDA}.
  3955. This function blocks until the function has been executed on every appropriate
  3956. processing units, so that it may not be called from a callback function for
  3957. instance.
  3958. @end deftypefun
  3959. @c ---------------------------------------------------------------------
  3960. @c Advanced Topics
  3961. @c ---------------------------------------------------------------------
  3962. @node Advanced Topics
  3963. @chapter Advanced Topics
  3964. @menu
  3965. * Defining a new data interface::
  3966. * Defining a new scheduling policy::
  3967. @end menu
  3968. @node Defining a new data interface
  3969. @section Defining a new data interface
  3970. @menu
  3971. * struct starpu_data_interface_ops_t:: Per-interface methods
  3972. * struct starpu_data_copy_methods:: Per-interface data transfer methods
  3973. * An example of data interface:: An example of data interface
  3974. @end menu
  3975. @c void *starpu_data_get_interface_on_node(starpu_data_handle handle, unsigned memory_node); TODO
  3976. @node struct starpu_data_interface_ops_t
  3977. @subsection @code{struct starpu_data_interface_ops_t} -- Per-interface methods
  3978. @table @asis
  3979. @item @emph{Description}:
  3980. TODO describe all the different fields
  3981. @end table
  3982. @node struct starpu_data_copy_methods
  3983. @subsection @code{struct starpu_data_copy_methods} -- Per-interface data transfer methods
  3984. @table @asis
  3985. @item @emph{Description}:
  3986. TODO describe all the different fields
  3987. @end table
  3988. @node An example of data interface
  3989. @subsection An example of data interface
  3990. @table @asis
  3991. TODO
  3992. See @code{src/datawizard/interfaces/vector_interface.c} for now.
  3993. @end table
  3994. @node Defining a new scheduling policy
  3995. @section Defining a new scheduling policy
  3996. TODO
  3997. A full example showing how to define a new scheduling policy is available in
  3998. the StarPU sources in the directory @code{examples/scheduler/}.
  3999. @menu
  4000. * struct starpu_sched_policy_s::
  4001. * starpu_worker_set_sched_condition::
  4002. * starpu_sched_set_min_priority:: Set the minimum priority level
  4003. * starpu_sched_set_max_priority:: Set the maximum priority level
  4004. * starpu_push_local_task:: Assign a task to a worker
  4005. * Source code::
  4006. @end menu
  4007. @node struct starpu_sched_policy_s
  4008. @subsection @code{struct starpu_sched_policy_s} -- Scheduler methods
  4009. @table @asis
  4010. @item @emph{Description}:
  4011. This structure contains all the methods that implement a scheduling policy. An
  4012. application may specify which scheduling strategy in the @code{sched_policy}
  4013. field of the @code{starpu_conf} structure passed to the @code{starpu_init}
  4014. function.
  4015. @item @emph{Fields}:
  4016. @table @asis
  4017. @item @code{init_sched}:
  4018. Initialize the scheduling policy.
  4019. @item @code{deinit_sched}:
  4020. Cleanup the scheduling policy.
  4021. @item @code{push_task}:
  4022. Insert a task into the scheduler.
  4023. @item @code{push_prio_task}:
  4024. Insert a priority task into the scheduler.
  4025. @item @code{push_prio_notify}:
  4026. Notify the scheduler that a task was pushed on the worker. This method is
  4027. called when a task that was explicitely assigned to a worker is scheduled. This
  4028. method therefore permits to keep the state of of the scheduler coherent even
  4029. when StarPU bypasses the scheduling strategy.
  4030. @item @code{pop_task}:
  4031. Get a task from the scheduler. The mutex associated to the worker is already
  4032. taken when this method is called. If this method is defined as @code{NULL}, the
  4033. worker will only execute tasks from its local queue. In this case, the
  4034. @code{push_task} method should use the @code{starpu_push_local_task} method to
  4035. assign tasks to the different workers.
  4036. @item @code{pop_every_task}:
  4037. Remove all available tasks from the scheduler (tasks are chained by the means
  4038. of the prev and next fields of the starpu_task structure). The mutex associated
  4039. to the worker is already taken when this method is called.
  4040. @item @code{post_exec_hook} (optionnal):
  4041. This method is called every time a task has been executed.
  4042. @item @code{policy_name}:
  4043. Name of the policy (optionnal).
  4044. @item @code{policy_description}:
  4045. Description of the policy (optionnal).
  4046. @end table
  4047. @end table
  4048. @node starpu_worker_set_sched_condition
  4049. @subsection @code{starpu_worker_set_sched_condition} -- Specify the condition variable associated to a worker
  4050. @deftypefun void starpu_worker_set_sched_condition (int @var{workerid}, pthread_cond_t *@var{sched_cond}, pthread_mutex_t *@var{sched_mutex})
  4051. When there is no available task for a worker, StarPU blocks this worker on a
  4052. condition variable. This function specifies which condition variable (and the
  4053. associated mutex) should be used to block (and to wake up) a worker. Note that
  4054. multiple workers may use the same condition variable. For instance, in the case
  4055. of a scheduling strategy with a single task queue, the same condition variable
  4056. would be used to block and wake up all workers.
  4057. The initialization method of a scheduling strategy (@code{init_sched}) must
  4058. call this function once per worker.
  4059. @end deftypefun
  4060. @node starpu_sched_set_min_priority
  4061. @subsection @code{starpu_sched_set_min_priority}
  4062. @deftypefun void starpu_sched_set_min_priority (int @var{min_prio})
  4063. Defines the minimum priority level supported by the scheduling policy. The
  4064. default minimum priority level is the same as the default priority level which
  4065. is 0 by convention. The application may access that value by calling the
  4066. @code{starpu_sched_get_min_priority} function. This function should only be
  4067. called from the initialization method of the scheduling policy, and should not
  4068. be used directly from the application.
  4069. @end deftypefun
  4070. @node starpu_sched_set_max_priority
  4071. @subsection @code{starpu_sched_set_max_priority}
  4072. @deftypefun void starpu_sched_set_min_priority (int @var{max_prio})
  4073. Defines the maximum priority level supported by the scheduling policy. The
  4074. default maximum priority level is 1. The application may access that value by
  4075. calling the @code{starpu_sched_get_max_priority} function. This function should
  4076. only be called from the initialization method of the scheduling policy, and
  4077. should not be used directly from the application.
  4078. @end deftypefun
  4079. @node starpu_push_local_task
  4080. @subsection @code{starpu_push_local_task}
  4081. @deftypefun int starpu_push_local_task (int @var{workerid}, {struct starpu_task} *@var{task}, int @var{back})
  4082. The scheduling policy may put tasks directly into a worker's local queue so
  4083. that it is not always necessary to create its own queue when the local queue
  4084. is sufficient. If "back" not null, the task is put at the back of the queue
  4085. where the worker will pop tasks first. Setting "back" to 0 therefore ensures
  4086. a FIFO ordering.
  4087. @end deftypefun
  4088. @node Source code
  4089. @subsection Source code
  4090. @cartouche
  4091. @smallexample
  4092. static struct starpu_sched_policy_s dummy_sched_policy = @{
  4093. .init_sched = init_dummy_sched,
  4094. .deinit_sched = deinit_dummy_sched,
  4095. .push_task = push_task_dummy,
  4096. .push_prio_task = NULL,
  4097. .pop_task = pop_task_dummy,
  4098. .post_exec_hook = NULL,
  4099. .pop_every_task = NULL,
  4100. .policy_name = "dummy",
  4101. .policy_description = "dummy scheduling strategy"
  4102. @};
  4103. @end smallexample
  4104. @end cartouche
  4105. @c ---------------------------------------------------------------------
  4106. @c C Extensions
  4107. @c ---------------------------------------------------------------------
  4108. @include c-extensions.texi
  4109. @c ---------------------------------------------------------------------
  4110. @c Appendices
  4111. @c ---------------------------------------------------------------------
  4112. @c ---------------------------------------------------------------------
  4113. @c Full source code for the 'Scaling a Vector' example
  4114. @c ---------------------------------------------------------------------
  4115. @node Full source code for the 'Scaling a Vector' example
  4116. @appendix Full source code for the 'Scaling a Vector' example
  4117. @menu
  4118. * Main application::
  4119. * CPU Kernel::
  4120. * CUDA Kernel::
  4121. * OpenCL Kernel::
  4122. @end menu
  4123. @node Main application
  4124. @section Main application
  4125. @smallexample
  4126. @include vector_scal_c.texi
  4127. @end smallexample
  4128. @node CPU Kernel
  4129. @section CPU Kernel
  4130. @smallexample
  4131. @include vector_scal_cpu.texi
  4132. @end smallexample
  4133. @node CUDA Kernel
  4134. @section CUDA Kernel
  4135. @smallexample
  4136. @include vector_scal_cuda.texi
  4137. @end smallexample
  4138. @node OpenCL Kernel
  4139. @section OpenCL Kernel
  4140. @menu
  4141. * Invoking the kernel::
  4142. * Source of the kernel::
  4143. @end menu
  4144. @node Invoking the kernel
  4145. @subsection Invoking the kernel
  4146. @smallexample
  4147. @include vector_scal_opencl.texi
  4148. @end smallexample
  4149. @node Source of the kernel
  4150. @subsection Source of the kernel
  4151. @smallexample
  4152. @include vector_scal_opencl_codelet.texi
  4153. @end smallexample
  4154. @c
  4155. @c Indices.
  4156. @c
  4157. @node Function Index
  4158. @unnumbered Function Index
  4159. @printindex fn
  4160. @bye