openmp_runtime_support.c 76 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430
  1. /* StarPU --- Runtime system for heterogeneous multicore architectures.
  2. *
  3. * Copyright (C) 2014 INRIA
  4. *
  5. * StarPU is free software; you can redistribute it and/or modify
  6. * it under the terms of the GNU Lesser General Public License as published by
  7. * the Free Software Foundation; either version 2.1 of the License, or (at
  8. * your option) any later version.
  9. *
  10. * StarPU is distributed in the hope that it will be useful, but
  11. * WITHOUT ANY WARRANTY; without even the implied warranty of
  12. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
  13. *
  14. * See the GNU Lesser General Public License in COPYING.LGPL for more details.
  15. */
  16. #include <starpu.h>
  17. #ifdef STARPU_OPENMP
  18. /*
  19. * locally disable -Wdeprecated-declarations to avoid
  20. * lots of deprecated warnings for ucontext related functions
  21. */
  22. #pragma GCC diagnostic push
  23. #pragma GCC diagnostic ignored "-Wdeprecated-declarations"
  24. #include <util/openmp_runtime_support.h>
  25. #include <core/task.h>
  26. #include <core/workers.h>
  27. #include <common/list.h>
  28. #include <common/starpu_spinlock.h>
  29. #include <common/uthash.h>
  30. #include <datawizard/interfaces/data_interface.h>
  31. #include <stdlib.h>
  32. #include <ctype.h>
  33. #include <strings.h>
  34. #define _STARPU_INITIAL_THREAD_STACKSIZE 2097152
  35. static struct starpu_omp_global _global_state;
  36. static starpu_pthread_key_t omp_thread_key;
  37. static starpu_pthread_key_t omp_task_key;
  38. struct starpu_omp_global *_starpu_omp_global_state = NULL;
  39. double _starpu_omp_clock_ref = 0.0; /* clock reference for starpu_omp_get_wtick */
  40. static struct starpu_omp_critical *create_omp_critical_struct(void);
  41. static void destroy_omp_critical_struct(struct starpu_omp_critical *critical);
  42. static struct starpu_omp_device *create_omp_device_struct(void);
  43. static void destroy_omp_device_struct(struct starpu_omp_device *device);
  44. static struct starpu_omp_region *create_omp_region_struct(struct starpu_omp_region *parent_region, struct starpu_omp_device *owner_device);
  45. static void destroy_omp_region_struct(struct starpu_omp_region *region);
  46. static struct starpu_omp_thread *create_omp_thread_struct(struct starpu_omp_region *owner_region);
  47. static void destroy_omp_thread_struct(struct starpu_omp_thread *thread);
  48. static struct starpu_omp_task *create_omp_task_struct(struct starpu_omp_task *parent_task,
  49. struct starpu_omp_thread *owner_thread, struct starpu_omp_region *owner_region, int is_implicit);
  50. static void destroy_omp_task_struct(struct starpu_omp_task *task);
  51. static void wake_up_and_unlock_task(struct starpu_omp_task *task);
  52. static void wake_up_barrier(struct starpu_omp_region *parallel_region);
  53. static void starpu_omp_task_preempt(void);
  54. struct starpu_omp_thread *_starpu_omp_get_thread(void)
  55. {
  56. struct starpu_omp_thread *thread = STARPU_PTHREAD_GETSPECIFIC(omp_thread_key);
  57. return thread;
  58. }
  59. static inline void _starpu_omp_set_thread(struct starpu_omp_thread *thread)
  60. {
  61. STARPU_PTHREAD_SETSPECIFIC(omp_thread_key, thread);
  62. }
  63. struct starpu_omp_task *_starpu_omp_get_task(void)
  64. {
  65. struct starpu_omp_task *task = STARPU_PTHREAD_GETSPECIFIC(omp_task_key);
  66. return task;
  67. }
  68. static inline void _starpu_omp_set_task(struct starpu_omp_task *task)
  69. {
  70. STARPU_PTHREAD_SETSPECIFIC(omp_task_key, task);
  71. }
  72. struct starpu_omp_region *_starpu_omp_get_region_at_level(int level)
  73. {
  74. const struct starpu_omp_task *task = _starpu_omp_get_task();
  75. struct starpu_omp_region *parallel_region;
  76. if (!task)
  77. return NULL;
  78. parallel_region = task->owner_region;
  79. if (level < 0 || level > parallel_region->icvs.levels_var)
  80. return NULL;
  81. while (level < parallel_region->icvs.levels_var)
  82. {
  83. parallel_region = parallel_region->parent_region;
  84. }
  85. return parallel_region;
  86. }
  87. int _starpu_omp_get_region_thread_num(const struct starpu_omp_region * const region)
  88. {
  89. struct starpu_omp_thread *thread = _starpu_omp_get_thread();
  90. STARPU_ASSERT(thread != NULL);
  91. if (thread == region->master_thread)
  92. return 0;
  93. int tid = starpu_omp_thread_list_member(&region->thread_list, thread);
  94. if (tid >= 0)
  95. return tid+1;
  96. _STARPU_ERROR("unrecognized omp thread\n");
  97. }
  98. static void weak_task_lock(struct starpu_omp_task *task)
  99. {
  100. _starpu_spin_lock(&task->lock);
  101. while (task->transaction_pending)
  102. {
  103. _starpu_spin_unlock(&task->lock);
  104. STARPU_UYIELD();
  105. _starpu_spin_lock(&task->lock);
  106. }
  107. }
  108. static void weak_task_unlock(struct starpu_omp_task *task)
  109. {
  110. _starpu_spin_unlock(&task->lock);
  111. }
  112. static void wake_up_and_unlock_task(struct starpu_omp_task *task)
  113. {
  114. STARPU_ASSERT(task->transaction_pending == 0);
  115. if (task->wait_on == 0)
  116. {
  117. weak_task_unlock(task);
  118. int ret = starpu_task_submit(task->starpu_task);
  119. STARPU_CHECK_RETURN_VALUE(ret, "starpu_task_submit");
  120. } else {
  121. weak_task_unlock(task);
  122. }
  123. }
  124. static void transaction_callback(void *_task)
  125. {
  126. struct starpu_omp_task *task = _task;
  127. _starpu_spin_lock(&task->lock);
  128. STARPU_ASSERT(task->transaction_pending != 0);
  129. task->transaction_pending = 0;
  130. _starpu_spin_unlock(&task->lock);
  131. }
  132. static void condition_init(struct starpu_omp_condition *condition)
  133. {
  134. condition->contention_list_head = NULL;
  135. }
  136. static void condition_exit(struct starpu_omp_condition *condition)
  137. {
  138. STARPU_ASSERT(condition->contention_list_head == NULL);
  139. condition->contention_list_head = NULL;
  140. }
  141. static void condition_wait(struct starpu_omp_condition *condition, struct _starpu_spinlock *lock, enum starpu_omp_task_wait_on flag)
  142. {
  143. struct starpu_omp_task *task = _starpu_omp_get_task();
  144. struct starpu_omp_task_link link;
  145. _starpu_spin_lock(&task->lock);
  146. task->wait_on |= flag;
  147. link.task = task;
  148. link.next = condition->contention_list_head;
  149. condition->contention_list_head = &link;
  150. task->transaction_pending = 1;
  151. _starpu_spin_unlock(&task->lock);
  152. _starpu_spin_unlock(lock);
  153. _starpu_task_prepare_for_continuation_ext(0, transaction_callback, task);
  154. starpu_omp_task_preempt();
  155. /* re-acquire the lock released by the callback */
  156. _starpu_spin_lock(lock);
  157. }
  158. #if 0
  159. /* unused for now */
  160. static void condition_signal(struct starpu_omp_condition *condition)
  161. {
  162. if (condition->contention_list_head != NULL)
  163. {
  164. struct starpu_omp_task *next_task = condition->contention_list_head->task;
  165. weak_task_lock(next_task);
  166. condition->contention_list_head = condition->contention_list_head->next;
  167. STARPU_ASSERT(next_task->wait_on & starpu_omp_task_wait_on_condition);
  168. next_task->wait_on &= ~starpu_omp_task_wait_on_condition;
  169. wake_up_and_unlock_task(next_task);
  170. }
  171. }
  172. #endif
  173. static void condition_broadcast(struct starpu_omp_condition *condition, enum starpu_omp_task_wait_on flag)
  174. {
  175. while (condition->contention_list_head != NULL)
  176. {
  177. struct starpu_omp_task *next_task = condition->contention_list_head->task;
  178. weak_task_lock(next_task);
  179. condition->contention_list_head = condition->contention_list_head->next;
  180. STARPU_ASSERT(next_task->wait_on & flag);
  181. next_task->wait_on &= ~flag;
  182. wake_up_and_unlock_task(next_task);
  183. }
  184. }
  185. static void register_thread_worker(struct starpu_omp_thread *thread)
  186. {
  187. STARPU_ASSERT(thread->worker != NULL);
  188. _starpu_spin_lock(&_global_state.hash_workers_lock);
  189. struct _starpu_worker *check = thread->worker;
  190. struct starpu_omp_thread *tmp = NULL;
  191. HASH_FIND_PTR(_global_state.hash_workers, &check, tmp);
  192. STARPU_ASSERT(tmp == NULL);
  193. HASH_ADD_PTR(_global_state.hash_workers, worker, thread);
  194. _starpu_spin_unlock(&_global_state.hash_workers_lock);
  195. }
  196. static struct starpu_omp_thread *get_worker_thread(struct _starpu_worker *starpu_worker)
  197. {
  198. struct starpu_omp_thread *thread = NULL;
  199. _starpu_spin_lock(&_global_state.hash_workers_lock);
  200. HASH_FIND_PTR(_global_state.hash_workers, &starpu_worker, thread);
  201. _starpu_spin_unlock(&_global_state.hash_workers_lock);
  202. return thread;
  203. }
  204. static struct starpu_omp_thread *get_local_thread(void)
  205. {
  206. struct starpu_omp_thread *thread = _starpu_omp_get_thread();
  207. if (thread == NULL)
  208. {
  209. struct _starpu_worker *starpu_worker = _starpu_get_local_worker_key();
  210. STARPU_ASSERT(starpu_worker != NULL);
  211. thread = get_worker_thread(starpu_worker);
  212. if (
  213. #ifdef STARPU_USE_CUDA
  214. (starpu_worker->arch != STARPU_CUDA_WORKER)
  215. &&
  216. #endif
  217. #ifdef STARPU_USE_OPENCL
  218. (starpu_worker->arch != STARPU_OPENCL_WORKER)
  219. &&
  220. #endif
  221. 1
  222. )
  223. {
  224. STARPU_ASSERT(thread != NULL);
  225. }
  226. if (thread != NULL)
  227. {
  228. _starpu_omp_set_thread(thread);
  229. }
  230. }
  231. return thread;
  232. }
  233. static struct starpu_omp_critical *create_omp_critical_struct(void)
  234. {
  235. struct starpu_omp_critical *critical;
  236. _STARPU_CALLOC(critical, 1, sizeof(*critical));
  237. _starpu_spin_init(&critical->lock);
  238. return critical;
  239. }
  240. static void destroy_omp_critical_struct(struct starpu_omp_critical *critical)
  241. {
  242. STARPU_ASSERT(critical->state == 0);
  243. STARPU_ASSERT(critical->contention_list_head == NULL);
  244. _starpu_spin_destroy(&critical->lock);
  245. critical->name = NULL;
  246. free(critical);
  247. }
  248. static struct starpu_omp_device *create_omp_device_struct(void)
  249. {
  250. struct starpu_omp_device *device;
  251. _STARPU_CALLOC(device, 1, sizeof(*device));
  252. _starpu_spin_init(&device->atomic_lock);
  253. return device;
  254. }
  255. static void destroy_omp_device_struct(struct starpu_omp_device *device)
  256. {
  257. _starpu_spin_destroy(&device->atomic_lock);
  258. memset(device, 0, sizeof(*device));
  259. free(device);
  260. }
  261. static struct starpu_omp_device *get_caller_device(void)
  262. {
  263. struct starpu_omp_task *task = _starpu_omp_get_task();
  264. struct starpu_omp_device *device;
  265. if (task)
  266. {
  267. STARPU_ASSERT(task->owner_region != NULL);
  268. device = task->owner_region->owner_device;
  269. }
  270. else
  271. {
  272. device = _global_state.initial_device;
  273. }
  274. STARPU_ASSERT(device != NULL);
  275. return device;
  276. }
  277. static struct starpu_omp_region *create_omp_region_struct(struct starpu_omp_region *parent_region, struct starpu_omp_device *owner_device)
  278. {
  279. struct starpu_omp_region *region;
  280. _STARPU_CALLOC(region, 1, sizeof(*region));
  281. region->parent_region = parent_region;
  282. region->owner_device = owner_device;
  283. starpu_omp_thread_list_init(&region->thread_list);
  284. starpu_omp_task_list_init(&region->implicit_task_list);
  285. _starpu_spin_init(&region->lock);
  286. _starpu_spin_init(&region->registered_handles_lock);
  287. region->level = (parent_region != NULL)?parent_region->level+1:0;
  288. return region;
  289. }
  290. static void destroy_omp_region_struct(struct starpu_omp_region *region)
  291. {
  292. STARPU_ASSERT(region->nb_threads == 0);
  293. STARPU_ASSERT(starpu_omp_thread_list_empty(&region->thread_list));
  294. STARPU_ASSERT(starpu_omp_task_list_empty(&region->implicit_task_list));
  295. STARPU_ASSERT(region->continuation_starpu_task == NULL);
  296. _starpu_spin_destroy(&region->registered_handles_lock);
  297. _starpu_spin_destroy(&region->lock);
  298. memset(region, 0, sizeof(*region));
  299. free(region);
  300. }
  301. static void omp_initial_thread_func(void)
  302. {
  303. struct starpu_omp_thread *initial_thread = _global_state.initial_thread;
  304. struct starpu_omp_task *initial_task = _global_state.initial_task;
  305. while (1)
  306. {
  307. struct starpu_task *continuation_starpu_task = initial_task->nested_region->continuation_starpu_task;
  308. starpu_driver_run_once(&initial_thread->starpu_driver);
  309. /*
  310. * if we are leaving the first nested region we give control back to initial task
  311. * otherwise, we should continue to execute work
  312. */
  313. if (_starpu_task_test_termination(continuation_starpu_task))
  314. {
  315. initial_task->nested_region->continuation_starpu_task = NULL;
  316. _starpu_omp_set_task(initial_task);
  317. swapcontext(&initial_thread->ctx, &initial_task->ctx);
  318. }
  319. }
  320. }
  321. static struct starpu_omp_thread *create_omp_thread_struct(struct starpu_omp_region *owner_region)
  322. {
  323. struct starpu_omp_thread *thread = starpu_omp_thread_new();
  324. if (thread == NULL)
  325. _STARPU_ERROR("memory allocation failed");
  326. memset(thread, 0, sizeof(*thread));
  327. thread->owner_region = owner_region;
  328. return thread;
  329. }
  330. static void destroy_omp_thread_struct(struct starpu_omp_thread *thread)
  331. {
  332. STARPU_ASSERT(thread->current_task == NULL);
  333. memset(thread, 0, sizeof(*thread));
  334. starpu_omp_thread_delete(thread);
  335. }
  336. static void starpu_omp_explicit_task_entry(struct starpu_omp_task *task)
  337. {
  338. STARPU_ASSERT(!(task->flags & STARPU_OMP_TASK_FLAGS_IMPLICIT));
  339. struct _starpu_worker *starpu_worker = _starpu_get_local_worker_key();
  340. if (starpu_worker->arch == STARPU_CPU_WORKER)
  341. {
  342. task->cpu_f(task->starpu_buffers, task->starpu_cl_arg);
  343. }
  344. #ifdef STARPU_USE_CUDA
  345. else if (starpu_worker->arch == STARPU_CUDA_WORKER)
  346. {
  347. task->cuda_f(task->starpu_buffers, task->starpu_cl_arg);
  348. }
  349. #endif
  350. #ifdef STARPU_USE_OPENCL
  351. else if (starpu_worker->arch == STARPU_OPENCL_WORKER)
  352. {
  353. task->opencl_f(task->starpu_buffers, task->starpu_cl_arg);
  354. }
  355. #endif
  356. else
  357. _STARPU_ERROR("invalid worker architecture");
  358. _starpu_omp_unregister_task_handles(task);
  359. _starpu_spin_lock(&task->lock);
  360. task->state = starpu_omp_task_state_terminated;
  361. task->transaction_pending=1;
  362. _starpu_spin_unlock(&task->lock);
  363. struct starpu_omp_thread *thread = _starpu_omp_get_thread();
  364. /*
  365. * the task reached the terminated state, definitively give hand back to the worker code.
  366. *
  367. * about to run on the worker stack...
  368. */
  369. setcontext(&thread->ctx);
  370. STARPU_ASSERT(0); /* unreachable code */
  371. }
  372. static void starpu_omp_implicit_task_entry(struct starpu_omp_task *task)
  373. {
  374. struct starpu_omp_thread *thread = _starpu_omp_get_thread();
  375. STARPU_ASSERT(task->flags & STARPU_OMP_TASK_FLAGS_IMPLICIT);
  376. task->cpu_f(task->starpu_buffers, task->starpu_cl_arg);
  377. starpu_omp_barrier();
  378. if (thread == task->owner_region->master_thread)
  379. {
  380. _starpu_omp_unregister_region_handles(task->owner_region);
  381. }
  382. task->state = starpu_omp_task_state_terminated;
  383. /*
  384. * the task reached the terminated state, definitively give hand back to the worker code.
  385. *
  386. * about to run on the worker stack...
  387. */
  388. setcontext(&thread->ctx);
  389. STARPU_ASSERT(0); /* unreachable code */
  390. }
  391. /*
  392. * stop executing a task that is about to block
  393. * and give hand back to the thread
  394. */
  395. static void starpu_omp_task_preempt(void)
  396. {
  397. struct starpu_omp_task *task = _starpu_omp_get_task();
  398. struct starpu_omp_thread *thread = _starpu_omp_get_thread();
  399. task->state = starpu_omp_task_state_preempted;
  400. /*
  401. * the task reached a blocked state, give hand back to the worker code.
  402. *
  403. * about to run on the worker stack...
  404. */
  405. swapcontext(&task->ctx, &thread->ctx);
  406. /* now running on the task stack again */
  407. }
  408. /*
  409. * wrap a task function to allow the task to be preempted
  410. */
  411. static void starpu_omp_implicit_task_exec(void *buffers[], void *cl_arg)
  412. {
  413. struct starpu_omp_task *task = starpu_task_get_current()->omp_task;
  414. STARPU_ASSERT(task->flags & STARPU_OMP_TASK_FLAGS_IMPLICIT);
  415. _starpu_omp_set_task(task);
  416. struct starpu_omp_thread *thread = get_local_thread();
  417. if (task->state != starpu_omp_task_state_preempted)
  418. {
  419. task->starpu_buffers = buffers;
  420. task->starpu_cl_arg = cl_arg;
  421. STARPU_ASSERT(task->stack == NULL);
  422. STARPU_ASSERT(task->stacksize > 0);
  423. _STARPU_MALLOC(task->stack, task->stacksize);
  424. getcontext(&task->ctx);
  425. /*
  426. * we do not use uc_link, starpu_omp_task_entry will handle
  427. * the end of the task
  428. */
  429. task->ctx.uc_link = NULL;
  430. task->ctx.uc_stack.ss_sp = task->stack;
  431. task->ctx.uc_stack.ss_size = task->stacksize;
  432. task->stack_vg_id = VALGRIND_STACK_REGISTER(task->stack, task->stack+task->stacksize);
  433. makecontext(&task->ctx, (void (*) ()) starpu_omp_implicit_task_entry, 1, task);
  434. }
  435. task->state = starpu_omp_task_state_clear;
  436. /*
  437. * start the task execution, or restore a previously preempted task.
  438. * about to run on the task stack...
  439. * */
  440. swapcontext(&thread->ctx, &task->ctx);
  441. /* now running on the worker stack again */
  442. STARPU_ASSERT(task->state == starpu_omp_task_state_preempted
  443. || task->state == starpu_omp_task_state_terminated);
  444. _starpu_omp_set_task(NULL);
  445. /* TODO: analyse the cause of the return and take appropriate steps */
  446. if (task->state == starpu_omp_task_state_terminated)
  447. {
  448. task->starpu_task->omp_task = NULL;
  449. task->starpu_task = NULL;
  450. VALGRIND_STACK_DEREGISTER(task->stack_vg_id);
  451. task->stack_vg_id = 0;
  452. free(task->stack);
  453. task->stack = NULL;
  454. memset(&task->ctx, 0, sizeof(task->ctx));
  455. }
  456. else if (task->state != starpu_omp_task_state_preempted)
  457. _STARPU_ERROR("invalid omp task state");
  458. }
  459. static void starpu_omp_task_completion_accounting(struct starpu_omp_task *task)
  460. {
  461. struct starpu_omp_task *parent_task = task->parent_task;
  462. struct starpu_omp_region *parallel_region = task->owner_region;
  463. weak_task_lock(parent_task);
  464. if (STARPU_ATOMIC_ADD(&parent_task->child_task_count, -1) == 0)
  465. {
  466. if (parent_task->state == starpu_omp_task_state_zombie)
  467. {
  468. STARPU_ASSERT(!(parent_task->flags & STARPU_OMP_TASK_FLAGS_IMPLICIT));
  469. weak_task_unlock(parent_task);
  470. destroy_omp_task_struct(parent_task);
  471. }
  472. else if (parent_task->wait_on & starpu_omp_task_wait_on_task_childs)
  473. {
  474. parent_task->wait_on &= ~starpu_omp_task_wait_on_task_childs;
  475. wake_up_and_unlock_task(parent_task);
  476. }
  477. else
  478. {
  479. weak_task_unlock(parent_task);
  480. }
  481. }
  482. else
  483. {
  484. weak_task_unlock(parent_task);
  485. }
  486. _starpu_spin_lock(&parallel_region->lock);
  487. if (STARPU_ATOMIC_ADD(&parallel_region->bound_explicit_task_count, -1) == 0)
  488. {
  489. struct starpu_omp_task *waiting_task = parallel_region->waiting_task;
  490. _starpu_spin_unlock(&parallel_region->lock);
  491. if (waiting_task)
  492. {
  493. weak_task_lock(waiting_task);
  494. _starpu_spin_lock(&parallel_region->lock);
  495. parallel_region->waiting_task = NULL;
  496. STARPU_ASSERT(waiting_task->wait_on & starpu_omp_task_wait_on_region_tasks);
  497. waiting_task->wait_on &= ~starpu_omp_task_wait_on_region_tasks;
  498. _starpu_spin_unlock(&parallel_region->lock);
  499. wake_up_and_unlock_task(waiting_task);
  500. }
  501. }
  502. else
  503. {
  504. _starpu_spin_unlock(&parallel_region->lock);
  505. }
  506. if (task->task_group)
  507. {
  508. struct starpu_omp_task *leader_task = task->task_group->leader_task;
  509. STARPU_ASSERT(leader_task != task);
  510. weak_task_lock(leader_task);
  511. if (STARPU_ATOMIC_ADD(&task->task_group->descendent_task_count, -1) == 0)
  512. {
  513. if (leader_task->wait_on & starpu_omp_task_wait_on_group
  514. && task->task_group == leader_task->task_group)
  515. /* only wake the leader_task if it is actually
  516. * waiting for the current task's task_group */
  517. {
  518. leader_task->wait_on &= ~starpu_omp_task_wait_on_group;
  519. wake_up_and_unlock_task(leader_task);
  520. }
  521. else
  522. {
  523. weak_task_unlock(leader_task);
  524. }
  525. }
  526. else
  527. {
  528. weak_task_unlock(leader_task);
  529. }
  530. }
  531. }
  532. /*
  533. * wrap a task function to allow the task to be preempted
  534. */
  535. static void starpu_omp_explicit_task_exec(void *buffers[], void *cl_arg)
  536. {
  537. struct starpu_omp_task *task = starpu_task_get_current()->omp_task;
  538. STARPU_ASSERT(!(task->flags & STARPU_OMP_TASK_FLAGS_IMPLICIT));
  539. _starpu_omp_set_task(task);
  540. struct starpu_omp_thread *thread = get_local_thread();
  541. if (task->state != starpu_omp_task_state_preempted)
  542. {
  543. if (thread == NULL)
  544. {
  545. struct _starpu_worker *starpu_worker = _starpu_get_local_worker_key();
  546. if (starpu_worker->arch != STARPU_CPU_WORKER)
  547. {
  548. if (
  549. #ifdef STARPU_USE_CUDA
  550. (starpu_worker->arch != STARPU_CUDA_WORKER)
  551. &&
  552. #endif
  553. #ifdef STARPU_USE_OPENCL
  554. (starpu_worker->arch != STARPU_OPENCL_WORKER)
  555. &&
  556. #endif
  557. 1
  558. )
  559. {
  560. _STARPU_ERROR("invalid worker architecture");
  561. }
  562. struct starpu_omp_thread *new_thread;
  563. new_thread = create_omp_thread_struct(NULL);
  564. new_thread->worker = starpu_worker;
  565. register_thread_worker(new_thread);
  566. thread = get_local_thread();
  567. STARPU_ASSERT(thread == new_thread);
  568. }
  569. else
  570. {
  571. _STARPU_ERROR("orphaned CPU thread");
  572. }
  573. }
  574. STARPU_ASSERT(thread != NULL);
  575. if (!(task->flags & STARPU_OMP_TASK_FLAGS_UNTIED))
  576. {
  577. struct _starpu_worker *starpu_worker = _starpu_get_local_worker_key();
  578. task->starpu_task->workerid = starpu_worker->workerid;
  579. task->starpu_task->execute_on_a_specific_worker = 1;
  580. }
  581. task->starpu_buffers = buffers;
  582. task->starpu_cl_arg = cl_arg;
  583. STARPU_ASSERT(task->stack == NULL);
  584. STARPU_ASSERT(task->stacksize > 0);
  585. _STARPU_MALLOC(task->stack, task->stacksize);
  586. getcontext(&task->ctx);
  587. /*
  588. * we do not use uc_link, starpu_omp_task_entry will handle
  589. * the end of the task
  590. */
  591. task->ctx.uc_link = NULL;
  592. task->ctx.uc_stack.ss_sp = task->stack;
  593. task->ctx.uc_stack.ss_size = task->stacksize;
  594. makecontext(&task->ctx, (void (*) ()) starpu_omp_explicit_task_entry, 1, task);
  595. }
  596. task->state = starpu_omp_task_state_clear;
  597. /*
  598. * start the task execution, or restore a previously preempted task.
  599. * about to run on the task stack...
  600. * */
  601. swapcontext(&thread->ctx, &task->ctx);
  602. /* now running on the worker stack again */
  603. STARPU_ASSERT(task->state == starpu_omp_task_state_preempted
  604. || task->state == starpu_omp_task_state_terminated);
  605. _starpu_omp_set_task(NULL);
  606. /* TODO: analyse the cause of the return and take appropriate steps */
  607. if (task->state == starpu_omp_task_state_terminated)
  608. {
  609. free(task->stack);
  610. task->stack = NULL;
  611. memset(&task->ctx, 0, sizeof(task->ctx));
  612. starpu_omp_task_completion_accounting(task);
  613. }
  614. else if (task->state != starpu_omp_task_state_preempted)
  615. _STARPU_ERROR("invalid omp task state");
  616. }
  617. static struct starpu_omp_task *create_omp_task_struct(struct starpu_omp_task *parent_task,
  618. struct starpu_omp_thread *owner_thread, struct starpu_omp_region *owner_region, int is_implicit)
  619. {
  620. struct starpu_omp_task *task = starpu_omp_task_new();
  621. if (task == NULL)
  622. _STARPU_ERROR("memory allocation failed");
  623. memset(task, 0, sizeof(*task));
  624. task->parent_task = parent_task;
  625. task->owner_thread = owner_thread;
  626. task->owner_region = owner_region;
  627. if (is_implicit)
  628. {
  629. task->flags |= STARPU_OMP_TASK_FLAGS_IMPLICIT;
  630. }
  631. _starpu_spin_init(&task->lock);
  632. /* TODO: initialize task->data_env_icvs with proper values */
  633. memset(&task->data_env_icvs, 0, sizeof(task->data_env_icvs));
  634. if (is_implicit)
  635. {
  636. /* TODO: initialize task->implicit_task_icvs with proper values */
  637. memset(&task->implicit_task_icvs, 0, sizeof(task->implicit_task_icvs));
  638. }
  639. if (owner_region->level > 0)
  640. {
  641. STARPU_ASSERT(owner_region->owner_device->icvs.stacksize_var > 0);
  642. task->stacksize = owner_region->owner_device->icvs.stacksize_var;
  643. }
  644. return task;
  645. }
  646. static void destroy_omp_task_struct(struct starpu_omp_task *task)
  647. {
  648. STARPU_ASSERT(task->state == starpu_omp_task_state_terminated || (task->state == starpu_omp_task_state_zombie && task->child_task_count == 0) || task->state == starpu_omp_task_state_target);
  649. if (task->state == starpu_omp_task_state_target)
  650. {
  651. starpu_omp_task_completion_accounting(task);
  652. }
  653. STARPU_ASSERT(task->nested_region == NULL);
  654. STARPU_ASSERT(task->starpu_task == NULL);
  655. STARPU_ASSERT(task->stack == NULL);
  656. _starpu_spin_destroy(&task->lock);
  657. memset(task, 0, sizeof(*task));
  658. starpu_omp_task_delete(task);
  659. }
  660. /*
  661. * setup the main application thread to handle the possible preemption of the initial task
  662. */
  663. static void omp_initial_thread_setup(void)
  664. {
  665. struct starpu_omp_thread *initial_thread = _global_state.initial_thread;
  666. struct starpu_omp_task *initial_task = _global_state.initial_task;
  667. /* .current_task */
  668. initial_thread->current_task = initial_task;
  669. /* .owner_region already set in create_omp_thread_struct */
  670. /* .initial_thread_stack */
  671. _STARPU_MALLOC(initial_thread->initial_thread_stack, _STARPU_INITIAL_THREAD_STACKSIZE);
  672. if (initial_thread->initial_thread_stack == NULL)
  673. _STARPU_ERROR("memory allocation failed");
  674. /* .ctx */
  675. getcontext(&initial_thread->ctx);
  676. /*
  677. * we do not use uc_link, the initial thread always should give hand back to the initial task
  678. */
  679. initial_thread->ctx.uc_link = NULL;
  680. initial_thread->ctx.uc_stack.ss_sp = initial_thread->initial_thread_stack;
  681. initial_thread->ctx.uc_stack.ss_size = _STARPU_INITIAL_THREAD_STACKSIZE;
  682. initial_thread->initial_thread_stack_vg_id = VALGRIND_STACK_REGISTER(initial_thread->initial_thread_stack, initial_thread->initial_thread_stack+_STARPU_INITIAL_THREAD_STACKSIZE);
  683. makecontext(&initial_thread->ctx, omp_initial_thread_func, 0);
  684. /* .starpu_driver */
  685. /*
  686. * we configure starpu to not launch CPU worker 0
  687. * because we will use the main thread to play the role of worker 0
  688. */
  689. struct starpu_conf omp_starpu_conf;
  690. int ret = starpu_conf_init(&omp_starpu_conf);
  691. STARPU_CHECK_RETURN_VALUE(ret, "starpu_conf_init");
  692. initial_thread->starpu_driver.type = STARPU_CPU_WORKER;
  693. initial_thread->starpu_driver.id.cpu_id = 0;
  694. omp_starpu_conf.not_launched_drivers = &initial_thread->starpu_driver;
  695. omp_starpu_conf.n_not_launched_drivers = 1;
  696. /* we are now ready to start StarPU */
  697. ret = starpu_init(&omp_starpu_conf);
  698. STARPU_CHECK_RETURN_VALUE(ret, "starpu_init");
  699. ret = starpu_driver_init(&initial_thread->starpu_driver);
  700. STARPU_CHECK_RETURN_VALUE(ret, "starpu_driver_init");
  701. _starpu_omp_set_task(initial_task);
  702. _global_state.nb_starpu_cpu_workers = starpu_worker_get_count_by_type(STARPU_CPU_WORKER);
  703. _STARPU_MALLOC(_global_state.starpu_cpu_worker_ids, _global_state.nb_starpu_cpu_workers * sizeof(int));
  704. if (_global_state.starpu_cpu_worker_ids == NULL)
  705. _STARPU_ERROR("memory allocation failed");
  706. ret = starpu_worker_get_ids_by_type(STARPU_CPU_WORKER, _global_state.starpu_cpu_worker_ids, _global_state.nb_starpu_cpu_workers);
  707. STARPU_ASSERT(ret == _global_state.nb_starpu_cpu_workers);
  708. initial_thread->worker = _starpu_get_worker_struct(_global_state.starpu_cpu_worker_ids[0]);
  709. STARPU_ASSERT(initial_thread->worker);
  710. STARPU_ASSERT(initial_thread->worker->arch == STARPU_CPU_WORKER);
  711. _starpu_omp_set_thread(initial_thread);
  712. register_thread_worker(initial_thread);
  713. }
  714. static void omp_initial_thread_exit()
  715. {
  716. struct starpu_omp_thread *initial_thread = _global_state.initial_thread;
  717. int ret = starpu_driver_deinit(&initial_thread->starpu_driver);
  718. STARPU_CHECK_RETURN_VALUE(ret, "starpu_driver_deinit");
  719. memset(&initial_thread->starpu_driver, 0, sizeof (initial_thread->starpu_driver));
  720. /* the driver for the main thread is now de-inited, we can shutdown Starpu */
  721. starpu_shutdown();
  722. free(_global_state.starpu_cpu_worker_ids);
  723. _global_state.starpu_cpu_worker_ids = NULL;
  724. _global_state.nb_starpu_cpu_workers = 0;
  725. VALGRIND_STACK_DEREGISTER(initial_thread->initial_thread_stack_vg_id);
  726. free(initial_thread->initial_thread_stack);
  727. initial_thread->initial_thread_stack = NULL;
  728. memset(&initial_thread->ctx, 0, sizeof (initial_thread->ctx));
  729. initial_thread->current_task = NULL;
  730. }
  731. static void omp_initial_region_setup(void)
  732. {
  733. omp_initial_thread_setup();
  734. const int max_active_levels = _starpu_omp_initial_icv_values->max_active_levels_var;
  735. const int max_threads = (int)starpu_cpu_worker_get_count();
  736. /* implementation specific initial ICV values override */
  737. if (_starpu_omp_initial_icv_values->nthreads_var[0] == 0)
  738. {
  739. _starpu_omp_initial_icv_values->nthreads_var[0] = max_threads;
  740. _starpu_omp_initial_icv_values->nthreads_var[1] = 0;
  741. }
  742. else
  743. {
  744. int i;
  745. for (i = 0; i < max_active_levels; i++)
  746. {
  747. if (_starpu_omp_initial_icv_values->nthreads_var[i] == 0)
  748. break;
  749. if (_starpu_omp_initial_icv_values->nthreads_var[i] > max_threads)
  750. {
  751. _starpu_omp_initial_icv_values->nthreads_var[i] = max_threads;
  752. }
  753. }
  754. }
  755. _starpu_omp_initial_icv_values->dyn_var = 0;
  756. _starpu_omp_initial_icv_values->nest_var = 0;
  757. _global_state.initial_device->icvs.max_active_levels_var = max_active_levels;
  758. _global_state.initial_device->icvs.def_sched_var = _starpu_omp_initial_icv_values->def_sched_var;
  759. _global_state.initial_device->icvs.def_sched_chunk_var = _starpu_omp_initial_icv_values->def_sched_chunk_var;
  760. _global_state.initial_device->icvs.stacksize_var = _starpu_omp_initial_icv_values->stacksize_var;
  761. _global_state.initial_device->icvs.wait_policy_var = _starpu_omp_initial_icv_values->wait_policy_var;
  762. _global_state.initial_region->master_thread = _global_state.initial_thread;
  763. _global_state.initial_region->nb_threads++;
  764. _global_state.initial_region->icvs.dyn_var = _starpu_omp_initial_icv_values->dyn_var;
  765. _global_state.initial_region->icvs.nest_var = _starpu_omp_initial_icv_values->nest_var;
  766. if (_starpu_omp_initial_icv_values->nthreads_var[1] != 0)
  767. {
  768. _STARPU_MALLOC(_global_state.initial_region->icvs.nthreads_var, (1+max_active_levels-_global_state.initial_region->level) * sizeof(*_global_state.initial_region->icvs.nthreads_var));
  769. int i,j;
  770. for (i = _global_state.initial_region->level, j = 0; i < max_active_levels; i++, j++)
  771. {
  772. _global_state.initial_region->icvs.nthreads_var[j] = _starpu_omp_initial_icv_values->nthreads_var[j];
  773. }
  774. _global_state.initial_region->icvs.nthreads_var[j] = 0;
  775. }
  776. else
  777. {
  778. _STARPU_MALLOC(_global_state.initial_region->icvs.nthreads_var, 2 * sizeof(*_global_state.initial_region->icvs.nthreads_var));
  779. _global_state.initial_region->icvs.nthreads_var[0] = _starpu_omp_initial_icv_values->nthreads_var[0];
  780. _global_state.initial_region->icvs.nthreads_var[1] = 0;
  781. }
  782. if (_starpu_omp_initial_icv_values->bind_var[1] != starpu_omp_proc_bind_undefined)
  783. {
  784. _STARPU_MALLOC(_global_state.initial_region->icvs.bind_var, (1+max_active_levels-_global_state.initial_region->level) * sizeof(*_global_state.initial_region->icvs.bind_var));
  785. int i,j;
  786. for (i = _global_state.initial_region->level, j = 0; i < max_active_levels; i++, j++)
  787. {
  788. _global_state.initial_region->icvs.bind_var[j] = _starpu_omp_initial_icv_values->bind_var[j];
  789. }
  790. _global_state.initial_region->icvs.bind_var[j] = starpu_omp_proc_bind_undefined;
  791. }
  792. else
  793. {
  794. _STARPU_MALLOC(_global_state.initial_region->icvs.bind_var, 2 * sizeof(*_global_state.initial_region->icvs.bind_var));
  795. _global_state.initial_region->icvs.bind_var[0] = _starpu_omp_initial_icv_values->bind_var[0];
  796. _global_state.initial_region->icvs.bind_var[1] = starpu_omp_proc_bind_undefined;
  797. }
  798. _global_state.initial_region->icvs.thread_limit_var = _starpu_omp_initial_icv_values->thread_limit_var;
  799. _global_state.initial_region->icvs.active_levels_var = 0;
  800. _global_state.initial_region->icvs.levels_var = 0;
  801. _global_state.initial_region->icvs.run_sched_var = _starpu_omp_initial_icv_values->run_sched_var;
  802. _global_state.initial_region->icvs.run_sched_chunk_var = _starpu_omp_initial_icv_values->run_sched_chunk_var;
  803. _global_state.initial_region->icvs.default_device_var = _starpu_omp_initial_icv_values->default_device_var;
  804. _global_state.initial_region->icvs.max_task_priority_var = _starpu_omp_initial_icv_values->max_task_priority_var;
  805. starpu_omp_task_list_push_back(&_global_state.initial_region->implicit_task_list,
  806. _global_state.initial_task);
  807. }
  808. static void omp_initial_region_exit(void)
  809. {
  810. omp_initial_thread_exit();
  811. _global_state.initial_task->state = starpu_omp_task_state_terminated;
  812. starpu_omp_task_list_pop_front(&_global_state.initial_region->implicit_task_list);
  813. _global_state.initial_region->master_thread = NULL;
  814. free(_global_state.initial_region->icvs.nthreads_var);
  815. free(_global_state.initial_region->icvs.bind_var);
  816. _global_state.initial_region->nb_threads--;
  817. }
  818. /*
  819. * If StarPU was compiled with --enable-openmp, but the OpenMP runtime support
  820. * is not in use, starpu_init() may have been called directly instead of
  821. * through starpu_omp_init(). However, some starpu_omp functions may be still
  822. * be called such as _starpu_omp_get_task(). So let's setup a basic environment
  823. * for them.
  824. */
  825. void _starpu_omp_dummy_init(void)
  826. {
  827. if (_starpu_omp_global_state != &_global_state)
  828. {
  829. STARPU_PTHREAD_KEY_CREATE(&omp_thread_key, NULL);
  830. STARPU_PTHREAD_KEY_CREATE(&omp_task_key, NULL);
  831. }
  832. }
  833. /*
  834. * Free data structures allocated by _starpu_omp_dummy_init().
  835. */
  836. void _starpu_omp_dummy_shutdown(void)
  837. {
  838. if (_starpu_omp_global_state != &_global_state)
  839. {
  840. STARPU_PTHREAD_KEY_DELETE(omp_thread_key);
  841. STARPU_PTHREAD_KEY_DELETE(omp_task_key);
  842. }
  843. }
  844. /*
  845. * Entry point to be called by the OpenMP runtime constructor
  846. */
  847. int starpu_omp_init(void)
  848. {
  849. _starpu_omp_global_state = &_global_state;
  850. STARPU_PTHREAD_KEY_CREATE(&omp_thread_key, NULL);
  851. STARPU_PTHREAD_KEY_CREATE(&omp_task_key, NULL);
  852. _global_state.initial_device = create_omp_device_struct();
  853. _global_state.initial_region = create_omp_region_struct(NULL, _global_state.initial_device);
  854. _global_state.initial_thread = create_omp_thread_struct(_global_state.initial_region);
  855. _global_state.initial_task = create_omp_task_struct(NULL,
  856. _global_state.initial_thread, _global_state.initial_region, 1);
  857. _global_state.default_critical = create_omp_critical_struct();
  858. _global_state.default_arbiter = starpu_arbiter_create();
  859. _global_state.named_criticals = NULL;
  860. _starpu_spin_init(&_global_state.named_criticals_lock);
  861. _global_state.hash_workers = NULL;
  862. _starpu_spin_init(&_global_state.hash_workers_lock);
  863. _starpu_omp_environment_init();
  864. _global_state.icvs.cancel_var = _starpu_omp_initial_icv_values->cancel_var;
  865. omp_initial_region_setup();
  866. /* init clock reference for starpu_omp_get_wtick */
  867. _starpu_omp_clock_ref = starpu_timing_now();
  868. return 0;
  869. }
  870. void starpu_omp_shutdown(void)
  871. {
  872. omp_initial_region_exit();
  873. /* TODO: free ICV variables */
  874. /* TODO: free task/thread/region/device structures */
  875. destroy_omp_task_struct(_global_state.initial_task);
  876. _global_state.initial_task = NULL;
  877. _global_state.initial_thread = NULL;
  878. destroy_omp_region_struct(_global_state.initial_region);
  879. _global_state.initial_region = NULL;
  880. destroy_omp_device_struct(_global_state.initial_device);
  881. _global_state.initial_device = NULL;
  882. destroy_omp_critical_struct(_global_state.default_critical);
  883. _global_state.default_critical = NULL;
  884. starpu_arbiter_destroy(_global_state.default_arbiter);
  885. _global_state.default_arbiter = NULL;
  886. _starpu_spin_lock(&_global_state.named_criticals_lock);
  887. {
  888. struct starpu_omp_critical *critical, *tmp;
  889. HASH_ITER(hh, _global_state.named_criticals, critical, tmp)
  890. {
  891. STARPU_ASSERT(critical != NULL);
  892. HASH_DEL(_global_state.named_criticals, critical);
  893. destroy_omp_critical_struct(critical);
  894. }
  895. }
  896. STARPU_ASSERT(_global_state.named_criticals == NULL);
  897. _starpu_spin_unlock(&_global_state.named_criticals_lock);
  898. _starpu_spin_destroy(&_global_state.named_criticals_lock);
  899. _starpu_spin_lock(&_global_state.hash_workers_lock);
  900. {
  901. struct starpu_omp_thread *thread, *tmp;
  902. HASH_ITER(hh, _global_state.hash_workers, thread, tmp)
  903. {
  904. STARPU_ASSERT(thread != NULL);
  905. HASH_DEL(_global_state.hash_workers, thread);
  906. destroy_omp_thread_struct(thread);
  907. }
  908. }
  909. STARPU_ASSERT(_global_state.hash_workers == NULL);
  910. _starpu_spin_unlock(&_global_state.hash_workers_lock);
  911. _starpu_spin_destroy(&_global_state.hash_workers_lock);
  912. _starpu_omp_environment_exit();
  913. STARPU_PTHREAD_KEY_DELETE(omp_task_key);
  914. STARPU_PTHREAD_KEY_DELETE(omp_thread_key);
  915. }
  916. void starpu_omp_parallel_region(const struct starpu_omp_parallel_region_attr *attr)
  917. {
  918. struct starpu_omp_thread *master_thread = _starpu_omp_get_thread();
  919. struct starpu_omp_task *task = _starpu_omp_get_task();
  920. struct starpu_omp_region *generating_region = task->owner_region;
  921. const int max_active_levels = generating_region->owner_device->icvs.max_active_levels_var;
  922. struct starpu_omp_region *new_region =
  923. create_omp_region_struct(generating_region, _global_state.initial_device);
  924. int ret;
  925. int nb_threads = 1;
  926. /* TODO: for now, nested parallel sections are not supported, thus we
  927. * open an active parallel section only if the generating region is the
  928. * initial region */
  929. if (attr->if_clause != 0)
  930. {
  931. const int max_threads = (int)starpu_cpu_worker_get_count();
  932. if (attr->num_threads > 0)
  933. {
  934. nb_threads = attr->num_threads;
  935. }
  936. else
  937. {
  938. nb_threads = generating_region->icvs.nthreads_var[0];
  939. }
  940. if (nb_threads > max_threads)
  941. {
  942. nb_threads = max_threads;
  943. }
  944. if (nb_threads > 1 && generating_region->icvs.active_levels_var+1 > max_active_levels)
  945. {
  946. nb_threads = 1;
  947. }
  948. }
  949. STARPU_ASSERT(nb_threads > 0);
  950. new_region->icvs.dyn_var = generating_region->icvs.dyn_var;
  951. new_region->icvs.nest_var = generating_region->icvs.nest_var;
  952. /* the nthreads_var and bind_var arrays do not hold more than
  953. * max_active_levels entries at most, even if some in-between levels
  954. * are inactive */
  955. if (new_region->level < max_active_levels)
  956. {
  957. if (generating_region->icvs.nthreads_var[1] != 0)
  958. {
  959. _STARPU_MALLOC(new_region->icvs.nthreads_var, (1+max_active_levels-new_region->level) * sizeof(*new_region->icvs.nthreads_var));
  960. int i,j;
  961. for (i = new_region->level, j = 0; i < max_active_levels; i++, j++)
  962. {
  963. new_region->icvs.nthreads_var[j] = generating_region->icvs.nthreads_var[j+1];
  964. }
  965. new_region->icvs.nthreads_var[j] = 0;
  966. }
  967. else
  968. {
  969. _STARPU_MALLOC(new_region->icvs.nthreads_var, 2 * sizeof(*new_region->icvs.nthreads_var));
  970. new_region->icvs.nthreads_var[0] = generating_region->icvs.nthreads_var[0];
  971. new_region->icvs.nthreads_var[1] = 0;
  972. }
  973. if (generating_region->icvs.bind_var[1] != starpu_omp_proc_bind_undefined)
  974. {
  975. _STARPU_MALLOC(new_region->icvs.bind_var, (1+max_active_levels-new_region->level) * sizeof(*new_region->icvs.bind_var));
  976. int i,j;
  977. for (i = new_region->level, j = 0; i < max_active_levels; i++, j++)
  978. {
  979. new_region->icvs.bind_var[j] = generating_region->icvs.bind_var[j+1];
  980. }
  981. new_region->icvs.bind_var[j] = starpu_omp_proc_bind_undefined;
  982. }
  983. else
  984. {
  985. _STARPU_MALLOC(new_region->icvs.bind_var, 2 * sizeof(*new_region->icvs.bind_var));
  986. new_region->icvs.bind_var[0] = generating_region->icvs.bind_var[0];
  987. new_region->icvs.bind_var[1] = starpu_omp_proc_bind_undefined;
  988. }
  989. }
  990. else
  991. {
  992. _STARPU_MALLOC(new_region->icvs.nthreads_var, sizeof(*new_region->icvs.nthreads_var));
  993. new_region->icvs.nthreads_var[0] = generating_region->icvs.nthreads_var[0];
  994. _STARPU_MALLOC(new_region->icvs.bind_var, sizeof(*new_region->icvs.bind_var));
  995. new_region->icvs.bind_var[0] = generating_region->icvs.bind_var[0];
  996. }
  997. new_region->icvs.thread_limit_var = generating_region->icvs.thread_limit_var;
  998. new_region->icvs.active_levels_var = (nb_threads > 1)?generating_region->icvs.active_levels_var+1:generating_region->icvs.active_levels_var;
  999. new_region->icvs.levels_var = generating_region->icvs.levels_var+1;
  1000. new_region->icvs.run_sched_var = generating_region->icvs.run_sched_var;
  1001. new_region->icvs.run_sched_chunk_var = generating_region->icvs.run_sched_chunk_var;
  1002. new_region->icvs.default_device_var = generating_region->icvs.default_device_var;
  1003. new_region->icvs.max_task_priority_var = generating_region->icvs.max_task_priority_var;
  1004. int i;
  1005. for (i = 0; i < nb_threads; i++)
  1006. {
  1007. struct starpu_omp_thread *new_thread;
  1008. if (i == 0)
  1009. {
  1010. new_thread = master_thread;
  1011. new_region->master_thread = master_thread;
  1012. }
  1013. else
  1014. {
  1015. /* TODO: specify actual starpu worker */
  1016. /* TODO: use a less arbitrary thread/worker mapping scheme */
  1017. if (generating_region->level == 0)
  1018. {
  1019. struct _starpu_worker *worker = _starpu_get_worker_struct(_global_state.starpu_cpu_worker_ids[i]);
  1020. new_thread = get_worker_thread(worker);
  1021. if (new_thread == NULL)
  1022. {
  1023. new_thread = create_omp_thread_struct(new_region);
  1024. new_thread->worker = _starpu_get_worker_struct(_global_state.starpu_cpu_worker_ids[i]);
  1025. register_thread_worker(new_thread);
  1026. }
  1027. }
  1028. else
  1029. {
  1030. new_thread = master_thread;
  1031. }
  1032. starpu_omp_thread_list_push_back(&new_region->thread_list, new_thread);
  1033. }
  1034. struct starpu_omp_task *new_task = create_omp_task_struct(task, new_thread, new_region, 1);
  1035. new_task->rank = new_region->nb_threads;
  1036. new_region->nb_threads++;
  1037. starpu_omp_task_list_push_back(&new_region->implicit_task_list, new_task);
  1038. }
  1039. STARPU_ASSERT(new_region->nb_threads == nb_threads);
  1040. /*
  1041. * if task == initial_task, create a starpu task as a continuation to all the implicit
  1042. * tasks of the new region, else prepare the task for preemption,
  1043. * to become itself a continuation to the implicit tasks of the new region
  1044. */
  1045. if (task == _global_state.initial_task)
  1046. {
  1047. new_region->continuation_starpu_task = starpu_task_create();
  1048. /* in that case, the continuation starpu task is only used for synchronisation */
  1049. new_region->continuation_starpu_task->cl = NULL;
  1050. new_region->continuation_starpu_task->workerid = master_thread->worker->workerid;
  1051. new_region->continuation_starpu_task->execute_on_a_specific_worker = 1;
  1052. /* this sync task will be tested for completion in omp_initial_thread_func() */
  1053. new_region->continuation_starpu_task->detach = 0;
  1054. }
  1055. else
  1056. {
  1057. /* through the preemption, the parent starpu task becomes the continuation task */
  1058. _starpu_task_prepare_for_continuation();
  1059. new_region->continuation_starpu_task = task->starpu_task;
  1060. }
  1061. task->nested_region = new_region;
  1062. /*
  1063. * create the starpu tasks for the implicit omp tasks,
  1064. * create explicit dependencies between these starpu tasks and the continuation starpu task
  1065. */
  1066. struct starpu_omp_task * implicit_task;
  1067. for (implicit_task = starpu_omp_task_list_begin(&new_region->implicit_task_list);
  1068. implicit_task != starpu_omp_task_list_end(&new_region->implicit_task_list);
  1069. implicit_task = starpu_omp_task_list_next(implicit_task))
  1070. {
  1071. implicit_task->cl = attr->cl;
  1072. /*
  1073. * save pointer to the regions user function from the parallel region codelet
  1074. *
  1075. * TODO: add support for multiple/heterogeneous implementations
  1076. */
  1077. implicit_task->cpu_f = implicit_task->cl.cpu_funcs[0];
  1078. /*
  1079. * plug the task wrapper into the parallel region codelet instead, to support task preemption
  1080. */
  1081. implicit_task->cl.cpu_funcs[0] = starpu_omp_implicit_task_exec;
  1082. implicit_task->starpu_task = starpu_task_create();
  1083. implicit_task->starpu_task->cl = &implicit_task->cl;
  1084. {
  1085. for (i = 0; i < implicit_task->cl.nbuffers; i++)
  1086. {
  1087. implicit_task->starpu_task->handles[i] = attr->handles[i];
  1088. }
  1089. }
  1090. implicit_task->starpu_task->cl_arg = attr->cl_arg;
  1091. implicit_task->starpu_task->cl_arg_size = attr->cl_arg_size;
  1092. implicit_task->starpu_task->cl_arg_free = attr->cl_arg_free;
  1093. implicit_task->starpu_task->omp_task = implicit_task;
  1094. implicit_task->starpu_task->workerid = implicit_task->owner_thread->worker->workerid;
  1095. implicit_task->starpu_task->execute_on_a_specific_worker = 1;
  1096. starpu_task_declare_deps_array(new_region->continuation_starpu_task, 1, &implicit_task->starpu_task);
  1097. }
  1098. attr = NULL;
  1099. /*
  1100. * submit all the region implicit starpu tasks
  1101. */
  1102. for (implicit_task = starpu_omp_task_list_begin(&new_region->implicit_task_list);
  1103. implicit_task != starpu_omp_task_list_end(&new_region->implicit_task_list);
  1104. implicit_task = starpu_omp_task_list_next(implicit_task))
  1105. {
  1106. ret = starpu_task_submit(implicit_task->starpu_task);
  1107. STARPU_CHECK_RETURN_VALUE(ret, "starpu_task_submit");
  1108. }
  1109. /*
  1110. * submit the region continuation starpu task if task == initial_task
  1111. */
  1112. if (task == _global_state.initial_task)
  1113. {
  1114. ret = _starpu_task_submit_internally(new_region->continuation_starpu_task);
  1115. STARPU_CHECK_RETURN_VALUE(ret, "_starpu_task_submit_internally");
  1116. }
  1117. /*
  1118. * preempt for completion of the region
  1119. */
  1120. starpu_omp_task_preempt();
  1121. if (task == _global_state.initial_task)
  1122. {
  1123. STARPU_ASSERT(new_region->continuation_starpu_task == NULL);
  1124. }
  1125. else
  1126. {
  1127. STARPU_ASSERT(new_region->continuation_starpu_task != NULL);
  1128. new_region->continuation_starpu_task = NULL;
  1129. }
  1130. /*
  1131. * TODO: free region resources
  1132. */
  1133. for (i = 0; i < nb_threads; i++)
  1134. {
  1135. if (i == 0)
  1136. {
  1137. new_region->master_thread = NULL;
  1138. }
  1139. else
  1140. {
  1141. starpu_omp_thread_list_pop_front(&new_region->thread_list);
  1142. /* TODO: cleanup unused threads */
  1143. }
  1144. new_region->nb_threads--;
  1145. struct starpu_omp_task *implicit_task_p = starpu_omp_task_list_pop_front(&new_region->implicit_task_list);
  1146. destroy_omp_task_struct(implicit_task_p);
  1147. }
  1148. STARPU_ASSERT(new_region->nb_threads == 0);
  1149. task->nested_region = NULL;
  1150. free(new_region->icvs.bind_var);
  1151. free(new_region->icvs.nthreads_var);
  1152. destroy_omp_region_struct(new_region);
  1153. }
  1154. static void wake_up_barrier(struct starpu_omp_region *parallel_region)
  1155. {
  1156. struct starpu_omp_task *task = _starpu_omp_get_task();
  1157. struct starpu_omp_task *implicit_task;
  1158. for (implicit_task = starpu_omp_task_list_begin(&parallel_region->implicit_task_list);
  1159. implicit_task != starpu_omp_task_list_end(&parallel_region->implicit_task_list);
  1160. implicit_task = starpu_omp_task_list_next(implicit_task))
  1161. {
  1162. if (implicit_task == task)
  1163. continue;
  1164. weak_task_lock(implicit_task);
  1165. STARPU_ASSERT(implicit_task->wait_on & starpu_omp_task_wait_on_barrier);
  1166. implicit_task->wait_on &= ~starpu_omp_task_wait_on_barrier;
  1167. wake_up_and_unlock_task(implicit_task);
  1168. }
  1169. }
  1170. void starpu_omp_barrier(void)
  1171. {
  1172. struct starpu_omp_task *task = _starpu_omp_get_task();
  1173. /* Assume barriers are performed in by the implicit tasks of a parallel_region */
  1174. STARPU_ASSERT(task->flags & STARPU_OMP_TASK_FLAGS_IMPLICIT);
  1175. struct starpu_omp_region *parallel_region = task->owner_region;
  1176. _starpu_spin_lock(&task->lock);
  1177. int inc_barrier_count = STARPU_ATOMIC_ADD(&parallel_region->barrier_count, 1);
  1178. if (inc_barrier_count == parallel_region->nb_threads)
  1179. {
  1180. /* last task reaching the barrier */
  1181. _starpu_spin_lock(&parallel_region->lock);
  1182. ANNOTATE_HAPPENS_AFTER(&parallel_region->barrier_count);
  1183. ANNOTATE_HAPPENS_BEFORE_FORGET_ALL(&parallel_region->barrier_count);
  1184. parallel_region->barrier_count = 0;
  1185. ANNOTATE_HAPPENS_AFTER(&parallel_region->barrier_count);
  1186. ANNOTATE_HAPPENS_BEFORE_FORGET_ALL(&parallel_region->barrier_count);
  1187. if (parallel_region->bound_explicit_task_count > 0)
  1188. {
  1189. task->wait_on |= starpu_omp_task_wait_on_region_tasks;
  1190. parallel_region->waiting_task = task;
  1191. task->transaction_pending = 1;
  1192. _starpu_spin_unlock(&parallel_region->lock);
  1193. _starpu_spin_unlock(&task->lock);
  1194. _starpu_task_prepare_for_continuation_ext(0, transaction_callback, task);
  1195. starpu_omp_task_preempt();
  1196. }
  1197. else
  1198. {
  1199. _starpu_spin_unlock(&parallel_region->lock);
  1200. _starpu_spin_unlock(&task->lock);
  1201. }
  1202. wake_up_barrier(parallel_region);
  1203. }
  1204. else
  1205. {
  1206. ANNOTATE_HAPPENS_BEFORE(&parallel_region->barrier_count);
  1207. /* not the last task reaching the barrier
  1208. * . prepare for conditional continuation
  1209. * . sleep
  1210. */
  1211. task->wait_on |= starpu_omp_task_wait_on_barrier;
  1212. task->transaction_pending = 1;
  1213. _starpu_spin_unlock(&task->lock);
  1214. _starpu_task_prepare_for_continuation_ext(0, transaction_callback, task);
  1215. starpu_omp_task_preempt();
  1216. STARPU_ASSERT(task->child_task_count == 0);
  1217. }
  1218. }
  1219. void starpu_omp_master(void (*f)(void *arg), void *arg)
  1220. {
  1221. if (starpu_omp_master_inline())
  1222. f(arg);
  1223. }
  1224. /* variant of omp_master for inlined code
  1225. * return !0 for the task that should perform the master section
  1226. * return 0 for the tasks that should not perform the master section */
  1227. int starpu_omp_master_inline(void)
  1228. {
  1229. struct starpu_omp_task *task = _starpu_omp_get_task();
  1230. struct starpu_omp_thread *thread = _starpu_omp_get_thread();
  1231. /* Assume master is performed in by the implicit tasks of a region */
  1232. STARPU_ASSERT(task->flags & STARPU_OMP_TASK_FLAGS_IMPLICIT);
  1233. struct starpu_omp_region *region = task->owner_region;
  1234. return thread == region->master_thread;
  1235. }
  1236. void starpu_omp_single(void (*f)(void *arg), void *arg, int nowait)
  1237. {
  1238. if (starpu_omp_single_inline())
  1239. f(arg);
  1240. if (!nowait)
  1241. starpu_omp_barrier();
  1242. }
  1243. /* variant of omp_single for inlined code
  1244. * return !0 for the task that should perform the single section
  1245. * return 0 for the tasks that should not perform the single section
  1246. * wait/nowait should be handled directly by the calling code using starpu_omp_barrier */
  1247. int starpu_omp_single_inline(void)
  1248. {
  1249. struct starpu_omp_task *task = _starpu_omp_get_task();
  1250. /* Assume singles are performed in by the implicit tasks of a region */
  1251. STARPU_ASSERT(task->flags & STARPU_OMP_TASK_FLAGS_IMPLICIT);
  1252. struct starpu_omp_region *region = task->owner_region;
  1253. int first = STARPU_BOOL_COMPARE_AND_SWAP(&region->single_id, task->single_id, task->single_id+1);
  1254. task->single_id++;
  1255. return first;
  1256. }
  1257. void starpu_omp_single_copyprivate(void (*f)(void *arg, void *data, unsigned long long data_size), void *arg, void *data, unsigned long long data_size)
  1258. {
  1259. struct starpu_omp_task *task = _starpu_omp_get_task();
  1260. struct starpu_omp_region *region = task->owner_region;
  1261. int first = starpu_omp_single_inline();
  1262. if (first)
  1263. {
  1264. region->copy_private_data = data;
  1265. f(arg, data, data_size);
  1266. }
  1267. starpu_omp_barrier();
  1268. if (!first)
  1269. memcpy(data, region->copy_private_data, data_size);
  1270. starpu_omp_barrier();
  1271. }
  1272. void *starpu_omp_single_copyprivate_inline_begin(void *data)
  1273. {
  1274. struct starpu_omp_task *task = _starpu_omp_get_task();
  1275. struct starpu_omp_region *region = task->owner_region;
  1276. int first = starpu_omp_single_inline();
  1277. if (first)
  1278. {
  1279. task->single_first = 1;
  1280. region->copy_private_data = data;
  1281. return NULL;
  1282. }
  1283. starpu_omp_barrier();
  1284. return region->copy_private_data;
  1285. }
  1286. void starpu_omp_single_copyprivate_inline_end(void)
  1287. {
  1288. struct starpu_omp_task *task = _starpu_omp_get_task();
  1289. /* Assume singles are performed in by the implicit tasks of a region */
  1290. STARPU_ASSERT(task->flags & STARPU_OMP_TASK_FLAGS_IMPLICIT);
  1291. if (task->single_first)
  1292. {
  1293. task->single_first = 0;
  1294. starpu_omp_barrier();
  1295. }
  1296. starpu_omp_barrier();
  1297. }
  1298. void starpu_omp_critical(void (*f)(void *arg), void *arg, const char *name)
  1299. {
  1300. starpu_omp_critical_inline_begin(name);
  1301. f(arg);
  1302. starpu_omp_critical_inline_end(name);
  1303. }
  1304. void starpu_omp_critical_inline_begin(const char *name)
  1305. {
  1306. struct starpu_omp_task *task = _starpu_omp_get_task();
  1307. struct starpu_omp_critical *critical = NULL;
  1308. struct starpu_omp_task_link link;
  1309. if (name)
  1310. {
  1311. _starpu_spin_lock(&_global_state.named_criticals_lock);
  1312. HASH_FIND_STR(_global_state.named_criticals, name, critical);
  1313. if (critical == NULL)
  1314. {
  1315. critical = create_omp_critical_struct();
  1316. critical->name = name;
  1317. HASH_ADD_STR(_global_state.named_criticals, name, critical);
  1318. }
  1319. _starpu_spin_unlock(&_global_state.named_criticals_lock);
  1320. }
  1321. else
  1322. {
  1323. critical = _global_state.default_critical;
  1324. }
  1325. _starpu_spin_lock(&critical->lock);
  1326. while (critical->state != 0)
  1327. {
  1328. _starpu_spin_lock(&task->lock);
  1329. task->wait_on |= starpu_omp_task_wait_on_critical;
  1330. task->transaction_pending = 1;
  1331. link.task = task;
  1332. link.next = critical->contention_list_head;
  1333. critical->contention_list_head = &link;
  1334. _starpu_spin_unlock(&task->lock);
  1335. _starpu_spin_unlock(&critical->lock);
  1336. _starpu_task_prepare_for_continuation_ext(0, transaction_callback, task);
  1337. starpu_omp_task_preempt();
  1338. /* re-acquire the spin lock */
  1339. _starpu_spin_lock(&critical->lock);
  1340. }
  1341. critical->state = 1;
  1342. _starpu_spin_unlock(&critical->lock);
  1343. }
  1344. void starpu_omp_critical_inline_end(const char *name)
  1345. {
  1346. struct starpu_omp_critical *critical = NULL;
  1347. if (name)
  1348. {
  1349. _starpu_spin_lock(&_global_state.named_criticals_lock);
  1350. HASH_FIND_STR(_global_state.named_criticals, name, critical);
  1351. STARPU_ASSERT(critical != NULL);
  1352. _starpu_spin_unlock(&_global_state.named_criticals_lock);
  1353. }
  1354. else
  1355. {
  1356. critical = _global_state.default_critical;
  1357. }
  1358. _starpu_spin_lock(&critical->lock);
  1359. STARPU_ASSERT(critical->state == 1);
  1360. critical->state = 0;
  1361. if (critical->contention_list_head != NULL)
  1362. {
  1363. struct starpu_omp_task *next_task = critical->contention_list_head->task;
  1364. weak_task_lock(next_task);
  1365. critical->contention_list_head = critical->contention_list_head->next;
  1366. STARPU_ASSERT(next_task->wait_on & starpu_omp_task_wait_on_critical);
  1367. next_task->wait_on &= ~starpu_omp_task_wait_on_critical;
  1368. wake_up_and_unlock_task(next_task);
  1369. }
  1370. _starpu_spin_unlock(&critical->lock);
  1371. }
  1372. static void explicit_task__destroy_callback(void *_task)
  1373. {
  1374. struct starpu_omp_task *task = _task;
  1375. STARPU_ASSERT(!(task->flags & STARPU_OMP_TASK_FLAGS_IMPLICIT));
  1376. task->starpu_task->omp_task = NULL;
  1377. task->starpu_task = NULL;
  1378. _starpu_spin_lock(&task->lock);
  1379. if (task->state != starpu_omp_task_state_target)
  1380. {
  1381. STARPU_ASSERT(task->transaction_pending == 1);
  1382. task->transaction_pending = 0;
  1383. if (task->child_task_count != 0)
  1384. {
  1385. task->state = starpu_omp_task_state_zombie;
  1386. _starpu_spin_unlock(&task->lock);
  1387. return;
  1388. }
  1389. }
  1390. _starpu_spin_unlock(&task->lock);
  1391. destroy_omp_task_struct(task);
  1392. }
  1393. void starpu_omp_task_region(const struct starpu_omp_task_region_attr *attr)
  1394. {
  1395. struct starpu_omp_task *generating_task = _starpu_omp_get_task();
  1396. struct starpu_omp_region *parallel_region = generating_task->owner_region;
  1397. int is_undeferred = 0;
  1398. int is_final = 0;
  1399. int is_included = 0;
  1400. int is_merged = 0;
  1401. int ret;
  1402. if (generating_task == _global_state.initial_task)
  1403. {
  1404. is_undeferred = 1;
  1405. is_final = 1;
  1406. is_included = 1;
  1407. }
  1408. else
  1409. {
  1410. if (!attr->if_clause)
  1411. {
  1412. is_undeferred = 1;
  1413. }
  1414. if (generating_task->flags & STARPU_OMP_TASK_FLAGS_FINAL)
  1415. {
  1416. is_final = 1;
  1417. is_included = 1;
  1418. }
  1419. else if (attr->final_clause)
  1420. {
  1421. is_final = 1;
  1422. }
  1423. if (is_included)
  1424. {
  1425. is_undeferred = 1;
  1426. }
  1427. if ((is_undeferred || is_included) & attr->mergeable_clause)
  1428. {
  1429. is_merged = 1;
  1430. }
  1431. }
  1432. if (is_merged || is_included)
  1433. {
  1434. if (is_included) {
  1435. /* TODO: backup current ICVs and setup new ICVs for the included task */
  1436. }
  1437. int i;
  1438. unsigned n = attr->cl.nbuffers;
  1439. if (n == 0)
  1440. n = 1;
  1441. void *data_interfaces[n];
  1442. for (i = 0; i < attr->cl.nbuffers; i++)
  1443. {
  1444. starpu_data_handle_t handle = attr->handles[i];
  1445. ret = starpu_data_acquire(handle, attr->cl.modes[i]);
  1446. STARPU_CHECK_RETURN_VALUE(ret, "starpu_data_acquire");
  1447. data_interfaces[i] = starpu_data_get_interface_on_node(handle, handle->home_node);
  1448. }
  1449. void (*f)(void **starpu_buffers, void *starpu_cl_arg) = attr->cl.cpu_funcs[0];
  1450. f(data_interfaces, attr->cl_arg);
  1451. for (i = 0; i < attr->cl.nbuffers; i++)
  1452. {
  1453. starpu_data_release(attr->handles[i]);
  1454. }
  1455. if (attr->cl_arg_free)
  1456. {
  1457. free(attr->cl_arg);
  1458. }
  1459. if (is_included) {
  1460. /* TODO: restore backuped ICVs */
  1461. }
  1462. }
  1463. else
  1464. {
  1465. struct starpu_omp_task *generated_task =
  1466. create_omp_task_struct(generating_task, NULL, parallel_region, 0);
  1467. generated_task->cl = attr->cl;
  1468. if (attr->untied_clause)
  1469. {
  1470. generated_task->flags |= STARPU_OMP_TASK_FLAGS_UNTIED;
  1471. }
  1472. if (is_final)
  1473. {
  1474. generated_task->flags |= STARPU_OMP_TASK_FLAGS_FINAL;
  1475. }
  1476. if (is_undeferred)
  1477. {
  1478. generated_task->flags |= STARPU_OMP_TASK_FLAGS_UNDEFERRED;
  1479. }
  1480. generated_task->task_group = generating_task->task_group;
  1481. generated_task->rank = -1;
  1482. /*
  1483. * save pointer to the regions user function from the task region codelet
  1484. *
  1485. * TODO: add support for multiple/heterogeneous implementations
  1486. */
  1487. if (generated_task->cl.cpu_funcs[0])
  1488. {
  1489. generated_task->cpu_f = generated_task->cl.cpu_funcs[0];
  1490. /*
  1491. * plug the task wrapper into the task region codelet instead, to support task preemption
  1492. */
  1493. generated_task->cl.cpu_funcs[0] = starpu_omp_explicit_task_exec;
  1494. }
  1495. #ifdef STARPU_USE_CUDA
  1496. if (generated_task->cl.cuda_funcs[0])
  1497. {
  1498. generated_task->cuda_f = generated_task->cl.cuda_funcs[0];
  1499. #if 1
  1500. /* we assume for now that Cuda task won't block, thus we don't need
  1501. * to initialize the StarPU OpenMP Runtime Support context for enabling
  1502. * continuations on Cuda tasks */
  1503. generated_task->state = starpu_omp_task_state_target;
  1504. #else
  1505. generated_task->cl.cuda_funcs[0] = starpu_omp_explicit_task_exec;
  1506. #endif
  1507. }
  1508. #endif
  1509. #ifdef STARPU_USE_OPENCL
  1510. if (generated_task->cl.opencl_funcs[0])
  1511. {
  1512. generated_task->opencl_f = generated_task->cl.opencl_funcs[0];
  1513. #if 1
  1514. /* we assume for now that OpenCL task won't block, thus we don't need
  1515. * to initialize the StarPU OpenMP Runtime Support context for enabling
  1516. * continuations on OpenCL tasks */
  1517. generated_task->state = starpu_omp_task_state_target;
  1518. #else
  1519. generated_task->cl.opencl_funcs[0] = starpu_omp_explicit_task_exec;
  1520. #endif
  1521. }
  1522. #endif
  1523. /* TODO: add other accelerator support */
  1524. generated_task->starpu_task = starpu_task_create();
  1525. generated_task->starpu_task->cl = &generated_task->cl;
  1526. generated_task->starpu_task->cl_arg = attr->cl_arg;
  1527. generated_task->starpu_task->cl_arg_size = attr->cl_arg_size;
  1528. generated_task->starpu_task->cl_arg_free = attr->cl_arg_free;
  1529. generated_task->starpu_task->priority = attr->priority;
  1530. {
  1531. int i;
  1532. for (i = 0; i < generated_task->cl.nbuffers; i++)
  1533. {
  1534. generated_task->starpu_task->handles[i] = attr->handles[i];
  1535. }
  1536. }
  1537. generated_task->starpu_task->omp_task = generated_task;
  1538. _starpu_task_set_omp_cleanup_callback(generated_task->starpu_task, explicit_task__destroy_callback, generated_task);
  1539. /* if the task is tied, execute_on_a_specific_worker will be changed to 1
  1540. * upon the first preemption of the generated task, once we know
  1541. * which worker thread has been selected */
  1542. generated_task->starpu_task->execute_on_a_specific_worker = 0;
  1543. (void)STARPU_ATOMIC_ADD(&generating_task->child_task_count, 1);
  1544. (void)STARPU_ATOMIC_ADD(&parallel_region->bound_explicit_task_count, 1);
  1545. if (generated_task->task_group)
  1546. {
  1547. (void)STARPU_ATOMIC_ADD(&generated_task->task_group->descendent_task_count, 1);
  1548. }
  1549. /* do not use the attribute struct afterward as it may become out of scope */
  1550. attr = NULL;
  1551. if (is_undeferred)
  1552. {
  1553. _starpu_task_prepare_for_continuation();
  1554. starpu_task_declare_deps_array(generating_task->starpu_task, 1,
  1555. &generated_task->starpu_task);
  1556. }
  1557. ret = starpu_task_submit(generated_task->starpu_task);
  1558. STARPU_CHECK_RETURN_VALUE(ret, "starpu_task_submit");
  1559. if (is_undeferred)
  1560. {
  1561. starpu_omp_task_preempt();
  1562. }
  1563. }
  1564. }
  1565. void starpu_omp_taskwait(void)
  1566. {
  1567. struct starpu_omp_task *task = _starpu_omp_get_task();
  1568. _starpu_spin_lock(&task->lock);
  1569. if (task->child_task_count > 0)
  1570. {
  1571. task->wait_on |= starpu_omp_task_wait_on_task_childs;
  1572. task->transaction_pending = 1;
  1573. _starpu_spin_unlock(&task->lock);
  1574. _starpu_task_prepare_for_continuation_ext(0, transaction_callback, task);
  1575. starpu_omp_task_preempt();
  1576. STARPU_ASSERT(task->child_task_count == 0);
  1577. }
  1578. else
  1579. {
  1580. _starpu_spin_unlock(&task->lock);
  1581. }
  1582. }
  1583. void starpu_omp_taskgroup(void (*f)(void *arg), void *arg)
  1584. {
  1585. struct starpu_omp_task *task = _starpu_omp_get_task();
  1586. struct starpu_omp_task_group task_group;
  1587. task_group.p_previous_task_group = task->task_group;
  1588. task_group.descendent_task_count = 0;
  1589. task_group.leader_task = task;
  1590. task->task_group = &task_group;
  1591. f(arg);
  1592. _starpu_spin_lock(&task->lock);
  1593. if (task_group.descendent_task_count > 0)
  1594. {
  1595. task->wait_on |= starpu_omp_task_wait_on_group;
  1596. task->transaction_pending = 1;
  1597. _starpu_spin_unlock(&task->lock);
  1598. _starpu_task_prepare_for_continuation_ext(0, transaction_callback, task);
  1599. starpu_omp_task_preempt();
  1600. STARPU_ASSERT(task_group.descendent_task_count == 0);
  1601. }
  1602. else
  1603. {
  1604. _starpu_spin_unlock(&task->lock);
  1605. }
  1606. task->task_group = task_group.p_previous_task_group;
  1607. }
  1608. void starpu_omp_taskgroup_inline_begin(void)
  1609. {
  1610. struct starpu_omp_task *task = _starpu_omp_get_task();
  1611. struct starpu_omp_task_group *p_task_group;
  1612. _STARPU_MALLOC(p_task_group, sizeof(*p_task_group));
  1613. p_task_group->p_previous_task_group = task->task_group;
  1614. p_task_group->descendent_task_count = 0;
  1615. p_task_group->leader_task = task;
  1616. task->task_group = p_task_group;
  1617. }
  1618. void starpu_omp_taskgroup_inline_end(void)
  1619. {
  1620. struct starpu_omp_task *task = _starpu_omp_get_task();
  1621. _starpu_spin_lock(&task->lock);
  1622. struct starpu_omp_task_group *p_task_group = task->task_group;
  1623. if (p_task_group->descendent_task_count > 0)
  1624. {
  1625. task->wait_on |= starpu_omp_task_wait_on_group;
  1626. task->transaction_pending = 1;
  1627. _starpu_spin_unlock(&task->lock);
  1628. _starpu_task_prepare_for_continuation_ext(0, transaction_callback, task);
  1629. starpu_omp_task_preempt();
  1630. STARPU_ASSERT(p_task_group->descendent_task_count == 0);
  1631. }
  1632. else
  1633. {
  1634. _starpu_spin_unlock(&task->lock);
  1635. }
  1636. task->task_group = p_task_group->p_previous_task_group;
  1637. free(p_task_group);
  1638. }
  1639. static inline void _starpu_omp_for_loop(struct starpu_omp_region *parallel_region, struct starpu_omp_task *task,
  1640. struct starpu_omp_loop *loop, int first_call,
  1641. unsigned long long nb_iterations, unsigned long long chunk, int schedule, int ordered, unsigned long long *_first_i, unsigned long long *_nb_i)
  1642. {
  1643. *_nb_i = 0;
  1644. if (schedule == starpu_omp_sched_undefined)
  1645. {
  1646. schedule = parallel_region->owner_device->icvs.def_sched_var;
  1647. chunk = parallel_region->owner_device->icvs.def_sched_chunk_var;
  1648. }
  1649. else if (schedule == starpu_omp_sched_runtime)
  1650. {
  1651. schedule = parallel_region->icvs.run_sched_var;
  1652. chunk = parallel_region->icvs.run_sched_chunk_var;
  1653. }
  1654. STARPU_ASSERT( schedule == starpu_omp_sched_static
  1655. || schedule == starpu_omp_sched_dynamic
  1656. || schedule == starpu_omp_sched_guided
  1657. || schedule == starpu_omp_sched_auto);
  1658. if (schedule == starpu_omp_sched_auto)
  1659. {
  1660. schedule = starpu_omp_sched_static;
  1661. chunk = 0;
  1662. }
  1663. if (schedule == starpu_omp_sched_static)
  1664. {
  1665. if (chunk > 0)
  1666. {
  1667. if (first_call)
  1668. {
  1669. *_first_i = task->rank * chunk;
  1670. }
  1671. else
  1672. {
  1673. *_first_i += parallel_region->nb_threads * chunk;
  1674. }
  1675. if (*_first_i < nb_iterations)
  1676. {
  1677. if (*_first_i + chunk > nb_iterations)
  1678. {
  1679. *_nb_i = nb_iterations - *_first_i;
  1680. }
  1681. else
  1682. {
  1683. *_nb_i = chunk;
  1684. }
  1685. }
  1686. }
  1687. else
  1688. {
  1689. if (first_call)
  1690. {
  1691. *_nb_i = nb_iterations / parallel_region->nb_threads;
  1692. *_first_i = (unsigned)task->rank * (*_nb_i);
  1693. unsigned long long remainder = nb_iterations % parallel_region->nb_threads;
  1694. if (remainder > 0)
  1695. {
  1696. if ((unsigned)task->rank < remainder)
  1697. {
  1698. (*_nb_i)++;
  1699. *_first_i += (unsigned)task->rank;
  1700. }
  1701. else
  1702. {
  1703. *_first_i += remainder;
  1704. }
  1705. }
  1706. }
  1707. }
  1708. }
  1709. else if (schedule == starpu_omp_sched_dynamic)
  1710. {
  1711. if (chunk == 0)
  1712. {
  1713. chunk = 1;
  1714. }
  1715. if (first_call)
  1716. {
  1717. *_first_i = 0;
  1718. }
  1719. _starpu_spin_lock(&parallel_region->lock);
  1720. if (loop->next_iteration < nb_iterations)
  1721. {
  1722. *_first_i = loop->next_iteration;
  1723. if (*_first_i + chunk > nb_iterations)
  1724. {
  1725. *_nb_i = nb_iterations - *_first_i;
  1726. }
  1727. else
  1728. {
  1729. *_nb_i = chunk;
  1730. }
  1731. loop->next_iteration += *_nb_i;
  1732. }
  1733. _starpu_spin_unlock(&parallel_region->lock);
  1734. }
  1735. else if (schedule == starpu_omp_sched_guided)
  1736. {
  1737. if (chunk == 0)
  1738. {
  1739. chunk = 1;
  1740. }
  1741. if (first_call)
  1742. {
  1743. *_first_i = 0;
  1744. }
  1745. _starpu_spin_lock(&parallel_region->lock);
  1746. if (loop->next_iteration < nb_iterations)
  1747. {
  1748. *_first_i = loop->next_iteration;
  1749. *_nb_i = (nb_iterations - *_first_i)/parallel_region->nb_threads;
  1750. if (*_nb_i < chunk)
  1751. {
  1752. if (*_first_i+chunk > nb_iterations)
  1753. {
  1754. *_nb_i = nb_iterations - *_first_i;
  1755. }
  1756. else
  1757. {
  1758. *_nb_i = chunk;
  1759. }
  1760. }
  1761. loop->next_iteration += *_nb_i;
  1762. }
  1763. _starpu_spin_unlock(&parallel_region->lock);
  1764. }
  1765. if (ordered)
  1766. {
  1767. task->ordered_first_i = *_first_i;
  1768. task->ordered_nb_i = *_nb_i;
  1769. }
  1770. }
  1771. static inline struct starpu_omp_loop *_starpu_omp_for_get_loop(struct starpu_omp_region *parallel_region, struct starpu_omp_task *task)
  1772. {
  1773. struct starpu_omp_loop *loop;
  1774. loop = parallel_region->loop_list;
  1775. while (loop && loop->id != task->loop_id)
  1776. {
  1777. loop = loop->next_loop;
  1778. }
  1779. return loop;
  1780. }
  1781. static inline struct starpu_omp_loop *_starpu_omp_for_loop_begin(struct starpu_omp_region *parallel_region, struct starpu_omp_task *task,
  1782. int ordered)
  1783. {
  1784. struct starpu_omp_loop *loop;
  1785. _starpu_spin_lock(&parallel_region->lock);
  1786. loop = _starpu_omp_for_get_loop(parallel_region, task);
  1787. if (!loop)
  1788. {
  1789. _STARPU_MALLOC(loop, sizeof(*loop));
  1790. loop->id = task->loop_id;
  1791. loop->next_iteration = 0;
  1792. loop->nb_completed_threads = 0;
  1793. loop->next_loop = parallel_region->loop_list;
  1794. parallel_region->loop_list = loop;
  1795. if (ordered)
  1796. {
  1797. loop->ordered_iteration = 0;
  1798. _starpu_spin_init(&loop->ordered_lock);
  1799. condition_init(&loop->ordered_cond);
  1800. }
  1801. }
  1802. _starpu_spin_unlock(&parallel_region->lock);
  1803. return loop;
  1804. }
  1805. static inline void _starpu_omp_for_loop_end(struct starpu_omp_region *parallel_region, struct starpu_omp_task *task,
  1806. struct starpu_omp_loop *loop, int ordered)
  1807. {
  1808. _starpu_spin_lock(&parallel_region->lock);
  1809. loop->nb_completed_threads++;
  1810. if (loop->nb_completed_threads == parallel_region->nb_threads)
  1811. {
  1812. struct starpu_omp_loop **p_loop;
  1813. if (ordered)
  1814. {
  1815. loop->ordered_iteration = 0;
  1816. condition_exit(&loop->ordered_cond);
  1817. _starpu_spin_destroy(&loop->ordered_lock);
  1818. }
  1819. STARPU_ASSERT(loop->next_loop == NULL);
  1820. p_loop = &(parallel_region->loop_list);
  1821. while (*p_loop != loop)
  1822. {
  1823. p_loop = &((*p_loop)->next_loop);
  1824. }
  1825. *p_loop = NULL;
  1826. free(loop);
  1827. }
  1828. _starpu_spin_unlock(&parallel_region->lock);
  1829. task->loop_id++;
  1830. }
  1831. int starpu_omp_for_inline_first(unsigned long long nb_iterations, unsigned long long chunk, int schedule, int ordered, unsigned long long *_first_i, unsigned long long *_nb_i)
  1832. {
  1833. struct starpu_omp_task *task = _starpu_omp_get_task();
  1834. struct starpu_omp_region *parallel_region = task->owner_region;
  1835. struct starpu_omp_loop *loop = _starpu_omp_for_loop_begin(parallel_region, task, ordered);
  1836. _starpu_omp_for_loop(parallel_region, task, loop, 1, nb_iterations, chunk, schedule, ordered, _first_i, _nb_i);
  1837. if (*_nb_i == 0)
  1838. {
  1839. _starpu_omp_for_loop_end(parallel_region, task, loop, ordered);
  1840. }
  1841. return (*_nb_i != 0);
  1842. }
  1843. int starpu_omp_for_inline_next(unsigned long long nb_iterations, unsigned long long chunk, int schedule, int ordered, unsigned long long *_first_i, unsigned long long *_nb_i)
  1844. {
  1845. struct starpu_omp_task *task = _starpu_omp_get_task();
  1846. struct starpu_omp_region *parallel_region = task->owner_region;
  1847. struct starpu_omp_loop *loop = _starpu_omp_for_loop_begin(parallel_region, task, ordered);
  1848. _starpu_omp_for_loop(parallel_region, task, loop, 0, nb_iterations, chunk, schedule, ordered, _first_i, _nb_i);
  1849. if (*_nb_i == 0)
  1850. {
  1851. _starpu_omp_for_loop_end(parallel_region, task, loop, ordered);
  1852. }
  1853. return (*_nb_i != 0);
  1854. }
  1855. int starpu_omp_for_inline_first_alt(unsigned long long nb_iterations, unsigned long long chunk, int schedule, int ordered, unsigned long long *_begin_i, unsigned long long *_end_i)
  1856. {
  1857. unsigned long long nb_i;
  1858. int end = starpu_omp_for_inline_first(nb_iterations, chunk, schedule, ordered, _begin_i, &nb_i);
  1859. *_end_i = *_begin_i + nb_i;
  1860. return end;
  1861. }
  1862. int starpu_omp_for_inline_next_alt(unsigned long long nb_iterations, unsigned long long chunk, int schedule, int ordered, unsigned long long *_begin_i, unsigned long long *_end_i)
  1863. {
  1864. unsigned long long nb_i;
  1865. int end = starpu_omp_for_inline_next(nb_iterations, chunk, schedule, ordered, _begin_i, &nb_i);
  1866. *_end_i = *_begin_i + nb_i;
  1867. return end;
  1868. }
  1869. void starpu_omp_for(void (*f)(unsigned long long _first_i, unsigned long long _nb_i, void *arg), void *arg, unsigned long long nb_iterations, unsigned long long chunk, int schedule, int ordered, int nowait)
  1870. {
  1871. unsigned long long _first_i = 0;
  1872. unsigned long long _nb_i = 0;
  1873. if (starpu_omp_for_inline_first(nb_iterations, chunk, schedule, ordered, &_first_i, &_nb_i))
  1874. {
  1875. do
  1876. {
  1877. f(_first_i, _nb_i, arg);
  1878. }
  1879. while (starpu_omp_for_inline_next(nb_iterations, chunk, schedule, ordered, &_first_i, &_nb_i));
  1880. }
  1881. if (!nowait)
  1882. {
  1883. starpu_omp_barrier();
  1884. }
  1885. }
  1886. void starpu_omp_for_alt(void (*f)(unsigned long long _begin_i, unsigned long long _end_i, void *arg), void *arg, unsigned long long nb_iterations, unsigned long long chunk, int schedule, int ordered, int nowait)
  1887. {
  1888. unsigned long long _begin_i = 0;
  1889. unsigned long long _end_i = 0;
  1890. if (starpu_omp_for_inline_first_alt(nb_iterations, chunk, schedule, ordered, &_begin_i, &_end_i))
  1891. {
  1892. do
  1893. {
  1894. f(_begin_i, _end_i, arg);
  1895. }
  1896. while (starpu_omp_for_inline_next_alt(nb_iterations, chunk, schedule, ordered, &_begin_i, &_end_i));
  1897. }
  1898. if (!nowait)
  1899. {
  1900. starpu_omp_barrier();
  1901. }
  1902. }
  1903. void starpu_omp_ordered(void (*f)(void *arg), void *arg)
  1904. {
  1905. starpu_omp_ordered_inline_begin();
  1906. f(arg);
  1907. starpu_omp_ordered_inline_end();
  1908. }
  1909. void starpu_omp_ordered_inline_begin(void)
  1910. {
  1911. struct starpu_omp_task *task = _starpu_omp_get_task();
  1912. struct starpu_omp_region *parallel_region = task->owner_region;
  1913. struct starpu_omp_loop *loop = _starpu_omp_for_get_loop(parallel_region, task);
  1914. unsigned long long i;
  1915. STARPU_ASSERT(task->ordered_nb_i > 0);
  1916. i = task->ordered_first_i;
  1917. task->ordered_first_i++;
  1918. task->ordered_nb_i--;
  1919. _starpu_spin_lock(&loop->ordered_lock);
  1920. while (i != loop->ordered_iteration)
  1921. {
  1922. STARPU_ASSERT(i > loop->ordered_iteration);
  1923. condition_wait(&loop->ordered_cond, &loop->ordered_lock, starpu_omp_task_wait_on_ordered);
  1924. }
  1925. }
  1926. void starpu_omp_ordered_inline_end(void)
  1927. {
  1928. struct starpu_omp_task *task = _starpu_omp_get_task();
  1929. struct starpu_omp_region *parallel_region = task->owner_region;
  1930. struct starpu_omp_loop *loop = _starpu_omp_for_get_loop(parallel_region, task);
  1931. loop->ordered_iteration++;
  1932. condition_broadcast(&loop->ordered_cond, starpu_omp_task_wait_on_ordered);
  1933. _starpu_spin_unlock(&loop->ordered_lock);
  1934. }
  1935. static inline struct starpu_omp_sections *_starpu_omp_get_sections(struct starpu_omp_region *parallel_region, struct starpu_omp_task *task)
  1936. {
  1937. struct starpu_omp_sections *sections;
  1938. sections = parallel_region->sections_list;
  1939. while (sections && sections->id != task->sections_id)
  1940. {
  1941. sections = sections->next_sections;
  1942. }
  1943. return sections;
  1944. }
  1945. static inline struct starpu_omp_sections *_starpu_omp_sections_begin(struct starpu_omp_region *parallel_region, struct starpu_omp_task *task)
  1946. {
  1947. struct starpu_omp_sections *sections;
  1948. _starpu_spin_lock(&parallel_region->lock);
  1949. sections = _starpu_omp_get_sections(parallel_region, task);
  1950. if (!sections)
  1951. {
  1952. _STARPU_MALLOC(sections, sizeof(*sections));
  1953. sections->id = task->sections_id;
  1954. sections->next_section_num = 0;
  1955. sections->nb_completed_threads = 0;
  1956. sections->next_sections = parallel_region->sections_list;
  1957. parallel_region->sections_list = sections;
  1958. }
  1959. _starpu_spin_unlock(&parallel_region->lock);
  1960. return sections;
  1961. }
  1962. static inline void _starpu_omp_sections_end(struct starpu_omp_region *parallel_region, struct starpu_omp_task *task,
  1963. struct starpu_omp_sections *sections)
  1964. {
  1965. _starpu_spin_lock(&parallel_region->lock);
  1966. sections->nb_completed_threads++;
  1967. if (sections->nb_completed_threads == parallel_region->nb_threads)
  1968. {
  1969. struct starpu_omp_sections **p_sections;
  1970. STARPU_ASSERT(sections->next_sections == NULL);
  1971. p_sections = &(parallel_region->sections_list);
  1972. while (*p_sections != sections)
  1973. {
  1974. p_sections = &((*p_sections)->next_sections);
  1975. }
  1976. *p_sections = NULL;
  1977. free(sections);
  1978. }
  1979. _starpu_spin_unlock(&parallel_region->lock);
  1980. task->sections_id++;
  1981. }
  1982. void starpu_omp_sections(unsigned long long nb_sections, void (**section_f)(void *arg), void **section_arg, int nowait)
  1983. {
  1984. struct starpu_omp_task *task = _starpu_omp_get_task();
  1985. struct starpu_omp_region *parallel_region = task->owner_region;
  1986. struct starpu_omp_sections *sections = _starpu_omp_sections_begin(parallel_region, task);
  1987. for (;;)
  1988. {
  1989. void (*f)(void *arg) = NULL;
  1990. void *arg = NULL;
  1991. _starpu_spin_lock(&parallel_region->lock);
  1992. if (sections->next_section_num < nb_sections)
  1993. {
  1994. f = section_f[sections->next_section_num];
  1995. arg = section_arg[sections->next_section_num];
  1996. sections->next_section_num ++;
  1997. }
  1998. _starpu_spin_unlock(&parallel_region->lock);
  1999. if (f == NULL)
  2000. break;
  2001. f(arg);
  2002. }
  2003. _starpu_omp_sections_end(parallel_region, task, sections);
  2004. if (!nowait)
  2005. {
  2006. starpu_omp_barrier();
  2007. }
  2008. }
  2009. void starpu_omp_sections_combined(unsigned long long nb_sections, void (*section_f)(unsigned long long section_num, void *arg), void *section_arg, int nowait)
  2010. {
  2011. struct starpu_omp_task *task = _starpu_omp_get_task();
  2012. struct starpu_omp_region *parallel_region = task->owner_region;
  2013. struct starpu_omp_sections *sections = _starpu_omp_sections_begin(parallel_region, task);
  2014. for (;;)
  2015. {
  2016. unsigned long long section_num;
  2017. void *arg = NULL;
  2018. _starpu_spin_lock(&parallel_region->lock);
  2019. if (sections->next_section_num < nb_sections)
  2020. {
  2021. section_num = sections->next_section_num;
  2022. arg = section_arg;
  2023. sections->next_section_num ++;
  2024. }
  2025. else
  2026. {
  2027. _starpu_spin_unlock(&parallel_region->lock);
  2028. break;
  2029. }
  2030. _starpu_spin_unlock(&parallel_region->lock);
  2031. section_f(section_num, arg);
  2032. }
  2033. _starpu_omp_sections_end(parallel_region, task, sections);
  2034. if (!nowait)
  2035. {
  2036. starpu_omp_barrier();
  2037. }
  2038. }
  2039. static void _starpu_omp_lock_init(void **_internal)
  2040. {
  2041. struct _starpu_omp_lock_internal *_lock;
  2042. _STARPU_CALLOC(_lock, 1, sizeof(*_lock));
  2043. _starpu_spin_init(&_lock->lock);
  2044. condition_init(&_lock->cond);
  2045. *_internal = _lock;
  2046. }
  2047. static void _starpu_omp_lock_destroy(void **_internal)
  2048. {
  2049. struct _starpu_omp_lock_internal * const _lock = *_internal;
  2050. STARPU_ASSERT(_lock->state == 0);
  2051. condition_exit(&_lock->cond);
  2052. _starpu_spin_destroy(&_lock->lock);
  2053. memset(_lock, 0, sizeof(*_lock));
  2054. free(_lock);
  2055. *_internal = NULL;
  2056. }
  2057. static void _starpu_omp_lock_set(void **_internal)
  2058. {
  2059. struct _starpu_omp_lock_internal * const _lock = *_internal;
  2060. _starpu_spin_lock(&_lock->lock);
  2061. while (_lock->state != 0)
  2062. {
  2063. condition_wait(&_lock->cond, &_lock->lock, starpu_omp_task_wait_on_lock);
  2064. }
  2065. _lock->state = 1;
  2066. _starpu_spin_unlock(&_lock->lock);
  2067. }
  2068. static void _starpu_omp_lock_unset(void **_internal)
  2069. {
  2070. struct _starpu_omp_lock_internal * const _lock = *_internal;
  2071. _starpu_spin_lock(&_lock->lock);
  2072. STARPU_ASSERT(_lock->state == 1);
  2073. _lock->state = 0;
  2074. condition_broadcast(&_lock->cond, starpu_omp_task_wait_on_lock);
  2075. _starpu_spin_unlock(&_lock->lock);
  2076. }
  2077. static int _starpu_omp_lock_test(void **_internal)
  2078. {
  2079. struct _starpu_omp_lock_internal * const _lock = *_internal;
  2080. int ret = 0;
  2081. _starpu_spin_lock(&_lock->lock);
  2082. if (_lock->state == 0)
  2083. {
  2084. _lock->state = 1;
  2085. ret = 1;
  2086. }
  2087. _starpu_spin_unlock(&_lock->lock);
  2088. return ret;
  2089. }
  2090. static void _starpu_omp_nest_lock_init(void **_internal)
  2091. {
  2092. struct _starpu_omp_nest_lock_internal *_nest_lock;
  2093. _STARPU_CALLOC(_nest_lock, 1, sizeof(*_nest_lock));
  2094. _starpu_spin_init(&_nest_lock->lock);
  2095. condition_init(&_nest_lock->cond);
  2096. *_internal = _nest_lock;
  2097. }
  2098. static void _starpu_omp_nest_lock_destroy(void **_internal)
  2099. {
  2100. struct _starpu_omp_nest_lock_internal * const _nest_lock = *_internal;
  2101. STARPU_ASSERT(_nest_lock->state == 0);
  2102. STARPU_ASSERT(_nest_lock->nesting == 0);
  2103. STARPU_ASSERT(_nest_lock->owner_task == NULL);
  2104. condition_exit(&_nest_lock->cond);
  2105. _starpu_spin_destroy(&_nest_lock->lock);
  2106. memset(_nest_lock, 0, sizeof(*_nest_lock));
  2107. free(_nest_lock);
  2108. *_internal = NULL;
  2109. }
  2110. static void _starpu_omp_nest_lock_set(void **_internal)
  2111. {
  2112. struct _starpu_omp_nest_lock_internal * const _nest_lock = *_internal;
  2113. struct starpu_omp_task * const task = _starpu_omp_get_task();
  2114. _starpu_spin_lock(&_nest_lock->lock);
  2115. if (_nest_lock->owner_task == task)
  2116. {
  2117. STARPU_ASSERT(_nest_lock->state == 1);
  2118. STARPU_ASSERT(_nest_lock->nesting > 0);
  2119. _nest_lock->nesting++;
  2120. }
  2121. else
  2122. {
  2123. while (_nest_lock->state != 0)
  2124. {
  2125. condition_wait(&_nest_lock->cond, &_nest_lock->lock, starpu_omp_task_wait_on_nest_lock);
  2126. }
  2127. STARPU_ASSERT(_nest_lock->nesting == 0);
  2128. STARPU_ASSERT(_nest_lock->owner_task == NULL);
  2129. _nest_lock->state = 1;
  2130. _nest_lock->owner_task = task;
  2131. _nest_lock->nesting = 1;
  2132. }
  2133. _starpu_spin_unlock(&_nest_lock->lock);
  2134. }
  2135. static void _starpu_omp_nest_lock_unset(void **_internal)
  2136. {
  2137. struct _starpu_omp_nest_lock_internal * const _nest_lock = *_internal;
  2138. struct starpu_omp_task * const task = _starpu_omp_get_task();
  2139. _starpu_spin_lock(&_nest_lock->lock);
  2140. STARPU_ASSERT(_nest_lock->owner_task == task);
  2141. STARPU_ASSERT(_nest_lock->state == 1);
  2142. STARPU_ASSERT(_nest_lock->nesting > 0);
  2143. _nest_lock->nesting--;
  2144. if (_nest_lock->nesting == 0)
  2145. {
  2146. _nest_lock->state = 0;
  2147. _nest_lock->owner_task = NULL;
  2148. condition_broadcast(&_nest_lock->cond, starpu_omp_task_wait_on_nest_lock);
  2149. }
  2150. _starpu_spin_unlock(&_nest_lock->lock);
  2151. }
  2152. static int _starpu_omp_nest_lock_test(void **_internal)
  2153. {
  2154. struct _starpu_omp_nest_lock_internal * const _nest_lock = *_internal;
  2155. struct starpu_omp_task * const task = _starpu_omp_get_task();
  2156. int ret = 0;
  2157. _starpu_spin_lock(&_nest_lock->lock);
  2158. if (_nest_lock->state == 0)
  2159. {
  2160. STARPU_ASSERT(_nest_lock->nesting == 0);
  2161. STARPU_ASSERT(_nest_lock->owner_task == NULL);
  2162. _nest_lock->state = 1;
  2163. _nest_lock->owner_task = task;
  2164. _nest_lock->nesting = 1;
  2165. ret = _nest_lock->nesting;
  2166. }
  2167. else if (_nest_lock->owner_task == task)
  2168. {
  2169. STARPU_ASSERT(_nest_lock->state == 1);
  2170. STARPU_ASSERT(_nest_lock->nesting > 0);
  2171. _nest_lock->nesting++;
  2172. ret = _nest_lock->nesting;
  2173. }
  2174. _starpu_spin_unlock(&_nest_lock->lock);
  2175. return ret;
  2176. }
  2177. void starpu_omp_init_lock (starpu_omp_lock_t *lock)
  2178. {
  2179. _starpu_omp_lock_init(&lock->internal);
  2180. }
  2181. void starpu_omp_destroy_lock (starpu_omp_lock_t *lock)
  2182. {
  2183. _starpu_omp_lock_destroy(&lock->internal);
  2184. }
  2185. void starpu_omp_set_lock (starpu_omp_lock_t *lock)
  2186. {
  2187. _starpu_omp_lock_set(&lock->internal);
  2188. }
  2189. void starpu_omp_unset_lock (starpu_omp_lock_t *lock)
  2190. {
  2191. _starpu_omp_lock_unset(&lock->internal);
  2192. }
  2193. int starpu_omp_test_lock (starpu_omp_lock_t *lock)
  2194. {
  2195. return _starpu_omp_lock_test(&lock->internal);
  2196. }
  2197. void starpu_omp_init_nest_lock (starpu_omp_nest_lock_t *nest_lock)
  2198. {
  2199. _starpu_omp_nest_lock_init(&nest_lock->internal);
  2200. }
  2201. void starpu_omp_destroy_nest_lock (starpu_omp_nest_lock_t *nest_lock)
  2202. {
  2203. _starpu_omp_nest_lock_destroy(&nest_lock->internal);
  2204. }
  2205. void starpu_omp_set_nest_lock (starpu_omp_nest_lock_t *nest_lock)
  2206. {
  2207. _starpu_omp_nest_lock_set(&nest_lock->internal);
  2208. }
  2209. void starpu_omp_unset_nest_lock (starpu_omp_nest_lock_t *nest_lock)
  2210. {
  2211. _starpu_omp_nest_lock_unset(&nest_lock->internal);
  2212. }
  2213. int starpu_omp_test_nest_lock (starpu_omp_nest_lock_t *nest_lock)
  2214. {
  2215. return _starpu_omp_nest_lock_test(&nest_lock->internal);
  2216. }
  2217. void starpu_omp_atomic_fallback_inline_begin(void)
  2218. {
  2219. struct starpu_omp_device *device = get_caller_device();
  2220. _starpu_spin_lock(&device->atomic_lock);
  2221. }
  2222. void starpu_omp_atomic_fallback_inline_end(void)
  2223. {
  2224. struct starpu_omp_device *device = get_caller_device();
  2225. _starpu_spin_unlock(&device->atomic_lock);
  2226. }
  2227. void starpu_omp_vector_annotate(starpu_data_handle_t handle, uint32_t slice_base)
  2228. {
  2229. struct starpu_vector_interface *vector_interface = (struct starpu_vector_interface *)
  2230. starpu_data_get_interface_on_node(handle, STARPU_MAIN_RAM);
  2231. assert(vector_interface->id == STARPU_VECTOR_INTERFACE_ID);
  2232. vector_interface->slice_base = slice_base;
  2233. }
  2234. struct starpu_arbiter *starpu_omp_get_default_arbiter(void)
  2235. {
  2236. return _global_state.default_arbiter;
  2237. }
  2238. /*
  2239. * restore deprecated diagnostics (-Wdeprecated-declarations)
  2240. */
  2241. #pragma GCC diagnostic pop
  2242. #endif /* STARPU_OPENMP */