data_interface.c 40 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386
  1. /* StarPU --- Runtime system for heterogeneous multicore architectures.
  2. *
  3. * Copyright (C) 2009-2021 Université de Bordeaux, CNRS (LaBRI UMR 5800), Inria
  4. *
  5. * StarPU is free software; you can redistribute it and/or modify
  6. * it under the terms of the GNU Lesser General Public License as published by
  7. * the Free Software Foundation; either version 2.1 of the License, or (at
  8. * your option) any later version.
  9. *
  10. * StarPU is distributed in the hope that it will be useful, but
  11. * WITHOUT ANY WARRANTY; without even the implied warranty of
  12. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
  13. *
  14. * See the GNU Lesser General Public License in COPYING.LGPL for more details.
  15. */
  16. #include <stdint.h>
  17. #include <stdarg.h>
  18. #include <datawizard/datawizard.h>
  19. #include <datawizard/memory_nodes.h>
  20. #include <datawizard/memstats.h>
  21. #include <datawizard/malloc.h>
  22. #include <core/dependencies/data_concurrency.h>
  23. #include <common/uthash.h>
  24. #include <common/starpu_spinlock.h>
  25. #include <core/task.h>
  26. #include <core/workers.h>
  27. #ifdef STARPU_OPENMP
  28. #include <util/openmp_runtime_support.h>
  29. #endif
  30. static struct starpu_data_interface_ops **_id_to_ops_array;
  31. static unsigned _id_to_ops_array_size;
  32. /* Entry in the `registered_handles' hash table. */
  33. struct handle_entry
  34. {
  35. UT_hash_handle hh;
  36. void *pointer;
  37. starpu_data_handle_t handle;
  38. };
  39. /* Hash table mapping host pointers to data handles. */
  40. static int nregistered, maxnregistered;
  41. static struct handle_entry *registered_handles;
  42. static struct _starpu_spinlock registered_handles_lock;
  43. static int _data_interface_number = STARPU_MAX_INTERFACE_ID;
  44. starpu_arbiter_t _starpu_global_arbiter;
  45. static void _starpu_data_unregister(starpu_data_handle_t handle, unsigned coherent, unsigned nowait);
  46. void _starpu_data_interface_init(void)
  47. {
  48. _starpu_spin_init(&registered_handles_lock);
  49. /* Just for testing purpose */
  50. if (starpu_get_env_number_default("STARPU_GLOBAL_ARBITER", 0) > 0)
  51. _starpu_global_arbiter = starpu_arbiter_create();
  52. }
  53. void _starpu_data_interface_shutdown()
  54. {
  55. struct handle_entry *entry=NULL, *tmp=NULL;
  56. if (registered_handles)
  57. {
  58. _STARPU_DISP("[warning] The application has not unregistered all data handles.\n");
  59. }
  60. _starpu_spin_destroy(&registered_handles_lock);
  61. free(_id_to_ops_array);
  62. _id_to_ops_array = NULL;
  63. _id_to_ops_array_size = 0;
  64. HASH_ITER(hh, registered_handles, entry, tmp)
  65. {
  66. HASH_DEL(registered_handles, entry);
  67. nregistered--;
  68. free(entry);
  69. }
  70. if (starpu_get_env_number_default("STARPU_MAX_MEMORY_USE", 0))
  71. _STARPU_DISP("Memory used for %d data handles: %lu MiB\n", maxnregistered, (unsigned long) (maxnregistered * sizeof(struct _starpu_data_state)) >> 20);
  72. STARPU_ASSERT(nregistered == 0);
  73. registered_handles = NULL;
  74. }
  75. #ifdef STARPU_OPENMP
  76. void _starpu_omp_unregister_region_handles(struct starpu_omp_region *region)
  77. {
  78. _starpu_spin_lock(&region->registered_handles_lock);
  79. struct handle_entry *entry=NULL, *tmp=NULL;
  80. HASH_ITER(hh, (region->registered_handles), entry, tmp)
  81. {
  82. entry->handle->removed_from_context_hash = 1;
  83. HASH_DEL(region->registered_handles, entry);
  84. starpu_data_unregister(entry->handle);
  85. free(entry);
  86. }
  87. _starpu_spin_unlock(&region->registered_handles_lock);
  88. }
  89. void _starpu_omp_unregister_task_handles(struct starpu_omp_task *task)
  90. {
  91. struct handle_entry *entry=NULL, *tmp=NULL;
  92. HASH_ITER(hh, task->registered_handles, entry, tmp)
  93. {
  94. entry->handle->removed_from_context_hash = 1;
  95. HASH_DEL(task->registered_handles, entry);
  96. starpu_data_unregister(entry->handle);
  97. free(entry);
  98. }
  99. }
  100. #endif
  101. struct starpu_data_interface_ops *_starpu_data_interface_get_ops(unsigned interface_id)
  102. {
  103. switch (interface_id)
  104. {
  105. case STARPU_MATRIX_INTERFACE_ID:
  106. return &starpu_interface_matrix_ops;
  107. case STARPU_BLOCK_INTERFACE_ID:
  108. return &starpu_interface_block_ops;
  109. case STARPU_VECTOR_INTERFACE_ID:
  110. return &starpu_interface_vector_ops;
  111. case STARPU_CSR_INTERFACE_ID:
  112. return &starpu_interface_csr_ops;
  113. case STARPU_BCSR_INTERFACE_ID:
  114. return &starpu_interface_bcsr_ops;
  115. case STARPU_VARIABLE_INTERFACE_ID:
  116. return &starpu_interface_variable_ops;
  117. case STARPU_VOID_INTERFACE_ID:
  118. return &starpu_interface_void_ops;
  119. case STARPU_MULTIFORMAT_INTERFACE_ID:
  120. return &starpu_interface_multiformat_ops;
  121. default:
  122. {
  123. if (interface_id-STARPU_MAX_INTERFACE_ID > _id_to_ops_array_size || _id_to_ops_array[interface_id-STARPU_MAX_INTERFACE_ID]==NULL)
  124. {
  125. _STARPU_MSG("There is no 'struct starpu_data_interface_ops' registered for interface %d\n", interface_id);
  126. STARPU_ABORT();
  127. return NULL;
  128. }
  129. else
  130. return _id_to_ops_array[interface_id-STARPU_MAX_INTERFACE_ID];
  131. }
  132. }
  133. }
  134. /* Register the mapping from PTR to HANDLE. If PTR is already mapped to
  135. * some handle, the new mapping shadows the previous one. */
  136. void _starpu_data_register_ram_pointer(starpu_data_handle_t handle, void *ptr)
  137. {
  138. struct handle_entry *entry;
  139. _STARPU_MALLOC(entry, sizeof(*entry));
  140. entry->pointer = ptr;
  141. entry->handle = handle;
  142. #ifdef STARPU_OPENMP
  143. struct starpu_omp_task *task = _starpu_omp_get_task();
  144. if (task)
  145. {
  146. if (task->flags & STARPU_OMP_TASK_FLAGS_IMPLICIT)
  147. {
  148. struct starpu_omp_region *parallel_region = task->owner_region;
  149. _starpu_spin_lock(&parallel_region->registered_handles_lock);
  150. HASH_ADD_PTR(parallel_region->registered_handles, pointer, entry);
  151. _starpu_spin_unlock(&parallel_region->registered_handles_lock);
  152. }
  153. else
  154. {
  155. HASH_ADD_PTR(task->registered_handles, pointer, entry);
  156. }
  157. }
  158. else
  159. #endif
  160. {
  161. struct handle_entry *old_entry;
  162. _starpu_spin_lock(&registered_handles_lock);
  163. HASH_FIND_PTR(registered_handles, &ptr, old_entry);
  164. if (old_entry)
  165. {
  166. /* Already registered this pointer, avoid undefined
  167. * behavior of duplicate in hash table */
  168. _starpu_spin_unlock(&registered_handles_lock);
  169. free(entry);
  170. }
  171. else
  172. {
  173. nregistered++;
  174. if (nregistered > maxnregistered)
  175. maxnregistered = nregistered;
  176. HASH_ADD_PTR(registered_handles, pointer, entry);
  177. _starpu_spin_unlock(&registered_handles_lock);
  178. }
  179. }
  180. }
  181. starpu_data_handle_t starpu_data_lookup(const void *ptr)
  182. {
  183. starpu_data_handle_t result;
  184. #ifdef STARPU_OPENMP
  185. struct starpu_omp_task *task = _starpu_omp_get_task();
  186. if (task)
  187. {
  188. if (task->flags & STARPU_OMP_TASK_FLAGS_IMPLICIT)
  189. {
  190. struct starpu_omp_region *parallel_region = task->owner_region;
  191. _starpu_spin_lock(&parallel_region->registered_handles_lock);
  192. {
  193. struct handle_entry *entry;
  194. HASH_FIND_PTR(parallel_region->registered_handles, &ptr, entry);
  195. if(STARPU_UNLIKELY(entry == NULL))
  196. result = NULL;
  197. else
  198. result = entry->handle;
  199. }
  200. _starpu_spin_unlock(&parallel_region->registered_handles_lock);
  201. }
  202. else
  203. {
  204. struct handle_entry *entry;
  205. HASH_FIND_PTR(task->registered_handles, &ptr, entry);
  206. if(STARPU_UNLIKELY(entry == NULL))
  207. result = NULL;
  208. else
  209. result = entry->handle;
  210. }
  211. }
  212. else
  213. #endif
  214. {
  215. _starpu_spin_lock(&registered_handles_lock);
  216. {
  217. struct handle_entry *entry;
  218. HASH_FIND_PTR(registered_handles, &ptr, entry);
  219. if(STARPU_UNLIKELY(entry == NULL))
  220. result = NULL;
  221. else
  222. result = entry->handle;
  223. }
  224. _starpu_spin_unlock(&registered_handles_lock);
  225. }
  226. return result;
  227. }
  228. /*
  229. * Start monitoring a piece of data
  230. */
  231. static void _starpu_register_new_data(starpu_data_handle_t handle,
  232. int home_node, uint32_t wt_mask)
  233. {
  234. void *ptr;
  235. STARPU_ASSERT(handle);
  236. /* first take care to properly lock the data */
  237. _starpu_spin_lock(&handle->header_lock);
  238. handle->root_handle = handle;
  239. //handle->father_handle = NULL;
  240. //handle->nsiblings = 0;
  241. //handle->siblings = NULL;
  242. //handle->sibling_index = 0; /* could be anything for the root */
  243. handle->depth = 1; /* the tree is just a node yet */
  244. handle->active = 1;
  245. /* Store some values directly in the handle not to recompute them all
  246. * the time. */
  247. handle->footprint = _starpu_compute_data_footprint(handle);
  248. handle->home_node = home_node;
  249. handle->wt_mask = wt_mask;
  250. //handle->aliases = 0;
  251. //handle->readonly_dup = NULL;
  252. //handle->readonly_dup_of = NULL;
  253. //handle->is_not_important = 0;
  254. handle->sequential_consistency =
  255. starpu_data_get_default_sequential_consistency_flag();
  256. handle->initialized = home_node != -1;
  257. //handle->readonly = 0;
  258. handle->ooc = 1;
  259. /* By default, there are no methods available to perform a reduction */
  260. //handle->redux_cl = NULL;
  261. //handle->init_cl = NULL;
  262. /* that new data is invalid from all nodes perpective except for the
  263. * home node */
  264. unsigned node;
  265. for (node = 0; node < STARPU_MAXNODES; node++)
  266. {
  267. struct _starpu_data_replicate *replicate;
  268. replicate = &handle->per_node[node];
  269. replicate->memory_node = node;
  270. //replicate->relaxed_coherency = 0;
  271. //replicate->refcnt = 0;
  272. //replicate->nb_tasks_prefetch = 0;
  273. if ((int) node == home_node)
  274. {
  275. /* this is the home node with the only valid copy */
  276. replicate->state = STARPU_OWNER;
  277. replicate->allocated = 1;
  278. //replicate->automatically_allocated = 0;
  279. replicate->initialized = 1;
  280. }
  281. else
  282. {
  283. /* the value is not available here yet */
  284. replicate->state = STARPU_INVALID;
  285. //replicate->allocated = 0;
  286. //replicate->initialized = 0;
  287. }
  288. }
  289. /* now the data is available ! */
  290. _starpu_spin_unlock(&handle->header_lock);
  291. for (node = 0; node < STARPU_MAXNODES; node++)
  292. {
  293. if (starpu_node_get_kind(node) != STARPU_CPU_RAM)
  294. continue;
  295. ptr = starpu_data_handle_to_pointer(handle, node);
  296. if (ptr != NULL)
  297. _starpu_data_register_ram_pointer(handle, ptr);
  298. }
  299. }
  300. void
  301. _starpu_data_initialize_per_worker(starpu_data_handle_t handle)
  302. {
  303. unsigned worker;
  304. unsigned nworkers = starpu_worker_get_count();
  305. _starpu_spin_checklocked(&handle->header_lock);
  306. _STARPU_CALLOC(handle->per_worker, nworkers, sizeof(*handle->per_worker));
  307. size_t interfacesize = handle->ops->interface_size;
  308. for (worker = 0; worker < nworkers; worker++)
  309. {
  310. struct _starpu_data_replicate *replicate;
  311. //unsigned node;
  312. replicate = &handle->per_worker[worker];
  313. //replicate->allocated = 0;
  314. //replicate->automatically_allocated = 0;
  315. replicate->state = STARPU_INVALID;
  316. //replicate->refcnt = 0;
  317. replicate->handle = handle;
  318. //replicate->nb_tasks_prefetch = 0;
  319. //for (node = 0; node < STARPU_MAXNODES; node++)
  320. //{
  321. // replicate->request[node] = NULL;
  322. //}
  323. /* Assuming being used for SCRATCH for now, patched when entering REDUX mode */
  324. replicate->relaxed_coherency = 1;
  325. //replicate->initialized = 0;
  326. replicate->memory_node = starpu_worker_get_memory_node(worker);
  327. _STARPU_CALLOC(replicate->data_interface, 1, interfacesize);
  328. /* duplicate the content of the interface on node 0 */
  329. memcpy(replicate->data_interface, handle->per_node[STARPU_MAIN_RAM].data_interface, interfacesize);
  330. }
  331. }
  332. void starpu_data_ptr_register(starpu_data_handle_t handle, unsigned node)
  333. {
  334. struct _starpu_data_replicate *replicate = &handle->per_node[node];
  335. _starpu_spin_lock(&handle->header_lock);
  336. STARPU_ASSERT_MSG(replicate->allocated == 0, "starpu_data_ptr_register must be called right after starpu_data_register");
  337. replicate->allocated = 1;
  338. replicate->automatically_allocated = 0;
  339. _starpu_spin_unlock(&handle->header_lock);
  340. }
  341. int _starpu_data_handle_init(starpu_data_handle_t handle, struct starpu_data_interface_ops *interface_ops, unsigned int mf_node)
  342. {
  343. unsigned node;
  344. /* Tell helgrind that our access to busy_count in
  345. * starpu_data_unregister is actually safe */
  346. STARPU_HG_DISABLE_CHECKING(handle->busy_count);
  347. handle->magic = 42;
  348. /* When not specified, the fields are initialized in _starpu_register_new_data and _starpu_data_partition */
  349. _starpu_data_requester_prio_list_init0(&handle->req_list);
  350. //handle->refcnt = 0;
  351. //handle->unlocking_reqs = 0;
  352. //handle->current_mode = STARPU_NONE;
  353. _starpu_spin_init(&handle->header_lock);
  354. //handle->busy_count = 0;
  355. //handle->busy_waiting = 0;
  356. STARPU_PTHREAD_MUTEX_INIT0(&handle->busy_mutex, NULL);
  357. STARPU_PTHREAD_COND_INIT0(&handle->busy_cond, NULL);
  358. //handle->root_handle
  359. //handle->father_handle
  360. //handle->active_children = NULL;
  361. //handle->active_nchildren = 0;
  362. //handle->active_readonly_children = NULL;
  363. //handle->active_readonly_nchildren = NULL;
  364. //handle->nactive_readonly_children = 0;
  365. //handle->nsiblings
  366. //handle->siblings
  367. //handle->sibling_index
  368. //handle->depth
  369. /* there is no hierarchy yet */
  370. //handle->children = NULL;
  371. //handle->nchildren = 0;
  372. //handle->nplans = 0;
  373. //handle->switch_cl = NULL;
  374. //handle->switch_cl_nparts = 0;
  375. //handle->partitioned = 0;
  376. //handle->part_readonly = 0;
  377. //handle->active
  378. //handle->active_ro = 0;
  379. //handle->per_node below
  380. handle->ops = interface_ops;
  381. size_t interfacesize = interface_ops->interface_size;
  382. for (node = 0; node < STARPU_MAXNODES; node++)
  383. {
  384. _starpu_memory_stats_init_per_node(handle, node);
  385. struct _starpu_data_replicate *replicate;
  386. replicate = &handle->per_node[node];
  387. /* relaxed_coherency = 0 */
  388. replicate->handle = handle;
  389. _STARPU_CALLOC(replicate->data_interface, 1, interfacesize);
  390. if (handle->ops->init) handle->ops->init(replicate->data_interface);
  391. }
  392. //handle->per_worker = NULL;
  393. //handle->ops above
  394. //handle->footprint
  395. //handle->home_node
  396. //handle->wt_mask
  397. //handle->aliases = 0;
  398. //handle->is_not_important
  399. //handle->sequential_consistency
  400. //handle->initialized
  401. //handle->readonly
  402. //handle->ooc
  403. //handle->lazy_unregister = 0;
  404. //handle->partition_automatic_disabled = 0;
  405. //handle->removed_from_context_hash = 0;
  406. STARPU_PTHREAD_MUTEX_INIT0(&handle->sequential_consistency_mutex, NULL);
  407. handle->last_submitted_mode = STARPU_R;
  408. //handle->last_sync_task = NULL;
  409. //handle->last_submitted_accessors.task = NULL;
  410. handle->last_submitted_accessors.next = &handle->last_submitted_accessors;
  411. handle->last_submitted_accessors.prev = &handle->last_submitted_accessors;
  412. #ifdef STARPU_USE_FXT
  413. //handle->last_submitted_ghost_sync_id_is_valid = 0;
  414. //handle->last_submitted_ghost_sync_id = 0;
  415. //handle->last_submitted_ghost_accessors_id = NULL;
  416. #endif
  417. //handle->post_sync_tasks = NULL;
  418. /* Tell helgrind that the race in _starpu_unlock_post_sync_tasks is fine */
  419. STARPU_HG_DISABLE_CHECKING(handle->post_sync_tasks_cnt);
  420. //handle->post_sync_tasks_cnt = 0;
  421. //handle->redux_cl
  422. //handle->init_cl
  423. //handle->reduction_refcnt = 0;
  424. _starpu_data_requester_prio_list_init0(&handle->reduction_req_list);
  425. //handle->reduction_tmp_handles = NULL;
  426. //handle->write_invalidation_req = NULL;
  427. //handle->mpi_data = NULL; /* invalid until set */
  428. _starpu_memory_stats_init(handle);
  429. handle->mf_node = mf_node;
  430. //handle->unregister_hook = NULL;
  431. if (_starpu_global_arbiter)
  432. /* Just for testing purpose */
  433. starpu_data_assign_arbiter(handle, _starpu_global_arbiter);
  434. else
  435. {
  436. //handle->arbiter = NULL;
  437. }
  438. _starpu_data_requester_prio_list_init0(&handle->arbitered_req_list);
  439. handle->last_locality = -1;
  440. //handle->dimensions = 0;
  441. //handle->coordinates = {};
  442. //handle->user_data = NULL;
  443. return 0;
  444. }
  445. static
  446. starpu_data_handle_t _starpu_data_handle_allocate(struct starpu_data_interface_ops *interface_ops, unsigned int mf_node)
  447. {
  448. starpu_data_handle_t handle;
  449. _STARPU_CALLOC(handle, 1, sizeof(struct _starpu_data_state));
  450. _starpu_data_handle_init(handle, interface_ops, mf_node);
  451. return handle;
  452. }
  453. void starpu_data_register(starpu_data_handle_t *handleptr, int home_node,
  454. void *data_interface,
  455. struct starpu_data_interface_ops *ops)
  456. {
  457. starpu_data_handle_t handle = _starpu_data_handle_allocate(ops, home_node);
  458. STARPU_ASSERT(handleptr);
  459. *handleptr = handle;
  460. /* check the interfaceid is set */
  461. STARPU_ASSERT(ops->interfaceid != STARPU_UNKNOWN_INTERFACE_ID);
  462. /* fill the interface fields with the appropriate method */
  463. STARPU_ASSERT(ops->register_data_handle);
  464. ops->register_data_handle(handle, home_node, data_interface);
  465. if ((unsigned)ops->interfaceid >= STARPU_MAX_INTERFACE_ID)
  466. {
  467. if ((unsigned)ops->interfaceid > _id_to_ops_array_size)
  468. {
  469. if (!_id_to_ops_array_size)
  470. {
  471. _id_to_ops_array_size = 16;
  472. }
  473. else
  474. {
  475. _id_to_ops_array_size *= 2;
  476. }
  477. _STARPU_REALLOC(_id_to_ops_array, _id_to_ops_array_size * sizeof(struct starpu_data_interface_ops *));
  478. }
  479. _id_to_ops_array[ops->interfaceid-STARPU_MAX_INTERFACE_ID] = ops;
  480. }
  481. _starpu_register_new_data(handle, home_node, 0);
  482. _STARPU_TRACE_HANDLE_DATA_REGISTER(handle);
  483. }
  484. void starpu_data_register_same(starpu_data_handle_t *handledst, starpu_data_handle_t handlesrc)
  485. {
  486. void *local_interface = starpu_data_get_interface_on_node(handlesrc, STARPU_MAIN_RAM);
  487. starpu_data_register(handledst, -1, local_interface, handlesrc->ops);
  488. }
  489. void *starpu_data_handle_to_pointer(starpu_data_handle_t handle, unsigned node)
  490. {
  491. /* Check whether the operation is supported and the node has actually
  492. * been allocated. */
  493. if (!starpu_data_test_if_allocated_on_node(handle, node))
  494. return NULL;
  495. if (handle->ops->to_pointer)
  496. {
  497. return handle->ops->to_pointer(starpu_data_get_interface_on_node(handle, node), node);
  498. }
  499. /* Deprecated */
  500. if (handle->ops->handle_to_pointer)
  501. {
  502. return handle->ops->handle_to_pointer(handle, node);
  503. }
  504. return NULL;
  505. }
  506. int starpu_data_pointer_is_inside(starpu_data_handle_t handle, unsigned node, void *ptr)
  507. {
  508. /* Check whether the operation is supported and the node has actually
  509. * been allocated. */
  510. if (!starpu_data_test_if_allocated_on_node(handle, node))
  511. return 0;
  512. if (handle->ops->pointer_is_inside)
  513. {
  514. return handle->ops->pointer_is_inside(starpu_data_get_interface_on_node(handle, node), node, ptr);
  515. }
  516. /* Don't know :/ */
  517. return -1;
  518. }
  519. void *starpu_data_get_local_ptr(starpu_data_handle_t handle)
  520. {
  521. return starpu_data_handle_to_pointer(handle, starpu_worker_get_local_memory_node());
  522. }
  523. struct starpu_data_interface_ops* starpu_data_get_interface_ops(starpu_data_handle_t handle)
  524. {
  525. return handle->ops;
  526. }
  527. /*
  528. * Stop monitoring a piece of data
  529. */
  530. void _starpu_data_unregister_ram_pointer(starpu_data_handle_t handle, unsigned node)
  531. {
  532. if (starpu_node_get_kind(node) != STARPU_CPU_RAM)
  533. return;
  534. #ifdef STARPU_OPENMP
  535. if (handle->removed_from_context_hash)
  536. return;
  537. #endif
  538. const void *ram_ptr = starpu_data_handle_to_pointer(handle, node);
  539. if (ram_ptr != NULL)
  540. {
  541. /* Remove the PTR -> HANDLE mapping. If a mapping from PTR
  542. * to another handle existed before (e.g., when using
  543. * filters), it becomes visible again. */
  544. struct handle_entry *entry;
  545. #ifdef STARPU_OPENMP
  546. struct starpu_omp_task *task = _starpu_omp_get_task();
  547. if (task)
  548. {
  549. if (task->flags & STARPU_OMP_TASK_FLAGS_IMPLICIT)
  550. {
  551. struct starpu_omp_region *parallel_region = task->owner_region;
  552. _starpu_spin_lock(&parallel_region->registered_handles_lock);
  553. HASH_FIND_PTR(parallel_region->registered_handles, &ram_ptr, entry);
  554. STARPU_ASSERT(entry != NULL);
  555. HASH_DEL(registered_handles, entry);
  556. _starpu_spin_unlock(&parallel_region->registered_handles_lock);
  557. }
  558. else
  559. {
  560. HASH_FIND_PTR(task->registered_handles, &ram_ptr, entry);
  561. STARPU_ASSERT(entry != NULL);
  562. HASH_DEL(task->registered_handles, entry);
  563. }
  564. }
  565. else
  566. #endif
  567. {
  568. _starpu_spin_lock(&registered_handles_lock);
  569. HASH_FIND_PTR(registered_handles, &ram_ptr, entry);
  570. if (entry)
  571. {
  572. if (entry->handle == handle)
  573. {
  574. nregistered--;
  575. HASH_DEL(registered_handles, entry);
  576. }
  577. else
  578. /* don't free it, it's not ours */
  579. entry = NULL;
  580. }
  581. _starpu_spin_unlock(&registered_handles_lock);
  582. }
  583. free(entry);
  584. }
  585. }
  586. void _starpu_data_free_interfaces(starpu_data_handle_t handle)
  587. {
  588. unsigned node;
  589. unsigned nworkers = starpu_worker_get_count();
  590. for (node = 0; node < STARPU_MAXNODES; node++)
  591. free(handle->per_node[node].data_interface);
  592. if (handle->per_worker)
  593. {
  594. unsigned worker;
  595. for (worker = 0; worker < nworkers; worker++)
  596. free(handle->per_worker[worker].data_interface);
  597. free(handle->per_worker);
  598. }
  599. }
  600. struct _starpu_unregister_callback_arg
  601. {
  602. unsigned memory_node;
  603. starpu_data_handle_t handle;
  604. unsigned terminated;
  605. starpu_pthread_mutex_t mutex;
  606. starpu_pthread_cond_t cond;
  607. };
  608. /* Check whether we should tell starpu_data_unregister that the data handle is
  609. * not busy any more.
  610. * The header is supposed to be locked.
  611. * This may free the handle, if it was lazily unregistered (1 is returned in
  612. * that case). The handle pointer thus becomes invalid for the caller.
  613. *
  614. * Note: we inline some of the tests in the _starpu_data_check_not_busy macro.
  615. */
  616. int __starpu_data_check_not_busy(starpu_data_handle_t handle)
  617. {
  618. if (STARPU_LIKELY(handle->busy_count))
  619. return 0;
  620. /* Not busy any more, perhaps have to unregister etc. */
  621. if (STARPU_UNLIKELY(handle->busy_waiting))
  622. {
  623. STARPU_PTHREAD_MUTEX_LOCK(&handle->busy_mutex);
  624. STARPU_PTHREAD_COND_BROADCAST(&handle->busy_cond);
  625. STARPU_PTHREAD_MUTEX_UNLOCK(&handle->busy_mutex);
  626. }
  627. /* The handle has been destroyed in between (eg. this was a temporary
  628. * handle created for a reduction.) */
  629. if (STARPU_UNLIKELY(handle->lazy_unregister))
  630. {
  631. handle->lazy_unregister = 0;
  632. _starpu_spin_unlock(&handle->header_lock);
  633. _starpu_data_unregister(handle, 0, 1);
  634. /* Warning: in case we unregister the handle, we must be sure
  635. * that the caller will not try to unlock the header after
  636. * !*/
  637. return 1;
  638. }
  639. return 0;
  640. }
  641. static
  642. void _starpu_check_if_valid_and_fetch_data_on_node(starpu_data_handle_t handle, struct _starpu_data_replicate *replicate, const char *origin)
  643. {
  644. unsigned node;
  645. unsigned nnodes = starpu_memory_nodes_get_count();
  646. int valid = 0;
  647. for (node = 0; node < nnodes; node++)
  648. {
  649. if (handle->per_node[node].state != STARPU_INVALID)
  650. {
  651. /* we found a copy ! */
  652. valid = 1;
  653. }
  654. }
  655. if (valid)
  656. {
  657. int ret = _starpu_fetch_data_on_node(handle, handle->home_node, replicate, STARPU_R, 0, STARPU_FETCH, 0, NULL, NULL, 0, origin);
  658. STARPU_ASSERT(!ret);
  659. _starpu_release_data_on_node(handle, 0, STARPU_NONE, replicate);
  660. }
  661. else
  662. {
  663. _starpu_spin_lock(&handle->header_lock);
  664. if (!_starpu_notify_data_dependencies(handle, STARPU_NONE))
  665. _starpu_spin_unlock(&handle->header_lock);
  666. }
  667. }
  668. static void _starpu_data_unregister_fetch_data_callback(void *_arg)
  669. {
  670. struct _starpu_unregister_callback_arg *arg = (struct _starpu_unregister_callback_arg *) _arg;
  671. starpu_data_handle_t handle = arg->handle;
  672. STARPU_ASSERT(handle);
  673. struct _starpu_data_replicate *replicate = &handle->per_node[arg->memory_node];
  674. _starpu_check_if_valid_and_fetch_data_on_node(handle, replicate, "_starpu_data_unregister_fetch_data_callback");
  675. /* unlock the caller */
  676. STARPU_PTHREAD_MUTEX_LOCK(&arg->mutex);
  677. arg->terminated = 1;
  678. STARPU_PTHREAD_COND_SIGNAL(&arg->cond);
  679. STARPU_PTHREAD_MUTEX_UNLOCK(&arg->mutex);
  680. }
  681. void _starpu_data_set_unregister_hook(starpu_data_handle_t handle, _starpu_data_handle_unregister_hook func)
  682. {
  683. STARPU_ASSERT(handle->unregister_hook == NULL);
  684. handle->unregister_hook = func;
  685. }
  686. /*
  687. * We are about to unregister this R/O data. There might be still other aliases,
  688. * in which case this returns 0. If not, users are not supposed to see it
  689. * any more, so detach it from their sight and return 1 to let unregistration happen.
  690. */
  691. static int _starpu_ro_data_detach(starpu_data_handle_t handle)
  692. {
  693. _starpu_spin_lock(&handle->header_lock);
  694. if (handle->aliases)
  695. {
  696. handle->aliases--;
  697. _starpu_spin_unlock(&handle->header_lock);
  698. return 0;
  699. }
  700. if (handle->readonly_dup)
  701. {
  702. STARPU_ASSERT(handle->readonly_dup->readonly_dup_of == handle);
  703. handle->readonly_dup->readonly_dup_of = NULL;
  704. handle->readonly_dup = NULL;
  705. }
  706. if (handle->readonly_dup_of)
  707. {
  708. STARPU_ASSERT(handle->readonly_dup_of->readonly_dup == handle);
  709. handle->readonly_dup_of->readonly_dup = NULL;
  710. handle->readonly_dup_of = NULL;
  711. }
  712. /* So that unregistration can use write dependencies to wait for
  713. * anything to finish */
  714. handle->readonly = 0;
  715. _starpu_spin_unlock(&handle->header_lock);
  716. return 1;
  717. }
  718. /* Unregister the data handle, perhaps we don't need to update the home_node
  719. * (in that case coherent is set to 0)
  720. * nowait is for internal use when we already know for sure that we won't have to wait.
  721. */
  722. static void _starpu_data_unregister(starpu_data_handle_t handle, unsigned coherent, unsigned nowait)
  723. {
  724. STARPU_ASSERT(handle);
  725. STARPU_ASSERT_MSG(handle->nchildren == 0, "data %p needs to be unpartitioned before unregistration", handle);
  726. STARPU_ASSERT_MSG(handle->nplans == 0, "data %p needs its partition plans to be cleaned before unregistration", handle);
  727. STARPU_ASSERT_MSG(handle->partitioned == 0, "data %p needs its partitioned plans to be unpartitioned before unregistration", handle);
  728. /* TODO: also check that it has the latest coherency */
  729. STARPU_ASSERT(!(nowait && handle->busy_count != 0));
  730. if (!_starpu_ro_data_detach(handle))
  731. return;
  732. int sequential_consistency = handle->sequential_consistency;
  733. if (sequential_consistency && !nowait)
  734. {
  735. /* We will acquire it in write mode to catch all dependencies,
  736. * but possibly it's not actually initialized. Fake it to avoid
  737. getting caught doing it */
  738. handle->initialized = 1;
  739. STARPU_ASSERT_MSG(_starpu_worker_may_perform_blocking_calls(), "starpu_data_unregister must not be called from a task or callback, perhaps you can use starpu_data_unregister_submit instead");
  740. /* If sequential consistency is enabled, wait until data is available */
  741. _starpu_data_wait_until_available(handle, handle->readonly?STARPU_R:STARPU_RW, "starpu_data_unregister");
  742. }
  743. if (coherent && !nowait)
  744. {
  745. STARPU_ASSERT_MSG(_starpu_worker_may_perform_blocking_calls(), "starpu_data_unregister must not be called from a task or callback, perhaps you can use starpu_data_unregister_submit instead");
  746. /* Fetch data in the home of the data to ensure we have a valid copy
  747. * where we registered it */
  748. int home_node = handle->home_node;
  749. if (home_node >= 0)
  750. {
  751. struct _starpu_unregister_callback_arg arg = { 0 };
  752. arg.handle = handle;
  753. arg.memory_node = (unsigned)home_node;
  754. arg.terminated = 0;
  755. STARPU_PTHREAD_MUTEX_INIT0(&arg.mutex, NULL);
  756. STARPU_PTHREAD_COND_INIT0(&arg.cond, NULL);
  757. if (!_starpu_attempt_to_submit_data_request_from_apps(handle, STARPU_R,
  758. _starpu_data_unregister_fetch_data_callback, &arg))
  759. {
  760. /* no one has locked this data yet, so we proceed immediately */
  761. struct _starpu_data_replicate *home_replicate = &handle->per_node[home_node];
  762. _starpu_check_if_valid_and_fetch_data_on_node(handle, home_replicate, "_starpu_data_unregister");
  763. }
  764. else
  765. {
  766. STARPU_PTHREAD_MUTEX_LOCK(&arg.mutex);
  767. while (!arg.terminated)
  768. STARPU_PTHREAD_COND_WAIT(&arg.cond, &arg.mutex);
  769. STARPU_PTHREAD_MUTEX_UNLOCK(&arg.mutex);
  770. }
  771. STARPU_PTHREAD_MUTEX_DESTROY(&arg.mutex);
  772. STARPU_PTHREAD_COND_DESTROY(&arg.cond);
  773. }
  774. /* If this handle uses a multiformat interface, we may have to convert
  775. * this piece of data back into the CPU format.
  776. * XXX : This is quite hacky, could we submit a task instead ?
  777. */
  778. if (_starpu_data_is_multiformat_handle(handle) && (starpu_node_get_kind(handle->mf_node) != STARPU_CPU_RAM))
  779. {
  780. _STARPU_DEBUG("Conversion needed\n");
  781. void *buffers[1];
  782. struct starpu_multiformat_interface *format_interface;
  783. home_node = handle->home_node;
  784. if (home_node < 0 || (starpu_node_get_kind(home_node) != STARPU_CPU_RAM))
  785. home_node = STARPU_MAIN_RAM;
  786. format_interface = (struct starpu_multiformat_interface *) starpu_data_get_interface_on_node(handle, home_node);
  787. struct starpu_codelet *cl = NULL;
  788. enum starpu_node_kind node_kind = starpu_node_get_kind(handle->mf_node);
  789. switch (node_kind)
  790. {
  791. #ifdef STARPU_USE_CUDA
  792. case STARPU_CUDA_RAM:
  793. {
  794. struct starpu_multiformat_data_interface_ops *mf_ops;
  795. mf_ops = (struct starpu_multiformat_data_interface_ops *) handle->ops->get_mf_ops(format_interface);
  796. cl = mf_ops->cuda_to_cpu_cl;
  797. break;
  798. }
  799. #endif
  800. #ifdef STARPU_USE_OPENCL
  801. case STARPU_OPENCL_RAM:
  802. {
  803. struct starpu_multiformat_data_interface_ops *mf_ops;
  804. mf_ops = (struct starpu_multiformat_data_interface_ops *) handle->ops->get_mf_ops(format_interface);
  805. cl = mf_ops->opencl_to_cpu_cl;
  806. break;
  807. }
  808. #endif
  809. #ifdef STARPU_USE_MIC
  810. case STARPU_MIC_RAM:
  811. {
  812. struct starpu_multiformat_data_interface_ops *mf_ops;
  813. mf_ops = (struct starpu_multiformat_data_interface_ops *) handle->ops->get_mf_ops(format_interface);
  814. cl = mf_ops->mic_to_cpu_cl;
  815. break;
  816. }
  817. #endif
  818. case STARPU_CPU_RAM: /* Impossible ! */
  819. default:
  820. STARPU_ABORT();
  821. }
  822. buffers[0] = format_interface;
  823. _starpu_cl_func_t func = _starpu_task_get_cpu_nth_implementation(cl, 0);
  824. STARPU_ASSERT(func);
  825. func(buffers, NULL);
  826. }
  827. }
  828. /* Prevent any further unregistration */
  829. handle->magic = 0;
  830. _starpu_spin_lock(&handle->header_lock);
  831. if (!coherent)
  832. {
  833. /* Should we postpone the unregister operation ? */
  834. if ((handle->busy_count > 0) && handle->lazy_unregister)
  835. {
  836. _starpu_spin_unlock(&handle->header_lock);
  837. return;
  838. }
  839. }
  840. /* Tell holders of references that we're starting waiting */
  841. handle->busy_waiting = 1;
  842. _starpu_spin_unlock(&handle->header_lock);
  843. retry_busy:
  844. /* Wait for all requests to finish (notably WT requests) */
  845. STARPU_PTHREAD_MUTEX_LOCK(&handle->busy_mutex);
  846. while (1)
  847. {
  848. /* Here helgrind would shout that this an unprotected access,
  849. * but this is actually fine: all threads who do busy_count--
  850. * are supposed to call _starpu_data_check_not_busy, which will
  851. * wake us up through the busy_mutex/busy_cond. */
  852. if (!handle->busy_count)
  853. break;
  854. /* This is woken by _starpu_data_check_not_busy, always called
  855. * after decrementing busy_count */
  856. STARPU_PTHREAD_COND_WAIT(&handle->busy_cond, &handle->busy_mutex);
  857. }
  858. STARPU_PTHREAD_MUTEX_UNLOCK(&handle->busy_mutex);
  859. /* Unregister MPI things after having waiting for MPI reqs etc. to settle down */
  860. if (handle->unregister_hook)
  861. {
  862. handle->unregister_hook(handle);
  863. handle->unregister_hook = NULL;
  864. }
  865. /* Wait for finished requests to release the handle */
  866. _starpu_spin_lock(&handle->header_lock);
  867. if (handle->busy_count)
  868. {
  869. /* Bad luck: some request went in in between, wait again... */
  870. _starpu_spin_unlock(&handle->header_lock);
  871. goto retry_busy;
  872. }
  873. size_t size = _starpu_data_get_alloc_size(handle);
  874. /* Destroy the data now */
  875. unsigned node;
  876. for (node = 0; node < STARPU_MAXNODES; node++)
  877. {
  878. struct _starpu_data_replicate *local = &handle->per_node[node];
  879. STARPU_ASSERT(!local->refcnt);
  880. if (local->allocated)
  881. {
  882. _starpu_data_unregister_ram_pointer(handle, node);
  883. /* free the data copy in a lazy fashion */
  884. if (local->automatically_allocated)
  885. _starpu_request_mem_chunk_removal(handle, local, node, size);
  886. }
  887. }
  888. if (handle->per_worker)
  889. {
  890. unsigned worker;
  891. unsigned nworkers = starpu_worker_get_count();
  892. for (worker = 0; worker < nworkers; worker++)
  893. {
  894. struct _starpu_data_replicate *local = &handle->per_worker[worker];
  895. STARPU_ASSERT(!local->refcnt);
  896. /* free the data copy in a lazy fashion */
  897. if (local->allocated && local->automatically_allocated)
  898. _starpu_request_mem_chunk_removal(handle, local, starpu_worker_get_memory_node(worker), size);
  899. }
  900. }
  901. _starpu_data_free_interfaces(handle);
  902. _starpu_memory_stats_free(handle);
  903. _starpu_spin_unlock(&handle->header_lock);
  904. _starpu_spin_destroy(&handle->header_lock);
  905. _starpu_data_clear_implicit(handle);
  906. free(handle->active_readonly_children);
  907. free(handle->active_readonly_nchildren);
  908. STARPU_PTHREAD_MUTEX_DESTROY(&handle->busy_mutex);
  909. STARPU_PTHREAD_COND_DESTROY(&handle->busy_cond);
  910. STARPU_PTHREAD_MUTEX_DESTROY(&handle->sequential_consistency_mutex);
  911. STARPU_HG_ENABLE_CHECKING(handle->post_sync_tasks_cnt);
  912. STARPU_HG_ENABLE_CHECKING(handle->busy_count);
  913. _starpu_data_requester_prio_list_deinit(&handle->req_list);
  914. _starpu_data_requester_prio_list_deinit(&handle->reduction_req_list);
  915. if (handle->switch_cl)
  916. {
  917. free(handle->switch_cl->dyn_nodes);
  918. free(handle->switch_cl);
  919. }
  920. _STARPU_TRACE_HANDLE_DATA_UNREGISTER(handle);
  921. free(handle);
  922. }
  923. void starpu_data_unregister(starpu_data_handle_t handle)
  924. {
  925. STARPU_ASSERT_MSG(handle->magic == 42, "data %p is invalid (was it already registered?)", handle);
  926. STARPU_ASSERT_MSG(!handle->lazy_unregister, "data %p can not be unregistered twice", handle);
  927. _starpu_data_unregister(handle, 1, 0);
  928. }
  929. void starpu_data_unregister_no_coherency(starpu_data_handle_t handle)
  930. {
  931. STARPU_ASSERT_MSG(handle->magic == 42, "data %p is invalid (was it already registered?)", handle);
  932. _starpu_data_unregister(handle, 0, 0);
  933. }
  934. static void _starpu_data_unregister_submit_cb(void *arg)
  935. {
  936. starpu_data_handle_t handle = arg;
  937. _starpu_spin_lock(&handle->header_lock);
  938. handle->lazy_unregister = 1;
  939. /* The handle should be busy since we are working on it.
  940. * when we releases the handle below, it will be destroyed by
  941. * _starpu_data_check_not_busy */
  942. STARPU_ASSERT(handle->busy_count);
  943. _starpu_spin_unlock(&handle->header_lock);
  944. starpu_data_release_on_node(handle, STARPU_ACQUIRE_NO_NODE_LOCK_ALL);
  945. }
  946. void starpu_data_unregister_submit(starpu_data_handle_t handle)
  947. {
  948. STARPU_ASSERT_MSG(handle->magic == 42, "data %p is invalid (was it already registered?)", handle);
  949. STARPU_ASSERT_MSG(!handle->lazy_unregister, "data %p can not be unregistered twice", handle);
  950. if (!_starpu_ro_data_detach(handle))
  951. return;
  952. /* Wait for all task dependencies on this handle before putting it for free */
  953. starpu_data_acquire_on_node_cb(handle, STARPU_ACQUIRE_NO_NODE_LOCK_ALL, handle->initialized?STARPU_RW:STARPU_W, _starpu_data_unregister_submit_cb, handle);
  954. }
  955. static void _starpu_data_invalidate(void *data)
  956. {
  957. starpu_data_handle_t handle = data;
  958. size_t size = _starpu_data_get_alloc_size(handle);
  959. _starpu_spin_lock(&handle->header_lock);
  960. //_STARPU_DEBUG("Really invalidating data %p\n", data);
  961. #ifdef STARPU_DEBUG
  962. {
  963. /* There shouldn't be any pending request since we acquired the data in W mode */
  964. unsigned i, j, nnodes = starpu_memory_nodes_get_count();
  965. for (i = 0; i < nnodes; i++)
  966. for (j = 0; j < nnodes; j++)
  967. STARPU_ASSERT_MSG(!handle->per_node[i].request[j], "request for handle %p pending from %u to %u while invalidating data!", handle, j, i);
  968. }
  969. #endif
  970. unsigned node;
  971. for (node = 0; node < STARPU_MAXNODES; node++)
  972. {
  973. struct _starpu_data_replicate *local = &handle->per_node[node];
  974. if (local->mc && local->allocated && local->automatically_allocated)
  975. {
  976. _starpu_data_unregister_ram_pointer(handle, node);
  977. /* free the data copy in a lazy fashion */
  978. _starpu_request_mem_chunk_removal(handle, local, node, size);
  979. }
  980. if (local->state != STARPU_INVALID)
  981. _STARPU_TRACE_DATA_STATE_INVALID(handle, node);
  982. local->state = STARPU_INVALID;
  983. }
  984. if (handle->per_worker)
  985. {
  986. unsigned worker;
  987. unsigned nworkers = starpu_worker_get_count();
  988. for (worker = 0; worker < nworkers; worker++)
  989. {
  990. struct _starpu_data_replicate *local = &handle->per_worker[worker];
  991. if (local->mc && local->allocated && local->automatically_allocated)
  992. /* free the data copy in a lazy fashion */
  993. _starpu_request_mem_chunk_removal(handle, local, starpu_worker_get_memory_node(worker), size);
  994. local->state = STARPU_INVALID;
  995. }
  996. }
  997. _starpu_spin_unlock(&handle->header_lock);
  998. starpu_data_release_on_node(handle, STARPU_ACQUIRE_NO_NODE_LOCK_ALL);
  999. }
  1000. void starpu_data_invalidate(starpu_data_handle_t handle)
  1001. {
  1002. STARPU_ASSERT(handle);
  1003. starpu_data_acquire_on_node(handle, STARPU_ACQUIRE_NO_NODE_LOCK_ALL, STARPU_W);
  1004. _starpu_data_invalidate(handle);
  1005. handle->initialized = 0;
  1006. }
  1007. void starpu_data_invalidate_submit(starpu_data_handle_t handle)
  1008. {
  1009. STARPU_ASSERT(handle);
  1010. starpu_data_acquire_on_node_cb(handle, STARPU_ACQUIRE_NO_NODE_LOCK_ALL, STARPU_W, _starpu_data_invalidate, handle);
  1011. handle->initialized = 0;
  1012. }
  1013. enum starpu_data_interface_id starpu_data_get_interface_id(starpu_data_handle_t handle)
  1014. {
  1015. return handle->ops->interfaceid;
  1016. }
  1017. void *starpu_data_get_interface_on_node(starpu_data_handle_t handle, unsigned memory_node)
  1018. {
  1019. return handle->per_node[memory_node].data_interface;
  1020. }
  1021. int starpu_data_interface_get_next_id(void)
  1022. {
  1023. _data_interface_number += 1;
  1024. return _data_interface_number-1;
  1025. }
  1026. int starpu_data_pack_node(starpu_data_handle_t handle, unsigned node, void **ptr, starpu_ssize_t *count)
  1027. {
  1028. STARPU_ASSERT_MSG(handle->ops->pack_data, "The datatype interface %s (%d) does not have a pack operation", handle->ops->name, handle->ops->interfaceid);
  1029. return handle->ops->pack_data(handle, node, ptr, count);
  1030. }
  1031. int starpu_data_pack(starpu_data_handle_t handle, void **ptr, starpu_ssize_t *count)
  1032. {
  1033. return starpu_data_pack_node(handle, starpu_worker_get_local_memory_node(), ptr, count);
  1034. }
  1035. int starpu_data_peek_node(starpu_data_handle_t handle, unsigned node, void *ptr, size_t count)
  1036. {
  1037. STARPU_ASSERT_MSG(handle->ops->peek_data, "The datatype interface %s (%d) does not have a peek operation", handle->ops->name, handle->ops->interfaceid);
  1038. int ret;
  1039. ret = handle->ops->peek_data(handle, node, ptr, count);
  1040. return ret;
  1041. }
  1042. int starpu_data_peek(starpu_data_handle_t handle, void *ptr, size_t count)
  1043. {
  1044. return starpu_data_peek_node(handle, starpu_worker_get_local_memory_node(), ptr, count);
  1045. }
  1046. int starpu_data_unpack_node(starpu_data_handle_t handle, unsigned node, void *ptr, size_t count)
  1047. {
  1048. STARPU_ASSERT_MSG(handle->ops->unpack_data, "The datatype interface %s (%d) does not have an unpack operation", handle->ops->name, handle->ops->interfaceid);
  1049. int ret;
  1050. ret = handle->ops->unpack_data(handle, node, ptr, count);
  1051. return ret;
  1052. }
  1053. int starpu_data_unpack(starpu_data_handle_t handle, void *ptr, size_t count)
  1054. {
  1055. return starpu_data_unpack_node(handle, starpu_worker_get_local_memory_node(), ptr, count);
  1056. }
  1057. size_t starpu_data_get_size(starpu_data_handle_t handle)
  1058. {
  1059. return handle->ops->get_size(handle);
  1060. }
  1061. size_t starpu_data_get_alloc_size(starpu_data_handle_t handle)
  1062. {
  1063. if (handle->ops->get_alloc_size)
  1064. return handle->ops->get_alloc_size(handle);
  1065. else
  1066. return handle->ops->get_size(handle);
  1067. }
  1068. void starpu_data_set_name(starpu_data_handle_t handle STARPU_ATTRIBUTE_UNUSED, const char *name STARPU_ATTRIBUTE_UNUSED)
  1069. {
  1070. _STARPU_TRACE_DATA_NAME(handle, name);
  1071. }
  1072. int starpu_data_get_home_node(starpu_data_handle_t handle)
  1073. {
  1074. return handle->home_node;
  1075. }
  1076. void starpu_data_set_coordinates_array(starpu_data_handle_t handle, unsigned dimensions, int dims[])
  1077. {
  1078. unsigned i;
  1079. unsigned max_dimensions = sizeof(handle->coordinates)/sizeof(handle->coordinates[0]);
  1080. if (dimensions > max_dimensions)
  1081. dimensions = max_dimensions;
  1082. handle->dimensions = dimensions;
  1083. for (i = 0; i < dimensions; i++)
  1084. handle->coordinates[i] = dims[i];
  1085. _STARPU_TRACE_DATA_COORDINATES(handle, dimensions, dims);
  1086. }
  1087. void starpu_data_set_coordinates(starpu_data_handle_t handle, unsigned dimensions, ...)
  1088. {
  1089. int dims[dimensions];
  1090. unsigned i;
  1091. va_list varg_list;
  1092. va_start(varg_list, dimensions);
  1093. for (i = 0; i < dimensions; i++)
  1094. dims[i] = va_arg(varg_list, int);
  1095. va_end(varg_list);
  1096. starpu_data_set_coordinates_array(handle, dimensions, dims);
  1097. }
  1098. unsigned starpu_data_get_coordinates_array(starpu_data_handle_t handle, unsigned dimensions, int dims[])
  1099. {
  1100. unsigned i;
  1101. if (dimensions > handle->dimensions)
  1102. dimensions = handle->dimensions;
  1103. for (i = 0; i < dimensions; i++)
  1104. dims[i] = handle->coordinates[i];
  1105. return dimensions;
  1106. }
  1107. void starpu_data_print(starpu_data_handle_t handle, unsigned node, FILE *stream)
  1108. {
  1109. if (handle->ops == NULL)
  1110. fprintf(stream, "Undefined");
  1111. else
  1112. {
  1113. switch (handle->ops->interfaceid)
  1114. {
  1115. case(STARPU_MATRIX_INTERFACE_ID):
  1116. fprintf(stream, "Matrix");
  1117. break;
  1118. case(STARPU_BLOCK_INTERFACE_ID):
  1119. fprintf(stream, "Block");
  1120. break;
  1121. case(STARPU_VECTOR_INTERFACE_ID):
  1122. fprintf(stream, "Vector");
  1123. break;
  1124. case(STARPU_CSR_INTERFACE_ID):
  1125. fprintf(stream, "CSR");
  1126. break;
  1127. case(STARPU_BCSR_INTERFACE_ID):
  1128. fprintf(stream, "BCSR");
  1129. break;
  1130. case(STARPU_VARIABLE_INTERFACE_ID):
  1131. fprintf(stream, "Variable");
  1132. break;
  1133. case(STARPU_VOID_INTERFACE_ID):
  1134. fprintf(stream, "Void");
  1135. break;
  1136. case(STARPU_MULTIFORMAT_INTERFACE_ID):
  1137. fprintf(stream, "Multfiformat");
  1138. break;
  1139. case(STARPU_COO_INTERFACE_ID):
  1140. fprintf(stream, "COO");
  1141. break;
  1142. case(STARPU_TENSOR_INTERFACE_ID):
  1143. fprintf(stream, "Tensor");
  1144. break;
  1145. case(STARPU_UNKNOWN_INTERFACE_ID ):
  1146. fprintf(stream, "UNKNOWN");
  1147. break;
  1148. default:
  1149. fprintf(stream, "User interface with id %d", handle->ops->interfaceid);
  1150. break;
  1151. }
  1152. }
  1153. void *data_interface = NULL;
  1154. if (starpu_data_test_if_allocated_on_node(handle, node))
  1155. data_interface = starpu_data_get_interface_on_node(handle, node);
  1156. if (starpu_data_test_if_allocated_on_node(handle, handle->home_node))
  1157. data_interface = starpu_data_get_interface_on_node(handle, handle->home_node);
  1158. if (handle->ops && handle->ops->describe && data_interface)
  1159. {
  1160. char buffer[1024];
  1161. handle->ops->describe(data_interface, buffer, sizeof(buffer));
  1162. fprintf(stream, " %s\n", buffer);
  1163. }
  1164. else
  1165. fprintf(stream, "\n");
  1166. }