xlu_kernels.c 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723
  1. /* StarPU --- Runtime system for heterogeneous multicore architectures.
  2. *
  3. * Copyright (C) 2009, 2010-2012, 2014-2017 Université de Bordeaux
  4. * Copyright (C) 2010, 2011, 2012, 2015 CNRS
  5. *
  6. * StarPU is free software; you can redistribute it and/or modify
  7. * it under the terms of the GNU Lesser General Public License as published by
  8. * the Free Software Foundation; either version 2.1 of the License, or (at
  9. * your option) any later version.
  10. *
  11. * StarPU is distributed in the hope that it will be useful, but
  12. * WITHOUT ANY WARRANTY; without even the implied warranty of
  13. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
  14. *
  15. * See the GNU Lesser General Public License in COPYING.LGPL for more details.
  16. */
  17. /* LU Kernels */
  18. #include "xlu.h"
  19. #include <math.h>
  20. #include <complex.h>
  21. #ifdef STARPU_USE_CUDA
  22. #include <cublas.h>
  23. #include <starpu_cublas_v2.h>
  24. #endif
  25. #define str(s) #s
  26. #define xstr(s) str(s)
  27. #define STARPU_LU_STR(name) xstr(STARPU_LU(name))
  28. #ifdef STARPU_USE_CUDA
  29. static const TYPE p1 = 1.0f;
  30. static const TYPE m1 = -1.0f;
  31. #endif
  32. /*
  33. * U22
  34. */
  35. static inline void STARPU_LU(common_u22)(void *descr[],
  36. int s, STARPU_ATTRIBUTE_UNUSED void *_args)
  37. {
  38. TYPE *right = (TYPE *)STARPU_MATRIX_GET_PTR(descr[0]);
  39. TYPE *left = (TYPE *)STARPU_MATRIX_GET_PTR(descr[1]);
  40. TYPE *center = (TYPE *)STARPU_MATRIX_GET_PTR(descr[2]);
  41. unsigned dx = STARPU_MATRIX_GET_NX(descr[2]);
  42. unsigned dy = STARPU_MATRIX_GET_NY(descr[2]);
  43. unsigned dz = STARPU_MATRIX_GET_NY(descr[0]);
  44. unsigned ld12 = STARPU_MATRIX_GET_LD(descr[0]);
  45. unsigned ld21 = STARPU_MATRIX_GET_LD(descr[1]);
  46. unsigned ld22 = STARPU_MATRIX_GET_LD(descr[2]);
  47. #ifdef STARPU_USE_CUDA
  48. cublasStatus status;
  49. cudaError_t cures;
  50. #endif
  51. switch (s)
  52. {
  53. case 0:
  54. CPU_GEMM("N", "N", dy, dx, dz,
  55. (TYPE)-1.0, right, ld21, left, ld12,
  56. (TYPE)1.0, center, ld22);
  57. break;
  58. #ifdef STARPU_USE_CUDA
  59. case 1:
  60. {
  61. status = CUBLAS_GEMM(starpu_cublas_get_local_handle(),
  62. CUBLAS_OP_N, CUBLAS_OP_N, dx, dy, dz,
  63. (CUBLAS_TYPE *)&m1, (CUBLAS_TYPE *)right, ld21, (CUBLAS_TYPE *)left, ld12,
  64. (CUBLAS_TYPE *)&p1, (CUBLAS_TYPE *)center, ld22);
  65. if (STARPU_UNLIKELY(status != CUBLAS_STATUS_SUCCESS))
  66. STARPU_CUBLAS_REPORT_ERROR(status);
  67. break;
  68. }
  69. #endif
  70. default:
  71. STARPU_ABORT();
  72. break;
  73. }
  74. }
  75. void STARPU_LU(cpu_u22)(void *descr[], void *_args)
  76. {
  77. STARPU_LU(common_u22)(descr, 0, _args);
  78. }
  79. #ifdef STARPU_USE_CUDA
  80. void STARPU_LU(cublas_u22)(void *descr[], void *_args)
  81. {
  82. STARPU_LU(common_u22)(descr, 1, _args);
  83. }
  84. #endif /* STARPU_USE_CUDA */
  85. static struct starpu_perfmodel STARPU_LU(model_22) =
  86. {
  87. .type = STARPU_HISTORY_BASED,
  88. #ifdef STARPU_ATLAS
  89. .symbol = STARPU_LU_STR(lu_model_22_atlas)
  90. #elif defined(STARPU_GOTO)
  91. .symbol = STARPU_LU_STR(lu_model_22_goto)
  92. #else
  93. .symbol = STARPU_LU_STR(lu_model_22)
  94. #endif
  95. };
  96. #ifdef STARPU_USE_CUDA
  97. static int can_execute(unsigned workerid, struct starpu_task *task, unsigned nimpl)
  98. {
  99. enum starpu_worker_archtype type = starpu_worker_get_type(workerid);
  100. if (type == STARPU_CPU_WORKER || type == STARPU_MIC_WORKER || type == STARPU_SCC_WORKER)
  101. return 1;
  102. #ifdef STARPU_SIMGRID
  103. /* We don't know, let's assume it can */
  104. return 1;
  105. #else
  106. /* Cuda device */
  107. const struct cudaDeviceProp *props;
  108. props = starpu_cuda_get_device_properties(workerid);
  109. if (props->major >= 2 || props->minor >= 3)
  110. {
  111. /* At least compute capability 1.3, supports doubles */
  112. return 1;
  113. }
  114. else
  115. {
  116. /* Old card does not support doubles */
  117. return 0;
  118. }
  119. #endif
  120. }
  121. #endif
  122. #define STRINGIFY_(x) #x
  123. #define STRINGIFY(x) STRINGIFY_(x)
  124. struct starpu_codelet cl22 =
  125. {
  126. .cpu_funcs = {STARPU_LU(cpu_u22)},
  127. .cpu_funcs_name = {STRINGIFY(STARPU_LU(cpu_u22))},
  128. #ifdef STARPU_USE_CUDA
  129. .cuda_funcs = {STARPU_LU(cublas_u22)},
  130. CAN_EXECUTE
  131. #elif defined(STARPU_SIMGRID)
  132. .cuda_funcs = {(void*)1},
  133. #endif
  134. .cuda_flags = {STARPU_CUDA_ASYNC},
  135. .nbuffers = 3,
  136. .modes = {STARPU_R, STARPU_R, STARPU_RW},
  137. .model = &STARPU_LU(model_22)
  138. };
  139. /*
  140. * U12
  141. */
  142. static inline void STARPU_LU(common_u12)(void *descr[],
  143. int s, STARPU_ATTRIBUTE_UNUSED void *_args)
  144. {
  145. TYPE *sub11;
  146. TYPE *sub12;
  147. sub11 = (TYPE *)STARPU_MATRIX_GET_PTR(descr[0]);
  148. sub12 = (TYPE *)STARPU_MATRIX_GET_PTR(descr[1]);
  149. unsigned ld11 = STARPU_MATRIX_GET_LD(descr[0]);
  150. unsigned ld12 = STARPU_MATRIX_GET_LD(descr[1]);
  151. unsigned nx12 = STARPU_MATRIX_GET_NX(descr[1]);
  152. unsigned ny12 = STARPU_MATRIX_GET_NY(descr[1]);
  153. #ifdef STARPU_USE_CUDA
  154. cublasStatus status;
  155. cudaError_t cures;
  156. #endif
  157. /* solve L11 U12 = A12 (find U12) */
  158. switch (s)
  159. {
  160. case 0:
  161. CPU_TRSM("L", "L", "N", "N", nx12, ny12,
  162. (TYPE)1.0, sub11, ld11, sub12, ld12);
  163. break;
  164. #ifdef STARPU_USE_CUDA
  165. case 1:
  166. status = CUBLAS_TRSM(starpu_cublas_get_local_handle(),
  167. CUBLAS_SIDE_LEFT, CUBLAS_FILL_MODE_LOWER, CUBLAS_OP_N, CUBLAS_DIAG_NON_UNIT,
  168. ny12, nx12,
  169. (CUBLAS_TYPE*)&p1, (CUBLAS_TYPE*)sub11, ld11, (CUBLAS_TYPE*)sub12, ld12);
  170. if (STARPU_UNLIKELY(status != CUBLAS_STATUS_SUCCESS))
  171. STARPU_CUBLAS_REPORT_ERROR(status);
  172. break;
  173. #endif
  174. default:
  175. STARPU_ABORT();
  176. break;
  177. }
  178. }
  179. void STARPU_LU(cpu_u12)(void *descr[], void *_args)
  180. {
  181. STARPU_LU(common_u12)(descr, 0, _args);
  182. }
  183. #ifdef STARPU_USE_CUDA
  184. void STARPU_LU(cublas_u12)(void *descr[], void *_args)
  185. {
  186. STARPU_LU(common_u12)(descr, 1, _args);
  187. }
  188. #endif /* STARPU_USE_CUDA */
  189. static struct starpu_perfmodel STARPU_LU(model_12) =
  190. {
  191. .type = STARPU_HISTORY_BASED,
  192. #ifdef STARPU_ATLAS
  193. .symbol = STARPU_LU_STR(lu_model_12_atlas)
  194. #elif defined(STARPU_GOTO)
  195. .symbol = STARPU_LU_STR(lu_model_12_goto)
  196. #else
  197. .symbol = STARPU_LU_STR(lu_model_12)
  198. #endif
  199. };
  200. struct starpu_codelet cl12 =
  201. {
  202. .cpu_funcs = {STARPU_LU(cpu_u12)},
  203. .cpu_funcs_name = {STRINGIFY(STARPU_LU(cpu_u12))},
  204. #ifdef STARPU_USE_CUDA
  205. .cuda_funcs = {STARPU_LU(cublas_u12)},
  206. CAN_EXECUTE
  207. #elif defined(STARPU_SIMGRID)
  208. .cuda_funcs = {(void*)1},
  209. #endif
  210. .cuda_flags = {STARPU_CUDA_ASYNC},
  211. .nbuffers = 2,
  212. .modes = {STARPU_R, STARPU_RW},
  213. .model = &STARPU_LU(model_12)
  214. };
  215. /*
  216. * U21
  217. */
  218. static inline void STARPU_LU(common_u21)(void *descr[],
  219. int s, STARPU_ATTRIBUTE_UNUSED void *_args)
  220. {
  221. TYPE *sub11;
  222. TYPE *sub21;
  223. sub11 = (TYPE *)STARPU_MATRIX_GET_PTR(descr[0]);
  224. sub21 = (TYPE *)STARPU_MATRIX_GET_PTR(descr[1]);
  225. unsigned ld11 = STARPU_MATRIX_GET_LD(descr[0]);
  226. unsigned ld21 = STARPU_MATRIX_GET_LD(descr[1]);
  227. unsigned nx21 = STARPU_MATRIX_GET_NX(descr[1]);
  228. unsigned ny21 = STARPU_MATRIX_GET_NY(descr[1]);
  229. #ifdef STARPU_USE_CUDA
  230. cublasStatus status;
  231. #endif
  232. switch (s)
  233. {
  234. case 0:
  235. CPU_TRSM("R", "U", "N", "U", nx21, ny21,
  236. (TYPE)1.0, sub11, ld11, sub21, ld21);
  237. break;
  238. #ifdef STARPU_USE_CUDA
  239. case 1:
  240. status = CUBLAS_TRSM(starpu_cublas_get_local_handle(),
  241. CUBLAS_SIDE_RIGHT, CUBLAS_FILL_MODE_UPPER, CUBLAS_OP_N, CUBLAS_DIAG_UNIT,
  242. ny21, nx21,
  243. (CUBLAS_TYPE*)&p1, (CUBLAS_TYPE*)sub11, ld11, (CUBLAS_TYPE*)sub21, ld21);
  244. if (status != CUBLAS_STATUS_SUCCESS)
  245. STARPU_CUBLAS_REPORT_ERROR(status);
  246. break;
  247. #endif
  248. default:
  249. STARPU_ABORT();
  250. break;
  251. }
  252. }
  253. void STARPU_LU(cpu_u21)(void *descr[], void *_args)
  254. {
  255. STARPU_LU(common_u21)(descr, 0, _args);
  256. }
  257. #ifdef STARPU_USE_CUDA
  258. void STARPU_LU(cublas_u21)(void *descr[], void *_args)
  259. {
  260. STARPU_LU(common_u21)(descr, 1, _args);
  261. }
  262. #endif
  263. static struct starpu_perfmodel STARPU_LU(model_21) =
  264. {
  265. .type = STARPU_HISTORY_BASED,
  266. #ifdef STARPU_ATLAS
  267. .symbol = STARPU_LU_STR(lu_model_21_atlas)
  268. #elif defined(STARPU_GOTO)
  269. .symbol = STARPU_LU_STR(lu_model_21_goto)
  270. #else
  271. .symbol = STARPU_LU_STR(lu_model_21)
  272. #endif
  273. };
  274. struct starpu_codelet cl21 =
  275. {
  276. .cpu_funcs = {STARPU_LU(cpu_u21)},
  277. .cpu_funcs_name = {STRINGIFY(STARPU_LU(cpu_u21))},
  278. #ifdef STARPU_USE_CUDA
  279. .cuda_funcs = {STARPU_LU(cublas_u21)},
  280. CAN_EXECUTE
  281. #elif defined(STARPU_SIMGRID)
  282. .cuda_funcs = {(void*)1},
  283. #endif
  284. .cuda_flags = {STARPU_CUDA_ASYNC},
  285. .nbuffers = 2,
  286. .modes = {STARPU_R, STARPU_RW},
  287. .model = &STARPU_LU(model_21)
  288. };
  289. /*
  290. * U11
  291. */
  292. static inline void STARPU_LU(common_u11)(void *descr[],
  293. int s, STARPU_ATTRIBUTE_UNUSED void *_args)
  294. {
  295. TYPE *sub11;
  296. sub11 = (TYPE *)STARPU_MATRIX_GET_PTR(descr[0]);
  297. unsigned long nx = STARPU_MATRIX_GET_NX(descr[0]);
  298. unsigned long ld = STARPU_MATRIX_GET_LD(descr[0]);
  299. unsigned long z;
  300. #ifdef STARPU_USE_CUDA
  301. cublasStatus status;
  302. cublasHandle_t handle;
  303. cudaStream_t stream;
  304. #endif
  305. switch (s)
  306. {
  307. case 0:
  308. for (z = 0; z < nx; z++)
  309. {
  310. TYPE pivot;
  311. pivot = sub11[z+z*ld];
  312. STARPU_ASSERT(!ISZERO(pivot));
  313. CPU_SCAL(nx - z - 1, (1.0/pivot), &sub11[z+(z+1)*ld], ld);
  314. CPU_GER(nx - z - 1, nx - z - 1, -1.0,
  315. &sub11[(z+1)+z*ld], 1,
  316. &sub11[z+(z+1)*ld], ld,
  317. &sub11[(z+1) + (z+1)*ld],ld);
  318. }
  319. break;
  320. #ifdef STARPU_USE_CUDA
  321. case 1:
  322. handle = starpu_cublas_get_local_handle();
  323. stream = starpu_cuda_get_local_stream();
  324. for (z = 0; z < nx; z++)
  325. {
  326. TYPE pivot;
  327. TYPE inv_pivot;
  328. cudaMemcpyAsync(&pivot, &sub11[z+z*ld], sizeof(TYPE), cudaMemcpyDeviceToHost, stream);
  329. cudaStreamSynchronize(stream);
  330. STARPU_ASSERT(!ISZERO(pivot));
  331. inv_pivot = 1.0/pivot;
  332. status = CUBLAS_SCAL(handle,
  333. nx - z - 1,
  334. (CUBLAS_TYPE*)&inv_pivot, (CUBLAS_TYPE*)&sub11[z+(z+1)*ld], ld);
  335. if (status != CUBLAS_STATUS_SUCCESS)
  336. STARPU_CUBLAS_REPORT_ERROR(status);
  337. status = CUBLAS_GER(handle,
  338. nx - z - 1, nx - z - 1,
  339. (CUBLAS_TYPE*)&m1,
  340. (CUBLAS_TYPE*)&sub11[(z+1)+z*ld], 1,
  341. (CUBLAS_TYPE*)&sub11[z+(z+1)*ld], ld,
  342. (CUBLAS_TYPE*)&sub11[(z+1) + (z+1)*ld],ld);
  343. if (status != CUBLAS_STATUS_SUCCESS)
  344. STARPU_CUBLAS_REPORT_ERROR(status);
  345. }
  346. cudaStreamSynchronize(stream);
  347. break;
  348. #endif
  349. default:
  350. STARPU_ABORT();
  351. break;
  352. }
  353. }
  354. void STARPU_LU(cpu_u11)(void *descr[], void *_args)
  355. {
  356. STARPU_LU(common_u11)(descr, 0, _args);
  357. }
  358. #ifdef STARPU_USE_CUDA
  359. void STARPU_LU(cublas_u11)(void *descr[], void *_args)
  360. {
  361. STARPU_LU(common_u11)(descr, 1, _args);
  362. }
  363. #endif /* STARPU_USE_CUDA */
  364. static struct starpu_perfmodel STARPU_LU(model_11) =
  365. {
  366. .type = STARPU_HISTORY_BASED,
  367. #ifdef STARPU_ATLAS
  368. .symbol = STARPU_LU_STR(lu_model_11_atlas)
  369. #elif defined(STARPU_GOTO)
  370. .symbol = STARPU_LU_STR(lu_model_11_goto)
  371. #else
  372. .symbol = STARPU_LU_STR(lu_model_11)
  373. #endif
  374. };
  375. struct starpu_codelet cl11 =
  376. {
  377. .cpu_funcs = {STARPU_LU(cpu_u11)},
  378. .cpu_funcs_name = {STRINGIFY(STARPU_LU(cpu_u11))},
  379. #ifdef STARPU_USE_CUDA
  380. .cuda_funcs = {STARPU_LU(cublas_u11)},
  381. CAN_EXECUTE
  382. #elif defined(STARPU_SIMGRID)
  383. .cuda_funcs = {(void*)1},
  384. #endif
  385. .nbuffers = 1,
  386. .modes = {STARPU_RW},
  387. .model = &STARPU_LU(model_11)
  388. };
  389. /*
  390. * U11 with pivoting
  391. */
  392. static inline void STARPU_LU(common_u11_pivot)(void *descr[],
  393. int s, void *_args)
  394. {
  395. TYPE *sub11;
  396. sub11 = (TYPE *)STARPU_MATRIX_GET_PTR(descr[0]);
  397. unsigned long nx = STARPU_MATRIX_GET_NX(descr[0]);
  398. unsigned long ld = STARPU_MATRIX_GET_LD(descr[0]);
  399. unsigned long z;
  400. struct piv_s *piv = _args;
  401. unsigned *ipiv = piv->piv;
  402. unsigned first = piv->first;
  403. #ifdef STARPU_USE_CUDA
  404. cublasStatus status;
  405. cublasHandle_t handle;
  406. cudaStream_t stream;
  407. #endif
  408. switch (s)
  409. {
  410. case 0:
  411. for (z = 0; z < nx; z++)
  412. {
  413. TYPE pivot;
  414. pivot = sub11[z+z*ld];
  415. if (fabs((double)(pivot)) < PIVOT_THRESHHOLD)
  416. {
  417. /* find the pivot */
  418. int piv_ind = CPU_IAMAX(nx - z, &sub11[z*(ld+1)], ld);
  419. ipiv[z + first] = piv_ind + z + first;
  420. /* swap if needed */
  421. if (piv_ind != 0)
  422. {
  423. CPU_SWAP(nx, &sub11[z*ld], 1, &sub11[(z+piv_ind)*ld], 1);
  424. }
  425. pivot = sub11[z+z*ld];
  426. }
  427. STARPU_ASSERT(!ISZERO(pivot));
  428. CPU_SCAL(nx - z - 1, (1.0/pivot), &sub11[z+(z+1)*ld], ld);
  429. CPU_GER(nx - z - 1, nx - z - 1, -1.0,
  430. &sub11[(z+1)+z*ld], 1,
  431. &sub11[z+(z+1)*ld], ld,
  432. &sub11[(z+1) + (z+1)*ld],ld);
  433. }
  434. break;
  435. #ifdef STARPU_USE_CUDA
  436. case 1:
  437. handle = starpu_cublas_get_local_handle();
  438. stream = starpu_cuda_get_local_stream();
  439. for (z = 0; z < nx; z++)
  440. {
  441. TYPE pivot;
  442. TYPE inv_pivot;
  443. cudaMemcpyAsync(&pivot, &sub11[z+z*ld], sizeof(TYPE), cudaMemcpyDeviceToHost, stream);
  444. cudaStreamSynchronize(stream);
  445. if (fabs((double)(pivot)) < PIVOT_THRESHHOLD)
  446. {
  447. /* find the pivot */
  448. int piv_ind;
  449. status = CUBLAS_IAMAX(handle,
  450. nx - z, (CUBLAS_TYPE*)&sub11[z*(ld+1)], ld, &piv_ind);
  451. piv_ind -= 1;
  452. if (status != CUBLAS_STATUS_SUCCESS)
  453. STARPU_CUBLAS_REPORT_ERROR(status);
  454. ipiv[z + first] = piv_ind + z + first;
  455. /* swap if needed */
  456. if (piv_ind != 0)
  457. {
  458. status = CUBLAS_SWAP(handle,
  459. nx,
  460. (CUBLAS_TYPE*)&sub11[z*ld], 1,
  461. (CUBLAS_TYPE*)&sub11[(z+piv_ind)*ld], 1);
  462. if (status != CUBLAS_STATUS_SUCCESS)
  463. STARPU_CUBLAS_REPORT_ERROR(status);
  464. }
  465. cudaMemcpyAsync(&pivot, &sub11[z+z*ld], sizeof(TYPE), cudaMemcpyDeviceToHost, stream);
  466. cudaStreamSynchronize(stream);
  467. }
  468. STARPU_ASSERT(!ISZERO(pivot));
  469. inv_pivot = 1.0/pivot;
  470. status = CUBLAS_SCAL(handle,
  471. nx - z - 1,
  472. (CUBLAS_TYPE*)&inv_pivot,
  473. (CUBLAS_TYPE*)&sub11[z+(z+1)*ld], ld);
  474. if (status != CUBLAS_STATUS_SUCCESS)
  475. STARPU_CUBLAS_REPORT_ERROR(status);
  476. status = CUBLAS_GER(handle,
  477. nx - z - 1, nx - z - 1,
  478. (CUBLAS_TYPE*)&m1,
  479. (CUBLAS_TYPE*)&sub11[(z+1)+z*ld], 1,
  480. (CUBLAS_TYPE*)&sub11[z+(z+1)*ld], ld,
  481. (CUBLAS_TYPE*)&sub11[(z+1) + (z+1)*ld],ld);
  482. if (status != CUBLAS_STATUS_SUCCESS)
  483. STARPU_CUBLAS_REPORT_ERROR(status);
  484. }
  485. cudaStreamSynchronize(stream);
  486. break;
  487. #endif
  488. default:
  489. STARPU_ABORT();
  490. break;
  491. }
  492. }
  493. void STARPU_LU(cpu_u11_pivot)(void *descr[], void *_args)
  494. {
  495. STARPU_LU(common_u11_pivot)(descr, 0, _args);
  496. }
  497. #ifdef STARPU_USE_CUDA
  498. void STARPU_LU(cublas_u11_pivot)(void *descr[], void *_args)
  499. {
  500. STARPU_LU(common_u11_pivot)(descr, 1, _args);
  501. }
  502. #endif /* STARPU_USE_CUDA */
  503. static struct starpu_perfmodel STARPU_LU(model_11_pivot) =
  504. {
  505. .type = STARPU_HISTORY_BASED,
  506. #ifdef STARPU_ATLAS
  507. .symbol = STARPU_LU_STR(lu_model_11_pivot_atlas)
  508. #elif defined(STARPU_GOTO)
  509. .symbol = STARPU_LU_STR(lu_model_11_pivot_goto)
  510. #else
  511. .symbol = STARPU_LU_STR(lu_model_11_pivot)
  512. #endif
  513. };
  514. struct starpu_codelet cl11_pivot =
  515. {
  516. .cpu_funcs = {STARPU_LU(cpu_u11_pivot)},
  517. .cpu_funcs_name = {STRINGIFY(STARPU_LU(cpu_u11_pivot))},
  518. #ifdef STARPU_USE_CUDA
  519. .cuda_funcs = {STARPU_LU(cublas_u11_pivot)},
  520. CAN_EXECUTE
  521. #elif defined(STARPU_SIMGRID)
  522. .cuda_funcs = {(void*)1},
  523. #endif
  524. .nbuffers = 1,
  525. .modes = {STARPU_RW},
  526. .model = &STARPU_LU(model_11_pivot)
  527. };
  528. /*
  529. * Pivoting
  530. */
  531. static inline void STARPU_LU(common_pivot)(void *descr[],
  532. int s, void *_args)
  533. {
  534. TYPE *matrix;
  535. matrix = (TYPE *)STARPU_MATRIX_GET_PTR(descr[0]);
  536. unsigned long nx = STARPU_MATRIX_GET_NX(descr[0]);
  537. unsigned long ld = STARPU_MATRIX_GET_LD(descr[0]);
  538. unsigned row;
  539. struct piv_s *piv = _args;
  540. unsigned *ipiv = piv->piv;
  541. unsigned first = piv->first;
  542. #ifdef STARPU_USE_CUDA
  543. cublasStatus status;
  544. cublasHandle_t handle;
  545. #endif
  546. switch (s)
  547. {
  548. case 0:
  549. for (row = 0; row < nx; row++)
  550. {
  551. unsigned rowpiv = ipiv[row+first] - first;
  552. if (rowpiv != row)
  553. {
  554. CPU_SWAP(nx, &matrix[row*ld], 1, &matrix[rowpiv*ld], 1);
  555. }
  556. }
  557. break;
  558. #ifdef STARPU_USE_CUDA
  559. case 1:
  560. handle = starpu_cublas_get_local_handle();
  561. for (row = 0; row < nx; row++)
  562. {
  563. unsigned rowpiv = ipiv[row+first] - first;
  564. if (rowpiv != row)
  565. {
  566. status = CUBLAS_SWAP(handle,
  567. nx,
  568. (CUBLAS_TYPE*)&matrix[row*ld], 1,
  569. (CUBLAS_TYPE*)&matrix[rowpiv*ld], 1);
  570. if (status != CUBLAS_STATUS_SUCCESS)
  571. STARPU_CUBLAS_REPORT_ERROR(status);
  572. }
  573. }
  574. break;
  575. #endif
  576. default:
  577. STARPU_ABORT();
  578. break;
  579. }
  580. }
  581. void STARPU_LU(cpu_pivot)(void *descr[], void *_args)
  582. {
  583. STARPU_LU(common_pivot)(descr, 0, _args);
  584. }
  585. #ifdef STARPU_USE_CUDA
  586. void STARPU_LU(cublas_pivot)(void *descr[], void *_args)
  587. {
  588. STARPU_LU(common_pivot)(descr, 1, _args);
  589. }
  590. #endif /* STARPU_USE_CUDA */
  591. static struct starpu_perfmodel STARPU_LU(model_pivot) =
  592. {
  593. .type = STARPU_HISTORY_BASED,
  594. #ifdef STARPU_ATLAS
  595. .symbol = STARPU_LU_STR(lu_model_pivot_atlas)
  596. #elif defined(STARPU_GOTO)
  597. .symbol = STARPU_LU_STR(lu_model_pivot_goto)
  598. #else
  599. .symbol = STARPU_LU_STR(lu_model_pivot)
  600. #endif
  601. };
  602. struct starpu_codelet cl_pivot =
  603. {
  604. .cpu_funcs = {STARPU_LU(cpu_pivot)},
  605. .cpu_funcs_name = {STRINGIFY(STARPU_LU(cpu_pivot))},
  606. #ifdef STARPU_USE_CUDA
  607. .cuda_funcs = {STARPU_LU(cublas_pivot)},
  608. CAN_EXECUTE
  609. #elif defined(STARPU_SIMGRID)
  610. .cuda_funcs = {(void*)1},
  611. #endif
  612. .cuda_flags = {STARPU_CUDA_ASYNC},
  613. .nbuffers = 1,
  614. .modes = {STARPU_RW},
  615. .model = &STARPU_LU(model_pivot)
  616. };