csi_attacher_test.go 46 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496
  1. /*
  2. Copyright 2017 The Kubernetes Authors.
  3. Licensed under the Apache License, Version 2.0 (the "License");
  4. you may not use this file except in compliance with the License.
  5. You may obtain a copy of the License at
  6. http://www.apache.org/licenses/LICENSE-2.0
  7. Unless required by applicable law or agreed to in writing, software
  8. distributed under the License is distributed on an "AS IS" BASIS,
  9. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  10. See the License for the specific language governing permissions and
  11. limitations under the License.
  12. */
  13. package csi
  14. import (
  15. "fmt"
  16. "io/ioutil"
  17. "os"
  18. "path/filepath"
  19. "reflect"
  20. "sync"
  21. "testing"
  22. "time"
  23. storage "k8s.io/api/storage/v1"
  24. apierrs "k8s.io/apimachinery/pkg/api/errors"
  25. meta "k8s.io/apimachinery/pkg/apis/meta/v1"
  26. "k8s.io/apimachinery/pkg/runtime"
  27. "k8s.io/apimachinery/pkg/types"
  28. "k8s.io/apimachinery/pkg/util/wait"
  29. "k8s.io/apimachinery/pkg/watch"
  30. utilfeature "k8s.io/apiserver/pkg/util/feature"
  31. "k8s.io/client-go/informers"
  32. clientset "k8s.io/client-go/kubernetes"
  33. fakeclient "k8s.io/client-go/kubernetes/fake"
  34. core "k8s.io/client-go/testing"
  35. utiltesting "k8s.io/client-go/util/testing"
  36. featuregatetesting "k8s.io/component-base/featuregate/testing"
  37. "k8s.io/kubernetes/pkg/features"
  38. "k8s.io/kubernetes/pkg/volume"
  39. volumetest "k8s.io/kubernetes/pkg/volume/testing"
  40. )
  41. var (
  42. bFalse = false
  43. bTrue = true
  44. )
  45. func makeTestAttachment(attachID, nodeName, pvName string) *storage.VolumeAttachment {
  46. return &storage.VolumeAttachment{
  47. ObjectMeta: meta.ObjectMeta{
  48. Name: attachID,
  49. },
  50. Spec: storage.VolumeAttachmentSpec{
  51. NodeName: nodeName,
  52. Attacher: "mock",
  53. Source: storage.VolumeAttachmentSource{
  54. PersistentVolumeName: &pvName,
  55. },
  56. },
  57. Status: storage.VolumeAttachmentStatus{
  58. Attached: false,
  59. AttachError: nil,
  60. DetachError: nil,
  61. },
  62. }
  63. }
  64. func markVolumeAttached(t *testing.T, client clientset.Interface, watch *watch.RaceFreeFakeWatcher, attachID string, status storage.VolumeAttachmentStatus) {
  65. ticker := time.NewTicker(10 * time.Millisecond)
  66. var attach *storage.VolumeAttachment
  67. var err error
  68. defer ticker.Stop()
  69. // wait for attachment to be saved
  70. for i := 0; i < 100; i++ {
  71. attach, err = client.StorageV1().VolumeAttachments().Get(attachID, meta.GetOptions{})
  72. if err != nil {
  73. if apierrs.IsNotFound(err) {
  74. <-ticker.C
  75. continue
  76. }
  77. t.Error(err)
  78. }
  79. if attach != nil {
  80. t.Logf("attachment found on try %d, stopping wait...", i)
  81. break
  82. }
  83. }
  84. t.Logf("stopped waiting for attachment")
  85. if attach == nil {
  86. t.Logf("attachment not found for id:%v", attachID)
  87. } else {
  88. attach.Status = status
  89. t.Logf("updating attachment %s with attach status %v", attachID, status)
  90. _, err := client.StorageV1().VolumeAttachments().Update(attach)
  91. if err != nil {
  92. t.Error(err)
  93. }
  94. watch.Modify(attach)
  95. }
  96. }
  97. func TestAttacherAttach(t *testing.T) {
  98. testCases := []struct {
  99. name string
  100. nodeName string
  101. driverName string
  102. volumeName string
  103. attachID string
  104. spec *volume.Spec
  105. injectAttacherError bool
  106. shouldFail bool
  107. }{
  108. {
  109. name: "test ok 1",
  110. nodeName: "testnode-01",
  111. driverName: "testdriver-01",
  112. volumeName: "testvol-01",
  113. attachID: getAttachmentName("testvol-01", "testdriver-01", "testnode-01"),
  114. spec: volume.NewSpecFromPersistentVolume(makeTestPV("pv01", 10, "testdriver-01", "testvol-01"), false),
  115. },
  116. {
  117. name: "test ok 2",
  118. nodeName: "node02",
  119. driverName: "driver02",
  120. volumeName: "vol02",
  121. attachID: getAttachmentName("vol02", "driver02", "node02"),
  122. spec: volume.NewSpecFromPersistentVolume(makeTestPV("pv01", 10, "driver02", "vol02"), false),
  123. },
  124. {
  125. name: "mismatch vol",
  126. nodeName: "node02",
  127. driverName: "driver02",
  128. volumeName: "vol01",
  129. attachID: getAttachmentName("vol02", "driver02", "node02"),
  130. spec: volume.NewSpecFromPersistentVolume(makeTestPV("pv01", 10, "driver02", "vol01"), false),
  131. shouldFail: true,
  132. },
  133. {
  134. name: "mismatch driver",
  135. nodeName: "node02",
  136. driverName: "driver000",
  137. volumeName: "vol02",
  138. attachID: getAttachmentName("vol02", "driver02", "node02"),
  139. spec: volume.NewSpecFromPersistentVolume(makeTestPV("pv01", 10, "driver01", "vol02"), false),
  140. shouldFail: true,
  141. },
  142. {
  143. name: "mismatch node",
  144. nodeName: "node000",
  145. driverName: "driver000",
  146. volumeName: "vol02",
  147. attachID: getAttachmentName("vol02", "driver02", "node02"),
  148. spec: volume.NewSpecFromPersistentVolume(makeTestPV("pv01", 10, "driver02", "vol02"), false),
  149. shouldFail: true,
  150. },
  151. {
  152. name: "attacher error",
  153. nodeName: "node02",
  154. driverName: "driver02",
  155. volumeName: "vol02",
  156. attachID: getAttachmentName("vol02", "driver02", "node02"),
  157. spec: volume.NewSpecFromPersistentVolume(makeTestPV("pv01", 10, "driver02", "vol02"), false),
  158. injectAttacherError: true,
  159. shouldFail: true,
  160. },
  161. {
  162. name: "test with volume source",
  163. nodeName: "node000",
  164. driverName: "driver000",
  165. volumeName: "vol02",
  166. attachID: getAttachmentName("vol02", "driver02", "node02"),
  167. spec: volume.NewSpecFromVolume(makeTestVol("pv01", "driver02")),
  168. shouldFail: true, // csi not enabled
  169. },
  170. {
  171. name: "missing spec",
  172. nodeName: "node000",
  173. driverName: "driver000",
  174. volumeName: "vol02",
  175. attachID: getAttachmentName("vol02", "driver02", "node02"),
  176. shouldFail: true, // csi not enabled
  177. },
  178. }
  179. // attacher loop
  180. for _, tc := range testCases {
  181. t.Logf("test case: %s", tc.name)
  182. plug, fakeWatcher, tmpDir, _ := newTestWatchPlugin(t, nil)
  183. defer os.RemoveAll(tmpDir)
  184. attacher, err := plug.NewAttacher()
  185. if err != nil {
  186. t.Fatalf("failed to create new attacher: %v", err)
  187. }
  188. csiAttacher := attacher.(*csiAttacher)
  189. go func(spec *volume.Spec, nodename string, fail bool) {
  190. attachID, err := csiAttacher.Attach(spec, types.NodeName(nodename))
  191. if !fail && err != nil {
  192. t.Errorf("expecting no failure, but got err: %v", err)
  193. }
  194. if fail && err == nil {
  195. t.Errorf("expecting failure, but got no err")
  196. }
  197. if attachID != "" {
  198. t.Errorf("expecting empty attachID, got %v", attachID)
  199. }
  200. }(tc.spec, tc.nodeName, tc.shouldFail)
  201. var status storage.VolumeAttachmentStatus
  202. if tc.injectAttacherError {
  203. status.Attached = false
  204. status.AttachError = &storage.VolumeError{
  205. Message: "attacher error",
  206. }
  207. } else {
  208. status.Attached = true
  209. }
  210. markVolumeAttached(t, csiAttacher.k8s, fakeWatcher, tc.attachID, status)
  211. }
  212. }
  213. func TestAttacherAttachWithInline(t *testing.T) {
  214. defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSIInlineVolume, true)()
  215. testCases := []struct {
  216. name string
  217. nodeName string
  218. driverName string
  219. volumeName string
  220. attachID string
  221. spec *volume.Spec
  222. injectAttacherError bool
  223. shouldFail bool
  224. }{
  225. {
  226. name: "test ok 1 with PV",
  227. nodeName: "node01",
  228. attachID: getAttachmentName("vol01", "driver01", "node01"),
  229. spec: volume.NewSpecFromPersistentVolume(makeTestPV("pv01", 10, "driver01", "vol01"), false),
  230. },
  231. {
  232. name: "test failure, attach with volSrc",
  233. nodeName: "node01",
  234. attachID: getAttachmentName("vol01", "driver01", "node01"),
  235. spec: volume.NewSpecFromVolume(makeTestVol("vol01", "driver01")),
  236. shouldFail: true,
  237. },
  238. {
  239. name: "attacher error",
  240. nodeName: "node02",
  241. attachID: getAttachmentName("vol02", "driver02", "node02"),
  242. spec: volume.NewSpecFromPersistentVolume(makeTestPV("pv02", 10, "driver02", "vol02"), false),
  243. injectAttacherError: true,
  244. shouldFail: true,
  245. },
  246. {
  247. name: "missing spec",
  248. nodeName: "node02",
  249. attachID: getAttachmentName("vol02", "driver02", "node02"),
  250. shouldFail: true,
  251. },
  252. }
  253. // attacher loop
  254. for _, tc := range testCases {
  255. t.Logf("test case: %s", tc.name)
  256. plug, fakeWatcher, tmpDir, _ := newTestWatchPlugin(t, nil)
  257. defer os.RemoveAll(tmpDir)
  258. attacher, err := plug.NewAttacher()
  259. if err != nil {
  260. t.Fatalf("failed to create new attacher: %v", err)
  261. }
  262. csiAttacher := attacher.(*csiAttacher)
  263. go func(spec *volume.Spec, nodename string, fail bool) {
  264. attachID, err := csiAttacher.Attach(spec, types.NodeName(nodename))
  265. if fail != (err != nil) {
  266. t.Errorf("expecting no failure, but got err: %v", err)
  267. }
  268. if attachID != "" {
  269. t.Errorf("expecting empty attachID, got %v", attachID)
  270. }
  271. }(tc.spec, tc.nodeName, tc.shouldFail)
  272. var status storage.VolumeAttachmentStatus
  273. if tc.injectAttacherError {
  274. status.Attached = false
  275. status.AttachError = &storage.VolumeError{
  276. Message: "attacher error",
  277. }
  278. } else {
  279. status.Attached = true
  280. }
  281. markVolumeAttached(t, csiAttacher.k8s, fakeWatcher, tc.attachID, status)
  282. }
  283. }
  284. func TestAttacherWithCSIDriver(t *testing.T) {
  285. defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSIDriverRegistry, true)()
  286. tests := []struct {
  287. name string
  288. driver string
  289. expectVolumeAttachment bool
  290. }{
  291. {
  292. name: "CSIDriver not attachable",
  293. driver: "not-attachable",
  294. expectVolumeAttachment: false,
  295. },
  296. {
  297. name: "CSIDriver is attachable",
  298. driver: "attachable",
  299. expectVolumeAttachment: true,
  300. },
  301. {
  302. name: "CSIDriver.AttachRequired not set -> failure",
  303. driver: "nil",
  304. expectVolumeAttachment: true,
  305. },
  306. {
  307. name: "CSIDriver does not exist not set -> failure",
  308. driver: "unknown",
  309. expectVolumeAttachment: true,
  310. },
  311. }
  312. for _, test := range tests {
  313. t.Run(test.name, func(t *testing.T) {
  314. fakeClient := fakeclient.NewSimpleClientset(
  315. getTestCSIDriver("not-attachable", nil, &bFalse),
  316. getTestCSIDriver("attachable", nil, &bTrue),
  317. getTestCSIDriver("nil", nil, nil),
  318. )
  319. plug, fakeWatcher, tmpDir, _ := newTestWatchPlugin(t, fakeClient)
  320. defer os.RemoveAll(tmpDir)
  321. attacher, err := plug.NewAttacher()
  322. if err != nil {
  323. t.Fatalf("failed to create new attacher: %v", err)
  324. }
  325. csiAttacher := attacher.(*csiAttacher)
  326. spec := volume.NewSpecFromPersistentVolume(makeTestPV("test-pv", 10, test.driver, "test-vol"), false)
  327. pluginCanAttach, err := plug.CanAttach(spec)
  328. if err != nil {
  329. t.Fatalf("attacher.CanAttach failed: %s", err)
  330. }
  331. if pluginCanAttach != test.expectVolumeAttachment {
  332. t.Errorf("attacher.CanAttach does not match expected attachment status %t", test.expectVolumeAttachment)
  333. }
  334. if !pluginCanAttach {
  335. t.Log("plugin is not attachable")
  336. return
  337. }
  338. var wg sync.WaitGroup
  339. wg.Add(1)
  340. go func(volSpec *volume.Spec, expectAttach bool) {
  341. attachID, err := csiAttacher.Attach(volSpec, types.NodeName("node"))
  342. defer wg.Done()
  343. if err != nil {
  344. t.Errorf("Attach() failed: %s", err)
  345. }
  346. if attachID != "" {
  347. t.Errorf("Expected empty attachID, got %q", attachID)
  348. }
  349. }(spec, test.expectVolumeAttachment)
  350. if test.expectVolumeAttachment {
  351. expectedAttachID := getAttachmentName("test-vol", test.driver, "node")
  352. status := storage.VolumeAttachmentStatus{
  353. Attached: true,
  354. }
  355. markVolumeAttached(t, csiAttacher.k8s, fakeWatcher, expectedAttachID, status)
  356. }
  357. wg.Wait()
  358. })
  359. }
  360. }
  361. func TestAttacherWaitForVolumeAttachmentWithCSIDriver(t *testing.T) {
  362. defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSIDriverRegistry, true)()
  363. // In order to detect if the volume plugin would skip WaitForAttach for non-attachable drivers,
  364. // we do not instantiate any VolumeAttachment. So if the plugin does not skip attach, WaitForVolumeAttachment
  365. // will return an error that volume attachment was not found.
  366. tests := []struct {
  367. name string
  368. driver string
  369. expectError bool
  370. }{
  371. {
  372. name: "CSIDriver not attachable -> success",
  373. driver: "not-attachable",
  374. expectError: false,
  375. },
  376. {
  377. name: "CSIDriver is attachable -> failure",
  378. driver: "attachable",
  379. expectError: true,
  380. },
  381. {
  382. name: "CSIDriver.AttachRequired not set -> failure",
  383. driver: "nil",
  384. expectError: true,
  385. },
  386. {
  387. name: "CSIDriver does not exist not set -> failure",
  388. driver: "unknown",
  389. expectError: true,
  390. },
  391. }
  392. for _, test := range tests {
  393. t.Run(test.name, func(t *testing.T) {
  394. fakeClient := fakeclient.NewSimpleClientset(
  395. getTestCSIDriver("not-attachable", nil, &bFalse),
  396. getTestCSIDriver("attachable", nil, &bTrue),
  397. getTestCSIDriver("nil", nil, nil),
  398. )
  399. plug, tmpDir := newTestPlugin(t, fakeClient)
  400. defer os.RemoveAll(tmpDir)
  401. attacher, err := plug.NewAttacher()
  402. if err != nil {
  403. t.Fatalf("failed to create new attacher: %v", err)
  404. }
  405. csiAttacher := attacher.(*csiAttacher)
  406. spec := volume.NewSpecFromPersistentVolume(makeTestPV("test-pv", 10, test.driver, "test-vol"), false)
  407. pluginCanAttach, err := plug.CanAttach(spec)
  408. if err != nil {
  409. t.Fatalf("plugin.CanAttach test failed: %s", err)
  410. }
  411. if !pluginCanAttach {
  412. t.Log("plugin is not attachable")
  413. return
  414. }
  415. _, err = csiAttacher.WaitForAttach(spec, "", nil, time.Second)
  416. if err != nil && !test.expectError {
  417. t.Errorf("Unexpected error: %s", err)
  418. }
  419. if err == nil && test.expectError {
  420. t.Errorf("Expected error, got none")
  421. }
  422. })
  423. }
  424. }
  425. func TestAttacherWaitForAttach(t *testing.T) {
  426. tests := []struct {
  427. name string
  428. driver string
  429. makeAttachment func() *storage.VolumeAttachment
  430. spec *volume.Spec
  431. expectedAttachID string
  432. expectError bool
  433. }{
  434. {
  435. name: "successful attach",
  436. driver: "attachable",
  437. makeAttachment: func() *storage.VolumeAttachment {
  438. testAttachID := getAttachmentName("test-vol", "attachable", "node")
  439. successfulAttachment := makeTestAttachment(testAttachID, "node", "test-pv")
  440. successfulAttachment.Status.Attached = true
  441. return successfulAttachment
  442. },
  443. spec: volume.NewSpecFromPersistentVolume(makeTestPV("test-pv", 10, "attachable", "test-vol"), false),
  444. expectedAttachID: getAttachmentName("test-vol", "attachable", "node"),
  445. expectError: false,
  446. },
  447. {
  448. name: "failed attach with vol source",
  449. makeAttachment: func() *storage.VolumeAttachment {
  450. testAttachID := getAttachmentName("test-vol", "attachable", "node")
  451. successfulAttachment := makeTestAttachment(testAttachID, "node", "volSrc01")
  452. successfulAttachment.Status.Attached = true
  453. return successfulAttachment
  454. },
  455. spec: volume.NewSpecFromVolume(makeTestVol("volSrc01", "attachable")),
  456. expectError: true,
  457. },
  458. {
  459. name: "failed attach",
  460. driver: "attachable",
  461. expectError: true,
  462. },
  463. }
  464. for _, test := range tests {
  465. t.Run(test.name, func(t *testing.T) {
  466. plug, _, tmpDir, _ := newTestWatchPlugin(t, nil)
  467. defer os.RemoveAll(tmpDir)
  468. attacher, err := plug.NewAttacher()
  469. if err != nil {
  470. t.Fatalf("failed to create new attacher: %v", err)
  471. }
  472. csiAttacher := attacher.(*csiAttacher)
  473. if test.makeAttachment != nil {
  474. attachment := test.makeAttachment()
  475. _, err = csiAttacher.k8s.StorageV1().VolumeAttachments().Create(attachment)
  476. if err != nil {
  477. t.Fatalf("failed to create VolumeAttachment: %v", err)
  478. }
  479. gotAttachment, err := csiAttacher.k8s.StorageV1().VolumeAttachments().Get(attachment.Name, meta.GetOptions{})
  480. if err != nil {
  481. t.Fatalf("failed to get created VolumeAttachment: %v", err)
  482. }
  483. t.Logf("created test VolumeAttachment %+v", gotAttachment)
  484. }
  485. attachID, err := csiAttacher.WaitForAttach(test.spec, "", nil, time.Second)
  486. if err != nil && !test.expectError {
  487. t.Errorf("Unexpected error: %s", err)
  488. }
  489. if err == nil && test.expectError {
  490. t.Errorf("Expected error, got none")
  491. }
  492. if attachID != test.expectedAttachID {
  493. t.Errorf("Expected attachID %q, got %q", test.expectedAttachID, attachID)
  494. }
  495. })
  496. }
  497. }
  498. func TestAttacherWaitForAttachWithInline(t *testing.T) {
  499. defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSIInlineVolume, true)()
  500. tests := []struct {
  501. name string
  502. driver string
  503. makeAttachment func() *storage.VolumeAttachment
  504. spec *volume.Spec
  505. expectedAttachID string
  506. expectError bool
  507. }{
  508. {
  509. name: "successful attach with PV",
  510. makeAttachment: func() *storage.VolumeAttachment {
  511. testAttachID := getAttachmentName("test-vol", "attachable", "node")
  512. successfulAttachment := makeTestAttachment(testAttachID, "node", "test-pv")
  513. successfulAttachment.Status.Attached = true
  514. return successfulAttachment
  515. },
  516. spec: volume.NewSpecFromPersistentVolume(makeTestPV("test-pv", 10, "attachable", "test-vol"), false),
  517. expectedAttachID: getAttachmentName("test-vol", "attachable", "node"),
  518. expectError: false,
  519. },
  520. {
  521. name: "failed attach with volSrc",
  522. makeAttachment: func() *storage.VolumeAttachment {
  523. testAttachID := getAttachmentName("test-vol", "attachable", "node")
  524. successfulAttachment := makeTestAttachment(testAttachID, "node", "volSrc01")
  525. successfulAttachment.Status.Attached = true
  526. return successfulAttachment
  527. },
  528. spec: volume.NewSpecFromVolume(makeTestVol("volSrc01", "attachable")),
  529. expectError: true,
  530. },
  531. {
  532. name: "failed attach",
  533. driver: "non-attachable",
  534. spec: volume.NewSpecFromPersistentVolume(makeTestPV("test-pv", 10, "non-attachable", "test-vol"), false),
  535. expectError: true,
  536. },
  537. }
  538. for _, test := range tests {
  539. t.Run(test.name, func(t *testing.T) {
  540. plug, _, tmpDir, _ := newTestWatchPlugin(t, nil)
  541. defer os.RemoveAll(tmpDir)
  542. attacher, err := plug.NewAttacher()
  543. if err != nil {
  544. t.Fatalf("failed to create new attacher: %v", err)
  545. }
  546. csiAttacher := attacher.(*csiAttacher)
  547. if test.makeAttachment != nil {
  548. attachment := test.makeAttachment()
  549. _, err = csiAttacher.k8s.StorageV1().VolumeAttachments().Create(attachment)
  550. if err != nil {
  551. t.Fatalf("failed to create VolumeAttachment: %v", err)
  552. }
  553. gotAttachment, err := csiAttacher.k8s.StorageV1().VolumeAttachments().Get(attachment.Name, meta.GetOptions{})
  554. if err != nil {
  555. t.Fatalf("failed to get created VolumeAttachment: %v", err)
  556. }
  557. t.Logf("created test VolumeAttachment %+v", gotAttachment)
  558. }
  559. attachID, err := csiAttacher.WaitForAttach(test.spec, "", nil, time.Second)
  560. if test.expectError != (err != nil) {
  561. t.Errorf("Unexpected error: %s", err)
  562. return
  563. }
  564. if attachID != test.expectedAttachID {
  565. t.Errorf("Expected attachID %q, got %q", test.expectedAttachID, attachID)
  566. }
  567. })
  568. }
  569. }
  570. func TestAttacherWaitForVolumeAttachment(t *testing.T) {
  571. nodeName := "test-node"
  572. testCases := []struct {
  573. name string
  574. initAttached bool
  575. finalAttached bool
  576. trigerWatchEventTime time.Duration
  577. initAttachErr *storage.VolumeError
  578. finalAttachErr *storage.VolumeError
  579. timeout time.Duration
  580. shouldFail bool
  581. }{
  582. {
  583. name: "attach success at get",
  584. initAttached: true,
  585. timeout: 50 * time.Millisecond,
  586. shouldFail: false,
  587. },
  588. {
  589. name: "attachment error ant get",
  590. initAttachErr: &storage.VolumeError{Message: "missing volume"},
  591. timeout: 30 * time.Millisecond,
  592. shouldFail: true,
  593. },
  594. {
  595. name: "attach success at watch",
  596. initAttached: false,
  597. finalAttached: true,
  598. trigerWatchEventTime: 5 * time.Millisecond,
  599. timeout: 50 * time.Millisecond,
  600. shouldFail: false,
  601. },
  602. {
  603. name: "attachment error ant watch",
  604. initAttached: false,
  605. finalAttached: false,
  606. finalAttachErr: &storage.VolumeError{Message: "missing volume"},
  607. trigerWatchEventTime: 5 * time.Millisecond,
  608. timeout: 30 * time.Millisecond,
  609. shouldFail: true,
  610. },
  611. {
  612. name: "time ran out",
  613. initAttached: false,
  614. finalAttached: true,
  615. trigerWatchEventTime: 100 * time.Millisecond,
  616. timeout: 50 * time.Millisecond,
  617. shouldFail: true,
  618. },
  619. }
  620. for i, tc := range testCases {
  621. plug, fakeWatcher, tmpDir, _ := newTestWatchPlugin(t, nil)
  622. defer os.RemoveAll(tmpDir)
  623. attacher, err := plug.NewAttacher()
  624. if err != nil {
  625. t.Fatalf("failed to create new attacher: %v", err)
  626. }
  627. csiAttacher := attacher.(*csiAttacher)
  628. t.Logf("running test: %v", tc.name)
  629. pvName := fmt.Sprintf("test-pv-%d", i)
  630. volID := fmt.Sprintf("test-vol-%d", i)
  631. attachID := getAttachmentName(volID, testDriver, nodeName)
  632. attachment := makeTestAttachment(attachID, nodeName, pvName)
  633. attachment.Status.Attached = tc.initAttached
  634. attachment.Status.AttachError = tc.initAttachErr
  635. _, err = csiAttacher.k8s.StorageV1().VolumeAttachments().Create(attachment)
  636. if err != nil {
  637. t.Fatalf("failed to attach: %v", err)
  638. }
  639. trigerWatchEventTime := tc.trigerWatchEventTime
  640. finalAttached := tc.finalAttached
  641. finalAttachErr := tc.finalAttachErr
  642. // after timeout, fakeWatcher will be closed by csiAttacher.waitForVolumeAttachment
  643. if tc.trigerWatchEventTime > 0 && tc.trigerWatchEventTime < tc.timeout {
  644. go func() {
  645. time.Sleep(trigerWatchEventTime)
  646. attachment := makeTestAttachment(attachID, nodeName, pvName)
  647. attachment.Status.Attached = finalAttached
  648. attachment.Status.AttachError = finalAttachErr
  649. fakeWatcher.Modify(attachment)
  650. }()
  651. }
  652. retID, err := csiAttacher.waitForVolumeAttachment(volID, attachID, tc.timeout)
  653. if tc.shouldFail && err == nil {
  654. t.Error("expecting failure, but err is nil")
  655. }
  656. if tc.initAttachErr != nil {
  657. if tc.initAttachErr.Message != err.Error() {
  658. t.Errorf("expecting error [%v], got [%v]", tc.initAttachErr.Message, err.Error())
  659. }
  660. }
  661. if err == nil && retID != attachID {
  662. t.Errorf("attacher.WaitForAttach not returning attachment ID")
  663. }
  664. }
  665. }
  666. func TestAttacherVolumesAreAttached(t *testing.T) {
  667. type attachedSpec struct {
  668. volName string
  669. spec *volume.Spec
  670. attached bool
  671. }
  672. testCases := []struct {
  673. name string
  674. attachedSpecs []attachedSpec
  675. }{
  676. {
  677. "attach and detach",
  678. []attachedSpec{
  679. {"vol0", volume.NewSpecFromPersistentVolume(makeTestPV("pv0", 10, testDriver, "vol0"), false), true},
  680. {"vol1", volume.NewSpecFromPersistentVolume(makeTestPV("pv1", 20, testDriver, "vol1"), false), true},
  681. {"vol2", volume.NewSpecFromPersistentVolume(makeTestPV("pv2", 10, testDriver, "vol2"), false), false},
  682. {"vol3", volume.NewSpecFromPersistentVolume(makeTestPV("pv3", 10, testDriver, "vol3"), false), false},
  683. {"vol4", volume.NewSpecFromPersistentVolume(makeTestPV("pv4", 20, testDriver, "vol4"), false), true},
  684. },
  685. },
  686. {
  687. "all detached",
  688. []attachedSpec{
  689. {"vol0", volume.NewSpecFromPersistentVolume(makeTestPV("pv0", 10, testDriver, "vol0"), false), false},
  690. {"vol1", volume.NewSpecFromPersistentVolume(makeTestPV("pv1", 20, testDriver, "vol1"), false), false},
  691. {"vol2", volume.NewSpecFromPersistentVolume(makeTestPV("pv2", 10, testDriver, "vol2"), false), false},
  692. },
  693. },
  694. {
  695. "all attached",
  696. []attachedSpec{
  697. {"vol0", volume.NewSpecFromPersistentVolume(makeTestPV("pv0", 10, testDriver, "vol0"), false), true},
  698. {"vol1", volume.NewSpecFromPersistentVolume(makeTestPV("pv1", 20, testDriver, "vol1"), false), true},
  699. },
  700. },
  701. {
  702. "include non-attable",
  703. []attachedSpec{
  704. {"vol0", volume.NewSpecFromPersistentVolume(makeTestPV("pv0", 10, testDriver, "vol0"), false), true},
  705. {"vol1", volume.NewSpecFromVolume(makeTestVol("pv1", testDriver)), false},
  706. },
  707. },
  708. }
  709. for _, tc := range testCases {
  710. t.Run(tc.name, func(t *testing.T) {
  711. plug, tmpDir := newTestPlugin(t, nil)
  712. defer os.RemoveAll(tmpDir)
  713. attacher, err := plug.NewAttacher()
  714. if err != nil {
  715. t.Fatalf("failed to create new attacher: %v", err)
  716. }
  717. csiAttacher := attacher.(*csiAttacher)
  718. nodeName := "test-node"
  719. var specs []*volume.Spec
  720. // create and save volume attchments
  721. for _, attachedSpec := range tc.attachedSpecs {
  722. specs = append(specs, attachedSpec.spec)
  723. attachID := getAttachmentName(attachedSpec.volName, testDriver, nodeName)
  724. attachment := makeTestAttachment(attachID, nodeName, attachedSpec.spec.Name())
  725. attachment.Status.Attached = attachedSpec.attached
  726. _, err := csiAttacher.k8s.StorageV1().VolumeAttachments().Create(attachment)
  727. if err != nil {
  728. t.Fatalf("failed to attach: %v", err)
  729. }
  730. }
  731. // retrieve attached status
  732. stats, err := csiAttacher.VolumesAreAttached(specs, types.NodeName(nodeName))
  733. if err != nil {
  734. t.Fatal(err)
  735. }
  736. if len(tc.attachedSpecs) != len(stats) {
  737. t.Errorf("expecting %d attachment status, got %d", len(tc.attachedSpecs), len(stats))
  738. }
  739. // compare attachment status for each spec
  740. for _, attached := range tc.attachedSpecs {
  741. stat, ok := stats[attached.spec]
  742. if attached.attached && !ok {
  743. t.Error("failed to retrieve attached status for:", attached.spec)
  744. }
  745. if attached.attached != stat {
  746. t.Errorf("expecting volume attachment %t, got %t", attached.attached, stat)
  747. }
  748. }
  749. })
  750. }
  751. }
  752. func TestAttacherVolumesAreAttachedWithInline(t *testing.T) {
  753. defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSIInlineVolume, true)()
  754. type attachedSpec struct {
  755. volName string
  756. spec *volume.Spec
  757. attached bool
  758. }
  759. testCases := []struct {
  760. name string
  761. attachedSpecs []attachedSpec
  762. }{
  763. {
  764. "attach and detach with volume sources",
  765. []attachedSpec{
  766. {"vol0", volume.NewSpecFromPersistentVolume(makeTestPV("pv0", 10, testDriver, "vol0"), false), true},
  767. {"vol1", volume.NewSpecFromVolume(makeTestVol("pv1", testDriver)), false},
  768. {"vol2", volume.NewSpecFromPersistentVolume(makeTestPV("pv2", 10, testDriver, "vol2"), false), true},
  769. {"vol3", volume.NewSpecFromVolume(makeTestVol("pv3", testDriver)), false},
  770. {"vol4", volume.NewSpecFromPersistentVolume(makeTestPV("pv4", 20, testDriver, "vol4"), false), true},
  771. },
  772. },
  773. }
  774. for _, tc := range testCases {
  775. t.Run(tc.name, func(t *testing.T) {
  776. plug, tmpDir := newTestPlugin(t, nil)
  777. defer os.RemoveAll(tmpDir)
  778. attacher, err := plug.NewAttacher()
  779. if err != nil {
  780. t.Fatalf("failed to create new attacher: %v", err)
  781. }
  782. csiAttacher := attacher.(*csiAttacher)
  783. nodeName := "test-node"
  784. var specs []*volume.Spec
  785. // create and save volume attchments
  786. for _, attachedSpec := range tc.attachedSpecs {
  787. specs = append(specs, attachedSpec.spec)
  788. attachID := getAttachmentName(attachedSpec.volName, testDriver, nodeName)
  789. attachment := makeTestAttachment(attachID, nodeName, attachedSpec.spec.Name())
  790. attachment.Status.Attached = attachedSpec.attached
  791. _, err := csiAttacher.k8s.StorageV1().VolumeAttachments().Create(attachment)
  792. if err != nil {
  793. t.Fatalf("failed to attach: %v", err)
  794. }
  795. }
  796. // retrieve attached status
  797. stats, err := csiAttacher.VolumesAreAttached(specs, types.NodeName(nodeName))
  798. if err != nil {
  799. t.Fatal(err)
  800. }
  801. if len(tc.attachedSpecs) != len(stats) {
  802. t.Errorf("expecting %d attachment status, got %d", len(tc.attachedSpecs), len(stats))
  803. }
  804. // compare attachment status for each spec
  805. for _, attached := range tc.attachedSpecs {
  806. stat, ok := stats[attached.spec]
  807. if attached.attached && !ok {
  808. t.Error("failed to retrieve attached status for:", attached.spec)
  809. }
  810. if attached.attached != stat {
  811. t.Errorf("expecting volume attachment %t, got %t", attached.attached, stat)
  812. }
  813. }
  814. })
  815. }
  816. }
  817. func TestAttacherDetach(t *testing.T) {
  818. nodeName := "test-node"
  819. testCases := []struct {
  820. name string
  821. volID string
  822. attachID string
  823. shouldFail bool
  824. reactor func(action core.Action) (handled bool, ret runtime.Object, err error)
  825. }{
  826. {name: "normal test", volID: "vol-001", attachID: getAttachmentName("vol-001", testDriver, nodeName)},
  827. {name: "normal test 2", volID: "vol-002", attachID: getAttachmentName("vol-002", testDriver, nodeName)},
  828. {name: "object not found", volID: "vol-non-existing", attachID: getAttachmentName("vol-003", testDriver, nodeName)},
  829. {
  830. name: "API error",
  831. volID: "vol-004",
  832. attachID: getAttachmentName("vol-004", testDriver, nodeName),
  833. shouldFail: true, // All other API errors should be propagated to caller
  834. reactor: func(action core.Action) (handled bool, ret runtime.Object, err error) {
  835. // return Forbidden to all DELETE requests
  836. if action.Matches("delete", "volumeattachments") {
  837. return true, nil, apierrs.NewForbidden(action.GetResource().GroupResource(), action.GetNamespace(), fmt.Errorf("mock error"))
  838. }
  839. return false, nil, nil
  840. },
  841. },
  842. }
  843. for _, tc := range testCases {
  844. t.Logf("running test: %v", tc.name)
  845. plug, fakeWatcher, tmpDir, client := newTestWatchPlugin(t, nil)
  846. defer os.RemoveAll(tmpDir)
  847. if tc.reactor != nil {
  848. client.PrependReactor("*", "*", tc.reactor)
  849. }
  850. attacher, err0 := plug.NewAttacher()
  851. if err0 != nil {
  852. t.Fatalf("failed to create new attacher: %v", err0)
  853. }
  854. csiAttacher := attacher.(*csiAttacher)
  855. pv := makeTestPV("test-pv", 10, testDriver, tc.volID)
  856. spec := volume.NewSpecFromPersistentVolume(pv, pv.Spec.PersistentVolumeSource.CSI.ReadOnly)
  857. attachment := makeTestAttachment(tc.attachID, nodeName, "test-pv")
  858. _, err := csiAttacher.k8s.StorageV1().VolumeAttachments().Create(attachment)
  859. if err != nil {
  860. t.Fatalf("failed to attach: %v", err)
  861. }
  862. volumeName, err := plug.GetVolumeName(spec)
  863. if err != nil {
  864. t.Errorf("test case %s failed: %v", tc.name, err)
  865. }
  866. go func() {
  867. fakeWatcher.Delete(attachment)
  868. }()
  869. err = csiAttacher.Detach(volumeName, types.NodeName(nodeName))
  870. if tc.shouldFail && err == nil {
  871. t.Fatal("expecting failure, but err = nil")
  872. }
  873. if !tc.shouldFail && err != nil {
  874. t.Fatalf("unexpected err: %v", err)
  875. }
  876. attach, err := csiAttacher.k8s.StorageV1().VolumeAttachments().Get(tc.attachID, meta.GetOptions{})
  877. if err != nil {
  878. if !apierrs.IsNotFound(err) {
  879. t.Fatalf("unexpected err: %v", err)
  880. }
  881. } else {
  882. if attach == nil {
  883. t.Errorf("expecting attachment not to be nil, but it is")
  884. }
  885. }
  886. }
  887. }
  888. func TestAttacherGetDeviceMountPath(t *testing.T) {
  889. // Setup
  890. // Create a new attacher
  891. plug, _, tmpDir, _ := newTestWatchPlugin(t, nil)
  892. defer os.RemoveAll(tmpDir)
  893. attacher, err0 := plug.NewAttacher()
  894. if err0 != nil {
  895. t.Fatalf("failed to create new attacher: %v", err0)
  896. }
  897. csiAttacher := attacher.(*csiAttacher)
  898. pluginDir := csiAttacher.plugin.host.GetPluginDir(plug.GetPluginName())
  899. testCases := []struct {
  900. testName string
  901. pvName string
  902. expectedMountPath string
  903. shouldFail bool
  904. }{
  905. {
  906. testName: "normal test",
  907. pvName: "test-pv1",
  908. expectedMountPath: pluginDir + "/pv/test-pv1/globalmount",
  909. },
  910. {
  911. testName: "no pv name",
  912. pvName: "",
  913. expectedMountPath: pluginDir + "/pv/test-pv1/globalmount",
  914. shouldFail: true,
  915. },
  916. }
  917. for _, tc := range testCases {
  918. t.Logf("Running test case: %s", tc.testName)
  919. var spec *volume.Spec
  920. // Create spec
  921. pv := makeTestPV(tc.pvName, 10, testDriver, "testvol")
  922. spec = volume.NewSpecFromPersistentVolume(pv, pv.Spec.PersistentVolumeSource.CSI.ReadOnly)
  923. // Run
  924. mountPath, err := csiAttacher.GetDeviceMountPath(spec)
  925. // Verify
  926. if err != nil && !tc.shouldFail {
  927. t.Errorf("test should not fail, but error occurred: %v", err)
  928. } else if err == nil {
  929. if tc.shouldFail {
  930. t.Errorf("test should fail, but no error occurred")
  931. } else if mountPath != tc.expectedMountPath {
  932. t.Errorf("mountPath does not equal expectedMountPath. Got: %s. Expected: %s", mountPath, tc.expectedMountPath)
  933. }
  934. }
  935. }
  936. }
  937. func TestAttacherMountDevice(t *testing.T) {
  938. pvName := "test-pv"
  939. testCases := []struct {
  940. testName string
  941. volName string
  942. devicePath string
  943. deviceMountPath string
  944. stageUnstageSet bool
  945. shouldFail bool
  946. spec *volume.Spec
  947. }{
  948. {
  949. testName: "normal PV",
  950. volName: "test-vol1",
  951. devicePath: "path1",
  952. deviceMountPath: "path2",
  953. stageUnstageSet: true,
  954. spec: volume.NewSpecFromPersistentVolume(makeTestPV(pvName, 10, testDriver, "test-vol1"), false),
  955. },
  956. {
  957. testName: "normal PV with mount options",
  958. volName: "test-vol1",
  959. devicePath: "path1",
  960. deviceMountPath: "path2",
  961. stageUnstageSet: true,
  962. spec: volume.NewSpecFromPersistentVolume(makeTestPVWithMountOptions(pvName, 10, testDriver, "test-vol1", []string{"test-op"}), false),
  963. },
  964. {
  965. testName: "no vol name",
  966. volName: "",
  967. devicePath: "path1",
  968. deviceMountPath: "path2",
  969. stageUnstageSet: true,
  970. shouldFail: true,
  971. spec: volume.NewSpecFromPersistentVolume(makeTestPV(pvName, 10, testDriver, ""), false),
  972. },
  973. {
  974. testName: "no device path",
  975. volName: "test-vol1",
  976. devicePath: "",
  977. deviceMountPath: "path2",
  978. stageUnstageSet: true,
  979. shouldFail: false,
  980. spec: volume.NewSpecFromPersistentVolume(makeTestPV(pvName, 10, testDriver, "test-vol1"), false),
  981. },
  982. {
  983. testName: "no device mount path",
  984. volName: "test-vol1",
  985. devicePath: "path1",
  986. deviceMountPath: "",
  987. stageUnstageSet: true,
  988. shouldFail: true,
  989. spec: volume.NewSpecFromPersistentVolume(makeTestPV(pvName, 10, testDriver, "test-vol1"), false),
  990. },
  991. {
  992. testName: "stage_unstage cap not set",
  993. volName: "test-vol1",
  994. devicePath: "path1",
  995. deviceMountPath: "path2",
  996. stageUnstageSet: false,
  997. spec: volume.NewSpecFromPersistentVolume(makeTestPV(pvName, 10, testDriver, "test-vol1"), false),
  998. },
  999. {
  1000. testName: "failure with volume source",
  1001. volName: "test-vol1",
  1002. devicePath: "path1",
  1003. deviceMountPath: "path2",
  1004. shouldFail: true,
  1005. spec: volume.NewSpecFromVolume(makeTestVol(pvName, testDriver)),
  1006. },
  1007. }
  1008. for _, tc := range testCases {
  1009. t.Logf("Running test case: %s", tc.testName)
  1010. // Setup
  1011. // Create a new attacher
  1012. plug, fakeWatcher, tmpDir, _ := newTestWatchPlugin(t, nil)
  1013. defer os.RemoveAll(tmpDir)
  1014. attacher, err0 := plug.NewAttacher()
  1015. if err0 != nil {
  1016. t.Fatalf("failed to create new attacher: %v", err0)
  1017. }
  1018. csiAttacher := attacher.(*csiAttacher)
  1019. csiAttacher.csiClient = setupClient(t, tc.stageUnstageSet)
  1020. if tc.deviceMountPath != "" {
  1021. tc.deviceMountPath = filepath.Join(tmpDir, tc.deviceMountPath)
  1022. }
  1023. nodeName := string(csiAttacher.plugin.host.GetNodeName())
  1024. attachID := getAttachmentName(tc.volName, testDriver, nodeName)
  1025. // Set up volume attachment
  1026. attachment := makeTestAttachment(attachID, nodeName, pvName)
  1027. _, err := csiAttacher.k8s.StorageV1().VolumeAttachments().Create(attachment)
  1028. if err != nil {
  1029. t.Fatalf("failed to attach: %v", err)
  1030. }
  1031. go func() {
  1032. fakeWatcher.Delete(attachment)
  1033. }()
  1034. // Run
  1035. err = csiAttacher.MountDevice(tc.spec, tc.devicePath, tc.deviceMountPath)
  1036. // Verify
  1037. if err != nil {
  1038. if !tc.shouldFail {
  1039. t.Errorf("test should not fail, but error occurred: %v", err)
  1040. }
  1041. continue
  1042. }
  1043. if err == nil && tc.shouldFail {
  1044. t.Errorf("test should fail, but no error occurred")
  1045. }
  1046. // Verify call goes through all the way
  1047. numStaged := 1
  1048. if !tc.stageUnstageSet {
  1049. numStaged = 0
  1050. }
  1051. cdc := csiAttacher.csiClient.(*fakeCsiDriverClient)
  1052. staged := cdc.nodeClient.GetNodeStagedVolumes()
  1053. if len(staged) != numStaged {
  1054. t.Errorf("got wrong number of staged volumes, expecting %v got: %v", numStaged, len(staged))
  1055. }
  1056. if tc.stageUnstageSet {
  1057. vol, ok := staged[tc.volName]
  1058. if !ok {
  1059. t.Errorf("could not find staged volume: %s", tc.volName)
  1060. }
  1061. if vol.Path != tc.deviceMountPath {
  1062. t.Errorf("expected mount path: %s. got: %s", tc.deviceMountPath, vol.Path)
  1063. }
  1064. if !reflect.DeepEqual(vol.MountFlags, tc.spec.PersistentVolume.Spec.MountOptions) {
  1065. t.Errorf("expected mount options: %v, got: %v", tc.spec.PersistentVolume.Spec.MountOptions, vol.MountFlags)
  1066. }
  1067. }
  1068. }
  1069. }
  1070. func TestAttacherMountDeviceWithInline(t *testing.T) {
  1071. defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CSIInlineVolume, true)()
  1072. pvName := "test-pv"
  1073. testCases := []struct {
  1074. testName string
  1075. volName string
  1076. devicePath string
  1077. deviceMountPath string
  1078. stageUnstageSet bool
  1079. shouldFail bool
  1080. spec *volume.Spec
  1081. }{
  1082. {
  1083. testName: "normal PV",
  1084. volName: "test-vol1",
  1085. devicePath: "path1",
  1086. deviceMountPath: "path2",
  1087. stageUnstageSet: true,
  1088. spec: volume.NewSpecFromPersistentVolume(makeTestPV(pvName, 10, testDriver, "test-vol1"), false),
  1089. },
  1090. {
  1091. testName: "failure with volSrc",
  1092. volName: "test-vol1",
  1093. devicePath: "path1",
  1094. deviceMountPath: "path2",
  1095. shouldFail: true,
  1096. spec: volume.NewSpecFromVolume(makeTestVol(pvName, testDriver)),
  1097. },
  1098. {
  1099. testName: "no vol name",
  1100. volName: "",
  1101. devicePath: "path1",
  1102. deviceMountPath: "path2",
  1103. stageUnstageSet: true,
  1104. shouldFail: true,
  1105. spec: volume.NewSpecFromPersistentVolume(makeTestPV(pvName, 10, testDriver, ""), false),
  1106. },
  1107. {
  1108. testName: "no device path",
  1109. volName: "test-vol1",
  1110. devicePath: "",
  1111. deviceMountPath: "path2",
  1112. stageUnstageSet: true,
  1113. shouldFail: false,
  1114. spec: volume.NewSpecFromPersistentVolume(makeTestPV(pvName, 10, testDriver, "test-vol1"), false),
  1115. },
  1116. {
  1117. testName: "no device mount path",
  1118. volName: "test-vol1",
  1119. devicePath: "path1",
  1120. deviceMountPath: "",
  1121. stageUnstageSet: true,
  1122. shouldFail: true,
  1123. spec: volume.NewSpecFromPersistentVolume(makeTestPV(pvName, 10, testDriver, "test-vol1"), false),
  1124. },
  1125. {
  1126. testName: "stage_unstage cap not set",
  1127. volName: "test-vol1",
  1128. devicePath: "path1",
  1129. deviceMountPath: "path2",
  1130. stageUnstageSet: false,
  1131. spec: volume.NewSpecFromPersistentVolume(makeTestPV(pvName, 10, testDriver, "test-vol1"), false),
  1132. },
  1133. {
  1134. testName: "missing spec",
  1135. volName: "test-vol1",
  1136. devicePath: "path1",
  1137. deviceMountPath: "path2",
  1138. shouldFail: true,
  1139. },
  1140. }
  1141. for _, tc := range testCases {
  1142. t.Logf("Running test case: %s", tc.testName)
  1143. // Setup
  1144. // Create a new attacher
  1145. plug, fakeWatcher, tmpDir, _ := newTestWatchPlugin(t, nil)
  1146. defer os.RemoveAll(tmpDir)
  1147. attacher, err0 := plug.NewAttacher()
  1148. if err0 != nil {
  1149. t.Fatalf("failed to create new attacher: %v", err0)
  1150. }
  1151. csiAttacher := attacher.(*csiAttacher)
  1152. csiAttacher.csiClient = setupClient(t, tc.stageUnstageSet)
  1153. if tc.deviceMountPath != "" {
  1154. tc.deviceMountPath = filepath.Join(tmpDir, tc.deviceMountPath)
  1155. }
  1156. nodeName := string(csiAttacher.plugin.host.GetNodeName())
  1157. attachID := getAttachmentName(tc.volName, testDriver, nodeName)
  1158. // Set up volume attachment
  1159. attachment := makeTestAttachment(attachID, nodeName, pvName)
  1160. _, err := csiAttacher.k8s.StorageV1().VolumeAttachments().Create(attachment)
  1161. if err != nil {
  1162. t.Fatalf("failed to attach: %v", err)
  1163. }
  1164. go func() {
  1165. fakeWatcher.Delete(attachment)
  1166. }()
  1167. // Run
  1168. err = csiAttacher.MountDevice(tc.spec, tc.devicePath, tc.deviceMountPath)
  1169. // Verify
  1170. if err != nil {
  1171. if !tc.shouldFail {
  1172. t.Errorf("test should not fail, but error occurred: %v", err)
  1173. }
  1174. continue
  1175. }
  1176. if err == nil && tc.shouldFail {
  1177. t.Errorf("test should fail, but no error occurred")
  1178. }
  1179. // Verify call goes through all the way
  1180. numStaged := 1
  1181. if !tc.stageUnstageSet {
  1182. numStaged = 0
  1183. }
  1184. cdc := csiAttacher.csiClient.(*fakeCsiDriverClient)
  1185. staged := cdc.nodeClient.GetNodeStagedVolumes()
  1186. if len(staged) != numStaged {
  1187. t.Errorf("got wrong number of staged volumes, expecting %v got: %v", numStaged, len(staged))
  1188. }
  1189. if tc.stageUnstageSet {
  1190. vol, ok := staged[tc.volName]
  1191. if !ok {
  1192. t.Errorf("could not find staged volume: %s", tc.volName)
  1193. }
  1194. if vol.Path != tc.deviceMountPath {
  1195. t.Errorf("expected mount path: %s. got: %s", tc.deviceMountPath, vol.Path)
  1196. }
  1197. }
  1198. }
  1199. }
  1200. func TestAttacherUnmountDevice(t *testing.T) {
  1201. testCases := []struct {
  1202. testName string
  1203. volID string
  1204. deviceMountPath string
  1205. jsonFile string
  1206. createPV bool
  1207. stageUnstageSet bool
  1208. shouldFail bool
  1209. }{
  1210. {
  1211. testName: "normal, json file exists",
  1212. volID: "project/zone/test-vol1",
  1213. deviceMountPath: "plugins/csi/pv/test-pv-name/globalmount",
  1214. jsonFile: `{"driverName": "csi", "volumeHandle":"project/zone/test-vol1"}`,
  1215. createPV: false,
  1216. stageUnstageSet: true,
  1217. },
  1218. {
  1219. testName: "normal, json file doesn't exist -> use PV",
  1220. volID: "project/zone/test-vol1",
  1221. deviceMountPath: "plugins/csi/pv/test-pv-name/globalmount",
  1222. jsonFile: "",
  1223. createPV: true,
  1224. stageUnstageSet: true,
  1225. },
  1226. {
  1227. testName: "invalid json -> use PV",
  1228. volID: "project/zone/test-vol1",
  1229. deviceMountPath: "plugins/csi/pv/test-pv-name/globalmount",
  1230. jsonFile: `{"driverName"}}`,
  1231. createPV: true,
  1232. stageUnstageSet: true,
  1233. },
  1234. {
  1235. testName: "no json, no PV.volID",
  1236. volID: "",
  1237. deviceMountPath: "plugins/csi/pv/test-pv-name/globalmount",
  1238. jsonFile: "",
  1239. createPV: true,
  1240. shouldFail: true,
  1241. },
  1242. {
  1243. testName: "no json, no PV",
  1244. volID: "project/zone/test-vol1",
  1245. deviceMountPath: "plugins/csi/pv/test-pv-name/globalmount",
  1246. jsonFile: "",
  1247. createPV: false,
  1248. stageUnstageSet: true,
  1249. shouldFail: true,
  1250. },
  1251. {
  1252. testName: "stage_unstage not set no vars should not fail",
  1253. deviceMountPath: "plugins/csi/pv/test-pv-name/globalmount",
  1254. jsonFile: `{"driverName":"test-driver","volumeHandle":"test-vol1"}`,
  1255. stageUnstageSet: false,
  1256. },
  1257. }
  1258. for _, tc := range testCases {
  1259. t.Logf("Running test case: %s", tc.testName)
  1260. // Setup
  1261. // Create a new attacher
  1262. plug, _, tmpDir, _ := newTestWatchPlugin(t, nil)
  1263. defer os.RemoveAll(tmpDir)
  1264. attacher, err0 := plug.NewAttacher()
  1265. if err0 != nil {
  1266. t.Fatalf("failed to create new attacher: %v", err0)
  1267. }
  1268. csiAttacher := attacher.(*csiAttacher)
  1269. csiAttacher.csiClient = setupClient(t, tc.stageUnstageSet)
  1270. if tc.deviceMountPath != "" {
  1271. tc.deviceMountPath = filepath.Join(tmpDir, tc.deviceMountPath)
  1272. }
  1273. // Add the volume to NodeStagedVolumes
  1274. cdc := csiAttacher.csiClient.(*fakeCsiDriverClient)
  1275. cdc.nodeClient.AddNodeStagedVolume(tc.volID, tc.deviceMountPath, nil)
  1276. // Make JSON for this object
  1277. if tc.deviceMountPath != "" {
  1278. if err := os.MkdirAll(tc.deviceMountPath, 0755); err != nil {
  1279. t.Fatalf("error creating directory %s: %s", tc.deviceMountPath, err)
  1280. }
  1281. }
  1282. dir := filepath.Dir(tc.deviceMountPath)
  1283. if tc.jsonFile != "" {
  1284. dataPath := filepath.Join(dir, volDataFileName)
  1285. if err := ioutil.WriteFile(dataPath, []byte(tc.jsonFile), 0644); err != nil {
  1286. t.Fatalf("error creating %s: %s", dataPath, err)
  1287. }
  1288. }
  1289. if tc.createPV {
  1290. // Make the PV for this object
  1291. pvName := filepath.Base(dir)
  1292. pv := makeTestPV(pvName, 5, "csi", tc.volID)
  1293. _, err := csiAttacher.k8s.CoreV1().PersistentVolumes().Create(pv)
  1294. if err != nil && !tc.shouldFail {
  1295. t.Fatalf("Failed to create PV: %v", err)
  1296. }
  1297. }
  1298. // Run
  1299. err := csiAttacher.UnmountDevice(tc.deviceMountPath)
  1300. // Verify
  1301. if err != nil {
  1302. if !tc.shouldFail {
  1303. t.Errorf("test should not fail, but error occurred: %v", err)
  1304. }
  1305. continue
  1306. }
  1307. if err == nil && tc.shouldFail {
  1308. t.Errorf("test should fail, but no error occurred")
  1309. }
  1310. // Verify call goes through all the way
  1311. expectedSet := 0
  1312. if !tc.stageUnstageSet {
  1313. expectedSet = 1
  1314. }
  1315. staged := cdc.nodeClient.GetNodeStagedVolumes()
  1316. if len(staged) != expectedSet {
  1317. t.Errorf("got wrong number of staged volumes, expecting %v got: %v", expectedSet, len(staged))
  1318. }
  1319. _, ok := staged[tc.volID]
  1320. if ok && tc.stageUnstageSet {
  1321. t.Errorf("found unexpected staged volume: %s", tc.volID)
  1322. } else if !ok && !tc.stageUnstageSet {
  1323. t.Errorf("could not find expected staged volume: %s", tc.volID)
  1324. }
  1325. if tc.jsonFile != "" && !tc.shouldFail {
  1326. dataPath := filepath.Join(dir, volDataFileName)
  1327. if _, err := os.Stat(dataPath); !os.IsNotExist(err) {
  1328. if err != nil {
  1329. t.Errorf("error checking file %s: %s", dataPath, err)
  1330. } else {
  1331. t.Errorf("json file %s should not exists, but it does", dataPath)
  1332. }
  1333. } else {
  1334. t.Logf("json file %s was correctly removed", dataPath)
  1335. }
  1336. }
  1337. }
  1338. }
  1339. // create a plugin mgr to load plugins and setup a fake client
  1340. func newTestWatchPlugin(t *testing.T, fakeClient *fakeclient.Clientset) (*csiPlugin, *watch.RaceFreeFakeWatcher, string, *fakeclient.Clientset) {
  1341. tmpDir, err := utiltesting.MkTmpdir("csi-test")
  1342. if err != nil {
  1343. t.Fatalf("can't create temp dir: %v", err)
  1344. }
  1345. if fakeClient == nil {
  1346. fakeClient = fakeclient.NewSimpleClientset()
  1347. }
  1348. fakeWatcher := watch.NewRaceFreeFake()
  1349. fakeClient.Fake.PrependWatchReactor("volumeattachments", core.DefaultWatchReactor(fakeWatcher, nil))
  1350. // Start informer for CSIDrivers.
  1351. factory := informers.NewSharedInformerFactory(fakeClient, csiResyncPeriod)
  1352. csiDriverInformer := factory.Storage().V1beta1().CSIDrivers()
  1353. csiDriverLister := csiDriverInformer.Lister()
  1354. factory.Start(wait.NeverStop)
  1355. host := volumetest.NewFakeVolumeHostWithCSINodeName(
  1356. tmpDir,
  1357. fakeClient,
  1358. nil,
  1359. "node",
  1360. csiDriverLister,
  1361. )
  1362. plugMgr := &volume.VolumePluginMgr{}
  1363. plugMgr.InitPlugins(ProbeVolumePlugins(), nil /* prober */, host)
  1364. plug, err := plugMgr.FindPluginByName(CSIPluginName)
  1365. if err != nil {
  1366. t.Fatalf("can't find plugin %v", CSIPluginName)
  1367. }
  1368. csiPlug, ok := plug.(*csiPlugin)
  1369. if !ok {
  1370. t.Fatalf("cannot assert plugin to be type csiPlugin")
  1371. }
  1372. if utilfeature.DefaultFeatureGate.Enabled(features.CSIDriverRegistry) {
  1373. // Wait until the informer in CSI volume plugin has all CSIDrivers.
  1374. wait.PollImmediate(testInformerSyncPeriod, testInformerSyncTimeout, func() (bool, error) {
  1375. return csiDriverInformer.Informer().HasSynced(), nil
  1376. })
  1377. }
  1378. return csiPlug, fakeWatcher, tmpDir, fakeClient
  1379. }