spec_runner.go 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412
  1. package specrunner
  2. import (
  3. "fmt"
  4. "os"
  5. "os/signal"
  6. "sync"
  7. "syscall"
  8. "github.com/onsi/ginkgo/internal/spec_iterator"
  9. "github.com/onsi/ginkgo/config"
  10. "github.com/onsi/ginkgo/internal/leafnodes"
  11. "github.com/onsi/ginkgo/internal/spec"
  12. Writer "github.com/onsi/ginkgo/internal/writer"
  13. "github.com/onsi/ginkgo/reporters"
  14. "github.com/onsi/ginkgo/types"
  15. "time"
  16. )
  17. type SpecRunner struct {
  18. description string
  19. beforeSuiteNode leafnodes.SuiteNode
  20. iterator spec_iterator.SpecIterator
  21. afterSuiteNode leafnodes.SuiteNode
  22. reporters []reporters.Reporter
  23. startTime time.Time
  24. suiteID string
  25. runningSpec *spec.Spec
  26. writer Writer.WriterInterface
  27. config config.GinkgoConfigType
  28. interrupted bool
  29. processedSpecs []*spec.Spec
  30. lock *sync.Mutex
  31. }
  32. func New(description string, beforeSuiteNode leafnodes.SuiteNode, iterator spec_iterator.SpecIterator, afterSuiteNode leafnodes.SuiteNode, reporters []reporters.Reporter, writer Writer.WriterInterface, config config.GinkgoConfigType) *SpecRunner {
  33. return &SpecRunner{
  34. description: description,
  35. beforeSuiteNode: beforeSuiteNode,
  36. iterator: iterator,
  37. afterSuiteNode: afterSuiteNode,
  38. reporters: reporters,
  39. writer: writer,
  40. config: config,
  41. suiteID: randomID(),
  42. lock: &sync.Mutex{},
  43. }
  44. }
  45. func (runner *SpecRunner) Run() bool {
  46. if runner.config.DryRun {
  47. runner.performDryRun()
  48. return true
  49. }
  50. runner.reportSuiteWillBegin()
  51. signalRegistered := make(chan struct{})
  52. go runner.registerForInterrupts(signalRegistered)
  53. <-signalRegistered
  54. suitePassed := runner.runBeforeSuite()
  55. if suitePassed {
  56. suitePassed = runner.runSpecs()
  57. }
  58. runner.blockForeverIfInterrupted()
  59. suitePassed = runner.runAfterSuite() && suitePassed
  60. runner.reportSuiteDidEnd(suitePassed)
  61. return suitePassed
  62. }
  63. func (runner *SpecRunner) performDryRun() {
  64. runner.reportSuiteWillBegin()
  65. if runner.beforeSuiteNode != nil {
  66. summary := runner.beforeSuiteNode.Summary()
  67. summary.State = types.SpecStatePassed
  68. runner.reportBeforeSuite(summary)
  69. }
  70. for {
  71. spec, err := runner.iterator.Next()
  72. if err == spec_iterator.ErrClosed {
  73. break
  74. }
  75. if err != nil {
  76. fmt.Println("failed to iterate over tests:\n" + err.Error())
  77. break
  78. }
  79. runner.processedSpecs = append(runner.processedSpecs, spec)
  80. summary := spec.Summary(runner.suiteID)
  81. runner.reportSpecWillRun(summary)
  82. if summary.State == types.SpecStateInvalid {
  83. summary.State = types.SpecStatePassed
  84. }
  85. runner.reportSpecDidComplete(summary, false)
  86. }
  87. if runner.afterSuiteNode != nil {
  88. summary := runner.afterSuiteNode.Summary()
  89. summary.State = types.SpecStatePassed
  90. runner.reportAfterSuite(summary)
  91. }
  92. runner.reportSuiteDidEnd(true)
  93. }
  94. func (runner *SpecRunner) runBeforeSuite() bool {
  95. if runner.beforeSuiteNode == nil || runner.wasInterrupted() {
  96. return true
  97. }
  98. runner.writer.Truncate()
  99. conf := runner.config
  100. passed := runner.beforeSuiteNode.Run(conf.ParallelNode, conf.ParallelTotal, conf.SyncHost)
  101. if !passed {
  102. runner.writer.DumpOut()
  103. }
  104. runner.reportBeforeSuite(runner.beforeSuiteNode.Summary())
  105. return passed
  106. }
  107. func (runner *SpecRunner) runAfterSuite() bool {
  108. if runner.afterSuiteNode == nil {
  109. return true
  110. }
  111. runner.writer.Truncate()
  112. conf := runner.config
  113. passed := runner.afterSuiteNode.Run(conf.ParallelNode, conf.ParallelTotal, conf.SyncHost)
  114. if !passed {
  115. runner.writer.DumpOut()
  116. }
  117. runner.reportAfterSuite(runner.afterSuiteNode.Summary())
  118. return passed
  119. }
  120. func (runner *SpecRunner) runSpecs() bool {
  121. suiteFailed := false
  122. skipRemainingSpecs := false
  123. for {
  124. spec, err := runner.iterator.Next()
  125. if err == spec_iterator.ErrClosed {
  126. break
  127. }
  128. if err != nil {
  129. fmt.Println("failed to iterate over tests:\n" + err.Error())
  130. suiteFailed = true
  131. break
  132. }
  133. runner.processedSpecs = append(runner.processedSpecs, spec)
  134. if runner.wasInterrupted() {
  135. break
  136. }
  137. if skipRemainingSpecs {
  138. spec.Skip()
  139. }
  140. if !spec.Skipped() && !spec.Pending() {
  141. if passed := runner.runSpec(spec); !passed {
  142. suiteFailed = true
  143. }
  144. } else if spec.Pending() && runner.config.FailOnPending {
  145. runner.reportSpecWillRun(spec.Summary(runner.suiteID))
  146. suiteFailed = true
  147. runner.reportSpecDidComplete(spec.Summary(runner.suiteID), spec.Failed())
  148. } else {
  149. runner.reportSpecWillRun(spec.Summary(runner.suiteID))
  150. runner.reportSpecDidComplete(spec.Summary(runner.suiteID), spec.Failed())
  151. }
  152. if spec.Failed() && runner.config.FailFast {
  153. skipRemainingSpecs = true
  154. }
  155. }
  156. return !suiteFailed
  157. }
  158. func (runner *SpecRunner) runSpec(spec *spec.Spec) (passed bool) {
  159. maxAttempts := 1
  160. if runner.config.FlakeAttempts > 0 {
  161. // uninitialized configs count as 1
  162. maxAttempts = runner.config.FlakeAttempts
  163. }
  164. for i := 0; i < maxAttempts; i++ {
  165. runner.reportSpecWillRun(spec.Summary(runner.suiteID))
  166. runner.runningSpec = spec
  167. spec.Run(runner.writer)
  168. runner.runningSpec = nil
  169. runner.reportSpecDidComplete(spec.Summary(runner.suiteID), spec.Failed())
  170. if !spec.Failed() {
  171. return true
  172. }
  173. }
  174. return false
  175. }
  176. func (runner *SpecRunner) CurrentSpecSummary() (*types.SpecSummary, bool) {
  177. if runner.runningSpec == nil {
  178. return nil, false
  179. }
  180. return runner.runningSpec.Summary(runner.suiteID), true
  181. }
  182. func (runner *SpecRunner) registerForInterrupts(signalRegistered chan struct{}) {
  183. c := make(chan os.Signal, 1)
  184. signal.Notify(c, os.Interrupt, syscall.SIGTERM)
  185. close(signalRegistered)
  186. <-c
  187. signal.Stop(c)
  188. runner.markInterrupted()
  189. go runner.registerForHardInterrupts()
  190. runner.writer.DumpOutWithHeader(`
  191. Received interrupt. Emitting contents of GinkgoWriter...
  192. ---------------------------------------------------------
  193. `)
  194. if runner.afterSuiteNode != nil {
  195. fmt.Fprint(os.Stderr, `
  196. ---------------------------------------------------------
  197. Received interrupt. Running AfterSuite...
  198. ^C again to terminate immediately
  199. `)
  200. runner.runAfterSuite()
  201. }
  202. runner.reportSuiteDidEnd(false)
  203. os.Exit(1)
  204. }
  205. func (runner *SpecRunner) registerForHardInterrupts() {
  206. c := make(chan os.Signal, 1)
  207. signal.Notify(c, os.Interrupt, syscall.SIGTERM)
  208. <-c
  209. fmt.Fprintln(os.Stderr, "\nReceived second interrupt. Shutting down.")
  210. os.Exit(1)
  211. }
  212. func (runner *SpecRunner) blockForeverIfInterrupted() {
  213. runner.lock.Lock()
  214. interrupted := runner.interrupted
  215. runner.lock.Unlock()
  216. if interrupted {
  217. select {}
  218. }
  219. }
  220. func (runner *SpecRunner) markInterrupted() {
  221. runner.lock.Lock()
  222. defer runner.lock.Unlock()
  223. runner.interrupted = true
  224. }
  225. func (runner *SpecRunner) wasInterrupted() bool {
  226. runner.lock.Lock()
  227. defer runner.lock.Unlock()
  228. return runner.interrupted
  229. }
  230. func (runner *SpecRunner) reportSuiteWillBegin() {
  231. runner.startTime = time.Now()
  232. summary := runner.suiteWillBeginSummary()
  233. for _, reporter := range runner.reporters {
  234. reporter.SpecSuiteWillBegin(runner.config, summary)
  235. }
  236. }
  237. func (runner *SpecRunner) reportBeforeSuite(summary *types.SetupSummary) {
  238. for _, reporter := range runner.reporters {
  239. reporter.BeforeSuiteDidRun(summary)
  240. }
  241. }
  242. func (runner *SpecRunner) reportAfterSuite(summary *types.SetupSummary) {
  243. for _, reporter := range runner.reporters {
  244. reporter.AfterSuiteDidRun(summary)
  245. }
  246. }
  247. func (runner *SpecRunner) reportSpecWillRun(summary *types.SpecSummary) {
  248. runner.writer.Truncate()
  249. for _, reporter := range runner.reporters {
  250. reporter.SpecWillRun(summary)
  251. }
  252. }
  253. func (runner *SpecRunner) reportSpecDidComplete(summary *types.SpecSummary, failed bool) {
  254. if failed && len(summary.CapturedOutput) == 0 {
  255. summary.CapturedOutput = string(runner.writer.Bytes())
  256. }
  257. for i := len(runner.reporters) - 1; i >= 1; i-- {
  258. runner.reporters[i].SpecDidComplete(summary)
  259. }
  260. if failed {
  261. runner.writer.DumpOut()
  262. }
  263. runner.reporters[0].SpecDidComplete(summary)
  264. }
  265. func (runner *SpecRunner) reportSuiteDidEnd(success bool) {
  266. summary := runner.suiteDidEndSummary(success)
  267. summary.RunTime = time.Since(runner.startTime)
  268. for _, reporter := range runner.reporters {
  269. reporter.SpecSuiteDidEnd(summary)
  270. }
  271. }
  272. func (runner *SpecRunner) countSpecsThatRanSatisfying(filter func(ex *spec.Spec) bool) (count int) {
  273. count = 0
  274. for _, spec := range runner.processedSpecs {
  275. if filter(spec) {
  276. count++
  277. }
  278. }
  279. return count
  280. }
  281. func (runner *SpecRunner) suiteDidEndSummary(success bool) *types.SuiteSummary {
  282. numberOfSpecsThatWillBeRun := runner.countSpecsThatRanSatisfying(func(ex *spec.Spec) bool {
  283. return !ex.Skipped() && !ex.Pending()
  284. })
  285. numberOfPendingSpecs := runner.countSpecsThatRanSatisfying(func(ex *spec.Spec) bool {
  286. return ex.Pending()
  287. })
  288. numberOfSkippedSpecs := runner.countSpecsThatRanSatisfying(func(ex *spec.Spec) bool {
  289. return ex.Skipped()
  290. })
  291. numberOfPassedSpecs := runner.countSpecsThatRanSatisfying(func(ex *spec.Spec) bool {
  292. return ex.Passed()
  293. })
  294. numberOfFlakedSpecs := runner.countSpecsThatRanSatisfying(func(ex *spec.Spec) bool {
  295. return ex.Flaked()
  296. })
  297. numberOfFailedSpecs := runner.countSpecsThatRanSatisfying(func(ex *spec.Spec) bool {
  298. return ex.Failed()
  299. })
  300. if runner.beforeSuiteNode != nil && !runner.beforeSuiteNode.Passed() && !runner.config.DryRun {
  301. var known bool
  302. numberOfSpecsThatWillBeRun, known = runner.iterator.NumberOfSpecsThatWillBeRunIfKnown()
  303. if !known {
  304. numberOfSpecsThatWillBeRun = runner.iterator.NumberOfSpecsPriorToIteration()
  305. }
  306. numberOfFailedSpecs = numberOfSpecsThatWillBeRun
  307. }
  308. return &types.SuiteSummary{
  309. SuiteDescription: runner.description,
  310. SuiteSucceeded: success,
  311. SuiteID: runner.suiteID,
  312. NumberOfSpecsBeforeParallelization: runner.iterator.NumberOfSpecsPriorToIteration(),
  313. NumberOfTotalSpecs: len(runner.processedSpecs),
  314. NumberOfSpecsThatWillBeRun: numberOfSpecsThatWillBeRun,
  315. NumberOfPendingSpecs: numberOfPendingSpecs,
  316. NumberOfSkippedSpecs: numberOfSkippedSpecs,
  317. NumberOfPassedSpecs: numberOfPassedSpecs,
  318. NumberOfFailedSpecs: numberOfFailedSpecs,
  319. NumberOfFlakedSpecs: numberOfFlakedSpecs,
  320. }
  321. }
  322. func (runner *SpecRunner) suiteWillBeginSummary() *types.SuiteSummary {
  323. numTotal, known := runner.iterator.NumberOfSpecsToProcessIfKnown()
  324. if !known {
  325. numTotal = -1
  326. }
  327. numToRun, known := runner.iterator.NumberOfSpecsThatWillBeRunIfKnown()
  328. if !known {
  329. numToRun = -1
  330. }
  331. return &types.SuiteSummary{
  332. SuiteDescription: runner.description,
  333. SuiteID: runner.suiteID,
  334. NumberOfSpecsBeforeParallelization: runner.iterator.NumberOfSpecsPriorToIteration(),
  335. NumberOfTotalSpecs: numTotal,
  336. NumberOfSpecsThatWillBeRun: numToRun,
  337. NumberOfPendingSpecs: -1,
  338. NumberOfSkippedSpecs: -1,
  339. NumberOfPassedSpecs: -1,
  340. NumberOfFailedSpecs: -1,
  341. NumberOfFlakedSpecs: -1,
  342. }
  343. }