fight flake in select2 test (#521)

Author: bahmutov

Development.md

@@ -107,3 +107,16 @@ Simply commit an empty set of changes with `git commit --allow-empty` and set th
 and push to the branch. The CircleCI will install Cypress from the above links and then will go through the CI run, allowing you to confirm that the new recipes are working.
 
 Once the new Cypress version is published, merge the branch.
+
+## Fighting flake
+
+If you notice a recipe test failing sometimes, run more of it! In [circle.yml](circle.yml) a job can take `repeat: N` parameter to run the recipe N times. Increase the number of times to run the recipe to flush out flaky tests and fix them.
+
+Example recipe to run 5 times
+
+```
+- testing-dom__select2:
+    requires:
+      - build
+    repeat: 5
+```

circle.yml

@@ -2,6 +2,11 @@ version: 2.1
 
 # testing jobs are all the same, so just use a template
 defaults: &defaults
+  parameters:
+    repeat:
+      description: Number of times to run this particular example
+      type: integer
+      default: 1
   parallelism: 1
   working_directory: ~/app
   docker:
@@ -26,11 +31,7 @@ defaults: &defaults
         # are not passed to the external PRs
         command: |
           cd examples/$CIRCLE_JOB
-          if [ -z "$CYPRESS_RECORD_KEY" ]; then
-            npm run cypress:run
-          else
-            npm run cypress:run -- --record --group $CIRCLE_JOB
-          fi
+          node ../../test-repeat -n << parameters.repeat >> --group $CIRCLE_JOB
         name: Cypress tests
 
 jobs:
@@ -451,6 +452,7 @@ all_jobs: &all_jobs
   - testing-dom__select2:
       requires:
         - build
+      repeat: 5
   - unit-testing__application-code:
       requires:
         - build

examples/server-communication__env-variables/.env

@@ -0,0 +1,3 @@
+FOO=42
+BAR=baz
+CYPRESS_ping=123

examples/server-communication__env-variables/README.md

@@ -2,6 +2,8 @@
 
 This recipe shows how to pass environment variables to your tests
 
-- See [package.json](package.json) which runs Cypress with environment variables set. The variables that start with `CYPRESS_` are extracted automatically. Other variables are copied from `process.env` in the script [cypress/plugins/index.js](cypress/plugins/index.js)
+- See [package.json](package.json) file which runs Cypress with environment variables set. The variables that start with `CYPRESS_` are extracted automatically. Other variables are copied from `process.env` in the script [cypress/plugins/index.js](cypress/plugins/index.js)
 - Additional variables can be passed via `env` object in [cypress.json](cypress.json)
 - Extract any other variable from `process.env` using `cypress/plugins/index.js` callback.
+
+Note: for testing scripts, the same environment variables set in `package.json` are also set in [.env](.env) file

examples/server-communication__xhr-assertions/cypress/integration/spok-spec.js

@@ -23,7 +23,7 @@ describe('network', () => {
       url: spok.endsWith('posts'),
       // network request takes at least 10ms
       // but should finish in less than 1 second
-      duration: spok.range(10, 1000),
+      duration: spok.range(10, 1500),
       statusMessage: spok.string,
       // check the request inside XHR object
       request: {

examples/testing-dom__select2/cypress/integration/spec.js

@@ -237,14 +237,31 @@ describe('select2', () => {
 
     it('selects a value by typing and selecting', () => {
       cy.server()
+      cy.route('https://jsonplaceholder.cypress.io/users?_type=query').as('query')
       cy.route('https://jsonplaceholder.cypress.io/users?term=clem&_type=query&q=clem').as('user_search')
 
       // first open the container, which makes the initial ajax call
       cy.get('#select2-user-container').click()
 
-      // then type into the input element to trigger search, and wait for results
-      cy.get('input[aria-controls="select2-user-results"]').type('clem{enter}')
+      // let's wait for Select2 widget to finish its full query
+      cy.wait('@query')
+
+      cy.get('.select2-results__option')
+      .should('not.have.class', 'loading-results')
+
+      // then type into the input element to trigger search
+      cy.get('input[aria-controls="select2-user-results"]')
+      .type('clem', { delay: 150 })
+
+      // and wait for results from XHR "query=clem"
       cy.wait('@user_search')
+      .its('responseBody.length').then((numberOfResults) => {
+        // make sure the Select2 has updated to show just
+        // the above number of items. Otherwise the next
+        // step might fail since the widget suddenly re-renders
+        // to show only the new results
+        cy.get('.select2-results__option').should('have.length', numberOfResults)
+      })
 
       // select a value, again by retrying command
       // https://on.cypress.io/retry-ability

package.json

@@ -108,6 +108,7 @@
     "cypress-react-unit-test": "4.9.0",
     "cypress-select-tests": "1.5.3",
     "cypress-wait-until": "1.7.1",
+    "dotenv": "8.2.0",
     "eslint": "6.0.1",
     "eslint-plugin-json-format": "2.0.1",
     "eslint-plugin-mocha": "5.0.0",

test-repeat.js

@@ -0,0 +1,68 @@
+/* eslint-disable no-console */
+// a small utility script to run Cypress N times
+// use (from examples/X folder)
+//    node ../../test-repeat.js -n 3
+// if you want to record, a good idea is to pass a group name
+//    node ../../test-repeat.js -n 3 --group my-example
+
+// if there is an .env file, lots it and add to process.env
+require('dotenv').config()
+
+const arg = require('arg')
+const cypress = require('cypress')
+const Bluebird = require('bluebird')
+
+const args = arg({
+  '-n': Number,
+  '--group': String,
+})
+
+const repeatNtimes = args['-n'] ? args['-n'] : 1
+
+console.log('will repeat Cypress run %d time(s)', repeatNtimes)
+
+const allRunOptions = []
+
+for (let k = 0; k < repeatNtimes; k += 1) {
+  const runOptions = {}
+
+  if (process.env.CYPRESS_RECORD_KEY) {
+    runOptions.record = true
+    runOptions.group = args['--group']
+
+    if (runOptions.group && repeatNtimes > 1) {
+      // make sure if we are repeating this example
+      // then the recording has group names on the Dashboard
+      // like "example-1-of-20", "example-2-of-20", ...
+      runOptions.group += `-${k + 1}-of-${repeatNtimes}`
+    }
+  }
+
+  allRunOptions.push(runOptions)
+}
+
+const onError = (err) => {
+  console.error(err)
+  process.exit(1)
+}
+
+Bluebird.mapSeries(allRunOptions, (runOptions, k, n) => {
+  console.log('***** %d of %d *****', k + 1, n)
+
+  const onTestResults = (testResults) => {
+    if (testResults.failures) {
+      console.error(testResults.message)
+      process.exit(testResults.failures)
+    }
+
+    if (testResults.totalFailed) {
+      console.error('run %d of %d failed', k + 1, n)
+      process.exit(testResults.totalFailed)
+    }
+  }
+
+  return cypress.run(runOptions).then(onTestResults)
+}).then(() => {
+  console.log('***** finished %d run(s) successfully *****', repeatNtimes)
+})
+.catch(onError)