Mailspring/spec/n1-spec-reporter.coffee
Evan Morikawa c3ecca2692 feat(tests): add integration tests
comment

Adding test harness

Using key strokes in main window test

Tests work now

Clean up argument variables

Rename list manager and get rid of old spec-helper methods

Extract out time overrides from spec-helper

Spectron test for contenteditable

fix spec exit codes and boot mode

fix(spec): cleanup N1.sh and make specs fail with exit code 1

Revert tests and get it working in window

Move to spec_integration and add window load tester

Specs pass. Console logs still in

Remove console logs

Extract N1 Launcher ready method

Make integrated unit test runner

feat(tests): adding integration tests

Summary:
The /spectron folder got moved to /spec_integration

There are now unit tests (the old ones) run via the renamed
`script/grunt run-unit-tests`

There are now integration tests run via the command `script/grunt
run-integration-tests`.

There are two types of integration tests:
1. Tests that operate on the whole app via Selenium/Chromedriver. These
tests have access to Spectron APIs but do NOT have access to any JS object
running inside the application. See the `app-boot-spec.es6` for an example
of these tests. This is tricky because we want to test the Main window,
but Spectron may latch onto any other of our loading windows. Code in
`integration-helper` give us an API that finds and loads the main window
so we can test it

2. Tests that run in the unit test suite that need Spectron to perform
integration-like behavior. These are the contentedtiable specs. The
Spectron server is accessed from the app and can be used to trigger
actions on the running app, from the app. These tests use the
windowed-test runner so Spectron can identify whether the tests have
completed, passed, or failed. Unfortunately Spectron can't access the logs
, nor the exit code of the test script thereby forcing us to parse the
HTML DOM. (Note this is still a WIP)

I also revamped the `N1.sh` file when getting the launch arguments to work
properly. It's much cleaner. We didn't need most of the data.

Test Plan: new tests

Reviewers: juan, bengotow

Differential Revision: https://phab.nylas.com/D2289

Fix composer specs

Tests can properly detect when Spectron is in the environment

Report plain text output in specs

fixing contenteditable specs

Testing slow keymaps on contenteditable specs

Move to DOm mutation

Spell as `subtree` not `subTree`
2015-12-02 13:41:14 -08:00

317 lines
10 KiB
CoffeeScript

path = require 'path'
_ = require 'underscore'
_str = require 'underscore.string'
{convertStackTrace} = require 'coffeestack'
{View, $, $$} = require '../src/space-pen-extensions'
grim = require 'grim'
marked = require 'marked'
sourceMaps = {}
formatStackTrace = (spec, message='', stackTrace, indent="") ->
return stackTrace unless stackTrace
jasminePattern = /^\s*at\s+.*\(?.*[/\\]jasmine(-[^/\\]*)?\.js:\d+:\d+\)?\s*$/
firstJasmineLinePattern = /^\s*at [/\\].*[/\\]jasmine(-[^/\\]*)?\.js:\d+:\d+\)?\s*$/
convertedLines = []
for line in stackTrace.split('\n')
convertedLines.push(line) unless jasminePattern.test(line)
break if firstJasmineLinePattern.test(line)
stackTrace = convertStackTrace(convertedLines.join('\n'), sourceMaps)
lines = stackTrace.split('\n')
# Remove first line of stack when it is the same as the error message
errorMatch = lines[0]?.match(/^Error: (.*)/)
lines.shift() if message.trim() is errorMatch?[1]?.trim()
for line, index in lines
# Remove prefix of lines matching: at [object Object].<anonymous> (path:1:2)
prefixMatch = line.match(/at \[object Object\]\.<anonymous> \(([^)]+)\)/)
line = "at #{prefixMatch[1]}" if prefixMatch
# Relativize locations to spec directory
lines[index] = line.replace("at #{spec.specDirectory}#{path.sep}", 'at ')
lines = lines.map (line) -> indent + line.trim()
lines.join('\n')
module.exports =
class N1SpecReporter extends View
@content: ->
@div class: 'spec-reporter', =>
@div class: 'padded pull-right', =>
@button outlet: 'reloadButton', class: 'btn btn-small reload-button', 'Reload Specs'
@div outlet: 'coreArea', class: 'symbol-area', =>
@div outlet: 'coreHeader', class: 'symbol-header'
@ul outlet: 'coreSummary', class: 'symbol-summary list-unstyled'
@div outlet: 'bundledArea', class: 'symbol-area', =>
@div outlet: 'bundledHeader', class: 'symbol-header'
@ul outlet: 'bundledSummary', class: 'symbol-summary list-unstyled'
@div outlet: 'userArea', class: 'symbol-area', =>
@div outlet: 'userHeader', class: 'symbol-header'
@ul outlet: 'userSummary', class: 'symbol-summary list-unstyled'
@div outlet: "status", class: 'status alert alert-info', =>
@div outlet: "time", class: 'time'
@div outlet: "specCount", class: 'spec-count'
@div outlet: "message", class: 'message'
@div outlet: "results", class: 'results'
@div outlet: "deprecations", class: 'status alert alert-warning', style: 'display: none', =>
@span outlet: 'deprecationStatus', '0 deprecations'
@div class: 'deprecation-toggle'
@div outlet: 'deprecationList', class: 'deprecation-list'
@pre outlet: "plainTextOutput", class: 'plain-text-output'
startedAt: null
runningSpecCount: 0
completeSpecCount: 0
passedCount: 0
failedCount: 0
skippedCount: 0
totalSpecCount: 0
deprecationCount: 0
@timeoutId: 0
reportRunnerStarting: (runner) ->
@handleEvents()
@startedAt = Date.now()
specs = runner.specs()
@totalSpecCount = specs.length
@addSpecs(specs)
$(document.body).append this
@on 'click', '.stack-trace', ->
$(this).toggleClass('expanded')
@reloadButton.on 'click', -> require('electron').ipcRenderer.send('call-window-method', 'restart')
reportRunnerResults: (runner) ->
@updateSpecCounts()
@status.addClass('alert-success').removeClass('alert-info') if @failedCount is 0
if @failedCount is 1
@message.text "#{@failedCount} failure"
else
@message.text "#{@failedCount} failures"
@status.addClass("specs-complete")
reportSuiteResults: (suite) ->
reportSpecResults: (spec) ->
@completeSpecCount++
spec.endedAt = Date.now()
@specComplete(spec)
@updateStatusView(spec)
@reportPlainTextSpecResult(spec)
reportPlainTextSpecResult: (spec) ->
str = ""
if spec.results().failedCount > 0
str += @suiteString(spec) + "\n"
indent = @indentationString(spec.suite, 1)
stackIndent = @indentationString(spec.suite, 2)
description = spec.description
description = "it #{description}" if description.indexOf('it ') isnt 0
str += indent + description + "\n"
for result in spec.results().getItems()
continue if result.passed()
str += indent + result.message + "\n"
stackTrace = formatStackTrace(spec, result.message, result.trace.stack, stackIndent)
str += stackTrace + "\n"
str += "\n\n"
@plainTextOutput.append(str)
indentationString: (suite, plus=0) ->
rootSuite = suite
indentLevel = 0 + plus
while rootSuite.parentSuite
rootSuite = rootSuite.parentSuite
indentLevel += 1
return [0...indentLevel].map(-> " ").join("")
suiteString: (spec) ->
descriptions = [spec.suite.description]
rootSuite = spec.suite
while rootSuite.parentSuite
indent = @indentationString(rootSuite)
descriptions.unshift(indent + rootSuite.description)
rootSuite = rootSuite.parentSuite
descriptions.join("\n")
reportSpecStarting: (spec) ->
@specStarted(spec)
addDeprecations: (spec) ->
deprecations = grim.getDeprecations()
@deprecationCount += deprecations.length
@deprecations.show() if @deprecationCount > 0
if @deprecationCount is 1
@deprecationStatus.text("1 deprecation")
else
@deprecationStatus.text("#{@deprecationCount} deprecations")
for deprecation in deprecations
@deprecationList.append $$ ->
@div class: 'padded', =>
@div class: 'result-message fail deprecation-message', =>
@raw marked(deprecation.message)
for stack in deprecation.getStacks()
fullStack = stack.map ({functionName, location}) ->
if functionName is '<unknown>'
" at #{location}"
else
" at #{functionName} (#{location})"
@pre class: 'stack-trace padded', formatStackTrace(spec, deprecation.message, fullStack.join('\n'))
grim.clearDeprecations()
handleEvents: ->
$(document).on "click", ".spec-toggle", ({currentTarget}) ->
element = $(currentTarget)
specFailures = element.parent().find('.spec-failures')
specFailures.toggle()
element.toggleClass('folded')
false
$(document).on "click", ".deprecation-toggle", ({currentTarget}) ->
element = $(currentTarget)
deprecationList = $(document).find('.deprecation-list')
deprecationList.toggle()
element.toggleClass('folded')
false
updateSpecCounts: ->
if @skippedCount
specCount = "#{@completeSpecCount - @skippedCount}/#{@totalSpecCount - @skippedCount} (#{@skippedCount} skipped)"
else
specCount = "#{@completeSpecCount}/#{@totalSpecCount}"
@specCount[0].textContent = specCount
updateStatusView: (spec) ->
if @failedCount > 0
@status.addClass('alert-danger').removeClass('alert-info')
@updateSpecCounts()
rootSuite = spec.suite
rootSuite = rootSuite.parentSuite while rootSuite.parentSuite
@message.text rootSuite.description
time = "#{Math.round((spec.endedAt - @startedAt) / 10)}"
time = "0#{time}" if time.length < 3
@time[0].textContent = "#{time[0...-2]}.#{time[-2..]}s"
addSpecs: (specs) ->
coreSpecs = 0
bundledPackageSpecs = 0
userPackageSpecs = 0
for spec in specs
symbol = $$ -> @li id: "spec-summary-#{spec.id}", class: "spec-summary pending"
switch spec.specType
when 'core'
coreSpecs++
@coreSummary.append symbol
when 'bundled'
bundledPackageSpecs++
@bundledSummary.append symbol
when 'user'
userPackageSpecs++
@userSummary.append symbol
if coreSpecs > 0
@coreHeader.text("Core Specs (#{coreSpecs})")
else
@coreArea.hide()
if bundledPackageSpecs > 0
@bundledHeader.text("Bundled Package Specs (#{bundledPackageSpecs})")
else
@bundledArea.hide()
if userPackageSpecs > 0
if coreSpecs is 0 and bundledPackageSpecs is 0
# Package specs being run, show a more descriptive label
{specDirectory} = specs[0]
packageFolderName = path.basename(path.dirname(specDirectory))
packageName = _str.humanize(packageFolderName)
@userHeader.text("#{packageName} Specs")
else
@userHeader.text("User Package Specs (#{userPackageSpecs})")
else
@userArea.hide()
specStarted: (spec) ->
@runningSpecCount++
specComplete: (spec) ->
specSummaryElement = $("#spec-summary-#{spec.id}")
specSummaryElement.removeClass('pending')
specSummaryElement.attr('title', spec.getFullName())
results = spec.results()
if results.skipped
specSummaryElement.addClass("skipped")
@skippedCount++
else if results.passed()
specSummaryElement.addClass("passed")
@passedCount++
else
specSummaryElement.addClass("failed")
specView = new SpecResultView(spec)
specView.attach()
@failedCount++
@addDeprecations(spec)
class SuiteResultView extends View
@content: ->
@div class: 'suite', =>
@div outlet: 'description', class: 'description'
initialize: (@suite) ->
@attr('id', "suite-view-#{@suite.id}")
@description.text(@suite.description)
attach: ->
(@parentSuiteView() or $('.results')).append this
parentSuiteView: ->
return unless @suite.parentSuite
if not suiteView = $("#suite-view-#{@suite.parentSuite.id}").view()
suiteView = new SuiteResultView(@suite.parentSuite)
suiteView.attach()
suiteView
class SpecResultView extends View
@content: ->
@div class: 'spec', =>
@div class: 'spec-toggle'
@div outlet: 'description', class: 'description'
@div outlet: 'specFailures', class: 'spec-failures'
initialize: (@spec) ->
@addClass("spec-view-#{@spec.id}")
description = @spec.description
description = "it #{description}" if description.indexOf('it ') isnt 0
@description.text(description)
for result in @spec.results().getItems() when not result.passed()
stackTrace = formatStackTrace(@spec, result.message, result.trace.stack)
@specFailures.append $$ ->
@div result.message, class: 'result-message fail'
@pre stackTrace, class: 'stack-trace padded' if stackTrace
attach: ->
@parentSuiteView().append this
parentSuiteView: ->
if not suiteView = $("#suite-view-#{@spec.suite.id}").view()
suiteView = new SuiteResultView(@spec.suite)
suiteView.attach()
suiteView