| 1 |
require 'builder' |
| 2 |
require 'fileutils' |
| 3 |
require 'erb' |
| 4 |
|
| 5 |
module Minitest |
| 6 |
module Reporters |
| 7 |
# A reporter for generating HTML test reports |
| 8 |
# This is recommended to be used with a CI server, where the report is kept as an artifact and is accessible via |
| 9 |
# a shared link |
| 10 |
# |
| 11 |
# The reporter sorts the results alphabetically and then by results |
| 12 |
# so that failing and skipped tests are at the top. |
| 13 |
# |
| 14 |
# When using Minitest Specs, the number prefix is dropped from the name of the test so that it reads well |
| 15 |
# |
| 16 |
# On each test run all files in the reports directory are deleted, this prevents a build up of old reports |
| 17 |
# |
| 18 |
# The report is generated using ERB. A custom ERB template can be provided but it is not required |
| 19 |
# The default ERB template uses JQuery and Bootstrap, both of these are included by referencing the CDN sites |
| 20 |
class HtmlReporter < BaseReporter |
| 21 |
# The title of the report |
| 22 |
attr_reader :title |
| 23 |
|
| 24 |
# The number of tests that passed |
| 25 |
def passes |
| 26 |
count - failures - errors - skips |
| 27 |
end |
| 28 |
|
| 29 |
# The percentage of tests that passed, calculated in a way that avoids rounding errors |
| 30 |
def percent_passes |
| 31 |
100 - percent_skips - percent_errors_failures |
| 32 |
end |
| 33 |
|
| 34 |
# The percentage of tests that were skipped |
| 35 |
def percent_skips |
| 36 |
(skips / count.to_f * 100).to_i |
| 37 |
end |
| 38 |
|
| 39 |
# Keeping old method name with typo for backwards compatibility in custom templates (for now) |
| 40 |
alias_method :percent_skipps, :percent_skips |
| 41 |
|
| 42 |
# The percentage of tests that failed |
| 43 |
def percent_errors_failures |
| 44 |
((errors + failures) / count.to_f * 100).to_i |
| 45 |
end |
| 46 |
|
| 47 |
# Trims off the number prefix on test names when using Minitest Specs |
| 48 |
def friendly_name(test) |
| 49 |
groups = test.name.scan(/(test_\d+_)(.*)/i) |
| 50 |
return test.name if groups.empty? |
| 51 |
"it #{groups[0][1]}" |
| 52 |
end |
| 53 |
|
| 54 |
# The constructor takes a hash, and uses the following keys: |
| 55 |
# :title - the title that will be used in the report, defaults to 'Test Results' |
| 56 |
# :reports_dir - the directory the reports should be written to, defaults to 'test/html_reports' |
| 57 |
# :erb_template - the path to a custom ERB template, defaults to the supplied ERB template |
| 58 |
# :mode - Useful for debugging, :terse suppresses errors and is the default, :verbose lets errors bubble up |
| 59 |
# :output_filename - the report's filename, defaults to 'index.html' |
| 60 |
def initialize(args = {}) |
| 61 |
super({}) |
| 62 |
|
| 63 |
defaults = { |
| 64 |
:title => 'Test Results', |
| 65 |
:erb_template => "#{File.dirname(__FILE__)}/../templates/index.html.erb", |
| 66 |
:reports_dir => ENV['MINITEST_HTML_REPORTS_DIR'] || 'test/html_reports', |
| 67 |
:mode => :safe, |
| 68 |
:output_filename => ENV['MINITEST_HTML_REPORTS_FILENAME'] || 'index.html', |
| 69 |
} |
| 70 |
|
| 71 |
settings = defaults.merge(args) |
| 72 |
|
| 73 |
@mode = settings[:mode] |
| 74 |
@title = settings[:title] |
| 75 |
@erb_template = settings[:erb_template] |
| 76 |
@output_filename = settings[:output_filename] |
| 77 |
reports_dir = settings[:reports_dir] |
| 78 |
|
| 79 |
@reports_path = File.absolute_path(reports_dir) |
| 80 |
end |
| 81 |
|
| 82 |
def start |
| 83 |
super |
| 84 |
|
| 85 |
puts "Emptying #{@reports_path}" |
| 86 |
FileUtils.mkdir_p(@reports_path) |
| 87 |
File.delete(html_file) if File.exist?(html_file) |
| 88 |
end |
| 89 |
|
| 90 |
# Called by the framework to generate the report |
| 91 |
def report |
| 92 |
super |
| 93 |
|
| 94 |
begin |
| 95 |
puts "Writing HTML reports to #{@reports_path}" |
| 96 |
erb_str = File.read(@erb_template) |
| 97 |
renderer = ERB.new(erb_str) |
| 98 |
|
| 99 |
tests_by_suites = tests.group_by { |test| test_class(test) } # taken from the JUnit reporter |
| 100 |
|
| 101 |
suites = tests_by_suites.map do |suite, tests| |
| 102 |
suite_summary = summarize_suite(suite, tests) |
| 103 |
suite_summary[:tests] = tests.sort { |a, b| compare_tests(a, b) } |
| 104 |
suite_summary |
| 105 |
end |
| 106 |
|
| 107 |
suites.sort! { |a, b| compare_suites(a, b) } |
| 108 |
|
| 109 |
result = renderer.result(binding) |
| 110 |
File.open(html_file, 'w') do |f| |
| 111 |
f.write(result) |
| 112 |
end |
| 113 |
|
| 114 |
# rubocop:disable Lint/RescueException |
| 115 |
rescue Exception => e |
| 116 |
puts 'There was an error writing the HTML report' |
| 117 |
puts 'This may have been caused by cancelling the test run' |
| 118 |
puts 'Use mode => :verbose in the HTML reporters constructor to see more detail' if @mode == :terse |
| 119 |
puts 'Use mode => :terse in the HTML reporters constructor to see less detail' if @mode != :terse |
| 120 |
raise e if @mode != :terse |
| 121 |
end |
| 122 |
# rubocop:enable Lint/RescueException |
| 123 |
end |
| 124 |
|
| 125 |
private |
| 126 |
|
| 127 |
def html_file |
| 128 |
"#{@reports_path}/#{@output_filename}" |
| 129 |
end |
| 130 |
|
| 131 |
def compare_suites_by_name(suite_a, suite_b) |
| 132 |
suite_a[:name] <=> suite_b[:name] |
| 133 |
end |
| 134 |
|
| 135 |
def compare_tests_by_name(test_a, test_b) |
| 136 |
friendly_name(test_a) <=> friendly_name(test_b) |
| 137 |
end |
| 138 |
|
| 139 |
# Test suites are first ordered by evaluating the results of the tests, then by test suite name |
| 140 |
# Test suites which have failing tests are given highest order |
| 141 |
# Tests suites which have skipped tests are given second highest priority |
| 142 |
def compare_suites(suite_a, suite_b) |
| 143 |
return compare_suites_by_name(suite_a, suite_b) if suite_a[:has_errors_or_failures] && suite_b[:has_errors_or_failures] |
| 144 |
return -1 if suite_a[:has_errors_or_failures] && !suite_b[:has_errors_or_failures] |
| 145 |
return 1 if !suite_a[:has_errors_or_failures] && suite_b[:has_errors_or_failures] |
| 146 |
|
| 147 |
return compare_suites_by_name(suite_a, suite_b) if suite_a[:has_skipps] && suite_b[:has_skipps] |
| 148 |
return -1 if suite_a[:has_skipps] && !suite_b[:has_skipps] |
| 149 |
return 1 if !suite_a[:has_skipps] && suite_b[:has_skipps] |
| 150 |
|
| 151 |
compare_suites_by_name(suite_a, suite_b) |
| 152 |
end |
| 153 |
|
| 154 |
# Tests are first ordered by evaluating the results of the tests, then by tests names |
| 155 |
# Tess which fail are given highest order |
| 156 |
# Tests which are skipped are given second highest priority |
| 157 |
def compare_tests(test_a, test_b) |
| 158 |
return compare_tests_by_name(test_a, test_b) if test_fail_or_error?(test_a) && test_fail_or_error?(test_b) |
| 159 |
|
| 160 |
return -1 if test_fail_or_error?(test_a) && !test_fail_or_error?(test_b) |
| 161 |
return 1 if !test_fail_or_error?(test_a) && test_fail_or_error?(test_b) |
| 162 |
|
| 163 |
return compare_tests_by_name(test_a, test_b) if test_a.skipped? && test_b.skipped? |
| 164 |
return -1 if test_a.skipped? && !test_b.skipped? |
| 165 |
return 1 if !test_a.skipped? && test_b.skipped? |
| 166 |
|
| 167 |
compare_tests_by_name(test_a, test_b) |
| 168 |
end |
| 169 |
|
| 170 |
def test_fail_or_error?(test) |
| 171 |
test.error? || test.failure |
| 172 |
end |
| 173 |
|
| 174 |
# based on analyze_suite from the JUnit reporter |
| 175 |
def summarize_suite(suite, tests) |
| 176 |
summary = Hash.new(0) |
| 177 |
summary[:name] = suite.to_s |
| 178 |
tests.each do |test| |
| 179 |
summary[:"#{result(test)}_count"] += 1 |
| 180 |
summary[:assertion_count] += test.assertions |
| 181 |
summary[:test_count] += 1 |
| 182 |
summary[:time] += test.time |
| 183 |
end |
| 184 |
summary[:has_errors_or_failures] = (summary[:fail_count] + summary[:error_count]) > 0 |
| 185 |
summary[:has_skipps] = summary[:skip_count] > 0 |
| 186 |
summary |
| 187 |
end |
| 188 |
|
| 189 |
# based on message_for(test) from the JUnit reporter |
| 190 |
def message_for(test) |
| 191 |
suite = test.class |
| 192 |
name = test.name |
| 193 |
e = test.failure |
| 194 |
|
| 195 |
if test.passed? |
| 196 |
nil |
| 197 |
elsif test.skipped? |
| 198 |
"Skipped:\n#{name}(#{suite}) [#{location(e)}]:\n#{e.message}\n" |
| 199 |
elsif test.failure |
| 200 |
"Failure:\n#{name}(#{suite}) [#{location(e)}]:\n#{e.message}\n" |
| 201 |
elsif test.error? |
| 202 |
"Error:\n#{name}(#{suite}):\n#{e.message}" |
| 203 |
end |
| 204 |
end |
| 205 |
|
| 206 |
# taken from the JUnit reporter |
| 207 |
def location(exception) |
| 208 |
last_before_assertion = '' |
| 209 |
exception.backtrace.reverse_each do |s| |
| 210 |
break if s =~ /in .(assert|refute|flunk|pass|fail|raise|must|wont)/ |
| 211 |
last_before_assertion = s |
| 212 |
end |
| 213 |
last_before_assertion.sub(/:in .*$/, '') |
| 214 |
end |
| 215 |
|
| 216 |
def total_time_to_hms |
| 217 |
return ('%.2fs' % total_time) if total_time < 1 |
| 218 |
|
| 219 |
hours = (total_time / (60 * 60)).round |
| 220 |
minutes = ((total_time / 60) % 60).round.to_s.rjust(2, '0') |
| 221 |
seconds = (total_time % 60).round.to_s.rjust(2, '0') |
| 222 |
|
| 223 |
"#{hours}h#{minutes}m#{seconds}s" |
| 224 |
end |
| 225 |
end |
| 226 |
end |
| 227 |
end |