Debian Codegraph @

lib/minitest/reporters/mean_time_reporter.rb

raw
1 require 'minitest/reporters'
2 require 'tmpdir'
3 require 'yaml'
4
5 module Minitest
6 module Reporters
7 # This reporter creates a report providing the average (mean), minimum and
8 # maximum times for a test to run. Running this for all your tests will
9 # allow you to:
10 #
11 # 1) Identify the slowest running tests over time as potential candidates
12 # for improvements or refactoring.
13 # 2) Identify (and fix) regressions in test run speed caused by changes to
14 # your tests or algorithms in your code.
15 # 3) Provide an abundance of statistics to enjoy.
16 #
17 # This is achieved by creating a (configurable) 'previous runs' statistics
18 # file which is parsed at the end of each run to provide a new
19 # (configurable) report. These statistics can be reset at any time by using
20 # a simple rake task:
21 #
22 # rake reset_statistics
23 #
24 class MeanTimeReporter < Minitest::Reporters::DefaultReporter
25 class InvalidOrder < StandardError; end
26 class InvalidSortColumn < StandardError; end
27
28 # Reset the statistics file for this reporter. Called via a rake task:
29 #
30 # rake reset_statistics
31 #
32 # @return [Boolean]
33 def self.reset_statistics!
34 new.reset_statistics!
35 end
36
37 # @param options [Hash]
38 # @option previous_runs_filename [String] Contains the times for each test
39 # by description. Defaults to '/tmp/minitest_reporters_previous_run'.
40 # @option report_filename [String] Contains the parsed results for the
41 # last test run. Defaults to '/tmp/minitest_reporters_report'.
42 # @option show_count [Fixnum] The number of tests to show in the report
43 # summary at the end of the test run. Default is 15.
44 # @option show_progress [Boolean] If true it prints pass/skip/fail marks.
45 # Default is true.
46 # @option show_all_runs [Boolean] If true it shows all recorded suit results.
47 # Default is true.
48 # @option sort_column [Symbol] One of :avg (default), :min, :max, :last.
49 # Determines the column by which the report summary is sorted.
50 # @option order [Symbol] One of :desc (default), or :asc. By default the
51 # report summary is listed slowest to fastest (:desc). :asc will order
52 # the report summary as fastest to slowest.
53 # @return [Minitest::Reporters::MeanTimeReporter]
54 def initialize(options = {})
55 super
56
57 @all_suite_times = []
58 end
59
60 # Copies the suite times from the
61 # {Minitest::Reporters::DefaultReporter#after_suite} method, making them
62 # available to this class.
63 #
64 # @return [Hash<String => Float>]
65 def after_suite(suite)
66 super
67
68 @all_suite_times = @suite_times
69 end
70
71 # Runs the {Minitest::Reporters::DefaultReporter#report} method and then
72 # enhances it by storing the results to the 'previous_runs_filename' and
73 # outputs the parsed results to both the 'report_filename' and the
74 # terminal.
75 #
76 def report
77 super
78
79 create_or_update_previous_runs!
80
81 create_new_report!
82
83 write_to_screen!
84 end
85
86 def on_start
87 super if options[:show_progress]
88 end
89
90 def on_record(test)
91 super if options[:show_progress]
92 end
93
94 def on_report
95 super if options[:show_progress]
96 end
97
98 # Resets the 'previous runs' file, essentially removing all previous
99 # statistics gathered.
100 #
101 # @return [void]
102 def reset_statistics!
103 File.delete(previous_runs_filename) if File.exist?(previous_runs_filename)
104 end
105
106 protected
107
108 attr_accessor :all_suite_times
109
110 private
111
112 # @return [Hash<String => Float>]
113 def current_run
114 Hash[all_suite_times]
115 end
116
117 # @return [Hash] Sets default values for the filenames used by this class,
118 # and the number of tests to output to output to the screen after each
119 # run.
120 def defaults
121 {
122 order: :desc,
123 show_count: 15,
124 show_progress: true,
125 show_all_runs: true,
126 sort_column: :avg,
127 previous_runs_filename: File.join(Dir.tmpdir, 'minitest_reporters_previous_run'),
128 report_filename: File.join(Dir.tmpdir, 'minitest_reporters_report'),
129 }
130 end
131
132 # Added to the top of the report file and to the screen output.
133 #
134 # @return [String]
135 def report_title
136 "\n\e[4mMinitest Reporters: Mean Time Report\e[24m " \
137 "(Samples: #{samples}, Order: #{sort_column.inspect} " \
138 "#{order.inspect})\n"
139 end
140
141 # The report itself. Displays statistics about all runs, ideal for use
142 # with the Unix 'head' command. Listed in slowest average descending
143 # order.
144 #
145 # @return [String]
146 def report_body
147 order_sorted_body.each_with_object([]) do |result, obj|
148 rating = rate(result[:last], result[:min], result[:max])
149
150 obj << "#{avg_label} #{result[:avg].to_s.ljust(12)} " \
151 "#{min_label} #{result[:min].to_s.ljust(12)} " \
152 "#{max_label} #{result[:max].to_s.ljust(12)} " \
153 "#{run_label(rating)} #{result[:last].to_s.ljust(12)} " \
154 "#{des_label} #{result[:desc]}\n"
155 end.join
156 end
157
158 # @return [String] All of the column-sorted results sorted by the :order
159 # option. (Defaults to :desc).
160 def order_sorted_body
161 if desc?
162 column_sorted_body.reverse
163
164 elsif asc?
165 column_sorted_body
166
167 end
168 end
169
170 # @return [Array<Hash<Symbol => String>>] All of the results sorted by
171 # the :sort_column option. (Defaults to :avg).
172 def column_sorted_body
173 runs = options[:show_all_runs] ? previous_run : current_run
174 runs.keys.each_with_object([]) do |description, obj|
175 timings = previous_run[description]
176 size = Array(timings).size
177 sum = Array(timings).inject { |total, x| total + x }
178 obj << {
179 avg: (sum / size).round(9),
180 min: Array(timings).min.round(9),
181 max: Array(timings).max.round(9),
182 last: Array(timings).last.round(9),
183 desc: description,
184 }
185 end.sort_by { |k| k[sort_column] }
186 end
187
188 # @return [Hash]
189 def options
190 defaults.merge!(@options)
191 end
192
193 # @return [Fixnum] The number of tests to output to output to the screen
194 # after each run.
195 def show_count
196 options[:show_count]
197 end
198
199 # @return [Hash<String => Array<Float>]
200 def previous_run
201 @previous_run ||= YAML.load_file(previous_runs_filename)
202 end
203
204 # @return [String] The path to the file which contains all the durations
205 # for each test run. The previous runs file is in YAML format, using the
206 # test name for the key and an array containing the time taken to run
207 # this test for values.
208 def previous_runs_filename
209 options[:previous_runs_filename]
210 end
211
212 # Returns a boolean indicating whether a previous runs file exists.
213 #
214 # @return [Boolean]
215 def previously_ran?
216 File.exist?(previous_runs_filename)
217 end
218
219 # @return [String] The path to the file which contains the parsed test
220 # results. The results file contains a line for each test with the
221 # average time of the test, the minimum time the test took to run,
222 # the maximum time the test took to run and a description of the test
223 # (which is the test name as emitted by Minitest).
224 def report_filename
225 options[:report_filename]
226 end
227
228 # A barbaric way to find out how many runs are in the previous runs file;
229 # this method takes the first test listed, and counts its samples
230 # trusting (naively) all runs to be the same number of samples. This will
231 # produce incorrect averages when new tests are added, so it is advised
232 # to restart the statistics by removing the 'previous runs' file. A rake
233 # task is provided to make this more convenient.
234 #
235 # rake reset_statistics
236 #
237 # @return [Fixnum]
238 def samples
239 return 1 unless previous_run.first[1].is_a?(Array)
240
241 previous_run.first[1].size
242 end
243
244 # Creates a new 'previous runs' file, or updates the existing one with
245 # the latest timings.
246 #
247 # @return [void]
248 def create_or_update_previous_runs!
249 if previously_ran?
250 current_run.each do |description, elapsed|
251 new_times = if previous_run[description.to_s]
252 Array(previous_run[description.to_s]) << elapsed
253 else
254 Array(elapsed)
255 end
256
257 previous_run.store(description.to_s, new_times)
258 end
259
260 File.write(previous_runs_filename, previous_run.to_yaml)
261
262 else
263
264 File.write(previous_runs_filename, current_run.to_yaml)
265
266 end
267 end
268
269 # Creates a new report file in the 'report_filename'. This file contains
270 # a line for each test of the following example format: (this is a single
271 # line despite explicit wrapping)
272 #
273 # Avg: 0.0555555 Min: 0.0498765 Max: 0.0612345 Last: 0.0499421
274 # Description: The test name
275 #
276 # Note however the timings are to 9 decimal places, and padded to 12
277 # characters and each label is coloured, Avg (yellow), Min (green),
278 # Max (red), Last (multi), and Description (blue). It looks pretty!
279 #
280 # The 'Last' label is special in that it will be colour coded depending
281 # on whether the last run was faster (bright green) or slower (bright red)
282 # or inconclusive (purple). This helps to identify changes on a per run
283 # basis.
284 #
285 # @return [void]
286 def create_new_report!
287 File.write(report_filename, report_title + report_body)
288 end
289
290 # Writes a number of tests (configured via the 'show_count' option) to the
291 # screen after creating the report. See '#create_new_report!' for example
292 # output information.
293 #
294 # @return [void]
295 def write_to_screen!
296 puts report_title
297 puts report_body.lines.take(show_count)
298 end
299
300 # @return [String] A yellow 'Avg:' label.
301 def avg_label
302 ANSI::Code.yellow('Avg:')
303 end
304
305 # @return [String] A blue 'Description:' label.
306 def des_label
307 ANSI::Code.blue('Description:')
308 end
309
310 # @return [String] A red 'Max:' label.
311 def max_label
312 ANSI::Code.red('Max:')
313 end
314
315 # @return [String] A green 'Min:' label.
316 def min_label
317 ANSI::Code.green('Min:')
318 end
319
320 # @param rating [Symbol] One of :faster, :slower or :inconclusive.
321 # @return [String] A purple 'Last:' label.
322 def run_label(rating)
323 case rating
324 when :faster then ANSI::Code.green('Last:')
325 when :slower then ANSI::Code.red('Last:')
326 else
327 ANSI::Code.magenta('Last:')
328 end
329 end
330
331 # @param run [Float] The last run time.
332 # @param min [Float] The minimum run time.
333 # @param max [Float] The maximum run time.
334 # @return [Symbol] One of :faster, :slower or :inconclusive.
335 def rate(run, min, max)
336 if run == min
337 :faster
338 elsif run == max
339 :slower
340 else
341 :inconclusive
342 end
343 end
344
345 # @return [Boolean] Whether the given :order option is :asc.
346 def asc?
347 order == :asc
348 end
349
350 # @return [Boolean] Whether the given :order option is :desc (default).
351 def desc?
352 order == :desc
353 end
354
355 # @raise [Minitest::Reporters::MeanTimeReporter::InvalidOrder]
356 # When the given :order option is invalid.
357 # @return [Symbol] The :order option, or by default; :desc.
358 def order
359 orders = [:desc, :asc]
360
361 if orders.include?(options[:order])
362 options[:order]
363
364 else
365 fail Minitest::Reporters::MeanTimeReporter::InvalidOrder,
366 "`:order` option must be one of #{orders.inspect}."
367
368 end
369 end
370
371 # @raise [Minitest::Reporters::MeanTimeReporter::InvalidSortColumn]
372 # When the given :sort_column option is invalid.
373 # @return [Symbol] The :sort_column option, or by default; :avg.
374 def sort_column
375 sort_columns = [:avg, :min, :max, :last]
376
377 if sort_columns.include?(options[:sort_column])
378 options[:sort_column]
379
380 else
381 fail Minitest::Reporters::MeanTimeReporter::InvalidSortColumn,
382 "`:sort_column` option must be one of #{sort_columns.inspect}."
383
384 end
385 end
386 end
387 end
388 end