test: expose boxplot and the extra stats test keeps track of

[breaking-change]

Author: erickt

src/compiletest/compiletest.rs

@@ -286,6 +286,9 @@ pub fn test_opts(config: &Config) -> test::TestOpts {
         test_shard: config.test_shard.clone(),
         nocapture: false,
         color: test::AutoColor,
+        show_boxplot: false,
+        boxplot_width: 50,
+        show_all_stats: false,
     }
 }
 

src/libtest/lib.rs

@@ -286,6 +286,9 @@ pub struct TestOpts {
     pub logfile: Option<Path>,
     pub nocapture: bool,
     pub color: ColorConfig,
+    pub show_boxplot: bool,
+    pub boxplot_width: uint,
+    pub show_all_stats: bool,
 }
 
 impl TestOpts {
@@ -303,6 +306,9 @@ impl TestOpts {
             logfile: None,
             nocapture: false,
             color: AutoColor,
+            show_boxplot: false,
+            boxplot_width: 50,
+            show_all_stats: false,
         }
     }
 }
@@ -333,7 +339,10 @@ fn optgroups() -> Vec<getopts::OptGroup> {
       getopts::optopt("", "color", "Configure coloring of output:
             auto   = colorize if stdout is a tty and tests are run on serially (default);
             always = always colorize output;
-            never  = never colorize output;", "auto|always|never"))
+            never  = never colorize output;", "auto|always|never"),
+      getopts::optflag("", "boxplot", "Display a boxplot of the benchmark statistics"),
+      getopts::optopt("", "boxplot-width", "Set the boxplot width (default 50)", "WIDTH"),
+      getopts::optflag("", "stats", "Display the benchmark min, max, and quartiles"))
 }
 
 fn usage(binary: &str) {
@@ -424,6 +433,21 @@ pub fn parse_opts(args: &[String]) -> Option<OptRes> {
                                             v))),
     };
 
+    let show_boxplot = matches.opt_present("boxplot");
+    let boxplot_width = match matches.opt_str("boxplot-width") {
+        Some(width) => {
+            match FromStr::from_str(width.as_slice()) {
+                Some(width) => width,
+                None => {
+                    return Some(Err(format!("argument for --boxplot-width must be a uint")));
+                }
+            }
+        }
+        None => 50,
+    };
+
+    let show_all_stats = matches.opt_present("stats");
+
     let test_opts = TestOpts {
         filter: filter,
         run_ignored: run_ignored,
@@ -436,6 +460,9 @@ pub fn parse_opts(args: &[String]) -> Option<OptRes> {
         logfile: logfile,
         nocapture: nocapture,
         color: color,
+        show_boxplot: show_boxplot,
+        boxplot_width: boxplot_width,
+        show_all_stats: show_all_stats,
     };
 
     Some(Ok(test_opts))
@@ -486,6 +513,9 @@ struct ConsoleTestState<T> {
     log_out: Option<File>,
     out: OutputLocation<T>,
     use_color: bool,
+    show_boxplot: bool,
+    boxplot_width: uint,
+    show_all_stats: bool,
     total: uint,
     passed: uint,
     failed: uint,
@@ -512,6 +542,9 @@ impl<T: Writer> ConsoleTestState<T> {
             out: out,
             log_out: log_out,
             use_color: use_color(opts),
+            show_boxplot: opts.show_boxplot,
+            boxplot_width: opts.boxplot_width,
+            show_all_stats: opts.show_all_stats,
             total: 0u,
             passed: 0u,
             failed: 0u,
@@ -607,8 +640,31 @@ impl<T: Writer> ConsoleTestState<T> {
             }
             TrBench(ref bs) => {
                 try!(self.write_bench());
-                self.write_plain(format!(": {}",
-                                         fmt_bench_samples(bs)).as_slice())
+
+                if self.show_boxplot {
+                    let mut wr = Vec::new();
+
+                    try!(stats::write_boxplot(&mut wr, &bs.ns_iter_summ, self.boxplot_width));
+
+                    let s = String::from_utf8(wr).unwrap();
+
+                    try!(self.write_plain(format!(": {}", s).as_slice()));
+                }
+
+                if self.show_all_stats {
+                    let mut wr = Vec::new();
+
+                    try!(stats::write_5_number_summary(&mut wr, &bs.ns_iter_summ));
+
+                    let s = String::from_utf8(wr).unwrap();
+
+                    try!(self.write_plain(format!(": {}", s).as_slice()));
+                } else {
+                    try!(self.write_plain(format!(": {}",
+                                                  fmt_bench_samples(bs)).as_slice()));
+                }
+
+                Ok(())
             }
         });
         self.write_plain("\n")
@@ -681,14 +737,14 @@ impl<T: Writer> ConsoleTestState<T> {
                 }
                 Improvement(pct) => {
                     improved += 1;
-                    try!(self.write_plain(format!(": {}", *k).as_slice()));
+                    try!(self.write_plain(format!(": {} ", *k).as_slice()));
                     try!(self.write_improved());
                     try!(self.write_plain(format!(" by {:.2}%\n",
                                                   pct as f64).as_slice()));
                 }
                 Regression(pct) => {
                     regressed += 1;
-                    try!(self.write_plain(format!(": {}", *k).as_slice()));
+                    try!(self.write_plain(format!(": {} ", *k).as_slice()));
                     try!(self.write_regressed());
                     try!(self.write_plain(format!(" by {:.2}%\n",
                                                   pct as f64).as_slice()));
@@ -860,6 +916,9 @@ fn should_sort_failures_before_printing_them() {
         log_out: None,
         out: Raw(Vec::new()),
         use_color: false,
+        show_boxplot: false,
+        boxplot_width: 0,
+        show_all_stats: false,
         total: 0u,
         passed: 0u,
         failed: 0u,

src/libtest/stats.rs

@@ -331,8 +331,8 @@ pub fn winsorize<T: Float + FromPrimitive>(samples: &mut [T], pct: T) {
 }
 
 /// Render writes the min, max and quartiles of the provided `Summary` to the provided `Writer`.
-pub fn write_5_number_summary<T: Float + Show>(w: &mut io::Writer,
-                                               s: &Summary<T>) -> io::IoResult<()> {
+pub fn write_5_number_summary<W: Writer, T: Float + Show>(w: &mut W,
+                                                          s: &Summary<T>) -> io::IoResult<()> {
     let (q1,q2,q3) = s.quartiles;
     write!(w, "(min={}, q1={}, med={}, q3={}, max={})",
                      s.min,
@@ -353,8 +353,8 @@ pub fn write_5_number_summary<T: Float + Show>(w: &mut io::Writer,
 /// ```{.ignore}
 ///   10 |        [--****#******----------]          | 40
 /// ```
-pub fn write_boxplot<T: Float + Show + FromPrimitive>(
-                     w: &mut io::Writer,
+pub fn write_boxplot<W: Writer, T: Float + Show + FromPrimitive>(
+                     w: &mut W,
                      s: &Summary<T>,
                      width_hint: uint)
                       -> io::IoResult<()> {
@@ -473,7 +473,7 @@ mod tests {
         let summ2 = Summary::new(samples);
 
         let mut w = io::stdout();
-        let w = &mut w as &mut io::Writer;
+        let w = &mut w;
         (write!(w, "\n")).unwrap();
         write_5_number_summary(w, &summ2).unwrap();
         (write!(w, "\n")).unwrap();
@@ -1028,7 +1028,7 @@ mod tests {
     fn test_boxplot_nonpositive() {
         fn t(s: &Summary<f64>, expected: String) {
             let mut m = Vec::new();
-            write_boxplot(&mut m as &mut io::Writer, s, 30).unwrap();
+            write_boxplot(&mut m, s, 30).unwrap();
             let out = String::from_utf8(m).unwrap();
             assert_eq!(out, expected);
         }