mirror of
https://github.com/zebrajr/react.git
synced 2026-01-15 12:15:22 +00:00
844ca8b6b2
This uses wall-clock time (for now) so it's noisier than alternatives
(cachegrind, CPU perf-counters), but it's still valuable. In a future diff we
can make it use those.
`measure.py` outputs something that `analyze.py` can understand, but you can use `analyze.py` without `measure.py` too. The file format is simple:
```
$ cat measurements.txt
factory_ms_jsc_jit 13.580322265625
factory_ms_jsc_jit 13.659912109375
factory_ms_jsc_jit 13.67919921875
factory_ms_jsc_nojit 12.827880859375
factory_ms_jsc_nojit 13.105224609375
factory_ms_jsc_nojit 13.195068359375
factory_ms_node 40.4891400039196
factory_ms_node 40.6669420003891
factory_ms_node 43.52413299679756
ssr_pe_cold_ms_jsc_jit 43.06005859375
...
```
(The lines do not need to be sorted.)
Comparing 0.14.0 vs master:
```
$ ./measure.py react-0.14.0.min.js >014.txt
Measuring SSR for PE benchmark (30 trials)
..............................
Measuring SSR for PE with warm JIT (3 slow trials)
...
$ ./measure.py react.min.js >master.txt
Measuring SSR for PE benchmark (30 trials)
..............................
Measuring SSR for PE with warm JIT (3 slow trials)
...
$ ./analyze.py 014.txt master.txt
Comparing 014.txt (control) vs master.txt (test)
Significant differences marked by ***
% change from control to test, with 99% CIs:
* factory_ms_jsc_jit
% change: -0.56% [ -2.51%, +1.39%]
means: 14.037 (control), 13.9593 (test)
* factory_ms_jsc_nojit
% change: +1.23% [ -1.18%, +3.64%]
means: 13.2586 (control), 13.4223 (test)
* factory_ms_node
% change: +3.53% [ +0.29%, +6.77%] ***
means: 42.0529 (control), 43.54 (test)
* ssr_pe_cold_ms_jsc_jit
% change: -6.84% [ -9.04%, -4.65%] ***
means: 44.2444 (control), 41.2187 (test)
* ssr_pe_cold_ms_jsc_nojit
% change: -11.81% [-14.66%, -8.96%] ***
means: 52.9449 (control), 46.6953 (test)
* ssr_pe_cold_ms_node
% change: -2.70% [ -4.52%, -0.88%] ***
means: 96.8909 (control), 94.2741 (test)
* ssr_pe_warm_ms_jsc_jit
% change: -17.60% [-22.04%, -13.16%] ***
means: 13.763 (control), 11.3439 (test)
* ssr_pe_warm_ms_jsc_nojit
% change: -20.65% [-22.62%, -18.68%] ***
means: 30.8829 (control), 24.5074 (test)
* ssr_pe_warm_ms_node
% change: -8.76% [-13.48%, -4.03%] ***
means: 30.0193 (control), 27.3964 (test)
$
```
112 lines
3.3 KiB
Python
Executable File
112 lines
3.3 KiB
Python
Executable File
#!/usr/bin/env python
|
|
# Copyright 2015, Facebook, Inc.
|
|
# All rights reserved.
|
|
#
|
|
# This source code is licensed under the BSD-style license found in the
|
|
# LICENSE file in the root directory of this source tree. An additional grant
|
|
# of patent rights can be found in the PATENTS file in the same directory.
|
|
|
|
import math
|
|
import sys
|
|
|
|
import numpy as np
|
|
import numpy.random as npr
|
|
import scipy.stats
|
|
|
|
|
|
def _bootstrap_mean_sem(samples):
|
|
"""Return the estimated standard error for a distribution's mean."""
|
|
samples = np.array(samples)
|
|
n = len(samples)
|
|
indices = npr.randint(0, n, (10000, n))
|
|
samples = samples[indices]
|
|
means = np.sort(np.mean(samples, axis=1))
|
|
return np.std(means, ddof=1)
|
|
|
|
|
|
def _read_measurements(f):
|
|
"""Read measurements from a file.
|
|
|
|
Returns {'a': [1.0, 2.0, 3.0], 'b': [5.0, 5.0, 5.0]} for a file containing
|
|
the six lines: ['a 1', 'a 2', 'a 3', 'b 5', 'b 5', 'b 5'].
|
|
"""
|
|
measurements = {}
|
|
for line in f:
|
|
label, value = line.split(None, 1)
|
|
measurements.setdefault(label, []).append(float(value))
|
|
return measurements
|
|
|
|
|
|
def _compute_mean_and_sd_of_ratio_from_delta_method(
|
|
mean_test,
|
|
sem_test,
|
|
mean_control,
|
|
sem_control
|
|
):
|
|
mean = (
|
|
((mean_test - mean_control) / mean_control) -
|
|
(pow(sem_control, 2) * mean_test / pow(mean_control, 3))
|
|
)
|
|
var = (
|
|
pow(sem_test / mean_control, 2) +
|
|
(pow(sem_control * mean_test, 2) / pow(mean_control, 4))
|
|
)
|
|
return (mean, math.sqrt(var))
|
|
|
|
|
|
def _main():
|
|
if len(sys.argv) != 3:
|
|
sys.stderr.write("usage: analyze.py control.txt test.txt\n")
|
|
return 1
|
|
|
|
ci_size = 0.99
|
|
p_value = scipy.stats.norm.ppf(0.5 * (1 + ci_size))
|
|
|
|
control, test = sys.argv[1:]
|
|
with open(control) as f:
|
|
control_measurements = _read_measurements(f)
|
|
with open(test) as f:
|
|
test_measurements = _read_measurements(f)
|
|
keys = set()
|
|
keys.update(control_measurements.iterkeys())
|
|
keys.update(test_measurements.iterkeys())
|
|
|
|
print "Comparing %s (control) vs %s (test)" % (control, test)
|
|
print "Significant differences marked by ***"
|
|
print "%% change from control to test, with %g%% CIs:" % (ci_size * 100,)
|
|
print
|
|
|
|
any_sig = False
|
|
for key in sorted(keys):
|
|
print "* %s" % (key,)
|
|
control_nums = control_measurements.get(key, [])
|
|
test_nums = test_measurements.get(key, [])
|
|
if not control_nums or not test_nums:
|
|
print " skipping..."
|
|
continue
|
|
|
|
mean_control = np.mean(control_nums)
|
|
mean_test = np.mean(test_nums)
|
|
sem_control = _bootstrap_mean_sem(control_nums)
|
|
sem_test = _bootstrap_mean_sem(test_nums)
|
|
|
|
rat_mean, rat_sem = _compute_mean_and_sd_of_ratio_from_delta_method(
|
|
mean_test, sem_test, mean_control, sem_control
|
|
)
|
|
rat_low = rat_mean - p_value * rat_sem
|
|
rat_high = rat_mean + p_value * rat_sem
|
|
|
|
sig = rat_high < 0 or rat_low > 0
|
|
any_sig = any_sig or sig
|
|
|
|
print " %% change: %+6.2f%% [%+6.2f%%, %+6.2f%%]%s" % (
|
|
100 * rat_mean,
|
|
100 * rat_low,
|
|
100 * rat_high,
|
|
' ***' if sig else ''
|
|
)
|
|
print " means: %g (control), %g (test)" % (mean_control, mean_test)
|
|
|
|
if __name__ == '__main__':
|
|
sys.exit(_main())
|