From 55e3387e2d7f0f37ae78af1794343d279b99a2d1 Mon Sep 17 00:00:00 2001 From: Lukas Heinrich Date: Sat, 25 Jan 2020 13:53:53 +0100 Subject: [PATCH 01/24] hmm --- src/pyhf/infer/__init__.py | 26 +++++++++++++++--- src/pyhf/infer/utils.py | 55 ++++++-------------------------------- 2 files changed, 30 insertions(+), 51 deletions(-) diff --git a/src/pyhf/infer/__init__.py b/src/pyhf/infer/__init__.py index 598e34db0a..e6b371b305 100644 --- a/src/pyhf/infer/__init__.py +++ b/src/pyhf/infer/__init__.py @@ -2,11 +2,11 @@ from .test_statistics import qmu from .utils import ( - generate_asimov_data, pvals_from_teststat, pvals_from_teststat_expected, ) from .. import get_backend +from .calculators import AsymptoticTestStatDistribution, AsymptoticCalculator, generate_asimov_data def hypotest( @@ -81,21 +81,39 @@ def hypotest( asimov_mu = 0.0 asimov_data = generate_asimov_data(asimov_mu, data, pdf, init_pars, par_bounds) + + qmu_v = qmu(poi_test, data, pdf, init_pars, par_bounds) sqrtqmu_v = tensorlib.sqrt(qmu_v) qmuA_v = qmu(poi_test, asimov_data, pdf, init_pars, par_bounds) sqrtqmuA_v = tensorlib.sqrt(qmuA_v) - CLsb, CLb, CLs = pvals_from_teststat(sqrtqmu_v, sqrtqmuA_v, qtilde=qtilde) + calc = AsymptoticCalculator(data,pdf,init_pars,par_bounds,qtilde=qtilde) + sb_dist, b_dist = calc.distributions(poi_test) + teststat = calc.teststatistic(poi_test) + + sqrtqmuA_v = calc.sqrtqmuA_v + + + CLsb = sb_dist.pvalue(teststat) + CLb = b_dist.pvalue(teststat) + CLs = CLsb / CLb + CLsb, CLb, CLs = ( + tensorlib.reshape(CLsb, (1,)), + tensorlib.reshape(CLb, (1,)), + tensorlib.reshape(CLs, (1,)), + ) _returns = [CLs] if kwargs.get('return_tail_probs'): _returns.append([CLsb, CLb]) if kwargs.get('return_expected_set'): CLs_exp = [] - for n_sigma in [-2, -1, 0, 1, 2]: - CLs_exp.append(pvals_from_teststat_expected(sqrtqmuA_v, nsigma=n_sigma)[-1]) + for n_sigma in [2,1,0,-1,-2]: + CLs = sb_dist.pvalue(n_sigma)/b_dist.pvalue(n_sigma) + v = tensorlib.reshape(CLs,(1,)) + CLs_exp.append(v) CLs_exp = tensorlib.astensor(CLs_exp) if kwargs.get('return_expected'): _returns.append(CLs_exp[2]) diff --git a/src/pyhf/infer/utils.py b/src/pyhf/infer/utils.py index 5a44cccebd..003c2e8378 100644 --- a/src/pyhf/infer/utils.py +++ b/src/pyhf/infer/utils.py @@ -1,15 +1,13 @@ """Utility Functions for model inference.""" from .. import get_backend from .mle import fixed_poi_fit +from .test_statistics import qmu -def generate_asimov_data(asimov_mu, data, pdf, init_pars, par_bounds): - """Compute Asimov Dataset (expected yields at best-fit values) for a given POI value.""" - bestfit_nuisance_asimov = fixed_poi_fit(asimov_mu, data, pdf, init_pars, par_bounds) - return pdf.expected_data(bestfit_nuisance_asimov) - - -def pvals_from_teststat(sqrtqmu_v, sqrtqmuA_v, qtilde=False): +def pvals_from_teststat( + sqrtqmu_v, + poi_test, data, pdf, init_pars, par_bounds, + qtilde=False): r""" Compute p-values from test-statistic values. @@ -36,38 +34,9 @@ def pvals_from_teststat(sqrtqmu_v, sqrtqmuA_v, qtilde=False): Tuple of Floats: The :math:`p`-values for the signal + background, background only, and signal only hypotheses respectivley """ - tensorlib, _ = get_backend() - if not qtilde: # qmu - teststat = sqrtqmu_v - sqrtqmuA_v - else: # qtilde - - def _true_case(): - teststat = sqrtqmu_v - sqrtqmuA_v - return teststat - - def _false_case(): - qmu = tensorlib.power(sqrtqmu_v, 2) - qmu_A = tensorlib.power(sqrtqmuA_v, 2) - teststat = (qmu - qmu_A) / (2 * sqrtqmuA_v) - return teststat - - teststat = tensorlib.conditional( - (sqrtqmu_v < sqrtqmuA_v), _true_case, _false_case - ) - - nullval = teststat + sqrtqmuA_v - altval = teststat - - CLsb = 1 - tensorlib.normal_cdf(nullval) - CLb = 1 - tensorlib.normal_cdf(altval) - CLs = CLsb / CLb - return ( - tensorlib.reshape(CLsb, (1,)), - tensorlib.reshape(CLb, (1,)), - tensorlib.reshape(CLs, (1,)), - ) - + raise RuntimeError() +from .calculators import AsymptoticTestStatDistribution, AsymptoticCalculator def pvals_from_teststat_expected(sqrtqmuA_v, nsigma=0): r""" Compute the expected :math:`p`-values CLsb, CLb and CLs for data corresponding to a given percentile of the alternate hypothesis. @@ -87,12 +56,4 @@ def pvals_from_teststat_expected(sqrtqmuA_v, nsigma=0): # was used. However, we can make a shortcut by just computing the p-values in mu^/sigma # space, where the p-values are Clsb = cdf(x-sqrt(lambda)) and CLb=cdf(x) - tensorlib, _ = get_backend() - CLsb = tensorlib.normal_cdf(nsigma - sqrtqmuA_v) - CLb = tensorlib.normal_cdf(nsigma) - CLs = CLsb / CLb - return ( - tensorlib.reshape(CLsb, (1,)), - tensorlib.reshape(CLb, (1,)), - tensorlib.reshape(CLs, (1,)), - ) + raise RuntimeError() From 2ad406a3766ea5c69930fc4c102c74f92768a527 Mon Sep 17 00:00:00 2001 From: Lukas Heinrich Date: Sat, 25 Jan 2020 14:21:39 +0100 Subject: [PATCH 02/24] works out --- src/pyhf/infer/__init__.py | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/src/pyhf/infer/__init__.py b/src/pyhf/infer/__init__.py index e6b371b305..7e5a354e31 100644 --- a/src/pyhf/infer/__init__.py +++ b/src/pyhf/infer/__init__.py @@ -81,14 +81,6 @@ def hypotest( asimov_mu = 0.0 asimov_data = generate_asimov_data(asimov_mu, data, pdf, init_pars, par_bounds) - - - qmu_v = qmu(poi_test, data, pdf, init_pars, par_bounds) - sqrtqmu_v = tensorlib.sqrt(qmu_v) - - qmuA_v = qmu(poi_test, asimov_data, pdf, init_pars, par_bounds) - sqrtqmuA_v = tensorlib.sqrt(qmuA_v) - calc = AsymptoticCalculator(data,pdf,init_pars,par_bounds,qtilde=qtilde) sb_dist, b_dist = calc.distributions(poi_test) teststat = calc.teststatistic(poi_test) @@ -119,7 +111,10 @@ def hypotest( _returns.append(CLs_exp[2]) _returns.append(CLs_exp) elif kwargs.get('return_expected'): - _returns.append(pvals_from_teststat_expected(sqrtqmuA_v)[-1]) + n_sigma = 0 + CLs = sb_dist.pvalue(n_sigma)/b_dist.pvalue(n_sigma) + v = tensorlib.reshape(CLs,(1,)) + _returns.append(v) # Enforce a consistent return type of the observed CLs return tuple(_returns) if len(_returns) > 1 else _returns[0] From 71d40d1b5e9372e884ffcbb4182c9e94e6e46079 Mon Sep 17 00:00:00 2001 From: Lukas Heinrich Date: Sat, 25 Jan 2020 14:32:53 +0100 Subject: [PATCH 03/24] adjust test --- tests/test_validation.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/test_validation.py b/tests/test_validation.py index 19859fdb27..0b992e47e1 100644 --- a/tests/test_validation.py +++ b/tests/test_validation.py @@ -189,11 +189,11 @@ def expected_result_1bin_normsys(mu=1.0): if mu == 1: expected_result = { "exp": [ - 7.471694618861785e-10, + 7.471684419037561e-10, 5.7411551509088054e-08, - 3.6898088058290313e-06, - 0.000169657315363677, - 0.004392708998183163, + 3.6898088062731205e-06, + 0.00016965731538267896, + 0.004392708998555453 ], "obs": 0.0006735317023683173, } @@ -701,6 +701,10 @@ def validate_hypotest(pdf, data, mu_test, expected_result, tolerance=1e-6): return_expected_set=True, qtilde=False, ) + import numpy as np + for x in CLs_exp_set: + for y in x: + print(y) assert abs(CLs_obs - expected_result['obs']) / expected_result['obs'] < tolerance for result, expected in zip(CLs_exp_set, expected_result['exp']): From f92f1f20a4cb906b78c00bc6ee5b4070ffb368be Mon Sep 17 00:00:00 2001 From: Lukas Heinrich Date: Sat, 25 Jan 2020 14:34:34 +0100 Subject: [PATCH 04/24] adjust test --- src/pyhf/infer/__init__.py | 19 +++++++++++-------- src/pyhf/infer/utils.py | 8 +++++--- tests/test_validation.py | 3 ++- 3 files changed, 18 insertions(+), 12 deletions(-) diff --git a/src/pyhf/infer/__init__.py b/src/pyhf/infer/__init__.py index 7e5a354e31..e5553b93a1 100644 --- a/src/pyhf/infer/__init__.py +++ b/src/pyhf/infer/__init__.py @@ -6,7 +6,11 @@ pvals_from_teststat_expected, ) from .. import get_backend -from .calculators import AsymptoticTestStatDistribution, AsymptoticCalculator, generate_asimov_data +from .calculators import ( + AsymptoticTestStatDistribution, + AsymptoticCalculator, + generate_asimov_data, +) def hypotest( @@ -81,13 +85,12 @@ def hypotest( asimov_mu = 0.0 asimov_data = generate_asimov_data(asimov_mu, data, pdf, init_pars, par_bounds) - calc = AsymptoticCalculator(data,pdf,init_pars,par_bounds,qtilde=qtilde) + calc = AsymptoticCalculator(data, pdf, init_pars, par_bounds, qtilde=qtilde) sb_dist, b_dist = calc.distributions(poi_test) teststat = calc.teststatistic(poi_test) sqrtqmuA_v = calc.sqrtqmuA_v - CLsb = sb_dist.pvalue(teststat) CLb = b_dist.pvalue(teststat) CLs = CLsb / CLb @@ -102,9 +105,9 @@ def hypotest( _returns.append([CLsb, CLb]) if kwargs.get('return_expected_set'): CLs_exp = [] - for n_sigma in [2,1,0,-1,-2]: - CLs = sb_dist.pvalue(n_sigma)/b_dist.pvalue(n_sigma) - v = tensorlib.reshape(CLs,(1,)) + for n_sigma in [2, 1, 0, -1, -2]: + CLs = sb_dist.pvalue(n_sigma) / b_dist.pvalue(n_sigma) + v = tensorlib.reshape(CLs, (1,)) CLs_exp.append(v) CLs_exp = tensorlib.astensor(CLs_exp) if kwargs.get('return_expected'): @@ -112,8 +115,8 @@ def hypotest( _returns.append(CLs_exp) elif kwargs.get('return_expected'): n_sigma = 0 - CLs = sb_dist.pvalue(n_sigma)/b_dist.pvalue(n_sigma) - v = tensorlib.reshape(CLs,(1,)) + CLs = sb_dist.pvalue(n_sigma) / b_dist.pvalue(n_sigma) + v = tensorlib.reshape(CLs, (1,)) _returns.append(v) # Enforce a consistent return type of the observed CLs return tuple(_returns) if len(_returns) > 1 else _returns[0] diff --git a/src/pyhf/infer/utils.py b/src/pyhf/infer/utils.py index 003c2e8378..686f18a939 100644 --- a/src/pyhf/infer/utils.py +++ b/src/pyhf/infer/utils.py @@ -5,9 +5,8 @@ def pvals_from_teststat( - sqrtqmu_v, - poi_test, data, pdf, init_pars, par_bounds, - qtilde=False): + sqrtqmu_v, poi_test, data, pdf, init_pars, par_bounds, qtilde=False +): r""" Compute p-values from test-statistic values. @@ -36,7 +35,10 @@ def pvals_from_teststat( """ raise RuntimeError() + from .calculators import AsymptoticTestStatDistribution, AsymptoticCalculator + + def pvals_from_teststat_expected(sqrtqmuA_v, nsigma=0): r""" Compute the expected :math:`p`-values CLsb, CLb and CLs for data corresponding to a given percentile of the alternate hypothesis. diff --git a/tests/test_validation.py b/tests/test_validation.py index 0b992e47e1..c6df8d16fc 100644 --- a/tests/test_validation.py +++ b/tests/test_validation.py @@ -193,7 +193,7 @@ def expected_result_1bin_normsys(mu=1.0): 5.7411551509088054e-08, 3.6898088062731205e-06, 0.00016965731538267896, - 0.004392708998555453 + 0.004392708998555453, ], "obs": 0.0006735317023683173, } @@ -702,6 +702,7 @@ def validate_hypotest(pdf, data, mu_test, expected_result, tolerance=1e-6): qtilde=False, ) import numpy as np + for x in CLs_exp_set: for y in x: print(y) From 383acabb1a5805c60fff0ac26c40fcdbc2b056cd Mon Sep 17 00:00:00 2001 From: Lukas Heinrich Date: Sat, 25 Jan 2020 14:36:07 +0100 Subject: [PATCH 05/24] adjust test --- src/pyhf/infer/__init__.py | 4 --- src/pyhf/infer/utils.py | 61 -------------------------------------- 2 files changed, 65 deletions(-) delete mode 100644 src/pyhf/infer/utils.py diff --git a/src/pyhf/infer/__init__.py b/src/pyhf/infer/__init__.py index e5553b93a1..a3fffa5646 100644 --- a/src/pyhf/infer/__init__.py +++ b/src/pyhf/infer/__init__.py @@ -1,10 +1,6 @@ """Inference for Statistical Models.""" from .test_statistics import qmu -from .utils import ( - pvals_from_teststat, - pvals_from_teststat_expected, -) from .. import get_backend from .calculators import ( AsymptoticTestStatDistribution, diff --git a/src/pyhf/infer/utils.py b/src/pyhf/infer/utils.py deleted file mode 100644 index 686f18a939..0000000000 --- a/src/pyhf/infer/utils.py +++ /dev/null @@ -1,61 +0,0 @@ -"""Utility Functions for model inference.""" -from .. import get_backend -from .mle import fixed_poi_fit -from .test_statistics import qmu - - -def pvals_from_teststat( - sqrtqmu_v, poi_test, data, pdf, init_pars, par_bounds, qtilde=False -): - r""" - Compute p-values from test-statistic values. - - The :math:`p`-values for signal strength :math:`\mu` and Asimov strength :math:`\mu'` as defined in Equations (59) and (57) of :xref:`arXiv:1007.1727` - - .. math:: - - p_{\mu} = 1-F\left(q_{\mu}\middle|\mu'\right) = 1- \Phi\left(q_{\mu} - \frac{\left(\mu-\mu'\right)}{\sigma}\right) - - with Equation (29) - - .. math:: - - \frac{(\mu-\mu')}{\sigma} = \sqrt{\Lambda}= \sqrt{q_{\mu,A}} - - given the observed test statistics :math:`q_{\mu}` and :math:`q_{\mu,A}`. - - Args: - sqrtqmu_v (Number or Tensor): The root of the calculated test statistic, :math:`\sqrt{q_{\mu}}` - sqrtqmuA_v (Number or Tensor): The root of the calculated test statistic given the Asimov data, :math:`\sqrt{q_{\mu,A}}` - qtilde (Bool): When ``True`` perform the calculation using the alternative test statistic, :math:`\tilde{q}`, as defined in Equation (62) of :xref:`arXiv:1007.1727` - - Returns: - Tuple of Floats: The :math:`p`-values for the signal + background, background only, and signal only hypotheses respectivley - - """ - raise RuntimeError() - - -from .calculators import AsymptoticTestStatDistribution, AsymptoticCalculator - - -def pvals_from_teststat_expected(sqrtqmuA_v, nsigma=0): - r""" - Compute the expected :math:`p`-values CLsb, CLb and CLs for data corresponding to a given percentile of the alternate hypothesis. - - Args: - sqrtqmuA_v (Number or Tensor): The root of the calculated test statistic given the Asimov data, :math:`\sqrt{q_{\mu,A}}` - nsigma (Number or Tensor): The number of standard deviations of variations of the signal strength from the background only hypothesis :math:`\left(\mu=0\right)` - - Returns: - Tuple of Floats: The :math:`p`-values for the signal + background, background only, and signal only hypotheses respectivley - - """ - # NOTE: - # To compute the expected p-value, one would need to first compute a hypothetical - # observed test-statistic for a dataset whose best-fit value is mu^ = mu'-n*sigma: - # $q_n$, and the proceed with the normal p-value computation for whatever test-statistic - # was used. However, we can make a shortcut by just computing the p-values in mu^/sigma - # space, where the p-values are Clsb = cdf(x-sqrt(lambda)) and CLb=cdf(x) - - raise RuntimeError() From 9d328513f5996fdb4d252fa50aaad273bb8b71f9 Mon Sep 17 00:00:00 2001 From: Lukas Heinrich Date: Sat, 25 Jan 2020 14:36:59 +0100 Subject: [PATCH 06/24] adjust test --- tests/test_validation.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tests/test_validation.py b/tests/test_validation.py index c6df8d16fc..2133529928 100644 --- a/tests/test_validation.py +++ b/tests/test_validation.py @@ -701,12 +701,6 @@ def validate_hypotest(pdf, data, mu_test, expected_result, tolerance=1e-6): return_expected_set=True, qtilde=False, ) - import numpy as np - - for x in CLs_exp_set: - for y in x: - print(y) - assert abs(CLs_obs - expected_result['obs']) / expected_result['obs'] < tolerance for result, expected in zip(CLs_exp_set, expected_result['exp']): assert abs(result - expected) / expected < tolerance From 3eaa6f5da5621a3cd61f729948716a57c6b6a2c7 Mon Sep 17 00:00:00 2001 From: Lukas Heinrich Date: Sat, 25 Jan 2020 14:37:29 +0100 Subject: [PATCH 07/24] adjust test --- src/pyhf/infer/calculators.py | 66 +++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 src/pyhf/infer/calculators.py diff --git a/src/pyhf/infer/calculators.py b/src/pyhf/infer/calculators.py new file mode 100644 index 0000000000..a876dc3e83 --- /dev/null +++ b/src/pyhf/infer/calculators.py @@ -0,0 +1,66 @@ +from .mle import fixed_poi_fit +from .. import get_backend +from .test_statistics import qmu + + +def generate_asimov_data(asimov_mu, data, pdf, init_pars, par_bounds): + """Compute Asimov Dataset (expected yields at best-fit values) for a given POI value.""" + bestfit_nuisance_asimov = fixed_poi_fit(asimov_mu, data, pdf, init_pars, par_bounds) + return pdf.expected_data(bestfit_nuisance_asimov) + + +class AsymptoticTestStatDistribution(object): + def __init__(self, shift): + self.shift = shift + + def pvalue(self, value): + tensorlib, _ = get_backend() + return 1 - tensorlib.normal_cdf(value - self.shift) + + def expected_value(self, nsigma): + return nsigma + + +class AsymptoticCalculator(object): + def __init__(self, data, pdf, init_pars=None, par_bounds=None, qtilde=False): + self.data = data + self.pdf = pdf + self.init_pars = init_pars or pdf.config.suggested_init() + self.par_bounds = par_bounds or pdf.config.suggested_bounds() + self.qtilde = qtilde + + def distributions(self, poi_test): + tensorlib, _ = get_backend() + asimov_mu = 0.0 + asimov_data = generate_asimov_data( + asimov_mu, self.data, self.pdf, self.init_pars, self.par_bounds + ) + qmuA_v = qmu(poi_test, asimov_data, self.pdf, self.init_pars, self.par_bounds) + self.sqrtqmuA_v = tensorlib.sqrt(qmuA_v) + + sb_dist = AsymptoticTestStatDistribution(-self.sqrtqmuA_v) + b_dist = AsymptoticTestStatDistribution(0.0) + return sb_dist, b_dist + + def teststatistic(self, poi_test): + tensorlib, _ = get_backend() + qmu_v = qmu(poi_test, self.data, self.pdf, self.init_pars, self.par_bounds) + sqrtqmu_v = tensorlib.sqrt(qmu_v) + if not self.qtilde: # qmu + teststat = sqrtqmu_v - self.sqrtqmuA_v + else: # qtilde + + def _true_case(): + teststat = sqrtqmu_v - self.sqrtqmuA_v + return teststat + + def _false_case(): + qmu = tensorlib.power(sqrtqmu_v, 2) + qmu_A = tensorlib.power(self.sqrtqmuA_v, 2) + teststat = (qmu - qmu_A) / (2 * self.sqrtqmuA_v) + return teststat + + teststat = tensorlib.conditional( + (sqrtqmu_v < self.sqrtqmuA_v), _true_case, _false_case + ) + return teststat From 8aa7aadb32556032c1734ff736e087e2e6e5ebdd Mon Sep 17 00:00:00 2001 From: Lukas Heinrich Date: Sat, 25 Jan 2020 14:45:54 +0100 Subject: [PATCH 08/24] adjust test --- src/pyhf/infer/__init__.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/pyhf/infer/__init__.py b/src/pyhf/infer/__init__.py index a3fffa5646..ce8e3ee360 100644 --- a/src/pyhf/infer/__init__.py +++ b/src/pyhf/infer/__init__.py @@ -85,8 +85,6 @@ def hypotest( sb_dist, b_dist = calc.distributions(poi_test) teststat = calc.teststatistic(poi_test) - sqrtqmuA_v = calc.sqrtqmuA_v - CLsb = sb_dist.pvalue(teststat) CLb = b_dist.pvalue(teststat) CLs = CLsb / CLb From 881e7e55e0c9ed9c32830feb341ddb5d969e6c71 Mon Sep 17 00:00:00 2001 From: Lukas Heinrich Date: Sat, 25 Jan 2020 14:52:30 +0100 Subject: [PATCH 09/24] adjust test --- src/pyhf/infer/__init__.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/src/pyhf/infer/__init__.py b/src/pyhf/infer/__init__.py index ce8e3ee360..9aa00d7ef3 100644 --- a/src/pyhf/infer/__init__.py +++ b/src/pyhf/infer/__init__.py @@ -2,11 +2,7 @@ from .test_statistics import qmu from .. import get_backend -from .calculators import ( - AsymptoticTestStatDistribution, - AsymptoticCalculator, - generate_asimov_data, -) +from .calculators import AsymptoticCalculator def hypotest( @@ -78,9 +74,6 @@ def hypotest( par_bounds = par_bounds or pdf.config.suggested_bounds() tensorlib, _ = get_backend() - asimov_mu = 0.0 - asimov_data = generate_asimov_data(asimov_mu, data, pdf, init_pars, par_bounds) - calc = AsymptoticCalculator(data, pdf, init_pars, par_bounds, qtilde=qtilde) sb_dist, b_dist = calc.distributions(poi_test) teststat = calc.teststatistic(poi_test) From d12d85d0841274a15ab5552f86d022879a4f32e3 Mon Sep 17 00:00:00 2001 From: Lukas Heinrich Date: Sat, 25 Jan 2020 15:54:30 +0100 Subject: [PATCH 10/24] docstrings --- src/pyhf/infer/__init__.py | 2 +- src/pyhf/infer/calculators.py | 94 ++++++++++++++++++++++++++++++++--- 2 files changed, 88 insertions(+), 8 deletions(-) diff --git a/src/pyhf/infer/__init__.py b/src/pyhf/infer/__init__.py index 9aa00d7ef3..9ad1b58a1f 100644 --- a/src/pyhf/infer/__init__.py +++ b/src/pyhf/infer/__init__.py @@ -75,8 +75,8 @@ def hypotest( tensorlib, _ = get_backend() calc = AsymptoticCalculator(data, pdf, init_pars, par_bounds, qtilde=qtilde) - sb_dist, b_dist = calc.distributions(poi_test) teststat = calc.teststatistic(poi_test) + sb_dist, b_dist = calc.distributions(poi_test) CLsb = sb_dist.pvalue(teststat) CLb = b_dist.pvalue(teststat) diff --git a/src/pyhf/infer/calculators.py b/src/pyhf/infer/calculators.py index a876dc3e83..023abbf762 100644 --- a/src/pyhf/infer/calculators.py +++ b/src/pyhf/infer/calculators.py @@ -1,3 +1,12 @@ +""" +Calculators for Hypothesis Testing. + +The role of the calculators is to compute test statistic and +provide distributions of said test statistic under various +hypotheses. + +Using the calculators hypothesis tests can then be performed. +""" from .mle import fixed_poi_fit from .. import get_backend from .test_statistics import qmu @@ -10,19 +19,70 @@ def generate_asimov_data(asimov_mu, data, pdf, init_pars, par_bounds): class AsymptoticTestStatDistribution(object): + """ + The distribution the test statistic in the asymptotic case. + + Note: These distributions are in -µ^/sigma space. In the ROOT + implementation the same sigma is assumed for both hypotheses + and p-values etc are computed in that space. This assumption + is necessarily valid, but we keep this for compatibility + reasons. + + In the -µ^/sigma space, the test statistic (i.e. µ^/sigma) is + normally distributed with unit variance and its mean at + the -µ', where µ' is the true poi value of the hypothesis. + """ + def __init__(self, shift): + """ + Asymptotic test statistic distribution. + + Args: + shift: the displacement of the test statistic distribus + + """ self.shift = shift def pvalue(self, value): + """ + Compute the p-value for a given value of the test statistic. + + Args: + value: the test statistic value. + + Returns; + pvalue (float): the integrated probability to observe + a value at least as large as the observed one. + + """ tensorlib, _ = get_backend() return 1 - tensorlib.normal_cdf(value - self.shift) def expected_value(self, nsigma): + """ + Return the expected value of the test statistic. + + Args: + nsigma: number of standard deviations. + + Returns; + expected value (float): the expected value of the test statistic. + + """ return nsigma class AsymptoticCalculator(object): + """The Asymptotic Calculator.""" + def __init__(self, data, pdf, init_pars=None, par_bounds=None, qtilde=False): + """ + Asymptotic Calculator. + + Args: + data: data + + """ self.data = data self.pdf = pdf self.init_pars = init_pars or pdf.config.suggested_init() @@ -30,22 +90,42 @@ def __init__(self, data, pdf, init_pars=None, par_bounds=None, qtilde=False): self.qtilde = qtilde def distributions(self, poi_test): - tensorlib, _ = get_backend() - asimov_mu = 0.0 - asimov_data = generate_asimov_data( - asimov_mu, self.data, self.pdf, self.init_pars, self.par_bounds - ) - qmuA_v = qmu(poi_test, asimov_data, self.pdf, self.init_pars, self.par_bounds) - self.sqrtqmuA_v = tensorlib.sqrt(qmuA_v) + """ + Probability Distributions of the test statistic value under the signal + background and and background-only hypothesis. + Args: + poi_test: the value for the parameter of interest. + + Returns + distributions (Tuple of `AsymptoticTestStatDistribution`): the distributions under the hypotheses. + + """ sb_dist = AsymptoticTestStatDistribution(-self.sqrtqmuA_v) b_dist = AsymptoticTestStatDistribution(0.0) return sb_dist, b_dist def teststatistic(self, poi_test): + """ + Compute the test statistic for the observed data under the studied model. + + Args: + poi_test: the value for the parameter of interest. + + Returns: + test statistic (Float): the value of the test statistic. + + """ tensorlib, _ = get_backend() qmu_v = qmu(poi_test, self.data, self.pdf, self.init_pars, self.par_bounds) sqrtqmu_v = tensorlib.sqrt(qmu_v) + + asimov_mu = 0.0 + asimov_data = generate_asimov_data( + asimov_mu, self.data, self.pdf, self.init_pars, self.par_bounds + ) + qmuA_v = qmu(poi_test, asimov_data, self.pdf, self.init_pars, self.par_bounds) + self.sqrtqmuA_v = tensorlib.sqrt(qmuA_v) + if not self.qtilde: # qmu teststat = sqrtqmu_v - self.sqrtqmuA_v else: # qtilde From b722809e81dae6325b01c36ac421324b054efbab Mon Sep 17 00:00:00 2001 From: Lukas Heinrich Date: Sat, 25 Jan 2020 17:46:27 +0100 Subject: [PATCH 11/24] docstrings --- docs/api.rst | 6 +++--- src/pyhf/infer/calculators.py | 15 ++++++++++++--- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index d2665483f7..4b2d821350 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -122,9 +122,9 @@ Inference mle.twice_nll mle.fit mle.fixed_poi_fit - utils.generate_asimov_data - utils.pvals_from_teststat - utils.pvals_from_teststat_expected + calculators.generate_asimov_data + calculators.AsymptoticTestStatDistribution + calculators.AsymptoticCalculator Exceptions ---------- diff --git a/src/pyhf/infer/calculators.py b/src/pyhf/infer/calculators.py index 023abbf762..b0baf1a8c9 100644 --- a/src/pyhf/infer/calculators.py +++ b/src/pyhf/infer/calculators.py @@ -38,7 +38,10 @@ def __init__(self, shift): Asymptotic test statistic distribution. Args: - shift: the displacement of the test statistic distribus + shift: the displacement of the test statistic distribution + + Returns: + distribution """ self.shift = shift @@ -80,8 +83,14 @@ def __init__(self, data, pdf, init_pars=None, par_bounds=None, qtilde=False): Asymptotic Calculator. Args: - data: data - + data: the observed data + pdf: the statistical model + init_pars: the initial parameters to be used for fitting + par_bounds: the parameter bounds used for fitting + + Returns: + calculator + """ self.data = data self.pdf = pdf From ec18211d4e96d0590182ecd9a2aea433e89cab7d Mon Sep 17 00:00:00 2001 From: Matthew Feickert Date: Fri, 31 Jan 2020 17:43:00 -0600 Subject: [PATCH 12/24] Apply formatting suggestions --- src/pyhf/infer/calculators.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pyhf/infer/calculators.py b/src/pyhf/infer/calculators.py index b0baf1a8c9..a0bc9d01e5 100644 --- a/src/pyhf/infer/calculators.py +++ b/src/pyhf/infer/calculators.py @@ -53,7 +53,7 @@ def pvalue(self, value): Args: value: the test statistic value. - Returns; + Returns: pvalue (float): the integrated probability to observe a value at least as large as the observed one. @@ -68,7 +68,7 @@ def expected_value(self, nsigma): Args: nsigma: number of standard deviations. - Returns; + Returns: expected value (float): the expected value of the test statistic. """ @@ -106,7 +106,7 @@ def distributions(self, poi_test): poi_test: the value for the parameter of interest. Returns - distributions (Tuple of `AsymptoticTestStatDistribution`): the distributions under the hypotheses. + distributions (Tuple of ~pyhf.infer.calculators.AsymptoticTestStatDistribution): the distributions under the hypotheses. """ sb_dist = AsymptoticTestStatDistribution(-self.sqrtqmuA_v) From c1bbd54e9bb8d614fa47d9dd59974d59f96d7a1d Mon Sep 17 00:00:00 2001 From: Matthew Feickert Date: Fri, 31 Jan 2020 17:46:20 -0600 Subject: [PATCH 13/24] Combine reshape and append to same line --- src/pyhf/infer/__init__.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/pyhf/infer/__init__.py b/src/pyhf/infer/__init__.py index 9ad1b58a1f..91ceef1f2d 100644 --- a/src/pyhf/infer/__init__.py +++ b/src/pyhf/infer/__init__.py @@ -94,8 +94,7 @@ def hypotest( CLs_exp = [] for n_sigma in [2, 1, 0, -1, -2]: CLs = sb_dist.pvalue(n_sigma) / b_dist.pvalue(n_sigma) - v = tensorlib.reshape(CLs, (1,)) - CLs_exp.append(v) + CLs_exp.append(tensorlib.reshape(CLs, (1,))) CLs_exp = tensorlib.astensor(CLs_exp) if kwargs.get('return_expected'): _returns.append(CLs_exp[2]) @@ -103,8 +102,7 @@ def hypotest( elif kwargs.get('return_expected'): n_sigma = 0 CLs = sb_dist.pvalue(n_sigma) / b_dist.pvalue(n_sigma) - v = tensorlib.reshape(CLs, (1,)) - _returns.append(v) + _returns.append(tensorlib.reshape(CLs, (1,))) # Enforce a consistent return type of the observed CLs return tuple(_returns) if len(_returns) > 1 else _returns[0] From 7f0fd94032e3f2a97c7edac7dcca98fb7f1000c3 Mon Sep 17 00:00:00 2001 From: Matthew Feickert Date: Fri, 31 Jan 2020 17:55:45 -0600 Subject: [PATCH 14/24] use math mode LaTeX in docstrings --- src/pyhf/infer/calculators.py | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/src/pyhf/infer/calculators.py b/src/pyhf/infer/calculators.py index a0bc9d01e5..9c83ff20c1 100644 --- a/src/pyhf/infer/calculators.py +++ b/src/pyhf/infer/calculators.py @@ -2,7 +2,7 @@ Calculators for Hypothesis Testing. The role of the calculators is to compute test statistic and -provide distributions of said test statistic under various +provide distributions of said test statistic under various hypotheses. Using the calculators hypothesis tests can then be performed. @@ -22,15 +22,14 @@ class AsymptoticTestStatDistribution(object): """ The distribution the test statistic in the asymptotic case. - Note: These distributions are in -µ^/sigma space. In the ROOT - implementation the same sigma is assumed for both hypotheses - and p-values etc are computed in that space. This assumption - is necessarily valid, but we keep this for compatibility - reasons. + Note: These distributions are in :math:`-\hat{\mu}/\sigma` space. + In the ROOT implementation the same sigma is assumed for both hypotheses + and :math:`p`-values etc are computed in that space. + This assumption is necessarily valid, but we keep this for compatibility reasons. - In the -µ^/sigma space, the test statistic (i.e. µ^/sigma) is - normally distributed with unit variance and its mean at - the -µ', where µ' is the true poi value of the hypothesis. + In the :math:`-\hat{\mu}/\sigma` space, the test statistic (i.e. :math:`\hat{\mu}/\sigma`) is + normally distributed with unit variance and its mean at + the :math:`-\mu'`, where :math:`\mu'` is the true poi value of the hypothesis. """ def __init__(self, shift): @@ -42,14 +41,14 @@ def __init__(self, shift): Returns: distribution - + """ self.shift = shift def pvalue(self, value): """ Compute the p-value for a given value of the test statistic. - + Args: value: the test statistic value. @@ -70,7 +69,7 @@ def expected_value(self, nsigma): Returns: expected value (float): the expected value of the test statistic. - + """ return nsigma @@ -90,7 +89,7 @@ def __init__(self, data, pdf, init_pars=None, par_bounds=None, qtilde=False): Returns: calculator - + """ self.data = data self.pdf = pdf @@ -107,7 +106,7 @@ def distributions(self, poi_test): Returns distributions (Tuple of ~pyhf.infer.calculators.AsymptoticTestStatDistribution): the distributions under the hypotheses. - + """ sb_dist = AsymptoticTestStatDistribution(-self.sqrtqmuA_v) b_dist = AsymptoticTestStatDistribution(0.0) @@ -122,7 +121,7 @@ def teststatistic(self, poi_test): Returns: test statistic (Float): the value of the test statistic. - + """ tensorlib, _ = get_backend() qmu_v = qmu(poi_test, self.data, self.pdf, self.init_pars, self.par_bounds) From 0c0e98b7b035da90bae1436b4a02a611d440d3c1 Mon Sep 17 00:00:00 2001 From: Matthew Feickert Date: Fri, 31 Jan 2020 23:29:16 -0600 Subject: [PATCH 15/24] Show full signatures in API docs --- docs/api.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/api.rst b/docs/api.rst index 4b2d821350..a3732da9f6 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -116,6 +116,8 @@ Inference .. autosummary:: :toctree: _generated/ + :nosignatures: + :template: modifierclass.rst hypotest test_statistics.qmu From e37f932c47057fcd0e2134c6be98b7370882a3f2 Mon Sep 17 00:00:00 2001 From: Matthew Feickert Date: Fri, 31 Jan 2020 23:29:50 -0600 Subject: [PATCH 16/24] Add dump of placeholder documentation infomation for API --- src/pyhf/infer/calculators.py | 35 +++++++++++++++++++++++------------ 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/src/pyhf/infer/calculators.py b/src/pyhf/infer/calculators.py index 9c83ff20c1..013f09c4ff 100644 --- a/src/pyhf/infer/calculators.py +++ b/src/pyhf/infer/calculators.py @@ -13,7 +13,20 @@ def generate_asimov_data(asimov_mu, data, pdf, init_pars, par_bounds): - """Compute Asimov Dataset (expected yields at best-fit values) for a given POI value.""" + """ + Compute Asimov Dataset (expected yields at best-fit values) for a given POI value. + + Args: + asimov_mu (`float`): asimov_mu temp + data (`tensor`): data temp + pdf (~pyhf.pdf.Model): pdf temp + init_pars (`tensor`): init_pars temp + par_bounds (`tensor`): par_bounds temp + + Returns: + Tensor: The Asimov dataset. + + """ bestfit_nuisance_asimov = fixed_poi_fit(asimov_mu, data, pdf, init_pars, par_bounds) return pdf.expected_data(bestfit_nuisance_asimov) @@ -37,7 +50,7 @@ def __init__(self, shift): Asymptotic test statistic distribution. Args: - shift: the displacement of the test statistic distribution + shift (`float`): The displacement of the test statistic distribution Returns: distribution @@ -47,14 +60,13 @@ def __init__(self, shift): def pvalue(self, value): """ - Compute the p-value for a given value of the test statistic. + Compute the :math:`p`-value for a given value of the test statistic. Args: - value: the test statistic value. + value (`float`): The test statistic value. Returns: - pvalue (float): the integrated probability to observe - a value at least as large as the observed one. + Float: The integrated probability to observe a value at least as large as the observed one. """ tensorlib, _ = get_backend() @@ -68,8 +80,7 @@ def expected_value(self, nsigma): nsigma: number of standard deviations. Returns: - expected value (float): the expected value of the test statistic. - + Float: The expected value of the test statistic. """ return nsigma @@ -102,10 +113,10 @@ def distributions(self, poi_test): Probability Distributions of the test statistic value under the signal + background and and background-only hypothesis. Args: - poi_test: the value for the parameter of interest. + poi_test: The value for the parameter of interest. - Returns - distributions (Tuple of ~pyhf.infer.calculators.AsymptoticTestStatDistribution): the distributions under the hypotheses. + Returns: + Tuple (~pyhf.infer.calculators.AsymptoticTestStatDistribution): The distributions under the hypotheses. """ sb_dist = AsymptoticTestStatDistribution(-self.sqrtqmuA_v) @@ -120,7 +131,7 @@ def teststatistic(self, poi_test): poi_test: the value for the parameter of interest. Returns: - test statistic (Float): the value of the test statistic. + Float: the value of the test statistic. """ tensorlib, _ = get_backend() From 3bd47778841185a41a6ed63a2645523655804252 Mon Sep 17 00:00:00 2001 From: Matthew Feickert Date: Sat, 29 Feb 2020 15:06:12 -0600 Subject: [PATCH 17/24] Dump some more correct/informative docstrings --- src/pyhf/infer/calculators.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/pyhf/infer/calculators.py b/src/pyhf/infer/calculators.py index 013f09c4ff..d215a89462 100644 --- a/src/pyhf/infer/calculators.py +++ b/src/pyhf/infer/calculators.py @@ -17,11 +17,11 @@ def generate_asimov_data(asimov_mu, data, pdf, init_pars, par_bounds): Compute Asimov Dataset (expected yields at best-fit values) for a given POI value. Args: - asimov_mu (`float`): asimov_mu temp - data (`tensor`): data temp - pdf (~pyhf.pdf.Model): pdf temp - init_pars (`tensor`): init_pars temp - par_bounds (`tensor`): par_bounds temp + asimov_mu (`float`): The value for the parameter of interest to be used. + data (`tensor`): The measurement data. + pdf (~pyhf.pdf.Model): The statistical model adhering to the schema ``model.json``. + init_pars (`tensor`): The initial parameter values to be used for minimization. + par_bounds (`tensor`): The parameter value bounds to be used for minimization. Returns: Tensor: The Asimov dataset. From 09cd08296049614a9d32de1ad6ea6948c69aad26 Mon Sep 17 00:00:00 2001 From: Matthew Feickert Date: Sat, 29 Feb 2020 15:10:42 -0600 Subject: [PATCH 18/24] Revert "Show full signatures in API docs" This reverts commit 0c0e98b7b035da90bae1436b4a02a611d440d3c1. --- docs/api.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index a3732da9f6..4b2d821350 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -116,8 +116,6 @@ Inference .. autosummary:: :toctree: _generated/ - :nosignatures: - :template: modifierclass.rst hypotest test_statistics.qmu From bdefe5574604dad432198d9d9ee3c956f521ba81 Mon Sep 17 00:00:00 2001 From: Matthew Feickert Date: Sat, 29 Feb 2020 15:20:23 -0600 Subject: [PATCH 19/24] Add docstrings for AsymptoticCalculator class --- src/pyhf/infer/calculators.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/pyhf/infer/calculators.py b/src/pyhf/infer/calculators.py index d215a89462..3c6bc3820f 100644 --- a/src/pyhf/infer/calculators.py +++ b/src/pyhf/infer/calculators.py @@ -18,10 +18,10 @@ def generate_asimov_data(asimov_mu, data, pdf, init_pars, par_bounds): Args: asimov_mu (`float`): The value for the parameter of interest to be used. - data (`tensor`): The measurement data. + data (`tensor`): The observed data. pdf (~pyhf.pdf.Model): The statistical model adhering to the schema ``model.json``. - init_pars (`tensor`): The initial parameter values to be used for minimization. - par_bounds (`tensor`): The parameter value bounds to be used for minimization. + init_pars (`tensor`): The initial parameter values to be used for fitting. + par_bounds (`tensor`): The parameter value bounds to be used for fitting. Returns: Tensor: The Asimov dataset. @@ -93,13 +93,13 @@ def __init__(self, data, pdf, init_pars=None, par_bounds=None, qtilde=False): Asymptotic Calculator. Args: - data: the observed data - pdf: the statistical model - init_pars: the initial parameters to be used for fitting - par_bounds: the parameter bounds used for fitting + data (`tensor`): The observed data. + pdf (~pyhf.pdf.Model): The statistical model adhering to the schema ``model.json``. + init_pars (`tensor`): The initial parameter values to be used for fitting. + par_bounds (`tensor`): The parameter value bounds to be used for fitting. Returns: - calculator + ~pyhf.infer.calculators.AsymptoticCalculator: The calculator for asymptotic quantities. """ self.data = data @@ -128,7 +128,7 @@ def teststatistic(self, poi_test): Compute the test statistic for the observed data under the studied model. Args: - poi_test: the value for the parameter of interest. + poi_test: The value for the parameter of interest. Returns: Float: the value of the test statistic. From 68d6a3c3729677abb68f264001072eaa3281986a Mon Sep 17 00:00:00 2001 From: Matthew Feickert Date: Sat, 29 Feb 2020 15:24:04 -0600 Subject: [PATCH 20/24] Add docstring to the AsymptoticTestStatDistribution class --- src/pyhf/infer/calculators.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pyhf/infer/calculators.py b/src/pyhf/infer/calculators.py index 3c6bc3820f..8a60463507 100644 --- a/src/pyhf/infer/calculators.py +++ b/src/pyhf/infer/calculators.py @@ -50,10 +50,10 @@ def __init__(self, shift): Asymptotic test statistic distribution. Args: - shift (`float`): The displacement of the test statistic distribution + shift (`float`): The displacement of the test statistic distribution. Returns: - distribution + ~pyhf.infer.calculators.AsymptoticTestStatDistribution: The asymptotic distribution of test statistic. """ self.shift = shift @@ -77,7 +77,7 @@ def expected_value(self, nsigma): Return the expected value of the test statistic. Args: - nsigma: number of standard deviations. + nsigma (`int` or `tensor`): The number of standard deviations. Returns: Float: The expected value of the test statistic. From 04f83fe96e529d2e519905c234ddab58aa7b4bda Mon Sep 17 00:00:00 2001 From: Lukas Heinrich Date: Wed, 4 Mar 2020 18:52:55 +0100 Subject: [PATCH 21/24] giordon --- src/pyhf/infer/__init__.py | 10 +++++----- src/pyhf/infer/calculators.py | 3 +++ 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/pyhf/infer/__init__.py b/src/pyhf/infer/__init__.py index 91ceef1f2d..8599502220 100644 --- a/src/pyhf/infer/__init__.py +++ b/src/pyhf/infer/__init__.py @@ -76,10 +76,10 @@ def hypotest( calc = AsymptoticCalculator(data, pdf, init_pars, par_bounds, qtilde=qtilde) teststat = calc.teststatistic(poi_test) - sb_dist, b_dist = calc.distributions(poi_test) + sig_plus_bkg_distribution, b_only_distribution = calc.distributions(poi_test) - CLsb = sb_dist.pvalue(teststat) - CLb = b_dist.pvalue(teststat) + CLsb = sig_plus_bkg_distribution.pvalue(teststat) + CLb = b_only_distribution.pvalue(teststat) CLs = CLsb / CLb CLsb, CLb, CLs = ( tensorlib.reshape(CLsb, (1,)), @@ -93,7 +93,7 @@ def hypotest( if kwargs.get('return_expected_set'): CLs_exp = [] for n_sigma in [2, 1, 0, -1, -2]: - CLs = sb_dist.pvalue(n_sigma) / b_dist.pvalue(n_sigma) + CLs = sig_plus_bkg_distribution.pvalue(n_sigma) / b_only_distribution.pvalue(n_sigma) CLs_exp.append(tensorlib.reshape(CLs, (1,))) CLs_exp = tensorlib.astensor(CLs_exp) if kwargs.get('return_expected'): @@ -101,7 +101,7 @@ def hypotest( _returns.append(CLs_exp) elif kwargs.get('return_expected'): n_sigma = 0 - CLs = sb_dist.pvalue(n_sigma) / b_dist.pvalue(n_sigma) + CLs = sig_plus_bkg_distribution.pvalue(n_sigma) / b_only_distribution.pvalue(n_sigma) _returns.append(tensorlib.reshape(CLs, (1,))) # Enforce a consistent return type of the observed CLs return tuple(_returns) if len(_returns) > 1 else _returns[0] diff --git a/src/pyhf/infer/calculators.py b/src/pyhf/infer/calculators.py index 8a60463507..a1f80d70a7 100644 --- a/src/pyhf/infer/calculators.py +++ b/src/pyhf/infer/calculators.py @@ -57,6 +57,7 @@ def __init__(self, shift): """ self.shift = shift + self.sqrtqmuA_v = None def pvalue(self, value): """ @@ -119,6 +120,8 @@ def distributions(self, poi_test): Tuple (~pyhf.infer.calculators.AsymptoticTestStatDistribution): The distributions under the hypotheses. """ + if self.sqrtqmuA_v is None; + raise RuntimeError('need to call .teststatistic(poi_test) first') sb_dist = AsymptoticTestStatDistribution(-self.sqrtqmuA_v) b_dist = AsymptoticTestStatDistribution(0.0) return sb_dist, b_dist From d52f512ec1093852461d06d5fd834c6748a75d3e Mon Sep 17 00:00:00 2001 From: Lukas Heinrich Date: Wed, 4 Mar 2020 18:54:07 +0100 Subject: [PATCH 22/24] giordon --- src/pyhf/infer/__init__.py | 8 ++++++-- src/pyhf/infer/calculators.py | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/pyhf/infer/__init__.py b/src/pyhf/infer/__init__.py index 8599502220..155f1e28e7 100644 --- a/src/pyhf/infer/__init__.py +++ b/src/pyhf/infer/__init__.py @@ -93,7 +93,9 @@ def hypotest( if kwargs.get('return_expected_set'): CLs_exp = [] for n_sigma in [2, 1, 0, -1, -2]: - CLs = sig_plus_bkg_distribution.pvalue(n_sigma) / b_only_distribution.pvalue(n_sigma) + CLs = sig_plus_bkg_distribution.pvalue( + n_sigma + ) / b_only_distribution.pvalue(n_sigma) CLs_exp.append(tensorlib.reshape(CLs, (1,))) CLs_exp = tensorlib.astensor(CLs_exp) if kwargs.get('return_expected'): @@ -101,7 +103,9 @@ def hypotest( _returns.append(CLs_exp) elif kwargs.get('return_expected'): n_sigma = 0 - CLs = sig_plus_bkg_distribution.pvalue(n_sigma) / b_only_distribution.pvalue(n_sigma) + CLs = sig_plus_bkg_distribution.pvalue(n_sigma) / b_only_distribution.pvalue( + n_sigma + ) _returns.append(tensorlib.reshape(CLs, (1,))) # Enforce a consistent return type of the observed CLs return tuple(_returns) if len(_returns) > 1 else _returns[0] diff --git a/src/pyhf/infer/calculators.py b/src/pyhf/infer/calculators.py index a1f80d70a7..587157d28f 100644 --- a/src/pyhf/infer/calculators.py +++ b/src/pyhf/infer/calculators.py @@ -120,7 +120,7 @@ def distributions(self, poi_test): Tuple (~pyhf.infer.calculators.AsymptoticTestStatDistribution): The distributions under the hypotheses. """ - if self.sqrtqmuA_v is None; + if self.sqrtqmuA_v is None: raise RuntimeError('need to call .teststatistic(poi_test) first') sb_dist = AsymptoticTestStatDistribution(-self.sqrtqmuA_v) b_dist = AsymptoticTestStatDistribution(0.0) From 9c69fb5936cb86d9d09d74ff05ccec0986271253 Mon Sep 17 00:00:00 2001 From: Lukas Heinrich Date: Wed, 4 Mar 2020 19:01:17 +0100 Subject: [PATCH 23/24] giordon --- src/pyhf/infer/calculators.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/pyhf/infer/calculators.py b/src/pyhf/infer/calculators.py index 587157d28f..354a775346 100644 --- a/src/pyhf/infer/calculators.py +++ b/src/pyhf/infer/calculators.py @@ -98,6 +98,7 @@ def __init__(self, data, pdf, init_pars=None, par_bounds=None, qtilde=False): pdf (~pyhf.pdf.Model): The statistical model adhering to the schema ``model.json``. init_pars (`tensor`): The initial parameter values to be used for fitting. par_bounds (`tensor`): The parameter value bounds to be used for fitting. + qtilde (`bool`): Whether to use qtilde as the test statistic. Returns: ~pyhf.infer.calculators.AsymptoticCalculator: The calculator for asymptotic quantities. From 2f828d318006eaa341e42ea6e785d24bde3d99a2 Mon Sep 17 00:00:00 2001 From: Giordon Stark Date: Wed, 4 Mar 2020 11:03:10 -0800 Subject: [PATCH 24/24] add missing test for coverage --- src/pyhf/infer/calculators.py | 1 + tests/test_infer.py | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/src/pyhf/infer/calculators.py b/src/pyhf/infer/calculators.py index 354a775346..8767f2cf4c 100644 --- a/src/pyhf/infer/calculators.py +++ b/src/pyhf/infer/calculators.py @@ -109,6 +109,7 @@ def __init__(self, data, pdf, init_pars=None, par_bounds=None, qtilde=False): self.init_pars = init_pars or pdf.config.suggested_init() self.par_bounds = par_bounds or pdf.config.suggested_bounds() self.qtilde = qtilde + self.sqrtqmuA_v = None def distributions(self, poi_test): """ diff --git a/tests/test_infer.py b/tests/test_infer.py index de8bbcecc8..731d3024c7 100644 --- a/tests/test_infer.py +++ b/tests/test_infer.py @@ -148,3 +148,12 @@ def logpdf(self, pars, data): ) assert np.isclose(cls[0], 0.7267836451638846) + + +@pytest.mark.parametrize("qtilde", [True, False]) +def test_calculator_distributions_without_teststatistic(qtilde): + calc = pyhf.infer.AsymptoticCalculator( + [0.0], {}, [1.0], [(0.0, 10.0)], qtilde=qtilde + ) + with pytest.raises(RuntimeError): + calc.distributions(1.0)