From dd5c4e69c0082aa57c64a0152188f49c3f9b6d4b Mon Sep 17 00:00:00 2001 From: Matthew Feickert Date: Mon, 15 Feb 2021 01:31:17 -0600 Subject: [PATCH 1/8] Ensure CLs values are 0-d tensors --- src/pyhf/infer/calculators.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/pyhf/infer/calculators.py b/src/pyhf/infer/calculators.py index 6b808c0b8b..d744ce8bfb 100644 --- a/src/pyhf/infer/calculators.py +++ b/src/pyhf/infer/calculators.py @@ -393,9 +393,11 @@ def pvalues(self, teststat, sig_plus_bkg_distribution, bkg_only_distribution): corresponding to the :math:`\mathrm{CL}_{s+b}`, :math:`\mathrm{CL}_{b}`, and :math:`\mathrm{CL}_{s}`. """ + tensorlib, _ = get_backend() + CLsb = sig_plus_bkg_distribution.pvalue(teststat) CLb = bkg_only_distribution.pvalue(teststat) - CLs = CLsb / CLb + CLs = tensorlib.astensor(CLsb / CLb) return CLsb, CLb, CLs def expected_pvalues(self, sig_plus_bkg_distribution, bkg_only_distribution): From 848ea0887a8b023d470bdc8dbf161b875c559b6c Mon Sep 17 00:00:00 2001 From: Matthew Feickert Date: Mon, 15 Feb 2021 01:48:29 -0600 Subject: [PATCH 2/8] Ensure for emperical distributions too --- src/pyhf/infer/calculators.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/pyhf/infer/calculators.py b/src/pyhf/infer/calculators.py index d744ce8bfb..be75e331b7 100644 --- a/src/pyhf/infer/calculators.py +++ b/src/pyhf/infer/calculators.py @@ -527,7 +527,7 @@ def pvalue(self, value): """ tensorlib, _ = get_backend() - return ( + return tensorlib.astensor( tensorlib.sum( tensorlib.where( self.samples >= value, tensorlib.astensor(1), tensorlib.astensor(0) @@ -769,9 +769,11 @@ def pvalues(self, teststat, sig_plus_bkg_distribution, bkg_only_distribution): corresponding to the :math:`\mathrm{CL}_{s+b}`, :math:`\mathrm{CL}_{b}`, and :math:`\mathrm{CL}_{s}`. """ + tensorlib, _ = get_backend() + CLsb = sig_plus_bkg_distribution.pvalue(teststat) CLb = bkg_only_distribution.pvalue(teststat) - CLs = CLsb / CLb + CLs = tensorlib.astensor(CLsb / CLb) return CLsb, CLb, CLs def expected_pvalues(self, sig_plus_bkg_distribution, bkg_only_distribution): From 8da46e7696ce37339058b414121329e8bfc6e4c0 Mon Sep 17 00:00:00 2001 From: Matthew Feickert Date: Mon, 15 Feb 2021 01:56:02 -0600 Subject: [PATCH 3/8] Ensure test stat is tensor --- src/pyhf/infer/calculators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pyhf/infer/calculators.py b/src/pyhf/infer/calculators.py index be75e331b7..131ce750d1 100644 --- a/src/pyhf/infer/calculators.py +++ b/src/pyhf/infer/calculators.py @@ -355,7 +355,7 @@ def _false_case(): teststat = tensorlib.conditional( (sqrtqmu_v < self.sqrtqmuA_v), _true_case, _false_case ) - return teststat + return tensorlib.astensor(teststat) def pvalues(self, teststat, sig_plus_bkg_distribution, bkg_only_distribution): r""" From cd8c759637d3a6e2c2685ef7774c734f20c70bcf Mon Sep 17 00:00:00 2001 From: Matthew Feickert Date: Mon, 15 Feb 2021 02:10:58 -0600 Subject: [PATCH 4/8] Fixup docstring --- src/pyhf/infer/calculators.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pyhf/infer/calculators.py b/src/pyhf/infer/calculators.py index 131ce750d1..ab9ab4d797 100644 --- a/src/pyhf/infer/calculators.py +++ b/src/pyhf/infer/calculators.py @@ -295,7 +295,7 @@ def teststatistic(self, poi_test): >>> mu_test = 1.0 >>> asymptotic_calculator = pyhf.infer.calculators.AsymptoticCalculator(data, model, test_stat="qtilde") >>> asymptotic_calculator.teststatistic(mu_test) - 0.14043184405388176 + array(0.14043184) Args: poi_test (:obj:`float` or :obj:`tensor`): The value for the parameter of interest. @@ -379,7 +379,7 @@ def pvalues(self, teststat, sig_plus_bkg_distribution, bkg_only_distribution): >>> sig_plus_bkg_dist, bkg_dist = asymptotic_calculator.distributions(mu_test) >>> CLsb, CLb, CLs = asymptotic_calculator.pvalues(q_tilde, sig_plus_bkg_dist, bkg_dist) >>> CLsb, CLb, CLs - (array(0.02332502), array(0.4441594), 0.05251497423736956) + (array(0.02332502), array(0.4441594), array(0.05251497)) Args: teststat (:obj:`tensor`): The test statistic. @@ -424,7 +424,7 @@ def expected_pvalues(self, sig_plus_bkg_distribution, bkg_only_distribution): >>> sig_plus_bkg_dist, bkg_dist = asymptotic_calculator.distributions(mu_test) >>> CLsb_exp_band, CLb_exp_band, CLs_exp_band = asymptotic_calculator.expected_pvalues(sig_plus_bkg_dist, bkg_dist) >>> CLs_exp_band - [0.0026062609501074576, 0.01382005356161206, 0.06445320535890459, 0.23525643861460702, 0.573036205919389] + [array(0.00260626), array(0.01382005), array(0.06445321), array(0.23525644), array(0.57303621)] Args: sig_plus_bkg_distribution (~pyhf.infer.calculators.AsymptoticTestStatDistribution): From 6183d8d4a5b753c9da98ad1eaa7b4a5bf85e08d9 Mon Sep 17 00:00:00 2001 From: Matthew Feickert Date: Mon, 15 Feb 2021 02:39:28 -0600 Subject: [PATCH 5/8] Match return type of asymp --- src/pyhf/infer/calculators.py | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/src/pyhf/infer/calculators.py b/src/pyhf/infer/calculators.py index ab9ab4d797..448dfea7f6 100644 --- a/src/pyhf/infer/calculators.py +++ b/src/pyhf/infer/calculators.py @@ -496,7 +496,7 @@ def pvalue(self, value): >>> samples = normal.sample((100,)) >>> dist = pyhf.infer.calculators.EmpiricalDistribution(samples) >>> dist.pvalue(7) - 0.02 + array(0.02) >>> import pyhf >>> import numpy.random as random @@ -517,7 +517,7 @@ def pvalue(self, value): ... ) ... ) >>> test_stat_dist.pvalue(test_stat_dist.samples[9]) - 0.3 + array(0.3) Args: value (:obj:`float`): The test statistic value. @@ -667,7 +667,7 @@ def distributions(self, poi_test, track_progress=None): ... ) >>> sig_plus_bkg_dist, bkg_dist = toy_calculator.distributions(mu_test) >>> sig_plus_bkg_dist.pvalue(mu_test), bkg_dist.pvalue(mu_test) - (0.14, 0.76) + (array(0.14), array(0.76)) Args: poi_test (:obj:`float` or :obj:`tensor`): The value for the parameter of interest. @@ -755,7 +755,7 @@ def pvalues(self, teststat, sig_plus_bkg_distribution, bkg_only_distribution): >>> sig_plus_bkg_dist, bkg_dist = toy_calculator.distributions(mu_test) >>> CLsb, CLb, CLs = toy_calculator.pvalues(q_tilde, sig_plus_bkg_dist, bkg_dist) >>> CLsb, CLb, CLs - (0.01, 0.41, 0.024390243902439025) + (array(0.01), array(0.41), array(0.02439024)) Args: teststat (:obj:`tensor`): The test statistic. @@ -801,7 +801,7 @@ def expected_pvalues(self, sig_plus_bkg_distribution, bkg_only_distribution): >>> sig_plus_bkg_dist, bkg_dist = toy_calculator.distributions(mu_test) >>> CLsb_exp_band, CLb_exp_band, CLs_exp_band = toy_calculator.expected_pvalues(sig_plus_bkg_dist, bkg_dist) >>> CLs_exp_band - [0.0, 0.0, 0.06186224489795918, 0.2845003327965815, 1.0] + [array(0.), array(0.), array(0.06186224), array(0.28450033), array(1.)] Args: sig_plus_bkg_distribution (~pyhf.infer.calculators.EmpiricalDistribution): @@ -815,16 +815,14 @@ def expected_pvalues(self, sig_plus_bkg_distribution, bkg_only_distribution): :math:`\mathrm{CL}_{b}`, and :math:`\mathrm{CL}_{s}`. """ tb, _ = get_backend() - pvalues = tb.astensor( - [ - self.pvalues( - tb.astensor(test_stat), - sig_plus_bkg_distribution, - bkg_only_distribution, - ) - for test_stat in bkg_only_distribution.samples - ] - ) + pvalues = [ + self.pvalues( + tb.astensor(test_stat), + sig_plus_bkg_distribution, + bkg_only_distribution, + ) + for test_stat in bkg_only_distribution.samples + ] # TODO: Add percentile to tensorlib # c.f. Issue #815, PR #817 import numpy as np @@ -832,11 +830,11 @@ def expected_pvalues(self, sig_plus_bkg_distribution, bkg_only_distribution): # percentiles for -2, -1, 0, 1, 2 standard deviations of the Normal distribution normal_percentiles = [2.27501319, 15.86552539, 50.0, 84.13447461, 97.72498681] pvalues_exp_band = np.percentile( - tb.tolist(pvalues), + pvalues, normal_percentiles, axis=0, ).T.tolist() - return pvalues_exp_band + return [[tb.astensor(pvalue) for pvalue in band] for band in pvalues_exp_band] def teststatistic(self, poi_test): """ From 2d5d3bb7bd5d23634f25471464e10d150a3d80d9 Mon Sep 17 00:00:00 2001 From: Matthew Feickert Date: Mon, 15 Feb 2021 02:50:38 -0600 Subject: [PATCH 6/8] Test stat docstring --- src/pyhf/infer/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pyhf/infer/utils.py b/src/pyhf/infer/utils.py index 46539fd82e..15c0d9dacf 100644 --- a/src/pyhf/infer/utils.py +++ b/src/pyhf/infer/utils.py @@ -33,7 +33,7 @@ def create_calculator(calctype, *args, **kwargs): ... ) >>> qmu_sig, qmu_bkg = toy_calculator.distributions(mu_test) >>> qmu_sig.pvalue(mu_test), qmu_bkg.pvalue(mu_test) - (0.14, 0.76) + (array(0.14), array(0.76)) Args: calctype (:obj:`str`): The calculator to create. Choose either From 6aa52ee31b847b42c109dc9e20fa62da385e7719 Mon Sep 17 00:00:00 2001 From: Matthew Feickert Date: Mon, 15 Feb 2021 03:07:31 -0600 Subject: [PATCH 7/8] Reduce casts to tensor --- src/pyhf/infer/calculators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pyhf/infer/calculators.py b/src/pyhf/infer/calculators.py index 448dfea7f6..34bab8ed73 100644 --- a/src/pyhf/infer/calculators.py +++ b/src/pyhf/infer/calculators.py @@ -817,7 +817,7 @@ def expected_pvalues(self, sig_plus_bkg_distribution, bkg_only_distribution): tb, _ = get_backend() pvalues = [ self.pvalues( - tb.astensor(test_stat), + test_stat, sig_plus_bkg_distribution, bkg_only_distribution, ) From 79e7a34b7bc2c113cd37d8ae8beeb29ebe9c5d57 Mon Sep 17 00:00:00 2001 From: Matthew Feickert Date: Mon, 15 Feb 2021 03:40:17 -0600 Subject: [PATCH 8/8] update return type to tensor --- src/pyhf/infer/calculators.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/pyhf/infer/calculators.py b/src/pyhf/infer/calculators.py index 34bab8ed73..a36399cffb 100644 --- a/src/pyhf/infer/calculators.py +++ b/src/pyhf/infer/calculators.py @@ -132,7 +132,7 @@ def pvalue(self, value): value (:obj:`float`): The test statistic value. Returns: - Float: The integrated probability to observe a value at least as large as the observed one. + Tensor: The integrated probability to observe a value at least as large as the observed one. """ tensorlib, _ = get_backend() @@ -301,7 +301,7 @@ def teststatistic(self, poi_test): poi_test (:obj:`float` or :obj:`tensor`): The value for the parameter of interest. Returns: - Float: The value of the test statistic. + Tensor: The value of the test statistic. """ tensorlib, _ = get_backend() @@ -389,7 +389,7 @@ def pvalues(self, teststat, sig_plus_bkg_distribution, bkg_only_distribution): The distribution for the background-only hypothesis. Returns: - Tuple (:obj:`float`): The :math:`p`-values for the test statistic + Tuple (:obj:`tensor`): The :math:`p`-values for the test statistic corresponding to the :math:`\mathrm{CL}_{s+b}`, :math:`\mathrm{CL}_{b}`, and :math:`\mathrm{CL}_{s}`. """ @@ -433,7 +433,7 @@ def expected_pvalues(self, sig_plus_bkg_distribution, bkg_only_distribution): The distribution for the background-only hypothesis. Returns: - Tuple (:obj:`float`): The :math:`p`-values for the test statistic + Tuple (:obj:`tensor`): The :math:`p`-values for the test statistic corresponding to the :math:`\mathrm{CL}_{s+b}`, :math:`\mathrm{CL}_{b}`, and :math:`\mathrm{CL}_{s}`. """ @@ -523,7 +523,7 @@ def pvalue(self, value): value (:obj:`float`): The test statistic value. Returns: - Float: The integrated probability to observe a value at least as large as the observed one. + Tensor: The integrated probability to observe a value at least as large as the observed one. """ tensorlib, _ = get_backend() @@ -765,7 +765,7 @@ def pvalues(self, teststat, sig_plus_bkg_distribution, bkg_only_distribution): The distribution for the background-only hypothesis. Returns: - Tuple (:obj:`float`): The :math:`p`-values for the test statistic + Tuple (:obj:`tensor`): The :math:`p`-values for the test statistic corresponding to the :math:`\mathrm{CL}_{s+b}`, :math:`\mathrm{CL}_{b}`, and :math:`\mathrm{CL}_{s}`. """ @@ -810,7 +810,7 @@ def expected_pvalues(self, sig_plus_bkg_distribution, bkg_only_distribution): The distribution for the background-only hypothesis. Returns: - Tuple (:obj:`float`): The :math:`p`-values for the test statistic + Tuple (:obj:`tensor`): The :math:`p`-values for the test statistic corresponding to the :math:`\mathrm{CL}_{s+b}`, :math:`\mathrm{CL}_{b}`, and :math:`\mathrm{CL}_{s}`. """ @@ -862,7 +862,7 @@ def teststatistic(self, poi_test): poi_test (:obj:`float` or :obj:`tensor`): The value for the parameter of interest. Returns: - Float: The value of the test statistic. + Tensor: The value of the test statistic. """ teststat_func = utils.get_test_stat(self.test_stat)