Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: Make p-value-like values 0-d tensors #1311

Merged
merged 8 commits into from
Feb 15, 2021
Merged

fix: Make p-value-like values 0-d tensors #1311

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
dd5c4e6
Ensure CLs values are 0-d tensors
matthewfeickert Feb 15, 2021
848ea08
Ensure for emperical distributions too
matthewfeickert Feb 15, 2021
8da46e7
Ensure test stat is tensor
matthewfeickert Feb 15, 2021
cd8c759
Fixup docstring
matthewfeickert Feb 15, 2021
6183d8d
Match return type of asymp
matthewfeickert Feb 15, 2021
2d5d3bb
Test stat docstring
matthewfeickert Feb 15, 2021
6aa52ee
Reduce casts to tensor
matthewfeickert Feb 15, 2021
79e7a34
update return type to tensor
matthewfeickert Feb 15, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
66 changes: 34 additions & 32 deletions src/pyhf/infer/calculators.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,7 @@ def pvalue(self, value):
value (:obj:`float`): The test statistic value.

Returns:
Float: The integrated probability to observe a value at least as large as the observed one.
Tensor: The integrated probability to observe a value at least as large as the observed one.

"""
tensorlib, _ = get_backend()
Expand Down Expand Up @@ -295,13 +295,13 @@ def teststatistic(self, poi_test):
>>> mu_test = 1.0
>>> asymptotic_calculator = pyhf.infer.calculators.AsymptoticCalculator(data, model, test_stat="qtilde")
>>> asymptotic_calculator.teststatistic(mu_test)
0.14043184405388176
array(0.14043184)

Args:
poi_test (:obj:`float` or :obj:`tensor`): The value for the parameter of interest.

Returns:
Float: The value of the test statistic.
Tensor: The value of the test statistic.

"""
tensorlib, _ = get_backend()
Expand Down Expand Up @@ -355,7 +355,7 @@ def _false_case():
teststat = tensorlib.conditional(
(sqrtqmu_v < self.sqrtqmuA_v), _true_case, _false_case
)
return teststat
return tensorlib.astensor(teststat)

def pvalues(self, teststat, sig_plus_bkg_distribution, bkg_only_distribution):
r"""
Expand All @@ -379,7 +379,7 @@ def pvalues(self, teststat, sig_plus_bkg_distribution, bkg_only_distribution):
>>> sig_plus_bkg_dist, bkg_dist = asymptotic_calculator.distributions(mu_test)
>>> CLsb, CLb, CLs = asymptotic_calculator.pvalues(q_tilde, sig_plus_bkg_dist, bkg_dist)
>>> CLsb, CLb, CLs
(array(0.02332502), array(0.4441594), 0.05251497423736956)
(array(0.02332502), array(0.4441594), array(0.05251497))

Args:
teststat (:obj:`tensor`): The test statistic.
Expand All @@ -389,13 +389,15 @@ def pvalues(self, teststat, sig_plus_bkg_distribution, bkg_only_distribution):
The distribution for the background-only hypothesis.

Returns:
Tuple (:obj:`float`): The :math:`p`-values for the test statistic
Tuple (:obj:`tensor`): The :math:`p`-values for the test statistic
corresponding to the :math:`\mathrm{CL}_{s+b}`,
:math:`\mathrm{CL}_{b}`, and :math:`\mathrm{CL}_{s}`.
"""
tensorlib, _ = get_backend()

CLsb = sig_plus_bkg_distribution.pvalue(teststat)
CLb = bkg_only_distribution.pvalue(teststat)
CLs = CLsb / CLb
CLs = tensorlib.astensor(CLsb / CLb)
return CLsb, CLb, CLs

def expected_pvalues(self, sig_plus_bkg_distribution, bkg_only_distribution):
Expand All @@ -422,7 +424,7 @@ def expected_pvalues(self, sig_plus_bkg_distribution, bkg_only_distribution):
>>> sig_plus_bkg_dist, bkg_dist = asymptotic_calculator.distributions(mu_test)
>>> CLsb_exp_band, CLb_exp_band, CLs_exp_band = asymptotic_calculator.expected_pvalues(sig_plus_bkg_dist, bkg_dist)
>>> CLs_exp_band
[0.0026062609501074576, 0.01382005356161206, 0.06445320535890459, 0.23525643861460702, 0.573036205919389]
[array(0.00260626), array(0.01382005), array(0.06445321), array(0.23525644), array(0.57303621)]

Args:
sig_plus_bkg_distribution (~pyhf.infer.calculators.AsymptoticTestStatDistribution):
Expand All @@ -431,7 +433,7 @@ def expected_pvalues(self, sig_plus_bkg_distribution, bkg_only_distribution):
The distribution for the background-only hypothesis.

Returns:
Tuple (:obj:`float`): The :math:`p`-values for the test statistic
Tuple (:obj:`tensor`): The :math:`p`-values for the test statistic
corresponding to the :math:`\mathrm{CL}_{s+b}`,
:math:`\mathrm{CL}_{b}`, and :math:`\mathrm{CL}_{s}`.
"""
Expand Down Expand Up @@ -494,7 +496,7 @@ def pvalue(self, value):
>>> samples = normal.sample((100,))
>>> dist = pyhf.infer.calculators.EmpiricalDistribution(samples)
>>> dist.pvalue(7)
0.02
array(0.02)

>>> import pyhf
>>> import numpy.random as random
Expand All @@ -515,17 +517,17 @@ def pvalue(self, value):
... )
... )
>>> test_stat_dist.pvalue(test_stat_dist.samples[9])
0.3
array(0.3)

Args:
value (:obj:`float`): The test statistic value.

Returns:
Float: The integrated probability to observe a value at least as large as the observed one.
Tensor: The integrated probability to observe a value at least as large as the observed one.

"""
tensorlib, _ = get_backend()
return (
return tensorlib.astensor(
tensorlib.sum(
tensorlib.where(
self.samples >= value, tensorlib.astensor(1), tensorlib.astensor(0)
Expand Down Expand Up @@ -665,7 +667,7 @@ def distributions(self, poi_test, track_progress=None):
... )
>>> sig_plus_bkg_dist, bkg_dist = toy_calculator.distributions(mu_test)
>>> sig_plus_bkg_dist.pvalue(mu_test), bkg_dist.pvalue(mu_test)
(0.14, 0.76)
(array(0.14), array(0.76))

Args:
poi_test (:obj:`float` or :obj:`tensor`): The value for the parameter of interest.
Expand Down Expand Up @@ -753,7 +755,7 @@ def pvalues(self, teststat, sig_plus_bkg_distribution, bkg_only_distribution):
>>> sig_plus_bkg_dist, bkg_dist = toy_calculator.distributions(mu_test)
>>> CLsb, CLb, CLs = toy_calculator.pvalues(q_tilde, sig_plus_bkg_dist, bkg_dist)
>>> CLsb, CLb, CLs
(0.01, 0.41, 0.024390243902439025)
(array(0.01), array(0.41), array(0.02439024))

Args:
teststat (:obj:`tensor`): The test statistic.
Expand All @@ -763,13 +765,15 @@ def pvalues(self, teststat, sig_plus_bkg_distribution, bkg_only_distribution):
The distribution for the background-only hypothesis.

Returns:
Tuple (:obj:`float`): The :math:`p`-values for the test statistic
Tuple (:obj:`tensor`): The :math:`p`-values for the test statistic
corresponding to the :math:`\mathrm{CL}_{s+b}`,
:math:`\mathrm{CL}_{b}`, and :math:`\mathrm{CL}_{s}`.
"""
tensorlib, _ = get_backend()

CLsb = sig_plus_bkg_distribution.pvalue(teststat)
CLb = bkg_only_distribution.pvalue(teststat)
CLs = CLsb / CLb
CLs = tensorlib.astensor(CLsb / CLb)
return CLsb, CLb, CLs

def expected_pvalues(self, sig_plus_bkg_distribution, bkg_only_distribution):
Expand Down Expand Up @@ -797,7 +801,7 @@ def expected_pvalues(self, sig_plus_bkg_distribution, bkg_only_distribution):
>>> sig_plus_bkg_dist, bkg_dist = toy_calculator.distributions(mu_test)
>>> CLsb_exp_band, CLb_exp_band, CLs_exp_band = toy_calculator.expected_pvalues(sig_plus_bkg_dist, bkg_dist)
>>> CLs_exp_band
[0.0, 0.0, 0.06186224489795918, 0.2845003327965815, 1.0]
[array(0.), array(0.), array(0.06186224), array(0.28450033), array(1.)]

Args:
sig_plus_bkg_distribution (~pyhf.infer.calculators.EmpiricalDistribution):
Expand All @@ -806,33 +810,31 @@ def expected_pvalues(self, sig_plus_bkg_distribution, bkg_only_distribution):
The distribution for the background-only hypothesis.

Returns:
Tuple (:obj:`float`): The :math:`p`-values for the test statistic
Tuple (:obj:`tensor`): The :math:`p`-values for the test statistic
corresponding to the :math:`\mathrm{CL}_{s+b}`,
:math:`\mathrm{CL}_{b}`, and :math:`\mathrm{CL}_{s}`.
"""
tb, _ = get_backend()
pvalues = tb.astensor(
[
self.pvalues(
tb.astensor(test_stat),
sig_plus_bkg_distribution,
bkg_only_distribution,
)
for test_stat in bkg_only_distribution.samples
]
)
pvalues = [
self.pvalues(
test_stat,
sig_plus_bkg_distribution,
bkg_only_distribution,
)
for test_stat in bkg_only_distribution.samples
]
# TODO: Add percentile to tensorlib
# c.f. Issue #815, PR #817
import numpy as np

# percentiles for -2, -1, 0, 1, 2 standard deviations of the Normal distribution
normal_percentiles = [2.27501319, 15.86552539, 50.0, 84.13447461, 97.72498681]
pvalues_exp_band = np.percentile(
tb.tolist(pvalues),
pvalues,
kratsg marked this conversation as resolved.
Show resolved Hide resolved
normal_percentiles,
axis=0,
).T.tolist()
return pvalues_exp_band
return [[tb.astensor(pvalue) for pvalue in band] for band in pvalues_exp_band]

def teststatistic(self, poi_test):
"""
Expand Down Expand Up @@ -860,7 +862,7 @@ def teststatistic(self, poi_test):
poi_test (:obj:`float` or :obj:`tensor`): The value for the parameter of interest.

Returns:
Float: The value of the test statistic.
Tensor: The value of the test statistic.

"""
teststat_func = utils.get_test_stat(self.test_stat)
Expand Down
2 changes: 1 addition & 1 deletion src/pyhf/infer/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ def create_calculator(calctype, *args, **kwargs):
... )
>>> qmu_sig, qmu_bkg = toy_calculator.distributions(mu_test)
>>> qmu_sig.pvalue(mu_test), qmu_bkg.pvalue(mu_test)
(0.14, 0.76)
(array(0.14), array(0.76))

Args:
calctype (:obj:`str`): The calculator to create. Choose either
Expand Down