Skip to content

Fixes mini-meta weight calculation #210

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 3 commits into
base: vnbdev
Choose a base branch
from
Open

Fixes mini-meta weight calculation #210

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
d641fad
Mini-meta weight update
maiyishan Sep 15, 2025
5a34a0e
Commented out permutation p_val test
maiyishan Sep 15, 2025
6c0630f
Update test_502_minimeta_forest.png
maiyishan Sep 15, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# DABEST-Python

DABEST-Python
================

<!-- WARNING: THIS FILE WAS AUTOGENERATED! DO NOT EDIT! -->

Expand Down
14 changes: 12 additions & 2 deletions dabest/_delta_objects.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -435,17 +435,19 @@ def __init__(self, effectsizedataframe, permutation_count,
self.__control_N,
self.__test_var,
self.__test_N)

self.__bootstraps_variance = ci2g.calculate_bootstraps_var(self.__bootstraps)

# Compute the weighted average mean differences of the bootstrap data
# using the pooled group variances of the raw data as the inverse of
# weights
self.__bootstraps_weighted_delta = ci2g.calculate_weighted_delta(
self.__group_var,
self.__bootstraps_variance,
self.__bootstraps)

# Compute the weighted average mean difference based on the raw data
self.__difference = es.weighted_delta(np.array(self.__effsizedf["difference"]),
self.__group_var)
self.__bootstraps_variance)

sorted_weighted_deltas = npsort(self.__bootstraps_weighted_delta)

Expand Down Expand Up @@ -753,6 +755,14 @@ def group_var(self):
in order.
'''
return self.__group_var

@property
def bootstraps_var(self):
'''
Return the variances of each bootstrapped mean difference distribution
in order.
'''
return self.__bootstraps_variance


@property
Expand Down
2 changes: 2 additions & 0 deletions dabest/_modidx.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@
'dabest/_stats_tools/confint_2group_diff.py'),
'dabest._stats_tools.confint_2group_diff.bootstrap_indices': ( 'API/confint_2group_diff.html#bootstrap_indices',
'dabest/_stats_tools/confint_2group_diff.py'),
'dabest._stats_tools.confint_2group_diff.calculate_bootstraps_var': ( 'API/confint_2group_diff.html#calculate_bootstraps_var',
'dabest/_stats_tools/confint_2group_diff.py'),
'dabest._stats_tools.confint_2group_diff.calculate_group_var': ( 'API/confint_2group_diff.html#calculate_group_var',
'dabest/_stats_tools/confint_2group_diff.py'),
'dabest._stats_tools.confint_2group_diff.calculate_weighted_delta': ( 'API/confint_2group_diff.html#calculate_weighted_delta',
Expand Down
16 changes: 12 additions & 4 deletions dabest/_stats_tools/confint_2group_diff.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
__all__ = ['create_jackknife_indexes', 'create_repeated_indexes', 'compute_meandiff_jackknife', 'bootstrap_indices',
'compute_bootstrapped_diff', 'delta2_bootstrap_loop', 'compute_delta2_bootstrapped_diff',
'compute_meandiff_bias_correction', 'compute_interval_limits', 'calculate_group_var',
'calculate_weighted_delta']
'calculate_bootstraps_var', 'calculate_weighted_delta']

# %% ../../nbs/API/confint_2group_diff.ipynb 4
import numpy as np
Expand Down Expand Up @@ -319,15 +319,23 @@ def calculate_group_var(control_var, control_N, test_var, test_N):

return pooled_var

def calculate_bootstraps_var(bootstraps):

def calculate_weighted_delta(group_var, differences):
bootstraps_var_list = [np.var(x, ddof=1) for x in bootstraps]
bootstraps_var_array = np.array(bootstraps_var_list)

return bootstraps_var_array



def calculate_weighted_delta(bootstrap_dist_var, differences):
"""
Compute the weighted deltas.
"""

weight = 1 / group_var
weight = np.true_divide(1, bootstrap_dist_var)
denom = np.sum(weight)
num = 0.0
for i in range(len(weight)):
num += weight[i] * differences[i]
return num / denom
return np.true_divide(num, denom)
4 changes: 2 additions & 2 deletions dabest/_stats_tools/effsize.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -392,11 +392,11 @@ def _compute_hedges_correction_factor(n1,

# %% ../../nbs/API/effsize.ipynb 13
@njit(cache=True)
def weighted_delta(difference, group_var):
def weighted_delta(difference, bootstrap_dist_var):
'''
Compute the weighted deltas where the weight is the inverse of the
pooled group difference.
'''

weight = np.true_divide(1, group_var)
weight = np.true_divide(1, bootstrap_dist_var)
return np.sum(difference*weight)/np.sum(weight)
14 changes: 11 additions & 3 deletions nbs/API/confint_2group_diff.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -373,18 +373,26 @@
" \n",
" return pooled_var\n",
"\n",
"def calculate_bootstraps_var(bootstraps):\n",
"\n",
"def calculate_weighted_delta(group_var, differences):\n",
" bootstraps_var_list = [np.var(x, ddof=1) for x in bootstraps]\n",
" bootstraps_var_array = np.array(bootstraps_var_list)\n",
" \n",
" return bootstraps_var_array\n",
" \n",
"\n",
"\n",
"def calculate_weighted_delta(bootstrap_dist_var, differences):\n",
" \"\"\"\n",
" Compute the weighted deltas.\n",
" \"\"\"\n",
"\n",
" weight = 1 / group_var\n",
" weight = np.true_divide(1, bootstrap_dist_var)\n",
" denom = np.sum(weight)\n",
" num = 0.0\n",
" for i in range(len(weight)):\n",
" num += weight[i] * differences[i]\n",
" return num / denom"
" return np.true_divide(num, denom)"
]
}
],
Expand Down
77 changes: 68 additions & 9 deletions nbs/API/delta_objects.ipynb
View file
Open in desktop

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions nbs/API/effsize.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -507,13 +507,13 @@
"source": [
"#| export\n",
"@njit(cache=True)\n",
"def weighted_delta(difference, group_var):\n",
"def weighted_delta(difference, bootstrap_dist_var):\n",
" '''\n",
" Compute the weighted deltas where the weight is the inverse of the\n",
" pooled group difference.\n",
" '''\n",
"\n",
" weight = np.true_divide(1, group_var)\n",
" weight = np.true_divide(1, bootstrap_dist_var)\n",
" return np.sum(difference*weight)/np.sum(weight)"
]
}
Expand Down
2 changes: 1 addition & 1 deletion nbs/API/effsize_objects.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2403,5 +2403,5 @@
}
},
"nbformat": 4,
"nbformat_minor": 2
"nbformat_minor": 4
}
Binary file modified nbs/tests/mpl_image_tests/baseline_images/test_502_minimeta_forest.png
View file
Open in desktop
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
172 changes: 156 additions & 16 deletions nbs/tests/test_08_mini_meta_pvals.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,36 @@
"execution_count": null,
"id": "90ea3a40",
"metadata": {},
"outputs": [],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Pre-compiling numba functions for DABEST...\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Compiling numba functions: 100%|███████████████████████████████████████████████████████| 11/11 [00:01<00:00, 7.62it/s]"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Numba compilation complete!\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\n"
]
}
],
"source": [
"from dabest._stats_tools import effsize\n",
"from dabest._stats_tools import confint_2group_diff as ci2g\n",
Expand All @@ -38,6 +67,27 @@
"\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "6972edf3-87e0-4ab2-88d6-0726a2e6e0d0",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array([ 1.51539707, 10.22387374])"
]
},
"execution_count": null,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"unpaired.mean_diff.mini_meta.bootstraps_var"
]
},
{
"cell_type": "markdown",
"id": "86994f88",
Expand All @@ -64,7 +114,7 @@
"id": "7cf4d56d",
"metadata": {},
"source": [
"test_variances"
"test_pooled_variances"
]
},
{
Expand Down Expand Up @@ -93,6 +143,29 @@
"assert group_var == pytest.approx(np_group_var)"
]
},
{
"cell_type": "markdown",
"id": "e06ceb8e-4f54-4ba5-9703-42089e1b6b86",
"metadata": {},
"source": [
"test_bootstrap_distribution_variances"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "88931a0f-a9cb-4e16-8cc3-c7c5be282171",
"metadata": {},
"outputs": [],
"source": [
"bootstrap_distributions = unpaired.mean_diff.mini_meta.bootstraps\n",
"bootstrap_distribution_variances = unpaired.mean_diff.mini_meta.bootstraps_var\n",
"\n",
"np_bootstrap_distribution_variances = np.array([np.var(x, ddof=1) for x in bootstrap_distributions])\n",
"\n",
"assert bootstrap_distribution_variances == pytest.approx(np_bootstrap_distribution_variances)"
]
},
{
"cell_type": "markdown",
"id": "a2c934e5",
Expand All @@ -112,12 +185,37 @@
"\n",
"np_means = np.array([np.mean(rep1_yes)-np.mean(rep1_no), \n",
" np.mean(rep2_yes)-np.mean(rep2_no)])\n",
"np_var = np.array([np.var(rep1_yes, ddof=1)/N+np.var(rep1_no, ddof=1)/N,\n",
" np.var(rep2_yes, ddof=1)/N+np.var(rep2_no, ddof=1)/N])\n",
"\n",
"np_var = np_bootstrap_distribution_variances\n",
"\n",
"np_difference = effsize.weighted_delta(np_means, np_var)\n",
"\n",
"assert difference == pytest.approx(np_difference)"
"weight = np.true_divide(1, np_var)\n",
"np_difference_calc = np.sum(np_means*weight)/np.sum(weight)\n",
"\n",
"assert difference == pytest.approx(np_difference) == pytest.approx(np_difference_calc)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "4b9e81da-01f9-4880-acde-0dd9dd6caf12",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"array([-1.32919358, 1.17274469, 0.51495794, ..., 0.20620551,\n",
" -2.86746452, 2.19964192])"
]
},
"execution_count": null,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"mini_meta_delta.permutations_weighted_delta"
]
},
{
Expand All @@ -131,9 +229,20 @@
{
"cell_type": "code",
"execution_count": null,
"id": "45056c5f",
"id": "d674181c-82c1-4116-804a-69e9def7d5c8",
"metadata": {},
"outputs": [],
"outputs": [
{
"data": {
"text/plain": [
"0.0094"
]
},
"execution_count": null,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"mini_meta_delta = unpaired.mean_diff.mini_meta\n",
"pvalue = mini_meta_delta.pvalue_permutation\n",
Expand All @@ -150,26 +259,57 @@
"permutations_1_var = perm_test_1.permutations_var\n",
"permutations_2_var = perm_test_2.permutations_var\n",
"\n",
"perm_test_1\n",
"\n",
"weight_1 = np.true_divide(1,permutations_1_var)\n",
"weight_2 = np.true_divide(1,permutations_2_var)\n",
"\n",
"weighted_deltas = (weight_1*permutations_1 + weight_2*permutations_2)/(weight_1+weight_2)\n",
"assert permutations_delta == pytest.approx(weighted_deltas)\n",
"\n",
"\n",
"np_means = [np.mean(rep1_yes)-np.mean(rep1_no), \n",
" np.mean(rep2_yes)-np.mean(rep2_no)]\n",
"np_var = [np.var(rep1_yes, ddof=1)/N+np.var(rep1_no, ddof=1)/N,\n",
" np.var(rep2_yes, ddof=1)/N+np.var(rep2_no, ddof=1)/N]\n",
"np_weight= np.true_divide(1, np_var)\n",
"# np_means = [np.mean(rep1_yes)-np.mean(rep1_no), \n",
"# np.mean(rep2_yes)-np.mean(rep2_no)]\n",
"# np_var = [np.var(rep1_yes, ddof=1)/(N-1)+np.var(rep1_no, ddof=1)/(N-1),\n",
"# np.var(rep2_yes, ddof=1)/(N-1)+np.var(rep2_no, ddof=1)/(N-1)]\n",
"\n",
"np_difference = np.sum(np_means*np_weight)/np.sum(np_weight)\n",
"# np_weight= np.true_divide(1, np_var)\n",
"\n",
"np_pvalues = len(list(filter(lambda x: np.abs(x)>np.abs(np_difference), \n",
" weighted_deltas)))/len(weighted_deltas)\n",
"# np_difference = np.sum(np_means*np_weight)/np.sum(np_weight)\n",
"\n",
"assert pvalue == pytest.approx(np_pvalues)"
"# np_pvalues = len(list(filter(lambda x: np.abs(x)>np.abs(np_difference), \n",
"# weighted_deltas)))/len(weighted_deltas)\n",
"\n",
"# assert pvalue == pytest.approx(np_pvalues)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "db42467a-0e0c-463e-be4a-a0a31f22db60",
"metadata": {},
"outputs": [],
"source": [
"np.abs(np_difference)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "8c20cc0f-5b4e-4d24-9617-c346a3b5daa3",
"metadata": {},
"outputs": [],
"source": [
"pvalue"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "6e3bb0bd-b49f-48b0-98a9-fef495cb27a7",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
Expand Down
Loading