From ffc3d5fed00966540ddf604f98ce294863de9d39 Mon Sep 17 00:00:00 2001 From: ivana Date: Sat, 6 Dec 2025 13:06:13 +0100 Subject: [PATCH 1/5] Add group-wise inference risks --- .../evaluators/inference_evaluator.py | 123 +++++++++++++++--- tests/test_inference_evaluator.py | 30 ++++- 2 files changed, 131 insertions(+), 22 deletions(-) diff --git a/src/anonymeter/evaluators/inference_evaluator.py b/src/anonymeter/evaluators/inference_evaluator.py index a9ad545..c1a42d0 100644 --- a/src/anonymeter/evaluators/inference_evaluator.py +++ b/src/anonymeter/evaluators/inference_evaluator.py @@ -14,16 +14,16 @@ def _run_attack( - target: pd.DataFrame, - syn: pd.DataFrame, - n_attacks: int, - aux_cols: list[str], - secret: str, - n_jobs: int, - naive: bool, - regression: Optional[bool], - inference_model: Optional[InferencePredictor], -) -> int: + target: pd.DataFrame, + syn: pd.DataFrame, + n_attacks: int, + aux_cols: list[str], + secret: str, + n_jobs: int, + naive: bool, + regression: Optional[bool], + inference_model: Optional[InferencePredictor], +) -> tuple[int, pd.Series]: if regression is None: regression = pd.api.types.is_numeric_dtype(target[secret]) @@ -31,13 +31,18 @@ def _run_attack( if naive: guesses = syn.sample(n_attacks)[secret] + guesses.index = targets.index else: # Instantiate the default KNN model if no other model is passed through `inference_model`. if inference_model is None: inference_model = KNNInferencePredictor(data=syn, columns=aux_cols, target_col=secret, n_jobs=n_jobs) + guesses = inference_model.predict(targets) - return evaluate_inference_guesses(guesses=guesses, secrets=targets[secret], regression=regression).sum() + if not guesses.index.equals(targets.index): + raise RuntimeError("The predictions indices do not match the target indices. Check your inference model.") + + return evaluate_inference_guesses(guesses=guesses, secrets=targets[secret], regression=regression).sum(), guesses def evaluate_inference_guesses( @@ -152,7 +157,7 @@ def __init__( syn: pd.DataFrame, aux_cols: list[str], secret: str, - regression: Optional[bool] = None, + regression: bool = False, n_attacks: int = 500, control: Optional[pd.DataFrame] = None, inference_model: Optional[InferencePredictor] = None @@ -180,7 +185,7 @@ def __init__( self._aux_cols = aux_cols self._evaluated = False - def _attack(self, target: pd.DataFrame, naive: bool, n_jobs: int, n_attacks: int) -> int: + def _attack(self, target: pd.DataFrame, naive: bool, n_jobs: int, n_attacks: int) -> tuple[int, pd.Series]: return _run_attack( target=target, syn=self._syn, @@ -207,14 +212,21 @@ def evaluate(self, n_jobs: int = -2) -> "InferenceEvaluator": The evaluated ``InferenceEvaluator`` object. """ - self._n_baseline = self._attack(target=self._ori, naive=True, n_jobs=n_jobs, - n_attacks=self._n_attacks_baseline) - self._n_success = self._attack(target=self._ori, naive=False, n_jobs=n_jobs, - n_attacks=self._n_attacks_ori) - self._n_control = ( - None if self._control is None else self._attack(target=self._control, naive=False, n_jobs=n_jobs, - n_attacks=self._n_attacks_control) - ) + # n_attacks is effective here + self._n_baseline, self._guesses_baseline = self._attack( + target=self._ori, naive=True, n_jobs=n_jobs, n_attacks=self._n_attacks_baseline + ) + + # n_attacks is not effective here, just needed for the baseline + self._n_success, self._guesses_success = self._attack( + target=self._ori, naive=False, n_jobs=n_jobs, n_attacks=self._n_attacks_ori + ) + # n_attacks is not effective here, just needed for the baseline + self._n_control, self._guesses_control = ( + (None, None) + if self._control is None + else self._attack(target=self._control, naive=False, n_jobs=n_jobs, n_attacks=self._n_attacks_control) + ) self._evaluated = True return self @@ -269,3 +281,72 @@ def risk(self, confidence_level: float = 0.95, baseline: bool = False) -> Privac """ results = self.results(confidence_level=confidence_level) return results.risk(baseline=baseline) + + def risk_for_groups(self, confidence_level: float = 0.95) -> dict[str, tuple[EvaluationResults, PrivacyRisk]]: + """Compute the attack risks on a group level, for every unique value of `self._data_groups`. + + Parameters + ---------- + confidence_level : float, default is 0.95 + Confidence level for the error bound calculation. + + Returns + ------- + dict[str, tuple[EvaluationResults | PrivacyRisk] + The group as a key, and then for every group the results (EvaluationResults), + and the risks (PrivacyRisk) as a tuple. + + """ + if not self._evaluated: + self.evaluate(n_jobs=-2) + + all_results = {} + + # For every unique group in `self._data_groups` + for group, data_ori in self._ori.groupby(self._secret): + # Get the targets for the current group + common_indices = data_ori.index.intersection(self._guesses_success.index) + # Get the guesses for the current group + data_ori = data_ori.loc[common_indices] + n_attacks_ori = len(data_ori) + + # Count the number of success attacks + n_success = evaluate_inference_guesses( + guesses=self._guesses_success.loc[common_indices], + secrets=data_ori[self._secret], + regression=self._regression, + ).sum() + + if self._control is not None: + # Get the targets for the current control group + data_control = self._control[self._control[self._secret] == group] + n_attacks_control = len(data_control) + + # Get the guesses for the current control group + common_indices = data_control.index.intersection(self._guesses_control.index) + + # Count the number of success control attacks + n_control = evaluate_inference_guesses( + guesses=self._guesses_control.loc[common_indices], + secrets=data_control[self._secret], + regression=self._regression, + ).sum() + else: + n_control = None + n_attacks_control = -1 + + # Recreate the EvaluationResults for the current group + assert n_attacks_ori == n_success + results = EvaluationResults( + n_attacks=(n_attacks_ori, self._n_attacks_baseline, n_attacks_control), + n_success=n_success, + n_baseline=self._n_baseline, # The baseline risk should be the same independent of the group + n_control=n_control, + confidence_level=confidence_level, + ) + # Compute the risk + risk = results.risk() + + all_results[group] = (results, risk) + + return all_results diff --git a/tests/test_inference_evaluator.py b/tests/test_inference_evaluator.py index eeaa934..f3424cf 100644 --- a/tests/test_inference_evaluator.py +++ b/tests/test_inference_evaluator.py @@ -8,6 +8,7 @@ import pytest from anonymeter.evaluators.inference_evaluator import InferenceEvaluator, evaluate_inference_guesses +from anonymeter.stats.confidence import EvaluationResults from tests.fixtures import get_adult @@ -104,7 +105,10 @@ def test_inference_evaluator_rates( @pytest.mark.parametrize("secret", ["education", "marital", "capital_gain"]) def test_inference_evaluator_leaks(aux_cols, secret): ori = get_adult("ori", n_samples=10) - evaluator = InferenceEvaluator(ori=ori, syn=ori, control=ori, aux_cols=aux_cols, secret=secret, n_attacks=10) + ori = ori.drop_duplicates(subset=aux_cols) + evaluator = InferenceEvaluator( + ori=ori, syn=ori, control=ori, aux_cols=aux_cols, secret=secret, n_attacks=ori.shape[0] + ) evaluator.evaluate(n_jobs=1) results = evaluator.results(confidence_level=0) @@ -123,3 +127,27 @@ def test_evaluator_not_evaluated(): ) with pytest.raises(RuntimeError): evaluator.risk() + + +@pytest.mark.parametrize( + "aux_cols", + [ + ["type_employer", "capital_loss", "hr_per_week", "age"], + ["education_num", "marital", "capital_loss"], + ], +) +@pytest.mark.parametrize("secret", ["education", "marital"]) +def test_inference_evaluator_group_wise(aux_cols, secret): + ori = get_adult("ori", n_samples=10) + ori = ori.drop_duplicates(subset=aux_cols) + evaluator = InferenceEvaluator( + ori=ori, syn=ori, control=ori, aux_cols=aux_cols, secret=secret, n_attacks=ori.shape[0] + ) + evaluator.evaluate(n_jobs=1) + + group_wise = evaluator.risk_for_groups(confidence_level=0) + + for _, res in group_wise.items(): + results: EvaluationResults = res[0] + np.testing.assert_equal(results.attack_rate, (1, 0)) + np.testing.assert_equal(results.control_rate, (1, 0)) From cd887671bd680f1cc211a75ede25360736bcc4a8 Mon Sep 17 00:00:00 2001 From: ivana Date: Wed, 17 Dec 2025 15:34:21 +0100 Subject: [PATCH 2/5] Address code refactoring comments; Make test fixture code drop duplicates; Remove old comments; Add RuntimeError test. --- .../evaluators/inference_evaluator.py | 23 ++++++++----------- tests/fixtures.py | 7 ++++-- tests/test_inference_evaluator.py | 23 +++++++++++++++---- tests/test_mixed_types_kneigbors.py | 4 ++-- tests/test_singling_out_evaluator.py | 8 +++---- tests/test_sklearn_inference_model.py | 2 +- 6 files changed, 39 insertions(+), 28 deletions(-) diff --git a/src/anonymeter/evaluators/inference_evaluator.py b/src/anonymeter/evaluators/inference_evaluator.py index c1a42d0..a250a01 100644 --- a/src/anonymeter/evaluators/inference_evaluator.py +++ b/src/anonymeter/evaluators/inference_evaluator.py @@ -31,16 +31,13 @@ def _run_attack( if naive: guesses = syn.sample(n_attacks)[secret] - guesses.index = targets.index else: # Instantiate the default KNN model if no other model is passed through `inference_model`. if inference_model is None: inference_model = KNNInferencePredictor(data=syn, columns=aux_cols, target_col=secret, n_jobs=n_jobs) guesses = inference_model.predict(targets) - - if not guesses.index.equals(targets.index): - raise RuntimeError("The predictions indices do not match the target indices. Check your inference model.") + guesses = guesses.reindex_like(targets) return evaluate_inference_guesses(guesses=guesses, secrets=targets[secret], regression=regression).sum(), guesses @@ -77,6 +74,9 @@ def evaluate_inference_guesses( Array of boolean values indicating the correcteness of each guess. """ + if not guesses.index.equals(secrets.index): + raise RuntimeError("The predictions indices do not match the target indices. Check your inference model.") + guesses_np = guesses.to_numpy() secrets_np = secrets.to_numpy() @@ -212,16 +212,12 @@ def evaluate(self, n_jobs: int = -2) -> "InferenceEvaluator": The evaluated ``InferenceEvaluator`` object. """ - # n_attacks is effective here self._n_baseline, self._guesses_baseline = self._attack( target=self._ori, naive=True, n_jobs=n_jobs, n_attacks=self._n_attacks_baseline ) - - # n_attacks is not effective here, just needed for the baseline self._n_success, self._guesses_success = self._attack( target=self._ori, naive=False, n_jobs=n_jobs, n_attacks=self._n_attacks_ori ) - # n_attacks is not effective here, just needed for the baseline self._n_control, self._guesses_control = ( (None, None) if self._control is None @@ -283,7 +279,7 @@ def risk(self, confidence_level: float = 0.95, baseline: bool = False) -> Privac return results.risk(baseline=baseline) def risk_for_groups(self, confidence_level: float = 0.95) -> dict[str, tuple[EvaluationResults, PrivacyRisk]]: - """Compute the attack risks on a group level, for every unique value of `self._data_groups`. + """Compute the inference risk for each group of targets with the same value of the secret attribute. Parameters ---------- @@ -298,7 +294,7 @@ def risk_for_groups(self, confidence_level: float = 0.95) -> dict[str, tuple[Eva """ if not self._evaluated: - self.evaluate(n_jobs=-2) + raise RuntimeError("The inference evaluator wasn't evaluated yet. Please, run `evaluate()` first.") all_results = {} @@ -307,13 +303,13 @@ def risk_for_groups(self, confidence_level: float = 0.95) -> dict[str, tuple[Eva # Get the targets for the current group common_indices = data_ori.index.intersection(self._guesses_success.index) # Get the guesses for the current group - data_ori = data_ori.loc[common_indices] - n_attacks_ori = len(data_ori) + target_group = data_ori.loc[common_indices] + n_attacks_ori = len(target_group) # Count the number of success attacks n_success = evaluate_inference_guesses( guesses=self._guesses_success.loc[common_indices], - secrets=data_ori[self._secret], + secrets=target_group[self._secret], regression=self._regression, ).sum() @@ -336,7 +332,6 @@ def risk_for_groups(self, confidence_level: float = 0.95) -> dict[str, tuple[Eva n_attacks_control = -1 # Recreate the EvaluationResults for the current group - assert n_attacks_ori == n_success results = EvaluationResults( n_attacks=(n_attacks_ori, self._n_attacks_baseline, n_attacks_control), n_success=n_success, diff --git a/tests/fixtures.py b/tests/fixtures.py index d29fc90..0633c10 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -11,7 +11,7 @@ TEST_DIR_PATH = os.path.dirname(os.path.realpath(__file__)) -def get_adult(which: str, n_samples: Optional[int] = None) -> pd.DataFrame: +def get_adult(which: str, deduplicate_on: Optional[list[str]] = None, n_samples: Optional[int] = None) -> pd.DataFrame: """Fixture for the adult dataset. For details see: @@ -21,6 +21,8 @@ def get_adult(which: str, n_samples: Optional[int] = None) -> pd.DataFrame: ---------- which : str, in ['ori', 'syn'] Whether to return the "original" or "synthetic" samples. + deduplicate_on: list of str + A list of columns based on which we'd drop duplicates in the samples. n_samples : int Number of sample records to return. If `None` - return all samples. @@ -37,4 +39,5 @@ def get_adult(which: str, n_samples: Optional[int] = None) -> pd.DataFrame: else: raise ValueError(f"Invalid value {which} for parameter `which`. Available are: 'ori' or 'syn'.") - return pd.read_csv(os.path.join(TEST_DIR_PATH, "datasets", fname), nrows=n_samples) + samples = pd.read_csv(os.path.join(TEST_DIR_PATH, "datasets", fname), nrows=n_samples) + return samples.drop_duplicates(subset=deduplicate_on) if deduplicate_on else samples diff --git a/tests/test_inference_evaluator.py b/tests/test_inference_evaluator.py index f3424cf..95b83c1 100644 --- a/tests/test_inference_evaluator.py +++ b/tests/test_inference_evaluator.py @@ -30,6 +30,21 @@ def test_evaluate_inference_guesses_classification(guesses, secrets, expected): np.testing.assert_equal(out, expected) +@pytest.mark.parametrize( + "guesses, secrets, expected", + [ + (("a", "b"), ("a", "b"), (True, True)), + ((np.nan, "b"), (np.nan, "b"), (True, True)) + ], +) +def test_evaluate_inference_guesses_secrets_indices(guesses, secrets, expected): + secrets = pd.Series(secrets).sort_index(ascending=False) + with pytest.raises(Exception) as runtime_error: + evaluate_inference_guesses(guesses=pd.Series(guesses), secrets=secrets, regression=False) + assert runtime_error.type is RuntimeError + assert "The predictions indices do not match the target indices" in str(runtime_error.value) + + @pytest.mark.parametrize( "guesses, secrets, expected", [ @@ -104,8 +119,7 @@ def test_inference_evaluator_rates( ) @pytest.mark.parametrize("secret", ["education", "marital", "capital_gain"]) def test_inference_evaluator_leaks(aux_cols, secret): - ori = get_adult("ori", n_samples=10) - ori = ori.drop_duplicates(subset=aux_cols) + ori = get_adult("ori", deduplicate_on=aux_cols, n_samples=10) evaluator = InferenceEvaluator( ori=ori, syn=ori, control=ori, aux_cols=aux_cols, secret=secret, n_attacks=ori.shape[0] ) @@ -117,7 +131,7 @@ def test_inference_evaluator_leaks(aux_cols, secret): def test_evaluator_not_evaluated(): - df = get_adult("ori", n_samples=10) + df = get_adult("ori", deduplicate_on=None, n_samples=10) evaluator = InferenceEvaluator( ori=df, syn=df, @@ -138,8 +152,7 @@ def test_evaluator_not_evaluated(): ) @pytest.mark.parametrize("secret", ["education", "marital"]) def test_inference_evaluator_group_wise(aux_cols, secret): - ori = get_adult("ori", n_samples=10) - ori = ori.drop_duplicates(subset=aux_cols) + ori = get_adult("ori", deduplicate_on=aux_cols, n_samples=10) evaluator = InferenceEvaluator( ori=ori, syn=ori, control=ori, aux_cols=aux_cols, secret=secret, n_attacks=ori.shape[0] ) diff --git a/tests/test_mixed_types_kneigbors.py b/tests/test_mixed_types_kneigbors.py index 21e0b26..9347af9 100644 --- a/tests/test_mixed_types_kneigbors.py +++ b/tests/test_mixed_types_kneigbors.py @@ -13,7 +13,7 @@ def test_mixed_type_kNN(): - df = get_adult("ori", n_samples=10) + df = get_adult("ori", deduplicate_on=None, n_samples=10) nn = MixedTypeKNeighbors().fit(df) shuffled_idx = rng.integers(10, size=10) dist, ids = nn.kneighbors(df.iloc[shuffled_idx], n_neighbors=1, return_distance=True) @@ -43,7 +43,7 @@ def test_mixed_type_kNN_numerical_scaling(): @pytest.mark.parametrize("n_neighbors, n_queries", [(1, 10), (3, 5)]) def test_mixed_type_kNN_shape(n_neighbors, n_queries): - df = get_adult("ori", n_samples=10) + df = get_adult("ori", deduplicate_on=None, n_samples=10) nn = MixedTypeKNeighbors(n_neighbors=n_neighbors).fit(df) ids = nn.kneighbors(df.head(n_queries)) assert isinstance(ids, np.ndarray) diff --git a/tests/test_singling_out_evaluator.py b/tests/test_singling_out_evaluator.py index 05a79cb..8b1da7b 100644 --- a/tests/test_singling_out_evaluator.py +++ b/tests/test_singling_out_evaluator.py @@ -23,8 +23,8 @@ @pytest.mark.parametrize("mode", ["univariate", "multivariate"]) def test_so_general(mode: str) -> None: - ori = get_adult("ori", n_samples=10) - syn = get_adult("syn", n_samples=10) + ori = get_adult("ori", deduplicate_on=None, n_samples=10) + syn = get_adult("syn", deduplicate_on=None, n_samples=10) soe = SinglingOutEvaluator(ori=ori, syn=syn, n_attacks=5).evaluate(mode=mode) for q in soe.queries(): @@ -150,7 +150,7 @@ def test_singling_out_query_generator() -> None: @pytest.mark.parametrize("confidence_level", [0.5, 0.68, 0.95, 0.99]) @pytest.mark.parametrize("mode", ["univariate", "multivariate"]) def test_singling_out_risk_estimate(confidence_level: float, mode: str) -> None: - ori = get_adult("ori", 10) + ori = get_adult("ori", deduplicate_on=None, n_samples=10) soe = SinglingOutEvaluator(ori=ori, syn=ori, n_attacks=5) soe.evaluate(mode=mode) _, ci = soe.risk(confidence_level=confidence_level) @@ -176,7 +176,7 @@ def _so_probability(n: int, w: float): @pytest.mark.parametrize("max_attempts", [1, 2, 3]) def test_so_evaluator_max_attempts(max_attempts: int) -> None: - ori = get_adult("ori", 10) + ori = get_adult("ori", deduplicate_on=None, n_samples=10) soe = SinglingOutEvaluator(ori=ori, syn=ori, n_attacks=10, max_attempts=max_attempts) soe.evaluate(mode="multivariate") diff --git a/tests/test_sklearn_inference_model.py b/tests/test_sklearn_inference_model.py index ac45d2a..d352310 100644 --- a/tests/test_sklearn_inference_model.py +++ b/tests/test_sklearn_inference_model.py @@ -25,7 +25,7 @@ ) @pytest.mark.parametrize("secret", ["capital_gain", "capital_loss"]) def test_inference_evaluator_custom_model_regressor(aux_cols, secret): - ori = get_adult("ori", n_samples=10) + ori = get_adult("ori", deduplicate_on=aux_cols, n_samples=10) # Inference model prep categorical_cols = ori[aux_cols].select_dtypes(include=["object"]).columns From 3b01d5a3683a34d4acc0cc2e256f95eb3f60c9df Mon Sep 17 00:00:00 2001 From: ivana Date: Wed, 17 Dec 2025 15:45:08 +0100 Subject: [PATCH 3/5] Address code refactoring comments. --- src/anonymeter/evaluators/inference_evaluator.py | 10 +++------- tests/test_inference_evaluator.py | 4 +--- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/src/anonymeter/evaluators/inference_evaluator.py b/src/anonymeter/evaluators/inference_evaluator.py index a250a01..51682ea 100644 --- a/src/anonymeter/evaluators/inference_evaluator.py +++ b/src/anonymeter/evaluators/inference_evaluator.py @@ -278,7 +278,7 @@ def risk(self, confidence_level: float = 0.95, baseline: bool = False) -> Privac results = self.results(confidence_level=confidence_level) return results.risk(baseline=baseline) - def risk_for_groups(self, confidence_level: float = 0.95) -> dict[str, tuple[EvaluationResults, PrivacyRisk]]: + def risk_for_groups(self, confidence_level: float = 0.95) -> dict[str, EvaluationResults]: """Compute the inference risk for each group of targets with the same value of the secret attribute. Parameters @@ -298,7 +298,7 @@ def risk_for_groups(self, confidence_level: float = 0.95) -> dict[str, tuple[Eva all_results = {} - # For every unique group in `self._data_groups` + # For every unique group in `self._secret` for group, data_ori in self._ori.groupby(self._secret): # Get the targets for the current group common_indices = data_ori.index.intersection(self._guesses_success.index) @@ -332,16 +332,12 @@ def risk_for_groups(self, confidence_level: float = 0.95) -> dict[str, tuple[Eva n_attacks_control = -1 # Recreate the EvaluationResults for the current group - results = EvaluationResults( + all_results[group] = EvaluationResults( n_attacks=(n_attacks_ori, self._n_attacks_baseline, n_attacks_control), n_success=n_success, n_baseline=self._n_baseline, # The baseline risk should be the same independent of the group n_control=n_control, confidence_level=confidence_level, ) - # Compute the risk - risk = results.risk() - - all_results[group] = (results, risk) return all_results diff --git a/tests/test_inference_evaluator.py b/tests/test_inference_evaluator.py index 95b83c1..401def5 100644 --- a/tests/test_inference_evaluator.py +++ b/tests/test_inference_evaluator.py @@ -8,7 +8,6 @@ import pytest from anonymeter.evaluators.inference_evaluator import InferenceEvaluator, evaluate_inference_guesses -from anonymeter.stats.confidence import EvaluationResults from tests.fixtures import get_adult @@ -160,7 +159,6 @@ def test_inference_evaluator_group_wise(aux_cols, secret): group_wise = evaluator.risk_for_groups(confidence_level=0) - for _, res in group_wise.items(): - results: EvaluationResults = res[0] + for _, results in group_wise.items(): np.testing.assert_equal(results.attack_rate, (1, 0)) np.testing.assert_equal(results.control_rate, (1, 0)) From 291c1f520e3c1d8158f7285d09422c60c8bde354 Mon Sep 17 00:00:00 2001 From: ivana Date: Thu, 8 Jan 2026 11:48:31 +0100 Subject: [PATCH 4/5] Add risk check test; Cleanup get_adult call. --- tests/fixtures.py | 6 +++-- tests/test_inference_evaluator.py | 38 ++++++++++++++++++++++++---- tests/test_mixed_types_kneigbors.py | 4 +-- tests/test_singling_out_evaluator.py | 8 +++--- 4 files changed, 43 insertions(+), 13 deletions(-) diff --git a/tests/fixtures.py b/tests/fixtures.py index 0633c10..e7edc1c 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -39,5 +39,7 @@ def get_adult(which: str, deduplicate_on: Optional[list[str]] = None, n_samples: else: raise ValueError(f"Invalid value {which} for parameter `which`. Available are: 'ori' or 'syn'.") - samples = pd.read_csv(os.path.join(TEST_DIR_PATH, "datasets", fname), nrows=n_samples) - return samples.drop_duplicates(subset=deduplicate_on) if deduplicate_on else samples + samples = pd.read_csv(os.path.join(TEST_DIR_PATH, "datasets", fname)) + if deduplicate_on: + samples = samples.drop_duplicates(subset=deduplicate_on) + return samples.iloc[:n_samples] diff --git a/tests/test_inference_evaluator.py b/tests/test_inference_evaluator.py index 401def5..3b1afbb 100644 --- a/tests/test_inference_evaluator.py +++ b/tests/test_inference_evaluator.py @@ -38,10 +38,9 @@ def test_evaluate_inference_guesses_classification(guesses, secrets, expected): ) def test_evaluate_inference_guesses_secrets_indices(guesses, secrets, expected): secrets = pd.Series(secrets).sort_index(ascending=False) - with pytest.raises(Exception) as runtime_error: + with pytest.raises(RuntimeError) as runtime_error: evaluate_inference_guesses(guesses=pd.Series(guesses), secrets=secrets, regression=False) - assert runtime_error.type is RuntimeError - assert "The predictions indices do not match the target indices" in str(runtime_error.value) + assert "The predictions indices do not match the target indices" in str(runtime_error.value) @pytest.mark.parametrize( @@ -130,7 +129,7 @@ def test_inference_evaluator_leaks(aux_cols, secret): def test_evaluator_not_evaluated(): - df = get_adult("ori", deduplicate_on=None, n_samples=10) + df = get_adult("ori", n_samples=10) evaluator = InferenceEvaluator( ori=df, syn=df, @@ -150,7 +149,7 @@ def test_evaluator_not_evaluated(): ], ) @pytest.mark.parametrize("secret", ["education", "marital"]) -def test_inference_evaluator_group_wise(aux_cols, secret): +def test_inference_evaluator_group_wise_rates(aux_cols, secret): ori = get_adult("ori", deduplicate_on=aux_cols, n_samples=10) evaluator = InferenceEvaluator( ori=ori, syn=ori, control=ori, aux_cols=aux_cols, secret=secret, n_attacks=ori.shape[0] @@ -162,3 +161,32 @@ def test_inference_evaluator_group_wise(aux_cols, secret): for _, results in group_wise.items(): np.testing.assert_equal(results.attack_rate, (1, 0)) np.testing.assert_equal(results.control_rate, (1, 0)) + +@pytest.mark.parametrize( + "aux_cols", + [ + ["type_employer", "capital_loss", "hr_per_week", "age"], + ["education_num", "marital", "capital_loss"], + ], +) +@pytest.mark.parametrize("secret", ["education", "marital"]) +def test_inference_evaluator_group_wise_risks(aux_cols, secret): + ori = get_adult("ori", deduplicate_on=aux_cols, n_samples=10) + evaluator = InferenceEvaluator( + ori=ori, syn=ori, control=ori, aux_cols=aux_cols, secret=secret, n_attacks=ori.shape[0] + ) + evaluator.evaluate(n_jobs=1) + main_risk = evaluator.risk(confidence_level=0.95) + + group_wise = evaluator.risk_for_groups(confidence_level=0.95) + + sum_risks = 0 + for _, results in group_wise.items(): + risk = results.risk().value + np.testing.assert_equal(risk, 0) + + sum_risks += risk + + np.testing.assert_allclose(sum_risks, main_risk.value) + +#%% diff --git a/tests/test_mixed_types_kneigbors.py b/tests/test_mixed_types_kneigbors.py index 9347af9..21e0b26 100644 --- a/tests/test_mixed_types_kneigbors.py +++ b/tests/test_mixed_types_kneigbors.py @@ -13,7 +13,7 @@ def test_mixed_type_kNN(): - df = get_adult("ori", deduplicate_on=None, n_samples=10) + df = get_adult("ori", n_samples=10) nn = MixedTypeKNeighbors().fit(df) shuffled_idx = rng.integers(10, size=10) dist, ids = nn.kneighbors(df.iloc[shuffled_idx], n_neighbors=1, return_distance=True) @@ -43,7 +43,7 @@ def test_mixed_type_kNN_numerical_scaling(): @pytest.mark.parametrize("n_neighbors, n_queries", [(1, 10), (3, 5)]) def test_mixed_type_kNN_shape(n_neighbors, n_queries): - df = get_adult("ori", deduplicate_on=None, n_samples=10) + df = get_adult("ori", n_samples=10) nn = MixedTypeKNeighbors(n_neighbors=n_neighbors).fit(df) ids = nn.kneighbors(df.head(n_queries)) assert isinstance(ids, np.ndarray) diff --git a/tests/test_singling_out_evaluator.py b/tests/test_singling_out_evaluator.py index 8b1da7b..066abc0 100644 --- a/tests/test_singling_out_evaluator.py +++ b/tests/test_singling_out_evaluator.py @@ -23,8 +23,8 @@ @pytest.mark.parametrize("mode", ["univariate", "multivariate"]) def test_so_general(mode: str) -> None: - ori = get_adult("ori", deduplicate_on=None, n_samples=10) - syn = get_adult("syn", deduplicate_on=None, n_samples=10) + ori = get_adult("ori", n_samples=10) + syn = get_adult("syn", n_samples=10) soe = SinglingOutEvaluator(ori=ori, syn=syn, n_attacks=5).evaluate(mode=mode) for q in soe.queries(): @@ -150,7 +150,7 @@ def test_singling_out_query_generator() -> None: @pytest.mark.parametrize("confidence_level", [0.5, 0.68, 0.95, 0.99]) @pytest.mark.parametrize("mode", ["univariate", "multivariate"]) def test_singling_out_risk_estimate(confidence_level: float, mode: str) -> None: - ori = get_adult("ori", deduplicate_on=None, n_samples=10) + ori = get_adult("ori", n_samples=10) soe = SinglingOutEvaluator(ori=ori, syn=ori, n_attacks=5) soe.evaluate(mode=mode) _, ci = soe.risk(confidence_level=confidence_level) @@ -176,7 +176,7 @@ def _so_probability(n: int, w: float): @pytest.mark.parametrize("max_attempts", [1, 2, 3]) def test_so_evaluator_max_attempts(max_attempts: int) -> None: - ori = get_adult("ori", deduplicate_on=None, n_samples=10) + ori = get_adult("ori", n_samples=10) soe = SinglingOutEvaluator(ori=ori, syn=ori, n_attacks=10, max_attempts=max_attempts) soe.evaluate(mode="multivariate") From 030fbfb1473ca6601a3429994abce577a4f348b6 Mon Sep 17 00:00:00 2001 From: ivana Date: Thu, 8 Jan 2026 11:56:13 +0100 Subject: [PATCH 5/5] Add risk check test; Cleanup get_adult call. --- tests/test_inference_evaluator.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/test_inference_evaluator.py b/tests/test_inference_evaluator.py index 3b1afbb..c572ac4 100644 --- a/tests/test_inference_evaluator.py +++ b/tests/test_inference_evaluator.py @@ -180,7 +180,7 @@ def test_inference_evaluator_group_wise_risks(aux_cols, secret): group_wise = evaluator.risk_for_groups(confidence_level=0.95) - sum_risks = 0 + sum_risks = 0.0 for _, results in group_wise.items(): risk = results.risk().value np.testing.assert_equal(risk, 0) @@ -188,5 +188,3 @@ def test_inference_evaluator_group_wise_risks(aux_cols, secret): sum_risks += risk np.testing.assert_allclose(sum_risks, main_risk.value) - -#%%