gradient-regularisation-interpretability/p_value.py at master · saeyslab/gradient-regularisation-interpretability · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
import pickle

import numpy as np
import pandas as pd
from matplotlib import pyplot as plt
from quantus import FaithfulnessEstimate, LocalLipschitzEstimate, MaxSensitivity, Complexity
from scipy.stats import sem, pearsonr, spearmanr, permutation_test

from main import CHECKPOINTS_FOLDER
from modules.constants import OPTIONS
from modules.parser import DATA_ROOT
from plot_saliency import read_files, DISTINGUISHERS
from saliency import USED_METRICS

ALL_CHECKPOINTS_FOLDER = CHECKPOINTS_FOLDER.parent
TRAINING_CONFIGURATIONS = "LeNet_kmnist", "LeNet_fashmnist", "LeNet_mnist", "ResNet_mnist", "resnet_cifar", "resnette_single"

TRAINING_CONFIGURATION_TO_OPTION = {tc.lower(): opt for tc, opt in zip(TRAINING_CONFIGURATIONS, OPTIONS)}

np.seterr(all='raise')

galpha_cutoff = {}

for checkpoints_folder in ALL_CHECKPOINTS_FOLDER.iterdir():
    with (checkpoints_folder / "regular_backprop_result.pickle").open("rb") as file:
        accuracies = pickle.load(file)
    galpha_list = sorted(accuracies)
    try:
        training_configuration = [tc for tc in TRAINING_CONFIGURATIONS if tc in checkpoints_folder.name][0].lower()
        option = TRAINING_CONFIGURATION_TO_OPTION[training_configuration]
    except IndexError:
        continue
    if checkpoints_folder.name == "resnette_double":
        training_configuration = "new_resnette"

    scores = []
    for galpha in galpha_list:
        if galpha <= 1e5:
            scores.append(accuracies[galpha][0])
        elif accuracies[galpha][0] < 0.8 * np.mean(scores):
            galpha_cutoff[option] = galpha * (1 + 1e-7)
            break
    else:
        galpha_cutoff[option] = galpha * (1 + 1e-7)

assert len(galpha_cutoff) == len(TRAINING_CONFIGURATIONS)

SALIENCY_FOLDERS = DATA_ROOT / "saliency_evaluation"
DEEPLIFT = "DeepLift"
GRADIENT = "Saliency"
SALIENCY_METHODS = GRADIENT, DEEPLIFT
p_folder = DATA_ROOT / "pdata"
latex_macros = ""
assert len(DISTINGUISHERS) == 1
for metric in USED_METRICS:
    out = []
    for saliency_folder in SALIENCY_FOLDERS.iterdir():
        option = [opt for opt in OPTIONS if opt in saliency_folder.name][0]
        for saliency_method in SALIENCY_METHODS:
            folder = saliency_folder / saliency_method
            data = read_files(folder)[0]
            for distinguisher in DISTINGUISHERS:
                alphas, scores = zip(*sorted(data[distinguisher][metric.name].items()))
                scores = np.array(scores)[np.array(alphas) < galpha_cutoff[option]]
                alphas = np.array(alphas)[np.array(alphas) < galpha_cutoff[option]]
                plt.errorbar(alphas, np.mean(scores, axis=1), sem(scores, axis=1), linestyle='None', marker='o',
                             capsize=3)
                plt.xscale("log")
                plt.close()
                identifier = option + "_" + saliency_method
                identifier = identifier.replace("FMNIST", "Fashion-MNIST")
                identifier = identifier.replace("CIFAR", "CIFAR-10")
                identifier = identifier.replace("Saliency", "gradient").replace("_", " ")
                pearson_result = pearsonr(np.log(alphas), np.mean(scores, axis=1))
                print(f"pearson {identifier}: {pearson_result}, {pearson_result.confidence_interval()}")


                def statistic(x):
                    return spearmanr(x, np.mean(scores, axis=1)).statistic


                correlation = statistic(np.log(alphas))
                ptest = permutation_test((np.log(alphas),), statistic, permutation_type='pairings')
                pvalue = ptest.pvalue
                print(f"spearman: {correlation} +- {pvalue}")
                #out.append((identifier, correlation, pvalue))
                #out.append((identifier.replace("_", " "), "%.4g" % result.correlation, "%.3e" % result.pvalue))
                out_folder = p_folder / metric.name.replace(" ", "_") / saliency_method / option
                out_folder.mkdir(parents=True, exist_ok=True)
                rho_path = out_folder / "rho.txt"
                p_path = out_folder / "p.txt"
                rho_path.write_text("%.3f" % correlation)
                p_path.write_text(f"{pvalue}")
                # latex macro here
                cmd_name = f"{metric.name.replace(' ', '')}{identifier.replace(' ', '')}values".replace("-", "").replace(".", "").replace("10", "").lower()

                latex_macros += f"\\DeclareRobustCommand{{\\{cmd_name}}}{{\n"
                latex_macros += f"  $\\rho = \\input{{{rho_path.relative_to(DATA_ROOT).as_posix()}}}, p = \\input{{{p_path.relative_to(DATA_ROOT).as_posix()}}}$\n"
                latex_macros += "}\n\n"

    #df = pd.DataFrame(data=out, columns=("Training configuration", "correlation", "p-value"))
    #df.to_csv(DATA_ROOT / f"pvalues/{metric.name.replace(' ', '_')}.txt", index=False)

(p_folder / "constants.tex").write_text(latex_macros)