-
Notifications
You must be signed in to change notification settings - Fork 14
/
Copy pathvisualize_switch_weights.py
120 lines (87 loc) · 3.35 KB
/
visualize_switch_weights.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
import os
from collections import OrderedDict
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
from keras.layers import Layer, Input, Dense, Activation, Conv2D, GlobalAveragePooling2D
from keras.models import Model
from keras.datasets import cifar10
from keras import backend as K
from switchnorm import SwitchNormalization
nb_classes = 10
img_rows, img_cols = 32, 32
img_channels = 3
img_dim = (img_channels, img_rows, img_cols) if K.image_dim_ordering() == "th" else (img_rows, img_cols, img_channels)
bottleneck = False
width = 1
weight_decay = 1e-4
ip = Input(shape=img_dim)
x = Conv2D(64, (3, 3), padding='same', kernel_initializer='he_normal')(ip)
x = SwitchNormalization(axis=-1, momentum=0.98)(x)
x = Activation('relu')(x)
x = Conv2D(64, (3, 3), padding='same', kernel_initializer='he_normal')(x)
x = SwitchNormalization(axis=-1, momentum=0.98)(x)
x = Activation('relu')(x)
x = Conv2D(128, (3, 3), padding='same', kernel_initializer='he_normal', strides=(2, 2))(x)
x = SwitchNormalization(axis=-1, momentum=0.98)(x)
x = Activation('relu')(x)
x = Conv2D(128, (3, 3), padding='same', kernel_initializer='he_normal')(x)
x = SwitchNormalization(axis=-1, momentum=0.98)(x)
x = Activation('relu')(x)
x = Conv2D(256, (3, 3), padding='same', kernel_initializer='he_normal', strides=(2, 2))(x)
x = SwitchNormalization(axis=-1, momentum=0.98)(x)
x = Activation('relu')(x)
x = Conv2D(256, (3, 3), padding='same', kernel_initializer='he_normal')(x)
x = SwitchNormalization(axis=-1, momentum=0.98)(x)
x = Activation('relu')(x)
x = GlobalAveragePooling2D()(x)
x = Dense(nb_classes, activation='softmax')(x)
model = Model(ip, x)
print("Model created")
print("Model created")
model.summary()
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=["accuracy"])
print("Finished compiling")
print("Building model...")
(trainX, trainY), (testX, testY) = cifar10.load_data()
trainX = trainX.astype('float32')
testX = testX.astype('float32')
trainX = (trainX / 127.5) - 1.0
testX = (testX / 127.5) - 1.0
# Load model
weights_file = "weights/Baseline-CIFAR10.h5"
if os.path.exists(weights_file):
model.load_weights(weights_file, by_name=True)
print("Model loaded.")
else:
print("Please train a CIFAR model first !")
exit()
layer_dict = OrderedDict([(i, layer.name)
for i, layer in enumerate(model.layers)
if layer.__class__.__name__ == 'SwitchNormalization'])
def softmax(x):
e_x = np.exp(x - np.max(x))
return e_x / e_x.sum(axis=0)
fig, ax = plt.subplots(len(layer_dict), 2, squeeze=False, sharex=True, sharey=True)
fig.subplots_adjust(hspace=.5)
for i, (id, name) in enumerate(layer_dict.items()):
print("Found %s at index %d" % (name, id))
layer = model.layers[id] # type: Layer
weights = layer.get_weights()
mean_weights = softmax(weights[2])
variance_weights = softmax(weights[3])
print("mean", mean_weights)
print("variance", variance_weights)
print()
plt.sca(ax[i, 0])
plt.barh(list(range(3)), mean_weights)
plt.yticks(list(range(3)), ['instance', 'layer', 'batch'])
plt.xlim([0.0, 1.0])
plt.title(name + ' - mean')
plt.sca(ax[i, 1])
plt.barh(list(range(3)), variance_weights)
plt.yticks(list(range(3)), ['instance', 'layer', 'batch'])
plt.xlim([0.0, 1.0])
plt.title(name + ' - var')
plt.show()