Commit b021739e authored by mjboos's avatar mjboos

validation

parent 483ae041
......@@ -238,12 +238,21 @@ def simple_attention_1d(trainable=False, prune=True):
'compilation_args' : {'optimizer_func' : optimizers.Adam, 'optimizer_args' : {'lr' : 0.0005, 'clipnorm' : 1.}, 'loss':{'main_output': 'binary_crossentropy'}, 'loss_weights' : [1.]}}
return model_params
def one_gru_attention(trainable=False, prune=True):
model_func = partial(models.RNN_diff_attention, rnn_func=keras.layers.CuDNNGRU, no_rnn_layers=2, hidden_rnn=96, dropout_dense=0.5, dropout=0.5, train_embedding=False)
model_params = {
'max_features' : 500000, 'model_function' : model_func, 'maxlen' : 500,
'embedding_dim' : 300, 'trainable' : trainable, 'prune' : prune,
'compilation_args' : {'optimizer_func' : optimizers.Adam, 'optimizer_args' : {'lr' : 0.0005, 'clipnorm' : 1.}, 'loss':{'main_output': 'binary_crossentropy'}, 'loss_weights' : [1.]}}
return model_params
def conc_attention(trainable=False, prune=True):
model_func = partial(models.RNN_diff_attention, rnn_func=keras.layers.CuDNNGRU, no_rnn_layers=2, hidden_rnn=96, dropout_dense=0.5, dropout=0.5, train_embedding=False)
model_params = {
'max_features' : 500000, 'model_function' : model_func, 'maxlen' : 500,
'embedding_dim' : 300, 'trainable' : trainable, 'prune' : prune,
'compilation_args' : {'optimizer_func' : optimizers.Adam, 'optimizer_args' : {'lr' : 0.001, 'clipnorm' : 1.}, 'loss':{'main_output': 'binary_crossentropy'}, 'loss_weights' : [1.]}}
'compilation_args' : {'optimizer_func' : optimizers.Adam, 'optimizer_args' : {'lr' : 0.0005, 'clipnorm' : 1.}, 'loss':{'main_output': 'binary_crossentropy'}, 'loss_weights' : [1.]}}
return model_params
def simple_attention(trainable=False, prune=True):
......@@ -270,6 +279,15 @@ def simple_attention_word_dropout(trainable=False, prune=True):
model_params['model_function'] = partial(models.RNN_time_dropout_attention, rnn_func=keras.layers.CuDNNGRU, no_rnn_layers=2, hidden_rnn=96, dropout_dense=0.5, dropout=0.5, train_embedding=False)
return model_params
def simple_huge_net(trainable=False, prune=True):
model_func = partial(models.RNN_conc, rnn_func=keras.layers.CuDNNGRU, no_rnn_layers=2, hidden_rnn=96)
model_params = {
'max_features' : 500000, 'model_function' : model_func, 'maxlen' : 500,
'embedding_dim' : 300, 'trainable' : trainable, 'prune' : prune,
'compilation_args' : {'optimizer_func' : optimizers.Adam, 'optimizer_args' : {'lr' : 0.0005, 'clipnorm' : 1.}, 'loss':{'main_output': 'binary_crossentropy'}, 'loss_weights' : [1.]}}
return model_params
def simple_net(trainable=False, prune=True):
model_func = partial(models.RNN_general, rnn_func=keras.layers.CuDNNGRU, no_rnn_layers=2, hidden_rnn=96, hidden_dense=128)
model_params = {
......
......@@ -584,6 +584,31 @@ def RNN_attention_1d(x, no_rnn_layers=2, hidden_rnn=48, hidden_dense=48, rnn_fun
x = Dense(1, activation="sigmoid", name='main_output')(x)
return x, None
def RNN_one_gru_attention(x, no_rnn_layers=2, hidden_rnn=48, hidden_dense=48, rnn_func=None, dropout_embed=0.2, dropout=0.5, dropout_dense=0.5, input_len=500, train_embedding=False):
if rnn_func is None:
rnn_func = CuDNNLSTM
if not isinstance(hidden_rnn, list):
hidden_rnn = [hidden_rnn] * no_rnn_layers
if len(hidden_rnn) != no_rnn_layers:
raise ValueError('list of recurrent units needs to be equal to no_rnn_layers')
if train_embedding:
vals = [x]
else:
vals = []
for rnn_size in hidden_rnn:
x = Dropout(dropout)(x)
x = Bidirectional(rnn_func(int(rnn_size), return_sequences=True))(x)
att = AttentionWeightedAverage(name='attlayer')(x)
x = concatenate([att, GlobalMaxPool1D()(x), Lambda(lambda x : x[:,-1, :])(x)])
# x = Dropout(dropout)(x)
# x = BatchNormalization(x)
# x = Dense(int(hidden_dense), activation='relu')(x)
x = Dropout(dropout_dense)(x)
x = Dense(6, activation="sigmoid", name='main_output')(x)
return x, None
def RNN_diff_attention(x, no_rnn_layers=2, hidden_rnn=48, hidden_dense=48, rnn_func=None, dropout_embed=0.2, dropout=0.5, dropout_dense=0.5, input_len=500, train_embedding=False):
if rnn_func is None:
rnn_func = CuDNNLSTM
......@@ -802,10 +827,12 @@ def RNN_conc(x, no_rnn_layers=2, hidden_rnn=48, hidden_dense=48, rnn_func=None,
hidden_rnn = [hidden_rnn] * no_rnn_layers
if len(hidden_rnn) != no_rnn_layers:
raise ValueError('list of recurrent units needs to be equal to no_rnn_layers')
vals = []
for rnn_size in hidden_rnn:
x = Dropout(dropout)(x)
x = Bidirectional(rnn_func(int(rnn_size), return_sequences=True))(x)
x = concatenate([GlobalAveragePooling1D()(x), GlobalMaxPool1D()(x), Lambda(lambda x : x[:,-1, :])(x)])
vals.append(x)
x = concatenate([GlobalAveragePooling1D()(x)] + [GlobalMaxPool1D()(val) for val in vals] + [Lambda(lambda x : x[:,-1, :])(val) for val in vals])
x = Dropout(dropout)(x)
# x = BatchNormalization(x)
# x = Dense(int(hidden_dense), activation='relu')(x)
......
......@@ -138,13 +138,13 @@ def do_hyperparameter_search():
joblib.dump(best, 'best_{}.pkl'.format(model_name))
def test_models():
fit_args = {'batch_size' : 80, 'epochs' : 10,
fit_args = {'batch_size' : 80, 'epochs' : 20,
'validation_split' : 0.2}
fixed_args = DNN.simple_attention()
fixed_args = DNN.conc_attention()
kwargs = {}
train_text, train_y = pre.load_data()
test_text, _ = pre.load_data('test.csv')
fixed_args['compilation_args']['optimizer_args'] = {'clipnorm' : 1., 'lr' : 0.001}
fixed_args['compilation_args']['optimizer_args'] = {'clipnorm' : 1., 'lr' : 0.0005}
fixed_args['compilation_args']['optimizer_func'] = optimizers.Adam
frozen_tokenizer = pre.KerasPaddingTokenizer(max_features=fixed_args['max_features'], maxlen=fixed_args['maxlen'])
frozen_tokenizer.fit(pd.concat([train_text, test_text]))
......@@ -152,6 +152,12 @@ def test_models():
kwargs['embedding'] = embedding
kwargs['tokenizer'] = frozen_tokenizer
DNN_model_validate(train_text, train_y, fit_args, fixed_args, kwargs, cv=6)
fixed_args = DNN.one_gru_attention()
DNN_model_validate(train_text, train_y, fit_args, fixed_args, kwargs, cv=6)
fixed_args = DNN.conc_attention()
DNN_model_validate(train_text, train_y, fit_args, fixed_args, kwargs, cv=6)
fixed_args = DNN.simple_huge_attention()
DNN_model_validate(train_text, train_y, fit_args, fixed_args, kwargs, cv=6)
if __name__=='__main__':
test_models()
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment