Commit d64001e4 authored by mjboos's avatar mjboos

refactor val

parent 4e96a780
......@@ -346,7 +346,7 @@ def simple_net(trainable=False, prune=True):
return model_params
def shallow_CNN(trainable=False, prune=True):
model_func = partial(models.CNN_shallow, n_filters=96, kernel_sizes=[3,4,5], dropout=0.3, dropout_embed=0.3, act='relu')
model_func = partial(models.CNN_shallow, n_filters=64, kernel_sizes=[3,4,5], dropout=0.5, dropout_embed=0.5, act=None)
model_params = {
'max_features' : 500000, 'model_function' : model_func, 'maxlen' : 500,
'embedding_dim' : 300, 'trainable' : trainable, 'prune' : prune,
......
......@@ -3,6 +3,7 @@
from __future__ import division
import numpy as np
import pandas as pd
from functools import partial
import matplotlib.pyplot as plt
import joblib
import pandas as pd, numpy as np
......@@ -15,6 +16,7 @@ from sklearn.base import BaseEstimator, TransformerMixin
import string
import langid
import preprocessing as pre
import models
bad_word_dict = joblib.load('bad_words_misspellings.pkl')
some_bad_words = joblib.load('some_bad_words.pkl')
......@@ -86,8 +88,16 @@ def contains_bad_word2(row):
match = re.search(bad_word_regex2, row)
return match is not None
def NMF_features():
from sklearn.decomposition import NMF
train_text, _ = pre.load_data()
test_text, _ = pre.load_data('test.csv')
tfidf = models.get_tfidf_model_model()
feature_mapping_dict = {
'count_symbol' : count_symbol,
'count_exclamation' : count_symbol,
'count_question' : partial(count_symbol, symbol='?'),
'bad_word' : contains_bad_word,
# 'bad_word2' : contains_bad_word2,
# 'count_capitals' : count_capitals,
......
......@@ -166,7 +166,7 @@ def make_training_set_preds(model, train_data, train_y, split=0.2):
def dump_trials(trials, fname=''):
import time
joblib.dump(trials, '../validation_logs/trial_{}_{}.json'.format(fname, time.strftime("%m%d-%H%M")))
joblib.dump(trials, '../validation_logs/trial_{}_{}.pkl'.format(fname, time.strftime("%m%d-%H%M")))
def update_embedding_vec(word_dict, path):
other_words = get_fasttext_embedding(path)
......
This diff is collapsed.
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment