{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom sklearn import model_selection\nimport nltk\nfrom nltk.corpus import stopwords\nimport string\nfrom collections import defaultdict\nfrom wordcloud import WordCloud, STOPWORDS\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport seaborn as sns\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.metrics import f1_score\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom scipy.sparse import csr_matrix, hstack\nimport math\nfrom matplotlib import colors\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"train_df = pd.read_csv(\"../input/train.csv\")\ntest_df = pd.read_csv(\"../input/test.csv\")\nsub_df = pd.read_csv('../input/sample_submission.csv')\nprint(\"Train shape : \", train_df.shape)\nprint(\"Test shape : \", test_df.shape)\nprint(\"Submission shape : \", sub_df.shape)\n# train_df.head()\n# test_df.head()\n\n# Copy train_df\ntrain_df_copy = train_df.copy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"# Drop qid\ntrain_df.drop(columns=\"qid\", inplace=True)\n# test_df.drop(columns=\"qid\", inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":false,"trusted":true},"cell_type":"code","source":"def ngram_extractor(text, n_gram):\n    token = [token for token in text.lower().strip(string.punctuation).split(\" \") if token != \"\" if token not in STOPWORDS]\n    ngrams = zip(*[token[i:] for i in range(n_gram)])\n    return [\" \".join(ngram) for ngram in ngrams]\n\n# Function to generate a dataframe with n_gram and top max_row frequencies\ndef generate_ngrams(df, col, n_gram, max_row):\n    temp_dict = defaultdict(int)\n    for question in df[col]:\n        for word in ngram_extractor(question, n_gram):\n            temp_dict[word] += 1\n    temp_df = pd.DataFrame(sorted(temp_dict.items(), key=lambda x: x[1])[::-1]).head(max_row)\n    temp_df.columns = [\"word\", \"wordcount\"]\n    return temp_df\n\ndef comparison_plot(df_1,df_2,col_1,col_2, space, ngram):\n    plt.rcParams.update({'font.size': 14})  \n    fig, ax = plt.subplots(1, 2, figsize=(20,5))\n#     ax[0].set(xscale=\"log\")\n#     ax[1].set(xscale=\"log\")\n    sns.barplot(x=col_2, y=col_1, data=df_1, ax=ax[0], color=colors.CSS4_COLORS.get('lightblue'))\n    sns.barplot(x=col_2, y=col_1, data=df_2, ax=ax[1], color=colors.CSS4_COLORS.get('lightsalmon'))\n\n    ax[0].set_xlabel('Word count', size=14, color=\"black\")\n    ax[0].set_ylabel(ngram.capitalize(), size=14, color=\"black\")\n    ax[0].set_title('Top '+ ngram + ' in sincere questions', size=15)\n\n    ax[1].set_xlabel('Word count', size=14, color=\"black\")\n    ax[1].set_ylabel(ngram.capitalize(), size=14, color=\"black\")\n    ax[1].set_title('Top '+ ngram + ' in insincere questions', size=15)\n\n    #fig.subplots_adjust(wspace=space)\n    fig.tight_layout()\n    \n    plt.show()\n    fig.savefig(ngram + '.pdf', format='pdf')\n\nsincere_1gram = generate_ngrams(train_df[train_df[\"target\"]==0], 'question_text', 1, 20)\ninsincere_1gram = generate_ngrams(train_df[train_df[\"target\"]==1], 'question_text', 1, 20)\n\ncomparison_plot(sincere_1gram,insincere_1gram,'word','wordcount', 0.25, \"unigrams\")\n\nsincere_2gram = generate_ngrams(train_df[train_df[\"target\"]==0], 'question_text', 2, 20)\ninsincere_2gram = generate_ngrams(train_df[train_df[\"target\"]==1], 'question_text', 2, 20)\n\ncomparison_plot(sincere_2gram,insincere_2gram,'word','wordcount', 0.25, \"bigrams\")\n\nsincere_3gram = generate_ngrams(train_df[train_df[\"target\"]==0], 'question_text', 3, 20)\ninsincere_3gram = generate_ngrams(train_df[train_df[\"target\"]==1], 'question_text', 3, 20)\n\ncomparison_plot(sincere_3gram,insincere_3gram,'word','wordcount', 0.25, \"trigrams\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"misspells = [\"ain't\", \"aren't\", \"can't\", \"'cause\", \"could've\", \"couldn't\", \"didn't\", \n    \"doesn't\", \"don't\", \"hadn't\", \"hasn't\", \"haven't\", \"he'd\", \"he'll\", \"he's\", \"how'd\", \n    \"how'd'y\", \"how'll\", \"how's\", \"i'd\", \"i'd've\", \"i'll\", \"i'll've\", \"i'm\", \"i've\", \"isn't\", \n    \"it'd\", \"it'd've\", \"it'll\", \"it'll've\", \"it's\", \"let's\", \"ma'am\", \"mayn't\", \"might've\", \n    \"mightn't\", \"mightn't've\", \"must've\", \"mustn't\", \"mustn't've\", \"needn't\", \"needn't've\", \n    \"o'clock\", \"oughtn't\", \"oughtn't've\", \"shan't\", \"sha'n't\", \"shan't've\", \"she'd\", \"she'd've\", \n    \"she'll\", \"she'll've\", \"she's\", \"should've\", \"shouldn't\", \"shouldn't've\", \"so've\", \"so's\", \n    \"this's\", \"that'd\", \"that'd've\", \"that's\", \"there'd\", \"there'd've\", \"there's\", \"here's\", \n    \"they'd\", \"they'd've\", \"they'll\", \"they'll've\", \"they're\", \"they've\", \"to've\", \"wasn't\", \n    \"we'd\", \"we'd've\", \"we'll\", \"we'll've\", \"we're\", \"we've\", \"weren't\", \"what'll\", \"what'll've\", \n    \"what're\", \"what's\", \"what've\", \"when's\", \"when've\", \"where'd\", \"where's\", \"where've\", \"who'll\", \n    \"who'll've\", \"who's\", \"who've\", \"why's\", \"why've\", \"will've\", \"won't\", \"won't've\", \"would've\", \n    \"wouldn't\", \"wouldn't've\", \"y'all\", \"y'all'd\", \"y'all'd've\", \"y'all're\", \"y'all've\", \"you'd\", \n    \"you'd've\", \"you'll\", \"you'll've\", \"you're\", \"you've\", \"colour\", \"centre\", \"favourite\", \n    \"travelling\", \"counselling\", \"theatre\", \"cancelled\", \"labour\", \"organisation\", \"wwii\", \n    \"citicise\", \"youtu \", \"qoura\", \"sallary\", \"whta\", \"narcisist\", \"howdo\", \"whatare\", \"howcan\", \n    \"howmuch\", \"howmany\", \"whydo\", \"doi\", \"thebest\", \"howdoes\", \"mastrubation\", \"mastrubate\", \n    \"mastrubating\", \"pennis\", \"etherium\", \"narcissit\", \"bigdata\", \"2k17\", \"2k18\", \"qouta\", \n    \"exboyfriend\", \"airhostess\", \"whst\", \"watsapp\", \"demonitisation\", \"demonitization\", \"demonetisation\",\n    'demonitization']\n\n# List Of Bad Words by Google-Profanity Words \nbad_words = ['cockknocker', 'n1gger', 'ing', 'fukker', 'nympho', 'fcuking', 'gook', 'freex', \n             'arschloch', 'fistfucked', 'chinc', 'raunch', 'fellatio', 'splooge',\n             'nutsack', 'lmfao', 'wigger', 'bastard', 'asses', 'fistfuckings', 'blue', 'waffle', \n             'beeyotch', 'pissin', 'dominatrix', 'fisting', 'vullva', 'paki', 'cyberfucker', 'chuj',\n             'penuus', 'masturbate', 'b00b*', 'fuks', 'sucked', 'fuckingshitmotherfucker', 'feces', 'panty', \n             'coital', 'wh00r.', 'whore', 'condom', 'hells', 'foreskin', 'wanker', 'hoer', 'sh1tz', 'shittings', \n             'wtf', 'recktum', 'dick*', 'pr0n', 'pasty', 'spik', 'phukked', 'assfuck', 'xxx', 'nigger*', 'ugly',\n             's_h_i_t', 'mamhoon', 'pornos', 'masterbates', 'mothafucks', 'Mother', 'Fukkah', 'chink', 'pussy', \n             'palace', 'azazel', 'fistfucking', 'ass-fucker', 'shag', 'chincs', 'duche', 'orgies', 'vag1na', 'molest', \n             'bollock', 'a-hole', 'seduce', 'Cock*', 'dog-fucker', 'shitz', 'Mother', 'Fucker', 'penial', 'biatch',\n             'junky', 'orifice', '5hit', 'kunilingus', 'cuntbag', 'hump', 'butt', 'fuck', 'titwank', 'schaffer', \n             'cracker', 'f.u.c.k', 'breasts', 'd1ld0', 'polac', 'boobs', 'ritard', 'fuckup', 'rape', 'hard', 'on', \n             'skanks', 'coksucka', 'cl1t', 'herpy', 's.o.b.', 'Motha', 'Fucker', 'penus', 'Fukker', 'p.u.s.s.y.', \n             'faggitt', 'b!tch', 'doosh', 'titty', 'pr1k', 'r-tard', 'gigolo', 'perse', 'lezzies', 'bollock*', \n             'pedophiliac', 'Ass', 'Monkey', 'mothafucker', 'amcik', 'b*tch', 'beaner', 'masterbat*', 'fucka', \n             'phuk', 'menses', 'pedophile', 'climax', 'cocksucking', 'fingerfucked', 'asswhole', 'basterdz',\n             'cahone', 'ahole', 'dickflipper', 'diligaf', 'Lesbian', 'sperm', 'pisser', 'dykes', 'Skanky',\n             'puuker', 'gtfo', 'orgasim', 'd0ng', 'testicle*', 'pen1s', 'piss-off', '@$$', 'fuck', 'trophy', \n             'arse*', 'fag', 'organ', 'potty', 'queerz', 'fannybandit', 'muthafuckaz', 'booger', 'pussypounder',\n             'titt', 'fuckoff', 'bootee', 'schlong', 'spunk', 'rumprammer', 'weed', 'bi7ch', 'pusse', 'blow', 'job', \n             'kusi*', 'assbanged', 'dumbass', 'kunts', 'chraa', 'cock', 'sucker', 'l3i+ch', 'cabron', 'arrse', 'cnut', \n             'murdep', 'fcuk', 'phuked', 'gang-bang', 'kuksuger', 'mothafuckers', 'ghey', 'clit', 'licker', 'feg', \n             'ma5terbate', 'd0uche', 'pcp', 'ejaculate', 'nigur', 'clits', 'd0uch3', 'b00bs', 'fucked', 'assbang', \n             'mutha', 'goddamned', 'cazzo', 'lmao', 'godamn', 'kill', 'coon', 'penis-breath', 'kyke', 'heshe', 'homo',\n             'tawdry', 'pissing', 'cumshot', 'motherfucker', 'menstruation', 'n1gr', 'rectus', 'oral', 'twats', \n             'scrot', 'God', 'damn', 'jerk', 'nigga', 'motherfuckin', 'kawk', 'homey', 'hooters', 'rump', \n             'dickheads', 'scrud', 'fist', 'fuck', 'carpet', 'muncher', 'cipa', 'cocaine', 'fanyy', 'frigga', \n             'massa', '5h1t', 'brassiere', 'inbred', 'spooge', 'shitface', 'tush', 'Fuken', 'boiolas', 'fuckass', 'wop*',\n             'cuntlick', 'fucker', 'bodily', 'bullshits', 'hom0', 'sumofabiatch', 'jackass', 'dilld0', 'puuke', 'cums', \n             'pakie', 'cock-sucker', 'pubic', 'pron', 'puta', 'penas', 'weiner', 'vaj1na', 'mthrfucker', 'souse', 'loin',\n             'clitoris', 'f.ck', 'dickface', 'rectal', 'whored', 'bookie', 'chota', 'bags', 'sh!t', 'pornography', 'spick', 'seamen',\n             'Phukker', 'beef', 'curtain', 'eat', 'hair', 'pie', 'mother', 'fucker', 'faigt', 'yeasty', 'Clit', 'kraut', 'CockSucker', \n             'Ekrem*', 'screwing', 'scrote', 'fubar', 'knob', 'end', 'sleazy', 'dickwhipper', 'ass', 'fuck', 'fellate', 'lesbos', \n             'nobjokey', 'dogging', 'fuck', 'hole', 'hymen', 'damn', 'dego', 'sphencter', 'queef*', 'gaylord', 'va1jina', 'a55', \n             'fuck', 'douchebag', 'blowjob', 'mibun', 'fucking', 'dago', 'heroin', 'tw4t', 'raper', 'muff', 'fitt*', 'wetback*',\n             'mo-fo', 'fuk*', 'klootzak', 'sux', 'damnit', 'pimmel', 'assh0lez', 'cntz', 'fux', 'gonads', 'bullshit', 'nigg3r', \n             'fack', 'weewee', 'shi+', 'shithead', 'pecker', 'Shytty', 'wh0re', 'a2m', 'kkk', 'penetration', 'kike', 'naked', \n             'kooch', 'ejaculation', 'bang', 'hoare', 'jap', 'foad', 'queef', 'buttwipe', 'Shity', 'dildo', 'dickripper', \n             'crackwhore', 'beaver', 'kum', 'sh!+', 'qweers', 'cocksuka', 'sexy', 'masterbating', 'peeenus', 'gays', \n             'cocksucks', 'b17ch', 'nad', 'j3rk0ff', 'fannyflaps',\n             'God-damned', 'masterbate', 'erotic', 'sadism', 'turd', 'flipping', 'the', 'bird',\n             'schizo', 'whiz', 'fagg1t', 'cop', 'some', \n             'wood', 'banger', 'Shyty', 'f', 'you', 'scag', 'soused', 'scank',\n             'clitorus', 'kumming', 'quim', 'penis', 'bestial', 'bimbo', 'gfy',\n             'spiks', 'shitings', 'phuking', 'paddy', 'mulkku', 'anal', \n             'leakage', 'bestiality', 'smegma', 'bull', 'shit', 'pillu*', 'schmuck',\n             'cuntsicle', 'fistfucker', 'shitdick', 'dirsa', 'm0f0']\n\nstopwords = set(stopwords.words('english'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def get_features(dataframe):\n    dataframe[\"text_size\"] = dataframe[\"question_text\"].apply(len).astype('uint16')\n    dataframe[\"capital_size\"] = dataframe[\"question_text\"].apply(lambda x: sum(1 for c in x if c.isupper())).astype('uint16')\n    dataframe[\"capital_rate\"] = dataframe.apply(lambda x: float(x[\"capital_size\"]) / float(x[\"text_size\"]), axis=1).astype('float16')\n    dataframe[\"exc_count\"] = dataframe[\"question_text\"].apply(lambda x: x.count(\"!\")).astype('uint16')\n    dataframe[\"question_count\"] = dataframe[\"question_text\"].apply(lambda x: x.count(\"?\")).astype('uint16')\n    dataframe[\"unq_punctuation_count\"] = dataframe[\"question_text\"].apply(lambda x: sum(x.count(p) for p in '∞θ÷α•à−β∅³π‘₹´°£€\\×™√²')).astype('uint16')\n    dataframe[\"symbol_count\"] = dataframe[\"question_text\"].apply(lambda x: sum(x.count(p) for p in '*&$%')).astype('uint16')\n    dataframe[\"words_count\"] = dataframe[\"question_text\"].apply(lambda x: len(x.split())).astype('uint16')\n    dataframe[\"unique_words\"] = dataframe[\"question_text\"].apply(lambda x: (len(set(x.split())))).astype('uint16')\n    dataframe[\"unique_rate\"] = dataframe[\"unique_words\"] / dataframe[\"words_count\"]\n    dataframe[\"word_max_length\"] = dataframe[\"question_text\"].apply(lambda x: max([len(word) for word in x.split()]) ).astype('uint16')  # useless\n    dataframe[\"mistake_count\"] = dataframe[\"question_text\"].apply(lambda x: sum(x.lower().count(w) for w in misspells)).astype('uint16')\n    dataframe['num_stopwords'] = dataframe[\"question_text\"].apply(lambda x: len([w for w in str(x).lower().split() if w in stopwords])).astype('uint16')\n    dataframe['num_punctuation_chars'] = dataframe[\"question_text\"].apply(lambda x: len([c for c in str(x) if c in string.punctuation])).astype('uint16')\n    dataframe['num_word_tokens_Title'] = dataframe[\"question_text\"].apply(lambda x: len([w for w in str(x).split() if w.istitle()])).astype('uint16')\n    dataframe['avg_word_token_length'] = dataframe[\"question_text\"].apply(lambda x: np.mean([len(w) for w in str(x).split()])).astype('uint16')\n    dataframe['num_smilies'] = dataframe['question_text'].apply(lambda x: sum(x.count(w) for w in (':-)', ':)', ';-)', ';)')))\n    dataframe['num_sad'] = dataframe['question_text'].apply(lambda x: sum(x.count(w) for w in (':-<', ':()', ';-()', ';(')))\n    dataframe[\"badwordcount\"] = dataframe['question_text'].apply(lambda comment: sum(comment.count(w) for w in bad_words))\n    dataframe['num_chars'] =    dataframe['question_text'].apply(len)\n    dataframe[\"normchar_badwords\"] = dataframe[\"badwordcount\"]/dataframe['num_chars']\n    dataframe[\"normword_badwords\"] = dataframe[\"badwordcount\"]/dataframe['text_size']\n    return dataframe","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"# # Get features\n# get_features(train_df)\n\n# # Split training data into train and validaiton sets (70/20/10)\n# train_x, temp_x, train_y, temp_y = model_selection.train_test_split(train_df.drop(columns='target'), train_df['target'], test_size=0.3)\n# valid_x, test_x, valid_y, test_y = model_selection.train_test_split(temp_x, temp_y, test_size=1/3)\n# print(\"No samples in training set: {}\".format(len(train_x)))\n# print(\"No samples in validation set: {}\".format(len(valid_x)))\n# print(\"No samples in test set: {}\".format(len(test_x)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"# Simple logistic regression model fitting/training\n# model = LogisticRegression(solver='lbfgs', dual=False, class_weight='balanced', C=0.5, max_iter=10000)\n# model.fit(train_x.drop(columns='question_text'), train_y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"# # Logistic regression prediction\n# # Probability output\n# model.predict_proba(valid_x.drop(columns='question_text'))\n# # Model score\n# model.score(valid_x.drop(columns='question_text'), valid_y)\n# # Class output\n# pred_valid_y = model.predict(valid_x.drop(columns='question_text'))\n# pred_test_y = model.predict(test_x.drop(columns='question_text'))\n# print(\"F1-score (Valid): {}\".format(f1_score(valid_y, pred_valid_y)))\n# print(\"F1-score (Test): {}\".format(f1_score(test_y, pred_test_y)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"# Submission\n# get_features(test_df)\n# pred_sub_test_y = model.predict(test_df.drop(columns='question_text'))\n# sub_df.prediction = pred_sub_test_y\n# sub_df.to_csv(\"submission.csv\", index=False)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#TFIDF Vectorizer\n# TODO Try with different tokenizer\n# tfidf_vectorizer = TfidfVectorizer(\n#         #ngram_range=(1,2),\n#         min_df=3,\n#         max_df=0.9,\n#         strip_accents='unicode',\n#         use_idf=True,\n#         smooth_idf=True,\n#         sublinear_tf=True,\n#         max_features=9000\n#     ).fit(pd.concat([train_df['question_text'], test_df['question_text']]))\n\ntfidf_vectorizer = TfidfVectorizer(\n    ngram_range=(1,1),\n    max_features=9000,\n    sublinear_tf=True, \n    strip_accents='unicode', \n    analyzer='word', \n    token_pattern=\"\\w{1,}\", \n    stop_words=\"english\",\n    max_df=0.95,\n    min_df=2\n).fit(pd.concat([train_df['question_text'], test_df['question_text']]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Get features\nget_features(train_df)\n\n# Split training data into train and validaiton sets (70/20/10)\ntrain_x, temp_x, train_y, temp_y = model_selection.train_test_split(train_df.drop(columns='target'), train_df['target'], test_size=0.3)\nvalid_x, test_x, valid_y, test_y = model_selection.train_test_split(temp_x, temp_y, test_size=1/3)\nprint(\"No samples in training set, shape, type: {}, {}, {}\".format(len(train_x), train_x.shape, type(train_x)))\nprint(\"No samples in validation set, shape, type: {}, {}, {}\".format(len(valid_x), valid_x.shape, type(valid_x)))\nprint(\"No samples in test set, shape, type: {}, {}, {}\".format(len(test_x), test_x.shape, type(test_x)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Transform data\ntrain_x_tfidf = tfidf_vectorizer.transform(train_x['question_text'].fillna(\"na_\").values).tocsr()\nvalid_x_tfidf = tfidf_vectorizer.transform(valid_x['question_text'].fillna(\"na_\").values).tocsr()\ntest_x_tfidf = tfidf_vectorizer.transform(test_x['question_text'].fillna(\"na_\").values).tocsr()\n\n# Stack features\ntrain_x_tfidf_f = hstack([csr_matrix(train_x.drop(columns='question_text')), train_x_tfidf])\nvalid_x_tfidf_f = hstack([csr_matrix(valid_x.drop(columns='question_text')), valid_x_tfidf])\ntest_x_tfidf_f = hstack([csr_matrix(test_x.drop(columns='question_text')), test_x_tfidf])\nprint(type(train_x), type(train_x_tfidf), type(train_x_tfidf_f))\nprint(train_x.shape, train_x_tfidf.shape, train_x_tfidf_f.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Fit model on transformed data\nmodel = LogisticRegression(solver='lbfgs', dual=False, class_weight='balanced', C=0.5, max_iter=10000)\nmodel.fit(train_x_tfidf_f, train_y)\n#model.fit(train_x_tfidf, train_y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Prediction\n# pred_valid_y = model.predict(valid_x_tfidf)\n# pred_test_y = model.predict(test_x_tfidf)\npred_valid_y = model.predict_proba(valid_x_tfidf_f)\npred_test_y = model.predict_proba(test_x_tfidf_f)\n# print(\"F1-score (Valid): {}\".format(f1_score(valid_y, pred_valid_y)))\n# print(\"F1-score (Test): {}\".format(f1_score(test_y, pred_test_y)))\n\ndef threshold_search(y_true, y_proba):\n    best_threshold = 0\n    best_score = 0\n    for threshold in [i * 0.01 for i in range(100)]:\n        score = f1_score(y_true=y_true, y_pred=y_proba > threshold)\n        if score > best_score:\n            best_threshold = threshold\n            best_score = score\n    search_result = {'threshold': best_threshold, 'f1': best_score}\n    return search_result\n\nt_result_0 = threshold_search(valid_y, pred_valid_y[:,0])\nt_result_1 = threshold_search(valid_y, pred_valid_y[:,1])\n                 \n# Find optimal threshold\n#print(\"Threshold (0) and F1-score: {} {}\".format(t_result_0['threshold'], t_result_0['f1']))\nprint(\"Threshold (1) and F1-score: {} {}\".format(t_result_1['threshold'], t_result_1['f1']))\n\npred_test_y_thresh = np.zeros(pred_test_y.shape[0])\npred_test_y_thresh[pred_test_y[:,1] > t_result_1['threshold']] = 1\n\nprint(\"F1-score (Test): {}\".format(f1_score(test_y, pred_test_y_thresh)))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}