{"cells":[{"metadata":{"_uuid":"8cfd52f7253edfbb02eb5399e088f70e4e3399bf"},"cell_type":"markdown","source":"**This is the first time I wrote the Pulibc kernel. I hope everyone can help me upvote. Thx**"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cfd5e8452f6c1496b23b8ce4d21105607d6efa8a"},"cell_type":"code","source":"#import required packages\n#basics\nimport pandas as pd \nimport numpy as np\n\n#misc\nimport gc\nimport time\nimport warnings\n\n#stats\nfrom scipy.misc import imread\nfrom scipy import sparse\nimport scipy.stats as ss\n\n#viz\nimport matplotlib.pyplot as plt\nimport matplotlib.gridspec as gridspec \nimport seaborn as sns\nfrom wordcloud import WordCloud ,STOPWORDS\ncolor = sns.color_palette()\n\n%matplotlib inline\n\nfrom plotly import tools\nimport plotly.offline as py\npy.init_notebook_mode(connected=True)\nimport plotly.graph_objs as go\n\n\n#nlp\nimport string\nimport re    #for regex\nimport nltk\nfrom nltk.corpus import stopwords\nimport spacy\nfrom nltk import pos_tag\nfrom nltk.stem.wordnet import WordNetLemmatizer \nfrom nltk.tokenize import word_tokenize\n# Tweet tokenizer does not split at apostophes which is what we want\nfrom nltk.tokenize import TweetTokenizer   \n\n\n#FeatureEngineering\nfrom sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer, HashingVectorizer\nfrom sklearn.decomposition import TruncatedSVD\nfrom sklearn.base import BaseEstimator, ClassifierMixin\nfrom sklearn.utils.validation import check_X_y, check_is_fitted\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn import metrics\nfrom sklearn.metrics import log_loss\nfrom sklearn.model_selection import StratifiedKFold, KFold\nfrom sklearn.metrics import f1_score\n\n# model\nimport lightgbm as lgb\n\neng_stopwords = set(stopwords.words(\"english\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d8c4dfc43f8986089c0ace57903ad1e5b1a85da6"},"cell_type":"code","source":"train_df = pd.read_csv(\"../input/train.csv\")\ntest_df = pd.read_csv(\"../input/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c83ddcb45f7381c820602113d246efbf5daeaaea"},"cell_type":"code","source":"print(\"The shape of train data frame: %s\" % str(train_df.shape))\nprint(\"The shape of test data frame: %s\" % str(test_df.shape))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c17c3d56a9869a89d8863f0b4aef1b7bdf40ffed"},"cell_type":"code","source":"print(\"The main columns: %s\" % str(train_df.columns.values))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d64c5d22575dc4f582730a1f76f0a6f6389d3929"},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"35129b1d2cfbc6b7fdea242eb8b14d21ab7fffa8"},"cell_type":"markdown","source":"## check the number of positive and negative"},{"metadata":{"trusted":true,"_uuid":"ac6dbf78a86c99a7b0e808987d2e3d147a4d83cf"},"cell_type":"code","source":"pos_num = train_df[train_df.target == 1].shape[0]\nneg_num = train_df[train_df.target == 0].shape[0]\nprint(\"The number of positive %d, and the number of negative %d\" % (pos_num, neg_num))\nprint(\"The rate of postive %.5f\" % (pos_num / train_df.shape[0]))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7658ac183525eb0fe144df8399b14ae9a014b16a"},"cell_type":"markdown","source":"we can see the rate of postive sample is 6.1%, just little postive samples. So if we use deep learning to predict the test samples, we should pay attention to overfit to negative sample."},{"metadata":{"_kg_hide-input":true,"trusted":true,"_uuid":"44cc9e620b93685bb44fb276058f36b6a7d0e47e"},"cell_type":"code","source":"train_df[train_df.target == 1].question_text.values[:10].tolist()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e94e42f07f79a820b8622a0ef35b27b8445343b3"},"cell_type":"markdown","source":"we can find the topic of positive samples are sensitive"},{"metadata":{"_uuid":"06b7d638ab4cc13dab0443a801aae8ecb366516d"},"cell_type":"markdown","source":"## analysis features"},{"metadata":{"trusted":true,"_uuid":"f2475a11d4ceda159533bb57ca084e46ad1c5464"},"cell_type":"code","source":"def generate_indirect_features(df):\n    df['count_word'] = df.question_text.apply(lambda x: len(str(x).split()))\n    df['count_unique_word']=df.question_text.apply(lambda x: len(set(str(x).split())))\n    df['count_letters']=df.question_text.apply(lambda x: len(str(x)))\n    df[\"count_punctuations\"] =df.question_text.apply(lambda x: len([c for c in str(x) if c in string.punctuation]))\n    df[\"count_words_upper\"] = df.question_text.apply(lambda x: len([w for w in str(x).split() if w.isupper()]))\n    df[\"count_words_title\"] = df.question_text.apply(lambda x: len([w for w in str(x).split() if w.istitle()]))\n    df[\"count_stopwords\"] = df.question_text.apply(lambda x: len([w for w in str(x).lower().split() if w in eng_stopwords]))\n    df[\"mean_word_len\"] = df.question_text.apply(lambda x: np.mean([len(w) for w in str(x).split()]))\n    df['word_unique_percent']=df['count_unique_word']*100/df['count_word']\n    df['punct_percent']=df['count_punctuations']*100/df['count_word']\n    return df\n\ntrain_df = generate_indirect_features(train_df)\ntest_df = generate_indirect_features(test_df)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b599014c0ba695c27f81a771dfc9f0e5e84974dc"},"cell_type":"markdown","source":"### 1. violin chart"},{"metadata":{"trusted":true,"_uuid":"d7e2b00a058998176813306ad8ea49107ce5e2d9"},"cell_type":"code","source":"def violin_chart(df, column_name, min_clip=None, max_clip=None, title=None):\n    title = column_name if title is None else title\n    plt.figure(figsize=(12, 6))\n    plt.title(title, fontsize=15)\n    sub_df = df[[column_name, 'target']]\n    min_clip = np.min(sub_df[column_name]) if min_clip is None else min_clip\n    max_clip = np.max(sub_df[column_name]) if max_clip is None else max_clip\n    sub_df[column_name] = np.clip(df[column_name].values, min_clip, max_clip)\n    sns.violinplot(y=column_name, x='target', data=sub_df, split=True, innert=\"quart\")\n    plt.xlabel(\"Is Isincere?\", fontsize=12)\n    plt.ylabel(column_name, fontsize=12)\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"469fdf7529726d3446f04faa3917d5bb7cd03bba"},"cell_type":"code","source":"for column_name in train_df.columns.values:\n    if column_name not in ['qid', 'question_text', 'target']:\n        print(column_name, end=\", \")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"76ef1dca704ed8e84e254ccb2abdc76b9ed52941","_kg_hide-input":true},"cell_type":"code","source":"violin_chart(train_df, \"count_word\")\nviolin_chart(train_df, \"count_unique_word\")\nviolin_chart(train_df, \"count_letters\")\nviolin_chart(train_df, \"count_punctuations\", max_clip=20)\nviolin_chart(train_df, \"count_words_upper\", max_clip=25)\nviolin_chart(train_df, \"count_words_title\")\nviolin_chart(train_df, \"count_stopwords\")\nviolin_chart(train_df, \"mean_word_len\")\nviolin_chart(train_df, \"word_unique_percent\")\nviolin_chart(train_df, \"punct_percent\", max_clip=70)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cf0986b157e65dfdac6640e171dd289165e4bb7f"},"cell_type":"markdown","source":"we can find that \n1. The words used in most texts are unique.\n2. The number of uppercase words in a positive sample is relatively small\n\n### 2. TF-IDF analysis"},{"metadata":{"trusted":true,"_uuid":"bceb5db0dcb510854816b2b70d2451e08c77d2db"},"cell_type":"code","source":"tf_idf_vector = TfidfVectorizer(strip_accents='unicode', analyzer='word',ngram_range=(1,1),\n                                use_idf=1, smooth_idf=1, sublinear_tf=1, stop_words = 'english')\ntrain_vect = tf_idf_vector.fit_transform(train_df.question_text.values)\nprint(\"The shape of TF-IDF train matrix: %s\" % str(train_vect.shape))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f53c77489d201288ae84954db2604caaf49e70c8"},"cell_type":"code","source":"def top_tfidf_words(tfidf_, words, top_n=25):\n    topn_ids = np.argsort(tfidf_)[::-1][:top_n]\n    top_words = [(words[i], tfidf_[i]) for i in topn_ids]\n    df = pd.DataFrame(top_words)\n    df.columns = ['word', 'tfidf']\n    return df\n\ndef top_mean_words(tf_idf_matrix, words, grp_ids, min_tfidf=0.1, top_n=25):\n    _matrix = tf_idf_matrix[grp_ids]\n#     _matrix[_matrix < min_tfidf] = 0\n    tfidf_means = _matrix.mean(axis=0)\n    tfidf_means = np.asarray(tfidf_means).reshape(-1)\n    return top_tfidf_words(tfidf_means, words, top_n)\n\ndef top_words_by_target(tf_idf_matrix, words, min_tfidf=0.1, top_n=20):\n    pos_idx = train_df.index[train_df.target == 1].values\n    neg_idx = train_df.index[train_df.target == 0].values\n    return top_mean_words(tf_idf_matrix, words, pos_idx, min_tfidf, top_n), top_mean_words(tf_idf_matrix, words, neg_idx, min_tfidf, top_n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9cbb261ae14cf8d21a2de250b89ab0245fe73b44"},"cell_type":"code","source":"pos_top_tfidf, neg_top_tfidf = top_words_by_target(train_vect, tf_idf_vector.get_feature_names())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4a85d62975e06592414f26bea8f820c6655ea365","_kg_hide-input":false},"cell_type":"code","source":"pos_top_tfidf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"87d2a924d143d88ee3874039a0eb141087dcc078","_kg_hide-input":true},"cell_type":"code","source":"trace = go.Bar(\n    x = pos_top_tfidf.word,\n    y = pos_top_tfidf.tfidf\n)\nlayout = dict(\n    title = \"Mean TF-IDF of word in positive\",\n    xaxis = dict(title = 'Word'),\n    yaxis = dict(title = 'Mean TF-IDF')\n)\ndata = [trace]\n\npy.iplot(dict(data = data, layout = layout), filename = 'basic-line')","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":false,"trusted":true,"_uuid":"8757834d0bf463f76efc4bd425a2165d06078d64"},"cell_type":"code","source":"neg_top_tfidf.head()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true,"_uuid":"c3375cb1e3d225c764b794b61899f6f7d6b3b191"},"cell_type":"code","source":"trace = go.Bar(\n    x = neg_top_tfidf.word,\n    y = neg_top_tfidf.tfidf\n)\n\nlayout = dict(\n    title = \"Mean TF-IDF of word in negative\",\n    xaxis = dict(title = 'Word'),\n    yaxis = dict(title = 'Mean TF-IDF')\n)\n\ndata = [trace]\n\npy.iplot(dict(data = data, layout = layout), filename = 'basic-line')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"aa75d2df0cd7bb78eed9fce4635bc2de0bf36021"},"cell_type":"markdown","source":"We found that in the positive sample, there are some keywords: Muslim, white, black, sex. Some of the key factors in the negative sample may be: best, good. Maybe we can catch those in our features.\n\n**Bi-gram TF-IDF**"},{"metadata":{"trusted":true,"_uuid":"068ace2a45e1053dde8bf510479dd29324e0b406"},"cell_type":"code","source":"tf_idf_vector = TfidfVectorizer(strip_accents='unicode', analyzer='word',ngram_range=(2,2),\n                                use_idf=1, smooth_idf=1, sublinear_tf=1, stop_words = 'english')\ntrain_vect = tf_idf_vector.fit_transform(train_df.question_text.values)\nprint(\"The shape of bi-gram TF-IDF train matrix: %s\" % str(train_vect.shape))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"625f03d59ab19d768030d96a84d1ea71f13edbed"},"cell_type":"code","source":"pos_top_tfidf, neg_top_tfidf = top_words_by_target(train_vect, tf_idf_vector.get_feature_names())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b12c0977c44502ab3e5711c787d4a0442d905603"},"cell_type":"code","source":"pos_top_tfidf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"79c9aa493a50d269199d306745a4c5a7e310f32b"},"cell_type":"code","source":"neg_top_tfidf.head()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true,"_uuid":"5b79b1313353a04bb5a6576ced79b7d469d4e2ab"},"cell_type":"code","source":"trace = go.Bar(\n    x = pos_top_tfidf.word,\n    y = pos_top_tfidf.tfidf\n)\n\nlayout = dict(\n    title = \"Mean TF-IDF of word in positive (Bigram)\",\n    xaxis = dict(title = 'Word'),\n    yaxis = dict(title = 'Mean TF-IDF')\n)\n\ndata = [trace]\n\npy.iplot(dict(data = data, layout = layout), filename = 'basic-line')","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true,"_uuid":"df95afb4e56076f82f8bd5190310573b581d277a"},"cell_type":"code","source":"trace = go.Bar(\n    x = neg_top_tfidf.word,\n    y = neg_top_tfidf.tfidf\n)\n\nlayout = dict(\n    title = \"Mean TF-IDF of word in negative (Bigram)\",\n    xaxis = dict(title = 'Word'),\n    yaxis = dict(title = 'Mean TF-IDF')\n)\n\ndata = [trace]\n\npy.iplot(dict(data = data, layout = layout), filename = 'basic-line')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"986ceb833771b57cf759d6a2227f0e03f9b4864e"},"cell_type":"markdown","source":"Here in the positive sample n-gram keywords: Donald Trump, chinese people, black people, white people. So we can know why “people” become the keyword in the word. Maybe bi-gram can catch best keywords for model as features.\n\n### Count words analysis"},{"metadata":{"trusted":true,"_uuid":"d8cb610c4b06ddcb89b0a0249fda4a2e7fae8585"},"cell_type":"code","source":"count_vector = CountVectorizer(strip_accents='unicode', analyzer='word',ngram_range=(1,1),  stop_words = 'english')\ntrain_vect = count_vector.fit_transform(train_df.question_text.values)\nprint(\"The shape of count train matrix: %s\" % str(train_vect.shape))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0016830508c18223e4d3d9ffaa5c3fc23a509926"},"cell_type":"code","source":"pos_top_count, neg_top_count = top_words_by_target(train_vect, count_vector.get_feature_names())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"549a83069cdd5684a4efd14e07d6b6f666b3d001"},"cell_type":"code","source":"pos_top_count.head()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true,"_uuid":"2800e5e55006bdad0a4cf0b7eb338a35243e0fad"},"cell_type":"code","source":"trace = go.Bar(\n    x = pos_top_count.word,\n    y = pos_top_count.tfidf\n)\n\nlayout = dict(\n    title = \"Mean Count Rate of word in positive (Bigram)\",\n    xaxis = dict(title = 'Word'),\n    yaxis = dict(title = 'Mean Count Rate')\n)\n\ndata = [trace]\n\npy.iplot(dict(data = data, layout = layout), filename = 'basic-line')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c4c24fc1f9d3e3edd7158de229bc14decdf483ed"},"cell_type":"code","source":"neg_top_count.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8cf510dbb16366f789314590f1967c29214cb516"},"cell_type":"code","source":"trace = go.Bar(\n    x = neg_top_count.word,\n    y = neg_top_count.tfidf\n)\n\nlayout = dict(\n    title = \"Mean Count Rate of word in negative (Bigram)\",\n    xaxis = dict(title = 'Word'),\n    yaxis = dict(title = 'Mean Count Rate')\n)\n\ndata = [trace]\n\npy.iplot(dict(data = data, layout = layout), filename = 'basic-line')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1c02ca9dd9badb4982d7e950f7a3da99f9814642"},"cell_type":"markdown","source":"emmm..... It is possible that the keywords not captured by countvector and tf-idf vector are basically the same.\n\n### keywords analysis\n\nAs we have guessed some of the keywords above, we try to draw a distribution containing the number of these keywords.\n\n#### Muslim\n\nThroughout the years, due to terrorist attacks and Trump’s appointment, the word Muslim has become racist."},{"metadata":{"trusted":true,"_uuid":"79d29a8a09c6b1bfb007134aa9ef0c37f4dced10"},"cell_type":"code","source":"def count_keywords(df, word):\n    df[\"count_%s\" % word] = df.question_text.apply(lambda x: x.lower().count(word))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e3b9209fcd0507d2a51d315e08a24c85f52b3c60"},"cell_type":"code","source":"train_df['count_muslim'] = train_df.question_text.apply(lambda x: x.lower().count(\"muslim\"))\ntest_df['count_muslim'] = test_df.question_text.apply(lambda x: x.lower().count(\"muslim\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1019e36b914a9d17e1de18c06e2946ff97b88a18"},"cell_type":"code","source":"violin_chart(train_df, \"count_muslim\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ddae5c207b8a13465770d7fadccdc50438ac7977"},"cell_type":"markdown","source":"We found that the negative sample basically does not have the word Muslim, and the positive sample appears relatively more.\n\n### Build Models"},{"metadata":{"trusted":true,"_uuid":"945c1d1dce52cf600b0b431a7e8ccb382e0854ee"},"cell_type":"code","source":"keywords = [\"trump\", \"chinese people\", \"black\", \"white people\", \"indians\", \"muslims\", \"sex\", \"india\"]\nfor keyword in keywords:\n    count_keywords(train_df, keyword)\n    count_keywords(test_df, keyword)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"76992c451936ee01ce7273b4982e16a692e00f6e"},"cell_type":"code","source":"tf_idf_word_vector = TfidfVectorizer(strip_accents='unicode', analyzer='word',ngram_range=(1, 2), \n                                     use_idf=1, smooth_idf=1, sublinear_tf=1, stop_words = 'english', \n                                     max_features=200000)\n\n# tf_idf_char_vector = TfidfVectorizer(sublinear_tf=True, strip_accents='unicode', \n#                                      analyzer='char', token_pattern=r'\\w{1,}',stop_words='english',\n#                                      ngram_range=(2, 5), max_features=50000)\n# tf_idf_char_vector = TfidfVectorizer(sublinear_tf=True, strip_accents='unicode', analyzer='char_wb', token_pattern=r'\\w{1,}',\n#                                      stop_words='english', ngram_range=(2, 5), max_features=50000)\n\ntrain_word_tfidf = tf_idf_word_vector.fit_transform(train_df.question_text)\n# train_char_tfidf = tf_idf_char_vector.fit_transform(train_df.question_text)\n# train_tfidf = sparse.hstack([train_word_tfidf, train_char_tfidf]).tocsr()\ntrain_tfidf = train_word_tfidf\ndel train_word_tfidf\n# del train_char_tfidf\ngc.collect()\nprint(\"The train tf-idf shape %s\" % str(train_tfidf.shape))\n\ntest_word_tfidf = tf_idf_word_vector.transform(test_df.question_text)\n# test_char_tfidf = tf_idf_char_vector.transform(test_df.question_text)\n# test_tfidf = sparse.hstack([test_word_tfidf, test_char_tfidf]).tocsr()\ntest_tfidf = test_word_tfidf\ndel test_word_tfidf\n# del test_char_tfidf\ngc.collect()\n\nprint(\"The test tf-idf shape %s\" % str(test_tfidf.shape))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6394e2703d3a18bc44ec4a0ed5ce9df8d05847f3"},"cell_type":"code","source":"indirect_features_name = [feat_name for feat_name in train_df.columns.values if feat_name not in [\"qid\", \"question_text\", \"target\"]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5c9f564728d11d99119efc5d70757a679a7b11da"},"cell_type":"code","source":"# indirect_features_name\ntrain_indirect_features = train_df[indirect_features_name].values\ntest_indirect_features = test_df[indirect_features_name].values\ntarget = train_df.target.values\n\n# prepare to delete the train dataframe and test dataframe\nnum_train = train_df.shape[0]\nsubmission_df = pd.DataFrame({\"qid\":test_df[\"qid\"].values})\nvalidation_df = pd.DataFrame({\"qid\":train_df[\"qid\"].values})\ndel train_df\ndel test_df\ngc.collect()\n\nX_train = sparse.hstack([train_tfidf, train_indirect_features]).tocsr()\nX_test = sparse.hstack([test_tfidf, test_indirect_features]).tocsr()\ndel train_tfidf\ndel test_tfidf\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"685bdd92a248dd4b0ed60e27e022ca62b89f315d"},"cell_type":"code","source":"def lgb_f1_score(y_pre, data):\n    y_true = data.get_label()\n    best_f1 = f1_score(y_true, (y_pre>0.5).astype(int))\n#     best_f1 = 0\n#     for thresh in np.arange(0.1, 0.501, 0.01):\n#         thresh = np.round(thresh, 2)\n#         _f1 = f1_score(y_true, (y_pre>thresh).astype(int))\n#         if _f1 > best_f1:\n#             best_f1 = _f1\n    return 'f1', best_f1, True","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"322085e85ad85ff6de03395e270dc89dd794337a","scrolled":true},"cell_type":"code","source":"# Set LGBM parameters\nparams = {\n    \"objective\": \"binary\",\n    'metric': {'auc'},\n    \"boosting_type\": \"gbdt\",\n    \"verbosity\": -1,\n    \"num_threads\": 4,\n    \"bagging_fraction\": 0.8,\n    \"feature_fraction\": 0.8,\n    \"learning_rate\": 0.1,\n    \"num_leaves\": 31,\n    \"verbose\": -1,\n    \"min_split_gain\": .1,\n    \"reg_alpha\": .1,\n    \"device_type\": \"gpu\",\n    \"seed\": 2018\n}\n\nscores = []\nfolds = KFold(n_splits=5)\nindices = np.arange(num_train)\ntrn_lgbset = lgb.Dataset(data=X_train, label=target, free_raw_data=False)\nvalid_predict = np.zeros(num_train, dtype=np.float32)\nmean_best_iter = 0\nfor n_fold, (trn_idx, val_idx) in enumerate(folds.split(indices)):\n    print(\"valid in the %d fold\" % (n_fold + 1))\n    model = lgb.train(\n        params=params,\n        train_set=trn_lgbset.subset(trn_idx),\n        num_boost_round=1000,\n        valid_sets=[trn_lgbset.subset(val_idx)],\n        early_stopping_rounds=50,\n#         feval=lgb_f1_score,\n        verbose_eval=200\n    )\n    mean_best_iter += model.best_iteration / 5\n    valid_predict[val_idx] = model.predict(trn_lgbset.data[val_idx], num_iteration=model.best_iteration)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"256e87910f21cba62409c0b5770b949f2c6ce51b"},"cell_type":"code","source":"best_thresh = 0\nbest_f1 = 0\nfor thresh in np.arange(0.1, 0.501, 0.01):\n    thresh = np.round(thresh, 2)\n    _f1 = f1_score(target, (valid_predict>thresh).astype(int))\n    if _f1 > best_f1:\n        best_f1 = _f1\n        best_thresh = thresh\n    print(\"\\tF1 score at threshold {0} is {1}\".format(thresh, _f1))\n\nprint(\"Best F1 score {0}, Best thresh {1}\".format(best_f1, best_thresh))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6afdd87b2f8ed6f7d758b0daa31796834e4c4617"},"cell_type":"code","source":"model = lgb.train(\n    params=params,\n    train_set=trn_lgbset,\n    feval=lgb_f1_score,\n    num_boost_round=int(mean_best_iter)\n)\n\npredict = model.predict(X_test, num_iteration=model.best_iteration)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"82f0603d81824e71468f7dee565db455e31fc6af"},"cell_type":"code","source":"validation_df[\"prediction\"] = valid_predict\nvalidation_df.to_csv(\"validation.csv\", index=False)\n\nsubmission_df[\"prediction\"] = (predict > best_thresh).astype(int)\nsubmission_df.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}