{"cells":[{"metadata":{"trusted":true,"_uuid":"04410d2c16645c40e91aa280040e456904eb761f"},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.feature_extraction.text import TfidfVectorizer","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train = pd.read_csv(\"../input/train.csv\")\ntest = pd.read_csv(\"../input/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7ad939b5c277e9dfca55a8423813b978522ca335"},"cell_type":"code","source":"import nltk\nfrom nltk.corpus import stopwords\nimport string\n\neng_stopwords = set(stopwords.words(\"english\"))\n\n## Number of words in the text ##\ntrain[\"num_words\"] = train[\"question_text\"].apply(lambda x: len(str(x).split()))\ntest[\"num_words\"] = test[\"question_text\"].apply(lambda x: len(str(x).split()))\n\n## Number of unique words in the text ##\ntrain[\"num_unique_words\"] = train[\"question_text\"].apply(lambda x: len(set(str(x).split())))\ntest[\"num_unique_words\"] = test[\"question_text\"].apply(lambda x: len(set(str(x).split())))\n\n## Number of characters in the text ##\ntrain[\"num_chars\"] = train[\"question_text\"].apply(lambda x: len(str(x)))\ntest[\"num_chars\"] = test[\"question_text\"].apply(lambda x: len(str(x)))\n\n## Number of stopwords in the text ##\ntrain[\"num_stopwords\"] = train[\"question_text\"].apply(lambda x: len([w for w in str(x).lower().split() if w in eng_stopwords]))\ntest[\"num_stopwords\"] = test[\"question_text\"].apply(lambda x: len([w for w in str(x).lower().split() if w in eng_stopwords]))\n\n## Number of punctuations in the text ##\ntrain[\"num_punctuations\"] =train['question_text'].apply(lambda x: len([c for c in str(x) if c in string.punctuation]) )\ntest[\"num_punctuations\"] =test['question_text'].apply(lambda x: len([c for c in str(x) if c in string.punctuation]) )\n\n## Number of title case words in the text ##\ntrain[\"num_words_upper\"] = train[\"question_text\"].apply(lambda x: len([w for w in str(x).split() if w.isupper()]))\ntest[\"num_words_upper\"] = test[\"question_text\"].apply(lambda x: len([w for w in str(x).split() if w.isupper()]))\n\n## Number of title case words in the text ##\ntrain[\"num_words_title\"] = train[\"question_text\"].apply(lambda x: len([w for w in str(x).split() if w.istitle()]))\ntest[\"num_words_title\"] = test[\"question_text\"].apply(lambda x: len([w for w in str(x).split() if w.istitle()]))\n\n## Average length of the words in the text ##\ntrain[\"mean_word_len\"] = train[\"question_text\"].apply(lambda x: np.mean([len(w) for w in str(x).split()]))\ntest[\"mean_word_len\"] = test[\"question_text\"].apply(lambda x: np.mean([len(w) for w in str(x).split()]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"87d53d52ca515edb114565bf9630bc72ca18475b"},"cell_type":"code","source":"train_text = train['question_text']\ntest_text = test['question_text']\nall_text = pd.concat([train_text, test_text])\n\nword_vectorizer = TfidfVectorizer(\n    sublinear_tf=True,\n    strip_accents='unicode',\n    analyzer='word',\n    token_pattern=r'\\w{1,}',\n    stop_words='english',\n    ngram_range=(1, 1),\n    max_features=5000)\nword_vectorizer.fit(all_text)\ntrain_word_features = word_vectorizer.transform(train_text)\ntest_word_features = word_vectorizer.transform(test_text)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a4aa342cd6cb9a98ad4b641f132c9aa01d2fb48c"},"cell_type":"code","source":"eng_features = ['num_words', 'num_unique_words', 'num_chars', \n                'num_stopwords', 'num_punctuations', 'num_words_upper', \n                'num_words_title', 'mean_word_len']\ntrain_ = train[eng_features]\ntrain_.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"69cb846cae03476680374d2df5162832f7f13000"},"cell_type":"code","source":"from scipy.sparse import hstack, csr_matrix\ntrain_ = hstack((csr_matrix(train_), train_word_features))\nprint(train_.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a5a6ba25c258627500d45c8e7d95286e84635397"},"cell_type":"code","source":"test_ = test[eng_features]\ntest_ = hstack((csr_matrix(test_), test_word_features))\nprint(test_.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"91c6c7a7a1b0cc93b8cac1966481eef2636c1614"},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\ny = train['target']\nX_tr, X_va, y_tr, y_va = train_test_split(train_, y, test_size=0.2, random_state=42)\nprint(X_tr.shape, X_va.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"29853fb2ccb79240e2197e1a1ff725dbdd2a055b"},"cell_type":"code","source":"y_va.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ebab38326d99986c6803951293e90168ae33b410"},"cell_type":"code","source":"%%time\nimport lightgbm as lgb\n\nfrom sklearn.metrics import f1_score\n\ndef lgb_f1_score(y_hat, data):\n    y_true = data.get_label()\n    y_hat = np.round(y_hat) # scikits f1 doesn't like probabilities\n    return 'f1', f1_score(y_true, y_hat), True\n\nparams = {'application': 'binary',\n          'metric': 'binary_logloss',\n          'learning_rate': 0.05,   \n          'max_depth': 9,\n          'num_leaves': 100,\n          'verbosity': -1,\n          'data_random_seed': 3,\n          'bagging_fraction': 0.8,\n          'feature_fraction': 0.4,\n          'nthread': 16,\n          'lambda_l1': 1,\n          'lambda_l2': 1,\n          'num_rounds': 2700,\n          'verbose_eval': 100}\n\nd_train = lgb.Dataset(X_tr, label=y_tr.values)\nd_valid = lgb.Dataset(X_va, label=y_va.values)\nprint('Train LGB')\nnum_rounds = params.pop('num_rounds')\nverbose_eval = params.pop('verbose_eval')\nmodel = lgb.train(params,\n                  train_set=d_train,\n                  num_boost_round=num_rounds,\n                  valid_sets=[d_train, d_valid],\n                  verbose_eval=verbose_eval,\n                  valid_names=['train', 'val'],\n                  feval=lgb_f1_score)\nprint('Predict')\npred_test_va = model.predict(X_va)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"99c6909f29f94e6d7fb689ba5f204b4161441d5b"},"cell_type":"code","source":"%%time\nbest_threshold = 0.01\nbest_score = 0.0\nfor threshold in range(1, 100):\n    threshold = threshold / 100\n    score = f1_score(y_va, pred_test_va > threshold)\n    if score > best_score:\n        best_threshold = threshold\n        best_score = score\nprint(0.5, f1_score(y_va, pred_test_va > 0.5))\nprint(best_threshold, best_score)\n# 0.24 0.5918758665447358","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"82ab58fd59b30c537604b7c065601e0cd9b491bd"},"cell_type":"code","source":"%%time\npred_test_y = model.predict(test_)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"221ad851c0aec189222f58c5432d2326a8824c4f"},"cell_type":"code","source":"submit_df = pd.DataFrame({\"qid\": test[\"qid\"], \"prediction\": (pred_test_y > best_threshold).astype(np.int)})\nsubmit_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5d89a01935cd2ec1de0dd27601197944abcb7660"},"cell_type":"code","source":"submit_df['prediction'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f9ce309f24a0a9d3f742a3f1fae8cd69e42f1a61"},"cell_type":"code","source":"submit_df.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}