{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import nltk\n#nltk.download('popular')\n\nimport re\nfrom nltk.corpus import stopwords\nfrom nltk.stem import WordNetLemmatizer\nfrom nltk.tokenize import word_tokenize\nfrom nltk.tokenize import TweetTokenizer\n\n# text cleaning & tokenization\ndef tokenize(text, stop_set = None, lemmatizer = None):\n    \n    # clean text\n    text = text.encode('ascii', 'ignore').decode('ascii')\n    #text = text.lower()\n    \n    text = re.sub(r'\\b(?:(?:https?|ftp)://)?\\w[\\w-]*(?:\\.[\\w-]+)+\\S*', ' ', text) # remove hyperlink,subs charact in the brackets\n    text = re.sub(\"[\\r\\n]\", ' ', text) # remove new line characters\n    #text = re.sub(r'[^\\w\\s]','',text)\n    text = text.strip()\n    \n    #tokens = word_tokenize(text)\n    # use TweetTokenizer instead of word_tokenize -> to prevent splitting at apostrophies\n    tknzr = TweetTokenizer()\n    tokens = tknzr.tokenize(text)\n    \n    # retain tokens with at least two words\n    tokens = [token for token in tokens if re.match(r'.*[a-z]{1,}.*', token)]\n    \n    # remove stopwords - optional\n    # removing stopwords lost important information\n    if stop_set != None:\n        tokens = [token for token in tokens if token not in stop_set]\n    \n    # lemmmatization - optional\n    if lemmatizer != None:\n        tokens = [lemmatizer.lemmatize(token) for token in tokens]\n    return tokens\n\n\ntrain['tokens'] = train['question_text'].map(lambda x: tokenize(x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from gensim.models import KeyedVectors\n\nnews_path = '../input/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin'\nembeddings_index = KeyedVectors.load_word2vec_format(news_path, binary=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Google News Embeddings\n# replace not found words\nto_remove = ['to','of','and', 'a']\n\n#replace_dict = {}\n#replace_dict = {'quora':'Quora', 'i\\'ve':'I\\'ve', 'instagram':'Instagram', 'upsc':'UPSC', 'bitcoin':'Bitcoin', 'trump\\'s':'Trump',\n#               'mbbs':'MBBS', 'whatsapp':'WhatsApp', 'favourite':'favorite', 'ece':'ECE', 'aiims':'AIIMS', 'colour':'color',\n#               'doesnt':'doesn\\'t','centre':'center','sbi':'SBI','cgl':'CGL','iim':'IIM','btech':'BTech'}\n\nreplace_dict = {'favourite':'favorite', 'bitcoin':'Bitcoin', 'colour':'color', 'doesnt':'doesn\\'t', 'centre':'center', 'Quorans':'Quora',\n               'travelling':'traveling', 'counselling':'counseling', 'didnt':'didn\\'t', 'btech':'BTech','isnt':'isn\\'t',\n               'Shouldn\\'t':'shouldn\\'t', 'programme':'program', 'realise':'realize', 'Wouldn\\'t':'wouldn\\'t', 'defence':'defense',\n               'Aren\\'t':'aren\\'t', 'organisation':'organization', 'How\\'s':'how\\'s', 'e-commerce':'ecommerce', 'grey':'gray',\n               'bitcoins':'Bitcoin', 'honours':'honors', 'learnt':'learned', 'licence':'license', 'mtech':'MTech', 'colours':'colors',\n               'e-mail':'email', 't-shirt':'tshirt', 'Whatis':'What\\'s', 'theatre':'theater', 'labour':'labor', 'Isnt':'Isn\\'t',\n               'behaviour':'behavior','aadhar':'Aadhar', 'Qoura':'Quora', 'aluminium':'aluminum'}\n\ndef clean_token(tokens, remove_list, re_dict, embedding):\n    \n    c_tokens = []\n    for token in tokens:\n        if token not in remove_list:\n            token2 = token\n            if token2 in embedding:\n                c_tokens.append(token2)\n            elif token2 in re_dict:\n                token2 = re_dict[token2]\n                c_tokens.append(token2)\n            else:    \n                # apostrophe\n                if token2.endswith('\\'s'):\n                    token2 = token2[:-2]\n                    \n                if (token2.endswith('s')) & (token2[:-1] in embedding):\n                    token2 = token2[:-1]\n                    \n                # break dash\n                if \"-\" in token2:\n                    token2 = token2.split('-')\n                    c_tokens += token2\n                else:\n                    c_tokens.append(token2)\n        \n\n    return c_tokens\n\ntrain['clean_tokens'] = train['tokens'].map(lambda x: clean_token(x, to_remove, replace_dict, embeddings_index))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def doc_mean(tokens, embedding):\n    \n    e_values = []\n    e_values = [embedding[token] for token in tokens if token in embedding]\n    \n    if len(e_values) > 0:\n        return np.mean(np.array(e_values), axis=0)\n        #return np.sum(np.array(e_values), axis=0)\n    else:\n        #return np.ones(300)*-999\n        return np.zeros(300)\n      \nX = np.vstack(train['clean_tokens'].apply(lambda x: doc_mean(x, embeddings_index)))\n#X = np.vstack(train['tokens'].apply(lambda x: doc_mean(x, embeddings_index)))\n\ny = train['target'].values\nindices = train.index","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# free up RAM\nimport gc\n\ndel embeddings_index\n#del train\n\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn import linear_model, tree, ensemble, metrics, model_selection, exceptions\n\n\ndef print_score(y_true, y_pred):\n    print(' accuracy : ', metrics.accuracy_score(y_true, y_pred))\n    print('precision : ', metrics.precision_score(y_true, y_pred))\n    print('   recall : ', metrics.recall_score(y_true, y_pred))\n    print('       F1 : ', metrics.f1_score(y_true, y_pred))\n\n    \n# train-test split\nX_train, X_test, y_train, y_test, train_indices, test_indices = model_selection.train_test_split(X, y, indices, test_size = 0.2, random_state = 2019)\n\n# train-test split - small sample\n#X_train, X_test, y_train, y_test = model_selection.train_test_split(X, y, test_size = 0.2, train_size = 0.2, random_state = 2019)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# free up RAM\nimport gc\n\n#del X\ndel y\n#del train\n\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Iterate 1 - Achieve high accuracy + Threshold search"},{"metadata":{"trusted":true},"cell_type":"code","source":"# import lightgbm as lgb\n\n# lgb_c = lgb.LGBMClassifier(learning_rate = 0.02,n_estimators = 2000)\n\n# lgb_c.fit(X_train, y_train,\n#           eval_set = [(X_test, y_test)],\n#           early_stopping_rounds = 5,\n#           eval_metric = 'auc',\n#           verbose = 100)\n\n\n# y_pred = lgb_c.predict(X_test, num_iteration=lgb_c.best_iteration_)\n# print_score(y_test, y_pred)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# treshold search\n\n# y_pred_proba = lgb_c.predict_proba(X_test,num_iteration=lgb_c.best_iteration_)[:,1]\n\n# thresholds = []\n# for thresh in np.arange(0.1, 0.901, 0.01):\n#     res = metrics.f1_score(y_test, (y_pred_proba > thresh).astype(int))\n#     thresholds.append([thresh, res])\n#     # print(\"F1 score at threshold {0} is {1}\".format(thresh, res))\n\n# thresholds.sort(key=lambda x: x[1], reverse=True)\n# best_thresh = thresholds[0][0]\n# print(\"Best threshold: \", best_thresh)\n# print(\"Best F1: \", thresholds[0][1])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Iterate 2 - Tune scale_pos_weight"},{"metadata":{"trusted":true},"cell_type":"code","source":"# import lightgbm as lgb\n\n# gridParams = {\n#     'scale_pos_weight': [3, 3.5, 4, 4.5]\n#     }\n\n# lgb_c = lgb.LGBMClassifier(learning_rate = 0.02,n_estimators = 2000)\n# grid_lgb = model_selection.GridSearchCV(lgb_c, gridParams, scoring='f1', cv = 3)\n\n\n# grid_lgb.fit(X_train, y_train, \n#           eval_set = [(X_test, y_test)],\n#           early_stopping_rounds = 5,\n#           eval_metric = 'auc',\n#           verbose = 500)\n\n# print(grid_lgb.best_params_)\n# print(grid_lgb.best_score_)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import lightgbm as lgb\n\nlgb_c = lgb.LGBMClassifier(learning_rate = 0.04,n_estimators = 3200, boosting_type = 'dart')\n\nlgb_c.fit(X_train, y_train,\n          eval_set = [(X_test, y_test)],\n          eval_metric = 'auc',\n          verbose = 500)\n\n\ny_pred = lgb_c.predict(X_test, num_iteration=lgb_c.best_iteration_)\nprint_score(y_test, y_pred)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# treshold search\n\ny_pred_proba = lgb_c.predict_proba(X_test,num_iteration=lgb_c.best_iteration_)[:,1]\n\nthresholds = []\nfor thresh in np.arange(0.1, 0.91, 0.01):\n    thresh = np.round(thresh, 2)\n    res = metrics.f1_score(y_test, (y_pred_proba > thresh).astype(int))\n    thresholds.append([thresh, res])\n    # print(\"F1 score at threshold {0} is {1}\".format(thresh, res))\n\nthresholds.sort(key=lambda x: x[1], reverse=True)\nbest_thresh = thresholds[0][0]\nprint(\"Best threshold: \", best_thresh)\nprint(\"Best F1: \", thresholds[0][1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import lightgbm as lgb\n\nlgb_c_weight = lgb.LGBMClassifier(learning_rate = 0.04,n_estimators = 3200, boosting_type = 'dart', scale_pos_weight = 3.5)\n\nlgb_c_weight.fit(X_train, y_train,\n          eval_set = [(X_test, y_test)],\n          eval_metric = 'auc',\n          verbose = 500)\n\n\ny_pred_weight = lgb_c_weight.predict(X_test, num_iteration=lgb_c_weight.best_iteration_)\nprint_score(y_test, y_pred_weight)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"p_labels = lgb_c_weight.predict(X, num_iteration=lgb_c_weight.best_iteration_)\np_proba = lgb_c_weight.predict_proba(X, num_iteration=lgb_c_weight.best_iteration_)\n\noutput_np = np.concatenate((p_labels.reshape(len(p_labels), 1), p_proba), axis = 1)\noutput = pd.DataFrame(output_np)\noutput.to_csv('label with proba.csv')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}