{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train_org = pd.read_csv(\"../input/train.csv\")\ntest_org = pd.read_csv(\"../input/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7b9c3a5e701cc80696be353baf2662f07e48a5cd"},"cell_type":"code","source":"from sklearn.feature_extraction.text import TfidfVectorizer","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"52d8dc1a20265d48cdba5bf975fc0338345d705d"},"cell_type":"code","source":"vectorizer = TfidfVectorizer(max_features=10000)\nX = vectorizer.fit_transform(train_org[\"question_text\"].values.tolist())\n#print(vectorizer.get_feature_names())\nprint(X.shape)\nY = train_org.target","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"72abd726cf5e4709dbf138d1e834a474c99a9fb9"},"cell_type":"code","source":"test_X = vectorizer.transform(test_org.question_text.values.tolist())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8af30e6afd4e7734d9ca790ae8dadbe8328eefde"},"cell_type":"code","source":"from sklearn.model_selection import StratifiedKFold\nkfold_indexes = []\nkfold = StratifiedKFold(n_splits=5, random_state=2018, shuffle=True)\nall_val_index = []\nfor i, (train_index, valid_index) in enumerate(kfold.split(X, Y)):\n    kfold_indexes.append([train_index, valid_index])\n    all_val_index.extend(valid_index)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"044ff202821ee0ab578757be9715cf0c84a4e8dd"},"cell_type":"code","source":"from sklearn.naive_bayes import MultinomialNB\nfrom sklearn.linear_model import LogisticRegression","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a3e77b7df8fffd085ea3643d3e9ca5692ff755b0"},"cell_type":"code","source":"def return_org_index(pred_shuf):\n    pred = np.zeros(len(all_val_index))\n    for i, id_ in enumerate(all_val_index):\n        pred[id_] = pred_shuf[i]\n    return pred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"30d3b639d76c9b9edb09989746b1a56aecd50c0f"},"cell_type":"code","source":"def one_model_predict(clf,kfold_indexes):\n    pred_train = []\n    train_true_label = []\n    pred_test = []\n    for train_index,valid_index in kfold_indexes:\n        X_train, X_val, Y_train, Y_val = X[train_index], X[valid_index], Y[train_index], Y[valid_index]\n        #clf = MultinomialNB(alpha=0.01)\n        clf.fit(X_train, Y_train)\n        pred_val = clf.predict_proba(X_val)\n        pred_train.extend(pred_val)\n        train_true_label.extend(Y_val)\n        pred_test_one_fold = clf.predict_proba(test_X)\n        pred_test.append(pred_test_one_fold)\n    train_true_label = np.array(train_true_label)\n    pred_train = np.array(pred_train)[:,1]\n    pred_test = np.mean(pred_test,axis=0)[:,1]\n    train_true_label = return_org_index(train_true_label)\n    pred_train = return_org_index(pred_train)\n    return train_true_label,pred_train,pred_test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dd89ecd3b97f9723efcd5885c53df1ffbc8e31eb"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0fc7c9689e570ae9c44e52a33e9a534c3c406bda"},"cell_type":"code","source":"clf = MultinomialNB(alpha=0.01)\ntrain_true_label1,pred_train1,pred_test1 = one_model_predict(clf,kfold_indexes)\nclf = LogisticRegression()\ntrain_true_label2,pred_train2,pred_test2 = one_model_predict(clf,kfold_indexes)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cd3091b16063123a25f5dd4237ad49540d363d5b"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f65017708b2f4995121dd1fbb5f28ddf8a92d997"},"cell_type":"code","source":"np.shape(pred_train1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f038a0d991cc5f2629ce709061046583185665c6"},"cell_type":"code","source":"first_layer_pred_train = np.vstack((pred_train1,pred_train2)).T\nfirst_layer_pred_test = np.vstack((pred_test1,pred_test2)).T","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"57d8ef2f23a2392924dde42103d321e522692f12"},"cell_type":"code","source":"np.shape(first_layer_pred_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e0bdb2bbe7f463d5eae9df56d68c8511adb97711"},"cell_type":"code","source":"from sklearn.linear_model import Ridge","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6712d650f8b4f51a2a56c9f81fd1b1d833919b80"},"cell_type":"code","source":"# stacking\ntrain_second_pred = []\ntest_second_pred =[]\ntrain_second_true_label = []\nfor id_train,id_valid in kfold_indexes:\n    train_onelabel_onedire = []\n    Xvalid = first_layer_pred_train[id_valid]\n    Xtrain = first_layer_pred_train[id_train]\n    Y_train = Y[id_train]\n    Y_valid = Y[id_valid]\n    model = Ridge(alpha=2.0)\n    model.fit(Xtrain, Y_train)\n    train_second_pred.extend(model.predict(Xvalid))\n    test_second_pred.append(model.predict(first_layer_pred_test))\n    train_second_true_label.extend(Y_valid)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1541c30ffa5f10aa808ebf45f5cd98548023eeb0"},"cell_type":"code","source":"from sklearn.metrics import f1_score","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"229df7c5cc50b841f7e089aaa7d789d07c442eb4"},"cell_type":"code","source":"pred_second_test = np.mean(test_second_pred,axis=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bd6f0e9c49f930dda89b678ec433814dd8e2f319"},"cell_type":"code","source":"def optimise_f2_thresholds(y, p, verbose=True, resolution=100):\n    def mf(x):\n        p2  =  (p > x).astype(np.int)\n        score = f1_score(y, p2)#fbeta_score(np.array(y), np.array(p2), beta=2, average='samples')\n        return score\n    x = 0.2\n    best_i2 = 0\n    best_score = 0\n    for i2 in range(resolution):\n        i2 /= 1.0 * resolution\n        x = i2\n        score = mf(x)\n        if score > best_score:\n            best_i2 = i2\n            best_score = score\n    x = best_i2\n    if verbose:\n        print(best_i2, best_score)\n    return best_i2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f30a53ad3f431f3efa5815dc6a56451e33aaf32a"},"cell_type":"code","source":"threhold = optimise_f2_thresholds(np.array(train_second_true_label), np.array(train_second_pred))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"44b5c18dab677cfa061e624c45e1484674872acf"},"cell_type":"code","source":"print(threhold)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d04f4afde63834eee94413b5797bfd58989a0a6e"},"cell_type":"code","source":"sub = pd.read_csv('../input/sample_submission.csv')\nsub.prediction = (pred_second_test > threhold).astype(np.int)\nsub.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f96dc788f9cd78af9b3c534df8c0c35549be1eed"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}