{"cells":[{"metadata":{"_uuid":"ff2269dc51c35002cea15de2cad1dbd0989201dc"},"cell_type":"markdown","source":"Goal this notebook is to demonstrate how to apply different algorithms to data, but without embeddings. Why? Because here I am not getting you maximum score; I just want to give you fast and simple example. \nHuge thanks for  awesome kernel https://www.kaggle.com/shujian/single-rnn-with-4-folds-clr by Shujian Liu.\nAlso https://www.kaggle.com/sudalairajkumar/a-look-at-different-embeddings where I got examples of neural networks models."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom tqdm import tqdm\nimport math\nfrom sklearn.model_selection import train_test_split\nfrom sklearn import metrics\n\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\n\nfrom sklearn.model_selection import GridSearchCV, KFold, StratifiedKFold\nfrom keras.wrappers.scikit_learn import KerasClassifier\nfrom sklearn.model_selection import cross_val_score\nfrom keras.models import Sequential, Model # initialize neural network library\nfrom keras.layers import Dense, Input, LSTM, Embedding, Dropout, Activation, CuDNNGRU, Conv1D, Bidirectional, GlobalMaxPool1D, GlobalAveragePooling1D, concatenate # build our layers library\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\nimport time\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.\n\n\nfrom keras import initializers, regularizers, constraints, optimizers, layers\nfrom sklearn.feature_extraction.text import CountVectorizer\n\nfrom keras.preprocessing import text, sequence\nfrom sklearn.metrics import f1_score","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"# load train and test datasets\ntrain_df = pd.read_csv(\"../input/train.csv\")\ntest_df = pd.read_csv(\"../input/test.csv\")\nprint(\"Train datasets shape:\", train_df.shape)\nprint(\"Test datasets shape:\", test_df.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a6c7fb86c645c98f648caaa15893d7b885f65434"},"cell_type":"code","source":"## split to train and val\ntrain_df, val_df = train_test_split(train_df, test_size=0.1, random_state=2018)\n\n## some config values \nembed_size = 300 # how big is each word vector\nmax_features = 50000 # how many unique words to use (i.e num rows in embedding vector)\nmaxlen = 100 # max number of words in a question to use\n\n## fill up the missing values\ntrain_X = train_df[\"question_text\"].fillna(\"_na_\").values\nval_X = val_df[\"question_text\"].fillna(\"_na_\").values\ntest_X = test_df[\"question_text\"].fillna(\"_na_\").values\n\n## Tokenize the sentences\ntokenizer = Tokenizer(num_words=max_features)\ntokenizer.fit_on_texts(list(train_X))\ntrain_X = tokenizer.texts_to_sequences(train_X)\nval_X = tokenizer.texts_to_sequences(val_X)\ntest_X = tokenizer.texts_to_sequences(test_X)\n\n## Pad the sentences \ntrain_X = pad_sequences(train_X, maxlen=maxlen)\nval_X = pad_sequences(val_X, maxlen=maxlen)\ntest_X = pad_sequences(test_X, maxlen=maxlen)\n\n## Get the target values\ntrain_y = train_df['target'].values\nval_y = val_df['target'].values","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bb2e9b953f666bb4cf43b9592431a74fe7a58c80"},"cell_type":"markdown","source":"There are few models, enable only the one model. But you can try any of those models just for better understanding which is best for you"},{"metadata":{"trusted":true,"_uuid":"f5a6d375f1ea89403d86a6d1ab6070c204f60633"},"cell_type":"code","source":"def first_nn_model():    \n    inp = Input(shape=(maxlen,))\n    x = Embedding(max_features, embed_size)(inp)\n    x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)\n    x = GlobalMaxPool1D()(x)\n    x = Dense(16, activation=\"relu\")(x)\n    x = Dropout(0.1)(x)\n    x = Dense(1, activation=\"sigmoid\")(x)\n    model = Model(inputs=inp, outputs=x)\n    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n\n    print(model.summary())\n    return model\n#model = first_nn_model()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0e979e53861cfac3dc4eaf33f53b2c6059305f55"},"cell_type":"code","source":"from sklearn.cluster import KMeans\ndef kmeans_model():\n    model = KMeans()\n    return model\n#model = kmeans_model()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d5a0fecb6a6481c043cc136f635da65a42783cb6"},"cell_type":"code","source":"from sklearn.naive_bayes import GaussianNB\ndef nbgauss_model():\n    model = GaussianNB()\n    return model\n#model = nbgauss_model()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3adfdb611a5a70b992257c294d2a1fa8ecc5240f"},"cell_type":"code","source":"from sklearn.naive_bayes import MultinomialNB\ndef nbmulti_model():\n    model = MultinomialNB()\n    return model\n#model = nbmulti_model()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"aca34ca9e9817737afef917b5aa48e1d08eaa4bd"},"cell_type":"code","source":"def second_nn_model():\n    inp = Input(shape=(maxlen, ))\n    x = Embedding(max_features, 100)(inp)\n    x = CuDNNGRU(64, return_sequences=True)(x)\n    avg_pool = GlobalAveragePooling1D()(x)\n    max_pool = GlobalMaxPooling1D()(x)\n    conc = concatenate([avg_pool, max_pool])\n    outp = Dense(1, activation=\"sigmoid\")(conc)\n    \n    model = Model(inputs=inp, outputs=outp)\n    model.compile(loss='binary_crossentropy',\n                  optimizer='adam',\n                  metrics=['accuracy'])\n    print(model.summary())\n    return model\n\n#model = gsecond_nn_model()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f160fb9b42d009648ab8544f7d6af52c93e61684"},"cell_type":"code","source":"# Evaluating the ANN \ndef simple_nn_model():\n    model = Sequential() # initialize neural network\n    model.add(Dense(units = 8, kernel_initializer = 'uniform', activation = 'relu', input_dim = val_X.shape[1]))\n    model.add(Dense(units = 4, kernel_initializer = 'uniform', activation = 'relu'))\n    model.add(Dense(units = 1, kernel_initializer = 'uniform', activation = 'sigmoid'))\n    model.compile(optimizer = 'adam', loss = 'binary_crossentropy', metrics = ['accuracy'])\n    return model\nmodel = simple_nn_model()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0b6b08d9998e7fcc48157a04fb8e5ea108f01e96"},"cell_type":"code","source":"from sklearn.svm import LinearSVC\ndef svc_model():\n    model = LinearSVC()\n    return model\n#model = svc_model()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"83be1b30fa0ffce1dd43e4f7978044aca72473c8"},"cell_type":"code","source":"from sklearn.linear_model import LinearRegression\ndef linr_model():\n    model = LinearRegression()\n    return model\n#model = lr_model()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"67ad36c11410a39b0736aa8a190988a618fc01a7"},"cell_type":"code","source":"from sklearn.linear_model import LogisticRegression\ndef logr_model():\n    model = LogisticRegression()\n    return model\n#model = logr_model()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b8c4e0a8e16c4b5575fd495b16dbe7508898ef81"},"cell_type":"code","source":"# Tried to run it but it doesn't work (need some preparation with dataset)\nimport lightgbm as lgb\ndef lgb_model():\n    lgb_train = lgb.Dataset(train_X, train_y)\n    lgb_eval = lgb.Dataset(test_X, val_y, reference=lgb_train)\n    # specify your configurations as a dict\n    params = {\n    'boosting_type': 'gbdt',\n    'objective': 'regression',\n    'metric': {'l2', 'l1'},\n    'num_leaves': 31,\n    'learning_rate': 0.05,\n    'feature_fraction': 0.9,\n    'bagging_fraction': 0.8,\n    'bagging_freq': 5,\n    'verbose': 0\n    }\n    \n    model = lgb.train(params,\n                      lgb_train,\n                      num_boost_round=20,\n                      valid_sets=lgb_eval,\n                      early_stopping_rounds=5)\n    # predict\n    pred_noemb_val_y = gbm.predict(X_test, num_iteration=gbm.best_iteration)\n    y_pred_val = (pred_noemb_val_y > 0.5).astype(int)\n    # eval\n    print('The f1score of prediction is:', f1score(val_y, y_pred_val) ** 0.5)\n#lgb_model()    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"529a1f0429a18b5c1c85a6ad11426f77b6cbdb09"},"cell_type":"code","source":"from xgboost import XGBClassifier\ndef xgb_model():\n    model = XGBClassifier()\n    return model\n#model = xgb_model()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"81dcabdb7e0924275c355ce1822a56b95f0a7a59"},"cell_type":"code","source":"#Create model, train and predict\n\nmodel.fit(train_X, train_y)\npred_noemb_val_y = model.predict(val_X)\npred_noemb_test_y = model.predict(test_X)\n\npred_noemb_test_y.shape\ntrain_meta = np.zeros(train_y.shape)\ntest_meta = np.zeros(test_X.shape[0])\ny_pred_val = (pred_noemb_val_y > 0.5).astype(int)\ny_pred_test = (pred_noemb_test_y > 0.5).astype(int)\n# Print predict using f1score from sklearn\nscore = f1_score(val_y, y_pred_val)\nprint(score)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ec3c5180e61aa752febd9f59e2d357ac067f5fd1"},"cell_type":"code","source":"# Collect garbage\nimport gc; gc.collect()\ntime.sleep(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b1c5a5f535e27c8d82fa7ee4be3baae2fb279e75"},"cell_type":"code","source":"# Create a submission\nsub_df = pd.DataFrame({'qid':test_df.qid.values})\nsub_df['prediction'] = y_pred_test\nsub_df.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}