{ "cells": [ { "cell_type": "markdown", "metadata": { "_cell_guid": "dc8a186e-d299-a558-3c90-e476f4e8ab49" }, "source": [ "# Quora using Logistic Regression" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "a94bbb2d-c58b-0b72-72ad-b48274e69ea2" }, "outputs": [], "source": [ "%reset -f\n", "from sklearn.feature_extraction.text import TfidfVectorizer\n", "from nltk.corpus import stopwords\n", "from nltk import word_tokenize\n", "from datetime import *\n", "from sklearn.model_selection import train_test_split\n", "from sklearn.metrics import log_loss\n", "import time\n", "#from pyxdameraulevenshtein import damerau_levenshtein_distance, normalized_damerau_levenshtein_distance\n", "import numpy as np\n", "import pandas as pd\n", "from sklearn.linear_model import LogisticRegression\n", "from sklearn.cross_validation import train_test_split\n", "from sklearn.metrics import roc_auc_score\n", "import numpy as np\n", "import pandas as pd\n", "from sklearn.ensemble import RandomForestClassifier\n", "from nltk.stem.porter import *\n", "stemmer = PorterStemmer()\n", "import random\n", "import re\n", "import numpy as np # linear algebra\n", "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n", "from sklearn.ensemble import RandomForestClassifier\n", "\n", "import scipy.io\n", "import scipy.io\n", "import scipy.io\n", "# import pysptk\n", "import scipy.io.wavfile\n", "import warnings\n", "\n", "warnings.filterwarnings('ignore')\n", "\n", "# numerical processing and scientific libraries\n", "import scipy\n", "\n", "from sklearn.cross_validation import StratifiedKFold\n", "from scipy.stats import norm\n", "import scipy.sparse\n", "from sklearn.feature_extraction.text import TfidfVectorizer\n", "\n", "import numpy as np # linear algebra\n", "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n", "import matplotlib.pyplot as plt\n", "import seaborn as sns\n", "from nltk.corpus import stopwords\n", "from nltk import word_tokenize, ngrams\n", "import codecs, difflib, Levenshtein\n", "import random\n", "import numpy as np\n", "import sys\n", "import random\n", "from datetime import *\n", "import numpy as np\n", "import os\n", "from sklearn.metrics import roc_curve\n", "import xgboost as xgb\n", "from sklearn.model_selection import train_test_split\n", "from sklearn.model_selection import GridSearchCV\n", "import pandas\n", "from sklearn.metrics import log_loss\n", "from sklearn.preprocessing import PolynomialFeatures\n", "import time\n", "import datetime\n", "import numpy as np\n", "import pandas as pd\n", "from sklearn.decomposition import PCA\n", "from sklearn.naive_bayes import GaussianNB\n", "from sklearn.linear_model import LogisticRegression, SGDClassifier\n", "from sklearn.ensemble import BaggingClassifier\n", "from sklearn.cross_validation import train_test_split\n", "from sklearn.pipeline import Pipeline\n", "from sklearn.metrics import roc_auc_score\n", "eng_stopwords = set(stopwords.words('english'))\n", "\n", "df_train = pd.read_csv('../input/train.csv', encoding=\"ISO-8859-1\")\n", "df_test = pd.read_csv('../input/test.csv', encoding=\"ISO-8859-1\")\n", "\n", "num_train = df_train.shape[0]\n", "print (num_train)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "2ab37e88-5b93-ff99-dbd0-73e286dd41ce" }, "outputs": [], "source": [ "def str_stem(str1):\n", " str1 = str(str1)\n", " str1 = re.sub(r'[^a-zA-Z0-9 ]',r'',str1)\n", " str1 = str1.lower()\n", " #str1 = (\" \").join([stemmer.stem(z) for z in str1.split(\" \")])\n", " return str1\n", "\n", "def str_common_word(str1, str2):\n", " str1, str2 = str1.lower(), str2.lower()\n", " words, cnt = str1.split(), 0\n", " for word in words:\n", " if str2.find(word)>=0:\n", " cnt+=1\n", " return cnt\n", "def ngram(tokens, n):\n", " grams =[tokens[i:i+n] for i in range(len(tokens)-(n-1))]\n", " return grams\n", "\n", "def get_sim(a_tri,b_tri):\n", " intersect = len(set(a_tri) & set(b_tri))\n", " union = len(set(a_tri) | set(b_tri))\n", " if union == 0:\n", " return 0\n", " return float(intersect)/(union)\n", "\n", "def jaccard_similarity(str1,str2):\n", " sentence_gram1 = str1\n", " sentence_gram2 = str2\n", " grams1 = ngram(sentence_gram1, 5)\n", " grams2 = ngram(sentence_gram2, 5)\n", " similarity = get_sim(grams1, grams2)\n", " return similarity\n", " \n", " \n", "df_all = pd.concat((df_train, df_test), axis=0, ignore_index=True)\n", "\n", "\n", "df_all['question1'] = df_all['question1'].map(lambda x:str_stem(x))\n", "df_all['question2'] = df_all['question2'].map(lambda x:str_stem(x))\n", "\n", "df_all['len_of_q1'] = df_all['question1'].map(lambda x:len(x.split())).astype(np.int64)\n", "df_all['len_of_q2'] = df_all['question2'].map(lambda x:len(x.split())).astype(np.int64)\n", "\n", "df_all['questions'] = df_all['question1']+\"|\"+df_all['question2']\n", "print (\"Questions combined...\")\n", "df_all['q2_in_q1'] = df_all['questions'].map(lambda x:str_common_word(x.split('|')[0],x.split('|')[1]))\n", "df_all['q1_in_q2'] = df_all['questions'].map(lambda x:str_common_word(x.split('|')[1],x.split('|')[0]))\n", "print (\"Common words found ...\")\n", "df_all['jaccard'] = df_all['questions'].map(lambda x:jaccard_similarity(x.split('|')[0],x.split('|')[1]))\n", "print (\"Jaccard similarities computed...\")\n", "#df_all['lev_distance'] = df_all['questions'].map(lambda x:normalized_damerau_levenshtein_distance(x.split('|')[0],x.split('|')[1]))\n", "#print (\"Levenshtein distances computed...\")\n", "\n", "df_all_orig=df_all.copy(deep=True)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "c7be7e9d-57fb-af97-395d-39f37490aaee" }, "outputs": [], "source": [ "df_all.head()" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "ee951743-270f-c81f-b8af-e0994f90655a" }, "outputs": [], "source": [ "\n", "df_all = df_all.drop(['id','qid1','qid2','question1','question2','questions'],axis=1)\n", "\n", "df_train = df_all.iloc[:num_train]\n", "df_test = df_all.iloc[num_train:]\n", "id_test = df_test['test_id']\n", "\n", "y_train = df_train['is_duplicate'].values\n", "X_train = df_train.drop(['test_id','is_duplicate'],axis=1).values\n", "X_test = df_test.drop(['test_id','is_duplicate'],axis=1).values\n", "\n", "from sklearn.cross_validation import train_test_split\n", "import xgboost as xgb\n", "\n", "X_df_train_SINGLE = X_train\n", "answers_1_SINGLE = list(y_train)\n", "\n", "#X_df_train_SINGLE = X_df_train_SINGLE.apply(lambda x: pandas.to_numeric(x, errors='ignore'))\n", "trainX, testX, trainY, testY = train_test_split(X_df_train_SINGLE, answers_1_SINGLE, test_size=.22) # CV\n", " \n", " \n", "xgbm = xgb.XGBClassifier(base_score=0.5, colsample_bytree=0.5,\n", " gamma=0.017, learning_rate=0.15, max_delta_step=0,\n", " max_depth=100, min_child_weight=3, n_estimators=500,\n", " nthread=-1, objective='binary:logistic', seed=0,\n", " silent=1, subsample=0.9)\n", "\n", "print ('Running:' + str(xgbm) + 'shape:' + str(X_df_train_SINGLE.shape)) \n", "\n", "model_train = xgbm.fit(trainX, trainY, early_stopping_rounds=100, \n", " eval_metric=\"logloss\",eval_set=[(testX, testY)], \n", " verbose=True)\n", "\n", "# print model_train\n", "predictions = xgbm.predict_proba(testX)[:, 1]\n", "\n", "print ('ROC AUC:' + str(roc_auc_score(testY, predictions)))\n", "print ('LOG LOSS:' + str(log_loss(testY, predictions)))" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "33309a0f-1360-7e87-e404-e5a4c702cd0a" }, "outputs": [], "source": [ "d_test = xgb.DMatrix(X_test)\n", "p_test = xgbm.predict(d_test)\n", "\n", "sub = pd.DataFrame()\n", "sub['test_id'] = np.int32(id_test)\n", "sub['is_duplicate'] = p_test\n", "sub.to_csv('simple_xgb.csv', index=False)" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "099177b4-e02e-5dd0-75ac-ba876d3a8a43" }, "outputs": [], "source": [ "" ] } ], "metadata": { "_change_revision": 0, "_is_fork": false, "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.0" } }, "nbformat": 4, "nbformat_minor": 0 }