{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom tqdm import tqdm\ntqdm.pandas()\n%matplotlib inline\n\nfrom plotly import tools\nimport plotly.offline as py\npy.init_notebook_mode(connected=True)\nimport plotly.graph_objs as go\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\nimport math\nfrom sklearn.model_selection import train_test_split\nfrom sklearn import metrics\nimport gensim.models.keyedvectors as word2vec\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.layers import Dense, Input, LSTM, Embedding, Dropout, Activation, CuDNNGRU, Conv1D, CuDNNLSTM, concatenate\nfrom keras.layers import Bidirectional, GlobalMaxPool1D, Dropout, SpatialDropout1D, GlobalAveragePooling1D, GlobalMaxPooling1D\nfrom keras.models import Model\nfrom keras import initializers, regularizers, constraints, optimizers, layers\nimport os\nfrom xgboost import XGBClassifier\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nfrom sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\nfrom nltk.corpus import stopwords\nfrom scipy.sparse import coo_matrix\nfrom nltk.stem.porter import *\n\nfrom nltk.stem import WordNetLemmatizer, SnowballStemmer\nstopWords = set(stopwords.words('english'))\n","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"df = pd.read_csv('../input/train.csv')\ntest = pd.read_csv('../input/test.csv') ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c88f464ba5ab092e271ff07b27b6ff45a5f10dd6"},"cell_type":"code","source":"import re\nstemmer = SnowballStemmer(language='english')\ndef clean_text(x):\n    x = str(x).lower()\n    for punct in \"/-'\":\n        x = x.replace(punct, ' ')\n    for punct in '&':\n        x = x.replace(punct, f' {punct} ')\n    for punct in '?!.,\"#$%\\'()*+-/:;<=>@[\\\\]^_`{|}~' + '“”’':\n        x = x.replace(punct, '')\n    return x\ndef clean_numbers(x):\n    x = re.sub('[0-9]{5,}', '#####', x)\n    x = re.sub('[0-9]{4}', '####', x)\n    x = re.sub('[0-9]{3}', '###', x)\n    x = re.sub('[0-9]{2}',\"##\",x)\n    return x\n\ndef lemma_stema(sentence):\n    new_sent = \"\"\n    for word in sentence.split(\" \"):\n        new_sent = new_sent + \" \" + stemmer.stem(WordNetLemmatizer().lemmatize(word, pos='v'))\n    new_sent = new_sent.strip(\" \")\n    return new_sent\ndef remove_stopwords(sentence):\n    new_sent = \"\"\n    for word in sentence.split(\" \"):\n        if word not in stopWords:\n            new_sent = new_sent + \" \" + word\n    new_sent = new_sent.strip(\" \")\n    return new_sent","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"571c61d3e692398f645d77eeb484762ebbed78b9"},"cell_type":"code","source":"df[\"question_text\"] = df[\"question_text\"].progress_apply(lambda x: clean_numbers(x))\ntest[\"question_text\"] = test[\"question_text\"].progress_apply(lambda x: clean_numbers(x))\n\ndf[\"question_text\"] = df[\"question_text\"].progress_apply(lambda x: clean_text(x))\ntest[\"question_text\"] = test[\"question_text\"].progress_apply(lambda x: clean_text(x))\n\ndf[\"question_text\"] = df[\"question_text\"].progress_apply(lambda x: lemma_stema(x))\ntest[\"question_text\"] = test[\"question_text\"].progress_apply(lambda x: lemma_stema(x))\n\ndf[\"question_text\"] = df[\"question_text\"].progress_apply(lambda x: remove_stopwords(x))\ntest[\"question_text\"] = test[\"question_text\"].progress_apply(lambda x: remove_stopwords(x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c1918981807639cbb9b7a75d352da0204f23682b"},"cell_type":"code","source":"from sklearn.feature_extraction.text import TfidfVectorizer\ntfvecor = TfidfVectorizer(strip_accents='unicode', ngram_range=(1,2))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7699b261a65eba72dbdde2086d491b3699fb3b55"},"cell_type":"code","source":"tfvecor.fit(df['question_text'].append(test['question_text']))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f70f46771337b1abc1522a893b78fa9a50dcd671"},"cell_type":"code","source":"dfx = tfvecor.transform(df['question_text'])\ntestx = tfvecor.transform(test['question_text'])\n# len(tfvecor.get_feature_names())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a996666cde3a085ff9d78badb468d05bd9423268"},"cell_type":"code","source":"dfx.shape, testx.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d15c45c3973d56b2f246da947985d26c3c33f8f0"},"cell_type":"code","source":"# from sklearn.model_selection import train_test_split\n# # from sklearn.feature_extraction.text import TfidfVectorizer,CountVectorizer\n\n\n# X_train, X_test, y_train, y_test = train_test_split(dfx,\n#                                                     df['target'],\n#                                                     test_size=0.2)\ndef downsample(df):\n    print(df.shape)\n    np.random.seed=42\n    dfzero = df[df['target']==0]\n    dfone = df[df['target']==1]\n    zeroind = np.random.choice(dfzero.index,100000, replace=False)\n    dfzerodown = dfzero.loc[zeroind, : ]\n    dffinal = pd.concat([dfzerodown, dfone], axis=0)\n    return dffinal","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0c5b8a46eb815b6fdda9a5437ad864a0e6f60e50"},"cell_type":"code","source":"def get_imp_feature(df):\n    from sklearn.linear_model import LogisticRegression\n    dfx = tfvecor.transform(df['question_text'])\n    clf=LogisticRegression(C=0.76,multi_class='multinomial',penalty='l1', solver='saga',n_jobs=-1)\n    clf.fit(dfx, df['target'])\n    feature_imp = clf.coef_.nonzero()[1]\n#     dfimp = dfx[: ,feature_imp]\n#     dfimp.todense()\n    # clf.fit(X_train, y_train)\n    # #clf = MultinomialNB().fit(X_train, y_train)\n    # predicted = clf.predict(X_test)\n    return feature_imp","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"33907a469ff21d44dc112a9e63d15c3f94a4ec77"},"cell_type":"code","source":"df_for_ft= downsample(df)\n# df_for_ft_x = tfvecor.transform(df_for_ft['question_text'])\n\nfeature_imp = get_imp_feature(df_for_ft)\n# get the important feature from the df\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b9c1c0064e090805701331da3af887b9257a7386"},"cell_type":"code","source":"len(feature_imp)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d6aeb8b7e91a5b9ea21d0cbff960c83d98462dd3"},"cell_type":"code","source":"dfimp = dfx[: ,feature_imp]\ntestimp = testx[:, feature_imp]\n# dfimp.todense()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cac7e4a3cda4a92fc2aecda1cd81a70597b9e4ce"},"cell_type":"code","source":"from lightgbm import LGBMClassifier\nmodel = LGBMClassifier(max_depth=20, num_leaves=50, learning_rate=0.1, n_jobs=-1, n_estimators=500, feature_fraction= 0.05,\n                       bagging_fraction=0.8)\nmodel.fit(dfimp, df['target'])\n#predcv = model.predict(X_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0ceaeda8a3679b9e72eb75f3d4a57dfc6e1e10ed"},"cell_type":"code","source":"predicted_test = model.predict(testimp)\n\ntest['prediction'] = predicted_test\nsubmission = test.drop(columns=['question_text'])\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"06faf022ad2ee9731aa7d1c9aba2b159214566e8"},"cell_type":"code","source":"submission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}