{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom nltk.tokenize import word_tokenize\nimport re\nimport random\n#from gensim.models import word2vec\nimport csv\nimport tensorflow as tf\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.layers import Dense, Dropout, Embedding,LSTM, CuDNNLSTM ,ZeroPadding2D, Conv1D, MaxPooling1D\nfrom keras.models import Sequential, Model\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.models import model_from_json\nfrom tqdm import tqdm,tqdm_notebook \nimport spacy\nfrom keras.models import load_model\nimport h5py\n\ntrain = pd.read_csv(\"../input/train.csv\")\ntest = pd.read_csv(\"../input/test.csv\")\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The source of the punct string, dictionaries mispell_dict and contraction_mapping:\nhttps://www.kaggle.com/theoviel/improve-your-score-with-some-text-preprocessing"},{"metadata":{"trusted":true},"cell_type":"code","source":"punct = \"/-'?!.,#$%\\'()*+-/:;<=>@[\\\\]^_`{|}~\" + '\"\"“”’' + '∞θ÷α•à−β∅³π‘₹´°£€\\×™√²—–&'\nmispell_dict = {'colour': 'color', 'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling', 'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor', 'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize', 'youtu ': 'youtube ', 'Qoura': 'Quora', 'sallary': 'salary', 'Whta': 'What', 'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do', 'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does', 'mastrubation': 'masturbation', 'mastrubate': 'masturbate', \"mastrubating\": 'masturbating', 'pennis': 'penis', 'Etherium': 'Ethereum', 'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017', '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', \"whst\": 'what', 'watsapp': 'whatsapp', 'demonitisation': 'demonetization', 'demonitization': 'demonetization', 'demonetisation': 'demonetization'}\ncontraction_mapping = {\"ain't\": \"is not\", \"aren't\": \"are not\",\"can't\": \"cannot\", \"'cause\": \"because\", \"could've\": \"could have\", \"couldn't\": \"could not\", \"didn't\": \"did not\",  \"doesn't\": \"does not\", \"don't\": \"do not\", \"hadn't\": \"had not\", \"hasn't\": \"has not\", \"haven't\": \"have not\", \"he'd\": \"he would\",\"he'll\": \"he will\", \"he's\": \"he is\", \"how'd\": \"how did\", \"how'd'y\": \"how do you\", \"how'll\": \"how will\", \"how's\": \"how is\",  \"I'd\": \"I would\", \"I'd've\": \"I would have\", \"I'll\": \"I will\", \"I'll've\": \"I will have\",\"I'm\": \"I am\", \"I've\": \"I have\", \"i'd\": \"i would\", \"i'd've\": \"i would have\", \"i'll\": \"i will\",  \"i'll've\": \"i will have\",\"i'm\": \"i am\", \"i've\": \"i have\", \"isn't\": \"is not\", \"it'd\": \"it would\", \"it'd've\": \"it would have\", \"it'll\": \"it will\", \"it'll've\": \"it will have\",\"it's\": \"it is\", \"let's\": \"let us\", \"ma'am\": \"madam\", \"mayn't\": \"may not\", \"might've\": \"might have\",\"mightn't\": \"might not\",\"mightn't've\": \"might not have\", \"must've\": \"must have\", \"mustn't\": \"must not\", \"mustn't've\": \"must not have\", \"needn't\": \"need not\", \"needn't've\": \"need not have\",\"o'clock\": \"of the clock\", \"oughtn't\": \"ought not\", \"oughtn't've\": \"ought not have\", \"shan't\": \"shall not\", \"sha'n't\": \"shall not\", \"shan't've\": \"shall not have\", \"she'd\": \"she would\", \"she'd've\": \"she would have\", \"she'll\": \"she will\", \"she'll've\": \"she will have\", \"she's\": \"she is\", \"should've\": \"should have\", \"shouldn't\": \"should not\", \"shouldn't've\": \"should not have\", \"so've\": \"so have\",\"so's\": \"so as\", \"this's\": \"this is\",\"that'd\": \"that would\", \"that'd've\": \"that would have\", \"that's\": \"that is\", \"there'd\": \"there would\", \"there'd've\": \"there would have\", \"there's\": \"there is\", \"here's\": \"here is\",\"they'd\": \"they would\", \"they'd've\": \"they would have\", \"they'll\": \"they will\", \"they'll've\": \"they will have\", \"they're\": \"they are\", \"they've\": \"they have\", \"to've\": \"to have\", \"wasn't\": \"was not\", \"we'd\": \"we would\", \"we'd've\": \"we would have\", \"we'll\": \"we will\", \"we'll've\": \"we will have\", \"we're\": \"we are\", \"we've\": \"we have\", \"weren't\": \"were not\", \"what'll\": \"what will\", \"what'll've\": \"what will have\", \"what're\": \"what are\",  \"what's\": \"what is\", \"what've\": \"what have\", \"when's\": \"when is\", \"when've\": \"when have\", \"where'd\": \"where did\", \"where's\": \"where is\", \"where've\": \"where have\", \"who'll\": \"who will\", \"who'll've\": \"who will have\", \"who's\": \"who is\", \"who've\": \"who have\", \"why's\": \"why is\", \"why've\": \"why have\", \"will've\": \"will have\", \"won't\": \"will not\", \"won't've\": \"will not have\", \"would've\": \"would have\", \"wouldn't\": \"would not\", \"wouldn't've\": \"would not have\", \"y'all\": \"you all\", \"y'all'd\": \"you all would\",\"y'all'd've\": \"you all would have\",\"y'all're\": \"you all are\",\"y'all've\": \"you all have\",\"you'd\": \"you would\", \"you'd've\": \"you would have\", \"you'll\": \"you will\", \"you'll've\": \"you will have\", \"you're\": \"you are\", \"you've\": \"you have\" }\nspell=dict(mispell_dict)\nspell.update(contraction_mapping)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":""},{"metadata":{"trusted":true},"cell_type":"code","source":"question_list=[]\n\nfor index,row in tqdm_notebook(train.iterrows()):\n    question_list.append([row['question_text'],int(row['target'])])\n    \n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":""},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"def preproc(words):\n    newwords=[]\n    for word in words:\n        punc=0\n        for p in punct:\n            if word==p:\n                punc=1\n        if punc==0:\n            word=re.sub('[0-9]{1,}','#',word)\n            for mispelling in spell.keys():\n                word=word.replace(mispelling,spell[mispelling])\n            newwords.append(word)\n    \n    return newwords\n\n      \n\ndef vectorize(text,t):\n    tokenized_questions=[]\n    for item in tqdm_notebook(text):\n        if t==0:\n            i=item[0]\n        else:\n            i=item\n           \n        i=word_tokenize(i)\n        i=preproc(i)\n        i=' '.join(i)\n        #i=glove(i).vector\n        \n        if t==0:\n            tokenized_questions.append([i,item[1]])\n        else:\n            tokenized_questions.append(i)\n    \n        \n    return tokenized_questions\n\n\ntrain_data=vectorize(question_list,0)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"random.shuffle(train_data)\ndef folds(k):\n    m=len(train_data)//5\n    if(k==0):\n        test=train_data[0:m]\n        train=train_data[m:5*m]\n    else:\n        test=train_data[m*k:(k+1)*m]\n        train=train_data[0:m*k] + train_data[(k+1)*m:5*m]\n   \n    return test,train\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_samples,train_samples=folds(2)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"https://towardsdatascience.com/cnn-application-on-structured-data-automated-feature-extraction-8f2cd28d9a7e"},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndef build_vocab(text):\n    vocabulary={}\n    ttl_docs=0\n    for question in tqdm_notebook(text):\n        ttl_docs+=1\n        q=word_tokenize(question[0])\n        temp={}\n        for word in q:\n            \n            try:\n                temp[word]\n            except:\n                try:\n                    vocabulary[word] +=1\n                except KeyError:\n                    vocabulary[word] =1\n    return ttl_docs,vocabulary\n\n\ntotal,vocab=build_vocab(train_data)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import operator\nl=sorted(vocab.items(), key=operator.itemgetter(1),reverse=True)\nfeat_vocab=l[:50000]\n\nfeat_vocab=dict(feat_vocab)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"glove_index={}\nfile='../input/embeddings/glove.840B.300d/glove.840B.300d.txt'\nfound=0\nf=open(file)\nfor line in tqdm_notebook(f):\n    components=line.split()\n    word=components[0]\n    vector=np.asarray(components[1:])\n    if len(vector)<301:\n        try:\n            feat_vocab[word]\n            found+=1\n            glove_index[word]=vector\n        except KeyError:\n            pass\n    \nf.close()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from math import log10\ndef tf_idf(question):\n    words={}\n    for word in question:\n        try:\n            words[word] +=1\n        except KeyError:\n            words[word] =1\n    \n    vec=np.zeros(300, dtype='float64')\n    for item in words.keys():\n        try:\n            weight= words[item]*log10(total/feat_vocab[item])\n           \n            vec=np.add(vec,glove_index[item].astype(np.float)*weight)\n        except KeyError:\n            pass\n        \n    return vec\n\nx_train=[]\ny_train=[]\n\nfor row in tqdm_notebook(train_samples):\n    x_train.append(tf_idf(row[0]))\n    y_train.append(row[1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del train_samples,train ,train_data,question_list","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_array=np.vstack(x_train)\nx_array = np.expand_dims(x_array, axis=2)\n\n\"\"\"y_array=np.array(y_train\"\"\"\ny_array=np.zeros((len(y_train),2))\n\nfor i in range(len(y_train)):\n    if y_train[i]==0:\n        y_array[i]=np.array([1,0])\n    else:\n        y_array[i]=np.array([0,1])\n   \n\nxs=[]\ny_val=[]\n\n\nfor row in tqdm_notebook(test_samples):\n    \n    xs.append(tf_idf(row[0]))\n    y_val.append(row[1]) \n\n\nx_validate=np.vstack(xs)\nx_validate = np.expand_dims(x_validate, axis=2)\n\n\"\"\"y_validate=np.array(y_val)\"\"\"\n\ny_validate=np.zeros((len(y_val),2))\n\nfor i in range (len(y_val)):\n    if y_val[i]==0:\n        y_validate[i]=np.array([1,0])\n    else:\n        y_validate[i]=np.array([0,1])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del test_samples","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.layers import Flatten\ncnn=Sequential()\n\ncnn.add(Conv1D(16, 3, activation='relu',input_shape=(x_array.shape[1],1)))\ncnn.add(Conv1D(16, 3, activation='relu'))\ncnn.add(Conv1D(16, 3, activation='relu'))\ncnn.add(MaxPooling1D(pool_size=2))\ncnn.add(Flatten())\ncnn.add(Dense(128,activation='relu'))\nlayer=Dense(100, activation='relu')\ncnn.add(layer)\ncnn.add(Dense(2, activation='softmax'))\ncnn.compile(loss='binary_crossentropy',\n              optimizer='adam',\n              metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cnn.fit(x_array,y_array,epochs=5,batch_size=128,validation_data=(x_validate,y_validate))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cnn.save(\"cnn.h5\")","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"feature_layer = Model(inputs=cnn.input,outputs=layer.output)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nbatch_size=100000\nrng=len(x_array)//batch_size + 1\ndef predict(array,batch,units):\n    for m in range(units):\n        start=m*batch\n        end=(m+1)*batch\n        if m==0:\n            new_features=feature_layer.predict(array[:end])\n        elif m==units :\n            new_features=np.concatenate((new_features,feature_layer.predict(array[start:])))\n        else:\n            new_features=np.concatenate((new_features,feature_layer.predict(array[start:end])))\n    \n    return new_features","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del vocab\ndel x_train,y_train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"features_fit=predict(x_array,batch_size,rng)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del x_array","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"iters=len(x_validate)//batch_size + 1\nfeatures_validate=predict(x_validate,batch_size,iters)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import xgboost as xgb\n\ndtrain=xgb.DMatrix(features_fit, label=y_array)\ndvalidate=xgb.DMatrix(features_validate,label=y_validate)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del features_fit , features_validate","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n#for i in range(1,6):\nprmtrs = {'objective': 'binary:logistic',\n          'max_delta_step':1,\n          'max_depth':10,\n          'min_child_weight':8,\n          'scale_pos_weigh':1,\n          'subsample':0.55,\n          'reg_alpha':0.001,\n          'learning_rate':0.1\n           }\nprmtrs['eval_metric'] = 'auc'\ndataset = [(dvalidate, 'eval'), (dtrain, 'train')]\n\nepochs=10\nxgbmodel=xgb.train(prmtrs,dtrain,epochs,dataset)\nprint(\"----------------------------------------\")\nprint('\\n')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del dtrain,dvalidate","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib\nxgb.plot_importance(xgbmodel)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"xgbmodel.save_model('xbgm.model')\nxgbmodel.dump_model('dump.raw.txt')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"import xgboost as xgb\nloaded_model = load_model('xgbm.model')\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import gc\n\ngc.collect()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def results(test_samples):\n    qid=[]\n    questions=[]\n    res={}\n    for index,row in test_samples.iterrows():\n        qid.append(row[0])\n        questions.append(row[1])\n    questions=vectorize(questions,1)\n    glove=[tf_idf(q) for q in questions]\n    questions=np.vstack(glove)\n    questions = np.expand_dims(questions, axis=2)\n    it=len(questions)//batch_size + 1\n    features=predict(questions,batch_size,it)\n    data=xgb.DMatrix(features)\n    predictions=xgbmodel.predict(data)\n    \n\n    for m,ids in tqdm_notebook(enumerate(qid)):\n        res[ids]=predictions[m]\n    \n    return res\n\nresults_dict=results(test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del test","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":false,"trusted":true},"cell_type":"code","source":"\ndef writeOutput(results):\n    header = [\"qid\", \"prediction\"]\n    output_file=open(\"submission.csv\", \"w\")\n    writer = csv.DictWriter(output_file,fieldnames=header)\n    writer.writeheader()\n    \n    m=0\n    k=0\n    \n    for item in results.keys():\n        if results[item]>0.501:\n            value=1\n            k+=1\n        else:\n            value=0\n            m+=1\n        ro={\"qid\":item,\"prediction\":value}\n        writer.writerow(ro)\n    print(k)\n    print(k/len(results))\n    print(m)\n    print(m/len(results))\n    \n    output_file.close() \n    \n\nwriteOutput(results_dict)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(len(results_dict))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.4"}},"nbformat":4,"nbformat_minor":1}