{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4d54738896577b8d36a38782ab8d1581cac4bdf3"},"cell_type":"code","source":"fram = pd.read_csv(\"../input/train.csv\")\nprint(fram.columns)\nprint(fram.groupby('target').count())\n\nprint(1225312 / (1225312 + 80810))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"455ed4100f2274cba538fe595990d06631981239"},"cell_type":"code","source":"from nltk.tokenize import TreebankWordTokenizer\ndef tokenize(strq):\n    rt = TreebankWordTokenizer().tokenize(strq)\n    rtstr = ' '.join(rt)\n    return rt , rtstr\n\nprint(tokenize('How is this day?'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6126cc6465c5c9255df93b0b283b8736500d0412"},"cell_type":"code","source":"def makeVocab(datafram):\n    vocab0 = list()\n    vocab1 = list()\n    dict0 = dict()\n    dict1 = dict()\n    for _ , data in datafram.iterrows():\n        curTxt = data['question_text'].lower() \n        curLab = data['target']\n        \n        tok , curTxt = tokenize(curTxt)\n        #print(tok , curTxt )\n        if int(curLab) == 0:\n            vocab0.append(curTxt)\n            for ctk in tok:\n                if ctk not in dict0:\n                    dict0[ctk] = 0\n                dict0[ctk] = dict0[ctk] + 1 \n        elif int(curLab) == 1:\n            vocab1.append(curTxt)\n            for ctk in tok:\n                if ctk not in dict1:\n                    dict1[ctk] = 0\n                dict1[ctk] = dict1[ctk] + 1    \n    \n    busttrm = 300\n    for ky in dict0.keys():\n        if ky not in dict1:\n            dict0[ky] = dict0[ky] + busttrm \n        \n            \n    for ky in dict1.keys():\n        if ky not in dict0:\n            dict1[ky] = dict1[ky] + busttrm\n    \n    import operator\n    sorted_dict0 = sorted(dict0.items(), key=operator.itemgetter(1) , reverse = True)\n    sorted_dict1 = sorted(dict1.items(), key=operator.itemgetter(1), reverse = True)\n    return sorted_dict0 , sorted_dict1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f3b5bea6bf1351eb6305e15f88f55449e276be64"},"cell_type":"code","source":"sorted_dict0 , sorted_dict1 = makeVocab(fram)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"784130385cdd89049ad5126897bd1802af6f2c80"},"cell_type":"code","source":"finalVocab = set()\ncnt = 0\n\nfor wrd , _ in sorted_dict0:\n    if cnt == 40000:\n        break\n    if True and wrd not in finalVocab:\n        cnt = cnt + 1\n        finalVocab.add(wrd)\n        \ncnt = 0\nfor wrd,_ in sorted_dict1:\n    if cnt == 40000:\n        break;\n    if (True) and (wrd not in finalVocab):\n        cnt = cnt + 1\n        finalVocab.add(wrd)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bf54f6c120dadc654a5b81b5ca00a41932ed5b7b"},"cell_type":"code","source":"print('openai' in finalVocab)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3ffc2b8a04b6bed78c853d581e6466d6a10bd998"},"cell_type":"code","source":"from nltk.tokenize import TreebankWordTokenizer\ndef vocabTokenize(strq):\n    tr = list()\n    rt = TreebankWordTokenizer().tokenize(strq)\n    for trm in rt:\n        if trm in finalVocab:\n            tr.append(trm)\n    rtstr = ' '.join(tr)\n    return rtstr\n\nprint(vocabTokenize('How is this day?'))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e9d0b2468d9e9f122217a0846c898025b71d890e"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"05a321048065dd4c67fc57aa7c246f5c4c9be15f"},"cell_type":"code","source":"def makeKerasSequence(datafram):\n    vocab = list()\n    wordlist = datafram.question_text.str.lower()\n    mx = 0\n    for currow in wordlist:\n        cursen = vocabTokenize(currow)\n        if len(cursen) > mx:\n            mx = len(cursen)\n        vocab.append(cursen)\n    print(mx)  \n    return vocab","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5029ad07ee0337d8351ec253be0c45c3169d738f"},"cell_type":"code","source":"import operator\ndef printCntVal(vocab):\n    cntdict = dict()\n    for cur in vocab:\n        if len(cur) not in cntdict:\n            cntdict[len(cur)] = 0 \n        cntdict[len(cur)] = cntdict[len(cur)] + 1\n    \n    cntdict = sorted(cntdict.items(), key=operator.itemgetter(1) , reverse = True)\n    print(cntdict)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"945af3b59cb511eaec8819ded1b1bffd07786444"},"cell_type":"code","source":"from keras.models import Model , Sequential\nfrom keras.layers import Input , Flatten , Bidirectional ,Dropout, CuDNNGRU , GlobalMaxPool1D , Dense , MaxPooling1D , Conv1D , Embedding \nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.engine.input_layer import Input\nfrom keras.utils import to_categorical","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3daee47211f0b2449d1aad7bdba7192f3928f675"},"cell_type":"code","source":"def conv1DModel(vocab_size , outdim , maxln):\n    model = Sequential()\n    model.add(Embedding(vocab_size + 1, outdim))\n    model.add(Conv1D(128 , 5 ,activation='relu'))\n    model.add(MaxPooling1D(5))\n    model.add(Conv1D(128 , 5 , activation='relu'))\n    model.add(MaxPooling1D(5))\n    model.add(Conv1D(128 , 5 , activation='relu'))\n    model.add(MaxPooling1D(6))\n    model.add(Flatten())\n    model.add(Dense(128 , activation='relu'))\n    model.add(Dense(1 , activation='sigmoid'))\n    \n    model.compile(loss = 'binary_crossentropy',\n                 optimizer = 'rmsprop')\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5bb0e65ce1375eaa0239134ec6f33307d0b7fe20"},"cell_type":"code","source":"def conv1DModelFn(vocab_size , outdim , maxln):\n    inp = Input(shape = (maxln, ) , dtype='int32')\n    xt = Embedding(vocab_size + 1, outdim)(inp)\n    x = Conv1D(128, 5, activation='relu')(xt)\n    x = MaxPooling1D(3)(x)\n    x = Conv1D(128, 5, activation='relu')(x)\n    x = MaxPooling1D(3)(x)\n    x = Conv1D(128, 5, activation='relu')(x)\n    x = MaxPooling1D(5)(x)  # global max pooling\n    x = Flatten()(x)\n    x = Dense(128, activation='relu')(x)\n    prd = Dense(1, activation='sigmoid')(x)\n    \n    model = Model(inp , prd)\n    \n    model.compile(loss = 'binary_crossentropy',\n                 optimizer = 'rmsprop' , metrics = ['accuracy'])\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f88d5051fb023a06e1914bac181d7f6123d2e56e"},"cell_type":"code","source":"def lstmModel(vocab_size , outdim , maxln):\n    inp = Input(shape = (maxln, ) , dtype='int32')\n    xt = Embedding(vocab_size + 1, outdim)(inp)\n     \n    #xt = Embedding(vocab_size + 1, outdim , \n    #               weights=[rpMat],trainable = False)(inp)\n    #x = Dropout(.3)(xt)\n    x = Bidirectional(CuDNNGRU(64, return_sequences=True))(xt)\n    x = GlobalMaxPool1D()(x)\n    x = Dense(16, activation='relu')(x)\n    x = Dropout(.1)(x)\n    prd = Dense(1, activation='sigmoid')(x)\n    \n    model = Model(inp , prd)\n    \n    model.compile(loss = 'binary_crossentropy',\n                 optimizer = 'adam' , metrics = ['accuracy'])\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"51d7bd2100daac73b40f49ee1093c137204b0681"},"cell_type":"code","source":"def prepareDataSequence(tokenizer , dataList , maxLn):\n    dataSq = tokenizer.texts_to_sequences(dataList)\n    dataSq = pad_sequences(dataSq , maxLn)\n    wordInx = tokenizer.word_index\n    return dataSq , wordInx","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"810ab97e3f5bece4966f30fd14ef20361e4c4033"},"cell_type":"code","source":"lab = fram.target.values\n#lab = lab.reshape(lab.shape[0] , 1)\n#lab = to_categorical(lab , 2)\nprint(lab.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fc7049823320c742bf7bb138cee22b4baca6aba2"},"cell_type":"code","source":"txtStor = makeKerasSequence(fram)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4e378079569b14c69da729b696e490893845828f"},"cell_type":"code","source":"printCntVal(txtStor)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"424688bf47984c86dee42538a3e3cc2114d9d2e9"},"cell_type":"code","source":"vocab_size = 80000\nmaxLn = 100\noutdim = 300\ntokenizer = Tokenizer(vocab_size)\ntokenizer.fit_on_texts(txtStor)\n\n\n'''\nprint(wordInx)\nmodel = conv1DModel(vocab_size , outdim , maxLn)\nprint(model.summary)\n\nmodel.fit(dataSq , lab , epochs = 2)\n'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b53ae213c531671ebe23f03bb6b5fa87c4a2e83c"},"cell_type":"code","source":"dataSq , wordInx = prepareDataSequence(tokenizer , txtStor , maxLn)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"142d921894c9e1d922a5fcff6842ad97487b468f"},"cell_type":"code","source":"def posDataInx(lab):\n    inx_0 = list()\n    inx_1 = list()\n    cur = 0\n    for clb in lab:\n        if clb[0] == 1:\n            inx_0.append(cur)\n        elif clb[1] == 1:\n            inx_1.append(cur)\n        cur = cur + 1\n    return inx_0, inx_1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"00d067e9c19474cce2a9d0d74ac7df8a6970f484"},"cell_type":"code","source":"def genData(dataSq , lab):\n    inx_0 , inx_1 = posDataInx(lab)\n    np.random.shuffle(inx_0)\n    np.random.shuffle(inx_1)\n    train_x = list()\n    train_y = list()\n    test_x = list()\n    test_y = list()\n    \n    for tstCn in range(5000):\n        test_x.append(dataSq[inx_0[tstCn]])\n        test_y.append(lab[inx_0[tstCn]])\n        \n    for tstCn in range(5000):\n        test_x.append(dataSq[inx_1[tstCn]])\n        test_y.append(lab[inx_1[tstCn]])\n        \n        \n    for tstCn in range(5000 , 60000):\n        train_x.append(dataSq[inx_0[tstCn]])\n        train_y.append(lab[inx_0[tstCn]])\n        \n    for tstCn in range(5000 ,len(inx_1)):\n        train_x.append(dataSq[inx_1[tstCn]])\n        train_y.append(lab[inx_1[tstCn]])    \n    \n    return train_x , train_y ,  test_x , test_y\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"e5e6c3b26f058a5a93f7a550b31bd06eb9e1406a"},"cell_type":"code","source":"print(dataSq.shape)\nprint(txtStor[0])\nprint(wordInx['did'])\n\n#train_x , train_y , test_x , test_y = genData(dataSq , lab)\n\n\n#scor = model.evaluate(test_x, test_y, batch_size=256)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ee0984bdba608ce6b3f1da6c134f40638bdad06f","scrolled":true},"cell_type":"code","source":"model = lstmModel(len(wordInx) , outdim , maxLn)\nmodel.fit(dataSq , lab , epochs = 2 , batch_size = 512 , class_weight={0:.4 , 1:1})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c43a3bc9ca3bcc36d2bf403cbc317fafedb8f052"},"cell_type":"code","source":"testDataFram = pd.read_csv(\"../input/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"022374df40df8a69af58c03a2b9b9fa86ac56b82"},"cell_type":"code","source":"testdata = makeKerasSequence(testDataFram)\ntestDataSq , wordInx = prepareDataSequence(tokenizer , testdata , maxLn)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9aad59caba91feb313e4248dd50de1f748381525"},"cell_type":"code","source":"printCntVal(testdata)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0f0f911ac3d9e60639b1f692271d3903379e050b"},"cell_type":"code","source":"scor = model.predict(testDataSq, batch_size=256)\nprint(scor)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"db335f68984c5f1fb8d831e35e5a2b499119b722"},"cell_type":"code","source":"#yout = np.argmax(scor , axis = 1)\nyout = (scor > .5).astype(int)#yout.reshape(testDataFram.shape[0])\nyout = yout.reshape(yout.shape[0])\nprint(yout)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f980760bcd9e1dc9c173d56a1e283f7cc7da0b79"},"cell_type":"code","source":"\n#yout = np.ones((testDataFram.shape[0] , ) , dtype = 'int32')\n#yout = np.random.randint(2, size=testDataFram.shape[0])\nprint(testDataFram['qid'])\nprint(yout)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6de045fc39e025b1368e59b4c4dce8476c8ce22f"},"cell_type":"code","source":"pred_df = pd.DataFrame({'qid':testDataFram['qid'],'prediction':yout}, columns=[\"qid\" , \"prediction\"])\n#print(pred_df)\npred_df.to_csv(\"submission.csv\" , index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"65c162243192834d8a07dc83c70fd1b662f36dc6"},"cell_type":"code","source":"pred_df.groupby('prediction').count()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3d5f6d675c77e1fb58086d8d9f7094e864c92619"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}