{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport keras\nimport pickle\n\nfrom collections import Counter\nnonalphanums = ''.join(c for c in map(chr, range(256)) if not c.isalnum())\nnonalphanums = nonalphanums.translate(str.maketrans('','',' '))\nraw = pd.read_csv(\"../input/train.csv\")\nquestiondata = np.array(raw.iloc[:,1])\ny = np.array(raw.iloc[:,2])\nappendation = \" \".join(questiondata).lower()\ntranslator = str.maketrans('', '', nonalphanums)\nappendation = appendation.translate(translator)\nwordcollection = appendation.split(\" \")\nocc = dict(Counter(wordcollection))\nwordocc =sorted(occ, key=occ.get, reverse=True)\ntopnumber = 1001\ntopocc = (wordocc[:topnumber-1])\nprint(topocc)\ntopocc.insert(0,\"<pad>\")\nprint(topocc)\ndef dechunker(chunked):\n    return np.array(chunked.split(\" \"))\ndechunked = np.array(list(map(dechunker,questiondata)))\ndef cleanlink(assortment):\n    returns = []\n    for i in assortment:\n        cleansed = i.lower().translate(translator)\n        if (cleansed in topocc):\n            returns.append(topocc.index(cleansed))\n    return np.array(returns)\ncleanlinked = np.array(list(map(cleanlink,dechunked)))\nlongestlinked = max(list(map(len,cleanlinked)))\nX = keras.preprocessing.sequence.pad_sequences(cleanlinked,\n                                                        value=0,\n                                                        padding='post',\n                                                        maxlen=longestlinked)\nY = y\n","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","collapsed":true,"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"code","source":"import pickle\nimport numpy as np\nimport pandas as pd\nfrom sklearn.model_selection import train_test_split\nimport keras\nfrom keras import Sequential\nfrom keras.layers import Embedding, GlobalAveragePooling1D, Dense, Dropout,LSTM","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3d2cf1d95c7b2499bdf279b59623817a09315c7a"},"cell_type":"code","source":"x,y = X,Y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ac8ba5a1da769b923087399e523e14807521be63"},"cell_type":"code","source":"x_train,x_test,y_train,y_test = train_test_split(x, y, test_size=0.2)\nepochs = 30\nbsize = 100","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b991ba4413e6956cfad35e62f086301c7ab279a9"},"cell_type":"code","source":"model = Sequential()\nmodel.add(Embedding(topnumber, 16))\nmodel.add(LSTM(16))\nmodel.add(Dense(16, activation=\"relu\"))\nmodel.add(Dropout(0.3))\nmodel.add(Dense(1, activation=\"sigmoid\"))\nmodel.summary()\nmodel.compile(optimizer='adam',\n              loss='binary_crossentropy',\n              metrics=['accuracy'])\nmodel.fit(x_train, y_train, epochs=epochs, batch_size=bsize, validation_data=(x_test, y_test), verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ed319e58955de501920147d3d509ca7dc6a062e3"},"cell_type":"code","source":"nonalphanums = ''.join(c for c in map(chr, range(256)) if not c.isalnum())\nnonalphanums = nonalphanums.translate(str.maketrans('','',' '))\nraw = pd.read_csv(\"../input/test.csv\")\nquestiondata = np.array(raw.iloc[:,1])\ny = np.array(raw.iloc[:,0])\nappendation = \" \".join(questiondata).lower()\ntranslator = str.maketrans('', '', nonalphanums)\nappendation = appendation.translate(translator)\nwordcollection = appendation.split(\" \")\nocc = dict(Counter(wordcollection))\nwordocc =sorted(occ, key=occ.get, reverse=True)\ntopnumber = 1000\ntopocc = (wordocc[:topnumber])\nprint(topocc)\ntopocc.insert(0,\"<pad>\")\nprint(topocc)\ndef dechunker(chunked):\n    return np.array(chunked.split(\" \"))\ndechunked = np.array(list(map(dechunker,questiondata)))\ndef cleanlink(assortment):\n    returns = []\n    for i in assortment:\n        cleansed = i.lower().translate(translator)\n        if (cleansed in topocc):\n            returns.append(topocc.index(cleansed))\n    return np.array(returns)\ncleanlinked = np.array(list(map(cleanlink,dechunked)))\nlongestlinked = max(list(map(len,cleanlinked)))\nx = keras.preprocessing.sequence.pad_sequences(cleanlinked,\n                                                        value=0,\n                                                        padding='post',\n                                                        maxlen=longestlinked)\ntest_x = x\nids = y\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"047d7d67c8c7be1355f2b6889bb8b7ead77e3da6"},"cell_type":"code","source":"results = model.predict(test_x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dd070a306e11049a4d013285a0993e3baaf0a470"},"cell_type":"code","source":"#package for submission\nimport csv\nfinal = []\ng = ['qid','prediction']\nfor it in range(0,len(results)):\n  tx = int(not (results[it]>np.mean(results)))\n  appendition = [ids[it],tx]\n  final.append(appendition)\nfinal = np.array(final)\nprint(final)\nar = final\nwith open('submission.csv', 'w+') as fp:\n    writer = csv.writer(fp, quoting=csv.QUOTE_NONNUMERIC)\n    writer.writerow(i for i in g)\n    writer.writerows(ar.tolist())","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}