{"cells":[{"metadata":{"_cell_guid":"a29d5f6e-9fd0-4a09-af21-746de9257ffc","_uuid":"c3a6593a32441ca844a7075d12e40b8406765dcb"},"cell_type":"markdown","source":"# Coding Log\n---\n## Start with Linear Regression \n    - variable - region, parent_category_name, category_name, user_type >> 0.2472 \n\n## Try LightGBM\n    - variable - region, parent_category_name, category_name, user_type >> 0.2412 \n    \n#### Hard to keep all matrix due to memory limitation\n#### So.....\n\n## Change Model to NN (feat. Keras)\n|  Model        | Variables                            | Val_loss  | LB |\n| ------------- |:-------------------------------- | -----:| -----:|\n|||||\n| Linear Reg | region, pcn, cn, ut  | Unknown | 0.2472 |\n| LightGBM | region, pcn, cn, ut  | 0.2351 | 0.2412 |\n|||||\n| NN      | region | 0.2588 | 0.3032 | \n| NN      | region, pcn      | 0.2475 | 0.2537 |\n| NN      | region, pcn, cn    |  0.2430 |  0.2488 |\n| NN      | region, pcn, cn, ut    | 0.2407 |  0.2459 | \n| NN      | region, pcn, cn, ut, city   |   0.2402 |  Unkonwn |\n| NN      | region, pcn, cn, ut, city, price  |    0.2370  | Unkonwn |\n| NN      | region, pcn, cn, ut, city, price, p1, p2, p3  |    0.2268  | 0.2320 |\n| NN      | region, pcn, cn, ut, city, price, p1, p2, p3, itemseq  |    0.2255  | 0.2304 |\n\n***\n***"},{"metadata":{"_cell_guid":"48c52923-464a-4664-b9b4-66021826d1d9","_uuid":"d32d260f9f826e8a94762adc952d105c47ece99c"},"cell_type":"markdown","source":"# Import  basic"},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","collapsed":true,"trusted":false},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport gc\nimport os\nprint(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"eaae9bc9-a72d-433f-a177-d95df02e4785","_uuid":"1353606773406222b81dcc991c85696a3b7302b3"},"cell_type":"markdown","source":"# Data Load"},{"metadata":{"scrolled":false,"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","collapsed":true,"trusted":false},"cell_type":"code","source":"df_train = pd.read_csv('../input/train.csv')\ndf_test = pd.read_csv('../input/test.csv')\n\ndf_y_train = df_train['deal_probability']\ndf_x_train = df_train.drop(['deal_probability'], axis=1)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"34d30103-7b60-49c1-a3f5-40a1dd56598c","_uuid":"7519fb76b564cda52e56b7ac6f6c6932826c6691","collapsed":true,"trusted":false},"cell_type":"code","source":"df_x_train.head(10)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"53b3bb60-487f-4e25-872c-050bb0a17845","_uuid":"4558541b14e15dabfa88ee129822f1bf53926bb0","collapsed":true,"trusted":false},"cell_type":"code","source":"df_x_train.info()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"d1391a20-2eac-4d97-a3dd-a5612a9580b3","_uuid":"0a1b77cd1a5d2f36765ce37dda8f70770dc2a51e","collapsed":true,"trusted":false},"cell_type":"code","source":"df_test.info()","execution_count":null,"outputs":[]},{"metadata":{"scrolled":false,"_cell_guid":"b58b8e82-c855-4b5e-a940-f273981df267","_uuid":"4f31d12083169e0ab67bcbf3f368bc26b6a7f479","collapsed":true,"trusted":false},"cell_type":"code","source":"# viewing # of unique value in each column \nfor col in df_x_train.columns:\n    print(col, len(df_x_train[col].unique()))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"b58151ec-07cb-46aa-9947-6846cd11fb07","_uuid":"e39a1f24aa68c81517786e4df447f2119ec8796e"},"cell_type":"markdown","source":"## Variable Plan\n\n    item_id 1503424              => drop\n    user_id 771769               => drop\n    region 28                    => (added)categorical embedding\n    city 1733                    => (added)categorical embedding\n    parent_category_name 9       => (added)categorical embedding\n    category_name 47             => (added)categorical embedding\n    param_1 372                  => (added)categorical embedding\n    param_2 272                  => (added)categorical embedding\n    param_3 1220                 => (added)categorical embedding \n    title 788377                 => deal with nlp\n    description 1317103          => (added)word embedding, LSTM\n    price 17007                  => (added)continuous, log\n    item_seq_number 28232        => continuous, log\n    activation_date 21           => (added)weekday, categorical\n    user_type 3                  => (added)categorical embedding\n    image 1390837                => image_feature (VGG, Resnet, inception) ***\n    image_top_1 3063             => (added)categorical"},{"metadata":{"_cell_guid":"5badf52d-f945-4b67-89d5-1c5a9a6f702e","_uuid":"5e010e9190a8de22282de646389c2429cef62182"},"cell_type":"markdown","source":"# Fill NA"},{"metadata":{"_cell_guid":"7b077114-1eb6-48a1-ad6c-fd75234a35c8","_uuid":"b2e69dbf36eafbb5b9b3967a508d8a6e3820f20e","collapsed":true,"trusted":false},"cell_type":"code","source":"df_x_train['image_top_1'].fillna(value=3067, inplace=True)\ndf_test['image_top_1'].fillna(value=3067, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"ad2be95e-4c42-4fe3-812b-60009284967b","_uuid":"f31899cd9496fd9d124d3100604746451cb88bd2","collapsed":true,"trusted":false},"cell_type":"code","source":"df_x_train['param_1'].fillna(value='_NA_', inplace=True)\ndf_test['param_1'].fillna(value='_NA_', inplace=True)\n\ndf_x_train['param_2'].fillna(value='_NA_', inplace=True)\ndf_test['param_2'].fillna(value='_NA_', inplace=True)\n\ndf_x_train['param_3'].fillna(value='_NA_', inplace=True)\ndf_test['param_3'].fillna(value='_NA_', inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"e81e9660-e704-49c6-9a92-be0ad5699389","_uuid":"ae4a3c669c84c9840f3cbf033311e50c447a99af","collapsed":true,"trusted":false},"cell_type":"code","source":"df_x_train['description'].fillna(value='_NA_', inplace=True)\ndf_test['description'].fillna(value='_NA_', inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"b2ef16d8-737c-4a54-bc05-dd7199dc4201","_uuid":"e695990fbefb069d248fde92a9cac56407629175"},"cell_type":"markdown","source":"# Preprocess"},{"metadata":{"_cell_guid":"8e9f8877-4e4a-4977-bb8b-3e1fb07055a6","_uuid":"841180183c42ce5aa95026c8f8afec3fc513ca99","collapsed":true,"trusted":false},"cell_type":"code","source":"from keras.preprocessing.text import Tokenizer\nfrom keras.utils import to_categorical\nimport argparse\n\n#create config init\nconfig = argparse.Namespace()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"eab706e3-c174-4f5d-a3fb-e4064b0baa23","_uuid":"55413b00e659a1321f1b16a97ffac1f2651a134c"},"cell_type":"markdown","source":"### Categorical Variables - region, pcn, cn, ut, city, p1, p2, p3, week, imgt1"},{"metadata":{"_cell_guid":"65a712fd-5897-4067-9a91-ecb017c72e6d","_uuid":"b1353289b55198ff1726e6f13057ec2c76b3f670","collapsed":true,"trusted":false},"cell_type":"code","source":"def tknzr_fit(col, df_trn, df_test):\n    tknzr = Tokenizer(filters='', lower=False, split='뷁', oov_token='oov' )\n    tknzr.fit_on_texts(df_trn[col])\n    return np.array(tknzr.texts_to_sequences(df_trn[col])), np.array(tknzr.texts_to_sequences(df_test[col])), tknzr","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"_cell_guid":"3b4818ae-98f3-411e-912c-da16144af252","_uuid":"3f53a78d9b223cf3020666aeafe2c5408da57ec4","collapsed":true,"trusted":false},"cell_type":"code","source":"tr_reg, te_reg, tknzr_reg = tknzr_fit('region', df_x_train, df_test)\ntr_pcn, te_pcn, tknzr_pcn = tknzr_fit('parent_category_name', df_x_train, df_test)\ntr_cn, te_cn, tknzr_cn = tknzr_fit('category_name', df_x_train, df_test)\ntr_ut, te_ut, tknzr_ut = tknzr_fit('user_type', df_x_train, df_test)\ntr_city, te_city, tknzr_city = tknzr_fit('city', df_x_train, df_test)\n\ntr_p1, te_p1, tknzr_p1 = tknzr_fit('param_1', df_x_train, df_test)\ntr_p2, te_p2, tknzr_p2 = tknzr_fit('param_2', df_x_train, df_test)\ntr_p3, te_p3, tknzr_p3 = tknzr_fit('param_3', df_x_train, df_test)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"520f0ea0-f3b1-4c5c-aee3-678a18d67526","_uuid":"d228ff2b548bde291075162be60048801b9724b4","collapsed":true,"trusted":false},"cell_type":"code","source":"tr_week = pd.to_datetime(df_x_train['activation_date']).dt.weekday.astype(np.int32).values\nte_week = pd.to_datetime(df_test['activation_date']).dt.weekday.astype(np.int32).values\ntr_week = np.expand_dims(tr_week, axis=-1)\nte_week = np.expand_dims(te_week, axis=-1)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"2adbebc9-efca-4e6f-b768-775ebda8e1ba","_uuid":"c83fd0d2ff22933164472789d5e6faf1891c9c06","collapsed":true,"trusted":false},"cell_type":"code","source":"tr_imgt1 = df_x_train['image_top_1'].astype(np.int32).values\nte_imgt1 = df_test['image_top_1'].astype(np.int32).values\ntr_imgt1 = np.expand_dims(tr_imgt1, axis=-1)\nte_imgt1 = np.expand_dims(te_imgt1, axis=-1)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"3791a4c1-425d-4b11-966d-05dd25185015","_uuid":"31efa367dd424e1b7bad7d05dc2d6682251f95f2"},"cell_type":"markdown","source":"### Continuous Variables"},{"metadata":{"_cell_guid":"0a65c473-2b54-4647-8561-2d22efc45e32","_uuid":"b5469cdbd9cb1e0edfb0002d16459b565c3f9140","collapsed":true,"trusted":false},"cell_type":"code","source":"eps = 1e-10\ntr_price = np.log(df_x_train['price']+eps)\nte_price = np.log(df_test['price']+eps)\ntr_price[tr_price.isna()] = -1.\nte_price[te_price.isna()] = -1.\n\ntr_price = np.expand_dims(tr_price, axis=-1)\nte_price = np.expand_dims(te_price, axis=-1)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"89011c19-9e10-4179-bb23-830ab8f1647c","_uuid":"fb17dddcb17351952a22dfdb8601de8a4aa792db","collapsed":true,"trusted":false},"cell_type":"code","source":"tr_itemseq = np.log(df_x_train['item_seq_number'])\nte_itemseq = np.log(df_test['item_seq_number'])\n# price_tr[price_tr.isna()] = -1.\n# price_te[price_te.isna()] = -1.\n\ntr_itemseq = np.expand_dims(tr_itemseq, axis=-1)\nte_itemseq = np.expand_dims(te_itemseq, axis=-1)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"422b3d0d-05b8-4cf3-9b61-95af172a8167","_uuid":"57540565977a60260b34355d9456e97ba4ce445f"},"cell_type":"markdown","source":"## Text Variable"},{"metadata":{"_cell_guid":"7b5bf168-68da-4f50-bd27-cc1e7d58c718","_uuid":"d6c7e7dd59566eb843eb2fa56a3241de77841227"},"cell_type":"markdown","source":"## keras_Tokenizer"},{"metadata":{"_cell_guid":"e1615772-eb25-4d5e-83e9-62bf5c280a1d","_uuid":"d054d41f5e7a4d6cf4bcefb3d8cb04be736d29df","collapsed":true,"trusted":false},"cell_type":"code","source":"config.len_desc = 100000","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"740ad94c-84b5-4e5e-892a-7b1e75a73d6c","_uuid":"431a91ae41e7faef0387ec4775f583897baa927e","collapsed":true,"trusted":false},"cell_type":"code","source":"from keras.preprocessing.sequence import pad_sequences","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"57287768-ede6-44b0-ab94-20cbb0bc9b0d","_uuid":"62c44b53a0375fee6b60f6142e33d065ca961872","collapsed":true,"trusted":false},"cell_type":"code","source":"tknzr_desc = Tokenizer(num_words=config.len_desc, lower='True')","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"b7bdb044-6450-41b1-b1bb-5047ee19e66b","_uuid":"e8ee7166b48c6bb1efa9eaca10b4ea98e08c544c","collapsed":true,"trusted":false},"cell_type":"code","source":"tknzr_desc.fit_on_texts(df_x_train['description'].values)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"cccfa4c6-2a53-403f-b82d-8555323c7bcc","_uuid":"825c131a586146bc3accda37134f23532594418b","collapsed":true,"trusted":false},"cell_type":"code","source":"tr_desc_seq = tknzr_desc.texts_to_sequences(df_x_train['description'].values)\nte_desc_seq = tknzr_desc.texts_to_sequences(df_test['description'].values)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"77437010-ba87-418d-96e9-a778d2001c90","_uuid":"5405922baf1e522f178a797beb48b9de6534278d","collapsed":true,"trusted":false},"cell_type":"code","source":"config.maxlen= 75","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"0482299c-e5c4-4ef1-ad98-d471417448f5","_uuid":"2e76db6a8d9bcfdeec58f009e99cdd5293f67c41","collapsed":true,"trusted":false},"cell_type":"code","source":"tr_desc_pad = pad_sequences(tr_desc_seq, maxlen=config.maxlen)\nte_desc_pad = pad_sequences(te_desc_seq, maxlen=config.maxlen)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"6354c9d3-97d6-4c6f-8eee-1956bd8d881d","_uuid":"fa71b9c2db5cdfe6a4973f4617d97cb841d14d12","collapsed":true,"trusted":false},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"c64c57ed-baff-45c1-8a04-da649bf9e14b","_uuid":"fcee45a22e9a39bce0e086645e2d21b5dadc78d1"},"cell_type":"markdown","source":"### Configuration"},{"metadata":{"_cell_guid":"a825d2f0-7476-4986-a251-4ddabe42aac9","_uuid":"e29d6ff492ada7407a3974d4abde794e4452d619","collapsed":true,"trusted":false},"cell_type":"code","source":"## categorical\nconfig.len_reg = len(tknzr_reg.word_index)\nconfig.len_pcn = len(tknzr_pcn.word_index)\nconfig.len_cn = len(tknzr_cn.word_index) \nconfig.len_ut = len(tknzr_ut.word_index)\nconfig.len_city = len(tknzr_city.word_index) +1\nconfig.len_week = 7\nconfig.len_imgt1 = int(df_x_train['image_top_1'].max())+1\nconfig.len_p1 = len(tknzr_p1.word_index)+1\nconfig.len_p2 = len(tknzr_p2.word_index)+1\nconfig.len_p3 = len(tknzr_p3.word_index)+1\n\n## continuous\nconfig.len_price = 1\nconfig.len_itemseq = 1\n\n#text\n# config.len_desc = len(tknzr_desc.word_index)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"70bfea20-1a16-49c4-8fe9-6058b3f49bd7","_uuid":"6cf195149b03af4ebb84a3b849520ab7cfc500cd","collapsed":true,"trusted":false},"cell_type":"code","source":"## categorical\nconfig.emb_reg = 8\nconfig.emb_pcn = 4\nconfig.emb_cn = 8\nconfig.emb_ut = 2\nconfig.emb_city = 16\nconfig.emb_week = 4\nconfig.emb_imgt1 = 16\nconfig.emb_p1 = 8\nconfig.emb_p2 = 16\nconfig.emb_p3 = 16\n\n#continuous\nconfig.emb_price = 16\nconfig.emb_itemseq = 16\n\n#text\nconfig.emb_desc = 100","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"409a9be3-01d3-4453-9a06-79b9b99f0d7c","_uuid":"595df98bafbd8340d3e8d2f715ff164170490ca8","collapsed":true,"trusted":false},"cell_type":"code","source":"valid_idx = df_y_train.sample(frac=0.2, random_state=1991).index\ntrain_idx = df_y_train[np.invert(df_y_train.index.isin(valid_idx))].index","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"fd0a03f1-a357-4fe3-bee5-ae08bf9699b4","_uuid":"cb1f6c3dffe81a6bbce795e72b70cf24e0cb7658","collapsed":true,"trusted":false},"cell_type":"code","source":"X = np.array([tr_reg, tr_pcn, tr_cn, tr_ut, tr_city, tr_week, tr_imgt1, tr_p1, tr_p2, tr_p3,\n              tr_price, tr_itemseq])\nX_test = np.array([te_reg, te_pcn, te_cn, te_ut, te_city, te_week, te_imgt1, te_p1, te_p2, te_p3,\n                   te_price, te_itemseq])\nY = df_y_train","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"c7b6bb1f-c1ff-4282-90ae-dc0a1bb7003a","_uuid":"c32c9cd92eb06490eeb303c872967aba6cdc4021","collapsed":true,"trusted":false},"cell_type":"code","source":"X_train = [x[train_idx] for x in X]\nX_valid = [x[valid_idx] for x in X]\nX_test = [x for x in X_test]\n\nY_train = Y[train_idx]\nY_valid = Y[valid_idx]","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"1e1db0cf-aa4d-4e93-a6bd-e00e88165252","_uuid":"59981ab6bb73eb1dd5d4a441c3f5bca240ac8c76","collapsed":true,"trusted":false},"cell_type":"code","source":"X_train.append(tr_desc_pad[train_idx])\nX_valid.append(tr_desc_pad[valid_idx])\nX_test.append(te_desc_pad)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"12ad97c9-d83c-46be-af4c-e367046c6598","_uuid":"5b60346095ebe8b377ecd8f5d0f2f96d3656a0b4","collapsed":true,"trusted":false},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"8dc175d0-5a9e-4339-b3ea-62d276a77a01","_uuid":"e946b112fec39acfedc6828fde3bb29a005c0e05"},"cell_type":"markdown","source":"# Keras Model & RMSE Loss"},{"metadata":{"_cell_guid":"4f476e70-bc9b-461b-b8cb-27013292ca82","_uuid":"408b5c2b4768f6b5535ebfc01a49b2dc338b361b","collapsed":true,"trusted":false},"cell_type":"code","source":"from keras.layers import Input, Embedding, Dense\nfrom keras.layers import GlobalMaxPool1D, GlobalMaxPool2D\nfrom keras.layers import concatenate\nfrom keras.layers import LSTM, CuDNNGRU, CuDNNLSTM, GRU\nfrom keras.models import Model\n\nfrom keras import backend as K\n\nfrom keras.optimizers import RMSprop, Adam\nfrom keras.callbacks import ModelCheckpoint, EarlyStopping\n\n### rmse loss for keras\ndef root_mean_squared_error(y_true, y_pred):\n    return K.sqrt(K.mean(K.square(y_pred - y_true))) ","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"5e4e5c4a-f902-472c-834d-4db66d4d47b5","_uuid":"2d7b88b2dfd0c422b9e1417d85e59bc1511fd03d","collapsed":true,"trusted":false},"cell_type":"code","source":"config.batch_size = 4096","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"8a7ed6f4-20ff-47fc-9aa2-840f094df7c1","_uuid":"3eabb145c56176b7c40d2e4a98a0db7c382ffc6a","collapsed":true,"trusted":false},"cell_type":"code","source":"def get_model():\n    K.clear_session()\n    inp_reg = Input(shape=(1, ), name='inp_region')\n    emb_reg = Embedding(config.len_reg, config.emb_reg, name='emb_region')(inp_reg)\n    \n    inp_pcn = Input(shape=(1, ), name='inp_parent_category_name')\n    emb_pcn = Embedding(config.len_pcn, config.emb_pcn, name='emb_parent_category_name')(inp_pcn)\n\n    inp_cn = Input(shape=(1, ), name='inp_category_name')\n    emb_cn = Embedding(config.len_cn, config.emb_cn, name=\"emb_category_name\" )(inp_cn)\n    \n    inp_ut = Input(shape=(1, ), name='inp_user_type')\n    emb_ut = Embedding(config.len_ut, config.emb_ut, name='emb_user_type' )(inp_ut)\n    \n    inp_city = Input(shape=(1, ), name='inp_city')\n    emb_city = Embedding(config.len_city, config.emb_city, name='emb_city' )(inp_city)\n\n    inp_week = Input(shape=(1, ), name='inp_week')\n    emb_week = Embedding(config.len_week, config.emb_week, name='emb_week' )(inp_week)\n\n    inp_imgt1 = Input(shape=(1, ), name='inp_imgt1')\n    emb_imgt1 = Embedding(config.len_imgt1, config.emb_imgt1, name='emb_imgt1')(inp_imgt1)\n    \n    inp_p1 = Input(shape=(1, ), name='inp_p1')\n    emb_p1 = Embedding(config.len_p1, config.emb_p1, name='emb_p1')(inp_p1)\n    \n    inp_p2 = Input(shape=(1, ), name='inp_p2')\n    emb_p2 = Embedding(config.len_p2, config.emb_p2, name='emb_p2')(inp_p2)\n    \n    inp_p3 = Input(shape=(1, ), name='inp_p3')\n    emb_p3 = Embedding(config.len_p3, config.emb_p3, name='emb_p3')(inp_p3)\n    \n    conc_cate = concatenate([emb_reg, emb_pcn,  emb_cn, emb_ut, emb_city, emb_week, emb_imgt1, emb_p1, emb_p2, emb_p3], axis=-1, name='concat_categorcal_vars')\n    conc_cate = GlobalMaxPool1D()(conc_cate)\n    \n    inp_price = Input(shape=(1, ), name='inp_price')\n    emb_price = Dense(config.emb_price, activation='tanh', name='emb_price')(inp_price)\n\n    inp_itemseq = Input(shape=(1, ), name='inp_itemseq')\n    emb_itemseq = Dense(config.emb_itemseq, activation='tanh', name='emb_itemseq')(inp_itemseq)\n    \n    conc_cont = concatenate([conc_cate, emb_price, emb_itemseq], axis=-1)\n    x = Dense(200, activation='relu')(conc_cont)\n    x = Dense(50, activation='relu')(x)\n\n    ### text\n    inp_desc = Input(shape=(config.maxlen, ), name='inp_desc')\n    emb_desc = Embedding(config.len_desc, config.emb_desc, name='emb_desc')(inp_desc)\n    \n    desc_layer = GRU(40, return_sequences=False)(emb_desc)\n    \n    conc_desc = concatenate([x, desc_layer], axis=-1)\n    ###\n\n    outp = Dense(1, activation='sigmoid', name='output')(conc_desc)\n\n    model = Model(inputs = [inp_reg, inp_pcn, inp_cn, inp_ut, inp_city, inp_week, inp_imgt1, inp_p1, inp_p2, inp_p3,\n                            inp_price, inp_itemseq, inp_desc], outputs = outp)\n    return model","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"_cell_guid":"c9fde9f1-9d76-4694-a3f5-0920141dfff9","_uuid":"1080f57fea3f92360d43aac0275ad02deaabf208","collapsed":true,"trusted":false},"cell_type":"code","source":"model = get_model()\nmodel.compile(optimizer='adam', loss = root_mean_squared_error, metrics=[root_mean_squared_error])\n# model.compile(optimizer=RMSprop(lr=0.0005, decay=0.00001), loss = root_mean_squared_error, metrics=['mse', root_mean_squared_error])\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"7aafb282-d489-4e19-82ac-4c77b47c4cd6","_uuid":"acb940740b5e9bb7a8f5401808be04230fff7735","collapsed":true,"trusted":false},"cell_type":"code","source":"### callbacks\ncheckpoint = ModelCheckpoint('best.hdf5', monitor='val_loss', verbose=1, save_best_only=True)\nearly = EarlyStopping(patience=3, mode='min')","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"7615ebcb-65aa-40b9-b628-025fd34b95ac","_uuid":"36cc1e8b0c90b9ea430afae17828b387e6522f9f","trusted":false,"collapsed":true},"cell_type":"code","source":"model.fit(x=X_train, y=np.array(Y_train), validation_data=(X_valid, Y_valid), batch_size=config.batch_size, epochs=6, callbacks=[checkpoint,early], verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"fdf57697-27b7-42e1-b620-d03afd737489","_uuid":"ba95dd5930060b90fd8fb7ac007ff3ddefcc591e"},"cell_type":"markdown","source":"# Model Visualization"},{"metadata":{"_cell_guid":"14d89fdc-c27c-4008-906a-999c40b8a02c","_uuid":"fb9e88ff42d1b8e67c4e7072c5fafac2aa1bfae6","collapsed":true,"trusted":false},"cell_type":"code","source":"from IPython.display import SVG\nfrom keras.utils.vis_utils import model_to_dot\n\nSVG(model_to_dot(model).create(prog='dot', format='svg'))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"b45907dd-0b43-4e88-b194-2b795ee80143","_uuid":"f9dfb3a8a3415e23fa0992e7ed223f946fc7ffd6"},"cell_type":"markdown","source":"# Test & submit"},{"metadata":{"_cell_guid":"2621d285-b76a-441f-9ca8-5e6a226178f3","_uuid":"00ef5e5626fec032d21009b71e68f53919a58e16","collapsed":true,"trusted":false},"cell_type":"code","source":"model.load_weights('best.hdf5')\npred = model.predict(X_test)\n\nsubm = pd.read_csv(\"../input/sample_submission.csv\")\nsubm['deal_probability'] = pred\nsubm.to_csv('submit_{}_{:.4f}.csv'.format('nn_p3', 0.226), index=False)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"b0a123c7-5f2c-40b5-8b98-e607bf28091f","_uuid":"21161d9f98c1460710cb5b1c867799742e660ab8"},"cell_type":"markdown","source":"# To do Next\n\n    ***** adding all variables\n    ***** embedding nlp for title & description\n    \n    **** hyper-parameters tuning\n    **** prevent overfitting (dropout, BN, etc...)\n    \n    *** image-feature adding (vgg? inception? resnet?)\n    *** K-fold validation (5 ~ 10)\n    \n    ** Ensemble with other model (lgbm)\n    ** Staking on Ensemble (OOF)\n    \n    * find partners "},{"metadata":{"_cell_guid":"11200214-3e3d-4cd3-a426-5058e543b712","_uuid":"a422b66b1ab824d7a5372f93d432c840545e01ec","collapsed":true,"trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.5","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}