{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","collapsed":true,"trusted":true},"cell_type":"code","source":"import pandas as pd \nimport numpy as np \nimport time \nimport gc ","execution_count":1,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"np.random.seed(42)\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import roc_auc_score\n\nfrom keras.models import Model\nfrom keras.layers import Input, Dropout, Dense, Embedding, SpatialDropout1D, concatenate\nfrom keras.layers import GRU, Bidirectional, GlobalAveragePooling1D, GlobalMaxPooling1D, BatchNormalization, Conv1D, MaxPooling1D, Flatten\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.preprocessing import text, sequence\nfrom keras.callbacks import Callback\nfrom keras import backend as K\nfrom keras.models import Model\n\nfrom sklearn.preprocessing import LabelBinarizer\nfrom sklearn.preprocessing import LabelEncoder, MinMaxScaler, StandardScaler\n\nimport warnings\nwarnings.filterwarnings('ignore')\n\nimport os\nos.environ['OMP_NUM_THREADS'] = '4'\n\nimport threading\nimport multiprocessing\nfrom multiprocessing import Pool, cpu_count\nfrom contextlib import closing\ncores = 4\n\nfrom keras import backend as K\nfrom keras.optimizers import RMSprop, Adam\nfrom keras.callbacks import ModelCheckpoint, EarlyStopping\n\n### rmse loss for keras\ndef root_mean_squared_error(y_true, y_pred):\n    return K.sqrt(K.mean(K.square(y_true - y_pred))) ","execution_count":2,"outputs":[]},{"metadata":{"_cell_guid":"f29ac0de-f5c8-4b83-8925-c0c41faab355","_uuid":"5ac680da23b4055f75a1b0454808308ef3ccb924"},"cell_type":"markdown","source":"# Choosing a Sequence length \n#train['title_description']= (train['title']+\" \"+train['description']).astype(str)\n#train['length'] = train['title_description'].apply(lambda x: len(x.split(\" \")))\nprint(train['length'].mean())\nprint(train[train['length']>50].shape)\nmax_seq_title_description_length = 100"},{"metadata":{"_cell_guid":"6d1a6395-fb0c-4f97-9d07-46e3f764f389","_uuid":"fe99b725c4f55ef4cf471e685107536cd428b98d","collapsed":true,"trusted":true},"cell_type":"code","source":"def preprocess_dataset(dataset):\n    \n    t1 = time.time()\n    print(\"Filling Missing Values.....\")\n    dataset['price'] = dataset['price'].fillna(0).astype('float32')\n    dataset['param_1'].fillna(value='missing', inplace=True)\n    dataset['param_2'].fillna(value='missing', inplace=True)\n    dataset['param_3'].fillna(value='missing', inplace=True)\n    \n    dataset['param_1'] = dataset['param_1'].astype(str)\n    dataset['param_2'] = dataset['param_2'].astype(str)\n    dataset['param_3'] = dataset['param_3'].astype(str)\n    \n    print(\"Casting data types to type Category.......\")\n    dataset['category_name'] = dataset['category_name'].astype('category')\n    dataset['parent_category_name'] = dataset['parent_category_name'].astype('category')\n    dataset['region'] = dataset['region'].astype('category')\n    dataset['city'] = dataset['city'].astype('category')\n\n    print(\"Creating New Feature.....\")\n    dataset['param123'] = (dataset['param_1']+'_'+dataset['param_2']+'_'+dataset['param_3']).astype(str)\n    del dataset['param_2'], dataset['param_3']\n    gc.collect()\n        \n    print(\"PreProcessing Function completed.\")\n    \n    return dataset\n\ndef keras_fit(train):\n    \n    t1 = time.time()\n    train['title_description']= (train['title']+\" \"+train['description']).astype(str)\n    del train['description'], train['title']\n    gc.collect()\n    \n    print(\"Start Tokenization.....\")\n    tokenizer = text.Tokenizer(num_words = max_words_title_description)\n    all_text = np.hstack([train['title_description'].str.lower()])\n    tokenizer.fit_on_texts(all_text)\n    del all_text\n    del train['activation_date']\n    gc.collect()\n    \n    print(\"Loading Test for Label Encoding on Train + Test\")\n    use_cols_test = ['region', 'city', 'parent_category_name', 'category_name', 'param_1', 'param_2', 'param_3']\n    test = pd.read_csv(\"../input/avito-demand-prediction/test.csv\", usecols = use_cols_test)\n    \n    test['param_1'].fillna(value='missing', inplace=True)\n    test['param_1'] = test['param_1'].astype(str)\n    test['param_2'].fillna(value='missing', inplace=True)\n    test['param_2'] = test['param_2'].astype(str)\n    test['param_3'].fillna(value='missing', inplace=True)\n    test['param_3'] = test['param_3'].astype(str)\n\n    print(\"Creating New Feature.....\")\n    test['param123'] = (test['param_1']+'_'+test['param_2']+'_'+test['param_3']).astype(str)\n    del test['param_2'], test['param_3']\n    gc.collect()\n    \n    ntrain = train.shape[0]\n    DF = pd.concat([train, test], axis = 0)\n    del train, test\n    gc.collect()\n    print(DF.shape)\n    \n    print(\"Start Label Encoding process....\")\n    le_region = LabelEncoder()\n    le_region.fit(DF.region)\n    \n    le_city = LabelEncoder()\n    le_city.fit(DF.city)\n    \n    le_category_name = LabelEncoder()\n    le_category_name.fit(DF.category_name)\n    \n    le_parent_category_name = LabelEncoder()\n    le_parent_category_name.fit(DF.parent_category_name)\n    \n    le_param_1 = LabelEncoder()\n    le_param_1.fit(DF.param_1)\n    \n    le_param123 = LabelEncoder()\n    le_param123.fit(DF.param123)\n\n    train = DF[0:ntrain]\n    test = DF[ntrain:]\n    del DF \n    gc.collect()\n    \n    train['price'] = np.log1p(train['price'])\n    train['item_seq_number'] = np.log(train['item_seq_number'])\n    print(\"Fit on Train Function completed.\")\n    \n    return train, tokenizer, le_region, le_city, le_category_name, le_parent_category_name, le_param_1, le_param123\n\ndef keras_train_transform(dataset):\n    \n    t1 = time.time()\n    \n    dataset['seq_title_description']= tokenizer.texts_to_sequences(dataset.title_description.str.lower())\n    print(\"Transform done for test\")\n    print(\"Time taken for Sequence Tokens is\"+str(time.time()-t1))\n    del train['title_description']\n    gc.collect()\n\n    dataset['region'] = le_region.transform(dataset['region'])\n    dataset['city'] = le_city.transform(dataset['city'])\n    dataset['category_name'] = le_category_name.transform(dataset['category_name'])\n    dataset['parent_category_name'] = le_parent_category_name.transform(dataset['parent_category_name'])\n    dataset['param_1'] = le_param_1.transform(dataset['param_1'])\n    dataset['param123'] = le_param123.transform(dataset['param123'])\n    \n    print(\"Transform on test function completed.\")\n    \n    return dataset\n    \ndef keras_test_transform(dataset):\n    \n    t1 = time.time()\n    dataset['title_description']= (dataset['title']+\" \"+dataset['description']).astype(str)\n    del dataset['description'], dataset['title']\n    gc.collect()\n    \n    dataset['seq_title_description']= tokenizer.texts_to_sequences(dataset.title_description.str.lower())\n    print(\"Transform done for test\")\n    print(\"Time taken for Sequence Tokens is\"+str(time.time()-t1))\n    \n    del dataset['activation_date'], dataset['title_description']\n    gc.collect()\n\n    dataset['region'] = le_region.transform(dataset['region'])\n    dataset['city'] = le_city.transform(dataset['city'])\n    dataset['category_name'] = le_category_name.transform(dataset['category_name'])\n    dataset['parent_category_name'] = le_parent_category_name.transform(dataset['parent_category_name'])\n    dataset['param_1'] = le_param_1.transform(dataset['param_1'])\n    dataset['param123'] = le_param123.transform(dataset['param123'])\n    \n    dataset['price'] = np.log1p(dataset['price'])\n    dataset['item_seq_number'] = np.log(dataset['item_seq_number'])\n    \n    print(\"Transform on test function completed.\")\n    \n    return dataset\n    \ndef get_keras_data(dataset):\n    X = {\n        'seq_title_description': pad_sequences(dataset.seq_title_description, maxlen=max_seq_title_description_length)\n        ,'region': np.array(dataset.region)\n        ,'city': np.array(dataset.city)\n        ,'category_name': np.array(dataset.category_name)\n        ,'parent_category_name': np.array(dataset.parent_category_name)\n        ,'param_1': np.array(dataset.param_1)\n        ,'param123': np.array(dataset.param123)\n        ,'price': np.array(dataset[[\"price\"]])\n        ,'item_seq_number': np.array(dataset[[\"item_seq_number\"]])\n    }\n    \n    print(\"Data ready for Vectorization\")\n    \n    return X","execution_count":3,"outputs":[]},{"metadata":{"_cell_guid":"eb0f1322-66ef-4b66-b100-1084481f19fb","_uuid":"3bb2f824ccf9d59d6eb228708d5de36c7b975e0e","collapsed":true},"cell_type":"markdown","source":"# Understanding Number of Tokens \n# What should be the value of num_words in Tokenizer \n\nL = tokenizer.word_counts  ##len(L) = 748124\nprint(len(L))\nfrom collections import OrderedDict \nL1 = [ value for (key, value) in sorted(L.items(), reverse=True)]\nL1.sort(reverse = True)\nK = L1[0:200000]"},{"metadata":{"_cell_guid":"d44a4688-a22f-469c-84eb-09dd7106f888","_uuid":"5447f21f8b2898960ed09e5332c7222a6b18c59f","trusted":true},"cell_type":"code","source":"# Loading Train data - No Params, No Image data \ndtypes_train = {\n                'price': 'float32',\n                'deal probability': 'float32',\n                'item_seq_number': 'uint32'\n}\n\n# No user_id\nuse_cols = ['item_id', 'region', 'city', 'parent_category_name', 'category_name', 'param_1', 'param_2', 'param_3', 'title', 'description', 'price', 'item_seq_number', 'activation_date', 'deal_probability']\ntrain = pd.read_csv(\"../input/avito-demand-prediction/train.csv\", parse_dates=[\"activation_date\"], usecols = use_cols, dtype = dtypes_train)\n\ny_train = train['deal_probability']\ndel train['deal_probability']\ngc.collect()\n\nmax_seq_title_description_length = 100\nmax_words_title_description = 200000\n\ntrain = preprocess_dataset(train)\ntrain, tokenizer, le_region, le_city, le_category_name, le_parent_category_name, le_param_1, le_param123 = keras_fit(train)\ntrain = keras_train_transform(train)\nprint(\"Tokenization done and TRAIN READY FOR Validation splitting\")\n\n# Calculation of max values for Categorical fields \n        \nmax_region = np.max(train.region.max())+1\nmax_city= np.max(train.city.max())+1\nmax_category_name = np.max(train.category_name.max())+1\nmax_parent_category_name = np.max(train.parent_category_name.max())+1\nmax_param_1 = np.max(train.param_1.max())+1\nmax_param123 = np.max(train.param123.max())+1\n    \nprint(\"Train Test Split\")\nx_train_f, x_valid_f, y_train_f, y_valid_f = train_test_split(train, y_train, train_size=0.95, random_state=233)\nprint(x_train_f.shape, x_valid_f.shape)\nprint(y_train_f.shape, y_valid_f.shape)\n\ndel train, y_train\ngc.collect()\n\nX_train = get_keras_data(x_train_f)\nX_valid = get_keras_data(x_valid_f)\ndel x_train_f, x_valid_f\ngc.collect()\n","execution_count":4,"outputs":[]},{"metadata":{"_cell_guid":"8b07a2cb-4208-4a18-98c7-e60a717b5cd9","_uuid":"323f100f2ab9ac36f586eba48ccec7a2f0a784f2","trusted":true},"cell_type":"code","source":"# EMBEDDINGS COMBINATION \n# FASTTEXT\n\nEMBEDDING_DIM1 = 300\nEMBEDDING_FILE1 = '../input/fasttest-common-crawl-russian/cc.ru.300.vec'\ndef get_coefs(word, *arr): return word, np.asarray(arr, dtype='float32')\nembeddings_index1 = dict(get_coefs(*o.rstrip().rsplit(' ')) for o in open(EMBEDDING_FILE1))\n\nvocab_size = len(tokenizer.word_index)+1\nEMBEDDING_DIM1 = 300# this is from the pretrained vectors\nembedding_matrix1 = np.zeros((vocab_size, EMBEDDING_DIM1))\nprint(embedding_matrix1.shape)\n# Creating Embedding matrix \nc = 0 \nc1 = 0 \nw_Y = []\nw_No = []\nfor word, i in tokenizer.word_index.items():\n    if word in embeddings_index1:\n        c +=1\n        embedding_vector = embeddings_index1[word]\n        w_Y.append(word)\n    else:\n        embedding_vector = None\n        w_No.append(word)\n        c1 +=1\n    if embedding_vector is not None:    \n        embedding_matrix1[i] = embedding_vector\n\nprint(c,c1, len(w_No), len(w_Y))\nprint(embedding_matrix1.shape)\ndel embeddings_index1\ngc.collect()\n\nprint(\" FAST TEXT DONE\")\n","execution_count":5,"outputs":[]},{"metadata":{"_cell_guid":"3d75792e-16b1-424c-8f67-20f7ffe618f3","_uuid":"06e0676f0d5bb1b96a185e6276d8b79d86b925a6"},"cell_type":"markdown","source":"# RUSSIAN GLOVE\n\nEMBEDDING_DIM2 = 300\nEMBEDDING_FILE2 = '../input/russian-glove/multilingual_embeddings.ru'\ndef get_coefs(word, *arr): return word, np.asarray(arr, dtype='float32')\nembeddings_index2 = dict(get_coefs(*o.rstrip().rsplit(' ')) for o in open(EMBEDDING_FILE2))\n\nvocab_size = len(tokenizer.word_index)+1\nEMBEDDING_DIM2 = 300# this is from the pretrained vectors\nembedding_matrix2 = np.zeros((vocab_size, EMBEDDING_DIM2))\nprint(embedding_matrix2.shape)\n# Creating Embedding matrix \nc = 0 \nc1 = 0 \nw_Y = []\nw_No = []\nfor word, i in tokenizer.word_index.items():\n    if word in embeddings_index2:\n        c +=1\n        embedding_vector = embeddings_index2[word]\n        w_Y.append(word)\n    else:\n        embedding_vector = None\n        w_No.append(word)\n        c1 +=1\n    if embedding_vector is not None:    \n        embedding_matrix2[i] = embedding_vector\n\nprint(c,c1, len(w_No), len(w_Y))\nprint(embedding_matrix2.shape)\ndel embeddings_index2\ngc.collect()\n\n# COMBINATION \n\nembedding_matrix_final = np.concatenate([embedding_matrix1, embedding_matrix2], axis = 1)\ndel embedding_matrix1, embedding_matrix2\ngc.collect()"},{"metadata":{"_cell_guid":"c0a3837a-e7ca-4618-a672-ba40210d005f","_uuid":"e74c68e6105ce951262e9f50b7da365660e6c59e","collapsed":true,"trusted":true},"cell_type":"code","source":"def RNN_model():\n\n    #Inputs\n    seq_title_description = Input(shape=[X_train[\"seq_title_description\"].shape[1]], name=\"seq_title_description\")\n    region = Input(shape=[1], name=\"region\")\n    city = Input(shape=[1], name=\"city\")\n    category_name = Input(shape=[1], name=\"category_name\")\n    parent_category_name = Input(shape=[1], name=\"parent_category_name\")\n    param_1 = Input(shape=[1], name=\"param_1\")\n    param123 = Input(shape=[1], name=\"param123\")\n    price = Input(shape=[1], name=\"price\")\n    item_seq_number = Input(shape = [1], name = 'item_seq_number')\n    \n    #Embeddings layers\n\n    emb_seq_title_description = Embedding(vocab_size, EMBEDDING_DIM1, weights = [embedding_matrix1], trainable = True)(seq_title_description)\n    emb_region = Embedding(max_region, 10)(region)\n    emb_city = Embedding(max_city, 10)(city)\n    emb_category_name = Embedding(max_category_name, 10)(category_name)\n    emb_parent_category_name = Embedding(max_parent_category_name, 10)(parent_category_name)\n    emb_param_1 = Embedding(max_param_1, 10)(param_1)\n    emb_param123 = Embedding(max_param123, 10)(param123)\n\n    rnn_layer1 = GRU(25) (emb_seq_title_description)\n    \n    #main layer\n    main_l = concatenate([\n          rnn_layer1\n        , Flatten() (emb_region)\n        , Flatten() (emb_city)\n        , Flatten() (emb_category_name)\n        , Flatten() (emb_parent_category_name)\n        , Flatten() (emb_param_1)\n        , Flatten() (emb_param123)\n        , price\n        , item_seq_number\n    ])\n    \n    main_l = Dropout(0.1)(Dense(512,activation='relu') (main_l))\n    main_l = Dropout(0.1)(Dense(64,activation='relu') (main_l))\n    \n    #output\n    output = Dense(1,activation=\"sigmoid\") (main_l)\n    \n    #model\n    model = Model([seq_title_description, region, city, category_name, parent_category_name, param_1, param123, price, item_seq_number ], output)\n    model.compile(optimizer = 'adam',\n                  loss= root_mean_squared_error,\n                  metrics = [root_mean_squared_error])\n    return model\n\ndef rmse(y, y_pred):\n\n    Rsum = np.sum((y - y_pred)**2)\n    n = y.shape[0]\n    RMSE = np.sqrt(Rsum/n)\n    return RMSE \n\ndef eval_model(model):\n    val_preds = model.predict(X_valid)\n    y_pred = val_preds[:, 0]\n    \n    y_true = np.array(y_valid_f)\n    \n    yt = pd.DataFrame(y_true)\n    yp = pd.DataFrame(y_pred)\n    \n    print(yt.isnull().any())\n    print(yp.isnull().any())\n    \n    v_rmse = rmse(y_true, y_pred)\n    print(\" RMSE for VALIDATION SET: \"+str(v_rmse))\n    return v_rmse\n\nexp_decay = lambda init, fin, steps: (init/fin)**(1/(steps-1)) - 1","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"c24f4ac2-e862-42b1-82ae-8d7ba4bfc73f","_uuid":"2484fc26cc0ecc6db58690d2b99c1feff53ccb95","trusted":true},"cell_type":"code","source":"from keras import optimizers\n\nepochs = 1\nBATCH_SIZE = 512 * 3\nsteps = int(len(X_train['seq_title_description'])/BATCH_SIZE) * epochs\nlr_init, lr_fin = 0.009, 0.0045\nlr_decay = exp_decay(lr_init, lr_fin, steps)\nmodelRNN = RNN_model()\nK.set_value(modelRNN.optimizer.lr, lr_init)\nK.set_value(modelRNN.optimizer.decay, lr_decay)\n\ndel embedding_matrix1\ngc.collect()\n\nfor i in range(3):\n    history = modelRNN.fit(X_train, y_train_f\n                    , epochs=epochs\n                    , batch_size= (BATCH_SIZE+(BATCH_SIZE*(i)))\n                    , validation_data = (X_valid, y_valid_f)\n                    , verbose=1\n                    )\n    # Evaluate RMSLE \n    v_rmse = eval_model(modelRNN)\n    \nprint(\"Finished Fitting the model\")","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"50701fda-fea2-4a2d-9376-507c7c7ef5bf","_uuid":"3bf1070bde87e426e6c87fa00d50abf7386552b5","trusted":true},"cell_type":"code","source":"del X_train, y_train_f, X_valid, y_valid_f\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"cb53cd3d-827f-494b-93e4-bd9f1b7a8372","_uuid":"0f9d5f94582d2ae83b9ea3954503da31389982e2","trusted":true},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"9c7d0ba0-b7ca-4d0a-b9ee-e963d0ae323b","_uuid":"165e7a3faa7c6f8e85acd6fb712f028021f249de","trusted":true},"cell_type":"code","source":"import time\nt1 = time.time()\ndef load_test():\n    for df in pd.read_csv('../input/avito-demand-prediction/test.csv', chunksize= 250000):\n        yield df\n\nitem_ids = np.array([], dtype=np.int32)\npreds= np.array([], dtype=np.float32)\n\ni = 0 \n    \nfor df in load_test():\n    \n    i +=1\n    print(df.dtypes)\n    item_id = df['item_id']\n    print(\" Chunk number is \"+str(i))\n    test = preprocess_dataset(df)\n    test = keras_test_transform(df)\n    del df\n    gc.collect()\n    \n    X_test = get_keras_data(test)\n    del test \n    gc.collect()\n    \n    preds1 = modelRNN.predict(X_test, batch_size = BATCH_SIZE, verbose = 1)\n    print(preds1.shape)\n    del X_test\n    gc.collect()\n    print(\"RNN Prediction is done\")\n\n    preds1 = preds1.reshape(-1,1)\n    #print(predsl.shape)\n    preds1 = np.clip(preds1, 0, 1)\n    print(preds1.shape)\n    item_ids = np.append(item_ids, item_id)\n    print(item_ids.shape)\n    preds = np.append(preds, preds1)\n    print(preds.shape)\n    \nprint(\"All chunks done\")\nt2 = time.time()\nprint(\"Total time for Parallel Batch Prediction is \"+str(t2-t1))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"e6833179-8039-44d9-867b-553c2d32dea3","_uuid":"22eb2aa29ced6806f75b62a677c60a2449577c7d","trusted":true},"cell_type":"code","source":"submission = pd.DataFrame( columns = ['item_id', 'deal_probability'])\nsubmission['item_id'] = item_ids\nsubmission['deal_probability'] = preds\n\nprint(\"Check Submission NOW!!!!!!!!@\")\nsubmission.to_csv(\"Avito_Shanth_RNN.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"0f5e8669-c86e-4c29-b0d9-9e935c72a0a3","_uuid":"1dd570ec2e4277ec005686c96fe8540befc39102","collapsed":true,"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.5","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}