{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"collapsed":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\nfrom sklearn.decomposition import TruncatedSVD\nfrom sklearn import preprocessing ,metrics\nfrom sklearn.svm import SVR\nfrom sklearn.linear_model import Ridge, Lasso, ElasticNet\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.model_selection import KFold, cross_validate, GridSearchCV\nimport xgboost as xgb\nimport lightgbm.sklearn as lgb\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport sklearn\nfrom sklearn.utils.testing import all_estimators\nimport sys\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\nfrom scipy import sparse as sp\n# Any results you write to the current directory are saved as output.\nfrom scipy.special import lambertw\nfrom scipy.stats import kurtosis, norm, rankdata, boxcox, zscore\nfrom scipy.optimize import fmin  # TODO: Explore efficacy of other opt. methods\nimport gc\nimport psutil\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b90fb21f3d03eb768936a2a74c71777ca0711370","collapsed":true},"cell_type":"code","source":"class wrapped_logger():\n    def __init__(self,log_file_name = 'debug.log'):\n        from logging import getLogger, StreamHandler,DEBUG,Formatter, FileHandler\n        import psutil \n        import time\n        self.time=time.time\n        self.logger = getLogger(__name__)\n        self.START_TIME=self.time()\n        self.psutil=psutil\n\n        handler_format = Formatter(f'%(asctime)s -  %(name)s - %(levelname)s - %(message)s')\n\n        handler = StreamHandler()\n        handler.setFormatter(handler_format)\n        handler.setLevel(DEBUG)\n        file_handler = FileHandler(log_file_name)\n        file_handler.setFormatter(handler_format)\n        file_handler.setLevel(DEBUG)\n        self.logger.setLevel(DEBUG)\n        self.logger.addHandler(handler)\n        self.logger.addHandler(file_handler)\n    def info(self,message):\n        self.logger.info(self.__message(message))\n    def debug(self,message):\n        self.logger.debug(self.__message(message))\n    def warning(self,message):\n        self.logger.warning(self.__message(message))\n        \n    def __message(self,message):\n        return f'{self.time()-self.START_TIME}s - mem usage:{self.psutil.virtual_memory().used/1024/1024} - {message}'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"48514181d2a8fb96ea5e6e76113eae4bc752a64c","collapsed":true},"cell_type":"code","source":"logger=wrapped_logger()\nlogger.info('start logging')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"d5e1d91de1a95691ea66729738b8f9fabb2a60c7"},"cell_type":"code","source":"FRAC = 0.01\nMY_DATASET='../input/avito-demand-prediction-challenge-private-dataset'\nDATASET='../input/avito-demand-prediction'\ndef read_csv(frac,fanc):\n    data = fanc\n    #data=data.iloc[:10000,:]\n    #if FRAC != 1: data = data.sample(frac = FRAC,random_state=0).reset_index(drop=True)\n    return data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"56df990757bc9fd0e0afeb958c615e3c74d7a37e","collapsed":true},"cell_type":"code","source":"train = read_csv(FRAC,pd.read_csv(f'{DATASET}/train.csv', parse_dates=[\"activation_date\"]))\ntest = read_csv(FRAC,pd.read_csv(f'{DATASET}/test.csv', parse_dates=[\"activation_date\"]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"26cbaf8cc138210cd27c772b578f8c15eb156d98","collapsed":true},"cell_type":"code","source":"train.nunique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f731c3153734983f4e2519076177c46a44bf4630","collapsed":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"9d4caa2a9d94a1e22d23453c7a16c9126cdcfb88"},"cell_type":"code","source":"def fill_na(col_names,train,test,fill_what):\n    for col_name in col_names:\n        train[col_name].fillna(fill_what, inplace=True)\n        test[col_name].fillna(fill_what, inplace=True)\n    return train, test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"1153c1a62d7f067e2ccd9bc509059f59663b982d"},"cell_type":"code","source":"def get_len(col_names):\n    for col_name in col_names:\n        train[f'{col_name}_len'] = train[col_name].apply(lambda x: len(x.split()))\n        test[f'{col_name}_len'] = test[col_name].apply(lambda x: len(x.split()))\n    return train, test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6ee9731e9d3cd8a6662fe7939c037a014f7fa5af","collapsed":true},"cell_type":"code","source":"train.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"ce8818e9519539e3e489a4fef2f633d6d1a5dac9"},"cell_type":"code","source":"def get_zscore(train,test,col_names):\n    full_data = pd.concat([train, test], axis = 0)\n    for col in col_names:\n        full_data[col] =((full_data[col]-full_data[col].mean())/full_data[col].std())\n        train[col]=full_data[col][:train.shape[0]]\n        test[col] =full_data[col][train.shape[0]:]\n        \n    del full_data\n    gc.collect()\n    return train, test\n        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"03b05602cae4fd41d30abe854b003f396c6e61e6","collapsed":true},"cell_type":"code","source":"col_names_fillna = [\n    'description',\n    'title',\n    \"param_1\", \n    \"param_2\", \n    \"param_3\",\n    'city',\n    \"region\",\n    \"parent_category_name\", \n    \"category_name\", \n    \"user_type\",\n    'user_id',\n    'activation_date'\n]\ntrain,test=fill_na(col_names_fillna,train,test,'NaN')\n\ntrain['image_bool'] = train['image'].apply(lambda x: 1 if str == type(x) else 0)\ntest['image_bool'] = test['image'].apply(lambda x: 1 if str == type(x) else 0)\n\ncol_names_len = [\n    'description',\n    'title',\n    \"param_1\", \n    \"param_2\", \n    \"param_3\"\n]\n\ntrain,test=get_len(col_names_len)\ncol_names_len = [\n    'description_len',\n    'title_len',\n    \"param_1_len\", \n    \"param_2_len\", \n    \"param_3_len\",\n    'price',\n    'item_seq_number'\n]\ntrain,test=get_zscore(train,test,col_names_len)\ny_train =train[\"deal_probability\"].ravel()\n\ntest_id = test[\"item_id\"].values\n\ncols_to_drop = [\"item_id\", 'image']\ntrain = train.drop(cols_to_drop + [\"deal_probability\"], axis = 1)\ntest = test.drop(cols_to_drop, axis = 1)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d8b0d2e2e1f421a17b9b513cbd25fc6e65e8c635","collapsed":true},"cell_type":"code","source":"def get_svd(fit_data, transform_data, n_comp, col_name):\n    #get svd\n    svd_obj = TruncatedSVD(n_components=n_comp, algorithm='arpack')\n    svd_obj.fit(fit_data)\n    df_svd = pd.DataFrame(svd_obj.transform(transform_data))\n    df_svd.columns = ['svd_' + col_name + '_'+str(i+1) for i in range(n_comp)]\n    del svd_obj\n    gc.collect()\n    return df_svd\n\ndef get_tfidf_svd(train, test, col_name):\n    #get tfidf\n    tfidf_vec = TfidfVectorizer(ngram_range=(1,1), max_features=100000)\n    full_tfidf = tfidf_vec.fit_transform(pd.concat([train[col_name], test[col_name]], axis=0))\n    train_tfidf = tfidf_vec.transform(train[col_name])\n    test_tfidf = tfidf_vec.transform(test[col_name])\n    \n    #get svd\n    n_comp = 10\n    train_svd = get_svd(full_tfidf, train_tfidf, n_comp,col_name)\n    test_svd = get_svd(full_tfidf, test_tfidf, n_comp,col_name)\n\n    #clean\n    del full_tfidf, train_tfidf, test_tfidf, tfidf_vec\n    gc.collect()\n    \n    return train_svd, test_svd\n\ndef engineering_tfidf(how,col_name):\n    if how == 'read': \n        logger.info(f'Read {col_name}')\n        train_svd = read_csv(FRAC,pd.read_csv(f'{MY_DATASET}/train_{col_name}_svd.csv'))\n        test_svd = read_csv(FRAC,pd.read_csv(f'{MY_DATASET}/test_{col_name}_svd.csv'))\n    elif how == 'write':\n        train_svd, test_svd = get_tfidf_svd(train = train, test = test, col_name = col_name)\n        train_svd.to_csv(f'train_{col_name}_svd.csv',index=False)\n        test_svd.to_csv(f'test_{col_name}_svd.csv',index=False)\n    else:\n        train_svd, test_svd = get_tfidf_svd(train = train, test = test, col_name = col_name)\n    return train_svd, test_svd\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"577cc86d13739580a6f6b0eea705f16aa6d83799","collapsed":true},"cell_type":"code","source":"train_title_svd, test_title_svd = engineering_tfidf('read','title')\ntrain_description_svd, test_description_svd = engineering_tfidf('read','description')\ntrain = pd.concat([train, train_title_svd,train_description_svd], axis=1)\ntest = pd.concat([test, test_title_svd,test_description_svd], axis=1)\ndel train_title_svd, test_title_svd, train_description_svd, test_description_svd\ngc.collect()\ncols_to_drop = ['description', 'title']\ntrain.drop(cols_to_drop, axis = 1, inplace=True)\ntest.drop(cols_to_drop, axis = 1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b0d5bccc7026fe984035d1f5b3b55b007807a075","collapsed":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4bf6977662bf088e8bcc135f1472925ff519f274","collapsed":true},"cell_type":"code","source":"train.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6a432798fe21b83e806b476e05d813bf3b9fafe2","collapsed":true},"cell_type":"code","source":"def num_to_cat(col_names,train,test,n_comp=-1):\n    \n    for col_name in col_names:\n        full_col = pd.concat([train[col_name], test[col_name]])\n        n_unique=full_col.nunique(dropna=False)\n        le = preprocessing.LabelEncoder()\n        oe = preprocessing.OneHotEncoder()\n        le.fit(full_col.values.astype('str'))\n\n        train_le = le.transform(train[col_name].values.astype('str')).reshape(-1,1)\n        test_le = le.transform(test[col_name].values.astype('str')).reshape(-1,1)\n        full_le = np.append(train_le , test_le).reshape(-1,1)\n        oe.fit(full_le)\n        train_oe = oe.transform(train_le)\n        test_oe = oe.transform(test_le)\n        \n        full_oe =sp.vstack((train_oe  ,test_oe))\n        if n_unique <= n_comp or n_comp == -1: \n            col_names_svd=[f'le_{col_name}_{i}' for i in range(1, n_unique+1) ]\n            train_svd=pd.SparseDataFrame(train_oe, columns=col_names_svd)\n            test_svd=pd.SparseDataFrame(test_oe, columns=col_names_svd)\n        else:\n            train_svd=get_svd(full_oe, train_oe, n_comp, col_name)\n            test_svd=get_svd(full_oe, test_oe, n_comp, col_name)\n        train = pd.concat([train,train_svd], axis=1)\n        test = pd.concat([test, test_svd], axis=1)\n        print(train.head())\n        train = train.drop(col_name, axis = 1)\n        test = test.drop(col_name, axis = 1)\n\n    return train,test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"3719a63d5112a35fec5ec00ff95b2079abf24852"},"cell_type":"code","source":"def num_to_label(col_names,train,test):\n    for col in col_names:\n        lbl = preprocessing.LabelEncoder()\n        lbl.fit(list(train[col].values.astype('str')) + list(test[col].values.astype('str')))\n        train[col] = lbl.transform(list(train[col].values.astype('str')))\n        test[col] = lbl.transform(list(test[col].values.astype('str')))\n    return train,test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"af85058bdeb5cae6f52252e80ae996cfbdf9ef77","scrolled":true,"collapsed":true},"cell_type":"code","source":"\ntrain[\"activation_weekday\"] = train[\"activation_date\"].dt.weekday\ntest[\"activation_weekday\"] = test[\"activation_date\"].dt.weekday\ntrain[\"activation_month\"] = train[\"activation_date\"].dt.month\ntest[\"activation_month\"] = test[\"activation_date\"].dt.month\ncols_to_drop = [\n    \"activation_date\",\n    'user_id'\n]\n\ntrain = train.drop(cols_to_drop, axis = 1)\ntest = test.drop(cols_to_drop, axis = 1)\ncol_vars_cat = [\n    #'user_id',\n    \"city\",\n    \"region\", \n    \"parent_category_name\", \n    \"category_name\", \n    \"user_type\", \n    \"param_1\", \n    \"param_2\", \n    \"param_3\",\n    'image_top_1',\n    'activation_weekday',\n    'activation_month'\n]\ncol_vars_lb=[\n    'user_id'\n]\ntrain,test=num_to_cat(col_vars_cat,train,test)\n#train,test=num_to_label(col_vars_lb,train,test)\n\ntrain.info()\ntrain = train.to_sparse()\ntest=test.to_sparse()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"70608d07c48f25f5203eac97dc9dd685ede5ef95","collapsed":true},"cell_type":"code","source":"train = train.to_coo().tocsr()\ntest = test.to_coo().tocsr()\nX=sp.vstack((train,test))\n\ndel train,test\ngc.collect()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true,"_uuid":"c991d8f61876c48655b531b47fa56443705e93b8"},"cell_type":"code","source":"def RMSLE(y, pred):\n    return metrics.mean_squared_error(y, pred)**0.5\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8a5272fb84156963f0fee6704979d730c1366391","collapsed":true},"cell_type":"code","source":"### train denoising autoencoder\nfrom keras.layers import Input, Dense\nfrom keras import Model\nfrom keras.callbacks import ModelCheckpoint, Callback, EarlyStopping\n\ndef get_DAE():\n    # denoising autoencoder\n    inputs = Input((X.shape[1],))\n    x = Dense(1500, activation='relu')(inputs) # 1500 original\n    x = Dense(1500, activation='relu', name=\"feature\")(x) # 1500 original\n    x = Dense(1500, activation='relu')(x) # 1500 original\n    outputs = Dense(X.shape[1], activation='relu')(x)\n    model = Model(inputs=inputs, outputs=outputs)\n    model.compile(optimizer='adam', loss='mse')\n\n    return model\n\n\ndef x_generator(x, batch_size, shuffle=True):\n    # batch generator of input\n    batch_index = 0\n    n = x.shape[0]\n    while True:\n        if batch_index == 0:\n            index_array = np.arange(n)\n            if shuffle:\n                index_array = np.random.permutation(n)\n\n        current_index = (batch_index * batch_size) % n\n        if n >= current_index + batch_size:\n            current_batch_size = batch_size\n            batch_index += 1\n        else:\n            current_batch_size = n - current_index\n            batch_index = 0\n\n        batch_x = x[index_array[current_index: current_index + current_batch_size]]\n\n        yield batch_x\n\n\ndef mix_generator(x, batch_size, swaprate=0.15, shuffle=True):\n    # generator of noized input and output\n    # swap 0.15% of values of data with values of another\n    num_value = X.shape[1]\n    num_swap = int(num_value * swaprate)\n    gen1 = x_generator(x, batch_size, shuffle)\n    gen2 = x_generator(x, batch_size, shuffle)\n    while True:\n        batch1 = next(gen1)\n        batch2 = next(gen2)\n        new_batch = batch1.copy().tolil()\n        for i in range(batch1.shape[0]):\n            swap_idx = np.random.choice(num_value, num_swap, replace=False)\n            new_batch[i, swap_idx] = batch2[i, swap_idx]\n\n        yield (new_batch.toarray(), batch1.toarray())\n    \ndef get_callbacks(save_path):\n    save_checkpoint = ModelCheckpoint(filepath=save_path, monitor='loss', save_best_only=True)\n    early_stopping = EarlyStopping(monitor='loss',\n                                   patience=4,\n                                   verbose=1,\n                                   min_delta=1e-4,\n                                   mode='min')\n    Callbacks = [ save_checkpoint, early_stopping]\n    return Callbacks\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3e1bc87869412f7be528807ee5621bca5a4ec585","collapsed":true},"cell_type":"code","source":"# training\nbatch_size = 128\nnum_epoch = 15 # 1000 original\n#gen = mix_generator(X, batch_size)\ncallbacks = get_callbacks('weight_dae.hdf5')\ndae = get_DAE()\ndae.load_weights(f'{MY_DATASET}/weight_dae.hdf5')\n#dae.load_weights(f'weight_dae.hdf5')\n'''\nhist = dae.fit_generator(generator=gen,\n                  steps_per_epoch=np.ceil(X.shape[0] / batch_size),\n                  epochs=num_epoch,\n                  callbacks=callbacks,\n                  verbose=1,)\n'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"52b910e87f3b02310de0329e5c25b26907486b95","collapsed":true},"cell_type":"code","source":"#logger.debug(hist.history)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5bf56773cf5010e4c4383bd585e68971f8603cb5","collapsed":true},"cell_type":"code","source":"### train NN with feature of DAE\nfrom keras.layers import Dropout\nfrom  keras.regularizers import l2\n\ndef get_NN(DAE):\n    l2_loss = l2(0.05)\n    DAE.trainable = False\n    x = dae.get_layer(\"feature\").output\n    x = Dropout(0.1)(x)\n    x = Dense(500, activation='relu', kernel_regularizer=l2_loss)(x) # 4500 original\n    x = Dropout(0.5)(x)\n    x = Dense(100, activation='relu', kernel_regularizer=l2_loss)(x) # 1000 original\n    x = Dropout(0.5)(x)\n    x = Dense(100, activation='relu', kernel_regularizer=l2_loss)(x) # 1000 original\n    x = Dropout(0.5)(x)\n    predictions = Dense(1, activation='relu', kernel_regularizer=l2_loss)(x)\n\n    model = Model(inputs=dae.input, outputs=predictions)\n    model.compile(loss='mse',optimizer='adam')\n\n    return model\n\n\ndef train_generator(x, y, batch_size, shuffle=True):\n    batch_index = 0\n    n = x.shape[0]\n    while True:\n        if batch_index == 0:\n            index_array = np.arange(n)\n            if shuffle:\n                index_array = np.random.permutation(n)\n\n        current_index = (batch_index * batch_size) % n\n        if n >= current_index + batch_size:\n            current_batch_size = batch_size\n            batch_index += 1\n        else:\n            current_batch_size = n - current_index\n            batch_index = 0\n\n        batch_x = x[index_array[current_index: current_index + current_batch_size]]\n        batch_y = y[index_array[current_index: current_index + current_batch_size]]\n\n        yield batch_x.toarray(), batch_y\n\n\ndef test_generator(x, batch_size, shuffle=False):\n    batch_index = 0\n    n = x.shape[0]\n    while True:\n        if batch_index == 0:\n            index_array = np.arange(n)\n            if shuffle:\n                index_array = np.random.permutation(n)\n\n        current_index = (batch_index * batch_size) % n\n        if n >= current_index + batch_size:\n            current_batch_size = batch_size\n            batch_index += 1\n        else:\n            current_batch_size = n - current_index\n            batch_index = 0\n\n        batch_x = x[index_array[current_index: current_index + current_batch_size]]\n\n        yield batch_x.toarray()\n\n        \n    \ndef get_callbacks(save_path):\n    save_checkpoint = ModelCheckpoint(filepath=save_path, monitor='val_loss', save_best_only=True)\n    early_stopping = EarlyStopping(monitor='val_loss',\n                                   patience=4,\n                                   verbose=1,\n                                   min_delta=1e-4,\n                                   mode='min')\n    Callbacks = [ save_checkpoint, early_stopping]\n    return Callbacks","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"921735852dd3a0f5bc177a16cd263cf9e4a00c1a","collapsed":true},"cell_type":"code","source":"y_train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0c0374380c0088f632f11d211951ee742290fde6","collapsed":true},"cell_type":"code","source":"import types\ndef get_memory_use():\n    for k,v in globals().items():\n        if hasattr(v, 'size') and not k.startswith('_') and not isinstance(v,types.ModuleType):\n            print(f'{k}|{v.size}')\n\nget_memory_use()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"321a540bbfb954953e5d39593ee1d0e36034864f","collapsed":true},"cell_type":"code","source":"\nfrom sklearn.model_selection import KFold\n\nbatch_size = 128\nnum_epoch = 5 # 150 original\nnum_fold = 5 # 5 original\n\nY_train = y_train\nX_train = X[:Y_train.shape[0]]\nX_test = X[Y_train.shape[0]:]\ndel X\ngc.collect()\n\ny_test = np.zeros([num_fold, X_test.shape[0]])\ny_valid = np.zeros([X_train.shape[0]])\n\nfolds = list(KFold(n_splits=num_fold, shuffle=True, random_state=42).split(X_train))\nfor j, (ids_train_split, ids_valid_split) in enumerate(folds):\n    print(\"fold\", j+1, \"==================\")\n    model = get_NN(dae)\n\n    # Fit model\n    if  j==-1:\n        ids_train_split=np.loadtxt(f'{MY_DATASET}/ids_train_split_{j}.csv',delimiter=',').astype('int32')\n        ids_valid_split=np.loadtxt(f'{MY_DATASET}/ids_valid_split_{j}.csv',delimiter=',').astype('int32')\n        callbacks = get_callbacks(\"weight\" + str(j) + \".hdf5\")\n    else:\n        #np.savetxt(f'ids_train_split_{j}.csv',ids_train_split,delimiter=',')\n        #np.savetxt(f'ids_valid_split_{j}.csv',ids_valid_split,delimiter=',')\n        #continue\n        callbacks = [get_callbacks(\"weight\" + str(j) + \".hdf5\")[1]]\n\n    if  j==-1 :\n        gen_train = train_generator(X_train[ids_train_split], Y_train[ids_train_split], batch_size)\n        gen_val = train_generator(X_train[ids_valid_split], Y_train[ids_valid_split], batch_size, shuffle=False)\n        callbacks = get_callbacks(\"weight\" + str(j) + \".hdf5\")\n        hist = model.fit_generator(\n            generator=gen_train,\n            steps_per_epoch=np.ceil(ids_train_split.shape[0] / batch_size),\n            epochs=num_epoch,\n            verbose=1,\n            callbacks=callbacks,\n            validation_data=gen_val,\n            validation_steps=np.ceil(ids_valid_split.shape[0] / batch_size),\n        )\n        logger.debug(hist.history)\n        del gen_train,gen_val\n        gc.collect()\n    else:\n        model.load_weights(f'{MY_DATASET}/weight{j}.hdf5') # load best epoch weight\n        pass\n\n\n    \n    # Predict on train, val and test\n    gen_val_pred = test_generator(X_train[ids_valid_split], batch_size, shuffle=False)\n    gen_test_pred = test_generator(X_test, batch_size, shuffle=False)\n    \n    y_valid[ids_valid_split] = model.predict_generator(generator=gen_val_pred,\n                                        steps=np.ceil(ids_valid_split.shape[0] / batch_size))[:,0]\n    y_test[j] = model.predict_generator(generator=gen_test_pred,\n                                        steps=np.ceil(X_test.shape[0] / batch_size))[:,0]\n    del gen_test_pred,gen_val_pred,model\n    gc.collect()\n\nscore = RMSLE(y_valid, Y_train)\nlogger.info(f'valid score: {score}')\ny_test_mean = np.mean(y_test, axis=0)\nnp.savetxt(f'dae_nn_train.csv',y_valid,delimiter=',')\nnp.savetxt(f'dae_nn_test.csv',y_test_mean,delimiter=',')\nsubmission = pd.DataFrame({\"item_id\":test_id, 'deal_probability': y_test_mean})\nsubmission.to_csv('submission_dae.csv', index=False)\n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}