{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Objective\n\n* Make a baseline model that predict the validation (28 days). \n* This competition has 2 stages, so the main objective is to make a model that can predict the demand for the next 28 days","metadata":{}},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings('ignore')\nimport pandas as pd\nimport numpy as np\nimport dask.dataframe as dd\npd.set_option('display.max_columns', 500)\npd.set_option('display.max_rows', 500)\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport lightgbm as lgb\nimport dask_xgboost as xgb\nimport dask.dataframe as dd\nfrom sklearn import preprocessing, metrics\nimport gc\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-22T06:40:27.608649Z","iopub.execute_input":"2022-07-22T06:40:27.608950Z","iopub.status.idle":"2022-07-22T06:40:27.622138Z","shell.execute_reply.started":"2022-07-22T06:40:27.608899Z","shell.execute_reply":"2022-07-22T06:40:27.621190Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def reduce_mem_usage(df, verbose=True):\n    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n    start_mem = df.memory_usage().sum() / 1024**2    \n    for col in df.columns:\n        col_type = df[col].dtypes\n        if col_type in numerics:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)    \n    end_mem = df.memory_usage().sum() / 1024**2\n    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n    return df\n\n\n# function to read the data and merge it (ignoring some columns, this is a very fst model)\n\n\ndef read_data():\n    print('Reading files...')\n    calendar = pd.read_csv('/kaggle/input/m5-forecasting-accuracy/calendar.csv')\n    calendar = reduce_mem_usage(calendar)\n    print('Calendar has {} rows and {} columns'.format(calendar.shape[0], calendar.shape[1]))\n    sell_prices = pd.read_csv('/kaggle/input/m5-forecasting-accuracy/sell_prices.csv')\n    sell_prices = reduce_mem_usage(sell_prices)\n    print('Sell prices has {} rows and {} columns'.format(sell_prices.shape[0], sell_prices.shape[1]))\n    sales_train_validation = pd.read_csv('/kaggle/input/m5-forecasting-accuracy/sales_train_validation.csv')\n    print('Sales train validation has {} rows and {} columns'.format(sales_train_validation.shape[0], sales_train_validation.shape[1]))\n    submission = pd.read_csv('/kaggle/input/m5-forecasting-accuracy/sample_submission.csv')\n    return calendar, sell_prices, sales_train_validation, submission\n\n\ndef melt_and_merge(calendar, sell_prices, sales_train_validation, submission, nrows = 55000000, merge = False):\n    \n    # melt sales data, get it ready for training\n    sales_train_validation = pd.melt(sales_train_validation, id_vars = ['id', 'item_id', 'dept_id', 'cat_id', 'store_id', 'state_id'], var_name = 'day', value_name = 'demand')\n    print(sales_train_validation.head())\n    print('Melted sales train validation has {} rows and {} columns'.format(sales_train_validation.shape[0], sales_train_validation.shape[1]))\n    sales_train_validation = reduce_mem_usage(sales_train_validation)\n    \n    # seperate test dataframes\n    test1_rows = [row for row in submission['id'] if 'validation' in row]\n    test2_rows = [row for row in submission['id'] if 'evaluation' in row]\n    test1 = submission[submission['id'].isin(test1_rows)]\n    test2 = submission[submission['id'].isin(test2_rows)]\n    \n    # change column names\n    test1.columns = ['id', 'd_1914', 'd_1915', 'd_1916', 'd_1917', 'd_1918', 'd_1919', 'd_1920', 'd_1921', 'd_1922', 'd_1923', 'd_1924', 'd_1925', 'd_1926', 'd_1927', 'd_1928', 'd_1929', 'd_1930', 'd_1931', \n                      'd_1932', 'd_1933', 'd_1934', 'd_1935', 'd_1936', 'd_1937', 'd_1938', 'd_1939', 'd_1940', 'd_1941']\n    test2.columns = ['id', 'd_1942', 'd_1943', 'd_1944', 'd_1945', 'd_1946', 'd_1947', 'd_1948', 'd_1949', 'd_1950', 'd_1951', 'd_1952', 'd_1953', 'd_1954', 'd_1955', 'd_1956', 'd_1957', 'd_1958', 'd_1959', \n                      'd_1960', 'd_1961', 'd_1962', 'd_1963', 'd_1964', 'd_1965', 'd_1966', 'd_1967', 'd_1968', 'd_1969']\n    \n    # get product table\n    product = sales_train_validation[['id', 'item_id', 'dept_id', 'cat_id', 'store_id', 'state_id']].drop_duplicates()\n    \n    # merge with product table\n    test2['id'] = test2['id'].str.replace('_evaluation','_validation')\n    test1 = test1.merge(product, how = 'left', on = 'id')\n    test2 = test2.merge(product, how = 'left', on = 'id')\n    test2['id'] = test2['id'].str.replace('_validation','_evaluation')\n    \n    # \n    test1 = pd.melt(test1, id_vars = ['id', 'item_id', 'dept_id', 'cat_id', 'store_id', 'state_id'], var_name = 'day', value_name = 'demand')\n    test2 = pd.melt(test2, id_vars = ['id', 'item_id', 'dept_id', 'cat_id', 'store_id', 'state_id'], var_name = 'day', value_name = 'demand')\n    \n    sales_train_validation['part'] = 'train'\n    test1['part'] = 'test1'\n    test2['part'] = 'test2'\n    \n    data = pd.concat([sales_train_validation, test1, test2], axis = 0)\n    \n    del sales_train_validation, test1, test2\n    \n    # get only a sample for fst training\n    data = data.loc[nrows:]\n    \n    # drop some calendar features\n    calendar.drop(['weekday', 'wday', 'month', 'year'], inplace = True, axis = 1)\n    \n    # delete test2 for now\n    data = data[data['part'] != 'test2']\n    \n    if merge:\n        # notebook crash with the entire dataset (maybee use tensorflow, dask, pyspark xD)\n        data = pd.merge(data, calendar, how = 'left', left_on = ['day'], right_on = ['d'])\n        data.drop(['d', 'day'], inplace = True, axis = 1)\n        # get the sell price data (this feature should be very important)\n        data = data.merge(sell_prices, on = ['store_id', 'item_id', 'wm_yr_wk'], how = 'left')\n        print('Our final dataset to train has {} rows and {} columns'.format(data.shape[0], data.shape[1]))\n    else: \n        pass\n    \n    gc.collect()\n    \n    return data\n        \ncalendar, sell_prices, sales_train_validation, submission = read_data()\ndata = melt_and_merge(calendar, sell_prices, sales_train_validation, submission, nrows = 27500000, merge = True)","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","execution":{"iopub.status.busy":"2022-07-22T06:44:17.440121Z","iopub.execute_input":"2022-07-22T06:44:17.440545Z","iopub.status.idle":"2022-07-22T06:47:54.366837Z","shell.execute_reply.started":"2022-07-22T06:44:17.440466Z","shell.execute_reply":"2022-07-22T06:47:54.365828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"* We have the data to build our first model, let's build a baseline and predict the validation data (in our case is test1)","metadata":{}},{"cell_type":"code","source":"def transform(data):\n    \n    nan_features = ['event_name_1', 'event_type_1', 'event_name_2', 'event_type_2']\n    for feature in nan_features:\n        data[feature].fillna('unknown', inplace = True)\n        \n    cat = ['item_id', 'dept_id', 'cat_id', 'store_id', 'state_id', 'event_name_1', 'event_type_1', 'event_name_2', 'event_type_2']\n    for feature in cat:\n        encoder = preprocessing.LabelEncoder()\n        data[feature] = encoder.fit_transform(data[feature])\n    \n    return data\n\ndef simple_fe(data):\n    \n    # rolling demand features\n    data['lag_t28'] = data.groupby(['id'])['demand'].transform(lambda x: x.shift(28))\n    data['lag_t29'] = data.groupby(['id'])['demand'].transform(lambda x: x.shift(29))\n    data['lag_t30'] = data.groupby(['id'])['demand'].transform(lambda x: x.shift(30))\n    data['rolling_mean_t7'] = data.groupby(['id'])['demand'].transform(lambda x: x.shift(28).rolling(7).mean())\n    data['rolling_std_t7'] = data.groupby(['id'])['demand'].transform(lambda x: x.shift(28).rolling(7).std())\n    data['rolling_mean_t30'] = data.groupby(['id'])['demand'].transform(lambda x: x.shift(28).rolling(30).mean())\n    data['rolling_mean_t90'] = data.groupby(['id'])['demand'].transform(lambda x: x.shift(28).rolling(90).mean())\n    data['rolling_mean_t180'] = data.groupby(['id'])['demand'].transform(lambda x: x.shift(28).rolling(180).mean())\n    data['rolling_std_t30'] = data.groupby(['id'])['demand'].transform(lambda x: x.shift(28).rolling(30).std())\n    data['rolling_skew_t30'] = data.groupby(['id'])['demand'].transform(lambda x: x.shift(28).rolling(30).skew())\n    data['rolling_kurt_t30'] = data.groupby(['id'])['demand'].transform(lambda x: x.shift(28).rolling(30).kurt())\n    \n    \n    # price features\n    data['lag_price_t1'] = data.groupby(['id'])['sell_price'].transform(lambda x: x.shift(1))\n    data['price_change_t1'] = (data['lag_price_t1'] - data['sell_price']) / (data['lag_price_t1'])\n    data['rolling_price_max_t365'] = data.groupby(['id'])['sell_price'].transform(lambda x: x.shift(1).rolling(365).max())\n    data['price_change_t365'] = (data['rolling_price_max_t365'] - data['sell_price']) / (data['rolling_price_max_t365'])\n    data['rolling_price_std_t7'] = data.groupby(['id'])['sell_price'].transform(lambda x: x.rolling(7).std())\n    data['rolling_price_std_t30'] = data.groupby(['id'])['sell_price'].transform(lambda x: x.rolling(30).std())\n    data.drop(['rolling_price_max_t365', 'lag_price_t1'], inplace = True, axis = 1)\n    \n    # time features\n    data['date'] = pd.to_datetime(data['date'])\n    data['year'] = data['date'].dt.year\n    data['month'] = data['date'].dt.month\n    data['week'] = data['date'].dt.week\n    data['day'] = data['date'].dt.day\n    data['dayofweek'] = data['date'].dt.dayofweek\n    \n    \n    return data\n\ndef run_lgb(data):\n    \n    # going to evaluate with the last 28 days\n    x_train = data[data['date'] <= '2016-03-27']\n    y_train = x_train['demand']\n    x_val = data[(data['date'] > '2016-03-27') & (data['date'] <= '2016-04-24')]\n    y_val = x_val['demand']\n    test = data[(data['date'] > '2016-04-24')]\n    del data\n    gc.collect()\n\n    # define random hyperparammeters\n    params = {\n        'boosting_type': 'gbdt',\n        'metric': 'rmse',\n        'objective': 'regression',\n        'n_jobs': -1,\n        'seed': 236,\n        'learning_rate': 0.1,\n        'bagging_fraction': 0.75,\n        'bagging_freq': 10, \n        'colsample_bytree': 0.75}\n\n    train_set = lgb.Dataset(x_train[features], y_train)\n    val_set = lgb.Dataset(x_val[features], y_val)\n    \n    del x_train, y_train\n\n    model = lgb.train(params, train_set, num_boost_round = 2500, early_stopping_rounds = 50, valid_sets = [train_set, val_set], verbose_eval = 100)\n    val_pred = model.predict(x_val[features])\n    val_score = np.sqrt(metrics.mean_squared_error(val_pred, y_val))\n    print(f'Our val rmse score is {val_score}')\n    y_pred = model.predict(test[features])\n    test['demand'] = y_pred\n    return test\n\ndef predict(test, submission):\n    predictions = test[['id', 'date', 'demand']]\n    predictions = pd.pivot(predictions, index = 'id', columns = 'date', values = 'demand').reset_index()\n    predictions.columns = ['id'] + ['F' + str(i + 1) for i in range(28)]\n\n    evaluation_rows = [row for row in submission['id'] if 'evaluation' in row] \n    evaluation = submission[submission['id'].isin(evaluation_rows)]\n\n    validation = submission[['id']].merge(predictions, on = 'id')\n    final = pd.concat([validation, evaluation])\n    final.to_csv('submission.csv', index = False)\n    \n\n# define list of features\nfeatures = ['item_id', 'dept_id', 'cat_id', 'store_id', 'state_id', 'year', 'month', 'week', 'day', 'dayofweek', 'event_name_1', 'event_type_1', 'event_name_2', 'event_type_2', \n            'snap_CA', 'snap_TX', 'snap_WI', 'sell_price', 'lag_t28', 'lag_t29', 'lag_t30', 'rolling_mean_t7', 'rolling_std_t7', 'rolling_mean_t30', 'rolling_mean_t90', \n            'rolling_mean_t180', 'rolling_std_t30', 'price_change_t1', 'price_change_t365', 'rolling_price_std_t7', 'rolling_price_std_t30', 'rolling_skew_t30', 'rolling_kurt_t30']\n\n\ndef transform_train_and_eval(data):\n    data = transform(data)\n    data = simple_fe(data)\n    # reduce memory for new features so we can train\n    data = reduce_mem_usage(data)\n    test = run_lgb(data)\n    predict(test, submission)\n    \ntransform_train_and_eval(data)","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}