{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"I'm currently plagued with submission errors.\n\nThis error must be caused by my \"create_daily_lag_feature\" function　to create target shift feature group by player.\nI am sure that the error is caused by this function. But I have no idea why.\n\nWhen I check the sample iteration test, It doesn't cause any errors. But only when I submit it, caused Notebook Threw Exception.\n\n\nI tried a lot of checking(reduce_memory, change dtypes, reduce dataframe rows), but none of it has helped\n\nPlease help me.","metadata":{}},{"cell_type":"code","source":"# ----- Import common library -----　\nimport pandas as pd\nimport numpy as np\nfrom datetime import datetime, timedelta\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport seaborn as sns\nsns.set()\nfrom tqdm import tqdm_notebook as tqdm\nfrom glob import glob\nimport gc\nimport pickle\nfrom time import time, sleep\nimport json\nimport pytz\nimport random\npd.set_option('display.max_columns', 500)\nimport warnings\nwarnings.filterwarnings('ignore')\nfrom IPython.core.display import display\nimport tempfile","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-07-03T08:40:49.335678Z","iopub.execute_input":"2021-07-03T08:40:49.336056Z","iopub.status.idle":"2021-07-03T08:40:50.190800Z","shell.execute_reply.started":"2021-07-03T08:40:49.335976Z","shell.execute_reply":"2021-07-03T08:40:50.189659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nfrom tqdm import tqdm\nfrom pandas.api.types import is_datetime64_any_dtype as is_datetime\nfrom pandas.api.types import is_categorical_dtype\nfrom tqdm import tqdm_notebook as tqdm\n\ndef reduce_mem_usage(df, use_float16=False):\n    \"\"\" iterate through all the columns of a dataframe and modify the data type\n        to reduce memory usage.        \n    \"\"\"\n    start_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n\n    for col in tqdm(df.columns):\n        if is_datetime(df[col]) or is_categorical_dtype(df[col]) or str(df[col].dtype)=='timedelta64[ns]':\n            # skip datetime type or categorical type\n            continue\n        col_type = df[col].dtype\n\n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)\n            else:\n                if use_float16 and c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        else:\n            #df[col] = df[col].astype('category')\n            df[col] = df[col].astype(str)\n\n    end_mem = df.memory_usage().sum() / 1024**2\n    print ('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n    print ('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n\n    return df","metadata":{"execution":{"iopub.status.busy":"2021-07-03T08:40:50.192191Z","iopub.execute_input":"2021-07-03T08:40:50.192507Z","iopub.status.idle":"2021-07-03T08:40:50.206017Z","shell.execute_reply.started":"2021-07-03T08:40:50.192479Z","shell.execute_reply":"2021-07-03T08:40:50.205031Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def read_data():\n    train = pd.read_pickle('../input/mlb-engagement-preprocessed/train_nextDayPlayerEngagement.pkl')\n    train['date'] = pd.to_datetime(train['engagementMetricsDate']).map(lambda x: x - timedelta(days=1))\n    train.drop(columns=['engagementMetricsDate'], inplace=True)\n    \n    #playerTwitterFollowers = pd.read_pickle('../input/local_eval/train_playerTwitterFollowers.pkl')\n    \n    teams = pd.read_csv('../input/mlb-player-digital-engagement-forecasting/teams.csv')\n    awards = pd.read_csv('../input/mlb-player-digital-engagement-forecasting/awards.csv')\n    seasons = pd.read_csv('../input/mlb-player-digital-engagement-forecasting/seasons.csv')\n    players = pd.read_csv('../input/mlb-player-digital-engagement-forecasting/players.csv')\n    example_test = pd.read_csv('../input/mlb-player-digital-engagement-forecasting/example_test.csv') \n    example_sample_submission = pd.read_csv('../input/mlb-player-digital-engagement-forecasting/example_sample_submission.csv')\n\n    return train, teams, awards, seasons, players, example_test, example_sample_submission\n\ndf, teams, awards, seasons, players, example_test, example_sample_submission = read_data()\ndf","metadata":{"execution":{"iopub.status.busy":"2021-07-03T08:40:50.207797Z","iopub.execute_input":"2021-07-03T08:40:50.208134Z","iopub.status.idle":"2021-07-03T08:41:13.128703Z","shell.execute_reply.started":"2021-07-03T08:40:50.208105Z","shell.execute_reply":"2021-07-03T08:41:13.126194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def create_daily_lag_feature(df, datetime_col, groupby_cols, target_col, lags=[1]):\n    df_copy = df[[datetime_col, target_col] + groupby_cols].copy()\n    new_cols = []\n    for lag in tqdm(lags):\n        new_col = target_col + '_' + '_'.join(groupby_cols) + f'_lag{lag}days'\n        if new_col in df.columns:\n            df.drop(columns=new_col, inplace=True)\n        lag_df = df_copy.set_index(datetime_col).groupby(groupby_cols)[target_col].shift(\n            freq=f'{lag}D').reset_index().rename(columns={target_col: new_col})\n        df = df.merge(lag_df, on=[datetime_col] + groupby_cols, how='left')\n        new_cols.append(new_col)\n    del df_copy\n    gc.collect()\n    return df, new_cols","metadata":{"execution":{"iopub.status.busy":"2021-07-03T08:41:13.131703Z","iopub.execute_input":"2021-07-03T08:41:13.132226Z","iopub.status.idle":"2021-07-03T08:41:13.141564Z","shell.execute_reply.started":"2021-07-03T08:41:13.132173Z","shell.execute_reply":"2021-07-03T08:41:13.140250Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import time\nstart_time = time.time()\n\ntarget_cols = ['target1', 'target2', 'target3', 'target4']\nfor target in target_cols:\n    df, new_cols = create_daily_lag_feature(\n        df=df,\n        datetime_col='date', \n        groupby_cols=['playerId'], \n        target_col=target, \n        lags=[31, 32, 33, 34, 35, 36, 37,]\n    )\n\ndf = reduce_mem_usage(df)\n\ntrain_df = df.reset_index(drop=True).copy()\n\ndel df\ngc.collect()\n\nend_time = time.time()\nprint ('{} sec'.format(end_time-start_time))\n\ntrain_df","metadata":{"execution":{"iopub.status.busy":"2021-07-03T08:41:13.143072Z","iopub.execute_input":"2021-07-03T08:41:13.143468Z","iopub.status.idle":"2021-07-03T08:42:29.880018Z","shell.execute_reply.started":"2021-07-03T08:41:13.143433Z","shell.execute_reply":"2021-07-03T08:42:29.878294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import mlb\nenv = mlb.make_env() # initialize the environment\niter_test = env.iter_test() # iterator which loops over each date in test set\n\ntarget_cols = ['target1', 'target2', 'target3', 'target4']\nrequired_days = 37\n\nfor (test_df, sample_prediction_df) in iter_test:\n    \n    #display(test_df)\n    #display(sample_prediction_df)\n    \n    test_start_date = test_df.index.map(lambda x: datetime.strptime(str(x), '%Y%m%d')).unique()[0]\n    train_df = train_df[train_df['date']<test_start_date]\n    train_start_date = test_start_date - timedelta(days=required_days)\n    train_df = train_df[train_df['date']>=train_start_date]\n    #train_df = train_df.sort_values(['date', 'playerId']).reset_index(drop=True)\n    \n    sub_df = sample_prediction_df.copy()\n    sample_prediction_df = sample_prediction_df.reset_index()\n    sample_prediction_df['date'] = pd.to_datetime(sample_prediction_df['date'].map(lambda x: datetime.strptime(str(x), '%Y%m%d')))\n    sample_prediction_df['playerId'] = sample_prediction_df['date_playerId'].map(lambda x: int(x.split('_')[1]))\n    \n    train_df['test'] = 0\n    sample_prediction_df['test'] = 1\n    df = pd.concat([train_df, sample_prediction_df])\n    df = df.drop_duplicates(['date', 'playerId']).reset_index(drop=True)\n    df = reduce_mem_usage(df)\n    del train_df, sample_prediction_df\n    gc.collect()\n\n    df['playerId'] = df['playerId'].astype(str)\n    df[target_cols] = df[target_cols].astype(np.float64)\n    \n    \"\"\"\n    This following block caused Submission Error(Notebook Threw Exception) only when submission. No error will occur when sample-prediction.\n    If this block is commented out, it does not caused Submission Error, the score will be calculated normally (all zero submission score)\n    \n    Even if lags = [1], it happens same error. So I wonder May dataset has some difference with public dataset.\n    \n    WHY???\n    \"\"\"\n    for target in target_cols:\n        df, new_cols = create_daily_lag_feature(\n            df=df,\n            datetime_col='date', \n            groupby_cols=['playerId'], \n            target_col=target, \n            lags=[31, 32, 33, 34, 35, 36, 37,]\n        )\n    \"\"\"\"\"\"\n        \n        \n    test = df[df['test']==1].reset_index(drop=True)\n    #display(test_df)\n    \n    test_preds = np.zeros((len(test), 4)).astype(np.float64)\n    \n    for seed in tqdm([2021]):\n        for fold in range(1):\n            for i, target in enumerate(target_cols):\n                test_x = test.copy()\n                preds = np.zeros((len(test_x), )).astype(np.float64)\n                test_preds[:, i] += preds / (len([2021]) * 1)\n    \n    train_df = df.copy()\n    del df\n    gc.collect()\n    \n    sub_df[target_cols] = np.clip(test_preds, 0, 100).astype(np.float64)\n    sub_df[target_cols] = sub_df[target_cols].fillna(0.)\n    env.predict(sub_df)\n    gc.collect()\n    #break","metadata":{"execution":{"iopub.status.busy":"2021-07-03T08:42:29.881178Z","iopub.status.idle":"2021-07-03T08:42:29.881626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sub_df","metadata":{"execution":{"iopub.status.busy":"2021-07-03T08:42:29.882771Z","iopub.status.idle":"2021-07-03T08:42:29.883417Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}