{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import gc\nimport sys\nimport warnings\nfrom joblib import Parallel, delayed\nfrom pathlib import Path\n\nimport ipywidgets as widgets\nimport matplotlib.pyplot as plt\nimport matplotlib.dates as dates\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport time\n\nglobal_time = time.time()\n\nwarnings.simplefilter(\"ignore\")\n\n# Set Matplotlib defaults\nplt.style.use(\"seaborn-whitegrid\")\nplt.rc(\"figure\", autolayout=True, figsize=(11, 5))\nplt.rc(\n    \"axes\",\n    labelweight=\"bold\",\n    labelsize=\"large\",\n    titleweight=\"bold\",\n    titlesize=14,\n    titlepad=10,\n)\nplot_params = dict(\n    color=\"0.75\",\n    style=\".-\",\n    markeredgecolor=\"0.25\",\n    markerfacecolor=\"0.25\",\n    legend=False,\n)","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":7.400126,"end_time":"2021-06-18T03:20:09.162691","exception":false,"start_time":"2021-06-18T03:20:01.762565","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-07-30T23:03:30.380824Z","iopub.execute_input":"2021-07-30T23:03:30.381147Z","iopub.status.idle":"2021-07-30T23:03:31.132571Z","shell.execute_reply.started":"2021-07-30T23:03:30.381071Z","shell.execute_reply":"2021-07-30T23:03:31.131657Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Helper function to unpack json found in daily data\ndef unpack_json(json_str):\n    return pd.DataFrame() if pd.isna(json_str) else pd.read_json(json_str)\n\n\ndef unpack_data(data, dfs=None, n_jobs=-1):\n    if dfs is not None:\n        data = data.loc[:, dfs]\n    unnested_dfs = {}\n    for name, column in data.iteritems():\n        daily_dfs = Parallel(n_jobs=n_jobs)(\n            delayed(unpack_json)(item) for date, item in column.iteritems())\n        df = pd.concat(daily_dfs)\n        unnested_dfs[name] = df\n    return unnested_dfs","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.032719,"end_time":"2021-06-18T03:20:09.355306","exception":false,"start_time":"2021-06-18T03:20:09.322587","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-07-30T23:03:31.134229Z","iopub.execute_input":"2021-07-30T23:03:31.134544Z","iopub.status.idle":"2021-07-30T23:03:31.142839Z","shell.execute_reply.started":"2021-07-30T23:03:31.134509Z","shell.execute_reply":"2021-07-30T23:03:31.142052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_dir = Path('../input/mlb-player-digital-engagement-forecasting/')\n\ndf_names = ['seasons', 'teams', 'players', 'awards']\n\nfor name in df_names:\n    globals()[name] = pd.read_csv(data_dir / f\"{name}.csv\")\n\nkaggle_data_tabs = widgets.Tab()\n# Add Output widgets for each pandas DF as tabs' children\nkaggle_data_tabs.children = list([widgets.Output() for df_name in df_names])\n\n\n\n# display(kaggle_data_tabs)","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.243274,"end_time":"2021-06-18T03:20:09.665119","exception":false,"start_time":"2021-06-18T03:20:09.421845","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-07-30T23:03:31.145305Z","iopub.execute_input":"2021-07-30T23:03:31.145579Z","iopub.status.idle":"2021-07-30T23:03:31.235664Z","shell.execute_reply.started":"2021-07-30T23:03:31.145548Z","shell.execute_reply":"2021-07-30T23:03:31.234889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import time\nglobal_time = time.time()\nmax_sec = 5*3600 ","metadata":{"execution":{"iopub.status.busy":"2021-07-30T23:03:31.237237Z","iopub.execute_input":"2021-07-30T23:03:31.237780Z","iopub.status.idle":"2021-07-30T23:03:31.241868Z","shell.execute_reply.started":"2021-07-30T23:03:31.237744Z","shell.execute_reply":"2021-07-30T23:03:31.241033Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# Define dataframes to load from training set\ndfs = [\n    'nextDayPlayerEngagement',  # targets\n    'playerBoxScores',  # features\n    # Other dataframes available for features:\n    'games',\n    'rosters',\n    # 'teamBoxScores',\n    # 'transactions',\n    # 'standings',\n    # 'awards',\n    # 'events',\n    'playerTwitterFollowers',\n    # 'teamTwitterFollowers',\n]\n\n# Read training data\ntraining = pd.read_csv(\n    data_dir / 'train_updated.csv',\n    usecols=['date'] + dfs,\n)\n\n# Convert training data date field to datetime type\ntraining['date'] = pd.to_datetime(training['date'], format=\"%Y%m%d\")\ntraining = training.set_index('date').to_period('D')\n# print(training.info())","metadata":{"_kg_hide-input":false,"papermill":{"duration":64.466592,"end_time":"2021-06-18T03:21:14.203064","exception":false,"start_time":"2021-06-18T03:20:09.736472","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2021-07-30T23:03:31.243437Z","iopub.execute_input":"2021-07-30T23:03:31.244234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%time\n# Unpack nested dataframes and store in dictionary `training_dfs`\ntraining_dfs = unpack_data(training, dfs=dfs)\nprint('\\n', training_dfs.keys())","metadata":{"papermill":{"duration":26.236092,"end_time":"2021-06-18T03:21:40.462615","exception":false,"start_time":"2021-06-18T03:21:14.226523","status":"completed"},"tags":[],"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def add_agg_feats(df,infer=True):\n    f = 1\n    if infer:\n        f = 0.78\n    df['num_games'] = f*df.groupby('gameDate')['home'].transform(lambda x: 1 - (x!=x).mean()).fillna(0)\n    # df['num_runs']  = f2*df.groupby('gameDate')['homeRuns'].transform('mean')\n    return df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"info_feats = ['gameDate','playerId']\ncat_feats_base = ['positionCode','teamId','statusCode','home']\ncat_feats = ['positionCode','teamId','statusCode','month']\nfeats = ['num_games','numberOfFollowers','diff', 'flyOuts',\n        'groundOuts', 'runsScored', 'doubles', 'triples', 'homeRuns',\n        'strikeOuts', 'baseOnBalls', 'intentionalWalks', 'hits', 'hitByPitch',\n        'atBats', 'caughtStealing', 'stolenBases', 'groundIntoDoublePlay',\n        'plateAppearances', 'totalBases', 'rbi',\n        'leftOnBase', 'sacBunts', 'sacFlies', 'catchersInterference',\n        'pickoffs', 'gamesPlayedPitching', 'gamesStartedPitching',\n        'completeGamesPitching', 'shutoutsPitching', 'winsPitching',\n        'lossesPitching', 'flyOutsPitching', 'airOutsPitching',\n        'groundOutsPitching', 'runsPitching', 'doublesPitching',\n        'triplesPitching', 'homeRunsPitching', 'strikeOutsPitching',\n        'baseOnBallsPitching', 'intentionalWalksPitching', 'hitsPitching',\n        'hitByPitchPitching', 'atBatsPitching', 'caughtStealingPitching',\n        'stolenBasesPitching', 'inningsPitched', 'saveOpportunities',\n        'earnedRuns', 'battersFaced', 'outsPitching', 'pitchesThrown', 'balls',\n        'strikes', 'hitBatsmen', 'balks', 'wildPitches', 'pickoffsPitching',\n        'rbiPitching', 'gamesFinishedPitching', 'inheritedRunners',\n        'inheritedRunnersScored', 'catchersInterferencePitching',\n        'sacBuntsPitching', 'sacFliesPitching', 'saves', 'holds', 'blownSaves',\n        'assists', 'putOuts', 'errors', 'chances']\n\ntar_cols = ['target1','target2','target3','target4']\n\nfeat_std = {'num_games': 0.2,\n     'numberOfFollowers': 2.0203003106791173,\n     'diff': 4.5429028238121605,\n     'flyOuts': 0.5994564164812675,\n     'groundOuts': 0.880031028873754,\n     'runsScored': 0.6354249280932919,\n     'doubles': 0.3742551786113168,\n     'triples': 0.2,\n     'homeRuns': 0.32624555685689094,\n     'strikeOuts': 0.8406402289572776,\n     'baseOnBalls': 0.5371370684035869,\n     'intentionalWalks': 0.2,\n     'hits': 0.8583428342850036,\n     'hitByPitch': 0.2,\n     'atBats': 1.6478300549887628,\n     'caughtStealing': 0.2,\n     'stolenBases': 0.21019790941512004,\n     'groundIntoDoublePlay': 0.23986575244480057,\n     'plateAppearances': 1.7689265657741993,\n     'totalBases': 1.7100020604796016,\n     'rbi': 0.7721294644017079,\n     'leftOnBase': 1.3728075259808412,\n     'sacBunts': 0.2,\n     'sacFlies': 0.2,\n     'catchersInterference': 0.2,\n     'pickoffs': 0.2,\n     'gamesPlayedPitching': 0.2,\n     'gamesStartedPitching': 0.4190423436473443,\n     'completeGamesPitching': 0.2,\n     'shutoutsPitching': 0.2,\n     'winsPitching': 0.33595393562809456,\n     'lossesPitching': 0.3351618729886152,\n     'flyOutsPitching': 1.3295187236130297,\n     'airOutsPitching': 2.3056747952654946,\n     'groundOutsPitching': 2.3446545443246705,\n     'runsPitching': 1.6076997348681283,\n     'doublesPitching': 0.7169228004951924,\n     'triplesPitching': 0.2,\n     'homeRunsPitching': 0.6054885071757616,\n     'strikeOutsPitching': 2.2404138070954094,\n     'baseOnBallsPitching': 1.0345850662281675,\n     'intentionalWalksPitching': 0.2,\n     'hitsPitching': 2.2465570005332784,\n     'hitByPitchPitching': 0.31987897336605925,\n     'atBatsPitching': 7.387801783173313,\n     'caughtStealingPitching': 0.203205178873725,\n     'stolenBasesPitching': 0.38069606988055466,\n     'inningsPitched': 1.9883795796409067,\n     'saveOpportunities': 0.28274591448900555,\n     'earnedRuns': 1.5271776986732541,\n     'battersFaced': 8.091299548597068,\n     'outsPitching': 5.895419109629862,\n     'pitchesThrown': 31.36018795374351,\n     'balls': 11.643084300453825,\n     'strikes': 20.21716920309324,\n     'hitBatsmen': 0.31987897336605925,\n     'balks': 0.2,\n     'wildPitches': 0.3076736135218784,\n     'pickoffsPitching': 0.2,\n     'rbiPitching': 1.5291092688313006,\n     'gamesFinishedPitching': 0.41729463178456044,\n     'inheritedRunners': 0.7408924572465002,\n     'inheritedRunnersScored': 0.4191755714981905,\n     'catchersInterferencePitching': 0.2,\n     'sacBuntsPitching': 0.2,\n     'sacFliesPitching': 0.23749239476534356,\n     'saves': 0.2515259923185149,\n     'holds': 0.35140201229516216,\n     'blownSaves': 0.2,\n     'assists': 1.2426148963769565,\n     'putOuts': 3.090758413548205,\n     'errors': 0.2244738592201668,\n     'chances': 3.2862113104708066}\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from datetime import timedelta\n\ndef merge_df(test_df,dfs,sort=False,infer=True):\n    test_df['gameDate'] = test_df['engagementMetricsDate'].astype('datetime64[ns]')\n    if not infer:\n        test_df['gameDate'] = test_df['gameDate'] - timedelta(days=1)\n    dfs['playerTwitterFollowers']['gameDate'] = dfs['playerTwitterFollowers']['date'].astype('datetime64[ns]')\n    for key in dfs:\n        dfs[key]['gameDate'] = dfs[key]['gameDate'].astype('datetime64[ns]')\n    df_train = dfs['playerBoxScores'].merge(dfs['games'],how='outer',on = ['gameDate','gamePk'])\n    df_train['diff'] = (df_train['homeScore'] - df_train['awayScore']).fillna(0).values\n    df_train = df_train.merge(dfs['rosters'],how='outer',on = ['gameDate','teamId','playerId'])\n    df_train = df_train.merge(dfs['playerTwitterFollowers'],how='outer',on = ['gameDate','playerId'])\n    \n    df_train = add_agg_feats(df_train,infer)\n    df_train = df_train[info_feats + cat_feats_base + feats]\n    ###\n    df_train = df_train.groupby(['gameDate','playerId']).head(1)\n    ###\n    df_train = test_df.merge(df_train,how='left',on = ['gameDate','playerId'])\n    if sort:\n        df_train = df_train.sort_values('gameDate')\n    ### \n    df_train['numberOfFollowers'] = df_train['numberOfFollowers'].values/100000\n    df_train['month'] = df_train['gameDate'].apply(lambda x: int(x.month)).values.astype('int')\n    \n    for f,s in feat_std.items():\n        df_train[f] = df_train[f].values/s\n\n    return df_train\n\n\ndf_train = merge_df(training_dfs['nextDayPlayerEngagement'],{key:df for key,df in training_dfs.items() if key!='nextDayPlayerEngagement'},sort=True,infer=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in range(1,5):\n    df_train[f'agg_tars_{i}'] = df_train.groupby('gameDate')[tar_cols[i-1]].transform('mean').fillna(0)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gc\n\nfor key in training_dfs:\n    training_dfs[key] = training_dfs[key].head()\n    \ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mapper = {}\nfor cat in cat_feats:\n    mapper_ = df_train[cat].unique()\n    mapper_ = {n:k for k,n in enumerate(mapper_)}\n    mapper[cat] = mapper_\nprint(mapper)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mapper = {'home':{np.nan:0,0:1,1:1},\n        'positionCode': {np.nan: 0, 11.0: 1, 1.0: 2, 8.0: 3, 9.0: 4, 3.0: 5, 7.0: 6, 2.0: 7, 10.0: 8, 4.0: 9, 6.0: 10, 5.0: 11, 12.0: 12}, \n        'teamId': { np.nan: 0,119.0: 1, 115.0: 2, 120.0: 3, 145.0: 4, 118.0: 5, 134.0: 6, 139.0: 7, 135.0: 8, 111.0: 9, 140.0: 10, 121.0: 11, 143.0: 12, 109.0: 13, 147.0: 14, 117.0: 15, 146.0: 16, 158.0: 17, 112.0: 18, 133.0: 19, 141.0: 20, 142.0: 21, 114.0: 22, 108.0: 23, 136.0: 24, 144.0: 25, 110.0: 26, 138.0: 27, 113.0: 28, 137.0: 29, 116.0: 30, 159.0: 31, 160.0: 32}, \n        'statusCode': {np.nan: 0,'A': 1, 'D60': 2, 'RM': 3, 'D10': 4, 'D7': 5, 'PL': 6, 'SU': 7, 'FME': 8, 'BRV': 9, 'RES': 10, 'DEC': 11}, \n        'month': {1: 0, 2: 1, 3: 2, 4: 3, 5: 4, 6: 5, 7: 6, 8: 7, 9: 8, 10: 9, 11: 10, 12: 11}}\n\ndef cat_transform(df):\n    for cat in cat_feats:\n        map_ = mapper[cat]\n        df[cat] = df[cat].map(map_).fillna(0).astype(int)\n    return df\n\ndf_train = cat_transform(df_train)\n\nNUM_CATS = [df_train[c].max()+1 for c in cat_feats]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.reset_index(drop=True, inplace=True)\ndf_train.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import math\nimport torch\nfrom torch.utils.data import Dataset, DataLoader\nimport random\n\nNUM = 32\nNUM_1 = 3\n\nBATCH_SIZE  = 64\nUNITS = 128\n\nclass MLBdata(Dataset):\n    def __init__(self,data,ind,infer=False):\n        self.infer = infer\n        self.ind = np.arange(len(ind))[ind]\n        self.data = data\n        self.date = data['gameDate'].values\n        self.feats = torch.from_numpy(data[feats].fillna(-1).values).float()\n        self.cat_feats = data[cat_feats].values.astype('int')\n        self.num_cats = NUM_CATS\n        self.tars = torch.from_numpy(data[tar_cols].fillna(0).values).float()\n        self.player_idx = data.groupby('playerId').groups\n        self.player_idx = {g:list(l) for g,l in self.player_idx.items()}\n        self.agg_tars = torch.from_numpy(data[[f'agg_tars_{k}' for k in range(1,5)]].values).float()\n                \n    def __len__(self):\n        if not self.infer:\n            return int(8*len(self.ind)/BATCH_SIZE)\n        else:\n            return len(self.ind)\n\n        \n    def __getitem__(self,idx):\n        if not self.infer:\n            idx_0 = random.choice(self.ind)\n        else:\n            idx_0 = self.ind[idx]\n        ind = self.player_idx[self.data.loc[idx_0,'playerId']] # all ind of player\n        time_Ok = False\n        while not time_Ok:\n            if self.infer:\n                idx = ind.index(idx_0)\n            else:\n                idx = random.randint(NUM*NUM_1,len(ind)-1)  \n            idx_1 = random.randint(1,min(len(ind),NUM))\n            time_Ok = True\n            \n        num = NUM_1*NUM\n\n        ind_1 = np.array(ind)[max(0,idx+1-num):idx+1] # all\n        ind_2 = ind[max(0,idx+1-num):idx+1-idx_1] # tar range        \n        ###\n        # date = np.concatenate([np.zeros(1),np.diff(self.date[ind_1]).astype('float')/(3600*24*1e9)]).astype('int')\n\n        ###\n        feats = self.feats[ind_1]\n        ###\n        tar_feats = self.tars[ind_2]\n        ###\n        tar_feats_1 = torch.cat([tar_feats.mean(0),tar_feats.median(0)[0]],-1)\n        tar_feats_1 = tar_feats_1[None,...] + 0*torch.zeros(len(feats),1)\n        ###\n        tar_feats = torch.cat([self.tars[ind_2],0*tar_feats[:,:1],self.agg_tars[ind_2]],-1)\n        tar_pad = torch.zeros((len(feats)-len(tar_feats),9))\n        tar_pad[:,4] = 1\n        tar_feats = torch.cat([torch.log(1+tar_feats),tar_pad],0)\n        feats = torch.cat([tar_feats,feats,tar_feats_1],-1)\n        ###\n        \n        cat_feats = []\n        for k,num in enumerate(self.num_cats):\n            cat_feats.append(torch.eye(num)[self.cat_feats[ind_1,k]]) \n\n        cat_feats = torch.cat(cat_feats,-1)\n        ###\n        tars = self.tars[ind_1[-NUM:]]\n        \n        if len(feats) < NUM:\n            d_num = NUM-len(feats)\n            feats = torch.cat([torch.zeros(d_num,feats.size(1)),feats])\n            cat_feats = torch.cat([torch.zeros(d_num,cat_feats.size(1)),cat_feats])\n            tars = torch.cat([-torch.ones(d_num,tars.size(1)),tars])\n\n        tars[:-idx_1,:] = -1\n\n        return feats,cat_feats,tars\n                        \nx,x1,t = MLBdata(df_train,np.ones(len(df_train))==1)[10000]\nNUM_F,NUM_C = x.shape[1],x1.shape[1]\nlen(MLBdata(df_train,np.ones(len(df_train))==1))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from torch import nn\nimport torch.nn.functional as F\n\n\nclass ResBlock(nn.Module):\n    def __init__(self,in_units,units=None):\n        super().__init__()\n        \n        if units is None:\n            units = 2*in_units\n        \n        # self.norm = nn.LayerNorm(normalized_shape=in_units, eps=1e-12) # nn.BatchNorm1d(in_units)\n        self.hidden = nn.Linear(in_units,units) \n        self.out = nn.Linear(units,in_units) \n    \n    def forward(self,x0):\n        x = F.leaky_relu(self.hidden(x0))    # self.norm(x0)))\n        x = F.leaky_relu(x0 + self.out(x))\n        return x\n    \n    \nclass Wave_Block(nn.Module):\n    def __init__(self, in_channels=UNITS, out_channels=UNITS, dilation_rates=6, kernel_size=2):\n        super(Wave_Block, self).__init__()\n        self.num_rates = dilation_rates\n        self.convs = nn.ModuleList()\n        self.filter_convs = nn.ModuleList()\n        self.gate_convs = nn.ModuleList()\n\n        self.convs.append(nn.Conv1d(in_channels, out_channels, kernel_size=1))\n        dilation_rates = [2 ** i for i in range(dilation_rates)]\n        for dilation_rate in dilation_rates:\n            self.filter_convs.append(\n                nn.Conv1d(out_channels, out_channels, kernel_size=kernel_size, padding=0, dilation=dilation_rate))\n            self.gate_convs.append(\n                nn.Conv1d(out_channels, out_channels, kernel_size=kernel_size, padding=0, dilation=dilation_rate))\n            self.convs.append(nn.Conv1d(out_channels, out_channels, kernel_size=1))\n\n    def forward(self, x):\n        b,s,u = x.size()\n        x = self.convs[0](x.transpose(1,2))\n        res = x\n        for i in range(self.num_rates):\n            x = F.leaky_relu(self.filter_convs[i](x)) * torch.sigmoid(self.gate_convs[i](x))\n            x = F.leaky_relu(self.convs[i + 1](x))\n            res = res[...,-x.size(-1):] + x\n        res = res.transpose(1,2)\n\n        return res\n    \n    \nclass MLB_Model(nn.Module):\n    def __init__(self,units):\n        super().__init__()\n        \n        num_c = 24\n        num_process = 2\n        drop_0 = .12\n        drop_1 = .16\n        \n\n        self.in_lay = nn.Linear(NUM_F,NUM_F)\n        self.in_lay_c = nn.Linear(NUM_C,num_c)\n        \n        self.merge = nn.Linear(NUM_F+num_c,units)\n        self.process = nn.Sequential(*[ResBlock(units) for _ in range(num_process)])\n        \n        self.drop_0 = nn.Dropout(drop_0)\n        self.drop_1 = nn.Dropout(drop_1)\n        \n        # \n        # self.wave = Wave_Block() \n        self.rnn = nn.GRU(units,units,batch_first=True) # LayerNormGRU(units,units) \n                \n        self.out_1 = nn.Linear(2*units,2*units)\n        self.out_2 = nn.Linear(2*units,2*units)\n        self.out_3 = nn.Linear(2*units,4)\n        \n    def forward(self,x,x_c):\n        x_0 = F.leaky_relu(self.in_lay(x))\n        x_1 = F.leaky_relu(self.in_lay_c(x_c))\n        x = F.leaky_relu(self.merge(torch.cat([x_0,x_1],-1)))\n        \n        x = self.drop_0(x)\n        # b,s,u = x.size()\n        x = self.process(x) # .reshape(-1,u)).reshape(b,s,u)\n        ###\n        # x_0 = self.wave(x)\n        x_0 = self.rnn(x)[0]\n        x = torch.cat([x[:,-NUM:,:],x_0[:,-NUM:,:]],-1)\n\n        x = self.drop_1(x)\n        ###\n        x = F.leaky_relu(self.out_1(x))\n        x = F.leaky_relu(self.out_2(x))\n        x = F.relu(self.out_3(x))\n        \n        return x\n        \n        \n# MLB_Model(UNITS)(x[None,],x1[None,]).shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nclass Trainer():\n    def __init__(self,data,pretrained=None):\n        val_ind = df_train['gameDate'].apply(lambda x: (x.year==2021) & (x.month==5))\n\n        train_ind = ~val_ind\n        self.data_t = MLBdata(data,train_ind)\n        self.data_v = MLBdata(data,val_ind,True)\n        self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n        self.model = MLB_Model(UNITS).to(self.device)\n        if pretrained is not None:\n            self.model.load_state_dict(torch.load(pretrained))\n        \n    def train_step(self,batch,val=False): \n        t = batch[-1].to(self.device)\n        batch = [x.to(self.device) for x in batch[:-1]]\n    \n        if val:\n            with torch.no_grad():\n                p = self.model(*batch)\n            return nn.L1Loss()(p[:,-1,:],t[:,-1,:]).item(),p[:,-1,:],t[:,-1,:]\n        \n        self.opt.zero_grad()\n        p = self.model(*batch)\n        mask = t!=-1\n        loss = torch.abs(p[mask]-t[mask]).mean()\n        loss.backward()\n        # if random.randint(0,100)==0:\n        #     plot_grad_flow(self.model.named_parameters())\n        \n        l = loss.item()\n        torch.nn.utils.clip_grad_norm_(self.model.parameters(), 1.,norm_type='inf')\n        self.opt.step()\n        return l\n        \n    def train_one_epoch(self,loader,val=False):\n        if val:\n            self.infer = True\n            self.preds = []\n            self.tars = []\n        else:\n            self.infer = False\n            \n        losses = []\n        self.iter = 0\n        for b in loader:\n            loss = self.train_step(b,val)\n            if self.infer:\n                self.preds.append(loss[1].detach().cpu().numpy())\n                self.tars.append(loss[2].detach().cpu().numpy())\n                loss = loss[0]\n            losses.append(loss)\n            if time.time() - global_time > max_sec:\n                self.end = True\n                break\n        if self.infer:\n            self.preds = np.concatenate(self.preds)\n            self.tars = np.concatenate(self.tars)\n        \n        return losses\n    \n    def validate(self):\n        self.model.eval()\n        loader = DataLoader(self.data_v,batch_size=BATCH_SIZE)\n        losses = self.train_one_epoch(loader,True)\n        print('Validation loss:',np.array(losses).mean())\n        self.model.train()\n        return np.array(losses).mean()\n    \n    def train(self,epochs,lr=2e-3,end_lr=5e-4):\n        self.num_epochs = epochs\n        self.best_loss = 100\n        self.model_paths = {}\n        self.end = False\n        \n        fact = (end_lr/lr)**(1/(1+epochs))\n        print(fact)\n        self.opt = torch.optim.Adam(self.model.parameters(),lr=lr,betas=(0.9,0.95),weight_decay=6e-6)\n        loader = DataLoader(self.data_t,batch_size=BATCH_SIZE,shuffle=True)\n        \n        for ep in range(epochs):\n            losses = self.train_one_epoch(loader)\n            plt.plot(losses)\n            plt.show()\n            print(f'loss for epoch {ep}:',np.array(losses).mean())\n            val_loss = self.validate()\n            # if (ep >= self.num_epochs-10):\n            path = f'weights_{ep}.path'\n            torch.save(self.model.state_dict(),path)\n            self.model_paths[path] = val_loss\n            if val_loss < self.best_loss:\n                self.best_loss = val_loss\n            lr *= fact\n            for g in self.opt.param_groups:\n                g['lr'] = lr\n            if self.end:\n                break\n        paths = sorted(self.model_paths.items(),key=lambda x: x[1],reverse=False)\n        self.model_paths = [p for p,_ in paths[:6]]\n        print(self.model_paths)\n                \ntr = Trainer(df_train,pretrained=None) # './weights_2.path')\n\ntr.train(30,lr=1.6e-3,end_lr=8e-4)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Evaluator():\n    def __init__(self,models,data):\n        self.data = data\n        self.models = models\n        self.dfs = [d for d in dfs if d!='nextDayPlayerEngagement']\n        self.num_cats = NUM_CATS\n        self.tars = torch.from_numpy(data[tar_cols].values).float()\n        self.update()\n        self.player_idx_0 = self.data.groupby('playerId').groups\n        self.player_idx_0 = {g:list(l) for g,l in self.player_idx.items()}\n        self.agg_tars = torch.from_numpy(data[[f'agg_tars_{k}' for k in range(1,5)]].values).float()\n        self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n        \n    def read_json(self,samp_df,test_df):\n        dfs_test = {}\n        for key in self.dfs:\n            if test_df[key].iloc[0] == test_df[key].iloc[0]:\n                df_temp = pd.read_json((test_df[key].iloc[0]))\n            else:\n                df_temp = pd.DataFrame({'playerId': samp_df['playerId']})\n                for col in  training_dfs[key].keys():\n                    if 'date' in col.lower():\n                        df_temp[col] = samp_df['engagementMetricsDate'].values\n                        continue\n                    elif col == 'playerId': \n                        continue\n                    df_temp[col] = np.nan\n            dfs_test[key] = df_temp\n        return dfs_test\n        \n    def unpack(self,samp_sub,test_dfs):\n        test_dfs = self.read_json(samp_sub,test_dfs)\n        pred_df = merge_df(samp_sub,test_dfs).reset_index(drop=True)\n        pred_df = cat_transform(pred_df)\n        self.start_index = self.data.index.max()\n        pred_df.index += self.start_index\n        self.data = pd.concat([self.data,pred_df]).reset_index(drop=True)\n        self.start_index += 1\n        self.ind = self.start_index + np.arange(len(pred_df))\n        \n    def update(self):\n        self.date = self.data['gameDate'].values\n        self.feats = torch.from_numpy(self.data[feats].fillna(-1).values).float()\n        self.cat_feats = self.data[cat_feats].values.astype('int')\n        self.player_idx = self.data.groupby('playerId').groups\n        self.player_idx = {g:list(l) for g,l in self.player_idx.items()}\n            \n    def __getitem__(self,idx):\n        idx_0 = self.ind[idx]\n        id_ = self.data.loc[idx_0,'playerId']\n        ind = self.player_idx[id_] # ind updated data\n        ind_0 = self.player_idx_0[id_] # ind old data\n        \n        delta = len(ind) - len(ind_0)\n\n        \n        num = NUM_1*NUM \n        ##\n        ind_1 = ind[-num:]\n        ind_2 = ind_0[delta-num:]\n        ##########\n        feats = self.feats[ind_1]\n        ###\n        tar_feats = self.tars[ind_2]\n        tar_feats_1 = torch.cat([tar_feats.mean(0),tar_feats.median(0)[0]],-1)\n        tar_feats_1 = tar_feats_1[None,...] + 0*torch.zeros(len(feats),1)\n        ###\n        tar_feats = torch.cat([self.tars[ind_2],0*tar_feats[:,:1],self.agg_tars[ind_2]],-1)\n        tar_pad = torch.zeros((len(feats)-len(tar_feats),9))\n        tar_pad[:,4] = 1\n        tar_feats = torch.cat([torch.log(1+tar_feats),tar_pad],0)\n        feats = torch.cat([tar_feats,feats,tar_feats_1],-1)\n        ###\n        \n        cat_feats = []\n        for k,num in enumerate(self.num_cats):\n            cat_feats.append(torch.eye(num)[self.cat_feats[ind_1,k]]) \n        cat_feats = torch.cat(cat_feats,-1)\n\n        return feats,cat_feats\n    \n    def make_preds(self):\n        preds = []\n        # print(self.ind)\n        # print(self.start_index)\n        for idx in range(len(self.ind)):\n            inp = self.__getitem__(idx)\n            inp = [x[None,...].to(self.device) for x in inp]\n            p = 0\n            for m in self.models:\n                with torch.no_grad():\n                    p += (m(*inp).cpu().numpy()/len(self.models))[:,-1,:]\n            preds.append(p)\n        preds = np.concatenate(preds,0)\n        \n        return preds\n    \n    def infer(self,sub_df,test_inp):\n        test_inp = self.unpack(sub_df,test_inp)\n        self.update()\n        pred = self.make_preds()\n        # preds /= np.arange(len(deltas)+1).sum()\n        self.pred_df = pd.DataFrame(pred,columns=['target1','target2','target3','target4'])\n        return self.pred_df\n        \n        \n# E = Evaluator([tr.model],df_train.head(100))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Create Submission #","metadata":{"papermill":{"duration":0.188102,"end_time":"2021-06-18T03:23:22.630419","exception":false,"start_time":"2021-06-18T03:23:22.442317","status":"completed"},"tags":[]}},{"cell_type":"code","source":"%%time\nimport mlb\nimport gc\n\nmodels = []\nfor p in tr.model_paths:\n    model = MLB_Model(UNITS).to(tr.device).eval()\n    model.load_state_dict(torch.load(p))\n    models.append(model)\n    \ndel tr\ngc.collect()\n\ndf_train = df_train.tail(1000000).reset_index(drop=True)\nE = Evaluator(models,df_train)\nenv = mlb.make_env()\niter_test = env.iter_test()\nstep = 0\nfor (test_df, sample_prediction_df) in iter_test:\n    # Unpack features from test_df\n    print(step)\n    gc.collect()\n    step += 1\n    # print(sample_prediction_df.head())\n    samp_df = sample_prediction_df.reset_index(drop=True).copy()\n    samp_df['engagementMetricsDate'] = sample_prediction_df.index.astype(str)\n    samp_df['engagementMetricsDate']  = samp_df['engagementMetricsDate'] .astype('datetime64[ns]')\n    \n    # print(samp_df.head())\n    samp_df['playerId'] = samp_df['date_playerId'].map(lambda x: int(x.split('_')[1]))\n    \n    pred_df = E.infer(samp_df,test_df)\n\n    sample_prediction_df.loc[:,tar_cols] =  np.clip(pred_df.loc[:,tar_cols].values, 0, 100)\n    \n    print(sample_prediction_df[['target1','target2','target3','target4']].mean())\n\n    # Submit predictions\n    env.predict(sample_prediction_df)  # constructs submissions.csv","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}