{"cells":[{"metadata":{"_uuid":"3c49edc658c81a58f3419b61f1b27e41ab7a42d7"},"cell_type":"markdown","source":"**Here is the overview of the process:**\n\n1. Make a table of group and match-based statistical features (min, max, mean, std, sum, rank)\n\n2. Train a neural net on those features to predict winPlacePerc\n    - I used pure tensorflow in part to get some practice using it, in part to be more able to tune my model.\n    - Some extra features that helped improve the training include:\n        * learning rate is dropped if there's no improvement after N epochs\n        * loss is a hybrid between MSE in the region [0,1] and 2*abs(MAE) for (1,inf)\n        * each term in loss is also multiplied by the group size, so that larger groups have a higer weight.\n\n3. Apply the post-processing trick that was reported in other kernels\n\n4. Drop all match-level stats.\n\n5. Train another neural net that 2 groups from the same match and tries to predict which group has a higher score\n\n6. Run the second net on the results of the first net, and if the second net contradicts the result of the first one, swap the winPlacePerc values.\n\n7. Repeat step 6 several times.\n\n8. Done!\n\n(version 14 of this kernel had the best score before I ran this version)"},{"metadata":{"_uuid":"63f8cc270c3b7abe7f132d2087b84d685c30aa29"},"cell_type":"markdown","source":"# Step 1. Process the data."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nimport re\nimport gc\nimport os\nimport datetime\n\nkernel_start_time = datetime.datetime.now()\n\ntrain_path = '../input/train_V2.csv'\ntest_path = '../input/test_V2.csv'\n\nprint(os.listdir(\"../input\"))\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"# Memory saving function credit to https://www.kaggle.com/gemartin/load-data-reduce-memory-usage\ndef reduce_mem_usage(df):\n    \"\"\" iterate through all the columns of a dataframe and modify the data type\n        to reduce memory usage.        \n    \"\"\"\n    #start_mem = df.memory_usage().sum() / 1024**2\n    #print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n    #print('{')\n    for col in df.columns:\n        col_type = df[col].dtype\n\n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        \n        col_type = str(df[col].dtype)\n        if col_type == 'object':\n            col_type = 'str'\n        else:\n            col_type = 'np.' + col_type    \n        #print('\\'' + col + '\\':' + col_type + ',')\n    #end_mem = df.memory_usage().sum() / 1024**2\n    #print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n    #print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n    #print('}')\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"205f7aabedac6117341c536a5af42d2c527e4ab9"},"cell_type":"code","source":"# optimal column data types\ndata_types = {'Id':str,'groupId':str,'matchId':str,'assists':np.int8,'boosts':np.int8,'damageDealt':np.float16,'DBNOs':np.int8,\n'headshotKills':np.int8,'heals':np.int8,'killPlace':np.int8,'killPoints':np.int16,'kills':np.int8,'killStreaks':np.int8,'longestKill':np.float16,\n'matchDuration':np.int16,'matchType':str,'maxPlace':np.int8,'numGroups':np.int8,'rankPoints':np.int16,'revives':np.int8,'rideDistance':np.float16,\n'roadKills':np.int8,'swimDistance':np.float16,'teamKills':np.int8,'vehicleDestroys':np.int8,'walkDistance':np.float16,'weaponsAcquired':np.int16,\n'winPoints':np.int16,'winPlacePerc':np.float16}\n\ndata_types['walkDistance'] = np.float32\ndata_types['rideDistance'] = np.float32\ndata_types['damageDealt'] = np.float32","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"411af1f8ebdec742e8ae4dc7418932b6df69a3c6"},"cell_type":"code","source":"# most useless features that was determined by LightGBM feature importance (may need to reassess)\nzero_stdev_cols = ['median_match_roadKills', 'median_match_vehicleDestroys', 'median_match_road_kills_per_rideDistance', 'std_match_longestKill']\nulesess_stat_cols = ['median_match_revives','median_match_teamKills','median_match_swimDistance_norm',\n'max_match_kill_streak_rate','max_group_roadKills','min_group_roadKills','median_match_assists',\n'sum_group_roadKills','sum_group_vehicleDestroys','min_group_vehicleDestroys']\nuseless_match_type_names = ['normal-duo','flarefpp','normal-solo','crashtpp','normal-squad','flaretpp','normal-duo-fpp',\n                           'normal-squadfpp', 'normal-squad-fpp', 'crashfpp','duo','squad-fpp','normal-solo-fpp']\n\nall_useless_cols = zero_stdev_cols + ulesess_stat_cols + useless_match_type_names\nall_useless_cols = set(all_useless_cols)\n\nmin_match_useful_cols = ['min_match_rankPoints', 'min_match_winPoints', 'min_match_killPoints']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d55168618bbe6fac143ce8161f285fc536e770bf"},"cell_type":"code","source":"# replace missing ranks\ndef fix_missing_ranks(X, mean_ranks=None, rank_stds=None, rank_cols=['rankPoints', 'winPoints']):\n    if (mean_ranks is None) or (rank_stds is None):\n        mean_ranks = {}\n        rank_stds = {}\n\n        for rank_col in rank_cols:\n            mean_ranks[rank_col] = X.loc[X[rank_col] > 1, rank_col].mean()\n            rank_stds[rank_col] = X.loc[X[rank_col] > 1, rank_col].std()\n    \n    # deviation from the mean value expressed in number of standard deviations\n    rank_deltas = (X.loc[(X['rankPoints'] > 1) & (X['winPoints'] <= 1), 'rankPoints'] - mean_ranks['rankPoints']) / rank_stds['rankPoints']\n    X.loc[(X['rankPoints'] > 1) & (X['winPoints'] <= 1), 'winPoints'] = mean_ranks['winPoints'] + rank_stds['winPoints']*rank_deltas\n    \n    win_deltas = (X.loc[(X['rankPoints'] <= 1) & (X['winPoints'] > 1), 'winPoints'] - mean_ranks['winPoints']) / rank_stds['winPoints']\n    X.loc[(X['rankPoints'] <= 1) & (X['winPoints'] > 1), 'rankPoints'] = mean_ranks['rankPoints'] + rank_stds['rankPoints']*win_deltas\n    \n    X.loc[(X['rankPoints'] <= 1) & (X['winPoints'] <= 1), 'rankPoints'] = mean_ranks['rankPoints']\n    X.loc[(X['rankPoints'] <= 1) & (X['winPoints'] <= 1), 'winPoints'] = mean_ranks['winPoints']\n\n    return X, mean_ranks, rank_stds","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c1cd9288ed34ebb0898ec4cd8a4913ff4ef3bdf3"},"cell_type":"code","source":"# add basic player-level features by combining other features together\ndef add_player_features(X):\n    X['headshot_rate'] = X['headshotKills'] / (X['kills'] + 0.00001)\n    X['kill_streak_rate'] = X['killStreaks'] / (X['kills'] + 0.00001)\n    X['kills_assists'] = X['kills'] + X['assists']\n    X['heals_boosts'] = X['heals'] + X['boosts']\n    X['total_distance'] = X['walkDistance'] + X['rideDistance'] + X['swimDistance']\n    X['kills_assists_per_heal_boost'] = X['kills_assists'] / (X['heals_boosts'] + 1)\n    X['damageDealt_per_heal_boost'] = X['damageDealt'] / (X['heals_boosts'] + 1)\n    X['road_kills_per_rideDistance'] = X['roadKills'] / (X['rideDistance'] + 0.01)\n    X['maxPlace_per_numGroups'] = X['maxPlace'] / X['numGroups']\n    X['assists_per_kill'] = X['assists'] / (X['kills'] + X['assists'] + 0.0001)\n    X['killPlace'] = X['killPlace'] - 1\n    return X","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"27cd11d1c80f91fb1aadba004553b6735011c7f0"},"cell_type":"code","source":"# create some group-level features\n# also creates some additional player-based features\ndef create_basic_group_info(X):\n    group_cols = ['matchId', 'groupId', 'matchDuration', 'matchType', 'maxPlace', 'numGroups', 'maxPlace_per_numGroups', 'winPlacePerc', 'killPlace']\n    if 'winPlacePerc' not in X.columns:\n        group_cols.remove('winPlacePerc')\n        \n    pl_data_grouped = X[group_cols].groupby(['matchId', 'groupId'])\n    gr_data = pl_data_grouped.first()\n    gr_data.drop(columns='killPlace', inplace=True)\n    \n    gr_data['raw_groupSize'] = pl_data_grouped['numGroups'].count()\n    gr_data['groupSize'] = gr_data['raw_groupSize']\n    gr_data['group_size_overflow'] = (gr_data['groupSize'] > 4).astype(np.int8)\n    gr_data.loc[gr_data['groupSize'] > 4, ['groupSize']] = 2 # replace group sizes with median, since it's a bug, max group size is 4\n    \n    \n    gr_data['meanGroupSize'] = gr_data.groupby('matchId')['groupSize'].transform(np.mean)\n    gr_data['medianGroupSize'] = gr_data.groupby('matchId')['groupSize'].transform(np.median)\n    #gr_data['maxGroupSize'] = gr_data.groupby('matchId')['groupSize'].transform(np.max)\n    #gr_data['minGroupSize'] = gr_data.groupby('matchId')['groupSize'].transform(np.min)\n    gr_data['maxKillPlace'] = pl_data_grouped['killPlace'].max().groupby('matchId').transform(np.max)\n    \n    gr_data['totalPlayers'] = gr_data.groupby('matchId')['groupSize'].transform(sum)\n    # some matches have missing players, so I adjust the total number of players to account for that\n    gr_data['totalPlayersAdjusted'] = gr_data['maxPlace'].astype(float) * gr_data['totalPlayers'] / (gr_data['numGroups'] + 0.01)\n    # trim total number to 100 as it can't be higher than that \n    gr_data['totalPlayersAdjusted'] = gr_data['totalPlayersAdjusted'].apply(lambda x: np.minimum(100.0, x))\n    #gr_data.drop(columns=['totalPlayers'], inplace=True)\n    gr_data['num_opponents'] = gr_data['totalPlayersAdjusted'] - gr_data['groupSize']\n    \n    X = X.merge(gr_data[['num_opponents', 'totalPlayersAdjusted', 'groupSize', 'raw_groupSize', 'maxKillPlace']], on=['matchId', 'groupId'])\n    \n    print('group size counts:')\n    print(X['raw_groupSize'].value_counts())\n    \n    # normalizing some features\n    X['revives_per_groupSize'] = X['revives'] / (X['groupSize'] - 1 + 0.001)\n    X['kills_assists_norm_both'] = X['kills_assists'].astype(np.float32) / X['num_opponents'] / X['matchDuration']\n    \n    X['killPlace_norm'] = X['killPlace']/ (X['maxKillPlace'] + 0.000001)\n    \n    #X['kills_assists_norm_opp_n'] = X['kills_assists'].astype(np.float32) / X['num_opponents']\n    #X['kills_assists_norm_dur'] = X['kills_assists'].astype(np.float32) / X['matchDuration']\n    X['damageDealt_norm_both'] = X['damageDealt'].astype(np.float32) / X['num_opponents'] / X['matchDuration']\n    #X['damageDealt_norm_opp_n'] = X['damageDealt'].astype(np.float32) / X['num_opponents']\n    #X['damageDealt_norm_dur'] = X['damageDealt'].astype(np.float32) / X['matchDuration']\n    X['DBNOs_norm'] = X['DBNOs'].astype(np.float32) / X['num_opponents'] / X['matchDuration']\n    X['heals_norm'] = X['heals'].astype(np.float32) / X['matchDuration']\n    X['boosts_norm'] = X['boosts'].astype(np.float32) / X['matchDuration'] # - lowers correlation, don't do\n    X['walkDistance_norm'] = X['walkDistance'].astype(np.float32) / X['matchDuration']\n    X['rideDistance_norm'] = X['rideDistance'].astype(np.float32) / X['matchDuration']\n    X['swimDistance_norm'] = X['swimDistance'].astype(np.float32) / X['matchDuration']\n    \n    #gr_data.drop(columns=['groupSize'], inplace=True)\n    \n    gr_data = reduce_mem_usage(gr_data)\n    gr_data.drop(columns=list(set(gr_data.columns)&all_useless_cols))\n    return gr_data, X","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b82e03224cf27801f23490ccb56659491bb2f90c"},"cell_type":"code","source":"# create group- and match- level stat features. Merge all useful features together while removing useless ones.\ndef create_group_and_match_stats(data, gr_data):\n    \n    group_stats_cols = ['assists','boosts','DBNOs','killPoints','longestKill','rankPoints', 'road_kills_per_rideDistance',\n                        'kills_assists_norm_both', 'damageDealt_norm_both', 'DBNOs_norm', 'heals_boosts', 'assists_per_kill', 'killPlace_norm',\n                        'revives','roadKills','teamKills','vehicleDestroys','weaponsAcquired','winPoints','headshot_rate','kill_streak_rate',\n                        'kills_assists', 'heals_norm','walkDistance_norm','rideDistance_norm','swimDistance_norm',\n                       'damageDealt_per_heal_boost','kills_assists_per_heal_boost']\n    # removed damageDealt\n    match_stats_cols = ['assists','boosts','DBNOs','killPoints','longestKill','rankPoints', 'road_kills_per_rideDistance',\n                        'kills_assists_norm_both', 'damageDealt_norm_both', 'DBNOs_norm', 'heals_boosts', 'assists_per_kill',\n                        'revives','roadKills','teamKills','vehicleDestroys','weaponsAcquired','winPoints','headshot_rate','kill_streak_rate',\n                        'kills_assists', 'heals_norm','walkDistance_norm','rideDistance_norm','swimDistance_norm',\n                       'damageDealt_per_heal_boost','kills_assists_per_heal_boost']\n    \n    pl_data_grouped_by_group = data.groupby(['matchId', 'groupId'])\n    pl_data_grouped_by_match = data.groupby(['matchId'])\n\n    #group_sizes = pl_data_grouped_by_group['groupSize'].count().values.reshape([-1])\n    #fixed_group_sizes = pl_data_grouped_by_group['groupSize'].first().values.reshape([-1])\n    #sum_multipliers = fixed_group_sizes.astype(np.float32) / group_sizes\n    #print('min multiplier: {:.2f}, max multiplier: {:.2f}'.format(np.min(sum_multipliers), np.max(sum_multipliers)))\n    #print('min group size: {:d}, max group size: {:d}'.format(np.min(group_sizes), np.max(group_sizes)))\n    #print('min fixed group size: {:d}, max fixed group size: {:d}'.format(np.min(fixed_group_sizes), np.max(fixed_group_sizes)))\n    #print(pd.Series(sum_multipliers).value_counts())\n    \n    group_funcs = {'min':np.min, 'max':np.max, 'sum':np.sum, 'median': np.mean}\n    match_funcs = {'min':np.min,'max':np.max, 'sum':np.sum, 'median': np.median, 'std':np.std}\n    extra_group_stats = pl_data_grouped_by_group[['matchId', 'groupId']].first().reset_index(drop=True)\n    extra_match_stats = pl_data_grouped_by_match[['matchId']].first().reset_index(drop=True)    \n\n    for colname in group_stats_cols:\n        for f_name, func in group_funcs.items():\n            gr_col_name = f_name + '_group_' + colname\n            if (gr_col_name not in all_useless_cols) or ((gr_col_name + '_rank') not in all_useless_cols):\n                if func is not sum:\n                    extra_group_stats[gr_col_name] = pl_data_grouped_by_group[colname].agg(func).values\n                else:\n                    extra_group_stats[gr_col_name] = pl_data_grouped_by_group[colname].agg(func).values * sum_multipliers\n    \n    for colname in match_stats_cols:    \n        for f_name, func in match_funcs.items():\n            m_col_name = f_name + '_match_' + colname\n            if m_col_name not in all_useless_cols:\n                if func is np.std:\n                    extra_match_stats[m_col_name] = pl_data_grouped_by_match[colname].agg(func).fillna(0).values\n                elif func is np.min:\n                    if m_col_name in min_match_useful_cols:\n                        extra_match_stats[m_col_name] = pl_data_grouped_by_match[colname].agg(func).values\n                else:\n                    extra_match_stats[m_col_name] = pl_data_grouped_by_match[colname].agg(func).values\n            \n            \n    extra_group_stats.set_index(['matchId', 'groupId'], inplace=True)\n    extra_match_stats.set_index(['matchId'], inplace=True)\n\n    pl_data_grouped_by_group = None\n    pl_data_grouped_by_match = None\n\n    select_cols = []\n    for col in extra_group_stats.columns:\n        if ((col + '_rank') not in all_useless_cols) and (col not in ['matchId', 'groupId']):\n            select_cols.append(col)\n    \n    # adding rank information\n    rank_data = extra_group_stats.groupby(['matchId'])\n    rank_data = rank_data[select_cols].rank() - 1 # method='dense'\n    gc.collect()\n    max_rank_data = rank_data.groupby(['matchId']).transform(np.max)\n    rank_data = rank_data / (max_rank_data + 0.0001)\n    max_rank_data = None\n    gc.collect()\n    print('rank data created')\n    \n    gr_col_to_drop = list(set(extra_group_stats.columns) & all_useless_cols)\n    extra_group_stats.drop(columns=gr_col_to_drop, inplace=True)\n    gc.collect()\n\n    extra_group_stats = extra_group_stats.join(rank_data, on=['matchId', 'groupId'], rsuffix='_rank')\n    extra_group_stats.reset_index(level=1, inplace=True) # put groupId back into the columns\n\n    rank_data = None\n    gc.collect()\n    print('rank data merged')\n    extra_group_stats = reduce_mem_usage(extra_group_stats)\n    extra_match_stats = reduce_mem_usage(extra_match_stats)\n\n    merged_features = extra_group_stats.merge(extra_match_stats, on=['matchId'])\n    extra_group_stats = None\n    extra_match_stats = None\n    gc.collect()\n    print('extra match and group stats merged')\n    merged_features = merged_features.merge(gr_data, on=['matchId', 'groupId'])\n    gr_data = None\n    gc.collect()\n    print('group data and stats merged')\n    \n    #one hot encoding of match type\n    cats = merged_features['matchType'].unique()\n    cats = set(cats) - set(all_useless_cols)\n    encoded_data = np.empty(shape=(merged_features.shape[0], 0), dtype=np.int8)\n    for category in cats:\n        encoded_data = np.c_[encoded_data, (merged_features[['matchType']] == category).values.reshape(-1,1).astype(np.int8)]\n    encoded_data = pd.DataFrame(encoded_data, columns=cats, index=merged_features.index, dtype=np.int8)\n    print('matchType data created')\n    for col in encoded_data.columns:\n        merged_features[col] = encoded_data[col]\n    encoded_data = None\n    gc.collect()\n    print('match type data merged')\n    cols_to_drop = ['matchType']\n    merged_features = merged_features.drop(columns=cols_to_drop)\n    \n    return merged_features","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b55661ba55d0b83d295c30bba1833f0181cafca3"},"cell_type":"code","source":"# remove outliers. Also returns data for the outliers. Maye I need to keep them...\ndef remove_outliers(X):\n    outliers = (X['walkDistance'] > 10000) | (X['rideDistance'] > 15000) | (X['swimDistance'] > 1000) | ((X['kills'] > 0) & (X['total_distance'] == 0))\n    outliers = outliers | (X['kills'] > 30) | (X['longestKill'] > 800) | (X['weaponsAcquired'] > 40)\n    X = X.loc[~outliers]\n    outlier_data = X.loc[outliers]\n    return X, outlier_data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"27c041b1e0c06efd71f054beaa18ed96d022f957"},"cell_type":"code","source":"# pipeline for processing the datasets:\n# use fit_transform to prepare the training data\n# use transform to prepare the test data. It will use some data calculated in fit_transform.\nclass DataPipeline():\n    def __init__(self, pipeline=None):\n        if pipeline is not None:\n            self.rank_means = pipeline.rank_means\n            self.rank_stds = pipeline.rank_stds\n        return\n    \n    def fit_transform(self, data_path):\n        data = pd.read_csv(data_path)\n        \n        # reduce memory usage\n        for col_name, col_type in data_types.items():\n            data[col_name] = data[col_name].astype(col_type)\n        \n        # remove crappy data\n        data = data.loc[~data['winPlacePerc'].isna()]\n        data = data.loc[data['maxPlace'] > 1]\n        \n        # remove too large groups\n        #group_sizes = data.groupby(['matchId', 'groupId'])[['maxPlace']].count()\n        #group_sizes.columns = ['groupSize']\n        #data = data.merge(group_sizes, on=['groupId', 'matchId'])\n        #data = data.loc[data['groupSize'] <= 8]\n        #data.drop(columns=['groupSize'], inplace=True)\n        #group_sizes = None\n        \n        data, self.rank_means, self.rank_stds = fix_missing_ranks(data)\n        data = add_player_features(data)\n        gr_data, data = create_basic_group_info(data)\n        \n        #data = remove_outliers(data)[0]\n        cols_to_drop = ['kills', 'headshotKills', 'killStreaks', 'walkDistance', 'rideDistance', 'swimDistance', 'heals']\n        data.drop(columns=cols_to_drop, inplace=True)\n        data_ids = data[['Id', 'matchId', 'groupId']]\n        \n        # generate and combine all features\n        merged_features = create_group_and_match_stats(data, gr_data)\n\n        # removing columns with zero stdev\n        #cols_to_remove = []\n        #for col in merged_features.columns:\n        #    if col != 'winPlacePerc':\n        #        min_val = merged_features[col].min()\n        #        max_val = merged_features[col].max()\n        #        if min_val == max_val:\n        #            cols_to_remove.append(col)\n        #            merged_features.drop(columns=col)\n        #            print('dropped column: ' + col)\n        #self.cols_to_remove = cols_to_remove\n        gc.collect()\n        print('final feature dataframe shape:', merged_features.shape)\n        \n        return merged_features.reset_index(), data_ids\n    \n    def transform(self, data_path):\n        data = pd.read_csv(data_path)\n        \n        # reduce memory usage\n        for col_name, col_type in data_types.items():\n            if col_name != 'winPlacePerc':\n                data[col_name] = data[col_name].astype(col_type)\n        \n        data = fix_missing_ranks(data, self.rank_means, self.rank_stds)[0]\n        data = add_player_features(data)\n        gr_data, data = create_basic_group_info(data)\n        \n        cols_to_drop = ['kills', 'headshotKills', 'killStreaks', 'walkDistance', 'rideDistance', 'swimDistance', 'heals']\n        data.drop(columns=cols_to_drop, inplace=True)\n        data_ids = data[['Id', 'matchId', 'groupId']]\n        \n        # generate and combine all features\n        merged_features = create_group_and_match_stats(data, gr_data)\n\n        # removing columns with zero stdev\n        #for col in self.useless_cols:\n        #    merged_features.drop(columns=col, inplace=True)\n        #    gc.collect()  \n        print('final feature dataframe shape:', merged_features.shape)\n        return merged_features.reset_index(), data_ids","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1644c18426cc0d6c93dd8a40c194193306c21cef"},"cell_type":"code","source":"gc.collect()\npipeline = DataPipeline()\ntrain, initial_ids = pipeline.fit_transform(train_path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"45be68039f59405d92c786a7909c2be57ec97409"},"cell_type":"code","source":"# standard normalization in a memory-efficient way\nmean_vals = pd.Series()\nstd_vals = pd.Series()\n\nfor num, col in enumerate(train.columns):\n    if col not in ['winPlacePerc', 'groupId', 'matchId']:\n        train[col] = train[col].astype(np.float32)\n        mean = train[col].mean()\n        stdev = train[col].std()\n        if stdev == 0:\n            train.drop(columns=col, inplace=True)\n            #print(num, 'drop column:\\t', col)\n        else:\n            train[col] = ((train[col] - mean)/stdev).astype(np.float32)\n            #print(num, '\\tname', col, 'mean', mean, 'stdev', stdev)\n            mean_vals[col] = mean\n            std_vals[col] = stdev\n        gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a63f0f589d5080e34241d7a9af1b03c39c9450cf"},"cell_type":"markdown","source":"# Step 2. Train the first neural net."},{"metadata":{"trusted":true,"_uuid":"641b371cd8957385e0eb3da618203e7eee1a3393"},"cell_type":"code","source":"# memory-efficient function to feed train and validation data into the neural net\ndef get_data_batch(batch_size, randomize=False):\n    if randomize:\n        curr_idx = np.random.permutation(train.shape[0])\n    else:\n        curr_idx = range(train.shape[0])\n    for batch_n in range(int(np.ceil(train.shape[0] / batch_size))):\n        batch_data = train.iloc[curr_idx[batch_n*batch_size:(batch_n+1)*batch_size]]\n        group_sizes = batch_data['groupSize'].values*std_vals['groupSize'] + mean_vals['groupSize']\n        yield batch_data.drop(columns=['winPlacePerc', 'matchId', 'groupId']).values, batch_data['winPlacePerc'].values, group_sizes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1deb6bfa3895cb4a6a739997025dede9911125f1"},"cell_type":"code","source":"import tensorflow as tf\n\ndropout_rate = [0.0, 0.0, 0.0, 0.0, 0.0, 0.5, 0.5]\n\ndef leaky_relu(z, name=None):\n    return tf.maximum(0.01*z, z, name=name)\n\ntf.reset_default_graph()\ndynamic_dropout = tf.placeholder_with_default(0.5, shape=(None), name='dynamic_dropout')\n\nX = tf.placeholder(dtype=tf.float32, shape=[None, train.shape[1]-3], name='X')\ny = tf.placeholder(dtype=tf.float32, shape=(None), name='y')\ngroup_sizes = tf.placeholder(dtype=tf.float32, shape=(None), name='group_sizes')\n\n#X_conv = tf.reshape(X[:,:144], shape=[-1, 2, 72, 1])\n#X_rest = X[:,144:]\n\ntraining = tf.placeholder_with_default(False, shape=(None), name='training') \n\nwith tf.name_scope('layers') as scope:\n    #X_drop = tf.layers.dropout(X, rate=dropout_rate[0], training=training, name='input_dropout')\n\n    hidden1 = tf.layers.dense(X, 100, name='hidden1',\n                             kernel_initializer=tf.variance_scaling_initializer())\n    hidden1 = tf.layers.batch_normalization(hidden1, training=training, momentum=0.9, name='hidden1_bn')\n    #hidden1 = leaky_relu(hidden1, name='hidden1_activation')\n    hidden1 = tf.nn.elu(hidden1, name='hidden1_activation')\n    #hidden1 = tf.layers.dropout(hidden1, rate=dropout_rate[1], training=training, name='dropout1')\n    \n    hidden2 = tf.layers.dense(hidden1, 60, name='hidden2',\n                             kernel_initializer=tf.variance_scaling_initializer())\n    hidden2 = tf.layers.batch_normalization(hidden2, training=training, momentum=0.9, name='hidden2_bn')\n    #hidden2 = leaky_relu(hidden2, name='hidden2_activation')\n    hidden2 = tf.nn.elu(hidden2, name='hidden2_activation')\n    #hidden2 = tf.layers.dropout(hidden2, rate=dropout_rate[2], training=training, name='dropout2')\n    \n    \n    hidden3 = tf.layers.dense(hidden2, 60, name='hidden3',\n                             kernel_initializer=tf.variance_scaling_initializer())\n    hidden3 = tf.layers.batch_normalization(hidden3, training=training, momentum=0.9, name='hidden3_bn')\n    hidden3 = tf.nn.elu(hidden3, name='hidden3_activation')\n    #hidden3 = leaky_relu(hidden3, name='hidden3_activation')\n    #hidden3 = tf.layers.dropout(hidden3, rate=dropout_rate[3], training=training, name='dropout3')\n    \n    hidden4 = tf.layers.dense(hidden3, 60, name='hidden4',\n                             kernel_initializer=tf.variance_scaling_initializer())\n    hidden4 = tf.layers.batch_normalization(hidden4, training=training, momentum=0.9, name='hidden4_bn')\n    hidden4 = tf.nn.elu(hidden4, name='hidden4_activation')\n    #hidden4 = leaky_relu(hidden4, name='hidden4_activation')\n    #hidden4 = tf.layers.dropout(hidden4, rate=dropout_rate[4], training=training, name='dropout4')\n    \n    hidden5 = tf.layers.dense(hidden4, 60, name='hidden5',\n                             kernel_initializer=tf.variance_scaling_initializer())\n    hidden5 = tf.layers.batch_normalization(hidden5, training=training, momentum=0.9, name='hidden5_bn')\n    hidden5 = tf.nn.elu(hidden5, name='hidden5_activation')\n    #hidden5 = leaky_relu(hidden5, name='hidden5_activation')\n    hidden5 = tf.layers.dropout(hidden5, rate=dropout_rate[5], training=training, name='dropout5')\n    \n    #hidden6 = tf.layers.dense(hidden5, 30, name='hidden6',\n    #                         kernel_initializer=tf.variance_scaling_initializer())\n    #hidden6 = tf.layers.batch_normalization(hidden6, training=training, momentum=0.9, name='hidden6_bn')\n    #hidden6 = tf.nn.elu(hidden6, name='hidden5_activation')\n    #hidden6 = tf.layers.dropout(hidden6, rate=dropout_rate[6], training=training, name='dropout6')\n    \n    output = tf.layers.dense(hidden5, 1, name='output',\n                             kernel_initializer=tf.variance_scaling_initializer())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1b5dc45499c08efd1e151ceee832c5403479608c"},"cell_type":"code","source":"with tf.name_scope('loss') as scope:\n    # MSE is the loss here, although the final score is determined based on MAE\n    diff_vector = tf.abs(tf.reshape(output, shape=[-1]) - y)\n    sum_group_sizes = tf.reduce_sum(group_sizes)\n    loss_mse = tf.reduce_sum(tf.square(diff_vector)*group_sizes)/sum_group_sizes\n    loss_mae = tf.reduce_sum(diff_vector*group_sizes)/sum_group_sizes\n    clipped_diff_vector = tf.clip_by_value(diff_vector, 0.0, 1.0)\n    lin_err = 2*(diff_vector - clipped_diff_vector)\n    loss_hybrid = tf.reduce_sum((tf.square(clipped_diff_vector) + lin_err)*group_sizes)/sum_group_sizes\n    #loss_hybrid = tf.reduce_mean(tf.square(clipped_diff_vector) + lin_err)\n\nwith tf.name_scope('training') as scope:\n    lr_high = 0.0008\n    lr_low = 0.00001\n    learning_rate = tf.Variable(0.0001, trainable=False, name='learning_rate')\n    optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate, beta1=0.6)\n    training_op = optimizer.minimize(loss_hybrid)\n\nval_batch_size = 10000\n\ndef get_mse_mae(sess, batch_size=10000):\n    y_diffs = np.empty(shape=0)\n    curr_group_sizes = np.empty(shape=0)\n    for X_batch, y_batch, batch_gr_sizes in get_data_batch(batch_size=batch_size):\n        curr_output_vals = sess.run(output, feed_dict = {X:X_batch}).reshape(-1)\n        y_diffs = np.r_[y_diffs, np.abs(y_batch - curr_output_vals)]\n        curr_group_sizes = np.r_[curr_group_sizes, batch_gr_sizes]\n    loss_mae_val = np.sum(y_diffs*curr_group_sizes)/np.sum(curr_group_sizes)\n    loss_mse_val = np.sum((y_diffs**2)*curr_group_sizes)/np.sum(curr_group_sizes)\n    #loss_mae_val = np.mean(y_diffs)\n    #loss_mse_val = np.mean(y_diffs**2)\n    return loss_mse_val, loss_mae_val\n    \nextra_training_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)\n\ninit = tf.global_variables_initializer()\nsaver = tf.train.Saver()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"de67bc7583c33e2e9c71d07f68fa38400025d68e"},"cell_type":"code","source":"gc.collect()\n\nbatch_size = 5000\nn_epochs = 1000\nmax_epochs_wo_improvement = 50\nmax_epochs_wo_lr_change = 10\nmax_time = 2 # hours\n\nloss_data = {'mse_train':[], 'mae_train':[], 'lr':[]}\n\nstart_time = datetime.datetime.now()\n\ndef get_dropout(initial_dropout, epoch):\n    return np.maximum(0, initial_dropout*(1 - epoch/200))\n\nwith tf.Session() as sess:\n    init.run()\n    min_loss_mae = np.inf\n    epochs_wo_improvement = 0\n    epochs_since_lr_change = 0\n    batch_lr = lr_high\n    \n    for epoch in range(n_epochs):\n        for X_batch, y_batch, batch_gr_sizes in get_data_batch(batch_size=batch_size, randomize=True):\n            feed_dict = {X: X_batch, y: y_batch, training:True, learning_rate:batch_lr, group_sizes:batch_gr_sizes}\n            sess.run([training_op, extra_training_ops], feed_dict=feed_dict)\n\n        loss_mse_train, loss_mae_train = get_mse_mae(sess)\n        loss_data['mse_train'].append(loss_mse_train)\n        loss_data['mae_train'].append(loss_mae_train)\n        loss_data['lr'].append(batch_lr)\n        \n        print('epoch', epoch, 'time passed:', (datetime.datetime.now() - start_time), 'learning rate: {:<5.5f}'.format(batch_lr)) #'learning rate:', sess.run(learning_rate))\n        print('MSE:', loss_mse_train, 'MAE:', loss_mae_train)\n        \n        if loss_mae_train <= min_loss_mae:\n            epochs_wo_improvement = 0\n            epochs_since_lr_change = 0\n            min_loss_mae = loss_mae_train\n            save_path = saver.save(sess, '../DNN_data/dnn_state.ckpt')\n            print('- best so far!')\n        else:\n            epochs_wo_improvement += 1\n            epochs_since_lr_change += 1\n        \n            if (epochs_wo_improvement > max_epochs_wo_improvement) or ((datetime.datetime.now() - start_time).seconds > max_time*60*60):\n                print('early breaking!')\n                break\n\n            if epochs_since_lr_change >= max_epochs_wo_lr_change:\n                    batch_lr = batch_lr - (batch_lr - lr_low)*0.3\n                    epochs_since_lr_change = 0\n                    #saver.restore(sess, save_path)\n\n    saver.restore(sess, save_path)\n    loss_mse_val, loss_mae_val = get_mse_mae(sess)\n    print('Final MSE:', loss_mse_val, 'final MAE:', loss_mae_val)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1da19a89c1c71f6522fe1ec26f0c812f473616c0"},"cell_type":"code","source":"# plot MAE and MSE vs. epoch\nplt.figure(figsize=[20,9])\nplt.subplot(3,1,1)\nplt.plot(loss_data['mse_train'], label='mse_train')\nplt.legend()\n\nplt.subplot(3,1,2)\nplt.plot(loss_data['mae_train'], label='mae_train')\nplt.ylim([0.02,0.045])\n#plt.ylim([0,plt.ylim()[1]])\nplt.legend()\n\nplt.subplot(3,1,3)\nplt.plot(loss_data['lr'], label='learning rate')\nplt.ylim([0,plt.ylim()[1]])\nplt.legend()\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"022aa6cad26a0bb3e0ea8673919dd919ed325e1d"},"cell_type":"markdown","source":"# Step 3. Train the second neural net to compare 2 groups."},{"metadata":{"trusted":true,"_uuid":"9414e22bdaeef4282166f48a5847cc2bef06e16c"},"cell_type":"code","source":"# select columns for the second DNN\ngroup_cols = [col for col in train.columns if re.search(r'group', col) is not None]\ngroup_cols = [col for col in group_cols if re.search(r'_rank', col) is None]\ngroup_cols.remove('groupId')\nother_cols = [col for col in train.columns if re.search(r'match|group', col) is None]\nother_cols.remove('num_opponents')\nother_cols.remove('winPlacePerc')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4476ee704f1e7ecddcca512dba35b0abf80f776b"},"cell_type":"code","source":"def get_match_comp_indices(x, gaps):\n    gap_lim = x.shape[0]\n    gaps = [gap for gap in gaps if gap < gap_lim]\n    if len(gaps) > 0:\n        x = x.sort_values(by=['winPlacePerc'])\n        results = []\n        for gap in gaps:\n            result = np.empty(shape=[x.shape[0]-gap, 2], dtype=np.int32)\n            result[:,0] = np.array(x.iloc[gap:].index).astype(np.int32)\n            result[:,1] = np.array(x.iloc[:-gap].index).astype(np.int32)\n            results.append(result)\n        return np.vstack(results)\n    else:\n        return np.empty(shape=[0,2], dtype=np.int32)\n\ndef get_comparator_indices(X, gap=1):\n    result = X[['matchId', 'winPlacePerc']].groupby(['matchId']).apply(lambda x: get_match_comp_indices(x, gap))\n    print('stage 1 complete')\n    result = np.concatenate(result, axis=0)\n    return result","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0d6a59b6e1c0fddbfd20239b3d3617ce7c0f4918"},"cell_type":"code","source":"# For training I select groups that are winthin 4 winPlacePerc ranks.\ngaps = [i for i in range(1, 4)]\ntrain_comp_inds = get_comparator_indices(train, gaps)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f65602a7b49e3bfbbd8bac431507a6b0b62028e9"},"cell_type":"code","source":"train_array = train.drop(columns=['matchId', 'groupId', 'winPlacePerc']).values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4c71660d93edb4e625f9fa929f8d6d9373130b89"},"cell_type":"code","source":"group_col_inds = [train.columns.drop(['matchId', 'groupId', 'winPlacePerc']).get_loc(col) for col in group_cols]\nother_col_inds = [train.columns.drop(['matchId', 'groupId', 'winPlacePerc']).get_loc(col) for col in other_cols]\n\ndef get_comp_batch(inds, batch_size, omit_last=True):\n    rand_inds = np.random.permutation(inds.shape[0])\n    if omit_last:\n        func = np.floor\n    else:\n        func = np.ceil\n    for batch_n in range(int(func(inds.shape[0]/batch_size))):\n        batch_inds = inds[rand_inds[batch_n*batch_size:(batch_n+1)*batch_size]]\n        X1_batch = train_array[batch_inds[:,0]][:,group_col_inds]\n        X2_batch = train_array[batch_inds[:,1]][:,group_col_inds]\n        X3_batch = train_array[batch_inds[:,0]][:,other_col_inds]\n        yield X1_batch, X2_batch, X3_batch","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2b80309ed7dd8c14f95fea0009ce5365c9977324"},"cell_type":"code","source":"dropout_rate = [0.0, 0.0, 0.0, 0.0, 0.1, 0.2, 0.2, 0.2]\nlayer_sizes = [100, 100, 60, 60, 60, 30, 30, 30]\n\ntf.reset_default_graph()\n\ndef leaky_relu(z, name=None):\n    return tf.maximum(0.01*z, z, name=name)\n\n#X = tf.placeholder(dtype=tf.float32, shape=[None, (len(group_cols)*2 + len(other_cols))], name='X')\n#X = tf.placeholder(dtype=tf.float32, shape=[None, (len(group_cols) + len(other_cols)), 2], name='X')\nX1 = tf.placeholder(dtype=tf.float32, shape=[None, len(group_cols)], name='X1')\nX2 = tf.placeholder(dtype=tf.float32, shape=[None, len(group_cols)], name='X2')\nX3 = tf.placeholder(dtype=tf.float32, shape=[None, len(other_cols)], name='X3')\n\nX = tf.concat([tf.concat([X1, X2, X3], axis=1), tf.concat([X2, X1, X3], axis=1)], axis=0)\n\ny = tf.placeholder(dtype=tf.float32, shape=(None), name='y')\ny_all = tf.concat([y, 1-y], axis=0)\n\ntraining = tf.placeholder_with_default(False, shape=(None), name='training') \n\nwith tf.name_scope('layers') as scope:\n    hidden = tf.layers.dropout(X, rate=dropout_rate[0], training=training, name='input_dropout')\n    \n    for i in range(8):\n        hidden = tf.layers.dense(hidden, layer_sizes[i], name=f'hidden_{i}',\n                                 kernel_initializer=tf.contrib.layers.variance_scaling_initializer())\n        hidden = tf.layers.batch_normalization(hidden, training=training, momentum=0.9, name=f'hidden_{i}_bn')\n        hidden = tf.nn.elu(hidden, name=f'hidden_{i}_activation')\n        hidden = leaky_relu(hidden, name=f'hidden_{i}_activation')\n        hidden = tf.layers.dropout(hidden, rate=dropout_rate[i], training=training, name=f'dropout_{i}')\n       \n    logits = tf.layers.dense(hidden, 1, name='logits',\n                             kernel_initializer=tf.contrib.layers.variance_scaling_initializer())\n    output = tf.nn.sigmoid(logits, name='output')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bc0bcd015c439358b0fa56f4d803bc486036a2de"},"cell_type":"code","source":"with tf.name_scope('loss') as scope:\n    # MSE is the loss here, although the final score is determined based on MAE\n    xentropy = tf.nn.sigmoid_cross_entropy_with_logits(labels=y_all,logits=tf.reshape(logits, [-1]),name='xentropy')\n    loss = tf.reduce_mean(xentropy)\n\nwith tf.name_scope('training') as scope:\n    #global_step = tf.train.global_step()\n    lr_low = 0.00001\n    lr_high = 0.003\n    lr_high_2 = 0.0003\n    decay_rate = 15\n    lr = tf.Variable(lr_high, trainable=False, name='learning_rate')\n    optimizer = tf.train.AdamOptimizer(learning_rate=lr, epsilon=1e-10)\n    #grads_and_vars = optimizer.compute_gradients(loss_mse)\n    #gradients = [grad for grad, var in grads_and_vars]\n    #names = [var.name for grad, var in grads_and_vars]\n    training_op = optimizer.minimize(loss)\n\nwith tf.name_scope('evaluation') as scope:\n    accuracy = tf.reduce_mean(tf.cast(tf.equal(tf.round(tf.reshape(output,[-1])),y_all), tf.float32), name='accuracy')\n\ndef get_full_set_accuracy(sess, inds, batch_size=1000):\n    y_batch = np.ones(shape=[batch_size])\n    y_for_comp = np.r_[y_batch, 1 - y_batch]\n    acc_data = np.empty(shape=0)\n    for X1_batch, X2_batch, X3_batch in get_comp_batch(inds=inds, batch_size=batch_size):\n        #X1_batch = train.loc[batch_inds[:,0], group_cols].values\n        #X2_batch = train.loc[batch_inds[:,1], group_cols].values\n        #X3_batch = train.loc[batch_inds[:,0], other_cols].values\n        feed_dict = {X1:X1_batch, X2:X2_batch, X3:X3_batch, y:y_batch}\n        y_pred = sess.run(output, feed_dict=feed_dict).reshape(-1)\n        batch_acc_data = ((y_pred > 0.5).astype(np.float32) == y_for_comp).astype(np.float32)\n        acc_data = np.r_[acc_data, batch_acc_data]\n    return np.mean(acc_data)\n    \nextra_training_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)\n\ninit = tf.global_variables_initializer()\nsaver_comp = tf.train.Saver()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"543527c9d6fa1f437174c6e315126474d76ab174"},"cell_type":"code","source":"import gc\nimport datetime\ngc.collect()\n\nbatch_size = 500\nn_epochs = 1000\nmax_epochs_without_improving = 50\nmax_epochs_wo_lr_change = 7\nmax_time = 5.5\n\nacc_data = {'acc_train':[], 'acc_val':[]}\nbatch_acc_data = []\nbatch_lr_data = []\n\nstart_time = datetime.datetime.now()\n\nwith tf.Session() as sess:\n    init.run()\n    max_val_acc = 0\n    epochs_wo_improvement = 0\n    epochs_since_lr_update = 0\n    gap = 1\n    #batches_per_epoch = int(len(train_match_ids) / batch_size)\n    #period = 1 # epochs\n    #period_batches = period * batches_per_epoch\n    batch_lr = lr_high\n    y_batch = np.ones(shape=[batch_size])\n    \n    for epoch in range(n_epochs):\n        n_batch = 0\n        \n        if epoch == 15:\n            batch_size = 1000\n            y_batch = np.ones(shape=[batch_size])\n            max_epochs_without_improving = 100\n            max_epochs_wo_lr_change = 10\n        if epoch == 25:\n            batch_size = 2000\n            y_batch = np.ones(shape=[batch_size])\n        if epoch == 40:\n            batch_size = 4000\n            y_batch = np.ones(shape=[batch_size])\n            max_epochs_wo_lr_change = 15\n        if epoch == 60:\n            batch_size = 8000\n            y_batch = np.ones(shape=[batch_size])\n            max_epochs_wo_lr_change = 20\n        \n        #curr_lr_high = lr_high_2 + (lr_high-lr_high_2)*np.exp(-(epoch - (epoch%period))/decay_rate)\n        for X1_batch, X2_batch, X3_batch in get_comp_batch(inds=train_comp_inds, batch_size=batch_size):\n        #for X_batch, y_batch in get_comparator_batch_2(train, train_match_ids, batch_size, gap):\n            #print(n_batch)\n            #if n_batch > batches_per_epoch:\n            #    batches_per_epoch = n_batch\n                \n            #curr_batch = (epoch % period)*batches_per_epoch + n_batch\n            #coeff = curr_batch/period_batches\n            #coeff = np.minimum(coeff, 1-coeff)\n            #coeff = 2*np.clip(coeff, 0, 0.5)\n            #batch_lr = lr_low + (curr_lr_high-lr_low)*(coeff)\n            \n            #print(X_batch.shape, y_batch.shape)\n            \n            #X1_batch = train.loc[inds[:,0], group_cols].values\n            #X2_batch = train.loc[inds[:,1], group_cols].values\n            #X3_batch = train.loc[inds[:,0], other_cols].values\n            feed_dict = {X1:X1_batch, X2:X2_batch, X3:X3_batch, y:y_batch}\n            \n            batch_lr_data.append(batch_lr)\n            batch_acc_data.append(sess.run(accuracy, feed_dict=feed_dict))\n            \n            feed_dict = {X1:X1_batch, X2:X2_batch, X3:X3_batch, y:y_batch, training:True, lr:batch_lr}\n            sess.run([training_op, extra_training_ops], feed_dict=feed_dict)\n            n_batch += 1\n        if (epoch >= 20) and (epoch % 1 == 0):\n            curr_ids = np.random.permutation(train_comp_inds.shape[0])[:100000]\n            train_acc = get_full_set_accuracy(sess, train_comp_inds[curr_ids], batch_size=10000)\n            #curr_ids = np.random.permutation(val_comp_inds.shape[0])[:10000]\n            #val_acc = get_full_set_accuracy(sess, val_comp_inds[curr_ids], batch_size=1000)\n            acc_data['acc_train'].append(train_acc)\n            #acc_data['acc_val'].append(val_acc)\n\n            print('epoch', epoch, 'time passed:', (datetime.datetime.now() - start_time), 'max lr:', batch_lr)\n            print('Accuracy: train:', train_acc)\n\n            if train_acc > max_val_acc:\n                epochs_wo_improvement = 0\n                epochs_since_lr_update = 0\n                max_val_acc = train_acc\n                save_path_comp = saver_comp.save(sess, '../DNN_data/dnn_state_comp.ckpt')\n                print('- best so far!')\n            else:\n                epochs_wo_improvement += 1\n                epochs_since_lr_update += 1\n                \n                if (epochs_wo_improvement > max_epochs_without_improving) or ((datetime.datetime.now() - kernel_start_time).seconds/60/60 > max_time):\n                    print('early breaking!')\n                    break\n                \n                if epochs_since_lr_update >= max_epochs_wo_lr_change:\n                    batch_lr = batch_lr - (batch_lr-lr_high_2)*0.33\n                    epochs_since_lr_update = 0\n                    # restore the best state before lowering the learning rate\n                    #saver_comp.restore(sess, save_path_comp)\n                    \n    saver_comp.restore(sess, save_path_comp)\n    curr_ids = np.random.permutation(train_comp_inds.shape[0])[:100000]\n    train_acc = get_full_set_accuracy(sess, train_comp_inds[curr_ids], batch_size=10000)\n    #val_acc = get_full_set_accuracy(sess, val_match_ids[:10000], batch_size=1000, gap=1)\n    print('Final accuracy: train:', train_acc)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ad41c7c528464d22b5cf45b3404127a371f5b320"},"cell_type":"code","source":"plt.figure(figsize=[15,10])\nplt.subplot(2,1,1)\nplt.plot(batch_acc_data)\n\nplt.subplot(2,1,2)\nplt.plot(batch_lr_data)\nplt.ylim([0, plt.ylim()[1]])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6c369b4744a6811a7a2ec2f95020c0a4f11b1ee6"},"cell_type":"code","source":"#plot accuracy vs batch number\nplt.figure(figsize=[10,6])\nplt.plot(acc_data['acc_train'])\nplt.grid()\nplt.title('Train accuracy') \nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"221fbdba7095e2848f6074332303ae2ec04ac150"},"cell_type":"markdown","source":"# Step 4. Swap the groups' predicted winPlacePercs if the second neural net predicts that they should be swapped"},{"metadata":{"trusted":true,"_uuid":"694bd9bb331debc51bd07adb1c96e8b9f518a273"},"cell_type":"code","source":"# functions for generating data for the second DNN to rearrange the group placements\ndef generate_val_comp_data(X, gap):\n    if X.shape[0] > gap:\n        X = X.sort_values(by=['winPlacePerc'])\n        group_data = X[group_cols].values\n        part0 = X[['matchId', 'groupId']].values[gap:,:]\n        part0 = np.c_[part0, X[['matchId', 'groupId']].values[:-gap,:]]\n        part1 = group_data[gap:,:]\n        part2 = group_data[:-gap,:]\n        part3 = X[other_cols].values[:-gap,:]\n        return part0, part1, part2, part3\n    return np.empty(shape=(0, 4)), np.empty(shape=(0, len(group_cols))), np.empty(shape=(0, len(group_cols))), np.empty(shape=(0, len(other_cols)))\n\ndef get_val_batch(data, batch_size=1000, gap=1):\n    matchIds = data['matchId'].unique().tolist()\n    for i in range(int(np.ceil(len(matchIds)/batch_size))):\n        match_ids_batch = matchIds[i*batch_size:(i+1)*batch_size]\n        grouped_data = data.loc[data['matchId'].isin(match_ids_batch)].groupby('matchId').apply(lambda x: generate_val_comp_data(x, gap))\n        part0, part1, part2, part3 = list(map(np.vstack, zip(*grouped_data)))\n        yield part0, part1, part2, part3","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"24abf1fdd27b5e0669eb336615a62bc039613111"},"cell_type":"code","source":"# reorder the groups placement\ndef reorder_val(data, ids, y_comp_pred):    \n    df = pd.DataFrame(np.c_[ids, y_comp_pred.reshape(-1,1)], columns=['matchId', 'groupId', 'ordered'])\n    data = data.merge(df, on=['matchId', 'groupId'], how='left')\n    data['ordered'].fillna(1, inplace=True)\n    data.sort_values(by=['matchId','winPlacePerc'], inplace=True)\n    zero_inds = np.nonzero(data['ordered'].values == 0)[0]\n    \n    i = 0\n    old_order = []\n    new_order = []\n    wPP_col_num = data.columns.get_loc('winPlacePerc')\n    while i < len(zero_inds):\n        start = zero_inds[i]\n        while (i+1 < len(zero_inds)) and (zero_inds[i+1] - zero_inds[i] == 1):\n            i += 1\n        end = zero_inds[i]\n        length = end - start + 1\n        i += 1\n        old_order.extend([i for i in range(start-1, start+length, 1)])\n        new_order.extend([i for i in range(start+length-1, start-2, -1)])\n        \n    data.iloc[old_order, wPP_col_num] = data.iloc[new_order, wPP_col_num].values\n\n    print('rows swapped')\n    data.drop(columns=['ordered'], inplace=True)\n    \n    return data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"65213b59c2652bd366682d280fc301f493d68589"},"cell_type":"code","source":"train = None\ntrain_idx = None\ntrain_ids = None\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f0ca9bbac4162b3f39483010a1d4a3496ef0ddf5"},"cell_type":"code","source":"test, test_ids = pipeline.transform(test_path)\n\n# standard normalization in a memory-efficient way\nfor num, col in enumerate(test.columns):\n    if col not in ['winPlacePerc', 'groupId', 'matchId']:\n        test[col] = test[col].astype(np.float32)\n        test[col] = ((test[col] - mean_vals[col])/std_vals[col]).astype(np.float32)\n        gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fe0b53c903b4de8f33eb443524968c9e3ef768e0"},"cell_type":"code","source":"# y_val is a data frame containing columns: \n# [matchId, groupId, winPlacePerc, winPlacePerc_pred]\ny_test_dnn = pd.DataFrame(np.zeros(shape=[1,3]), columns=['matchId', 'groupId', 'winPlacePerc_pred'])\npredict_batch_size = 10000\ntf.reset_default_graph()\nsaver = tf.train.import_meta_graph('../DNN_data/dnn_state.ckpt.meta')\n\nX = tf.get_default_graph().get_tensor_by_name('X:0')\noutput = tf.get_default_graph().get_tensor_by_name('layers/output/BiasAdd:0')\n\ninit = tf.global_variables_initializer()\n\n\nwith tf.Session() as sess:\n    init.run()\n    saver.restore(sess, '../DNN_data/dnn_state.ckpt')\n    n_batches = int(np.ceil(test.shape[0]/predict_batch_size))\n    for batch_n in range(n_batches):\n        data_batch = test.iloc[batch_n*predict_batch_size:(batch_n+1)*predict_batch_size]\n        X_batch = data_batch.drop(columns=['matchId','groupId'])\n        y_batch = data_batch[['matchId','groupId']]\n        y_batch['winPlacePerc_pred'] = output.eval(session=sess, feed_dict={X: X_batch})\n        y_test_dnn = y_test_dnn.append(y_batch, ignore_index=True)\ny_test_dnn = y_test_dnn.iloc[1:,:]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"13ec629955df17d99702ed78af0eef22d2e41c5c"},"cell_type":"code","source":"# aligning winPlacePerc based on the predicted rank and numGroups\ndef rank_align_predictions(X, y, scaled=False):\n    X['winPlacePerc'] = y\n    X['rank'] = X.groupby(['matchId'])['winPlacePerc'].rank(method='dense')\n    X['max_rank'] = X.groupby(['matchId'])['rank'].transform(np.max)\n    adj_winPlacePerc = (X['rank'] - 1) / (X['max_rank'] - 1 + 0.0000000001)\n    X.drop(columns=['winPlacePerc', 'rank', 'max_rank'], inplace=True)\n    return adj_winPlacePerc\n\n#further align predictions based on maxPlace\ndef fix_predictions(X, y, scaled=False):\n    y = y.copy()\n    y[y > 1.0] = 1.0\n    y[y < 0.0] = 0.0\n    \n    if scaled:\n        max_places = X['maxPlace'].values*std_vals['maxPlace'] + mean_vals['maxPlace']\n        num_groups = X['numGroups'].values*std_vals['numGroups'] + mean_vals['numGroups']\n    else:\n        max_places = X['maxPlace'].values\n        num_groups = X['numGroups'].values\n    \n    multiplier = (max_places[max_places > 1] - 1).astype(np.float32)\n    y[max_places > 1] = np.round(y[max_places > 1] * multiplier) / multiplier\n    y[max_places == 1] = 1.0\n    y[max_places <= 0] = 0.0\n    y[num_groups <= 1] = 0.0\n\n    return y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"be80c3d034ff9676908c7632f1e36d792bce2045"},"cell_type":"code","source":"# align and fix predictions\ny_test_dnn['winPlacePerc_pred'] = rank_align_predictions(y_test_dnn.drop(columns=['winPlacePerc_pred']), y_test_dnn['winPlacePerc_pred'])\ny_test_dnn = y_test_dnn.merge(test[['matchId', 'groupId', 'numGroups', 'maxPlace']], on=['matchId', 'groupId'])\ny_test_dnn['winPlacePerc_pred'] = fix_predictions(y_test_dnn.drop(columns=['winPlacePerc_pred']), y_test_dnn['winPlacePerc_pred'], scaled=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0c1f6399f5f6cdea816f382fd92900103dfc3b75"},"cell_type":"code","source":"test = test.merge(y_test_dnn[['matchId', 'groupId', 'winPlacePerc_pred']], on=['matchId', 'groupId'])\ntest.rename(columns={'winPlacePerc_pred':'winPlacePerc'}, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"747a4b6cc5ac239984609f00f7b9d6d19899c75c"},"cell_type":"code","source":"n_epochs = 6\nbatch_size = 10000\ncomp_thresh = [-0.1, 0, -0.1]\n\ny_comp_diff_data = {}\ny_comp_pred_data = {}\n\ntf.reset_default_graph()\nsaver_comp = tf.train.import_meta_graph('../DNN_data/dnn_state_comp.ckpt.meta')\n\nX1 = tf.get_default_graph().get_tensor_by_name('X1:0')\nX2 = tf.get_default_graph().get_tensor_by_name('X2:0')\nX3 = tf.get_default_graph().get_tensor_by_name('X3:0')\n\noutput = tf.get_default_graph().get_tensor_by_name('layers/output:0')\n\ninit = tf.global_variables_initializer()\n\nwith tf.Session() as sess:\n    saver_comp.restore(sess, save_path_comp)\n    \n    for epoch in range(n_epochs):\n        y_comp_pred = np.empty(shape=[0])\n        y_comp_diff = np.empty(shape=[0])\n        all_ids = np.empty(shape=[0, 2])\n        for batch_ids, X1_batch, X2_batch, X3_batch in get_val_batch(test, batch_size):\n            feed_dict = {X1:X1_batch, X2:X2_batch, X3:X3_batch}\n            batch_output = output.eval(feed_dict=feed_dict).reshape(-1)\n            assert len(batch_output) % 2 ==0\n            y_batch_pred = batch_output[:int(len(batch_output)/2)]\n            y_batch_comp = batch_output[:int(len(batch_output)/2)] - batch_output[int(len(batch_output)/2):]\n            \n            y_comp_pred = np.r_[y_comp_pred, y_batch_pred]\n            y_comp_diff = np.r_[y_comp_diff, y_batch_comp]\n            all_ids = np.r_[all_ids, batch_ids[:,:2]]\n        \n        y_comp_pred_data[epoch] = y_comp_pred\n        y_comp_diff_data[epoch] = y_comp_diff\n        thresh_ind = np.minimum(len(comp_thresh)-1, epoch)\n        y_comp_diff = y_comp_diff > comp_thresh[thresh_ind]\n        print('epoch:', epoch, 'number of zeros:', len(np.nonzero(y_comp_diff == 0)[0]))\n        \n        if np.min(y_comp_diff) == 0:\n            test = reorder_val(test, all_ids, y_comp_diff)\n        else:\n            break","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"06631f44e73246d37aedee6d9edfaab87c0bcb42"},"cell_type":"code","source":"a = 0.9\nfor epoch in y_comp_pred_data.keys():\n    plt.figure(figsize=[15,4])\n    plt.suptitle('EPOCH {:d}'.format(epoch))\n    y_diff = y_comp_diff_data[epoch]\n    y_pred = y_comp_pred_data[epoch]\n    \n    plt.subplot(1,2,1)\n    counts, bins, _ = plt.hist(y_diff, label='y_diff', alpha=1, bins=100, range=[-a, +a], stacked=False, histtype='stepfilled')\n    plt.plot([0,0], plt.ylim(), 'k--')\n    plt.grid()\n    plt.legend()\n    plt.xlabel('y_comp_diff')\n    plt.ylabel('count')\n\n    plt.subplot(1,2,2)\n    counts2, bins2, _ = plt.hist(y_pred, label='y_pred', alpha=1, bins=100, range=[0.5-a/2, 0.5+a/2], stacked=False, histtype='stepfilled')\n    plt.plot([0.5,0.5], plt.ylim(), 'k--')\n    plt.grid()\n    plt.legend()\n    plt.xlabel('y_comp_predicted')\n    plt.ylabel('count')\n    \n    plt.tight_layout()\n    \n    plt.show()\n    ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a48efbd238c776bd4ece03c6e4f0fcfabd5d1d9b"},"cell_type":"markdown","source":"# Step 5. Submission"},{"metadata":{"trusted":true,"_uuid":"b5f6ba67c60536af9d157b9b6e530ffc44c45577"},"cell_type":"code","source":"def create_submission_table(X, y, id_table):\n    out = X[['matchId', 'groupId']]\n    out['winPlacePerc'] = y\n    out = id_table.merge(out, on=['matchId', 'groupId'])\n    out = out.drop(columns=['groupId', 'matchId'])\n    return out","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6bcee1d81b10cf9ee5fb0e096fdcce083dd45834"},"cell_type":"code","source":"submission = create_submission_table(test, test['winPlacePerc'], test_ids)\nsubmission.to_csv('submission.csv', index=False)\nsubmission.head(50)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ea26a4c46300e4257ed8c03fc8b53440e300a5e4"},"cell_type":"code","source":"submission.shape","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}