{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"%load_ext autoreload\n%autoreload 2\n\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"88229e833719ad9f043c143fc1006bef22952538"},"cell_type":"code","source":"import os\nfrom pandas_summary import DataFrameSummary\nfrom sklearn.ensemble import RandomForestRegressor, RandomForestClassifier\nfrom IPython.display import display\n\nfrom sklearn import metrics\nfrom sklearn_pandas import DataFrameMapper\nfrom sklearn.preprocessing import LabelEncoder, Imputer, StandardScaler\nfrom pandas.api.types import is_string_dtype, is_numeric_dtype\nfrom sklearn.ensemble import forest\nfrom sklearn.tree import export_graphviz\nimport IPython, graphviz, re","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"dba1d33191e179943360f486cee7386bc7e0468e"},"cell_type":"markdown","source":"# **Helper functions**"},{"metadata":{"_kg_hide-input":true,"trusted":true,"_uuid":"4f4eda724671dc9d1e84fc5d9a5edaf7b095cc1a"},"cell_type":"code","source":"def draw_tree(t, df, size=10, ratio=0.6, precision=0):\n    \"\"\" Draws a representation of a random forest in IPython.\n\n    Parameters:\n    -----------\n    t: The tree you wish to draw\n    df: The data used to train the tree. This is used to get the names of the features.\n    \"\"\"\n    s=export_graphviz(t, out_file=None, feature_names=df.columns, filled=True,\n                      special_characters=True, rotate=True, precision=precision)\n    IPython.display.display(graphviz.Source(re.sub('Tree {',\n       f'Tree {{ size={size}; ratio={ratio}', s)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5d11aedb4242c1ba02c61b647eccfdcee3f9c20c","_kg_hide-input":true},"cell_type":"code","source":"def get_sample(df,n):\n    \"\"\" Gets a random sample of n rows from df, without replacement.\n\n    Parameters:\n    -----------\n    df: A pandas data frame, that you wish to sample from.\n    n: The number of rows you wish to sample.\n\n    Returns:\n    --------\n    return value: A random sample of n rows of df.\n\n    Examples:\n    ---------\n    >>> df = pd.DataFrame({'col1' : [1, 2, 3], 'col2' : ['a', 'b', 'a']})\n    >>> df\n       col1 col2\n    0     1    a\n    1     2    b\n    2     3    a\n\n    >>> get_sample(df, 2)\n       col1 col2\n    1     2    b\n    2     3    a\n    \"\"\"\n    idxs = sorted(np.random.permutation(len(df))[:n])\n    return df.iloc[idxs].copy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c832a302fafaffed225314bdb1b9701b8b8dc797","_kg_hide-input":true},"cell_type":"code","source":"def train_cats(df):\n    \"\"\"Change any columns of strings in a panda's dataframe to a column of\n    categorical values. This applies the changes inplace.\n\n    Parameters:\n    -----------\n    df: A pandas dataframe. Any columns of strings will be changed to\n        categorical values.\n\n    Examples:\n    ---------\n\n    >>> df = pd.DataFrame({'col1' : [1, 2, 3], 'col2' : ['a', 'b', 'a']})\n    >>> df\n       col1 col2\n    0     1    a\n    1     2    b\n    2     3    a\n\n    note the type of col2 is string\n\n    >>> train_cats(df)\n    >>> df\n\n       col1 col2\n    0     1    a\n    1     2    b\n    2     3    a\n\n    now the type of col2 is category\n    \"\"\"\n    for n,c in df.items():\n        if is_string_dtype(c): df[n] = c.astype('category').cat.as_ordered()\n\ndef apply_cats(df, trn):\n    \"\"\"Changes any columns of strings in df into categorical variables using trn as\n    a template for the category codes.\n\n    Parameters:\n    -----------\n    df: A pandas dataframe. Any columns of strings will be changed to\n        categorical values. The category codes are determined by trn.\n\n    trn: A pandas dataframe. When creating a category for df, it looks up the\n        what the category's code were in trn and makes those the category codes\n        for df.\n\n    Examples:\n    ---------\n    >>> df = pd.DataFrame({'col1' : [1, 2, 3], 'col2' : ['a', 'b', 'a']})\n    >>> df\n       col1 col2\n    0     1    a\n    1     2    b\n    2     3    a\n\n    note the type of col2 is string\n\n    >>> train_cats(df)\n    >>> df\n\n       col1 col2\n    0     1    a\n    1     2    b\n    2     3    a\n\n    now the type of col2 is category {a : 1, b : 2}\n\n    >>> df2 = pd.DataFrame({'col1' : [1, 2, 3], 'col2' : ['b', 'a', 'a']})\n    >>> apply_cats(df2, df)\n\n           col1 col2\n        0     1    b\n        1     2    a\n        2     3    a\n\n    now the type of col is category {a : 1, b : 2}\n    \"\"\"\n    for n,c in df.items():\n        if (n in trn.columns) and (trn[n].dtype.name=='category'):\n            df[n] = c.astype('category').cat.as_ordered()\n            df[n].cat.set_categories(trn[n].cat.categories, ordered=True, inplace=True)\n\ndef fix_missing(df, col, name, na_dict):\n    \"\"\" Fill missing data in a column of df with the median, and add a {name}_na column\n    which specifies if the data was missing.\n\n    Parameters:\n    -----------\n    df: The data frame that will be changed.\n\n    col: The column of data to fix by filling in missing data.\n\n    name: The name of the new filled column in df.\n\n    na_dict: A dictionary of values to create na's of and the value to insert. If\n        name is not a key of na_dict the median will fill any missing data. Also\n        if name is not a key of na_dict and there is no missing data in col, then\n        no {name}_na column is not created.\n\n\n    Examples:\n    ---------\n    >>> df = pd.DataFrame({'col1' : [1, np.NaN, 3], 'col2' : [5, 2, 2]})\n    >>> df\n       col1 col2\n    0     1    5\n    1   nan    2\n    2     3    2\n\n    >>> fix_missing(df, df['col1'], 'col1', {})\n    >>> df\n       col1 col2 col1_na\n    0     1    5   False\n    1     2    2    True\n    2     3    2   False\n\n\n    >>> df = pd.DataFrame({'col1' : [1, np.NaN, 3], 'col2' : [5, 2, 2]})\n    >>> df\n       col1 col2\n    0     1    5\n    1   nan    2\n    2     3    2\n\n    >>> fix_missing(df, df['col2'], 'col2', {})\n    >>> df\n       col1 col2\n    0     1    5\n    1   nan    2\n    2     3    2\n\n\n    >>> df = pd.DataFrame({'col1' : [1, np.NaN, 3], 'col2' : [5, 2, 2]})\n    >>> df\n       col1 col2\n    0     1    5\n    1   nan    2\n    2     3    2\n\n    >>> fix_missing(df, df['col1'], 'col1', {'col1' : 500})\n    >>> df\n       col1 col2 col1_na\n    0     1    5   False\n    1   500    2    True\n    2     3    2   False\n    \"\"\"\n    if is_numeric_dtype(col):\n        if pd.isnull(col).sum() or (name in na_dict):\n            df[name+'_na'] = pd.isnull(col)\n            filler = na_dict[name] if name in na_dict else col.median()\n            df[name] = col.fillna(filler)\n            na_dict[name] = filler\n    return na_dict\n\ndef numericalize(df, col, name, max_n_cat):\n    \"\"\" Changes the column col from a categorical type to it's integer codes.\n\n    Parameters:\n    -----------\n    df: A pandas dataframe. df[name] will be filled with the integer codes from\n        col.\n\n    col: The column you wish to change into the categories.\n    name: The column name you wish to insert into df. This column will hold the\n        integer codes.\n\n    max_n_cat: If col has more categories than max_n_cat it will not change the\n        it to its integer codes. If max_n_cat is None, then col will always be\n        converted.\n\n    Examples:\n    ---------\n    >>> df = pd.DataFrame({'col1' : [1, 2, 3], 'col2' : ['a', 'b', 'a']})\n    >>> df\n       col1 col2\n    0     1    a\n    1     2    b\n    2     3    a\n\n    note the type of col2 is string\n\n    >>> train_cats(df)\n    >>> df\n\n       col1 col2\n    0     1    a\n    1     2    b\n    2     3    a\n\n    now the type of col2 is category { a : 1, b : 2}\n\n    >>> numericalize(df, df['col2'], 'col3', None)\n\n       col1 col2 col3\n    0     1    a    1\n    1     2    b    2\n    2     3    a    1\n    \"\"\"\n    if not is_numeric_dtype(col) and ( max_n_cat is None or len(col.cat.categories)>max_n_cat):\n        df[name] = col.cat.codes+1\n\ndef scale_vars(df, mapper):\n    warnings.filterwarnings('ignore', category=sklearn.exceptions.DataConversionWarning)\n    if mapper is None:\n        map_f = [([n],StandardScaler()) for n in df.columns if is_numeric_dtype(df[n])]\n        mapper = DataFrameMapper(map_f).fit(df)\n    df[mapper.transformed_names_] = mapper.transform(df)\n    return mapper\n\ndef proc_df(df, y_fld=None, skip_flds=None, ignore_flds=None, do_scale=False, na_dict=None,\n            preproc_fn=None, max_n_cat=None, subset=None, mapper=None):\n    \"\"\" proc_df takes a data frame df and splits off the response variable, and\n    changes the df into an entirely numeric dataframe. For each column of df \n    which is not in skip_flds nor in ignore_flds, na values are replaced by the\n    median value of the column.\n\n    Parameters:\n    -----------\n    df: The data frame you wish to process.\n\n    y_fld: The name of the response variable\n\n    skip_flds: A list of fields that dropped from df.\n\n    ignore_flds: A list of fields that are ignored during processing.\n\n    do_scale: Standardizes each column in df. Takes Boolean Values(True,False)\n\n    na_dict: a dictionary of na columns to add. Na columns are also added if there\n        are any missing values.\n\n    preproc_fn: A function that gets applied to df.\n\n    max_n_cat: The maximum number of categories to break into dummy values, instead\n        of integer codes.\n\n    subset: Takes a random subset of size subset from df.\n\n    mapper: If do_scale is set as True, the mapper variable\n        calculates the values used for scaling of variables during training time (mean and standard deviation).\n\n    Returns:\n    --------\n    [x, y, nas, mapper(optional)]:\n\n        x: x is the transformed version of df. x will not have the response variable\n            and is entirely numeric.\n\n        y: y is the response variable\n\n        nas: returns a dictionary of which nas it created, and the associated median.\n\n        mapper: A DataFrameMapper which stores the mean and standard deviation of the corresponding continuous\n        variables which is then used for scaling of during test-time.\n\n    Examples:\n    ---------\n    >>> df = pd.DataFrame({'col1' : [1, 2, 3], 'col2' : ['a', 'b', 'a']})\n    >>> df\n       col1 col2\n    0     1    a\n    1     2    b\n    2     3    a\n\n    note the type of col2 is string\n\n    >>> train_cats(df)\n    >>> df\n\n       col1 col2\n    0     1    a\n    1     2    b\n    2     3    a\n\n    now the type of col2 is category { a : 1, b : 2}\n\n    >>> x, y, nas = proc_df(df, 'col1')\n    >>> x\n\n       col2\n    0     1\n    1     2\n    2     1\n\n    >>> data = DataFrame(pet=[\"cat\", \"dog\", \"dog\", \"fish\", \"cat\", \"dog\", \"cat\", \"fish\"],\n                 children=[4., 6, 3, 3, 2, 3, 5, 4],\n                 salary=[90, 24, 44, 27, 32, 59, 36, 27])\n\n    >>> mapper = DataFrameMapper([(:pet, LabelBinarizer()),\n                          ([:children], StandardScaler())])\n\n    >>>round(fit_transform!(mapper, copy(data)), 2)\n\n    8x4 Array{Float64,2}:\n    1.0  0.0  0.0   0.21\n    0.0  1.0  0.0   1.88\n    0.0  1.0  0.0  -0.63\n    0.0  0.0  1.0  -0.63\n    1.0  0.0  0.0  -1.46\n    0.0  1.0  0.0  -0.63\n    1.0  0.0  0.0   1.04\n    0.0  0.0  1.0   0.21\n    \"\"\"\n    if not ignore_flds: ignore_flds=[]\n    if not skip_flds: skip_flds=[]\n    if subset: df = get_sample(df,subset)\n    else: df = df.copy()\n    ignored_flds = df.loc[:, ignore_flds]\n    df.drop(ignore_flds, axis=1, inplace=True)\n    if preproc_fn: preproc_fn(df)\n    if y_fld is None: y = None\n    else:\n        if not is_numeric_dtype(df[y_fld]): df[y_fld] = df[y_fld].cat.codes\n        y = df[y_fld].values\n        skip_flds += [y_fld]\n    df.drop(skip_flds, axis=1, inplace=True)\n\n    if na_dict is None: na_dict = {}\n    else: na_dict = na_dict.copy()\n    na_dict_initial = na_dict.copy()\n    for n,c in df.items(): na_dict = fix_missing(df, c, n, na_dict)\n    if len(na_dict_initial.keys()) > 0:\n        df.drop([a + '_na' for a in list(set(na_dict.keys()) - set(na_dict_initial.keys()))], axis=1, inplace=True)\n    if do_scale: mapper = scale_vars(df, mapper)\n    for n,c in df.items(): numericalize(df, c, n, max_n_cat)\n    df = pd.get_dummies(df, dummy_na=True)\n    df = pd.concat([ignored_flds, df], axis=1)\n    res = [df, y, na_dict]\n    if do_scale: res = res + [mapper]\n    return res\n\ndef rf_feat_importance(m, df):\n    return pd.DataFrame({'cols':df.columns, 'imp':m.feature_importances_}\n                       ).sort_values('imp', ascending=False)\n\ndef set_rf_samples(n):\n    \"\"\" Changes Scikit learn's random forests to give each tree a random sample of\n    n random rows.\n    \"\"\"\n    forest._generate_sample_indices = (lambda rs, n_samples:\n        forest.check_random_state(rs).randint(0, n_samples, n))\n\ndef reset_rf_samples():\n    \"\"\" Undoes the changes produced by set_rf_samples.\n    \"\"\"\n    forest._generate_sample_indices = (lambda rs, n_samples:\n        forest.check_random_state(rs).randint(0, n_samples, n_samples))\n\ndef get_nn_mappers(df, cat_vars, contin_vars):\n    # Replace nulls with 0 for continuous, \"\" for categorical.\n    for v in contin_vars: df[v] = df[v].fillna(df[v].max()+100,)\n    for v in cat_vars: df[v].fillna('#NA#', inplace=True)\n\n    # list of tuples, containing variable and instance of a transformer for that variable\n    # for categoricals, use LabelEncoder to map to integers. For continuous, standardize\n    cat_maps = [(o, LabelEncoder()) for o in cat_vars]\n    contin_maps = [([o], StandardScaler()) for o in contin_vars]\n    return DataFrameMapper(cat_maps).fit(df), DataFrameMapper(contin_maps).fit(df)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"19a1d7b4a5de03e5c2d8c33319a318a3520e4429"},"cell_type":"markdown","source":"# **Helper Method to reduce memory Usage**"},{"metadata":{"trusted":true,"_uuid":"647915e756e6f51b7ed34348ba9371fbec0bc183","_kg_hide-input":true},"cell_type":"code","source":"# Memory saving function credit to https://www.kaggle.com/gemartin/load-data-reduce-memory-usage\ndef reduce_mem_usage(df):\n    \"\"\" iterate through all the columns of a dataframe and modify the data type\n        to reduce memory usage.\n    \"\"\"\n    start_mem = df.memory_usage().sum() / 1024**2\n    \n    for col in df.columns:\n        col_type = df[col].dtype\n        \n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                #if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                #    df[col] = df[col].astype(np.float16)\n                #el\n                if c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        #else:\n            #df[col] = df[col].astype('category')\n\n    end_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage of dataframe is {:.2f} MB --> {:.2f} MB (Decreased by {:.1f}%)'.format(\n        start_mem, end_mem, 100 * (start_mem - end_mem) / start_mem))\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"af56fc0e42e167aa9b004fabca0debe14be27d9a"},"cell_type":"code","source":"PATH = \"../input/\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d118b2c41493ef045fd8c7b4954e960d6789aca0"},"cell_type":"code","source":"%%time \ndf_raw_train = pd.read_csv(f'{PATH}train_V2.csv', low_memory=False)\ndf_raw_train = reduce_mem_usage(df_raw_train)\ndf_raw_test = pd.read_csv(f'{PATH}test_V2.csv', low_memory = False)\ndf_raw_test = reduce_mem_usage(df_raw_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a91a86dde9459c51356aa6d51b48687ff70222fd"},"cell_type":"code","source":"def display_all(df):\n    with pd.option_context(\"display.max_rows\", 1000, \"display.max_columns\", 1000): \n        display(df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"254574e6b58991f2b96a0d66cdaac3ce1d22470c"},"cell_type":"code","source":"display_all(df_raw_train.describe(include='all').T)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8a68e1eeefb4fbb716765cc6958569a016a205bc"},"cell_type":"markdown","source":"# **Handling null values**"},{"metadata":{"trusted":true,"_uuid":"24665d736208eabafc6db61a8a3213455331426e"},"cell_type":"code","source":"null_cnt = df_raw_train.isnull().sum().sort_values()\nprint('null count:', null_cnt[null_cnt > 0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4da7d3624c5855fecd6403164208249690901b03"},"cell_type":"code","source":"df_raw_train.dropna(inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a973bf3e15f5bd375195d05f9699617b7de7dc87"},"cell_type":"code","source":"df_raw_train = df_raw_train.sort_values(by = ['matchId'])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d0dcbf7e7218454a9b8f1176c18238d7d2fbee43"},"cell_type":"markdown","source":"**There are lot of variations of MatchType but basically we can categorise them all to solo, duo or squad**"},{"metadata":{"trusted":true,"_uuid":"56b2aad56057441e9ee9e2fbc7dbfd90d0761b51"},"cell_type":"code","source":"mapper = lambda x: 'solo' if ('solo' in x) else 'duo' if ('duo' in x) or ('crash' in x) else 'squad'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1f3f09348319117158422cd2f65f21337a69a92d"},"cell_type":"code","source":"df_raw_train['matchType'] = df_raw_train['matchType'].apply(mapper)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ea2ea00296a0a507ce63c0d5290364081685cb54"},"cell_type":"code","source":"train_cats(df_raw_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0437d39b6af6ced0f94257bb019c6f4a981d6745"},"cell_type":"code","source":"df, y, nas = proc_df(df_raw_train, 'winPlacePerc')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1052ba5719968a6a8473e8a34b09fa211447a6e5"},"cell_type":"code","source":"def split_vals(a,n): return a[:n].copy(), a[n:].copy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"151b5e5ebc81d4a75a9b77f7e47e3e7649478903"},"cell_type":"code","source":"n_valid = 1000000\nn_trn = len(df)-n_valid\nraw_train, raw_valid = split_vals(df, n_trn)\nX_train, X_valid = split_vals(df, n_trn)\ny_train, y_valid = split_vals(y, n_trn)\n\nX_train.shape, y_train.shape, X_valid.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d54dbd78245a767e40907bc710a564af15ddbf62"},"cell_type":"code","source":"def rmse(x,y): return np.sqrt(((x-y)**2).mean())\n\ndef print_score(m):\n    res = [rmse(m.predict(X_train[features]), y_train), rmse(m.predict(X_valid[features]), y_valid),\n                m.score(X_train[features], y_train), m.score(X_valid[features], y_valid)]\n    if hasattr(m, 'oob_score_'): res.append(m.oob_score_)\n    print(res)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"545699d79e8d39b2015b4a8b2d5d287d890109b9"},"cell_type":"code","source":"# Training on subset of the data initially to do analysis fast.\nX_train = X_train[:1000000]\ny_train = y_train[:1000000]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4ded5eaa4ff42e8014621722211c48d39feb5888"},"cell_type":"code","source":"X_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fc5c352176a0fe09b6e662e5fed5ed4838331d25"},"cell_type":"code","source":"# Training on simple model without removing matchId, Id and groupId. Just to visualise the decision tree\nm = RandomForestRegressor(n_estimators=1, max_depth=3, bootstrap=False, n_jobs=-1)\nm.fit(X_train, y_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dc9563239042e389556b1abaf6e7208c8ee5de0f"},"cell_type":"code","source":"rmse(m.predict(X_train), y_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6c4020b4ec1ed443c1e8d061fc4ca98fe563b0af"},"cell_type":"code","source":"draw_tree(m.estimators_[0], X_train, precision=3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7f000d96e1215696db7eb32d47fc1f0f9be512a3"},"cell_type":"code","source":"def fillInf(df, val):\n    numcols = df.select_dtypes(include='number').columns\n    cols = numcols[numcols != 'winPlacePerc']\n    df[df == np.Inf] = np.NaN\n    df[df == np.NINF] = np.NaN\n    for c in cols: df[c].fillna(val, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"636354bc5c252c4edfe3a30ff9f20f8369c3df6a"},"cell_type":"code","source":"features = list(X_train.columns)\nfeatures","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e0cd4878f979653dc1a5b3e376b352be239ee6ce"},"cell_type":"code","source":"features.remove('Id')\nfeatures.remove('groupId')\nfeatures.remove('matchId')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"626976aa001adda9bab3706fa6afe1a199f957dd"},"cell_type":"code","source":"# Fitting using a basic Random Forest\nm = RandomForestRegressor(n_jobs=-1)\nm.fit(X_train[features], y_train)\nprint_score(m)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"844a95ab243216b7cbcc3dae22cb955d17ed016d"},"cell_type":"markdown","source":"# **Analysing the feature importance with basic features**"},{"metadata":{"trusted":true,"_uuid":"a11fbc5be671fa3f3fa34c5ab9127abba48346ef"},"cell_type":"code","source":"fi = rf_feat_importance(m, X_train[features]); fi","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"93e2ab687e6aebe4317e9093eec0d47c9799f83e"},"cell_type":"code","source":"def plot_fi(fi): return fi.plot('cols', 'imp', 'barh', figsize=(12,7), legend=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"06066891423d7bb263417ec4666e765407bd386a"},"cell_type":"code","source":"plot_fi(fi[:30])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"56b03f06635bb5f0e59372e82f75ad9399ca2385"},"cell_type":"markdown","source":"# **Adding new features.**\nThe features which we can be broadly categorised into three categories. Individual, Group wise, Match wise. In **solo** matchtype no of groups will  be equal to no of players and in **duo** players will participate in groups of two and in **squad** players will participate in groups of 4. Using stats which summarise the group and match may be helpful"},{"metadata":{"trusted":true,"_uuid":"e8d882ade2e7e982a1863a5d5cf463d4818e9ea8"},"cell_type":"code","source":"X_train['headshot_percentage'] = X_train['headshotKills']/X_train['kills']\nX_valid['headshot_percentage'] = X_valid['headshotKills']/X_valid['kills']\n\nX_train['totalDistance'] = X_train['rideDistance'] + X_train['walkDistance'] + X_train['swimDistance']\nX_valid['totalDistance'] = X_valid['rideDistance'] + X_valid['walkDistance'] + X_valid['swimDistance']\n\nX_train['health_items'] = X_train['heals'] + X_train['boosts']\nX_train['killPlaceOverMaxPlace'] = X_train['killPlace'] / X_train['maxPlace']\nX_train['killsOverWalkDistance'] = X_train['kills'] / X_train['walkDistance']\n\nX_valid['health_items'] = X_valid['heals'] + X_train['boosts']\nX_valid['killPlaceOverMaxPlace'] = X_valid['killPlace'] / X_train['maxPlace']\nX_valid['killsOverWalkDistance'] = X_valid['kills'] / X_train['walkDistance']\n\nX_train['killStreakRate'] = X_train['killStreaks']/X_train['kills']\nX_valid['killStreakRate'] = X_valid['killStreaks']/X_valid['kills']\n\nX_train['killMinute'] = X_train['kills'] / X_train['matchDuration']\nX_valid['killMinute'] = X_valid['kills'] / X_valid['matchDuration']\n\nX_train['damageDealtMinute'] = X_train['damageDealt'] / X_train['matchDuration']\nX_valid['damageDealtMinute'] = X_valid['damageDealt'] / X_valid['matchDuration']\n\nX_train['participateKills'] = X_train['kills'] + X_train['assists'] + X_train['DBNOs']\nX_valid['participateKills'] = X_valid['kills'] + X_valid['assists'] + X_valid['DBNOs']\n\nX_train['vehicleDestroysMinute'] = X_train['vehicleDestroys'] / X_train['matchDuration']\nX_valid['vehicleDestroysMinute'] = X_valid['vehicleDestroys'] / X_valid['matchDuration']\n\nX_train['killsMiter'] = X_train['roadKills'] / X_train['rideDistance']\nX_valid['killsMiter'] = X_valid['roadKills'] / X_valid['rideDistance']\n\nX_train['playersJoined'] = X_train.groupby('matchId')['matchId'].transform('count')\n\nX_train['killsNorm'] = X_train['kills']*((100-X_train['playersJoined'])/100)\nX_train['damageDealtNorm'] = X_train['damageDealt']*((100-X_train['playersJoined'])/100)\n\nX_train['boostsPerWalkDistance'] = X_train['boosts']/(X_train['walkDistance']) \n\nX_train['healsPerWalkDistance'] = X_train['heals']/(X_train['walkDistance'])\n\nX_train['healsAndBoosts'] = X_train['heals']+X_train['boosts']\n\nX_train['healsAndBoostsPerWalkDistance'] = X_train['healsAndBoosts']/(X_train['walkDistance'])\n\nX_valid['playersJoined'] = X_valid.groupby('matchId')['matchId'].transform('count')\n\nX_valid['killsNorm'] = X_valid['kills']*((100-X_valid['playersJoined'])/100)\nX_valid['damageDealtNorm'] = X_valid['damageDealt']*((100-X_valid['playersJoined'])/100)\n\nX_valid['boostsPerWalkDistance'] = X_valid['boosts']/(X_train['walkDistance']) \n\nX_valid['healsPerWalkDistance'] = X_valid['heals']/(X_train['walkDistance'])\n\nX_valid['healsAndBoosts'] = X_valid['heals'] + X_valid['boosts']\n\nX_valid['healsAndBoostsPerWalkDistance'] = X_valid['healsAndBoosts']/(X_valid['walkDistance'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"08ab9cb13b91d2a64adc18d48d827b698ac5a760"},"cell_type":"code","source":"train_group_agg = X_train.groupby(['matchId', 'groupId', 'matchType'])\nval_group_agg =  X_valid.groupby(['matchId', 'groupId', 'matchType'])\n\ntrain_match_agg = X_train.groupby(['matchId'])\nval_match_agg = X_valid.groupby(['matchId'])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"493f9f211b6b33b33026c15fb63094d3cb75ca24"},"cell_type":"code","source":"X_train.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"39d96fd38effa3b32daafa090867f29f7afbbe94"},"cell_type":"code","source":"features = list(X_train.columns)\nfeatures.remove('matchId')\nfeatures.remove('groupId')\nfeatures.remove('Id')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"071892658e07073af94eb67765806c4822e7108a","_kg_hide-output":true},"cell_type":"code","source":"agg_col = features\nagg_col.remove('matchType')\nagg_col.remove('numGroups')\nagg_col.remove('maxPlace')\nagg_col","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"trusted":true,"_uuid":"9601b799edd8ac7fbe95291c4d0879c48e65dacd"},"cell_type":"code","source":"features","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"de67de0c77522b0a102ce8bec656de79ecbcc704"},"cell_type":"code","source":"fillInf(X_train, 0)\nfillInf(X_valid, 0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7e7e2edc29b312cb71f90047acb45524cafc5159"},"cell_type":"code","source":"for col in agg_col :\n    X_train['percentage_match_' + col] = train_match_agg[col].rank(pct=True).values\n    X_valid['percentage_match_' + col] = val_match_agg[col].rank(pct=True).values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"09d31c0e889b6470f752462307a3e140f484d076"},"cell_type":"code","source":"X_train = X_train.merge(train_match_agg[agg_col].max().rename(columns=lambda s: 'match_max_' + s), on = 'matchId', how = 'left')\nX_valid = X_valid.merge(val_match_agg[agg_col].max().rename(columns=lambda s: 'match_max_' + s), on = 'matchId', how = 'left')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9823160d46eb6110fae43d870b3c5b2a93c474b4"},"cell_type":"code","source":"X_train = X_train.merge(train_match_agg[agg_col].min().rename(columns=lambda s: 'match_min_' + s), on = 'matchId', how = 'left')\nX_valid = X_valid.merge(val_match_agg[agg_col].min().rename(columns=lambda s: 'match_min_' + s), on = 'matchId', how = 'left')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0b57d36e733c6f0c9a367d0b55efaee8248429b1"},"cell_type":"code","source":"X_train = X_train.merge(train_match_agg[agg_col].mean().rename(columns=lambda s: 'match_mean_' + s), on = 'matchId', how = 'left')\nX_valid = X_valid.merge(val_match_agg[agg_col].mean().rename(columns=lambda s: 'match_mean_' + s), on = 'matchId', how = 'left')","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"trusted":true,"_uuid":"35eb97d3c334d55a5503406a1fdced9d25e2d17c"},"cell_type":"code","source":"display_all(X_train.head().T)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"trusted":true,"_uuid":"48296e9792a173ea595af1bb2c06da8848f9e00f"},"cell_type":"code","source":"features = list(X_train.columns)\nfeatures.remove('matchId')\nfeatures.remove('groupId')\nfeatures.remove('Id')\nfeatures","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1c49b669c933a9d4e5dfb2029c468f4e95e8fd7d"},"cell_type":"code","source":"# Chooses random set for each tree. Helpful in reducing overfitting as it increases variance per each treee \nset_rf_samples(100000)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7ea964820fa93ee92eb1f61b2b4c29e526176ddf"},"cell_type":"code","source":"m = RandomForestRegressor(n_jobs=-1, n_estimators=40, max_features=0.5,min_samples_leaf=3)\nm.fit(X_train[features], y_train)\nprint_score(m)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8d8148910b15c1e35fe03aedbaf745c5d88a0240","_kg_hide-output":true},"cell_type":"code","source":"fi = rf_feat_importance(m, X_train[features]); \ndisplay_all(fi)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"96356aa7ea0909f58d7fea2ddbb0a3801d665e70"},"cell_type":"code","source":"plot_fi(fi[:30])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6d82ca4eb3f7d8f74d07b4be3ffbc319be0d6e95"},"cell_type":"markdown","source":"**Removing columns which are not at all important**"},{"metadata":{"trusted":true,"_uuid":"d71b0fbfa8e455f937022c69f7d2ab5962728ac7"},"cell_type":"code","source":"to_keep = fi[fi.imp > 0.0005].cols\nlen(to_keep)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"aed81c56452e091d14b594a4508a4d8fa197ee3f"},"cell_type":"code","source":"X_train = X_train[to_keep]\nX_valid = X_valid[to_keep]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"04af0280d6125ad7cd6f3352e90b6bbfff7176f4"},"cell_type":"code","source":"features = list(X_train.columns)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"trusted":true,"_uuid":"337c858f0488951b09845e84a7bda9b81af53d47"},"cell_type":"code","source":"features","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3d48287b780009db91ccef9efdc2828d42200e59"},"cell_type":"code","source":"m = RandomForestRegressor(n_jobs=-1, n_estimators=40, max_features=0.5,min_samples_leaf=3)\nm.fit(X_train[features], y_train)\nprint_score(m)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"45c437ce967f71e76e0a5ff6f3796b3730e3b5e1"},"cell_type":"code","source":"m = RandomForestRegressor(n_jobs=-1, n_estimators=40, max_features=0.5,min_samples_leaf=10)\nm.fit(X_train[features], y_train)\nprint_score(m)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"60ef86a37ac1fba05afe6738a7fcbc001f174fad"},"cell_type":"code","source":"m = RandomForestRegressor(n_jobs=-1, n_estimators=40, max_features=0.5,min_samples_leaf=25)\nm.fit(X_train[features], y_train)\nprint_score(m)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"trusted":true,"_uuid":"64dd690d9a9c5bb75e81222c2dc5ddc86d25124d"},"cell_type":"code","source":"fi = rf_feat_importance(m, X_train[features]); \ndisplay_all(fi)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b4df8bed8aec35a7bc1e2178657cdc4b27b7efa2"},"cell_type":"code","source":"plot_fi(fi[:15])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2369646579eb53e07dfd5251e560936a12ab58df"},"cell_type":"markdown","source":"**Removing some more features**"},{"metadata":{"trusted":true,"_uuid":"f85ab5dee51037d3b5c834a7693f62d968c287c5"},"cell_type":"code","source":"to_keep = fi[fi.imp > 0.0005].cols\nlen(to_keep)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1a53c0c795f7e2388bd884d284cba846945c8a2e"},"cell_type":"code","source":"X_train = X_train[to_keep]\nX_valid = X_valid[to_keep]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a15825e146ca5a6c4b866b700c4a8f4e68373208"},"cell_type":"code","source":"features = X_train.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c60a533193672d6cf8b330ab80b4278b87d4045d"},"cell_type":"code","source":"m = RandomForestRegressor(n_jobs=-1, n_estimators=40, max_features=0.6,min_samples_leaf=3)\nm.fit(X_train[features], y_train)\nprint_score(m)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1059d97c8d700669f398485f54f90a5fd98b3c7b"},"cell_type":"markdown","source":"# **Removing redundant features**\nVariable which have similar meaning makes it very hard to interpret the model. So here we try to remove such features with help of dendogram"},{"metadata":{"trusted":true,"_uuid":"579e19a766bf97cee84612c65b769df7b17ad438"},"cell_type":"code","source":"from scipy.cluster import hierarchy as hc\nimport scipy\nfrom matplotlib import pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3741a7dda4fe62e877c1775ac7f813222ce50ea9"},"cell_type":"code","source":"corr = np.round(scipy.stats.spearmanr(X_train[features]).correlation, 4)\ncorr_condensed = hc.distance.squareform(1-corr)\nz = hc.linkage(corr_condensed, method='average')\nfig = plt.figure(figsize=(16,10))\ndendrogram = hc.dendrogram(z, labels=X_train[features].columns, orientation='left', leaf_font_size=16)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9833d27409cd059a3209a2014df44cd3b4c1585e"},"cell_type":"code","source":"def get_score(X_train, y_train, X_valid, y_valid) :\n    m = RandomForestRegressor(n_jobs=-1, n_estimators=40, max_features=0.6,min_samples_leaf=3)\n    m.fit(X_train, y_train)\n    res = [rmse(m.predict(X_train), y_train), rmse(m.predict(X_valid), y_valid),\n                m.score(X_train, y_train), m.score(X_valid, y_valid)]    \n    return res","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3a37bd1ad5feac3b0d488e5a1591a4a13ae5a920"},"cell_type":"markdown","source":"**Let's try removing some of these related features to see if the model can be simplified without impacting the accuracy.**"},{"metadata":{"trusted":true,"_uuid":"d6fe88e7ebbc9d38b7682d33e3ec2366956044ef"},"cell_type":"code","source":"for c in ('percentage_match_killPlaceOverMaxPlace', 'percentage_match_killPlace','percentage_match_health_items', 'percentage_match_killMinute'):\n    print(c, get_score(X_train.drop(c, axis=1), y_train, X_valid.drop(c, axis = 1), y_valid))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e50b2a66bda57e61f69ea306d86f07a3e3bd211d"},"cell_type":"markdown","source":"**From the above results, we can safely remove percentage_match_killPlaceOverMaxPlace **"},{"metadata":{"trusted":true,"_uuid":"c059a091bc5ce8c088b3fcaa1998ea153f93ac1a"},"cell_type":"code","source":"X_train = X_train.drop(['percentage_match_killPlaceOverMaxPlace'], axis = 1)\nX_valid = X_valid.drop(['percentage_match_killPlaceOverMaxPlace'], axis = 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0ceccbbc1726f1d559e7b52f5eda197176c6dda0"},"cell_type":"code","source":"features = list(X_train.columns)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5417600e24a630515ccdc670bbfce3af03e4898c"},"cell_type":"markdown","source":"**Final feature importance interpretation**"},{"metadata":{"trusted":true,"_uuid":"2ce6772441cf207fe00432947c8d206b7d84159e"},"cell_type":"code","source":"m = RandomForestRegressor(n_jobs=-1, n_estimators=40, max_features=0.6,min_samples_leaf=3)\nm.fit(X_train, y_train)\nprint_score(m)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f3f391e521b62164697539170f4ac95b0f85664f"},"cell_type":"code","source":"fi = rf_feat_importance(m, X_train[features]); \ndisplay_all(fi)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ff7b4820a49e4d4fa62b2af7b34f1def2ccb296d"},"cell_type":"code","source":"plot_fi(fi[:15])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8dfe2db348faca0909539b037fb6084b0e65acb1"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}