{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-07T09:36:18.230553Z","iopub.execute_input":"2022-08-07T09:36:18.231183Z","iopub.status.idle":"2022-08-07T09:36:18.242960Z","shell.execute_reply.started":"2022-08-07T09:36:18.231140Z","shell.execute_reply":"2022-08-07T09:36:18.241603Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = pd.read_csv('/kaggle/input/digital-turbine-auction-bid-price-prediction/train_data.csv')\ntest_df = pd.read_csv('/kaggle/input/digital-turbine-auction-bid-price-prediction/test_data.csv')","metadata":{"execution":{"iopub.status.busy":"2022-08-07T09:36:18.245323Z","iopub.execute_input":"2022-08-07T09:36:18.246454Z","iopub.status.idle":"2022-08-07T09:36:34.933958Z","shell.execute_reply.started":"2022-08-07T09:36:18.246418Z","shell.execute_reply":"2022-08-07T09:36:34.932976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# imports","metadata":{}},{"cell_type":"code","source":"from matplotlib import pyplot as plt\nimport seaborn as sns\nfrom catboost import CatBoostRegressor\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import mean_squared_error ","metadata":{"execution":{"iopub.status.busy":"2022-08-07T09:36:34.935687Z","iopub.execute_input":"2022-08-07T09:36:34.936031Z","iopub.status.idle":"2022-08-07T09:36:34.941301Z","shell.execute_reply.started":"2022-08-07T09:36:34.935991Z","shell.execute_reply":"2022-08-07T09:36:34.940102Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# dataframe info stats\n\ndef stats(data):\n    \n    maxx = []\n    minn = []\n    for i in data.columns:\n        maxx.append(data[i].value_counts().max())\n        minn.append(data[i].value_counts().min())\n\n    return pd.DataFrame(\n        {'nunique': data.nunique(),\n         'len': len(data),\n        # 'nunique/len': data.nunique()/len(data),\n         'types':data.dtypes,\n         'Nulls' : data.isna().sum(),\n        # 'Nullpercent' : data.isna().sum()/len(data),\n         \"Value counts Max\": maxx,\n         'Value counts Min':minn \n        },\n        columns = ['nunique', 'len','types','Nulls'#,'Nullpercent', 'nunique/len'\n                   ,\"Value counts Max\",'Value counts Min']).\\\n        sort_values(by ='nunique',ascending = False)\n\n\n\ndef countPlot(col,num = 6,hue = None):\n    sns.set(rc={'figure.figsize':(6,6)})\n    ax = sns.countplot(x=col, data=train_df, hue = hue,\n                   order=train_df[col].value_counts().iloc[:num].index)\n    \n    return plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-07T09:36:34.942633Z","iopub.execute_input":"2022-08-07T09:36:34.943140Z","iopub.status.idle":"2022-08-07T09:36:34.955324Z","shell.execute_reply.started":"2022-08-07T09:36:34.943105Z","shell.execute_reply":"2022-08-07T09:36:34.954220Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-07T09:36:34.959690Z","iopub.execute_input":"2022-08-07T09:36:34.960035Z","iopub.status.idle":"2022-08-07T09:36:34.983679Z","shell.execute_reply.started":"2022-08-07T09:36:34.959992Z","shell.execute_reply":"2022-08-07T09:36:34.982622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-07T09:36:34.985080Z","iopub.execute_input":"2022-08-07T09:36:34.985648Z","iopub.status.idle":"2022-08-07T09:36:35.008712Z","shell.execute_reply.started":"2022-08-07T09:36:34.985613Z","shell.execute_reply":"2022-08-07T09:36:35.007513Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stats(train_df)","metadata":{"execution":{"iopub.status.busy":"2022-08-07T09:36:35.011592Z","iopub.execute_input":"2022-08-07T09:36:35.012631Z","iopub.status.idle":"2022-08-07T09:37:02.156541Z","shell.execute_reply.started":"2022-08-07T09:36:35.012595Z","shell.execute_reply":"2022-08-07T09:37:02.155589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stats(test_df)","metadata":{"execution":{"iopub.status.busy":"2022-08-07T09:37:02.158330Z","iopub.execute_input":"2022-08-07T09:37:02.158730Z","iopub.status.idle":"2022-08-07T09:37:02.328547Z","shell.execute_reply.started":"2022-08-07T09:37:02.158691Z","shell.execute_reply":"2022-08-07T09:37:02.327375Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Eda","metadata":{}},{"cell_type":"code","source":"fig, axes = plt.subplots(2, 2)\naxes = axes.ravel()\ntrain_df['unitDisplayType'].hist(figsize = (8, 8),ax=axes[0])\ntrain_df['connectionType'].hist(figsize = (8, 8),ax=axes[1])\ntrain_df['c3'].hist(figsize = (8, 8),ax=axes[2])\ntrain_df['has_won'].hist(figsize = (8, 8),ax=axes[3])\n\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-07T09:37:02.330362Z","iopub.execute_input":"2022-08-07T09:37:02.330772Z","iopub.status.idle":"2022-08-07T09:37:09.606026Z","shell.execute_reply.started":"2022-08-07T09:37:02.330734Z","shell.execute_reply":"2022-08-07T09:37:09.603951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"countPlot('countryCode',10)","metadata":{"execution":{"iopub.status.busy":"2022-08-07T09:37:09.607498Z","iopub.execute_input":"2022-08-07T09:37:09.608109Z","iopub.status.idle":"2022-08-07T09:37:13.162705Z","shell.execute_reply.started":"2022-08-07T09:37:09.608072Z","shell.execute_reply":"2022-08-07T09:37:13.161715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"countPlot('winBid',10)","metadata":{"execution":{"iopub.status.busy":"2022-08-07T09:37:13.167278Z","iopub.execute_input":"2022-08-07T09:37:13.168217Z","iopub.status.idle":"2022-08-07T09:37:14.190632Z","shell.execute_reply.started":"2022-08-07T09:37:13.168175Z","shell.execute_reply":"2022-08-07T09:37:14.189606Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"countPlot('sentPrice',6,'has_won')","metadata":{"execution":{"iopub.status.busy":"2022-08-07T09:37:14.191913Z","iopub.execute_input":"2022-08-07T09:37:14.192501Z","iopub.status.idle":"2022-08-07T09:37:15.744647Z","shell.execute_reply.started":"2022-08-07T09:37:14.192458Z","shell.execute_reply":"2022-08-07T09:37:15.743614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"countPlot('mediationProviderVersion')","metadata":{"execution":{"iopub.status.busy":"2022-08-07T09:37:15.745933Z","iopub.execute_input":"2022-08-07T09:37:15.746815Z","iopub.status.idle":"2022-08-07T09:37:18.953610Z","shell.execute_reply.started":"2022-08-07T09:37:15.746775Z","shell.execute_reply":"2022-08-07T09:37:18.952610Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def percentile80(x):\n    return np.percentile(x,80)\n\npivoted = train_df.pivot_table(index = ['countryCode'], values = ['winBid'], \n               aggfunc = [np.mean, np.median, np.std,'count', percentile80])\n\npivoted.columns = pivoted.columns.get_level_values(0)\n\n\npivoted.sort_values('count',ascending=False).head(17)","metadata":{"execution":{"iopub.status.busy":"2022-08-07T09:37:18.955289Z","iopub.execute_input":"2022-08-07T09:37:18.955701Z","iopub.status.idle":"2022-08-07T09:37:22.512275Z","shell.execute_reply.started":"2022-08-07T09:37:18.955598Z","shell.execute_reply":"2022-08-07T09:37:22.511173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# preprocessing","metadata":{}},{"cell_type":"code","source":"# Q&D\n# try with missing\ntrain_df.fillna({'countryCode':'US', 'connectionType':'UNKNOWN'}, inplace=True)\ntest_df.fillna({'countryCode':'US', 'connectionType':'UNKNOWN'}, inplace=True)\n                                    ","metadata":{"execution":{"iopub.status.busy":"2022-08-07T09:37:22.513902Z","iopub.execute_input":"2022-08-07T09:37:22.514267Z","iopub.status.idle":"2022-08-07T09:37:23.080833Z","shell.execute_reply.started":"2022-08-07T09:37:22.514230Z","shell.execute_reply":"2022-08-07T09:37:23.079813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"## TODO  fill na in country code by other columns\n\n# columns = ['brandName','correctModelName','connectionType','countryCode']\n\n# fill_null_df = train_df[columns + ['winBid']]\\\n# .groupby(columns, as_index=False,sort=True).count()\n\n# fillcountrycode = fill_null_df.sort_values(by=['brandName','correctModelName','connectionType','winBid'],ascending=False).drop_duplicates(['brandName','correctModelName','connectionType'])\n# fillconnectiontype = fill_null_df.sort_values(by=['brandName','correctModelName','countryCode','winBid'],ascending=False).drop_duplicates(['brandName','correctModelName','countryCode'])\n","metadata":{"execution":{"iopub.status.busy":"2022-08-07T09:37:23.082514Z","iopub.execute_input":"2022-08-07T09:37:23.082941Z","iopub.status.idle":"2022-08-07T09:37:23.087903Z","shell.execute_reply.started":"2022-08-07T09:37:23.082903Z","shell.execute_reply":"2022-08-07T09:37:23.086622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cols = ['c2', 'c4']\ntrain_df[cols] = train_df[cols].applymap(np.int16)\ntest_df[cols] = test_df[cols].applymap(np.int16)","metadata":{"execution":{"iopub.status.busy":"2022-08-07T09:37:23.089669Z","iopub.execute_input":"2022-08-07T09:37:23.090421Z","iopub.status.idle":"2022-08-07T09:37:31.150140Z","shell.execute_reply.started":"2022-08-07T09:37:23.090385Z","shell.execute_reply":"2022-08-07T09:37:31.149168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# select columns","metadata":{}},{"cell_type":"code","source":"train_columns = [ 'unitDisplayType', 'brandName', 'bundleId',\n       'appVersion', 'correctModelName', 'countryCode',# 'deviceId',\n       'osAndVersion', 'connectionType', 'c1', 'c2', 'c3', 'c4', 'size',\n       'mediationProviderVersion', 'bidFloorPrice'#,'has_won'\n       ]\n\ncat_columns = [ 'unitDisplayType', 'brandName', 'bundleId',\n       'appVersion', 'correctModelName', 'countryCode', #'deviceId',\n       'osAndVersion', 'connectionType', 'c1', 'c2', 'c3', 'c4', 'size',\n       'mediationProviderVersion',#'has_won'\n       ]\n\ntarget = ['winBid']","metadata":{"execution":{"iopub.status.busy":"2022-08-07T09:37:31.151494Z","iopub.execute_input":"2022-08-07T09:37:31.151974Z","iopub.status.idle":"2022-08-07T09:37:31.158603Z","shell.execute_reply.started":"2022-08-07T09:37:31.151936Z","shell.execute_reply":"2022-08-07T09:37:31.157400Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# train model","metadata":{}},{"cell_type":"code","source":"X_train, X_val, y_train, y_val = train_test_split(train_df[train_columns],train_df[target], test_size = 0.27, random_state=17) ","metadata":{"execution":{"iopub.status.busy":"2022-08-07T09:37:31.160273Z","iopub.execute_input":"2022-08-07T09:37:31.160622Z","iopub.status.idle":"2022-08-07T09:37:42.824167Z","shell.execute_reply.started":"2022-08-07T09:37:31.160587Z","shell.execute_reply":"2022-08-07T09:37:42.823060Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time \n\ncboost =  CatBoostRegressor(random_state=17,cat_features=cat_columns,task_type='GPU',learning_rate = 0.1,verbose=False)\n\ny_pred = cboost.fit(X_train, y_train).predict(X_val)","metadata":{"execution":{"iopub.status.busy":"2022-08-07T09:37:42.825581Z","iopub.execute_input":"2022-08-07T09:37:42.825988Z","iopub.status.idle":"2022-08-07T09:46:39.810792Z","shell.execute_reply.started":"2022-08-07T09:37:42.825941Z","shell.execute_reply":"2022-08-07T09:46:39.809715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mse = mean_squared_error(y_val, y_pred)\nnp.sqrt(mse)","metadata":{"execution":{"iopub.status.busy":"2022-08-07T09:46:39.812147Z","iopub.execute_input":"2022-08-07T09:46:39.813097Z","iopub.status.idle":"2022-08-07T09:46:39.835263Z","shell.execute_reply.started":"2022-08-07T09:46:39.813057Z","shell.execute_reply":"2022-08-07T09:46:39.834378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_sub = cboost.predict(test_df[train_columns])","metadata":{"execution":{"iopub.status.busy":"2022-08-07T09:46:39.837589Z","iopub.execute_input":"2022-08-07T09:46:39.838016Z","iopub.status.idle":"2022-08-07T09:46:40.448862Z","shell.execute_reply.started":"2022-08-07T09:46:39.837980Z","shell.execute_reply":"2022-08-07T09:46:40.447838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_sub[0:7]","metadata":{"execution":{"iopub.status.busy":"2022-08-07T09:46:40.450128Z","iopub.execute_input":"2022-08-07T09:46:40.450498Z","iopub.status.idle":"2022-08-07T09:46:40.459065Z","shell.execute_reply.started":"2022-08-07T09:46:40.450459Z","shell.execute_reply":"2022-08-07T09:46:40.458059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_sub = y_sub.clip(0.02)","metadata":{"execution":{"iopub.status.busy":"2022-08-07T09:46:40.460493Z","iopub.execute_input":"2022-08-07T09:46:40.461467Z","iopub.status.idle":"2022-08-07T09:46:40.468004Z","shell.execute_reply.started":"2022-08-07T09:46:40.461424Z","shell.execute_reply":"2022-08-07T09:46:40.467019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# TODO bid > bidFloorPrice\n# TODO B.R","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subdf = pd.DataFrame({'deviceId':test_df['deviceId'], 'winBid':y_sub})\nsubdf.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-07T09:46:40.469393Z","iopub.execute_input":"2022-08-07T09:46:40.470399Z","iopub.status.idle":"2022-08-07T09:46:40.485241Z","shell.execute_reply.started":"2022-08-07T09:46:40.470363Z","shell.execute_reply":"2022-08-07T09:46:40.484342Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# submission","metadata":{}},{"cell_type":"code","source":"sub_example = pd.read_csv('/kaggle/input/digital-turbine-auction-bid-price-prediction/sample_submission.csv')\n\nsub_example.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-07T09:46:40.487007Z","iopub.execute_input":"2022-08-07T09:46:40.487407Z","iopub.status.idle":"2022-08-07T09:46:40.518062Z","shell.execute_reply.started":"2022-08-07T09:46:40.487372Z","shell.execute_reply":"2022-08-07T09:46:40.517223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subdf = subdf.set_index('deviceId')\nsubdf = subdf.reindex(index=sub_example['deviceId'])\nsubdf = subdf.reset_index()\nsubdf.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-07T09:46:40.520555Z","iopub.execute_input":"2022-08-07T09:46:40.521729Z","iopub.status.idle":"2022-08-07T09:46:40.545508Z","shell.execute_reply.started":"2022-08-07T09:46:40.521691Z","shell.execute_reply":"2022-08-07T09:46:40.544662Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"VER = 'BL'","metadata":{"execution":{"iopub.status.busy":"2022-08-07T09:46:40.546885Z","iopub.execute_input":"2022-08-07T09:46:40.547266Z","iopub.status.idle":"2022-08-07T09:46:40.551995Z","shell.execute_reply.started":"2022-08-07T09:46:40.547198Z","shell.execute_reply":"2022-08-07T09:46:40.550949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subdf.to_csv(f'submission_{VER}.csv', index = False)","metadata":{"execution":{"iopub.status.busy":"2022-08-07T09:46:40.557632Z","iopub.execute_input":"2022-08-07T09:46:40.558397Z","iopub.status.idle":"2022-08-07T09:46:40.651193Z","shell.execute_reply.started":"2022-08-07T09:46:40.558362Z","shell.execute_reply":"2022-08-07T09:46:40.650274Z"},"trusted":true},"execution_count":null,"outputs":[]}]}