{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-10T09:25:42.006002Z","iopub.execute_input":"2022-08-10T09:25:42.006796Z","iopub.status.idle":"2022-08-10T09:25:42.035113Z","shell.execute_reply.started":"2022-08-10T09:25:42.006668Z","shell.execute_reply":"2022-08-10T09:25:42.033788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntrain_df = pd.read_csv('/kaggle/input/digital-turbine-auction-bid-price-prediction/train_data.csv')\nprint(train_df.shape)\ntest_df = pd.read_csv('/kaggle/input/digital-turbine-auction-bid-price-prediction/test_data.csv')\nprint(train_df.shape)","metadata":{"execution":{"iopub.status.busy":"2022-08-10T09:25:42.037272Z","iopub.execute_input":"2022-08-10T09:25:42.037646Z","iopub.status.idle":"2022-08-10T09:26:17.282156Z","shell.execute_reply.started":"2022-08-10T09:25:42.037613Z","shell.execute_reply":"2022-08-10T09:26:17.280747Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntrain_df[\"eventTimestamp\"] = pd.to_datetime(train_df[\"eventTimestamp\"],unit='ms',infer_datetime_format=True)\nprint(\"train times\")\nprint(train_df[\"eventTimestamp\"].describe())\n\ntest_df[\"eventTimestamp\"] = pd.to_datetime(test_df[\"eventTimestamp\"],unit='ms',infer_datetime_format=True)\nprint(\"test times\")\nprint(test_df[\"eventTimestamp\"].describe())","metadata":{"execution":{"iopub.status.busy":"2022-08-10T09:26:17.283934Z","iopub.execute_input":"2022-08-10T09:26:17.284449Z","iopub.status.idle":"2022-08-10T09:26:20.149373Z","shell.execute_reply.started":"2022-08-10T09:26:17.284383Z","shell.execute_reply":"2022-08-10T09:26:20.148071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(train_df[\"eventTimestamp\"].dt.date.describe())\n\ndisplay(test_df[\"eventTimestamp\"].dt.date.describe())","metadata":{"execution":{"iopub.status.busy":"2022-08-10T09:31:34.712521Z","iopub.execute_input":"2022-08-10T09:31:34.714198Z","iopub.status.idle":"2022-08-10T09:31:38.700011Z","shell.execute_reply.started":"2022-08-10T09:31:34.714148Z","shell.execute_reply":"2022-08-10T09:31:38.698794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## check further train/test overlap","metadata":{}},{"cell_type":"code","source":"test_df.select_dtypes(\"O\").nunique()","metadata":{"execution":{"iopub.status.busy":"2022-08-10T09:26:24.240881Z","iopub.execute_input":"2022-08-10T09:26:24.241917Z","iopub.status.idle":"2022-08-10T09:26:24.320924Z","shell.execute_reply.started":"2022-08-10T09:26:24.241865Z","shell.execute_reply":"2022-08-10T09:26:24.319803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# cats = test_df.select_dtypes(\"O\").uniques()\n# cats\nfor c in test_df.select_dtypes(\"O\").columns:\n    print(c)\n    test_c = set(test_df[c])\n    print(len(test_c),\"uniques in test\")\n    train_c = set(train_df[c])\n    overlap_percent = 100*len(train_c.intersection(test_c))/len(train_c)\n    if overlap_percent<100:\n        print(\"train overlap %:\",overlap_percent)\n        print(\"train overlap:\",len(train_c.intersection(test_c)))","metadata":{"execution":{"iopub.status.busy":"2022-08-10T09:30:20.928383Z","iopub.execute_input":"2022-08-10T09:30:20.928886Z","iopub.status.idle":"2022-08-10T09:30:30.791657Z","shell.execute_reply.started":"2022-08-10T09:30:20.928850Z","shell.execute_reply":"2022-08-10T09:30:30.790438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.dtypes","metadata":{"execution":{"iopub.status.busy":"2022-08-09T14:18:02.004611Z","iopub.execute_input":"2022-08-09T14:18:02.005Z","iopub.status.idle":"2022-08-09T14:18:02.014648Z","shell.execute_reply.started":"2022-08-09T14:18:02.004967Z","shell.execute_reply":"2022-08-09T14:18:02.01336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# imports","metadata":{}},{"cell_type":"code","source":"from matplotlib import pyplot as plt\nimport seaborn as sns\nfrom catboost import CatBoostRegressor\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import mean_squared_error ","metadata":{"execution":{"iopub.status.busy":"2022-08-09T14:04:12.961312Z","iopub.execute_input":"2022-08-09T14:04:12.961869Z","iopub.status.idle":"2022-08-09T14:04:14.322985Z","shell.execute_reply.started":"2022-08-09T14:04:12.961816Z","shell.execute_reply":"2022-08-09T14:04:14.320151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# dataframe info stats\n\ndef stats(data):\n    \n    maxx = []\n    minn = []\n    for i in data.columns:\n        maxx.append(data[i].value_counts().max())\n        minn.append(data[i].value_counts().min())\n\n    return pd.DataFrame(\n        {'nunique': data.nunique(),\n         'len': len(data),\n\n         'types':data.dtypes,\n         'Nulls' : data.isna().sum(),\n                 # 'nunique/len': data.nunique()/len(data),\n        # 'Nullpercent' : data.isna().sum()/len(data),\n         \"Value counts Max\": maxx,\n         'Value counts Min':minn \n        },\n        columns = ['nunique', 'len','types','Nulls'#,'Nullpercent', 'nunique/len'\n                   ,\"Value counts Max\",'Value counts Min']).\\\n        sort_values(by ='nunique',ascending = False)\n\n\n\ndef countPlot(col,num = 6,hue = None):\n    sns.set(rc={'figure.figsize':(6,6)})\n    ax = sns.countplot(x=col, data=train_df, hue = hue,\n                   order=train_df[col].value_counts().iloc[:num].index)\n    \n    return plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-09T14:04:14.329587Z","iopub.execute_input":"2022-08-09T14:04:14.331351Z","iopub.status.idle":"2022-08-09T14:04:14.344873Z","shell.execute_reply.started":"2022-08-09T14:04:14.331296Z","shell.execute_reply":"2022-08-09T14:04:14.343002Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-09T14:04:14.347342Z","iopub.execute_input":"2022-08-09T14:04:14.347766Z","iopub.status.idle":"2022-08-09T14:04:14.396947Z","shell.execute_reply.started":"2022-08-09T14:04:14.34773Z","shell.execute_reply":"2022-08-09T14:04:14.395576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### export","metadata":{}},{"cell_type":"code","source":"# train_df.to_parquet(\"digital-turbine-auction-bid-price_train.parquet\",index=False)\n# test_df.to_parquet(\"digital-turbine-auction-bid-price_test.parquet\",index=False)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stats(train_df)","metadata":{"execution":{"iopub.status.busy":"2022-08-09T14:04:14.435244Z","iopub.execute_input":"2022-08-09T14:04:14.43618Z","iopub.status.idle":"2022-08-09T14:04:51.772236Z","shell.execute_reply.started":"2022-08-09T14:04:14.436135Z","shell.execute_reply":"2022-08-09T14:04:51.77056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stats(test_df)","metadata":{"execution":{"iopub.status.busy":"2022-08-09T14:04:51.77374Z","iopub.execute_input":"2022-08-09T14:04:51.774649Z","iopub.status.idle":"2022-08-09T14:04:52.006906Z","shell.execute_reply.started":"2022-08-09T14:04:51.774608Z","shell.execute_reply":"2022-08-09T14:04:52.004914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Eda","metadata":{}},{"cell_type":"code","source":"fig, axes = plt.subplots(2, 2)\naxes = axes.ravel()\ntrain_df['unitDisplayType'].hist(figsize = (8, 8),ax=axes[0])\ntrain_df['connectionType'].hist(figsize = (8, 8),ax=axes[1])\ntrain_df['c3'].hist(figsize = (8, 8),ax=axes[2])\ntrain_df['has_won'].hist(figsize = (8, 8),ax=axes[3])\n\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-09T14:04:52.008522Z","iopub.execute_input":"2022-08-09T14:04:52.008974Z","iopub.status.idle":"2022-08-09T14:05:01.89863Z","shell.execute_reply.started":"2022-08-09T14:04:52.008931Z","shell.execute_reply":"2022-08-09T14:05:01.897401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"countPlot('countryCode',10)","metadata":{"execution":{"iopub.status.busy":"2022-08-09T14:05:01.903074Z","iopub.execute_input":"2022-08-09T14:05:01.904106Z","iopub.status.idle":"2022-08-09T14:05:06.047294Z","shell.execute_reply.started":"2022-08-09T14:05:01.904062Z","shell.execute_reply":"2022-08-09T14:05:06.045784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"countPlot('winBid',10)","metadata":{"execution":{"iopub.status.busy":"2022-08-09T14:05:06.049391Z","iopub.execute_input":"2022-08-09T14:05:06.049897Z","iopub.status.idle":"2022-08-09T14:05:07.575676Z","shell.execute_reply.started":"2022-08-09T14:05:06.049854Z","shell.execute_reply":"2022-08-09T14:05:07.574447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"countPlot('sentPrice',6,'has_won')","metadata":{"execution":{"iopub.status.busy":"2022-08-09T14:05:07.577338Z","iopub.execute_input":"2022-08-09T14:05:07.578531Z","iopub.status.idle":"2022-08-09T14:05:09.815123Z","shell.execute_reply.started":"2022-08-09T14:05:07.578481Z","shell.execute_reply":"2022-08-09T14:05:09.813698Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"countPlot('mediationProviderVersion')","metadata":{"execution":{"iopub.status.busy":"2022-08-09T14:05:09.817616Z","iopub.execute_input":"2022-08-09T14:05:09.818126Z","iopub.status.idle":"2022-08-09T14:05:13.913197Z","shell.execute_reply.started":"2022-08-09T14:05:09.818081Z","shell.execute_reply":"2022-08-09T14:05:13.911188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def percentile80(x):\n    return np.percentile(x,80)\n\npivoted = train_df.pivot_table(index = ['countryCode'], values = ['winBid'], \n               aggfunc = [np.mean, np.median, np.std,'count', percentile80])\n\npivoted.columns = pivoted.columns.get_level_values(0)\n\n\npivoted.sort_values('count',ascending=False).head(17)","metadata":{"execution":{"iopub.status.busy":"2022-08-09T14:05:13.915777Z","iopub.execute_input":"2022-08-09T14:05:13.917076Z","iopub.status.idle":"2022-08-09T14:05:18.817098Z","shell.execute_reply.started":"2022-08-09T14:05:13.917008Z","shell.execute_reply":"2022-08-09T14:05:18.816186Z"},"trusted":true},"execution_count":null,"outputs":[]}]}