{"cells":[{"metadata":{"toc":true,"_uuid":"efbbfec6dd93a26bd09c7f3388f8b7e0414656cf"},"cell_type":"markdown","source":"

Table of Contents

\n
"},{"metadata":{"_uuid":"b700f8fface83da956028bf5b5dca271a0868d66"},"cell_type":"markdown","source":"# Preprocess "},{"metadata":{"trusted":false,"_uuid":"9e237058178a5091c8d9e8df32213be466c44593","collapsed":true},"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nfrom pathlib import Path\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport gc\nimport dask.dataframe as dd\n\n%matplotlib inline\n%load_ext memory_profiler","execution_count":18,"outputs":[]},{"metadata":{"_uuid":"f11eb222f4c717edf714fdcde1d335ba336bacb1"},"cell_type":"markdown","source":"# Setting"},{"metadata":{"_uuid":"60fed709d3e5e56af8ce412982b53d868ece686b"},"cell_type":"markdown","source":"## Path"},{"metadata":{"trusted":false,"_uuid":"b27ad152fb1da86173d57ee2af2e4ac586e04cd6","collapsed":true},"cell_type":"code","source":"HOME_PATH = Path(os.getcwd()).parent\ndire = str(HOME_PATH)\nprint(os.path.exists(dire))\nprint(dire)\n \ndata_Path = HOME_PATH / \"data\"\ndire = str(data_Path)\nprint(os.path.exists(dire))\nprint(dire)\n\nraw_data_Path = data_Path / \"raw\"\ndire = str(raw_data_Path)\nprint(os.path.exists(dire))\nprint(dire)\n\ntrain_csv_Path = raw_data_Path / \"train.csv\"\ndire = str(train_csv_Path)\nprint(os.path.exists(dire))\nprint(dire)\n\ntrain_sample_csv_Path = raw_data_Path / \"train_sample.csv\"\ndire = str(train_sample_csv_Path)\nprint(os.path.exists(dire))\nprint(dire)\n\ntest_csv_Path = raw_data_Path / \"test.csv\"\ndire = str(test_csv_Path)\nprint(os.path.exists(dire))\nprint(dire)\n\ninterim_data_Path = data_Path / \"interim\"\ndire = str(interim_data_Path)\nprint(os.path.exists(dire))\nprint(dire)\n","execution_count":19,"outputs":[]},{"metadata":{"_uuid":"d42582f192ceadc0c6d085565fdd3fcde0bbd15e"},"cell_type":"markdown","source":"## dtype"},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"3733cd25f3825d8677a1cf39f2a486b8fc88e2b5"},"cell_type":"code","source":"dtypes = {\n \"ip\" : \"uint32\",\n \"app\" : \"uint16\",\n \"device\" : \"uint16\",\n \"os\" : \"uint16\",\n \"channel\" : \"uint16\",\n \"is_attributed\" : \"uint8\",\n}","execution_count":20,"outputs":[]},{"metadata":{"_uuid":"0ecb6eaf11672dc33b7c278fa500631c8436e3dc"},"cell_type":"markdown","source":"## read_csv"},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"419cf1acc0d641098b69537e8ca0bbca8862f4ef"},"cell_type":"code","source":"df_train = dd.read_csv(urlpath=str(train_csv_Path),\n dtype=dtypes,\n parse_dates=[\"click_time\", \"attributed_time\"],\n )","execution_count":21,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"68251d1a025541e97479e5578dc985c874e12d0b","collapsed":true},"cell_type":"code","source":"df_train.head()","execution_count":22,"outputs":[]},{"metadata":{"_uuid":"3758379ecbcece8dd96641cb2835caa0811254ad"},"cell_type":"markdown","source":"# IsNull"},{"metadata":{"trusted":false,"_uuid":"b305f5a3eb1a501268c5896592da98d989580fc5","collapsed":true},"cell_type":"code","source":"%%time\n%%memit\ntrain_isnull = df_train.isnull().sum().compute()\ntrain_isnull.to_csv(path=str(interim_data_Path / \"train_isnull.csv\"))","execution_count":7,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"19e9790d240fb352403bdba0efb24fced9856680","collapsed":true},"cell_type":"code","source":"train_isnull = pd.read_csv(filepath_or_buffer=str(interim_data_Path / \"train_isnull.csv\"), header=None)\ntrain_isnull","execution_count":8,"outputs":[]},{"metadata":{"scrolled":true,"trusted":false,"_uuid":"039dbb8ee933d17b5ffd7e736611871014f0919f","collapsed":true},"cell_type":"code","source":"%%time\n%%memit\ngc.collect()","execution_count":9,"outputs":[]},{"metadata":{"_uuid":"dad8024c33397595c9714e0c07c17a0bd82efe33"},"cell_type":"markdown","source":"# Describe"},{"metadata":{"trusted":false,"_uuid":"162ee81cc468809c55313c265d7c6f3a14247411","collapsed":true},"cell_type":"code","source":"%%time\n%%memit\ntrain_desc = df_train.describe().compute()\n# save\ntrain_desc.to_csv(path_or_buf=str(interim_data_Path / \"train_desc.csv\"))","execution_count":12,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"77a236849362bc6a00c393168b180b63db245add","collapsed":true},"cell_type":"code","source":"# load\ntrain_desc = pd.read_csv(filepath_or_buffer=str(interim_data_Path / \"train_desc.csv\"),)\ntrain_desc","execution_count":13,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"add2ed6aba793b059353a7aa49bcc2c69190f348","collapsed":true},"cell_type":"code","source":"%%time\n%%memit\ngc.collect()","execution_count":14,"outputs":[]},{"metadata":{"_uuid":"256d7c6c486b5c5c4ff58fadc042954208641683"},"cell_type":"markdown","source":"# Mean"},{"metadata":{"trusted":false,"_uuid":"da9a6f34ae16dbd601a1c840ff53eea54060181a","collapsed":true},"cell_type":"code","source":"%%time\n%%memit\nmean = df_train[\"is_attributed\"].mean().compute()\nprint(mean)\nprint(type(mean))\n\nwith open(file=str(interim_data_Path / \"train_mean_of_is_attributed.txt\"), mode='wt') as f:\n f.write(str(mean))","execution_count":15,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"d15332d85b471069f34eb02afe79ead1ded3689e","collapsed":true},"cell_type":"code","source":"with open(file=str(interim_data_Path /\"train_mean_of_is_attributed.txt\"), mode=\"rt\") as f:\n mean = np.float64(f.readline())\nmean","execution_count":16,"outputs":[]},{"metadata":{"_uuid":"a399dded42b0732d1f6f8387eed8ac8467f7018f"},"cell_type":"markdown","source":"# Unique\nthe number of unique features"},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"41461f2286624cec5a51e6d8aa217cd65e085736"},"cell_type":"code","source":"# exclude \"click_time\", \"attribute\"\n # \"click_time\" : too many unique values\n # \"attribute_time\" : too many missing value\nfeatures = [\"ip\", \"app\", \"device\", \"os\", \"channel\"]","execution_count":23,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"7e54ab8bbfd67a01043e1914fdcc9ff0cb2193b8","collapsed":true},"cell_type":"code","source":"%%time\n%%memit\nuniques = [len(df_train[f].unique().compute()) for f in features]\n\nprint(type(uniques[0]))\n\nuniques = pd.DataFrame(data=np.array(uniques).reshape(1, 5), columns=features)\nuniques.to_csv(path_or_buf=str(interim_data_Path / \"train_features_uniques.csv\"))\nuniques","execution_count":17,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"f9195fc7b37f224380d286805cbc75f42b5e2546","collapsed":true},"cell_type":"code","source":"uniques = pd.read_csv(filepath_or_buffer=str(interim_data_Path / \"train_features_uniques.csv\"),\n index_col=0,\n #dtype=,\n )\nuniques","execution_count":24,"outputs":[]},{"metadata":{"_uuid":"99d5c59ebaf98b14a0b48d87def83e379a8f1720"},"cell_type":"markdown","source":"## Histgram"},{"metadata":{"trusted":false,"_uuid":"3c54facb783b8aa5618a63779fcc3c16183b0492","collapsed":true},"cell_type":"code","source":"%%time\n\nfig = plt.figure(figsize=(10, 6))\nax1 = fig.add_subplot(1, 1, 1)\nval = np.log10(uniques.values[0])\nprint(val)\n\nidx = np.arange(len(uniques.columns))\nb = ax1.bar(idx, val)\nax1.set_xticks(idx)\nax1.set_xticklabels(uniques.columns)\n#ax1.set_ylim([0, 100])\nax1.set_ylabel('log10(y)')\n\n\ncolor_list = [\"#6666ff\", \"#ffb266\", \"#009900\", \"#000099\", \"#cc99ff\"]\nfor i in idx:\n b[i].set_facecolor(color_list[i])","execution_count":26,"outputs":[]},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"d0e3f11cc51e0d8d9f25e0b2304f64339761d86b"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"56ffedff594b11987f3ee248f76c769ceec3d761"},"cell_type":"markdown","source":"# IP"},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"de0b97a06e1f65298752776d277cfe9c3c25a14d"},"cell_type":"code","source":"fea = \"ip\"","execution_count":136,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"f5c1f312ab8d5b92f00721d4baecd1171e9b8811","collapsed":true},"cell_type":"code","source":"df_fea_val_counts_tmp = df_train[fea].value_counts()\nprint(type(df_fea_val_counts_tmp))\n\ndf_fea_val_counts_tmp = df_fea_val_counts_tmp.to_frame()\nprint(type(df_fea_val_counts_tmp))\n\ndf_fea_val_counts = df_fea_val_counts_tmp.reset_index(drop=False)\n\ndel df_fea_val_counts_tmp\n\ndf_fea_val_counts.columns = [fea, 'counts']\n\n# save df_ip_val_counts\ndf_fea_val_counts.to_csv(filename=str(interim_data_Path / (\"train_\"+fea+\"_val_counts-*.csv\")))\n\ndel df_fea_val_counts\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"5629417e0096b6165adb83440838b887dbe60a1f"},"cell_type":"code","source":"# load\ntmp = dd.read_csv(urlpath=str(interim_data_Path / (\"train_\"+fea+\"_val_counts-0.csv\")),)\ndf_fea_val_counts = tmp.drop(labels=tmp.columns[0], axis=1)","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":false,"collapsed":true,"_uuid":"159e78a970652292d2a47d14730424e0cc5c0725"},"cell_type":"code","source":"df_fea_val_counts.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1f40b79f65540c90bd8affe97b2fbcd299290302"},"cell_type":"markdown","source":"## merge"},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"ca2c4c970edea4cd23135b4a16265a8a960f9df0"},"cell_type":"code","source":"df_train_tmp = dd.merge(left=df_train,\n right=df_fea_val_counts,\n how=\"left\",\n on=fea,\n )","execution_count":33,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"89e9071ff96225d09533bdebfe2881518203426f","collapsed":true},"cell_type":"code","source":"df_train_tmp.head()","execution_count":34,"outputs":[]},{"metadata":{"_uuid":"50c021b38af8104e44b3dc951ad78672d4ca1255"},"cell_type":"markdown","source":"## sort all\nip unique : 277396"},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"62e3aac88cbf634bf9349ddd89922fb87a5848cd"},"cell_type":"code","source":"n = 1000000 # 1000000 > 277396","execution_count":35,"outputs":[]},{"metadata":{"_uuid":"9c5c5b59867d0439955e5a978ddf0d4a8630b28c"},"cell_type":"markdown","source":"### proportion"},{"metadata":{"scrolled":false,"trusted":false,"collapsed":true,"_uuid":"d1f8be9ebdf6e4b3827bd2de7830bdbcc2268a85"},"cell_type":"code","source":"proportion = df_train_tmp[[fea, 'is_attributed']].groupby(by=fea).mean().reset_index().nlargest(n=n, columns='is_attributed').reset_index(drop=True)","execution_count":36,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"f892b48c79db0c1f43c36a50a4e0c9ecef4d33c2","collapsed":true},"cell_type":"code","source":"# save\nproportion.to_csv(filename=str(interim_data_Path / (\"train_\"+fea+\"_isAttributed_proportion-*.csv\")))","execution_count":37,"outputs":[]},{"metadata":{"_uuid":"72191f96ac8ce9085633984bcb2de8bfe8a6397c"},"cell_type":"markdown","source":"#### load proportion"},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"a5fde6cefa866c73b5bf8e33c42ba546af38aa82"},"cell_type":"code","source":"# load\n#tmp = dd.read_csv(urlpath=str(interim_data_Path / (\"train_\"+fea+\"_isAttributed_proportion-0.csv\")),)\n#proportion = tmp.drop(labels=tmp.columns[0], axis=1)\n#\nproportion = pd.read_csv(filepath_or_buffer=str(interim_data_Path / (\"train_\"+fea+\"_isAttributed_proportion-0.csv\")),\n index_col=0,\n )","execution_count":131,"outputs":[]},{"metadata":{"_uuid":"08378928db9bcfefdb9ff92fd351b7d946069315"},"cell_type":"markdown","source":"### counts"},{"metadata":{"scrolled":false,"trusted":false,"collapsed":true,"_uuid":"7396b97b6d7dd98a3119997680efcf38f0011f74"},"cell_type":"code","source":"counts = df_train_tmp[[fea, 'is_attributed']].groupby(by=fea).count().reset_index().nlargest(n=n, columns='is_attributed').reset_index(drop=True)","execution_count":39,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"5083ca8a4d872470b6a92ced52b368ff1c7d6725","collapsed":true},"cell_type":"code","source":"# save\ncounts.to_csv(filename=str(interim_data_Path / (\"train_\"+fea+\"_isAttributed_counts-*.csv\")))","execution_count":40,"outputs":[]},{"metadata":{"_uuid":"46f16f6ed75967bb88ca93691c187727981cf614"},"cell_type":"markdown","source":"#### load counts"},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"238f30611e6e66abf7e66ebf138b5862305bde77"},"cell_type":"code","source":"# load\n#tmp = dd.read_csv(urlpath=str(interim_data_Path / (\"train_\"+fea+\"_isAttributed_counts-0.csv\")),)\n#counts = tmp.drop(labels=tmp.columns[0], axis=1)\n#\ncounts = pd.read_csv(filepath_or_buffer=str(interim_data_Path / (\"train_\"+fea+\"_isAttributed_counts-0.csv\")),\n index_col=0,\n )","execution_count":132,"outputs":[]},{"metadata":{"_uuid":"01961e0b207704af17397ad6b323d22d967507d5"},"cell_type":"markdown","source":"### Head"},{"metadata":{"scrolled":true,"trusted":false,"_uuid":"e7c7196d38ec6367f8def6118453a5f69c69b5ff","collapsed":true},"cell_type":"code","source":"proportion.head()","execution_count":42,"outputs":[]},{"metadata":{"scrolled":false,"trusted":false,"_uuid":"6c85a5b368188ac277894d62f3c1b69ed18832ec","collapsed":true},"cell_type":"code","source":"counts.head()","execution_count":43,"outputs":[]},{"metadata":{"_uuid":"001239ea6ae767568eeaf05b9f7dd321d788d9c2"},"cell_type":"markdown","source":"## Merge"},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"01ad5f93eb525b720252f68d005050f9407cfcc4"},"cell_type":"code","source":"df_merge = counts.merge(right=proportion, on=fea, how='left', )\ndf_merge.columns = [fea, 'click_count', 'prop_downloaded']","execution_count":133,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"7fdb8ac30ebbdb87d9a50f4e9df8e33159be6d15","collapsed":true},"cell_type":"code","source":"type(df_merge['click_count'])","execution_count":45,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"e3b7cebe0d9ca486a7c4dc703367d155360b894c","collapsed":true},"cell_type":"code","source":"df_merge.head()","execution_count":46,"outputs":[]},{"metadata":{"_uuid":"b2fd2bd3a320d1eadb68ea502f1f20e8e6f2f921"},"cell_type":"markdown","source":"## Plot"},{"metadata":{"scrolled":false,"trusted":false,"_uuid":"9f09a3e6c2d2713099da137b51ed9eb7c9a23464","collapsed":true},"cell_type":"code","source":"title = \"Conversion_Rates_over_Counts_of_300_Most_Popular_\"+fea.upper()+\"s\"\n\nfig = plt.figure(figsize=(15, 6))\nax1 = fig.add_subplot(1, 1, 1)\n\nn = 300\n\nax1.set_yscale(\"linear\")\nax1.yaxis.tick_left()\ndf_merge[['click_count']][:n].plot(ax=ax1, color=\"blue\",)\nax1.yaxis.set_label_position(\"left\")\nax1.set_ylabel(ylabel='Count of clicks')\nax1.legend(loc=3, bbox_to_anchor=(1.05, 1), borderaxespad=0.)\n\nax2 = ax1.twinx()\nax2.yaxis.tick_right()\ndf_merge[['prop_downloaded']][:n].plot(ax=ax2, color=\"green\",)\nax2.yaxis.set_label_position(\"right\")\nax2.set_ylabel(ylabel='Proportion Downloaded')\nax2.legend(loc=3, bbox_to_anchor=(1.05, 1 + 0.07), borderaxespad=0.)\n\nax2.set_title(title)\nax2.set_xlabel(\"Sort by \" + fea)\n#ax.legend(loc=\"best\")\n#ax.set_ylabel('Proportion Downloaded')\n#plt.show()\nfig.savefig(fname=str(data_Path / \"plot\" / title))\nplt.show()\n","execution_count":134,"outputs":[]},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"307d7140012179243b6ee7eeacd93eb4fe66a7bf"},"cell_type":"code","source":"print(df_merge[:20])","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"d2494716bc61f84aa61a15da0ccb7cb185419eaf","collapsed":true},"cell_type":"code","source":"gc.collect()","execution_count":51,"outputs":[]},{"metadata":{"_uuid":"4b7db636ab189178d2b65a6eda8b2a1bd94a460d"},"cell_type":"markdown","source":"# APP"},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"a720edbe1a56fce61523406274c3777e0bee08b4"},"cell_type":"code","source":"fea = \"app\"","execution_count":52,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"cf81ad5fe69f871aba244cc21f3feec0d71dc952","collapsed":true},"cell_type":"code","source":"df_fea_val_counts_tmp = df_train[fea].value_counts()\nprint(type(df_fea_val_counts_tmp))\n\ndf_fea_val_counts_tmp = df_fea_val_counts_tmp.to_frame()\nprint(type(df_fea_val_counts_tmp))\n\ndf_fea_val_counts = df_fea_val_counts_tmp.reset_index(drop=False)\n\ndel df_fea_val_counts_tmp\n\ndf_fea_val_counts.columns = [fea, 'counts']\n\n# save df_ip_val_counts\ndf_fea_val_counts.to_csv(filename=str(interim_data_Path / (\"train_\"+fea+\"_val_counts-*.csv\")))\n\ndel df_fea_val_counts\ngc.collect()","execution_count":53,"outputs":[]},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"85bd3fc91234c08641c703621ead312b74475f54"},"cell_type":"code","source":"# load\ntmp = dd.read_csv(urlpath=str(interim_data_Path / (\"train_\"+fea+\"_val_counts-0.csv\")),)\ndf_fea_val_counts = tmp.drop(labels=tmp.columns[0], axis=1)","execution_count":54,"outputs":[]},{"metadata":{"scrolled":true,"trusted":false,"_uuid":"08f00850a37c7272621d7e39824b6fce48dbfbae","collapsed":true},"cell_type":"code","source":"df_fea_val_counts.head()","execution_count":55,"outputs":[]},{"metadata":{"_uuid":"dd36a857e4ac7fe1a4399213e43f44b6a27791b0"},"cell_type":"markdown","source":"## merge"},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"ff2ba37e189a8c041c33608aeb64baa4e21a5355"},"cell_type":"code","source":"df_train_tmp = dd.merge(left=df_train,\n right=df_fea_val_counts,\n how=\"left\",\n on=fea,\n )","execution_count":56,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"345b721f98d758095f8b11b75e1cd98b708f325e","collapsed":true},"cell_type":"code","source":"df_train_tmp.head()","execution_count":57,"outputs":[]},{"metadata":{"_uuid":"93cbc579eddbdf068c3ce5e3df509eeb6cb217fe"},"cell_type":"markdown","source":"## sort all\nip unique : 277396"},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"fd53ded13c7b561d82212d430e9f91bbca6f2a62"},"cell_type":"code","source":"n = 1000000 # 1000000 > 277396","execution_count":58,"outputs":[]},{"metadata":{"_uuid":"7b101d4748cbf5c679e2f2e46843b98449359df8"},"cell_type":"markdown","source":"### proportion"},{"metadata":{"scrolled":false,"trusted":false,"collapsed":true,"_uuid":"681a2ce1bf4ed803ffc2468899aff4a212171662"},"cell_type":"code","source":"proportion = df_train_tmp[[fea, 'is_attributed']].groupby(by=fea).mean().reset_index().nlargest(n=n, columns='is_attributed').reset_index(drop=True)","execution_count":59,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"f78816ec3a83a6d10330105863a9fb2fb1c2f4b7","collapsed":true},"cell_type":"code","source":"# save\nproportion.to_csv(filename=str(interim_data_Path / (\"train_\"+fea+\"_isAttributed_proportion-*.csv\")))","execution_count":60,"outputs":[]},{"metadata":{"_uuid":"989b9b7a4e4855dd51f4397087846b86364a31fb"},"cell_type":"markdown","source":"#### load proportion"},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"04725d61f5b3d71fe452e02af4b3cd4749b151dd"},"cell_type":"code","source":"# load\n#tmp = dd.read_csv(urlpath=str(interim_data_Path / (\"train_\"+fea+\"_isAttributed_proportion-0.csv\")),)\n#proportion = tmp.drop(labels=tmp.columns[0], axis=1)\n#\nproportion = pd.read_csv(filepath_or_buffer=str(interim_data_Path / (\"train_\"+fea+\"_isAttributed_proportion-0.csv\")),\n index_col=0,\n )","execution_count":61,"outputs":[]},{"metadata":{"_uuid":"c573d989b4330f8f6b1d862d373dd93d4187c7cc"},"cell_type":"markdown","source":"### counts"},{"metadata":{"scrolled":false,"trusted":false,"collapsed":true,"_uuid":"01e3ae54d47549cedf0a91fc09f944a2126681f4"},"cell_type":"code","source":"counts = df_train_tmp[[fea, 'is_attributed']].groupby(by=fea).count().reset_index().nlargest(n=n, columns='is_attributed').reset_index(drop=True)","execution_count":62,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"932c4671e5ae08335cebd0ccdd48896190094e56","collapsed":true},"cell_type":"code","source":"# save\ncounts.to_csv(filename=str(interim_data_Path / (\"train_\"+fea+\"_isAttributed_counts-*.csv\")))","execution_count":63,"outputs":[]},{"metadata":{"_uuid":"46316d2ba06a924356f7126dac5a48868aa2eddb"},"cell_type":"markdown","source":"#### load counts"},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"1b04ba9ff1ceb7d5d5bf614a7efaf90ee5b6c82b"},"cell_type":"code","source":"# load\n#tmp = dd.read_csv(urlpath=str(interim_data_Path / (\"train_\"+fea+\"_isAttributed_counts-0.csv\")),)\n#counts = tmp.drop(labels=tmp.columns[0], axis=1)\n#\ncounts = pd.read_csv(filepath_or_buffer=str(interim_data_Path / (\"train_\"+fea+\"_isAttributed_counts-0.csv\")),\n index_col=0,\n )","execution_count":64,"outputs":[]},{"metadata":{"_uuid":"dc05a00fe7460d20ac266d31c97345f01b300008"},"cell_type":"markdown","source":"### Head"},{"metadata":{"scrolled":true,"trusted":false,"_uuid":"e60456fb53093121625994decbbb0feaa1b8638b","collapsed":true},"cell_type":"code","source":"proportion.head()","execution_count":65,"outputs":[]},{"metadata":{"scrolled":false,"trusted":false,"_uuid":"c1e34c34b963748e75b7f44877ed8f90e79520a2","collapsed":true},"cell_type":"code","source":"counts.head()","execution_count":66,"outputs":[]},{"metadata":{"_uuid":"3328bf9079868bb48f936807930da9b14cb865a0"},"cell_type":"markdown","source":"## Merge"},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"010d5b29bbaa21dd0d73f26ab3c59df3d604a947"},"cell_type":"code","source":"df_merge = counts.merge(right=proportion, on=fea, how='left', )\n\ndf_merge.columns = [fea, 'click_count', 'prop_downloaded']","execution_count":67,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"0c9a302500b1645dd83796102c7d7e734881c488","collapsed":true},"cell_type":"code","source":"type(df_merge['click_count'])","execution_count":68,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"2eb9099a8a26ed5023bea49888255d89094757f2","collapsed":true},"cell_type":"code","source":"df_merge['click_count'].head()","execution_count":69,"outputs":[]},{"metadata":{"_uuid":"c2e821cf96990358315fdd2f073447333d2bb28b"},"cell_type":"markdown","source":"## Plot"},{"metadata":{"scrolled":false,"trusted":false,"_uuid":"1600d796fa8d19418cbf689fbf94935aa09563db","collapsed":true},"cell_type":"code","source":"title = \"Conversion_Rates_over_Counts_of_300_Most_Popular_\"+fea.upper()+\"s\"\n\nfig = plt.figure(figsize=(15, 6))\nax1 = fig.add_subplot(1, 1, 1)\n\nn = 300\n\nax1.set_yscale(\"linear\")\nax1.yaxis.tick_left()\ndf_merge[['click_count']][:n].plot(ax=ax1, color=\"blue\",)\nax1.yaxis.set_label_position(\"left\")\nax1.set_ylabel(ylabel='Count of clicks')\nax1.legend(loc=3, bbox_to_anchor=(1.05, 1), borderaxespad=0.)\n\nax2 = ax1.twinx()\nax2.yaxis.tick_right()\ndf_merge[['prop_downloaded']][:n].plot(ax=ax2, color=\"green\",)\nax2.yaxis.set_label_position(\"right\")\nax2.set_ylabel(ylabel='Proportion Downloaded')\nax2.legend(loc=3, bbox_to_anchor=(1.05, 1 + 0.07), borderaxespad=0.)\n\nax2.set_title(title)\nax2.set_xlabel(\"Sort by \" + fea)\n#ax.legend(loc=\"best\")\n#ax.set_ylabel('Proportion Downloaded')\n#plt.show()\nfig.savefig(fname=str(data_Path / \"plot\" / title))\nplt.show()\n\nprint(df_merge[:20])","execution_count":70,"outputs":[]},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"984d3cf2fe024ecc01e4e882f15df7155133fd00"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"fd82c497172dc727c3557b248dae0673c62c4038"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"75a4a3669231206285c389036c7288ebf34ffcea"},"cell_type":"markdown","source":"# Device"},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"9decd2bacf97ad2a8289e4b3ec895e4e1a77bd8d"},"cell_type":"code","source":"fea = \"device\"","execution_count":71,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"e88ffdb1edd69f26875a927d1a5a48ca71566f00","collapsed":true},"cell_type":"code","source":"df_fea_val_counts_tmp = df_train[fea].value_counts()\nprint(type(df_fea_val_counts_tmp))\n\ndf_fea_val_counts_tmp = df_fea_val_counts_tmp.to_frame()\nprint(type(df_fea_val_counts_tmp))\n\ndf_fea_val_counts = df_fea_val_counts_tmp.reset_index(drop=False)\n\ndel df_fea_val_counts_tmp\n\ndf_fea_val_counts.columns = [fea, 'counts']\n\n# save df_ip_val_counts\ndf_fea_val_counts.to_csv(filename=str(interim_data_Path / (\"train_\"+fea+\"_val_counts-*.csv\")))\n\ndel df_fea_val_counts\ngc.collect()","execution_count":72,"outputs":[]},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"bec0af766c58a0fadcd6c4c1a976791795132cec"},"cell_type":"code","source":"# load\ntmp = dd.read_csv(urlpath=str(interim_data_Path / (\"train_\"+fea+\"_val_counts-0.csv\")),)\ndf_fea_val_counts = tmp.drop(labels=tmp.columns[0], axis=1)","execution_count":73,"outputs":[]},{"metadata":{"scrolled":true,"trusted":false,"_uuid":"756e2a660c654aeda9726e9edea7a67b918362b5","collapsed":true},"cell_type":"code","source":"df_fea_val_counts.head()","execution_count":74,"outputs":[]},{"metadata":{"_uuid":"ceecaf804393318b50d011e2499ac28bcd60e564"},"cell_type":"markdown","source":"## merge"},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"5158504b350fa818d432cdb0f97502020cb2b1ee"},"cell_type":"code","source":"df_train_tmp = dd.merge(left=df_train,\n right=df_fea_val_counts,\n how=\"left\",\n on=fea,\n )","execution_count":75,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"09bad2ae8613664c5623d8232fe1bd2c41412790","collapsed":true},"cell_type":"code","source":"df_train_tmp.head()","execution_count":76,"outputs":[]},{"metadata":{"_uuid":"150796b5f722d16b33f1bf783cd471d58a97daee"},"cell_type":"markdown","source":"## sort all\nip unique : 277396"},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"5203b95595b27021e984b7ead0604ac3a265a223"},"cell_type":"code","source":"n = 1000000 # 1000000 > 277396","execution_count":77,"outputs":[]},{"metadata":{"_uuid":"dec0fab808d0f31a681212fb3efe5718e82f04b9"},"cell_type":"markdown","source":"### proportion"},{"metadata":{"scrolled":false,"trusted":false,"collapsed":true,"_uuid":"720b81e2a396fa5038f0588647e5a457040e82bd"},"cell_type":"code","source":"proportion = df_train_tmp[[fea, 'is_attributed']].groupby(by=fea).mean().reset_index().nlargest(n=n, columns='is_attributed').reset_index(drop=True)","execution_count":78,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"efab33ad0dfd9548cb3ef1ea1fda66b51786b2d4","collapsed":true},"cell_type":"code","source":"# save\nproportion.to_csv(filename=str(interim_data_Path / (\"train_\"+fea+\"_isAttributed_proportion-*.csv\")))","execution_count":79,"outputs":[]},{"metadata":{"_uuid":"0da660ca9c2d25d21f561952d03056bfb5b8c885"},"cell_type":"markdown","source":"#### load proportion"},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"d22af0a098a4c1999e5abbb90a995f57c4732227"},"cell_type":"code","source":"# load\n#tmp = dd.read_csv(urlpath=str(interim_data_Path / (\"train_\"+fea+\"_isAttributed_proportion-0.csv\")),)\n#proportion = tmp.drop(labels=tmp.columns[0], axis=1)\n#\nproportion = pd.read_csv(filepath_or_buffer=str(interim_data_Path / (\"train_\"+fea+\"_isAttributed_proportion-0.csv\")),\n index_col=0,\n )","execution_count":80,"outputs":[]},{"metadata":{"_uuid":"c675700160330b9a321da9c3c2d9dfe1b6e3bb0f"},"cell_type":"markdown","source":"### counts"},{"metadata":{"scrolled":false,"trusted":false,"collapsed":true,"_uuid":"2d057e385c59500c59ed1b61d02734dbe94883bd"},"cell_type":"code","source":"counts = df_train_tmp[[fea, 'is_attributed']].groupby(by=fea).count().reset_index().nlargest(n=n, columns='is_attributed').reset_index(drop=True)","execution_count":81,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"a50d3da52662915a41324dc49be112a7e2a7bce3","collapsed":true},"cell_type":"code","source":"# save\ncounts.to_csv(filename=str(interim_data_Path / (\"train_\"+fea+\"_isAttributed_counts-*.csv\")))","execution_count":82,"outputs":[]},{"metadata":{"_uuid":"f59c933ef121ee92da5d042d51804353aba9397e"},"cell_type":"markdown","source":"#### load counts"},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"5ae6c997da9f42c9fa766637cb1d60c84c57a195"},"cell_type":"code","source":"# load\n#tmp = dd.read_csv(urlpath=str(interim_data_Path / (\"train_\"+fea+\"_isAttributed_counts-0.csv\")),)\n#counts = tmp.drop(labels=tmp.columns[0], axis=1)\n#\ncounts = pd.read_csv(filepath_or_buffer=str(interim_data_Path / (\"train_\"+fea+\"_isAttributed_counts-0.csv\")),\n index_col=0,\n )","execution_count":83,"outputs":[]},{"metadata":{"_uuid":"fd24cb643e15b3c2c8dc6966a94a502262234178"},"cell_type":"markdown","source":"### Head"},{"metadata":{"scrolled":true,"trusted":false,"_uuid":"a528ff36bf7ead8d411b781b1cdf3316ff87f3ce","collapsed":true},"cell_type":"code","source":"proportion.head()","execution_count":84,"outputs":[]},{"metadata":{"scrolled":false,"trusted":false,"_uuid":"f40e749e416902c5155bad154ad8e9808493f124","collapsed":true},"cell_type":"code","source":"counts.head()","execution_count":85,"outputs":[]},{"metadata":{"_uuid":"3cba2bd48c0d553e48ac379df1e28dcf51c69cb5"},"cell_type":"markdown","source":"## Merge"},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"899fb331dc99158426e8f10a614aa0ec664f29a5"},"cell_type":"code","source":"df_merge = counts.merge(right=proportion, on=fea, how='left', )\n\ndf_merge.columns = [fea, 'click_count', 'prop_downloaded']","execution_count":86,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"c68e9e728e84a33d682ee18754e0093790b92f1e","collapsed":true},"cell_type":"code","source":"type(df_merge['click_count'])","execution_count":87,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"25f20c18d6ab97575e6121ab95c4498b70f45b86","collapsed":true},"cell_type":"code","source":"df_merge['click_count'].head()","execution_count":88,"outputs":[]},{"metadata":{"_uuid":"7e633eb1043d18f4f5f22b9b1513b51334e0c366"},"cell_type":"markdown","source":"## Plot"},{"metadata":{"scrolled":false,"trusted":false,"_uuid":"b80546abed364c408317b430f0636e3f2856b669","collapsed":true},"cell_type":"code","source":"title = \"Conversion_Rates_over_Counts_of_300_Most_Popular_\"+fea.upper()+\"s\"\n\nfig = plt.figure(figsize=(15, 6))\nax1 = fig.add_subplot(1, 1, 1)\n\nn = 300\n\nax1.set_yscale(\"linear\")\nax1.yaxis.tick_left()\ndf_merge[['click_count']][:n].plot(ax=ax1, color=\"blue\",)\nax1.yaxis.set_label_position(\"left\")\nax1.set_ylabel(ylabel='Count of clicks')\nax1.legend(loc=3, bbox_to_anchor=(1.05, 1), borderaxespad=0.)\n\nax2 = ax1.twinx()\nax2.yaxis.tick_right()\ndf_merge[['prop_downloaded']][:n].plot(ax=ax2, color=\"green\",)\nax2.yaxis.set_label_position(\"right\")\nax2.set_ylabel(ylabel='Proportion Downloaded')\nax2.legend(loc=3, bbox_to_anchor=(1.05, 1 + 0.07), borderaxespad=0.)\n\nax2.set_title(title)\nax2.set_xlabel(\"Sort by \" + fea)\n#ax.legend(loc=\"best\")\n#ax.set_ylabel('Proportion Downloaded')\n#plt.show()\nfig.savefig(fname=str(data_Path / \"plot\" / title))\nplt.show()\n\nprint(df_merge[:20])","execution_count":89,"outputs":[]},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"74fa2904697edc88536a1493db24095f1228ff43"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"647e8bd9bd38dc6ac1a509bab63291555e72df49"},"cell_type":"markdown","source":"# OS"},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"a2673fa188f31cd9146f2f26bc67b0baf5ccea67"},"cell_type":"code","source":"fea = \"os\"","execution_count":90,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"e394d7eb975a7d30d04c5c0afca7759d03bd4517","collapsed":true},"cell_type":"code","source":"df_fea_val_counts_tmp = df_train[fea].value_counts()\nprint(type(df_fea_val_counts_tmp))\n\ndf_fea_val_counts_tmp = df_fea_val_counts_tmp.to_frame()\nprint(type(df_fea_val_counts_tmp))\n\ndf_fea_val_counts = df_fea_val_counts_tmp.reset_index(drop=False)\n\ndel df_fea_val_counts_tmp\n\ndf_fea_val_counts.columns = [fea, 'counts']\n\n# save df_ip_val_counts\ndf_fea_val_counts.to_csv(filename=str(interim_data_Path / (\"train_\"+fea+\"_val_counts-*.csv\")))\n\ndel df_fea_val_counts\ngc.collect()","execution_count":91,"outputs":[]},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"58255611ab11a834acca6e6342b5a5e3fc7ff525"},"cell_type":"code","source":"# load\ntmp = dd.read_csv(urlpath=str(interim_data_Path / (\"train_\"+fea+\"_val_counts-0.csv\")),)\ndf_fea_val_counts = tmp.drop(labels=tmp.columns[0], axis=1)","execution_count":92,"outputs":[]},{"metadata":{"scrolled":true,"trusted":false,"_uuid":"a81d26034c28fd8e3805f689ecae50f23d657d70","collapsed":true},"cell_type":"code","source":"df_fea_val_counts.head()","execution_count":93,"outputs":[]},{"metadata":{"_uuid":"9a42e26c253b510aa4bc4d35c742dbd1472ef6f4"},"cell_type":"markdown","source":"## merge"},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"7b7274529dd518ce93338f5920671b883e4340da"},"cell_type":"code","source":"df_train_tmp = dd.merge(left=df_train,\n right=df_fea_val_counts,\n how=\"left\",\n on=fea,\n )","execution_count":94,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"7d10f17d58bd47e2dd9e532f49eac6cd06c85eda","collapsed":true},"cell_type":"code","source":"df_train_tmp.head()","execution_count":95,"outputs":[]},{"metadata":{"_uuid":"8315fadc4238bac7f42aca162b7a404c35feb2ac"},"cell_type":"markdown","source":"## sort all\nip unique : 277396"},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"b2053edf5146664b5d3b9604fb81ec38d09f29a0"},"cell_type":"code","source":"n = 1000000 # 1000000 > 277396","execution_count":96,"outputs":[]},{"metadata":{"_uuid":"b3e169c87c2609af4c2623a057d492fe5072fce7"},"cell_type":"markdown","source":"### proportion"},{"metadata":{"scrolled":false,"trusted":false,"collapsed":true,"_uuid":"89b1487cf2f3c931491bdf1691a9d4513518463f"},"cell_type":"code","source":"proportion = df_train_tmp[[fea, 'is_attributed']].groupby(by=fea).mean().reset_index().nlargest(n=n, columns='is_attributed').reset_index(drop=True)","execution_count":97,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"7c4e49e622ffa48c2f3aaa315cde7fa025f0019d","collapsed":true},"cell_type":"code","source":"# save\nproportion.to_csv(filename=str(interim_data_Path / (\"train_\"+fea+\"_isAttributed_proportion-*.csv\")))","execution_count":98,"outputs":[]},{"metadata":{"_uuid":"bf3e91bf2aa01d362cad6e53bba1539fea08a163"},"cell_type":"markdown","source":"#### load proportion"},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"d3a2802ed963fcd116d5e675dcc704af9b49a7aa"},"cell_type":"code","source":"# load\n#tmp = dd.read_csv(urlpath=str(interim_data_Path / (\"train_\"+fea+\"_isAttributed_proportion-0.csv\")),)\n#proportion = tmp.drop(labels=tmp.columns[0], axis=1)\n#\nproportion = pd.read_csv(filepath_or_buffer=str(interim_data_Path / (\"train_\"+fea+\"_isAttributed_proportion-0.csv\")),\n index_col=0,\n )","execution_count":99,"outputs":[]},{"metadata":{"_uuid":"d5e8882ef7b09fe72cd26f7ca1eb760cd0fd8eb4"},"cell_type":"markdown","source":"### counts"},{"metadata":{"scrolled":false,"trusted":false,"collapsed":true,"_uuid":"2d0db22e16423a99c68f651182cbba7d83499572"},"cell_type":"code","source":"counts = df_train_tmp[[fea, 'is_attributed']].groupby(by=fea).count().reset_index().nlargest(n=n, columns='is_attributed').reset_index(drop=True)","execution_count":100,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"39934cde3ee0c5d6898fe1b509c3499cb6e136b1","collapsed":true},"cell_type":"code","source":"# save\ncounts.to_csv(filename=str(interim_data_Path / (\"train_\"+fea+\"_isAttributed_counts-*.csv\")))","execution_count":101,"outputs":[]},{"metadata":{"_uuid":"9bceb45db479fee76f632266d86820380f74ef76"},"cell_type":"markdown","source":"#### load counts"},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"ed16cc8d04f1735774b9f2924aee7a8b9898894c"},"cell_type":"code","source":"# load\n#tmp = dd.read_csv(urlpath=str(interim_data_Path / (\"train_\"+fea+\"_isAttributed_counts-0.csv\")),)\n#counts = tmp.drop(labels=tmp.columns[0], axis=1)\n#\ncounts = pd.read_csv(filepath_or_buffer=str(interim_data_Path / (\"train_\"+fea+\"_isAttributed_counts-0.csv\")),\n index_col=0,\n )","execution_count":102,"outputs":[]},{"metadata":{"_uuid":"e5b2a800a777cc3a25b65630f9fc25213dee6bcc"},"cell_type":"markdown","source":"### Head"},{"metadata":{"scrolled":true,"trusted":false,"_uuid":"f3d4370411017fd5e00452aac47dd180f367966f","collapsed":true},"cell_type":"code","source":"proportion.head()","execution_count":103,"outputs":[]},{"metadata":{"scrolled":false,"trusted":false,"_uuid":"773c0dd29c13e44a2746bc6441c2f728fde04d4a","collapsed":true},"cell_type":"code","source":"counts.head()","execution_count":104,"outputs":[]},{"metadata":{"_uuid":"28bf0091ec01587e9e81fc40eca850435ec0a829"},"cell_type":"markdown","source":"## Merge"},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"5594d5f92b36f58a14a7f645647518a60395e842"},"cell_type":"code","source":"df_merge = counts.merge(right=proportion, on=fea, how='left', )\n\ndf_merge.columns = [fea, 'click_count', 'prop_downloaded']","execution_count":105,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"007048ef0e2c814057d1959d4de6368af87dbe91","collapsed":true},"cell_type":"code","source":"type(df_merge['click_count'])","execution_count":106,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"67304345dde62f793eaab89c15379db0e97b93c2","collapsed":true},"cell_type":"code","source":"df_merge['click_count'].head()","execution_count":107,"outputs":[]},{"metadata":{"_uuid":"8147a513062c79e0bdf1e75121b5dc1e1895ca92"},"cell_type":"markdown","source":"## Plot"},{"metadata":{"scrolled":false,"trusted":false,"_uuid":"634cd34151d667b908f492bc1333e949992d77e0","collapsed":true},"cell_type":"code","source":"title = \"Conversion_Rates_over_Counts_of_300_Most_Popular_\"+fea.upper()+\"s\"\n\nfig = plt.figure(figsize=(15, 6))\nax1 = fig.add_subplot(1, 1, 1)\n\nn = 300\n\nax1.set_yscale(\"linear\")\nax1.yaxis.tick_left()\ndf_merge[['click_count']][:n].plot(ax=ax1, color=\"blue\",)\nax1.yaxis.set_label_position(\"left\")\nax1.set_ylabel(ylabel='Count of clicks')\nax1.legend(loc=3, bbox_to_anchor=(1.05, 1), borderaxespad=0.)\n\nax2 = ax1.twinx()\nax2.yaxis.tick_right()\ndf_merge[['prop_downloaded']][:n].plot(ax=ax2, color=\"green\",)\nax2.yaxis.set_label_position(\"right\")\nax2.set_ylabel(ylabel='Proportion Downloaded')\nax2.legend(loc=3, bbox_to_anchor=(1.05, 1 + 0.07), borderaxespad=0.)\n\nax2.set_title(title)\nax2.set_xlabel(\"Sort by \" + fea)\n#ax.legend(loc=\"best\")\n#ax.set_ylabel('Proportion Downloaded')\n#plt.show()\nfig.savefig(fname=str(data_Path / \"plot\" / title))\nplt.show()\n\nprint(df_merge[:20])","execution_count":108,"outputs":[]},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"3a5a834400ddafb5beeb214dbf34a2551553a9a4"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"cebbad4db5d300f331b7edb2023ff7f9db106d8f"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"9248770d9a648592175359f4f2fceb441a32c790"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"36cce44db890a6be6fe2f82aae5735e1216120c7"},"cell_type":"markdown","source":"# Channel"},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"8719a76c0882179028ed4b536f694774a44b37ed"},"cell_type":"code","source":"fea = \"channel\"","execution_count":109,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"04979dda504e0e8af369880a73239487e02bdb53","collapsed":true},"cell_type":"code","source":"df_fea_val_counts_tmp = df_train[fea].value_counts()\nprint(type(df_fea_val_counts_tmp))\n\ndf_fea_val_counts_tmp = df_fea_val_counts_tmp.to_frame()\nprint(type(df_fea_val_counts_tmp))\n\ndf_fea_val_counts = df_fea_val_counts_tmp.reset_index(drop=False)\n\ndel df_fea_val_counts_tmp\n\ndf_fea_val_counts.columns = [fea, 'counts']\n\n# save df_ip_val_counts\ndf_fea_val_counts.to_csv(filename=str(interim_data_Path / (\"train_\"+fea+\"_val_counts-*.csv\")))\n\ndel df_fea_val_counts\ngc.collect()","execution_count":110,"outputs":[]},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"7a908b471be132cb38a50c89e71869095d032185"},"cell_type":"code","source":"# load\ntmp = dd.read_csv(urlpath=str(interim_data_Path / (\"train_\"+fea+\"_val_counts-0.csv\")),)\ndf_fea_val_counts = tmp.drop(labels=tmp.columns[0], axis=1)","execution_count":111,"outputs":[]},{"metadata":{"scrolled":true,"trusted":false,"_uuid":"35bdebd7a4368be3c0a4ef04c1a843c6d224be87","collapsed":true},"cell_type":"code","source":"df_fea_val_counts.head()","execution_count":112,"outputs":[]},{"metadata":{"_uuid":"7870237746f00d7fbadf3239b506c840070c1b15"},"cell_type":"markdown","source":"## merge"},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"e9f4ac9b84d298bc151770dbff0ac010ef195604"},"cell_type":"code","source":"df_train_tmp = dd.merge(left=df_train,\n right=df_fea_val_counts,\n how=\"left\",\n on=fea,\n )","execution_count":113,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"10315a10bc4359ebf418a8d389b0ba801e4efe9c","collapsed":true},"cell_type":"code","source":"df_train_tmp.head()","execution_count":114,"outputs":[]},{"metadata":{"_uuid":"f1195956f2ecdbcd26d6088fa062e7231f2db056"},"cell_type":"markdown","source":"## sort all\nip unique : 277396"},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"9e085c49befac639e058b3f3a88ce0e51b5c543a"},"cell_type":"code","source":"n = 1000000 # 1000000 > 277396","execution_count":115,"outputs":[]},{"metadata":{"_uuid":"1d5ac12adc6ff344db043c24b6d4d1e3a5405123"},"cell_type":"markdown","source":"### proportion"},{"metadata":{"scrolled":false,"trusted":false,"collapsed":true,"_uuid":"d68cc6edeff422a2b8182467d1ac4b1e0b513173"},"cell_type":"code","source":"proportion = df_train_tmp[[fea, 'is_attributed']].groupby(by=fea).mean().reset_index().nlargest(n=n, columns='is_attributed').reset_index(drop=True)","execution_count":116,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"d8548e298672b7ae261849ca44884984dbe66edc","collapsed":true},"cell_type":"code","source":"# save\nproportion.to_csv(filename=str(interim_data_Path / (\"train_\"+fea+\"_isAttributed_proportion-*.csv\")))","execution_count":117,"outputs":[]},{"metadata":{"_uuid":"5869cb4a03aba63df0981fd7fcad2f2b43d64970"},"cell_type":"markdown","source":"#### load proportion"},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"f087e0a3ad24a87d97ce51af4c005e9574f4369c"},"cell_type":"code","source":"# load\n#tmp = dd.read_csv(urlpath=str(interim_data_Path / (\"train_\"+fea+\"_isAttributed_proportion-0.csv\")),)\n#proportion = tmp.drop(labels=tmp.columns[0], axis=1)\n#\nproportion = pd.read_csv(filepath_or_buffer=str(interim_data_Path / (\"train_\"+fea+\"_isAttributed_proportion-0.csv\")),\n index_col=0,\n )","execution_count":118,"outputs":[]},{"metadata":{"_uuid":"6b8b0626f4bbfc3f7324fdb8590e0ead74b2b06e"},"cell_type":"markdown","source":"### counts"},{"metadata":{"scrolled":false,"trusted":false,"collapsed":true,"_uuid":"f369002dd82a6594870688aa7041c4d0bee344be"},"cell_type":"code","source":"counts = df_train_tmp[[fea, 'is_attributed']].groupby(by=fea).count().reset_index().nlargest(n=n, columns='is_attributed').reset_index(drop=True)","execution_count":119,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"2e2245cebad6e5cc4f3ce912f19d41f4d21e3cdb","collapsed":true},"cell_type":"code","source":"# save\ncounts.to_csv(filename=str(interim_data_Path / (\"train_\"+fea+\"_isAttributed_counts-*.csv\")))","execution_count":120,"outputs":[]},{"metadata":{"_uuid":"46ba1dd0d5cfc33b3867b7b25029535860b198eb"},"cell_type":"markdown","source":"#### load counts"},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"5e30696fe6f6581e5a3af7781ea81bc6a993cb8f"},"cell_type":"code","source":"# load\n#tmp = dd.read_csv(urlpath=str(interim_data_Path / (\"train_\"+fea+\"_isAttributed_counts-0.csv\")),)\n#counts = tmp.drop(labels=tmp.columns[0], axis=1)\n#\ncounts = pd.read_csv(filepath_or_buffer=str(interim_data_Path / (\"train_\"+fea+\"_isAttributed_counts-0.csv\")),\n index_col=0,\n )","execution_count":121,"outputs":[]},{"metadata":{"_uuid":"0e8d87cd839f2499c6dc468308add9dd4f13ab04"},"cell_type":"markdown","source":"### Head"},{"metadata":{"scrolled":true,"trusted":false,"_uuid":"536632378cc381396f4ce48ca3379394009ac1e5","collapsed":true},"cell_type":"code","source":"proportion.head()","execution_count":122,"outputs":[]},{"metadata":{"scrolled":false,"trusted":false,"_uuid":"5ff10eec1e906560203205a0ed6168388853527a","collapsed":true},"cell_type":"code","source":"counts.head()","execution_count":123,"outputs":[]},{"metadata":{"_uuid":"36cf3227634290d426201701684700d4b5aff65e"},"cell_type":"markdown","source":"## Merge"},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"b634fa1a518bad83ac7de1c457317842191c7443"},"cell_type":"code","source":"df_merge = counts.merge(right=proportion, on=fea, how='left', )\n\ndf_merge.columns = [fea, 'click_count', 'prop_downloaded']","execution_count":124,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"bd3aabe9272b355cc137ae19d3f8dce2945b818e","collapsed":true},"cell_type":"code","source":"type(df_merge['click_count'])","execution_count":125,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"f38525b67118c7d558870cab16603723980eed8d","collapsed":true},"cell_type":"code","source":"df_merge['click_count'].head()","execution_count":126,"outputs":[]},{"metadata":{"_uuid":"f25cf324983e372eeeb433ce79d5552c6defb80c"},"cell_type":"markdown","source":"## Plot"},{"metadata":{"scrolled":false,"trusted":false,"_uuid":"dfeceacaf42c428e3aa553e48f1e1bebeda27db7","collapsed":true},"cell_type":"code","source":"title = \"Conversion_Rates_over_Counts_of_300_Most_Popular_\"+fea.upper()+\"s\"\n\nfig = plt.figure(figsize=(15, 6))\nax1 = fig.add_subplot(1, 1, 1)\n\nn = 300\n\nax1.set_yscale(\"linear\")\nax1.yaxis.tick_left()\ndf_merge[['click_count']][:n].plot(ax=ax1, color=\"blue\",)\nax1.yaxis.set_label_position(\"left\")\nax1.set_ylabel(ylabel='Count of clicks')\nax1.legend(loc=3, bbox_to_anchor=(1.05, 1), borderaxespad=0.)\n\nax2 = ax1.twinx()\nax2.yaxis.tick_right()\ndf_merge[['prop_downloaded']][:n].plot(ax=ax2, color=\"green\",)\nax2.yaxis.set_label_position(\"right\")\nax2.set_ylabel(ylabel='Proportion Downloaded')\nax2.legend(loc=3, bbox_to_anchor=(1.05, 1 + 0.07), borderaxespad=0.)\n\nax2.set_title(title)\nax2.set_xlabel(\"Sort by \" + fea)\n#ax.legend(loc=\"best\")\n#ax.set_ylabel('Proportion Downloaded')\n#plt.show()\nfig.savefig(fname=str(data_Path / \"plot\" / title))\nplt.show()\n\nprint(df_merge[:20])","execution_count":127,"outputs":[]},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"cdb7890d7b66ceac705dc28f154922e134c51d89"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"b323b07b4c10d475b4fc6f2faac0ca9a429bb69a"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"287f5d110b18d67feef10b1d1ebf1886c0e57b04"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"6be905636bb5d1a9f7e469c3acad6c40f6bafa05"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"99f17ad094326597ddde7ab478830297334131bc"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"21d0d0a802516f45eb12b4d52a2c4189b370ab44"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"050b6257b793c2606d9e61c564a917ad8558d9c3"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"collapsed":true,"_uuid":"6486e3e82ca837cf1f4b4df5b875d2d6df47cb11"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.5","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"toc":{"nav_menu":{},"number_sections":true,"sideBar":true,"skip_h1_title":false,"title_cell":"Table of Contents","title_sidebar":"Contents","toc_cell":true,"toc_position":{"height":"490px","left":"358px","top":"111px","width":"263px"},"toc_section_display":true,"toc_window_display":true}},"nbformat":4,"nbformat_minor":1}