{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-03T03:15:24.583688Z","iopub.execute_input":"2022-08-03T03:15:24.584621Z","iopub.status.idle":"2022-08-03T03:15:24.612231Z","shell.execute_reply.started":"2022-08-03T03:15:24.584493Z","shell.execute_reply":"2022-08-03T03:15:24.611360Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport seaborn as sn\nimport matplotlib.pyplot as plt\n\nimport re\nfrom collections import defaultdict, Counter","metadata":{"execution":{"iopub.status.busy":"2022-08-03T03:15:24.613781Z","iopub.execute_input":"2022-08-03T03:15:24.614315Z","iopub.status.idle":"2022-08-03T03:15:25.569747Z","shell.execute_reply.started":"2022-08-03T03:15:24.614283Z","shell.execute_reply":"2022-08-03T03:15:25.568829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 1. 데이터 불러오기","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv(\"/kaggle/input/nlp-getting-started/train.csv\")\ntest = pd.read_csv(\"/kaggle/input/nlp-getting-started/test.csv\")\n\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-03T03:15:25.571735Z","iopub.execute_input":"2022-08-03T03:15:25.572764Z","iopub.status.idle":"2022-08-03T03:15:25.661215Z","shell.execute_reply.started":"2022-08-03T03:15:25.572720Z","shell.execute_reply":"2022-08-03T03:15:25.659880Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.shape","metadata":{"execution":{"iopub.status.busy":"2022-08-03T03:15:25.662917Z","iopub.execute_input":"2022-08-03T03:15:25.663391Z","iopub.status.idle":"2022-08-03T03:15:25.672020Z","shell.execute_reply.started":"2022-08-03T03:15:25.663348Z","shell.execute_reply":"2022-08-03T03:15:25.670789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2022-08-03T03:15:25.674892Z","iopub.execute_input":"2022-08-03T03:15:25.676070Z","iopub.status.idle":"2022-08-03T03:15:25.689558Z","shell.execute_reply.started":"2022-08-03T03:15:25.676023Z","shell.execute_reply":"2022-08-03T03:15:25.688382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[\"keyword\"].unique()","metadata":{"execution":{"iopub.status.busy":"2022-08-03T03:15:25.691098Z","iopub.execute_input":"2022-08-03T03:15:25.691539Z","iopub.status.idle":"2022-08-03T03:15:25.712465Z","shell.execute_reply.started":"2022-08-03T03:15:25.691497Z","shell.execute_reply":"2022-08-03T03:15:25.711360Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"text_len = train[\"text\"].apply(lambda x: len(x.split()))\ntext_len.min(), text_len.max(), text_len.mean()","metadata":{"execution":{"iopub.status.busy":"2022-08-03T03:15:25.716140Z","iopub.execute_input":"2022-08-03T03:15:25.716841Z","iopub.status.idle":"2022-08-03T03:15:25.739177Z","shell.execute_reply.started":"2022-08-03T03:15:25.716807Z","shell.execute_reply":"2022-08-03T03:15:25.737849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sn.displot(text_len)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-03T03:15:25.740589Z","iopub.execute_input":"2022-08-03T03:15:25.741050Z","iopub.status.idle":"2022-08-03T03:15:26.122246Z","shell.execute_reply.started":"2022-08-03T03:15:25.741006Z","shell.execute_reply":"2022-08-03T03:15:26.121269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[train[\"text\"].apply(lambda x: '@' in x) == True][\"text\"]","metadata":{"execution":{"iopub.status.busy":"2022-08-03T03:15:26.123775Z","iopub.execute_input":"2022-08-03T03:15:26.124143Z","iopub.status.idle":"2022-08-03T03:15:26.139609Z","shell.execute_reply.started":"2022-08-03T03:15:26.124111Z","shell.execute_reply":"2022-08-03T03:15:26.138287Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[train[\"text\"].apply(lambda x: '#' in x) == True][\"text\"]","metadata":{"execution":{"iopub.status.busy":"2022-08-03T03:15:26.141776Z","iopub.execute_input":"2022-08-03T03:15:26.142239Z","iopub.status.idle":"2022-08-03T03:15:26.158599Z","shell.execute_reply.started":"2022-08-03T03:15:26.142199Z","shell.execute_reply":"2022-08-03T03:15:26.157584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"true_cnt = train.groupby(\"keyword\").sum()[\"target\"]\nall_cnt = train.groupby(\"keyword\").count()[\"target\"]\n\ntrue_cnt","metadata":{"execution":{"iopub.status.busy":"2022-08-03T03:15:26.162322Z","iopub.execute_input":"2022-08-03T03:15:26.162891Z","iopub.status.idle":"2022-08-03T03:15:26.183038Z","shell.execute_reply.started":"2022-08-03T03:15:26.162859Z","shell.execute_reply":"2022-08-03T03:15:26.181973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"keyword_cnt_ratio = true_cnt / all_cnt\n\nplt.figure(figsize=(12,8))\nkeyword_cnt_ratio.sort_values().plot()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-03T03:15:26.184307Z","iopub.execute_input":"2022-08-03T03:15:26.185134Z","iopub.status.idle":"2022-08-03T03:15:26.393139Z","shell.execute_reply.started":"2022-08-03T03:15:26.185100Z","shell.execute_reply":"2022-08-03T03:15:26.392187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"true_cnt = train.groupby(\"location\").sum()[\"target\"]\nfalse_cnt = train.groupby(\"location\").count()[\"target\"] - true_cnt\n\nlocation_cnt_diff = true_cnt - false_cnt\n\nplt.figure(figsize=(12,8))\nlocation_cnt_diff.sort_values().plot()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-03T03:15:26.394758Z","iopub.execute_input":"2022-08-03T03:15:26.395568Z","iopub.status.idle":"2022-08-03T03:15:26.634382Z","shell.execute_reply.started":"2022-08-03T03:15:26.395522Z","shell.execute_reply":"2022-08-03T03:15:26.633111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[train[\"keyword\"].isna()][\"text\"]","metadata":{"execution":{"iopub.status.busy":"2022-08-03T03:15:26.636182Z","iopub.execute_input":"2022-08-03T03:15:26.636575Z","iopub.status.idle":"2022-08-03T03:15:26.647097Z","shell.execute_reply.started":"2022-08-03T03:15:26.636531Z","shell.execute_reply":"2022-08-03T03:15:26.646238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"id_tags = train[\"text\"].str.split().apply(lambda x: list(filter(lambda xx: \"@\" in xx, x)))\nid_tags = id_tags[id_tags.apply(lambda x: len(x)) > 0].apply(lambda x: \" \".join(x))\nid_tags","metadata":{"execution":{"iopub.status.busy":"2022-08-03T03:15:26.648326Z","iopub.execute_input":"2022-08-03T03:15:26.649457Z","iopub.status.idle":"2022-08-03T03:15:26.712184Z","shell.execute_reply.started":"2022-08-03T03:15:26.649424Z","shell.execute_reply":"2022-08-03T03:15:26.710937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"without_id = train[\"text\"].str.split().apply(lambda x: list(filter(lambda xx: \"@\" not in xx, x)))\nwithout_id = without_id.apply(lambda x: \" \".join(x))\nwithout_id","metadata":{"execution":{"iopub.status.busy":"2022-08-03T03:15:26.713846Z","iopub.execute_input":"2022-08-03T03:15:26.714958Z","iopub.status.idle":"2022-08-03T03:15:26.771282Z","shell.execute_reply.started":"2022-08-03T03:15:26.714894Z","shell.execute_reply":"2022-08-03T03:15:26.770128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"without_id_url = without_id.str.split().apply(lambda x: list(filter(lambda xx: \"://\" not in xx, x)))\nwithout_id_url = without_id_url.apply(lambda x: \" \".join(x))\nwithout_id_url","metadata":{"execution":{"iopub.status.busy":"2022-08-03T03:15:26.772971Z","iopub.execute_input":"2022-08-03T03:15:26.773679Z","iopub.status.idle":"2022-08-03T03:15:26.829188Z","shell.execute_reply.started":"2022-08-03T03:15:26.773636Z","shell.execute_reply":"2022-08-03T03:15:26.827970Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"without_id_url_special = without_id_url.apply(lambda x: re.sub(r\"[^a-zA-Z0-9.!? ]\",\"\",x))\nwithout_id_url_special","metadata":{"execution":{"iopub.status.busy":"2022-08-03T03:15:26.830809Z","iopub.execute_input":"2022-08-03T03:15:26.831525Z","iopub.status.idle":"2022-08-03T03:15:26.860626Z","shell.execute_reply.started":"2022-08-03T03:15:26.831480Z","shell.execute_reply":"2022-08-03T03:15:26.859570Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hashtags = train[\"text\"].str.split().apply(lambda x: list(filter(lambda xx: \"#\" in xx, x)))\nhashtags_without_special = hashtags.apply(lambda x: list(map(lambda xx: re.sub(r\"[^a-zA-Z0-9]\",\"\",xx), x)))\nhashtags_without_special","metadata":{"execution":{"iopub.status.busy":"2022-08-03T03:15:26.862235Z","iopub.execute_input":"2022-08-03T03:15:26.863192Z","iopub.status.idle":"2022-08-03T03:15:27.058946Z","shell.execute_reply.started":"2022-08-03T03:15:26.863148Z","shell.execute_reply":"2022-08-03T03:15:27.057610Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[\"hashtags\"] = hashtags_without_special\ntrain[\"text_clean\"] = without_id_url_special\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-03T03:15:27.060728Z","iopub.execute_input":"2022-08-03T03:15:27.061196Z","iopub.status.idle":"2022-08-03T03:15:27.081424Z","shell.execute_reply.started":"2022-08-03T03:15:27.061152Z","shell.execute_reply":"2022-08-03T03:15:27.080010Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"true_hashtags = []\nfalse_hashtags = []\n\nfor row in train[train[\"target\"] == 1][\"hashtags\"].to_list():\n    true_hashtags = true_hashtags + list(map(lambda x: x.lower(), row))\nfor row in train[train[\"target\"] == 0][\"hashtags\"].to_list():\n    false_hashtags = false_hashtags + list(map(lambda x: x.lower(), row))\n\ntrue_hashtags_cnt = Counter(true_hashtags)\nfalse_hashtags_cnt = Counter(false_hashtags)\ntop20_true_hashtags = sorted(true_hashtags_cnt.items(), key=lambda x: -x[1])[:20]\ntop20_true_hashtags","metadata":{"execution":{"iopub.status.busy":"2022-08-03T03:15:27.083501Z","iopub.execute_input":"2022-08-03T03:15:27.083999Z","iopub.status.idle":"2022-08-03T03:15:27.155763Z","shell.execute_reply.started":"2022-08-03T03:15:27.083951Z","shell.execute_reply":"2022-08-03T03:15:27.154753Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"top20_false_hashtags = sorted(false_hashtags_cnt.items(), key=lambda x: -x[1])[:20]\ntop20_false_hashtags","metadata":{"execution":{"iopub.status.busy":"2022-08-03T03:15:27.157826Z","iopub.execute_input":"2022-08-03T03:15:27.158541Z","iopub.status.idle":"2022-08-03T03:15:27.168579Z","shell.execute_reply.started":"2022-08-03T03:15:27.158496Z","shell.execute_reply":"2022-08-03T03:15:27.167305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hashtags_cnt_ratio = dict()\nfor key, val in (true_hashtags_cnt+false_hashtags_cnt).items():\n    hashtags_cnt_ratio[key] = true_hashtags_cnt[key] / val\n  \nsorted(hashtags_cnt_ratio.items(), key=lambda x: -x[1])[:20]","metadata":{"execution":{"iopub.status.busy":"2022-08-03T03:15:27.169965Z","iopub.execute_input":"2022-08-03T03:15:27.170397Z","iopub.status.idle":"2022-08-03T03:15:27.190608Z","shell.execute_reply.started":"2022-08-03T03:15:27.170354Z","shell.execute_reply":"2022-08-03T03:15:27.189436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def make_hashtag_ratio(df):\n    true_hashtags = sum(df[df[\"target\"] == 1][\"hashtags\"].to_list(), [])\n    false_hashtags = sum(df[df[\"target\"] == 0][\"hashtags\"].to_list(), [])\n    true_hashtags_cnt = Counter(true_hashtags)\n    false_hashtags_cnt = Counter(false_hashtags)\n    \n    hashtags_cnt_ratio = dict()\n    for key, val in (true_hashtags_cnt+false_hashtags_cnt).items():\n        hashtags_cnt_ratio[key] = true_hashtags_cnt[key] / val\n    return hashtags_cnt_ratio","metadata":{"execution":{"iopub.status.busy":"2022-08-03T03:15:27.192387Z","iopub.execute_input":"2022-08-03T03:15:27.193858Z","iopub.status.idle":"2022-08-03T03:15:27.211986Z","shell.execute_reply.started":"2022-08-03T03:15:27.193807Z","shell.execute_reply":"2022-08-03T03:15:27.208189Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hashtags_score_max = train[\"hashtags\"].apply(lambda x: np.nan if x == [] else max(map(lambda xx: hashtags_cnt_ratio[xx.lower()], x)))\nhashtags_score_max","metadata":{"execution":{"iopub.status.busy":"2022-08-03T03:15:27.214662Z","iopub.execute_input":"2022-08-03T03:15:27.215489Z","iopub.status.idle":"2022-08-03T03:15:27.240948Z","shell.execute_reply.started":"2022-08-03T03:15:27.215442Z","shell.execute_reply":"2022-08-03T03:15:27.239681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hashtags_score_mean = train[\"hashtags\"].apply(lambda x: np.nan if x == [] else sum(map(lambda xx: hashtags_cnt_ratio[xx.lower()], x))/len(x))\nhashtags_score_mean","metadata":{"execution":{"iopub.status.busy":"2022-08-03T03:15:27.243567Z","iopub.execute_input":"2022-08-03T03:15:27.244482Z","iopub.status.idle":"2022-08-03T03:15:27.269901Z","shell.execute_reply.started":"2022-08-03T03:15:27.244435Z","shell.execute_reply":"2022-08-03T03:15:27.268688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"keyword_score = train[\"keyword\"].apply(lambda x: x if x not in keyword_cnt_ratio.keys() else keyword_cnt_ratio[x])\nkeyword_score","metadata":{"execution":{"iopub.status.busy":"2022-08-03T03:15:27.271662Z","iopub.execute_input":"2022-08-03T03:15:27.272413Z","iopub.status.idle":"2022-08-03T03:15:27.338779Z","shell.execute_reply.started":"2022-08-03T03:15:27.272366Z","shell.execute_reply":"2022-08-03T03:15:27.337613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[\"hashtags_score_max\"] = hashtags_score_max\ntrain[\"hashtags_score_mean\"] = hashtags_score_mean\ntrain[\"keyword_score\"] = keyword_score\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-03T03:15:27.340444Z","iopub.execute_input":"2022-08-03T03:15:27.341158Z","iopub.status.idle":"2022-08-03T03:15:27.363813Z","shell.execute_reply.started":"2022-08-03T03:15:27.341115Z","shell.execute_reply":"2022-08-03T03:15:27.362584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_uni = train.drop_duplicates(\"text_clean\")\ntrain_uni.shape","metadata":{"execution":{"iopub.status.busy":"2022-08-03T03:15:27.371357Z","iopub.execute_input":"2022-08-03T03:15:27.372138Z","iopub.status.idle":"2022-08-03T03:15:27.400164Z","shell.execute_reply.started":"2022-08-03T03:15:27.372092Z","shell.execute_reply":"2022-08-03T03:15:27.399000Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scores = train_uni[[\"hashtags_score_mean\", \"hashtags_score_max\", \"keyword_score\", \"target\"]]\nscores[scores[\"hashtags_score_mean\"].notna() & scores[\"keyword_score\"].notna()].corr()","metadata":{"execution":{"iopub.status.busy":"2022-08-03T03:15:27.401324Z","iopub.execute_input":"2022-08-03T03:15:27.402119Z","iopub.status.idle":"2022-08-03T03:15:27.425970Z","shell.execute_reply.started":"2022-08-03T03:15:27.402072Z","shell.execute_reply":"2022-08-03T03:15:27.425065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scores.fillna(0).corr()","metadata":{"execution":{"iopub.status.busy":"2022-08-03T03:15:27.427405Z","iopub.execute_input":"2022-08-03T03:15:27.428094Z","iopub.status.idle":"2022-08-03T03:15:27.444876Z","shell.execute_reply.started":"2022-08-03T03:15:27.428050Z","shell.execute_reply":"2022-08-03T03:15:27.443753Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scores.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2022-08-03T03:15:27.446450Z","iopub.execute_input":"2022-08-03T03:15:27.447582Z","iopub.status.idle":"2022-08-03T03:15:27.462409Z","shell.execute_reply.started":"2022-08-03T03:15:27.447538Z","shell.execute_reply":"2022-08-03T03:15:27.461202Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hashtag_set = set(sum(train[\"hashtags\"].to_list(),[]))\nkeyword_set = set(train[\"keyword\"].unique())\n\nlen(hashtag_set), len(keyword_set), len(hashtag_set.intersection(keyword_set))","metadata":{"execution":{"iopub.status.busy":"2022-08-03T03:15:27.464768Z","iopub.execute_input":"2022-08-03T03:15:27.465681Z","iopub.status.idle":"2022-08-03T03:15:27.569649Z","shell.execute_reply.started":"2022-08-03T03:15:27.465646Z","shell.execute_reply":"2022-08-03T03:15:27.568471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"hashtag_plus_keyword = train_uni.apply(lambda x: x[\"hashtags\"] if pd.isna(x[\"keyword\"]) else x[\"hashtags\"]+[x[\"keyword\"]], axis=1)\nhashtag_plus_keyword = hashtag_plus_keyword.apply(lambda x: list(map(lambda xx: xx.lower(), x)))\nhashtag_plus_keyword","metadata":{"execution":{"iopub.status.busy":"2022-08-03T03:15:27.571428Z","iopub.execute_input":"2022-08-03T03:15:27.572160Z","iopub.status.idle":"2022-08-03T03:15:27.796654Z","shell.execute_reply.started":"2022-08-03T03:15:27.572114Z","shell.execute_reply":"2022-08-03T03:15:27.794552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"(hashtag_plus_keyword.apply(lambda x:len(x)) == 0).sum()","metadata":{"execution":{"iopub.status.busy":"2022-08-03T03:15:27.798276Z","iopub.execute_input":"2022-08-03T03:15:27.798983Z","iopub.status.idle":"2022-08-03T03:15:27.818153Z","shell.execute_reply.started":"2022-08-03T03:15:27.798938Z","shell.execute_reply":"2022-08-03T03:15:27.816995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tmp_df = pd.DataFrame()\ntmp_df[\"target\"] = train_uni[\"target\"]\ntmp_df[\"hashtags\"] = hashtag_plus_keyword\nnew_hashtag_ratio = make_hashtag_ratio(tmp_df)\nsorted(new_hashtag_ratio.items(), key=lambda x: -x[1])[:20]","metadata":{"execution":{"iopub.status.busy":"2022-08-03T03:15:27.819669Z","iopub.execute_input":"2022-08-03T03:15:27.820892Z","iopub.status.idle":"2022-08-03T03:15:27.961841Z","shell.execute_reply.started":"2022-08-03T03:15:27.820857Z","shell.execute_reply":"2022-08-03T03:15:27.960752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"new_hashtag_score = hashtag_plus_keyword.apply(lambda x: np.nan if x == [] else sum(map(lambda xx: new_hashtag_ratio[xx.lower()], x))/len(x))\nnew_hashtag_score","metadata":{"execution":{"iopub.status.busy":"2022-08-03T03:15:27.963073Z","iopub.execute_input":"2022-08-03T03:15:27.963888Z","iopub.status.idle":"2022-08-03T03:15:27.984438Z","shell.execute_reply.started":"2022-08-03T03:15:27.963854Z","shell.execute_reply":"2022-08-03T03:15:27.983118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_uni = pd.DataFrame(train_uni)\ntrain_uni[\"hashtag_plus_keyword\"] = hashtag_plus_keyword\ntrain_uni[\"new_hashtag_score\"] = new_hashtag_score\ntrain_uni[[\"new_hashtag_score\", \"target\"]].corr()","metadata":{"execution":{"iopub.status.busy":"2022-08-03T03:15:27.986252Z","iopub.execute_input":"2022-08-03T03:15:27.986606Z","iopub.status.idle":"2022-08-03T03:15:28.005298Z","shell.execute_reply.started":"2022-08-03T03:15:27.986577Z","shell.execute_reply":"2022-08-03T03:15:28.004306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_uni[\"new_hashtag_score\"].isna().sum()","metadata":{"execution":{"iopub.status.busy":"2022-08-03T03:15:28.006777Z","iopub.execute_input":"2022-08-03T03:15:28.007457Z","iopub.status.idle":"2022-08-03T03:15:28.015030Z","shell.execute_reply.started":"2022-08-03T03:15:28.007425Z","shell.execute_reply":"2022-08-03T03:15:28.013890Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mean_when_false = train_uni[train_uni[\"target\"]==0 & train_uni[\"new_hashtag_score\"].notna()][\"new_hashtag_score\"].mean()\nmean_when_true = train_uni[train_uni[\"target\"]==1 & train_uni[\"new_hashtag_score\"].notna()][\"new_hashtag_score\"].mean()\nmean_total = train_uni[train_uni[\"new_hashtag_score\"].notna()][\"new_hashtag_score\"].mean()\nmean_when_false, mean_when_true, mean_total","metadata":{"execution":{"iopub.status.busy":"2022-08-03T03:15:28.016552Z","iopub.execute_input":"2022-08-03T03:15:28.017505Z","iopub.status.idle":"2022-08-03T03:15:28.036675Z","shell.execute_reply.started":"2022-08-03T03:15:28.017464Z","shell.execute_reply":"2022-08-03T03:15:28.035407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_uni[\"new_hashtag_score\"] = train_uni.apply(\n    lambda x: x[\"new_hashtag_score\"] if pd.notna(x[\"new_hashtag_score\"]) else (mean_when_true if x[\"target\"] == 1 else mean_when_false), axis=1)\ntrain_uni","metadata":{"execution":{"iopub.status.busy":"2022-08-03T03:15:28.038610Z","iopub.execute_input":"2022-08-03T03:15:28.039582Z","iopub.status.idle":"2022-08-03T03:15:28.199585Z","shell.execute_reply.started":"2022-08-03T03:15:28.039535Z","shell.execute_reply":"2022-08-03T03:15:28.198383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preprocessed_data = train_uni[[\"id\", \"text_clean\", \"new_hashtag_score\", \"target\"]]\npreprocessed_data","metadata":{"execution":{"iopub.status.busy":"2022-08-03T03:15:28.201082Z","iopub.execute_input":"2022-08-03T03:15:28.202005Z","iopub.status.idle":"2022-08-03T03:15:28.221826Z","shell.execute_reply.started":"2022-08-03T03:15:28.201959Z","shell.execute_reply":"2022-08-03T03:15:28.220507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preprocessed_data.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2022-08-03T03:15:28.223649Z","iopub.execute_input":"2022-08-03T03:15:28.224596Z","iopub.status.idle":"2022-08-03T03:15:28.237033Z","shell.execute_reply.started":"2022-08-03T03:15:28.224550Z","shell.execute_reply":"2022-08-03T03:15:28.235829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-03T03:15:28.238894Z","iopub.execute_input":"2022-08-03T03:15:28.240115Z","iopub.status.idle":"2022-08-03T03:15:28.254328Z","shell.execute_reply.started":"2022-08-03T03:15:28.240068Z","shell.execute_reply":"2022-08-03T03:15:28.252885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test[\"text_clean\"] = test[\"text\"].str.split().apply(lambda x: list(filter(lambda xx: \"@\" not in xx, x)))\ntest[\"text_clean\"] = test[\"text_clean\"].apply(lambda x: \" \".join(x))\ntest[\"text_clean\"] = test[\"text\"].str.split().apply(lambda x: list(filter(lambda xx: \"://\" not in xx, x)))\ntest[\"text_clean\"] = test[\"text_clean\"].apply(lambda x: \" \".join(x))\ntest[\"text_clean\"] = test[\"text_clean\"].apply(lambda x: re.sub(r\"[^a-zA-Z0-9.!? ]\",\"\",x))\ntest.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-03T03:15:28.256135Z","iopub.execute_input":"2022-08-03T03:15:28.256564Z","iopub.status.idle":"2022-08-03T03:15:28.334056Z","shell.execute_reply.started":"2022-08-03T03:15:28.256521Z","shell.execute_reply":"2022-08-03T03:15:28.332682Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test[\"hashtags\"] = test[\"text\"].str.split().apply(lambda x: list(filter(lambda xx: \"#\" in xx, x)))\ntest[\"hashtags\"] = test[\"hashtags\"].apply(lambda x: list(map(lambda xx: re.sub(r\"[^a-zA-Z0-9]\",\"\",xx), x)))\ntest.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-03T03:16:26.826439Z","iopub.execute_input":"2022-08-03T03:16:26.826865Z","iopub.status.idle":"2022-08-03T03:16:26.874157Z","shell.execute_reply.started":"2022-08-03T03:16:26.826835Z","shell.execute_reply":"2022-08-03T03:16:26.872717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test[\"hashtags\"] = test.apply(lambda x: x[\"hashtags\"] if pd.isna(x[\"keyword\"]) else x[\"hashtags\"]+[x[\"keyword\"]], axis=1)\ntest[\"hashtags\"] = test[\"hashtags\"].apply(lambda x: list(map(lambda xx: xx.lower(), x)))\ntest.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-03T03:17:30.090470Z","iopub.execute_input":"2022-08-03T03:17:30.090983Z","iopub.status.idle":"2022-08-03T03:17:30.193253Z","shell.execute_reply.started":"2022-08-03T03:17:30.090937Z","shell.execute_reply":"2022-08-03T03:17:30.192016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"new_hashtag_ratio = defaultdict(int, new_hashtag_ratio)\nlen(new_hashtag_ratio)","metadata":{"execution":{"iopub.status.busy":"2022-08-03T03:20:05.354822Z","iopub.execute_input":"2022-08-03T03:20:05.355807Z","iopub.status.idle":"2022-08-03T03:20:05.362433Z","shell.execute_reply.started":"2022-08-03T03:20:05.355768Z","shell.execute_reply":"2022-08-03T03:20:05.361250Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test[\"hashtag_score\"] = test[\"hashtags\"].apply(lambda x: np.nan if x == [] else sum(map(lambda xx: new_hashtag_ratio[xx.lower()], x))/len(x))\ntest.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-03T03:20:06.025469Z","iopub.execute_input":"2022-08-03T03:20:06.026226Z","iopub.status.idle":"2022-08-03T03:20:06.048491Z","shell.execute_reply.started":"2022-08-03T03:20:06.026179Z","shell.execute_reply":"2022-08-03T03:20:06.047345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test[\"hashtag_score\"] = test[\"hashtag_score\"].fillna(mean_total)\ntest.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2022-08-03T03:20:41.963566Z","iopub.execute_input":"2022-08-03T03:20:41.964006Z","iopub.status.idle":"2022-08-03T03:20:41.976242Z","shell.execute_reply.started":"2022-08-03T03:20:41.963970Z","shell.execute_reply":"2022-08-03T03:20:41.975334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preprocessed_test_data = test[[\"id\", \"text_clean\", \"hashtag_score\"]]\npreprocessed_test_data.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-03T03:21:54.327003Z","iopub.execute_input":"2022-08-03T03:21:54.327395Z","iopub.status.idle":"2022-08-03T03:21:54.343389Z","shell.execute_reply.started":"2022-08-03T03:21:54.327363Z","shell.execute_reply":"2022-08-03T03:21:54.342015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preprocessed_data.to_csv(\"/kaggle/working/train_preprocessed.csv\", index=False)\npreprocessed_test_data.to_csv(\"/kaggle/working/test_preprocessed.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-03T03:25:51.032731Z","iopub.execute_input":"2022-08-03T03:25:51.033138Z","iopub.status.idle":"2022-08-03T03:25:51.085813Z","shell.execute_reply.started":"2022-08-03T03:25:51.033104Z","shell.execute_reply":"2022-08-03T03:25:51.084448Z"},"trusted":true},"execution_count":null,"outputs":[]}]}