{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":3136,"databundleVersionId":26502,"sourceType":"competition"},{"sourceId":31254,"databundleVersionId":3103714,"sourceType":"competition"},{"sourceId":3078823,"sourceType":"datasetVersion","datasetId":1883326}],"dockerImageVersionId":30684,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-04-21T06:58:33.364721Z","iopub.execute_input":"2024-04-21T06:58:33.365141Z","iopub.status.idle":"2024-04-21T06:58:34.640089Z","shell.execute_reply.started":"2024-04-21T06:58:33.365109Z","shell.execute_reply":"2024-04-21T06:58:34.638925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Some Guideline for EDA that i compiled,\n\nsome sources:\n- https://www.kaggle.com/code/kenjee/basic-eda-example-section-6\n- https://www.kaggle.com/code/computervisi/titanic-eda\n- https://www.kaggle.com/code/vanguarde/h-m-eda-first-look","metadata":{}},{"cell_type":"code","source":"#import visualization libraries\nimport matplotlib.pyplot as plt\nimport seaborn as sns","metadata":{"execution":{"iopub.status.busy":"2024-04-21T06:59:01.034781Z","iopub.execute_input":"2024-04-21T06:59:01.035280Z","iopub.status.idle":"2024-04-21T06:59:02.614477Z","shell.execute_reply.started":"2024-04-21T06:59:01.035243Z","shell.execute_reply":"2024-04-21T06:59:02.613377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#load data \ndf_agg = pd.read_csv('/kaggle/input/ken-jee-youtube-data/Aggregated_Metrics_By_Video.csv',encoding='utf-8')\ndf_agg_country_sub = pd.read_csv('/kaggle/input/ken-jee-youtube-data/Aggregated_Metrics_By_Country_And_Subscriber_Status.csv', encoding='utf-8')\ndf_ts = pd.read_csv('/kaggle/input/ken-jee-youtube-data/Video_Performance_Over_Time.csv', encoding='utf-8')\ndf_comments = pd.read_csv('/kaggle/input/ken-jee-youtube-data/All_Comments_Final.csv', encoding='utf-8')","metadata":{"execution":{"iopub.status.busy":"2024-04-17T14:59:32.897194Z","iopub.execute_input":"2024-04-17T14:59:32.897710Z","iopub.status.idle":"2024-04-17T14:59:33.775699Z","shell.execute_reply.started":"2024-04-17T14:59:32.897672Z","shell.execute_reply":"2024-04-17T14:59:33.774408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_comments.head()","metadata":{"execution":{"iopub.status.busy":"2024-04-16T03:36:05.854779Z","iopub.execute_input":"2024-04-16T03:36:05.855735Z","iopub.status.idle":"2024-04-16T03:36:05.869158Z","shell.execute_reply.started":"2024-04-16T03:36:05.855696Z","shell.execute_reply":"2024-04-16T03:36:05.867883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# the column headers have some extra non-ascii \n# characters, we need to clean them up before we do our analysis\n# this goes through each column and removes all the non-ascii characters \n\nnewcols =[x.encode(\"ascii\", \"ignore\").decode('utf-8') for x in df_agg.columns]\ndf_agg.columns = newcols","metadata":{"execution":{"iopub.status.busy":"2024-04-16T03:38:54.698073Z","iopub.execute_input":"2024-04-16T03:38:54.698517Z","iopub.status.idle":"2024-04-16T03:38:54.704657Z","shell.execute_reply.started":"2024-04-16T03:38:54.698486Z","shell.execute_reply":"2024-04-16T03:38:54.703673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_agg.columns","metadata":{"execution":{"iopub.status.busy":"2024-04-16T03:38:57.688861Z","iopub.execute_input":"2024-04-16T03:38:57.689774Z","iopub.status.idle":"2024-04-16T03:38:57.697821Z","shell.execute_reply.started":"2024-04-16T03:38:57.689733Z","shell.execute_reply":"2024-04-16T03:38:57.696402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_agg.describe()","metadata":{"execution":{"iopub.status.busy":"2024-04-16T03:39:00.264550Z","iopub.execute_input":"2024-04-16T03:39:00.265405Z","iopub.status.idle":"2024-04-16T03:39:00.341136Z","shell.execute_reply.started":"2024-04-16T03:39:00.265360Z","shell.execute_reply":"2024-04-16T03:39:00.339974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Single Variable Plots\n","metadata":{}},{"cell_type":"code","source":"df_agg.head()","metadata":{"execution":{"iopub.status.busy":"2024-04-16T03:42:15.552034Z","iopub.execute_input":"2024-04-16T03:42:15.552479Z","iopub.status.idle":"2024-04-16T03:42:15.579293Z","shell.execute_reply.started":"2024-04-16T03:42:15.552440Z","shell.execute_reply":"2024-04-16T03:42:15.577926Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# lets see the distribution of likes\n#you can see the right skewed data\n# and we can see there are outliers\ndf_agg.Likes.hist(bins = 100)","metadata":{"execution":{"iopub.status.busy":"2024-04-16T03:42:58.072963Z","iopub.execute_input":"2024-04-16T03:42:58.073882Z","iopub.status.idle":"2024-04-16T03:42:58.682376Z","shell.execute_reply.started":"2024-04-16T03:42:58.073815Z","shell.execute_reply":"2024-04-16T03:42:58.681361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# From this boxplot, we can see that\n# there are quite a few outliers this data. \n# because of this we should not use averages\n\nplt.boxplot(df_agg['Likes'])","metadata":{"execution":{"iopub.status.busy":"2024-04-16T03:56:36.575754Z","iopub.execute_input":"2024-04-16T03:56:36.576189Z","iopub.status.idle":"2024-04-16T03:56:36.803013Z","shell.execute_reply.started":"2024-04-16T03:56:36.576159Z","shell.execute_reply":"2024-04-16T03:56:36.801855Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# lets look at the average percentage viewed (%)\n# with this case there will no outlier\n\nplt.hist(df_agg['Average percentage viewed (%)'])","metadata":{"execution":{"iopub.status.busy":"2024-04-16T03:47:27.023026Z","iopub.execute_input":"2024-04-16T03:47:27.024648Z","iopub.status.idle":"2024-04-16T03:47:27.336115Z","shell.execute_reply.started":"2024-04-16T03:47:27.024590Z","shell.execute_reply":"2024-04-16T03:47:27.334761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#lets see the boxplot average percentage viewed\n#50% of the data is between 25-45% \nplt.boxplot(df_agg['Average percentage viewed (%)'])\n","metadata":{"execution":{"iopub.status.busy":"2024-04-16T03:59:35.576632Z","iopub.execute_input":"2024-04-16T03:59:35.577088Z","iopub.status.idle":"2024-04-16T03:59:35.825262Z","shell.execute_reply.started":"2024-04-16T03:59:35.577057Z","shell.execute_reply":"2024-04-16T03:59:35.824329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#this data has a significant amount of right skew,\n#we may want to transform this data if we were planning to\n#use linear regression.\n\ndf_agg['Impressions click-through rate (%)'].hist()","metadata":{"execution":{"iopub.status.busy":"2024-04-16T03:53:19.376636Z","iopub.execute_input":"2024-04-16T03:53:19.377840Z","iopub.status.idle":"2024-04-16T03:53:19.703514Z","shell.execute_reply.started":"2024-04-16T03:53:19.377770Z","shell.execute_reply":"2024-04-16T03:53:19.702166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# with CTR, there are a few outlier\nplt.boxplot(df_agg['Impressions click-through rate (%)'])","metadata":{"execution":{"iopub.status.busy":"2024-04-16T04:01:23.461954Z","iopub.execute_input":"2024-04-16T04:01:23.462807Z","iopub.status.idle":"2024-04-16T04:01:23.710649Z","shell.execute_reply.started":"2024-04-16T04:01:23.462769Z","shell.execute_reply":"2024-04-16T04:01:23.709192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Categorical Data\n\nfor example look at our revenue data and make it into different categories that are relevant to us.\nfor this case we make the categories, less than < $100, $100 - $1000, > $1000\n","metadata":{}},{"cell_type":"code","source":"#Make data interval for revenue\nbins = pd.IntervalIndex.from_tuples([(0,100),(100,1000),(1000, float(\"inf\"))])","metadata":{"execution":{"iopub.status.busy":"2024-04-16T04:35:49.790066Z","iopub.execute_input":"2024-04-16T04:35:49.790454Z","iopub.status.idle":"2024-04-16T04:35:49.797732Z","shell.execute_reply.started":"2024-04-16T04:35:49.790426Z","shell.execute_reply":"2024-04-16T04:35:49.796079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bins","metadata":{"execution":{"iopub.status.busy":"2024-04-16T04:35:50.863926Z","iopub.execute_input":"2024-04-16T04:35:50.864310Z","iopub.status.idle":"2024-04-16T04:35:50.872283Z","shell.execute_reply.started":"2024-04-16T04:35:50.864282Z","shell.execute_reply":"2024-04-16T04:35:50.871049Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#now we can use pandas cut function, that cut data into sections\n#for example\nbins_ex = pd.IntervalIndex.from_tuples([(0, 1), (2, 3), (4, 5)])\nsegment_data = pd.cut([0, 0.5, 1.5, 2.5, 4.5], bins_ex)\nsegment_data","metadata":{"execution":{"iopub.status.busy":"2024-04-16T04:40:59.097392Z","iopub.execute_input":"2024-04-16T04:40:59.097797Z","iopub.status.idle":"2024-04-16T04:40:59.111032Z","shell.execute_reply.started":"2024-04-16T04:40:59.097767Z","shell.execute_reply":"2024-04-16T04:40:59.109765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# now bins \ndf_agg['rev_buckets'] = pd.cut(df_agg['Your estimated revenue (USD)'],bins)","metadata":{"execution":{"iopub.status.busy":"2024-04-16T04:41:01.021184Z","iopub.execute_input":"2024-04-16T04:41:01.021646Z","iopub.status.idle":"2024-04-16T04:41:01.030583Z","shell.execute_reply.started":"2024-04-16T04:41:01.021616Z","shell.execute_reply":"2024-04-16T04:41:01.029292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_agg[[\"Your estimated revenue (USD)\",\"rev_buckets\"]].head()","metadata":{"execution":{"iopub.status.busy":"2024-04-16T04:41:39.635159Z","iopub.execute_input":"2024-04-16T04:41:39.635579Z","iopub.status.idle":"2024-04-16T04:41:39.650765Z","shell.execute_reply.started":"2024-04-16T04:41:39.635551Z","shell.execute_reply":"2024-04-16T04:41:39.649860Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#get count of number of videos by reveune bucket\nrev_values = df_agg['rev_buckets'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2024-04-16T04:41:48.585708Z","iopub.execute_input":"2024-04-16T04:41:48.586138Z","iopub.status.idle":"2024-04-16T04:41:48.594678Z","shell.execute_reply.started":"2024-04-16T04:41:48.586107Z","shell.execute_reply":"2024-04-16T04:41:48.592935Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rev_values","metadata":{"execution":{"iopub.status.busy":"2024-04-16T04:41:54.665958Z","iopub.execute_input":"2024-04-16T04:41:54.666392Z","iopub.status.idle":"2024-04-16T04:41:54.676323Z","shell.execute_reply.started":"2024-04-16T04:41:54.666358Z","shell.execute_reply":"2024-04-16T04:41:54.675026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rev_values.plot.bar()\n#now we can see the distribution of the revenue based on the bins","metadata":{"execution":{"iopub.status.busy":"2024-04-16T04:41:59.697232Z","iopub.execute_input":"2024-04-16T04:41:59.697653Z","iopub.status.idle":"2024-04-16T04:41:59.982506Z","shell.execute_reply.started":"2024-04-16T04:41:59.697621Z","shell.execute_reply":"2024-04-16T04:41:59.981004Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Relationships and Multi-Variable Plots\nsomething we can see in the plots:\n1. scatter plots\n2. correlation matrices\n3. pivot tables\n4. bar chart\n5. line charts","metadata":{}},{"cell_type":"code","source":"# Scatter Plots\n# lets make a scatter plot to understand if the average percentage \n# viewed of the video is related to the cost per mili on the video \n# (the amount youtube makes for 1000 views)\n\nplt.scatter(df_agg['Average percentage viewed (%)'],\n           df_agg['CPM (USD)'])","metadata":{"execution":{"iopub.status.busy":"2024-04-16T05:11:25.858393Z","iopub.execute_input":"2024-04-16T05:11:25.858873Z","iopub.status.idle":"2024-04-16T05:11:26.193042Z","shell.execute_reply.started":"2024-04-16T05:11:25.858807Z","shell.execute_reply":"2024-04-16T05:11:26.190560Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#a scatter plot with trendline\nsns.regplot(x='Average percentage viewed (%)',\n           y='CPM (USD)', data = df_agg)","metadata":{"execution":{"iopub.status.busy":"2024-04-16T05:13:56.400636Z","iopub.execute_input":"2024-04-16T05:13:56.401127Z","iopub.status.idle":"2024-04-16T05:13:56.879773Z","shell.execute_reply.started":"2024-04-16T05:13:56.401093Z","shell.execute_reply":"2024-04-16T05:13:56.878544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Correlation Matrices","metadata":{}},{"cell_type":"code","source":"df_agg.head()","metadata":{"execution":{"iopub.status.busy":"2024-04-16T05:18:28.709380Z","iopub.execute_input":"2024-04-16T05:18:28.709799Z","iopub.status.idle":"2024-04-16T05:18:28.736682Z","shell.execute_reply.started":"2024-04-16T05:18:28.709768Z","shell.execute_reply":"2024-04-16T05:18:28.735224Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#lets start with the numerical features\nnumerical_features = df_agg.select_dtypes(include=[np.number]) #filtering the numerical features\nnumerical_features.head()","metadata":{"execution":{"iopub.status.busy":"2024-04-16T05:34:55.776548Z","iopub.execute_input":"2024-04-16T05:34:55.777027Z","iopub.status.idle":"2024-04-16T05:34:55.803596Z","shell.execute_reply.started":"2024-04-16T05:34:55.776993Z","shell.execute_reply":"2024-04-16T05:34:55.802320Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# this is the default of correlation matrices in seaborn\ncorr = numerical_features.corr()\n\nsns.heatmap(corr)","metadata":{"execution":{"iopub.status.busy":"2024-04-16T05:35:21.180243Z","iopub.execute_input":"2024-04-16T05:35:21.180719Z","iopub.status.idle":"2024-04-16T05:35:21.820794Z","shell.execute_reply.started":"2024-04-16T05:35:21.180677Z","shell.execute_reply":"2024-04-16T05:35:21.819779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# it looks kinda awful, and hard to iterpret.\n#lets try a better version\n\nsns.set_theme(style=\"white\")\n\ncorr = numerical_features.corr()\n\n#correlate graphs\nmask = np.triu(np.ones_like(corr, dtype=bool))\nf, ax = plt.subplots(figsize=(15, 10))\ncmap = sns.diverging_palette(230, 20, as_cmap=True)\nsns.heatmap(corr, mask=mask, cmap=cmap, center=0,\n            square=True, linewidths=.5, cbar_kws={\"shrink\": .5}, annot=True, annot_kws={\"fontsize\":8})","metadata":{"execution":{"iopub.status.busy":"2024-04-16T05:42:15.326906Z","iopub.execute_input":"2024-04-16T05:42:15.327342Z","iopub.status.idle":"2024-04-16T05:42:16.521060Z","shell.execute_reply.started":"2024-04-16T05:42:15.327312Z","shell.execute_reply":"2024-04-16T05:42:16.519792Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Pivot Tables\n","metadata":{}},{"cell_type":"code","source":"df_agg_country_sub.head()","metadata":{"execution":{"iopub.status.busy":"2024-04-16T05:44:21.113089Z","iopub.execute_input":"2024-04-16T05:44:21.113536Z","iopub.status.idle":"2024-04-16T05:44:21.135507Z","shell.execute_reply.started":"2024-04-16T05:44:21.113506Z","shell.execute_reply":"2024-04-16T05:44:21.134180Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#lets use pivot table\n#the default function is mean\n\npd.pivot_table(df_agg_country_sub\n               , index = 'Country Code', #this is the row\n               values = 'Average View Percentage').sort_values('Average View Percentage')","metadata":{"execution":{"iopub.status.busy":"2024-04-16T06:50:31.932842Z","iopub.execute_input":"2024-04-16T06:50:31.933248Z","iopub.status.idle":"2024-04-16T06:50:31.962967Z","shell.execute_reply.started":"2024-04-16T06:50:31.933216Z","shell.execute_reply":"2024-04-16T06:50:31.962096Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#another way to use pivot\npd.pivot_table(df_agg_country_sub, index = 'Country Code',\n               columns = 'Is Subscribed',\n               values = 'Average View Percentage')","metadata":{"execution":{"iopub.status.busy":"2024-04-16T06:53:21.665719Z","iopub.execute_input":"2024-04-16T06:53:21.666176Z","iopub.status.idle":"2024-04-16T06:53:21.710604Z","shell.execute_reply.started":"2024-04-16T06:53:21.666147Z","shell.execute_reply":"2024-04-16T06:53:21.709227Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#to put plot it after pivot\npd.pivot_table(df_agg_country_sub, index = 'Is Subscribed', \n               values = 'Average View Percentage').plot.bar()","metadata":{"execution":{"iopub.status.busy":"2024-04-16T06:54:57.442409Z","iopub.execute_input":"2024-04-16T06:54:57.442838Z","iopub.status.idle":"2024-04-16T06:54:57.789777Z","shell.execute_reply.started":"2024-04-16T06:54:57.442795Z","shell.execute_reply":"2024-04-16T06:54:57.788550Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Line Chart","metadata":{}},{"cell_type":"code","source":"df_ts.info()\n#the date column tyoe is object, let","metadata":{"execution":{"iopub.status.busy":"2024-04-17T14:59:40.661555Z","iopub.execute_input":"2024-04-17T14:59:40.662029Z","iopub.status.idle":"2024-04-17T14:59:40.743864Z","shell.execute_reply.started":"2024-04-17T14:59:40.661996Z","shell.execute_reply":"2024-04-17T14:59:40.742631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#we replace the string Sept to Sep because \"Sept\" is not recognized by pandas.to_datetime\ndf_ts['Date'] = df_ts['Date'].str.replace('Sept','Sep')","metadata":{"execution":{"iopub.status.busy":"2024-04-17T15:06:57.691487Z","iopub.execute_input":"2024-04-17T15:06:57.692003Z","iopub.status.idle":"2024-04-17T15:06:57.749468Z","shell.execute_reply.started":"2024-04-17T15:06:57.691969Z","shell.execute_reply":"2024-04-17T15:06:57.748192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#first lets change the date format to datetime\ndf_ts['Date'] = pd.to_datetime(df_ts['Date'])","metadata":{"execution":{"iopub.status.busy":"2024-04-17T15:07:03.260055Z","iopub.execute_input":"2024-04-17T15:07:03.261602Z","iopub.status.idle":"2024-04-17T15:07:03.298878Z","shell.execute_reply.started":"2024-04-17T15:07:03.261559Z","shell.execute_reply":"2024-04-17T15:07:03.297763Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_ts.head(2)","metadata":{"execution":{"iopub.status.busy":"2024-04-17T15:08:17.300926Z","iopub.execute_input":"2024-04-17T15:08:17.301437Z","iopub.status.idle":"2024-04-17T15:08:17.319588Z","shell.execute_reply.started":"2024-04-17T15:08:17.301403Z","shell.execute_reply":"2024-04-17T15:08:17.318257Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#pivot data\n#because we want to see the daily user unsubscribed daily\nunsubscribe = pd.pivot_table(\ndf_ts, index=\"Date\", values=\"User Subscriptions Removed\",\naggfunc=\"sum\").reset_index() #reset index bcause we dont want date as index\n\nunsubscribe.head(3)","metadata":{"execution":{"iopub.status.busy":"2024-04-17T15:11:37.130879Z","iopub.execute_input":"2024-04-17T15:11:37.131438Z","iopub.status.idle":"2024-04-17T15:11:37.157637Z","shell.execute_reply.started":"2024-04-17T15:11:37.131400Z","shell.execute_reply":"2024-04-17T15:11:37.156373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#lets visualize it\ng = sns.lineplot(data=unsubscribe, x='Date', \n             y='User Subscriptions Removed')\ng.tick_params(axis='x', rotation=30)","metadata":{"execution":{"iopub.status.busy":"2024-04-17T15:17:49.724479Z","iopub.execute_input":"2024-04-17T15:17:49.724975Z","iopub.status.idle":"2024-04-17T15:17:50.230713Z","shell.execute_reply.started":"2024-04-17T15:17:49.724936Z","shell.execute_reply":"2024-04-17T15:17:50.229510Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#lets put multiple line in the graph\n#.replace function is used to replace the contents of datetime object\n# day parameter meaning the new date value (1-31)\n\n#now we want to get the months (like date_trunc in sql)\ndf_ts['Month_Year'] = df_ts['Date'].apply(lambda x: x.replace(day=1))\n","metadata":{"execution":{"iopub.status.busy":"2024-04-17T16:07:39.597809Z","iopub.execute_input":"2024-04-17T16:07:39.598391Z","iopub.status.idle":"2024-04-17T16:07:41.079950Z","shell.execute_reply.started":"2024-04-17T16:07:39.598345Z","shell.execute_reply":"2024-04-17T16:07:41.078852Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_ts[['Month_Year','Date']].head()","metadata":{"execution":{"iopub.status.busy":"2024-04-17T16:07:43.420810Z","iopub.execute_input":"2024-04-17T16:07:43.421288Z","iopub.status.idle":"2024-04-17T16:07:43.436554Z","shell.execute_reply.started":"2024-04-17T16:07:43.421240Z","shell.execute_reply":"2024-04-17T16:07:43.435168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#now lets create 3 pivot tables\nsubs = pd.pivot_table(df_ts, index=\"Month_Year\",\n                      values=\"User Subscriptions Removed\", aggfunc=\"sum\").reset_index()\nlikes =pd.pivot_table(df_ts, index=\"Month_Year\",\n                      values=\"Video Likes Removed\", aggfunc=\"sum\").reset_index()\ndislikes = pd.pivot_table(df_ts, index=\"Month_Year\",\n                      values=\"Video Dislikes Added\", aggfunc=\"sum\").reset_index()\n#create 3 line plots\nsns.lineplot(data=subs, x='Month_Year',y='User Subscriptions Removed', label=\"Subs removed\")\nsns.lineplot(data=likes, x='Month_Year',y='Video Likes Removed', label=\"likes removed\")\nsns.lineplot(data=dislikes, x='Month_Year',y='Video Dislikes Added', label=\"dislikes removed\")","metadata":{"execution":{"iopub.status.busy":"2024-04-17T16:21:16.192289Z","iopub.execute_input":"2024-04-17T16:21:16.194223Z","iopub.status.idle":"2024-04-17T16:21:16.848997Z","shell.execute_reply.started":"2024-04-17T16:21:16.194160Z","shell.execute_reply":"2024-04-17T16:21:16.847494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Now lets work on another Data!\nin this case we used the titanic data","metadata":{}},{"cell_type":"code","source":"from plotnine import *  \n# plotnine is an implementation of a grammar of graphics in Python, it is based on ggplot2. \n#The grammar allows users to compose plots by explicitly mapping data to the visual objects that make up the plot.\nimport matplotlib.pyplot as plt  \nimport seaborn as sns  \nimport pandas as pd  \nimport numpy as np \nimport warnings  \nwarnings.filterwarnings('ignore')  ","metadata":{"execution":{"iopub.status.busy":"2024-04-25T14:14:59.830603Z","iopub.execute_input":"2024-04-25T14:14:59.831018Z","iopub.status.idle":"2024-04-25T14:15:03.820368Z","shell.execute_reply.started":"2024-04-25T14:14:59.830984Z","shell.execute_reply":"2024-04-25T14:15:03.818810Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"now some overview of Titanic data:\n1. PassengerId\n2. Survived -> the target variable (1 survived, 0 not)\n3. Pclass -> is the socio economic status passenger\nhas 3 unique values:\n1 = upper class\n2 = middle class\n3 = lower class\n4. Emabarked -> is port of embarkation an it is categorical feature:\nC = Cherbourg\nQ = Queenstown\nS = Southampton\n\n","metadata":{}},{"cell_type":"code","source":"#function to combine data and separate data\ndef concat_df(train_data, test_data):\n    # Returns a concatenated df of training and test set\n    return pd.concat([train_data, test_data], sort=True).reset_index(drop=True)\n\ndef divide_df(all_data):\n    # Returns divided dfs of training and test set\n    return all_data.loc[:890], all_data.loc[891:].drop(['Survived'], axis=1)\n","metadata":{"execution":{"iopub.status.busy":"2024-04-25T14:17:52.990929Z","iopub.execute_input":"2024-04-25T14:17:52.991357Z","iopub.status.idle":"2024-04-25T14:17:52.999306Z","shell.execute_reply.started":"2024-04-25T14:17:52.991323Z","shell.execute_reply":"2024-04-25T14:17:52.997571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#lets load the data\ndf_train = pd.read_csv('/kaggle/input/titanic/train.csv')  # load train data\ndf_test = pd.read_csv('/kaggle/input/titanic/test.csv')  # load test data\n\ndf_all = concat_df(df_train, df_test)  # we apply the function described above, the union of two dataframes.","metadata":{"execution":{"iopub.status.busy":"2024-04-25T14:17:57.431589Z","iopub.execute_input":"2024-04-25T14:17:57.432063Z","iopub.status.idle":"2024-04-25T14:17:57.479475Z","shell.execute_reply.started":"2024-04-25T14:17:57.432025Z","shell.execute_reply":"2024-04-25T14:17:57.478154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.head()","metadata":{"execution":{"iopub.status.busy":"2024-04-25T14:18:00.724296Z","iopub.execute_input":"2024-04-25T14:18:00.724852Z","iopub.status.idle":"2024-04-25T14:18:00.758903Z","shell.execute_reply.started":"2024-04-25T14:18:00.724808Z","shell.execute_reply":"2024-04-25T14:18:00.757577Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#set name for each data set\ndf_train.name = 'Training Set' \ndf_test.name = 'Test Set'  \ndf_all.name = 'All Set' ","metadata":{"execution":{"iopub.status.busy":"2024-04-21T07:00:35.095657Z","iopub.execute_input":"2024-04-21T07:00:35.096348Z","iopub.status.idle":"2024-04-21T07:00:35.102214Z","shell.execute_reply.started":"2024-04-21T07:00:35.096304Z","shell.execute_reply":"2024-04-21T07:00:35.100926Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dfs = [df_train, df_test]","metadata":{"execution":{"iopub.status.busy":"2024-04-25T14:18:08.846560Z","iopub.execute_input":"2024-04-25T14:18:08.846990Z","iopub.status.idle":"2024-04-25T14:18:08.853919Z","shell.execute_reply.started":"2024-04-25T14:18:08.846956Z","shell.execute_reply":"2024-04-25T14:18:08.852063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Finding Missing Values 🥸","metadata":{}},{"cell_type":"code","source":"#function to analyze missing column","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def display_missing(df):    \n    for col in df.columns.tolist():          \n        print('{} column missing values: {}'.format(col, df[col].isnull().sum()))\n    print('\\n')\n    \nfor df in dfs:\n    print('{}'.format(df.name))\n    display_missing(df)","metadata":{"execution":{"iopub.status.busy":"2024-04-20T07:51:20.267973Z","iopub.execute_input":"2024-04-20T07:51:20.268468Z","iopub.status.idle":"2024-04-20T07:51:20.296851Z","shell.execute_reply.started":"2024-04-20T07:51:20.268434Z","shell.execute_reply":"2024-04-20T07:51:20.295292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"we missing a lot of data in age and cabin!","metadata":{}},{"cell_type":"code","source":"#group by data\ndf_train[['Sex','Survived']].groupby(['Sex'],as_index=False).mean().sort_values(\n    by=\"Survived\",ascending=False)","metadata":{"execution":{"iopub.status.busy":"2024-04-20T07:58:31.968728Z","iopub.execute_input":"2024-04-20T07:58:31.969146Z","iopub.status.idle":"2024-04-20T07:58:31.984348Z","shell.execute_reply.started":"2024-04-20T07:58:31.969115Z","shell.execute_reply":"2024-04-20T07:58:31.982782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#try it pivot, same result!\npd.pivot_table(data=df_train[['Sex','Survived']], index='Sex',\n              values='Survived' , aggfunc='sum').reset_index()","metadata":{"execution":{"iopub.status.busy":"2024-04-20T07:59:53.844395Z","iopub.execute_input":"2024-04-20T07:59:53.844938Z","iopub.status.idle":"2024-04-20T07:59:53.872698Z","shell.execute_reply.started":"2024-04-20T07:59:53.844903Z","shell.execute_reply":"2024-04-20T07:59:53.871024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Using gg plot in Python!\n#default funciton is SUM\n(ggplot(df_train)\n + aes(x='Sex', y='Survived')\n + geom_col()\n + ggtitle('gender survival rate')\n + theme(text=element_text(family='NanumBarunGothic'))\n)","metadata":{"execution":{"iopub.status.busy":"2024-04-20T07:59:10.663936Z","iopub.execute_input":"2024-04-20T07:59:10.664720Z","iopub.status.idle":"2024-04-20T07:59:11.282209Z","shell.execute_reply.started":"2024-04-20T07:59:10.664678Z","shell.execute_reply":"2024-04-20T07:59:11.280497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#survived rate with embarked!\n(ggplot(df_train)\n + aes(x='Sex', y='Survived', fill='Embarked')\n + geom_col()\n)","metadata":{"execution":{"iopub.status.busy":"2024-04-20T08:00:29.289034Z","iopub.execute_input":"2024-04-20T08:00:29.289478Z","iopub.status.idle":"2024-04-20T08:00:29.856831Z","shell.execute_reply.started":"2024-04-20T08:00:29.289443Z","shell.execute_reply":"2024-04-20T08:00:29.855452Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Now Visualization with Scatter plot! 🌻","metadata":{}},{"cell_type":"code","source":"(ggplot(df_train)\n + aes(x='Age', y='Fare')\n + geom_point()\n + ggtitle('Fares by age group')\n + theme(text=element_text(family='NanumBarunGothic'))\n)","metadata":{"execution":{"iopub.status.busy":"2024-04-21T07:01:35.425887Z","iopub.execute_input":"2024-04-21T07:01:35.426256Z","iopub.status.idle":"2024-04-21T07:01:36.029990Z","shell.execute_reply.started":"2024-04-21T07:01:35.426228Z","shell.execute_reply":"2024-04-21T07:01:36.028727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train['Survived'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2024-04-21T07:02:45.658747Z","iopub.execute_input":"2024-04-21T07:02:45.659186Z","iopub.status.idle":"2024-04-21T07:02:45.668983Z","shell.execute_reply.started":"2024-04-21T07:02:45.659154Z","shell.execute_reply":"2024-04-21T07:02:45.667857Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train['Survived'] = df_train['Survived'].astype('category')","metadata":{"execution":{"iopub.status.busy":"2024-04-21T07:03:08.247765Z","iopub.execute_input":"2024-04-21T07:03:08.248457Z","iopub.status.idle":"2024-04-21T07:03:08.253798Z","shell.execute_reply.started":"2024-04-21T07:03:08.248422Z","shell.execute_reply":"2024-04-21T07:03:08.252656Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"(ggplot(df_train)\n + aes(x='Age', y='Fare', color='Survived')\n + geom_point()\n + stat_smooth()\n + ggtitle('Age group and fare survival rate')\n + theme(text=element_text(family='NanumBarunGothic'))\n)","metadata":{"execution":{"iopub.status.busy":"2024-04-21T07:03:17.375644Z","iopub.execute_input":"2024-04-21T07:03:17.376312Z","iopub.status.idle":"2024-04-21T07:03:18.144179Z","shell.execute_reply.started":"2024-04-21T07:03:17.376270Z","shell.execute_reply":"2024-04-21T07:03:18.142946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#we can also split the graph\n(ggplot(df_train) \n + aes(x='Age', y='Fare', color='Sex')\n + geom_point()\n + stat_smooth()\n + facet_wrap('~Survived')\n)","metadata":{"execution":{"iopub.status.busy":"2024-04-21T07:04:01.787333Z","iopub.execute_input":"2024-04-21T07:04:01.787827Z","iopub.status.idle":"2024-04-21T07:04:02.760988Z","shell.execute_reply.started":"2024-04-21T07:04:01.787784Z","shell.execute_reply":"2024-04-21T07:04:02.759705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Box plot\n(ggplot(df_train) \n + aes(x='Sex', y='Fare', fill='Survived')\n + geom_boxplot()\n + ggtitle('Sex Fare Survival Rate')\n + theme(text=element_text(family='NanumBarunGothic'))\n)","metadata":{"execution":{"iopub.status.busy":"2024-04-21T07:04:42.676924Z","iopub.execute_input":"2024-04-21T07:04:42.677382Z","iopub.status.idle":"2024-04-21T07:04:43.377359Z","shell.execute_reply.started":"2024-04-21T07:04:42.677346Z","shell.execute_reply":"2024-04-21T07:04:43.376155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#survival count based on age\n\ndf_all.head()","metadata":{"execution":{"iopub.status.busy":"2024-04-21T07:23:48.926574Z","iopub.execute_input":"2024-04-21T07:23:48.927008Z","iopub.status.idle":"2024-04-21T07:23:48.947339Z","shell.execute_reply.started":"2024-04-21T07:23:48.926977Z","shell.execute_reply":"2024-04-21T07:23:48.946508Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Creating Bins 🥑","metadata":{}},{"cell_type":"code","source":"#first let create bin with age\ndf_all['Age_Bin'] = pd.cut(df_all['Age'],10)","metadata":{"execution":{"iopub.status.busy":"2024-04-21T08:32:38.720334Z","iopub.execute_input":"2024-04-21T08:32:38.720777Z","iopub.status.idle":"2024-04-21T08:32:38.738777Z","shell.execute_reply.started":"2024-04-21T08:32:38.720744Z","shell.execute_reply":"2024-04-21T08:32:38.737450Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_all[['Age_Bin']].value_counts()","metadata":{"execution":{"iopub.status.busy":"2024-04-21T08:33:55.051360Z","iopub.execute_input":"2024-04-21T08:33:55.053779Z","iopub.status.idle":"2024-04-21T08:33:55.068978Z","shell.execute_reply.started":"2024-04-21T08:33:55.053730Z","shell.execute_reply":"2024-04-21T08:33:55.067737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#lets create the cart survival counts based on age\nfig, axs = plt.subplots(figsize=(22, 9))\nsns.countplot(x='Age_Bin', hue='Survived', data=df_all)\n\nplt.xlabel('Age_Bin', size=15, labelpad=20)\nplt.ylabel('Passenger Count', size=15, labelpad=20)\nplt.tick_params(axis='x', labelsize=15)\nplt.tick_params(axis='y', labelsize=15)\n\nplt.legend(['Not Survived', 'Survived'], loc='upper right', prop={'size': 15})\nplt.title('Survival Counts in {} Feature'.format('Age'), size=15, y=1.05)\n\nplt.show()\n\n#well to nobody surprise many young people are survived","metadata":{"execution":{"iopub.status.busy":"2024-04-21T08:57:35.742906Z","iopub.execute_input":"2024-04-21T08:57:35.743326Z","iopub.status.idle":"2024-04-21T08:57:36.220174Z","shell.execute_reply.started":"2024-04-21T08:57:35.743297Z","shell.execute_reply":"2024-04-21T08:57:36.219318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Cabin data\n#lets see how many missing value we have here\nprint(\"Train Cabin missing: \" + str(df_train.Cabin.isnull().sum()/len(df_train.Cabin)))\nprint(\"Test Cabin missing: \" + str(df_test.Cabin.isnull().sum()/len(df_test.Cabin)))","metadata":{"execution":{"iopub.status.busy":"2024-04-25T14:20:12.295699Z","iopub.execute_input":"2024-04-25T14:20:12.296121Z","iopub.status.idle":"2024-04-25T14:20:12.307391Z","shell.execute_reply.started":"2024-04-25T14:20:12.296086Z","shell.execute_reply":"2024-04-25T14:20:12.305956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#theres a lot of missing data arounf 77-78%\n#lets assign all the null values to NA\ndf_all.Cabin.fillna(\"NA\", inplace=True)","metadata":{"execution":{"iopub.status.busy":"2024-04-25T14:20:14.477050Z","iopub.execute_input":"2024-04-25T14:20:14.477474Z","iopub.status.idle":"2024-04-25T14:20:14.485898Z","shell.execute_reply.started":"2024-04-25T14:20:14.477426Z","shell.execute_reply":"2024-04-25T14:20:14.484304Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_all['Cabin'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2024-04-21T09:29:08.480111Z","iopub.execute_input":"2024-04-21T09:29:08.480523Z","iopub.status.idle":"2024-04-21T09:29:08.491324Z","shell.execute_reply.started":"2024-04-21T09:29:08.480490Z","shell.execute_reply":"2024-04-21T09:29:08.489893Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#to clean the data, lets get only the first letter of cabin\ndf_all.Cabin = [str(i)[0] for i in df_all.Cabin]","metadata":{"execution":{"iopub.status.busy":"2024-04-25T14:20:19.784770Z","iopub.execute_input":"2024-04-25T14:20:19.785165Z","iopub.status.idle":"2024-04-25T14:20:19.793618Z","shell.execute_reply.started":"2024-04-25T14:20:19.785136Z","shell.execute_reply":"2024-04-25T14:20:19.792161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_all['Cabin'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2024-04-25T14:20:22.427694Z","iopub.execute_input":"2024-04-25T14:20:22.428223Z","iopub.status.idle":"2024-04-25T14:20:22.446657Z","shell.execute_reply.started":"2024-04-25T14:20:22.428187Z","shell.execute_reply":"2024-04-25T14:20:22.444905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def percent_value_counts(df, feature):\n    \"\"\"This function takes in a dataframe and a column and finds the percentage of the value_counts\"\"\"\n    percent = pd.DataFrame(round(df.loc[:,feature].value_counts(dropna=False, normalize=True)*100,2))\n    ## creating a df with th\n    total = pd.DataFrame(df.loc[:,feature].value_counts(dropna=False))\n    ## concating percent and total dataframe\n\n    total.columns = [\"Total\"]\n    percent.columns = ['Percent']\n    return pd.concat([total, percent], axis = 1)\n\npercent_value_counts(df_all, \"Cabin\")","metadata":{"execution":{"iopub.status.busy":"2024-04-25T14:20:26.703093Z","iopub.execute_input":"2024-04-25T14:20:26.703514Z","iopub.status.idle":"2024-04-25T14:20:26.727073Z","shell.execute_reply.started":"2024-04-25T14:20:26.703482Z","shell.execute_reply":"2024-04-25T14:20:26.725646Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#lets see the average price based on cabin number\ndf_all.groupby(\"Cabin\")['Fare'].mean().sort_values()","metadata":{"execution":{"iopub.status.busy":"2024-04-25T14:20:50.047778Z","iopub.execute_input":"2024-04-25T14:20:50.048153Z","iopub.status.idle":"2024-04-25T14:20:50.074551Z","shell.execute_reply.started":"2024-04-25T14:20:50.048124Z","shell.execute_reply":"2024-04-25T14:20:50.072889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#based on the average fare, we can predict what is the cabin\ndef cabin_estimator(i):\n    \"\"\"Grouping cabin feature by the first letter\"\"\"\n    a = 0\n    if i<16:\n        a = \"G\"\n    elif i>=16 and i<27:\n        a = \"F\"\n    elif i>=27 and i<38:\n        a = \"T\"\n    elif i>=38 and i<47:\n        a = \"A\"\n    elif i>= 47 and i<53:\n        a = \"E\"\n    elif i>= 53 and i<54:\n        a = \"D\"\n    elif i>=54 and i<116:\n        a = 'C'\n    else:\n        a = \"B\"\n    return a\n","metadata":{"execution":{"iopub.status.busy":"2024-04-25T14:20:54.066320Z","iopub.execute_input":"2024-04-25T14:20:54.066790Z","iopub.status.idle":"2024-04-25T14:20:54.076298Z","shell.execute_reply.started":"2024-04-25T14:20:54.066754Z","shell.execute_reply":"2024-04-25T14:20:54.074831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#apply value to the missing cabin\ndf_all['Cabin'] = df_all.Fare.apply(lambda x: cabin_estimator(x))\n\npercent_value_counts(df_all, \"Cabin\")","metadata":{"execution":{"iopub.status.busy":"2024-04-25T14:20:57.963233Z","iopub.execute_input":"2024-04-25T14:20:57.964004Z","iopub.status.idle":"2024-04-25T14:20:57.984222Z","shell.execute_reply.started":"2024-04-25T14:20:57.963964Z","shell.execute_reply":"2024-04-25T14:20:57.982905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_all.head()","metadata":{"execution":{"iopub.status.busy":"2024-04-25T14:21:02.604795Z","iopub.execute_input":"2024-04-25T14:21:02.605298Z","iopub.status.idle":"2024-04-25T14:21:02.628284Z","shell.execute_reply.started":"2024-04-25T14:21:02.605248Z","shell.execute_reply":"2024-04-25T14:21:02.626842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Creating Deck column from the first letter of the Cabin column (M stands for Missing)\ndf_all['Deck'] = df_all['Cabin'].apply(lambda s: s[0] if pd.notnull(s) else 'M')","metadata":{"execution":{"iopub.status.busy":"2024-04-25T14:21:15.587028Z","iopub.execute_input":"2024-04-25T14:21:15.587535Z","iopub.status.idle":"2024-04-25T14:21:15.598360Z","shell.execute_reply.started":"2024-04-25T14:21:15.587495Z","shell.execute_reply":"2024-04-25T14:21:15.596856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_all['Deck'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2024-04-25T14:21:32.384897Z","iopub.execute_input":"2024-04-25T14:21:32.385362Z","iopub.status.idle":"2024-04-25T14:21:32.398731Z","shell.execute_reply.started":"2024-04-25T14:21:32.385324Z","shell.execute_reply":"2024-04-25T14:21:32.396901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Passenger in the T deck is changed to A\nidx = df_all[df_all['Deck'] == 'T'].index\ndf_all.loc[idx, 'Deck'] = 'A'\n\ndf_all_decks_survived = df_all.groupby(['Deck',\n                                        'Survived']).count().drop(columns=['Sex', 'Age', \n                                                                                   'SibSp', 'Parch', 'Fare', \n                                                                                   'Embarked', 'Pclass', 'Cabin', \n                                                                                   'PassengerId', 'Ticket']).rename(columns={'Name':'Count'}) #.transpose()","metadata":{"execution":{"iopub.status.busy":"2024-04-25T14:22:39.085915Z","iopub.execute_input":"2024-04-25T14:22:39.086324Z","iopub.status.idle":"2024-04-25T14:22:39.109070Z","shell.execute_reply.started":"2024-04-25T14:22:39.086291Z","shell.execute_reply":"2024-04-25T14:22:39.107541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#trying to see the total survived/not survived based on the deck\ndf_all_decks_survived","metadata":{"execution":{"iopub.status.busy":"2024-04-25T14:22:44.529148Z","iopub.execute_input":"2024-04-25T14:22:44.529708Z","iopub.status.idle":"2024-04-25T14:22:44.546989Z","shell.execute_reply.started":"2024-04-25T14:22:44.529642Z","shell.execute_reply":"2024-04-25T14:22:44.545567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#lets tranpose it for better readebility\ndf_all_decks_survived = df_all_decks_survived.transpose()\ndf_all_decks_survived","metadata":{"execution":{"iopub.status.busy":"2024-04-25T14:32:27.887174Z","iopub.execute_input":"2024-04-25T14:32:27.887665Z","iopub.status.idle":"2024-04-25T14:32:27.905316Z","shell.execute_reply.started":"2024-04-25T14:32:27.887628Z","shell.execute_reply":"2024-04-25T14:32:27.903549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#this is a function how many the survived and the not survived for each deck into dictionary\ndef get_survived_dist(df):\n    \n    # Creating a dictionary for every survival count in every deck\n    surv_counts = {'A':{}, 'B':{}, 'C':{}, 'D':{}, 'E':{}, 'F':{}, 'G':{}, 'M':{}}\n    decks = df.columns.levels[0]    \n\n    for deck in decks:\n        for survive in range(0, 2):\n            surv_counts[deck][survive] = df[deck][survive][0]\n            \n    df_surv = pd.DataFrame(surv_counts)\n    surv_percentages = {}\n\n    for col in df_surv.columns:\n        surv_percentages[col] = [(count / df_surv[col].sum()) * 100 for count in df_surv[col]]\n        \n    return surv_counts, surv_percentages","metadata":{"execution":{"iopub.status.busy":"2024-04-25T14:24:45.221647Z","iopub.execute_input":"2024-04-25T14:24:45.223848Z","iopub.status.idle":"2024-04-25T14:24:45.233145Z","shell.execute_reply.started":"2024-04-25T14:24:45.223791Z","shell.execute_reply":"2024-04-25T14:24:45.231693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#this is a plot for\ndef display_surv_dist(percentages):\n    \n    df_survived_percentages = pd.DataFrame(percentages).transpose()\n    deck_names = ('A', 'B', 'C', 'D', 'E', 'F', 'G', 'M')\n    bar_count = np.arange(len(deck_names))  \n    bar_width = 0.85    \n\n    not_survived = df_survived_percentages[0]\n    survived = df_survived_percentages[1]\n    \n    plt.figure(figsize=(20, 10))\n    plt.bar(bar_count, not_survived, color='#b5ffb9', edgecolor='white', width=bar_width, label=\"Not Survived\")\n    plt.bar(bar_count, survived, bottom=not_survived, color='#f9bc86', edgecolor='white', width=bar_width, label=\"Survived\")\n \n    plt.xlabel('Deck', size=15, labelpad=20)\n    plt.ylabel('Survival Percentage', size=15, labelpad=20)\n    plt.xticks(bar_count, deck_names)    \n    plt.tick_params(axis='x', labelsize=15)\n    plt.tick_params(axis='y', labelsize=15)\n    \n    plt.legend(loc='upper left', bbox_to_anchor=(1, 1), prop={'size': 15})\n    plt.title('Survival Percentage in Decks', size=18, y=1.05)\n    \n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-04-25T14:25:07.508968Z","iopub.execute_input":"2024-04-25T14:25:07.509398Z","iopub.status.idle":"2024-04-25T14:25:07.522001Z","shell.execute_reply.started":"2024-04-25T14:25:07.509367Z","shell.execute_reply":"2024-04-25T14:25:07.520618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_surv_count, all_surv_per = get_survived_dist(df_all_decks_survived)","metadata":{"execution":{"iopub.status.busy":"2024-04-25T14:25:21.212229Z","iopub.execute_input":"2024-04-25T14:25:21.213610Z","iopub.status.idle":"2024-04-25T14:25:21.236170Z","shell.execute_reply.started":"2024-04-25T14:25:21.213560Z","shell.execute_reply":"2024-04-25T14:25:21.234830Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#lets see what all_surv_count\nall_surv_count","metadata":{"execution":{"iopub.status.busy":"2024-04-25T14:26:24.969749Z","iopub.execute_input":"2024-04-25T14:26:24.970175Z","iopub.status.idle":"2024-04-25T14:26:24.979509Z","shell.execute_reply.started":"2024-04-25T14:26:24.970142Z","shell.execute_reply":"2024-04-25T14:26:24.978249Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_surv_per","metadata":{"execution":{"iopub.status.busy":"2024-04-25T14:27:41.845892Z","iopub.execute_input":"2024-04-25T14:27:41.846584Z","iopub.status.idle":"2024-04-25T14:27:41.856131Z","shell.execute_reply.started":"2024-04-25T14:27:41.846522Z","shell.execute_reply":"2024-04-25T14:27:41.854669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_survived_percentages = pd.DataFrame(all_surv_per).transpose()\ndf_survived_percentages\n#the table below is the input for the chart below, we can see how is the percentage for each bar","metadata":{"execution":{"iopub.status.busy":"2024-04-25T14:34:34.403921Z","iopub.execute_input":"2024-04-25T14:34:34.404389Z","iopub.status.idle":"2024-04-25T14:34:34.420783Z","shell.execute_reply.started":"2024-04-25T14:34:34.404355Z","shell.execute_reply":"2024-04-25T14:34:34.419284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display_surv_dist(all_surv_per)","metadata":{"execution":{"iopub.status.busy":"2024-04-25T14:25:23.737436Z","iopub.execute_input":"2024-04-25T14:25:23.738971Z","iopub.status.idle":"2024-04-25T14:25:24.200437Z","shell.execute_reply.started":"2024-04-25T14:25:23.738913Z","shell.execute_reply":"2024-04-25T14:25:24.198988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Lets see another Dataset which is the H&M dataset! 🎽","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport seaborn as sns\nfrom matplotlib import pyplot as plt\nfrom tqdm.notebook import tqdm","metadata":{"execution":{"iopub.status.busy":"2024-04-25T14:37:46.697021Z","iopub.execute_input":"2024-04-25T14:37:46.697484Z","iopub.status.idle":"2024-04-25T14:37:46.716449Z","shell.execute_reply.started":"2024-04-25T14:37:46.697435Z","shell.execute_reply":"2024-04-25T14:37:46.714615Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"articles = pd.read_csv(\"../input/h-and-m-personalized-fashion-recommendations/articles.csv\")\ncustomers = pd.read_csv(\"../input/h-and-m-personalized-fashion-recommendations/customers.csv\")\ntransactions = pd.read_csv(\"../input/h-and-m-personalized-fashion-recommendations/transactions_train.csv\")","metadata":{"execution":{"iopub.status.busy":"2024-04-25T14:37:56.974348Z","iopub.execute_input":"2024-04-25T14:37:56.974776Z","iopub.status.idle":"2024-04-25T14:39:41.093484Z","shell.execute_reply.started":"2024-04-25T14:37:56.974743Z","shell.execute_reply":"2024-04-25T14:39:41.092143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Lets see the data!\n","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}