{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"}],"dockerImageVersionId":30787,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"

⏳TASK - measuring problematic internet use in children and adolescents

","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-10-28T04:26:40.461953Z","iopub.execute_input":"2024-10-28T04:26:40.462489Z","iopub.status.idle":"2024-10-28T04:26:40.466683Z","shell.execute_reply.started":"2024-10-28T04:26:40.462447Z","shell.execute_reply":"2024-10-28T04:26:40.465781Z"}}},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"

🛠 Necessary tools

","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport tensorflow as tf\nimport tensorflow.keras as kr\n\nfrom sklearn.model_selection import train_test_split as Split\nfrom sklearn.ensemble import RandomForestClassifier , AdaBoostClassifier\nfrom sklearn.metrics import classification_report, accuracy_score\nfrom sklearn.preprocessing import OrdinalEncoder\n\nfrom warnings import filterwarnings\nfilterwarnings('ignore', category=FutureWarning)","metadata":{"execution":{"iopub.status.busy":"2024-11-26T11:09:20.969692Z","iopub.execute_input":"2024-11-26T11:09:20.970633Z","iopub.status.idle":"2024-11-26T11:09:20.975988Z","shell.execute_reply.started":"2024-11-26T11:09:20.970593Z","shell.execute_reply":"2024-11-26T11:09:20.974937Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# directories\ntrain_dir = '/kaggle/input/child-mind-institute-problematic-internet-use/train.csv'\ntest_dir = '/kaggle/input/child-mind-institute-problematic-internet-use/test.csv'\ntrain_par = '/kaggle/input/child-mind-institute-problematic-internet-use/series_train.parquet'\ntest_par = '/kaggle/input/child-mind-institute-problematic-internet-use/series_test.parquet'\nsubmission_dir = '/kaggle/input/child-mind-institute-problematic-internet-use/sample_submission.csv'\ndata_dict = '/kaggle/input/child-mind-institute-problematic-internet-use/test.csv'","metadata":{"execution":{"iopub.status.busy":"2024-11-26T11:04:28.088754Z","iopub.execute_input":"2024-11-26T11:04:28.089144Z","iopub.status.idle":"2024-11-26T11:04:28.105092Z","shell.execute_reply.started":"2024-11-26T11:04:28.089117Z","shell.execute_reply":"2024-11-26T11:04:28.104216Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\"\"\"df_tr_par = pd.read_parquet(train_par)\ndf_ts_par = pd.read_parquet(test_par)\n# Save as CSV\ndf_tr_par.to_csv('series_train.csv', index=False)\ndf_ts_par.to_csv('series_test.csv', index=False)\"\"\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-26T11:03:18.186983Z","iopub.execute_input":"2024-11-26T11:03:18.187831Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"

📩Loading data

","metadata":{}},{"cell_type":"code","source":"def load_data(directory):\n df = pd.read_csv(directory)\n return df\ndata_train = load_data(train_dir)\ndata_test = load_data(test_dir)\nsubmit = load_data(submission_dir)\ndata_train.head(10)","metadata":{"execution":{"iopub.status.busy":"2024-11-26T11:04:57.850623Z","iopub.execute_input":"2024-11-26T11:04:57.851486Z","iopub.status.idle":"2024-11-26T11:04:57.929239Z","shell.execute_reply.started":"2024-11-26T11:04:57.851451Z","shell.execute_reply":"2024-11-26T11:04:57.928124Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_train.info()","metadata":{"execution":{"iopub.status.busy":"2024-11-26T11:05:00.678174Z","iopub.execute_input":"2024-11-26T11:05:00.678523Z","iopub.status.idle":"2024-11-26T11:05:00.703215Z","shell.execute_reply.started":"2024-11-26T11:05:00.678493Z","shell.execute_reply":"2024-11-26T11:05:00.702353Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_train.describe()","metadata":{"execution":{"iopub.status.busy":"2024-11-26T11:05:04.575466Z","iopub.execute_input":"2024-11-26T11:05:04.575856Z","iopub.status.idle":"2024-11-26T11:05:04.703982Z","shell.execute_reply.started":"2024-11-26T11:05:04.575818Z","shell.execute_reply":"2024-11-26T11:05:04.703105Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"

♻️ Data cleaning

","metadata":{}},{"cell_type":"code","source":"data_train.sii.value_counts()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-26T11:05:07.429724Z","iopub.execute_input":"2024-11-26T11:05:07.430110Z","iopub.status.idle":"2024-11-26T11:05:07.439256Z","shell.execute_reply.started":"2024-11-26T11:05:07.430076Z","shell.execute_reply":"2024-11-26T11:05:07.438284Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#The first thing here that we need to compare the number of \n# input columns for both train and test datas\n\nbefore = f'Before:\\n The number of columns in train data : {len(data_train.columns)} \\n The number of columns in test data : {len(data_test.columns)}\\n'\n\nclmn = list(data_test.columns)\n\nclmn = clmn+['sii']\n\ndata_train = data_train[clmn]\n\nafter = f\"\\nAfter:\\n The number of columns in train data:{len(data_train.columns)}\\n The number of columns in test data : {len(data_test.columns)}\"\n\nprint(before , after)","metadata":{"execution":{"iopub.status.busy":"2024-11-26T11:05:09.759373Z","iopub.execute_input":"2024-11-26T11:05:09.759771Z","iopub.status.idle":"2024-11-26T11:05:09.767475Z","shell.execute_reply.started":"2024-11-26T11:05:09.759735Z","shell.execute_reply":"2024-11-26T11:05:09.766526Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def Nan_values(df):\n values = df.isna().sum()\n percentages = (df.isna().sum()/(len(df)/100)).round(1)\n nan_data = pd.DataFrame({\n \"number_of_nan\" : values,\n \"percentages\" : percentages\n })\n return nan_data\nNANs = Nan_values(data_train)\nNANs","metadata":{"execution":{"iopub.status.busy":"2024-11-26T11:05:50.687261Z","iopub.execute_input":"2024-11-26T11:05:50.688143Z","iopub.status.idle":"2024-11-26T11:05:50.708375Z","shell.execute_reply.started":"2024-11-26T11:05:50.688107Z","shell.execute_reply":"2024-11-26T11:05:50.707526Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(\"The percentage of nan_values over 50% may affect the accuracy of our predictions,\\\nso that we can drop them from the dataset\")\nNANs[NANs.percentages>60]","metadata":{"execution":{"iopub.status.busy":"2024-11-26T11:05:56.350413Z","iopub.execute_input":"2024-11-26T11:05:56.351093Z","iopub.status.idle":"2024-11-26T11:05:56.361249Z","shell.execute_reply.started":"2024-11-26T11:05:56.351055Z","shell.execute_reply":"2024-11-26T11:05:56.360271Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def clean_data(df):\n # to calculate the threshold for more than 50% NaN values\n threshold_c = int(df.shape[0] * 0.5)\n threshold_r = int(df.shape[1] * 0.5)\n # to drop columns with more than 50% NaN values\n df = df.dropna(axis=1, thresh=threshold_c)\n # to drop rows with more than 50% NaN values\n df = df.dropna(thresh=threshold_r)\n return df\ndata_train = clean_data(data_train)\ndata_train","metadata":{"execution":{"iopub.status.busy":"2024-11-26T11:06:01.641415Z","iopub.execute_input":"2024-11-26T11:06:01.641797Z","iopub.status.idle":"2024-11-26T11:06:01.676716Z","shell.execute_reply.started":"2024-11-26T11:06:01.641762Z","shell.execute_reply":"2024-11-26T11:06:01.675910Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"under_fifty = Nan_values(data_train)\nunder_fifty","metadata":{"execution":{"iopub.status.busy":"2024-11-26T11:06:17.288416Z","iopub.execute_input":"2024-11-26T11:06:17.289121Z","iopub.status.idle":"2024-11-26T11:06:17.304615Z","shell.execute_reply.started":"2024-11-26T11:06:17.289083Z","shell.execute_reply":"2024-11-26T11:06:17.303762Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"under_fifty[under_fifty.percentages>1]","metadata":{"execution":{"iopub.status.busy":"2024-11-26T11:06:21.423697Z","iopub.execute_input":"2024-11-26T11:06:21.424135Z","iopub.status.idle":"2024-11-26T11:06:21.436625Z","shell.execute_reply.started":"2024-11-26T11:06:21.424095Z","shell.execute_reply":"2024-11-26T11:06:21.435583Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"data_train.info()","metadata":{"execution":{"iopub.status.busy":"2024-11-26T11:06:24.430524Z","iopub.execute_input":"2024-11-26T11:06:24.431363Z","iopub.status.idle":"2024-11-26T11:06:24.443129Z","shell.execute_reply.started":"2024-11-26T11:06:24.431327Z","shell.execute_reply":"2024-11-26T11:06:24.442235Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"

📩Filling missing values

","metadata":{}},{"cell_type":"code","source":"def fill_data(df):\n # to fill float value columns with mean of them\n for col in df.select_dtypes(include='float').columns:\n mean_value = df[col].mean() # for calculating mean for each float column\n df[col] = df[col].fillna(mean_value)\n # to fill object columns with the most frequent value\n for colmn in df.columns:\n most_freq = df[colmn].mode()[0] #for calculating different mode for each column \n if most_freq is not None:\n df[col] = df[col].fillna(most_freq)\n else:\n most_freq = df[colmn].mode()[1]\n df[col] = df[col].fillna(most_freq)\n df[df.columns] = df[df.columns].fillna(method='ffill')\n df[df.columns] = df[df.columns].fillna(method='bfill')\n return df\ndf = fill_data(data_train)\ndf","metadata":{"execution":{"iopub.status.busy":"2024-11-26T11:06:28.632186Z","iopub.execute_input":"2024-11-26T11:06:28.632544Z","iopub.status.idle":"2024-11-26T11:06:28.709823Z","shell.execute_reply.started":"2024-11-26T11:06:28.632506Z","shell.execute_reply":"2024-11-26T11:06:28.708957Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.isna().sum().max()","metadata":{"execution":{"iopub.status.busy":"2024-11-26T11:06:31.719206Z","iopub.execute_input":"2024-11-26T11:06:31.719722Z","iopub.status.idle":"2024-11-26T11:06:31.734364Z","shell.execute_reply.started":"2024-11-26T11:06:31.719667Z","shell.execute_reply":"2024-11-26T11:06:31.733318Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df.sii.value_counts()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-26T11:06:34.375480Z","iopub.execute_input":"2024-11-26T11:06:34.375864Z","iopub.status.idle":"2024-11-26T11:06:34.383318Z","shell.execute_reply.started":"2024-11-26T11:06:34.375829Z","shell.execute_reply":"2024-11-26T11:06:34.382458Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"

🔢 Encoding

","metadata":{}},{"cell_type":"code","source":"import category_encoders as ce\ndef encoding(df):\n # to fill float value columns with mean of them\n for col in df.select_dtypes(include='object').columns:\n encoder = ce.LeaveOneOutEncoder(cols=[col])\n df[col] = encoder.fit_transform(df[col], df['sii'])\n return df\ndata_encoded = encoding(df)\ndata_encoded","metadata":{"execution":{"iopub.status.busy":"2024-11-26T11:07:51.835240Z","iopub.execute_input":"2024-11-26T11:07:51.835611Z","iopub.status.idle":"2024-11-26T11:07:52.248085Z","shell.execute_reply.started":"2024-11-26T11:07:51.835577Z","shell.execute_reply":"2024-11-26T11:07:52.247240Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"Y = data_encoded['sii'].values\nX = data_encoded.drop(['id','sii'],axis=1).values\n\nx_train,x_test,y_train,y_test = Split(X,Y,test_size=0.2,random_state=10)","metadata":{"execution":{"iopub.status.busy":"2024-11-26T11:07:56.971152Z","iopub.execute_input":"2024-11-26T11:07:56.972349Z","iopub.status.idle":"2024-11-26T11:07:56.981691Z","shell.execute_reply.started":"2024-11-26T11:07:56.972308Z","shell.execute_reply":"2024-11-26T11:07:56.980986Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"x_train.shape","metadata":{"execution":{"iopub.status.busy":"2024-11-26T11:07:59.970351Z","iopub.execute_input":"2024-11-26T11:07:59.970766Z","iopub.status.idle":"2024-11-26T11:07:59.977187Z","shell.execute_reply.started":"2024-11-26T11:07:59.970730Z","shell.execute_reply":"2024-11-26T11:07:59.976106Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"threshold = np.median(y_train)\ny_train_class = np.where(y_train > threshold, 1, 0) ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-26T11:08:02.280421Z","iopub.execute_input":"2024-11-26T11:08:02.281263Z","iopub.status.idle":"2024-11-26T11:08:02.285979Z","shell.execute_reply.started":"2024-11-26T11:08:02.281215Z","shell.execute_reply":"2024-11-26T11:08:02.284940Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"

📈 Model training

","metadata":{}},{"cell_type":"code","source":"model = AdaBoostClassifier()\n\nmodel.fit(x_train,y_train_class)","metadata":{"execution":{"iopub.status.busy":"2024-11-26T11:09:44.274671Z","iopub.execute_input":"2024-11-26T11:09:44.275059Z","iopub.status.idle":"2024-11-26T11:09:44.601666Z","shell.execute_reply.started":"2024-11-26T11:09:44.275028Z","shell.execute_reply":"2024-11-26T11:09:44.600781Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"threshold_t = np.median(y_test)\ny_test_class = np.where(y_test > threshold_t, 1, 0) ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2024-11-26T11:09:48.128139Z","iopub.execute_input":"2024-11-26T11:09:48.129007Z","iopub.status.idle":"2024-11-26T11:09:48.133491Z","shell.execute_reply.started":"2024-11-26T11:09:48.128969Z","shell.execute_reply":"2024-11-26T11:09:48.132575Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"model.score(x_test,y_test_class)","metadata":{"execution":{"iopub.status.busy":"2024-11-26T11:09:51.045705Z","iopub.execute_input":"2024-11-26T11:09:51.046389Z","iopub.status.idle":"2024-11-26T11:09:51.066200Z","shell.execute_reply.started":"2024-11-26T11:09:51.046349Z","shell.execute_reply":"2024-11-26T11:09:51.065429Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"Y_pred = model.predict(x_test)","metadata":{"execution":{"iopub.status.busy":"2024-11-26T11:09:54.502621Z","iopub.execute_input":"2024-11-26T11:09:54.503515Z","iopub.status.idle":"2024-11-26T11:09:54.520809Z","shell.execute_reply.started":"2024-11-26T11:09:54.503474Z","shell.execute_reply":"2024-11-26T11:09:54.520105Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"Y_pred = Y_pred.round(0)\nY_pred = Y_pred.flatten().tolist()","metadata":{"execution":{"iopub.status.busy":"2024-11-26T11:09:57.348831Z","iopub.execute_input":"2024-11-26T11:09:57.349153Z","iopub.status.idle":"2024-11-26T11:09:57.353942Z","shell.execute_reply.started":"2024-11-26T11:09:57.349125Z","shell.execute_reply":"2024-11-26T11:09:57.352961Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# 🔎 Testing","metadata":{}},{"cell_type":"code","source":"data_test = load_data(test_dir)\ndata_test.head(10)","metadata":{"execution":{"iopub.status.busy":"2024-11-26T11:10:00.359472Z","iopub.execute_input":"2024-11-26T11:10:00.360178Z","iopub.status.idle":"2024-11-26T11:10:00.388012Z","shell.execute_reply.started":"2024-11-26T11:10:00.360142Z","shell.execute_reply":"2024-11-26T11:10:00.387119Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"clm = data_encoded.drop('sii',axis=1).columns\ndata_test = data_test[clm]","metadata":{"execution":{"iopub.status.busy":"2024-11-26T11:10:03.755308Z","iopub.execute_input":"2024-11-26T11:10:03.755674Z","iopub.status.idle":"2024-11-26T11:10:03.763570Z","shell.execute_reply.started":"2024-11-26T11:10:03.755622Z","shell.execute_reply":"2024-11-26T11:10:03.762800Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"NAN_test = Nan_values(data_test)\nNAN_test","metadata":{"execution":{"iopub.status.busy":"2024-11-26T11:10:06.095994Z","iopub.execute_input":"2024-11-26T11:10:06.096827Z","iopub.status.idle":"2024-11-26T11:10:06.110266Z","shell.execute_reply.started":"2024-11-26T11:10:06.096788Z","shell.execute_reply":"2024-11-26T11:10:06.109373Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_test = fill_data(data_test)\ndf_test","metadata":{"execution":{"iopub.status.busy":"2024-11-26T11:10:09.437583Z","iopub.execute_input":"2024-11-26T11:10:09.438280Z","iopub.status.idle":"2024-11-26T11:10:09.505155Z","shell.execute_reply.started":"2024-11-26T11:10:09.438238Z","shell.execute_reply":"2024-11-26T11:10:09.504129Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"ids = df_test['id']\nfinal_test = df_test.copy()\nfinal_test.drop(\"id\",axis=1,inplace=True)\nfinal_test","metadata":{"execution":{"iopub.status.busy":"2024-11-26T11:10:13.244938Z","iopub.execute_input":"2024-11-26T11:10:13.245789Z","iopub.status.idle":"2024-11-26T11:10:13.278042Z","shell.execute_reply.started":"2024-11-26T11:10:13.245752Z","shell.execute_reply":"2024-11-26T11:10:13.277322Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"ordin = OrdinalEncoder()\ncol = final_test.select_dtypes(include='object').columns\nfinal_test[col] = ordin.fit_transform(final_test[col])\nfinal_test","metadata":{"execution":{"iopub.status.busy":"2024-11-26T11:10:16.571233Z","iopub.execute_input":"2024-11-26T11:10:16.571599Z","iopub.status.idle":"2024-11-26T11:10:16.613831Z","shell.execute_reply.started":"2024-11-26T11:10:16.571566Z","shell.execute_reply":"2024-11-26T11:10:16.612885Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y = final_test.values\ny","metadata":{"execution":{"iopub.status.busy":"2024-11-26T11:10:19.580086Z","iopub.execute_input":"2024-11-26T11:10:19.580959Z","iopub.status.idle":"2024-11-26T11:10:19.593506Z","shell.execute_reply.started":"2024-11-26T11:10:19.580922Z","shell.execute_reply":"2024-11-26T11:10:19.592375Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"

📊 Predictions

","metadata":{}},{"cell_type":"code","source":"y_pred = model.predict(y)\ny_pred","metadata":{"execution":{"iopub.status.busy":"2024-11-26T11:10:25.503372Z","iopub.execute_input":"2024-11-26T11:10:25.504133Z","iopub.status.idle":"2024-11-26T11:10:25.520927Z","shell.execute_reply.started":"2024-11-26T11:10:25.504094Z","shell.execute_reply":"2024-11-26T11:10:25.520004Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"y_pred = y_pred.round(0)","metadata":{"execution":{"iopub.status.busy":"2024-11-26T11:10:30.774770Z","iopub.execute_input":"2024-11-26T11:10:30.775575Z","iopub.status.idle":"2024-11-26T11:10:30.779476Z","shell.execute_reply.started":"2024-11-26T11:10:30.775540Z","shell.execute_reply":"2024-11-26T11:10:30.778409Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission = pd.DataFrame({\n 'id' : ids,\n 'sii' : y_pred\n})\nsubmission","metadata":{"execution":{"iopub.status.busy":"2024-11-26T11:10:43.971022Z","iopub.execute_input":"2024-11-26T11:10:43.971876Z","iopub.status.idle":"2024-11-26T11:10:43.981834Z","shell.execute_reply.started":"2024-11-26T11:10:43.971831Z","shell.execute_reply":"2024-11-26T11:10:43.980865Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"submission.to_csv('submission.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2024-11-26T11:10:52.373022Z","iopub.execute_input":"2024-11-26T11:10:52.373409Z","iopub.status.idle":"2024-11-26T11:10:52.381320Z","shell.execute_reply.started":"2024-11-26T11:10:52.373375Z","shell.execute_reply":"2024-11-26T11:10:52.380570Z"},"trusted":true},"outputs":[],"execution_count":null}]}