{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"}],"dockerImageVersionId":30786,"isInternetEnabled":false,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.cdfdfom/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-11-20T08:06:29.559337Z","iopub.execute_input":"2024-11-20T08:06:29.559762Z","iopub.status.idle":"2024-11-20T08:06:34.939665Z","shell.execute_reply.started":"2024-11-20T08:06:29.559715Z","shell.execute_reply":"2024-11-20T08:06:34.938240Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport os\nimport pyarrow.parquet as pq\nfrom sklearn.cluster import KMeans\nfrom sklearn.preprocessing import MinMaxScaler\nfrom sklearn.metrics import accuracy_score\nimport lightgbm as lgb\nfrom sklearn.model_selection import train_test_split","metadata":{"execution":{"iopub.status.busy":"2024-11-20T08:06:34.941128Z","iopub.execute_input":"2024-11-20T08:06:34.941781Z","iopub.status.idle":"2024-11-20T08:06:37.005602Z","shell.execute_reply.started":"2024-11-20T08:06:34.941741Z","shell.execute_reply":"2024-11-20T08:06:37.004422Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def preprocessing(name):\n    df = pd.read_csv(name)\n    \n    #delete the columns with \"PCIAT\"\n    columns_to_delete = [col for col in df.columns if col.startswith('PCIAT')]\n    df = df.drop(columns=columns_to_delete)\n    \n    #delete the column of sii in training set and retrieve the information of labels\n    if (name.endswith('train.csv')):\n        labels = df['sii']\n        df = df.drop(columns='sii')\n    else:\n        #if the name = 'test.csv', the labels won't be used\n        labels = 0\n        \n    #retrieve the information of ids and then delete it from dataframe\n    df_result = pd.DataFrame(columns=['id'])\n    df_result['id'] = df['id']\n    df = df.drop('id', axis=1)\n    \n    #one-hot-encoding for those categorical data\n    df.drop('Basic_Demos-Enroll_Season', axis=1, inplace=True)\n    columns_to_drop = ['PAQ_C-Season', 'PAQ_A-Season']\n    df.drop(columns=columns_to_drop, inplace=True)\n    object_list = df.select_dtypes(include=['object']).columns.tolist()\n    df_encoded_filled = pd.get_dummies(df, columns=object_list) #reserve the na\n    \n    df_grp_age_sex = df_encoded_filled.groupby(['Basic_Demos-Age', 'Basic_Demos-Sex'])\n    print(df_grp_age_sex)\n    fill_na_dict = {'Physical-Diastolic_BP': df_grp_age_sex['Physical-Diastolic_BP'].transform('median'),\n\n                'Physical-Systolic_BP': df_grp_age_sex['Physical-Systolic_BP'].transform('median')}\n\n    df_encoded_filled.fillna(fill_na_dict, inplace=True)\n    fill_na_dict = {'Physical-Height': df_grp_age_sex['Physical-Height'].transform('median'),\n                'Physical-Weight': df_grp_age_sex['Physical-Weight'].transform('median')}\n    \n    df_encoded_filled.fillna(value=fill_na_dict, inplace=True)\n    \n    df_encoded_filled.fillna(value={'Physical-BMI': 703 * df_encoded_filled['Physical-Weight'] / df_encoded_filled['Physical-Height'] ** 2}, inplace=True)\n    \n    fill_na_dict = {'Physical-Waist_Circumference': df_grp_age_sex['Physical-Waist_Circumference'].transform('median')}\n    df_encoded_filled.fillna(value=fill_na_dict, inplace=True)\n    \n    fill_na_dict = {'Physical-HeartRate': df_grp_age_sex['Physical-HeartRate'].transform('median')}\n\n    df_encoded_filled.fillna(value={'Physical-HeartRate': df_encoded_filled['Physical-HeartRate'].median()}, inplace=True)\n    \n    fill_na_list = ['Fitness_Endurance-Time_Mins', 'Fitness_Endurance-Time_Sec',\n\n                'FGC-FGC_CU', 'FGC-FGC_GSND', 'FGC-FGC_GSD',\n\n                'FGC-FGC_PU', 'FGC-FGC_SRL', 'FGC-FGC_SRR', 'FGC-FGC_TL']\n\n    fill_na_dict = {feature: df_grp_age_sex[feature].transform('median') for feature in fill_na_list}\n    df_encoded_filled.fillna(value=fill_na_dict, inplace=True)\n    \n    \n    df_encoded_filled = pd.get_dummies(df_encoded_filled, columns=['BIA-BIA_Activity_Level_num','BIA-BIA_Frame_num'])\n    \n    \n    fill_na_list = ['BIA-BIA_BMC','BIA-BIA_BMI','BIA-BIA_BMR','BIA-BIA_DEE','BIA-BIA_ECW',\n                    'BIA-BIA_FFM','BIA-BIA_FFMI','BIA-BIA_FMI','BIA-BIA_Fat','BIA-BIA_ICW','BIA-BIA_LDM',\n                    'BIA-BIA_LST','BIA-BIA_SMM','BIA-BIA_TBW']\n    fill_na_dict = {feature: df_grp_age_sex[feature].transform('median') for feature in fill_na_list}\n    \n    df_encoded_filled.fillna(value=fill_na_dict, inplace=True)\n    \n    \n    df_encoded_filled['Fitness_Endurance-Time'] = df_encoded_filled['Fitness_Endurance-Time_Mins'] * 60 + df_encoded_filled['Fitness_Endurance-Time_Sec']\n    \n    df_encoded_filled['PAQ_Total'] = df_encoded_filled['PAQ_A-PAQ_A_Total'].fillna(df_encoded_filled['PAQ_C-PAQ_C_Total'])\n    df_grp_age_sex = df_encoded_filled.groupby(['Basic_Demos-Age', 'Basic_Demos-Sex'])\n    fill_na_dict = {'PAQ_Total': df_grp_age_sex['PAQ_Total'].transform('median')}\n    df_encoded_filled.fillna(value=fill_na_dict, inplace=True)\n    df_encoded_filled.drop(columns = ['PAQ_A-PAQ_A_Total', 'PAQ_C-PAQ_C_Total'])\n    \n    \n    fill_na_list = ['SDS-SDS_Total_Raw','SDS-SDS_Total_T']\n    fill_na_dict = {feature: df_grp_age_sex[feature].transform('median') for feature in fill_na_list}\n    df_encoded_filled.fillna(value=fill_na_dict, inplace=True)\n    \n    \n    \n    df_encoded_filled.fillna(value={'PreInt_EduHx-computerinternet_hoursday': df_encoded_filled['PreInt_EduHx-computerinternet_hoursday'].mode()[0]}, inplace=True)\n    fill_na_dict = {'CGAS-CGAS_Score': df_grp_age_sex['CGAS-CGAS_Score'].transform('median')}\n    df_encoded_filled.fillna(value=fill_na_dict, inplace=True)\n    \n    \n    columns_to_drop = ['Physical-BMI','Fitness_Endurance-Time_Mins','Fitness_Endurance-Time_Sec', 'FGC-FGC_CU_Zone', \n                      'FGC-FGC_GSND_Zone', 'FGC-FGC_GSD_Zone', 'FGC-FGC_PU_Zone', 'FGC-FGC_SRL_Zone', 'FGC-FGC_SRR_Zone', 'FGC-FGC_TL_Zone']\n    df_encoded_filled.drop(columns=columns_to_drop, inplace=True)\n    print(df_encoded_filled.columns)\n    return [df_encoded_filled, df_result, labels]\n\n\"\"\"\nDescription: do the preprocessing of the dataset\n\ninput: \n    name: the filename of the dataset\n    \noutput: \n    df_encoded_filled: the df after preprocessing\n    df_result: the df saving the information of ids\n    labels: the labels of the data if the filename is train.csv     \n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2024-11-20T08:06:37.008453Z","iopub.execute_input":"2024-11-20T08:06:37.009207Z","iopub.status.idle":"2024-11-20T08:06:37.034542Z","shell.execute_reply.started":"2024-11-20T08:06:37.009154Z","shell.execute_reply":"2024-11-20T08:06:37.033402Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def model(df_encoded_filled, labels, leaves = 61, learning_rate = 0.01):\n    class_weights = [1, 12, 24, 300]\n    #Step1: training of the model using labeled data(only 2/3 of the train dataset)\n    train_set = df_encoded_filled[labels.notnull()]\n    label_input = labels[labels.notnull()]\n    X_train, X_test, y_train, y_test = train_test_split(train_set, label_input, test_size=0.2, random_state=42)\n    train_data = lgb.Dataset(X_train, label=y_train, params={'use_missing': True}, weight=np.array([class_weights[int(y)] for y in y_train]))\n    test_data = lgb.Dataset(X_test, label=y_test, weight=np.array([class_weights[int(y)] for y in y_test]), reference=train_data)\n    params = {\n    'objective': 'multiclass',\n    'num_class': 4,\n    'metric': 'multi_logloss',\n    'boosting_type': 'goss',\n    'num_leaves': leaves,\n    'learning_rate': learning_rate,\n    'feature_fraction': 0.9,\n    }\n    num_round = 200\n    model = lgb.train(params, train_data, num_round, valid_sets=[test_data])\n    \n    #Step2: make prediction of the remaining 1/3 data which are unlabeled in training set\n    df_encoded_filled_reset = df_encoded_filled.reset_index(drop=True)\n    y_pred = model.predict(df_encoded_filled[labels.isnull()])\n    \n    y_pred_max = [x.argmax() for x in y_pred]\n    nan_index = labels.isnull()\n    row_of_nan = df_encoded_filled.index[nan_index]\n    for index, pred_values in zip(row_of_nan, y_pred_max):\n        labels[index] = pred_values\n        \n    #Step3: re-train the model using the full labeled dataset\n    train_set = df_encoded_filled\n    X_train, X_test, y_train, y_test = train_test_split(train_set, labels, test_size=0.2, random_state=42)\n    train_data = lgb.Dataset(X_train, label=y_train, params={'use_missing': True}, weight=np.array([class_weights[int(y)] for y in y_train]))\n    test_data = lgb.Dataset(X_test, label=y_test, weight=np.array([class_weights[int(y)] for y in y_test]), reference=train_data)\n    model_retrained = lgb.train(params, train_data, num_round, valid_sets=[test_data])   \n    return model_retrained\n\n\"\"\"\nDescription: Do the process of semi-supervised learning\n\ninput:  \n    df_encoded_filled: the df of training set after preprocessing\n    labels: the value of sii in training set\n    \noutput: \n    model_retrained: the resulted model\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2024-11-20T08:06:37.036153Z","iopub.execute_input":"2024-11-20T08:06:37.036757Z","iopub.status.idle":"2024-11-20T08:06:37.055176Z","shell.execute_reply.started":"2024-11-20T08:06:37.036704Z","shell.execute_reply":"2024-11-20T08:06:37.054129Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def prediction(name_test, model_retrained):\n    [df_encoded_filled, df_result, _] = preprocessing(name_test)\n    \n    #Ensure that the input of the model contains the same format with training data\n    columns_list = ['Basic_Demos-Age', 'Basic_Demos-Sex', 'CGAS-CGAS_Score',\n       'Physical-Height', 'Physical-Weight', 'Physical-Waist_Circumference',\n       'Physical-Diastolic_BP', 'Physical-HeartRate', 'Physical-Systolic_BP',\n       'Fitness_Endurance-Max_Stage', 'FGC-FGC_CU', 'FGC-FGC_GSND',\n       'FGC-FGC_GSD', 'FGC-FGC_PU', 'FGC-FGC_SRL', 'FGC-FGC_SRR', 'FGC-FGC_TL',\n       'BIA-BIA_BMC', 'BIA-BIA_BMI', 'BIA-BIA_BMR', 'BIA-BIA_DEE',\n       'BIA-BIA_ECW', 'BIA-BIA_FFM', 'BIA-BIA_FFMI', 'BIA-BIA_FMI',\n       'BIA-BIA_Fat', 'BIA-BIA_ICW', 'BIA-BIA_LDM', 'BIA-BIA_LST',\n       'BIA-BIA_SMM', 'BIA-BIA_TBW', 'PAQ_A-PAQ_A_Total', 'PAQ_C-PAQ_C_Total',\n       'SDS-SDS_Total_Raw', 'SDS-SDS_Total_T',\n       'PreInt_EduHx-computerinternet_hoursday', 'CGAS-Season_Fall',\n       'CGAS-Season_Spring', 'CGAS-Season_Summer', 'CGAS-Season_Winter',\n       'Physical-Season_Fall', 'Physical-Season_Spring',\n       'Physical-Season_Summer', 'Physical-Season_Winter',\n       'Fitness_Endurance-Season_Fall', 'Fitness_Endurance-Season_Spring',\n       'Fitness_Endurance-Season_Summer', 'Fitness_Endurance-Season_Winter',\n       'FGC-Season_Fall', 'FGC-Season_Spring', 'FGC-Season_Summer',\n       'FGC-Season_Winter', 'BIA-Season_Fall', 'BIA-Season_Spring',\n       'BIA-Season_Summer', 'BIA-Season_Winter', 'SDS-Season_Fall',\n       'SDS-Season_Spring', 'SDS-Season_Summer', 'SDS-Season_Winter',\n       'PreInt_EduHx-Season_Fall', 'PreInt_EduHx-Season_Spring',\n       'PreInt_EduHx-Season_Summer', 'PreInt_EduHx-Season_Winter',\n       'BIA-BIA_Activity_Level_num_1.0', 'BIA-BIA_Activity_Level_num_2.0',\n       'BIA-BIA_Activity_Level_num_3.0', 'BIA-BIA_Activity_Level_num_4.0',\n       'BIA-BIA_Activity_Level_num_5.0', 'BIA-BIA_Frame_num_1.0',\n       'BIA-BIA_Frame_num_2.0', 'BIA-BIA_Frame_num_3.0',\n       'Fitness_Endurance-Time', 'PAQ_Total']\n    for col in columns_list:\n        if col not in df_encoded_filled.columns:\n            df_encoded_filled[col] = np.nan\n    df_encoded_filled = df_encoded_filled.reindex(columns=columns_list)\n    \n    #make prediction\n    labels = model_retrained.predict(df_encoded_filled)\n    pred = [x.argmax() for x in labels]\n    df_result['sii'] = pred\n    return df_result\n\n\"\"\"\nDescription: make the prediction of the test set\n\ninput:  \n    name_test: the filename of test set\n    model_retrained: the resulted model from semi-supervised learning\n    \noutput: \n    df_result: the resulted prediction df which aligns the format of submission\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2024-11-20T08:06:37.056734Z","iopub.execute_input":"2024-11-20T08:06:37.057139Z","iopub.status.idle":"2024-11-20T08:06:37.074657Z","shell.execute_reply.started":"2024-11-20T08:06:37.057104Z","shell.execute_reply":"2024-11-20T08:06:37.073314Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def main(train_set, test_set):\n    \n    #do preprocessing on training set\n    score = []\n    df_result_array = []\n    #df_sol = pd.read_csv('test.csv')\n    #ans = df_sol['sii']\n    #train the model\n    for leave in [61]:\n        for rate in [0.04]:\n            [df_train_encoded_filled, train_result, labels] = preprocessing(train_set)\n            model_retrained = model(df_train_encoded_filled, labels, leave, rate)\n            \n            #make prediction\n            df_result = prediction(test_set, model_retrained)\n            \"\"\"\n            predi = df_result['sii']\n            print(f\"F1: {f1_score(ans, predi, average='weighted')}\")\n            score.append(f1_score(ans, predi, average='weighted'))\n            df_result_array.append(df_result['sii'])\n            \"\"\"\n            \n    return df_result\n\n\"\"\"\nDescription: main function of whole program\n\ninput:  \n    train_set: the filename of train set\n    test_set: the filename of test set\n    \noutput: \n    df_result: the resulted prediction df which aligns the format of submission\n\"\"\"","metadata":{"execution":{"iopub.status.busy":"2024-11-20T08:06:37.076305Z","iopub.execute_input":"2024-11-20T08:06:37.076660Z","iopub.status.idle":"2024-11-20T08:06:37.092872Z","shell.execute_reply.started":"2024-11-20T08:06:37.076628Z","shell.execute_reply":"2024-11-20T08:06:37.091572Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_result = main('/kaggle/input/child-mind-institute-problematic-internet-use/train.csv', '/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')","metadata":{"execution":{"iopub.status.busy":"2024-11-20T08:06:37.094298Z","iopub.execute_input":"2024-11-20T08:06:37.094680Z","iopub.status.idle":"2024-11-20T08:06:58.366846Z","shell.execute_reply.started":"2024-11-20T08:06:37.094644Z","shell.execute_reply":"2024-11-20T08:06:58.365319Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_result.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2024-11-20T08:06:58.369257Z","iopub.execute_input":"2024-11-20T08:06:58.369642Z","iopub.status.idle":"2024-11-20T08:06:58.379326Z","shell.execute_reply.started":"2024-11-20T08:06:58.369606Z","shell.execute_reply":"2024-11-20T08:06:58.378012Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_result","metadata":{"execution":{"iopub.status.busy":"2024-11-20T08:06:58.381011Z","iopub.execute_input":"2024-11-20T08:06:58.381381Z","iopub.status.idle":"2024-11-20T08:06:58.396863Z","shell.execute_reply.started":"2024-11-20T08:06:58.381331Z","shell.execute_reply":"2024-11-20T08:06:58.395515Z"},"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{},"outputs":[],"execution_count":null}]}