{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"}],"dockerImageVersionId":30775,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"%%time\nimport pandas as pd\nfrom sklearn.decomposition import PCA\n# explicitly require this experimental feature\nfrom sklearn.experimental import enable_iterative_imputer  # noqa\n# now you can import normally from sklearn.impute\nfrom sklearn.impute import IterativeImputer\nimport numpy as np\nimport seaborn as sns\n\ndf_train = pd.read_csv(\"/kaggle/input/child-mind-institute-problematic-internet-use/train.csv\")\ndf_test = pd.read_csv('/kaggle/input/child-mind-institute-problematic-internet-use/test.csv')\n\nfeatures = df_test.columns.tolist()\nfeatures.remove('id')\n\ncat_features = df_test.drop('id', axis=1).select_dtypes(include='object').columns.tolist()\ncat_features += ['Basic_Demos-Sex']\nnum_features = [x for x in features if x not in cat_features]\n\n\ndef impute_num_features(train_data, test_data):\n    data = pd.concat([train_data, test_data])\n    imputer = IterativeImputer(random_state=2024)\n    imputer.set_output(transform='pandas')\n    imputer.fit(data)\n    data = imputer.transform(data)\n    return data.iloc[:len(train_data)], data.iloc[len(train_data):]\n\n\ndef reduce_dimension(train_data, test_data, dim):\n    data = pd.concat([train_data, test_data])\n    pca = PCA(n_components=dim, random_state=2024)\n    pca.set_output(transform='pandas')\n    pca.fit(data)\n    data = pca.transform(data)\n    return data.iloc[:len(train_data)], data.iloc[len(train_data):], pca\n\n\ndef cal_winsorize(train_data, test_data, limits=[0.01, 0.01]):\n    data = pd.concat([train_data, test_data])\n    for col in data.columns:\n        data[col] = np.clip(data[col].values, np.nanquantile(data[col].values, limits[0]), np.nanquantile(data[col].values, 1 - limits[1]))\n    return data.iloc[:len(train_data)], data.iloc[len(train_data):]\n\ndef preprocess_categorical_cols(data):\n    return data.fillna('NaN').astype('category')\n\ndf_train[cat_features] = preprocess_categorical_cols(df_train[cat_features])\ndf_test[cat_features] = preprocess_categorical_cols(df_test[cat_features])\n\n\ndf_train[num_features], df_test[num_features] = cal_winsorize(df_train[num_features], df_test[num_features])\ndf_train[num_features], df_test[num_features] = impute_num_features(df_train[num_features], df_test[num_features])\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-09-25T10:08:11.823254Z","iopub.execute_input":"2024-09-25T10:08:11.823713Z","iopub.status.idle":"2024-09-25T10:08:47.019992Z","shell.execute_reply.started":"2024-09-25T10:08:11.823670Z","shell.execute_reply":"2024-09-25T10:08:47.018437Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\ndf_train_num_pca, df_test_num_pca, pca = reduce_dimension(df_train[num_features], df_test[num_features], 6)\n\ndef scatter_plot(df_features, target):\n    data = pd.concat([df_features, target], axis=1)\n    data = data[~target.isna()]\n    sns.scatterplot(data=data, x=\"pca0\", y=\"pca1\", hue=\"sii\")\n\nprint(pca.explained_variance_ratio_)\nscatter_plot(df_train_num_pca, df_train['sii'])","metadata":{"execution":{"iopub.status.busy":"2024-09-25T10:08:47.023083Z","iopub.execute_input":"2024-09-25T10:08:47.024283Z","iopub.status.idle":"2024-09-25T10:08:48.023548Z","shell.execute_reply.started":"2024-09-25T10:08:47.024207Z","shell.execute_reply":"2024-09-25T10:08:48.022337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\n","metadata":{},"execution_count":null,"outputs":[]}]}