{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.11.11","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":81933,"databundleVersionId":9643020,"sourceType":"competition"}],"dockerImageVersionId":31012,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n\ndictionary = pd.read_csv(\"/kaggle/input/child-mind-institute-problematic-internet-use/data_dictionary.csv\")\ndataset = pd.read_csv(\"/kaggle/input/child-mind-institute-problematic-internet-use/train.csv\")\ndataset_test = pd.read_csv(\"/kaggle/input/child-mind-institute-problematic-internet-use/test.csv\")\n\n\n\n# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        # print(os.path.join(dirname, filename))\n        pass\n        \n        \n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-22T07:57:30.833306Z","iopub.execute_input":"2025-09-22T07:57:30.833656Z","iopub.status.idle":"2025-09-22T07:57:32.039552Z","shell.execute_reply.started":"2025-09-22T07:57:30.833608Z","shell.execute_reply":"2025-09-22T07:57:32.038264Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dictionary","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-22T07:57:32.041271Z","iopub.execute_input":"2025-09-22T07:57:32.041532Z","iopub.status.idle":"2025-09-22T07:57:32.052700Z","shell.execute_reply.started":"2025-09-22T07:57:32.041512Z","shell.execute_reply":"2025-09-22T07:57:32.051911Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#definizione costanti\n\nLABEL = 'PreInt_EduHx-computerinternet_hoursday'","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-22T07:57:32.053339Z","iopub.execute_input":"2025-09-22T07:57:32.053533Z","iopub.status.idle":"2025-09-22T07:57:32.070566Z","shell.execute_reply.started":"2025-09-22T07:57:32.053517Z","shell.execute_reply":"2025-09-22T07:57:32.069804Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\ntrain = None\ntest = None\n\ntrain, test = train_test_split(dataset, train_size=0.8, random_state=0)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-22T07:57:32.071538Z","iopub.execute_input":"2025-09-22T07:57:32.072221Z","iopub.status.idle":"2025-09-22T07:57:32.092203Z","shell.execute_reply.started":"2025-09-22T07:57:32.072202Z","shell.execute_reply":"2025-09-22T07:57:32.091474Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Analisi iniziale\n\nnon avendo alcuna conoscenza del dataset, mi affideró inizialmente alla documentazione, con l'obiettivo di capire meglio cosa trattano i dati e rendermi piu facile la loro pulizia","metadata":{}},{"cell_type":"markdown","source":"\nnella documentazione sono specificati i tipi delle colonne, str, categorical int, int, float. Useró questa informazione per trovare le colonne da normalizzare in seguito","metadata":{}},{"cell_type":"code","source":"normalizable_types = set([\"int\", \"float\"])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-22T07:57:32.094262Z","iopub.execute_input":"2025-09-22T07:57:32.094485Z","iopub.status.idle":"2025-09-22T07:57:32.109151Z","shell.execute_reply.started":"2025-09-22T07:57:32.094468Z","shell.execute_reply":"2025-09-22T07:57:32.108484Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"La documentazione raggruppa le feature per \"Instrument\", dando cosi il modo di capire che sono collegate, eseguo ora dei controlli per vedere se i dati sono coerenti tra loro o se ci sono delle discrepanze","metadata":{}},{"cell_type":"code","source":"dictionary[dictionary[\"Instrument\"] == \"Children's Global Assessment Scale\"] \n\n#DATI SPORCHI: CGAS-Season\te CGAS-CGAS_Score  sono due colonne collegate, capita che la prima sia valorizzata mentre la seconda no.\ntrain[~train[\"CGAS-Season\"].isnull() & train[\"CGAS-CGAS_Score\"].isnull()][[\"CGAS-Season\", \"CGAS-CGAS_Score\"]]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-22T07:57:32.110145Z","iopub.execute_input":"2025-09-22T07:57:32.110410Z","iopub.status.idle":"2025-09-22T07:57:32.135516Z","shell.execute_reply.started":"2025-09-22T07:57:32.110381Z","shell.execute_reply":"2025-09-22T07:57:32.134680Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"both_nan = train[~train[\"CGAS-Season\"].isnull() & train[\"CGAS-CGAS_Score\"].isnull()][[\"CGAS-Season\", \"CGAS-CGAS_Score\"]]\nboth_nan[\"CGAS-Season\"] = np.nan\nboth_nan","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-22T07:57:32.136291Z","iopub.execute_input":"2025-09-22T07:57:32.136471Z","iopub.status.idle":"2025-09-22T07:57:32.155025Z","shell.execute_reply.started":"2025-09-22T07:57:32.136457Z","shell.execute_reply":"2025-09-22T07:57:32.154442Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"controllo di valori NaN per le feature dell'instrument **Physical Measures**","metadata":{}},{"cell_type":"code","source":"physical_measures_fields =  dictionary[dictionary[\"Instrument\"] == \"Physical Measures\"][\"Field\"]\n\ntrain[train[physical_measures_fields.iloc[0]].isnull() &\n(        \n    ~train[physical_measures_fields.iloc[1]].isnull() |\n    ~train[physical_measures_fields.iloc[2]].isnull() |\n    ~train[physical_measures_fields.iloc[3]].isnull() |\n    ~train[physical_measures_fields.iloc[4]].isnull() |\n    ~train[physical_measures_fields.iloc[5]].isnull() |\n    ~train[physical_measures_fields.iloc[6]].isnull() |\n    ~train[physical_measures_fields.iloc[7]].isnull() \n)\n][physical_measures_fields].head(50)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-22T07:57:32.155807Z","iopub.execute_input":"2025-09-22T07:57:32.156024Z","iopub.status.idle":"2025-09-22T07:57:32.173949Z","shell.execute_reply.started":"2025-09-22T07:57:32.155999Z","shell.execute_reply":"2025-09-22T07:57:32.173113Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"controllo di valori NaN per le feature dell'instrument **FitnessGram Vitals and Treadmill**\n","metadata":{}},{"cell_type":"code","source":"FitnessGram_Vitals_and_Treadmill_cols = dictionary[dictionary[\"Instrument\"] == \"FitnessGram Vitals and Treadmill\"][\"Field\"]\ntrain[\ntrain[\"Fitness_Endurance-Season\"].isnull() &\n(\n    ~train[\"Fitness_Endurance-Max_Stage\"].isnull() &\n    ~train[\"Fitness_Endurance-Time_Mins\"].isnull()\n)]\n[FitnessGram_Vitals_and_Treadmill_cols]\n#le colonne Fitness_Endurance-Time_Mins e\tFitness_Endurance-Time_Sec possono essere condensate in una unica colonna dei secondi","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-22T07:57:32.174727Z","iopub.execute_input":"2025-09-22T07:57:32.174980Z","iopub.status.idle":"2025-09-22T07:57:32.189210Z","shell.execute_reply.started":"2025-09-22T07:57:32.174949Z","shell.execute_reply":"2025-09-22T07:57:32.188272Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#questa categoria ha 15 colonne, non posso effettuare lo stesso controllo di prima\nprint(f'N colonne: {dictionary[dictionary[\"Instrument\"] == \"FitnessGram Child\"].shape[0]}')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-09-22T07:57:32.190164Z","iopub.execute_input":"2025-09-22T07:57:32.190473Z","iopub.status.idle":"2025-09-22T07:57:32.207059Z","shell.execute_reply.started":"2025-09-22T07:57:32.190450Z","shell.execute_reply":"2025-09-22T07:57:32.206198Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#nuovo controllo: prendo le colonne numeriche e ne estraggo la media il minimo e il massimo, eventualmente le normalizzo\n\ndef print_col_stats(dataframe):\n    copy = dataframe[\n        dictionary[\n            dictionary[\"Field\"].isin(dataframe.columns) &\n            dictionary[\"Type\"].isin(normalizable_types)]\n        [\"Field\"]\n    ]\n\n    for i in copy.columns:\n        print(f'Colonna: {i:<15} media:{dataframe[i].mean():<8.2f} std deviation: {dataframe[i].std():<8.2f} massimo: {dataframe[i].max():<8.2f} minimo: {dataframe[i].min():<8.2f}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-22T07:57:32.207957Z","iopub.execute_input":"2025-09-22T07:57:32.208285Z","iopub.status.idle":"2025-09-22T07:57:32.224553Z","shell.execute_reply.started":"2025-09-22T07:57:32.208254Z","shell.execute_reply":"2025-09-22T07:57:32.223800Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"fitnessgram_child_cols =  dictionary[dictionary[\"Instrument\"] == \"FitnessGram Child\"][\"Field\"]\nprint_col_stats(train[fitnessgram_child_cols])\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-22T07:57:32.225504Z","iopub.execute_input":"2025-09-22T07:57:32.225798Z","iopub.status.idle":"2025-09-22T07:57:32.254413Z","shell.execute_reply.started":"2025-09-22T07:57:32.225758Z","shell.execute_reply":"2025-09-22T07:57:32.253667Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"queste informazioni non mi aiuta molto a capire se ci sono dei valori fuori posto...\ndecido comunque che tutte le colonne int, float vanno normalizzate","metadata":{}},{"cell_type":"code","source":"train[dictionary[dictionary[\"Instrument\"] == \"Bio-electric Impedance Analysis\"][\"Field\"]]\n\nbio_electric_cols = dictionary[dictionary[\"Instrument\"] == \"Bio-electric Impedance Analysis\"][\"Field\"]\n\nprint_col_stats(train[bio_electric_cols])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-22T07:57:32.255294Z","iopub.execute_input":"2025-09-22T07:57:32.255580Z","iopub.status.idle":"2025-09-22T07:57:32.273799Z","shell.execute_reply.started":"2025-09-22T07:57:32.255551Z","shell.execute_reply":"2025-09-22T07:57:32.272869Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Fine analisi 'manuale' del dataset\n\nda questo momento i raggruppamenti diventano troppo grandi ed é difficile accorgersi dei dettagli, passo a creare la funzione che prepara i dati","metadata":{}},{"cell_type":"markdown","source":"# Preparazione dei dati","metadata":{}},{"cell_type":"markdown","source":"viene detto nella documentazione che dalle colonne relative a Parent-Child Internet Addiction Test (**PCIAT**) viene derivato il risultato \nche sta nella colonna 'sii', queste colonne non sono presenti nel test quindi, le rimuoveró","metadata":{}},{"cell_type":"code","source":"PCIAT_cols = dictionary[dictionary[\"Instrument\"] == \"Parent-Child Internet Addiction Test\"][\"Field\"]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-22T07:57:32.277951Z","iopub.execute_input":"2025-09-22T07:57:32.278188Z","iopub.status.idle":"2025-09-22T07:57:32.284376Z","shell.execute_reply.started":"2025-09-22T07:57:32.278169Z","shell.execute_reply":"2025-09-22T07:57:32.283570Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Alcune istanze hanno la **label non valorizzata**, non potendo essere d'aiuto ad un modello di supervised learning, le elimineró","metadata":{}},{"cell_type":"code","source":"#alcune istanze hanno la label non valorizzata, non potendo essere d'aiuto le elimineró\ntrain[train[\"sii\"].isnull()][[\"id\", \"sii\"]]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-22T07:57:32.285227Z","iopub.execute_input":"2025-09-22T07:57:32.285447Z","iopub.status.idle":"2025-09-22T07:57:32.305840Z","shell.execute_reply.started":"2025-09-22T07:57:32.285424Z","shell.execute_reply":"2025-09-22T07:57:32.305005Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#definisco tutte le correzioni che vanno fatte ai dati\n\ndef clear_CGAS_cols(dataset):\n    copy = dataset.copy()\n    condition = copy[\"CGAS-Season\"].isnull() | copy[\"CGAS-CGAS_Score\"].isnull()\n    copy.loc[condition, \"CGAS-Season\"] = np.nan\n    copy.loc[condition, \"CGAS-CGAS_Score\"] = np.nan\n    return copy\n\ndef drop_PCIAT_cols(dataset):\n    return dataset.drop(columns=PCIAT_cols)\n\ndef drop_null_label_rows(dataset):     \n    return dataset.drop(dataset[dataset[\"sii\"].isnull()].index)\n\ndef drop_id_column(dataset):\n    return dataset.drop(\"id\", axis=1)\n    \n\ndef merge_min_sec_cols(dataset):\n    copy = dataset.copy()\n    copy[\"Fitness_Endurance-Time_Sec\"] += copy[\"Fitness_Endurance-Time_Mins\"]*60\n    return copy.drop(columns=[\"Fitness_Endurance-Time_Mins\"])\n\ndef to_float(dataset):\n    numerical = dataset.select_dtypes(include=\"number\")\n    classified = dataset.select_dtypes(exclude=\"number\")\n    # result = pd.concat([numerical.astype(\"float\"), classified], axis=1)\n    return pd.concat([numerical.astype(\"float\"), classified], axis=1)\n\n\n\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.preprocessing import FunctionTransformer\n\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-22T07:57:32.306781Z","iopub.execute_input":"2025-09-22T07:57:32.307136Z","iopub.status.idle":"2025-09-22T07:57:32.317329Z","shell.execute_reply.started":"2025-09-22T07:57:32.307111Z","shell.execute_reply":"2025-09-22T07:57:32.316676Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Scelta delle classi per la pulizia dei dati\n\nHo diverse trasformazioni da eseguire, mi avvarró della **Pipeline** di SKlearn per vari motivi, tra cui i piu importanti:\n- Override delle funzioni fit e transform per poter preservare il pandas.Dataframe: dopo aver usato una classe di sklearn.preprocessing, quello che in genere succede é che viene ritornato un numpy array, questo mi é scomodo se devo fare altre trasformazioni perche alcune sono fatte apposta per essere applicate su colonne specifiche\n- Mantenimento di una struttura del codice pulita e poter capire a colpo d'occhio l'ordine delle operazioni eseguito","metadata":{}},{"cell_type":"markdown","source":"### Codifica colonne stagioni\n\ndevo codificare le colonne di tipo stringa, che secondo il dizionario sono le colonne delle stagioni \nle scelte sono LabelEncoder, OrdinalEncoder e OneHotEncoder. dato che non posso dare un ordine di importanza o di\nsomiglianza alle stagioni, preferisco trattare l'informazione in modo piu indipendente e quindi useró \nOneHotEncoder per creare 4 colonne separate","metadata":{}},{"cell_type":"code","source":"from sklearn.preprocessing import OneHotEncoder, StandardScaler\nfrom sklearn.compose import ColumnTransformer\n\n\n#creato un OneHotEncoder che agisce solo sulle colonne necessarie\nseason_columns = dictionary[\n    (dictionary[\"Field\"].str.contains(\"Season\")) &\n    (dictionary[\"Instrument\"] != \"Parent-Child Internet Addiction Test\") #PCIAT columns\n][\"Field\"]\n\n\nfrom sklearn.base import BaseEstimator, TransformerMixin\n\n#classe che implementa fit e transform senza perdere il pandas.DataFrame, cosi da poterlo mantenere in pipeline e poter usare le colonne\nclass DataFrameTransformer(\n    BaseEstimator, #per avere get_params()/set_params() \n):\n\n    def __init__(self, preprocessor, columns=None):\n        self.preprocessor = preprocessor\n        self.columns = columns\n        \n    def fit(self, X, y=None):\n        self.columns_ = X.columns if (self.columns is None) else self.columns\n        copy = pd.DataFrame(data=X, columns=self.columns_, index=X.index)\n        return self.preprocessor.fit(copy[self.columns_])\n\n    def transform(self, X):\n        np_array = self.preprocessor.transform(X[self.columns_])\n        new_df = pd.DataFrame(np_array, columns=self.preprocessor.get_feature_names_out(), index=X.index)\n        result =  pd.concat([X.drop(columns=self.columns_), new_df], axis=1)\n\n        return result\n\n    def fit_transform(self, X, y=None):\n        self.fit(X)\n        return self.transform(X)\n\n\nseason_encoder = DataFrameTransformer(\n    OneHotEncoder(sparse_output=False),\n    columns=season_columns.tolist()\n)\n\nnormalizer = DataFrameTransformer(\n    StandardScaler(),\n    columns=dictionary[dictionary[\"Type\"].isin([\"int\", \"float\"])][\"Field\"].tolist()\n)\n\n\nfrom sklearn.impute import SimpleImputer\n\nnan_filler = DataFrameTransformer(\n    SimpleImputer(strategy=\"median\")\n)\n\ncleaning_pipeline = Pipeline([\n    ('normalize', normalizer),\n    ('clear CGAS columns', FunctionTransformer(clear_CGAS_cols)),\n    ('drop PCIAT cols', FunctionTransformer(drop_PCIAT_cols)),\n    ('drop null label rows',FunctionTransformer(drop_null_label_rows)),\n    ('drop id col', FunctionTransformer(drop_id_column)),\n    ('merge_mins_sec_cols', FunctionTransformer(merge_min_sec_cols)),\n    ('convert to float', FunctionTransformer(to_float)),\n    ('encode seasons', season_encoder) ,\n    ('remove NaN', nan_filler)\n])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-22T07:57:32.318095Z","iopub.execute_input":"2025-09-22T07:57:32.318388Z","iopub.status.idle":"2025-09-22T07:57:32.339701Z","shell.execute_reply.started":"2025-09-22T07:57:32.318346Z","shell.execute_reply":"2025-09-22T07:57:32.338838Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Training del modello\n\nfacciamo uso dei dati che abbiamo appena trasformato, primissima cosa é creare un test set da far adottare ai modelli","metadata":{}},{"cell_type":"code","source":"\ntrain, test = train_test_split(dataset, train_size=0.8, random_state=0)\ntrain = cleaning_pipeline.fit_transform(train)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-22T07:57:32.340862Z","iopub.execute_input":"2025-09-22T07:57:32.341160Z","iopub.status.idle":"2025-09-22T07:57:32.425703Z","shell.execute_reply.started":"2025-09-22T07:57:32.341139Z","shell.execute_reply":"2025-09-22T07:57:32.425064Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Comparazione di modelli\n\nSceglieró due modelli, uno di sklearn e uno di una libreria esterna, e vedró qual é il piú adatto per questo task di classificazione\n\n### Primo modello:\nAlcuni modelli visti a lezione sono: \n* K-Nearest Neighbor\n* Decision Tree Classifier\n* Linear/logistic regression (secondo me non utilizzabile perché le istanze non sono legate da un concetto di tempo)\n* Support Vector Machines\n\nIo adotteró un Decision Tree Classifier","metadata":{}},{"cell_type":"code","source":"#inizio scelta e fine-tuning dei modelli\n\nvalid_X, train_X, valid_y, train_y  = train_test_split(train.drop(\"sii\", axis=1), train[\"sii\"], train_size=0.25, random_state=0)\nX = pd.concat([train_X, valid_X], axis=0)\ny = pd.concat([train_y, valid_y], axis=0)\n\ndef accuracy(y_true, y_pred):\n    matches=0\n    for t,p in zip(y_true, y_pred):\n        if (t == p):\n            matches+=1\n    return matches/len(y_true)\n\n\nprint(f'baseline accuracy: {(train[\"sii\"].value_counts().max())/train[\"sii\"].value_counts().sum():.3f}')\n\n\nfrom sklearn.tree import DecisionTreeClassifier\n\ndecision_tree_classifier = DecisionTreeClassifier()\ndecision_tree_classifier.fit(train_X, train_y)\npred_y = decision_tree_classifier.predict(valid_X)\nprint(f'accuracy: {accuracy(valid_y, pred_y):.3f}')\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-22T07:57:32.426481Z","iopub.execute_input":"2025-09-22T07:57:32.426693Z","iopub.status.idle":"2025-09-22T07:57:32.494272Z","shell.execute_reply.started":"2025-09-22T07:57:32.426677Z","shell.execute_reply":"2025-09-22T07:57:32.493474Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"L'accuracy del modello é anche piu bassa della baseline, ora provo a fare il tuning dei parametri e vedere se ottengo un risultato migliore","metadata":{}},{"cell_type":"code","source":"#modello 1:\n\nmax_depth=[3,5,7,10,15,20, 99]\nmin_samples_split=[32,16,8,4,2]\n\n\n\nfor d in max_depth:\n    for msp in min_samples_split:\n        decision_tree_classifier = DecisionTreeClassifier(\n            splitter='random', #per alleviare il bias\n            max_depth=d,\n            random_state=0,\n            min_samples_split=msp\n        )\n        decision_tree_classifier.fit(train_X, train_y)\n        pred_y = decision_tree_classifier.predict(valid_X)\n        print(f' max_depth: {d:>2}, min_samples_split: {msp:>2} accuracy: {accuracy(valid_y, pred_y):>.3f}, depth reached: {decision_tree_classifier.get_depth():>2}, n leaves: {decision_tree_classifier.get_n_leaves()}')  \n    print()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-22T07:57:32.495521Z","iopub.execute_input":"2025-09-22T07:57:32.495763Z","iopub.status.idle":"2025-09-22T07:57:32.982342Z","shell.execute_reply.started":"2025-09-22T07:57:32.495744Z","shell.execute_reply":"2025-09-22T07:57:32.981528Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Conclusioni\n\nsembra che il DecisionTreeClassifier non sia efficace per questo problema, l'accuracy supera appena la baseline, per giunta solo quando l'albero é piccolo,\nper cui sembra vada in overfitting piuttosto presto. Essendo poi cosí piccolo ha un alto bias e difficilmente indovina istanze diverse da quelle piu semplici","metadata":{}},{"cell_type":"markdown","source":"## Scelta del secondo modello\n\nper la scelta del secondo modello sono stati proposti Catboost,  LightGBM, e XGBoost.\n\nPer scegliere li proveró tutti e 3 senza tuning e adotteró il modello con l'accuracy migliore","metadata":{}},{"cell_type":"code","source":"from catboost import CatBoostClassifier\n\nimport xgboost as xgb\nfrom sklearn.metrics import accuracy_score\n\ncbc = CatBoostClassifier()\n\ncbc.fit(train_X, train_y)\ny_pred = cbc.predict(valid_X)\nprint(f'accuracy di CatBoostClassifier: {accuracy_score(valid_y, pred_y):>.3f}')  ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-22T07:57:32.983232Z","iopub.execute_input":"2025-09-22T07:57:32.983500Z","iopub.status.idle":"2025-09-22T07:57:45.114962Z","shell.execute_reply.started":"2025-09-22T07:57:32.983480Z","shell.execute_reply":"2025-09-22T07:57:45.114291Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from lightgbm import LGBMClassifier\n\nlgbmc = LGBMClassifier()\nlgbmc.fit(train_X, train_y)\ny_pred = lgbmc.predict(valid_X)\nprint(f'accuracy di LGBMClassifier: {accuracy_score(valid_y, pred_y):>.3f}')  ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-22T07:57:45.115660Z","iopub.execute_input":"2025-09-22T07:57:45.115894Z","iopub.status.idle":"2025-09-22T07:57:45.901260Z","shell.execute_reply.started":"2025-09-22T07:57:45.115876Z","shell.execute_reply":"2025-09-22T07:57:45.900498Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import xgboost as xgb\nfrom xgboost import XGBClassifier, plot_tree\n\nxgbmodel = XGBClassifier(random_state=0)\nxgbmodel.fit(train_X,train_y)\npred_y = xgbmodel.predict(valid_X)\nprint(f'accuracy: {accuracy_score(pred_y, valid_y)}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-22T07:57:45.902175Z","iopub.execute_input":"2025-09-22T07:57:45.902920Z","iopub.status.idle":"2025-09-22T07:57:46.624930Z","shell.execute_reply.started":"2025-09-22T07:57:45.902890Z","shell.execute_reply":"2025-09-22T07:57:46.624385Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"mi occuperó del tuning di un **XGBClassifier** dato che di base ha performato meglio degli altri","metadata":{}},{"cell_type":"markdown","source":"### Utilizzo del RandomForest\n\ndato che l'XGBClassifier usa un algoritmo Gradient Boosted Tree non sarebbe equo compararlo con un DecisionTreeClassifier, quindi adotteró un RandomForest al posto di un semplice DecisionTree.","metadata":{}},{"cell_type":"code","source":"from sklearn.ensemble import RandomForestClassifier as RandomForest\n\nn_estimator = [10,25,50,100,125]\nmax_features = ['log2', 'sqrt', 15, 50] # ~60 feature totali,  log2=6, sqrt=8\nmax_depth = [3,5,8,12,15,20]\n\nclass RandomForestConfig:\n    def __init__(self, n_estimators, max_depth, max_features, accuracy):\n        self.n_estimators = n_estimators\n        self.max_depth = max_depth\n        self.max_features = max_features\n        self.accuracy = accuracy\n        \n\nrf_config_list = []\n\nfor est in n_estimator:\n    for mf in max_features:\n        for md in max_depth:\n            model = RandomForest(\n                est,\n                max_features=mf,\n                max_depth=md,\n                random_state=0\n            )\n            model.fit(train_X, train_y)\n            pred_y = model.predict(valid_X)\n            acc = accuracy_score(pred_y, valid_y)\n            rf_config_list.append(RandomForestConfig(est, md, mf, accuracy=acc))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-22T07:57:46.625407Z","iopub.execute_input":"2025-09-22T07:57:46.625583Z","iopub.status.idle":"2025-09-22T07:58:34.322070Z","shell.execute_reply.started":"2025-09-22T07:57:46.625567Z","shell.execute_reply":"2025-09-22T07:58:34.321073Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"#migliori scelte di parametri\nrf_config_list.sort(key = lambda conf: -conf.accuracy)\nfor c in rf_config_list[:20]: \n    print(f'  accuracy: {c.accuracy:>.3f} n_estimators: {c.n_estimators:>3} max_depth: {c.max_depth if (c.max_depth is not None) else \"None\":>4}, max_features: {c.max_features if (c.max_features is not None) else \"None\":>4}') ","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-22T07:58:34.322953Z","iopub.execute_input":"2025-09-22T07:58:34.323187Z","iopub.status.idle":"2025-09-22T07:58:34.329341Z","shell.execute_reply.started":"2025-09-22T07:58:34.323168Z","shell.execute_reply":"2025-09-22T07:58:34.328422Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Conclusioni iniziali\n\ni parametri sono molto diversi... sembra che max_depth debba essere bassa\n\nIn questo caso un modello che predica la moda sembra essere piú indicato, perché evidentemente il task di classificazione é difficile. Per questo motivo i modelli con una bassa varianza (e quindi con max_depth bassa) risultano piu efficienti.","metadata":{}},{"cell_type":"code","source":"model1 = RandomForest(\n    100,\n    max_depth=8,\n    min_samples_leaf=10\n)\nmodel1.fit(train_X, train_y)\npred_y = model1.predict(valid_X)\n\nprint(f'accuracy: {accuracy_score(pred_y, valid_y):>.3f}')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-22T07:58:34.330303Z","iopub.execute_input":"2025-09-22T07:58:34.330576Z","iopub.status.idle":"2025-09-22T07:58:34.730290Z","shell.execute_reply.started":"2025-09-22T07:58:34.330558Z","shell.execute_reply":"2025-09-22T07:58:34.729343Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Train di XGBClassifier\ntestiamo un range di parametri in modo simile a come abbiamo fatto per il random forest","metadata":{}},{"cell_type":"code","source":"class XGBConfig:\n    def __init__(self, gamma, subsample, accuracy, max_depth):\n        self.gamma = gamma\n        self.subsample = subsample\n        self.accuracy = accuracy\n        self.max_depth = max_depth\n\n    def __repr__(self):\n        return f\"accuracy: {self.accuracy: <3.2f} gamma: {self.gamma: <2} subsample: {self.subsample: <2} max_depth: {self.max_depth: <2}\"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-22T07:58:34.731159Z","iopub.execute_input":"2025-09-22T07:58:34.731356Z","iopub.status.idle":"2025-09-22T07:58:34.736466Z","shell.execute_reply.started":"2025-09-22T07:58:34.731340Z","shell.execute_reply":"2025-09-22T07:58:34.735339Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"xgb_config_list = []\n\n#loss minima necessaria per eseguire uno split\ngamma = [0, 5, 10, 20]\n#quanta porzione di dataset prende ogni tree\nsubsample = [1, .66, .5]\n#max depth\nmax_depth = [3,5,8,12,15,20]\n\nfor g in gamma:\n    for s in subsample:\n        for md in max_depth:\n            model = XGBClassifier(\n                subsample=s,\n                gamma=g,\n                max_depth = md,\n                random_state=0\n            )\n            model.fit(train_X, train_y)\n            pred_y = model.predict(valid_X)\n            acc = accuracy_score(pred_y, valid_y)\n            xgb_config_list.append(XGBConfig(gamma=g, subsample=s, accuracy=acc, max_depth=md))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-22T07:58:34.737308Z","iopub.execute_input":"2025-09-22T07:58:34.737568Z","iopub.status.idle":"2025-09-22T07:59:03.311462Z","shell.execute_reply.started":"2025-09-22T07:58:34.737545Z","shell.execute_reply":"2025-09-22T07:59:03.310865Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def printXgbScores(XGBConfigArray):\n    pass\n\nxgb_config_list.sort(key= lambda el : el.accuracy, reverse=True)\nfor el in xgb_config_list:\n    print(el)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-22T07:59:03.312109Z","iopub.execute_input":"2025-09-22T07:59:03.312611Z","iopub.status.idle":"2025-09-22T07:59:03.318859Z","shell.execute_reply.started":"2025-09-22T07:59:03.312590Z","shell.execute_reply":"2025-09-22T07:59:03.317615Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# analisi\nQuesta volta la **max_depth** varia parecchio, data la scelta dei parametri e dei loro valori potrebbe non essere quindi un fattore determinante per il modello. possiamo invece notare che il valore **gamma** impostato tra 5 e 10 porta sempre dei buoni risultati e se guardiamo le configurazioni dalle ultime posizioni  possiamo vedere che gamma é valorizzato con  0 op 20, quindi il modello é fortemente influenzato da questo parametro e il suo valore ottimale si aggira tra i 5 e i 10. Per quanto riguarda **subsample** é difficile trovare una correlazione con la accuracy, in quanto i valori sono distribuiti equamente per le configurazioni basse e alte. L'univa cosa che possiamo notare é l'assenza di subsample=1 nelle prime posizioni.\n\n### conclusione\neviteró di usare max_depth come parametro in quanto apparentemente irrilevante, imposto *gamma = 8* e *subsample = 0.66*","metadata":{}},{"cell_type":"code","source":"model2 = XGBClassifier(\n    subsamples=0.66,\n    gamma=8\n)\n\nmodel2.fit(train_X, train_y)\npred_y2 = model2.predict(valid_X)\n\n\nprint(f\"\"\"\n    accuracy Random Forest: {accuracy_score(valid_y, pred_y):<3.3f}\n    accuracy XGB classifier: {accuracy_score(valid_y, pred_y2):<3.3f}\n\"\"\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-22T07:59:03.320487Z","iopub.execute_input":"2025-09-22T07:59:03.320882Z","iopub.status.idle":"2025-09-22T07:59:03.527474Z","shell.execute_reply.started":"2025-09-22T07:59:03.320861Z","shell.execute_reply":"2025-09-22T07:59:03.525610Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Feature importance\n\nAvvaliamoci dei metodi di clustering per analizzare l'importanza delle features","metadata":{}},{"cell_type":"markdown","source":"### uso di KMeans\n\nfaremo delle prove con il sample del testing per provare a capire quale numero di centri sia corretto adottare","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nfrom sklearn.cluster import KMeans\n\nMAX_CLUSTERS = 15\npreviousGain = 0\ngain = 0\ngains = []\nclusters_range = range(1, MAX_CLUSTERS + 1)\n\nfor c in clusters_range:\n    k_means = KMeans(\n        n_clusters=c,\n        max_iter=10,\n        init='k-means++',\n        random_state=0,\n        n_init='auto'\n    )\n    k_means.fit(train_X)\n    gain = previousGain - k_means.inertia_ if previousGain > 0 else k_means.inertia_\n    print(f\"clusters: {c:<2} SSE = {k_means.inertia_:<.2f} gain = {gain:<.2f}\")\n    gains.append( k_means.inertia_)\n    previousGain = k_means.inertia_\n\n# Plot gain curve\nplt.plot(clusters_range, gains, marker='o')\nplt.xlabel('Number of Clusters')\nplt.ylabel('Gain in SSE')\nplt.title('Gain Curve of KMeans Clustering')\nplt.grid(True)\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-22T07:59:03.528666Z","iopub.execute_input":"2025-09-22T07:59:03.528916Z","iopub.status.idle":"2025-09-22T07:59:03.858375Z","shell.execute_reply.started":"2025-09-22T07:59:03.528898Z","shell.execute_reply":"2025-09-22T07:59:03.857567Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"MAX_CLUSTERS = 15\npreviousSSE = 0\n\nfor c in range(1, MAX_CLUSTERS+1):\n    k_means = KMeans(\n        n_clusters=c,\n        max_iter=10,\n        init='random',\n        random_state=0,\n        n_init='auto'\n    )\n    k_means.fit(train_X)\n    print(f\"clusters: {c}, SSE = {k_means.inertia_:<.2f}, gain = {previousSSE - k_means.inertia_:<.2f}\")\n    previousSSE = k_means.inertia_","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-22T07:59:03.859237Z","iopub.execute_input":"2025-09-22T07:59:03.859505Z","iopub.status.idle":"2025-09-22T07:59:04.651966Z","shell.execute_reply.started":"2025-09-22T07:59:03.859486Z","shell.execute_reply":"2025-09-22T07:59:04.651346Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Centri\nCon un numero di centri fino al 6 abbiamo ottenuto un buon miglioramento rispetto alla volta precedente, guardando i valori numerici, c'é stato un certo miglioramento anche con l'uso di 9 centri ma é probabilmente dovuto a un caso dato che il clustering con 8 centri ha prodotto pochi risultati. \n\nNon é mostrato in codice, ma per capire se eravamo finiti in un caso fortunato/sfortunato sono stati fatti altri tentativi modificando **random_state** e molti di essi hanno portato un buon miglioramento fino ai 6 centri. Come avevamo previsto, il gain dato dalla configurazione con 9 centri non era costante, ma dovuto solo a un caso.","metadata":{}},{"cell_type":"code","source":"def print_clustering_stats(labels):\n    value, count = np.unique(labels, return_counts=True)\n    for i in value:\n        print(f\"value: {value[i]} instances: {count[i]}\")\n\n\ndef do_kmeans(n_clusters = 1):\n    k_means = KMeans(\n        n_clusters=n_clusters,\n        max_iter=10,\n        init='k-means++',\n        random_state=0,\n        n_init='auto'\n    )\n    \n    k_means.fit(train_X)\n    train_X_with_cluster = train_X.copy()\n    train_X_with_cluster['cluster'] = k_means.labels_\n    print_clustering_stats(k_means.labels_)\n    \n    return train_X_with_cluster","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-22T07:59:04.652782Z","iopub.execute_input":"2025-09-22T07:59:04.653014Z","iopub.status.idle":"2025-09-22T07:59:04.658960Z","shell.execute_reply.started":"2025-09-22T07:59:04.652994Z","shell.execute_reply":"2025-09-22T07:59:04.658433Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"do_kmeans(6)\nprint() #evita di printare il dataset ritornato da do_kmeans","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-22T07:59:04.659950Z","iopub.execute_input":"2025-09-22T07:59:04.660320Z","iopub.status.idle":"2025-09-22T07:59:04.686574Z","shell.execute_reply.started":"2025-09-22T07:59:04.660301Z","shell.execute_reply":"2025-09-22T07:59:04.685920Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Controllando il clustering in sé senza l'ausilio dell'SSE vediamo che le istanze vengono distribuite in modo assai diseguale, penso quindi che il numero di centri piu corretto sia minore","metadata":{}},{"cell_type":"code","source":"do_kmeans(4)\nprint()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-22T07:59:04.687618Z","iopub.execute_input":"2025-09-22T07:59:04.687839Z","iopub.status.idle":"2025-09-22T07:59:04.702910Z","shell.execute_reply.started":"2025-09-22T07:59:04.687822Z","shell.execute_reply":"2025-09-22T07:59:04.702319Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## conclusioni su kmeans\n\nDato che otteniamo sempre un clustering all'apparenza molto sbilanciato proviamo ad adottare delle tecniche differenti, ci avvaliamo ora dell' **Agglomerative clustering**","metadata":{}},{"cell_type":"code","source":"from sklearn.cluster import AgglomerativeClustering\n\nlinkage_types = ['ward', 'complete', 'average', 'single']\n\nfor l in linkage_types:\n    clustering = AgglomerativeClustering(\n        n_clusters=4,\n        linkage=l,\n        memory='/kaggle/tmp'\n    )\n    clustering.fit(X)\n\n    print(f\"\\nlinkage: {l}\")\n    print_clustering_stats(clustering.labels_)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-22T07:59:04.703352Z","iopub.execute_input":"2025-09-22T07:59:04.703659Z","iopub.status.idle":"2025-09-22T07:59:04.755877Z","shell.execute_reply.started":"2025-09-22T07:59:04.703640Z","shell.execute_reply":"2025-09-22T07:59:04.755014Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Osservazioni\n\nqualunque sia la misura di **n_clusters** il clustering é sproporzionato, questo penso sia dovuto all'esistenza di outliers che compromettono l'abilitá dell'Agglomerative Clustering di suddividere bene il dataset\n","metadata":{}},{"cell_type":"markdown","source":"## DBScan\nutilizziamo **dbscan** dato che é un metodo di clustering piú resistente agli outliers e proviamo a identificarli","metadata":{}},{"cell_type":"code","source":"from sklearn.cluster import DBSCAN\n\ndbscan = DBSCAN(\n    eps=15,\n    min_samples = 30\n)\ndbscan.fit(X)\n\nprint_clustering_stats(dbscan.labels_)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-22T07:59:04.756833Z","iopub.execute_input":"2025-09-22T07:59:04.757087Z","iopub.status.idle":"2025-09-22T07:59:04.839490Z","shell.execute_reply.started":"2025-09-22T07:59:04.757061Z","shell.execute_reply":"2025-09-22T07:59:04.838688Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### exploiting del Random Forest\n\nutilizziamo la misura della similarity del random forest come appoggio per un analisi con dbscan","metadata":{}},{"cell_type":"code","source":"from sklearn.ensemble import RandomForestClassifier\n\nrf = RandomForestClassifier(n_estimators=50)\nrf.fit(X,y)\n\nfrom sklearn.metrics import pairwise_distances\n\n# pairwise distance\nleaves = rf.apply(X)\npair_wise_dist = pairwise_distances( \n    X=leaves, \n    metric= lambda oi, oj: 1.0-np.mean(oi==oj)\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-22T07:59:04.840688Z","iopub.execute_input":"2025-09-22T07:59:04.840897Z","iopub.status.idle":"2025-09-22T07:59:22.087026Z","shell.execute_reply.started":"2025-09-22T07:59:04.840873Z","shell.execute_reply":"2025-09-22T07:59:22.086266Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dbscan.fit(pair_wise_dist)\nprint_clustering_stats(dbscan.labels_)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-22T07:59:22.087991Z","iopub.execute_input":"2025-09-22T07:59:22.088297Z","iopub.status.idle":"2025-09-22T07:59:22.574132Z","shell.execute_reply.started":"2025-09-22T07:59:22.088271Z","shell.execute_reply":"2025-09-22T07:59:22.573387Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dbscan2 = DBSCAN(\n    eps=1,\n    min_samples = 30\n)\n\ndbscan2.fit(pair_wise_dist)\nprint_clustering_stats(dbscan2.labels_)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-22T07:59:22.574892Z","iopub.execute_input":"2025-09-22T07:59:22.575211Z","iopub.status.idle":"2025-09-22T07:59:22.956568Z","shell.execute_reply.started":"2025-09-22T07:59:22.575186Z","shell.execute_reply":"2025-09-22T07:59:22.955924Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### trovare il giusto esp:\n\nho fatto un po di prove con i valori di esp ed é difficile trovare un valore che dia origine a un buon clustering, se esp é troppo piccolo allora ci sará un'eccessiva classificazione di outliers, se esp é troppo grande ci troveremo ci troveremo con un unico cluster di tutti i punti del dataset\n\nInvece di provare valori a caso definiamo un algoritmo che trovi un valore iniziale su cui possiamo fare affidamento","metadata":{}},{"cell_type":"code","source":"def find_decent_esp(dataset, min_samples, logging=True, custom_esp_max=None):\n\n    esp_min = 0\n    esp_max = custom_esp_max if custom_esp_max != None else 100 #custom_esp_max aggiunto perche se un esp decente fosse >100 l'algoritmo non sarebbe accurato\n    esp = esp_max\n    \n    test_dbscan = DBSCAN(\n            eps=esp,\n            min_samples = min_samples\n    )\n    test_dbscan.fit(dataset)\n\n    iterationsCount = 0\n    \n    while(not esp_is_decent(test_dbscan)):\n\n        iterationsCount+=1\n        if (iterationsCount > 20):\n            print(\"no decent esp value found\")\n            return 0\n        \n        esp = (esp_max+esp_min)/2\n        \n        test_dbscan = DBSCAN(\n            eps=esp,\n            min_samples = min_samples\n        )\n        test_dbscan.fit(dataset)\n\n        if (esp_is_too_big(test_dbscan)):\n            esp_max = esp\n\n        if (esp_is_too_small(test_dbscan)):\n            esp_min = esp     \n\n        if logging:\n            print(f\"[find_decent_esp] - current esp: {esp}\")\n        \n    return esp\n\n\ndef esp_is_too_big(clustering_model):\n    labels = clustering_model.labels_\n\n    n_outliers = np.sum(labels == -1)\n    n_instances_first_cluster = np.sum(labels == 0)\n    n_instances_other_clusters = np.sum(labels > 0)\n\n    return n_instances_other_clusters == 0 and n_instances_first_cluster > n_outliers\n\n\ndef esp_is_too_small(clustering_model):\n    labels = clustering_model.labels_\n\n    n_outliers = np.sum(labels == -1)\n    n_instances_clusters = np.sum(labels != -1)\n\n    return n_outliers > n_instances_clusters\n\ndef esp_is_decent(clustering_model):\n    labels = clustering_model.labels_\n\n    n_outliers = np.sum(labels == -1)\n    n_instances_first_cluster = np.sum(labels == 0)\n    n_instances_other_clusters = np.sum(labels > 0)\n\n    return n_instances_other_clusters > 1","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-22T07:59:22.963256Z","iopub.execute_input":"2025-09-22T07:59:22.964438Z","iopub.status.idle":"2025-09-22T07:59:22.972912Z","shell.execute_reply.started":"2025-09-22T07:59:22.964414Z","shell.execute_reply":"2025-09-22T07:59:22.972325Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(find_decent_esp(X, 30))","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-22T07:59:22.973802Z","iopub.execute_input":"2025-09-22T07:59:22.974216Z","iopub.status.idle":"2025-09-22T07:59:23.352517Z","shell.execute_reply.started":"2025-09-22T07:59:22.974173Z","shell.execute_reply":"2025-09-22T07:59:23.351765Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Migliorare l'algoritmo\n\npossiamo spingerci ancora piu in la e usare i criteri di misura dell'esp sopra definiti per definire un range di valori che ha senso usare per esp","metadata":{}},{"cell_type":"code","source":"def find_esp_range(dataset, min_samples, step_size=0.05, logging=True):\n    decent_esp = find_decent_esp(dataset, min_samples, logging=logging)\n\n    step = decent_esp*step_size\n    \n    #test limite superiore\n    test_dbscan = DBSCAN(\n        eps=decent_esp,\n        min_samples = min_samples\n    )\n    test_dbscan.fit(dataset)\n    \n    max_decent_esp = decent_esp\n    while(not esp_is_too_big(test_dbscan)):\n\n        test_dbscan = DBSCAN(\n            eps=max_decent_esp+step,\n            min_samples = min_samples\n        )        \n        test_dbscan.fit(dataset)\n\n        if not esp_is_too_big(test_dbscan):\n            max_decent_esp+=step\n\n        if logging:\n            print(f\"[find_esp_range] - current max: {max_decent_esp}\")\n    \n    #test limite inferiore\n    test_dbscan = DBSCAN(\n        eps=decent_esp,\n        min_samples = min_samples\n    )\n    test_dbscan.fit(dataset)\n    \n    min_decent_esp = decent_esp\n    while(not esp_is_too_small(test_dbscan)):\n\n        test_dbscan = DBSCAN(\n            eps=min_decent_esp-step,\n            min_samples = min_samples\n        )        \n        test_dbscan.fit(dataset)\n\n        if not esp_is_too_small(test_dbscan):\n            min_decent_esp-=step\n\n        if logging:\n            print(f\"[find_esp_range] - current min: {min_decent_esp}\")\n    \n    return min_decent_esp, max_decent_esp","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-22T07:59:23.353350Z","iopub.execute_input":"2025-09-22T07:59:23.353615Z","iopub.status.idle":"2025-09-22T07:59:23.360100Z","shell.execute_reply.started":"2025-09-22T07:59:23.353596Z","shell.execute_reply":"2025-09-22T07:59:23.359458Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"find_esp_range(X, 30, step_size=0.05)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-22T07:59:23.360874Z","iopub.execute_input":"2025-09-22T07:59:23.361076Z","iopub.status.idle":"2025-09-22T07:59:26.240927Z","shell.execute_reply.started":"2025-09-22T07:59:23.361059Z","shell.execute_reply":"2025-09-22T07:59:26.240379Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Ora possiamo dedicarci ad analizzare i risultati del dbscan sul nostro dataset e sulla similarity del random forest","metadata":{}},{"cell_type":"code","source":"#nota: ho avuto problemi a usare il dbscan per pair_wise_dist, find_decent_esp potrebbe non riuscire a trovare un parametro, dipende dalla randomness del random forest. \nfind_esp_range(pair_wise_dist, 30)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-22T07:59:26.241390Z","iopub.execute_input":"2025-09-22T07:59:26.241569Z","iopub.status.idle":"2025-09-22T07:59:32.610076Z","shell.execute_reply.started":"2025-09-22T07:59:26.241554Z","shell.execute_reply":"2025-09-22T07:59:32.609469Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.metrics import silhouette_score\n\ndbscan = DBSCAN(\n    eps = 1.46,\n    min_samples = 20\n)\ndbscan.fit(pair_wise_dist)\nprint_clustering_stats(dbscan.labels_)\n\nlabels = dbscan.labels_ != -1\nX_no_noise = X[labels]\nlabels_no_noise = dbscan.labels_[labels]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-22T07:59:32.611068Z","iopub.execute_input":"2025-09-22T07:59:32.611264Z","iopub.status.idle":"2025-09-22T07:59:33.029069Z","shell.execute_reply.started":"2025-09-22T07:59:32.611248Z","shell.execute_reply":"2025-09-22T07:59:33.028388Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"dbscan = DBSCAN(\n    eps = 16,\n    min_samples = 30\n)\ndbscan.fit(X)\nprint_clustering_stats(dbscan.labels_)\n\nlabels = dbscan.labels_ != -1\nX_no_noise = X[labels]\nlabels_no_noise = dbscan.labels_[labels]\nprint(f\"Silhouette Score: { silhouette_score(X_no_noise, labels_no_noise):.3f}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-22T07:59:33.029645Z","iopub.execute_input":"2025-09-22T07:59:33.029877Z","iopub.status.idle":"2025-09-22T07:59:33.207155Z","shell.execute_reply.started":"2025-09-22T07:59:33.029858Z","shell.execute_reply":"2025-09-22T07:59:33.206238Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"print(y.value_counts())","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-22T07:59:33.208109Z","iopub.execute_input":"2025-09-22T07:59:33.208391Z","iopub.status.idle":"2025-09-22T07:59:33.214424Z","shell.execute_reply.started":"2025-09-22T07:59:33.208370Z","shell.execute_reply":"2025-09-22T07:59:33.213787Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"**Osservazione:**\nil dataset é parecchio sbilanciato e si finisce sempre con l'ottenere un cluster grande e altri cluster molto piu piccoli o outliers.","metadata":{}},{"cell_type":"markdown","source":"### Conclusioni\n\nIl dataset fornito non permette una distribuzione abbastanza omogenea dei dati da rendere utile o quantomeno informativa la pratica del clustering.","metadata":{}},{"cell_type":"markdown","source":"# Feature importance tramite random forest\n\nUtilizziamo il random forest precedentemente allenato per scoprire quali feature influiscono di piú e quali di meno nelle scelte dell'algoritmo","metadata":{}},{"cell_type":"code","source":"features_with_score = []\n\nscores = rf.feature_importances_\ncols = train_X.columns\nfor i in range(len(cols)):\n    features_with_score.append((cols[i], scores[i]))\n\nfeatures_with_score.sort(key=lambda it : it[1], reverse=True)\nprint(f\"feature importance of [{len(features_with_score)}] features:\\n\")\nfor item in features_with_score:\n    print(f\"{item[0]}, score: {item[1]}\")\n    \n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-22T07:59:33.215208Z","iopub.execute_input":"2025-09-22T07:59:33.215416Z","iopub.status.idle":"2025-09-22T07:59:33.235507Z","shell.execute_reply.started":"2025-09-22T07:59:33.215391Z","shell.execute_reply":"2025-09-22T07:59:33.234753Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"feature_names = [f[0] for f in features_with_score]\nfeature_scores = [f[1] for f in features_with_score]\n\nplt.figure(figsize=(10, 20))\nplt.barh(feature_names, feature_scores)\nplt.xlabel(\"Importance Score\")\nplt.title(f\"Feature Importance of {len(feature_names)} Features\")\nplt.gca().invert_yaxis()\nplt.grid(axis='x')\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-22T07:59:33.236364Z","iopub.execute_input":"2025-09-22T07:59:33.236641Z","iopub.status.idle":"2025-09-22T07:59:34.112281Z","shell.execute_reply.started":"2025-09-22T07:59:33.236599Z","shell.execute_reply":"2025-09-22T07:59:34.111541Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Rimuoviamo tutte le feature sotto lo 0.01 di importance","metadata":{}},{"cell_type":"code","source":"important_features = [f for f in features_with_score if f[1] > 0.01]\nimportant_features.sort()\nimportant_cols = [f[0] for f in important_features]\n\nN_IMPORTANT_FEATURES = len(important_features)\n\nX[important_cols]\n\nrf.fit(train_X[important_cols], train_y)\nnew_pred_y = rf.predict(valid_X[important_cols])\n\naccuracy_score(new_pred_y, valid_y)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-22T07:59:34.113031Z","iopub.execute_input":"2025-09-22T07:59:34.113254Z","iopub.status.idle":"2025-09-22T07:59:34.408498Z","shell.execute_reply.started":"2025-09-22T07:59:34.113237Z","shell.execute_reply":"2025-09-22T07:59:34.407665Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# print(tuned_model.predict_proba(train_X))\nfrom sklearn.metrics import ConfusionMatrixDisplay\nConfusionMatrixDisplay.from_estimator(\n    estimator = model1,\n    X=valid_X, y=valid_y\n)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-22T07:59:34.409393Z","iopub.execute_input":"2025-09-22T07:59:34.409797Z","iopub.status.idle":"2025-09-22T07:59:34.667517Z","shell.execute_reply.started":"2025-09-22T07:59:34.409772Z","shell.execute_reply":"2025-09-22T07:59:34.666795Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Istanze sottovalutate\nda questa matrice posso vedere che la maggior parte degli errori avviene perche le istanze vengono \"sottovalutate\" o classificate tendenzialmente con label = 0\nSarebbe interessante capire quale obbiettivo si ha in mente, se avere un accuracy piu alta possibile anche se con un approccio piú conservativo quindi con il metodo attuale, o se cercare di trovare i casi di internet addiction nonostante si vada a classificare erroneamente dei casi normali","metadata":{}},{"cell_type":"code","source":"class Prediction_Info:\n    def __init__(self, idx, predicted_value, predicted_prob, correct_value, correct_prob):\n        self.idx = idx\n        self.predicted_value = predicted_value\n        self.predicted_prob = predicted_prob\n        self.correct_value = correct_value\n        self.correct_prob = correct_prob\n        self.is_correct = predicted_value == correct_value\n\n    def __repr__(self):\n        return f\"idx: {self.idx:<4.0f} | value predicted: {self.predicted_value:<1.0f} | predicted prob: {self.predicted_prob:<1.2f} | correct value: {self.correct_value:<1.0f} | correct prob: {self.correct_prob:<1.2f} | {'RIGHT' if self.is_correct else 'WRONG'}\"\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-22T07:59:34.668357Z","iopub.execute_input":"2025-09-22T07:59:34.668604Z","iopub.status.idle":"2025-09-22T07:59:34.674180Z","shell.execute_reply.started":"2025-09-22T07:59:34.668587Z","shell.execute_reply":"2025-09-22T07:59:34.673566Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"from sklearn.model_selection import StratifiedKFold\n\nskf = StratifiedKFold(n_splits=5, shuffle=True, random_state=0)\n    \npredictions = []\nfor train_idx, valid_idx in skf.split(X, y):\n    \n    rf.fit(X.iloc[train_idx], y.iloc[train_idx])\n    preds = rf.predict(X.iloc[valid_idx])\n    probs = rf.predict_proba(X.iloc[valid_idx])\n\n    for i, idx in enumerate(valid_idx):\n        prediction = Prediction_Info(\n            idx,\n            preds[i],\n            probs[i].max(),\n            y.iloc[idx],\n            probs[i, int(y.iloc[idx])]\n        )\n        predictions.append(prediction)\n\npredictions.sort(key=lambda item:item.idx)\n\n\nfor p in predictions[:5]:\n    print(p)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-22T07:59:34.674936Z","iopub.execute_input":"2025-09-22T07:59:34.675247Z","iopub.status.idle":"2025-09-22T07:59:36.086405Z","shell.execute_reply.started":"2025-09-22T07:59:34.675221Z","shell.execute_reply":"2025-09-22T07:59:36.085668Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"right_pred = [p for p in predictions if p.is_correct]\nwrong_pred = [p for p in predictions if not p.is_correct]","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-22T07:59:36.087124Z","iopub.execute_input":"2025-09-22T07:59:36.087323Z","iopub.status.idle":"2025-09-22T07:59:36.092085Z","shell.execute_reply.started":"2025-09-22T07:59:36.087307Z","shell.execute_reply":"2025-09-22T07:59:36.091197Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Analisi predizioni piu imprecise\n\nPrendiamo le feature piú impattanti per i casi piu imprecisi e vediamo in che scala sono nell'importanza delle feature.","metadata":{}},{"cell_type":"code","source":"importan_cols_set = set(important_cols)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-22T07:59:36.093093Z","iopub.execute_input":"2025-09-22T07:59:36.093491Z","iopub.status.idle":"2025-09-22T07:59:36.107588Z","shell.execute_reply.started":"2025-09-22T07:59:36.093447Z","shell.execute_reply":"2025-09-22T07:59:36.106830Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"class FeaturesVarianceRank:\n    def __init__(self, name, rank, localRank):\n        self.name = name\n        self.rank = rank\n        self.localRank = localRank\n\n    def __repr__(self):\n        return f\"nome: {self.name:<30} | rank (global): {self.rank:<3} | rank (local): {self.localRank:<3}\"\n\n#sortate per importanza decrescente\nfeatures_with_score.sort(key=lambda it : it[1], reverse=True)\n\nranked_features = {\n    feature: FeaturesVarianceRank(feature, i + 1, None)\n    for i, (feature, score) in enumerate(features_with_score)\n}\n\nwrong_pred.sort(key=lambda el: el.correct_prob)\n\n#estraiamo le 50 istanze peggiori e vediamo come si comportano le features\n\nwrong_idx = [el.idx for el in wrong_pred[:50]]\nfeature_variances = X.iloc[wrong_idx].var()\n\nsorted_features = feature_variances.sort_values(ascending=False)\n\nfor rank, feature_name in enumerate(sorted_features.index, start=1):\n    ranked_features[feature_name].localRank = rank","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-22T07:59:36.108280Z","iopub.execute_input":"2025-09-22T07:59:36.108521Z","iopub.status.idle":"2025-09-22T07:59:36.125516Z","shell.execute_reply.started":"2025-09-22T07:59:36.108497Z","shell.execute_reply":"2025-09-22T07:59:36.124805Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"l = list(ranked_features.values())\nl = [el for el in l if el.name in importan_cols_set]\nl.sort(key = lambda el: el.rank, reverse=False)\nl","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-22T07:59:36.126303Z","iopub.execute_input":"2025-09-22T07:59:36.126591Z","iopub.status.idle":"2025-09-22T07:59:36.174069Z","shell.execute_reply.started":"2025-09-22T07:59:36.126564Z","shell.execute_reply":"2025-09-22T07:59:36.173158Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"l2 =[(el.name, el.rank-el.localRank) for el in l]\nl2.sort(key = lambda el: el[1], reverse=True)\nfor el in l2:\n    string = \"variance increased by\" if el[1]>=0 else \"variance decreased by\"\n    print(f\"nome: {el[0]:<35} {string}: {el[1]}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-22T07:59:36.174996Z","iopub.execute_input":"2025-09-22T07:59:36.175321Z","iopub.status.idle":"2025-09-22T07:59:36.189093Z","shell.execute_reply.started":"2025-09-22T07:59:36.175292Z","shell.execute_reply":"2025-09-22T07:59:36.188198Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Struttura dati:\n\nper rappresentare i dati ho usato un oggetto **FeaturesVarianceRank** che esprime:\n- **nome**: il nome della colonna\n- **rank**: l'importanza della colonna utilizzando la varianza di tutto il dataset\n- **variance**: la varianza, in questo caso é stata calcolata solo guardando alle 50 feature peggiori\n\n## Conclusioni\n\noltre ad aver scoperto che **Fitness_Endurance-Time_Sec** non era stata normalizzata (1793 di varianza), ci accorgiamo che le feature del test **BIA** hanno un aumento di varianza spropositato quando si va ad osservare le feature con predizioni peggiori","metadata":{}},{"cell_type":"markdown","source":"Possiamo eseguire lo stesso tipo di analisi con le feature con le migliori predizioni","metadata":{}},{"cell_type":"code","source":"#sortate per importanza decrescente\nfeatures_with_score.sort(key=lambda it : it[1], reverse=True)\n\nranked_features = {\n    feature: FeaturesVarianceRank(feature, i + 1, None)\n    for i, (feature, score) in enumerate(features_with_score)\n}\n\nright_pred.sort(key=lambda el: el.predicted_prob, reverse=True)\n\nright_idx = [el.idx for el in right_pred[:100]]\nfeature_variances = X.iloc[right_idx].var()\n\nsorted_features = feature_variances.sort_values(ascending=False)\n\nfor rank, feature_name in enumerate(sorted_features.index, start=1):\n    ranked_features[feature_name].localRank = rank","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-22T07:59:36.190052Z","iopub.execute_input":"2025-09-22T07:59:36.190331Z","iopub.status.idle":"2025-09-22T07:59:36.206582Z","shell.execute_reply.started":"2025-09-22T07:59:36.190304Z","shell.execute_reply":"2025-09-22T07:59:36.205966Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"l = list(ranked_features.values())\nl = [el for el in l if el.name in importan_cols_set]\nl.sort(key = lambda el: el.rank, reverse=False)\nl","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-22T07:59:36.207464Z","iopub.execute_input":"2025-09-22T07:59:36.207675Z","iopub.status.idle":"2025-09-22T07:59:36.223215Z","shell.execute_reply.started":"2025-09-22T07:59:36.207654Z","shell.execute_reply":"2025-09-22T07:59:36.222484Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"l2 =[(el.name, el.rank-el.localRank) for el in l]\nl2.sort(key = lambda el: el[1], reverse=True)\nfor el in l2:\n    string = \"variance increased by\" if el[1]>=0 else \"variance decreased by\"\n    print(f\"nome: {el[0]:<35} {string}: {el[1]}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-09-22T07:59:36.224034Z","iopub.execute_input":"2025-09-22T07:59:36.224262Z","iopub.status.idle":"2025-09-22T07:59:36.242992Z","shell.execute_reply.started":"2025-09-22T07:59:36.224241Z","shell.execute_reply":"2025-09-22T07:59:36.242345Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"## Conclusioni\n\nAnche qua possiamo notare che le colonne **BIA** hanno una varianza piuttosto diversa per le istanze migliori. Questo suggerisce che il modello si affida parecchio a quella feature, potrebbe essere un'idea diminuire la sua importanza e vedere su quali altri dati il modello faccia affidamento","metadata":{}}]}