{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Another summer out of office notebook with TPOT","metadata":{}},{"cell_type":"markdown","source":"<div align=\"center\"><img src=\"https://raw.githubusercontent.com/EpistasisLab/tpot/master/images/tpot-logo.jpg\" alt=\"drawing\" width=\"300\"/></div>\nTPOT is a Python Automated Machine Learning tool that optimizes machine learning pipelines using genetic programming.\n\nYou can find more about it <a href=\"https://epistasislab.github.io/tpot/\">here</a>.\n\nLet's see how to use it in this playground to have the machines do most of the work while we enjoy the summer time !\n\n<u>*Side note:*</u> \n\n*TPOT use genetic programming to find the best model. It takes time as it assess a population of solutions, then add some new solutions or mix solutions together to try to have better solutions at each generations. I've seen people having TPOT running for a couple of days.*\n\n*So My first intent was to run the notebook for about the maximum before notebook timeout. Then serialize and save the TPOT model in a dataset in order to run the notebook again. I could by this process resume the solutions search process where it was.*\n\n*Unfortunately, TPOT models can't be serialized for later re-use as far as I found. So I can't re-run the notebook over and over to find better solution.*\n\n*See the feature request for this that is still open since about 2019 <a href=\"https://github.com/EpistasisLab/tpot/issues/977\">TPOT GitHub Repo - issue #977</a>*","metadata":{}},{"cell_type":"code","source":"# first install the TPOT package\n! pip install tpot","metadata":{"execution":{"iopub.status.busy":"2022-08-05T08:14:44.409510Z","iopub.execute_input":"2022-08-05T08:14:44.410106Z","iopub.status.idle":"2022-08-05T08:14:56.942664Z","shell.execute_reply.started":"2022-08-05T08:14:44.409987Z","shell.execute_reply":"2022-08-05T08:14:56.941447Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# let's import the needed pacakages\n\nimport numpy as np \nimport pandas as pd\nimport os\n\nimport tpot","metadata":{"execution":{"iopub.status.busy":"2022-08-05T08:14:56.944663Z","iopub.execute_input":"2022-08-05T08:14:56.945032Z","iopub.status.idle":"2022-08-05T08:15:00.437771Z","shell.execute_reply.started":"2022-08-05T08:14:56.944990Z","shell.execute_reply":"2022-08-05T08:15:00.436414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# and now load the data\n\nroot = os.path.join(\"..\",\"input\",\"tabular-playground-series-aug-2022\")\ndf_train = pd.read_csv(os.path.join(root, \"train.csv\"), index_col=0)\ndf_test = pd.read_csv(os.path.join(root, \"test.csv\"), index_col=0)\n\ndf_all = pd.concat([df_train.copy(), df_test.copy()])","metadata":{"execution":{"iopub.status.busy":"2022-08-05T08:15:00.439549Z","iopub.execute_input":"2022-08-05T08:15:00.440822Z","iopub.status.idle":"2022-08-05T08:15:00.736691Z","shell.execute_reply.started":"2022-08-05T08:15:00.440767Z","shell.execute_reply":"2022-08-05T08:15:00.735454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# for TPOT, we need to give only numerical features\n# so we will now encode text features\n\nfrom sklearn.preprocessing import LabelEncoder\n\ndef create_encoders(df, col):\n    \n    encoder = LabelEncoder()\n    encoder.fit(df[col].values)\n    \n    df[col] = encoder.transform(df[col].values)\n    \n    return encoder, df\n\ndef apply_encoders(encoders, df):\n    for col in encoders.keys():\n        if col in df.columns:\n            df[col] = encoders[col].transform(df[col].values)\n\n    return df\n\n# first we create the encoders with all the data\nencoders = {}\ndf = df_all.copy()\nd_unique = df_all.nunique().to_dict()\nfor k in d_unique.keys():\n    if df_all[k].dtype==\"object\":\n        encoders[k], df = create_encoders(df.copy(), k)\n        \n# then we apply them on test and train datasets\ndf_test = apply_encoders(encoders, df_test.copy())    \ndf_train = apply_encoders(encoders, df_train.copy())        ","metadata":{"execution":{"iopub.status.busy":"2022-08-05T08:15:00.739946Z","iopub.execute_input":"2022-08-05T08:15:00.740482Z","iopub.status.idle":"2022-08-05T08:15:00.890249Z","shell.execute_reply.started":"2022-08-05T08:15:00.740427Z","shell.execute_reply":"2022-08-05T08:15:00.888837Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# we will specify to TPOT the models to investigate\n# primarily, to avoid models that do not provide the \"predict_proba\" method \n# but also to refine the base models settings\n\nclassifier_config_dict = {\n\n    # Classifiers\n    'sklearn.naive_bayes.GaussianNB': {\n    },\n\n    'sklearn.naive_bayes.BernoulliNB': {\n        'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.],\n        'fit_prior': [True, False]\n    },\n\n    'sklearn.naive_bayes.MultinomialNB': {\n        'alpha': [1e-3, 1e-2, 1e-1, 1., 10., 100.],\n        'fit_prior': [True, False]\n    },\n\n    'sklearn.tree.DecisionTreeClassifier': {\n        'criterion': [\"gini\", \"entropy\"],\n        'max_depth': range(1, 11),\n        'min_samples_split': range(2, 21),\n        'min_samples_leaf': range(1, 21)\n    },\n\n    'sklearn.ensemble.ExtraTreesClassifier': {\n        'n_estimators': [50, 100, 200],\n        'criterion': [\"gini\", \"entropy\"],\n        'max_features': np.arange(0.05, 1.01, 0.05),\n        'min_samples_split': range(2, 21),\n        'min_samples_leaf': range(1, 21),\n        'bootstrap': [True, False]\n    },\n\n    'sklearn.ensemble.RandomForestClassifier': {\n        'n_estimators': [50, 100, 200],\n        'criterion': [\"gini\", \"entropy\"],\n        'max_features': np.arange(0.05, 1.01, 0.05),\n        'min_samples_split': range(2, 21),\n        'min_samples_leaf':  range(1, 21),\n        'bootstrap': [True, False]\n    },\n\n    'sklearn.ensemble.GradientBoostingClassifier': {\n        'n_estimators': [50, 100, 200],\n        'learning_rate': [1e-3, 1e-2, 1e-1, 0.5, 1.],\n        'max_depth': range(1, 11),\n        'min_samples_split': range(2, 21),\n        'min_samples_leaf': range(1, 21),\n        'subsample': np.arange(0.05, 1.01, 0.05),\n        'max_features': np.arange(0.05, 1.01, 0.05)\n    },\n\n    'sklearn.neighbors.KNeighborsClassifier': {\n        'n_neighbors': range(1, 101),\n        'weights': [\"uniform\", \"distance\"],\n        'p': [1, 2]\n    },\n\n# no predict_proba() method available for this model\n#     'sklearn.svm.LinearSVC': {\n#         'penalty': [\"l1\", \"l2\"],\n#         'loss': [\"hinge\", \"squared_hinge\"],\n#         'dual': [True, False],\n#         'tol': [1e-5, 1e-4, 1e-3, 1e-2, 1e-1],\n#         'C': [1e-4, 1e-3, 1e-2, 1e-1, 0.5, 1., 5., 10., 15., 20., 25.]\n#     },\n\n    'sklearn.linear_model.LogisticRegression': {\n        'penalty': [\"l1\", \"l2\"],\n        'C': [1e-4, 1e-3, 1e-2, 1e-1, 0.5, 1., 5., 10., 15., 20., 25.],\n        'dual': [True, False]\n    },\n\n    'xgboost.XGBClassifier': {\n        'n_estimators': [50, 100, 200],\n        'max_depth': range(1, 11),\n        'learning_rate': [1e-3, 1e-2, 1e-1, 0.5, 1.],\n        'subsample': np.arange(0.05, 1.01, 0.05),\n        'min_child_weight': range(1, 21),\n        'n_jobs': [1],\n        'verbosity': [0]\n    },\n\n    'sklearn.linear_model.SGDClassifier': {\n        'loss': ['log', 'hinge', 'modified_huber', 'squared_hinge', 'perceptron'],\n        'penalty': ['elasticnet'],\n        'alpha': [0.0, 0.01, 0.001],\n        'learning_rate': ['invscaling', 'constant'],\n        'fit_intercept': [True, False],\n        'l1_ratio': [0.25, 0.0, 1.0, 0.75, 0.5],\n        'eta0': [0.1, 1.0, 0.01],\n        'power_t': [0.5, 0.0, 1.0, 0.1, 100.0, 10.0, 50.0]\n    },\n\n    'sklearn.neural_network.MLPClassifier': {\n        'alpha': [1e-4, 1e-3, 1e-2, 1e-1],\n        'learning_rate_init': [1e-3, 1e-2, 1e-1, 0.5, 1.]\n    },\n\n    # Preprocesssors\n    'sklearn.preprocessing.Binarizer': {\n        'threshold': np.arange(0.0, 1.01, 0.05)\n    },\n\n    'sklearn.decomposition.FastICA': {\n        'tol': np.arange(0.0, 1.01, 0.05)\n    },\n\n    'sklearn.cluster.FeatureAgglomeration': {\n        'linkage': ['ward', 'complete', 'average'],\n        'affinity': ['euclidean', 'l1', 'l2', 'manhattan', 'cosine']\n    },\n\n    'sklearn.preprocessing.MaxAbsScaler': {\n    },\n\n    'sklearn.preprocessing.MinMaxScaler': {\n    },\n\n    'sklearn.preprocessing.Normalizer': {\n        'norm': ['l1', 'l2', 'max']\n    },\n\n    'sklearn.kernel_approximation.Nystroem': {\n        'kernel': ['rbf', 'cosine', 'chi2', 'laplacian', 'polynomial', 'poly', 'linear', 'additive_chi2', 'sigmoid'],\n        'gamma': np.arange(0.0, 1.01, 0.05),\n        'n_components': range(1, 11)\n    },\n\n    'sklearn.decomposition.PCA': {\n        'svd_solver': ['randomized'],\n        'iterated_power': range(1, 11)\n    },\n\n    'sklearn.preprocessing.PolynomialFeatures': {\n        'degree': [2],\n        'include_bias': [False],\n        'interaction_only': [False]\n    },\n\n    'sklearn.kernel_approximation.RBFSampler': {\n        'gamma': np.arange(0.0, 1.01, 0.05)\n    },\n\n    'sklearn.preprocessing.RobustScaler': {\n    },\n\n    'sklearn.preprocessing.StandardScaler': {\n    },\n\n    'tpot.builtins.ZeroCount': {\n    },\n\n    'tpot.builtins.OneHotEncoder': {\n        'minimum_fraction': [0.05, 0.1, 0.15, 0.2, 0.25],\n        'sparse': [False],\n        'threshold': [10]\n    },\n\n    # Selectors\n    'sklearn.feature_selection.SelectFwe': {\n        'alpha': np.arange(0, 0.05, 0.001),\n        'score_func': {\n            'sklearn.feature_selection.f_classif': None\n        }\n    },\n\n    'sklearn.feature_selection.SelectPercentile': {\n        'percentile': range(1, 100),\n        'score_func': {\n            'sklearn.feature_selection.f_classif': None\n        }\n    },\n\n    'sklearn.feature_selection.VarianceThreshold': {\n        'threshold': [0.0001, 0.0005, 0.001, 0.005, 0.01, 0.05, 0.1, 0.2]\n    },\n\n    'sklearn.feature_selection.RFE': {\n        'step': np.arange(0.05, 1.01, 0.05),\n        'estimator': {\n            'sklearn.ensemble.ExtraTreesClassifier': {\n                'n_estimators': [100],\n                'criterion': ['gini', 'entropy'],\n                'max_features': np.arange(0.05, 1.01, 0.05)\n            }\n        }\n    },\n\n    'sklearn.feature_selection.SelectFromModel': {\n        'threshold': np.arange(0, 1.01, 0.05),\n        'estimator': {\n            'sklearn.ensemble.ExtraTreesClassifier': {\n                'n_estimators': [100],\n                'criterion': ['gini', 'entropy'],\n                'max_features': np.arange(0.05, 1.01, 0.05)\n            }\n        }\n    },\n\n    'tpot.builtins.PytorchLRClassifier': {\n        'learning_rate': [1e-3, 1e-2, 1e-1],\n        'batch_size': [4, 8, 16, 32, 64, 128, 256, 512, 1024],\n        'num_epochs': [10, 25, 50, 100, 250],\n        'weight_decay': [0, 1e-4, 1e-3, 1e-2]\n    },\n\n    'tpot.builtins.PytorchMLPClassifier': {\n        'learning_rate': [1e-3, 1e-2, 1e-1],\n        'batch_size': [4, 8, 16, 32, 64, 128, 256, 512],\n        'num_epochs': [10, 25, 50, 100, 250],\n        'weight_decay': [0, 1e-4, 1e-3, 1e-2]\n    },\n}","metadata":{"execution":{"iopub.status.busy":"2022-08-05T08:15:00.892297Z","iopub.execute_input":"2022-08-05T08:15:00.892836Z","iopub.status.idle":"2022-08-05T08:15:00.924345Z","shell.execute_reply.started":"2022-08-05T08:15:00.892787Z","shell.execute_reply":"2022-08-05T08:15:00.923261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# instanciante the TPot classifier\n# TPot wil generate a intial population \n# then refresh and make mutation in the population \n# I've specified a folder where TPot will store the best models found at each generation\n# it is interesting to look at what is investigated \n\nmodel_tpot = tpot.TPOTClassifier(\n    scoring=\"roc_auc\", \n    verbosity=2, \n    population_size=100,\n    offspring_size=50,\n    warm_start=True,\n    generations=10000,\n    cv=5,\n    n_jobs=-1,\n    periodic_checkpoint_folder=\"tpot\",\n    max_time_mins = 11 * 60,\n    max_eval_time_mins = 7, \n    config_dict = classifier_config_dict,\n    template='Selector-Transformer-Classifier'\n)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T08:15:00.925643Z","iopub.execute_input":"2022-08-05T08:15:00.926802Z","iopub.status.idle":"2022-08-05T08:15:00.942096Z","shell.execute_reply.started":"2022-08-05T08:15:00.926758Z","shell.execute_reply":"2022-08-05T08:15:00.940995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# fit the TOPT model with features and labels\n# export the model\n\nfeatures = [c for c in df_train.columns if c != \"failure\"]\nmodel_tpot.fit(df_train[features].values, df_train[\"failure\"].values)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T08:15:00.943521Z","iopub.execute_input":"2022-08-05T08:15:00.944148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 0.5947982151377031","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# export the selected model as a file and have a quick glance at it\n\nmodel_tpot.export('model_tpot.py')\n\n! cat model_tpot.py","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# make prediction using the optimized pipeline\n\ndf_test[\"pred\"] = model_tpot.predict_proba(df_test[features].values)[:, 1]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# create the submission file\n\ndf_submission = pd.read_csv(os.path.join(root, \"sample_submission.csv\"), index_col=0)\n\ndf_submission = df_submission.join(df_test[\"pred\"])\ndf_submission[\"failure\"] = df_submission[\"pred\"]\ndf_submission[\"failure\"].to_csv(\"submission.csv\")\ndf_submission[\"failure\"]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# quick viz of the predictions\n\ndf_submission[\"failure\"].hist(bins=100)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}