{"cells":[{"metadata":{"_uuid":"0739e2d7d07a0a9bbd81d33b71b81e0e884c3fa9"},"cell_type":"markdown","source":"Work in progress:\n\n- Catboost, LGBM? - seems to be doing very well in other kernels\n- Parameters tuning very basic right now... need to work on this + find out which specific parameters are important\n- Need to apply model ensembling\n- Try working with images and sentiment"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n%matplotlib inline\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import train_test_split\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\")) #this is the directory for inputs\n\n# Any results you write to the current directory are saved as output.\n\n# For notebook plotting\n%matplotlib inline\n\n# Data Viz\nimport matplotlib.pyplot as plt\n\nfrom time import time\nfrom scipy.stats import randint as sp_randint\n\n# Scikit + xgb library\nfrom sklearn.naive_bayes import GaussianNB\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.metrics import cohen_kappa_score, make_scorer\nfrom sklearn.model_selection import cross_val_score, train_test_split\nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn.model_selection import RandomizedSearchCV\nfrom sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier, AdaBoostClassifier, GradientBoostingClassifier\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.datasets import load_digits\nfrom sklearn.ensemble import RandomForestClassifier\nimport xgboost as xgb\n\n\n# Seed for reproducability\nseed = 123\nnp.random.seed(seed)\n\n# Directory\nKAGGLE_DIR = '../input/'\n\n# Info about dataset - referenced from another kernel\nprint('Files and directories: \\n{}\\n'.format(os.listdir(KAGGLE_DIR)))\nprint('Within the train directory: \\n{}\\n'.format(os.listdir(KAGGLE_DIR + 'train')))\nprint('Within the test directory: \\n{}\\n'.format(os.listdir(KAGGLE_DIR + 'test')))\n\nprint('\\n# File sizes')\nfor file in os.listdir(KAGGLE_DIR):\n    print('{}| {} MB'.format(file.ljust(30), \n                             str(round(os.path.getsize(KAGGLE_DIR + file) / 1000000, 2))))\n        \nprint('\\n# File sizes in train: ')\nfor file in os.listdir(KAGGLE_DIR + 'train/'):\n    print('{}| {} MB'.format(file.ljust(30), \n                             str(round(os.path.getsize(KAGGLE_DIR + 'train/' + file) / 1000000, 2))))\n        \nprint('\\n# File sizes in test: ')\nfor file in os.listdir(KAGGLE_DIR + 'test/'):\n    print('{}| {} MB'.format(file.ljust(30), \n                             str(round(os.path.getsize(KAGGLE_DIR + 'test/' + file) / 1000000, 2))))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"423f1ad1910cfeb79962f428eb474a14f9e2b17a"},"cell_type":"code","source":"#kappa score calculator\nfrom sklearn.metrics import cohen_kappa_score\nfrom sklearn.metrics import make_scorer\ndef kappa(y_true, y_pred):\n    return cohen_kappa_score(y_true, y_pred, weights='quadratic')\nscorer = make_scorer(kappa)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"#load data\nbreeds = pd.read_csv('../input/breed_labels.csv') \ncolors = pd.read_csv('../input/color_labels.csv')\nstates = pd.read_csv('../input/state_labels.csv')\n\ntrain = pd.read_csv('../input/train/train.csv')\ntest = pd.read_csv('../input/test/test.csv')\nsub = pd.read_csv('../input/test/sample_submission.csv')\ntarget = train['AdoptionSpeed']\n\ntrain['dataset_type'] = 'train'\ntest['dataset_type'] = 'test'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fbc04f63fc672e8401ea340fd55e2a34d7130bd7"},"cell_type":"code","source":"train.info() #look at basic info\n#seems as if some names are missing - but probably irrelavant\n#description\n#photo and video may be used for additional analysis\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"95af5ea5e1f4ce3f2f1e252b53cdbd67b80bbf91"},"cell_type":"code","source":"#look at all data except description\ntrain.drop('Description', axis=1).head() ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"89e00e7704c159736d4f125bebc2f4a77ed024af"},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1cd1447eb693c33172e2eb99b19e11be813b15b9"},"cell_type":"code","source":"train.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e5ea8190799efaaba6c9a133392adb7347fad2cc"},"cell_type":"code","source":"train.hist(bins=50, figsize = (20,15))\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b972ef7ee9000269203ebddda5920dc66e0cbb58"},"cell_type":"code","source":"corr_matrix = train.corr()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b9e78edaca3a809e4d7412451c51eb8e4cc3d7ed"},"cell_type":"code","source":"corr_matrix[\"AdoptionSpeed\"].sort_values(ascending=False) #see what is correlated\n#seems that there is very little correlation","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"562c6dca559f8269b297efbaa424d0d8cba8c6fd"},"cell_type":"markdown","source":"Prepare the Data (Data Cleaning... etc)"},{"metadata":{"trusted":true,"_uuid":"4392b13c10b4b9b6eca444e257f6685c8b8a4291"},"cell_type":"code","source":"# Data clean for initial model\n# Probably need to implement these models for higher results\ntarget = train['AdoptionSpeed']\nclean_df = train.drop(columns=['Name', 'RescuerID', 'Description', 'PetID', 'AdoptionSpeed', 'dataset_type'])\nclean_test = test.drop(columns=['Name', 'RescuerID', 'Description', 'PetID', 'dataset_type'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7eaa88f68d60fe2e09c3ef6a9bea9362a6c61877"},"cell_type":"code","source":"#split training set for training and validation\nx_train, x_valid, y_train, y_valid = train_test_split(clean_df, \n                                                      target, \n                                                      test_size=0.2, \n                                                      random_state=seed)\n\n# Preparation for XGBoost\nd_train = xgb.DMatrix(x_train, label=y_train)\nd_valid = xgb.DMatrix(x_valid, label=y_valid)\n\nwatchlist = [(d_train, 'train'), (d_valid, 'valid')]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"71afbbc8d6eaf7b83e211ee627e371b12a17f022"},"cell_type":"code","source":"#see if data is clean... seems as if petfinder.my did most of the cleaning for us\nclean_df.isnull().values.any()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ef70f0f6f921d700666e704b4046bf0d9a484274"},"cell_type":"code","source":"#lets see what clean data looks like\nclean_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a51eb6aeed74b8acc0299d78f53c0671dc4d0128"},"cell_type":"code","source":"#check structure of all data that we cleaned\n\nx_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"34e393bc400ee82d6c77a027bdd236a00846f555"},"cell_type":"code","source":"y_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"076978e826c04dd75e38252fbade605c60f541e0"},"cell_type":"code","source":"clean_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"72088511241b0e6f0e97bc6df30c8d05187228a8"},"cell_type":"code","source":"target.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2cedf784fdf4a07c32ef751ec3a3931fc8cc3a08"},"cell_type":"code","source":"# Fit XGBoost - got initial params from another kernel... need to tune this for better \n# performance -> gridsearch\n\nxgb_params = {'objective' : 'multi:softmax',\n              'eval_metric' : 'mlogloss',\n              'eta' : 0.05,\n              'max_depth' : 4,\n              'num_class' : 5,\n              'lambda' : 0.8\n}\n\n\nprint('Fitting XGBoost: ')\nbst = xgb.train(xgb_params, \n                d_train, \n                400, \n                watchlist, \n                early_stopping_rounds=50, \n                verbose_eval=0)\n\n#Fit RandomTree\nclf_rfc = RandomForestClassifier()\nclf_rfc.fit(x_train, y_train)\n\nclf_etc = ExtraTreesClassifier()\nclf_etc.fit(x_train, y_train)\n\nclf_ada = AdaBoostClassifier()\nclf_ada.fit(x_train, y_train)\n\nclf_gdc = GradientBoostingClassifier()\nclf_gdc.fit(x_train, y_train)\n\n#this is the dict struct that will run our models\nmodels = {'XGBoost' : bst , \n         'RandomTree':  clf_rfc ,\n         'ExtraTrees' : clf_etc,\n         'Adaboost' : clf_ada,\n         'GradientBoost' : clf_gdc}\n\nbst.predict(xgb.DMatrix(clean_test)).astype(int)\n\n\n#training scores report\nprint('Training set scores... Check for overfitting, underfitting, etc...:\\n')\ntrain_scores = []\nfor name, model in models.items():\n    if name == 'XGBoost':\n        score = kappa(bst.predict(xgb.DMatrix(clean_df)).astype(int), target)\n        print('{} score: {}'.format(str(name), round(score, 5)))\n    else:    \n        score = kappa(model.predict(x_valid), y_valid)\n        print('{} score: {}'.format(str(name), round(score, 5)))\n    train_scores.append(score)\n\nprint('\\nMean Score: {0:10.4f}'.format(np.mean(train_scores)))\n\nprint('\\nStandard Deviation of Scores: {0:10.4f}'.format(np.std(train_scores)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"085bc125e2c973214fd1d157995db6016cc48645"},"cell_type":"code","source":"# Utility function to report best scores for gridsearch/randomsearch\ndef report(results, n_top=3):\n    for i in range(1, n_top + 1):\n        candidates = np.flatnonzero(results['rank_test_score'] == i)\n        for candidate in candidates:\n            print(\"Model with rank: {0}\".format(i))\n            print(\"Mean validation score: {0:.3f} (std: {1:.3f})\".format(\n                  results['mean_test_score'][candidate],\n                  results['std_test_score'][candidate]))\n            print(\"Parameters: {0}\".format(results['params'][candidate]))\n            print(\"\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4cd1a0747f09b2975b0d0093289b894e5960e416"},"cell_type":"code","source":"#gridsearch / random search for random forest classifier\n#RANDOM SEARCH for random forest classifier\n\n\n# param, distribution options that random search, grid search will run.\nparam_dist = {\"max_depth\": [3, None],\n              \"max_features\": sp_randint(1, 11),\n              \"min_samples_split\": sp_randint(2, 11),\n              \"bootstrap\": [True, False],\n              \"criterion\": [\"gini\", \"entropy\"]}\n\n# run randomized search\nn_iter_search = 20\nrandom_search = RandomizedSearchCV(clf_rfc, param_distributions=param_dist,\n                                   n_iter=n_iter_search, cv=5)\n\nstart = time()\nrandom_search.fit(x_train, y_train)\nprint(\"RandomizedSearchCV took %.2f seconds for %d candidates\"\n      \" parameter settings.\" % ((time() - start), n_iter_search))\nreport(random_search.cv_results_)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6f38166239a580ffadfcc2b8f1967728b1c5e09c"},"cell_type":"code","source":"#GRID SEARCH for random forest classifier\n# use a full grid over all parameters\nparam_grid = {\"max_depth\": [3, None],\n              \"max_features\": [1, 3, 10],\n              \"min_samples_split\": [2, 3, 10],\n              \"bootstrap\": [True, False],\n              \"criterion\": [\"gini\", \"entropy\"]}\n\n# run grid search\ngrid_search = GridSearchCV(clf_rfc, param_grid=param_grid, cv=5)\nstart = time()\ngrid_search.fit(x_valid, y_valid)\n\nprint(\"GridSearchCV took %.2f seconds for %d candidate parameter settings.\"\n      % (time() - start, len(grid_search.cv_results_['params'])))\nreport(grid_search.cv_results_)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"15b430295a5736c7a19ccb9386d20ec8a7998b8a"},"cell_type":"code","source":"#Fit RandomTree with modified Params\nclf_rfc_modparam = RandomForestClassifier(bootstrap= False, criterion= 'gini', max_depth = None, \n                             max_features= 8, min_samples_split= 8)\nclf_rfc_modparam.fit(x_train, y_train)\n\n#report score\nscore = kappa(clf_rfc_modparam.predict(x_valid), y_valid)\nprint('{} score: {}'.format(\"clf_rfc_modparam\", round(score, 4)))\n\n#wow this dropped scores","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"612a245d6640b0e82c432a417fff7a1b56ab4da5"},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3e02acb2f30ae07a548fd67e3bda32f80c7087ed"},"cell_type":"code","source":"#submitting files\n\ntest_pet_ID = test['PetID']\nfinal = bst.predict(xgb.DMatrix(clean_test))\n\nsubmission = pd.DataFrame(data={'PetID' : test_pet_ID.tolist(), \n                                   'AdoptionSpeed' : final})\nsubmission.AdoptionSpeed = submission.AdoptionSpeed.astype(int)\nsubmission.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}