{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import json\n\nimport scipy as sp\nimport pandas as pd\nimport numpy as np\n\nfrom functools import partial\nfrom math import sqrt\n\nfrom sklearn.metrics import cohen_kappa_score, mean_squared_error\nfrom sklearn.metrics import confusion_matrix as sk_cmatrix\nfrom sklearn.model_selection import StratifiedKFold\n\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.decomposition import TruncatedSVD, LatentDirichletAllocation\nfrom sklearn.preprocessing import LabelEncoder, FunctionTransformer\n\nfrom collections import Counter\n\nimport lightgbm as lgb\n\nfrom sklearn.model_selection import KFold, ShuffleSplit\nfrom sklearn.pipeline import Pipeline, FeatureUnion\nfrom sklearn.model_selection import RandomizedSearchCV\nfrom scipy.stats import randint as sp_randint\nfrom sklearn.base import TransformerMixin, BaseEstimator, ClassifierMixin\nimport string\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn_pandas import DataFrameMapper\nfrom nltk.stem.snowball import SnowballStemmer\nnp.random.seed(369)\n\ndef confusion_matrix(rater_a, rater_b, min_rating=None, max_rating=None):\n    \"\"\"\n    Returns the confusion matrix between rater's ratings\n    \"\"\"\n    assert(len(rater_a) == len(rater_b))\n    if min_rating is None:\n        min_rating = min(rater_a + rater_b)\n    if max_rating is None:\n        max_rating = max(rater_a + rater_b)\n    num_ratings = int(max_rating - min_rating + 1)\n    conf_mat = [[0 for i in range(num_ratings)]\n                for j in range(num_ratings)]\n    for a, b in zip(rater_a, rater_b):\n        conf_mat[a - min_rating][b - min_rating] += 1\n    return conf_mat\n\ndef histogram(ratings, min_rating=None, max_rating=None):\n    \"\"\"\n    Returns the counts of each type of rating that a rater made\n    \"\"\"\n    if min_rating is None:\n        min_rating = min(ratings)\n    if max_rating is None:\n        max_rating = max(ratings)\n    num_ratings = int(max_rating - min_rating + 1)\n    hist_ratings = [0 for x in range(num_ratings)]\n    for r in ratings:\n        hist_ratings[r - min_rating] += 1\n    return hist_ratings\n\ndef quadratic_weighted_kappa(y, y_pred):\n    \"\"\"\n    Calculates the quadratic weighted kappa\n    axquadratic_weighted_kappa calculates the quadratic weighted kappa\n    value, which is a measure of inter-rater agreement between two raters\n    that provide discrete numeric ratings.  Potential values range from -1\n    (representing complete disagreement) to 1 (representing complete\n    agreement).  A kappa value of 0 is expected if all agreement is due to\n    chance.\n    quadratic_weighted_kappa(rater_a, rater_b), where rater_a and rater_b\n    each correspond to a list of integer ratings.  These lists must have the\n    same length.\n    The ratings should be integers, and it is assumed that they contain\n    the complete range of possible ratings.\n    quadratic_weighted_kappa(X, min_rating, max_rating), where min_rating\n    is the minimum possible rating, and max_rating is the maximum possible\n    rating\n    \"\"\"\n    rater_a = y\n    rater_b = y_pred\n    min_rating=None\n    max_rating=None\n    rater_a = np.array(rater_a, dtype=int)\n    rater_b = np.array(rater_b, dtype=int)\n    assert(len(rater_a) == len(rater_b))\n    if min_rating is None:\n        min_rating = min(min(rater_a), min(rater_b))\n    if max_rating is None:\n        max_rating = max(max(rater_a), max(rater_b))\n    conf_mat = confusion_matrix(rater_a, rater_b,\n                                min_rating, max_rating)\n    num_ratings = len(conf_mat)\n    num_scored_items = float(len(rater_a))\n\n    hist_rater_a = histogram(rater_a, min_rating, max_rating)\n    hist_rater_b = histogram(rater_b, min_rating, max_rating)\n\n    numerator = 0.0\n    denominator = 0.0\n\n    for i in range(num_ratings):\n        for j in range(num_ratings):\n            expected_count = (hist_rater_a[i] * hist_rater_b[j]\n                              / num_scored_items)\n            d = pow(i - j, 2.0) / pow(num_ratings - 1, 2.0)\n            numerator += d * conf_mat[i][j] / num_scored_items\n            denominator += d * expected_count / num_scored_items\n\n    return (1.0 - numerator / denominator)\n\nclass SmallModalityAsOthers(BaseEstimator, TransformerMixin):\n    \"\"\"Group small frequencies modality as an other modality\n\n    Parameters\n    ----\n    cols: columns name to apply modality fgroup\n    threshold: int - number threshold to switch a modality to \"other\"\n\n    Attributes\n    ----\n    Return pandas dataframe with new modality \"other\" for given columns\n    \"\"\"\n    def __init__(self, cols, threshold=10, other_name=\"other\"):\n        self.cols = cols\n        self.threshold = threshold\n        self.other_name = other_name\n\n    def fit(self, df, y=None, **fit_params):\n        self.modality_to_others = dict()\n        for col in self.cols:\n            table = df.loc[:, col].value_counts().reset_index()\n            self.modality_to_others[col] =\\\n                list(table.loc[table.loc[:, col] < self.threshold, \"index\"])\n        return self\n\n    def transform(self, df, **transform_params):\n        for col in self.cols:\n            df.loc[df.loc[:, col].map(lambda x: x in self.modality_to_others[col]),\n                   col] =\\\n                self.other_name\n        return df\n    \nclass NumberOfRowByValue(BaseEstimator, TransformerMixin):\n    \"\"\"Count number of rows for a specific categorical variable\n\n    Parameters\n    ----\n    col_groupby: column  name to groupby from\n    \n    Attributes\n    ----\n    Return pandas dataframe with news column. \n    Number of rows which take this specific categorical variable in training\n    \"\"\"\n    def __init__(self, col_groupby, new_col_name=\"n_row_by_category\"):\n        self.col_groupby = col_groupby\n        self.new_col_name = new_col_name\n    def fit(self, df, y=None, **fit_params):\n        self.n_row =\\\n            df.groupby(self.col_groupby).size().reset_index()\n        self.n_row.columns = [self.col_groupby, self.new_col_name]\n        return self\n    def transform(self, df, **transform_params):\n        df =\\\n            df.merge(self.n_row, how=\"left\", on=self.col_groupby)\n        return df\n\nfrom sklearn.utils.validation import check_is_fitted\nfrom sklearn.utils import column_or_1d\n\ndef _get_unseen():\n    \"\"\"Basically just a static method\n    instead of a class attribute to avoid\n    someone accidentally changing it.\"\"\"\n    return 99999\n\n\nclass SafeLabelEncoder(LabelEncoder):\n    \"\"\"An extension of LabelEncoder that will\n    not throw an exception for unseen data, but will\n    instead return a default value of 99999\n\n    Attributes\n    ----------\n\n    classes_ : the classes that are encoded\n    \"\"\"\n\n    def transform(self, y):\n        \"\"\"Perform encoding if already fit.\n\n        Parameters\n        ----------\n\n        y : array_like, shape=(n_samples,)\n            The array to encode\n\n        Returns\n        -------\n\n        e : array_like, shape=(n_samples,)\n            The encoded array\n        \"\"\"\n        check_is_fitted(self, 'classes_')\n        y = column_or_1d(y, warn=True)\n\n        classes = np.unique(y)\n        # _check_numpy_unicode_bug(classes)\n\n        # Check not too many:\n        unseen = _get_unseen()\n        if len(classes) >= unseen:\n            raise ValueError('Too many factor levels in feature. Max is %i' % unseen)\n\n        e = np.array([\n                         np.searchsorted(self.classes_, x) if x in self.classes_ else unseen\n                         for x in y\n                         ])\n\n        return e\n    \nclass CrossFeatures(BaseEstimator, TransformerMixin):\n    \"\"\" Create new column as multiplication between two columns\n    Attributes\n    ----------\n    cols_tuple: list of string tuple (a, b) such as result will be a * b\n\n    \"\"\"\n    def __init__(self, cols_tuple=None):\n        self.cols_tuple = cols_tuple\n    def fit(self, df, y=None, **fit_params):\n        return self\n    def transform(self, df, **transform_params):\n        for a, b in self.cols_tuple:\n            df.loc[:, str(a) + \"_MULTIPLICATED_BY_\" + str(b)] =\\\n                pd.to_numeric(df.loc[:, a], errors='coerce') * pd.to_numeric(df.loc[:, b], errors='coerce')\n        return df\n    \nclass DivBetweenCols(BaseEstimator, TransformerMixin):\n    \"\"\" Create new column as division between two columns\n    Attributes\n    ----------\n    cols_tuple: list of string tuple (a, b) such as result will be a / b\n\n    \"\"\"\n    def __init__(self, cols_tuple=None):\n        self.cols_tuple = cols_tuple\n    def fit(self, df, y=None, **fit_params):\n        return self\n    def transform(self, df, **transform_params):\n        for a, b in self.cols_tuple:\n            df.loc[:, str(a) + \"_DIVIDED_BY_\" + str(b)] =\\\n                pd.to_numeric(df.loc[:, a], errors='coerce') / pd.to_numeric(df.loc[:, b], errors='coerce')\n        return df\n    \nclass ColumnsSelector(BaseEstimator, TransformerMixin):\n    \"\"\" Create new Dataframe with columns selected\n    Attributes\n    ----------\n    colnames_list: list of string - columns name\n\n    \"\"\"\n    def __init__(self, colnames_list=None):\n        self.colnames_list = colnames_list\n    def fit(self, df, y=None, **fit_params):\n        return self\n    def transform(self, df, **transform_params):\n        if len(self.colnames_list) == 1:\n            return(df.loc[:, self.colnames_list[0]])\n        else:\n            return df.loc[:, self.colnames_list]\n    \nclass MeanYByCategories(BaseEstimator, TransformerMixin):\n    \"\"\"For columns that should not accept negative value:\n        if negative value is found then value become nan\n\n    Parameters\n    ----\n    cols: list of column names that sould not accept negative values\n\n    Attributes\n    ----\n    Return pandas dataframe with cols columns contain missing value if value are below 0\n    \"\"\"\n    def __init__(self, col, new_col_name):\n        self.col = col\n        self.new_col_name = new_col_name\n    def fit(self, df, y=None, **fit_params):\n        full_df = pd.concat([df.reset_index(drop=True),\n                             pd.Series(y).reset_index(drop=True)], axis=1, ignore_index=True)\n        full_df.columns = list(df.columns) + [\"y_target\"]\n        self.median_by_cat =\\\n            full_df.groupby(self.col)[\"y_target\"].mean().reset_index()\n        return self\n    def transform(self, df, **transform_params):\n        df =\\\n            df.merge(self.median_by_cat, how=\"left\", on=self.col)\n        df.drop(self.col, axis=1, inplace=True)\n        df = df.rename(columns={\"y_target\":self.new_col_name})\n        return df\n    \n\nclass CreateGroupByFeature(BaseEstimator, TransformerMixin):\n    \"\"\"\n    Parameters\n    ----\n    cols: list of column names to compute mean\n    by: column to groupby\n\n    Attributes\n    ----\n    Return pandas dataframe wnew columns as average values\n    \"\"\"\n    def __init__(self, col, by, percentile=False):\n        self.col = col\n        self.by = by\n        self.percentile = percentile\n    def fit(self, df, y=None, **fit_params):\n        operations = [\"max\", \"mean\", \"std\"]\n        self.median_by_cat =\\\n            df.groupby(self.by)[self.col].agg(operations).reset_index()\n        self.median_by_cat.columns =\\\n            [self.by] + [x + \"_\" + y + \"_\" + self.by for x in self.col for y in operations]\n        if self.percentile:\n            self.percentile_table =\\\n                df.groupby(self.by)[self.col].quantile([0.1, 0.25, 0.75, 0.9]).reset_index()\n            self.percentile_table =\\\n                self.percentile_table.pivot(self.by, columns=\"level_1\")\n            self.percentile_table.columns = [x + str(y) + \"_by_\" + self.by for x in self.col for y in [0.1, 0.25, 0.75, 0.9]]\n        return self\n    def transform(self, df, **transform_params):\n        df =\\\n            df.merge(self.median_by_cat, how=\"left\", on=self.by)\n        # df.drop(self.by, axis=1, inplace=True)\n        if self.percentile:\n            df =\\\n                df.merge(self.percentile_table, how=\"left\", on=self.by)\n        # df = df.rename(columns={\"y_target\":self.new_col_name})\n        return df\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bcc99d2c0cc3281b6bcd21f69eff302cbd075e71"},"cell_type":"markdown","source":"## 1 - Load Data"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"print('Train')\ntrain = pd.read_csv(\"../input/train/train.csv\")\nprint(train.shape)\n\nprint('Test')\ntest = pd.read_csv(\"../input/test/test.csv\")\nprint(test.shape)\n\nprint('Breeds')\nbreeds = pd.read_csv(\"../input/breed_labels.csv\")\nprint(breeds.shape)\n\nprint('Colors')\ncolors = pd.read_csv(\"../input/color_labels.csv\")\nprint(colors.shape)\n\nprint('States')\nstates = pd.read_csv(\"../input/state_labels.csv\")\nprint(states.shape)\n\ntarget = train['AdoptionSpeed']\ntrain_id = train['PetID']\ntest_id = test['PetID']\ntrain.drop(['AdoptionSpeed', 'PetID'], axis=1, inplace=True)\ntest.drop(['PetID'], axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"291274bd70b16fc50539b93856e53628dcf7b3ce"},"cell_type":"markdown","source":"## 2 - Feature enginerring"},{"metadata":{"_uuid":"109fe4c88d4f383297a0561ce2dcf84e60d208bc"},"cell_type":"markdown","source":"#TODO\n - ADD tfidf for description\n - add sentiment file\n - add image metadata file"},{"metadata":{"trusted":true,"_uuid":"a0fb8d13a66699caa5425ad436ab9e1d95545c28"},"cell_type":"code","source":"def name_size(name):\n    if pd.notnull(name):\n        return(len(name))\n    else:\n        return(0)\n    \ndef n_words_in_name(name):\n    if pd.notnull(name):\n        return(len(name.split(\" \")))\n    else:\n        return(0)\n    \ndef hasNumbers(inputString):\n    if pd.notnull(inputString):\n        return int(any(char.isdigit() for char in inputString))\n    else:\n        return(0)\n\ndef n_punctuation(inputString):\n    if pd.notnull(inputString):\n        return sum([j in string.punctuation for j in inputString])\n    else:\n        return(0)\n    \nCOLUMNS_TO_DROP = [\"RescuerID\"]\nCATEGORICAL_FEATURES = [\"Gender\", \"Vaccinated\", \"Dewormed\", \n                        \"Sterilized\", \"State\",\n                        \"Name\", \"MaturitySize\", \"Type\",\n                        \"Health\"]\n\nCOLUMNS_WITH_SMALL_MODALITY =\\\n    [\"Breed1\", \"Breed2\", \"Color1\", \"Color2\", \"Color3\", \"Name\"]\n\n# Create transformers instance\nto_other = SmallModalityAsOthers(cols=COLUMNS_WITH_SMALL_MODALITY, other_name=-1, threshold=30)\nname_frequencies = NumberOfRowByValue(col_groupby=\"Name\", new_col_name=\"name_frequency\")\ncross_feature = CrossFeatures(cols_tuple=[(\"Age\", \"Health\")])\ndiv_features = DivBetweenCols(cols_tuple=[(\"Age\", \"Health\")])\nmedian_Y = MeanYByCategories(col=\"Type\", new_col_name=\"Type_y_frequencies\")\nmedian_Y_by_breed1 = MeanYByCategories(col=\"Breed1\", new_col_name=\"Breed1\")\nmedian_Y_by_breed2 = MeanYByCategories(col=\"Breed2\", new_col_name=\"Breed2\")\nmedian_Y_by_color1 = MeanYByCategories(col=\"Color1\", new_col_name=\"Color1\")\nmedian_Y_by_color2 = MeanYByCategories(col=\"Color2\", new_col_name=\"Color2\")\nmedian_Y_by_color3 = MeanYByCategories(col=\"Color3\", new_col_name=\"Color3\")\ngrouper = CreateGroupByFeature(col=[\"Fee\", \"Quantity\"], by=\"Breed1\", percentile=False)\ngrouper2 = CreateGroupByFeature(col=[\"Fee\", \"Quantity\"], by=\"Breed2\", percentile=False)\nstemmer = SnowballStemmer(\"english\")\n\nall_data = [train, test]\n\nfor i in range(len(all_data)):\n    # Name size\n    all_data[i].loc[:, \"name_size\"] =\\\n        all_data[i].Name.map(name_size)\n    \n    all_data[i].loc[:, \"n_words_in_name\"] =\\\n        all_data[i].Name.map(n_words_in_name)\n    \n    all_data[i].loc[:, \"name_contain_number\"] =\\\n        all_data[i].Name.map(hasNumbers)\n    \n    all_data[i].loc[:, \"n_punctuations_in_name\"] =\\\n        all_data[i].Name.map(n_punctuation)\n    \n    all_data[i].loc[:, \"photo_and_video\"] =\\\n        all_data[i].PhotoAmt + all_data[i].VideoAmt\n   \n\nall_data[0] = median_Y.fit_transform(all_data[0], target)\nall_data[1] = median_Y.transform(all_data[1])\n\nall_data[0] = median_Y_by_breed1.fit_transform(all_data[0], target)\nall_data[1] = median_Y_by_breed1.transform(all_data[1])\nall_data[0] = median_Y_by_breed2.fit_transform(all_data[0], target)\nall_data[1] = median_Y_by_breed2.transform(all_data[1])\nall_data[0] = median_Y_by_color1.fit_transform(all_data[0], target)\nall_data[1] = median_Y_by_color1.transform(all_data[1])\nall_data[0] = median_Y_by_color2.fit_transform(all_data[0], target)\nall_data[1] = median_Y_by_color2.transform(all_data[1])\nall_data[0] = median_Y_by_color3.fit_transform(all_data[0], target)\nall_data[1] = median_Y_by_color3.transform(all_data[1])\n        \nall_data[0] = name_frequencies.fit_transform(all_data[0])\nall_data[1] = name_frequencies.transform(all_data[1])\nall_data[1].name_frequency = all_data[1].name_frequency.fillna(0)\nall_data[0].name_frequency = all_data[0].name_frequency.fillna(0)\n\n# Label encode Name\nname_le = SafeLabelEncoder()\nall_data[0].Name = name_le.fit_transform(all_data[0].Name.fillna(\"missing\"))\nall_data[1].Name = name_le.transform(all_data[1].Name.fillna(\"missing\"))\n\nall_data[0] = to_other.fit_transform(all_data[0])\nall_data[1] = to_other.transform(all_data[1])\n\n# cross and div feature\nall_data[0] = cross_feature.fit_transform(all_data[0])\nall_data[1] = cross_feature.transform(all_data[1])\n\nall_data[0] = div_features.fit_transform(all_data[0])\nall_data[1] = div_features.transform(all_data[1])\n\nall_data[0] = all_data[0].drop(COLUMNS_TO_DROP, axis=1)\nall_data[1] = all_data[1].drop(COLUMNS_TO_DROP, axis=1)\n\nall_data[0] = grouper.fit_transform(all_data[0])\nall_data[1] = grouper.transform(all_data[1])\nall_data[0] = grouper2.fit_transform(all_data[0])\nall_data[1] = grouper2.transform(all_data[1])\n\nfor i in range(len(all_data)):\n    all_data[i].loc[:, \"relative_fee\"] =\\\n        all_data[i].Fee / all_data[i].Fee_mean_Breed1\n    all_data[i].loc[:, \"relative_fee2\"] =\\\n        all_data[i].Fee / all_data[i].Fee_mean_Breed2\n    all_data[i].loc[:, \"relative_Quantity\"] =\\\n        all_data[i].Fee / all_data[i].Quantity_mean_Breed1\n    all_data[i].loc[:, \"relative_Quantity2\"] =\\\n        all_data[i].Fee / all_data[i].Quantity_mean_Breed2\n    \n    all_data[i].Description = all_data[i].Description.fillna(\"None\")\n    all_data[i].Description =\\\n        all_data[i].Description.map(lambda x: stemmer.stem(x))\n\ntrain = all_data[0]\ntest = all_data[1]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"395755f995ba4f933f6dac390fce994efcc9b2b2"},"cell_type":"markdown","source":"## 3 - simple lgbm"},{"metadata":{"trusted":true,"_uuid":"5ec06823ceca177b7d463a0541a736431337a2df"},"cell_type":"code","source":"# lgbm = lgb.LGBMClassifier(categorical_features=CATEGORICAL_FEATURES)\nlgbm = lgb.LGBMClassifier(categorical_features=CATEGORICAL_FEATURES)\ndescription_selector = ColumnsSelector(colnames_list=[\"Description\"])\nnodescription_selector =\\\n    ColumnsSelector(colnames_list=[x for x in train.columns if x !=\"Description\"])\ntf_idf = TfidfVectorizer(stop_words=\"english\", ngram_range=(1, 2),\n                        norm=\"l1\", max_df=0.9, min_df=0.05)\nlda = LatentDirichletAllocation(n_jobs=4)\n\npipe = Pipeline([(\"union\", FeatureUnion([\n                ('tf_idf', \n                  Pipeline([('extract_field',\n                              description_selector),\n                            ('tfidf', \n                              tf_idf),\n                           (\"lda\", lda)])),\n                ('no_tfidf',\n                  nodescription_selector)])),\n                 (\"classifier\", lgbm)])\n\n# pipe = Pipeline([(\"classifier\", lgbm)])\n\nRECOMPUTE_BEST_PARAMS = True\n\n# Find best param with random search\nparam_dist = {\"classifier__num_leaves\": sp_randint(2, 20),\n              \"classifier__learning_rate\": [0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9],\n              \"classifier__max_bin\": sp_randint(13, 23),\n              \"classifier__bagging_freq\": [20, 21, 22, 23, 24, 25, 26, 27],\n              \"classifier__max_depth\": sp_randint(100, 250),\n              \"classifier__feature_fraction\": [0.7, 0.8, 0.9, 1],\n             \"classifier__n_estimators\":sp_randint(150, 800),\n             \"classifier__reg_alpha\":[0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6]}\n\nbest_params = {'classifier__bagging_freq': 26,\n             'classifier__feature_fraction': 0.9,\n             'classifier__learning_rate': 0.3,\n             'classifier__max_bin': 21,\n             'classifier__max_depth': 219,\n             'classifier__n_estimators': 304,\n             'classifier__num_leaves': 3,\n             'classifier__reg_alpha': 0.6}\n\nif RECOMPUTE_BEST_PARAMS:\n    n_iter_search = 20\n    random_search = RandomizedSearchCV(pipe, param_distributions=param_dist,\n                                        n_iter=n_iter_search, cv=5,\n                                      scoring=\"neg_log_loss\",\n                                      n_jobs=4)\n\n    random_search.fit(train, target)\n    \n    best_params = random_search.best_params_\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"037a4ba0ccdf97f195d8aeee1f7c4e7ddbcabb04"},"cell_type":"code","source":"# affect best params\npipe = pipe.set_params(**best_params)\n\n# k-fold to evaluate\nkf = ShuffleSplit(n_splits=5, test_size=0.20, random_state=50)\n\nqwk_train_list = []\nqwk_test_list = []\npipe_list = []\nfor train_index, test_index in kf.split(train):\n    X_train, X_test = train.iloc[train_index], train.iloc[test_index]\n    y_train, y_test = target[train_index], target[test_index]\n    \n    if RECOMPUTE_BEST_PARAMS:\n        random_search.fit(X_train, y_train)\n        train_predictions = random_search.predict(X_train)\n        test_predictions = random_search.predict(X_test)\n        pipe_list.append(random_search)\n    else:\n        pipe.fit(X_train, y_train)\n\n        train_predictions = pipe.predict(X_train)\n        test_predictions = pipe.predict(X_test)\n        pipe_list.append(pipe)\n    \n    qwk_train = quadratic_weighted_kappa(y_train, train_predictions)\n    qwk_test = quadratic_weighted_kappa(y_test, test_predictions)\n    \n    print(\"QWK train = \" + str(qwk_train))\n    print(\"QWK test = \" + str(qwk_test))\n    \n    qwk_train_list = qwk_train_list + [qwk_train]\n    qwk_test_list = qwk_test_list + [qwk_test]\n    \nprint(\"Average of QWK train = \" + str(sum(qwk_train_list) / len(qwk_train_list)))\nprint(\"Average of QWK test = \" + str(sum(qwk_test_list) / len(qwk_test_list)))\n\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e6a3f0433b0ae40889d0a669dfba219778c15c05"},"cell_type":"markdown","source":"## 4 - Apply to test"},{"metadata":{"trusted":true,"_uuid":"2a5b39c770592062e2c8475fb7491c7ac879c8db"},"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nif RECOMPUTE_BEST_PARAMS == False:\n    feat_imp =\\\n        pd.concat([pd.Series([\"lda_component\" + str(i) for i in range(1, 10)] + \n                             list(train.columns)), \n                   pd.Series(pipe.named_steps[\"classifier\"].feature_importances_)], axis=1)\n\n    feat_imp.columns = [\"var\", \"importance\"]\n\n    feat_imp = feat_imp.sort_values(by=\"importance\", ascending=False)\n\n    feat_imp.plot.bar(x=\"var\", y=\"importance\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7bc4970b04fc7e4da4933ff738a64a7a009aea25"},"cell_type":"code","source":"test_predictions_proba = pipe_list[0].predict_proba(test)\nfor temp_pipe in pipe_list[1:]:\n    # temp_pipe.fit(train, target)\n    test_predictions_proba = test_predictions_proba + temp_pipe.predict_proba(test)\n\ntest_predictions_proba = test_predictions_proba / 5\ntest_predictions = np.apply_along_axis(np.argmax, arr=test_predictions_proba, axis=1)\n\nsubmission = pd.DataFrame({'PetID': test_id, 'AdoptionSpeed': test_predictions})\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8f06c1397845c12272011a71731b0a15c0353d5f"},"cell_type":"code","source":"submission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cd12acd97624bca6daffc7e6bbd0288a374c2f02"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}