{"cells":[
 {
  "cell_type": "code",
  "execution_count": null,
  "metadata": {
   "collapsed": false
  },
  "outputs": [],
  "source": "import numpy as np\nimport pandas as pd\nimport sys\n\nfrom scipy import sparse as sp\n\nfrom sklearn.base import BaseEstimator, TransformerMixin\nfrom sklearn.cross_validation import train_test_split\nfrom sklearn.feature_extraction.text import CountVectorizer\nfrom sklearn.feature_selection import VarianceThreshold\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.metrics import get_scorer\nfrom sklearn.pipeline import FeatureUnion, Pipeline\nfrom sklearn.preprocessing import FunctionTransformer, LabelBinarizer, OneHotEncoder\n\nfrom xgboost import XGBClassifier\n\n\nclass TextTransformer(BaseEstimator, TransformerMixin):\n    def __init__(self, est):\n        super(TextTransformer, self).__init__()\n        self.est = est\n        pass\n    \n    def fit(self, X, y=None):\n        self.est.fit(X.ravel())\n        return self\n    \n    def transform(self, X):\n        Xs = [\n            self.est.transform(X[:,_])\n            for _ in range(X.shape[-1])\n        ]\n        result = (Xs[0]>0).astype(int)\n        for _ in range(len(Xs)-1):\n            result += (Xs[_+1]>0).astype(int)\n        return sp.hstack((\n                (result == len(Xs)).astype(float), # Some kind of binary AND\n                (result == 1).astype(float) # Binary XOR\n            )).tocsr()\n\nclass SupervisedTransformer(BaseEstimator, TransformerMixin):\n    def __init__(self, est, method):\n        super(SupervisedTransformer, self).__init__()\n        self.est = est\n        self.method = method\n        pass\n    \n    def fit(self, X, y=None):\n        self.est.fit(X, y)\n        return self\n    \n    def transform(self, X):\n        return getattr(self.est, self.method)(X)\n\nclass EqNotEqBinarizer(BaseEstimator, TransformerMixin):\n    def __init__(self):\n        super(EqNotEqBinarizer, self).__init__()\n        pass\n    \n    def fit(self, X, y=None):\n        assert X.shape[-1] == 2, 'Only two-column arrays'\n        self.bin_ = LabelBinarizer(sparse_output=True)\n        self.bin_.fit(X.ravel())\n        return self\n    \n    def transform(self, X):\n        z = np.zeros((X.shape[0], 3), dtype=int)\n        eqmask = X[:,0] == X[:,1]\n        noteqmask = X[:,0] != X[:,1]\n        z[eqmask,0] = X[eqmask,0]\n        z[noteqmask,1] = X[noteqmask,0]\n        z[noteqmask,2] = X[noteqmask,1]\n        return sp.hstack((\n                self.bin_.transform(z[:,0]),\n                self.bin_.transform(z[:,1]) + self.bin_.transform(z[:,2])\n            ))\n\ndef get_file(mode):\n    params = {}\n    if mode == 'test':\n        params['index_col'] = 0\n    items = pd.read_csv('../input/ItemInfo_%s.csv' % mode, index_col=0)[[\n            'categoryID', 'title', 'locationID', 'metroID', 'lon', 'lat']]\n    items['title'] = items['title'].fillna('nan')\n    items['metroID'] = items['metroID'].fillna(-1)\n    parent_categories = pd.read_csv('../input/Category.csv', index_col=0)\n    regions = pd.read_csv('../input/Location.csv', index_col=0)\n    items = pd.merge(items, parent_categories, left_on='categoryID', right_index=True, how='inner', sort=False)\n    items = pd.merge(items, regions, left_on='locationID', right_index=True, how='inner', sort=False)\n    del parent_categories\n    del regions\n\n    pr = pd.read_csv('../input/ItemPairs_%s.csv' % mode, **params)\n    pr = pd.merge(pr, items, left_on='itemID_1', right_index=True, how='inner', sort=False)\n    pr = pd.merge(pr, items, left_on='itemID_2', right_index=True, how='inner', sort=False)\n    del items\n\n    print('Columns: ' + str(pr.columns), file=sys.stderr)\n\n    fields = [\n        'categoryID_x', 'parentCategoryID_x', 'title_x', 'title_y',\n        'locationID_x', 'locationID_y', 'regionID_x', 'regionID_y', 'metroID_x', 'metroID_y',\n        'lon_x', 'lon_y', 'lat_x', 'lat_y',\n    ]\n    if mode == 'train':\n        return pr[fields + ['isDuplicate']]\n    else:\n        return pr[fields]\n\n\ndef get_balanced_train_indices(column=\"categoryID\"):\n    from sklearn.utils import shuffle\n    \n    prtest = pd.read_csv(\"../input/ItemPairs_test.csv\", index_col=0)\n    prtest = pd.merge(prtest,\n                     pd.read_csv(\"../input/ItemInfo_test.csv\", index_col=0),\n                     left_on=\"itemID_1\", right_index=True, how=\"inner\", sort=False)\n    catdist = prtest[column].value_counts() / len(prtest)\n    del prtest\n    \n    prtrain = pd.read_csv(\"../input/ItemPairs_train.csv\")\n    prtrain = pd.merge(prtrain,\n                       pd.read_csv(\"../input/ItemInfo_train.csv\", index_col=0),\n                       left_on=\"itemID_1\", right_index=True, how=\"inner\", sort=False)\n    \n    indices = np.array([])\n    trainsize = len(prtrain)\n    for cat, dist in catdist.iteritems():\n        trcatdist = len(prtrain[prtrain[column] == cat])\n        if trcatdist < int(1.0 * dist * trainsize):\n            trainsize = int(1.0 * trainsize * trcatdist / (dist * trainsize))\n    for cat, dist in catdist.iteritems():\n        indices = np.hstack((indices, shuffle(prtrain[prtrain[column] == cat].index, random_state=1)[:int(dist*trainsize)]))\n    \n    indices = pd.Index(np.sort(indices.astype(int)))\n    return indices\n\n    \ndef _print_shape(X):\n    print(\"SHAPE: \", X.shape, file=sys.stderr)\n    return X\n    \nest = Pipeline([\n        ('shape1', FunctionTransformer(_print_shape, validate=False)),\n        ('feats', FeatureUnion(transformer_list=[\n                    ('categories', Pipeline([\n                                ('filter', FunctionTransformer(lambda X: X[:,[0]], validate=False)),\n                                ('binarizer', OneHotEncoder()),\n                                ('shape1', FunctionTransformer(_print_shape, validate=False)),\n                            ])),\n                    ('parentCategories', Pipeline([\n                                ('filter', FunctionTransformer(lambda X: X[:,[1]], validate=False)),\n                                ('binarizer', OneHotEncoder()),\n                                ('shape1', FunctionTransformer(_print_shape, validate=False)),\n                            ])),\n                    ('titles', Pipeline([\n                                ('filter', FunctionTransformer(lambda X: X[:,[2,3]], validate=False)),\n                                ('titleswitch', TextTransformer(CountVectorizer(binary=True))),\n                                ('logreg', SupervisedTransformer(LogisticRegression(C=0.01), 'predict_proba')),\n                                ('selector', FunctionTransformer(lambda X: X[:,[1]])),\n                                ('shape1', FunctionTransformer(_print_shape, validate=False)),\n                            ])),\n#                     ('locationID', Pipeline([\n#                                 ('filter', FunctionTransformer(lambda X: X[:,[4,5]].astype(int), validate=False)),\n#                                 ('binarizer', EqNotEqBinarizer()),\n#                                 ('threshold', VarianceThreshold(0.0001)),\n#                                 ('shape1', FunctionTransformer(_print_shape, validate=False)),\n#                             ])),\n                    ('regionID', Pipeline([\n                                ('filter', FunctionTransformer(lambda X: X[:,[6,7]].astype(int), validate=False)),\n                                ('binarizer', EqNotEqBinarizer()),\n                                ('threshold', VarianceThreshold(0.0001)),\n                                ('shape1', FunctionTransformer(_print_shape, validate=False)),\n                            ])),\n#                     ('metroID', Pipeline([\n#                                 ('filter', FunctionTransformer(lambda X: X[:,[8,9]].astype(int), validate=False)),\n#                                 ('binarizer', EqNotEqBinarizer()),\n#                                 ('threshold', VarianceThreshold(0.0001)),\n#                                 ('shape1', FunctionTransformer(_print_shape, validate=False)),\n#                             ])),\n                    ('coords', Pipeline([\n                                ('filter', FunctionTransformer(lambda X: X[:,[10,11,12,13]].astype(float), validate=False)),\n                                ('shape1', FunctionTransformer(_print_shape, validate=False)),\n                            ])),\n                ])),\n        ('shape2', FunctionTransformer(_print_shape, validate=False)),\n        ('est', XGBClassifier()),\n    ])\n\npr = get_file('train')\nprint('Columns: ' + str(pr.columns), file=sys.stderr)\nprint('FITTING...', file=sys.stderr)\nest.fit(pr.drop('isDuplicate', axis=1).values, pr['isDuplicate'].values)\nprint('FITTED', file=sys.stderr)\n\ndel pr\n\npr = get_file('test')\nprint('Columns: ' + str(pr.columns), file=sys.stderr)\n\npr['probability'] = est.predict_proba(pr.values)[:,1]\n\npr[['probability']].to_csv('submission.csv')\n"
 }
],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"}}, "nbformat": 4, "nbformat_minor": 0}