{"cells":[{"metadata":{"_cell_guid":"becbe6bf-2d52-4f9e-a7e1-b76d55bc0bed","collapsed":true,"_uuid":"080f9f1383191f7381f0e2f7570d2f72b8a47181","trusted":true},"cell_type":"code","source":"import csv\nimport time\nfrom csv import DictReader\nfrom math import exp, log, sqrt\nfrom numba import jit\nimport pandas as pd\nfrom random import randint","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"8d08d5de-cf00-4612-8e5b-dc34f883b6ac","collapsed":true,"_uuid":"7b41890aa2bd879e94dee99cb00a087f4e49e689","trusted":true},"cell_type":"code","source":"data_path = \"../input/\"\ntrain = data_path+'train.csv'\ntrain_s = data_path+'train_sample.csv'                                        # path to training file\ntest = data_path+'test.csv'                 # path to testing file\nsubmission = 'submission.csv' ","execution_count":14,"outputs":[]},{"metadata":{"_cell_guid":"358fcba3-600d-4cf0-93cc-159516a9645d","collapsed":true,"_uuid":"0e3e2c617e35dc35a4fa5762e768741fcd9c9900","trusted":true},"cell_type":"code","source":"alpha = 0.011 # learning rate\nbeta = 0.000000001   # smoothing parameter for adaptive learning rate\nL1 = 0.0000001    # L1 regularization, larger value means more regularized\nL2 = 0.0001","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"126fe099-501f-4dc7-a879-72485d78871f","collapsed":true,"_uuid":"6db8d8b5161f5781f08e435edb9ed445ede0622d","trusted":true},"cell_type":"code","source":"D = 2 ** 26          # number of weights to use\ninteraction = False","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"e88e2eae-d18a-41c8-b412-240c5bab4659","collapsed":true,"_uuid":"cc8479922e312e0892e57adfc6f978f9e9c01a01","trusted":true},"cell_type":"code","source":"epoch = 2       # learn training data for N passes\nholdday = '' ","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"6e6580a6-3007-4754-a408-ddef2705e1e8","collapsed":true,"_uuid":"97eeb8291e601ab72e1ed3eedb8eae5a07a05b89","trusted":true},"cell_type":"code","source":"class ftrl_proximal(object):\n    ''' Our main algorithm: Follow the regularized leader - proximal\n\n        In short,\n        this is an adaptive-learning-rate sparse logistic-regression with\n        efficient L1-L2-regularization\n\n        Reference:\n        http://www.eecs.tufts.edu/~dsculley/papers/ad-click-prediction.pdf\n    '''\n\n    def __init__(self, alpha, beta, L1, L2, D):\n        # parameters\n        self.alpha = alpha\n        self.beta = beta\n        self.L1 = L1\n        self.L2 = L2\n\n        # feature related parameters\n        self.D = D\n        \n\n        # model\n        # n: squared sum of past gradients\n        # z: weights\n        # w: lazy weights\n        self.n = [0.] * D\n        self.z = [0.] * D\n        self.w = {}\n\n    def _indices(self, x):\n        ''' A helper generator that yields the indices in x\n\n            The purpose of this generator is to make the following\n            code a bit cleaner when doing feature interaction.\n        '''\n\n        # first yield index of the bias term\n        yield 0\n\n        # then yield the normal indices\n        for index in x:\n            yield index\n     \n\n    def predict(self, x):\n        ''' Get probability estimation on x\n\n            INPUT:\n                x: features\n\n            OUTPUT:\n                probability of p(y = 1 | x; w)\n        '''\n\n        # parameters\n        alpha = self.alpha\n        beta = self.beta\n        L1 = self.L1\n        L2 = self.L2\n\n        # model\n        n = self.n\n        z = self.z\n        w = {}\n\n        # wTx is the inner product of w and x\n        wTx = 0.\n        for i in self._indices(x):\n            sign = -1. if z[i] < 0 else 1.  # get sign of z[i]\n\n            # build w on the fly using z and n, hence the name - lazy weights\n            # we are doing this at prediction instead of update time is because\n            # this allows us for not storing the complete w\n            if sign * z[i] <= L1:\n                # w[i] vanishes due to L1 regularization\n                w[i] = 0.\n            else:\n                # apply prediction time L1, L2 regularization to z and get w\n                w[i] = (sign * L1 - z[i]) / ((beta + sqrt(n[i])) / alpha + L2)\n\n            wTx += w[i]\n\n        # cache the current w for update stage\n        self.w = w\n\n        # bounded sigmoid function, this is the probability estimation\n        return 1. / (1. + exp(-max(min(wTx, 35.), -35.)))\n\n    def update(self, x, p, y):\n        ''' Update model using x, p, y\n\n            INPUT:\n                x: feature, a list of indices\n                p: click probability prediction of our model\n                y: answer\n\n            MODIFIES:\n                self.n: increase by squared gradient\n                self.z: weights\n        '''\n\n        # parameter\n        alpha = self.alpha\n\n        # model\n        n = self.n\n        z = self.z\n        w = self.w\n\n        # gradient under logloss\n        g = p - y\n       \n        f = 1 if y == 0 else randint(300, 400)\n        ############################################################################\n        ############################################################################\n\n\n        # update z and n\n        for i in self._indices(x):\n            sigma = (sqrt(n[i] + g * g) - sqrt(n[i])) / alpha\n            z[i] += f*(g - sigma * w[i])\n            n[i] += f*(g * g)\n            ","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"87b40d1c-f5df-4954-85de-b4d512b7201c","collapsed":true,"_uuid":"a42a6e5e2caa4c5a8b5cfc04d367196e3cbb469e","trusted":true},"cell_type":"code","source":"def temps(df):\n    date, time = df['click_time'].split(' ')\n    hour = time.split(':')[0]\n    h=int(hour)\n    df[\"Nuit\"]=0\n    df[\"Matin\"]=0\n    df[\"Apres-midi\"]=0\n    df[\"Soir\"]=0\n   \n    if(h>=18):\n        \n        df[\"Soir\"]=1\n        \n    if(h>=0 and h<8):\n        df[\"Nuit\"]=1\n    \n    if(h>=8 and h<12):\n        df[\"Matin\"]=1\n    \n    if(h>=12 and h<18):\n        df[\"Apres-midi\"]=1","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"8d776693-2f15-48c8-8c51-fa9543a1f84a","collapsed":true,"_uuid":"f02798caa98ec23322db26608d73109142eee8cb","trusted":true},"cell_type":"code","source":"def data(path, D):\n    ''' GENERATOR: Apply hash-trick to the original csv row\n                   and for simplicity, we one-hot-encode everything\n\n        INPUT:\n            path: path to training or testing file\n            D: the max index that we can hash to\n\n        YIELDS:\n            ID: id of the instance, mainly useless\n            x: a list of hashed and one-hot-encoded 'indices'\n               we only need the index since all values are either 0 or 1\n            y: y = 1 if we have a click, else we have y = 0\n    '''\n    for t, row in enumerate(DictReader(open(path))):\n        x = []\n        y = 0.\n        \n        # Parse hour and date\n        temps(row)\n        \n        del row['click_time']\n        \n        # process clicks        \n        if 'is_attributed' in row:\n            if row['is_attributed'] == '1':\n                y = 1.\n                #date, tim = row['attributed_time'].split(' ')\n                #ahour = tim.split(':')[0]\n            del row['is_attributed'], row['attributed_time']\n            #x.append(abs(hash('chour_%s___%s_ahour'%(chour, ahour))) % D)\n            \n        try:\n            click_id = row['click_id']\n        except:\n            click_id = ''\n            \n        \n        # Add the rest of the features\n        for k, v in row.items():\n            x.append(abs(hash('%s_%s'%(k, v))) % D)\n        \n        # Add an interaction\n        x.append(abs(hash('%s_os__chl_%s'%(row['channel'], row['os']))) % D)\n        x.append(abs(hash('%s_app_chl_%s'%(row['channel'], row['app']))) % D)\n        x.append(abs(hash('%s_app_os_%s'%(row['os'], row['app']))) % D)\n        x.append(abs(hash('%s_ip_%s_app__device_%s'%(row['ip'],row['app'], row['device']))) % D)\n        \n        yield t, x, y, click_id","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"0cfca30c-3f00-4187-9613-bb325f4c85e0","_uuid":"44b8283e66cb9d486a4c1430205b2b77e8f6bfec","trusted":true,"collapsed":true},"cell_type":"code","source":"learner = ftrl_proximal(alpha, beta, L1, L2, D)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"4e316fab-832a-42c0-b796-70b85a35c575","_uuid":"3077588039a7195f9b117bcb5a2ea13de90fb801","trusted":true},"cell_type":"code","source":"start_time = time.time()\nfor e in range(epoch):\n    for t, x, y, _ in data(train, D):  # data is a generator\n        p = learner.predict(x)\n        learner.update(x, p, y)\n        if t%1000000 == 0:\n            print(\"ligne: %sM ; %sMin \"%( int(t/1e+6), '%0.0f'%((time.time()-start_time)/60)))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"1fbb2358-ee11-4583-ad35-9bd229e5ddfa","scrolled":true,"_uuid":"0039f0bc87dc6774010e0dd272d2b5ae7f55c107","trusted":true},"cell_type":"code","source":"start_time = time.time()\n\nwith open(submission, 'w') as outfile:\n    outfile.write('click_id,is_attributed\\n')\n    for t, x, y, click_id in data(test, D):\n        p = learner.predict(x)\n        outfile.write('%s,%s\\n' % (click_id, str(p)))\n        if t%1000000 == 0:\n            print(\"Test Rows Processed: %sM ; %ss \"%( int(t/1e+6), '%0.0f'%(time.time()-start_time)))\n\nprint('Commit & Run !')","execution_count":15,"outputs":[]},{"metadata":{"_cell_guid":"1888cef1-1cac-4f97-8631-e839a87cd140","_uuid":"80a7a0f028a3479c95f1527dfe0be773100cfa52","trusted":true},"cell_type":"code","source":"import os\nos.remove('./submution1.csv')\nprint(os.listdir(\"./\"))\n","execution_count":16,"outputs":[]},{"metadata":{"_cell_guid":"d429c26f-5686-4e1b-8486-2c3fc33f1d5b","collapsed":true,"_uuid":"cfa4216bf5591a9c38d172c6de66cc2548545595","trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"05d11a4c-ffb7-46d6-8b13-eb3e361186cb","_uuid":"d3d551bdec48ff0575d217aebd50df3544d5acb3","trusted":true},"cell_type":"code","source":"d","execution_count":12,"outputs":[]},{"metadata":{"_cell_guid":"9c42698b-30b5-4cb4-9bda-43dffe32c9bd","collapsed":true,"_uuid":"7547c46782a4f1f793f54a21272730fbf5fd73a9","trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"e12f7cac-56ad-499a-a8b9-42099f2056dd","collapsed":true,"_uuid":"58beed2b6b7d0e1fd83634baef9beda66190e5de","trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"language_info":{"name":"python","version":"3.6.5","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"}},"nbformat":4,"nbformat_minor":1}