{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","collapsed":true,"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":false},"cell_type":"code","source":"#Inspirational Notebooks\n#https://www.kaggle.com/nanomathias/feature-engineering-importance-testing\n#https://www.kaggle.com/shep312/single-generalised-lightgbm-lb-0-9686/code\n#\nimport pandas as pd\nfrom IPython.display import display\nimport numpy as np\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.model_selection import cross_val_score\nimport gc\nimport os\nimport matplotlib.pyplot as plt\nfrom keras.models import Sequential\nfrom keras.layers import Dense,Dropout\nfrom keras.utils import np_utils\nfrom keras import optimizers\nfrom sklearn.preprocessing import MinMaxScaler\n\nos.environ['OMP_NUM_THREADS'] = '4'  # Number of threads on the Kaggle server\n\ntrain_cols = ['ip', 'app', 'device', 'os', 'channel', 'click_time', 'is_attributed']\ntest_cols = ['ip', 'app', 'device', 'os', 'channel', 'click_time']\n\ndtypes = {\n    'ip'            : 'uint32',\n    'app'           : 'uint16',\n    'device'        : 'uint16',\n    'os'            : 'uint16',\n    'channel'       : 'uint16',\n    'is_attributed' : 'uint8',\n    'click_id'      : 'uint32'\n}\n\nn_rows_to_train = 3000000\nn_rows_to_test = 1000000\n\ndummycol = ['ip_cat','app','os']\nnotcol = ['click_id','is_attributed']\n\n\ndef load_talking_train_big_data(n_rows):\n    data = pd.read_csv(\"D:/Kaggle/TalkingData/train.csv\", dtype=dtypes, nrows=n_rows, usecols=train_cols)   \n    return data\n\ndef load_test_model_data(n_rows,skiprows):\n    train_cols = ['ip', 'app', 'device', 'os', 'channel', 'click_time', 'is_attributed']\n    data_test = pd.read_csv(\"D:/Kaggle/TalkingData/train.csv\",\n                            dtype=dtypes, nrows=n_rows, skiprows = range(1,skiprows), usecols=train_cols)\n    return data_test\n\ndef load_test_submission_data():\n    data_test = pd.read_csv(\"D:/Kaggle/TalkingData/test.csv\", dtype=dtypes, usecols=test_cols)\n    return data_test\n\ndef get_features(td):\n    \n    GROUP_AGG_FLAG = True\n    GROUP_NEXT_CLICKS_FLAG = True\n    HISTORY_CLICKS_FLAG = True\n\n    td['click_time'] = pd.to_datetime(td.click_time)\n    td['hour'] = td.click_time.dt.hour.astype('uint8')\n    td['day'] = td.click_time.dt.day.astype('uint8')\n    \n    #add a section of day\n    day_section = 0\n    for start_time, end_time in zip([0,6,12,18],[6,12,18,24]):\n        td.loc[(td['hour'] >= start_time) & (td['hour'] < end_time), 'day_section'] = day_section\n        day_section +=1\n    td['day_section'] = td['day_section'].astype('uint8')\n    gc.collect()\n    \n    if(GROUP_AGG_FLAG==True):\n        \n        GROUPBY_AGGREGATIONS = [\n\n            # V1 - GroupBy Features #\n            #########################    \n            # Count, for ip\n            {'groupby': ['ip'], 'select': 'channel', 'agg': 'count'},\n            # Count, for ip-day-hour\n            {'groupby': ['ip','day','hour'], 'select': 'channel', 'agg': 'count'},\n            # Count, for ip-app\n            {'groupby': ['ip', 'app'], 'select': 'channel', 'agg': 'count'},        \n            # Count, for ip-device\n            {'groupby': ['ip', 'device'], 'select': 'channel', 'agg': 'count'},\n            # Count, for ip-device-os\n            {'groupby': ['ip', 'device','os'], 'select': 'channel', 'agg': 'count'},\n\n            # V2 - GroupBy Features #\n            #########################\n            # Average clicks on app by distinct users; is it an app they return to?\n            {'groupby': ['app'], \n             'select': 'ip', \n             'agg': lambda x: float(len(x)) / len(x.unique()), \n             'agg_name': 'AvgViewPerDistinct'\n            },\n            # How popular is the app or channel?\n            {'groupby': ['app'], 'select': 'channel', 'agg': 'count'},\n            {'groupby': ['channel'], 'select': 'app', 'agg': 'count'}\n        ]\n\n        for spec in GROUPBY_AGGREGATIONS:\n            agg_name = spec['agg_name'] if 'agg_name' in spec else spec['agg']\n            print(\"Grouping by {}, and aggregating {} with {}\".format(spec['groupby'], spec['select'], agg_name))        \n            list_features = list(set(spec['groupby']+[spec['select']]))\n            name_feature = \"{}_{}_{}\".format('_'.join(spec['groupby']),agg_name,spec['select'])\n\n            new_feature = td[list_features].groupby(spec['groupby'])[spec['select']].agg(spec['agg']).\\\n                reset_index().rename(index=str,columns={spec['select']:name_feature}).fillna(0)\n            gc.collect()\n            td = td.merge(new_feature,on=spec['groupby'],how=\"left\")\n            del(new_feature)\n            gc.collect()\n        \n        #add category of ip\n        td.loc[td['ip_count_channel'] == 1, 'ip_cat'] = 0\n        cat = 1\n        for start, end in zip([2,5,10,15,25,50],[5,10,15,25,50,100]):\n            td.loc[(td['ip_count_channel'] >= start) & (td['ip_count_channel'] < end) , 'ip_cat'] = cat\n            cat +=1\n        td.loc[td['ip_count_channel'] >= 100, 'ip_cat'] = cat\n        td['ip_cat'] = td['ip_cat'].astype('uint8')\n        gc.collect()\n\n    if(GROUP_NEXT_CLICKS_FLAG == True):  \n        \n        #add features of time till next click\n        GROUP_BY_NEXT_CLICKS = [\n            {'groupby': ['ip']},\n            {'groupby': ['ip', 'app']},\n            {'groupby': ['ip', 'channel']},\n        ]\n\n        for spec in GROUP_BY_NEXT_CLICKS:\n            new_feature = '{}_nextClick'.format('_'.join(spec['groupby']))    \n            # Unique list of features to select\n            list_features = spec['groupby'] + ['click_time']\n            td[new_feature] = td[list_features].groupby(spec['groupby']).click_time.\\\n                transform(lambda x: x.diff().shift(-1)).dt.seconds.fillna(0)\n            gc.collect()\n            \n    if(HISTORY_CLICKS_FLAG==True):\n        \n        # add features of prev and next clicks\n        HISTORY_CLICKS = {\n            'identical_clicks': ['ip', 'app', 'device', 'os', 'channel'],\n            'ip_device_os_clicks': ['ip', 'device', 'os'],\n            'app_clicks': ['ip', 'app']\n        }\n\n        for fname, fset in HISTORY_CLICKS.items():\n            # prev clicks\n            td['prev_'+fname] = td.groupby(fset).cumcount().rename('prev_'+fname)\n            # next clicks\n            td['future_'+fname] = td.iloc[::-1].groupby(fset).cumcount().rename('future_'+fname).iloc[::-1]\n        gc.collect()\n    del(td['ip'])\n    gc.collect()\n    \n    return td\n\ndef convert_preds(raw_preds):\n    preds = []\n    for p in raw_preds:\n        preds.append(1 - p[0])\n    return preds\n\ndef check_model(model,scaler):\n    test_check = load_test_model_data(n_rows_to_test,n_rows_to_train)\n    print(\"test data uploaded\")\n    test_check = get_features(test_check)\n    y_test = test_check['is_attributed']\n    y_test = np_utils.to_categorical(y_test, 2)\n    \n    X_test = test_check.drop('is_attributed', axis=1).select_dtypes(include=[np.number])\n    del(test_check)\n    gc.collect()\n    \n    scalercol = [col for col in X_test.columns if col not in (dummycol+notcol)]\n    \n    for col_title in dummycol:\n        dummy = pd.get_dummies(X_test[X_test[col_title].isin(uniq_dict[col_title])][col_title],\n                               columns=[col_title], prefix=col_title)\n        \n        if(len(uniq_dict[col_title]) > dummy.shape[1]):\n            for num in uniq_dict[col_title]:\n                if('_'.join([col_title,num.astype('str')]) not in dummy.columns):\n                    dummy['_'.join([col_title,num.astype('str')])] = 0\n        X_test = pd.concat([dummy,X_test],axis=1)\n        \n        del(dummy)\n        gc.collect()\n        \n    X_test[scalercol] = scaler.transform(X_test[scalercol])\n    \n    del(scaler)\n    gc.collect()\n        \n    scores_test = model.evaluate(X_test, y_test, batch_size=100, verbose=0)\n    print(\"Result on test data: {:.2f}\".format(scores_test[1]*100))\n    \n    del(y_test,X_test)\n    gc.collect()\n\n\ndef make_submission(model_data,scaler):\n    test_data = load_test_submission_data()\n    print(\"Submission-data uploaded\")\n    test_data = get_features(test_data).select_dtypes(include=[np.number])\n    \n    scalercol = [col for col in test_data.columns if col not in (dummycol+notcol)]\n    \n    for col_title in dummycol:\n        dummy = pd.get_dummies(test_data[test_data[col_title].isin(uniq_dict[col_title])][col_title],\n                               columns=[col_title], prefix=col_title)\n        if(len(uniq_dict[col_title]) > dummy.shape[1]):\n            for num in uniq_dict[col_title]:\n                if('_'.join([col_title,num.astype('str')]) not in dummy.columns):\n                    dummy['_'.join([col_title,num.astype('str')])] = 0\n        test_data = pd.concat([dummy,test_data],axis=1)\n        del(dummy)\n        gc.collect() \n        \n    test_data[scalercol] = scaler.transform(test_data[scalercol])\n    submit = pd.read_csv(\"D:/Kaggle/TalkingData/test.csv\", dtype='int', usecols=['click_id'],nrows=500000)\n    gc.collect()\n    submit['is_attributed'] = convert_preds(model_data.predict_proba(test_data, batch_size=100))\n    del(test_data)\n    gc.collect()\n    submit.to_csv('D:/Kaggle/TalkingData/result_keras.csv', index=False)\n    print(\"submission-file is created\")\n\n    \ntrain_data = load_talking_train_big_data(n_rows_to_train)\nprint(\"Train data uploaded. Count of lines: {}\".format(train_data.shape[0]))\ntrain_data = get_features(train_data)\ny = train_data['is_attributed']\ny = np_utils.to_categorical(y, 2)\nX = train_data.drop('is_attributed', axis=1).select_dtypes(include=[np.number])\ndel(train_data)\ngc.collect()\n\nscalercol = [col for col in X.columns if col not in (dummycol+notcol)]\nuniq_dict = {col:[] for col in dummycol}\n\n#add dummy-variables for selected features\nfor col_title in dummycol:\n    uniq_dict[col_title] = pd.unique(X[col_title].values)\n    dummy = pd.get_dummies(X[col_title], columns=[col_title], prefix=col_title)\n    X = pd.concat([dummy,X],axis=1)\n    del(dummy)\n    gc.collect()\n    \n#scale other features\nscaler = MinMaxScaler()\nX[scalercol] = scaler.fit_transform(X[scalercol])\n\nmodel = Sequential()\nmodel.add(Dense(100,input_dim=X.shape[1],activation=\"relu\", kernel_initializer=\"normal\"))\nmodel.add(Dense(100, activation=\"relu\"))\nmodel.add(Dense(50, activation=\"relu\"))\nmodel.add(Dense(2, activation=\"softmax\"))\n#sgd = optimizers.SGD(lr=0.01, momentum=0.9, decay=1e-6, nesterov=True)\nada = optimizers.Adadelta(lr=0.1, rho=0.95, epsilon=None, decay=0.0)\nmodel.compile(loss=\"categorical_crossentropy\", optimizer=ada, metrics=[\"accuracy\"])\n\nmodel.fit(X, y, batch_size=100, epochs=3, validation_split=0.15)\n\nprint(\"Model trained\")\nscores_train = model.evaluate(X, y, batch_size=100, verbose=0)\nprint(\"Result on train data: {:.2f}\".format(scores_train[1]*100))\n\ndel(X,y)\ngc.collect()\n\n#check_model(model,scaler)\nmake_submission(model,scaler)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","collapsed":true,"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"language_info":{"name":"python","version":"3.6.5","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"}},"nbformat":4,"nbformat_minor":1}