{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.ensemble import RandomForestClassifier\nimport os\nimport random\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"aef65a4810bb484b0fd834619796e09406a7b46a"},"cell_type":"code","source":"#Machine Learning","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f45dfa37f3b130aea930691edc9a572972a6abab"},"cell_type":"code","source":"def OHEncode (df, cols):\n    for x in cols:\n        x_ohe = pd.get_dummies(df[x], prefix=x)\n        df = pd.concat([df, x_ohe], axis=1)\n        df = df.drop([x], axis=1)\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9bc613159b7e531466d88e5a085ba2980233c301"},"cell_type":"code","source":"# define columns we want to OHE\n# cols = ['platform','geo_location','dayofweek','hour']\n# cols = ['platform','geo_location','hour']\ncols = ['platform']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1013c88f463608b5b4292dd0bf16ddfd50976964"},"cell_type":"code","source":"testing=False\nchunksize=50000\nclf = LogisticRegression()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e15b0772f265beab48ec8b0cf740272bb59340f4"},"cell_type":"code","source":"train = pd.read_csv(\"../input/train-featured/clicks_events_full.csv\", iterator=True,chunksize=chunksize) #Load data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dca73c4ef65206bdd307e98f00b9126eb06b434c"},"cell_type":"code","source":"print('Chunks training')\nfor chunk in train:\n    chunk = chunk.drop(['geo_location','dayofweek','hour'], axis=1)\n    chunk = OHEncode(chunk,cols) # Perform OHE\n    predictors=[x for x in chunk.columns if x not in ['display_id','clicked']] # Select columns for prediction\n    chunk=chunk.fillna(0.0)\n    clf.fit(chunk[predictors], chunk[\"clicked\"]) #Fit classifier\n    if testing:\n        break\ntrain='' #remove train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fe30fc19f1c53c1c4b909a1fb540c347aeecb812"},"cell_type":"code","source":"chunk.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"31edb90e1384753f9e05f17355a9cad6f01d262e"},"cell_type":"code","source":"test =  pd.read_csv(\"../input/test-featured/test_events_full.csv\",iterator=True,chunksize=chunksize) #Load data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"63e25c3c2355d210eefe5aa51bc2a2b6bbe0f98e"},"cell_type":"code","source":"print('Testing')\npredY=[]\nfor chunk in test:\n    init_chunk_size=len(chunk)\n    chunk = chunk.drop(['geo_location','dayofweek','hour'], axis=1)\n    chunk = OHEncode(chunk,cols)\n    chunk=chunk.fillna(0.0)\n    chunk_pred=list(clf.predict_proba(chunk[predictors]).astype(float)[:,1])\n    predY += chunk_pred\n    if testing:\n        break\nprint('Done Testing')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7b09772c563980cd899888193a234f97349da478"},"cell_type":"code","source":"print('Preparing for Submission')\ntest='' #remove test\ntest= pd.read_csv('../input/outbrain-click-prediction/clicks_test.csv') #load full test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e35de1ecc5620d738142d6bff8e5f48e0568b657"},"cell_type":"code","source":"results=pd.concat((test,pd.DataFrame(predY)) ,axis=1,ignore_index=True) #Combine the predicted values with the ids\nprint(results.head(10))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"94f0c28a3dac42883dfe68edd1074d83e9465817"},"cell_type":"code","source":"results.columns = ['display_id','ad_id','clicked']#Rename the columns\nprint(results.head(10))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"abbafae46837ce1fb5650abf05146c1990db03b5"},"cell_type":"code","source":"results = results.sort_values(by=['display_id','clicked'], ascending=[True, False])\nresults = results.reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5b562428c6dea39f64e382151075e860621ab1d4"},"cell_type":"code","source":"results2 = results.copy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b16e6c5d34a22a3482ae75d3d5104dee989ddeeb"},"cell_type":"code","source":"submission_data = results2.groupby('display_id').ad_id.apply(lambda x: \" \".join(map(str,x))).reset_index()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a854de4c7aef629718251dae8df5240805b4dc1f","scrolled":false},"cell_type":"code","source":"submission_data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9c419acd398107e061f52cda60348475394417b0"},"cell_type":"code","source":"submission_data.to_csv('submission_feature_removed.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e84a5afe6338dff3c0cd270d5df31d1222d5032a"},"cell_type":"code","source":"#results=results[['display_id','ad_id']].groupby('display_id')['ad_id'].agg(lambda col: ' '.join(map(str,col)))\n#results.columns=[['display_id','ad_id']]\n#results.to_csv('submission_final.csv')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}