{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# IMPORTING NECESSARY MODULES FOR DATA ANALYSIS AND PREDICTIVE MODELLING\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.metrics import accuracy_score,confusion_matrix\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.model_selection import train_test_split\nimport xgboost as xgb\nimport lightgbm as lgb\nimport re\nimport gc\nimport os\nimport psutil\nimport humanize\nfrom sklearn.model_selection import KFold, StratifiedKFold\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nfrom IPython.display import HTML, display, clear_output\nimport warnings\nwarnings.filterwarnings('ignore')\n%matplotlib inline\npd.set_option('display.max_rows', 500)\npd.set_option('display.max_columns', 500)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"print(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"743e7859c3d5722d9cf36bee2eacb188de388126"},"cell_type":"code","source":"DIR = '../input/test/'\nprint(len([name for name in os.listdir(DIR) if os.path.isfile(os.path.join(DIR, name))]))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c21b6c3325e6396bbc3d2b5cf34b004bdbba282a"},"cell_type":"markdown","source":"# From above code patch we can see that we have around 2624 test files "},{"metadata":{"trusted":true,"_uuid":"26b9bb8982777003ed1ab3a06c427d3c8a869074"},"cell_type":"code","source":"TrainDataPath = '../input/train.csv'\nTestDataPath = '../input/test/seg_00030f.csv' # Randomly taking a sample test data\nSubDataPath = '../input/sample_submission.csv'\n\n# Loading the Training Dataset and Submission File\nTrainData = pd.read_csv(TrainDataPath, nrows=10000000, dtype={'acoustic_data': np.int16, 'time_to_failure': np.float64})\nTestData = pd.read_csv(TestDataPath, dtype={'acoustic_data': np.int16, 'time_to_failure': np.float64})\nSubData = pd.read_csv(SubDataPath)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7fd0daab8307c7dc13dd37c312606b2b9ba50f50"},"cell_type":"code","source":"print(\"Training Dataset Shape:\")\nprint(TrainData.shape)\nprint(\"\\n\")\nprint(\"Training Dataset Columns/Features:\")\nprint(TrainData.dtypes)\nTrainData.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f1e6ac69cb6eeb3ad8da039f38f32488ca42ab94"},"cell_type":"code","source":"print(\"Test Dataset Shape:\")\nprint(TestData.shape)\nprint(\"\\n\")\nprint(\"Test Dataset Columns/Features:\")\nprint(TestData.dtypes)\nTestData.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"78f96228d306401b2e3c9171df84c8d72b84f730"},"cell_type":"code","source":"print(\"Submission Dataset Shape:\")\nprint(SubData.shape)\nprint(\"\\n\")\nprint(\"Submission Dataset Columns/Features:\")\nprint(SubData.dtypes)\nSubData.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"aeead7ad7dfb8d58fba34169c9119de4c6c91bc5"},"cell_type":"code","source":"# checking missing data percentage in train data\ntotal = TrainData.isnull().sum().sort_values(ascending = False)\npercent = (TrainData.isnull().sum()/TrainData.isnull().count()*100).sort_values(ascending = False)\nmissing_TrainData  = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])\nmissing_TrainData.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0a8c3c6a8d7ba60f8dddde23c055658a7f8cd3b1"},"cell_type":"code","source":"# checking missing data percentage in test data\ntotal = TestData.isnull().sum().sort_values(ascending = False)\npercent = (TestData.isnull().sum()/TestData.isnull().count()*100).sort_values(ascending = False)\nmissing_TrainData  = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])\nmissing_TrainData.head(10)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ef97503fae3d6bc08110252e270ea5367ff5da46"},"cell_type":"markdown","source":"# HELPER FUNCTIONS"},{"metadata":{"trusted":true,"_uuid":"dd4e7eb47fae015f652be568fafe86fdee3ed252"},"cell_type":"code","source":"def printmemusage():\n process = psutil.Process(os.getpid())\n print(\"Gen RAM Free: \" + humanize.naturalsize( psutil.virtual_memory().available ), \" | Proc size: \" + humanize.naturalsize( process.memory_info().rss))\nprintmemusage()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6adb82b4194b55ee194c52c3339633754d074c0f"},"cell_type":"code","source":"def plot_bar_counts_categorical(data_se, title, figsize, sort_by_counts=False):\n    info = data_se.value_counts()\n    info_norm = data_se.value_counts(normalize=True)\n    categories = info.index.values\n    counts = info.values\n    counts_norm = info_norm.values\n    fig, ax = plt.subplots(figsize=figsize)\n    if data_se.dtype in ['object']:\n        if sort_by_counts == False:\n            inds = categories.argsort()\n            counts = counts[inds]\n            counts_norm = counts_norm[inds]\n            categories = categories[inds]\n        ax = sns.barplot(counts, categories, orient = \"h\", ax=ax)\n        ax.set(xlabel=\"count\", ylabel=data_se.name)\n        ax.set_title(\"Distribution of \" + title)\n        for n, da in enumerate(counts):\n            ax.text(da, n, str(da)+ \",  \" + str(round(counts_norm[n]*100,2)) + \" %\", fontsize=10, va='center')\n    else:\n        inds = categories.argsort()\n        counts_sorted = counts[inds]\n        counts_norm_sorted = counts_norm[inds]\n        ax = sns.barplot(categories, counts, orient = \"v\", ax=ax)\n        ax.set(xlabel=data_se.name, ylabel='count')\n        ax.set_title(\"Distribution of \" + title)\n        for n, da in enumerate(counts_sorted):\n            ax.text(n, da, str(da)+ \",  \" + str(round(counts_norm_sorted[n]*100,2)) + \" %\", fontsize=10, ha='center')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"333ca6c71e6225b34c656de70e43955d1485c73a"},"cell_type":"code","source":"def count_plot_by_hue(data_se, hue_se, title, figsize, sort_by_counts=False):\n    if sort_by_counts == False:\n        order = data_se.unique()\n        order.sort()\n    else:\n        order = data_se.value_counts().index.values\n    off_hue = hue_se.nunique()\n    off = len(order)\n    fig, ax = plt.subplots(figsize=figsize)\n    ax = sns.countplot(y=data_se, hue=hue_se, order=order, ax=ax)\n    ax.set_title(title)\n    patches = ax.patches\n    for i, p in enumerate(ax.patches):\n        x=p.get_bbox().get_points()[1,0]\n        y=p.get_bbox().get_points()[:,1]\n        total = x\n        p = i\n        q = i\n        while(q < (off_hue*off)):\n            p = p - off\n            if p >= 0:\n                total = total + (patches[p].get_bbox().get_points()[1,0] if not np.isnan(patches[p].get_bbox().get_points()[1,0]) else 0)\n            else:\n                q = q + off\n                if q < (off*off_hue):\n                    total = total + (patches[q].get_bbox().get_points()[1,0] if not np.isnan(patches[q].get_bbox().get_points()[1,0]) else 0)\n       \n        perc = str(round(100*(x/total), 2)) + \" %\"\n        \n        if not np.isnan(x):\n            ax.text(x, y.mean(), str(int(x)) + \",  \" + perc, va='center')\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ada6efa0bc89bba3df882b38a026d14a29b707f2"},"cell_type":"code","source":"def show_unique(data_se):\n    display(HTML('<h5><font color=\"green\"> Shape Of Dataset Is: ' + str(data_se.shape) + '</font></h5>'))\n    for i in data_se.columns:\n        if data_se[i].nunique() == data_se.shape[0]:\n            display(HTML('<font color=\"red\"> ATTENTION!!! ' + str(i+' --> '+str(data_se[i].nunique())) + '</font>'))\n        elif (data_se[i].nunique() == 1):\n            display(HTML('<font color=\"Blue\"> ATTENTION!!! ' + str(i+' --> '+str(data_se[i].nunique())) + '</font>'))\n        else:\n            print(i+' -->', data_se[i].nunique())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c339ce0288685a828177c7246a29d8eb187e44f7"},"cell_type":"code","source":"def show_countplot(data_se):\n    display(HTML('<h2><font color=\"blue\"> Dataset CountPlot Visualization: </font></h2>'))\n    for i in data_se.columns:\n        if (data_se[i].nunique() <= 10):\n            plot_bar_counts_categorical(data_se[i].astype(str), 'Dataset Column: '+ i, (15,7))\n        elif (data_se[i].nunique() > 10 and data_se[i].nunique() <= 20):\n            plot_bar_counts_categorical(data_se[i].astype(str), 'Dataset Column: '+ i, (15,12))\n        else:\n            print('Columns do not fit in display '+i+' -->', data_se[i].nunique())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"da4c48816f1f85aefa1a3e07ec062015986227d3"},"cell_type":"code","source":"gc.collect() # Python garbage collection module for dereferencing the memory pointers and making memory available for better usage","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"beec76a9e4fedf0b86b2efcbf4e5109f592c24b2"},"cell_type":"markdown","source":"# Ok Now We Should Start Getting Some Insights About the Data"},{"metadata":{"trusted":true,"_uuid":"ccb2501dbfd87a481f0182b6d0abb905f988ca41"},"cell_type":"code","source":"TrainData.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"78dec2ff6fd80a618f903d7aec8f4c7346cf9932"},"cell_type":"code","source":"TestData.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0d9504715a18ab80c0198e413e3541b30113bba4"},"cell_type":"code","source":"# show_unique function shows the no of unique values present in the each column of the dataset\nshow_unique(TrainData)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"140c8187b93eec17526d17b7197e84dd65252b8a"},"cell_type":"code","source":"show_unique(TestData)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f10fa8fe8ef0e9e6b76edfde6cc9eec8f641a4b8"},"cell_type":"code","source":"# TRAIN DATA HeatMap\nf,ax = plt.subplots(figsize=(10, 5))\nsns.heatmap(TrainData.corr(), annot=True, linewidths=.2, fmt= '.1f',ax=ax,cmap='Blues')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a1a14717eafd3512be813adf917d00e786e8025e"},"cell_type":"markdown","source":"### We can't see any correlation among the columns "},{"metadata":{"trusted":true,"_uuid":"d7307215dd568603000cc06ee68ee84df76214f1"},"cell_type":"code","source":"plt.figure(figsize=(18, 5))\nsns.distplot((TrainData[\"acoustic_data\"]))\nplt.title('TRAIN DATA')\nplt.show()\n\nplt.figure(figsize=(18, 5))\nsns.distplot((TrainData[\"time_to_failure\"]))\nplt.title('TRAIN DATA')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"66c1fe05bc0212c7de014093f99c502a958668eb"},"cell_type":"code","source":"fig, ax = plt.subplots(2,1, figsize=(18,12))\nax[1].plot(TrainData.index.values, TrainData.time_to_failure.values, c=\"darkred\")\nax[1].set_title(\"Quaketime of 10 Mio rows\")\nax[1].set_xlabel(\"Index\")\nax[1].set_ylabel(\"Quaketime in ms\");\nax[0].plot(TrainData.index.values, TrainData.acoustic_data.values, c=\"mediumseagreen\")\nax[0].set_title(\"Signal of 10 Mio rows\")\nax[0].set_xlabel(\"Index\")\nax[0].set_ylabel(\"Acoustic Signal\");","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8862d4e9fa77add5bb034b536a2a960307cd9aef"},"cell_type":"code","source":"fig, ax = plt.subplots(3,1,figsize=(18,18))\nax[0].plot(TrainData.index.values[0:50000], TrainData.time_to_failure.values[0:50000], c=\"Red\")\nax[0].set_xlabel(\"Index\")\nax[0].set_ylabel(\"Time to quake\")\nax[0].set_title(\"How does the second quaketime pattern look like?\")\nax[1].plot(TrainData.index.values[0:49999], np.diff(TrainData.time_to_failure.values[0:50000]))\nax[1].set_xlabel(\"Index\")\nax[1].set_ylabel(\"Difference between quaketimes\")\nax[1].set_title(\"Are the jumps always the same?\")\nax[2].plot(TrainData.index.values[0:4000], TrainData.time_to_failure.values[0:4000])\nax[2].set_xlabel(\"Index from 0 to 4000\")\nax[2].set_ylabel(\"Quaketime\")\nax[2].set_title(\"How does the quaketime changes within the first block?\");","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"92c24ecf139cfcfe309ad8ea3b1bd55cec3f03a0"},"cell_type":"markdown","source":"# Please Upvote, Your Support Is Very Much Required."},{"metadata":{"_uuid":"0b039a8fcf64aa1310e916cf7da8523ff198df0e"},"cell_type":"markdown","source":"# I Will Be Soon Updating The Whole Notebook."},{"metadata":{"trusted":true,"_uuid":"f1733aa01879658cb498d44dd240f7d3e7c59491"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}