{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Feel free to download the compressed file containing .pkl datasets from this folder. I have already run this notebook and uploaded the output. All float64 and int64 columns are converted to float16 and int8. In order to read the files, you can use **pd.read_pickle()**. This method will save loading time and reduce RAM usage dramatically. ","metadata":{}},{"cell_type":"markdown","source":"#### Reduce Dataset Size and Create Pickle format","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport gc","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = pd.read_csv('../input/amex-default-prediction/train_data.csv', index_col='customer_ID')","metadata":{"execution":{"iopub.status.busy":"2022-05-28T08:19:09.915262Z","iopub.execute_input":"2022-05-28T08:19:09.916219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.info()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.dtypes.value_counts()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"float64_columns = train_data.columns[train_data.dtypes == 'float64'].values\nobject_columns = train_data.columns[train_data.dtypes == 'O'].values\nint64_columns = train_data.columns[train_data.dtypes == 'int64'].values","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"float64_columns","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"object_columns","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"int64_columns","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data[float64_columns] = train_data[float64_columns].astype('float16')\ntrain_data[int64_columns] = train_data[int64_columns].astype('int8')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.info()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.to_pickle('train_data.pkl')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train_data\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data = pd.read_csv('../input/amex-default-prediction/test_data.csv', index_col='customer_ID')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data.info()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data[float64_columns] = test_data[float64_columns].astype('float16')\ntest_data[int64_columns] = test_data[int64_columns].astype('int8')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data.info()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data.to_pickle('test_data.pkl')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del test_data\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_labels = pd.read_csv('../input/amex-default-prediction/train_labels.csv', index_col='customer_ID')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_labels.info()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_labels['target'] = train_labels['target'].astype('int8')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_labels.info()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_labels.to_pickle('train_labels.pkl')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission = pd.read_csv('../input/amex-default-prediction/sample_submission.csv', index_col='customer_ID')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission.info()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission['prediction'] = sample_submission['prediction'].astype('int8')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission.info()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission.to_pickle('sample_submission.pkl')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train_labels, sample_submission, float64_columns, int64_columns, object_columns\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}