{
  "id": 41305,
  "title": "some code to split the user_logs.csv without lots of ram",
  "url": "/competitions/kkbox-churn-prediction-challenge/discussion/41305",
  "author_name": "liulongxiao",
  "post_date": "2017-10-16T09:59:53.203000",
  "votes": 1,
  "comment_count": 0,
  "views": 0,
  "content": "<p>train_set=set(pd.read_csv('kkbox_data/train.csv').msno)##992931\ntest_set=set(pd.read_csv('kkbox_data/sample_submission_zero.csv').msno)##970960</p>\n\n<p>class filewriter:\n    def <strong>init</strong>(self,filename,chunksize):\n        self.filename=filename\n        self.chunksize=chunksize\n        self.line_count=0\n        self.cursor=None\n        self.open()</p>\n\n<pre><code>def close(self):\n    self.cursor.close()\n\ndef open(self):\n    self.cursor=open(self.filename,'a')\n    self.line_count=0\n\ndef write(self,string):\n    if self.line_count&gt;self.chunksize:\n        self.close()\n        self.open()\n    self.cursor.write(string)\n    self.line_count+=1\n\ndef __eq__(self,other):\n    return self.filename==other\n</code></pre>\n\n<p>def split_csv(filename,dir,msno_set):\n    f=open(filename,'r')\n    columns=f.readline()\n    filewriters={}\n    while  True:\n        line = f.readline()\n        if len(line) == 0:\n            break\n        data = line.replace('\\n', '').split(',')\n        if data[0] not in msno_set:\n            continue\n        if data[0][0] not in filewriters.keys():\n            filewriters[data[0][0]]=filewriter(\"{}/{}.csv\".format(dir,data[0][0]),100000)\n            filewriters[data[0][0]].write(columns)\n        filewriters[data[0][0]].write(line)</p>\n\n<p>split_csv('kkbox_data/user_logs.csv','split_userlog',train_set.union(test_set))</p>",
  "messages": [
    {
      "id": 231877,
      "postDate": "2017-10-16T09:59:53.203Z",
      "content": "<p>train_set=set(pd.read_csv('kkbox_data/train.csv').msno)##992931\ntest_set=set(pd.read_csv('kkbox_data/sample_submission_zero.csv').msno)##970960</p>\n\n<p>class filewriter:\n    def <strong>init</strong>(self,filename,chunksize):\n        self.filename=filename\n        self.chunksize=chunksize\n        self.line_count=0\n        self.cursor=None\n        self.open()</p>\n\n<pre><code>def close(self):\n    self.cursor.close()\n\ndef open(self):\n    self.cursor=open(self.filename,'a')\n    self.line_count=0\n\ndef write(self,string):\n    if self.line_count&gt;self.chunksize:\n        self.close()\n        self.open()\n    self.cursor.write(string)\n    self.line_count+=1\n\ndef __eq__(self,other):\n    return self.filename==other\n</code></pre>\n\n<p>def split_csv(filename,dir,msno_set):\n    f=open(filename,'r')\n    columns=f.readline()\n    filewriters={}\n    while  True:\n        line = f.readline()\n        if len(line) == 0:\n            break\n        data = line.replace('\\n', '').split(',')\n        if data[0] not in msno_set:\n            continue\n        if data[0][0] not in filewriters.keys():\n            filewriters[data[0][0]]=filewriter(\"{}/{}.csv\".format(dir,data[0][0]),100000)\n            filewriters[data[0][0]].write(columns)\n        filewriters[data[0][0]].write(line)</p>\n\n<p>split_csv('kkbox_data/user_logs.csv','split_userlog',train_set.union(test_set))</p>",
      "rawMarkdown": "train_set=set(pd.read_csv('kkbox_data/train.csv').msno)##992931\ntest_set=set(pd.read_csv('kkbox_data/sample_submission_zero.csv').msno)##970960\n\nclass filewriter:\n    def __init__(self,filename,chunksize):\n        self.filename=filename\n        self.chunksize=chunksize\n        self.line_count=0\n        self.cursor=None\n        self.open()\n\n    def close(self):\n        self.cursor.close()\n\n    def open(self):\n        self.cursor=open(self.filename,'a')\n        self.line_count=0\n\n    def write(self,string):\n        if self.line_count&gt;self.chunksize:\n            self.close()\n            self.open()\n        self.cursor.write(string)\n        self.line_count+=1\n\n    def __eq__(self,other):\n        return self.filename==other\n\ndef split_csv(filename,dir,msno_set):\n    f=open(filename,'r')\n    columns=f.readline()\n    filewriters={}\n    while  True:\n        line = f.readline()\n        if len(line) == 0:\n            break\n        data = line.replace('\\n', '').split(',')\n        if data[0] not in msno_set:\n            continue\n        if data[0][0] not in filewriters.keys():\n            filewriters[data[0][0]]=filewriter(\"{}/{}.csv\".format(dir,data[0][0]),100000)\n            filewriters[data[0][0]].write(columns)\n        filewriters[data[0][0]].write(line)\n\nsplit_csv('kkbox_data/user_logs.csv','split_userlog',train_set.union(test_set))",
      "votes": 1
    }
  ],
  "comments": [],
  "raw_markdown_by_id": {
    "231877": "train_set=set(pd.read_csv('kkbox_data/train.csv').msno)##992931\ntest_set=set(pd.read_csv('kkbox_data/sample_submission_zero.csv').msno)##970960\n\nclass filewriter:\n    def __init__(self,filename,chunksize):\n        self.filename=filename\n        self.chunksize=chunksize\n        self.line_count=0\n        self.cursor=None\n        self.open()\n\n    def close(self):\n        self.cursor.close()\n\n    def open(self):\n        self.cursor=open(self.filename,'a')\n        self.line_count=0\n\n    def write(self,string):\n        if self.line_count&gt;self.chunksize:\n            self.close()\n            self.open()\n        self.cursor.write(string)\n        self.line_count+=1\n\n    def __eq__(self,other):\n        return self.filename==other\n\ndef split_csv(filename,dir,msno_set):\n    f=open(filename,'r')\n    columns=f.readline()\n    filewriters={}\n    while  True:\n        line = f.readline()\n        if len(line) == 0:\n            break\n        data = line.replace('\\n', '').split(',')\n        if data[0] not in msno_set:\n            continue\n        if data[0][0] not in filewriters.keys():\n            filewriters[data[0][0]]=filewriter(\"{}/{}.csv\".format(dir,data[0][0]),100000)\n            filewriters[data[0][0]].write(columns)\n        filewriters[data[0][0]].write(line)\n\nsplit_csv('kkbox_data/user_logs.csv','split_userlog',train_set.union(test_set))"
  }
}