# -*- coding: utf-8 -*-

import sys
import couchdb
from mailboxes__jwzthreading import thread, Message
from mailboxes__CouchDBBulkReader import CouchDBBulkReader
from datetime import datetime as dt
from prettytable import PrettyTable

try:
    import jsonlib2 as json
except:
    import json

DB = sys.argv[1]
NUM_PROC_THREADS = 3 # Recommendation: ~1 thread/core

# Pull the data as efficient as possible from CouchDB by using a thread
# pool to get as close as possible to being I/O bound.
# A single request to _all_docs works except that it CPU bound to a single core

now = dt.now()
print >> sys.stderr, 'Bulk reading from CouchDB...'
br = CouchDBBulkReader(DB, NUM_PROC_THREADS)
docs = br.read()
print >> sys.stderr, '	%s' % (dt.now() - now, )

now = dt.now()
print >> sys.stderr, 'Threading in Memory...'
threads = thread([Message(doc) for doc in docs])
print >> sys.stderr, '	%s' % (dt.now() - now, )

# Write out threading info into another database.
# Note that writes to CouchDB are serialized to append-only
# databases, so threading is unlikely to help here, and besides,
# the average document size is very small, making this a quick operation

now = dt.now()
print >> sys.stderr, 'Bulk writing to CouchDB...'
server = couchdb.Server('http://localhost:5984')
db = server.create(DB + '-threads')
results = db.update([{'thread': thread} for thread in threads],
                    all_or_nothing=True)
print >> sys.stderr, '	%s' % (dt.now() - now, )

# Some basic stats

print >> sys.stderr, 'Total number of threads: %s' % len(threads)
print >> sys.stderr

# Compute (_id, len(thread)) tuples
# You could also compute thread length directly in CouchDB using a simple reducer function

stats = sorted(zip([result[1] for result in results], [len(t) for t in threads]),
               key=lambda x: x[1])

fields = ['Thread Id', 'Thread Length']
pt = PrettyTable(fields=fields)
[pt.set_field_align(f, 'l') for f in fields]

for stat in stats:
    pt.add_row(stat)

pt.printt()

{
   "_id": "b6d4f96224bc546acd34c405e6fff62f",
   "_rev": "1-1bf63dcdd94067ad647afe2ea3ade63c",
   "thread": [
       {
           "external_id": "24a30d62545728e26eb3311d63ae6e02",
           "subject": "FW: Sitara EOL Bridge Problem Today"
       },
       {
           "external_id": "bb808c9081912f5861295bf1d105dd02",
           "subject": "FW: Sitara EOL Bridge Problem Today"
       },
       {
           "external_id": "3b2c340c28782c8986737c35a332cd88",
           "subject": "FW: Sitara EOL Bridge Problem Today"
       }
   ]
   }

# -*- coding: utf-8 -*-

from datetime import datetime as dt
from math import ceil
import httplib
import urllib
import time
import threadpool
try:
    import jsonlib2 as json
except:
    import json


class CouchDBBulkReader:

    def __init__(
        self,
        db,
        num_threads,
        host='localhost',
        port=5984,
        ):

        self.db = db
        self.num_threads = num_threads
        self.host = host
        self.port = port
        self.results = []

        id_buckets = self._getDocIds()
        self.pool = threadpool.ThreadPool(self.num_threads)
        requests = threadpool.makeRequests(self._getDocs, id_buckets,
                self._callback, self._errCallback)
        [self.pool.putRequest(req) for req in requests]
        self.pool.wait()

    def read(self):
        while True:
            try:
                time.sleep(0.5)
                self.pool.poll()
            except threadpool.NoResultsPending:
                return self.results
            except KeyboardInterrupt:
                print 'Keyboard interrupt. Exiting'
                sys.exit()
            finally:
                self.pool.joinAllDismissedWorkers()

    # Called to quickly get all of the document ids which can be sorted and dibbied out

    def _getDocIds(self):

        # Helper function to tersely compute a list of indices that evenly distribute
        # the items in it

        def partition(alist, indices):
            return [alist[i:j] for (i, j) in zip([0] + indices, indices
                    + [None])][:-1]

        try:
            conn = httplib.HTTPConnection(self.host, self.port)
            conn.request('GET', '/%s/_all_docs' % (self.db, ))
            response = conn.getresponse()
            if response.status != 200:  #  OK
                print 'Unable to get docs: %s %s' % (response.status,
                        response.reason)
                sys.exit()

            ids = [i['id'] for i in json.loads(response.read())['rows']
                   if not i['id'].startswith('_')]
            ids.sort()
        finally:
            conn.close()

        partition_size = int(ceil(1.0 * len(ids) / self.num_threads))
        indices = []

        _len = len(ids)
        idx = 0
        while idx < _len:
            idx += partition_size
            indices.append(idx)

        return partition(ids, indices)

    def _getDocs(self, ids):
        try:
            (startkey, endkey) = (ids[0], ids[-1])
            conn = httplib.HTTPConnection(self.host, self.port)
            conn.request('GET',
                         '/%s/_all_docs?startkey="%s"&endkey="%s"&include_docs=true'
                          % (self.db, startkey, endkey))
            response = conn.getresponse()
            if response.status != 200:  #  OK
                print 'Unable to get docs: %s %s' % (response.status,
                        response.reason)
                sys.exit()
            return response.read()
        finally:
            conn.close()

    def _errCallback(self, request, result):
        print 'An Error occurred:', request, result
        sys.exit()

    def _callback(self, request, result):
        rows = json.loads(result)['rows']
        self.results.extend([row['doc'] for row in rows])