vnpy/archive/datayes/api.py

#encoding: UTF-8
import os
import json
import time
import requests
import pymongo
import pandas as pd

from datetime import datetime, timedelta
from Queue import Queue, Empty
from threading import Thread, Timer
from pymongo import MongoClient

from requests.exceptions import ConnectionError
from errors import (VNPAST_ConfigError, VNPAST_RequestError,
VNPAST_DataConstructorError)

class Config(object):
	"""
	Json-like config object.

	The Config contains all kinds of settings and user info that 
	could be useful in the implementation of Api wrapper.

	privates
	--------
	* head: string; the name of config file.
	* token: string; user's token.
	* body: dictionary; the main content of config.
			- domain: string, api domain.
			- ssl:  boolean, specifes http or https usage.
			- version: string, version of the api. Currently 'v1'.
			- header: dictionary; the request header which contains 
					  authorization infomation.

	"""
	head = 'my config'

	toke_ = '44ebc0f058981f85382595f9f15f967' + \
			'0c7eaf2695de30dd752e8f33e9022baa0'

	token = '575593eb7696aec7339224c0fac2313780d8645f68b77369dcb35f8bcb419a0b'

	body = {
		'ssl': False,
		'domain': 'api.wmcloud.com/data',
		'version': 'v1',
		'header': {
			'Connection' : 'keep-alive',
			'Authorization': 'Bearer ' + token
		}
	}

	def __init__(self, head=None, token=None, body=None):
		""" 
		Reloaded constructor. 

		parameters
		----------
		* head: string; the name of config file. Default is None.
		* token: string; user's token.
		* body: dictionary; the main content of config
		"""
		if head:
			self.head = head
		if token: 
			self.token = token
		if body:
			self.body = body

	def view(self):
		""" Prettify printing method. """
		config_view = {
			'config_head' : self.head,
			'config_body' : self.body,
			'user_token' : self.token
		}
		print json.dumps(config_view, 
						 indent=4, 
						 sort_keys=True)

#----------------------------------------------------------------------
# Data containers.

class BaseDataContainer(object):
	"""
	Basic data container. The fundamental of all other data 
	container objects defined within this module.

	privates
	--------
	* head: string; the head(type) of data container.
	* body: dictionary; data content. Among all sub-classes that inherit 
	BaseDataContainer, type(body) varies according to the financial meaning
	that the child data container stands for. 
		- History:
		- Bar

	"""
	head = 'ABSTRACT_DATA'
	body = dict()
	pass

class History(BaseDataContainer):
	"""
	Historical data container. The foundation of all other pandas
	DataFrame-like two dimensional data containers for this module.

	privates
	--------
	* head: string; the head(type) of data container.
	* body: pd.DataFrame object; contains data contents.

	"""
	head = 'HISTORY'
	body = pd.DataFrame()

	def __init__(self, data):
		"""
		Reloaded constructor. 

		parameters
		----------
		* data: dictionary; usually a Json-like response from
		  web based api. For our purposes, data is exactly resp.json()
		  where resp is the response from datayes developer api.

		  		- example: {'data': [
								{
									'closePrice': 15.88,
									'date': 20150701, ...
								},
								{
									'closePrice': 15.99,
									'date': 20150702, ...
								}, ...], 
							'retCode': 1, 
							'retMsg': 'Success'}.

		  So the body of data is actually in data['data'], which is
		  our target when constructing the container.

		"""
		try:
			assert 'data' in data
			self.body = pd.DataFrame(data['data'])
		except AssertionError:
			msg = '[{}]: Unable to construct history data; '.format(
					self.head) + 'input is not a dataframe.'
			raise VNPAST_DataConstructorError(msg)
		except Exception,e:
			msg = '[{}]: Unable to construct history data; '.format(
					self.head) + str(e)
			raise VNPAST_DataConstructorError(msg)

class Bar(History):
	"""
	Historical Bar data container. Inherits from History()
	DataFrame-like two dimensional data containers for Bar data.

	privates
	--------
	* head: string; the head(type) of data container.
	* body: pd.DataFrame object; contains data contents.
	"""
	head = 'HISTORY_BAR'
	body = pd.DataFrame()

	def __init__(self, data):
		"""
		Reloaded constructor. 

		parameters
		----------
		* data: dictionary; usually a Json-like response from
		  web based api. For our purposes, data is exactly resp.json()
		  where resp is the response from datayes developer api.

		  		- example: {'data': [{
		  					'exchangeCD': 'XSHG', 
		  					'utcOffset': '+08:00', 
		  					'unit': 1, 
		  					'currencyCD': 'CNY', 
		  					'barBodys': [
			  						{
			  							'closePrice': 15.88,
										'date': 20150701, ...
			  						},
			  						{
			  							'closePrice': 15.99,
										'date': 20150702, ...
			  						}, ... ],
			  				'ticker': '000001', 
			  				'shortNM': u'\u4e0a\u8bc1\u6307\u6570'
			  				}, ...(other tickers) ],
							'retCode': 1, 
							'retMsg': 'Success'}.

		  When requesting 1 ticker, json['data'] layer has only one element;
		  we expect that this is for data collectioning for multiple tickers,
		  which is currently impossible nevertheless.
 
		  So we want resp.json()['data'][0]['barBodys'] for Bar data contents,
		  and that is what we go into when constructing Bar.
		"""
		try:
			assert 'data' in data
			assert 'barBodys' in data['data'][0]
			self.body = pd.DataFrame(data['data'][0]['barBodys'])
		except AssertionError:
			msg = '[{}]: Unable to construct history data; '.format(
					self.head) + 'input is not a dataframe.'
			raise VNPAST_DataConstructorError(msg)
		except Exception,e:
			msg = '[{}]: Unable to construct history data; '.format(
					self.head) + str(e)
			raise VNPAST_DataConstructorError(msg)


#----------------------------------------------------------------------
# Datayes Api class

class PyApi(object):
	"""
	Python based Datayes Api object.

	PyApi should be initialized with a Config json. The config must be complete,
	in that once constructed, the private variables like request headers, 
	tokens, etc. become constant values (inherited from config), and will be 
	consistantly referred to whenever make requests.


	privates
	--------
	* _config: Config object; a container of all useful settings when making 
	  requests.
	* _ssl, _domain, _domain_stream, _version, _header, _account_id: 
	  boolean, string, string, string, dictionary, integer;
	  just private references to the items in Config. See the docs of Config().
	* _session: requests.session object.


	examples
	--------


	"""
	_config = Config()

	# request stuffs
	_ssl = False
	_domain = ''
	_version = 'v1'
	_header = dict()
	_token = None

	_session = requests.session()

	def __init__(self, config):
		"""
		Constructor.

		parameters
		----------
		* config: Config object; specifies user and connection configs.
		"""
		if config.body:
			try:
				self._config = config
				self._ssl = config.body['ssl']
				self._domain = config.body['domain']
				self._version = config.body['version']
				self._header = config.body['header']
			except KeyError:
				msg = '[API]: Unable to configure api; ' + \
					  'config file is incomplete.'
				raise VNPAST_ConfigError(msg)
			except Exception,e:
				msg = '[API]: Unable to configure api; ' + str(e)
				raise VNPAST_ConfigError(msg)

		# configure protocol
		if self._ssl:
			self._domain = 'https://' + self._domain
		else:
			self._domain = 'http://' + self._domain

	def __access(self, url, params, method='GET'):
		"""
		request specific data from given url with parameters.

		parameters
		----------
		* url: string.
		* params: dictionary.
		* method: string; 'GET' or 'POST', request method.

		"""
		try:
			assert type(url) == str
			assert type(params) == dict
		except AssertionError,e:
			raise e('[API]: Unvalid url or parameter input.')
		if not self._session:
			s = requests.session()
		else: s = self._session

		# prepare and send the request.
		try:
			req = requests.Request(method,
								   url = url,
								   headers = self._header,
								   params = params)
			prepped = s.prepare_request(req) # prepare the request
			resp = s.send(prepped, stream=False, verify=True)
			if method == 'GET':
				assert resp.status_code == 200
			elif method == 'POST':
				assert resp.status_code == 201
			return resp
		except AssertionError:
			msg = '[API]: Bad request, unexpected response status: ' + \
				  str(resp.status_code)
			raise VNPAST_RequestError(msg)
			pass
		except Exception,e:
			msg = '[API]: Bad request.' + str(e)
			raise VNPAST_RequestError(msg)

	#----------------------------------------------------------------------
	# directly get methods - Market data

	def get_equity_M1_one(self,
					  	  start='', end='', secID='000001.XSHG'):
		"""
		Get 1-minute intraday bar data of one security.

		parameters
		----------
		* start, end: string; Time mark formatted in 'HH:MM'. Specifies the 
		  start/end point of bar. Note that the requested date is the 
		  latest trading day (only one day), and the default start/end time is
		  '09:30' and min(now, '15:00'). Effective minute bars range from 
		  09:30 - 11:30 in the morning and 13:01 - 15:00 in the afternoon.
		* secID: string; the security ID in the form of '000001.XSHG', i.e.
		  ticker.exchange

		"""
		url = '{}/{}/api/market/getBarRTIntraDay.json'.format(
			   self._domain, self._version)
		params = {
			'startTime': start,
			'endTime': end,
			'securityID': secID,
		}
		try:
			resp = self.__access(url=url, params=params)
			assert len(resp.json()) > 0
			print resp.json()
			data = Bar(resp.json())
			return data
		except AssertionError: return 0


	def get_equity_M1(self, field='', start='20130701', end='20130730', 
					  secID='000001.XSHG', output='df'):
		"""
		1-minute bar in a month, currently unavailable.

		parameters
		----------
		* field: string; variables that are to be requested.
		* start, end: string; Time mark formatted in 'YYYYMMDD'.
		* secID: string; the security ID in the form of '000001.XSHG', i.e.
		  ticker.exchange
		* output: enumeration of strings; the format of output that will be 
		  returned. default is 'df', optionals are:
		  		- 'df': returns History object, 
		  			  where ret.body is a dataframe.
		  		- 'list': returns a list of dictionaries.

		"""
		url = '{}/{}/api/market/getBarHistDateRange.json'.format(
			   self._domain, self._version)
		params = {
			'field': field,
			'startDate': start,
			'endDate': end,
			'securityID': secID,
		}
		try:
			resp = self.__access(url=url, params=params)
			assert len(resp.json()) > 0
			if output == 'df':
				data = Bar(resp.json())
			elif output == 'list':
				data = resp.json()['data'][0]['barBodys']
			return data
		except AssertionError: return 0


	def get_equity_D1(self, field='', start='', end='', secID='',
					  ticker='', one=20150513, output='df'):
		"""
		Get 1-day interday bar data of one security.

		parameters
		----------
		* field: string; variables that are to be requested. Available variables
		  are: (* is unique for securities)

		  		- secID 				string.
		  		- tradeDate 			date(?).
		  		- ticker 				string.
		  		- secShortName 			string.
		  		- exchangeCD 			string.
		  		- preClosePrice 		double.
		  		- actPreClosePrice*		double.
		  		- openPrice		 		double.
		  		- highestPrice	 		double.
		  		- lowestPrice 			double.
		  		- closePrice 			double.
		  		- turnoverVol 			double.
		  		- turnoverValue 		double.
		  		- dealAmount* 			integer.
		  		- turnoverRate	 		double.
		  		- accumAdjFactor* 		double.
		  		- negMarketValue* 		double.
		  		- marketValue* 			double.
		  		- PE* 					double.
		  		- PE1* 					double.
		  		- PB* 					double.

		  Field is an optional parameter, default setting returns all fields.

		* start, end: string; Date mark formatted in 'YYYYMMDD'. Specifies the 
		  start/end point of bar. Start and end are optional parameters. If 
		  start, end and ticker are all specified, default 'one' value will be
		  abandoned.

		* secID: string; the security ID in the form of '000001.XSHG', i.e.
		  ticker.exchange.

		* ticker: string; the trading code in the form of '000001'.

		* one: string; Date mark formatted in 'YYYYMMDD'. 
		  Specifies one date on which data of all tickers are to be requested. 
		  Note that to get effective json data response, at least one parameter 
		  in {secID, ticker, tradeDate} should be entered.

		* output: enumeration of strings; the format of output that will be 
		  returned. default is 'df', optionals are:
		  		- 'df': returns History object, 
		  			  where ret.body is a dataframe.
		  		- 'list': returns a list of dictionaries.

		"""
		if start and end and ticker:
			one = '' # while user specifies start/end, covers tradeDate.

		url = '{}/{}/api/market/getMktEqud.json'.format(
			   self._domain, self._version)
		params = {
			'field': field,
			'beginDate': start,
			'endDate': end,
			'secID': secID,
			'ticker': ticker,
			'tradeDate': one
		}
		try:
			resp = self.__access(url=url, params=params)
			assert len(resp.json()) > 0
			if output == 'df':
				data = History(resp.json())
			elif output == 'list':
				data = resp.json()['data']
			return data
			#return resp
		except AssertionError: return 0

	def get_block_D1(self, field='', start='', end='', secID='',
					 ticker='', one=20150513):
		"""

		"""
		pass

	def get_repo_D1(self, field='', start='', end='', secID='',
					ticker='', one=20150513):
		"""

		"""
		pass

	def get_bond_D1(self, field='', start='', end='', secID='',
					ticker='', one=20150513, output='df'):
		"""
		Get 1-day interday bar data of one bond instrument.

		parameters
		----------

		* field: string; variables that are to be requested. Available variables
		  are: (* is unique for bonds)

		  		- secID 				string.
		  		- tradeDate 			date(?).
		  		- ticker 				string.
		  		- secShortName 			string.
		  		- exchangeCD 			string.
		  		- preClosePrice 		double.
		  		- openPrice		 		double.
		  		- highestPrice	 		double.
		  		- lowestPrice 			double.
		  		- closePrice 			double.
		  		- turnoverVol 			double.
		  		- turnoverValue 		double.
		  		- turnoverRate	 		double.
		  		- dealAmount*			integer.
		  		- accrInterest*			double.
		  		- YTM(yieldToMaturity)* double.	

		  Field is an optional parameter, default setting returns all fields.

		* start, end, secID, ticker, one, output
		  string, string, string, string, string, string(enum)
		  Same as above, reference: get_equity_D1().

		"""
		if start and end and ticker:
			one = '' # while user specifies start/end, covers tradeDate.

		url = '{}/{}/api/market/getMktBondd.json'.format(
			   self._domain, self._version)
		params = {
			'field': field,
			'beginDate': start,
			'endDate': end,
			'secID': secID,
			'ticker': ticker,
			'tradeDate': one
		}
		try:
			resp = self.__access(url=url, params=params)
			assert len(resp.json()) > 0
			if output == 'df':
				data = History(resp.json())
			elif output == 'list':
				data = resp.json()['data']
			return data
		except AssertionError: return 0

	def get_future_D1(self, field='', start='', end='', secID='',
					ticker='', one=20150513, output='df'):
		"""
		Get 1-day interday bar data of one future contract.

		parameters
		----------

		* field: string; variables that are to be requested. Available variables
		  are: (* is unique for future contracts)

		  		- secID 				string.
		  		- tradeDate 			date(?).
		  		- ticker 				string.
		  		- secShortName 			string.
		  		- exchangeCD 			string.
		  		- contractObject*		string.
		  		- contractMark*			string.
		  		- preSettlePrice*		double.
		  		- preClosePrice 		double.
		  		- openPrice		 		double.
		  		- highestPrice	 		double.
		  		- lowestPrice 			double.
		  		- closePrice 			double.
		  		- settlePrice* 			double.
		  		- turnoverVol 			integer.
		  		- turnoverValue 		integer.
		  		- openInt*				integer.
		  		- CHG* 					double.
		  		- CHG1* 				double.
		  		- CHGPct*				double.
		  		- mainCon*				integer (0/1 flag).
		  		- smainCon*				integer (0/1 flag).

		  Field is an optional parameter, default setting returns all fields.

		* start, end, secID, ticker, one, output
		  string, string, string, string, string, string(enum)
		  Same as above, reference: get_equity_D1().
		"""
		if start and end and ticker:
			one = '' # while user specifies start/end, covers tradeDate.

		url = '{}/{}/api/market/getMktFutd.json'.format(
			   self._domain, self._version)
		params = {
			'field': field,
			'beginDate': start,
			'endDate': end,
			'secID': secID,
			'ticker': ticker,
			'tradeDate': one
		}
		try:
			resp = self.__access(url=url, params=params)
			assert len(resp.json()) > 0
			if output == 'df':
				data = History(resp.json())
			elif output == 'list':
				data = resp.json()['data']
			return data
		except AssertionError: return 0

	def get_future_main_D1(self, field='', start='', end='', mark='',
						   obj='', main=1, one=20150513):
		"""

		"""
		pass

	def get_fund_D1(self, field='', start='', end='', secID='',
					ticker='', one=20150513, output='df'):
		"""
		Get 1-day interday bar data of one mutual fund.

		parameters
		----------

		* field: string; variables that are to be requested. Available variables
		  are: (* is unique for funds)

		  		- secID 				string.
		  		- tradeDate 			date(?).
		  		- ticker 				string.
		  		- secShortName 			string.
		  		- exchangeCD 			string.
		  		- preClosePrice 		double.
		  		- openPrice		 		double.
		  		- highestPrice	 		double.
		  		- lowestPrice 			double.
		  		- closePrice 			double.
		  		- turnoverVol 			double.
		  		- turnoverValue 		double.
		  		- CHG* 					double.
		  		- CHGPct*				double.
		  		- discount* 			double.
		  		- discountRatio*		double.
		  		- circulationShares*	double.

		  Field is an optional parameter, default setting returns all fields.

		* start, end, secID, ticker, one, output
		  string, string, string, string, string, string(enum)
		  Same as above, reference: get_equity_D1().

		"""
		if start and end and ticker:
			one = '' # while user specifies start/end, covers tradeDate.

		url = '{}/{}/api/market/getMktFundd.json'.format(
			   self._domain, self._version)
		params = {
			'field': field,
			'beginDate': start,
			'endDate': end,
			'secID': secID,
			'ticker': ticker,
			'tradeDate': one
		}
		try:
			resp = self.__access(url=url, params=params)
			assert len(resp.json()) > 0
			if output == 'df':
				data = History(resp.json())
			elif output == 'list':
				data = resp.json()['data']
			return data
		except AssertionError: return 0

	def get_index_D1(self, field='', start='', end='', indexID='',
					 ticker='', one=20150513, output='df'):
		"""
		Get 1-day interday bar data of one stock index.

		parameters
		----------

		* field: string; variables that are to be requested. Available variables
		  are: (* is unique for indices)

		  		- indexID 				string.
		  		- tradeDate 			date(?).
		  		- ticker 				string.
		  		- secShortName 			string.
		  		- porgFullName*			string.
		  		- exchangeCD 			string.
		  		- preCloseIndex 		double.
		  		- openIndex		 		double.
		  		- highestIndex	 		double.
		  		- lowestIndex 			double.
		  		- closeIndex 			double.
		  		- turnoverVol 			double.
		  		- turnoverValue 		double.
		  		- CHG* 					double.
		  		- CHGPct*				double.

		  Field is an optional parameter, default setting returns all fields.

		* start, end, secID, ticker, one, output
		  string, string, string, string, string, string(enum)
		  Same as above, reference: get_equity_D1().

		"""
		if start and end and ticker:
			one = '' # while user specifies start/end, covers tradeDate.

		url = '{}/{}/api/market/getMktIdxd.json'.format(
			   self._domain, self._version)
		params = {
			'field': field,
			'beginDate': start,
			'endDate': end,
			'indexID': indexID,
			'ticker': ticker,
			'tradeDate': one
		}
		try:
			resp = self.__access(url=url, params=params)
			assert len(resp.json()) > 0
			if output == 'df':
				data = History(resp.json())
			elif output == 'list':
				data = resp.json()['data']
			return data
		except AssertionError: return 0

	def get_option_D1(self, field='', start='', end='', secID='',
					  optID='' ,ticker='', one=20150513, output='df'):
		"""
		Get 1-day interday bar data of one option contact.

		parameters
		----------

		* field: string; variables that are to be requested. Available variables
		  are: (* is unique for options)

		  		- secID 				string.
		  		- optID* 				string.
		  		- tradeDate 			date(?).
		  		- ticker 				string.
		  		- secShortName 			string.
		  		- exchangeCD 			string.
		  		- preClosePrice 		double.
		  		- openPrice		 		double.
		  		- highestPrice	 		double.
		  		- lowestPrice 			double.
		  		- closePrice 			double.
		  		- settlePrice*			double.
		  		- turnoverVol 			double.
		  		- turnoverValue 		double.
		  		- openInt*				integer.

		  Field is an optional parameter, default setting returns all fields.

		* start, end, secID, ticker, one, output
		  string, string, string, string, string, string(enum)
		  Same as above, reference: get_equity_D1().

		"""
		if start and end and ticker:
			one = '' # while user specifies start/end, covers tradeDate.

		url = '{}/{}/api/market/getMktOptd.json'.format(
			   self._domain, self._version)
		params = {
			'field': field,
			'beginDate': start,
			'endDate': end,
			'secID': secID,
			'optID': optID,
			'ticker': ticker,
			'tradeDate': one
		}
		try:
			resp = self.__access(url=url, params=params)
			assert len(resp.json()) > 0
			if output == 'df':
				data = History(resp.json())
			elif output == 'list':
				data = resp.json()['data']
			return data
		except AssertionError: return 0

	def get_stockFactor_D1(self, field='', secID='',
					  	   ticker='000001', start=20130701, end=20130801):
		"""
		Get 1-day interday factor data for stocks.

		parameters
		----------

		* field: string; variables that are to be requested. 
		  Field is an optional parameter, default setting returns all fields.

		* start, end, secID, ticker, one, output
		  string, string, string, string, string, string(enum)
		  Same as above, reference: get_equity_D1().
		"""
		url = '{}/{}/api/market/getStockFactorsDateRange.json'.format(
			   self._domain, self._version)
		params = {
			'field': field,
			'beginDate': start,
			'endDate': end,
			'secID': secID,
			'ticker': ticker
		}
		try:
			resp = self.__access(url=url, params=params)
			assert len(resp.json()) > 0
			data = History(resp.json())
			return data
		except AssertionError: return 0
		
	#----------------------------------------------------------------------
	# directly get methods - Fundamental Data

	def get_balanceSheet(self, field='', secID='',
						 start='', end='', pubStart='', pubEnd='',
						 reportType='', ticker='000001'):
		"""

		"""
		url = '{}/{}/api/fundamental/getFdmtBS.json'.format(
			  self._domain, self._version)
		params = {
			'field': field,
			'secID': secID,
			'ticker': ticker,
			'beginDate': start,
			'endDate': end,
			'publishDateBegin': pubStart,
			'publishDateEnd': pubEnd,
			'reportType': reportType
		}
		try:
			resp = self.__access(url=url, params=params)
			assert len(resp.json()) > 0
			data = History(resp.json())
			return data
		except AssertionError: return 0

	def get_balanceSheet_bnk(self):
		"""

		"""
		pass

	def get_balanceSheet_sec(self):
		"""

		"""
		pass

	def get_balanceSheet_ins(self):
		"""

		"""
		pass

	def get_balanceSheet_ind(self):
		"""

		"""
		pass

	def get_cashFlow(self, field='', secID='',
					 start='', end='', pubStart='', pubEnd='',
					 reportType='', ticker='000001'):
		"""

		"""
		url = '{}/{}/api/fundamental/getFdmtCF.json'.format(
			  self._domain, self._version)
		params = {
			'field': field,
			'secID': secID,
			'ticker': ticker,
			'beginDate': start,
			'endDate': end,
			'publishDateBegin': pubStart,
			'publishDateEnd': pubEnd,
			'reportType': reportType
		}
		try:
			resp = self.__access(url=url, params=params)
			assert len(resp.json()) > 0
			data = History(resp.json())
			return data
		except AssertionError: return 0

	def get_cashFlow_bnk(self):
		"""

		"""
		pass

	def get_cashFlow_sec(self):
		"""

		"""
		pass

	def get_cashFlow_ins(self):
		"""

		"""
		pass

	def get_cashFlow_ind(self):
		"""

		"""
		pass

	def get_incomeStatement(self, field='', secID='',
					 		start='', end='', pubStart='', pubEnd='',
					 		reportType='', ticker='000001'):
		"""

		"""
		url = '{}/{}/api/fundamental/getFdmtIS.json'.format(
			  self._domain, self._version)
		params = {
			'field': field,
			'secID': secID,
			'ticker': ticker,
			'beginDate': start,
			'endDate': end,
			'publishDateBegin': pubStart,
			'publishDateEnd': pubEnd,
			'reportType': reportType
		}
		try:
			resp = self.__access(url=url, params=params)
			assert len(resp.json()) > 0
			data = History(resp.json())
			return data
		except AssertionError: return 0

	def get_incomeStatement_bnk(self):
		"""

		"""
		pass

	def get_incomeStatement_sec(self):
		"""

		"""
		pass

	def get_incomeStatement_ins(self):
		"""

		"""
		pass

	def get_incomeStatement_ind(self):
		"""

		"""
		pass

	#----------------------------------------------------------------------
	# multi-threading download for database storage.

	def __drudgery(self, id, db, indexType,
				  start, end, tasks, target):
		"""
		basic drudgery function.
		This method loops over a list of tasks(tickers) and get data using
		target api.get_# method for all those tickers.
		A new feature 'date' or 'dateTime'(for intraday) will be automatically
		added into every json-like documents, and specifies the datetime.
		datetime() formatted date(time) mark. With the setting of MongoDB
		in this module, this feature should be the unique index for all 
		collections.

		By programatically assigning creating and assigning tasks to drudgery
		functions, multi-threading download of data can be achieved.

		parameters
		----------
		* id: integer; the ID of Drudgery session.

		* db: pymongo.db object; the database which collections of bars will
		  go into.

		* indexType: string(enum): 'date' or 'datetime', specifies what
		  is the collection index formatted.

		* start, end: string; Date mark formatted in 'YYYYMMDD'. Specifies the 
		  start/end point of collections of bars.

		* tasks: list of strings; the tickers that this drudgery function
		  loops over.

		* target: method; the api.get_# method that is to be called by 
		  drudgery function.
		"""
		if len(tasks) == 0:
			return 0

		# str to datetime inline functions.
		if indexType == 'date':
			todt = lambda str_dt: datetime.strptime(str_dt,'%Y-%m-%d')
			update_dt = lambda d: d.update({'date':todt(d['tradeDate'])})
		elif indexType == 'datetime':
			todt = lambda str_d, str_t: datetime.strptime(
				str_d + ' ' + str_t,'%Y-%m-%d %H:%M')
			update_dt = lambda d: d.update(
				{'dateTime':todt(d['dataDate'], d['barTime'])})
		else:
			raise ValueError

		# loop over all tickers in task list.
		k, n = 1, len(tasks)
		for ticker in tasks:
			try:
				data = target(start = start,
							  end = end, 
							  ticker = ticker,
							  output = 'list')
				assert len(data) >= 1
				map(update_dt, data) # add datetime feature to docs.
				coll = db[ticker]
				coll.insert_many(data)
				print '[API|Session{}]: '.format(id) + \
					  'Finished {} in {}.'.format(k, n)
				k += 1
			except AssertionError:
				msg = '[API|Session{}]: '.format(id) + \
					  'Empty dataset in the response.'
				print msg
				pass
			except Exception, e:
				msg = '[API|Session{}]: '.format(id) + \
					  'Exception encountered when ' + \
					  'requesting data; ' + str(e)
				print msg
				pass

	def get_equity_D1_drudgery(self, id, db, start, end, tasks=[]):
		"""
		call __drudgery targeting at get_equity_D1()
		"""
		self.__drudgery(id=id, db=db,
					   indexType = 'date',
					   start = start, 
					   end = end, 
					   tasks = tasks,
					   target = self.get_equity_D1)

	def get_future_D1_drudgery(self, id, db, start, end, tasks=[]):
		"""
		call __drudgery targeting at get_future_D1()
		"""
		self.__drudgery(id=id, db=db, 
					   indexType = 'date',
					   start = start, 
					   end = end, 
					   tasks = tasks,
					   target = self.get_future_D1)

	def get_index_D1_drudgery(self, id, db, start, end, tasks=[]):
		"""
		call __drudgery targeting at get_index_D1()
		"""
		self.__drudgery(id=id, db=db, 
					   indexType = 'date',
					   start = start, 
					   end = end, 
					   tasks = tasks,
					   target = self.get_index_D1)

	def get_bond_D1_drudgery(self, id, db, start, end, tasks=[]):
		"""
		call __drudgery targeting at get_bond_D1()
		"""
		self.__drudgery(id=id, db=db, 
					   indexType = 'date',
					   start = start, 
					   end = end, 
					   tasks = tasks,
					   target = self.get_bond_D1)

	def get_fund_D1_drudgery(self, id, db, start, end, tasks=[]):
		"""
		call __drudgery targeting at get_fund_D1()
		"""
		self.__drudgery(id=id, db=db, 
					   indexType = 'date',
					   start = start, 
					   end = end, 
					   tasks = tasks,
					   target = self.get_fund_D1)

	def get_option_D1_drudgery(self, id, db, start, end, tasks=[]):
		"""
		call __drudgery targeting at get_option_D1()
		"""
		self.__drudgery(id=id, db=db, 
					   indexType = 'date',
					   start = start, 
					   end = end, 
					   tasks = tasks,
					   target = self.get_option_D1)

	#----------------------------------------------------------------------

	def __overlord(self, db, start, end, dName, 
				   target1, target2, sessionNum):
		"""
		Basic controller of multithreading request.
		Generates a list of all tickers, creates threads and distribute 
		tasks to individual #_drudgery() functions.

		parameters
		----------
		* db: pymongo.db object; the database which collections of bars will
		  go into. Note that this database will be transferred to every 
		  drudgery functions created by controller.

		* start, end: string; Date mark formatted in 'YYYYMMDD'. Specifies the 
		  start/end point of collections of bars.

		* dName: string; the path of file where all tickers' infomation
		  are stored in. 

		* target1: method; targetting api method that overlord calls
		  to get tasks list.

		* target2: method; the corresponding drudgery function.

		* sessionNum: integer; the number of threads that will be deploied.
		  Concretely, the list of all tickers will be sub-divided into chunks,
		  where chunkSize = len(allTickers)/sessionNum.

		"""
		if os.path.isfile(dName):
			# if directory exists, read from it.
			jsonFile = open(dName,'r')
			allTickers = json.loads(jsonFile.read())
			jsonFile.close()
		else:
			data = target1()
			allTickers = list(data.body['ticker'])
		
		chunkSize = len(allTickers)/sessionNum
		taskLists = [allTickers[k:k+chunkSize] for k in range(
						0, len(allTickers), chunkSize)]
		k = 0
		for tasks in taskLists:
			thrd = Thread(target = target2,
						  args = (k, db, start, end, tasks))
			thrd.start()
			k += 1
		return 1

	def get_equity_D1_mongod(self, db, start, end, sessionNum=30):
		"""
		Controller of get equity D1 method.
		"""
		self.__overlord(db = db,
						start = start,
						end = end,
						dName = 'names/equTicker.json',
						target1 = self.get_equity_D1,
						target2 = self.get_equity_D1_drudgery,
						sessionNum = sessionNum)

	def get_future_D1_mongod(self, db, start, end, sessionNum=30):
		"""
		Controller of get future D1 method.
		"""
		self.__overlord(db = db,
						start = start,
						end = end,
						dName = 'names/futTicker.json',
						target1 = self.get_future_D1,
						target2 = self.get_future_D1_drudgery,
						sessionNum = sessionNum)

	def get_index_D1_mongod(self, db, start, end, sessionNum=30):
		"""
		Controller of get index D1 method.
		"""
		self.__overlord(db = db,
						start = start,
						end = end,
						dName = 'names/idxTicker.json',
						target1 = self.get_index_D1,
						target2 = self.get_index_D1_drudgery,
						sessionNum = sessionNum)

	def get_bond_D1_mongod(self, db, start, end, sessionNum=30):
		"""
		Controller of get bond D1 method.
		"""
		self.__overlord(db = db,
						start = start,
						end = end,
						dName = 'names/bndTicker.json',
						target1 = self.get_bond_D1,
						target2 = self.get_bond_D1_drudgery,
						sessionNum = sessionNum)

	def get_fund_D1_mongod(self, db, start, end, sessionNum=30):
		"""
		Controller of get fund D1 method.
		"""
		self.__overlord(db = db,
						start = start,
						end = end,
						dName = 'names/fudTicker.json',
						target1 = self.get_fund_D1,
						target2 = self.get_fund_D1_drudgery,
						sessionNum = sessionNum)

	def get_option_D1_mongod(self, db, start, end, sessionNum=30):
		"""
		Controller of get option D1 method.
		"""
		self.__overlord(db = db,
						start = start,
						end = end,
						dName = 'names/optTicker.json',
						target1 = self.get_option_D1,
						target2 = self.get_option_D1_drudgery,
						sessionNum = sessionNum)

	def get_equity_D1_mongod_(self, db, start, end, sessionNum=30):
		"""
		Outer controller of get equity D1 method. 
		Generates a list of all tickers, creates threads and distribute 
		tasks to individual get_equity_D1_drudgery() functions.

		parameters
		----------
		* db: pymongo.db object; the database which collections of bars will
		  go into. Note that this database will be transferred to every 
		  drudgery functions created by controller.
		* start, end: string; Date mark formatted in 'YYYYMMDD'. Specifies the 
		  start/end point of collections of bars.
		* sessionNum: integer; the number of threads that will be deploied.
		  Concretely, the list of all tickers will be sub-divided into chunks,
		  where chunkSize = len(allTickers)/sessionNum.

		"""
		# initialize task list.
		dName = 'names/equTicker.json'
		if os.path.isfile(dName):
			# if directory exists, read from it.
			jsonFile = open(dName,'r')
			allTickers = json.loads(jsonFile.read())
			jsonFile.close()
		else:
			data = self.get_equity_D1()
			allTickers = list(data.body['ticker'])
			
		chunkSize = len(allTickers)/sessionNum
		taskLists = [allTickers[k:k+chunkSize] for k in range(
						0, len(allTickers), chunkSize)]
		k = 0
		for tasks in taskLists:
			thrd = Thread(target = self.get_equity_D1_drudgery,
						  args = (k, db, start, end, tasks))
			thrd.start()
			k += 1
		return 1


	#----------------------------------------------------------------------#
	# to be deprecated

	def get_equity_D1_drudgery_(self, id, db, 
							    start, end, tasks=[]):
		"""
		Drudgery function of getting equity_D1 bars.
		This method loops over a list of tasks(tickers) and get D1 bar
		for all these tickers. A new feature 'date' will be automatically
		added into every json-like documents, and specifies the datetime.
		datetime() formatted date mark. With the default setting of MongoDB
		in this module, this feature should be the unique index for all 
		collections.

		By programatically assigning creating and assigning tasks to drudgery
		functions, multi-threading download of data can be achieved.

		parameters
		----------
		* id: integer; the ID of Drudgery session.
		* db: pymongo.db object; the database which collections of bars will
		  go into.
		* start, end: string; Date mark formatted in 'YYYYMMDD'. Specifies the 
		  start/end point of collections of bars.
		* tasks: list of strings; the tickers that this drudgery function
		  loops over.

		"""
		if len(tasks) == 0:
			return 0
		# str to datetime inline functions.
		todt = lambda str_dt: datetime.strptime(str_dt,'%Y-%m-%d')
		update_dt = lambda d: d.update({'date':todt(d['tradeDate'])})
		# loop over all tickers in task list.
		k, n = 1, len(tasks)
		for ticker in tasks:
			try:
				data = self.get_equity_D1(start = start,
										  end = end, 
										  ticker = ticker,
										  output = 'list')
				assert len(data) >= 1
				map(update_dt, data) # add datetime feature to docs.
				coll = db[ticker]
				coll.insert_many(data)
				print '[API|Session{}]: '.format(id) + \
					  'Finished {} in {}.'.format(k, n)
				k += 1
			except ConnectionError:
			# If choke connection, standby for 1sec an invoke again.
				time.sleep(1)
				self.get_equity_D1_drudgery(
					 id, db, start, end, tasks)
			except AssertionError:
				msg = '[API|Session{}]: '.format(id) + \
					  'Empty dataset in the response.'
				print msg
				pass
			except Exception, e:
				msg = '[API|Session{}]: '.format(id) + \
					  'Exception encountered when ' + \
					  'requesting data; ' + str(e)
				print msg
				pass

	def get_equity_D1_mongod_(self, db, start, end, sessionNum=30):
		"""
		Outer controller of get equity D1 method. 
		Generates a list of all tickers, creates threads and distribute 
		tasks to individual get_equity_D1_drudgery() functions.

		parameters
		----------
		* db: pymongo.db object; the database which collections of bars will
		  go into. Note that this database will be transferred to every 
		  drudgery functions created by controller.
		* start, end: string; Date mark formatted in 'YYYYMMDD'. Specifies the 
		  start/end point of collections of bars.
		* sessionNum: integer; the number of threads that will be deploied.
		  Concretely, the list of all tickers will be sub-divided into chunks,
		  where chunkSize = len(allTickers)/sessionNum.

		"""
		# initialize task list.
		dName = 'names/equTicker.json'
		if os.path.isfile(dName):
			# if directory exists, read from it.
			jsonFile = open(dName,'r')
			allTickers = json.loads(jsonFile.read())
			jsonFile.close()
		else:
			data = self.get_equity_D1()
			allTickers = list(data.body['ticker'])
			
		chunkSize = len(allTickers)/sessionNum
		taskLists = [allTickers[k:k+chunkSize] for k in range(
						0, len(allTickers), chunkSize)]
		k = 0
		for tasks in taskLists:
			thrd = Thread(target = self.get_equity_D1_drudgery,
						  args = (k, db, start, end, tasks))
			thrd.start()
			k += 1
		return 1

	#----------------------------------------------------------------------#

	def get_equity_M1_drudgery(self, id, db, 
							   start, end, tasks=[]):
		"""
		Drudgery function of getting equity_D1 bars.
		This method loops over a list of tasks(tickers) and get D1 bar
		for all these tickers. A new feature 'dateTime', combined by Y-m-d 
		formatted date part and H:M time part, will be automatically added into 
		every json-like documents. It would be a datetime.datetime() timestamp 
		object. In this module, this feature should be the unique index for all 
		collections.

		By programatically assigning creating and assigning tasks to drudgery
		functions, multi-threading download of data can be achieved.

		parameters
		----------
		* id: integer; the ID of Drudgery session.
		* db: pymongo.db object; the database which collections of bars will
		  go into.
		* start, end: string; Date mark formatted in 'YYYYMMDD'. Specifies the 
		  start/end point of collections of bars. Note that to ensure the 
		  success of every requests, the range amid start and end had better be 
		  no more than one month. 
		* tasks: list of strings; the tickers that this drudgery function
		  loops over.

		"""
		if len(tasks) == 0:
			return 0

		# str to datetime inline functions.
		todt = lambda str_d, str_t: datetime.strptime(
				str_d + ' ' + str_t,'%Y-%m-%d %H:%M')
		update_dt = lambda d: d.update(
				{'dateTime':todt(d['dataDate'], d['barTime'])})

		k, n = 1, len(tasks)
		for secID in tasks:
			try:
				data = self.get_equity_M1(start = start,
										  end = end, 
										  secID = secID,
										  output = 'list')
				map(update_dt, data) # add datetime feature to docs.
				coll = db[secID]
				coll.insert_many(data)
				print '[API|Session{}]: '.format(id) + \
					  'Finished {} in {}.'.format(k, n)
				k += 1
			except ConnectionError:
			# If choke connection, standby for 1sec an invoke again.
				time.sleep(1)
				self.get_equity_D1_drudgery(
					 id, db, start, end, tasks)
			except AssertionError:
				msg = '[API|Session{}]: '.format(id) + \
					  'Empty dataset in the response.'
				print msg
				pass
			except Exception, e:
				msg = '[API|Session{}]: '.format(id) + \
					  'Exception encountered when ' + \
					  'requesting data; ' + str(e)
				print msg
				pass

	def get_equity_M1_interMonth(self, db, id,
							     startYr=datetime.now().year-2, 
							     endYr=datetime.now().year, 
							     tasks=[]):
		"""
		Mid-level wrapper of get equity M1 method. 
		Get 1-minute bar between specified start year and ending year for
		more than one tickers in tasks list. 

		parameters
		----------
		* db: pymongo.db object; the database which collections of bars will
		  go into. Note that this database will be transferred to every 
		  drudgery functions created by controller.
		* id: integer; the ID of wrapper session.
		* startYr, endYr: integer; the start and ending year amid which the
		  1-minute bar data is gotten one month by another employing 
		  get_equity_M1_drudgery() function. 
		  Default values are this year and two years before now. 
		  the complete time range will be sub-divided into months. And threads
		  are deployed for each of these months.

		  		- example
		  		  -------
		  		  Suppose .now() is Auguest 15th 2015. (20150815)
		  		  startYr, endYr = 2014, 2015.
		  		  then two list of strings will be generated:
		  		  ymdStringStart = ['20140102','20140202', ... '20150802']
		  		  ymdStringEnd = ['20140101','20140201', ... '20150801']
		  		  the sub-timeRanges passed to drudgeries will be:
		  		  (start, end): (20140102, 20140201), (20140202, 20140301),
		  		  ..., (20150702, 20150801).
		  		  So the actual time range is 20140102 - 20150801.

		* sessionNum: integer; the number of threads that will be deploied.
		  Concretely, the list of all tickers will be sub-divided into chunks,
		  where chunkSize = len(allTickers)/sessionNum.

		"""
		# Construct yyyymmdd strings.(as ymdStrings list)
		now = datetime.now()
		years = [str(y) for y in range(startYr, endYr+1)]
		monthDates = [(2-len(str(k)))*'0'+str(k)+'02' for k in range(1,13)]
		ymdStringStart = [y+md for y in years for md in monthDates if (
					  	  datetime.strptime(y+md,'%Y%m%d')<=now)]
		monthDates = [(2-len(str(k)))*'0'+str(k)+'01' for k in range(1,13)]
		ymdStringEnd = [y+md for y in years for md in monthDates if (
					  	datetime.strptime(y+md,'%Y%m%d')<=now)]
		k = 0
		for t in range(len(ymdStringEnd)-1):
			start = ymdStringStart[t]
			end = ymdStringEnd[t+1]
			subID = str(id) + '_' + str(k)
			thrd = Thread(target = self.get_equity_M1_drudgery,
						  args = (subID, db, start, end, tasks))
			thrd.start()
			k += 1


	def get_equity_M1_all(self, db, 
							 startYr=datetime.now().year-2, 
							 endYr=datetime.now().year, 
							 splitNum=10):
		"""


		"""
		"""
		# initialize task list.
		data = self.get_equity_D1()
		allTickers = list(data.body['ticker'])
		exchangeCDs = list(data.body['exchangeCD'])
		allSecIds = [allTickers[k]+'.'+exchangeCDs[k] for k in range(
						len(allTickers))]
		chunkSize = len(allSecIds)/splitNum
		taskLists = [allSecIds[k:k+chunkSize] for k in range(
						0, len(allSecIds), chunkSize)]

		# Construct yyyymmdd strings.(as ymdStrings list)
		now = datetime.now()
		years = [str(y) for y in range(startYr, endYr+1)]
		monthDates = [(2-len(str(k)))*'0'+str(k)+'01' for k in range(1,13)]
		ymdStrings = [y+md for y in years for md in monthDates if (
					  datetime.strptime(y+md,'%Y%m%d')<=now)]

		print taskLists[0]
		print ymdStrings
		
		k = 0
		for t in range(len(ymdStrings)-1):
			start = ymdStrings[t]
			end = ymdStrings[t+1]
			thrd = Thread(target = self.get_equity_M1_drudgery,
							  args = (k, db, start, end, taskLists[0]))
			thrd.start()
			k += 1
		return 1
		"""
		pass